v1.13.8

`tokenize.FSTRING_MIDDLE` was introduced, changing the
representation of `f"x{{y"` from `STRING(f"x{{y")` to:

* `FSTRING_START('f"')`
* `FSTRING_MIDDLE('x{')`
* `FSTRING_MIDDLE('y')`
* `FSTRING_END('"')`

each literal `{` (encoded as `{{` in the input) now appears as a
single `{` as the final character of its `FSTRING_MIDDLE`, with
additional consecutive `FSTRING_MIDDLE` tokens if necessary

regular interpolating `{` are encoded as separate `OP` tokens

the fact that the literal `{` is encoded as a single `{` instead
of `{{` breaks the assumption that the string-value of each token
maps directly to the original code

fix this by replacing `{` with `{{` and `}` with `}}` in
`FSTRING_MIDDLE` tokens, and not adding whitespace after
`FSTRING_MIDDLE` tokens

.vscode/settings.json

@@ -22,6 +22,9 @@
         "terminal.ansiBrightCyan": "#9cf0ed",
         "terminal.ansiBrightWhite": "#ffffff",
     },
+    "python.terminal.activateEnvironment": false,
+    "python.analysis.enablePytestSupport": false,
+    "python.analysis.typeCheckingMode": "standard",
     "python.testing.pytestEnabled": false,
     "python.testing.unittestEnabled": true,
     "python.testing.unittestArgs": [
@@ -31,23 +34,8 @@
         "-p",
         "test_*.py"
     ],
-    "python.linting.pylintEnabled": true,
-    "python.linting.flake8Enabled": true,
-    "python.linting.banditEnabled": true,
-    "python.linting.mypyEnabled": true,
-    "python.linting.flake8Args": [
-        "--max-line-length=120",
-        "--ignore=E722,F405,E203,W503,W293,E402,E501,E128,E226",
-    ],
-    "python.linting.banditArgs": [
-        "--ignore=B104,B110,B112"
-    ],
-    // python3 -m isort --py=27 --profile=black copyparty/
-    "python.formatting.provider": "none",
-    "[python]": {
-        "editor.defaultFormatter": "ms-python.black-formatter"
-    },
-    "editor.formatOnSave": true,
+    // python3 -m isort --py=27 --profile=black ~/dev/copyparty/{copyparty,tests}/*.py && python3 -m black -t py27 ~/dev/copyparty/{copyparty,tests,bin}/*.py $(find ~/dev/copyparty/copyparty/stolen -iname '*.py')
+    "editor.formatOnSave": false,
     "[html]": {
         "editor.formatOnSave": false,
         "editor.autoIndent": "keep",
@@ -58,6 +46,4 @@
     "files.associations": {
         "*.makefile": "makefile"
     },
-    "python.linting.enabled": true,
-    "python.pythonPath": "/usr/bin/python3"
 }

README.md

@@ -85,6 +85,7 @@ turn almost any device into a file server with resumable uploads/downloads using
     * [prometheus](#prometheus) - metrics/stats can be enabled
     * [other extremely specific features](#other-extremely-specific-features) - you'll never find a use for these
         * [custom mimetypes](#custom-mimetypes) - change the association of a file extension
+        * [feature chickenbits](#feature-chickenbits) - buggy feature? rip it out
 * [packages](#packages) - the party might be closer than you think
     * [arch package](#arch-package) - now [available on aur](https://aur.archlinux.org/packages/copyparty) maintained by [@icxes](https://github.com/icxes)
     * [fedora package](#fedora-package) - does not exist yet
@@ -111,6 +112,7 @@ turn almost any device into a file server with resumable uploads/downloads using
 * [HTTP API](#HTTP-API) - see [devnotes](./docs/devnotes.md#http-api)
 * [dependencies](#dependencies) - mandatory deps
     * [optional dependencies](#optional-dependencies) - install these to enable bonus features
+        * [dependency chickenbits](#dependency-chickenbits) - prevent loading an optional dependency
     * [optional gpl stuff](#optional-gpl-stuff)
 * [sfx](#sfx) - the self-contained "binary" (recommended!)
     * [copyparty.exe](#copypartyexe) - download [copyparty.exe](https://github.com/9001/copyparty/releases/latest/download/copyparty.exe) (win8+) or [copyparty32.exe](https://github.com/9001/copyparty/releases/latest/download/copyparty32.exe) (win7+)
@@ -124,7 +126,7 @@ turn almost any device into a file server with resumable uploads/downloads using
 
 just run **[copyparty-sfx.py](https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.py)** -- that's it! 🎉
 
-* or install through pypi: `python3 -m pip install --user -U copyparty`
+* or install through [pypi](https://pypi.org/project/copyparty/): `python3 -m pip install --user -U copyparty`
 * or if you cannot install python, you can use [copyparty.exe](#copypartyexe) instead
 * or install [on arch](#arch-package) ╱ [on NixOS](#nixos-module) ╱ [through nix](#nix-package)
 * or if you are on android, [install copyparty in termux](#install-on-android)
@@ -194,7 +196,7 @@ firewall-cmd --reload
 also see [comparison to similar software](./docs/versus.md)
 
 * backend stuff
-  * ☑ IPv6
+  * ☑ IPv6 + unix-sockets
   * ☑ [multiprocessing](#performance) (actual multithreading)
   * ☑ volumes (mountpoints)
   * ☑ [accounts](#accounts-and-volumes)
@@ -209,7 +211,7 @@ also see [comparison to similar software](./docs/versus.md)
 * upload
   * ☑ basic: plain multipart, ie6 support
   * ☑ [up2k](#uploading): js, resumable, multithreaded
-    * unaffected by cloudflare's max-upload-size (100 MiB)
+    * **no filesize limit!** ...unless you use Cloudflare, then it's 383.9 GiB
   * ☑ stash: simple PUT filedropper
   * ☑ filename randomizer
   * ☑ write-only folders
@@ -225,6 +227,7 @@ also see [comparison to similar software](./docs/versus.md)
   * ☑ [navpane](#navpane) (directory tree sidebar)
   * ☑ file manager (cut/paste, delete, [batch-rename](#batch-rename))
   * ☑ audio player (with [OS media controls](https://user-images.githubusercontent.com/241032/215347492-b4250797-6c90-4e09-9a4c-721edf2fb15c.png) and opus/mp3 transcoding)
+    * ☑ play video files as audio (converted on server)
   * ☑ image gallery with webm player
   * ☑ textfile browser with syntax hilighting
   * ☑ [thumbnails](#thumbnails)
@@ -578,9 +581,6 @@ images with the following names (see `--th-covers`) become the thumbnail of the
 * the order is significant, so if both `cover.png` and `folder.jpg` exist in a folder, it will pick the first matching `--th-covers` entry (`folder.jpg`)
 * and, if you enable [file indexing](#file-indexing), it will also try those names as dotfiles (`.folder.jpg` and so), and then fallback on the first picture in the folder (if it has any pictures at all)
 
-in the grid/thumbnail view, if the audio player panel is open, songs will start playing when clicked
-* indicated by the audio files having the ▶ icon instead of 💾
-
 enabling `multiselect` lets you click files to select them, and then shift-click another file for range-select
 * `multiselect` is mostly intended for phones/tablets, but the `sel` option in the `[⚙️] settings` tab is better suited for desktop use, allowing selection by CTRL-clicking and range-selection with SHIFT-click, all without affecting regular clicking
   * the `sel` option can be made default globally with `--gsel` or per-volume with volflag `gsel`
@@ -646,6 +646,7 @@ up2k has several advantages:
   * uploads resume if you reboot your browser or pc, just upload the same files again
   * server detects any corruption; the client reuploads affected chunks
   * the client doesn't upload anything that already exists on the server
+  * no filesize limit unless imposed by a proxy, for example Cloudflare, which blocks uploads over 383.9 GiB
 * much higher speeds than ftp/scp/tarpipe on some internet connections (mainly american ones) thanks to parallel connections
 * the last-modified timestamp of the file is preserved
 
@@ -800,6 +801,7 @@ some hilights:
 * OS integration; control playback from your phone's lockscreen ([windows](https://user-images.githubusercontent.com/241032/233213022-298a98ba-721a-4cf1-a3d4-f62634bc53d5.png) // [iOS](https://user-images.githubusercontent.com/241032/142711926-0700be6c-3e31-47b3-9928-53722221f722.png) // [android](https://user-images.githubusercontent.com/241032/233212311-a7368590-08c7-4f9f-a1af-48ccf3f36fad.png))
 * shows the audio waveform in the seekbar
 * not perfectly gapless but can get really close (see settings + eq below); good enough to enjoy gapless albums as intended
+* videos can be played as audio, without wasting bandwidth on the video
 
 click the `play` link next to an audio file, or copy the link target to [share it](https://a.ocv.me/pub/demo/music/Ubiktune%20-%20SOUNDSHOCK%202%20-%20FM%20FUNK%20TERRROR!!/#af-1fbfba61&t=18) (optionally with a timestamp to start playing from, like that example does)
 
@@ -987,7 +989,7 @@ some recommended FTP / FTPS clients; `wark` = example password:
 
 ## webdav server
 
-with read-write support,  supports winXP and later, macos, nautilus/gvfs
+with read-write support,  supports winXP and later, macos, nautilus/gvfs  ... a greay way to [access copyparty straight from the file explorer in your OS](#mount-as-drive)
 
 click the [connect](http://127.0.0.1:3923/?hc) button in the control-panel to see connection instructions for windows, linux, macos
 
@@ -1311,6 +1313,8 @@ you can set hooks before and/or after an event happens, and currently you can ho
 
 there's a bunch of flags and stuff, see `--help-hooks`
 
+if you want to write your own hooks, see [devnotes](./docs/devnotes.md#event-hooks)
+
 
 ### upload events
 
@@ -1455,6 +1459,8 @@ some reverse proxies (such as [Caddy](https://caddyserver.com/)) can automatical
 * **warning:** nginx-QUIC (HTTP/3) is still experimental and can make uploads much slower, so HTTP/1.1 is recommended for now
 * depending on server/client, HTTP/1.1 can also be 5x faster than HTTP/2
 
+for improved security (and a tiny performance boost) consider listening on a unix-socket with `-i /tmp/party.sock` instead of `-i 127.0.0.1`
+
 example webserver configs:
 
 * [nginx config](contrib/nginx/copyparty.conf) -- entire domain/subdomain
@@ -1555,6 +1561,23 @@ in a config-file, this is the same as:
 run copyparty with `--mimes` to list all the default mappings
 
 
+### feature chickenbits
+
+buggy feature? rip it out  by setting any of the following environment variables to disable its associated bell or whistle,
+
+| env-var              | what it does |
+| -------------------- | ------------ |
+| `PRTY_NO_IFADDR`     | disable ip/nic discovery by poking into your OS with ctypes |
+| `PRTY_NO_IPV6`       | disable some ipv6 support (should not be necessary since windows 2000) |
+| `PRTY_NO_LZMA`       | disable streaming xz compression of incoming uploads |
+| `PRTY_NO_MP`         | disable all use of the python `multiprocessing` module (actual multithreading, cpu-count for parsers/thumbnailers) |
+| `PRTY_NO_SQLITE`     | disable all database-related functionality (file indexing, metadata indexing, most file deduplication logic) |
+| `PRTY_NO_TLS`        | disable native HTTPS support; if you still want to accept HTTPS connections then TLS must now be terminated by a reverse-proxy |
+| `PRTY_NO_TPOKE`      | disable systemd-tmpfilesd avoider |
+
+example: `PRTY_NO_IFADDR=1 python3 copyparty-sfx.py`
+
+
 # packages
 
 the party might be closer than you think
@@ -1798,7 +1821,7 @@ alternatively, some alternatives roughly sorted by speed (unreproducible benchma
 * [rclone-http](./docs/rclone.md) (26s), read-only
 * [partyfuse.py](./bin/#partyfusepy) (35s), read-only
 * [rclone-ftp](./docs/rclone.md) (47s), read/WRITE
-* davfs2 (103s), read/WRITE, *very fast* on small files
+* davfs2 (103s), read/WRITE
 * [win10-webdav](#webdav-server) (138s), read/WRITE
 * [win10-smb2](#smb-server) (387s), read/WRITE
 
@@ -1877,6 +1900,7 @@ some notes on hardening
   * cors doesn't work right otherwise
 * if you allow anonymous uploads or otherwise don't trust the contents of a volume, you can prevent XSS with volflag `nohtml`
   * this returns html documents as plaintext, and also disables markdown rendering
+* when running behind a reverse-proxy, listen on a unix-socket with `-i /tmp/party.sock` instead of `-i 127.0.0.1` for tighter access control (plus you get a tiny performance boost for free)
 
 safety profiles:
 
@@ -2041,6 +2065,37 @@ enable [smb](#smb-server) support (**not** recommended):
 `pyvips` gives higher quality thumbnails than `Pillow` and is 320% faster, using 270% more ram: `sudo apt install libvips42 && python3 -m pip install --user -U pyvips`
 
 
+### dependency chickenbits
+
+prevent loading an optional dependency  , for example if:
+
+* you have an incompatible version installed and it causes problems
+* you just don't want copyparty to use it, maybe to save ram
+
+set any of the following environment variables to disable its associated optional feature,
+
+| env-var              | what it does |
+| -------------------- | ------------ |
+| `PRTY_NO_ARGON2`     | disable argon2-cffi password hashing |
+| `PRTY_NO_CFSSL`      | never attempt to generate self-signed certificates using [cfssl](https://github.com/cloudflare/cfssl) |
+| `PRTY_NO_FFMPEG`     | **audio transcoding** goes byebye, **thumbnailing** must be handled by Pillow/libvips |
+| `PRTY_NO_FFPROBE`    | **audio transcoding** goes byebye, **thumbnailing** must be handled by Pillow/libvips, **metadata-scanning** must be handled by mutagen |
+| `PRTY_NO_MUTAGEN`    | do not use [mutagen](https://pypi.org/project/mutagen/) for reading metadata from media files; will fallback to ffprobe |
+| `PRTY_NO_PIL`        | disable all [Pillow](https://pypi.org/project/pillow/)-based thumbnail support; will fallback to libvips or ffmpeg |
+| `PRTY_NO_PILF`       | disable Pillow `ImageFont` text rendering, used for folder thumbnails |
+| `PRTY_NO_PIL_AVIF`   | disable 3rd-party Pillow plugin for [AVIF support](https://pypi.org/project/pillow-avif-plugin/) |
+| `PRTY_NO_PIL_HEIF`   | disable 3rd-party Pillow plugin for [HEIF support](https://pypi.org/project/pyheif-pillow-opener/) |
+| `PRTY_NO_PIL_WEBP`   | disable use of native webp support in Pillow |
+| `PRTY_NO_PSUTIL`     | do not use [psutil](https://pypi.org/project/psutil/) for reaping stuck hooks and plugins on Windows |
+| `PRTY_NO_VIPS`       | disable all [libvips](https://pypi.org/project/pyvips/)-based thumbnail support; will fallback to Pillow or ffmpeg |
+
+example: `PRTY_NO_PIL=1 python3 copyparty-sfx.py`
+
+* `PRTY_NO_PIL` saves ram
+* `PRTY_NO_VIPS` saves ram and startup time
+* python2.7 on windows: `PRTY_NO_FFMPEG` + `PRTY_NO_FFPROBE` saves startup time
+
+
 ## optional gpl stuff
 
 some bundled tools have copyleft dependencies, see [./bin/#mtag](bin/#mtag)

bin/hooks/README.md

@@ -2,7 +2,7 @@ standalone programs which are executed by copyparty when an event happens (uploa
 
 these programs either take zero arguments, or a filepath (the affected file), or a json message with filepath + additional info
 
-run copyparty with `--help-hooks` for usage details / hook type explanations (xbu/xau/xiu/xbr/xar/xbd/xad)
+run copyparty with `--help-hooks` for usage details / hook type explanations (xm/xbu/xau/xiu/xbr/xar/xbd/xad/xban)
 
 > **note:** in addition to event hooks (the stuff described here), copyparty has another api to run your programs/scripts while providing way more information such as audio tags / video codecs / etc and optionally daisychaining data between scripts in a processing pipeline; if that's what you want then see [mtp plugins](../mtag/) instead
 
@@ -13,6 +13,7 @@ run copyparty with `--help-hooks` for usage details / hook type explanations (xb
 * [image-noexif.py](image-noexif.py) removes image exif by overwriting / directly editing the uploaded file
 * [discord-announce.py](discord-announce.py) announces new uploads on discord using webhooks ([example](https://user-images.githubusercontent.com/241032/215304439-1c1cb3c8-ec6f-4c17-9f27-81f969b1811a.png))
 * [reject-mimetype.py](reject-mimetype.py) rejects uploads unless the mimetype is acceptable
+* [into-the-cache-it-goes.py](into-the-cache-it-goes.py) avoids bugs in caching proxies by immediately downloading each file that is uploaded
 
 
 # upload batches
@@ -23,6 +24,7 @@ these are `--xiu` hooks; unlike `xbu` and `xau` (which get executed on every sin
 
 # before upload
 * [reject-extension.py](reject-extension.py) rejects uploads if they match a list of file extensions
+* [reloc-by-ext.py](reloc-by-ext.py) redirects an upload to another destination based on the file extension
 
 
 # on message

bin/hooks/into-the-cache-it-goes.py

@@ -0,0 +1,140 @@
+#!/usr/bin/env python3
+
+import sys
+import json
+import shutil
+import platform
+import subprocess as sp
+from urllib.parse import quote
+
+
+_ = r"""
+try to avoid race conditions in caching proxies
+(primarily cloudflare, but probably others too)
+by means of the most obvious solution possible:
+
+just as each file has finished uploading, use
+the server's external URL to download the file
+so that it ends up in the cache, warm and snug
+
+this intentionally delays the upload response
+as it waits for the file to finish downloading
+before copyparty is allowed to return the URL
+
+NOTE: you must edit this script before use,
+  replacing https://example.com with your URL
+
+NOTE: if the files are only accessible with a
+  password and/or filekey, you must also add
+  a cromulent password in the PASSWORD field
+
+NOTE: needs either wget, curl, or "requests":
+  python3 -m pip install --user -U requests
+
+
+example usage as global config:
+    --xau j,t10,bin/hooks/into-the-cache-it-goes.py
+
+parameters explained,
+    xau = execute after upload
+    j   = this hook needs upload information as json (not just the filename)
+    t10 = abort download and continue if it takes longer than 10sec
+
+example usage as a volflag (per-volume config):
+    -v srv/inc:inc:r:rw,ed:c,xau=j,t10,bin/hooks/into-the-cache-it-goes.py
+                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+    (share filesystem-path srv/inc as volume /inc,
+     readable by everyone, read-write for user 'ed',
+     running this plugin on all uploads with params explained above)
+
+example usage as a volflag in a copyparty config file:
+    [/inc]
+      srv/inc
+      accs:
+        r: *
+        rw: ed
+      flags:
+        xau: j,t10,bin/hooks/into-the-cache-it-goes.py
+"""
+
+
+# replace this with your site's external URL
+# (including the :portnumber if necessary)
+SITE_URL = "https://example.com"
+
+# if downloading is protected by passwords or filekeys,
+# specify a valid password between the quotes below:
+PASSWORD = ""
+
+# if file is larger than this, skip download
+MAX_MEGABYTES = 8
+
+# =============== END OF CONFIG ===============
+
+
+WINDOWS = platform.system() == "Windows"
+
+
+def main():
+    fun = download_with_python
+    if shutil.which("curl"):
+        fun = download_with_curl
+    elif shutil.which("wget"):
+        fun = download_with_wget
+
+    inf = json.loads(sys.argv[1])
+
+    if inf["sz"] > 1024 * 1024 * MAX_MEGABYTES:
+        print("[into-the-cache] file is too large; will not download")
+        return
+
+    file_url = "/"
+    if inf["vp"]:
+        file_url += inf["vp"] + "/"
+    file_url += inf["ap"].replace("\\", "/").split("/")[-1]
+    file_url = SITE_URL.rstrip("/") + quote(file_url, safe=b"/")
+
+    print("[into-the-cache] %s(%s)" % (fun.__name__, file_url))
+    fun(file_url, PASSWORD.strip())
+
+    print("[into-the-cache] Download OK")
+
+
+def download_with_curl(url, pw):
+    cmd = ["curl"]
+
+    if pw:
+        cmd += ["-HPW:%s" % (pw,)]
+
+    nah = sp.DEVNULL
+    sp.check_call(cmd + [url], stdout=nah, stderr=nah)
+
+
+def download_with_wget(url, pw):
+    cmd = ["wget", "-O"]
+
+    cmd += ["nul" if WINDOWS else "/dev/null"]
+
+    if pw:
+        cmd += ["--header=PW:%s" % (pw,)]
+
+    nah = sp.DEVNULL
+    sp.check_call(cmd + [url], stdout=nah, stderr=nah)
+
+
+def download_with_python(url, pw):
+    import requests
+
+    headers = {}
+    if pw:
+        headers["PW"] = pw
+
+    with requests.get(url, headers=headers, stream=True) as r:
+        r.raise_for_status()
+        for _ in r.iter_content(chunk_size=1024 * 256):
+            pass
+
+
+if __name__ == "__main__":
+    main()

bin/hooks/qbittorrent-magnet.py

@@ -23,17 +23,18 @@ because the keyword "anime" is in the DESTS config below
 needs python3
 
 example usage as global config (not a good idea):
-    python copyparty-sfx.py --xm f,j,t60,bin/hooks/qbittorrent-magnet.py
+    python copyparty-sfx.py --xm aw,f,j,t60,bin/hooks/qbittorrent-magnet.py
 
 parameters explained,
     xm = execute on message (📟)
+    aw = only users with write-access can use this
     f = fork; don't delay other hooks while this is running
     j = provide message information as json (not just the text)
     t60 = abort if qbittorrent has to think about it for more than 1 min
 
 example usage as a volflag (per-volume config, much better):
-    -v srv/qb:qb:A,ed:c,xm=f,j,t60,bin/hooks/qbittorrent-magnet.py
-                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+    -v srv/qb:qb:A,ed:c,xm=aw,f,j,t60,bin/hooks/qbittorrent-magnet.py
+                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
     (share filesystem-path srv/qb as volume /qb with Admin for user 'ed',
      running this plugin on all messages with the params explained above)
@@ -44,7 +45,7 @@ example usage as a volflag in a copyparty config file:
       accs:
         A: ed
       flags:
-        xm: f,j,t60,bin/hooks/qbittorrent-magnet.py
+        xm: aw,f,j,t60,bin/hooks/qbittorrent-magnet.py
 
 the volflag examples only kicks in if you send the torrent magnet
 while you're in the /qb folder (or any folder below there)

bin/hooks/reloc-by-ext.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+
+import json
+import os
+import sys
+
+
+_ = r"""
+relocate/redirect incoming uploads according to file extension
+
+example usage as global config:
+    --xbu j,c1,bin/hooks/reloc-by-ext.py
+
+parameters explained,
+    xbu = execute before upload
+    j   = this hook needs upload information as json (not just the filename)
+    c1  = this hook returns json on stdout, so tell copyparty to read that
+
+example usage as a volflag (per-volume config):
+    -v srv/inc:inc:r:rw,ed:c,xbu=j,c1,bin/hooks/reloc-by-ext.py
+                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+    (share filesystem-path srv/inc as volume /inc,
+     readable by everyone, read-write for user 'ed',
+     running this plugin on all uploads with the params explained above)
+
+example usage as a volflag in a copyparty config file:
+    [/inc]
+      srv/inc
+      accs:
+        r: *
+        rw: ed
+      flags:
+        xbu: j,c1,bin/hooks/reloc-by-ext.py
+
+note: this could also work as an xau hook (after-upload), but
+  because it doesn't need to read the file contents its better
+  as xbu (before-upload) since that's safer / less buggy,
+  and only xbu works with up2k (dragdrop into browser)
+"""
+
+
+PICS = "avif bmp gif heic heif jpeg jpg jxl png psd qoi tga tif tiff webp"
+VIDS = "3gp asf avi flv mkv mov mp4 mpeg mpeg2 mpegts mpg mpg2 nut ogm ogv rm ts vob webm wmv"
+MUSIC = "aac aif aiff alac amr ape dfpwm flac m4a mp3 ogg opus ra tak tta wav wma wv"
+
+
+def main():
+    inf = json.loads(sys.argv[1])
+    vdir, fn = os.path.split(inf["vp"])
+
+    try:
+        fn, ext = fn.rsplit(".", 1)
+    except:
+        # no file extension; abort
+        return
+
+    ext = ext.lower()
+
+    ##
+    ## some example actions to take; pick one by
+    ## selecting it inside the print at the end:
+    ##
+
+    # create a subfolder named after the filetype and move it into there
+    into_subfolder = {"vp": ext}
+
+    # move it into a toplevel folder named after the filetype
+    into_toplevel = {"vp": "/" + ext}
+
+    # move it into a filetype-named folder next to the target folder
+    into_sibling = {"vp": "../" + ext}
+
+    # move images into "/just/pics", vids into "/just/vids",
+    # music into "/just/tunes", and anything else as-is
+    if ext in PICS.split():
+        by_category = {"vp": "/just/pics"}
+    elif ext in VIDS.split():
+        by_category = {"vp": "/just/vids"}
+    elif ext in MUSIC.split():
+        by_category = {"vp": "/just/tunes"}
+    else:
+        by_category = {}
+
+    # now choose the effect to apply; can be any of these:
+    # into_subfolder  into_toplevel  into_sibling  by_category
+    effect = into_subfolder
+    print(json.dumps({"reloc": effect}))
+
+
+if __name__ == "__main__":
+    main()

bin/hooks/wget.py

@@ -12,18 +12,19 @@ application/x-www-form-urlencoded (for example using the
 📟 message-to-server-log in the web-ui)
 
 example usage as global config:
-    --xm f,j,t3600,bin/hooks/wget.py
+    --xm aw,f,j,t3600,bin/hooks/wget.py
 
 parameters explained,
     xm = execute on message-to-server-log
+    aw = only users with write-access can use this
     f = fork; don't delay other hooks while this is running
     j = provide message information as json (not just the text)
     c3 = mute all output
     t3600 = timeout and abort download after 1 hour
 
 example usage as a volflag (per-volume config):
-    -v srv/inc:inc:r:rw,ed:c,xm=f,j,t3600,bin/hooks/wget.py
-                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+    -v srv/inc:inc:r:rw,ed:c,xm=aw,f,j,t3600,bin/hooks/wget.py
+                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
     (share filesystem-path srv/inc as volume /inc,
      readable by everyone, read-write for user 'ed',
@@ -36,7 +37,7 @@ example usage as a volflag in a copyparty config file:
         r: *
         rw: ed
       flags:
-        xm: f,j,t3600,bin/hooks/wget.py     
+        xm: aw,f,j,t3600,bin/hooks/wget.py
 
 the volflag examples only kicks in if you send the message
 while you're in the /inc folder (or any folder below there)

bin/u2c.py

@@ -1,8 +1,8 @@
 #!/usr/bin/env python3
 from __future__ import print_function, unicode_literals
 
-S_VERSION = "1.18"
-S_BUILD_DT = "2024-06-01"
+S_VERSION = "1.22"
+S_BUILD_DT = "2024-08-08"
 
 """
 u2c.py: upload to copyparty
@@ -20,6 +20,7 @@ import sys
 import stat
 import math
 import time
+import json
 import atexit
 import signal
 import socket
@@ -79,7 +80,7 @@ req_ses = requests.Session()
 
 
 class Daemon(threading.Thread):
-    def __init__(self, target, name = None, a = None):
+    def __init__(self, target, name=None, a=None):
         threading.Thread.__init__(self, name=name)
         self.a = a or ()
         self.fun = target
@@ -110,18 +111,22 @@ class File(object):
         # set by get_hashlist
         self.cids = []  # type: list[tuple[str, int, int]]  # [ hash, ofs, sz ]
         self.kchunks = {}  # type: dict[str, tuple[int, int]]  # hash: [ ofs, sz ]
+        self.t_hash = 0.0  # type: float
 
         # set by handshake
         self.recheck = False  # duplicate; redo handshake after all files done
         self.ucids = []  # type: list[str]  # chunks which need to be uploaded
         self.wark = ""  # type: str
         self.url = ""  # type: str
-        self.nhs = 0
+        self.nhs = 0  # type: int
 
         # set by upload
+        self.t0_up = 0.0  # type: float
+        self.t1_up = 0.0  # type: float
+        self.nojoin = 0  # type: int
         self.up_b = 0  # type: int
         self.up_c = 0  # type: int
-        self.cd = 0
+        self.cd = 0  # type: int
 
         # t = "size({}) lmod({}) top({}) rel({}) abs({}) name({})\n"
         # eprint(t.format(self.size, self.lmod, self.top, self.rel, self.abs, self.name))
@@ -130,10 +135,20 @@ class File(object):
 class FileSlice(object):
     """file-like object providing a fixed window into a file"""
 
-    def __init__(self, file, cid):
+    def __init__(self, file, cids):
         # type: (File, str) -> None
 
-        self.car, self.len = file.kchunks[cid]
+        self.file = file
+        self.cids = cids
+
+        self.car, tlen = file.kchunks[cids[0]]
+        for cid in cids[1:]:
+            ofs, clen = file.kchunks[cid]
+            if ofs != self.car + tlen:
+                raise Exception(9)
+            tlen += clen
+
+        self.len = tlen
         self.cdr = self.car + self.len
         self.ofs = 0  # type: int
         self.f = open(file.abs, "rb", 512 * 1024)
@@ -357,7 +372,7 @@ def undns(url):
     usp = urlsplit(url)
     hn = usp.hostname
     gai = None
-    eprint("resolving host [{0}] ...".format(hn), end="")
+    eprint("resolving host [%s] ..." % (hn,))
     try:
         gai = socket.getaddrinfo(hn, None)
         hn = gai[0][4][0]
@@ -375,7 +390,7 @@ def undns(url):
 
     usp = usp._replace(netloc=hn)
     url = urlunsplit(usp)
-    eprint(" {0}".format(url))
+    eprint(" %s\n" % (url,))
     return url
 
 
@@ -518,6 +533,8 @@ def get_hashlist(file, pcb, mth):
     file_ofs = 0
     ret = []
     with open(file.abs, "rb", 512 * 1024) as f:
+        t0 = time.time()
+
         if mth and file.size >= 1024 * 512:
             ret = mth.hash(f, file.size, chunk_sz, pcb, file)
             file_rem = 0
@@ -544,10 +561,12 @@ def get_hashlist(file, pcb, mth):
             if pcb:
                 pcb(file, file_ofs)
 
+    file.t_hash = time.time() - t0
     file.cids = ret
     file.kchunks = {}
     for k, v1, v2 in ret:
-        file.kchunks[k] = [v1, v2]
+        if k not in file.kchunks:
+            file.kchunks[k] = [v1, v2]
 
 
 def handshake(ar, file, search):
@@ -589,7 +608,8 @@ def handshake(ar, file, search):
         sc = 600
         txt = ""
         try:
-            r = req_ses.post(url, headers=headers, json=req)
+            zs = json.dumps(req, separators=(",\n", ": "))
+            r = req_ses.post(url, headers=headers, data=zs)
             sc = r.status_code
             txt = r.text
             if sc < 400:
@@ -636,13 +656,20 @@ def handshake(ar, file, search):
     return r["hash"], r["sprs"]
 
 
-def upload(file, cid, pw, stats):
-    # type: (File, str, str, str) -> None
-    """upload one specific chunk, `cid` (a chunk-hash)"""
+def upload(fsl, pw, stats):
+    # type: (FileSlice, str, str) -> None
+    """upload a range of file data, defined by one or more `cid` (chunk-hash)"""
+
+    ctxt = fsl.cids[0]
+    if len(fsl.cids) > 1:
+        n = 192 // len(fsl.cids)
+        n = 9 if n > 9 else 2 if n < 2 else n
+        zsl = [zs[:n] for zs in fsl.cids[1:]]
+        ctxt += ",%d,%s" % (n, "".join(zsl))
 
     headers = {
-        "X-Up2k-Hash": cid,
-        "X-Up2k-Wark": file.wark,
+        "X-Up2k-Hash": ctxt,
+        "X-Up2k-Wark": fsl.file.wark,
         "Content-Type": "application/octet-stream",
     }
 
@@ -652,15 +679,24 @@ def upload(file, cid, pw, stats):
     if pw:
         headers["Cookie"] = "=".join(["cppwd", pw])
 
-    f = FileSlice(file, cid)
     try:
-        r = req_ses.post(file.url, headers=headers, data=f)
+        r = req_ses.post(fsl.file.url, headers=headers, data=fsl)
+
+        if r.status_code == 400:
+            txt = r.text
+            if (
+                "already being written" in txt
+                or "already got that" in txt
+                or "only sibling chunks" in txt
+            ):
+                fsl.file.nojoin = 1
+
         if not r:
             raise Exception(repr(r))
 
         _ = r.content
     finally:
-        f.f.close()
+        fsl.f.close()
 
 
 class Ctl(object):
@@ -724,6 +760,9 @@ class Ctl(object):
         if ar.safe:
             self._safe()
         else:
+            self.at_hash = 0.0
+            self.at_up = 0.0
+            self.at_upr = 0.0
             self.hash_f = 0
             self.hash_c = 0
             self.hash_b = 0
@@ -743,7 +782,7 @@ class Ctl(object):
 
             self.mutex = threading.Lock()
             self.q_handshake = Queue()  # type: Queue[File]
-            self.q_upload = Queue()  # type: Queue[tuple[File, str]]
+            self.q_upload = Queue()  # type: Queue[FileSlice]
 
             self.st_hash = [None, "(idle, starting...)"]  # type: tuple[File, int]
             self.st_up = [None, "(idle, starting...)"]  # type: tuple[File, int]
@@ -788,7 +827,8 @@ class Ctl(object):
                 for nc, cid in enumerate(hs):
                     print("  {0} up {1}".format(ncs - nc, cid))
                     stats = "{0}/0/0/{1}".format(nf, self.nfiles - nf)
-                    upload(file, cid, self.ar.a, stats)
+                    fslice = FileSlice(file, [cid])
+                    upload(fslice, self.ar.a, stats)
 
             print("  ok!")
             if file.recheck:
@@ -797,7 +837,7 @@ class Ctl(object):
         if not self.recheck:
             return
 
-        eprint("finalizing {0} duplicate files".format(len(self.recheck)))
+        eprint("finalizing %d duplicate files\n" % (len(self.recheck),))
         for file in self.recheck:
             handshake(self.ar, file, search)
 
@@ -871,10 +911,17 @@ class Ctl(object):
             t = "{0} eta @ {1}/s, {2}, {3}# left".format(self.eta, spd, sleft, nleft)
             eprint(txt + "\033]0;{0}\033\\\r{0}{1}".format(t, tail))
 
+        if self.hash_b and self.at_hash:
+            spd = humansize(self.hash_b / self.at_hash)
+            eprint("\nhasher: %.2f sec, %s/s\n" % (self.at_hash, spd))
+        if self.up_b and self.at_up:
+            spd = humansize(self.up_b / self.at_up)
+            eprint("upload: %.2f sec, %s/s\n" % (self.at_up, spd))
+
         if not self.recheck:
             return
 
-        eprint("finalizing {0} duplicate files".format(len(self.recheck)))
+        eprint("finalizing %d duplicate files\n" % (len(self.recheck),))
         for file in self.recheck:
             handshake(self.ar, file, False)
 
@@ -1060,21 +1107,62 @@ class Ctl(object):
                 self.handshaker_busy -= 1
 
             if not hs:
-                kw = "uploaded" if file.up_b else "   found"
-                print("{0} {1}".format(kw, upath))
-            for cid in hs:
-                self.q_upload.put([file, cid])
+                self.at_hash += file.t_hash
+
+                if self.ar.spd:
+                    if VT100:
+                        c1 = "\033[36m"
+                        c2 = "\033[0m"
+                    else:
+                        c1 = c2 = ""
+
+                    spd_h = humansize(file.size / file.t_hash, True)
+                    if file.up_b:
+                        t_up = file.t1_up - file.t0_up
+                        spd_u = humansize(file.size / t_up, True)
+
+                        t = "uploaded %s %s(h:%.2fs,%s/s,up:%.2fs,%s/s)%s"
+                        print(t % (upath, c1, file.t_hash, spd_h, t_up, spd_u, c2))
+                    else:
+                        t = "   found %s %s(%.2fs,%s/s)%s"
+                        print(t % (upath, c1, file.t_hash, spd_h, c2))
+                else:
+                    kw = "uploaded" if file.up_b else "   found"
+                    print("{0} {1}".format(kw, upath))
+
+            chunksz = up2k_chunksize(file.size)
+            njoin = (self.ar.sz * 1024 * 1024) // chunksz
+            cs = hs[:]
+            while cs:
+                fsl = FileSlice(file, cs[:1])
+                try:
+                    if file.nojoin:
+                        raise Exception()
+                    for n in range(2, min(len(cs), njoin + 1)):
+                        fsl = FileSlice(file, cs[:n])
+                except:
+                    pass
+                cs = cs[len(fsl.cids) :]
+                self.q_upload.put(fsl)
 
     def uploader(self):
         while True:
-            task = self.q_upload.get()
-            if not task:
+            fsl = self.q_upload.get()
+            if not fsl:
                 self.st_up = [None, "(finished)"]
                 break
 
+            file = fsl.file
+            cids = fsl.cids
+
             with self.mutex:
+                if not self.uploader_busy:
+                    self.at_upr = time.time()
                 self.uploader_busy += 1
-                self.t0_up = self.t0_up or time.time()
+                if not file.t0_up:
+                    file.t0_up = time.time()
+                    if not self.t0_up:
+                        self.t0_up = file.t0_up
 
             stats = "%d/%d/%d/%d %d/%d %s" % (
                 self.up_f,
@@ -1086,28 +1174,30 @@ class Ctl(object):
                 self.eta,
             )
 
-            file, cid = task
             try:
-                upload(file, cid, self.ar.a, stats)
+                upload(fsl, self.ar.a, stats)
             except Exception as ex:
-                t = "upload failed, retrying: {0} #{1} ({2})\n"
-                eprint(t.format(file.name, cid[:8], ex))
+                t = "upload failed, retrying: %s #%s+%d (%s)\n"
+                eprint(t % (file.name, cids[0][:8], len(cids) - 1, ex))
                 file.cd = time.time() + self.ar.cd
                 # handshake will fix it
 
             with self.mutex:
-                sz = file.kchunks[cid][1]
-                file.ucids = [x for x in file.ucids if x != cid]
+                sz = fsl.len
+                file.ucids = [x for x in file.ucids if x not in cids]
                 if not file.ucids:
+                    file.t1_up = time.time()
                     self.q_handshake.put(file)
 
-                self.st_up = [file, cid]
+                self.st_up = [file, cids[0]]
                 file.up_b += sz
                 self.up_b += sz
                 self.up_br += sz
                 file.up_c += 1
                 self.up_c += 1
                 self.uploader_busy -= 1
+                if not self.uploader_busy:
+                    self.at_up += time.time() - self.at_upr
 
     def up_done(self, file):
         if self.ar.dl:
@@ -1150,6 +1240,7 @@ source file/folder selection uses rsync syntax, meaning that:
     ap.add_argument("--ok", action="store_true", help="continue even if some local files are inaccessible")
     ap.add_argument("--touch", action="store_true", help="if last-modified timestamps differ, push local to server (need write+delete perms)")
     ap.add_argument("--ow", action="store_true", help="overwrite existing files instead of autorenaming")
+    ap.add_argument("--spd", action="store_true", help="print speeds for each file")
     ap.add_argument("--version", action="store_true", help="show version and exit")
 
     ap = app.add_argument_group("compatibility")
@@ -1164,6 +1255,7 @@ source file/folder selection uses rsync syntax, meaning that:
     ap = app.add_argument_group("performance tweaks")
     ap.add_argument("-j", type=int, metavar="CONNS", default=2, help="parallel connections")
     ap.add_argument("-J", type=int, metavar="CORES", default=hcores, help="num cpu-cores to use for hashing; set 0 or 1 for single-core hashing")
+    ap.add_argument("--sz", type=int, metavar="MiB", default=64, help="try to make each POST this big")
     ap.add_argument("-nh", action="store_true", help="disable hashing while uploading")
     ap.add_argument("-ns", action="store_true", help="no status panel (for slow consoles and macos)")
     ap.add_argument("--cd", type=float, metavar="SEC", default=5, help="delay before reattempting a failed handshake/upload")
@@ -1208,7 +1300,7 @@ source file/folder selection uses rsync syntax, meaning that:
     ar.url = ar.url.rstrip("/") + "/"
     if "://" not in ar.url:
         ar.url = "http://" + ar.url
-    
+
     if "https://" in ar.url.lower():
         try:
             import ssl, zipfile

contrib/package/arch/PKGBUILD

@@ -1,6 +1,6 @@
 # Maintainer: icxes <dev.null@need.moe>
 pkgname=copyparty
-pkgver="1.13.3"
+pkgver="1.13.6"
 pkgrel=1
 pkgdesc="File server with accelerated resumable uploads, dedup, WebDAV, FTP, TFTP, zeroconf, media indexer, thumbnails++"
 arch=("any")
@@ -21,7 +21,7 @@ optdepends=("ffmpeg: thumbnails for videos, images (slower) and audio, music tag
 )
 source=("https://github.com/9001/${pkgname}/releases/download/v${pkgver}/${pkgname}-${pkgver}.tar.gz")
 backup=("etc/${pkgname}.d/init" )
-sha256sums=("35845d6335fba4a13d153d7062f365dad529202bc865b93267d899e19a0a6da3")
+sha256sums=("abb9d0df24c4082594adc2070b7e9fb84a57733d0a3fbf83f4ec2a02813dd717")
 
 build() {
     cd "${srcdir}/${pkgname}-${pkgver}"

contrib/package/nix/copyparty/pin.json

@@ -1,5 +1,5 @@
 {
-    "url": "https://github.com/9001/copyparty/releases/download/v1.13.3/copyparty-sfx.py",
-    "version": "1.13.3",
-    "hash": "sha256-LtbdioAYtWGC4+5frzUjXwm0thubkyMhc86YU/rXIuo="
+    "url": "https://github.com/9001/copyparty/releases/download/v1.13.6/copyparty-sfx.py",
+    "version": "1.13.6",
+    "hash": "sha256-eQSHZYXIbS4piCU/zOyEz3E6V/lS6348vENMLwONYUM="
 }

contrib/plugins/README.md

@@ -20,6 +20,13 @@ point `--js-browser` to one of these by URL:
 
 
 
+## example any-js
+point `--js-browser` and/or `--js-other` to one of these by URL:
+
+* [`banner.js`](banner.js) shows a very enterprise [legal-banner](https://github.com/user-attachments/assets/8ae8e087-b209-449c-b08d-74e040f0284b)
+
+
+
 ## example browser-css
 point `--css-browser` to one of these by URL:
 

contrib/plugins/banner.js

@@ -0,0 +1,93 @@
+(function() {
+
+// usage: copy this to '.banner.js' in your webroot,
+// and run copyparty with the following argument:
+// --body-foot '<script src="/.banner.js"></script>'
+
+
+
+// had to pick the most chuuni one as the default
+var bannertext = '' +
+'<h3>You are accessing a U.S. Government (USG) Information System (IS) that is provided for USG-authorized use only.</h3>' +
+'<p>By using this IS (which includes any device attached to this IS), you consent to the following conditions:</p>' +
+'<ul>' +
+'<li>The USG routinely intercepts and monitors communications on this IS for purposes including, but not limited to, penetration testing, COMSEC monitoring, network operations and defense, personnel misconduct (PM), law enforcement (LE), and counterintelligence (CI) investigations.</li>' +
+'<li>At any time, the USG may inspect and seize data stored on this IS.</li>' +
+'<li>Communications using, or data stored on, this IS are not private, are subject to routine monitoring, interception, and search, and may be disclosed or used for any USG-authorized purpose.</li>' +
+'<li>This IS includes security measures (e.g., authentication and access controls) to protect USG interests -- not for your personal benefit or privacy.</li>' +
+'<li>Notwithstanding the above, using this IS does not constitute consent to PM, LE or CI investigative searching or monitoring of the content of privileged communications, or work product, related to personal representation or services by attorneys, psychotherapists, or clergy, and their assistants. Such communications and work product are private and confidential. See User Agreement for details.</li>' +
+'</ul>';
+
+
+
+// fancy div to insert into pages
+function bannerdiv(border) {
+    var ret = mknod('div', null, bannertext);
+    if (border)
+        ret.setAttribute("style", "border:1em solid var(--fg); border-width:.3em 0; margin:3em 0");
+    return ret;
+}
+
+
+
+// keep all of these false and then selectively enable them in the if-blocks below
+var show_msgbox = false,
+    login_top = false,
+    top = false,
+    bottom = false,
+    top_bordered = false,
+    bottom_bordered = false;
+
+if (QS("h1#cc") && QS("a#k")) {
+    // this is the controlpanel
+    // (you probably want to keep just one of these enabled)
+    show_msgbox = true;
+    login_top = true;
+    bottom = true;
+}
+else if (ebi("swin") && ebi("smac")) {
+    // this is the connect-page, same deal here
+    show_msgbox = true;
+    top_bordered = true;
+    bottom_bordered = true;
+}
+else if (ebi("op_cfg") || ebi("div#mw") ) {
+    // we're running in the main filebrowser (op_cfg) or markdown-viewer/editor (div#mw),
+    // fragile pages which break if you do something too fancy
+    show_msgbox = true;
+}
+
+
+
+// shows a fullscreen messagebox; works on all pages
+if (show_msgbox) {
+    var now = Math.floor(Date.now() / 1000),
+        last_shown = sread("bannerts") || 0;
+
+    // 60 * 60 * 17 = 17 hour cooldown
+    if (now - last_shown > 60 * 60 * 17) {
+        swrite("bannerts", now);
+        modal.confirm(bannertext, null, function () {
+            location = 'https://this-page-intentionally-left-blank.org/';
+        });
+    }
+}
+
+// show a message on the page footer; only works on the connect-page
+if (top || top_bordered) {
+    var dst = ebi('wrap');
+    dst.insertBefore(bannerdiv(top_bordered), dst.firstChild);
+}
+
+// show a message on the page footer; only works on the controlpanel and connect-page
+if (bottom || bottom_bordered) {
+    ebi('wrap').appendChild(bannerdiv(bottom_bordered));
+}
+
+// show a message on the top of the page; only works on the controlpanel
+if (login_top) {
+    var dst = QS('h1');
+    dst.parentNode.insertBefore(bannerdiv(false), dst);
+}
+
+})();

contrib/themes/bsod.css

@@ -0,0 +1,118 @@
+/* copy bsod.* into a folder named ".themes" in your webroot and then
+     --themes=10 --theme=9 --css-browser=/.themes/bsod.css
+*/
+
+html.ey {
+    --w2: #3d7bbc;
+    --w3: #5fcbec;
+
+    --fg: #fff;
+    --fg-max: #fff;
+    --fg-weak: var(--w3);
+
+    --bg: #2067b2;
+    --bg-d3: var(--bg);
+    --bg-d2: var(--w2);
+    --bg-d1: var(--fg-weak);
+    --bg-u2: var(--bg);
+    --bg-u3: var(--bg);
+    --bg-u5: var(--w2);
+
+    --tab-alt: var(--fg-weak);
+	--row-alt: var(--w2);
+
+    --scroll: var(--w3);
+
+    --a: #fff;
+    --a-b: #fff;
+    --a-hil: #fff;
+    --a-h-bg: var(--fg-weak);
+    --a-dark: var(--a);
+    --a-gray: var(--fg-weak);
+
+	--btn-fg: var(--a);
+	--btn-bg: var(--w2);
+    --btn-h-fg: var(--w2);
+	--btn-1-fg: var(--bg);
+	--btn-1-bg: var(--a);
+    --txt-sh: a;
+    --txt-bg: var(--w2);
+
+	--u2-b1-bg: var(--w2);
+	--u2-b2-bg: var(--w2);
+	--u2-o-bg: var(--w2);
+	--u2-o-1-bg: var(--a);
+    --u2-txt-bg: var(--w2);
+	--u2-tab-bg: a;
+	--u2-tab-1-bg: var(--w2);
+
+    --sort-1: var(--a);
+    --sort-1: var(--fg-weak);
+
+    --tree-bg: var(--bg);
+
+	--g-b1: a;
+	--g-b2: a;
+    --g-f-bg: var(--w2);
+
+    --f-sh1: 0.1;
+    --f-sh2: 0.02;
+    --f-sh3: 0.1;
+    --f-h-b1: a;
+
+    --srv-1: var(--a);
+    --srv-3: var(--a);
+
+    --mp-sh: a;
+}
+
+html.ey {
+    background: url('bsod.png') top 5em right 4.5em no-repeat fixed var(--bg);
+}
+html.ey body#b {
+    background: var(--bg);  /*sandbox*/
+}
+html.ey #ops {
+    margin: 1.7em 1.5em 0 1.5em;
+	border-radius: .3em;
+	border-width: 1px 0;
+}
+html.ey #ops a {
+    text-shadow: 1px 1px 0 rgba(0,0,0,0.5);
+}
+html.ey .opbox {
+	margin: 1.5em 0 0 0;
+}
+html.ey #tree {
+    box-shadow: none;
+}
+html.ey #tt {
+    border-color: var(--w2);
+    background: var(--w2);
+}
+html.ey .mdo a {
+    background: none;
+    text-decoration: underline;
+}
+html.ey .mdo pre,
+html.ey .mdo code {
+    color: #fff;
+    background: var(--w2);
+    border: none;
+}
+html.ey .mdo h1,
+html.ey .mdo h2 {
+    background: none;
+    border-color: var(--w2);
+}
+html.ey .mdo ul ul,
+html.ey .mdo ul ol,
+html.ey .mdo ol ul,
+html.ey .mdo ol ol {
+	border-color: var(--w2);
+}
+html.ey .mdo p>em,
+html.ey .mdo li>em,
+html.ey .mdo td>em {
+	color: #fd0;
+}

copyparty/__main__.py

@@ -67,7 +67,13 @@ if True:  # pylint: disable=using-constant-test
 
     from typing import Any, Optional
 
+if PY2:
+    range = xrange  # type: ignore
+
 try:
+    if os.environ.get("PRTY_NO_TLS"):
+        raise Exception()
+
     HAVE_SSL = True
     import ssl
 except:
@@ -344,7 +350,7 @@ def configure_ssl_ver(al: argparse.Namespace) -> None:
     # oh man i love openssl
     # check this out
     # hold my beer
-    assert ssl
+    assert ssl  # type: ignore
     ptn = re.compile(r"^OP_NO_(TLS|SSL)v")
     sslver = terse_sslver(al.ssl_ver).split(",")
     flags = [k for k in ssl.__dict__ if ptn.match(k)]
@@ -378,7 +384,7 @@ def configure_ssl_ver(al: argparse.Namespace) -> None:
 
 
 def configure_ssl_ciphers(al: argparse.Namespace) -> None:
-    assert ssl
+    assert ssl  # type: ignore
     ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
     if al.ssl_ver:
         ctx.options &= ~al.ssl_flags_en
@@ -491,11 +497,20 @@ def disable_quickedit() -> None:
 
 
 def sfx_tpoke(top: str):
-    files = [os.path.join(dp, p) for dp, dd, df in os.walk(top) for p in dd + df]
+    if os.environ.get("PRTY_NO_TPOKE"):
+        return
+
+    files = [top] + [
+        os.path.join(dp, p) for dp, dd, df in os.walk(top) for p in dd + df
+    ]
     while True:
         t = int(time.time())
-        for f in [top] + files:
-            os.utime(f, (t, t))
+        for f in list(files):
+            try:
+                os.utime(f, (t, t))
+            except Exception as ex:
+                lprint("<TPOKE> [%s] %r" % (f, ex))
+                files.remove(f)
 
         time.sleep(78123)
 
@@ -689,6 +704,11 @@ def get_sects():
             \033[36mxban\033[0m can be used to overrule / cancel a user ban event;
             if the program returns 0 (true/OK) then the ban will NOT happen
 
+            effects can be used to redirect uploads into other
+            locations, and to delete or index other files based
+            on new uploads, but with certain limitations. See
+            bin/hooks/reloc* and docs/devnotes.md#hook-effects
+
             except for \033[36mxm\033[0m, only one hook / one action can run at a time,
             so it's recommended to use the \033[36mf\033[0m flag unless you really need
             to wait for the hook to finish before continuing (without \033[36mf\033[0m
@@ -942,14 +962,15 @@ def add_upload(ap):
     ap2.add_argument("--sparse", metavar="MiB", type=int, default=4, help="windows-only: minimum size of incoming uploads through up2k before they are made into sparse files")
     ap2.add_argument("--turbo", metavar="LVL", type=int, default=0, help="configure turbo-mode in up2k client; [\033[32m-1\033[0m] = forbidden/always-off, [\033[32m0\033[0m] = default-off and warn if enabled, [\033[32m1\033[0m] = default-off, [\033[32m2\033[0m] = on, [\033[32m3\033[0m] = on and disable datecheck")
     ap2.add_argument("--u2j", metavar="JOBS", type=int, default=2, help="web-client: number of file chunks to upload in parallel; 1 or 2 is good for low-latency (same-country) connections, 4-8 for android clients, 16 for cross-atlantic (max=64)")
+    ap2.add_argument("--u2sz", metavar="N,N,N", type=u, default="1,64,96", help="web-client: default upload chunksize (MiB); sets \033[33mmin,default,max\033[0m in the settings gui. Each HTTP POST will aim for this size. Cloudflare max is 96. Big values are good for cross-atlantic but may increase HDD fragmentation on some FS. Disable this optimization with [\033[32m1,1,1\033[0m]")
     ap2.add_argument("--u2sort", metavar="TXT", type=u, default="s", help="upload order; [\033[32ms\033[0m]=smallest-first, [\033[32mn\033[0m]=alphabetical, [\033[32mfs\033[0m]=force-s, [\033[32mfn\033[0m]=force-n -- alphabetical is a bit slower on fiber/LAN but makes it easier to eyeball if everything went fine")
     ap2.add_argument("--write-uplog", action="store_true", help="write POST reports to textfiles in working-directory")
 
 
 def add_network(ap):
     ap2 = ap.add_argument_group('network options')
-    ap2.add_argument("-i", metavar="IP", type=u, default="::", help="ip to bind (comma-sep.), default: all IPv4 and IPv6")
-    ap2.add_argument("-p", metavar="PORT", type=u, default="3923", help="ports to bind (comma/range)")
+    ap2.add_argument("-i", metavar="IP", type=u, default="::", help="ip to bind (comma-sep.) and/or [\033[32munix:/tmp/a.sock\033[0m], default: all IPv4 and IPv6")
+    ap2.add_argument("-p", metavar="PORT", type=u, default="3923", help="ports to bind (comma/range); ignored for unix-sockets")
     ap2.add_argument("--ll", action="store_true", help="include link-local IPv4/IPv6 in mDNS replies, even if the NIC has routable IPs (breaks some mDNS clients)")
     ap2.add_argument("--rproxy", metavar="DEPTH", type=int, default=1, help="which ip to associate clients with; [\033[32m0\033[0m]=tcp, [\033[32m1\033[0m]=origin (first x-fwd, unsafe), [\033[32m2\033[0m]=outermost-proxy, [\033[32m3\033[0m]=second-proxy, [\033[32m-1\033[0m]=closest-proxy")
     ap2.add_argument("--xff-hdr", metavar="NAME", type=u, default="x-forwarded-for", help="if reverse-proxied, which http header to read the client's real ip from")
@@ -1116,6 +1137,7 @@ def add_hooks(ap):
     ap2.add_argument("--xad", metavar="CMD", type=u, action="append", help="execute \033[33mCMD\033[0m after  a file delete")
     ap2.add_argument("--xm", metavar="CMD", type=u, action="append", help="execute \033[33mCMD\033[0m on message")
     ap2.add_argument("--xban", metavar="CMD", type=u, action="append", help="execute \033[33mCMD\033[0m if someone gets banned (pw/404/403/url)")
+    ap2.add_argument("--hook-v", action="store_true", help="verbose hooks")
 
 
 def add_stats(ap):
@@ -1344,9 +1366,10 @@ def add_ui(ap, retry):
     ap2.add_argument("--unlist", metavar="REGEX", type=u, default="", help="don't show files matching \033[33mREGEX\033[0m in file list. Purely cosmetic! Does not affect API calls, just the browser. Example: [\033[32m\\.(js|css)$\033[0m] (volflag=unlist)")
     ap2.add_argument("--favico", metavar="TXT", type=u, default="c 000 none" if retry else "🎉 000 none", help="\033[33mfavicon-text\033[0m [ \033[33mforeground\033[0m [ \033[33mbackground\033[0m ] ], set blank to disable")
     ap2.add_argument("--mpmc", metavar="URL", type=u, default="", help="change the mediaplayer-toggle mouse cursor; URL to a folder with {2..5}.png inside (or disable with [\033[32m.\033[0m])")
-    ap2.add_argument("--js-browser", metavar="L", type=u, default="", help="URL to additional JS to include")
-    ap2.add_argument("--css-browser", metavar="L", type=u, default="", help="URL to additional CSS to include")
-    ap2.add_argument("--html-head", metavar="TXT", type=u, default="", help="text to append to the <head> of all HTML pages; can be @PATH to send the contents of a file at PATH, and/or begin with %% to render as jinja2 template (volflag=html_head)")
+    ap2.add_argument("--css-browser", metavar="L", type=u, default="", help="URL to additional CSS to include in the filebrowser html")
+    ap2.add_argument("--js-browser", metavar="L", type=u, default="", help="URL to additional JS to include in the filebrowser html")
+    ap2.add_argument("--js-other", metavar="L", type=u, default="", help="URL to additional JS to include in all other pages")
+    ap2.add_argument("--html-head", metavar="TXT", type=u, default="", help="text to append to the <head> of all HTML pages (except for basic-browser); can be @PATH to send the contents of a file at PATH, and/or begin with %% to render as jinja2 template (volflag=html_head)")
     ap2.add_argument("--ih", action="store_true", help="if a folder contains index.html, show that instead of the directory listing by default (can be changed in the client settings UI, or add ?v to URL for override)")
     ap2.add_argument("--textfiles", metavar="CSV", type=u, default="txt,nfo,diz,cue,readme", help="file extensions to present as plaintext")
     ap2.add_argument("--txt-max", metavar="KiB", type=int, default=64, help="max size of embedded textfiles on ?doc= (anything bigger will be lazy-loaded by JS)")
@@ -1365,12 +1388,14 @@ def add_debug(ap):
     ap2 = ap.add_argument_group('debug options')
     ap2.add_argument("--vc", action="store_true", help="verbose config file parser (explain config)")
     ap2.add_argument("--cgen", action="store_true", help="generate config file from current config (best-effort; probably buggy)")
+    ap2.add_argument("--deps", action="store_true", help="list information about detected optional dependencies")
     if hasattr(select, "poll"):
         ap2.add_argument("--no-poll", action="store_true", help="kernel-bug workaround: disable poll; use select instead (limits max num clients to ~700)")
     ap2.add_argument("--no-sendfile", action="store_true", help="kernel-bug workaround: disable sendfile; do a safe and slow read-send-loop instead")
     ap2.add_argument("--no-scandir", action="store_true", help="kernel-bug workaround: disable scandir; do a listdir + stat on each file instead")
     ap2.add_argument("--no-fastboot", action="store_true", help="wait for initial filesystem indexing before accepting client requests")
     ap2.add_argument("--no-htp", action="store_true", help="disable httpserver threadpool, create threads as-needed instead")
+    ap2.add_argument("--rm-sck", action="store_true", help="when listening on unix-sockets, do a basic delete+bind instead of the default atomic bind")
     ap2.add_argument("--srch-dbg", action="store_true", help="explain search processing, and do some extra expensive sanity checks")
     ap2.add_argument("--rclone-mdns", action="store_true", help="use mdns-domain instead of server-ip on /?hc")
     ap2.add_argument("--stackmon", metavar="P,S", type=u, default="", help="write stacktrace to \033[33mP\033[0math every \033[33mS\033[0m second, for example --stackmon=\033[32m./st/%%Y-%%m/%%d/%%H%%M.xz,60")

copyparty/__version__.py

@@ -1,8 +1,8 @@
 # coding: utf-8
 
-VERSION = (1, 13, 4)
+VERSION = (1, 13, 8)
 CODENAME = "race the beam"
-BUILD_DT = (2024, 7, 16)
+BUILD_DT = (2024, 8, 13)
 
 S_VERSION = ".".join(map(str, VERSION))
 S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT)

copyparty/authsrv.py

@@ -12,7 +12,7 @@ import threading
 import time
 from datetime import datetime
 
-from .__init__ import ANYWIN, TYPE_CHECKING, WINDOWS, E
+from .__init__ import ANYWIN, PY2, TYPE_CHECKING, WINDOWS, E
 from .bos import bos
 from .cfg import flagdescs, permdescs, vf_bmap, vf_cmap, vf_vmap
 from .pwhash import PWHash
@@ -56,6 +56,9 @@ if TYPE_CHECKING:
     # Vflags: TypeAlias = dict[str, Any]
     # Mflags: TypeAlias = dict[str, Vflags]
 
+if PY2:
+    range = xrange  # type: ignore
+
 
 LEELOO_DALLAS = "leeloo_dallas"
 
@@ -441,7 +444,7 @@ class VFS(object):
 
     def _find(self, vpath: str) -> tuple["VFS", str]:
         """return [vfs,remainder]"""
-        if vpath == "":
+        if not vpath:
             return self, ""
 
         if "/" in vpath:
@@ -451,7 +454,7 @@ class VFS(object):
             rem = ""
 
         if name in self.nodes:
-            return self.nodes[name]._find(undot(rem))
+            return self.nodes[name]._find(rem)
 
         return self, vpath
 
@@ -518,8 +521,8 @@ class VFS(object):
                     t = "{} has no {} in [{}] => [{}] => [{}]"
                     self.log("vfs", t.format(uname, msg, vpath, cvpath, ap), 6)
 
-                t = 'you don\'t have %s-access in "/%s"'
-                raise Pebkac(err, t % (msg, cvpath))
+                t = 'you don\'t have %s-access in "/%s" or below "/%s"'
+                raise Pebkac(err, t % (msg, cvpath, vn.vpath))
 
         return vn, rem
 
@@ -1895,7 +1898,7 @@ class AuthSrv(object):
                     self.log(t.format(vol.vpath), 1)
                     del vol.flags["lifetime"]
 
-                needs_e2d = [x for x in hooks if x != "xm"]
+                needs_e2d = [x for x in hooks if x in ("xau", "xiu")]
                 drop = [x for x in needs_e2d if vol.flags.get(x)]
                 if drop:
                     t = 'removing [{}] from volume "/{}" because e2d is disabled'

copyparty/broker_util.py

@@ -28,7 +28,7 @@ class ExceptionalQueue(Queue, object):
                 if rv[1] == "pebkac":
                     raise Pebkac(*rv[2:])
                 else:
-                    raise Exception(rv[2])
+                    raise rv[2]
 
         return rv
 
@@ -65,8 +65,8 @@ def try_exec(want_retval: Union[bool, int], func: Any, *args: list[Any]) -> Any:
 
         return ["exception", "pebkac", ex.code, str(ex)]
 
-    except:
+    except Exception as ex:
         if not want_retval:
             raise
 
-        return ["exception", "stack", traceback.format_exc()]
+        return ["exception", "stack", ex]

copyparty/cert.py

@@ -9,7 +9,7 @@ import time
 from .__init__ import ANYWIN
 from .util import Netdev, runcmd, wrename, wunlink
 
-HAVE_CFSSL = True
+HAVE_CFSSL = not os.environ.get("PRTY_NO_CFSSL")
 
 if True:  # pylint: disable=using-constant-test
     from .util import NamedLogger, RootLogger

copyparty/fsutil.py

@@ -9,12 +9,12 @@ import time
 from .__init__ import ANYWIN, MACOS
 from .authsrv import AXS, VFS
 from .bos import bos
-from .util import chkcmd, min_ex
+from .util import chkcmd, min_ex, undot
 
 if True:  # pylint: disable=using-constant-test
     from typing import Optional, Union
 
-    from .util import RootLogger
+    from .util import RootLogger, undot
 
 
 class Fstab(object):
@@ -52,7 +52,7 @@ class Fstab(object):
                 self.log(msg.format(path, fs, min_ex()), 3)
                 return fs
 
-        path = path.lstrip("/")
+        path = undot(path)
         try:
             return self.cache[path]
         except:
@@ -124,7 +124,7 @@ class Fstab(object):
         if ANYWIN:
             path = self._winpath(path)
 
-        path = path.lstrip("/")
+        path = undot(path)
         ptn = re.compile(r"^[^\\/]*")
         vn, rem = self.tab._find(path)
         if not self.trusted:

copyparty/ftpd.py

@@ -41,6 +41,9 @@ if True:  # pylint: disable=using-constant-test
     import typing
     from typing import Any, Optional, Union
 
+if PY2:
+    range = xrange  # type: ignore
+
 
 class FSE(FilesystemError):
     def __init__(self, msg: str, severity: int = 0) -> None:
@@ -350,7 +353,7 @@ class FtpFs(AbstractedFS):
         svp = join(self.cwd, src).lstrip("/")
         dvp = join(self.cwd, dst).lstrip("/")
         try:
-            self.hub.up2k.handle_mv(self.uname, svp, dvp)
+            self.hub.up2k.handle_mv(self.uname, self.h.cli_ip, svp, dvp)
         except Exception as ex:
             raise FSE(str(ex))
 
@@ -468,6 +471,9 @@ class FtpHandler(FTPHandler):
         xbu = vfs.flags.get("xbu")
         if xbu and not runhook(
             None,
+            None,
+            self.hub.up2k,
+            "xbu.ftpd",
             xbu,
             ap,
             vp,
@@ -477,7 +483,7 @@ class FtpHandler(FTPHandler):
             0,
             0,
             self.cli_ip,
-            0,
+            time.time(),
             "",
         ):
             raise FSE("Upload blocked by xbu server config")
@@ -580,9 +586,15 @@ class Ftpd(object):
         if "::" in ips:
             ips.append("0.0.0.0")
 
+        ips = [x for x in ips if "unix:" not in x]
+
         if self.args.ftp4:
             ips = [x for x in ips if ":" not in x]
 
+        if not ips:
+            lgr.fatal("cannot start ftp-server; no compatible IPs in -i")
+            return
+
         ips = list(ODict.fromkeys(ips))  # dedup
 
         ioloop = IOLoop()

copyparty/httpcli.py

@@ -13,18 +13,22 @@ import json
 import os
 import random
 import re
+import socket
 import stat
 import string
 import threading  # typechk
 import time
 import uuid
 from datetime import datetime
-from email.utils import formatdate, parsedate
+from email.utils import parsedate
 from operator import itemgetter
 
 import jinja2  # typechk
 
 try:
+    if os.environ.get("PRTY_NO_LZMA"):
+        raise Exception()
+
     import lzma
 except:
     pass
@@ -54,6 +58,7 @@ from .util import (
     alltrace,
     atomic_move,
     exclude_dotfiles,
+    formatdate,
     fsenc,
     gen_filekey,
     gen_filekey_dbg,
@@ -69,7 +74,9 @@ from .util import (
     humansize,
     ipnorm,
     loadpy,
+    log_reloc,
     min_ex,
+    pathmod,
     quotep,
     rand_name,
     read_header,
@@ -105,6 +112,9 @@ if True:  # pylint: disable=using-constant-test
 if TYPE_CHECKING:
     from .httpconn import HttpConn
 
+if not hasattr(socket, "AF_UNIX"):
+    setattr(socket, "AF_UNIX", -9001)
+
 _ = (argparse, threading)
 
 NO_CACHE = {"Cache-Control": "no-cache"}
@@ -222,6 +232,11 @@ class HttpCli(object):
         ka["s_doctitle"] = self.args.doctitle
         ka["tcolor"] = self.vn.flags["tcolor"]
 
+        if self.args.js_other and "js" not in ka:
+            zs = self.args.js_other
+            zs += "&" if "?" in zs else "?"
+            ka["js"] = zs
+
         zso = self.vn.flags.get("html_head")
         if zso:
             ka["this"] = self
@@ -303,8 +318,11 @@ class HttpCli(object):
         )
         self.host = self.headers.get("host") or ""
         if not self.host:
-            zs = "%s:%s" % self.s.getsockname()[:2]
-            self.host = zs[7:] if zs.startswith("::ffff:") else zs
+            if self.s.family == socket.AF_UNIX:
+                self.host = self.args.name
+            else:
+                zs = "%s:%s" % self.s.getsockname()[:2]
+                self.host = zs[7:] if zs.startswith("::ffff:") else zs
 
         trusted_xff = False
         n = self.args.rproxy
@@ -646,11 +664,8 @@ class HttpCli(object):
                 if not self._check_nonfatal(pex, post):
                     self.keepalive = False
 
-                if pex is ex:
-                    em = msg = str(ex)
-                else:
-                    em = repr(ex)
-                    msg = min_ex()
+                em = str(ex)
+                msg = em if pex is ex else min_ex()
 
                 if pex.code != 404 or self.do_log:
                     self.log(
@@ -694,6 +709,9 @@ class HttpCli(object):
         xban = self.vn.flags.get("xban")
         if not xban or not runhook(
             self.log,
+            self.conn.hsrv.broker,
+            None,
+            "xban",
             xban,
             self.vn.canonical(self.rem),
             self.vpath,
@@ -790,7 +808,7 @@ class HttpCli(object):
 
         # close if unknown length, otherwise take client's preference
         response.append("Connection: " + ("Keep-Alive" if self.keepalive else "Close"))
-        response.append("Date: " + formatdate(usegmt=True))
+        response.append("Date: " + formatdate())
 
         # headers{} overrides anything set previously
         if headers:
@@ -814,9 +832,9 @@ class HttpCli(object):
                 self.cbonk(self.conn.hsrv.gmal, zs, "cc_hdr", "Cc in out-hdr")
                 raise Pebkac(999)
 
+        response.append("\r\n")
         try:
-            # best practice to separate headers and body into different packets
-            self.s.sendall("\r\n".join(response).encode("utf-8") + b"\r\n\r\n")
+            self.s.sendall("\r\n".join(response).encode("utf-8"))
         except:
             raise Pebkac(400, "client d/c while replying headers")
 
@@ -1149,7 +1167,7 @@ class HttpCli(object):
             return self.tx_mounts()
 
         # conditional redirect to single volumes
-        if self.vpath == "" and not self.ouparam:
+        if not self.vpath and not self.ouparam:
             nread = len(self.rvol)
             nwrite = len(self.wvol)
             if nread + nwrite == 1 or (self.rvol == self.wvol and nread == 1):
@@ -1171,7 +1189,8 @@ class HttpCli(object):
         if self.args.no_dav:
             raise Pebkac(405, "WebDAV is disabled in server config")
 
-        vn, rem = self.asrv.vfs.get(self.vpath, self.uname, False, False, err=401)
+        vn = self.vn
+        rem = self.rem
         tap = vn.canonical(rem)
 
         if "davauth" in vn.flags and self.uname == "*":
@@ -1308,7 +1327,7 @@ class HttpCli(object):
 
             pvs: dict[str, str] = {
                 "displayname": html_escape(rp.split("/")[-1]),
-                "getlastmodified": formatdate(mtime, usegmt=True),
+                "getlastmodified": formatdate(mtime),
                 "resourcetype": '<D:collection xmlns:D="DAV:"/>' if isdir else "",
                 "supportedlock": '<D:lockentry xmlns:D="DAV:"><D:lockscope><D:exclusive/></D:lockscope><D:locktype><D:write/></D:locktype></D:lockentry>',
             }
@@ -1555,8 +1574,8 @@ class HttpCli(object):
         self.log("PUT %s @%s" % (self.req, self.uname))
 
         if not self.can_write:
-            t = "user {} does not have write-access here"
-            raise Pebkac(403, t.format(self.uname))
+            t = "user %s does not have write-access under /%s"
+            raise Pebkac(403, t % (self.uname, self.vn.vpath))
 
         if not self.args.no_dav and self._applesan():
             return self.headers.get("content-length") == "0"
@@ -1631,6 +1650,9 @@ class HttpCli(object):
                             if xm:
                                 runhook(
                                     self.log,
+                                    self.conn.hsrv.broker,
+                                    None,
+                                    "xm",
                                     xm,
                                     self.vn.canonical(self.rem),
                                     self.vpath,
@@ -1779,11 +1801,15 @@ class HttpCli(object):
 
         if xbu:
             at = time.time() - lifetime
-            if not runhook(
+            vp = vjoin(self.vpath, fn) if nameless else self.vpath
+            hr = runhook(
                 self.log,
+                self.conn.hsrv.broker,
+                None,
+                "xbu.http.dump",
                 xbu,
                 path,
-                self.vpath,
+                vp,
                 self.host,
                 self.uname,
                 self.asrv.vfs.get_perms(self.vpath, self.uname),
@@ -1792,10 +1818,25 @@ class HttpCli(object):
                 self.ip,
                 at,
                 "",
-            ):
+            )
+            if not hr:
                 t = "upload blocked by xbu server config"
                 self.log(t, 1)
                 raise Pebkac(403, t)
+            if hr.get("reloc"):
+                x = pathmod(self.asrv.vfs, path, vp, hr["reloc"])
+                if x:
+                    if self.args.hook_v:
+                        log_reloc(self.log, hr["reloc"], x, path, vp, fn, vfs, rem)
+                    fdir, self.vpath, fn, (vfs, rem) = x
+                    if self.args.nw:
+                        fn = os.devnull
+                    else:
+                        bos.makedirs(fdir)
+                        path = os.path.join(fdir, fn)
+                        if not nameless:
+                            self.vpath = vjoin(self.vpath, fn)
+                        params["fdir"] = fdir
 
         if is_put and not (self.args.no_dav or self.args.nw) and bos.path.exists(path):
             # allow overwrite if...
@@ -1870,24 +1911,45 @@ class HttpCli(object):
                 fn = fn2
                 path = path2
 
-        if xau and not runhook(
-            self.log,
-            xau,
-            path,
-            self.vpath,
-            self.host,
-            self.uname,
-            self.asrv.vfs.get_perms(self.vpath, self.uname),
-            mt,
-            post_sz,
-            self.ip,
-            at,
-            "",
-        ):
-            t = "upload blocked by xau server config"
-            self.log(t, 1)
-            wunlink(self.log, path, vfs.flags)
-            raise Pebkac(403, t)
+        if xau:
+            vp = vjoin(self.vpath, fn) if nameless else self.vpath
+            hr = runhook(
+                self.log,
+                self.conn.hsrv.broker,
+                None,
+                "xau.http.dump",
+                xau,
+                path,
+                vp,
+                self.host,
+                self.uname,
+                self.asrv.vfs.get_perms(self.vpath, self.uname),
+                mt,
+                post_sz,
+                self.ip,
+                at,
+                "",
+            )
+            if not hr:
+                t = "upload blocked by xau server config"
+                self.log(t, 1)
+                wunlink(self.log, path, vfs.flags)
+                raise Pebkac(403, t)
+            if hr.get("reloc"):
+                x = pathmod(self.asrv.vfs, path, vp, hr["reloc"])
+                if x:
+                    if self.args.hook_v:
+                        log_reloc(self.log, hr["reloc"], x, path, vp, fn, vfs, rem)
+                    fdir, self.vpath, fn, (vfs, rem) = x
+                    bos.makedirs(fdir)
+                    path2 = os.path.join(fdir, fn)
+                    atomic_move(self.log, path, path2, vfs.flags)
+                    path = path2
+                    if not nameless:
+                        self.vpath = vjoin(self.vpath, fn)
+            sz = bos.path.getsize(path)
+        else:
+            sz = post_sz
 
         vfs, rem = vfs.get_dbv(rem)
         self.conn.hsrv.broker.say(
@@ -1910,7 +1972,7 @@ class HttpCli(object):
                 alg,
                 self.args.fk_salt,
                 path,
-                post_sz,
+                sz,
                 0 if ANYWIN else bos.stat(path).st_ino,
             )[: vfs.flags["fk"]]
 
@@ -2202,33 +2264,47 @@ class HttpCli(object):
 
     def handle_post_binary(self) -> bool:
         try:
-            remains = int(self.headers["content-length"])
+            postsize = remains = int(self.headers["content-length"])
         except:
             raise Pebkac(400, "you must supply a content-length for binary POST")
 
         try:
-            chash = self.headers["x-up2k-hash"]
+            chashes = self.headers["x-up2k-hash"].split(",")
             wark = self.headers["x-up2k-wark"]
         except KeyError:
             raise Pebkac(400, "need hash and wark headers for binary POST")
 
+        chashes = [x.strip() for x in chashes]
+        if len(chashes) == 3 and len(chashes[1]) == 1:
+            # the first hash, then length of consecutive hashes,
+            # then a list of stitched hashes as one long string
+            clen = int(chashes[1])
+            siblings = chashes[2]
+            chashes = [chashes[0]]
+            for n in range(0, len(siblings), clen):
+                chashes.append(siblings[n : n + clen])
+
         vfs, _ = self.asrv.vfs.get(self.vpath, self.uname, False, True)
         ptop = (vfs.dbv or vfs).realpath
 
-        x = self.conn.hsrv.broker.ask("up2k.handle_chunk", ptop, wark, chash)
+        x = self.conn.hsrv.broker.ask("up2k.handle_chunks", ptop, wark, chashes)
         response = x.get()
-        chunksize, cstart, path, lastmod, sprs = response
+        chashes, chunksize, cstarts, path, lastmod, sprs = response
+        maxsize = chunksize * len(chashes)
+        cstart0 = cstarts[0]
 
         try:
             if self.args.nw:
                 path = os.devnull
 
-            if remains > chunksize:
-                raise Pebkac(400, "your chunk is too big to fit")
+            if remains > maxsize:
+                t = "your client is sending %d bytes which is too much (server expected %d bytes at most)"
+                raise Pebkac(400, t % (remains, maxsize))
 
-            self.log("writing {} #{} @{} len {}".format(path, chash, cstart, remains))
-
-            reader = read_socket(self.sr, self.args.s_rd_sz, remains)
+            t = "writing %s %s+%d #%d+%d %s"
+            chunkno = cstart0[0] // chunksize
+            zs = " ".join([chashes[0][:15]] + [x[:9] for x in chashes[1:]])
+            self.log(t % (path, cstart0, remains, chunkno, len(chashes), zs))
 
             f = None
             fpool = not self.args.no_fpool and sprs
@@ -2242,37 +2318,43 @@ class HttpCli(object):
             f = f or open(fsenc(path), "rb+", self.args.iobuf)
 
             try:
-                f.seek(cstart[0])
-                post_sz, _, sha_b64 = hashcopy(reader, f, self.args.s_wr_slp)
-
-                if sha_b64 != chash:
-                    try:
-                        self.bakflip(f, cstart[0], post_sz, sha_b64, vfs.flags)
-                    except:
-                        self.log("bakflip failed: " + min_ex())
-
-                    t = "your chunk got corrupted somehow (received {} bytes); expected vs received hash:\n{}\n{}"
-                    raise Pebkac(400, t.format(post_sz, chash, sha_b64))
-
-                if len(cstart) > 1 and path != os.devnull:
-                    self.log(
-                        "clone {} to {}".format(
-                            cstart[0], " & ".join(unicode(x) for x in cstart[1:])
-                        )
+                for chash, cstart in zip(chashes, cstarts):
+                    f.seek(cstart[0])
+                    reader = read_socket(
+                        self.sr, self.args.s_rd_sz, min(remains, chunksize)
                     )
-                    ofs = 0
-                    while ofs < chunksize:
-                        bufsz = max(4 * 1024 * 1024, self.args.iobuf)
-                        bufsz = min(chunksize - ofs, bufsz)
-                        f.seek(cstart[0] + ofs)
-                        buf = f.read(bufsz)
-                        for wofs in cstart[1:]:
-                            f.seek(wofs + ofs)
-                            f.write(buf)
+                    post_sz, _, sha_b64 = hashcopy(reader, f, self.args.s_wr_slp)
 
-                        ofs += len(buf)
+                    if sha_b64 != chash:
+                        try:
+                            self.bakflip(f, cstart[0], post_sz, sha_b64, vfs.flags)
+                        except:
+                            self.log("bakflip failed: " + min_ex())
 
-                    self.log("clone {} done".format(cstart[0]))
+                        t = "your chunk got corrupted somehow (received {} bytes); expected vs received hash:\n{}\n{}"
+                        raise Pebkac(400, t.format(post_sz, chash, sha_b64))
+
+                    remains -= chunksize
+
+                    if len(cstart) > 1 and path != os.devnull:
+                        self.log(
+                            "clone {} to {}".format(
+                                cstart[0], " & ".join(unicode(x) for x in cstart[1:])
+                            )
+                        )
+                        ofs = 0
+                        while ofs < chunksize:
+                            bufsz = max(4 * 1024 * 1024, self.args.iobuf)
+                            bufsz = min(chunksize - ofs, bufsz)
+                            f.seek(cstart[0] + ofs)
+                            buf = f.read(bufsz)
+                            for wofs in cstart[1:]:
+                                f.seek(wofs + ofs)
+                                f.write(buf)
+
+                            ofs += len(buf)
+
+                        self.log("clone {} done".format(cstart[0]))
 
                 if not fpool:
                     f.close()
@@ -2284,10 +2366,10 @@ class HttpCli(object):
                 f.close()
                 raise
         finally:
-            x = self.conn.hsrv.broker.ask("up2k.release_chunk", ptop, wark, chash)
+            x = self.conn.hsrv.broker.ask("up2k.release_chunks", ptop, wark, chashes)
             x.get()  # block client until released
 
-        x = self.conn.hsrv.broker.ask("up2k.confirm_chunk", ptop, wark, chash)
+        x = self.conn.hsrv.broker.ask("up2k.confirm_chunks", ptop, wark, chashes)
         ztis = x.get()
         try:
             num_left, fin_path = ztis
@@ -2306,7 +2388,7 @@ class HttpCli(object):
 
         cinf = self.headers.get("x-up2k-stat", "")
 
-        spd = self._spd(post_sz)
+        spd = self._spd(postsize)
         self.log("{:70} thank {}".format(spd, cinf))
         self.reply(b"thank")
         return True
@@ -2515,18 +2597,15 @@ class HttpCli(object):
                 fname = sanitize_fn(
                     p_file or "", "", [".prologue.html", ".epilogue.html"]
                 )
+                abspath = os.path.join(fdir, fname)
+                suffix = "-%.6f-%s" % (time.time(), dip)
                 if p_file and not nullwrite:
                     if rnd:
                         fname = rand_name(fdir, fname, rnd)
 
-                    if not bos.path.isdir(fdir):
-                        raise Pebkac(404, "that folder does not exist")
-
-                    suffix = "-{:.6f}-{}".format(time.time(), dip)
                     open_args = {"fdir": fdir, "suffix": suffix}
 
                     if "replace" in self.uparam:
-                        abspath = os.path.join(fdir, fname)
                         if not self.can_delete:
                             self.log("user not allowed to overwrite with ?replace")
                         elif bos.path.exists(abspath):
@@ -2536,6 +2615,58 @@ class HttpCli(object):
                             except:
                                 t = "toctou while deleting for ?replace: %s"
                             self.log(t % (abspath,))
+                else:
+                    open_args = {}
+                    tnam = fname = os.devnull
+                    fdir = abspath = ""
+
+                if xbu:
+                    at = time.time() - lifetime
+                    hr = runhook(
+                        self.log,
+                        self.conn.hsrv.broker,
+                        None,
+                        "xbu.http.bup",
+                        xbu,
+                        abspath,
+                        vjoin(upload_vpath, fname),
+                        self.host,
+                        self.uname,
+                        self.asrv.vfs.get_perms(upload_vpath, self.uname),
+                        at,
+                        0,
+                        self.ip,
+                        at,
+                        "",
+                    )
+                    if not hr:
+                        t = "upload blocked by xbu server config"
+                        self.log(t, 1)
+                        raise Pebkac(403, t)
+                    if hr.get("reloc"):
+                        zs = vjoin(upload_vpath, fname)
+                        x = pathmod(self.asrv.vfs, abspath, zs, hr["reloc"])
+                        if x:
+                            if self.args.hook_v:
+                                log_reloc(
+                                    self.log,
+                                    hr["reloc"],
+                                    x,
+                                    abspath,
+                                    zs,
+                                    fname,
+                                    vfs,
+                                    rem,
+                                )
+                            fdir, upload_vpath, fname, (vfs, rem) = x
+                            abspath = os.path.join(fdir, fname)
+                            if nullwrite:
+                                fdir = abspath = ""
+                            else:
+                                open_args["fdir"] = fdir
+
+                if p_file and not nullwrite:
+                    bos.makedirs(fdir)
 
                     # reserve destination filename
                     with ren_open(fname, "wb", fdir=fdir, suffix=suffix) as zfw:
@@ -2551,26 +2682,6 @@ class HttpCli(object):
                     tnam = fname = os.devnull
                     fdir = abspath = ""
 
-                if xbu:
-                    at = time.time() - lifetime
-                    if not runhook(
-                        self.log,
-                        xbu,
-                        abspath,
-                        self.vpath,
-                        self.host,
-                        self.uname,
-                        self.asrv.vfs.get_perms(self.vpath, self.uname),
-                        at,
-                        0,
-                        self.ip,
-                        at,
-                        "",
-                    ):
-                        t = "upload blocked by xbu server config"
-                        self.log(t, 1)
-                        raise Pebkac(403, t)
-
                 if lim:
                     lim.chk_bup(self.ip)
                     lim.chk_nup(self.ip)
@@ -2613,29 +2724,58 @@ class HttpCli(object):
 
                     tabspath = ""
 
+                    at = time.time() - lifetime
+                    if xau:
+                        hr = runhook(
+                            self.log,
+                            self.conn.hsrv.broker,
+                            None,
+                            "xau.http.bup",
+                            xau,
+                            abspath,
+                            vjoin(upload_vpath, fname),
+                            self.host,
+                            self.uname,
+                            self.asrv.vfs.get_perms(upload_vpath, self.uname),
+                            at,
+                            sz,
+                            self.ip,
+                            at,
+                            "",
+                        )
+                        if not hr:
+                            t = "upload blocked by xau server config"
+                            self.log(t, 1)
+                            wunlink(self.log, abspath, vfs.flags)
+                            raise Pebkac(403, t)
+                        if hr.get("reloc"):
+                            zs = vjoin(upload_vpath, fname)
+                            x = pathmod(self.asrv.vfs, abspath, zs, hr["reloc"])
+                            if x:
+                                if self.args.hook_v:
+                                    log_reloc(
+                                        self.log,
+                                        hr["reloc"],
+                                        x,
+                                        abspath,
+                                        zs,
+                                        fname,
+                                        vfs,
+                                        rem,
+                                    )
+                                fdir, upload_vpath, fname, (vfs, rem) = x
+                                ap2 = os.path.join(fdir, fname)
+                                if nullwrite:
+                                    fdir = ap2 = ""
+                                else:
+                                    bos.makedirs(fdir)
+                                    atomic_move(self.log, abspath, ap2, vfs.flags)
+                                abspath = ap2
+                        sz = bos.path.getsize(abspath)
+
                     files.append(
                         (sz, sha_hex, sha_b64, p_file or "(discarded)", fname, abspath)
                     )
-                    at = time.time() - lifetime
-                    if xau and not runhook(
-                        self.log,
-                        xau,
-                        abspath,
-                        self.vpath,
-                        self.host,
-                        self.uname,
-                        self.asrv.vfs.get_perms(self.vpath, self.uname),
-                        at,
-                        sz,
-                        self.ip,
-                        at,
-                        "",
-                    ):
-                        t = "upload blocked by xau server config"
-                        self.log(t, 1)
-                        wunlink(self.log, abspath, vfs.flags)
-                        raise Pebkac(403, t)
-
                     dbv, vrem = vfs.get_dbv(rem)
                     self.conn.hsrv.broker.say(
                         "up2k.hash_file",
@@ -2691,13 +2831,14 @@ class HttpCli(object):
         for sz, sha_hex, sha_b64, ofn, lfn, ap in files:
             vsuf = ""
             if (self.can_read or self.can_upget) and "fk" in vfs.flags:
+                st = bos.stat(ap)
                 alg = 2 if "fka" in vfs.flags else 1
                 vsuf = "?k=" + self.gen_fk(
                     alg,
                     self.args.fk_salt,
                     ap,
-                    sz,
-                    0 if ANYWIN or not ap else bos.stat(ap).st_ino,
+                    st.st_size,
+                    0 if ANYWIN or not ap else st.st_ino,
                 )[: vfs.flags["fk"]]
 
             if "media" in self.uparam or "medialinks" in vfs.flags:
@@ -2864,6 +3005,9 @@ class HttpCli(object):
         if xbu:
             if not runhook(
                 self.log,
+                self.conn.hsrv.broker,
+                None,
+                "xbu.http.txt",
                 xbu,
                 fp,
                 self.vpath,
@@ -2903,6 +3047,9 @@ class HttpCli(object):
         xau = vfs.flags.get("xau")
         if xau and not runhook(
             self.log,
+            self.conn.hsrv.broker,
+            None,
+            "xau.http.txt",
             xau,
             fp,
             self.vpath,
@@ -2943,7 +3090,7 @@ class HttpCli(object):
         return True
 
     def _chk_lastmod(self, file_ts: int) -> tuple[str, bool]:
-        file_lastmod = formatdate(file_ts, usegmt=True)
+        file_lastmod = formatdate(file_ts)
         cli_lastmod = self.headers.get("if-modified-since")
         if cli_lastmod:
             try:
@@ -3025,8 +3172,8 @@ class HttpCli(object):
             for n, fn in enumerate([".prologue.html", ".epilogue.html"]):
                 if lnames is not None and fn not in lnames:
                     continue
-                fn = os.path.join(abspath, fn)
-                if bos.path.exists(fn):
+                fn = "%s/%s" % (abspath, fn)
+                if bos.path.isfile(fn):
                     with open(fsenc(fn), "rb") as f:
                         logues[n] = f.read().decode("utf-8")
                     if "exp" in vn.flags:
@@ -3044,7 +3191,7 @@ class HttpCli(object):
                 fns = []
 
             for fn in fns:
-                fn = os.path.join(abspath, fn)
+                fn = "%s/%s" % (abspath, fn)
                 if bos.path.isfile(fn):
                     with open(fsenc(fn), "rb") as f:
                         readme = f.read().decode("utf-8")
@@ -3579,7 +3726,7 @@ class HttpCli(object):
         # (useragent-sniffing kinshi due to caching proxies)
         mime, ico = self.ico.get(txt, not small, "raster" in self.uparam)
 
-        lm = formatdate(self.E.t0, usegmt=True)
+        lm = formatdate(self.E.t0)
         self.reply(ico, mime=mime, headers={"Last-Modified": lm})
         return True
 
@@ -3658,6 +3805,11 @@ class HttpCli(object):
             "arg_base": arg_base,
         }
 
+        if self.args.js_other and "js" not in targs:
+            zs = self.args.js_other
+            zs += "&" if "?" in zs else "?"
+            targs["js"] = zs
+
         zfv = self.vn.flags.get("html_head")
         if zfv:
             targs["this"] = self
@@ -4135,7 +4287,7 @@ class HttpCli(object):
         if self.args.no_mv:
             raise Pebkac(403, "the rename/move feature is disabled in server config")
 
-        x = self.conn.hsrv.broker.ask("up2k.handle_mv", self.uname, vsrc, vdst)
+        x = self.conn.hsrv.broker.ask("up2k.handle_mv", self.uname, self.ip, vsrc, vdst)
         self.loud_reply(x.get(), status=201)
         return True
 
@@ -4503,6 +4655,7 @@ class HttpCli(object):
             "themes": self.args.themes,
             "turbolvl": self.args.turbo,
             "u2j": self.args.u2j,
+            "u2sz": self.args.u2sz,
             "idxh": int(self.args.ih),
             "u2sort": self.args.u2sort,
         }

copyparty/httpconn.py

@@ -9,6 +9,9 @@ import threading  # typechk
 import time
 
 try:
+    if os.environ.get("PRTY_NO_TLS"):
+        raise Exception()
+
     HAVE_SSL = True
     import ssl
 except:

copyparty/httpsrv.py

@@ -12,7 +12,7 @@ import time
 
 import queue
 
-from .__init__ import ANYWIN, CORES, EXE, MACOS, TYPE_CHECKING, EnvParams, unicode
+from .__init__ import ANYWIN, CORES, EXE, MACOS, PY2, TYPE_CHECKING, EnvParams, unicode
 
 try:
     MNFE = ModuleNotFoundError
@@ -84,6 +84,12 @@ if TYPE_CHECKING:
 if True:  # pylint: disable=using-constant-test
     from typing import Any, Optional
 
+if PY2:
+    range = xrange  # type: ignore
+
+if not hasattr(socket, "AF_UNIX"):
+    setattr(socket, "AF_UNIX", -9001)
+
 
 class HttpSrv(object):
     """
@@ -240,15 +246,24 @@ class HttpSrv(object):
                     return
 
     def listen(self, sck: socket.socket, nlisteners: int) -> None:
+        tcp = sck.family != socket.AF_UNIX
+
         if self.args.j != 1:
             # lost in the pickle; redefine
             if not ANYWIN or self.args.reuseaddr:
                 sck.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
 
-            sck.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
+            if tcp:
+                sck.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
+
             sck.settimeout(None)  # < does not inherit, ^ opts above do
 
-        ip, port = sck.getsockname()[:2]
+        if tcp:
+            ip, port = sck.getsockname()[:2]
+        else:
+            ip = re.sub(r"\.[0-9]+$", "", sck.getsockname().split("/")[-1])
+            port = 0
+
         self.srvs.append(sck)
         self.bound.add((ip, port))
         self.nclimax = math.ceil(self.args.nc * 1.0 / nlisteners)
@@ -260,10 +275,19 @@ class HttpSrv(object):
 
     def thr_listen(self, srv_sck: socket.socket) -> None:
         """listens on a shared tcp server"""
-        ip, port = srv_sck.getsockname()[:2]
         fno = srv_sck.fileno()
-        hip = "[{}]".format(ip) if ":" in ip else ip
-        msg = "subscribed @ {}:{}  f{} p{}".format(hip, port, fno, os.getpid())
+        if srv_sck.family == socket.AF_UNIX:
+            ip = re.sub(r"\.[0-9]+$", "", srv_sck.getsockname())
+            msg = "subscribed @ %s  f%d p%d" % (ip, fno, os.getpid())
+            ip = ip.split("/")[-1]
+            port = 0
+            tcp = False
+        else:
+            tcp = True
+            ip, port = srv_sck.getsockname()[:2]
+            hip = "[%s]" % (ip,) if ":" in ip else ip
+            msg = "subscribed @ %s:%d  f%d p%d" % (hip, port, fno, os.getpid())
+
         self.log(self.name, msg)
 
         Daemon(self.broker.say, "sig-hsrv-up1", ("cb_httpsrv_up",))
@@ -335,11 +359,13 @@ class HttpSrv(object):
 
             try:
                 sck, saddr = srv_sck.accept()
-                cip = unicode(saddr[0])
-                if cip.startswith("::ffff:"):
-                    cip = cip[7:]
-
-                addr = (cip, saddr[1])
+                if tcp:
+                    cip = unicode(saddr[0])
+                    if cip.startswith("::ffff:"):
+                        cip = cip[7:]
+                    addr = (cip, saddr[1])
+                else:
+                    addr = (ip, sck.fileno())
             except (OSError, socket.error) as ex:
                 if self.stopping:
                     break

copyparty/ico.py

@@ -74,7 +74,7 @@ class Ico(object):
                 try:
                     _, _, tw, th = pb.textbbox((0, 0), ext)
                 except:
-                    tw, th = pb.textsize(ext)
+                    tw, th = pb.textsize(ext)  # type: ignore
 
                 tw += len(ext)
                 cw = tw // len(ext)

copyparty/mtag.py

@@ -32,6 +32,17 @@ if True:  # pylint: disable=using-constant-test
     from .util import NamedLogger, RootLogger
 
 
+try:
+    if os.environ.get("PRTY_NO_MUTAGEN"):
+        raise Exception()
+
+    from mutagen import version  # noqa: F401
+
+    HAVE_MUTAGEN = True
+except:
+    HAVE_MUTAGEN = False
+
+
 def have_ff(scmd: str) -> bool:
     if ANYWIN:
         scmd += ".exe"
@@ -48,8 +59,8 @@ def have_ff(scmd: str) -> bool:
         return bool(shutil.which(scmd))
 
 
-HAVE_FFMPEG = have_ff("ffmpeg")
-HAVE_FFPROBE = have_ff("ffprobe")
+HAVE_FFMPEG = not os.environ.get("PRTY_NO_FFMPEG") and have_ff("ffmpeg")
+HAVE_FFPROBE = not os.environ.get("PRTY_NO_FFPROBE") and have_ff("ffprobe")
 
 
 class MParser(object):
@@ -336,9 +347,7 @@ class MTag(object):
 
         if self.backend == "mutagen":
             self._get = self.get_mutagen
-            try:
-                from mutagen import version  # noqa: F401
-            except:
+            if not HAVE_MUTAGEN:
                 self.log("could not load Mutagen, trying FFprobe instead", c=3)
                 self.backend = "ffprobe"
 
@@ -578,7 +587,7 @@ class MTag(object):
                 continue
 
             if k == ".aq":
-                v /= 1000
+                v /= 1000  # type: ignore
 
             if k == "ac" and v.startswith("mp4a.40."):
                 v = "aac"

copyparty/pwhash.py

@@ -4,11 +4,21 @@ from __future__ import print_function, unicode_literals
 import argparse
 import base64
 import hashlib
+import os
 import sys
 import threading
 
 from .__init__ import unicode
 
+try:
+    if os.environ.get("PRTY_NO_ARGON2"):
+        raise Exception()
+
+    HAVE_ARGON2 = True
+    from argon2 import __version__ as argon2ver
+except:
+    HAVE_ARGON2 = False
+
 
 class PWHash(object):
     def __init__(self, args: argparse.Namespace):

copyparty/smbd.py

@@ -187,6 +187,8 @@ class SMB(object):
 
         debug('%s("%s", %s) %s @%s\033[K\033[0m', caller, vpath, str(a), perms, uname)
         vfs, rem = self.asrv.vfs.get(vpath, uname, *perms)
+        if not vfs.realpath:
+            raise Exception("unmapped vfs")
         return vfs, vfs.canonical(rem)
 
     def _listdir(self, vpath: str, *a: Any, **ka: Any) -> list[str]:
@@ -195,6 +197,8 @@ class SMB(object):
         uname = self._uname()
         # debug('listdir("%s", %s) @%s\033[K\033[0m', vpath, str(a), uname)
         vfs, rem = self.asrv.vfs.get(vpath, uname, False, False)
+        if not vfs.realpath:
+            raise Exception("unmapped vfs")
         _, vfs_ls, vfs_virt = vfs.ls(
             rem, uname, not self.args.no_scandir, [[False, False]]
         )
@@ -240,7 +244,21 @@ class SMB(object):
 
             xbu = vfs.flags.get("xbu")
             if xbu and not runhook(
-                self.nlog, xbu, ap, vpath, "", "", "", 0, 0, "1.7.6.2", 0, ""
+                self.nlog,
+                None,
+                self.hub.up2k,
+                "xbu.smb",
+                xbu,
+                ap,
+                vpath,
+                "",
+                "",
+                "",
+                0,
+                0,
+                "1.7.6.2",
+                time.time(),
+                "",
             ):
                 yeet("blocked by xbu server config: " + vpath)
 
@@ -297,7 +315,7 @@ class SMB(object):
             t = "blocked rename (no-move-acc %s): /%s @%s"
             yeet(t % (vfs1.axs.umove, vp1, uname))
 
-        self.hub.up2k.handle_mv(uname, vp1, vp2)
+        self.hub.up2k.handle_mv(uname, "1.7.6.2", vp1, vp2)
         try:
             bos.makedirs(ap2)
         except:

copyparty/ssdp.py

@@ -5,11 +5,11 @@ import errno
 import re
 import select
 import socket
-from email.utils import formatdate
+import time
 
 from .__init__ import TYPE_CHECKING
 from .multicast import MC_Sck, MCast
-from .util import CachedSet, html_escape, min_ex
+from .util import CachedSet, formatdate, html_escape, min_ex
 
 if TYPE_CHECKING:
     from .broker_util import BrokerCli
@@ -229,7 +229,7 @@ CONFIGID.UPNP.ORG: 1
 
 """
         v4 = srv.ip.replace("::ffff:", "")
-        zs = zs.format(formatdate(usegmt=True), v4, srv.hport, self.args.zsid)
+        zs = zs.format(formatdate(), v4, srv.hport, self.args.zsid)
         zb = zs[1:].replace("\n", "\r\n").encode("utf-8", "replace")
         srv.sck.sendto(zb, addr[:2])
 

copyparty/stolen/dnslib/dns.py

@@ -12,6 +12,12 @@ from .label import DNSBuffer, DNSLabel
 from .ranges import IP4, IP6, H, I, check_bytes
 
 
+try:
+    range = xrange
+except:
+    pass
+
+
 class DNSError(Exception):
     pass
 

copyparty/stolen/ifaddr/__init__.py

@@ -11,7 +11,21 @@ import os
 
 from ._shared import IP, Adapter
 
-if os.name == "nt":
+
+def nope(include_unconfigured=False):
+    return []
+
+
+try:
+    S390X = os.uname().machine == "s390x"
+except:
+    S390X = False
+
+
+if os.environ.get("PRTY_NO_IFADDR") or S390X:
+    # s390x deadlocks at libc.getifaddrs
+    get_adapters = nope
+elif os.name == "nt":
     from ._win32 import get_adapters
 elif os.name == "posix":
     from ._posix import get_adapters

copyparty/stolen/ifaddr/_shared.py

@@ -17,6 +17,7 @@ if not PY2:
     U: Callable[[str], str] = str
 else:
     U = unicode  # noqa: F821  # pylint: disable=undefined-variable,self-assigning-variable
+    range = xrange  # noqa: F821  # pylint: disable=undefined-variable,self-assigning-variable
 
 
 class Adapter(object):

copyparty/stolen/qrcodegen.py

@@ -16,6 +16,11 @@ if True:  # pylint: disable=using-constant-test
 
     from typing import Callable, List, Optional, Tuple, Union
 
+try:
+    range = xrange
+except:
+    pass
+
 
 def num_char_count_bits(ver: int) -> int:
     return 16 if (ver + 7) // 17 else 8

copyparty/sutil.py

@@ -6,7 +6,7 @@ import tempfile
 from datetime import datetime
 
 from .__init__ import CORES
-from .authsrv import AuthSrv, VFS
+from .authsrv import VFS, AuthSrv
 from .bos import bos
 from .th_cli import ThumbCli
 from .util import UTC, vjoin, vol_san

copyparty/svchub.py

@@ -28,18 +28,30 @@ if True:  # pylint: disable=using-constant-test
     import typing
     from typing import Any, Optional, Union
 
-from .__init__ import ANYWIN, EXE, MACOS, TYPE_CHECKING, E, EnvParams, unicode
+from .__init__ import ANYWIN, EXE, MACOS, PY2, TYPE_CHECKING, E, EnvParams, unicode
 from .authsrv import BAD_CFG, AuthSrv
 from .cert import ensure_cert
-from .mtag import HAVE_FFMPEG, HAVE_FFPROBE
+from .mtag import HAVE_FFMPEG, HAVE_FFPROBE, HAVE_MUTAGEN
+from .pwhash import HAVE_ARGON2
 from .tcpsrv import TcpSrv
-from .th_srv import HAVE_PIL, HAVE_VIPS, HAVE_WEBP, ThumbSrv
+from .th_srv import (
+    HAVE_AVIF,
+    HAVE_FFMPEG,
+    HAVE_FFPROBE,
+    HAVE_HEIF,
+    HAVE_PIL,
+    HAVE_VIPS,
+    HAVE_WEBP,
+    ThumbSrv,
+)
 from .up2k import Up2k
 from .util import (
     DEF_EXP,
     DEF_MTE,
     DEF_MTH,
     FFMPEG_URL,
+    HAVE_PSUTIL,
+    HAVE_SQLITE3,
     UTC,
     VERSIONS,
     Daemon,
@@ -65,6 +77,9 @@ if TYPE_CHECKING:
     except:
         pass
 
+if PY2:
+    range = xrange  # type: ignore
+
 
 class SvcHub(object):
     """
@@ -232,6 +247,8 @@ class SvcHub(object):
 
         self.up2k = Up2k(self)
 
+        self._feature_test()
+
         decs = {k: 1 for k in self.args.th_dec.split(",")}
         if not HAVE_VIPS:
             decs.pop("vips", None)
@@ -420,6 +437,58 @@ class SvcHub(object):
 
         Daemon(self.sd_notify, "sd-notify")
 
+    def _feature_test(self) -> None:
+        fok = []
+        fng = []
+        t_ff = "transcode audio, create spectrograms, video thumbnails"
+        to_check = [
+            (HAVE_SQLITE3, "sqlite", "file and media indexing"),
+            (HAVE_PIL, "pillow", "image thumbnails (plenty fast)"),
+            (HAVE_VIPS, "vips", "image thumbnails (faster, eats more ram)"),
+            (HAVE_WEBP, "pillow-webp", "create thumbnails as webp files"),
+            (HAVE_FFMPEG, "ffmpeg", t_ff + ", good-but-slow image thumbnails"),
+            (HAVE_FFPROBE, "ffprobe", t_ff + ", read audio/media tags"),
+            (HAVE_MUTAGEN, "mutagen", "read audio tags (ffprobe is better but slower)"),
+            (HAVE_ARGON2, "argon2", "secure password hashing (advanced users only)"),
+            (HAVE_HEIF, "pillow-heif", "read .heif images with pillow (rarely useful)"),
+            (HAVE_AVIF, "pillow-avif", "read .avif images with pillow (rarely useful)"),
+        ]
+        if ANYWIN:
+            to_check += [
+                (HAVE_PSUTIL, "psutil", "improved plugin cleanup  (rarely useful)")
+            ]
+
+        verbose = self.args.deps
+        if verbose:
+            self.log("dependencies", "")
+
+        for have, feat, what in to_check:
+            lst = fok if have else fng
+            lst.append((feat, what))
+            if verbose:
+                zi = 2 if have else 5
+                sgot = "found" if have else "missing"
+                t = "%7s: %s \033[36m(%s)"
+                self.log("dependencies", t % (sgot, feat, what), zi)
+
+        if verbose:
+            self.log("dependencies", "")
+            return
+
+        sok = ", ".join(x[0] for x in fok)
+        sng = ", ".join(x[0] for x in fng)
+
+        t = ""
+        if sok:
+            t += "OK: \033[32m" + sok
+        if sng:
+            if t:
+                t += ", "
+            t += "\033[0mNG: \033[35m" + sng
+
+        t += "\033[0m, see --deps"
+        self.log("dependencies", t, 6)
+
     def _check_env(self) -> None:
         try:
             files = os.listdir(E.cfg)

copyparty/szip.py

@@ -37,9 +37,7 @@ def dostime2unix(buf: bytes) -> int:
 
 
 def unixtime2dos(ts: int) -> bytes:
-    tt = time.gmtime(ts + 1)
-    dy, dm, dd, th, tm, ts = list(tt)[:6]
-
+    dy, dm, dd, th, tm, ts, _, _, _ = time.gmtime(ts + 1)
     bd = ((dy - 1980) << 9) + (dm << 5) + dd
     bt = (th << 11) + (tm << 5) + ts // 2
     try:

copyparty/tcpsrv.py

@@ -17,18 +17,23 @@ from .util import (
     E_UNREACH,
     HAVE_IPV6,
     IP6ALL,
+    VF_CAREFUL,
     Netdev,
+    atomic_move,
     min_ex,
     sunpack,
     termsize,
 )
 
 if True:
-    from typing import Generator
+    from typing import Generator, Union
 
 if TYPE_CHECKING:
     from .svchub import SvcHub
 
+if not hasattr(socket, "AF_UNIX"):
+    setattr(socket, "AF_UNIX", -9001)
+
 if not hasattr(socket, "IPPROTO_IPV6"):
     setattr(socket, "IPPROTO_IPV6", 41)
 
@@ -217,14 +222,29 @@ class TcpSrv(object):
         if self.args.qr or self.args.qrs:
             self.qr = self._qr(qr1, qr2)
 
+    def nlog(self, msg: str, c: Union[int, str] = 0) -> None:
+        self.log("tcpsrv", msg, c)
+
     def _listen(self, ip: str, port: int) -> None:
-        ipv = socket.AF_INET6 if ":" in ip else socket.AF_INET
+        if "unix:" in ip:
+            tcp = False
+            ipv = socket.AF_UNIX
+            ip = ip.split("unix:")[1]
+        elif ":" in ip:
+            tcp = True
+            ipv = socket.AF_INET6
+        else:
+            tcp = True
+            ipv = socket.AF_INET
+
         srv = socket.socket(ipv, socket.SOCK_STREAM)
 
         if not ANYWIN or self.args.reuseaddr:
             srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
 
-        srv.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
+        if tcp:
+            srv.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
+
         srv.settimeout(None)  # < does not inherit, ^ opts above do
 
         try:
@@ -236,8 +256,19 @@ class TcpSrv(object):
             srv.setsockopt(socket.SOL_IP, socket.IP_FREEBIND, 1)
 
         try:
-            srv.bind((ip, port))
-            sport = srv.getsockname()[1]
+            if tcp:
+                srv.bind((ip, port))
+            else:
+                if ANYWIN or self.args.rm_sck:
+                    if os.path.exists(ip):
+                        os.unlink(ip)
+                    srv.bind(ip)
+                else:
+                    tf = "%s.%d" % (ip, os.getpid())
+                    srv.bind(tf)
+                    atomic_move(self.nlog, tf, ip, VF_CAREFUL)
+
+            sport = srv.getsockname()[1] if tcp else port
             if port != sport:
                 # linux 6.0.16 lets you bind a port which is in use
                 # except it just gives you a random port instead
@@ -249,12 +280,23 @@ class TcpSrv(object):
             except:
                 pass
 
+            e = ""
             if ex.errno in E_ADDR_IN_USE:
                 e = "\033[1;31mport {} is busy on interface {}\033[0m".format(port, ip)
+                if not tcp:
+                    e = "\033[1;31munix-socket {} is busy\033[0m".format(ip)
             elif ex.errno in E_ADDR_NOT_AVAIL:
                 e = "\033[1;31minterface {} does not exist\033[0m".format(ip)
-            else:
+
+            if not e:
+                if not tcp:
+                    t = "\n\n\n  NOTE: this crash may be due to a unix-socket bug; try --rm-sck\n"
+                    self.log("tcpsrv", t, 2)
                 raise
+
+            if not tcp and not self.args.rm_sck:
+                e += "; maybe this is a bug? try --rm-sck"
+
             raise Exception(e)
 
     def run(self) -> None:
@@ -262,7 +304,14 @@ class TcpSrv(object):
         bound: list[tuple[str, int]] = []
         srvs: list[socket.socket] = []
         for srv in self.srv:
-            ip, port = srv.getsockname()[:2]
+            if srv.family == socket.AF_UNIX:
+                tcp = False
+                ip = re.sub(r"\.[0-9]+$", "", srv.getsockname())
+                port = 0
+            else:
+                tcp = True
+                ip, port = srv.getsockname()[:2]
+
             if ip == IP6ALL:
                 ip = "::"  # jython
 
@@ -294,8 +343,12 @@ class TcpSrv(object):
             bound.append((ip, port))
             srvs.append(srv)
             fno = srv.fileno()
-            hip = "[{}]".format(ip) if ":" in ip else ip
-            msg = "listening @ {}:{}  f{} p{}".format(hip, port, fno, os.getpid())
+            if tcp:
+                hip = "[{}]".format(ip) if ":" in ip else ip
+                msg = "listening @ {}:{}  f{} p{}".format(hip, port, fno, os.getpid())
+            else:
+                msg = "listening @ {}  f{} p{}".format(ip, fno, os.getpid())
+
             self.log("tcpsrv", msg)
             if self.args.q:
                 print(msg)
@@ -348,6 +401,8 @@ class TcpSrv(object):
     def detect_interfaces(self, listen_ips: list[str]) -> dict[str, Netdev]:
         from .stolen.ifaddr import get_adapters
 
+        listen_ips = [x for x in listen_ips if "unix:" not in x]
+
         nics = get_adapters(True)
         eps: dict[str, Netdev] = {}
         for nic in nics:

copyparty/tftpd.py

@@ -36,7 +36,7 @@ from partftpy.TftpShared import TftpException
 from .__init__ import EXE, PY2, TYPE_CHECKING
 from .authsrv import VFS
 from .bos import bos
-from .util import BytesIO, Daemon, ODict, exclude_dotfiles, min_ex, runhook, undot
+from .util import UTC, BytesIO, Daemon, ODict, exclude_dotfiles, min_ex, runhook, undot
 
 if True:  # pylint: disable=using-constant-test
     from typing import Any, Union
@@ -44,6 +44,9 @@ if True:  # pylint: disable=using-constant-test
 if TYPE_CHECKING:
     from .svchub import SvcHub
 
+if PY2:
+    range = xrange  # type: ignore
+
 
 lg = logging.getLogger("tftp")
 debug, info, warning, error = (lg.debug, lg.info, lg.warning, lg.error)
@@ -163,9 +166,16 @@ class Tftpd(object):
         if "::" in ips:
             ips.append("0.0.0.0")
 
+        ips = [x for x in ips if "unix:" not in x]
+
         if self.args.tftp4:
             ips = [x for x in ips if ":" not in x]
 
+        if not ips:
+            t = "cannot start tftp-server; no compatible IPs in -i"
+            self.nlog(t, 1)
+            return
+
         ips = list(ODict.fromkeys(ips))  # dedup
 
         for ip in ips:
@@ -241,6 +251,8 @@ class Tftpd(object):
 
         debug('%s("%s", %s) %s\033[K\033[0m', caller, vpath, str(a), perms)
         vfs, rem = self.asrv.vfs.get(vpath, "*", *perms)
+        if not vfs.realpath:
+            raise Exception("unmapped vfs")
         return vfs, vfs.canonical(rem)
 
     def _ls(self, vpath: str, raddress: str, rport: int, force=False) -> Any:
@@ -262,7 +274,7 @@ class Tftpd(object):
         dirs1 = [(v.st_mtime, v.st_size, k + "/") for k, v in vfs_ls if k in dnames]
         fils1 = [(v.st_mtime, v.st_size, k) for k, v in vfs_ls if k not in dnames]
         real1 = dirs1 + fils1
-        realt = [(datetime.fromtimestamp(mt), sz, fn) for mt, sz, fn in real1]
+        realt = [(datetime.fromtimestamp(mt, UTC), sz, fn) for mt, sz, fn in real1]
         reals = [
             (
                 "%04d-%02d-%02d %02d:%02d:%02d"
@@ -328,7 +340,21 @@ class Tftpd(object):
 
             xbu = vfs.flags.get("xbu")
             if xbu and not runhook(
-                self.nlog, xbu, ap, vpath, "", "", "", 0, 0, "8.3.8.7", 0, ""
+                self.nlog,
+                None,
+                self.hub.up2k,
+                "xbu.tftpd",
+                xbu,
+                ap,
+                vpath,
+                "",
+                "",
+                "",
+                0,
+                0,
+                "8.3.8.7",
+                time.time(),
+                "",
             ):
                 yeet("blocked by xbu server config: " + vpath)
 
@@ -336,7 +362,7 @@ class Tftpd(object):
             return self._ls(vpath, "", 0, True)
 
         if not a:
-            a = [self.args.iobuf]
+            a = (self.args.iobuf,)
 
         return open(ap, mode, *a, **ka)
 

copyparty/th_cli.py

@@ -59,7 +59,8 @@ class ThumbCli(object):
 
         want_opus = fmt in ("opus", "caf", "mp3")
         is_au = ext in self.fmt_ffa
-        if is_au:
+        is_vau = want_opus and ext in self.fmt_ffv
+        if is_au or is_vau:
             if want_opus:
                 if self.args.no_acode:
                     return None
@@ -107,7 +108,7 @@ class ThumbCli(object):
 
             fmt = sfmt
 
-        elif fmt[:1] == "p" and not is_au:
+        elif fmt[:1] == "p" and not is_au and not is_vid:
             t = "cannot thumbnail [%s]: png only allowed for waveforms"
             self.log(t % (rem), 6)
             return None

copyparty/th_srv.py

@@ -12,7 +12,7 @@ import time
 
 from queue import Queue
 
-from .__init__ import ANYWIN, TYPE_CHECKING
+from .__init__ import ANYWIN, PY2, TYPE_CHECKING
 from .authsrv import VFS
 from .bos import bos
 from .mtag import HAVE_FFMPEG, HAVE_FFPROBE, au_unpk, ffprobe
@@ -38,6 +38,9 @@ if True:  # pylint: disable=using-constant-test
 if TYPE_CHECKING:
     from .svchub import SvcHub
 
+if PY2:
+    range = xrange  # type: ignore
+
 HAVE_PIL = False
 HAVE_PILF = False
 HAVE_HEIF = False
@@ -45,22 +48,34 @@ HAVE_AVIF = False
 HAVE_WEBP = False
 
 try:
+    if os.environ.get("PRTY_NO_PIL"):
+        raise Exception()
+
     from PIL import ExifTags, Image, ImageFont, ImageOps
 
     HAVE_PIL = True
     try:
+        if os.environ.get("PRTY_NO_PILF"):
+            raise Exception()
+
         ImageFont.load_default(size=16)
         HAVE_PILF = True
     except:
         pass
 
     try:
+        if os.environ.get("PRTY_NO_PIL_WEBP"):
+            raise Exception()
+
         Image.new("RGB", (2, 2)).save(BytesIO(), format="webp")
         HAVE_WEBP = True
     except:
         pass
 
     try:
+        if os.environ.get("PRTY_NO_PIL_HEIF"):
+            raise Exception()
+
         from pyheif_pillow_opener import register_heif_opener
 
         register_heif_opener()
@@ -69,6 +84,9 @@ try:
         pass
 
     try:
+        if os.environ.get("PRTY_NO_PIL_AVIF"):
+            raise Exception()
+
         import pillow_avif  # noqa: F401  # pylint: disable=unused-import
 
         HAVE_AVIF = True
@@ -80,6 +98,9 @@ except:
     pass
 
 try:
+    if os.environ.get("PRTY_NO_VIPS"):
+        raise Exception()
+
     HAVE_VIPS = True
     import pyvips
 
@@ -304,23 +325,31 @@ class ThumbSrv(object):
                 ap_unpk = abspath
 
             if not bos.path.exists(tpath):
+                want_mp3 = tpath.endswith(".mp3")
+                want_opus = tpath.endswith(".opus") or tpath.endswith(".caf")
+                want_png = tpath.endswith(".png")
+                want_au = want_mp3 or want_opus
                 for lib in self.args.th_dec:
+                    can_au = lib == "ff" and (
+                        ext in self.fmt_ffa or ext in self.fmt_ffv
+                    )
+
                     if lib == "pil" and ext in self.fmt_pil:
                         funs.append(self.conv_pil)
                     elif lib == "vips" and ext in self.fmt_vips:
                         funs.append(self.conv_vips)
-                    elif lib == "ff" and ext in self.fmt_ffi or ext in self.fmt_ffv:
-                        funs.append(self.conv_ffmpeg)
-                    elif lib == "ff" and ext in self.fmt_ffa:
-                        if tpath.endswith(".opus") or tpath.endswith(".caf"):
+                    elif can_au and (want_png or want_au):
+                        if want_opus:
                             funs.append(self.conv_opus)
-                        elif tpath.endswith(".mp3"):
+                        elif want_mp3:
                             funs.append(self.conv_mp3)
-                        elif tpath.endswith(".png"):
+                        elif want_png:
                             funs.append(self.conv_waves)
                             png_ok = True
-                        else:
-                            funs.append(self.conv_spec)
+                    elif lib == "ff" and (ext in self.fmt_ffi or ext in self.fmt_ffv):
+                        funs.append(self.conv_ffmpeg)
+                    elif lib == "ff" and ext in self.fmt_ffa and not want_au:
+                        funs.append(self.conv_spec)
 
             tdir, tfn = os.path.split(tpath)
             ttpath = os.path.join(tdir, "w", tfn)

copyparty/u2idx.py

@@ -8,7 +8,7 @@ import threading
 import time
 from operator import itemgetter
 
-from .__init__ import ANYWIN, TYPE_CHECKING, unicode
+from .__init__ import ANYWIN, PY2, TYPE_CHECKING, unicode
 from .authsrv import LEELOO_DALLAS, VFS
 from .bos import bos
 from .up2k import up2k_wark_from_hashlist
@@ -38,6 +38,9 @@ if True:  # pylint: disable=using-constant-test
 if TYPE_CHECKING:
     from .httpsrv import HttpSrv
 
+if PY2:
+    range = xrange  # type: ignore
+
 
 class U2idx(object):
     def __init__(self, hsrv: "HttpSrv") -> None:

copyparty/up2k.py

@@ -28,8 +28,8 @@ from .fsutil import Fstab
 from .mtag import MParser, MTag
 from .util import (
     HAVE_SQLITE3,
-    VF_CAREFUL,
     SYMTIME,
+    VF_CAREFUL,
     Daemon,
     MTHash,
     Pebkac,
@@ -46,6 +46,7 @@ from .util import (
     hidedir,
     humansize,
     min_ex,
+    pathmod,
     quotep,
     rand_name,
     ren_open,
@@ -165,6 +166,7 @@ class Up2k(object):
         self.xiu_ptn = re.compile(r"(?:^|,)i([0-9]+)")
         self.xiu_busy = False  # currently running hook
         self.xiu_asleep = True  # needs rescan_cond poke to schedule self
+        self.fx_backlog: list[tuple[str, dict[str, str], str]] = []
 
         self.cur: dict[str, "sqlite3.Cursor"] = {}
         self.mem_cur = None
@@ -545,7 +547,7 @@ class Up2k(object):
                     nrm += 1
 
             if nrm:
-                self.log("{} files graduated in {}".format(nrm, vp))
+                self.log("%d files graduated in /%s" % (nrm, vp))
 
             if timeout < 10:
                 continue
@@ -1296,7 +1298,7 @@ class Up2k(object):
                         not cv
                         or liname not in th_cvds
                         or cv.lower() not in th_cvds
-                        or th_cvd.index(iname) < th_cvd.index(cv)
+                        or th_cvd.index(liname) < th_cvd.index(cv.lower())
                     )
                 ):
                     cv = iname
@@ -2544,7 +2546,7 @@ class Up2k(object):
             if self.mutex.acquire(timeout=10):
                 got_lock = True
                 with self.reg_mutex:
-                    return self._handle_json(cj)
+                    ret = self._handle_json(cj)
             else:
                 t = "cannot receive uploads right now;\nserver busy with {}.\nPlease wait; the client will retry..."
                 raise Pebkac(503, t.format(self.blocked or "[unknown]"))
@@ -2552,11 +2554,16 @@ class Up2k(object):
             if not PY2:
                 raise
             with self.mutex, self.reg_mutex:
-                return self._handle_json(cj)
+                ret = self._handle_json(cj)
         finally:
             if got_lock:
                 self.mutex.release()
 
+        if self.fx_backlog:
+            self.do_fx_backlog()
+
+        return ret
+
     def _handle_json(self, cj: dict[str, Any]) -> dict[str, Any]:
         ptop = cj["ptop"]
         if not self.register_vpath(ptop, cj["vcfg"]):
@@ -2750,7 +2757,7 @@ class Up2k(object):
                         job = deepcopy(job)
                         job["wark"] = wark
                         job["at"] = cj.get("at") or time.time()
-                        for k in "lmod ptop vtop prel host user addr".split():
+                        for k in "lmod ptop vtop prel name host user addr".split():
                             job[k] = cj.get(k) or ""
 
                         pdir = djoin(cj["ptop"], cj["prel"])
@@ -2758,28 +2765,43 @@ class Up2k(object):
                             job["name"] = rand_name(
                                 pdir, cj["name"], vfs.flags["nrand"]
                             )
-                        else:
-                            job["name"] = self._untaken(pdir, cj, now)
 
                         dst = djoin(job["ptop"], job["prel"], job["name"])
                         xbu = vfs.flags.get("xbu")
-                        if xbu and not runhook(
-                            self.log,
-                            xbu,  # type: ignore
-                            dst,
-                            job["vtop"],
-                            job["host"],
-                            job["user"],
-                            self.asrv.vfs.get_perms(job["vtop"], job["user"]),
-                            job["lmod"],
-                            job["size"],
-                            job["addr"],
-                            job["at"],
-                            "",
-                        ):
-                            t = "upload blocked by xbu server config: {}".format(dst)
-                            self.log(t, 1)
-                            raise Pebkac(403, t)
+                        if xbu:
+                            vp = djoin(job["vtop"], job["prel"], job["name"])
+                            hr = runhook(
+                                self.log,
+                                None,
+                                self,
+                                "xbu.up2k.dupe",
+                                xbu,  # type: ignore
+                                dst,
+                                vp,
+                                job["host"],
+                                job["user"],
+                                self.asrv.vfs.get_perms(job["vtop"], job["user"]),
+                                job["lmod"],
+                                job["size"],
+                                job["addr"],
+                                job["at"],
+                                "",
+                            )
+                            if not hr:
+                                t = "upload blocked by xbu server config: %s" % (dst,)
+                                self.log(t, 1)
+                                raise Pebkac(403, t)
+                            if hr.get("reloc"):
+                                x = pathmod(self.asrv.vfs, dst, vp, hr["reloc"])
+                                if x:
+                                    pdir, _, job["name"], (vfs, rem) = x
+                                    dst = os.path.join(pdir, job["name"])
+                                    job["ptop"] = vfs.realpath
+                                    job["vtop"] = vfs.vpath
+                                    job["prel"] = rem
+                                    bos.makedirs(pdir)
+
+                        job["name"] = self._untaken(pdir, job, now)
 
                         if not self.args.nw:
                             dvf: dict[str, Any] = vfs.flags
@@ -2851,11 +2873,11 @@ class Up2k(object):
                 # one chunk may occur multiple times in a file;
                 # filter to unique values for the list of missing chunks
                 # (preserve order to reduce disk thrashing)
-                lut = {}
+                lut = set()
                 for k in cj["hash"]:
                     if k not in lut:
                         job["need"].append(k)
-                        lut[k] = 1
+                        lut.add(k)
 
                 try:
                     self._new_upload(job)
@@ -3013,9 +3035,9 @@ class Up2k(object):
             times = (int(time.time()), int(lmod))
             bos.utime(dst, times, False)
 
-    def handle_chunk(
-        self, ptop: str, wark: str, chash: str
-    ) -> tuple[int, list[int], str, float, bool]:
+    def handle_chunks(
+        self, ptop: str, wark: str, chashes: list[str]
+    ) -> tuple[list[str], int, list[list[int]], str, float, bool]:
         with self.mutex, self.reg_mutex:
             self.db_act = self.vol_act[ptop] = time.time()
             job = self.registry[ptop].get(wark)
@@ -3024,49 +3046,91 @@ class Up2k(object):
                 self.log("unknown wark [{}], known: {}".format(wark, known))
                 raise Pebkac(400, "unknown wark" + SSEELOG)
 
-            if chash not in job["need"]:
-                msg = "chash = {} , need:\n".format(chash)
-                msg += "\n".join(job["need"])
-                self.log(msg)
-                raise Pebkac(400, "already got that but thanks??")
+            if len(chashes) > 1 and len(chashes[1]) < 44:
+                # first hash is full-length; expand remaining ones
+                uniq = []
+                lut = set()
+                for chash in job["hash"]:
+                    if chash not in lut:
+                        uniq.append(chash)
+                        lut.add(chash)
+                try:
+                    nchunk = uniq.index(chashes[0])
+                except:
+                    raise Pebkac(400, "unknown chunk0 [%s]" % (chashes[0]))
+                expanded = [chashes[0]]
+                for prefix in chashes[1:]:
+                    nchunk += 1
+                    chash = uniq[nchunk]
+                    if not chash.startswith(prefix):
+                        t = "next sibling chunk does not start with expected prefix [%s]: [%s]"
+                        raise Pebkac(400, t % (prefix, chash))
+                    expanded.append(chash)
+                chashes = expanded
 
-            nchunk = [n for n, v in enumerate(job["hash"]) if v == chash]
-            if not nchunk:
-                raise Pebkac(400, "unknown chunk")
+            for chash in chashes:
+                if chash not in job["need"]:
+                    msg = "chash = {} , need:\n".format(chash)
+                    msg += "\n".join(job["need"])
+                    self.log(msg)
+                    t = "already got that (%s) but thanks??"
+                    if chash not in job["hash"]:
+                        t = "unknown chunk wtf: %s"
+                    raise Pebkac(400, t % (chash,))
 
-            if chash in job["busy"]:
-                nh = len(job["hash"])
-                idx = job["hash"].index(chash)
-                t = "that chunk is already being written to:\n  {}\n  {} {}/{}\n  {}"
-                raise Pebkac(400, t.format(wark, chash, idx, nh, job["name"]))
+                if chash in job["busy"]:
+                    nh = len(job["hash"])
+                    idx = job["hash"].index(chash)
+                    t = "that chunk is already being written to:\n  {}\n  {} {}/{}\n  {}"
+                    raise Pebkac(400, t.format(wark, chash, idx, nh, job["name"]))
+
+            assert chash  # type: ignore
+            chunksize = up2k_chunksize(job["size"])
+
+            coffsets = []
+            nchunks = []
+            for chash in chashes:
+                nchunk = [n for n, v in enumerate(job["hash"]) if v == chash]
+                if not nchunk:
+                    raise Pebkac(400, "unknown chunk %s" % (chash))
+
+                ofs = [chunksize * x for x in nchunk]
+                coffsets.append(ofs)
+                nchunks.append(nchunk)
+
+            for ofs1, ofs2 in zip(coffsets, coffsets[1:]):
+                gap = (ofs2[0] - ofs1[0]) - chunksize
+                if gap:
+                    t = "only sibling chunks can be stitched; gap of %d bytes between offsets %d and %d in %s"
+                    raise Pebkac(400, t % (gap, ofs1[0], ofs2[0], job["name"]))
 
             path = djoin(job["ptop"], job["prel"], job["tnam"])
 
-            chunksize = up2k_chunksize(job["size"])
-            ofs = [chunksize * x for x in nchunk]
-
             if not job["sprs"]:
                 cur_sz = bos.path.getsize(path)
-                if ofs[0] > cur_sz:
+                if coffsets[0][0] > cur_sz:
                     t = "please upload sequentially using one thread;\nserver filesystem does not support sparse files.\n  file: {}\n  chunk: {}\n  cofs: {}\n  flen: {}"
-                    t = t.format(job["name"], nchunk[0], ofs[0], cur_sz)
+                    t = t.format(job["name"], nchunks[0][0], coffsets[0][0], cur_sz)
                     raise Pebkac(400, t)
 
             job["busy"][chash] = 1
 
         job["poke"] = time.time()
 
-        return chunksize, ofs, path, job["lmod"], job["sprs"]
+        return chashes, chunksize, coffsets, path, job["lmod"], job["sprs"]
 
-    def release_chunk(self, ptop: str, wark: str, chash: str) -> bool:
+    def release_chunks(self, ptop: str, wark: str, chashes: list[str]) -> bool:
         with self.reg_mutex:
             job = self.registry[ptop].get(wark)
             if job:
-                job["busy"].pop(chash, None)
+                for chash in chashes:
+                    job["busy"].pop(chash, None)
 
         return True
 
-    def confirm_chunk(self, ptop: str, wark: str, chash: str) -> tuple[int, str]:
+    def confirm_chunks(
+        self, ptop: str, wark: str, chashes: list[str]
+    ) -> tuple[int, str]:
         with self.mutex, self.reg_mutex:
             self.db_act = self.vol_act[ptop] = time.time()
             try:
@@ -3075,14 +3139,16 @@ class Up2k(object):
                 src = djoin(pdir, job["tnam"])
                 dst = djoin(pdir, job["name"])
             except Exception as ex:
-                return "confirm_chunk, wark, " + repr(ex)  # type: ignore
+                return "confirm_chunk, wark(%r)" % (ex,)  # type: ignore
 
-            job["busy"].pop(chash, None)
+            for chash in chashes:
+                job["busy"].pop(chash, None)
 
             try:
-                job["need"].remove(chash)
+                for chash in chashes:
+                    job["need"].remove(chash)
             except Exception as ex:
-                return "confirm_chunk, chash, " + repr(ex)  # type: ignore
+                return "confirm_chunk, chash(%s) %r" % (chash, ex)  # type: ignore
 
             ret = len(job["need"])
             if ret > 0:
@@ -3098,6 +3164,9 @@ class Up2k(object):
         with self.mutex, self.reg_mutex:
             self._finish_upload(ptop, wark)
 
+        if self.fx_backlog:
+            self.do_fx_backlog()
+
     def _finish_upload(self, ptop: str, wark: str) -> None:
         """mutex(main,reg) me"""
         try:
@@ -3291,25 +3360,30 @@ class Up2k(object):
 
         xau = False if skip_xau else vflags.get("xau")
         dst = djoin(ptop, rd, fn)
-        if xau and not runhook(
-            self.log,
-            xau,
-            dst,
-            djoin(vtop, rd, fn),
-            host,
-            usr,
-            self.asrv.vfs.get_perms(djoin(vtop, rd, fn), usr),
-            int(ts),
-            sz,
-            ip,
-            at or time.time(),
-            "",
-        ):
-            t = "upload blocked by xau server config"
-            self.log(t, 1)
-            wunlink(self.log, dst, vflags)
-            self.registry[ptop].pop(wark, None)
-            raise Pebkac(403, t)
+        if xau:
+            hr = runhook(
+                self.log,
+                None,
+                self,
+                "xau.up2k",
+                xau,
+                dst,
+                djoin(vtop, rd, fn),
+                host,
+                usr,
+                self.asrv.vfs.get_perms(djoin(vtop, rd, fn), usr),
+                ts,
+                sz,
+                ip,
+                at or time.time(),
+                "",
+            )
+            if not hr:
+                t = "upload blocked by xau server config"
+                self.log(t, 1)
+                wunlink(self.log, dst, vflags)
+                self.registry[ptop].pop(wark, None)
+                raise Pebkac(403, t)
 
         xiu = vflags.get("xiu")
         if xiu:
@@ -3493,6 +3567,9 @@ class Up2k(object):
                 if xbd:
                     if not runhook(
                         self.log,
+                        None,
+                        self,
+                        "xbd",
                         xbd,
                         abspath,
                         vpath,
@@ -3502,7 +3579,7 @@ class Up2k(object):
                         stl.st_mtime,
                         st.st_size,
                         ip,
-                        0,
+                        time.time(),
                         "",
                     ):
                         t = "delete blocked by xbd server config: {}"
@@ -3527,6 +3604,9 @@ class Up2k(object):
                 if xad:
                     runhook(
                         self.log,
+                        None,
+                        self,
+                        "xad",
                         xad,
                         abspath,
                         vpath,
@@ -3536,7 +3616,7 @@ class Up2k(object):
                         stl.st_mtime,
                         st.st_size,
                         ip,
-                        0,
+                        time.time(),
                         "",
                     )
 
@@ -3552,7 +3632,7 @@ class Up2k(object):
 
         return n_files, ok + ok2, ng + ng2
 
-    def handle_mv(self, uname: str, svp: str, dvp: str) -> str:
+    def handle_mv(self, uname: str, ip: str, svp: str, dvp: str) -> str:
         if svp == dvp or dvp.startswith(svp + "/"):
             raise Pebkac(400, "mv: cannot move parent into subfolder")
 
@@ -3569,7 +3649,7 @@ class Up2k(object):
         if stat.S_ISREG(st.st_mode) or stat.S_ISLNK(st.st_mode):
             with self.mutex:
                 try:
-                    ret = self._mv_file(uname, svp, dvp, curs)
+                    ret = self._mv_file(uname, ip, svp, dvp, curs)
                 finally:
                     for v in curs:
                         v.connection.commit()
@@ -3602,7 +3682,7 @@ class Up2k(object):
                             raise Pebkac(500, "mv: bug at {}, top {}".format(svpf, svp))
 
                         dvpf = dvp + svpf[len(svp) :]
-                        self._mv_file(uname, svpf, dvpf, curs)
+                        self._mv_file(uname, ip, svpf, dvpf, curs)
                 finally:
                     for v in curs:
                         v.connection.commit()
@@ -3627,7 +3707,7 @@ class Up2k(object):
         return "k"
 
     def _mv_file(
-        self, uname: str, svp: str, dvp: str, curs: set["sqlite3.Cursor"]
+        self, uname: str, ip: str, svp: str, dvp: str, curs: set["sqlite3.Cursor"]
     ) -> str:
         """mutex(main) me;  will mutex(reg)"""
         svn, srem = self.asrv.vfs.get(svp, uname, True, False, True)
@@ -3661,21 +3741,27 @@ class Up2k(object):
             except:
                 pass  # broken symlink; keep as-is
 
+        ftime = stl.st_mtime
+        fsize = st.st_size
+
         xbr = svn.flags.get("xbr")
         xar = dvn.flags.get("xar")
         if xbr:
             if not runhook(
                 self.log,
+                None,
+                self,
+                "xbr",
                 xbr,
                 sabs,
                 svp,
                 "",
                 uname,
                 self.asrv.vfs.get_perms(svp, uname),
-                stl.st_mtime,
-                st.st_size,
-                "",
-                0,
+                ftime,
+                fsize,
+                ip,
+                time.time(),
                 "",
             ):
                 t = "move blocked by xbr server config: {}".format(svp)
@@ -3703,16 +3789,19 @@ class Up2k(object):
             if xar:
                 runhook(
                     self.log,
+                    None,
+                    self,
+                    "xar.ln",
                     xar,
                     dabs,
                     dvp,
                     "",
                     uname,
                     self.asrv.vfs.get_perms(dvp, uname),
-                    0,
-                    0,
-                    "",
-                    0,
+                    ftime,
+                    fsize,
+                    ip,
+                    time.time(),
                     "",
                 )
 
@@ -3721,13 +3810,6 @@ class Up2k(object):
         c1, w, ftime_, fsize_, ip, at = self._find_from_vpath(svn.realpath, srem)
         c2 = self.cur.get(dvn.realpath)
 
-        if ftime_ is None:
-            ftime = stl.st_mtime
-            fsize = st.st_size
-        else:
-            ftime = ftime_
-            fsize = fsize_ or 0
-
         has_dupes = False
         if w:
             assert c1
@@ -3735,7 +3817,9 @@ class Up2k(object):
                 self._copy_tags(c1, c2, w)
 
             with self.reg_mutex:
-                has_dupes = self._forget_file(svn.realpath, srem, c1, w, is_xvol, fsize)
+                has_dupes = self._forget_file(
+                    svn.realpath, srem, c1, w, is_xvol, fsize_ or fsize
+                )
 
             if not is_xvol:
                 has_dupes = self._relink(w, svn.realpath, srem, dabs)
@@ -3805,7 +3889,7 @@ class Up2k(object):
 
             if is_link:
                 try:
-                    times = (int(time.time()), int(stl.st_mtime))
+                    times = (int(time.time()), int(ftime))
                     bos.utime(dabs, times, False)
                 except:
                     pass
@@ -3815,16 +3899,19 @@ class Up2k(object):
         if xar:
             runhook(
                 self.log,
+                None,
+                self,
+                "xar.mv",
                 xar,
                 dabs,
                 dvp,
                 "",
                 uname,
                 self.asrv.vfs.get_perms(dvp, uname),
-                0,
-                0,
-                "",
-                0,
+                ftime,
+                fsize,
+                ip,
+                time.time(),
                 "",
             )
 
@@ -4108,23 +4195,35 @@ class Up2k(object):
         xbu = self.flags[job["ptop"]].get("xbu")
         ap_chk = djoin(pdir, job["name"])
         vp_chk = djoin(job["vtop"], job["prel"], job["name"])
-        if xbu and not runhook(
-            self.log,
-            xbu,
-            ap_chk,
-            vp_chk,
-            job["host"],
-            job["user"],
-            self.asrv.vfs.get_perms(vp_chk, job["user"]),
-            int(job["lmod"]),
-            job["size"],
-            job["addr"],
-            int(job["t0"]),
-            "",
-        ):
-            t = "upload blocked by xbu server config: {}".format(vp_chk)
-            self.log(t, 1)
-            raise Pebkac(403, t)
+        if xbu:
+            hr = runhook(
+                self.log,
+                None,
+                self,
+                "xbu.up2k",
+                xbu,
+                ap_chk,
+                vp_chk,
+                job["host"],
+                job["user"],
+                self.asrv.vfs.get_perms(vp_chk, job["user"]),
+                job["lmod"],
+                job["size"],
+                job["addr"],
+                job["t0"],
+                "",
+            )
+            if not hr:
+                t = "upload blocked by xbu server config: {}".format(vp_chk)
+                self.log(t, 1)
+                raise Pebkac(403, t)
+            if hr.get("reloc"):
+                x = pathmod(self.asrv.vfs, ap_chk, vp_chk, hr["reloc"])
+                if x:
+                    pdir, _, job["name"], (vfs, rem) = x
+                    job["ptop"] = vfs.realpath
+                    job["vtop"] = vfs.vpath
+                    job["prel"] = rem
 
         tnam = job["name"] + ".PARTIAL"
         if self.args.dotpart:
@@ -4398,6 +4497,9 @@ class Up2k(object):
             with self.rescan_cond:
                 self.rescan_cond.notify_all()
 
+        if self.fx_backlog:
+            self.do_fx_backlog()
+
         return True
 
     def hash_file(
@@ -4429,6 +4531,48 @@ class Up2k(object):
             self.hashq.put(zt)
             self.n_hashq += 1
 
+    def do_fx_backlog(self):
+        with self.mutex, self.reg_mutex:
+            todo = self.fx_backlog
+            self.fx_backlog = []
+        for act, hr, req_vp in todo:
+            self.hook_fx(act, hr, req_vp)
+
+    def hook_fx(self, act: str, hr: dict[str, str], req_vp: str) -> None:
+        bad = [k for k in hr if k != "vp"]
+        if bad:
+            t = "got unsupported key in %s from hook: %s"
+            raise Exception(t % (act, bad))
+
+        for fvp in hr.get("vp") or []:
+            # expect vpath including filename; either absolute
+            # or relative to the client's vpath (request url)
+            if fvp.startswith("/"):
+                fvp, fn = vsplit(fvp[1:])
+                fvp = "/" + fvp
+            else:
+                fvp, fn = vsplit(fvp)
+
+            x = pathmod(self.asrv.vfs, "", req_vp, {"vp": fvp, "fn": fn})
+            if not x:
+                t = "hook_fx(%s): failed to resolve %s based on %s"
+                self.log(t % (act, fvp, req_vp))
+                continue
+
+            ap, rd, fn, (vn, rem) = x
+            vp = vjoin(rd, fn)
+            if not vp:
+                raise Exception("hook_fx: blank vp from pathmod")
+
+            if act == "idx":
+                rd = rd[len(vn.vpath) :].strip("/")
+                self.hash_file(
+                    vn.realpath, vn.vpath, vn.flags, rd, fn, "", time.time(), "", True
+                )
+
+            if act == "del":
+                self._handle_rm(LEELOO_DALLAS, "", vp, [], False, False)
+
     def shutdown(self) -> None:
         self.stop = True
 

copyparty/util.py

@@ -26,7 +26,6 @@ import threading
 import time
 import traceback
 from collections import Counter
-from email.utils import formatdate
 
 from ipaddress import IPv4Address, IPv4Network, IPv6Address, IPv6Network
 from queue import Queue
@@ -60,6 +59,10 @@ except:
     UTC = _UTC()
 
 
+if PY2:
+    range = xrange  # type: ignore
+
+
 if sys.version_info >= (3, 7) or (
     sys.version_info >= (3, 6) and platform.python_implementation() == "CPython"
 ):
@@ -99,6 +102,9 @@ except:
     pass
 
 try:
+    if os.environ.get("PRTY_NO_SQLITE"):
+        raise Exception()
+
     HAVE_SQLITE3 = True
     import sqlite3
 
@@ -107,6 +113,9 @@ except:
     HAVE_SQLITE3 = False
 
 try:
+    if os.environ.get("PRTY_NO_PSUTIL"):
+        raise Exception()
+
     HAVE_PSUTIL = True
     import psutil
 except:
@@ -137,10 +146,15 @@ if TYPE_CHECKING:
     import magic
 
     from .authsrv import VFS
+    from .broker_util import BrokerCli
+    from .up2k import Up2k
 
 FAKE_MP = False
 
 try:
+    if os.environ.get("PRTY_NO_MP"):
+        raise ImportError()
+
     import multiprocessing as mp
 
     # import multiprocessing.dummy as mp
@@ -159,6 +173,9 @@ else:
 
 
 try:
+    if os.environ.get("PRTY_NO_IPV6"):
+        raise Exception()
+
     socket.inet_pton(socket.AF_INET6, "::1")
     HAVE_IPV6 = True
 except:
@@ -794,7 +811,7 @@ class CachedSet(object):
 
         c = self.c = {k: v for k, v in self.c.items() if now - v < self.maxage}
         try:
-            self.oldest = c[min(c, key=c.get)]
+            self.oldest = c[min(c, key=c.get)]  # type: ignore
         except:
             self.oldest = now
 
@@ -1377,7 +1394,7 @@ def vol_san(vols: list["VFS"], txt: bytes) -> bytes:
 def min_ex(max_lines: int = 8, reverse: bool = False) -> str:
     et, ev, tb = sys.exc_info()
     stb = traceback.extract_tb(tb) if tb else traceback.extract_stack()[:-1]
-    fmt = "%s @ %d <%s>: %s"
+    fmt = "%s:%d <%s>: %s"
     ex = [fmt % (fp.split(os.sep)[-1], ln, fun, txt) for fp, ln, fun, txt in stb]
     if et or ev or tb:
         ex.append("[%s] %s" % (et.__name__ if et else "(anonymous)", ev))
@@ -1821,10 +1838,21 @@ def gen_filekey_dbg(
     return ret
 
 
+WKDAYS = "Mon Tue Wed Thu Fri Sat Sun".split()
+MONTHS = "Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec".split()
+RFC2822 = "%s, %02d %s %04d %02d:%02d:%02d GMT"
+
+
+def formatdate(ts: Optional[float] = None) -> str:
+    # gmtime ~= datetime.fromtimestamp(ts, UTC).timetuple()
+    y, mo, d, h, mi, s, wd, _, _ = time.gmtime(ts)
+    return RFC2822 % (WKDAYS[wd], d, MONTHS[mo - 1], y, h, mi, s)
+
+
 def gencookie(k: str, v: str, r: str, tls: bool, dur: int = 0, txt: str = "") -> str:
     v = v.replace("%", "%25").replace(";", "%3B")
     if dur:
-        exp = formatdate(time.time() + dur, usegmt=True)
+        exp = formatdate(time.time() + dur)
     else:
         exp = "Fri, 15 Aug 1997 01:00:00 GMT"
 
@@ -1839,12 +1867,10 @@ def humansize(sz: float, terse: bool = False) -> str:
 
         sz /= 1024.0
 
-    ret = " ".join([str(sz)[:4].rstrip("."), unit])
-
-    if not terse:
-        return ret
-
-    return ret.replace("iB", "").replace(" ", "")
+    if terse:
+        return "%s%s" % (str(sz)[:4].rstrip("."), unit[:1])
+    else:
+        return "%s %s" % (str(sz)[:4].rstrip("."), unit)
 
 
 def unhumanize(sz: str) -> int:
@@ -1896,7 +1922,7 @@ def uncyg(path: str) -> str:
 def undot(path: str) -> str:
     ret: list[str] = []
     for node in path.split("/"):
-        if node in ["", "."]:
+        if node == "." or not node:
             continue
 
         if node == "..":
@@ -2049,7 +2075,7 @@ def _quotep2(txt: str) -> str:
     """url quoter which deals with bytes correctly"""
     btxt = w8enc(txt)
     quot = quote(btxt, safe=b"/")
-    return w8dec(quot.replace(b" ", b"+"))
+    return w8dec(quot.replace(b" ", b"+"))  # type: ignore
 
 
 def _quotep3(txt: str) -> str:
@@ -2093,6 +2119,72 @@ def ujoin(rd: str, fn: str) -> str:
         return rd or fn
 
 
+def log_reloc(
+    log: "NamedLogger",
+    re: dict[str, str],
+    pm: tuple[str, str, str, tuple["VFS", str]],
+    ap: str,
+    vp: str,
+    fn: str,
+    vn: "VFS",
+    rem: str,
+) -> None:
+    nap, nvp, nfn, (nvn, nrem) = pm
+    t = "reloc %s:\nold ap [%s]\nnew ap [%s\033[36m/%s\033[0m]\nold vp [%s]\nnew vp [%s\033[36m/%s\033[0m]\nold fn [%s]\nnew fn [%s]\nold vfs [%s]\nnew vfs [%s]\nold rem [%s]\nnew rem [%s]"
+    log(t % (re, ap, nap, nfn, vp, nvp, nfn, fn, nfn, vn.vpath, nvn.vpath, rem, nrem))
+
+
+def pathmod(
+    vfs: "VFS", ap: str, vp: str, mod: dict[str, str]
+) -> Optional[tuple[str, str, str, tuple["VFS", str]]]:
+    # vfs: authsrv.vfs
+    # ap: original abspath to a file
+    # vp: original urlpath to a file
+    # mod: modification (ap/vp/fn)
+
+    nvp = "\n"  # new vpath
+    ap = os.path.dirname(ap)
+    vp, fn = vsplit(vp)
+    if mod.get("fn"):
+        fn = mod["fn"]
+        nvp = vp
+
+    for ref, k in ((ap, "ap"), (vp, "vp")):
+        if k not in mod:
+            continue
+
+        ms = mod[k].replace(os.sep, "/")
+        if ms.startswith("/"):
+            np = ms
+        elif k == "vp":
+            np = undot(vjoin(ref, ms))
+        else:
+            np = os.path.abspath(os.path.join(ref, ms))
+
+        if k == "vp":
+            nvp = np.lstrip("/")
+            continue
+
+        # try to map abspath to vpath
+        np = np.replace("/", os.sep)
+        for vn_ap, vn in vfs.all_aps:
+            if not np.startswith(vn_ap):
+                continue
+            zs = np[len(vn_ap) :].replace(os.sep, "/")
+            nvp = vjoin(vn.vpath, zs)
+            break
+
+    if nvp == "\n":
+        return None
+
+    vn, rem = vfs.get(nvp, "*", False, False)
+    if not vn.realpath:
+        raise Exception("unmapped vfs")
+
+    ap = vn.canonical(rem)
+    return ap, nvp, fn, (vn, rem)
+
+
 def _w8dec2(txt: bytes) -> str:
     """decodes filesystem-bytes to wtf8"""
     return surrogateescape.decodefilename(txt)
@@ -2709,30 +2801,30 @@ def rmdirs_up(top: str, stop: str) -> tuple[list[str], list[str]]:
 
 def unescape_cookie(orig: str) -> str:
     # mw=idk; doot=qwe%2Crty%3Basd+fgh%2Bjkl%25zxc%26vbn  # qwe,rty;asd fgh+jkl%zxc&vbn
-    ret = ""
+    ret = []
     esc = ""
     for ch in orig:
         if ch == "%":
-            if len(esc) > 0:
-                ret += esc
+            if esc:
+                ret.append(esc)
             esc = ch
 
-        elif len(esc) > 0:
+        elif esc:
             esc += ch
             if len(esc) == 3:
                 try:
-                    ret += chr(int(esc[1:], 16))
+                    ret.append(chr(int(esc[1:], 16)))
                 except:
-                    ret += esc
+                    ret.append(esc)
                 esc = ""
 
         else:
-            ret += ch
+            ret.append(ch)
 
-    if len(esc) > 0:
-        ret += esc
+    if esc:
+        ret.append(esc)
 
-    return ret
+    return "".join(ret)
 
 
 def guess_mime(url: str, fallback: str = "application/octet-stream") -> str:
@@ -3106,6 +3198,7 @@ def runihook(
 
 def _runhook(
     log: Optional["NamedLogger"],
+    src: str,
     cmd: str,
     ap: str,
     vp: str,
@@ -3117,14 +3210,16 @@ def _runhook(
     ip: str,
     at: float,
     txt: str,
-) -> bool:
+) -> dict[str, Any]:
+    ret = {"rc": 0}
     areq, chk, fork, jtxt, wait, sp_ka, acmd = _parsehook(log, cmd)
     if areq:
         for ch in areq:
             if ch not in perms:
                 t = "user %s not allowed to run hook %s; need perms %s, have %s"
-                log(t % (uname, cmd, areq, perms))
-                return True  # fallthrough to next hook
+                if log:
+                    log(t % (uname, cmd, areq, perms))
+                return ret  # fallthrough to next hook
     if jtxt:
         ja = {
             "ap": ap,
@@ -3136,6 +3231,7 @@ def _runhook(
             "host": host,
             "user": uname,
             "perms": perms,
+            "src": src,
             "txt": txt,
         }
         arg = json.dumps(ja)
@@ -3154,18 +3250,34 @@ def _runhook(
     else:
         rc, v, err = runcmd(bcmd, **sp_ka)  # type: ignore
         if chk and rc:
+            ret["rc"] = rc
             retchk(rc, bcmd, err, log, 5)
-            return False
+        else:
+            try:
+                ret = json.loads(v)
+            except:
+                ret = {}
+
+            try:
+                if "stdout" not in ret:
+                    ret["stdout"] = v
+                if "rc" not in ret:
+                    ret["rc"] = rc
+            except:
+                ret = {"rc": rc, "stdout": v}
 
     wait -= time.time() - t0
     if wait > 0:
         time.sleep(wait)
 
-    return True
+    return ret
 
 
 def runhook(
     log: Optional["NamedLogger"],
+    broker: Optional["BrokerCli"],
+    up2k: Optional["Up2k"],
+    src: str,
     cmds: list[str],
     ap: str,
     vp: str,
@@ -3177,19 +3289,42 @@ def runhook(
     ip: str,
     at: float,
     txt: str,
-) -> bool:
+) -> dict[str, Any]:
+    assert broker or up2k
+    asrv = (broker or up2k).asrv
+    args = (broker or up2k).args
     vp = vp.replace("\\", "/")
+    ret = {"rc": 0}
     for cmd in cmds:
         try:
-            if not _runhook(log, cmd, ap, vp, host, uname, perms, mt, sz, ip, at, txt):
-                return False
+            hr = _runhook(
+                log, src, cmd, ap, vp, host, uname, perms, mt, sz, ip, at, txt
+            )
+            if log and args.hook_v:
+                log("hook(%s) [%s] => \033[32m%s" % (src, cmd, hr), 6)
+            if not hr:
+                return {}
+            for k, v in hr.items():
+                if k in ("idx", "del") and v:
+                    if broker:
+                        broker.say("up2k.hook_fx", k, v, vp)
+                    else:
+                        up2k.fx_backlog.append((k, v, vp))
+                elif k == "reloc" and v:
+                    # idk, just take the last one ig
+                    ret["reloc"] = v
+                elif k in ret:
+                    if k == "rc" and v:
+                        ret[k] = v
+                else:
+                    ret[k] = v
         except Exception as ex:
             (log or print)("hook: {}".format(ex))
             if ",c," in "," + cmd:
-                return False
+                return {}
             break
 
-    return True
+    return ret
 
 
 def loadpy(ap: str, hot: bool) -> Any:

copyparty/web/baguettebox.js

@@ -29,6 +29,7 @@ window.baguetteBox = (function () {
         isOverlayVisible = false,
         touch = {},  // start-pos
         touchFlag = false,  // busy
+        scrollCSS = ['', ''],
         scrollTimer = 0,
         re_i = /^[^?]+\.(a?png|avif|bmp|gif|heif|jpe?g|jfif|svg|webp)(\?|$)/i,
         re_v = /^[^?]+\.(webm|mkv|mp4)(\?|$)/i,
@@ -567,6 +568,12 @@ window.baguetteBox = (function () {
 
     function showOverlay(chosenImageIndex) {
         if (options.noScrollbars) {
+            var a = document.documentElement.style.overflowY,
+                b = document.body.style.overflowY;
+
+            if (a != 'hidden' || b != 'scroll')
+                scrollCSS = [a, b];
+
             document.documentElement.style.overflowY = 'hidden';
             document.body.style.overflowY = 'scroll';
         }
@@ -615,8 +622,8 @@ window.baguetteBox = (function () {
         playvid(false);
         removeFromCache('#files');
         if (options.noScrollbars) {
-            document.documentElement.style.overflowY = 'auto';
-            document.body.style.overflowY = 'auto';
+            document.documentElement.style.overflowY = scrollCSS[0];
+            document.body.style.overflowY = scrollCSS[1];
         }
 
         try {
@@ -1060,7 +1067,7 @@ window.baguetteBox = (function () {
                 return showNextImage();
 
             show_buttons('t');
-            
+
             if (Date.now() - ctime <= 500 && !IPHONE)
                 tglfull();
 

copyparty/web/browser.css

@@ -1017,9 +1017,6 @@ html.y #path a:hover {
 	color: var(--g-dfg);
 }
 #ggrid>a.au:before {
-	content: '💾';
-}
-html.np_open #ggrid>a.au:before {
 	content: '▶';
 }
 #ggrid>a:before {

copyparty/web/browser.js

@@ -210,6 +210,8 @@ var Ls = {
 
 		"cut_datechk": "has no effect unless the turbo button is enabled$N$Nreduces the yolo factor by a tiny amount; checks whether the file timestamps on the server matches yours$N$Nshould <em>theoretically</em> catch most unfinished / corrupted uploads, but is not a substitute for doing a verification pass with turbo disabled afterwards\">date-chk",
 
+		"cut_u2sz": "size (in MiB) of each upload chunk; big values fly better across the atlantic. Try low values on very unreliable connections",
+
 		"cut_flag": "ensure only one tab is uploading at a time $N -- other tabs must have this enabled too $N -- only affects tabs on the same domain",
 
 		"cut_az": "upload files in alphabetical order, rather than smallest-file-first$N$Nalphabetical order can make it easier to eyeball if something went wrong on the server, but it makes uploading slightly slower on fiber / LAN",
@@ -268,6 +270,8 @@ var Ls = {
 		"mb_play": "play",
 		"mm_hashplay": "play this audio file?",
 		"mp_breq": "need firefox 82+ or chrome 73+ or iOS 15+",
+		"mm_bload": "now loading...",
+		"mm_bconv": "converting to {0}, please wait...",
 		"mm_opusen": "your browser cannot play aac / m4a files;\ntranscoding to opus is now enabled",
 		"mm_playerr": "playback failed: ",
 		"mm_eabrt": "The playback attempt was cancelled",
@@ -358,6 +362,7 @@ var Ls = {
 		"tvt_sel": "select file &nbsp; ( for cut / delete / ... )$NHotkey: S\">sel",
 		"tvt_edit": "open file in text editor$NHotkey: E\">✏️ edit",
 
+		"gt_vau": "don't show videos, just play the audio\">🎧",
 		"gt_msel": "enable file selection; ctrl-click a file to override$N$N&lt;em&gt;when active: doubleclick a file / folder to open it&lt;/em&gt;$N$NHotkey: S\">multiselect",
 		"gt_crop": "center-crop thumbnails\">crop",
 		"gt_3x": "hi-res thumbnails\">3x",
@@ -461,7 +466,7 @@ var Ls = {
 		"u_badf": 'These {0} files (of {1} total) were skipped, possibly due to filesystem permissions:\n\n',
 		"u_blankf": 'These {0} files (of {1} total) are blank / empty; upload them anyways?\n\n',
 		"u_just1": '\nMaybe it works better if you select just one file',
-		"u_ff_many": "This amount of files <em>may</em> cause Firefox to skip some files, or crash.\nPlease try again with fewer files (or use Chrome) if that happens.",
+		"u_ff_many": "if you're using <b>Linux / MacOS / Android,</b> then this amount of files <a href=\"https://bugzilla.mozilla.org/show_bug.cgi?id=1790500\" target=\"_blank\"><em>may</em> crash Firefox!</a>\nif that happens, please try again (or use Chrome).",
 		"u_up_life": "This upload will be deleted from the server\n{0} after it completes",
 		"u_asku": 'upload these {0} files to <code>{1}</code>',
 		"u_unpt": "you can undo / delete this upload using the top-left 🧯",
@@ -478,12 +483,13 @@ var Ls = {
 		"u_ehsinit": "server rejected the request to initiate upload; retrying...",
 		"u_eneths": "network error while performing upload handshake; retrying...",
 		"u_enethd": "network error while testing target existence; retrying...",
+		"u_cbusy": "waiting for server to trust us again after a network glitch...",
 		"u_ehsdf": "server ran out of disk space!\n\nwill keep retrying, in case someone\nfrees up enough space to continue",
 		"u_emtleak1": "it looks like your webbrowser may have a memory leak;\nplease",
 		"u_emtleak2": ' <a href="{0}">switch to https (recommended)</a> or ',
 		"u_emtleak3": ' ',
-		"u_emtleakc": 'try the following:\n<ul><li>hit <code>F5</code> to refresh the page</li><li>then disable the &nbsp;<code>mt</code>&nbsp; button in the &nbsp;<code>⚙️ settings</code></li><li>and try that upload again</li></ul>Uploads will be a bit slower, but oh well.\nSorry for the trouble !\n\nPS: chrome v107 <a href="https://bugs.chromium.org/p/chromium/issues/detail?id=1354816">has a bugfix</a> for this',
-		"u_emtleakf": 'try the following:\n<ul><li>hit <code>F5</code> to refresh the page</li><li>then enable <code>🥔</code> (potato) in the upload UI<li>and try that upload again</li></ul>\nPS: firefox <a href="https://bugzilla.mozilla.org/show_bug.cgi?id=1790500">will hopefully have a bugfix</a> at some point',
+		"u_emtleakc": 'try the following:\n<ul><li>hit <code>F5</code> to refresh the page</li><li>then disable the &nbsp;<code>mt</code>&nbsp; button in the &nbsp;<code>⚙️ settings</code></li><li>and try that upload again</li></ul>Uploads will be a bit slower, but oh well.\nSorry for the trouble !\n\nPS: chrome v107 <a href="https://bugs.chromium.org/p/chromium/issues/detail?id=1354816" target="_blank">has a bugfix</a> for this',
+		"u_emtleakf": 'try the following:\n<ul><li>hit <code>F5</code> to refresh the page</li><li>then enable <code>🥔</code> (potato) in the upload UI<li>and try that upload again</li></ul>\nPS: firefox <a href="https://bugzilla.mozilla.org/show_bug.cgi?id=1790500" target="_blank">will hopefully have a bugfix</a> at some point',
 		"u_s404": "not found on server",
 		"u_expl": "explain",
 		"u_maxconn": "most browsers limit this to 6, but firefox lets you raise it with <code>connections-per-server</code> in <code>about:config</code>",
@@ -721,6 +727,8 @@ var Ls = {
 
 		"cut_datechk": "har ingen effekt dersom turbo er avslått$N$Ngjør turbo bittelitt tryggere ved å sjekke datostemplingen på filene (i tillegg til filstørrelse)$N$N<em>burde</em> oppdage og gjenoppta de fleste ufullstendige opplastninger, men er <em>ikke</em> en fullverdig erstatning for å deaktivere turbo og gjøre en skikkelig sjekk\">date-chk",
 
+		"cut_u2sz": "størrelse i megabyte for hvert bruddstykke for opplastning. Store verdier flyr bedre over atlanteren. Små verdier kan være bedre på særdeles ustabile forbindelser",
+
 		"cut_flag": "samkjører nettleserfaner slik at bare én $N kan holde på med befaring / opplastning $N -- andre faner må også ha denne skrudd på $N -- fungerer kun innenfor samme domene",
 
 		"cut_az": "last opp filer i alfabetisk rekkefølge, istedenfor minste-fil-først$N$Nalfabetisk kan gjøre det lettere å anslå om alt gikk bra, men er bittelitt tregere på fiber / LAN",
@@ -779,6 +787,8 @@ var Ls = {
 		"mb_play": "lytt",
 		"mm_hashplay": "spill denne sangen?",
 		"mp_breq": "krever firefox 82+, chrome 73+, eller iOS 15+",
+		"mm_bload": "laster inn...",
+		"mm_bconv": "konverterer til {0}, vent litt...",
 		"mm_opusen": "nettleseren din forstår ikke aac / m4a;\nkonvertering til opus er nå aktivert",
 		"mm_playerr": "avspilling feilet: ",
 		"mm_eabrt": "Avspillingsforespørselen ble avbrutt",
@@ -869,6 +879,7 @@ var Ls = {
 		"tvt_sel": "markér filen &nbsp; ( for utklipp / sletting / ... )$NSnarvei: S\">merk",
 		"tvt_edit": "redigér filen$NSnarvei: E\">✏️ endre",
 
+		"gt_vau": "ikke vis videofiler, bare spill lyden\">🎧",
 		"gt_msel": "markér filer istedenfor å åpne dem; ctrl-klikk filer for å overstyre$N$N&lt;em&gt;når aktiv: dobbelklikk en fil / mappe for å åpne&lt;/em&gt;$N$NSnarvei: S\">markering",
 		"gt_crop": "beskjær ikonene så de passer bedre\">✂",
 		"gt_3x": "høyere oppløsning på ikoner\">3x",
@@ -972,7 +983,7 @@ var Ls = {
 		"u_badf": 'Disse {0} filene (av totalt {1}) kan ikke leses, kanskje pga rettighetsproblemer i filsystemet på datamaskinen din:\n\n',
 		"u_blankf": 'Disse {0} filene (av totalt {1}) er blanke / uten innhold; ønsker du å laste dem opp uansett?\n\n',
 		"u_just1": '\nFunker kanskje bedre hvis du bare tar én fil om gangen',
-		"u_ff_many": "Det var mange filer! Mulig at Firefox kommer til å krasje, eller\nhoppe over et par av dem. Smart å ha Chrome på lur i tilfelle.",
+		"u_ff_many": 'Hvis du bruker <b>Linux / MacOS / Android,</b> så kan dette antallet filer<br /><a href="https://bugzilla.mozilla.org/show_bug.cgi?id=1790500" target="_blank"><em>kanskje</em> krasje Firefox!</a> Hvis det skjer, så prøv igjen (eller bruk Chrome).',
 		"u_up_life": "Filene slettes fra serveren {0}\netter at opplastningen er fullført",
 		"u_asku": 'Laste opp disse {0} filene til <code>{1}</code>',
 		"u_unpt": "Du kan angre / slette opplastningen med 🧯 oppe til venstre",
@@ -989,12 +1000,13 @@ var Ls = {
 		"u_ehsinit": "server nektet forespørselen om å begynne en ny opplastning; prøver igjen...",
 		"u_eneths": "et problem med nettverket gjorde at avtale om opplastning ikke kunne inngås; prøver igjen...",
 		"u_enethd": "et problem med nettverket gjorde at filsjekk ikke kunne utføres; prøver igjen...",
+		"u_cbusy": "venter på klarering ifra server etter et lite nettverksglipp...",
 		"u_ehsdf": "serveren er full!\n\nprøver igjen regelmessig,\ni tilfelle noen rydder litt...",
 		"u_emtleak1": "uff, det er mulig at nettleseren din har en minnelekkasje...\nForeslår",
 		"u_emtleak2": ' helst at du <a href="{0}">bytter til https</a>, eller ',
 		"u_emtleak3": ' at du ',
-		"u_emtleakc": 'prøver følgende:\n<ul><li>trykk F5 for å laste siden på nytt</li><li>så skru av &nbsp;<code>mt</code>&nbsp; bryteren under &nbsp;<code>⚙️ innstillinger</code></li><li>og forsøk den samme opplastningen igjen</li></ul>Opplastning vil gå litt tregere, men det får så være.\nBeklager bryderiet !\n\nPS: feilen <a href="<a href="https://bugs.chromium.org/p/chromium/issues/detail?id=1354816">skal være fikset</a> i chrome v107',
-		"u_emtleakf": 'prøver følgende:\n<ul><li>trykk F5 for å laste siden på nytt</li><li>så skru på <code>🥔</code> ("enkelt UI") i opplasteren</li><li>og forsøk den samme opplastningen igjen</li></ul>\nPS: Firefox <a href="https://bugzilla.mozilla.org/show_bug.cgi?id=1790500">fikser forhåpentligvis feilen</a> en eller annen gang',
+		"u_emtleakc": 'prøver følgende:\n<ul><li>trykk F5 for å laste siden på nytt</li><li>så skru av &nbsp;<code>mt</code>&nbsp; bryteren under &nbsp;<code>⚙️ innstillinger</code></li><li>og forsøk den samme opplastningen igjen</li></ul>Opplastning vil gå litt tregere, men det får så være.\nBeklager bryderiet !\n\nPS: feilen <a href="https://bugs.chromium.org/p/chromium/issues/detail?id=1354816" target="_blank">skal være fikset</a> i chrome v107',
+		"u_emtleakf": 'prøver følgende:\n<ul><li>trykk F5 for å laste siden på nytt</li><li>så skru på <code>🥔</code> ("enkelt UI") i opplasteren</li><li>og forsøk den samme opplastningen igjen</li></ul>\nPS: Firefox <a href="https://bugzilla.mozilla.org/show_bug.cgi?id=1790500" target="_blank">fikser forhåpentligvis feilen</a> en eller annen gang',
 		"u_s404": "ikke funnet på serveren",
 		"u_expl": "forklar",
 		"u_maxconn": "de fleste nettlesere tillater ikke mer enn 6, men firefox lar deg øke grensen med <code>connections-per-server</code> i <code>about:config</code>",
@@ -1251,6 +1263,7 @@ ebi('op_cfg').innerHTML = (
 	'		<a id="hashw" class="tgl btn" href="#" tt="' + L.cut_mt + '</a>\n' +
 	'		<a id="u2turbo" class="tgl btn ttb" href="#" tt="' + L.cut_turbo + '</a>\n' +
 	'		<a id="u2tdate" class="tgl btn ttb" href="#" tt="' + L.cut_datechk + '</a>\n' +
+	'		<input type="text" id="u2szg" value="" ' + NOAC + ' style="width:3em" tt="' + L.cut_u2sz + '" />' +
 	'		<a id="flag_en" class="tgl btn" href="#" tt="' + L.cut_flag + '">💤</a>\n' +
 	'		<a id="u2sort" class="tgl btn" href="#" tt="' + L.cut_az + '">az</a>\n' +
 	'		<a id="upnag" class="tgl btn" href="#" tt="' + L.cut_nag + '">🔔</a>\n' +
@@ -1593,6 +1606,8 @@ var mpl = (function () {
 	r.pp = function () {
 		var adur, apos, playing = mp.au && !mp.au.paused;
 
+		clearTimeout(mpl.t_eplay);
+
 		clmod(ebi('np_inf'), 'playing', playing);
 
 		if (mp.au && isNum(adur = mp.au.duration) && isNum(apos = mp.au.currentTime) && apos >= 0)
@@ -1702,7 +1717,7 @@ catch (ex) { }
 
 
 var re_au_native = (can_ogg || have_acode) ? /\.(aac|flac|m4a|mp3|ogg|opus|wav)$/i : /\.(aac|flac|m4a|mp3|wav)$/i,
-	re_au_all = /\.(aac|ac3|aif|aiff|alac|alaw|amr|ape|au|dfpwm|dts|flac|gsm|it|itgz|itxz|itz|m4a|mdgz|mdxz|mdz|mo3|mod|mp2|mp3|mpc|mptm|mt2|mulaw|ogg|okt|opus|ra|s3m|s3gz|s3xz|s3z|tak|tta|ulaw|wav|wma|wv|xm|xmgz|xmxz|xmz|xpk)$/i;
+	re_au_all = /\.(aac|ac3|aif|aiff|alac|alaw|amr|ape|au|dfpwm|dts|flac|gsm|it|itgz|itxz|itz|m4a|mdgz|mdxz|mdz|mo3|mod|mp2|mp3|mpc|mptm|mt2|mulaw|ogg|okt|opus|ra|s3m|s3gz|s3xz|s3z|tak|tta|ulaw|wav|wma|wv|xm|xmgz|xmxz|xmz|xpk|3gp|asf|avi|flv|m4v|mkv|mov|mp4|mpeg|mpeg2|mpegts|mpg|mpg2|nut|ogm|ogv|rm|ts|vob|webm|wmv)$/i;
 
 
 // extract songs + add play column
@@ -2178,8 +2193,21 @@ var pbar = (function () {
 		}
 		pctx.clearRect(0, 0, pc.w, pc.h);
 
-		if (!mp || !mp.au || !isNum(adur = mp.au.duration) || !isNum(apos = mp.au.currentTime) || apos < 0 || adur < apos)
+		if (!mp || !mp.au)
+			return;  // not-init
+
+		if (!isNum(adur = mp.au.duration) || !isNum(apos = mp.au.currentTime) || apos < 0 || adur < apos) {
+			if (Date.now() - mp.au.pt0 < 500)
+				return;
+
+			pctx.fillStyle = light ? 'rgba(0,0,0,0.5)' : 'rgba(255,255,255,0.5)';
+			var m = /[?&]th=(opus|caf|mp3)/.exec('' + mp.au.rsrc),
+				txt = mp.au.ded ? L.mm_playerr.replace(':', ' ;_;') :
+					m ? L.mm_bconv.format(m[1]) : L.mm_bload;
+
+			pctx.fillText(txt, 16, pc.h / 1.5);
 			return;  // not-init || unsupp-codec
+		}
 
 		if (bau != mp.au)
 			r.drawbuf();
@@ -3145,7 +3173,9 @@ function play(tid, is_ev, seek) {
 		mpl.unbuffer(url);
 	}, 500);
 
+	mp.au.ded = 0;
 	mp.au.tid = tid;
+	mp.au.pt0 = Date.now();
 	mp.au.evp = get_evpath();
 	mp.au.volume = mp.expvol(mp.vol);
 	var trs = QSA('#files tr.play');
@@ -3213,6 +3243,8 @@ function evau_error(e) {
 	var err = '',
 		eplaya = (e && e.target) || (window.event && window.event.srcElement);
 
+	eplaya.ded = 1;
+
 	switch (eplaya.error.code) {
 		case eplaya.error.MEDIA_ERR_ABORTED:
 			err = L.mm_eabrt;
@@ -3327,6 +3359,7 @@ function scan_hash(v) {
 
 
 function eval_hash() {
+	document.onkeydown = ahotkeys;
 	window.onpopstate = treectl.onpopfun;
 
 	if (hash0 && window.og_fn) {
@@ -4403,7 +4436,7 @@ var showfile = (function () {
 
 			var td = ebi(link.id).closest('tr').getElementsByTagName('td')[0];
 
-			if (lang == 'md' && td.textContent != '-')
+			if (lang == 'ts' || (lang == 'md' && td.textContent != '-'))
 				continue;
 
 			td.innerHTML = '<a href="#" id="t' +
@@ -4670,6 +4703,7 @@ var thegrid = (function () {
 	gfiles.style.display = 'none';
 	gfiles.innerHTML = (
 		'<div id="ghead" class="ghead">' +
+		'<a href="#" class="tgl btn" id="gridvau" tt="' + L.gt_vau + '</a> ' +
 		'<a href="#" class="tgl btn" id="gridsel" tt="' + L.gt_msel + '</a> ' +
 		'<a href="#" class="tgl btn" id="gridcrop" tt="' + L.gt_crop + '</a> ' +
 		'<a href="#" class="tgl btn" id="grid3x" tt="' + L.gt_3x + '</a> ' +
@@ -4831,7 +4865,7 @@ var thegrid = (function () {
 		else if (oth.hasAttribute('download'))
 			oth.click();
 
-		else if (widget.is_open && aplay)
+		else if (aplay && (r.vau || !is_img))
 			aplay.click();
 
 		else if (is_dir && !have_sel)
@@ -5122,6 +5156,7 @@ var thegrid = (function () {
 
 	bcfg_bind(r, 'thumbs', 'thumbs', true, r.setdirty);
 	bcfg_bind(r, 'ihop', 'ihop', true);
+	bcfg_bind(r, 'vau', 'gridvau', false);
 	bcfg_bind(r, 'crop', 'gridcrop', !dcrop.endsWith('n'), r.set_crop);
 	bcfg_bind(r, 'x3', 'grid3x', dth3x.endsWith('y'), r.set_x3);
 	bcfg_bind(r, 'sel', 'gridsel', false, r.loadsel);
@@ -5221,7 +5256,9 @@ function tree_up(justgo) {
 		if (!justgo)
 			return;
 	}
-	act.parentNode.parentNode.parentNode.getElementsByTagName('a')[1].click();
+	var a = act.parentNode.parentNode.parentNode.getElementsByTagName('a')[1];
+	if (a.parentNode.tagName == 'LI')
+		a.click();
 }
 
 
@@ -5284,7 +5321,7 @@ function fselfunw(e, ae, d, rem) {
 	}
 	selfun();
 }
-document.onkeydown = function (e) {
+var ahotkeys = function (e) {
 	if (e.altKey || e.isComposing)
 		return;
 
@@ -7992,7 +8029,8 @@ function sandbox(tgt, rules, cls, html) {
 			env = js.split(/\blogues *=/)[0] + 'a;';
 	}
 
-	html = '<html class="iframe ' + document.documentElement.className + '"><head><style>' + globalcss() +
+	html = '<html class="iframe ' + document.documentElement.className +
+		'"><head><style>html{background:#eee;color:#000}\n' + globalcss() +
 		'</style><base target="_parent"></head><body id="b" class="logue ' + cls + '">' + html +
 		'<script>' + env + '</script>' + sandboxjs() +
 		'<script>var d=document.documentElement,TS="' + TS + '",' +

copyparty/web/browser2.html

@@ -11,7 +11,6 @@
 		td{border:1px solid #999;border-width:1px 1px 0 0;padding:0 5px}
 		a{display:block}
 	</style>
-{{ html_head }}
 </head>
 
 <body>

copyparty/web/md.html

@@ -159,5 +159,8 @@ try { l.light = drk? 0:1; } catch (ex) { }
 	{%- if edit %}
 	<script src="{{ r }}/.cpr/md2.js?_={{ ts }}"></script>
 	{%- endif %}
+	{%- if js %}
+	<script src="{{ js }}_={{ ts }}"></script>
+	{%- endif %}
 </body></html>
 

copyparty/web/mde.html

@@ -53,5 +53,8 @@ try { l.light = drk? 0:1; } catch (ex) { }
 	<script src="{{ r }}/.cpr/deps/marked.js?_={{ ts }}"></script>
 	<script src="{{ r }}/.cpr/deps/easymde.js?_={{ ts }}"></script>
 	<script src="{{ r }}/.cpr/mde.js?_={{ ts }}"></script>
+	{%- if js %}
+	<script src="{{ js }}_={{ ts }}"></script>
+	{%- endif %}
 </body></html>
 

copyparty/web/msg.html

@@ -46,6 +46,9 @@
 		}, 1000);
 	</script>
 	{%- endif %}
+	{%- if js %}
+	<script src="{{ js }}_={{ ts }}"></script>
+	{%- endif %}
 </body>
 
 </html>

copyparty/web/splash.html

@@ -119,6 +119,9 @@ document.documentElement.className = (STG && STG.cpp_thm) || "{{ this.args.theme
 </script>
 <script src="{{ r }}/.cpr/util.js?_={{ ts }}"></script>
 <script src="{{ r }}/.cpr/splash.js?_={{ ts }}"></script>
+{%- if js %}
+<script src="{{ js }}_={{ ts }}"></script>
+{%- endif %}
 </body>
 </html>
 

copyparty/web/svcs.html

@@ -56,7 +56,7 @@
                 <li>running <code>rclone mount</code> as root? add <code>--allow-other</code></li>
                 <li>old version of rclone? replace all <code>=</code> with <code>&nbsp;</code> (space)</li>
             </ul>
-            
+
             <p>if you want to use the native WebDAV client in windows instead (slow and buggy), first run <a href="{{ r }}/.cpr/a/webdav-cfg.bat">webdav-cfg.bat</a> to remove the 47 MiB filesize limit (also fixes latency and password login), then connect:</p>
             <pre>
                 net use <b>w:</b> http{{ s }}://{{ ep }}/{{ rvp }}{% if accs %} k /user:<b>{{ pw }}</b>{% endif %}
@@ -64,16 +64,7 @@
         </div>
 
         <div class="os lin">
-            <pre>
-                yum install davfs2
-                {% if accs %}printf '%s\n' <b>{{ pw }}</b> k | {% endif %}mount -t davfs -ouid=1000 http{{ s }}://{{ ep }}/{{ rvp }} <b>mp</b>
-            </pre>
-            <p>make it automount on boot:</p>
-            <pre>
-                printf '%s\n' "http{{ s }}://{{ ep }}/{{ rvp }} <b>{{ pw }}</b> k" >> /etc/davfs2/secrets
-                printf '%s\n' "http{{ s }}://{{ ep }}/{{ rvp }} <b>mp</b> davfs rw,user,uid=1000,noauto 0 0" >> /etc/fstab
-            </pre>
-            <p>or you can use rclone instead, which is much slower but doesn't require root (plus it keeps lastmodified on upload):</p>
+            <p>rclone (v1.63 or later) is recommended:</p>
             <pre>
                 rclone config create {{ aname }}-dav webdav url=http{{ s }}://{{ rip }}{{ hport }} vendor=owncloud pacer_min_sleep=0.01ms{% if accs %} user=k pass=<b>{{ pw }}</b>{% endif %}
                 rclone mount --vfs-cache-mode writes --dir-cache-time 5s {{ aname }}-dav:{{ rvp }} <b>mp</b>
@@ -85,6 +76,16 @@
                 <li>running <code>rclone mount</code> as root? add <code>--allow-other</code></li>
                 <li>old version of rclone? replace all <code>=</code> with <code>&nbsp;</code> (space)</li>
             </ul>
+            <p>alternatively use davfs2 (requires root, is slower, forgets lastmodified-timestamp on upload):</p>
+            <pre>
+                yum install davfs2
+                {% if accs %}printf '%s\n' <b>{{ pw }}</b> k | {% endif %}mount -t davfs -ouid=1000 http{{ s }}://{{ ep }}/{{ rvp }} <b>mp</b>
+            </pre>
+            <p>make davfs2 automount on boot:</p>
+            <pre>
+                printf '%s\n' "http{{ s }}://{{ ep }}/{{ rvp }} <b>{{ pw }}</b> k" >> /etc/davfs2/secrets
+                printf '%s\n' "http{{ s }}://{{ ep }}/{{ rvp }} <b>mp</b> davfs rw,user,uid=1000,noauto 0 0" >> /etc/fstab
+            </pre>
             <p>or the emergency alternative (gnome/gui-only):</p>
             <!-- gnome-bug: ignores vp -->
             <pre>
@@ -104,7 +105,7 @@
             <pre>
                 http{{ s }}://k:<b>{{ pw }}</b>@{{ ep }}/{{ rvp }}
             </pre>
-            
+
             {% if s %}
             <p><em>replace <code>https</code> with <code>http</code> if it doesn't work</em></p>
             {% endif %}
@@ -244,6 +245,9 @@ document.documentElement.className = (STG && STG.cpp_thm) || "{{ args.theme }}";
 </script>
 <script src="{{ r }}/.cpr/util.js?_={{ ts }}"></script>
 <script src="{{ r }}/.cpr/svcs.js?_={{ ts }}"></script>
+{%- if js %}
+<script src="{{ js }}_={{ ts }}"></script>
+{%- endif %}
 </body>
 </html>
 

copyparty/web/ui.css

@@ -265,7 +265,11 @@ html.y #tth {
     box-shadow: 0 .3em 3em rgba(0,0,0,0.5);
 	max-width: 50em;
 	max-height: 30em;
-	overflow: auto;
+	overflow-x: auto;
+	overflow-y: scroll;
+}
+#modalc.yk {
+	overflow-y: auto;
 }
 #modalc td {
 	text-align: unset;

copyparty/web/up2k.js

@@ -658,7 +658,9 @@ function Donut(uc, st) {
     }
 
     function pos() {
-        return uc.fsearch ? Math.max(st.bytes.hashed, st.bytes.finished) : st.bytes.finished;
+        return uc.fsearch ?
+            Math.max(st.bytes.hashed, st.bytes.finished) :
+            st.bytes.inflight + st.bytes.finished;
     }
 
     r.on = function (ya) {
@@ -853,6 +855,7 @@ function up2k_init(subtle) {
     setmsg(suggest_up2k, 'msg');
 
     var parallel_uploads = ebi('nthread').value = icfg_get('nthread', u2j),
+        stitch_tgt = ebi('u2szg').value = icfg_get('u2sz', u2sz.split(',')[1]),
         uc = {},
         fdom_ctr = 0,
         biggest_file = 0;
@@ -1207,7 +1210,7 @@ function up2k_init(subtle) {
                     match = false;
 
             if (match) {
-                var msg = ['directory iterator got stuck on the following {0} items; good chance your browser is about to spinlock:<ul>'.format(missing.length)];
+                var msg = ['directory iterator got stuck trying to access the following {0} items; will skip:<ul>'.format(missing.length)];
                 for (var a = 0; a < Math.min(20, missing.length); a++)
                     msg.push('<li>' + esc(missing[a]) + '</li>');
 
@@ -1736,6 +1739,11 @@ function up2k_init(subtle) {
                     }
                 }
 
+                if (st.bytes.inflight && (st.bytes.inflight < 0 || !st.busy.upload.length)) {
+                    console.log('insane inflight ' + st.bytes.inflight);
+                    st.bytes.inflight = 0;
+                }
+
                 var mou_ikkai = false;
 
                 if (st.busy.handshake.length &&
@@ -2178,7 +2186,7 @@ function up2k_init(subtle) {
         st.busy.head.push(t);
 
         var xhr = new XMLHttpRequest();
-        xhr.onerror = function () {
+        xhr.onerror = xhr.ontimeout = function () {
             console.log('head onerror, retrying', t.name, t);
             if (!toast.visible)
                 toast.warn(9.98, L.u_enethd + "\n\nfile: " + t.name, t);
@@ -2222,6 +2230,7 @@ function up2k_init(subtle) {
             try { orz(e); } catch (ex) { vis_exh(ex + '', 'up2k.js', '', '', ex); }
         };
 
+        xhr.timeout = 34000;
         xhr.open('HEAD', t.purl + uricom_enc(t.name), true);
         xhr.send();
     }
@@ -2247,7 +2256,7 @@ function up2k_init(subtle) {
             console.log("sending keepalive handshake", t.name, t);
 
         var xhr = new XMLHttpRequest();
-        xhr.onerror = function () {
+        xhr.onerror = xhr.ontimeout = function () {
             if (t.t_busied != me)  // t.done ok
                 return console.log('zombie handshake onerror', t.name, t);
 
@@ -2374,11 +2383,39 @@ function up2k_init(subtle) {
                     var arr = st.todo.upload,
                         sort = arr.length && arr[arr.length - 1].nfile > t.n;
 
-                    for (var a = 0; a < t.postlist.length; a++)
+                    if (!t.stitch_sz) {
+                        // keep all connections busy
+                        var bpc = (st.bytes.total - st.bytes.finished) / (parallel_uploads || 1),
+                            ocs = 1024 * 1024,
+                            stp = 1024 * 512,
+                            ccs = ocs;
+                        while (ccs < bpc) {
+                            ocs = ccs;
+                            ccs += stp; if (ccs < bpc) ocs = ccs;
+                            ccs += stp; stp *= 2;
+                        }
+                        ocs = Math.floor(ocs / 1024 / 1024);
+                        t.stitch_sz = Math.min(ocs, stitch_tgt);
+                    }
+
+                    for (var a = 0; a < t.postlist.length; a++) {
+                        var nparts = [], tbytes = 0, stitch = t.stitch_sz;
+                        if (t.nojoin && t.nojoin - t.postlist.length < 6)
+                            stitch = 1;
+
+                        --a;
+                        for (var b = 0; b < stitch; b++) {
+                            nparts.push(t.postlist[++a]);
+                            tbytes += chunksize;
+                            if (tbytes + chunksize > stitch * 1024 * 1024 || t.postlist[a + 1] - t.postlist[a] !== 1)
+                                break;
+                        }
                         arr.push({
                             'nfile': t.n,
-                            'npart': t.postlist[a]
+                            'nparts': nparts
                         });
+                    }
+                    t.nojoin = 0;
 
                     msg = null;
                     done = false;
@@ -2387,7 +2424,7 @@ function up2k_init(subtle) {
                         arr.sort(function (a, b) {
                             return a.nfile < b.nfile ? -1 :
                             /*  */ a.nfile > b.nfile ? 1 :
-                                    a.npart < b.npart ? -1 : 1;
+                            /*  */ a.nparts[0] < b.nparts[0] ? -1 : 1;
                         });
                 }
 
@@ -2493,6 +2530,7 @@ function up2k_init(subtle) {
 
         xhr.open('POST', t.purl, true);
         xhr.responseType = 'text';
+        xhr.timeout = 42000;
         xhr.send(JSON.stringify(req));
     }
 
@@ -2534,7 +2572,10 @@ function up2k_init(subtle) {
     function exec_upload() {
         var upt = st.todo.upload.shift(),
             t = st.files[upt.nfile],
-            npart = upt.npart,
+            nparts = upt.nparts,
+            pcar = nparts[0],
+            pcdr = nparts[nparts.length - 1],
+            snpart = pcar == pcdr ? pcar : ('' + pcar + '~' + pcdr),
             tries = 0;
 
         if (t.done)
@@ -2549,8 +2590,8 @@ function up2k_init(subtle) {
         pvis.seth(t.n, 1, "🚀 send");
 
         var chunksize = get_chunksize(t.size),
-            car = npart * chunksize,
-            cdr = car + chunksize;
+            car = pcar * chunksize,
+            cdr = (pcdr + 1) * chunksize;
 
         if (cdr >= t.size)
             cdr = t.size;
@@ -2560,14 +2601,19 @@ function up2k_init(subtle) {
             var txt = unpre((xhr.response && xhr.response.err) || xhr.responseText);
             if (txt.indexOf('upload blocked by x') + 1) {
                 apop(st.busy.upload, upt);
-                apop(t.postlist, npart);
+                for (var a = pcar; a <= pcdr; a++)
+                    apop(t.postlist, a);
                 pvis.seth(t.n, 1, "ERROR");
                 pvis.seth(t.n, 2, txt.split(/\n/)[0]);
                 pvis.move(t.n, 'ng');
                 return;
             }
             if (xhr.status == 200) {
-                pvis.prog(t, npart, cdr - car);
+                var bdone = cdr - car;
+                for (var a = pcar; a <= pcdr; a++) {
+                    pvis.prog(t, a, Math.min(bdone, chunksize));
+                    bdone -= chunksize;
+                }
                 st.bytes.finished += cdr - car;
                 st.bytes.uploaded += cdr - car;
                 t.bytes_uploaded += cdr - car;
@@ -2576,18 +2622,21 @@ function up2k_init(subtle) {
             }
             else if (txt.indexOf('already got that') + 1 ||
                 txt.indexOf('already being written') + 1) {
-                console.log("ignoring dupe-segment error", t.name, t);
+                t.nojoin = t.nojoin || t.postlist.length;
+                console.log("ignoring dupe-segment with backoff", t.nojoin, t.name, t);
+                if (!toast.visible && st.todo.upload.length < 4)
+                    toast.msg(10, L.u_cbusy);
             }
             else {
-                xhrchk(xhr, L.u_cuerr2.format(npart, Math.ceil(t.size / chunksize), t.name), "404, target folder not found (???)", "warn", t);
-
+                xhrchk(xhr, L.u_cuerr2.format(snpart, Math.ceil(t.size / chunksize), t.name), "404, target folder not found (???)", "warn", t);
                 chill(t);
             }
             orz2(xhr);
         }
         var orz2 = function (xhr) {
             apop(st.busy.upload, upt);
-            apop(t.postlist, npart);
+            for (var a = pcar; a <= pcdr; a++)
+                apop(t.postlist, a);
             if (!t.postlist.length) {
                 t.t_uploaded = Date.now();
                 pvis.seth(t.n, 1, 'verifying');
@@ -2601,28 +2650,48 @@ function up2k_init(subtle) {
                 btot = Math.floor(st.bytes.total / 1024 / 1024);
 
             xhr.upload.onprogress = function (xev) {
-                var nb = xev.loaded;
-                st.bytes.inflight += nb - xhr.bsent;
+                var nb = xev.loaded,
+                    db = nb - xhr.bsent;
+
+                if (!db)
+                    return;
+
+                st.bytes.inflight += db;
                 xhr.bsent = nb;
-                pvis.prog(t, npart, nb);
+                xhr.timeout = 64000 + Date.now() - xhr.t0;
+                pvis.prog(t, pcar, nb);
             };
             xhr.onload = function (xev) {
                 try { orz(xhr); } catch (ex) { vis_exh(ex + '', 'up2k.js', '', '', ex); }
             };
-            xhr.onerror = function (xev) {
+            xhr.onerror = xhr.ontimeout = function (xev) {
                 if (crashed)
                     return;
 
                 st.bytes.inflight -= (xhr.bsent || 0);
+                xhr.bsent = 0;
 
                 if (!toast.visible)
-                    toast.warn(9.98, L.u_cuerr.format(npart, Math.ceil(t.size / chunksize), t.name), t);
+                    toast.warn(9.98, L.u_cuerr.format(snpart, Math.ceil(t.size / chunksize), t.name), t);
 
+                t.nojoin = t.nojoin || t.postlist.length;  // maybe rproxy postsize limit
                 console.log('chunkpit onerror,', ++tries, t.name, t);
                 orz2(xhr);
             };
+
+            var chashes = [],
+                ctxt = t.hash[pcar],
+                plen = Math.floor(192 / nparts.length);
+
+            plen = plen > 9 ? 9 : plen < 2 ? 2 : plen;
+            for (var a = pcar + 1; a <= pcdr; a++)
+                chashes.push(t.hash[a].slice(0, plen));
+
+            if (chashes.length)
+                ctxt += ',' + plen + ',' + chashes.join('');
+
             xhr.open('POST', t.purl, true);
-            xhr.setRequestHeader("X-Up2k-Hash", t.hash[npart]);
+            xhr.setRequestHeader("X-Up2k-Hash", ctxt);
             xhr.setRequestHeader("X-Up2k-Wark", t.wark);
             xhr.setRequestHeader("X-Up2k-Stat", "{0}/{1}/{2}/{3} {4}/{5} {6}".format(
                 pvis.ctr.ok, pvis.ctr.ng, pvis.ctr.bz, pvis.ctr.q, btot, btot - bfin,
@@ -2632,6 +2701,8 @@ function up2k_init(subtle) {
                 xhr.overrideMimeType('Content-Type', 'application/octet-stream');
 
             xhr.bsent = 0;
+            xhr.t0 = Date.now();
+            xhr.timeout = 42000;
             xhr.responseType = 'text';
             xhr.send(t.fobj.slice(car, cdr));
         }
@@ -2732,13 +2803,34 @@ function up2k_init(subtle) {
         if (parallel_uploads > 16)
             parallel_uploads = 16;
 
-        if (parallel_uploads > 7)
+        if (parallel_uploads > 6)
             toast.warn(10, L.u_maxconn);
+        else if (toast.txt == L.u_maxconn)
+            toast.hide();
 
         obj.value = parallel_uploads;
         bumpthread({ "target": 1 });
     }
 
+    var read_u2sz = function () {
+        var el = ebi('u2szg'), n = parseInt(el.value), dv = u2sz.split(',');
+        stitch_tgt = n = (
+            isNaN(n) ? dv[1] :
+            n < dv[0] ? dv[0] :
+            n > dv[2] ? dv[2] : n
+        );
+        if (n == dv[1]) sdrop('u2sz'); else swrite('u2sz', n);
+        if (el.value != n) el.value = n;
+    };
+    ebi('u2szg').addEventListener('blur', read_u2sz);
+    ebi('u2szg').onkeydown = function (e) {
+        if (anymod(e)) return;
+        var n = e.code == 'ArrowUp' ? 1 : e.code == 'ArrowDown' ? -1 : 0;
+        if (!n) return;
+        this.value = parseInt(this.value) + n;
+        read_u2sz();
+    }
+
     function tgl_fsearch() {
         set_fsearch(!uc.fsearch);
     }

copyparty/web/util.js

@@ -127,13 +127,13 @@ if ((document.location + '').indexOf(',rej,') + 1)
 
 try {
     console.hist = [];
-    var CMAXHIST = 1000;
+    var CMAXHIST = MOBILE ? 9000 : 44000;
     var hook = function (t) {
         var orig = console[t].bind(console),
             cfun = function () {
                 console.hist.push(Date.now() + ' ' + t + ': ' + Array.from(arguments).join(', '));
                 if (console.hist.length > CMAXHIST)
-                    console.hist = console.hist.slice(CMAXHIST / 2);
+                    console.hist = console.hist.slice(CMAXHIST / 4);
 
                 orig.apply(console, arguments);
             };
@@ -1396,10 +1396,10 @@ var tt = (function () {
             o = ctr.querySelectorAll('*[tt]');
 
         for (var a = o.length - 1; a >= 0; a--) {
-            o[a].onfocus = _cshow;
-            o[a].onblur = _hide;
-            o[a].onmouseenter = _dshow;
-            o[a].onmouseleave = _hide;
+            o[a].addEventListener('focus', _cshow);
+            o[a].addEventListener('blur', _hide);
+            o[a].addEventListener('mouseenter', _dshow);
+            o[a].addEventListener('mouseleave', _hide);
         }
         r.hide();
     }
@@ -1536,6 +1536,7 @@ var modal = (function () {
     var r = {},
         q = [],
         o = null,
+        scrolling = null,
         cb_up = null,
         cb_ok = null,
         cb_ng = null,
@@ -1556,6 +1557,7 @@ var modal = (function () {
     r.nofocus = 0;
 
     r.show = function (html) {
+        tt.hide();
         o = mknod('div', 'modal');
         o.innerHTML = '<table><tr><td><div id="modalc">' + html + '</div></td></tr></table>';
         document.body.appendChild(o);
@@ -1579,6 +1581,7 @@ var modal = (function () {
 
         document.addEventListener('focus', onfocus);
         document.addEventListener('selectionchange', onselch);
+        timer.add(scrollchk, 1);
         timer.add(onfocus);
         if (cb_up)
             setTimeout(cb_up, 1);
@@ -1586,6 +1589,8 @@ var modal = (function () {
 
     r.hide = function () {
         timer.rm(onfocus);
+        timer.rm(scrollchk);
+        scrolling = null;
         try {
             ebi('modal-ok').removeEventListener('blur', onblur);
         }
@@ -1604,13 +1609,28 @@ var modal = (function () {
         r.hide();
         if (cb_ok)
             cb_ok(v);
-    }
+    };
     var ng = function (e) {
         ev(e);
         r.hide();
         if (cb_ng)
             cb_ng(null);
-    }
+    };
+
+    var scrollchk = function () {
+        if (scrolling === true)
+            return;
+
+        var o = ebi('modalc'),
+            vis = o.offsetHeight,
+            all = o.scrollHeight,
+            nsc = 8 + vis < all;
+
+        if (scrolling !== nsc)
+            clmod(o, 'yk', !nsc);
+
+        scrolling = nsc;
+    };
 
     var onselch = function () {
         try {

docs/changelog.md

@@ -1,3 +1,124 @@
+▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀  
+# 2024-0729-2028  `v1.13.6`  not that big
+
+## new features
+
+* up2k.js: set clientside timeouts on http connections during upload 85e54980
+  * some reverse-proxy setups could cause uploads to hang indefinitely by eating requests; should recover nicely now
+* audio-player shows statustext while loading 662541c6
+* [bsod theme](https://github.com/9001/copyparty/tree/hovudstraum/contrib/themes) [(live demo)](https://cd.ocv.me/c/) 15ddcf53
+
+## bugfixes
+
+* fix bugs in the [long-distance upload optimizations](https://github.com/9001/copyparty/releases/tag/v1.13.5) in the previous version:
+  * up2k.js didn't necessarily use the expected chunksize when stitching 225bd80e
+  * u2c (commandline uploader): 8916bce3
+    * use the correct chunksize instead of overshooting like crazy
+    * could crash on exit if `-z` was enabled (so basically harmless)
+    * the "time spent uploading" statustext that was printed on exit could multiply by `-j` and exceed walltime
+* misc ux 9bb6e0dc
+  * don't accept hotkeys until it's safe to do so
+  * improve messages regarding the [firefox crash](https://bugzilla.mozilla.org/show_bug.cgi?id=1790500)
+  * keep more console logs in memory (easier to debug)
+  * fix wordwrap in messageboxes on firefox a19a0fa9
+
+## other changes
+
+* changed the `xm` / "on message" [hook examples](https://github.com/9001/copyparty/tree/hovudstraum/bin/hooks#on-message) to reject users without write-access 99edba4f
+* docker images were rebuilt on 2024-08-02, 23:30 UTC with new optimizations: 98ffaadf
+  * 😃 RAM usage decreased by `5-6 MiB` for most flavors; `10 MiB` for dj/iv
+  * 😕 image size grew by `4 MiB` (min), `6 MiB` (ac/im/iv), `9 MiB` (dj)
+  * 😃 startup time reduced to about half
+  * and avoids a deadlock on IBM mainframes
+* updated comparison to other software 6b54972e
+  * `hfs2` is dead, `hfs3` and `filebrowser` improved
+
+
+
+▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀  
+# 2024-0722-2323  `v1.13.5`  american sized
+
+## new features
+
+* long-distance uploads are now **twice as fast** on average 132a8350
+  * boost tcp windowsize scaling by stitching together smaller chunks into bigger chonks so they fly better across the atlantic
+  * i'm not kidding, on the two routes we've tested this on we gained 1.6x / 160% (from US-West to Finland) and **2.6x / 260%** (Norway to US-East)
+    * files that are between 4 MiB and 256 MiB see the biggest improvement; 70% faster <= 768 MiB, 40% <= 1.5 GiB, 10% <= 6G
+  * if this turns out to be buggy, disable it serverside with `--u2sz 1,1,1` or clientside in the browser-ui: `[⚙️]` -> `up2k switches` -> change `64` to `1`
+* u2c.py (CLI uploader): support stitching (☝️) + print a summary with hashing and upload speeds  987bce21
+* video files can play as audio 53f1e3c9
+  * audio is extracted serverside to avoid wasting bandwidth
+  * extraction is lossy (converted to opus or mp3 depending on browser)
+  * togglebutton `🎧` in the gridview toolbar to enable/disable
+* new hook: [into-the-cache-it-goes.py](https://github.com/9001/copyparty/tree/hovudstraum/bin/hooks#after-upload) d26a944d
+  * avoids a cloudflare bug (race condition?) where it will send truncated files to visitors on the very first load if several people simultaneously access a file that hasn't been viewed before
+
+## bugfixes
+
+* inline markdown/logues rendered black-on-black in firefox 54 and some other browsers from 2017 and older eeef8091
+* unintuitive folder thumbnail selection if folder contains both `Cover.jpg` and `cover.jpg` f955d2bd
+* the gridview toolbar got undocked after viewing a pic/vid dc449bf8
+
+## other changes
+
+* #90 recommend rclone in favor of davfs2 ef0ecf87
+* improved some error messages e565ad5f
+* added helptext exporters to generate the online [html](https://ocv.me/copyparty/helptext.html) and [txt](https://ocv.me/copyparty/helptext.txt) editions 59533990
+* mention that cloudflare is incompatible with uploading files larger than 383.9 GiB 
+
+
+
+▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀  
+# 2024-0716-0457  `v1.13.4`  descript.ion
+
+## new features
+
+* "medialinks"; instead of the usual hotlink, the basic-uploader (as used by sharex and such) can return a link that opens the file in the media viewer c9281f89
+  * enable for all uploads with volflag `medialinks`, or just for one upload by adding `?media` to the post url
+* thumbnails are now fully compatible with dirkeys/filekeys 52e06226
+* `--th-covers` will respect filename order, selecting the first matching filename as the folder thumbnail 1cdb1702
+* new hook: [bittorrent downloader](https://github.com/9001/copyparty/tree/hovudstraum/bin/hooks#on-message) bd3b3863 803e1565
+* hooks: d749683d
+  * can be restricted to only run when user has specific permissions
+  * user permissions are also included in the json message to the hook
+  * new syntax to prepend args to the hook's command
+  * (all this will be better documented after some additional upcoming hook-related features, see `--help-hooks` for now)
+* support `descript.ion` usenet metadata; will parse and render into directory listings when possible 927c3bce
+  * directory listings are now 2% slower, eh who's keeping count anyways
+* tftp-server: 45259251
+  * improved support for buggy clients
+  * improved ipv6 support, especially on macos
+  * improved robustness on unreliable networks
+* #85 new option `--gsel` to default-enable the client setting to select files by ctrl-clicking them in the grid 9a87ee2f
+* music player: set audio volume by scrollwheel 36d6d29a
+
+## bugfixes
+
+* race-the-beam (downloading an unfinished upload) could get interrupted near the end, requiring a manual resume in the browser's download manager to finish f37187a0
+* ftp-server: when accessing the root folder of servers without a root folder, it could mention inaccessible folders 84e8e1dd
+* ftp-server: uploads will automatically replace existing files if user has delete perms 0a9f4c60
+  * windows 2000 expects this behavior, otherwise it'll freak out and delete stuff and then not actually upload it, nice
+  * new option `--ftp-no-ow` restores old default behavior of rejecting upload if target filename exists
+* music player:
+  * stop trying to recover from a corrupted file if the user already fixed it manually 55a011b9
+  * support downloading the currently playing song regardless of current folder c06aa683
+* music player preloader: db6059e1
+  * stop searching after 5 folders of nothing
+  * don't crash playback by walking into error-pages
+* `--og` (rich discord embeds) was incompatible with viewing markdown docs d75a2c77
+* `--cgen` (configfile generator) much less jank d5de3f2f
+
+## other changes
+
+* mention that HTTP/2 is still usually slower than HTTP/1.1 dfe7f1d9
+* give up much sooner if a client is supposed to send a request body but isn't c549f367
+* support running copyparty as a server on windows 2000 and winXP 8c73e0cb 2fd12a83
+* updated deps 6e58514b
+  * copyparty.exe: python 3.12, pillow 10.4, pyinstaller 6.9
+  * dompurify 3.1.6
+
+
+
 ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀  
 # 2024-0601-2324  `v1.13.3`  700+
 

docs/devnotes.md

@@ -12,6 +12,8 @@
     * [write](#write)
     * [admin](#admin)
     * [general](#general)
+* [event hooks](#event-hooks) - on writing your own [hooks](../README.md#event-hooks)
+    * [hook effects](#hook-effects) - hooks can cause intentional side-effects
 * [assumptions](#assumptions)
     * [mdns](#mdns)
 * [sfx repack](#sfx-repack) - reduce the size of an sfx by removing features
@@ -55,8 +57,8 @@ quick outline of the up2k protocol,  see [uploading](https://github.com/9001/cop
 * server creates the `wark`, an identifier for this upload
   * `sha512( salt + filesize + chunk_hashes )`
   * and a sparse file is created for the chunks to drop into
-* client uploads each chunk
-  * header entries for the chunk-hash and wark
+* client sends a series of POSTs, with one or more consecutive chunks in each
+  * header entries for the chunk-hashes (comma-separated) and wark
   * server writes chunks into place based on the hash
 * client does another handshake with the hashlist; server replies with OK or a list of chunks to reupload
 
@@ -204,6 +206,32 @@ upload modifiers:
 | GET | `?pw=x` | logout |
 
 
+# event hooks
+
+on writing your own [hooks](../README.md#event-hooks)
+
+## hook effects
+
+hooks can cause intentional side-effects,  such as redirecting an upload into another location, or creating+indexing additional files, or deleting existing files, by returning json on stdout
+
+* `reloc` can redirect uploads before/after uploading has finished, based on filename, extension, file contents, uploader ip/name etc.
+* `idx` informs copyparty about a new file to index as a consequence of this upload
+* `del` tells copyparty to delete an unrelated file by vpath
+
+for these to take effect, the hook must be defined with the `c1` flag; see example [reloc-by-ext](https://github.com/9001/copyparty/blob/hovudstraum/bin/hooks/reloc-by-ext.py)
+
+a subset of effect types are available for a subset of hook types,
+
+* most hook types (xbu/xau/xbr/xar/xbd/xad/xm) support `idx` and `del` for all http protocols (up2k / basic-uploader / webdav), but not ftp/tftp/smb
+* most hook types will abort/reject the action if the hook returns nonzero, assuming flag `c` is given, see examples [reject-extension](https://github.com/9001/copyparty/blob/hovudstraum/bin/hooks/reject-extension.py) and [reject-mimetype](https://github.com/9001/copyparty/blob/hovudstraum/bin/hooks/reject-mimetype.py)
+* `xbu` supports `reloc` for all http protocols (up2k / basic-uploader / webdav), but not ftp/tftp/smb
+* `xau` supports `reloc` for basic-uploader / webdav only, not up2k or ftp/tftp/smb
+  * so clients like sharex are supported, but not dragdrop into browser
+
+to trigger indexing of files `/foo/1.txt` and `/foo/bar/2.txt`, a hook can `print(json.dumps({"idx":{"vp":["/foo/1.txt","/foo/bar/2.txt"]}}))` (and replace "idx" with "del" to delete instead)
+* note: paths starting with `/` are absolute URLs, but you can also do `../3.txt` relative to the destination folder of each uploaded file
+
+
 # assumptions
 
 ## mdns
@@ -327,10 +355,6 @@ can be reproduced with `--no-sendfile --s-wr-sz 8192 --s-wr-slp 0.3 --rsp-slp 6`
   * remove brokers / multiprocessing stuff; https://github.com/9001/copyparty/tree/no-broker
   * reduce the nesting / indirections in `HttpCli` / `httpcli.py`
     * nearly zero benefit from stuff like replacing all the `self.conn.hsrv` with a local `hsrv` variable
-* reduce up2k roundtrips
-  * start from a chunk index and just go
-  * terminate client on bad data
-    * not worth the effort, just throw enough conncetions at it
 * single sha512 across all up2k chunks?
   * crypto.subtle cannot into streaming, would have to use hashwasm, expensive
 * separate sqlite table per tag

docs/examples/docker/portainer.md

@@ -0,0 +1,45 @@
+the following setup appears to work (copyparty starts, accepts uploads, is able to persist config)
+
+tested on debian 12 using [portainer-ce](https://docs.portainer.io/start/install-ce/server/docker/linux) with [docker-ce](https://docs.docker.com/engine/install/debian/) as root (not rootless)
+
+before making the container, first `mkdir /etc/copyparty /srv/pub` which will be bind-mounts into the container
+
+> both `/etc/copyparty` and `/srv/pub` are examples; you can change them if you'd like
+
+put your copyparty config files directly into `/etc/copyparty` and the files to share inside `/srv/pub`
+
+on first startup, copyparty will create a subfolder inside `/etc/copyparty` called `copyparty` where it puts some runtime state; for example replacing `/etc/copyparty/copyparty/cert.pem` with another TLS certificate is a quick and dirty way to get valid HTTPS (if you really want copyparty to handle that and not a reverse-proxy)
+
+
+## in portainer:
+
+```
+environments -> local -> containers -> add container:
+
+       name = copyparty-ac
+   registry = docker hub
+      image = copyparty/ac
+always pull = no
+
+manual network port publishing:
+  3923 to 3923 [TCP]
+
+advanced -> command & logging:
+  console = interactive & tty
+
+advanced -> volumes -> map additional volume:
+  container = /cfg  [Bind]
+  host = /etc/copyparty  [Writable]
+
+advanced -> volumes -> map additional volume:
+  container = /w  [Bind]
+  host = /srv/pub  [Writable]
+```
+
+notes:
+
+* `/cfg` is where copyparty expects to find its config files; `/etc/copyparty` is just an example mapping to that
+
+* `/w` is where copyparty expects to find the folder to share; `/srv/pub` is just an example mapping to that
+
+* the volumes must be bind-mounts to avoid permission issues (or so the theory goes)

docs/versus.md

@@ -20,6 +20,7 @@ currently up to date with [awesome-selfhosted](https://github.com/awesome-selfho
   * 💾 = what copyparty offers as an alternative
 * 🔵 = similarities
 * ⚠️ = disadvantages (something copyparty does "better")
+* 🔥 = hazards
 
 
 ## toc
@@ -37,7 +38,7 @@ currently up to date with [awesome-selfhosted](https://github.com/awesome-selfho
     * [another matrix](#another-matrix)
 * [reviews](#reviews)
     * [copyparty](#copyparty)
-    * [hfs2](#hfs2)
+    * [hfs2](#hfs2) 🔥
     * [hfs3](#hfs3)
     * [nextcloud](#nextcloud)
     * [seafile](#seafile)
@@ -83,8 +84,8 @@ the table headers in the matrixes below are the different softwares, with a quic
 
 the softwares,
 * `a` = [copyparty](https://github.com/9001/copyparty)
-* `b` = [hfs2](https://rejetto.com/hfs/)
-* `c` = [hfs3](https://github.com/rejetto/hfs)
+* `b` = [hfs2](https://github.com/rejetto/hfs2/) 🔥
+* `c` = [hfs3](https://rejetto.com/hfs/)
 * `d` = [nextcloud](https://github.com/nextcloud/server)
 * `e` = [seafile](https://github.com/haiwen/seafile)
 * `f` = [rclone](https://github.com/rclone/rclone), specifically `rclone serve webdav .`
@@ -152,19 +153,20 @@ symbol legend,
 | feature / software      | a | b | c | d | e | f | g | h | i | j | k | l | m |
 | ----------------------- | - | - | - | - | - | - | - | - | - | - | - | - | - |
 | download folder as zip  | █ | █ | █ | █ | ╱ |   | █ |   | █ | █ | ╱ | █ | ╱ |
-| download folder as tar  | █ |   |   |   |   |   |   |   |   | █ |   |   |   |
-| upload                  | █ | █ | █ | █ | █ | █ | █ | █ | █ | █ | █ | █ | █ |
-| parallel uploads        | █ |   |   | █ | █ |   | • |   | █ |   | █ |   | █ |
-| resumable uploads       | █ |   |   |   |   |   |   |   | █ |   | █ | ╱ |   |
-| upload segmenting       | █ |   |   |   |   |   |   | █ | █ |   | █ | ╱ | █ |
+| download folder as tar  | █ |   |   |   |   |   |   |   |   |   |   |   |   |
+| upload                  | █ | █ | █ | █ | █ | █ | █ | █ | █ | █ | ╱ | █ | █ |
+| parallel uploads        | █ |   |   | █ | █ |   | • |   | █ | █ | █ |   | █ |
+| resumable uploads       | █ |   | █ |   |   |   |   |   | █ | █ | █ | ╱ |   |
+| upload segmenting       | █ |   |   |   |   |   |   | █ | █ | █ | █ | ╱ | █ |
 | upload acceleration     | █ |   |   |   |   |   |   |   | █ |   | █ |   |   |
 | upload verification     | █ |   |   | █ | █ |   |   |   | █ |   |   |   |   |
 | upload deduplication    | █ |   |   |   | █ |   |   |   | █ |   |   |   |   |
 | upload a 999 TiB file   | █ |   |   |   | █ | █ | • |   | █ |   | █ | ╱ | ╱ |
-| race the beam ("p2p")   | █ |   |   |   |   |   |   |   |   | • |   |   |   |
+| CTRL-V from device      | █ |   |   | █ |   |   |   |   |   |   |   |   |   |
+| race the beam ("p2p")   | █ |   |   |   |   |   |   |   |   |   |   |   |   |
 | keep last-modified time | █ |   |   | █ | █ | █ |   |   |   |   |   | █ |   |
 | upload rules            | ╱ | ╱ | ╱ | ╱ | ╱ |   |   | ╱ | ╱ |   | ╱ | ╱ | ╱ |
-| ┗ max disk usage        | █ | █ |   |   | █ |   |   |   | █ |   |   | █ | █ |
+| ┗ max disk usage        | █ | █ | █ |   | █ |   |   |   | █ |   |   | █ | █ |
 | ┗ max filesize          | █ |   |   |   |   |   |   | █ |   |   | █ | █ | █ |
 | ┗ max items in folder   | █ |   |   |   |   |   |   |   |   |   |   | ╱ |   |
 | ┗ max file age          | █ |   |   |   |   |   |   |   | █ |   |   |   |   |
@@ -173,6 +175,7 @@ symbol legend,
 | ┗ randomize filename    | █ |   |   |   |   |   |   | █ | █ |   |   |   |   |
 | ┗ mimetype reject-list  | ╱ |   |   |   |   |   |   |   | • | ╱ |   | ╱ | • |
 | ┗ extension reject-list | ╱ |   |   |   |   |   |   | █ | • | ╱ |   | ╱ | • |
+| ┗ upload routing        | █ |   |   |   |   |   |   |   |   |   |   |   |   |
 | checksums provided      |   |   |   | █ | █ |   |   |   | █ | ╱ |   |   |   |
 | cloud storage backend   | ╱ | ╱ | ╱ | █ | █ | █ | ╱ |   |   | ╱ | █ | █ | ╱ |
 
@@ -182,8 +185,13 @@ symbol legend,
 
 * `upload verification` = uploads are checksummed or otherwise confirmed to have been transferred correctly
 
+* `CTRL-V from device` = press CTRL-C in Windows Explorer (or whatever) and paste into the webbrowser to upload it
+
 * `race the beam` = files can be downloaded while they're still uploading; downloaders are slowed down such that the uploader is always ahead
 
+* `upload routing` = depending on filetype / contents / uploader etc., the file can be redirected to another location or otherwise transformed; mitigates limitations such as [sharex#3992](https://github.com/ShareX/ShareX/issues/3992)
+  * copyparty example: [reloc-by-ext](https://github.com/9001/copyparty/tree/hovudstraum/bin/hooks#before-upload)
+
 * `checksums provided` = when downloading a file from the server, the file's checksum is provided for verification client-side
 
 * `cloud storage backend` = able to serve files from (and write to) s3 or similar cloud services; `╱` means the software can do this with some help from `rclone mount` as a bridge
@@ -213,7 +221,7 @@ symbol legend,
 | serve sftp (ssh)        |   |   |   |   |   | █ |   |   |   |   |   | █ | █ |
 | serve smb/cifs          | ╱ |   |   |   |   | █ |   |   |   |   |   |   |   |
 | serve dlna              |   |   |   |   |   | █ |   |   |   |   |   |   |   |
-| listen on unix-socket   |   |   |   | █ | █ |   | █ | █ | █ |   | █ | █ |   |
+| listen on unix-socket   | █ |   |   | █ | █ |   | █ | █ | █ | █ | █ | █ |   |
 | zeroconf                | █ |   |   |   |   |   |   |   |   |   |   |   | █ |
 | supports netscape 4     | ╱ |   |   |   |   | █ |   |   |   |   | • |   | ╱ |
 | ...internet explorer 6  | ╱ | █ |   | █ |   | █ |   |   |   |   | • |   | ╱ |
@@ -243,7 +251,7 @@ symbol legend,
 | listen multiple ports   | █ |   |   |   |   |   |   |   |   |   |   | █ |   |
 | virtual file system     | █ | █ | █ |   |   |   | █ |   |   |   |   | █ |   |
 | reverse-proxy ok        | █ |   | █ | █ | █ | █ | █ | █ | • | • | • | █ | ╱ |
-| folder-rproxy ok        | █ |   |   |   | █ | █ |   | • | • | • | • |   | • |
+| folder-rproxy ok        | █ |   | █ |   | █ | █ |   | • | • | █ | • |   | • |
 
 * `folder-rproxy` = reverse-proxying without dedicating an entire (sub)domain, using a subfolder instead
 * `l`/sftpgo:
@@ -266,9 +274,9 @@ symbol legend,
 | per-folder permissions  | ╱ |   |   | █ | █ |   | █ |   | █ | █ | ╱ | █ | █ |
 | per-file permissions    |   |   |   | █ | █ |   | █ |   | █ |   |   |   | █ |
 | per-file passwords      | █ |   |   | █ | █ |   | █ |   | █ |   |   |   | █ |
-| unmap subfolders        | █ |   |   |   |   |   | █ |   |   | █ | ╱ | • |   |
+| unmap subfolders        | █ |   | █ |   |   |   | █ |   |   | █ | ╱ | • |   |
 | index.html blocks list  | ╱ |   |   |   |   |   | █ |   |   | • |   |   |   |
-| write-only folders      | █ |   |   |   |   |   |   |   |   |   | █ | █ |   |
+| write-only folders      | █ |   | █ |   |   |   |   |   |   |   | █ | █ |   |
 | files stored as-is      | █ | █ | █ | █ |   | █ | █ |   |   | █ | █ | █ | █ |
 | file versioning         |   |   |   | █ | █ |   |   |   |   |   |   |   |   |
 | file encryption         |   |   |   | █ | █ | █ |   |   |   |   |   | █ |   |
@@ -298,6 +306,7 @@ symbol legend,
 * `file action event hooks` = run script before/after upload, move, rename, ...
 * `one-way folder sync` = like rsync, optionally deleting unexpected files at target
 * `full sync` = stateful, dropbox-like sync
+* `speed throttle` = rate limiting (per ip, per user, per connection, anything like that)
 * `curl-friendly ls` = returns a [sortable plaintext folder listing](https://user-images.githubusercontent.com/241032/215322619-ea5fd606-3654-40ad-94ee-2bc058647bb2.png) when curled
 * `curl-friendly upload` = uploading with curl is just `curl -T some.bin http://.../`
 * `a`/copyparty remarks:
@@ -323,14 +332,14 @@ symbol legend,
 | feature / software      | a | b | c | d | e | f | g | h | i | j | k | l | m |
 | ----------------------  | - | - | - | - | - | - | - | - | - | - | - | - | - |
 | single-page app         | █ |   | █ | █ | █ |   |   | █ | █ | █ | █ |   | █ |
-| themes                  | █ | █ |   | █ |   |   |   |   | █ |   |   |   |   |
+| themes                  | █ | █ | █ | █ |   |   |   |   | █ |   |   |   |   |
 | directory tree nav      | █ | ╱ |   |   | █ |   |   |   | █ |   | ╱ |   |   |
 | multi-column sorting    | █ |   |   |   |   |   |   |   |   |   |   |   |   |
 | thumbnails              | █ |   |   | ╱ | ╱ |   |   | █ | █ | ╱ |   |   | █ |
 | ┗ image thumbnails      | █ |   |   | █ | █ |   |   | █ | █ | █ |   |   | █ |
 | ┗ video thumbnails      | █ |   |   | █ | █ |   |   |   | █ |   |   |   | █ |
 | ┗ audio spectrograms    | █ |   |   |   |   |   |   |   |   |   |   |   |   |
-| audio player            | █ |   |   | █ | █ |   |   |   | █ | ╱ |   |   | █ |
+| audio player            | █ |   | ╱ | █ | █ |   |   |   | █ | ╱ |   |   | █ |
 | ┗ gapless playback      | █ |   |   |   |   |   |   |   | • |   |   |   |   |
 | ┗ audio equalizer       | █ |   |   |   |   |   |   |   |   |   |   |   |   |
 | ┗ waveform seekbar      | █ |   |   |   |   |   |   |   |   |   |   |   |   |
@@ -348,16 +357,16 @@ symbol legend,
 | search by custom parser | █ |   |   |   |   |   |   |   |   |   |   |   |   |
 | find local file         | █ |   |   |   |   |   |   |   |   |   |   |   |   |
 | undo recent uploads     | █ |   |   |   |   |   |   |   |   |   |   |   |   |
-| create directories      | █ |   |   | █ | █ | ╱ | █ | █ | █ | █ | █ | █ | █ |
-| image viewer            | █ |   |   | █ | █ |   |   |   | █ | █ | █ |   | █ |
+| create directories      | █ |   | █ | █ | █ | ╱ | █ | █ | █ | █ | █ | █ | █ |
+| image viewer            | █ |   | █ | █ | █ |   |   |   | █ | █ | █ |   | █ |
 | markdown viewer         | █ |   |   |   | █ |   |   |   | █ | ╱ | ╱ |   | █ |
 | markdown editor         | █ |   |   |   | █ |   |   |   | █ | ╱ | ╱ |   | █ |
 | readme.md in listing    | █ |   |   | █ |   |   |   |   |   |   |   |   |   |
 | rename files            | █ | █ | █ | █ | █ | ╱ | █ |   | █ | █ | █ | █ | █ |
 | batch rename            | █ |   |   |   |   |   |   |   | █ |   |   |   |   |
 | cut / paste files       | █ | █ |   | █ | █ |   |   |   | █ |   |   |   | █ |
-| move files              | █ | █ |   | █ | █ |   | █ |   | █ | █ | █ |   | █ |
-| delete files            | █ | █ |   | █ | █ | ╱ | █ | █ | █ | █ | █ | █ | █ |
+| move files              | █ | █ | █ | █ | █ |   | █ |   | █ | █ | █ |   | █ |
+| delete files            | █ | █ | █ | █ | █ | ╱ | █ | █ | █ | █ | █ | █ | █ |
 | copy files              |   |   |   |   | █ |   |   |   | █ | █ | █ |   | █ |
 
 * `single-page app` = multitasking; possible to continue navigating while uploading
@@ -367,8 +376,12 @@ symbol legend,
 * `undo recent uploads` = accounts without delete permissions have a time window where they can undo their own uploads
 * `a`/copyparty has teeny-tiny skips playing gapless albums depending on audio codec (opus best)
 * `b`/hfs2 has a very basic directory tree view, not showing sibling folders
+* `c`/hfs3 remarks:
+  * audio playback does not continue into next song
 * `f`/rclone can do some file management (mkdir, rename, delete) when hosting througn webdav
-* `j`/filebrowser has a plaintext viewer/editor
+* `j`/filebrowser remarks:
+  * audio playback does not continue into next song
+  * plaintext viewer/editor
 * `k`/filegator directory tree is a modal window
 
 
@@ -424,6 +437,7 @@ symbol legend,
   * 💾 are what copyparty offers as an alternative
 * 🔵 are similarities
 * ⚠️ are disadvantages (something copyparty does "better")
+* 🔥 are hazards
 
 ## [copyparty](https://github.com/9001/copyparty)
 * resumable uploads which are verified server-side
@@ -431,8 +445,9 @@ symbol legend,
   * both of the above are surprisingly uncommon features
 * very cross-platform (python, no dependencies)
 
-## [hfs2](https://rejetto.com/hfs/)
-* the OG, the legend
+## [hfs2](https://github.com/rejetto/hfs2/)
+* the OG, the legend (now replaced by [hfs3](#hfs3))
+* 🔥 hfs2 is dead and dangerous! unfixed RCE: [info](https://github.com/rejetto/hfs2/issues/44), [info](https://github.com/drapid/hfs/issues/3), [info](https://asec.ahnlab.com/en/67650/)
 * ⚠️ uploads not resumable / accelerated / integrity-checked
   * ⚠️ on cloudflare: max upload size 100 MiB
 * ⚠️ windows-only
@@ -440,10 +455,19 @@ symbol legend,
 * vfs with gui config, per-volume permissions
 * starting to show its age, hence the rewrite:
 
-## [hfs3](https://github.com/rejetto/hfs)
+## [hfs3](https://rejetto.com/hfs/)
 * nodejs; cross-platform
 * vfs with gui config, per-volume permissions
-* still early development, let's revisit later
+* 🔵 uploads are resumable
+* ⚠️ uploads are not segmented; max upload size 100 MiB on cloudflare
+* ⚠️ uploads are not accelerated (copyparty is 3x faster across the atlantic)
+* ⚠️ uploads are not integrity-checked
+* ⚠️ copies the file after upload; need twice filesize free disk space
+* ⚠️ doesn't support crazy filenames
+* ✅ config GUI
+* ✅ download counter
+* ✅ watch active connections
+* ✅ plugins
 
 ## [nextcloud](https://github.com/nextcloud/server)
 * php, mariadb
@@ -497,6 +521,7 @@ symbol legend,
 * rust; cross-platform (windows, linux, macos)
 * ⚠️ uploads not resumable / accelerated / integrity-checked
   * ⚠️ on cloudflare: max upload size 100 MiB
+  * ⚠️ across the atlantic, copyparty is 3x faster
 * ⚠️ doesn't support crazy filenames
 * ✅ per-url access control (copyparty is per-volume)
 * 🔵 basic but really snappy ui
@@ -539,8 +564,10 @@ symbol legend,
 
 ## [filebrowser](https://github.com/filebrowser/filebrowser)
 * go; cross-platform (windows, linux, mac)
-* ⚠️ uploads not resumable / accelerated / integrity-checked
-  * ⚠️ on cloudflare: max upload size 100 MiB
+* 🔵 uploads are resumable and segmented
+* 🔵 multiple files are uploaded in parallel, but...
+  * ⚠️ big files are not accelerated (copyparty is 5x faster across the atlantic)
+* ⚠️ uploads are not integrity-checked
 * ⚠️ http only; no webdav / ftp / zeroconf
 * ⚠️ doesn't support crazy filenames
 * ⚠️ no directory tree nav
@@ -550,12 +577,14 @@ symbol legend,
   * ⚠️ but no directory tree for navigation
 * ✅ user signup
 * ✅ command runner / remote shell
-* 🔵 supposed to have write-only folders but couldn't get it to work
+* ✅ more efficient; can handle around twice as much simultaneous traffic
 
 ## [filegator](https://github.com/filegator/filegator)
-* go; cross-platform (windows, linux, mac)
+* php; cross-platform (windows, linux, mac)
 * 🔵 *it has upload segmenting and acceleration*
   * ⚠️ but uploads are still not integrity-checked
+  * ⚠️ on copyparty, uploads are 40x faster
+    * compared to the official filegator docker example which might be bad
 * ⚠️ http only; no webdav / ftp / zeroconf
 * ⚠️ does not support symlinks
 * ⚠️ expensive download-as-zip feature
@@ -566,6 +595,7 @@ symbol legend,
 * go; cross-platform (windows, linux, mac)
 * ⚠️ http uploads not resumable / accelerated / integrity-checked
   * ⚠️ on cloudflare: max upload size 100 MiB
+  * ⚠️ across the atlantic, copyparty is 2.5x faster
   * 🔵 sftp uploads are resumable
 * ⚠️ web UI is very minimal + a bit slow
   * ⚠️ no thumbnails / image viewer / audio player
@@ -573,6 +603,7 @@ symbol legend,
 * ⚠️ no filesystem indexing / search
 * ⚠️ doesn't run on phones, tablets
 * ⚠️ no zeroconf (mdns/ssdp)
+* ⚠️ impractical directory URLs
 * ⚠️ AGPL licensed
 * 🔵 ftp, ftps, webdav
 * ✅ sftp server
@@ -589,11 +620,13 @@ symbol legend,
 ## [arozos](https://github.com/tobychui/arozos)
 * big suite of applications similar to [kodbox](#kodbox), copyparty is better at downloading/uploading/music/indexing but arozos has other advantages
 * go; primarily linux (limited support for windows)
+* ⚠️ needs root
 * ⚠️ uploads not resumable / integrity-checked
 * ⚠️ uploading small files to copyparty is 2.7x faster
 * ⚠️ uploading large files to copyparty is at least 10% faster
   * arozos is websocket-based, 512 KiB chunks; writes each chunk to separate files and then merges
   * copyparty splices directly into the final file; faster and better for the HDD and filesystem
+* ⚠️ across the atlantic, uploading to copyparty is 6x faster
 * ⚠️ no directory tree navpane; not as easy to navigate
 * ⚠️ download-as-zip is not streaming; creates a temp.file on the server
 * ⚠️ not self-contained (pulls from jsdelivr)

scripts/docker/Dockerfile.ac

@@ -5,19 +5,16 @@ LABEL   org.opencontainers.image.url="https://github.com/9001/copyparty" \
         org.opencontainers.image.licenses="MIT" \
         org.opencontainers.image.title="copyparty-ac" \
         org.opencontainers.image.description="copyparty with Pillow and FFmpeg (image/audio/video thumbnails, audio transcoding, media tags)"
-ENV     PYTHONPYCACHEPREFIX=/tmp/pyc \
-        XDG_CONFIG_HOME=/cfg
+ENV     XDG_CONFIG_HOME=/cfg
 
 RUN     apk --no-cache add !pyc \
             wget \
-            py3-argon2-cffi py3-pillow \
-            ffmpeg \
-        && rm -rf /tmp/pyc \
-        && mkdir /cfg /w \
-        && chmod 777 /cfg /w \
-        && echo % /cfg > initcfg
+            py3-jinja2 py3-argon2-cffi py3-pillow \
+            ffmpeg
+
+COPY    i/dist/copyparty-sfx.py innvikler.sh ./
+RUN     ash innvikler.sh && rm innvikler.sh
 
-COPY    i/dist/copyparty-sfx.py ./
 WORKDIR /w
 EXPOSE  3923
-ENTRYPOINT ["python3", "/z/copyparty-sfx.py", "--no-crt", "-c", "/z/initcfg"]
+ENTRYPOINT ["python3", "-m", "copyparty", "--no-crt", "-c", "/z/initcfg"]

scripts/docker/Dockerfile.dj

@@ -5,15 +5,14 @@ LABEL   org.opencontainers.image.url="https://github.com/9001/copyparty" \
         org.opencontainers.image.licenses="MIT" \
         org.opencontainers.image.title="copyparty-dj" \
         org.opencontainers.image.description="copyparty with all optional dependencies, including musical key / bpm detection"
-ENV     PYTHONPYCACHEPREFIX=/tmp/pyc \
-        XDG_CONFIG_HOME=/cfg
+ENV     XDG_CONFIG_HOME=/cfg
 
 COPY    i/bin/mtag/install-deps.sh ./
 COPY    i/bin/mtag/audio-bpm.py /mtag/
 COPY    i/bin/mtag/audio-key.py /mtag/
 RUN     apk add -U !pyc \
             wget \
-            py3-argon2-cffi py3-pillow py3-pip py3-cffi \
+            py3-jinja2 py3-argon2-cffi py3-pillow py3-pip py3-cffi \
             ffmpeg \
             vips-jxl vips-heif vips-poppler vips-magick \
             py3-numpy fftw libsndfile \
@@ -27,18 +26,12 @@ RUN     apk add -U !pyc \
         && python3 -m pip install pyvips \
         && bash install-deps.sh \
         && apk del py3-pip .bd \
-        && rm -rf /var/cache/apk/* /tmp/pyc \
         && chmod 777 /root \
-        && ln -s /root/vamp /root/.local / \
-        && mkdir /cfg /w \
-        && chmod 777 /cfg /w \
-        && echo % /cfg > initcfg
+        && ln -s /root/vamp /root/.local /
+
+COPY    i/dist/copyparty-sfx.py innvikler.sh ./
+RUN     ash innvikler.sh && rm innvikler.sh
 
-COPY    i/dist/copyparty-sfx.py ./
 WORKDIR /w
 EXPOSE  3923
-ENTRYPOINT ["python3", "/z/copyparty-sfx.py", "--no-crt", "-c", "/z/initcfg"]
-
-# size: 286 MB
-# bpm/key: 529 sec
-# idx-bench: 2352 MB/s
+ENTRYPOINT ["python3", "-m", "copyparty", "--no-crt", "-c", "/z/initcfg"]

scripts/docker/Dockerfile.im

@@ -5,18 +5,15 @@ LABEL   org.opencontainers.image.url="https://github.com/9001/copyparty" \
         org.opencontainers.image.licenses="MIT" \
         org.opencontainers.image.title="copyparty-im" \
         org.opencontainers.image.description="copyparty with Pillow and Mutagen (image thumbnails, media tags)"
-ENV     PYTHONPYCACHEPREFIX=/tmp/pyc \
-        XDG_CONFIG_HOME=/cfg
+ENV     XDG_CONFIG_HOME=/cfg
 
 RUN     apk --no-cache add !pyc \
             wget \
-            py3-argon2-cffi py3-pillow py3-mutagen \
-        && rm -rf /tmp/pyc \
-        && mkdir /cfg /w \
-        && chmod 777 /cfg /w \
-        && echo % /cfg > initcfg
+            py3-jinja2 py3-argon2-cffi py3-pillow py3-mutagen
+
+COPY    i/dist/copyparty-sfx.py innvikler.sh ./
+RUN     ash innvikler.sh && rm innvikler.sh
 
-COPY    i/dist/copyparty-sfx.py ./
 WORKDIR /w
 EXPOSE  3923
-ENTRYPOINT ["python3", "/z/copyparty-sfx.py", "--no-crt", "-c", "/z/initcfg"]
+ENTRYPOINT ["python3", "-m", "copyparty", "--no-crt", "-c", "/z/initcfg"]

scripts/docker/Dockerfile.iv

@@ -5,12 +5,11 @@ LABEL   org.opencontainers.image.url="https://github.com/9001/copyparty" \
         org.opencontainers.image.licenses="MIT" \
         org.opencontainers.image.title="copyparty-iv" \
         org.opencontainers.image.description="copyparty with Pillow, FFmpeg, libvips (image/audio/video thumbnails, audio transcoding, media tags)"
-ENV     PYTHONPYCACHEPREFIX=/tmp/pyc \
-        XDG_CONFIG_HOME=/cfg
+ENV     XDG_CONFIG_HOME=/cfg
 
 RUN     apk add -U !pyc \
             wget \
-            py3-argon2-cffi py3-pillow py3-pip py3-cffi \
+            py3-jinja2 py3-argon2-cffi py3-pillow py3-pip py3-cffi \
             ffmpeg \
             vips-jxl vips-heif vips-poppler vips-magick \
         && apk add -t .bd \
@@ -18,13 +17,11 @@ RUN     apk add -U !pyc \
             python3-dev py3-wheel \
         && rm -f /usr/lib/python3*/EXTERNALLY-MANAGED \
         && python3 -m pip install pyvips \
-        && apk del py3-pip .bd \
-        && rm -rf /var/cache/apk/* /tmp/pyc \
-        && mkdir /cfg /w \
-        && chmod 777 /cfg /w \
-        && echo % /cfg > initcfg
+        && apk del py3-pip .bd
+
+COPY    i/dist/copyparty-sfx.py innvikler.sh ./
+RUN     ash innvikler.sh && rm innvikler.sh
 
-COPY    i/dist/copyparty-sfx.py ./
 WORKDIR /w
 EXPOSE  3923
-ENTRYPOINT ["python3", "/z/copyparty-sfx.py", "--no-crt", "-c", "/z/initcfg"]
+ENTRYPOINT ["python3", "-m", "copyparty", "--no-crt", "-c", "/z/initcfg"]

scripts/docker/Dockerfile.min

@@ -5,17 +5,14 @@ LABEL   org.opencontainers.image.url="https://github.com/9001/copyparty" \
         org.opencontainers.image.licenses="MIT" \
         org.opencontainers.image.title="copyparty-min" \
         org.opencontainers.image.description="just copyparty, no thumbnails / media tags / audio transcoding"
-ENV     PYTHONPYCACHEPREFIX=/tmp/pyc \
-        XDG_CONFIG_HOME=/cfg
+ENV     XDG_CONFIG_HOME=/cfg
 
 RUN     apk --no-cache add !pyc \
-            python3 \
-        && rm -rf /tmp/pyc \
-        && mkdir /cfg /w \
-        && chmod 777 /cfg /w \
-        && echo % /cfg > initcfg
+            py3-jinja2
+
+COPY    i/dist/copyparty-sfx.py innvikler.sh ./
+RUN     ash innvikler.sh && rm innvikler.sh
 
-COPY    i/dist/copyparty-sfx.py ./
 WORKDIR /w
 EXPOSE  3923
-ENTRYPOINT ["python3", "/z/copyparty-sfx.py", "--no-crt", "--no-thumb", "-c", "/z/initcfg"]
+ENTRYPOINT ["python3", "-m", "copyparty", "--no-crt", "--no-thumb", "-c", "/z/initcfg"]

scripts/docker/README.md

@@ -22,6 +22,11 @@ this example is also available as a podman-compatible [docker-compose yaml](http
 i'm not very familiar with containers, so let me know if this section could be better 🙏
 
 
+## portainer
+
+* there is a [portainer howto](https://github.com/9001/copyparty/blob/hovudstraum/docs/examples/docker/portainer.md) which is mostly untested
+
+
 ## configuration
 
 > this section basically explains how the [docker-compose yaml](https://github.com/9001/copyparty/blob/hovudstraum/docs/examples/docker/basic-docker-compose) works, so you may look there instead

scripts/docker/devnotes.md

@@ -17,3 +17,19 @@ but I don't really know what i'm doing here 💩
   `podman login docker.io`  
   `podman login ghcr.io -u 9001`  
   [about gchq](https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-container-registry) (takes a classic token as password)
+
+
+## building on alpine
+
+```bash
+apk add podman{,-docker}
+rc-update add cgroups
+service cgroups start
+vim /etc/containers/storage.conf  # driver = "btrfs"
+modprobe tun
+echo ed:100000:65536 >/etc/subuid
+echo ed:100000:65536 >/etc/subgid
+apk add qemu-openrc qemu-tools qemu-{arm,armeb,aarch64,s390x,ppc64le}
+rc-update add qemu-binfmt
+service qemu-binfmt start
+```

scripts/docker/innvikler.sh

@@ -0,0 +1,46 @@
+#!/bin/ash
+set -ex
+
+# cleanup for flavors with python build steps (dj/iv)
+rm -rf /var/cache/apk/* /root/.cache
+
+# initial config; common for all flavors
+mkdir /cfg /w
+chmod 777 /cfg /w
+echo % /cfg > initcfg
+
+# unpack sfx and dive in
+python3 copyparty-sfx.py --version
+cd /tmp/pe-copyparty.0
+
+# steal the stuff we need
+mv copyparty partftpy ftp/* /usr/lib/python3.*/site-packages/
+
+# golf
+cd /usr/lib/python3.*/
+rm -rf \
+  /tmp/pe-* /z/copyparty-sfx.py \
+  ensurepip pydoc_data turtle.py turtledemo lib2to3
+
+# drop bytecode
+find / -xdev -name __pycache__ -print0 | xargs -0 rm -rf
+
+# build the stuff we want
+python3 -m compileall -qj4 site-packages sqlite3 xml
+
+# drop the stuff we dont
+find -name __pycache__ |
+  grep -E 'ty/web/|/pycpar' |
+  tr '\n' '\0' | xargs -0 rm -rf
+
+# two-for-one:
+# 1) smoketest copyparty even starts
+# 2) build any bytecode we missed
+# this tends to race other builders (alle gode ting er tre)
+cd /z
+python3 -m copyparty \
+  --ign-ebind -p$((1024+RANDOM)),$((1024+RANDOM)),$((1024+RANDOM)) \
+  --no-crt -qi127.1 --exit=idx -e2dsa -e2ts
+
+# output from -e2d
+rm -rf .hist

scripts/docker/make.sh

@@ -6,6 +6,8 @@ set -e
     exit 1
 }
 
+suf=-b1
+suf=
 sarchs="386 amd64 arm/v7 arm64/v8 ppc64le s390x"
 archs="amd64 arm s390x 386 arm64 ppc64le"
 imgs="dj iv min im ac"
@@ -103,11 +105,12 @@ filt=
             # --pull=never does nothing at all btw
             (set -x
             $nice podman build \
+                --squash \
                 --pull=never \
                 --from localhost/alpine-$a \
-                -t copyparty-$i-$a \
+                -t copyparty-$i-$a$suf \
                 -f Dockerfile.$i . ||
-                    (echo $? $i-$a >> err)
+                    (echo $? $i-$a >> err; printf '%096d\n' $(seq 1 42))
             rm -f .blk
             ) 2> >(tee $a.err | sed "s/^/$aa:/" >&2) > >(tee $a.out | sed "s/^/$aa:/") &
         done
@@ -134,9 +137,10 @@ filt=
         variants=
         for a in $archs; do
             [[ " ${ngs[*]} " =~ " $i-$a " ]] && continue
-            variants="$variants containers-storage:localhost/copyparty-$i-$a"
+            variants="$variants containers-storage:localhost/copyparty-$i-$a$suf"
         done
-        podman manifest create copyparty-$i $variants
+        podman manifest rm copyparty-$i$suf || echo "(that's fine btw)"
+        podman manifest create copyparty-$i$suf $variants
     done
 }
 

scripts/help2html.py

@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+
+import re
+import subprocess as sp
+
+
+# to convert the copyparty --help to html, run this in xfce4-terminal @ 140x43:
+_ = r""""
+echo; for a in '' -accounts -flags -handlers -hooks -urlform -exp -ls -dbd -pwhash -zm; do
+./copyparty-sfx.py --help$a 2>/dev/null; printf '\n\n\n%0139d\n\n\n'; done  # xfce4-terminal @ 140x43
+"""
+# click [edit] => [select all]
+# click [edit] => [copy as html]
+# and then run this script
+
+
+def readclip():
+    cmds = [
+        "xsel -ob",
+        "xclip -selection CLIPBOARD -o",
+        "pbpaste",
+    ]
+    for cmd in cmds:
+        try:
+            return sp.check_output(cmd.split()).decode("utf-8")
+        except:
+            pass
+
+
+def cnv(src):
+    yield '<html style="background:#222;color:#fff"><body>'
+    skip_sfx = False
+    in_sfx = 0
+    in_salt = 0
+
+    while True:
+        ln = next(src)
+        if "<font" in ln:
+            if not ln.startswith("<pre>"):
+                ln = "<pre>" + ln
+            yield ln
+            break
+
+    for ln in src:
+        ln = ln.rstrip()
+        if re.search(r"^<font[^>]+>copyparty v[0-9]", ln):
+            in_sfx = 3
+        if in_sfx:
+            in_sfx -= 1
+            if not skip_sfx:
+                yield ln
+            continue
+        if '">uuid:' in ln:
+            ln = re.sub(r">uuid:[0-9a-f-]{36}<", ">autogenerated<", ln)
+        if "-salt SALT" in ln:
+            in_salt = 3
+        if in_salt:
+            in_salt -= 1
+            t = ln
+            ln = re.sub(r">[0-9a-zA-Z/+]{24}<", ">24-character-autogenerated<", ln)
+            ln = re.sub(r">[0-9a-zA-Z/+]{40}<", ">40-character-autogenerated<", ln)
+            if t != ln:
+                in_salt = 0
+        ln = ln.replace(">/home/ed/", ">~/")
+        if ln.startswith("0" * 20):
+            skip_sfx = True
+        yield ln
+
+    yield "</pre>eof</body></html>"
+
+
+def main():
+    src = readclip()
+    src = re.split("0{100,200}", src[::-1], 1)[1][::-1]
+    with open("helptext.html", "wb") as fo:
+        for ln in cnv(iter(src.split("\n")[:-3])):
+            fo.write(ln.encode("utf-8") + b"\r\n")
+
+
+if __name__ == "__main__":
+    main()

scripts/help2txt.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+set -e
+
+( xsel -ob | sed -r '
+s`/home/ed/`~/`;
+s/uuid:[0-9a-f-]{36}/autogenerated/;
+s/(-salt SALT.*default: )[0-9a-zA-Z/+]{24}\)/\124-character-autogenerated)/;
+s/(-salt SALT.*default: )[0-9a-zA-Z/+]{40}\)/\140-character-autogenerated)/;
+' | awk '
+/^copyparty/{a=1} !a{next}
+/^0{20}/{b=1} b&&/^copyparty v[0-9]+\./{s=3}
+s{s-=1;next} 1' |
+head -n-6; echo eof ) >helptext.txt
+exit 0
+
+
+# =====================================================================
+# end of script;  below is the explanation how to use this:
+
+
+# first open an infinitely wide console (this is why you own an ultrawide) and copypaste this into it:
+for a in '' -accounts -flags -handlers -hooks -urlform -exp -ls -dbd -pwhash -zm; do
+./copyparty-sfx.py --help$a 2>/dev/null; printf '\n\n\n%0255d\n\n\n'; done
+
+# then copypaste all of the output by pressing ctrl-shift-a, ctrl-shift-c
+# and finally actually run this script which should produce helptext.txt

scripts/make-sfx.sh

@@ -202,14 +202,14 @@ necho() {
 	mv {markupsafe,jinja2} j2/
 
 	necho collecting pyftpdlib
-	f="../build/pyftpdlib-1.5.9.tar.gz"
+	f="../build/pyftpdlib-1.5.10.tar.gz"
 	[ -e "$f" ] ||
-		(url=https://github.com/giampaolo/pyftpdlib/archive/refs/tags/release-1.5.9.tar.gz;
+		(url=https://files.pythonhosted.org/packages/cf/31/8d910cf40317dd0db74ba0b8558d0dee23c8b002468c14d3a5dec0e6e9fd/pyftpdlib-1.5.10.tar.gz;
 		wget -O$f "$url" || curl -L "$url" >$f)
 
 	tar -zxf $f
-	mv pyftpdlib-release-*/pyftpdlib .
-	rm -rf pyftpdlib-release-* pyftpdlib/test
+	mv pyftpdlib-*/pyftpdlib .
+	rm -rf pyftpdlib-* pyftpdlib/test
 	for f in pyftpdlib/_async{hat,ore}.py; do
 		[ -e "$f" ] || continue;
 		iawk 'NR<4||NR>27||!/^#/;NR==4{print"# license: https://opensource.org/licenses/ISC\n"}' $f
@@ -413,7 +413,7 @@ rm have
 
 ised /fork_process/d ftp/pyftpdlib/servers.py
 iawk '/^class _Base/{s=1}!s' ftp/pyftpdlib/authorizers.py
-iawk '/^ {0,4}[^ ]/{s=0}/^ {4}def (serve_forever|_loop)/{s=1}!s' ftp/pyftpdlib/servers.py
+iawk '/^ {0,4}[a-zA-Z]/{s=0}/^ {4}def (serve_forever|_loop)/{s=1}!s' ftp/pyftpdlib/servers.py
 rm -f ftp/pyftpdlib/{__main__,prefork}.py
 
 [ $no_ftp ] &&

scripts/pyinstaller/deps.sha512

@@ -36,4 +36,4 @@ d1420c8417fad7888766dd26b9706a87c63e8f33dceeb8e26d0056d5127b0b3ed9272e44b4b76113
 2be320b4191f208cdd6af183c77ba2cf460ea52164ee45ac3ff17d6dfa57acd9deff016636c2dd42a21f4f6af977d5f72df7dacf599bebcf41757272354d14c1  pillow-10.4.0-cp312-cp312-win_amd64.whl
 776378f5414efd26ec8a1cb3228a7b5fdf6afca3fa335a0e9b071266d55d9d9e66ee157c25a468a05bfa70ccd33c48b101998523fc6ff6bcf5e82a1d81ed0af8  pyinstaller-6.9.0-py3-none-win_amd64.whl
 c0af77d2a57cb063ab038dc986ed3582bc5acc8c8bd91d726101935d6388f50854ddbca26bc846ed5d1022cdee4d96242938c66f0ddc4565c36b60d691064db8  pyinstaller_hooks_contrib-2024.7-py2.py3-none-any.whl
-2f9a11ffae6d9f1ed76bf816f28812fcba71f87080b0c92e52bfccb46243118c5803a7e25dd78003ca7d66501bfcdce8ff7c691c63c0038b0d409ca3842dcc89  python-3.12.4-amd64.exe
+0572c6345f6a4f7f3e5c2ff858e3ca7ca54ae4478f3d59d8e18cb0f596e61dcf12aef579db229e83d63b30f15d6684ee6bb3feaea9413e5e636a503933057678  python-3.12.5-amd64.exe

scripts/pyinstaller/notes.txt

@@ -41,7 +41,7 @@ fns=(
   pillow-10.4.0-cp312-cp312-win_amd64.whl
   pyinstaller-6.9.0-py3-none-win_amd64.whl
   pyinstaller_hooks_contrib-2024.7-py2.py3-none-any.whl
-  python-3.12.4-amd64.exe
+  python-3.12.5-amd64.exe
 )
 [ $w7 ] && fns+=(  # u2c stuff
   certifi-2024.2.2-py3-none-any.whl

scripts/pyinstaller/up2k.sh

@@ -25,7 +25,7 @@ grep -E '^from .ssl_ import' $APPDATA/python/python37/site-packages/urllib3/util
     echo golfing
     echo > $APPDATA/python/python37/site-packages/requests/certs.py
     sed -ri 's/^(DEFAULT_CA_BUNDLE_PATH = ).*/\1""/' $APPDATA/python/python37/site-packages/requests/utils.py
-    sed -ri '/^import zipfile$/d' $APPDATA/python/python37/site-packages/requests/utils.py             
+    sed -ri '/^import zipfile$/d' $APPDATA/python/python37/site-packages/requests/utils.py
     sed -ri 's/"idna"//' $APPDATA/python/python37/site-packages/requests/packages.py
     sed -ri 's/import charset_normalizer.*/pass/' $APPDATA/python/python37/site-packages/requests/compat.py
     sed -ri 's/raise.*charset_normalizer.*/pass/' $APPDATA/python/python37/site-packages/requests/__init__.py

scripts/test/tftp.sh

@@ -3,34 +3,39 @@ set -ex
 
 # PYTHONPATH=.:~/dev/partftpy/ taskset -c 0 python3 -m copyparty -v srv::r -v srv/junk:junk:A --tftp 3969 
 
-get_src=~/dev/copyparty/srv/palette.flac
-get_fn=${get_src##*/}
+get_src=~/dev/copyparty/srv/ro/palette.flac
+get_fp=ro/${get_src##*/}  # server url
+get_fn=${get_fp##*/}  # just filename
 
 put_src=~/Downloads/102.zip
 put_dst=~/dev/copyparty/srv/junk/102.zip
 
+export PATH="$PATH:$HOME/src/atftp-0.8.0"
+
 cd /dev/shm
 
-echo curl get 1428 v4; curl --tftp-blksize 1428 tftp://127.0.0.1:3969/$get_fn | cmp $get_src || exit 1
-echo curl get 1428 v6; curl --tftp-blksize 1428 tftp://[::1]:3969/$get_fn | cmp $get_src || exit 1
+echo curl get 1428 v4; curl --tftp-blksize 1428 tftp://127.0.0.1:3969/$get_fp | cmp $get_src || exit 1
+echo curl get 1428 v6; curl --tftp-blksize 1428 tftp://[::1]:3969/$get_fp | cmp $get_src || exit 1
 
 echo curl put 1428 v4; rm -f $put_dst && curl --tftp-blksize 1428 -T $put_src tftp://127.0.0.1:3969/junk/ && cmp $put_src $put_dst || exit 1
 echo curl put 1428 v6; rm -f $put_dst && curl --tftp-blksize 1428 -T $put_src tftp://[::1]:3969/junk/ && cmp $put_src $put_dst || exit 1
 
-echo atftp get 1428; rm -f $get_fn && ~/src/atftp/atftp --option "blksize 1428" -g -r $get_fn 127.0.0.1 3969 && cmp $get_fn $get_src || exit 1
+echo atftp get 1428; rm -f $get_fn && atftp --option "blksize 1428" -g -r $get_fp -l $get_fn 127.0.0.1 3969 && cmp $get_fn $get_src || exit 1
 
-echo atftp put 1428; rm -f $put_dst && ~/src/atftp/atftp --option "blksize 1428" 127.0.0.1 3969 -p -l $put_src -r junk/102.zip && cmp $put_src $put_dst || exit 1
+echo atftp put 1428; rm -f $put_dst && atftp --option "blksize 1428" 127.0.0.1 3969 -p -l $put_src -r junk/102.zip && cmp $put_src $put_dst || exit 1
 
-echo tftp-hpa get; rm -f $put_dst && tftp -v -m binary 127.0.0.1 3969 -c get $get_fn && cmp $get_src $get_fn || exit 1
+echo tftp-hpa get; rm -f $get_fn && tftp -v -m binary 127.0.0.1 3969 -c get $get_fp && cmp $get_src $get_fn || exit 1
 
 echo tftp-hpa put; rm -f $put_dst && tftp -v -m binary 127.0.0.1 3969 -c put $put_src junk/102.zip && cmp $put_src $put_dst || exit 1
 
-echo curl get 512; curl tftp://127.0.0.1:3969/$get_fn | cmp $get_src || exit 1
+echo curl get 512; curl tftp://127.0.0.1:3969/$get_fp | cmp $get_src || exit 1
 
 echo curl put 512; rm -f $put_dst && curl -T $put_src tftp://127.0.0.1:3969/junk/ && cmp $put_src $put_dst || exit 1
 
-echo atftp get 512; rm -f $get_fn && ~/src/atftp/atftp -g -r $get_fn 127.0.0.1 3969 && cmp $get_fn $get_src || exit 1
+echo atftp get 512; rm -f $get_fn && atftp -g -r $get_fp -l $get_fn 127.0.0.1 3969 && cmp $get_fn $get_src || exit 1
 
-echo atftp put 512; rm -f $put_dst && ~/src/atftp/atftp 127.0.0.1 3969 -p -l $put_src -r junk/102.zip && cmp $put_src $put_dst || exit 1
+echo atftp put 512; rm -f $put_dst && atftp 127.0.0.1 3969 -p -l $put_src -r junk/102.zip && cmp $put_src $put_dst || exit 1
 
 echo nice
+
+rm -f $get_fn

scripts/uncomment.py

@@ -8,6 +8,12 @@ import sys
 import tokenize
 
 
+try:
+    FSTRING_MIDDLE = tokenize.FSTRING_MIDDLE
+except:
+    FSTRING_MIDDLE = -9001
+
+
 def uncomment(fpath):
     """modified https://stackoverflow.com/a/62074206"""
 
@@ -31,7 +37,7 @@ def uncomment(fpath):
         if start_line > last_lineno:
             last_col = 0
 
-        if start_col > last_col:
+        if start_col > last_col and prev_toktype != FSTRING_MIDDLE:
             out += " " * (start_col - last_col)
 
         is_legalese = (
@@ -48,6 +54,10 @@ def uncomment(fpath):
                 out += token_string
             else:
                 out += '"a"'
+        elif token_type == FSTRING_MIDDLE:
+            out += token_string.replace(r"{", r"{{").replace(r"}", r"}}")
+            if not code and token_string.strip():
+                code = True
         elif token_type != tokenize.COMMENT:
             out += token_string
             if not code and token_string.strip():

tests/test_hooks.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python3
+# coding: utf-8
+from __future__ import print_function, unicode_literals
+
+import os
+import shutil
+import tempfile
+import unittest
+
+from copyparty.authsrv import AuthSrv
+from copyparty.httpcli import HttpCli
+from tests import util as tu
+from tests.util import Cfg
+
+
+def hdr(query):
+    h = "GET /{} HTTP/1.1\r\nPW: o\r\nConnection: close\r\n\r\n"
+    return h.format(query).encode("utf-8")
+
+
+class TestHooks(unittest.TestCase):
+    def setUp(self):
+        self.td = tu.get_ramdisk()
+
+    def tearDown(self):
+        os.chdir(tempfile.gettempdir())
+        shutil.rmtree(self.td)
+
+    def reset(self):
+        td = os.path.join(self.td, "vfs")
+        if os.path.exists(td):
+            shutil.rmtree(td)
+        os.mkdir(td)
+        os.chdir(td)
+        return td
+
+    def test(self):
+        vcfg = ["a/b/c/d:c/d:A", "a:a:r"]
+
+        scenarios = (
+            ('{"vp":"x/y"}', "c/d/a.png", "c/d/x/y/a.png"),
+            ('{"vp":"x/y"}', "c/d/e/a.png", "c/d/e/x/y/a.png"),
+            ('{"vp":"../x/y"}', "c/d/e/a.png", "c/d/x/y/a.png"),
+            ('{"ap":"x/y"}', "c/d/a.png", "c/d/x/y/a.png"),
+            ('{"ap":"x/y"}', "c/d/e/a.png", "c/d/e/x/y/a.png"),
+            ('{"ap":"../x/y"}', "c/d/e/a.png", "c/d/x/y/a.png"),
+            ('{"ap":"../x/y"}', "c/d/a.png", "a/b/c/x/y/a.png"),
+            ('{"fn":"b.png"}', "c/d/a.png", "c/d/b.png"),
+            ('{"vp":"x","fn":"b.png"}', "c/d/a.png", "c/d/x/b.png"),
+        )
+
+        for x in scenarios:
+            print("\n\n\n", x)
+            hooktxt, url_up, url_dl = x
+            for hooktype in ("xbu", "xau"):
+                for upfun in (self.put, self.bup):
+                    self.reset()
+                    self.makehook("""print('{"reloc":%s}')""" % (hooktxt,))
+                    ka = {hooktype: ["j,c1,h.py"]}
+                    self.args = Cfg(v=vcfg, a=["o:o"], e2d=True, **ka)
+                    self.asrv = AuthSrv(self.args, self.log)
+
+                    h, b = upfun(url_up)
+                    self.assertIn("201 Created", h)
+                    h, b = self.curl(url_dl)
+                    self.assertEqual(b, "ok %s\n" % (url_up))
+
+    def makehook(self, hs):
+        with open("h.py", "wb") as f:
+            f.write(hs.encode("utf-8"))
+
+    def put(self, url):
+        buf = "PUT /{0} HTTP/1.1\r\nPW: o\r\nConnection: close\r\nContent-Length: {1}\r\n\r\nok {0}\n"
+        buf = buf.format(url, len(url) + 4).encode("utf-8")
+        print("PUT -->", buf)
+        conn = tu.VHttpConn(self.args, self.asrv, self.log, buf)
+        HttpCli(conn).run()
+        ret = conn.s._reply.decode("utf-8").split("\r\n\r\n", 1)
+        print("PUT <--", ret)
+        return ret
+
+    def bup(self, url):
+        hdr = "POST /%s HTTP/1.1\r\nPW: o\r\nConnection: close\r\nContent-Type: multipart/form-data; boundary=XD\r\nContent-Length: %d\r\n\r\n"
+        bdy = '--XD\r\nContent-Disposition: form-data; name="act"\r\n\r\nbput\r\n--XD\r\nContent-Disposition: form-data; name="f"; filename="%s"\r\n\r\n'
+        ftr = "\r\n--XD--\r\n"
+        try:
+            url, fn = url.rsplit("/", 1)
+        except:
+            fn = url
+            url = ""
+
+        buf = (bdy % (fn,) + "ok %s/%s\n" % (url, fn) + ftr).encode("utf-8")
+        buf = (hdr % (url, len(buf))).encode("utf-8") + buf
+        print("PoST -->", buf)
+        conn = tu.VHttpConn(self.args, self.asrv, self.log, buf)
+        HttpCli(conn).run()
+        ret = conn.s._reply.decode("utf-8").split("\r\n\r\n", 1)
+        print("POST <--", ret)
+        return ret
+
+    def curl(self, url, binary=False):
+        conn = tu.VHttpConn(self.args, self.asrv, self.log, hdr(url))
+        HttpCli(conn).run()
+        if binary:
+            h, b = conn.s._reply.split(b"\r\n\r\n", 1)
+            return [h.decode("utf-8"), b]
+
+        return conn.s._reply.decode("utf-8").split("\r\n\r\n", 1)
+
+    def log(self, src, msg, c=0):
+        print(msg)

tests/test_httpcli.py

@@ -10,6 +10,7 @@ import tarfile
 import tempfile
 import time
 import unittest
+import zipfile
 
 from copyparty.authsrv import AuthSrv
 from copyparty.httpcli import HttpCli
@@ -31,6 +32,9 @@ class TestHttpCli(unittest.TestCase):
         shutil.rmtree(self.td)
 
     def test(self):
+        test_tar = True
+        test_zip = True
+
         td = os.path.join(self.td, "vfs")
         os.mkdir(td)
         os.chdir(td)
@@ -40,6 +44,7 @@ class TestHttpCli(unittest.TestCase):
         self.can_write = ["wa", "wo", "aa", "ao"]
         self.fn = "g{:x}g".format(int(time.time() * 3))
 
+        tctr = 0
         allfiles = []
         allvols = []
         for top in self.dtypes:
@@ -83,6 +88,7 @@ class TestHttpCli(unittest.TestCase):
             self.asrv = AuthSrv(self.args, self.log)
             vfiles = [x for x in allfiles if x.startswith(top)]
             for fp in vfiles:
+                tctr += 1
                 rok, wok = self.can_rw(fp)
                 furl = fp.split("/", 1)[1]
                 durl = furl.rsplit("/", 1)[0] if "/" in furl else ""
@@ -112,39 +118,61 @@ class TestHttpCli(unittest.TestCase):
                     eprint("\033[33m{}\n# {}\033[0m".format(ret, url))
                     self.fail()
 
-                # tar
-                url = durl + "?tar"
-                h, b = self.curl(url, True)
-                # with open(os.path.join(td, "tar"), "wb") as f:
-                #    f.write(b)
-                try:
-                    tar = tarfile.open(fileobj=io.BytesIO(b), mode="r|").getnames()
-                except:
-                    if "HTTP/1.1 403 Forbidden" not in h and b != b"\nJ2EOT":
-                        eprint("bad tar?", url, h, b)
-                        raise
-                    tar = []
-                tar = [x.split("/", 1)[1] for x in tar]
-                tar = ["/".join([y for y in [top, durl, x] if y]) for x in tar]
-                tar = [[x] + self.can_rw(x) for x in tar]
-                tar_ok = [x[0] for x in tar if x[1]]
-                tar_ng = [x[0] for x in tar if not x[1]]
-                self.assertEqual([], tar_ng)
-
-                if durl.split("/")[-1] in self.can_read:
+                # expected files in archives
+                if rok:
                     ref = [x for x in vfiles if self.in_dive(top + "/" + durl, x)]
-                    for f in ref:
-                        ok = f in tar_ok
-                        pr = print if ok else eprint
-                        pr("{}: {}".format("ok" if ok else "NG", f))
                     ref.sort()
+                else:
+                    ref = []
+
+                if test_tar:
+                    url = durl + "?tar"
+                    h, b = self.curl(url, True)
+                    try:
+                        tar = tarfile.open(fileobj=io.BytesIO(b), mode="r|").getnames()
+                    except:
+                        if "HTTP/1.1 403 Forbidden" not in h and b != b"\nJ2EOT":
+                            eprint("bad tar?", url, h, b)
+                            raise
+                        tar = []
+                    tar = [x.split("/", 1)[1] for x in tar]
+                    tar = ["/".join([y for y in [top, durl, x] if y]) for x in tar]
+                    tar = [[x] + self.can_rw(x) for x in tar]
+                    tar_ok = [x[0] for x in tar if x[1]]
+                    tar_ng = [x[0] for x in tar if not x[1]]
                     tar_ok.sort()
                     self.assertEqual(ref, tar_ok)
+                    self.assertEqual([], tar_ng)
+
+                if test_zip:
+                    url = durl + "?zip"
+                    h, b = self.curl(url, True)
+                    try:
+                        with zipfile.ZipFile(io.BytesIO(b), "r") as zf:
+                            zfi = zf.infolist()
+                    except:
+                        if "HTTP/1.1 403 Forbidden" not in h and b != b"\nJ2EOT":
+                            eprint("bad zip?", url, h, b)
+                            raise
+                        zfi = []
+                    zfn = [x.filename.split("/", 1)[1] for x in zfi]
+                    zfn = ["/".join([y for y in [top, durl, x] if y]) for x in zfn]
+                    zfn = [[x] + self.can_rw(x) for x in zfn]
+                    zf_ok = [x[0] for x in zfn if x[1]]
+                    zf_ng = [x[0] for x in zfn if not x[1]]
+                    zf_ok.sort()
+                    self.assertEqual(ref, zf_ok)
+                    self.assertEqual([], zf_ng)
 
                 # stash
                 h, ret = self.put(url)
                 res = h.startswith("HTTP/1.1 201 ")
                 self.assertEqual(res, wok)
+                if wok:
+                    vp = h.split("\nLocation: http://a:1/")[1].split("\r")[0]
+                    vn, rem = self.asrv.vfs.get(vp, "*", False, False)
+                    ap = os.path.join(vn.realpath, rem)
+                    os.unlink(ap)
 
     def can_rw(self, fp):
         # lowest non-neutral folder declares permissions

tests/test_vfs.py

@@ -235,7 +235,7 @@ class TestVFS(unittest.TestCase):
                     """
                     u a:123
                     u asd:fgh:jkl
-                    
+
                     ./src
                     /dst
                     r a

tests/util.py

@@ -6,6 +6,7 @@ import os
 import platform
 import re
 import shutil
+import socket
 import subprocess as sp
 import sys
 import tempfile
@@ -68,6 +69,13 @@ def chkcmd(argv):
 
 def get_ramdisk():
     def subdir(top):
+        for d in os.listdir(top):
+            if not d.startswith("cptd-"):
+                continue
+            p = os.path.join(top, d)
+            st = os.stat(p)
+            if time.time() - st.st_mtime > 300:
+                shutil.rmtree(p)
         ret = os.path.join(top, "cptd-{}".format(os.getpid()))
         shutil.rmtree(ret, True)
         os.mkdir(ret)
@@ -111,16 +119,16 @@ class Cfg(Namespace):
     def __init__(self, a=None, v=None, c=None, **ka0):
         ka = {}
 
-        ex = "daw dav_auth dav_inf dav_mac dav_rt e2d e2ds e2dsa e2t e2ts e2tsr e2v e2vu e2vp early_ban ed emp exp force_js getmod grid gsel hardlink ih ihead magic never_symlink nid nih no_acode no_athumb no_dav no_dedup no_del no_dupe no_lifetime no_logues no_mv no_pipe no_poll no_readme no_robots no_sb_md no_sb_lg no_scandir no_tarcmp no_thumb no_vthumb no_zip nrand nw og og_no_head og_s_title q rand smb srch_dbg stats uqe vague_403 vc ver xdev xlink xvol"
+        ex = "daw dav_auth dav_inf dav_mac dav_rt e2d e2ds e2dsa e2t e2ts e2tsr e2v e2vu e2vp early_ban ed emp exp force_js getmod grid gsel hardlink ih ihead magic never_symlink nid nih no_acode no_athumb no_dav no_dedup no_del no_dupe no_lifetime no_logues no_mv no_pipe no_poll no_readme no_robots no_sb_md no_sb_lg no_scandir no_tarcmp no_thumb no_vthumb no_zip nrand nw og og_no_head og_s_title q rand smb srch_dbg stats uqe vague_403 vc ver write_uplog xdev xlink xvol"
         ka.update(**{k: False for k in ex.split()})
 
-        ex = "dotpart dotsrch no_dhash no_fastboot no_rescan no_sendfile no_snap no_voldump re_dhash plain_ip"
+        ex = "dotpart dotsrch hook_v no_dhash no_fastboot no_rescan no_sendfile no_snap no_voldump re_dhash plain_ip"
         ka.update(**{k: True for k in ex.split()})
 
-        ex = "ah_cli ah_gen css_browser hist js_browser mime mimes no_forget no_hash no_idx nonsus_urls og_tpl og_ua"
+        ex = "ah_cli ah_gen css_browser hist js_browser js_other mime mimes no_forget no_hash no_idx nonsus_urls og_tpl og_ua"
         ka.update(**{k: None for k in ex.split()})
 
-        ex = "hash_mt srch_time u2abort u2j"
+        ex = "hash_mt srch_time u2abort u2j u2sz"
         ka.update(**{k: 1 for k in ex.split()})
 
         ex = "au_vol mtab_age reg_cap s_thead s_tbody th_convt"
@@ -178,6 +186,10 @@ class Cfg(Namespace):
 
 
 class NullBroker(object):
+    def __init__(self, args, asrv):
+        self.args = args
+        self.asrv = asrv
+
     def say(self, *args):
         pass
 
@@ -189,6 +201,7 @@ class VSock(object):
     def __init__(self, buf):
         self._query = buf
         self._reply = b""
+        self.family = socket.AF_INET
         self.sendall = self.send
 
     def recv(self, sz):
@@ -213,7 +226,7 @@ class VHttpSrv(object):
         self.asrv = asrv
         self.log = log
 
-        self.broker = NullBroker()
+        self.broker = NullBroker(args, asrv)
         self.prism = None
         self.bans = {}
         self.nreq = 0