v1.13.8

`tokenize.FSTRING_MIDDLE` was introduced, changing the
representation of `f"x{{y"` from `STRING(f"x{{y")` to:

* `FSTRING_START('f"')`
* `FSTRING_MIDDLE('x{')`
* `FSTRING_MIDDLE('y')`
* `FSTRING_END('"')`

each literal `{` (encoded as `{{` in the input) now appears as a
single `{` as the final character of its `FSTRING_MIDDLE`, with
additional consecutive `FSTRING_MIDDLE` tokens if necessary

regular interpolating `{` are encoded as separate `OP` tokens

the fact that the literal `{` is encoded as a single `{` instead
of `{{` breaks the assumption that the string-value of each token
maps directly to the original code

fix this by replacing `{` with `{{` and `}` with `}}` in
`FSTRING_MIDDLE` tokens, and not adding whitespace after
`FSTRING_MIDDLE` tokens

.vscode/settings.json

@@ -22,6 +22,9 @@
         "terminal.ansiBrightCyan": "#9cf0ed",
         "terminal.ansiBrightWhite": "#ffffff",
     },
+    "python.terminal.activateEnvironment": false,
+    "python.analysis.enablePytestSupport": false,
+    "python.analysis.typeCheckingMode": "standard",
     "python.testing.pytestEnabled": false,
     "python.testing.unittestEnabled": true,
     "python.testing.unittestArgs": [
@@ -31,23 +34,8 @@
         "-p",
         "test_*.py"
     ],
-    "python.linting.pylintEnabled": true,
-    "python.linting.flake8Enabled": true,
-    "python.linting.banditEnabled": true,
-    "python.linting.mypyEnabled": true,
-    "python.linting.flake8Args": [
-        "--max-line-length=120",
-        "--ignore=E722,F405,E203,W503,W293,E402,E501,E128,E226",
-    ],
-    "python.linting.banditArgs": [
-        "--ignore=B104,B110,B112"
-    ],
-    // python3 -m isort --py=27 --profile=black copyparty/
-    "python.formatting.provider": "none",
-    "[python]": {
-        "editor.defaultFormatter": "ms-python.black-formatter"
-    },
-    "editor.formatOnSave": true,
+    // python3 -m isort --py=27 --profile=black ~/dev/copyparty/{copyparty,tests}/*.py && python3 -m black -t py27 ~/dev/copyparty/{copyparty,tests,bin}/*.py $(find ~/dev/copyparty/copyparty/stolen -iname '*.py')
+    "editor.formatOnSave": false,
     "[html]": {
         "editor.formatOnSave": false,
         "editor.autoIndent": "keep",
@@ -58,6 +46,4 @@
     "files.associations": {
         "*.makefile": "makefile"
     },
-    "python.linting.enabled": true,
-    "python.pythonPath": "/usr/bin/python3"
 }

README.md

@@ -85,6 +85,7 @@ turn almost any device into a file server with resumable uploads/downloads using
     * [prometheus](#prometheus) - metrics/stats can be enabled
     * [other extremely specific features](#other-extremely-specific-features) - you'll never find a use for these
         * [custom mimetypes](#custom-mimetypes) - change the association of a file extension
+        * [feature chickenbits](#feature-chickenbits) - buggy feature? rip it out
 * [packages](#packages) - the party might be closer than you think
     * [arch package](#arch-package) - now [available on aur](https://aur.archlinux.org/packages/copyparty) maintained by [@icxes](https://github.com/icxes)
     * [fedora package](#fedora-package) - does not exist yet
@@ -111,6 +112,7 @@ turn almost any device into a file server with resumable uploads/downloads using
 * [HTTP API](#HTTP-API) - see [devnotes](./docs/devnotes.md#http-api)
 * [dependencies](#dependencies) - mandatory deps
     * [optional dependencies](#optional-dependencies) - install these to enable bonus features
+        * [dependency chickenbits](#dependency-chickenbits) - prevent loading an optional dependency
     * [optional gpl stuff](#optional-gpl-stuff)
 * [sfx](#sfx) - the self-contained "binary" (recommended!)
     * [copyparty.exe](#copypartyexe) - download [copyparty.exe](https://github.com/9001/copyparty/releases/latest/download/copyparty.exe) (win8+) or [copyparty32.exe](https://github.com/9001/copyparty/releases/latest/download/copyparty32.exe) (win7+)
@@ -124,7 +126,7 @@ turn almost any device into a file server with resumable uploads/downloads using
 
 just run **[copyparty-sfx.py](https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.py)** -- that's it! 🎉
 
-* or install through pypi: `python3 -m pip install --user -U copyparty`
+* or install through [pypi](https://pypi.org/project/copyparty/): `python3 -m pip install --user -U copyparty`
 * or if you cannot install python, you can use [copyparty.exe](#copypartyexe) instead
 * or install [on arch](#arch-package) ╱ [on NixOS](#nixos-module) ╱ [through nix](#nix-package)
 * or if you are on android, [install copyparty in termux](#install-on-android)
@@ -194,7 +196,7 @@ firewall-cmd --reload
 also see [comparison to similar software](./docs/versus.md)
 
 * backend stuff
-  * ☑ IPv6
+  * ☑ IPv6 + unix-sockets
   * ☑ [multiprocessing](#performance) (actual multithreading)
   * ☑ volumes (mountpoints)
   * ☑ [accounts](#accounts-and-volumes)
@@ -579,9 +581,6 @@ images with the following names (see `--th-covers`) become the thumbnail of the
 * the order is significant, so if both `cover.png` and `folder.jpg` exist in a folder, it will pick the first matching `--th-covers` entry (`folder.jpg`)
 * and, if you enable [file indexing](#file-indexing), it will also try those names as dotfiles (`.folder.jpg` and so), and then fallback on the first picture in the folder (if it has any pictures at all)
 
-in the grid/thumbnail view, if the audio player panel is open, songs will start playing when clicked
-* indicated by the audio files having the ▶ icon instead of 💾
-
 enabling `multiselect` lets you click files to select them, and then shift-click another file for range-select
 * `multiselect` is mostly intended for phones/tablets, but the `sel` option in the `[⚙️] settings` tab is better suited for desktop use, allowing selection by CTRL-clicking and range-selection with SHIFT-click, all without affecting regular clicking
   * the `sel` option can be made default globally with `--gsel` or per-volume with volflag `gsel`
@@ -1314,6 +1313,8 @@ you can set hooks before and/or after an event happens, and currently you can ho
 
 there's a bunch of flags and stuff, see `--help-hooks`
 
+if you want to write your own hooks, see [devnotes](./docs/devnotes.md#event-hooks)
+
 
 ### upload events
 
@@ -1458,6 +1459,8 @@ some reverse proxies (such as [Caddy](https://caddyserver.com/)) can automatical
 * **warning:** nginx-QUIC (HTTP/3) is still experimental and can make uploads much slower, so HTTP/1.1 is recommended for now
 * depending on server/client, HTTP/1.1 can also be 5x faster than HTTP/2
 
+for improved security (and a tiny performance boost) consider listening on a unix-socket with `-i /tmp/party.sock` instead of `-i 127.0.0.1`
+
 example webserver configs:
 
 * [nginx config](contrib/nginx/copyparty.conf) -- entire domain/subdomain
@@ -1558,6 +1561,23 @@ in a config-file, this is the same as:
 run copyparty with `--mimes` to list all the default mappings
 
 
+### feature chickenbits
+
+buggy feature? rip it out  by setting any of the following environment variables to disable its associated bell or whistle,
+
+| env-var              | what it does |
+| -------------------- | ------------ |
+| `PRTY_NO_IFADDR`     | disable ip/nic discovery by poking into your OS with ctypes |
+| `PRTY_NO_IPV6`       | disable some ipv6 support (should not be necessary since windows 2000) |
+| `PRTY_NO_LZMA`       | disable streaming xz compression of incoming uploads |
+| `PRTY_NO_MP`         | disable all use of the python `multiprocessing` module (actual multithreading, cpu-count for parsers/thumbnailers) |
+| `PRTY_NO_SQLITE`     | disable all database-related functionality (file indexing, metadata indexing, most file deduplication logic) |
+| `PRTY_NO_TLS`        | disable native HTTPS support; if you still want to accept HTTPS connections then TLS must now be terminated by a reverse-proxy |
+| `PRTY_NO_TPOKE`      | disable systemd-tmpfilesd avoider |
+
+example: `PRTY_NO_IFADDR=1 python3 copyparty-sfx.py`
+
+
 # packages
 
 the party might be closer than you think
@@ -1880,6 +1900,7 @@ some notes on hardening
   * cors doesn't work right otherwise
 * if you allow anonymous uploads or otherwise don't trust the contents of a volume, you can prevent XSS with volflag `nohtml`
   * this returns html documents as plaintext, and also disables markdown rendering
+* when running behind a reverse-proxy, listen on a unix-socket with `-i /tmp/party.sock` instead of `-i 127.0.0.1` for tighter access control (plus you get a tiny performance boost for free)
 
 safety profiles:
 
@@ -2044,6 +2065,37 @@ enable [smb](#smb-server) support (**not** recommended):
 `pyvips` gives higher quality thumbnails than `Pillow` and is 320% faster, using 270% more ram: `sudo apt install libvips42 && python3 -m pip install --user -U pyvips`
 
 
+### dependency chickenbits
+
+prevent loading an optional dependency  , for example if:
+
+* you have an incompatible version installed and it causes problems
+* you just don't want copyparty to use it, maybe to save ram
+
+set any of the following environment variables to disable its associated optional feature,
+
+| env-var              | what it does |
+| -------------------- | ------------ |
+| `PRTY_NO_ARGON2`     | disable argon2-cffi password hashing |
+| `PRTY_NO_CFSSL`      | never attempt to generate self-signed certificates using [cfssl](https://github.com/cloudflare/cfssl) |
+| `PRTY_NO_FFMPEG`     | **audio transcoding** goes byebye, **thumbnailing** must be handled by Pillow/libvips |
+| `PRTY_NO_FFPROBE`    | **audio transcoding** goes byebye, **thumbnailing** must be handled by Pillow/libvips, **metadata-scanning** must be handled by mutagen |
+| `PRTY_NO_MUTAGEN`    | do not use [mutagen](https://pypi.org/project/mutagen/) for reading metadata from media files; will fallback to ffprobe |
+| `PRTY_NO_PIL`        | disable all [Pillow](https://pypi.org/project/pillow/)-based thumbnail support; will fallback to libvips or ffmpeg |
+| `PRTY_NO_PILF`       | disable Pillow `ImageFont` text rendering, used for folder thumbnails |
+| `PRTY_NO_PIL_AVIF`   | disable 3rd-party Pillow plugin for [AVIF support](https://pypi.org/project/pillow-avif-plugin/) |
+| `PRTY_NO_PIL_HEIF`   | disable 3rd-party Pillow plugin for [HEIF support](https://pypi.org/project/pyheif-pillow-opener/) |
+| `PRTY_NO_PIL_WEBP`   | disable use of native webp support in Pillow |
+| `PRTY_NO_PSUTIL`     | do not use [psutil](https://pypi.org/project/psutil/) for reaping stuck hooks and plugins on Windows |
+| `PRTY_NO_VIPS`       | disable all [libvips](https://pypi.org/project/pyvips/)-based thumbnail support; will fallback to Pillow or ffmpeg |
+
+example: `PRTY_NO_PIL=1 python3 copyparty-sfx.py`
+
+* `PRTY_NO_PIL` saves ram
+* `PRTY_NO_VIPS` saves ram and startup time
+* python2.7 on windows: `PRTY_NO_FFMPEG` + `PRTY_NO_FFPROBE` saves startup time
+
+
 ## optional gpl stuff
 
 some bundled tools have copyleft dependencies, see [./bin/#mtag](bin/#mtag)

bin/hooks/README.md

@@ -24,6 +24,7 @@ these are `--xiu` hooks; unlike `xbu` and `xau` (which get executed on every sin
 
 # before upload
 * [reject-extension.py](reject-extension.py) rejects uploads if they match a list of file extensions
+* [reloc-by-ext.py](reloc-by-ext.py) redirects an upload to another destination based on the file extension
 
 
 # on message

bin/hooks/into-the-cache-it-goes.py

@@ -41,8 +41,8 @@ parameters explained,
     t10 = abort download and continue if it takes longer than 10sec
 
 example usage as a volflag (per-volume config):
-    -v srv/inc:inc:r:rw,ed:xau=j,t10,bin/hooks/into-the-cache-it-goes.py
-                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+    -v srv/inc:inc:r:rw,ed:c,xau=j,t10,bin/hooks/into-the-cache-it-goes.py
+                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
     (share filesystem-path srv/inc as volume /inc,
      readable by everyone, read-write for user 'ed',

bin/hooks/reloc-by-ext.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+
+import json
+import os
+import sys
+
+
+_ = r"""
+relocate/redirect incoming uploads according to file extension
+
+example usage as global config:
+    --xbu j,c1,bin/hooks/reloc-by-ext.py
+
+parameters explained,
+    xbu = execute before upload
+    j   = this hook needs upload information as json (not just the filename)
+    c1  = this hook returns json on stdout, so tell copyparty to read that
+
+example usage as a volflag (per-volume config):
+    -v srv/inc:inc:r:rw,ed:c,xbu=j,c1,bin/hooks/reloc-by-ext.py
+                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+    (share filesystem-path srv/inc as volume /inc,
+     readable by everyone, read-write for user 'ed',
+     running this plugin on all uploads with the params explained above)
+
+example usage as a volflag in a copyparty config file:
+    [/inc]
+      srv/inc
+      accs:
+        r: *
+        rw: ed
+      flags:
+        xbu: j,c1,bin/hooks/reloc-by-ext.py
+
+note: this could also work as an xau hook (after-upload), but
+  because it doesn't need to read the file contents its better
+  as xbu (before-upload) since that's safer / less buggy,
+  and only xbu works with up2k (dragdrop into browser)
+"""
+
+
+PICS = "avif bmp gif heic heif jpeg jpg jxl png psd qoi tga tif tiff webp"
+VIDS = "3gp asf avi flv mkv mov mp4 mpeg mpeg2 mpegts mpg mpg2 nut ogm ogv rm ts vob webm wmv"
+MUSIC = "aac aif aiff alac amr ape dfpwm flac m4a mp3 ogg opus ra tak tta wav wma wv"
+
+
+def main():
+    inf = json.loads(sys.argv[1])
+    vdir, fn = os.path.split(inf["vp"])
+
+    try:
+        fn, ext = fn.rsplit(".", 1)
+    except:
+        # no file extension; abort
+        return
+
+    ext = ext.lower()
+
+    ##
+    ## some example actions to take; pick one by
+    ## selecting it inside the print at the end:
+    ##
+
+    # create a subfolder named after the filetype and move it into there
+    into_subfolder = {"vp": ext}
+
+    # move it into a toplevel folder named after the filetype
+    into_toplevel = {"vp": "/" + ext}
+
+    # move it into a filetype-named folder next to the target folder
+    into_sibling = {"vp": "../" + ext}
+
+    # move images into "/just/pics", vids into "/just/vids",
+    # music into "/just/tunes", and anything else as-is
+    if ext in PICS.split():
+        by_category = {"vp": "/just/pics"}
+    elif ext in VIDS.split():
+        by_category = {"vp": "/just/vids"}
+    elif ext in MUSIC.split():
+        by_category = {"vp": "/just/tunes"}
+    else:
+        by_category = {}
+
+    # now choose the effect to apply; can be any of these:
+    # into_subfolder  into_toplevel  into_sibling  by_category
+    effect = into_subfolder
+    print(json.dumps({"reloc": effect}))
+
+
+if __name__ == "__main__":
+    main()

bin/u2c.py

@@ -1,8 +1,8 @@
 #!/usr/bin/env python3
 from __future__ import print_function, unicode_literals
 
-S_VERSION = "1.21"
-S_BUILD_DT = "2024-07-26"
+S_VERSION = "1.22"
+S_BUILD_DT = "2024-08-08"
 
 """
 u2c.py: upload to copyparty
@@ -660,8 +660,15 @@ def upload(fsl, pw, stats):
     # type: (FileSlice, str, str) -> None
     """upload a range of file data, defined by one or more `cid` (chunk-hash)"""
 
+    ctxt = fsl.cids[0]
+    if len(fsl.cids) > 1:
+        n = 192 // len(fsl.cids)
+        n = 9 if n > 9 else 2 if n < 2 else n
+        zsl = [zs[:n] for zs in fsl.cids[1:]]
+        ctxt += ",%d,%s" % (n, "".join(zsl))
+
     headers = {
-        "X-Up2k-Hash": ",".join(fsl.cids),
+        "X-Up2k-Hash": ctxt,
         "X-Up2k-Wark": fsl.file.wark,
         "Content-Type": "application/octet-stream",
     }

contrib/package/arch/PKGBUILD

@@ -1,6 +1,6 @@
 # Maintainer: icxes <dev.null@need.moe>
 pkgname=copyparty
-pkgver="1.13.5"
+pkgver="1.13.6"
 pkgrel=1
 pkgdesc="File server with accelerated resumable uploads, dedup, WebDAV, FTP, TFTP, zeroconf, media indexer, thumbnails++"
 arch=("any")
@@ -21,7 +21,7 @@ optdepends=("ffmpeg: thumbnails for videos, images (slower) and audio, music tag
 )
 source=("https://github.com/9001/${pkgname}/releases/download/v${pkgver}/${pkgname}-${pkgver}.tar.gz")
 backup=("etc/${pkgname}.d/init" )
-sha256sums=("83bf52ac03256ee6fe405a912e2767578692760f9554f821dfcab0700dd58082")
+sha256sums=("abb9d0df24c4082594adc2070b7e9fb84a57733d0a3fbf83f4ec2a02813dd717")
 
 build() {
     cd "${srcdir}/${pkgname}-${pkgver}"

contrib/package/nix/copyparty/pin.json

@@ -1,5 +1,5 @@
 {
-    "url": "https://github.com/9001/copyparty/releases/download/v1.13.5/copyparty-sfx.py",
-    "version": "1.13.5",
-    "hash": "sha256-I+dqsiScYPcX6JpLgwVoLs7l0FlbXabc/Ofqye9RQI0="
+    "url": "https://github.com/9001/copyparty/releases/download/v1.13.6/copyparty-sfx.py",
+    "version": "1.13.6",
+    "hash": "sha256-eQSHZYXIbS4piCU/zOyEz3E6V/lS6348vENMLwONYUM="
 }

contrib/plugins/README.md

@@ -20,6 +20,13 @@ point `--js-browser` to one of these by URL:
 
 
 
+## example any-js
+point `--js-browser` and/or `--js-other` to one of these by URL:
+
+* [`banner.js`](banner.js) shows a very enterprise [legal-banner](https://github.com/user-attachments/assets/8ae8e087-b209-449c-b08d-74e040f0284b)
+
+
+
 ## example browser-css
 point `--css-browser` to one of these by URL:
 

contrib/plugins/banner.js

@@ -0,0 +1,93 @@
+(function() {
+
+// usage: copy this to '.banner.js' in your webroot,
+// and run copyparty with the following argument:
+// --body-foot '<script src="/.banner.js"></script>'
+
+
+
+// had to pick the most chuuni one as the default
+var bannertext = '' +
+'<h3>You are accessing a U.S. Government (USG) Information System (IS) that is provided for USG-authorized use only.</h3>' +
+'<p>By using this IS (which includes any device attached to this IS), you consent to the following conditions:</p>' +
+'<ul>' +
+'<li>The USG routinely intercepts and monitors communications on this IS for purposes including, but not limited to, penetration testing, COMSEC monitoring, network operations and defense, personnel misconduct (PM), law enforcement (LE), and counterintelligence (CI) investigations.</li>' +
+'<li>At any time, the USG may inspect and seize data stored on this IS.</li>' +
+'<li>Communications using, or data stored on, this IS are not private, are subject to routine monitoring, interception, and search, and may be disclosed or used for any USG-authorized purpose.</li>' +
+'<li>This IS includes security measures (e.g., authentication and access controls) to protect USG interests -- not for your personal benefit or privacy.</li>' +
+'<li>Notwithstanding the above, using this IS does not constitute consent to PM, LE or CI investigative searching or monitoring of the content of privileged communications, or work product, related to personal representation or services by attorneys, psychotherapists, or clergy, and their assistants. Such communications and work product are private and confidential. See User Agreement for details.</li>' +
+'</ul>';
+
+
+
+// fancy div to insert into pages
+function bannerdiv(border) {
+    var ret = mknod('div', null, bannertext);
+    if (border)
+        ret.setAttribute("style", "border:1em solid var(--fg); border-width:.3em 0; margin:3em 0");
+    return ret;
+}
+
+
+
+// keep all of these false and then selectively enable them in the if-blocks below
+var show_msgbox = false,
+    login_top = false,
+    top = false,
+    bottom = false,
+    top_bordered = false,
+    bottom_bordered = false;
+
+if (QS("h1#cc") && QS("a#k")) {
+    // this is the controlpanel
+    // (you probably want to keep just one of these enabled)
+    show_msgbox = true;
+    login_top = true;
+    bottom = true;
+}
+else if (ebi("swin") && ebi("smac")) {
+    // this is the connect-page, same deal here
+    show_msgbox = true;
+    top_bordered = true;
+    bottom_bordered = true;
+}
+else if (ebi("op_cfg") || ebi("div#mw") ) {
+    // we're running in the main filebrowser (op_cfg) or markdown-viewer/editor (div#mw),
+    // fragile pages which break if you do something too fancy
+    show_msgbox = true;
+}
+
+
+
+// shows a fullscreen messagebox; works on all pages
+if (show_msgbox) {
+    var now = Math.floor(Date.now() / 1000),
+        last_shown = sread("bannerts") || 0;
+
+    // 60 * 60 * 17 = 17 hour cooldown
+    if (now - last_shown > 60 * 60 * 17) {
+        swrite("bannerts", now);
+        modal.confirm(bannertext, null, function () {
+            location = 'https://this-page-intentionally-left-blank.org/';
+        });
+    }
+}
+
+// show a message on the page footer; only works on the connect-page
+if (top || top_bordered) {
+    var dst = ebi('wrap');
+    dst.insertBefore(bannerdiv(top_bordered), dst.firstChild);
+}
+
+// show a message on the page footer; only works on the controlpanel and connect-page
+if (bottom || bottom_bordered) {
+    ebi('wrap').appendChild(bannerdiv(bottom_bordered));
+}
+
+// show a message on the top of the page; only works on the controlpanel
+if (login_top) {
+    var dst = QS('h1');
+    dst.parentNode.insertBefore(bannerdiv(false), dst);
+}
+
+})();

copyparty/__main__.py

@@ -67,7 +67,13 @@ if True:  # pylint: disable=using-constant-test
 
     from typing import Any, Optional
 
+if PY2:
+    range = xrange  # type: ignore
+
 try:
+    if os.environ.get("PRTY_NO_TLS"):
+        raise Exception()
+
     HAVE_SSL = True
     import ssl
 except:
@@ -344,7 +350,7 @@ def configure_ssl_ver(al: argparse.Namespace) -> None:
     # oh man i love openssl
     # check this out
     # hold my beer
-    assert ssl
+    assert ssl  # type: ignore
     ptn = re.compile(r"^OP_NO_(TLS|SSL)v")
     sslver = terse_sslver(al.ssl_ver).split(",")
     flags = [k for k in ssl.__dict__ if ptn.match(k)]
@@ -378,7 +384,7 @@ def configure_ssl_ver(al: argparse.Namespace) -> None:
 
 
 def configure_ssl_ciphers(al: argparse.Namespace) -> None:
-    assert ssl
+    assert ssl  # type: ignore
     ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
     if al.ssl_ver:
         ctx.options &= ~al.ssl_flags_en
@@ -491,6 +497,9 @@ def disable_quickedit() -> None:
 
 
 def sfx_tpoke(top: str):
+    if os.environ.get("PRTY_NO_TPOKE"):
+        return
+
     files = [top] + [
         os.path.join(dp, p) for dp, dd, df in os.walk(top) for p in dd + df
     ]
@@ -695,6 +704,11 @@ def get_sects():
             \033[36mxban\033[0m can be used to overrule / cancel a user ban event;
             if the program returns 0 (true/OK) then the ban will NOT happen
 
+            effects can be used to redirect uploads into other
+            locations, and to delete or index other files based
+            on new uploads, but with certain limitations. See
+            bin/hooks/reloc* and docs/devnotes.md#hook-effects
+
             except for \033[36mxm\033[0m, only one hook / one action can run at a time,
             so it's recommended to use the \033[36mf\033[0m flag unless you really need
             to wait for the hook to finish before continuing (without \033[36mf\033[0m
@@ -955,8 +969,8 @@ def add_upload(ap):
 
 def add_network(ap):
     ap2 = ap.add_argument_group('network options')
-    ap2.add_argument("-i", metavar="IP", type=u, default="::", help="ip to bind (comma-sep.), default: all IPv4 and IPv6")
-    ap2.add_argument("-p", metavar="PORT", type=u, default="3923", help="ports to bind (comma/range)")
+    ap2.add_argument("-i", metavar="IP", type=u, default="::", help="ip to bind (comma-sep.) and/or [\033[32munix:/tmp/a.sock\033[0m], default: all IPv4 and IPv6")
+    ap2.add_argument("-p", metavar="PORT", type=u, default="3923", help="ports to bind (comma/range); ignored for unix-sockets")
     ap2.add_argument("--ll", action="store_true", help="include link-local IPv4/IPv6 in mDNS replies, even if the NIC has routable IPs (breaks some mDNS clients)")
     ap2.add_argument("--rproxy", metavar="DEPTH", type=int, default=1, help="which ip to associate clients with; [\033[32m0\033[0m]=tcp, [\033[32m1\033[0m]=origin (first x-fwd, unsafe), [\033[32m2\033[0m]=outermost-proxy, [\033[32m3\033[0m]=second-proxy, [\033[32m-1\033[0m]=closest-proxy")
     ap2.add_argument("--xff-hdr", metavar="NAME", type=u, default="x-forwarded-for", help="if reverse-proxied, which http header to read the client's real ip from")
@@ -1123,6 +1137,7 @@ def add_hooks(ap):
     ap2.add_argument("--xad", metavar="CMD", type=u, action="append", help="execute \033[33mCMD\033[0m after  a file delete")
     ap2.add_argument("--xm", metavar="CMD", type=u, action="append", help="execute \033[33mCMD\033[0m on message")
     ap2.add_argument("--xban", metavar="CMD", type=u, action="append", help="execute \033[33mCMD\033[0m if someone gets banned (pw/404/403/url)")
+    ap2.add_argument("--hook-v", action="store_true", help="verbose hooks")
 
 
 def add_stats(ap):
@@ -1351,9 +1366,10 @@ def add_ui(ap, retry):
     ap2.add_argument("--unlist", metavar="REGEX", type=u, default="", help="don't show files matching \033[33mREGEX\033[0m in file list. Purely cosmetic! Does not affect API calls, just the browser. Example: [\033[32m\\.(js|css)$\033[0m] (volflag=unlist)")
     ap2.add_argument("--favico", metavar="TXT", type=u, default="c 000 none" if retry else "🎉 000 none", help="\033[33mfavicon-text\033[0m [ \033[33mforeground\033[0m [ \033[33mbackground\033[0m ] ], set blank to disable")
     ap2.add_argument("--mpmc", metavar="URL", type=u, default="", help="change the mediaplayer-toggle mouse cursor; URL to a folder with {2..5}.png inside (or disable with [\033[32m.\033[0m])")
-    ap2.add_argument("--js-browser", metavar="L", type=u, default="", help="URL to additional JS to include")
-    ap2.add_argument("--css-browser", metavar="L", type=u, default="", help="URL to additional CSS to include")
-    ap2.add_argument("--html-head", metavar="TXT", type=u, default="", help="text to append to the <head> of all HTML pages; can be @PATH to send the contents of a file at PATH, and/or begin with %% to render as jinja2 template (volflag=html_head)")
+    ap2.add_argument("--css-browser", metavar="L", type=u, default="", help="URL to additional CSS to include in the filebrowser html")
+    ap2.add_argument("--js-browser", metavar="L", type=u, default="", help="URL to additional JS to include in the filebrowser html")
+    ap2.add_argument("--js-other", metavar="L", type=u, default="", help="URL to additional JS to include in all other pages")
+    ap2.add_argument("--html-head", metavar="TXT", type=u, default="", help="text to append to the <head> of all HTML pages (except for basic-browser); can be @PATH to send the contents of a file at PATH, and/or begin with %% to render as jinja2 template (volflag=html_head)")
     ap2.add_argument("--ih", action="store_true", help="if a folder contains index.html, show that instead of the directory listing by default (can be changed in the client settings UI, or add ?v to URL for override)")
     ap2.add_argument("--textfiles", metavar="CSV", type=u, default="txt,nfo,diz,cue,readme", help="file extensions to present as plaintext")
     ap2.add_argument("--txt-max", metavar="KiB", type=int, default=64, help="max size of embedded textfiles on ?doc= (anything bigger will be lazy-loaded by JS)")
@@ -1372,12 +1388,14 @@ def add_debug(ap):
     ap2 = ap.add_argument_group('debug options')
     ap2.add_argument("--vc", action="store_true", help="verbose config file parser (explain config)")
     ap2.add_argument("--cgen", action="store_true", help="generate config file from current config (best-effort; probably buggy)")
+    ap2.add_argument("--deps", action="store_true", help="list information about detected optional dependencies")
     if hasattr(select, "poll"):
         ap2.add_argument("--no-poll", action="store_true", help="kernel-bug workaround: disable poll; use select instead (limits max num clients to ~700)")
     ap2.add_argument("--no-sendfile", action="store_true", help="kernel-bug workaround: disable sendfile; do a safe and slow read-send-loop instead")
     ap2.add_argument("--no-scandir", action="store_true", help="kernel-bug workaround: disable scandir; do a listdir + stat on each file instead")
     ap2.add_argument("--no-fastboot", action="store_true", help="wait for initial filesystem indexing before accepting client requests")
     ap2.add_argument("--no-htp", action="store_true", help="disable httpserver threadpool, create threads as-needed instead")
+    ap2.add_argument("--rm-sck", action="store_true", help="when listening on unix-sockets, do a basic delete+bind instead of the default atomic bind")
     ap2.add_argument("--srch-dbg", action="store_true", help="explain search processing, and do some extra expensive sanity checks")
     ap2.add_argument("--rclone-mdns", action="store_true", help="use mdns-domain instead of server-ip on /?hc")
     ap2.add_argument("--stackmon", metavar="P,S", type=u, default="", help="write stacktrace to \033[33mP\033[0math every \033[33mS\033[0m second, for example --stackmon=\033[32m./st/%%Y-%%m/%%d/%%H%%M.xz,60")

copyparty/__version__.py

@@ -1,8 +1,8 @@
 # coding: utf-8
 
-VERSION = (1, 13, 6)
+VERSION = (1, 13, 8)
 CODENAME = "race the beam"
-BUILD_DT = (2024, 7, 29)
+BUILD_DT = (2024, 8, 13)
 
 S_VERSION = ".".join(map(str, VERSION))
 S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT)

copyparty/authsrv.py

@@ -12,7 +12,7 @@ import threading
 import time
 from datetime import datetime
 
-from .__init__ import ANYWIN, TYPE_CHECKING, WINDOWS, E
+from .__init__ import ANYWIN, PY2, TYPE_CHECKING, WINDOWS, E
 from .bos import bos
 from .cfg import flagdescs, permdescs, vf_bmap, vf_cmap, vf_vmap
 from .pwhash import PWHash
@@ -56,6 +56,9 @@ if TYPE_CHECKING:
     # Vflags: TypeAlias = dict[str, Any]
     # Mflags: TypeAlias = dict[str, Vflags]
 
+if PY2:
+    range = xrange  # type: ignore
+
 
 LEELOO_DALLAS = "leeloo_dallas"
 
@@ -441,7 +444,7 @@ class VFS(object):
 
     def _find(self, vpath: str) -> tuple["VFS", str]:
         """return [vfs,remainder]"""
-        if vpath == "":
+        if not vpath:
             return self, ""
 
         if "/" in vpath:
@@ -451,7 +454,7 @@ class VFS(object):
             rem = ""
 
         if name in self.nodes:
-            return self.nodes[name]._find(undot(rem))
+            return self.nodes[name]._find(rem)
 
         return self, vpath
 
@@ -518,8 +521,8 @@ class VFS(object):
                     t = "{} has no {} in [{}] => [{}] => [{}]"
                     self.log("vfs", t.format(uname, msg, vpath, cvpath, ap), 6)
 
-                t = 'you don\'t have %s-access in "/%s"'
-                raise Pebkac(err, t % (msg, cvpath))
+                t = 'you don\'t have %s-access in "/%s" or below "/%s"'
+                raise Pebkac(err, t % (msg, cvpath, vn.vpath))
 
         return vn, rem
 
@@ -1895,7 +1898,7 @@ class AuthSrv(object):
                     self.log(t.format(vol.vpath), 1)
                     del vol.flags["lifetime"]
 
-                needs_e2d = [x for x in hooks if x != "xm"]
+                needs_e2d = [x for x in hooks if x in ("xau", "xiu")]
                 drop = [x for x in needs_e2d if vol.flags.get(x)]
                 if drop:
                     t = 'removing [{}] from volume "/{}" because e2d is disabled'

copyparty/cert.py

@@ -9,7 +9,7 @@ import time
 from .__init__ import ANYWIN
 from .util import Netdev, runcmd, wrename, wunlink
 
-HAVE_CFSSL = True
+HAVE_CFSSL = not os.environ.get("PRTY_NO_CFSSL")
 
 if True:  # pylint: disable=using-constant-test
     from .util import NamedLogger, RootLogger

copyparty/fsutil.py

@@ -9,12 +9,12 @@ import time
 from .__init__ import ANYWIN, MACOS
 from .authsrv import AXS, VFS
 from .bos import bos
-from .util import chkcmd, min_ex
+from .util import chkcmd, min_ex, undot
 
 if True:  # pylint: disable=using-constant-test
     from typing import Optional, Union
 
-    from .util import RootLogger
+    from .util import RootLogger, undot
 
 
 class Fstab(object):
@@ -52,7 +52,7 @@ class Fstab(object):
                 self.log(msg.format(path, fs, min_ex()), 3)
                 return fs
 
-        path = path.lstrip("/")
+        path = undot(path)
         try:
             return self.cache[path]
         except:
@@ -124,7 +124,7 @@ class Fstab(object):
         if ANYWIN:
             path = self._winpath(path)
 
-        path = path.lstrip("/")
+        path = undot(path)
         ptn = re.compile(r"^[^\\/]*")
         vn, rem = self.tab._find(path)
         if not self.trusted:

copyparty/ftpd.py

@@ -41,6 +41,9 @@ if True:  # pylint: disable=using-constant-test
     import typing
     from typing import Any, Optional, Union
 
+if PY2:
+    range = xrange  # type: ignore
+
 
 class FSE(FilesystemError):
     def __init__(self, msg: str, severity: int = 0) -> None:
@@ -350,7 +353,7 @@ class FtpFs(AbstractedFS):
         svp = join(self.cwd, src).lstrip("/")
         dvp = join(self.cwd, dst).lstrip("/")
         try:
-            self.hub.up2k.handle_mv(self.uname, svp, dvp)
+            self.hub.up2k.handle_mv(self.uname, self.h.cli_ip, svp, dvp)
         except Exception as ex:
             raise FSE(str(ex))
 
@@ -468,6 +471,9 @@ class FtpHandler(FTPHandler):
         xbu = vfs.flags.get("xbu")
         if xbu and not runhook(
             None,
+            None,
+            self.hub.up2k,
+            "xbu.ftpd",
             xbu,
             ap,
             vp,
@@ -477,7 +483,7 @@ class FtpHandler(FTPHandler):
             0,
             0,
             self.cli_ip,
-            0,
+            time.time(),
             "",
         ):
             raise FSE("Upload blocked by xbu server config")
@@ -580,9 +586,15 @@ class Ftpd(object):
         if "::" in ips:
             ips.append("0.0.0.0")
 
+        ips = [x for x in ips if "unix:" not in x]
+
         if self.args.ftp4:
             ips = [x for x in ips if ":" not in x]
 
+        if not ips:
+            lgr.fatal("cannot start ftp-server; no compatible IPs in -i")
+            return
+
         ips = list(ODict.fromkeys(ips))  # dedup
 
         ioloop = IOLoop()

copyparty/httpcli.py

@@ -13,18 +13,22 @@ import json
 import os
 import random
 import re
+import socket
 import stat
 import string
 import threading  # typechk
 import time
 import uuid
 from datetime import datetime
-from email.utils import formatdate, parsedate
+from email.utils import parsedate
 from operator import itemgetter
 
 import jinja2  # typechk
 
 try:
+    if os.environ.get("PRTY_NO_LZMA"):
+        raise Exception()
+
     import lzma
 except:
     pass
@@ -54,6 +58,7 @@ from .util import (
     alltrace,
     atomic_move,
     exclude_dotfiles,
+    formatdate,
     fsenc,
     gen_filekey,
     gen_filekey_dbg,
@@ -69,7 +74,9 @@ from .util import (
     humansize,
     ipnorm,
     loadpy,
+    log_reloc,
     min_ex,
+    pathmod,
     quotep,
     rand_name,
     read_header,
@@ -105,6 +112,9 @@ if True:  # pylint: disable=using-constant-test
 if TYPE_CHECKING:
     from .httpconn import HttpConn
 
+if not hasattr(socket, "AF_UNIX"):
+    setattr(socket, "AF_UNIX", -9001)
+
 _ = (argparse, threading)
 
 NO_CACHE = {"Cache-Control": "no-cache"}
@@ -222,6 +232,11 @@ class HttpCli(object):
         ka["s_doctitle"] = self.args.doctitle
         ka["tcolor"] = self.vn.flags["tcolor"]
 
+        if self.args.js_other and "js" not in ka:
+            zs = self.args.js_other
+            zs += "&" if "?" in zs else "?"
+            ka["js"] = zs
+
         zso = self.vn.flags.get("html_head")
         if zso:
             ka["this"] = self
@@ -303,8 +318,11 @@ class HttpCli(object):
         )
         self.host = self.headers.get("host") or ""
         if not self.host:
-            zs = "%s:%s" % self.s.getsockname()[:2]
-            self.host = zs[7:] if zs.startswith("::ffff:") else zs
+            if self.s.family == socket.AF_UNIX:
+                self.host = self.args.name
+            else:
+                zs = "%s:%s" % self.s.getsockname()[:2]
+                self.host = zs[7:] if zs.startswith("::ffff:") else zs
 
         trusted_xff = False
         n = self.args.rproxy
@@ -691,6 +709,9 @@ class HttpCli(object):
         xban = self.vn.flags.get("xban")
         if not xban or not runhook(
             self.log,
+            self.conn.hsrv.broker,
+            None,
+            "xban",
             xban,
             self.vn.canonical(self.rem),
             self.vpath,
@@ -787,7 +808,7 @@ class HttpCli(object):
 
         # close if unknown length, otherwise take client's preference
         response.append("Connection: " + ("Keep-Alive" if self.keepalive else "Close"))
-        response.append("Date: " + formatdate(usegmt=True))
+        response.append("Date: " + formatdate())
 
         # headers{} overrides anything set previously
         if headers:
@@ -811,9 +832,9 @@ class HttpCli(object):
                 self.cbonk(self.conn.hsrv.gmal, zs, "cc_hdr", "Cc in out-hdr")
                 raise Pebkac(999)
 
+        response.append("\r\n")
         try:
-            # best practice to separate headers and body into different packets
-            self.s.sendall("\r\n".join(response).encode("utf-8") + b"\r\n\r\n")
+            self.s.sendall("\r\n".join(response).encode("utf-8"))
         except:
             raise Pebkac(400, "client d/c while replying headers")
 
@@ -1146,7 +1167,7 @@ class HttpCli(object):
             return self.tx_mounts()
 
         # conditional redirect to single volumes
-        if self.vpath == "" and not self.ouparam:
+        if not self.vpath and not self.ouparam:
             nread = len(self.rvol)
             nwrite = len(self.wvol)
             if nread + nwrite == 1 or (self.rvol == self.wvol and nread == 1):
@@ -1168,7 +1189,8 @@ class HttpCli(object):
         if self.args.no_dav:
             raise Pebkac(405, "WebDAV is disabled in server config")
 
-        vn, rem = self.asrv.vfs.get(self.vpath, self.uname, False, False, err=401)
+        vn = self.vn
+        rem = self.rem
         tap = vn.canonical(rem)
 
         if "davauth" in vn.flags and self.uname == "*":
@@ -1305,7 +1327,7 @@ class HttpCli(object):
 
             pvs: dict[str, str] = {
                 "displayname": html_escape(rp.split("/")[-1]),
-                "getlastmodified": formatdate(mtime, usegmt=True),
+                "getlastmodified": formatdate(mtime),
                 "resourcetype": '<D:collection xmlns:D="DAV:"/>' if isdir else "",
                 "supportedlock": '<D:lockentry xmlns:D="DAV:"><D:lockscope><D:exclusive/></D:lockscope><D:locktype><D:write/></D:locktype></D:lockentry>',
             }
@@ -1552,8 +1574,8 @@ class HttpCli(object):
         self.log("PUT %s @%s" % (self.req, self.uname))
 
         if not self.can_write:
-            t = "user {} does not have write-access here"
-            raise Pebkac(403, t.format(self.uname))
+            t = "user %s does not have write-access under /%s"
+            raise Pebkac(403, t % (self.uname, self.vn.vpath))
 
         if not self.args.no_dav and self._applesan():
             return self.headers.get("content-length") == "0"
@@ -1628,6 +1650,9 @@ class HttpCli(object):
                             if xm:
                                 runhook(
                                     self.log,
+                                    self.conn.hsrv.broker,
+                                    None,
+                                    "xm",
                                     xm,
                                     self.vn.canonical(self.rem),
                                     self.vpath,
@@ -1776,11 +1801,15 @@ class HttpCli(object):
 
         if xbu:
             at = time.time() - lifetime
-            if not runhook(
+            vp = vjoin(self.vpath, fn) if nameless else self.vpath
+            hr = runhook(
                 self.log,
+                self.conn.hsrv.broker,
+                None,
+                "xbu.http.dump",
                 xbu,
                 path,
-                self.vpath,
+                vp,
                 self.host,
                 self.uname,
                 self.asrv.vfs.get_perms(self.vpath, self.uname),
@@ -1789,10 +1818,25 @@ class HttpCli(object):
                 self.ip,
                 at,
                 "",
-            ):
+            )
+            if not hr:
                 t = "upload blocked by xbu server config"
                 self.log(t, 1)
                 raise Pebkac(403, t)
+            if hr.get("reloc"):
+                x = pathmod(self.asrv.vfs, path, vp, hr["reloc"])
+                if x:
+                    if self.args.hook_v:
+                        log_reloc(self.log, hr["reloc"], x, path, vp, fn, vfs, rem)
+                    fdir, self.vpath, fn, (vfs, rem) = x
+                    if self.args.nw:
+                        fn = os.devnull
+                    else:
+                        bos.makedirs(fdir)
+                        path = os.path.join(fdir, fn)
+                        if not nameless:
+                            self.vpath = vjoin(self.vpath, fn)
+                        params["fdir"] = fdir
 
         if is_put and not (self.args.no_dav or self.args.nw) and bos.path.exists(path):
             # allow overwrite if...
@@ -1867,24 +1911,45 @@ class HttpCli(object):
                 fn = fn2
                 path = path2
 
-        if xau and not runhook(
-            self.log,
-            xau,
-            path,
-            self.vpath,
-            self.host,
-            self.uname,
-            self.asrv.vfs.get_perms(self.vpath, self.uname),
-            mt,
-            post_sz,
-            self.ip,
-            at,
-            "",
-        ):
-            t = "upload blocked by xau server config"
-            self.log(t, 1)
-            wunlink(self.log, path, vfs.flags)
-            raise Pebkac(403, t)
+        if xau:
+            vp = vjoin(self.vpath, fn) if nameless else self.vpath
+            hr = runhook(
+                self.log,
+                self.conn.hsrv.broker,
+                None,
+                "xau.http.dump",
+                xau,
+                path,
+                vp,
+                self.host,
+                self.uname,
+                self.asrv.vfs.get_perms(self.vpath, self.uname),
+                mt,
+                post_sz,
+                self.ip,
+                at,
+                "",
+            )
+            if not hr:
+                t = "upload blocked by xau server config"
+                self.log(t, 1)
+                wunlink(self.log, path, vfs.flags)
+                raise Pebkac(403, t)
+            if hr.get("reloc"):
+                x = pathmod(self.asrv.vfs, path, vp, hr["reloc"])
+                if x:
+                    if self.args.hook_v:
+                        log_reloc(self.log, hr["reloc"], x, path, vp, fn, vfs, rem)
+                    fdir, self.vpath, fn, (vfs, rem) = x
+                    bos.makedirs(fdir)
+                    path2 = os.path.join(fdir, fn)
+                    atomic_move(self.log, path, path2, vfs.flags)
+                    path = path2
+                    if not nameless:
+                        self.vpath = vjoin(self.vpath, fn)
+            sz = bos.path.getsize(path)
+        else:
+            sz = post_sz
 
         vfs, rem = vfs.get_dbv(rem)
         self.conn.hsrv.broker.say(
@@ -1907,7 +1972,7 @@ class HttpCli(object):
                 alg,
                 self.args.fk_salt,
                 path,
-                post_sz,
+                sz,
                 0 if ANYWIN else bos.stat(path).st_ino,
             )[: vfs.flags["fk"]]
 
@@ -2210,13 +2275,21 @@ class HttpCli(object):
             raise Pebkac(400, "need hash and wark headers for binary POST")
 
         chashes = [x.strip() for x in chashes]
+        if len(chashes) == 3 and len(chashes[1]) == 1:
+            # the first hash, then length of consecutive hashes,
+            # then a list of stitched hashes as one long string
+            clen = int(chashes[1])
+            siblings = chashes[2]
+            chashes = [chashes[0]]
+            for n in range(0, len(siblings), clen):
+                chashes.append(siblings[n : n + clen])
 
         vfs, _ = self.asrv.vfs.get(self.vpath, self.uname, False, True)
         ptop = (vfs.dbv or vfs).realpath
 
         x = self.conn.hsrv.broker.ask("up2k.handle_chunks", ptop, wark, chashes)
         response = x.get()
-        chunksize, cstarts, path, lastmod, sprs = response
+        chashes, chunksize, cstarts, path, lastmod, sprs = response
         maxsize = chunksize * len(chashes)
         cstart0 = cstarts[0]
 
@@ -2524,18 +2597,15 @@ class HttpCli(object):
                 fname = sanitize_fn(
                     p_file or "", "", [".prologue.html", ".epilogue.html"]
                 )
+                abspath = os.path.join(fdir, fname)
+                suffix = "-%.6f-%s" % (time.time(), dip)
                 if p_file and not nullwrite:
                     if rnd:
                         fname = rand_name(fdir, fname, rnd)
 
-                    if not bos.path.isdir(fdir):
-                        raise Pebkac(404, "that folder does not exist")
-
-                    suffix = "-{:.6f}-{}".format(time.time(), dip)
                     open_args = {"fdir": fdir, "suffix": suffix}
 
                     if "replace" in self.uparam:
-                        abspath = os.path.join(fdir, fname)
                         if not self.can_delete:
                             self.log("user not allowed to overwrite with ?replace")
                         elif bos.path.exists(abspath):
@@ -2545,6 +2615,58 @@ class HttpCli(object):
                             except:
                                 t = "toctou while deleting for ?replace: %s"
                             self.log(t % (abspath,))
+                else:
+                    open_args = {}
+                    tnam = fname = os.devnull
+                    fdir = abspath = ""
+
+                if xbu:
+                    at = time.time() - lifetime
+                    hr = runhook(
+                        self.log,
+                        self.conn.hsrv.broker,
+                        None,
+                        "xbu.http.bup",
+                        xbu,
+                        abspath,
+                        vjoin(upload_vpath, fname),
+                        self.host,
+                        self.uname,
+                        self.asrv.vfs.get_perms(upload_vpath, self.uname),
+                        at,
+                        0,
+                        self.ip,
+                        at,
+                        "",
+                    )
+                    if not hr:
+                        t = "upload blocked by xbu server config"
+                        self.log(t, 1)
+                        raise Pebkac(403, t)
+                    if hr.get("reloc"):
+                        zs = vjoin(upload_vpath, fname)
+                        x = pathmod(self.asrv.vfs, abspath, zs, hr["reloc"])
+                        if x:
+                            if self.args.hook_v:
+                                log_reloc(
+                                    self.log,
+                                    hr["reloc"],
+                                    x,
+                                    abspath,
+                                    zs,
+                                    fname,
+                                    vfs,
+                                    rem,
+                                )
+                            fdir, upload_vpath, fname, (vfs, rem) = x
+                            abspath = os.path.join(fdir, fname)
+                            if nullwrite:
+                                fdir = abspath = ""
+                            else:
+                                open_args["fdir"] = fdir
+
+                if p_file and not nullwrite:
+                    bos.makedirs(fdir)
 
                     # reserve destination filename
                     with ren_open(fname, "wb", fdir=fdir, suffix=suffix) as zfw:
@@ -2560,26 +2682,6 @@ class HttpCli(object):
                     tnam = fname = os.devnull
                     fdir = abspath = ""
 
-                if xbu:
-                    at = time.time() - lifetime
-                    if not runhook(
-                        self.log,
-                        xbu,
-                        abspath,
-                        self.vpath,
-                        self.host,
-                        self.uname,
-                        self.asrv.vfs.get_perms(self.vpath, self.uname),
-                        at,
-                        0,
-                        self.ip,
-                        at,
-                        "",
-                    ):
-                        t = "upload blocked by xbu server config"
-                        self.log(t, 1)
-                        raise Pebkac(403, t)
-
                 if lim:
                     lim.chk_bup(self.ip)
                     lim.chk_nup(self.ip)
@@ -2622,29 +2724,58 @@ class HttpCli(object):
 
                     tabspath = ""
 
+                    at = time.time() - lifetime
+                    if xau:
+                        hr = runhook(
+                            self.log,
+                            self.conn.hsrv.broker,
+                            None,
+                            "xau.http.bup",
+                            xau,
+                            abspath,
+                            vjoin(upload_vpath, fname),
+                            self.host,
+                            self.uname,
+                            self.asrv.vfs.get_perms(upload_vpath, self.uname),
+                            at,
+                            sz,
+                            self.ip,
+                            at,
+                            "",
+                        )
+                        if not hr:
+                            t = "upload blocked by xau server config"
+                            self.log(t, 1)
+                            wunlink(self.log, abspath, vfs.flags)
+                            raise Pebkac(403, t)
+                        if hr.get("reloc"):
+                            zs = vjoin(upload_vpath, fname)
+                            x = pathmod(self.asrv.vfs, abspath, zs, hr["reloc"])
+                            if x:
+                                if self.args.hook_v:
+                                    log_reloc(
+                                        self.log,
+                                        hr["reloc"],
+                                        x,
+                                        abspath,
+                                        zs,
+                                        fname,
+                                        vfs,
+                                        rem,
+                                    )
+                                fdir, upload_vpath, fname, (vfs, rem) = x
+                                ap2 = os.path.join(fdir, fname)
+                                if nullwrite:
+                                    fdir = ap2 = ""
+                                else:
+                                    bos.makedirs(fdir)
+                                    atomic_move(self.log, abspath, ap2, vfs.flags)
+                                abspath = ap2
+                        sz = bos.path.getsize(abspath)
+
                     files.append(
                         (sz, sha_hex, sha_b64, p_file or "(discarded)", fname, abspath)
                     )
-                    at = time.time() - lifetime
-                    if xau and not runhook(
-                        self.log,
-                        xau,
-                        abspath,
-                        self.vpath,
-                        self.host,
-                        self.uname,
-                        self.asrv.vfs.get_perms(self.vpath, self.uname),
-                        at,
-                        sz,
-                        self.ip,
-                        at,
-                        "",
-                    ):
-                        t = "upload blocked by xau server config"
-                        self.log(t, 1)
-                        wunlink(self.log, abspath, vfs.flags)
-                        raise Pebkac(403, t)
-
                     dbv, vrem = vfs.get_dbv(rem)
                     self.conn.hsrv.broker.say(
                         "up2k.hash_file",
@@ -2700,13 +2831,14 @@ class HttpCli(object):
         for sz, sha_hex, sha_b64, ofn, lfn, ap in files:
             vsuf = ""
             if (self.can_read or self.can_upget) and "fk" in vfs.flags:
+                st = bos.stat(ap)
                 alg = 2 if "fka" in vfs.flags else 1
                 vsuf = "?k=" + self.gen_fk(
                     alg,
                     self.args.fk_salt,
                     ap,
-                    sz,
-                    0 if ANYWIN or not ap else bos.stat(ap).st_ino,
+                    st.st_size,
+                    0 if ANYWIN or not ap else st.st_ino,
                 )[: vfs.flags["fk"]]
 
             if "media" in self.uparam or "medialinks" in vfs.flags:
@@ -2873,6 +3005,9 @@ class HttpCli(object):
         if xbu:
             if not runhook(
                 self.log,
+                self.conn.hsrv.broker,
+                None,
+                "xbu.http.txt",
                 xbu,
                 fp,
                 self.vpath,
@@ -2912,6 +3047,9 @@ class HttpCli(object):
         xau = vfs.flags.get("xau")
         if xau and not runhook(
             self.log,
+            self.conn.hsrv.broker,
+            None,
+            "xau.http.txt",
             xau,
             fp,
             self.vpath,
@@ -2952,7 +3090,7 @@ class HttpCli(object):
         return True
 
     def _chk_lastmod(self, file_ts: int) -> tuple[str, bool]:
-        file_lastmod = formatdate(file_ts, usegmt=True)
+        file_lastmod = formatdate(file_ts)
         cli_lastmod = self.headers.get("if-modified-since")
         if cli_lastmod:
             try:
@@ -3034,8 +3172,8 @@ class HttpCli(object):
             for n, fn in enumerate([".prologue.html", ".epilogue.html"]):
                 if lnames is not None and fn not in lnames:
                     continue
-                fn = os.path.join(abspath, fn)
-                if bos.path.exists(fn):
+                fn = "%s/%s" % (abspath, fn)
+                if bos.path.isfile(fn):
                     with open(fsenc(fn), "rb") as f:
                         logues[n] = f.read().decode("utf-8")
                     if "exp" in vn.flags:
@@ -3053,7 +3191,7 @@ class HttpCli(object):
                 fns = []
 
             for fn in fns:
-                fn = os.path.join(abspath, fn)
+                fn = "%s/%s" % (abspath, fn)
                 if bos.path.isfile(fn):
                     with open(fsenc(fn), "rb") as f:
                         readme = f.read().decode("utf-8")
@@ -3588,7 +3726,7 @@ class HttpCli(object):
         # (useragent-sniffing kinshi due to caching proxies)
         mime, ico = self.ico.get(txt, not small, "raster" in self.uparam)
 
-        lm = formatdate(self.E.t0, usegmt=True)
+        lm = formatdate(self.E.t0)
         self.reply(ico, mime=mime, headers={"Last-Modified": lm})
         return True
 
@@ -3667,6 +3805,11 @@ class HttpCli(object):
             "arg_base": arg_base,
         }
 
+        if self.args.js_other and "js" not in targs:
+            zs = self.args.js_other
+            zs += "&" if "?" in zs else "?"
+            targs["js"] = zs
+
         zfv = self.vn.flags.get("html_head")
         if zfv:
             targs["this"] = self
@@ -4144,7 +4287,7 @@ class HttpCli(object):
         if self.args.no_mv:
             raise Pebkac(403, "the rename/move feature is disabled in server config")
 
-        x = self.conn.hsrv.broker.ask("up2k.handle_mv", self.uname, vsrc, vdst)
+        x = self.conn.hsrv.broker.ask("up2k.handle_mv", self.uname, self.ip, vsrc, vdst)
         self.loud_reply(x.get(), status=201)
         return True
 

copyparty/httpconn.py

@@ -9,6 +9,9 @@ import threading  # typechk
 import time
 
 try:
+    if os.environ.get("PRTY_NO_TLS"):
+        raise Exception()
+
     HAVE_SSL = True
     import ssl
 except:

copyparty/httpsrv.py

@@ -12,7 +12,7 @@ import time
 
 import queue
 
-from .__init__ import ANYWIN, CORES, EXE, MACOS, TYPE_CHECKING, EnvParams, unicode
+from .__init__ import ANYWIN, CORES, EXE, MACOS, PY2, TYPE_CHECKING, EnvParams, unicode
 
 try:
     MNFE = ModuleNotFoundError
@@ -84,6 +84,12 @@ if TYPE_CHECKING:
 if True:  # pylint: disable=using-constant-test
     from typing import Any, Optional
 
+if PY2:
+    range = xrange  # type: ignore
+
+if not hasattr(socket, "AF_UNIX"):
+    setattr(socket, "AF_UNIX", -9001)
+
 
 class HttpSrv(object):
     """
@@ -240,15 +246,24 @@ class HttpSrv(object):
                     return
 
     def listen(self, sck: socket.socket, nlisteners: int) -> None:
+        tcp = sck.family != socket.AF_UNIX
+
         if self.args.j != 1:
             # lost in the pickle; redefine
             if not ANYWIN or self.args.reuseaddr:
                 sck.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
 
-            sck.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
+            if tcp:
+                sck.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
+
             sck.settimeout(None)  # < does not inherit, ^ opts above do
 
-        ip, port = sck.getsockname()[:2]
+        if tcp:
+            ip, port = sck.getsockname()[:2]
+        else:
+            ip = re.sub(r"\.[0-9]+$", "", sck.getsockname().split("/")[-1])
+            port = 0
+
         self.srvs.append(sck)
         self.bound.add((ip, port))
         self.nclimax = math.ceil(self.args.nc * 1.0 / nlisteners)
@@ -260,10 +275,19 @@ class HttpSrv(object):
 
     def thr_listen(self, srv_sck: socket.socket) -> None:
         """listens on a shared tcp server"""
-        ip, port = srv_sck.getsockname()[:2]
         fno = srv_sck.fileno()
-        hip = "[{}]".format(ip) if ":" in ip else ip
-        msg = "subscribed @ {}:{}  f{} p{}".format(hip, port, fno, os.getpid())
+        if srv_sck.family == socket.AF_UNIX:
+            ip = re.sub(r"\.[0-9]+$", "", srv_sck.getsockname())
+            msg = "subscribed @ %s  f%d p%d" % (ip, fno, os.getpid())
+            ip = ip.split("/")[-1]
+            port = 0
+            tcp = False
+        else:
+            tcp = True
+            ip, port = srv_sck.getsockname()[:2]
+            hip = "[%s]" % (ip,) if ":" in ip else ip
+            msg = "subscribed @ %s:%d  f%d p%d" % (hip, port, fno, os.getpid())
+
         self.log(self.name, msg)
 
         Daemon(self.broker.say, "sig-hsrv-up1", ("cb_httpsrv_up",))
@@ -335,11 +359,13 @@ class HttpSrv(object):
 
             try:
                 sck, saddr = srv_sck.accept()
-                cip = unicode(saddr[0])
-                if cip.startswith("::ffff:"):
-                    cip = cip[7:]
-
-                addr = (cip, saddr[1])
+                if tcp:
+                    cip = unicode(saddr[0])
+                    if cip.startswith("::ffff:"):
+                        cip = cip[7:]
+                    addr = (cip, saddr[1])
+                else:
+                    addr = (ip, sck.fileno())
             except (OSError, socket.error) as ex:
                 if self.stopping:
                     break

copyparty/ico.py

@@ -74,7 +74,7 @@ class Ico(object):
                 try:
                     _, _, tw, th = pb.textbbox((0, 0), ext)
                 except:
-                    tw, th = pb.textsize(ext)
+                    tw, th = pb.textsize(ext)  # type: ignore
 
                 tw += len(ext)
                 cw = tw // len(ext)

copyparty/mtag.py

@@ -32,6 +32,17 @@ if True:  # pylint: disable=using-constant-test
     from .util import NamedLogger, RootLogger
 
 
+try:
+    if os.environ.get("PRTY_NO_MUTAGEN"):
+        raise Exception()
+
+    from mutagen import version  # noqa: F401
+
+    HAVE_MUTAGEN = True
+except:
+    HAVE_MUTAGEN = False
+
+
 def have_ff(scmd: str) -> bool:
     if ANYWIN:
         scmd += ".exe"
@@ -48,8 +59,8 @@ def have_ff(scmd: str) -> bool:
         return bool(shutil.which(scmd))
 
 
-HAVE_FFMPEG = have_ff("ffmpeg")
-HAVE_FFPROBE = have_ff("ffprobe")
+HAVE_FFMPEG = not os.environ.get("PRTY_NO_FFMPEG") and have_ff("ffmpeg")
+HAVE_FFPROBE = not os.environ.get("PRTY_NO_FFPROBE") and have_ff("ffprobe")
 
 
 class MParser(object):
@@ -336,9 +347,7 @@ class MTag(object):
 
         if self.backend == "mutagen":
             self._get = self.get_mutagen
-            try:
-                from mutagen import version  # noqa: F401
-            except:
+            if not HAVE_MUTAGEN:
                 self.log("could not load Mutagen, trying FFprobe instead", c=3)
                 self.backend = "ffprobe"
 
@@ -578,7 +587,7 @@ class MTag(object):
                 continue
 
             if k == ".aq":
-                v /= 1000
+                v /= 1000  # type: ignore
 
             if k == "ac" and v.startswith("mp4a.40."):
                 v = "aac"

copyparty/pwhash.py

@@ -4,11 +4,21 @@ from __future__ import print_function, unicode_literals
 import argparse
 import base64
 import hashlib
+import os
 import sys
 import threading
 
 from .__init__ import unicode
 
+try:
+    if os.environ.get("PRTY_NO_ARGON2"):
+        raise Exception()
+
+    HAVE_ARGON2 = True
+    from argon2 import __version__ as argon2ver
+except:
+    HAVE_ARGON2 = False
+
 
 class PWHash(object):
     def __init__(self, args: argparse.Namespace):

copyparty/smbd.py

@@ -187,6 +187,8 @@ class SMB(object):
 
         debug('%s("%s", %s) %s @%s\033[K\033[0m', caller, vpath, str(a), perms, uname)
         vfs, rem = self.asrv.vfs.get(vpath, uname, *perms)
+        if not vfs.realpath:
+            raise Exception("unmapped vfs")
         return vfs, vfs.canonical(rem)
 
     def _listdir(self, vpath: str, *a: Any, **ka: Any) -> list[str]:
@@ -195,6 +197,8 @@ class SMB(object):
         uname = self._uname()
         # debug('listdir("%s", %s) @%s\033[K\033[0m', vpath, str(a), uname)
         vfs, rem = self.asrv.vfs.get(vpath, uname, False, False)
+        if not vfs.realpath:
+            raise Exception("unmapped vfs")
         _, vfs_ls, vfs_virt = vfs.ls(
             rem, uname, not self.args.no_scandir, [[False, False]]
         )
@@ -240,7 +244,21 @@ class SMB(object):
 
             xbu = vfs.flags.get("xbu")
             if xbu and not runhook(
-                self.nlog, xbu, ap, vpath, "", "", "", 0, 0, "1.7.6.2", 0, ""
+                self.nlog,
+                None,
+                self.hub.up2k,
+                "xbu.smb",
+                xbu,
+                ap,
+                vpath,
+                "",
+                "",
+                "",
+                0,
+                0,
+                "1.7.6.2",
+                time.time(),
+                "",
             ):
                 yeet("blocked by xbu server config: " + vpath)
 
@@ -297,7 +315,7 @@ class SMB(object):
             t = "blocked rename (no-move-acc %s): /%s @%s"
             yeet(t % (vfs1.axs.umove, vp1, uname))
 
-        self.hub.up2k.handle_mv(uname, vp1, vp2)
+        self.hub.up2k.handle_mv(uname, "1.7.6.2", vp1, vp2)
         try:
             bos.makedirs(ap2)
         except:

copyparty/ssdp.py

@@ -5,11 +5,11 @@ import errno
 import re
 import select
 import socket
-from email.utils import formatdate
+import time
 
 from .__init__ import TYPE_CHECKING
 from .multicast import MC_Sck, MCast
-from .util import CachedSet, html_escape, min_ex
+from .util import CachedSet, formatdate, html_escape, min_ex
 
 if TYPE_CHECKING:
     from .broker_util import BrokerCli
@@ -229,7 +229,7 @@ CONFIGID.UPNP.ORG: 1
 
 """
         v4 = srv.ip.replace("::ffff:", "")
-        zs = zs.format(formatdate(usegmt=True), v4, srv.hport, self.args.zsid)
+        zs = zs.format(formatdate(), v4, srv.hport, self.args.zsid)
         zb = zs[1:].replace("\n", "\r\n").encode("utf-8", "replace")
         srv.sck.sendto(zb, addr[:2])
 

copyparty/stolen/dnslib/dns.py

@@ -12,6 +12,12 @@ from .label import DNSBuffer, DNSLabel
 from .ranges import IP4, IP6, H, I, check_bytes
 
 
+try:
+    range = xrange
+except:
+    pass
+
+
 class DNSError(Exception):
     pass
 

copyparty/stolen/ifaddr/__init__.py

@@ -11,7 +11,21 @@ import os
 
 from ._shared import IP, Adapter
 
-if os.name == "nt":
+
+def nope(include_unconfigured=False):
+    return []
+
+
+try:
+    S390X = os.uname().machine == "s390x"
+except:
+    S390X = False
+
+
+if os.environ.get("PRTY_NO_IFADDR") or S390X:
+    # s390x deadlocks at libc.getifaddrs
+    get_adapters = nope
+elif os.name == "nt":
     from ._win32 import get_adapters
 elif os.name == "posix":
     from ._posix import get_adapters

copyparty/stolen/ifaddr/_shared.py

@@ -17,6 +17,7 @@ if not PY2:
     U: Callable[[str], str] = str
 else:
     U = unicode  # noqa: F821  # pylint: disable=undefined-variable,self-assigning-variable
+    range = xrange  # noqa: F821  # pylint: disable=undefined-variable,self-assigning-variable
 
 
 class Adapter(object):

copyparty/stolen/qrcodegen.py

@@ -16,6 +16,11 @@ if True:  # pylint: disable=using-constant-test
 
     from typing import Callable, List, Optional, Tuple, Union
 
+try:
+    range = xrange
+except:
+    pass
+
 
 def num_char_count_bits(ver: int) -> int:
     return 16 if (ver + 7) // 17 else 8

copyparty/sutil.py

@@ -6,7 +6,7 @@ import tempfile
 from datetime import datetime
 
 from .__init__ import CORES
-from .authsrv import AuthSrv, VFS
+from .authsrv import VFS, AuthSrv
 from .bos import bos
 from .th_cli import ThumbCli
 from .util import UTC, vjoin, vol_san

copyparty/svchub.py

@@ -28,18 +28,30 @@ if True:  # pylint: disable=using-constant-test
     import typing
     from typing import Any, Optional, Union
 
-from .__init__ import ANYWIN, EXE, MACOS, TYPE_CHECKING, E, EnvParams, unicode
+from .__init__ import ANYWIN, EXE, MACOS, PY2, TYPE_CHECKING, E, EnvParams, unicode
 from .authsrv import BAD_CFG, AuthSrv
 from .cert import ensure_cert
-from .mtag import HAVE_FFMPEG, HAVE_FFPROBE
+from .mtag import HAVE_FFMPEG, HAVE_FFPROBE, HAVE_MUTAGEN
+from .pwhash import HAVE_ARGON2
 from .tcpsrv import TcpSrv
-from .th_srv import HAVE_PIL, HAVE_VIPS, HAVE_WEBP, ThumbSrv
+from .th_srv import (
+    HAVE_AVIF,
+    HAVE_FFMPEG,
+    HAVE_FFPROBE,
+    HAVE_HEIF,
+    HAVE_PIL,
+    HAVE_VIPS,
+    HAVE_WEBP,
+    ThumbSrv,
+)
 from .up2k import Up2k
 from .util import (
     DEF_EXP,
     DEF_MTE,
     DEF_MTH,
     FFMPEG_URL,
+    HAVE_PSUTIL,
+    HAVE_SQLITE3,
     UTC,
     VERSIONS,
     Daemon,
@@ -65,6 +77,9 @@ if TYPE_CHECKING:
     except:
         pass
 
+if PY2:
+    range = xrange  # type: ignore
+
 
 class SvcHub(object):
     """
@@ -232,6 +247,8 @@ class SvcHub(object):
 
         self.up2k = Up2k(self)
 
+        self._feature_test()
+
         decs = {k: 1 for k in self.args.th_dec.split(",")}
         if not HAVE_VIPS:
             decs.pop("vips", None)
@@ -420,6 +437,58 @@ class SvcHub(object):
 
         Daemon(self.sd_notify, "sd-notify")
 
+    def _feature_test(self) -> None:
+        fok = []
+        fng = []
+        t_ff = "transcode audio, create spectrograms, video thumbnails"
+        to_check = [
+            (HAVE_SQLITE3, "sqlite", "file and media indexing"),
+            (HAVE_PIL, "pillow", "image thumbnails (plenty fast)"),
+            (HAVE_VIPS, "vips", "image thumbnails (faster, eats more ram)"),
+            (HAVE_WEBP, "pillow-webp", "create thumbnails as webp files"),
+            (HAVE_FFMPEG, "ffmpeg", t_ff + ", good-but-slow image thumbnails"),
+            (HAVE_FFPROBE, "ffprobe", t_ff + ", read audio/media tags"),
+            (HAVE_MUTAGEN, "mutagen", "read audio tags (ffprobe is better but slower)"),
+            (HAVE_ARGON2, "argon2", "secure password hashing (advanced users only)"),
+            (HAVE_HEIF, "pillow-heif", "read .heif images with pillow (rarely useful)"),
+            (HAVE_AVIF, "pillow-avif", "read .avif images with pillow (rarely useful)"),
+        ]
+        if ANYWIN:
+            to_check += [
+                (HAVE_PSUTIL, "psutil", "improved plugin cleanup  (rarely useful)")
+            ]
+
+        verbose = self.args.deps
+        if verbose:
+            self.log("dependencies", "")
+
+        for have, feat, what in to_check:
+            lst = fok if have else fng
+            lst.append((feat, what))
+            if verbose:
+                zi = 2 if have else 5
+                sgot = "found" if have else "missing"
+                t = "%7s: %s \033[36m(%s)"
+                self.log("dependencies", t % (sgot, feat, what), zi)
+
+        if verbose:
+            self.log("dependencies", "")
+            return
+
+        sok = ", ".join(x[0] for x in fok)
+        sng = ", ".join(x[0] for x in fng)
+
+        t = ""
+        if sok:
+            t += "OK: \033[32m" + sok
+        if sng:
+            if t:
+                t += ", "
+            t += "\033[0mNG: \033[35m" + sng
+
+        t += "\033[0m, see --deps"
+        self.log("dependencies", t, 6)
+
     def _check_env(self) -> None:
         try:
             files = os.listdir(E.cfg)

copyparty/szip.py

@@ -37,9 +37,7 @@ def dostime2unix(buf: bytes) -> int:
 
 
 def unixtime2dos(ts: int) -> bytes:
-    tt = time.gmtime(ts + 1)
-    dy, dm, dd, th, tm, ts = list(tt)[:6]
-
+    dy, dm, dd, th, tm, ts, _, _, _ = time.gmtime(ts + 1)
     bd = ((dy - 1980) << 9) + (dm << 5) + dd
     bt = (th << 11) + (tm << 5) + ts // 2
     try:

copyparty/tcpsrv.py

@@ -17,18 +17,23 @@ from .util import (
     E_UNREACH,
     HAVE_IPV6,
     IP6ALL,
+    VF_CAREFUL,
     Netdev,
+    atomic_move,
     min_ex,
     sunpack,
     termsize,
 )
 
 if True:
-    from typing import Generator
+    from typing import Generator, Union
 
 if TYPE_CHECKING:
     from .svchub import SvcHub
 
+if not hasattr(socket, "AF_UNIX"):
+    setattr(socket, "AF_UNIX", -9001)
+
 if not hasattr(socket, "IPPROTO_IPV6"):
     setattr(socket, "IPPROTO_IPV6", 41)
 
@@ -217,14 +222,29 @@ class TcpSrv(object):
         if self.args.qr or self.args.qrs:
             self.qr = self._qr(qr1, qr2)
 
+    def nlog(self, msg: str, c: Union[int, str] = 0) -> None:
+        self.log("tcpsrv", msg, c)
+
     def _listen(self, ip: str, port: int) -> None:
-        ipv = socket.AF_INET6 if ":" in ip else socket.AF_INET
+        if "unix:" in ip:
+            tcp = False
+            ipv = socket.AF_UNIX
+            ip = ip.split("unix:")[1]
+        elif ":" in ip:
+            tcp = True
+            ipv = socket.AF_INET6
+        else:
+            tcp = True
+            ipv = socket.AF_INET
+
         srv = socket.socket(ipv, socket.SOCK_STREAM)
 
         if not ANYWIN or self.args.reuseaddr:
             srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
 
-        srv.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
+        if tcp:
+            srv.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
+
         srv.settimeout(None)  # < does not inherit, ^ opts above do
 
         try:
@@ -236,8 +256,19 @@ class TcpSrv(object):
             srv.setsockopt(socket.SOL_IP, socket.IP_FREEBIND, 1)
 
         try:
-            srv.bind((ip, port))
-            sport = srv.getsockname()[1]
+            if tcp:
+                srv.bind((ip, port))
+            else:
+                if ANYWIN or self.args.rm_sck:
+                    if os.path.exists(ip):
+                        os.unlink(ip)
+                    srv.bind(ip)
+                else:
+                    tf = "%s.%d" % (ip, os.getpid())
+                    srv.bind(tf)
+                    atomic_move(self.nlog, tf, ip, VF_CAREFUL)
+
+            sport = srv.getsockname()[1] if tcp else port
             if port != sport:
                 # linux 6.0.16 lets you bind a port which is in use
                 # except it just gives you a random port instead
@@ -249,12 +280,23 @@ class TcpSrv(object):
             except:
                 pass
 
+            e = ""
             if ex.errno in E_ADDR_IN_USE:
                 e = "\033[1;31mport {} is busy on interface {}\033[0m".format(port, ip)
+                if not tcp:
+                    e = "\033[1;31munix-socket {} is busy\033[0m".format(ip)
             elif ex.errno in E_ADDR_NOT_AVAIL:
                 e = "\033[1;31minterface {} does not exist\033[0m".format(ip)
-            else:
+
+            if not e:
+                if not tcp:
+                    t = "\n\n\n  NOTE: this crash may be due to a unix-socket bug; try --rm-sck\n"
+                    self.log("tcpsrv", t, 2)
                 raise
+
+            if not tcp and not self.args.rm_sck:
+                e += "; maybe this is a bug? try --rm-sck"
+
             raise Exception(e)
 
     def run(self) -> None:
@@ -262,7 +304,14 @@ class TcpSrv(object):
         bound: list[tuple[str, int]] = []
         srvs: list[socket.socket] = []
         for srv in self.srv:
-            ip, port = srv.getsockname()[:2]
+            if srv.family == socket.AF_UNIX:
+                tcp = False
+                ip = re.sub(r"\.[0-9]+$", "", srv.getsockname())
+                port = 0
+            else:
+                tcp = True
+                ip, port = srv.getsockname()[:2]
+
             if ip == IP6ALL:
                 ip = "::"  # jython
 
@@ -294,8 +343,12 @@ class TcpSrv(object):
             bound.append((ip, port))
             srvs.append(srv)
             fno = srv.fileno()
-            hip = "[{}]".format(ip) if ":" in ip else ip
-            msg = "listening @ {}:{}  f{} p{}".format(hip, port, fno, os.getpid())
+            if tcp:
+                hip = "[{}]".format(ip) if ":" in ip else ip
+                msg = "listening @ {}:{}  f{} p{}".format(hip, port, fno, os.getpid())
+            else:
+                msg = "listening @ {}  f{} p{}".format(ip, fno, os.getpid())
+
             self.log("tcpsrv", msg)
             if self.args.q:
                 print(msg)
@@ -348,6 +401,8 @@ class TcpSrv(object):
     def detect_interfaces(self, listen_ips: list[str]) -> dict[str, Netdev]:
         from .stolen.ifaddr import get_adapters
 
+        listen_ips = [x for x in listen_ips if "unix:" not in x]
+
         nics = get_adapters(True)
         eps: dict[str, Netdev] = {}
         for nic in nics:

copyparty/tftpd.py

@@ -36,7 +36,7 @@ from partftpy.TftpShared import TftpException
 from .__init__ import EXE, PY2, TYPE_CHECKING
 from .authsrv import VFS
 from .bos import bos
-from .util import BytesIO, Daemon, ODict, exclude_dotfiles, min_ex, runhook, undot
+from .util import UTC, BytesIO, Daemon, ODict, exclude_dotfiles, min_ex, runhook, undot
 
 if True:  # pylint: disable=using-constant-test
     from typing import Any, Union
@@ -44,6 +44,9 @@ if True:  # pylint: disable=using-constant-test
 if TYPE_CHECKING:
     from .svchub import SvcHub
 
+if PY2:
+    range = xrange  # type: ignore
+
 
 lg = logging.getLogger("tftp")
 debug, info, warning, error = (lg.debug, lg.info, lg.warning, lg.error)
@@ -163,9 +166,16 @@ class Tftpd(object):
         if "::" in ips:
             ips.append("0.0.0.0")
 
+        ips = [x for x in ips if "unix:" not in x]
+
         if self.args.tftp4:
             ips = [x for x in ips if ":" not in x]
 
+        if not ips:
+            t = "cannot start tftp-server; no compatible IPs in -i"
+            self.nlog(t, 1)
+            return
+
         ips = list(ODict.fromkeys(ips))  # dedup
 
         for ip in ips:
@@ -241,6 +251,8 @@ class Tftpd(object):
 
         debug('%s("%s", %s) %s\033[K\033[0m', caller, vpath, str(a), perms)
         vfs, rem = self.asrv.vfs.get(vpath, "*", *perms)
+        if not vfs.realpath:
+            raise Exception("unmapped vfs")
         return vfs, vfs.canonical(rem)
 
     def _ls(self, vpath: str, raddress: str, rport: int, force=False) -> Any:
@@ -262,7 +274,7 @@ class Tftpd(object):
         dirs1 = [(v.st_mtime, v.st_size, k + "/") for k, v in vfs_ls if k in dnames]
         fils1 = [(v.st_mtime, v.st_size, k) for k, v in vfs_ls if k not in dnames]
         real1 = dirs1 + fils1
-        realt = [(datetime.fromtimestamp(mt), sz, fn) for mt, sz, fn in real1]
+        realt = [(datetime.fromtimestamp(mt, UTC), sz, fn) for mt, sz, fn in real1]
         reals = [
             (
                 "%04d-%02d-%02d %02d:%02d:%02d"
@@ -328,7 +340,21 @@ class Tftpd(object):
 
             xbu = vfs.flags.get("xbu")
             if xbu and not runhook(
-                self.nlog, xbu, ap, vpath, "", "", "", 0, 0, "8.3.8.7", 0, ""
+                self.nlog,
+                None,
+                self.hub.up2k,
+                "xbu.tftpd",
+                xbu,
+                ap,
+                vpath,
+                "",
+                "",
+                "",
+                0,
+                0,
+                "8.3.8.7",
+                time.time(),
+                "",
             ):
                 yeet("blocked by xbu server config: " + vpath)
 
@@ -336,7 +362,7 @@ class Tftpd(object):
             return self._ls(vpath, "", 0, True)
 
         if not a:
-            a = [self.args.iobuf]
+            a = (self.args.iobuf,)
 
         return open(ap, mode, *a, **ka)
 

copyparty/th_srv.py

@@ -12,7 +12,7 @@ import time
 
 from queue import Queue
 
-from .__init__ import ANYWIN, TYPE_CHECKING
+from .__init__ import ANYWIN, PY2, TYPE_CHECKING
 from .authsrv import VFS
 from .bos import bos
 from .mtag import HAVE_FFMPEG, HAVE_FFPROBE, au_unpk, ffprobe
@@ -38,6 +38,9 @@ if True:  # pylint: disable=using-constant-test
 if TYPE_CHECKING:
     from .svchub import SvcHub
 
+if PY2:
+    range = xrange  # type: ignore
+
 HAVE_PIL = False
 HAVE_PILF = False
 HAVE_HEIF = False
@@ -45,22 +48,34 @@ HAVE_AVIF = False
 HAVE_WEBP = False
 
 try:
+    if os.environ.get("PRTY_NO_PIL"):
+        raise Exception()
+
     from PIL import ExifTags, Image, ImageFont, ImageOps
 
     HAVE_PIL = True
     try:
+        if os.environ.get("PRTY_NO_PILF"):
+            raise Exception()
+
         ImageFont.load_default(size=16)
         HAVE_PILF = True
     except:
         pass
 
     try:
+        if os.environ.get("PRTY_NO_PIL_WEBP"):
+            raise Exception()
+
         Image.new("RGB", (2, 2)).save(BytesIO(), format="webp")
         HAVE_WEBP = True
     except:
         pass
 
     try:
+        if os.environ.get("PRTY_NO_PIL_HEIF"):
+            raise Exception()
+
         from pyheif_pillow_opener import register_heif_opener
 
         register_heif_opener()
@@ -69,6 +84,9 @@ try:
         pass
 
     try:
+        if os.environ.get("PRTY_NO_PIL_AVIF"):
+            raise Exception()
+
         import pillow_avif  # noqa: F401  # pylint: disable=unused-import
 
         HAVE_AVIF = True
@@ -80,6 +98,9 @@ except:
     pass
 
 try:
+    if os.environ.get("PRTY_NO_VIPS"):
+        raise Exception()
+
     HAVE_VIPS = True
     import pyvips
 

copyparty/u2idx.py

@@ -8,7 +8,7 @@ import threading
 import time
 from operator import itemgetter
 
-from .__init__ import ANYWIN, TYPE_CHECKING, unicode
+from .__init__ import ANYWIN, PY2, TYPE_CHECKING, unicode
 from .authsrv import LEELOO_DALLAS, VFS
 from .bos import bos
 from .up2k import up2k_wark_from_hashlist
@@ -38,6 +38,9 @@ if True:  # pylint: disable=using-constant-test
 if TYPE_CHECKING:
     from .httpsrv import HttpSrv
 
+if PY2:
+    range = xrange  # type: ignore
+
 
 class U2idx(object):
     def __init__(self, hsrv: "HttpSrv") -> None:

copyparty/up2k.py

@@ -28,8 +28,8 @@ from .fsutil import Fstab
 from .mtag import MParser, MTag
 from .util import (
     HAVE_SQLITE3,
-    VF_CAREFUL,
     SYMTIME,
+    VF_CAREFUL,
     Daemon,
     MTHash,
     Pebkac,
@@ -46,6 +46,7 @@ from .util import (
     hidedir,
     humansize,
     min_ex,
+    pathmod,
     quotep,
     rand_name,
     ren_open,
@@ -165,6 +166,7 @@ class Up2k(object):
         self.xiu_ptn = re.compile(r"(?:^|,)i([0-9]+)")
         self.xiu_busy = False  # currently running hook
         self.xiu_asleep = True  # needs rescan_cond poke to schedule self
+        self.fx_backlog: list[tuple[str, dict[str, str], str]] = []
 
         self.cur: dict[str, "sqlite3.Cursor"] = {}
         self.mem_cur = None
@@ -2544,7 +2546,7 @@ class Up2k(object):
             if self.mutex.acquire(timeout=10):
                 got_lock = True
                 with self.reg_mutex:
-                    return self._handle_json(cj)
+                    ret = self._handle_json(cj)
             else:
                 t = "cannot receive uploads right now;\nserver busy with {}.\nPlease wait; the client will retry..."
                 raise Pebkac(503, t.format(self.blocked or "[unknown]"))
@@ -2552,11 +2554,16 @@ class Up2k(object):
             if not PY2:
                 raise
             with self.mutex, self.reg_mutex:
-                return self._handle_json(cj)
+                ret = self._handle_json(cj)
         finally:
             if got_lock:
                 self.mutex.release()
 
+        if self.fx_backlog:
+            self.do_fx_backlog()
+
+        return ret
+
     def _handle_json(self, cj: dict[str, Any]) -> dict[str, Any]:
         ptop = cj["ptop"]
         if not self.register_vpath(ptop, cj["vcfg"]):
@@ -2750,7 +2757,7 @@ class Up2k(object):
                         job = deepcopy(job)
                         job["wark"] = wark
                         job["at"] = cj.get("at") or time.time()
-                        for k in "lmod ptop vtop prel host user addr".split():
+                        for k in "lmod ptop vtop prel name host user addr".split():
                             job[k] = cj.get(k) or ""
 
                         pdir = djoin(cj["ptop"], cj["prel"])
@@ -2758,28 +2765,43 @@ class Up2k(object):
                             job["name"] = rand_name(
                                 pdir, cj["name"], vfs.flags["nrand"]
                             )
-                        else:
-                            job["name"] = self._untaken(pdir, cj, now)
 
                         dst = djoin(job["ptop"], job["prel"], job["name"])
                         xbu = vfs.flags.get("xbu")
-                        if xbu and not runhook(
-                            self.log,
-                            xbu,  # type: ignore
-                            dst,
-                            job["vtop"],
-                            job["host"],
-                            job["user"],
-                            self.asrv.vfs.get_perms(job["vtop"], job["user"]),
-                            job["lmod"],
-                            job["size"],
-                            job["addr"],
-                            job["at"],
-                            "",
-                        ):
-                            t = "upload blocked by xbu server config: {}".format(dst)
-                            self.log(t, 1)
-                            raise Pebkac(403, t)
+                        if xbu:
+                            vp = djoin(job["vtop"], job["prel"], job["name"])
+                            hr = runhook(
+                                self.log,
+                                None,
+                                self,
+                                "xbu.up2k.dupe",
+                                xbu,  # type: ignore
+                                dst,
+                                vp,
+                                job["host"],
+                                job["user"],
+                                self.asrv.vfs.get_perms(job["vtop"], job["user"]),
+                                job["lmod"],
+                                job["size"],
+                                job["addr"],
+                                job["at"],
+                                "",
+                            )
+                            if not hr:
+                                t = "upload blocked by xbu server config: %s" % (dst,)
+                                self.log(t, 1)
+                                raise Pebkac(403, t)
+                            if hr.get("reloc"):
+                                x = pathmod(self.asrv.vfs, dst, vp, hr["reloc"])
+                                if x:
+                                    pdir, _, job["name"], (vfs, rem) = x
+                                    dst = os.path.join(pdir, job["name"])
+                                    job["ptop"] = vfs.realpath
+                                    job["vtop"] = vfs.vpath
+                                    job["prel"] = rem
+                                    bos.makedirs(pdir)
+
+                        job["name"] = self._untaken(pdir, job, now)
 
                         if not self.args.nw:
                             dvf: dict[str, Any] = vfs.flags
@@ -2851,11 +2873,11 @@ class Up2k(object):
                 # one chunk may occur multiple times in a file;
                 # filter to unique values for the list of missing chunks
                 # (preserve order to reduce disk thrashing)
-                lut = {}
+                lut = set()
                 for k in cj["hash"]:
                     if k not in lut:
                         job["need"].append(k)
-                        lut[k] = 1
+                        lut.add(k)
 
                 try:
                     self._new_upload(job)
@@ -3015,7 +3037,7 @@ class Up2k(object):
 
     def handle_chunks(
         self, ptop: str, wark: str, chashes: list[str]
-    ) -> tuple[list[int], list[list[int]], str, float, bool]:
+    ) -> tuple[list[str], int, list[list[int]], str, float, bool]:
         with self.mutex, self.reg_mutex:
             self.db_act = self.vol_act[ptop] = time.time()
             job = self.registry[ptop].get(wark)
@@ -3024,12 +3046,37 @@ class Up2k(object):
                 self.log("unknown wark [{}], known: {}".format(wark, known))
                 raise Pebkac(400, "unknown wark" + SSEELOG)
 
+            if len(chashes) > 1 and len(chashes[1]) < 44:
+                # first hash is full-length; expand remaining ones
+                uniq = []
+                lut = set()
+                for chash in job["hash"]:
+                    if chash not in lut:
+                        uniq.append(chash)
+                        lut.add(chash)
+                try:
+                    nchunk = uniq.index(chashes[0])
+                except:
+                    raise Pebkac(400, "unknown chunk0 [%s]" % (chashes[0]))
+                expanded = [chashes[0]]
+                for prefix in chashes[1:]:
+                    nchunk += 1
+                    chash = uniq[nchunk]
+                    if not chash.startswith(prefix):
+                        t = "next sibling chunk does not start with expected prefix [%s]: [%s]"
+                        raise Pebkac(400, t % (prefix, chash))
+                    expanded.append(chash)
+                chashes = expanded
+
             for chash in chashes:
                 if chash not in job["need"]:
                     msg = "chash = {} , need:\n".format(chash)
                     msg += "\n".join(job["need"])
                     self.log(msg)
-                    raise Pebkac(400, "already got that (%s) but thanks??" % (chash,))
+                    t = "already got that (%s) but thanks??"
+                    if chash not in job["hash"]:
+                        t = "unknown chunk wtf: %s"
+                    raise Pebkac(400, t % (chash,))
 
                 if chash in job["busy"]:
                     nh = len(job["hash"])
@@ -3037,9 +3084,11 @@ class Up2k(object):
                     t = "that chunk is already being written to:\n  {}\n  {} {}/{}\n  {}"
                     raise Pebkac(400, t.format(wark, chash, idx, nh, job["name"]))
 
+            assert chash  # type: ignore
             chunksize = up2k_chunksize(job["size"])
 
             coffsets = []
+            nchunks = []
             for chash in chashes:
                 nchunk = [n for n, v in enumerate(job["hash"]) if v == chash]
                 if not nchunk:
@@ -3047,6 +3096,7 @@ class Up2k(object):
 
                 ofs = [chunksize * x for x in nchunk]
                 coffsets.append(ofs)
+                nchunks.append(nchunk)
 
             for ofs1, ofs2 in zip(coffsets, coffsets[1:]):
                 gap = (ofs2[0] - ofs1[0]) - chunksize
@@ -3058,16 +3108,16 @@ class Up2k(object):
 
             if not job["sprs"]:
                 cur_sz = bos.path.getsize(path)
-                if ofs[0] > cur_sz:
+                if coffsets[0][0] > cur_sz:
                     t = "please upload sequentially using one thread;\nserver filesystem does not support sparse files.\n  file: {}\n  chunk: {}\n  cofs: {}\n  flen: {}"
-                    t = t.format(job["name"], nchunk[0], ofs[0], cur_sz)
+                    t = t.format(job["name"], nchunks[0][0], coffsets[0][0], cur_sz)
                     raise Pebkac(400, t)
 
             job["busy"][chash] = 1
 
         job["poke"] = time.time()
 
-        return chunksize, coffsets, path, job["lmod"], job["sprs"]
+        return chashes, chunksize, coffsets, path, job["lmod"], job["sprs"]
 
     def release_chunks(self, ptop: str, wark: str, chashes: list[str]) -> bool:
         with self.reg_mutex:
@@ -3114,6 +3164,9 @@ class Up2k(object):
         with self.mutex, self.reg_mutex:
             self._finish_upload(ptop, wark)
 
+        if self.fx_backlog:
+            self.do_fx_backlog()
+
     def _finish_upload(self, ptop: str, wark: str) -> None:
         """mutex(main,reg) me"""
         try:
@@ -3307,25 +3360,30 @@ class Up2k(object):
 
         xau = False if skip_xau else vflags.get("xau")
         dst = djoin(ptop, rd, fn)
-        if xau and not runhook(
-            self.log,
-            xau,
-            dst,
-            djoin(vtop, rd, fn),
-            host,
-            usr,
-            self.asrv.vfs.get_perms(djoin(vtop, rd, fn), usr),
-            int(ts),
-            sz,
-            ip,
-            at or time.time(),
-            "",
-        ):
-            t = "upload blocked by xau server config"
-            self.log(t, 1)
-            wunlink(self.log, dst, vflags)
-            self.registry[ptop].pop(wark, None)
-            raise Pebkac(403, t)
+        if xau:
+            hr = runhook(
+                self.log,
+                None,
+                self,
+                "xau.up2k",
+                xau,
+                dst,
+                djoin(vtop, rd, fn),
+                host,
+                usr,
+                self.asrv.vfs.get_perms(djoin(vtop, rd, fn), usr),
+                ts,
+                sz,
+                ip,
+                at or time.time(),
+                "",
+            )
+            if not hr:
+                t = "upload blocked by xau server config"
+                self.log(t, 1)
+                wunlink(self.log, dst, vflags)
+                self.registry[ptop].pop(wark, None)
+                raise Pebkac(403, t)
 
         xiu = vflags.get("xiu")
         if xiu:
@@ -3509,6 +3567,9 @@ class Up2k(object):
                 if xbd:
                     if not runhook(
                         self.log,
+                        None,
+                        self,
+                        "xbd",
                         xbd,
                         abspath,
                         vpath,
@@ -3518,7 +3579,7 @@ class Up2k(object):
                         stl.st_mtime,
                         st.st_size,
                         ip,
-                        0,
+                        time.time(),
                         "",
                     ):
                         t = "delete blocked by xbd server config: {}"
@@ -3543,6 +3604,9 @@ class Up2k(object):
                 if xad:
                     runhook(
                         self.log,
+                        None,
+                        self,
+                        "xad",
                         xad,
                         abspath,
                         vpath,
@@ -3552,7 +3616,7 @@ class Up2k(object):
                         stl.st_mtime,
                         st.st_size,
                         ip,
-                        0,
+                        time.time(),
                         "",
                     )
 
@@ -3568,7 +3632,7 @@ class Up2k(object):
 
         return n_files, ok + ok2, ng + ng2
 
-    def handle_mv(self, uname: str, svp: str, dvp: str) -> str:
+    def handle_mv(self, uname: str, ip: str, svp: str, dvp: str) -> str:
         if svp == dvp or dvp.startswith(svp + "/"):
             raise Pebkac(400, "mv: cannot move parent into subfolder")
 
@@ -3585,7 +3649,7 @@ class Up2k(object):
         if stat.S_ISREG(st.st_mode) or stat.S_ISLNK(st.st_mode):
             with self.mutex:
                 try:
-                    ret = self._mv_file(uname, svp, dvp, curs)
+                    ret = self._mv_file(uname, ip, svp, dvp, curs)
                 finally:
                     for v in curs:
                         v.connection.commit()
@@ -3618,7 +3682,7 @@ class Up2k(object):
                             raise Pebkac(500, "mv: bug at {}, top {}".format(svpf, svp))
 
                         dvpf = dvp + svpf[len(svp) :]
-                        self._mv_file(uname, svpf, dvpf, curs)
+                        self._mv_file(uname, ip, svpf, dvpf, curs)
                 finally:
                     for v in curs:
                         v.connection.commit()
@@ -3643,7 +3707,7 @@ class Up2k(object):
         return "k"
 
     def _mv_file(
-        self, uname: str, svp: str, dvp: str, curs: set["sqlite3.Cursor"]
+        self, uname: str, ip: str, svp: str, dvp: str, curs: set["sqlite3.Cursor"]
     ) -> str:
         """mutex(main) me;  will mutex(reg)"""
         svn, srem = self.asrv.vfs.get(svp, uname, True, False, True)
@@ -3677,21 +3741,27 @@ class Up2k(object):
             except:
                 pass  # broken symlink; keep as-is
 
+        ftime = stl.st_mtime
+        fsize = st.st_size
+
         xbr = svn.flags.get("xbr")
         xar = dvn.flags.get("xar")
         if xbr:
             if not runhook(
                 self.log,
+                None,
+                self,
+                "xbr",
                 xbr,
                 sabs,
                 svp,
                 "",
                 uname,
                 self.asrv.vfs.get_perms(svp, uname),
-                stl.st_mtime,
-                st.st_size,
-                "",
-                0,
+                ftime,
+                fsize,
+                ip,
+                time.time(),
                 "",
             ):
                 t = "move blocked by xbr server config: {}".format(svp)
@@ -3719,16 +3789,19 @@ class Up2k(object):
             if xar:
                 runhook(
                     self.log,
+                    None,
+                    self,
+                    "xar.ln",
                     xar,
                     dabs,
                     dvp,
                     "",
                     uname,
                     self.asrv.vfs.get_perms(dvp, uname),
-                    0,
-                    0,
-                    "",
-                    0,
+                    ftime,
+                    fsize,
+                    ip,
+                    time.time(),
                     "",
                 )
 
@@ -3737,13 +3810,6 @@ class Up2k(object):
         c1, w, ftime_, fsize_, ip, at = self._find_from_vpath(svn.realpath, srem)
         c2 = self.cur.get(dvn.realpath)
 
-        if ftime_ is None:
-            ftime = stl.st_mtime
-            fsize = st.st_size
-        else:
-            ftime = ftime_
-            fsize = fsize_ or 0
-
         has_dupes = False
         if w:
             assert c1
@@ -3751,7 +3817,9 @@ class Up2k(object):
                 self._copy_tags(c1, c2, w)
 
             with self.reg_mutex:
-                has_dupes = self._forget_file(svn.realpath, srem, c1, w, is_xvol, fsize)
+                has_dupes = self._forget_file(
+                    svn.realpath, srem, c1, w, is_xvol, fsize_ or fsize
+                )
 
             if not is_xvol:
                 has_dupes = self._relink(w, svn.realpath, srem, dabs)
@@ -3821,7 +3889,7 @@ class Up2k(object):
 
             if is_link:
                 try:
-                    times = (int(time.time()), int(stl.st_mtime))
+                    times = (int(time.time()), int(ftime))
                     bos.utime(dabs, times, False)
                 except:
                     pass
@@ -3831,16 +3899,19 @@ class Up2k(object):
         if xar:
             runhook(
                 self.log,
+                None,
+                self,
+                "xar.mv",
                 xar,
                 dabs,
                 dvp,
                 "",
                 uname,
                 self.asrv.vfs.get_perms(dvp, uname),
-                0,
-                0,
-                "",
-                0,
+                ftime,
+                fsize,
+                ip,
+                time.time(),
                 "",
             )
 
@@ -4124,23 +4195,35 @@ class Up2k(object):
         xbu = self.flags[job["ptop"]].get("xbu")
         ap_chk = djoin(pdir, job["name"])
         vp_chk = djoin(job["vtop"], job["prel"], job["name"])
-        if xbu and not runhook(
-            self.log,
-            xbu,
-            ap_chk,
-            vp_chk,
-            job["host"],
-            job["user"],
-            self.asrv.vfs.get_perms(vp_chk, job["user"]),
-            int(job["lmod"]),
-            job["size"],
-            job["addr"],
-            int(job["t0"]),
-            "",
-        ):
-            t = "upload blocked by xbu server config: {}".format(vp_chk)
-            self.log(t, 1)
-            raise Pebkac(403, t)
+        if xbu:
+            hr = runhook(
+                self.log,
+                None,
+                self,
+                "xbu.up2k",
+                xbu,
+                ap_chk,
+                vp_chk,
+                job["host"],
+                job["user"],
+                self.asrv.vfs.get_perms(vp_chk, job["user"]),
+                job["lmod"],
+                job["size"],
+                job["addr"],
+                job["t0"],
+                "",
+            )
+            if not hr:
+                t = "upload blocked by xbu server config: {}".format(vp_chk)
+                self.log(t, 1)
+                raise Pebkac(403, t)
+            if hr.get("reloc"):
+                x = pathmod(self.asrv.vfs, ap_chk, vp_chk, hr["reloc"])
+                if x:
+                    pdir, _, job["name"], (vfs, rem) = x
+                    job["ptop"] = vfs.realpath
+                    job["vtop"] = vfs.vpath
+                    job["prel"] = rem
 
         tnam = job["name"] + ".PARTIAL"
         if self.args.dotpart:
@@ -4414,6 +4497,9 @@ class Up2k(object):
             with self.rescan_cond:
                 self.rescan_cond.notify_all()
 
+        if self.fx_backlog:
+            self.do_fx_backlog()
+
         return True
 
     def hash_file(
@@ -4445,6 +4531,48 @@ class Up2k(object):
             self.hashq.put(zt)
             self.n_hashq += 1
 
+    def do_fx_backlog(self):
+        with self.mutex, self.reg_mutex:
+            todo = self.fx_backlog
+            self.fx_backlog = []
+        for act, hr, req_vp in todo:
+            self.hook_fx(act, hr, req_vp)
+
+    def hook_fx(self, act: str, hr: dict[str, str], req_vp: str) -> None:
+        bad = [k for k in hr if k != "vp"]
+        if bad:
+            t = "got unsupported key in %s from hook: %s"
+            raise Exception(t % (act, bad))
+
+        for fvp in hr.get("vp") or []:
+            # expect vpath including filename; either absolute
+            # or relative to the client's vpath (request url)
+            if fvp.startswith("/"):
+                fvp, fn = vsplit(fvp[1:])
+                fvp = "/" + fvp
+            else:
+                fvp, fn = vsplit(fvp)
+
+            x = pathmod(self.asrv.vfs, "", req_vp, {"vp": fvp, "fn": fn})
+            if not x:
+                t = "hook_fx(%s): failed to resolve %s based on %s"
+                self.log(t % (act, fvp, req_vp))
+                continue
+
+            ap, rd, fn, (vn, rem) = x
+            vp = vjoin(rd, fn)
+            if not vp:
+                raise Exception("hook_fx: blank vp from pathmod")
+
+            if act == "idx":
+                rd = rd[len(vn.vpath) :].strip("/")
+                self.hash_file(
+                    vn.realpath, vn.vpath, vn.flags, rd, fn, "", time.time(), "", True
+                )
+
+            if act == "del":
+                self._handle_rm(LEELOO_DALLAS, "", vp, [], False, False)
+
     def shutdown(self) -> None:
         self.stop = True
 

copyparty/util.py

@@ -26,7 +26,6 @@ import threading
 import time
 import traceback
 from collections import Counter
-from email.utils import formatdate
 
 from ipaddress import IPv4Address, IPv4Network, IPv6Address, IPv6Network
 from queue import Queue
@@ -60,6 +59,10 @@ except:
     UTC = _UTC()
 
 
+if PY2:
+    range = xrange  # type: ignore
+
+
 if sys.version_info >= (3, 7) or (
     sys.version_info >= (3, 6) and platform.python_implementation() == "CPython"
 ):
@@ -99,6 +102,9 @@ except:
     pass
 
 try:
+    if os.environ.get("PRTY_NO_SQLITE"):
+        raise Exception()
+
     HAVE_SQLITE3 = True
     import sqlite3
 
@@ -107,6 +113,9 @@ except:
     HAVE_SQLITE3 = False
 
 try:
+    if os.environ.get("PRTY_NO_PSUTIL"):
+        raise Exception()
+
     HAVE_PSUTIL = True
     import psutil
 except:
@@ -137,10 +146,15 @@ if TYPE_CHECKING:
     import magic
 
     from .authsrv import VFS
+    from .broker_util import BrokerCli
+    from .up2k import Up2k
 
 FAKE_MP = False
 
 try:
+    if os.environ.get("PRTY_NO_MP"):
+        raise ImportError()
+
     import multiprocessing as mp
 
     # import multiprocessing.dummy as mp
@@ -159,6 +173,9 @@ else:
 
 
 try:
+    if os.environ.get("PRTY_NO_IPV6"):
+        raise Exception()
+
     socket.inet_pton(socket.AF_INET6, "::1")
     HAVE_IPV6 = True
 except:
@@ -794,7 +811,7 @@ class CachedSet(object):
 
         c = self.c = {k: v for k, v in self.c.items() if now - v < self.maxage}
         try:
-            self.oldest = c[min(c, key=c.get)]
+            self.oldest = c[min(c, key=c.get)]  # type: ignore
         except:
             self.oldest = now
 
@@ -1821,10 +1838,21 @@ def gen_filekey_dbg(
     return ret
 
 
+WKDAYS = "Mon Tue Wed Thu Fri Sat Sun".split()
+MONTHS = "Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec".split()
+RFC2822 = "%s, %02d %s %04d %02d:%02d:%02d GMT"
+
+
+def formatdate(ts: Optional[float] = None) -> str:
+    # gmtime ~= datetime.fromtimestamp(ts, UTC).timetuple()
+    y, mo, d, h, mi, s, wd, _, _ = time.gmtime(ts)
+    return RFC2822 % (WKDAYS[wd], d, MONTHS[mo - 1], y, h, mi, s)
+
+
 def gencookie(k: str, v: str, r: str, tls: bool, dur: int = 0, txt: str = "") -> str:
     v = v.replace("%", "%25").replace(";", "%3B")
     if dur:
-        exp = formatdate(time.time() + dur, usegmt=True)
+        exp = formatdate(time.time() + dur)
     else:
         exp = "Fri, 15 Aug 1997 01:00:00 GMT"
 
@@ -1839,12 +1867,10 @@ def humansize(sz: float, terse: bool = False) -> str:
 
         sz /= 1024.0
 
-    ret = " ".join([str(sz)[:4].rstrip("."), unit])
-
-    if not terse:
-        return ret
-
-    return ret.replace("iB", "").replace(" ", "")
+    if terse:
+        return "%s%s" % (str(sz)[:4].rstrip("."), unit[:1])
+    else:
+        return "%s %s" % (str(sz)[:4].rstrip("."), unit)
 
 
 def unhumanize(sz: str) -> int:
@@ -1896,7 +1922,7 @@ def uncyg(path: str) -> str:
 def undot(path: str) -> str:
     ret: list[str] = []
     for node in path.split("/"):
-        if node in ["", "."]:
+        if node == "." or not node:
             continue
 
         if node == "..":
@@ -2049,7 +2075,7 @@ def _quotep2(txt: str) -> str:
     """url quoter which deals with bytes correctly"""
     btxt = w8enc(txt)
     quot = quote(btxt, safe=b"/")
-    return w8dec(quot.replace(b" ", b"+"))
+    return w8dec(quot.replace(b" ", b"+"))  # type: ignore
 
 
 def _quotep3(txt: str) -> str:
@@ -2093,6 +2119,72 @@ def ujoin(rd: str, fn: str) -> str:
         return rd or fn
 
 
+def log_reloc(
+    log: "NamedLogger",
+    re: dict[str, str],
+    pm: tuple[str, str, str, tuple["VFS", str]],
+    ap: str,
+    vp: str,
+    fn: str,
+    vn: "VFS",
+    rem: str,
+) -> None:
+    nap, nvp, nfn, (nvn, nrem) = pm
+    t = "reloc %s:\nold ap [%s]\nnew ap [%s\033[36m/%s\033[0m]\nold vp [%s]\nnew vp [%s\033[36m/%s\033[0m]\nold fn [%s]\nnew fn [%s]\nold vfs [%s]\nnew vfs [%s]\nold rem [%s]\nnew rem [%s]"
+    log(t % (re, ap, nap, nfn, vp, nvp, nfn, fn, nfn, vn.vpath, nvn.vpath, rem, nrem))
+
+
+def pathmod(
+    vfs: "VFS", ap: str, vp: str, mod: dict[str, str]
+) -> Optional[tuple[str, str, str, tuple["VFS", str]]]:
+    # vfs: authsrv.vfs
+    # ap: original abspath to a file
+    # vp: original urlpath to a file
+    # mod: modification (ap/vp/fn)
+
+    nvp = "\n"  # new vpath
+    ap = os.path.dirname(ap)
+    vp, fn = vsplit(vp)
+    if mod.get("fn"):
+        fn = mod["fn"]
+        nvp = vp
+
+    for ref, k in ((ap, "ap"), (vp, "vp")):
+        if k not in mod:
+            continue
+
+        ms = mod[k].replace(os.sep, "/")
+        if ms.startswith("/"):
+            np = ms
+        elif k == "vp":
+            np = undot(vjoin(ref, ms))
+        else:
+            np = os.path.abspath(os.path.join(ref, ms))
+
+        if k == "vp":
+            nvp = np.lstrip("/")
+            continue
+
+        # try to map abspath to vpath
+        np = np.replace("/", os.sep)
+        for vn_ap, vn in vfs.all_aps:
+            if not np.startswith(vn_ap):
+                continue
+            zs = np[len(vn_ap) :].replace(os.sep, "/")
+            nvp = vjoin(vn.vpath, zs)
+            break
+
+    if nvp == "\n":
+        return None
+
+    vn, rem = vfs.get(nvp, "*", False, False)
+    if not vn.realpath:
+        raise Exception("unmapped vfs")
+
+    ap = vn.canonical(rem)
+    return ap, nvp, fn, (vn, rem)
+
+
 def _w8dec2(txt: bytes) -> str:
     """decodes filesystem-bytes to wtf8"""
     return surrogateescape.decodefilename(txt)
@@ -2709,30 +2801,30 @@ def rmdirs_up(top: str, stop: str) -> tuple[list[str], list[str]]:
 
 def unescape_cookie(orig: str) -> str:
     # mw=idk; doot=qwe%2Crty%3Basd+fgh%2Bjkl%25zxc%26vbn  # qwe,rty;asd fgh+jkl%zxc&vbn
-    ret = ""
+    ret = []
     esc = ""
     for ch in orig:
         if ch == "%":
-            if len(esc) > 0:
-                ret += esc
+            if esc:
+                ret.append(esc)
             esc = ch
 
-        elif len(esc) > 0:
+        elif esc:
             esc += ch
             if len(esc) == 3:
                 try:
-                    ret += chr(int(esc[1:], 16))
+                    ret.append(chr(int(esc[1:], 16)))
                 except:
-                    ret += esc
+                    ret.append(esc)
                 esc = ""
 
         else:
-            ret += ch
+            ret.append(ch)
 
-    if len(esc) > 0:
-        ret += esc
+    if esc:
+        ret.append(esc)
 
-    return ret
+    return "".join(ret)
 
 
 def guess_mime(url: str, fallback: str = "application/octet-stream") -> str:
@@ -3106,6 +3198,7 @@ def runihook(
 
 def _runhook(
     log: Optional["NamedLogger"],
+    src: str,
     cmd: str,
     ap: str,
     vp: str,
@@ -3117,14 +3210,16 @@ def _runhook(
     ip: str,
     at: float,
     txt: str,
-) -> bool:
+) -> dict[str, Any]:
+    ret = {"rc": 0}
     areq, chk, fork, jtxt, wait, sp_ka, acmd = _parsehook(log, cmd)
     if areq:
         for ch in areq:
             if ch not in perms:
                 t = "user %s not allowed to run hook %s; need perms %s, have %s"
-                log(t % (uname, cmd, areq, perms))
-                return True  # fallthrough to next hook
+                if log:
+                    log(t % (uname, cmd, areq, perms))
+                return ret  # fallthrough to next hook
     if jtxt:
         ja = {
             "ap": ap,
@@ -3136,6 +3231,7 @@ def _runhook(
             "host": host,
             "user": uname,
             "perms": perms,
+            "src": src,
             "txt": txt,
         }
         arg = json.dumps(ja)
@@ -3154,18 +3250,34 @@ def _runhook(
     else:
         rc, v, err = runcmd(bcmd, **sp_ka)  # type: ignore
         if chk and rc:
+            ret["rc"] = rc
             retchk(rc, bcmd, err, log, 5)
-            return False
+        else:
+            try:
+                ret = json.loads(v)
+            except:
+                ret = {}
+
+            try:
+                if "stdout" not in ret:
+                    ret["stdout"] = v
+                if "rc" not in ret:
+                    ret["rc"] = rc
+            except:
+                ret = {"rc": rc, "stdout": v}
 
     wait -= time.time() - t0
     if wait > 0:
         time.sleep(wait)
 
-    return True
+    return ret
 
 
 def runhook(
     log: Optional["NamedLogger"],
+    broker: Optional["BrokerCli"],
+    up2k: Optional["Up2k"],
+    src: str,
     cmds: list[str],
     ap: str,
     vp: str,
@@ -3177,19 +3289,42 @@ def runhook(
     ip: str,
     at: float,
     txt: str,
-) -> bool:
+) -> dict[str, Any]:
+    assert broker or up2k
+    asrv = (broker or up2k).asrv
+    args = (broker or up2k).args
     vp = vp.replace("\\", "/")
+    ret = {"rc": 0}
     for cmd in cmds:
         try:
-            if not _runhook(log, cmd, ap, vp, host, uname, perms, mt, sz, ip, at, txt):
-                return False
+            hr = _runhook(
+                log, src, cmd, ap, vp, host, uname, perms, mt, sz, ip, at, txt
+            )
+            if log and args.hook_v:
+                log("hook(%s) [%s] => \033[32m%s" % (src, cmd, hr), 6)
+            if not hr:
+                return {}
+            for k, v in hr.items():
+                if k in ("idx", "del") and v:
+                    if broker:
+                        broker.say("up2k.hook_fx", k, v, vp)
+                    else:
+                        up2k.fx_backlog.append((k, v, vp))
+                elif k == "reloc" and v:
+                    # idk, just take the last one ig
+                    ret["reloc"] = v
+                elif k in ret:
+                    if k == "rc" and v:
+                        ret[k] = v
+                else:
+                    ret[k] = v
         except Exception as ex:
             (log or print)("hook: {}".format(ex))
             if ",c," in "," + cmd:
-                return False
+                return {}
             break
 
-    return True
+    return ret
 
 
 def loadpy(ap: str, hot: bool) -> Any:

copyparty/web/browser.css

@@ -1017,9 +1017,6 @@ html.y #path a:hover {
 	color: var(--g-dfg);
 }
 #ggrid>a.au:before {
-	content: '💾';
-}
-html.np_open #ggrid>a.au:before {
 	content: '▶';
 }
 #ggrid>a:before {

copyparty/web/browser2.html

@@ -11,7 +11,6 @@
 		td{border:1px solid #999;border-width:1px 1px 0 0;padding:0 5px}
 		a{display:block}
 	</style>
-{{ html_head }}
 </head>
 
 <body>

copyparty/web/md.html

@@ -159,5 +159,8 @@ try { l.light = drk? 0:1; } catch (ex) { }
 	{%- if edit %}
 	<script src="{{ r }}/.cpr/md2.js?_={{ ts }}"></script>
 	{%- endif %}
+	{%- if js %}
+	<script src="{{ js }}_={{ ts }}"></script>
+	{%- endif %}
 </body></html>
 

copyparty/web/mde.html

@@ -53,5 +53,8 @@ try { l.light = drk? 0:1; } catch (ex) { }
 	<script src="{{ r }}/.cpr/deps/marked.js?_={{ ts }}"></script>
 	<script src="{{ r }}/.cpr/deps/easymde.js?_={{ ts }}"></script>
 	<script src="{{ r }}/.cpr/mde.js?_={{ ts }}"></script>
+	{%- if js %}
+	<script src="{{ js }}_={{ ts }}"></script>
+	{%- endif %}
 </body></html>
 

copyparty/web/msg.html

@@ -46,6 +46,9 @@
 		}, 1000);
 	</script>
 	{%- endif %}
+	{%- if js %}
+	<script src="{{ js }}_={{ ts }}"></script>
+	{%- endif %}
 </body>
 
 </html>

copyparty/web/splash.html

@@ -119,6 +119,9 @@ document.documentElement.className = (STG && STG.cpp_thm) || "{{ this.args.theme
 </script>
 <script src="{{ r }}/.cpr/util.js?_={{ ts }}"></script>
 <script src="{{ r }}/.cpr/splash.js?_={{ ts }}"></script>
+{%- if js %}
+<script src="{{ js }}_={{ ts }}"></script>
+{%- endif %}
 </body>
 </html>
 

copyparty/web/svcs.html

@@ -245,6 +245,9 @@ document.documentElement.className = (STG && STG.cpp_thm) || "{{ args.theme }}";
 </script>
 <script src="{{ r }}/.cpr/util.js?_={{ ts }}"></script>
 <script src="{{ r }}/.cpr/svcs.js?_={{ ts }}"></script>
+{%- if js %}
+<script src="{{ js }}_={{ ts }}"></script>
+{%- endif %}
 </body>
 </html>
 

copyparty/web/up2k.js

@@ -2383,8 +2383,23 @@ function up2k_init(subtle) {
                     var arr = st.todo.upload,
                         sort = arr.length && arr[arr.length - 1].nfile > t.n;
 
+                    if (!t.stitch_sz) {
+                        // keep all connections busy
+                        var bpc = (st.bytes.total - st.bytes.finished) / (parallel_uploads || 1),
+                            ocs = 1024 * 1024,
+                            stp = 1024 * 512,
+                            ccs = ocs;
+                        while (ccs < bpc) {
+                            ocs = ccs;
+                            ccs += stp; if (ccs < bpc) ocs = ccs;
+                            ccs += stp; stp *= 2;
+                        }
+                        ocs = Math.floor(ocs / 1024 / 1024);
+                        t.stitch_sz = Math.min(ocs, stitch_tgt);
+                    }
+
                     for (var a = 0; a < t.postlist.length; a++) {
-                        var nparts = [], tbytes = 0, stitch = stitch_tgt;
+                        var nparts = [], tbytes = 0, stitch = t.stitch_sz;
                         if (t.nojoin && t.nojoin - t.postlist.length < 6)
                             stitch = 1;
 
@@ -2400,6 +2415,7 @@ function up2k_init(subtle) {
                             'nparts': nparts
                         });
                     }
+                    t.nojoin = 0;
 
                     msg = null;
                     done = false;
@@ -2606,7 +2622,7 @@ function up2k_init(subtle) {
             }
             else if (txt.indexOf('already got that') + 1 ||
                 txt.indexOf('already being written') + 1) {
-                t.nojoin = t.postlist.length;
+                t.nojoin = t.nojoin || t.postlist.length;
                 console.log("ignoring dupe-segment with backoff", t.nojoin, t.name, t);
                 if (!toast.visible && st.todo.upload.length < 4)
                     toast.msg(10, L.u_cbusy);
@@ -2653,19 +2669,29 @@ function up2k_init(subtle) {
                     return;
 
                 st.bytes.inflight -= (xhr.bsent || 0);
+                xhr.bsent = 0;
 
                 if (!toast.visible)
                     toast.warn(9.98, L.u_cuerr.format(snpart, Math.ceil(t.size / chunksize), t.name), t);
 
+                t.nojoin = t.nojoin || t.postlist.length;  // maybe rproxy postsize limit
                 console.log('chunkpit onerror,', ++tries, t.name, t);
                 orz2(xhr);
             };
-            var chashes = [];
-            for (var a = pcar; a <= pcdr; a++)
-                chashes.push(t.hash[a]);
+
+            var chashes = [],
+                ctxt = t.hash[pcar],
+                plen = Math.floor(192 / nparts.length);
+
+            plen = plen > 9 ? 9 : plen < 2 ? 2 : plen;
+            for (var a = pcar + 1; a <= pcdr; a++)
+                chashes.push(t.hash[a].slice(0, plen));
+
+            if (chashes.length)
+                ctxt += ',' + plen + ',' + chashes.join('');
 
             xhr.open('POST', t.purl, true);
-            xhr.setRequestHeader("X-Up2k-Hash", chashes.join(","));
+            xhr.setRequestHeader("X-Up2k-Hash", ctxt);
             xhr.setRequestHeader("X-Up2k-Wark", t.wark);
             xhr.setRequestHeader("X-Up2k-Stat", "{0}/{1}/{2}/{3} {4}/{5} {6}".format(
                 pvis.ctr.ok, pvis.ctr.ng, pvis.ctr.bz, pvis.ctr.q, btot, btot - bfin,

docs/changelog.md

@@ -1,3 +1,40 @@
+▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀  
+# 2024-0729-2028  `v1.13.6`  not that big
+
+## new features
+
+* up2k.js: set clientside timeouts on http connections during upload 85e54980
+  * some reverse-proxy setups could cause uploads to hang indefinitely by eating requests; should recover nicely now
+* audio-player shows statustext while loading 662541c6
+* [bsod theme](https://github.com/9001/copyparty/tree/hovudstraum/contrib/themes) [(live demo)](https://cd.ocv.me/c/) 15ddcf53
+
+## bugfixes
+
+* fix bugs in the [long-distance upload optimizations](https://github.com/9001/copyparty/releases/tag/v1.13.5) in the previous version:
+  * up2k.js didn't necessarily use the expected chunksize when stitching 225bd80e
+  * u2c (commandline uploader): 8916bce3
+    * use the correct chunksize instead of overshooting like crazy
+    * could crash on exit if `-z` was enabled (so basically harmless)
+    * the "time spent uploading" statustext that was printed on exit could multiply by `-j` and exceed walltime
+* misc ux 9bb6e0dc
+  * don't accept hotkeys until it's safe to do so
+  * improve messages regarding the [firefox crash](https://bugzilla.mozilla.org/show_bug.cgi?id=1790500)
+  * keep more console logs in memory (easier to debug)
+  * fix wordwrap in messageboxes on firefox a19a0fa9
+
+## other changes
+
+* changed the `xm` / "on message" [hook examples](https://github.com/9001/copyparty/tree/hovudstraum/bin/hooks#on-message) to reject users without write-access 99edba4f
+* docker images were rebuilt on 2024-08-02, 23:30 UTC with new optimizations: 98ffaadf
+  * 😃 RAM usage decreased by `5-6 MiB` for most flavors; `10 MiB` for dj/iv
+  * 😕 image size grew by `4 MiB` (min), `6 MiB` (ac/im/iv), `9 MiB` (dj)
+  * 😃 startup time reduced to about half
+  * and avoids a deadlock on IBM mainframes
+* updated comparison to other software 6b54972e
+  * `hfs2` is dead, `hfs3` and `filebrowser` improved
+
+
+
 ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀  
 # 2024-0722-2323  `v1.13.5`  american sized
 

docs/devnotes.md

@@ -12,6 +12,8 @@
     * [write](#write)
     * [admin](#admin)
     * [general](#general)
+* [event hooks](#event-hooks) - on writing your own [hooks](../README.md#event-hooks)
+    * [hook effects](#hook-effects) - hooks can cause intentional side-effects
 * [assumptions](#assumptions)
     * [mdns](#mdns)
 * [sfx repack](#sfx-repack) - reduce the size of an sfx by removing features
@@ -204,6 +206,32 @@ upload modifiers:
 | GET | `?pw=x` | logout |
 
 
+# event hooks
+
+on writing your own [hooks](../README.md#event-hooks)
+
+## hook effects
+
+hooks can cause intentional side-effects,  such as redirecting an upload into another location, or creating+indexing additional files, or deleting existing files, by returning json on stdout
+
+* `reloc` can redirect uploads before/after uploading has finished, based on filename, extension, file contents, uploader ip/name etc.
+* `idx` informs copyparty about a new file to index as a consequence of this upload
+* `del` tells copyparty to delete an unrelated file by vpath
+
+for these to take effect, the hook must be defined with the `c1` flag; see example [reloc-by-ext](https://github.com/9001/copyparty/blob/hovudstraum/bin/hooks/reloc-by-ext.py)
+
+a subset of effect types are available for a subset of hook types,
+
+* most hook types (xbu/xau/xbr/xar/xbd/xad/xm) support `idx` and `del` for all http protocols (up2k / basic-uploader / webdav), but not ftp/tftp/smb
+* most hook types will abort/reject the action if the hook returns nonzero, assuming flag `c` is given, see examples [reject-extension](https://github.com/9001/copyparty/blob/hovudstraum/bin/hooks/reject-extension.py) and [reject-mimetype](https://github.com/9001/copyparty/blob/hovudstraum/bin/hooks/reject-mimetype.py)
+* `xbu` supports `reloc` for all http protocols (up2k / basic-uploader / webdav), but not ftp/tftp/smb
+* `xau` supports `reloc` for basic-uploader / webdav only, not up2k or ftp/tftp/smb
+  * so clients like sharex are supported, but not dragdrop into browser
+
+to trigger indexing of files `/foo/1.txt` and `/foo/bar/2.txt`, a hook can `print(json.dumps({"idx":{"vp":["/foo/1.txt","/foo/bar/2.txt"]}}))` (and replace "idx" with "del" to delete instead)
+* note: paths starting with `/` are absolute URLs, but you can also do `../3.txt` relative to the destination folder of each uploaded file
+
+
 # assumptions
 
 ## mdns

docs/examples/docker/portainer.md

@@ -0,0 +1,45 @@
+the following setup appears to work (copyparty starts, accepts uploads, is able to persist config)
+
+tested on debian 12 using [portainer-ce](https://docs.portainer.io/start/install-ce/server/docker/linux) with [docker-ce](https://docs.docker.com/engine/install/debian/) as root (not rootless)
+
+before making the container, first `mkdir /etc/copyparty /srv/pub` which will be bind-mounts into the container
+
+> both `/etc/copyparty` and `/srv/pub` are examples; you can change them if you'd like
+
+put your copyparty config files directly into `/etc/copyparty` and the files to share inside `/srv/pub`
+
+on first startup, copyparty will create a subfolder inside `/etc/copyparty` called `copyparty` where it puts some runtime state; for example replacing `/etc/copyparty/copyparty/cert.pem` with another TLS certificate is a quick and dirty way to get valid HTTPS (if you really want copyparty to handle that and not a reverse-proxy)
+
+
+## in portainer:
+
+```
+environments -> local -> containers -> add container:
+
+       name = copyparty-ac
+   registry = docker hub
+      image = copyparty/ac
+always pull = no
+
+manual network port publishing:
+  3923 to 3923 [TCP]
+
+advanced -> command & logging:
+  console = interactive & tty
+
+advanced -> volumes -> map additional volume:
+  container = /cfg  [Bind]
+  host = /etc/copyparty  [Writable]
+
+advanced -> volumes -> map additional volume:
+  container = /w  [Bind]
+  host = /srv/pub  [Writable]
+```
+
+notes:
+
+* `/cfg` is where copyparty expects to find its config files; `/etc/copyparty` is just an example mapping to that
+
+* `/w` is where copyparty expects to find the folder to share; `/srv/pub` is just an example mapping to that
+
+* the volumes must be bind-mounts to avoid permission issues (or so the theory goes)

docs/versus.md

@@ -175,6 +175,7 @@ symbol legend,
 | ┗ randomize filename    | █ |   |   |   |   |   |   | █ | █ |   |   |   |   |
 | ┗ mimetype reject-list  | ╱ |   |   |   |   |   |   |   | • | ╱ |   | ╱ | • |
 | ┗ extension reject-list | ╱ |   |   |   |   |   |   | █ | • | ╱ |   | ╱ | • |
+| ┗ upload routing        | █ |   |   |   |   |   |   |   |   |   |   |   |   |
 | checksums provided      |   |   |   | █ | █ |   |   |   | █ | ╱ |   |   |   |
 | cloud storage backend   | ╱ | ╱ | ╱ | █ | █ | █ | ╱ |   |   | ╱ | █ | █ | ╱ |
 
@@ -188,6 +189,9 @@ symbol legend,
 
 * `race the beam` = files can be downloaded while they're still uploading; downloaders are slowed down such that the uploader is always ahead
 
+* `upload routing` = depending on filetype / contents / uploader etc., the file can be redirected to another location or otherwise transformed; mitigates limitations such as [sharex#3992](https://github.com/ShareX/ShareX/issues/3992)
+  * copyparty example: [reloc-by-ext](https://github.com/9001/copyparty/tree/hovudstraum/bin/hooks#before-upload)
+
 * `checksums provided` = when downloading a file from the server, the file's checksum is provided for verification client-side
 
 * `cloud storage backend` = able to serve files from (and write to) s3 or similar cloud services; `╱` means the software can do this with some help from `rclone mount` as a bridge
@@ -217,7 +221,7 @@ symbol legend,
 | serve sftp (ssh)        |   |   |   |   |   | █ |   |   |   |   |   | █ | █ |
 | serve smb/cifs          | ╱ |   |   |   |   | █ |   |   |   |   |   |   |   |
 | serve dlna              |   |   |   |   |   | █ |   |   |   |   |   |   |   |
-| listen on unix-socket   |   |   |   | █ | █ |   | █ | █ | █ | █ | █ | █ |   |
+| listen on unix-socket   | █ |   |   | █ | █ |   | █ | █ | █ | █ | █ | █ |   |
 | zeroconf                | █ |   |   |   |   |   |   |   |   |   |   |   | █ |
 | supports netscape 4     | ╱ |   |   |   |   | █ |   |   |   |   | • |   | ╱ |
 | ...internet explorer 6  | ╱ | █ |   | █ |   | █ |   |   |   |   | • |   | ╱ |

scripts/docker/Dockerfile.ac

@@ -5,19 +5,16 @@ LABEL   org.opencontainers.image.url="https://github.com/9001/copyparty" \
         org.opencontainers.image.licenses="MIT" \
         org.opencontainers.image.title="copyparty-ac" \
         org.opencontainers.image.description="copyparty with Pillow and FFmpeg (image/audio/video thumbnails, audio transcoding, media tags)"
-ENV     PYTHONPYCACHEPREFIX=/tmp/pyc \
-        XDG_CONFIG_HOME=/cfg
+ENV     XDG_CONFIG_HOME=/cfg
 
 RUN     apk --no-cache add !pyc \
             wget \
-            py3-argon2-cffi py3-pillow \
-            ffmpeg \
-        && rm -rf /tmp/pyc \
-        && mkdir /cfg /w \
-        && chmod 777 /cfg /w \
-        && echo % /cfg > initcfg
+            py3-jinja2 py3-argon2-cffi py3-pillow \
+            ffmpeg
+
+COPY    i/dist/copyparty-sfx.py innvikler.sh ./
+RUN     ash innvikler.sh && rm innvikler.sh
 
-COPY    i/dist/copyparty-sfx.py ./
 WORKDIR /w
 EXPOSE  3923
-ENTRYPOINT ["python3", "/z/copyparty-sfx.py", "--no-crt", "-c", "/z/initcfg"]
+ENTRYPOINT ["python3", "-m", "copyparty", "--no-crt", "-c", "/z/initcfg"]

scripts/docker/Dockerfile.dj

@@ -5,15 +5,14 @@ LABEL   org.opencontainers.image.url="https://github.com/9001/copyparty" \
         org.opencontainers.image.licenses="MIT" \
         org.opencontainers.image.title="copyparty-dj" \
         org.opencontainers.image.description="copyparty with all optional dependencies, including musical key / bpm detection"
-ENV     PYTHONPYCACHEPREFIX=/tmp/pyc \
-        XDG_CONFIG_HOME=/cfg
+ENV     XDG_CONFIG_HOME=/cfg
 
 COPY    i/bin/mtag/install-deps.sh ./
 COPY    i/bin/mtag/audio-bpm.py /mtag/
 COPY    i/bin/mtag/audio-key.py /mtag/
 RUN     apk add -U !pyc \
             wget \
-            py3-argon2-cffi py3-pillow py3-pip py3-cffi \
+            py3-jinja2 py3-argon2-cffi py3-pillow py3-pip py3-cffi \
             ffmpeg \
             vips-jxl vips-heif vips-poppler vips-magick \
             py3-numpy fftw libsndfile \
@@ -27,18 +26,12 @@ RUN     apk add -U !pyc \
         && python3 -m pip install pyvips \
         && bash install-deps.sh \
         && apk del py3-pip .bd \
-        && rm -rf /var/cache/apk/* /tmp/pyc \
         && chmod 777 /root \
-        && ln -s /root/vamp /root/.local / \
-        && mkdir /cfg /w \
-        && chmod 777 /cfg /w \
-        && echo % /cfg > initcfg
+        && ln -s /root/vamp /root/.local /
+
+COPY    i/dist/copyparty-sfx.py innvikler.sh ./
+RUN     ash innvikler.sh && rm innvikler.sh
 
-COPY    i/dist/copyparty-sfx.py ./
 WORKDIR /w
 EXPOSE  3923
-ENTRYPOINT ["python3", "/z/copyparty-sfx.py", "--no-crt", "-c", "/z/initcfg"]
-
-# size: 286 MB
-# bpm/key: 529 sec
-# idx-bench: 2352 MB/s
+ENTRYPOINT ["python3", "-m", "copyparty", "--no-crt", "-c", "/z/initcfg"]

scripts/docker/Dockerfile.im

@@ -5,18 +5,15 @@ LABEL   org.opencontainers.image.url="https://github.com/9001/copyparty" \
         org.opencontainers.image.licenses="MIT" \
         org.opencontainers.image.title="copyparty-im" \
         org.opencontainers.image.description="copyparty with Pillow and Mutagen (image thumbnails, media tags)"
-ENV     PYTHONPYCACHEPREFIX=/tmp/pyc \
-        XDG_CONFIG_HOME=/cfg
+ENV     XDG_CONFIG_HOME=/cfg
 
 RUN     apk --no-cache add !pyc \
             wget \
-            py3-argon2-cffi py3-pillow py3-mutagen \
-        && rm -rf /tmp/pyc \
-        && mkdir /cfg /w \
-        && chmod 777 /cfg /w \
-        && echo % /cfg > initcfg
+            py3-jinja2 py3-argon2-cffi py3-pillow py3-mutagen
+
+COPY    i/dist/copyparty-sfx.py innvikler.sh ./
+RUN     ash innvikler.sh && rm innvikler.sh
 
-COPY    i/dist/copyparty-sfx.py ./
 WORKDIR /w
 EXPOSE  3923
-ENTRYPOINT ["python3", "/z/copyparty-sfx.py", "--no-crt", "-c", "/z/initcfg"]
+ENTRYPOINT ["python3", "-m", "copyparty", "--no-crt", "-c", "/z/initcfg"]

scripts/docker/Dockerfile.iv

@@ -5,12 +5,11 @@ LABEL   org.opencontainers.image.url="https://github.com/9001/copyparty" \
         org.opencontainers.image.licenses="MIT" \
         org.opencontainers.image.title="copyparty-iv" \
         org.opencontainers.image.description="copyparty with Pillow, FFmpeg, libvips (image/audio/video thumbnails, audio transcoding, media tags)"
-ENV     PYTHONPYCACHEPREFIX=/tmp/pyc \
-        XDG_CONFIG_HOME=/cfg
+ENV     XDG_CONFIG_HOME=/cfg
 
 RUN     apk add -U !pyc \
             wget \
-            py3-argon2-cffi py3-pillow py3-pip py3-cffi \
+            py3-jinja2 py3-argon2-cffi py3-pillow py3-pip py3-cffi \
             ffmpeg \
             vips-jxl vips-heif vips-poppler vips-magick \
         && apk add -t .bd \
@@ -18,13 +17,11 @@ RUN     apk add -U !pyc \
             python3-dev py3-wheel \
         && rm -f /usr/lib/python3*/EXTERNALLY-MANAGED \
         && python3 -m pip install pyvips \
-        && apk del py3-pip .bd \
-        && rm -rf /var/cache/apk/* /tmp/pyc \
-        && mkdir /cfg /w \
-        && chmod 777 /cfg /w \
-        && echo % /cfg > initcfg
+        && apk del py3-pip .bd
+
+COPY    i/dist/copyparty-sfx.py innvikler.sh ./
+RUN     ash innvikler.sh && rm innvikler.sh
 
-COPY    i/dist/copyparty-sfx.py ./
 WORKDIR /w
 EXPOSE  3923
-ENTRYPOINT ["python3", "/z/copyparty-sfx.py", "--no-crt", "-c", "/z/initcfg"]
+ENTRYPOINT ["python3", "-m", "copyparty", "--no-crt", "-c", "/z/initcfg"]

scripts/docker/Dockerfile.min

@@ -5,17 +5,14 @@ LABEL   org.opencontainers.image.url="https://github.com/9001/copyparty" \
         org.opencontainers.image.licenses="MIT" \
         org.opencontainers.image.title="copyparty-min" \
         org.opencontainers.image.description="just copyparty, no thumbnails / media tags / audio transcoding"
-ENV     PYTHONPYCACHEPREFIX=/tmp/pyc \
-        XDG_CONFIG_HOME=/cfg
+ENV     XDG_CONFIG_HOME=/cfg
 
 RUN     apk --no-cache add !pyc \
-            python3 \
-        && rm -rf /tmp/pyc \
-        && mkdir /cfg /w \
-        && chmod 777 /cfg /w \
-        && echo % /cfg > initcfg
+            py3-jinja2
+
+COPY    i/dist/copyparty-sfx.py innvikler.sh ./
+RUN     ash innvikler.sh && rm innvikler.sh
 
-COPY    i/dist/copyparty-sfx.py ./
 WORKDIR /w
 EXPOSE  3923
-ENTRYPOINT ["python3", "/z/copyparty-sfx.py", "--no-crt", "--no-thumb", "-c", "/z/initcfg"]
+ENTRYPOINT ["python3", "-m", "copyparty", "--no-crt", "--no-thumb", "-c", "/z/initcfg"]

scripts/docker/README.md

@@ -22,6 +22,11 @@ this example is also available as a podman-compatible [docker-compose yaml](http
 i'm not very familiar with containers, so let me know if this section could be better 🙏
 
 
+## portainer
+
+* there is a [portainer howto](https://github.com/9001/copyparty/blob/hovudstraum/docs/examples/docker/portainer.md) which is mostly untested
+
+
 ## configuration
 
 > this section basically explains how the [docker-compose yaml](https://github.com/9001/copyparty/blob/hovudstraum/docs/examples/docker/basic-docker-compose) works, so you may look there instead

scripts/docker/innvikler.sh

@@ -0,0 +1,46 @@
+#!/bin/ash
+set -ex
+
+# cleanup for flavors with python build steps (dj/iv)
+rm -rf /var/cache/apk/* /root/.cache
+
+# initial config; common for all flavors
+mkdir /cfg /w
+chmod 777 /cfg /w
+echo % /cfg > initcfg
+
+# unpack sfx and dive in
+python3 copyparty-sfx.py --version
+cd /tmp/pe-copyparty.0
+
+# steal the stuff we need
+mv copyparty partftpy ftp/* /usr/lib/python3.*/site-packages/
+
+# golf
+cd /usr/lib/python3.*/
+rm -rf \
+  /tmp/pe-* /z/copyparty-sfx.py \
+  ensurepip pydoc_data turtle.py turtledemo lib2to3
+
+# drop bytecode
+find / -xdev -name __pycache__ -print0 | xargs -0 rm -rf
+
+# build the stuff we want
+python3 -m compileall -qj4 site-packages sqlite3 xml
+
+# drop the stuff we dont
+find -name __pycache__ |
+  grep -E 'ty/web/|/pycpar' |
+  tr '\n' '\0' | xargs -0 rm -rf
+
+# two-for-one:
+# 1) smoketest copyparty even starts
+# 2) build any bytecode we missed
+# this tends to race other builders (alle gode ting er tre)
+cd /z
+python3 -m copyparty \
+  --ign-ebind -p$((1024+RANDOM)),$((1024+RANDOM)),$((1024+RANDOM)) \
+  --no-crt -qi127.1 --exit=idx -e2dsa -e2ts
+
+# output from -e2d
+rm -rf .hist

scripts/docker/make.sh

@@ -6,6 +6,8 @@ set -e
     exit 1
 }
 
+suf=-b1
+suf=
 sarchs="386 amd64 arm/v7 arm64/v8 ppc64le s390x"
 archs="amd64 arm s390x 386 arm64 ppc64le"
 imgs="dj iv min im ac"
@@ -103,11 +105,12 @@ filt=
             # --pull=never does nothing at all btw
             (set -x
             $nice podman build \
+                --squash \
                 --pull=never \
                 --from localhost/alpine-$a \
-                -t copyparty-$i-$a \
+                -t copyparty-$i-$a$suf \
                 -f Dockerfile.$i . ||
-                    (echo $? $i-$a >> err)
+                    (echo $? $i-$a >> err; printf '%096d\n' $(seq 1 42))
             rm -f .blk
             ) 2> >(tee $a.err | sed "s/^/$aa:/" >&2) > >(tee $a.out | sed "s/^/$aa:/") &
         done
@@ -134,9 +137,10 @@ filt=
         variants=
         for a in $archs; do
             [[ " ${ngs[*]} " =~ " $i-$a " ]] && continue
-            variants="$variants containers-storage:localhost/copyparty-$i-$a"
+            variants="$variants containers-storage:localhost/copyparty-$i-$a$suf"
         done
-        podman manifest create copyparty-$i $variants
+        podman manifest rm copyparty-$i$suf || echo "(that's fine btw)"
+        podman manifest create copyparty-$i$suf $variants
     done
 }
 

scripts/make-sfx.sh

@@ -202,14 +202,14 @@ necho() {
 	mv {markupsafe,jinja2} j2/
 
 	necho collecting pyftpdlib
-	f="../build/pyftpdlib-1.5.9.tar.gz"
+	f="../build/pyftpdlib-1.5.10.tar.gz"
 	[ -e "$f" ] ||
-		(url=https://github.com/giampaolo/pyftpdlib/archive/refs/tags/release-1.5.9.tar.gz;
+		(url=https://files.pythonhosted.org/packages/cf/31/8d910cf40317dd0db74ba0b8558d0dee23c8b002468c14d3a5dec0e6e9fd/pyftpdlib-1.5.10.tar.gz;
 		wget -O$f "$url" || curl -L "$url" >$f)
 
 	tar -zxf $f
-	mv pyftpdlib-release-*/pyftpdlib .
-	rm -rf pyftpdlib-release-* pyftpdlib/test
+	mv pyftpdlib-*/pyftpdlib .
+	rm -rf pyftpdlib-* pyftpdlib/test
 	for f in pyftpdlib/_async{hat,ore}.py; do
 		[ -e "$f" ] || continue;
 		iawk 'NR<4||NR>27||!/^#/;NR==4{print"# license: https://opensource.org/licenses/ISC\n"}' $f
@@ -413,7 +413,7 @@ rm have
 
 ised /fork_process/d ftp/pyftpdlib/servers.py
 iawk '/^class _Base/{s=1}!s' ftp/pyftpdlib/authorizers.py
-iawk '/^ {0,4}[^ ]/{s=0}/^ {4}def (serve_forever|_loop)/{s=1}!s' ftp/pyftpdlib/servers.py
+iawk '/^ {0,4}[a-zA-Z]/{s=0}/^ {4}def (serve_forever|_loop)/{s=1}!s' ftp/pyftpdlib/servers.py
 rm -f ftp/pyftpdlib/{__main__,prefork}.py
 
 [ $no_ftp ] &&

scripts/pyinstaller/deps.sha512

@@ -36,4 +36,4 @@ d1420c8417fad7888766dd26b9706a87c63e8f33dceeb8e26d0056d5127b0b3ed9272e44b4b76113
 2be320b4191f208cdd6af183c77ba2cf460ea52164ee45ac3ff17d6dfa57acd9deff016636c2dd42a21f4f6af977d5f72df7dacf599bebcf41757272354d14c1  pillow-10.4.0-cp312-cp312-win_amd64.whl
 776378f5414efd26ec8a1cb3228a7b5fdf6afca3fa335a0e9b071266d55d9d9e66ee157c25a468a05bfa70ccd33c48b101998523fc6ff6bcf5e82a1d81ed0af8  pyinstaller-6.9.0-py3-none-win_amd64.whl
 c0af77d2a57cb063ab038dc986ed3582bc5acc8c8bd91d726101935d6388f50854ddbca26bc846ed5d1022cdee4d96242938c66f0ddc4565c36b60d691064db8  pyinstaller_hooks_contrib-2024.7-py2.py3-none-any.whl
-2f9a11ffae6d9f1ed76bf816f28812fcba71f87080b0c92e52bfccb46243118c5803a7e25dd78003ca7d66501bfcdce8ff7c691c63c0038b0d409ca3842dcc89  python-3.12.4-amd64.exe
+0572c6345f6a4f7f3e5c2ff858e3ca7ca54ae4478f3d59d8e18cb0f596e61dcf12aef579db229e83d63b30f15d6684ee6bb3feaea9413e5e636a503933057678  python-3.12.5-amd64.exe

scripts/pyinstaller/notes.txt

@@ -41,7 +41,7 @@ fns=(
   pillow-10.4.0-cp312-cp312-win_amd64.whl
   pyinstaller-6.9.0-py3-none-win_amd64.whl
   pyinstaller_hooks_contrib-2024.7-py2.py3-none-any.whl
-  python-3.12.4-amd64.exe
+  python-3.12.5-amd64.exe
 )
 [ $w7 ] && fns+=(  # u2c stuff
   certifi-2024.2.2-py3-none-any.whl

scripts/test/tftp.sh

@@ -3,34 +3,39 @@ set -ex
 
 # PYTHONPATH=.:~/dev/partftpy/ taskset -c 0 python3 -m copyparty -v srv::r -v srv/junk:junk:A --tftp 3969 
 
-get_src=~/dev/copyparty/srv/palette.flac
-get_fn=${get_src##*/}
+get_src=~/dev/copyparty/srv/ro/palette.flac
+get_fp=ro/${get_src##*/}  # server url
+get_fn=${get_fp##*/}  # just filename
 
 put_src=~/Downloads/102.zip
 put_dst=~/dev/copyparty/srv/junk/102.zip
 
+export PATH="$PATH:$HOME/src/atftp-0.8.0"
+
 cd /dev/shm
 
-echo curl get 1428 v4; curl --tftp-blksize 1428 tftp://127.0.0.1:3969/$get_fn | cmp $get_src || exit 1
-echo curl get 1428 v6; curl --tftp-blksize 1428 tftp://[::1]:3969/$get_fn | cmp $get_src || exit 1
+echo curl get 1428 v4; curl --tftp-blksize 1428 tftp://127.0.0.1:3969/$get_fp | cmp $get_src || exit 1
+echo curl get 1428 v6; curl --tftp-blksize 1428 tftp://[::1]:3969/$get_fp | cmp $get_src || exit 1
 
 echo curl put 1428 v4; rm -f $put_dst && curl --tftp-blksize 1428 -T $put_src tftp://127.0.0.1:3969/junk/ && cmp $put_src $put_dst || exit 1
 echo curl put 1428 v6; rm -f $put_dst && curl --tftp-blksize 1428 -T $put_src tftp://[::1]:3969/junk/ && cmp $put_src $put_dst || exit 1
 
-echo atftp get 1428; rm -f $get_fn && ~/src/atftp/atftp --option "blksize 1428" -g -r $get_fn 127.0.0.1 3969 && cmp $get_fn $get_src || exit 1
+echo atftp get 1428; rm -f $get_fn && atftp --option "blksize 1428" -g -r $get_fp -l $get_fn 127.0.0.1 3969 && cmp $get_fn $get_src || exit 1
 
-echo atftp put 1428; rm -f $put_dst && ~/src/atftp/atftp --option "blksize 1428" 127.0.0.1 3969 -p -l $put_src -r junk/102.zip && cmp $put_src $put_dst || exit 1
+echo atftp put 1428; rm -f $put_dst && atftp --option "blksize 1428" 127.0.0.1 3969 -p -l $put_src -r junk/102.zip && cmp $put_src $put_dst || exit 1
 
-echo tftp-hpa get; rm -f $put_dst && tftp -v -m binary 127.0.0.1 3969 -c get $get_fn && cmp $get_src $get_fn || exit 1
+echo tftp-hpa get; rm -f $get_fn && tftp -v -m binary 127.0.0.1 3969 -c get $get_fp && cmp $get_src $get_fn || exit 1
 
 echo tftp-hpa put; rm -f $put_dst && tftp -v -m binary 127.0.0.1 3969 -c put $put_src junk/102.zip && cmp $put_src $put_dst || exit 1
 
-echo curl get 512; curl tftp://127.0.0.1:3969/$get_fn | cmp $get_src || exit 1
+echo curl get 512; curl tftp://127.0.0.1:3969/$get_fp | cmp $get_src || exit 1
 
 echo curl put 512; rm -f $put_dst && curl -T $put_src tftp://127.0.0.1:3969/junk/ && cmp $put_src $put_dst || exit 1
 
-echo atftp get 512; rm -f $get_fn && ~/src/atftp/atftp -g -r $get_fn 127.0.0.1 3969 && cmp $get_fn $get_src || exit 1
+echo atftp get 512; rm -f $get_fn && atftp -g -r $get_fp -l $get_fn 127.0.0.1 3969 && cmp $get_fn $get_src || exit 1
 
-echo atftp put 512; rm -f $put_dst && ~/src/atftp/atftp 127.0.0.1 3969 -p -l $put_src -r junk/102.zip && cmp $put_src $put_dst || exit 1
+echo atftp put 512; rm -f $put_dst && atftp 127.0.0.1 3969 -p -l $put_src -r junk/102.zip && cmp $put_src $put_dst || exit 1
 
 echo nice
+
+rm -f $get_fn

scripts/uncomment.py

@@ -8,6 +8,12 @@ import sys
 import tokenize
 
 
+try:
+    FSTRING_MIDDLE = tokenize.FSTRING_MIDDLE
+except:
+    FSTRING_MIDDLE = -9001
+
+
 def uncomment(fpath):
     """modified https://stackoverflow.com/a/62074206"""
 
@@ -31,7 +37,7 @@ def uncomment(fpath):
         if start_line > last_lineno:
             last_col = 0
 
-        if start_col > last_col:
+        if start_col > last_col and prev_toktype != FSTRING_MIDDLE:
             out += " " * (start_col - last_col)
 
         is_legalese = (
@@ -48,6 +54,10 @@ def uncomment(fpath):
                 out += token_string
             else:
                 out += '"a"'
+        elif token_type == FSTRING_MIDDLE:
+            out += token_string.replace(r"{", r"{{").replace(r"}", r"}}")
+            if not code and token_string.strip():
+                code = True
         elif token_type != tokenize.COMMENT:
             out += token_string
             if not code and token_string.strip():

tests/test_hooks.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python3
+# coding: utf-8
+from __future__ import print_function, unicode_literals
+
+import os
+import shutil
+import tempfile
+import unittest
+
+from copyparty.authsrv import AuthSrv
+from copyparty.httpcli import HttpCli
+from tests import util as tu
+from tests.util import Cfg
+
+
+def hdr(query):
+    h = "GET /{} HTTP/1.1\r\nPW: o\r\nConnection: close\r\n\r\n"
+    return h.format(query).encode("utf-8")
+
+
+class TestHooks(unittest.TestCase):
+    def setUp(self):
+        self.td = tu.get_ramdisk()
+
+    def tearDown(self):
+        os.chdir(tempfile.gettempdir())
+        shutil.rmtree(self.td)
+
+    def reset(self):
+        td = os.path.join(self.td, "vfs")
+        if os.path.exists(td):
+            shutil.rmtree(td)
+        os.mkdir(td)
+        os.chdir(td)
+        return td
+
+    def test(self):
+        vcfg = ["a/b/c/d:c/d:A", "a:a:r"]
+
+        scenarios = (
+            ('{"vp":"x/y"}', "c/d/a.png", "c/d/x/y/a.png"),
+            ('{"vp":"x/y"}', "c/d/e/a.png", "c/d/e/x/y/a.png"),
+            ('{"vp":"../x/y"}', "c/d/e/a.png", "c/d/x/y/a.png"),
+            ('{"ap":"x/y"}', "c/d/a.png", "c/d/x/y/a.png"),
+            ('{"ap":"x/y"}', "c/d/e/a.png", "c/d/e/x/y/a.png"),
+            ('{"ap":"../x/y"}', "c/d/e/a.png", "c/d/x/y/a.png"),
+            ('{"ap":"../x/y"}', "c/d/a.png", "a/b/c/x/y/a.png"),
+            ('{"fn":"b.png"}', "c/d/a.png", "c/d/b.png"),
+            ('{"vp":"x","fn":"b.png"}', "c/d/a.png", "c/d/x/b.png"),
+        )
+
+        for x in scenarios:
+            print("\n\n\n", x)
+            hooktxt, url_up, url_dl = x
+            for hooktype in ("xbu", "xau"):
+                for upfun in (self.put, self.bup):
+                    self.reset()
+                    self.makehook("""print('{"reloc":%s}')""" % (hooktxt,))
+                    ka = {hooktype: ["j,c1,h.py"]}
+                    self.args = Cfg(v=vcfg, a=["o:o"], e2d=True, **ka)
+                    self.asrv = AuthSrv(self.args, self.log)
+
+                    h, b = upfun(url_up)
+                    self.assertIn("201 Created", h)
+                    h, b = self.curl(url_dl)
+                    self.assertEqual(b, "ok %s\n" % (url_up))
+
+    def makehook(self, hs):
+        with open("h.py", "wb") as f:
+            f.write(hs.encode("utf-8"))
+
+    def put(self, url):
+        buf = "PUT /{0} HTTP/1.1\r\nPW: o\r\nConnection: close\r\nContent-Length: {1}\r\n\r\nok {0}\n"
+        buf = buf.format(url, len(url) + 4).encode("utf-8")
+        print("PUT -->", buf)
+        conn = tu.VHttpConn(self.args, self.asrv, self.log, buf)
+        HttpCli(conn).run()
+        ret = conn.s._reply.decode("utf-8").split("\r\n\r\n", 1)
+        print("PUT <--", ret)
+        return ret
+
+    def bup(self, url):
+        hdr = "POST /%s HTTP/1.1\r\nPW: o\r\nConnection: close\r\nContent-Type: multipart/form-data; boundary=XD\r\nContent-Length: %d\r\n\r\n"
+        bdy = '--XD\r\nContent-Disposition: form-data; name="act"\r\n\r\nbput\r\n--XD\r\nContent-Disposition: form-data; name="f"; filename="%s"\r\n\r\n'
+        ftr = "\r\n--XD--\r\n"
+        try:
+            url, fn = url.rsplit("/", 1)
+        except:
+            fn = url
+            url = ""
+
+        buf = (bdy % (fn,) + "ok %s/%s\n" % (url, fn) + ftr).encode("utf-8")
+        buf = (hdr % (url, len(buf))).encode("utf-8") + buf
+        print("PoST -->", buf)
+        conn = tu.VHttpConn(self.args, self.asrv, self.log, buf)
+        HttpCli(conn).run()
+        ret = conn.s._reply.decode("utf-8").split("\r\n\r\n", 1)
+        print("POST <--", ret)
+        return ret
+
+    def curl(self, url, binary=False):
+        conn = tu.VHttpConn(self.args, self.asrv, self.log, hdr(url))
+        HttpCli(conn).run()
+        if binary:
+            h, b = conn.s._reply.split(b"\r\n\r\n", 1)
+            return [h.decode("utf-8"), b]
+
+        return conn.s._reply.decode("utf-8").split("\r\n\r\n", 1)
+
+    def log(self, src, msg, c=0):
+        print(msg)

tests/test_httpcli.py

@@ -10,6 +10,7 @@ import tarfile
 import tempfile
 import time
 import unittest
+import zipfile
 
 from copyparty.authsrv import AuthSrv
 from copyparty.httpcli import HttpCli
@@ -31,6 +32,9 @@ class TestHttpCli(unittest.TestCase):
         shutil.rmtree(self.td)
 
     def test(self):
+        test_tar = True
+        test_zip = True
+
         td = os.path.join(self.td, "vfs")
         os.mkdir(td)
         os.chdir(td)
@@ -40,6 +44,7 @@ class TestHttpCli(unittest.TestCase):
         self.can_write = ["wa", "wo", "aa", "ao"]
         self.fn = "g{:x}g".format(int(time.time() * 3))
 
+        tctr = 0
         allfiles = []
         allvols = []
         for top in self.dtypes:
@@ -83,6 +88,7 @@ class TestHttpCli(unittest.TestCase):
             self.asrv = AuthSrv(self.args, self.log)
             vfiles = [x for x in allfiles if x.startswith(top)]
             for fp in vfiles:
+                tctr += 1
                 rok, wok = self.can_rw(fp)
                 furl = fp.split("/", 1)[1]
                 durl = furl.rsplit("/", 1)[0] if "/" in furl else ""
@@ -112,39 +118,61 @@ class TestHttpCli(unittest.TestCase):
                     eprint("\033[33m{}\n# {}\033[0m".format(ret, url))
                     self.fail()
 
-                # tar
-                url = durl + "?tar"
-                h, b = self.curl(url, True)
-                # with open(os.path.join(td, "tar"), "wb") as f:
-                #    f.write(b)
-                try:
-                    tar = tarfile.open(fileobj=io.BytesIO(b), mode="r|").getnames()
-                except:
-                    if "HTTP/1.1 403 Forbidden" not in h and b != b"\nJ2EOT":
-                        eprint("bad tar?", url, h, b)
-                        raise
-                    tar = []
-                tar = [x.split("/", 1)[1] for x in tar]
-                tar = ["/".join([y for y in [top, durl, x] if y]) for x in tar]
-                tar = [[x] + self.can_rw(x) for x in tar]
-                tar_ok = [x[0] for x in tar if x[1]]
-                tar_ng = [x[0] for x in tar if not x[1]]
-                self.assertEqual([], tar_ng)
-
-                if durl.split("/")[-1] in self.can_read:
+                # expected files in archives
+                if rok:
                     ref = [x for x in vfiles if self.in_dive(top + "/" + durl, x)]
-                    for f in ref:
-                        ok = f in tar_ok
-                        pr = print if ok else eprint
-                        pr("{}: {}".format("ok" if ok else "NG", f))
                     ref.sort()
+                else:
+                    ref = []
+
+                if test_tar:
+                    url = durl + "?tar"
+                    h, b = self.curl(url, True)
+                    try:
+                        tar = tarfile.open(fileobj=io.BytesIO(b), mode="r|").getnames()
+                    except:
+                        if "HTTP/1.1 403 Forbidden" not in h and b != b"\nJ2EOT":
+                            eprint("bad tar?", url, h, b)
+                            raise
+                        tar = []
+                    tar = [x.split("/", 1)[1] for x in tar]
+                    tar = ["/".join([y for y in [top, durl, x] if y]) for x in tar]
+                    tar = [[x] + self.can_rw(x) for x in tar]
+                    tar_ok = [x[0] for x in tar if x[1]]
+                    tar_ng = [x[0] for x in tar if not x[1]]
                     tar_ok.sort()
                     self.assertEqual(ref, tar_ok)
+                    self.assertEqual([], tar_ng)
+
+                if test_zip:
+                    url = durl + "?zip"
+                    h, b = self.curl(url, True)
+                    try:
+                        with zipfile.ZipFile(io.BytesIO(b), "r") as zf:
+                            zfi = zf.infolist()
+                    except:
+                        if "HTTP/1.1 403 Forbidden" not in h and b != b"\nJ2EOT":
+                            eprint("bad zip?", url, h, b)
+                            raise
+                        zfi = []
+                    zfn = [x.filename.split("/", 1)[1] for x in zfi]
+                    zfn = ["/".join([y for y in [top, durl, x] if y]) for x in zfn]
+                    zfn = [[x] + self.can_rw(x) for x in zfn]
+                    zf_ok = [x[0] for x in zfn if x[1]]
+                    zf_ng = [x[0] for x in zfn if not x[1]]
+                    zf_ok.sort()
+                    self.assertEqual(ref, zf_ok)
+                    self.assertEqual([], zf_ng)
 
                 # stash
                 h, ret = self.put(url)
                 res = h.startswith("HTTP/1.1 201 ")
                 self.assertEqual(res, wok)
+                if wok:
+                    vp = h.split("\nLocation: http://a:1/")[1].split("\r")[0]
+                    vn, rem = self.asrv.vfs.get(vp, "*", False, False)
+                    ap = os.path.join(vn.realpath, rem)
+                    os.unlink(ap)
 
     def can_rw(self, fp):
         # lowest non-neutral folder declares permissions

tests/util.py

@@ -6,6 +6,7 @@ import os
 import platform
 import re
 import shutil
+import socket
 import subprocess as sp
 import sys
 import tempfile
@@ -68,6 +69,13 @@ def chkcmd(argv):
 
 def get_ramdisk():
     def subdir(top):
+        for d in os.listdir(top):
+            if not d.startswith("cptd-"):
+                continue
+            p = os.path.join(top, d)
+            st = os.stat(p)
+            if time.time() - st.st_mtime > 300:
+                shutil.rmtree(p)
         ret = os.path.join(top, "cptd-{}".format(os.getpid()))
         shutil.rmtree(ret, True)
         os.mkdir(ret)
@@ -111,13 +119,13 @@ class Cfg(Namespace):
     def __init__(self, a=None, v=None, c=None, **ka0):
         ka = {}
 
-        ex = "daw dav_auth dav_inf dav_mac dav_rt e2d e2ds e2dsa e2t e2ts e2tsr e2v e2vu e2vp early_ban ed emp exp force_js getmod grid gsel hardlink ih ihead magic never_symlink nid nih no_acode no_athumb no_dav no_dedup no_del no_dupe no_lifetime no_logues no_mv no_pipe no_poll no_readme no_robots no_sb_md no_sb_lg no_scandir no_tarcmp no_thumb no_vthumb no_zip nrand nw og og_no_head og_s_title q rand smb srch_dbg stats uqe vague_403 vc ver xdev xlink xvol"
+        ex = "daw dav_auth dav_inf dav_mac dav_rt e2d e2ds e2dsa e2t e2ts e2tsr e2v e2vu e2vp early_ban ed emp exp force_js getmod grid gsel hardlink ih ihead magic never_symlink nid nih no_acode no_athumb no_dav no_dedup no_del no_dupe no_lifetime no_logues no_mv no_pipe no_poll no_readme no_robots no_sb_md no_sb_lg no_scandir no_tarcmp no_thumb no_vthumb no_zip nrand nw og og_no_head og_s_title q rand smb srch_dbg stats uqe vague_403 vc ver write_uplog xdev xlink xvol"
         ka.update(**{k: False for k in ex.split()})
 
-        ex = "dotpart dotsrch no_dhash no_fastboot no_rescan no_sendfile no_snap no_voldump re_dhash plain_ip"
+        ex = "dotpart dotsrch hook_v no_dhash no_fastboot no_rescan no_sendfile no_snap no_voldump re_dhash plain_ip"
         ka.update(**{k: True for k in ex.split()})
 
-        ex = "ah_cli ah_gen css_browser hist js_browser mime mimes no_forget no_hash no_idx nonsus_urls og_tpl og_ua"
+        ex = "ah_cli ah_gen css_browser hist js_browser js_other mime mimes no_forget no_hash no_idx nonsus_urls og_tpl og_ua"
         ka.update(**{k: None for k in ex.split()})
 
         ex = "hash_mt srch_time u2abort u2j u2sz"
@@ -178,6 +186,10 @@ class Cfg(Namespace):
 
 
 class NullBroker(object):
+    def __init__(self, args, asrv):
+        self.args = args
+        self.asrv = asrv
+
     def say(self, *args):
         pass
 
@@ -189,6 +201,7 @@ class VSock(object):
     def __init__(self, buf):
         self._query = buf
         self._reply = b""
+        self.family = socket.AF_INET
         self.sendall = self.send
 
     def recv(self, sz):
@@ -213,7 +226,7 @@ class VHttpSrv(object):
         self.asrv = asrv
         self.log = log
 
-        self.broker = NullBroker()
+        self.broker = NullBroker(args, asrv)
         self.prism = None
         self.bans = {}
         self.nreq = 0