v1.9.5

permissions are now per-account instead of coalescing

also stops windows from freaking out if there's an offline volume

.github/pull_request_template.md

@@ -1,2 +1,2 @@
-Please include the following text somewhere in this PR description:  
+To show that your contribution is compatible with the MIT License, please include the following text somewhere in this PR description:  
 This PR complies with the DCO; https://developercertificate.org/  

.gitignore

@@ -37,6 +37,7 @@ up.*.txt
 .hist/
 scripts/docker/*.out
 scripts/docker/*.err
+/perf.*
 
 # nix build output link
 result

.vscode/launch.py

@@ -30,9 +30,17 @@ except:
 
 argv = [os.path.expanduser(x) if x.startswith("~") else x for x in argv]
 
+sfx = ""
+if len(sys.argv) > 1 and os.path.isfile(sys.argv[1]):
+    sfx = sys.argv[1]
+    sys.argv = [sys.argv[0]] + sys.argv[2:]
+
 argv += sys.argv[1:]
 
-if re.search(" -j ?[0-9]", " ".join(argv)):
+if sfx:
+    argv = [sys.executable, sfx] + argv
+    sp.check_call(argv)
+elif re.search(" -j ?[0-9]", " ".join(argv)):
     argv = [sys.executable, "-m", "copyparty"] + argv
     sp.check_call(argv)
 else:

.vscode/settings.json

@@ -35,34 +35,18 @@
     "python.linting.flake8Enabled": true,
     "python.linting.banditEnabled": true,
     "python.linting.mypyEnabled": true,
-    "python.linting.mypyArgs": [
-        "--ignore-missing-imports",
-        "--follow-imports=silent",
-        "--show-column-numbers",
-        "--strict"
-    ],
     "python.linting.flake8Args": [
         "--max-line-length=120",
-        "--ignore=E722,F405,E203,W503,W293,E402,E501,E128",
+        "--ignore=E722,F405,E203,W503,W293,E402,E501,E128,E226",
     ],
     "python.linting.banditArgs": [
-        "--ignore=B104"
-    ],
-    "python.linting.pylintArgs": [
-        "--disable=missing-module-docstring",
-        "--disable=missing-class-docstring",
-        "--disable=missing-function-docstring",
-        "--disable=import-outside-toplevel",
-        "--disable=wrong-import-position",
-        "--disable=raise-missing-from",
-        "--disable=bare-except",
-        "--disable=broad-except",
-        "--disable=invalid-name",
-        "--disable=line-too-long",
-        "--disable=consider-using-f-string"
+        "--ignore=B104,B110,B112"
     ],
     // python3 -m isort --py=27 --profile=black copyparty/
-    "python.formatting.provider": "black",
+    "python.formatting.provider": "none",
+    "[python]": {
+        "editor.defaultFormatter": "ms-python.black-formatter"
+    },
     "editor.formatOnSave": true,
     "[html]": {
         "editor.formatOnSave": false,
@@ -74,10 +58,6 @@
     "files.associations": {
         "*.makefile": "makefile"
     },
-    "python.formatting.blackArgs": [
-        "-t",
-        "py27"
-    ],
     "python.linting.enabled": true,
     "python.pythonPath": "/usr/bin/python3"
 }

README.md

@@ -41,6 +41,7 @@ turn almost any device into a file server with resumable uploads/downloads using
     * [batch rename](#batch-rename) - select some files and press `F2` to bring up the rename UI
     * [media player](#media-player) - plays almost every audio format there is
         * [audio equalizer](#audio-equalizer) - bass boosted
+        * [fix unreliable playback on android](#fix-unreliable-playback-on-android) - due to phone / app settings
     * [markdown viewer](#markdown-viewer) - and there are *two* editors
     * [other tricks](#other-tricks)
     * [searching](#searching) - search by size, date, path/name, mp3-tags, ...
@@ -65,10 +66,15 @@ turn almost any device into a file server with resumable uploads/downloads using
     * [file parser plugins](#file-parser-plugins) - provide custom parsers to index additional tags
     * [event hooks](#event-hooks) - trigger a program on uploads, renames etc ([examples](./bin/hooks/))
         * [upload events](#upload-events) - the older, more powerful approach ([examples](./bin/mtag/))
+    * [handlers](#handlers) - redefine behavior with plugins ([examples](./bin/handlers/))
     * [hiding from google](#hiding-from-google) - tell search engines you dont wanna be indexed
     * [themes](#themes)
     * [complete examples](#complete-examples)
     * [reverse-proxy](#reverse-proxy) - running copyparty next to other websites
+    * [prometheus](#prometheus) - metrics/stats can be enabled
+* [packages](#packages) - the party might be closer than you think
+    * [arch package](#arch-package) - now [available on aur](https://aur.archlinux.org/packages/copyparty) maintained by [@icxes](https://github.com/icxes)
+    * [fedora package](#fedora-package) - now [available on copr-pypi](https://copr.fedorainfracloud.org/coprs/g/copr/PyPI/)
     * [nix package](#nix-package) - `nix profile install github:9001/copyparty`
     * [nixos module](#nixos-module)
 * [browser support](#browser-support) - TLDR: yes
@@ -79,9 +85,10 @@ turn almost any device into a file server with resumable uploads/downloads using
 * [iOS shortcuts](#iOS-shortcuts) - there is no iPhone app, but
 * [performance](#performance) - defaults are usually fine - expect `8 GiB/s` download, `1 GiB/s` upload
     * [client-side](#client-side) - when uploading files
-* [security](#security) - some notes on hardening
+* [security](#security) - there is a [discord server](https://discord.gg/25J8CdTT6G)
     * [gotchas](#gotchas) - behavior that might be unexpected
     * [cors](#cors) - cross-site request config
+    * [password hashing](#password-hashing) - you can hash passwords
     * [https](#https) - both HTTP and HTTPS are accepted
 * [recovering from crashes](#recovering-from-crashes)
     * [client crashes](#client-crashes)
@@ -101,9 +108,9 @@ turn almost any device into a file server with resumable uploads/downloads using
 
 just run **[copyparty-sfx.py](https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.py)** -- that's it! 🎉
 
-* or install through pypi (python3 only): `python3 -m pip install --user -U copyparty`
+* or install through pypi: `python3 -m pip install --user -U copyparty`
 * or if you cannot install python, you can use [copyparty.exe](#copypartyexe) instead
-* or [install through nix](#nix-package), or [on NixOS](#nixos-module)
+* or install [on arch](#arch-package) ╱ [on fedora](#fedora-package) ╱ [on NixOS](#nixos-module) ╱ [through nix](#nix-package)
 * or if you are on android, [install copyparty in termux](#install-on-android)
 * or if you prefer to [use docker](./scripts/docker/) 🐋 you can do that too
   * docker has all deps built-in, so skip this step:
@@ -123,7 +130,7 @@ enable thumbnails (images/audio/video), media indexing, and audio transcoding by
 
 running copyparty without arguments (for example doubleclicking it on Windows) will give everyone read/write access to the current folder; you may want [accounts and volumes](#accounts-and-volumes)
 
-or see [complete windows example](./docs/examples/windows.md)
+or see [some usage examples](#complete-examples) for inspiration, or the [complete windows example](./docs/examples/windows.md)
 
 some recommended options:
 * `-e2dsa` enables general [file indexing](#file-indexing)
@@ -275,9 +282,14 @@ server notes:
 
 * [Firefox issue 1790500](https://bugzilla.mozilla.org/show_bug.cgi?id=1790500) -- entire browser can crash after uploading ~4000 small files
 
+* Android: music playback randomly stops due to [battery usage settings](#fix-unreliable-playback-on-android)
+
 * iPhones: the volume control doesn't work because [apple doesn't want it to](https://developer.apple.com/library/archive/documentation/AudioVideo/Conceptual/Using_HTML5_Audio_Video/Device-SpecificConsiderations/Device-SpecificConsiderations.html#//apple_ref/doc/uid/TP40009523-CH5-SW11)
-  * *future workaround:* enable the equalizer, make it all-zero, and set a negative boost to reduce the volume
-    * "future" because `AudioContext` can't maintain a stable playback speed in the current iOS version (15.7), maybe one day...
+  * `AudioContext` will probably never be a viable workaround as apple introduces new issues faster than they fix current ones
+
+* iPhones: the preload feature (in the media-player-options tab) can cause a tiny audio glitch 20sec before the end of each song, but disabling it may cause worse iOS bugs to appear instead
+  * just a hunch, but disabling preloading may cause playback to stop entirely, or possibly mess with bluetooth speakers
+  * tried to add a tooltip regarding this but looks like apple broke my tooltips
 
 * Windows: folders cannot be accessed if the name ends with `.`
   * python or windows bug
@@ -287,6 +299,7 @@ server notes:
 
 * VirtualBox: sqlite throws `Disk I/O Error` when running in a VM and the up2k database is in a vboxsf
   * use `--hist` or the `hist` volflag (`-v [...]:c,hist=/tmp/foo`) to place the db inside the vm instead
+  * also happens on mergerfs, so put the db elsewhere
 
 * Ubuntu: dragging files from certain folders into firefox or chrome is impossible
   * due to snap security policies -- see `snap connections firefox` for the allowlist, `removable-media` permits all of `/mnt` and `/media` apparently
@@ -300,7 +313,7 @@ upgrade notes
   * http-api: delete/move is now `POST` instead of `GET`
   * everything other than `GET` and `HEAD` must pass [cors validation](#cors)
 * `1.5.0` (2022-12-03): [new chunksize formula](https://github.com/9001/copyparty/commit/54e1c8d261df) for files larger than 128 GiB
-  * **users:** upgrade to the latest [cli uploader](https://github.com/9001/copyparty/blob/hovudstraum/bin/up2k.py) if you use that
+  * **users:** upgrade to the latest [cli uploader](https://github.com/9001/copyparty/blob/hovudstraum/bin/u2c.py) if you use that
   * **devs:** update third-party up2k clients (if those even exist)
 
 
@@ -319,7 +332,7 @@ upgrade notes
 # accounts and volumes
 
 per-folder, per-user permissions  - if your setup is getting complex, consider making a [config file](./docs/example.conf) instead of using arguments
-* much easier to manage, and you can modify the config at runtime with `systemctl reload copyparty` or more conveniently using the `[reload cfg]` button in the control-panel (if logged in as admin)
+* much easier to manage, and you can modify the config at runtime with `systemctl reload copyparty` or more conveniently using the `[reload cfg]` button in the control-panel (if the user has `a`/admin in any volume)
   * changes to the `[global]` config section requires a restart to take effect
 
 a quick summary can be seen using `--help-accounts`
@@ -338,6 +351,8 @@ permissions:
 * `d` (delete): delete files/folders
 * `g` (get): only download files, cannot see folder contents or zip/tar
 * `G` (upget): same as `g` except uploaders get to see their own filekeys (see `fk` in examples below)
+* `h` (html): same as `g` except folders return their index.html, and filekeys are not necessary for index.html
+* `a` (admin): can see uploader IPs, config-reload
 
 examples:
 * add accounts named u1, u2, u3 with passwords p1, p2, p3: `-a u1:p1 -a u2:p2 -a u3:p3`
@@ -466,6 +481,7 @@ click the `🌲` or pressing the `B` hotkey to toggle between breadcrumbs path (
 ## thumbnails
 
 press `g` or `田` to toggle grid-view instead of the file listing  and `t` toggles icons / thumbnails
+* can be made default globally with `--grid` or per-volume with volflag `grid`
 
 ![copyparty-thumbs-fs8](https://user-images.githubusercontent.com/241032/129636211-abd20fa2-a953-4366-9423-1c88ebb96ba9.png)
 
@@ -476,10 +492,14 @@ it does static images with Pillow / pyvips / FFmpeg, and uses FFmpeg for video f
 audio files are covnerted into spectrograms using FFmpeg unless you `--no-athumb` (and some FFmpeg builds may need `--th-ff-swr`)
 
 images with the following names (see `--th-covers`) become the thumbnail of the folder they're in: `folder.png`, `folder.jpg`, `cover.png`, `cover.jpg`
+* and, if you enable [file indexing](#file-indexing), all remaining folders will also get thumbnails (as long as they contain any pics at all)
 
 in the grid/thumbnail view, if the audio player panel is open, songs will start playing when clicked
 * indicated by the audio files having the ▶ icon instead of 💾
 
+enabling `multiselect` lets you click files to select them, and then shift-click another file for range-select
+* `multiselect` is mostly intended for phones/tablets, but the `sel` option in the `[⚙️] settings` tab is better suited for desktop use, allowing selection by CTRL-clicking and range-selection with SHIFT-click, all without affecting regular clicking
+
 
 ## zip downloads
 
@@ -490,10 +510,16 @@ select which type of archive you want in the `[⚙️] config` tab:
 | name | url-suffix | description |
 |--|--|--|
 | `tar` | `?tar` | plain gnutar, works great with `curl \| tar -xv` |
+| `pax` | `?tar=pax` | pax-format tar, futureproof, not as fast |
+| `tgz` | `?tar=gz` | gzip compressed gnu-tar (slow), for `curl \| tar -xvz` |
+| `txz` | `?tar=xz` | gnu-tar with xz / lzma compression (v.slow) |
 | `zip` | `?zip=utf8` | works everywhere, glitchy filenames on win7 and older |
 | `zip_dos` | `?zip` | traditional cp437 (no unicode) to fix glitchy filenames |
 | `zip_crc` | `?zip=crc` | cp437 with crc32 computed early for truly ancient software |
 
+* gzip default level is `3` (0=fast, 9=best), change with `?tar=gz:9`
+* xz default level is `1` (0=fast, 9=best), change with `?tar=xz:9`
+* bz2 default level is `2` (1=fast, 9=best), change with `?tar=bz2:9`
 * hidden files (dotfiles) are excluded unless `-ed`
   * `up2k.db` and `dir.txt` is always excluded
 * `zip_crc` will take longer to download since the server has to read each file twice
@@ -504,10 +530,14 @@ you can also zip a selection of files or folders by clicking them in the browser
 
 ![copyparty-zipsel-fs8](https://user-images.githubusercontent.com/241032/129635374-e5136e01-470a-49b1-a762-848e8a4c9cdc.png)
 
+cool trick: download a folder by appending url-params `?tar&opus` to transcode all audio files (except aac|m4a|mp3|ogg|opus|wma) to opus before they're added to the archive
+* super useful if you're 5 minutes away from takeoff and realize you don't have any music on your phone but your server only has flac files and downloading those will burn through all your data + there wouldn't be enough time anyways
+* and url-params `&j` / `&w` produce jpeg/webm thumbnails/spectrograms instead of the original audio/video/images
+
 
 ## uploading
 
-drag files/folders into the web-browser to upload  (or use the [command-line uploader](https://github.com/9001/copyparty/tree/hovudstraum/bin#up2kpy))
+drag files/folders into the web-browser to upload  (or use the [command-line uploader](https://github.com/9001/copyparty/tree/hovudstraum/bin#u2cpy))
 
 this initiates an upload using `up2k`; there are two uploaders available:
 * `[🎈] bup`, the basic uploader, supports almost every browser since netscape 4.0
@@ -602,6 +632,7 @@ file selection: click somewhere on the line (not the link itsef), then:
 * `up/down` to move
 * `shift-up/down` to move-and-select
 * `ctrl-shift-up/down` to also scroll
+* shift-click another line for range-select
 
 * cut: select some files and `ctrl-x`
 * paste: `ctrl-v` in another folder
@@ -685,7 +716,7 @@ open the `[🎺]` media-player-settings tab to configure it,
   * `[loop]` keeps looping the folder
   * `[next]` plays into the next folder
 * transcode:
-  * `[flac]` convers `flac` and `wav` files into opus
+  * `[flac]` converts `flac` and `wav` files into opus
   * `[aac]` converts `aac` and `m4a` files into opus
   * `[oth]` converts all other known formats into opus
     * `aac|ac3|aif|aiff|alac|alaw|amr|ape|au|dfpwm|dts|flac|gsm|it|m4a|mo3|mod|mp2|mp3|mpc|mptm|mt2|mulaw|ogg|okt|opus|ra|s3m|tak|tta|ulaw|wav|wma|wv|xm|xpk`
@@ -700,6 +731,13 @@ can also boost the volume in general, or increase/decrease stereo width (like [c
 
 has the convenient side-effect of reducing the pause between songs, so gapless albums play better with the eq enabled (just make it flat)
 
+not available on iPhones / iPads because AudioContext currently breaks background audio playback on iOS (15.7.8)
+
+
+### fix unreliable playback on android
+
+due to phone / app settings,  android phones may randomly stop playing music when the power saver kicks in, especially at the end of an album -- you can fix it by [disabling power saving](https://user-images.githubusercontent.com/241032/235262123-c328cca9-3930-4948-bd18-3949b9fd3fcf.png) in the [app settings](https://user-images.githubusercontent.com/241032/235262121-2ffc51ae-7821-4310-a322-c3b7a507890c.png) of the browser you use for music streaming (preferably a dedicated one)
+
 
 ## markdown viewer
 
@@ -758,7 +796,7 @@ for the above example to work, add the commandline argument `-e2ts` to also scan
 using arguments or config files, or a mix of both:
 * config files (`-c some.conf`) can set additional commandline arguments; see [./docs/example.conf](docs/example.conf) and [./docs/example2.conf](docs/example2.conf)
 * `kill -s USR1` (same as `systemctl reload copyparty`) to reload accounts and volumes from config files without restarting
-  * or click the `[reload cfg]` button in the control-panel when logged in as admin 
+  * or click the `[reload cfg]` button in the control-panel if the user has `a`/admin in any volume
   * changes to the `[global]` config section requires a restart to take effect
 
 
@@ -866,15 +904,13 @@ unsafe, slow, not recommended for wan,  enable with `--smb` for read-only or `--
 
 click the [connect](http://127.0.0.1:3923/?hc) button in the control-panel to see connection instructions for windows, linux, macos
 
-dependencies: `python3 -m pip install --user -U impacket==0.10.0`
+dependencies: `python3 -m pip install --user -U impacket==0.11.0`
 * newer versions of impacket will hopefully work just fine but there is monkeypatching so maybe not
 
 some **BIG WARNINGS** specific to SMB/CIFS, in decreasing importance:
 * not entirely confident that read-only is read-only
 * the smb backend is not fully integrated with vfs, meaning there could be security issues (path traversal). Please use `--smb-port` (see below) and [prisonparty](./bin/prisonparty.sh)
-  * account passwords work per-volume as expected, but account permissions are coalesced; all accounts have read-access to all volumes, and if a single account has write-access to some volume then all other accounts also do
-    * if no accounts have write-access to a specific volume, or if `--smbw` is not set, then writing to that volume from smb *should* be impossible
-    * will be fixed once [impacket v0.11.0](https://github.com/SecureAuthCorp/impacket/commit/d923c00f75d54b972bca573a211a82f09b55261a) is released
+  * account passwords work per-volume as expected, and so does account permissions (read/write/move/delete), but `--smbw` must be given to allow write-access from smb
   * [shadowing](#shadowing) probably works as expected but no guarantees
 
 and some minor issues,
@@ -885,7 +921,7 @@ and some minor issues,
   * win10 onwards does not allow connecting anonymously / without accounts
 * on windows, creating a new file through rightclick --> new --> textfile throws an error due to impacket limitations -- hit OK and F5 to get your file
 * python3 only
-* slow
+* slow (the builtin webdav support in windows is 5x faster, and rclone-webdav is 30x faster)
 
 known client bugs:
 * on win7 only, `--smb1` is much faster than smb2 (default) because it keeps rescanning folders on smb2
@@ -919,14 +955,13 @@ through arguments:
 * `--xlink` enables deduplication across volumes
 
 the same arguments can be set as volflags, in addition to `d2d`, `d2ds`, `d2t`, `d2ts`, `d2v` for disabling:
-* `-v ~/music::r:c,e2dsa,e2tsr` does a full reindex of everything on startup
+* `-v ~/music::r:c,e2ds,e2tsr` does a full reindex of everything on startup
 * `-v ~/music::r:c,d2d` disables **all** indexing, even if any `-e2*` are on
 * `-v ~/music::r:c,d2t` disables all `-e2t*` (tags), does not affect `-e2d*`
 * `-v ~/music::r:c,d2ds` disables on-boot scans; only index new uploads
 * `-v ~/music::r:c,d2ts` same except only affecting tags
 
 note:
-* the parser can finally handle `c,e2dsa,e2tsr` so you no longer have to `c,e2dsa:c,e2tsr`
 * `e2tsr` is probably always overkill, since `e2ds`/`e2dsa` would pick up any file modifications and `e2ts` would then reindex those, unless there is a new copyparty version with new parsers and the release note says otherwise
 * the rescan button in the admin panel has no effect unless the volume has `-e2ds` or higher
 * deduplication is possible on windows if you run copyparty as administrator (not saying you should!)
@@ -948,7 +983,11 @@ avoid traversing into other filesystems  using `--xdev` / volflag `:c,xdev`, ski
 
 and/or you can `--xvol` / `:c,xvol` to ignore all symlinks leaving the volume's top directory, but still allow bind-mounts pointing elsewhere
 
-**NB: only affects the indexer** -- users can still access anything inside a volume, unless shadowed by another volume
+* symlinks are permitted with `xvol` if they point into another volume where the user has the same level of access
+
+these options will reduce performance; unlikely worst-case estimates are 14% reduction for directory listings, 35% for download-as-tar
+
+as of copyparty v1.7.0 these options also prevent file access at runtime -- in previous versions it was just hints for the indexer
 
 ### periodic rescan
 
@@ -965,6 +1004,8 @@ set upload rules using volflags,  some examples:
 
 * `:c,sz=1k-3m` sets allowed filesize between 1 KiB and 3 MiB inclusive (suffixes: `b`, `k`, `m`, `g`)
 * `:c,df=4g` block uploads if there would be less than 4 GiB free disk space afterwards
+* `:c,vmaxb=1g` block uploads if total volume size would exceed 1 GiB afterwards
+* `:c,vmaxn=4k` block uploads if volume would contain more than 4096 files afterwards
 * `:c,nosub` disallow uploading into subdirectories; goes well with `rotn` and `rotf`:
 * `:c,rotn=1000,2` moves uploads into subfolders, up to 1000 files in each folder before making a new one, two levels deep (must be at least 1)
 * `:c,rotf=%Y/%m/%d/%H` enforces files to be uploaded into a structure of subfolders according to that date format
@@ -978,6 +1019,9 @@ you can also set transaction limits which apply per-IP and per-volume, but these
 * `:c,maxn=250,3600` allows 250 files over 1 hour from each IP (tracked per-volume)
 * `:c,maxb=1g,300` allows 1 GiB total over 5 minutes from each IP (tracked per-volume)
 
+notes:
+* `vmaxb` and `vmaxn` requires either the `e2ds` volflag or `-e2dsa` global-option
+
 
 ## compress uploads
 
@@ -1108,6 +1152,13 @@ note that this is way more complicated than the new [event hooks](#event-hooks)
 note that it will occupy the parsing threads, so fork anything expensive (or set `kn` to have copyparty fork it for you) -- otoh if you want to intentionally queue/singlethread you can combine it with `--mtag-mt 1`
 
 
+## handlers
+
+redefine behavior with plugins ([examples](./bin/handlers/))
+
+replace 404 and 403 errors with something completely different (that's it for now)
+
+
 ## hiding from google
 
 tell search engines you dont wanna be indexed,  either using the good old [robots.txt](https://www.robotstxt.org/robotstxt.html) or through copyparty settings:
@@ -1144,9 +1195,33 @@ see the top of [./copyparty/web/browser.css](./copyparty/web/browser.css) where
 
 ## complete examples
 
-* [running on windows](./docs/examples/windows.md)
+* see [running on windows](./docs/examples/windows.md) for a fancy windows setup
 
-* read-only music server  
+  * or use any of the examples below, just replace `python copyparty-sfx.py` with `copyparty.exe` if you're using the exe edition
+
+* allow anyone to download or upload files into the current folder:  
+  `python copyparty-sfx.py`
+
+  * enable searching and music indexing with `-e2dsa -e2ts`
+
+  * start an FTP server on port 3921 with `--ftp 3921`
+
+  * announce it on your LAN with `-z` so it appears in windows/Linux file managers
+
+* anyone can upload, but nobody can see any files (even the uploader):  
+  `python copyparty-sfx.py -e2dsa -v .::w`
+
+  * block uploads if there's less than 4 GiB free disk space with `--df 4`
+
+  * show a popup on new uploads with `--xau bin/hooks/notify.py`
+
+* anyone can upload, and receive "secret" links for each upload they do:  
+  `python copyparty-sfx.py -e2dsa -v .::wG:c,fk=8`
+
+* anyone can browse, only `kevin` (password `okgo`) can upload/move/delete files:  
+  `python copyparty-sfx.py -e2dsa -a kevin:okgo -v .::r:rwmd,kevin`
+
+* read-only music server:  
   `python copyparty-sfx.py -v /mnt/nas/music:/music:r -e2dsa -e2ts --no-robots --force-js --theme 2`
   
   * ...with bpm and key scanning  
@@ -1169,6 +1244,7 @@ you can either:
   * if copyparty says `incorrect --rp-loc or webserver config; expected vpath starting with [...]` it's likely because the webserver is stripping away the proxy location from the request URLs -- see the `ProxyPass` in the apache example below
 
 some reverse proxies (such as [Caddy](https://caddyserver.com/)) can automatically obtain a valid https/tls certificate for you, and some support HTTP/2 and QUIC which could be a nice speed boost
+* **warning:** nginx-QUIC is still experimental and can make uploads much slower, so HTTP/2 is recommended for now
 
 example webserver configs:
 
@@ -1176,6 +1252,74 @@ example webserver configs:
 * [apache2 config](contrib/apache/copyparty.conf) -- location-based
 
 
+## prometheus
+
+metrics/stats can be enabled  at URL `/.cpr/metrics` for grafana / prometheus / etc (openmetrics 1.0.0)
+
+must be enabled with `--stats` since it reduces startup time a tiny bit, and you probably want `-e2dsa` too
+
+the endpoint is only accessible by `admin` accounts, meaning the `a` in `rwmda` in the following example commandline: `python3 -m copyparty -a ed:wark -v /mnt/nas::rwmda,ed --stats -e2dsa`
+
+follow a guide for setting up `node_exporter` except have it read from copyparty instead; example `/etc/prometheus/prometheus.yml` below
+
+```yaml
+scrape_configs:
+  - job_name: copyparty
+    metrics_path: /.cpr/metrics
+    basic_auth:
+      password: wark
+    static_configs:
+      - targets: ['192.168.123.1:3923']
+```
+
+currently the following metrics are available,
+* `cpp_uptime_seconds`
+* `cpp_bans` number of banned IPs
+
+and these are available per-volume only:
+* `cpp_disk_size_bytes` total HDD size
+* `cpp_disk_free_bytes` free HDD space
+
+and these are per-volume and `total`:
+* `cpp_vol_bytes` size of all files in volume
+* `cpp_vol_files` number of files
+* `cpp_dupe_bytes` disk space presumably saved by deduplication
+* `cpp_dupe_files` number of dupe files
+* `cpp_unf_bytes` currently unfinished / incoming uploads
+
+some of the metrics have additional requirements to function correctly,
+* `cpp_vol_*` requires either the `e2ds` volflag or `-e2dsa` global-option
+
+the following options are available to disable some of the metrics:
+* `--nos-hdd` disables `cpp_disk_*` which can prevent spinning up HDDs
+* `--nos-vol` disables `cpp_vol_*` which reduces server startup time
+* `--nos-dup` disables `cpp_dupe_*` which reduces the server load caused by prometheus queries
+* `--nos-unf` disables `cpp_unf_*` for no particular purpose
+
+
+# packages
+
+the party might be closer than you think
+
+
+## arch package
+
+now [available on aur](https://aur.archlinux.org/packages/copyparty) maintained by [@icxes](https://github.com/icxes)
+
+
+## fedora package
+
+now [available on copr-pypi](https://copr.fedorainfracloud.org/coprs/g/copr/PyPI/)  , maintained autonomously -- [track record](https://copr.fedorainfracloud.org/coprs/g/copr/PyPI/package/python-copyparty/) seems OK
+
+```bash
+dnf copr enable @copr/PyPI
+dnf install python3-copyparty  # just a minimal install, or... 
+dnf install python3-{copyparty,pillow,argon2-cffi,pyftpdlib,pyOpenSSL} ffmpeg-free  # with recommended deps
+```
+
+this *may* also work on RHEL but [I'm not paying IBM to verify that](https://www.jeffgeerling.com/blog/2023/dear-red-hat-are-you-dumb)
+
+
 ## nix package
 
 `nix profile install github:9001/copyparty`
@@ -1350,10 +1494,10 @@ interact with copyparty using non-browser clients
   * `(printf 'PUT /junk?pw=wark HTTP/1.1\r\n\r\n'; cat movie.mkv) | nc 127.0.0.1 3923`
   * `(printf 'PUT / HTTP/1.1\r\n\r\n'; cat movie.mkv) >/dev/tcp/127.0.0.1/3923`
 
-* python: [up2k.py](https://github.com/9001/copyparty/blob/hovudstraum/bin/up2k.py) is a command-line up2k client [(webm)](https://ocv.me/stuff/u2cli.webm)
+* python: [u2c.py](https://github.com/9001/copyparty/blob/hovudstraum/bin/u2c.py) is a command-line up2k client [(webm)](https://ocv.me/stuff/u2cli.webm)
   * file uploads, file-search, [folder sync](#folder-sync), autoresume of aborted/broken uploads
-  * can be downloaded from copyparty: controlpanel -> connect -> [up2k.py](http://127.0.0.1:3923/.cpr/a/up2k.py)
-  * see [./bin/README.md#up2kpy](bin/README.md#up2kpy)
+  * can be downloaded from copyparty: controlpanel -> connect -> [u2c.py](http://127.0.0.1:3923/.cpr/a/u2c.py)
+  * see [./bin/README.md#u2cpy](bin/README.md#u2cpy)
 
 * FUSE: mount a copyparty server as a local filesystem
   * cross-platform python client available in [./bin/](bin/)
@@ -1376,11 +1520,11 @@ NOTE: curl will not send the original filename if you use `-T` combined with url
 
 sync folders to/from copyparty
 
-the commandline uploader [up2k.py](https://github.com/9001/copyparty/tree/hovudstraum/bin#up2kpy) with `--dr` is the best way to sync a folder to copyparty; verifies checksums and does files in parallel, and deletes unexpected files on the server after upload has finished which makes file-renames really cheap (it'll rename serverside and skip uploading)
+the commandline uploader [u2c.py](https://github.com/9001/copyparty/tree/hovudstraum/bin#u2cpy) with `--dr` is the best way to sync a folder to copyparty; verifies checksums and does files in parallel, and deletes unexpected files on the server after upload has finished which makes file-renames really cheap (it'll rename serverside and skip uploading)
 
 alternatively there is [rclone](./docs/rclone.md) which allows for bidirectional sync and is *way* more flexible (stream files straight from sftp/s3/gcs to copyparty, ...), although there is no integrity check and it won't work with files over 100 MiB if copyparty is behind cloudflare
 
-* starting from rclone v1.63 (currently [in beta](https://beta.rclone.org/?filter=latest)), rclone will also be faster than up2k.py
+* starting from rclone v1.63 (currently [in beta](https://beta.rclone.org/?filter=latest)), rclone will also be faster than u2c.py
 
 
 ## mount as drive
@@ -1447,7 +1591,7 @@ when uploading files,
 * chrome is recommended, at least compared to firefox:
   * up to 90% faster when hashing, especially on SSDs
   * up to 40% faster when uploading over extremely fast internets
-  * but [up2k.py](https://github.com/9001/copyparty/blob/hovudstraum/bin/up2k.py) can be 40% faster than chrome again
+  * but [u2c.py](https://github.com/9001/copyparty/blob/hovudstraum/bin/u2c.py) can be 40% faster than chrome again
 
 * if you're cpu-bottlenecked, or the browser is maxing a cpu core:
   * up to 30% faster uploads if you hide the upload status list by switching away from the `[🚀]` up2k ui-tab (or closing it)
@@ -1458,10 +1602,14 @@ when uploading files,
 
 # security
 
+there is a [discord server](https://discord.gg/25J8CdTT6G)  with an `@everyone` for all important updates (at the lack of better ideas)
+
 some notes on hardening
 
 * set `--rproxy 0` if your copyparty is directly facing the internet (not through a reverse-proxy)
   * cors doesn't work right otherwise
+* if you allow anonymous uploads or otherwise don't trust the contents of a volume, you can prevent XSS with volflag `nohtml`
+  * this returns html documents as plaintext, and also disables markdown rendering
 
 safety profiles:
 
@@ -1475,9 +1623,9 @@ safety profiles:
   * `--unpost 0`, `--no-del`, `--no-mv` disables all move/delete support
   * `--hardlink` creates hardlinks instead of symlinks when deduplicating uploads, which is less maintenance
     * however note if you edit one file it will also affect the other copies
-  * `--vague-401` returns a "404 not found" instead of "401 unauthorized" which is a common enterprise meme
+  * `--vague-403` returns a "404 not found" instead of "401 unauthorized" which is a common enterprise meme
   * `--ban-404=50,60,1440` ban client for 1440min (24h) if they hit 50 404's in 60min
-    * **NB:** will ban anyone who enables up2k turbo
+    * `--turbo=-1` to force-disable turbo-mode in the uploader which could otherwise hit the 404-ban
   * `--nih` removes the server hostname from directory listings
 
 * option `-sss` is a shortcut for the above plus:
@@ -1492,6 +1640,8 @@ other misc notes:
   * combine this with volflag `c,fk` to generate filekeys (per-file accesskeys); users which have full read-access will then see URLs with `?k=...` appended to the end, and `g` users must provide that URL including the correct key to avoid a 404
     * the default filekey entropy is fairly small so give `--fk-salt` around 30 characters if you want filekeys longer than 16 chars
   * permissions `wG` lets users upload files and receive their own filekeys, still without being able to see other uploads
+  * permission `h` instead of `r` makes copyparty behave like a traditional webserver with directory listing/index disabled, returning index.html instead
+    * compatibility with filekeys: index.html itself can be retrieved without the correct filekey, but all other files are protected
 
 
 ## gotchas
@@ -1499,10 +1649,12 @@ other misc notes:
 behavior that might be unexpected
 
 * users without read-access to a folder can still see the `.prologue.html` / `.epilogue.html` / `README.md` contents, for the purpose of showing a description on how to use the uploader for example
-* users can submit `<script>`s which autorun for other visitors in a few ways;
+* users can submit `<script>`s which autorun (in a sandbox) for other visitors in a few ways;
   * uploading a `README.md` -- avoid with `--no-readme`
   * renaming `some.html` to `.epilogue.html` -- avoid with either `--no-logues` or `--no-dot-ren`
-  * the directory-listing embed is sandboxed (so any malicious scripts can't do any damage) but the markdown editor is not
+  * the directory-listing embed is sandboxed (so any malicious scripts can't do any damage) but the markdown editor is not 100% safe, see below
+* markdown documents can contain html and `<script>`s; attempts are made to prevent scripts from executing (unless `-emp` is specified) but this is not 100% bulletproof, so setting the `nohtml` volflag is still the safest choice
+  * or eliminate the problem entirely by only giving write-access to trustworthy people :^)
 
 
 ## cors
@@ -1517,12 +1669,28 @@ by default, except for `GET` and `HEAD` operations, all requests must either:
 cors can be configured with `--acao` and `--acam`, or the protections entirely disabled with `--allow-csrf`
 
 
+## password hashing
+
+you can hash passwords  before putting them into config files / providing them as arguments; see `--help-pwhash` for all the details
+
+`--ah-alg argon2` enables it, and if you have any plaintext passwords then it'll print the hashed versions on startup so you can replace them
+
+optionally also specify `--ah-cli` to enter an interactive mode where it will hash passwords without ever writing the plaintext ones to disk
+
+the default configs take about 0.4 sec and 256 MiB RAM to process a new password on a decent laptop
+
+
 ## https
 
 both HTTP and HTTPS are accepted  by default, but letting a [reverse proxy](#reverse-proxy) handle the https/tls/ssl would be better (probably more secure by default)
 
 copyparty doesn't speak HTTP/2 or QUIC, so using a reverse proxy would solve that as well
 
+if [cfssl](https://github.com/cloudflare/cfssl/releases/latest) is installed, copyparty will automatically create a CA and server-cert on startup
+* the certs are written to `--crt-dir` for distribution, see `--help` for the other `--crt` options
+* this will be a self-signed certificate so you must install your `ca.pem` into all your browsers/devices
+* if you want to avoid the hassle of distributing certs manually, please consider using a reverse proxy
+
 
 # recovering from crashes
 
@@ -1559,6 +1727,8 @@ mandatory deps:
 
 install these to enable bonus features
 
+enable hashed passwords in config: `argon2-cffi`
+
 enable ftp-server:
 * for just plaintext FTP, `pyftpdlib` (is built into the SFX)
 * with TLS encryption, `pyftpdlib pyopenssl`
@@ -1575,7 +1745,7 @@ enable [thumbnails](#thumbnails) of...
 * **JPEG XL pictures:** `pyvips` or `ffmpeg`
 
 enable [smb](#smb-server) support (**not** recommended):
-* `impacket==0.10.0`
+* `impacket==0.11.0`
 
 `pyvips` gives higher quality thumbnails than `Pillow` and is 320% faster, using 270% more ram: `sudo apt install libvips42 && python3 -m pip install --user -U pyvips`
 

bin/README.md

@@ -1,4 +1,4 @@
-# [`up2k.py`](up2k.py)
+# [`u2c.py`](u2c.py)
 * command-line up2k client [(webm)](https://ocv.me/stuff/u2cli.webm)
 * file uploads, file-search, autoresume of aborted/broken uploads
 * sync local folder to server

bin/handlers/README.md

@@ -0,0 +1,35 @@
+replace the standard 404 / 403 responses with plugins
+
+
+# usage
+
+load plugins either globally with `--on404 ~/dev/copyparty/bin/handlers/sorry.py` or for a specific volume with `:c,on404=~/handlers/sorry.py`
+
+
+# api
+
+each plugin must define a `main()` which takes 3 arguments;
+
+* `cli` is an instance of [copyparty/httpcli.py](https://github.com/9001/copyparty/blob/hovudstraum/copyparty/httpcli.py) (the monstrosity itself)
+* `vn` is the VFS which overlaps with the requested URL, and
+* `rem` is the URL remainder below the VFS mountpoint
+    * so `vn.vpath + rem` == `cli.vpath` == original request
+
+
+# examples
+
+## on404
+
+* [sorry.py](answer.py) replies with a custom message instead of the usual 404
+* [nooo.py](nooo.py) replies with an endless noooooooooooooo
+* [never404.py](never404.py) 100% guarantee that 404 will never be a thing again as it automatically creates dummy files whenever necessary
+* [caching-proxy.py](caching-proxy.py) transforms copyparty into a squid/varnish knockoff
+
+## on403
+
+* [ip-ok.py](ip-ok.py) disables security checks if client-ip is 1.2.3.4
+
+
+# notes
+
+* on403 only works for trivial stuff (basic http access) since I haven't been able to think of any good usecases for it (was just easy to add while doing on404)

bin/handlers/caching-proxy.py

@@ -0,0 +1,36 @@
+# assume each requested file exists on another webserver and
+# download + mirror them as they're requested
+# (basically pretend we're warnish)
+
+import os
+import requests
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from copyparty.httpcli import HttpCli
+
+
+def main(cli: "HttpCli", vn, rem):
+    url = "https://mirrors.edge.kernel.org/alpine/" + rem
+    abspath = os.path.join(vn.realpath, rem)
+
+    # sneaky trick to preserve a requests-session between downloads
+    # so it doesn't have to spend ages reopening https connections;
+    # luckily we can stash it inside the copyparty client session,
+    # name just has to be definitely unused so "hacapo_req_s" it is
+    req_s = getattr(cli.conn, "hacapo_req_s", None) or requests.Session()
+    setattr(cli.conn, "hacapo_req_s", req_s)
+
+    try:
+        os.makedirs(os.path.dirname(abspath), exist_ok=True)
+        with req_s.get(url, stream=True, timeout=69) as r:
+            r.raise_for_status()
+            with open(abspath, "wb", 64 * 1024) as f:
+                for buf in r.iter_content(chunk_size=64 * 1024):
+                    f.write(buf)
+    except:
+        os.unlink(abspath)
+        return "false"
+
+    return "retry"

bin/handlers/ip-ok.py

@@ -0,0 +1,6 @@
+# disable permission checks and allow access if client-ip is 1.2.3.4
+
+
+def main(cli, vn, rem):
+    if cli.ip == "1.2.3.4":
+        return "allow"

bin/handlers/never404.py

@@ -0,0 +1,11 @@
+# create a dummy file and let copyparty return it
+
+
+def main(cli, vn, rem):
+    print("hello", cli.ip)
+
+    abspath = vn.canonical(rem)
+    with open(abspath, "wb") as f:
+        f.write(b"404? not on MY watch!")
+
+    return "retry"

bin/handlers/nooo.py

@@ -0,0 +1,16 @@
+# reply with an endless "noooooooooooooooooooooooo"
+
+
+def say_no():
+    yield b"n"
+    while True:
+        yield b"o" * 4096
+
+
+def main(cli, vn, rem):
+    cli.send_headers(None, 404, "text/plain")
+
+    for chunk in say_no():
+        cli.s.sendall(chunk)
+
+    return "false"

bin/handlers/sorry.py

@@ -0,0 +1,7 @@
+# sends a custom response instead of the usual 404
+
+
+def main(cli, vn, rem):
+    msg = f"sorry {cli.ip} but {cli.vpath} doesn't exist"
+
+    return str(cli.reply(msg.encode("utf-8"), 404, "text/plain"))

bin/hooks/wget.py

@@ -37,6 +37,10 @@ def main():
     if "://" not in url:
         url = "https://" + url
 
+    proto = url.split("://")[0].lower()
+    if proto not in ("http", "https", "ftp", "ftps"):
+        raise Exception("bad proto {}".format(proto))
+
     os.chdir(inf["ap"])
 
     name = url.split("?")[0].split("/")[-1]

bin/mtag/README.md

@@ -24,6 +24,15 @@ these do not have any problematic dependencies at all:
   * also available as an [event hook](../hooks/wget.py)
 
 
+## dangerous plugins
+
+plugins in this section should only be used with appropriate precautions:
+
+* [very-bad-idea.py](./very-bad-idea.py) combined with [meadup.js](https://github.com/9001/copyparty/blob/hovudstraum/contrib/plugins/meadup.js) converts copyparty into a janky yet extremely flexible chromecast clone
+  * also adds a virtual keyboard by @steinuil to the basic-upload tab for comfy couch crowd control
+  * anything uploaded through the [android app](https://github.com/9001/party-up) (files or links) are executed on the server, meaning anyone can infect your PC with malware... so protect this with a password and keep it on a LAN!
+
+
 # dependencies
 
 run [`install-deps.sh`](install-deps.sh) to build/install most dependencies required by these programs (supports windows/linux/macos)

bin/mtag/very-bad-idea.py

@@ -1,6 +1,11 @@
 #!/usr/bin/env python3
 
 """
+WARNING -- DANGEROUS PLUGIN --
+  if someone is able to upload files to a copyparty which is
+  running this plugin, they can execute malware on your machine
+  so please keep this on a LAN and protect it with a password
+
 use copyparty as a chromecast replacement:
   * post a URL and it will open in the default browser
   * upload a file and it will open in the default application
@@ -10,16 +15,17 @@ use copyparty as a chromecast replacement:
 
 the android app makes it a breeze to post pics and links:
   https://github.com/9001/party-up/releases
-  (iOS devices have to rely on the web-UI)
 
-goes without saying, but this is HELLA DANGEROUS,
-  GIVES RCE TO ANYONE WHO HAVE UPLOAD PERMISSIONS
+iOS devices can use the web-UI or the shortcut instead:
+  https://github.com/9001/copyparty#ios-shortcuts
 
-example copyparty config to use this:
-  --urlform save,get -v.::w:c,e2d,e2t,mte=+a1:c,mtp=a1=ad,kn,c0,bin/mtag/very-bad-idea.py
+example copyparty config to use this;
+lets the user "kevin" with password "hunter2" use this plugin:
+  -a kevin:hunter2 --urlform save,get -v.::w,kevin:c,e2d,e2t,mte=+a1:c,mtp=a1=ad,kn,c0,bin/mtag/very-bad-idea.py
 
 recommended deps:
-  apt install xdotool libnotify-bin
+  apt install xdotool libnotify-bin mpv
+  python3 -m pip install --user -U streamlink yt-dlp
   https://github.com/9001/copyparty/blob/hovudstraum/contrib/plugins/meadup.js
 
 and you probably want `twitter-unmute.user.js` from the res folder
@@ -63,8 +69,10 @@ set -e
 EOF
 chmod 755 /usr/local/bin/chromium-browser
 
-# start the server  (note: replace `-v.::rw:` with `-v.::w:` to disallow retrieving uploaded stuff)
-cd ~/Downloads; python3 copyparty-sfx.py --urlform save,get -v.::rw:c,e2d,e2t,mte=+a1:c,mtp=a1=ad,kn,very-bad-idea.py
+# start the server
+# note 1: replace hunter2 with a better password to access the server
+# note 2: replace `-v.::rw` with `-v.::w` to disallow retrieving uploaded stuff
+cd ~/Downloads; python3 copyparty-sfx.py -a kevin:hunter2 --urlform save,get -v.::rw,kevin:c,e2d,e2t,mte=+a1:c,mtp=a1=ad,kn,very-bad-idea.py
 
 """
 
@@ -72,11 +80,23 @@ cd ~/Downloads; python3 copyparty-sfx.py --urlform save,get -v.::rw:c,e2d,e2t,mt
 import os
 import sys
 import time
+import shutil
 import subprocess as sp
 from urllib.parse import unquote_to_bytes as unquote
+from urllib.parse import quote
+
+have_mpv = shutil.which("mpv")
+have_vlc = shutil.which("vlc")
 
 
 def main():
+    if len(sys.argv) > 2 and sys.argv[1] == "x":
+        # invoked on commandline for testing;
+        # python3 very-bad-idea.py x msg=https://youtu.be/dQw4w9WgXcQ
+        txt = " ".join(sys.argv[2:])
+        txt = quote(txt.replace(" ", "+"))
+        return open_post(txt.encode("utf-8"))
+
     fp = os.path.abspath(sys.argv[1])
     with open(fp, "rb") as f:
         txt = f.read(4096)
@@ -92,7 +112,7 @@ def open_post(txt):
     try:
         k, v = txt.split(" ", 1)
     except:
-        open_url(txt)
+        return open_url(txt)
 
     if k == "key":
         sp.call(["xdotool", "key"] + v.split(" "))
@@ -128,6 +148,17 @@ def open_url(txt):
     # else:
     #    sp.call(["xdotool", "getactivewindow", "windowminimize"])  # minimizes the focused windo
 
+    # mpv is probably smart enough to use streamlink automatically
+    if try_mpv(txt):
+        print("mpv got it")
+        return
+
+    # or maybe streamlink would be a good choice to open this
+    if try_streamlink(txt):
+        print("streamlink got it")
+        return
+
+    # nope,
     # close any error messages:
     sp.call(["xdotool", "search", "--name", "Error", "windowclose"])
     # sp.call(["xdotool", "key", "ctrl+alt+d"])  # doesnt work at all
@@ -136,4 +167,39 @@ def open_url(txt):
     sp.call(["xdg-open", txt])
 
 
+def try_mpv(url):
+    t0 = time.time()
+    try:
+        print("trying mpv...")
+        sp.check_call(["mpv", "--fs", url])
+        return True
+    except:
+        # if it ran for 15 sec it probably succeeded and terminated
+        t = time.time()
+        return t - t0 > 15
+
+
+def try_streamlink(url):
+    t0 = time.time()
+    try:
+        import streamlink
+
+        print("trying streamlink...")
+        streamlink.Streamlink().resolve_url(url)
+
+        if have_mpv:
+            args = "-m streamlink -p mpv -a --fs"
+        else:
+            args = "-m streamlink"
+
+        cmd = [sys.executable] + args.split() + [url, "best"]
+        t0 = time.time()
+        sp.check_call(cmd)
+        return True
+    except:
+        # if it ran for 10 sec it probably succeeded and terminated
+        t = time.time()
+        return t - t0 > 10
+
+
 main()

bin/mtag/wget.py

@@ -65,6 +65,10 @@ def main():
     if "://" not in url:
         url = "https://" + url
 
+    proto = url.split("://")[0].lower()
+    if proto not in ("http", "https", "ftp", "ftps"):
+        raise Exception("bad proto {}".format(proto))
+
     os.chdir(fdir)
 
     name = url.split("?")[0].split("/")[-1]

bin/u2c.py

@@ -1,19 +1,20 @@
 #!/usr/bin/env python3
 from __future__ import print_function, unicode_literals
 
-S_VERSION = "1.6"
-S_BUILD_DT = "2023-04-20"
+S_VERSION = "1.10"
+S_BUILD_DT = "2023-08-15"
 
 """
-up2k.py: upload to copyparty
+u2c.py: upload to copyparty
 2021, ed <irc.rizon.net>, MIT-Licensed
-https://github.com/9001/copyparty/blob/hovudstraum/bin/up2k.py
+https://github.com/9001/copyparty/blob/hovudstraum/bin/u2c.py
 
 - dependencies: requests
 - supports python 2.6, 2.7, and 3.3 through 3.12
 - if something breaks just try again and it'll autoresume
 """
 
+import re
 import os
 import sys
 import stat
@@ -21,6 +22,7 @@ import math
 import time
 import atexit
 import signal
+import socket
 import base64
 import hashlib
 import platform
@@ -38,7 +40,7 @@ except:
 
 try:
     import requests
-except ImportError:
+except ImportError as ex:
     if EXE:
         raise
     elif sys.version_info > (2, 7):
@@ -49,7 +51,7 @@ except ImportError:
         m = "\n  ERROR: need these:\n" + "\n".join(m) + "\n"
         m += "\n  for f in *.whl; do unzip $f; done; rm -r *.dist-info\n"
 
-    print(m.format(sys.executable))
+    print(m.format(sys.executable), "\nspecifically,", ex)
     sys.exit(1)
 
 
@@ -58,6 +60,7 @@ PY2 = sys.version_info < (3,)
 if PY2:
     from Queue import Queue
     from urllib import quote, unquote
+    from urlparse import urlsplit, urlunsplit
 
     sys.dont_write_bytecode = True
     bytes = str
@@ -65,6 +68,7 @@ else:
     from queue import Queue
     from urllib.parse import unquote_to_bytes as unquote
     from urllib.parse import quote_from_bytes as quote
+    from urllib.parse import urlsplit, urlunsplit
 
     unicode = str
 
@@ -337,6 +341,32 @@ class CTermsize(object):
 ss = CTermsize()
 
 
+def undns(url):
+    usp = urlsplit(url)
+    hn = usp.hostname
+    gai = None
+    eprint("resolving host [{0}] ...".format(hn), end="")
+    try:
+        gai = socket.getaddrinfo(hn, None)
+        hn = gai[0][4][0]
+    except KeyboardInterrupt:
+        raise
+    except:
+        t = "\n\033[31mfailed to resolve upload destination host;\033[0m\ngai={0}\n"
+        eprint(t.format(repr(gai)))
+        raise
+
+    if usp.port:
+        hn = "{0}:{1}".format(hn, usp.port)
+    if usp.username or usp.password:
+        hn = "{0}:{1}@{2}".format(usp.username, usp.password, hn)
+
+    usp = usp._replace(netloc=hn)
+    url = urlunsplit(usp)
+    eprint(" {0}".format(url))
+    return url
+
+
 def _scd(err, top):
     """non-recursive listing of directory contents, along with stat() info"""
     with os.scandir(top) as dh:
@@ -382,10 +412,11 @@ def walkdir(err, top, seen):
                 err.append((ap, str(ex)))
 
 
-def walkdirs(err, tops):
+def walkdirs(err, tops, excl):
     """recursive statdir for a list of tops, yields [top, relpath, stat]"""
     sep = "{0}".format(os.sep).encode("ascii")
     if not VT100:
+        excl = excl.replace("/", r"\\")
         za = []
         for td in tops:
             try:
@@ -402,6 +433,8 @@ def walkdirs(err, tops):
         za = [x.replace(b"/", b"\\") for x in za]
         tops = za
 
+    ptn = re.compile(excl.encode("utf-8") or b"\n")
+
     for top in tops:
         isdir = os.path.isdir(top)
         if top[-1:] == sep:
@@ -414,6 +447,8 @@ def walkdirs(err, tops):
 
         if isdir:
             for ap, inf in walkdir(err, top, []):
+                if ptn.match(ap):
+                    continue
                 yield stop, ap[len(stop) :].lstrip(sep), inf
         else:
             d, n = top.rsplit(sep, 1)
@@ -625,7 +660,7 @@ class Ctl(object):
         nfiles = 0
         nbytes = 0
         err = []
-        for _, _, inf in walkdirs(err, ar.files):
+        for _, _, inf in walkdirs(err, ar.files, ar.x):
             if stat.S_ISDIR(inf.st_mode):
                 continue
 
@@ -667,7 +702,7 @@ class Ctl(object):
         if ar.te:
             req_ses.verify = ar.te
 
-        self.filegen = walkdirs([], ar.files)
+        self.filegen = walkdirs([], ar.files, ar.x)
         self.recheck = []  # type: list[File]
 
         if ar.safe:
@@ -853,7 +888,7 @@ class Ctl(object):
                         print("      ls ~{0}".format(srd))
                         zb = self.ar.url.encode("utf-8")
                         zb += quotep(rd.replace(b"\\", b"/"))
-                        r = req_ses.get(zb + b"?ls&dots", headers=headers)
+                        r = req_ses.get(zb + b"?ls&lt&dots", headers=headers)
                         if not r:
                             raise Exception("HTTP {0}".format(r.status_code))
 
@@ -931,7 +966,7 @@ class Ctl(object):
 
             upath = file.abs.decode("utf-8", "replace")
             if not VT100:
-                upath = upath[4:]
+                upath = upath.lstrip("\\?")
 
             hs, sprs = handshake(self.ar, file, search)
             if search:
@@ -1068,11 +1103,13 @@ source file/folder selection uses rsync syntax, meaning that:
     ap.add_argument("-v", action="store_true", help="verbose")
     ap.add_argument("-a", metavar="PASSWORD", help="password or $filepath")
     ap.add_argument("-s", action="store_true", help="file-search (disables upload)")
+    ap.add_argument("-x", type=unicode, metavar="REGEX", default="", help="skip file if filesystem-abspath matches REGEX, example: '.*/\.hist/.*'")
     ap.add_argument("--ok", action="store_true", help="continue even if some local files are inaccessible")
     ap.add_argument("--version", action="store_true", help="show version and exit")
 
     ap = app.add_argument_group("compatibility")
     ap.add_argument("--cls", action="store_true", help="clear screen before start")
+    ap.add_argument("--rh", type=int, metavar="TRIES", default=0, help="resolve server hostname before upload (good for buggy networks, but TLS certs will break)")
 
     ap = app.add_argument_group("folder sync")
     ap.add_argument("--dl", action="store_true", help="delete local files after uploading")
@@ -1083,7 +1120,7 @@ source file/folder selection uses rsync syntax, meaning that:
     ap.add_argument("-j", type=int, metavar="THREADS", default=4, help="parallel connections")
     ap.add_argument("-J", type=int, metavar="THREADS", default=hcores, help="num cpu-cores to use for hashing; set 0 or 1 for single-core hashing")
     ap.add_argument("-nh", action="store_true", help="disable hashing while uploading")
-    ap.add_argument("-ns", action="store_true", help="no status panel (for slow consoles)")
+    ap.add_argument("-ns", action="store_true", help="no status panel (for slow consoles and macos)")
     ap.add_argument("--safe", action="store_true", help="use simple fallback approach")
     ap.add_argument("-z", action="store_true", help="ZOOMIN' (skip uploading files if they exist at the destination with the ~same last-modified timestamp, so same as yolo / turbo with date-chk but even faster)")
 
@@ -1096,7 +1133,7 @@ source file/folder selection uses rsync syntax, meaning that:
         ar = app.parse_args()
     finally:
         if EXE and not sys.argv[1:]:
-            print("*** hit enter to exit ***")
+            eprint("*** hit enter to exit ***")
             try:
                 input()
             except:
@@ -1129,8 +1166,18 @@ source file/folder selection uses rsync syntax, meaning that:
         with open(fn, "rb") as f:
             ar.a = f.read().decode("utf-8").strip()
 
+    for n in range(ar.rh):
+        try:
+            ar.url = undns(ar.url)
+            break
+        except KeyboardInterrupt:
+            raise
+        except:
+            if n > ar.rh - 2:
+                raise
+
     if ar.cls:
-        print("\x1b\x5b\x48\x1b\x5b\x32\x4a\x1b\x5b\x33\x4a", end="")
+        eprint("\x1b\x5b\x48\x1b\x5b\x32\x4a\x1b\x5b\x33\x4a", end="")
 
     ctl = Ctl(ar)
 

contrib/cfssl.sh

@@ -1,14 +1,44 @@
 #!/bin/bash
 set -e
 
+cat >/dev/null <<'EOF'
+
+NOTE: copyparty is now able to do this automatically;
+however you may wish to use this script instead if
+you have specific needs (or if copyparty breaks)
+
+this script generates a new self-signed TLS certificate and
+replaces the default insecure one that comes with copyparty
+
+as it is trivial to impersonate a copyparty server using the
+default certificate, it is highly recommended to do this
+
+this will create a self-signed CA, and a Server certificate
+which gets signed by that CA -- you can run it multiple times
+with different server-FQDNs / IPs to create additional certs
+for all your different servers / (non-)copyparty services
+
+EOF
+
+
 # ca-name and server-fqdn
 ca_name="$1"
 srv_fqdn="$2"
 
-[ -z "$srv_fqdn" ] && {
-	echo "need arg 1: ca name"
-	echo "need arg 2: server fqdn and/or IPs, comma-separated"
-	echo "optional arg 3: if set, write cert into copyparty cfg"
+[ -z "$srv_fqdn" ] && { cat <<'EOF'
+need arg 1: ca name
+need arg 2: server fqdn and/or IPs, comma-separated
+optional arg 3: if set, write cert into copyparty cfg
+
+example:
+  ./cfssl.sh PartyCo partybox.local y
+EOF
+	exit 1
+}
+
+
+command -v cfssljson 2>/dev/null || {
+	echo please install cfssl and try again
 	exit 1
 }
 
@@ -59,12 +89,14 @@ show() {
 }
 show ca.pem
 show "$srv_fqdn.pem"
-
+echo
+echo "successfully generated new certificates"
 
 # write cert into copyparty config
 [ -z "$3" ] || {
 	mkdir -p ~/.config/copyparty
 	cat "$srv_fqdn".{key,pem} ca.pem >~/.config/copyparty/cert.pem 
+	echo "successfully replaced copyparty certificate"
 }
 
 

contrib/nginx/copyparty.conf

@@ -34,6 +34,8 @@ server {
 		proxy_set_header   Host              $host;
 		proxy_set_header   X-Real-IP         $remote_addr;
 		proxy_set_header   X-Forwarded-For   $proxy_add_x_forwarded_for;
+		# NOTE: with cloudflare you want this instead:
+		#proxy_set_header   X-Forwarded-For   $http_cf_connecting_ip;
 		proxy_set_header   X-Forwarded-Proto $scheme;
 		proxy_set_header   Connection        "Keep-Alive";
 	}

contrib/nixos/modules/copyparty.nix

@@ -138,6 +138,8 @@ in {
                 "d" (delete): permanently delete files and folders
                 "g" (get):    download files, but cannot see folder contents
                 "G" (upget):  "get", but can see filekeys of their own uploads
+                "h" (html):   "get", but folders return their index.html
+                "a" (admin):  can see uploader IPs, config-reload
 
               For example: "rwmd"
 

contrib/package/arch/PKGBUILD

@@ -1,50 +1,48 @@
 # Maintainer: icxes <dev.null@need.moe>
 pkgname=copyparty
-pkgver="1.6.12"
+pkgver="1.9.4"
 pkgrel=1
 pkgdesc="Portable file sharing hub"
 arch=("any")
 url="https://github.com/9001/${pkgname}"
 license=('MIT')
-depends=("python" "lsof")
-optdepends=("ffmpeg: thumbnails for videos, images (slower) and audio, music tags" 
-            "python-jinja: faster html generator" 
+depends=("python" "lsof" "python-jinja")
+makedepends=("python-wheel" "python-setuptools" "python-build" "python-installer" "make" "pigz")
+optdepends=("ffmpeg: thumbnails for videos, images (slower) and audio, music tags"
             "python-mutagen: music tags (alternative)" 
             "python-pillow: thumbnails for images" 
             "python-pyvips: thumbnails for images (higher quality, faster, uses more ram)" 
             "libkeyfinder-git: detection of musical keys" 
             "qm-vamp-plugins: BPM detection" 
             "python-pyopenssl: ftps functionality" 
+            "python-argon2_cffi: hashed passwords in config" 
             "python-impacket-git: smb support (bad idea)"
 )
-source=("${url}/releases/download/v${pkgver}/${pkgname}-sfx.py" 
-        "${pkgname}.conf"
-        "${pkgname}.service"
-        "prisonparty.service"
-        "index.md"
-        "https://raw.githubusercontent.com/9001/${pkgname}/v${pkgver}/bin/prisonparty.sh"
-        "https://raw.githubusercontent.com/9001/${pkgname}/v${pkgver}/LICENSE"
-)
+source=("https://github.com/9001/${pkgname}/releases/download/v${pkgver}/${pkgname}-${pkgver}.tar.gz")
 backup=("etc/${pkgname}.d/init" )
-sha256sums=("ce3702634fecf7cd6a5c79709a8e6cb1e9fe0748ec4754a278be7a60b618dd42"
-            "b8565eba5e64dedba1cf6c7aac7e31c5a731ed7153d6810288a28f00a36c28b2"
-            "f65c207e0670f9d78ad2e399bda18d5502ff30d2ac79e0e7fc48e7fbdc39afdc"
-            "c4f396b083c9ec02ad50b52412c84d2a82be7f079b2d016e1c9fad22d68285ff"
-            "dba701de9fd584405917e923ea1e59dbb249b96ef23bad479cf4e42740b774c8"
-            "8e89d281483e22d11d111bed540652af35b66af6f14f49faae7b959f6cdc6475"
-            "cb2ce3d6277bf2f5a82ecf336cc44963bc6490bcf496ffbd75fc9e21abaa75f3"
-)
+sha256sums=("c327ac35deaa5e6cc86b3b1a251cc78517be5578c37c4dff90e98465ced82abc")
+
+build() {
+    cd "${srcdir}/${pkgname}-${pkgver}"
+    
+    pushd copyparty/web
+    make -j$(nproc)
+    rm Makefile
+    popd
+    
+    python3 -m build -wn
+}
 
 package() {
-    cd "${srcdir}/"
+    cd "${srcdir}/${pkgname}-${pkgver}"
+    python3 -m installer -d "$pkgdir" dist/*.whl
 
     install -dm755 "${pkgdir}/etc/${pkgname}.d"
-    install -Dm755 "${pkgname}-sfx.py" "${pkgdir}/usr/bin/${pkgname}"
-    install -Dm755 "prisonparty.sh" "${pkgdir}/usr/bin/prisonparty"
-    install -Dm644 "${pkgname}.conf" "${pkgdir}/etc/${pkgname}.d/init"
-    install -Dm644 "${pkgname}.service" "${pkgdir}/usr/lib/systemd/system/${pkgname}.service"
-    install -Dm644 "prisonparty.service" "${pkgdir}/usr/lib/systemd/system/prisonparty.service"
-    install -Dm644 "index.md" "${pkgdir}/var/lib/${pkgname}-jail/README.md"
+    install -Dm755 "bin/prisonparty.sh" "${pkgdir}/usr/bin/prisonparty"
+    install -Dm644 "contrib/package/arch/${pkgname}.conf" "${pkgdir}/etc/${pkgname}.d/init"
+    install -Dm644 "contrib/package/arch/${pkgname}.service" "${pkgdir}/usr/lib/systemd/system/${pkgname}.service"
+    install -Dm644 "contrib/package/arch/prisonparty.service" "${pkgdir}/usr/lib/systemd/system/prisonparty.service"
+    install -Dm644 "contrib/package/arch/index.md" "${pkgdir}/var/lib/${pkgname}-jail/README.md"
     install -Dm644 "LICENSE" "${pkgdir}/usr/share/licenses/${pkgname}/LICENSE"
 
     find /etc/${pkgname}.d -iname '*.conf' 2>/dev/null | grep -qE . && return

contrib/package/nix/copyparty/default.nix

@@ -1,4 +1,7 @@
-{ lib, stdenv, makeWrapper, fetchurl, utillinux, python, jinja2, impacket, pyftpdlib, pyopenssl, pillow, pyvips, ffmpeg, mutagen,
+{ lib, stdenv, makeWrapper, fetchurl, utillinux, python, jinja2, impacket, pyftpdlib, pyopenssl, argon2-cffi, pillow, pyvips, ffmpeg, mutagen,
+
+# use argon2id-hashed passwords in config files (sha2 is always available)
+withHashedPasswords ? true,
 
 # create thumbnails with Pillow; faster than FFmpeg / MediaProcessing
 withThumbnails ? true,
@@ -35,6 +38,7 @@ let
     ++ lib.optional withFastThumbnails pyvips
     ++ lib.optional withMediaProcessing ffmpeg
     ++ lib.optional withBasicAudioMetadata mutagen
+    ++ lib.optional withHashedPasswords argon2-cffi
     );
 in stdenv.mkDerivation {
   pname = "copyparty";

contrib/package/nix/copyparty/pin.json

@@ -1,5 +1,5 @@
 {
-    "url": "https://github.com/9001/copyparty/releases/download/v1.6.12/copyparty-sfx.py",
-    "version": "1.6.12",
-    "hash": "sha256-zjcCY0/s981qXHlwmo5ssen+B0jsR1SieL56YLYY3UI="
+    "url": "https://github.com/9001/copyparty/releases/download/v1.9.4/copyparty-sfx.py",
+    "version": "1.9.4",
+    "hash": "sha256-17dBvr6uUzaQtzunZUX84BLTWCGe1Hucz9b48BKowHs="
 }

contrib/systemd/cfssl.service

@@ -1,3 +1,6 @@
+# NOTE: this is now a built-in feature in copyparty
+# but you may still want this if you have specific needs
+#
 # systemd service which generates a new TLS certificate on each boot,
 # that way the one-year expiry time won't cause any issues --
 # just have everyone trust the ca.pem once every 10 years

contrib/systemd/copyparty.service

@@ -22,6 +22,7 @@
 #   add '-i 127.0.0.1' to only allow local connections
 #   add '-e2dsa' to enable filesystem scanning + indexing
 #   add '-e2ts' to enable metadata indexing
+#   remove '--ansi' to disable colored logs
 #
 # with `Type=notify`, copyparty will signal systemd when it is ready to
 #   accept connections; correctly delaying units depending on copyparty.
@@ -59,7 +60,7 @@ ExecStartPre=+nft add rule ip nat prerouting tcp dport 443 redirect to :3923
 ExecStartPre=+/bin/bash -c 'mkdir -p /run/tmpfiles.d/ && echo "x /tmp/pe-copyparty*" > /run/tmpfiles.d/copyparty.conf'
 
 # copyparty settings
-ExecStart=/usr/bin/python3 /usr/local/bin/copyparty-sfx.py -e2d -v /mnt::rw
+ExecStart=/usr/bin/python3 /usr/local/bin/copyparty-sfx.py --ansi -e2d -v /mnt::rw
 
 [Install]
 WantedBy=multi-user.target

copyparty/__init__.py

@@ -6,6 +6,10 @@ import platform
 import sys
 import time
 
+# fmt: off
+_:tuple[int,int]=(0,0)  # _____________________________________________________________________  hey there! if you are reading this, your python is too old to run copyparty without some help. Please use https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.py or the pypi package instead, or see https://github.com/9001/copyparty/blob/hovudstraum/docs/devnotes.md#building if you want to build it yourself :-)  ************************************************************************************************************************************************
+# fmt: on
+
 try:
     from typing import TYPE_CHECKING
 except:
@@ -27,7 +31,12 @@ WINDOWS: Any = (
     else False
 )
 
-VT100 = not WINDOWS or WINDOWS >= [10, 0, 14393]
+VT100 = "--ansi" in sys.argv or (
+    os.environ.get("NO_COLOR", "").lower() in ("", "0", "false")
+    and sys.stdout.isatty()
+    and "--no-ansi" not in sys.argv
+    and (not WINDOWS or WINDOWS >= [10, 0, 14393])
+)
 # introduced in anniversary update
 
 ANYWIN = WINDOWS or sys.platform in ["msys", "cygwin"]

copyparty/__main__.py

@@ -10,11 +10,9 @@ __url__ = "https://github.com/9001/copyparty/"
 
 import argparse
 import base64
-import filecmp
 import locale
 import os
 import re
-import shutil
 import socket
 import sys
 import threading
@@ -186,7 +184,7 @@ def init_E(E: EnvParams) -> None:
 
         with open_binary("copyparty", "z.tar") as tgz:
             with tarfile.open(fileobj=tgz) as tf:
-                tf.extractall(tdn)
+                tf.extractall(tdn)  # nosec (archive is safe)
 
         return tdn
 
@@ -201,7 +199,7 @@ def init_E(E: EnvParams) -> None:
         E.mod = _unpack()
 
     if sys.platform == "win32":
-        bdir = os.environ.get("APPDATA") or os.environ.get("TEMP")
+        bdir = os.environ.get("APPDATA") or os.environ.get("TEMP") or "."
         E.cfg = os.path.normpath(bdir + "/copyparty")
     elif sys.platform == "darwin":
         E.cfg = os.path.expanduser("~/Library/Preferences/copyparty")
@@ -242,6 +240,37 @@ def get_srvname() -> str:
     return ret
 
 
+def get_fk_salt(cert_path) -> str:
+    fp = os.path.join(E.cfg, "fk-salt.txt")
+    try:
+        with open(fp, "rb") as f:
+            ret = f.read().strip()
+    except:
+        if os.path.exists(cert_path):
+            zi = os.path.getmtime(cert_path)
+            ret = "{}".format(zi).encode("utf-8")
+        else:
+            ret = base64.b64encode(os.urandom(18))
+
+        with open(fp, "wb") as f:
+            f.write(ret + b"\n")
+
+    return ret.decode("utf-8")
+
+
+def get_ah_salt() -> str:
+    fp = os.path.join(E.cfg, "ah-salt.txt")
+    try:
+        with open(fp, "rb") as f:
+            ret = f.read().strip()
+    except:
+        ret = base64.b64encode(os.urandom(18))
+        with open(fp, "wb") as f:
+            f.write(ret + b"\n")
+
+    return ret.decode("utf-8")
+
+
 def ensure_locale() -> None:
     safe = "en_US.UTF-8"
     for x in [
@@ -261,45 +290,22 @@ def ensure_locale() -> None:
     warn(t.format(safe))
 
 
-def ensure_cert(al: argparse.Namespace) -> None:
+def ensure_webdeps() -> None:
+    ap = os.path.join(E.mod, "web/deps/mini-fa.woff")
+    if os.path.exists(ap):
+        return
+
+    warn(
+        """could not find webdeps;
+  if you are running the sfx, or exe, or pypi package, or docker image,
+  then this is a bug! Please let me know so I can fix it, thanks :-)
+  https://github.com/9001/copyparty/issues/new?labels=bug&template=bug_report.md
+
+  however, if you are a dev, or running copyparty from source, and you want
+  full client functionality, you will need to build or obtain the webdeps:
+  https://github.com/9001/copyparty/blob/hovudstraum/docs/devnotes.md#building
     """
-    the default cert (and the entire TLS support) is only here to enable the
-    crypto.subtle javascript API, which is necessary due to the webkit guys
-    being massive memers (https://www.chromium.org/blink/webcrypto)
-
-    i feel awful about this and so should they
-    """
-    cert_insec = os.path.join(E.mod, "res/insecure.pem")
-    cert_appdata = os.path.join(E.cfg, "cert.pem")
-    if not os.path.isfile(al.cert):
-        if cert_appdata != al.cert:
-            raise Exception("certificate file does not exist: " + al.cert)
-
-        shutil.copy(cert_insec, al.cert)
-
-    with open(al.cert, "rb") as f:
-        buf = f.read()
-        o1 = buf.find(b"-BEGIN PRIVATE KEY-")
-        o2 = buf.find(b"-BEGIN CERTIFICATE-")
-        m = "unsupported certificate format: "
-        if o1 < 0:
-            raise Exception(m + "no private key inside pem")
-        if o2 < 0:
-            raise Exception(m + "no server certificate inside pem")
-        if o1 > o2:
-            raise Exception(m + "private key must appear before server certificate")
-
-    try:
-        if filecmp.cmp(al.cert, cert_insec):
-            lprint(
-                "\033[33musing default TLS certificate; https will be insecure."
-                + "\033[36m\ncertificate location: {}\033[0m\n".format(al.cert)
-            )
-    except:
-        pass
-
-    # speaking of the default cert,
-    # printf 'NO\n.\n.\n.\n.\ncopyparty-insecure\n.\n' | faketime '2000-01-01 00:00:00' openssl req -x509 -sha256 -newkey rsa:2048 -keyout insecure.pem -out insecure.pem -days $((($(printf %d 0x7fffffff)-$(date +%s --date=2000-01-01T00:00:00Z))/(60*60*24))) -nodes && ls -al insecure.pem && openssl x509 -in insecure.pem -text -noout
+    )
 
 
 def configure_ssl_ver(al: argparse.Namespace) -> None:
@@ -486,6 +492,8 @@ def get_sects():
               "d" (delete): permanently delete files and folders
               "g" (get):    download files, but cannot see folder contents
               "G" (upget):  "get", but can see filekeys of their own uploads
+              "h" (html):   "get", but folders return their index.html
+              "a" (admin):  can see uploader IPs, config-reload
 
             too many volflags to list here, see --help-flags
 
@@ -522,6 +530,50 @@ def get_sects():
             ).rstrip()
             + build_flags_desc(),
         ],
+        [
+            "handlers",
+            "use plugins to handle certain events",
+            dedent(
+                """
+            usually copyparty returns a \033[33m404\033[0m if a file does not exist, and
+            \033[33m403\033[0m if a user tries to access a file they don't have access to
+
+            you can load a plugin which will be invoked right before this
+            happens, and the plugin can choose to override this behavior
+
+            load the plugin using --args or volflags; for example \033[36m
+             --on404 ~/partyhandlers/not404.py
+             -v .::r:c,on404=~/partyhandlers/not404.py
+            \033[0m
+            the file must define the function \033[35mmain(cli,vn,rem)\033[0m:
+             \033[35mcli\033[0m: the copyparty HttpCli instance
+             \033[35mvn\033[0m:  the VFS which overlaps with the requested URL
+             \033[35mrem\033[0m: the remainder of the URL below the VFS mountpoint
+
+            `main` must return a string; one of the following:
+
+            > \033[32m"true"\033[0m: the plugin has responded to the request,
+                and the TCP connection should be kept open
+
+            > \033[32m"false"\033[0m: the plugin has responded to the request,
+                and the TCP connection should be terminated
+
+            > \033[32m"retry"\033[0m: the plugin has done something to resolve the 404
+                situation, and copyparty should reattempt reading the file.
+                if it still fails, a regular 404 will be returned
+
+            > \033[32m"allow"\033[0m: should ignore the insufficient permissions
+                and let the client continue anyways
+
+            > \033[32m""\033[0m: the plugin has not handled the request;
+                try the next plugin or return the usual 404 or 403
+
+            \033[1;35mPS!\033[0m the folder that contains the python file should ideally
+              not contain many other python files, and especially nothing
+              with filenames that overlap with modules used by copyparty
+            """
+            ),
+        ],
         [
             "hooks",
             "execute commands before/after various events",
@@ -536,6 +588,7 @@ def get_sects():
              \033[36mxbd\033[35m executes CMD before a file delete
              \033[36mxad\033[35m executes CMD after  a file delete
              \033[36mxm\033[35m executes CMD on message
+             \033[36mxban\033[35m executes CMD if someone gets banned
             \033[0m
             can be defined as --args or volflags; for example \033[36m
              --xau notify-send
@@ -571,6 +624,9 @@ def get_sects():
             executed program on STDIN instead of as argv arguments, and
             it also includes the wark (file-id/hash) as a json property
 
+            \033[36mxban\033[0m can be used to overrule / cancel a user ban event;
+            if the program returns 0 (true/OK) then the ban will NOT happen
+
             except for \033[36mxm\033[0m, only one hook / one action can run at a time,
             so it's recommended to use the \033[36mf\033[0m flag unless you really need
             to wait for the hook to finish before continuing (without \033[36mf\033[0m
@@ -619,9 +675,9 @@ def get_sects():
 
             \033[32macid\033[0m = extremely safe but slow; the old default. Should never lose any data no matter what
 
-            \033[32mswal\033[0m = 2.4x faster uploads yet 99.9%% as safe -- theoretical chance of losing metadata for the ~200 most recently uploaded files if there's a power-loss or your OS crashes
+            \033[32mswal\033[0m = 2.4x faster uploads yet 99.9% as safe -- theoretical chance of losing metadata for the ~200 most recently uploaded files if there's a power-loss or your OS crashes
 
-            \033[32mwal\033[0m = another 21x faster on HDDs yet 90%% as safe; same pitfall as \033[33mswal\033[0m except more likely
+            \033[32mwal\033[0m = another 21x faster on HDDs yet 90% as safe; same pitfall as \033[33mswal\033[0m except more likely
 
             \033[32myolo\033[0m = another 1.5x faster, and removes the occasional sudden upload-pause while the disk syncs, but now you're at risk of losing the entire database in a powerloss / OS-crash
 
@@ -629,6 +685,72 @@ def get_sects():
             """
             ),
         ],
+        [
+            "pwhash",
+            "password hashing",
+            dedent(
+                """
+            when \033[36m--ah-alg\033[0m is not the default [\033[32mnone\033[0m], all account passwords must be hashed
+
+            passwords can be hashed on the commandline with \033[36m--ah-gen\033[0m, but copyparty will also hash and print any passwords that are non-hashed (password which do not start with '+') and then terminate afterwards
+
+            \033[36m--ah-alg\033[0m specifies the hashing algorithm and a list of optional comma-separated arguments:
+
+            \033[36m--ah-alg argon2\033[0m  # which is the same as:
+            \033[36m--ah-alg argon2,3,256,4,19\033[0m
+            use argon2id with timecost 3, 256 MiB, 4 threads, version 19 (0x13/v1.3)
+
+            \033[36m--ah-alg scrypt\033[0m  # which is the same as:
+            \033[36m--ah-alg scrypt,13,2,8,4\033[0m
+            use scrypt with cost 2**13, 2 iterations, blocksize 8, 4 threads
+
+            \033[36m--ah-alg sha2\033[0m  # which is the same as:
+            \033[36m--ah-alg sha2,424242\033[0m
+            use sha2-512 with 424242 iterations
+
+            recommended: \033[32m--ah-alg argon2\033[0m
+              (takes about 0.4 sec and 256M RAM to process a new password)
+
+            argon2 needs python-package argon2-cffi,
+            scrypt needs openssl,
+            sha2 is always available
+            """
+            ),
+        ],
+        [
+            "zm",
+            "mDNS debugging",
+            dedent(
+                """
+            the mDNS protocol is multicast-based, which means there are thousands
+            of fun and intersesting ways for it to break unexpectedly
+
+            things to check if it does not work at all:
+
+            * is there a firewall blocking port 5353 on either the server or client?
+              (for example, clients may be able to send queries to copyparty,
+               but the replies could get lost)
+
+            * is multicast accidentally disabled on either the server or client?
+              (look for mDNS log messages saying "new client on [...]")
+
+            * the router/switch must be multicast and igmp capable
+
+            things to check if it works for a while but then it doesn't:
+
+            * is there a firewall blocking port 5353 on either the server or client?
+              (copyparty may be unable to see the queries from the clients, but the
+               clients may still be able to see the initial unsolicited announce,
+               so it works for about 2 minutes after startup until TTL expires)
+
+            * does the client have multiple IPs on its interface, and some of the
+              IPs are in subnets which the copyparty server is not a member of?
+
+            for both of the above intermittent issues, try --zm-spam 30
+            (not spec-compliant but nothing will mind)
+            """
+            ),
+        ],
     ]
 
 
@@ -694,7 +816,7 @@ def add_upload(ap):
     ap2.add_argument("--magic", action="store_true", help="enable filetype detection on nameless uploads (volflag=magic)")
     ap2.add_argument("--df", metavar="GiB", type=float, default=0, help="ensure GiB free disk space by rejecting upload requests")
     ap2.add_argument("--sparse", metavar="MiB", type=int, default=4, help="windows-only: minimum size of incoming uploads through up2k before they are made into sparse files")
-    ap2.add_argument("--turbo", metavar="LVL", type=int, default=0, help="configure turbo-mode in up2k client; [\033[32m0\033[0m] = off and warn if enabled, [\033[32m1\033[0m] = off, [\033[32m2\033[0m] = on, [\033[32m3\033[0m] = on and disable datecheck")
+    ap2.add_argument("--turbo", metavar="LVL", type=int, default=0, help="configure turbo-mode in up2k client; [\033[32m-1\033[0m] = forbidden/always-off, [\033[32m0\033[0m] = default-off and warn if enabled, [\033[32m1\033[0m] = default-off, [\033[32m2\033[0m] = on, [\033[32m3\033[0m] = on and disable datecheck")
     ap2.add_argument("--u2sort", metavar="TXT", type=u, default="s", help="upload order; [\033[32ms\033[0m]=smallest-first, [\033[32mn\033[0m]=alphabetical, [\033[32mfs\033[0m]=force-s, [\033[32mfn\033[0m]=force-n -- alphabetical is a bit slower on fiber/LAN but makes it easier to eyeball if everything went fine")
     ap2.add_argument("--write-uplog", action="store_true", help="write POST reports to textfiles in working-directory")
 
@@ -710,6 +832,8 @@ def add_network(ap):
         ap2.add_argument("--reuseaddr", action="store_true", help="set reuseaddr on listening sockets on windows; allows rapid restart of copyparty at the expense of being able to accidentally start multiple instances")
     else:
         ap2.add_argument("--freebind", action="store_true", help="allow listening on IPs which do not yet exist, for example if the network interfaces haven't finished going up. Only makes sense for IPs other than '0.0.0.0', '127.0.0.1', '::', and '::1'. May require running as root (unless net.ipv6.ip_nonlocal_bind)")
+    ap2.add_argument("--s-thead", metavar="SEC", type=int, default=120, help="socket timeout (read request header)")
+    ap2.add_argument("--s-tbody", metavar="SEC", type=float, default=186, help="socket timeout (read/write request/response bodies). Use 60 on fast servers (default is extremely safe). Disable with 0 if reverse-proxied for a 2%% speed boost")
     ap2.add_argument("--s-wr-sz", metavar="B", type=int, default=256*1024, help="socket write size in bytes")
     ap2.add_argument("--s-wr-slp", metavar="SEC", type=float, default=0, help="debug: socket write delay in seconds")
     ap2.add_argument("--rsp-slp", metavar="SEC", type=float, default=0, help="debug: response delay in seconds")
@@ -727,6 +851,25 @@ def add_tls(ap, cert_path):
     ap2.add_argument("--ssl-log", metavar="PATH", type=u, help="log master secrets for later decryption in wireshark")
 
 
+def add_cert(ap, cert_path):
+    cert_dir = os.path.dirname(cert_path)
+    ap2 = ap.add_argument_group('TLS certificate generator options')
+    ap2.add_argument("--no-crt", action="store_true", help="disable automatic certificate creation")
+    ap2.add_argument("--crt-ns", metavar="N,N", type=u, default="", help="comma-separated list of FQDNs (domains) to add into the certificate")
+    ap2.add_argument("--crt-exact", action="store_true", help="do not add wildcard entries for each --crt-ns")
+    ap2.add_argument("--crt-noip", action="store_true", help="do not add autodetected IP addresses into cert")
+    ap2.add_argument("--crt-nolo", action="store_true", help="do not add 127.0.0.1 / localhost into cert")
+    ap2.add_argument("--crt-nohn", action="store_true", help="do not add mDNS names / hostname into cert")
+    ap2.add_argument("--crt-dir", metavar="PATH", default=cert_dir, help="where to save the CA cert")
+    ap2.add_argument("--crt-cdays", metavar="D", type=float, default=3650, help="ca-certificate expiration time in days")
+    ap2.add_argument("--crt-sdays", metavar="D", type=float, default=365, help="server-cert expiration time in days")
+    ap2.add_argument("--crt-cn", metavar="TXT", type=u, default="partyco", help="CA/server-cert common-name")
+    ap2.add_argument("--crt-cnc", metavar="TXT", type=u, default="--crt-cn", help="override CA name")
+    ap2.add_argument("--crt-cns", metavar="TXT", type=u, default="--crt-cn cpp", help="override server-cert name")
+    ap2.add_argument("--crt-back", metavar="HRS", type=float, default=72, help="backdate in hours")
+    ap2.add_argument("--crt-alg", metavar="S-N", type=u, default="ecdsa-256", help="algorithm and keysize; one of these: ecdsa-256 rsa-4096 rsa-2048")
+
+
 def add_zeroconf(ap):
     ap2 = ap.add_argument_group("Zeroconf options")
     ap2.add_argument("-z", action="store_true", help="enable all zeroconf backends (mdns, ssdp)")
@@ -738,7 +881,7 @@ def add_zeroconf(ap):
 
 
 def add_zc_mdns(ap):
-    ap2 = ap.add_argument_group("Zeroconf-mDNS options")
+    ap2 = ap.add_argument_group("Zeroconf-mDNS options; also see --help-zm")
     ap2.add_argument("--zm", action="store_true", help="announce the enabled protocols over mDNS (multicast DNS-SD) -- compatible with KDE, gnome, macOS, ...")
     ap2.add_argument("--zm-on", metavar="NETS", type=u, default="", help="enable zeroconf ONLY on the comma-separated list of subnets and/or interface names/indexes")
     ap2.add_argument("--zm-off", metavar="NETS", type=u, default="", help="disable zeroconf on the comma-separated list of subnets and/or interface names/indexes")
@@ -752,8 +895,9 @@ def add_zc_mdns(ap):
     ap2.add_argument("--zm-lf", metavar="PATH", type=u, default="", help="link a specific folder for ftp shares")
     ap2.add_argument("--zm-ls", metavar="PATH", type=u, default="", help="link a specific folder for smb shares")
     ap2.add_argument("--zm-mnic", action="store_true", help="merge NICs which share subnets; assume that same subnet means same network")
-    ap2.add_argument("--zm-msub", action="store_true", help="merge subnets on each NIC -- always enabled for ipv6 -- reduces network load, but gnome-gvfs clients may stop working")
+    ap2.add_argument("--zm-msub", action="store_true", help="merge subnets on each NIC -- always enabled for ipv6 -- reduces network load, but gnome-gvfs clients may stop working, and clients cannot be in subnets that the server is not")
     ap2.add_argument("--zm-noneg", action="store_true", help="disable NSEC replies -- try this if some clients don't see copyparty")
+    ap2.add_argument("--zm-spam", metavar="SEC", type=float, default=0, help="send unsolicited announce every SEC; useful if clients have IPs in a subnet which doesn't overlap with the server")
 
 
 def add_zc_ssdp(ap):
@@ -771,6 +915,7 @@ def add_ftp(ap):
     ap2.add_argument("--ftp", metavar="PORT", type=int, help="enable FTP server on PORT, for example \033[32m3921")
     ap2.add_argument("--ftps", metavar="PORT", type=int, help="enable FTPS server on PORT, for example \033[32m3990")
     ap2.add_argument("--ftpv", action="store_true", help="verbose")
+    ap2.add_argument("--ftp4", action="store_true", help="only listen on IPv4")
     ap2.add_argument("--ftp-wt", metavar="SEC", type=int, default=7, help="grace period for resuming interrupted uploads (any client can write to any file last-modified more recently than SEC seconds ago)")
     ap2.add_argument("--ftp-nat", metavar="ADDR", type=u, help="the NAT address to use for passive connections")
     ap2.add_argument("--ftp-pr", metavar="P-P", type=u, help="the range of TCP ports to use for passive connections, for example \033[32m12000-13000")
@@ -781,21 +926,31 @@ def add_webdav(ap):
     ap2.add_argument("--daw", action="store_true", help="enable full write support, even if client may not be webdav. \033[1;31mWARNING:\033[0m This has side-effects -- PUT-operations will now \033[1;31mOVERWRITE\033[0m existing files, rather than inventing new filenames to avoid loss of data. You might want to instead set this as a volflag where needed. By not setting this flag, uploaded files can get written to a filename which the client does not expect (which might be okay, depending on client)")
     ap2.add_argument("--dav-inf", action="store_true", help="allow depth:infinite requests (recursive file listing); extremely server-heavy but required for spec compliance -- luckily few clients rely on this")
     ap2.add_argument("--dav-mac", action="store_true", help="disable apple-garbage filter -- allow macos to create junk files (._* and .DS_Store, .Spotlight-*, .fseventsd, .Trashes, .AppleDouble, __MACOS)")
+    ap2.add_argument("--dav-rt", action="store_true", help="show symlink-destination's lastmodified instead of the link itself; always enabled for recursive listings (volflag=davrt)")
+    ap2.add_argument("--dav-auth", action="store_true", help="force auth for all folders (required by davfs2 when only some folders are world-readable) (volflag=davauth)")
 
 
 def add_smb(ap):
     ap2 = ap.add_argument_group('SMB/CIFS options')
-    ap2.add_argument("--smb", action="store_true", help="enable smb (read-only) -- this requires running copyparty as root on linux and macos unless --smb-port is set above 1024 and your OS does port-forwarding from 445 to that.\n\033[1;31mWARNING:\033[0m this protocol is dangerous! Never expose to the internet. Account permissions are coalesced; if one account has write-access to a volume, then all accounts do.")
+    ap2.add_argument("--smb", action="store_true", help="enable smb (read-only) -- this requires running copyparty as root on linux and macos unless --smb-port is set above 1024 and your OS does port-forwarding from 445 to that.\n\033[1;31mWARNING:\033[0m this protocol is dangerous! Never expose to the internet!")
     ap2.add_argument("--smbw", action="store_true", help="enable write support (please dont)")
     ap2.add_argument("--smb1", action="store_true", help="disable SMBv2, only enable SMBv1 (CIFS)")
     ap2.add_argument("--smb-port", metavar="PORT", type=int, default=445, help="port to listen on -- if you change this value, you must NAT from TCP:445 to this port using iptables or similar")
     ap2.add_argument("--smb-nwa-1", action="store_true", help="disable impacket#1433 workaround (truncate directory listings to 64kB)")
     ap2.add_argument("--smb-nwa-2", action="store_true", help="disable impacket workaround for filecopy globs")
+    ap2.add_argument("--smba", action="store_true", help="small performance boost: disable per-account permissions, enables account coalescing instead (if one user has write/delete-access, then everyone does)")
     ap2.add_argument("--smbv", action="store_true", help="verbose")
     ap2.add_argument("--smbvv", action="store_true", help="verboser")
     ap2.add_argument("--smbvvv", action="store_true", help="verbosest")
 
 
+def add_handlers(ap):
+    ap2 = ap.add_argument_group('handlers (see --help-handlers)')
+    ap2.add_argument("--on404", metavar="PY", type=u, action="append", help="handle 404s by executing PY file")
+    ap2.add_argument("--on403", metavar="PY", type=u, action="append", help="handle 403s by executing PY file")
+    ap2.add_argument("--hot-handlers", action="store_true", help="reload handlers on each request -- expensive but convenient when hacking on stuff")
+
+
 def add_hooks(ap):
     ap2 = ap.add_argument_group('event hooks (see --help-hooks)')
     ap2.add_argument("--xbu", metavar="CMD", type=u, action="append", help="execute CMD before a file upload starts")
@@ -806,6 +961,16 @@ def add_hooks(ap):
     ap2.add_argument("--xbd", metavar="CMD", type=u, action="append", help="execute CMD before a file delete")
     ap2.add_argument("--xad", metavar="CMD", type=u, action="append", help="execute CMD after  a file delete")
     ap2.add_argument("--xm", metavar="CMD", type=u, action="append", help="execute CMD on message")
+    ap2.add_argument("--xban", metavar="CMD", type=u, action="append", help="execute CMD if someone gets banned (pw/404/403/url)")
+
+
+def add_stats(ap):
+    ap2 = ap.add_argument_group('grafana/prometheus metrics endpoint')
+    ap2.add_argument("--stats", action="store_true", help="enable openmetrics at /.cpr/metrics for admin accounts")
+    ap2.add_argument("--nos-hdd", action="store_true", help="disable disk-space metrics (used/free space)")
+    ap2.add_argument("--nos-vol", action="store_true", help="disable volume size metrics (num files, total bytes, vmaxb/vmaxn)")
+    ap2.add_argument("--nos-dup", action="store_true", help="disable dupe-files metrics (good idea; very slow)")
+    ap2.add_argument("--nos-unf", action="store_true", help="disable unfinished-uploads metrics")
 
 
 def add_yolo(ap):
@@ -821,21 +986,23 @@ def add_optouts(ap):
     ap2.add_argument("--no-dav", action="store_true", help="disable webdav support")
     ap2.add_argument("--no-del", action="store_true", help="disable delete operations")
     ap2.add_argument("--no-mv", action="store_true", help="disable move/rename operations")
+    ap2.add_argument("-nth", action="store_true", help="no title hostname; don't show --name in <title>")
     ap2.add_argument("-nih", action="store_true", help="no info hostname -- don't show in UI")
     ap2.add_argument("-nid", action="store_true", help="no info disk-usage -- don't show in UI")
     ap2.add_argument("-nb", action="store_true", help="no powered-by-copyparty branding in UI")
     ap2.add_argument("--no-zip", action="store_true", help="disable download as zip/tar")
+    ap2.add_argument("--no-tarcmp", action="store_true", help="disable download as compressed tar (?tar=gz, ?tar=bz2, ?tar=xz, ?tar=gz:9, ...)")
     ap2.add_argument("--no-lifetime", action="store_true", help="disable automatic deletion of uploads after a certain time (as specified by the 'lifetime' volflag)")
 
 
-def add_safety(ap, fk_salt):
+def add_safety(ap):
     ap2 = ap.add_argument_group('safety options')
     ap2.add_argument("-s", action="count", default=0, help="increase safety: Disable thumbnails / potentially dangerous software (ffmpeg/pillow/vips), hide partial uploads, avoid crawlers.\n └─Alias of\033[32m --dotpart --no-thumb --no-mtag-ff --no-robots --force-js")
-    ap2.add_argument("-ss", action="store_true", help="further increase safety: Prevent js-injection, accidental move/delete, broken symlinks, webdav, 404 on 403, ban on excessive 404s.\n └─Alias of\033[32m -s --unpost=0 --no-del --no-mv --hardlink --vague-403 --ban-404=50,60,1440 -nih")
+    ap2.add_argument("-ss", action="store_true", help="further increase safety: Prevent js-injection, accidental move/delete, broken symlinks, webdav, 404 on 403, ban on excessive 404s.\n └─Alias of\033[32m -s --unpost=0 --no-del --no-mv --hardlink --vague-403 --ban-404=50,60,1440 --turbo=-1 -nih")
     ap2.add_argument("-sss", action="store_true", help="further increase safety: Enable logging to disk, scan for dangerous symlinks.\n └─Alias of\033[32m -ss --no-dav --no-logues --no-readme -lo=cpp-%%Y-%%m%%d-%%H%%M%%S.txt.xz --ls=**,*,ln,p,r")
     ap2.add_argument("--ls", metavar="U[,V[,F]]", type=u, help="do a sanity/safety check of all volumes on startup; arguments \033[33mUSER\033[0m,\033[33mVOL\033[0m,\033[33mFLAGS\033[0m; example [\033[32m**,*,ln,p,r\033[0m]")
-    ap2.add_argument("--salt", type=u, default="hunter2", help="up2k file-hash salt; serves no purpose, no reason to change this (but delete all databases if you do)")
-    ap2.add_argument("--fk-salt", metavar="SALT", type=u, default=fk_salt, help="per-file accesskey salt; used to generate unpredictable URLs for hidden files -- this one DOES matter")
+    ap2.add_argument("--xvol", action="store_true", help="never follow symlinks leaving the volume root, unless the link is into another volume where the user has similar access (volflag=xvol)")
+    ap2.add_argument("--xdev", action="store_true", help="stay within the filesystem of the volume root; do not descend into other devices (symlink or bind-mount to another HDD, ...) (volflag=xdev)")
     ap2.add_argument("--no-dot-mv", action="store_true", help="disallow moving dotfiles; makes it impossible to move folders containing dotfiles")
     ap2.add_argument("--no-dot-ren", action="store_true", help="disallow renaming dotfiles; makes it impossible to make something a dotfile")
     ap2.add_argument("--no-logues", action="store_true", help="disable rendering .prologue/.epilogue.html into directory listings")
@@ -846,12 +1013,27 @@ def add_safety(ap, fk_salt):
     ap2.add_argument("--logout", metavar="H", type=float, default="8086", help="logout clients after H hours of inactivity; [\033[32m0.0028\033[0m]=10sec, [\033[32m0.1\033[0m]=6min, [\033[32m24\033[0m]=day, [\033[32m168\033[0m]=week, [\033[32m720\033[0m]=month, [\033[32m8760\033[0m]=year)")
     ap2.add_argument("--ban-pw", metavar="N,W,B", type=u, default="9,60,1440", help="more than \033[33mN\033[0m wrong passwords in \033[33mW\033[0m minutes = ban for \033[33mB\033[0m minutes; disable with [\033[32mno\033[0m]")
     ap2.add_argument("--ban-404", metavar="N,W,B", type=u, default="no", help="hitting more than \033[33mN\033[0m 404's in \033[33mW\033[0m minutes = ban for \033[33mB\033[0m minutes (disabled by default since turbo-up2k counts as 404s)")
+    ap2.add_argument("--ban-403", metavar="N,W,B", type=u, default="9,2,1440", help="hitting more than \033[33mN\033[0m 403's in \033[33mW\033[0m minutes = ban for \033[33mB\033[0m minutes; [\033[32m1440\033[0m]=day, [\033[32m10080\033[0m]=week, [\033[32m43200\033[0m]=month")
+    ap2.add_argument("--ban-422", metavar="N,W,B", type=u, default="9,2,1440", help="hitting more than \033[33mN\033[0m 422's in \033[33mW\033[0m minutes = ban for \033[33mB\033[0m minutes (422 is server fuzzing, invalid POSTs and so)")
+    ap2.add_argument("--ban-url", metavar="N,W,B", type=u, default="9,2,1440", help="hitting more than \033[33mN\033[0m sus URL's in \033[33mW\033[0m minutes = ban for \033[33mB\033[0m minutes (decent replacement for --ban-404 if that can't be used)")
+    ap2.add_argument("--sus-urls", metavar="REGEX", type=u, default=r"\.php$|(^|/)wp-(admin|content|includes)/", help="URLs which are considered sus / eligible for banning; disable with blank or [\033[32mno\033[0m]")
+    ap2.add_argument("--nonsus-urls", metavar="REGEX", type=u, default=r"^(favicon\.ico|robots\.txt)$|^apple-touch-icon|^\.well-known", help="harmless URLs ignored from 404-bans; disable with blank or [\033[32mno\033[0m]")
     ap2.add_argument("--aclose", metavar="MIN", type=int, default=10, help="if a client maxes out the server connection limit, downgrade it from connection:keep-alive to connection:close for MIN minutes (and also kill its active connections) -- disable with 0")
     ap2.add_argument("--loris", metavar="B", type=int, default=60, help="if a client maxes out the server connection limit without sending headers, ban it for B minutes; disable with [\033[32m0\033[0m]")
     ap2.add_argument("--acao", metavar="V[,V]", type=u, default="*", help="Access-Control-Allow-Origin; list of origins (domains/IPs without port) to accept requests from; [\033[32mhttps://1.2.3.4\033[0m]. Default [\033[32m*\033[0m] allows requests from all sites but removes cookies and http-auth; only ?pw=hunter2 survives")
     ap2.add_argument("--acam", metavar="V[,V]", type=u, default="GET,HEAD", help="Access-Control-Allow-Methods; list of methods to accept from offsite ('*' behaves like described in --acao)")
 
 
+def add_salt(ap, fk_salt, ah_salt):
+    ap2 = ap.add_argument_group('salting options')
+    ap2.add_argument("--ah-alg", metavar="ALG", type=u, default="none", help="account-pw hashing algorithm; one of these, best to worst: argon2 scrypt sha2 none (each optionally followed by alg-specific comma-sep. config)")
+    ap2.add_argument("--ah-salt", metavar="SALT", type=u, default=ah_salt, help="account-pw salt; ignored if --ah-alg is none (default)")
+    ap2.add_argument("--ah-gen", metavar="PW", type=u, default="", help="generate hashed password for \033[33mPW\033[0m, or read passwords from STDIN if \033[33mPW\033[0m is [\033[32m-\033[0m]")
+    ap2.add_argument("--ah-cli", action="store_true", help="interactive shell which hashes passwords without ever storing or displaying the original passwords")
+    ap2.add_argument("--fk-salt", metavar="SALT", type=u, default=fk_salt, help="per-file accesskey salt; used to generate unpredictable URLs for hidden files")
+    ap2.add_argument("--warksalt", metavar="SALT", type=u, default="hunter2", help="up2k file-hash salt; serves no purpose, no reason to change this (but delete all databases if you do)")
+
+
 def add_shutdown(ap):
     ap2 = ap.add_argument_group('shutdown options')
     ap2.add_argument("--ign-ebind", action="store_true", help="continue running even if it's impossible to listen on some of the requested endpoints")
@@ -863,7 +1045,10 @@ def add_logging(ap):
     ap2 = ap.add_argument_group('logging options')
     ap2.add_argument("-q", action="store_true", help="quiet")
     ap2.add_argument("-lo", metavar="PATH", type=u, help="logfile, example: \033[32mcpp-%%Y-%%m%%d-%%H%%M%%S.txt.xz")
+    ap2.add_argument("--no-ansi", action="store_true", default=not VT100, help="disable colors; same as environment-variable NO_COLOR")
+    ap2.add_argument("--ansi", action="store_true", help="force colors; overrides environment-variable NO_COLOR")
     ap2.add_argument("--no-voldump", action="store_true", help="do not list volumes and permissions on startup")
+    ap2.add_argument("--log-tdec", type=int, default=3, help="timestamp resolution / number of timestamp decimals")
     ap2.add_argument("--log-conn", action="store_true", help="debug: print tcp-server msgs")
     ap2.add_argument("--log-htp", action="store_true", help="debug: print http-server threadpool scaling")
     ap2.add_argument("--ihead", metavar="HEADER", type=u, action='append', help="dump incoming header")
@@ -882,10 +1067,10 @@ def add_thumbnail(ap):
     ap2.add_argument("--no-thumb", action="store_true", help="disable all thumbnails (volflag=dthumb)")
     ap2.add_argument("--no-vthumb", action="store_true", help="disable video thumbnails (volflag=dvthumb)")
     ap2.add_argument("--no-athumb", action="store_true", help="disable audio thumbnails (spectrograms) (volflag=dathumb)")
-    ap2.add_argument("--th-size", metavar="WxH", default="320x256", help="thumbnail res")
+    ap2.add_argument("--th-size", metavar="WxH", default="320x256", help="thumbnail res (volflag=thsize)")
     ap2.add_argument("--th-mt", metavar="CORES", type=int, default=CORES, help="num cpu cores to use for generating thumbnails")
-    ap2.add_argument("--th-convt", metavar="SEC", type=int, default=60, help="conversion timeout in seconds")
-    ap2.add_argument("--th-no-crop", action="store_true", help="dynamic height; show full image")
+    ap2.add_argument("--th-convt", metavar="SEC", type=float, default=60, help="conversion timeout in seconds (volflag=convt)")
+    ap2.add_argument("--th-no-crop", action="store_true", help="dynamic height; show full image (volflag=nocrop)")
     ap2.add_argument("--th-dec", metavar="LIBS", default="vips,pil,ff", help="image decoders, in order of preference")
     ap2.add_argument("--th-no-jpg", action="store_true", help="disable jpg output")
     ap2.add_argument("--th-no-webp", action="store_true", help="disable webp output")
@@ -894,7 +1079,7 @@ def add_thumbnail(ap):
     ap2.add_argument("--th-poke", metavar="SEC", type=int, default=300, help="activity labeling cooldown -- avoids doing keepalive pokes (updating the mtime) on thumbnail folders more often than SEC seconds")
     ap2.add_argument("--th-clean", metavar="SEC", type=int, default=43200, help="cleanup interval; 0=disabled")
     ap2.add_argument("--th-maxage", metavar="SEC", type=int, default=604800, help="max folder age -- folders which haven't been poked for longer than --th-poke seconds will get deleted every --th-clean seconds")
-    ap2.add_argument("--th-covers", metavar="N,N", type=u, default="folder.png,folder.jpg,cover.png,cover.jpg", help="folder thumbnails to stat/look for; case-insensitive if -e2d")
+    ap2.add_argument("--th-covers", metavar="N,N", type=u, default="folder.png,folder.jpg,cover.png,cover.jpg", help="folder thumbnails to stat/look for; enabling -e2d will make these case-insensitive, and also automatically select thumbnails for all folders that contain pics, even if none match this pattern")
     # https://pillow.readthedocs.io/en/stable/handbook/image-file-formats.html
     # https://github.com/libvips/libvips
     # ffmpeg -hide_banner -demuxers | awk '/^ D  /{print$2}' | while IFS= read -r x; do ffmpeg -hide_banner -h demuxer=$x; done | grep -E '^Demuxer |extensions:'
@@ -908,6 +1093,7 @@ def add_thumbnail(ap):
 def add_transcoding(ap):
     ap2 = ap.add_argument_group('transcoding options')
     ap2.add_argument("--no-acode", action="store_true", help="disable audio transcoding")
+    ap2.add_argument("--no-bacode", action="store_true", help="disable batch audio transcoding by folder download (zip/tar)")
     ap2.add_argument("--ac-maxage", metavar="SEC", type=int, default=86400, help="delete cached transcode output after SEC seconds")
 
 
@@ -927,8 +1113,6 @@ def add_db_general(ap, hcores):
     ap2.add_argument("--no-forget", action="store_true", help="never forget indexed files, even when deleted from disk -- makes it impossible to ever upload the same file twice (volflag=noforget)")
     ap2.add_argument("--dbd", metavar="PROFILE", default="wal", help="database durability profile; sets the tradeoff between robustness and speed, see --help-dbd (volflag=dbd)")
     ap2.add_argument("--xlink", action="store_true", help="on upload: check all volumes for dupes, not just the target volume (volflag=xlink)")
-    ap2.add_argument("--xdev", action="store_true", help="do not descend into other filesystems (symlink or bind-mount to another HDD, ...) (volflag=xdev)")
-    ap2.add_argument("--xvol", action="store_true", help="skip symlinks leaving the volume root (volflag=xvol)")
     ap2.add_argument("--hash-mt", metavar="CORES", type=int, default=hcores, help="num cpu cores to use for file hashing; set 0 or 1 for single-core hashing")
     ap2.add_argument("--re-maxage", metavar="SEC", type=int, default=0, help="disk rescan volume interval, 0=off (volflag=scan)")
     ap2.add_argument("--db-act", metavar="SEC", type=float, default=10, help="defer any scheduled volume reindexing until SEC seconds after last db write (uploads, renames, ...)")
@@ -950,7 +1134,7 @@ def add_db_metadata(ap):
     ap2.add_argument("--mtag-vv", action="store_true", help="debug mtp settings and mutagen/ffprobe parsers")
     ap2.add_argument("-mtm", metavar="M=t,t,t", type=u, action="append", help="add/replace metadata mapping")
     ap2.add_argument("-mte", metavar="M,M,M", type=u, help="tags to index/display (comma-sep.)",
-        default="circle,album,.tn,artist,title,.bpm,key,.dur,.q,.vq,.aq,vc,ac,fmt,res,.fps,ahash,vhash")
+        default="circle,album,.tn,artist,title,.bpm,key,.dur,.q,.vq,.aq,vc,ac,fmt,res,.fps,ahash,vhash,up_ip,.up_at")
     ap2.add_argument("-mth", metavar="M,M,M", type=u, help="tags to hide by default (comma-sep.)",
         default=".vq,.aq,vc,ac,fmt,res,.fps")
     ap2.add_argument("-mtp", metavar="M=[f,]BIN", type=u, action="append", help="read tag M using program BIN to parse the file")
@@ -958,20 +1142,23 @@ def add_db_metadata(ap):
 
 def add_ui(ap, retry):
     ap2 = ap.add_argument_group('ui options')
+    ap2.add_argument("--grid", action="store_true", help="show grid/thumbnails by default (volflag=grid)")
     ap2.add_argument("--lang", metavar="LANG", type=u, default="eng", help="language")
     ap2.add_argument("--theme", metavar="NUM", type=int, default=0, help="default theme to use")
     ap2.add_argument("--themes", metavar="NUM", type=int, default=8, help="number of themes installed")
+    ap2.add_argument("--unlist", metavar="REGEX", type=u, default="", help="don't show files matching REGEX in file list. Purely cosmetic! Does not affect API calls, just the browser. Example: [\033[32m\\.(js|css)$\033[0m] (volflag=unlist)")
     ap2.add_argument("--favico", metavar="TXT", type=u, default="c 000 none" if retry else "🎉 000 none", help="\033[33mfavicon-text\033[0m [ \033[33mforeground\033[0m [ \033[33mbackground\033[0m ] ], set blank to disable")
     ap2.add_argument("--mpmc", metavar="URL", type=u, default="", help="change the mediaplayer-toggle mouse cursor; URL to a folder with {2..5}.png inside (or disable with [\033[32m.\033[0m])")
     ap2.add_argument("--js-browser", metavar="L", type=u, help="URL to additional JS to include")
     ap2.add_argument("--css-browser", metavar="L", type=u, help="URL to additional CSS to include")
     ap2.add_argument("--html-head", metavar="TXT", type=u, default="", help="text to append to the <head> of all HTML pages")
-    ap2.add_argument("--ih", action="store_true", help="if a folder contains index.html, show that instead of the directory listing by default (can be changed in the client settings UI)")
+    ap2.add_argument("--ih", action="store_true", help="if a folder contains index.html, show that instead of the directory listing by default (can be changed in the client settings UI, or add ?v to URL for override)")
     ap2.add_argument("--textfiles", metavar="CSV", type=u, default="txt,nfo,diz,cue,readme", help="file extensions to present as plaintext")
     ap2.add_argument("--txt-max", metavar="KiB", type=int, default=64, help="max size of embedded textfiles on ?doc= (anything bigger will be lazy-loaded by JS)")
-    ap2.add_argument("--doctitle", metavar="TXT", type=u, default="copyparty", help="title / service-name to show in html documents")
+    ap2.add_argument("--doctitle", metavar="TXT", type=u, default="copyparty @ --name", help="title / service-name to show in html documents")
+    ap2.add_argument("--bname", metavar="TXT", type=u, default="--name", help="server name (displayed in filebrowser document title)")
     ap2.add_argument("--pb-url", metavar="URL", type=u, default="https://github.com/9001/copyparty", help="powered-by link; disable with -np")
-    ap2.add_argument("--ver", action="store_true", help="show version on the control panel (incompatible by -np)")
+    ap2.add_argument("--ver", action="store_true", help="show version on the control panel (incompatible with -nb)")
     ap2.add_argument("--md-sbf", metavar="FLAGS", type=u, default="downloads forms popups scripts top-navigation-by-user-activation", help="list of capabilities to ALLOW for README.md docs (volflag=md_sbf); see https://developer.mozilla.org/en-US/docs/Web/HTML/Element/iframe#attr-sandbox")
     ap2.add_argument("--lg-sbf", metavar="FLAGS", type=u, default="downloads forms popups scripts top-navigation-by-user-activation", help="list of capabilities to ALLOW for prologue/epilogue docs  (volflag=lg_sbf)")
     ap2.add_argument("--no-sb-md", action="store_true", help="don't sandbox README.md documents (volflags: no_sb_md | sb_md)")
@@ -1009,10 +1196,8 @@ def run_argparse(
 
     cert_path = os.path.join(E.cfg, "cert.pem")
 
-    try:
-        fk_salt = unicode(os.path.getmtime(cert_path))
-    except:
-        fk_salt = "hunter2"
+    fk_salt = get_fk_salt(cert_path)
+    ah_salt = get_ah_salt()
 
     hcores = min(CORES, 4)  # optimal on py3.11 @ r5-4500U
 
@@ -1023,6 +1208,7 @@ def run_argparse(
     add_general(ap, nc, srvname)
     add_network(ap)
     add_tls(ap, cert_path)
+    add_cert(ap, cert_path)
     add_qr(ap, tty)
     add_zeroconf(ap)
     add_zc_mdns(ap)
@@ -1035,11 +1221,14 @@ def run_argparse(
     add_ftp(ap)
     add_webdav(ap)
     add_smb(ap)
-    add_safety(ap, fk_salt)
+    add_safety(ap)
+    add_salt(ap, fk_salt, ah_salt)
     add_optouts(ap)
     add_shutdown(ap)
     add_yolo(ap)
+    add_handlers(ap)
     add_hooks(ap)
+    add_stats(ap)
     add_ui(ap, retry)
     add_admin(ap)
     add_logging(ap)
@@ -1107,6 +1296,8 @@ def main(argv: Optional[list[str]] = None) -> None:
 
     ensure_locale()
 
+    ensure_webdeps()
+
     for k, v in zip(argv[1:], argv[2:]):
         if k == "-c" and os.path.isfile(v):
             supp = args_from_cfg(v)
@@ -1118,16 +1309,22 @@ def main(argv: Optional[list[str]] = None) -> None:
             supp = args_from_cfg(v)
             argv.extend(supp)
 
-    deprecated: list[tuple[str, str]] = []
+    deprecated: list[tuple[str, str]] = [("--salt", "--warksalt")]
     for dk, nk in deprecated:
-        try:
-            idx = argv.index(dk)
-        except:
+        idx = -1
+        ov = ""
+        for n, k in enumerate(argv):
+            if k == dk or k.startswith(dk + "="):
+                idx = n
+                if "=" in k:
+                    ov = "=" + k.split("=", 1)[1]
+
+        if idx < 0:
             continue
 
         msg = "\033[1;31mWARNING:\033[0;1m\n  {} \033[0;33mwas replaced with\033[0;1m {} \033[0;33mand will be removed\n\033[0m"
         lprint(msg.format(dk, nk))
-        argv[idx] = nk
+        argv[idx] = nk + ov
         time.sleep(2)
 
     da = len(argv) == 1
@@ -1173,16 +1370,18 @@ def main(argv: Optional[list[str]] = None) -> None:
     except:
         sys.exit(1)
 
-    if HAVE_SSL:
-        ensure_cert(al)
+    if al.ansi:
+        al.no_ansi = False
+    elif not al.no_ansi:
+        al.ansi = VT100
 
-    if WINDOWS and not al.keep_qem:
+    if WINDOWS and not al.keep_qem and not al.ah_cli:
         try:
             disable_quickedit()
         except:
             lprint("\nfailed to disable quick-edit-mode:\n" + min_ex() + "\n")
 
-    if not VT100:
+    if al.ansi:
         al.wintitle = ""
 
     nstrs: list[str] = []
@@ -1201,11 +1400,9 @@ def main(argv: Optional[list[str]] = None) -> None:
             if re.match("c[^,]", opt):
                 mod = True
                 na.append("c," + opt[1:])
-            elif re.sub("^[rwmdgG]*", "", opt) and "," not in opt:
+            elif re.sub("^[rwmdgGha]*", "", opt) and "," not in opt:
                 mod = True
                 perm = opt[0]
-                if perm == "a":
-                    perm = "rw"
                 na.append(perm + "," + opt[1:])
             else:
                 na.append(opt)
@@ -1261,6 +1458,7 @@ def main(argv: Optional[list[str]] = None) -> None:
             configure_ssl_ciphers(al)
     else:
         warn("ssl module does not exist; cannot enable https")
+        al.http_only = True
 
     if PY2 and WINDOWS and al.e2d:
         warn(

copyparty/__version__.py

@@ -1,8 +1,8 @@
 # coding: utf-8
 
-VERSION = (1, 6, 13)
-CODENAME = "cors k"
-BUILD_DT = (2023, 4, 23)
+VERSION = (1, 9, 5)
+CODENAME = "prometheable"
+BUILD_DT = (2023, 9, 9)
 
 S_VERSION = ".".join(map(str, VERSION))
 S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT)

copyparty/authsrv.py

@@ -15,6 +15,7 @@ from datetime import datetime
 from .__init__ import ANYWIN, TYPE_CHECKING, WINDOWS
 from .bos import bos
 from .cfg import flagdescs, permdescs, vf_bmap, vf_cmap, vf_vmap
+from .pwhash import PWHash
 from .util import (
     IMPLICATIONS,
     META_NOBOTS,
@@ -40,7 +41,10 @@ if True:  # pylint: disable=using-constant-test
     from .util import NamedLogger, RootLogger
 
 if TYPE_CHECKING:
-    pass
+    from .broker_mp import BrokerMp
+    from .broker_thr import BrokerThr
+    from .broker_util import BrokerCli
+
     # Vflags: TypeAlias = dict[str, str | bool | float | list[str]]
     # Vflags: TypeAlias = dict[str, Any]
     # Mflags: TypeAlias = dict[str, Vflags]
@@ -48,6 +52,11 @@ if TYPE_CHECKING:
 
 LEELOO_DALLAS = "leeloo_dallas"
 
+SEE_LOG = "see log for details"
+SSEELOG = " ({})".format(SEE_LOG)
+BAD_CFG = "invalid config; {}".format(SEE_LOG)
+SBADCFG = " ({})".format(BAD_CFG)
+
 
 class AXS(object):
     def __init__(
@@ -58,6 +67,8 @@ class AXS(object):
         udel: Optional[Union[list[str], set[str]]] = None,
         uget: Optional[Union[list[str], set[str]]] = None,
         upget: Optional[Union[list[str], set[str]]] = None,
+        uhtml: Optional[Union[list[str], set[str]]] = None,
+        uadmin: Optional[Union[list[str], set[str]]] = None,
     ) -> None:
         self.uread: set[str] = set(uread or [])
         self.uwrite: set[str] = set(uwrite or [])
@@ -65,14 +76,12 @@ class AXS(object):
         self.udel: set[str] = set(udel or [])
         self.uget: set[str] = set(uget or [])
         self.upget: set[str] = set(upget or [])
+        self.uhtml: set[str] = set(uhtml or [])
+        self.uadmin: set[str] = set(uadmin or [])
 
     def __repr__(self) -> str:
-        return "AXS({})".format(
-            ", ".join(
-                "{}={!r}".format(k, self.__dict__[k])
-                for k in "uread uwrite umove udel uget upget".split()
-            )
-        )
+        ks = "uread uwrite umove udel uget upget uhtml uadmin".split()
+        return "AXS(%s)" % (", ".join("%s=%r" % (k, self.__dict__[k]) for k in ks),)
 
 
 class Lim(object):
@@ -90,6 +99,8 @@ class Lim(object):
         self.dfl = 0  # free disk space limit
         self.dft = 0  # last-measured time
         self.dfv = 0  # currently free
+        self.vbmax = 0  # volume bytes max
+        self.vnmax = 0  # volume max num files
 
         self.smin = 0  # filesize min
         self.smax = 0  # filesize max
@@ -119,8 +130,11 @@ class Lim(object):
         ip: str,
         rem: str,
         sz: int,
+        ptop: str,
         abspath: str,
+        broker: Optional[Union["BrokerCli", "BrokerMp", "BrokerThr"]] = None,
         reg: Optional[dict[str, dict[str, Any]]] = None,
+        volgetter: str = "up2k.get_volsize",
     ) -> tuple[str, str]:
         if reg is not None and self.reg is None:
             self.reg = reg
@@ -131,6 +145,7 @@ class Lim(object):
         self.chk_rem(rem)
         if sz != -1:
             self.chk_sz(sz)
+            self.chk_vsz(broker, ptop, sz, volgetter)
             self.chk_df(abspath, sz)  # side effects; keep last-ish
 
         ap2, vp2 = self.rot(abspath)
@@ -146,6 +161,25 @@ class Lim(object):
         if self.smax and sz > self.smax:
             raise Pebkac(400, "file too big")
 
+    def chk_vsz(
+        self,
+        broker: Optional[Union["BrokerCli", "BrokerMp", "BrokerThr"]],
+        ptop: str,
+        sz: int,
+        volgetter: str = "up2k.get_volsize",
+    ) -> None:
+        if not broker or not self.vbmax + self.vnmax:
+            return
+
+        x = broker.ask(volgetter, ptop)
+        nbytes, nfiles = x.get()
+
+        if self.vbmax and self.vbmax < nbytes + sz:
+            raise Pebkac(400, "volume has exceeded max size")
+
+        if self.vnmax and self.vnmax < nfiles + 1:
+            raise Pebkac(400, "volume has exceeded max num.files")
+
     def chk_df(self, abspath: str, sz: int, already_written: bool = False) -> None:
         if not self.dfl:
             return
@@ -266,7 +300,7 @@ class Lim(object):
 
         self.bupc[ip] = mark
         if mark >= self.bmax:
-            raise Pebkac(429, "ingress saturated")
+            raise Pebkac(429, "upload size limit exceeded")
 
 
 class VFS(object):
@@ -285,6 +319,8 @@ class VFS(object):
         self.vpath = vpath  # absolute path in the virtual filesystem
         self.axs = axs
         self.flags = flags  # config options
+        self.root = self
+        self.dev = 0  # st_dev
         self.nodes: dict[str, VFS] = {}  # child nodes
         self.histtab: dict[str, str] = {}  # all realpath->histpath
         self.dbv: Optional[VFS] = None  # closest full/non-jump parent
@@ -295,28 +331,46 @@ class VFS(object):
         self.adel: dict[str, list[str]] = {}
         self.aget: dict[str, list[str]] = {}
         self.apget: dict[str, list[str]] = {}
+        self.ahtml: dict[str, list[str]] = {}
+        self.aadmin: dict[str, list[str]] = {}
 
         if realpath:
+            rp = realpath + ("" if realpath.endswith(os.sep) else os.sep)
+            vp = vpath + ("/" if vpath else "")
             self.histpath = os.path.join(realpath, ".hist")  # db / thumbcache
             self.all_vols = {vpath: self}  # flattened recursive
+            self.all_aps = [(rp, self)]
+            self.all_vps = [(vp, self)]
         else:
             self.histpath = ""
             self.all_vols = {}
+            self.all_aps = []
+            self.all_vps = []
 
     def __repr__(self) -> str:
-        return "VFS({})".format(
+        return "VFS(%s)" % (
             ", ".join(
-                "{}={!r}".format(k, self.__dict__[k])
+                "%s=%r" % (k, self.__dict__[k])
                 for k in "realpath vpath axs flags".split()
             )
         )
 
-    def get_all_vols(self, outdict: dict[str, "VFS"]) -> None:
+    def get_all_vols(
+        self,
+        vols: dict[str, "VFS"],
+        aps: list[tuple[str, "VFS"]],
+        vps: list[tuple[str, "VFS"]],
+    ) -> None:
         if self.realpath:
-            outdict[self.vpath] = self
+            vols[self.vpath] = self
+            rp = self.realpath
+            rp += "" if rp.endswith(os.sep) else os.sep
+            vp = self.vpath + ("/" if self.vpath else "")
+            aps.append((rp, self))
+            vps.append((vp, self))
 
         for v in self.nodes.values():
-            v.get_all_vols(outdict)
+            v.get_all_vols(vols, aps, vps)
 
     def add(self, src: str, dst: str) -> "VFS":
         """get existing, or add new path to the vfs"""
@@ -388,9 +442,13 @@ class VFS(object):
 
     def can_access(
         self, vpath: str, uname: str
-    ) -> tuple[bool, bool, bool, bool, bool, bool]:
-        """can Read,Write,Move,Delete,Get,Upget"""
-        vn, _ = self._find(undot(vpath))
+    ) -> tuple[bool, bool, bool, bool, bool, bool, bool]:
+        """can Read,Write,Move,Delete,Get,Upget,Admin"""
+        if vpath:
+            vn, _ = self._find(undot(vpath))
+        else:
+            vn = self
+
         c = vn.axs
         return (
             uname in c.uread or "*" in c.uread,
@@ -399,7 +457,9 @@ class VFS(object):
             uname in c.udel or "*" in c.udel,
             uname in c.uget or "*" in c.uget,
             uname in c.upget or "*" in c.upget,
+            uname in c.uadmin or "*" in c.uadmin,
         )
+        # skip uhtml because it's rarely needed
 
     def get(
         self,
@@ -420,7 +480,8 @@ class VFS(object):
                     self.log("vfs", "invalid relpath [{}]".format(vpath))
                 raise Pebkac(404)
 
-        vn, rem = self._find(undot(vpath))
+        cvpath = undot(vpath)
+        vn, rem = self._find(cvpath)
         c: AXS = vn.axs
 
         for req, d, msg in [
@@ -431,6 +492,11 @@ class VFS(object):
             (will_get, c.uget, "get"),
         ]:
             if req and (uname not in d and "*" not in d) and uname != LEELOO_DALLAS:
+                if vpath != cvpath and vpath != "." and self.log:
+                    ap = vn.canonical(rem)
+                    t = "{} has no {} in [{}] => [{}] => [{}]"
+                    self.log("vfs", t.format(uname, msg, vpath, cvpath, ap), 6)
+
                 t = "you don't have {}-access for this location"
                 raise Pebkac(err, t.format(msg))
 
@@ -545,9 +611,20 @@ class VFS(object):
                 self.log("vfs.walk", t.format(seen[-1], fsroot, self.vpath, rem), 3)
             return
 
+        if "xdev" in self.flags or "xvol" in self.flags:
+            rm1 = []
+            for le in vfs_ls:
+                ap = absreal(os.path.join(fsroot, le[0]))
+                vn2 = self.chk_ap(ap)
+                if not vn2 or not vn2.get("", uname, True, False):
+                    rm1.append(le)
+            _ = [vfs_ls.remove(x) for x in rm1]  # type: ignore
+
         seen = seen[:] + [fsroot]
         rfiles = [x for x in vfs_ls if not stat.S_ISDIR(x[1].st_mode)]
         rdirs = [x for x in vfs_ls if stat.S_ISDIR(x[1].st_mode)]
+        # if lstat: ignore folder symlinks since copyparty will never make those
+        #            (and we definitely don't want to descend into them)
 
         rfiles.sort()
         rdirs.sort()
@@ -641,6 +718,44 @@ class VFS(object):
             for d in [{"vp": v, "ap": a, "st": n} for v, a, n in ret2]:
                 yield d
 
+    def chk_ap(self, ap: str, st: Optional[os.stat_result] = None) -> Optional["VFS"]:
+        aps = ap + os.sep
+        if "xdev" in self.flags and not ANYWIN:
+            if not st:
+                ap2 = ap.replace("\\", "/") if ANYWIN else ap
+                while ap2:
+                    try:
+                        st = bos.stat(ap2)
+                        break
+                    except:
+                        if "/" not in ap2:
+                            raise
+                        ap2 = ap2.rsplit("/", 1)[0]
+                assert st
+
+            vdev = self.dev
+            if not vdev:
+                vdev = self.dev = bos.stat(self.realpath).st_dev
+
+            if vdev != st.st_dev:
+                if self.log:
+                    t = "xdev: {}[{}] => {}[{}]"
+                    self.log("vfs", t.format(vdev, self.realpath, st.st_dev, ap), 3)
+
+                return None
+
+        if "xvol" in self.flags:
+            for vap, vn in self.root.all_aps:
+                if aps.startswith(vap):
+                    return vn
+
+            if self.log:
+                self.log("vfs", "xvol: [{}]".format(ap), 3)
+
+            return None
+
+        return self
+
 
 if WINDOWS:
     re_vol = re.compile(r"^([a-zA-Z]:[\\/][^:]*|[^:]*):([^:]*):(.*)$")
@@ -658,6 +773,7 @@ class AuthSrv(object):
         warn_anonwrite: bool = True,
         dargs: Optional[argparse.Namespace] = None,
     ) -> None:
+        self.ah = PWHash(args)
         self.args = args
         self.dargs = dargs or args
         self.log_func = log_func
@@ -694,7 +810,7 @@ class AuthSrv(object):
         if dst in mount:
             t = "multiple filesystem-paths mounted at [/{}]:\n  [{}]\n  [{}]"
             self.log(t.format(dst, mount[dst], src), c=1)
-            raise Exception("invalid config")
+            raise Exception(BAD_CFG)
 
         if src in mount.values():
             t = "filesystem-path [{}] mounted in multiple locations:"
@@ -703,7 +819,7 @@ class AuthSrv(object):
                 t += "\n  /{}".format(v)
 
             self.log(t, c=3)
-            raise Exception("invalid config")
+            raise Exception(BAD_CFG)
 
         if not bos.path.isdir(src):
             self.log("warning: filesystem-path does not exist: {}".format(src), 3)
@@ -770,6 +886,9 @@ class AuthSrv(object):
             if not ln.split("#")[0].strip():
                 continue
 
+            if re.match(r"^\[.*\]:$", ln):
+                ln = ln[:-1]
+
             subsection = ln in (catx, catf)
             if ln.startswith("[") or subsection:
                 self._e()
@@ -799,7 +918,7 @@ class AuthSrv(object):
                         t = "volume-specific config (anything from --help-flags)"
                         self._l(ln, 6, t)
                 else:
-                    raise Exception("invalid section header")
+                    raise Exception("invalid section header" + SBADCFG)
 
                 self.indent = "    " if subsection else "  "
                 continue
@@ -822,7 +941,7 @@ class AuthSrv(object):
                     acct[u] = p
                 except:
                     t = 'lines inside the [accounts] section must be "username: password"'
-                    raise Exception(t)
+                    raise Exception(t + SBADCFG)
                 continue
 
             if vp is not None and ap is None:
@@ -840,7 +959,7 @@ class AuthSrv(object):
                 try:
                     self._l(ln, 5, "volume access config:")
                     sk, sv = ln.split(":")
-                    if re.sub("[rwmdgG]", "", sk) or not sk:
+                    if re.sub("[rwmdgGha]", "", sk) or not sk:
                         err = "invalid accs permissions list; "
                         raise Exception(err)
                     if " " in re.sub(", *", "", sv).strip():
@@ -849,8 +968,8 @@ class AuthSrv(object):
                     self._read_vol_str(sk, sv.replace(" ", ""), daxs[vp], mflags[vp])
                     continue
                 except:
-                    err += "accs entries must be 'rwmdgG: user1, user2, ...'"
-                    raise Exception(err)
+                    err += "accs entries must be 'rwmdgGha: user1, user2, ...'"
+                    raise Exception(err + SBADCFG)
 
             if cat == catf:
                 err = ""
@@ -863,7 +982,7 @@ class AuthSrv(object):
                         if bad:
                             err = "bad characters [{}] in volflag name [{}]; "
                             err = err.format(bad, sk)
-                            raise Exception(err)
+                            raise Exception(err + SBADCFG)
                         if sv is True:
                             fstr += "," + sk
                         else:
@@ -875,9 +994,9 @@ class AuthSrv(object):
                     continue
                 except:
                     err += "flags entries (volflags) must be one of the following:\n  'flag1, flag2, ...'\n  'key: value'\n  'flag1, flag2, key: value'"
-                    raise Exception(err)
+                    raise Exception(err + SBADCFG)
 
-            raise Exception("unprocessable line in config")
+            raise Exception("unprocessable line in config" + SBADCFG)
 
         self._e()
         self.line_ctr = 0
@@ -885,7 +1004,7 @@ class AuthSrv(object):
     def _read_vol_str(
         self, lvl: str, uname: str, axs: AXS, flags: dict[str, Any]
     ) -> None:
-        if lvl.strip("crwmdgG"):
+        if lvl.strip("crwmdgGha"):
             raise Exception("invalid volflag: {},{}".format(lvl, uname))
 
         if lvl == "c":
@@ -914,6 +1033,9 @@ class AuthSrv(object):
                 ("w", axs.uwrite),
                 ("m", axs.umove),
                 ("d", axs.udel),
+                ("a", axs.uadmin),
+                ("h", axs.uhtml),
+                ("h", axs.uget),
                 ("g", axs.uget),
                 ("G", axs.uget),
                 ("G", axs.upget),
@@ -943,7 +1065,8 @@ class AuthSrv(object):
             flags[name] = True
             return
 
-        if name not in "mtp xbu xau xiu xbr xar xbd xad xm".split():
+        zs = "mtp on403 on404 xbu xau xiu xbr xar xbd xad xm xban"
+        if name not in zs.split():
             if value is True:
                 t = "└─add volflag [{}] = {}  ({})"
             else:
@@ -988,7 +1111,7 @@ class AuthSrv(object):
 
         if self.args.v:
             # list of src:dst:permset:permset:...
-            # permset is <rwmdgG>[,username][,username] or <c>,<flag>[=args]
+            # permset is <rwmdgGha>[,username][,username] or <c>,<flag>[=args]
             for v_str in self.args.v:
                 m = re_vol.match(v_str)
                 if not m:
@@ -1032,6 +1155,8 @@ class AuthSrv(object):
                     self.log("\n{0}\n{1}{0}".format(t, "\n".join(slns)))
                     raise
 
+        self.setup_pwhash(acct)
+
         # case-insensitive; normalize
         if WINDOWS:
             cased = {}
@@ -1067,9 +1192,15 @@ class AuthSrv(object):
 
         assert vfs
         vfs.all_vols = {}
-        vfs.get_all_vols(vfs.all_vols)
+        vfs.all_aps = []
+        vfs.all_vps = []
+        vfs.get_all_vols(vfs.all_vols, vfs.all_aps, vfs.all_vps)
+        for vol in vfs.all_vols.values():
+            vol.all_aps.sort(key=lambda x: len(x[0]), reverse=True)
+            vol.all_vps.sort(key=lambda x: len(x[0]), reverse=True)
+            vol.root = vfs
 
-        for perm in "read write move del get pget".split():
+        for perm in "read write move del get pget html admin".split():
             axs_key = "u" + perm
             unames = ["*"] + list(acct.keys())
             umap: dict[str, list[str]] = {x: [] for x in unames}
@@ -1084,7 +1215,16 @@ class AuthSrv(object):
         all_users = {}
         missing_users = {}
         for axs in daxs.values():
-            for d in [axs.uread, axs.uwrite, axs.umove, axs.udel, axs.uget, axs.upget]:
+            for d in [
+                axs.uread,
+                axs.uwrite,
+                axs.umove,
+                axs.udel,
+                axs.uget,
+                axs.upget,
+                axs.uhtml,
+                axs.uadmin,
+            ]:
                 for usr in d:
                     all_users[usr] = 1
                     if usr != "*" and usr not in acct:
@@ -1096,11 +1236,19 @@ class AuthSrv(object):
                 + ", ".join(k for k in sorted(missing_users)),
                 c=1,
             )
-            raise Exception("invalid config")
+            raise Exception(BAD_CFG)
 
         if LEELOO_DALLAS in all_users:
             raise Exception("sorry, reserved username: " + LEELOO_DALLAS)
 
+        seenpwds = {}
+        for usr, pwd in acct.items():
+            if pwd in seenpwds:
+                t = "accounts [{}] and [{}] have the same password; this is not supported"
+                self.log(t.format(seenpwds[pwd], usr), 1)
+                raise Exception(BAD_CFG)
+            seenpwds[pwd] = usr
+
         promote = []
         demote = []
         for vol in vfs.all_vols.values():
@@ -1202,6 +1350,16 @@ class AuthSrv(object):
                 use = True
                 lim.bmax, lim.bwin = [unhumanize(x) for x in zs.split(",")]
 
+            zs = vol.flags.get("vmaxb")
+            if zs:
+                use = True
+                lim.vbmax = unhumanize(zs)
+
+            zs = vol.flags.get("vmaxn")
+            if zs:
+                use = True
+                lim.vnmax = unhumanize(zs)
+
             if use:
                 vol.lim = lim
 
@@ -1290,6 +1448,10 @@ class AuthSrv(object):
                 if k in vol.flags:
                     vol.flags[k] = int(vol.flags[k])
 
+            for k in ("convt",):
+                if k in vol.flags:
+                    vol.flags[k] = float(vol.flags[k])
+
             for k1, k2 in IMPLICATIONS:
                 if k1 in vol.flags:
                     vol.flags[k2] = True
@@ -1315,8 +1477,8 @@ class AuthSrv(object):
                 vol.flags["mth"] = self.args.mth
 
             # append additive args from argv to volflags
-            hooks = "xbu xau xiu xbr xar xbd xad xm".split()
-            for name in ["mtp"] + hooks:
+            hooks = "xbu xau xiu xbr xar xbd xad xm xban".split()
+            for name in "mtp on404 on403".split() + hooks:
                 self._read_volflag(vol.flags, name, getattr(self.args, name), True)
 
             for hn in hooks:
@@ -1338,6 +1500,10 @@ class AuthSrv(object):
                         hfs = [x for x in hfs if x != "f"]
                         ocmd = ",".join(hfs + [cmd])
 
+                    if "c" not in hfs and "f" not in hfs and hn == "xban":
+                        hfs = ["c"] + hfs
+                        ocmd = ",".join(hfs + [cmd])
+
                     ncmds.append(ocmd)
                 vol.flags[hn] = ncmds
 
@@ -1448,6 +1614,10 @@ class AuthSrv(object):
             self.log(t, 1)
             errors = True
 
+        if self.args.smb and self.ah.on and acct:
+            self.log("--smb can only be used when --ah-alg is none", 1)
+            errors = True
+
         for vol in vfs.all_vols.values():
             for k in list(vol.flags.keys()):
                 if re.match("^-[^-]+$", k):
@@ -1460,6 +1630,7 @@ class AuthSrv(object):
         vfs.bubble_flags()
 
         have_e2d = False
+        have_e2t = False
         t = "volumes and permissions:\n"
         for zv in vfs.all_vols.values():
             if not self.warn_anonwrite:
@@ -1473,6 +1644,8 @@ class AuthSrv(object):
                 ["delete", "udel"],
                 ["   get", "uget"],
                 [" upget", "upget"],
+                ["  html", "uhtml"],
+                ["uadmin", "uadmin"],
             ]:
                 u = list(sorted(getattr(zv.axs, attr)))
                 u = ", ".join("\033[35meverybody\033[0m" if x == "*" else x for x in u)
@@ -1482,6 +1655,9 @@ class AuthSrv(object):
             if "e2d" in zv.flags:
                 have_e2d = True
 
+            if "e2t" in zv.flags:
+                have_e2t = True
+
             t += "\n"
 
         if self.warn_anonwrite:
@@ -1493,8 +1669,21 @@ class AuthSrv(object):
                 if t:
                     self.log("\n\033[{}\033[0m\n".format(t))
 
+                if not have_e2t:
+                    t = "hint: argument -e2ts enables multimedia indexing (artist/title/...)"
+                    self.log(t, 6)
+            else:
+                t = "hint: argument -e2dsa enables searching, upload-undo, and better deduplication"
+                self.log(t, 6)
+
+            zv, _ = vfs.get("/", "*", False, False)
+            zs = zv.realpath.lower()
+            if zs in ("/", "c:\\") or zs.startswith(r"c:\windows"):
+                t = "you are sharing a system directory: {}\n"
+                self.log(t.format(zv.realpath), c=1)
+
         try:
-            zv, _ = vfs.get("/", "*", False, True)
+            zv, _ = vfs.get("", "*", False, True, err=999)
             if self.warn_anonwrite and os.getcwd() == zv.realpath:
                 t = "anyone can write to the current directory: {}\n"
                 self.log(t.format(zv.realpath), c=1)
@@ -1511,7 +1700,51 @@ class AuthSrv(object):
             self.re_pwd = None
             pwds = [re.escape(x) for x in self.iacct.keys()]
             if pwds:
-                self.re_pwd = re.compile("=(" + "|".join(pwds) + ")([]&; ]|$)")
+                if self.ah.on:
+                    zs = r"(\[H\] pw:.*|[?&]pw=)([^&]+)"
+                else:
+                    zs = r"(\[H\] pw:.*|=)(" + "|".join(pwds) + r")([]&; ]|$)"
+
+                self.re_pwd = re.compile(zs)
+
+    def setup_pwhash(self, acct: dict[str, str]) -> None:
+        self.ah = PWHash(self.args)
+        if not self.ah.on:
+            return
+
+        if self.args.ah_cli:
+            self.ah.cli()
+            sys.exit()
+        elif self.args.ah_gen == "-":
+            self.ah.stdin()
+            sys.exit()
+        elif self.args.ah_gen:
+            print(self.ah.hash(self.args.ah_gen))
+            sys.exit()
+
+        if not acct:
+            return
+
+        changed = False
+        for uname, pw in list(acct.items())[:]:
+            if pw.startswith("+") and len(pw) == 33:
+                continue
+
+            changed = True
+            hpw = self.ah.hash(pw)
+            acct[uname] = hpw
+            t = "hashed password for account {}: {}"
+            self.log(t.format(uname, hpw), 3)
+
+        if not changed:
+            return
+
+        lns = []
+        for uname, pw in acct.items():
+            lns.append("  {}: {}".format(uname, pw))
+
+        t = "please use the following hashed passwords in your config:\n{}"
+        self.log(t.format("\n".join(lns)), 3)
 
     def chk_sqlite_threadsafe(self) -> str:
         v = SQLITE_VER[-1:]
@@ -1568,10 +1801,20 @@ class AuthSrv(object):
                 raise Exception("volume not found: " + zs)
 
         self.log(str({"users": users, "vols": vols, "flags": flags}))
-        t = "/{}: read({}) write({}) move({}) del({}) get({}) upget({})"
+        t = "/{}: read({}) write({}) move({}) del({}) get({}) upget({}) uadmin({})"
         for k, zv in self.vfs.all_vols.items():
             vc = zv.axs
-            vs = [k, vc.uread, vc.uwrite, vc.umove, vc.udel, vc.uget, vc.upget]
+            vs = [
+                k,
+                vc.uread,
+                vc.uwrite,
+                vc.umove,
+                vc.udel,
+                vc.uget,
+                vc.upget,
+                vc.uhtml,
+                vc.uadmin,
+            ]
             self.log(t.format(*vs))
 
         flag_v = "v" in flags
@@ -1651,7 +1894,8 @@ class AuthSrv(object):
         ]
 
         csv = set("i p".split())
-        lst = set("c ihead mtm mtp xad xar xau xiu xbd xbr xbu xm".split())
+        zs = "c ihead mtm mtp on403 on404 xad xar xau xiu xban xbd xbr xbu xm"
+        lst = set(zs.split())
         askip = set("a v c vc cgen theme".split())
 
         # keymap from argv to vflag
@@ -1710,6 +1954,8 @@ class AuthSrv(object):
                 "d": "udel",
                 "g": "uget",
                 "G": "upget",
+                "h": "uhtml",
+                "a": "uadmin",
             }
             users = {}
             for pkey in perms.values():
@@ -1801,13 +2047,19 @@ def expand_config_file(ret: list[str], fp: str, ipath: str) -> None:
 
     if os.path.isdir(fp):
         names = os.listdir(fp)
-        ret.append("#\033[36m cfg files in {} => {}\033[0m".format(fp, names))
+        crumb = "#\033[36m cfg files in {} => {}\033[0m".format(fp, names)
+        ret.append(crumb)
         for fn in sorted(names):
             fp2 = os.path.join(fp, fn)
             if not fp2.endswith(".conf") or fp2 in ipath:
                 continue
 
             expand_config_file(ret, fp2, ipath)
+
+        if ret[-1] == crumb:
+            # no config files below; remove breadcrumb
+            ret.pop()
+
         return
 
     ipath += " -> " + fp
@@ -1906,7 +2158,7 @@ def upgrade_cfg_fmt(
             else:
                 sn = sn.replace(",", ", ")
             ret.append("    " + sn)
-        elif sn[:1] in "rwmdgG":
+        elif sn[:1] in "rwmdgGha":
             if cat != catx:
                 cat = catx
                 ret.append(cat)

copyparty/broker_mp.py

@@ -9,7 +9,7 @@ import queue
 
 from .__init__ import CORES, TYPE_CHECKING
 from .broker_mpw import MpWorker
-from .broker_util import try_exec
+from .broker_util import ExceptionalQueue, try_exec
 from .util import Daemon, mp
 
 if TYPE_CHECKING:
@@ -107,6 +107,19 @@ class BrokerMp(object):
                 if retq_id:
                     proc.q_pend.put((retq_id, "retq", rv))
 
+    def ask(self, dest: str, *args: Any) -> ExceptionalQueue:
+
+        # new non-ipc invoking managed service in hub
+        obj = self.hub
+        for node in dest.split("."):
+            obj = getattr(obj, node)
+
+        rv = try_exec(True, obj, *args)
+
+        retq = ExceptionalQueue(1)
+        retq.put(rv)
+        return retq
+
     def say(self, dest: str, *args: Any) -> None:
         """
         send message to non-hub component in other process,

copyparty/cert.py

@@ -0,0 +1,226 @@
+import calendar
+import errno
+import filecmp
+import json
+import os
+import shutil
+import time
+
+from .util import Netdev, runcmd
+
+HAVE_CFSSL = True
+
+if True:  # pylint: disable=using-constant-test
+    from .util import RootLogger
+
+
+def ensure_cert(log: "RootLogger", args) -> None:
+    """
+    the default cert (and the entire TLS support) is only here to enable the
+    crypto.subtle javascript API, which is necessary due to the webkit guys
+    being massive memers (https://www.chromium.org/blink/webcrypto)
+
+    i feel awful about this and so should they
+    """
+    cert_insec = os.path.join(args.E.mod, "res/insecure.pem")
+    cert_appdata = os.path.join(args.E.cfg, "cert.pem")
+    if not os.path.isfile(args.cert):
+        if cert_appdata != args.cert:
+            raise Exception("certificate file does not exist: " + args.cert)
+
+        shutil.copy(cert_insec, args.cert)
+
+    with open(args.cert, "rb") as f:
+        buf = f.read()
+        o1 = buf.find(b" PRIVATE KEY-")
+        o2 = buf.find(b" CERTIFICATE-")
+        m = "unsupported certificate format: "
+        if o1 < 0:
+            raise Exception(m + "no private key inside pem")
+        if o2 < 0:
+            raise Exception(m + "no server certificate inside pem")
+        if o1 > o2:
+            raise Exception(m + "private key must appear before server certificate")
+
+    try:
+        if filecmp.cmp(args.cert, cert_insec):
+            t = "using default TLS certificate; https will be insecure:\033[36m {}"
+            log("cert", t.format(args.cert), 3)
+    except:
+        pass
+
+    # speaking of the default cert,
+    # printf 'NO\n.\n.\n.\n.\ncopyparty-insecure\n.\n' | faketime '2000-01-01 00:00:00' openssl req -x509 -sha256 -newkey rsa:2048 -keyout insecure.pem -out insecure.pem -days $((($(printf %d 0x7fffffff)-$(date +%s --date=2000-01-01T00:00:00Z))/(60*60*24))) -nodes && ls -al insecure.pem && openssl x509 -in insecure.pem -text -noout
+
+
+def _read_crt(args, fn):
+    try:
+        if not os.path.exists(os.path.join(args.crt_dir, fn)):
+            return 0, {}
+
+        acmd = ["cfssl-certinfo", "-cert", fn]
+        rc, so, se = runcmd(acmd, cwd=args.crt_dir)
+        if rc:
+            return 0, {}
+
+        inf = json.loads(so)
+        zs = inf["not_after"]
+        expiry = calendar.timegm(time.strptime(zs, "%Y-%m-%dT%H:%M:%SZ"))
+        return expiry, inf
+    except OSError as ex:
+        if ex.errno == errno.ENOENT:
+            raise
+        return 0, {}
+    except:
+        return 0, {}
+
+
+def _gen_ca(log: "RootLogger", args):
+    expiry = _read_crt(args, "ca.pem")[0]
+    if time.time() + args.crt_cdays * 60 * 60 * 24 * 0.1 < expiry:
+        return
+
+    backdate = "{}m".format(int(args.crt_back * 60))
+    expiry = "{}m".format(int(args.crt_cdays * 60 * 24))
+    cn = args.crt_cnc.replace("--crt-cn", args.crt_cn)
+    algo, ksz = args.crt_alg.split("-")
+    req = {
+        "CN": cn,
+        "CA": {"backdate": backdate, "expiry": expiry, "pathlen": 0},
+        "key": {"algo": algo, "size": int(ksz)},
+        "names": [{"O": cn}],
+    }
+    sin = json.dumps(req).encode("utf-8")
+    log("cert", "creating new ca ...", 6)
+
+    cmd = "cfssl gencert -initca -"
+    rc, so, se = runcmd(cmd.split(), 30, sin=sin)
+    if rc:
+        raise Exception("failed to create ca-cert: {}, {}".format(rc, se), 3)
+
+    cmd = "cfssljson -bare ca"
+    sin = so.encode("utf-8")
+    rc, so, se = runcmd(cmd.split(), 10, sin=sin, cwd=args.crt_dir)
+    if rc:
+        raise Exception("failed to translate ca-cert: {}, {}".format(rc, se), 3)
+
+    bname = os.path.join(args.crt_dir, "ca")
+    os.rename(bname + "-key.pem", bname + ".key")
+    os.unlink(bname + ".csr")
+
+    log("cert", "new ca OK", 2)
+
+
+def _gen_srv(log: "RootLogger", args, netdevs: dict[str, Netdev]):
+    names = args.crt_ns.split(",") if args.crt_ns else []
+    if not args.crt_exact:
+        for n in names[:]:
+            names.append("*.{}".format(n))
+    if not args.crt_noip:
+        for ip in netdevs.keys():
+            names.append(ip.split("/")[0])
+    if args.crt_nolo:
+        names = [x for x in names if x not in ("localhost", "127.0.0.1", "::1")]
+    if not args.crt_nohn:
+        names.append(args.name)
+        names.append(args.name + ".local")
+    if not names:
+        names = ["127.0.0.1"]
+    if "127.0.0.1" in names or "::1" in names:
+        names.append("localhost")
+    names = list({x: 1 for x in names}.keys())
+
+    try:
+        expiry, inf = _read_crt(args, "srv.pem")
+        expired = time.time() + args.crt_sdays * 60 * 60 * 24 * 0.1 > expiry
+        cert_insec = os.path.join(args.E.mod, "res/insecure.pem")
+        for n in names:
+            if n not in inf["sans"]:
+                raise Exception("does not have {}".format(n))
+        if expired:
+            raise Exception("old server-cert has expired")
+        if not filecmp.cmp(args.cert, cert_insec):
+            return
+    except Exception as ex:
+        log("cert", "will create new server-cert; {}".format(ex))
+
+    log("cert", "creating server-cert ...", 6)
+
+    backdate = "{}m".format(int(args.crt_back * 60))
+    expiry = "{}m".format(int(args.crt_sdays * 60 * 24))
+    cfg = {
+        "signing": {
+            "default": {
+                "backdate": backdate,
+                "expiry": expiry,
+                "usages": ["signing", "key encipherment", "server auth"],
+            }
+        }
+    }
+    with open(os.path.join(args.crt_dir, "cfssl.json"), "wb") as f:
+        f.write(json.dumps(cfg).encode("utf-8"))
+
+    cn = args.crt_cns.replace("--crt-cn", args.crt_cn)
+    algo, ksz = args.crt_alg.split("-")
+    req = {
+        "key": {"algo": algo, "size": int(ksz)},
+        "names": [{"O": cn}],
+    }
+    sin = json.dumps(req).encode("utf-8")
+
+    cmd = "cfssl gencert -config=cfssl.json -ca ca.pem -ca-key ca.key -profile=www"
+    acmd = cmd.split() + ["-hostname=" + ",".join(names), "-"]
+    rc, so, se = runcmd(acmd, 30, sin=sin, cwd=args.crt_dir)
+    if rc:
+        raise Exception("failed to create cert: {}, {}".format(rc, se))
+
+    cmd = "cfssljson -bare srv"
+    sin = so.encode("utf-8")
+    rc, so, se = runcmd(cmd.split(), 10, sin=sin, cwd=args.crt_dir)
+    if rc:
+        raise Exception("failed to translate cert: {}, {}".format(rc, se))
+
+    bname = os.path.join(args.crt_dir, "srv")
+    try:
+        os.unlink(bname + ".key")
+    except:
+        pass
+    os.rename(bname + "-key.pem", bname + ".key")
+    os.unlink(bname + ".csr")
+
+    with open(os.path.join(args.crt_dir, "ca.pem"), "rb") as f:
+        ca = f.read()
+
+    with open(bname + ".key", "rb") as f:
+        skey = f.read()
+
+    with open(bname + ".pem", "rb") as f:
+        scrt = f.read()
+
+    with open(args.cert, "wb") as f:
+        f.write(skey + scrt + ca)
+
+    log("cert", "new server-cert OK", 2)
+
+
+def gencert(log: "RootLogger", args, netdevs: dict[str, Netdev]):
+    global HAVE_CFSSL
+
+    if args.http_only:
+        return
+
+    if args.no_crt or not HAVE_CFSSL:
+        ensure_cert(log, args)
+        return
+
+    try:
+        _gen_ca(log, args)
+        _gen_srv(log, args, netdevs)
+    except Exception as ex:
+        HAVE_CFSSL = False
+        log("cert", "could not create TLS certificates: {}".format(ex), 3)
+        if getattr(ex, "errno", 0) == errno.ENOENT:
+            t = "install cfssl if you want to fix this; https://github.com/cloudflare/cfssl/releases/latest  (cfssl, cfssljson, cfssl-certinfo)"
+            log("cert", t, 6)
+
+        ensure_cert(log, args)

copyparty/cfg.py

@@ -13,6 +13,9 @@ def vf_bmap() -> dict[str, str]:
         "no_dedup": "copydupes",
         "no_dupe": "nodupe",
         "no_forget": "noforget",
+        "th_no_crop": "nocrop",
+        "dav_auth": "davauth",
+        "dav_rt": "davrt",
     }
     for k in (
         "dotsrch",
@@ -22,6 +25,7 @@ def vf_bmap() -> dict[str, str]:
         "e2v",
         "e2vu",
         "e2vp",
+        "grid",
         "hardlink",
         "magic",
         "no_sb_md",
@@ -37,8 +41,8 @@ def vf_bmap() -> dict[str, str]:
 
 def vf_vmap() -> dict[str, str]:
     """argv-to-volflag: simple values"""
-    ret = {}
-    for k in ("lg_sbf", "md_sbf"):
+    ret = {"th_convt": "convt", "th_size": "thsize"}
+    for k in ("dbd", "lg_sbf", "md_sbf", "nrand", "unlist"):
         ret[k] = k
     return ret
 
@@ -46,7 +50,7 @@ def vf_vmap() -> dict[str, str]:
 def vf_cmap() -> dict[str, str]:
     """argv-to-volflag: complex/lists"""
     ret = {}
-    for k in ("dbd", "html_head", "mte", "mth", "nrand"):
+    for k in ("html_head", "mte", "mth"):
         ret[k] = k
     return ret
 
@@ -58,6 +62,8 @@ permdescs = {
     "d": "delete; permanently delete files and folders",
     "g": "get; download files, but cannot see folder contents",
     "G": 'upget; same as "g" but can see filekeys of their own uploads',
+    "h": 'html; same as "g" but folders return their index.html',
+    "a": "admin; can see uploader IPs, config-reload",
 }
 
 
@@ -75,7 +81,9 @@ flagcats = {
     },
     "upload rules": {
         "maxn=250,600": "max 250 uploads over 15min",
-        "maxb=1g,300": "max 1 GiB over 5min (suffixes: b, k, m, g)",
+        "maxb=1g,300": "max 1 GiB over 5min (suffixes: b, k, m, g, t)",
+        "vmaxb=1g": "total volume size max 1 GiB (suffixes: b, k, m, g, t)",
+        "vmaxn=4k": "max 4096 files in volume (suffixes: b, k, m, g, t)",
         "rand": "force randomized filenames, 9 chars long by default",
         "nrand=N": "randomized filenames are N chars long",
         "sz=1k-3m": "allow filesizes between 1 KiB and 3MiB",
@@ -103,10 +111,11 @@ flagcats = {
         "nohash=\\.iso$": "skips hashing file contents if path matches *.iso",
         "noidx=\\.iso$": "fully ignores the contents at paths matching *.iso",
         "noforget": "don't forget files when deleted from disk",
+        "fat32": "avoid excessive reindexing on android sdcardfs",
         "dbd=[acid|swal|wal|yolo]": "database speed-durability tradeoff",
         "xlink": "cross-volume dupe detection / linking",
         "xdev": "do not descend into other filesystems",
-        "xvol": "skip symlinks leaving the volume root",
+        "xvol": "do not follow symlinks leaving the volume root",
         "dotsrch": "show dotfiles in search results",
         "nodotsrch": "hide dotfiles in search results (default)",
     },
@@ -119,6 +128,13 @@ flagcats = {
         "dvthumb": "disables video thumbnails",
         "dathumb": "disables audio thumbnails (spectrograms)",
         "dithumb": "disables image thumbnails",
+        "thsize": "thumbnail res; WxH",
+        "nocrop": "disable center-cropping",
+        "convt": "conversion timeout in seconds",
+    },
+    "handlers\n(better explained in --help-handlers)": {
+        "on404=PY": "handle 404s by executing PY file",
+        "on403=PY": "handle 403s by executing PY file",
     },
     "event hooks\n(better explained in --help-hooks)": {
         "xbu=CMD": "execute CMD before a file upload starts",
@@ -129,8 +145,11 @@ flagcats = {
         "xbd=CMD": "execute CMD before a file delete",
         "xad=CMD": "execute CMD after  a file delete",
         "xm=CMD": "execute CMD on message",
+        "xban=CMD": "execute CMD if someone gets banned",
     },
     "client and ux": {
+        "grid": "show grid/thumbnails by default",
+        "unlist": "dont list files matching REGEX",
         "html_head=TXT": "includes TXT in the <head>",
         "robots": "allows indexing by search engines (default)",
         "norobots": "kindly asks search engines to leave",
@@ -140,9 +159,12 @@ flagcats = {
         "sb_lg": "enable js sandbox for prologue/epilogue (default)",
         "md_sbf": "list of markdown-sandbox safeguards to disable",
         "lg_sbf": "list of *logue-sandbox safeguards to disable",
+        "nohtml": "return html and markdown as text/html",
     },
     "others": {
-        "fk=8": 'generates per-file accesskeys,\nwhich will then be required at the "g" permission'
+        "fk=8": 'generates per-file accesskeys,\nwhich will then be required at the "g" permission',
+        "davauth": "ask webdav clients to login for all folders",
+        "davrt": "show lastmod time of symlink destination, not the link itself\n(note: this option is always enabled for recursive listings)",
     },
 }
 

copyparty/ftpd.py

@@ -2,18 +2,26 @@
 from __future__ import print_function, unicode_literals
 
 import argparse
+import errno
 import logging
 import os
 import stat
 import sys
 import time
 
+from .__init__ import ANYWIN, PY2, TYPE_CHECKING, E
+
+try:
+    import asynchat
+except:
+    sys.path.append(os.path.join(E.mod, "vend"))
+
 from pyftpdlib.authorizers import AuthenticationFailed, DummyAuthorizer
 from pyftpdlib.filesystems import AbstractedFS, FilesystemError
 from pyftpdlib.handlers import FTPHandler
+from pyftpdlib.ioloop import IOLoop
 from pyftpdlib.servers import FTPServer
 
-from .__init__ import ANYWIN, PY2, TYPE_CHECKING, E
 from .authsrv import VFS
 from .bos import bos
 from .util import (
@@ -29,15 +37,6 @@ from .util import (
     vjoin,
 )
 
-try:
-    from pyftpdlib.ioloop import IOLoop
-except ImportError:
-    p = os.path.join(E.mod, "vend")
-    print("loading asynchat from " + p)
-    sys.path.append(p)
-    from pyftpdlib.ioloop import IOLoop
-
-
 if TYPE_CHECKING:
     from .svchub import SvcHub
 
@@ -46,6 +45,12 @@ if True:  # pylint: disable=using-constant-test
     from typing import Any, Optional
 
 
+class FSE(FilesystemError):
+    def __init__(self, msg: str, severity: int = 0) -> None:
+        super(FilesystemError, self).__init__(msg)
+        self.severity = severity
+
+
 class FtpAuth(DummyAuthorizer):
     def __init__(self, hub: "SvcHub") -> None:
         super(FtpAuth, self).__init__()
@@ -72,10 +77,13 @@ class FtpAuth(DummyAuthorizer):
                 raise AuthenticationFailed("banned")
 
         asrv = self.hub.asrv
-        if username == "anonymous":
-            uname = "*"
-        else:
-            uname = asrv.iacct.get(password, "") or asrv.iacct.get(username, "") or "*"
+        uname = "*"
+        if username != "anonymous":
+            for zs in (password, username):
+                zs = asrv.iacct.get(asrv.ah.hash(zs), "")
+                if zs:
+                    uname = zs
+                    break
 
         if not uname or not (asrv.vfs.aread.get(uname) or asrv.vfs.awrite.get(uname)):
             g = self.hub.gpwd
@@ -87,7 +95,7 @@ class FtpAuth(DummyAuthorizer):
 
             raise AuthenticationFailed("Authentication failed.")
 
-        handler.uname = uname
+        handler.uname = handler.username = uname
 
     def get_home_dir(self, username: str) -> str:
         return "/"
@@ -113,7 +121,8 @@ class FtpFs(AbstractedFS):
     def __init__(
         self, root: str, cmd_channel: Any
     ) -> None:  # pylint: disable=super-init-not-called
-        self.h = self.cmd_channel = cmd_channel  # type: FTPHandler
+        self.h = cmd_channel  # type: FTPHandler
+        self.cmd_channel = cmd_channel  # type: FTPHandler
         self.hub: "SvcHub" = cmd_channel.hub
         self.args = cmd_channel.args
         self.uname = cmd_channel.uname
@@ -123,14 +132,11 @@ class FtpFs(AbstractedFS):
 
         self.can_read = self.can_write = self.can_move = False
         self.can_delete = self.can_get = self.can_upget = False
+        self.can_admin = False
 
         self.listdirinfo = self.listdir
         self.chdir(".")
 
-    def die(self, msg):
-        self.h.die(msg)
-        raise Exception()
-
     def v2a(
         self,
         vpath: str,
@@ -140,21 +146,34 @@ class FtpFs(AbstractedFS):
         d: bool = False,
     ) -> tuple[str, VFS, str]:
         try:
-            vpath = vpath.replace("\\", "/").lstrip("/")
+            vpath = vpath.replace("\\", "/").strip("/")
             rd, fn = os.path.split(vpath)
             if ANYWIN and relchk(rd):
                 logging.warning("malicious vpath: %s", vpath)
-                self.die("Unsupported characters in filepath")
+                t = "Unsupported characters in [{}]"
+                raise FSE(t.format(vpath), 1)
 
             fn = sanitize_fn(fn or "", "", [".prologue.html", ".epilogue.html"])
             vpath = vjoin(rd, fn)
             vfs, rem = self.hub.asrv.vfs.get(vpath, self.uname, r, w, m, d)
             if not vfs.realpath:
-                self.die("No filesystem mounted at this path")
+                t = "No filesystem mounted at [{}]"
+                raise FSE(t.format(vpath))
+
+            if "xdev" in vfs.flags or "xvol" in vfs.flags:
+                ap = vfs.canonical(rem)
+                avfs = vfs.chk_ap(ap)
+                t = "Permission denied in [{}]"
+                if not avfs:
+                    raise FSE(t.format(vpath), 1)
+
+                cr, cw, cm, cd, _, _, _ = avfs.can_access("", self.h.uname)
+                if r and not cr or w and not cw or m and not cm or d and not cd:
+                    raise FSE(t.format(vpath), 1)
 
             return os.path.join(vfs.realpath, rem), vfs, rem
         except Pebkac as ex:
-            self.die(str(ex))
+            raise FSE(str(ex))
 
     def rv2a(
         self,
@@ -177,7 +196,7 @@ class FtpFs(AbstractedFS):
     def validpath(self, path: str) -> bool:
         if "/.hist/" in path:
             if "/up2k." in path or path.endswith("/dir.txt"):
-                self.die("Access to this file is forbidden")
+                raise FSE("Access to this file is forbidden", 1)
 
         return True
 
@@ -194,7 +213,7 @@ class FtpFs(AbstractedFS):
                 td = 0
 
             if td < -1 or td > self.args.ftp_wt:
-                self.die("Cannot open existing file for writing")
+                raise FSE("Cannot open existing file for writing")
 
         self.validpath(ap)
         return open(fsenc(ap), mode)
@@ -203,9 +222,17 @@ class FtpFs(AbstractedFS):
         nwd = join(self.cwd, path)
         vfs, rem = self.hub.asrv.vfs.get(nwd, self.uname, False, False)
         ap = vfs.canonical(rem)
-        if not bos.path.isdir(ap):
+        try:
+            st = bos.stat(ap)
+            if not stat.S_ISDIR(st.st_mode):
+                raise Exception()
+        except:
             # returning 550 is library-default and suitable
-            self.die("Failed to change directory")
+            raise FSE("No such file or directory")
+
+        avfs = vfs.chk_ap(ap, st)
+        if not avfs:
+            raise FSE("Permission denied", 1)
 
         self.cwd = nwd
         (
@@ -215,16 +242,19 @@ class FtpFs(AbstractedFS):
             self.can_delete,
             self.can_get,
             self.can_upget,
-        ) = self.hub.asrv.vfs.can_access(self.cwd.lstrip("/"), self.h.uname)
+            self.can_admin,
+        ) = avfs.can_access("", self.h.uname)
 
     def mkdir(self, path: str) -> None:
         ap = self.rv2a(path, w=True)[0]
         bos.makedirs(ap)  # filezilla expects this
 
     def listdir(self, path: str) -> list[str]:
-        vpath = join(self.cwd, path).lstrip("/")
+        vpath = join(self.cwd, path)
         try:
-            vfs, rem = self.hub.asrv.vfs.get(vpath, self.uname, True, False)
+            ap, vfs, rem = self.v2a(vpath, True, False)
+            if not bos.path.isdir(ap):
+                raise FSE("No such file or directory", 1)
 
             fsroot, vfs_ls1, vfs_virt = vfs.ls(
                 rem,
@@ -240,8 +270,12 @@ class FtpFs(AbstractedFS):
 
             vfs_ls.sort()
             return vfs_ls
-        except:
-            if vpath:
+        except Exception as ex:
+            # panic on malicious names
+            if getattr(ex, "severity", 0):
+                raise
+
+            if vpath.strip("/"):
                 # display write-only folders as empty
                 return []
 
@@ -251,31 +285,35 @@ class FtpFs(AbstractedFS):
 
     def rmdir(self, path: str) -> None:
         ap = self.rv2a(path, d=True)[0]
-        bos.rmdir(ap)
+        try:
+            bos.rmdir(ap)
+        except OSError as e:
+            if e.errno != errno.ENOENT:
+                raise
 
     def remove(self, path: str) -> None:
         if self.args.no_del:
-            self.die("The delete feature is disabled in server config")
+            raise FSE("The delete feature is disabled in server config")
 
         vp = join(self.cwd, path).lstrip("/")
         try:
-            self.hub.up2k.handle_rm(self.uname, self.h.cli_ip, [vp], [])
+            self.hub.up2k.handle_rm(self.uname, self.h.cli_ip, [vp], [], False)
         except Exception as ex:
-            self.die(str(ex))
+            raise FSE(str(ex))
 
     def rename(self, src: str, dst: str) -> None:
         if not self.can_move:
-            self.die("Not allowed for user " + self.h.uname)
+            raise FSE("Not allowed for user " + self.h.uname)
 
         if self.args.no_mv:
-            self.die("The rename/move feature is disabled in server config")
+            raise FSE("The rename/move feature is disabled in server config")
 
         svp = join(self.cwd, src).lstrip("/")
         dvp = join(self.cwd, dst).lstrip("/")
         try:
             self.hub.up2k.handle_mv(self.uname, svp, dvp)
         except Exception as ex:
-            self.die(str(ex))
+            raise FSE(str(ex))
 
     def chmod(self, path: str, mode: str) -> None:
         pass
@@ -284,7 +322,10 @@ class FtpFs(AbstractedFS):
         try:
             ap = self.rv2a(path, r=True)[0]
             return bos.stat(ap)
-        except:
+        except FSE as ex:
+            if ex.severity:
+                raise
+
             ap = self.rv2a(path)[0]
             st = bos.stat(ap)
             if not stat.S_ISDIR(st.st_mode):
@@ -304,7 +345,10 @@ class FtpFs(AbstractedFS):
         try:
             st = self.stat(path)
             return stat.S_ISREG(st.st_mode)
-        except:
+        except Exception as ex:
+            if getattr(ex, "severity", 0):
+                raise
+
             return False  # expected for mojibake in ftp_SIZE()
 
     def islink(self, path: str) -> bool:
@@ -315,7 +359,10 @@ class FtpFs(AbstractedFS):
         try:
             st = self.stat(path)
             return stat.S_ISDIR(st.st_mode)
-        except:
+        except Exception as ex:
+            if getattr(ex, "severity", 0):
+                raise
+
             return True
 
     def getsize(self, path: str) -> int:
@@ -365,14 +412,10 @@ class FtpHandler(FTPHandler):
         # reduce non-debug logging
         self.log_cmds_list = [x for x in self.log_cmds_list if x not in ("CWD", "XCWD")]
 
-    def die(self, msg):
-        self.respond("550 {}".format(msg))
-        raise FilesystemError(msg)
-
     def ftp_STOR(self, file: str, mode: str = "w") -> Any:
         # Optional[str]
         vp = join(self.fs.cwd, file).lstrip("/")
-        ap, vfs, rem = self.fs.v2a(vp)
+        ap, vfs, rem = self.fs.v2a(vp, w=True)
         self.vfs_map[ap] = vp
         xbu = vfs.flags.get("xbu")
         if xbu and not runhook(
@@ -388,7 +431,7 @@ class FtpHandler(FTPHandler):
             0,
             "",
         ):
-            self.die("Upload blocked by xbu server config")
+            raise FSE("Upload blocked by xbu server config")
 
         # print("ftp_STOR: {} {} => {}".format(vp, mode, ap))
         ret = FTPHandler.ftp_STOR(self, file, mode)
@@ -462,9 +505,9 @@ class Ftpd(object):
 
         for h_lp in hs:
             h2, lp = h_lp
-            h2.hub = hub
-            h2.args = hub.args
-            h2.authorizer = FtpAuth(hub)
+            FtpHandler.hub = h2.hub = hub
+            FtpHandler.args = h2.args = hub.args
+            FtpHandler.authorizer = h2.authorizer = FtpAuth(hub)
 
             if self.args.ftp_pr:
                 p1, p2 = [int(x) for x in self.args.ftp_pr.split("-")]
@@ -488,6 +531,9 @@ class Ftpd(object):
         if "::" in ips:
             ips.append("0.0.0.0")
 
+        if self.args.ftp4:
+            ips = [x for x in ips if ":" not in x]
+
         ioloop = IOLoop()
         for ip in ips:
             for h, lp in hs:

copyparty/httpconn.py

@@ -54,7 +54,6 @@ class HttpConn(object):
         self.args: argparse.Namespace = hsrv.args  # mypy404
         self.E: EnvParams = self.args.E
         self.asrv: AuthSrv = hsrv.asrv  # mypy404
-        self.cert_path = hsrv.cert_path
         self.u2fh: Util.FHC = hsrv.u2fh  # mypy404
         self.iphash: HMaccas = hsrv.broker.iphash
         self.bans: dict[str, int] = hsrv.bans
@@ -114,7 +113,7 @@ class HttpConn(object):
 
     def _detect_https(self) -> bool:
         method = None
-        if self.cert_path:
+        if True:
             try:
                 method = self.s.recv(4, socket.MSG_PEEK)
             except socket.timeout:
@@ -148,7 +147,7 @@ class HttpConn(object):
         self.sr = None
         if self.args.https_only:
             is_https = True
-        elif self.args.http_only or not HAVE_SSL:
+        elif self.args.http_only:
             is_https = False
         else:
             # raise Exception("asdf")
@@ -162,7 +161,7 @@ class HttpConn(object):
             self.log_src = self.log_src.replace("[36m", "[35m")
             try:
                 ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
-                ctx.load_cert_chain(self.cert_path)
+                ctx.load_cert_chain(self.args.cert)
                 if self.args.ssl_ver:
                     ctx.options &= ~self.args.ssl_flags_en
                     ctx.options |= self.args.ssl_flags_de

copyparty/httpsrv.py

@@ -4,6 +4,7 @@ from __future__ import print_function, unicode_literals
 import base64
 import math
 import os
+import re
 import socket
 import sys
 import threading
@@ -33,13 +34,29 @@ except MNFE:
    * (try another python version, if you have one)
    * (try copyparty.sfx instead)
 """.format(
-            os.path.basename(sys.executable)
+            sys.executable
+        )
+    )
+    sys.exit(1)
+except SyntaxError:
+    if EXE:
+        raise
+
+    print(
+        """\033[1;31m
+  your jinja2 version is incompatible with your python version;\033[33m
+  please try to replace it with an older version:\033[0m
+   * {} -m pip install --user jinja2==2.11.3
+   * (try another python version, if you have one)
+   * (try copyparty.sfx instead)
+""".format(
+            sys.executable
         )
     )
     sys.exit(1)
 
-from .bos import bos
 from .httpconn import HttpConn
+from .metrics import Metrics
 from .u2idx import U2idx
 from .util import (
     E_SCK,
@@ -49,6 +66,7 @@ from .util import (
     Magician,
     Netdev,
     NetMap,
+    absreal,
     ipnorm,
     min_ex,
     shut_socket,
@@ -82,12 +100,16 @@ class HttpSrv(object):
         # redefine in case of multiprocessing
         socket.setdefaulttimeout(120)
 
+        self.t0 = time.time()
         nsuf = "-n{}-i{:x}".format(nid, os.getpid()) if nid else ""
         self.magician = Magician()
         self.nm = NetMap([], {})
         self.ssdp: Optional["SSDPr"] = None
         self.gpwd = Garda(self.args.ban_pw)
         self.g404 = Garda(self.args.ban_404)
+        self.g403 = Garda(self.args.ban_403)
+        self.g422 = Garda(self.args.ban_422, False)
+        self.gurl = Garda(self.args.ban_url)
         self.bans: dict[str, int] = {}
         self.aclose: dict[str, int] = {}
 
@@ -105,6 +127,7 @@ class HttpSrv(object):
         self.t_periodic: Optional[threading.Thread] = None
 
         self.u2fh = FHC()
+        self.metrics = Metrics(self)
         self.srvs: list[socket.socket] = []
         self.ncli = 0  # exact
         self.clients: set[HttpConn] = set()  # laggy
@@ -122,6 +145,11 @@ class HttpSrv(object):
         zs = os.path.join(self.E.mod, "web", "deps", "prism.js.gz")
         self.prism = os.path.exists(zs)
 
+        self.statics: set[str] = set()
+        self._build_statics()
+
+        self.ptn_cc = re.compile(r"[\x00-\x1f]")
+
         self.mallow = "GET HEAD POST PUT DELETE OPTIONS".split()
         if not self.args.no_dav:
             zs = "PROPFIND PROPPATCH LOCK UNLOCK MKCOL COPY MOVE"
@@ -132,12 +160,6 @@ class HttpSrv(object):
 
             self.ssdp = SSDPr(broker)
 
-        cert_path = self.args.cert
-        if bos.path.exists(cert_path):
-            self.cert_path = cert_path
-        else:
-            self.cert_path = ""
-
         if self.tp_q:
             self.start_threads(4)
 
@@ -158,6 +180,14 @@ class HttpSrv(object):
         except:
             pass
 
+    def _build_statics(self) -> None:
+        for dp, _, df in os.walk(os.path.join(self.E.mod, "web")):
+            for fn in df:
+                ap = absreal(os.path.join(dp, fn))
+                self.statics.add(ap)
+                if ap.endswith(".gz") or ap.endswith(".br"):
+                    self.statics.add(ap[:-3])
+
     def set_netdevs(self, netdevs: dict[str, Netdev]) -> None:
         ips = set()
         for ip, _ in self.bound:

copyparty/ico.py

@@ -17,7 +17,9 @@ class Ico(object):
     def get(self, ext: str, as_thumb: bool, chrome: bool) -> tuple[str, bytes]:
         """placeholder to make thumbnails not break"""
 
-        zb = hashlib.sha1(ext.encode("utf-8")).digest()[2:4]
+        bext = ext.encode("ascii", "replace")
+        ext = bext.decode("utf-8")
+        zb = hashlib.sha1(bext).digest()[2:4]
         if PY2:
             zb = [ord(x) for x in zb]
 
@@ -33,7 +35,7 @@ class Ico(object):
             h = int(100 / (float(sw) / float(sh)))
             w = 100
 
-        if chrome and as_thumb:
+        if chrome:
             # cannot handle more than ~2000 unique SVGs
             if HAVE_PIL:
                 # svg: 3s, cache: 6s, this: 8s
@@ -43,8 +45,19 @@ class Ico(object):
                 w = 64
                 img = Image.new("RGB", (w, h), "#" + c[:6])
                 pb = ImageDraw.Draw(img)
-                tw, th = pb.textsize(ext)
-                pb.text(((w - tw) // 2, (h - th) // 2), ext, fill="#" + c[6:])
+                try:
+                    _, _, tw, th = pb.textbbox((0, 0), ext)
+                except:
+                    tw, th = pb.textsize(ext)
+
+                tw += len(ext)
+                cw = tw // len(ext)
+                x = ((w - tw) // 2) - (cw * 2) // 3
+                fill = "#" + c[6:]
+                for ch in ext:
+                    pb.text((x, (h - th) // 2), " %s " % (ch,), fill=fill)
+                    x += cw
+
                 img = img.resize((w * 3, h * 3), Image.NEAREST)
 
                 buf = BytesIO()

copyparty/mdns.py

@@ -1,6 +1,7 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
+import errno
 import random
 import select
 import socket
@@ -277,12 +278,26 @@ class MDNS(MCast):
         zf = time.time() + 2
         self.probing = zf  # cant unicast so give everyone an extra sec
         self.unsolicited = [zf, zf + 1, zf + 3, zf + 7]  # rfc-8.3
+
+        try:
+            self.run2()
+        except OSError as ex:
+            if ex.errno != errno.EBADF:
+                raise
+
+            self.log("stopping due to {}".format(ex), "90")
+
+        self.log("stopped", 2)
+
+    def run2(self) -> None:
         last_hop = time.time()
         ihop = self.args.mc_hop
         while self.running:
             timeout = (
                 0.02 + random.random() * 0.07
-                if self.probing or self.q or self.defend or self.unsolicited
+                if self.probing or self.q or self.defend
+                else max(0.05, self.unsolicited[0] - time.time())
+                if self.unsolicited
                 else (last_hop + ihop if ihop else 180)
             )
             rdy = select.select(self.srv, [], [], timeout)
@@ -314,8 +329,6 @@ class MDNS(MCast):
                 self.log(t.format(self.hn[:-1]), 2)
                 self.probing = 0
 
-        self.log("stopped", 2)
-
     def stop(self, panic=False) -> None:
         self.running = False
         for srv in self.srv.values():
@@ -502,6 +515,10 @@ class MDNS(MCast):
             for srv in self.srv.values():
                 tx.add(srv)
 
+            if not self.unsolicited and self.args.zm_spam:
+                zf = time.time() + self.args.zm_spam + random.random() * 0.07
+                self.unsolicited.append(zf)
+
         for srv, deadline in list(self.defend.items()):
             if now < deadline:
                 continue

copyparty/metrics.py

@@ -0,0 +1,165 @@
+# coding: utf-8
+from __future__ import print_function, unicode_literals
+
+import json
+import time
+
+from .__init__ import TYPE_CHECKING
+from .util import Pebkac, get_df, unhumanize
+
+if TYPE_CHECKING:
+    from .httpcli import HttpCli
+    from .httpsrv import HttpSrv
+
+
+class Metrics(object):
+    def __init__(self, hsrv: "HttpSrv") -> None:
+        self.hsrv = hsrv
+
+    def tx(self, cli: "HttpCli") -> bool:
+        if not cli.avol:
+            raise Pebkac(403, "not allowed for user " + cli.uname)
+
+        args = cli.args
+        if not args.stats:
+            raise Pebkac(403, "the stats feature is not enabled in server config")
+
+        conn = cli.conn
+        vfs = conn.asrv.vfs
+        allvols = list(sorted(vfs.all_vols.items()))
+
+        idx = conn.get_u2idx()
+        if not idx or not hasattr(idx, "p_end"):
+            idx = None
+
+        ret: list[str] = []
+
+        def addc(k: str, unit: str, v: str, desc: str) -> None:
+            if unit:
+                k += "_" + unit
+                zs = "# TYPE %s counter\n# UNIT %s %s\n# HELP %s %s\n%s_created %s\n%s_total %s"
+                ret.append(zs % (k, k, unit, k, desc, k, int(self.hsrv.t0), k, v))
+            else:
+                zs = "# TYPE %s counter\n# HELP %s %s\n%s_created %s\n%s_total %s"
+                ret.append(zs % (k, k, desc, k, int(self.hsrv.t0), k, v))
+
+        def addh(k: str, typ: str, desc: str) -> None:
+            zs = "# TYPE %s %s\n# HELP %s %s"
+            ret.append(zs % (k, typ, k, desc))
+
+        def addbh(k: str, desc: str) -> None:
+            zs = "# TYPE %s gauge\n# UNIT %s bytes\n# HELP %s %s"
+            ret.append(zs % (k, k, k, desc))
+
+        def addv(k: str, v: str) -> None:
+            ret.append("%s %s" % (k, v))
+
+        v = "{:.3f}".format(time.time() - self.hsrv.t0)
+        addc("cpp_uptime", "seconds", v, "time since last server restart")
+
+        v = str(len(conn.bans or []))
+        addc("cpp_bans", "", v, "number of banned IPs")
+
+        if not args.nos_hdd:
+            addbh("cpp_disk_size_bytes", "total HDD size of volume")
+            addbh("cpp_disk_free_bytes", "free HDD space in volume")
+            for vpath, vol in allvols:
+                free, total = get_df(vol.realpath)
+                addv('cpp_disk_size_bytes{vol="/%s"}' % (vpath), str(total))
+                addv('cpp_disk_free_bytes{vol="/%s"}' % (vpath), str(free))
+
+        if idx and not args.nos_vol:
+            addbh("cpp_vol_bytes", "num bytes of data in volume")
+            addh("cpp_vol_files", "gauge", "num files in volume")
+            addbh("cpp_vol_free_bytes", "free space (vmaxb) in volume")
+            addh("cpp_vol_free_files", "gauge", "free space (vmaxn) in volume")
+            tnbytes = 0
+            tnfiles = 0
+
+            volsizes = []
+            try:
+                ptops = [x.realpath for _, x in allvols]
+                x = self.hsrv.broker.ask("up2k.get_volsizes", ptops)
+                volsizes = x.get()
+            except Exception as ex:
+                cli.log("tx_stats get_volsizes: {!r}".format(ex), 3)
+
+            for (vpath, vol), (nbytes, nfiles) in zip(allvols, volsizes):
+                tnbytes += nbytes
+                tnfiles += nfiles
+                addv('cpp_vol_bytes{vol="/%s"}' % (vpath), str(nbytes))
+                addv('cpp_vol_files{vol="/%s"}' % (vpath), str(nfiles))
+
+                if vol.flags.get("vmaxb") or vol.flags.get("vmaxn"):
+
+                    zi = unhumanize(vol.flags.get("vmaxb") or "0")
+                    if zi:
+                        v = str(zi - nbytes)
+                        addv('cpp_vol_free_bytes{vol="/%s"}' % (vpath), v)
+
+                    zi = unhumanize(vol.flags.get("vmaxn") or "0")
+                    if zi:
+                        v = str(zi - nfiles)
+                        addv('cpp_vol_free_files{vol="/%s"}' % (vpath), v)
+
+            if volsizes:
+                addv('cpp_vol_bytes{vol="total"}', str(tnbytes))
+                addv('cpp_vol_files{vol="total"}', str(tnfiles))
+
+        if idx and not args.nos_dup:
+            addbh("cpp_dupe_bytes", "num dupe bytes in volume")
+            addh("cpp_dupe_files", "gauge", "num dupe files in volume")
+            tnbytes = 0
+            tnfiles = 0
+            for vpath, vol in allvols:
+                cur = idx.get_cur(vol.realpath)
+                if not cur:
+                    continue
+
+                nbytes = 0
+                nfiles = 0
+                q = "select sz, count(*)-1 c from up group by w having c"
+                for sz, c in cur.execute(q):
+                    nbytes += sz * c
+                    nfiles += c
+
+                tnbytes += nbytes
+                tnfiles += nfiles
+                addv('cpp_dupe_bytes{vol="/%s"}' % (vpath), str(nbytes))
+                addv('cpp_dupe_files{vol="/%s"}' % (vpath), str(nfiles))
+
+            addv('cpp_dupe_bytes{vol="total"}', str(tnbytes))
+            addv('cpp_dupe_files{vol="total"}', str(tnfiles))
+
+        if not args.nos_unf:
+            addbh("cpp_unf_bytes", "incoming/unfinished uploads (num bytes)")
+            addh("cpp_unf_files", "gauge", "incoming/unfinished uploads (num files)")
+            tnbytes = 0
+            tnfiles = 0
+            try:
+                x = self.hsrv.broker.ask("up2k.get_unfinished")
+                xs = x.get()
+                xj = json.loads(xs)
+                for ptop, (nbytes, nfiles) in xj.items():
+                    tnbytes += nbytes
+                    tnfiles += nfiles
+                    vol = next((x[1] for x in allvols if x[1].realpath == ptop), None)
+                    if not vol:
+                        t = "tx_stats get_unfinished: could not map {}"
+                        cli.log(t.format(ptop), 3)
+                        continue
+
+                    addv('cpp_unf_bytes{vol="/%s"}' % (vol.vpath), str(nbytes))
+                    addv('cpp_unf_files{vol="/%s"}' % (vol.vpath), str(nfiles))
+
+                addv('cpp_unf_bytes{vol="total"}', str(tnbytes))
+                addv('cpp_unf_files{vol="total"}', str(tnfiles))
+
+            except Exception as ex:
+                cli.log("tx_stats get_unfinished: {!r}".format(ex), 3)
+
+        ret.append("# EOF")
+
+        mime = "application/openmetrics-text; version=1.0.0; charset=utf-8"
+        cli.reply("\n".join(ret).encode("utf-8"), mime=mime)
+        return True

copyparty/multicast.py

@@ -15,7 +15,7 @@ from ipaddress import (
 )
 
 from .__init__ import MACOS, TYPE_CHECKING
-from .util import Netdev, find_prefix, min_ex, spack
+from .util import Daemon, Netdev, find_prefix, min_ex, spack
 
 if TYPE_CHECKING:
     from .svchub import SvcHub
@@ -228,6 +228,7 @@ class MCast(object):
         for srv in self.srv.values():
             assert srv.ip in self.sips
 
+        Daemon(self.hopper, "mc-hop")
         return bound
 
     def setup_socket(self, srv: MC_Sck) -> None:
@@ -299,33 +300,57 @@ class MCast(object):
                 t = "failed to set IPv4 TTL/LOOP; announcements may not survive multiple switches/routers"
                 self.log(t, 3)
 
-        self.hop(srv)
+        if self.hop(srv, False):
+            self.log("igmp was already joined?? chilling for a sec", 3)
+            time.sleep(1.2)
+
+        self.hop(srv, True)
         self.b4.sort(reverse=True)
         self.b6.sort(reverse=True)
 
-    def hop(self, srv: MC_Sck) -> None:
+    def hop(self, srv: MC_Sck, on: bool) -> bool:
         """rejoin to keepalive on routers/switches without igmp-snooping"""
         sck = srv.sck
         req = srv.mreq
         if ":" in srv.ip:
-            try:
-                sck.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_LEAVE_GROUP, req)
-                # linux does leaves/joins twice with 0.2~1.05s spacing
-                time.sleep(1.2)
-            except:
-                pass
-
-            sck.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_JOIN_GROUP, req)
+            if not on:
+                try:
+                    sck.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_LEAVE_GROUP, req)
+                    return True
+                except:
+                    return False
+            else:
+                sck.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_JOIN_GROUP, req)
         else:
-            try:
-                sck.setsockopt(socket.IPPROTO_IP, socket.IP_DROP_MEMBERSHIP, req)
-                time.sleep(1.2)
-            except:
-                pass
+            if not on:
+                try:
+                    sck.setsockopt(socket.IPPROTO_IP, socket.IP_DROP_MEMBERSHIP, req)
+                    return True
+                except:
+                    return False
+            else:
+                # t = "joining {} from ip {} idx {} with mreq {}"
+                # self.log(t.format(srv.grp, srv.ip, srv.idx, repr(srv.mreq)), 6)
+                sck.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, req)
 
-            # t = "joining {} from ip {} idx {} with mreq {}"
-            # self.log(t.format(srv.grp, srv.ip, srv.idx, repr(srv.mreq)), 6)
-            sck.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, req)
+        return True
+
+    def hopper(self):
+        while self.args.mc_hop and self.running:
+            time.sleep(self.args.mc_hop)
+            if not self.running:
+                return
+
+            for srv in self.srv.values():
+                self.hop(srv, False)
+
+            # linux does leaves/joins twice with 0.2~1.05s spacing
+            time.sleep(1.2)
+            if not self.running:
+                return
+
+            for srv in self.srv.values():
+                self.hop(srv, True)
 
     def map_client(self, cip: str) -> Optional[MC_Sck]:
         try:

copyparty/pwhash.py

@@ -0,0 +1,145 @@
+# coding: utf-8
+from __future__ import print_function, unicode_literals
+
+import argparse
+import base64
+import hashlib
+import sys
+import threading
+
+from .__init__ import unicode
+
+
+class PWHash(object):
+    def __init__(self, args: argparse.Namespace):
+        self.args = args
+
+        try:
+            alg, ac = args.ah_alg.split(",")
+        except:
+            alg = args.ah_alg
+            ac = {}
+
+        if alg == "none":
+            alg = ""
+
+        self.alg = alg
+        self.ac = ac
+        if not alg:
+            self.on = False
+            self.hash = unicode
+            return
+
+        self.on = True
+        self.salt = args.ah_salt.encode("utf-8")
+        self.cache: dict[str, str] = {}
+        self.mutex = threading.Lock()
+        self.hash = self._cache_hash
+
+        if alg == "sha2":
+            self._hash = self._gen_sha2
+        elif alg == "scrypt":
+            self._hash = self._gen_scrypt
+        elif alg == "argon2":
+            self._hash = self._gen_argon2
+        else:
+            t = "unsupported password hashing algorithm [{}], must be one of these: argon2 scrypt sha2 none"
+            raise Exception(t.format(alg))
+
+    def _cache_hash(self, plain: str) -> str:
+        with self.mutex:
+            try:
+                return self.cache[plain]
+            except:
+                pass
+
+            if not plain:
+                return ""
+
+            if len(plain) > 255:
+                raise Exception("password too long")
+
+            if len(self.cache) > 9000:
+                self.cache = {}
+
+            ret = self._hash(plain)
+            self.cache[plain] = ret
+            return ret
+
+    def _gen_sha2(self, plain: str) -> str:
+        its = int(self.ac[0]) if self.ac else 424242
+        bplain = plain.encode("utf-8")
+        ret = b"\n"
+        for _ in range(its):
+            ret = hashlib.sha512(self.salt + bplain + ret).digest()
+
+        return "+" + base64.urlsafe_b64encode(ret[:24]).decode("utf-8")
+
+    def _gen_scrypt(self, plain: str) -> str:
+        cost = 2 << 13
+        its = 2
+        blksz = 8
+        para = 4
+        try:
+            cost = 2 << int(self.ac[0])
+            its = int(self.ac[1])
+            blksz = int(self.ac[2])
+            para = int(self.ac[3])
+        except:
+            pass
+
+        ret = plain.encode("utf-8")
+        for _ in range(its):
+            ret = hashlib.scrypt(ret, salt=self.salt, n=cost, r=blksz, p=para, dklen=24)
+
+        return "+" + base64.urlsafe_b64encode(ret).decode("utf-8")
+
+    def _gen_argon2(self, plain: str) -> str:
+        from argon2.low_level import Type as ArgonType
+        from argon2.low_level import hash_secret
+
+        time_cost = 3
+        mem_cost = 256
+        parallelism = 4
+        version = 19
+        try:
+            time_cost = int(self.ac[0])
+            mem_cost = int(self.ac[1])
+            parallelism = int(self.ac[2])
+            version = int(self.ac[3])
+        except:
+            pass
+
+        bplain = plain.encode("utf-8")
+
+        bret = hash_secret(
+            secret=bplain,
+            salt=self.salt,
+            time_cost=time_cost,
+            memory_cost=mem_cost * 1024,
+            parallelism=parallelism,
+            hash_len=24,
+            type=ArgonType.ID,
+            version=version,
+        )
+        ret = bret.split(b"$")[-1].decode("utf-8")
+        return "+" + ret.replace("/", "_").replace("+", "-")
+
+    def stdin(self) -> None:
+        while True:
+            ln = sys.stdin.readline().strip()
+            if not ln:
+                break
+            print(self.hash(ln))
+
+    def cli(self) -> None:
+        import getpass
+
+        while True:
+            p1 = getpass.getpass("password> ")
+            p2 = getpass.getpass("again or just hit ENTER> ")
+            if p2 and p1 != p2:
+                print("\033[31minputs don't match; try again\033[0m", file=sys.stderr)
+                continue
+            print(self.hash(p1))
+            print()

copyparty/smbd.py

@@ -32,6 +32,8 @@ class SMB(object):
         self.asrv = hub.asrv
         self.log = hub.log
         self.files: dict[int, tuple[float, str]] = {}
+        self.noacc = self.args.smba
+        self.accs = not self.args.smba
 
         lg.setLevel(logging.DEBUG if self.args.smbvvv else logging.INFO)
         for x in ["impacket", "impacket.smbserver"]:
@@ -94,6 +96,14 @@ class SMB(object):
 
         port = int(self.args.smb_port)
         srv = smbserver.SimpleSMBServer(listenAddress=ip, listenPort=port)
+        try:
+            if self.accs:
+                srv.setAuthCallback(self._auth_cb)
+        except:
+            self.accs = False
+            self.noacc = True
+            t = "impacket too old; access permissions will not work! all accounts are admin!"
+            self.log("smb", t, 1)
 
         ro = "no" if self.args.smbw else "yes"  # (does nothing)
         srv.addShare("A", "/", readOnly=ro)
@@ -119,24 +129,73 @@ class SMB(object):
     def start(self) -> None:
         Daemon(self.srv.start)
 
-    def _v2a(self, caller: str, vpath: str, *a: Any) -> tuple[VFS, str]:
+    def _auth_cb(self, *a, **ka):
+        debug("auth-result: %s %s", a, ka)
+        conndata = ka["connData"]
+        auth_ok = conndata["Authenticated"]
+        uname = ka["user_name"] if auth_ok else "*"
+        uname = self.asrv.iacct.get(uname, uname) or "*"
+        oldname = conndata.get("partygoer", "*") or "*"
+        cli_ip = conndata["ClientIP"]
+        cli_hn = ka["host_name"]
+        if uname != "*":
+            conndata["partygoer"] = uname
+            info("client %s [%s] authed as %s", cli_ip, cli_hn, uname)
+        elif oldname != "*":
+            info("client %s [%s] keeping old auth as %s", cli_ip, cli_hn, oldname)
+        elif auth_ok:
+            info("client %s [%s] authed as [*] (anon)", cli_ip, cli_hn)
+        else:
+            info("client %s [%s] rejected", cli_ip, cli_hn)
+
+    def _uname(self) -> str:
+        if self.noacc:
+            return LEELOO_DALLAS
+
+        try:
+            # you found it! my single worst bit of code so far
+            # (if you can think of a better way to track users through impacket i'm all ears)
+            cf0 = inspect.currentframe().f_back.f_back
+            cf = cf0.f_back
+            for n in range(3):
+                cl = cf.f_locals
+                if "connData" in cl:
+                    return cl["connData"]["partygoer"]
+                cf = cf.f_back
+        except:
+            warning(
+                "nyoron... %s <<-- %s <<-- %s <<-- %s",
+                cf0.f_code.co_name,
+                cf0.f_back.f_code.co_name,
+                cf0.f_back.f_back.f_code.co_name,
+                cf0.f_back.f_back.f_back.f_code.co_name,
+            )
+            return "*"
+
+    def _v2a(
+        self, caller: str, vpath: str, *a: Any, uname="", perms=None
+    ) -> tuple[VFS, str]:
         vpath = vpath.replace("\\", "/").lstrip("/")
         # cf = inspect.currentframe().f_back
         # c1 = cf.f_back.f_code.co_name
         # c2 = cf.f_code.co_name
-        debug('%s("%s", %s)\033[K\033[0m', caller, vpath, str(a))
+        if not uname:
+            uname = self._uname()
+        if not perms:
+            perms = [True, True]
 
-        # TODO find a way to grab `identity` in smbComSessionSetupAndX and smb2SessionSetup
-        vfs, rem = self.asrv.vfs.get(vpath, LEELOO_DALLAS, True, True)
+        debug('%s("%s", %s) %s @%s\033[K\033[0m', caller, vpath, str(a), perms, uname)
+        vfs, rem = self.asrv.vfs.get(vpath, uname, *perms)
         return vfs, vfs.canonical(rem)
 
     def _listdir(self, vpath: str, *a: Any, **ka: Any) -> list[str]:
         vpath = vpath.replace("\\", "/").lstrip("/")
         # caller = inspect.currentframe().f_back.f_code.co_name
-        debug('listdir("%s", %s)\033[K\033[0m', vpath, str(a))
-        vfs, rem = self.asrv.vfs.get(vpath, LEELOO_DALLAS, False, False)
+        uname = self._uname()
+        # debug('listdir("%s", %s) @%s\033[K\033[0m', vpath, str(a), uname)
+        vfs, rem = self.asrv.vfs.get(vpath, uname, False, False)
         _, vfs_ls, vfs_virt = vfs.ls(
-            rem, LEELOO_DALLAS, not self.args.no_scandir, [[False, False]]
+            rem, uname, not self.args.no_scandir, [[False, False]]
         )
         dirs = [x[0] for x in vfs_ls if stat.S_ISDIR(x[1].st_mode)]
         fils = [x[0] for x in vfs_ls if x[0] not in dirs]
@@ -149,8 +208,8 @@ class SMB(object):
         sz = 112 * 2  # ['.', '..']
         for n, fn in enumerate(ls):
             if sz >= 64000:
-                t = "listing only %d of %d files (%d byte); see impacket#1433"
-                warning(t, n, len(ls), sz)
+                t = "listing only %d of %d files (%d byte) in /%s; see impacket#1433"
+                warning(t, n, len(ls), sz, vpath)
                 break
 
             nsz = len(fn.encode("utf-16", "replace"))
@@ -171,10 +230,12 @@ class SMB(object):
         if wr and not self.args.smbw:
             yeet("blocked write (no --smbw): " + vpath)
 
-        vfs, ap = self._v2a("open", vpath, *a)
+        uname = self._uname()
+        vfs, ap = self._v2a("open", vpath, *a, uname=uname, perms=[True, wr])
         if wr:
             if not vfs.axs.uwrite:
-                yeet("blocked write (no-write-acc): " + vpath)
+                t = "blocked write (no-write-acc %s): /%s @%s"
+                yeet(t % (vfs.axs.uwrite, vpath, uname))
 
             xbu = vfs.flags.get("xbu")
             if xbu and not runhook(
@@ -204,7 +265,7 @@ class SMB(object):
 
         _, vp = self.files.pop(fd)
         vp, fn = os.path.split(vp)
-        vfs, rem = self.hub.asrv.vfs.get(vp, LEELOO_DALLAS, False, True)
+        vfs, rem = self.hub.asrv.vfs.get(vp, self._uname(), False, True)
         vfs, rem = vfs.get_dbv(rem)
         self.hub.up2k.hash_file(
             vfs.realpath,
@@ -224,15 +285,18 @@ class SMB(object):
         vp1 = vp1.lstrip("/")
         vp2 = vp2.lstrip("/")
 
-        vfs2, ap2 = self._v2a("rename", vp2, vp1)
+        uname = self._uname()
+        vfs2, ap2 = self._v2a("rename", vp2, vp1, uname=uname)
         if not vfs2.axs.uwrite:
-            yeet("blocked rename (no-write-acc): " + vp2)
+            t = "blocked write (no-write-acc %s): /%s @%s"
+            yeet(t % (vfs2.axs.uwrite, vp2, uname))
 
-        vfs1, _ = self.asrv.vfs.get(vp1, LEELOO_DALLAS, True, True)
+        vfs1, _ = self.asrv.vfs.get(vp1, uname, True, True, True)
         if not vfs1.axs.umove:
-            yeet("blocked rename (no-move-acc): " + vp1)
+            t = "blocked rename (no-move-acc %s): /%s @%s"
+            yeet(t % (vfs1.axs.umove, vp1, uname))
 
-        self.hub.up2k.handle_mv(LEELOO_DALLAS, vp1, vp2)
+        self.hub.up2k.handle_mv(uname, vp1, vp2)
         try:
             bos.makedirs(ap2)
         except:
@@ -242,52 +306,74 @@ class SMB(object):
         if not self.args.smbw:
             yeet("blocked mkdir (no --smbw): " + vpath)
 
-        vfs, ap = self._v2a("mkdir", vpath)
+        uname = self._uname()
+        vfs, ap = self._v2a("mkdir", vpath, uname=uname)
         if not vfs.axs.uwrite:
-            yeet("blocked mkdir (no-write-acc): " + vpath)
+            t = "blocked mkdir (no-write-acc %s): /%s @%s"
+            yeet(t % (vfs.axs.uwrite, vpath, uname))
 
         return bos.mkdir(ap)
 
     def _stat(self, vpath: str, *a: Any, **ka: Any) -> os.stat_result:
-        return bos.stat(self._v2a("stat", vpath, *a)[1], *a, **ka)
+        try:
+            ap = self._v2a("stat", vpath, *a, perms=[True, False])[1]
+            ret = bos.stat(ap, *a, **ka)
+            # debug(" `-stat:ok")
+            return ret
+        except:
+            # white lie: windows freaks out if we raise due to an offline volume
+            # debug(" `-stat:NOPE (faking a directory)")
+            ts = int(time.time())
+            return os.stat_result((16877, -1, -1, 1, 1000, 1000, 8, ts, ts, ts))
 
     def _unlink(self, vpath: str) -> None:
         if not self.args.smbw:
             yeet("blocked delete (no --smbw): " + vpath)
 
         # return bos.unlink(self._v2a("stat", vpath, *a)[1])
-        vfs, ap = self._v2a("delete", vpath)
+        uname = self._uname()
+        vfs, ap = self._v2a(
+            "delete", vpath, uname=uname, perms=[True, False, False, True]
+        )
         if not vfs.axs.udel:
             yeet("blocked delete (no-del-acc): " + vpath)
 
         vpath = vpath.replace("\\", "/").lstrip("/")
-        self.hub.up2k.handle_rm(LEELOO_DALLAS, "1.7.6.2", [vpath], [])
+        self.hub.up2k.handle_rm(uname, "1.7.6.2", [vpath], [], False)
 
     def _utime(self, vpath: str, times: tuple[float, float]) -> None:
         if not self.args.smbw:
             yeet("blocked utime (no --smbw): " + vpath)
 
-        vfs, ap = self._v2a("utime", vpath)
+        uname = self._uname()
+        vfs, ap = self._v2a("utime", vpath, uname=uname)
         if not vfs.axs.uwrite:
-            yeet("blocked utime (no-write-acc): " + vpath)
+            t = "blocked utime (no-write-acc %s): /%s @%s"
+            yeet(t % (vfs.axs.uwrite, vpath, uname))
 
         return bos.utime(ap, times)
 
     def _p_exists(self, vpath: str) -> bool:
+        # ap = "?"
         try:
-            bos.stat(self._v2a("p.exists", vpath)[1])
+            ap = self._v2a("p.exists", vpath, perms=[True, False])[1]
+            bos.stat(ap)
+            # debug(" `-exists((%s)->(%s)):ok", vpath, ap)
             return True
         except:
+            # debug(" `-exists((%s)->(%s)):NOPE", vpath, ap)
             return False
 
     def _p_getsize(self, vpath: str) -> int:
-        st = bos.stat(self._v2a("p.getsize", vpath)[1])
+        st = bos.stat(self._v2a("p.getsize", vpath, perms=[True, False])[1])
         return st.st_size
 
     def _p_isdir(self, vpath: str) -> bool:
         try:
-            st = bos.stat(self._v2a("p.isdir", vpath)[1])
-            return stat.S_ISDIR(st.st_mode)
+            st = bos.stat(self._v2a("p.isdir", vpath, perms=[True, False])[1])
+            ret = stat.S_ISDIR(st.st_mode)
+            # debug(" `-isdir:%s:%s", st.st_mode, ret)
+            return ret
         except:
             return False
 

copyparty/ssdp.py

@@ -1,6 +1,7 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
+import errno
 import re
 import select
 import socket
@@ -80,7 +81,7 @@ class SSDPr(object):
         ubase = "{}://{}:{}".format(proto, sip, sport)
         zsl = self.args.zsl
         url = zsl if "://" in zsl else ubase + "/" + zsl.lstrip("/")
-        name = "{} @ {}".format(self.args.doctitle, self.args.name)
+        name = self.args.doctitle
         zs = zs.strip().format(c(ubase), c(url), c(name), c(self.args.zsid))
         hc.reply(zs.encode("utf-8", "replace"))
         return False  # close connectino
@@ -129,6 +130,17 @@ class SSDPd(MCast):
             srv.hport = hp
 
         self.log("listening")
+        try:
+            self.run2()
+        except OSError as ex:
+            if ex.errno != errno.EBADF:
+                raise
+
+            self.log("stopping due to {}".format(ex), "90")
+
+        self.log("stopped", 2)
+
+    def run2(self) -> None:
         while self.running:
             rdy = select.select(self.srv, [], [], self.args.z_chk or 180)
             rx: list[socket.socket] = rdy[0]  # type: ignore
@@ -148,8 +160,6 @@ class SSDPd(MCast):
                     )
                     self.log(t, 6)
 
-        self.log("stopped", 2)
-
     def stop(self) -> None:
         self.running = False
         for srv in self.srv.values():
@@ -204,7 +214,7 @@ CONFIGID.UPNP.ORG: 1
         srv.sck.sendto(zb, addr[:2])
 
         if cip not in self.txc.c:
-            self.log("{} [{}] --> {}".format(srv.name, srv.ip, cip), "6")
+            self.log("{} [{}] --> {}".format(srv.name, srv.ip, cip), 6)
 
         self.txc.add(cip)
         self.txc.cln()

copyparty/star.py

@@ -1,6 +1,7 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
+import re
 import stat
 import tarfile
 
@@ -44,6 +45,7 @@ class StreamTar(StreamArc):
         self,
         log: "NamedLogger",
         fgen: Generator[dict[str, Any], None, None],
+        cmp: str = "",
         **kwargs: Any
     ):
         super(StreamTar, self).__init__(log, fgen)
@@ -53,14 +55,41 @@ class StreamTar(StreamArc):
         self.qfile = QFile()
         self.errf: dict[str, Any] = {}
 
-        # python 3.8 changed to PAX_FORMAT as default,
-        # waste of space and don't care about the new features
+        # python 3.8 changed to PAX_FORMAT as default;
+        # slower, bigger, and no particular advantage
         fmt = tarfile.GNU_FORMAT
-        self.tar = tarfile.open(fileobj=self.qfile, mode="w|", format=fmt)  # type: ignore
+        if "pax" in cmp:
+            # unless a client asks for it (currently
+            # gnu-tar has wider support than pax-tar)
+            fmt = tarfile.PAX_FORMAT
+            cmp = re.sub(r"[^a-z0-9]*pax[^a-z0-9]*", "", cmp)
+
+        try:
+            cmp, lv = cmp.replace(":", ",").split(",")
+            lv = int(lv)
+        except:
+            lv = None
+
+        arg = {"name": None, "fileobj": self.qfile, "mode": "w", "format": fmt}
+        if cmp == "gz":
+            fun = tarfile.TarFile.gzopen
+            arg["compresslevel"] = lv if lv is not None else 3
+        elif cmp == "bz2":
+            fun = tarfile.TarFile.bz2open
+            arg["compresslevel"] = lv if lv is not None else 2
+        elif cmp == "xz":
+            fun = tarfile.TarFile.xzopen
+            arg["preset"] = lv if lv is not None else 1
+        else:
+            fun = tarfile.open
+            arg["mode"] = "w|"
+
+        self.tar = fun(**arg)
 
         Daemon(self._gen, "star-gen")
 
     def gen(self) -> Generator[Optional[bytes], None, None]:
+        buf = b""
         try:
             while True:
                 buf = self.qfile.q.get()
@@ -72,6 +101,12 @@ class StreamTar(StreamArc):
 
             yield None
         finally:
+            while buf:
+                try:
+                    buf = self.qfile.q.get()
+                except:
+                    pass
+
             if self.errf:
                 bos.unlink(self.errf["ap"])
 
@@ -101,6 +136,9 @@ class StreamTar(StreamArc):
                 errors.append((f["vp"], f["err"]))
                 continue
 
+            if self.stopped:
+                break
+
             try:
                 self.ser(f)
             except:

copyparty/sutil.py

@@ -1,10 +1,14 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
+import os
 import tempfile
 from datetime import datetime
 
+from .__init__ import CORES
 from .bos import bos
+from .th_cli import ThumbCli
+from .util import vjoin
 
 if True:  # pylint: disable=using-constant-test
     from typing import Any, Generator, Optional
@@ -21,10 +25,78 @@ class StreamArc(object):
     ):
         self.log = log
         self.fgen = fgen
+        self.stopped = False
 
     def gen(self) -> Generator[Optional[bytes], None, None]:
         raise Exception("override me")
 
+    def stop(self) -> None:
+        self.stopped = True
+
+
+def gfilter(
+    fgen: Generator[dict[str, Any], None, None],
+    thumbcli: ThumbCli,
+    uname: str,
+    vtop: str,
+    fmt: str,
+) -> Generator[dict[str, Any], None, None]:
+    from concurrent.futures import ThreadPoolExecutor
+
+    pend = []
+    with ThreadPoolExecutor(max_workers=CORES) as tp:
+        try:
+            for f in fgen:
+                task = tp.submit(enthumb, thumbcli, uname, vtop, f, fmt)
+                pend.append((task, f))
+                if pend[0][0].done() or len(pend) > CORES * 4:
+                    task, f = pend.pop(0)
+                    try:
+                        f = task.result(600)
+                    except:
+                        pass
+                    yield f
+
+            for task, f in pend:
+                try:
+                    f = task.result(600)
+                except:
+                    pass
+                yield f
+        except Exception as ex:
+            thumbcli.log("gfilter flushing ({})".format(ex))
+            for task, f in pend:
+                try:
+                    task.result(600)
+                except:
+                    pass
+            thumbcli.log("gfilter flushed")
+
+
+def enthumb(
+    thumbcli: ThumbCli, uname: str, vtop: str, f: dict[str, Any], fmt: str
+) -> dict[str, Any]:
+    rem = f["vp"]
+    ext = rem.rsplit(".", 1)[-1].lower()
+    if fmt == "opus" and ext in "aac|m4a|mp3|ogg|opus|wma".split("|"):
+        raise Exception()
+
+    vp = vjoin(vtop, rem.split("/", 1)[1])
+    vn, rem = thumbcli.asrv.vfs.get(vp, uname, True, False)
+    dbv, vrem = vn.get_dbv(rem)
+    thp = thumbcli.get(dbv, vrem, f["st"].st_mtime, fmt)
+    if not thp:
+        raise Exception()
+
+    ext = "jpg" if fmt == "j" else "webp" if fmt == "w" else fmt
+    sz = bos.path.getsize(thp)
+    st: os.stat_result = f["st"]
+    ts = st.st_mtime
+    f["ap"] = thp
+    f["vp"] = f["vp"].rsplit(".", 1)[0] + "." + ext
+    f["st"] = os.stat_result((st.st_mode, -1, -1, 1, 1000, 1000, sz, ts, ts, ts))
+    return f
+
 
 def errdesc(errors: list[tuple[str, str]]) -> tuple[dict[str, Any], list[str]]:
     report = ["copyparty failed to add the following files to the archive:", ""]

copyparty/svchub.py

@@ -28,8 +28,9 @@ if True:  # pylint: disable=using-constant-test
     import typing
     from typing import Any, Optional, Union
 
-from .__init__ import ANYWIN, EXE, MACOS, TYPE_CHECKING, VT100, EnvParams, unicode
-from .authsrv import AuthSrv
+from .__init__ import ANYWIN, EXE, MACOS, TYPE_CHECKING, EnvParams, unicode
+from .authsrv import BAD_CFG, AuthSrv
+from .cert import ensure_cert
 from .mtag import HAVE_FFMPEG, HAVE_FFPROBE
 from .tcpsrv import TcpSrv
 from .th_srv import HAVE_PIL, HAVE_VIPS, HAVE_WEBP, ThumbSrv
@@ -80,6 +81,7 @@ class SvcHub(object):
         self.dargs = dargs
         self.argv = argv
         self.E: EnvParams = args.E
+        self.no_ansi = args.no_ansi
         self.logf: Optional[typing.TextIO] = None
         self.logf_base_fn = ""
         self.stop_req = False
@@ -98,11 +100,6 @@ class SvcHub(object):
 
         self.iphash = HMaccas(os.path.join(self.E.cfg, "iphash"), 8)
 
-        # for non-http clients (ftp)
-        self.bans: dict[str, int] = {}
-        self.gpwd = Garda(self.args.ban_pw)
-        self.g404 = Garda(self.args.ban_404)
-
         if args.sss or args.s >= 3:
             args.ss = True
             args.no_dav = True
@@ -119,6 +116,7 @@ class SvcHub(object):
             args.hardlink = True
             args.vague_403 = True
             args.ban_404 = "50,60,1440"
+            args.turbo = -1
             args.nih = True
 
         if args.s:
@@ -129,8 +127,19 @@ class SvcHub(object):
             args.force_js = True
 
         if not self._process_config():
-            raise Exception("bad config")
+            raise Exception(BAD_CFG)
 
+        # for non-http clients (ftp)
+        self.bans: dict[str, int] = {}
+        self.gpwd = Garda(self.args.ban_pw)
+        self.g404 = Garda(self.args.ban_404)
+        self.g403 = Garda(self.args.ban_403)
+        self.g422 = Garda(self.args.ban_422)
+        self.gurl = Garda(self.args.ban_url)
+
+        self.log_div = 10 ** (6 - args.log_tdec)
+        self.log_efmt = "%02d:%02d:%02d.%0{}d".format(args.log_tdec)
+        self.log_dfmt = "%04d-%04d-%06d.%0{}d".format(args.log_tdec)
         self.log = self._log_disabled if args.q else self._log_enabled
         if args.lo:
             self._setup_logfile(printed)
@@ -160,6 +169,14 @@ class SvcHub(object):
         ch = "abcdefghijklmnopqrstuvwx"[int(args.theme / 2)]
         args.theme = "{0}{1} {0} {1}".format(ch, bri)
 
+        if args.nih:
+            args.vname = ""
+            args.doctitle = args.doctitle.replace(" @ --name", "")
+        else:
+            args.vname = args.name
+        args.doctitle = args.doctitle.replace("--name", args.vname)
+        args.bname = args.bname.replace("--name", args.vname) or args.vname
+
         if args.log_fk:
             args.log_fk = re.compile(args.log_fk)
 
@@ -238,7 +255,8 @@ class SvcHub(object):
         if args.ftp or args.ftps:
             from .ftpd import Ftpd
 
-            self.ftpd = Ftpd(self)
+            self.ftpd: Optional[Ftpd] = None
+            Daemon(self.start_ftpd, "start_ftpd")
             zms += "f" if args.ftp else "F"
 
         if args.smb:
@@ -268,6 +286,28 @@ class SvcHub(object):
 
         self.broker = Broker(self)
 
+    def start_ftpd(self) -> None:
+        time.sleep(30)
+        if self.ftpd:
+            return
+
+        self.restart_ftpd()
+
+    def restart_ftpd(self) -> None:
+        if not hasattr(self, "ftpd"):
+            return
+
+        from .ftpd import Ftpd
+
+        if self.ftpd:
+            return  # todo
+
+        if not os.path.exists(self.args.cert):
+            ensure_cert(self.log, self.args)
+
+        self.ftpd = Ftpd(self)
+        self.log("root", "started FTPd")
+
     def thr_httpsrv_up(self) -> None:
         time.sleep(1 if self.args.ign_ebind_all else 5)
         expected = self.broker.num_workers * self.tcpsrv.nsrv
@@ -363,6 +403,18 @@ class SvcHub(object):
             if vs and vs.startswith("~"):
                 setattr(al, k, os.path.expanduser(vs))
 
+        for k in "sus_urls nonsus_urls".split(" "):
+            vs = getattr(al, k)
+            if not vs or vs == "no":
+                setattr(al, k, None)
+            else:
+                setattr(al, k, re.compile(vs))
+
+        if not al.sus_urls:
+            al.ban_url = "no"
+        elif al.ban_url == "no":
+            al.sus_urls = None
+
         return True
 
     def _setlimits(self) -> None:
@@ -647,8 +699,14 @@ class SvcHub(object):
             return
 
         with self.log_mutex:
-            ts = datetime.utcnow().strftime("%Y-%m%d-%H%M%S.%f")[:-3]
-            self.logf.write("@{} [{}\033[0m] {}\n".format(ts, src, msg))
+            zd = datetime.utcnow()
+            ts = self.log_dfmt % (
+                zd.year,
+                zd.month * 100 + zd.day,
+                (zd.hour * 100 + zd.minute) * 100 + zd.second,
+                zd.microsecond // self.log_div,
+            )
+            self.logf.write("@%s [%s\033[0m] %s\n" % (ts, src, msg))
 
             now = time.time()
             if now >= self.next_day:
@@ -675,26 +733,36 @@ class SvcHub(object):
             now = time.time()
             if now >= self.next_day:
                 dt = datetime.utcfromtimestamp(now)
-                print("\033[36m{}\033[0m\n".format(dt.strftime("%Y-%m-%d")), end="")
+                zs = "{}\n" if self.no_ansi else "\033[36m{}\033[0m\n"
+                zs = zs.format(dt.strftime("%Y-%m-%d"))
+                print(zs, end="")
                 self._set_next_day()
+                if self.logf:
+                    self.logf.write(zs)
 
-            fmt = "\033[36m{} \033[33m{:21} \033[0m{}\n"
-            if not VT100:
-                fmt = "{} {:21} {}\n"
+            fmt = "\033[36m%s \033[33m%-21s \033[0m%s\n"
+            if self.no_ansi:
+                fmt = "%s %-21s %s\n"
                 if "\033" in msg:
                     msg = ansi_re.sub("", msg)
                 if "\033" in src:
                     src = ansi_re.sub("", src)
             elif c:
                 if isinstance(c, int):
-                    msg = "\033[3{}m{}\033[0m".format(c, msg)
+                    msg = "\033[3%sm%s\033[0m" % (c, msg)
                 elif "\033" not in c:
-                    msg = "\033[{}m{}\033[0m".format(c, msg)
+                    msg = "\033[%sm%s\033[0m" % (c, msg)
                 else:
-                    msg = "{}{}\033[0m".format(c, msg)
+                    msg = "%s%s\033[0m" % (c, msg)
 
-            ts = datetime.utcfromtimestamp(now).strftime("%H:%M:%S.%f")[:-3]
-            msg = fmt.format(ts, src, msg)
+            zd = datetime.utcfromtimestamp(now)
+            ts = self.log_efmt % (
+                zd.hour,
+                zd.minute,
+                zd.second,
+                zd.microsecond // self.log_div,
+            )
+            msg = fmt % (ts, src, msg)
             try:
                 print(msg, end="")
             except UnicodeEncodeError:

copyparty/szip.py

@@ -221,6 +221,7 @@ class StreamZip(StreamArc):
         fgen: Generator[dict[str, Any], None, None],
         utf8: bool = False,
         pre_crc: bool = False,
+        **kwargs: Any
     ) -> None:
         super(StreamZip, self).__init__(log, fgen)
 
@@ -275,6 +276,7 @@ class StreamZip(StreamArc):
     def gen(self) -> Generator[bytes, None, None]:
         errf: dict[str, Any] = {}
         errors = []
+        mbuf = b""
         try:
             for f in self.fgen:
                 if "err" in f:
@@ -283,13 +285,20 @@ class StreamZip(StreamArc):
 
                 try:
                     for x in self.ser(f):
-                        yield x
+                        mbuf += x
+                        if len(mbuf) >= 16384:
+                            yield mbuf
+                            mbuf = b""
                 except GeneratorExit:
                     raise
                 except:
                     ex = min_ex(5, True).replace("\n", "\n-- ")
                     errors.append((f["vp"], ex))
 
+            if mbuf:
+                yield mbuf
+                mbuf = b""
+
             if errors:
                 errf, txt = errdesc(errors)
                 self.log("\n".join(([repr(errf)] + txt[1:])))
@@ -299,20 +308,23 @@ class StreamZip(StreamArc):
             cdir_pos = self.pos
             for name, sz, ts, crc, h_pos in self.items:
                 buf = gen_hdr(h_pos, name, sz, ts, self.utf8, crc, self.pre_crc)
-                yield self._ct(buf)
+                mbuf += self._ct(buf)
+                if len(mbuf) >= 16384:
+                    yield mbuf
+                    mbuf = b""
             cdir_end = self.pos
 
             _, need_64 = gen_ecdr(self.items, cdir_pos, cdir_end)
             if need_64:
                 ecdir64_pos = self.pos
                 buf = gen_ecdr64(self.items, cdir_pos, cdir_end)
-                yield self._ct(buf)
+                mbuf += self._ct(buf)
 
                 buf = gen_ecdr64_loc(ecdir64_pos)
-                yield self._ct(buf)
+                mbuf += self._ct(buf)
 
             ecdr, _ = gen_ecdr(self.items, cdir_pos, cdir_end)
-            yield self._ct(ecdr)
+            yield mbuf + self._ct(ecdr)
         finally:
             if errf:
                 bos.unlink(errf["ap"])

copyparty/tcpsrv.py

@@ -7,7 +7,8 @@ import socket
 import sys
 import time
 
-from .__init__ import ANYWIN, PY2, TYPE_CHECKING, VT100, unicode
+from .__init__ import ANYWIN, PY2, TYPE_CHECKING, unicode
+from .cert import gencert
 from .stolen.qrcodegen import QrCode
 from .util import (
     E_ACCESS,
@@ -295,6 +296,8 @@ class TcpSrv(object):
     def _distribute_netdevs(self):
         self.hub.broker.say("set_netdevs", self.netdevs)
         self.hub.start_zeroconf()
+        gencert(self.log, self.args, self.netdevs)
+        self.hub.restart_ftpd()
 
     def shutdown(self) -> None:
         self.stopping = True
@@ -501,7 +504,7 @@ class TcpSrv(object):
                 zoom = 1
 
         qr = qrc.render(zoom, pad)
-        if not VT100:
+        if self.args.no_ansi:
             return "{}\n{}".format(txt, qr)
 
         halfc = "\033[40;48;5;{0}m{1}\033[47;48;5;{2}m"

copyparty/th_cli.py

@@ -108,6 +108,7 @@ class ThumbCli(object):
                 if st.st_size:
                     ret = tpath = tp
                     fmt = ret.rsplit(".")[1]
+                    break
                 else:
                     abort = True
             except:

copyparty/th_srv.py

@@ -13,6 +13,7 @@ import time
 from queue import Queue
 
 from .__init__ import ANYWIN, TYPE_CHECKING
+from .authsrv import VFS
 from .bos import bos
 from .mtag import HAVE_FFMPEG, HAVE_FFPROBE, ffprobe
 from .util import (
@@ -110,8 +111,6 @@ class ThumbSrv(object):
         self.args = hub.args
         self.log_func = hub.log
 
-        res = hub.args.th_size.split("x")
-        self.res = tuple([int(x) for x in res])
         self.poke_cd = Cooldown(self.args.th_poke)
 
         self.mutex = threading.Lock()
@@ -119,7 +118,7 @@ class ThumbSrv(object):
         self.stopping = False
         self.nthr = max(1, self.args.th_mt)
 
-        self.q: Queue[Optional[tuple[str, str]]] = Queue(self.nthr * 4)
+        self.q: Queue[Optional[tuple[str, str, VFS]]] = Queue(self.nthr * 4)
         for n in range(self.nthr):
             Daemon(self.worker, "thumb-{}-{}".format(n, self.nthr))
 
@@ -184,6 +183,10 @@ class ThumbSrv(object):
         with self.mutex:
             return not self.nthr
 
+    def getres(self, vn: VFS) -> tuple[int, int]:
+        w, h = vn.flags["thsize"].split("x")
+        return int(w), int(h)
+
     def get(self, ptop: str, rem: str, mtime: float, fmt: str) -> Optional[str]:
         histpath = self.asrv.vfs.histtab.get(ptop)
         if not histpath:
@@ -211,7 +214,13 @@ class ThumbSrv(object):
                 do_conv = True
 
         if do_conv:
-            self.q.put((abspath, tpath))
+            allvols = list(self.asrv.vfs.all_vols.values())
+            vn = next((x for x in allvols if x.realpath == ptop), None)
+            if not vn:
+                self.log("ptop [{}] not in {}".format(ptop, allvols), 3)
+                vn = self.asrv.vfs.all_aps[0][1]
+
+            self.q.put((abspath, tpath, vn))
             self.log("conv {} \033[0m{}".format(tpath, abspath), c=6)
 
         while not self.stopping:
@@ -248,7 +257,7 @@ class ThumbSrv(object):
             if not task:
                 break
 
-            abspath, tpath = task
+            abspath, tpath, vn = task
             ext = abspath.split(".")[-1].lower()
             png_ok = False
             funs = []
@@ -274,10 +283,14 @@ class ThumbSrv(object):
 
             tdir, tfn = os.path.split(tpath)
             ttpath = os.path.join(tdir, "w", tfn)
+            try:
+                bos.unlink(ttpath)
+            except:
+                pass
 
             for fun in funs:
                 try:
-                    fun(abspath, ttpath)
+                    fun(abspath, ttpath, vn)
                     break
                 except Exception as ex:
                     msg = "{} could not create thumbnail of {}\n{}"
@@ -311,9 +324,10 @@ class ThumbSrv(object):
         with self.mutex:
             self.nthr -= 1
 
-    def fancy_pillow(self, im: "Image.Image") -> "Image.Image":
+    def fancy_pillow(self, im: "Image.Image", vn: VFS) -> "Image.Image":
         # exif_transpose is expensive (loads full image + unconditional copy)
-        r = max(*self.res) * 2
+        res = self.getres(vn)
+        r = max(*res) * 2
         im.thumbnail((r, r), resample=Image.LANCZOS)
         try:
             k = next(k for k, v in ExifTags.TAGS.items() if v == "Orientation")
@@ -327,23 +341,23 @@ class ThumbSrv(object):
         if rot in rots:
             im = im.transpose(rots[rot])
 
-        if self.args.th_no_crop:
-            im.thumbnail(self.res, resample=Image.LANCZOS)
+        if "nocrop" in vn.flags:
+            im.thumbnail(res, resample=Image.LANCZOS)
         else:
             iw, ih = im.size
-            dw, dh = self.res
+            dw, dh = res
             res = (min(iw, dw), min(ih, dh))
             im = ImageOps.fit(im, res, method=Image.LANCZOS)
 
         return im
 
-    def conv_pil(self, abspath: str, tpath: str) -> None:
+    def conv_pil(self, abspath: str, tpath: str, vn: VFS) -> None:
         with Image.open(fsenc(abspath)) as im:
             try:
-                im = self.fancy_pillow(im)
+                im = self.fancy_pillow(im, vn)
             except Exception as ex:
                 self.log("fancy_pillow {}".format(ex), "90")
-                im.thumbnail(self.res)
+                im.thumbnail(self.getres(vn))
 
             fmts = ["RGB", "L"]
             args = {"quality": 40}
@@ -366,12 +380,12 @@ class ThumbSrv(object):
 
             im.save(tpath, **args)
 
-    def conv_vips(self, abspath: str, tpath: str) -> None:
+    def conv_vips(self, abspath: str, tpath: str, vn: VFS) -> None:
         crops = ["centre", "none"]
-        if self.args.th_no_crop:
+        if "nocrop" in vn.flags:
             crops = ["none"]
 
-        w, h = self.res
+        w, h = self.getres(vn)
         kw = {"height": h, "size": "down", "intent": "relative"}
 
         for c in crops:
@@ -385,8 +399,8 @@ class ThumbSrv(object):
 
         img.write_to_file(tpath, Q=40)
 
-    def conv_ffmpeg(self, abspath: str, tpath: str) -> None:
-        ret, _ = ffprobe(abspath, int(self.args.th_convt / 2))
+    def conv_ffmpeg(self, abspath: str, tpath: str, vn: VFS) -> None:
+        ret, _ = ffprobe(abspath, int(vn.flags["convt"] / 2))
         if not ret:
             return
 
@@ -398,12 +412,13 @@ class ThumbSrv(object):
             seek = [b"-ss", "{:.0f}".format(dur / 3).encode("utf-8")]
 
         scale = "scale={0}:{1}:force_original_aspect_ratio="
-        if self.args.th_no_crop:
+        if "nocrop" in vn.flags:
             scale += "decrease,setsar=1:1"
         else:
             scale += "increase,crop={0}:{1},setsar=1:1"
 
-        bscale = scale.format(*list(self.res)).encode("utf-8")
+        res = self.getres(vn)
+        bscale = scale.format(*list(res)).encode("utf-8")
         # fmt: off
         cmd = [
             b"ffmpeg",
@@ -435,11 +450,11 @@ class ThumbSrv(object):
             ]
 
         cmd += [fsenc(tpath)]
-        self._run_ff(cmd)
+        self._run_ff(cmd, vn)
 
-    def _run_ff(self, cmd: list[bytes]) -> None:
+    def _run_ff(self, cmd: list[bytes], vn: VFS) -> None:
         # self.log((b" ".join(cmd)).decode("utf-8"))
-        ret, _, serr = runcmd(cmd, timeout=self.args.th_convt)
+        ret, _, serr = runcmd(cmd, timeout=vn.flags["convt"])
         if not ret:
             return
 
@@ -482,8 +497,8 @@ class ThumbSrv(object):
         self.log(t + txt, c=c)
         raise sp.CalledProcessError(ret, (cmd[0], b"...", cmd[-1]))
 
-    def conv_waves(self, abspath: str, tpath: str) -> None:
-        ret, _ = ffprobe(abspath, int(self.args.th_convt / 2))
+    def conv_waves(self, abspath: str, tpath: str, vn: VFS) -> None:
+        ret, _ = ffprobe(abspath, int(vn.flags["convt"] / 2))
         if "ac" not in ret:
             raise Exception("not audio")
 
@@ -508,10 +523,10 @@ class ThumbSrv(object):
         # fmt: on
 
         cmd += [fsenc(tpath)]
-        self._run_ff(cmd)
+        self._run_ff(cmd, vn)
 
-    def conv_spec(self, abspath: str, tpath: str) -> None:
-        ret, _ = ffprobe(abspath, int(self.args.th_convt / 2))
+    def conv_spec(self, abspath: str, tpath: str, vn: VFS) -> None:
+        ret, _ = ffprobe(abspath, int(vn.flags["convt"] / 2))
         if "ac" not in ret:
             raise Exception("not audio")
 
@@ -551,13 +566,13 @@ class ThumbSrv(object):
             ]
 
         cmd += [fsenc(tpath)]
-        self._run_ff(cmd)
+        self._run_ff(cmd, vn)
 
-    def conv_opus(self, abspath: str, tpath: str) -> None:
+    def conv_opus(self, abspath: str, tpath: str, vn: VFS) -> None:
         if self.args.no_acode:
             raise Exception("disabled in server config")
 
-        ret, _ = ffprobe(abspath, int(self.args.th_convt / 2))
+        ret, _ = ffprobe(abspath, int(vn.flags["convt"] / 2))
         if "ac" not in ret:
             raise Exception("not audio")
 
@@ -570,11 +585,15 @@ class ThumbSrv(object):
         want_caf = tpath.endswith(".caf")
         tmp_opus = tpath
         if want_caf:
-            tmp_opus = tpath.rsplit(".", 1)[0] + ".opus"
+            tmp_opus = tpath + ".opus"
+            try:
+                bos.unlink(tmp_opus)
+            except:
+                pass
 
         caf_src = abspath if src_opus else tmp_opus
 
-        if not want_caf or (not src_opus and not bos.path.isfile(tmp_opus)):
+        if not want_caf or not src_opus:
             # fmt: off
             cmd = [
                 b"ffmpeg",
@@ -589,7 +608,7 @@ class ThumbSrv(object):
                 fsenc(tmp_opus)
             ]
             # fmt: on
-            self._run_ff(cmd)
+            self._run_ff(cmd, vn)
 
         # iOS fails to play some "insufficiently complex" files
         # (average file shorter than 8 seconds), so of course we
@@ -613,7 +632,7 @@ class ThumbSrv(object):
                 fsenc(tpath)
             ]
             # fmt: on
-            self._run_ff(cmd)
+            self._run_ff(cmd, vn)
 
         elif want_caf:
             # simple remux should be safe
@@ -631,7 +650,13 @@ class ThumbSrv(object):
                 fsenc(tpath)
             ]
             # fmt: on
-            self._run_ff(cmd)
+            self._run_ff(cmd, vn)
+
+        if tmp_opus != tpath:
+            try:
+                bos.unlink(tmp_opus)
+            except:
+                pass
 
     def poke(self, tdir: str) -> None:
         if not self.poke_cd.poke(tdir):

copyparty/u2idx.py

@@ -69,7 +69,7 @@ class U2idx(object):
 
         fsize = body["size"]
         fhash = body["hash"]
-        wark = up2k_wark_from_hashlist(self.args.salt, fsize, fhash)
+        wark = up2k_wark_from_hashlist(self.args.warksalt, fsize, fhash)
 
         uq = "substr(w,1,16) = ? and w = ?"
         uv: list[Union[str, int]] = [wark[:16], wark]

copyparty/up2k.py

@@ -22,8 +22,9 @@ from copy import deepcopy
 from queue import Queue
 
 from .__init__ import ANYWIN, PY2, TYPE_CHECKING, WINDOWS
-from .authsrv import LEELOO_DALLAS, VFS, AuthSrv
+from .authsrv import LEELOO_DALLAS, SSEELOG, VFS, AuthSrv
 from .bos import bos
+from .cfg import vf_bmap, vf_vmap
 from .fsutil import Fstab
 from .mtag import MParser, MTag
 from .util import (
@@ -41,6 +42,7 @@ from .util import (
     gen_filekey,
     gen_filekey_dbg,
     hidedir,
+    humansize,
     min_ex,
     quotep,
     rand_name,
@@ -56,6 +58,7 @@ from .util import (
     sfsenc,
     spack,
     statdir,
+    unhumanize,
     vjoin,
     vsplit,
     w8b64dec,
@@ -73,8 +76,8 @@ if True:  # pylint: disable=using-constant-test
 if TYPE_CHECKING:
     from .svchub import SvcHub
 
-zs = "avif,avifs,bmp,gif,heic,heics,heif,heifs,ico,j2p,j2k,jp2,jpeg,jpg,jpx,png,tga,tif,tiff,webp"
-CV_EXTS = set(zs.split(","))
+zsg = "avif,avifs,bmp,gif,heic,heics,heif,heifs,ico,j2p,j2k,jp2,jpeg,jpg,jpx,png,tga,tif,tiff,webp"
+CV_EXTS = set(zsg.split(","))
 
 
 class Dbw(object):
@@ -110,7 +113,7 @@ class Up2k(object):
         self.args = hub.args
         self.log_func = hub.log
 
-        self.salt = self.args.salt
+        self.salt = self.args.warksalt
         self.r_hash = re.compile("^[0-9a-zA-Z_-]{44}$")
 
         self.gid = 0
@@ -125,6 +128,8 @@ class Up2k(object):
         self.registry: dict[str, dict[str, dict[str, Any]]] = {}
         self.flags: dict[str, dict[str, Any]] = {}
         self.droppable: dict[str, list[str]] = {}
+        self.volnfiles: dict["sqlite3.Cursor", int] = {}
+        self.volsize: dict["sqlite3.Cursor", int] = {}
         self.volstate: dict[str, str] = {}
         self.vol_act: dict[str, float] = {}
         self.busy_aps: set[str] = set()
@@ -195,7 +200,8 @@ class Up2k(object):
         if self.stop:
             # up-mt consistency not guaranteed if init is interrupted;
             # drop caches for a full scan on next boot
-            self._drop_caches()
+            with self.mutex:
+                self._drop_caches()
 
             if self.pp:
                 self.pp.end = True
@@ -261,6 +267,58 @@ class Up2k(object):
         }
         return json.dumps(ret, indent=4)
 
+    def get_unfinished(self) -> str:
+        if PY2 or not self.mutex.acquire(timeout=0.5):
+            return "{}"
+
+        ret: dict[str, tuple[int, int]] = {}
+        try:
+            for ptop, tab2 in self.registry.items():
+                nbytes = 0
+                nfiles = 0
+                drp = self.droppable.get(ptop, {})
+                for wark, job in tab2.items():
+                    if wark in drp:
+                        continue
+
+                    nfiles += 1
+                    try:
+                        # close enough on average
+                        nbytes += len(job["need"]) * job["size"] // len(job["hash"])
+                    except:
+                        pass
+
+                ret[ptop] = (nbytes, nfiles)
+        finally:
+            self.mutex.release()
+
+        return json.dumps(ret, indent=4)
+
+    def get_volsize(self, ptop: str) -> tuple[int, int]:
+        with self.mutex:
+            return self._get_volsize(ptop)
+
+    def get_volsizes(self, ptops: list[str]) -> list[tuple[int, int]]:
+        ret = []
+        with self.mutex:
+            for ptop in ptops:
+                ret.append(self._get_volsize(ptop))
+
+        return ret
+
+    def _get_volsize(self, ptop: str) -> tuple[int, int]:
+        if "e2ds" not in self.flags.get(ptop, {}):
+            return (0, 0)
+
+        cur = self.cur[ptop]
+        nbytes = self.volsize[cur]
+        nfiles = self.volnfiles[cur]
+        for j in list(self.registry.get(ptop, {}).values()):
+            nbytes += j["size"]
+            nfiles += 1
+
+        return (nbytes, nfiles)
+
     def rescan(
         self, all_vols: dict[str, VFS], scan_vols: list[str], wait: bool, fscan: bool
     ) -> str:
@@ -380,11 +438,11 @@ class Up2k(object):
                         if rd.startswith("//") or fn.startswith("//"):
                             rd, fn = s3dec(rd, fn)
 
-                        fvp = "{}/{}".format(rd, fn).strip("/")
+                        fvp = ("%s/%s" % (rd, fn)).strip("/")
                         if vp:
-                            fvp = "{}/{}".format(vp, fvp)
+                            fvp = "%s/%s" % (vp, fvp)
 
-                        self._handle_rm(LEELOO_DALLAS, "", fvp, [])
+                        self._handle_rm(LEELOO_DALLAS, "", fvp, [], True)
                         nrm += 1
 
                 if nrm:
@@ -575,7 +633,8 @@ class Up2k(object):
 
         if self.args.re_dhash or [zv for zv in vols if "e2tsr" in zv.flags]:
             self.args.re_dhash = False
-            self._drop_caches()
+            with self.mutex:
+                self._drop_caches()
 
         for vol in vols:
             if self.stop:
@@ -739,8 +798,9 @@ class Up2k(object):
         ff = "\033[0;35m{}{:.0}"
         fv = "\033[0;36m{}:\033[90m{}"
         fx = set(("html_head",))
-        fdl = ("dbd", "lg_sbf", "md_sbf", "mte", "mth", "mtp", "nrand", "rand")
-        fd = {x: x for x in fdl}
+        fd = vf_bmap()
+        fd.update(vf_vmap())
+        fd = {v: k for k, v in fd.items()}
         fl = {
             k: v
             for k, v in flags.items()
@@ -751,6 +811,9 @@ class Up2k(object):
             for k, v in fl.items()
             if k not in fx
         ]
+        if not a:
+            a = ["\033[90mall-default"]
+
         if a:
             vpath = "?"
             for k, v in self.asrv.vfs.all_vols.items():
@@ -766,9 +829,9 @@ class Up2k(object):
 
         reg = {}
         drp = None
-        path = os.path.join(histpath, "up2k.snap")
-        if bos.path.exists(path):
-            with gzip.GzipFile(path, "rb") as f:
+        snap = os.path.join(histpath, "up2k.snap")
+        if bos.path.exists(snap):
+            with gzip.GzipFile(snap, "rb") as f:
                 j = f.read().decode("utf-8")
 
             reg2 = json.loads(j)
@@ -779,20 +842,20 @@ class Up2k(object):
                 pass
 
             for k, job in reg2.items():
-                path = djoin(job["ptop"], job["prel"], job["name"])
-                if bos.path.exists(path):
+                fp = djoin(job["ptop"], job["prel"], job["name"])
+                if bos.path.exists(fp):
                     reg[k] = job
                     job["poke"] = time.time()
                     job["busy"] = {}
                 else:
-                    self.log("ign deleted file in snap: [{}]".format(path))
+                    self.log("ign deleted file in snap: [{}]".format(fp))
 
             if drp is None:
                 drp = [k for k, v in reg.items() if not v.get("need", [])]
             else:
                 drp = [x for x in drp if x in reg]
 
-            t = "loaded snap {} |{}| ({})".format(path, len(reg.keys()), len(drp or []))
+            t = "loaded snap {} |{}| ({})".format(snap, len(reg.keys()), len(drp or []))
             ta = [t] + self._vis_reg_progress(reg)
             self.log("\n".join(ta))
 
@@ -804,12 +867,17 @@ class Up2k(object):
         if not HAVE_SQLITE3 or "e2d" not in flags or "d2d" in flags:
             return None
 
-        if bos.makedirs(histpath):
-            hidedir(histpath)
+        try:
+            if bos.makedirs(histpath):
+                hidedir(histpath)
+        except:
+            return None
 
         try:
             cur = self._open_db(db_path)
             self.cur[ptop] = cur
+            self.volsize[cur] = 0
+            self.volnfiles[cur] = 0
 
             # speeds measured uploading 520 small files on a WD20SPZX (SMR 2.5" 5400rpm 4kb)
             dbd = flags["dbd"]
@@ -856,6 +924,7 @@ class Up2k(object):
         rei = vol.flags.get("noidx")
         reh = vol.flags.get("nohash")
         n4g = bool(vol.flags.get("noforget"))
+        ffat = "fat32" in vol.flags
         cst = bos.stat(top)
         dev = cst.st_dev if vol.flags.get("xdev") else 0
 
@@ -892,6 +961,7 @@ class Up2k(object):
                     rei,
                     reh,
                     n4g,
+                    ffat,
                     [],
                     cst,
                     dev,
@@ -917,6 +987,28 @@ class Up2k(object):
 
             db.c.connection.commit()
 
+            if (
+                vol.flags.get("vmaxb")
+                or vol.flags.get("vmaxn")
+                or (self.args.stats and not self.args.nos_vol)
+            ):
+                zs = "select count(sz), sum(sz) from up"
+                vn, vb = db.c.execute(zs).fetchone()
+                vb = vb or 0
+                vb += vn * 2048
+                self.volsize[db.c] = vb
+                self.volnfiles[db.c] = vn
+                vmaxb = unhumanize(vol.flags.get("vmaxb") or "0")
+                vmaxn = unhumanize(vol.flags.get("vmaxn") or "0")
+                t = "{:>5} / {:>5}  ( {:>5} / {:>5} files) in {}".format(
+                    humansize(vb, True),
+                    humansize(vmaxb, True),
+                    humansize(vn, True).rstrip("B"),
+                    humansize(vmaxn, True).rstrip("B"),
+                    vol.realpath,
+                )
+                self.log(t)
+
             return True, bool(n_add or n_rm or do_vac)
 
     def _build_dir(
@@ -929,6 +1021,7 @@ class Up2k(object):
         rei: Optional[Pattern[str]],
         reh: Optional[Pattern[str]],
         n4g: bool,
+        ffat: bool,
         seen: list[str],
         cst: os.stat_result,
         dev: int,
@@ -957,7 +1050,7 @@ class Up2k(object):
         if WINDOWS:
             rd = rd.replace("\\", "/").strip("/")
 
-        g = statdir(self.log_func, not self.args.no_scandir, False, cdir)
+        g = statdir(self.log_func, not self.args.no_scandir, True, cdir)
         gl = sorted(g)
         partials = set([x[0] for x in gl if "PARTIAL" in x[0]])
         for iname, inf in gl:
@@ -972,8 +1065,14 @@ class Up2k(object):
                 continue
 
             lmod = int(inf.st_mtime)
+            if stat.S_ISLNK(inf.st_mode):
+                try:
+                    inf = bos.stat(abspath)
+                except:
+                    continue
+
             sz = inf.st_size
-            if fat32 and inf.st_mtime % 2:
+            if fat32 and not ffat and inf.st_mtime % 2:
                 fat32 = False
 
             if stat.S_ISDIR(inf.st_mode):
@@ -990,7 +1089,19 @@ class Up2k(object):
                 # self.log(" dir: {}".format(abspath))
                 try:
                     ret += self._build_dir(
-                        db, top, excl, abspath, rap, rei, reh, n4g, seen, inf, dev, xvol
+                        db,
+                        top,
+                        excl,
+                        abspath,
+                        rap,
+                        rei,
+                        reh,
+                        n4g,
+                        fat32,
+                        seen,
+                        inf,
+                        dev,
+                        xvol,
                     )
                 except:
                     t = "failed to index subdir [{}]:\n{}"
@@ -1092,7 +1203,7 @@ class Up2k(object):
                         top, rp, dts, lmod, dsz, sz
                     )
                     self.log(t)
-                    self.db_rm(db.c, rd, fn)
+                    self.db_rm(db.c, rd, fn, 0)
                     ret += 1
                     db.n += 1
                     in_db = []
@@ -1175,7 +1286,7 @@ class Up2k(object):
         rm_files = [x for x in hits if x not in seen_files]
         n_rm = len(rm_files)
         for fn in rm_files:
-            self.db_rm(db.c, rd, fn)
+            self.db_rm(db.c, rd, fn, 0)
 
         if n_rm:
             self.log("forgot {} deleted files".format(n_rm))
@@ -1340,9 +1451,11 @@ class Up2k(object):
                 pf = "v{}, {:.0f}+".format(n_left, b_left / 1024 / 1024)
                 self.pp.msg = pf + abspath
 
-                st = bos.stat(abspath)
+                # throws on broken symlinks (always did)
+                stl = bos.lstat(abspath)
+                st = bos.stat(abspath) if stat.S_ISLNK(stl.st_mode) else stl
+                mt2 = int(stl.st_mtime)
                 sz2 = st.st_size
-                mt2 = int(st.st_mtime)
 
                 if nohash or not sz2:
                     w2 = up2k_wark_from_metadata(self.salt, sz2, mt2, rd, fn)
@@ -1364,6 +1477,13 @@ class Up2k(object):
                 if w == w2:
                     continue
 
+                # symlink mtime was inconsistent before v1.9.4; check if that's it
+                if st != stl and (nohash or not sz2):
+                    mt2b = int(st.st_mtime)
+                    w2b = up2k_wark_from_metadata(self.salt, sz2, mt2b, rd, fn)
+                    if w == w2b:
+                        continue
+
                 rewark.append((drd, dfn, w2, sz2, mt2))
 
                 t = "hash mismatch: {}\n  db: {} ({} byte, {})\n  fs: {} ({} byte, {})"
@@ -2284,7 +2404,9 @@ class Up2k(object):
             if lost:
                 c2 = None
                 for cur, dp_dir, dp_fn in lost:
-                    self.db_rm(cur, dp_dir, dp_fn)
+                    t = "forgetting deleted file: /{}"
+                    self.log(t.format(vjoin(vjoin(vfs.vpath, dp_dir), dp_fn)))
+                    self.db_rm(cur, dp_dir, dp_fn, cj["size"])
                     if c2 and c2 != cur:
                         c2.connection.commit()
 
@@ -2296,27 +2418,31 @@ class Up2k(object):
             cur = jcur
             ptop = None  # use cj or job as appropriate
 
+            if not job and wark in reg:
+                # ensure the files haven't been deleted manually
+                rj = reg[wark]
+                names = [rj[x] for x in ["name", "tnam"] if x in rj]
+                for fn in names:
+                    path = djoin(rj["ptop"], rj["prel"], fn)
+                    try:
+                        if bos.path.getsize(path) > 0 or not rj["need"]:
+                            # upload completed or both present
+                            break
+                    except:
+                        # missing; restart
+                        if not self.args.nw and not n4g:
+                            t = "forgetting deleted partial upload at {}"
+                            self.log(t.format(path))
+                            del reg[wark]
+                        break
+
             if job or wark in reg:
                 job = job or reg[wark]
                 if (
-                    job["ptop"] == cj["ptop"]
-                    and job["prel"] == cj["prel"]
-                    and job["name"] == cj["name"]
+                    job["ptop"] != cj["ptop"]
+                    or job["prel"] != cj["prel"]
+                    or job["name"] != cj["name"]
                 ):
-                    # ensure the files haven't been deleted manually
-                    names = [job[x] for x in ["name", "tnam"] if x in job]
-                    for fn in names:
-                        path = djoin(job["ptop"], job["prel"], fn)
-                        try:
-                            if bos.path.getsize(path) > 0:
-                                # upload completed or both present
-                                break
-                        except:
-                            # missing; restart
-                            if not self.args.nw and not n4g:
-                                job = None
-                            break
-                else:
                     # file contents match, but not the path
                     src = djoin(job["ptop"], job["prel"], job["name"])
                     dst = djoin(cj["ptop"], cj["prel"], cj["name"])
@@ -2418,7 +2544,14 @@ class Up2k(object):
 
                 if vfs.lim:
                     ap2, cj["prel"] = vfs.lim.all(
-                        cj["addr"], cj["prel"], cj["size"], ap1, reg
+                        cj["addr"],
+                        cj["prel"],
+                        cj["size"],
+                        cj["ptop"],
+                        ap1,
+                        self.hub.broker,
+                        reg,
+                        "up2k._get_volsize",
                     )
                     bos.makedirs(ap2)
                     vfs.lim.nup(cj["addr"])
@@ -2564,7 +2697,7 @@ class Up2k(object):
 
             try:
                 if "hardlink" in flags:
-                    os.link(fsenc(src), fsenc(dst))
+                    os.link(fsenc(absreal(src)), fsenc(dst))
                     linked = True
             except Exception as ex:
                 self.log("cannot hardlink: " + repr(ex))
@@ -2591,7 +2724,7 @@ class Up2k(object):
             if not job:
                 known = " ".join([x for x in self.registry[ptop].keys()])
                 self.log("unknown wark [{}], known: {}".format(wark, known))
-                raise Pebkac(400, "unknown wark")
+                raise Pebkac(400, "unknown wark" + SSEELOG)
 
             if chash not in job["need"]:
                 msg = "chash = {} , need:\n".format(chash)
@@ -2736,7 +2869,7 @@ class Up2k(object):
 
             self._symlink(dst, d2, self.flags[ptop], lmod=lmod)
             if cur:
-                self.db_rm(cur, rd, fn)
+                self.db_rm(cur, rd, fn, job["size"])
                 self.db_add(cur, vflags, rd, fn, lmod, *z2[3:])
 
         if cur:
@@ -2779,7 +2912,7 @@ class Up2k(object):
 
         self.db_act = self.vol_act[ptop] = time.time()
         try:
-            self.db_rm(cur, rd, fn)
+            self.db_rm(cur, rd, fn, sz)
             self.db_add(
                 cur,
                 vflags,
@@ -2809,13 +2942,17 @@ class Up2k(object):
 
         return True
 
-    def db_rm(self, db: "sqlite3.Cursor", rd: str, fn: str) -> None:
+    def db_rm(self, db: "sqlite3.Cursor", rd: str, fn: str, sz: int) -> None:
         sql = "delete from up where rd = ? and fn = ?"
         try:
-            db.execute(sql, (rd, fn))
+            r = db.execute(sql, (rd, fn))
         except:
             assert self.mem_cur
-            db.execute(sql, s3enc(self.mem_cur, rd, fn))
+            r = db.execute(sql, s3enc(self.mem_cur, rd, fn))
+
+        if r.rowcount:
+            self.volsize[db] -= sz
+            self.volnfiles[db] -= 1
 
     def db_add(
         self,
@@ -2844,6 +2981,9 @@ class Up2k(object):
             v = (wark, int(ts), sz, rd, fn, ip or "", int(at or 0))
             db.execute(sql, v)
 
+        self.volsize[db] += sz
+        self.volnfiles[db] += 1
+
         xau = False if skip_xau else vflags.get("xau")
         dst = djoin(ptop, rd, fn)
         if xau and not runhook(
@@ -2897,7 +3037,9 @@ class Up2k(object):
             except:
                 pass
 
-    def handle_rm(self, uname: str, ip: str, vpaths: list[str], lim: list[int]) -> str:
+    def handle_rm(
+        self, uname: str, ip: str, vpaths: list[str], lim: list[int], rm_up: bool
+    ) -> str:
         n_files = 0
         ok = {}
         ng = {}
@@ -2906,7 +3048,7 @@ class Up2k(object):
                 self.log("hit delete limit of {} files".format(lim[1]), 3)
                 break
 
-            a, b, c = self._handle_rm(uname, ip, vp, lim)
+            a, b, c = self._handle_rm(uname, ip, vp, lim, rm_up)
             n_files += a
             for k in b:
                 ok[k] = 1
@@ -2920,7 +3062,7 @@ class Up2k(object):
         return "deleted {} files (and {}/{} folders)".format(n_files, iok, iok + ing)
 
     def _handle_rm(
-        self, uname: str, ip: str, vpath: str, lim: list[int]
+        self, uname: str, ip: str, vpath: str, lim: list[int], rm_up: bool
     ) -> tuple[int, list[str], list[str]]:
         self.db_act = time.time()
         try:
@@ -2937,7 +3079,8 @@ class Up2k(object):
             permsets = [[False, True]]
             vn, rem = self.asrv.vfs.get(vpath, uname, *permsets[0])
             vn, rem = vn.get_dbv(rem)
-            _, _, _, _, dip, dat = self._find_from_vpath(vn.realpath, rem)
+            with self.mutex:
+                _, _, _, _, dip, dat = self._find_from_vpath(vn.realpath, rem)
 
             t = "you cannot delete this: "
             if not dip:
@@ -2989,12 +3132,12 @@ class Up2k(object):
                         break
 
                 abspath = djoin(adir, fn)
+                st = bos.stat(abspath)
                 volpath = "{}/{}".format(vrem, fn).strip("/")
                 vpath = "{}/{}".format(dbv.vpath, volpath).strip("/")
                 self.log("rm {}\n  {}".format(vpath, abspath))
                 _ = dbv.get(volpath, uname, *permsets[0])
                 if xbd:
-                    st = bos.stat(abspath)
                     if not runhook(
                         self.log,
                         xbd,
@@ -3018,25 +3161,43 @@ class Up2k(object):
                     try:
                         ptop = dbv.realpath
                         cur, wark, _, _, _, _ = self._find_from_vpath(ptop, volpath)
-                        self._forget_file(ptop, volpath, cur, wark, True)
+                        self._forget_file(ptop, volpath, cur, wark, True, st.st_size)
                     finally:
                         if cur:
                             cur.connection.commit()
 
                 bos.unlink(abspath)
                 if xad:
-                    runhook(self.log, xad, abspath, vpath, "", uname, 0, 0, ip, 0, "")
+                    runhook(
+                        self.log,
+                        xad,
+                        abspath,
+                        vpath,
+                        "",
+                        uname,
+                        st.st_mtime,
+                        st.st_size,
+                        ip,
+                        0,
+                        "",
+                    )
 
-        ok: list[str] = []
-        ng: list[str] = []
         if is_dir:
             ok, ng = rmdirs(self.log_func, scandir, True, atop, 1)
+        else:
+            ok = ng = []
 
-        ok2, ng2 = rmdirs_up(os.path.dirname(atop), ptop)
+        if rm_up:
+            ok2, ng2 = rmdirs_up(os.path.dirname(atop), ptop)
+        else:
+            ok2 = ng2 = []
 
         return n_files, ok + ok2, ng + ng2
 
     def handle_mv(self, uname: str, svp: str, dvp: str) -> str:
+        if svp == dvp or dvp.startswith(svp + "/"):
+            raise Pebkac(400, "mv: cannot move parent into subfolder")
+
         svn, srem = self.asrv.vfs.get(svp, uname, True, False, True)
         svn, srem = svn.get_dbv(srem)
         sabs = svn.canonical(srem, False)
@@ -3090,8 +3251,21 @@ class Up2k(object):
 
             curs.clear()
 
-        rmdirs(self.log_func, scandir, True, sabs, 1)
-        rmdirs_up(os.path.dirname(sabs), svn.realpath)
+        rm_ok, rm_ng = rmdirs(self.log_func, scandir, True, sabs, 1)
+
+        for zsl in (rm_ok, rm_ng):
+            for ap in reversed(zsl):
+                if not ap.startswith(sabs):
+                    raise Pebkac(500, "mv_d: bug at {}, top {}".format(ap, sabs))
+
+                rem = ap[len(sabs) :].replace(os.sep, "/").lstrip("/")
+                vp = vjoin(dvp, rem)
+                try:
+                    dvn, drem = self.asrv.vfs.get(vp, uname, False, True)
+                    bos.mkdir(dvn.canonical(drem))
+                except:
+                    pass
+
         return "k"
 
     def _mv_file(
@@ -3182,7 +3356,7 @@ class Up2k(object):
             if c2 and c2 != c1:
                 self._copy_tags(c1, c2, w)
 
-            self._forget_file(svn.realpath, srem, c1, w, c1 != c2)
+            self._forget_file(svn.realpath, srem, c1, w, c1 != c2, fsize)
             self._relink(w, svn.realpath, srem, dabs)
             curs.add(c1)
 
@@ -3258,6 +3432,7 @@ class Up2k(object):
         cur: Optional["sqlite3.Cursor"],
         wark: Optional[str],
         drop_tags: bool,
+        sz: int,
     ) -> None:
         """forgets file in db, fixes symlinks, does not delete"""
         srd, sfn = vsplit(vrem)
@@ -3272,7 +3447,7 @@ class Up2k(object):
                 q = "delete from mt where w=?"
                 cur.execute(q, (wark[:16],))
 
-            self.db_rm(cur, srd, sfn)
+            self.db_rm(cur, srd, sfn, sz)
 
         reg = self.registry.get(ptop)
         if reg:

copyparty/util.py

@@ -171,6 +171,7 @@ HTTPCODE = {
     500: "Internal Server Error",
     501: "Not Implemented",
     503: "Service Unavailable",
+    999: "MissingNo",
 }
 
 
@@ -296,11 +297,11 @@ REKOBO_LKEY = {k.lower(): v for k, v in REKOBO_KEY.items()}
 pybin = sys.executable or ""
 if EXE:
     pybin = ""
-    for p in "python3 python".split():
+    for zsg in "python3 python".split():
         try:
-            p = shutil.which(p)
-            if p:
-                pybin = p
+            zsg = shutil.which(zsg)
+            if zsg:
+                pybin = zsg
                 break
         except:
             pass
@@ -537,7 +538,7 @@ class _Unrecv(object):
         self.log = log
         self.buf: bytes = b""
 
-    def recv(self, nbytes: int) -> bytes:
+    def recv(self, nbytes: int, spins: int = 1) -> bytes:
         if self.buf:
             ret = self.buf[:nbytes]
             self.buf = self.buf[nbytes:]
@@ -548,6 +549,10 @@ class _Unrecv(object):
                 ret = self.s.recv(nbytes)
                 break
             except socket.timeout:
+                spins -= 1
+                if spins <= 0:
+                    ret = b""
+                    break
                 continue
             except:
                 ret = b""
@@ -590,7 +595,7 @@ class _LUnrecv(object):
         self.log = log
         self.buf = b""
 
-    def recv(self, nbytes: int) -> bytes:
+    def recv(self, nbytes: int, spins: int) -> bytes:
         if self.buf:
             ret = self.buf[:nbytes]
             self.buf = self.buf[nbytes:]
@@ -609,7 +614,7 @@ class _LUnrecv(object):
     def recv_ex(self, nbytes: int, raise_on_trunc: bool = True) -> bytes:
         """read an exact number of bytes"""
         try:
-            ret = self.recv(nbytes)
+            ret = self.recv(nbytes, 1)
             err = False
         except:
             ret = b""
@@ -617,7 +622,7 @@ class _LUnrecv(object):
 
         while not err and len(ret) < nbytes:
             try:
-                ret += self.recv(nbytes - len(ret))
+                ret += self.recv(nbytes - len(ret), 1)
             except OSError:
                 err = True
 
@@ -921,7 +926,8 @@ class Magician(object):
 class Garda(object):
     """ban clients for repeated offenses"""
 
-    def __init__(self, cfg: str) -> None:
+    def __init__(self, cfg: str, uniq: bool = True) -> None:
+        self.uniq = uniq
         try:
             a, b, c = cfg.strip().split(",")
             self.lim = int(a)
@@ -967,7 +973,7 @@ class Garda(object):
             # assume /64 clients; drop 4 groups
             ip = IPv6Address(ip).exploded[:-20]
 
-        if prev:
+        if prev and self.uniq:
             if self.prev.get(ip) == prev:
                 return 0, ip
 
@@ -1223,12 +1229,15 @@ def ren_open(
         except OSError as ex_:
             ex = ex_
 
-            if ex.errno == errno.EINVAL and not asciified:
+            # EPERM: android13
+            if ex.errno in (errno.EINVAL, errno.EPERM) and not asciified:
                 asciified = True
-                bname, fname = [
-                    zs.encode("ascii", "replace").decode("ascii").replace("?", "_")
-                    for zs in [bname, fname]
-                ]
+                zsl = []
+                for zs in (bname, fname):
+                    zs = zs.encode("ascii", "replace").decode("ascii")
+                    zs = re.sub(r"[^][a-zA-Z0-9(){}.,+=!-]", "_", zs)
+                    zsl.append(zs)
+                bname, fname = zsl
                 continue
 
             # ENOTSUP: zfs on ubuntu 20.04
@@ -1292,7 +1301,7 @@ class MultipartParser(object):
         rfc1341/rfc1521/rfc2047/rfc2231/rfc2388/rfc6266/the-real-world
         (only the fallback non-js uploader relies on these filenames)
         """
-        for ln in read_header(self.sr):
+        for ln in read_header(self.sr, 2, 2592000):
             self.log(ln)
 
             m = self.re_ctype.match(ln)
@@ -1439,7 +1448,7 @@ class MultipartParser(object):
         for buf in iterable:
             ret += buf
             if len(ret) > max_len:
-                raise Pebkac(400, "field length is too long")
+                raise Pebkac(422, "field length is too long")
 
         return ret
 
@@ -1492,15 +1501,15 @@ def get_boundary(headers: dict[str, str]) -> str:
     return m.group(2)
 
 
-def read_header(sr: Unrecv) -> list[str]:
+def read_header(sr: Unrecv, t_idle: int, t_tot: int) -> list[str]:
     t0 = time.time()
     ret = b""
     while True:
-        if time.time() - t0 > 120:
+        if time.time() - t0 >= t_tot:
             return []
 
         try:
-            ret += sr.recv(1024)
+            ret += sr.recv(1024, t_idle // 2)
         except:
             if not ret:
                 return []
@@ -1549,7 +1558,7 @@ def rand_name(fdir: str, fn: str, rnd: int) -> str:
 def gen_filekey(salt: str, fspath: str, fsize: int, inode: int) -> str:
     return base64.urlsafe_b64encode(
         hashlib.sha512(
-            "{} {} {} {}".format(salt, fspath, fsize, inode).encode("utf-8", "replace")
+            ("%s %s %s %s" % (salt, fspath, fsize, inode)).encode("utf-8", "replace")
         ).digest()
     ).decode("ascii")
 
@@ -1589,7 +1598,7 @@ def gen_filekey_dbg(
 
 
 def gencookie(k: str, v: str, r: str, tls: bool, dur: Optional[int]) -> str:
-    v = v.replace(";", "")
+    v = v.replace("%", "%25").replace(";", "%3B")
     if dur:
         exp = formatdate(time.time() + dur, usegmt=True)
     else:
@@ -1622,7 +1631,12 @@ def unhumanize(sz: str) -> int:
         pass
 
     mc = sz[-1:].lower()
-    mi = {"k": 1024, "m": 1024 * 1024, "g": 1024 * 1024 * 1024}.get(mc, 1)
+    mi = {
+        "k": 1024,
+        "m": 1024 * 1024,
+        "g": 1024 * 1024 * 1024,
+        "t": 1024 * 1024 * 1024 * 1024,
+    }.get(mc, 1)
     return int(float(sz[:-1]) * mi)
 
 
@@ -1658,7 +1672,7 @@ def uncyg(path: str) -> str:
     if len(path) > 2 and path[2] != "/":
         return path
 
-    return "{}:\\{}".format(path[1], path[3:])
+    return "%s:\\%s" % (path[1], path[3:])
 
 
 def undot(path: str) -> str:
@@ -1701,7 +1715,7 @@ def sanitize_fn(fn: str, ok: str, bad: list[str]) -> str:
 
         bad = ["con", "prn", "aux", "nul"]
         for n in range(1, 10):
-            bad += "com{0} lpt{0}".format(n).split(" ")
+            bad += ("com%s lpt%s" % (n, n)).split(" ")
 
         if fn.lower().split(".")[0] in bad:
             fn = "_" + fn
@@ -2015,6 +2029,8 @@ def shut_socket(log: "NamedLogger", sck: socket.socket, timeout: int = 3) -> Non
             sck.shutdown(socket.SHUT_RDWR)
         except:
             pass
+    except Exception as ex:
+        log("shut({}): {}".format(fd, ex), "90")
     finally:
         td = time.time() - t0
         if td >= 1:
@@ -2109,7 +2125,7 @@ def list_ips() -> list[str]:
 def yieldfile(fn: str) -> Generator[bytes, None, None]:
     with open(fsenc(fn), "rb", 512 * 1024) as f:
         while True:
-            buf = f.read(64 * 1024)
+            buf = f.read(128 * 1024)
             if not buf:
                 break
 
@@ -2266,7 +2282,7 @@ def rmdirs(
     dirs = [os.path.join(top, x) for x in dirs]
     ok = []
     ng = []
-    for d in dirs[::-1]:
+    for d in reversed(dirs):
         a, b = rmdirs(logger, scandir, lstat, d, depth + 1)
         ok += a
         ng += b
@@ -2316,7 +2332,7 @@ def unescape_cookie(orig: str) -> str:
                     ret += chr(int(esc[1:], 16))
                 except:
                     ret += esc
-                    esc = ""
+                esc = ""
 
         else:
             ret += ch
@@ -2416,7 +2432,7 @@ def killtree(root: int) -> None:
 
 
 def runcmd(
-    argv: Union[list[bytes], list[str]], timeout: Optional[int] = None, **ka: Any
+    argv: Union[list[bytes], list[str]], timeout: Optional[float] = None, **ka: Any
 ) -> tuple[int, str, str]:
     kill = ka.pop("kill", "t")  # [t]ree [m]ain [n]one
     capture = ka.pop("capture", 3)  # 0=none 1=stdout 2=stderr 3=both
@@ -2469,7 +2485,7 @@ def chkcmd(argv: Union[list[bytes], list[str]], **ka: Any) -> tuple[str, str]:
     return sout, serr
 
 
-def mchkcmd(argv: Union[list[bytes], list[str]], timeout: int = 10) -> None:
+def mchkcmd(argv: Union[list[bytes], list[str]], timeout: float = 10) -> None:
     if PY2:
         with open(os.devnull, "wb") as f:
             rv = sp.call(argv, stdout=f, stderr=f)
@@ -2716,6 +2732,34 @@ def runhook(
     return True
 
 
+def loadpy(ap: str, hot: bool) -> Any:
+    """
+    a nice can of worms capable of causing all sorts of bugs
+    depending on what other inconveniently named files happen
+    to be in the same folder
+    """
+    if ap.startswith("~"):
+        ap = os.path.expanduser(ap)
+
+    mdir, mfile = os.path.split(absreal(ap))
+    mname = mfile.rsplit(".", 1)[0]
+    sys.path.insert(0, mdir)
+
+    if PY2:
+        mod = __import__(mname)
+        if hot:
+            reload(mod)
+    else:
+        import importlib
+
+        mod = importlib.import_module(mname)
+        if hot:
+            importlib.reload(mod)
+
+    sys.path.remove(mdir)
+    return mod
+
+
 def gzip_orig_sz(fn: str) -> int:
     with open(fsenc(fn), "rb") as f:
         f.seek(-4, 2)

copyparty/web/Makefile

@@ -6,7 +6,7 @@ pk: $(addsuffix .gz, $(wildcard *.js *.css))
 un: $(addsuffix .un, $(wildcard *.gz))
 
 %.gz: %
-	pigz -11 -J 34 -I 5730 $<
+	pigz -11 -J 34 -I 573 $<
 
 %.un: %
 	pigz -d $<

copyparty/web/a/u2c.py

@@ -0,0 +1 @@
+../../../bin/u2c.py

copyparty/web/a/up2k.py

@@ -1 +0,0 @@
-../../../bin/up2k.py

copyparty/web/baguettebox.js

@@ -127,7 +127,7 @@ window.baguetteBox = (function () {
             var gallery = [];
             [].forEach.call(tagsNodeList, function (imageElement, imageIndex) {
                 var imageElementClickHandler = function (e) {
-                    if (ctrl(e))
+                    if (ctrl(e) || e && e.shiftKey)
                         return true;
 
                     e.preventDefault ? e.preventDefault() : e.returnValue = false;
@@ -261,7 +261,7 @@ window.baguetteBox = (function () {
             setloop(1);
         else if (k == "BracketRight")
             setloop(2);
-        else if (e.shiftKey)
+        else if (e.shiftKey && k != 'KeyR')
             return;
         else if (k == "ArrowLeft" || k == "KeyJ")
             showPreviousImage();
@@ -310,7 +310,7 @@ window.baguetteBox = (function () {
         options = {};
         setOptions(o);
         if (tt.en)
-            tt.show.bind(this)();
+            tt.show.call(this);
     }
 
     function setVmode() {
@@ -356,7 +356,7 @@ window.baguetteBox = (function () {
 
         setVmode();
         if (tt.en)
-            tt.show.bind(this)();
+            tt.show.call(this);
     }
 
     function findfile() {
@@ -376,7 +376,12 @@ window.baguetteBox = (function () {
             else
                 (vid() || ebi('bbox-overlay')).requestFullscreen();
         }
-        catch (ex) { alert(ex); }
+        catch (ex) {
+            if (IPHONE)
+                alert('sorry, apple decided to make this impossible on iphones (should work on ipad tho)');
+            else
+                alert(ex);
+        }
     }
 
     function tglsel() {
@@ -968,7 +973,7 @@ window.baguetteBox = (function () {
             clmod(btnPrev, 'off', 't');
             clmod(btnNext, 'off', 't');
 
-            if (Date.now() - ctime <= 500)
+            if (Date.now() - ctime <= 500 && !IPHONE)
                 tglfull();
 
             ctime = Date.now();

copyparty/web/browser.css

@@ -55,6 +55,7 @@
 	--u2-sbtn-b1: #999;
 	--u2-txt-bg: var(--bg-u5);
 	--u2-tab-bg: linear-gradient(to bottom, var(--bg), var(--bg-u1));
+	--u2-tab-b1: rgba(128,128,128,0.8);
 	--u2-tab-1-fg: #fd7;
 	--u2-tab-1-bg: linear-gradient(to bottom, var(#353), var(--bg) 80%);
 	--u2-tab-1-b1: #7c5;
@@ -270,6 +271,7 @@ html.bz {
 	--btn-1h-fg: #000;
 	--txt-sh: a;
 
+	--u2-tab-b1: var(--bg-u5);
 	--u2-tab-1-fg: var(--fg-max);
 	--u2-tab-1-bg: var(--bg);
 
@@ -329,6 +331,7 @@ html.c {
 html.cz {
 	--bgg: var(--bg-u2);
 	--srv-3: #fff;
+	--u2-tab-b1: var(--bg-d3);
 }
 html.cy {
 	--fg: #fff;
@@ -411,10 +414,11 @@ html.dz {
 	--op-aa-bg: var(--bg-d2);
 	--op-a-sh: rgba(0,0,0,0.5);
 
-	--u2-btn-b1: #999;
-	--u2-sbtn-b1: #999;
+	--u2-btn-b1: var(--fg-weak);
+	--u2-sbtn-b1: var(--fg-weak);
 	--u2-txt-bg: var(--bg-u5);
 	--u2-tab-bg: linear-gradient(to bottom, var(--bg), var(--bg-u1));
+	--u2-tab-b1: var(--fg-weak);
 	--u2-tab-1-fg: #fff;
 	--u2-tab-1-bg: linear-gradient(to bottom, var(#353), var(--bg) 80%);
 	--u2-tab-1-b1: #7c5;
@@ -423,6 +427,12 @@ html.dz {
 	--u2-b-fg: #fff;
 	--u2-b1-bg: #3a3;
 	--u2-b2-bg: #3a3;
+	--u2-o-bg: var(--btn-bg);
+	--u2-o-b1: var(--bg-u5);
+	--u2-o-h-bg: var(--fg-weak);
+	--u2-o-1-bg: var(--fg-weak);
+	--u2-o-1-b1: var(--a);
+	--u2-o-1h-bg: var(--a);
 	--u2-inf-bg: #07a;
 	--u2-inf-b1: #0be;
 	--u2-ok-bg: #380;
@@ -483,6 +493,7 @@ html.dz {
 	--err-ts: #500;
 
 	text-shadow: none;
+	font-family: 'scp', monospace, monospace;
 }
 html.dy {
 	--fg: #000;
@@ -721,14 +732,13 @@ html.y #files thead th {
 	margin: 0;
 	padding: .3em .5em;
 	background: var(--bg);
+	max-width: var(--file-td-w);
+	word-wrap: break-word;
+	overflow: hidden;
 }
 #files tr:nth-child(2n) td {
 	background: var(--row-alt);
 }
-#files td+td+td {
-	max-width: 30em;
-	overflow: hidden;
-}
 #files td+td {
 	box-shadow: 1px 0 0 0 rgba(128,128,128,var(--f-sh1)) inset, 0 1px 0 rgba(255,255,255,var(--f-sh2)) inset, 0 -1px 0 rgba(255,255,255,var(--f-sh2)) inset;
 }
@@ -851,7 +861,7 @@ html.y #path a:hover {
 }
 .mdo,
 .mdo * {
-	line-height: 1.4em;
+	line-height: 1.5em;
 }
 #srv_info,
 #srv_info2,
@@ -1159,10 +1169,10 @@ html.y #widget.open {
 	background: #fff;
 	background: var(--bg-u3);
 }
-#wfm, #wzip, #wnp {
+#wfs, #wfm, #wzip, #wnp {
 	display: none;
 }
-#wzip, #wnp {
+#wfs, #wzip, #wnp {
 	margin-right: .2em;
 	padding-right: .2em;
 	border: 1px solid var(--bg-u5);
@@ -1174,6 +1184,7 @@ html.y #widget.open {
 	padding-left: .2em;
 	border-left-width: .1em;
 }
+#wfs.act,
 #wfm.act {
 	display: inline-block;
 }
@@ -1197,6 +1208,13 @@ html.y #widget.open {
 	position: relative;
 	display: inline-block;
 }
+#wfs {
+	font-size: .36em;
+	text-align: right;
+	line-height: 1.3em;
+	padding: 0 .3em 0 0;
+	border-width: 0 .25em 0 0;
+}
 #wfm span,
 #wnp span {
 	font-size: .6em;
@@ -1212,7 +1230,8 @@ html.y #widget.open {
 #wfm a.hide {
 	display: none;
 }
-#files tbody tr.fcut td {
+#files tbody tr.fcut td,
+#ggrid>a.fcut {
 	animation: fcut .5s ease-out;
 }
 @keyframes fcut {
@@ -1379,6 +1398,9 @@ input[type="checkbox"]:checked+label {
 	color: #0e0;
 	color: var(--a);
 }
+html.dz input {
+	font-family: 'scp', monospace, monospace;
+}
 .opwide div>span>input+label {
 	padding: .3em 0 .3em .3em;
 	margin: 0 0 0 -.3em;
@@ -1559,6 +1581,7 @@ html.cz .btn {
 	border-bottom: .2em solid #709;
 }
 html.dz .btn {
+	font-size: 1em;
 	box-shadow: 0 0 0 .1em #080 inset;
 }
 html.dz .tgl.btn.on {
@@ -1602,6 +1625,12 @@ html.cz .tgl.btn.on {
 	list-style: none;
 	border-top: 1px solid var(--bg-u5);
 }
+#tree li.offline>a:first-child:before {
+	content: '❌';
+	position: absolute;
+	margin-left: -.25em;
+	z-index: 3;
+}
 #tree ul a.sel {
 	background: #000;
 	background: var(--bg-d3);
@@ -1743,6 +1772,7 @@ html.y #tree.nowrap .ntree a+a:hover {
 	display: none;
 }
 .ghead {
+	background: var(--bg-u2);
 	border-radius: .3em;
 	padding: .2em .5em;
 	line-height: 2.3em;
@@ -2456,7 +2486,7 @@ html.y #bbox-overlay figcaption a {
 	width: 21em;
 }
 #u2cards {
-	padding: 1em 1em .3em 1em;
+	padding: 1em 1em .42em 1em;
 	margin: 0 auto;
 	white-space: nowrap;
 	text-align: center;
@@ -2481,7 +2511,8 @@ html.y #bbox-overlay figcaption a {
 #u2cards a {
 	padding: .2em 1em;
 	background: var(--u2-tab-bg);
-	border: 1px solid rgba(128,128,128,0.8);
+	border: 1px solid #999;
+	border-color: var(--u2-tab-b1);
 	border-width: 0 0 1px 0;
 }
 #u2cards a:first-child {
@@ -2740,6 +2771,9 @@ html.c .opbox,
 html.a .opbox {
 	margin: 1.5em 0 0 0;
 }
+html.dz .opview input.i {
+	width: calc(100% - 17em);
+}
 html.c #tree,
 html.c #treeh,
 html.a #tree,
@@ -2792,6 +2826,9 @@ html.a #u2btn {
 html.ay #u2btn {
 	box-shadow: .4em .4em 0 #ccc;
 }
+html.dz #u2btn {
+	letter-spacing: -.033em;
+}
 html.c #u2conf.ww #u2btn,
 html.a #u2conf.ww #u2btn {
 	margin: -2em .5em -3em 0;
@@ -2939,6 +2976,7 @@ html.b #treepar {
 html.b #wrap {
 	margin-top: 2em;
 }
+html.by .ghead,
 html.bz .ghead {
 	background: var(--bg);
 	padding: .2em 0;

copyparty/web/browser.html

@@ -11,7 +11,7 @@
 	<link rel="stylesheet" media="screen" href="{{ r }}/.cpr/ui.css?_={{ ts }}">
 	<link rel="stylesheet" media="screen" href="{{ r }}/.cpr/browser.css?_={{ ts }}">
 	{%- if css %}
-	<link rel="stylesheet" media="screen" href="{{ css }}?_={{ ts }}">
+	<link rel="stylesheet" media="screen" href="{{ css }}_={{ ts }}">
 	{%- endif %}
 </head>
 
@@ -29,7 +29,7 @@
 
 	<div id="op_player" class="opview opbox opwide"></div>
 
-	<div id="op_bup" class="opview opbox act">
+	<div id="op_bup" class="opview opbox {% if not ls0 %}act{% endif %}">
 		<div id="u2err"></div>
 		<form method="post" enctype="multipart/form-data" accept-charset="utf-8" action="{{ url_suf }}">
 			<input type="hidden" name="act" value="bput" />
@@ -39,7 +39,7 @@
 		<a id="bbsw" href="?b=u" rel="nofollow"><br />switch to basic browser</a>
 	</div>
 
-	<div id="op_mkdir" class="opview opbox act">
+	<div id="op_mkdir" class="opview opbox {% if not ls0 %}act{% endif %}">
 		<form method="post" enctype="multipart/form-data" accept-charset="utf-8" action="{{ url_suf }}">
 			<input type="hidden" name="act" value="mkdir" />
 			📂<input type="text" name="name" class="i" placeholder="awesome mix vol.1">
@@ -55,7 +55,7 @@
 		</form>
 	</div>
 
-	<div id="op_msg" class="opview opbox act">
+	<div id="op_msg" class="opview opbox {% if not ls0 %}act{% endif %}">
 		<form method="post" enctype="application/x-www-form-urlencoded" accept-charset="utf-8" action="{{ url_suf }}">
 			📟<input type="text" name="msg" class="i" placeholder="lorem ipsum dolor sit amet">
 			<input type="submit" value="send msg to srv log">
@@ -138,9 +138,11 @@
 			TS = "{{ ts }}",
 			acct = "{{ acct }}",
 			perms = {{ perms }},
+			dgrid = {{ dgrid|tojson }},
 			themes = {{ themes }},
 			dtheme = "{{ dtheme }}",
 			srvinf = "{{ srv_info }}",
+			s_name = "{{ s_name }}",
 			lang = "{{ lang }}",
 			dfavico = "{{ favico }}",
 			def_hcols = {{ def_hcols|tojson }},
@@ -171,7 +173,7 @@
 	<script src="{{ r }}/.cpr/browser.js?_={{ ts }}"></script>
 	<script src="{{ r }}/.cpr/up2k.js?_={{ ts }}"></script>
 	{%- if js %}
-	<script src="{{ js }}?_={{ ts }}"></script>
+	<script src="{{ js }}_={{ ts }}"></script>
 	{%- endif %}
 </body>
 

copyparty/web/cf.html

@@ -3,7 +3,7 @@
 
 <head>
 	<meta charset="utf-8">
-	<title>{{ svcname }}</title>
+	<title>{{ s_doctitle }}</title>
 	<meta http-equiv="X-UA-Compatible" content="IE=edge">
 	<meta name="viewport" content="width=device-width, initial-scale=0.8">
 </head>

copyparty/web/md.js

@@ -212,6 +212,8 @@ function convert_markdown(md_text, dest_dom) {
 
     try {
         var md_html = marked.parse(md_text, marked_opts);
+        if (!have_emp)
+            md_html = DOMPurify.sanitize(md_html);
     }
     catch (ex) {
         if (ext)

copyparty/web/md2.js

@@ -278,6 +278,7 @@ function Modpoll() {
             return;
 
         var new_md = this.responseText,
+            new_mt = this.getResponseHeader('X-Lastmod3') || r.lastmod,
             server_ref = server_md.replace(/\r/g, ''),
             server_now = new_md.replace(/\r/g, '');
 
@@ -285,6 +286,7 @@ function Modpoll() {
         if (r.initial && server_ref != server_now)
             return modal.confirm('Your browser decided to show an outdated copy of the document!\n\nDo you want to load the latest version from the server instead?', function () {
                 dom_src.value = server_md = new_md;
+                last_modified = new_mt;
                 draw_md();
             }, null);
 

copyparty/web/msg.html

@@ -3,7 +3,7 @@
 
 <head>
 	<meta charset="utf-8">
-	<title>{{ svcname }}</title>
+	<title>{{ s_doctitle }}</title>
 	<meta http-equiv="X-UA-Compatible" content="IE=edge">
 	<meta name="viewport" content="width=device-width, initial-scale=0.8">
 	<meta name="theme-color" content="#333">

copyparty/web/splash.html

@@ -3,7 +3,7 @@
 
 <head>
 	<meta charset="utf-8">
-	<title>{{ svcname }}</title>
+	<title>{{ s_doctitle }}</title>
 	<meta http-equiv="X-UA-Compatible" content="IE=edge">
 	<meta name="viewport" content="width=device-width, initial-scale=0.8">
 	<meta name="theme-color" content="#333">

copyparty/web/svcs.html

@@ -3,7 +3,7 @@
 
 <head>
 	<meta charset="utf-8">
-	<title>{{ args.doctitle }} @ {{ args.name }}</title>
+	<title>{{ s_doctitle }}</title>
 	<meta http-equiv="X-UA-Compatible" content="IE=edge">
 	<meta name="viewport" content="width=device-width, initial-scale=0.8">
 	<meta name="theme-color" content="#333">
@@ -181,7 +181,7 @@
         <p><em>note: if you are on LAN (or just dont have valid certificates), add <code>-td</code></em></p>
         {% endif %}
         <p>
-            you can use <a href="{{ r }}/.cpr/a/up2k.py">up2k.py</a> to upload (sometimes faster than web-browsers)
+            you can use <a href="{{ r }}/.cpr/a/u2c.py">u2c.py</a> to upload (sometimes faster than web-browsers)
         </p>
 
 

copyparty/web/ui.css

@@ -4,6 +4,8 @@
 	src: local('Source Code Pro Regular'), local('SourceCodePro-Regular'), url(deps/scp.woff2) format('woff2');
 }
 html {
+	text-size-adjust: 100%;
+	-webkit-text-size-adjust: 100%;
 	touch-action: manipulation;
 }
 #tt, #toast {
@@ -73,7 +75,7 @@ html {
 #toastb {
 	max-height: 70vh;
 	overflow-y: auto;
-	padding: 1px;
+	padding: .1em;
 }
 #toast.scroll #toastb {
 	overflow-y: scroll;

copyparty/web/up2k.js

@@ -723,7 +723,7 @@ function Donut(uc, st) {
 
     function strobe() {
         var txt = strobes.pop();
-        wintitle(txt);
+        wintitle(txt, false);
         if (!txt)
             clearInterval(tstrober);
     }
@@ -971,7 +971,7 @@ function up2k_init(subtle) {
         if (++nenters <= 0)
             nenters = 1;
 
-        if (onover.bind(this)(e))
+        if (onover.call(this, e))
             return true;
 
         var mup, up = QS('#up_zd');
@@ -995,16 +995,29 @@ function up2k_init(subtle) {
     function onoverb(e) {
         // zones are alive; disable cuo2duo branch
         document.body.ondragover = document.body.ondrop = null;
-        return onover.bind(this)(e);
+        return onover.call(this, e);
     }
     function onover(e) {
+        return onovercmn(this, e, false);
+    }
+    function onoverbtn(e) {
+        return onovercmn(this, e, true);
+    }
+    function onovercmn(self, e, btn) {
         try {
             var ok = false, dt = e.dataTransfer.types;
             for (var a = 0; a < dt.length; a++)
                 if (dt[a] == 'Files')
                     ok = true;
-                else if (dt[a] == 'text/uri-list')
-                    return true;
+                else if (dt[a] == 'text/uri-list') {
+                    if (btn) {
+                        ok = true;
+                        if (toast.txt == L.u_uri)
+                            toast.hide();
+                    }
+                    else
+                        return toast.inf(10, L.u_uri) || true;
+                }
 
             if (!ok)
                 return true;
@@ -1020,8 +1033,11 @@ function up2k_init(subtle) {
             document.body.ondragenter = document.body.ondragleave = document.body.ondragover = null;
             return modal.alert('your browser does not support drag-and-drop uploading');
         }
+        if (btn)
+            return;
+
         clmod(ebi('drops'), 'vis', 1);
-        var v = this.getAttribute('v');
+        var v = self.getAttribute('v');
         if (v)
             clmod(ebi(v), 'hl', 1);
     }
@@ -1045,6 +1061,8 @@ function up2k_init(subtle) {
     document.body.ondragleave = offdrag;
     document.body.ondragover = onover;
     document.body.ondrop = gotfile;
+    ebi('u2btn').ondrop = gotfile;
+    ebi('u2btn').ondragover = onoverbtn;
 
     var drops = [ebi('up_dz'), ebi('srch_dz')];
     for (var a = 0; a < 2; a++) {
@@ -1088,7 +1106,7 @@ function up2k_init(subtle) {
     function gotfile(e) {
         ev(e);
         nenters = 0;
-        offdrag.bind(this)();
+        offdrag.call(this);
         var dz = this && this.getAttribute('id');
         if (!dz && e && e.clientY)
             // cuo2duo fallback
@@ -1132,7 +1150,7 @@ function up2k_init(subtle) {
                 dst = good_files;
 
             if (is_itemlist) {
-                if (fobj.kind !== 'file')
+                if (fobj.kind !== 'file' && fobj.type !== 'text/uri-list')
                     continue;
 
                 try {
@@ -1144,6 +1162,8 @@ function up2k_init(subtle) {
                 }
                 catch (ex) { }
                 fobj = fobj.getAsFile();
+                if (!fobj)
+                    continue;
             }
             try {
                 if (fobj.size < 1)
@@ -1826,6 +1846,7 @@ function up2k_init(subtle) {
 
         timer.rm(etafun);
         timer.rm(donut.do);
+        ebi('u2tabw').style.minHeight = '0px';
         utw_minh = 0;
     }
 
@@ -2609,8 +2630,7 @@ function up2k_init(subtle) {
             }
         }
     }
-    window.addEventListener('resize', onresize);
-    onresize();
+    onresize100.add(onresize, true);
 
     if (MOBILE) {
         // android-chrome wobbles for a bit; firefox / iOS-safari are OK
@@ -2678,6 +2698,11 @@ function up2k_init(subtle) {
     }
 
     function draw_turbo() {
+        if (turbolvl < 0 && uc.turbo) {
+            bcfg_set('u2turbo', uc.turbo = false);
+            toast.err(10, "turbo is disabled in server config");
+        }
+
         var msg = (turbolvl || !uc.turbo) ? null : uc.fsearch ? L.u_ts : L.u_tu,
             html = ebi('u2foot').innerHTML;
 

copyparty/web/util.js

@@ -7,6 +7,7 @@ if (!window.console || !console.log)
 
 
 var wah = '',
+    NOAC = 'autocorrect="off" autocapitalize="off"',
     L, tt, treectl, thegrid, up2k, asmCrypto, hashwasm, vbar, marked,
     CB = '?_=' + Date.now(),
     R = SR.slice(1),
@@ -159,8 +160,8 @@ function vis_exh(msg, url, lineNo, columnNo, error) {
     window.onerror = undefined;
     var html = [
         '<h1>you hit a bug!</h1>',
-        '<p style="font-size:1.3em;margin:0">try to <a href="#" onclick="localStorage.clear();location.reload();">reset copyparty settings</a> if you are stuck here, or <a href="#" onclick="ignex();">ignore this</a> / <a href="#" onclick="ignex(true);">ignore all</a> / <a href="?b=u">basic</a></p>',
-        '<p style="color:#fff">please send me a screenshot arigathanks gozaimuch: <a href="<ghi>" target="_blank">github issue</a> or <code>ed#2644</code></p>',
+        '<p style="font-size:1.3em;margin:0;line-height:2em">try to <a href="#" onclick="localStorage.clear();location.reload();">reset copyparty settings</a> if you are stuck here, or <a href="#" onclick="ignex();">ignore this</a> / <a href="#" onclick="ignex(true);">ignore all</a> / <a href="?b=u">basic</a></p>',
+        '<p style="color:#fff">please send me a screenshot arigathanks gozaimuch: <a href="<ghi>" target="_blank">new github issue</a></p>',
         '<p class="b">' + esc(url + ' @' + lineNo + ':' + columnNo), '<br />' + esc(String(msg)).replace(/\n/g, '<br />') + '</p>',
         '<p><b>UA:</b> ' + esc(navigator.userAgent + '')
     ];
@@ -225,7 +226,7 @@ function vis_exh(msg, url, lineNo, columnNo, error) {
                 '#exbox{background:#222;color:#ddd;font-family:sans-serif;font-size:0.8em;padding:0 1em 1em 1em;z-index:80386;position:fixed;top:0;left:0;right:0;bottom:0;width:100%;height:100%;overflow:auto;width:calc(100% - 2em)} ' +
                 '#exbox,#exbox *{line-height:1.5em;overflow-wrap:break-word} ' +
                 '#exbox code{color:#bf7;background:#222;padding:.1em;margin:.2em;font-size:1.1em;font-family:monospace,monospace} ' +
-                '#exbox a{text-decoration:underline;color:#fc0} ' +
+                '#exbox a{text-decoration:underline;color:#fc0;background:#222;border:none} ' +
                 '#exbox h1{margin:.5em 1em 0 0;padding:0} ' +
                 '#exbox p.b{border-top:1px solid #999;margin:1em 0 0 0;font-size:1em} ' +
                 '#exbox ul, #exbox li {margin:0 0 0 .5em;padding:0} ' +
@@ -368,6 +369,15 @@ function import_js(url, cb) {
 }
 
 
+function unsmart(txt) {
+    return !IPHONE ? txt : (txt.
+        replace(/[\u2014]/g, "--").
+        replace(/[\u2022]/g, "*").
+        replace(/[\u2018\u2019]/g, "'").
+        replace(/[\u201c\u201d]/g, '"'));
+}
+
+
 var crctab = (function () {
     var c, tab = [];
     for (var n = 0; n < 256; n++) {
@@ -742,7 +752,7 @@ function get_pwd() {
     if (pwd.length < 2)
         return null;
 
-    return pwd[1].split(';')[0];
+    return decodeURIComponent(pwd[1].split(';')[0]);
 }
 
 
@@ -1051,10 +1061,73 @@ function cliptxt(txt, ok) {
 }
 
 
+function Debounce(delay) {
+    var r = this;
+    r.delay = delay;
+    r.timer = 0;
+    r.t_hit = 0;
+    r.t_run = 0;
+    r.q = [];
+
+    r.add = function (fun, run) {
+        r.rm(fun);
+        r.q.push(fun);
+
+        if (run)
+            fun();
+    };
+
+    r.rm = function (fun) {
+        apop(r.q, fun);
+    };
+
+    r.run = function () {
+        if (crashed)
+            return;
+
+        r.t_run = Date.now();
+
+        var q = r.q.slice(0);
+        for (var a = 0; a < q.length; a++)
+            q[a]();
+    };
+
+    r.hit = function () {
+        if (crashed)
+            return;
+
+        var now = Date.now(),
+            td_hit = now - r.t_hit,
+            td_run = now - r.t_run;
+
+        if (td_run >= r.delay * 2)
+            r.t_run = now;
+
+        if (td_run >= r.delay && td_run <= r.delay * 2) {
+            // r.delay is also deadline
+            clearTimeout(r.timer);
+            return r.run();
+        }
+
+        if (td_hit < r.delay / 5)
+            return;
+
+        clearTimeout(r.timer);
+        r.timer = setTimeout(r.run, r.delay);
+        r.t_hit = now;
+    };
+};
+
+var onresize100 = new Debounce(100);
+window.addEventListener('resize', onresize100.hit);
+
+
 var timer = (function () {
     var r = {};
     r.q = [];
     r.last = 0;
+    r.fs = 0;
+    r.fc = 0;
 
     r.add = function (fun, run) {
         r.rm(fun);
@@ -1080,6 +1153,7 @@ var timer = (function () {
             q[a]();
 
         r.last = Date.now();
+        //r.fc++; if (r.last - r.fs >= 2000) { console.log(r.last - r.fs, r.fc); r.fs = r.last; r.fc = 0; }
     }
     setInterval(doevents, 100);
 
@@ -1104,7 +1178,7 @@ var tt = (function () {
     var prev = null;
     r.cshow = function () {
         if (this !== prev)
-            r.show.bind(this)();
+            r.show.call(this);
 
         prev = this;
     };
@@ -1116,7 +1190,7 @@ var tt = (function () {
             return;
 
         if (Date.now() - r.lvis < 400)
-            return r.show.bind(this)();
+            return r.show.call(this);
 
         tev = setTimeout(r.show.bind(this), 800);
         if (TOUCH)
@@ -1274,6 +1348,9 @@ var toast = (function () {
     r.visible = false;
     r.txt = null;
     r.tag = obj;  // filler value (null is scary)
+    r.p_txt = '';
+    r.p_sec = 0;
+    r.p_t = 0;
 
     function scrollchk() {
         if (scrolling)
@@ -1306,10 +1383,23 @@ var toast = (function () {
     };
 
     r.show = function (cl, sec, txt, tag) {
+        var same = r.visible && txt == r.p_txt && r.p_sec == sec,
+            delta = Date.now() - r.p_t;
+
+        if (same && delta < 100)
+            return;
+
+        r.p_txt = txt;
+        r.p_sec = sec;
+        r.p_t = Date.now();
+
         clearTimeout(te);
         if (sec)
             te = setTimeout(r.hide, sec * 1000);
 
+        if (same && delta < 1000)
+            return;
+
         if (txt.indexOf('<body>') + 1)
             txt = txt.slice(0, txt.indexOf('<')) + ' [...]';
 
@@ -1488,7 +1578,7 @@ var modal = (function () {
         cb_ok = cok;
         cb_ng = cng === undefined ? cok : null;
         cb_up = fun;
-        html += '<input id="modali" type="text" /><div id="modalb">' + ok_cancel + '</div>';
+        html += '<input id="modali" type="text" ' + NOAC + ' /><div id="modalb">' + ok_cancel + '</div>';
         r.show(html);
 
         ebi('modali').value = v || '';
@@ -1520,7 +1610,7 @@ function repl_load() {
             ret = [
                 'var v=Object.keys(localStorage); v.sort(); JSON.stringify(v)',
                 "for (var a of QSA('#files a[id]')) a.setAttribute('download','')",
-                'console.hist.slice(-10).join("\\n")'
+                'console.hist.slice(-50).join("\\n")'
             ];
 
         ipre.innerHTML = '<option value=""></option>';
@@ -1576,6 +1666,8 @@ function repl(e) {
         if (!cmd)
             return toast.inf(3, 'eval aborted');
 
+        cmd = unsmart(cmd);
+
         if (cmd.startsWith(',')) {
             evalex_fatal = true;
             return modal.alert(esc(eval(cmd.slice(1)) + ''));
@@ -1769,7 +1861,6 @@ function cprop(name) {
 
 
 function bchrome() {
-    console.log(document.documentElement.className);
     var v, o = QS('meta[name=theme-color]');
     if (!o)
         return;
@@ -1787,16 +1878,17 @@ function xhrchk(xhr, prefix, e404, lvl, tag) {
     if (xhr.status < 400 && xhr.status >= 200)
         return true;
 
-    if (xhr.status == 403)
+    var errtxt = (xhr.response && xhr.response.err) || xhr.responseText,
+        fun = toast[lvl || 'err'],
+        is_cf = /[Cc]loud[f]lare|>Just a mo[m]ent|#cf-b[u]bbles|Chec[k]ing your br[o]wser|\/chall[e]nge-platform|"chall[e]nge-error|nable Ja[v]aScript and cook/.test(errtxt);
+
+    if (xhr.status == 403 && !is_cf)
         return toast.err(0, prefix + (L && L.xhr403 || "403: access denied\n\ntry pressing F5, maybe you got logged out"), tag);
 
     if (xhr.status == 404)
         return toast.err(0, prefix + e404, tag);
 
-    var errtxt = (xhr.response && xhr.response.err) || xhr.responseText,
-        fun = toast[lvl || 'err'];
-
-    if (xhr.status == 503 && /[Cc]loud[f]lare|>Just a mo[m]ent|#cf-b[u]bbles|Chec[k]ing your br[o]wser/.test(errtxt)) {
+    if (is_cf && (xhr.status == 403 || xhr.status == 503)) {
         var now = Date.now(), td = now - cf_cha_t;
         if (td < 15000)
             return;

docs/changelog.md

@@ -1,3 +1,475 @@
+▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀  
+# 2023-0902-0018  `v1.9.4`  yes symlink times
+
+hello! it's been a while, an entire day even...
+
+## new features
+* download folder as tar.gz, tar.bz2, tar.xz
+  * single-threaded, so extremely slow, but nice for easily compressed data or challenged networks
+  * append `?tar=gz`, `?tar=bz2` or `?tar=xz` to a folder URL to do it
+  * default compression levels are gz:3, bz2:2, xz:1; override with `?tar=gz:9`
+
+# bugfixes
+* c1efd227b7377144a5760bc6cff64f4e87b626d9 symlink-deduplicated files got indexed with the wrong last-modified timestamp
+  * mostly inconsequential; would cause the dupe's uploader-ip to be forgotten on the next server restart since it would reindex to "fix" the timestamp
+* when linking [a search query](https://a.ocv.me/pub/#q=tags%20like%20soundsho*) it loads the results faster
+
+# other changes
+* update readme to mention that iPhones and iPads dislike the preload feature and respond by glitching the audio a bit when a song is exactly 20 seconds away from ending and yet how it's probably a bad idea to disable preloading since i bet it's load-bearing against other iOS bugs
+  * speaking of iPhones and iPads, the [previous version](https://github.com/9001/copyparty/releases/tag/v1.9.3) should have fixed album playback on those
+
+
+
+▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀  
+# 2023-0831-2211  `v1.9.3`  iOS and http fixes
+
+## new features
+* iPhones and iPads are now able to...
+  * 9986136dfb2364edb35aa9fbb87410641c6d6af3 play entire albums while the screen is off without the music randomly stopping
+    * apple keeps breaking AudioContext in new and interesting ways; time to give up (no more equalizer)
+  * 1c0d978979a703edeb792e552b18d3b7695b2d90 perform search queries and execude js code
+    * by translating [smart-quotes](https://stackoverflow.com/questions/48678359/ios-11-safari-html-disable-smart-punctuation) into regular `'` and `"` characters
+* python 3.12 support
+  * technically a bugfix since it was added [a year ago](https://github.com/9001/copyparty/commit/32e22dfe84d5e0b13914b4d0e15c1b8c9725a76d) way before the first py3.12 alpha was released but turns out i botched it, oh well
+* filter error messages so they never include the filesystem path where copyparty's python files reside
+* print more context in server logs if someone hits an unexpected permission-denied
+
+# bugfixes
+found some iffy stuff combing over the code but, as far as I can tell, luckily none of these were dangerous:
+* URL normalization was a bit funky, but it appears everything access-control-related was unaffected
+* some url parameters were double-decoded, causing the unpost filtering and file renaming to fail if the values contained `%`
+* clients could cause the server to return an invalid cache-control header, but newlines and control-characters got rejected correctly
+* minor cosmetics / qol fixes:
+  * reduced flickering on page load in chrome
+  * fixed some console spam in search results
+  * markdown documents now have the same line-height in directory listings and the editor
+
+
+
+▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀  
+# 2023-0826-2116  `v1.9.2`  bigger hammer
+
+## new features
+* more ways to automatically ban users! three new sensors, all default-enabled, giving a 1 day ban after 9 hits in 2 minutes:
+  * `--ban-403`: trying to access volumes that dont exist or require authentication
+  * `--ban-422`: invalid POST messages (from brutefocing POST parameters and such)
+  * `--ban-url`: URLs which 404 and also match `--sus-urls` (scanners/crawlers)
+  * if you want to run a vulnerability scan on copyparty, please just [download the server](https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.py) and do it locally! takes less than 30 seconds to set up, you get lower latency, and you won't be filling up the logfiles on the demo server with junk, thank you 🙏
+* more ban-related stuff,
+  * new global option `--nonsus-urls` specifies regex of URLs which are OK to 404 and shouldn't ban people
+  * `--turbo` now accepts the value `-1` which makes it impossible for clients to enable it, making `--ban-404` safe to use
+* range-selecting files in the list-view by shift-pgup/pgdn
+* volumes which are currently unavailable (dead nfs share, external HDD which is off, ...) are marked with a ❌ in the directory tree sidebar
+* the toggle-button to see dotfiles is now persisted as a cookie so it also applies on the initial page load
+* more effort is made to prevent `<script>`s inside markdown documents from running in the markdown editor and the fullpage viewer
+  * anyone who wanted to use markdown files for malicious stuff can still just upload an html file instead, so this doesn't make anything more secure, just less confusing
+  * the safest approach is still the `nohtml` volflag which disables markdown rendering outside sandboxes entirely, or only giving out write-access to trustworthy people
+  * enabling markdown plugins with `-emp` now has the side-effect of cancelling this band-aid too
+
+## bugfixes
+* textfile navigation hotkeys broke in the previous version
+
+## other changes
+* example [nginx config](https://github.com/9001/copyparty/blob/hovudstraum/contrib/nginx/copyparty.conf) was not compatible with cloudflare (suggest `$http_cf_connecting_ip` instead of `$proxy_add_x_forwarded_for`)
+* `copyparty.exe` is now built with python 3.11.5 which fixes [CVE-2023-40217](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-40217)
+  * `copyparty32.exe` is not, because python understandably ended win7 support 
+* [similar software](https://github.com/9001/copyparty/blob/hovudstraum/docs/versus.md):
+  * copyparty appears to be 30x faster than nextcloud and seafile at receiving uploads of many small files
+  * seafile has a size limit when zip-downloading folders
+
+
+
+▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀  
+# 2023-0820-2338  `v1.9.1`  prometheable
+
+## new features
+* #49 prometheus / grafana / openmetrics integration ([see readme](https://github.com/9001/copyparty#prometheus))
+  * read metrics from http://127.0.0.1:3923/.cpr/metrics after enabling with `--stats`
+* download a folder with all music transcoded to opus by adding `?tar=opus` or `?zip&opus` to the URL
+  * can also be used to download thumbnails instead of full images; `?tar=w` for webp, `?tar=j` for jpg
+    * so i guess the long-time requested feature of pre-generating thumbnails kind of happened after all, if you schedule a `curl http://127.0.0.1:3923/?tar=w >/dev/null` after server startup
+* u2c (commandline uploader): argument `-x` to exclude files by regex (compares absolute filesystem paths)
+* `--zm-spam 30` can be used to improve zeroconf / mDNS reliability on crazy networks
+  * only necessary if there are clients with multiple IPs and some of the IPs are outside the subnets that copyparty are in -- not spec-compliant, not really recommended, but shouldn't cause any issues either
+  * and `--mc-hop` wasn't actually implemented until now
+* dragging an image from another browser window onto the upload button is now possible
+  * only works on chrome, and only on windows or linux (not macos)
+* server hostname is prefixed in all window titles
+  * can be adjusted with `--bname` (the file explorer) and `--doctitle` (all other documents)
+  * can be disabled with `--nth` (just window title) or `--nih` (title + header)
+
+## bugfixes
+* docker: the autogenerated seeds for filekeys and account passwords now get persisted to the config volume (thx noktuas)
+* uploading files with fancy filenames could fail if the copyparty server is running on android
+* improve workarounds for some apple/iphone/ios jank (thx noktuas and spiky)
+  * some ui elements had their font-size selected by fair dice roll
+  * the volume control does nothing because [apple disabled it](https://developer.apple.com/library/archive/documentation/AudioVideo/Conceptual/Using_HTML5_Audio_Video/Device-SpecificConsiderations/Device-SpecificConsiderations.html#//apple_ref/doc/uid/TP40009523-CH5-SW11), so add a warning
+  * the image gallery cannot be fullscreened [as apple intended](https://developer.mozilla.org/en-US/docs/Web/API/Element/requestFullscreen#browser_compatibility) so add a warning
+
+## other changes
+* file table columns are now limited to browser window width
+* readme: mention that nginx-QUIC is currently very slow (thx noktuas)
+* #50 add a safeguard to the wget plugin in case wget at some point adds support for `file://` or similar
+* show a suggestion on startup to enable the database
+
+
+
+▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀  
+# 2023-0725-1550  `v1.8.8`  just boring bugfixes
+
+final release until late august unless something bad happens and i end up building this thing on a shinkansen
+
+## recent security / vulnerability fixes
+* there is a [discord server](https://discord.gg/25J8CdTT6G) with an `@everyone` in case of future important updates
+* [v1.8.7](https://github.com/9001/copyparty/releases/tag/v1.8.7) (2023-07-23) - [CVE-2023-38501](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-38501) - reflected XSS
+* [v1.8.2](https://github.com/9001/copyparty/releases/tag/v1.8.2) (2023-07-14) - [CVE-2023-37474](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-37474) - path traversal (first CVE)
+  * all serverlogs reviewed so far (5 public servers) showed no signs of exploitation
+
+## bugfixes
+* range-select with shiftclick:
+  * don't crash when entering another folder and shift-clicking some more
+  * remember selection origin when lazy-loading more stuff into the viewport
+* markdown editor:
+  * fix confusing warnings when the browser cache decides it *really* wants to cache
+  * and when a document starts with a newline
+* remember intended actions such as `?edit` on login prompts
+* Windows: TLS-cert generation (triggered by network changes) could occasionally fail
+
+
+
+▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀  
+# 2023-0723-1543  `v1.8.7`  XSS for days
+
+at the lack of better ideas, there is now a [discord server](https://discord.gg/25J8CdTT6G) with an `@everyone` for all future important updates such as this one
+
+## bugfixes
+* reflected XSS through `/?k304` and `/?setck`
+  * if someone tricked you into clicking a URL containing a chain of `%0d` and `%0a` they could potentially have moved/deleted existing files on the server, or uploaded new files, using your account
+  * if you use a reverse proxy, you can check if you have been exploited like so:
+    * nginx: grep your logs for URLs containing `%0d%0a%0d%0a`, for example using the following command:
+      ```bash
+      (gzip -dc access.log*.gz; cat access.log) | sed -r 's/" [0-9]+ .*//' | grep -iE '%0[da]%0[da]%0[da]%0[da]'
+      ```
+  * if you find any traces of exploitation (or just want to be on the safe side) it's recommended to change the passwords of your copyparty accounts
+  * huge thanks *again* to @TheHackyDog !
+* the original fix for CVE-2023-37474 broke the download links for u2c.py and partyfuse.py
+* fix mediaplayer spinlock if the server only has a single audio file
+
+
+
+▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀  
+# 2023-0721-0036  `v1.8.6`  fix reflected XSS
+
+## bugfixes
+* reflected XSS through `/?hc` (the optional subfolder parameter to the [connect](https://a.ocv.me/?hc) page)
+  * if someone tricked you into clicking `http://127.0.0.1:3923/?hc=<script>alert(1)</script>` they could potentially have moved/deleted existing files on the server, or uploaded new files, using your account
+  * if you use a reverse proxy, you can check if you have been exploited like so:
+    * nginx: grep your logs for URLs containing `?hc=` with `<` somewhere in its value, for example using the following command:
+      ```bash
+      (gzip -dc access.log*.gz; cat access.log) | sed -r 's/" [0-9]+ .*//' | grep -E '[?&](hc|pw)=.*[<>]'
+      ```
+  * if you find any traces of exploitation (or just want to be on the safe side) it's recommended to change the passwords of your copyparty accounts
+  * thanks again to @TheHackyDog !
+
+
+
+▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀  
+# 2023-0718-0746  `v1.8.4`  range-select v2
+
+**IMPORTANT:** `v1.8.2` (previous release) fixed [CVE-2023-37474](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-37474) ; please see the [1.8.2 release notes](https://github.com/9001/copyparty/releases/tag/v1.8.2) (all serverlogs reviewed so far showed no signs of exploitation)
+
+* read-only demo server at https://a.ocv.me/pub/demo/
+* [docker image](https://github.com/9001/copyparty/tree/hovudstraum/scripts/docker) ╱ [similar software](https://github.com/9001/copyparty/blob/hovudstraum/docs/versus.md) ╱ [client testbed](https://cd.ocv.me/b/)
+
+## new features
+* #47 file selection by shift-clicking
+  * in list-view: click a table row to select it, then shift-click another to select all files in-between
+  * in grid-view: either enable the `multiselect` button (mainly for phones/tablets), or the new `sel` button in the `[⚙️] settings` tab (better for mouse+keyboard), then shift-click two files
+* volflag `fat32` avoids a bug in android's sdcardfs causing excessive reindexing on startup if any files were modified on the sdcard since last reboot
+
+## bugfixes
+* minor corrections to the new features from #45
+  * uploader IPs are now visible for `a`dmin accounts in `d2t` volumes as well
+
+## other changes
+* the admin-panel is only accessible for accounts which have the `a` (admin) permission-level in one or more volumes; so instead of giving your user `rwmd` access, you'll want `rwmda` instead:
+  ```bash
+  python3 copyparty-sfx.py -a joe:hunter2 -v /mnt/nas/pub:pub:rwmda,joe
+  ```
+  or in a settings file,
+  ```yaml
+  [/pub]
+    /mnt/nas/pub
+    accs:
+      rwmda: joe
+  ```
+  * until now, `rw` was enough, however most readwrite users don't need access to those features
+  * grabbing a stacktrace with `?stack` is permitted for both `rw` and `a`
+
+
+
+▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀  
+# 2023-0714-1558  `v1.8.2`  URGENT: fix path traversal vulnerability
+
+* read-only demo server at https://a.ocv.me/pub/demo/
+* [docker image](https://github.com/9001/copyparty/tree/hovudstraum/scripts/docker) ╱ [similar software](https://github.com/9001/copyparty/blob/hovudstraum/docs/versus.md) ╱ [client testbed](https://cd.ocv.me/b/)
+
+Starting with the bad and important news; this release fixes https://github.com/9001/copyparty/security/advisories/GHSA-pxfv-7rr3-2qjg / [CVE-2023-37474](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-37474) -- so please upgrade!
+
+Every version until now had a [path traversal vulnerability](https://owasp.org/www-community/attacks/Path_Traversal) which allowed read-access to any file on the server's filesystem. To summarize,
+* Every file that the copyparty process had the OS-level permissions to read, could be retrieved over HTTP without password authentication
+* However, an attacker would need to know the full (or copyparty-module-relative) path to the file; it was luckily impossible to list directory contents to discover files on the server
+* You may have been running copyparty with some mitigations against this:
+  * [prisonparty](https://github.com/9001/copyparty/tree/hovudstraum/bin#prisonpartysh) limited the scope of access to files which were intentionally given to copyparty for sharing; meaning all volumes, as well as the following read-only filesystem locations: `/bin`, `/lib`, `/lib32`, `/lib64`, `/sbin`, `/usr`, `/etc/alternatives`
+  * the [nix package](https://github.com/9001/copyparty#nix-package) has a similar mitigation implemented using systemd concepts
+  * [docker containers](https://github.com/9001/copyparty/tree/hovudstraum/scripts/docker) would only expose the files which were intentionally mounted into the container, so even better
+* More conventional setups, such as just running the sfx (python or exe editions), would unfortunately expose all files readable by the current user
+* The following configurations would have made the impact much worse:
+  * running copyparty as root
+
+So, three years, and finally a CVE -- which has been there since day one... Not great huh. There is a list of all the copyparty alternatives that I know of in the `similar software` link above.
+
+Thanks for flying copyparty! And especially if you decide to continue doing so :-)
+
+## new features
+* #43 volflags to specify thumbnailer behavior per-volume;
+  * `--th-no-crop` / volflag `nocrop` to specify whether autocrop should be disabled
+  * `--th-size` / volflag `thsize` to set a custom thumbnail resolution
+  * `--th-convt` / volflag `convt` to specify conversion timeout
+* #45 resulted in a handful of opportunities to tighten security in intentionally-dangerous setups (public folders with anonymous uploads enabled):
+  * a new permission, `a` (in addition to the existing `rwmdgG`), to show the uploader-IP and upload-time for each file in the file listing
+    * accidentally incompatible with the `d2t` volflag (will be fixed in the next ver)
+  * volflag `nohtml` is a good defense against (un)intentional XSS; it returns HTML-files and markdown-files as plaintext instead of rendering them, meaning any malicious `<script>` won't run -- bad idea for regular use since it breaks fundamental functionality, but good when you really need it
+    * the README-previews below the file-listing still renders as usual, as this is fine thanks to the sandbox
+  * a new eventhook `--xban` to run a plugin when copyparty decides to ban someone (for password bruteforcing or excessive 404's), for example to blackhole the IP using fail2ban or similar
+
+## bugfixes
+* **fixes a path traversal vulnerability,** https://github.com/9001/copyparty/security/advisories/GHSA-pxfv-7rr3-2qjg / [CVE-2023-37474](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-37474)
+  * HUGE thanks to @TheHackyDog for reporting this !!
+  * if you use a reverse proxy, you can check if you have been exploited like so:
+    * nginx: grep your logs for URLs containing both `.cpr/` and `%2[^0]`, for example using the following command:
+      ```bash
+      (gzip -dc access.log.*.gz; cat access.log) | sed -r 's/" [0-9]+ .*//' | grep -E 'cpr/.*%2[^0]' | grep -vF data:image/svg
+      ```
+* 77f1e5144455eb946db7368792ea11c934f0f6da fixes an extremely unlikely race-condition (see the commit for details)
+* 8f59afb1593a75b8ce8c91ceee304097a07aea6e fixes another race-condition which is a bit worse:
+  * the unpost feature could collide with other database activity, with the worst-case outcome being aborted batch operations, for example a directory move or a batch-rename which stops halfways
+
+----
+
+# 💾 what to download?
+| download link | is it good? | description |
+| -- | -- | -- |
+| **[copyparty-sfx.py](https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.py)** | ✅ the best 👍 | runs anywhere! only needs python |
+| [a docker image](https://github.com/9001/copyparty/blob/hovudstraum/scripts/docker/README.md) | it's ok | good if you prefer docker 🐋 |
+| [copyparty.exe](https://github.com/9001/copyparty/releases/latest/download/copyparty.exe) |  ⚠️ [acceptable](https://github.com/9001/copyparty#copypartyexe) | for [win8](https://user-images.githubusercontent.com/241032/221445946-1e328e56-8c5b-44a9-8b9f-dee84d942535.png) or later; built-in thumbnailer |
+| [u2c.exe](https://github.com/9001/copyparty/releases/download/v1.7.1/u2c.exe) | ⚠️ acceptable | [CLI uploader](https://github.com/9001/copyparty/blob/hovudstraum/bin/u2c.py) as a win7+ exe ([video](https://a.ocv.me/pub/demo/pics-vids/u2cli.webm)) |
+| [copyparty32.exe](https://github.com/9001/copyparty/releases/latest/download/copyparty32.exe) | ⛔️ [dangerous](https://github.com/9001/copyparty#copypartyexe) | for [win7](https://user-images.githubusercontent.com/241032/221445944-ae85d1f4-d351-4837-b130-82cab57d6cca.png) -- never expose to the internet! |
+| [cpp-winpe64.exe](https://github.com/9001/copyparty/releases/download/v1.8.2/copyparty-winpe64.exe) | ⛔️ dangerous | runs on [64bit WinPE](https://user-images.githubusercontent.com/241032/205454984-e6b550df-3c49-486d-9267-1614078dd0dd.png), otherwise useless |
+
+* except for [u2c.exe](https://github.com/9001/copyparty/releases/download/v1.7.1/u2c.exe), all of the options above are equivalent
+* the zip and tar.gz files below are just source code
+* python packages are available at [PyPI](https://pypi.org/project/copyparty/#files)
+
+
+
+
+▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀  
+# 2023-0707-2220  `v1.8.1`  in case of 404
+
+## new features
+* [handlers](https://github.com/9001/copyparty/tree/hovudstraum/bin/handlers); change the behavior of 404 / 403 with plugins
+  * makes it possible to use copyparty as a [caching proxy](https://github.com/9001/copyparty/blob/hovudstraum/bin/handlers/caching-proxy.py)
+* #42 add mpv + streamlink support to [very-bad-idea](https://github.com/9001/copyparty/tree/hovudstraum/bin/mtag#dangerous-plugins)
+* add support for Pillow 10
+  * also improved text rendering in icons
+* mention the [fedora package](https://github.com/9001/copyparty#fedora-package) in the readme
+
+## bugfixes
+* theme 6 (hacker) didn't show the state of some toggle-switches
+* windows: keep quickedit enabled when hashing passwords interactively
+
+
+
+▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀  
+# 2023-0626-0005  `v1.8.0`  argon
+
+News: if you use rclone as a copyparty webdav client, upgrading to [rclone v1.63](https://github.com/rclone/rclone/releases/tag/v1.63.0) (just released) will give you [a huge speed boost](https://github.com/rclone/rclone/pull/6897) for small files
+
+## new features
+* #39 hashed passwords
+  * instead of keeping plaintext account passwords in config files, you can now store hashed ones instead
+  * `--ah-alg` specifies algorithm; best to worst: `argon2`, `scrypt`, `sha2`, or the default `none`
+  * the default settings of each algorithm takes `0.4 sec` to hash a password, and argon2 eats `256 MiB` RAM
+    * can be adjusted with optional comma-separated args after the algorithm name; see `--help-pwhash`
+  * `--ah-salt` is the [static salt](https://github.com/9001/copyparty/blob/hovudstraum/docs/devnotes.md#hashed-passwords) for all passwords, and is autogenerated-and-persisted if not specified
+  * `--ah-cli` switches copyparty into a shell where you can hash passwords interactively
+    * but copyparty will also autoconvert any unhashed passwords on startup and give you the values to insert into the config anyways
+* #40 volume size limit
+  * volflag `vmaxb` specifies max size of a volume
+  * volflag `vmaxn` specifies max number of files in a volume
+  * example: `-v [...]:c,vmaxb=900g:c,vmaxn=20k` blocks uploads if the volume reaches 900 GiB or a total of 20480 files
+  * good alternative to `--df` since it works per-volume
+
+## bugfixes
+* autogenerated TLS certs didn't include the mDNS name
+
+## other changes
+* improved cloudflare challenge detection
+* markdown edits will now trigger upload hooks
+
+
+
+▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀  
+# 2023-0611-0814  `v1.7.6`  NO_COLOR
+
+## new features
+* #31 `--grid` shows thumbnails instead of file-list by default
+* #28 `--unlist` regex-exclude files from browser listings
+  * for example `--unlist '\.(js|css)$'` hides all `.js` and `.css` files
+  * **purely cosmetic!** the files are still fully accessible, and still appear in API calls
+* auto-generate TLS certificates on startup / network-change
+  * mostly good for LAN, requires [cfssl](https://github.com/cloudflare/cfssl/releases/latest), can be disabled with `--no-crt`
+  * creates a self-signed CA and certs with SANs of all detected server IPs
+    * so it's still recommended to use a reverse-proxy / letsencrypt for WAN servers
+* the default `--fk-salt` is now much stronger
+  * all existing installations will keep the previously selected seed -- you can choose to upgrade by deleting `~/.config/copyparty/cert.pem` but this will change all filekeys / per-file passwords
+* the `NO_COLOR` environment-variable is now supported, removing colors from stdout
+  * see https://no-color.org/ and more importantly https://youtu.be/biW5UVGkPMA?t=150
+  * `--ansi` and `--no-ansi` can also be used to force-enable/disable colored output
+* #33 disable colors when stdout is redirected to a pipe/file -- by @clach04 
+* #32 simplify building sfx from source
+* upgraded [copyparty.exe](https://github.com/9001/copyparty/releases/latest/download/copyparty.exe) to [python 3.11.4](https://pythoninsider.blogspot.com/2023/06/python-3114-31012-3917-3817-3717-and.html)
+
+## bugfixes
+* #30 `--ftps` didn't work without `--ftp`
+* tiny css bug in light themes (opaque thumbnail controls)
+
+
+
+▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀  
+# 2023-0513-0000  `v1.7.2`  hard resolve
+
+## new features
+* print a warning if `c:\`, `c:\windows*`, or all of `/` are shared
+* upgraded the docker image to v3.18 which enables the [chiptune player](https://a.ocv.me/pub/demo/music/chiptunes/#af-f6fb2e5f)
+* in config files, allow trailing `:` in section headers
+
+## bugfixes
+* when `--hardlink` (or the volflag) is set, resolve symlinks before hardlinking
+  * uploads could fail due to relative symlinks
+* really minor ux fixes
+  * left-align `GET` in access logs
+  * the upload panel didn't always shrink back down after uploads completed
+
+
+
+▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀  
+# 2023-0507-1834  `v1.7.1`  CräzY;PWDs
+
+## new features
+* webdav:
+  * support write-only folders
+  * option `--dav-auth` / volflag `davauth` forces clients to always auth
+    * helps clients such as `davfs2` see all folders if the root is anon-readable but some subfolders are not
+    * alternatively you could configure your client to always send the password in the `PW` header
+* include usernames in http request logs
+* audio player:
+  * consumes less power on phones when the screen is off
+  * smoother playback cursor on short songs
+
+## bugfixes
+* the characters `;` and `%` can now be used in passwords
+  * but non-ascii characters (such as the ä in the release title) can, in fact, not
+* verify that all accounts have unique passwords on startup (#25)
+
+## other changes
+* ftpd: log incorrect passwords only, not correct ones
+* `up2k.py` (the upload, folder-sync, and file-search client) has been renamed to [u2c.py](https://github.com/9001/copyparty/tree/hovudstraum/bin#u2cpy)
+  * `u2c` as in `up2k client`, or `up2k CLI`, or `upload-to-copyparty` -- good name
+  * now the only things named "up2k" are the web-ui and the server backend which is way less confusing
+* upgrade packaging from [setup.py](https://github.com/9001/copyparty/blob/hovudstraum/setup.py) to [pyproject.toml](https://github.com/9001/copyparty/blob/hovudstraum/pyproject.toml)
+  * no practical consequences aside from a warm fuzzy feeling of being in the future
+* the docker images ~~will be~~ got rebuilt 2023-05-11 ~~in a few days (when [alpine](https://alpinelinux.org/) 3.18 is released)~~ enabling [the chiptune player](https://a.ocv.me/pub/demo/music/chiptunes/#af-f6fb2e5f)
+
+
+
+▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀  
+# 2023-0429-2114  `v1.7.0`  unlinked
+
+don't get excited! nothing new and revolutionary, but `xvol` and `xdev` changed behavior so there's an above-average chance of fresh bugs
+
+## new features
+* (#24): `xvol` and `xdev`, previously just hints to the filesystem indexer, now actively block access as well:
+  * `xvol` stops users following symlinks leaving the volumes they have access to
+    * so if you symlink `/home/ed/music` into `/srv/www/music` it'll get blocked
+    * ...unless both folders are accessible through volumes, and the user has read-access to both
+  * `xdev` stops users crossing the filesystem boundary of the volumes they have access to
+    * so if you symlink another HDD into a volume it'll get blocked, but you can still symlink from other places on the same FS
+  * enabling these will add a slight performance hit; the unlikely worst-case is `14%` slower directory listings, `35%` slower download-as-tar
+* file selection summary (num files, size, audio duration) in the bottom right
+* [u2cli](https://github.com/9001/copyparty/blob/hovudstraum/bin/up2k.py): more aggressive resolving with `--rh`
+* [add a warning](https://github.com/9001/copyparty#fix-unreliable-playback-on-android) that the default powersave settings in android may stop playing music during album changes
+  * also appears [in the media player](https://user-images.githubusercontent.com/241032/235327191-7aaefff9-5d41-4e42-b71f-042a8247f29d.png) if the issue is detected at runtime (playback halts for 30sec while screen is off)
+
+## bugfixes
+* (#23): stop autodeleting empty folders when moving or deleting files
+  * but files which expire / [self-destruct](https://github.com/9001/copyparty#self-destruct) still clean up parent directories like before
+* ftp-server: some clients could fail to `mkdir` at first attempt (and also complain during rmdir)
+
+## other changes
+* new version of [cpp-winpe64.exe](https://github.com/9001/copyparty/releases/download/v1.7.0/copyparty-winpe64.exe) since the ftp-server fix might be relevant
+
+
+
+▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀  
+# 2023-0426-2300  `v1.6.15`  unexpected boost
+
+## new features
+* 30% faster folder listings due to [the very last thing](https://github.com/9001/copyparty/commit/55c74ad164633a0a64dceb51f7f534da0422cbb5) i'd ever expect to be a bottleneck, [thx perf](https://docs.python.org/3.12/howto/perf_profiling.html)
+* option to see the lastmod timestamps of symlinks instead of the target files
+  * makes the turbo mode of [u2cli, the commandline uploader and folder-sync tool](https://github.com/9001/copyparty/blob/hovudstraum/bin/up2k.py) more turbo since copyparty dedupes uploads by symlinking to an existing copy and the symlink is stamped with the deduped file's lastmod
+  * **webdav:** enabled by default (because rclone will want this), can be disabled with arg `--dav-rt` or volflag `davrt`
+  * **http:** disabled by default, can be enabled per-request with urlparam `lt`
+* [u2cli](https://github.com/9001/copyparty/blob/hovudstraum/bin/up2k.py): option `--rh` to resolve server hostname only once at start of upload
+  * fantastic for buggy networks, but it'll break TLS
+
+## bugfixes
+* new arg `--s-tbody` specifies the network timeout before a dead connection gets dropped (default 3min)
+  * before there was no timeout at all, which could hang uploads or possibly consume all server resources
+  * ...but this is only relevant if your copyparty is directly exposed to the internet with no reverse proxy
+    * with nginx/caddy/etc you can disable the timeout with `--s-tbody 0` for a 3% performance boost (*wow!*)
+* iPhone audio transcoder could turn bad and stop transcoding
+* ~~maybe android phones no longer pause playback at the end of an album~~
+  * nope, that was due to [android's powersaver](https://github.com/9001/copyparty#fix-unreliable-playback-on-android), oh well
+  * ***bonus unintended feature:*** navigate into other folders while a song is plaing
+* [installing from the source tarball](https://github.com/9001/copyparty/blob/hovudstraum/docs/devnotes.md#build-from-release-tarball) should be ok now
+  * good base for making distro packages probably
+
+## other changes
+* since the network timeout fix is relevant for the single usecase that [cpp-winpe64.exe](https://github.com/9001/copyparty/releases/download/v1.6.15/copyparty-winpe64.exe) covers, there is now a new version of that
+
+
+
+▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀  
+# 2023-0424-0609  `v1.6.14`  unsettable flags
+
+## new features
+* unset a volflag (override a global option) by negating it (setting volflag `-flagname`)
+* new argument `--cert` to specify TLS certificate location
+  * defaults to `~/.config/copyparty/cert.pem` like before
+
+## bugfixes
+* in zip/tar downloads, always use the parent-folder name as the archive root
+* more reliable ftp authentication when providing password as username
+* connect-page: fix rclone ftps example
+
+## other changes
+* stop suggesting `--http-only` and `--https-only` for performance since the difference is negligible
+* mention how some antivirus (avast, avg, mcafee) thinks that pillow's webp encoder is a virus, affecting `copyparty.exe`
+
+
+
 ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀  
 # 2023-0420-2141  `v1.6.12`  as seen on nixos
 

docs/devnotes.md

@@ -4,8 +4,9 @@
 * [future plans](#future-plans) - some improvement ideas
 * [design](#design)
     * [up2k](#up2k) - quick outline of the up2k protocol
-    * [why not tus](#why-not-tus) - I didn't know about [tus](https://tus.io/)
-    * [why chunk-hashes](#why-chunk-hashes) - a single sha512 would be better, right?
+        * [why not tus](#why-not-tus) - I didn't know about [tus](https://tus.io/)
+        * [why chunk-hashes](#why-chunk-hashes) - a single sha512 would be better, right?
+* [hashed passwords](#hashed-passwords) - regarding the curious decisions
 * [http api](#http-api)
     * [read](#read)
     * [write](#write)
@@ -17,6 +18,7 @@
 * [building](#building)
     * [dev env setup](#dev-env-setup)
     * [just the sfx](#just-the-sfx)
+    * [build from release tarball](#build-from-release-tarball) - uses the included prebuilt webdeps
     * [complete release](#complete-release)
 * [todo](#todo) - roughly sorted by priority
     * [discarded ideas](#discarded-ideas)
@@ -67,14 +69,14 @@ regarding the frequent server log message during uploads;
 * on this http connection, `2.77 GiB` transferred, `102.9 MiB/s` average, `948` chunks handled
 * client says `4` uploads OK, `0` failed, `3` busy, `1` queued, `10042 MiB` total size, `7198 MiB` and `00:01:09` left
 
-## why not tus
+### why not tus
 
 I didn't know about [tus](https://tus.io/)  when I made this, but:
 * up2k has the advantage that it supports parallel uploading of non-contiguous chunks straight into the final file -- [tus does a merge at the end](https://tus.io/protocols/resumable-upload.html#concatenation) which is slow and taxing on the server HDD / filesystem (unless i'm misunderstanding)
 * up2k has the slight disadvantage of requiring the client to hash the entire file before an upload can begin, but this has the benefit of immediately skipping duplicate files
   * and the hashing happens in a separate thread anyways so it's usually not a bottleneck
 
-## why chunk-hashes
+### why chunk-hashes
 
 a single sha512 would be better, right?
 
@@ -84,13 +86,22 @@ as a result, the hashes are much less useful than they could have been (search t
 
 however it allows for hashing multiple chunks in parallel, greatly increasing upload speed from fast storage (NVMe, raid-0 and such)
 
-* both the [browser uploader](https://github.com/9001/copyparty#uploading) and the [commandline one](https://github.com/9001/copyparty/blob/hovudstraum/bin/up2k.py) does this now, allowing for fast uploading even from plaintext http
+* both the [browser uploader](https://github.com/9001/copyparty#uploading) and the [commandline one](https://github.com/9001/copyparty/tree/hovudstraum/bin#u2cpy) does this now, allowing for fast uploading even from plaintext http
 
 hashwasm would solve the streaming issue but reduces hashing speed for sha512 (xxh128 does 6 GiB/s), and it would make old browsers and [iphones](https://bugs.webkit.org/show_bug.cgi?id=228552) unsupported
 
 * blake2 might be a better choice since xxh is non-cryptographic, but that gets ~15 MiB/s on slower androids
 
 
+# hashed passwords
+
+regarding the curious decisions
+
+there is a static salt for all passwords;
+* because most copyparty APIs allow users to authenticate using only their password, making the username unknown, so impossible to do per-account salts
+* the drawback of this is that an attacker can bruteforce all accounts in parallel, however most copyparty instances only have a handful of accounts in the first place, and it can be compensated by increasing the hashing cost anyways
+
+
 # http api
 
 * table-column `params` = URL parameters; `?foo=bar&qux=...`
@@ -110,11 +121,18 @@ authenticate using header `Cookie: cppwd=foo` or url param `&pw=foo`
 | GET | `?ls&dots` | list files/folders at URL as JSON, including dotfiles |
 | GET | `?ls=t` | list files/folders at URL as plaintext |
 | GET | `?ls=v` | list files/folders at URL, terminal-formatted |
+| GET | `?lt` | in listings, use symlink timestamps rather than targets |
 | GET | `?b` | list files/folders at URL as simplified HTML |
 | GET | `?tree=.` | list one level of subdirectories inside URL |
 | GET | `?tree` | list one level of subdirectories for each level until URL |
-| GET | `?tar` | download everything below URL as a tar file |
-| GET | `?zip=utf-8` | download everything below URL as a zip file |
+| GET | `?tar` | download everything below URL as a gnu-tar file |
+| GET | `?tar=gz:9` | ...as a gzip-level-9 gnu-tar file |
+| GET | `?tar=xz:9` | ...as an xz-level-9 gnu-tar file |
+| GET | `?tar=pax` | ...as a pax-tar file |
+| GET | `?tar=pax,xz` | ...as an xz-level-1 pax-tar file |
+| GET | `?zip=utf-8` | ...as a zip file |
+| GET | `?zip` | ...as a WinXP-compatible zip file |
+| GET | `?zip=crc` | ...as an MSDOS-compatible zip file |
 | GET | `?ups` | show recent uploads from your IP |
 | GET | `?ups&filter=f` | ...where URL contains `f` |
 | GET | `?mime=foo` | specify return mimetype `foo` |
@@ -226,39 +244,55 @@ pip install mutagen  # audio metadata
 pip install pyftpdlib  # ftp server
 pip install impacket  # smb server -- disable Windows Defender if you REALLY need this on windows
 pip install Pillow pyheif-pillow-opener pillow-avif-plugin  # thumbnails
+pip install pyvips  # faster thumbnails
+pip install psutil  # better cleanup of stuck metadata parsers on windows 
 pip install black==21.12b0 click==8.0.2 bandit pylint flake8 isort mypy  # vscode tooling
 ```
 
 
 ## just the sfx
 
-first grab the web-dependencies from a previous sfx (assuming you don't need to modify something in those):
+if you just want to modify the copyparty source code (py/html/css/js) then this is the easiest approach
 
-```sh
-rm -rf copyparty/web/deps
-curl -L https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.py >x.py
-python3 x.py --version
-rm x.py
-cp -R /tmp/pe-copyparty.$(id -u)/copyparty/web/deps copyparty/web/
-```
-
-or you could build the web-dependencies from source instead (NB: does not include prismjs, need to grab that manually):
-```sh
-make -C scripts/deps-docker
-```
-
-then build the sfx using any of the following examples:
+build the sfx using any of the following examples:
 
 ```sh
 ./scripts/make-sfx.sh           # regular edition
+./scripts/make-sfx.sh fast      # build faster (worse js/css compression)
 ./scripts/make-sfx.sh gz no-cm  # gzip-compressed + no fancy markdown editor
 ```
 
 
+## build from release tarball
+
+uses the included prebuilt webdeps
+
+if you downloaded a [release](https://github.com/9001/copyparty/releases) source tarball from github (for example [copyparty-1.6.15.tar.gz](https://github.com/9001/copyparty/releases/download/v1.6.15/copyparty-1.6.15.tar.gz) so not the autogenerated one) you can build it like so,
+
+```bash
+python3 -m pip install --user -U build setuptools wheel jinja2 strip_hints
+bash scripts/run-tests.sh python3  # optional
+python3 -m build
+```
+
+if you are unable to use `build`, you can use the old setuptools approach instead,
+
+```bash
+python3 setup.py install --user setuptools wheel jinja2
+python3 setup.py build
+# you now have a wheel which you can install. or extract and repackage:
+python3 setup.py install --skip-build --prefix=/usr --root=$HOME/pe/copyparty
+```
+
+
 ## complete release
 
 also builds the sfx so skip the sfx section above
 
+*WARNING: `rls.sh` has not yet been updated with the docker-images and arch/nix packaging*
+
+does everything completely from scratch, straight from your local repo
+
 in the `scripts` folder:
 
 * run `make -C deps-docker` to build all dependencies

docs/notes.sh

@@ -194,6 +194,9 @@ sqlite3 .hist/up2k.db 'select * from mt where k="fgsfds" or k="t:mtp"' | tee /de
 for ((f=420;f<1200;f++)); do sz=$(ffmpeg -y -f lavfi -i sine=frequency=$f:duration=2 -vf volume=0.1 -ac 1 -ar 44100 -f s16le /dev/shm/a.wav 2>/dev/null; base64 -w0 </dev/shm/a.wav | gzip -c | wc -c); printf '%d %d\n' $f $sz; done | tee /dev/stderr | sort -nrk2,2
 ffmpeg -y -f lavfi -i sine=frequency=1050:duration=2 -vf volume=0.1 -ac 1 -ar 44100 /dev/shm/a.wav
 
+# better sine
+sox -DnV -r8000 -b8 -c1 /dev/shm/a.wav synth 1.1 sin 400 vol 0.02
+
 # play icon calibration pics
 for w in 150 170 190 210 230 250; do for h in 130 150 170 190 210; do /c/Program\ Files/ImageMagick-7.0.11-Q16-HDRI/magick.exe convert -size ${w}x${h} xc:brown -fill orange -draw "circle $((w/2)),$((h/2)) $((w/2)),$((h/3))" $w-$h.png; done; done
 

docs/rclone.md

@@ -72,7 +72,7 @@ rclone.exe mount --vfs-cache-mode writes --vfs-cache-max-age 5s --attr-timeout 5
 
 # sync folders to/from copyparty
 
-note that the up2k client [up2k.py](https://github.com/9001/copyparty/tree/hovudstraum/bin#up2kpy) (available on the "connect" page of your copyparty server) does uploads much faster and safer, but rclone is bidirectional and more ubiquitous
+note that the up2k client [u2c.py](https://github.com/9001/copyparty/tree/hovudstraum/bin#u2cpy) (available on the "connect" page of your copyparty server) does uploads much faster and safer, but rclone is bidirectional and more ubiquitous
 
 ```
 rclone sync /usr/share/icons/ cpp-rw:fds/

docs/versus.md

@@ -148,7 +148,7 @@ symbol legend,
 
 | feature / software      | a | b | c | d | e | f | g | h | i | j | k | l |
 | ----------------------- | - | - | - | - | - | - | - | - | - | - | - | - |
-| download folder as zip  | █ | █ | █ | █ | █ |   | █ |   | █ | █ | ╱ | █ |
+| download folder as zip  | █ | █ | █ | █ | ╱ |   | █ |   | █ | █ | ╱ | █ |
 | download folder as tar  | █ |   |   |   |   |   |   |   |   | █ |   |   |
 | upload                  | █ | █ | █ | █ | █ | █ | █ | █ | █ | █ | █ | █ |
 | parallel uploads        | █ |   |   | █ | █ |   | • |   | █ |   | █ |   |
@@ -183,6 +183,7 @@ symbol legend,
 * `cloud storage backend` = able to serve files from (and write to) s3 or similar cloud services; `╱` means the software can do this with some help from `rclone mount` as a bridge
 
 * `a`/copyparty can reject uploaded files (based on complex conditions), for example [by extension](https://github.com/9001/copyparty/blob/hovudstraum/bin/hooks/reject-extension.py) or [mimetype](https://github.com/9001/copyparty/blob/hovudstraum/bin/hooks/reject-mimetype.py)
+* `e`/seafile download-as-zip is not streaming; it creates the full zipfile before download can start, and fails on big folders
 * `j`/filebrowser remarks:
   * can provide checksums for single files on request
   * can probably do extension/mimetype rejection similar to copyparty
@@ -254,7 +255,7 @@ symbol legend,
 | per-file permissions    |   |   |   | █ | █ |   | █ |   | █ |   |   |   |
 | per-file passwords      | █ |   |   | █ | █ |   | █ |   | █ |   |   |   |
 | unmap subfolders        | █ |   |   |   |   |   | █ |   |   | █ | ╱ | • |
-| index.html blocks list  |   |   |   |   |   |   | █ |   |   | • |   |   |
+| index.html blocks list  | ╱ |   |   |   |   |   | █ |   |   | • |   |   |
 | write-only folders      | █ |   |   |   |   |   |   |   |   |   | █ | █ |
 | files stored as-is      | █ | █ | █ | █ |   | █ | █ |   |   | █ | █ | █ |
 | file versioning         |   |   |   | █ | █ |   |   |   |   |   |   |   |
@@ -287,9 +288,10 @@ symbol legend,
 * `curl-friendly ls` = returns a [sortable plaintext folder listing](https://user-images.githubusercontent.com/241032/215322619-ea5fd606-3654-40ad-94ee-2bc058647bb2.png) when curled
 * `curl-friendly upload` = uploading with curl is just `curl -T some.bin http://.../`
 * `a`/copyparty remarks:
-  * one-way folder sync from local to server can be done efficiently with [up2k.py](https://github.com/9001/copyparty/blob/hovudstraum/bin/up2k.py), or with webdav and conventional rsync
+  * one-way folder sync from local to server can be done efficiently with [u2c.py](https://github.com/9001/copyparty/tree/hovudstraum/bin#u2cpy), or with webdav and conventional rsync
   * can hot-reload config files (with just a few exceptions)
   * can set per-folder permissions if that folder is made into a separate volume, so there is configuration overhead
+  * `index.html` on its own does not prevent directory listing, but permission `h` (instead of `r`) enforces index.html to be returned instead of folder contents
   * [event hooks](https://github.com/9001/copyparty/tree/hovudstraum/bin/hooks) ([discord](https://user-images.githubusercontent.com/241032/215304439-1c1cb3c8-ec6f-4c17-9f27-81f969b1811a.png), [desktop](https://user-images.githubusercontent.com/241032/215335767-9c91ed24-d36e-4b6b-9766-fb95d12d163f.png)) inspired by filebrowser, as well as the more complex [media parser](https://github.com/9001/copyparty/tree/hovudstraum/bin/mtag) alternative
   * upload history can be visualized using [partyjournal](https://github.com/9001/copyparty/blob/hovudstraum/bin/partyjournal.py)
 * `k`/filegator remarks:
@@ -432,6 +434,7 @@ symbol legend,
   * not that bad, can probably be remedied with bindmounts or maybe symlinks
 * ⚠️ uploads not resumable / accelerated / integrity-checked
   * ⚠️ on cloudflare: max upload size 100 MiB
+* ⚠️ uploading small files is slow; `2.2` files per sec (copyparty does `87`/sec), tested locally with [linuxserver/nextcloud](https://hub.docker.com/r/linuxserver/nextcloud) (sqlite)
 * ⚠️ no write-only / upload-only folders
 * ⚠️ http/webdav only; no ftp, zeroconf
 * ⚠️ less awesome music player
@@ -451,7 +454,9 @@ symbol legend,
   * *much worse than nextcloud* in that regard
 * ⚠️ uploads not resumable / accelerated / integrity-checked
   * ⚠️ on cloudflare: max upload size 100 MiB
+* ⚠️ uploading small files is slow; `2.7` files per sec (copyparty does `87`/sec), tested locally with [official container](https://manual.seafile.com/docker/deploy_seafile_with_docker/)
 * ⚠️ no write-only / upload-only folders
+* ⚠️ big folders cannot be zip-downloaded
 * ⚠️ http/webdav only; no ftp, zeroconf
 * ⚠️ less awesome music player
 * ⚠️ doesn't run on android or ipads

pyproject.toml

@@ -0,0 +1,146 @@
+[project]
+name = "copyparty"
+description = """
+  Portable file server with accelerated resumable uploads, \
+  deduplication, WebDAV, FTP, zeroconf, media indexer, \
+  video thumbnails, audio transcoding, and write-only folders"""
+readme = "README.md"
+authors = [{ name = "ed", email = "copyparty@ocv.me" }]
+license = { text = "MIT" }
+requires-python = ">=3.3"
+dependencies = ["Jinja2"]
+dynamic = ["version"]
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.3",
+    "Programming Language :: Python :: 3.4",
+    "Programming Language :: Python :: 3.5",
+    "Programming Language :: Python :: 3.6",
+    "Programming Language :: Python :: 3.7",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: Implementation :: CPython",
+    "Programming Language :: Python :: Implementation :: Jython",
+    "Programming Language :: Python :: Implementation :: PyPy",
+    "Environment :: Console",
+    "Environment :: No Input/Output (Daemon)",
+    "Intended Audience :: End Users/Desktop",
+    "Intended Audience :: System Administrators",
+    "Topic :: Communications :: File Sharing",
+    "Topic :: Internet :: File Transfer Protocol (FTP)",
+    "Topic :: Internet :: WWW/HTTP :: HTTP Servers",
+]
+
+[project.urls]
+"Source Code" = "https://github.com/9001/copyparty"
+"Bug Tracker" = "https://github.com/9001/copyparty/issues"
+"Demo Server" = "https://a.ocv.me/pub/demo/"
+
+[project.optional-dependencies]
+thumbnails = ["Pillow"]
+thumbnails2 = ["pyvips"]
+audiotags = ["mutagen"]
+ftpd = ["pyftpdlib"]
+ftps = ["pyftpdlib", "pyopenssl"]
+pwhash = ["argon2-cffi"]
+
+[project.scripts]
+copyparty = "copyparty.__main__:main"
+"u2c" = "copyparty.web.a.u2c:main"
+"partyfuse" = "copyparty.web.a.partyfuse:main"
+
+# =====================================================================
+
+[build-system]
+requires = ["setuptools", "wheel"]
+build-backend = "setuptools.build_meta"
+# requires = ["hatchling"]
+# build-backend = "hatchling.build"
+
+[tool.hatch.version]
+source = "code"
+path = "copyparty/__version__.py"
+
+[tool.setuptools.dynamic]
+version = { attr = "copyparty.__version__.__version__" }
+
+[tool.setuptools.packages.find]
+include = ["copyparty*"]
+
+[tool.setuptools.package-data]
+copyparty = [
+    "res/COPYING.txt",
+    "res/insecure.pem",
+    "web/*.gz",
+    "web/*.js",
+    "web/*.css",
+    "web/*.html",
+    "web/a/*.bat",
+    "web/dd/*.png",
+    "web/deps/*.gz",
+    "web/deps/*.woff*",
+]
+
+# =====================================================================
+
+[tool.black]
+required-version = '21.12b0'
+target-version = ['py27']
+
+[tool.isort]
+profile = "black"
+include_trailing_comma = true
+
+[tool.bandit]
+skips = ["B104", "B110", "B112"]
+
+# =====================================================================
+
+[tool.pylint.MAIN]
+py-version = "3.11"
+jobs = 2
+
+[tool.pylint."MESSAGES CONTROL"]
+disable = [
+    "missing-module-docstring",
+    "missing-class-docstring",
+    "missing-function-docstring",
+    "import-outside-toplevel",
+    "wrong-import-position",
+    "raise-missing-from",
+    "bare-except",
+    "broad-exception-raised",
+    "broad-exception-caught",
+    "invalid-name",
+    "line-too-long",
+    "too-many-lines",
+    "consider-using-f-string",
+    "pointless-string-statement",
+]
+
+[tool.pylint.FORMAT]
+expected-line-ending-format = "LF"
+
+# =====================================================================
+
+[tool.mypy]
+python_version = "3.11"
+files = ["copyparty"]
+show_error_codes = true
+show_column_numbers = true
+pretty = true
+strict = true
+local_partial_types = true
+strict_equality = true
+warn_unreachable = true
+ignore_missing_imports = true
+follow_imports = "silent"
+
+[[tool.mypy.overrides]]
+no_implicit_reexport = false

scripts/copyparty-repack.sh

@@ -34,7 +34,7 @@ set -e
 #   4823  copyparty-extras/copyparty-repack.sh
 #           `- source files from github
 #
-#  23663  copyparty-extras/up2k.py
+#  23663  copyparty-extras/u2c.py
 #           `- standalone utility to upload or search for files
 #
 #  32280  copyparty-extras/partyfuse.py
@@ -147,7 +147,7 @@ repack sfx-lite "re no-dd no-cm no-hl gz"
 # copy lite-sfx.py to ./copyparty,
 # delete extracted source code
 ( cd copyparty-extras/
-mv copyparty-*/bin/up2k.py .
+mv copyparty-*/bin/u2c.py .
 mv copyparty-*/bin/partyfuse.py .
 cp -pv sfx-lite/copyparty-sfx.py ../copyparty
 rm -rf copyparty-{0..9}*.*.*{0..9}

scripts/deps-docker/Dockerfile

@@ -1,9 +1,9 @@
-# TODO easymde embeds codemirror on 3.17 due to new npm probably
-FROM    alpine:3.16
+FROM    alpine:3.18
 WORKDIR /z
 ENV     ver_asmcrypto=c72492f4a66e17a0e5dd8ad7874de354f3ccdaa5 \
         ver_hashwasm=4.9.0 \
         ver_marked=4.3.0 \
+        ver_dompf=3.0.5 \
         ver_mde=2.18.0 \
         ver_codemirror=5.65.12 \
         ver_fontawesome=5.13.0 \
@@ -14,6 +14,7 @@ ENV     ver_asmcrypto=c72492f4a66e17a0e5dd8ad7874de354f3ccdaa5 \
 # https://github.com/markedjs/marked/releases
 # https://github.com/Ionaru/easy-markdown-editor/tags
 # https://github.com/codemirror/codemirror5/releases
+# https://github.com/cure53/DOMPurify/releases
 # https://github.com/Daninet/hash-wasm/releases
 # https://github.com/openpgpjs/asmcrypto.js
 # https://github.com/google/zopfli/tags
@@ -28,6 +29,7 @@ RUN     mkdir -p /z/dist/no-pk \
         && wget https://github.com/markedjs/marked/archive/v$ver_marked.tar.gz -O marked.tgz \
         && wget https://github.com/Ionaru/easy-markdown-editor/archive/$ver_mde.tar.gz -O mde.tgz \
         && wget https://github.com/codemirror/codemirror5/archive/$ver_codemirror.tar.gz -O codemirror.tgz \
+        && wget https://github.com/cure53/DOMPurify/archive/refs/tags/$ver_dompf.tar.gz -O dompurify.tgz \
         && wget https://github.com/FortAwesome/Font-Awesome/releases/download/$ver_fontawesome/fontawesome-free-$ver_fontawesome-web.zip -O fontawesome.zip \
         && wget https://github.com/google/zopfli/archive/zopfli-$ver_zopfli.tar.gz -O zopfli.tgz \
         && wget https://github.com/Daninet/hash-wasm/releases/download/v$ver_hashwasm/hash-wasm@$ver_hashwasm.zip -O hash-wasm.zip \
@@ -49,6 +51,7 @@ RUN     mkdir -p /z/dist/no-pk \
             && cd easy-markdown-editor* \
             && npm install \
             && npm i gulp-cli -g ) \
+        && tar -xf dompurify.tgz \
         && tar -xf prism.tgz \
         && unzip fontawesome.zip \
         && tar -xf zopfli.tgz
@@ -121,6 +124,10 @@ RUN     cd easy-markdown-editor-$ver_mde \
         && cp -pv dist/easymde.min.js /z/dist/easymde.js
 
 
+# build dompurify
+RUN     (echo; cat DOMPurify-$ver_dompf/dist/purify.min.js) >> /z/dist/marked.js
+
+
 # build fontawesome and scp
 COPY    mini-fa.sh /z
 COPY    mini-fa.css /z

scripts/docker/Dockerfile.ac

@@ -5,11 +5,14 @@ LABEL   org.opencontainers.image.url="https://github.com/9001/copyparty" \
         org.opencontainers.image.licenses="MIT" \
         org.opencontainers.image.title="copyparty-ac" \
         org.opencontainers.image.description="copyparty with Pillow and FFmpeg (image/audio/video thumbnails, audio transcoding, media tags)"
+ENV     PYTHONPYCACHEPREFIX=/tmp/pyc \
+        XDG_CONFIG_HOME=/cfg
 
-RUN     apk --no-cache add \
+RUN     apk --no-cache add !pyc \
             wget \
-            py3-pillow \
+            py3-argon2-cffi py3-pillow \
             ffmpeg \
+        && rm -rf /tmp/pyc \
         && mkdir /cfg /w \
         && chmod 777 /cfg /w \
         && echo % /cfg > initcfg
@@ -17,4 +20,4 @@ RUN     apk --no-cache add \
 COPY    i/dist/copyparty-sfx.py ./
 WORKDIR /w
 EXPOSE  3923
-ENTRYPOINT ["python3", "/z/copyparty-sfx.py", "-c", "/z/initcfg"]
+ENTRYPOINT ["python3", "/z/copyparty-sfx.py", "--no-crt", "-c", "/z/initcfg"]

scripts/docker/Dockerfile.dj

@@ -5,26 +5,28 @@ LABEL   org.opencontainers.image.url="https://github.com/9001/copyparty" \
         org.opencontainers.image.licenses="MIT" \
         org.opencontainers.image.title="copyparty-dj" \
         org.opencontainers.image.description="copyparty with all optional dependencies, including musical key / bpm detection"
+ENV     PYTHONPYCACHEPREFIX=/tmp/pyc \
+        XDG_CONFIG_HOME=/cfg
 
 COPY    i/bin/mtag/install-deps.sh ./
 COPY    i/bin/mtag/audio-bpm.py /mtag/
 COPY    i/bin/mtag/audio-key.py /mtag/
-RUN     apk add -U \
+RUN     apk add -U !pyc \
             wget \
-            py3-pillow py3-pip py3-cffi \
+            py3-argon2-cffi py3-pillow py3-pip py3-cffi \
             ffmpeg \
             vips-jxl vips-heif vips-poppler vips-magick \
             py3-numpy fftw libsndfile \
             vamp-sdk vamp-sdk-libs \
-        && python3 -m pip install pyvips \
-        && apk --no-cache add -t .bd \
+        && apk add -t .bd \
             bash wget gcc g++ make cmake patchelf \
             python3-dev ffmpeg-dev fftw-dev libsndfile-dev \
             py3-wheel py3-numpy-dev \
             vamp-sdk-dev \
+        && python3 -m pip install pyvips \
         && bash install-deps.sh \
         && apk del py3-pip .bd \
-        && rm -rf /var/cache/apk/* \
+        && rm -rf /var/cache/apk/* /tmp/pyc \
         && chmod 777 /root \
         && ln -s /root/vamp /root/.local / \
         && mkdir /cfg /w \
@@ -34,4 +36,4 @@ RUN     apk add -U \
 COPY    i/dist/copyparty-sfx.py ./
 WORKDIR /w
 EXPOSE  3923
-ENTRYPOINT ["python3", "/z/copyparty-sfx.py", "-c", "/z/initcfg"]
+ENTRYPOINT ["python3", "/z/copyparty-sfx.py", "--no-crt", "-c", "/z/initcfg"]

scripts/docker/Dockerfile.im

@@ -5,10 +5,13 @@ LABEL   org.opencontainers.image.url="https://github.com/9001/copyparty" \
         org.opencontainers.image.licenses="MIT" \
         org.opencontainers.image.title="copyparty-im" \
         org.opencontainers.image.description="copyparty with Pillow and Mutagen (image thumbnails, media tags)"
+ENV     PYTHONPYCACHEPREFIX=/tmp/pyc \
+        XDG_CONFIG_HOME=/cfg
 
-RUN     apk --no-cache add \
+RUN     apk --no-cache add !pyc \
             wget \
-            py3-pillow py3-mutagen \
+            py3-argon2-cffi py3-pillow py3-mutagen \
+        && rm -rf /tmp/pyc \
         && mkdir /cfg /w \
         && chmod 777 /cfg /w \
         && echo % /cfg > initcfg
@@ -16,4 +19,4 @@ RUN     apk --no-cache add \
 COPY    i/dist/copyparty-sfx.py ./
 WORKDIR /w
 EXPOSE  3923
-ENTRYPOINT ["python3", "/z/copyparty-sfx.py", "-c", "/z/initcfg"]
+ENTRYPOINT ["python3", "/z/copyparty-sfx.py", "--no-crt", "-c", "/z/initcfg"]

scripts/docker/Dockerfile.iv

@@ -5,14 +5,20 @@ LABEL   org.opencontainers.image.url="https://github.com/9001/copyparty" \
         org.opencontainers.image.licenses="MIT" \
         org.opencontainers.image.title="copyparty-iv" \
         org.opencontainers.image.description="copyparty with Pillow, FFmpeg, libvips (image/audio/video thumbnails, audio transcoding, media tags)"
+ENV     PYTHONPYCACHEPREFIX=/tmp/pyc \
+        XDG_CONFIG_HOME=/cfg
 
-RUN     apk --no-cache add \
+RUN     apk add -U !pyc \
             wget \
-            py3-pillow py3-pip py3-cffi \
+            py3-argon2-cffi py3-pillow py3-pip py3-cffi \
             ffmpeg \
             vips-jxl vips-heif vips-poppler vips-magick \
+        && apk add -t .bd \
+            bash wget gcc g++ make cmake patchelf \
+            python3-dev py3-wheel \
         && python3 -m pip install pyvips \
-        && apk del py3-pip \
+        && apk del py3-pip .bd \
+        && rm -rf /var/cache/apk/* /tmp/pyc \
         && mkdir /cfg /w \
         && chmod 777 /cfg /w \
         && echo % /cfg > initcfg
@@ -20,4 +26,4 @@ RUN     apk --no-cache add \
 COPY    i/dist/copyparty-sfx.py ./
 WORKDIR /w
 EXPOSE  3923
-ENTRYPOINT ["python3", "/z/copyparty-sfx.py", "-c", "/z/initcfg"]
+ENTRYPOINT ["python3", "/z/copyparty-sfx.py", "--no-crt", "-c", "/z/initcfg"]

scripts/docker/Dockerfile.min

@@ -5,9 +5,12 @@ LABEL   org.opencontainers.image.url="https://github.com/9001/copyparty" \
         org.opencontainers.image.licenses="MIT" \
         org.opencontainers.image.title="copyparty-min" \
         org.opencontainers.image.description="just copyparty, no thumbnails / media tags / audio transcoding"
+ENV     PYTHONPYCACHEPREFIX=/tmp/pyc \
+        XDG_CONFIG_HOME=/cfg
 
-RUN     apk --no-cache add \
+RUN     apk --no-cache add !pyc \
             python3 \
+        && rm -rf /tmp/pyc \
         && mkdir /cfg /w \
         && chmod 777 /cfg /w \
         && echo % /cfg > initcfg
@@ -15,4 +18,4 @@ RUN     apk --no-cache add \
 COPY    i/dist/copyparty-sfx.py ./
 WORKDIR /w
 EXPOSE  3923
-ENTRYPOINT ["python3", "/z/copyparty-sfx.py", "-c", "/z/initcfg"]
+ENTRYPOINT ["python3", "/z/copyparty-sfx.py", "--no-crt", "--no-thumb", "-c", "/z/initcfg"]

scripts/docker/Dockerfile.min.pip

@@ -5,14 +5,17 @@ LABEL   org.opencontainers.image.url="https://github.com/9001/copyparty" \
         org.opencontainers.image.licenses="MIT" \
         org.opencontainers.image.title="copyparty-min-pip" \
         org.opencontainers.image.description="just copyparty, no thumbnails, no media tags, no audio transcoding"
+ENV     PYTHONPYCACHEPREFIX=/tmp/pyc \
+        XDG_CONFIG_HOME=/cfg
 
-RUN     apk --no-cache add python3 py3-pip \
+RUN     apk --no-cache add python3 py3-pip !pyc \
         && python3 -m pip install copyparty \
         && apk del py3-pip \
+        && rm -rf /tmp/pyc \
         && mkdir /cfg /w \
         && chmod 777 /cfg /w \
         && echo % /cfg > initcfg
 
 WORKDIR /w
 EXPOSE  3923
-ENTRYPOINT ["python3", "-m", "copyparty", "-c", "/z/initcfg"]
+ENTRYPOINT ["python3", "-m", "copyparty", "--no-crt", "--no-thumb", "-c", "/z/initcfg"]

scripts/docker/README.md

@@ -15,6 +15,7 @@ docker run --rm -it -u 1000 -p 3923:3923 -v /mnt/nas:/w -v $PWD/cfgdir:/cfg copy
 * `copyparty/ac` is the recommended [image edition](#editions)
 * you can download the image from github instead by replacing `copyparty/ac` with `ghcr.io/9001/copyparty-ac`
 * if you are using rootless podman, remove `-u 1000`
+* if you have selinux, append `:z` to all `-v` args (for example `-v /mnt/nas:/w:z`)
 
 i'm unfamiliar with docker-compose and alternatives so let me know if this section could be better 🙏
 
@@ -77,8 +78,3 @@ or using commandline arguments,
 # build the images yourself
 
 basically `./make.sh hclean pull img push` but see [devnotes.md](./devnotes.md)
-
-
-# notes
-
-* currently unable to play [tracker music](https://en.wikipedia.org/wiki/Module_file) (mod/s3m/xm/it/...) -- will be fixed in june 2023 (Alpine 3.18)

scripts/docker/make.sh

@@ -95,7 +95,7 @@ filt=
                 [ $(jobs -p | wc -l) -lt $(nproc) ] && break
                 while [ -e .blk ]; do sleep 0.2; done
             done
-            aa="$(printf '%7s' $a)"
+            aa="$(printf '%11s' $a-$i)"
 
             # arm takes forever so make it top priority
             [ ${a::3} == arm ] && nice= || nice=nice

scripts/make-pypi-release.sh

@@ -73,14 +73,17 @@ pydir="$(
 }
 
 function have() {
-	python -c "import $1; $1; $1.__version__"
+	python -c "import $1; $1; getattr($1,'__version__',0)"
 }
 
 function load_env() {
 	. buildenv/bin/activate
 	have setuptools
 	have wheel
+	have build
 	have twine
+	have jinja2
+	have strip_hints
 }
 
 load_env || {
@@ -88,19 +91,32 @@ load_env || {
 	deactivate || true
 	rm -rf buildenv
 	python3 -m venv buildenv
-	(. buildenv/bin/activate && pip install twine wheel)
+	(. buildenv/bin/activate && pip install \
+		setuptools wheel build twine jinja2 strip_hints )
 	load_env
 }
 
+# cleanup
+rm -rf unt build/pypi
+
 # grab licenses
 scripts/genlic.sh copyparty/res/COPYING.txt
 
-# remove type hints to support python < 3.9
+# clean-ish packaging env
 rm -rf build/pypi
 mkdir -p build/pypi
-cp -pR setup.py README.md LICENSE copyparty contrib bin scripts/strip_hints build/pypi/
+cp -pR pyproject.toml README.md LICENSE copyparty contrib bin scripts/strip_hints build/pypi/
 tar -c docs/lics.txt scripts/genlic.sh build/*.txt | tar -xC build/pypi/
 cd build/pypi
+
+# delete junk
+find -name '*.pyc' -delete
+find -name __pycache__ -delete
+find -name py.typed -delete
+find -type f \( -name .DS_Store -or -name ._.DS_Store \) -delete
+find -type f -name ._\* | while IFS= read -r f; do cmp <(printf '\x00\x05\x16') <(head -c 3 -- "$f") && rm -f -- "$f"; done
+
+# remove type hints to support python < 3.9
 f=../strip-hints-0.1.10.tar.gz
 [ -e $f ] || 
 	(url=https://files.pythonhosted.org/packages/9c/d4/312ddce71ee10f7e0ab762afc027e07a918f1c0e1be5b0069db5b0e7542d/strip-hints-0.1.10.tar.gz;
@@ -132,24 +148,13 @@ while IFS= read -r x; do
 done
 
 rm -rf contrib
-[ $fast ] && sed -ri s/5730/10/ copyparty/web/Makefile
+[ $fast ] && sed -ri s/573/10/ copyparty/web/Makefile
 (cd copyparty/web && make -j$(nproc) && rm Makefile)
 
 # build
-./setup.py clean2
-./setup.py sdist bdist_wheel --universal
+python3 -m build
 
 [ "$mode" == t ] && twine upload -r pypitest dist/*
 [ "$mode" == u ] && twine upload -r pypi dist/*
 
-cat <<EOF
-
-
-    all done!
-    
-    to clean up the source tree:
-    
-       cd ~/dev/copyparty
-       ./setup.py clean2
-   
-EOF
+true

scripts/make-sfx.sh

@@ -2,10 +2,14 @@
 set -e
 echo
 
+berr() { p=$(head -c 72 </dev/zero | tr '\0' =); printf '\n%s\n\n' $p; cat; printf '\n%s\n\n' $p; }
+
 help() { exec cat <<'EOF'
 
 # optional args:
 #
+# `fast` builds faster, with cheaper js/css compression
+#
 # `clean` uses files from git (everything except web/deps),
 #   so local changes won't affect the produced sfx
 #
@@ -42,6 +46,13 @@ help() { exec cat <<'EOF'
 #
 # `no-dd` saves ~2k by removing the mouse cursor
 #
+# _____________________________________________________________________
+# build behavior:
+#
+# `dl-wd` automatically downloads webdeps if necessary
+#
+# `ign-wd` allows building an sfx without webdeps
+#
 # ---------------------------------------------------------------------
 #
 # if you are on windows, you can use msys2:
@@ -109,6 +120,8 @@ while [ ! -z "$1" ]; do
 		no-hl)  no_hl=1  ; ;;
 		no-dd)  no_dd=1  ; ;;
 		no-cm)  no_cm=1  ; ;;
+		dl-wd)  dl_wd=1  ; ;;
+		ign-wd) ign_wd=1 ; ;;
 		fast)   zopf=    ; ;;
 		ultra)  ultra=1  ; ;;
 		lang)   shift;langs="$1"; ;;
@@ -223,7 +236,7 @@ necho() {
 
 	# enable this to dynamically remove type hints at startup,
 	# in case a future python version can use them for performance
-	true || (
+	true && (
 		necho collecting strip-hints
 		f=../build/strip-hints-0.1.10.tar.gz
 		[ -e $f ] ||
@@ -283,12 +296,56 @@ necho() {
 	rm -f copyparty/stolen/*/README.md
 
 	# remove type hints before build instead
-	(cd copyparty; "$pybin" ../../scripts/strip_hints/a.py; rm uh)
+	(cd copyparty; PYTHONPATH="..:$PYTHONPATH" "$pybin" ../../scripts/strip_hints/a.py; rm uh)
 
 	licfile=$(realpath copyparty/res/COPYING.txt)
 	(cd ../scripts; ./genlic.sh "$licfile")
 }
 
+[ ! -e copyparty/web/deps/mini-fa.woff ] && [ $dl_wd ] && {
+	echo "could not find webdeps; downloading..."
+	url=https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.py
+	wget -Ox.py "$url" || curl -L "$url" >x.py
+
+	echo "extracting webdeps..."
+	wdsrc="$("$pybin" x.py --version 2>&1 | tee /dev/stderr | awk '/sfxdir:/{sub(/.*: /,"");print;exit}')"
+	[ "$wdsrc" ] || {
+		echo failed to discover tempdir of reference copyparty-sfx.py
+		exit 1
+	}
+	rm -rf copyparty/web/deps
+	cp -pvR "$wdsrc/copyparty/web/deps" copyparty/web/
+
+	# also copy it out into the source-tree for next time
+	rm -rf ../copyparty/web/deps
+	cp -pR copyparty/web/deps ../copyparty/web
+
+	rm x.py
+}
+
+[ -e copyparty/web/deps/mini-fa.woff ] || [ $ign_wd ] || { berr <<'EOF'
+ERROR:
+  could not find webdeps; the front-end will not be fully functional
+
+please choose one of the following:
+
+A) add the argument "dl-wd" to fix it automatically; this will
+    download copyparty-sfx.py and extract the webdeps from there
+
+B) build the webdeps from source:  make -C scripts/deps-docker
+
+C) add the argument "ign-wd" to continue building the sfx without webdeps
+
+alternative A is a good choice if you are only intending to
+modify the copyparty source code (py/html/css/js) and do not
+plan to make any changes to the mostly-third-party webdeps
+
+there may be additional hints in the devnotes:
+https://github.com/9001/copyparty/blob/hovudstraum/docs/devnotes.md#building
+EOF
+	exit 1
+}
+
 ver=
 [ -z "$repack" ] &&
 git describe --tags >/dev/null 2>/dev/null && {
@@ -335,9 +392,9 @@ find -name '*.pyc' -delete
 find -name __pycache__ -delete
 find -name py.typed -delete
 
-# especially prevent osx from leaking your lan ip (wtf apple)
+# especially prevent macos/osx from leaking your lan ip (wtf apple)
 find -type f \( -name .DS_Store -or -name ._.DS_Store \) -delete
-find -type f -name ._\* | while IFS= read -r f; do cmp <(printf '\x00\x05\x16') <(head -c 3 -- "$f") && rm -f -- "$f"; done
+find -type f -name ._\* | while IFS= read -r f; do cmp <(printf '\x00\x05\x16') <(head -c 3 -- "$f") && rm -fv -- "$f"; done
 
 rm -f copyparty/web/deps/*.full.* copyparty/web/dbg-* copyparty/web/Makefile
 
@@ -421,7 +478,7 @@ while IFS= read -r f; do
 done
 
 # up2k goes from 28k to 22k laff
-awk 'BEGIN{gensub(//,"",1)}' </dev/null &&
+awk 'BEGIN{gensub(//,"",1)}' </dev/null 2>/dev/null &&
 echo entabbening &&
 find | grep -E '\.css$' | while IFS= read -r f; do
 	awk '{
@@ -435,7 +492,9 @@ find | grep -E '\.css$' | while IFS= read -r f; do
 	1
 	' <$f | sed -r 's/;\}$/}/; /\{\}$/d' >t
 	tmv "$f"
-done
+done ||
+	echo "WARNING: your awk does not have gensub, so the sfx will not have optimal compression"
+
 unexpand -h 2>/dev/null &&
 find | grep -E '\.(js|html)$' | while IFS= read -r f; do
 	unexpand -t 4 --first-only <"$f" >t
@@ -529,7 +588,7 @@ sed -r 's/([^ ]*) (.*)/\2.\1/' | grep -vE '/list1?$' > list1
 for n in {1..50}; do
 	(grep -vE '\.(gz|br)$' list1; grep -E '\.(gz|br)$' list1 | (shuf||gshuf) ) >list || true
 	s=$( (sha1sum||shasum) < list | cut -c-16)
-	grep -q $s "$zdir/h" && continue
+	grep -q $s "$zdir/h" 2>/dev/null && continue
 	echo $s >> "$zdir/h"
 	break
 done

scripts/make-tgz-release.sh

@@ -64,6 +64,8 @@ git archive hovudstraum | tar -xC "$rls_dir"
 echo ">>> export untracked deps"
 tar -c copyparty/web/deps | tar -xC "$rls_dir"
 
+scripts/genlic.sh "$rls_dir/copyparty/res/COPYING.txt"
+
 cd "$rls_dir"
 find -type d -exec chmod 755 '{}' \+
 find -type f -exec chmod 644 '{}' \+
@@ -93,7 +95,8 @@ rm \
   .gitattributes \
   .gitignore
 
-mv LICENSE LICENSE.txt
+cp -pv LICENSE LICENSE.txt
+mv setup.py{,.disabled}
 
 # the regular cleanup memes
 find -name '*.pyc' -delete

scripts/prep.sh

@@ -11,30 +11,12 @@ update_arch_pkgbuild() {
     rm -rf x
     mkdir x
 
-    (echo "$self/../dist/copyparty-sfx.py"
-    awk -v self="$self" '
-        /^\)/{o=0}
-        /^source=/{o=1;next}
-        {
-            sub(/..pkgname./,"copyparty");
-            sub(/.*pkgver./,self "/..");
-            sub(/^ +"/,"");sub(/"/,"")
-        }
-        o&&!/https/' PKGBUILD
-    ) |
-    xargs sha256sum > x/sums
+    sha=$(sha256sum "$self/../dist/copyparty-$ver.tar.gz" | awk '{print$1}')
 
-    (awk -v ver=$ver '
+    awk -v ver=$ver -v sha=$sha '
         /^pkgver=/{sub(/[0-9\.]+/,ver)};
-        /^sha256sums=/{exit};
-        1' PKGBUILD
-    echo -n 'sha256sums=('
-    p=; cat x/sums | while read s _; do
-        echo "$p\"$s\""
-        p='            '
-    done
-    awk '/^sha256sums=/{o=1} o&&/^\)/{o=2} o==2' PKGBUILD
-    ) >a
+        /^sha256sums=/{sub(/[0-9a-f]{64}/,sha)};
+        1' PKGBUILD >a
     mv a PKGBUILD
 
     rm -rf x

scripts/pyinstaller/depchk.sh

@@ -44,7 +44,7 @@ ckgh() {
 			curl -s https://api.github.com/repos/$dep/releases | tee h |
 			jq -r 'first|.assets|.[]|.name'
 		)
-		[ -z "$k" ] && echo "outdated: $dep" && cp h "ng-$dep" e=1
+		[ -z "$k" ] && echo "outdated: $dep" && cp h "ng-$dep" && e=1
 	done
 	true
 }

scripts/pyinstaller/deps.sha512

@@ -1,13 +1,13 @@
 d5510a24cb5e15d6d30677335bbc7624c319b371c0513981843dc51d9b3a1e027661096dfcfc540634222bb2634be6db55bf95185b30133cb884f1e47652cf53  altgraph-0.17.3-py2.py3-none-any.whl
 eda6c38fc4d813fee897e969ff9ecc5acc613df755ae63df0392217bbd67408b5c1f6c676f2bf5497b772a3eb4e1a360e1245e1c16ee83f0af555f1ab82c3977  Git-2.39.1-32-bit.exe
 17ce52ba50692a9d964f57a23ac163fb74c77fdeb2ca988a6d439ae1fe91955ff43730c073af97a7b3223093ffea3479a996b9b50ee7fba0869247a56f74baa6  pefile-2023.2.7-py3-none-any.whl
-d68c78bc83f4f48c604912b2d1ca4772b0e6ed676cd2eb439411e0a74d63fe215aac93dd9dab04ed341909a4a6a1efc13ec982516e3cb0fc7c355055e63d9178  pyinstaller-5.10.1-py3-none-win32.whl
-fe62705893c86eeb2d5b841da8debe05dedda98364dec190b487e718caad8a8735503bf93739a7a27ea793a835bf976fb919ceec1424b8fc550b936bae4a54e9  pyinstaller-5.10.1-py3-none-win_amd64.whl
-61c543983ff67e2bdff94d2d6198023679437363db8c660fa81683aff87c5928cd800720488e18d09be89fe45d6ab99be3ccb912cb2e03e2bca385b4338e1e42  pyinstaller_hooks_contrib-2023.2-py2.py3-none-any.whl
-132a5380f33a245f2e744413a0e1090bc42b7356376de5121397cec5976b04b79f7c9ebe28af222c9c7b01461f7d7920810d220e337694727e0d7cd9e91fa667  pywin32_ctypes-0.2.0-py2.py3-none-any.whl
+f298e34356b5590dde7477d7b3a88ad39c622a2bcf3fcd7c53870ce8384dd510f690af81b8f42e121a22d3968a767d2e07595036b2ed7049c8ef4d112bcf3a61  pyinstaller-5.13.2-py3-none-win32.whl
+ea73aa54cc6d5db20dfb127e54562dabf890e4cd6171a91b10a51af2bcfc76e1d64cbdce4546df2dcfe42b624724c85b1cd05934be2413425b1f880222727b4f  pyinstaller-5.13.2-py3-none-win_amd64.whl
+2f4e3927a38cf7757bc9a1c06370d79209669a285a80f1b09cf9917137825c7022a50a56b351807e6e687e2c3a7bd7b2c5cc6daeb4d90e11920284c1a04a1cc3  pyinstaller_hooks_contrib-2023.8-py2.py3-none-any.whl
+749a473646c6d4c7939989649733d4c7699fd1c359c27046bf5bc9c070d1a4b8b986bbc65f60d7da725baf16dbfdd75a4c2f5bb8335f2cb5685073f5fee5c2d1  pywin32_ctypes-0.2.2-py3-none-any.whl
 3c5adf0a36516d284a2ede363051edc1bcc9df925c5a8a9fa2e03cab579dd8d847fdad42f7fd5ba35992e08234c97d2dbfec40a9d12eec61c8dc03758f2bd88e  typing_extensions-4.4.0-py3-none-any.whl
-4b6e9ae967a769fe32be8cf0bc0d5a213b138d1e0344e97656d08a3d15578d81c06c45b334c872009db2db8f39db0c77c94ff6c35168d5e13801917667c08678  upx-4.0.2-win32.zip
-# up2k (win7)
+8d16a967a0a7872a7575b1005cf66915deacda6ee8611fbb52f42fc3e3beb2f901a5140c942a5d146bd412b92bfa9cbadd82beeba83df6d70930c6dc26608a5b  upx-4.1.0-win32.zip
+# u2c (win7)
 a7d259277af4948bf960682bc9fb45a44b9ae9a19763c8a7c313cef4aa9ec2d447d843e4a7c409e9312c8c8f863a24487a8ee4ffa6891e9b1c4e111bb4723861  certifi-2022.12.7-py3-none-any.whl
 2822c0dae180b1c8cfb7a70c8c00bad62af9afdbb18b656236680def9d3f1fcdcb8ef5eb64fc3b4c934385cd175ad5992a2284bcba78a243130de75b2d1650db  charset_normalizer-3.1.0-cp37-cp37m-win32.whl
 ffdd45326f4e91c02714f7a944cbcc2fdd09299f709cfa8aec0892053eef0134fb80d9ba3790afd319538a86feb619037cbf533e2f5939cb56b35bb17f56c858  idna-3.4-py3-none-any.whl
@@ -24,7 +24,7 @@ c06b3295d1d0b0f0a6f9a6cd0be861b9b643b4a5ea37857f0bd41c45deaf27bb927b71922dab74e6
 ba91ab0518c61eff13e5612d9e6b532940813f6b56e6ed81ea6c7c4d45acee4d98136a383a25067512b8f75538c67c987cf3944bfa0229e3cb677e2fb81e763e  zipp-3.10.0-py3-none-any.whl
 # win10
 00558cca2e0ac813d404252f6e5aeacb50546822ecb5d0570228b8ddd29d94e059fbeb6b90393dee5abcddaca1370aca784dc9b095cbb74e980b3c024767fb24  Jinja2-3.1.2-py3-none-any.whl
-b1db6f5a79fc15391547643e5973cf5946c0acfa6febb68bc90fc3f66369681100cc100f32dd04256dcefa510e7864c718515a436a4af3a10fe205c413c7e693  MarkupSafe-2.1.2-cp311-cp311-win_amd64.whl
-4a20aeb52d4fde6aabcba05ee261595eeb5482c72ee27332690f34dd6e7a49c0b3ba3813202ac15c9d21e29f1cd803f2e79ccc1c45ec314fcd0a937016bcbc56  mutagen-1.46.0-py3-none-any.whl
-78414808cb9a5fa74e7b23360b8f46147952530e3cc78a3ad4b80be3e26598080537ac691a1be1f35b7428a22c1f65a6adf45986da2752fbe9d9819d77a58bf8  Pillow-9.5.0-cp311-cp311-win_amd64.whl
-4b7711b950858f459d47145b88ccde659279c6af47144d58a1c54ea2ce4b80ec43eb7f69c68f12f8f6bc54c86a44e77441993257f7ad43aab364655de5c51bb1  python-3.11.2-amd64.exe
+7f8f4daa4f4f2dbf24cdd534b2952ee3fba6334eb42b37465ccda3aa1cccc3d6204aa6bfffb8a83bf42ec59c702b5b5247d4c8ee0d4df906334ae53072ef8c4c  MarkupSafe-2.1.3-cp311-cp311-win_amd64.whl
+8a6e2b13a2ec4ef914a5d62aad3db6464d45e525a82e07f6051ed10474eae959069e165dba011aefb8207cdfd55391d73d6f06362c7eb247b08763106709526e  mutagen-1.47.0-py3-none-any.whl
+926d408a886059a75cf12706fa061146f9f042b27fb6e65be7d49f398ed23fb0227639d84804586ac014c6bcf7d08cd86a09c1a20793d341aa0802d3d32a546b  Pillow-10.0.0-cp311-cp311-win_amd64.whl
+c86bbeacad3ae3c7bde747f5b4f09c11eced841add14e79ec4a064e5e29ebca35460e543ba735b11bfb882837d5ff4371ce64492d28d096b4686233c9a8cda6d  python-3.11.5-amd64.exe

scripts/pyinstaller/deps.txt

@@ -13,7 +13,7 @@ https://pypi.org/project/MarkupSafe/#files
 https://pypi.org/project/mutagen/#files
 https://pypi.org/project/Pillow/#files
 
-# up2k (win7) additionals
+# u2c (win7) additionals
 https://pypi.org/project/certifi/#files
 https://pypi.org/project/charset-normalizer/#files  # cp37-cp37m-win32.whl
 https://pypi.org/project/idna/#files

scripts/pyinstaller/notes.txt

@@ -17,13 +17,13 @@ uname -s | grep NT-10 && w10=1 || {
 fns=(
   altgraph-0.17.3-py2.py3-none-any.whl
   pefile-2023.2.7-py3-none-any.whl
-  pyinstaller-5.10.1-py3-none-win_amd64.whl
-  pyinstaller_hooks_contrib-2023.2-py2.py3-none-any.whl
-  pywin32_ctypes-0.2.0-py2.py3-none-any.whl
-  upx-4.0.2-win32.zip
+  pyinstaller-5.13.2-py3-none-win_amd64.whl
+  pyinstaller_hooks_contrib-2023.7-py2.py3-none-any.whl
+  pywin32_ctypes-0.2.2-py3-none-any.whl
+  upx-4.1.0-win32.zip
 )
 [ $w10 ] && fns+=(
-  mutagen-1.46.0-py3-none-any.whl
+  mutagen-1.47.0-py3-none-any.whl
   Pillow-9.4.0-cp311-cp311-win_amd64.whl
   python-3.11.3-amd64.exe
 }
@@ -43,12 +43,11 @@ fns=(
 )
 [ $w7x64 ] && fns+=(
   windows6.1-kb2533623-x64.msu
-  pyinstaller-5.10.1-py3-none-win_amd64.whl
   python-3.7.9-amd64.exe
 )
 [ $w7x32 ] && fns+=(
   windows6.1-kb2533623-x86.msu
-  pyinstaller-5.10.1-py3-none-win32.whl
+  pyinstaller-5.13.2-py3-none-win32.whl
   python-3.7.9.exe
 )
 dl() { curl -fkLOC- "$1" && return 0; echo "$1"; return 1; }

scripts/pyinstaller/up2k.rc

@@ -18,9 +18,9 @@ VSVersionInfo(
         [StringStruct('CompanyName', 'ocv.me'),
         StringStruct('FileDescription', 'copyparty uploader / filesearch command'),
         StringStruct('FileVersion', '1.2.3'),
-        StringStruct('InternalName', 'up2k'),
+        StringStruct('InternalName', 'u2c'),
         StringStruct('LegalCopyright', '2019, ed'),
-        StringStruct('OriginalFilename', 'up2k.exe'),
+        StringStruct('OriginalFilename', 'u2c.exe'),
         StringStruct('ProductName', 'copyparty up2k client'),
         StringStruct('ProductVersion', '1.2.3')])
       ]), 

scripts/pyinstaller/up2k.sh

@@ -14,7 +14,7 @@ uname -s | grep -E 'WOW64|NT-10' && echo need win7-32 && exit 1
 dl() { curl -fkLO "$1"; }
 cd ~/Downloads
 
-dl https://192.168.123.1:3923/cpp/bin/up2k.py
+dl https://192.168.123.1:3923/cpp/bin/u2c.py
 dl https://192.168.123.1:3923/cpp/scripts/pyinstaller/up2k.ico
 dl https://192.168.123.1:3923/cpp/scripts/pyinstaller/up2k.rc
 dl https://192.168.123.1:3923/cpp/scripts/pyinstaller/up2k.spec
@@ -37,12 +37,12 @@ grep -E '^from .ssl_ import' $APPDATA/python/python37/site-packages/urllib3/util
     echo golfed
 }
 
-read a b _ < <(awk -F\" '/^S_VERSION =/{$0=$2;sub(/\./," ");print}' < up2k.py)
+read a b _ < <(awk -F\" '/^S_VERSION =/{$0=$2;sub(/\./," ");print}' < u2c.py)
 sed -r 's/1,2,3,0/'$a,$b,0,0'/;s/1\.2\.3/'$a.$b.0/ <up2k.rc >up2k.rc2
 
-#python uncomment.py up2k.py
+#python uncomment.py u2c.py
 $APPDATA/python/python37/scripts/pyinstaller -y --clean --upx-dir=. up2k.spec
 
-./dist/up2k.exe --version
+./dist/u2c.exe --version
 
-curl -fkT dist/up2k.exe -HPW:wark https://192.168.123.1:3923/
+curl -fkT dist/u2c.exe -HPW:wark https://192.168.123.1:3923/

scripts/pyinstaller/up2k.spec

@@ -5,7 +5,7 @@ block_cipher = None
 
 
 a = Analysis(
-    ['up2k.py'],
+    ['u2c.py'],
     pathex=[],
     binaries=[],
     datas=[],
@@ -49,7 +49,7 @@ a = Analysis(
 
 # this is the only change to the autogenerated specfile:
 xdll = ["libcrypto-1_1.dll"]
-a.binaries = TOC([x for x in a.binaries if x[0] not in xdll])
+a.binaries = [x for x in a.binaries if x[0] not in xdll]
 
 pyz = PYZ(a.pure, a.zipped_data, cipher=block_cipher)
 
@@ -60,7 +60,7 @@ exe = EXE(
     a.zipfiles,
     a.datas,
     [],
-    name='up2k',
+    name='u2c',
     debug=False,
     bootloader_ignore_signals=False,
     strip=False,

scripts/pyinstaller/up2k.spec.sh

@@ -11,4 +11,4 @@ ex=(
   encodings.{zlib_codec,base64_codec,bz2_codec,charmap,hex_codec,palmos,punycode,rot_13}
 );
 cex=(); for a in "${ex[@]}"; do cex+=(--exclude "$a"); done
-$APPDATA/python/python37/scripts/pyi-makespec --version-file up2k.rc2 -i up2k.ico -n up2k -c -F up2k.py "${cex[@]}"
+$APPDATA/python/python37/scripts/pyi-makespec --version-file up2k.rc2 -i up2k.ico -n u2c -c -F u2c.py "${cex[@]}"

scripts/rls.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 set -e
 
-parallel=2
+parallel=1
 
 [ -e make-sfx.sh ] || cd scripts
 [ -e make-sfx.sh ] && [ -e deps-docker ] || {

scripts/sfx.ls

@@ -11,6 +11,7 @@ copyparty/broker_mp.py,
 copyparty/broker_mpw.py,
 copyparty/broker_thr.py,
 copyparty/broker_util.py,
+copyparty/cert.py,
 copyparty/cfg.py,
 copyparty/dxml.py,
 copyparty/fsutil.py,
@@ -20,9 +21,12 @@ copyparty/httpconn.py,
 copyparty/httpsrv.py,
 copyparty/ico.py,
 copyparty/mdns.py,
+copyparty/metrics.py,
 copyparty/mtag.py,
 copyparty/multicast.py,
+copyparty/pwhash.py,
 copyparty/res,
+copyparty/res/__init__.py,
 copyparty/res/COPYING.txt,
 copyparty/res/insecure.pem,
 copyparty/smbd.py,
@@ -62,7 +66,7 @@ copyparty/web,
 copyparty/web/a,
 copyparty/web/a/__init__.py,
 copyparty/web/a/partyfuse.py,
-copyparty/web/a/up2k.py,
+copyparty/web/a/u2c.py,
 copyparty/web/a/webdav-cfg.bat,
 copyparty/web/baguettebox.js,
 copyparty/web/browser.css,