v1.4.6

onscroll doesnt trigger so files dont load in

.gitignore

@@ -5,13 +5,16 @@ __pycache__/
 MANIFEST.in
 MANIFEST
 copyparty.egg-info/
-buildenv/
-build/
-dist/
-sfx/
-py2/
 .venv/
 
+/buildenv/
+/build/
+/dist/
+/py2/
+/sfx*
+/unt/
+/log/
+
 # ide
 *.sublime-workspace
 
@@ -19,6 +22,7 @@ py2/
 *.bak
 
 # derived
+copyparty/res/COPYING.txt
 copyparty/web/deps/
 srv/
 

.vscode/settings.json

@@ -23,7 +23,6 @@
         "terminal.ansiBrightWhite": "#ffffff",
     },
     "python.testing.pytestEnabled": false,
-    "python.testing.nosetestsEnabled": false,
     "python.testing.unittestEnabled": true,
     "python.testing.unittestArgs": [
         "-v",
@@ -35,18 +34,40 @@
     "python.linting.pylintEnabled": true,
     "python.linting.flake8Enabled": true,
     "python.linting.banditEnabled": true,
+    "python.linting.mypyEnabled": true,
+    "python.linting.mypyArgs": [
+        "--ignore-missing-imports",
+        "--follow-imports=silent",
+        "--show-column-numbers",
+        "--strict"
+    ],
     "python.linting.flake8Args": [
         "--max-line-length=120",
-        "--ignore=E722,F405,E203,W503,W293,E402",
+        "--ignore=E722,F405,E203,W503,W293,E402,E501,E128",
     ],
     "python.linting.banditArgs": [
         "--ignore=B104"
     ],
+    "python.linting.pylintArgs": [
+        "--disable=missing-module-docstring",
+        "--disable=missing-class-docstring",
+        "--disable=missing-function-docstring",
+        "--disable=wrong-import-position",
+        "--disable=raise-missing-from",
+        "--disable=bare-except",
+        "--disable=invalid-name",
+        "--disable=line-too-long",
+        "--disable=consider-using-f-string"
+    ],
+    // python3 -m isort --py=27 --profile=black copyparty/
     "python.formatting.provider": "black",
     "editor.formatOnSave": true,
     "[html]": {
         "editor.formatOnSave": false,
     },
+    "[css]": {
+        "editor.formatOnSave": false,
+    },
     "files.associations": {
         "*.makefile": "makefile"
     },

README.md

@@ -9,11 +9,12 @@
 turn your phone or raspi into a portable file server with resumable uploads/downloads using *any* web browser
 
 * server only needs `py2.7` or `py3.3+`, all dependencies optional
-* browse/upload with IE4 / netscape4.0 on win3.11 (heh)
-* *resumable* uploads need `firefox 34+` / `chrome 41+` / `safari 7+` for full speed
-* code standard: `black`
+* browse/upload with [IE4](#browser-support) / netscape4.0 on win3.11 (heh)
+* *resumable* uploads need `firefox 34+` / `chrome 41+` / `safari 7+`
 
-📷 **screenshots:** [browser](#the-browser) // [upload](#uploading) // [unpost](#unpost) // [thumbnails](#thumbnails) // [search](#searching) // [fsearch](#file-search) // [zip-DL](#zip-downloads) // [md-viewer](#markdown-viewer) // [ie4](#browser-support)
+try the **[read-only demo server](https://a.ocv.me/pub/demo/)** 👀 running from a basement in finland
+
+📷 **screenshots:** [browser](#the-browser) // [upload](#uploading) // [unpost](#unpost) // [thumbnails](#thumbnails) // [search](#searching) // [fsearch](#file-search) // [zip-DL](#zip-downloads) // [md-viewer](#markdown-viewer)
 
 
 ## get the app
@@ -43,26 +44,33 @@ turn your phone or raspi into a portable file server with resumable uploads/down
     * [tabs](#tabs) - the main tabs in the ui
     * [hotkeys](#hotkeys) - the browser has the following hotkeys
     * [navpane](#navpane) - switching between breadcrumbs or navpane
-    * [thumbnails](#thumbnails) - press `g` to toggle grid-view instead of the file listing
+    * [thumbnails](#thumbnails) - press `g` or `田` to toggle grid-view instead of the file listing
     * [zip downloads](#zip-downloads) - download folders (or file selections) as `zip` or `tar` files
     * [uploading](#uploading) - drag files/folders into the web-browser to upload
         * [file-search](#file-search) - dropping files into the browser also lets you see if they exist on the server
         * [unpost](#unpost) - undo/delete accidental uploads
+        * [self-destruct](#self-destruct) - uploads can be given a lifetime
     * [file manager](#file-manager) - cut/paste, rename, and delete files/folders (if you have permission)
     * [batch rename](#batch-rename) - select some files and press `F2` to bring up the rename UI
     * [markdown viewer](#markdown-viewer) - and there are *two* editors
     * [other tricks](#other-tricks)
     * [searching](#searching) - search by size, date, path/name, mp3-tags, ...
 * [server config](#server-config) - using arguments or config files, or a mix of both
+    * [qr-code](#qr-code) - print a qr-code [(screenshot)](https://user-images.githubusercontent.com/241032/194728533-6f00849b-c6ac-43c6-9359-83e454d11e00.png) for quick access
     * [ftp-server](#ftp-server) - an FTP server can be started using `--ftp 3921`
-    * [file indexing](#file-indexing)
-    * [upload rules](#upload-rules) - set upload rules using volume flags
+    * [file indexing](#file-indexing) - enables dedup and music search ++
+        * [exclude-patterns](#exclude-patterns) - to save some time
+        * [filesystem guards](#filesystem-guards) - avoid traversing into other filesystems
+        * [periodic rescan](#periodic-rescan) - filesystem monitoring
+    * [upload rules](#upload-rules) - set upload rules using volflags
     * [compress uploads](#compress-uploads) - files can be autocompressed on upload
+    * [other flags](#other-flags)
     * [database location](#database-location) - in-volume (`.hist/up2k.db`, default) or somewhere else
     * [metadata from audio files](#metadata-from-audio-files) - set `-e2t` to index tags on upload
-    * [file parser plugins](#file-parser-plugins) - provide custom parsers to index additional tags, also see [./bin/mtag/README.md](./bin/mtag/README.md)
+    * [file parser plugins](#file-parser-plugins) - provide custom parsers to index additional tags
     * [upload events](#upload-events) - trigger a script/program on each upload
     * [hiding from google](#hiding-from-google) - tell search engines you dont wanna be indexed
+    * [themes](#themes)
     * [complete examples](#complete-examples)
 * [browser support](#browser-support) - TLDR: yes
 * [client examples](#client-examples) - interact with copyparty using non-browser clients
@@ -86,6 +94,7 @@ turn your phone or raspi into a portable file server with resumable uploads/down
     * [optional gpl stuff](#optional-gpl-stuff)
 * [sfx](#sfx) - the self-contained "binary"
     * [sfx repack](#sfx-repack) - reduce the size of an sfx by removing features
+    * [copyparty.exe](#copypartyexe)
 * [install on android](#install-on-android)
 * [reporting bugs](#reporting-bugs) - ideas for context to include in bug reports
 * [building](#building)
@@ -100,15 +109,17 @@ turn your phone or raspi into a portable file server with resumable uploads/down
 
 download **[copyparty-sfx.py](https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.py)** and you're all set!
 
-running the sfx without arguments (for example doubleclicking it on Windows) will give everyone read/write access to the current folder; see `-h` for help if you want [accounts and volumes](#accounts-and-volumes) etc
+if you cannot install python, you can use [copyparty.exe](#copypartyexe) instead
+
+running the sfx without arguments (for example doubleclicking it on Windows) will give everyone read/write access to the current folder; you may want [accounts and volumes](#accounts-and-volumes)
 
 some recommended options:
 * `-e2dsa` enables general [file indexing](#file-indexing)
 * `-e2ts` enables audio metadata indexing (needs either FFprobe or Mutagen), see [optional dependencies](#optional-dependencies)
 * `-v /mnt/music:/music:r:rw,foo -a foo:bar` shares `/mnt/music` as `/music`, `r`eadable by anyone, and read-write for user `foo`, password `bar`
   * replace `:r:rw,foo` with `:r,foo` to only make the folder readable by `foo` and nobody else
-  * see [accounts and volumes](#accounts-and-volumes) for the syntax and other permissions (`r`ead, `w`rite, `m`ove, `d`elete, `g`et)
-* `--ls '**,*,ln,p,r'` to crash on startup if any of the volumes contain a symlink which point outside the volume, as that could give users unintended access
+  * see [accounts and volumes](#accounts-and-volumes) for the syntax and other permissions (`r`ead, `w`rite, `m`ove, `d`elete, `g`et, up`G`et)
+* `--ls '**,*,ln,p,r'` to crash on startup if any of the volumes contain a symlink which point outside the volume, as that could give users unintended access (see `--help-ls`)
 
 
 ### on servers
@@ -157,16 +168,18 @@ feature summary
   * ☑ volumes (mountpoints)
   * ☑ [accounts](#accounts-and-volumes)
   * ☑ [ftp-server](#ftp-server)
+  * ☑ [qr-code](#qr-code) for quick access
 * upload
   * ☑ basic: plain multipart, ie6 support
   * ☑ [up2k](#uploading): js, resumable, multithreaded
   * ☑ stash: simple PUT filedropper
   * ☑ [unpost](#unpost): undo/delete accidental uploads
+  * ☑ [self-destruct](#self-destruct) (specified server-side or client-side)
   * ☑ symlink/discard existing files (content-matching)
 * download
   * ☑ single files in browser
   * ☑ [folders as zip / tar files](#zip-downloads)
-  * ☑ FUSE client (read-only)
+  * ☑ [FUSE client](https://github.com/9001/copyparty/tree/hovudstraum/bin#copyparty-fusepy) (read-only)
 * browser
   * ☑ [navpane](#navpane) (directory tree sidebar)
   * ☑ file manager (cut/paste, delete, [batch-rename](#batch-rename))
@@ -174,7 +187,7 @@ feature summary
   * ☑ image gallery with webm player
   * ☑ textfile browser with syntax hilighting
   * ☑ [thumbnails](#thumbnails)
-    * ☑ ...of images using Pillow
+    * ☑ ...of images using Pillow, pyvips, or FFmpeg
     * ☑ ...of videos using FFmpeg
     * ☑ ...of audio (spectrograms) using FFmpeg
     * ☑ cache eviction (max-age; maybe max-size eventually)
@@ -202,6 +215,7 @@ project goals / philosophy
 * inverse linux philosophy -- do all the things, and do an *okay* job
   * quick drop-in service to get a lot of features in a pinch
   * there are probably [better alternatives](https://github.com/awesome-selfhosted/awesome-selfhosted) if you have specific/long-term needs
+    * but the resumable multithreaded uploads are p slick ngl
 * run anywhere, support everything
   * as many web-browsers and python versions as possible
     * every browser should at least be able to browse, download, upload files
@@ -232,7 +246,6 @@ some improvement ideas
 
 # bugs
 
-* Windows: python 3.7 and older cannot read tags with FFprobe, so use Mutagen or upgrade
 * Windows: python 2.7 cannot index non-ascii filenames with `-e2d`
 * Windows: python 2.7 cannot handle filenames with mojibake
 * `--th-ff-jpg` may fix video thumbnails on some FFmpeg versions (macos, some linux)
@@ -240,13 +253,27 @@ some improvement ideas
 
 ## general bugs
 
-* Windows: if the up2k db is on a samba-share or network disk, you'll get unpredictable behavior if the share is disconnected for a bit
+* Windows: if the `up2k.db` (filesystem index) is on a samba-share or network disk, you'll get unpredictable behavior if the share is disconnected for a bit
   * use `--hist` or the `hist` volflag (`-v [...]:c,hist=/tmp/foo`) to place the db on a local disk instead
 * all volumes must exist / be available on startup; up2k (mtp especially) gets funky otherwise
+* [the database can get stuck](https://github.com/9001/copyparty/issues/10)
+  * has only happened once but that is once too many
+  * luckily not dangerous for file integrity and doesn't really stop uploads or anything like that
+  * but would really appreciate some logs if anyone ever runs into it again
 * probably more, pls let me know
 
 ## not my bugs
 
+* [Chrome issue 1317069](https://bugs.chromium.org/p/chromium/issues/detail?id=1317069) -- if you try to upload a folder which contains symlinks by dragging it into the browser, the symlinked files will not get uploaded
+
+* [Chrome issue 1354816](https://bugs.chromium.org/p/chromium/issues/detail?id=1354816) -- chrome may eat all RAM uploading over plaintext http with `mt` enabled
+
+  * more amusingly, [Chrome issue 1354800](https://bugs.chromium.org/p/chromium/issues/detail?id=1354800) -- chrome may eat all RAM uploading in general (altho you probably won't run into this one)
+
+* [Chrome issue 1352210](https://bugs.chromium.org/p/chromium/issues/detail?id=1352210) -- plaintext http may be faster at filehashing than https (but also extremely CPU-intensive and likely to run into the above gc bugs)
+
+* [Firefox issue 1790500](https://bugzilla.mozilla.org/show_bug.cgi?id=1790500) -- sometimes forgets to close filedescriptors during upload so the browser can crash after ~4000 files
+
 * iPhones: the volume control doesn't work because [apple doesn't want it to](https://developer.apple.com/library/archive/documentation/AudioVideo/Conceptual/Using_HTML5_Audio_Video/Device-SpecificConsiderations/Device-SpecificConsiderations.html#//apple_ref/doc/uid/TP40009523-CH5-SW11)
   * *future workaround:* enable the equalizer, make it all-zero, and set a negative boost to reduce the volume
     * "future" because `AudioContext` is broken in the current iOS version (15.1), maybe one day...
@@ -260,6 +287,9 @@ some improvement ideas
 * VirtualBox: sqlite throws `Disk I/O Error` when running in a VM and the up2k database is in a vboxsf
   * use `--hist` or the `hist` volflag (`-v [...]:c,hist=/tmp/foo`) to place the db inside the vm instead
 
+* Ubuntu: dragging files from certain folders into firefox or chrome is impossible
+  * due to snap security policies -- see `snap connections firefox` for the allowlist, `removable-media` permits all of `/mnt` and `/media` apparently
+
 
 # FAQ
 
@@ -270,7 +300,7 @@ some improvement ideas
   * you can also do this with linux filesystem permissions; `chmod 111 music` will make it possible to access files and folders inside the `music` folder but not list the immediate contents -- also works with other software, not just copyparty
 
 * can I make copyparty download a file to my server if I give it a URL?
-  * not officially, but there is a [terrible hack](https://github.com/9001/copyparty/blob/hovudstraum/bin/mtag/wget.py) which makes it possible
+  * not really, but there is a [terrible hack](https://github.com/9001/copyparty/blob/hovudstraum/bin/mtag/wget.py) which makes it possible
 
 
 # accounts and volumes
@@ -278,6 +308,8 @@ some improvement ideas
 per-folder, per-user permissions  - if your setup is getting complex, consider making a [config file](./docs/example.conf) instead of using arguments
 * much easier to manage, and you can modify the config at runtime with `systemctl reload copyparty` or more conveniently using the `[reload cfg]` button in the control-panel (if logged in as admin)
 
+a quick summary can be seen using `--help-accounts`
+
 configuring accounts/volumes with arguments:
 * `-a usr:pwd` adds account `usr` with password `pwd`
 * `-v .::r` adds current-folder `.` as the webroot, `r`eadable by anyone
@@ -291,6 +323,7 @@ permissions:
 * `m` (move): move files/folders *from* this folder
 * `d` (delete): delete files/folders
 * `g` (get): only download files, cannot see folder contents or zip/tar
+* `G` (upget): same as `g` except uploaders get to see their own filekeys (see `fk` in examples below)
 
 examples:
 * add accounts named u1, u2, u3 with passwords p1, p2, p3: `-a u1:p1 -a u2:p2 -a u3:p3`
@@ -301,17 +334,21 @@ examples:
   * unauthorized users accessing the webroot can see that the `inc` folder exists, but cannot open it
   * `u1` can open the `inc` folder, but cannot see the contents, only upload new files to it
   * `u2` can browse it and move files *from* `/inc` into any folder where `u2` has write-access
-* make folder `/mnt/ss` available at `/i`, read-write for u1, get-only for everyone else, and enable accesskeys: `-v /mnt/ss:i:rw,u1:g:c,fk=4`
-  * `c,fk=4` sets the `fk` volume-flag to 4, meaning each file gets a 4-character accesskey
-  * `u1` can upload files, browse the folder, and see the generated accesskeys
-  * other users cannot browse the folder, but can access the files if they have the full file URL with the accesskey
+* make folder `/mnt/ss` available at `/i`, read-write for u1, get-only for everyone else, and enable filekeys: `-v /mnt/ss:i:rw,u1:g:c,fk=4`
+  * `c,fk=4` sets the `fk` (filekey) volflag to 4, meaning each file gets a 4-character accesskey
+  * `u1` can upload files, browse the folder, and see the generated filekeys
+  * other users cannot browse the folder, but can access the files if they have the full file URL with the filekey
+  * replacing the `g` permission with `wg` would let anonymous users upload files, but not see the required filekey to access it
+  * replacing the `g` permission with `wG` would let anonymous users upload files, receiving a working direct link in return
+
+anyone trying to bruteforce a password gets banned according to `--ban-pw`; default is 24h ban for 9 failed attempts in 1 hour
 
 
 # the browser
 
 accessing a copyparty server using a web-browser
 
-![copyparty-browser-fs8](https://user-images.githubusercontent.com/241032/129635359-d6dd9b07-8079-4020-ad77-2bfdb9ebd8d5.png)
+![copyparty-browser-fs8](https://user-images.githubusercontent.com/241032/192042695-522b3ec7-6845-494a-abdb-d1c0d0e23801.png)
 
 
 ## tabs
@@ -334,7 +371,7 @@ the browser has the following hotkeys  (always qwerty)
 * `I/K` prev/next folder
 * `M` parent folder (or unexpand current)
 * `V` toggle folders / textfiles in the navpane
-* `G` toggle list / [grid view](#thumbnails)
+* `G` toggle list / [grid view](#thumbnails) -- same as `田` bottom-right
 * `T` toggle thumbnails / icons
 * `ESC` close various things
 * `ctrl-X` cut selected files/folders
@@ -355,19 +392,24 @@ the browser has the following hotkeys  (always qwerty)
   * `U/O` skip 10sec back/forward
   * `0..9` jump to 0%..90%
   * `P` play/pause (also starts playing the folder)
+  * `Y` download file
 * when viewing images / playing videos:
   * `J/L, Left/Right` prev/next file
   * `Home/End` first/last file
+  * `F` toggle fullscreen
   * `S` toggle selection
   * `R` rotate clockwise (shift=ccw)
+  * `Y` download file
   * `Esc` close viewer
   * videos:
     * `U/O` skip 10sec back/forward
+    * `0..9` jump to 0%..90%
     * `P/K/Space` play/pause
-    * `F` fullscreen
-    * `C` continue playing next video
-    * `V` loop
     * `M` mute
+    * `C` continue playing next video
+    * `V` loop entire file
+    * `[` loop range (start)
+    * `]` loop range (end)
 * when the navpane is open:
   * `A/D` adjust tree width
 * in the [grid view](#thumbnails):
@@ -399,11 +441,13 @@ click the `🌲` or pressing the `B` hotkey to toggle between breadcrumbs path (
 
 ## thumbnails
 
-press `g` to toggle grid-view instead of the file listing,  and `t` toggles icons / thumbnails
+press `g` or `田` to toggle grid-view instead of the file listing  and `t` toggles icons / thumbnails
 
 ![copyparty-thumbs-fs8](https://user-images.githubusercontent.com/241032/129636211-abd20fa2-a953-4366-9423-1c88ebb96ba9.png)
 
-it does static images with Pillow and uses FFmpeg for video files, so you may want to `--no-thumb` or maybe just `--no-vthumb` depending on how dangerous your users are
+it does static images with Pillow / pyvips / FFmpeg, and uses FFmpeg for video files, so you may want to `--no-thumb` or maybe just `--no-vthumb` depending on how dangerous your users are
+* pyvips is 3x faster than Pillow, Pillow is 3x faster than FFmpeg
+* disable thumbnails for specific volumes with volflag `dthumb` for all, or `dvthumb` / `dathumb` / `dithumb` for video/audio/images only
 
 audio files are covnerted into spectrograms using FFmpeg unless you `--no-athumb` (and some FFmpeg builds may need `--th-ff-swr`)
 
@@ -439,13 +483,13 @@ you can also zip a selection of files or folders by clicking them in the browser
 
 ## uploading
 
-drag files/folders into the web-browser to upload
+drag files/folders into the web-browser to upload  (or use the [command-line uploader](https://github.com/9001/copyparty/tree/hovudstraum/bin#up2kpy))
 
 this initiates an upload using `up2k`; there are two uploaders available:
 * `[🎈] bup`, the basic uploader, supports almost every browser since netscape 4.0
-* `[🚀] up2k`, the fancy one
+* `[🚀] up2k`, the good / fancy one
 
-you can also undo/delete uploads by using `[🧯]` [unpost](#unpost)
+NB: you can undo/delete your own uploads with `[🧯]` [unpost](#unpost)
 
 up2k has several advantages:
 * you can drop folders into the browser (files are added recursively)
@@ -457,19 +501,19 @@ up2k has several advantages:
 * much higher speeds than ftp/scp/tarpipe on some internet connections (mainly american ones) thanks to parallel connections
 * the last-modified timestamp of the file is preserved
 
-see [up2k](#up2k) for details on how it works
+see [up2k](#up2k) for details on how it works, or watch a [demo video](https://a.ocv.me/pub/demo/pics-vids/#gf-0f6f5c0d)
 
 ![copyparty-upload-fs8](https://user-images.githubusercontent.com/241032/129635371-48fc54ca-fa91-48e3-9b1d-ba413e4b68cb.png)
 
 **protip:** you can avoid scaring away users with [contrib/plugins/minimal-up2k.html](contrib/plugins/minimal-up2k.html) which makes it look [much simpler](https://user-images.githubusercontent.com/241032/118311195-dd6ca380-b4ef-11eb-86f3-75a3ff2e1332.png)
 
-**protip:** if you enable `favicon` in the `[⚙️] settings` tab (by typing something into the textbox), the icon in the browser tab will indicate upload progress
+**protip:** if you enable `favicon` in the `[⚙️] settings` tab (by typing something into the textbox), the icon in the browser tab will indicate upload progress -- also, the `[🔔]` and/or `[🔊]` switches enable visible and/or audible notifications on upload completion
 
 the up2k UI is the epitome of polished inutitive experiences:
 * "parallel uploads" specifies how many chunks to upload at the same time
 * `[🏃]` analysis of other files should continue while one is uploading
+* `[🥔]` shows a simpler UI for faster uploads from slow devices
 * `[💭]` ask for confirmation before files are added to the queue
-* `[💤]` sync uploading between other copyparty browser-tabs so only one is active
 * `[🔎]` switch between upload and [file-search](#file-search) mode
   * ignore `[🔎]` if you add files by dragging them into the browser
 
@@ -481,7 +525,7 @@ and then theres the tabs below it,
   * plus up to 3 entries each from `[done]` and `[que]` for context
 * `[que]` is all the files that are still queued
 
-note that since up2k has to read each file twice, `[🎈 bup]` can *theoretically* be up to 2x faster in some extreme cases (files bigger than your ram, combined with an internet connection faster than the read-speed of your HDD, or if you're uploading from a cuo2duo)
+note that since up2k has to read each file twice, `[🎈] bup` can *theoretically* be up to 2x faster in some extreme cases (files bigger than your ram, combined with an internet connection faster than the read-speed of your HDD, or if you're uploading from a cuo2duo)
 
 if you are resuming a massive upload and want to skip hashing the files which already finished, you can enable `turbo` in the `[⚙️] config` tab, but please read the tooltip on that button
 
@@ -511,6 +555,17 @@ undo/delete accidental uploads
 you can unpost even if you don't have regular move/delete access, however only for files uploaded within the past `--unpost` seconds (default 12 hours) and the server must be running with `-e2d`
 
 
+### self-destruct
+
+uploads can be given a lifetime,  afer which they expire / self-destruct
+
+the feature must be enabled per-volume with the `lifetime` [upload rule](#upload-rules) which sets the upper limit for how long a file gets to stay on the server
+
+clients can specify a shorter expiration time using the [up2k ui](#uploading) -- the relevant options become visible upon navigating into a folder with `lifetimes` enabled -- or by using the `life` [upload modifier](#write)
+
+specifying a custom expiration time client-side will affect the timespan in which unposts are permitted, so keep an eye on the estimates in the up2k ui
+
+
 ## file manager
 
 cut/paste, rename, and delete files/folders (if you have permission)
@@ -592,7 +647,7 @@ and there are *two* editors
 
 * get a plaintext file listing by adding `?ls=t` to a URL, or a compact colored one with `?ls=v` (for unix terminals)
 
-* if you are using media hotkeys to switch songs and are getting tired of seeing the OSD popup which Windows doesn't let you disable, consider https://ocv.me/dev/?media-osd-bgone.ps1
+* if you are using media hotkeys to switch songs and are getting tired of seeing the OSD popup which Windows doesn't let you disable, consider [./contrib/media-osd-bgone.ps1](contrib/#media-osd-bgoneps1)
 
 * click the bottom-left `π` to open a javascript prompt for debugging
 
@@ -615,7 +670,9 @@ path/name queries are space-separated, AND'ed together, and words are negated wi
 * path: `shibayan -bossa` finds all files where one of the folders contain `shibayan` but filters out any results where `bossa` exists somewhere in the path
 * name: `demetori styx` gives you [good stuff](https://www.youtube.com/watch?v=zGh0g14ZJ8I&list=PL3A147BD151EE5218&index=9)
 
-add the argument `-e2ts` to also scan/index tags from music files, which brings us over to:
+the `raw` field allows for more complex stuff such as `( tags like *nhato* or tags like *taishi* ) and ( not tags like *nhato* or not tags like *taishi* )` which finds all songs by either nhato or taishi, excluding collabs (terrible example, why would you do that)
+
+for the above example to work, add the commandline argument `-e2ts` to also scan/index tags from music files, which brings us over to:
 
 
 # server config
@@ -626,6 +683,19 @@ using arguments or config files, or a mix of both:
   * or click the `[reload cfg]` button in the control-panel when logged in as admin 
 
 
+## qr-code
+
+print a qr-code [(screenshot)](https://user-images.githubusercontent.com/241032/194728533-6f00849b-c6ac-43c6-9359-83e454d11e00.png) for quick access,  great between phones on android hotspots which keep changing the subnet
+
+* `--qr` enables it
+* `--qrs` does https instead of http
+* `--qrl lootbox/?pw=hunter2` appends to the url, linking to the `lootbox` folder with password `hunter2`
+* `--qrz 1` forces 1x zoom instead of autoscaling to fit the terminal size
+  * 1x may render incorrectly on some terminals/fonts, but 2x should always work
+
+it will use your external ip (default route) unless `--qri` specifies an ip-prefix or domain
+
+
 ## ftp-server
 
 an FTP server can be started using `--ftp 3921`,  and/or `--ftps` for explicit TLS (ftpes)
@@ -640,7 +710,9 @@ an FTP server can be started using `--ftp 3921`,  and/or `--ftps` for explicit T
 
 ## file indexing
 
-file indexing relies on two database tables, the up2k filetree (`-e2d`) and the metadata tags (`-e2t`), stored in `.hist/up2k.db`. Configuration can be done through arguments, volume flags, or a mix of both.
+enables dedup and music search ++
+
+file indexing relies on two database tables, the up2k filetree (`-e2d`) and the metadata tags (`-e2t`), stored in `.hist/up2k.db`. Configuration can be done through arguments, volflags, or a mix of both.
 
 through arguments:
 * `-e2d` enables file indexing on upload
@@ -649,8 +721,11 @@ through arguments:
 * `-e2t` enables metadata indexing on upload
 * `-e2ts` also scans for tags in all files that don't have tags yet
 * `-e2tsr` also deletes all existing tags, doing a full reindex
+* `-e2v` verfies file integrity at startup, comparing hashes from the db
+* `-e2vu` patches the database with the new hashes from the filesystem
+* `-e2vp` panics and kills copyparty instead
 
-the same arguments can be set as volume flags, in addition to `d2d`, `d2ds`, `d2t`, `d2ts` for disabling:
+the same arguments can be set as volflags, in addition to `d2d`, `d2ds`, `d2t`, `d2ts`, `d2v` for disabling:
 * `-v ~/music::r:c,e2dsa,e2tsr` does a full reindex of everything on startup
 * `-v ~/music::r:c,d2d` disables **all** indexing, even if any `-e2*` are on
 * `-v ~/music::r:c,d2t` disables all `-e2t*` (tags), does not affect `-e2d*`
@@ -661,8 +736,11 @@ note:
 * the parser can finally handle `c,e2dsa,e2tsr` so you no longer have to `c,e2dsa:c,e2tsr`
 * `e2tsr` is probably always overkill, since `e2ds`/`e2dsa` would pick up any file modifications and `e2ts` would then reindex those, unless there is a new copyparty version with new parsers and the release note says otherwise
 * the rescan button in the admin panel has no effect unless the volume has `-e2ds` or higher
+* deduplication is possible on windows if you run copyparty as administrator (not saying you should!)
 
-to save some time, you can provide a regex pattern for filepaths to only index by filename/path/size/last-modified (and not the hash of the file contents) by setting `--no-hash \.iso$` or the volume-flag `:c,nohash=\.iso$`, this has the following consequences:
+### exclude-patterns
+
+to save some time,  you can provide a regex pattern for filepaths to only index by filename/path/size/last-modified (and not the hash of the file contents) by setting `--no-hash \.iso$` or the volflag `:c,nohash=\.iso$`, this has the following consequences:
 * initial indexing is way faster, especially when the volume is on a network disk
 * makes it impossible to [file-search](#file-search)
 * if someone uploads the same file contents, the upload will not be detected as a dupe, so it will not get symlinked or rejected
@@ -671,12 +749,29 @@ similarly, you can fully ignore files/folders using `--no-idx [...]` and `:c,noi
 
 if you set `--no-hash [...]` globally, you can enable hashing for specific volumes using flag `:c,nohash=`
 
+### filesystem guards
+
+avoid traversing into other filesystems  using `--xdev` / volflag `:c,xdev`, skipping any symlinks or bind-mounts to another HDD for example
+
+and/or you can `--xvol` / `:c,xvol` to ignore all symlinks leaving the volume's top directory, but still allow bind-mounts pointing elsewhere
+
+**NB: only affects the indexer** -- users can still access anything inside a volume, unless shadowed by another volume
+
+### periodic rescan
+
+filesystem monitoring;  if copyparty is not the only software doing stuff on your filesystem, you may want to enable periodic rescans to keep the index up to date
+
+argument `--re-maxage 60` will rescan all volumes every 60 sec, same as volflag `:c,scan=60` to specify it per-volume
+
+uploads are disabled while a rescan is happening, so rescans will be delayed by `--db-act` (default 10 sec) when there is write-activity going on (uploads, renames, ...)
+
 
 ## upload rules
 
-set upload rules using volume flags,  some examples:
+set upload rules using volflags,  some examples:
 
 * `:c,sz=1k-3m` sets allowed filesize between 1 KiB and 3 MiB inclusive (suffixes: `b`, `k`, `m`, `g`)
+* `:c,df=4g` block uploads if there would be less than 4 GiB free disk space afterwards
 * `:c,nosub` disallow uploading into subdirectories; goes well with `rotn` and `rotf`:
 * `:c,rotn=1000,2` moves uploads into subfolders, up to 1000 files in each folder before making a new one, two levels deep (must be at least 1)
 * `:c,rotf=%Y/%m/%d/%H` enforces files to be uploaded into a structure of subfolders according to that date format
@@ -695,16 +790,16 @@ you can also set transaction limits which apply per-IP and per-volume, but these
 
 files can be autocompressed on upload,  either on user-request (if config allows) or forced by server-config
 
-* volume flag `gz` allows gz compression
-* volume flag `xz` allows lzma compression
-* volume flag `pk` **forces** compression on all files
+* volflag `gz` allows gz compression
+* volflag `xz` allows lzma compression
+* volflag `pk` **forces** compression on all files
 * url parameter `pk` requests compression with server-default algorithm
 * url parameter `gz` or `xz` requests compression with a specific algorithm
 * url parameter `xz` requests xz compression
 
 things to note,
 * the `gz` and `xz` arguments take a single optional argument, the compression level (range 0 to 9)
-* the `pk` volume flag takes the optional argument `ALGORITHM,LEVEL` which will then be forced for all uploads, for example `gz,9` or `xz,0`
+* the `pk` volflag takes the optional argument `ALGORITHM,LEVEL` which will then be forced for all uploads, for example `gz,9` or `xz,0`
 * default compression is gzip level 9
 * all upload methods except up2k are supported
 * the files will be indexed after compression, so dupe-detection and file-search will not work as expected
@@ -718,13 +813,18 @@ some examples,
   allows (but does not force) gz compression if client uploads to `/inc?pk` or `/inc?gz` or `/inc?gz=4`
 
 
+## other flags
+
+* `:c,magic` enables filetype detection for nameless uploads, same as `--magic`
+
+
 ## database location
 
 in-volume (`.hist/up2k.db`, default) or somewhere else
 
 copyparty creates a subfolder named `.hist` inside each volume where it stores the database, thumbnails, and some other stuff
 
-this can instead be kept in a single place using the `--hist` argument, or the `hist=` volume flag, or a mix of both:
+this can instead be kept in a single place using the `--hist` argument, or the `hist=` volflag, or a mix of both:
 * `--hist ~/.cache/copyparty -v ~/music::r:c,hist=-` sets `~/.cache/copyparty` as the default place to put volume info, but `~/music` gets the regular `.hist` subfolder (`-` restores default behavior)
 
 note:
@@ -757,21 +857,31 @@ see the beautiful mess of a dictionary in [mtag.py](https://github.com/9001/copy
 * avoids pulling any GPL code into copyparty
 * more importantly runs FFprobe on incoming files which is bad if your FFmpeg has a cve
 
+`--mtag-to` sets the tag-scan timeout; very high default (60 sec) to cater for zfs and other randomly-freezing filesystems. Lower values like 10 are usually safe, allowing for faster processing of tricky files
+
 
 ## file parser plugins
 
-provide custom parsers to index additional tags, also see [./bin/mtag/README.md](./bin/mtag/README.md)
+provide custom parsers to index additional tags,  also see [./bin/mtag/README.md](./bin/mtag/README.md)
 
-copyparty can invoke external programs to collect additional metadata for files using `mtp` (either as argument or volume flag), there is a default timeout of 30sec
+copyparty can invoke external programs to collect additional metadata for files using `mtp` (either as argument or volflag), there is a default timeout of 60sec, and only files which contain audio get analyzed by default (see ay/an/ad below)
 
 * `-mtp .bpm=~/bin/audio-bpm.py` will execute `~/bin/audio-bpm.py` with the audio file as argument 1 to provide the `.bpm` tag, if that does not exist in the audio metadata
 * `-mtp key=f,t5,~/bin/audio-key.py` uses `~/bin/audio-key.py` to get the `key` tag, replacing any existing metadata tag (`f,`), aborting if it takes longer than 5sec (`t5,`)
 * `-v ~/music::r:c,mtp=.bpm=~/bin/audio-bpm.py:c,mtp=key=f,t5,~/bin/audio-key.py` both as a per-volume config wow this is getting ugly
 
-*but wait, there's more!* `-mtp` can be used for non-audio files as well using the `a` flag: `ay` only do audio files, `an` only do non-audio files, or `ad` do all files (d as in dontcare) 
+*but wait, there's more!* `-mtp` can be used for non-audio files as well using the `a` flag: `ay` only do audio files (default), `an` only do non-audio files, or `ad` do all files (d as in dontcare)
 
+* "audio file" also means videos btw, as long as there is an audio stream
 * `-mtp ext=an,~/bin/file-ext.py` runs `~/bin/file-ext.py` to get the `ext` tag only if file is not audio (`an`)
 * `-mtp arch,built,ver,orig=an,eexe,edll,~/bin/exe.py` runs `~/bin/exe.py` to get properties about windows-binaries only if file is not audio (`an`) and file extension is exe or dll
+* if you want to daisychain parsers, use the `p` flag to set processing order
+  * `-mtp foo=p1,~/a.py` runs before `-mtp foo=p2,~/b.py` and will forward all the tags detected so far as json to the stdin of b.py
+* option `c0` disables capturing of stdout/stderr, so copyparty will not receive any tags from the process at all -- instead the invoked program is free to print whatever to the console, just using copyparty as a launcher
+  * `c1` captures stdout only, `c2` only stderr, and `c3` (default) captures both
+* you can control how the parser is killed if it times out with option `kt` killing the entire process tree (default), `km` just the main process, or `kn` let it continue running until copyparty is terminated
+
+if something doesn't work, try `--mtag-v` for verbose error messages
 
 
 ## upload events
@@ -779,16 +889,16 @@ copyparty can invoke external programs to collect additional metadata for files
 trigger a script/program on each upload  like so:
 
 ```
--v /mnt/inc:inc:w:c,mte=+a1:c,mtp=a1=ad,/usr/bin/notify-send
+-v /mnt/inc:inc:w:c,mte=+x1:c,mtp=x1=ad,kn,/usr/bin/notify-send
 ```
 
-so filesystem location `/mnt/inc` shared at `/inc`, write-only for everyone, appending `a1` to the list of tags to index, and using `/usr/bin/notify-send` to "provide" that tag
+so filesystem location `/mnt/inc` shared at `/inc`, write-only for everyone, appending `x1` to the list of tags to index (`mte`), and using `/usr/bin/notify-send` to "provide" tag `x1` for any filetype (`ad`) with kill-on-timeout disabled (`kn`)
 
 that'll run the command `notify-send` with the path to the uploaded file as the first and only argument (so on linux it'll show a notification on-screen)
 
 note that it will only trigger on new unique files, not dupes
 
-and it will occupy the parsing threads, so fork anything expensive, or if you want to intentionally queue/singlethread you can combine it with `--mtag-mt 1`
+and it will occupy the parsing threads, so fork anything expensive (or set `kn` to have copyparty fork it for you) -- otoh if you want to intentionally queue/singlethread you can combine it with `--mtag-mt 1`
 
 if this becomes popular maybe there should be a less janky way to do it actually
 
@@ -798,16 +908,48 @@ if this becomes popular maybe there should be a less janky way to do it actually
 tell search engines you dont wanna be indexed,  either using the good old [robots.txt](https://www.robotstxt.org/robotstxt.html) or through copyparty settings:
 
 * `--no-robots` adds HTTP (`X-Robots-Tag`) and HTML (`<meta>`) headers with `noindex, nofollow` globally
-* volume-flag `[...]:c,norobots` does the same thing for that single volume
-* volume-flag `[...]:c,robots` ALLOWS search-engine crawling for that volume, even if `--no-robots` is set globally
+* volflag `[...]:c,norobots` does the same thing for that single volume
+* volflag `[...]:c,robots` ALLOWS search-engine crawling for that volume, even if `--no-robots` is set globally
 
 also, `--force-js` disables the plain HTML folder listing, making things harder to parse for search engines
 
 
+## themes
+
+you can change the default theme with `--theme 2`, and add your own themes by modifying `browser.css` or providing your own css to `--css-browser`, then telling copyparty they exist by increasing `--themes`
+
+<table><tr><td width="33%" align="center"><a href="https://user-images.githubusercontent.com/241032/165864907-17e2ac7d-319d-4f25-8718-2f376f614b51.png"><img src="https://user-images.githubusercontent.com/241032/165867551-fceb35dd-38f0-42bb-bef3-25ba651ca69b.png"></a>
+0. classic dark</td><td width="33%" align="center"><a href="https://user-images.githubusercontent.com/241032/168644399-68938de5-da9b-445f-8d92-b51c74b5f345.png"><img src="https://user-images.githubusercontent.com/241032/168644404-8e1a2fdc-6e59-4c41-905e-ba5399ed686f.png"></a>
+2. flat pm-monokai</td><td width="33%" align="center"><a href="https://user-images.githubusercontent.com/241032/165864901-db13a429-a5da-496d-8bc6-ce838547f69d.png"><img src="https://user-images.githubusercontent.com/241032/165867560-aa834aef-58dc-4abe-baef-7e562b647945.png"></a>
+4. vice</td></tr><tr><td align="center"><a href="https://user-images.githubusercontent.com/241032/165864905-692682eb-6fb4-4d40-b6fe-27d2c7d3e2a7.png"><img src="https://user-images.githubusercontent.com/241032/165867555-080b73b6-6d85-41bb-a7c6-ad277c608365.png"></a>
+1. classic light</td><td align="center"><a href="https://user-images.githubusercontent.com/241032/168645276-fb02fd19-190a-407a-b8d3-d58fee277e02.png"><img src="https://user-images.githubusercontent.com/241032/168645280-f0662b3c-9764-4875-a2e2-d91cc8199b23.png"></a>
+3. flat light
+</td><td align="center"><a href="https://user-images.githubusercontent.com/241032/165864898-10ce7052-a117-4fcf-845b-b56c91687908.png"><img src="https://user-images.githubusercontent.com/241032/165867562-f3003d45-dd2a-4564-8aae-fed44c1ae064.png"></a>
+5. <a href="https://blog.codinghorror.com/a-tribute-to-the-windows-31-hot-dog-stand-color-scheme/">hotdog stand</a></td></tr></table>
+
+the classname of the HTML tag is set according to the selected theme, which is used to set colors as css variables ++
+
+* each theme *generally* has a dark theme (even numbers) and a light theme (odd numbers), showing in pairs
+* the first theme (theme 0 and 1) is `html.a`, second theme (2 and 3) is `html.b`
+* if a light theme is selected, `html.y` is set, otherwise `html.z` is
+* so if the dark edition of the 2nd theme is selected, you use any of `html.b`, `html.z`, `html.bz` to specify rules
+
+see the top of [./copyparty/web/browser.css](./copyparty/web/browser.css) where the color variables are set, and there's layout-specific stuff near the bottom
+
+
 ## complete examples
 
-* read-only music server with bpm and key scanning  
-  `python copyparty-sfx.py -v /mnt/nas/music:/music:r -e2dsa -e2ts -mtp .bpm=f,audio-bpm.py -mtp key=f,audio-key.py`
+* read-only music server  
+  `python copyparty-sfx.py -v /mnt/nas/music:/music:r -e2dsa -e2ts --no-robots --force-js --theme 2`
+  
+  * ...with bpm and key scanning  
+    `-mtp .bpm=f,audio-bpm.py -mtp key=f,audio-key.py`
+  
+  * ...with a read-write folder for `kevin` whose password is `okgo`  
+    `-a kevin:okgo -v /mnt/nas/inc:/inc:rw,kevin`
+  
+  * ...with logging to disk  
+    `-lo log/cpp-%Y-%m%d-%H%M%S.txt.xz`
 
 
 # browser support
@@ -853,7 +995,8 @@ quick summary of more eccentric web-browsers trying to view a directory index:
 | **w3m** (0.5.3/macports)  | can browse, login, upload at 100kB/s, mkdir/msg |
 | **netsurf** (3.10/arch)   | is basically ie6 with much better css (javascript has almost no effect) | 
 | **opera** (11.60/winxp)   | OK: thumbnails, image-viewer, zip-selection, rename/cut/paste. NG: up2k, navpane, markdown, audio |
-| **ie4** and **netscape** 4.0  | can browse, upload with `?b=u` |
+| **ie4** and **netscape** 4.0  | can browse, upload with `?b=u`, auth with `&pw=wark` |
+| **ncsa mosaic** 2.7       | does not get a pass, [pic1](https://user-images.githubusercontent.com/241032/174189227-ae816026-cf6f-4be5-a26e-1b3b072c1b2f.png) - [pic2](https://user-images.githubusercontent.com/241032/174189225-5651c059-5152-46e9-ac26-7e98e497901b.png) |
 | **SerenityOS** (7e98457)  | hits a page fault, works with `?b=u`, file upload not-impl |
 
 
@@ -866,7 +1009,9 @@ interact with copyparty using non-browser clients
   * `var xhr = new XMLHttpRequest(); xhr.open('POST', '//127.0.0.1:3923/msgs?raw'); xhr.send('foo');`
 
 * curl/wget: upload some files (post=file, chunk=stdin)
-  * `post(){ curl -b cppwd=wark -F act=bput -F f=@"$1" http://127.0.0.1:3923/;}`  
+  * `post(){ curl -F act=bput -F f=@"$1" http://127.0.0.1:3923/?pw=wark;}`  
+    `post movie.mkv`
+  * `post(){ curl -b cppwd=wark -H rand:8 -T "$1" http://127.0.0.1:3923/;}`  
     `post movie.mkv`
   * `post(){ wget --header='Cookie: cppwd=wark' --post-file="$1" -O- http://127.0.0.1:3923/?raw;}`  
     `post movie.mkv`
@@ -892,7 +1037,9 @@ copyparty returns a truncated sha512sum of your PUT/POST as base64; you can gene
     b512(){ printf "$((sha512sum||shasum -a512)|sed -E 's/ .*//;s/(..)/\\x\1/g')"|base64|tr '+/' '-_'|head -c44;}
     b512 <movie.mkv
 
-you can provide passwords using cookie `cppwd=hunter2`, as a url query `?pw=hunter2`, or with basic-authentication (either as the username or password)
+you can provide passwords using cookie `cppwd=hunter2`, as a url-param `?pw=hunter2`, or with basic-authentication (either as the username or password)
+
+NOTE: curl will not send the original filename if you use `-T` combined with url-params! Also, make sure to always leave a trailing slash in URLs unless you want to override the filename
 
 
 # up2k
@@ -910,19 +1057,33 @@ quick outline of the up2k protocol, see [uploading](#uploading) for the web-clie
   * server writes chunks into place based on the hash
 * client does another handshake with the hashlist; server replies with OK or a list of chunks to reupload
 
-up2k has saved a few uploads from becoming corrupted in-transfer already; caught an android phone on wifi redhanded in wireshark with a bitflip, however bup with https would *probably* have noticed as well (thanks to tls also functioning as an integrity check)
+up2k has saved a few uploads from becoming corrupted in-transfer already;
+* caught an android phone on wifi redhanded in wireshark with a bitflip, however bup with https would *probably* have noticed as well (thanks to tls also functioning as an integrity check)
+* also stopped someone from uploading because their ram was bad
+
+regarding the frequent server log message during uploads;  
+`6.0M 106M/s 2.77G 102.9M/s n948 thank 4/0/3/1 10042/7198 00:01:09`
+* this chunk was `6 MiB`, uploaded at `106 MiB/s`
+* on this http connection, `2.77 GiB` transferred, `102.9 MiB/s` average, `948` chunks handled
+* client says `4` uploads OK, `0` failed, `3` busy, `1` queued, `10042 MiB` total size, `7198 MiB` and `00:01:09` left
 
 
 ## why chunk-hashes
 
 a single sha512 would be better, right?
 
-this is due to `crypto.subtle` not providing a streaming api (or the option to seed the sha512 hasher with a starting hash)
+this is due to `crypto.subtle` [not yet](https://github.com/w3c/webcrypto/issues/73) providing a streaming api (or the option to seed the sha512 hasher with a starting hash)
 
 as a result, the hashes are much less useful than they could have been (search the server by sha512, provide the sha512 in the response http headers, ...)
 
+however it allows for hashing multiple chunks in parallel, greatly increasing upload speed from fast storage (NVMe, raid-0 and such)
+
+* both the [browser uploader](#uploading) and the [commandline one](https://github.com/9001/copyparty/blob/hovudstraum/bin/up2k.py) does this now, allowing for fast uploading even from plaintext http
+
 hashwasm would solve the streaming issue but reduces hashing speed for sha512 (xxh128 does 6 GiB/s), and it would make old browsers and [iphones](https://bugs.webkit.org/show_bug.cgi?id=228552) unsupported
 
+* blake2 might be a better choice since xxh is non-cryptographic, but that gets ~15 MiB/s on slower androids
+
 
 # performance
 
@@ -934,6 +1095,7 @@ below are some tweaks roughly ordered by usefulness:
 * `--http-only` or `--https-only` (unless you want to support both protocols) will reduce the delay before a new connection is established
 * `--hist` pointing to a fast location (ssd) will make directory listings and searches faster when `-e2d` or `-e2t` is set
 * `--no-hash .` when indexing a network-disk if you don't care about the actual filehashes and only want the names/tags searchable
+* `--no-htp --hash-mt=0 --th-mt=1` minimizes the number of threads; can help in some eccentric environments (like the vscode debugger)
 * `-j` enables multiprocessing (actual multithreading) and can make copyparty perform better in cpu-intensive workloads, for example:
   * huge amount of short-lived connections
   * really heavy traffic (downloads/uploads)
@@ -952,6 +1114,7 @@ when uploading files,
 
 * if you're cpu-bottlenecked, or the browser is maxing a cpu core:
   * up to 30% faster uploads if you hide the upload status list by switching away from the `[🚀]` up2k ui-tab (or closing it)
+    * optionally you can switch to the lightweight potato ui by clicking the `[🥔]`
     * switching to another browser-tab also works, the favicon will update every 10 seconds in that case
   * unlikely to be a problem, but can happen when uploding many small files, or your internet is too fast, or PC too slow
 
@@ -960,16 +1123,31 @@ when uploading files,
 
 some notes on hardening
 
-on public copyparty instances with anonymous upload enabled:
+* option `-s` is a shortcut to set the following options:
+  * `--no-thumb` disables thumbnails and audio transcoding to stop copyparty from running `FFmpeg`/`Pillow`/`VIPS` on uploaded files, which is a [good idea](https://www.cvedetails.com/vulnerability-list.php?vendor_id=3611) if anonymous upload is enabled
+  * `--no-mtag-ff` uses `mutagen` to grab music tags instead of `FFmpeg`, which is safer and faster but less accurate
+  * `--dotpart` hides uploads from directory listings while they're still incoming
+  * `--no-robots` and `--force-js` makes life harder for crawlers, see [hiding from google](#hiding-from-google)
 
-* users can upload html/css/js which will evaluate for other visitors in a few ways,
-  * unless `--no-readme` is set: by uploading/modifying a file named `readme.md`
-  * if `move` access is granted AND none of `--no-logues`, `--no-dot-mv`, `--no-dot-ren` is set: by uploading some .html file and renaming it to `.epilogue.html` (uploading it directly is blocked)
+* option `-ss` is a shortcut for the above plus:
+  * `--no-logues` and `--no-readme` disables support for readme's and prologues / epilogues in directory listings, which otherwise lets people upload arbitrary `<script>` tags
+  * `--unpost 0`, `--no-del`, `--no-mv` disables all move/delete support
+  * `--hardlink` creates hardlinks instead of symlinks when deduplicating uploads, which is less maintenance
+    * however note if you edit one file it will also affect the other copies
+  * `--vague-403` returns a "404 not found" instead of "403 forbidden" which is a common enterprise meme
+  * `--ban-404=50,60,1440` ban client for 1440min (24h) if they hit 50 404's in 60min
+    * **NB:** will ban anyone who enables up2k turbo
+  * `--nih` removes the server hostname from directory listings
 
-other misc:
+* option `-sss` is a shortcut for the above plus:
+  * `-lo cpp-%Y-%m%d-%H%M%S.txt.xz` enables logging to disk
+  * `-ls **,*,ln,p,r` does a scan on startup for any dangerous symlinks
+
+other misc notes:
 
 * you can disable directory listings by giving permission `g` instead of `r`, only accepting direct URLs to files
-  * combine this with volume-flag `c,fk` to generate per-file accesskeys; users which have full read-access will then see URLs with `?k=...` appended to the end, and `g` users must provide that URL including the correct key to avoid a 404
+  * combine this with volflag `c,fk` to generate filekeys (per-file accesskeys); users which have full read-access will then see URLs with `?k=...` appended to the end, and `g` users must provide that URL including the correct key to avoid a 404
+  * permissions `wG` lets users upload files and receive their own filekeys, still without being able to see other uploads
 
 
 ## gotchas
@@ -1060,7 +1238,17 @@ authenticate using header `Cookie: cppwd=foo` or url param `&pw=foo`
 | uPOST | | `msg=foo` | send message `foo` into server log |
 | mPOST | | `act=tput`, `body=TEXT` | overwrite markdown document at URL |
 
-server behavior of `msg` can be reconfigured with `--urlform`
+upload modifiers:
+
+| http-header | url-param | effect |
+|--|--|--|
+| `Accept: url` | `want=url` | return just the file URL |
+| `Rand: 4` | `rand=4` | generate random filename with 4 characters |
+| `Life: 30` | `life=30` | delete file after 30 seconds |
+
+* `life` only has an effect if the volume has a lifetime, and the volume lifetime must be greater than the file's
+
+* server behavior of `msg` can be reconfigured with `--urlform`
 
 ## admin
 
@@ -1096,10 +1284,13 @@ enable music tags:
 * or `ffprobe` (20x slower, more accurate, possibly dangerous depending on your distro and users)
 
 enable [thumbnails](#thumbnails) of...
-* **images:** `Pillow` (requires py2.7 or py3.5+)
+* **images:** `Pillow` and/or `pyvips` and/or `ffmpeg` (requires py2.7 or py3.5+)
 * **videos/audio:** `ffmpeg` and `ffprobe` somewhere in `$PATH`
-* **HEIF pictures:** `pyheif-pillow-opener` (requires Linux or a C compiler)
-* **AVIF pictures:** `pillow-avif-plugin`
+* **HEIF pictures:** `pyvips` or `ffmpeg` or `pyheif-pillow-opener` (requires Linux or a C compiler)
+* **AVIF pictures:** `pyvips` or `ffmpeg` or `pillow-avif-plugin`
+* **JPEG XL pictures:** `pyvips` or `ffmpeg`
+
+`pyvips` gives higher quality thumbnails than `Pillow` and is 320% faster, using 270% more ram: `sudo apt install libvips42 && python3 -m pip install --user -U pyvips`
 
 
 ## install recommended deps
@@ -1140,40 +1331,64 @@ for the `re`pack to work, first run one of the sfx'es once to unpack it
 **note:** you can also just download and run [scripts/copyparty-repack.sh](scripts/copyparty-repack.sh) -- this will grab the latest copyparty release from github and do a few repacks; works on linux/macos (and windows with msys2 or WSL)
 
 
+## copyparty.exe
+
+![copyparty-exe-fs8](https://user-images.githubusercontent.com/241032/194707422-cb7f66c9-41a2-4cb9-8dbc-2ab866cd4338.png)
+
+[copyparty.exe](https://github.com/9001/copyparty/releases/latest/download/copyparty.exe) can be convenient on old machines where installing python is problematic, however is **not recommended** and should be considered a last resort -- if possible, please use **[copyparty-sfx.py](https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.py)** instead
+
+the exe is compatible with 32bit windows7, which means it uses an ancient copy of python (3.7.9) which cannot be upgraded and will definitely become a security hazard at some point
+
+meanwhile [copyparty-sfx.py](https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.py) instead relies on your system python which gives better performance and will stay safe as long as you keep your python install up-to-date
+
+then again, if you are already into downloading shady binaries from the internet, you may also want my [minimal builds](./scripts/pyinstaller#ffmpeg) of [ffmpeg](https://ocv.me/stuff/bin/ffmpeg.exe) and [ffprobe](https://ocv.me/stuff/bin/ffprobe.exe) which enables copyparty to extract multimedia-info, do audio-transcoding, and thumbnails/spectrograms/waveforms, however it's much better to instead grab a [recent official build](https://github.com/BtbN/FFmpeg-Builds/releases/download/latest/ffmpeg-master-latest-win64-gpl.zip) every once ina while if you can afford the size
+
+
 # install on android
 
 install [Termux](https://termux.com/) (see [ocv.me/termux](https://ocv.me/termux/)) and then copy-paste this into Termux (long-tap) all at once:
 ```sh
-apt update && apt -y full-upgrade && termux-setup-storage && apt -y install python && python -m ensurepip && python -m pip install -U copyparty
+apt update && apt -y full-upgrade && apt update && termux-setup-storage && apt -y install python && python -m ensurepip && python -m pip install --user -U copyparty
 echo $?
 ```
 
-after the initial setup, you can launch copyparty at any time by running `copyparty` anywhere in Termux
+after the initial setup, you can launch copyparty at any time by running `copyparty` anywhere in Termux -- and if you run it with `--qr` you'll get a [neat qr-code](#qr-code) pointing to your external ip
+
+if you want thumbnails, `apt -y install ffmpeg`
+
+* or if you want to use vips instead, `apt -y install libvips && python -m pip install --user -U wheel && python -m pip install --user -U pyvips && (cd /data/data/com.termux/files/usr/lib/; ln -s libgobject-2.0.so{,.0}; ln -s libvips.so{,.42})`
 
 
 # reporting bugs
 
 ideas for context to include in bug reports
 
+in general, commandline arguments (and config file if any)
+
 if something broke during an upload (replacing FILENAME with a part of the filename that broke):
 ```
 journalctl -aS '48 hour ago' -u copyparty | grep -C10 FILENAME | tee bug.log
 ```
 
+if there's a wall of base64 in the log (thread stacks) then please include that, especially if you run into something freezing up or getting stuck, for example `OperationalError('database is locked')` -- alternatively you can visit `/?stack` to see the stacks live, so http://127.0.0.1:3923/?stack for example
+
 
 # building
 
 ## dev env setup
 
-mostly optional; if you need a working env for vscode or similar
+you need python 3.9 or newer due to type hints
+
+the rest is mostly optional; if you need a working env for vscode or similar
 
 ```sh
 python3 -m venv .venv
 . .venv/bin/activate
-pip install jinja2  # mandatory
+pip install jinja2 strip_hints  # MANDATORY
 pip install mutagen  # audio metadata
+pip install pyftpdlib  # ftp server
 pip install Pillow pyheif-pillow-opener pillow-avif-plugin  # thumbnails
-pip install black bandit pylint flake8  # vscode tooling
+pip install black==21.12b0 click==8.0.2 bandit pylint flake8 isort mypy  # vscode tooling
 ```
 
 
@@ -1184,7 +1399,7 @@ first grab the web-dependencies from a previous sfx (assuming you don't need to
 ```sh
 rm -rf copyparty/web/deps
 curl -L https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.py >x.py
-python3 x.py -h
+python3 x.py --version
 rm x.py
 mv /tmp/pe-copyparty/copyparty/web/deps/ copyparty/web/deps/
 ```
@@ -1204,10 +1419,7 @@ also builds the sfx so skip the sfx section above
 in the `scripts` folder:
 
 * run `make -C deps-docker` to build all dependencies
-* `git tag v1.2.3 && git push origin --tags`
-* upload to pypi with `make-pypi-release.(sh|bat)`
-* create github release with `make-tgz-release.sh`
-* create sfx with `make-sfx.sh`
+* run `./rls.sh 1.2.3` which uploads to pypi + creates github release + sfx
 
 
 # todo
@@ -1234,7 +1446,7 @@ roughly sorted by priority
 * up2k partials ui
   * feels like there isn't much point
 * cache sha512 chunks on client
-  * too dangerous
+  * too dangerous -- overtaken by turbo mode
 * comment field
   * nah
 * look into android thumbnail cache file format

bin/copyparty-fuse-streaming.py

@@ -42,6 +42,7 @@ import threading
 import traceback
 import http.client  # py2: httplib
 import urllib.parse
+import calendar
 from datetime import datetime
 from urllib.parse import quote_from_bytes as quote
 from urllib.parse import unquote_to_bytes as unquote
@@ -495,7 +496,7 @@ class Gateway(object):
                 ts = 60 * 60 * 24 * 2
                 try:
                     sz = int(fsize)
-                    ts = datetime.strptime(fdate, "%Y-%m-%d %H:%M:%S").timestamp()
+                    ts = calendar.timegm(time.strptime(fdate, "%Y-%m-%d %H:%M:%S"))
                 except:
                     info("bad HTML or OS [{}] [{}]".format(fdate, fsize))
                     # python cannot strptime(1959-01-01) on windows

bin/copyparty-fuse.py

@@ -45,6 +45,7 @@ import threading
 import traceback
 import http.client  # py2: httplib
 import urllib.parse
+import calendar
 from datetime import datetime
 from urllib.parse import quote_from_bytes as quote
 from urllib.parse import unquote_to_bytes as unquote
@@ -443,7 +444,7 @@ class Gateway(object):
                 ts = 60 * 60 * 24 * 2
                 try:
                     sz = int(fsize)
-                    ts = datetime.strptime(fdate, "%Y-%m-%d %H:%M:%S").timestamp()
+                    ts = calendar.timegm(time.strptime(fdate, "%Y-%m-%d %H:%M:%S"))
                 except:
                     info("bad HTML or OS [{}] [{}]".format(fdate, fsize))
                     # python cannot strptime(1959-01-01) on windows

bin/dbtool.py

@@ -8,7 +8,10 @@ import sqlite3
 import argparse
 
 DB_VER1 = 3
-DB_VER2 = 4
+DB_VER2 = 5
+
+BY_PATH = None
+NC = None
 
 
 def die(msg):
@@ -57,8 +60,13 @@ def compare(n1, d1, n2, d2, verbose):
         if rd.split("/", 1)[0] == ".hist":
             continue
 
-        q = "select w from up where rd = ? and fn = ?"
-        hit = d2.execute(q, (rd, fn)).fetchone()
+        if BY_PATH:
+            q = "select w from up where rd = ? and fn = ?"
+            hit = d2.execute(q, (rd, fn)).fetchone()
+        else:
+            q = "select w from up where substr(w,1,16) = ? and +w = ?"
+            hit = d2.execute(q, (w1[:16], w1)).fetchone()
+
         if not hit:
             miss += 1
             if verbose:
@@ -70,27 +78,32 @@ def compare(n1, d1, n2, d2, verbose):
     n = 0
     miss = {}
     nmiss = 0
-    for w1, k, v in d1.execute("select * from mt"):
+    for w1s, k, v in d1.execute("select * from mt"):
 
         n += 1
         if n % 100_000 == 0:
             m = f"\033[36mchecked {n:,} of {nt:,} tags in {n1} against {n2}, so far {nmiss} missing tags\033[0m"
             print(m)
 
-        q = "select rd, fn from up where substr(w,1,16) = ?"
-        rd, fn = d1.execute(q, (w1,)).fetchone()
+        q = "select w, rd, fn from up where substr(w,1,16) = ?"
+        w1, rd, fn = d1.execute(q, (w1s,)).fetchone()
         if rd.split("/", 1)[0] == ".hist":
             continue
 
-        q = "select substr(w,1,16) from up where rd = ? and fn = ?"
-        w2 = d2.execute(q, (rd, fn)).fetchone()
+        if BY_PATH:
+            q = "select w from up where rd = ? and fn = ?"
+            w2 = d2.execute(q, (rd, fn)).fetchone()
+        else:
+            q = "select w from up where substr(w,1,16) = ? and +w = ?"
+            w2 = d2.execute(q, (w1s, w1)).fetchone()
+
         if w2:
             w2 = w2[0]
 
         v2 = None
         if w2:
             v2 = d2.execute(
-                "select v from mt where w = ? and +k = ?", (w2, k)
+                "select v from mt where w = ? and +k = ?", (w2[:16], k)
             ).fetchone()
             if v2:
                 v2 = v2[0]
@@ -124,7 +137,7 @@ def compare(n1, d1, n2, d2, verbose):
 
     for k, v in sorted(miss.items()):
         if v:
-            print(f"{n1} has {v:6} more {k:<6} tags than {n2}")
+            print(f"{n1} has {v:7} more {k:<7} tags than {n2}")
 
     print(f"in total, {nmiss} missing tags in {n2}\n")
 
@@ -132,47 +145,75 @@ def compare(n1, d1, n2, d2, verbose):
 def copy_mtp(d1, d2, tag, rm):
     nt = next(d1.execute("select count(w) from mt where k = ?", (tag,)))[0]
     n = 0
-    ndone = 0
-    for w1, k, v in d1.execute("select * from mt where k = ?", (tag,)):
+    ncopy = 0
+    nskip = 0
+    for w1s, k, v in d1.execute("select * from mt where k = ?", (tag,)):
         n += 1
         if n % 25_000 == 0:
-            m = f"\033[36m{n:,} of {nt:,} tags checked, so far {ndone} copied\033[0m"
+            m = f"\033[36m{n:,} of {nt:,} tags checked, so far {ncopy} copied, {nskip} skipped\033[0m"
             print(m)
 
-        q = "select rd, fn from up where substr(w,1,16) = ?"
-        rd, fn = d1.execute(q, (w1,)).fetchone()
+        q = "select w, rd, fn from up where substr(w,1,16) = ?"
+        w1, rd, fn = d1.execute(q, (w1s,)).fetchone()
         if rd.split("/", 1)[0] == ".hist":
             continue
 
-        q = "select substr(w,1,16) from up where rd = ? and fn = ?"
-        w2 = d2.execute(q, (rd, fn)).fetchone()
+        if BY_PATH:
+            q = "select w from up where rd = ? and fn = ?"
+            w2 = d2.execute(q, (rd, fn)).fetchone()
+        else:
+            q = "select w from up where substr(w,1,16) = ? and +w = ?"
+            w2 = d2.execute(q, (w1s, w1)).fetchone()
+
         if not w2:
             continue
 
-        w2 = w2[0]
-        hit = d2.execute("select v from mt where w = ? and +k = ?", (w2, k)).fetchone()
+        w2s = w2[0][:16]
+        hit = d2.execute("select v from mt where w = ? and +k = ?", (w2s, k)).fetchone()
         if hit:
             hit = hit[0]
 
         if hit != v:
-            ndone += 1
-            if hit is not None:
-                d2.execute("delete from mt where w = ? and +k = ?", (w2, k))
+            if NC and hit is not None:
+                nskip += 1
+                continue
 
-            d2.execute("insert into mt values (?,?,?)", (w2, k, v))
+            ncopy += 1
+            if hit is not None:
+                d2.execute("delete from mt where w = ? and +k = ?", (w2s, k))
+
+            d2.execute("insert into mt values (?,?,?)", (w2s, k, v))
             if rm:
-                d2.execute("delete from mt where w = ? and +k = 't:mtp'", (w2,))
+                d2.execute("delete from mt where w = ? and +k = 't:mtp'", (w2s,))
 
     d2.commit()
-    print(f"copied {ndone} {tag} tags over")
+    print(f"copied {ncopy} {tag} tags over, skipped {nskip}")
+
+
+def examples():
+    print(
+        """
+# clearing the journal
+./dbtool.py up2k.db
+
+# copy tags ".bpm" and "key" from old.db to up2k.db, and remove the mtp flag from matching files (so copyparty won't run any mtps on it)
+./dbtool.py -ls up2k.db
+./dbtool.py -src old.db up2k.db -cmp
+./dbtool.py -src old.v3 up2k.db -rm-mtp-flag -copy key
+./dbtool.py -src old.v3 up2k.db -rm-mtp-flag -copy .bpm -vac
+
+"""
+    )
 
 
 def main():
+    global NC, BY_PATH
     os.system("")
     print()
 
     ap = argparse.ArgumentParser()
     ap.add_argument("db", help="database to work on")
+    ap.add_argument("-h2", action="store_true", help="show examples")
     ap.add_argument("-src", metavar="DB", type=str, help="database to copy from")
 
     ap2 = ap.add_argument_group("informational / read-only stuff")
@@ -185,11 +226,29 @@ def main():
     ap2.add_argument(
         "-rm-mtp-flag",
         action="store_true",
-        help="when an mtp tag is copied over, also mark that as done, so copyparty won't run mtp on it",
+        help="when an mtp tag is copied over, also mark that file as done, so copyparty won't run any mtps on those files",
     )
     ap2.add_argument("-vac", action="store_true", help="optimize DB")
 
+    ap2 = ap.add_argument_group("behavior modifiers")
+    ap2.add_argument(
+        "-nc",
+        action="store_true",
+        help="no-clobber; don't replace/overwrite existing tags",
+    )
+    ap2.add_argument(
+        "-by-path",
+        action="store_true",
+        help="match files based on location rather than warks (content-hash), use this if the databases have different wark salts",
+    )
+
     ar = ap.parse_args()
+    if ar.h2:
+        examples()
+        return
+
+    NC = ar.nc
+    BY_PATH = ar.by_path
 
     for v in [ar.db, ar.src]:
         if v and not os.path.exists(v):

bin/mtag/README.md

@@ -42,7 +42,7 @@ run [`install-deps.sh`](install-deps.sh) to build/install most dependencies requ
 * `mtp` modules will not run if a file has existing tags in the db, so clear out the tags with `-e2tsr` the first time you launch with new `mtp` options
 
 
-## usage with volume-flags
+## usage with volflags
 
 instead of affecting all volumes, you can set the options for just one volume like so:
 

bin/mtag/cksum.py

@@ -17,7 +17,7 @@ except:
 
 """
 calculates various checksums for uploads,
-usage: -mtp crc32,md5,sha1,sha256b=bin/mtag/cksum.py
+usage: -mtp crc32,md5,sha1,sha256b=ad,bin/mtag/cksum.py
 """
 
 

bin/mtag/guestbook-read.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+
+"""
+fetch latest msg from guestbook and return as tag
+
+example copyparty config to use this:
+  --urlform save,get -vsrv/hello:hello:w:c,e2ts,mtp=guestbook=t10,ad,p,bin/mtag/guestbook-read.py:mte=+guestbook
+
+explained:
+  for realpath srv/hello (served at /hello), write-only for eveyrone,
+  enable file analysis on upload (e2ts),
+  use mtp plugin "bin/mtag/guestbook-read.py" to provide metadata tag "guestbook",
+  do this on all uploads regardless of extension,
+  t10 = 10 seconds timeout for each dwonload,
+  ad = parse file regardless if FFmpeg thinks it is audio or not
+  p = request upload info as json on stdin (need ip)
+  mte=+guestbook enabled indexing of that tag for this volume
+
+PS: this requires e2ts to be functional,
+  meaning you need to do at least one of these:
+   * apt install ffmpeg
+   * pip3 install mutagen
+"""
+
+
+import json
+import os
+import sqlite3
+import sys
+
+
+# set 0 to allow infinite msgs from one IP,
+# other values delete older messages to make space,
+# so 1 only keeps latest msg
+NUM_MSGS_TO_KEEP = 1
+
+
+def main():
+    fp = os.path.abspath(sys.argv[1])
+    fdir = os.path.dirname(fp)
+
+    zb = sys.stdin.buffer.read()
+    zs = zb.decode("utf-8", "replace")
+    md = json.loads(zs)
+
+    ip = md["up_ip"]
+
+    # can put the database inside `fdir` if you'd like,
+    # by default it saves to PWD:
+    # os.chdir(fdir)
+
+    db = sqlite3.connect("guestbook.db3")
+    with db:
+        t = "select msg from gb where ip = ? order by ts desc"
+        r = db.execute(t, (ip,)).fetchone()
+        if r:
+            print(r[0])
+
+
+if __name__ == "__main__":
+    main()

bin/mtag/guestbook.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python3
+
+"""
+store messages from users in an sqlite database
+which can be read from another mtp for example
+
+takes input from application/x-www-form-urlencoded POSTs,
+for example using the message/pager function on the website
+
+example copyparty config to use this:
+  --urlform save,get -vsrv/hello:hello:w:c,e2ts,mtp=xgb=ebin,t10,ad,p,bin/mtag/guestbook.py:mte=+xgb
+
+explained:
+  for realpath srv/hello (served at /hello),write-only for eveyrone,
+  enable file analysis on upload (e2ts),
+  use mtp plugin "bin/mtag/guestbook.py" to provide metadata tag "xgb",
+  do this on all uploads with the file extension "bin",
+  t300 = 300 seconds timeout for each dwonload,
+  ad = parse file regardless if FFmpeg thinks it is audio or not
+  p = request upload info as json on stdin
+  mte=+xgb enabled indexing of that tag for this volume
+
+PS: this requires e2ts to be functional,
+  meaning you need to do at least one of these:
+   * apt install ffmpeg
+   * pip3 install mutagen
+"""
+
+
+import json
+import os
+import sqlite3
+import sys
+from urllib.parse import unquote_to_bytes as unquote
+
+
+# set 0 to allow infinite msgs from one IP,
+# other values delete older messages to make space,
+# so 1 only keeps latest msg
+NUM_MSGS_TO_KEEP = 1
+
+
+def main():
+    fp = os.path.abspath(sys.argv[1])
+    fdir = os.path.dirname(fp)
+    fname = os.path.basename(fp)
+    if not fname.startswith("put-") or not fname.endswith(".bin"):
+        raise Exception("not a post file")
+
+    zb = sys.stdin.buffer.read()
+    zs = zb.decode("utf-8", "replace")
+    md = json.loads(zs)
+
+    buf = b""
+    with open(fp, "rb") as f:
+        while True:
+            b = f.read(4096)
+            buf += b
+            if len(buf) > 4096:
+                raise Exception("too big")
+
+            if not b:
+                break
+
+    if not buf:
+        raise Exception("file is empty")
+
+    buf = unquote(buf.replace(b"+", b" "))
+    txt = buf.decode("utf-8")
+
+    if not txt.startswith("msg="):
+        raise Exception("does not start with msg=")
+
+    ip = md["up_ip"]
+    ts = md["up_at"]
+    txt = txt[4:]
+
+    # can put the database inside `fdir` if you'd like,
+    # by default it saves to PWD:
+    # os.chdir(fdir)
+
+    db = sqlite3.connect("guestbook.db3")
+    try:
+        db.execute("select 1 from gb").fetchone()
+    except:
+        with db:
+            db.execute("create table gb (ip text, ts real, msg text)")
+            db.execute("create index gb_ip on gb(ip)")
+
+    with db:
+        if NUM_MSGS_TO_KEEP == 1:
+            t = "delete from gb where ip = ?"
+            db.execute(t, (ip,))
+
+        t = "insert into gb values (?,?,?)"
+        db.execute(t, (ip, ts, txt))
+
+        if NUM_MSGS_TO_KEEP > 1:
+            t = "select ts from gb where ip = ? order by ts desc"
+            hits = db.execute(t, (ip,)).fetchall()
+
+            if len(hits) > NUM_MSGS_TO_KEEP:
+                lim = hits[NUM_MSGS_TO_KEEP][0]
+                t = "delete from gb where ip = ? and ts <= ?"
+                db.execute(t, (ip, lim))
+
+    print(txt)
+
+
+if __name__ == "__main__":
+    main()

bin/mtag/image-noexif.py

@@ -43,7 +43,6 @@ PS: this requires e2ts to be functional,
 
 import os
 import sys
-import time
 import filecmp
 import subprocess as sp
 
@@ -90,4 +89,7 @@ def main():
 
 
 if __name__ == "__main__":
-    main()
+    try:
+        main()
+    except:
+        pass

bin/mtag/mousepad.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+import subprocess as sp
+
+
+"""
+mtp test -- opens a texteditor
+
+usage:
+  -vsrv/v1:v1:r:c,mte=+x1:c,mtp=x1=ad,p,bin/mtag/mousepad.py
+
+explained:
+  c,mte: list of tags to index in this volume
+  c,mtp: add new tag provider
+     x1: dummy tag to provide
+     ad: dontcare if audio or not
+      p: priority 1 (run after initial tag-scan with ffprobe or mutagen)
+"""
+
+
+def main():
+    env = os.environ.copy()
+    env["DISPLAY"] = ":0.0"
+
+    if False:
+        # open the uploaded file
+        fp = sys.argv[-1]
+    else:
+        # display stdin contents (`oth_tags`)
+        fp = "/dev/stdin"
+
+    p = sp.Popen(["/usr/bin/mousepad", fp])
+    p.communicate()
+
+
+main()

bin/mtag/rclone-upload.py

@@ -0,0 +1,76 @@
+#!/usr/bin/env python
+
+import json
+import os
+import subprocess as sp
+import sys
+import time
+
+try:
+    from copyparty.util import fsenc
+except:
+
+    def fsenc(p):
+        return p.encode("utf-8")
+
+
+_ = r"""
+first checks the tag "vidchk" which must be "ok" to continue,
+then uploads all files to some cloud storage (RCLONE_REMOTE)
+and DELETES THE ORIGINAL FILES if rclone returns 0 ("success")
+
+deps:
+  rclone
+
+usage:
+  -mtp x2=t43200,ay,p2,bin/mtag/rclone-upload.py
+
+explained:
+t43200: timeout 12h
+    ay: only process files which contain audio (including video with audio)
+    p2: set priority 2 (after vidchk's suggested priority of 1),
+          so the output of vidchk will be passed in here
+
+complete usage example as vflags along with vidchk:
+  -vsrv/vidchk:vidchk:r:rw,ed:c,e2dsa,e2ts,mtp=vidchk=t600,p,bin/mtag/vidchk.py:c,mtp=rupload=t43200,ay,p2,bin/mtag/rclone-upload.py:c,mte=+vidchk,rupload
+
+setup: see https://rclone.org/drive/
+
+if you wanna use this script standalone / separately from copyparty,
+either set CONDITIONAL_UPLOAD False or provide the following stdin:
+  {"vidchk":"ok"}
+"""
+
+
+RCLONE_REMOTE = "notmybox"
+CONDITIONAL_UPLOAD = True
+
+
+def main():
+    fp = sys.argv[1]
+    if CONDITIONAL_UPLOAD:
+        zb = sys.stdin.buffer.read()
+        zs = zb.decode("utf-8", "replace")
+        md = json.loads(zs)
+
+        chk = md.get("vidchk", None)
+        if chk != "ok":
+            print(f"vidchk={chk}", file=sys.stderr)
+            sys.exit(1)
+
+    dst = f"{RCLONE_REMOTE}:".encode("utf-8")
+    cmd = [b"rclone", b"copy", b"--", fsenc(fp), dst]
+
+    t0 = time.time()
+    try:
+        sp.check_call(cmd)
+    except:
+        print("rclone failed", file=sys.stderr)
+        sys.exit(1)
+
+    print(f"{time.time() - t0:.1f} sec")
+    os.unlink(fsenc(fp))
+
+
+if __name__ == "__main__":
+    main()

bin/mtag/very-bad-idea.py

@@ -16,7 +16,7 @@ goes without saying, but this is HELLA DANGEROUS,
   GIVES RCE TO ANYONE WHO HAVE UPLOAD PERMISSIONS
 
 example copyparty config to use this:
-  --urlform save,get -v.::w:c,e2d,e2t,mte=+a1:c,mtp=a1=ad,bin/mtag/very-bad-idea.py
+  --urlform save,get -v.::w:c,e2d,e2t,mte=+a1:c,mtp=a1=ad,kn,c0,bin/mtag/very-bad-idea.py
 
 recommended deps:
   apt install xdotool libnotify-bin
@@ -63,8 +63,8 @@ set -e
 EOF
 chmod 755 /usr/local/bin/chromium-browser
 
-# start the server  (note: replace `-v.::rw:` with `-v.::r:` to disallow retrieving uploaded stuff)
-cd ~/Downloads; python3 copyparty-sfx.py --urlform save,get -v.::rw:c,e2d,e2t,mte=+a1:c,mtp=a1=ad,very-bad-idea.py
+# start the server  (note: replace `-v.::rw:` with `-v.::w:` to disallow retrieving uploaded stuff)
+cd ~/Downloads; python3 copyparty-sfx.py --urlform save,get -v.::rw:c,e2d,e2t,mte=+a1:c,mtp=a1=ad,kn,very-bad-idea.py
 
 """
 

bin/mtag/vidchk.py

@@ -0,0 +1,131 @@
+#!/usr/bin/env python3
+
+import json
+import re
+import os
+import sys
+import subprocess as sp
+
+try:
+    from copyparty.util import fsenc
+except:
+
+    def fsenc(p):
+        return p.encode("utf-8")
+
+
+_ = r"""
+inspects video files for errors and such
+plus stores a bunch of metadata to filename.ff.json
+
+usage:
+  -mtp vidchk=t600,ay,p,bin/mtag/vidchk.py
+
+explained:
+t600: timeout 10min
+  ay: only process files which contain audio (including video with audio)
+   p: set priority 1 (lowest priority after initial ffprobe/mutagen for base tags),
+       makes copyparty feed base tags into this script as json
+
+if you wanna use this script standalone / separately from copyparty,
+provide the video resolution on stdin as json:  {"res":"1920x1080"}
+"""
+
+
+FAST = True  # parse entire file at container level
+# FAST = False  # fully decode audio and video streams
+
+
+# warnings to ignore
+harmless = re.compile(
+    r"Unsupported codec with id |Could not find codec parameters.*Attachment:|analyzeduration"
+    + r"|timescale not set"
+)
+
+
+def wfilter(lines):
+    return [x for x in lines if x.strip() and not harmless.search(x)]
+
+
+def errchk(so, se, rc, dbg):
+    if dbg:
+        with open(dbg, "wb") as f:
+            f.write(b"so:\n" + so + b"\nse:\n" + se + b"\n")
+
+    if rc:
+        err = (so + se).decode("utf-8", "replace").split("\n", 1)
+        err = wfilter(err) or err
+        return f"ERROR {rc}: {err[0]}"
+
+    if se:
+        err = se.decode("utf-8", "replace").split("\n", 1)
+        err = wfilter(err)
+        if err:
+            return f"Warning: {err[0]}"
+
+    return None
+
+
+def main():
+    fp = sys.argv[1]
+    zb = sys.stdin.buffer.read()
+    zs = zb.decode("utf-8", "replace")
+    md = json.loads(zs)
+
+    fdir = os.path.dirname(os.path.realpath(fp))
+    flag = os.path.join(fdir, ".processed")
+    if os.path.exists(flag):
+        return "already processed"
+
+    try:
+        w, h = [int(x) for x in md["res"].split("x")]
+        if not w + h:
+            raise Exception()
+    except:
+        return "could not determine resolution"
+
+    # grab streams/format metadata + 2 seconds of frames at the start and end
+    zs = "ffprobe -hide_banner -v warning -of json -show_streams -show_format -show_packets -show_data_hash crc32 -read_intervals %+2,999999%+2"
+    cmd = zs.encode("ascii").split(b" ") + [fsenc(fp)]
+    p = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE)
+    so, se = p.communicate()
+
+    # spaces to tabs, drops filesize from 69k to 48k
+    so = b"\n".join(
+        [
+            b"\t" * int((len(x) - len(x.lstrip())) / 4) + x.lstrip()
+            for x in (so or b"").split(b"\n")
+        ]
+    )
+    with open(fsenc(f"{fp}.ff.json"), "wb") as f:
+        f.write(so)
+
+    err = errchk(so, se, p.returncode, f"{fp}.vidchk")
+    if err:
+        return err
+
+    if max(w, h) < 1280 and min(w, h) < 720:
+        return "resolution too small"
+
+    zs = (
+        "ffmpeg -y -hide_banner -nostdin -v warning"
+        + " -err_detect +crccheck+bitstream+buffer+careful+compliant+aggressive+explode"
+        + " -xerror -i"
+    )
+
+    cmd = zs.encode("ascii").split(b" ") + [fsenc(fp)]
+
+    if FAST:
+        zs = "-c copy -f null -"
+    else:
+        zs = "-vcodec rawvideo -acodec pcm_s16le -f null -"
+
+    cmd += zs.encode("ascii").split(b" ")
+
+    p = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE)
+    so, se = p.communicate()
+    return errchk(so, se, p.returncode, f"{fp}.vidchk")
+
+
+if __name__ == "__main__":
+    print(main() or "ok")

bin/prisonparty.sh

@@ -11,13 +11,13 @@ sysdirs=( /bin /lib /lib32 /lib64 /sbin /usr )
 help() { cat <<'EOF'
 
 usage:
-  ./prisonparty.sh <ROOTDIR> <UID> <GID> [VOLDIR [VOLDIR...]] -- python3 copyparty-sfx.py [...]"
+  ./prisonparty.sh <ROOTDIR> <UID> <GID> [VOLDIR [VOLDIR...]] -- python3 copyparty-sfx.py [...]
 
 example:
-  ./prisonparty.sh /var/lib/copyparty-jail 1000 1000 /mnt/nas/music -- python3 copyparty-sfx.py -v /mnt/nas/music::rwmd"
+  ./prisonparty.sh /var/lib/copyparty-jail 1000 1000 /mnt/nas/music -- python3 copyparty-sfx.py -v /mnt/nas/music::rwmd
 
 example for running straight from source (instead of using an sfx):
-  PYTHONPATH=$PWD ./prisonparty.sh /var/lib/copyparty-jail 1000 1000 /mnt/nas/music -- python3 -um copyparty -v /mnt/nas/music::rwmd"
+  PYTHONPATH=$PWD ./prisonparty.sh /var/lib/copyparty-jail 1000 1000 /mnt/nas/music -- python3 -um copyparty -v /mnt/nas/music::rwmd
 
 note that if you have python modules installed as --user (such as bpm/key detectors),
   you should add /home/foo/.local as a VOLDIR

bin/unforget.py

@@ -0,0 +1,99 @@
+#!/usr/bin/env python3
+
+"""
+unforget.py: rebuild db from logfiles
+2022-09-07, v0.1, ed <irc.rizon.net>, MIT-Licensed
+https://github.com/9001/copyparty/blob/hovudstraum/bin/unforget.py
+
+only makes sense if running copyparty with --no-forget
+(e.g. immediately shifting uploads to other storage)
+
+usage:
+  xz -d < log | ./unforget.py .hist/up2k.db
+
+"""
+
+import re
+import sys
+import json
+import base64
+import sqlite3
+import argparse
+
+
+FS_ENCODING = sys.getfilesystemencoding()
+
+
+class APF(argparse.ArgumentDefaultsHelpFormatter, argparse.RawDescriptionHelpFormatter):
+    pass
+
+
+mem_cur = sqlite3.connect(":memory:").cursor()
+mem_cur.execute(r"create table a (b text)")
+
+
+def s3enc(rd: str, fn: str) -> tuple[str, str]:
+    ret: list[str] = []
+    for v in [rd, fn]:
+        try:
+            mem_cur.execute("select * from a where b = ?", (v,))
+            ret.append(v)
+        except:
+            wtf8 = v.encode(FS_ENCODING, "surrogateescape")
+            ret.append("//" + base64.urlsafe_b64encode(wtf8).decode("ascii"))
+
+    return ret[0], ret[1]
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("db")
+    ar = ap.parse_args()
+
+    db = sqlite3.connect(ar.db).cursor()
+    ptn_times = re.compile(r"no more chunks, setting times \(([0-9]+)")
+    at = 0
+    ctr = 0
+
+    for ln in [x.decode("utf-8", "replace").rstrip() for x in sys.stdin.buffer]:
+        if "no more chunks, setting times (" in ln:
+            m = ptn_times.search(ln)
+            if m:
+                at = int(m.group(1))
+
+        if '"hash": []' in ln:
+            try:
+                ofs = ln.find("{")
+                j = json.loads(ln[ofs:])
+            except:
+                pass
+
+            w = j["wark"]
+            if db.execute("select w from up where w = ?", (w,)).fetchone():
+                continue
+
+            # PYTHONPATH=/home/ed/dev/copyparty/ python3 -m copyparty -e2dsa  -v foo:foo:rwmd,ed -aed:wark --no-forget
+            # 05:34:43.845 127.0.0.1 42496       no more chunks, setting times (1662528883, 1658001882)
+            # 05:34:43.863 127.0.0.1 42496       {"name": "f\"2", "purl": "/foo/bar/baz/", "size": 1674, "lmod": 1658001882, "sprs": true, "hash": [], "wark": "LKIWpp2jEAh9dH3fu-DobuURFGEKlODXDGTpZ1otMhUg"}
+            # |                      w                       |     mt     |  sz  |   rd    | fn  |    ip     |     at     |
+            # | LKIWpp2jEAh9dH3fu-DobuURFGEKlODXDGTpZ1otMhUg | 1658001882 | 1674 | bar/baz | f"2 | 127.0.0.1 | 1662528883 |
+
+            rd, fn = s3enc(j["purl"].strip("/"), j["name"])
+            ip = ln.split(" ")[1].split("m")[-1]
+
+            q = "insert into up values (?,?,?,?,?,?,?)"
+            v = (w, int(j["lmod"]), int(j["size"]), rd, fn, ip, at)
+            db.execute(q, v)
+            ctr += 1
+            if ctr % 1024 == 1023:
+                print(f"{ctr} commit...")
+                db.connection.commit()
+
+    if ctr:
+        db.connection.commit()
+
+    print(f"unforgot {ctr} files")
+
+
+if __name__ == "__main__":
+    main()

bin/up2k.py

@@ -3,11 +3,11 @@ from __future__ import print_function, unicode_literals
 
 """
 up2k.py: upload to copyparty
-2021-11-28, v0.13, ed <irc.rizon.net>, MIT-Licensed
+2022-09-05, v0.19, ed <irc.rizon.net>, MIT-Licensed
 https://github.com/9001/copyparty/blob/hovudstraum/bin/up2k.py
 
 - dependencies: requests
-- supports python 2.6, 2.7, and 3.3 through 3.10
+- supports python 2.6, 2.7, and 3.3 through 3.11
 
 - almost zero error-handling
 - but if something breaks just try again and it'll autoresume
@@ -22,15 +22,33 @@ import atexit
 import signal
 import base64
 import hashlib
-import argparse
 import platform
 import threading
-import requests
 import datetime
 
+try:
+    import argparse
+except:
+    m = "\n  ERROR: need 'argparse'; download it here:\n   https://github.com/ThomasWaldmann/argparse/raw/master/argparse.py\n"
+    print(m)
+    raise
+
+try:
+    import requests
+except ImportError:
+    if sys.version_info > (2, 7):
+        m = "\nERROR: need 'requests'; please run this command:\n {0} -m pip install --user requests\n"
+    else:
+        m = "requests/2.18.4 urllib3/1.23 chardet/3.0.4 certifi/2020.4.5.1 idna/2.7"
+        m = ["   https://pypi.org/project/" + x + "/#files" for x in m.split()]
+        m = "\n  ERROR: need these:\n" + "\n".join(m) + "\n"
+
+    print(m.format(sys.executable))
+    sys.exit(1)
+
 
 # from copyparty/__init__.py
-PY2 = sys.version_info[0] == 2
+PY2 = sys.version_info < (3,)
 if PY2:
     from Queue import Queue
     from urllib import unquote
@@ -76,15 +94,15 @@ class File(object):
         self.up_b = 0  # type: int
         self.up_c = 0  # type: int
 
-        # m = "size({}) lmod({}) top({}) rel({}) abs({}) name({})\n"
-        # eprint(m.format(self.size, self.lmod, self.top, self.rel, self.abs, self.name))
+        # t = "size({}) lmod({}) top({}) rel({}) abs({}) name({})\n"
+        # eprint(t.format(self.size, self.lmod, self.top, self.rel, self.abs, self.name))
 
 
 class FileSlice(object):
     """file-like object providing a fixed window into a file"""
 
     def __init__(self, file, cid):
-        # type: (File, str) -> FileSlice
+        # type: (File, str) -> None
 
         self.car, self.len = file.kchunks[cid]
         self.cdr = self.car + self.len
@@ -125,6 +143,89 @@ class FileSlice(object):
         return ret
 
 
+class MTHash(object):
+    def __init__(self, cores):
+        self.f = None
+        self.sz = 0
+        self.csz = 0
+        self.omutex = threading.Lock()
+        self.imutex = threading.Lock()
+        self.work_q = Queue()
+        self.done_q = Queue()
+        self.thrs = []
+        for _ in range(cores):
+            t = threading.Thread(target=self.worker)
+            t.daemon = True
+            t.start()
+            self.thrs.append(t)
+
+    def hash(self, f, fsz, chunksz, pcb=None, pcb_opaque=None):
+        with self.omutex:
+            self.f = f
+            self.sz = fsz
+            self.csz = chunksz
+
+            chunks = {}
+            nchunks = int(math.ceil(fsz / chunksz))
+            for nch in range(nchunks):
+                self.work_q.put(nch)
+
+            ex = ""
+            for nch in range(nchunks):
+                qe = self.done_q.get()
+                try:
+                    nch, dig, ofs, csz = qe
+                    chunks[nch] = [dig, ofs, csz]
+                except:
+                    ex = ex or qe
+
+                if pcb:
+                    pcb(pcb_opaque, chunksz * nch)
+
+            if ex:
+                raise Exception(ex)
+
+            ret = []
+            for n in range(nchunks):
+                ret.append(chunks[n])
+
+            self.f = None
+            self.csz = 0
+            self.sz = 0
+            return ret
+
+    def worker(self):
+        while True:
+            ofs = self.work_q.get()
+            try:
+                v = self.hash_at(ofs)
+            except Exception as ex:
+                v = str(ex)
+
+            self.done_q.put(v)
+
+    def hash_at(self, nch):
+        f = self.f
+        ofs = ofs0 = nch * self.csz
+        hashobj = hashlib.sha512()
+        chunk_sz = chunk_rem = min(self.csz, self.sz - ofs)
+        while chunk_rem > 0:
+            with self.imutex:
+                f.seek(ofs)
+                buf = f.read(min(chunk_rem, 1024 * 1024 * 12))
+
+            if not buf:
+                raise Exception("EOF at " + str(ofs))
+
+            hashobj.update(buf)
+            chunk_rem -= len(buf)
+            ofs += len(buf)
+
+        digest = hashobj.digest()[:33]
+        digest = base64.urlsafe_b64encode(digest).decode("utf-8")
+        return nch, digest, ofs0, chunk_sz
+
+
 _print = print
 
 
@@ -150,13 +251,11 @@ if not VT100:
 
 
 def termsize():
-    import os
-
     env = os.environ
 
     def ioctl_GWINSZ(fd):
         try:
-            import fcntl, termios, struct, os
+            import fcntl, termios, struct
 
             cr = struct.unpack("hh", fcntl.ioctl(fd, termios.TIOCGWINSZ, "1234"))
         except:
@@ -217,8 +316,8 @@ class CTermsize(object):
             eprint("\033[s\033[r\033[u")
         else:
             self.g = 1 + self.h - margin
-            m = "{0}\033[{1}A".format("\n" * margin, margin)
-            eprint("{0}\033[s\033[1;{1}r\033[u".format(m, self.g - 1))
+            t = "{0}\033[{1}A".format("\n" * margin, margin)
+            eprint("{0}\033[s\033[1;{1}r\033[u".format(t, self.g - 1))
 
 
 ss = CTermsize()
@@ -231,8 +330,8 @@ def _scd(err, top):
             abspath = os.path.join(top, fh.name)
             try:
                 yield [abspath, fh.stat()]
-            except:
-                err.append(abspath)
+            except Exception as ex:
+                err.append((abspath, str(ex)))
 
 
 def _lsd(err, top):
@@ -241,25 +340,31 @@ def _lsd(err, top):
         abspath = os.path.join(top, name)
         try:
             yield [abspath, os.stat(abspath)]
-        except:
-            err.append(abspath)
+        except Exception as ex:
+            err.append((abspath, str(ex)))
 
 
-if hasattr(os, "scandir"):
+if hasattr(os, "scandir") and sys.version_info > (3, 6):
     statdir = _scd
 else:
     statdir = _lsd
 
 
-def walkdir(err, top):
+def walkdir(err, top, seen):
     """recursive statdir"""
+    atop = os.path.abspath(os.path.realpath(top))
+    if atop in seen:
+        err.append((top, "recursive-symlink"))
+        return
+
+    seen = seen[:] + [atop]
     for ap, inf in sorted(statdir(err, top)):
         if stat.S_ISDIR(inf.st_mode):
             try:
-                for x in walkdir(err, ap):
+                for x in walkdir(err, ap, seen):
                     yield x
-            except:
-                err.append(ap)
+            except Exception as ex:
+                err.append((ap, str(ex)))
         else:
             yield ap, inf
 
@@ -274,7 +379,7 @@ def walkdirs(err, tops):
             stop = os.path.dirname(top)
 
         if os.path.isdir(top):
-            for ap, inf in walkdir(err, top):
+            for ap, inf in walkdir(err, top, []):
                 yield stop, ap[len(stop) :].lstrip(sep), inf
         else:
             d, n = top.rsplit(sep, 1)
@@ -323,8 +428,8 @@ def up2k_chunksize(filesize):
 
 
 # mostly from copyparty/up2k.py
-def get_hashlist(file, pcb):
-    # type: (File, any) -> None
+def get_hashlist(file, pcb, mth):
+    # type: (File, any, any) -> None
     """generates the up2k hashlist from file contents, inserts it into `file`"""
 
     chunk_sz = up2k_chunksize(file.size)
@@ -332,7 +437,12 @@ def get_hashlist(file, pcb):
     file_ofs = 0
     ret = []
     with open(file.abs, "rb", 512 * 1024) as f:
+        if mth and file.size >= 1024 * 512:
+            ret = mth.hash(f, file.size, chunk_sz, pcb, file)
+            file_rem = 0
+
         while file_rem > 0:
+            # same as `hash_at` except for `imutex` / bufsz
             hashobj = hashlib.sha512()
             chunk_sz = chunk_rem = min(chunk_sz, file_rem)
             while chunk_rem > 0:
@@ -360,7 +470,7 @@ def get_hashlist(file, pcb):
 
 
 def handshake(req_ses, url, file, pw, search):
-    # type: (requests.Session, str, File, any, bool) -> List[str]
+    # type: (requests.Session, str, File, any, bool) -> list[str]
     """
     performs a handshake with the server; reply is:
       if search, a list of search results
@@ -389,8 +499,9 @@ def handshake(req_ses, url, file, pw, search):
         try:
             r = req_ses.post(url, headers=headers, json=req)
             break
-        except:
-            eprint("handshake failed, retrying: {0}\n".format(file.name))
+        except Exception as ex:
+            em = str(ex).split("SSLError(")[-1]
+            eprint("handshake failed, retrying: {0}\n  {1}\n\n".format(file.name, em))
             time.sleep(1)
 
     try:
@@ -399,7 +510,7 @@ def handshake(req_ses, url, file, pw, search):
         raise Exception(r.text)
 
     if search:
-        return r["hits"]
+        return r["hits"], False
 
     try:
         pre, url = url.split("://")
@@ -411,7 +522,7 @@ def handshake(req_ses, url, file, pw, search):
     file.name = r["name"]
     file.wark = r["wark"]
 
-    return r["hash"]
+    return r["hash"], r["sprs"]
 
 
 def upload(req_ses, file, cid, pw):
@@ -471,12 +582,19 @@ class Ctl(object):
 
         if err:
             eprint("\n# failed to access {0} paths:\n".format(len(err)))
-            for x in err:
-                eprint(x.decode("utf-8", "replace") + "\n")
+            for ap, msg in err:
+                if ar.v:
+                    eprint("{0}\n `-{1}\n\n".format(ap.decode("utf-8", "replace"), msg))
+                else:
+                    eprint(ap.decode("utf-8", "replace") + "\n")
 
             eprint("^ failed to access those {0} paths ^\n\n".format(len(err)))
+
+            if not ar.v:
+                eprint("hint: set -v for detailed error messages\n")
+
             if not ar.ok:
-                eprint("aborting because --ok is not set\n")
+                eprint("hint: aborting because --ok is not set\n")
                 return
 
         eprint("found {0} files, {1}\n\n".format(nfiles, humansize(nbytes)))
@@ -491,11 +609,37 @@ class Ctl(object):
 
         self.filegen = walkdirs([], ar.files)
         if ar.safe:
-            self.safe()
+            self._safe()
         else:
-            self.fancy()
+            self.hash_f = 0
+            self.hash_c = 0
+            self.hash_b = 0
+            self.up_f = 0
+            self.up_c = 0
+            self.up_b = 0
+            self.up_br = 0
+            self.hasher_busy = 1
+            self.handshaker_busy = 0
+            self.uploader_busy = 0
+            self.serialized = False
 
-    def safe(self):
+            self.t0 = time.time()
+            self.t0_up = None
+            self.spd = None
+
+            self.mutex = threading.Lock()
+            self.q_handshake = Queue()  # type: Queue[File]
+            self.q_recheck = Queue()  # type: Queue[File]  # partial upload exists [...]
+            self.q_upload = Queue()  # type: Queue[tuple[File, str]]
+
+            self.st_hash = [None, "(idle, starting...)"]  # type: tuple[File, int]
+            self.st_up = [None, "(idle, starting...)"]  # type: tuple[File, int]
+
+            self.mth = MTHash(ar.J) if ar.J > 1 else None
+
+            self._fancy()
+
+    def _safe(self):
         """minimal basic slow boring fallback codepath"""
         search = self.ar.s
         for nf, (top, rel, inf) in enumerate(self.filegen):
@@ -503,12 +647,12 @@ class Ctl(object):
             upath = file.abs.decode("utf-8", "replace")
 
             print("{0} {1}\n  hash...".format(self.nfiles - nf, upath))
-            get_hashlist(file, None)
+            get_hashlist(file, None, None)
 
             burl = self.ar.url[:12] + self.ar.url[8:].split("/")[0] + "/"
             while True:
                 print("  hs...")
-                hs = handshake(req_ses, self.ar.url, file, self.ar.a, search)
+                hs, _ = handshake(req_ses, self.ar.url, file, self.ar.a, search)
                 if search:
                     if hs:
                         for hit in hs:
@@ -529,29 +673,7 @@ class Ctl(object):
 
             print("  ok!")
 
-    def fancy(self):
-        self.hash_f = 0
-        self.hash_c = 0
-        self.hash_b = 0
-        self.up_f = 0
-        self.up_c = 0
-        self.up_b = 0
-        self.up_br = 0
-        self.hasher_busy = 1
-        self.handshaker_busy = 0
-        self.uploader_busy = 0
-
-        self.t0 = time.time()
-        self.t0_up = None
-        self.spd = None
-
-        self.mutex = threading.Lock()
-        self.q_handshake = Queue()  # type: Queue[File]
-        self.q_recheck = Queue()  # type: Queue[File]  # partial upload exists [...]
-        self.q_upload = Queue()  # type: Queue[tuple[File, str]]
-
-        self.st_hash = [None, "(idle, starting...)"]  # type: tuple[File, int]
-        self.st_up = [None, "(idle, starting...)"]  # type: tuple[File, int]
+    def _fancy(self):
         if VT100:
             atexit.register(self.cleanup_vt100)
             ss.scroll_region(3)
@@ -597,8 +719,8 @@ class Ctl(object):
                         if "/" in name:
                             name = "\033[36m{0}\033[0m/{1}".format(*name.rsplit("/", 1))
 
-                        m = "{0:6.1f}% {1} {2}\033[K"
-                        txt += m.format(p, self.nfiles - f, name)
+                        t = "{0:6.1f}% {1} {2}\033[K"
+                        txt += t.format(p, self.nfiles - f, name)
 
                 txt += "\033[{0}H ".format(ss.g + 2)
             else:
@@ -614,11 +736,12 @@ class Ctl(object):
 
             spd = humansize(spd)
             eta = str(datetime.timedelta(seconds=int(eta)))
-            left = humansize(self.nbytes - self.up_b)
+            sleft = humansize(self.nbytes - self.up_b)
+            nleft = self.nfiles - self.up_f
             tail = "\033[K\033[u" if VT100 else "\r"
 
-            m = "eta: {0} @ {1}/s, {2} left".format(eta, spd, left)
-            eprint(txt + "\033]0;{0}\033\\\r{1}{2}".format(m, m, tail))
+            t = "{0} eta @ {1}/s, {2}, {3}# left".format(eta, spd, sleft, nleft)
+            eprint(txt + "\033]0;{0}\033\\\r{0}{1}".format(t, tail))
 
     def cleanup_vt100(self):
         ss.scroll_region(None)
@@ -677,7 +800,7 @@ class Ctl(object):
 
                 time.sleep(0.05)
 
-            get_hashlist(file, self.cb_hasher)
+            get_hashlist(file, self.cb_hasher, self.mth)
             with self.mutex:
                 self.hash_f += 1
                 self.hash_c += len(file.cids)
@@ -709,7 +832,7 @@ class Ctl(object):
             upath = file.abs.decode("utf-8", "replace")
 
             try:
-                hs = handshake(req_ses, self.ar.url, file, self.ar.a, search)
+                hs, sprs = handshake(req_ses, self.ar.url, file, self.ar.a, search)
             except Exception as ex:
                 if q == self.q_handshake and "<pre>partial upload exists" in str(ex):
                     self.q_recheck.put(file)
@@ -720,8 +843,8 @@ class Ctl(object):
             if search:
                 if hs:
                     for hit in hs:
-                        m = "found: {0}\n  {1}{2}\n"
-                        print(m.format(upath, burl, hit["rp"]), end="")
+                        t = "found: {0}\n  {1}{2}\n"
+                        print(t.format(upath, burl, hit["rp"]), end="")
                 else:
                     print("NOT found: {0}\n".format(upath), end="")
 
@@ -734,6 +857,12 @@ class Ctl(object):
                 continue
 
             with self.mutex:
+                if not sprs and not self.serialized:
+                    t = "server filesystem does not support sparse files; serializing uploads\n"
+                    eprint(t)
+                    self.serialized = True
+                    for _ in range(self.ar.j - 1):
+                        self.q_upload.put(None)
                 if not hs:
                     # all chunks done
                     self.up_f += 1
@@ -800,6 +929,9 @@ def main():
     if not VT100:
         os.system("rem")  # enables colors
 
+    cores = (os.cpu_count() if hasattr(os, "cpu_count") else 0) or 2
+    hcores = min(cores, 3)  # 4% faster than 4+ on py3.9 @ r5-4500U
+
     # fmt: off
     ap = app = argparse.ArgumentParser(formatter_class=APF, epilog="""
 NOTE:
@@ -810,11 +942,13 @@ source file/folder selection uses rsync syntax, meaning that:
 
     ap.add_argument("url", type=unicode, help="server url, including destination folder")
     ap.add_argument("files", type=unicode, nargs="+", help="files and/or folders to process")
+    ap.add_argument("-v", action="store_true", help="verbose")
     ap.add_argument("-a", metavar="PASSWORD", help="password")
     ap.add_argument("-s", action="store_true", help="file-search (disables upload)")
     ap.add_argument("--ok", action="store_true", help="continue even if some local files are inaccessible")
     ap = app.add_argument_group("performance tweaks")
     ap.add_argument("-j", type=int, metavar="THREADS", default=4, help="parallel connections")
+    ap.add_argument("-J", type=int, metavar="THREADS", default=hcores, help="num cpu-cores to use for hashing; set 0 or 1 for single-core hashing")
     ap.add_argument("-nh", action="store_true", help="disable hashing while uploading")
     ap.add_argument("--safe", action="store_true", help="use simple fallback approach")
     ap.add_argument("-z", action="store_true", help="ZOOMIN' (skip uploading files if they exist at the destination with the ~same last-modified timestamp, so same as yolo / turbo with date-chk but even faster)")

contrib/README.md

@@ -22,6 +22,9 @@ however if your copyparty is behind a reverse-proxy, you may want to use [`share
 * `URL`: full URL to the root folder (with trailing slash) followed by `$regex:1|1$`
 * `pw`: password (remove `Parameters` if anon-write)
 
+### [`media-osd-bgone.ps1`](media-osd-bgone.ps1)
+* disables the [windows OSD popup](https://user-images.githubusercontent.com/241032/122821375-0e08df80-d2dd-11eb-9fd9-184e8aacf1d0.png) (the thing on the left) which appears every time you hit media hotkeys to adjust volume or change song while playing music with the copyparty web-ui, or most other audio players really
+
 ### [`explorer-nothumbs-nofoldertypes.reg`](explorer-nothumbs-nofoldertypes.reg)
 * disables thumbnails and folder-type detection in windows explorer
 * makes it way faster (especially for slow/networked locations (such as copyparty-fuse))
@@ -29,6 +32,7 @@ however if your copyparty is behind a reverse-proxy, you may want to use [`share
 ### [`cfssl.sh`](cfssl.sh)
 * creates CA and server certificates using cfssl
 * give a 3rd argument to install it to your copyparty config
+* systemd service at [`systemd/cfssl.service`](systemd/cfssl.service)
 
 # OS integration
 init-scripts to start copyparty as a service

contrib/cfssl.sh

@@ -7,7 +7,7 @@ srv_fqdn="$2"
 
 [ -z "$srv_fqdn" ] && {
 	echo "need arg 1: ca name"
-	echo "need arg 2: server fqdn"
+	echo "need arg 2: server fqdn and/or IPs, comma-separated"
 	echo "optional arg 3: if set, write cert into copyparty cfg"
 	exit 1
 }

contrib/media-osd-bgone.ps1

@@ -0,0 +1,104 @@
+# media-osd-bgone.ps1: disable media-control OSD on win10do
+# v1.1, 2021-06-25, ed <irc.rizon.net>, MIT-licensed
+# https://github.com/9001/copyparty/blob/hovudstraum/contrib/media-osd-bgone.ps1
+#
+# locates the first window that looks like the media OSD and minimizes it;
+# doing this once after each reboot should do the trick
+# (adjust the width/height filter if it doesn't work)
+#
+# ---------------------------------------------------------------------
+#
+# tip: save the following as "media-osd-bgone.bat" next to this script:
+#   start cmd /c "powershell -command ""set-executionpolicy -scope process bypass; .\media-osd-bgone.ps1"" & ping -n 2 127.1 >nul"
+#
+# then create a shortcut to that bat-file and move the shortcut here:
+#   %appdata%\Microsoft\Windows\Start Menu\Programs\Startup
+#
+# and now this will autorun on bootup
+
+
+Add-Type -TypeDefinition @"
+using System;
+using System.IO;
+using System.Threading;
+using System.Diagnostics;
+using System.Runtime.InteropServices;
+using System.Windows.Forms;
+
+namespace A {
+  public class B : Control {
+
+    [DllImport("user32.dll")]
+    static extern void keybd_event(byte bVk, byte bScan, uint dwFlags, int dwExtraInfo);
+
+    [DllImport("user32.dll", SetLastError = true)]
+    static extern IntPtr FindWindowEx(IntPtr hwndParent, IntPtr hwndChildAfter, string lpszClass, string lpszWindow);
+
+    [DllImport("user32.dll", SetLastError=true)]
+    static extern bool GetWindowRect(IntPtr hwnd, out RECT lpRect);
+
+    [DllImport("user32.dll")]
+    static extern bool ShowWindow(IntPtr hWnd, int nCmdShow);
+
+    [StructLayout(LayoutKind.Sequential)]
+    public struct RECT {
+      public int x;
+      public int y;
+      public int x2;
+      public int y2;
+    }
+    
+    bool fa() {
+      RECT r;
+      IntPtr it = IntPtr.Zero;
+      while ((it = FindWindowEx(IntPtr.Zero, it, "NativeHWNDHost", "")) != IntPtr.Zero) {
+        if (FindWindowEx(it, IntPtr.Zero, "DirectUIHWND", "") == IntPtr.Zero)
+          continue;
+        
+        if (!GetWindowRect(it, out r))
+          continue;
+
+        int w = r.x2 - r.x + 1;
+        int h = r.y2 - r.y + 1;
+
+        Console.WriteLine("[*] hwnd {0:x} @ {1}x{2} sz {3}x{4}", it, r.x, r.y, w, h);
+        if (h != 141)
+          continue;
+        
+        ShowWindow(it, 6);
+        Console.WriteLine("[+] poof");
+        return true;
+      }
+      return false;
+    }
+
+    void fb() {
+      keybd_event((byte)Keys.VolumeMute, 0, 0, 0);
+      keybd_event((byte)Keys.VolumeMute, 0, 2, 0);
+      Thread.Sleep(500);
+      keybd_event((byte)Keys.VolumeMute, 0, 0, 0);
+      keybd_event((byte)Keys.VolumeMute, 0, 2, 0);
+
+      while (true) {
+        if (fa()) {
+          break;
+        }
+        Console.WriteLine("[!] not found");
+        Thread.Sleep(1000);
+      }
+      this.Invoke((MethodInvoker)delegate {
+        Application.Exit();
+      });
+    }
+
+    public void Run() {
+      Console.WriteLine("[+] hi");
+      new Thread(new ThreadStart(fb)).Start();
+      Application.Run();
+      Console.WriteLine("[+] bye");
+    }
+  }
+}
+"@ -ReferencedAssemblies System.Windows.Forms
+
+(New-Object -TypeName A.B).Run()

contrib/plugins/README.md

@@ -11,10 +11,18 @@ save one of these as `.epilogue.html` inside a folder to customize it:
 
 
 
+## example browser-js
+point `--js-browser` to one of these by URL:
+
+* [`minimal-up2k.js`](minimal-up2k.js) is similar to the above `minimal-up2k.html` except it applies globally to all write-only folders
+* [`up2k-hooks.js`](up2k-hooks.js) lets you specify a ruleset for files to skip uploading
+  * [`up2k-hook-ytid.js`](up2k-hook-ytid.js) is a more specific example checking youtube-IDs against some API
+
+
+
 ## example browser-css
 point `--css-browser` to one of these by URL:
 
-* [`browser.css`](browser.css) changes the background
 * [`browser-icons.css`](browser-icons.css) adds filetype icons
 
 

contrib/plugins/browser.css

@@ -1,30 +0,0 @@
-html {
-    background: #222 url('/wp/wallhaven-mdjrqy.jpg') center / cover no-repeat fixed;
-}
-#files th {
-    background: rgba(32, 32, 32, 0.9) !important;
-}
-#ops,
-#tree,
-#files td {
-    background: rgba(32, 32, 32, 0.3) !important;
-}
-
-
-html.light {
-    background: #eee url('/wp/wallhaven-dpxl6l.png') center / cover no-repeat fixed;
-}
-html.light #files th {
-    background: rgba(255, 255, 255, 0.9) !important;
-}
-html.light .logue,
-html.light #ops,
-html.light #tree,
-html.light #files td {
-    background: rgba(248, 248, 248, 0.8) !important;
-}
-
-
-#files * {
-    background: transparent !important;
-}

contrib/plugins/minimal-up2k.html

@@ -7,7 +7,7 @@
 
     /* make the up2k ui REALLY minimal by hiding a bunch of stuff: */
 
-    #ops, #tree, #path, #wrap>h2:last-child,  /* main tabs and navigators (tree/breadcrumbs) */
+    #ops, #tree, #path, #epi+h2,  /* main tabs and navigators (tree/breadcrumbs) */
 
     #u2conf tr:first-child>td[rowspan]:not(#u2btn_cw),  /* most of the config options */
 

contrib/plugins/minimal-up2k.js

@@ -0,0 +1,59 @@
+/*
+
+makes the up2k ui REALLY minimal by hiding a bunch of stuff
+
+almost the same as minimal-up2k.html except this one...:
+
+ -- applies to every write-only folder when used with --js-browser
+
+ -- only applies if javascript is enabled
+
+ -- doesn't hide the total upload ETA display
+
+ -- looks slightly better
+
+*/
+
+var u2min = `
+<style>
+
+#ops, #path, #tree, #files, #epi+div+h2,
+#u2conf td.c+.c, #u2cards, #srch_dz, #srch_zd {
+  display: none !important;
+}
+#u2conf {margin:5em auto 0 auto !important}
+#u2conf.ww {width:70em}
+#u2conf.w {width:50em}
+#u2conf.w .c,
+#u2conf.w #u2btn_cw {text-align:left}
+#u2conf.w #u2btn_cw {width:70%}
+#u2etaw {margin:3em auto}
+#u2etaw.w {
+  text-align: center;
+  margin: -3.5em auto 5em auto;
+}
+#u2etaw.w #u2etas {margin-right:-37em}
+#u2etaw.w #u2etas.o {margin-top:-2.2em}
+#u2etaw.ww {margin:-1em auto}
+#u2etaw.ww #u2etas {padding-left:4em}
+#u2etas {
+  background: none !important;
+  border: none !important;
+}
+#wrap {margin-left:2em !important}
+.logue {
+  border: none !important;
+  margin: 2em auto !important;
+}
+.logue:before {content:'' !important}
+
+</style>
+
+<a href="#" onclick="this.parentNode.innerHTML='';">show advanced options</a>
+`;
+
+if (!has(perms, 'read')) {
+  var e2 = mknod('div');
+  e2.innerHTML = u2min;
+  ebi('wrap').insertBefore(e2, QS('#epi+h2'));
+}

contrib/plugins/up2k-hook-ytid.js

@@ -0,0 +1,297 @@
+// way more specific example --
+// assumes all files dropped into the uploader have a youtube-id somewhere in the filename,
+// locates the youtube-ids and passes them to an API which returns a list of IDs which should be uploaded
+//
+// also tries to find the youtube-id in the embedded metadata
+//
+// assumes copyparty is behind nginx as /ytq is a standalone service which must be rproxied in place
+
+function up2k_namefilter(good_files, nil_files, bad_files, hooks) {
+    var passthru = up2k.uc.fsearch;
+    if (passthru)
+        return hooks[0](good_files, nil_files, bad_files, hooks.slice(1));
+
+    a_up2k_namefilter(good_files, nil_files, bad_files, hooks).then(() => { });
+}
+
+// ebi('op_up2k').appendChild(mknod('input','unick'));
+
+function bstrpos(buf, ptn) {
+    var ofs = 0,
+        ch0 = ptn[0],
+        sz = buf.byteLength;
+
+    while (true) {
+        ofs = buf.indexOf(ch0, ofs);
+        if (ofs < 0 || ofs >= sz)
+            return -1;
+
+        for (var a = 1; a < ptn.length; a++)
+            if (buf[ofs + a] !== ptn[a])
+                break;
+
+        if (a === ptn.length)
+            return ofs;
+
+        ++ofs;
+    }
+}
+
+async function a_up2k_namefilter(good_files, nil_files, bad_files, hooks) {
+    var t0 = Date.now(),
+        yt_ids = new Set(),
+        textdec = new TextDecoder('latin1'),
+        md_ptn = new TextEncoder().encode('youtube.com/watch?v='),
+        file_ids = [],  // all IDs found for each good_files
+        md_only = [],  // `${id} ${fn}` where ID was only found in metadata
+        mofs = 0,
+        mnchk = 0,
+        mfile = '',
+        myid = localStorage.getItem('ytid_t0');
+
+    if (!myid)
+        localStorage.setItem('ytid_t0', myid = Date.now());
+
+    for (var a = 0; a < good_files.length; a++) {
+        var [fobj, name] = good_files[a],
+            cname = name,  // will clobber
+            sz = fobj.size,
+            ids = [],
+            fn_ids = [],
+            md_ids = [],
+            id_ok = false,
+            m;
+
+        // all IDs found in this file
+        file_ids.push(ids);
+
+        // look for ID in filename; reduce the
+        // metadata-scan intensity if the id looks safe
+        m = /[\[(-]([\w-]{11})[\])]?\.(?:mp4|webm|mkv|flv|opus|ogg|mp3|m4a|aac)$/i.exec(name);
+        id_ok = !!m;
+
+        while (true) {
+            // fuzzy catch-all;
+            // some ytdl fork did %(title)-%(id).%(ext) ...
+            m = /(?:^|[^\w])([\w-]{11})(?:$|[^\w-])/.exec(cname);
+            if (!m)
+                break;
+
+            cname = cname.replace(m[1], '');
+            yt_ids.add(m[1]);
+            fn_ids.unshift(m[1]);
+        }
+
+        // look for IDs in video metadata,
+        if (/\.(mp4|webm|mkv|flv|opus|ogg|mp3|m4a|aac)$/i.exec(name)) {
+            toast.show('inf r', 0, `analyzing file ${a + 1} / ${good_files.length} :\n${name}\n\nhave analysed ${++mnchk} files in ${(Date.now() - t0) / 1000} seconds, ${humantime((good_files.length - (a + 1)) * (((Date.now() - t0) / 1000) / mnchk))} remaining,\n\nbiggest offset so far is ${mofs}, in this file:\n\n${mfile}`);
+
+            // check first and last 128 MiB;
+            // pWxOroN5WCo.mkv @  6edb98 (6.92M)
+            // Nf-nN1wF5Xo.mp4 @ 4a98034 (74.6M)
+            var chunksz = 1024 * 1024 * 2,  // byte
+                aspan = id_ok ? 128 : 512;  // MiB
+
+            aspan = parseInt(Math.min(sz / 2, aspan * 1024 * 1024) / chunksz) * chunksz;
+            if (!aspan)
+                aspan = Math.min(sz, chunksz);
+
+            for (var side = 0; side < 2; side++) {
+                var ofs = side ? Math.max(0, sz - aspan) : 0,
+                    nchunks = aspan / chunksz;
+
+                for (var chunk = 0; chunk < nchunks; chunk++) {
+                    var bchunk = await fobj.slice(ofs, ofs + chunksz + 16).arrayBuffer(),
+                        uchunk = new Uint8Array(bchunk, 0, bchunk.byteLength),
+                        bofs = bstrpos(uchunk, md_ptn),
+                        absofs = Math.min(ofs + bofs, (sz - ofs) + bofs),
+                        txt = bofs < 0 ? '' : textdec.decode(uchunk.subarray(bofs)),
+                        m;
+
+                    //console.log(`side ${ side }, chunk ${ chunk }, ofs ${ ofs }, bchunk ${ bchunk.byteLength }, txt ${ txt.length }`);
+                    while (true) {
+                        // mkv/webm have [a-z] immediately after url
+                        m = /(youtube\.com\/watch\?v=[\w-]{11})/.exec(txt);
+                        if (!m)
+                            break;
+
+                        txt = txt.replace(m[1], '');
+                        m = m[1].slice(-11);
+
+                        console.log(`found ${m} @${bofs}, ${name} `);
+                        yt_ids.add(m);
+                        if (!has(fn_ids, m) && !has(md_ids, m)) {
+                            md_ids.push(m);
+                            md_only.push(`${m} ${name}`);
+                        }
+                        else
+                            // id appears several times; make it preferred
+                            md_ids.unshift(m);
+
+                        // bail after next iteration
+                        chunk = nchunks - 1;
+                        side = 9;
+
+                        if (mofs < absofs) {
+                            mofs = absofs;
+                            mfile = name;
+                        }
+                    }
+                    ofs += chunksz;
+                    if (ofs >= sz)
+                        break;
+                }
+            }
+        }
+
+        for (var yi of md_ids)
+            ids.push(yi);
+
+        for (var yi of fn_ids)
+            if (!has(ids, yi))
+                ids.push(yi);
+    }
+
+    if (md_only.length)
+        console.log('recovered the following youtube-IDs by inspecting metadata:\n\n' + md_only.join('\n'));
+    else if (yt_ids.size)
+        console.log('did not discover any additional youtube-IDs by inspecting metadata; all the IDs also existed in the filenames');
+    else
+        console.log('failed to find any youtube-IDs at all, sorry');
+
+    if (false) {
+        var msg = `finished analysing ${mnchk} files in ${(Date.now() - t0) / 1000} seconds,\n\nbiggest offset was ${mofs} in this file:\n\n${mfile}`,
+            mfun = function () { toast.ok(0, msg); };
+
+        mfun();
+        setTimeout(mfun, 200);
+
+        return hooks[0]([], [], [], hooks.slice(1));
+    }
+
+    var el = ebi('unick'), unick = el ? el.value : '';
+    if (unick) {
+        console.log(`sending uploader nickname [${unick}]`);
+        fetch(document.location, {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8' },
+            body: 'msg=' + encodeURIComponent(unick)
+        });
+    }
+
+    toast.inf(5, `running query for ${yt_ids.size} youtube-IDs...`);
+
+    var xhr = new XHR();
+    xhr.open('POST', '/ytq', true);
+    xhr.setRequestHeader('Content-Type', 'text/plain');
+    xhr.onload = xhr.onerror = function () {
+        if (this.status != 200)
+            return toast.err(0, `sorry, database query failed ;_;\n\nplease let us know so we can look at it, thx!!\n\nerror ${this.status}: ${(this.response && this.response.err) || this.responseText}`);
+
+        process_id_list(this.responseText);
+    };
+    xhr.send(Array.from(yt_ids).join('\n'));
+
+    function process_id_list(txt) {
+        var wanted_ids = new Set(txt.trim().split('\n')),
+            name_id = {},
+            wanted_names = new Set(),  // basenames with a wanted ID -- not including relpath
+            wanted_names_scoped = {},  // basenames with a wanted ID -> list of dirs to search under
+            wanted_files = new Set();  // filedrops
+
+        for (var a = 0; a < good_files.length; a++) {
+            var name = good_files[a][1];
+            for (var b = 0; b < file_ids[a].length; b++)
+                if (wanted_ids.has(file_ids[a][b])) {
+                    // let the next stage handle this to prevent dupes
+                    //wanted_files.add(good_files[a]);
+
+                    var m = /(.*)\.(mp4|webm|mkv|flv|opus|ogg|mp3|m4a|aac)$/i.exec(name);
+                    if (!m)
+                        continue;
+
+                    var [rd, fn] = vsplit(m[1]);
+
+                    if (fn in wanted_names_scoped)
+                        wanted_names_scoped[fn].push(rd);
+                    else
+                        wanted_names_scoped[fn] = [rd];
+
+                    wanted_names.add(fn);
+                    name_id[m[1]] = file_ids[a][b];
+
+                    break;
+                }
+        }
+
+        // add all files with the same basename as each explicitly wanted file
+        // (infojson/chatlog/etc when ID was discovered from metadata)
+        for (var a = 0; a < good_files.length; a++) {
+            var [rd, name] = vsplit(good_files[a][1]);
+            for (var b = 0; b < 3; b++) {
+                name = name.replace(/\.[^\.]+$/, '');
+                if (!wanted_names.has(name))
+                    continue;
+
+                var vid_fp = false;
+                for (var c of wanted_names_scoped[name])
+                    if (rd.startsWith(c))
+                        vid_fp = c + name;
+
+                if (!vid_fp)
+                    continue;
+
+                var subdir = name_id[vid_fp];
+                subdir = `v${subdir.slice(0, 1)}/${subdir}-${myid}`;
+                var newpath = subdir + '/' + good_files[a][1].split(/\//g).pop();
+
+                // check if this file is a dupe
+                for (var c of good_files)
+                    if (c[1] == newpath)
+                        newpath = null;
+
+                if (!newpath)
+                    break;
+
+                good_files[a][1] = newpath;
+                wanted_files.add(good_files[a]);
+                break;
+            }
+        }
+
+        function upload_filtered() {
+            if (!wanted_files.size)
+                return modal.alert('Good news -- turns out we already have all those.\n\nBut thank you for checking in!');
+
+            hooks[0](Array.from(wanted_files), nil_files, bad_files, hooks.slice(1));
+        }
+
+        function upload_all() {
+            hooks[0](good_files, nil_files, bad_files, hooks.slice(1));
+        }
+
+        var n_skip = good_files.length - wanted_files.size,
+            msg = `you added ${good_files.length} files; ${good_files.length == n_skip ? 'all' : n_skip} of them were skipped --\neither because we already have them,\nor because there is no youtube-ID in your filenames.\n\n<code>OK</code> / <code>Enter</code> = continue uploading just the ${wanted_files.size} files we definitely need\n\n<code>Cancel</code> / <code>ESC</code> = override the filter; upload ALL the files you added`;
+
+        if (!n_skip)
+            upload_filtered();
+        else
+            modal.confirm(msg, upload_filtered, upload_all);
+    };
+}
+
+up2k_hooks.push(function () {
+    up2k.gotallfiles.unshift(up2k_namefilter);
+});
+
+// persist/restore nickname field if present
+setInterval(function () {
+    var o = ebi('unick');
+    if (!o || document.activeElement == o)
+        return;
+
+    o.oninput = function () {
+        localStorage.setItem('unick', o.value);
+    };
+    o.value = localStorage.getItem('unick') || '';
+}, 1000);

contrib/plugins/up2k-hooks.js

@@ -0,0 +1,45 @@
+// hooks into up2k
+
+function up2k_namefilter(good_files, nil_files, bad_files, hooks) {
+    // is called when stuff is dropped into the browser,
+    // after iterating through the directory tree and discovering all files,
+    // before the upload confirmation dialogue is shown
+
+    // good_files will successfully upload
+    // nil_files are empty files and will show an alert in the final hook
+    // bad_files are unreadable and cannot be uploaded
+    var file_lists = [good_files, nil_files, bad_files];
+
+    // build a list of filenames
+    var filenames = [];
+    for (var lst of file_lists)
+        for (var ent of lst)
+            filenames.push(ent[1]);
+
+    toast.inf(5, "running database query...");
+
+    // simulate delay while passing the list to some api for checking
+    setTimeout(function () {
+
+        // only keep webm files as an example
+        var new_lists = [];
+        for (var lst of file_lists) {
+            var keep = [];
+            new_lists.push(keep);
+
+            for (var ent of lst)
+                if (/\.webm$/.test(ent[1]))
+                    keep.push(ent);
+        }
+
+        // finally, call the next hook in the chain
+        [good_files, nil_files, bad_files] = new_lists;
+        hooks[0](good_files, nil_files, bad_files, hooks.slice(1));
+
+    }, 1000);
+}
+
+// register
+up2k_hooks.push(function () {
+    up2k.gotallfiles.unshift(up2k_namefilter);
+});

contrib/systemd/cfssl.service

@@ -0,0 +1,23 @@
+# systemd service which generates a new TLS certificate on each boot,
+# that way the one-year expiry time won't cause any issues --
+# just have everyone trust the ca.pem once every 10 years
+#
+# assumptions/placeholder values:
+#  * this script and copyparty runs as user "cpp"
+#  * copyparty repo is at ~cpp/dev/copyparty
+#  * CA is named partylan
+#  * server IPs = 10.1.2.3 and 192.168.123.1
+#  * server hostname = party.lan
+
+[Unit]
+Description=copyparty certificate generator
+Before=copyparty.service
+
+[Service]
+User=cpp
+Type=oneshot
+SyslogIdentifier=cpp-cert
+ExecStart=/bin/bash -c 'cd ~/dev/copyparty/contrib && ./cfssl.sh partylan 10.1.2.3,192.168.123.1,party.lan y'
+
+[Install]
+WantedBy=multi-user.target

contrib/systemd/copyparty.service

@@ -2,16 +2,22 @@
 # and share '/mnt' with anonymous read+write
 #
 # installation:
-#   cp -pv copyparty.service /etc/systemd/system && systemctl enable --now copyparty
+#   cp -pv copyparty.service /etc/systemd/system
 #   restorecon -vr /etc/systemd/system/copyparty.service
-#   firewall-cmd --permanent --add-port={80,443,3923}/tcp
+#   firewall-cmd --permanent --add-port={80,443,3923}/tcp  # --zone=libvirt
 #   firewall-cmd --reload
+#   systemctl daemon-reload && systemctl enable --now copyparty
 #
 # you may want to:
+#   change "User=cpp" and "/home/cpp/" to another user
+#   remove the nft lines to only listen on port 3923
+# and in the ExecStart= line:
 #   change '/usr/bin/python3' to another interpreter
 #   change '/mnt::rw' to another location or permission-set
-#   remove '-p 80,443,3923' to only listen on port 3923
+#   add '-q' to disable logging on busy servers
 #   add '-i 127.0.0.1' to only allow local connections
+#   add '-e2dsa' to enable filesystem scanning + indexing
+#   add '-e2ts' to enable metadata indexing
 #
 # with `Type=notify`, copyparty will signal systemd when it is ready to
 #   accept connections; correctly delaying units depending on copyparty.
@@ -19,9 +25,11 @@
 #   python disabling line-buffering, so messages are out-of-order:
 #   https://user-images.githubusercontent.com/241032/126040249-cb535cc7-c599-4931-a796-a5d9af691bad.png
 #
-# if you remove -q to enable logging, you may also want to remove the
-#   following line to enable buffering (slightly better performance):
+# unless you add -q to disable logging, you may want to remove the
+#   following line to allow buffering (slightly better performance):
 #   Environment=PYTHONUNBUFFERED=x
+#
+# keep ExecStartPre before ExecStart, at least on rhel8
 
 [Unit]
 Description=copyparty file server
@@ -31,8 +39,23 @@ Type=notify
 SyslogIdentifier=copyparty
 Environment=PYTHONUNBUFFERED=x
 ExecReload=/bin/kill -s USR1 $MAINPID
-ExecStartPre=/bin/bash -c 'mkdir -p /run/tmpfiles.d/ && echo "x /tmp/pe-copyparty*" > /run/tmpfiles.d/copyparty.conf'
-ExecStart=/usr/bin/python3 /usr/local/bin/copyparty-sfx.py -q -p 80,443,3923 -v /mnt::rw
+
+# user to run as + where the TLS certificate is (if any)
+User=cpp
+Environment=XDG_CONFIG_HOME=/home/cpp/.config
+
+# setup forwarding from ports 80 and 443 to port 3923
+ExecStartPre=+/bin/bash -c 'nft -n -a list table nat | awk "/ to :3923 /{print\$NF}" | xargs -rL1 nft delete rule nat prerouting handle; true'
+ExecStartPre=+nft add table ip nat
+ExecStartPre=+nft -- add chain ip nat prerouting { type nat hook prerouting priority -100 \; }
+ExecStartPre=+nft add rule ip nat prerouting tcp dport 80 redirect to :3923
+ExecStartPre=+nft add rule ip nat prerouting tcp dport 443 redirect to :3923
+
+# stop systemd-tmpfiles-clean.timer from deleting copyparty while it's running
+ExecStartPre=+/bin/bash -c 'mkdir -p /run/tmpfiles.d/ && echo "x /tmp/pe-copyparty*" > /run/tmpfiles.d/copyparty.conf'
+
+# copyparty settings
+ExecStart=/usr/bin/python3 /usr/local/bin/copyparty-sfx.py -e2d -v /mnt::rw
 
 [Install]
 WantedBy=multi-user.target

copyparty/__init__.py

@@ -1,80 +1,48 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
-import platform
-import time
-import sys
 import os
+import platform
+import sys
+import time
 
-PY2 = sys.version_info[0] == 2
+try:
+    from typing import TYPE_CHECKING, Any
+except:
+    TYPE_CHECKING = False
+
+PY2 = sys.version_info < (3,)
 if PY2:
     sys.dont_write_bytecode = True
-    unicode = unicode
+    unicode = unicode  # noqa: F821  # pylint: disable=undefined-variable,self-assigning-variable
 else:
     unicode = str
 
-WINDOWS = False
-if platform.system() == "Windows":
-    WINDOWS = [int(x) for x in platform.version().split(".")]
+WINDOWS: Any = (
+    [int(x) for x in platform.version().split(".")]
+    if platform.system() == "Windows"
+    else False
+)
 
 VT100 = not WINDOWS or WINDOWS >= [10, 0, 14393]
 # introduced in anniversary update
 
-ANYWIN = WINDOWS or sys.platform in ["msys"]
+ANYWIN = WINDOWS or sys.platform in ["msys", "cygwin"]
 
 MACOS = platform.system() == "Darwin"
 
-
-def get_unixdir():
-    paths = [
-        (os.environ.get, "XDG_CONFIG_HOME"),
-        (os.path.expanduser, "~/.config"),
-        (os.environ.get, "TMPDIR"),
-        (os.environ.get, "TEMP"),
-        (os.environ.get, "TMP"),
-        (unicode, "/tmp"),
-    ]
-    for chk in [os.listdir, os.mkdir]:
-        for pf, pa in paths:
-            try:
-                p = pf(pa)
-                # print(chk.__name__, p, pa)
-                if not p or p.startswith("~"):
-                    continue
-
-                p = os.path.normpath(p)
-                chk(p)
-                p = os.path.join(p, "copyparty")
-                if not os.path.isdir(p):
-                    os.mkdir(p)
-
-                return p
-            except:
-                pass
-
-    raise Exception("could not find a writable path for config")
+try:
+    CORES = len(os.sched_getaffinity(0))
+except:
+    CORES = (os.cpu_count() if hasattr(os, "cpu_count") else 0) or 2
 
 
 class EnvParams(object):
-    def __init__(self):
+    def __init__(self) -> None:
         self.t0 = time.time()
-        self.mod = os.path.dirname(os.path.realpath(__file__))
-        if self.mod.endswith("__init__"):
-            self.mod = os.path.dirname(self.mod)
-
-        if sys.platform == "win32":
-            self.cfg = os.path.normpath(os.environ["APPDATA"] + "/copyparty")
-        elif sys.platform == "darwin":
-            self.cfg = os.path.expanduser("~/Library/Preferences/copyparty")
-        else:
-            self.cfg = get_unixdir()
-
-        self.cfg = self.cfg.replace("\\", "/")
-        try:
-            os.makedirs(self.cfg)
-        except:
-            if not os.path.isdir(self.cfg):
-                raise
+        self.mod = None
+        self.cfg = None
+        self.ox = getattr(sys, "oxidized", None)
 
 
 E = EnvParams()

copyparty/__main__.py

@@ -8,35 +8,60 @@ __copyright__ = 2019
 __license__ = "MIT"
 __url__ = "https://github.com/9001/copyparty/"
 
-import re
-import os
-import sys
-import time
-import shutil
+import argparse
 import filecmp
 import locale
-import argparse
+import os
+import re
+import shutil
+import sys
 import threading
+import time
 import traceback
 from textwrap import dedent
 
-from .__init__ import E, WINDOWS, ANYWIN, VT100, PY2, unicode
-from .__version__ import S_VERSION, S_BUILD_DT, CODENAME
-from .svchub import SvcHub
-from .util import py_desc, align_tab, IMPLICATIONS, ansi_re, min_ex
+from .__init__ import ANYWIN, CORES, PY2, VT100, WINDOWS, E, EnvParams, unicode
+from .__version__ import CODENAME, S_BUILD_DT, S_VERSION
 from .authsrv import re_vol
+from .svchub import SvcHub
+from .util import (
+    IMPLICATIONS,
+    JINJA_VER,
+    PYFTPD_VER,
+    SQLITE_VER,
+    align_tab,
+    ansi_re,
+    min_ex,
+    py_desc,
+    termsize,
+    wrap,
+)
 
-HAVE_SSL = True
 try:
+    from collections.abc import Callable
+    from types import FrameType
+
+    from typing import Any, Optional
+except:
+    pass
+
+try:
+    HAVE_SSL = True
     import ssl
 except:
     HAVE_SSL = False
 
-printed = ""
+printed: list[str] = []
 
 
 class RiceFormatter(argparse.HelpFormatter):
-    def _get_help_string(self, action):
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        if PY2:
+            kwargs["width"] = termsize()[0]
+
+        super(RiceFormatter, self).__init__(*args, **kwargs)
+
+    def _get_help_string(self, action: argparse.Action) -> str:
         """
         same as ArgumentDefaultsHelpFormatter(HelpFormatter)
         except the help += [...] line now has colors
@@ -45,41 +70,146 @@ class RiceFormatter(argparse.HelpFormatter):
         if not VT100:
             fmt = " (default: %(default)s)"
 
-        help = action.help
-        if "%(default)" not in action.help:
+        ret = unicode(action.help)
+        if "%(default)" not in ret:
             if action.default is not argparse.SUPPRESS:
                 defaulting_nargs = [argparse.OPTIONAL, argparse.ZERO_OR_MORE]
                 if action.option_strings or action.nargs in defaulting_nargs:
-                    help += fmt
-        return help
+                    ret += fmt
 
-    def _fill_text(self, text, width, indent):
+        if not VT100:
+            ret = re.sub("\033\[[0-9;]+m", "", ret)
+
+        return ret
+
+    def _fill_text(self, text: str, width: int, indent: str) -> str:
         """same as RawDescriptionHelpFormatter(HelpFormatter)"""
         return "".join(indent + line + "\n" for line in text.splitlines())
 
+    def __add_whitespace(self, idx: int, iWSpace: int, text: str) -> str:
+        return (" " * iWSpace) + text if idx else text
+
+    def _split_lines(self, text: str, width: int) -> list[str]:
+        # https://stackoverflow.com/a/35925919
+        textRows = text.splitlines()
+        ptn = re.compile(r"\s*[0-9\-]{0,}\.?\s*")
+        for idx, line in enumerate(textRows):
+            search = ptn.search(line)
+            if not line.strip():
+                textRows[idx] = " "
+            elif search:
+                lWSpace = search.end()
+                lines = [
+                    self.__add_whitespace(i, lWSpace, x)
+                    for i, x in enumerate(wrap(line, width, width - 1))
+                ]
+                textRows[idx] = lines
+
+        return [item for sublist in textRows for item in sublist]
+
 
 class Dodge11874(RiceFormatter):
-    def __init__(self, *args, **kwargs):
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
         kwargs["width"] = 9003
         super(Dodge11874, self).__init__(*args, **kwargs)
 
 
-def lprint(*a, **ka):
-    global printed
+class BasicDodge11874(
+    argparse.ArgumentDefaultsHelpFormatter, argparse.RawDescriptionHelpFormatter
+):
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        kwargs["width"] = 9003
+        super(BasicDodge11874, self).__init__(*args, **kwargs)
 
-    txt = " ".join(unicode(x) for x in a) + ka.get("end", "\n")
-    printed += txt
+
+def lprint(*a: Any, **ka: Any) -> None:
+    eol = ka.pop("end", "\n")
+    txt: str = " ".join(unicode(x) for x in a) + eol
+    printed.append(txt)
     if not VT100:
         txt = ansi_re.sub("", txt)
 
-    print(txt, **ka)
+    print(txt, end="", **ka)
 
 
-def warn(msg):
+def warn(msg: str) -> None:
     lprint("\033[1mwarning:\033[0;33m {}\033[0m\n".format(msg))
 
 
-def ensure_locale():
+def init_E(E: EnvParams) -> None:
+    # __init__ runs 18 times when oxidized; do expensive stuff here
+
+    def get_unixdir() -> str:
+        paths: list[tuple[Callable[..., str], str]] = [
+            (os.environ.get, "XDG_CONFIG_HOME"),
+            (os.path.expanduser, "~/.config"),
+            (os.environ.get, "TMPDIR"),
+            (os.environ.get, "TEMP"),
+            (os.environ.get, "TMP"),
+            (unicode, "/tmp"),
+        ]
+        for chk in [os.listdir, os.mkdir]:
+            for pf, pa in paths:
+                try:
+                    p = pf(pa)
+                    # print(chk.__name__, p, pa)
+                    if not p or p.startswith("~"):
+                        continue
+
+                    p = os.path.normpath(p)
+                    chk(p)  # type: ignore
+                    p = os.path.join(p, "copyparty")
+                    if not os.path.isdir(p):
+                        os.mkdir(p)
+
+                    return p
+                except:
+                    pass
+
+        raise Exception("could not find a writable path for config")
+
+    def _unpack() -> str:
+        import atexit
+        import tarfile
+        import tempfile
+        from importlib.resources import open_binary
+
+        td = tempfile.TemporaryDirectory(prefix="")
+        atexit.register(td.cleanup)
+        tdn = td.name
+
+        with open_binary("copyparty", "z.tar") as tgz:
+            with tarfile.open(fileobj=tgz) as tf:
+                tf.extractall(tdn)
+
+        return tdn
+
+    try:
+        E.mod = os.path.dirname(os.path.realpath(__file__))
+        if E.mod.endswith("__init__"):
+            E.mod = os.path.dirname(E.mod)
+    except:
+        if not E.ox:
+            raise
+
+        E.mod = _unpack()
+
+    if sys.platform == "win32":
+        E.cfg = os.path.normpath(os.environ["APPDATA"] + "/copyparty")
+    elif sys.platform == "darwin":
+        E.cfg = os.path.expanduser("~/Library/Preferences/copyparty")
+    else:
+        E.cfg = get_unixdir()
+
+    E.cfg = E.cfg.replace("\\", "/")
+    try:
+        os.makedirs(E.cfg)
+    except:
+        if not os.path.isdir(E.cfg):
+            raise
+
+
+def ensure_locale() -> None:
     for x in [
         "en_US.UTF-8",
         "English_United States.UTF8",
@@ -87,13 +217,13 @@ def ensure_locale():
     ]:
         try:
             locale.setlocale(locale.LC_ALL, x)
-            lprint("Locale:", x)
+            lprint("Locale: {}\n".format(x))
             break
         except:
             continue
 
 
-def ensure_cert():
+def ensure_cert() -> None:
     """
     the default cert (and the entire TLS support) is only here to enable the
     crypto.subtle javascript API, which is necessary due to the webkit guys
@@ -119,8 +249,8 @@ def ensure_cert():
     # printf 'NO\n.\n.\n.\n.\ncopyparty-insecure\n.\n' | faketime '2000-01-01 00:00:00' openssl req -x509 -sha256 -newkey rsa:2048 -keyout insecure.pem -out insecure.pem -days $((($(printf %d 0x7fffffff)-$(date +%s --date=2000-01-01T00:00:00Z))/(60*60*24))) -nodes && ls -al insecure.pem && openssl x509 -in insecure.pem -text -noout
 
 
-def configure_ssl_ver(al):
-    def terse_sslver(txt):
+def configure_ssl_ver(al: argparse.Namespace) -> None:
+    def terse_sslver(txt: str) -> str:
         txt = txt.lower()
         for c in ["_", "v", "."]:
             txt = txt.replace(c, "")
@@ -135,8 +265,8 @@ def configure_ssl_ver(al):
     flags = [k for k in ssl.__dict__ if ptn.match(k)]
     # SSLv2 SSLv3 TLSv1 TLSv1_1 TLSv1_2 TLSv1_3
     if "help" in sslver:
-        avail = [terse_sslver(x[6:]) for x in flags]
-        avail = " ".join(sorted(avail) + ["all"])
+        avail1 = [terse_sslver(x[6:]) for x in flags]
+        avail = " ".join(sorted(avail1) + ["all"])
         lprint("\navailable ssl/tls versions:\n  " + avail)
         sys.exit(0)
 
@@ -157,12 +287,12 @@ def configure_ssl_ver(al):
 
     for k in ["ssl_flags_en", "ssl_flags_de"]:
         num = getattr(al, k)
-        lprint("{}: {:8x} ({})".format(k, num, num))
+        lprint("{0}: {1:8x} ({1})".format(k, num))
 
     # think i need that beer now
 
 
-def configure_ssl_ciphers(al):
+def configure_ssl_ciphers(al: argparse.Namespace) -> None:
     ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
     if al.ssl_ver:
         ctx.options &= ~al.ssl_flags_en
@@ -186,8 +316,8 @@ def configure_ssl_ciphers(al):
         sys.exit(0)
 
 
-def args_from_cfg(cfg_path):
-    ret = []
+def args_from_cfg(cfg_path: str) -> list[str]:
+    ret: list[str] = []
     skip = False
     with open(cfg_path, "rb") as f:
         for ln in [x.decode("utf-8").strip() for x in f]:
@@ -212,29 +342,30 @@ def args_from_cfg(cfg_path):
     return ret
 
 
-def sighandler(sig=None, frame=None):
+def sighandler(sig: Optional[int] = None, frame: Optional[FrameType] = None) -> None:
     msg = [""] * 5
     for th in threading.enumerate():
+        stk = sys._current_frames()[th.ident]  # type: ignore
         msg.append(str(th))
-        msg.extend(traceback.format_stack(sys._current_frames()[th.ident]))
+        msg.extend(traceback.format_stack(stk))
 
     msg.append("\n")
     print("\n".join(msg))
 
 
-def disable_quickedit():
-    import ctypes
+def disable_quickedit() -> None:
     import atexit
+    import ctypes
     from ctypes import wintypes
 
-    def ecb(ok, fun, args):
+    def ecb(ok: bool, fun: Any, args: list[Any]) -> list[Any]:
         if not ok:
-            err = ctypes.get_last_error()
+            err: int = ctypes.get_last_error()  # type: ignore
             if err:
-                raise ctypes.WinError(err)
+                raise ctypes.WinError(err)  # type: ignore
         return args
 
-    k32 = ctypes.WinDLL("kernel32", use_last_error=True)
+    k32 = ctypes.WinDLL(str("kernel32"), use_last_error=True)  # type: ignore
     if PY2:
         wintypes.LPDWORD = ctypes.POINTER(wintypes.DWORD)
 
@@ -244,14 +375,14 @@ def disable_quickedit():
     k32.GetConsoleMode.argtypes = (wintypes.HANDLE, wintypes.LPDWORD)
     k32.SetConsoleMode.argtypes = (wintypes.HANDLE, wintypes.DWORD)
 
-    def cmode(out, mode=None):
+    def cmode(out: bool, mode: Optional[int] = None) -> int:
         h = k32.GetStdHandle(-11 if out else -10)
         if mode:
-            return k32.SetConsoleMode(h, mode)
+            return k32.SetConsoleMode(h, mode)  # type: ignore
 
-        mode = wintypes.DWORD()
-        k32.GetConsoleMode(h, ctypes.byref(mode))
-        return mode.value
+        cmode = wintypes.DWORD()
+        k32.GetConsoleMode(h, ctypes.byref(cmode))
+        return cmode.value
 
     # disable quickedit
     mode = orig_in = cmode(False)
@@ -270,7 +401,17 @@ def disable_quickedit():
             cmode(True, mode | 4)
 
 
-def run_argparse(argv, formatter):
+def showlic() -> None:
+    p = os.path.join(E.mod, "res", "COPYING.txt")
+    if not os.path.exists(p):
+        print("no relevant license info to display")
+        return
+
+    with open(p, "rb") as f:
+        print(f.read().decode("utf-8", "replace"))
+
+
+def run_argparse(argv: list[str], formatter: Any, retry: bool) -> argparse.Namespace:
     ap = argparse.ArgumentParser(
         formatter_class=formatter,
         prog="copyparty",
@@ -282,7 +423,7 @@ def run_argparse(argv, formatter):
     except:
         fk_salt = "hunter2"
 
-    cores = os.cpu_count() if hasattr(os, "cpu_count") else 4
+    hcores = min(CORES, 3)  # 4% faster than 4+ on py3.9 @ r5-4500U
 
     sects = [
         [
@@ -291,16 +432,17 @@ def run_argparse(argv, formatter):
             dedent(
                 """
             -a takes username:password,
-            -v takes src:dst:perm1:perm2:permN:volflag1:volflag2:volflagN:...
-               where "perm" is "permissions,username1,username2,..."
-               and "volflag" is config flags to set on this volume
-            
+            -v takes src:dst:\033[33mperm\033[0m1:\033[33mperm\033[0m2:\033[33mperm\033[0mN:\033[32mvolflag\033[0m1:\033[32mvolflag\033[0m2:\033[32mvolflag\033[0mN:...
+                * "\033[33mperm\033[0m" is "permissions,username1,username2,..."
+                * "\033[32mvolflag\033[0m" is config flags to set on this volume
+
             list of permissions:
               "r" (read):   list folder contents, download files
               "w" (write):  upload files; need "r" to see the uploads
               "m" (move):   move files and folders; need "w" at destination
               "d" (delete): permanently delete files and folders
               "g" (get):    download files, but cannot see folder contents
+              "G" (upget):  "get", but can see filekeys of their own uploads
 
             too many volflags to list here, see the other sections
 
@@ -313,7 +455,7 @@ def run_argparse(argv, formatter):
                * w (write-only) for everyone
                * rw (read+write) for ed
                * reject duplicate files  \033[0m
-            
+
             if no accounts or volumes are configured,
             current folder will be read/write for everyone
 
@@ -334,37 +476,54 @@ def run_argparse(argv, formatter):
             \033[0muploads, general:
               \033[36mnodupe\033[35m rejects existing files (instead of symlinking them)
               \033[36mnosub\033[35m forces all uploads into the top folder of the vfs
+              \033[36mmagic$\033[35m enables filetype detection for nameless uploads
               \033[36mgz\033[35m allows server-side gzip of uploads with ?gz (also c,xz)
               \033[36mpk\033[35m forces server-side compression, optional arg: xz,9
-            
+
             \033[0mupload rules:
               \033[36mmaxn=250,600\033[35m max 250 uploads over 15min
               \033[36mmaxb=1g,300\033[35m max 1 GiB over 5min (suffixes: b, k, m, g)
               \033[36msz=1k-3m\033[35m allow filesizes between 1 KiB and 3MiB
-            
+              \033[36mdf=1g\033[35m ensure 1 GiB free disk space
+
             \033[0mupload rotation:
             (moves all uploads into the specified folder structure)
               \033[36mrotn=100,3\033[35m 3 levels of subfolders with 100 entries in each
               \033[36mrotf=%Y-%m/%d-%H\033[35m date-formatted organizing
               \033[36mlifetime=3600\033[35m uploads are deleted after 1 hour
-            
+
             \033[0mdatabase, general:
               \033[36me2d\033[35m sets -e2d (all -e2* args can be set using ce2* volflags)
               \033[36md2ts\033[35m disables metadata collection for existing files
               \033[36md2ds\033[35m disables onboot indexing, overrides -e2ds*
               \033[36md2t\033[35m disables metadata collection, overrides -e2t*
+              \033[36md2v\033[35m disables file verification, overrides -e2v*
               \033[36md2d\033[35m disables all database stuff, overrides -e2*
-              \033[36mnohash=\\.iso$\033[35m skips hashing file contents if path matches *.iso
-              \033[36mnoidx=\\.iso$\033[35m fully ignores the contents at paths matching *.iso
               \033[36mhist=/tmp/cdb\033[35m puts thumbnails and indexes at that location
               \033[36mscan=60\033[35m scan for new files every 60sec, same as --re-maxage
-            
+              \033[36mnohash=\\.iso$\033[35m skips hashing file contents if path matches *.iso
+              \033[36mnoidx=\\.iso$\033[35m fully ignores the contents at paths matching *.iso
+              \033[36mnoforget$\033[35m don't forget files when deleted from disk
+              \033[36mxdev\033[35m do not descend into other filesystems
+              \033[36mxvol\033[35m skip symlinks leaving the volume root
+
             \033[0mdatabase, audio tags:
             "mte", "mth", "mtp", "mtm" all work the same as -mte, -mth, ...
               \033[36mmtp=.bpm=f,audio-bpm.py\033[35m uses the "audio-bpm.py" program to
                 generate ".bpm" tags from uploads (f = overwrite tags)
               \033[36mmtp=ahash,vhash=media-hash.py\033[35m collects two tags at once
-            
+
+            \033[0mthumbnails:
+              \033[36mdthumb\033[35m disables all thumbnails
+              \033[36mdvthumb\033[35m disables video thumbnails
+              \033[36mdathumb\033[35m disables audio thumbnails (spectrograms)
+              \033[36mdithumb\033[35m disables image thumbnails
+
+            \033[0mclient and ux:
+              \033[36mhtml_head=TXT\033[35m includes TXT in the <head>
+              \033[36mrobots\033[35m allows indexing by search engines (default)
+              \033[36mnorobots\033[35m kindly asks search engines to leave
+
             \033[0mothers:
               \033[36mfk=8\033[35m generates per-file accesskeys,
                 which will then be required at the "g" permission
@@ -373,7 +532,7 @@ def run_argparse(argv, formatter):
         ],
         [
             "urlform",
-            "",
+            "how to handle url-form POSTs",
             dedent(
                 """
             values for --urlform:
@@ -412,78 +571,105 @@ def run_argparse(argv, formatter):
     ap2.add_argument("-c", metavar="PATH", type=u, action="append", help="add config file")
     ap2.add_argument("-nc", metavar="NUM", type=int, default=64, help="max num clients")
     ap2.add_argument("-j", metavar="CORES", type=int, default=1, help="max num cpu cores, 0=all")
-    ap2.add_argument("-a", metavar="ACCT", type=u, action="append", help="add account, USER:PASS; example [ed:wark")
-    ap2.add_argument("-v", metavar="VOL", type=u, action="append", help="add volume, SRC:DST:FLAG; example [.::r], [/mnt/nas/music:/music:r:aed")
-    ap2.add_argument("-ed", action="store_true", help="enable ?dots")
-    ap2.add_argument("-emp", action="store_true", help="enable markdown plugins")
+    ap2.add_argument("-a", metavar="ACCT", type=u, action="append", help="add account, \033[33mUSER\033[0m:\033[33mPASS\033[0m; example [\033[32med:wark\033[0m]")
+    ap2.add_argument("-v", metavar="VOL", type=u, action="append", help="add volume, \033[33mSRC\033[0m:\033[33mDST\033[0m:\033[33mFLAG\033[0m; examples [\033[32m.::r\033[0m], [\033[32m/mnt/nas/music:/music:r:aed\033[0m]")
+    ap2.add_argument("-ed", action="store_true", help="enable the ?dots url parameter / client option which allows clients to see dotfiles / hidden files")
+    ap2.add_argument("-emp", action="store_true", help="enable markdown plugins -- neat but dangerous, big XSS risk")
     ap2.add_argument("-mcr", metavar="SEC", type=int, default=60, help="md-editor mod-chk rate")
-    ap2.add_argument("--urlform", metavar="MODE", type=u, default="print,get", help="how to handle url-forms; examples: [stash], [save,get]")
-    ap2.add_argument("--wintitle", metavar="TXT", type=u, default="cpp @ $pub", help="window title, for example '$ip-10.1.2.' or '$ip-'")
+    ap2.add_argument("--urlform", metavar="MODE", type=u, default="print,get", help="how to handle url-form POSTs; see --help-urlform")
+    ap2.add_argument("--wintitle", metavar="TXT", type=u, default="cpp @ $pub", help="window title, for example [\033[32m$ip-10.1.2.\033[0m] or [\033[32m$ip-]")
+    ap2.add_argument("--license", action="store_true", help="show licenses and exit")
+    ap2.add_argument("--version", action="store_true", help="show versions and exit")
+
+    ap2 = ap.add_argument_group('qr options')
+    ap2.add_argument("--qr", action="store_true", help="show http:// QR-code on startup")
+    ap2.add_argument("--qrs", action="store_true", help="show https:// QR-code on startup")
+    ap2.add_argument("--qrl", metavar="PATH", type=u, default="", help="location to include in the url, for example [\033[32mpriv/?pw=hunter2\033[0m]")
+    ap2.add_argument("--qri", metavar="PREFIX", type=u, default="", help="select IP which starts with PREFIX")
+    ap2.add_argument("--qr-fg", metavar="COLOR", type=int, default=16, help="foreground")
+    ap2.add_argument("--qr-bg", metavar="COLOR", type=int, default=229, help="background (white=255)")
+    ap2.add_argument("--qrp", metavar="CELLS", type=int, default=4, help="padding (spec says 4 or more, but 1 is usually fine)")
+    ap2.add_argument("--qrz", metavar="N", type=int, default=0, help="[\033[32m1\033[0m]=1x, [\033[32m2\033[0m]=2x, [\033[32m0\033[0m]=auto (try 2 on broken fonts)")
 
     ap2 = ap.add_argument_group('upload options')
-    ap2.add_argument("--dotpart", action="store_true", help="dotfile incomplete uploads")
-    ap2.add_argument("--sparse", metavar="MiB", type=int, default=4, help="up2k min.size threshold (mswin-only)")
+    ap2.add_argument("--dotpart", action="store_true", help="dotfile incomplete uploads, hiding them from clients unless -ed")
+    ap2.add_argument("--plain-ip", action="store_true", help="when avoiding filename collisions by appending the uploader's ip to the filename: append the plaintext ip instead of salting and hashing the ip")
     ap2.add_argument("--unpost", metavar="SEC", type=int, default=3600*12, help="grace period where uploads can be deleted by the uploader, even without delete permissions; 0=disabled")
+    ap2.add_argument("--reg-cap", metavar="N", type=int, default=38400, help="max number of uploads to keep in memory when running without -e2d; roughly 1 MiB RAM per 600")
     ap2.add_argument("--no-fpool", action="store_true", help="disable file-handle pooling -- instead, repeatedly close and reopen files during upload")
-    ap2.add_argument("--use-fpool", action="store_true", help="force file-handle pooling, even if copyparty thinks you're better off without")
-    ap2.add_argument("--no-symlink", action="store_true", help="duplicate file contents instead")
-    ap2.add_argument("--reg-cap", metavar="N", type=int, default=9000, help="max number of uploads to keep in memory when running without -e2d")
+    ap2.add_argument("--use-fpool", action="store_true", help="force file-handle pooling, even if copyparty thinks you're better off without -- probably useful on nfs and cow filesystems (zfs, btrfs)")
+    ap2.add_argument("--hardlink", action="store_true", help="prefer hardlinks instead of symlinks when possible (within same filesystem)")
+    ap2.add_argument("--never-symlink", action="store_true", help="do not fallback to symlinks when a hardlink cannot be made")
+    ap2.add_argument("--no-dedup", action="store_true", help="disable symlink/hardlink creation; copy file contents instead")
+    ap2.add_argument("--magic", action="store_true", help="enable filetype detection on nameless uploads")
+    ap2.add_argument("--df", metavar="GiB", type=float, default=0, help="ensure GiB free disk space by rejecting upload requests")
+    ap2.add_argument("--sparse", metavar="MiB", type=int, default=4, help="windows-only: minimum size of incoming uploads through up2k before they are made into sparse files")
+    ap2.add_argument("--turbo", metavar="LVL", type=int, default=0, help="configure turbo-mode in up2k client; [\033[32m0\033[0m] = off and warn if enabled, [\033[32m1\033[0m] = off, [\033[32m2\033[0m] = on, [\033[32m3\033[0m] = on and disable datecheck")
+    ap2.add_argument("--u2sort", metavar="TXT", type=u, default="s", help="upload order; [\033[32ms\033[0m]=smallest-first, [\033[32mn\033[0m]=alphabetical, [\033[32mfs\033[0m]=force-s, [\033[32mfn\033[0m]=force-n -- alphabetical is a bit slower on fiber/LAN but makes it easier to eyeball if everything went fine")
+    ap2.add_argument("--write-uplog", action="store_true", help="write POST reports to textfiles in working-directory")
 
     ap2 = ap.add_argument_group('network options')
     ap2.add_argument("-i", metavar="IP", type=u, default="0.0.0.0", help="ip to bind (comma-sep.)")
     ap2.add_argument("-p", metavar="PORT", type=u, default="3923", help="ports to bind (comma/range)")
-    ap2.add_argument("--rproxy", metavar="DEPTH", type=int, default=1, help="which ip to keep; 0 = tcp, 1 = origin (first x-fwd), 2 = cloudflare, 3 = nginx, -1 = closest proxy")
+    ap2.add_argument("--rproxy", metavar="DEPTH", type=int, default=1, help="which ip to keep; [\033[32m0\033[0m]=tcp, [\033[32m1\033[0m]=origin (first x-fwd), [\033[32m2\033[0m]=cloudflare, [\033[32m3\033[0m]=nginx, [\033[32m-1\033[0m]=closest proxy")
     ap2.add_argument("--s-wr-sz", metavar="B", type=int, default=256*1024, help="socket write size in bytes")
-    ap2.add_argument("--s-wr-slp", metavar="SEC", type=float, default=0, help="socket write delay in seconds")
-    ap2.add_argument("--rsp-slp", metavar="SEC", type=float, default=0, help="response delay in seconds")
+    ap2.add_argument("--s-wr-slp", metavar="SEC", type=float, default=0, help="debug: socket write delay in seconds")
+    ap2.add_argument("--rsp-slp", metavar="SEC", type=float, default=0, help="debug: response delay in seconds")
 
     ap2 = ap.add_argument_group('SSL/TLS options')
-    ap2.add_argument("--http-only", action="store_true", help="disable ssl/tls")
-    ap2.add_argument("--https-only", action="store_true", help="disable plaintext")
-    ap2.add_argument("--ssl-ver", metavar="LIST", type=u, help="set allowed ssl/tls versions; [help] shows available versions; default is what your python version considers safe")
-    ap2.add_argument("--ciphers", metavar="LIST", type=u, help="set allowed ssl/tls ciphers; [help] shows available ciphers")
+    ap2.add_argument("--http-only", action="store_true", help="disable ssl/tls -- force plaintext")
+    ap2.add_argument("--https-only", action="store_true", help="disable plaintext -- force tls")
+    ap2.add_argument("--ssl-ver", metavar="LIST", type=u, help="set allowed ssl/tls versions; [\033[32mhelp\033[0m] shows available versions; default is what your python version considers safe")
+    ap2.add_argument("--ciphers", metavar="LIST", type=u, help="set allowed ssl/tls ciphers; [\033[32mhelp\033[0m] shows available ciphers")
     ap2.add_argument("--ssl-dbg", action="store_true", help="dump some tls info")
-    ap2.add_argument("--ssl-log", metavar="PATH", type=u, help="log master secrets")
+    ap2.add_argument("--ssl-log", metavar="PATH", type=u, help="log master secrets for later decryption in wireshark")
 
     ap2 = ap.add_argument_group('FTP options')
-    ap2.add_argument("--ftp", metavar="PORT", type=int, help="enable FTP server on PORT, for example 3921")
-    ap2.add_argument("--ftps", metavar="PORT", type=int, help="enable FTPS server on PORT, for example 3990")
+    ap2.add_argument("--ftp", metavar="PORT", type=int, help="enable FTP server on PORT, for example \033[32m3921")
+    ap2.add_argument("--ftps", metavar="PORT", type=int, help="enable FTPS server on PORT, for example \033[32m3990")
     ap2.add_argument("--ftp-dbg", action="store_true", help="enable debug logging")
     ap2.add_argument("--ftp-nat", metavar="ADDR", type=u, help="the NAT address to use for passive connections")
-    ap2.add_argument("--ftp-pr", metavar="P-P", type=u, help="the range of TCP ports to use for passive connections, for example 12000-13000")
+    ap2.add_argument("--ftp-pr", metavar="P-P", type=u, help="the range of TCP ports to use for passive connections, for example \033[32m12000-13000")
 
     ap2 = ap.add_argument_group('opt-outs')
-    ap2.add_argument("-nw", action="store_true", help="disable writes (benchmark)")
-    ap2.add_argument("--keep-qem", action="store_true", help="do not disable quick-edit-mode on windows")
+    ap2.add_argument("-nw", action="store_true", help="never write anything to disk (debug/benchmark)")
+    ap2.add_argument("--keep-qem", action="store_true", help="do not disable quick-edit-mode on windows (it is disabled to avoid accidental text selection which will deadlock copyparty)")
     ap2.add_argument("--no-del", action="store_true", help="disable delete operations")
     ap2.add_argument("--no-mv", action="store_true", help="disable move/rename operations")
-    ap2.add_argument("-nih", action="store_true", help="no info hostname")
-    ap2.add_argument("-nid", action="store_true", help="no info disk-usage")
+    ap2.add_argument("-nih", action="store_true", help="no info hostname -- don't show in UI")
+    ap2.add_argument("-nid", action="store_true", help="no info disk-usage -- don't show in UI")
     ap2.add_argument("--no-zip", action="store_true", help="disable download as zip/tar")
     ap2.add_argument("--no-lifetime", action="store_true", help="disable automatic deletion of uploads after a certain time (lifetime volflag)")
 
     ap2 = ap.add_argument_group('safety options')
-    ap2.add_argument("--ls", metavar="U[,V[,F]]", type=u, help="scan all volumes; arguments USER,VOL,FLAGS; example [**,*,ln,p,r]")
-    ap2.add_argument("--salt", type=u, default="hunter2", help="up2k file-hash salt")
-    ap2.add_argument("--fk-salt", metavar="SALT", type=u, default=fk_salt, help="per-file accesskey salt")
+    ap2.add_argument("-s", action="count", default=0, help="increase safety: Disable thumbnails / potentially dangerous software (ffmpeg/pillow/vips), hide partial uploads, avoid crawlers.\n └─Alias of\033[32m --dotpart --no-thumb --no-mtag-ff --no-robots --force-js")
+    ap2.add_argument("-ss", action="store_true", help="further increase safety: Prevent js-injection, accidental move/delete, broken symlinks, 404 on 403, ban on excessive 404s.\n └─Alias of\033[32m -s --no-dot-mv --no-dot-ren --unpost=0 --no-del --no-mv --hardlink --vague-403 --ban-404=50,60,1440 -nih")
+    ap2.add_argument("-sss", action="store_true", help="further increase safety: Enable logging to disk, scan for dangerous symlinks.\n └─Alias of\033[32m -ss -lo=cpp-%%Y-%%m%%d-%%H%%M%%S.txt.xz --ls=**,*,ln,p,r")
+    ap2.add_argument("--ls", metavar="U[,V[,F]]", type=u, help="do a sanity/safety check of all volumes on startup; arguments \033[33mUSER\033[0m,\033[33mVOL\033[0m,\033[33mFLAGS\033[0m; example [\033[32m**,*,ln,p,r\033[0m]")
+    ap2.add_argument("--salt", type=u, default="hunter2", help="up2k file-hash salt; used to generate unpredictable internal identifiers for uploads -- doesn't really matter")
+    ap2.add_argument("--fk-salt", metavar="SALT", type=u, default=fk_salt, help="per-file accesskey salt; used to generate unpredictable URLs for hidden files -- this one DOES matter")
     ap2.add_argument("--no-dot-mv", action="store_true", help="disallow moving dotfiles; makes it impossible to move folders containing dotfiles")
     ap2.add_argument("--no-dot-ren", action="store_true", help="disallow renaming dotfiles; makes it impossible to make something a dotfile")
     ap2.add_argument("--no-logues", action="store_true", help="disable rendering .prologue/.epilogue.html into directory listings")
     ap2.add_argument("--no-readme", action="store_true", help="disable rendering readme.md into directory listings")
     ap2.add_argument("--vague-403", action="store_true", help="send 404 instead of 403 (security through ambiguity, very enterprise)")
-    ap2.add_argument("--force-js", action="store_true", help="don't send HTML folder listings, force clients to use the embedded json instead")
+    ap2.add_argument("--force-js", action="store_true", help="don't send folder listings as HTML, force clients to use the embedded json instead -- slight protection against misbehaving search engines which ignore --no-robots")
     ap2.add_argument("--no-robots", action="store_true", help="adds http and html headers asking search engines to not index anything")
+    ap2.add_argument("--logout", metavar="H", type=float, default="8086", help="logout clients after H hours of inactivity; [\033[32m0.0028\033[0m]=10sec, [\033[32m0.1\033[0m]=6min, [\033[32m24\033[0m]=day, [\033[32m168\033[0m]=week, [\033[32m720\033[0m]=month, [\033[32m8760\033[0m]=year)")
+    ap2.add_argument("--ban-pw", metavar="N,W,B", type=u, default="9,60,1440", help="more than \033[33mN\033[0m wrong passwords in \033[33mW\033[0m minutes = ban for \033[33mB\033[0m minutes; disable with [\033[32mno\033[0m]")
+    ap2.add_argument("--ban-404", metavar="N,W,B", type=u, default="no", help="hitting more than \033[33mN\033[0m 404's in \033[33mW\033[0m minutes = ban for \033[33mB\033[0m minutes (disabled by default since turbo-up2k counts as 404s)")
 
-    ap2 = ap.add_argument_group('yolo options')
+    ap2 = ap.add_argument_group('shutdown options')
     ap2.add_argument("--ign-ebind", action="store_true", help="continue running even if it's impossible to listen on some of the requested endpoints")
     ap2.add_argument("--ign-ebind-all", action="store_true", help="continue running even if it's impossible to receive connections at all")
+    ap2.add_argument("--exit", metavar="WHEN", type=u, default="", help="shutdown after WHEN has finished; for example [\033[32midx\033[0m] will do volume indexing + metadata analysis")
 
     ap2 = ap.add_argument_group('logging options')
     ap2.add_argument("-q", action="store_true", help="quiet")
-    ap2.add_argument("-lo", metavar="PATH", type=u, help="logfile, example: cpp-%%Y-%%m%%d-%%H%%M%%S.txt.xz")
+    ap2.add_argument("-lo", metavar="PATH", type=u, help="logfile, example: \033[32mcpp-%%Y-%%m%%d-%%H%%M%%S.txt.xz")
     ap2.add_argument("--no-voldump", action="store_true", help="do not list volumes and permissions on startup")
-    ap2.add_argument("--log-conn", action="store_true", help="print tcp-server msgs")
-    ap2.add_argument("--log-htp", action="store_true", help="print http-server threadpool scaling")
+    ap2.add_argument("--log-conn", action="store_true", help="debug: print tcp-server msgs")
+    ap2.add_argument("--log-htp", action="store_true", help="debug: print http-server threadpool scaling")
     ap2.add_argument("--ihead", metavar="HEADER", type=u, action='append', help="dump incoming header")
     ap2.add_argument("--lf-url", metavar="RE", type=u, default=r"^/\.cpr/|\?th=[wj]$", help="dont log URLs matching")
 
@@ -497,61 +683,91 @@ def run_argparse(argv, formatter):
     ap2.add_argument("--no-athumb", action="store_true", help="disable audio thumbnails (spectrograms)")
     ap2.add_argument("--no-vthumb", action="store_true", help="disable video thumbnails")
     ap2.add_argument("--th-size", metavar="WxH", default="320x256", help="thumbnail res")
-    ap2.add_argument("--th-mt", metavar="CORES", type=int, default=cores, help="num cpu cores to use for generating thumbnails")
+    ap2.add_argument("--th-mt", metavar="CORES", type=int, default=CORES, help="num cpu cores to use for generating thumbnails")
     ap2.add_argument("--th-convt", metavar="SEC", type=int, default=60, help="conversion timeout in seconds")
     ap2.add_argument("--th-no-crop", action="store_true", help="dynamic height; show full image")
+    ap2.add_argument("--th-dec", metavar="LIBS", default="vips,pil,ff", help="image decoders, in order of preference")
     ap2.add_argument("--th-no-jpg", action="store_true", help="disable jpg output")
     ap2.add_argument("--th-no-webp", action="store_true", help="disable webp output")
-    ap2.add_argument("--th-ff-jpg", action="store_true", help="force jpg for video thumbs")
+    ap2.add_argument("--th-ff-jpg", action="store_true", help="force jpg output for video thumbs")
     ap2.add_argument("--th-ff-swr", action="store_true", help="use swresample instead of soxr for audio thumbs")
-    ap2.add_argument("--th-poke", metavar="SEC", type=int, default=300, help="activity labeling cooldown")
+    ap2.add_argument("--th-poke", metavar="SEC", type=int, default=300, help="activity labeling cooldown -- avoids doing keepalive pokes (updating the mtime) on thumbnail folders more often than SEC seconds")
     ap2.add_argument("--th-clean", metavar="SEC", type=int, default=43200, help="cleanup interval; 0=disabled")
-    ap2.add_argument("--th-maxage", metavar="SEC", type=int, default=604800, help="max folder age")
-    ap2.add_argument("--th-covers", metavar="N,N", type=u, default="folder.png,folder.jpg,cover.png,cover.jpg", help="folder thumbnails to stat for")
+    ap2.add_argument("--th-maxage", metavar="SEC", type=int, default=604800, help="max folder age -- folders which haven't been poked for longer than --th-poke seconds will get deleted every --th-clean seconds")
+    ap2.add_argument("--th-covers", metavar="N,N", type=u, default="folder.png,folder.jpg,cover.png,cover.jpg", help="folder thumbnails to stat/look for")
+    # https://pillow.readthedocs.io/en/stable/handbook/image-file-formats.html
+    # https://github.com/libvips/libvips
+    # ffmpeg -hide_banner -demuxers | awk '/^ D  /{print$2}' | while IFS= read -r x; do ffmpeg -hide_banner -h demuxer=$x; done | grep -E '^Demuxer |extensions:'
+    ap2.add_argument("--th-r-pil", metavar="T,T", type=u, default="bmp,dib,gif,icns,ico,jpg,jpeg,jp2,jpx,pcx,png,pbm,pgm,ppm,pnm,sgi,tga,tif,tiff,webp,xbm,dds,xpm,heif,heifs,heic,heics,avif,avifs", help="image formats to decode using pillow")
+    ap2.add_argument("--th-r-vips", metavar="T,T", type=u, default="jpg,jpeg,jp2,jpx,jxl,tif,tiff,png,webp,heic,avif,fit,fits,fts,exr,svg,hdr,ppm,pgm,pfm,gif,nii", help="image formats to decode using pyvips")
+    ap2.add_argument("--th-r-ffi", metavar="T,T", type=u, default="apng,avif,avifs,bmp,dds,dib,fit,fits,fts,gif,heic,heics,heif,heifs,icns,ico,jp2,jpeg,jpg,jpx,jxl,pbm,pcx,pfm,pgm,png,pnm,ppm,psd,sgi,tga,tif,tiff,webp,xbm,xpm", help="image formats to decode using ffmpeg")
+    ap2.add_argument("--th-r-ffv", metavar="T,T", type=u, default="av1,asf,avi,flv,m4v,mkv,mjpeg,mjpg,mpg,mpeg,mpg2,mpeg2,h264,avc,mts,h265,hevc,mov,3gp,mp4,ts,mpegts,nut,ogv,ogm,rm,vob,webm,wmv", help="video formats to decode using ffmpeg")
+    ap2.add_argument("--th-r-ffa", metavar="T,T", type=u, default="aac,m4a,ogg,opus,flac,alac,mp3,mp2,ac3,dts,wma,ra,wav,aif,aiff,au,alaw,ulaw,mulaw,amr,gsm,ape,tak,tta,wv,mpc", help="audio formats to decode using ffmpeg")
 
     ap2 = ap.add_argument_group('transcoding options')
     ap2.add_argument("--no-acode", action="store_true", help="disable audio transcoding")
-    ap2.add_argument("--ac-maxage", metavar="SEC", type=int, default=86400, help="delete transcode output after SEC seconds")
+    ap2.add_argument("--ac-maxage", metavar="SEC", type=int, default=86400, help="delete cached transcode output after SEC seconds")
 
     ap2 = ap.add_argument_group('general db options')
-    ap2.add_argument("-e2d", action="store_true", help="enable up2k database")
-    ap2.add_argument("-e2ds", action="store_true", help="enable up2k db-scanner, sets -e2d")
-    ap2.add_argument("-e2dsa", action="store_true", help="scan all folders (for search), sets -e2ds")
+    ap2.add_argument("-e2d", action="store_true", help="enable up2k database, making files searchable + enables upload deduplocation")
+    ap2.add_argument("-e2ds", action="store_true", help="scan writable folders for new files on startup; sets -e2d")
+    ap2.add_argument("-e2dsa", action="store_true", help="scans all folders on startup; sets -e2ds")
+    ap2.add_argument("-e2v", action="store_true", help="verify file integrity; rehash all files and compare with db")
+    ap2.add_argument("-e2vu", action="store_true", help="on hash mismatch: update the database with the new hash")
+    ap2.add_argument("-e2vp", action="store_true", help="on hash mismatch: panic and quit copyparty")
     ap2.add_argument("--hist", metavar="PATH", type=u, help="where to store volume data (db, thumbs)")
     ap2.add_argument("--no-hash", metavar="PTN", type=u, help="regex: disable hashing of matching paths during e2ds folder scans")
     ap2.add_argument("--no-idx", metavar="PTN", type=u, help="regex: disable indexing of matching paths during e2ds folder scans")
+    ap2.add_argument("--no-dhash", action="store_true", help="disable rescan acceleration; do full database integrity check -- makes the db ~5%% smaller and bootup/rescans 3~10x slower")
+    ap2.add_argument("--no-forget", action="store_true", help="never forget indexed files, even when deleted from disk -- makes it impossible to ever upload the same file twice")
+    ap2.add_argument("--xdev", action="store_true", help="do not descend into other filesystems (symlink or bind-mount to another HDD, ...)")
+    ap2.add_argument("--xvol", action="store_true", help="skip symlinks leaving the volume root")
+    ap2.add_argument("--hash-mt", metavar="CORES", type=int, default=hcores, help="num cpu cores to use for file hashing; set 0 or 1 for single-core hashing")
     ap2.add_argument("--re-maxage", metavar="SEC", type=int, default=0, help="disk rescan volume interval, 0=off, can be set per-volume with the 'scan' volflag")
-    ap2.add_argument("--srch-time", metavar="SEC", type=int, default=30, help="search deadline")
-    ap2.add_argument("--srch-hits", metavar="N", type=int, default=1000, help="max search results")
-    
+    ap2.add_argument("--db-act", metavar="SEC", type=float, default=10, help="defer any scheduled volume reindexing until SEC seconds after last db write (uploads, renames, ...)")
+    ap2.add_argument("--srch-time", metavar="SEC", type=int, default=45, help="search deadline -- terminate searches running for more than SEC seconds")
+    ap2.add_argument("--srch-hits", metavar="N", type=int, default=7999, help="max search results to allow clients to fetch; 125 results will be shown initially")
+
     ap2 = ap.add_argument_group('metadata db options')
-    ap2.add_argument("-e2t", action="store_true", help="enable metadata indexing")
-    ap2.add_argument("-e2ts", action="store_true", help="enable metadata scanner, sets -e2t")
-    ap2.add_argument("-e2tsr", action="store_true", help="rescan all metadata, sets -e2ts")
-    ap2.add_argument("--no-mutagen", action="store_true", help="use FFprobe for tags instead")
-    ap2.add_argument("--no-mtag-ff", action="store_true", help="never use FFprobe as tag reader")
-    ap2.add_argument("--mtag-mt", metavar="CORES", type=int, default=cores, help="num cpu cores to use for tag scanning")
+    ap2.add_argument("-e2t", action="store_true", help="enable metadata indexing; makes it possible to search for artist/title/codec/resolution/...")
+    ap2.add_argument("-e2ts", action="store_true", help="scan existing files on startup; sets -e2t")
+    ap2.add_argument("-e2tsr", action="store_true", help="delete all metadata from DB and do a full rescan; sets -e2ts")
+    ap2.add_argument("--no-mutagen", action="store_true", help="use FFprobe for tags instead; will catch more tags")
+    ap2.add_argument("--no-mtag-ff", action="store_true", help="never use FFprobe as tag reader; is probably safer")
+    ap2.add_argument("--mtag-to", metavar="SEC", type=int, default=60, help="timeout for ffprobe tag-scan")
+    ap2.add_argument("--mtag-mt", metavar="CORES", type=int, default=CORES, help="num cpu cores to use for tag scanning")
+    ap2.add_argument("--mtag-v", action="store_true", help="verbose tag scanning; print errors from mtp subprocesses and such")
+    ap2.add_argument("--mtag-vv", action="store_true", help="debug mtp settings")
     ap2.add_argument("-mtm", metavar="M=t,t,t", type=u, action="append", help="add/replace metadata mapping")
     ap2.add_argument("-mte", metavar="M,M,M", type=u, help="tags to index/display (comma-sep.)",
-        default="circle,album,.tn,artist,title,.bpm,key,.dur,.q,.vq,.aq,vc,ac,res,.fps,ahash,vhash")
+        default="circle,album,.tn,artist,title,.bpm,key,.dur,.q,.vq,.aq,vc,ac,fmt,res,.fps,ahash,vhash")
     ap2.add_argument("-mth", metavar="M,M,M", type=u, help="tags to hide by default (comma-sep.)",
-        default=".vq,.aq,vc,ac,res,.fps")
-    ap2.add_argument("-mtp", metavar="M=[f,]bin", type=u, action="append", help="read tag M using bin")
+        default=".vq,.aq,vc,ac,fmt,res,.fps")
+    ap2.add_argument("-mtp", metavar="M=[f,]BIN", type=u, action="append", help="read tag M using program BIN to parse the file")
 
     ap2 = ap.add_argument_group('ui options')
+    ap2.add_argument("--lang", metavar="LANG", type=u, default="eng", help="language")
+    ap2.add_argument("--theme", metavar="NUM", type=int, default=0, help="default theme to use")
+    ap2.add_argument("--themes", metavar="NUM", type=int, default=8, help="number of themes installed")
+    ap2.add_argument("--favico", metavar="TXT", type=u, default="c 000 none" if retry else "🎉 000 none", help="\033[33mfavicon-text\033[0m [ \033[33mforeground\033[0m [ \033[33mbackground\033[0m ] ], set blank to disable")
     ap2.add_argument("--js-browser", metavar="L", type=u, help="URL to additional JS to include")
     ap2.add_argument("--css-browser", metavar="L", type=u, help="URL to additional CSS to include")
     ap2.add_argument("--html-head", metavar="TXT", type=u, default="", help="text to append to the <head> of all HTML pages")
     ap2.add_argument("--textfiles", metavar="CSV", type=u, default="txt,nfo,diz,cue,readme", help="file extensions to present as plaintext")
+    ap2.add_argument("--txt-max", metavar="KiB", type=int, default=64, help="max size of embedded textfiles on ?doc= (anything bigger will be lazy-loaded by JS)")
     ap2.add_argument("--doctitle", metavar="TXT", type=u, default="copyparty", help="title / service-name to show in html documents")
 
     ap2 = ap.add_argument_group('debug options')
-    ap2.add_argument("--no-sendfile", action="store_true", help="disable sendfile")
-    ap2.add_argument("--no-scandir", action="store_true", help="disable scandir")
-    ap2.add_argument("--no-fastboot", action="store_true", help="wait for up2k indexing")
+    ap2.add_argument("--no-sendfile", action="store_true", help="disable sendfile; instead using a traditional file read loop")
+    ap2.add_argument("--no-scandir", action="store_true", help="disable scandir; instead using listdir + stat on each file")
+    ap2.add_argument("--no-fastboot", action="store_true", help="wait for up2k indexing before starting the httpd")
     ap2.add_argument("--no-htp", action="store_true", help="disable httpserver threadpool, create threads as-needed instead")
-    ap2.add_argument("--stackmon", metavar="P,S", type=u, help="write stacktrace to Path every S second")
+    ap2.add_argument("--stackmon", metavar="P,S", type=u, help="write stacktrace to Path every S second, for example --stackmon=\033[32m./st/%%Y-%%m/%%d/%%H%%M.xz,60")
     ap2.add_argument("--log-thrs", metavar="SEC", type=float, help="list active threads every SEC")
+    ap2.add_argument("--log-fk", metavar="REGEX", type=u, default="", help="log filekey params for files where path matches REGEX; [\033[32m.\033[0m] (a single dot) = all files")
+    ap2.add_argument("--bak-flips", action="store_true", help="[up2k] if a client uploads a bitflipped/corrupted chunk, store a copy according to --bf-nc and --bf-dir")
+    ap2.add_argument("--bf-nc", metavar="NUM", type=int, default=200, help="bak-flips: stop if there's more than NUM files at --kf-dir already; default: 6.3 GiB max (200*32M)")
+    ap2.add_argument("--bf-dir", metavar="PATH", type=u, default="bf", help="bak-flips: store corrupted chunks at PATH; default: folder named 'bf' wherever copyparty was started")
     # fmt: on
 
     ap2 = ap.add_argument_group("help sections")
@@ -569,18 +785,33 @@ def run_argparse(argv, formatter):
     return ret
 
 
-def main(argv=None):
+def main(argv: Optional[list[str]] = None) -> None:
     time.strptime("19970815", "%Y%m%d")  # python#7980
     if WINDOWS:
         os.system("rem")  # enables colors
 
+    init_E(E)
     if argv is None:
         argv = sys.argv
 
-    desc = py_desc().replace("[", "\033[1;30m[")
+    f = '\033[36mcopyparty v{} "\033[35m{}\033[36m" ({})\n{}\033[0;36m\n   sqlite v{} | jinja2 v{} | pyftpd v{}\n\033[0m'
+    f = f.format(
+        S_VERSION,
+        CODENAME,
+        S_BUILD_DT,
+        py_desc().replace("[", "\033[1;30m["),
+        SQLITE_VER,
+        JINJA_VER,
+        PYFTPD_VER,
+    )
+    lprint(f)
 
-    f = '\033[36mcopyparty v{} "\033[35m{}\033[36m" ({})\n{}\033[0m\n'
-    lprint(f.format(S_VERSION, CODENAME, S_BUILD_DT, desc))
+    if "--version" in argv:
+        sys.exit(0)
+
+    if "--license" in argv:
+        showlic()
+        sys.exit(0)
 
     ensure_locale()
     if HAVE_SSL:
@@ -591,7 +822,7 @@ def main(argv=None):
             supp = args_from_cfg(v)
             argv.extend(supp)
 
-    deprecated = []
+    deprecated: list[tuple[str, str]] = []
     for dk, nk in deprecated:
         try:
             idx = argv.index(dk)
@@ -604,26 +835,36 @@ def main(argv=None):
         time.sleep(2)
 
     try:
-        if len(argv) == 1 and (ANYWIN or not os.geteuid()):
-            argv.extend(["-p80,443,3923", "--ign-ebind"])
+        if len(argv) == 1:
+            argv.extend(["--qr"])
+            if ANYWIN or not os.geteuid():
+                argv.extend(["-p80,443,3923", "--ign-ebind"])
     except:
         pass
 
-    try:
-        al = run_argparse(argv, RiceFormatter)
-    except AssertionError:
-        al = run_argparse(argv, Dodge11874)
+    retry = False
+    for fmtr in [RiceFormatter, RiceFormatter, Dodge11874, BasicDodge11874]:
+        try:
+            al = run_argparse(argv, fmtr, retry)
+        except SystemExit:
+            raise
+        except:
+            retry = True
+            lprint("\n[ {} ]:\n{}\n".format(fmtr, min_ex()))
+
+    assert al
+    al.E = E  # __init__ is not shared when oxidized
 
     if WINDOWS and not al.keep_qem:
         try:
             disable_quickedit()
         except:
-            print("\nfailed to disable quick-edit-mode:\n" + min_ex() + "\n")
+            lprint("\nfailed to disable quick-edit-mode:\n" + min_ex() + "\n")
 
     if not VT100:
         al.wintitle = ""
 
-    nstrs = []
+    nstrs: list[str] = []
     anymod = False
     for ostr in al.v or []:
         m = re_vol.match(ostr)
@@ -639,7 +880,7 @@ def main(argv=None):
             if re.match("c[^,]", opt):
                 mod = True
                 na.append("c," + opt[1:])
-            elif re.sub("^[rwmdg]*", "", opt) and "," not in opt:
+            elif re.sub("^[rwmdgG]*", "", opt) and "," not in opt:
                 mod = True
                 perm = opt[0]
                 if perm == "a":
@@ -674,6 +915,15 @@ def main(argv=None):
     except:
         raise Exception("invalid value for -p")
 
+    for arg, kname, okays in [["--u2sort", "u2sort", "s n fs fn"]]:
+        val = unicode(getattr(al, kname))
+        if val not in okays.split():
+            zs = "argument {} cannot be '{}'; try one of these: {}"
+            raise Exception(zs.format(arg, val, okays))
+
+    if not al.qrs and [k for k in argv if k.startswith("--qr")]:
+        al.qr = True
+
     if HAVE_SSL:
         if al.ssl_ver:
             configure_ssl_ver(al)
@@ -694,7 +944,7 @@ def main(argv=None):
 
     # signal.signal(signal.SIGINT, sighandler)
 
-    SvcHub(al, argv, printed).run()
+    SvcHub(al, argv, "".join(printed)).run()
 
 
 if __name__ == "__main__":

copyparty/__version__.py

@@ -1,8 +1,8 @@
 # coding: utf-8
 
-VERSION = (1, 2, 2)
-CODENAME = "ftp btw"
-BUILD_DT = (2022, 3, 20)
+VERSION = (1, 4, 6)
+CODENAME = "mostly reliable"
+BUILD_DT = (2022, 10, 13)
 
 S_VERSION = ".".join(map(str, VERSION))
 S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT)

copyparty/bos/bos.py

@@ -2,23 +2,30 @@
 from __future__ import print_function, unicode_literals
 
 import os
-from ..util import fsenc, fsdec, SYMTIME
+
+from ..util import SYMTIME, fsdec, fsenc
 from . import path
 
+try:
+    from typing import Optional
+except:
+    pass
+
+_ = (path,)
 
 # grep -hRiE '(^|[^a-zA-Z_\.-])os\.' . | gsed -r 's/ /\n/g;s/\(/(\n/g' | grep -hRiE '(^|[^a-zA-Z_\.-])os\.' | sort | uniq -c
 # printf 'os\.(%s)' "$(grep ^def bos/__init__.py | gsed -r 's/^def //;s/\(.*//' | tr '\n' '|' | gsed -r 's/.$//')"
 
 
-def chmod(p, mode):
+def chmod(p: str, mode: int) -> None:
     return os.chmod(fsenc(p), mode)
 
 
-def listdir(p="."):
+def listdir(p: str = ".") -> list[str]:
     return [fsdec(x) for x in os.listdir(fsenc(p))]
 
 
-def makedirs(name, mode=0o755, exist_ok=True):
+def makedirs(name: str, mode: int = 0o755, exist_ok: bool = True) -> None:
     bname = fsenc(name)
     try:
         os.makedirs(bname, mode)
@@ -27,31 +34,33 @@ def makedirs(name, mode=0o755, exist_ok=True):
             raise
 
 
-def mkdir(p, mode=0o755):
+def mkdir(p: str, mode: int = 0o755) -> None:
     return os.mkdir(fsenc(p), mode)
 
 
-def rename(src, dst):
+def rename(src: str, dst: str) -> None:
     return os.rename(fsenc(src), fsenc(dst))
 
 
-def replace(src, dst):
+def replace(src: str, dst: str) -> None:
     return os.replace(fsenc(src), fsenc(dst))
 
 
-def rmdir(p):
+def rmdir(p: str) -> None:
     return os.rmdir(fsenc(p))
 
 
-def stat(p):
+def stat(p: str) -> os.stat_result:
     return os.stat(fsenc(p))
 
 
-def unlink(p):
+def unlink(p: str) -> None:
     return os.unlink(fsenc(p))
 
 
-def utime(p, times=None, follow_symlinks=True):
+def utime(
+    p: str, times: Optional[tuple[float, float]] = None, follow_symlinks: bool = True
+) -> None:
     if SYMTIME:
         return os.utime(fsenc(p), times, follow_symlinks=follow_symlinks)
     else:
@@ -60,7 +69,7 @@ def utime(p, times=None, follow_symlinks=True):
 
 if hasattr(os, "lstat"):
 
-    def lstat(p):
+    def lstat(p: str) -> os.stat_result:
         return os.lstat(fsenc(p))
 
 else:

copyparty/bos/path.py

@@ -2,43 +2,44 @@
 from __future__ import print_function, unicode_literals
 
 import os
-from ..util import fsenc, fsdec, SYMTIME
+
+from ..util import SYMTIME, fsdec, fsenc
 
 
-def abspath(p):
+def abspath(p: str) -> str:
     return fsdec(os.path.abspath(fsenc(p)))
 
 
-def exists(p):
+def exists(p: str) -> bool:
     return os.path.exists(fsenc(p))
 
 
-def getmtime(p, follow_symlinks=True):
+def getmtime(p: str, follow_symlinks: bool = True) -> float:
     if not follow_symlinks and SYMTIME:
         return os.lstat(fsenc(p)).st_mtime
     else:
         return os.path.getmtime(fsenc(p))
 
 
-def getsize(p):
+def getsize(p: str) -> int:
     return os.path.getsize(fsenc(p))
 
 
-def isfile(p):
+def isfile(p: str) -> bool:
     return os.path.isfile(fsenc(p))
 
 
-def isdir(p):
+def isdir(p: str) -> bool:
     return os.path.isdir(fsenc(p))
 
 
-def islink(p):
+def islink(p: str) -> bool:
     return os.path.islink(fsenc(p))
 
 
-def lexists(p):
+def lexists(p: str) -> bool:
     return os.path.lexists(fsenc(p))
 
 
-def realpath(p):
+def realpath(p: str) -> str:
     return fsdec(os.path.realpath(fsenc(p)))

copyparty/broker_mp.py

@@ -1,37 +1,56 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
-import time
 import threading
+import time
 
-from .broker_util import try_exec
+import queue
+
+from .__init__ import CORES, TYPE_CHECKING
 from .broker_mpw import MpWorker
+from .broker_util import try_exec
 from .util import mp
 
+if TYPE_CHECKING:
+    from .svchub import SvcHub
+
+try:
+    from typing import Any
+except:
+    pass
+
+
+class MProcess(mp.Process):
+    def __init__(
+        self,
+        q_pend: queue.Queue[tuple[int, str, list[Any]]],
+        q_yield: queue.Queue[tuple[int, str, list[Any]]],
+        target: Any,
+        args: Any,
+    ) -> None:
+        super(MProcess, self).__init__(target=target, args=args)
+        self.q_pend = q_pend
+        self.q_yield = q_yield
+
 
 class BrokerMp(object):
     """external api; manages MpWorkers"""
 
-    def __init__(self, hub):
+    def __init__(self, hub: "SvcHub") -> None:
         self.hub = hub
         self.log = hub.log
         self.args = hub.args
 
         self.procs = []
-        self.retpend = {}
-        self.retpend_mutex = threading.Lock()
         self.mutex = threading.Lock()
 
-        self.num_workers = self.args.j or mp.cpu_count()
+        self.num_workers = self.args.j or CORES
         self.log("broker", "booting {} subprocesses".format(self.num_workers))
         for n in range(1, self.num_workers + 1):
-            q_pend = mp.Queue(1)
-            q_yield = mp.Queue(64)
+            q_pend: queue.Queue[tuple[int, str, list[Any]]] = mp.Queue(1)
+            q_yield: queue.Queue[tuple[int, str, list[Any]]] = mp.Queue(64)
 
-            proc = mp.Process(target=MpWorker, args=(q_pend, q_yield, self.args, n))
-            proc.q_pend = q_pend
-            proc.q_yield = q_yield
-            proc.clients = {}
+            proc = MProcess(q_pend, q_yield, MpWorker, (q_pend, q_yield, self.args, n))
 
             thr = threading.Thread(
                 target=self.collector, args=(proc,), name="mp-sink-{}".format(n)
@@ -42,11 +61,11 @@ class BrokerMp(object):
             self.procs.append(proc)
             proc.start()
 
-    def shutdown(self):
+    def shutdown(self) -> None:
         self.log("broker", "shutting down")
         for n, proc in enumerate(self.procs):
             thr = threading.Thread(
-                target=proc.q_pend.put([0, "shutdown", []]),
+                target=proc.q_pend.put((0, "shutdown", [])),
                 name="mp-shutdown-{}-{}".format(n, len(self.procs)),
             )
             thr.start()
@@ -62,12 +81,12 @@ class BrokerMp(object):
 
             procs.pop()
 
-    def reload(self):
+    def reload(self) -> None:
         self.log("broker", "reloading")
         for _, proc in enumerate(self.procs):
-            proc.q_pend.put([0, "reload", []])
+            proc.q_pend.put((0, "reload", []))
 
-    def collector(self, proc):
+    def collector(self, proc: MProcess) -> None:
         """receive message from hub in other process"""
         while True:
             msg = proc.q_yield.get()
@@ -78,10 +97,7 @@ class BrokerMp(object):
 
             elif dest == "retq":
                 # response from previous ipc call
-                with self.retpend_mutex:
-                    retq = self.retpend.pop(retq_id)
-
-                retq.put(args)
+                raise Exception("invalid broker_mp usage")
 
             else:
                 # new ipc invoking managed service in hub
@@ -93,9 +109,9 @@ class BrokerMp(object):
                 rv = try_exec(retq_id, obj, *args)
 
                 if retq_id:
-                    proc.q_pend.put([retq_id, "retq", rv])
+                    proc.q_pend.put((retq_id, "retq", rv))
 
-    def put(self, want_retval, dest, *args):
+    def say(self, dest: str, *args: Any) -> None:
         """
         send message to non-hub component in other process,
         returns a Queue object which eventually contains the response if want_retval
@@ -103,7 +119,7 @@ class BrokerMp(object):
         """
         if dest == "listen":
             for p in self.procs:
-                p.q_pend.put([0, dest, [args[0], len(self.procs)]])
+                p.q_pend.put((0, dest, [args[0], len(self.procs)]))
 
         elif dest == "cb_httpsrv_up":
             self.hub.cb_httpsrv_up()

copyparty/broker_mpw.py

@@ -1,20 +1,40 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
-import sys
+import argparse
+import os
 import signal
+import sys
 import threading
 
-from .broker_util import ExceptionalQueue
+import queue
+
+from .__init__ import ANYWIN
+from .authsrv import AuthSrv
+from .broker_util import BrokerCli, ExceptionalQueue
 from .httpsrv import HttpSrv
-from .util import FAKE_MP
-from copyparty.authsrv import AuthSrv
+from .util import FAKE_MP, HMaccas
+
+try:
+    from types import FrameType
+
+    from typing import Any, Optional, Union
+except:
+    pass
 
 
-class MpWorker(object):
+class MpWorker(BrokerCli):
     """one single mp instance"""
 
-    def __init__(self, q_pend, q_yield, args, n):
+    def __init__(
+        self,
+        q_pend: queue.Queue[tuple[int, str, list[Any]]],
+        q_yield: queue.Queue[tuple[int, str, list[Any]]],
+        args: argparse.Namespace,
+        n: int,
+    ) -> None:
+        super(MpWorker, self).__init__()
+
         self.q_pend = q_pend
         self.q_yield = q_yield
         self.args = args
@@ -22,20 +42,25 @@ class MpWorker(object):
 
         self.log = self._log_disabled if args.q and not args.lo else self._log_enabled
 
-        self.retpend = {}
+        self.retpend: dict[int, Any] = {}
         self.retpend_mutex = threading.Lock()
         self.mutex = threading.Lock()
 
         # we inherited signal_handler from parent,
         # replace it with something harmless
         if not FAKE_MP:
-            for sig in [signal.SIGINT, signal.SIGTERM, signal.SIGUSR1]:
+            sigs = [signal.SIGINT, signal.SIGTERM]
+            if not ANYWIN:
+                sigs.append(signal.SIGUSR1)
+
+            for sig in sigs:
                 signal.signal(sig, self.signal_handler)
 
         # starting to look like a good idea
         self.asrv = AuthSrv(args, None, False)
 
         # instantiate all services here (TODO: inheritance?)
+        self.iphash = HMaccas(os.path.join(self.args.E.cfg, "iphash"), 8)
         self.httpsrv = HttpSrv(self, n)
 
         # on winxp and some other platforms,
@@ -45,20 +70,20 @@ class MpWorker(object):
         thr.start()
         thr.join()
 
-    def signal_handler(self, sig, frame):
+    def signal_handler(self, sig: Optional[int], frame: Optional[FrameType]) -> None:
         # print('k')
         pass
 
-    def _log_enabled(self, src, msg, c=0):
-        self.q_yield.put([0, "log", [src, msg, c]])
+    def _log_enabled(self, src: str, msg: str, c: Union[int, str] = 0) -> None:
+        self.q_yield.put((0, "log", [src, msg, c]))
 
-    def _log_disabled(self, src, msg, c=0):
+    def _log_disabled(self, src: str, msg: str, c: Union[int, str] = 0) -> None:
         pass
 
-    def logw(self, msg, c=0):
+    def logw(self, msg: str, c: Union[int, str] = 0) -> None:
         self.log("mp{}".format(self.n), msg, c)
 
-    def main(self):
+    def main(self) -> None:
         while True:
             retq_id, dest, args = self.q_pend.get()
 
@@ -87,15 +112,14 @@ class MpWorker(object):
             else:
                 raise Exception("what is " + str(dest))
 
-    def put(self, want_retval, dest, *args):
-        if want_retval:
-            retq = ExceptionalQueue(1)
-            retq_id = id(retq)
-            with self.retpend_mutex:
-                self.retpend[retq_id] = retq
-        else:
-            retq = None
-            retq_id = 0
+    def ask(self, dest: str, *args: Any) -> ExceptionalQueue:
+        retq = ExceptionalQueue(1)
+        retq_id = id(retq)
+        with self.retpend_mutex:
+            self.retpend[retq_id] = retq
 
-        self.q_yield.put([retq_id, dest, args])
+        self.q_yield.put((retq_id, dest, list(args)))
         return retq
+
+    def say(self, dest: str, *args: Any) -> None:
+        self.q_yield.put((0, dest, list(args)))

copyparty/broker_thr.py

@@ -1,16 +1,29 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
+import os
 import threading
 
+from .__init__ import TYPE_CHECKING
+from .broker_util import BrokerCli, ExceptionalQueue, try_exec
 from .httpsrv import HttpSrv
-from .broker_util import ExceptionalQueue, try_exec
+from .util import HMaccas
+
+if TYPE_CHECKING:
+    from .svchub import SvcHub
+
+try:
+    from typing import Any
+except:
+    pass
 
 
-class BrokerThr(object):
+class BrokerThr(BrokerCli):
     """external api; behaves like BrokerMP but using plain threads"""
 
-    def __init__(self, hub):
+    def __init__(self, hub: "SvcHub") -> None:
+        super(BrokerThr, self).__init__()
+
         self.hub = hub
         self.log = hub.log
         self.args = hub.args
@@ -20,32 +33,39 @@ class BrokerThr(object):
         self.num_workers = 1
 
         # instantiate all services here (TODO: inheritance?)
+        self.iphash = HMaccas(os.path.join(self.args.E.cfg, "iphash"), 8)
         self.httpsrv = HttpSrv(self, None)
         self.reload = self.noop
 
-    def shutdown(self):
+    def shutdown(self) -> None:
         # self.log("broker", "shutting down")
         self.httpsrv.shutdown()
 
-    def noop(self):
+    def noop(self) -> None:
         pass
 
-    def put(self, want_retval, dest, *args):
+    def ask(self, dest: str, *args: Any) -> ExceptionalQueue:
+
+        # new ipc invoking managed service in hub
+        obj = self.hub
+        for node in dest.split("."):
+            obj = getattr(obj, node)
+
+        rv = try_exec(True, obj, *args)
+
+        # pretend we're broker_mp
+        retq = ExceptionalQueue(1)
+        retq.put(rv)
+        return retq
+
+    def say(self, dest: str, *args: Any) -> None:
         if dest == "listen":
             self.httpsrv.listen(args[0], 1)
+            return
 
-        else:
-            # new ipc invoking managed service in hub
-            obj = self.hub
-            for node in dest.split("."):
-                obj = getattr(obj, node)
+        # new ipc invoking managed service in hub
+        obj = self.hub
+        for node in dest.split("."):
+            obj = getattr(obj, node)
 
-            # TODO will deadlock if dest performs another ipc
-            rv = try_exec(want_retval, obj, *args)
-            if not want_retval:
-                return
-
-            # pretend we're broker_mp
-            retq = ExceptionalQueue(1)
-            retq.put(rv)
-            return retq
+        try_exec(False, obj, *args)

copyparty/broker_util.py

@@ -1,17 +1,30 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
-
+import argparse
 import traceback
 
-from .util import Pebkac, Queue
+from queue import Queue
+
+from .__init__ import TYPE_CHECKING
+from .authsrv import AuthSrv
+from .util import HMaccas, Pebkac
+
+try:
+    from typing import Any, Optional, Union
+
+    from .util import RootLogger
+except:
+    pass
+
+if TYPE_CHECKING:
+    from .httpsrv import HttpSrv
 
 
 class ExceptionalQueue(Queue, object):
-    def get(self, block=True, timeout=None):
+    def get(self, block: bool = True, timeout: Optional[float] = None) -> Any:
         rv = super(ExceptionalQueue, self).get(block, timeout)
 
-        # TODO: how expensive is this?
         if isinstance(rv, list):
             if rv[0] == "exception":
                 if rv[1] == "pebkac":
@@ -22,7 +35,27 @@ class ExceptionalQueue(Queue, object):
         return rv
 
 
-def try_exec(want_retval, func, *args):
+class BrokerCli(object):
+    """
+    helps mypy understand httpsrv.broker but still fails a few levels deeper,
+    for example resolving httpconn.* in httpcli -- see lines tagged #mypy404
+    """
+
+    def __init__(self) -> None:
+        self.log: "RootLogger" = None
+        self.args: argparse.Namespace = None
+        self.asrv: AuthSrv = None
+        self.httpsrv: "HttpSrv" = None
+        self.iphash: HMaccas = None
+
+    def ask(self, dest: str, *args: Any) -> ExceptionalQueue:
+        return ExceptionalQueue(1)
+
+    def say(self, dest: str, *args: Any) -> None:
+        pass
+
+
+def try_exec(want_retval: Union[bool, int], func: Any, *args: list[Any]) -> Any:
     try:
         return func(*args)
 

copyparty/fsutil.py

@@ -0,0 +1,154 @@
+# coding: utf-8
+from __future__ import print_function, unicode_literals
+
+import os
+import re
+import time
+
+from .__init__ import ANYWIN, MACOS
+from .authsrv import AXS, VFS
+from .bos import bos
+from .util import chkcmd, min_ex
+
+try:
+    from typing import Optional, Union
+
+    from .util import RootLogger
+except:
+    pass
+
+
+class Fstab(object):
+    def __init__(self, log: "RootLogger"):
+        self.log_func = log
+
+        self.trusted = False
+        self.tab: Optional[VFS] = None
+        self.cache: dict[str, str] = {}
+        self.age = 0.0
+
+    def log(self, msg: str, c: Union[int, str] = 0) -> None:
+        self.log_func("fstab", msg + "\033[K", c)
+
+    def get(self, path: str) -> str:
+        if len(self.cache) > 9000:
+            self.age = time.time()
+            self.tab = None
+            self.cache = {}
+
+        fs = "ext4"
+        msg = "failed to determine filesystem at [{}]; assuming {}\n{}"
+
+        if ANYWIN:
+            fs = "vfat"
+            try:
+                path = self._winpath(path)
+            except:
+                self.log(msg.format(path, fs, min_ex()), 3)
+                return fs
+
+        path = path.lstrip("/")
+        try:
+            return self.cache[path]
+        except:
+            pass
+
+        try:
+            fs = self.get_w32(path) if ANYWIN else self.get_unix(path)
+        except:
+            self.log(msg.format(path, fs, min_ex()), 3)
+
+        fs = fs.lower()
+        self.cache[path] = fs
+        self.log("found {} at {}".format(fs, path))
+        return fs
+
+    def _winpath(self, path: str) -> str:
+        # try to combine volume-label + st_dev (vsn)
+        path = path.replace("/", "\\")
+        vid = path.split(":", 1)[0].strip("\\").split("\\", 1)[0]
+        try:
+            return "{}*{}".format(vid, bos.stat(path).st_dev)
+        except:
+            return vid
+
+    def build_fallback(self) -> None:
+        self.tab = VFS(self.log_func, "idk", "/", AXS(), {})
+        self.trusted = False
+
+    def build_tab(self) -> None:
+        self.log("building tab")
+
+        sptn = r"^.*? on (.*) type ([^ ]+) \(.*"
+        if MACOS:
+            sptn = r"^.*? on (.*) \(([^ ]+), .*"
+
+        ptn = re.compile(sptn)
+        so, _ = chkcmd(["mount"])
+        tab1: list[tuple[str, str]] = []
+        for ln in so.split("\n"):
+            m = ptn.match(ln)
+            if not m:
+                continue
+
+            zs1, zs2 = m.groups()
+            tab1.append((str(zs1), str(zs2)))
+
+        tab1.sort(key=lambda x: (len(x[0]), x[0]))
+        path1, fs1 = tab1[0]
+        tab = VFS(self.log_func, fs1, path1, AXS(), {})
+        for path, fs in tab1[1:]:
+            tab.add(fs, path.lstrip("/"))
+
+        self.tab = tab
+
+    def relabel(self, path: str, nval: str) -> None:
+        assert self.tab
+        self.cache = {}
+        if ANYWIN:
+            path = self._winpath(path)
+
+        path = path.lstrip("/")
+        ptn = re.compile(r"^[^\\/]*")
+        vn, rem = self.tab._find(path)
+        if not self.trusted:
+            # no mtab access; have to build as we go
+            if "/" in rem:
+                self.tab.add("idk", os.path.join(vn.vpath, rem.split("/")[0]))
+            if rem:
+                self.tab.add(nval, path)
+            else:
+                vn.realpath = nval
+
+            return
+
+        visit = [vn]
+        while visit:
+            vn = visit.pop()
+            vn.realpath = ptn.sub(nval, vn.realpath)
+            visit.extend(list(vn.nodes.values()))
+
+    def get_unix(self, path: str) -> str:
+        if not self.tab:
+            try:
+                self.build_tab()
+                self.trusted = True
+            except:
+                # prisonparty or other restrictive environment
+                self.log("failed to build tab:\n{}".format(min_ex()), 3)
+                self.build_fallback()
+
+        assert self.tab
+        ret = self.tab._find(path)[0]
+        if self.trusted or path == ret.vpath:
+            return ret.realpath.split("/")[0]
+        else:
+            return "idk"
+
+    def get_w32(self, path: str) -> str:
+        if not self.tab:
+            self.build_fallback()
+
+        assert self.tab
+        ret = self.tab._find(path)[0]
+        return ret.realpath

copyparty/ftpd.py

@@ -1,16 +1,23 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
-import os
-import sys
-import stat
-import time
+import argparse
 import logging
+import os
+import stat
+import sys
 import threading
+import time
 
-from .__init__ import E, PY2
-from .util import Pebkac, fsenc, exclude_dotfiles
+from pyftpdlib.authorizers import AuthenticationFailed, DummyAuthorizer
+from pyftpdlib.filesystems import AbstractedFS, FilesystemError
+from pyftpdlib.handlers import FTPHandler
+from pyftpdlib.log import config_logging
+from pyftpdlib.servers import FTPServer
+
+from .__init__ import PY2, TYPE_CHECKING, E
 from .bos import bos
+from .util import Pebkac, exclude_dotfiles, fsenc
 
 try:
     from pyftpdlib.ioloop import IOLoop
@@ -20,65 +27,66 @@ except ImportError:
     sys.path.append(p)
     from pyftpdlib.ioloop import IOLoop
 
-from pyftpdlib.authorizers import DummyAuthorizer, AuthenticationFailed
-from pyftpdlib.filesystems import AbstractedFS, FilesystemError
-from pyftpdlib.handlers import FTPHandler
-from pyftpdlib.servers import FTPServer
-from pyftpdlib.log import config_logging
 
+if TYPE_CHECKING:
+    from .svchub import SvcHub
 
 try:
-    from typing import TYPE_CHECKING
-
-    if TYPE_CHECKING:
-        from .svchub import SvcHub
-except ImportError:
+    import typing
+    from typing import Any, Optional
+except:
     pass
 
 
 class FtpAuth(DummyAuthorizer):
-    def __init__(self):
+    def __init__(self, hub: "SvcHub") -> None:
         super(FtpAuth, self).__init__()
-        self.hub = None  # type: SvcHub
+        self.hub = hub
 
-    def validate_authentication(self, username, password, handler):
+    def validate_authentication(
+        self, username: str, password: str, handler: Any
+    ) -> None:
         asrv = self.hub.asrv
         if username == "anonymous":
             password = ""
 
         uname = "*"
         if password:
-            uname = asrv.iacct.get(password, None)
+            uname = asrv.iacct.get(password, "")
 
         handler.username = uname
 
-        if password and not uname:
+        if (password and not uname) or not (
+            asrv.vfs.aread.get(uname) or asrv.vfs.awrite.get(uname)
+        ):
             raise AuthenticationFailed("Authentication failed.")
 
-    def get_home_dir(self, username):
+    def get_home_dir(self, username: str) -> str:
         return "/"
 
-    def has_user(self, username):
+    def has_user(self, username: str) -> bool:
         asrv = self.hub.asrv
         return username in asrv.acct
 
-    def has_perm(self, username, perm, path=None):
+    def has_perm(self, username: str, perm: int, path: Optional[str] = None) -> bool:
         return True  # handled at filesystem layer
 
-    def get_perms(self, username):
+    def get_perms(self, username: str) -> str:
         return "elradfmwMT"
 
-    def get_msg_login(self, username):
+    def get_msg_login(self, username: str) -> str:
         return "sup {}".format(username)
 
-    def get_msg_quit(self, username):
+    def get_msg_quit(self, username: str) -> str:
         return "cya"
 
 
 class FtpFs(AbstractedFS):
-    def __init__(self, root, cmd_channel):
+    def __init__(
+        self, root: str, cmd_channel: Any
+    ) -> None:  # pylint: disable=super-init-not-called
         self.h = self.cmd_channel = cmd_channel  # type: FTPHandler
-        self.hub = cmd_channel.hub  # type: SvcHub
+        self.hub: "SvcHub" = cmd_channel.hub
         self.args = cmd_channel.args
 
         self.uname = self.hub.asrv.iacct.get(cmd_channel.password, "*")
@@ -86,10 +94,20 @@ class FtpFs(AbstractedFS):
         self.cwd = "/"  # pyftpdlib convention of leading slash
         self.root = "/var/lib/empty"
 
+        self.can_read = self.can_write = self.can_move = False
+        self.can_delete = self.can_get = self.can_upget = False
+
         self.listdirinfo = self.listdir
         self.chdir(".")
 
-    def v2a(self, vpath, r=False, w=False, m=False, d=False):
+    def v2a(
+        self,
+        vpath: str,
+        r: bool = False,
+        w: bool = False,
+        m: bool = False,
+        d: bool = False,
+    ) -> str:
         try:
             vpath = vpath.replace("\\", "/").lstrip("/")
             vfs, rem = self.hub.asrv.vfs.get(vpath, self.uname, r, w, m, d)
@@ -100,25 +118,32 @@ class FtpFs(AbstractedFS):
         except Pebkac as ex:
             raise FilesystemError(str(ex))
 
-    def rv2a(self, vpath, r=False, w=False, m=False, d=False):
+    def rv2a(
+        self,
+        vpath: str,
+        r: bool = False,
+        w: bool = False,
+        m: bool = False,
+        d: bool = False,
+    ) -> str:
         return self.v2a(os.path.join(self.cwd, vpath), r, w, m, d)
 
-    def ftp2fs(self, ftppath):
+    def ftp2fs(self, ftppath: str) -> str:
         # return self.v2a(ftppath)
         return ftppath  # self.cwd must be vpath
 
-    def fs2ftp(self, fspath):
+    def fs2ftp(self, fspath: str) -> str:
         # raise NotImplementedError()
         return fspath
 
-    def validpath(self, path):
+    def validpath(self, path: str) -> bool:
         if "/.hist/" in path:
             if "/up2k." in path or path.endswith("/dir.txt"):
                 raise FilesystemError("access to this file is forbidden")
 
         return True
 
-    def open(self, filename, mode):
+    def open(self, filename: str, mode: str) -> typing.IO[Any]:
         r = "r" in mode
         w = "w" in mode or "a" in mode or "+" in mode
 
@@ -129,24 +154,30 @@ class FtpFs(AbstractedFS):
         self.validpath(ap)
         return open(fsenc(ap), mode)
 
-    def chdir(self, path):
+    def chdir(self, path: str) -> None:
         self.cwd = join(self.cwd, path)
-        x = self.hub.asrv.vfs.can_access(self.cwd.lstrip("/"), self.h.username)
-        self.can_read, self.can_write, self.can_move, self.can_delete, self.can_get = x
+        (
+            self.can_read,
+            self.can_write,
+            self.can_move,
+            self.can_delete,
+            self.can_get,
+            self.can_upget,
+        ) = self.hub.asrv.vfs.can_access(self.cwd.lstrip("/"), self.h.username)
 
-    def mkdir(self, path):
+    def mkdir(self, path: str) -> None:
         ap = self.rv2a(path, w=True)
         bos.mkdir(ap)
 
-    def listdir(self, path):
+    def listdir(self, path: str) -> list[str]:
         vpath = join(self.cwd, path).lstrip("/")
         try:
             vfs, rem = self.hub.asrv.vfs.get(vpath, self.uname, True, False)
 
-            fsroot, vfs_ls, vfs_virt = vfs.ls(
+            fsroot, vfs_ls1, vfs_virt = vfs.ls(
                 rem, self.uname, not self.args.no_scandir, [[True], [False, True]]
             )
-            vfs_ls = [x[0] for x in vfs_ls]
+            vfs_ls = [x[0] for x in vfs_ls1]
             vfs_ls.extend(vfs_virt.keys())
 
             if not self.args.ed:
@@ -154,7 +185,7 @@ class FtpFs(AbstractedFS):
 
             vfs_ls.sort()
             return vfs_ls
-        except Exception as ex:
+        except:
             if vpath:
                 # display write-only folders as empty
                 return []
@@ -163,44 +194,39 @@ class FtpFs(AbstractedFS):
             r = {x.split("/")[0]: 1 for x in self.hub.asrv.vfs.all_vols.keys()}
             return list(sorted(list(r.keys())))
 
-    def rmdir(self, path):
+    def rmdir(self, path: str) -> None:
         ap = self.rv2a(path, d=True)
         bos.rmdir(ap)
 
-    def remove(self, path):
+    def remove(self, path: str) -> None:
         if self.args.no_del:
             raise FilesystemError("the delete feature is disabled in server config")
 
         vp = join(self.cwd, path).lstrip("/")
-        x = self.hub.broker.put(
-            True, "up2k.handle_rm", self.uname, self.h.remote_ip, [vp]
-        )
-
         try:
-            x.get()
+            self.hub.up2k.handle_rm(self.uname, self.h.remote_ip, [vp], [])
         except Exception as ex:
             raise FilesystemError(str(ex))
 
-    def rename(self, src, dst):
+    def rename(self, src: str, dst: str) -> None:
         if not self.can_move:
             raise FilesystemError("not allowed for user " + self.h.username)
 
         if self.args.no_mv:
-            m = "the rename/move feature is disabled in server config"
-            raise FilesystemError(m)
+            t = "the rename/move feature is disabled in server config"
+            raise FilesystemError(t)
 
         svp = join(self.cwd, src).lstrip("/")
         dvp = join(self.cwd, dst).lstrip("/")
-        x = self.hub.broker.put(True, "up2k.handle_mv", self.uname, svp, dvp)
         try:
-            x.get()
+            self.hub.up2k.handle_mv(self.uname, svp, dvp)
         except Exception as ex:
             raise FilesystemError(str(ex))
 
-    def chmod(self, path, mode):
+    def chmod(self, path: str, mode: str) -> None:
         pass
 
-    def stat(self, path):
+    def stat(self, path: str) -> os.stat_result:
         try:
             ap = self.rv2a(path, r=True)
             return bos.stat(ap)
@@ -212,64 +238,70 @@ class FtpFs(AbstractedFS):
 
             return st
 
-    def utime(self, path, timeval):
+    def utime(self, path: str, timeval: float) -> None:
         ap = self.rv2a(path, w=True)
         return bos.utime(ap, (timeval, timeval))
 
-    def lstat(self, path):
+    def lstat(self, path: str) -> os.stat_result:
         ap = self.rv2a(path)
         return bos.lstat(ap)
 
-    def isfile(self, path):
+    def isfile(self, path: str) -> bool:
         st = self.stat(path)
         return stat.S_ISREG(st.st_mode)
 
-    def islink(self, path):
+    def islink(self, path: str) -> bool:
         ap = self.rv2a(path)
         return bos.path.islink(ap)
 
-    def isdir(self, path):
+    def isdir(self, path: str) -> bool:
         try:
             st = self.stat(path)
             return stat.S_ISDIR(st.st_mode)
         except:
             return True
 
-    def getsize(self, path):
+    def getsize(self, path: str) -> int:
         ap = self.rv2a(path)
         return bos.path.getsize(ap)
 
-    def getmtime(self, path):
+    def getmtime(self, path: str) -> float:
         ap = self.rv2a(path)
         return bos.path.getmtime(ap)
 
-    def realpath(self, path):
+    def realpath(self, path: str) -> str:
         return path
 
-    def lexists(self, path):
+    def lexists(self, path: str) -> bool:
         ap = self.rv2a(path)
         return bos.path.lexists(ap)
 
-    def get_user_by_uid(self, uid):
+    def get_user_by_uid(self, uid: int) -> str:
         return "root"
 
-    def get_group_by_uid(self, gid):
+    def get_group_by_uid(self, gid: int) -> str:
         return "root"
 
 
 class FtpHandler(FTPHandler):
     abstracted_fs = FtpFs
+    hub: "SvcHub" = None
+    args: argparse.Namespace = None
+
+    def __init__(self, conn: Any, server: Any, ioloop: Any = None) -> None:
+        self.hub: "SvcHub" = FtpHandler.hub
+        self.args: argparse.Namespace = FtpHandler.args
 
-    def __init__(self, conn, server, ioloop=None):
         if PY2:
             FTPHandler.__init__(self, conn, server, ioloop)
         else:
             super(FtpHandler, self).__init__(conn, server, ioloop)
 
         # abspath->vpath mapping to resolve log_transfer paths
-        self.vfs_map = {}
+        self.vfs_map: dict[str, str] = {}
 
-    def ftp_STOR(self, file, mode="w"):
+    def ftp_STOR(self, file: str, mode: str = "w") -> Any:
+        # Optional[str]
         vp = join(self.fs.cwd, file).lstrip("/")
         ap = self.fs.v2a(vp)
         self.vfs_map[ap] = vp
@@ -278,7 +310,16 @@ class FtpHandler(FTPHandler):
         # print("ftp_STOR: {} {} OK".format(vp, mode))
         return ret
 
-    def log_transfer(self, cmd, filename, receive, completed, elapsed, bytes):
+    def log_transfer(
+        self,
+        cmd: str,
+        filename: bytes,
+        receive: bool,
+        completed: bool,
+        elapsed: float,
+        bytes: int,
+    ) -> Any:
+        # None
         ap = filename.decode("utf-8", "replace")
         vp = self.vfs_map.pop(ap, None)
         # print("xfer_end: {} => {}".format(ap, vp))
@@ -286,9 +327,7 @@ class FtpHandler(FTPHandler):
             vp, fn = os.path.split(vp)
             vfs, rem = self.hub.asrv.vfs.get(vp, self.username, False, True)
             vfs, rem = vfs.get_dbv(rem)
-            self.hub.broker.put(
-                False,
-                "up2k.hash_file",
+            self.hub.up2k.hash_file(
                 vfs.realpath,
                 vfs.flags,
                 rem,
@@ -313,7 +352,7 @@ except:
 
 
 class Ftpd(object):
-    def __init__(self, hub):
+    def __init__(self, hub: "SvcHub") -> None:
         self.hub = hub
         self.args = hub.args
 
@@ -322,24 +361,23 @@ class Ftpd(object):
             hs.append([FtpHandler, self.args.ftp])
         if self.args.ftps:
             try:
-                h = SftpHandler
+                h1 = SftpHandler
             except:
-                m = "\nftps requires pyopenssl;\nplease run the following:\n\n  {} -m pip install --user pyopenssl\n"
-                print(m.format(sys.executable))
+                t = "\nftps requires pyopenssl;\nplease run the following:\n\n  {} -m pip install --user pyopenssl\n"
+                print(t.format(sys.executable))
                 sys.exit(1)
 
-            h.certfile = os.path.join(E.cfg, "cert.pem")
-            h.tls_control_required = True
-            h.tls_data_required = True
+            h1.certfile = os.path.join(self.args.E.cfg, "cert.pem")
+            h1.tls_control_required = True
+            h1.tls_data_required = True
 
-            hs.append([h, self.args.ftps])
+            hs.append([h1, self.args.ftps])
 
-        for h in hs:
-            h, lp = h
-            h.hub = hub
-            h.args = hub.args
-            h.authorizer = FtpAuth()
-            h.authorizer.hub = hub
+        for h_lp in hs:
+            h2, lp = h_lp
+            h2.hub = hub
+            h2.args = hub.args
+            h2.authorizer = FtpAuth(hub)
 
             if self.args.ftp_pr:
                 p1, p2 = [int(x) for x in self.args.ftp_pr.split("-")]
@@ -351,10 +389,10 @@ class Ftpd(object):
                     else:
                         p1 += d + 1
 
-                h.passive_ports = list(range(p1, p2 + 1))
+                h2.passive_ports = list(range(p1, p2 + 1))
 
             if self.args.ftp_nat:
-                h.masquerade_address = self.args.ftp_nat
+                h2.masquerade_address = self.args.ftp_nat
 
         if self.args.ftp_dbg:
             config_logging(level=logging.DEBUG)
@@ -364,11 +402,11 @@ class Ftpd(object):
             for h, lp in hs:
                 FTPServer((ip, int(lp)), h, ioloop)
 
-        t = threading.Thread(target=ioloop.loop)
-        t.daemon = True
-        t.start()
+        thr = threading.Thread(target=ioloop.loop, name="ftp")
+        thr.daemon = True
+        thr.start()
 
 
-def join(p1, p2):
+def join(p1: str, p2: str) -> str:
     w = os.path.join(p1, p2.replace("\\", "/"))
     return os.path.normpath(w).replace("\\", "/")

copyparty/httpconn.py

@@ -1,24 +1,37 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
-import re
+import argparse  # typechk
 import os
-import time
+import re
 import socket
+import threading  # typechk
+import time
 
-HAVE_SSL = True
 try:
+    HAVE_SSL = True
     import ssl
 except:
     HAVE_SSL = False
 
-from .__init__ import E
-from .util import Unrecv
+from . import util as Util
+from .__init__ import TYPE_CHECKING, EnvParams
+from .authsrv import AuthSrv  # typechk
 from .httpcli import HttpCli
-from .u2idx import U2idx
-from .th_cli import ThumbCli
-from .th_srv import HAVE_PIL
 from .ico import Ico
+from .mtag import HAVE_FFMPEG
+from .th_cli import ThumbCli
+from .th_srv import HAVE_PIL, HAVE_VIPS
+from .u2idx import U2idx
+from .util import HMaccas, shut_socket
+
+try:
+    from typing import Optional, Pattern, Union
+except:
+    pass
+
+if TYPE_CHECKING:
+    from .httpsrv import HttpSrv
 
 
 class HttpConn(object):
@@ -27,39 +40,46 @@ class HttpConn(object):
     creates an HttpCli for each request (Connection: Keep-Alive)
     """
 
-    def __init__(self, sck, addr, hsrv):
+    def __init__(
+        self, sck: socket.socket, addr: tuple[str, int], hsrv: "HttpSrv"
+    ) -> None:
         self.s = sck
+        self.sr: Optional[Util._Unrecv] = None
         self.addr = addr
         self.hsrv = hsrv
 
-        self.mutex = hsrv.mutex
-        self.args = hsrv.args
-        self.asrv = hsrv.asrv
+        self.mutex: threading.Lock = hsrv.mutex  # mypy404
+        self.args: argparse.Namespace = hsrv.args  # mypy404
+        self.E: EnvParams = self.args.E
+        self.asrv: AuthSrv = hsrv.asrv  # mypy404
         self.cert_path = hsrv.cert_path
-        self.u2fh = hsrv.u2fh
+        self.u2fh: Util.FHC = hsrv.u2fh  # mypy404
+        self.iphash: HMaccas = hsrv.broker.iphash
 
-        enth = HAVE_PIL and not self.args.no_thumb
-        self.thumbcli = ThumbCli(hsrv) if enth else None
-        self.ico = Ico(self.args)
+        enth = (HAVE_PIL or HAVE_VIPS or HAVE_FFMPEG) and not self.args.no_thumb
+        self.thumbcli: Optional[ThumbCli] = ThumbCli(hsrv) if enth else None  # mypy404
+        self.ico: Ico = Ico(self.args)  # mypy404
 
-        self.t0 = time.time()
+        self.t0: float = time.time()  # mypy404
         self.stopping = False
-        self.nreq = 0
-        self.nbyte = 0
-        self.u2idx = None
-        self.log_func = hsrv.log
-        self.lf_url = re.compile(self.args.lf_url) if self.args.lf_url else None
+        self.nreq: int = 0  # mypy404
+        self.nbyte: int = 0  # mypy404
+        self.u2idx: Optional[U2idx] = None
+        self.log_func: "Util.RootLogger" = hsrv.log  # mypy404
+        self.log_src: str = "httpconn"  # mypy404
+        self.lf_url: Optional[Pattern[str]] = (
+            re.compile(self.args.lf_url) if self.args.lf_url else None
+        )  # mypy404
         self.set_rproxy()
 
-    def shutdown(self):
+    def shutdown(self) -> None:
         self.stopping = True
         try:
-            self.s.shutdown(socket.SHUT_RDWR)
-            self.s.close()
+            shut_socket(self.log, self.s, 1)
         except:
             pass
 
-    def set_rproxy(self, ip=None):
+    def set_rproxy(self, ip: Optional[str] = None) -> str:
         if ip is None:
             color = 36
             ip = self.addr[0]
@@ -72,35 +92,37 @@ class HttpConn(object):
         self.log_src = "{} \033[{}m{}".format(ip, color, self.addr[1]).ljust(26)
         return self.log_src
 
-    def respath(self, res_name):
-        return os.path.join(E.mod, "web", res_name)
+    def respath(self, res_name: str) -> str:
+        return os.path.join(self.E.mod, "web", res_name)
 
-    def log(self, msg, c=0):
+    def log(self, msg: str, c: Union[int, str] = 0) -> None:
         self.log_func(self.log_src, msg, c)
 
-    def get_u2idx(self):
+    def get_u2idx(self) -> U2idx:
+        # one u2idx per tcp connection;
+        # sqlite3 fully parallelizes under python threads
         if not self.u2idx:
             self.u2idx = U2idx(self)
 
         return self.u2idx
 
-    def _detect_https(self):
+    def _detect_https(self) -> bool:
         method = None
         if self.cert_path:
             try:
                 method = self.s.recv(4, socket.MSG_PEEK)
             except socket.timeout:
-                return
+                return False
             except AttributeError:
                 # jython does not support msg_peek; forget about https
                 method = self.s.recv(4)
-                self.sr = Unrecv(self.s)
+                self.sr = Util.Unrecv(self.s, self.log)
                 self.sr.buf = method
 
                 # jython used to do this, they stopped since it's broken
                 # but reimplementing sendall is out of scope for now
                 if not getattr(self.s, "sendall", None):
-                    self.s.sendall = self.s.send
+                    self.s.sendall = self.s.send  # type: ignore
 
             if len(method) != 4:
                 err = "need at least 4 bytes in the first packet; got {}".format(
@@ -110,17 +132,18 @@ class HttpConn(object):
                     self.log(err)
 
                 self.s.send(b"HTTP/1.1 400 Bad Request\r\n\r\n" + err.encode("utf-8"))
-                return
+                return False
 
         return method not in [None, b"GET ", b"HEAD", b"POST", b"PUT ", b"OPTI"]
 
-    def run(self):
+    def run(self) -> None:
         self.sr = None
         if self.args.https_only:
             is_https = True
         elif self.args.http_only or not HAVE_SSL:
             is_https = False
         else:
+            # raise Exception("asdf")
             is_https = self._detect_https()
 
         if is_https:
@@ -149,14 +172,15 @@ class HttpConn(object):
                 self.s = ctx.wrap_socket(self.s, server_side=True)
                 msg = [
                     "\033[1;3{:d}m{}".format(c, s)
-                    for c, s in zip([0, 5, 0], self.s.cipher())
+                    for c, s in zip([0, 5, 0], self.s.cipher())  # type: ignore
                 ]
                 self.log(" ".join(msg) + "\033[0m")
 
                 if self.args.ssl_dbg and hasattr(self.s, "shared_ciphers"):
-                    overlap = [y[::-1] for y in self.s.shared_ciphers()]
-                    lines = [str(x) for x in (["TLS cipher overlap:"] + overlap)]
-                    self.log("\n".join(lines))
+                    ciphers = self.s.shared_ciphers()
+                    assert ciphers
+                    overlap = [str(y[::-1]) for y in ciphers]
+                    self.log("TLS cipher overlap:" + "\n".join(overlap))
                     for k, v in [
                         ["compression", self.s.compression()],
                         ["ALPN proto", self.s.selected_alpn_protocol()],
@@ -167,11 +191,7 @@ class HttpConn(object):
             except Exception as ex:
                 em = str(ex)
 
-                if "ALERT_BAD_CERTIFICATE" in em:
-                    # firefox-linux if there is no exception yet
-                    self.log("client rejected our certificate (nice)")
-
-                elif "ALERT_CERTIFICATE_UNKNOWN" in em:
+                if "ALERT_CERTIFICATE_UNKNOWN" in em:
                     # android-chrome keeps doing this
                     pass
 
@@ -181,7 +201,7 @@ class HttpConn(object):
                 return
 
         if not self.sr:
-            self.sr = Unrecv(self.s)
+            self.sr = Util.Unrecv(self.s, self.log)
 
         while not self.stopping:
             self.nreq += 1

copyparty/httpsrv.py

@@ -1,13 +1,15 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
-import os
-import sys
-import time
-import math
 import base64
+import math
+import os
 import socket
+import sys
 import threading
+import time
+
+import queue
 
 try:
     import jinja2
@@ -26,15 +28,27 @@ except ImportError:
     )
     sys.exit(1)
 
-from .__init__ import E, PY2, MACOS
-from .util import FHC, spack, min_ex, start_stackmon, start_log_thrs
+from .__init__ import MACOS, TYPE_CHECKING, EnvParams
 from .bos import bos
 from .httpconn import HttpConn
+from .util import (
+    FHC,
+    Garda,
+    Magician,
+    min_ex,
+    shut_socket,
+    spack,
+    start_log_thrs,
+    start_stackmon,
+)
 
-if PY2:
-    import Queue as queue
-else:
-    import queue
+if TYPE_CHECKING:
+    from .broker_util import BrokerCli
+
+try:
+    from typing import Any, Optional
+except:
+    pass
 
 
 class HttpSrv(object):
@@ -43,14 +57,22 @@ class HttpSrv(object):
     relying on MpSrv for performance (HttpSrv is just plain threads)
     """
 
-    def __init__(self, broker, nid):
+    def __init__(self, broker: "BrokerCli", nid: Optional[int]) -> None:
         self.broker = broker
         self.nid = nid
         self.args = broker.args
+        self.E: EnvParams = self.args.E
         self.log = broker.log
         self.asrv = broker.asrv
 
+        # redefine in case of multiprocessing
+        socket.setdefaulttimeout(120)
+
         nsuf = "-n{}-i{:x}".format(nid, os.getpid()) if nid else ""
+        self.magician = Magician()
+        self.bans: dict[str, int] = {}
+        self.gpwd = Garda(self.args.ban_pw)
+        self.g404 = Garda(self.args.ban_404)
 
         self.name = "hsrv" + nsuf
         self.mutex = threading.Lock()
@@ -58,31 +80,34 @@ class HttpSrv(object):
 
         self.tp_nthr = 0  # actual
         self.tp_ncli = 0  # fading
-        self.tp_time = None  # latest worker collect
-        self.tp_q = None if self.args.no_htp else queue.LifoQueue()
-        self.t_periodic = None
+        self.tp_time = 0.0  # latest worker collect
+        self.tp_q: Optional[queue.LifoQueue[Any]] = (
+            None if self.args.no_htp else queue.LifoQueue()
+        )
+        self.t_periodic: Optional[threading.Thread] = None
 
         self.u2fh = FHC()
-        self.srvs = []
+        self.srvs: list[socket.socket] = []
         self.ncli = 0  # exact
-        self.clients = {}  # laggy
+        self.clients: set[HttpConn] = set()  # laggy
         self.nclimax = 0
-        self.cb_ts = 0
-        self.cb_v = 0
+        self.cb_ts = 0.0
+        self.cb_v = ""
 
         env = jinja2.Environment()
-        env.loader = jinja2.FileSystemLoader(os.path.join(E.mod, "web"))
+        env.loader = jinja2.FileSystemLoader(os.path.join(self.E.mod, "web"))
         self.j2 = {
             x: env.get_template(x + ".html")
-            for x in ["splash", "browser", "browser2", "msg", "md", "mde"]
+            for x in ["splash", "browser", "browser2", "msg", "md", "mde", "cf"]
         }
-        self.prism = os.path.exists(os.path.join(E.mod, "web", "deps", "prism.js.gz"))
+        zs = os.path.join(self.E.mod, "web", "deps", "prism.js.gz")
+        self.prism = os.path.exists(zs)
 
-        cert_path = os.path.join(E.cfg, "cert.pem")
+        cert_path = os.path.join(self.E.cfg, "cert.pem")
         if bos.path.exists(cert_path):
             self.cert_path = cert_path
         else:
-            self.cert_path = None
+            self.cert_path = ""
 
         if self.tp_q:
             self.start_threads(4)
@@ -94,7 +119,19 @@ class HttpSrv(object):
             if self.args.log_thrs:
                 start_log_thrs(self.log, self.args.log_thrs, nid)
 
-    def start_threads(self, n):
+        self.th_cfg: dict[str, Any] = {}
+        t = threading.Thread(target=self.post_init, name="hsrv-init2")
+        t.daemon = True
+        t.start()
+
+    def post_init(self) -> None:
+        try:
+            x = self.broker.ask("thumbsrv.getcfg")
+            self.th_cfg = x.get()
+        except:
+            pass
+
+    def start_threads(self, n: int) -> None:
         self.tp_nthr += n
         if self.args.log_htp:
             self.log(self.name, "workers += {} = {}".format(n, self.tp_nthr), 6)
@@ -107,15 +144,16 @@ class HttpSrv(object):
             thr.daemon = True
             thr.start()
 
-    def stop_threads(self, n):
+    def stop_threads(self, n: int) -> None:
         self.tp_nthr -= n
         if self.args.log_htp:
             self.log(self.name, "workers -= {} = {}".format(n, self.tp_nthr), 6)
 
+        assert self.tp_q
         for _ in range(n):
             self.tp_q.put(None)
 
-    def periodic(self):
+    def periodic(self) -> None:
         while True:
             time.sleep(2 if self.tp_ncli or self.ncli else 10)
             with self.mutex:
@@ -129,7 +167,13 @@ class HttpSrv(object):
                     self.t_periodic = None
                     return
 
-    def listen(self, sck, nlisteners):
+    def listen(self, sck: socket.socket, nlisteners: int) -> None:
+        if self.args.j != 1:
+            # lost in the pickle; redefine
+            sck.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+            sck.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
+            sck.settimeout(None)  # < does not inherit, ^ does
+
         ip, port = sck.getsockname()
         self.srvs.append(sck)
         self.nclimax = math.ceil(self.args.nc * 1.0 / nlisteners)
@@ -141,17 +185,17 @@ class HttpSrv(object):
         t.daemon = True
         t.start()
 
-    def thr_listen(self, srv_sck):
+    def thr_listen(self, srv_sck: socket.socket) -> None:
         """listens on a shared tcp server"""
         ip, port = srv_sck.getsockname()
         fno = srv_sck.fileno()
-        msg = "subscribed @ {}:{}  f{}".format(ip, port, fno)
+        msg = "subscribed @ {}:{}  f{} p{}".format(ip, port, fno, os.getpid())
         self.log(self.name, msg)
 
-        def fun():
-            self.broker.put(False, "cb_httpsrv_up")
+        def fun() -> None:
+            self.broker.say("cb_httpsrv_up")
 
-        threading.Thread(target=fun).start()
+        threading.Thread(target=fun, name="sig-hsrv-up1").start()
 
         while not self.stopping:
             if self.args.log_conn:
@@ -173,21 +217,21 @@ class HttpSrv(object):
                 continue
 
             if self.args.log_conn:
-                m = "|{}C-acc2 \033[0;36m{} \033[3{}m{}".format(
+                t = "|{}C-acc2 \033[0;36m{} \033[3{}m{}".format(
                     "-" * 3, ip, port % 8, port
                 )
-                self.log("%s %s" % addr, m, c="1;30")
+                self.log("%s %s" % addr, t, c="1;30")
 
             self.accept(sck, addr)
 
-    def accept(self, sck, addr):
+    def accept(self, sck: socket.socket, addr: tuple[str, int]) -> None:
         """takes an incoming tcp connection and creates a thread to handle it"""
         now = time.time()
 
         if now - (self.tp_time or now) > 300:
-            m = "httpserver threadpool died: tpt {:.2f}, now {:.2f}, nthr {}, ncli {}"
-            self.log(self.name, m.format(self.tp_time, now, self.tp_nthr, self.ncli), 1)
-            self.tp_time = None
+            t = "httpserver threadpool died: tpt {:.2f}, now {:.2f}, nthr {}, ncli {}"
+            self.log(self.name, t.format(self.tp_time, now, self.tp_nthr, self.ncli), 1)
+            self.tp_time = 0
             self.tp_q = None
 
         with self.mutex:
@@ -197,10 +241,10 @@ class HttpSrv(object):
                 if self.nid:
                     name += "-{}".format(self.nid)
 
-                t = threading.Thread(target=self.periodic, name=name)
-                self.t_periodic = t
-                t.daemon = True
-                t.start()
+                thr = threading.Thread(target=self.periodic, name=name)
+                self.t_periodic = thr
+                thr.daemon = True
+                thr.start()
 
             if self.tp_q:
                 self.tp_time = self.tp_time or now
@@ -212,8 +256,8 @@ class HttpSrv(object):
                 return
 
         if not self.args.no_htp:
-            m = "looks like the httpserver threadpool died; please make an issue on github and tell me the story of how you pulled that off, thanks and dog bless\n"
-            self.log(self.name, m, 1)
+            t = "looks like the httpserver threadpool died; please make an issue on github and tell me the story of how you pulled that off, thanks and dog bless\n"
+            self.log(self.name, t, 1)
 
         thr = threading.Thread(
             target=self.thr_client,
@@ -223,14 +267,15 @@ class HttpSrv(object):
         thr.daemon = True
         thr.start()
 
-    def thr_poolw(self):
+    def thr_poolw(self) -> None:
+        assert self.tp_q
         while True:
             task = self.tp_q.get()
             if not task:
                 break
 
             with self.mutex:
-                self.tp_time = None
+                self.tp_time = 0
 
             try:
                 sck, addr = task
@@ -240,10 +285,13 @@ class HttpSrv(object):
                 )
                 self.thr_client(sck, addr)
                 me.name = self.name + "-poolw"
-            except:
-                self.log(self.name, "thr_client: " + min_ex(), 3)
+            except Exception as ex:
+                if str(ex).startswith("client d/c "):
+                    self.log(self.name, "thr_client: " + str(ex), 6)
+                else:
+                    self.log(self.name, "thr_client: " + min_ex(), 3)
 
-    def shutdown(self):
+    def shutdown(self) -> None:
         self.stopping = True
         for srv in self.srvs:
             try:
@@ -251,12 +299,12 @@ class HttpSrv(object):
             except:
                 pass
 
-        clients = list(self.clients.keys())
+        thrs = []
+        clients = list(self.clients)
         for cli in clients:
-            try:
-                cli.shutdown()
-            except:
-                pass
+            t = threading.Thread(target=cli.shutdown)
+            thrs.append(t)
+            t.start()
 
         if self.tp_q:
             self.stop_threads(self.tp_nthr)
@@ -265,15 +313,16 @@ class HttpSrv(object):
                 if self.tp_q.empty():
                     break
 
+        for t in thrs:
+            t.join()
+
         self.log(self.name, "ok bye")
 
-    def thr_client(self, sck, addr):
+    def thr_client(self, sck: socket.socket, addr: tuple[str, int]) -> None:
         """thread managing one tcp client"""
-        sck.settimeout(120)
-
         cli = HttpConn(sck, addr, self)
         with self.mutex:
-            self.clients[cli] = 0
+            self.clients.add(cli)
 
         fno = sck.fileno()
         try:
@@ -297,8 +346,7 @@ class HttpSrv(object):
 
             try:
                 fno = sck.fileno()
-                sck.shutdown(socket.SHUT_RDWR)
-                sck.close()
+                shut_socket(cli.log, sck)
             except (OSError, socket.error) as ex:
                 if not MACOS:
                     self.log(
@@ -316,10 +364,10 @@ class HttpSrv(object):
                     raise
             finally:
                 with self.mutex:
-                    del self.clients[cli]
+                    self.clients.remove(cli)
                     self.ncli -= 1
 
-    def cachebuster(self):
+    def cachebuster(self) -> str:
         if time.time() - self.cb_ts < 1:
             return self.cb_v
 
@@ -327,9 +375,9 @@ class HttpSrv(object):
             if time.time() - self.cb_ts < 1:
                 return self.cb_v
 
-            v = E.t0
+            v = self.E.t0
             try:
-                with os.scandir(os.path.join(E.mod, "web")) as dh:
+                with os.scandir(os.path.join(self.E.mod, "web")) as dh:
                     for fh in dh:
                         inf = fh.stat()
                         v = max(v, inf.st_mtime)

copyparty/ico.py

@@ -1,33 +1,69 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
-import hashlib
+import argparse  # typechk
 import colorsys
+import hashlib
 
 from .__init__ import PY2
+from .th_srv import HAVE_PIL
+from .util import BytesIO
 
 
 class Ico(object):
-    def __init__(self, args):
+    def __init__(self, args: argparse.Namespace) -> None:
         self.args = args
 
-    def get(self, ext, as_thumb):
+    def get(self, ext: str, as_thumb: bool, chrome: bool) -> tuple[str, bytes]:
         """placeholder to make thumbnails not break"""
 
-        h = hashlib.md5(ext.encode("utf-8")).digest()[:2]
+        zb = hashlib.sha1(ext.encode("utf-8")).digest()[2:4]
         if PY2:
-            h = [ord(x) for x in h]
+            zb = [ord(x) for x in zb]
 
-        c1 = colorsys.hsv_to_rgb(h[0] / 256.0, 1, 0.3)
-        c2 = colorsys.hsv_to_rgb(h[0] / 256.0, 1, 1)
-        c = list(c1) + list(c2)
-        c = [int(x * 255) for x in c]
-        c = "".join(["{:02x}".format(x) for x in c])
+        c1 = colorsys.hsv_to_rgb(zb[0] / 256.0, 1, 0.3)
+        c2 = colorsys.hsv_to_rgb(zb[0] / 256.0, 1, 1)
+        ci = [int(x * 255) for x in list(c1) + list(c2)]
+        c = "".join(["{:02x}".format(x) for x in ci])
 
+        w = 100
         h = 30
         if not self.args.th_no_crop and as_thumb:
-            w, h = self.args.th_size.split("x")
-            h = int(100 / (float(w) / float(h)))
+            sw, sh = self.args.th_size.split("x")
+            h = int(100 / (float(sw) / float(sh)))
+            w = 100
+
+        if chrome and as_thumb:
+            # cannot handle more than ~2000 unique SVGs
+            if HAVE_PIL:
+                # svg: 3s, cache: 6s, this: 8s
+                from PIL import Image, ImageDraw
+
+                h = int(64 * h / w)
+                w = 64
+                img = Image.new("RGB", (w, h), "#" + c[:6])
+                pb = ImageDraw.Draw(img)
+                tw, th = pb.textsize(ext)
+                pb.text(((w - tw) // 2, (h - th) // 2), ext, fill="#" + c[6:])
+                img = img.resize((w * 3, h * 3), Image.NEAREST)
+
+                buf = BytesIO()
+                img.save(buf, format="PNG", compress_level=1)
+                return "image/png", buf.getvalue()
+
+            elif False:
+                # 48s, too slow
+                import pyvips
+
+                h = int(192 * h / w)
+                w = 192
+                img = pyvips.Image.text(
+                    ext, width=w, height=h, dpi=192, align=pyvips.Align.CENTRE
+                )
+                img = img.ifthenelse(ci[3:], ci[:3], blend=True)
+                # i = i.resize(3, kernel=pyvips.Kernel.NEAREST)
+                buf = img.write_to_buffer(".png[compression=1]")
+                return "image/png", buf
 
         svg = """\
 <?xml version="1.0" encoding="UTF-8"?>
@@ -37,6 +73,6 @@ class Ico(object):
   fill="#{}" font-family="monospace" font-size="14px" style="letter-spacing:.5px">{}</text>
 </g></svg>
 """
-        svg = svg.format(h, c[:6], c[6:], ext).encode("utf-8")
+        svg = svg.format(h, c[:6], c[6:], ext)
 
-        return ["image/svg+xml", svg]
+        return "image/svg+xml", svg.encode("utf-8")

copyparty/mtag.py

@@ -1,18 +1,26 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
-import os
-import sys
+import argparse
 import json
+import os
 import shutil
 import subprocess as sp
+import sys
 
-from .__init__ import PY2, WINDOWS, unicode
-from .util import fsenc, fsdec, uncyg, runcmd, REKOBO_LKEY
+from .__init__ import PY2, WINDOWS, E, unicode
 from .bos import bos
+from .util import REKOBO_LKEY, fsenc, min_ex, retchk, runcmd, uncyg
+
+try:
+    from typing import Any, Union
+
+    from .util import RootLogger
+except:
+    pass
 
 
-def have_ff(cmd):
+def have_ff(cmd: str) -> bool:
     if PY2:
         print("# checking {}".format(cmd))
         cmd = (cmd + " -version").encode("ascii").split(b" ")
@@ -30,13 +38,16 @@ HAVE_FFPROBE = have_ff("ffprobe")
 
 
 class MParser(object):
-    def __init__(self, cmdline):
+    def __init__(self, cmdline: str) -> None:
         self.tag, args = cmdline.split("=", 1)
         self.tags = self.tag.split(",")
 
-        self.timeout = 30
+        self.timeout = 60
         self.force = False
+        self.kill = "t"  # tree; all children recursively
+        self.capture = 3  # outputs to consume
         self.audio = "y"
+        self.pri = 0  # priority; higher = later
         self.ext = []
 
         while True:
@@ -58,6 +69,14 @@ class MParser(object):
                 self.audio = arg[1:]  # [r]equire [n]ot [d]ontcare
                 continue
 
+            if arg.startswith("k"):
+                self.kill = arg[1:]  # [t]ree [m]ain [n]one
+                continue
+
+            if arg.startswith("c"):
+                self.capture = int(arg[1:])  # 0=none 1=stdout 2=stderr 3=both
+                continue
+
             if arg == "f":
                 self.force = True
                 continue
@@ -70,10 +89,16 @@ class MParser(object):
                 self.ext.append(arg[1:])
                 continue
 
+            if arg.startswith("p"):
+                self.pri = int(arg[1:] or "1")
+                continue
+
             raise Exception()
 
 
-def ffprobe(abspath, timeout=10):
+def ffprobe(
+    abspath: str, timeout: int = 60
+) -> tuple[dict[str, tuple[int, Any]], dict[str, list[Any]]]:
     cmd = [
         b"ffprobe",
         b"-hide_banner",
@@ -82,19 +107,20 @@ def ffprobe(abspath, timeout=10):
         b"--",
         fsenc(abspath),
     ]
-    rc = runcmd(cmd, timeout=timeout)
-    return parse_ffprobe(rc[1])
+    rc, so, se = runcmd(cmd, timeout=timeout)
+    retchk(rc, cmd, se)
+    return parse_ffprobe(so)
 
 
-def parse_ffprobe(txt):
+def parse_ffprobe(txt: str) -> tuple[dict[str, tuple[int, Any]], dict[str, list[Any]]]:
     """ffprobe -show_format -show_streams"""
     streams = []
     fmt = {}
-    g = None
+    g = {}
     for ln in [x.rstrip("\r") for x in txt.split("\n")]:
         try:
-            k, v = ln.split("=", 1)
-            g[k] = v
+            sk, sv = ln.split("=", 1)
+            g[sk] = sv
             continue
         except:
             pass
@@ -108,8 +134,8 @@ def parse_ffprobe(txt):
             fmt = g
 
     streams = [fmt] + streams
-    ret = {}  # processed
-    md = {}  # raw tags
+    ret: dict[str, Any] = {}  # processed
+    md: dict[str, list[Any]] = {}  # raw tags
 
     is_audio = fmt.get("format_name") in ["mp3", "ogg", "flac", "wav"]
     if fmt.get("filename", "").split(".")[-1].lower() in ["m4a", "aac"]:
@@ -157,52 +183,55 @@ def parse_ffprobe(txt):
             ]
 
         if typ == "format":
-            kvm = [["duration", ".dur"], ["bit_rate", ".q"]]
+            kvm = [["duration", ".dur"], ["bit_rate", ".q"], ["format_name", "fmt"]]
 
         for sk, rk in kvm:
-            v = strm.get(sk)
-            if v is None:
+            v1 = strm.get(sk)
+            if v1 is None:
                 continue
 
             if rk.startswith("."):
                 try:
-                    v = float(v)
+                    zf = float(v1)
                     v2 = ret.get(rk)
-                    if v2 is None or v > v2:
-                        ret[rk] = v
+                    if v2 is None or zf > v2:
+                        ret[rk] = zf
                 except:
                     # sqlite doesnt care but the code below does
-                    if v not in ["N/A"]:
-                        ret[rk] = v
+                    if v1 not in ["N/A"]:
+                        ret[rk] = v1
             else:
-                ret[rk] = v
+                ret[rk] = v1
 
     if ret.get("vc") == "ansi":  # shellscript
         return {}, {}
 
     for strm in streams:
-        for k, v in strm.items():
-            if not k.startswith("TAG:"):
+        for sk, sv in strm.items():
+            if not sk.startswith("TAG:"):
                 continue
 
-            k = k[4:].strip()
-            v = v.strip()
-            if k and v and k not in md:
-                md[k] = [v]
+            sk = sk[4:].strip()
+            sv = sv.strip()
+            if sk and sv and sk not in md:
+                md[sk] = [sv]
 
-    for k in [".q", ".vq", ".aq"]:
-        if k in ret:
-            ret[k] /= 1000  # bit_rate=320000
+    for sk in [".q", ".vq", ".aq"]:
+        if sk in ret:
+            ret[sk] /= 1000  # bit_rate=320000
 
-    for k in [".q", ".vq", ".aq", ".resw", ".resh"]:
-        if k in ret:
-            ret[k] = int(ret[k])
+    for sk in [".q", ".vq", ".aq", ".resw", ".resh"]:
+        if sk in ret:
+            ret[sk] = int(ret[sk])
 
     if ".fps" in ret:
         fps = ret[".fps"]
         if "/" in fps:
             fa, fb = fps.split("/")
-            fps = int(fa) * 1.0 / int(fb)
+            try:
+                fps = int(fa) * 1.0 / int(fb)
+            except:
+                fps = 9001
 
         if fps < 1000 and fmt.get("format_name") not in ["image2", "png_pipe"]:
             ret[".fps"] = round(fps, 3)
@@ -215,33 +244,32 @@ def parse_ffprobe(txt):
             if ".q" in ret:
                 del ret[".q"]
 
+    if "fmt" in ret:
+        ret["fmt"] = ret["fmt"].split(",")[0]
+
     if ".resw" in ret and ".resh" in ret:
         ret["res"] = "{}x{}".format(ret[".resw"], ret[".resh"])
 
-    ret = {k: [0, v] for k, v in ret.items()}
+    zd = {k: (0, v) for k, v in ret.items()}
 
-    return ret, md
+    return zd, md
 
 
 class MTag(object):
-    def __init__(self, log_func, args):
+    def __init__(self, log_func: "RootLogger", args: argparse.Namespace) -> None:
         self.log_func = log_func
         self.args = args
         self.usable = True
         self.prefer_mt = not args.no_mtag_ff
         self.backend = "ffprobe" if args.no_mutagen else "mutagen"
-        self.can_ffprobe = (
-            HAVE_FFPROBE
-            and not args.no_mtag_ff
-            and (not WINDOWS or sys.version_info >= (3, 8))
-        )
+        self.can_ffprobe = HAVE_FFPROBE and not args.no_mtag_ff
         mappings = args.mtm
         or_ffprobe = " or FFprobe"
 
         if self.backend == "mutagen":
             self.get = self.get_mutagen
             try:
-                import mutagen
+                import mutagen  # noqa: F401  # pylint: disable=unused-import,import-outside-toplevel
             except:
                 self.log("could not load Mutagen, trying FFprobe instead", c=3)
                 self.backend = "ffprobe"
@@ -258,11 +286,6 @@ class MTag(object):
                 msg = "found FFprobe but it was disabled by --no-mtag-ff"
                 self.log(msg, c=3)
 
-            elif WINDOWS and sys.version_info < (3, 8):
-                or_ffprobe = " or python >= 3.8"
-                msg = "found FFprobe but your python is too old; need 3.8 or newer"
-                self.log(msg, c=1)
-
         if not self.usable:
             msg = "need Mutagen{} to read media tags so please run this:\n{}{} -m pip install --user mutagen\n"
             pybin = os.path.basename(sys.executable)
@@ -338,31 +361,33 @@ class MTag(object):
         }
         # self.get = self.compare
 
-    def log(self, msg, c=0):
+    def log(self, msg: str, c: Union[int, str] = 0) -> None:
         self.log_func("mtag", msg, c)
 
-    def normalize_tags(self, ret, md):
-        for k, v in dict(md).items():
-            if not v:
+    def normalize_tags(
+        self, parser_output: dict[str, tuple[int, Any]], md: dict[str, list[Any]]
+    ) -> dict[str, Union[str, float]]:
+        for sk, tv in dict(md).items():
+            if not tv:
                 continue
 
-            k = k.lower().split("::")[0].strip()
-            mk = self.rmap.get(k)
-            if not mk:
+            sk = sk.lower().split("::")[0].strip()
+            key_mapping = self.rmap.get(sk)
+            if not key_mapping:
                 continue
 
-            pref, mk = mk
-            if mk not in ret or ret[mk][0] > pref:
-                ret[mk] = [pref, v[0]]
+            priority, alias = key_mapping
+            if alias not in parser_output or parser_output[alias][0] > priority:
+                parser_output[alias] = (priority, tv[0])
 
-        # take first value
-        ret = {k: unicode(v[1]).strip() for k, v in ret.items()}
+        # take first value (lowest priority / most preferred)
+        ret = {sk: unicode(tv[1]).strip() for sk, tv in parser_output.items()}
 
         # track 3/7 => track 3
-        for k, v in ret.items():
-            if k[0] == ".":
-                v = v.split("/")[0].strip().lstrip("0")
-                ret[k] = v or 0
+        for sk, tv in ret.items():
+            if sk[0] == ".":
+                sv = str(tv).split("/")[0].strip().lstrip("0")
+                ret[sk] = sv or 0
 
         # normalize key notation to rkeobo
         okey = ret.get("key")
@@ -372,7 +397,7 @@ class MTag(object):
 
         return ret
 
-    def compare(self, abspath):
+    def compare(self, abspath: str) -> dict[str, Union[str, float]]:
         if abspath.endswith(".au"):
             return {}
 
@@ -410,7 +435,9 @@ class MTag(object):
 
         return r1
 
-    def get_mutagen(self, abspath):
+    def get_mutagen(self, abspath: str) -> dict[str, Union[str, float]]:
+        ret: dict[str, tuple[int, Any]] = {}
+
         if not bos.path.isfile(abspath):
             return {}
 
@@ -420,11 +447,14 @@ class MTag(object):
             md = mutagen.File(fsenc(abspath), easy=True)
             if not md.info.length and not md.info.codec:
                 raise Exception()
-        except Exception as ex:
+        except:
             return self.get_ffprobe(abspath) if self.can_ffprobe else {}
 
         sz = bos.path.getsize(abspath)
-        ret = {".q": [0, int((sz / md.info.length) / 128)]}
+        try:
+            ret[".q"] = (0, int((sz / md.info.length) / 128))
+        except:
+            pass
 
         for attr, k, norm in [
             ["codec", "ac", unicode],
@@ -455,54 +485,74 @@ class MTag(object):
             if k == "ac" and v.startswith("mp4a.40."):
                 v = "aac"
 
-            ret[k] = [0, norm(v)]
+            ret[k] = (0, norm(v))
 
         return self.normalize_tags(ret, md)
 
-    def get_ffprobe(self, abspath):
+    def get_ffprobe(self, abspath: str) -> dict[str, Union[str, float]]:
         if not bos.path.isfile(abspath):
             return {}
 
-        ret, md = ffprobe(abspath)
+        ret, md = ffprobe(abspath, self.args.mtag_to)
         return self.normalize_tags(ret, md)
 
-    def get_bin(self, parsers, abspath):
+    def get_bin(
+        self, parsers: dict[str, MParser], abspath: str, oth_tags: dict[str, Any]
+    ) -> dict[str, Any]:
         if not bos.path.isfile(abspath):
             return {}
 
-        pypath = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
-        pypath = [str(pypath)] + [str(x) for x in sys.path if x]
-        pypath = str(os.pathsep.join(pypath))
         env = os.environ.copy()
-        env["PYTHONPATH"] = pypath
+        try:
+            pypath = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
+            zsl = [str(pypath)] + [str(x) for x in sys.path if x]
+            pypath = str(os.pathsep.join(zsl))
+            env["PYTHONPATH"] = pypath
+        except:
+            if not E.ox:
+                raise
 
-        ret = {}
-        for tagname, mp in parsers.items():
+        ret: dict[str, Any] = {}
+        for tagname, parser in sorted(parsers.items(), key=lambda x: (x[1].pri, x[0])):
             try:
-                cmd = [mp.bin, abspath]
-                if mp.bin.endswith(".py"):
+                cmd = [parser.bin, abspath]
+                if parser.bin.endswith(".py"):
                     cmd = [sys.executable] + cmd
 
-                args = {"env": env, "timeout": mp.timeout}
+                args = {
+                    "env": env,
+                    "timeout": parser.timeout,
+                    "kill": parser.kill,
+                    "capture": parser.capture,
+                }
+
+                if parser.pri:
+                    zd = oth_tags.copy()
+                    zd.update(ret)
+                    args["sin"] = json.dumps(zd).encode("utf-8", "replace")
 
                 if WINDOWS:
                     args["creationflags"] = 0x4000
                 else:
                     cmd = ["nice"] + cmd
 
-                cmd = [fsenc(x) for x in cmd]
-                v = sp.check_output(cmd, **args).strip()
+                bcmd = [fsenc(x) for x in cmd]
+                rc, v, err = runcmd(bcmd, **args)  # type: ignore
+                retchk(rc, bcmd, err, self.log, 5, self.args.mtag_v)
+                v = v.strip()
                 if not v:
                     continue
 
                 if "," not in tagname:
-                    ret[tagname] = v.decode("utf-8")
+                    ret[tagname] = v
                 else:
-                    v = json.loads(v)
+                    zj = json.loads(v)
                     for tag in tagname.split(","):
-                        if tag and tag in v:
-                            ret[tag] = v[tag]
+                        if tag and tag in zj:
+                            ret[tag] = zj[tag]
             except:
-                pass
+                if self.args.mtag_v:
+                    t = "mtag error: tagname {}, parser {}, file {} => {}"
+                    self.log(t.format(tagname, parser.bin, abspath, min_ex()))
 
         return ret

copyparty/star.py

@@ -4,20 +4,29 @@ from __future__ import print_function, unicode_literals
 import tarfile
 import threading
 
-from .sutil import errdesc
-from .util import Queue, fsenc
+from queue import Queue
+
 from .bos import bos
+from .sutil import StreamArc, errdesc
+from .util import fsenc, min_ex
+
+try:
+    from typing import Any, Generator, Optional
+
+    from .util import NamedLogger
+except:
+    pass
 
 
-class QFile(object):
+class QFile(object):  # inherit io.StringIO for painful typing
     """file-like object which buffers writes into a queue"""
 
-    def __init__(self):
-        self.q = Queue(64)
-        self.bq = []
+    def __init__(self) -> None:
+        self.q: Queue[Optional[bytes]] = Queue(64)
+        self.bq: list[bytes] = []
         self.nq = 0
 
-    def write(self, buf):
+    def write(self, buf: Optional[bytes]) -> None:
         if buf is None or self.nq >= 240 * 1024:
             self.q.put(b"".join(self.bq))
             self.bq = []
@@ -30,40 +39,47 @@ class QFile(object):
             self.nq += len(buf)
 
 
-class StreamTar(object):
+class StreamTar(StreamArc):
     """construct in-memory tar file from the given path"""
 
-    def __init__(self, log, fgen, **kwargs):
+    def __init__(
+        self,
+        log: "NamedLogger",
+        fgen: Generator[dict[str, Any], None, None],
+        **kwargs: Any
+    ):
+        super(StreamTar, self).__init__(log, fgen)
+
         self.ci = 0
         self.co = 0
         self.qfile = QFile()
-        self.log = log
-        self.fgen = fgen
-        self.errf = None
+        self.errf: dict[str, Any] = {}
 
         # python 3.8 changed to PAX_FORMAT as default,
         # waste of space and don't care about the new features
         fmt = tarfile.GNU_FORMAT
-        self.tar = tarfile.open(fileobj=self.qfile, mode="w|", format=fmt)
+        self.tar = tarfile.open(fileobj=self.qfile, mode="w|", format=fmt)  # type: ignore
 
         w = threading.Thread(target=self._gen, name="star-gen")
         w.daemon = True
         w.start()
 
-    def gen(self):
-        while True:
-            buf = self.qfile.q.get()
-            if not buf:
-                break
+    def gen(self) -> Generator[Optional[bytes], None, None]:
+        try:
+            while True:
+                buf = self.qfile.q.get()
+                if not buf:
+                    break
 
-            self.co += len(buf)
-            yield buf
+                self.co += len(buf)
+                yield buf
 
-        yield None
-        if self.errf:
-            bos.unlink(self.errf["ap"])
+            yield None
+        finally:
+            if self.errf:
+                bos.unlink(self.errf["ap"])
 
-    def ser(self, f):
+    def ser(self, f: dict[str, Any]) -> None:
         name = f["vp"]
         src = f["ap"]
         fsi = f["st"]
@@ -76,20 +92,21 @@ class StreamTar(object):
         inf.gid = 0
 
         self.ci += inf.size
-        with open(fsenc(src), "rb", 512 * 1024) as f:
-            self.tar.addfile(inf, f)
+        with open(fsenc(src), "rb", 512 * 1024) as fo:
+            self.tar.addfile(inf, fo)
 
-    def _gen(self):
+    def _gen(self) -> None:
         errors = []
         for f in self.fgen:
             if "err" in f:
-                errors.append([f["vp"], f["err"]])
+                errors.append((f["vp"], f["err"]))
                 continue
 
             try:
                 self.ser(f)
-            except Exception as ex:
-                errors.append([f["vp"], repr(ex)])
+            except:
+                ex = min_ex(5, True).replace("\n", "\n-- ")
+                errors.append((f["vp"], ex))
 
         if errors:
             self.errf, txt = errdesc(errors)

copyparty/stolen/qrcodegen.py

@@ -0,0 +1,593 @@
+# coding: utf-8
+
+# modified copy of Project Nayuki's qrcodegen (MIT-licensed);
+# https://github.com/nayuki/QR-Code-generator/blob/daa3114/python/qrcodegen.py
+# the original ^ is extremely well commented so refer to that for explanations
+
+# hacks: binary-only, auto-ecc, render, py2-compat
+
+from __future__ import print_function, unicode_literals
+
+import collections
+import itertools
+
+try:
+    from collections.abc import Sequence
+
+    from typing import Callable, List, Optional, Tuple, Union
+except:
+    pass
+
+
+def num_char_count_bits(ver: int) -> int:
+    return 16 if (ver + 7) // 17 else 8
+
+
+class Ecc(object):
+    ordinal: int
+    formatbits: int
+
+    def __init__(self, i: int, fb: int) -> None:
+        self.ordinal = i
+        self.formatbits = fb
+
+    LOW: "Ecc"
+    MEDIUM: "Ecc"
+    QUARTILE: "Ecc"
+    HIGH: "Ecc"
+
+
+Ecc.LOW = Ecc(0, 1)
+Ecc.MEDIUM = Ecc(1, 0)
+Ecc.QUARTILE = Ecc(2, 3)
+Ecc.HIGH = Ecc(3, 2)
+
+
+class QrSegment(object):
+    @staticmethod
+    def make_seg(data: Union[bytes, Sequence[int]]) -> "QrSegment":
+        bb = _BitBuffer()
+        for b in data:
+            bb.append_bits(b, 8)
+        return QrSegment(len(data), bb)
+
+    numchars: int  # num bytes, not the same as the data's bit length
+    bitdata: List[int]  # The data bits of this segment
+
+    def __init__(self, numch: int, bitdata: Sequence[int]) -> None:
+        if numch < 0:
+            raise ValueError()
+        self.numchars = numch
+        self.bitdata = list(bitdata)
+
+    @staticmethod
+    def get_total_bits(segs: Sequence["QrSegment"], ver: int) -> Optional[int]:
+        result = 0
+        for seg in segs:
+            ccbits: int = num_char_count_bits(ver)
+            if seg.numchars >= (1 << ccbits):
+                return None  # segment length doesn't fit the field's bit width
+            result += 4 + ccbits + len(seg.bitdata)
+        return result
+
+
+class QrCode(object):
+    @staticmethod
+    def encode_binary(data: Union[bytes, Sequence[int]]) -> "QrCode":
+        return QrCode.encode_segments([QrSegment.make_seg(data)])
+
+    @staticmethod
+    def encode_segments(
+        segs: Sequence[QrSegment],
+        ecl: Ecc = Ecc.LOW,
+        minver: int = 2,
+        maxver: int = 40,
+        mask: int = -1,
+    ) -> "QrCode":
+        for ver in range(minver, maxver + 1):
+            datacapacitybits: int = QrCode._get_num_data_codewords(ver, ecl) * 8
+            datausedbits: Optional[int] = QrSegment.get_total_bits(segs, ver)
+            if (datausedbits is not None) and (datausedbits <= datacapacitybits):
+                break
+
+        assert datausedbits
+
+        for newecl in (
+            Ecc.MEDIUM,
+            Ecc.QUARTILE,
+            Ecc.HIGH,
+        ):
+            if datausedbits <= QrCode._get_num_data_codewords(ver, newecl) * 8:
+                ecl = newecl
+
+        # Concatenate all segments to create the data bit string
+        bb = _BitBuffer()
+        for seg in segs:
+            bb.append_bits(4, 4)
+            bb.append_bits(seg.numchars, num_char_count_bits(ver))
+            bb.extend(seg.bitdata)
+        assert len(bb) == datausedbits
+
+        # Add terminator and pad up to a byte if applicable
+        datacapacitybits = QrCode._get_num_data_codewords(ver, ecl) * 8
+        assert len(bb) <= datacapacitybits
+        bb.append_bits(0, min(4, datacapacitybits - len(bb)))
+        bb.append_bits(0, -len(bb) % 8)
+        assert len(bb) % 8 == 0
+
+        # Pad with alternating bytes until data capacity is reached
+        for padbyte in itertools.cycle((0xEC, 0x11)):
+            if len(bb) >= datacapacitybits:
+                break
+            bb.append_bits(padbyte, 8)
+
+        # Pack bits into bytes in big endian
+        datacodewords = bytearray([0] * (len(bb) // 8))
+        for (i, bit) in enumerate(bb):
+            datacodewords[i >> 3] |= bit << (7 - (i & 7))
+
+        return QrCode(ver, ecl, datacodewords, mask)
+
+    ver: int
+    size: int  # w/h; 21..177 (ver * 4 + 17)
+    ecclvl: Ecc
+    mask: int  # 0..7
+    modules: List[List[bool]]
+    unmaskable: List[List[bool]]
+
+    def __init__(
+        self,
+        ver: int,
+        ecclvl: Ecc,
+        datacodewords: Union[bytes, Sequence[int]],
+        msk: int,
+    ) -> None:
+        self.ver = ver
+        self.size = ver * 4 + 17
+        self.ecclvl = ecclvl
+
+        self.modules = [[False] * self.size for _ in range(self.size)]
+        self.unmaskable = [[False] * self.size for _ in range(self.size)]
+
+        # Compute ECC, draw modules
+        self._draw_function_patterns()
+        allcodewords: bytes = self._add_ecc_and_interleave(bytearray(datacodewords))
+        self._draw_codewords(allcodewords)
+
+        if msk == -1:  # automask
+            minpenalty: int = 1 << 32
+            for i in range(8):
+                self._apply_mask(i)
+                self._draw_format_bits(i)
+                penalty = self._get_penalty_score()
+                if penalty < minpenalty:
+                    msk = i
+                    minpenalty = penalty
+                self._apply_mask(i)  # xor/undo
+
+        assert 0 <= msk <= 7
+        self.mask = msk
+        self._apply_mask(msk)  # Apply the final choice of mask
+        self._draw_format_bits(msk)  # Overwrite old format bits
+
+    def render(self, zoom=1, pad=4) -> str:
+        tab = self.modules
+        sz = self.size
+        if sz % 2 and zoom == 1:
+            tab.append([False] * sz)
+
+        tab = [[False] * sz] * pad + tab + [[False] * sz] * pad
+        tab = [[False] * pad + x + [False] * pad for x in tab]
+
+        rows: list[str] = []
+        if zoom == 1:
+            for y in range(0, len(tab), 2):
+                row = ""
+                for x in range(len(tab[y])):
+                    v = 2 if tab[y][x] else 0
+                    v += 1 if tab[y + 1][x] else 0
+                    row += " ▄▀█"[v]
+                rows.append(row)
+        else:
+            for tr in tab:
+                row = ""
+                for zb in tr:
+                    row += " █"[int(zb)] * 2
+                rows.append(row)
+
+        return "\n".join(rows)
+
+    def _draw_function_patterns(self) -> None:
+        # Draw horizontal and vertical timing patterns
+        for i in range(self.size):
+            self._set_function_module(6, i, i % 2 == 0)
+            self._set_function_module(i, 6, i % 2 == 0)
+
+        # Draw 3 finder patterns (all corners except bottom right; overwrites some timing modules)
+        self._draw_finder_pattern(3, 3)
+        self._draw_finder_pattern(self.size - 4, 3)
+        self._draw_finder_pattern(3, self.size - 4)
+
+        # Draw numerous alignment patterns
+        alignpatpos: List[int] = self._get_alignment_pattern_positions()
+        numalign: int = len(alignpatpos)
+        skips: Sequence[Tuple[int, int]] = (
+            (0, 0),
+            (0, numalign - 1),
+            (numalign - 1, 0),
+        )
+        for i in range(numalign):
+            for j in range(numalign):
+                if (i, j) not in skips:  # avoid finder corners
+                    self._draw_alignment_pattern(alignpatpos[i], alignpatpos[j])
+
+        # draw config data with dummy mask value; ctor overwrites it
+        self._draw_format_bits(0)
+        self._draw_ver()
+
+    def _draw_format_bits(self, mask: int) -> None:
+        # Calculate error correction code and pack bits; ecclvl is uint2, mask is uint3
+        data: int = self.ecclvl.formatbits << 3 | mask
+        rem: int = data
+        for _ in range(10):
+            rem = (rem << 1) ^ ((rem >> 9) * 0x537)
+        bits: int = (data << 10 | rem) ^ 0x5412  # uint15
+        assert bits >> 15 == 0
+
+        # first copy
+        for i in range(0, 6):
+            self._set_function_module(8, i, _get_bit(bits, i))
+        self._set_function_module(8, 7, _get_bit(bits, 6))
+        self._set_function_module(8, 8, _get_bit(bits, 7))
+        self._set_function_module(7, 8, _get_bit(bits, 8))
+        for i in range(9, 15):
+            self._set_function_module(14 - i, 8, _get_bit(bits, i))
+
+        # second copy
+        for i in range(0, 8):
+            self._set_function_module(self.size - 1 - i, 8, _get_bit(bits, i))
+        for i in range(8, 15):
+            self._set_function_module(8, self.size - 15 + i, _get_bit(bits, i))
+        self._set_function_module(8, self.size - 8, True)  # Always dark
+
+    def _draw_ver(self) -> None:
+        if self.ver < 7:
+            return
+
+        # Calculate error correction code and pack bits
+        rem: int = self.ver  # ver is uint6, 7..40
+        for _ in range(12):
+            rem = (rem << 1) ^ ((rem >> 11) * 0x1F25)
+        bits: int = self.ver << 12 | rem  # uint18
+        assert bits >> 18 == 0
+
+        # Draw two copies
+        for i in range(18):
+            bit: bool = _get_bit(bits, i)
+            a: int = self.size - 11 + i % 3
+            b: int = i // 3
+            self._set_function_module(a, b, bit)
+            self._set_function_module(b, a, bit)
+
+    def _draw_finder_pattern(self, x: int, y: int) -> None:
+        for dy in range(-4, 5):
+            for dx in range(-4, 5):
+                xx, yy = x + dx, y + dy
+                if (0 <= xx < self.size) and (0 <= yy < self.size):
+                    # Chebyshev/infinity norm
+                    self._set_function_module(
+                        xx, yy, max(abs(dx), abs(dy)) not in (2, 4)
+                    )
+
+    def _draw_alignment_pattern(self, x: int, y: int) -> None:
+        for dy in range(-2, 3):
+            for dx in range(-2, 3):
+                self._set_function_module(x + dx, y + dy, max(abs(dx), abs(dy)) != 1)
+
+    def _set_function_module(self, x: int, y: int, isdark: bool) -> None:
+        self.modules[y][x] = isdark
+        self.unmaskable[y][x] = True
+
+    def _add_ecc_and_interleave(self, data: bytearray) -> bytes:
+        ver: int = self.ver
+        assert len(data) == QrCode._get_num_data_codewords(ver, self.ecclvl)
+
+        # Calculate parameter numbers
+        numblocks: int = QrCode._NUM_ERROR_CORRECTION_BLOCKS[self.ecclvl.ordinal][ver]
+        blockecclen: int = QrCode._ECC_CODEWORDS_PER_BLOCK[self.ecclvl.ordinal][ver]
+        rawcodewords: int = QrCode._get_num_raw_data_modules(ver) // 8
+        numshortblocks: int = numblocks - rawcodewords % numblocks
+        shortblocklen: int = rawcodewords // numblocks
+
+        # Split data into blocks and append ECC to each block
+        blocks: List[bytes] = []
+        rsdiv: bytes = QrCode._reed_solomon_compute_divisor(blockecclen)
+        k: int = 0
+        for i in range(numblocks):
+            dat: bytearray = data[
+                k : k + shortblocklen - blockecclen + (0 if i < numshortblocks else 1)
+            ]
+            k += len(dat)
+            ecc: bytes = QrCode._reed_solomon_compute_remainder(dat, rsdiv)
+            if i < numshortblocks:
+                dat.append(0)
+            blocks.append(dat + ecc)
+        assert k == len(data)
+
+        # Interleave (not concatenate) the bytes from every block into a single sequence
+        result = bytearray()
+        for i in range(len(blocks[0])):
+            for (j, blk) in enumerate(blocks):
+                # Skip the padding byte in short blocks
+                if (i != shortblocklen - blockecclen) or (j >= numshortblocks):
+                    result.append(blk[i])
+        assert len(result) == rawcodewords
+        return result
+
+    def _draw_codewords(self, data: bytes) -> None:
+        assert len(data) == QrCode._get_num_raw_data_modules(self.ver) // 8
+
+        i: int = 0  # Bit index into the data
+        for right in range(self.size - 1, 0, -2):
+            # idx of right column in each column pair
+            if right <= 6:
+                right -= 1
+            for vert in range(self.size):  # Vertical counter
+                for j in range(2):
+                    x: int = right - j
+                    upward: bool = (right + 1) & 2 == 0
+                    y: int = (self.size - 1 - vert) if upward else vert
+                    if (not self.unmaskable[y][x]) and (i < len(data) * 8):
+                        self.modules[y][x] = _get_bit(data[i >> 3], 7 - (i & 7))
+                        i += 1
+                    # any remainder bits (0..7) were set 0/false/light by ctor
+
+        assert i == len(data) * 8
+
+    def _apply_mask(self, mask: int) -> None:
+        masker: Callable[[int, int], int] = QrCode._MASK_PATTERNS[mask]
+        for y in range(self.size):
+            for x in range(self.size):
+                self.modules[y][x] ^= (masker(x, y) == 0) and (
+                    not self.unmaskable[y][x]
+                )
+
+    def _get_penalty_score(self) -> int:
+        result: int = 0
+        size: int = self.size
+        modules: List[List[bool]] = self.modules
+
+        # Adjacent modules in row having same color, and finder-like patterns
+        for y in range(size):
+            runcolor: bool = False
+            runx: int = 0
+            runhistory = collections.deque([0] * 7, 7)
+            for x in range(size):
+                if modules[y][x] == runcolor:
+                    runx += 1
+                    if runx == 5:
+                        result += QrCode._PENALTY_N1
+                    elif runx > 5:
+                        result += 1
+                else:
+                    self._finder_penalty_add_history(runx, runhistory)
+                    if not runcolor:
+                        result += (
+                            self._finder_penalty_count_patterns(runhistory)
+                            * QrCode._PENALTY_N3
+                        )
+                    runcolor = modules[y][x]
+                    runx = 1
+            result += (
+                self._finder_penalty_terminate_and_count(runcolor, runx, runhistory)
+                * QrCode._PENALTY_N3
+            )
+
+        # Adjacent modules in column having same color, and finder-like patterns
+        for x in range(size):
+            runcolor = False
+            runy = 0
+            runhistory = collections.deque([0] * 7, 7)
+            for y in range(size):
+                if modules[y][x] == runcolor:
+                    runy += 1
+                    if runy == 5:
+                        result += QrCode._PENALTY_N1
+                    elif runy > 5:
+                        result += 1
+                else:
+                    self._finder_penalty_add_history(runy, runhistory)
+                    if not runcolor:
+                        result += (
+                            self._finder_penalty_count_patterns(runhistory)
+                            * QrCode._PENALTY_N3
+                        )
+                    runcolor = modules[y][x]
+                    runy = 1
+            result += (
+                self._finder_penalty_terminate_and_count(runcolor, runy, runhistory)
+                * QrCode._PENALTY_N3
+            )
+
+        # 2*2 blocks of modules having same color
+        for y in range(size - 1):
+            for x in range(size - 1):
+                if (
+                    modules[y][x]
+                    == modules[y][x + 1]
+                    == modules[y + 1][x]
+                    == modules[y + 1][x + 1]
+                ):
+                    result += QrCode._PENALTY_N2
+
+        # Balance of dark and light modules
+        dark: int = sum((1 if cell else 0) for row in modules for cell in row)
+        total: int = size ** 2  # Note that size is odd, so dark/total != 1/2
+
+        # Compute the smallest integer k >= 0 such that (45-5k)% <= dark/total <= (55+5k)%
+        k: int = (abs(dark * 20 - total * 10) + total - 1) // total - 1
+        assert 0 <= k <= 9
+        result += k * QrCode._PENALTY_N4
+        assert 0 <= result <= 2568888
+        # ^ Non-tight upper bound based on default values of PENALTY_N1, ..., N4
+
+        return result
+
+    def _get_alignment_pattern_positions(self) -> List[int]:
+        ver: int = self.ver
+        if ver == 1:
+            return []
+
+        numalign: int = ver // 7 + 2
+        step: int = (
+            26
+            if (ver == 32)
+            else (ver * 4 + numalign * 2 + 1) // (numalign * 2 - 2) * 2
+        )
+        result: List[int] = [
+            (self.size - 7 - i * step) for i in range(numalign - 1)
+        ] + [6]
+        return list(reversed(result))
+
+    @staticmethod
+    def _get_num_raw_data_modules(ver: int) -> int:
+        result: int = (16 * ver + 128) * ver + 64
+        if ver >= 2:
+            numalign: int = ver // 7 + 2
+            result -= (25 * numalign - 10) * numalign - 55
+            if ver >= 7:
+                result -= 36
+        assert 208 <= result <= 29648
+        return result
+
+    @staticmethod
+    def _get_num_data_codewords(ver: int, ecl: Ecc) -> int:
+        return (
+            QrCode._get_num_raw_data_modules(ver) // 8
+            - QrCode._ECC_CODEWORDS_PER_BLOCK[ecl.ordinal][ver]
+            * QrCode._NUM_ERROR_CORRECTION_BLOCKS[ecl.ordinal][ver]
+        )
+
+    @staticmethod
+    def _reed_solomon_compute_divisor(degree: int) -> bytes:
+        if not (1 <= degree <= 255):
+            raise ValueError("Degree out of range")
+
+        # Polynomial coefficients are stored from highest to lowest power, excluding the leading term which is always 1.
+        # For example the polynomial x^3 + 255x^2 + 8x + 93 is stored as the uint8 array [255, 8, 93].
+        result = bytearray([0] * (degree - 1) + [1])  # start with monomial x^0
+
+        # Compute the product polynomial (x - r^0) * (x - r^1) * (x - r^2) * ... * (x - r^{degree-1}),
+        # and drop the highest monomial term which is always 1x^degree.
+        # Note that r = 0x02, which is a generator element of this field GF(2^8/0x11D).
+        root: int = 1
+        for _ in range(degree):
+            # Multiply the current product by (x - r^i)
+            for j in range(degree):
+                result[j] = QrCode._reed_solomon_multiply(result[j], root)
+                if j + 1 < degree:
+                    result[j] ^= result[j + 1]
+            root = QrCode._reed_solomon_multiply(root, 0x02)
+
+        return result
+
+    @staticmethod
+    def _reed_solomon_compute_remainder(data: bytes, divisor: bytes) -> bytes:
+        result = bytearray([0] * len(divisor))
+        for b in data:  # Polynomial division
+            factor: int = b ^ result.pop(0)
+            result.append(0)
+            for (i, coef) in enumerate(divisor):
+                result[i] ^= QrCode._reed_solomon_multiply(coef, factor)
+
+        return result
+
+    @staticmethod
+    def _reed_solomon_multiply(x: int, y: int) -> int:
+        if (x >> 8 != 0) or (y >> 8 != 0):
+            raise ValueError("Byte out of range")
+        z: int = 0  # Russian peasant multiplication
+        for i in reversed(range(8)):
+            z = (z << 1) ^ ((z >> 7) * 0x11D)
+            z ^= ((y >> i) & 1) * x
+        assert z >> 8 == 0
+        return z
+
+    def _finder_penalty_count_patterns(self, runhistory: collections.deque[int]) -> int:
+        n: int = runhistory[1]
+        assert n <= self.size * 3
+        core: bool = (
+            n > 0
+            and (runhistory[2] == runhistory[4] == runhistory[5] == n)
+            and runhistory[3] == n * 3
+        )
+        return (
+            1 if (core and runhistory[0] >= n * 4 and runhistory[6] >= n) else 0
+        ) + (1 if (core and runhistory[6] >= n * 4 and runhistory[0] >= n) else 0)
+
+    def _finder_penalty_terminate_and_count(
+        self,
+        currentruncolor: bool,
+        currentrunlength: int,
+        runhistory: collections.deque[int],
+    ) -> int:
+        if currentruncolor:  # Terminate dark run
+            self._finder_penalty_add_history(currentrunlength, runhistory)
+            currentrunlength = 0
+        currentrunlength += self.size  # Add light border to final run
+        self._finder_penalty_add_history(currentrunlength, runhistory)
+        return self._finder_penalty_count_patterns(runhistory)
+
+    def _finder_penalty_add_history(
+        self, currentrunlength: int, runhistory: collections.deque[int]
+    ) -> None:
+        if runhistory[0] == 0:
+            currentrunlength += self.size  # Add light border to initial run
+
+        runhistory.appendleft(currentrunlength)
+
+    _PENALTY_N1: int = 3
+    _PENALTY_N2: int = 3
+    _PENALTY_N3: int = 40
+    _PENALTY_N4: int = 10
+
+    # fmt: off
+    _ECC_CODEWORDS_PER_BLOCK: Sequence[Sequence[int]] = (
+        (-1,  7, 10, 15, 20, 26, 18, 20, 24, 30, 18, 20, 24, 26, 30, 22, 24, 28, 30, 28, 28, 28, 28, 30, 30, 26, 28, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30),  # noqa: E241  # L
+        (-1, 10, 16, 26, 18, 24, 16, 18, 22, 22, 26, 30, 22, 22, 24, 24, 28, 28, 26, 26, 26, 26, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28),  # noqa: E241  # M
+        (-1, 13, 22, 18, 26, 18, 24, 18, 22, 20, 24, 28, 26, 24, 20, 30, 24, 28, 28, 26, 30, 28, 30, 30, 30, 30, 28, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30),  # noqa: E241  # Q
+        (-1, 17, 28, 22, 16, 22, 28, 26, 26, 24, 28, 24, 28, 22, 24, 24, 30, 28, 28, 26, 28, 30, 24, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30))  # noqa: E241  # H
+
+    _NUM_ERROR_CORRECTION_BLOCKS: Sequence[Sequence[int]] = (
+        (-1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 4,  4,  4,  4,  4,  6,  6,  6,  6,  7,  8,  8,  9,  9, 10, 12, 12, 12, 13, 14, 15, 16, 17, 18, 19, 19, 20, 21, 22, 24, 25),  # noqa: E241  # L
+        (-1, 1, 1, 1, 2, 2, 4, 4, 4, 5, 5,  5,  8,  9,  9, 10, 10, 11, 13, 14, 16, 17, 17, 18, 20, 21, 23, 25, 26, 28, 29, 31, 33, 35, 37, 38, 40, 43, 45, 47, 49),  # noqa: E241  # M
+        (-1, 1, 1, 2, 2, 4, 4, 6, 6, 8, 8,  8, 10, 12, 16, 12, 17, 16, 18, 21, 20, 23, 23, 25, 27, 29, 34, 34, 35, 38, 40, 43, 45, 48, 51, 53, 56, 59, 62, 65, 68),  # noqa: E241  # Q
+        (-1, 1, 1, 2, 4, 4, 4, 5, 6, 8, 8, 11, 11, 16, 16, 18, 16, 19, 21, 25, 25, 25, 34, 30, 32, 35, 37, 40, 42, 45, 48, 51, 54, 57, 60, 63, 66, 70, 74, 77, 81))  # noqa: E241  # H
+    # fmt: on
+
+    _MASK_PATTERNS: Sequence[Callable[[int, int], int]] = (
+        (lambda x, y: (x + y) % 2),
+        (lambda x, y: y % 2),
+        (lambda x, y: x % 3),
+        (lambda x, y: (x + y) % 3),
+        (lambda x, y: (x // 3 + y // 2) % 2),
+        (lambda x, y: x * y % 2 + x * y % 3),
+        (lambda x, y: (x * y % 2 + x * y % 3) % 2),
+        (lambda x, y: ((x + y) % 2 + x * y % 3) % 2),
+    )
+
+
+class _BitBuffer(list):  # type: ignore
+    def append_bits(self, val: int, n: int) -> None:
+        if (n < 0) or (val >> n != 0):
+            raise ValueError("Value out of range")
+
+        self.extend(((val >> i) & 1) for i in reversed(range(n)))
+
+
+def _get_bit(x: int, i: int) -> bool:
+    return (x >> i) & 1 != 0
+
+
+class DataTooLongError(ValueError):
+    pass

copyparty/stolen/surrogateescape.py

@@ -12,27 +12,18 @@ Original source: misc/python/surrogateescape.py in https://bitbucket.org/haypo/m
 
 # This code is released under the Python license and the BSD 2-clause license
 
-import platform
 import codecs
+import platform
 import sys
 
-PY3 = sys.version_info[0] > 2
+PY3 = sys.version_info > (3,)
 WINDOWS = platform.system() == "Windows"
 FS_ERRORS = "surrogateescape"
 
-
-def u(text):
-    if PY3:
-        return text
-    else:
-        return text.decode("unicode_escape")
-
-
-def b(data):
-    if PY3:
-        return data.encode("latin1")
-    else:
-        return data
+try:
+    from typing import Any
+except:
+    pass
 
 
 if PY3:
@@ -43,7 +34,7 @@ else:
     bytes_chr = chr
 
 
-def surrogateescape_handler(exc):
+def surrogateescape_handler(exc: Any) -> tuple[str, int]:
     """
     Pure Python implementation of the PEP 383: the "surrogateescape" error
     handler of Python 3. Undecodable bytes will be replaced by a Unicode
@@ -74,7 +65,7 @@ class NotASurrogateError(Exception):
     pass
 
 
-def replace_surrogate_encode(mystring):
+def replace_surrogate_encode(mystring: str) -> str:
     """
     Returns a (unicode) string, not the more logical bytes, because the codecs
     register_error functionality expects this.
@@ -100,7 +91,7 @@ def replace_surrogate_encode(mystring):
     return str().join(decoded)
 
 
-def replace_surrogate_decode(mybytes):
+def replace_surrogate_decode(mybytes: bytes) -> str:
     """
     Returns a (unicode) string
     """
@@ -121,7 +112,7 @@ def replace_surrogate_decode(mybytes):
     return str().join(decoded)
 
 
-def encodefilename(fn):
+def encodefilename(fn: str) -> bytes:
     if FS_ENCODING == "ascii":
         # ASCII encoder of Python 2 expects that the error handler returns a
         # Unicode string encodable to ASCII, whereas our surrogateescape error
@@ -161,14 +152,11 @@ def encodefilename(fn):
         return fn.encode(FS_ENCODING, FS_ERRORS)
 
 
-def decodefilename(fn):
+def decodefilename(fn: bytes) -> str:
     return fn.decode(FS_ENCODING, FS_ERRORS)
 
 
 FS_ENCODING = sys.getfilesystemencoding()
-# FS_ENCODING = "ascii"; fn = b("[abc\xff]"); encoded = u("[abc\udcff]")
-# FS_ENCODING = 'cp932'; fn = b('[abc\x81\x00]'); encoded = u('[abc\udc81\x00]')
-# FS_ENCODING = 'UTF-8'; fn = b('[abc\xff]'); encoded = u('[abc\udcff]')
 
 
 if WINDOWS and not PY3:
@@ -181,7 +169,7 @@ if WINDOWS and not PY3:
 FS_ENCODING = codecs.lookup(FS_ENCODING).name
 
 
-def register_surrogateescape():
+def register_surrogateescape() -> None:
     """
     Registers the surrogateescape error handler on Python 2 (only)
     """

copyparty/sutil.py

@@ -1,14 +1,34 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
-import time
 import tempfile
 from datetime import datetime
 
 from .bos import bos
 
+try:
+    from typing import Any, Generator, Optional
 
-def errdesc(errors):
+    from .util import NamedLogger
+except:
+    pass
+
+
+class StreamArc(object):
+    def __init__(
+        self,
+        log: "NamedLogger",
+        fgen: Generator[dict[str, Any], None, None],
+        **kwargs: Any
+    ):
+        self.log = log
+        self.fgen = fgen
+
+    def gen(self) -> Generator[Optional[bytes], None, None]:
+        pass
+
+
+def errdesc(errors: list[tuple[str, str]]) -> tuple[dict[str, Any], list[str]]:
     report = ["copyparty failed to add the following files to the archive:", ""]
 
     for fn, err in errors:

copyparty/svchub.py

@@ -1,51 +1,103 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
+import argparse
+import base64
+import calendar
+import gzip
 import os
-import sys
-import time
+import re
 import shlex
-import string
 import signal
 import socket
+import string
+import sys
 import threading
+import time
 from datetime import datetime, timedelta
-import calendar
 
-from .__init__ import E, PY2, WINDOWS, ANYWIN, MACOS, VT100, unicode
-from .util import mp, start_log_thrs, start_stackmon, min_ex, ansi_re
+try:
+    from types import FrameType
+
+    import typing
+    from typing import Any, Optional, Union
+except:
+    pass
+
+from .__init__ import ANYWIN, MACOS, VT100, EnvParams, unicode
 from .authsrv import AuthSrv
-from .tcpsrv import TcpSrv
-from .up2k import Up2k
-from .th_srv import ThumbSrv, HAVE_PIL, HAVE_WEBP
 from .mtag import HAVE_FFMPEG, HAVE_FFPROBE
+from .tcpsrv import TcpSrv
+from .th_srv import HAVE_PIL, HAVE_VIPS, HAVE_WEBP, ThumbSrv
+from .up2k import Up2k
+from .util import (
+    VERSIONS,
+    HMaccas,
+    alltrace,
+    ansi_re,
+    min_ex,
+    mp,
+    start_log_thrs,
+    start_stackmon,
+)
 
 
 class SvcHub(object):
     """
     Hosts all services which cannot be parallelized due to reliance on monolithic resources.
     Creates a Broker which does most of the heavy stuff; hosted services can use this to perform work:
-        hub.broker.put(want_reply, destination, args_list).
+        hub.broker.<say|ask>(destination, args_list).
 
     Either BrokerThr (plain threads) or BrokerMP (multiprocessing) is used depending on configuration.
     Nothing is returned synchronously; if you want any value returned from the call,
     put() can return a queue (if want_reply=True) which has a blocking get() with the response.
     """
 
-    def __init__(self, args, argv, printed):
+    def __init__(self, args: argparse.Namespace, argv: list[str], printed: str) -> None:
         self.args = args
         self.argv = argv
-        self.logf = None
+        self.E: EnvParams = args.E
+        self.logf: Optional[typing.TextIO] = None
+        self.logf_base_fn = ""
         self.stop_req = False
-        self.reload_req = False
         self.stopping = False
+        self.stopped = False
+        self.reload_req = False
         self.reloading = False
         self.stop_cond = threading.Condition()
+        self.nsigs = 3
         self.retcode = 0
         self.httpsrv_up = 0
 
         self.log_mutex = threading.Lock()
         self.next_day = 0
+        self.tstack = 0.0
+
+        self.iphash = HMaccas(os.path.join(self.E.cfg, "iphash"), 8)
+
+        if args.sss or args.s >= 3:
+            args.ss = True
+            args.lo = args.lo or "cpp-%Y-%m%d-%H%M%S.txt.xz"
+            args.ls = args.ls or "**,*,ln,p,r"
+
+        if args.ss or args.s >= 2:
+            args.s = True
+            args.no_logues = True
+            args.no_readme = True
+            args.unpost = 0
+            args.no_del = True
+            args.no_mv = True
+            args.hardlink = True
+            args.vague_403 = True
+            args.ban_404 = "50,60,1440"
+            args.nih = True
+
+        if args.s:
+            args.dotpart = True
+            args.no_thumb = True
+            args.no_mtag_ff = True
+            args.no_robots = True
+            args.force_js = True
 
         self.log = self._log_disabled if args.q else self._log_enabled
         if args.lo:
@@ -59,16 +111,26 @@ class SvcHub(object):
 
         if not args.use_fpool and args.j != 1:
             args.no_fpool = True
-            m = "multithreading enabled with -j {}, so disabling fpool -- this can reduce upload performance on some filesystems"
-            self.log("root", m.format(args.j))
+            t = "multithreading enabled with -j {}, so disabling fpool -- this can reduce upload performance on some filesystems"
+            self.log("root", t.format(args.j))
 
         if not args.no_fpool and args.j != 1:
-            m = "WARNING: --use-fpool combined with multithreading is untested and can probably cause undefined behavior"
+            t = "WARNING: --use-fpool combined with multithreading is untested and can probably cause undefined behavior"
             if ANYWIN:
-                m = 'windows cannot do multithreading without --no-fpool, so enabling that -- note that upload performance will suffer if you have microsoft defender "real-time protection" enabled, so you probably want to use -j 1 instead'
+                t = 'windows cannot do multithreading without --no-fpool, so enabling that -- note that upload performance will suffer if you have microsoft defender "real-time protection" enabled, so you probably want to use -j 1 instead'
                 args.no_fpool = True
 
-            self.log("root", m, c=3)
+            self.log("root", t, c=3)
+
+        bri = "zy"[args.theme % 2 :][:1]
+        ch = "abcdefghijklmnopqrstuvwx"[int(args.theme / 2)]
+        args.theme = "{0}{1} {0} {1}".format(ch, bri)
+
+        if not args.hardlink and args.never_symlink:
+            args.no_dedup = True
+
+        if args.log_fk:
+            args.log_fk = re.compile(args.log_fk)
 
         # initiate all services to manage
         self.asrv = AuthSrv(self.args, self.log)
@@ -78,20 +140,30 @@ class SvcHub(object):
         self.tcpsrv = TcpSrv(self)
         self.up2k = Up2k(self)
 
+        decs = {k: 1 for k in self.args.th_dec.split(",")}
+        if not HAVE_VIPS:
+            decs.pop("vips", None)
+        if not HAVE_PIL:
+            decs.pop("pil", None)
+        if not HAVE_FFMPEG or not HAVE_FFPROBE:
+            decs.pop("ff", None)
+
+        self.args.th_dec = list(decs.keys())
         self.thumbsrv = None
         if not args.no_thumb:
-            if HAVE_PIL:
-                if not HAVE_WEBP:
-                    args.th_no_webp = True
-                    msg = "setting --th-no-webp because either libwebp is not available or your Pillow is too old"
-                    self.log("thumb", msg, c=3)
+            t = ", ".join(self.args.th_dec) or "(None available)"
+            self.log("thumb", "decoder preference: {}".format(t))
 
+            if "pil" in self.args.th_dec and not HAVE_WEBP:
+                msg = "disabling webp thumbnails because either libwebp is not available or your Pillow is too old"
+                self.log("thumb", msg, c=3)
+
+            if self.args.th_dec:
                 self.thumbsrv = ThumbSrv(self)
             else:
-                msg = "need Pillow to create thumbnails; for example:\n{}{} -m pip install --user Pillow\n"
-                self.log(
-                    "thumb", msg.format(" " * 37, os.path.basename(sys.executable)), c=3
-                )
+                msg = "need either Pillow, pyvips, or FFmpeg to create thumbnails; for example:\n{0}{1} -m pip install --user Pillow\n{0}{1} -m pip install --user pyvips\n{0}apt install ffmpeg"
+                msg = msg.format(" " * 37, os.path.basename(sys.executable))
+                self.log("thumb", msg, c=3)
 
         if not args.no_acode and args.no_thumb:
             msg = "setting --no-acode because --no-thumb (sorry)"
@@ -114,46 +186,56 @@ class SvcHub(object):
         if self.check_mp_enable():
             from .broker_mp import BrokerMp as Broker
         else:
-            from .broker_thr import BrokerThr as Broker
+            from .broker_thr import BrokerThr as Broker  # type: ignore
 
         self.broker = Broker(self)
 
-    def thr_httpsrv_up(self):
-        time.sleep(5)
+    def thr_httpsrv_up(self) -> None:
+        time.sleep(1 if self.args.ign_ebind_all else 5)
         expected = self.broker.num_workers * self.tcpsrv.nsrv
         failed = expected - self.httpsrv_up
         if not failed:
             return
 
         if self.args.ign_ebind_all:
+            if not self.tcpsrv.srv:
+                for _ in range(self.broker.num_workers):
+                    self.broker.say("cb_httpsrv_up")
             return
 
         if self.args.ign_ebind and self.tcpsrv.srv:
             return
 
-        m = "{}/{} workers failed to start"
-        m = m.format(failed, expected)
-        self.log("root", m, 1)
+        t = "{}/{} workers failed to start"
+        t = t.format(failed, expected)
+        self.log("root", t, 1)
 
         self.retcode = 1
+        self.sigterm()
+
+    def sigterm(self) -> None:
         os.kill(os.getpid(), signal.SIGTERM)
 
-    def cb_httpsrv_up(self):
+    def cb_httpsrv_up(self) -> None:
         self.httpsrv_up += 1
         if self.httpsrv_up != self.broker.num_workers:
             return
 
         time.sleep(0.1)  # purely cosmetic dw
-        self.log("root", "workers OK\n")
+        if self.tcpsrv.qr:
+            self.log("qr-code", self.tcpsrv.qr)
+        else:
+            self.log("root", "workers OK\n")
+
         self.up2k.init_vols()
 
         thr = threading.Thread(target=self.sd_notify, name="sd-notify")
         thr.daemon = True
         thr.start()
 
-    def _logname(self):
+    def _logname(self) -> str:
         dt = datetime.utcnow()
-        fn = self.args.lo
+        fn = str(self.args.lo)
         for fs in "YmdHMS":
             fs = "%" + fs
             if fs in fn:
@@ -161,7 +243,7 @@ class SvcHub(object):
 
         return fn
 
-    def _setup_logfile(self, printed):
+    def _setup_logfile(self, printed: str) -> None:
         base_fn = fn = sel_fn = self._logname()
         if fn != self.args.lo:
             ctr = 0
@@ -183,8 +265,6 @@ class SvcHub(object):
 
             lh = codecs.open(fn, "w", encoding="utf-8", errors="replace")
 
-        lh.base_fn = base_fn
-
         argv = [sys.executable] + self.argv
         if hasattr(shlex, "quote"):
             argv = [shlex.quote(x) for x in argv]
@@ -193,14 +273,16 @@ class SvcHub(object):
 
         msg = "[+] opened logfile [{}]\n".format(fn)
         printed += msg
-        lh.write("t0: {:.3f}\nargv: {}\n\n{}".format(E.t0, " ".join(argv), printed))
+        t = "t0: {:.3f}\nargv: {}\n\n{}"
+        lh.write(t.format(self.E.t0, " ".join(argv), printed))
         self.logf = lh
+        self.logf_base_fn = base_fn
         print(msg, end="")
 
-    def run(self):
+    def run(self) -> None:
         self.tcpsrv.run()
 
-        thr = threading.Thread(target=self.thr_httpsrv_up)
+        thr = threading.Thread(target=self.thr_httpsrv_up, name="sig-hsrv-up2")
         thr.daemon = True
         thr.start()
 
@@ -228,21 +310,23 @@ class SvcHub(object):
                 pass
 
             self.shutdown()
-            thr.join()
+            # cant join; eats signals on win10
+            while not self.stopped:
+                time.sleep(0.1)
         else:
             self.stop_thr()
 
-    def reload(self):
+    def reload(self) -> str:
         if self.reloading:
             return "cannot reload; already in progress"
 
         self.reloading = True
-        t = threading.Thread(target=self._reload)
+        t = threading.Thread(target=self._reload, name="reloading")
         t.daemon = True
         t.start()
         return "reload initiated"
 
-    def _reload(self):
+    def _reload(self) -> None:
         self.log("root", "reload scheduled")
         with self.up2k.mutex:
             self.asrv.reload()
@@ -251,7 +335,7 @@ class SvcHub(object):
 
         self.reloading = False
 
-    def stop_thr(self):
+    def stop_thr(self) -> None:
         while not self.stop_req:
             with self.stop_cond:
                 self.stop_cond.wait(9001)
@@ -262,11 +346,24 @@ class SvcHub(object):
 
         self.shutdown()
 
-    def signal_handler(self, sig, frame):
+    def signal_handler(self, sig: int, frame: Optional[FrameType]) -> None:
         if self.stopping:
-            return
+            if self.nsigs <= 0:
+                try:
+                    threading.Thread(target=self.pr, args=("OMBO BREAKER",)).start()
+                    time.sleep(0.1)
+                except:
+                    pass
 
-        if sig == signal.SIGUSR1:
+                if ANYWIN:
+                    os.system("taskkill /f /pid {}".format(os.getpid()))
+                else:
+                    os.kill(os.getpid(), signal.SIGKILL)
+            else:
+                self.nsigs -= 1
+                return
+
+        if not ANYWIN and sig == signal.SIGUSR1:
             self.reload_req = True
         else:
             self.stop_req = True
@@ -274,7 +371,7 @@ class SvcHub(object):
         with self.stop_cond:
             self.stop_cond.notify_all()
 
-    def shutdown(self):
+    def shutdown(self) -> None:
         if self.stopping:
             return
 
@@ -287,9 +384,7 @@ class SvcHub(object):
 
         ret = 1
         try:
-            with self.log_mutex:
-                print("OPYTHAT")
-
+            self.pr("OPYTHAT")
             self.tcpsrv.shutdown()
             self.broker.shutdown()
             self.up2k.shutdown()
@@ -302,35 +397,39 @@ class SvcHub(object):
                         break
 
                     if n == 3:
-                        print("waiting for thumbsrv (10sec)...")
+                        self.pr("waiting for thumbsrv (10sec)...")
 
-            print("nailed it", end="")
+            self.pr("nailed it", end="")
             ret = self.retcode
+        except:
+            self.pr("\033[31m[ error during shutdown ]\n{}\033[0m".format(min_ex()))
+            raise
         finally:
             if self.args.wintitle:
                 print("\033]0;\033\\", file=sys.stderr, end="")
                 sys.stderr.flush()
 
-            print("\033[0m")
+            self.pr("\033[0m")
             if self.logf:
                 self.logf.close()
 
+            self.stopped = True
             sys.exit(ret)
 
-    def _log_disabled(self, src, msg, c=0):
+    def _log_disabled(self, src: str, msg: str, c: Union[int, str] = 0) -> None:
         if not self.logf:
             return
 
         with self.log_mutex:
             ts = datetime.utcnow().strftime("%Y-%m%d-%H%M%S.%f")[:-3]
-            self.logf.write("@{} [{}] {}\n".format(ts, src, msg))
+            self.logf.write("@{} [{}\033[0m] {}\n".format(ts, src, msg))
 
             now = time.time()
             if now >= self.next_day:
                 self._set_next_day()
 
-    def _set_next_day(self):
-        if self.next_day and self.logf and self.logf.base_fn != self._logname():
+    def _set_next_day(self) -> None:
+        if self.next_day and self.logf and self.logf_base_fn != self._logname():
             self.logf.close()
             self._setup_logfile("")
 
@@ -344,7 +443,7 @@ class SvcHub(object):
         dt = dt.replace(hour=0, minute=0, second=0)
         self.next_day = calendar.timegm(dt.utctimetuple())
 
-    def _log_enabled(self, src, msg, c=0):
+    def _log_enabled(self, src: str, msg: str, c: Union[int, str] = 0) -> None:
         """handles logging from all components"""
         with self.log_mutex:
             now = time.time()
@@ -381,34 +480,34 @@ class SvcHub(object):
             if self.logf:
                 self.logf.write(msg)
 
-    def check_mp_support(self):
-        vmin = sys.version_info[1]
-        if WINDOWS:
-            msg = "need python 3.3 or newer for multiprocessing;"
-            if PY2 or vmin < 3:
-                return msg
-        elif MACOS:
+    def pr(self, *a: Any, **ka: Any) -> None:
+        with self.log_mutex:
+            print(*a, **ka)
+
+    def check_mp_support(self) -> str:
+        if MACOS:
             return "multiprocessing is wonky on mac osx;"
-        else:
-            msg = "need python 3.3+ for multiprocessing;"
-            if PY2 or vmin < 3:
-                return msg
+        elif sys.version_info < (3, 3):
+            return "need python 3.3 or newer for multiprocessing;"
 
         try:
-            x = mp.Queue(1)
-            x.put(["foo", "bar"])
+            x: mp.Queue[tuple[str, str]] = mp.Queue(1)
+            x.put(("foo", "bar"))
             if x.get()[0] != "foo":
                 raise Exception()
         except:
             return "multiprocessing is not supported on your platform;"
 
-        return None
+        return ""
 
-    def check_mp_enable(self):
+    def check_mp_enable(self) -> bool:
         if self.args.j == 1:
             return False
 
-        if mp.cpu_count() <= 1:
+        try:
+            if mp.cpu_count() <= 1:
+                raise Exception()
+        except:
             self.log("svchub", "only one CPU detected; multiprocessing disabled")
             return False
 
@@ -427,21 +526,34 @@ class SvcHub(object):
             self.log("svchub", "cannot efficiently use multiple CPU cores")
             return False
 
-    def sd_notify(self):
+    def sd_notify(self) -> None:
         try:
-            addr = os.getenv("NOTIFY_SOCKET")
-            if not addr:
+            zb = os.getenv("NOTIFY_SOCKET")
+            if not zb:
                 return
 
-            addr = unicode(addr)
+            addr = unicode(zb)
             if addr.startswith("@"):
                 addr = "\0" + addr[1:]
 
-            m = "".join(x for x in addr if x in string.printable)
-            self.log("sd_notify", m)
+            t = "".join(x for x in addr if x in string.printable)
+            self.log("sd_notify", t)
 
             sck = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
             sck.connect(addr)
             sck.sendall(b"READY=1")
         except:
             self.log("sd_notify", min_ex())
+
+    def log_stacks(self) -> None:
+        td = time.time() - self.tstack
+        if td < 300:
+            self.log("stacks", "cooldown {}".format(td))
+            return
+
+        self.tstack = time.time()
+        zs = "{}\n{}".format(VERSIONS, alltrace())
+        zb = zs.encode("utf-8", "replace")
+        zb = gzip.compress(zb)
+        zs = base64.b64encode(zb).decode("ascii")
+        self.log("stacks", zs)

copyparty/szip.py

@@ -1,17 +1,23 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
-import os
+import calendar
 import time
 import zlib
-from datetime import datetime
 
-from .sutil import errdesc
-from .util import yieldfile, sanitize_fn, spack, sunpack
 from .bos import bos
+from .sutil import StreamArc, errdesc
+from .util import min_ex, sanitize_fn, spack, sunpack, yieldfile
+
+try:
+    from typing import Any, Generator, Optional
+
+    from .util import NamedLogger
+except:
+    pass
 
 
-def dostime2unix(buf):
+def dostime2unix(buf: bytes) -> int:
     t, d = sunpack(b"<HH", buf)
 
     ts = (t & 0x1F) * 2
@@ -26,27 +32,38 @@ def dostime2unix(buf):
     tf = "{:04d}-{:02d}-{:02d} {:02d}:{:02d}:{:02d}"
     iso = tf.format(*tt)
 
-    dt = datetime.strptime(iso, "%Y-%m-%d %H:%M:%S")
-    return int(dt.timestamp())
+    dt = time.strptime(iso, "%Y-%m-%d %H:%M:%S")
+    return int(calendar.timegm(dt))
 
 
-def unixtime2dos(ts):
-    tt = time.gmtime(ts)
+def unixtime2dos(ts: int) -> bytes:
+    tt = time.gmtime(ts + 1)
     dy, dm, dd, th, tm, ts = list(tt)[:6]
 
     bd = ((dy - 1980) << 9) + (dm << 5) + dd
     bt = (th << 11) + (tm << 5) + ts // 2
-    return spack(b"<HH", bt, bd)
+    try:
+        return spack(b"<HH", bt, bd)
+    except:
+        return b"\x00\x00\x21\x00"
 
 
-def gen_fdesc(sz, crc32, z64):
+def gen_fdesc(sz: int, crc32: int, z64: bool) -> bytes:
     ret = b"\x50\x4b\x07\x08"
     fmt = b"<LQQ" if z64 else b"<LLL"
     ret += spack(fmt, crc32, sz, sz)
     return ret
 
 
-def gen_hdr(h_pos, fn, sz, lastmod, utf8, crc32, pre_crc):
+def gen_hdr(
+    h_pos: Optional[int],
+    fn: str,
+    sz: int,
+    lastmod: int,
+    utf8: bool,
+    icrc32: int,
+    pre_crc: bool,
+) -> bytes:
     """
     does regular file headers
     and the central directory meme if h_pos is set
@@ -65,8 +82,8 @@ def gen_hdr(h_pos, fn, sz, lastmod, utf8, crc32, pre_crc):
     # confusingly this doesn't bump if h_pos
     req_ver = b"\x2d\x00" if z64 else b"\x0a\x00"
 
-    if crc32:
-        crc32 = spack(b"<L", crc32)
+    if icrc32:
+        crc32 = spack(b"<L", icrc32)
     else:
         crc32 = b"\x00" * 4
 
@@ -74,7 +91,7 @@ def gen_hdr(h_pos, fn, sz, lastmod, utf8, crc32, pre_crc):
         # 4b magic, 2b min-ver
         ret = b"\x50\x4b\x03\x04" + req_ver
     else:
-        # 4b magic, 2b spec-ver, 2b min-ver
+        # 4b magic, 2b spec-ver (1b compat, 1b os (00 dos, 03 unix)), 2b min-ver
         ret = b"\x50\x4b\x01\x02\x1e\x03" + req_ver
 
     ret += b"\x00" if pre_crc else b"\x08"  # streaming
@@ -93,30 +110,43 @@ def gen_hdr(h_pos, fn, sz, lastmod, utf8, crc32, pre_crc):
     fn = sanitize_fn(fn, "/", [])
     bfn = fn.encode("utf-8" if utf8 else "cp437", "replace").replace(b"?", b"_")
 
+    # add ntfs (0x24) and/or unix (0x10) extrafields for utc, add z64 if requested
     z64_len = len(z64v) * 8 + 4 if z64v else 0
-    ret += spack(b"<HH", len(bfn), z64_len)
+    ret += spack(b"<HH", len(bfn), 0x10 + z64_len)
 
     if h_pos is not None:
         # 2b comment, 2b diskno
         ret += b"\x00" * 4
 
         # 2b internal.attr, 4b external.attr
-        # infozip-macos: 0100 0000 a481 file:644
-        # infozip-macos: 0100 0100 0080 file:000
-        ret += b"\x01\x00\x00\x00\xa4\x81"
+        # infozip-macos: 0100 0000 a481 (spec-ver 1e03) file:644
+        # infozip-macos: 0100 0100 0080 (spec-ver 1e03) file:000
+        #     win10-zip: 0000 2000 0000 (spec-ver xx00) FILE_ATTRIBUTE_ARCHIVE
+        ret += b"\x00\x00\x00\x00\xa4\x81"  # unx
+        # ret += b"\x00\x00\x20\x00\x00\x00"  # fat
 
         # 4b local-header-ofs
         ret += spack(b"<L", min(h_pos, 0xFFFFFFFF))
 
     ret += bfn
 
+    # ntfs: type 0a, size 20, rsvd, attr1, len 18, mtime, atime, ctime
+    # b"\xa3\x2f\x82\x41\x55\x68\xd8\x01"  1652616838.798941100  ~5.861518  132970904387989411  ~58615181
+    # nt = int((lastmod + 11644473600) * 10000000)
+    # ret += spack(b"<HHLHHQQQ", 0xA, 0x20, 0, 1, 0x18, nt, nt, nt)
+
+    # unix: type 0d, size 0c, atime, mtime, uid, gid
+    ret += spack(b"<HHLLHH", 0xD, 0xC, int(lastmod), int(lastmod), 1000, 1000)
+
     if z64v:
         ret += spack(b"<HH" + b"Q" * len(z64v), 1, len(z64v) * 8, *z64v)
 
     return ret
 
 
-def gen_ecdr(items, cdir_pos, cdir_end):
+def gen_ecdr(
+    items: list[tuple[str, int, int, int, int]], cdir_pos: int, cdir_end: int
+) -> tuple[bytes, bool]:
     """
     summary of all file headers,
     usually the zipfile footer unless something clamps
@@ -141,10 +171,12 @@ def gen_ecdr(items, cdir_pos, cdir_end):
     # 2b comment length
     ret += b"\x00\x00"
 
-    return [ret, need_64]
+    return ret, need_64
 
 
-def gen_ecdr64(items, cdir_pos, cdir_end):
+def gen_ecdr64(
+    items: list[tuple[str, int, int, int, int]], cdir_pos: int, cdir_end: int
+) -> bytes:
     """
     z64 end of central directory
     added when numfiles or a headerptr clamps
@@ -168,7 +200,7 @@ def gen_ecdr64(items, cdir_pos, cdir_end):
     return ret
 
 
-def gen_ecdr64_loc(ecdr64_pos):
+def gen_ecdr64_loc(ecdr64_pos: int) -> bytes:
     """
     z64 end of central directory locator
     points to ecdr64
@@ -183,31 +215,36 @@ def gen_ecdr64_loc(ecdr64_pos):
     return ret
 
 
-class StreamZip(object):
-    def __init__(self, log, fgen, utf8=False, pre_crc=False):
-        self.log = log
-        self.fgen = fgen
+class StreamZip(StreamArc):
+    def __init__(
+        self,
+        log: "NamedLogger",
+        fgen: Generator[dict[str, Any], None, None],
+        utf8: bool = False,
+        pre_crc: bool = False,
+    ) -> None:
+        super(StreamZip, self).__init__(log, fgen)
+
         self.utf8 = utf8
         self.pre_crc = pre_crc
 
         self.pos = 0
-        self.items = []
+        self.items: list[tuple[str, int, int, int, int]] = []
 
-    def _ct(self, buf):
+    def _ct(self, buf: bytes) -> bytes:
         self.pos += len(buf)
         return buf
 
-    def ser(self, f):
+    def ser(self, f: dict[str, Any]) -> Generator[bytes, None, None]:
         name = f["vp"]
         src = f["ap"]
         st = f["st"]
 
         sz = st.st_size
-        ts = st.st_mtime + 1
+        ts = st.st_mtime
 
-        crc = None
+        crc = 0
         if self.pre_crc:
-            crc = 0
             for buf in yieldfile(src):
                 crc = zlib.crc32(buf, crc)
 
@@ -217,7 +254,6 @@ class StreamZip(object):
         buf = gen_hdr(None, name, sz, ts, self.utf8, crc, self.pre_crc)
         yield self._ct(buf)
 
-        crc = crc or 0
         for buf in yieldfile(src):
             if not self.pre_crc:
                 crc = zlib.crc32(buf, crc)
@@ -226,7 +262,7 @@ class StreamZip(object):
 
         crc &= 0xFFFFFFFF
 
-        self.items.append([name, sz, ts, crc, h_pos])
+        self.items.append((name, sz, ts, crc, h_pos))
 
         z64 = sz >= 4 * 1024 * 1024 * 1024
 
@@ -234,42 +270,46 @@ class StreamZip(object):
             buf = gen_fdesc(sz, crc, z64)
             yield self._ct(buf)
 
-    def gen(self):
+    def gen(self) -> Generator[bytes, None, None]:
         errors = []
-        for f in self.fgen:
-            if "err" in f:
-                errors.append([f["vp"], f["err"]])
-                continue
+        try:
+            for f in self.fgen:
+                if "err" in f:
+                    errors.append((f["vp"], f["err"]))
+                    continue
 
-            try:
-                for x in self.ser(f):
+                try:
+                    for x in self.ser(f):
+                        yield x
+                except GeneratorExit:
+                    raise
+                except:
+                    ex = min_ex(5, True).replace("\n", "\n-- ")
+                    errors.append((f["vp"], ex))
+
+            if errors:
+                errf, txt = errdesc(errors)
+                self.log("\n".join(([repr(errf)] + txt[1:])))
+                for x in self.ser(errf):
                     yield x
-            except Exception as ex:
-                errors.append([f["vp"], repr(ex)])
 
-        if errors:
-            errf, txt = errdesc(errors)
-            self.log("\n".join(([repr(errf)] + txt[1:])))
-            for x in self.ser(errf):
-                yield x
+            cdir_pos = self.pos
+            for name, sz, ts, crc, h_pos in self.items:
+                buf = gen_hdr(h_pos, name, sz, ts, self.utf8, crc, self.pre_crc)
+                yield self._ct(buf)
+            cdir_end = self.pos
 
-        cdir_pos = self.pos
-        for name, sz, ts, crc, h_pos in self.items:
-            buf = gen_hdr(h_pos, name, sz, ts, self.utf8, crc, self.pre_crc)
-            yield self._ct(buf)
-        cdir_end = self.pos
+            _, need_64 = gen_ecdr(self.items, cdir_pos, cdir_end)
+            if need_64:
+                ecdir64_pos = self.pos
+                buf = gen_ecdr64(self.items, cdir_pos, cdir_end)
+                yield self._ct(buf)
 
-        _, need_64 = gen_ecdr(self.items, cdir_pos, cdir_end)
-        if need_64:
-            ecdir64_pos = self.pos
-            buf = gen_ecdr64(self.items, cdir_pos, cdir_end)
-            yield self._ct(buf)
+                buf = gen_ecdr64_loc(ecdir64_pos)
+                yield self._ct(buf)
 
-            buf = gen_ecdr64_loc(ecdir64_pos)
-            yield self._ct(buf)
-
-        ecdr, _ = gen_ecdr(self.items, cdir_pos, cdir_end)
-        yield self._ct(ecdr)
-
-        if errors:
-            bos.unlink(errf["ap"])
+            ecdr, _ = gen_ecdr(self.items, cdir_pos, cdir_end)
+            yield self._ct(ecdr)
+        finally:
+            if errors:
+                bos.unlink(errf["ap"])

copyparty/tcpsrv.py

@@ -1,12 +1,17 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
+import os
 import re
-import sys
 import socket
+import sys
 
-from .__init__ import MACOS, ANYWIN, unicode
-from .util import chkcmd
+from .__init__ import ANYWIN, MACOS, PY2, TYPE_CHECKING, VT100, unicode
+from .stolen.qrcodegen import QrCode
+from .util import chkcmd, sunpack, termsize
+
+if TYPE_CHECKING:
+    from .svchub import SvcHub
 
 
 class TcpSrv(object):
@@ -15,16 +20,19 @@ class TcpSrv(object):
     which then uses the least busy HttpSrv to handle it
     """
 
-    def __init__(self, hub):
+    def __init__(self, hub: "SvcHub"):
         self.hub = hub
         self.args = hub.args
         self.log = hub.log
 
-        self.stopping = False
+        # mp-safe since issue6056
+        socket.setdefaulttimeout(120)
 
-        self.srv = []
+        self.stopping = False
+        self.srv: list[socket.socket] = []
         self.nsrv = 0
-        ok = {}
+        self.qr = ""
+        ok: dict[str, list[int]] = {}
         for ip in self.args.i:
             ok[ip] = []
             for port in self.args.p:
@@ -34,8 +42,8 @@ class TcpSrv(object):
                     ok[ip].append(port)
                 except Exception as ex:
                     if self.args.ign_ebind or self.args.ign_ebind_all:
-                        m = "could not listen on {}:{}: {}"
-                        self.log("tcpsrv", m.format(ip, port, ex), c=3)
+                        t = "could not listen on {}:{}: {}"
+                        self.log("tcpsrv", t.format(ip, port, ex), c=3)
                     else:
                         raise
 
@@ -54,10 +62,12 @@ class TcpSrv(object):
                 for x in nonlocals:
                     eps[x] = "external"
 
+        qr1: dict[str, list[int]] = {}
+        qr2: dict[str, list[int]] = {}
         msgs = []
-        title_tab = {}
+        title_tab: dict[str, dict[str, int]] = {}
         title_vars = [x[1:] for x in self.args.wintitle.split(" ") if x.startswith("$")]
-        m = "available @ {}://{}:{}/  (\033[33m{}\033[0m)"
+        t = "available @ {}://{}:{}/  (\033[33m{}\033[0m)"
         for ip, desc in sorted(eps.items(), key=lambda x: x[1]):
             for port in sorted(self.args.p):
                 if port not in ok.get(ip, ok.get("0.0.0.0", [])):
@@ -69,7 +79,14 @@ class TcpSrv(object):
                 elif self.args.https_only or port == 443:
                     proto = "https"
 
-                msgs.append(m.format(proto, ip, port, desc))
+                msgs.append(t.format(proto, ip, port, desc))
+
+                is_ext = "external" in unicode(desc)
+                qrt = qr1 if is_ext else qr2
+                try:
+                    qrt[ip].append(port)
+                except:
+                    qrt[ip] = [port]
 
                 if not self.args.wintitle:
                     continue
@@ -80,7 +97,7 @@ class TcpSrv(object):
                     ep = "{}:{}".format(ip, port)
 
                 hits = []
-                if "pub" in title_vars and "external" in unicode(desc):
+                if "pub" in title_vars and is_ext:
                     hits.append(("pub", ep))
 
                 if "pub" in title_vars or "all" in title_vars:
@@ -98,16 +115,20 @@ class TcpSrv(object):
 
         if msgs:
             msgs[-1] += "\n"
-            for m in msgs:
-                self.log("tcpsrv", m)
+            for t in msgs:
+                self.log("tcpsrv", t)
 
         if self.args.wintitle:
             self._set_wintitle(title_tab)
 
-    def _listen(self, ip, port):
+        if self.args.qr or self.args.qrs:
+            self.qr = self._qr(qr1, qr2)
+
+    def _listen(self, ip: str, port: int) -> None:
         srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
         srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
         srv.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
+        srv.settimeout(None)  # < does not inherit, ^ does
         try:
             srv.bind((ip, port))
             self.srv.append(srv)
@@ -120,19 +141,19 @@ class TcpSrv(object):
                 raise
             raise Exception(e)
 
-    def run(self):
+    def run(self) -> None:
         for srv in self.srv:
             srv.listen(self.args.nc)
             ip, port = srv.getsockname()
             fno = srv.fileno()
-            msg = "listening @ {}:{}  f{}".format(ip, port, fno)
+            msg = "listening @ {}:{}  f{} p{}".format(ip, port, fno, os.getpid())
             self.log("tcpsrv", msg)
             if self.args.q:
                 print(msg)
 
-            self.hub.broker.put(False, "listen", srv)
+            self.hub.broker.say("listen", srv)
 
-    def shutdown(self):
+    def shutdown(self) -> None:
         self.stopping = True
         try:
             for srv in self.srv:
@@ -142,30 +163,59 @@ class TcpSrv(object):
 
         self.log("tcpsrv", "ok bye")
 
-    def ips_linux(self):
-        eps = {}
+    def ips_linux_ifconfig(self) -> dict[str, str]:
+        # for termux
+        try:
+            txt, _ = chkcmd(["ifconfig"])
+        except:
+            return {}
+
+        eps: dict[str, str] = {}
+        dev = None
+        ip = None
+        up = None
+        for ln in (txt + "\n").split("\n"):
+            if not ln.strip() and dev and ip:
+                eps[ip] = dev + ("" if up else ", \033[31mLINK-DOWN")
+                dev = ip = up = None
+                continue
+
+            if ln == ln.lstrip():
+                dev = re.split(r"[: ]", ln)[0]
+
+            if "UP" in re.split(r"[<>, \t]", ln):
+                up = True
+
+            m = re.match(r"^\s+inet\s+([^ ]+)", ln)
+            if m:
+                ip = m.group(1)
+
+        return eps
+
+    def ips_linux(self) -> dict[str, str]:
         try:
             txt, _ = chkcmd(["ip", "addr"])
         except:
-            return eps
+            return self.ips_linux_ifconfig()
 
         r = re.compile(r"^\s+inet ([^ ]+)/.* (.*)")
         ri = re.compile(r"^\s*[0-9]+\s*:.*")
         up = False
+        eps: dict[str, str] = {}
         for ln in txt.split("\n"):
             if ri.match(ln):
                 up = "UP" in re.split("[>,< ]", ln)
 
             try:
-                ip, dev = r.match(ln.rstrip()).groups()
+                ip, dev = r.match(ln.rstrip()).groups()  # type: ignore
                 eps[ip] = dev + ("" if up else ", \033[31mLINK-DOWN")
             except:
                 pass
 
         return eps
 
-    def ips_macos(self):
-        eps = {}
+    def ips_macos(self) -> dict[str, str]:
+        eps: dict[str, str] = {}
         try:
             txt, _ = chkcmd(["ifconfig"])
         except:
@@ -173,7 +223,7 @@ class TcpSrv(object):
 
         rdev = re.compile(r"^([^ ]+):")
         rip = re.compile(r"^\tinet ([0-9\.]+) ")
-        dev = None
+        dev = "UNKNOWN"
         for ln in txt.split("\n"):
             m = rdev.match(ln)
             if m:
@@ -182,17 +232,17 @@ class TcpSrv(object):
             m = rip.match(ln)
             if m:
                 eps[m.group(1)] = dev
-                dev = None
+                dev = "UNKNOWN"
 
         return eps
 
-    def ips_windows_ipconfig(self):
-        eps = {}
-        offs = {}
+    def ips_windows_ipconfig(self) -> tuple[dict[str, str], set[str]]:
+        eps: dict[str, str] = {}
+        offs: set[str] = set()
         try:
             txt, _ = chkcmd(["ipconfig"])
         except:
-            return eps
+            return eps, offs
 
         rdev = re.compile(r"(^[^ ].*):$")
         rip = re.compile(r"^ +IPv?4? [^:]+: *([0-9\.]{7,15})$")
@@ -202,12 +252,12 @@ class TcpSrv(object):
             m = rdev.match(ln)
             if m:
                 if dev and dev not in eps.values():
-                    offs[dev] = 1
+                    offs.add(dev)
 
                 dev = m.group(1).split(" adapter ", 1)[-1]
 
             if dev and roff.match(ln):
-                offs[dev] = 1
+                offs.add(dev)
                 dev = None
 
             m = rip.match(ln)
@@ -216,12 +266,12 @@ class TcpSrv(object):
                 dev = None
 
         if dev and dev not in eps.values():
-            offs[dev] = 1
+            offs.add(dev)
 
         return eps, offs
 
-    def ips_windows_netsh(self):
-        eps = {}
+    def ips_windows_netsh(self) -> dict[str, str]:
+        eps: dict[str, str] = {}
         try:
             txt, _ = chkcmd("netsh interface ip show address".split())
         except:
@@ -241,7 +291,7 @@ class TcpSrv(object):
 
         return eps
 
-    def detect_interfaces(self, listen_ips):
+    def detect_interfaces(self, listen_ips: list[str]) -> dict[str, str]:
         if MACOS:
             eps = self.ips_macos()
         elif ANYWIN:
@@ -268,7 +318,6 @@ class TcpSrv(object):
         ]:
             try:
                 s.connect((ip, 1))
-                # raise OSError(13, "a")
                 default_route = s.getsockname()[0]
                 break
             except (OSError, socket.error) as ex:
@@ -289,25 +338,95 @@ class TcpSrv(object):
 
         return eps
 
-    def _set_wintitle(self, vars):
-        vars["all"] = vars.get("all", {"Local-Only": 1})
-        vars["pub"] = vars.get("pub", vars["all"])
+    def _set_wintitle(self, vs: dict[str, dict[str, int]]) -> None:
+        vs["all"] = vs.get("all", {"Local-Only": 1})
+        vs["pub"] = vs.get("pub", vs["all"])
 
-        vars2 = {}
-        for k, eps in vars.items():
-            vars2[k] = {
-                ep: 1
-                for ep in eps.keys()
-                if ":" not in ep or ep.split(":")[0] not in eps
-            }
+        vs2 = {}
+        for k, eps in vs.items():
+            filt = {ep: 1 for ep in eps if ":" not in ep}
+            have = set(filt)
+            for ep in sorted(eps):
+                ip = ep.split(":")[0]
+                if ip not in have:
+                    have.add(ip)
+                    filt[ep] = 1
+
+            lo = [x for x in filt if x.startswith("127.")]
+            if len(filt) > 3 and lo:
+                for ip in lo:
+                    filt.pop(ip)
+
+            vs2[k] = filt
 
         title = ""
-        vars = vars2
+        vs = vs2
         for p in self.args.wintitle.split(" "):
             if p.startswith("$"):
-                p = " and ".join(sorted(vars.get(p[1:], {"(None)": 1}).keys()))
+                seps = list(sorted(vs.get(p[1:], {"(None)": 1}).keys()))
+                p = ", ".join(seps[:3])
+                if len(seps) > 3:
+                    p += ", ..."
 
             title += "{} ".format(p)
 
         print("\033]0;{}\033\\".format(title), file=sys.stderr, end="")
         sys.stderr.flush()
+
+    def _qr(self, t1: dict[str, list[int]], t2: dict[str, list[int]]) -> str:
+        ip = None
+        for ip in list(t1) + list(t2):
+            if ip.startswith(self.args.qri):
+                break
+            ip = ""
+
+        if not ip:
+            # maybe /bin/ip is missing or smth
+            ip = self.args.qri
+
+        if not ip:
+            return ""
+
+        if self.args.http_only:
+            https = ""
+        elif self.args.https_only:
+            https = "s"
+        else:
+            https = "s" if self.args.qrs else ""
+
+        ports = t1.get(ip, t2.get(ip, []))
+        dport = 443 if https else 80
+        port = "" if dport in ports or not ports else ":{}".format(ports[0])
+        txt = "http{}://{}{}/{}".format(https, ip, port, self.args.qrl)
+
+        btxt = txt.encode("utf-8")
+        if PY2:
+            btxt = sunpack(b"B" * len(btxt), btxt)
+
+        fg = self.args.qr_fg
+        bg = self.args.qr_bg
+        pad = self.args.qrp
+        zoom = self.args.qrz
+        qrc = QrCode.encode_binary(btxt)
+        if zoom == 0:
+            try:
+                tw, th = termsize()
+                tsz = min(tw // 2, th)
+                zoom = 1 if qrc.size + pad * 2 >= tsz else 2
+            except:
+                zoom = 1
+
+        qr = qrc.render(zoom, pad)
+        if not VT100:
+            return "{}\n{}".format(txt, qr)
+
+        def ansify(m: re.Match) -> str:
+            t = "\033[40;48;5;{}m{}\033[47;48;5;{}m"
+            return t.format(fg, " " * len(m.group(1)), bg)
+
+        if zoom > 1:
+            qr = re.sub("(█+)", ansify, qr)
+
+        qr = qr.replace("\n", "\033[K\n") + "\033[K"  # win10do
+        t = "{} \033[0;38;5;{};48;5;{}m\033[J\n{}\033[999G\033[0m\033[J"
+        return t.format(txt, fg, bg, qr)

copyparty/th_cli.py

@@ -3,13 +3,23 @@ from __future__ import print_function, unicode_literals
 
 import os
 
-from .util import Cooldown
-from .th_srv import thumb_path, THUMBABLE, FMT_FFV, FMT_FFA
+from .__init__ import TYPE_CHECKING
+from .authsrv import VFS
 from .bos import bos
+from .th_srv import HAVE_WEBP, thumb_path
+from .util import Cooldown
+
+try:
+    from typing import Optional, Union
+except:
+    pass
+
+if TYPE_CHECKING:
+    from .httpsrv import HttpSrv
 
 
 class ThumbCli(object):
-    def __init__(self, hsrv):
+    def __init__(self, hsrv: "HttpSrv") -> None:
         self.broker = hsrv.broker
         self.log_func = hsrv.log
         self.args = hsrv.args
@@ -18,38 +28,67 @@ class ThumbCli(object):
         # cache on both sides for less broker spam
         self.cooldown = Cooldown(self.args.th_poke)
 
-    def log(self, msg, c=0):
+        try:
+            c = hsrv.th_cfg
+            if not c:
+                raise Exception()
+        except:
+            c = {k: {} for k in ["thumbable", "pil", "vips", "ffi", "ffv", "ffa"]}
+
+        self.thumbable = c["thumbable"]
+        self.fmt_pil = c["pil"]
+        self.fmt_vips = c["vips"]
+        self.fmt_ffi = c["ffi"]
+        self.fmt_ffv = c["ffv"]
+        self.fmt_ffa = c["ffa"]
+
+        # defer args.th_ff_jpg, can change at runtime
+        d = next((x for x in self.args.th_dec if x in ("vips", "pil")), None)
+        self.can_webp = HAVE_WEBP or d == "vips"
+
+    def log(self, msg: str, c: Union[int, str] = 0) -> None:
         self.log_func("thumbcli", msg, c)
 
-    def get(self, ptop, rem, mtime, fmt):
+    def get(self, dbv: VFS, rem: str, mtime: float, fmt: str) -> Optional[str]:
+        ptop = dbv.realpath
         ext = rem.rsplit(".")[-1].lower()
-        if ext not in THUMBABLE:
+        if ext not in self.thumbable or "dthumb" in dbv.flags:
             return None
 
-        is_vid = ext in FMT_FFV
-        if is_vid and self.args.no_vthumb:
+        is_vid = ext in self.fmt_ffv
+        if is_vid and "dvthumb" in dbv.flags:
             return None
 
         want_opus = fmt in ("opus", "caf")
-        is_au = ext in FMT_FFA
+        is_au = ext in self.fmt_ffa
         if is_au:
             if want_opus:
                 if self.args.no_acode:
                     return None
             else:
-                if self.args.no_athumb:
+                if "dathumb" in dbv.flags:
                     return None
         elif want_opus:
             return None
 
-        if rem.startswith(".hist/th/") and rem.split(".")[-1] in ["webp", "jpg"]:
+        is_img = not is_vid and not is_au
+        if is_img and "dithumb" in dbv.flags:
+            return None
+
+        preferred = self.args.th_dec[0] if self.args.th_dec else ""
+
+        if rem.startswith(".hist/th/") and rem.split(".")[-1] in ["webp", "jpg", "png"]:
             return os.path.join(ptop, rem)
 
         if fmt == "j" and self.args.th_no_jpg:
             fmt = "w"
 
         if fmt == "w":
-            if self.args.th_no_webp or ((is_vid or is_au) and self.args.th_ff_jpg):
+            if (
+                self.args.th_no_webp
+                or (is_img and not self.can_webp)
+                or (self.args.th_ff_jpg and (not is_img or preferred == "ff"))
+            ):
                 fmt = "j"
 
         histpath = self.asrv.vfs.histtab.get(ptop)
@@ -58,27 +97,41 @@ class ThumbCli(object):
             return None
 
         tpath = thumb_path(histpath, rem, mtime, fmt)
+        tpaths = [tpath]
+        if fmt == "w":
+            # also check for jpg (maybe webp is unavailable)
+            tpaths.append(tpath.rsplit(".", 1)[0] + ".jpg")
+
         ret = None
-        try:
-            st = bos.stat(tpath)
-            if st.st_size:
-                ret = tpath
-            else:
-                return None
-        except:
-            pass
+        abort = False
+        for tp in tpaths:
+            try:
+                st = bos.stat(tp)
+                if st.st_size:
+                    ret = tpath = tp
+                    fmt = ret.rsplit(".")[1]
+                else:
+                    abort = True
+            except:
+                pass
 
         if ret:
             tdir = os.path.dirname(tpath)
             if self.cooldown.poke(tdir):
-                self.broker.put(False, "thumbsrv.poke", tdir)
+                self.broker.say("thumbsrv.poke", tdir)
 
             if want_opus:
                 # audio files expire individually
                 if self.cooldown.poke(tpath):
-                    self.broker.put(False, "thumbsrv.poke", tpath)
+                    self.broker.say("thumbsrv.poke", tpath)
 
             return ret
 
-        x = self.broker.put(True, "thumbsrv.get", ptop, rem, mtime, fmt)
-        return x.get()
+        if abort:
+            return None
+
+        if not bos.path.getsize(os.path.join(ptop, rem)):
+            return None
+
+        x = self.broker.ask("thumbsrv.get", ptop, rem, mtime, fmt)
+        return x.get()  # type: ignore

copyparty/th_srv.py

@@ -1,19 +1,28 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
-import os
-import time
-import shutil
 import base64
 import hashlib
-import threading
+import os
+import shutil
 import subprocess as sp
+import threading
+import time
 
-from .__init__ import PY2, unicode
-from .util import fsenc, vsplit, statdir, runcmd, Queue, Cooldown, BytesIO, min_ex
+from queue import Queue
+
+from .__init__ import TYPE_CHECKING
 from .bos import bos
 from .mtag import HAVE_FFMPEG, HAVE_FFPROBE, ffprobe
+from .util import BytesIO, Cooldown, Pebkac, fsenc, min_ex, runcmd, statdir, vsplit
 
+try:
+    from typing import Optional, Union
+except:
+    pass
+
+if TYPE_CHECKING:
+    from .svchub import SvcHub
 
 HAVE_PIL = False
 HAVE_HEIF = False
@@ -21,7 +30,7 @@ HAVE_AVIF = False
 HAVE_WEBP = False
 
 try:
-    from PIL import Image, ImageOps, ExifTags
+    from PIL import ExifTags, Image, ImageOps
 
     HAVE_PIL = True
     try:
@@ -39,7 +48,7 @@ try:
         pass
 
     try:
-        import pillow_avif
+        import pillow_avif  # noqa: F401  # pylint: disable=unused-import
 
         HAVE_AVIF = True
     except:
@@ -47,34 +56,14 @@ try:
 except:
     pass
 
-# https://pillow.readthedocs.io/en/stable/handbook/image-file-formats.html
-# ffmpeg -formats
-FMT_PIL = "bmp dib gif icns ico jpg jpeg jp2 jpx pcx png pbm pgm ppm pnm sgi tga tif tiff webp xbm dds xpm"
-FMT_FFV = "av1 asf avi flv m4v mkv mjpeg mjpg mpg mpeg mpg2 mpeg2 h264 avc mts h265 hevc mov 3gp mp4 ts mpegts nut ogv ogm rm vob webm wmv"
-FMT_FFA = "aac m4a ogg opus flac alac mp3 mp2 ac3 dts wma ra wav aif aiff au alaw ulaw mulaw amr gsm ape tak tta wv"
-
-if HAVE_HEIF:
-    FMT_PIL += " heif heifs heic heics"
-
-if HAVE_AVIF:
-    FMT_PIL += " avif avifs"
-
-FMT_PIL, FMT_FFV, FMT_FFA = [
-    {x: True for x in y.split(" ") if x} for y in [FMT_PIL, FMT_FFV, FMT_FFA]
-]
+try:
+    HAVE_VIPS = True
+    import pyvips
+except:
+    HAVE_VIPS = False
 
 
-THUMBABLE = {}
-
-if HAVE_PIL:
-    THUMBABLE.update(FMT_PIL)
-
-if HAVE_FFMPEG and HAVE_FFPROBE:
-    THUMBABLE.update(FMT_FFV)
-    THUMBABLE.update(FMT_FFA)
-
-
-def thumb_path(histpath, rem, mtime, fmt):
+def thumb_path(histpath: str, rem: str, mtime: float, fmt: str) -> str:
     # base16 = 16 = 256
     # b64-lc = 38 = 1444
     # base64 = 64 = 4096
@@ -93,14 +82,14 @@ def thumb_path(histpath, rem, mtime, fmt):
     if fmt in ("opus", "caf"):
         cat = "ac"
     else:
-        fmt = "webp" if fmt == "w" else "jpg"
+        fmt = "webp" if fmt == "w" else "png" if fmt == "p" else "jpg"
         cat = "th"
 
     return "{}/{}/{}/{}.{:x}.{}".format(histpath, cat, rd, fn, int(mtime), fmt)
 
 
 class ThumbSrv(object):
-    def __init__(self, hub):
+    def __init__(self, hub: "SvcHub") -> None:
         self.hub = hub
         self.asrv = hub.asrv
         self.args = hub.args
@@ -111,17 +100,17 @@ class ThumbSrv(object):
         self.poke_cd = Cooldown(self.args.th_poke)
 
         self.mutex = threading.Lock()
-        self.busy = {}
+        self.busy: dict[str, list[threading.Condition]] = {}
         self.stopping = False
         self.nthr = max(1, self.args.th_mt)
 
-        self.q = Queue(self.nthr * 4)
+        self.q: Queue[Optional[tuple[str, str]]] = Queue(self.nthr * 4)
         for n in range(self.nthr):
-            t = threading.Thread(
+            thr = threading.Thread(
                 target=self.worker, name="thumb-{}-{}".format(n, self.nthr)
             )
-            t.daemon = True
-            t.start()
+            thr.daemon = True
+            thr.start()
 
         want_ff = not self.args.no_vthumb or not self.args.no_athumb
         if want_ff and (not HAVE_FFMPEG or not HAVE_FFPROBE):
@@ -141,19 +130,50 @@ class ThumbSrv(object):
             t.daemon = True
             t.start()
 
-    def log(self, msg, c=0):
+        self.fmt_pil, self.fmt_vips, self.fmt_ffi, self.fmt_ffv, self.fmt_ffa = [
+            set(y.split(","))
+            for y in [
+                self.args.th_r_pil,
+                self.args.th_r_vips,
+                self.args.th_r_ffi,
+                self.args.th_r_ffv,
+                self.args.th_r_ffa,
+            ]
+        ]
+
+        if not HAVE_HEIF:
+            for f in "heif heifs heic heics".split(" "):
+                self.fmt_pil.discard(f)
+
+        if not HAVE_AVIF:
+            for f in "avif avifs".split(" "):
+                self.fmt_pil.discard(f)
+
+        self.thumbable: set[str] = set()
+
+        if "pil" in self.args.th_dec:
+            self.thumbable |= self.fmt_pil
+
+        if "vips" in self.args.th_dec:
+            self.thumbable |= self.fmt_vips
+
+        if "ff" in self.args.th_dec:
+            for zss in [self.fmt_ffi, self.fmt_ffv, self.fmt_ffa]:
+                self.thumbable |= zss
+
+    def log(self, msg: str, c: Union[int, str] = 0) -> None:
         self.log_func("thumb", msg, c)
 
-    def shutdown(self):
+    def shutdown(self) -> None:
         self.stopping = True
         for _ in range(self.nthr):
             self.q.put(None)
 
-    def stopped(self):
+    def stopped(self) -> bool:
         with self.mutex:
             return not self.nthr
 
-    def get(self, ptop, rem, mtime, fmt):
+    def get(self, ptop: str, rem: str, mtime: float, fmt: str) -> Optional[str]:
         histpath = self.asrv.vfs.histtab.get(ptop)
         if not histpath:
             self.log("no histpath for [{}]".format(ptop))
@@ -180,7 +200,7 @@ class ThumbSrv(object):
                 do_conv = True
 
         if do_conv:
-            self.q.put([abspath, tpath])
+            self.q.put((abspath, tpath))
             self.log("conv {} \033[0m{}".format(tpath, abspath), c=6)
 
         while not self.stopping:
@@ -201,7 +221,17 @@ class ThumbSrv(object):
 
         return None
 
-    def worker(self):
+    def getcfg(self) -> dict[str, set[str]]:
+        return {
+            "thumbable": self.thumbable,
+            "pil": self.fmt_pil,
+            "vips": self.fmt_vips,
+            "ffi": self.fmt_ffi,
+            "ffv": self.fmt_ffv,
+            "ffa": self.fmt_ffa,
+        }
+
+    def worker(self) -> None:
         while not self.stopping:
             task = self.q.get()
             if not task:
@@ -209,26 +239,47 @@ class ThumbSrv(object):
 
             abspath, tpath = task
             ext = abspath.split(".")[-1].lower()
+            png_ok = False
             fun = None
             if not bos.path.exists(tpath):
-                if ext in FMT_PIL:
-                    fun = self.conv_pil
-                elif ext in FMT_FFV:
-                    fun = self.conv_ffmpeg
-                elif ext in FMT_FFA:
-                    if tpath.endswith(".opus") or tpath.endswith(".caf"):
-                        fun = self.conv_opus
-                    else:
-                        fun = self.conv_spec
+                for lib in self.args.th_dec:
+                    if fun:
+                        break
+                    elif lib == "pil" and ext in self.fmt_pil:
+                        fun = self.conv_pil
+                    elif lib == "vips" and ext in self.fmt_vips:
+                        fun = self.conv_vips
+                    elif lib == "ff" and ext in self.fmt_ffi or ext in self.fmt_ffv:
+                        fun = self.conv_ffmpeg
+                    elif lib == "ff" and ext in self.fmt_ffa:
+                        if tpath.endswith(".opus") or tpath.endswith(".caf"):
+                            fun = self.conv_opus
+                        elif tpath.endswith(".png"):
+                            fun = self.conv_waves
+                            png_ok = True
+                        else:
+                            fun = self.conv_spec
+
+            if not png_ok and tpath.endswith(".png"):
+                raise Pebkac(400, "png only allowed for waveforms")
 
             if fun:
                 try:
                     fun(abspath, tpath)
-                except:
+                except Exception as ex:
                     msg = "{} could not create thumbnail of {}\n{}"
-                    self.log(msg.format(fun.__name__, abspath, min_ex()), "1;30")
-                    with open(tpath, "wb") as _:
-                        pass
+                    msg = msg.format(fun.__name__, abspath, min_ex())
+                    c: Union[str, int] = 1 if "<Signals.SIG" in msg else "1;30"
+                    self.log(msg, c)
+                    if getattr(ex, "returncode", 0) != 321:
+                        with open(tpath, "wb") as _:
+                            pass
+                    else:
+                        # ffmpeg may spawn empty files on windows
+                        try:
+                            os.unlink(tpath)
+                        except:
+                            pass
 
             with self.mutex:
                 subs = self.busy[tpath]
@@ -241,7 +292,7 @@ class ThumbSrv(object):
         with self.mutex:
             self.nthr -= 1
 
-    def fancy_pillow(self, im):
+    def fancy_pillow(self, im: "Image.Image") -> "Image.Image":
         # exif_transpose is expensive (loads full image + unconditional copy)
         r = max(*self.res) * 2
         im.thumbnail((r, r), resample=Image.LANCZOS)
@@ -267,7 +318,7 @@ class ThumbSrv(object):
 
         return im
 
-    def conv_pil(self, abspath, tpath):
+    def conv_pil(self, abspath: str, tpath: str) -> None:
         with Image.open(fsenc(abspath)) as im:
             try:
                 im = self.fancy_pillow(im)
@@ -296,16 +347,35 @@ class ThumbSrv(object):
 
             im.save(tpath, **args)
 
-    def conv_ffmpeg(self, abspath, tpath):
-        ret, _ = ffprobe(abspath)
+    def conv_vips(self, abspath: str, tpath: str) -> None:
+        crops = ["centre", "none"]
+        if self.args.th_no_crop:
+            crops = ["none"]
 
-        ext = abspath.rsplit(".")[-1]
-        if ext in ["h264", "h265"]:
-            seek = []
+        w, h = self.res
+        kw = {"height": h, "size": "down", "intent": "relative"}
+
+        for c in crops:
+            try:
+                kw["crop"] = c
+                img = pyvips.Image.thumbnail(abspath, w, **kw)
+                break
+            except:
+                pass
+
+        img.write_to_file(tpath, Q=40)
+
+    def conv_ffmpeg(self, abspath: str, tpath: str) -> None:
+        ret, _ = ffprobe(abspath, int(self.args.th_convt / 2))
+        if not ret:
+            return
+
+        ext = abspath.rsplit(".")[-1].lower()
+        if ext in ["h264", "h265"] or ext in self.fmt_ffi:
+            seek: list[bytes] = []
         else:
             dur = ret[".dur"][1] if ".dur" in ret else 4
-            seek = "{:.0f}".format(dur / 3)
-            seek = [b"-ss", seek.encode("utf-8")]
+            seek = [b"-ss", "{:.0f}".format(dur / 3).encode("utf-8")]
 
         scale = "scale={0}:{1}:force_original_aspect_ratio="
         if self.args.th_no_crop:
@@ -313,7 +383,7 @@ class ThumbSrv(object):
         else:
             scale += "increase,crop={0}:{1},setsar=1:1"
 
-        scale = scale.format(*list(self.res)).encode("utf-8")
+        bscale = scale.format(*list(self.res)).encode("utf-8")
         # fmt: off
         cmd = [
             b"ffmpeg",
@@ -325,7 +395,7 @@ class ThumbSrv(object):
         cmd += [
             b"-i", fsenc(abspath),
             b"-map", b"0:v:0",
-            b"-vf", scale,
+            b"-vf", bscale,
             b"-frames:v", b"1",
             b"-metadata:s:v:0", b"rotate=0",
         ]
@@ -347,17 +417,81 @@ class ThumbSrv(object):
         cmd += [fsenc(tpath)]
         self._run_ff(cmd)
 
-    def _run_ff(self, cmd):
+    def _run_ff(self, cmd: list[bytes]) -> None:
         # self.log((b" ".join(cmd)).decode("utf-8"))
-        ret, sout, serr = runcmd(cmd, timeout=self.args.th_convt)
-        if ret != 0:
-            m = "FFmpeg failed (probably a corrupt video file):\n"
-            m += "\n".join(["ff: {}".format(x) for x in serr.split("\n")])
-            self.log(m, c="1;30")
-            raise sp.CalledProcessError(ret, (cmd[0], b"...", cmd[-1]))
+        ret, _, serr = runcmd(cmd, timeout=self.args.th_convt)
+        if not ret:
+            return
 
-    def conv_spec(self, abspath, tpath):
-        ret, _ = ffprobe(abspath)
+        c: Union[str, int] = "1;30"
+        t = "FFmpeg failed (probably a corrupt video file):\n"
+        if (
+            (not self.args.th_ff_jpg or time.time() - int(self.args.th_ff_jpg) < 60)
+            and cmd[-1].lower().endswith(b".webp")
+            and (
+                "Error selecting an encoder" in serr
+                or "Automatic encoder selection failed" in serr
+                or "Default encoder for format webp" in serr
+                or "Please choose an encoder manually" in serr
+            )
+        ):
+            self.args.th_ff_jpg = time.time()
+            t = "FFmpeg failed because it was compiled without libwebp; enabling --th-ff-jpg to force jpeg output:\n"
+            ret = 321
+            c = 1
+
+        if (
+            not self.args.th_ff_swr or time.time() - int(self.args.th_ff_swr) < 60
+        ) and (
+            "Requested resampling engine is unavailable" in serr
+            or "output pad on Parsed_aresample_" in serr
+        ):
+            self.args.th_ff_swr = time.time()
+            t = "FFmpeg failed because it was compiled without libsox; enabling --th-ff-swr to force swr resampling:\n"
+            ret = 321
+            c = 1
+
+        lines = serr.strip("\n").split("\n")
+        if len(lines) > 50:
+            lines = lines[:25] + ["[...]"] + lines[-25:]
+
+        txt = "\n".join(["ff: " + str(x) for x in lines])
+        if len(txt) > 5000:
+            txt = txt[:2500] + "...\nff: [...]\nff: ..." + txt[-2500:]
+
+        self.log(t + txt, c=c)
+        raise sp.CalledProcessError(ret, (cmd[0], b"...", cmd[-1]))
+
+    def conv_waves(self, abspath: str, tpath: str) -> None:
+        ret, _ = ffprobe(abspath, int(self.args.th_convt / 2))
+        if "ac" not in ret:
+            raise Exception("not audio")
+
+        flt = (
+            b"[0:a:0]"
+            b"compand=.3|.3:1|1:-90/-60|-60/-40|-40/-30|-20/-20:6:0:-90:0.2"
+            b",volume=2"
+            b",showwavespic=s=2048x64:colors=white"
+            b",convolution=1 1 1 1 1 1 1 1 1:1 1 1 1 1 1 1 1 1:1 1 1 1 1 1 1 1 1:1 -1 1 -1 5 -1 1 -1 1"  # idk what im doing but it looks ok
+        )
+
+        # fmt: off
+        cmd = [
+            b"ffmpeg",
+            b"-nostdin",
+            b"-v", b"error",
+            b"-hide_banner",
+            b"-i", fsenc(abspath),
+            b"-filter_complex", flt,
+            b"-frames:v", b"1",
+        ]
+        # fmt: on
+
+        cmd += [fsenc(tpath)]
+        self._run_ff(cmd)
+
+    def conv_spec(self, abspath: str, tpath: str) -> None:
+        ret, _ = ffprobe(abspath, int(self.args.th_convt / 2))
         if "ac" not in ret:
             raise Exception("not audio")
 
@@ -378,7 +512,8 @@ class ThumbSrv(object):
             b"-hide_banner",
             b"-i", fsenc(abspath),
             b"-filter_complex", fc.encode("utf-8"),
-            b"-map", b"[o]"
+            b"-map", b"[o]",
+            b"-frames:v", b"1",
         ]
         # fmt: on
 
@@ -398,11 +533,11 @@ class ThumbSrv(object):
         cmd += [fsenc(tpath)]
         self._run_ff(cmd)
 
-    def conv_opus(self, abspath, tpath):
+    def conv_opus(self, abspath: str, tpath: str) -> None:
         if self.args.no_acode:
             raise Exception("disabled in server config")
 
-        ret, _ = ffprobe(abspath)
+        ret, _ = ffprobe(abspath, int(self.args.th_convt / 2))
         if "ac" not in ret:
             raise Exception("not audio")
 
@@ -446,7 +581,7 @@ class ThumbSrv(object):
             # fmt: on
             self._run_ff(cmd)
 
-    def poke(self, tdir):
+    def poke(self, tdir: str) -> None:
         if not self.poke_cd.poke(tdir):
             return
 
@@ -458,7 +593,7 @@ class ThumbSrv(object):
         except:
             pass
 
-    def cleaner(self):
+    def cleaner(self) -> None:
         interval = self.args.th_clean
         while True:
             time.sleep(interval)
@@ -473,26 +608,27 @@ class ThumbSrv(object):
 
             self.log("\033[Jcln ok; rm {} dirs".format(ndirs))
 
-    def clean(self, histpath):
+    def clean(self, histpath: str) -> int:
         ret = 0
         for cat in ["th", "ac"]:
-            ret += self._clean(histpath, cat, None)
+            top = os.path.join(histpath, cat)
+            if not bos.path.isdir(top):
+                continue
+
+            ret += self._clean(cat, top)
 
         return ret
 
-    def _clean(self, histpath, cat, thumbpath):
-        if not thumbpath:
-            thumbpath = os.path.join(histpath, cat)
-
+    def _clean(self, cat: str, thumbpath: str) -> int:
         # self.log("cln {}".format(thumbpath))
-        exts = ["jpg", "webp"] if cat == "th" else ["opus", "caf"]
+        exts = ["jpg", "webp", "png"] if cat == "th" else ["opus", "caf"]
         maxage = getattr(self.args, cat + "_maxage")
         now = time.time()
         prev_b64 = None
-        prev_fp = None
+        prev_fp = ""
         try:
-            ents = statdir(self.log, not self.args.no_scandir, False, thumbpath)
-            ents = sorted(list(ents))
+            t1 = statdir(self.log_func, not self.args.no_scandir, False, thumbpath)
+            ents = sorted(list(t1))
         except:
             return 0
 
@@ -507,7 +643,7 @@ class ThumbSrv(object):
                 if age > maxage:
                     with self.mutex:
                         safe = True
-                        for k in self.busy.keys():
+                        for k in self.busy:
                             if k.lower().replace("\\", "/").startswith(cmp):
                                 safe = False
                                 break
@@ -517,7 +653,7 @@ class ThumbSrv(object):
                             self.log("rm -rf [{}]".format(fp))
                             shutil.rmtree(fp, ignore_errors=True)
                 else:
-                    self._clean(histpath, cat, fp)
+                    ndirs += self._clean(cat, fp)
 
                 continue
 

copyparty/u2idx.py

@@ -1,28 +1,37 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
-import re
+import calendar
 import os
-import time
+import re
 import threading
-from datetime import datetime
+import time
 from operator import itemgetter
 
-from .__init__ import ANYWIN, unicode
-from .util import absreal, s3dec, Pebkac, min_ex, gen_filekey, quotep
+from .__init__ import ANYWIN, TYPE_CHECKING, unicode
 from .bos import bos
 from .up2k import up2k_wark_from_hashlist
+from .util import HAVE_SQLITE3, Pebkac, absreal, gen_filekey, min_ex, quotep, s3dec
 
+if HAVE_SQLITE3:
+    import sqlite3
 
 try:
-    HAVE_SQLITE3 = True
-    import sqlite3
+    from pathlib import Path
 except:
-    HAVE_SQLITE3 = False
+    pass
+
+try:
+    from typing import Any, Optional, Union
+except:
+    pass
+
+if TYPE_CHECKING:
+    from .httpconn import HttpConn
 
 
 class U2idx(object):
-    def __init__(self, conn):
+    def __init__(self, conn: "HttpConn") -> None:
         self.log_func = conn.log_func
         self.asrv = conn.asrv
         self.args = conn.args
@@ -32,17 +41,21 @@ class U2idx(object):
             self.log("your python does not have sqlite3; searching will be disabled")
             return
 
-        self.cur = {}
-        self.mem_cur = sqlite3.connect(":memory:")
+        self.active_id = ""
+        self.active_cur: Optional["sqlite3.Cursor"] = None
+        self.cur: dict[str, "sqlite3.Cursor"] = {}
+        self.mem_cur = sqlite3.connect(":memory:").cursor()
         self.mem_cur.execute(r"create table a (b text)")
 
-        self.p_end = None
-        self.p_dur = 0
+        self.p_end = 0.0
+        self.p_dur = 0.0
 
-    def log(self, msg, c=0):
+    def log(self, msg: str, c: Union[int, str] = 0) -> None:
         self.log_func("u2idx", msg, c)
 
-    def fsearch(self, vols, body):
+    def fsearch(
+        self, vols: list[tuple[str, str, dict[str, Any]]], body: dict[str, Any]
+    ) -> list[dict[str, Any]]:
         """search by up2k hashlist"""
         if not HAVE_SQLITE3:
             return []
@@ -52,14 +65,14 @@ class U2idx(object):
         wark = up2k_wark_from_hashlist(self.args.salt, fsize, fhash)
 
         uq = "substr(w,1,16) = ? and w = ?"
-        uv = [wark[:16], wark]
+        uv: list[Union[str, int]] = [wark[:16], wark]
 
         try:
-            return self.run_query(vols, uq, uv, True, False)[0]
+            return self.run_query(vols, uq, uv, True, False, 99999)[0]
         except:
             raise Pebkac(500, min_ex())
 
-    def get_cur(self, ptop):
+    def get_cur(self, ptop: str) -> Optional["sqlite3.Cursor"]:
         if not HAVE_SQLITE3:
             return None
 
@@ -76,17 +89,35 @@ class U2idx(object):
         if not bos.path.exists(db_path):
             return None
 
-        cur = sqlite3.connect(db_path, 2).cursor()
+        cur = None
+        if ANYWIN:
+            uri = ""
+            try:
+                uri = "{}?mode=ro&nolock=1".format(Path(db_path).as_uri())
+                cur = sqlite3.connect(uri, 2, uri=True).cursor()
+                self.log("ro: {}".format(db_path))
+            except:
+                self.log("could not open read-only: {}\n{}".format(uri, min_ex()))
+
+        if not cur:
+            # on windows, this steals the write-lock from up2k.deferred_init --
+            # seen on win 10.0.17763.2686, py 3.10.4, sqlite 3.37.2
+            cur = sqlite3.connect(db_path, 2).cursor()
+            self.log("opened {}".format(db_path))
+
         self.cur[ptop] = cur
         return cur
 
-    def search(self, vols, uq):
+    def search(
+        self, vols: list[tuple[str, str, dict[str, Any]]], uq: str, lim: int
+    ) -> tuple[list[dict[str, Any]], list[str]]:
         """search by query params"""
         if not HAVE_SQLITE3:
-            return []
+            return [], []
 
         q = ""
-        va = []
+        v: Union[str, int] = ""
+        va: list[Union[str, int]] = []
         have_up = False  # query has up.* operands
         have_mt = False
         is_key = True
@@ -169,18 +200,17 @@ class U2idx(object):
 
             if is_date:
                 is_date = False
-                v = v.upper().rstrip("Z").replace(",", " ").replace("T", " ")
-                while "  " in v:
-                    v = v.replace("  ", " ")
-
+                v = re.sub(r"[tzTZ, ]+", " ", v).strip()
                 for fmt in [
                     "%Y-%m-%d %H:%M:%S",
                     "%Y-%m-%d %H:%M",
                     "%Y-%m-%d %H",
                     "%Y-%m-%d",
+                    "%Y-%m",
+                    "%Y",
                 ]:
                     try:
-                        v = datetime.strptime(v, fmt).timestamp()
+                        v = calendar.timegm(time.strptime(str(v), fmt))
                         break
                     except:
                         pass
@@ -208,11 +238,12 @@ class U2idx(object):
 
             # lowercase tag searches
             m = ptn_lc.search(q)
-            if not m or not ptn_lcv.search(unicode(v)):
+            zs = unicode(v)
+            if not m or not ptn_lcv.search(zs):
                 continue
 
             va.pop()
-            va.append(v.lower())
+            va.append(zs.lower())
             q = q[: m.start()]
 
             field, oper = m.groups()
@@ -222,12 +253,20 @@ class U2idx(object):
                 q += " lower({}) {} ? ) ".format(field, oper)
 
         try:
-            return self.run_query(vols, q, va, have_up, have_mt)
+            return self.run_query(vols, q, va, have_up, have_mt, lim)
         except Exception as ex:
             raise Pebkac(500, repr(ex))
 
-    def run_query(self, vols, uq, uv, have_up, have_mt):
-        done_flag = []
+    def run_query(
+        self,
+        vols: list[tuple[str, str, dict[str, Any]]],
+        uq: str,
+        uv: list[Union[str, int]],
+        have_up: bool,
+        have_mt: bool,
+        lim: int,
+    ) -> tuple[list[dict[str, Any]], list[str]]:
+        done_flag: list[bool] = []
         self.active_id = "{:.6f}_{}".format(
             time.time(), threading.current_thread().ident
         )
@@ -244,18 +283,16 @@ class U2idx(object):
 
         if not uq or not uv:
             uq = "select * from up"
-            uv = ()
+            uv = []
         elif have_mt:
             uq = "select up.*, substr(up.w,1,16) mtw from up where " + uq
-            uv = tuple(uv)
         else:
             uq = "select up.* from up where " + uq
-            uv = tuple(uv)
 
         self.log("qs: {!r} {!r}".format(uq, uv))
 
         ret = []
-        lim = int(self.args.srch_hits)
+        lim = min(lim, int(self.args.srch_hits))
         taglist = {}
         for (vtop, ptop, flags) in vols:
             cur = self.get_cur(ptop)
@@ -270,15 +307,14 @@ class U2idx(object):
                     v = vtop + "/"
 
                 vuv.append(v)
-            vuv = tuple(vuv)
 
             sret = []
             fk = flags.get("fk")
-            c = cur.execute(uq, vuv)
+            c = cur.execute(uq, tuple(vuv))
             for hit in c:
                 w, ts, sz, rd, fn, ip, at = hit[:7]
                 lim -= 1
-                if lim <= 0:
+                if lim < 0:
                     break
 
                 if rd.startswith("//") or fn.startswith("//"):
@@ -307,7 +343,7 @@ class U2idx(object):
                 w = hit["w"]
                 del hit["w"]
                 tags = {}
-                q2 = "select k, v from mt where w = ? and k != 'x'"
+                q2 = "select k, v from mt where w = ? and +k != 'x'"
                 for k, v2 in cur.execute(q2, (w,)):
                     taglist[k] = True
                     tags[k] = v2
@@ -318,7 +354,7 @@ class U2idx(object):
             # print("[{}] {}".format(ptop, sret))
 
         done_flag.append(True)
-        self.active_id = None
+        self.active_id = ""
 
         # undupe hits from multiple metadata keys
         if len(ret) > 1:
@@ -332,11 +368,12 @@ class U2idx(object):
 
         return ret, list(taglist.keys())
 
-    def terminator(self, identifier, done_flag):
+    def terminator(self, identifier: str, done_flag: list[bool]) -> None:
         for _ in range(self.timeout):
             time.sleep(1)
             if done_flag:
                 return
 
         if identifier == self.active_id:
+            assert self.active_cur
             self.active_cur.connection.interrupt()

copyparty/web/baguettebox.js

@@ -17,12 +17,11 @@ window.baguetteBox = (function () {
             titleTag: false,
             async: false,
             preload: 2,
-            animation: 'slideIn',
             afterShow: null,
             afterHide: null,
             onChange: null,
         },
-        overlay, slider, btnPrev, btnNext, btnHelp, btnRotL, btnRotR, btnSel, btnVmode, btnClose,
+        overlay, slider, btnPrev, btnNext, btnHelp, btnAnim, btnRotL, btnRotR, btnSel, btnFull, btnVmode, btnClose,
         currentGallery = [],
         currentIndex = 0,
         isOverlayVisible = false,
@@ -30,13 +29,17 @@ window.baguetteBox = (function () {
         touchFlag = false,  // busy
         re_i = /.+\.(gif|jpe?g|png|webp)(\?|$)/i,
         re_v = /.+\.(webm|mp4)(\?|$)/i,
+        anims = ['slideIn', 'fadeIn', 'none'],
         data = {},  // all galleries
         imagesElements = [],
         documentLastFocus = null,
         isFullscreen = false,
         vmute = false,
-        vloop = false,
-        vnext = false,
+        vloop = sread('vmode') == 'L',
+        vnext = sread('vmode') == 'C',
+        loopA = null,
+        loopB = null,
+        url_ts = null,
         resume_mp = false;
 
     var onFSC = function (e) {
@@ -178,9 +181,11 @@ window.baguetteBox = (function () {
                 '<button id="bbox-next" class="bbox-btn" type="button" aria-label="Next">&gt;</button>' +
                 '<div id="bbox-btns">' +
                 '<button id="bbox-help" type="button">?</button>' +
+                '<button id="bbox-anim" type="button" tt="a">-</button>' +
                 '<button id="bbox-rotl" type="button">↶</button>' +
                 '<button id="bbox-rotr" type="button">↷</button>' +
                 '<button id="bbox-tsel" type="button">sel</button>' +
+                '<button id="bbox-full" type="button">⛶</button>' +
                 '<button id="bbox-vmode" type="button" tt="a"></button>' +
                 '<button id="bbox-close" type="button" aria-label="Close">X</button>' +
                 '</div></div>'
@@ -193,12 +198,13 @@ window.baguetteBox = (function () {
         btnPrev = ebi('bbox-prev');
         btnNext = ebi('bbox-next');
         btnHelp = ebi('bbox-help');
+        btnAnim = ebi('bbox-anim');
         btnRotL = ebi('bbox-rotl');
         btnRotR = ebi('bbox-rotr');
         btnSel = ebi('bbox-tsel');
+        btnFull = ebi('bbox-full');
         btnVmode = ebi('bbox-vmode');
         btnClose = ebi('bbox-close');
-        bindEvents();
     }
 
     function halp() {
@@ -213,23 +219,26 @@ window.baguetteBox = (function () {
             ['home', 'first file'],
             ['end', 'last file'],
             ['R', 'rotate (shift=ccw)'],
+            ['F', 'toggle fullscreen'],
             ['S', 'toggle file selection'],
             ['space, P, K', 'video: play / pause'],
             ['U', 'video: seek 10sec back'],
             ['P', 'video: seek 10sec ahead'],
+            ['0..9', 'video: seek 0%..90%'],
             ['M', 'video: toggle mute'],
             ['V', 'video: toggle loop'],
             ['C', 'video: toggle auto-next'],
-            ['F', 'video: toggle fullscreen'],
+            ['<code>[</code>, <code>]</code>', 'video: loop start / end'],
         ],
-            d = mknod('table'),
+            d = mknod('table', 'bbox-halp'),
             html = ['<tbody>'];
 
         for (var a = 0; a < list.length; a++)
             html.push('<tr><td>' + list[a][0] + '</td><td>' + list[a][1] + '</td></tr>');
 
+        html.push('<tr><td colspan="2">tap middle of img to hide btns</td></tr>');
+        html.push('<tr><td colspan="2">tap left/right sides for prev/next</td></tr>');
         d.innerHTML = html.join('\n') + '</tbody>';
-        d.setAttribute('id', 'bbox-halp');
         d.onclick = function () {
             overlay.removeChild(d);
         };
@@ -237,12 +246,18 @@ window.baguetteBox = (function () {
     }
 
     function keyDownHandler(e) {
-        if (e.ctrlKey || e.altKey || e.metaKey || e.isComposing || modal.busy)
+        if (anymod(e, true) || modal.busy)
             return;
 
-        var k = e.code + '', v = vid();
+        var k = e.code + '', v = vid(), pos = -1;
 
-        if (k == "ArrowLeft" || k == "KeyJ")
+        if (k == "BracketLeft")
+            setloop(1);
+        else if (k == "BracketRight")
+            setloop(2);
+        else if (e.shiftKey)
+            return;
+        else if (k == "ArrowLeft" || k == "KeyJ")
             showPreviousImage();
         else if (k == "ArrowRight" || k == "KeyL")
             showNextImage();
@@ -256,6 +271,8 @@ window.baguetteBox = (function () {
             playpause();
         else if (k == "KeyU" || k == "KeyO")
             relseek(k == "KeyU" ? -10 : 10);
+        else if (k.indexOf('Digit') === 0)
+            vid().currentTime = vid().duration * parseInt(k.slice(-1)) * 0.1;
         else if (k == "KeyM" && v) {
             v.muted = vmute = !vmute;
             mp_ctl();
@@ -271,17 +288,23 @@ window.baguetteBox = (function () {
             setVmode();
         }
         else if (k == "KeyF")
-            try {
-                if (isFullscreen)
-                    document.exitFullscreen();
-                else
-                    v.requestFullscreen();
-            }
-            catch (ex) { }
+            tglfull();
         else if (k == "KeyS")
             tglsel();
         else if (k == "KeyR")
             rotn(e.shiftKey ? -1 : 1);
+        else if (k == "KeyY")
+            dlpic();
+    }
+
+    function anim() {
+        var i = (anims.indexOf(options.animation) + 1) % anims.length,
+            o = options;
+        swrite('ganim', anims[i]);
+        options = {};
+        setOptions(o);
+        if (tt.en)
+            tt.show.bind(this)();
     }
 
     function setVmode() {
@@ -308,6 +331,7 @@ window.baguetteBox = (function () {
         btnVmode.setAttribute('aria-label', msg);
         btnVmode.setAttribute('tt', msg + tts);
         btnVmode.textContent = lbl;
+        swrite('vmode', lbl[0]);
 
         v.loop = vloop
         if (vloop && v.paused)
@@ -329,19 +353,39 @@ window.baguetteBox = (function () {
             tt.show.bind(this)();
     }
 
-    function tglsel() {
+    function findfile() {
         var thumb = currentGallery[currentIndex].imageElement,
             name = vsplit(thumb.href)[1].split('?')[0],
             files = msel.getall();
 
         for (var a = 0; a < files.length; a++)
             if (vsplit(files[a].vp)[1] == name)
-                clmod(ebi(files[a].id).closest('tr'), 'sel', 't');
+                return [name, a, files, ebi(files[a].id)];
+    }
 
+    function tglfull() {
+        try {
+            if (isFullscreen)
+                document.exitFullscreen();
+            else
+                (vid() || ebi('bbox-overlay')).requestFullscreen();
+        }
+        catch (ex) { alert(ex); }
+    }
+
+    function tglsel() {
+        var o = findfile()[3];
+        clmod(o.closest('tr'), 'sel', 't');
         msel.selui();
         selbg();
     }
 
+    function dlpic() {
+        var url = findfile()[3].href;
+        url += (url.indexOf('?') < 0 ? '?' : '&') + 'cache';
+        dl_file(url);
+    }
+
     function selbg() {
         var img = vidimg(),
             thumb = currentGallery[currentIndex].imageElement,
@@ -364,7 +408,7 @@ window.baguetteBox = (function () {
     }
 
     function keyUpHandler(e) {
-        if (e.ctrlKey || e.altKey || e.metaKey || e.isComposing)
+        if (anymod(e))
             return;
 
         var k = e.code + '';
@@ -391,15 +435,20 @@ window.baguetteBox = (function () {
     var nonPassiveEvent = passiveSupp ? { passive: true } : null;
 
     function bindEvents() {
+        bind(document, 'keydown', keyDownHandler);
+        bind(document, 'keyup', keyUpHandler);
+        bind(document, 'fullscreenchange', onFSC);
         bind(overlay, 'click', overlayClickHandler);
         bind(btnPrev, 'click', showPreviousImage);
         bind(btnNext, 'click', showNextImage);
         bind(btnClose, 'click', hideOverlay);
         bind(btnVmode, 'click', tglVmode);
         bind(btnHelp, 'click', halp);
+        bind(btnAnim, 'click', anim);
         bind(btnRotL, 'click', rotl);
         bind(btnRotR, 'click', rotr);
         bind(btnSel, 'click', tglsel);
+        bind(btnFull, 'click', tglfull);
         bind(slider, 'contextmenu', contextmenuHandler);
         bind(overlay, 'touchstart', touchstartHandler, nonPassiveEvent);
         bind(overlay, 'touchmove', touchmoveHandler, passiveEvent);
@@ -408,15 +457,20 @@ window.baguetteBox = (function () {
     }
 
     function unbindEvents() {
+        unbind(document, 'keydown', keyDownHandler);
+        unbind(document, 'keyup', keyUpHandler);
+        unbind(document, 'fullscreenchange', onFSC);
         unbind(overlay, 'click', overlayClickHandler);
         unbind(btnPrev, 'click', showPreviousImage);
         unbind(btnNext, 'click', showNextImage);
         unbind(btnClose, 'click', hideOverlay);
         unbind(btnVmode, 'click', tglVmode);
         unbind(btnHelp, 'click', halp);
+        unbind(btnAnim, 'click', anim);
         unbind(btnRotL, 'click', rotl);
         unbind(btnRotR, 'click', rotr);
         unbind(btnSel, 'click', tglsel);
+        unbind(btnFull, 'click', tglfull);
         unbind(slider, 'contextmenu', contextmenuHandler);
         unbind(overlay, 'touchstart', touchstartHandler, nonPassiveEvent);
         unbind(overlay, 'touchmove', touchmoveHandler, passiveEvent);
@@ -437,9 +491,8 @@ window.baguetteBox = (function () {
         var imagesFiguresIds = [];
         var imagesCaptionsIds = [];
         for (var i = 0, fullImage; i < gallery.length; i++) {
-            fullImage = mknod('div');
+            fullImage = mknod('div', 'baguette-img-' + i);
             fullImage.className = 'full-image';
-            fullImage.id = 'baguette-img-' + i;
             imagesElements.push(fullImage);
 
             imagesFiguresIds.push('bbox-figure-' + i);
@@ -459,7 +512,12 @@ window.baguetteBox = (function () {
             if (typeof newOptions[item] !== 'undefined')
                 options[item] = newOptions[item];
         }
-        slider.style.transition = (options.animation === 'fadeIn' ? 'opacity .4s ease' :
+
+        var an = options.animation = sread('ganim') || anims[ANIM ? 0 : 2];
+        btnAnim.textContent = ['⇄', '⮺', '⚡'][anims.indexOf(an)];
+        btnAnim.setAttribute('tt', 'animation: ' + an);
+
+        slider.style.transition = (options.animation === 'fadeIn' ? 'opacity .3s ease' :
             options.animation === 'slideIn' ? '' : 'none');
 
         if (options.buttons === 'auto' && ('ontouchstart' in window || currentGallery.length === 1))
@@ -476,9 +534,7 @@ window.baguetteBox = (function () {
         if (overlay.style.display === 'block')
             return;
 
-        bind(document, 'keydown', keyDownHandler);
-        bind(document, 'keyup', keyUpHandler);
-        bind(document, 'fullscreenchange', onFSC);
+        bindEvents();
         currentIndex = chosenImageIndex;
         touch = {
             count: 0,
@@ -490,6 +546,10 @@ window.baguetteBox = (function () {
             preloadPrev(currentIndex);
         });
 
+        clmod(ebi('bbox-btns'), 'off');
+        clmod(btnPrev, 'off');
+        clmod(btnNext, 'off');
+
         updateOffset();
         overlay.style.display = 'block';
         // Fade in overlay
@@ -502,9 +562,10 @@ window.baguetteBox = (function () {
                 options.afterShow();
         }, 50);
 
-        if (options.onChange)
+        if (options.onChange && !url_ts)
             options.onChange(currentIndex, imagesElements.length);
 
+        url_ts = null;
         documentLastFocus = document.activeElement;
         btnClose.focus();
         isOverlayVisible = true;
@@ -520,9 +581,14 @@ window.baguetteBox = (function () {
         if (overlay.style.display === 'none')
             return;
 
-        unbind(document, 'keydown', keyDownHandler);
-        unbind(document, 'keyup', keyUpHandler);
-        unbind(document, 'fullscreenchange', onFSC);
+        sethash('');
+        unbindEvents();
+        try {
+            document.exitFullscreen();
+            isFullscreen = false;
+        }
+        catch (ex) { }
+
         // Fade out and hide the overlay
         overlay.className = '';
         setTimeout(function () {
@@ -568,16 +634,14 @@ window.baguetteBox = (function () {
         if (is_vid && index != currentIndex)
             return;  // no preload
 
-        var figure = mknod('figure');
-        figure.id = 'bbox-figure-' + index;
+        var figure = mknod('figure', 'bbox-figure-' + index);
         figure.innerHTML = '<div class="bbox-spinner">' +
             '<div class="bbox-double-bounce1"></div>' +
             '<div class="bbox-double-bounce2"></div>' +
             '</div>';
 
         if (options.captions && imageCaption) {
-            var figcaption = mknod('figcaption');
-            figcaption.id = 'bbox-figcaption-' + index;
+            var figcaption = mknod('figcaption', 'bbox-figcaption-' + index);
             figcaption.innerHTML = imageCaption;
             figure.appendChild(figcaption);
         }
@@ -637,18 +701,12 @@ window.baguetteBox = (function () {
             showOverlay(index);
             return true;
         }
-        if (index < 0) {
-            if (options.animation)
-                bounceAnimation('left');
 
-            return false;
-        }
-        if (index >= imagesElements.length) {
-            if (options.animation)
-                bounceAnimation('right');
+        if (index < 0)
+            return bounceAnimation('left');
 
-            return false;
-        }
+        if (index >= imagesElements.length)
+            return bounceAnimation('right');
 
         var v = vid();
         if (v) {
@@ -756,8 +814,18 @@ window.baguetteBox = (function () {
     }
 
     function playvid(play) {
-        if (vid())
-            vid()[play ? 'play' : 'pause']();
+        if (!play) {
+            timer.rm(loopchk);
+            loopA = loopB = null;
+        }
+
+        var v = vid();
+        if (!v)
+            return;
+
+        v[play ? 'play' : 'pause']();
+        if (play && loopA !== null && v.currentTime < loopA)
+            v.currentTime = loopA;
     }
 
     function playpause() {
@@ -776,6 +844,38 @@ window.baguetteBox = (function () {
             showNextImage();
     }
 
+    function setloop(side) {
+        var v = vid();
+        if (!v)
+            return;
+
+        var t = v.currentTime;
+        if (side == 1) loopA = t;
+        if (side == 2) loopB = t;
+        if (side)
+            toast.inf(5, 'Loop' + (side == 1 ? 'A' : 'B') + ': ' + f2f(t, 2));
+
+        if (loopB !== null) {
+            timer.add(loopchk);
+            sethash(window.location.hash.slice(1).split('&')[0] + '&t=' + (loopA || 0) + '-' + loopB);
+        }
+    }
+
+    function loopchk() {
+        if (loopB === null)
+            return;
+
+        var v = vid();
+        if (!v || v.paused || v.currentTime < loopB)
+            return;
+
+        v.currentTime = loopA || 0;
+    }
+
+    function urltime(txt) {
+        url_ts = txt;
+    }
+
     function mp_ctl() {
         var v = vid();
         if (!vmute && v && mp.au && !mp.au.paused) {
@@ -789,10 +889,11 @@ window.baguetteBox = (function () {
     }
 
     function bounceAnimation(direction) {
-        slider.className = 'bounce-from-' + direction;
+        slider.className = options.animation == 'slideIn' ? 'bounce-from-' + direction : 'eog';
         setTimeout(function () {
             slider.className = '';
-        }, 400);
+        }, 300);
+        return false;
     }
 
     function updateOffset() {
@@ -806,7 +907,7 @@ window.baguetteBox = (function () {
                     slider.style.transform = 'translate3d(' + offset + ',0,0)' :
                     slider.style.left = offset;
                 slider.style.opacity = 1;
-            }, 400);
+            }, 100);
         } else {
             xform ?
                 slider.style.transform = 'translate3d(' + offset + ',0,0)' :
@@ -818,6 +919,15 @@ window.baguetteBox = (function () {
             playvid(true);
             v.muted = vmute;
             v.loop = vloop;
+            if (url_ts) {
+                var seek = ('' + url_ts).split('-');
+                v.currentTime = seek[0];
+                if (seek.length > 1) {
+                    loopA = parseFloat(seek[0]);
+                    loopB = parseFloat(seek[1]);
+                    setloop();
+                }
+            }
         }
         selbg();
         mp_ctl();
@@ -829,6 +939,28 @@ window.baguetteBox = (function () {
         else
             timer.rm(rotn);
 
+        var ctime = 0;
+        el.onclick = v ? null : function (e) {
+            var rc = e.target.getBoundingClientRect(),
+                x = e.clientX - rc.left,
+                fx = x / (rc.right - rc.left);
+
+            if (fx < 0.3)
+                return showPreviousImage();
+
+            if (fx > 0.7)
+                return showNextImage();
+
+            clmod(ebi('bbox-btns'), 'off', 't');
+            clmod(btnPrev, 'off', 't');
+            clmod(btnNext, 'off', 't');
+
+            if (Date.now() - ctime <= 500)
+                tglfull();
+
+            ctime = Date.now();
+        };
+
         var prev = QS('.full-image.vis');
         if (prev)
             clmod(prev, 'vis');
@@ -865,8 +997,6 @@ window.baguetteBox = (function () {
     function destroyPlugin() {
         unbindEvents();
         clearCachedData();
-        unbind(document, 'keydown', keyDownHandler);
-        unbind(document, 'keyup', keyUpHandler);
         document.getElementsByTagName('body')[0].removeChild(ebi('bbox-overlay'));
         data = {};
         currentGallery = [];
@@ -879,6 +1009,7 @@ window.baguetteBox = (function () {
         showNext: showNextImage,
         showPrevious: showPreviousImage,
         relseek: relseek,
+        urltime: urltime,
         playpause: playpause,
         hide: hideOverlay,
         destroy: destroyPlugin

copyparty/web/browser.html

@@ -3,9 +3,9 @@
 
 <head>
 	<meta charset="utf-8">
-	<title>⇆🎉 {{ title }}</title>
+	<title>{{ title }}</title>
 	<meta http-equiv="X-UA-Compatible" content="IE=edge">
-	<meta name="viewport" content="width=device-width, initial-scale=0.8">
+	<meta name="viewport" content="width=device-width, initial-scale=0.8, minimum-scale=0.6">
 {{ html_head }}
 	<link rel="stylesheet" media="screen" href="/.cpr/ui.css?_={{ ts }}">
 	<link rel="stylesheet" media="screen" href="/.cpr/browser.css?_={{ ts }}">
@@ -35,6 +35,7 @@
 			<input type="file" name="f" multiple /><br />
 			<input type="submit" value="start upload">
 		</form>
+		<a id="bbsw" href="?b=u"><br />switch to basic browser</a>
 	</div>
 
 	<div id="op_mkdir" class="opview opbox act">
@@ -67,7 +68,7 @@
 	<div id="op_cfg" class="opview opbox opwide"></div>
 	
 	<h1 id="path">
-		<a href="#" id="entree" tt="show navpane (directory tree sidebar)$NHotkey: B">🌲</a>
+		<a href="#" id="entree">🌲</a>
 		{%- for n in vpnodes %}
 		<a href="/{{ n[0] }}">{{ n[1] }}</a>
 		{%- endfor %}
@@ -119,7 +120,7 @@
 	
 	<div id="epi" class="logue">{{ logues[1] }}</div>
 
-	<h2><a href="/?h">control-panel</a></h2>
+	<h2><a href="/?h" id="goh">control-panel</a></h2>
 	
 	<a href="#" id="repl">π</a>
 
@@ -134,22 +135,32 @@
 	<script>
 		var acct = "{{ acct }}",
 			perms = {{ perms }},
+			themes = {{ themes }},
+			dtheme = "{{ dtheme }}",
+			srvinf = "{{ srv_info }}",
+			lang = "{{ lang }}",
+			dfavico = "{{ favico }}",
 			def_hcols = {{ def_hcols|tojson }},
 			have_up2k_idx = {{ have_up2k_idx|tojson }},
 			have_tags_idx = {{ have_tags_idx|tojson }},
 			have_acode = {{ have_acode|tojson }},
 			have_mv = {{ have_mv|tojson }},
 			have_del = {{ have_del|tojson }},
-			have_unpost = {{ have_unpost|tojson }},
+			have_unpost = {{ have_unpost }},
 			have_zip = {{ have_zip|tojson }},
+			lifetime = {{ lifetime }},
+			turbolvl = {{ turbolvl }},
+			u2sort = "{{ u2sort }}",
+			have_emp = {{ have_emp|tojson }},
 			txt_ext = "{{ txt_ext }}",
 			{% if no_prism %}no_prism = 1,{% endif %}
 			readme = {{ readme|tojson }},
 			ls0 = {{ ls0|tojson }};
 
-		document.documentElement.setAttribute("class", localStorage.lightmode == 1 ? "light" : "dark");
+		document.documentElement.className = localStorage.theme || dtheme;
 	</script>
 	<script src="/.cpr/util.js?_={{ ts }}"></script>
+	<script src="/.cpr/baguettebox.js?_={{ ts }}"></script>
 	<script src="/.cpr/browser.js?_={{ ts }}"></script>
 	<script src="/.cpr/up2k.js?_={{ ts }}"></script>
 	{%- if js %}

copyparty/web/browser2.html

@@ -45,7 +45,9 @@
 <tr><td></td><td><a href="../{{ url_suf }}">parent folder</a></td><td>-</td><td>-</td></tr>
 
 {%- for f in files %}
-<tr><td>{{ f.lead }}</td><td><a href="{{ f.href }}{{ url_suf }}">{{ f.name|e }}</a></td><td>{{ f.sz }}</td><td>{{ f.dt }}</td></tr>
+<tr><td>{{ f.lead }}</td><td><a href="{{ f.href }}{{
+	'&' + url_suf[1:] if url_suf[:1] == '?' and '?' in f.href else url_suf
+	}}">{{ f.name|e }}</a></td><td>{{ f.sz }}</td><td>{{ f.dt }}</td></tr>
 {%- endfor %}
 
 		</tbody>

copyparty/web/cf.html

@@ -0,0 +1,27 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+	<meta charset="utf-8">
+	<title>{{ svcname }}</title>
+	<meta http-equiv="X-UA-Compatible" content="IE=edge">
+	<meta name="viewport" content="width=device-width, initial-scale=0.8">
+</head>
+
+<body>
+	<div id="box" style="opacity: 0; font-family: sans-serif">
+		<h3>please press F5 to reload the page</h3>
+		<p>sorry for the inconvenience</p>
+	</div>
+
+	<script>
+		setTimeout(function() {
+			document.getElementById('box').style.opacity = 1;
+		}, 500);
+
+		parent.toast.ok(30, parent.L.cf_ok);
+		parent.qsr('#cf_frame');
+	</script>
+</body>
+
+</html>

copyparty/web/dbg-audio.js

@@ -13,8 +13,7 @@ audio_eq.apply = function () {
 
     var can = ebi('fft_can');
     if (!can) {
-        can = mknod('canvas');
-        can.setAttribute('id', 'fft_can');
+        can = mknod('canvas', 'fft_can');
         can.style.cssText = 'position:absolute;left:0;bottom:5em;width:' + w + 'px;height:' + h + 'px;z-index:9001';
         document.body.appendChild(can);
         can.width = w;

copyparty/web/md.css

@@ -161,7 +161,7 @@ blink {
 		height: 1.05em;
 		margin: -.2em .3em -.2em -.4em;
 		display: inline-block;
-		border: 1px solid rgba(0,0,0,0.2);
+		border: 1px solid rgba(154,154,154,0.6);
 		border-width: .2em .2em 0 0;
 		transform: rotate(45deg);
 	}
@@ -219,48 +219,45 @@ blink {
 
 
 
-	html.dark,
-	html.dark body {
+	html.z,
+	html.z body {
 		background: #222;
 		color: #ccc;
 	}
-	html.dark #toc a {
+	html.z #toc a {
 		color: #ccc;
 		border-left: .4em solid #444;
 		border-bottom: .1em solid #333;
 	}
-	html.dark #toc a.act {
+	html.z #toc a.act {
 		color: #fff;
 		border-left: .4em solid #3ad;
 	}
-	html.dark #toc li {
+	html.z #toc li {
 		border-width: 0;
 	}
-	html.dark #mn a:not(:last-child)::after {
-		border-color: rgba(255,255,255,0.3);
-	}
-	html.dark #mn a {
+	html.z #mn a {
 		color: #ccc;
 	}
-	html.dark #mn {
+	html.z #mn {
 		border-bottom: 1px solid #333;
 	}
-	html.dark #mn,
-	html.dark #mh {
+	html.z #mn,
+	html.z #mh {
 		background: #222;
 	}
-	html.dark #mh a {
+	html.z #mh a {
 		color: #ccc;
 		background: none;
 	}
-	html.dark #mh a:hover {
+	html.z #mh a:hover {
 		background: #333;
 		color: #fff;
 	}
-	html.dark #toolsbox {
+	html.z #toolsbox {
 		background: #222;
 	}
-	html.dark #toolsbox.open {
+	html.z #toolsbox.open {
 		box-shadow: 0 .2em .2em #069;
 		border-radius: 0 0 .4em .4em;
 	}
@@ -307,24 +304,24 @@ blink {
 	}
 
 
-	
-	html.dark #toc {
+
+	html.z #toc {
 		background: #282828;
 		border-top: 1px solid #2c2c2c;
 		box-shadow: 0 0 1em #181818;
 	}
-	html.dark #toc,
-	html.dark #mw {
+	html.z #toc,
+	html.z #mw {
 		scrollbar-color: #b80 #282828;
 	}
-	html.dark #toc::-webkit-scrollbar-track {
+	html.z #toc::-webkit-scrollbar-track {
 		background: #282828;
 	}
-	html.dark #toc::-webkit-scrollbar {
+	html.z #toc::-webkit-scrollbar {
 		background: #282828;
 		width: .8em;
 	}
-	html.dark #toc::-webkit-scrollbar-thumb {
+	html.z #toc::-webkit-scrollbar-thumb {
 		background: #b80;
 	}
 }
@@ -431,17 +428,17 @@ blink {
 	}
 
 
-	
-	html.dark .mdo a {
+
+	html.z .mdo a {
 		color: #000;
 	}
-	html.dark .mdo pre,
-	html.dark .mdo code {
+	html.z .mdo pre,
+	html.z .mdo code {
 		color: #240;
 	}
-	html.dark .mdo p>em,
-	html.dark .mdo li>em,
-	html.dark .mdo td>em {
+	html.z .mdo p>em,
+	html.z .mdo li>em,
+	html.z .mdo td>em {
 		color: #940;
 	}
 }

copyparty/web/md.html

@@ -1,6 +1,6 @@
 <!DOCTYPE html><html><head>
 	<meta charset="utf-8">
-	<title>📝🎉 {{ title }}</title>
+	<title>📝 {{ title }}</title>
 	<meta http-equiv="X-UA-Compatible" content="IE=edge">
 	<meta name="viewport" content="width=device-width, initial-scale=0.7">
 {{ html_head }}
@@ -127,30 +127,32 @@ write markdown (most html is 🙆 too)
 	
 	<script>
 
-var last_modified = {{ lastmod }};
+var last_modified = {{ lastmod }},
+	have_emp = {{ have_emp|tojson }},
+	dfavico = "{{ favico }}";
+
 var md_opt = {
 	link_md_as_html: false,
-	allow_plugins: {{ md_plug }},
 	modpoll_freq: {{ md_chk_rate }}
 };
 
 (function () {
-    var l = localStorage,
-		drk = l.lightmode != 1,
+	var l = localStorage,
+		drk = l.light != 1,
 		btn = document.getElementById("lightswitch"),
 		f = function (e) {
 if (e) { e.preventDefault(); drk = !drk; }
-document.documentElement.setAttribute("class", drk? "dark":"light");
+document.documentElement.className = drk? "z":"y";
 btn.innerHTML = "go " + (drk ? "light":"dark");
-l.lightmode = drk? 0:1;
-    	};
+l.light = drk? 0:1;
+		};
 	
 	btn.onclick = f;
 	f();
 })();
 
 	</script>
-    <script src="/.cpr/util.js?_={{ ts }}"></script>
+	<script src="/.cpr/util.js?_={{ ts }}"></script>
 	<script src="/.cpr/deps/marked.js?_={{ ts }}"></script>
 	<script src="/.cpr/md.js?_={{ ts }}"></script>
 	{%- if edit %}

copyparty/web/md.js

@@ -20,10 +20,6 @@ var dbg = function () { };
 // dbg = console.log
 
 
-// plugins
-var md_plug = {};
-
-
 // dodge browser issues
 (function () {
     var ua = navigator.userAgent;
@@ -44,7 +40,7 @@ var md_plug = {};
         link += parts[a] + (a < aa ? '/' : '');
         o = mknod('a');
         o.setAttribute('href', link);
-        o.textContent = uricom_dec(parts[a])[0] || 'top';
+        o.textContent = uricom_dec(parts[a]) || 'top';
         dom_nav.appendChild(o);
     }
 })();
@@ -160,7 +156,7 @@ function copydom(src, dst, lv) {
 }
 
 
-function md_plug_err(ex, js) {
+md_plug_err = function (ex, js) {
     qsr('#md_errbox');
     if (!ex)
         return;
@@ -177,8 +173,7 @@ function md_plug_err(ex, js) {
             o.textContent = lns[ln - 1];
         }
     }
-    var errbox = mknod('div');
-    errbox.setAttribute('id', 'md_errbox');
+    var errbox = mknod('div', 'md_errbox');
     errbox.style.cssText = 'position:absolute;top:0;left:0;padding:1em .5em;background:#2b2b2b;color:#fc5'
     errbox.textContent = msg;
     errbox.onclick = function () {
@@ -197,50 +192,12 @@ function md_plug_err(ex, js) {
 }
 
 
-function load_plug(md_text, plug_type) {
-    if (!md_opt.allow_plugins)
-        return md_text;
-
-    var find = '\n```copyparty_' + plug_type + '\n';
-    var ofs = md_text.indexOf(find);
-    if (ofs === -1)
-        return md_text;
-
-    var ofs2 = md_text.indexOf('\n```', ofs + 1);
-    if (ofs2 == -1)
-        return md_text;
-
-    var js = md_text.slice(ofs + find.length, ofs2 + 1);
-    var md = md_text.slice(0, ofs + 1) + md_text.slice(ofs2 + 4);
-
-    var old_plug = md_plug[plug_type];
-    if (!old_plug || old_plug[1] != js) {
-        js = 'const x = { ' + js + ' }; x;';
-        try {
-            var x = eval(js);
-        }
-        catch (ex) {
-            md_plug[plug_type] = null;
-            md_plug_err(ex, js);
-            return md;
-        }
-        if (x['ctor']) {
-            x['ctor']();
-            delete x['ctor'];
-        }
-        md_plug[plug_type] = [x, js];
-    }
-
-    return md;
-}
-
-
 function convert_markdown(md_text, dest_dom) {
     md_text = md_text.replace(/\r/g, '');
 
     md_plug_err(null);
-    md_text = load_plug(md_text, 'pre');
-    md_text = load_plug(md_text, 'post');
+    md_text = load_md_plug(md_text, 'pre');
+    md_text = load_md_plug(md_text, 'post');
 
     var marked_opts = {
         //headerPrefix: 'h-',
@@ -248,7 +205,7 @@ function convert_markdown(md_text, dest_dom) {
         gfm: true
     };
 
-    var ext = md_plug['pre'];
+    var ext = md_plug.pre;
     if (ext)
         Object.assign(marked_opts, ext[0]);
 
@@ -278,7 +235,7 @@ function convert_markdown(md_text, dest_dom) {
         if (!txt)
             nodes[a].textContent = href;
         else if (href !== txt)
-            nodes[a].setAttribute('class', 'vis');
+            nodes[a].className = 'vis';
     }
 
     // todo-lists (should probably be a marked extension)
@@ -294,7 +251,7 @@ function convert_markdown(md_text, dest_dom) {
         var clas = done ? 'done' : 'pend';
         var char = done ? 'Y' : 'N';
 
-        dom_li.setAttribute('class', 'task-list-item');
+        dom_li.className = 'task-list-item';
         dom_li.style.listStyleType = 'none';
         var html = dom_li.innerHTML;
         dom_li.innerHTML =
@@ -349,7 +306,7 @@ function convert_markdown(md_text, dest_dom) {
         el.innerHTML = '<a href="#' + id + '">' + el.innerHTML + '</a>';
     }
 
-    ext = md_plug['post'];
+    ext = md_plug.post;
     if (ext && ext[0].render)
         try {
             ext[0].render(md_dom);
@@ -438,7 +395,7 @@ function init_toc() {
 
     // collect vertical position of all toc items (headers in document)
     function freshen_offsets() {
-        var top = window.pageYOffset || document.documentElement.scrollTop;
+        var top = yscroll();
         for (var a = anchors.length - 1; a >= 0; a--) {
             var y = top + anchors[a].elm.getBoundingClientRect().top;
             y = Math.round(y * 10.0) / 10;
@@ -454,7 +411,7 @@ function init_toc() {
         if (anchors.length == 0)
             return;
 
-        var ptop = window.pageYOffset || document.documentElement.scrollTop;
+        var ptop = yscroll();
         var hit = anchors.length - 1;
         for (var a = 0; a < anchors.length; a++) {
             if (anchors[a].y >= ptop - 8) {  //???
@@ -468,11 +425,11 @@ function init_toc() {
             for (var a = 0; a < anchors.length; a++) {
                 if (anchors[a].active) {
                     anchors[a].active = false;
-                    links[a].setAttribute('class', '');
+                    links[a].className = '';
                 }
             }
             anchors[hit].active = true;
-            links[hit].setAttribute('class', 'act');
+            links[hit].className = 'act';
         }
 
         var pane_height = parseInt(getComputedStyle(dom_toc).height);
@@ -541,5 +498,5 @@ dom_navtgl.onclick = function () {
 if (sread('hidenav') == 1)
     dom_navtgl.onclick();
 
-if (window['tt'])
+if (window.tt && tt.init)
     tt.init();

copyparty/web/md2.css

@@ -36,6 +36,11 @@
 	width: 55em;
 	width: min(55em, calc(100% - 2em));
 }
+#mtw.single.editor,
+#mw.single.editor {
+	width: calc(100% - 1em);
+	left: .5em;
+}
 
 
 #mp {
@@ -61,7 +66,7 @@
 	position: relative;
 	scrollbar-color: #eb0 #f7f7f7;
 }
-html.dark #mt {
+html.z #mt {
 	color: #eee;
 	background: #222;
 	border: 1px solid #777;
@@ -77,7 +82,7 @@ html.dark #mt {
 	background: #f97;
 	border-radius: .15em;
 }
-html.dark #save.force-save {
+html.z #save.force-save {
 	color: #fca;
 	background: #720;
 }
@@ -102,7 +107,7 @@ html.dark #save.force-save {
 #helpclose {
 	display: block;
 }
-html.dark #helpbox {
+html.z #helpbox {
 	box-shadow: 0 .5em 2em #444;
 	background: #222;
 	border: 1px solid #079;

copyparty/web/md2.js

@@ -16,8 +16,7 @@ var dom_sbs = ebi('sbs');
 var dom_nsbs = ebi('nsbs');
 var dom_tbox = ebi('toolsbox');
 var dom_ref = (function () {
-    var d = mknod('div');
-    d.setAttribute('id', 'mtr');
+    var d = mknod('div', 'mtr');
     dom_swrap.appendChild(d);
     d = ebi('mtr');
     // hide behind the textarea (offsetTop is not computed if display:none)
@@ -144,16 +143,16 @@ redraw = (function () {
         map_pre = genmap(dom_pre, map_pre);
     }
     function setsbs() {
-        dom_wrap.setAttribute('class', '');
-        dom_swrap.setAttribute('class', '');
+        dom_wrap.className = '';
+        dom_swrap.className = '';
         onresize();
     }
     function modetoggle() {
         var mode = dom_nsbs.innerHTML;
         dom_nsbs.innerHTML = mode == 'editor' ? 'preview' : 'editor';
         mode += ' single';
-        dom_wrap.setAttribute('class', mode);
-        dom_swrap.setAttribute('class', mode);
+        dom_wrap.className = mode;
+        dom_swrap.className = mode;
         onresize();
     }
 
@@ -255,10 +254,10 @@ function Modpoll() {
 
         console.log('modpoll...');
         var url = (document.location + '').split('?')[0] + '?raw&_=' + Date.now();
-        var xhr = new XMLHttpRequest();
+        var xhr = new XHR();
         xhr.open('GET', url, true);
         xhr.responseType = 'text';
-        xhr.onreadystatechange = r.cb;
+        xhr.onload = xhr.onerror = r.cb;
         xhr.send();
     };
 
@@ -268,9 +267,6 @@ function Modpoll() {
             return;
         }
 
-        if (this.readyState != XMLHttpRequest.DONE)
-            return;
-
         if (this.status !== 200) {
             console.log('modpoll err ' + this.status + ": " + this.responseText);
             return;
@@ -309,7 +305,7 @@ var modpoll = new Modpoll();
 
 
 window.onbeforeunload = function (e) {
-    if ((ebi("save").getAttribute('class') + '').indexOf('disabled') >= 0)
+    if ((ebi("save").className + '').indexOf('disabled') >= 0)
         return; //nice (todo)
 
     e.preventDefault(); //ff
@@ -321,7 +317,7 @@ window.onbeforeunload = function (e) {
 function save(e) {
     if (e) e.preventDefault();
     var save_btn = ebi("save"),
-        save_cls = save_btn.getAttribute('class') + '';
+        save_cls = save_btn.className + '';
 
     if (save_cls.indexOf('disabled') >= 0)
         return toast.inf(2, "no changes");
@@ -336,10 +332,10 @@ function save(e) {
         fd.append("body", txt);
 
         var url = (document.location + '').split('?')[0];
-        var xhr = new XMLHttpRequest();
+        var xhr = new XHR();
         xhr.open('POST', url, true);
         xhr.responseType = 'text';
-        xhr.onreadystatechange = save_cb;
+        xhr.onload = xhr.onerror = save_cb;
         xhr.btn = save_btn;
         xhr.txt = txt;
 
@@ -356,9 +352,6 @@ function save(e) {
 }
 
 function save_cb() {
-    if (this.readyState != XMLHttpRequest.DONE)
-        return;
-
     if (this.status !== 200)
         return toast.err(0, 'Error!  The file was NOT saved.\n\n' + this.status + ": " + (this.responseText + '').replace(/^<pre>/, ""));
 
@@ -397,10 +390,10 @@ function save_cb() {
 function run_savechk(lastmod, txt, btn, ntry) {
     // download the saved doc from the server and compare
     var url = (document.location + '').split('?')[0] + '?raw&_=' + Date.now();
-    var xhr = new XMLHttpRequest();
+    var xhr = new XHR();
     xhr.open('GET', url, true);
     xhr.responseType = 'text';
-    xhr.onreadystatechange = savechk_cb;
+    xhr.onload = xhr.onerror = savechk_cb;
     xhr.lastmod = lastmod;
     xhr.txt = txt;
     xhr.btn = btn;
@@ -409,9 +402,6 @@ function run_savechk(lastmod, txt, btn, ntry) {
 }
 
 function savechk_cb() {
-    if (this.readyState != XMLHttpRequest.DONE)
-        return;
-
     if (this.status !== 200)
         return toast.err(0, 'Error!  The file was NOT saved.\n\n' + this.status + ": " + (this.responseText + '').replace(/^<pre>/, ""));
 
@@ -518,6 +508,20 @@ function setsel(s) {
 }
 
 
+// cut/copy current line
+function md_cut(cut) {
+    var s = linebounds();
+    if (s.car != s.cdr)
+        return;
+
+    dom_src.setSelectionRange(s.n1, s.n2 + 1, 'forward');
+    setTimeout(function () {
+        var i = cut ? s.n1 : s.car;
+        dom_src.setSelectionRange(i, i, 'forward');
+    }, 1);
+}
+
+
 // indent/dedent
 function md_indent(dedent) {
     var s = getsel(),
@@ -678,7 +682,7 @@ function reLastIndexOf(txt, ptn, end) {
 // table formatter
 function fmt_table(e) {
     if (e) e.preventDefault();
-    //dom_tbox.setAttribute('class', '');
+    //dom_tbox.className = '';
 
     var txt = dom_src.value,
         ofs = dom_src.selectionStart,
@@ -829,7 +833,7 @@ function fmt_table(e) {
 // show unicode
 function mark_uni(e) {
     if (e) e.preventDefault();
-    dom_tbox.setAttribute('class', '');
+    dom_tbox.className = '';
 
     var txt = dom_src.value,
         ptn = new RegExp('([^' + js_uni_whitelist + ']+)', 'g'),
@@ -964,6 +968,10 @@ var set_lno = (function () {
                 md_p_jump(dn);
                 return false;
             }
+            if (ev.code == "KeyX" || ev.code == "KeyC") {
+                md_cut(ev.code == "KeyX");
+                return true; //sic
+            }
         }
         else {
             if (ev.code == "Tab" || kc == 9) {
@@ -989,14 +997,14 @@ var set_lno = (function () {
 
 ebi('tools').onclick = function (e) {
     if (e) e.preventDefault();
-    var is_open = dom_tbox.getAttribute('class') != 'open';
-    dom_tbox.setAttribute('class', is_open ? 'open' : '');
+    var is_open = dom_tbox.className != 'open';
+    dom_tbox.className = is_open ? 'open' : '';
 };
 
 
 ebi('help').onclick = function (e) {
     if (e) e.preventDefault();
-    dom_tbox.setAttribute('class', '');
+    dom_tbox.className = '';
 
     var dom = ebi('helpbox');
     var dtxt = dom.getElementsByTagName('textarea');

copyparty/web/mde.css

@@ -84,24 +84,24 @@ html .editor-toolbar>button.save.force-save {
 
 
 /* darkmode */
-html.dark .mdo,
-html.dark .CodeMirror {
+html.z .mdo,
+html.z .CodeMirror {
 	border-color: #222;
 }
-html.dark,
-html.dark body,
-html.dark .CodeMirror {
+html.z,
+html.z body,
+html.z .CodeMirror {
 	background: #222;
 	color: #ccc;
 }
-html.dark .CodeMirror-cursor {
+html.z .CodeMirror-cursor {
 	border-color: #fff;
 }
-html.dark .CodeMirror-selected {
+html.z .CodeMirror-selected {
 	box-shadow: 0 0 1px #0cf inset;
 }
-html.dark .CodeMirror-selected,
-html.dark .CodeMirror-selectedtext {
+html.z .CodeMirror-selected,
+html.z .CodeMirror-selectedtext {
 	border-radius: .1em;
 	background: #246;
 	color: #fff;
@@ -109,37 +109,37 @@ html.dark .CodeMirror-selectedtext {
 
 
 
-html.dark #mn a {
+html.z #mn a {
 	color: #ccc;
 }
-html.dark #mn a:not(:last-child):after {
+html.z #mn a:not(:last-child):after {
 	border-color: rgba(255,255,255,0.3);
 }
-html.dark .editor-toolbar {
+html.z .editor-toolbar {
 	border-color: #2c2c2c;
 	background: #1c1c1c;
 }
-html.dark .editor-toolbar>i.separator {
+html.z .editor-toolbar>i.separator {
 	border-left: 1px solid #444;
 	border-right: 1px solid #111;
 }
-html.dark .editor-toolbar>button {
+html.z .editor-toolbar>button {
 	margin-left: -1px; border: 1px solid rgba(255,255,255,0.1);
 	color: #aaa;
 }
 
 
 
-html.dark .editor-toolbar>button:hover {
+html.z .editor-toolbar>button:hover {
 	color: #333;
 }
-html.dark .editor-toolbar>button.active {
+html.z .editor-toolbar>button.active {
 	color: #333;
 	border-color: #ec1;
 	background: #c90;
 }
-html.dark .editor-toolbar::after,
-html.dark .editor-toolbar::before {
+html.z .editor-toolbar::after,
+html.z .editor-toolbar::before {
 	background: none;
 }
 
@@ -150,6 +150,6 @@ html.dark .editor-toolbar::before {
 	padding: 1em;
 	background: #f7f7f7;
 }
-html.dark .mdo {
+html.z .mdo {
 	background: #1c1c1c;
 }

copyparty/web/mde.html

@@ -1,6 +1,6 @@
 <!DOCTYPE html><html><head>
 	<meta charset="utf-8">
-	<title>📝🎉 {{ title }}</title>
+	<title>📝 {{ title }}</title>
 	<meta http-equiv="X-UA-Compatible" content="IE=edge">
 	<meta name="viewport" content="width=device-width, initial-scale=0.7">
 {{ html_head }}
@@ -25,27 +25,29 @@
 	<a href="#" id="repl">π</a>
 	<script>
 
-var last_modified = {{ lastmod }};
+var last_modified = {{ lastmod }},
+	have_emp = {{ have_emp|tojson }},
+	dfavico = "{{ favico }}";
+
 var md_opt = {
 	link_md_as_html: false,
-	allow_plugins: {{ md_plug }},
 	modpoll_freq: {{ md_chk_rate }}
 };
 
 var lightswitch = (function () {
 	var l = localStorage,
-		drk = l.lightmode != 1,
+		drk = l.light != 1,
 		f = function (e) {
 if (e) drk = !drk;
-document.documentElement.setAttribute("class", drk? "dark":"light");
-l.lightmode = drk? 0:1;
+document.documentElement.className = drk? "z":"y";
+l.light = drk? 0:1;
 		};
 	f();
 	return f;
 })();
 
 	</script>
-    <script src="/.cpr/util.js?_={{ ts }}"></script>
+	<script src="/.cpr/util.js?_={{ ts }}"></script>
 	<script src="/.cpr/deps/marked.js?_={{ ts }}"></script>
 	<script src="/.cpr/deps/easymde.js?_={{ ts }}"></script>
 	<script src="/.cpr/mde.js?_={{ ts }}"></script>

copyparty/web/mde.js

@@ -15,7 +15,7 @@ var dom_md = ebi('mt');
         if (a > 0)
             loc.push(n[a]);
 
-        var dec = uricom_dec(n[a])[0].replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
+        var dec = uricom_dec(n[a]).replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
 
         nav.push('<a href="/' + loc.join('/') + '">' + dec + '</a>');
     }
@@ -114,10 +114,10 @@ function save(mde) {
         fd.append("body", txt);
 
         var url = (document.location + '').split('?')[0];
-        var xhr = new XMLHttpRequest();
+        var xhr = new XHR();
         xhr.open('POST', url, true);
         xhr.responseType = 'text';
-        xhr.onreadystatechange = save_cb;
+        xhr.onload = xhr.onerror = save_cb;
         xhr.btn = save_btn;
         xhr.mde = mde;
         xhr.txt = txt;
@@ -133,9 +133,6 @@ function save(mde) {
 }
 
 function save_cb() {
-    if (this.readyState != XMLHttpRequest.DONE)
-        return;
-
     if (this.status !== 200)
         return toast.err(0, 'Error!  The file was NOT saved.\n\n' + this.status + ": " + (this.responseText + '').replace(/^<pre>/, ""));
 
@@ -170,10 +167,10 @@ function save_cb() {
 
     // download the saved doc from the server and compare
     var url = (document.location + '').split('?')[0] + '?raw';
-    var xhr = new XMLHttpRequest();
+    var xhr = new XHR();
     xhr.open('GET', url, true);
     xhr.responseType = 'text';
-    xhr.onreadystatechange = save_chk;
+    xhr.onload = xhr.onerror = save_chk;
     xhr.btn = this.save_btn;
     xhr.mde = this.mde;
     xhr.txt = this.txt;
@@ -182,9 +179,6 @@ function save_cb() {
 }
 
 function save_chk() {
-    if (this.readyState != XMLHttpRequest.DONE)
-        return;
-
     if (this.status !== 200)
         return toast.err(0, 'Error!  The file was NOT saved.\n\n' + this.status + ": " + (this.responseText + '').replace(/^<pre>/, ""));
 

copyparty/web/msg.html

@@ -2,49 +2,49 @@
 <html lang="en">
 
 <head>
-    <meta charset="utf-8">
-    <title>{{ svcname }}</title>
-    <meta http-equiv="X-UA-Compatible" content="IE=edge">
-    <meta name="viewport" content="width=device-width, initial-scale=0.8">
+	<meta charset="utf-8">
+	<title>{{ svcname }}</title>
+	<meta http-equiv="X-UA-Compatible" content="IE=edge">
+	<meta name="viewport" content="width=device-width, initial-scale=0.8">
 	{{ html_head }}
-    <link rel="stylesheet" media="screen" href="/.cpr/msg.css?_={{ ts }}">
+	<link rel="stylesheet" media="screen" href="/.cpr/msg.css?_={{ ts }}">
 </head>
 
 <body>
-    <div id="box">
-        
-        {%- if h1 %}
-        <h1>{{ h1 }}</h1>
-        {%- endif %}
-        
-        {%- if h2 %}
-        <h2>{{ h2 }}</h2>
-        {%- endif %}
-        
-        {%- if p %}
-        <p>{{ p }}</p>
-        {%- endif %}
+	<div id="box">
 
-        {%- if pre %}
-        <pre>{{ pre }}</pre>
-        {%- endif %}
+		{%- if h1 %}
+		<h1>{{ h1 }}</h1>
+		{%- endif %}
 
-        {%- if html %}
-        {{ html }}
-        {%- endif %}
+		{%- if h2 %}
+		<h2>{{ h2 }}</h2>
+		{%- endif %}
 
-        {%- if click %}
-        <script>document.getElementsByTagName("a")[0].click()</script>
-        {%- endif %}
-    </div>
+		{%- if p %}
+		<p>{{ p }}</p>
+		{%- endif %}
 
-    {%- if redir %}
-    <script>
-        setTimeout(function() {
-            window.location.replace("{{ redir }}");
-        }, 1000);
-    </script>
-    {%- endif %}
+		{%- if pre %}
+		<pre>{{ pre }}</pre>
+		{%- endif %}
+
+		{%- if html %}
+		{{ html }}
+		{%- endif %}
+
+		{%- if click %}
+		<script>document.getElementsByTagName("a")[0].click()</script>
+		{%- endif %}
+	</div>
+
+	{%- if redir %}
+	<script>
+		setTimeout(function() {
+			window.location.replace("{{ redir }}");
+		}, 1000);
+	</script>
+	{%- endif %}
 </body>
 
 </html>

copyparty/web/splash.css

@@ -1,9 +1,7 @@
-html, body, #wrap {
+html {
 	color: #333;
 	background: #f7f7f7;
 	font-family: sans-serif;
-}
-html {
 	touch-action: manipulation;
 }
 #wrap {
@@ -38,7 +36,6 @@ a+a {
 	margin: -.2em 0 0 .5em;
 }
 .logout,
-.btns a,
 a.r {
 	color: #c04;
 	border-color: #c7a;
@@ -88,27 +85,24 @@ blockquote {
 }
 
 
-html.dark,
-html.dark body,
-html.dark #wrap {
+html.z {
 	background: #222;
 	color: #ccc;
 }
-html.dark h1 {
+html.z h1 {
 	border-color: #777;
 }
-html.dark a {
+html.z a {
 	color: #fff;
 	background: #057;
 	border-color: #37a;
 }
-html.dark .logout,
-html.dark .btns a,
-html.dark a.r {
+html.z .logout,
+html.z a.r {
 	background: #804;
 	border-color: #c28;
 }
-html.dark input {
+html.z input {
 	color: #fff;
 	background: #626;
 	border: 1px solid #c2c;
@@ -117,6 +111,12 @@ html.dark input {
 	padding: .5em .7em;
 	margin: 0 .5em 0 0;
 }
-html.dark .num {
+html.z .num {
 	border-color: #777;
 }
+
+
+html.bz {
+	color: #bbd;
+	background: #11121d;
+}

copyparty/web/splash.html

@@ -2,105 +2,109 @@
 <html lang="en">
 
 <head>
-    <meta charset="utf-8">
-    <title>{{ svcname }}</title>
-    <meta http-equiv="X-UA-Compatible" content="IE=edge">
-    <meta name="viewport" content="width=device-width, initial-scale=0.8">
+	<meta charset="utf-8">
+	<title>{{ svcname }}</title>
+	<meta http-equiv="X-UA-Compatible" content="IE=edge">
+	<meta name="viewport" content="width=device-width, initial-scale=0.8">
 {{ html_head }}
-    <link rel="stylesheet" media="screen" href="/.cpr/splash.css?_={{ ts }}">
-    <link rel="stylesheet" media="screen" href="/.cpr/ui.css?_={{ ts }}">
+	<link rel="stylesheet" media="screen" href="/.cpr/splash.css?_={{ ts }}">
+	<link rel="stylesheet" media="screen" href="/.cpr/ui.css?_={{ ts }}">
 </head>
 
 <body>
-    <div id="wrap">
-        <a href="/?h" class="refresh">refresh</a>
+	<div id="wrap">
+		<a id="a" href="/?h" class="refresh">refresh</a>
 
-        {%- if this.uname == '*' %}
-            <p>howdy stranger &nbsp; <small>(you're not logged in)</small></p>
-        {%- else %}
-            <a href="/?pw=x" class="logout">logout</a>
-            <p>welcome back, <strong>{{ this.uname }}</strong></p>
-        {%- endif %}
+		{%- if this.uname == '*' %}
+			<p id="b">howdy stranger &nbsp; <small>(you're not logged in)</small></p>
+		{%- else %}
+			<a id="c" href="/?pw=x" class="logout">logout</a>
+			<p><span id="m">welcome back,</span> <strong>{{ this.uname }}</strong></p>
+		{%- endif %}
 
-        {%- if msg %}
-        <div id="msg">
-            {{ msg }}
-        </div>
-        {%- endif %}
+		{%- if msg %}
+		<div id="msg">
+			{{ msg }}
+		</div>
+		{%- endif %}
 
-        {%- if avol %}
-        <h1>admin panel:</h1>
-        <table><tr><td> <!-- hehehe -->
-            <table class="num">
-                <tr><td>scanning</td><td>{{ scanning }}</td></tr>
-                <tr><td>hash-q</td><td>{{ hashq }}</td></tr>
-                <tr><td>tag-q</td><td>{{ tagq }}</td></tr>
-                <tr><td>mtp-q</td><td>{{ mtpq }}</td></tr>
-            </table>
-        </td><td>
-            <table class="vols">
-                <thead><tr><th>vol</th><th>action</th><th>status</th></tr></thead>
-                <tbody>
-                    {% for mp in avol %}
-                    {%- if mp in vstate and vstate[mp] %}
-                    <tr><td><a href="{{ mp }}{{ url_suf }}">{{ mp }}</a></td><td><a href="{{ mp }}?scan">rescan</a></td><td>{{ vstate[mp] }}</td></tr>
-                    {%- endif %}
-                    {% endfor %}
-                </tbody>
-            </table>
-        </td></tr></table>
-        <div class="btns">
-            <a href="/?stack" tt="shows the state of all active threads">dump stack</a>
-            <a href="/?reload=cfg" tt="reload config files (accounts/volumes/volflags),$Nand rescan all e2ds volumes">reload cfg</a>
-        </div>
-        {%- endif %}
+		{%- if avol %}
+		<h1>admin panel:</h1>
+		<table><tr><td> <!-- hehehe -->
+			<table class="num">
+				<tr><td>scanning</td><td>{{ scanning }}</td></tr>
+				<tr><td>hash-q</td><td>{{ hashq }}</td></tr>
+				<tr><td>tag-q</td><td>{{ tagq }}</td></tr>
+				<tr><td>mtp-q</td><td>{{ mtpq }}</td></tr>
+				<tr><td>db-act</td><td id="u">{{ dbwt }}</td></tr>
+			</table>
+		</td><td>
+			<table class="vols">
+				<thead><tr><th>vol</th><th id="t">action</th><th>status</th></tr></thead>
+				<tbody>
+					{% for mp in avol %}
+					{%- if mp in vstate and vstate[mp] %}
+					<tr><td><a href="{{ mp }}{{ url_suf }}">{{ mp }}</a></td><td><a class="s" href="{{ mp }}?scan">rescan</a></td><td>{{ vstate[mp] }}</td></tr>
+					{%- endif %}
+					{% endfor %}
+				</tbody>
+			</table>
+		</td></tr></table>
+		<div class="btns">
+			<a id="d" href="/?stack">dump stack</a>
+			<a id="e" href="/?reload=cfg">reload cfg</a>
+		</div>
+		{%- endif %}
 
-        {%- if rvol %}
-        <h1>you can browse these:</h1>
-        <ul>
-            {% for mp in rvol %}
-            <li><a href="{{ mp }}{{ url_suf }}">{{ mp }}</a></li>
-            {% endfor %}
-        </ul>
-        {%- endif %}
+		{%- if rvol %}
+		<h1 id="f">you can browse:</h1>
+		<ul>
+			{% for mp in rvol %}
+			<li><a href="{{ mp }}{{ url_suf }}">{{ mp }}</a></li>
+			{% endfor %}
+		</ul>
+		{%- endif %}
 
-        {%- if wvol %}
-        <h1>you can upload to:</h1>
-        <ul>
-            {% for mp in wvol %}
-            <li><a href="{{ mp }}{{ url_suf }}">{{ mp }}</a></li>
-            {% endfor %}
-        </ul>
-        {%- endif %}
+		{%- if wvol %}
+		<h1 id="g">you can upload to:</h1>
+		<ul>
+			{% for mp in wvol %}
+			<li><a href="{{ mp }}{{ url_suf }}">{{ mp }}</a></li>
+			{% endfor %}
+		</ul>
+		{%- endif %}
 
-        <h1 id="cc">client config:</h1>
-        <ul>
-            {% if k304 %}
-            <li><a href="/?k304=n">disable k304</a> (currently enabled)
-            {%- else %}
-            <li><a href="/?k304=y" class="r">enable k304</a> (currently disabled)
-            {% endif %}
-            <blockquote>enabling this will disconnect your client on every HTTP 304, which can prevent some buggy browsers/proxies from getting stuck (suddenly not being able to load pages), <em>but</em> it will also make things slower in general</blockquote></li>
-            
-            <li><a href="/?reset" class="r" onclick="localStorage.clear();return true">reset client settings</a></li>
-        </ul>
+		<h1 id="cc">client config:</h1>
+		<ul>
+			{% if k304 %}
+			<li><a id="h" href="/?k304=n">disable k304</a> (currently enabled)
+			{%- else %}
+			<li><a id="i" href="/?k304=y" class="r">enable k304</a> (currently disabled)
+			{% endif %}
+			<blockquote id="j">enabling this will disconnect your client on every HTTP 304, which can prevent some buggy proxies from getting stuck (suddenly not loading pages), <em>but</em> it will also make things slower in general</blockquote></li>
+			
+			<li><a id="k" href="/?reset" class="r" onclick="localStorage.clear();return true">reset client settings</a></li>
+		</ul>
 
-        <h1>login for more:</h1>
-        <ul>
-            <form method="post" enctype="multipart/form-data" action="/{{ qvpath }}">
-                <input type="hidden" name="act" value="login" />
-                <input type="password" name="cppwd" />
-                <input type="submit" value="Login" />
-            </form>
-        </ul>
-    </div>
+		<h1 id="l">login for more:</h1>
+		<ul>
+			<form method="post" enctype="multipart/form-data" action="/{{ qvpath }}">
+				<input type="hidden" name="act" value="login" />
+				<input type="password" name="cppwd" />
+				<input type="submit" value="Login" />
+			</form>
+		</ul>
+	</div>
 	<a href="#" id="repl">π</a>
-    <script>
+	<script>
 
-document.documentElement.setAttribute("class", localStorage.lightmode == 1 ? "light" : "dark");
+var lang="{{ lang }}",
+	dfavico="{{ favico }}";
+
+document.documentElement.className=localStorage.theme||"{{ this.args.theme }}";
 
 </script>
 <script src="/.cpr/util.js?_={{ ts }}"></script>
-<script>tt.init();</script>
+<script src="/.cpr/splash.js?_={{ ts }}"></script>
 </body>
 </html>

copyparty/web/splash.js

@@ -0,0 +1,55 @@
+var Ls = {
+	"nor": {
+		"a1": "oppdater",
+		"b1": "halloien &nbsp; <small>(du er ikke logget inn)</small>",
+		"c1": "logg ut",
+		"d1": "tilstand",
+		"d2": "vis tilstanden til alle tråder",
+		"e1": "last innst.",
+		"e2": "leser inn konfigurasjonsfiler på nytt$N(kontoer, volumer, volumbrytere)$Nog kartlegger alle e2ds-volumer",
+		"f1": "du kan betrakte:",
+		"g1": "du kan laste opp til:",
+		"cc1": "klient-konfigurasjon",
+		"h1": "skru av k304",
+		"i1": "skru på k304",
+		"j1": "k304 bryter tilkoplingen for hver HTTP 304. Dette hjelper mot visse mellomtjenere som kan sette seg fast / plutselig slutter å laste sider, men det reduserer også ytelsen betydelig",
+		"k1": "nullstill innstillinger",
+		"l1": "logg inn:",
+		"m1": "velkommen tilbake,",
+		"n1": "404: filen finnes ikke &nbsp;┐( ´ -`)┌",
+		"o1": 'eller kanskje du ikke har tilgang? prøv å logge inn eller <a href="/?h">gå hjem</a>',
+		"p1": "403: tilgang nektet &nbsp;~┻━┻",
+		"q1": 'du må logge inn eller <a href="/?h">gå hjem</a>',
+		"r1": "gå hjem",
+		".s1": "kartlegg",
+		"t1": "handling",
+		"u2": "tid siden noen sist skrev til serveren$N( opplastning / navneendring / ... )$N$N17d = 17 dager$N1h23 = 1 time 23 minutter$N4m56 = 4 minuter 56 sekunder",
+	},
+	"eng": {
+		"d2": "shows the state of all active threads",
+		"e2": "reload config files (accounts/volumes/volflags),$Nand rescan all e2ds volumes",
+		"u2": "time since the last server write$N( upload / rename / ... )$N$N17d = 17 days$N1h23 = 1 hour 23 minutes$N4m56 = 4 minutes 56 seconds",
+	}
+},
+	d = Ls[sread("lang") || lang];
+
+for (var k in (d || {})) {
+	var f = k.slice(-1),
+		i = k.slice(0, -1),
+		o = QSA(i.startsWith('.') ? i : '#' + i);
+
+	for (var a = 0; a < o.length; a++)
+		if (f == 1)
+			o[a].innerHTML = d[k];
+		else if (f == 2)
+			o[a].setAttribute("tt", d[k]);
+}
+
+tt.init();
+var o = QS('input[name="cppwd"]');
+if (!ebi('c') && o.offsetTop + o.offsetHeight < window.innerHeight)
+	o.focus();
+
+o = ebi('u');
+if (o && /[0-9]+$/.exec(o.innerHTML))
+	o.innerHTML = shumantime(o.innerHTML);

copyparty/web/ui.css

@@ -11,6 +11,7 @@ html {
 	max-width: 34em;
 	max-width: min(34em, 90%);
 	max-width: min(34em, calc(100% - 7em));
+	color: #ddd;
 	background: #333;
 	border: 0 solid #777;
 	box-shadow: 0 .2em .5em #111;
@@ -74,6 +75,9 @@ html {
 	margin-right: -1.2em;
 	padding-right: .7em;
 }
+#toast.r #toastb {
+	text-align: right;
+}
 #toast pre {
 	margin: 0;
 }
@@ -157,26 +161,47 @@ html {
 #tt em {
 	color: #f6a;
 }
-html.light #tt {
+html.y #tt {
+	color: #333;
 	background: #fff;
 	border-color: #888 #000 #777 #000;
 }
-html.light #tt,
-html.light #toast {
+html.bz #tt {
+	background: #202231;
+	border-color: #3b3f58;
+}
+html.y #tt,
+html.y #toast {
 	box-shadow: 0 .3em 1em rgba(0,0,0,0.4);
 }
-#modalc code,
-html.light #tt code {
+html.y #tt code {
 	background: #060;
 	color: #fff;
 }
-html.light #tt em {
+#modalc code {
+	color: #060;
+	background: transparent;
+	border: 1px solid #ccc;
+}
+html.y #tt em {
 	color: #d38;
 }
-html.light #tth {
+html.y #tth {
 	color: #000;
 	background: #fff;
 }
+#cf_frame {
+	position: fixed;
+	z-index: 573;
+	top: 3em;
+	left: 50%;
+	width: 40em;
+	height: 30em;
+	margin-left: -20.2em;
+	border-radius: .4em;
+	border: .4em solid var(--fg);
+	box-shadow: 0 2em 4em 1em var(--bg-max);
+}
 #modal {
 	position: fixed;
     overflow: auto;
@@ -211,6 +236,10 @@ html.light #tth {
 	max-height: 30em;
 	overflow: auto;
 }
+#modalc td {
+	text-align: unset;
+	padding: .2em;
+}
 @media (min-width: 40em) {
     #modalc {
         min-width: 30em;
@@ -226,6 +255,9 @@ html.light #tth {
 	padding: .3em;
 	text-align: center;
 }
+#modalc a {
+	color: #07b;
+}
 #modalb {
 	position: sticky;
 	text-align: right;
@@ -268,24 +300,28 @@ html.light #tth {
 	max-width: 24em;
 }
 *:focus,
+*:focus+label,
 #pctl *:focus,
 .btn:focus {
 	box-shadow: 0 .1em .2em #fc0 inset;
+	outline: #fc0 solid .1em;
 	border-radius: .2em;
 }
-html.light *:focus,
-html.light #pctl *:focus,
-html.light .btn:focus {
+html.y *:focus,
+html.y *:focus+label,
+html.y #pctl *:focus,
+html.y .btn:focus {
 	box-shadow: 0 .1em .2em #037 inset;
+	outline: #037 solid .1em;
 }
 input[type="text"]:focus,
 input:not([type]):focus,
 textarea:focus {
 	box-shadow: 0 .1em .3em #fc0, 0 -.1em .3em #fc0;
 }
-html.light input[type="text"]:focus,
-html.light input:not([type]):focus,
-html.light textarea:focus {
+html.y input[type="text"]:focus,
+html.y input:not([type]):focus,
+html.y textarea:focus {
 	box-shadow: 0 .1em .3em #037, 0 -.1em .3em #037;
 }
 
@@ -363,11 +399,13 @@ html.light textarea:focus {
 	padding-left: 2em;
 	border-left: .3em solid #ddd;
 }
-.mdo ul>li,
-.mdo ol>li {
+.mdo ul>li {
 	margin: .7em 0;
 	list-style-type: disc;
 }
+.mdo ol>li {
+	margin: .7em 0 .7em 2em;
+}
 .mdo strong {
 	color: #000;
 }
@@ -414,7 +452,7 @@ html.light textarea:focus {
 		overflow-wrap: break-word;
 		word-wrap: break-word; /*ie*/
 	}
-	html.light .mdo a,
+	html.y .mdo a,
 	.mdo a {
 		color: #fff;
 		background: #39b;
@@ -443,48 +481,58 @@ html.light textarea:focus {
 
 
 
-	html.dark .mdo a {
+	html.z .mdo a {
 		background: #057;
 	}
-	html.dark .mdo h1 a, html.dark .mdo h4 a,
-	html.dark .mdo h2 a, html.dark .mdo h5 a,
-	html.dark .mdo h3 a, html.dark .mdo h6 a {
+	html.z .mdo h1 a, html.z .mdo h4 a,
+	html.z .mdo h2 a, html.z .mdo h5 a,
+	html.z .mdo h3 a, html.z .mdo h6 a {
 		color: inherit;
 		background: none;
 	}
-	html.dark .mdo pre,
-	html.dark .mdo code {
+	html.z .mdo pre,
+	html.z .mdo code {
 		color: #8c0;
 		background: #1a1a1a;
 		border: .07em solid #333;
 	}
-	html.dark .mdo ul,
-	html.dark .mdo ol {
+	html.z .mdo ul,
+	html.z .mdo ol {
 		border-color: #444;
 	}
-	html.dark .mdo strong {
+	html.z .mdo strong {
 		color: #fff;
 	}
-	html.dark .mdo p>em,
-	html.dark .mdo li>em,
-	html.dark .mdo td>em {
+	html.z .mdo p>em,
+	html.z .mdo li>em,
+	html.z .mdo td>em {
 		color: #f94;
 		border-color: #666;
 	}
-	html.dark .mdo h1 {
+	html.z .mdo h1 {
 		background: #383838;
 		border-top: .4em solid #b80;
 		border-bottom: .4em solid #4c4c4c;
 	}
-	html.dark .mdo h2 {
+	html.bz .mdo h1 {
+		background: #202231;
+		border: 1px solid #2d2f45;
+		border-width: 0 0 .4em 0;
+	}
+	html.z .mdo h2 {
 		background: #444;
 		border-bottom: .22em solid #555;
 	}
-	html.dark .mdo td,
-	html.dark .mdo th {
+	html.bz .mdo h2,
+	html.bz .mdo h3 {
+		background: transparent;
+		border-color: #3b3f58;
+	}
+	html.z .mdo td,
+	html.z .mdo th {
 		border-color: #444;
 	}
-	html.dark .mdo blockquote {
+	html.z .mdo blockquote {
 		background: #282828;
 		border: .07em dashed #444;
 	}

copyparty/web/util.js

@@ -1,20 +1,60 @@
 "use strict";
 
-if (!window['console'])
-    window['console'] = {
+if (!window.console || !console.log)
+    window.console = {
         "log": function (msg) { }
     };
 
 
-var is_touch = 'ontouchstart' in window,
-    IPHONE = is_touch && /iPhone|iPad|iPod/i.test(navigator.userAgent),
+var wah = '',
+    L, tt, treectl, thegrid, up2k, asmCrypto, hashwasm, vbar, marked,
+    CB = '?_=' + Date.now(),
+    HALFMAX = 8192 * 8192 * 8192 * 8192,
+    HTTPS = (window.location + '').indexOf('https:') === 0,
+    TOUCH = 'ontouchstart' in window,
+    MOBILE = TOUCH,
+    CHROME = !!window.chrome,
+    FIREFOX = ('netscape' in window) && / rv:/.test(navigator.userAgent),
+    IPHONE = TOUCH && /iPhone|iPad|iPod/i.test(navigator.userAgent),
     WINDOWS = navigator.platform ? navigator.platform == 'Win32' : /Windows/.test(navigator.userAgent);
 
+if (!window.WebAssembly || !WebAssembly.Memory)
+    window.WebAssembly = false;
+
+if (!window.Notification || !Notification.permission)
+    window.Notification = false;
+
+if (!window.FormData)
+    window.FormData = false;
+
+try {
+    CB = '?' + document.currentScript.src.split('?').pop();
+
+    if (navigator.userAgentData.mobile)
+        MOBILE = true;
+
+    if (navigator.userAgentData.platform == 'Windows')
+        WINDOWS = true;
+
+    if (navigator.userAgentData.brands.some(function (d) { return d.brand == 'Chromium' }))
+        CHROME = true;
+}
+catch (ex) { }
+
 
 var ebi = document.getElementById.bind(document),
     QS = document.querySelector.bind(document),
     QSA = document.querySelectorAll.bind(document),
-    mknod = document.createElement.bind(document);
+    XHR = XMLHttpRequest;
+
+
+function mknod(et, eid) {
+    var ret = document.createElement(et);
+    if (eid)
+        ret.id = eid;
+
+    return ret;
+}
 
 
 function qsr(sel) {
@@ -81,14 +121,20 @@ catch (ex) {
         console.log = console.stdlog;
     console.log('console capture failed', ex);
 }
-var crashed = false, ignexd = {};
+var crashed = false, ignexd = {}, evalex_fatal = false;
 function vis_exh(msg, url, lineNo, columnNo, error) {
-    if ((msg + '').indexOf('ResizeObserver') !== -1)
+    if ((msg + '').indexOf('ResizeObserver') + 1)
         return;  // chrome issue 809574 (benign, from <video>)
 
-    if ((msg + '').indexOf('l2d.js') !== -1)
+    if ((msg + '').indexOf('l2d.js') + 1)
         return;  // `t` undefined in tapEvent -> hitTestSimpleCustom
 
+    if (!/\.js($|\?)/.exec('' + url))
+        return;  // chrome debugger
+
+    if ((url + '').indexOf(' > eval') + 1 && !evalex_fatal)
+        return;  // md timer
+
     var ekey = url + '\n' + lineNo + '\n' + msg;
     if (ignexd[ekey] || crashed)
         return;
@@ -99,7 +145,7 @@ function vis_exh(msg, url, lineNo, columnNo, error) {
         '<h1>you hit a bug!</h1>',
         '<p style="font-size:1.3em;margin:0">try to <a href="#" onclick="localStorage.clear();location.reload();">reset copyparty settings</a> if you are stuck here, or <a href="#" onclick="ignex();">ignore this</a> / <a href="#" onclick="ignex(true);">ignore all</a></p>',
         '<p style="color:#fff">please send me a screenshot arigathanks gozaimuch: <a href="<ghi>" target="_blank">github issue</a> or <code>ed#2644</code></p>',
-        '<p class="b">' + esc(url + ' @' + lineNo + ':' + columnNo), '<br />' + esc(String(msg)) + '</p>',
+        '<p class="b">' + esc(url + ' @' + lineNo + ':' + columnNo), '<br />' + esc(String(msg)).replace(/\n/g, '<br />') + '</p>',
         '<p><b>UA:</b> ' + esc(navigator.userAgent + '')
     ];
 
@@ -151,8 +197,7 @@ function vis_exh(msg, url, lineNo, columnNo, error) {
     try {
         var exbox = ebi('exbox');
         if (!exbox) {
-            exbox = mknod('div');
-            exbox.setAttribute('id', 'exbox');
+            exbox = mknod('div', 'exbox');
             document.body.appendChild(exbox);
 
             var s = mknod('style');
@@ -194,13 +239,18 @@ function ctrl(e) {
 }
 
 
+function anymod(e, shift_ok) {
+    return e && (e.ctrlKey || e.altKey || e.metaKey || e.isComposing || (!shift_ok && e.shiftKey));
+}
+
+
 function ev(e) {
     e = e || window.event;
     if (!e)
         return;
 
     if (e.preventDefault)
-        e.preventDefault()
+        e.preventDefault();
 
     if (e.stopPropagation)
         e.stopPropagation();
@@ -213,6 +263,11 @@ function ev(e) {
 }
 
 
+function noope(e) {
+    ev(e);
+}
+
+
 // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/endsWith
 if (!String.prototype.endsWith)
     String.prototype.endsWith = function (search, this_len) {
@@ -249,6 +304,14 @@ if (!Element.prototype.closest)
         } while (el !== null && el.nodeType === 1);
     };
 
+if (!String.prototype.format)
+    String.prototype.format = function () {
+        var args = arguments;
+        return this.replace(/{(\d+)}/g, function (match, number) {
+            return typeof args[number] != 'undefined' ?
+                args[number] : match;
+        });
+    };
 
 // https://stackoverflow.com/a/950146
 function import_js(url, cb) {
@@ -312,7 +375,7 @@ function clmod(el, cls, add) {
 
     var n2 = n1.replace(re, ' ') + (add ? ' ' + cls : '');
 
-    if (!n1 == !n2)
+    if (n1 == n2)
         return false;
 
     el.className = n2;
@@ -327,11 +390,37 @@ function clgot(el, cls) {
     if (el.classList)
         return el.classList.contains(cls);
 
-    var lst = (el.getAttribute('class') + '').split(/ /g);
+    var lst = (el.className + '').split(/ /g);
     return has(lst, cls);
 }
 
 
+var ANIM = true;
+try {
+    var mq = window.matchMedia('(prefers-reduced-motion: reduce)');
+    mq.onchange = function () {
+        ANIM = !mq.matches;
+    };
+    ANIM = !mq.matches;
+}
+catch (ex) { }
+
+
+function yscroll() {
+    if (document.documentElement.scrollTop) {
+        return (window.yscroll = function () {
+            return document.documentElement.scrollTop;
+        })();
+    }
+    if (window.pageYOffset) {
+        return (window.yscroll = function () {
+            return window.pageYOffset;
+        })();
+    }
+    return 0;
+}
+
+
 function showsort(tab) {
     var v, vn, v1, v2, th = tab.tHead,
         sopts = jread('fsort', [["href", 1, ""]]);
@@ -420,6 +509,16 @@ function sortTable(table, col, cb) {
         }
         return reverse * (a.localeCompare(b));
     });
+    if (sread('dir1st') !== '0') {
+        var r1 = [], r2 = [];
+        for (var i = 0; i < tr.length; i++) {
+            var cell = tr[vl[i][1]].cells[1],
+                href = cell.getAttribute('sortv') || cell.textContent.trim();
+
+            (href.split('?')[0].slice(-1) == '/' ? r1 : r2).push(vl[i]);
+        }
+        vl = r1.concat(r2);
+    }
     for (i = 0; i < tr.length; ++i) tb.appendChild(tr[vl[i][1]]);
     if (cb) cb();
 }
@@ -448,7 +547,7 @@ function linksplit(rp, id) {
         q = '?' + q[1];
     }
 
-    if (rp && rp.charAt(0) == '/')
+    if (rp && rp[0] == '/')
         rp = rp.slice(1);
 
     while (rp) {
@@ -461,7 +560,7 @@ function linksplit(rp, id) {
             link = rp.slice(0, ofs + 1);
             rp = rp.slice(ofs + 1);
         }
-        var vlink = esc(uricom_dec(link)[0]);
+        var vlink = esc(uricom_dec(link));
 
         if (link.indexOf('/') !== -1) {
             vlink = vlink.slice(0, -1) + '<span>/</span>';
@@ -519,6 +618,17 @@ function url_enc(txt) {
 
 
 function uricom_dec(txt) {
+    try {
+        return decodeURIComponent(txt);
+    }
+    catch (ex) {
+        console.log("ucd-err [" + txt + "]");
+        return txt;
+    }
+}
+
+
+function uricom_sdec(txt) {
     try {
         return [decodeURIComponent(txt), true];
     }
@@ -532,7 +642,7 @@ function uricom_dec(txt) {
 function uricom_adec(arr, li) {
     var ret = [];
     for (var a = 0; a < arr.length; a++) {
-        var txt = uricom_dec(arr[a])[0];
+        var txt = uricom_dec(arr[a]);
         ret.push(li ? '<li>' + esc(txt) + '</li>' : txt);
     }
 
@@ -554,7 +664,7 @@ function get_evpath() {
 
 
 function get_vpath() {
-    return uricom_dec(get_evpath())[0];
+    return uricom_dec(get_evpath());
 }
 
 
@@ -584,6 +694,14 @@ function s2ms(s) {
 }
 
 
+var isNum = function (v) {
+    var n = parseFloat(v);
+    return !isNaN(v - n) && n === v;
+};
+if (window.Number && Number.isFinite)
+    isNum = Number.isFinite;
+
+
 function f2f(val, nd) {
     // 10.toFixed(1) returns 10.00 for certain values of 10
     val = (val * Math.pow(10, nd)).toFixed(0).split('.')[0];
@@ -592,18 +710,19 @@ function f2f(val, nd) {
 
 
 function humansize(b, terse) {
-    var i = 0, u = terse ? ['B', 'K', 'M', 'G'] : ['B', 'KB', 'MB', 'GB'];
-    while (b >= 1000 && i < u.length) {
+    var i = 0, u = ['B', 'KB', 'MB', 'GB', 'TB', 'PB'];
+    while (b >= 1000 && i < u.length - 1) {
         b /= 1024;
         i += 1;
     }
-    return f2f(b, b >= 100 ? 0 : b >= 10 ? 1 : 2) + ' ' + u[i];
+    return (f2f(b, b >= 100 ? 0 : b >= 10 ? 1 : 2) +
+        ' ' + (terse ? u[i].charAt(0) : u[i]));
 }
 
 
 function humantime(v) {
     if (v >= 60 * 60 * 24)
-        return v;
+        return shumantime(v);
 
     try {
         return /.*(..:..:..).*/.exec(new Date(v * 1000).toUTCString())[1];
@@ -614,12 +733,55 @@ function humantime(v) {
 }
 
 
+function shumantime(v, long) {
+    if (v < 10)
+        return f2f(v, 2) + 's';
+    if (v < 60)
+        return f2f(v, 1) + 's';
+
+    v = parseInt(v);
+    var st = [[60 * 60 * 24, 60 * 60, 'd'], [60 * 60, 60, 'h'], [60, 1, 'm']];
+
+    for (var a = 0; a < st.length; a++) {
+        var m1 = st[a][0],
+            m2 = st[a][1],
+            ch = st[a][2];
+
+        if (v < m1)
+            continue;
+
+        var v1 = parseInt(v / m1),
+            v2 = ('0' + parseInt((v % m1) / m2)).slice(-2);
+
+        return v1 + ch + (v1 >= 10 || v2 == '00' ? '' : v2 + (
+            long && a < st.length - 1 ? st[a + 1][2] : ''));
+    }
+}
+
+
+function lhumantime(v) {
+    var t = shumantime(v, 1),
+        tp = t.replace(/([a-z])/g, " $1 ").split(/ /g).slice(0, -1);
+
+    if (!L || tp.length < 2 || tp[1].indexOf('$') + 1)
+        return t;
+
+    var ret = '';
+    for (var a = 0; a < tp.length; a += 2)
+        ret += tp[a] + ' ' + L['ht_' + tp[a + 1]].replace(tp[a] == 1 ? /!.*/ : /!/, '') + L.ht_and;
+
+    return ret.slice(0, -L.ht_and.length);
+}
+
+
 function clamp(v, a, b) {
     return Math.min(Math.max(v, a), b);
 }
 
 
 function has(haystack, needle) {
+    try { return haystack.includes(needle); } catch (ex) { }
+
     for (var a = 0; a < haystack.length; a++)
         if (haystack[a] == needle)
             return true;
@@ -682,7 +844,7 @@ function fcfg_get(name, defval) {
     var o = ebi(name),
         val = parseFloat(sread(name));
 
-    if (isNaN(val))
+    if (!isNum(val))
         return parseFloat(o ? o.value : defval);
 
     if (o)
@@ -773,14 +935,18 @@ function scfg_bind(obj, oname, cname, defval, cb) {
 
 function hist_push(url) {
     console.log("h-push " + url);
-    if (window.history && history.pushState)
+    try {
         history.pushState(url, url, url);
+    }
+    catch (ex) { }
 }
 
 function hist_replace(url) {
     console.log("h-repl " + url);
-    if (window.history && history.replaceState)
+    try {
         history.replaceState(url, url, url);
+    }
+    catch (ex) { }  // ff "The operation is insecure." on rapid switches
 }
 
 function sethash(hv) {
@@ -792,6 +958,14 @@ function sethash(hv) {
     }
 }
 
+function dl_file(url) {
+    console.log('DL [%s]', url);
+    var o = mknod('a');
+    o.setAttribute('href', url);
+    o.setAttribute('download', '');
+    o.click();
+}
+
 
 var timer = (function () {
     var r = {};
@@ -831,8 +1005,8 @@ var timer = (function () {
 
 var tt = (function () {
     var r = {
-        "tt": mknod("div"),
-        "th": mknod("div"),
+        "tt": mknod("div", 'tt'),
+        "th": mknod("div", 'tth'),
         "en": true,
         "el": null,
         "skip": false,
@@ -840,8 +1014,6 @@ var tt = (function () {
     };
 
     r.th.innerHTML = '?';
-    r.tt.setAttribute('id', 'tt');
-    r.th.setAttribute('id', 'tth');
     document.body.appendChild(r.tt);
     document.body.appendChild(r.th);
 
@@ -853,7 +1025,7 @@ var tt = (function () {
         prev = this;
     };
 
-    var tev;
+    var tev, vh;
     r.dshow = function (e) {
         clearTimeout(tev);
         if (!r.getmsg(this))
@@ -863,16 +1035,17 @@ var tt = (function () {
             return r.show.bind(this)();
 
         tev = setTimeout(r.show.bind(this), 800);
-        if (is_touch)
+        if (TOUCH)
             return;
 
+        vh = window.innerHeight;
         this.addEventListener('mousemove', r.move);
         clmod(r.th, 'act', 1);
         r.move(e);
     };
 
     r.getmsg = function (el) {
-        if (QS('body.bbox-open'))
+        if (IPHONE && QS('body.bbox-open'))
             return;
 
         var cfg = sread('tooltips');
@@ -927,7 +1100,7 @@ var tt = (function () {
     };
 
     r.hide = function (e) {
-        ev(e);
+        //ev(e);  // eats checkbox-label clicks
         clearTimeout(tev);
         window.removeEventListener('scroll', r.hide);
 
@@ -944,8 +1117,9 @@ var tt = (function () {
     };
 
     r.move = function (e) {
+        var sy = e.clientY + 128 > vh ? -1 : 1;
         r.th.style.left = (e.pageX + 12) + 'px';
-        r.th.style.top = (e.pageY + 12) + 'px';
+        r.th.style.top = (e.pageY + 12 * sy) + 'px';
     };
 
     if (IPHONE) {
@@ -1010,12 +1184,12 @@ var toast = (function () {
     var r = {},
         te = null,
         scrolling = false,
-        obj = mknod('div');
+        obj = mknod('div', 'toast');
 
-    obj.setAttribute('id', 'toast');
     document.body.appendChild(obj);
     r.visible = false;
     r.txt = null;
+    r.tag = obj;  // filler value (null is scary)
 
     function scrollchk() {
         if (scrolling)
@@ -1044,9 +1218,10 @@ var toast = (function () {
         clearTimeout(te);
         clmod(obj, 'vis');
         r.visible = false;
+        r.tag = obj;
     };
 
-    r.show = function (cl, sec, txt) {
+    r.show = function (cl, sec, txt, tag) {
         clearTimeout(te);
         if (sec)
             te = setTimeout(r.hide, sec * 1000);
@@ -1062,19 +1237,20 @@ var toast = (function () {
         timer.add(scrollchk);
         r.visible = true;
         r.txt = txt;
+        r.tag = tag;
     };
 
-    r.ok = function (sec, txt) {
-        r.show('ok', sec, txt);
+    r.ok = function (sec, txt, tag) {
+        r.show('ok', sec, txt, tag);
     };
-    r.inf = function (sec, txt) {
-        r.show('inf', sec, txt);
+    r.inf = function (sec, txt, tag) {
+        r.show('inf', sec, txt, tag);
     };
-    r.warn = function (sec, txt) {
-        r.show('warn', sec, txt);
+    r.warn = function (sec, txt, tag) {
+        r.show('warn', sec, txt, tag);
     };
-    r.err = function (sec, txt) {
-        r.show('err', sec, txt);
+    r.err = function (sec, txt, tag) {
+        r.show('err', sec, txt, tag);
     };
 
     return r;
@@ -1088,15 +1264,21 @@ var modal = (function () {
         cb_up = null,
         cb_ok = null,
         cb_ng = null,
-        prim = '<a href="#" id="modal-ok">OK</a>',
-        sec = '<a href="#" id="modal-ng">Cancel</a>',
+        tok, tng, prim, sec, ok_cancel;
+
+    r.load = function () {
+        tok = (L && L.m_ok) || 'OK';
+        tng = (L && L.m_ng) || 'Cancel';
+        prim = '<a href="#" id="modal-ok">' + tok + '</a>';
+        sec = '<a href="#" id="modal-ng">' + tng + '</a>';
         ok_cancel = WINDOWS ? prim + sec : sec + prim;
+    };
+    r.load();
 
     r.busy = false;
 
     r.show = function (html) {
-        o = mknod('div');
-        o.setAttribute('id', 'modal');
+        o = mknod('div', 'modal');
         o.innerHTML = '<table><tr><td><div id="modalc">' + html + '</div></td></tr></table>';
         document.body.appendChild(o);
         document.addEventListener('keydown', onkey);
@@ -1151,7 +1333,8 @@ var modal = (function () {
             return;
 
         setTimeout(function () {
-            ebi('modal-ok').focus();
+            if (ctr = ebi('modal-ok'))
+                ctr.focus();
         }, 20);
         ev(e);
     }
@@ -1172,6 +1355,9 @@ var modal = (function () {
             return ok();
         }
 
+        if ((k == 'ArrowLeft' || k == 'ArrowRight') && eng && (ae == eok || ae == eng))
+            return (ae == eok ? eng : eok).focus() || ev(e);
+
         if (k == 'Escape')
             return ng();
     }
@@ -1194,17 +1380,17 @@ var modal = (function () {
         r.show(html);
     }
 
-    r.confirm = function (html, cok, cng, fun) {
+    r.confirm = function (html, cok, cng, fun, btns) {
         q.push(function () {
-            _confirm(lf2br(html), cok, cng, fun);
+            _confirm(lf2br(html), cok, cng, fun, btns);
         });
         next();
     }
-    function _confirm(html, cok, cng, fun) {
+    function _confirm(html, cok, cng, fun, btns) {
         cb_ok = cok;
         cb_ng = cng === undefined ? cok : cng;
         cb_up = fun;
-        html += '<div id="modalb">' + ok_cancel + '</div>';
+        html += '<div id="modalb">' + (btns || ok_cancel) + '</div>';
         r.show(html);
     }
 
@@ -1306,8 +1492,10 @@ function repl(e) {
         if (!cmd)
             return toast.inf(3, 'eval aborted');
 
-        if (cmd.startsWith(','))
-            return modal.alert(esc(eval(cmd.slice(1)) + ''))
+        if (cmd.startsWith(',')) {
+            evalex_fatal = true;
+            return modal.alert(esc(eval(cmd.slice(1)) + ''));
+        }
 
         try {
             modal.alert(esc(eval(cmd) + ''));
@@ -1321,6 +1509,49 @@ if (ebi('repl'))
     ebi('repl').onclick = repl;
 
 
+var md_plug = {};
+var md_plug_err = function (ex, js) {
+    if (ex)
+        console.log(ex, js);
+};
+function load_md_plug(md_text, plug_type) {
+    if (!have_emp)
+        return md_text;
+
+    var find = '\n```copyparty_' + plug_type + '\n';
+    var ofs = md_text.indexOf(find);
+    if (ofs === -1)
+        return md_text;
+
+    var ofs2 = md_text.indexOf('\n```', ofs + 1);
+    if (ofs2 == -1)
+        return md_text;
+
+    var js = md_text.slice(ofs + find.length, ofs2 + 1);
+    var md = md_text.slice(0, ofs + 1) + md_text.slice(ofs2 + 4);
+
+    var old_plug = md_plug[plug_type];
+    if (!old_plug || old_plug[1] != js) {
+        js = 'const x = { ' + js + ' }; x;';
+        try {
+            var x = eval(js);
+            if (x['ctor']) {
+                x['ctor']();
+                delete x['ctor'];
+            }
+        }
+        catch (ex) {
+            md_plug[plug_type] = null;
+            md_plug_err(ex, js);
+            return md;
+        }
+        md_plug[plug_type] = [x, js];
+    }
+
+    return md;
+}
+
+
 var svg_decl = '<?xml version="1.0" encoding="UTF-8"?>\n';
 
 
@@ -1346,12 +1577,24 @@ var favico = (function () {
         var b64;
         try {
             b64 = btoa(svg ? svg_decl + svg : gx(r.txt));
+            //console.log('f1');
         }
-        catch (ex) {
-            b64 = encodeURIComponent(r.txt).replace(/%([0-9A-F]{2})/g,
-                function x(m, v) { return String.fromCharCode('0x' + v); });
-
-            b64 = btoa(gx(unescape(encodeURIComponent(r.txt))));
+        catch (e1) {
+            try {
+                b64 = btoa(gx(encodeURIComponent(r.txt).replace(/%([0-9A-F]{2})/g,
+                    function x(m, v) { return String.fromCharCode('0x' + v); })));
+                //console.log('f2');
+            }
+            catch (e2) {
+                try {
+                    b64 = btoa(gx(unescape(encodeURIComponent(r.txt))));
+                    //console.log('f3');
+                }
+                catch (e3) {
+                    //console.log('fe');
+                    return;
+                }
+            }
         }
 
         if (!r.tag) {
@@ -1364,12 +1607,49 @@ var favico = (function () {
 
     r.init = function () {
         clearTimeout(r.to);
-        scfg_bind(r, 'txt', 'icot', '', r.upd);
-        scfg_bind(r, 'fg', 'icof', 'fc5', r.upd);
-        scfg_bind(r, 'bg', 'icob', '222', r.upd);
+        var dv = (window.dfavico || '').trim().split(/ +/),
+            fg = dv.length < 2 ? 'fc5' : dv[1].toLowerCase() == 'none' ? '' : dv[1],
+            bg = dv.length < 3 ? '222' : dv[2].toLowerCase() == 'none' ? '' : dv[2];
+
+        scfg_bind(r, 'txt', 'icot', dv[0], r.upd);
+        scfg_bind(r, 'fg', 'icof', fg, r.upd);
+        scfg_bind(r, 'bg', 'icob', bg, r.upd);
         r.upd();
     };
 
     r.to = setTimeout(r.init, 100);
     return r;
 })();
+
+
+var cf_cha_t = 0;
+function xhrchk(xhr, prefix, e404, lvl, tag) {
+    if (xhr.status < 400 && xhr.status >= 200)
+        return true;
+
+    if (xhr.status == 403)
+        return toast.err(0, prefix + (L && L.xhr403 || "403: access denied\n\ntry pressing F5, maybe you got logged out"), tag);
+
+    if (xhr.status == 404)
+        return toast.err(0, prefix + e404, tag);
+
+    var errtxt = (xhr.response && xhr.response.err) || xhr.responseText,
+        fun = toast[lvl || 'err'];
+
+    if (xhr.status == 503 && /[Cc]loud[f]lare|>Just a mo[m]ent|#cf-b[u]bbles|Chec[k]ing your br[o]wser/.test(errtxt)) {
+        var now = Date.now(), td = now - cf_cha_t;
+        if (td < 15000)
+            return;
+
+        cf_cha_t = now;
+        errtxt = 'Clou' + wah + 'dflare protection kicked in\n\n<strong>trying to fix it...</strong>';
+        fun = toast.warn;
+
+        qsr('#cf_frame');
+        var fr = mknod('iframe', 'cf_frame');
+        fr.src = '/?cf_challenge';
+        document.body.appendChild(fr);
+    }
+
+    return fun(0, prefix + xhr.status + ": " + errtxt, tag);
+}

copyparty/web/w.hash.js

@@ -0,0 +1,106 @@
+"use strict";
+
+
+function hex2u8(txt) {
+    return new Uint8Array(txt.match(/.{2}/g).map(function (b) { return parseInt(b, 16); }));
+}
+
+
+var subtle = null;
+try {
+    subtle = crypto.subtle;
+    subtle.digest('SHA-512', new Uint8Array(1)).then(
+        function (x) { },
+        function (x) { load_fb(); }
+    );
+}
+catch (ex) {
+    load_fb();
+}
+function load_fb() {
+    subtle = null;
+    importScripts('/.cpr/deps/sha512.hw.js');
+}
+
+
+var reader = null,
+    gc1, gc2, gc3,
+    busy = false;
+
+
+onmessage = (d) => {
+    if (busy)
+        return postMessage(["panic", 'worker got another task while busy']);
+
+    if (!reader)
+        reader = new FileReader();
+
+    var [nchunk, fobj, car, cdr] = d.data,
+        t0 = Date.now();
+
+    reader.onload = function (e) {
+        try {
+            // chrome gc forgets the filereader output; remind it
+            // (for some chromes, also necessary for subtle)
+            gc1 = e.target.result;
+            gc2 = new Uint8Array(gc1, 0, 1);
+            gc3 = new Uint8Array(gc1, gc1.byteLength - 1);
+
+            //console.log('[ w] %d HASH bgin', nchunk);
+            postMessage(["read", nchunk, cdr - car, Date.now() - t0]);
+            hash_calc(gc1);
+        }
+        catch (ex) {
+            busy = false;
+            postMessage(["panic", ex + '']);
+        }
+    };
+    reader.onerror = function () {
+        busy = false;
+        var err = reader.error + '';
+
+        if (err.indexOf('NotReadableError') !== -1 || // win10-chrome defender
+            err.indexOf('NotFoundError') !== -1  // macos-firefox permissions
+        )
+            return postMessage(["fail", 'OS-error', err + ' @ ' + car]);
+
+        postMessage(["ferr", err]);
+    };
+    //console.log('[ w] %d read bgin', nchunk);
+    busy = true;
+    reader.readAsArrayBuffer(fobj.slice(car, cdr));
+
+
+    var hash_calc = function (buf) {
+        var hash_done = function (hashbuf) {
+            // stop gc from attempting to free early
+            if (!gc1 || !gc2 || !gc3)
+                return console.log('torch went out');
+
+            gc1 = gc2 = gc3 = null;
+            busy = false;
+            try {
+                var hslice = new Uint8Array(hashbuf).subarray(0, 33);
+                //console.log('[ w] %d HASH DONE', nchunk);
+                postMessage(["done", nchunk, hslice, cdr - car]);
+            }
+            catch (ex) {
+                postMessage(["panic", ex + '']);
+            }
+        };
+
+        // stop gc from attempting to free early
+        if (!gc1 || !gc2 || !gc3)
+            console.log('torch went out');
+
+        if (subtle)
+            subtle.digest('SHA-512', buf).then(hash_done);
+        else {
+            // note: lifting u8buf counterproductive for the chrome gc bug
+            var u8buf = new Uint8Array(buf);
+            hashwasm.sha512(u8buf).then(function (v) {
+                hash_done(hex2u8(v))
+            });
+        }
+    };
+}

docs/README.md

@@ -13,6 +13,9 @@
 
 # other stuff
 
+## [`changelog.md`](changelog.md)
+* occasionally grabbed from github release notes
+
 ## [`rclone.md`](rclone.md)
 * notes on using rclone as a fuse client/server
 

docs/lics.txt

@@ -0,0 +1,67 @@
+--- server-side --- software ---
+
+https://github.com/9001/copyparty/
+C: 2019 ed
+L: MIT
+
+https://github.com/pallets/jinja/
+C: 2007 Pallets
+L: BSD 3-Clause 
+
+https://github.com/pallets/markupsafe/
+C: 2010 Pallets
+L: BSD 3-Clause 
+
+https://github.com/giampaolo/pyftpdlib/
+C: 2007 Giampaolo Rodola'
+L: MIT
+
+https://github.com/nayuki/QR-Code-generator
+C: Project Nayuki
+L: MIT
+
+https://github.com/python/cpython/blob/3.10/Lib/asyncore.py
+C: 1996 Sam Rushing
+L: ISC
+
+https://github.com/ahupp/python-magic/
+C: 2001-2014 Adam Hupp
+L: MIT
+
+--- client-side --- software ---
+
+https://github.com/Daninet/hash-wasm/
+C: 2020 Dani Biró
+L: MIT
+
+https://github.com/openpgpjs/asmcrypto.js/
+C: 2013 Artem S Vybornov
+L: MIT
+
+https://github.com/feimosi/baguetteBox.js/
+C: 2017 Marek Grzybek
+L: MIT
+
+https://github.com/markedjs/marked/
+C: 2018+, MarkedJS
+C: 2011-2018, Christopher Jeffrey (https://github.com/chjj/)
+L: MIT
+
+https://github.com/codemirror/codemirror5/
+C: 2017 Marijn Haverbeke <marijnh@gmail.com> and others
+L: MIT
+
+https://github.com/Ionaru/easy-markdown-editor/
+C: 2015 Sparksuite, Inc.
+C: 2017 Jeroen Akkerman.
+L: MIT
+
+--- client-side --- fonts ---
+
+https://github.com/adobe-fonts/source-code-pro/
+C: 2010-2019 Adobe
+L: SIL OFL 1.1
+
+https://github.com/FortAwesome/Font-Awesome/
+C: 2022 Fonticons, Inc.
+L: SIL OFL 1.1

docs/notes.bat

@@ -0,0 +1,4 @@
+rem appending a static ip to a dhcp nic on windows 10-1703 or later
+netsh interface ipv4 show interface
+netsh interface ipv4 set interface interface="Ethernet 2" dhcpstaticipcoexistence=enabled
+netsh interface ipv4 add address "Ethernet 2" 10.1.2.4 255.255.255.0

docs/notes.md

@@ -0,0 +1,10 @@
+# up2k.js
+
+## potato detection
+
+* tsk 0.25/8.4/31.5 bzw 1.27/22.9/18 = 77% (38.4s, 49.7s)
+  * 4c locale #1313, ff-102,deb-11 @ ryzen4500u wifi -> win10
+  * profiling shows 2sec heavy gc every 2sec
+
+* tsk 0.41/4.1/10 bzw 1.41/9.9/7 = 73% (13.3s, 18.2s)
+  * 4c locale #1313, ch-103,deb-11 @ ryzen4500u wifi -> win10

docs/notes.sh

@@ -48,7 +48,15 @@ avg() { awk 'function pr(ncsz) {if (nsmp>0) {printf "%3s %s\n", csz, sum/nsmp} c
 ## time between first and last upload
 
 python3 -um copyparty -nw -v srv::rw -i 127.0.0.1 2>&1 | tee log 
-cat log | awk '!/"purl"/{next} {s=$1;sub(/[^m]+m/,"");gsub(/:/," ");t=60*(60*$1+$2)+$3} !a{a=t;sa=s} {b=t;sb=s} END {print b-a,sa,sb}'
+cat log | awk '!/"purl"/{next} {s=$1;sub(/[^m]+m/,"");gsub(/:/," ");t=60*(60*$1+$2)+$3} t<p{t+=86400} !a{a=t;sa=s} {b=t;sb=s} END {print b-a,sa,sb}'
+
+# or if the client youre measuring dies for ~15sec every once ina while and you wanna filter those out,
+cat log | awk '!/"purl"/{next} {s=$1;sub(/[^m]+m/,"");gsub(/:/," ");t=60*(60*$1+$2)+$3} t<p{t+=86400} !p{a=t;p=t;r=0;next} t-p>1{printf "%.3f += %.3f - %.3f  (%.3f)  # %.3f -> %.3f\n",r,p,a,p-a,p,t;r+=p-a;a=t} {p=t} END {print r+p-a}'
+
+
+##
+## find uploads blocked by slow i/o or maybe deadlocks
+awk '/^.\+. opened logfile/{print;next} {sub(/.$/,"")} !/^..36m[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{3} /{next} !/0m(POST|writing) /{next} {c=0;p=$3} /0mPOST/{c=1} {s=$1;sub(/[^m]+m/,"");gsub(/:/," ");s=60*(60*$1+$2)+$3} c{t[p]=s;next} {d=s-t[p]} d>10{print $0 "  # " d}'
 
 
 ##
@@ -59,6 +67,7 @@ mkdir -p "${dirs[@]}"
 for dir in "${dirs[@]}"; do for fn in ふが "$(printf \\xed\\x93)" 'qw,er;ty%20as df?gh+jkl%zxc&vbn <qwe>"rty'"'"'uio&asd&nbsp;fgh'; do echo "$dir" > "$dir/$fn.html"; done; done
 # qw er+ty%20ui%%20op<as>df&gh&amp;jk#zx'cv"bn`m=qw*er^ty?ui@op,as.df-gh_jk
 
+
 ##
 ## upload mojibake
 
@@ -135,6 +144,43 @@ sqlite3 -readonly up2k.db.key-full 'select w, v from mt where k = "key" order by
 sqlite3 -readonly up2k.db.key-full 'select w, v from mt where k = "key" order by w' > k1; sqlite3 -readonly up2k.db 'select mt.w, mt.v, up.rd, up.fn from mt inner join up on mt.w = substr(up.w,1,16) where mt.k = "key" order by up.rd, up.fn' > k2; ok=0; ng=0; while IFS='|' read w k2 path; do k1="$(grep -E "^$w" k1 | sed -r 's/.*\|//')"; [ "$k1" = "$k2" ] && ok=$((ok+1)) || { ng=$((ng+1)); printf '%3s %3s  %s\n' "$k1" "$k2" "$path"; }; done < <(cat k2); echo "match $ok   diff $ng"
 
 
+##
+## scanning for exceptions
+
+cd /dev/shm
+journalctl -aS '720 hour ago' -t python3 -o with-unit --utc | cut -d\  -f2,6- > cpp.log
+tac cpp.log | awk '/RuntimeError: generator ignored GeneratorExit/{n=1} n{n--;if(n==0)print} 1' | grep 'generator ignored GeneratorExit' -C7 | head -n 100
+awk '/Exception ignored in: <generator object StreamZip.gen/{s=1;next} /could not create thumbnail/{s=3;next} s{s--;next} 1' <cpp.log | less -R
+less-search:
+  >: |Exception|Traceback
+
+
+##
+## tracking bitflips
+
+l=log.tmux-1662316902  # your logfile (tmux-capture or decompressed -lo)
+
+# grab handshakes to a smaller logfile
+tr -d '\r' <$l | awk '/^.\[36m....-..-...\[0m.?$/{d=substr($0,6,10)} !d{next} /"purl": "/{t=substr($1,6);sub(/[^ ]+ /,"");sub(/ .\[34m[0-9]+ /," ");printf("%s %s %s %s\n",d,t,ip,$0)}' | while read d t ip f; do u=$(date +%s --date="${d}T${t}Z"); printf '%s\n' "$u $ip $f"; done > handshakes
+
+# quick list of affected files
+grep 'your chunk got corrupted somehow' -A1 $l | tr -d '\r' | grep -E '^[a-zA-Z0-9_-]{44}$' | sort | uniq | while IFS= read -r x; do grep -F "$x" handshakes | head -c 200; echo; done | sed -r 's/.*"name": "//' | sort | uniq -cw20
+
+# find all cases of corrupt chunks and print their respective handshakes (if any),
+# timestamps are when the corrupted chunk was received (and also the order they are displayed),
+# first checksum is the expected value from the handshake, second is what got uploaded
+awk <$l '/^.\[36m....-..-...\[0m.?$/{d=substr($0,6,10)} /your chunk got corrupted somehow/{n=2;t=substr($1,6);next} !n{next} {n--;sub(/\r$/,"")} n{a=$0;next} {sub(/.\[0m,.*/,"");printf "%s %s %s %s\n",d,t,a,$0}' |
+while read d t h1 h2; do printf '%s %s\n' $d $t; (
+printf '  %s [%s]\n' $h1 "$(grep -F $h1 <handshakes | head -n 1)"
+printf '  %s [%s]\n' $h2 "$(grep -F $h2 <handshakes | head -n 1)"
+) | sed 's/, "sprs":.*//'; done | less -R
+
+# notes; TODO clean up and put in the readme maybe --
+# quickest way to drop the bad files (if a client generated bad hashes for the initial handshake) is shutting down copyparty and moving aside the unfinished file (both the .PARTIAL and the empty placeholder)
+# BUT the clients will immediately re-handshake the upload with the same bitflipped hashes, so the uploaders have to refresh their browsers before you do that,
+# so maybe just ask them to refresh and do nothing for 6 hours so the timeout kicks in, which deletes the placeholders/name-reservations and you can then manually delete the .PARTIALs at some point later
+
+
 ##
 ## media
 
@@ -182,7 +228,7 @@ brew install python@2
 pip install virtualenv
 
 # readme toc
-cat README.md | awk 'function pr() { if (!h) {return}; if (/^ *[*!#|]/||!s) {printf "%s\n",h;h=0;return}; if (/.../) {printf "%s - %s\n",h,$0;h=0}; };    /^#/{s=1;pr()} /^#* *(file indexing|install on android|dev env setup|just the sfx|complete release|optional gpl stuff)|`$/{s=0} /^#/{lv=length($1);sub(/[^ ]+ /,"");bab=$0;gsub(/ /,"-",bab); h=sprintf("%" ((lv-1)*4+1) "s [%s](#%s)", "*",$0,bab);next} !h{next} {sub(/  .*/,"");sub(/[:,]$/,"")} {pr()}' > toc; grep -E '^## readme toc' -B1000 -A2 <README.md >p1; grep -E '^## quickstart' -B2 -A999999 <README.md >p2; (cat p1; grep quickstart -A1000 <toc; cat p2) >README.md; rm p1 p2 toc
+cat README.md | awk 'function pr() { if (!h) {return}; if (/^ *[*!#|]/||!s) {printf "%s\n",h;h=0;return}; if (/.../) {printf "%s - %s\n",h,$0;h=0}; };    /^#/{s=1;pr()} /^#* *(install on android|dev env setup|just the sfx|complete release|optional gpl stuff)|`$/{s=0} /^#/{lv=length($1);sub(/[^ ]+ /,"");bab=$0;gsub(/ /,"-",bab);gsub(/\./,"",bab); h=sprintf("%" ((lv-1)*4+1) "s [%s](#%s)", "*",$0,bab);next} !h{next} {sub(/  .*/,"");sub(/[:;,]$/,"")} {pr()}' > toc; grep -E '^## readme toc' -B1000 -A2 <README.md >p1; grep -E '^## quickstart' -B2 -A999999 <README.md >p2; (cat p1; grep quickstart -A1000 <toc; cat p2) >README.md; rm p1 p2 toc
 
 # fix firefox phantom breakpoints,
 # suggestions from bugtracker, doesnt work (debugger is not attachable)
@@ -200,6 +246,9 @@ git pull; git reset --hard origin/HEAD && git log --format=format:"%H %ai %d" --
 # download all sfx versions
 curl https://api.github.com/repos/9001/copyparty/releases?per_page=100 | jq -r '.[] | .tag_name + " " + .name' | tr -d '\r' | while read v t; do fn="$(printf '%s\n' "copyparty $v $t.py" | tr / -)"; [ -e "$fn" ] || curl https://github.com/9001/copyparty/releases/download/$v/copyparty-sfx.py -Lo "$fn"; done
 
+# convert releasenotes to changelog
+curl https://api.github.com/repos/9001/copyparty/releases?per_page=100 | jq -r '.[] | "▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀  \n# \(.created_at)  `\(.tag_name)`  \(.name)\n\n\(.body)\n\n\n"' | sed -r 's/^# ([0-9]{4}-)([0-9]{2})-([0-9]{2})T([0-9]{2}):([0-9]{2}):([0-9]{2})Z /# \1\2\3-\4\5 /' > changelog.md
+
 # push to multiple git remotes
 git config -l | grep '^remote'
 git remote add all git@github.com:9001/copyparty.git

docs/pyoxidizer.txt

@@ -0,0 +1,52 @@
+pyoxidizer doesn't crosscompile yet so need to build in a windows vm,
+luckily possible to do mostly airgapped (https-proxy for crates)
+
+none of this is version-specific but doing absolute links just in case
+(only exception is py3.8 which is the final win7 ver)
+
+# deps (download on linux host):
+https://www.python.org/ftp/python/3.10.7/python-3.10.7-amd64.exe
+https://github.com/indygreg/PyOxidizer/releases/download/pyoxidizer%2F0.22.0/pyoxidizer-0.22.0-x86_64-pc-windows-msvc.zip
+https://github.com/upx/upx/releases/download/v3.96/upx-3.96-win64.zip
+https://static.rust-lang.org/dist/rust-1.61.0-x86_64-pc-windows-msvc.msi
+https://github.com/indygreg/python-build-standalone/releases/download/20220528/cpython-3.8.13%2B20220528-i686-pc-windows-msvc-static-noopt-full.tar.zst
+
+# need cl.exe, prefer 2017 -- download on linux host:
+https://visualstudio.microsoft.com/downloads/?q=build+tools
+https://docs.microsoft.com/en-us/visualstudio/releases/2022/release-history#release-dates-and-build-numbers
+https://aka.ms/vs/15/release/vs_buildtools.exe  # 2017
+https://aka.ms/vs/16/release/vs_buildtools.exe  # 2019
+https://aka.ms/vs/17/release/vs_buildtools.exe  # 2022
+https://docs.microsoft.com/en-us/visualstudio/install/workload-component-id-vs-build-tools?view=vs-2017
+
+# use disposable w10 vm to prep offline installer; xfer to linux host with firefox to copyparty
+vs_buildtools-2017.exe --add Microsoft.VisualStudio.Workload.MSBuildTools --add Microsoft.VisualStudio.Workload.VCTools --add Microsoft.VisualStudio.Component.Windows10SDK.17763 --layout c:\msbt2017 --lang en-us
+
+# need two proxies on host; s5s or ssh for msys2(socks5), and tinyproxy for rust(http)
+UP=- python3 socks5server.py 192.168.123.1 4321
+ssh -vND 192.168.123.1:4321 localhost
+git clone https://github.com/tinyproxy/tinyproxy.git
+  ./autogen.sh
+  ./configure --prefix=/home/ed/pe/tinyproxy
+  make -j24 install
+  printf '%s\n' >cfg "Port 4380" "Listen 192.168.123.1"
+  ./tinyproxy -dccfg
+
+https://github.com/msys2/msys2-installer/releases/download/2022-09-04/msys2-x86_64-20220904.exe
+export all_proxy=socks5h://192.168.123.1:4321
+# if chat dies after auth (2 messages) it probably failed dns, note the h in socks5h to tunnel dns
+pacman -Syuu
+pacman -S git patch mingw64/mingw-w64-x86_64-zopfli
+cd /c && curl -k https://192.168.123.1:3923/ro/ox/msbt2017/?tar | tar -xv
+
+first install certs from msbt/certificates then admin-cmd `vs_buildtools.exe --noweb`,
+default selection (vc++2017-v15.9-v14.16, vc++redist, vc++bt-core) += win10sdk (for io.h)
+
+install rust without documentation, python 3.10, put upx and pyoxidizer into ~/bin,
+[cmd.exe] python -m pip install --user -U wheel-0.37.1.tar.gz strip-hints-0.1.10.tar.gz
+p=192.168.123.1:4380; export https_proxy=$p; export http_proxy=$p
+
+# and with all of the one-time-setup out of the way,
+mkdir /c/d; cd /c/d && curl -k https://192.168.123.1:3923/cpp/gb?pw=wark > gb && git clone gb copyparty
+cd /c/d/copyparty/ && curl -k https://192.168.123.1:3923/cpp/patch?pw=wark | patch -p1
+cd /c/d/copyparty/scripts && CARGO_HTTP_CHECK_REVOKE=false PATH=/c/Users/$USER/AppData/Local/Programs/Python/Python310:/c/Users/$USER/bin:"$(cygpath "C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\VC\Tools\MSVC\14.16.27023\bin\Hostx86\x86"):$PATH"  ./make-sfx.sh ox ultra

pyoxidizer.bzl

@@ -0,0 +1,48 @@
+# builds win7-i386 exe on win10-ltsc-1809(17763.316)
+# see docs/pyoxidizer.txt
+
+def make_exe():
+    dist = default_python_distribution(flavor="standalone_static", python_version="3.8")
+    policy = dist.make_python_packaging_policy()
+    policy.allow_files = True
+    policy.allow_in_memory_shared_library_loading = True
+    #policy.bytecode_optimize_level_zero = True
+    #policy.include_distribution_sources = False  # error instantiating embedded Python interpreter: during initializing Python main: init_fs_encoding: failed to get the Python codec of the filesystem encoding
+    policy.include_distribution_resources = False
+    policy.include_non_distribution_sources = False
+    policy.include_test = False
+    python_config = dist.make_python_interpreter_config()
+    #python_config.module_search_paths = ["$ORIGIN/lib"]
+
+    python_config.run_module = "copyparty"
+    exe = dist.to_python_executable(
+        name="copyparty",
+        config=python_config,
+        packaging_policy=policy,
+    )
+    exe.windows_runtime_dlls_mode = "never"
+    exe.windows_subsystem = "console"
+    exe.add_python_resources(exe.read_package_root(
+        path="sfx",
+        packages=[
+            "copyparty",
+            "jinja2",
+            "markupsafe",
+            "pyftpdlib",
+            "python-magic",
+        ]
+    ))
+    return exe
+
+def make_embedded_resources(exe):
+    return exe.to_embedded_resources()
+
+def make_install(exe):
+    files = FileManifest()
+    files.add_python_resource("copyparty", exe)
+    return files
+
+register_target("exe", make_exe)
+register_target("resources", make_embedded_resources, depends=["exe"], default_build_script=True)
+register_target("install", make_install, depends=["exe"], default=True)
+resolve_targets()

scripts/copyparty-repack.sh

@@ -119,7 +119,7 @@ chmod 755 \
 
 # extract the sfx
 ( cd copyparty-extras/sfx-full/
-./copyparty-sfx.py -h
+./copyparty-sfx.py --version
 )
 
 

scripts/deps-docker/Dockerfile

@@ -1,10 +1,10 @@
-FROM    alpine:3.15
+FROM    alpine:3
 WORKDIR /z
-ENV     ver_asmcrypto=5b994303a9d3e27e0915f72a10b6c2c51535a4dc \
+ENV     ver_asmcrypto=c72492f4a66e17a0e5dd8ad7874de354f3ccdaa5 \
         ver_hashwasm=4.9.0 \
-        ver_marked=4.0.12 \
-        ver_mde=2.16.1 \
-        ver_codemirror=5.65.2 \
+        ver_marked=4.0.18 \
+        ver_mde=2.18.0 \
+        ver_codemirror=5.65.9 \
         ver_fontawesome=5.13.0 \
         ver_zopfli=1.0.3
 
@@ -17,7 +17,7 @@ RUN     mkdir -p /z/dist/no-pk \
         && wget https://github.com/openpgpjs/asmcrypto.js/archive/$ver_asmcrypto.tar.gz -O asmcrypto.tgz \
         && wget https://github.com/markedjs/marked/archive/v$ver_marked.tar.gz -O marked.tgz \
         && wget https://github.com/Ionaru/easy-markdown-editor/archive/$ver_mde.tar.gz -O mde.tgz \
-        && wget https://github.com/codemirror/CodeMirror/archive/$ver_codemirror.tar.gz -O codemirror.tgz \
+        && wget https://github.com/codemirror/codemirror5/archive/$ver_codemirror.tar.gz -O codemirror.tgz \
         && wget https://github.com/FortAwesome/Font-Awesome/releases/download/$ver_fontawesome/fontawesome-free-$ver_fontawesome-web.zip -O fontawesome.zip \
         && wget https://github.com/google/zopfli/archive/zopfli-$ver_zopfli.tar.gz -O zopfli.tgz \
         && wget https://github.com/Daninet/hash-wasm/releases/download/v$ver_hashwasm/hash-wasm@$ver_hashwasm.zip -O hash-wasm.zip \
@@ -32,7 +32,7 @@ RUN     mkdir -p /z/dist/no-pk \
             && npm install \
             && npm i grunt uglify-js -g ) \
         && (tar -xf codemirror.tgz \
-            && cd CodeMirror-$ver_codemirror \
+            && cd codemirror5-$ver_codemirror \
             && npm install ) \
         && (tar -xf mde.tgz \
             && cd easy-markdown-editor* \
@@ -43,8 +43,6 @@ RUN     mkdir -p /z/dist/no-pk \
 
 
 # todo
-# https://cdn.jsdelivr.net/gh/highlightjs/cdn-release/build/highlight.min.js
-# https://cdn.jsdelivr.net/gh/highlightjs/cdn-release/build/styles/default.min.css
 # https://prismjs.com/download.html#themes=prism-funky&languages=markup+css+clike+javascript+autohotkey+bash+basic+batch+c+csharp+cpp+cmake+diff+docker+go+ini+java+json+kotlin+latex+less+lisp+lua+makefile+objectivec+perl+powershell+python+r+jsx+ruby+rust+sass+scss+sql+swift+systemd+toml+typescript+vbnet+verilog+vhdl+yaml&plugins=line-highlight+line-numbers+autolinker
 
 
@@ -89,7 +87,7 @@ RUN     cd marked-$ver_marked \
 
 # build codemirror
 COPY    codemirror.patch /z/
-RUN     cd CodeMirror-$ver_codemirror \
+RUN     cd codemirror5-$ver_codemirror \
         && patch -p1 < /z/codemirror.patch \
         && sed -ri '/^var urlRE = /d' mode/gfm/gfm.js \
         && npm run build \

scripts/deps-docker/Makefile

@@ -23,4 +23,4 @@ purge:
 
 sh:
 	@printf "\n\033[1;31mopening a shell in the most recently created docker image\033[0m\n"
-	docker run --rm -it `docker images -aq | head -n 1` /bin/bash
+	docker run --rm -it `docker images -aq | head -n 1` /bin/ash

scripts/deps-docker/marked-ln.patch

@@ -1,15 +1,15 @@
 diff --git a/src/Lexer.js b/src/Lexer.js
-adds linetracking to marked.js v4.0.6;
+adds linetracking to marked.js v4.0.17;
 add data-ln="%d" to most tags, %d is the source markdown line
 --- a/src/Lexer.js
 +++ b/src/Lexer.js
-@@ -50,4 +50,5 @@ function mangle(text) {
+@@ -52,4 +52,5 @@ function mangle(text) {
  export class Lexer {
    constructor(options) {
 +    this.ln = 1;  // like most editors, start couting from 1
      this.tokens = [];
      this.tokens.links = Object.create(null);
-@@ -127,4 +128,15 @@ export class Lexer {
+@@ -128,4 +129,15 @@ export class Lexer {
    }
  
 +  set_ln(token, ln = this.ln) {
@@ -25,9 +25,9 @@ add data-ln="%d" to most tags, %d is the source markdown line
 +
    /**
     * Lexing
-@@ -134,7 +146,11 @@ export class Lexer {
-       src = src.replace(/^ +$/gm, '');
+@@ -140,7 +152,11 @@ export class Lexer {
      }
+ 
 -    let token, lastToken, cutSrc, lastParagraphClipped;
 +    let token, lastToken, cutSrc, lastParagraphClipped, ln;
  
@@ -38,111 +38,112 @@ add data-ln="%d" to most tags, %d is the source markdown line
 +
        if (this.options.extensions
          && this.options.extensions.block
-@@ -142,4 +158,5 @@ export class Lexer {
+@@ -148,4 +164,5 @@ export class Lexer {
            if (token = extTokenizer.call({ lexer: this }, src, tokens)) {
              src = src.substring(token.raw.length);
 +            this.set_ln(token, ln);
              tokens.push(token);
              return true;
-@@ -153,4 +170,5 @@ export class Lexer {
+@@ -159,4 +176,5 @@ export class Lexer {
        if (token = this.tokenizer.space(src)) {
          src = src.substring(token.raw.length);
 +        this.set_ln(token, ln); // is \n if not type
-         if (token.type) {
-           tokens.push(token);
-@@ -162,4 +180,5 @@ export class Lexer {
+         if (token.raw.length === 1 && tokens.length > 0) {
+           // if there's a single \n as a spacer, it's terminating the last line,
+@@ -172,4 +190,5 @@ export class Lexer {
        if (token = this.tokenizer.code(src)) {
          src = src.substring(token.raw.length);
 +        this.set_ln(token, ln);
          lastToken = tokens[tokens.length - 1];
          // An indented code block cannot interrupt a paragraph.
-@@ -177,4 +196,5 @@ export class Lexer {
+@@ -187,4 +206,5 @@ export class Lexer {
        if (token = this.tokenizer.fences(src)) {
          src = src.substring(token.raw.length);
 +        this.set_ln(token, ln);
          tokens.push(token);
          continue;
-@@ -184,4 +204,5 @@ export class Lexer {
+@@ -194,4 +214,5 @@ export class Lexer {
        if (token = this.tokenizer.heading(src)) {
          src = src.substring(token.raw.length);
 +        this.set_ln(token, ln);
          tokens.push(token);
          continue;
-@@ -191,4 +212,5 @@ export class Lexer {
+@@ -201,4 +222,5 @@ export class Lexer {
        if (token = this.tokenizer.hr(src)) {
          src = src.substring(token.raw.length);
 +        this.set_ln(token, ln);
          tokens.push(token);
          continue;
-@@ -198,4 +220,5 @@ export class Lexer {
+@@ -208,4 +230,5 @@ export class Lexer {
        if (token = this.tokenizer.blockquote(src)) {
          src = src.substring(token.raw.length);
 +        this.set_ln(token, ln);
          tokens.push(token);
          continue;
-@@ -205,4 +228,5 @@ export class Lexer {
+@@ -215,4 +238,5 @@ export class Lexer {
        if (token = this.tokenizer.list(src)) {
          src = src.substring(token.raw.length);
 +        this.set_ln(token, ln);
          tokens.push(token);
          continue;
-@@ -212,4 +236,5 @@ export class Lexer {
+@@ -222,4 +246,5 @@ export class Lexer {
        if (token = this.tokenizer.html(src)) {
          src = src.substring(token.raw.length);
 +        this.set_ln(token, ln);
          tokens.push(token);
          continue;
-@@ -219,4 +244,5 @@ export class Lexer {
+@@ -229,4 +254,5 @@ export class Lexer {
        if (token = this.tokenizer.def(src)) {
          src = src.substring(token.raw.length);
 +        this.set_ln(token, ln);
          lastToken = tokens[tokens.length - 1];
          if (lastToken && (lastToken.type === 'paragraph' || lastToken.type === 'text')) {
-@@ -236,4 +262,5 @@ export class Lexer {
+@@ -246,4 +272,5 @@ export class Lexer {
        if (token = this.tokenizer.table(src)) {
          src = src.substring(token.raw.length);
 +        this.set_ln(token, ln);
          tokens.push(token);
          continue;
-@@ -243,4 +270,5 @@ export class Lexer {
+@@ -253,4 +280,5 @@ export class Lexer {
        if (token = this.tokenizer.lheading(src)) {
          src = src.substring(token.raw.length);
 +        this.set_ln(token, ln);
          tokens.push(token);
          continue;
-@@ -263,4 +291,5 @@ export class Lexer {
+@@ -273,4 +301,5 @@ export class Lexer {
        }
        if (this.state.top && (token = this.tokenizer.paragraph(cutSrc))) {
 +        this.set_ln(token, ln);
          lastToken = tokens[tokens.length - 1];
          if (lastParagraphClipped && lastToken.type === 'paragraph') {
-@@ -280,4 +309,6 @@ export class Lexer {
+@@ -290,4 +319,6 @@ export class Lexer {
        if (token = this.tokenizer.text(src)) {
          src = src.substring(token.raw.length);
 +        this.set_ln(token, ln);
 +        this.ln++;
          lastToken = tokens[tokens.length - 1];
          if (lastToken && lastToken.type === 'text') {
-@@ -355,4 +386,5 @@ export class Lexer {
+@@ -365,4 +396,5 @@ export class Lexer {
            if (token = extTokenizer.call({ lexer: this }, src, tokens)) {
              src = src.substring(token.raw.length);
 +            this.ln = token.ln || this.ln;
              tokens.push(token);
              return true;
-@@ -420,4 +452,6 @@ export class Lexer {
+@@ -430,4 +462,6 @@ export class Lexer {
        if (token = this.tokenizer.br(src)) {
          src = src.substring(token.raw.length);
 +        // no need to reset (no more blockTokens anyways)
 +        token.ln = this.ln++;
          tokens.push(token);
          continue;
-@@ -462,4 +496,5 @@ export class Lexer {
+@@ -472,4 +506,5 @@ export class Lexer {
        if (token = this.tokenizer.inlineText(cutSrc, smartypants)) {
          src = src.substring(token.raw.length);
 +        this.ln = token.ln || this.ln;
          if (token.raw.slice(-1) !== '_') { // Track prevChar before string of ____ started
            prevChar = token.raw.slice(-1);
 diff --git a/src/Parser.js b/src/Parser.js
+index a22a2bc..884ad66 100644
 --- a/src/Parser.js
 +++ b/src/Parser.js
 @@ -18,4 +18,5 @@ export class Parser {
@@ -205,6 +206,7 @@ diff --git a/src/Parser.js b/src/Parser.js
        // Run any renderer extensions
        if (this.options.extensions && this.options.extensions.renderers && this.options.extensions.renderers[token.type]) {
 diff --git a/src/Renderer.js b/src/Renderer.js
+index 7c36a75..aa1a53a 100644
 --- a/src/Renderer.js
 +++ b/src/Renderer.js
 @@ -11,6 +11,12 @@ export class Renderer {
@@ -214,10 +216,10 @@ diff --git a/src/Renderer.js b/src/Renderer.js
    }
  
 +  tag_ln(n) {
-+    this.ln = ' data-ln="' + n + '"';
++    this.ln = ` data-ln="${n}"`;
 +    return this;
 +  };
-+  
++
    code(code, infostring, escaped) {
      const lang = (infostring || '').match(/\S*/)[0];
 @@ -26,10 +32,10 @@ export class Renderer {
@@ -233,65 +235,65 @@ diff --git a/src/Renderer.js b/src/Renderer.js
 +    return '<pre' + this.ln + '><code class="'
        + this.options.langPrefix
        + escape(lang, true)
-@@ -40,5 +46,5 @@ export class Renderer {
- 
+@@ -43,5 +49,5 @@ export class Renderer {
+    */
    blockquote(quote) {
--    return '<blockquote>\n' + quote + '</blockquote>\n';
-+    return '<blockquote' + this.ln + '>\n' + quote + '</blockquote>\n';
+-    return `<blockquote>\n${quote}</blockquote>\n`;
++    return `<blockquote${this.ln}>\n${quote}</blockquote>\n`;
    }
  
-@@ -51,4 +57,5 @@ export class Renderer {
-       return '<h'
-         + level
-+        + this.ln
-         + ' id="'
-         + this.options.headerPrefix
-@@ -61,5 +68,5 @@ export class Renderer {
+@@ -59,9 +65,9 @@ export class Renderer {
+     if (this.options.headerIds) {
+       const id = this.options.headerPrefix + slugger.slug(raw);
+-      return `<h${level} id="${id}">${text}</h${level}>\n`;
++      return `<h${level}${this.ln} id="${id}">${text}</h${level}>\n`;
      }
+ 
      // ignore IDs
--    return '<h' + level + '>' + text + '</h' + level + '>\n';
-+    return '<h' + level + this.ln + '>' + text + '</h' + level + '>\n';
+-    return `<h${level}>${text}</h${level}>\n`;
++    return `<h${level}${this.ln}>${text}</h${level}>\n`;
    }
  
-@@ -75,5 +82,5 @@ export class Renderer {
- 
+@@ -80,5 +86,5 @@ export class Renderer {
+    */
    listitem(text) {
--    return '<li>' + text + '</li>\n';
-+    return '<li' + this.ln + '>' + text + '</li>\n';
+-    return `<li>${text}</li>\n`;
++    return `<li${this.ln}>${text}</li>\n`;
    }
  
-@@ -87,5 +94,5 @@ export class Renderer {
- 
+@@ -95,5 +101,5 @@ export class Renderer {
+    */
    paragraph(text) {
--    return '<p>' + text + '</p>\n';
-+    return '<p' + this.ln + '>' + text + '</p>\n';
+-    return `<p>${text}</p>\n`;
++    return `<p${this.ln}>${text}</p>\n`;
    }
  
-@@ -102,5 +109,5 @@ export class Renderer {
- 
+@@ -117,5 +123,5 @@ export class Renderer {
+    */
    tablerow(content) {
--    return '<tr>\n' + content + '</tr>\n';
-+    return '<tr' + this.ln + '>\n' + content + '</tr>\n';
+-    return `<tr>\n${content}</tr>\n`;
++    return `<tr${this.ln}>\n${content}</tr>\n`;
    }
  
-@@ -127,5 +134,5 @@ export class Renderer {
+@@ -151,5 +157,5 @@ export class Renderer {
  
    br() {
 -    return this.options.xhtml ? '<br/>' : '<br>';
-+    return this.options.xhtml ? '<br' + this.ln + '/>' : '<br' + this.ln + '>';
++    return this.options.xhtml ? `<br${this.ln}/>` : `<br${this.ln}>`;
    }
  
-@@ -153,5 +160,5 @@ export class Renderer {
+@@ -190,5 +196,5 @@ export class Renderer {
      }
  
--    let out = '<img src="' + href + '" alt="' + text + '"';
-+    let out = '<img' + this.ln + ' src="' + href + '" alt="' + text + '"';
+-    let out = `<img src="${href}" alt="${text}"`;
++    let out = `<img${this.ln} src="${href}" alt="${text}"`;
      if (title) {
-       out += ' title="' + title + '"';
+       out += ` title="${title}"`;
 diff --git a/src/Tokenizer.js b/src/Tokenizer.js
+index e8a69b6..2cc772b 100644
 --- a/src/Tokenizer.js
 +++ b/src/Tokenizer.js
-@@ -297,4 +297,7 @@ export class Tokenizer {
+@@ -302,4 +302,7 @@ export class Tokenizer {
        const l = list.items.length;
  
 +      // each nested list gets +1 ahead; this hack makes every listgroup -1 but atleast it doesn't get infinitely bad

scripts/genlic.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+set -e
+
+outfile="$(realpath "$1")"
+
+[ -e genlic.sh ] || cd scripts
+[ -e genlic.sh ]
+
+f=../build/mit.txt
+[ -e $f ] ||
+	curl https://opensource.org/licenses/MIT |
+	awk '/div>/{o=0}o>1;o{o++}/;COPYRIGHT HOLDER/{o=1}' |
+	awk '{gsub(/<[^>]+>/,"")};1' >$f
+
+f=../build/isc.txt
+[ -e $f ] ||
+	curl https://opensource.org/licenses/ISC |
+	awk '/div>/{o=0}o>2;o{o++}/;OWNER/{o=1}' |
+	awk '{gsub(/<[^>]+>/,"")};/./{b=0}!/./{b++}b>1{next}1' >$f
+
+f=../build/3bsd.txt
+[ -e $f ] ||
+	curl https://opensource.org/licenses/BSD-3-Clause |
+	awk '/div>/{o=0}o>1;o{o++}/HOLDER/{o=1}' |
+	awk '{gsub(/<[^>]+>/,"")};1' >$f
+
+f=../build/ofl.txt
+[ -e $f ] ||
+	curl https://opensource.org/licenses/OFL-1.1 |
+	awk '/PREAMBLE/{o=1}/sil\.org/{o=0}!o{next}/./{printf "%s ",$0;next}{print"\n"}' |
+	awk '{gsub(/<[^>]+>/,"");gsub(/^\s+/,"");gsub(/&amp;/,"\\&")}/./{b=0}!/./{b++}b>1{next}1' >$f
+
+(sed -r 's/^L: /License: /;s/^C: /Copyright (c) /' <../docs/lics.txt
+printf '\n\n--- MIT License ---\n\n'; cat ../build/mit.txt
+printf '\n\n--- ISC License ---\n\n'; cat ../build/isc.txt
+printf '\n\n--- BSD 3-Clause License ---\n\n'; cat ../build/3bsd.txt
+printf '\n\n--- SIL Open Font License v1.1 ---\n\n'; cat ../build/ofl.txt
+) |
+while IFS= read -r x; do
+	[ "${x:0:4}" = "--- " ] || {
+		printf '%s\n' "$x"
+		continue
+	}
+	n=${#x}
+	p=$(( (80-n)/2 ))
+	printf "%${p}s\033[07m%s\033[0m\n" "" "$x"
+done > "$outfile"

scripts/make-pypi-release.sh

@@ -14,10 +14,6 @@ gtar=$(command -v gtar || command -v gnutar) || true
 		realpath() { grealpath "$@"; }
 }
 
-which md5sum 2>/dev/null >/dev/null &&
-	md5sum=md5sum ||
-	md5sum="md5 -r"
-
 mode="$1"
 
 [ -z "$mode" ] &&
@@ -90,6 +86,15 @@ function have() {
 have setuptools
 have wheel
 have twine
+
+# remove type hints to support python < 3.9
+rm -rf build/pypi
+mkdir -p build/pypi
+cp -pR setup.py README.md LICENSE copyparty tests bin scripts/strip_hints build/pypi/
+cd build/pypi
+tar --strip-components=2 -xf ../strip-hints-0.1.10.tar.gz strip-hints-0.1.10/src/strip_hints
+python3 -c 'from strip_hints.a import uh; uh("copyparty")'
+
 ./setup.py clean2
 ./setup.py sdist bdist_wheel --universal
 

scripts/make-sfx.sh

@@ -12,8 +12,13 @@ help() { exec cat <<'EOF'
 # `re` does a repack of an sfx which you already executed once
 #   (grabs files from the sfx-created tempdir), overrides `clean`
 #
+# `ox` builds a pyoxidizer exe instead of py
+#
 # `gz` creates a gzip-compressed python sfx instead of bzip2
 #
+# `lang` limits which languages/translations to include,
+#   for example `lang eng` or `lang eng|nor`
+#
 # `no-cm` saves ~82k by removing easymde/codemirror
 #   (the fancy markdown editor)
 #
@@ -23,6 +28,11 @@ help() { exec cat <<'EOF'
 #   (browsers will try to use 'Consolas' instead)
 #
 # `no-dd` saves ~2k by removing the mouse cursor
+#
+# ---------------------------------------------------------------------
+#
+# if you are on windows, you can use msys2:
+#   PATH=/c/Users/$USER/AppData/Local/Programs/Python/Python310:"$PATH" ./make-sfx.sh fast
 
 EOF
 }
@@ -48,6 +58,10 @@ gtar=$(command -v gtar || command -v gnutar) || true
 gawk=$(command -v gawk || command -v gnuawk || command -v awk)
 awk() { $gawk "$@"; }
 
+targs=(--owner=1000 --group=1000)
+[ "$OSTYPE" = msys ] &&
+	targs=()
+
 pybin=$(command -v python3 || command -v python) || {
 	echo need python
 	exit 1
@@ -61,18 +75,26 @@ pybin=$(command -v python3 || command -v python) || {
 	exit 1
 }
 
+[ $CSN ] ||
+	CSN=sfx
+
+langs=
 use_gz=
 zopf=2560
 while [ ! -z "$1" ]; do
 	case $1 in
 		clean)  clean=1  ; ;;
 		re)     repack=1 ; ;;
+		ox)     use_ox=1 ; ;;
 		gz)     use_gz=1 ; ;;
+		gzz)    shift;use_gzz=$1;use_gz=1; ;;
 		no-fnt) no_fnt=1 ; ;;
 		no-hl)  no_hl=1  ; ;;
 		no-dd)  no_dd=1  ; ;;
 		no-cm)  no_cm=1  ; ;;
-		fast)   zopf=100 ; ;;
+		fast)   zopf=    ; ;;
+		ultra)  ultra=1  ; ;;
+		lang)   shift;langs="$1"; ;;
 		*)      help     ; ;;
 	esac
 	shift
@@ -89,9 +111,9 @@ stamp=$(
 	done | sort | tail -n 1 | sha1sum | cut -c-16
 )
 
-rm -rf sfx/*
-mkdir -p sfx build
-cd sfx
+rm -rf $CSN/*
+mkdir -p $CSN build
+cd $CSN
 
 tmpdir="$(
 	printf '%s\n' "$TMPDIR" /tmp |
@@ -99,9 +121,9 @@ tmpdir="$(
 )"
 
 [ $repack ] && {
-	old="$tmpdir/pe-copyparty"
+	old="$tmpdir/pe-copyparty.$(id -u)"
 	echo "repack of files in $old"
-	cp -pR "$old/"*{dep-j2,dep-ftp,copyparty} .
+	cp -pR "$old/"*{py2,j2,ftp,copyparty} .
 }
 
 [ $repack ] || {
@@ -125,8 +147,8 @@ tmpdir="$(
 	mv MarkupSafe-*/src/markupsafe .
 	rm -rf MarkupSafe-* markupsafe/_speedups.c
 
-	mkdir dep-j2/
-	mv {markupsafe,jinja2} dep-j2/
+	mkdir j2/
+	mv {markupsafe,jinja2} j2/
 
 	echo collecting pyftpdlib
 	f="../build/pyftpdlib-1.5.6.tar.gz"
@@ -138,8 +160,8 @@ tmpdir="$(
 	mv pyftpdlib-release-*/pyftpdlib .
 	rm -rf pyftpdlib-release-* pyftpdlib/test
 
-	mkdir dep-ftp/
-	mv pyftpdlib dep-ftp/
+	mkdir ftp/
+	mv pyftpdlib ftp/
 
 	echo collecting asyncore, asynchat
 	for n in asyncore.py asynchat.py; do
@@ -149,6 +171,41 @@ tmpdir="$(
 			wget -O$f "$url" || curl -L "$url" >$f)
 	done
 
+	echo collecting python-magic
+	v=0.4.27
+	f="../build/python-magic-$v.tar.gz"
+	[ -e "$f" ] ||
+		(url=https://files.pythonhosted.org/packages/da/db/0b3e28ac047452d079d375ec6798bf76a036a08182dbb39ed38116a49130/python-magic-0.4.27.tar.gz;
+		wget -O$f "$url" || curl -L "$url" >$f)
+
+	tar -zxf $f
+	mkdir magic
+	mv python-magic-*/magic .
+	rm -rf python-magic-*
+	rm magic/compat.py
+	f=magic/__init__.py
+	awk '/^def _add_compat/{o=1} !o; /^_add_compat/{o=0}' <$f >t
+	tmv "$f"
+	mv magic ftp/  # doesn't provide a version label anyways
+
+	# enable this to dynamically remove type hints at startup,
+	# in case a future python version can use them for performance
+	true || (
+		echo collecting strip-hints
+		f=../build/strip-hints-0.1.10.tar.gz
+		[ -e $f ] ||
+			(url=https://files.pythonhosted.org/packages/9c/d4/312ddce71ee10f7e0ab762afc027e07a918f1c0e1be5b0069db5b0e7542d/strip-hints-0.1.10.tar.gz;
+			wget -O$f "$url" || curl -L "$url" >$f)
+
+		tar -zxf $f
+		mv strip-hints-0.1.10/src/strip_hints .
+		rm -rf strip-hints-* strip_hints/import_hooks*
+		sed -ri 's/[a-z].* as import_hooks$/"""a"""/' strip_hints/*.py
+
+		cp -pR ../scripts/strip_hints/ .
+	)
+	cp -pR ../scripts/py2/ .
+
 	# msys2 tar is bad, make the best of it
 	echo collecting source
 	[ $clean ] && {
@@ -165,6 +222,12 @@ tmpdir="$(
 	for n in asyncore.py asynchat.py; do
 		awk 'NR<4||NR>27;NR==4{print"# license: https://opensource.org/licenses/ISC\n"}' ../build/$n >copyparty/vend/$n
 	done
+
+	# remove type hints before build instead
+	(cd copyparty; "$pybin" ../../scripts/strip_hints/a.py; rm uh)
+
+	licfile=$(realpath copyparty/res/COPYING.txt)
+	(cd ../scripts; ./genlic.sh "$licfile")
 }
 
 ver=
@@ -206,11 +269,12 @@ ts=$(date -u +%s)
 hts=$(date -u +%Y-%m%d-%H%M%S) # --date=@$ts (thx osx)
 
 mkdir -p ../dist
-sfx_out=../dist/copyparty-sfx
+sfx_out=../dist/copyparty-$CSN
 
 echo cleanup
 find -name '*.pyc' -delete
 find -name __pycache__ -delete
+find -name py.typed -delete
 
 # especially prevent osx from leaking your lan ip (wtf apple)
 find -type f \( -name .DS_Store -or -name ._.DS_Store \) -delete
@@ -262,17 +326,31 @@ rm have
 	tmv "$f"
 }
 
-[ $repack ] ||
-find | grep -E '\.py$' |
-  grep -vE '__version__' |
-  tr '\n' '\0' |
-  xargs -0 "$pybin" ../scripts/uncomment.py
+[ $langs ] &&
+	for f in copyparty/web/{browser.js,splash.js}; do
+		gzip -d "$f.gz" || true
+		awk '/^\}/{l=0} !l; /^var Ls =/{l=1;next} o; /^\t["}]/{o=0} /^\t"'"$langs"'"/{o=1;print}' <$f >t
+		tmv "$f"
+	done
 
-f=dep-j2/jinja2/constants.py
+[ ! $repack ] && [ ! $use_ox ] && {
+	# uncomment; oxidized drops 45 KiB but becomes undebuggable
+	find | grep -E '\.py$' |
+		grep -vE '__version__' |
+		tr '\n' '\0' |
+		xargs -0 "$pybin" ../scripts/uncomment.py
+
+	# py2-compat
+	#find | grep -E '\.py$' | while IFS= read -r x; do
+	#	sed -ri '/: TypeAlias = /d' "$x"; done
+}
+
+f=j2/jinja2/constants.py
 awk '/^LOREM_IPSUM_WORDS/{o=1;print "LOREM_IPSUM_WORDS = u\"a\"";next} !o; /"""/{o=0}' <$f >t
 tmv "$f"
+rm -f j2/jinja2/async*
 
-grep -rLE '^#[^a-z]*coding: utf-8' dep-j2 |
+grep -rLE '^#[^a-z]*coding: utf-8' j2 |
 while IFS= read -r f; do
 	(echo "# coding: utf-8"; cat "$f") >t
 	tmv "$f"
@@ -301,9 +379,9 @@ find | grep -E '\.(js|html)$' | while IFS= read -r f; do
 done
 
 gzres() {
-	command -v pigz &&
-		pk="pigz -11 -I $zopf" ||
-		pk='gzip'
+	[ $zopf ] && command -v zopfli && pk="zopfli --i$zopf"
+	[ $zopf ] && command -v pigz && pk="pigz -11 -I $zopf"
+	[ -z "$pk" ] && pk='gzip'
 
 	np=$(nproc)
 	echo "$pk #$np"
@@ -327,7 +405,7 @@ gzres() {
 }
 
 
-zdir="$tmpdir/cpp-mksfx"
+zdir="$tmpdir/cpp-mk$CSN"
 [ -e "$zdir/$stamp" ] || rm -rf "$zdir"
 mkdir -p "$zdir"
 echo a > "$zdir/$stamp"
@@ -342,7 +420,8 @@ nf=$(ls -1 "$zdir"/arc.* | wc -l)
 }
 [ $use_zdir ] && {
 	arcs=("$zdir"/arc.*)
-	arc="${arcs[$RANDOM % ${#arcs[@]} ] }"
+	n=$(( $RANDOM % ${#arcs[@]} ))
+	arc="${arcs[n]}"
 	echo "using $arc"
 	tar -xf "$arc"
 	for f in copyparty/web/*.gz; do
@@ -351,14 +430,41 @@ nf=$(ls -1 "$zdir"/arc.* | wc -l)
 }
 
 
+[ $use_ox ] && {
+	tgt=x86_64-pc-windows-msvc
+	tgt=i686-pc-windows-msvc  # 2M smaller (770k after upx)
+	bdir=build/$tgt/release/install/copyparty
+
+	t="res web"
+	(printf "\n\n\nBUT WAIT! THERE'S MORE!!\n\n";
+	cat ../$bdir/COPYING.txt) >> copyparty/res/COPYING.txt ||
+		echo "copying.txt 404 pls rebuild"
+
+	mv ftp/* j2/* copyparty/vend/* .
+	rm -rf ftp j2 py2 copyparty/vend
+	(cd copyparty; tar -cvf z.tar $t; rm -rf $t)
+	cd ..
+	pyoxidizer build --release --target-triple $tgt
+	mv $bdir/copyparty.exe dist/
+	cp -pv "$(for d in '/c/Program Files (x86)/Microsoft Visual Studio/'*'/BuildTools/VC/Redist/MSVC'; do
+		find "$d" -name vcruntime140.dll; done | sort | grep -vE '/x64/|/onecore/' | head -n 1)" dist/
+	dist/copyparty.exe --version
+	cp -pv dist/copyparty{,.orig}.exe
+	[ $ultra ] && a="--best --lzma" || a=-1
+	/bin/time -f %es upx $a dist/copyparty.exe >/dev/null
+	ls -al dist/copyparty{,.orig}.exe
+	exit 0
+}
+
+
 echo gen tarlist
-for d in copyparty dep-j2 dep-ftp; do find $d -type f; done |
+for d in copyparty j2 ftp py2; do find $d -type f; done |  # strip_hints
 sed -r 's/(.*)\.(.*)/\2 \1/' | LC_ALL=C sort |
 sed -r 's/([^ ]*) (.*)/\2.\1/' | grep -vE '/list1?$' > list1
 
 for n in {1..50}; do
-	(grep -vE '\.(gz|br)$' list1; grep -E '\.(gz|br)$' list1 | shuf) >list || true
-	s=$(md5sum list | cut -c-16)
+	(grep -vE '\.(gz|br)$' list1; grep -E '\.(gz|br)$' list1 | (shuf||gshuf) ) >list || true
+	s=$( (sha1sum||shasum) < list | cut -c-16)
 	grep -q $s "$zdir/h" && continue
 	echo $s >> "$zdir/h"
 	break
@@ -366,19 +472,20 @@ done
 [ $n -eq 50 ] && exit
 
 echo creating tar
-args=(--owner=1000 --group=1000)
-[ "$OSTYPE" = msys ] &&
-	args=()
+tar -cf tar "${targs[@]}" --numeric-owner -T list
 
-tar -cf tar "${args[@]}" --numeric-owner -T list
-
-pc=bzip2
-pe=bz2
-[ $use_gz ] && pc=gzip && pe=gz
+pc="bzip2 -"; pe=bz2
+[ $use_gz ] && pc="gzip -" && pe=gz
+[ $use_gzz ] && pc="pigz -11 -I$use_gzz" && pe=gz
 
 echo compressing tar
-# detect best level; bzip2 -7 is usually better than -9
-for n in {2..9}; do cp tar t.$n; $pc  -$n t.$n & done; wait; mv -v $(ls -1S t.*.$pe | tail -n 1) tar.bz2
+for n in {2..9}; do cp tar t.$n; nice $pc$n t.$n & done; wait
+minf=$(for f in t.*.$pe; do
+	s1=$(wc -c <$f)
+	s2=$(tr -d '\r\n\0' <$f | wc -c)
+	echo "$(( s2+(s1-s2)*3 )) $f"
+done | sort -n | awk '{print$2;exit}')
+mv -v $minf tar.bz2
 rm t.* || true
 exts=()
 

scripts/profile.py

@@ -8,7 +8,7 @@ cmd = sys.argv[1]
 if cmd == "cpp":
     from copyparty.__main__ import main
 
-    argv = ["__main__", "-v", "srv::r", "-v", "../../yt:yt:r"]
+    argv = ["__main__", "-vsrv::r:c,e2ds,e2ts"]
     main(argv=argv)
 
 elif cmd == "test":
@@ -29,6 +29,6 @@ else:
 #
 # python -m vmprof -o prof --lines ./scripts/profile.py test
 
-# linux: ~/.local/bin/vmprofshow prof tree | grep -vF '[1m  0.'
-# macos: ~/Library/Python/3.9/bin/vmprofshow prof tree | grep -vF '[1m  0.'
+# linux: ~/.local/bin/vmprofshow prof tree | awk '$2>1{n=5} !n{next} 1;{n--} !n{print""}'
+# macos: ~/Library/Python/3.9/bin/vmprofshow prof tree
 #   win: %appdata%\..\Roaming\Python\Python39\Scripts\vmprofshow.exe prof tree

scripts/py2/queue/__init__.py

@@ -0,0 +1,4 @@
+# coding: utf-8
+from __future__ import print_function, unicode_literals
+
+from Queue import Queue, LifoQueue, PriorityQueue, Empty, Full

scripts/pyinstaller/README.md

@@ -0,0 +1,12 @@
+builds a fully standalone copyparty.exe compatible with 32bit win7-sp1 and later
+
+requires a win7 vm which has never been connected to the internet and a host-only network with the linux host at 192.168.123.1
+
+first-time setup steps in notes.txt
+
+run build.sh in the vm to fetch src + compile + push a new exe to the linux host for manual publishing
+
+
+## ffmpeg
+
+built with [ffmpeg-windows-build-helpers](https://github.com/rdp/ffmpeg-windows-build-helpers) and [this patch](./ffmpeg.patch) using [these steps](./ffmpeg.txt)

scripts/pyinstaller/build.sh

@@ -0,0 +1,65 @@
+#!/bin/bash
+set -e
+
+curl http://192.168.123.1:3923/cpp/scripts/pyinstaller/build.sh |
+tee build2.sh | cmp build.sh && rm build2.sh || {
+    echo "new build script; upgrade y/n:"
+    while true; do read -u1 -n1 -r r; [[ $r =~ [yYnN] ]] && break; done
+    [[ $r =~ [yY] ]] && mv build{2,}.sh && exec ./build.sh
+}
+
+dl() { curl -fkLO "$1"; }
+
+cd ~/Downloads
+
+dl https://192.168.123.1:3923/cpp/dist/copyparty-sfx.py
+dl https://192.168.123.1:3923/cpp/scripts/pyinstaller/loader.ico
+dl https://192.168.123.1:3923/cpp/scripts/pyinstaller/loader.py
+dl https://192.168.123.1:3923/cpp/scripts/pyinstaller/loader.rc
+
+rm -rf $TEMP/pe-copyparty*
+python copyparty-sfx.py --version
+
+rm -rf mods; mkdir mods
+cp -pR $TEMP/pe-copyparty/copyparty/ $TEMP/pe-copyparty/{ftp,j2}/* mods/
+
+af() { awk "$1" <$2 >tf; mv tf "$2"; }
+
+rm -rf mods/magic/
+
+sed -ri /pickle/d mods/jinja2/_compat.py
+sed -ri '/(bccache|PackageLoader)/d' mods/jinja2/__init__.py
+af '/^class/{s=0}/^class PackageLoader/{s=1}!s' mods/jinja2/loaders.py
+
+sed -ri /fork_process/d mods/pyftpdlib/servers.py
+af '/^class _Base/{s=1}!s' mods/pyftpdlib/authorizers.py
+
+read a b c d _ < <(
+    grep -E '^VERSION =' mods/copyparty/__version__.py |
+    tail -n 1 |
+    sed -r 's/[^0-9]+//;s/[" )]//g;s/[-,]/ /g;s/$/ 0/'
+)
+sed -r 's/1,2,3,0/'$a,$b,$c,$d'/;s/1\.2\.3/'$a.$b.$c/ <loader.rc >loader.rc2
+
+$APPDATA/python/python37/scripts/pyinstaller \
+    -y --clean -p mods --upx-dir=. \
+    --exclude-module copyparty.broker_mp \
+    --exclude-module copyparty.broker_mpw \
+    --exclude-module curses \
+    --exclude-module ctypes.macholib \
+    --exclude-module inspect \
+    --exclude-module multiprocessing \
+    --exclude-module pdb \
+    --exclude-module pickle \
+    --exclude-module pyftpdlib.prefork \
+    --exclude-module urllib.request \
+    --exclude-module urllib.response \
+    --exclude-module urllib.robotparser \
+    --exclude-module zipfile \
+    --version-file loader.rc2 -i loader.ico -n copyparty -c -F loader.py \
+    --add-data 'mods/copyparty/res;copyparty/res' \
+    --add-data 'mods/copyparty/web;copyparty/web'
+
+# ./upx.exe --best --ultra-brute --lzma -k dist/copyparty.exe
+
+curl -fkT dist/copyparty.exe -b cppwd=wark https://192.168.123.1:3923/