v1.5.1

never been able to reproduce it intentionally but this should work

.gitignore

@@ -22,6 +22,7 @@ copyparty.egg-info/
 *.bak
 
 # derived
+copyparty/res/COPYING.txt
 copyparty/web/deps/
 srv/
 

.vscode/launch.py

@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+
 # takes arguments from launch.json
 # is used by no_dbg in tasks.json
 # launches 10x faster than mspython debugpy
@@ -9,15 +11,15 @@ import sys
 
 print(sys.executable)
 
+import json5
 import shlex
-import jstyleson
 import subprocess as sp
 
 
 with open(".vscode/launch.json", "r", encoding="utf-8") as f:
     tj = f.read()
 
-oj = jstyleson.loads(tj)
+oj = json5.loads(tj)
 argv = oj["configurations"][0]["args"]
 
 try:
@@ -28,6 +30,8 @@ except:
 
 argv = [os.path.expanduser(x) if x.startswith("~") else x for x in argv]
 
+argv += sys.argv[1:]
+
 if re.search(" -j ?[0-9]", " ".join(argv)):
     argv = [sys.executable, "-m", "copyparty"] + argv
     sp.check_call(argv)

README.md

@@ -8,9 +8,9 @@
 
 turn your phone or raspi into a portable file server with resumable uploads/downloads using *any* web browser
 
-* server only needs `py2.7` or `py3.3+`, all dependencies optional
+* server only needs Python (`2.7` or `3.3+`), all dependencies optional
 * browse/upload with [IE4](#browser-support) / netscape4.0 on win3.11 (heh)
-* *resumable* uploads need `firefox 34+` / `chrome 41+` / `safari 7+`
+* protocols: [http](#the-browser) // [ftp](#ftp-server) // [webdav](#webdav-server) // [smb/cifs](#smb-server)
 
 try the **[read-only demo server](https://a.ocv.me/pub/demo/)** 👀 running from a basement in finland
 
@@ -30,16 +30,17 @@ try the **[read-only demo server](https://a.ocv.me/pub/demo/)** 👀 running fro
     * [quickstart](#quickstart) - download **[copyparty-sfx.py](https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.py)** and you're all set!
         * [on servers](#on-servers) - you may also want these, especially on servers
         * [on debian](#on-debian) - recommended additional steps on debian
-    * [notes](#notes) - general notes
-    * [status](#status) - feature summary
+    * [features](#features)
     * [testimonials](#testimonials) - small collection of user feedback
 * [motivations](#motivations) - project goals / philosophy
-    * [future plans](#future-plans) - some improvement ideas
+    * [notes](#notes) - general notes
 * [bugs](#bugs)
     * [general bugs](#general-bugs)
     * [not my bugs](#not-my-bugs)
+* [breaking changes](#breaking-changes) - upgrade notes
 * [FAQ](#FAQ) - "frequently" asked questions
 * [accounts and volumes](#accounts-and-volumes) - per-folder, per-user permissions
+    * [shadowing](#shadowing) - hiding specific subfolders
 * [the browser](#the-browser) - accessing a copyparty server using a web-browser
     * [tabs](#tabs) - the main tabs in the ui
     * [hotkeys](#hotkeys) - the browser has the following hotkeys
@@ -56,8 +57,14 @@ try the **[read-only demo server](https://a.ocv.me/pub/demo/)** 👀 running fro
     * [other tricks](#other-tricks)
     * [searching](#searching) - search by size, date, path/name, mp3-tags, ...
 * [server config](#server-config) - using arguments or config files, or a mix of both
+    * [zeroconf](#zeroconf) - announce enabled services on the LAN
+        * [mdns](#mdns) - LAN domain-name and feature announcer
+        * [ssdp](#ssdp) - windows-explorer announcer
     * [qr-code](#qr-code) - print a qr-code [(screenshot)](https://user-images.githubusercontent.com/241032/194728533-6f00849b-c6ac-43c6-9359-83e454d11e00.png) for quick access
-    * [ftp-server](#ftp-server) - an FTP server can be started using `--ftp 3921`
+    * [ftp server](#ftp-server) - an FTP server can be started using `--ftp 3921`
+    * [webdav server](#webdav-server) - with read-write support
+        * [connecting to webdav from windows](#connecting-to-webdav-from-windows) - using the GUI
+    * [smb server](#smb-server) - unsafe, slow, not recommended for wan
     * [file indexing](#file-indexing) - enables dedup and music search ++
         * [exclude-patterns](#exclude-patterns) - to save some time
         * [filesystem guards](#filesystem-guards) - avoid traversing into other filesystems
@@ -74,8 +81,8 @@ try the **[read-only demo server](https://a.ocv.me/pub/demo/)** 👀 running fro
     * [complete examples](#complete-examples)
 * [browser support](#browser-support) - TLDR: yes
 * [client examples](#client-examples) - interact with copyparty using non-browser clients
+    * [mount as drive](#mount-as-drive) - a remote copyparty server as a local filesystem
 * [up2k](#up2k) - quick outline of the up2k protocol, see [uploading](#uploading) for the web-client
-    * [why chunk-hashes](#why-chunk-hashes) - a single sha512 would be better, right?
 * [performance](#performance) - defaults are usually fine - expect `8 GiB/s` download, `1 GiB/s` upload
     * [client-side](#client-side) - when uploading files
 * [security](#security) - some notes on hardening
@@ -93,16 +100,10 @@ try the **[read-only demo server](https://a.ocv.me/pub/demo/)** 👀 running fro
     * [install recommended deps](#install-recommended-deps)
     * [optional gpl stuff](#optional-gpl-stuff)
 * [sfx](#sfx) - the self-contained "binary"
-    * [sfx repack](#sfx-repack) - reduce the size of an sfx by removing features
-    * [copyparty.exe](#copypartyexe)
+    * [copyparty.exe](#copypartyexe) - download [copyparty.exe](https://github.com/9001/copyparty/releases/latest/download/copyparty.exe) or [copyparty64.exe](https://github.com/9001/copyparty/releases/latest/download/copyparty64.exe)
 * [install on android](#install-on-android)
 * [reporting bugs](#reporting-bugs) - ideas for context to include in bug reports
-* [building](#building)
-    * [dev env setup](#dev-env-setup)
-    * [just the sfx](#just-the-sfx)
-    * [complete release](#complete-release)
-* [todo](#todo) - roughly sorted by priority
-    * [discarded ideas](#discarded-ideas)
+* [devnotes](#devnotes) - for build instructions etc, see [./docs/devnotes.md](./docs/devnotes.md)
 
 
 ## quickstart
@@ -115,11 +116,10 @@ running the sfx without arguments (for example doubleclicking it on Windows) wil
 
 some recommended options:
 * `-e2dsa` enables general [file indexing](#file-indexing)
-* `-e2ts` enables audio metadata indexing (needs either FFprobe or Mutagen), see [optional dependencies](#optional-dependencies)
+* `-e2ts` enables audio metadata indexing (needs either FFprobe or Mutagen), see [optional dependencies](#optional-dependencies) to enable thumbnails and more
 * `-v /mnt/music:/music:r:rw,foo -a foo:bar` shares `/mnt/music` as `/music`, `r`eadable by anyone, and read-write for user `foo`, password `bar`
   * replace `:r:rw,foo` with `:r,foo` to only make the folder readable by `foo` and nobody else
   * see [accounts and volumes](#accounts-and-volumes) for the syntax and other permissions (`r`ead, `w`rite, `m`ove, `d`elete, `g`et, up`G`et)
-* `--ls '**,*,ln,p,r'` to crash on startup if any of the volumes contain a symlink which point outside the volume, as that could give users unintended access (see `--help-ls`)
 
 
 ### on servers
@@ -130,6 +130,14 @@ you may also want these, especially on servers:
 * [contrib/systemd/prisonparty.service](contrib/systemd/prisonparty.service) to run it in a chroot (for extra security)
 * [contrib/nginx/copyparty.conf](contrib/nginx/copyparty.conf) to reverse-proxy behind nginx (for better https)
 
+and remember to open the ports you want; here's a complete example including every feature copyparty has to offer:
+```
+firewall-cmd --permanent --add-port={80,443,3921,3923,3945,3990}/tcp  # --zone=libvirt
+firewall-cmd --permanent --add-port=12000-12099/tcp --permanent  # --zone=libvirt
+firewall-cmd --permanent --add-port={1900,5353}/udp  # --zone=libvirt
+firewall-cmd --reload
+```
+(1900:ssdp, 3921:ftp, 3923:http/https, 3945:smb, 3990:ftps, 5353:mdns, 12000:passive-ftp)
 
 ### on debian
 
@@ -144,31 +152,18 @@ recommended additional steps on debian  which enable audio metadata and thumbnai
 (skipped `pyheif-pillow-opener` because apparently debian is too old to build it)
 
 
-## notes
-
-general notes:
-* paper-printing is affected by dark/light-mode! use lightmode for color, darkmode for grayscale
-  * because no browsers currently implement the media-query to do this properly orz
-
-browser-specific:
-* iPhone/iPad: use Firefox to download files
-* Android-Chrome: increase "parallel uploads" for higher speed (android bug)
-* Android-Firefox: takes a while to select files (their fix for ☝️)
-* Desktop-Firefox: ~~may use gigabytes of RAM if your files are massive~~ *seems to be OK now*
-* Desktop-Firefox: may stop you from deleting files you've uploaded until you visit `about:memory` and click `Minimize memory usage`
-
-
-## status
-
-feature summary
+## features
 
 * backend stuff
-  * ☑ sanic multipart parser
-  * ☑ multiprocessing (actual multithreading)
+  * ☑ IPv6
+  * ☑ [multiprocessing](#performance) (actual multithreading)
   * ☑ volumes (mountpoints)
   * ☑ [accounts](#accounts-and-volumes)
-  * ☑ [ftp-server](#ftp-server)
+  * ☑ [ftp server](#ftp-server)
+  * ☑ [webdav server](#webdav-server)
+  * ☑ [smb/cifs server](#smb-server)
   * ☑ [qr-code](#qr-code) for quick access
+  * ☑ [upnp / zeroconf / mdns / ssdp](#zeroconf)
 * upload
   * ☑ basic: plain multipart, ie6 support
   * ☑ [up2k](#uploading): js, resumable, multithreaded
@@ -179,7 +174,7 @@ feature summary
 * download
   * ☑ single files in browser
   * ☑ [folders as zip / tar files](#zip-downloads)
-  * ☑ [FUSE client](https://github.com/9001/copyparty/tree/hovudstraum/bin#copyparty-fusepy) (read-only)
+  * ☑ [FUSE client](https://github.com/9001/copyparty/tree/hovudstraum/bin#partyfusepy) (read-only)
 * browser
   * ☑ [navpane](#navpane) (directory tree sidebar)
   * ☑ file manager (cut/paste, delete, [batch-rename](#batch-rename))
@@ -228,20 +223,21 @@ project goals / philosophy
   * no build steps; modify the js/python without needing node.js or anything like that
 
 
-## future plans
+## notes
 
-some improvement ideas
+general notes:
+* paper-printing is affected by dark/light-mode! use lightmode for color, darkmode for grayscale
+  * because no browsers currently implement the media-query to do this properly orz
 
-* the JS is a mess -- a preact rewrite would be nice
-  * preferably without build dependencies like webpack/babel/node.js, maybe a python thing to assemble js files into main.js
-  * good excuse to look at using virtual lists (browsers start to struggle when folders contain over 5000 files)
-* the UX is a mess -- a proper design would be nice
-  * very organic (much like the python/js), everything was an afterthought
-  * true for both the layout and the visual flair
-  * something like the tron board-room ui (or most other hollywood ones, like ironman) would be :100:
-* some of the python files are way too big
-  * `up2k.py` ended up doing all the file indexing / db management
-  * `httpcli.py` should be separated into modules in general
+browser-specific:
+* iPhone/iPad: use Firefox to download files
+* Android-Chrome: increase "parallel uploads" for higher speed (android bug)
+* Android-Firefox: takes a while to select files (their fix for ☝️)
+* Desktop-Firefox: ~~may use gigabytes of RAM if your files are massive~~ *seems to be OK now*
+* Desktop-Firefox: may stop you from deleting files you've uploaded until you visit `about:memory` and click `Minimize memory usage`
+
+server-os-specific:
+* RHEL8 / Rocky8: you can run copyparty using `/usr/libexec/platform-python`
 
 
 # bugs
@@ -291,6 +287,15 @@ some improvement ideas
   * due to snap security policies -- see `snap connections firefox` for the allowlist, `removable-media` permits all of `/mnt` and `/media` apparently
 
 
+# breaking changes
+
+upgrade notes
+
+* `1.5.0` (2022-12-03): [new chunksize formula](https://github.com/9001/copyparty/commit/54e1c8d261df) for files larger than 128 GiB
+  * **users:** upgrade to the latest [cli uploader](https://github.com/9001/copyparty/blob/hovudstraum/bin/up2k.py) if you use that
+  * **devs:** update third-party up2k clients (if those even exist)
+
+
 # FAQ
 
 "frequently" asked questions
@@ -344,6 +349,13 @@ examples:
 anyone trying to bruteforce a password gets banned according to `--ban-pw`; default is 24h ban for 9 failed attempts in 1 hour
 
 
+## shadowing
+
+hiding specific subfolders  by mounting another volume on top of them
+
+for example `-v /mnt::r -v /var/empty:web/certs:r` mounts the server folder `/mnt` as the webroot, but another volume is mounted at `/web/certs` -- so visitors can only see the contents of `/mnt` and `/mnt/web` (at URLs `/` and `/web`), but not `/mnt/web/certs` because URL `/web/certs` is mapped to `/var/empty`
+
+
 # the browser
 
 accessing a copyparty server using a web-browser
@@ -367,6 +379,7 @@ the main tabs in the ui
 ## hotkeys
 
 the browser has the following hotkeys  (always qwerty)
+* `?` show hotkeys help
 * `B` toggle breadcrumbs / [navpane](#navpane)
 * `I/K` prev/next folder
 * `M` parent folder (or unexpand current)
@@ -374,8 +387,10 @@ the browser has the following hotkeys  (always qwerty)
 * `G` toggle list / [grid view](#thumbnails) -- same as `田` bottom-right
 * `T` toggle thumbnails / icons
 * `ESC` close various things
+* `ctrl-K` delete selected files/folders
 * `ctrl-X` cut selected files/folders
 * `ctrl-V` paste
+* `Y` download selected files
 * `F2` [rename](#batch-rename) selected file/folder
 * when a file/folder is selected (in not-grid-view):
   * `Up/Down` move cursor
@@ -678,11 +693,38 @@ for the above example to work, add the commandline argument `-e2ts` to also scan
 # server config
 
 using arguments or config files, or a mix of both:
-* config files (`-c some.conf`) can set additional commandline arguments; see [./docs/example.conf](docs/example.conf)
+* config files (`-c some.conf`) can set additional commandline arguments; see [./docs/example.conf](docs/example.conf) and [./docs/example2.conf](docs/example2.conf)
 * `kill -s USR1` (same as `systemctl reload copyparty`) to reload accounts and volumes from config files without restarting
   * or click the `[reload cfg]` button in the control-panel when logged in as admin 
 
 
+## zeroconf
+
+announce enabled services on the LAN  if you specify the `-z` option, which enables [mdns](#mdns) and [ssdp](#ssdp)
+
+* `--z-on` / `--z-off`' limits the feature to certain networks
+
+
+### mdns
+
+LAN domain-name and feature announcer
+
+uses [multicast dns](https://en.wikipedia.org/wiki/Multicast_DNS) to give copyparty a domain which any machine on the LAN can use to access it
+
+all enabled services ([webdav](#webdav-server), [ftp](#ftp-server), [smb](#smb-server)) will appear in mDNS-aware file managers (KDE, gnome, macOS, ...)
+
+the domain will be http://partybox.local if the machine's hostname is `partybox` unless `--name` specifies soemthing else
+
+
+### ssdp
+
+windows-explorer announcer
+
+uses [ssdp](https://en.wikipedia.org/wiki/Simple_Service_Discovery_Protocol) to make copyparty appear in the windows file explorer on all machines on the LAN
+
+doubleclicking the icon opens the "connect" page which explains how to mount copyparty as a local filesystem
+
+
 ## qr-code
 
 print a qr-code [(screenshot)](https://user-images.githubusercontent.com/241032/194728533-6f00849b-c6ac-43c6-9359-83e454d11e00.png) for quick access,  great between phones on android hotspots which keep changing the subnet
@@ -693,10 +735,10 @@ print a qr-code [(screenshot)](https://user-images.githubusercontent.com/241032/
 * `--qrz 1` forces 1x zoom instead of autoscaling to fit the terminal size
   * 1x may render incorrectly on some terminals/fonts, but 2x should always work
 
-it will use your external ip (default route) unless `--qri` specifies an ip-prefix or domain
+it uses the server hostname if [mdns](#mdns) is enbled, otherwise it'll use your external ip (default route) unless `--qri` specifies a specific ip-prefix or domain
 
 
-## ftp-server
+## ftp server
 
 an FTP server can be started using `--ftp 3921`,  and/or `--ftps` for explicit TLS (ftpes)
 
@@ -706,6 +748,79 @@ an FTP server can be started using `--ftp 3921`,  and/or `--ftps` for explicit T
 * runs in active mode by default, you probably want `--ftp-pr 12000-13000`
   * if you enable both `ftp` and `ftps`, the port-range will be divided in half
   * some older software (filezilla on debian-stable) cannot passive-mode with TLS
+* login with any username + your password, or put your password in the username field
+
+
+## webdav server
+
+with read-write support,  supports winXP and later, macos, nautilus/gvfs
+
+click the [connect](http://127.0.0.1:3923/?hc) button in the control-panel to see connection instructions for windows, linux, macos
+
+general usage:
+* login with any username + your password, or put your password in the username field (password field can be empty/whatever)
+
+on macos, connect from finder:
+* [Go] -> [Connect to Server...] -> http://192.168.123.1:3923/
+
+
+### connecting to webdav from windows
+
+using the GUI  (winXP or later):
+* rightclick [my computer] -> [map network drive] -> Folder: `http://192.168.123.1:3923/`
+  * on winXP only, click the `Sign up for online storage` hyperlink instead and put the URL there
+  * providing your password as the username is recommended; the password field can be anything or empty
+
+known client bugs:
+* win7+ doesn't actually send the password to the server when reauthenticating after a reboot unless you first try to login with an incorrect password and then switch to the correct password
+  * or just type your password into the username field instead to get around it entirely
+* connecting to a folder which allows anonymous read will make writing impossible, as windows has decided it doesn't need to login
+  * workaround: connect twice; first to a folder which requires auth, then to the folder you actually want, and leave both of those mounted
+* win7+ may open a new tcp connection for every file and sometimes forgets to close them, eventually needing a reboot
+  * maybe NIC-related (??), happens with win10-ltsc on e1000e but not virtio
+* windows cannot access folders which contain filenames with invalid unicode or forbidden characters (`<>:"/\|?*`), or names ending with `.`
+* winxp cannot show unicode characters outside of *some range*
+  * latin-1 is fine, hiragana is not (not even as shift-jis on japanese xp)
+
+
+## smb server
+
+unsafe, slow, not recommended for wan,  enable with `--smb` for read-only or `--smbw` for read-write
+
+click the [connect](http://127.0.0.1:3923/?hc) button in the control-panel to see connection instructions for windows, linux, macos
+
+dependencies: `python3 -m pip install --user -U impacket==0.10.0`
+* newer versions of impacket will hopefully work just fine but there is monkeypatching so maybe not
+
+some **BIG WARNINGS** specific to SMB/CIFS, in decreasing importance:
+* not entirely confident that read-only is read-only
+* the smb backend is not fully integrated with vfs, meaning there could be security issues (path traversal). Please use `--smb-port` (see below) and [prisonparty](./bin/prisonparty.sh)
+  * account passwords work per-volume as expected, but account permissions are coalesced; all accounts have read-access to all volumes, and if a single account has write-access to some volume then all other accounts also do
+    * if no accounts have write-access to a specific volume, or if `--smbw` is not set, then writing to that volume from smb *should* be impossible
+    * will be fixed once [impacket v0.11.0](https://github.com/SecureAuthCorp/impacket/commit/d923c00f75d54b972bca573a211a82f09b55261a) is released
+  * [shadowing](#shadowing) probably works as expected but no guarantees
+
+and some minor issues,
+* clients only see the first ~400 files in big folders; [impacket#1433](https://github.com/SecureAuthCorp/impacket/issues/1433)
+* hot-reload of server config (`/?reload=cfg`) only works for volumes, not account passwords
+* listens on the first IPv4 `-i` interface only (default = :: = 0.0.0.0 = all)
+* login doesn't work on winxp, but anonymous access is ok -- remove all accounts from copyparty config for that to work
+  * win10 onwards does not allow connecting anonymously / without accounts
+* on windows, creating a new file through rightclick --> new --> textfile throws an error due to impacket limitations -- hit OK and F5 to get your file
+* python3 only
+* slow
+
+known client bugs:
+* on win7 only, `--smb1` is much faster than smb2 (default) because it keeps rescanning folders on smb2
+  * however smb1 is buggy and is not enabled by default on win10 onwards
+* windows cannot access folders which contain filenames with invalid unicode or forbidden characters (`<>:"/\|?*`), or names ending with `.`
+
+the smb protocol listens on TCP port 445, which is a privileged port on linux and macos, which would require running copyparty as root. However, this can be avoided by listening on another port using `--smb-port 3945` and then using NAT to forward the traffic from 445 to there;
+* on linux: `iptables -t nat -A PREROUTING -i eth0 -p tcp --dport 445 -j REDIRECT --to-port 3945`
+
+authenticate with one of the following:
+* username `$username`, password `$password`
+* username `$password`, password `k`
 
 
 ## file indexing
@@ -724,6 +839,7 @@ through arguments:
 * `-e2v` verfies file integrity at startup, comparing hashes from the db
 * `-e2vu` patches the database with the new hashes from the filesystem
 * `-e2vp` panics and kills copyparty instead
+* `--xlink` enables deduplication across volumes
 
 the same arguments can be set as volflags, in addition to `d2d`, `d2ds`, `d2t`, `d2ts`, `d2v` for disabling:
 * `-v ~/music::r:c,e2dsa,e2tsr` does a full reindex of everything on startup
@@ -1024,10 +1140,12 @@ interact with copyparty using non-browser clients
 
 * python: [up2k.py](https://github.com/9001/copyparty/blob/hovudstraum/bin/up2k.py) is a command-line up2k client [(webm)](https://ocv.me/stuff/u2cli.webm)
   * file uploads, file-search, autoresume of aborted/broken uploads
+  * can be downloaded from copyparty: controlpanel -> connect -> [up2k.py](http://127.0.0.1:3923/.cpr/a/up2k.py)
   * see [./bin/README.md#up2kpy](bin/README.md#up2kpy)
 
 * FUSE: mount a copyparty server as a local filesystem
   * cross-platform python client available in [./bin/](bin/)
+  * can be downloaded from copyparty: controlpanel -> connect -> [partyfuse.py](http://127.0.0.1:3923/.cpr/a/partyfuse.py)
   * [rclone](https://rclone.org/) as client can give ~5x performance, see [./docs/rclone.md](docs/rclone.md)
 
 * sharex (screenshot utility): see [./contrib/sharex.sxcu](contrib/#sharexsxcu)
@@ -1042,47 +1160,22 @@ you can provide passwords using cookie `cppwd=hunter2`, as a url-param `?pw=hunt
 NOTE: curl will not send the original filename if you use `-T` combined with url-params! Also, make sure to always leave a trailing slash in URLs unless you want to override the filename
 
 
-# up2k
+## mount as drive
 
-quick outline of the up2k protocol, see [uploading](#uploading) for the web-client
-* the up2k client splits a file into an "optimal" number of chunks
-  * 1 MiB each, unless that becomes more than 256 chunks
-  * tries 1.5M, 2M, 3, 4, 6, ... until <= 256 chunks or size >= 32M
-* client posts the list of hashes, filename, size, last-modified
-* server creates the `wark`, an identifier for this upload
-  * `sha512( salt + filesize + chunk_hashes )`
-  * and a sparse file is created for the chunks to drop into
-* client uploads each chunk
-  * header entries for the chunk-hash and wark
-  * server writes chunks into place based on the hash
-* client does another handshake with the hashlist; server replies with OK or a list of chunks to reupload
+a remote copyparty server as a local filesystem;  go to the control-panel and click `connect` to see a list of commands to do that
 
-up2k has saved a few uploads from becoming corrupted in-transfer already;
-* caught an android phone on wifi redhanded in wireshark with a bitflip, however bup with https would *probably* have noticed as well (thanks to tls also functioning as an integrity check)
-* also stopped someone from uploading because their ram was bad
+alternatively, some alternatives roughly sorted by speed (unreproducible benchmark), best first:
 
-regarding the frequent server log message during uploads;  
-`6.0M 106M/s 2.77G 102.9M/s n948 thank 4/0/3/1 10042/7198 00:01:09`
-* this chunk was `6 MiB`, uploaded at `106 MiB/s`
-* on this http connection, `2.77 GiB` transferred, `102.9 MiB/s` average, `948` chunks handled
-* client says `4` uploads OK, `0` failed, `3` busy, `1` queued, `10042 MiB` total size, `7198 MiB` and `00:01:09` left
+* [rclone-http](./docs/rclone.md) (25s), read-only
+* [rclone-ftp](./docs/rclone.md) (47s), read/WRITE
+* [rclone-webdav](./docs/rclone.md) (51s), read/WRITE
+  * copyparty-1.5.0's webdav server is faster than rclone-1.60.0 (69s)
+* [partyfuse.py](./bin/#partyfusepy) (71s), read-only
+* davfs2 (103s), read/WRITE, *very fast* on small files
+* [win10-webdav](#webdav-server) (138s), read/WRITE
+* [win10-smb2](#smb-server) (387s), read/WRITE
 
-
-## why chunk-hashes
-
-a single sha512 would be better, right?
-
-this is due to `crypto.subtle` [not yet](https://github.com/w3c/webcrypto/issues/73) providing a streaming api (or the option to seed the sha512 hasher with a starting hash)
-
-as a result, the hashes are much less useful than they could have been (search the server by sha512, provide the sha512 in the response http headers, ...)
-
-however it allows for hashing multiple chunks in parallel, greatly increasing upload speed from fast storage (NVMe, raid-0 and such)
-
-* both the [browser uploader](#uploading) and the [commandline one](https://github.com/9001/copyparty/blob/hovudstraum/bin/up2k.py) does this now, allowing for fast uploading even from plaintext http
-
-hashwasm would solve the streaming issue but reduces hashing speed for sha512 (xxh128 does 6 GiB/s), and it would make old browsers and [iphones](https://bugs.webkit.org/show_bug.cgi?id=228552) unsupported
-
-* blake2 might be a better choice since xxh is non-cryptographic, but that gets ~15 MiB/s on slower androids
+most clients will fail to mount the root of a copyparty server unless there is a root volume (so you get the admin-panel instead of a browser when accessing it) -- in that case, mount a specific volume instead
 
 
 # performance
@@ -1095,7 +1188,7 @@ below are some tweaks roughly ordered by usefulness:
 * `--http-only` or `--https-only` (unless you want to support both protocols) will reduce the delay before a new connection is established
 * `--hist` pointing to a fast location (ssd) will make directory listings and searches faster when `-e2d` or `-e2t` is set
 * `--no-hash .` when indexing a network-disk if you don't care about the actual filehashes and only want the names/tags searchable
-* `--no-htp --hash-mt=0 --th-mt=1` minimizes the number of threads; can help in some eccentric environments (like the vscode debugger)
+* `--no-htp --hash-mt=0 --mtag-mt=1 --th-mt=1` minimizes the number of threads; can help in some eccentric environments (like the vscode debugger)
 * `-j` enables multiprocessing (actual multithreading) and can make copyparty perform better in cpu-intensive workloads, for example:
   * huge amount of short-lived connections
   * really heavy traffic (downloads/uploads)
@@ -1134,12 +1227,13 @@ some notes on hardening
   * `--unpost 0`, `--no-del`, `--no-mv` disables all move/delete support
   * `--hardlink` creates hardlinks instead of symlinks when deduplicating uploads, which is less maintenance
     * however note if you edit one file it will also affect the other copies
-  * `--vague-403` returns a "404 not found" instead of "403 forbidden" which is a common enterprise meme
+  * `--vague-401` returns a "404 not found" instead of "401 unauthorized" which is a common enterprise meme
   * `--ban-404=50,60,1440` ban client for 1440min (24h) if they hit 50 404's in 60min
     * **NB:** will ban anyone who enables up2k turbo
   * `--nih` removes the server hostname from directory listings
 
 * option `-sss` is a shortcut for the above plus:
+  * `--no-dav` disables webdav support
   * `-lo cpp-%Y-%m%d-%H%M%S.txt.xz` enables logging to disk
   * `-ls **,*,ln,p,r` does a scan on startup for any dangerous symlinks
 
@@ -1179,90 +1273,7 @@ however you can hit `F12` in the up2k tab and use the devtools to see how far yo
 
 # HTTP API
 
-* table-column `params` = URL parameters; `?foo=bar&qux=...`
-* table-column `body` = POST payload
-* method `jPOST` = json post
-* method `mPOST` = multipart post
-* method `uPOST` = url-encoded post
-* `FILE` = conventional HTTP file upload entry (rfc1867 et al, filename in `Content-Disposition`)
-
-authenticate using header `Cookie: cppwd=foo` or url param `&pw=foo`
-
-## read
-
-| method | params | result |
-|--|--|--|
-| GET | `?ls` | list files/folders at URL as JSON |
-| GET | `?ls&dots` | list files/folders at URL as JSON, including dotfiles |
-| GET | `?ls=t` | list files/folders at URL as plaintext |
-| GET | `?ls=v` | list files/folders at URL, terminal-formatted |
-| GET | `?b` | list files/folders at URL as simplified HTML |
-| GET | `?tree=.` | list one level of subdirectories inside URL |
-| GET | `?tree` | list one level of subdirectories for each level until URL |
-| GET | `?tar` | download everything below URL as a tar file |
-| GET | `?zip=utf-8` | download everything below URL as a zip file |
-| GET | `?ups` | show recent uploads from your IP |
-| GET | `?ups&filter=f` | ...where URL contains `f` |
-| GET | `?mime=foo` | specify return mimetype `foo` |
-| GET | `?raw` | get markdown file at URL as plaintext |
-| GET | `?txt` | get file at URL as plaintext |
-| GET | `?txt=iso-8859-1` | ...with specific charset |
-| GET | `?th` | get image/video at URL as thumbnail |
-| GET | `?th=opus` | convert audio file to 128kbps opus |
-| GET | `?th=caf` | ...in the iOS-proprietary container |
-
-| method | body | result |
-|--|--|--|
-| jPOST | `{"q":"foo"}` | do a server-wide search; see the `[🔎]` search tab `raw` field for syntax |
-
-| method | params | body | result |
-|--|--|--|--|
-| jPOST | `?tar` | `["foo","bar"]` | download folders `foo` and `bar` inside URL as a tar file |
-
-## write
-
-| method | params | result |
-|--|--|--|
-| GET | `?move=/foo/bar` | move/rename the file/folder at URL to /foo/bar |
-
-| method | params | body | result |
-|--|--|--|--|
-| PUT | | (binary data) | upload into file at URL |
-| PUT | `?gz` | (binary data) | compress with gzip and write into file at URL |
-| PUT | `?xz` | (binary data) | compress with xz and write into file at URL |
-| mPOST | | `act=bput`, `f=FILE` | upload `FILE` into the folder at URL |
-| mPOST | `?j` | `act=bput`, `f=FILE` | ...and reply with json |
-| mPOST | | `act=mkdir`, `name=foo` | create directory `foo` at URL |
-| GET | `?delete` | | delete URL recursively |
-| jPOST | `?delete` | `["/foo","/bar"]` | delete `/foo` and `/bar` recursively |
-| uPOST | | `msg=foo` | send message `foo` into server log |
-| mPOST | | `act=tput`, `body=TEXT` | overwrite markdown document at URL |
-
-upload modifiers:
-
-| http-header | url-param | effect |
-|--|--|--|
-| `Accept: url` | `want=url` | return just the file URL |
-| `Rand: 4` | `rand=4` | generate random filename with 4 characters |
-| `Life: 30` | `life=30` | delete file after 30 seconds |
-
-* `life` only has an effect if the volume has a lifetime, and the volume lifetime must be greater than the file's
-
-* server behavior of `msg` can be reconfigured with `--urlform`
-
-## admin
-
-| method | params | result |
-|--|--|--|
-| GET | `?reload=cfg` | reload config files and rescan volumes |
-| GET | `?scan` | initiate a rescan of the volume which provides URL |
-| GET | `?stack` | show a stacktrace of all threads |
-
-## general
-
-| method | params | result |
-|--|--|--|
-| GET | `?pw=x` | logout |
+see [devnotes](#./docs/devnotes.md#http-api)
 
 
 # dependencies
@@ -1290,6 +1301,9 @@ enable [thumbnails](#thumbnails) of...
 * **AVIF pictures:** `pyvips` or `ffmpeg` or `pillow-avif-plugin`
 * **JPEG XL pictures:** `pyvips` or `ffmpeg`
 
+enable [smb](#smb-server) support:
+* `impacket==0.10.0`
+
 `pyvips` gives higher quality thumbnails than `Pillow` and is 320% faster, using 270% more ram: `sudo apt install libvips42 && python3 -m pip install --user -U pyvips`
 
 
@@ -1310,34 +1324,20 @@ these are standalone programs and will never be imported / evaluated by copypart
 
 the self-contained "binary"  [copyparty-sfx.py](https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.py) will unpack itself and run copyparty, assuming you have python installed of course
 
-
-## sfx repack
-
-reduce the size of an sfx by removing features
-
-if you don't need all the features, you can repack the sfx and save a bunch of space; all you need is an sfx and a copy of this repo (nothing else to download or build, except if you're on windows then you need msys2 or WSL)
-* `393k` size of original sfx.py as of v1.1.3
-* `310k` after `./scripts/make-sfx.sh re no-cm`
-* `269k` after `./scripts/make-sfx.sh re no-cm no-hl`
-
-the features you can opt to drop are
-* `cm`/easymde, the "fancy" markdown editor, saves ~82k
-* `hl`, prism, the syntax hilighter, saves ~41k
-* `fnt`, source-code-pro, the monospace font, saves ~9k
-* `dd`, the custom mouse cursor for the media player tray tab, saves ~2k
-
-for the `re`pack to work, first run one of the sfx'es once to unpack it
-
-**note:** you can also just download and run [scripts/copyparty-repack.sh](scripts/copyparty-repack.sh) -- this will grab the latest copyparty release from github and do a few repacks; works on linux/macos (and windows with msys2 or WSL)
+you can reduce the sfx size by repacking it; see [./docs/devnotes.md#sfx-repack](#./docs/devnotes.md#sfx-repack)
 
 
 ## copyparty.exe
 
+download [copyparty.exe](https://github.com/9001/copyparty/releases/latest/download/copyparty.exe) or [copyparty64.exe](https://github.com/9001/copyparty/releases/latest/download/copyparty64.exe)
+
 ![copyparty-exe-fs8](https://user-images.githubusercontent.com/241032/194707422-cb7f66c9-41a2-4cb9-8dbc-2ab866cd4338.png)
 
-[copyparty.exe](https://github.com/9001/copyparty/releases/latest/download/copyparty.exe) can be convenient on old machines where installing python is problematic, however is **not recommended** and should be considered a last resort -- if possible, please use **[copyparty-sfx.py](https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.py)** instead
+can be convenient on old machines where installing python is problematic, however is **not recommended** and should be considered a last resort -- if possible, please use **[copyparty-sfx.py](https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.py)** instead
 
-the exe is compatible with 32bit windows7, which means it uses an ancient copy of python (3.7.9) which cannot be upgraded and will definitely become a security hazard at some point
+* [copyparty.exe](https://github.com/9001/copyparty/releases/latest/download/copyparty.exe) is compatible with 32bit windows7, which means it uses an ancient copy of python (3.7.9) which cannot be upgraded and will definitely become a security hazard at some point
+
+* [copyparty64.exe](https://github.com/9001/copyparty/releases/latest/download/copyparty64.exe) is identical except 64bit so it [works in WinPE](https://user-images.githubusercontent.com/241032/205454984-e6b550df-3c49-486d-9267-1614078dd0dd.png)
 
 meanwhile [copyparty-sfx.py](https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.py) instead relies on your system python which gives better performance and will stay safe as long as you keep your python install up-to-date
 
@@ -1346,17 +1346,17 @@ then again, if you are already into downloading shady binaries from the internet
 
 # install on android
 
-install [Termux](https://termux.com/) (see [ocv.me/termux](https://ocv.me/termux/)) and then copy-paste this into Termux (long-tap) all at once:
+install [Termux](https://termux.com/) + its companion app `Termux:API` (see [ocv.me/termux](https://ocv.me/termux/)) and then copy-paste this into Termux (long-tap) all at once:
 ```sh
-apt update && apt -y full-upgrade && apt update && termux-setup-storage && apt -y install python && python -m ensurepip && python -m pip install --user -U copyparty
+yes | pkg upgrade && termux-setup-storage && yes | pkg install python termux-api && python -m ensurepip && python -m pip install --user -U copyparty && { grep -qE 'PATH=.*\.local/bin' ~/.bashrc 2>/dev/null || { echo 'PATH="$HOME/.local/bin:$PATH"' >> ~/.bashrc && . ~/.bashrc; }; }
 echo $?
 ```
 
 after the initial setup, you can launch copyparty at any time by running `copyparty` anywhere in Termux -- and if you run it with `--qr` you'll get a [neat qr-code](#qr-code) pointing to your external ip
 
-if you want thumbnails, `apt -y install ffmpeg`
+if you want thumbnails (photos+videos) and you're okay with spending another 132 MiB of storage, `pkg install ffmpeg && python3 -m pip install --user -U pillow`
 
-* or if you want to use vips instead, `apt -y install libvips && python -m pip install --user -U wheel && python -m pip install --user -U pyvips && (cd /data/data/com.termux/files/usr/lib/; ln -s libgobject-2.0.so{,.0}; ln -s libvips.so{,.42})`
+* or if you want to use `vips` for photo-thumbs instead, `pkg install libvips && python -m pip install --user -U wheel && python -m pip install --user -U pyvips && (cd /data/data/com.termux/files/usr/lib/; ln -s libgobject-2.0.so{,.0}; ln -s libvips.so{,.42})`
 
 
 # reporting bugs
@@ -1373,86 +1373,6 @@ journalctl -aS '48 hour ago' -u copyparty | grep -C10 FILENAME | tee bug.log
 if there's a wall of base64 in the log (thread stacks) then please include that, especially if you run into something freezing up or getting stuck, for example `OperationalError('database is locked')` -- alternatively you can visit `/?stack` to see the stacks live, so http://127.0.0.1:3923/?stack for example
 
 
-# building
+# devnotes
 
-## dev env setup
-
-you need python 3.9 or newer due to type hints
-
-the rest is mostly optional; if you need a working env for vscode or similar
-
-```sh
-python3 -m venv .venv
-. .venv/bin/activate
-pip install jinja2 strip_hints  # MANDATORY
-pip install mutagen  # audio metadata
-pip install pyftpdlib  # ftp server
-pip install Pillow pyheif-pillow-opener pillow-avif-plugin  # thumbnails
-pip install black==21.12b0 click==8.0.2 bandit pylint flake8 isort mypy  # vscode tooling
-```
-
-
-## just the sfx
-
-first grab the web-dependencies from a previous sfx (assuming you don't need to modify something in those):
-
-```sh
-rm -rf copyparty/web/deps
-curl -L https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.py >x.py
-python3 x.py -h
-rm x.py
-mv /tmp/pe-copyparty/copyparty/web/deps/ copyparty/web/deps/
-```
-
-then build the sfx using any of the following examples:
-
-```sh
-./scripts/make-sfx.sh           # regular edition
-./scripts/make-sfx.sh gz no-cm  # gzip-compressed + no fancy markdown editor
-```
-
-
-## complete release
-
-also builds the sfx so skip the sfx section above
-
-in the `scripts` folder:
-
-* run `make -C deps-docker` to build all dependencies
-* run `./rls.sh 1.2.3` which uploads to pypi + creates github release + sfx
-
-
-# todo
-
-roughly sorted by priority
-
-* nothing! currently
-
-
-## discarded ideas
-
-* reduce up2k roundtrips
-  * start from a chunk index and just go
-  * terminate client on bad data
-    * not worth the effort, just throw enough conncetions at it
-* single sha512 across all up2k chunks?
-  * crypto.subtle cannot into streaming, would have to use hashwasm, expensive
-* separate sqlite table per tag
-  * performance fixed by skipping some indexes (`+mt.k`)
-* audio fingerprinting
-  * only makes sense if there can be a wasm client and that doesn't exist yet (except for olaf which is agpl hence counts as not existing)
-* `os.copy_file_range` for up2k cloning
-  * almost never hit this path anyways
-* up2k partials ui
-  * feels like there isn't much point
-* cache sha512 chunks on client
-  * too dangerous -- overtaken by turbo mode
-* comment field
-  * nah
-* look into android thumbnail cache file format
-  * absolutely not
-* indexedDB for hashes, cfg enable/clear/sz, 2gb avail, ~9k for 1g, ~4k for 100m, 500k items before autoeviction
-  * blank hashlist when up-ok to skip handshake
-    * too many confusing side-effects
-* hls framework for Someone Else to drop code into :^)
-  * probably not, too much stuff to consider -- seeking, start at offset, task stitching (probably np-hard), conditional passthru, rate-control (especially multi-consumer), session keepalive, cache mgmt...
+for build instructions etc, see [./docs/devnotes.md](./docs/devnotes.md)

bin/README.md

@@ -11,7 +11,7 @@ produces a chronological list of all uploads by collecting info from up2k databa
 * optional mapping from IP-addresses to nicknames
 
 
-# [`copyparty-fuse.py`](copyparty-fuse.py)
+# [`partyfuse.py`](partyfuse.py)
 * mount a copyparty server as a local filesystem (read-only)
 * **supports Windows!** -- expect `194 MiB/s` sequential read
 * **supports Linux** -- expect `117 MiB/s` sequential read
@@ -30,19 +30,19 @@ also consider using [../docs/rclone.md](../docs/rclone.md) instead for 5x perfor
 * install [winfsp](https://github.com/billziss-gh/winfsp/releases/latest) and [python 3](https://www.python.org/downloads/)
   * [x] add python 3.x to PATH (it asks during install)
 * `python -m pip install --user fusepy`
-* `python ./copyparty-fuse.py n: http://192.168.1.69:3923/`
+* `python ./partyfuse.py n: http://192.168.1.69:3923/`
 
 10% faster in [msys2](https://www.msys2.org/), 700% faster if debug prints are enabled:
 * `pacman -S mingw64/mingw-w64-x86_64-python{,-pip}`
 * `/mingw64/bin/python3 -m pip install --user fusepy`
-* `/mingw64/bin/python3 ./copyparty-fuse.py [...]`
+* `/mingw64/bin/python3 ./partyfuse.py [...]`
 
 you could replace winfsp with [dokan](https://github.com/dokan-dev/dokany/releases/latest), let me know if you [figure out how](https://github.com/dokan-dev/dokany/wiki/FUSE)  
 (winfsp's sshfs leaks, doesn't look like winfsp itself does, should be fine)
 
 
 
-# [`copyparty-fuse🅱️.py`](copyparty-fuseb.py)
+# [`partyfuse2.py`](partyfuse2.py)
 * mount a copyparty server as a local filesystem (read-only)
 * does the same thing except more correct, `samba` approves
 * **supports Linux** -- expect `18 MiB/s` (wait what)
@@ -50,7 +50,7 @@ you could replace winfsp with [dokan](https://github.com/dokan-dev/dokany/releas
 
 
 
-# [`copyparty-fuse-streaming.py`](copyparty-fuse-streaming.py)
+# [`partyfuse-streaming.py`](partyfuse-streaming.py)
 * pretend this doesn't exist
 
 

bin/mtag/install-deps.sh

@@ -6,6 +6,7 @@ set -e
 #
 # linux/alpine: requires gcc g++ make cmake patchelf {python3,ffmpeg,fftw,libsndfile}-dev py3-{wheel,pip} py3-numpy{,-dev}
 # linux/debian: requires libav{codec,device,filter,format,resample,util}-dev {libfftw3,python3,libsndfile1}-dev python3-{numpy,pip} vamp-{plugin-sdk,examples} patchelf cmake
+# linux/fedora: requires gcc gcc-c++ make cmake patchelf {python3,ffmpeg,fftw,libsndfile}-devel python3-numpy vamp-plugin-sdk qm-vamp-plugins
 # win64: requires msys2-mingw64 environment
 # macos: requires macports
 #
@@ -160,12 +161,12 @@ install_keyfinder() {
 	
 	h="$HOME"
 	so="lib/libkeyfinder.so"
-	memes=()
+	memes=(-DBUILD_TESTING=OFF)
 
 	[ $win ] &&
 		so="bin/libkeyfinder.dll" &&
 		h="$(printf '%s\n' "$USERPROFILE" | tr '\\' '/')" &&
-		memes+=(-G "MinGW Makefiles" -DBUILD_TESTING=OFF)
+		memes+=(-G "MinGW Makefiles")
 	
 	[ $mac ] &&
 		so="lib/libkeyfinder.dylib"
@@ -185,7 +186,7 @@ install_keyfinder() {
 	}
 	
 	# rm -rf /Users/ed/Library/Python/3.9/lib/python/site-packages/*keyfinder*
-	CFLAGS="-I$h/pe/keyfinder/include -I/opt/local/include" \
+	CFLAGS="-I$h/pe/keyfinder/include -I/opt/local/include -I/usr/include/ffmpeg" \
 	LDFLAGS="-L$h/pe/keyfinder/lib -L$h/pe/keyfinder/lib64 -L/opt/local/lib" \
 	PKG_CONFIG_PATH=/c/msys64/mingw64/lib/pkgconfig \
 	$pybin -m pip install --user keyfinder

bin/partyfuse-streaming.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 from __future__ import print_function, unicode_literals
 
-"""copyparty-fuse-streaming: remote copyparty as a local filesystem"""
+"""partyfuse-streaming: remote copyparty as a local filesystem"""
 __author__ = "ed <copyparty@ocv.me>"
 __copyright__ = 2020
 __license__ = "MIT"
@@ -12,7 +12,7 @@ __url__ = "https://github.com/9001/copyparty/"
 mount a copyparty server (local or remote) as a filesystem
 
 usage:
-  python copyparty-fuse-streaming.py http://192.168.1.69:3923/  ./music
+  python partyfuse-streaming.py http://192.168.1.69:3923/  ./music
 
 dependencies:
   python3 -m pip install --user fusepy
@@ -21,7 +21,7 @@ dependencies:
   + on Windows: https://github.com/billziss-gh/winfsp/releases/latest
 
 this was a mistake:
-  fork of copyparty-fuse.py with a streaming cache rather than readahead,
+  fork of partyfuse.py with a streaming cache rather than readahead,
   thought this was gonna be way faster (and it kind of is)
   except the overhead of reopening connections on trunc totally kills it
 """
@@ -62,12 +62,12 @@ except:
     else:
         libfuse = "apt install libfuse\n    modprobe fuse"
 
-    print(
-        "\n  could not import fuse; these may help:"
-        + "\n    python3 -m pip install --user fusepy\n    "
-        + libfuse
-        + "\n"
-    )
+    m = """\033[33m
+  could not import fuse; these may help:
+    {} -m pip install --user fusepy
+    {}
+\033[0m"""
+    print(m.format(sys.executable, libfuse))
     raise
 
 
@@ -154,7 +154,7 @@ def dewin(txt):
 class RecentLog(object):
     def __init__(self):
         self.mtx = threading.Lock()
-        self.f = None  # open("copyparty-fuse.log", "wb")
+        self.f = None  # open("partyfuse.log", "wb")
         self.q = []
 
         thr = threading.Thread(target=self.printer)
@@ -185,9 +185,9 @@ class RecentLog(object):
             print("".join(q), end="")
 
 
-# [windows/cmd/cpy3]  python dev\copyparty\bin\copyparty-fuse.py q: http://192.168.1.159:1234/
-# [windows/cmd/msys2] C:\msys64\mingw64\bin\python3 dev\copyparty\bin\copyparty-fuse.py q: http://192.168.1.159:1234/
-# [windows/mty/msys2] /mingw64/bin/python3 /c/Users/ed/dev/copyparty/bin/copyparty-fuse.py q: http://192.168.1.159:1234/
+# [windows/cmd/cpy3]  python dev\copyparty\bin\partyfuse.py q: http://192.168.1.159:1234/
+# [windows/cmd/msys2] C:\msys64\mingw64\bin\python3 dev\copyparty\bin\partyfuse.py q: http://192.168.1.159:1234/
+# [windows/mty/msys2] /mingw64/bin/python3 /c/Users/ed/dev/copyparty/bin/partyfuse.py q: http://192.168.1.159:1234/
 #
 # [windows] find /q/music/albums/Phant*24bit -printf '%s %p\n' | sort -n | tail -n 8 | sed -r 's/^[0-9]+ //' | while IFS= read -r x; do dd if="$x" of=/dev/null bs=4k count=8192 & done
 # [alpine]  ll t; for x in t/2020_0724_16{2,3}*; do dd if="$x" of=/dev/null bs=4k count=10240 & done

bin/partyfuse.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 from __future__ import print_function, unicode_literals
 
-"""copyparty-fuse: remote copyparty as a local filesystem"""
+"""partyfuse: remote copyparty as a local filesystem"""
 __author__ = "ed <copyparty@ocv.me>"
 __copyright__ = 2019
 __license__ = "MIT"
@@ -12,7 +12,7 @@ __url__ = "https://github.com/9001/copyparty/"
 mount a copyparty server (local or remote) as a filesystem
 
 usage:
-  python copyparty-fuse.py http://192.168.1.69:3923/  ./music
+  python partyfuse.py http://192.168.1.69:3923/  ./music
 
 dependencies:
   python3 -m pip install --user fusepy
@@ -74,12 +74,12 @@ except:
     else:
         libfuse = "apt install libfuse3-3\n    modprobe fuse"
 
-    print(
-        "\n  could not import fuse; these may help:"
-        + "\n    python3 -m pip install --user fusepy\n    "
-        + libfuse
-        + "\n"
-    )
+    m = """\033[33m
+  could not import fuse; these may help:
+    {} -m pip install --user fusepy
+    {}
+\033[0m"""
+    print(m.format(sys.executable, libfuse))
     raise
 
 
@@ -166,7 +166,7 @@ def dewin(txt):
 class RecentLog(object):
     def __init__(self):
         self.mtx = threading.Lock()
-        self.f = None  # open("copyparty-fuse.log", "wb")
+        self.f = None  # open("partyfuse.log", "wb")
         self.q = []
 
         thr = threading.Thread(target=self.printer)
@@ -197,9 +197,9 @@ class RecentLog(object):
             print("".join(q), end="")
 
 
-# [windows/cmd/cpy3]  python dev\copyparty\bin\copyparty-fuse.py q: http://192.168.1.159:1234/
-# [windows/cmd/msys2] C:\msys64\mingw64\bin\python3 dev\copyparty\bin\copyparty-fuse.py q: http://192.168.1.159:1234/
-# [windows/mty/msys2] /mingw64/bin/python3 /c/Users/ed/dev/copyparty/bin/copyparty-fuse.py q: http://192.168.1.159:1234/
+# [windows/cmd/cpy3]  python dev\copyparty\bin\partyfuse.py q: http://192.168.1.159:1234/
+# [windows/cmd/msys2] C:\msys64\mingw64\bin\python3 dev\copyparty\bin\partyfuse.py q: http://192.168.1.159:1234/
+# [windows/mty/msys2] /mingw64/bin/python3 /c/Users/ed/dev/copyparty/bin/partyfuse.py q: http://192.168.1.159:1234/
 #
 # [windows] find /q/music/albums/Phant*24bit -printf '%s %p\n' | sort -n | tail -n 8 | sed -r 's/^[0-9]+ //' | while IFS= read -r x; do dd if="$x" of=/dev/null bs=4k count=8192 & done
 # [alpine]  ll t; for x in t/2020_0724_16{2,3}*; do dd if="$x" of=/dev/null bs=4k count=10240 & done

bin/partyfuse2.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 from __future__ import print_function, unicode_literals
 
-"""copyparty-fuseb: remote copyparty as a local filesystem"""
+"""partyfuse2: remote copyparty as a local filesystem"""
 __author__ = "ed <copyparty@ocv.me>"
 __copyright__ = 2020
 __license__ = "MIT"
@@ -32,9 +32,19 @@ try:
     if not hasattr(fuse, "__version__"):
         raise Exception("your fuse-python is way old")
 except:
-    print(
-        "\n  could not import fuse; these may help:\n    python3 -m pip install --user fuse-python\n    apt install libfuse\n    modprobe fuse\n"
-    )
+    if WINDOWS:
+        libfuse = "install https://github.com/billziss-gh/winfsp/releases/latest"
+    elif MACOS:
+        libfuse = "install https://osxfuse.github.io/"
+    else:
+        libfuse = "apt install libfuse\n    modprobe fuse"
+
+    m = """\033[33m
+  could not import fuse; these may help:
+    {} -m pip install --user fuse-python
+    {}
+\033[0m"""
+    print(m.format(sys.executable, libfuse))
     raise
 
 
@@ -42,13 +52,13 @@ except:
 mount a copyparty server (local or remote) as a filesystem
 
 usage:
-  python ./copyparty-fuseb.py -f -o allow_other,auto_unmount,nonempty,pw=wark,url=http://192.168.1.69:3923 /mnt/nas
+  python ./partyfuse2.py -f -o allow_other,auto_unmount,nonempty,pw=wark,url=http://192.168.1.69:3923 /mnt/nas
 
 dependencies:
   sudo apk add fuse-dev python3-dev
   python3 -m pip install --user fuse-python
 
-fork of copyparty-fuse.py based on fuse-python which
+fork of partyfuse.py based on fuse-python which
   appears to be more compliant than fusepy? since this works with samba
     (probably just my garbage code tbh)
 """
@@ -639,7 +649,7 @@ def main():
         print("  need argument: mount-path")
         print("example:")
         print(
-            "  ./copyparty-fuseb.py -f -o allow_other,auto_unmount,nonempty,pw=wark,url=http://192.168.1.69:3923 /mnt/nas"
+            "  ./partyfuse2.py -f -o allow_other,auto_unmount,nonempty,pw=wark,url=http://192.168.1.69:3923 /mnt/nas"
         )
         sys.exit(1)
 

bin/up2k.py

@@ -3,7 +3,7 @@ from __future__ import print_function, unicode_literals
 
 """
 up2k.py: upload to copyparty
-2022-09-05, v0.19, ed <irc.rizon.net>, MIT-Licensed
+2022-11-29, v0.22, ed <irc.rizon.net>, MIT-Licensed
 https://github.com/9001/copyparty/blob/hovudstraum/bin/up2k.py
 
 - dependencies: requests
@@ -51,8 +51,7 @@ except ImportError:
 PY2 = sys.version_info < (3,)
 if PY2:
     from Queue import Queue
-    from urllib import unquote
-    from urllib import quote
+    from urllib import quote, unquote
 
     sys.dont_write_bytecode = True
     bytes = str
@@ -69,6 +68,14 @@ VT100 = platform.system() != "Windows"
 req_ses = requests.Session()
 
 
+class Daemon(threading.Thread):
+    def __init__(self, target, name=None, a=None):
+        # type: (Any, Any, Any) -> None
+        threading.Thread.__init__(self, target=target, args=a or (), name=name)
+        self.daemon = True
+        self.start()
+
+
 class File(object):
     """an up2k upload task; represents a single file"""
 
@@ -86,6 +93,7 @@ class File(object):
         self.kchunks = {}  # type: dict[str, tuple[int, int]]  # hash: [ ofs, sz ]
 
         # set by handshake
+        self.recheck = False  # duplicate; redo handshake after all files done
         self.ucids = []  # type: list[str]  # chunks which need to be uploaded
         self.wark = None  # type: str
         self.url = None  # type: str
@@ -154,10 +162,7 @@ class MTHash(object):
         self.done_q = Queue()
         self.thrs = []
         for _ in range(cores):
-            t = threading.Thread(target=self.worker)
-            t.daemon = True
-            t.start()
-            self.thrs.append(t)
+            self.thrs.append(Daemon(self.worker))
 
     def hash(self, f, fsz, chunksz, pcb=None, pcb_opaque=None):
         with self.omutex:
@@ -290,9 +295,7 @@ class CTermsize(object):
         except:
             return
 
-        thr = threading.Thread(target=self.worker)
-        thr.daemon = True
-        thr.start()
+        Daemon(self.worker)
 
     def worker(self):
         while True:
@@ -420,7 +423,7 @@ def up2k_chunksize(filesize):
     while True:
         for mul in [1, 2]:
             nchunks = math.ceil(filesize * 1.0 / chunksize)
-            if nchunks <= 256 or chunksize >= 32 * 1024 * 1024:
+            if nchunks <= 256 or (chunksize >= 32 * 1024 * 1024 and nchunks < 4096):
                 return chunksize
 
             chunksize += stepsize
@@ -469,8 +472,8 @@ def get_hashlist(file, pcb, mth):
         file.kchunks[k] = [v1, v2]
 
 
-def handshake(req_ses, url, file, pw, search):
-    # type: (requests.Session, str, File, any, bool) -> list[str]
+def handshake(url, file, pw, search):
+    # type: (str, File, Any, bool) -> tuple[list[str], bool]
     """
     performs a handshake with the server; reply is:
       if search, a list of search results
@@ -504,6 +507,17 @@ def handshake(req_ses, url, file, pw, search):
             eprint("handshake failed, retrying: {0}\n  {1}\n\n".format(file.name, em))
             time.sleep(1)
 
+    sc = r.status_code
+    if sc >= 400:
+        txt = r.text
+        if sc == 422 or "<pre>partial upload exists at a different" in txt:
+            file.recheck = True
+            return [], False
+        elif sc == 409 or "<pre>upload rejected, file already exists" in txt:
+            return [], False
+
+        raise Exception("http {0}: {1}".format(sc, txt))
+
     try:
         r = r.json()
     except:
@@ -525,8 +539,8 @@ def handshake(req_ses, url, file, pw, search):
     return r["hash"], r["sprs"]
 
 
-def upload(req_ses, file, cid, pw):
-    # type: (requests.Session, File, str, any) -> None
+def upload(file, cid, pw):
+    # type: (File, str, Any) -> None
     """upload one specific chunk, `cid` (a chunk-hash)"""
 
     headers = {
@@ -548,12 +562,6 @@ def upload(req_ses, file, cid, pw):
         f.f.close()
 
 
-class Daemon(threading.Thread):
-    def __init__(self, *a, **ka):
-        threading.Thread.__init__(self, *a, **ka)
-        self.daemon = True
-
-
 class Ctl(object):
     """
     this will be the coordinator which runs everything in parallel
@@ -629,8 +637,8 @@ class Ctl(object):
 
             self.mutex = threading.Lock()
             self.q_handshake = Queue()  # type: Queue[File]
-            self.q_recheck = Queue()  # type: Queue[File]  # partial upload exists [...]
             self.q_upload = Queue()  # type: Queue[tuple[File, str]]
+            self.recheck = []  # type: list[File]
 
             self.st_hash = [None, "(idle, starting...)"]  # type: tuple[File, int]
             self.st_up = [None, "(idle, starting...)"]  # type: tuple[File, int]
@@ -652,7 +660,7 @@ class Ctl(object):
             burl = self.ar.url[:12] + self.ar.url[8:].split("/")[0] + "/"
             while True:
                 print("  hs...")
-                hs, _ = handshake(req_ses, self.ar.url, file, self.ar.a, search)
+                hs, _ = handshake(self.ar.url, file, self.ar.a, search)
                 if search:
                     if hs:
                         for hit in hs:
@@ -669,19 +677,28 @@ class Ctl(object):
                 ncs = len(hs)
                 for nc, cid in enumerate(hs):
                     print("  {0} up {1}".format(ncs - nc, cid))
-                    upload(req_ses, file, cid, self.ar.a)
+                    upload(file, cid, self.ar.a)
 
             print("  ok!")
+            if file.recheck:
+                self.recheck.append(file)
+
+        if not self.recheck:
+            return
+
+        eprint("finalizing {0} duplicate files".format(len(self.recheck)))
+        for file in self.recheck:
+            handshake(self.ar.url, file, self.ar.a, search)
 
     def _fancy(self):
         if VT100:
             atexit.register(self.cleanup_vt100)
             ss.scroll_region(3)
 
-        Daemon(target=self.hasher).start()
+        Daemon(self.hasher)
         for _ in range(self.ar.j):
-            Daemon(target=self.handshaker).start()
-            Daemon(target=self.uploader).start()
+            Daemon(self.handshaker)
+            Daemon(self.uploader)
 
         idles = 0
         while idles < 3:
@@ -743,6 +760,13 @@ class Ctl(object):
             t = "{0} eta @ {1}/s, {2}, {3}# left".format(eta, spd, sleft, nleft)
             eprint(txt + "\033]0;{0}\033\\\r{0}{1}".format(t, tail))
 
+        if not self.recheck:
+            return
+
+        eprint("finalizing {0} duplicate files".format(len(self.recheck)))
+        for file in self.recheck:
+            handshake(self.ar.url, file, self.ar.a, False)
+
     def cleanup_vt100(self):
         ss.scroll_region(None)
         eprint("\033[J\033]0;\033\\")
@@ -785,15 +809,17 @@ class Ctl(object):
             while True:
                 with self.mutex:
                     if (
-                        self.hash_b - self.up_b < 1024 * 1024 * 128
-                        and self.hash_c - self.up_c < 64
-                        and (
-                            not self.ar.nh
-                            or (
-                                self.q_upload.empty()
-                                and self.q_handshake.empty()
-                                and not self.uploader_busy
-                            )
+                        self.hash_f - self.up_f == 1
+                        or (
+                            self.hash_b - self.up_b < 1024 * 1024 * 1024
+                            and self.hash_c - self.up_c < 512
+                        )
+                    ) and (
+                        not self.ar.nh
+                        or (
+                            self.q_upload.empty()
+                            and self.q_handshake.empty()
+                            and not self.uploader_busy
                         )
                     ):
                         break
@@ -813,16 +839,10 @@ class Ctl(object):
 
     def handshaker(self):
         search = self.ar.s
-        q = self.q_handshake
         burl = self.ar.url[:8] + self.ar.url[8:].split("/")[0] + "/"
         while True:
-            file = q.get()
+            file = self.q_handshake.get()
             if not file:
-                if q == self.q_handshake:
-                    q = self.q_recheck
-                    q.put(None)
-                    continue
-
                 self.q_upload.put(None)
                 break
 
@@ -830,16 +850,7 @@ class Ctl(object):
                 self.handshaker_busy += 1
 
             upath = file.abs.decode("utf-8", "replace")
-
-            try:
-                hs, sprs = handshake(req_ses, self.ar.url, file, self.ar.a, search)
-            except Exception as ex:
-                if q == self.q_handshake and "<pre>partial upload exists" in str(ex):
-                    self.q_recheck.put(file)
-                    hs = []
-                else:
-                    raise
-
+            hs, sprs = handshake(self.ar.url, file, self.ar.a, search)
             if search:
                 if hs:
                     for hit in hs:
@@ -856,8 +867,11 @@ class Ctl(object):
 
                 continue
 
+            if file.recheck:
+                self.recheck.append(file)
+
             with self.mutex:
-                if not sprs and not self.serialized:
+                if hs and not sprs and not self.serialized:
                     t = "server filesystem does not support sparse files; serializing uploads\n"
                     eprint(t)
                     self.serialized = True
@@ -900,7 +914,7 @@ class Ctl(object):
 
             file, cid = task
             try:
-                upload(req_ses, file, cid, self.ar.a)
+                upload(file, cid, self.ar.a)
             except:
                 eprint("upload failed, retrying: {0} #{1}\n".format(file.name, cid[:8]))
                 pass  # handshake will fix it

contrib/README.md

@@ -27,7 +27,13 @@ however if your copyparty is behind a reverse-proxy, you may want to use [`share
 
 ### [`explorer-nothumbs-nofoldertypes.reg`](explorer-nothumbs-nofoldertypes.reg)
 * disables thumbnails and folder-type detection in windows explorer
-* makes it way faster (especially for slow/networked locations (such as copyparty-fuse))
+* makes it way faster (especially for slow/networked locations (such as partyfuse))
+
+### [`webdav-basicauth.reg`](webdav-basicauth.reg)
+* enables webdav basic-auth over plaintext http; takes effect after a reboot OR after running `webdav-unlimit.bat`
+
+### [`webdav-unlimit.bat`](webdav-unlimit.bat)
+* removes the 47.6 MiB filesize limit when downloading from webdav
 
 ### [`cfssl.sh`](cfssl.sh)
 * creates CA and server certificates using cfssl

contrib/nginx/copyparty.conf

@@ -1,11 +1,11 @@
 # when running copyparty behind a reverse proxy,
 # the following arguments are recommended:
 #
-#   -nc 512         important, see next paragraph
 #   --http-only     lower latency on initial connection
 #   -i 127.0.0.1    only accept connections from nginx
 #
 # -nc must match or exceed the webserver's max number of concurrent clients;
+# copyparty default is 1024 if OS permits it (see "max clients:" on startup),
 # nginx default is 512  (worker_processes 1, worker_connections 512)
 #
 # you may also consider adding -j0 for CPU-intensive configurations

contrib/webdav-cfg.bat

@@ -0,0 +1,51 @@
+@echo off
+rem removes the 47.6 MiB filesize limit when downloading from webdav
+rem + optionally allows/enables password-auth over plaintext http
+rem + optionally helps disable wpad
+
+setlocal enabledelayedexpansion
+
+net session >nul 2>&1
+if %errorlevel% neq 0 (
+    echo sorry, you must run this as administrator
+    pause
+    exit /b
+)
+
+reg add HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\services\WebClient\Parameters /v FileSizeLimitInBytes /t REG_DWORD /d 0xffffffff /f
+reg add HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Services\WebClient\Parameters /v FsCtlRequestTimeoutInSec /t REG_DWORD /d 0xffffffff /f
+
+echo(
+echo OK;
+echo allow webdav basic-auth over plaintext http?
+echo Y: login works, but the password will be visible in wireshark etc
+echo N: login will NOT work unless you use https and valid certificates
+set c=.
+set /p "c=(Y/N): "
+echo(
+if /i not "!c!"=="y" goto :g1
+reg add HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\services\WebClient\Parameters /v BasicAuthLevel /t REG_DWORD /d 0x2 /f
+rem default is 1 (require tls)
+
+:g1
+echo(
+echo OK;
+echo do you want to disable wpad?
+echo can give a HUGE speed boost depending on network settings
+set c=.
+set /p "c=(Y/N): "
+echo(
+if /i not "!c!"=="y" goto :g2
+echo(
+echo i'm about to open the [Connections] tab in [Internet Properties] for you;
+echo please click [LAN settings] and disable [Automatically detect settings]
+echo(
+pause
+control inetcpl.cpl,,4
+
+:g2
+net stop webclient
+net start webclient
+echo(
+echo OK; all done
+pause

copyparty/__init__.py

@@ -7,16 +7,19 @@ import sys
 import time
 
 try:
-    from typing import TYPE_CHECKING, Any
+    from typing import TYPE_CHECKING
 except:
     TYPE_CHECKING = False
 
+if True:
+    from typing import Any, Callable
+
 PY2 = sys.version_info < (3,)
-if PY2:
+if not PY2:
+    unicode: Callable[[Any], str] = str
+else:
     sys.dont_write_bytecode = True
     unicode = unicode  # noqa: F821  # pylint: disable=undefined-variable,self-assigning-variable
-else:
-    unicode = str
 
 WINDOWS: Any = (
     [int(x) for x in platform.version().split(".")]
@@ -40,8 +43,8 @@ except:
 class EnvParams(object):
     def __init__(self) -> None:
         self.t0 = time.time()
-        self.mod = None
-        self.cfg = None
+        self.mod = ""
+        self.cfg = ""
         self.ox = getattr(sys, "oxidized", None)
 
 

copyparty/__main__.py

@@ -9,26 +9,30 @@ __license__ = "MIT"
 __url__ = "https://github.com/9001/copyparty/"
 
 import argparse
+import base64
 import filecmp
 import locale
 import os
 import re
 import shutil
+import socket
 import sys
 import threading
 import time
 import traceback
+import uuid
 from textwrap import dedent
 
 from .__init__ import ANYWIN, CORES, PY2, VT100, WINDOWS, E, EnvParams, unicode
 from .__version__ import CODENAME, S_BUILD_DT, S_VERSION
-from .authsrv import re_vol
+from .authsrv import expand_config_file, re_vol
 from .svchub import SvcHub
 from .util import (
     IMPLICATIONS,
     JINJA_VER,
     PYFTPD_VER,
     SQLITE_VER,
+    UNPLICATIONS,
     align_tab,
     ansi_re,
     min_ex,
@@ -37,13 +41,11 @@ from .util import (
     wrap,
 )
 
-try:
+if True:  # pylint: disable=using-constant-test
     from collections.abc import Callable
     from types import FrameType
 
     from typing import Any, Optional
-except:
-    pass
 
 try:
     HAVE_SSL = True
@@ -78,9 +80,9 @@ class RiceFormatter(argparse.HelpFormatter):
                     ret += fmt
 
         if not VT100:
-            ret = re.sub("\033\[[0-9;]+m", "", ret)
+            ret = re.sub("\033\\[[0-9;]+m", "", ret)
 
-        return ret
+        return ret  # type: ignore
 
     def _fill_text(self, text: str, width: int, indent: str) -> str:
         """same as RawDescriptionHelpFormatter(HelpFormatter)"""
@@ -103,7 +105,7 @@ class RiceFormatter(argparse.HelpFormatter):
                     self.__add_whitespace(i, lWSpace, x)
                     for i, x in enumerate(wrap(line, width, width - 1))
                 ]
-                textRows[idx] = lines
+                textRows[idx] = lines  # type: ignore
 
         return [item for sublist in textRows for item in sublist]
 
@@ -140,7 +142,7 @@ def init_E(E: EnvParams) -> None:
     # __init__ runs 18 times when oxidized; do expensive stuff here
 
     def get_unixdir() -> str:
-        paths: list[tuple[Callable[..., str], str]] = [
+        paths: list[tuple[Callable[..., Any], str]] = [
             (os.environ.get, "XDG_CONFIG_HOME"),
             (os.path.expanduser, "~/.config"),
             (os.environ.get, "TMPDIR"),
@@ -162,7 +164,7 @@ def init_E(E: EnvParams) -> None:
                     if not os.path.isdir(p):
                         os.mkdir(p)
 
-                    return p
+                    return p  # type: ignore
                 except:
                     pass
 
@@ -195,7 +197,8 @@ def init_E(E: EnvParams) -> None:
         E.mod = _unpack()
 
     if sys.platform == "win32":
-        E.cfg = os.path.normpath(os.environ["APPDATA"] + "/copyparty")
+        bdir = os.environ.get("APPDATA") or os.environ.get("TEMP")
+        E.cfg = os.path.normpath(bdir + "/copyparty")
     elif sys.platform == "darwin":
         E.cfg = os.path.expanduser("~/Library/Preferences/copyparty")
     else:
@@ -209,6 +212,31 @@ def init_E(E: EnvParams) -> None:
             raise
 
 
+def get_srvname() -> str:
+    try:
+        ret: str = unicode(socket.gethostname()).split(".")[0]
+    except:
+        ret = ""
+
+    if ret not in ["", "localhost"]:
+        return ret
+
+    fp = os.path.join(E.cfg, "name.txt")
+    lprint("using hostname from {}\n".format(fp))
+    try:
+        with open(fp, "rb") as f:
+            ret = f.read().decode("utf-8", "replace").strip()
+    except:
+        ret = ""
+        while len(ret) < 7:
+            ret += base64.b32encode(os.urandom(4))[:7].decode("utf-8").lower()
+            ret = re.sub("[234567=]", "", ret)[:7]
+        with open(fp, "wb") as f:
+            f.write(ret.encode("utf-8") + b"\n")
+
+    return ret
+
+
 def ensure_locale() -> None:
     for x in [
         "en_US.UTF-8",
@@ -317,27 +345,29 @@ def configure_ssl_ciphers(al: argparse.Namespace) -> None:
 
 
 def args_from_cfg(cfg_path: str) -> list[str]:
+    lines: list[str] = []
+    expand_config_file(lines, cfg_path, "")
+
     ret: list[str] = []
     skip = False
-    with open(cfg_path, "rb") as f:
-        for ln in [x.decode("utf-8").strip() for x in f]:
-            if not ln:
-                skip = False
-                continue
+    for ln in lines:
+        if not ln:
+            skip = False
+            continue
 
-            if ln.startswith("#"):
-                continue
+        if ln.startswith("#"):
+            continue
 
-            if not ln.startswith("-"):
-                continue
+        if not ln.startswith("-"):
+            continue
 
-            if skip:
-                continue
+        if skip:
+            continue
 
-            try:
-                ret.extend(ln.split(" ", 1))
-            except:
-                ret.append(ln)
+        try:
+            ret.extend(ln.split(" ", 1))
+        except:
+            ret.append(ln)
 
     return ret
 
@@ -411,7 +441,9 @@ def showlic() -> None:
         print(f.read().decode("utf-8", "replace"))
 
 
-def run_argparse(argv: list[str], formatter: Any, retry: bool) -> argparse.Namespace:
+def run_argparse(
+    argv: list[str], formatter: Any, retry: bool, nc: int
+) -> argparse.Namespace:
     ap = argparse.ArgumentParser(
         formatter_class=formatter,
         prog="copyparty",
@@ -425,6 +457,10 @@ def run_argparse(argv: list[str], formatter: Any, retry: bool) -> argparse.Names
 
     hcores = min(CORES, 3)  # 4% faster than 4+ on py3.9 @ r5-4500U
 
+    tty = os.environ.get("TERM", "").lower() == "linux"
+
+    srvname = get_srvname()
+
     sects = [
         [
             "accounts",
@@ -504,6 +540,7 @@ def run_argparse(argv: list[str], formatter: Any, retry: bool) -> argparse.Names
               \033[36mnohash=\\.iso$\033[35m skips hashing file contents if path matches *.iso
               \033[36mnoidx=\\.iso$\033[35m fully ignores the contents at paths matching *.iso
               \033[36mnoforget$\033[35m don't forget files when deleted from disk
+              \033[36mxlink$\033[35m cross-volume dupe detection / linking
               \033[36mxdev\033[35m do not descend into other filesystems
               \033[36mxvol\033[35m skip symlinks leaving the volume root
 
@@ -569,7 +606,7 @@ def run_argparse(argv: list[str], formatter: Any, retry: bool) -> argparse.Names
     u = unicode
     ap2 = ap.add_argument_group('general options')
     ap2.add_argument("-c", metavar="PATH", type=u, action="append", help="add config file")
-    ap2.add_argument("-nc", metavar="NUM", type=int, default=64, help="max num clients")
+    ap2.add_argument("-nc", metavar="NUM", type=int, default=nc, help="max num clients")
     ap2.add_argument("-j", metavar="CORES", type=int, default=1, help="max num cpu cores, 0=all")
     ap2.add_argument("-a", metavar="ACCT", type=u, action="append", help="add account, \033[33mUSER\033[0m:\033[33mPASS\033[0m; example [\033[32med:wark\033[0m]")
     ap2.add_argument("-v", metavar="VOL", type=u, action="append", help="add volume, \033[33mSRC\033[0m:\033[33mDST\033[0m:\033[33mFLAG\033[0m; examples [\033[32m.::r\033[0m], [\033[32m/mnt/nas/music:/music:r:aed\033[0m]")
@@ -578,6 +615,7 @@ def run_argparse(argv: list[str], formatter: Any, retry: bool) -> argparse.Names
     ap2.add_argument("-mcr", metavar="SEC", type=int, default=60, help="md-editor mod-chk rate")
     ap2.add_argument("--urlform", metavar="MODE", type=u, default="print,get", help="how to handle url-form POSTs; see --help-urlform")
     ap2.add_argument("--wintitle", metavar="TXT", type=u, default="cpp @ $pub", help="window title, for example [\033[32m$ip-10.1.2.\033[0m] or [\033[32m$ip-]")
+    ap2.add_argument("--name", metavar="TXT", type=u, default=srvname, help="server name (displayed topleft in browser and in mDNS)")
     ap2.add_argument("--license", action="store_true", help="show licenses and exit")
     ap2.add_argument("--version", action="store_true", help="show versions and exit")
 
@@ -585,23 +623,25 @@ def run_argparse(argv: list[str], formatter: Any, retry: bool) -> argparse.Names
     ap2.add_argument("--qr", action="store_true", help="show http:// QR-code on startup")
     ap2.add_argument("--qrs", action="store_true", help="show https:// QR-code on startup")
     ap2.add_argument("--qrl", metavar="PATH", type=u, default="", help="location to include in the url, for example [\033[32mpriv/?pw=hunter2\033[0m]")
-    ap2.add_argument("--qri", metavar="PREFIX", type=u, default="", help="select IP which starts with PREFIX")
-    ap2.add_argument("--qr-fg", metavar="COLOR", type=int, default=16, help="foreground")
+    ap2.add_argument("--qri", metavar="PREFIX", type=u, default="", help="select IP which starts with PREFIX; [\033[32m.\033[0m] to force default IP when mDNS URL would have been used instead")
+    ap2.add_argument("--qr-fg", metavar="COLOR", type=int, default=0 if tty else 16, help="foreground; try [\033[32m0\033[0m] if the qr-code is unreadable")
     ap2.add_argument("--qr-bg", metavar="COLOR", type=int, default=229, help="background (white=255)")
     ap2.add_argument("--qrp", metavar="CELLS", type=int, default=4, help="padding (spec says 4 or more, but 1 is usually fine)")
-    ap2.add_argument("--qrz", metavar="N", type=int, default=0, help="[\033[32m1\033[0m]=1x, [\033[32m2\033[0m]=2x, [\033[32m0\033[0m]=auto (try 2 on broken fonts)")
+    ap2.add_argument("--qrz", metavar="N", type=int, default=0, help="[\033[32m1\033[0m]=1x, [\033[32m2\033[0m]=2x, [\033[32m0\033[0m]=auto (try [\033[32m2\033[0m] on broken fonts)")
 
     ap2 = ap.add_argument_group('upload options')
     ap2.add_argument("--dotpart", action="store_true", help="dotfile incomplete uploads, hiding them from clients unless -ed")
     ap2.add_argument("--plain-ip", action="store_true", help="when avoiding filename collisions by appending the uploader's ip to the filename: append the plaintext ip instead of salting and hashing the ip")
     ap2.add_argument("--unpost", metavar="SEC", type=int, default=3600*12, help="grace period where uploads can be deleted by the uploader, even without delete permissions; 0=disabled")
     ap2.add_argument("--reg-cap", metavar="N", type=int, default=38400, help="max number of uploads to keep in memory when running without -e2d; roughly 1 MiB RAM per 600")
-    ap2.add_argument("--no-fpool", action="store_true", help="disable file-handle pooling -- instead, repeatedly close and reopen files during upload")
-    ap2.add_argument("--use-fpool", action="store_true", help="force file-handle pooling, even if copyparty thinks you're better off without -- probably useful on nfs and cow filesystems (zfs, btrfs)")
+    ap2.add_argument("--no-fpool", action="store_true", help="disable file-handle pooling -- instead, repeatedly close and reopen files during upload (very slow on windows)")
+    ap2.add_argument("--use-fpool", action="store_true", help="force file-handle pooling, even when it might be dangerous (multiprocessing, filesystems lacking sparse-files support, ...)")
     ap2.add_argument("--hardlink", action="store_true", help="prefer hardlinks instead of symlinks when possible (within same filesystem)")
     ap2.add_argument("--never-symlink", action="store_true", help="do not fallback to symlinks when a hardlink cannot be made")
     ap2.add_argument("--no-dedup", action="store_true", help="disable symlink/hardlink creation; copy file contents instead")
-    ap2.add_argument("--magic", action="store_true", help="enable filetype detection on nameless uploads")
+    ap2.add_argument("--no-dupe", action="store_true", help="reject duplicate files during upload; only matches within the same volume (volflag=nodupe)")
+    ap2.add_argument("--no-snap", action="store_true", help="disable snapshots -- forget unfinished uploads on shutdown; don't create .hist/up2k.snap files -- abandoned/interrupted uploads must be cleaned up manually")
+    ap2.add_argument("--magic", action="store_true", help="enable filetype detection on nameless uploads (volflag=magic)")
     ap2.add_argument("--df", metavar="GiB", type=float, default=0, help="ensure GiB free disk space by rejecting upload requests")
     ap2.add_argument("--sparse", metavar="MiB", type=int, default=4, help="windows-only: minimum size of incoming uploads through up2k before they are made into sparse files")
     ap2.add_argument("--turbo", metavar="LVL", type=int, default=0, help="configure turbo-mode in up2k client; [\033[32m0\033[0m] = off and warn if enabled, [\033[32m1\033[0m] = off, [\033[32m2\033[0m] = on, [\033[32m3\033[0m] = on and disable datecheck")
@@ -609,7 +649,7 @@ def run_argparse(argv: list[str], formatter: Any, retry: bool) -> argparse.Names
     ap2.add_argument("--write-uplog", action="store_true", help="write POST reports to textfiles in working-directory")
 
     ap2 = ap.add_argument_group('network options')
-    ap2.add_argument("-i", metavar="IP", type=u, default="0.0.0.0", help="ip to bind (comma-sep.)")
+    ap2.add_argument("-i", metavar="IP", type=u, default="::", help="ip to bind (comma-sep.), default: all IPv4 and IPv6")
     ap2.add_argument("-p", metavar="PORT", type=u, default="3923", help="ports to bind (comma/range)")
     ap2.add_argument("--rproxy", metavar="DEPTH", type=int, default=1, help="which ip to keep; [\033[32m0\033[0m]=tcp, [\033[32m1\033[0m]=origin (first x-fwd), [\033[32m2\033[0m]=cloudflare, [\033[32m3\033[0m]=nginx, [\033[32m-1\033[0m]=closest proxy")
     ap2.add_argument("--s-wr-sz", metavar="B", type=int, default=256*1024, help="socket write size in bytes")
@@ -624,27 +664,77 @@ def run_argparse(argv: list[str], formatter: Any, retry: bool) -> argparse.Names
     ap2.add_argument("--ssl-dbg", action="store_true", help="dump some tls info")
     ap2.add_argument("--ssl-log", metavar="PATH", type=u, help="log master secrets for later decryption in wireshark")
 
+    ap2 = ap.add_argument_group("Zeroconf options")
+    ap2.add_argument("-z", action="store_true", help="enable all zeroconf backends (mdns, ssdp)")
+    ap2.add_argument("--z-on", metavar="NICS/NETS", type=u, default="", help="enable zeroconf ONLY on the comma-separated list of subnets and/or interface names/indexes\n └─example: \033[32meth0, wlo1, virhost0, 192.168.123.0/24, fd00:fda::/96\033[0m")
+    ap2.add_argument("--z-off", metavar="NICS/NETS", type=u, default="", help="disable zeroconf on the comma-separated list of subnets and/or interface names/indexes")
+    ap2.add_argument("-zv", action="store_true", help="verbose all zeroconf backends")
+    ap2.add_argument("--mc-hop", metavar="SEC", type=int, default=0, help="rejoin multicast groups every SEC seconds (workaround for some switches/routers which cause mDNS to suddenly stop working after some time); try [\033[32m300\033[0m] or [\033[32m180\033[0m]")
+
+    ap2 = ap.add_argument_group("Zeroconf-mDNS options:")
+    ap2.add_argument("--zm", action="store_true", help="announce the enabled protocols over mDNS (multicast DNS-SD) -- compatible with KDE, gnome, macOS, ...")
+    ap2.add_argument("--zm-on", metavar="NICS/NETS", type=u, default="", help="enable zeroconf ONLY on the comma-separated list of subnets and/or interface names/indexes")
+    ap2.add_argument("--zm-off", metavar="NICS/NETS", type=u, default="", help="disable zeroconf on the comma-separated list of subnets and/or interface names/indexes")
+    ap2.add_argument("--zm4", action="store_true", help="IPv4 only -- try this if some clients can't connect")
+    ap2.add_argument("--zm6", action="store_true", help="IPv6 only")
+    ap2.add_argument("--zmv", action="store_true", help="verbose mdns")
+    ap2.add_argument("--zmvv", action="store_true", help="verboser mdns")
+    ap2.add_argument("--zms", metavar="dhf", type=u, default="", help="list of services to announce -- d=webdav h=http f=ftp s=smb -- lowercase=plaintext uppercase=TLS -- default: all enabled services except http/https (\033[32mDdfs\033[0m if \033[33m--ftp\033[0m and \033[33m--smb\033[0m is set)")
+    ap2.add_argument("--zm-ld", metavar="PATH", type=u, default="", help="link a specific folder for webdav shares")
+    ap2.add_argument("--zm-lh", metavar="PATH", type=u, default="", help="link a specific folder for http shares")
+    ap2.add_argument("--zm-lf", metavar="PATH", type=u, default="", help="link a specific folder for ftp shares")
+    ap2.add_argument("--zm-ls", metavar="PATH", type=u, default="", help="link a specific folder for smb shares")
+    ap2.add_argument("--zm-mnic", action="store_true", help="merge NICs which share subnets; assume that same subnet means same network")
+    ap2.add_argument("--zm-msub", action="store_true", help="merge subnets on each NIC -- always enabled for ipv6 -- reduces network load, but gnome-gvfs clients may stop working")
+    ap2.add_argument("--zm-noneg", action="store_true", help="disable NSEC replies -- try this if some clients don't see copyparty")
+
+    ap2 = ap.add_argument_group("Zeroconf-SSDP options:")
+    ap2.add_argument("--zs", action="store_true", help="announce the enabled protocols over SSDP -- compatible with Windows")
+    ap2.add_argument("--zs-on", metavar="NICS/NETS", type=u, default="", help="enable zeroconf ONLY on the comma-separated list of subnets and/or interface names/indexes")
+    ap2.add_argument("--zs-off", metavar="NICS/NETS", type=u, default="", help="disable zeroconf on the comma-separated list of subnets and/or interface names/indexes")
+    ap2.add_argument("--zsv", action="store_true", help="verbose SSDP")
+    ap2.add_argument("--zsl", metavar="PATH", type=u, default="/?hc", help="location to include in the url (or a complete external URL), for example [\033[32mpriv/?pw=hunter2\033[0m] or [\033[32mpriv/?pw=hunter2\033[0m]")
+    ap2.add_argument("--zsid", metavar="UUID", type=u, default=uuid.uuid4().urn[4:], help="USN (device identifier) to announce")
+
     ap2 = ap.add_argument_group('FTP options')
     ap2.add_argument("--ftp", metavar="PORT", type=int, help="enable FTP server on PORT, for example \033[32m3921")
     ap2.add_argument("--ftps", metavar="PORT", type=int, help="enable FTPS server on PORT, for example \033[32m3990")
-    ap2.add_argument("--ftp-dbg", action="store_true", help="enable debug logging")
+    ap2.add_argument("--ftpv", action="store_true", help="verbose")
+    ap2.add_argument("--ftp-wt", metavar="SEC", type=int, default=7, help="grace period for resuming interrupted uploads (any client can write to any file last-modified more recently than SEC seconds ago)")
     ap2.add_argument("--ftp-nat", metavar="ADDR", type=u, help="the NAT address to use for passive connections")
     ap2.add_argument("--ftp-pr", metavar="P-P", type=u, help="the range of TCP ports to use for passive connections, for example \033[32m12000-13000")
 
+    ap2 = ap.add_argument_group('WebDAV options')
+    ap2.add_argument("--daw", action="store_true", help="enable full write support. \033[1;31mWARNING:\033[0m This has side-effects -- PUT-operations will now \033[1;31mOVERWRITE\033[0m existing files, rather than inventing new filenames to avoid loss of data. You might want to instead set this as a volflag where needed. By not setting this flag, uploaded files can get written to a filename which the client does not expect (which might be okay, depending on client)")
+    ap2.add_argument("--dav-inf", action="store_true", help="allow depth:infinite requests (recursive file listing); extremely server-heavy but required for spec compliance -- luckily few clients rely on this")
+    ap2.add_argument("--dav-mac", action="store_true", help="disable apple-garbage filter -- allow macos to create junk files (._* and .DS_Store, .Spotlight-*, .fseventsd, .Trashes, .AppleDouble, __MACOS)")
+
+    ap2 = ap.add_argument_group('SMB/CIFS options')
+    ap2.add_argument("--smb", action="store_true", help="enable smb (read-only) -- this requires running copyparty as root on linux and macos unless --smb-port is set above 1024 and your OS does port-forwarding from 445 to that.\n\033[1;31mWARNING:\033[0m this protocol is dangerous! Never expose to the internet. Account permissions are coalesced; if one account has write-access to a volume, then all accounts do.")
+    ap2.add_argument("--smbw", action="store_true", help="enable write support (please dont)")
+    ap2.add_argument("--smb1", action="store_true", help="disable SMBv2, only enable SMBv1 (CIFS)")
+    ap2.add_argument("--smb-port", metavar="PORT", type=int, default=445, help="port to listen on -- if you change this value, you must NAT from TCP:445 to this port using iptables or similar")
+    ap2.add_argument("--smb-nwa-1", action="store_true", help="disable impacket#1433 workaround (truncate directory listings to 64kB)")
+    ap2.add_argument("--smb-nwa-2", action="store_true", help="disable impacket workaround for filecopy globs")
+    ap2.add_argument("--smbv", action="store_true", help="verbose")
+    ap2.add_argument("--smbvv", action="store_true", help="verboser")
+    ap2.add_argument("--smbvvv", action="store_true", help="verbosest")
+
     ap2 = ap.add_argument_group('opt-outs')
     ap2.add_argument("-nw", action="store_true", help="never write anything to disk (debug/benchmark)")
     ap2.add_argument("--keep-qem", action="store_true", help="do not disable quick-edit-mode on windows (it is disabled to avoid accidental text selection which will deadlock copyparty)")
+    ap2.add_argument("--no-dav", action="store_true", help="disable webdav support")
     ap2.add_argument("--no-del", action="store_true", help="disable delete operations")
     ap2.add_argument("--no-mv", action="store_true", help="disable move/rename operations")
     ap2.add_argument("-nih", action="store_true", help="no info hostname -- don't show in UI")
     ap2.add_argument("-nid", action="store_true", help="no info disk-usage -- don't show in UI")
     ap2.add_argument("--no-zip", action="store_true", help="disable download as zip/tar")
-    ap2.add_argument("--no-lifetime", action="store_true", help="disable automatic deletion of uploads after a certain time (lifetime volflag)")
+    ap2.add_argument("--no-lifetime", action="store_true", help="disable automatic deletion of uploads after a certain time (as specified by the 'lifetime' volflag)")
 
     ap2 = ap.add_argument_group('safety options')
     ap2.add_argument("-s", action="count", default=0, help="increase safety: Disable thumbnails / potentially dangerous software (ffmpeg/pillow/vips), hide partial uploads, avoid crawlers.\n └─Alias of\033[32m --dotpart --no-thumb --no-mtag-ff --no-robots --force-js")
-    ap2.add_argument("-ss", action="store_true", help="further increase safety: Prevent js-injection, accidental move/delete, broken symlinks, 404 on 403, ban on excessive 404s.\n └─Alias of\033[32m -s --no-dot-mv --no-dot-ren --unpost=0 --no-del --no-mv --hardlink --vague-403 --ban-404=50,60,1440 -nih")
-    ap2.add_argument("-sss", action="store_true", help="further increase safety: Enable logging to disk, scan for dangerous symlinks.\n └─Alias of\033[32m -ss -lo=cpp-%%Y-%%m%%d-%%H%%M%%S.txt.xz --ls=**,*,ln,p,r")
+    ap2.add_argument("-ss", action="store_true", help="further increase safety: Prevent js-injection, accidental move/delete, broken symlinks, webdav, 404 on 403, ban on excessive 404s.\n └─Alias of\033[32m -s --no-dot-mv --no-dot-ren --unpost=0 --no-del --no-mv --hardlink --vague-403 --ban-404=50,60,1440 -nih")
+    ap2.add_argument("-sss", action="store_true", help="further increase safety: Enable logging to disk, scan for dangerous symlinks.\n └─Alias of\033[32m -ss --no-dav -lo=cpp-%%Y-%%m%%d-%%H%%M%%S.txt.xz --ls=**,*,ln,p,r")
     ap2.add_argument("--ls", metavar="U[,V[,F]]", type=u, help="do a sanity/safety check of all volumes on startup; arguments \033[33mUSER\033[0m,\033[33mVOL\033[0m,\033[33mFLAGS\033[0m; example [\033[32m**,*,ln,p,r\033[0m]")
     ap2.add_argument("--salt", type=u, default="hunter2", help="up2k file-hash salt; used to generate unpredictable internal identifiers for uploads -- doesn't really matter")
     ap2.add_argument("--fk-salt", metavar="SALT", type=u, default=fk_salt, help="per-file accesskey salt; used to generate unpredictable URLs for hidden files -- this one DOES matter")
@@ -654,10 +744,12 @@ def run_argparse(argv: list[str], formatter: Any, retry: bool) -> argparse.Names
     ap2.add_argument("--no-readme", action="store_true", help="disable rendering readme.md into directory listings")
     ap2.add_argument("--vague-403", action="store_true", help="send 404 instead of 403 (security through ambiguity, very enterprise)")
     ap2.add_argument("--force-js", action="store_true", help="don't send folder listings as HTML, force clients to use the embedded json instead -- slight protection against misbehaving search engines which ignore --no-robots")
-    ap2.add_argument("--no-robots", action="store_true", help="adds http and html headers asking search engines to not index anything")
+    ap2.add_argument("--no-robots", action="store_true", help="adds http and html headers asking search engines to not index anything (volflag=norobots)")
     ap2.add_argument("--logout", metavar="H", type=float, default="8086", help="logout clients after H hours of inactivity; [\033[32m0.0028\033[0m]=10sec, [\033[32m0.1\033[0m]=6min, [\033[32m24\033[0m]=day, [\033[32m168\033[0m]=week, [\033[32m720\033[0m]=month, [\033[32m8760\033[0m]=year)")
     ap2.add_argument("--ban-pw", metavar="N,W,B", type=u, default="9,60,1440", help="more than \033[33mN\033[0m wrong passwords in \033[33mW\033[0m minutes = ban for \033[33mB\033[0m minutes; disable with [\033[32mno\033[0m]")
     ap2.add_argument("--ban-404", metavar="N,W,B", type=u, default="no", help="hitting more than \033[33mN\033[0m 404's in \033[33mW\033[0m minutes = ban for \033[33mB\033[0m minutes (disabled by default since turbo-up2k counts as 404s)")
+    ap2.add_argument("--aclose", metavar="MIN", type=int, default=10, help="if a client maxes out the server connection limit, downgrade it from connection:keep-alive to connection:close for MIN minutes (and also kill its active connections) -- disable with 0")
+    ap2.add_argument("--loris", metavar="B", type=int, default=60, help="if a client maxes out the server connection limit without sending headers, ban it for B minutes; disable with [\033[32m0\033[0m]")
 
     ap2 = ap.add_argument_group('shutdown options')
     ap2.add_argument("--ign-ebind", action="store_true", help="continue running even if it's impossible to listen on some of the requested endpoints")
@@ -671,7 +763,7 @@ def run_argparse(argv: list[str], formatter: Any, retry: bool) -> argparse.Names
     ap2.add_argument("--log-conn", action="store_true", help="debug: print tcp-server msgs")
     ap2.add_argument("--log-htp", action="store_true", help="debug: print http-server threadpool scaling")
     ap2.add_argument("--ihead", metavar="HEADER", type=u, action='append', help="dump incoming header")
-    ap2.add_argument("--lf-url", metavar="RE", type=u, default=r"^/\.cpr/|\?th=[wj]$", help="dont log URLs matching")
+    ap2.add_argument("--lf-url", metavar="RE", type=u, default=r"^/\.cpr/|\?th=[wj]$|/\.(_|ql_|DS_Store$|localized$)", help="dont log URLs matching")
 
     ap2 = ap.add_argument_group('admin panel options')
     ap2.add_argument("--no-reload", action="store_true", help="disable ?reload=cfg (reload users/volumes/volflags from config file)")
@@ -679,9 +771,9 @@ def run_argparse(argv: list[str], formatter: Any, retry: bool) -> argparse.Names
     ap2.add_argument("--no-stack", action="store_true", help="disable ?stack (list all stacks)")
 
     ap2 = ap.add_argument_group('thumbnail options')
-    ap2.add_argument("--no-thumb", action="store_true", help="disable all thumbnails")
-    ap2.add_argument("--no-athumb", action="store_true", help="disable audio thumbnails (spectrograms)")
-    ap2.add_argument("--no-vthumb", action="store_true", help="disable video thumbnails")
+    ap2.add_argument("--no-thumb", action="store_true", help="disable all thumbnails (volflag=dthumb)")
+    ap2.add_argument("--no-vthumb", action="store_true", help="disable video thumbnails (volflag=dvthumb)")
+    ap2.add_argument("--no-athumb", action="store_true", help="disable audio thumbnails (spectrograms) (volflag=dathumb)")
     ap2.add_argument("--th-size", metavar="WxH", default="320x256", help="thumbnail res")
     ap2.add_argument("--th-mt", metavar="CORES", type=int, default=CORES, help="num cpu cores to use for generating thumbnails")
     ap2.add_argument("--th-convt", metavar="SEC", type=int, default=60, help="conversion timeout in seconds")
@@ -715,15 +807,16 @@ def run_argparse(argv: list[str], formatter: Any, retry: bool) -> argparse.Names
     ap2.add_argument("-e2v", action="store_true", help="verify file integrity; rehash all files and compare with db")
     ap2.add_argument("-e2vu", action="store_true", help="on hash mismatch: update the database with the new hash")
     ap2.add_argument("-e2vp", action="store_true", help="on hash mismatch: panic and quit copyparty")
-    ap2.add_argument("--hist", metavar="PATH", type=u, help="where to store volume data (db, thumbs)")
-    ap2.add_argument("--no-hash", metavar="PTN", type=u, help="regex: disable hashing of matching paths during e2ds folder scans")
-    ap2.add_argument("--no-idx", metavar="PTN", type=u, help="regex: disable indexing of matching paths during e2ds folder scans")
+    ap2.add_argument("--hist", metavar="PATH", type=u, help="where to store volume data (db, thumbs) (volflag=hist)")
+    ap2.add_argument("--no-hash", metavar="PTN", type=u, help="regex: disable hashing of matching paths during e2ds folder scans (volflag=nohash)")
+    ap2.add_argument("--no-idx", metavar="PTN", type=u, help="regex: disable indexing of matching paths during e2ds folder scans (volflag=noidx)")
     ap2.add_argument("--no-dhash", action="store_true", help="disable rescan acceleration; do full database integrity check -- makes the db ~5%% smaller and bootup/rescans 3~10x slower")
-    ap2.add_argument("--no-forget", action="store_true", help="never forget indexed files, even when deleted from disk -- makes it impossible to ever upload the same file twice")
-    ap2.add_argument("--xdev", action="store_true", help="do not descend into other filesystems (symlink or bind-mount to another HDD, ...)")
-    ap2.add_argument("--xvol", action="store_true", help="skip symlinks leaving the volume root")
+    ap2.add_argument("--no-forget", action="store_true", help="never forget indexed files, even when deleted from disk -- makes it impossible to ever upload the same file twice (volflag=noforget)")
+    ap2.add_argument("--xlink", action="store_true", help="on upload: check all volumes for dupes, not just the target volume (volflag=xlink)")
+    ap2.add_argument("--xdev", action="store_true", help="do not descend into other filesystems (symlink or bind-mount to another HDD, ...) (volflag=xdev)")
+    ap2.add_argument("--xvol", action="store_true", help="skip symlinks leaving the volume root (volflag=xvol)")
     ap2.add_argument("--hash-mt", metavar="CORES", type=int, default=hcores, help="num cpu cores to use for file hashing; set 0 or 1 for single-core hashing")
-    ap2.add_argument("--re-maxage", metavar="SEC", type=int, default=0, help="disk rescan volume interval, 0=off, can be set per-volume with the 'scan' volflag")
+    ap2.add_argument("--re-maxage", metavar="SEC", type=int, default=0, help="disk rescan volume interval, 0=off (volflag=scan)")
     ap2.add_argument("--db-act", metavar="SEC", type=float, default=10, help="defer any scheduled volume reindexing until SEC seconds after last db write (uploads, renames, ...)")
     ap2.add_argument("--srch-time", metavar="SEC", type=int, default=45, help="search deadline -- terminate searches running for more than SEC seconds")
     ap2.add_argument("--srch-hits", metavar="N", type=int, default=7999, help="max search results to allow clients to fetch; 125 results will be shown initially")
@@ -737,7 +830,7 @@ def run_argparse(argv: list[str], formatter: Any, retry: bool) -> argparse.Names
     ap2.add_argument("--mtag-to", metavar="SEC", type=int, default=60, help="timeout for ffprobe tag-scan")
     ap2.add_argument("--mtag-mt", metavar="CORES", type=int, default=CORES, help="num cpu cores to use for tag scanning")
     ap2.add_argument("--mtag-v", action="store_true", help="verbose tag scanning; print errors from mtp subprocesses and such")
-    ap2.add_argument("--mtag-vv", action="store_true", help="debug mtp settings")
+    ap2.add_argument("--mtag-vv", action="store_true", help="debug mtp settings and mutagen/ffprobe parsers")
     ap2.add_argument("-mtm", metavar="M=t,t,t", type=u, action="append", help="add/replace metadata mapping")
     ap2.add_argument("-mte", metavar="M,M,M", type=u, help="tags to index/display (comma-sep.)",
         default="circle,album,.tn,artist,title,.bpm,key,.dur,.q,.vq,.aq,vc,ac,fmt,res,.fps,ahash,vhash")
@@ -762,6 +855,7 @@ def run_argparse(argv: list[str], formatter: Any, retry: bool) -> argparse.Names
     ap2.add_argument("--no-scandir", action="store_true", help="disable scandir; instead using listdir + stat on each file")
     ap2.add_argument("--no-fastboot", action="store_true", help="wait for up2k indexing before starting the httpd")
     ap2.add_argument("--no-htp", action="store_true", help="disable httpserver threadpool, create threads as-needed instead")
+    ap2.add_argument("--rclone-mdns", action="store_true", help="use mdns-domain instead of server-ip on /?hc")
     ap2.add_argument("--stackmon", metavar="P,S", type=u, help="write stacktrace to Path every S second, for example --stackmon=\033[32m./st/%%Y-%%m/%%d/%%H%%M.xz,60")
     ap2.add_argument("--log-thrs", metavar="SEC", type=float, help="list active threads every SEC")
     ap2.add_argument("--log-fk", metavar="REGEX", type=u, default="", help="log filekey params for files where path matches REGEX; [\033[32m.\033[0m] (a single dot) = all files")
@@ -774,6 +868,19 @@ def run_argparse(argv: list[str], formatter: Any, retry: bool) -> argparse.Names
     for k, h, _ in sects:
         ap2.add_argument("--help-" + k, action="store_true", help=h)
 
+    try:
+        if not retry:
+            raise Exception()
+
+        for x in ap._actions:
+            if not x.help:
+                continue
+
+            a = ["ascii", "replace"]
+            x.help = x.help.encode(*a).decode(*a) + "\033[0m"
+    except:
+        pass
+
     ret = ap.parse_args(args=argv[1:])
     for k, h, t in sects:
         k2 = "help_" + k.replace("-", "_")
@@ -799,7 +906,7 @@ def main(argv: Optional[list[str]] = None) -> None:
         S_VERSION,
         CODENAME,
         S_BUILD_DT,
-        py_desc().replace("[", "\033[1;30m["),
+        py_desc().replace("[", "\033[90m["),
         SQLITE_VER,
         JINJA_VER,
         PYFTPD_VER,
@@ -818,7 +925,13 @@ def main(argv: Optional[list[str]] = None) -> None:
         ensure_cert()
 
     for k, v in zip(argv[1:], argv[2:]):
-        if k == "-c":
+        if k == "-c" and os.path.isfile(v):
+            supp = args_from_cfg(v)
+            argv.extend(supp)
+
+    for k in argv[1:]:
+        v = k[2:]
+        if k.startswith("-c") and v and os.path.isfile(v):
             supp = args_from_cfg(v)
             argv.extend(supp)
 
@@ -834,25 +947,41 @@ def main(argv: Optional[list[str]] = None) -> None:
         argv[idx] = nk
         time.sleep(2)
 
+    da = len(argv) == 1
     try:
-        if len(argv) == 1:
+        if da:
             argv.extend(["--qr"])
             if ANYWIN or not os.geteuid():
                 argv.extend(["-p80,443,3923", "--ign-ebind"])
     except:
         pass
 
+    if da:
+        t = "no arguments provided; will use {}\n"
+        lprint(t.format(" ".join(argv[1:])))
+
+    nc = 1024
+    try:
+        import resource
+
+        _, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
+        if hard > 0:  # -1 == infinite
+            nc = min(nc, hard // 4)
+    except:
+        nc = 512
+
     retry = False
     for fmtr in [RiceFormatter, RiceFormatter, Dodge11874, BasicDodge11874]:
         try:
-            al = run_argparse(argv, fmtr, retry)
+            al = run_argparse(argv, fmtr, retry, nc)
+            break
         except SystemExit:
             raise
         except:
             retry = True
             lprint("\n[ {} ]:\n{}\n".format(fmtr, min_ex()))
 
-    assert al
+    assert al  # type: ignore
     al.E = E  # __init__ is not shared when oxidized
 
     if WINDOWS and not al.keep_qem:
@@ -905,6 +1034,11 @@ def main(argv: Optional[list[str]] = None) -> None:
         if getattr(al, k1):
             setattr(al, k2, True)
 
+    # propagate unplications
+    for k1, k2 in UNPLICATIONS:
+        if getattr(al, k1):
+            setattr(al, k2, False)
+
     al.i = al.i.split(",")
     try:
         if "-" in al.p:
@@ -924,6 +1058,9 @@ def main(argv: Optional[list[str]] = None) -> None:
     if not al.qrs and [k for k in argv if k.startswith("--qr")]:
         al.qr = True
 
+    if al.ihead:
+        al.ihead = [x.lower() for x in al.ihead]
+
     if HAVE_SSL:
         if al.ssl_ver:
             configure_ssl_ver(al)
@@ -939,6 +1076,10 @@ def main(argv: Optional[list[str]] = None) -> None:
             + "  (if you crash with codec errors then that is why)"
         )
 
+    if PY2 and al.smb:
+        print("error: python2 cannot --smb")
+        return
+
     if sys.version_info < (3, 6):
         al.no_scandir = True
 

copyparty/__version__.py

@@ -1,8 +1,8 @@
 # coding: utf-8
 
-VERSION = (1, 4, 4)
-CODENAME = "mostly reliable"
-BUILD_DT = (2022, 10, 9)
+VERSION = (1, 5, 1)
+CODENAME = "babel"
+BUILD_DT = (2022, 12, 3)
 
 S_VERSION = ".".join(map(str, VERSION))
 S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT)

copyparty/authsrv.py

@@ -18,6 +18,7 @@ from .util import (
     IMPLICATIONS,
     META_NOBOTS,
     SQLITE_VER,
+    UNPLICATIONS,
     Pebkac,
     absreal,
     fsenc,
@@ -30,15 +31,12 @@ from .util import (
     unhumanize,
 )
 
-try:
+if True:  # pylint: disable=using-constant-test
     from collections.abc import Iterable
 
-    import typing
     from typing import Any, Generator, Optional, Union
 
     from .util import RootLogger
-except:
-    pass
 
 if TYPE_CHECKING:
     pass
@@ -411,6 +409,7 @@ class VFS(object):
         will_move: bool = False,
         will_del: bool = False,
         will_get: bool = False,
+        err: int = 403,
     ) -> tuple["VFS", str]:
         """returns [vfsnode,fs_remainder] if user has the requested permissions"""
         if ANYWIN:
@@ -432,7 +431,7 @@ class VFS(object):
         ]:
             if req and (uname not in d and "*" not in d) and uname != LEELOO_DALLAS:
                 t = "you don't have {}-access for this location"
-                raise Pebkac(403, t.format(msg))
+                raise Pebkac(err, t.format(msg))
 
         return vn, rem
 
@@ -577,14 +576,21 @@ class VFS(object):
                 yield x
 
     def zipgen(
-        self, vrem: str, flt: set[str], uname: str, dots: bool, scandir: bool
+        self,
+        vrem: str,
+        flt: set[str],
+        uname: str,
+        dots: bool,
+        dirs: bool,
+        scandir: bool,
+        wrap: bool = True,
     ) -> Generator[dict[str, Any], None, None]:
 
         # if multiselect: add all items to archive root
         # if single folder: the folder itself is the top-level item
-        folder = "" if flt else (vrem.split("/")[-1] or "top")
+        folder = "" if flt or not wrap else (vrem.split("/")[-1] or "top")
 
-        g = self.walk(folder, vrem, [], uname, [[True]], dots, scandir, False)
+        g = self.walk(folder, vrem, [], uname, [[True, False]], dots, scandir, False)
         for _, _, vpath, apath, files, rd, vd in g:
             if flt:
                 files = [x for x in files if x[0] in flt]
@@ -618,6 +624,21 @@ class VFS(object):
             for f in [{"vp": v, "ap": a, "st": n[1]} for v, a, n in ret]:
                 yield f
 
+            if not dirs:
+                continue
+
+            ts = int(time.time())
+            st = os.stat_result((16877, -1, -1, 1, 1000, 1000, 8, ts, ts, ts))
+            dnames = [n[0] for n in rd]
+            dstats = [n[1] for n in rd]
+            dnames += list(vd.keys())
+            dstats += [st] * len(vd)
+            vpaths = [vpath + "/" + n for n in dnames] if vpath else dnames
+            apaths = [os.path.join(apath, n) for n in dnames]
+            ret2 = list(zip(vpaths, apaths, dstats))
+            for d in [{"vp": v, "ap": a, "st": n} for v, a, n in ret2]:
+                yield d
+
 
 if WINDOWS:
     re_vol = re.compile(r"^([a-zA-Z]:[\\/][^:]*|[^:]*):([^:]*):(.*)$")
@@ -683,7 +704,8 @@ class AuthSrv(object):
 
     def _parse_config_file(
         self,
-        fd: typing.BinaryIO,
+        fp: str,
+        cfg_lines: list[str],
         acct: dict[str, str],
         daxs: dict[str, AXS],
         mflags: dict[str, dict[str, Any]],
@@ -693,7 +715,8 @@ class AuthSrv(object):
         vol_src = None
         vol_dst = None
         self.line_ctr = 0
-        for ln in [x.decode("utf-8").strip() for x in fd]:
+        expand_config_file(cfg_lines, fp, "")
+        for ln in cfg_lines:
             self.line_ctr += 1
             if not ln and vol_src is not None:
                 vol_src = None
@@ -722,6 +745,9 @@ class AuthSrv(object):
                 if not vol_dst.startswith("/"):
                     raise Exception('invalid mountpoint "{}"'.format(vol_dst))
 
+                if vol_src.startswith("~"):
+                    vol_src = os.path.expanduser(vol_src)
+
                 # cfg files override arguments and previous files
                 vol_src = absreal(vol_src)
                 vol_dst = vol_dst.strip("/")
@@ -738,6 +764,7 @@ class AuthSrv(object):
                 t = "WARNING (config-file): permission flag 'a' is deprecated; please use 'rw' instead"
                 self.log(t, 1)
 
+            assert vol_dst is not None
             self._read_vol_str(lvl, uname, daxs[vol_dst], mflags[vol_dst])
 
     def _read_vol_str(
@@ -846,13 +873,16 @@ class AuthSrv(object):
 
         if self.args.c:
             for cfg_fn in self.args.c:
-                with open(cfg_fn, "rb") as f:
-                    try:
-                        self._parse_config_file(f, acct, daxs, mflags, mount)
-                    except:
-                        t = "\n\033[1;31m\nerror in config file {} on line {}:\n\033[0m"
-                        self.log(t.format(cfg_fn, self.line_ctr), 1)
-                        raise
+                lns: list[str] = []
+                try:
+                    self._parse_config_file(cfg_fn, lns, acct, daxs, mflags, mount)
+                except:
+                    lns = lns[: self.line_ctr]
+                    slns = ["{:4}: {}".format(n, s) for n, s in enumerate(lns, 1)]
+                    t = "\033[1;31m\nerror @ line {}, included from {}\033[0m"
+                    t = t.format(self.line_ctr, cfg_fn)
+                    self.log("\n{0}\n{1}{0}".format(t, "\n".join(slns)))
+                    raise
 
         # case-insensitive; normalize
         if WINDOWS:
@@ -887,6 +917,7 @@ class AuthSrv(object):
             zv.flags = mflags[dst]
             zv.dbv = None
 
+        assert vfs
         vfs.all_vols = {}
         vfs.get_all_vols(vfs.all_vols)
 
@@ -1088,7 +1119,12 @@ class AuthSrv(object):
                 if getattr(self.args, k):
                     vol.flags[k] = True
 
-            for ga, vf in [["no_forget", "noforget"], ["magic", "magic"]]:
+            for ga, vf in (
+                ("no_forget", "noforget"),
+                ("no_dupe", "nodupe"),
+                ("magic", "magic"),
+                ("xlink", "xlink"),
+            ):
                 if getattr(self.args, ga):
                     vol.flags[vf] = True
 
@@ -1096,6 +1132,10 @@ class AuthSrv(object):
                 if k1 in vol.flags:
                     vol.flags[k2] = True
 
+            for k1, k2 in UNPLICATIONS:
+                if k1 in vol.flags:
+                    vol.flags[k2] = False
+
             # default tag cfgs if unset
             if "mte" not in vol.flags:
                 vol.flags["mte"] = self.args.mte
@@ -1192,6 +1232,18 @@ class AuthSrv(object):
                 self.log(t.format(mtp), 1)
                 errors = True
 
+        have_daw = False
+        for vol in vfs.all_vols.values():
+            daw = vol.flags.get("daw") or self.args.daw
+            if daw:
+                vol.flags["daw"] = True
+                have_daw = True
+
+        if have_daw and self.args.no_dav:
+            t = 'volume "/{}" has volflag "daw" (webdav write-access), but --no-dav is set'
+            self.log(t, 1)
+            errors = True
+
         if errors:
             sys.exit(1)
 
@@ -1335,7 +1387,7 @@ class AuthSrv(object):
                     "",
                     [],
                     u,
-                    [[True]],
+                    [[True, False]],
                     True,
                     not self.args.no_scandir,
                     False,
@@ -1379,3 +1431,33 @@ class AuthSrv(object):
 
         if not flag_r:
             sys.exit(0)
+
+
+def expand_config_file(ret: list[str], fp: str, ipath: str) -> None:
+    """expand all % file includes"""
+    fp = absreal(fp)
+    ipath += " -> " + fp
+    ret.append("#\033[36m opening cfg file{}\033[0m".format(ipath))
+    if len(ipath.split(" -> ")) > 64:
+        raise Exception("hit max depth of 64 includes")
+
+    if os.path.isdir(fp):
+        for fn in sorted(os.listdir(fp)):
+            fp2 = os.path.join(fp, fn)
+            if not os.path.isfile(fp2):
+                continue  # dont recurse
+
+            expand_config_file(ret, fp2, ipath)
+        return
+
+    with open(fp, "rb") as f:
+        for ln in [x.decode("utf-8").strip() for x in f]:
+            if ln.startswith("% "):
+                fp2 = ln[1:].strip()
+                fp2 = os.path.join(os.path.dirname(fp), fp2)
+                expand_config_file(ret, fp2, ipath)
+                continue
+
+            ret.append(ln)
+
+    ret.append("#\033[36m closed{}\033[0m".format(ipath))

copyparty/bos/bos.py

@@ -4,14 +4,13 @@ from __future__ import print_function, unicode_literals
 import os
 
 from ..util import SYMTIME, fsdec, fsenc
-from . import path
+from . import path as path
 
-try:
-    from typing import Optional
-except:
-    pass
+if True:  # pylint: disable=using-constant-test
+    from typing import Any, Optional
 
 _ = (path,)
+__all__ = ["path"]
 
 # grep -hRiE '(^|[^a-zA-Z_\.-])os\.' . | gsed -r 's/ /\n/g;s/\(/(\n/g' | grep -hRiE '(^|[^a-zA-Z_\.-])os\.' | sort | uniq -c
 # printf 'os\.(%s)' "$(grep ^def bos/__init__.py | gsed -r 's/^def //;s/\(.*//' | tr '\n' '|' | gsed -r 's/.$//')"
@@ -38,6 +37,10 @@ def mkdir(p: str, mode: int = 0o755) -> None:
     return os.mkdir(fsenc(p), mode)
 
 
+def open(p: str, *a, **ka) -> int:
+    return os.open(fsenc(p), *a, **ka)
+
+
 def rename(src: str, dst: str) -> None:
     return os.rename(fsenc(src), fsenc(dst))
 

copyparty/broker_mp.py

@@ -9,15 +9,13 @@ import queue
 from .__init__ import CORES, TYPE_CHECKING
 from .broker_mpw import MpWorker
 from .broker_util import try_exec
-from .util import mp
+from .util import Daemon, mp
 
 if TYPE_CHECKING:
     from .svchub import SvcHub
 
-try:
+if True:  # pylint: disable=using-constant-test
     from typing import Any
-except:
-    pass
 
 
 class MProcess(mp.Process):
@@ -51,13 +49,7 @@ class BrokerMp(object):
             q_yield: queue.Queue[tuple[int, str, list[Any]]] = mp.Queue(64)
 
             proc = MProcess(q_pend, q_yield, MpWorker, (q_pend, q_yield, self.args, n))
-
-            thr = threading.Thread(
-                target=self.collector, args=(proc,), name="mp-sink-{}".format(n)
-            )
-            thr.daemon = True
-            thr.start()
-
+            Daemon(self.collector, "mp-sink-{}".format(n), (proc,))
             self.procs.append(proc)
             proc.start()
 
@@ -121,6 +113,10 @@ class BrokerMp(object):
             for p in self.procs:
                 p.q_pend.put((0, dest, [args[0], len(self.procs)]))
 
+        elif dest == "set_netdevs":
+            for p in self.procs:
+                p.q_pend.put((0, dest, list(args)))
+
         elif dest == "cb_httpsrv_up":
             self.hub.cb_httpsrv_up()
 

copyparty/broker_mpw.py

@@ -13,14 +13,12 @@ from .__init__ import ANYWIN
 from .authsrv import AuthSrv
 from .broker_util import BrokerCli, ExceptionalQueue
 from .httpsrv import HttpSrv
-from .util import FAKE_MP, HMaccas
+from .util import FAKE_MP, Daemon, HMaccas
 
-try:
+if True:  # pylint: disable=using-constant-test
     from types import FrameType
 
     from typing import Any, Optional, Union
-except:
-    pass
 
 
 class MpWorker(BrokerCli):
@@ -65,10 +63,7 @@ class MpWorker(BrokerCli):
 
         # on winxp and some other platforms,
         # use thr.join() to block all signals
-        thr = threading.Thread(target=self.main, name="mpw-main")
-        thr.daemon = True
-        thr.start()
-        thr.join()
+        Daemon(self.main, "mpw-main").join()
 
     def signal_handler(self, sig: Optional[int], frame: Optional[FrameType]) -> None:
         # print('k')
@@ -102,6 +97,9 @@ class MpWorker(BrokerCli):
             elif dest == "listen":
                 self.httpsrv.listen(args[0], args[1])
 
+            elif dest == "set_netdevs":
+                self.httpsrv.set_netdevs(args[0])
+
             elif dest == "retq":
                 # response from previous ipc call
                 with self.retpend_mutex:

copyparty/broker_thr.py

@@ -12,10 +12,8 @@ from .util import HMaccas
 if TYPE_CHECKING:
     from .svchub import SvcHub
 
-try:
+if True:  # pylint: disable=using-constant-test
     from typing import Any
-except:
-    pass
 
 
 class BrokerThr(BrokerCli):
@@ -63,6 +61,10 @@ class BrokerThr(BrokerCli):
             self.httpsrv.listen(args[0], 1)
             return
 
+        if dest == "set_netdevs":
+            self.httpsrv.set_netdevs(args[0])
+            return
+
         # new ipc invoking managed service in hub
         obj = self.hub
         for node in dest.split("."):

copyparty/broker_util.py

@@ -10,12 +10,10 @@ from .__init__ import TYPE_CHECKING
 from .authsrv import AuthSrv
 from .util import HMaccas, Pebkac
 
-try:
+if True:  # pylint: disable=using-constant-test
     from typing import Any, Optional, Union
 
     from .util import RootLogger
-except:
-    pass
 
 if TYPE_CHECKING:
     from .httpsrv import HttpSrv
@@ -41,12 +39,14 @@ class BrokerCli(object):
     for example resolving httpconn.* in httpcli -- see lines tagged #mypy404
     """
 
+    log: "RootLogger"
+    args: argparse.Namespace
+    asrv: AuthSrv
+    httpsrv: "HttpSrv"
+    iphash: HMaccas
+
     def __init__(self) -> None:
-        self.log: "RootLogger" = None
-        self.args: argparse.Namespace = None
-        self.asrv: AuthSrv = None
-        self.httpsrv: "HttpSrv" = None
-        self.iphash: HMaccas = None
+        pass
 
     def ask(self, dest: str, *args: Any) -> ExceptionalQueue:
         return ExceptionalQueue(1)

copyparty/dxml.py

@@ -0,0 +1,72 @@
+import importlib
+import sys
+import xml.etree.ElementTree as ET
+
+from .__init__ import PY2
+
+if True:  # pylint: disable=using-constant-test
+    from typing import Any, Optional
+
+
+def get_ET() -> ET.XMLParser:
+    pn = "xml.etree.ElementTree"
+    cn = "_elementtree"
+
+    cmod = sys.modules.pop(cn, None)
+    if not cmod:
+        return ET.XMLParser  # type: ignore
+
+    pmod = sys.modules.pop(pn)
+    sys.modules[cn] = None  # type: ignore
+
+    ret = importlib.import_module(pn)
+    for name, mod in ((pn, pmod), (cn, cmod)):
+        if mod:
+            sys.modules[name] = mod
+        else:
+            sys.modules.pop(name, None)
+
+    sys.modules["xml.etree"].ElementTree = pmod  # type: ignore
+    ret.ParseError = ET.ParseError  # type: ignore
+    return ret.XMLParser  # type: ignore
+
+
+XMLParser: ET.XMLParser = get_ET()
+
+
+class DXMLParser(XMLParser):  # type: ignore
+    def __init__(self) -> None:
+        tb = ET.TreeBuilder()
+        super(DXMLParser, self).__init__(target=tb)
+
+        p = self._parser if PY2 else self.parser
+        p.StartDoctypeDeclHandler = self.nope
+        p.EntityDeclHandler = self.nope
+        p.UnparsedEntityDeclHandler = self.nope
+        p.ExternalEntityRefHandler = self.nope
+
+    def nope(self, *a: Any, **ka: Any) -> None:
+        raise BadXML("{}, {}".format(a, ka))
+
+
+class BadXML(Exception):
+    pass
+
+
+def parse_xml(txt: str) -> ET.Element:
+    parser = DXMLParser()
+    parser.feed(txt)
+    return parser.close()  # type: ignore
+
+
+def mktnod(name: str, text: str) -> ET.Element:
+    el = ET.Element(name)
+    el.text = text
+    return el
+
+
+def mkenod(name: str, sub_el: Optional[ET.Element] = None) -> ET.Element:
+    el = ET.Element(name)
+    if sub_el is not None:
+        el.append(sub_el)
+    return el

copyparty/fsutil.py

@@ -10,12 +10,10 @@ from .authsrv import AXS, VFS
 from .bos import bos
 from .util import chkcmd, min_ex
 
-try:
+if True:  # pylint: disable=using-constant-test
     from typing import Optional, Union
 
     from .util import RootLogger
-except:
-    pass
 
 
 class Fstab(object):
@@ -28,7 +26,7 @@ class Fstab(object):
         self.age = 0.0
 
     def log(self, msg: str, c: Union[int, str] = 0) -> None:
-        self.log_func("fstab", msg + "\033[K", c)
+        self.log_func("fstab", msg, c)
 
     def get(self, path: str) -> str:
         if len(self.cache) > 9000:

copyparty/ftpd.py

@@ -6,18 +6,16 @@ import logging
 import os
 import stat
 import sys
-import threading
 import time
 
 from pyftpdlib.authorizers import AuthenticationFailed, DummyAuthorizer
 from pyftpdlib.filesystems import AbstractedFS, FilesystemError
 from pyftpdlib.handlers import FTPHandler
-from pyftpdlib.log import config_logging
 from pyftpdlib.servers import FTPServer
 
 from .__init__ import PY2, TYPE_CHECKING, E
 from .bos import bos
-from .util import Pebkac, exclude_dotfiles, fsenc
+from .util import Daemon, Pebkac, exclude_dotfiles, fsenc, ipnorm
 
 try:
     from pyftpdlib.ioloop import IOLoop
@@ -31,11 +29,9 @@ except ImportError:
 if TYPE_CHECKING:
     from .svchub import SvcHub
 
-try:
+if True:  # pylint: disable=using-constant-test
     import typing
     from typing import Any, Optional
-except:
-    pass
 
 
 class FtpAuth(DummyAuthorizer):
@@ -46,21 +42,40 @@ class FtpAuth(DummyAuthorizer):
     def validate_authentication(
         self, username: str, password: str, handler: Any
     ) -> None:
+        handler.username = "{}:{}".format(username, password)
+
+        ip = handler.addr[0]
+        if ip.startswith("::ffff:"):
+            ip = ip[7:]
+
+        ip = ipnorm(ip)
+        bans = self.hub.bans
+        if ip in bans:
+            rt = bans[ip] - time.time()
+            if rt < 0:
+                logging.info("client unbanned")
+                del bans[ip]
+            else:
+                raise AuthenticationFailed("banned")
+
         asrv = self.hub.asrv
         if username == "anonymous":
-            password = ""
+            uname = "*"
+        else:
+            uname = asrv.iacct.get(password, "") or asrv.iacct.get(username, "") or "*"
 
-        uname = "*"
-        if password:
-            uname = asrv.iacct.get(password, "")
+        if not uname or not (asrv.vfs.aread.get(uname) or asrv.vfs.awrite.get(uname)):
+            g = self.hub.gpwd
+            if g.lim:
+                bonk, ip = g.bonk(ip, handler.username)
+                if bonk:
+                    logging.warning("client banned: invalid passwords")
+                    bans[ip] = bonk
+
+            raise AuthenticationFailed("Authentication failed.")
 
         handler.username = uname
 
-        if (password and not uname) or not (
-            asrv.vfs.aread.get(uname) or asrv.vfs.awrite.get(uname)
-        ):
-            raise AuthenticationFailed("Authentication failed.")
-
     def get_home_dir(self, username: str) -> str:
         return "/"
 
@@ -148,14 +163,28 @@ class FtpFs(AbstractedFS):
         w = "w" in mode or "a" in mode or "+" in mode
 
         ap = self.rv2a(filename, r, w)
-        if w and bos.path.exists(ap):
-            raise FilesystemError("cannot open existing file for writing")
+        if w:
+            try:
+                st = bos.stat(ap)
+                td = time.time() - st.st_mtime
+            except:
+                td = 0
+
+            if td < -1 or td > self.args.ftp_wt:
+                raise FilesystemError("cannot open existing file for writing")
 
         self.validpath(ap)
         return open(fsenc(ap), mode)
 
     def chdir(self, path: str) -> None:
-        self.cwd = join(self.cwd, path)
+        nwd = join(self.cwd, path)
+        vfs, rem = self.hub.asrv.vfs.get(nwd, self.uname, False, False)
+        ap = vfs.canonical(rem)
+        if not bos.path.isdir(ap):
+            # returning 550 is library-default and suitable
+            raise FilesystemError("Failed to change directory")
+
+        self.cwd = nwd
         (
             self.can_read,
             self.can_write,
@@ -175,7 +204,10 @@ class FtpFs(AbstractedFS):
             vfs, rem = self.hub.asrv.vfs.get(vpath, self.uname, True, False)
 
             fsroot, vfs_ls1, vfs_virt = vfs.ls(
-                rem, self.uname, not self.args.no_scandir, [[True], [False, True]]
+                rem,
+                self.uname,
+                not self.args.no_scandir,
+                [[True, False], [False, True]],
             )
             vfs_ls = [x[0] for x in vfs_ls1]
             vfs_ls.extend(vfs_virt.keys())
@@ -244,11 +276,14 @@ class FtpFs(AbstractedFS):
 
     def lstat(self, path: str) -> os.stat_result:
         ap = self.rv2a(path)
-        return bos.lstat(ap)
+        return bos.stat(ap)
 
     def isfile(self, path: str) -> bool:
-        st = self.stat(path)
-        return stat.S_ISREG(st.st_mode)
+        try:
+            st = self.stat(path)
+            return stat.S_ISREG(st.st_mode)
+        except:
+            return False  # expected for mojibake in ftp_SIZE()
 
     def islink(self, path: str) -> bool:
         ap = self.rv2a(path)
@@ -285,8 +320,8 @@ class FtpFs(AbstractedFS):
 
 class FtpHandler(FTPHandler):
     abstracted_fs = FtpFs
-    hub: "SvcHub" = None
-    args: argparse.Namespace = None
+    hub: "SvcHub"
+    args: argparse.Namespace
 
     def __init__(self, conn: Any, server: Any, ioloop: Any = None) -> None:
         self.hub: "SvcHub" = FtpHandler.hub
@@ -300,6 +335,9 @@ class FtpHandler(FTPHandler):
         # abspath->vpath mapping to resolve log_transfer paths
         self.vfs_map: dict[str, str] = {}
 
+        # reduce non-debug logging
+        self.log_cmds_list = [x for x in self.log_cmds_list if x not in ("CWD", "XCWD")]
+
     def ftp_STOR(self, file: str, mode: str = "w") -> Any:
         # Optional[str]
         vp = join(self.fs.cwd, file).lstrip("/")
@@ -394,17 +432,15 @@ class Ftpd(object):
             if self.args.ftp_nat:
                 h2.masquerade_address = self.args.ftp_nat
 
-        if self.args.ftp_dbg:
-            config_logging(level=logging.DEBUG)
+        lgr = logging.getLogger("pyftpdlib")
+        lgr.setLevel(logging.DEBUG if self.args.ftpv else logging.INFO)
 
         ioloop = IOLoop()
         for ip in self.args.i:
             for h, lp in hs:
                 FTPServer((ip, int(lp)), h, ioloop)
 
-        thr = threading.Thread(target=ioloop.loop, name="ftp")
-        thr.daemon = True
-        thr.start()
+        Daemon(ioloop.loop, "ftp")
 
 
 def join(p1: str, p2: str) -> str:

copyparty/httpconn.py

@@ -25,15 +25,16 @@ from .th_srv import HAVE_PIL, HAVE_VIPS
 from .u2idx import U2idx
 from .util import HMaccas, shut_socket
 
-try:
+if True:  # pylint: disable=using-constant-test
     from typing import Optional, Pattern, Union
-except:
-    pass
 
 if TYPE_CHECKING:
     from .httpsrv import HttpSrv
 
 
+PTN_HTTP = re.compile(br"[A-Z]{3}[A-Z ]")
+
+
 class HttpConn(object):
     """
     spawned by HttpSrv to handle an incoming client connection,
@@ -45,6 +46,7 @@ class HttpConn(object):
     ) -> None:
         self.s = sck
         self.sr: Optional[Util._Unrecv] = None
+        self.cli: Optional[HttpCli] = None
         self.addr = addr
         self.hsrv = hsrv
 
@@ -55,6 +57,8 @@ class HttpConn(object):
         self.cert_path = hsrv.cert_path
         self.u2fh: Util.FHC = hsrv.u2fh  # mypy404
         self.iphash: HMaccas = hsrv.broker.iphash
+        self.bans: dict[str, int] = hsrv.bans
+        self.aclose: dict[str, int] = hsrv.aclose
 
         enth = (HAVE_PIL or HAVE_VIPS or HAVE_FFMPEG) and not self.args.no_thumb
         self.thumbcli: Optional[ThumbCli] = ThumbCli(hsrv) if enth else None  # mypy404
@@ -62,7 +66,7 @@ class HttpConn(object):
 
         self.t0: float = time.time()  # mypy404
         self.stopping = False
-        self.nreq: int = 0  # mypy404
+        self.nreq: int = -1  # mypy404
         self.nbyte: int = 0  # mypy404
         self.u2idx: Optional[U2idx] = None
         self.log_func: "Util.RootLogger" = hsrv.log  # mypy404
@@ -134,9 +138,11 @@ class HttpConn(object):
                 self.s.send(b"HTTP/1.1 400 Bad Request\r\n\r\n" + err.encode("utf-8"))
                 return False
 
-        return method not in [None, b"GET ", b"HEAD", b"POST", b"PUT ", b"OPTI"]
+        return not method or not bool(PTN_HTTP.match(method))
 
     def run(self) -> None:
+        self.s.settimeout(10)
+
         self.sr = None
         if self.args.https_only:
             is_https = True
@@ -205,6 +211,6 @@ class HttpConn(object):
 
         while not self.stopping:
             self.nreq += 1
-            cli = HttpCli(self)
-            if not cli.run():
+            self.cli = HttpCli(self)
+            if not self.cli.run():
                 return

copyparty/httpsrv.py

@@ -32,9 +32,14 @@ from .__init__ import MACOS, TYPE_CHECKING, EnvParams
 from .bos import bos
 from .httpconn import HttpConn
 from .util import (
+    E_SCK,
     FHC,
+    Daemon,
     Garda,
     Magician,
+    Netdev,
+    NetMap,
+    ipnorm,
     min_ex,
     shut_socket,
     spack,
@@ -44,11 +49,10 @@ from .util import (
 
 if TYPE_CHECKING:
     from .broker_util import BrokerCli
+    from .ssdp import SSDPr
 
-try:
+if True:  # pylint: disable=using-constant-test
     from typing import Any, Optional
-except:
-    pass
 
 
 class HttpSrv(object):
@@ -70,10 +74,15 @@ class HttpSrv(object):
 
         nsuf = "-n{}-i{:x}".format(nid, os.getpid()) if nid else ""
         self.magician = Magician()
-        self.bans: dict[str, int] = {}
+        self.nm = NetMap([], {})
+        self.ssdp: Optional["SSDPr"] = None
         self.gpwd = Garda(self.args.ban_pw)
         self.g404 = Garda(self.args.ban_404)
+        self.bans: dict[str, int] = {}
+        self.aclose: dict[str, int] = {}
 
+        self.ip = ""
+        self.port = 0
         self.name = "hsrv" + nsuf
         self.mutex = threading.Lock()
         self.stopping = False
@@ -96,13 +105,16 @@ class HttpSrv(object):
 
         env = jinja2.Environment()
         env.loader = jinja2.FileSystemLoader(os.path.join(self.E.mod, "web"))
-        self.j2 = {
-            x: env.get_template(x + ".html")
-            for x in ["splash", "browser", "browser2", "msg", "md", "mde", "cf"]
-        }
+        jn = ["splash", "svcs", "browser", "browser2", "msg", "md", "mde", "cf"]
+        self.j2 = {x: env.get_template(x + ".html") for x in jn}
         zs = os.path.join(self.E.mod, "web", "deps", "prism.js.gz")
         self.prism = os.path.exists(zs)
 
+        if self.args.zs:
+            from .ssdp import SSDPr
+
+            self.ssdp = SSDPr(broker)
+
         cert_path = os.path.join(self.E.cfg, "cert.pem")
         if bos.path.exists(cert_path):
             self.cert_path = cert_path
@@ -120,9 +132,7 @@ class HttpSrv(object):
                 start_log_thrs(self.log, self.args.log_thrs, nid)
 
         self.th_cfg: dict[str, Any] = {}
-        t = threading.Thread(target=self.post_init, name="hsrv-init2")
-        t.daemon = True
-        t.start()
+        Daemon(self.post_init, "hsrv-init2")
 
     def post_init(self) -> None:
         try:
@@ -131,18 +141,16 @@ class HttpSrv(object):
         except:
             pass
 
+    def set_netdevs(self, netdevs: dict[str, Netdev]) -> None:
+        self.nm = NetMap([self.ip], netdevs)
+
     def start_threads(self, n: int) -> None:
         self.tp_nthr += n
         if self.args.log_htp:
             self.log(self.name, "workers += {} = {}".format(n, self.tp_nthr), 6)
 
         for _ in range(n):
-            thr = threading.Thread(
-                target=self.thr_poolw,
-                name=self.name + "-poolw",
-            )
-            thr.daemon = True
-            thr.start()
+            Daemon(self.thr_poolw, self.name + "-poolw")
 
     def stop_threads(self, n: int) -> None:
         self.tp_nthr -= n
@@ -170,26 +178,29 @@ class HttpSrv(object):
     def listen(self, sck: socket.socket, nlisteners: int) -> None:
         if self.args.j != 1:
             # lost in the pickle; redefine
+            try:
+                sck.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
+            except:
+                pass
             sck.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
             sck.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
             sck.settimeout(None)  # < does not inherit, ^ does
 
-        ip, port = sck.getsockname()
+        self.ip, self.port = sck.getsockname()[:2]
         self.srvs.append(sck)
         self.nclimax = math.ceil(self.args.nc * 1.0 / nlisteners)
-        t = threading.Thread(
-            target=self.thr_listen,
-            args=(sck,),
-            name="httpsrv-n{}-listen-{}-{}".format(self.nid or "0", ip, port),
+        Daemon(
+            self.thr_listen,
+            "httpsrv-n{}-listen-{}-{}".format(self.nid or "0", self.ip, self.port),
+            (sck,),
         )
-        t.daemon = True
-        t.start()
 
     def thr_listen(self, srv_sck: socket.socket) -> None:
         """listens on a shared tcp server"""
-        ip, port = srv_sck.getsockname()
+        ip, port = srv_sck.getsockname()[:2]
         fno = srv_sck.fileno()
-        msg = "subscribed @ {}:{}  f{} p{}".format(ip, port, fno, os.getpid())
+        hip = "[{}]".format(ip) if ":" in ip else ip
+        msg = "subscribed @ {}:{}  f{} p{}".format(hip, port, fno, os.getpid())
         self.log(self.name, msg)
 
         def fun() -> None:
@@ -199,19 +210,80 @@ class HttpSrv(object):
 
         while not self.stopping:
             if self.args.log_conn:
-                self.log(self.name, "|%sC-ncli" % ("-" * 1,), c="1;30")
+                self.log(self.name, "|%sC-ncli" % ("-" * 1,), c="90")
 
-            if self.ncli >= self.nclimax:
-                self.log(self.name, "at connection limit; waiting", 3)
-                while self.ncli >= self.nclimax:
-                    time.sleep(0.1)
+            spins = 0
+            while self.ncli >= self.nclimax:
+                if not spins:
+                    self.log(self.name, "at connection limit; waiting", 3)
+
+                spins += 1
+                time.sleep(0.1)
+                if spins != 50 or not self.args.aclose:
+                    continue
+
+                ipfreq: dict[str, int] = {}
+                with self.mutex:
+                    for c in self.clients:
+                        ip = ipnorm(c.ip)
+                        try:
+                            ipfreq[ip] += 1
+                        except:
+                            ipfreq[ip] = 1
+
+                ip, n = sorted(ipfreq.items(), key=lambda x: x[1], reverse=True)[0]
+                if n < self.nclimax / 2:
+                    continue
+
+                self.aclose[ip] = int(time.time() + self.args.aclose * 60)
+                nclose = 0
+                nloris = 0
+                nconn = 0
+                with self.mutex:
+                    for c in self.clients:
+                        cip = ipnorm(c.ip)
+                        if ip != cip:
+                            continue
+
+                        nconn += 1
+                        try:
+                            if (
+                                c.nreq >= 1
+                                or not c.cli
+                                or c.cli.in_hdr_recv
+                                or c.cli.keepalive
+                            ):
+                                Daemon(c.shutdown)
+                                nclose += 1
+                                if c.nreq <= 0 and (not c.cli or c.cli.in_hdr_recv):
+                                    nloris += 1
+                        except:
+                            pass
+
+                t = "{} downgraded to connection:close for {} min; dropped {}/{} connections"
+                self.log(self.name, t.format(ip, self.args.aclose, nclose, nconn), 1)
+
+                if nloris < nconn / 2:
+                    continue
+
+                t = "slowloris (idle-conn): {} banned for {} min"
+                self.log(self.name, t.format(ip, self.args.loris, nclose), 1)
+                self.bans[ip] = int(time.time() + self.args.loris * 60)
 
             if self.args.log_conn:
-                self.log(self.name, "|%sC-acc1" % ("-" * 2,), c="1;30")
+                self.log(self.name, "|%sC-acc1" % ("-" * 2,), c="90")
 
             try:
-                sck, addr = srv_sck.accept()
+                sck, saddr = srv_sck.accept()
+                cip, cport = saddr[:2]
+                if cip.startswith("::ffff:"):
+                    cip = cip[7:]
+
+                addr = (cip, cport)
             except (OSError, socket.error) as ex:
+                if self.stopping:
+                    break
+
                 self.log(self.name, "accept({}): {}".format(fno, ex), c=6)
                 time.sleep(0.02)
                 continue
@@ -220,7 +292,7 @@ class HttpSrv(object):
                 t = "|{}C-acc2 \033[0;36m{} \033[3{}m{}".format(
                     "-" * 3, ip, port % 8, port
                 )
-                self.log("%s %s" % addr, t, c="1;30")
+                self.log("%s %s" % addr, t, c="90")
 
             self.accept(sck, addr)
 
@@ -241,10 +313,7 @@ class HttpSrv(object):
                 if self.nid:
                     name += "-{}".format(self.nid)
 
-                thr = threading.Thread(target=self.periodic, name=name)
-                self.t_periodic = thr
-                thr.daemon = True
-                thr.start()
+                self.t_periodic = Daemon(self.periodic, name)
 
             if self.tp_q:
                 self.tp_time = self.tp_time or now
@@ -259,13 +328,11 @@ class HttpSrv(object):
             t = "looks like the httpserver threadpool died; please make an issue on github and tell me the story of how you pulled that off, thanks and dog bless\n"
             self.log(self.name, t, 1)
 
-        thr = threading.Thread(
-            target=self.thr_client,
-            args=(sck, addr),
-            name="httpconn-{}-{}".format(addr[0].split(".", 2)[-1][-6:], addr[1]),
+        Daemon(
+            self.thr_client,
+            "httpconn-{}-{}".format(addr[0].split(".", 2)[-1][-6:], addr[1]),
+            (sck, addr),
         )
-        thr.daemon = True
-        thr.start()
 
     def thr_poolw(self) -> None:
         assert self.tp_q
@@ -324,15 +391,16 @@ class HttpSrv(object):
         with self.mutex:
             self.clients.add(cli)
 
+        # print("{}\n".format(len(self.clients)), end="")
         fno = sck.fileno()
         try:
             if self.args.log_conn:
-                self.log("%s %s" % addr, "|%sC-crun" % ("-" * 4,), c="1;30")
+                self.log("%s %s" % addr, "|%sC-crun" % ("-" * 4,), c="90")
 
             cli.run()
 
         except (OSError, socket.error) as ex:
-            if ex.errno not in [10038, 10054, 107, 57, 49, 9]:
+            if ex.errno not in E_SCK:
                 self.log(
                     "%s %s" % addr,
                     "run({}): {}".format(fno, ex),
@@ -342,7 +410,7 @@ class HttpSrv(object):
         finally:
             sck = cli.s
             if self.args.log_conn:
-                self.log("%s %s" % addr, "|%sC-cdone" % ("-" * 5,), c="1;30")
+                self.log("%s %s" % addr, "|%sC-cdone" % ("-" * 5,), c="90")
 
             try:
                 fno = sck.fileno()
@@ -352,15 +420,9 @@ class HttpSrv(object):
                     self.log(
                         "%s %s" % addr,
                         "shut({}): {}".format(fno, ex),
-                        c="1;30",
+                        c="90",
                     )
-                if ex.errno not in [10038, 10054, 107, 57, 49, 9]:
-                    # 10038 No longer considered a socket
-                    # 10054 Foribly closed by remote
-                    #   107 Transport endpoint not connected
-                    #    57 Socket is not connected
-                    #    49 Can't assign requested address (wifi down)
-                    #     9 Bad file descriptor
+                if ex.errno not in E_SCK:
                     raise
             finally:
                 with self.mutex:

copyparty/mdns.py

@@ -0,0 +1,507 @@
+# coding: utf-8
+from __future__ import print_function, unicode_literals
+
+import random
+import select
+import socket
+import time
+
+from ipaddress import IPv4Network, IPv6Network
+
+from .__init__ import TYPE_CHECKING
+from .__init__ import unicode as U
+from .multicast import MC_Sck, MCast
+from .stolen.dnslib import CLASS as DC
+from .stolen.dnslib import (
+    NSEC,
+    PTR,
+    QTYPE,
+    RR,
+    SRV,
+    TXT,
+    A,
+    AAAA,
+    DNSHeader,
+    DNSQuestion,
+    DNSRecord,
+)
+from .util import CachedSet, Daemon, Netdev, min_ex
+
+if TYPE_CHECKING:
+    from .svchub import SvcHub
+
+if True:  # pylint: disable=using-constant-test
+    from typing import Any, Optional, Union
+
+
+MDNS4 = "224.0.0.251"
+MDNS6 = "ff02::fb"
+
+
+class MDNS_Sck(MC_Sck):
+    def __init__(
+        self,
+        sck: socket.socket,
+        nd: Netdev,
+        grp: str,
+        ip: str,
+        net: Union[IPv4Network, IPv6Network],
+    ):
+        super(MDNS_Sck, self).__init__(sck, nd, grp, ip, net)
+
+        self.bp_probe = b""
+        self.bp_ip = b""
+        self.bp_svc = b""
+        self.bp_bye = b""
+
+        self.last_tx = 0.0
+
+
+class MDNS(MCast):
+    def __init__(self, hub: "SvcHub") -> None:
+        al = hub.args
+        grp4 = "" if al.zm6 else MDNS4
+        grp6 = "" if al.zm4 else MDNS6
+        super(MDNS, self).__init__(
+            hub, MDNS_Sck, al.zm_on, al.zm_off, grp4, grp6, 5353, hub.args.zmv
+        )
+        self.srv: dict[socket.socket, MDNS_Sck] = {}
+
+        self.ttl = 300
+
+        zs = self.args.name + ".local."
+        zs = zs.encode("ascii", "replace").decode("ascii", "replace")
+        self.hn = "-".join(x for x in zs.split("?") if x) or (
+            "vault-{}".format(random.randint(1, 255))
+        )
+        self.lhn = self.hn.lower()
+
+        # requester ip -> (response deadline, srv, body):
+        self.q: dict[str, tuple[float, MDNS_Sck, bytes]] = {}
+        self.rx4 = CachedSet(0.42)  # 3 probes @ 250..500..750 => 500ms span
+        self.rx6 = CachedSet(0.42)
+        self.svcs, self.sfqdns = self.build_svcs()
+        self.lsvcs = {k.lower(): v for k, v in self.svcs.items()}
+        self.lsfqdns = set([x.lower() for x in self.sfqdns])
+
+        self.probing = 0.0
+        self.unsolicited: list[float] = []  # scheduled announces on all nics
+        self.defend: dict[MDNS_Sck, float] = {}  # server -> deadline
+
+    def log(self, msg: str, c: Union[int, str] = 0) -> None:
+        self.log_func("mDNS", msg, c)
+
+    def build_svcs(self) -> tuple[dict[str, dict[str, Any]], set[str]]:
+        zms = self.args.zms
+        http = {"port": 80 if 80 in self.args.p else self.args.p[0]}
+        https = {"port": 443 if 443 in self.args.p else self.args.p[0]}
+        webdav = http.copy()
+        webdavs = https.copy()
+        webdav["u"] = webdavs["u"] = "u"  # KDE requires username
+        ftp = {"port": (self.args.ftp if "f" in zms else self.args.ftps)}
+        smb = {"port": self.args.smb_port}
+
+        # some gvfs require path
+        zs = self.args.zm_ld or "/"
+        if zs:
+            webdav["path"] = zs
+            webdavs["path"] = zs
+
+        if self.args.zm_lh:
+            http["path"] = self.args.zm_lh
+            https["path"] = self.args.zm_lh
+
+        if self.args.zm_lf:
+            ftp["path"] = self.args.zm_lf
+
+        if self.args.zm_ls:
+            smb["path"] = self.args.zm_ls
+
+        svcs: dict[str, dict[str, Any]] = {}
+
+        if "d" in zms:
+            svcs["_webdav._tcp.local."] = webdav
+
+        if "D" in zms:
+            svcs["_webdavs._tcp.local."] = webdavs
+
+        if "h" in zms:
+            svcs["_http._tcp.local."] = http
+
+        if "H" in zms:
+            svcs["_https._tcp.local."] = https
+
+        if "f" in zms.lower():
+            svcs["_ftp._tcp.local."] = ftp
+
+        if "s" in zms.lower():
+            svcs["_smb._tcp.local."] = smb
+
+        sfqdns: set[str] = set()
+        for k, v in svcs.items():
+            name = "{}-c-{}".format(self.args.name, k.split(".")[0][1:])
+            v["name"] = name
+            sfqdns.add("{}.{}".format(name, k))
+
+        return svcs, sfqdns
+
+    def build_replies(self) -> None:
+        for srv in self.srv.values():
+            probe = DNSRecord(DNSHeader(0, 0), q=DNSQuestion(self.hn, QTYPE.ANY))
+            areply = DNSRecord(DNSHeader(0, 0x8400))
+            sreply = DNSRecord(DNSHeader(0, 0x8400))
+            bye = DNSRecord(DNSHeader(0, 0x8400))
+
+            have4 = have6 = False
+            for s2 in self.srv.values():
+                if srv.idx != s2.idx:
+                    continue
+
+                if s2.v6:
+                    have6 = True
+                else:
+                    have4 = True
+
+            for ip in srv.ips:
+                if ":" in ip:
+                    qt = QTYPE.AAAA
+                    ar = {"rclass": DC.F_IN, "rdata": AAAA(ip)}
+                else:
+                    qt = QTYPE.A
+                    ar = {"rclass": DC.F_IN, "rdata": A(ip)}
+
+                r0 = RR(self.hn, qt, ttl=0, **ar)
+                r120 = RR(self.hn, qt, ttl=120, **ar)
+                # rfc-10:
+                #   SHOULD rr ttl 120sec for A/AAAA/SRV
+                #   (and recommend 75min for all others)
+
+                probe.add_auth(r120)
+                areply.add_answer(r120)
+                sreply.add_answer(r120)
+                bye.add_answer(r0)
+
+            for sclass, props in self.svcs.items():
+                sname = props["name"]
+                sport = props["port"]
+                sfqdn = sname + "." + sclass
+
+                k = "_services._dns-sd._udp.local."
+                r = RR(k, QTYPE.PTR, DC.IN, 4500, PTR(sclass))
+                sreply.add_answer(r)
+
+                r = RR(sclass, QTYPE.PTR, DC.IN, 4500, PTR(sfqdn))
+                sreply.add_answer(r)
+
+                r = RR(sfqdn, QTYPE.SRV, DC.F_IN, 120, SRV(0, 0, sport, self.hn))
+                sreply.add_answer(r)
+                areply.add_answer(r)
+
+                r = RR(sfqdn, QTYPE.SRV, DC.F_IN, 0, SRV(0, 0, sport, self.hn))
+                bye.add_answer(r)
+
+                txts = []
+                for k in ("u", "path"):
+                    if k not in props:
+                        continue
+
+                    zb = "{}={}".format(k, props[k]).encode("utf-8")
+                    if len(zb) > 255:
+                        t = "value too long for mdns: [{}]"
+                        raise Exception(t.format(props[k]))
+
+                    txts.append(zb)
+
+                # gvfs really wants txt even if they're empty
+                r = RR(sfqdn, QTYPE.TXT, DC.F_IN, 4500, TXT(txts))
+                sreply.add_answer(r)
+
+            if not (have4 and have6) and not self.args.zm_noneg:
+                ns = NSEC(self.hn, ["AAAA" if have6 else "A"])
+                r = RR(self.hn, QTYPE.NSEC, DC.F_IN, 120, ns)
+                areply.add_ar(r)
+                if len(sreply.pack()) < 1400:
+                    sreply.add_ar(r)
+
+            srv.bp_probe = probe.pack()
+            srv.bp_ip = areply.pack()
+            srv.bp_svc = sreply.pack()
+            srv.bp_bye = bye.pack()
+
+            # since all replies are small enough to fit in one packet,
+            # always send full replies rather than just a/aaaa records
+            srv.bp_ip = srv.bp_svc
+
+    def send_probes(self) -> None:
+        slp = random.random() * 0.25
+        for _ in range(3):
+            time.sleep(slp)
+            slp = 0.25
+            if not self.running:
+                break
+
+            if self.args.zmv:
+                self.log("sending hostname probe...")
+
+            # ipv4: need to probe each ip (each server)
+            # ipv6: only need to probe each set of looped nics
+            probed6: set[str] = set()
+            for srv in self.srv.values():
+                if srv.ip in probed6:
+                    continue
+
+                try:
+                    srv.sck.sendto(srv.bp_probe, (srv.grp, 5353))
+                    if srv.v6:
+                        for ip in srv.ips:
+                            probed6.add(ip)
+                except Exception as ex:
+                    self.log("sendto failed: {} ({})".format(srv.ip, ex), "90")
+
+    def run(self) -> None:
+        try:
+            bound = self.create_servers()
+        except:
+            t = "no server IP matches the mdns config\n{}"
+            self.log(t.format(min_ex()), 1)
+            bound = []
+
+        if not bound:
+            self.log("failed to announce copyparty services on the network", 3)
+            return
+
+        self.build_replies()
+        Daemon(self.send_probes)
+        zf = time.time() + 2
+        self.probing = zf  # cant unicast so give everyone an extra sec
+        self.unsolicited = [zf, zf + 1, zf + 3, zf + 7]  # rfc-8.3
+        last_hop = time.time()
+        ihop = self.args.mc_hop
+        while self.running:
+            timeout = (
+                0.02 + random.random() * 0.07
+                if self.probing or self.q or self.defend or self.unsolicited
+                else (last_hop + ihop if ihop else 180)
+            )
+            rdy = select.select(self.srv, [], [], timeout)
+            rx: list[socket.socket] = rdy[0]  # type: ignore
+            self.rx4.cln()
+            self.rx6.cln()
+            for sck in rx:
+                buf, addr = sck.recvfrom(4096)
+                try:
+                    self.eat(buf, addr, sck)
+                except:
+                    if not self.running:
+                        return
+
+                    t = "{} {} \033[33m|{}| {}\n{}".format(
+                        self.srv[sck].name, addr, len(buf), repr(buf)[2:-1], min_ex()
+                    )
+                    self.log(t, 6)
+
+            if not self.probing:
+                self.process()
+                continue
+
+            if self.probing < time.time():
+                t = "probe ok; announcing [{}]"
+                self.log(t.format(self.hn[:-1]), 2)
+                self.probing = 0
+
+    def stop(self, panic=False) -> None:
+        self.running = False
+        if not panic:
+            for srv in self.srv.values():
+                srv.sck.sendto(srv.bp_bye, (srv.grp, 5353))
+
+        self.srv = {}
+
+    def eat(self, buf: bytes, addr: tuple[str, int], sck: socket.socket) -> None:
+        cip = addr[0]
+        v6 = ":" in cip
+        if cip.startswith("169.254") or v6 and not cip.startswith("fe80"):
+            return
+
+        cache = self.rx6 if v6 else self.rx4
+        if buf in cache.c:
+            return
+
+        cache.add(buf)
+        srv: Optional[MDNS_Sck] = self.srv[sck] if v6 else self.map_client(cip)  # type: ignore
+        if not srv:
+            return
+
+        now = time.time()
+
+        if self.args.zmv and cip != srv.ip and cip not in srv.ips:
+            t = "{} [{}] \033[36m{} \033[0m|{}|"
+            self.log(t.format(srv.name, srv.ip, cip, len(buf)), "90")
+
+        p = DNSRecord.parse(buf)
+        if self.args.zmvv:
+            self.log(str(p))
+
+        # check for incoming probes for our hostname
+        cips = [U(x.rdata) for x in p.auth if U(x.rname).lower() == self.lhn]
+        if cips and self.sips.isdisjoint(cips):
+            if not [x for x in cips if x not in ("::1", "127.0.0.1")]:
+                # avahi broadcasting 127.0.0.1-only packets
+                return
+
+            self.log("someone trying to steal our hostname: {}".format(cips), 3)
+            # immediately unicast
+            if not self.probing:
+                srv.sck.sendto(srv.bp_ip, (cip, 5353))
+
+            # and schedule multicast
+            self.defend[srv] = self.defend.get(srv, now + 0.1)
+            return
+
+        # check for someone rejecting our probe / hijacking our hostname
+        cips = [
+            U(x.rdata)
+            for x in p.rr
+            if U(x.rname).lower() == self.lhn and x.rclass == DC.F_IN
+        ]
+        if cips and self.sips.isdisjoint(cips):
+            if not [x for x in cips if x not in ("::1", "127.0.0.1")]:
+                # avahi broadcasting 127.0.0.1-only packets
+                return
+
+            t = "mdns zeroconf: "
+            if self.probing:
+                t += "Cannot start; hostname '{}' is occupied"
+            else:
+                t += "Emergency stop; hostname '{}' got stolen"
+
+            t += " on {}! Use --name to set another hostname.\n\nName taken by {}\n\nYour IPs: {}\n"
+            self.log(t.format(self.args.name, srv.name, cips, list(self.sips)), 1)
+            self.stop(True)
+            return
+
+        # then rfc-6.7; dns pretending to be mdns (android...)
+        if p.header.id or addr[1] != 5353:
+            rsp: Optional[DNSRecord] = None
+            for r in p.questions:
+                try:
+                    lhn = U(r.qname).lower()
+                except:
+                    self.log("invalid question: {}".format(r))
+                    continue
+
+                if lhn != self.lhn:
+                    continue
+
+                if p.header.id and r.qtype in (QTYPE.A, QTYPE.AAAA):
+                    rsp = rsp or DNSRecord(DNSHeader(p.header.id, 0x8400))
+                    rsp.add_question(r)
+                    for ip in srv.ips:
+                        qt = r.qtype
+                        v6 = ":" in ip
+                        if v6 == (qt == QTYPE.AAAA):
+                            rd = AAAA(ip) if v6 else A(ip)
+                            rr = RR(self.hn, qt, DC.IN, 10, rd)
+                            rsp.add_answer(rr)
+            if rsp:
+                srv.sck.sendto(rsp.pack(), addr[:2])
+                # but don't return in case it's a differently broken client
+
+        # then a/aaaa records
+        for r in p.questions:
+            try:
+                lhn = U(r.qname).lower()
+            except:
+                self.log("invalid question: {}".format(r))
+                continue
+
+            if lhn != self.lhn:
+                continue
+
+            # gvfs keeps repeating itself
+            found = False
+            unicast = False
+            for rr in p.rr:
+                try:
+                    rname = U(rr.rname).lower()
+                except:
+                    self.log("invalid rr: {}".format(rr))
+                    continue
+
+                if rname == self.lhn:
+                    if rr.ttl > 60:
+                        found = True
+                    if rr.rclass == DC.F_IN:
+                        unicast = True
+
+            if unicast:
+                # spec-compliant mDNS-over-unicast
+                srv.sck.sendto(srv.bp_ip, (cip, 5353))
+            elif addr[1] != 5353:
+                # just in case some clients use (and want us to use) invalid ports
+                srv.sck.sendto(srv.bp_ip, addr[:2])
+
+            if not found:
+                self.q[cip] = (0, srv, srv.bp_ip)
+                return
+
+        deadline = now + (0.5 if p.header.tc else 0.02)  # rfc-7.2
+
+        # and service queries
+        for r in p.questions:
+            if not r or not r.qname:
+                continue
+
+            qname = U(r.qname).lower()
+            if qname in self.lsvcs or qname == "_services._dns-sd._udp.local.":
+                self.q[cip] = (deadline, srv, srv.bp_svc)
+                break
+        # heed rfc-7.1 if there was an announce in the past 12sec
+        # (workaround gvfs race-condition where it occasionally
+        #  doesn't read/decode the full response...)
+        if now < srv.last_tx + 12:
+            for rr in p.rr:
+                if not rr.rdata:
+                    continue
+
+                rdata = U(rr.rdata).lower()
+                if rdata in self.lsfqdns:
+                    if rr.ttl > 2250:
+                        self.q.pop(cip, None)
+                    break
+
+    def process(self) -> None:
+        tx = set()
+        now = time.time()
+        cooldown = 0.9  # rfc-6: 1
+        if self.unsolicited and self.unsolicited[0] < now:
+            self.unsolicited.pop(0)
+            cooldown = 0.1
+            for srv in self.srv.values():
+                tx.add(srv)
+
+        for srv, deadline in list(self.defend.items()):
+            if now < deadline:
+                continue
+
+            if self._tx(srv, srv.bp_ip, 0.02):  # rfc-6: 0.25
+                self.defend.pop(srv)
+
+        for cip, (deadline, srv, msg) in list(self.q.items()):
+            if now < deadline:
+                continue
+
+            self.q.pop(cip)
+            self._tx(srv, msg, cooldown)
+
+        for srv in tx:
+            self._tx(srv, srv.bp_svc, cooldown)
+
+    def _tx(self, srv: MDNS_Sck, msg: bytes, cooldown: float) -> bool:
+        now = time.time()
+        if now < srv.last_tx + cooldown:
+            return False
+
+        srv.sck.sendto(msg, (srv.grp, 5353))
+        srv.last_tx = now
+        return True

copyparty/mtag.py

@@ -12,25 +12,23 @@ from .__init__ import PY2, WINDOWS, E, unicode
 from .bos import bos
 from .util import REKOBO_LKEY, fsenc, min_ex, retchk, runcmd, uncyg
 
-try:
+if True:  # pylint: disable=using-constant-test
     from typing import Any, Union
 
     from .util import RootLogger
-except:
-    pass
 
 
-def have_ff(cmd: str) -> bool:
+def have_ff(scmd: str) -> bool:
     if PY2:
-        print("# checking {}".format(cmd))
-        cmd = (cmd + " -version").encode("ascii").split(b" ")
+        print("# checking {}".format(scmd))
+        acmd = (scmd + " -version").encode("ascii").split(b" ")
         try:
-            sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE).communicate()
+            sp.Popen(acmd, stdout=sp.PIPE, stderr=sp.PIPE).communicate()
             return True
         except:
             return False
     else:
-        return bool(shutil.which(cmd))
+        return bool(shutil.which(scmd))
 
 
 HAVE_FFMPEG = have_ff("ffmpeg")
@@ -269,7 +267,7 @@ class MTag(object):
         if self.backend == "mutagen":
             self.get = self.get_mutagen
             try:
-                import mutagen  # noqa: F401  # pylint: disable=unused-import,import-outside-toplevel
+                from mutagen import version  # noqa: F401
             except:
                 self.log("could not load Mutagen, trying FFprobe instead", c=3)
                 self.backend = "ffprobe"
@@ -381,20 +379,26 @@ class MTag(object):
                 parser_output[alias] = (priority, tv[0])
 
         # take first value (lowest priority / most preferred)
-        ret = {sk: unicode(tv[1]).strip() for sk, tv in parser_output.items()}
+        ret: dict[str, Union[str, float]] = {
+            sk: unicode(tv[1]).strip() for sk, tv in parser_output.items()
+        }
 
         # track 3/7 => track 3
-        for sk, tv in ret.items():
+        for sk, zv in ret.items():
             if sk[0] == ".":
-                sv = str(tv).split("/")[0].strip().lstrip("0")
+                sv = str(zv).split("/")[0].strip().lstrip("0")
                 ret[sk] = sv or 0
 
         # normalize key notation to rkeobo
         okey = ret.get("key")
         if okey:
-            key = okey.replace(" ", "").replace("maj", "").replace("min", "m")
+            key = str(okey).replace(" ", "").replace("maj", "").replace("min", "m")
             ret["key"] = REKOBO_LKEY.get(key.lower(), okey)
 
+        if self.args.mtag_vv:
+            zl = " ".join("\033[36m{} \033[33m{}".format(k, v) for k, v in ret.items())
+            self.log("norm: {}\033[0m".format(zl), "90")
+
         return ret
 
     def compare(self, abspath: str) -> dict[str, Union[str, float]]:
@@ -441,10 +445,15 @@ class MTag(object):
         if not bos.path.isfile(abspath):
             return {}
 
-        import mutagen
+        from mutagen import File
 
         try:
-            md = mutagen.File(fsenc(abspath), easy=True)
+            md = File(fsenc(abspath), easy=True)
+            assert md
+            if self.args.mtag_vv:
+                for zd in (md.info.__dict__, dict(md.tags)):
+                    zl = ["\033[36m{} \033[33m{}".format(k, v) for k, v in zd.items()]
+                    self.log("mutagen: {}\033[0m".format(" ".join(zl)), "90")
             if not md.info.length and not md.info.codec:
                 raise Exception()
         except:
@@ -494,6 +503,12 @@ class MTag(object):
             return {}
 
         ret, md = ffprobe(abspath, self.args.mtag_to)
+
+        if self.args.mtag_vv:
+            for zd in (ret, dict(md)):
+                zl = ["\033[36m{} \033[33m{}".format(k, v) for k, v in zd.items()]
+                self.log("ffprobe: {}\033[0m".format(" ".join(zl)), "90")
+
         return self.normalize_tags(ret, md)
 
     def get_bin(

copyparty/multicast.py

@@ -0,0 +1,332 @@
+# coding: utf-8
+from __future__ import print_function, unicode_literals
+
+import socket
+import time
+
+import ipaddress
+from ipaddress import IPv4Address, IPv4Network, IPv6Address, IPv6Network
+
+from .__init__ import TYPE_CHECKING
+from .util import MACOS, Netdev, min_ex, spack
+
+if TYPE_CHECKING:
+    from .svchub import SvcHub
+
+if True:  # pylint: disable=using-constant-test
+    from typing import Optional, Union
+
+if not hasattr(socket, "IPPROTO_IPV6"):
+    setattr(socket, "IPPROTO_IPV6", 41)
+
+
+class NoIPs(Exception):
+    pass
+
+
+class MC_Sck(object):
+    """there is one socket for each server ip"""
+
+    def __init__(
+        self,
+        sck: socket.socket,
+        nd: Netdev,
+        grp: str,
+        ip: str,
+        net: Union[IPv4Network, IPv6Network],
+    ):
+        self.sck = sck
+        self.idx = nd.idx
+        self.name = nd.name
+        self.grp = grp
+        self.mreq = b""
+        self.ip = ip
+        self.net = net
+        self.ips = {ip: net}
+        self.v6 = ":" in ip
+        self.have4 = ":" not in ip
+        self.have6 = ":" in ip
+
+
+class MCast(object):
+    def __init__(
+        self,
+        hub: "SvcHub",
+        Srv: type[MC_Sck],
+        on: list[str],
+        off: list[str],
+        mc_grp_4: str,
+        mc_grp_6: str,
+        port: int,
+        vinit: bool,
+    ) -> None:
+        """disable ipv%d by setting mc_grp_%d empty"""
+        self.hub = hub
+        self.Srv = Srv
+        self.args = hub.args
+        self.asrv = hub.asrv
+        self.log_func = hub.log
+        self.on = on
+        self.off = off
+        self.grp4 = mc_grp_4
+        self.grp6 = mc_grp_6
+        self.port = port
+        self.vinit = vinit
+
+        self.srv: dict[socket.socket, MC_Sck] = {}  # listening sockets
+        self.sips: set[str] = set()  # all listening ips (including failed attempts)
+        self.b2srv: dict[bytes, MC_Sck] = {}  # binary-ip -> server socket
+        self.b4: list[bytes] = []  # sorted list of binary-ips
+        self.b6: list[bytes] = []  # sorted list of binary-ips
+        self.cscache: dict[str, Optional[MC_Sck]] = {}  # client ip -> server cache
+
+        self.running = True
+
+    def log(self, msg: str, c: Union[int, str] = 0) -> None:
+        self.log_func("multicast", msg, c)
+
+    def create_servers(self) -> list[str]:
+        bound: list[str] = []
+        netdevs = self.hub.tcpsrv.netdevs
+        ips = [x[0] for x in self.hub.tcpsrv.bound]
+
+        if "::" in ips:
+            ips = [x for x in ips if x != "::"] + list(
+                [x.split("/")[0] for x in netdevs if ":" in x]
+            )
+            ips.append("0.0.0.0")
+
+        if "0.0.0.0" in ips:
+            ips = [x for x in ips if x != "0.0.0.0"] + list(
+                [x.split("/")[0] for x in netdevs if ":" not in x]
+            )
+
+        ips = [x for x in ips if x not in ("::1", "127.0.0.1")]
+
+        # ip -> ip/prefix
+        ips = [[x for x in netdevs if x.startswith(y + "/")][0] for y in ips]
+
+        on = self.on[:]
+        off = self.off[:]
+        for lst in (on, off):
+            for av in list(lst):
+                for sk, sv in netdevs.items():
+                    if (av == str(sv.idx) or av == sv.name) and sk not in lst:
+                        lst.append(sk)
+
+        if on:
+            ips = [x for x in ips if x in on]
+        elif off:
+            ips = [x for x in ips if x not in off]
+
+        if not self.grp4:
+            ips = [x for x in ips if ":" in x]
+
+        if not self.grp6:
+            ips = [x for x in ips if ":" not in x]
+
+        ips = list(set(ips))
+        all_selected = ips[:]
+
+        # discard non-linklocal ipv6
+        ips = [x for x in ips if ":" not in x or x.startswith("fe80")]
+
+        if not ips:
+            raise NoIPs()
+
+        for ip in ips:
+            v6 = ":" in ip
+            netdev = netdevs[ip]
+            if not netdev.idx:
+                t = "using INADDR_ANY for ip [{}], netdev [{}]"
+                if not self.srv and ip not in ["::", "0.0.0.0"]:
+                    self.log(t.format(ip, netdev), 3)
+
+            ipv = socket.AF_INET6 if v6 else socket.AF_INET
+            sck = socket.socket(ipv, socket.SOCK_DGRAM, socket.IPPROTO_UDP)
+            sck.settimeout(None)
+            sck.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+            try:
+                sck.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
+            except:
+                pass
+
+            # most ipv6 clients expect multicast on linklocal ip only;
+            # add a/aaaa records for the other nic IPs
+            other_ips: set[str] = set()
+            if v6:
+                for nd in netdevs.values():
+                    if nd.idx == netdev.idx and nd.ip in all_selected and ":" in nd.ip:
+                        other_ips.add(nd.ip)
+
+            net = ipaddress.ip_network(ip, False)
+            ip = ip.split("/")[0]
+            srv = self.Srv(sck, netdev, self.grp6 if ":" in ip else self.grp4, ip, net)
+            for oth_ip in other_ips:
+                srv.ips[oth_ip.split("/")[0]] = ipaddress.ip_network(oth_ip, False)
+
+            # gvfs breaks if a linklocal ip appears in a dns reply
+            srv.ips = {k: v for k, v in srv.ips.items() if not k.startswith("fe80")}
+            if not srv.ips:
+                self.log("no routable IPs on {}; skipping [{}]".format(netdev, ip), 3)
+                continue
+
+            try:
+                self.setup_socket(srv)
+                self.srv[sck] = srv
+                bound.append(ip)
+            except:
+                t = "announce failed on {} [{}]:\n{}"
+                self.log(t.format(netdev, ip, min_ex()), 3)
+
+        if self.args.zm_msub:
+            for s1 in self.srv.values():
+                for s2 in self.srv.values():
+                    if s1.idx != s2.idx:
+                        continue
+
+                    if s1.ip not in s2.ips:
+                        s2.ips[s1.ip] = s1.net
+
+        if self.args.zm_mnic:
+            for s1 in self.srv.values():
+                for s2 in self.srv.values():
+                    for ip1, net1 in list(s1.ips.items()):
+                        for ip2, net2 in list(s2.ips.items()):
+                            if net1 == net2 and ip1 != ip2:
+                                s1.ips[ip2] = net2
+
+        self.sips = set([x.split("/")[0] for x in all_selected])
+        for srv in self.srv.values():
+            assert srv.ip in self.sips
+
+        return bound
+
+    def setup_socket(self, srv: MC_Sck) -> None:
+        sck = srv.sck
+        if srv.v6:
+            if self.vinit:
+                zsl = list(srv.ips.keys())
+                self.log("v6({}) idx({}) {}".format(srv.ip, srv.idx, zsl), 6)
+
+            for ip in srv.ips:
+                bip = socket.inet_pton(socket.AF_INET6, ip)
+                self.b2srv[bip] = srv
+                self.b6.append(bip)
+
+            grp = self.grp6 if srv.idx else ""
+            try:
+                if MACOS:
+                    raise Exception()
+
+                sck.bind((grp, self.port, 0, srv.idx))
+            except:
+                sck.bind(("", self.port, 0, srv.idx))
+
+            bgrp = socket.inet_pton(socket.AF_INET6, self.grp6)
+            dev = spack(b"@I", srv.idx)
+            srv.mreq = bgrp + dev
+            if srv.idx != socket.INADDR_ANY:
+                sck.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_MULTICAST_IF, dev)
+
+            try:
+                sck.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_MULTICAST_HOPS, 255)
+                sck.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_MULTICAST_LOOP, 1)
+            except:
+                # macos
+                t = "failed to set IPv6 TTL/LOOP; announcements may not survive multiple switches/routers"
+                self.log(t, 3)
+        else:
+            if self.vinit:
+                self.log("v4({}) idx({})".format(srv.ip, srv.idx), 6)
+
+            bip = socket.inet_aton(srv.ip)
+            self.b2srv[bip] = srv
+            self.b4.append(bip)
+
+            grp = self.grp4 if srv.idx else ""
+            try:
+                if MACOS:
+                    raise Exception()
+
+                sck.bind((grp, self.port))
+            except:
+                sck.bind(("", self.port))
+
+            bgrp = socket.inet_aton(self.grp4)
+            dev = (
+                spack(b"=I", socket.INADDR_ANY)
+                if srv.idx == socket.INADDR_ANY
+                else socket.inet_aton(srv.ip)
+            )
+            srv.mreq = bgrp + dev
+            if srv.idx != socket.INADDR_ANY:
+                sck.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_IF, dev)
+
+            try:
+                sck.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, 255)
+                sck.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_LOOP, 1)
+            except:
+                # probably can't happen but dontcare if it does
+                t = "failed to set IPv4 TTL/LOOP; announcements may not survive multiple switches/routers"
+                self.log(t, 3)
+
+        self.hop(srv)
+        self.b4.sort(reverse=True)
+        self.b6.sort(reverse=True)
+
+    def hop(self, srv: MC_Sck) -> None:
+        """rejoin to keepalive on routers/switches without igmp-snooping"""
+        sck = srv.sck
+        req = srv.mreq
+        if ":" in srv.ip:
+            try:
+                sck.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_LEAVE_GROUP, req)
+                # linux does leaves/joins twice with 0.2~1.05s spacing
+                time.sleep(1.2)
+            except:
+                pass
+
+            sck.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_JOIN_GROUP, req)
+        else:
+            try:
+                sck.setsockopt(socket.IPPROTO_IP, socket.IP_DROP_MEMBERSHIP, req)
+                time.sleep(1.2)
+            except:
+                pass
+
+            # t = "joining {} from ip {} idx {} with mreq {}"
+            # self.log(t.format(srv.grp, srv.ip, srv.idx, repr(srv.mreq)), 6)
+            sck.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, req)
+
+    def map_client(self, cip: str) -> Optional[MC_Sck]:
+        try:
+            return self.cscache[cip]
+        except:
+            pass
+
+        ret: Optional[MC_Sck] = None
+        v6 = ":" in cip
+        ci = IPv6Address(cip) if v6 else IPv4Address(cip)
+        for x in self.b6 if v6 else self.b4:
+            srv = self.b2srv[x]
+            if any([x for x in srv.ips.values() if ci in x]):
+                ret = srv
+                break
+
+        if not ret and cip in ("127.0.0.1", "::1"):
+            # just give it something
+            ret = list(self.srv.values())[0]
+
+        if ret:
+            t = "new client on {} ({}): {}"
+            self.log(t.format(ret.name, ret.net, cip), 6)
+        else:
+            t = "could not map client {} to known subnet; maybe forwarded from another network?"
+            self.log(t.format(cip), 3)
+
+        if len(self.cscache) > 9000:
+            self.cscache = {}
+
+        self.cscache[cip] = ret
+        return ret

copyparty/smbd.py

@@ -0,0 +1,321 @@
+# coding: utf-8
+
+import inspect
+import logging
+import os
+import random
+import stat
+import sys
+import time
+from types import SimpleNamespace
+
+from .__init__ import ANYWIN, TYPE_CHECKING
+from .authsrv import LEELOO_DALLAS, VFS
+from .bos import bos
+from .util import Daemon, min_ex
+
+if True:  # pylint: disable=using-constant-test
+    from typing import Any
+
+if TYPE_CHECKING:
+    from .svchub import SvcHub
+
+
+lg = logging.getLogger("smb")
+debug, info, warning, error = (lg.debug, lg.info, lg.warning, lg.error)
+
+
+class SMB(object):
+    def __init__(self, hub: "SvcHub") -> None:
+        self.hub = hub
+        self.args = hub.args
+        self.asrv = hub.asrv
+        self.log = hub.log
+        self.files: dict[int, tuple[float, str]] = {}
+
+        lg.setLevel(logging.DEBUG if self.args.smbvvv else logging.INFO)
+        for x in ["impacket", "impacket.smbserver"]:
+            lgr = logging.getLogger(x)
+            lgr.setLevel(logging.DEBUG if self.args.smbvv else logging.INFO)
+
+        try:
+            from impacket import smbserver
+            from impacket.ntlm import compute_lmhash, compute_nthash
+        except ImportError:
+            m = "\033[36m\n{}\033[31m\n\nERROR: need 'impacket'; please run this command:\033[33m\n {} -m pip install --user impacket\n\033[0m"
+            print(m.format(min_ex(), sys.executable))
+            sys.exit(1)
+
+        # patch vfs into smbserver.os
+        fos = SimpleNamespace()
+        for k in os.__dict__:
+            try:
+                setattr(fos, k, getattr(os, k))
+            except:
+                pass
+        fos.close = self._close
+        fos.listdir = self._listdir
+        fos.mkdir = self._mkdir
+        fos.open = self._open
+        fos.remove = self._unlink
+        fos.rename = self._rename
+        fos.stat = self._stat
+        fos.unlink = self._unlink
+        fos.utime = self._utime
+        smbserver.os = fos
+
+        # ...and smbserver.os.path
+        fop = SimpleNamespace()
+        for k in os.path.__dict__:
+            try:
+                setattr(fop, k, getattr(os.path, k))
+            except:
+                pass
+        fop.exists = self._p_exists
+        fop.getsize = self._p_getsize
+        fop.isdir = self._p_isdir
+        smbserver.os.path = fop
+
+        if not self.args.smb_nwa_2:
+            fop.join = self._p_join
+
+        # other patches
+        smbserver.isInFileJail = self._is_in_file_jail
+        self._disarm()
+
+        ip = next((x for x in self.args.i if ":" not in x), None)
+        if not ip:
+            self.log("smb", "IPv6 not supported for SMB; listening on 0.0.0.0", 3)
+            ip = "0.0.0.0"
+
+        port = int(self.args.smb_port)
+        srv = smbserver.SimpleSMBServer(listenAddress=ip, listenPort=port)
+
+        ro = "no" if self.args.smbw else "yes"  # (does nothing)
+        srv.addShare("A", "/", readOnly=ro)
+        srv.setSMB2Support(not self.args.smb1)
+
+        for name, pwd in self.asrv.acct.items():
+            for u, p in ((name, pwd), (pwd, "k")):
+                lmhash = compute_lmhash(p)
+                nthash = compute_nthash(p)
+                srv.addCredential(u, 0, lmhash, nthash)
+
+        chi = [random.randint(0, 255) for x in range(8)]
+        cha = "".join(["{:02x}".format(x) for x in chi])
+        srv.setSMBChallenge(cha)
+
+        self.srv = srv
+        self.stop = srv.stop
+        self.log("smb", "listening @ {}:{}".format(ip, port))
+
+    def start(self) -> None:
+        Daemon(self.srv.start)
+
+    def _v2a(self, caller: str, vpath: str, *a: Any) -> tuple[VFS, str]:
+        vpath = vpath.replace("\\", "/").lstrip("/")
+        # cf = inspect.currentframe().f_back
+        # c1 = cf.f_back.f_code.co_name
+        # c2 = cf.f_code.co_name
+        debug('%s("%s", %s)\033[K\033[0m', caller, vpath, str(a))
+
+        # TODO find a way to grab `identity` in smbComSessionSetupAndX and smb2SessionSetup
+        vfs, rem = self.asrv.vfs.get(vpath, LEELOO_DALLAS, True, True)
+        return vfs, vfs.canonical(rem)
+
+    def _listdir(self, vpath: str, *a: Any, **ka: Any) -> list[str]:
+        vpath = vpath.replace("\\", "/").lstrip("/")
+        # caller = inspect.currentframe().f_back.f_code.co_name
+        debug('listdir("%s", %s)\033[K\033[0m', vpath, str(a))
+        vfs, rem = self.asrv.vfs.get(vpath, LEELOO_DALLAS, False, False)
+        _, vfs_ls, vfs_virt = vfs.ls(
+            rem, LEELOO_DALLAS, not self.args.no_scandir, [[False, False]]
+        )
+        dirs = [x[0] for x in vfs_ls if stat.S_ISDIR(x[1].st_mode)]
+        fils = [x[0] for x in vfs_ls if x[0] not in dirs]
+        ls = list(vfs_virt.keys()) + dirs + fils
+        if self.args.smb_nwa_1:
+            return ls
+
+        # clients crash somewhere around 65760 byte
+        ret = []
+        sz = 112 * 2  # ['.', '..']
+        for n, fn in enumerate(ls):
+            if sz >= 64000:
+                t = "listing only %d of %d files (%d byte); see impacket#1433"
+                warning(t, n, len(ls), sz)
+                break
+
+            nsz = len(fn.encode("utf-16", "replace"))
+            nsz = ((nsz + 7) // 8) * 8
+            sz += 104 + nsz
+            ret.append(fn)
+
+        return ret
+
+    def _open(
+        self, vpath: str, flags: int, *a: Any, chmod: int = 0o777, **ka: Any
+    ) -> Any:
+        f_ro = os.O_RDONLY
+        if ANYWIN:
+            f_ro |= os.O_BINARY
+
+        wr = flags != f_ro
+        if wr and not self.args.smbw:
+            yeet("blocked write (no --smbw): " + vpath)
+
+        vfs, ap = self._v2a("open", vpath, *a)
+        if wr and not vfs.axs.uwrite:
+            yeet("blocked write (no-write-acc): " + vpath)
+
+        ret = bos.open(ap, flags, *a, mode=chmod, **ka)
+        if wr:
+            now = time.time()
+            nf = len(self.files)
+            if nf > 9000:
+                oldest = min([x[0] for x in self.files.values()])
+                cutoff = oldest + (now - oldest) / 2
+                self.files = {k: v for k, v in self.files.items() if v[0] > cutoff}
+                info("was tracking %d files, now %d", nf, len(self.files))
+
+            vpath = vpath.replace("\\", "/").lstrip("/")
+            self.files[ret] = (now, vpath)
+
+        return ret
+
+    def _close(self, fd: int) -> None:
+        os.close(fd)
+        if fd not in self.files:
+            return
+
+        _, vp = self.files.pop(fd)
+        vp, fn = os.path.split(vp)
+        vfs, rem = self.hub.asrv.vfs.get(vp, LEELOO_DALLAS, False, True)
+        vfs, rem = vfs.get_dbv(rem)
+        self.hub.up2k.hash_file(
+            vfs.realpath,
+            vfs.flags,
+            rem,
+            fn,
+            "1.7.6.2",
+            time.time(),
+        )
+
+    def _rename(self, vp1: str, vp2: str) -> None:
+        if not self.args.smbw:
+            yeet("blocked rename (no --smbw): " + vp1)
+
+        vp1 = vp1.lstrip("/")
+        vp2 = vp2.lstrip("/")
+
+        vfs2, ap2 = self._v2a("rename", vp2, vp1)
+        if not vfs2.axs.uwrite:
+            yeet("blocked rename (no-write-acc): " + vp2)
+
+        vfs1, _ = self.asrv.vfs.get(vp1, LEELOO_DALLAS, True, True)
+        if not vfs1.axs.umove:
+            yeet("blocked rename (no-move-acc): " + vp1)
+
+        self.hub.up2k.handle_mv(LEELOO_DALLAS, vp1, vp2)
+        try:
+            bos.makedirs(ap2)
+        except:
+            pass
+
+    def _mkdir(self, vpath: str) -> None:
+        if not self.args.smbw:
+            yeet("blocked mkdir (no --smbw): " + vpath)
+
+        vfs, ap = self._v2a("mkdir", vpath)
+        if not vfs.axs.uwrite:
+            yeet("blocked mkdir (no-write-acc): " + vpath)
+
+        return bos.mkdir(ap)
+
+    def _stat(self, vpath: str, *a: Any, **ka: Any) -> os.stat_result:
+        return bos.stat(self._v2a("stat", vpath, *a)[1], *a, **ka)
+
+    def _unlink(self, vpath: str) -> None:
+        if not self.args.smbw:
+            yeet("blocked delete (no --smbw): " + vpath)
+
+        # return bos.unlink(self._v2a("stat", vpath, *a)[1])
+        vfs, ap = self._v2a("delete", vpath)
+        if not vfs.axs.udel:
+            yeet("blocked delete (no-del-acc): " + vpath)
+
+        vpath = vpath.replace("\\", "/").lstrip("/")
+        self.hub.up2k.handle_rm(LEELOO_DALLAS, "1.7.6.2", [vpath], [])
+
+    def _utime(self, vpath: str, times: tuple[float, float]) -> None:
+        if not self.args.smbw:
+            yeet("blocked utime (no --smbw): " + vpath)
+
+        vfs, ap = self._v2a("utime", vpath)
+        if not vfs.axs.uwrite:
+            yeet("blocked utime (no-write-acc): " + vpath)
+
+        return bos.utime(ap, times)
+
+    def _p_exists(self, vpath: str) -> bool:
+        try:
+            bos.stat(self._v2a("p.exists", vpath)[1])
+            return True
+        except:
+            return False
+
+    def _p_getsize(self, vpath: str) -> int:
+        st = bos.stat(self._v2a("p.getsize", vpath)[1])
+        return st.st_size
+
+    def _p_isdir(self, vpath: str) -> bool:
+        try:
+            st = bos.stat(self._v2a("p.isdir", vpath)[1])
+            return stat.S_ISDIR(st.st_mode)
+        except:
+            return False
+
+    def _p_join(self, *a) -> str:
+        # impacket.smbserver reads globs from queryDirectoryRequest['Buffer']
+        # where somehow `fds.*` becomes `fds"*` so lets fix that
+        ret = os.path.join(*a)
+        return ret.replace('"', ".")  # type: ignore
+
+    def _hook(self, *a: Any, **ka: Any) -> None:
+        src = inspect.currentframe().f_back.f_code.co_name
+        error("\033[31m%s:hook(%s)\033[0m", src, a)
+        raise Exception("nope")
+
+    def _disarm(self) -> None:
+        from impacket import smbserver
+
+        smbserver.os.chmod = self._hook
+        smbserver.os.chown = self._hook
+        smbserver.os.ftruncate = self._hook
+        smbserver.os.lchown = self._hook
+        smbserver.os.link = self._hook
+        smbserver.os.lstat = self._hook
+        smbserver.os.replace = self._hook
+        smbserver.os.scandir = self._hook
+        smbserver.os.symlink = self._hook
+        smbserver.os.truncate = self._hook
+        smbserver.os.walk = self._hook
+
+        smbserver.os.path.abspath = self._hook
+        smbserver.os.path.expanduser = self._hook
+        smbserver.os.path.getatime = self._hook
+        smbserver.os.path.getctime = self._hook
+        smbserver.os.path.getmtime = self._hook
+        smbserver.os.path.isabs = self._hook
+        smbserver.os.path.isfile = self._hook
+        smbserver.os.path.islink = self._hook
+        smbserver.os.path.realpath = self._hook
+
+    def _is_in_file_jail(self, *a: Any) -> bool:
+        # handled by vfs
+        return True
+
+
+def yeet(msg: str) -> None:
+    info(msg)
+    raise Exception(msg)

copyparty/ssdp.py

@@ -0,0 +1,194 @@
+# coding: utf-8
+from __future__ import print_function, unicode_literals
+
+import re
+import select
+import socket
+from email.utils import formatdate
+
+from .__init__ import TYPE_CHECKING
+from .multicast import MC_Sck, MCast
+from .util import CachedSet, min_ex
+
+if TYPE_CHECKING:
+    from .broker_util import BrokerCli
+    from .httpcli import HttpCli
+    from .svchub import SvcHub
+
+if True:  # pylint: disable=using-constant-test
+    from typing import Optional, Union
+
+
+GRP = "239.255.255.250"
+
+
+class SSDP_Sck(MC_Sck):
+    def __init__(self, *a):
+        super(SSDP_Sck, self).__init__(*a)
+        self.hport = 0
+
+
+class SSDPr(object):
+    """generates http responses for httpcli"""
+
+    def __init__(self, broker: "BrokerCli") -> None:
+        self.broker = broker
+        self.args = broker.args
+
+    def reply(self, hc: "HttpCli") -> bool:
+        if hc.vpath.endswith("device.xml"):
+            return self.tx_device(hc)
+
+        hc.reply(b"unknown request", 400)
+        return False
+
+    def tx_device(self, hc: "HttpCli") -> bool:
+        zs = """
+<?xml version="1.0"?>
+<root xmlns="urn:schemas-upnp-org:device-1-0">
+    <specVersion>
+        <major>1</major>
+        <minor>0</minor>
+    </specVersion>
+    <URLBase>{}</URLBase>
+    <device>
+        <presentationURL>{}</presentationURL>
+        <deviceType>urn:schemas-upnp-org:device:Basic:1</deviceType>
+        <friendlyName>{}</friendlyName>
+        <modelDescription>file server</modelDescription>
+        <manufacturer>ed</manufacturer>
+        <manufacturerURL>https://ocv.me/</manufacturerURL>
+        <modelName>copyparty</modelName>
+        <modelURL>https://github.com/9001/copyparty/</modelURL>
+        <UDN>{}</UDN>
+        <serviceList>
+            <service>
+                <serviceType>urn:schemas-upnp-org:device:Basic:1</serviceType>
+                <serviceId>urn:schemas-upnp-org:device:Basic</serviceId>
+                <controlURL>/.cpr/ssdp/services.xml</controlURL>
+                <eventSubURL>/.cpr/ssdp/services.xml</eventSubURL>
+                <SCPDURL>/.cpr/ssdp/services.xml</SCPDURL>
+            </service>
+        </serviceList>
+    </device>
+</root>"""
+
+        sip, sport = hc.s.getsockname()[:2]
+        proto = "https" if self.args.https_only else "http"
+        ubase = "{}://{}:{}".format(proto, sip, sport)
+        zsl = self.args.zsl
+        url = zsl if "://" in zsl else ubase + "/" + zsl.lstrip("/")
+        name = "{} @ {}".format(self.args.doctitle, self.args.name)
+        zs = zs.strip().format(ubase, url, name, self.args.zsid)
+        hc.reply(zs.encode("utf-8", "replace"))
+        return False  # close connectino
+
+
+class SSDPd(MCast):
+    """communicates with ssdp clients over multicast"""
+
+    def __init__(self, hub: "SvcHub") -> None:
+        al = hub.args
+        vinit = al.zsv and not al.zmv
+        super(SSDPd, self).__init__(
+            hub, SSDP_Sck, al.zs_on, al.zs_off, GRP, "", 1900, vinit
+        )
+        self.srv: dict[socket.socket, SSDP_Sck] = {}
+        self.rxc = CachedSet(0.7)
+        self.txc = CachedSet(5)  # win10: every 3 sec
+        self.ptn_st = re.compile(b"\nst: *upnp:rootdevice", re.I)
+
+    def log(self, msg: str, c: Union[int, str] = 0) -> None:
+        self.log_func("SSDP", msg, c)
+
+    def run(self) -> None:
+        try:
+            bound = self.create_servers()
+        except:
+            t = "no server IP matches the ssdp config\n{}"
+            self.log(t.format(min_ex()), 1)
+            bound = []
+
+        if not bound:
+            self.log("failed to announce copyparty services on the network", 3)
+            return
+
+        # find http port for this listening ip
+        for srv in self.srv.values():
+            tcps = self.hub.tcpsrv.bound
+            hp = next((x[1] for x in tcps if x[0] in ("0.0.0.0", srv.ip)), 0)
+            hp = hp or next((x[1] for x in tcps if x[0] == "::"), 0)
+            if not hp:
+                hp = tcps[0][1]
+                self.log("assuming port {} for {}".format(hp, srv.ip), 3)
+            srv.hport = hp
+
+        self.log("listening")
+        while self.running:
+            rdy = select.select(self.srv, [], [], 180)
+            rx: list[socket.socket] = rdy[0]  # type: ignore
+            self.rxc.cln()
+            for sck in rx:
+                buf, addr = sck.recvfrom(4096)
+                try:
+                    self.eat(buf, addr)
+                except:
+                    if not self.running:
+                        return
+
+                    t = "{} {} \033[33m|{}| {}\n{}".format(
+                        self.srv[sck].name, addr, len(buf), repr(buf)[2:-1], min_ex()
+                    )
+                    self.log(t, 6)
+
+    def stop(self) -> None:
+        self.running = False
+        self.srv = {}
+
+    def eat(self, buf: bytes, addr: tuple[str, int]) -> None:
+        cip = addr[0]
+        if cip.startswith("169.254"):
+            return
+
+        if buf in self.rxc.c:
+            return
+
+        self.rxc.add(buf)
+        srv: Optional[SSDP_Sck] = self.map_client(cip)  # type: ignore
+        if not srv:
+            return
+
+        if not buf.startswith(b"M-SEARCH * HTTP/1."):
+            raise Exception("not an ssdp message")
+
+        if not self.ptn_st.search(buf):
+            return
+
+        if self.args.zsv:
+            t = "{} [{}] \033[36m{} \033[0m|{}|"
+            self.log(t.format(srv.name, srv.ip, cip, len(buf)), "90")
+
+        zs = """
+HTTP/1.1 200 OK
+CACHE-CONTROL: max-age=1800
+DATE: {0}
+EXT:
+LOCATION: http://{1}:{2}/.cpr/ssdp/device.xml
+OPT: "http://schemas.upnp.org/upnp/1/0/"; ns=01
+01-NLS: {3}
+SERVER: UPnP/1.0
+ST: upnp:rootdevice
+USN: {3}::upnp:rootdevice
+BOOTID.UPNP.ORG: 0
+CONFIGID.UPNP.ORG: 1
+
+"""
+        zs = zs.format(formatdate(usegmt=True), srv.ip, srv.hport, self.args.zsid)
+        zb = zs[1:].replace("\n", "\r\n").encode("utf-8", "replace")
+        srv.sck.sendto(zb, addr[:2])
+
+        if cip not in self.txc.c:
+            self.log("{} [{}] --> {}".format(srv.name, srv.ip, cip), "6")
+
+        self.txc.add(cip)
+        self.txc.cln()

copyparty/star.py

@@ -2,20 +2,17 @@
 from __future__ import print_function, unicode_literals
 
 import tarfile
-import threading
 
 from queue import Queue
 
 from .bos import bos
 from .sutil import StreamArc, errdesc
-from .util import fsenc, min_ex
+from .util import Daemon, fsenc, min_ex
 
-try:
+if True:  # pylint: disable=using-constant-test
     from typing import Any, Generator, Optional
 
     from .util import NamedLogger
-except:
-    pass
 
 
 class QFile(object):  # inherit io.StringIO for painful typing
@@ -60,9 +57,7 @@ class StreamTar(StreamArc):
         fmt = tarfile.GNU_FORMAT
         self.tar = tarfile.open(fileobj=self.qfile, mode="w|", format=fmt)  # type: ignore
 
-        w = threading.Thread(target=self._gen, name="star-gen")
-        w.daemon = True
-        w.start()
+        Daemon(self._gen, "star-gen")
 
     def gen(self) -> Generator[Optional[bytes], None, None]:
         try:

copyparty/stolen/dnslib/README.md

@@ -0,0 +1,5 @@
+`dnslib` but heavily simplified/feature-stripped
+
+L: MIT
+Copyright (c) 2010 - 2017 Paul Chakravarti
+https://github.com/paulc/dnslib/

copyparty/stolen/dnslib/__init__.py

@@ -0,0 +1,11 @@
+# coding: utf-8
+
+"""
+L: MIT
+Copyright (c) 2010 - 2017 Paul Chakravarti
+https://github.com/paulc/dnslib/tree/0.9.23
+"""
+
+from .dns import *
+
+version = "0.9.23"

copyparty/stolen/dnslib/bimap.py

@@ -0,0 +1,41 @@
+# coding: utf-8
+
+import types
+
+
+class BimapError(Exception):
+    pass
+
+
+class Bimap(object):
+    def __init__(self, name, forward, error=AttributeError):
+        self.name = name
+        self.error = error
+        self.forward = forward.copy()
+        self.reverse = dict([(v, k) for (k, v) in list(forward.items())])
+
+    def get(self, k, default=None):
+        try:
+            return self.forward[k]
+        except KeyError:
+            return default or str(k)
+
+    def __getitem__(self, k):
+        try:
+            return self.forward[k]
+        except KeyError:
+            if isinstance(self.error, types.FunctionType):
+                return self.error(self.name, k, True)
+            else:
+                raise self.error("%s: Invalid forward lookup: [%s]" % (self.name, k))
+
+    def __getattr__(self, k):
+        try:
+            if k == "__wrapped__":
+                raise AttributeError()
+            return self.reverse[k]
+        except KeyError:
+            if isinstance(self.error, types.FunctionType):
+                return self.error(self.name, k, False)
+            else:
+                raise self.error("%s: Invalid reverse lookup: [%s]" % (self.name, k))

copyparty/stolen/dnslib/bit.py

@@ -0,0 +1,15 @@
+# coding: utf-8
+
+from __future__ import print_function
+
+
+def get_bits(data, offset, bits=1):
+    mask = ((1 << bits) - 1) << offset
+    return (data & mask) >> offset
+
+
+def set_bits(data, value, offset, bits=1):
+    mask = ((1 << bits) - 1) << offset
+    clear = 0xFFFF ^ mask
+    data = (data & clear) | ((value << offset) & mask)
+    return data

copyparty/stolen/dnslib/buffer.py

@@ -0,0 +1,56 @@
+# coding: utf-8
+
+import binascii
+import struct
+
+
+class BufferError(Exception):
+    pass
+
+
+class Buffer(object):
+    def __init__(self, data=b""):
+        self.data = bytearray(data)
+        self.offset = 0
+
+    def remaining(self):
+        return len(self.data) - self.offset
+
+    def get(self, length):
+        if length > self.remaining():
+            raise BufferError(
+                "Not enough bytes [offset=%d,remaining=%d,requested=%d]"
+                % (self.offset, self.remaining(), length)
+            )
+        start = self.offset
+        end = self.offset + length
+        self.offset += length
+        return bytes(self.data[start:end])
+
+    def hex(self):
+        return binascii.hexlify(self.data)
+
+    def pack(self, fmt, *args):
+        self.offset += struct.calcsize(fmt)
+        self.data += struct.pack(fmt, *args)
+
+    def append(self, s):
+        self.offset += len(s)
+        self.data += s
+
+    def update(self, ptr, fmt, *args):
+        s = struct.pack(fmt, *args)
+        self.data[ptr : ptr + len(s)] = s
+
+    def unpack(self, fmt):
+        try:
+            data = self.get(struct.calcsize(fmt))
+            return struct.unpack(fmt, data)
+        except struct.error:
+            raise BufferError(
+                "Error unpacking struct '%s' <%s>"
+                % (fmt, binascii.hexlify(data).decode())
+            )
+
+    def __len__(self):
+        return len(self.data)

copyparty/stolen/dnslib/dns.py

@@ -0,0 +1,775 @@
+# coding: utf-8
+
+from __future__ import print_function
+
+import binascii
+from itertools import chain
+
+from .bimap import Bimap, BimapError
+from .bit import get_bits, set_bits
+from .buffer import BufferError
+from .label import DNSBuffer, DNSLabel
+from .ranges import IP4, IP6, H, I, check_bytes
+
+
+class DNSError(Exception):
+    pass
+
+
+def unknown_qtype(name, key, forward):
+    if forward:
+        try:
+            return "TYPE%d" % (key,)
+        except:
+            raise DNSError("%s: Invalid forward lookup: [%s]" % (name, key))
+    else:
+        if key.startswith("TYPE"):
+            try:
+                return int(key[4:])
+            except:
+                pass
+        raise DNSError("%s: Invalid reverse lookup: [%s]" % (name, key))
+
+
+QTYPE = Bimap(
+    "QTYPE",
+    {1: "A", 12: "PTR", 16: "TXT", 28: "AAAA", 33: "SRV", 47: "NSEC", 255: "ANY"},
+    unknown_qtype,
+)
+
+CLASS = Bimap("CLASS", {1: "IN", 254: "None", 255: "*", 0x8001: "F_IN"}, DNSError)
+
+QR = Bimap("QR", {0: "QUERY", 1: "RESPONSE"}, DNSError)
+
+RCODE = Bimap(
+    "RCODE",
+    {
+        0: "NOERROR",
+        1: "FORMERR",
+        2: "SERVFAIL",
+        3: "NXDOMAIN",
+        4: "NOTIMP",
+        5: "REFUSED",
+        6: "YXDOMAIN",
+        7: "YXRRSET",
+        8: "NXRRSET",
+        9: "NOTAUTH",
+        10: "NOTZONE",
+    },
+    DNSError,
+)
+
+OPCODE = Bimap(
+    "OPCODE", {0: "QUERY", 1: "IQUERY", 2: "STATUS", 4: "NOTIFY", 5: "UPDATE"}, DNSError
+)
+
+
+def label(label, origin=None):
+    if label.endswith("."):
+        return DNSLabel(label)
+    else:
+        return (origin if isinstance(origin, DNSLabel) else DNSLabel(origin)).add(label)
+
+
+class DNSRecord(object):
+    @classmethod
+    def parse(cls, packet) -> "DNSRecord":
+        buffer = DNSBuffer(packet)
+        try:
+            header = DNSHeader.parse(buffer)
+            questions = []
+            rr = []
+            auth = []
+            ar = []
+            for i in range(header.q):
+                questions.append(DNSQuestion.parse(buffer))
+            for i in range(header.a):
+                rr.append(RR.parse(buffer))
+            for i in range(header.auth):
+                auth.append(RR.parse(buffer))
+            for i in range(header.ar):
+                ar.append(RR.parse(buffer))
+            return cls(header, questions, rr, auth=auth, ar=ar)
+        except (BufferError, BimapError) as e:
+            raise DNSError(
+                "Error unpacking DNSRecord [offset=%d]: %s" % (buffer.offset, e)
+            )
+
+    @classmethod
+    def question(cls, qname, qtype="A", qclass="IN"):
+        return DNSRecord(
+            q=DNSQuestion(qname, getattr(QTYPE, qtype), getattr(CLASS, qclass))
+        )
+
+    def __init__(
+        self, header=None, questions=None, rr=None, q=None, a=None, auth=None, ar=None
+    ) -> None:
+        self.header = header or DNSHeader()
+        self.questions: list[DNSQuestion] = questions or []
+        self.rr: list[RR] = rr or []
+        self.auth: list[RR] = auth or []
+        self.ar: list[RR] = ar or []
+
+        if q:
+            self.questions.append(q)
+        if a:
+            self.rr.append(a)
+        self.set_header_qa()
+
+    def reply(self, ra=1, aa=1):
+        return DNSRecord(
+            DNSHeader(id=self.header.id, bitmap=self.header.bitmap, qr=1, ra=ra, aa=aa),
+            q=self.q,
+        )
+
+    def add_question(self, *q) -> None:
+        self.questions.extend(q)
+        self.set_header_qa()
+
+    def add_answer(self, *rr) -> None:
+        self.rr.extend(rr)
+        self.set_header_qa()
+
+    def add_auth(self, *auth) -> None:
+        self.auth.extend(auth)
+        self.set_header_qa()
+
+    def add_ar(self, *ar) -> None:
+        self.ar.extend(ar)
+        self.set_header_qa()
+
+    def set_header_qa(self) -> None:
+        self.header.q = len(self.questions)
+        self.header.a = len(self.rr)
+        self.header.auth = len(self.auth)
+        self.header.ar = len(self.ar)
+
+    def get_q(self):
+        return self.questions[0] if self.questions else DNSQuestion()
+
+    q = property(get_q)
+
+    def get_a(self):
+        return self.rr[0] if self.rr else RR()
+
+    a = property(get_a)
+
+    def pack(self) -> bytes:
+        self.set_header_qa()
+        buffer = DNSBuffer()
+        self.header.pack(buffer)
+        for q in self.questions:
+            q.pack(buffer)
+        for rr in self.rr:
+            rr.pack(buffer)
+        for auth in self.auth:
+            auth.pack(buffer)
+        for ar in self.ar:
+            ar.pack(buffer)
+        return buffer.data
+
+    def truncate(self):
+        return DNSRecord(DNSHeader(id=self.header.id, bitmap=self.header.bitmap, tc=1))
+
+    def format(self, prefix="", sort=False):
+        s = sorted if sort else lambda x: x
+        sections = [repr(self.header)]
+        sections.extend(s([repr(q) for q in self.questions]))
+        sections.extend(s([repr(rr) for rr in self.rr]))
+        sections.extend(s([repr(rr) for rr in self.auth]))
+        sections.extend(s([repr(rr) for rr in self.ar]))
+        return prefix + ("\n" + prefix).join(sections)
+
+    short = format
+
+    def __repr__(self):
+        return self.format()
+
+    __str__ = __repr__
+
+
+class DNSHeader(object):
+    id = H("id")
+    bitmap = H("bitmap")
+    q = H("q")
+    a = H("a")
+    auth = H("auth")
+    ar = H("ar")
+
+    @classmethod
+    def parse(cls, buffer):
+        try:
+            (id, bitmap, q, a, auth, ar) = buffer.unpack("!HHHHHH")
+            return cls(id, bitmap, q, a, auth, ar)
+        except (BufferError, BimapError) as e:
+            raise DNSError(
+                "Error unpacking DNSHeader [offset=%d]: %s" % (buffer.offset, e)
+            )
+
+    def __init__(self, id=None, bitmap=None, q=0, a=0, auth=0, ar=0, **args) -> None:
+        self.id = id if id else 0
+        if bitmap is None:
+            self.bitmap = 0
+        else:
+            self.bitmap = bitmap
+        self.q = q
+        self.a = a
+        self.auth = auth
+        self.ar = ar
+        for k, v in args.items():
+            if k.lower() == "qr":
+                self.qr = v
+            elif k.lower() == "opcode":
+                self.opcode = v
+            elif k.lower() == "aa":
+                self.aa = v
+            elif k.lower() == "tc":
+                self.tc = v
+            elif k.lower() == "rd":
+                self.rd = v
+            elif k.lower() == "ra":
+                self.ra = v
+            elif k.lower() == "z":
+                self.z = v
+            elif k.lower() == "ad":
+                self.ad = v
+            elif k.lower() == "cd":
+                self.cd = v
+            elif k.lower() == "rcode":
+                self.rcode = v
+
+    def get_qr(self):
+        return get_bits(self.bitmap, 15)
+
+    def set_qr(self, val):
+        self.bitmap = set_bits(self.bitmap, val, 15)
+
+    qr = property(get_qr, set_qr)
+
+    def get_opcode(self):
+        return get_bits(self.bitmap, 11, 4)
+
+    def set_opcode(self, val):
+        self.bitmap = set_bits(self.bitmap, val, 11, 4)
+
+    opcode = property(get_opcode, set_opcode)
+
+    def get_aa(self):
+        return get_bits(self.bitmap, 10)
+
+    def set_aa(self, val):
+        self.bitmap = set_bits(self.bitmap, val, 10)
+
+    aa = property(get_aa, set_aa)
+
+    def get_tc(self):
+        return get_bits(self.bitmap, 9)
+
+    def set_tc(self, val):
+        self.bitmap = set_bits(self.bitmap, val, 9)
+
+    tc = property(get_tc, set_tc)
+
+    def get_rd(self):
+        return get_bits(self.bitmap, 8)
+
+    def set_rd(self, val):
+        self.bitmap = set_bits(self.bitmap, val, 8)
+
+    rd = property(get_rd, set_rd)
+
+    def get_ra(self):
+        return get_bits(self.bitmap, 7)
+
+    def set_ra(self, val):
+        self.bitmap = set_bits(self.bitmap, val, 7)
+
+    ra = property(get_ra, set_ra)
+
+    def get_z(self):
+        return get_bits(self.bitmap, 6)
+
+    def set_z(self, val):
+        self.bitmap = set_bits(self.bitmap, val, 6)
+
+    z = property(get_z, set_z)
+
+    def get_ad(self):
+        return get_bits(self.bitmap, 5)
+
+    def set_ad(self, val):
+        self.bitmap = set_bits(self.bitmap, val, 5)
+
+    ad = property(get_ad, set_ad)
+
+    def get_cd(self):
+        return get_bits(self.bitmap, 4)
+
+    def set_cd(self, val):
+        self.bitmap = set_bits(self.bitmap, val, 4)
+
+    cd = property(get_cd, set_cd)
+
+    def get_rcode(self):
+        return get_bits(self.bitmap, 0, 4)
+
+    def set_rcode(self, val):
+        self.bitmap = set_bits(self.bitmap, val, 0, 4)
+
+    rcode = property(get_rcode, set_rcode)
+
+    def pack(self, buffer):
+        buffer.pack("!HHHHHH", self.id, self.bitmap, self.q, self.a, self.auth, self.ar)
+
+    def __repr__(self):
+        f = [
+            self.aa and "AA",
+            self.tc and "TC",
+            self.rd and "RD",
+            self.ra and "RA",
+            self.z and "Z",
+            self.ad and "AD",
+            self.cd and "CD",
+        ]
+        if OPCODE.get(self.opcode) == "UPDATE":
+            f1 = "zo"
+            f2 = "pr"
+            f3 = "up"
+            f4 = "ad"
+        else:
+            f1 = "q"
+            f2 = "a"
+            f3 = "ns"
+            f4 = "ar"
+        return (
+            "<DNS Header: id=0x%x type=%s opcode=%s flags=%s "
+            "rcode='%s' %s=%d %s=%d %s=%d %s=%d>"
+            % (
+                self.id,
+                QR.get(self.qr),
+                OPCODE.get(self.opcode),
+                ",".join(filter(None, f)),
+                RCODE.get(self.rcode),
+                f1,
+                self.q,
+                f2,
+                self.a,
+                f3,
+                self.auth,
+                f4,
+                self.ar,
+            )
+        )
+
+    __str__ = __repr__
+
+
+class DNSQuestion(object):
+    @classmethod
+    def parse(cls, buffer):
+        try:
+            qname = buffer.decode_name()
+            qtype, qclass = buffer.unpack("!HH")
+            return cls(qname, qtype, qclass)
+        except (BufferError, BimapError) as e:
+            raise DNSError(
+                "Error unpacking DNSQuestion [offset=%d]: %s" % (buffer.offset, e)
+            )
+
+    def __init__(self, qname=None, qtype=1, qclass=1) -> None:
+        self.qname = qname
+        self.qtype = qtype
+        self.qclass = qclass
+
+    def set_qname(self, qname):
+        if isinstance(qname, DNSLabel):
+            self._qname = qname
+        else:
+            self._qname = DNSLabel(qname)
+
+    def get_qname(self):
+        return self._qname
+
+    qname = property(get_qname, set_qname)
+
+    def pack(self, buffer):
+        buffer.encode_name(self.qname)
+        buffer.pack("!HH", self.qtype, self.qclass)
+
+    def __repr__(self):
+        return "<DNS Question: '%s' qtype=%s qclass=%s>" % (
+            self.qname,
+            QTYPE.get(self.qtype),
+            CLASS.get(self.qclass),
+        )
+
+    __str__ = __repr__
+
+
+class RR(object):
+    rtype = H("rtype")
+    rclass = H("rclass")
+    ttl = I("ttl")
+    rdlength = H("rdlength")
+
+    @classmethod
+    def parse(cls, buffer):
+        try:
+            rname = buffer.decode_name()
+            rtype, rclass, ttl, rdlength = buffer.unpack("!HHIH")
+            if rdlength:
+                rdata = RDMAP.get(QTYPE.get(rtype), RD).parse(buffer, rdlength)
+            else:
+                rdata = ""
+            return cls(rname, rtype, rclass, ttl, rdata)
+        except (BufferError, BimapError) as e:
+            raise DNSError("Error unpacking RR [offset=%d]: %s" % (buffer.offset, e))
+
+    def __init__(self, rname=None, rtype=1, rclass=1, ttl=0, rdata=None) -> None:
+        self.rname = rname
+        self.rtype = rtype
+        self.rclass = rclass
+        self.ttl = ttl
+        self.rdata = rdata
+
+    def set_rname(self, rname):
+        if isinstance(rname, DNSLabel):
+            self._rname = rname
+        else:
+            self._rname = DNSLabel(rname)
+
+    def get_rname(self):
+        return self._rname
+
+    rname = property(get_rname, set_rname)
+
+    def pack(self, buffer):
+        buffer.encode_name(self.rname)
+        buffer.pack("!HHI", self.rtype, self.rclass, self.ttl)
+        rdlength_ptr = buffer.offset
+        buffer.pack("!H", 0)
+        start = buffer.offset
+        self.rdata.pack(buffer)
+        end = buffer.offset
+        buffer.update(rdlength_ptr, "!H", end - start)
+
+    def __repr__(self):
+        return "<DNS RR: '%s' rtype=%s rclass=%s ttl=%d rdata='%s'>" % (
+            self.rname,
+            QTYPE.get(self.rtype),
+            CLASS.get(self.rclass),
+            self.ttl,
+            self.rdata,
+        )
+
+    __str__ = __repr__
+
+
+class RD(object):
+    @classmethod
+    def parse(cls, buffer, length):
+        try:
+            data = buffer.get(length)
+            return cls(data)
+        except (BufferError, BimapError) as e:
+            raise DNSError("Error unpacking RD [offset=%d]: %s" % (buffer.offset, e))
+
+    def __init__(self, data=b"") -> None:
+        check_bytes("data", data)
+        self.data = bytes(data)
+
+    def pack(self, buffer):
+        buffer.append(self.data)
+
+    def __repr__(self):
+        if len(self.data) > 0:
+            return "\\# %d %s" % (
+                len(self.data),
+                binascii.hexlify(self.data).decode().upper(),
+            )
+        else:
+            return "\\# 0"
+
+    attrs = ("data",)
+
+
+def _force_bytes(x):
+    if isinstance(x, bytes):
+        return x
+    else:
+        return x.encode()
+
+
+class TXT(RD):
+    @classmethod
+    def parse(cls, buffer, length):
+        try:
+            data = list()
+            start_bo = buffer.offset
+            now_length = 0
+            while buffer.offset < start_bo + length:
+                (txtlength,) = buffer.unpack("!B")
+
+                if now_length + txtlength < length:
+                    now_length += txtlength
+                    data.append(buffer.get(txtlength))
+                else:
+                    raise DNSError(
+                        "Invalid TXT record: len(%d) > RD len(%d)" % (txtlength, length)
+                    )
+            return cls(data)
+        except (BufferError, BimapError) as e:
+            raise DNSError("Error unpacking TXT [offset=%d]: %s" % (buffer.offset, e))
+
+    def __init__(self, data) -> None:
+        if type(data) in (tuple, list):
+            self.data = [_force_bytes(x) for x in data]
+        else:
+            self.data = [_force_bytes(data)]
+        if any([len(x) > 255 for x in self.data]):
+            raise DNSError("TXT record too long: %s" % self.data)
+
+    def pack(self, buffer):
+        for ditem in self.data:
+            if len(ditem) > 255:
+                raise DNSError("TXT record too long: %s" % ditem)
+            buffer.pack("!B", len(ditem))
+            buffer.append(ditem)
+
+    def __repr__(self):
+        return ",".join([repr(x) for x in self.data])
+
+
+class A(RD):
+
+    data = IP4("data")
+
+    @classmethod
+    def parse(cls, buffer, length):
+        try:
+            data = buffer.unpack("!BBBB")
+            return cls(data)
+        except (BufferError, BimapError) as e:
+            raise DNSError("Error unpacking A [offset=%d]: %s" % (buffer.offset, e))
+
+    def __init__(self, data) -> None:
+        if type(data) in (tuple, list):
+            self.data = tuple(data)
+        else:
+            self.data = tuple(map(int, data.rstrip(".").split(".")))
+
+    def pack(self, buffer):
+        buffer.pack("!BBBB", *self.data)
+
+    def __repr__(self):
+        return "%d.%d.%d.%d" % self.data
+
+
+def _parse_ipv6(a):
+    l, _, r = a.partition("::")
+    l_groups = list(chain(*[divmod(int(x, 16), 256) for x in l.split(":") if x]))
+    r_groups = list(chain(*[divmod(int(x, 16), 256) for x in r.split(":") if x]))
+    zeros = [0] * (16 - len(l_groups) - len(r_groups))
+    return tuple(l_groups + zeros + r_groups)
+
+
+def _format_ipv6(a):
+    left = []
+    right = []
+    current = "left"
+    for i in range(0, 16, 2):
+        group = (a[i] << 8) + a[i + 1]
+        if current == "left":
+            if group == 0 and i < 14:
+                if (a[i + 2] << 8) + a[i + 3] == 0:
+                    current = "right"
+                else:
+                    left.append("0")
+            else:
+                left.append("%x" % group)
+        else:
+            if group == 0 and len(right) == 0:
+                pass
+            else:
+                right.append("%x" % group)
+    if len(left) < 8:
+        return ":".join(left) + "::" + ":".join(right)
+    else:
+        return ":".join(left)
+
+
+class AAAA(RD):
+    data = IP6("data")
+
+    @classmethod
+    def parse(cls, buffer, length):
+        try:
+            data = buffer.unpack("!16B")
+            return cls(data)
+        except (BufferError, BimapError) as e:
+            raise DNSError("Error unpacking AAAA [offset=%d]: %s" % (buffer.offset, e))
+
+    def __init__(self, data) -> None:
+        if type(data) in (tuple, list):
+            self.data = tuple(data)
+        else:
+            self.data = _parse_ipv6(data)
+
+    def pack(self, buffer):
+        buffer.pack("!16B", *self.data)
+
+    def __repr__(self):
+        return _format_ipv6(self.data)
+
+
+class CNAME(RD):
+    @classmethod
+    def parse(cls, buffer, length):
+        try:
+            label = buffer.decode_name()
+            return cls(label)
+        except (BufferError, BimapError) as e:
+            raise DNSError("Error unpacking CNAME [offset=%d]: %s" % (buffer.offset, e))
+
+    def __init__(self, label=None) -> None:
+        self.label = label
+
+    def set_label(self, label):
+        if isinstance(label, DNSLabel):
+            self._label = label
+        else:
+            self._label = DNSLabel(label)
+
+    def get_label(self):
+        return self._label
+
+    label = property(get_label, set_label)
+
+    def pack(self, buffer):
+        buffer.encode_name(self.label)
+
+    def __repr__(self):
+        return "%s" % (self.label)
+
+    attrs = ("label",)
+
+
+class PTR(CNAME):
+    pass
+
+
+class SRV(RD):
+    priority = H("priority")
+    weight = H("weight")
+    port = H("port")
+
+    @classmethod
+    def parse(cls, buffer, length):
+        try:
+            priority, weight, port = buffer.unpack("!HHH")
+            target = buffer.decode_name()
+            return cls(priority, weight, port, target)
+        except (BufferError, BimapError) as e:
+            raise DNSError("Error unpacking SRV [offset=%d]: %s" % (buffer.offset, e))
+
+    def __init__(self, priority=0, weight=0, port=0, target=None) -> None:
+        self.priority = priority
+        self.weight = weight
+        self.port = port
+        self.target = target
+
+    def set_target(self, target):
+        if isinstance(target, DNSLabel):
+            self._target = target
+        else:
+            self._target = DNSLabel(target)
+
+    def get_target(self):
+        return self._target
+
+    target = property(get_target, set_target)
+
+    def pack(self, buffer):
+        buffer.pack("!HHH", self.priority, self.weight, self.port)
+        buffer.encode_name(self.target)
+
+    def __repr__(self):
+        return "%d %d %d %s" % (self.priority, self.weight, self.port, self.target)
+
+    attrs = ("priority", "weight", "port", "target")
+
+
+def decode_type_bitmap(type_bitmap):
+    rrlist = []
+    buf = DNSBuffer(type_bitmap)
+    while buf.remaining():
+        winnum, winlen = buf.unpack("BB")
+        bitmap = bytearray(buf.get(winlen))
+        for (pos, value) in enumerate(bitmap):
+            for i in range(8):
+                if (value << i) & 0x80:
+                    bitpos = (256 * winnum) + (8 * pos) + i
+                    rrlist.append(QTYPE[bitpos])
+    return rrlist
+
+
+def encode_type_bitmap(rrlist):
+    rrlist = sorted([getattr(QTYPE, rr) for rr in rrlist])
+    buf = DNSBuffer()
+    curWindow = rrlist[0] // 256
+    bitmap = bytearray(32)
+    n = len(rrlist) - 1
+    for i, rr in enumerate(rrlist):
+        v = rr - curWindow * 256
+        bitmap[v // 8] |= 1 << (7 - v % 8)
+
+        if i == n or rrlist[i + 1] >= (curWindow + 1) * 256:
+            while bitmap[-1] == 0:
+                bitmap = bitmap[:-1]
+            buf.pack("BB", curWindow, len(bitmap))
+            buf.append(bitmap)
+
+            if i != n:
+                curWindow = rrlist[i + 1] // 256
+                bitmap = bytearray(32)
+
+    return buf.data
+
+
+class NSEC(RD):
+    @classmethod
+    def parse(cls, buffer, length):
+        try:
+            end = buffer.offset + length
+            name = buffer.decode_name()
+            rrlist = decode_type_bitmap(buffer.get(end - buffer.offset))
+            return cls(name, rrlist)
+        except (BufferError, BimapError) as e:
+            raise DNSError("Error unpacking NSEC [offset=%d]: %s" % (buffer.offset, e))
+
+    def __init__(self, label, rrlist) -> None:
+        self.label = label
+        self.rrlist = rrlist
+
+    def set_label(self, label):
+        if isinstance(label, DNSLabel):
+            self._label = label
+        else:
+            self._label = DNSLabel(label)
+
+    def get_label(self):
+        return self._label
+
+    label = property(get_label, set_label)
+
+    def pack(self, buffer):
+        buffer.encode_name(self.label)
+        buffer.append(encode_type_bitmap(self.rrlist))
+
+    def __repr__(self):
+        return "%s %s" % (self.label, " ".join(self.rrlist))
+
+    attrs = ("label", "rrlist")
+
+
+RDMAP = {"A": A, "AAAA": AAAA, "TXT": TXT, "PTR": PTR, "SRV": SRV, "NSEC": NSEC}

copyparty/stolen/dnslib/label.py

@@ -0,0 +1,154 @@
+# coding: utf-8
+
+from __future__ import print_function
+
+import re
+
+from .bit import get_bits, set_bits
+from .buffer import Buffer, BufferError
+
+LDH = set(range(33, 127))
+ESCAPE = re.compile(r"\\([0-9][0-9][0-9])")
+
+
+class DNSLabelError(Exception):
+    pass
+
+
+class DNSLabel(object):
+    def __init__(self, label):
+        if type(label) == DNSLabel:
+            self.label = label.label
+        elif type(label) in (list, tuple):
+            self.label = tuple(label)
+        else:
+            if not label or label in (b".", "."):
+                self.label = ()
+            elif type(label) is not bytes:
+                if type("") != type(b""):
+
+                    label = ESCAPE.sub(lambda m: chr(int(m[1])), label)
+                self.label = tuple(label.encode("idna").rstrip(b".").split(b"."))
+            else:
+                if type("") == type(b""):
+
+                    label = ESCAPE.sub(lambda m: chr(int(m.groups()[0])), label)
+                self.label = tuple(label.rstrip(b".").split(b"."))
+
+    def add(self, name):
+        new = DNSLabel(name)
+        if self.label:
+            new.label += self.label
+        return new
+
+    def idna(self):
+        return ".".join([s.decode("idna") for s in self.label]) + "."
+
+    def _decode(self, s):
+        if set(s).issubset(LDH):
+
+            return s.decode()
+        else:
+
+            return "".join([(chr(c) if (c in LDH) else "\\%03d" % c) for c in s])
+
+    def __str__(self):
+        return ".".join([self._decode(bytearray(s)) for s in self.label]) + "."
+
+    def __repr__(self):
+        return "<DNSLabel: '%s'>" % str(self)
+
+    def __hash__(self):
+        return hash(tuple(map(lambda x: x.lower(), self.label)))
+
+    def __ne__(self, other):
+        return not self == other
+
+    def __eq__(self, other):
+        if type(other) != DNSLabel:
+            return self.__eq__(DNSLabel(other))
+        else:
+            return [l.lower() for l in self.label] == [l.lower() for l in other.label]
+
+    def __len__(self):
+        return len(b".".join(self.label))
+
+
+class DNSBuffer(Buffer):
+    def __init__(self, data=b""):
+        super(DNSBuffer, self).__init__(data)
+        self.names = {}
+
+    def decode_name(self, last=-1):
+        label = []
+        done = False
+        while not done:
+            (length,) = self.unpack("!B")
+            if get_bits(length, 6, 2) == 3:
+
+                self.offset -= 1
+                pointer = get_bits(self.unpack("!H")[0], 0, 14)
+                save = self.offset
+                if last == save:
+                    raise BufferError(
+                        "Recursive pointer in DNSLabel [offset=%d,pointer=%d,length=%d]"
+                        % (self.offset, pointer, len(self.data))
+                    )
+                if pointer < self.offset:
+                    self.offset = pointer
+                else:
+
+                    raise BufferError(
+                        "Invalid pointer in DNSLabel [offset=%d,pointer=%d,length=%d]"
+                        % (self.offset, pointer, len(self.data))
+                    )
+                label.extend(self.decode_name(save).label)
+                self.offset = save
+                done = True
+            else:
+                if length > 0:
+                    l = self.get(length)
+                    try:
+                        l.decode()
+                    except UnicodeDecodeError:
+                        raise BufferError("Invalid label <%s>" % l)
+                    label.append(l)
+                else:
+                    done = True
+        return DNSLabel(label)
+
+    def encode_name(self, name):
+        if not isinstance(name, DNSLabel):
+            name = DNSLabel(name)
+        if len(name) > 253:
+            raise DNSLabelError("Domain label too long: %r" % name)
+        name = list(name.label)
+        while name:
+            if tuple(name) in self.names:
+
+                pointer = self.names[tuple(name)]
+                pointer = set_bits(pointer, 3, 14, 2)
+                self.pack("!H", pointer)
+                return
+            else:
+                self.names[tuple(name)] = self.offset
+                element = name.pop(0)
+                if len(element) > 63:
+                    raise DNSLabelError("Label component too long: %r" % element)
+                self.pack("!B", len(element))
+                self.append(element)
+        self.append(b"\x00")
+
+    def encode_name_nocompress(self, name):
+        if not isinstance(name, DNSLabel):
+            name = DNSLabel(name)
+        if len(name) > 253:
+            raise DNSLabelError("Domain label too long: %r" % name)
+        name = list(name.label)
+        while name:
+            element = name.pop(0)
+            if len(element) > 63:
+                raise DNSLabelError("Label component too long: %r" % element)
+            self.pack("!B", len(element))
+            self.append(element)
+        self.append(b"\x00")

copyparty/stolen/dnslib/lex.py

@@ -0,0 +1,105 @@
+# coding: utf-8
+
+from __future__ import print_function
+
+import collections
+
+try:
+    from StringIO import StringIO
+except ImportError:
+    from io import StringIO
+
+
+class Lexer(object):
+
+    escape_chars = "\\"
+    escape = {"n": "\n", "t": "\t", "r": "\r"}
+
+    def __init__(self, f, debug=False):
+        if hasattr(f, "read"):
+            self.f = f
+        elif type(f) == str:
+            self.f = StringIO(f)
+        elif type(f) == bytes:
+            self.f = StringIO(f.decode())
+        else:
+            raise ValueError("Invalid input")
+        self.debug = debug
+        self.q = collections.deque()
+        self.state = self.lexStart
+        self.escaped = False
+        self.eof = False
+
+    def __iter__(self):
+        return self.parse()
+
+    def next_token(self):
+        if self.debug:
+            print("STATE", self.state)
+        (tok, self.state) = self.state()
+        return tok
+
+    def parse(self):
+        while self.state is not None and not self.eof:
+            tok = self.next_token()
+            if tok:
+                yield tok
+
+    def read(self, n=1):
+        s = ""
+        while self.q and n > 0:
+            s += self.q.popleft()
+            n -= 1
+        s += self.f.read(n)
+        if s == "":
+            self.eof = True
+        if self.debug:
+            print("Read: >%s<" % repr(s))
+        return s
+
+    def peek(self, n=1):
+        s = ""
+        i = 0
+        while len(self.q) > i and n > 0:
+            s += self.q[i]
+            i += 1
+            n -= 1
+        r = self.f.read(n)
+        if n > 0 and r == "":
+            self.eof = True
+        self.q.extend(r)
+        if self.debug:
+            print("Peek : >%s<" % repr(s + r))
+        return s + r
+
+    def pushback(self, s):
+        p = collections.deque(s)
+        p.extend(self.q)
+        self.q = p
+
+    def readescaped(self):
+        c = self.read(1)
+        if c in self.escape_chars:
+            self.escaped = True
+            n = self.peek(3)
+            if n.isdigit():
+                n = self.read(3)
+                if self.debug:
+                    print("Escape: >%s<" % n)
+                return chr(int(n, 8))
+            elif n[0] in "x":
+                x = self.read(3)
+                if self.debug:
+                    print("Escape: >%s<" % x)
+                return chr(int(x[1:], 16))
+            else:
+                c = self.read(1)
+                if self.debug:
+                    print("Escape: >%s<" % c)
+                return self.escape.get(c, c)
+        else:
+            self.escaped = False
+            return c
+
+    def lexStart(self):
+        return (None, None)

copyparty/stolen/dnslib/ranges.py

@@ -0,0 +1,81 @@
+# coding: utf-8
+
+import sys
+
+if sys.version_info < (3,):
+    int_types = (
+        int,
+        long,
+    )
+    byte_types = (str, bytearray)
+else:
+    int_types = (int,)
+    byte_types = (bytes, bytearray)
+
+
+def check_instance(name, val, types):
+    if not isinstance(val, types):
+        raise ValueError(
+            "Attribute '%s' must be instance of %s [%s]" % (name, types, type(val))
+        )
+
+
+def check_bytes(name, val):
+    return check_instance(name, val, byte_types)
+
+
+def range_property(attr, min, max):
+    def getter(obj):
+        return getattr(obj, "_%s" % attr)
+
+    def setter(obj, val):
+        if isinstance(val, int_types) and min <= val <= max:
+            setattr(obj, "_%s" % attr, val)
+        else:
+            raise ValueError(
+                "Attribute '%s' must be between %d-%d [%s]" % (attr, min, max, val)
+            )
+
+    return property(getter, setter)
+
+
+def B(attr):
+    return range_property(attr, 0, 255)
+
+
+def H(attr):
+    return range_property(attr, 0, 65535)
+
+
+def I(attr):
+    return range_property(attr, 0, 4294967295)
+
+
+def ntuple_range(attr, n, min, max):
+    f = lambda x: isinstance(x, int_types) and min <= x <= max
+
+    def getter(obj):
+        return getattr(obj, "_%s" % attr)
+
+    def setter(obj, val):
+        if len(val) != n:
+            raise ValueError(
+                "Attribute '%s' must be tuple with %d elements [%s]" % (attr, n, val)
+            )
+        if all(map(f, val)):
+            setattr(obj, "_%s" % attr, val)
+        else:
+            raise ValueError(
+                "Attribute '%s' elements must be between %d-%d [%s]"
+                % (attr, min, max, val)
+            )
+
+    return property(getter, setter)
+
+
+def IP4(attr):
+    return ntuple_range(attr, 4, 0, 255)
+
+
+def IP6(attr):
+    return ntuple_range(attr, 16, 0, 255)

copyparty/stolen/ifaddr/README.md

@@ -0,0 +1,5 @@
+`ifaddr` with py2.7 support enabled by make-sfx.sh which strips py3 hints using strip_hints and removes the `^if True:` blocks
+
+L: BSD-2-Clause
+Copyright (c) 2014 Stefan C. Mueller
+https://github.com/pydron/ifaddr/

copyparty/stolen/ifaddr/__init__.py

@@ -0,0 +1,21 @@
+# coding: utf-8
+from __future__ import print_function, unicode_literals
+
+"""
+L: BSD-2-Clause
+Copyright (c) 2014 Stefan C. Mueller
+https://github.com/pydron/ifaddr/tree/0.2.0
+"""
+
+import os
+
+from ._shared import IP, Adapter
+
+if os.name == "nt":
+    from ._win32 import get_adapters
+elif os.name == "posix":
+    from ._posix import get_adapters
+else:
+    raise RuntimeError("Unsupported Operating System: %s" % os.name)
+
+__all__ = ["Adapter", "IP", "get_adapters"]

copyparty/stolen/ifaddr/_posix.py

@@ -0,0 +1,84 @@
+# coding: utf-8
+from __future__ import print_function, unicode_literals
+
+import collections
+import ctypes.util
+import os
+import socket
+
+import ipaddress
+
+if True:  # pylint: disable=using-constant-test
+    from typing import Iterable, Optional
+
+from . import _shared as shared
+from ._shared import U
+
+
+class ifaddrs(ctypes.Structure):
+    pass
+
+
+ifaddrs._fields_ = [
+    ("ifa_next", ctypes.POINTER(ifaddrs)),
+    ("ifa_name", ctypes.c_char_p),
+    ("ifa_flags", ctypes.c_uint),
+    ("ifa_addr", ctypes.POINTER(shared.sockaddr)),
+    ("ifa_netmask", ctypes.POINTER(shared.sockaddr)),
+]
+
+libc = ctypes.CDLL(ctypes.util.find_library("socket" if os.uname()[0] == "SunOS" else "c"), use_errno=True)  # type: ignore
+
+
+def get_adapters(include_unconfigured: bool = False) -> Iterable[shared.Adapter]:
+
+    addr0 = addr = ctypes.POINTER(ifaddrs)()
+    retval = libc.getifaddrs(ctypes.byref(addr))
+    if retval != 0:
+        eno = ctypes.get_errno()
+        raise OSError(eno, os.strerror(eno))
+
+    ips = collections.OrderedDict()
+
+    def add_ip(adapter_name: str, ip: Optional[shared.IP]) -> None:
+        if adapter_name not in ips:
+            index = None  # type: Optional[int]
+            try:
+                # Mypy errors on this when the Windows CI runs:
+                #     error: Module has no attribute "if_nametoindex"
+                index = socket.if_nametoindex(adapter_name)  # type: ignore
+            except (OSError, AttributeError):
+                pass
+            ips[adapter_name] = shared.Adapter(
+                adapter_name, adapter_name, [], index=index
+            )
+        if ip is not None:
+            ips[adapter_name].ips.append(ip)
+
+    while addr:
+        name = addr[0].ifa_name.decode(encoding="UTF-8")
+        ip_addr = shared.sockaddr_to_ip(addr[0].ifa_addr)
+        if ip_addr:
+            if addr[0].ifa_netmask and not addr[0].ifa_netmask[0].sa_familiy:
+                addr[0].ifa_netmask[0].sa_familiy = addr[0].ifa_addr[0].sa_familiy
+            netmask = shared.sockaddr_to_ip(addr[0].ifa_netmask)
+            if isinstance(netmask, tuple):
+                netmaskStr = U(netmask[0])
+                prefixlen = shared.ipv6_prefixlength(ipaddress.IPv6Address(netmaskStr))
+            else:
+                if netmask is None:
+                    t = "sockaddr_to_ip({}) returned None"
+                    raise Exception(t.format(addr[0].ifa_netmask))
+
+                netmaskStr = U("0.0.0.0/" + netmask)
+                prefixlen = ipaddress.IPv4Network(netmaskStr).prefixlen
+            ip = shared.IP(ip_addr, prefixlen, name)
+            add_ip(name, ip)
+        else:
+            if include_unconfigured:
+                add_ip(name, None)
+        addr = addr[0].ifa_next
+
+    libc.freeifaddrs(addr0)
+
+    return ips.values()

copyparty/stolen/ifaddr/_shared.py

@@ -0,0 +1,203 @@
+# coding: utf-8
+from __future__ import print_function, unicode_literals
+
+import ctypes
+import platform
+import socket
+import sys
+
+import ipaddress
+
+if True:  # pylint: disable=using-constant-test
+    from typing import Callable, List, Optional, Union
+
+
+PY2 = sys.version_info < (3,)
+if not PY2:
+    U: Callable[[str], str] = str
+else:
+    U = unicode  # noqa: F821  # pylint: disable=undefined-variable,self-assigning-variable
+
+
+class Adapter(object):
+    """
+    Represents a network interface device controller (NIC), such as a
+    network card. An adapter can have multiple IPs.
+
+    On Linux aliasing (multiple IPs per physical NIC) is implemented
+    by creating 'virtual' adapters, each represented by an instance
+    of this class. Each of those 'virtual' adapters can have both
+    a IPv4 and an IPv6 IP address.
+    """
+
+    def __init__(
+        self, name: str, nice_name: str, ips: List["IP"], index: Optional[int] = None
+    ) -> None:
+
+        #: Unique name that identifies the adapter in the system.
+        #: On Linux this is of the form of `eth0` or `eth0:1`, on
+        #: Windows it is a UUID in string representation, such as
+        #: `{846EE342-7039-11DE-9D20-806E6F6E6963}`.
+        self.name = name
+
+        #: Human readable name of the adpater. On Linux this
+        #: is currently the same as :attr:`name`. On Windows
+        #: this is the name of the device.
+        self.nice_name = nice_name
+
+        #: List of :class:`ifaddr.IP` instances in the order they were
+        #: reported by the system.
+        self.ips = ips
+
+        #: Adapter index as used by some API (e.g. IPv6 multicast group join).
+        self.index = index
+
+    def __repr__(self) -> str:
+        return "Adapter(name={name}, nice_name={nice_name}, ips={ips}, index={index})".format(
+            name=repr(self.name),
+            nice_name=repr(self.nice_name),
+            ips=repr(self.ips),
+            index=repr(self.index),
+        )
+
+
+if True:
+    # Type of an IPv4 address (a string in "xxx.xxx.xxx.xxx" format)
+    _IPv4Address = str
+
+    # Type of an IPv6 address (a three-tuple `(ip, flowinfo, scope_id)`)
+    _IPv6Address = tuple[str, int, int]
+
+
+class IP(object):
+    """
+    Represents an IP address of an adapter.
+    """
+
+    def __init__(
+        self, ip: Union[_IPv4Address, _IPv6Address], network_prefix: int, nice_name: str
+    ) -> None:
+
+        #: IP address. For IPv4 addresses this is a string in
+        #: "xxx.xxx.xxx.xxx" format. For IPv6 addresses this
+        #: is a three-tuple `(ip, flowinfo, scope_id)`, where
+        #: `ip` is a string in the usual collon separated
+        #: hex format.
+        self.ip = ip
+
+        #: Number of bits of the IP that represent the
+        #: network. For a `255.255.255.0` netmask, this
+        #: number would be `24`.
+        self.network_prefix = network_prefix
+
+        #: Human readable name for this IP.
+        #: On Linux is this currently the same as the adapter name.
+        #: On Windows this is the name of the network connection
+        #: as configured in the system control panel.
+        self.nice_name = nice_name
+
+    @property
+    def is_IPv4(self) -> bool:
+        """
+        Returns `True` if this IP is an IPv4 address and `False`
+        if it is an IPv6 address.
+        """
+        return not isinstance(self.ip, tuple)
+
+    @property
+    def is_IPv6(self) -> bool:
+        """
+        Returns `True` if this IP is an IPv6 address and `False`
+        if it is an IPv4 address.
+        """
+        return isinstance(self.ip, tuple)
+
+    def __repr__(self) -> str:
+        return "IP(ip={ip}, network_prefix={network_prefix}, nice_name={nice_name})".format(
+            ip=repr(self.ip),
+            network_prefix=repr(self.network_prefix),
+            nice_name=repr(self.nice_name),
+        )
+
+
+if platform.system() == "Darwin" or "BSD" in platform.system():
+
+    # BSD derived systems use marginally different structures
+    # than either Linux or Windows.
+    # I still keep it in `shared` since we can use
+    # both structures equally.
+
+    class sockaddr(ctypes.Structure):
+        _fields_ = [
+            ("sa_len", ctypes.c_uint8),
+            ("sa_familiy", ctypes.c_uint8),
+            ("sa_data", ctypes.c_uint8 * 14),
+        ]
+
+    class sockaddr_in(ctypes.Structure):
+        _fields_ = [
+            ("sa_len", ctypes.c_uint8),
+            ("sa_familiy", ctypes.c_uint8),
+            ("sin_port", ctypes.c_uint16),
+            ("sin_addr", ctypes.c_uint8 * 4),
+            ("sin_zero", ctypes.c_uint8 * 8),
+        ]
+
+    class sockaddr_in6(ctypes.Structure):
+        _fields_ = [
+            ("sa_len", ctypes.c_uint8),
+            ("sa_familiy", ctypes.c_uint8),
+            ("sin6_port", ctypes.c_uint16),
+            ("sin6_flowinfo", ctypes.c_uint32),
+            ("sin6_addr", ctypes.c_uint8 * 16),
+            ("sin6_scope_id", ctypes.c_uint32),
+        ]
+
+else:
+
+    class sockaddr(ctypes.Structure):  # type: ignore
+        _fields_ = [("sa_familiy", ctypes.c_uint16), ("sa_data", ctypes.c_uint8 * 14)]
+
+    class sockaddr_in(ctypes.Structure):  # type: ignore
+        _fields_ = [
+            ("sin_familiy", ctypes.c_uint16),
+            ("sin_port", ctypes.c_uint16),
+            ("sin_addr", ctypes.c_uint8 * 4),
+            ("sin_zero", ctypes.c_uint8 * 8),
+        ]
+
+    class sockaddr_in6(ctypes.Structure):  # type: ignore
+        _fields_ = [
+            ("sin6_familiy", ctypes.c_uint16),
+            ("sin6_port", ctypes.c_uint16),
+            ("sin6_flowinfo", ctypes.c_uint32),
+            ("sin6_addr", ctypes.c_uint8 * 16),
+            ("sin6_scope_id", ctypes.c_uint32),
+        ]
+
+
+def sockaddr_to_ip(
+    sockaddr_ptr: "ctypes.pointer[sockaddr]",
+) -> Optional[Union[_IPv4Address, _IPv6Address]]:
+    if sockaddr_ptr:
+        if sockaddr_ptr[0].sa_familiy == socket.AF_INET:
+            ipv4 = ctypes.cast(sockaddr_ptr, ctypes.POINTER(sockaddr_in))
+            ippacked = bytes(bytearray(ipv4[0].sin_addr))
+            ip = U(ipaddress.ip_address(ippacked))
+            return ip
+        elif sockaddr_ptr[0].sa_familiy == socket.AF_INET6:
+            ipv6 = ctypes.cast(sockaddr_ptr, ctypes.POINTER(sockaddr_in6))
+            flowinfo = ipv6[0].sin6_flowinfo
+            ippacked = bytes(bytearray(ipv6[0].sin6_addr))
+            ip = U(ipaddress.ip_address(ippacked))
+            scope_id = ipv6[0].sin6_scope_id
+            return (ip, flowinfo, scope_id)
+    return None
+
+
+def ipv6_prefixlength(address: ipaddress.IPv6Address) -> int:
+    prefix_length = 0
+    for i in range(address.max_prefixlen):
+        if int(address) >> i & 1:
+            prefix_length = prefix_length + 1
+    return prefix_length

copyparty/stolen/ifaddr/_win32.py

@@ -0,0 +1,135 @@
+# coding: utf-8
+from __future__ import print_function, unicode_literals
+
+import ctypes
+from ctypes import wintypes
+
+if True:  # pylint: disable=using-constant-test
+    from typing import Iterable, List
+
+from . import _shared as shared
+
+NO_ERROR = 0
+ERROR_BUFFER_OVERFLOW = 111
+MAX_ADAPTER_NAME_LENGTH = 256
+MAX_ADAPTER_DESCRIPTION_LENGTH = 128
+MAX_ADAPTER_ADDRESS_LENGTH = 8
+AF_UNSPEC = 0
+
+
+class SOCKET_ADDRESS(ctypes.Structure):
+    _fields_ = [
+        ("lpSockaddr", ctypes.POINTER(shared.sockaddr)),
+        ("iSockaddrLength", wintypes.INT),
+    ]
+
+
+class IP_ADAPTER_UNICAST_ADDRESS(ctypes.Structure):
+    pass
+
+
+IP_ADAPTER_UNICAST_ADDRESS._fields_ = [
+    ("Length", wintypes.ULONG),
+    ("Flags", wintypes.DWORD),
+    ("Next", ctypes.POINTER(IP_ADAPTER_UNICAST_ADDRESS)),
+    ("Address", SOCKET_ADDRESS),
+    ("PrefixOrigin", ctypes.c_uint),
+    ("SuffixOrigin", ctypes.c_uint),
+    ("DadState", ctypes.c_uint),
+    ("ValidLifetime", wintypes.ULONG),
+    ("PreferredLifetime", wintypes.ULONG),
+    ("LeaseLifetime", wintypes.ULONG),
+    ("OnLinkPrefixLength", ctypes.c_uint8),
+]
+
+
+class IP_ADAPTER_ADDRESSES(ctypes.Structure):
+    pass
+
+
+IP_ADAPTER_ADDRESSES._fields_ = [
+    ("Length", wintypes.ULONG),
+    ("IfIndex", wintypes.DWORD),
+    ("Next", ctypes.POINTER(IP_ADAPTER_ADDRESSES)),
+    ("AdapterName", ctypes.c_char_p),
+    ("FirstUnicastAddress", ctypes.POINTER(IP_ADAPTER_UNICAST_ADDRESS)),
+    ("FirstAnycastAddress", ctypes.c_void_p),
+    ("FirstMulticastAddress", ctypes.c_void_p),
+    ("FirstDnsServerAddress", ctypes.c_void_p),
+    ("DnsSuffix", ctypes.c_wchar_p),
+    ("Description", ctypes.c_wchar_p),
+    ("FriendlyName", ctypes.c_wchar_p),
+]
+
+
+iphlpapi = ctypes.windll.LoadLibrary("Iphlpapi")  # type: ignore
+
+
+def enumerate_interfaces_of_adapter(
+    nice_name: str, address: IP_ADAPTER_UNICAST_ADDRESS
+) -> Iterable[shared.IP]:
+
+    # Iterate through linked list and fill list
+    addresses = []  # type: List[IP_ADAPTER_UNICAST_ADDRESS]
+    while True:
+        addresses.append(address)
+        if not address.Next:
+            break
+        address = address.Next[0]
+
+    for address in addresses:
+        ip = shared.sockaddr_to_ip(address.Address.lpSockaddr)
+        if ip is None:
+            t = "sockaddr_to_ip({}) returned None"
+            raise Exception(t.format(address.Address.lpSockaddr))
+
+        network_prefix = address.OnLinkPrefixLength
+        yield shared.IP(ip, network_prefix, nice_name)
+
+
+def get_adapters(include_unconfigured: bool = False) -> Iterable[shared.Adapter]:
+
+    # Call GetAdaptersAddresses() with error and buffer size handling
+
+    addressbuffersize = wintypes.ULONG(15 * 1024)
+    retval = ERROR_BUFFER_OVERFLOW
+    while retval == ERROR_BUFFER_OVERFLOW:
+        addressbuffer = ctypes.create_string_buffer(addressbuffersize.value)
+        retval = iphlpapi.GetAdaptersAddresses(
+            wintypes.ULONG(AF_UNSPEC),
+            wintypes.ULONG(0),
+            None,
+            ctypes.byref(addressbuffer),
+            ctypes.byref(addressbuffersize),
+        )
+    if retval != NO_ERROR:
+        raise ctypes.WinError()  # type: ignore
+
+    # Iterate through adapters fill array
+    address_infos = []  # type: List[IP_ADAPTER_ADDRESSES]
+    address_info = IP_ADAPTER_ADDRESSES.from_buffer(addressbuffer)
+    while True:
+        address_infos.append(address_info)
+        if not address_info.Next:
+            break
+        address_info = address_info.Next[0]
+
+    # Iterate through unicast addresses
+    result = []  # type: List[shared.Adapter]
+    for adapter_info in address_infos:
+
+        # We don't expect non-ascii characters here, so encoding shouldn't matter
+        name = adapter_info.AdapterName.decode()
+        nice_name = adapter_info.Description
+        index = adapter_info.IfIndex
+
+        if adapter_info.FirstUnicastAddress:
+            ips = enumerate_interfaces_of_adapter(
+                adapter_info.FriendlyName, adapter_info.FirstUnicastAddress[0]
+            )
+            ips = list(ips)
+            result.append(shared.Adapter(name, nice_name, ips, index=index))
+        elif include_unconfigured:
+            result.append(shared.Adapter(name, nice_name, [], index=index))
+
+    return result

copyparty/stolen/qrcodegen.py

@@ -11,12 +11,10 @@ from __future__ import print_function, unicode_literals
 import collections
 import itertools
 
-try:
+if True:  # pylint: disable=using-constant-test
     from collections.abc import Sequence
 
     from typing import Callable, List, Optional, Tuple, Union
-except:
-    pass
 
 
 def num_char_count_bits(ver: int) -> int:

copyparty/stolen/surrogateescape.py

@@ -20,10 +20,8 @@ PY3 = sys.version_info > (3,)
 WINDOWS = platform.system() == "Windows"
 FS_ERRORS = "surrogateescape"
 
-try:
+if True:  # pylint: disable=using-constant-test
     from typing import Any
-except:
-    pass
 
 
 if PY3:

copyparty/sutil.py

@@ -6,12 +6,10 @@ from datetime import datetime
 
 from .bos import bos
 
-try:
+if True:  # pylint: disable=using-constant-test
     from typing import Any, Generator, Optional
 
     from .util import NamedLogger
-except:
-    pass
 
 
 class StreamArc(object):
@@ -25,7 +23,7 @@ class StreamArc(object):
         self.fgen = fgen
 
     def gen(self) -> Generator[Optional[bytes], None, None]:
-        pass
+        raise Exception("override me")
 
 
 def errdesc(errors: list[tuple[str, str]]) -> tuple[dict[str, Any], list[str]]:

copyparty/svchub.py

@@ -5,6 +5,7 @@ import argparse
 import base64
 import calendar
 import gzip
+import logging
 import os
 import re
 import shlex
@@ -16,15 +17,17 @@ import threading
 import time
 from datetime import datetime, timedelta
 
-try:
+# from inspect import currentframe
+# print(currentframe().f_lineno)
+
+
+if True:  # pylint: disable=using-constant-test
     from types import FrameType
 
     import typing
     from typing import Any, Optional, Union
-except:
-    pass
 
-from .__init__ import ANYWIN, MACOS, VT100, EnvParams, unicode
+from .__init__ import ANYWIN, MACOS, TYPE_CHECKING, VT100, EnvParams, unicode
 from .authsrv import AuthSrv
 from .mtag import HAVE_FFMPEG, HAVE_FFPROBE
 from .tcpsrv import TcpSrv
@@ -32,6 +35,9 @@ from .th_srv import HAVE_PIL, HAVE_VIPS, HAVE_WEBP, ThumbSrv
 from .up2k import Up2k
 from .util import (
     VERSIONS,
+    Daemon,
+    Garda,
+    HLog,
     HMaccas,
     alltrace,
     ansi_re,
@@ -41,6 +47,13 @@ from .util import (
     start_stackmon,
 )
 
+if TYPE_CHECKING:
+    try:
+        from .mdns import MDNS
+        from .ssdp import SSDPd
+    except:
+        pass
+
 
 class SvcHub(object):
     """
@@ -75,8 +88,14 @@ class SvcHub(object):
 
         self.iphash = HMaccas(os.path.join(self.E.cfg, "iphash"), 8)
 
+        # for non-http clients (ftp)
+        self.bans: dict[str, int] = {}
+        self.gpwd = Garda(self.args.ban_pw)
+        self.g404 = Garda(self.args.ban_404)
+
         if args.sss or args.s >= 3:
             args.ss = True
+            args.no_dav = True
             args.lo = args.lo or "cpp-%Y-%m%d-%H%M%S.txt.xz"
             args.ls = args.ls or "**,*,ln,p,r"
 
@@ -103,6 +122,11 @@ class SvcHub(object):
         if args.lo:
             self._setup_logfile(printed)
 
+        lg = logging.getLogger()
+        lh = HLog(self.log)
+        lg.handlers = [lh]
+        lg.setLevel(logging.DEBUG)
+
         if args.stackmon:
             start_stackmon(args.stackmon, 0)
 
@@ -137,6 +161,14 @@ class SvcHub(object):
         if args.ls:
             self.asrv.dbg_ls()
 
+        if not ANYWIN:
+            self._setlimits()
+
+        self.log("root", "max clients: {}".format(self.args.nc))
+
+        if not self._process_config():
+            raise Exception("bad config")
+
         self.tcpsrv = TcpSrv(self)
         self.up2k = Up2k(self)
 
@@ -177,10 +209,35 @@ class SvcHub(object):
 
         args.th_poke = min(args.th_poke, args.th_maxage, args.ac_maxage)
 
+        zms = ""
+        if not args.https_only:
+            zms += "d"
+        if not args.http_only:
+            zms += "D"
+
         if args.ftp or args.ftps:
             from .ftpd import Ftpd
 
             self.ftpd = Ftpd(self)
+            zms += "f" if args.ftp else "F"
+
+        if args.smb:
+            # impacket.dcerpc is noisy about listen timeouts
+            sto = socket.getdefaulttimeout()
+            socket.setdefaulttimeout(None)
+
+            from .smbd import SMB
+
+            self.smbd = SMB(self)
+            socket.setdefaulttimeout(sto)
+            self.smbd.start()
+            zms += "s"
+
+        if not args.zms:
+            args.zms = zms
+
+        self.mdns: Optional["MDNS"] = None
+        self.ssdp: Optional["SSDPd"] = None
 
         # decide which worker impl to use
         if self.check_mp_enable():
@@ -229,9 +286,61 @@ class SvcHub(object):
 
         self.up2k.init_vols()
 
-        thr = threading.Thread(target=self.sd_notify, name="sd-notify")
-        thr.daemon = True
-        thr.start()
+        Daemon(self.sd_notify, "sd-notify")
+
+    def _process_config(self) -> bool:
+        al = self.args
+
+        al.zm_on = al.zm_on or al.z_on
+        al.zs_on = al.zs_on or al.z_on
+        al.zm_off = al.zm_off or al.z_off
+        al.zs_off = al.zs_off or al.z_off
+        for n in ("zm_on", "zm_off", "zs_on", "zs_off"):
+            vs = getattr(al, n).replace(" ", ",").split(",")
+            vs = [x for x in vs if x]
+            setattr(al, n, vs)
+
+        return True
+
+    def _setlimits(self) -> None:
+        try:
+            import resource
+
+            soft, hard = [
+                x if x > 0 else 1024 * 1024
+                for x in list(resource.getrlimit(resource.RLIMIT_NOFILE))
+            ]
+        except:
+            self.log("root", "failed to read rlimits from os", 6)
+            return
+
+        if not soft or not hard:
+            t = "got bogus rlimits from os ({}, {})"
+            self.log("root", t.format(soft, hard), 6)
+            return
+
+        want = self.args.nc * 4
+        new_soft = min(hard, want)
+        if new_soft < soft:
+            return
+
+        # t = "requesting rlimit_nofile({}), have {}"
+        # self.log("root", t.format(new_soft, soft), 6)
+
+        try:
+            import resource
+
+            resource.setrlimit(resource.RLIMIT_NOFILE, (new_soft, hard))
+            soft = new_soft
+        except:
+            t = "rlimit denied; max open files: {}"
+            self.log("root", t.format(soft), 3)
+            return
+
+        if soft < want:
+            t = "max open files: {} (wanted {} for -nc {})"
+            self.log("root", t.format(soft, want, self.args.nc), 3)
+            self.args.nc = min(self.args.nc, soft // 2)
 
     def _logname(self) -> str:
         dt = datetime.utcnow()
@@ -256,10 +365,12 @@ class SvcHub(object):
         fn = sel_fn
 
         try:
-            import lzma
-
-            lh = lzma.open(fn, "wt", encoding="utf-8", errors="replace", preset=0)
+            if fn.lower().endswith(".xz"):
+                import lzma
 
+                lh = lzma.open(fn, "wt", encoding="utf-8", errors="replace", preset=0)
+            else:
+                lh = open(fn, "wt", encoding="utf-8", errors="replace")
         except:
             import codecs
 
@@ -282,9 +393,25 @@ class SvcHub(object):
     def run(self) -> None:
         self.tcpsrv.run()
 
-        thr = threading.Thread(target=self.thr_httpsrv_up, name="sig-hsrv-up2")
-        thr.daemon = True
-        thr.start()
+        if getattr(self.args, "zm", False):
+            try:
+                from .mdns import MDNS
+
+                self.mdns = MDNS(self)
+                Daemon(self.mdns.run, "mdns")
+            except:
+                self.log("root", "mdns startup failed;\n" + min_ex(), 3)
+
+        if getattr(self.args, "zs", False):
+            try:
+                from .ssdp import SSDPd
+
+                self.ssdp = SSDPd(self)
+                Daemon(self.ssdp.run, "ssdp")
+            except:
+                self.log("root", "ssdp startup failed;\n" + min_ex(), 3)
+
+        Daemon(self.thr_httpsrv_up, "sig-hsrv-up2")
 
         sigs = [signal.SIGINT, signal.SIGTERM]
         if not ANYWIN:
@@ -299,9 +426,7 @@ class SvcHub(object):
         # never lucky
         if ANYWIN:
             # msys-python probably fine but >msys-python
-            thr = threading.Thread(target=self.stop_thr, name="svchub-sig")
-            thr.daemon = True
-            thr.start()
+            Daemon(self.stop_thr, "svchub-sig")
 
             try:
                 while not self.stop_req:
@@ -321,9 +446,7 @@ class SvcHub(object):
             return "cannot reload; already in progress"
 
         self.reloading = True
-        t = threading.Thread(target=self._reload, name="reloading")
-        t.daemon = True
-        t.start()
+        Daemon(self._reload, "reloading")
         return "reload initiated"
 
     def _reload(self) -> None:
@@ -346,6 +469,17 @@ class SvcHub(object):
 
         self.shutdown()
 
+    def kill9(self, delay: float = 0.0) -> None:
+        if delay > 0.01:
+            time.sleep(delay)
+            print("component stuck; issuing sigkill")
+            time.sleep(0.1)
+
+        if ANYWIN:
+            os.system("taskkill /f /pid {}".format(os.getpid()))
+        else:
+            os.kill(os.getpid(), signal.SIGKILL)
+
     def signal_handler(self, sig: int, frame: Optional[FrameType]) -> None:
         if self.stopping:
             if self.nsigs <= 0:
@@ -355,10 +489,7 @@ class SvcHub(object):
                 except:
                     pass
 
-                if ANYWIN:
-                    os.system("taskkill /f /pid {}".format(os.getpid()))
-                else:
-                    os.kill(os.getpid(), signal.SIGKILL)
+                self.kill9()
             else:
                 self.nsigs -= 1
                 return
@@ -385,8 +516,18 @@ class SvcHub(object):
         ret = 1
         try:
             self.pr("OPYTHAT")
-            self.tcpsrv.shutdown()
+            slp = 0.0
+
+            if self.mdns:
+                Daemon(self.mdns.stop)
+                slp = time.time() + 0.5
+
+            if self.ssdp:
+                Daemon(self.ssdp.stop)
+                slp = time.time() + 0.5
+
             self.broker.shutdown()
+            self.tcpsrv.shutdown()
             self.up2k.shutdown()
             if self.thumbsrv:
                 self.thumbsrv.shutdown()
@@ -399,6 +540,14 @@ class SvcHub(object):
                     if n == 3:
                         self.pr("waiting for thumbsrv (10sec)...")
 
+            if hasattr(self, "smbd"):
+                slp = max(slp, time.time() + 0.5)
+                Daemon(self.kill9, a=(1,))
+                Daemon(self.smbd.stop)
+
+            while time.time() < slp:
+                time.sleep(0.1)
+
             self.pr("nailed it", end="")
             ret = self.retcode
         except:

copyparty/szip.py

@@ -9,12 +9,10 @@ from .bos import bos
 from .sutil import StreamArc, errdesc
 from .util import min_ex, sanitize_fn, spack, sunpack, yieldfile
 
-try:
+if True:  # pylint: disable=using-constant-test
     from typing import Any, Generator, Optional
 
     from .util import NamedLogger
-except:
-    pass
 
 
 def dostime2unix(buf: bytes) -> int:
@@ -271,6 +269,7 @@ class StreamZip(StreamArc):
             yield self._ct(buf)
 
     def gen(self) -> Generator[bytes, None, None]:
+        errf: dict[str, Any] = {}
         errors = []
         try:
             for f in self.fgen:
@@ -311,5 +310,5 @@ class StreamZip(StreamArc):
             ecdr, _ = gen_ecdr(self.items, cdir_pos, cdir_end)
             yield self._ct(ecdr)
         finally:
-            if errors:
+            if errf:
                 bos.unlink(errf["ap"])

copyparty/tcpsrv.py

@@ -6,13 +6,28 @@ import re
 import socket
 import sys
 
-from .__init__ import ANYWIN, MACOS, PY2, TYPE_CHECKING, VT100, unicode
+from .__init__ import ANYWIN, PY2, TYPE_CHECKING, VT100, unicode
 from .stolen.qrcodegen import QrCode
-from .util import chkcmd, sunpack, termsize
+from .util import (
+    E_ACCESS,
+    E_ADDR_IN_USE,
+    E_ADDR_NOT_AVAIL,
+    E_UNREACH,
+    Netdev,
+    min_ex,
+    sunpack,
+    termsize,
+)
+
+if True:
+    from typing import Generator
 
 if TYPE_CHECKING:
     from .svchub import SvcHub
 
+if not hasattr(socket, "IPPROTO_IPV6"):
+    setattr(socket, "IPPROTO_IPV6", 41)
+
 
 class TcpSrv(object):
     """
@@ -30,47 +45,98 @@ class TcpSrv(object):
 
         self.stopping = False
         self.srv: list[socket.socket] = []
+        self.bound: list[tuple[str, int]] = []
         self.nsrv = 0
         self.qr = ""
+        pad = False
         ok: dict[str, list[int]] = {}
         for ip in self.args.i:
-            ok[ip] = []
+            if ip == "::":
+                if socket.has_ipv6:
+                    ips = ["::", "0.0.0.0"]
+                    dual = True
+                else:
+                    ips = ["0.0.0.0"]
+                    dual = False
+            else:
+                ips = [ip]
+                dual = False
+
+            for ipa in ips:
+                ok[ipa] = []
+
             for port in self.args.p:
-                self.nsrv += 1
+                successful_binds = 0
                 try:
-                    self._listen(ip, port)
-                    ok[ip].append(port)
+                    for ipa in ips:
+                        try:
+                            self._listen(ipa, port)
+                            ok[ipa].append(port)
+                            successful_binds += 1
+                        except:
+                            if dual and ":" in ipa:
+                                t = "listen on IPv6 [{}] failed; trying IPv4 {}...\n{}"
+                                self.log("tcpsrv", t.format(ipa, ips[1], min_ex()), 3)
+                                pad = True
+                                continue
+
+                            # binding 0.0.0.0 after :: fails on dualstack
+                            # but is necessary on non-dualstakc
+                            if successful_binds:
+                                continue
+
+                            raise
+
                 except Exception as ex:
                     if self.args.ign_ebind or self.args.ign_ebind_all:
                         t = "could not listen on {}:{}: {}"
                         self.log("tcpsrv", t.format(ip, port, ex), c=3)
+                        pad = True
                     else:
                         raise
 
         if not self.srv and not self.args.ign_ebind_all:
             raise Exception("could not listen on any of the given interfaces")
 
-        if self.nsrv != len(self.srv):
+        if pad:
             self.log("tcpsrv", "")
 
-        ip = "127.0.0.1"
-        eps = {ip: "local only"}
-        nonlocals = [x for x in self.args.i if x != ip]
+        eps = {
+            "127.0.0.1": Netdev("127.0.0.1", 0, "", "local only"),
+            "::1": Netdev("::1", 0, "", "local only"),
+        }
+        nonlocals = [x for x in self.args.i if x not in [k.split("/")[0] for k in eps]]
         if nonlocals:
-            eps = self.detect_interfaces(self.args.i)
+            try:
+                self.netdevs = self.detect_interfaces(self.args.i)
+            except:
+                t = "failed to discover server IP addresses\n"
+                self.log("tcpsrv", t + min_ex(), 3)
+                self.netdevs = {}
+
+            eps.update({k.split("/")[0]: v for k, v in self.netdevs.items()})
             if not eps:
                 for x in nonlocals:
-                    eps[x] = "external"
+                    eps[x] = Netdev(x, 0, "", "external")
+        else:
+            self.netdevs = {}
 
-        qr1 = {}
-        qr2 = {}
+        qr1: dict[str, list[int]] = {}
+        qr2: dict[str, list[int]] = {}
         msgs = []
         title_tab: dict[str, dict[str, int]] = {}
         title_vars = [x[1:] for x in self.args.wintitle.split(" ") if x.startswith("$")]
         t = "available @ {}://{}:{}/  (\033[33m{}\033[0m)"
         for ip, desc in sorted(eps.items(), key=lambda x: x[1]):
+            if ip.startswith("fe80"):
+                continue
+
             for port in sorted(self.args.p):
-                if port not in ok.get(ip, ok.get("0.0.0.0", [])):
+                if (
+                    port not in ok.get(ip, [])
+                    and port not in ok.get("::", [])
+                    and port not in ok.get("0.0.0.0", [])
+                ):
                     continue
 
                 proto = " http"
@@ -79,7 +145,8 @@ class TcpSrv(object):
                 elif self.args.https_only or port == 443:
                     proto = "https"
 
-                msgs.append(t.format(proto, ip, port, desc))
+                hip = "[{}]".format(ip) if ":" in ip else ip
+                msgs.append(t.format(proto, hip, port, desc))
 
                 is_ext = "external" in unicode(desc)
                 qrt = qr1 if is_ext else qr2
@@ -114,45 +181,82 @@ class TcpSrv(object):
                         title_tab[tk] = {tv: 1}
 
         if msgs:
-            msgs[-1] += "\n"
             for t in msgs:
                 self.log("tcpsrv", t)
 
         if self.args.wintitle:
             self._set_wintitle(title_tab)
+        else:
+            print("\n", end="")
 
         if self.args.qr or self.args.qrs:
             self.qr = self._qr(qr1, qr2)
 
     def _listen(self, ip: str, port: int) -> None:
-        srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        ipv = socket.AF_INET6 if ":" in ip else socket.AF_INET
+        srv = socket.socket(ipv, socket.SOCK_STREAM)
+        try:
+            srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
+        except:
+            pass
         srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
         srv.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
         srv.settimeout(None)  # < does not inherit, ^ does
+
+        try:
+            srv.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, False)
+        except:
+            pass  # will create another ipv4 socket instead
+
         try:
             srv.bind((ip, port))
             self.srv.append(srv)
         except (OSError, socket.error) as ex:
-            if ex.errno in [98, 48]:
+            if ex.errno in E_ADDR_IN_USE:
                 e = "\033[1;31mport {} is busy on interface {}\033[0m".format(port, ip)
-            elif ex.errno in [99, 49]:
+            elif ex.errno in E_ADDR_NOT_AVAIL:
                 e = "\033[1;31minterface {} does not exist\033[0m".format(ip)
             else:
                 raise
             raise Exception(e)
 
     def run(self) -> None:
+        all_eps = [x.getsockname()[:2] for x in self.srv]
+        bound: list[tuple[str, int]] = []
+        srvs: list[socket.socket] = []
         for srv in self.srv:
-            srv.listen(self.args.nc)
-            ip, port = srv.getsockname()
+            ip, port = srv.getsockname()[:2]
+            try:
+                srv.listen(self.args.nc)
+            except:
+                if ip == "0.0.0.0" and ("::", port) in bound:
+                    # dualstack
+                    srv.close()
+                    continue
+
+                if ip == "::" and ("0.0.0.0", port) in all_eps:
+                    # no ipv6
+                    srv.close()
+                    continue
+
+                raise
+
+            bound.append((ip, port))
+            srvs.append(srv)
             fno = srv.fileno()
-            msg = "listening @ {}:{}  f{} p{}".format(ip, port, fno, os.getpid())
+            hip = "[{}]".format(ip) if ":" in ip else ip
+            msg = "listening @ {}:{}  f{} p{}".format(hip, port, fno, os.getpid())
             self.log("tcpsrv", msg)
             if self.args.q:
                 print(msg)
 
             self.hub.broker.say("listen", srv)
 
+        self.srv = srvs
+        self.bound = bound
+        self.nsrv = len(srvs)
+        self.hub.broker.say("set_netdevs", self.netdevs)
+
     def shutdown(self) -> None:
         self.stopping = True
         try:
@@ -163,180 +267,88 @@ class TcpSrv(object):
 
         self.log("tcpsrv", "ok bye")
 
-    def ips_linux_ifconfig(self) -> dict[str, str]:
-        # for termux
-        try:
-            txt, _ = chkcmd(["ifconfig"])
-        except:
-            return {}
+    def detect_interfaces(self, listen_ips: list[str]) -> dict[str, Netdev]:
+        from .stolen.ifaddr import get_adapters
 
-        eps: dict[str, str] = {}
-        dev = None
-        ip = None
-        up = None
-        for ln in (txt + "\n").split("\n"):
-            if not ln.strip() and dev and ip:
-                eps[ip] = dev + ("" if up else ", \033[31mLINK-DOWN")
-                dev = ip = up = None
+        nics = get_adapters(True)
+        eps: dict[str, Netdev] = {}
+        for nic in nics:
+            for nip in nic.ips:
+                ipa = nip.ip[0] if ":" in str(nip.ip) else nip.ip
+                sip = "{}/{}".format(ipa, nip.network_prefix)
+                if sip.startswith("169.254"):
+                    # browsers dont impl linklocal
+                    continue
+
+                nd = Netdev(sip, nic.index or 0, nic.nice_name, "")
+                eps[sip] = nd
+                try:
+                    idx = socket.if_nametoindex(nd.name)
+                    if idx and idx != nd.idx:
+                        t = "netdev idx mismatch; ifaddr={} cpython={}"
+                        self.log("tcpsrv", t.format(nd.idx, idx), 3)
+                        nd.idx = idx
+                except:
+                    pass
+
+        if "0.0.0.0" not in listen_ips and "::" not in listen_ips:
+            eps = {k: v for k, v in eps.items() if k.split("/")[0] in listen_ips}
+
+        try:
+            ext_devs = list(self._extdevs_nix())
+            ext_ips = [k for k, v in eps.items() if v.name in ext_devs]
+            ext_ips = [x.split("/")[0] for x in ext_ips]
+            if not ext_ips:
+                raise Exception()
+        except:
+            rt = self._defroute()
+            ext_ips = [rt] if rt else []
+
+        for lip in listen_ips:
+            if not ext_ips or lip not in ["0.0.0.0", "::"] + ext_ips:
                 continue
 
-            if ln == ln.lstrip():
-                dev = re.split(r"[: ]", ln)[0]
-
-            if "UP" in re.split(r"[<>, \t]", ln):
-                up = True
-
-            m = re.match(r"^\s+inet\s+([^ ]+)", ln)
-            if m:
-                ip = m.group(1)
+            desc = "\033[32mexternal"
+            ips = ext_ips if lip in ["0.0.0.0", "::"] else [lip]
+            for ip in ips:
+                ip = next((x for x in eps if x.startswith(ip + "/")), "")
+                if ip and "external" not in eps[ip].desc:
+                    eps[ip].desc += ", " + desc
 
         return eps
 
-    def ips_linux(self) -> dict[str, str]:
-        try:
-            txt, _ = chkcmd(["ip", "addr"])
-        except:
-            return self.ips_linux_ifconfig()
+    def _extdevs_nix(self) -> Generator[str, None, None]:
+        with open("/proc/net/route", "rb") as f:
+            next(f)
+            for ln in f:
+                r = ln.decode("utf-8").strip().split()
+                if r[1] == "0" * 8 and int(r[3], 16) & 2:
+                    yield r[0]
 
-        r = re.compile(r"^\s+inet ([^ ]+)/.* (.*)")
-        ri = re.compile(r"^\s*[0-9]+\s*:.*")
-        up = False
-        eps: dict[str, str] = {}
-        for ln in txt.split("\n"):
-            if ri.match(ln):
-                up = "UP" in re.split("[>,< ]", ln)
-
-            try:
-                ip, dev = r.match(ln.rstrip()).groups()  # type: ignore
-                eps[ip] = dev + ("" if up else ", \033[31mLINK-DOWN")
-            except:
-                pass
-
-        return eps
-
-    def ips_macos(self) -> dict[str, str]:
-        eps: dict[str, str] = {}
-        try:
-            txt, _ = chkcmd(["ifconfig"])
-        except:
-            return eps
-
-        rdev = re.compile(r"^([^ ]+):")
-        rip = re.compile(r"^\tinet ([0-9\.]+) ")
-        dev = "UNKNOWN"
-        for ln in txt.split("\n"):
-            m = rdev.match(ln)
-            if m:
-                dev = m.group(1)
-
-            m = rip.match(ln)
-            if m:
-                eps[m.group(1)] = dev
-                dev = "UNKNOWN"
-
-        return eps
-
-    def ips_windows_ipconfig(self) -> tuple[dict[str, str], set[str]]:
-        eps: dict[str, str] = {}
-        offs: set[str] = set()
-        try:
-            txt, _ = chkcmd(["ipconfig"])
-        except:
-            return eps, offs
-
-        rdev = re.compile(r"(^[^ ].*):$")
-        rip = re.compile(r"^ +IPv?4? [^:]+: *([0-9\.]{7,15})$")
-        roff = re.compile(r".*: Media disconnected$")
-        dev = None
-        for ln in txt.replace("\r", "").split("\n"):
-            m = rdev.match(ln)
-            if m:
-                if dev and dev not in eps.values():
-                    offs.add(dev)
-
-                dev = m.group(1).split(" adapter ", 1)[-1]
-
-            if dev and roff.match(ln):
-                offs.add(dev)
-                dev = None
-
-            m = rip.match(ln)
-            if m and dev:
-                eps[m.group(1)] = dev
-                dev = None
-
-        if dev and dev not in eps.values():
-            offs.add(dev)
-
-        return eps, offs
-
-    def ips_windows_netsh(self) -> dict[str, str]:
-        eps: dict[str, str] = {}
-        try:
-            txt, _ = chkcmd("netsh interface ip show address".split())
-        except:
-            return eps
-
-        rdev = re.compile(r'.* "([^"]+)"$')
-        rip = re.compile(r".* IP\b.*: +([0-9\.]{7,15})$")
-        dev = None
-        for ln in txt.replace("\r", "").split("\n"):
-            m = rdev.match(ln)
-            if m:
-                dev = m.group(1)
-
-            m = rip.match(ln)
-            if m and dev:
-                eps[m.group(1)] = dev
-
-        return eps
-
-    def detect_interfaces(self, listen_ips: list[str]) -> dict[str, str]:
-        if MACOS:
-            eps = self.ips_macos()
-        elif ANYWIN:
-            eps, off = self.ips_windows_ipconfig()  # sees more interfaces + link state
-            eps.update(self.ips_windows_netsh())  # has better names
-            for k, v in eps.items():
-                if v in off:
-                    eps[k] += ", \033[31mLINK-DOWN"
-        else:
-            eps = self.ips_linux()
-
-        if "0.0.0.0" not in listen_ips:
-            eps = {k: v for k, v in eps.items() if k in listen_ips}
-
-        default_route = None
+    def _defroute(self) -> str:
+        ret = ""
         s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
         for ip in [
-            "10.255.255.255",
-            "172.31.255.255",
-            "192.168.255.255",
-            "239.255.255.255",
+            "10.254.39.23",
+            "172.31.39.23",
+            "192.168.39.23",
+            "239.254.39.23",
+            "169.254.39.23",
             # could add 1.1.1.1 as a final fallback
             # but external connections is kinshi
         ]:
             try:
                 s.connect((ip, 1))
-                default_route = s.getsockname()[0]
+                ret = s.getsockname()[0]
                 break
             except (OSError, socket.error) as ex:
-                if ex.errno == 13:
+                if ex.errno in E_ACCESS:
                     self.log("tcpsrv", "eaccess {} (trying next)".format(ip))
-                elif ex.errno not in [101, 10065, 10051]:
+                elif ex.errno not in E_UNREACH:
                     self.log("tcpsrv", "route lookup failed; err {}".format(ex.errno))
 
         s.close()
-
-        for lip in listen_ips:
-            if default_route and lip in ["0.0.0.0", default_route]:
-                desc = "\033[32mexternal"
-                try:
-                    eps[default_route] += ", " + desc
-                except:
-                    eps[default_route] = desc
-
-        return eps
+        return ret
 
     def _set_wintitle(self, vs: dict[str, dict[str, int]]) -> None:
         vs["all"] = vs.get("all", {"Local-Only": 1})
@@ -370,23 +382,33 @@ class TcpSrv(object):
 
             title += "{} ".format(p)
 
-        print("\033]0;{}\033\\".format(title), file=sys.stderr, end="")
+        print("\033]0;{}\033\\\n".format(title), file=sys.stderr, end="")
         sys.stderr.flush()
 
     def _qr(self, t1: dict[str, list[int]], t2: dict[str, list[int]]) -> str:
         ip = None
-        for ip in list(t1) + list(t2):
-            if ip.startswith(self.args.qri):
+        ips = list(t1) + list(t2)
+        qri = self.args.qri
+        if self.args.zm and not qri:
+            name = self.args.name + ".local"
+            t1[name] = next(v for v in (t1 or t2).values())
+            ips = [name] + ips
+
+        for ip in ips:
+            if ip.startswith(qri) or qri == ".":
                 break
             ip = ""
 
         if not ip:
             # maybe /bin/ip is missing or smth
-            ip = self.args.qri
+            ip = qri
 
         if not ip:
             return ""
 
+        if ":" in ip:
+            ip = "[{}]".format(ip)
+
         if self.args.http_only:
             https = ""
         elif self.args.https_only:
@@ -420,13 +442,22 @@ class TcpSrv(object):
         if not VT100:
             return "{}\n{}".format(txt, qr)
 
+        halfc = "\033[40;48;5;{0}m{1}\033[47;48;5;{2}m"
+        if not fg:
+            halfc = "\033[0;40m{1}\033[0;47m"
+
         def ansify(m: re.Match) -> str:
-            t = "\033[40;48;5;{}m{}\033[47;48;5;{}m"
-            return t.format(fg, " " * len(m.group(1)), bg)
+            return halfc.format(fg, " " * len(m.group(1)), bg)
 
         if zoom > 1:
             qr = re.sub("(█+)", ansify, qr)
 
         qr = qr.replace("\n", "\033[K\n") + "\033[K"  # win10do
-        t = "{} \033[0;38;5;{};48;5;{}m\033[J\n{}\033[999G\033[0m\033[J"
-        return t.format(txt, fg, bg, qr)
+        cc = " \033[0;38;5;{0};47;48;5;{1}m" if fg else " \033[0;30;47m"
+        t = cc + "\n{2}\033[999G\033[0m\033[J"
+        t = t.format(fg, bg, qr)
+        if ANYWIN:
+            # prevent color loss on terminal resize
+            t = t.replace("\n", "`\n`")
+
+        return txt + t

copyparty/th_cli.py

@@ -9,10 +9,8 @@ from .bos import bos
 from .th_srv import HAVE_WEBP, thumb_path
 from .util import Cooldown
 
-try:
+if True:  # pylint: disable=using-constant-test
     from typing import Optional, Union
-except:
-    pass
 
 if TYPE_CHECKING:
     from .httpsrv import HttpSrv

copyparty/th_srv.py

@@ -14,12 +14,20 @@ from queue import Queue
 from .__init__ import TYPE_CHECKING
 from .bos import bos
 from .mtag import HAVE_FFMPEG, HAVE_FFPROBE, ffprobe
-from .util import BytesIO, Cooldown, Pebkac, fsenc, min_ex, runcmd, statdir, vsplit
+from .util import (
+    BytesIO,
+    Cooldown,
+    Daemon,
+    Pebkac,
+    fsenc,
+    min_ex,
+    runcmd,
+    statdir,
+    vsplit,
+)
 
-try:
+if True:  # pylint: disable=using-constant-test
     from typing import Optional, Union
-except:
-    pass
 
 if TYPE_CHECKING:
     from .svchub import SvcHub
@@ -106,11 +114,7 @@ class ThumbSrv(object):
 
         self.q: Queue[Optional[tuple[str, str]]] = Queue(self.nthr * 4)
         for n in range(self.nthr):
-            thr = threading.Thread(
-                target=self.worker, name="thumb-{}-{}".format(n, self.nthr)
-            )
-            thr.daemon = True
-            thr.start()
+            Daemon(self.worker, "thumb-{}-{}".format(n, self.nthr))
 
         want_ff = not self.args.no_vthumb or not self.args.no_athumb
         if want_ff and (not HAVE_FFMPEG or not HAVE_FFPROBE):
@@ -126,9 +130,7 @@ class ThumbSrv(object):
             self.log(msg, c=3)
 
         if self.args.th_clean:
-            t = threading.Thread(target=self.cleaner, name="thumb.cln")
-            t.daemon = True
-            t.start()
+            Daemon(self.cleaner, "thumb.cln")
 
         self.fmt_pil, self.fmt_vips, self.fmt_ffi, self.fmt_ffv, self.fmt_ffa = [
             set(y.split(","))
@@ -269,7 +271,7 @@ class ThumbSrv(object):
                 except Exception as ex:
                     msg = "{} could not create thumbnail of {}\n{}"
                     msg = msg.format(fun.__name__, abspath, min_ex())
-                    c: Union[str, int] = 1 if "<Signals.SIG" in msg else "1;30"
+                    c: Union[str, int] = 1 if "<Signals.SIG" in msg else "90"
                     self.log(msg, c)
                     if getattr(ex, "returncode", 0) != 321:
                         with open(tpath, "wb") as _:
@@ -323,7 +325,7 @@ class ThumbSrv(object):
             try:
                 im = self.fancy_pillow(im)
             except Exception as ex:
-                self.log("fancy_pillow {}".format(ex), "1;30")
+                self.log("fancy_pillow {}".format(ex), "90")
                 im.thumbnail(self.res)
 
             fmts = ["RGB", "L"]
@@ -423,7 +425,7 @@ class ThumbSrv(object):
         if not ret:
             return
 
-        c: Union[str, int] = "1;30"
+        c: Union[str, int] = "90"
         t = "FFmpeg failed (probably a corrupt video file):\n"
         if (
             (not self.args.th_ff_jpg or time.time() - int(self.args.th_ff_jpg) < 60)
@@ -621,7 +623,7 @@ class ThumbSrv(object):
 
     def _clean(self, cat: str, thumbpath: str) -> int:
         # self.log("cln {}".format(thumbpath))
-        exts = ["jpg", "webp"] if cat == "th" else ["opus", "caf"]
+        exts = ["jpg", "webp", "png"] if cat == "th" else ["opus", "caf"]
         maxage = getattr(self.args, cat + "_maxage")
         now = time.time()
         prev_b64 = None

copyparty/u2idx.py

@@ -11,7 +11,16 @@ from operator import itemgetter
 from .__init__ import ANYWIN, TYPE_CHECKING, unicode
 from .bos import bos
 from .up2k import up2k_wark_from_hashlist
-from .util import HAVE_SQLITE3, Pebkac, absreal, gen_filekey, min_ex, quotep, s3dec
+from .util import (
+    HAVE_SQLITE3,
+    Daemon,
+    Pebkac,
+    absreal,
+    gen_filekey,
+    min_ex,
+    quotep,
+    s3dec,
+)
 
 if HAVE_SQLITE3:
     import sqlite3
@@ -21,10 +30,8 @@ try:
 except:
     pass
 
-try:
+if True:  # pylint: disable=using-constant-test
     from typing import Any, Optional, Union
-except:
-    pass
 
 if TYPE_CHECKING:
     from .httpconn import HttpConn
@@ -190,7 +197,7 @@ class U2idx(object):
                     v = "exists(select 1 from mt where mt.w = mtw and " + vq
 
                 else:
-                    raise Pebkac(400, "invalid key [" + v + "]")
+                    raise Pebkac(400, "invalid key [{}]".format(v))
 
                 q += v + " "
                 continue
@@ -270,16 +277,7 @@ class U2idx(object):
         self.active_id = "{:.6f}_{}".format(
             time.time(), threading.current_thread().ident
         )
-        thr = threading.Thread(
-            target=self.terminator,
-            args=(
-                self.active_id,
-                done_flag,
-            ),
-            name="u2idx-terminator",
-        )
-        thr.daemon = True
-        thr.start()
+        Daemon(self.terminator, "u2idx-terminator", (self.active_id, done_flag))
 
         if not uq or not uv:
             uq = "select * from up"

copyparty/up2k.py

@@ -2,6 +2,7 @@
 from __future__ import print_function, unicode_literals
 
 import base64
+import errno
 import gzip
 import hashlib
 import json
@@ -28,6 +29,7 @@ from .mtag import MParser, MTag
 from .util import (
     HAVE_SQLITE3,
     SYMTIME,
+    Daemon,
     MTHash,
     Pebkac,
     ProgressPrinter,
@@ -58,10 +60,8 @@ if HAVE_SQLITE3:
 
 DB_VER = 5
 
-try:
+if True:  # pylint: disable=using-constant-test
     from typing import Any, Optional, Pattern, Union
-except:
-    pass
 
 if TYPE_CHECKING:
     from .svchub import SvcHub
@@ -152,9 +152,7 @@ class Up2k(object):
         if ANYWIN:
             # usually fails to set lastmod too quickly
             self.lastmod_q: list[tuple[str, int, tuple[int, int], bool]] = []
-            thr = threading.Thread(target=self._lastmodder, name="up2k-lastmod")
-            thr.daemon = True
-            thr.start()
+            Daemon(self._lastmodder, "up2k-lastmod")
 
         self.fstab = Fstab(self.log_func)
 
@@ -170,9 +168,7 @@ class Up2k(object):
         if self.args.no_fastboot:
             return
 
-        t = threading.Thread(target=self.deferred_init, name="up2k-deferred-init")
-        t.daemon = True
-        t.start()
+        Daemon(self.deferred_init, "up2k-deferred-init")
 
     def reload(self) -> None:
         self.gid += 1
@@ -187,32 +183,21 @@ class Up2k(object):
         if not self.pp and self.args.exit == "idx":
             return self.hub.sigterm()
 
-        thr = threading.Thread(target=self._snapshot, name="up2k-snapshot")
-        thr.daemon = True
-        thr.start()
-
+        Daemon(self._snapshot, "up2k-snapshot")
         if have_e2d:
-            thr = threading.Thread(target=self._hasher, name="up2k-hasher")
-            thr.daemon = True
-            thr.start()
-
-            thr = threading.Thread(target=self._sched_rescan, name="up2k-rescan")
-            thr.daemon = True
-            thr.start()
-
+            Daemon(self._hasher, "up2k-hasher")
+            Daemon(self._sched_rescan, "up2k-rescan")
             if self.mtag:
                 for n in range(max(1, self.args.mtag_mt)):
-                    name = "tagger-{}".format(n)
-                    thr = threading.Thread(target=self._tagger, name=name)
-                    thr.daemon = True
-                    thr.start()
+                    Daemon(self._tagger, "tagger-{}".format(n))
 
-                thr = threading.Thread(target=self._run_all_mtp, name="up2k-mtp-init")
-                thr.daemon = True
-                thr.start()
+                Daemon(self._run_all_mtp, "up2k-mtp-init")
 
     def log(self, msg: str, c: Union[int, str] = 0) -> None:
-        self.log_func("up2k", msg + "\033[K", c)
+        if self.pp:
+            msg += "\033[K"
+
+        self.log_func("up2k", msg, c)
 
     def _block(self, why: str) -> None:
         self.blocked = why
@@ -255,13 +240,11 @@ class Up2k(object):
             return "cannot initiate; scan is already in progress"
 
         args = (all_vols, scan_vols)
-        t = threading.Thread(
-            target=self.init_indexes,
-            args=args,
-            name="up2k-rescan-{}".format(scan_vols[0] if scan_vols else "all"),
+        Daemon(
+            self.init_indexes,
+            "up2k-rescan-{}".format(scan_vols[0] if scan_vols else "all"),
+            args,
         )
-        t.daemon = True
-        t.start()
         return ""
 
     def _sched_rescan(self) -> None:
@@ -580,7 +563,7 @@ class Up2k(object):
         if self.mtag:
             t = "online (running mtp)"
             if scan_vols:
-                thr = threading.Thread(target=self._run_all_mtp, name="up2k-mtp-scan")
+                thr = Daemon(self._run_all_mtp, "up2k-mtp-scan", r=False)
         else:
             self.pp = None
             t = "online, idle"
@@ -589,7 +572,6 @@ class Up2k(object):
             self.volstate[vol.vpath] = t
 
         if thr:
-            thr.daemon = True
             thr.start()
 
         return have_e2d
@@ -613,7 +595,7 @@ class Up2k(object):
 
         ft = "\033[0;32m{}{:.0}"
         ff = "\033[0;35m{}{:.0}"
-        fv = "\033[0;36m{}:\033[1;30m{}"
+        fv = "\033[0;36m{}:\033[90m{}"
         fx = set(("html_head",))
         a = [
             (ft if v is True else ff if v is False else fv).format(k, str(v))
@@ -956,6 +938,7 @@ class Up2k(object):
             if n:
                 t = "forgetting {} shadowed autoindexed files in [{}] > [{}]"
                 self.log(t.format(n, top, sh_rd))
+                assert sh_erd
 
                 q = "delete from dh where (d = ? or d like ?||'%')"
                 db.c.execute(q, (sh_erd, sh_erd + "/"))
@@ -1440,7 +1423,7 @@ class Up2k(object):
 
         if self.args.mtag_vv:
             t = "parsers for {}: \033[0m{}"
-            self.log(t.format(ptop, list(parsers.keys())), "1;30")
+            self.log(t.format(ptop, list(parsers.keys())), "90")
 
         self.mtp_parsers[ptop] = parsers
 
@@ -1571,7 +1554,7 @@ class Up2k(object):
             all_parsers = self.mtp_parsers[ptop]
         except:
             if self.args.mtag_vv:
-                self.log("no mtp defined for {}".format(ptop), "1;30")
+                self.log("no mtp defined for {}".format(ptop), "90")
             return {}
 
         entags = self.entags[ptop]
@@ -1582,14 +1565,14 @@ class Up2k(object):
                     # is audio, require non-audio?
                     if v.audio == "n":
                         if self.args.mtag_vv:
-                            t = "skip mtp {}; is no-audio, have audio"
-                            self.log(t.format(k), "1;30")
+                            t = "skip mtp {}; want no-audio, got audio"
+                            self.log(t.format(k), "90")
                         continue
                 # is not audio, require audio?
                 elif v.audio == "y":
                     if self.args.mtag_vv:
-                        t = "skip mtp {}; is audio, have no-audio"
-                        self.log(t.format(k), "1;30")
+                        t = "skip mtp {}; want audio, got no-audio"
+                        self.log(t.format(k), "90")
                     continue
 
             if v.ext:
@@ -1601,8 +1584,8 @@ class Up2k(object):
 
                 if not match:
                     if self.args.mtag_vv:
-                        t = "skip mtp {}; need file-ext {}, have {}"
-                        self.log(t.format(k, v.ext, abspath.rsplit(".")[-1]), "1;30")
+                        t = "skip mtp {}; want file-ext {}, got {}"
+                        self.log(t.format(k, v.ext, abspath.rsplit(".")[-1]), "90")
                     continue
 
             parsers[k] = v
@@ -1621,11 +1604,7 @@ class Up2k(object):
 
         mpool: Queue[Mpqe] = Queue(nw)
         for _ in range(nw):
-            thr = threading.Thread(
-                target=self._tag_thr, args=(mpool,), name="up2k-mpool"
-            )
-            thr.daemon = True
-            thr.start()
+            Daemon(self._tag_thr, "up2k-mpool", (mpool,))
 
         return mpool
 
@@ -1650,13 +1629,13 @@ class Up2k(object):
                 if not qe.mtp:
                     if self.args.mtag_vv:
                         t = "tag-thr: {}({})"
-                        self.log(t.format(self.mtag.backend, qe.abspath), "1;30")
+                        self.log(t.format(self.mtag.backend, qe.abspath), "90")
 
                     tags = self.mtag.get(qe.abspath)
                 else:
                     if self.args.mtag_vv:
                         t = "tag-thr: {}({})"
-                        self.log(t.format(list(qe.mtp.keys()), qe.abspath), "1;30")
+                        self.log(t.format(list(qe.mtp.keys()), qe.abspath), "90")
 
                     tags = self.mtag.get_bin(qe.mtp, qe.abspath, qe.oth_tags)
                     vtags = [
@@ -1922,12 +1901,23 @@ class Up2k(object):
         sprs = self.fstab.get(pdir) != "ng"
 
         with self.mutex:
-            cur = self.cur.get(cj["ptop"])
-            reg = self.registry[cj["ptop"]]
+            ptop = cj["ptop"]
+            jcur = self.cur.get(ptop)
+            reg = self.registry[ptop]
             vfs = self.asrv.vfs.all_vols[cj["vtop"]]
             n4g = vfs.flags.get("noforget")
-            lost: list[tuple[str, str]] = []
-            if cur:
+            lost: list[tuple["sqlite3.Cursor", str, str]] = []
+
+            vols = [(ptop, jcur)] if jcur else []
+            if vfs.flags.get("xlink"):
+                vols += [(k, v) for k, v in self.cur.items() if k != ptop]
+
+            alts: list[tuple[int, int, dict[str, Any]]] = []
+            for ptop, cur in vols:
+                allv = self.asrv.vfs.all_vols
+                cvfs = next((v for v in allv.values() if v.realpath == ptop), vfs)
+                vtop = cj["vtop"] if cur == jcur else cvfs.vpath
+
                 if self.no_expr_idx:
                     q = r"select * from up where w = ?"
                     argv = [wark]
@@ -1935,13 +1925,12 @@ class Up2k(object):
                     q = r"select * from up where substr(w,1,16) = ? and w = ?"
                     argv = [wark[:16], wark]
 
-                alts: list[tuple[int, int, dict[str, Any]]] = []
-                cur = cur.execute(q, tuple(argv))
-                for _, dtime, dsize, dp_dir, dp_fn, ip, at in cur:
+                c2 = cur.execute(q, tuple(argv))
+                for _, dtime, dsize, dp_dir, dp_fn, ip, at in c2:
                     if dp_dir.startswith("//") or dp_fn.startswith("//"):
                         dp_dir, dp_fn = s3dec(dp_dir, dp_fn)
 
-                    dp_abs = "/".join([cj["ptop"], dp_dir, dp_fn])
+                    dp_abs = "/".join([ptop, dp_dir, dp_fn])
                     try:
                         st = bos.stat(dp_abs)
                         if stat.S_ISLNK(st.st_mode):
@@ -1951,14 +1940,14 @@ class Up2k(object):
                         if n4g:
                             st = os.stat_result((0, -1, -1, 0, 0, 0, 0, 0, 0, 0))
                         else:
-                            lost.append((dp_dir, dp_fn))
+                            lost.append((cur, dp_dir, dp_fn))
                             continue
 
                     j = {
                         "name": dp_fn,
                         "prel": dp_dir,
-                        "vtop": cj["vtop"],
-                        "ptop": cj["ptop"],
+                        "vtop": vtop,
+                        "ptop": ptop,
                         "sprs": sprs,  # dontcare; finished anyways
                         "size": dsize,
                         "lmod": dtime,
@@ -1979,20 +1968,33 @@ class Up2k(object):
                     )
                     alts.append((score, -len(alts), j))
 
-                job = sorted(alts, reverse=True)[0][2] if alts else None
-                if job and wark in reg:
-                    # self.log("pop " + wark + "  " + job["name"] + " handle_json db", 4)
-                    del reg[wark]
+            job = sorted(alts, reverse=True)[0][2] if alts else None
+            if job and wark in reg:
+                # self.log("pop " + wark + "  " + job["name"] + " handle_json db", 4)
+                del reg[wark]
 
-                if lost:
-                    for dp_dir, dp_fn in lost:
-                        self.db_rm(cur, dp_dir, dp_fn)
+            if lost:
+                c2 = None
+                for cur, dp_dir, dp_fn in lost:
+                    self.db_rm(cur, dp_dir, dp_fn)
+                    if c2 and c2 != cur:
+                        c2.connection.commit()
 
-                    cur.connection.commit()
+                    c2 = cur
+
+                assert c2
+                c2.connection.commit()
+
+            cur = jcur
+            ptop = None  # use cj or job as appropriate
 
             if job or wark in reg:
                 job = job or reg[wark]
-                if job["prel"] == cj["prel"] and job["name"] == cj["name"]:
+                if (
+                    job["ptop"] == cj["ptop"]
+                    and job["prel"] == cj["prel"]
+                    and job["name"] == cj["name"]
+                ):
                     # ensure the files haven't been deleted manually
                     names = [job[x] for x in ["name", "tnam"] if x in job]
                     for fn in names:
@@ -2026,13 +2028,13 @@ class Up2k(object):
                             except:
                                 self.dupesched[src] = [dupe]
 
-                        raise Pebkac(400, err)
+                        raise Pebkac(422, err)
 
-                    elif "nodupe" in self.flags[job["ptop"]]:
+                    elif "nodupe" in self.flags[cj["ptop"]]:
                         self.log("dupe-reject:\n  {0}\n  {1}".format(src, dst))
                         err = "upload rejected, file already exists:\n"
                         err += "/" + quotep(vsrc) + " "
-                        raise Pebkac(400, err)
+                        raise Pebkac(409, err)
                     else:
                         # symlink to the client-provided name,
                         # returning the previous upload info
@@ -2670,7 +2672,7 @@ class Up2k(object):
         try:
             atomic_move(sabs, dabs)
         except OSError as ex:
-            if ex.errno != 18:
+            if ex.errno != errno.EXDEV:
                 raise
 
             self.log("cross-device move:\n  {}\n  {}".format(sabs, dabs))
@@ -3012,12 +3014,15 @@ class Up2k(object):
             if x["need"] and now - x["poke"] > self.snap_discard_interval
         ]
 
-        lost = [
-            x
-            for x in reg.values()
-            if x["need"]
-            and not bos.path.exists(os.path.join(x["ptop"], x["prel"], x["name"]))
-        ]
+        if self.args.nw:
+            lost = []
+        else:
+            lost = [
+                x
+                for x in reg.values()
+                if x["need"]
+                and not bos.path.exists(os.path.join(x["ptop"], x["prel"], x["name"]))
+            ]
 
         if rm or lost:
             t = "dropping {} abandoned, {} deleted uploads in {}"
@@ -3043,7 +3048,7 @@ class Up2k(object):
                 except:
                     pass
 
-        if self.args.nw:
+        if self.args.nw or self.args.no_snap:
             return
 
         path = os.path.join(histpath, "up2k.snap")
@@ -3175,8 +3180,9 @@ class Up2k(object):
         for x in list(self.spools):
             self._unspool(x)
 
-        self.log("writing snapshot")
-        self.do_snapshot()
+        if not self.args.no_snap:
+            self.log("writing snapshot")
+            self.do_snapshot()
 
 
 def up2k_chunksize(filesize: int) -> int:
@@ -3185,7 +3191,7 @@ def up2k_chunksize(filesize: int) -> int:
     while True:
         for mul in [1, 2]:
             nchunks = math.ceil(filesize * 1.0 / chunksize)
-            if nchunks <= 256 or chunksize >= 32 * 1024 * 1024:
+            if nchunks <= 256 or (chunksize >= 32 * 1024 * 1024 and nchunks <= 4096):
                 return chunksize
 
             chunksize += stepsize

copyparty/util.py

@@ -3,8 +3,10 @@ from __future__ import print_function, unicode_literals
 
 import base64
 import contextlib
+import errno
 import hashlib
 import hmac
+import logging
 import math
 import mimetypes
 import os
@@ -22,13 +24,37 @@ import time
 import traceback
 from collections import Counter
 from datetime import datetime
+from email.utils import formatdate
 
+from ipaddress import IPv4Address, IPv4Network, IPv6Address, IPv6Network
 from queue import Queue
 
 from .__init__ import ANYWIN, MACOS, PY2, TYPE_CHECKING, VT100, WINDOWS
 from .__version__ import S_BUILD_DT, S_VERSION
 from .stolen import surrogateescape
 
+
+def _ens(want: str) -> tuple[int, ...]:
+    ret: list[int] = []
+    for v in want.split():
+        try:
+            ret.append(getattr(errno, v))
+        except:
+            pass
+
+    return tuple(ret)
+
+
+# WSAECONNRESET - foribly closed by remote
+# WSAENOTSOCK - no longer a socket
+# EUNATCH - can't assign requested address (wifi down)
+E_SCK = _ens("ENOTCONN EUNATCH EBADF WSAENOTSOCK WSAECONNRESET")
+E_ADDR_NOT_AVAIL = _ens("EADDRNOTAVAIL WSAEADDRNOTAVAIL")
+E_ADDR_IN_USE = _ens("EADDRINUSE WSAEADDRINUSE")
+E_ACCESS = _ens("EACCES WSAEACCES")
+E_UNREACH = _ens("EHOSTUNREACH WSAEHOSTUNREACH ENETUNREACH WSAENETUNREACH")
+
+
 try:
     import ctypes
     import fcntl
@@ -36,11 +62,6 @@ try:
 except:
     pass
 
-try:
-    from ipaddress import IPv6Address
-except:
-    pass
-
 try:
     HAVE_SQLITE3 = True
     import sqlite3  # pylint: disable=unused-import  # typechk
@@ -53,7 +74,7 @@ try:
 except:
     HAVE_PSUTIL = False
 
-try:
+if True:  # pylint: disable=using-constant-test
     import types
     from collections.abc import Callable, Iterable
 
@@ -68,8 +89,6 @@ try:
         def __call__(self, msg: str, c: Union[int, str] = 0) -> None:
             return None
 
-except:
-    pass
 
 if TYPE_CHECKING:
     import magic
@@ -79,10 +98,9 @@ if TYPE_CHECKING:
 FAKE_MP = False
 
 try:
-    if not FAKE_MP:
-        import multiprocessing as mp
-    else:
-        import multiprocessing.dummy as mp  # type: ignore
+    import multiprocessing as mp
+
+    # import multiprocessing.dummy as mp
 except ImportError:
     # support jython
     mp = None  # type: ignore
@@ -122,24 +140,28 @@ else:
 
 SYMTIME = sys.version_info > (3, 6) and os.utime in os.supports_follow_symlinks
 
-HTTP_TS_FMT = "%a, %d %b %Y %H:%M:%S GMT"
-
 META_NOBOTS = '<meta name="robots" content="noindex, nofollow">'
 
 HTTPCODE = {
     200: "OK",
+    201: "Created",
     204: "No Content",
     206: "Partial Content",
+    207: "Multi-Status",
     302: "Found",
     304: "Not Modified",
     400: "Bad Request",
+    401: "Unauthorized",
     403: "Forbidden",
     404: "Not Found",
     405: "Method Not Allowed",
+    409: "Conflict",
     411: "Length Required",
+    412: "Precondition Failed",
     413: "Payload Too Large",
     416: "Requested Range Not Satisfiable",
     422: "Unprocessable Entity",
+    423: "Locked",
     429: "Too Many Requests",
     500: "Internal Server Error",
     501: "Not Implemented",
@@ -156,7 +178,27 @@ IMPLICATIONS = [
     ["e2vu", "e2v"],
     ["e2vp", "e2v"],
     ["e2v", "e2d"],
+    ["smbw", "smb"],
+    ["smb1", "smb"],
+    ["smbvvv", "smbvv"],
+    ["smbvv", "smbv"],
+    ["smbv", "smb"],
+    ["zv", "zmv"],
+    ["zv", "zsv"],
+    ["z", "zm"],
+    ["z", "zs"],
+    ["zmvv", "zmv"],
+    ["zm4", "zm"],
+    ["zm6", "zm"],
+    ["zmv", "zm"],
+    ["zms", "zm"],
+    ["zsv", "zs"],
 ]
+if ANYWIN:
+    IMPLICATIONS.extend([["z", "zm4"]])
+
+
+UNPLICATIONS = [["no_dav", "daw"]]
 
 
 MIMES = {
@@ -311,6 +353,37 @@ _: Any = (mp, BytesIO, quote, unquote, SQLITE_VER, JINJA_VER, PYFTPD_VER)
 __all__ = ["mp", "BytesIO", "quote", "unquote", "SQLITE_VER", "JINJA_VER", "PYFTPD_VER"]
 
 
+class Daemon(threading.Thread):
+    def __init__(
+        self,
+        target: Any,
+        name: Optional[str] = None,
+        a: Optional[Iterable[Any]] = None,
+        r: bool = True,
+    ) -> None:
+        threading.Thread.__init__(self, target=target, name=name, args=a or ())
+        self.daemon = True
+        if r:
+            self.start()
+
+
+class Netdev(object):
+    def __init__(self, ip: str, idx: int, name: str, desc: str):
+        self.ip = ip
+        self.idx = idx
+        self.name = name
+        self.desc = desc
+
+    def __str__(self):
+        return "{}-{}{}".format(self.idx, self.name, self.desc)
+
+    def __lt__(self, rhs):
+        return str(self) < str(rhs)
+
+    def __eq__(self, rhs):
+        return str(self) == str(rhs)
+
+
 class Cooldown(object):
     def __init__(self, maxage: float) -> None:
         self.maxage = maxage
@@ -337,6 +410,96 @@ class Cooldown(object):
             return ret
 
 
+class HLog(logging.Handler):
+    def __init__(self, log_func: "RootLogger") -> None:
+        logging.Handler.__init__(self)
+        self.log_func = log_func
+        self.ptn_ftp = re.compile(r"^([0-9a-f:\.]+:[0-9]{1,5})-\[")
+        self.ptn_smb_ign = re.compile(r"^(Callback added|Config file parsed)")
+
+    def __repr__(self) -> str:
+        level = logging.getLevelName(self.level)
+        return "<%s cpp(%s)>" % (self.__class__.__name__, level)
+
+    def flush(self) -> None:
+        pass
+
+    def emit(self, record: logging.LogRecord) -> None:
+        msg = self.format(record)
+        lv = record.levelno
+        if lv < logging.INFO:
+            c = 6
+        elif lv < logging.WARNING:
+            c = 0
+        elif lv < logging.ERROR:
+            c = 3
+        else:
+            c = 1
+
+        if record.name.startswith("PIL") and lv < logging.WARNING:
+            return
+        elif record.name == "pyftpdlib":
+            m = self.ptn_ftp.match(msg)
+            if m:
+                ip = m.group(1)
+                msg = msg[len(ip) + 1 :]
+                if ip.startswith("::ffff:"):
+                    record.name = ip[7:]
+                else:
+                    record.name = ip
+        elif record.name.startswith("impacket"):
+            if self.ptn_smb_ign.match(msg):
+                return
+
+        self.log_func(record.name[-21:], msg, c)
+
+
+class NetMap(object):
+    def __init__(self, ips: list[str], netdevs: dict[str, Netdev]) -> None:
+        if "::" in ips:
+            ips = [x for x in ips if x != "::"] + list(
+                [x.split("/")[0] for x in netdevs if ":" in x]
+            )
+            ips.append("0.0.0.0")
+
+        if "0.0.0.0" in ips:
+            ips = [x for x in ips if x != "0.0.0.0"] + list(
+                [x.split("/")[0] for x in netdevs if ":" not in x]
+            )
+
+        ips = [x for x in ips if x not in ("::1", "127.0.0.1")]
+        ips = [[x for x in netdevs if x.startswith(y + "/")][0] for y in ips]
+
+        self.cache: dict[str, str] = {}
+        self.b2sip: dict[bytes, str] = {}
+        self.b2net: dict[bytes, Union[IPv4Network, IPv6Network]] = {}
+        self.bip: list[bytes] = []
+        for ip in ips:
+            v6 = ":" in ip
+            fam = socket.AF_INET6 if v6 else socket.AF_INET
+            bip = socket.inet_pton(fam, ip.split("/")[0])
+            self.bip.append(bip)
+            self.b2sip[bip] = ip.split("/")[0]
+            self.b2net[bip] = (IPv6Network if v6 else IPv4Network)(ip, False)
+
+        self.bip.sort(reverse=True)
+
+    def map(self, ip: str) -> str:
+        try:
+            return self.cache[ip]
+        except:
+            pass
+
+        v6 = ":" in ip
+        ci = IPv6Address(ip) if v6 else IPv4Address(ip)
+        bip = next((x for x in self.bip if ci in self.b2net[x]), None)
+        ret = self.b2sip[bip] if bip else ""
+        if len(self.cache) > 9000:
+            self.cache = {}
+        self.cache[ip] = ret
+        return ret
+
+
 class UnrecvEOF(OSError):
     pass
 
@@ -357,7 +520,16 @@ class _Unrecv(object):
             self.buf = self.buf[nbytes:]
             return ret
 
-        ret = self.s.recv(nbytes)
+        while True:
+            try:
+                ret = self.s.recv(nbytes)
+                break
+            except socket.timeout:
+                continue
+            except:
+                ret = b""
+                break
+
         if not ret:
             raise UnrecvEOF("client stopped sending data")
 
@@ -444,6 +616,27 @@ class _LUnrecv(object):
 Unrecv = _Unrecv
 
 
+class CachedSet(object):
+    def __init__(self, maxage: float) -> None:
+        self.c: dict[Any, float] = {}
+        self.maxage = maxage
+        self.oldest = 0.0
+
+    def add(self, v: Any) -> None:
+        self.c[v] = time.time()
+
+    def cln(self) -> None:
+        now = time.time()
+        if now - self.oldest < self.maxage:
+            return
+
+        c = self.c = {k: v for k, v in self.c.items() if now - v < self.maxage}
+        try:
+            self.oldest = c[min(c, key=c.get)]
+        except:
+            self.oldest = now
+
+
 class FHC(object):
     class CE(object):
         def __init__(self, fh: typing.BinaryIO) -> None:
@@ -540,9 +733,7 @@ class MTHash(object):
         self.done_q: Queue[tuple[int, str, int, int]] = Queue()
         self.thrs = []
         for n in range(cores):
-            t = threading.Thread(target=self.worker, name="mth-" + str(n))
-            t.daemon = True
-            t.start()
+            t = Daemon(self.worker, "mth-" + str(n))
             self.thrs.append(t)
 
     def hash(
@@ -746,7 +937,7 @@ class Garda(object):
         if not self.lim:
             return 0, ip
 
-        if ":" in ip and not PY2:
+        if ":" in ip:
             # assume /64 clients; drop 4 groups
             ip = IPv6Address(ip).exploded[:-20]
 
@@ -848,20 +1039,14 @@ def alltrace() -> str:
         else:
             rret += ret
 
-    return "\n".join(rret + bret)
+    return "\n".join(rret + bret) + "\n"
 
 
 def start_stackmon(arg_str: str, nid: int) -> None:
     suffix = "-{}".format(nid) if nid else ""
     fp, f = arg_str.rsplit(",", 1)
     zi = int(f)
-    t = threading.Thread(
-        target=stackmon,
-        args=(fp, zi, suffix),
-        name="stackmon" + suffix,
-    )
-    t.daemon = True
-    t.start()
+    Daemon(stackmon, "stackmon" + suffix, (fp, zi, suffix))
 
 
 def stackmon(fp: str, ival: float, suffix: str) -> None:
@@ -914,13 +1099,7 @@ def start_log_thrs(
         tname = "logthr-n{}-i{:x}".format(nid, os.getpid())
         lname = tname[3:]
 
-    t = threading.Thread(
-        target=log_thrs,
-        args=(logger, ival, lname),
-        name=tname,
-    )
-    t.daemon = True
-    t.start()
+    Daemon(log_thrs, tname, (logger, ival, lname))
 
 
 def log_thrs(log: Callable[[str, str, int], None], ival: float, name: str) -> None:
@@ -967,12 +1146,20 @@ def ren_open(
     fun = kwargs.pop("fun", open)
     fdir = kwargs.pop("fdir", None)
     suffix = kwargs.pop("suffix", None)
+    overwrite = kwargs.pop("overwrite", None)
 
     if fname == os.devnull:
         with fun(fname, *args, **kwargs) as f:
             yield {"orz": (f, fname)}
             return
 
+    if overwrite:
+        assert fdir
+        fpath = os.path.join(fdir, fname)
+        with fun(fsenc(fpath), *args, **kwargs) as f:
+            yield {"orz": (f, fname)}
+            return
+
     if suffix:
         ext = fname.split(".")[-1]
         if len(ext) < 7:
@@ -1006,6 +1193,7 @@ def ren_open(
 
             with fun(fsenc(fpath), *args, **kwargs) as f:
                 if b64:
+                    assert fdir
                     fp2 = "fn-trunc.{}.txt".format(b64)
                     fp2 = os.path.join(fdir, fp2)
                     with open(fsenc(fp2), "wb") as f2:
@@ -1017,7 +1205,7 @@ def ren_open(
         except OSError as ex_:
             ex = ex_
 
-            if ex.errno == 22 and not asciified:
+            if ex.errno == errno.EINVAL and not asciified:
                 asciified = True
                 bname, fname = [
                     zs.encode("ascii", "replace").decode("ascii").replace("?", "_")
@@ -1025,11 +1213,14 @@ def ren_open(
                 ]
                 continue
 
-            if ex.errno not in [36, 63, 95] and (not WINDOWS or ex.errno != 22):
+            # ENOTSUP: zfs on ubuntu 20.04
+            if ex.errno not in (errno.ENAMETOOLONG, errno.ENOSR, errno.ENOTSUP) and (
+                not WINDOWS or ex.errno != errno.EINVAL
+            ):
                 raise
 
         if not b64:
-            zs = (orig_name + "\n" + suffix).encode("utf-8", "replace")
+            zs = "{}\n{}".format(orig_name, suffix).encode("utf-8", "replace")
             zs = hashlib.sha512(zs).digest()[:12]
             b64 = base64.urlsafe_b64encode(zs).decode("utf-8")
 
@@ -1118,7 +1309,7 @@ class MultipartParser(object):
                 return field, None
 
             try:
-                is_webkit = self.headers["user-agent"].lower().find("applewebkit") >= 0
+                is_webkit = "applewebkit" in self.headers["user-agent"].lower()
             except:
                 is_webkit = False
 
@@ -1179,6 +1370,7 @@ class MultipartParser(object):
                     buf = buf[d:]
 
                 # look for boundary near the end of the buffer
+                n = 0
                 for n in range(1, len(buf) + 1):
                     if not buf[-n:] in self.boundary:
                         n -= 1
@@ -1283,8 +1475,12 @@ def get_boundary(headers: dict[str, str]) -> str:
 
 
 def read_header(sr: Unrecv) -> list[str]:
+    t0 = time.time()
     ret = b""
     while True:
+        if time.time() - t0 > 120:
+            return []
+
         try:
             ret += sr.recv(1024)
         except:
@@ -1333,12 +1529,12 @@ def gen_filekey_dbg(
         try:
             import inspect
 
-            ctx = ",".join(inspect.stack()[n][3] for n in range(2, 5))
+            ctx = ",".join(inspect.stack()[n].function for n in range(2, 5))
         except:
             ctx = ""
 
+        p2 = "a"
         try:
-            p2 = "a"
             p2 = absreal(fspath)
             if p2 != fspath:
                 raise Exception()
@@ -1355,8 +1551,7 @@ def gen_filekey_dbg(
 def gencookie(k: str, v: str, dur: Optional[int]) -> str:
     v = v.replace(";", "")
     if dur:
-        dt = datetime.utcfromtimestamp(time.time() + dur)
-        exp = dt.strftime("%a, %d %b %Y %H:%M:%S GMT")
+        exp = formatdate(time.time() + dur, usegmt=True)
     else:
         exp = "Fri, 15 Aug 1997 01:00:00 GMT"
 
@@ -1508,9 +1703,12 @@ def exclude_dotfiles(filepaths: list[str]) -> list[str]:
     return [x for x in filepaths if not x.split("/")[-1].startswith(".")]
 
 
-def http_ts(ts: int) -> str:
-    file_dt = datetime.utcfromtimestamp(ts)
-    return file_dt.strftime(HTTP_TS_FMT)
+def ipnorm(ip: str) -> str:
+    if ":" in ip:
+        # assume /64 clients; drop 4 groups
+        return IPv6Address(ip).exploded[:-20]
+
+    return ip
 
 
 def html_escape(s: str, quot: bool = False, crlf: bool = False) -> str:
@@ -1535,17 +1733,21 @@ def html_bescape(s: bytes, quot: bool = False, crlf: bool = False) -> bytes:
     return s
 
 
-def quotep(txt: str) -> str:
+def _quotep2(txt: str) -> str:
     """url quoter which deals with bytes correctly"""
     btxt = w8enc(txt)
-    quot1 = quote(btxt, safe=b"/")
-    if not PY2:
-        quot2 = quot1.encode("ascii")
-    else:
-        quot2 = quot1
+    quot = quote(btxt, safe=b"/")
+    return w8dec(quot.replace(b" ", b"+"))
 
-    quot3 = quot2.replace(b" ", b"+")
-    return w8dec(quot3)
+
+def _quotep3(txt: str) -> str:
+    """url quoter which deals with bytes correctly"""
+    btxt = w8enc(txt)
+    quot = quote(btxt, safe=b"/").encode("utf-8")
+    return w8dec(quot.replace(b" ", b"+"))
+
+
+quotep = _quotep3 if not PY2 else _quotep2
 
 
 def unquotep(txt: str) -> str:
@@ -1564,25 +1766,36 @@ def vsplit(vpath: str) -> tuple[str, str]:
 
 
 def vjoin(rd: str, fn: str) -> str:
-    return rd + "/" + fn if rd else fn
+    if rd and fn:
+        return rd + "/" + fn
+    else:
+        return rd or fn
 
 
-def w8dec(txt: bytes) -> str:
+def _w8dec2(txt: bytes) -> str:
     """decodes filesystem-bytes to wtf8"""
-    if PY2:
-        return surrogateescape.decodefilename(txt)
+    return surrogateescape.decodefilename(txt)
 
+
+def _w8enc2(txt: str) -> bytes:
+    """encodes wtf8 to filesystem-bytes"""
+    return surrogateescape.encodefilename(txt)
+
+
+def _w8dec3(txt: bytes) -> str:
+    """decodes filesystem-bytes to wtf8"""
     return txt.decode(FS_ENCODING, "surrogateescape")
 
 
-def w8enc(txt: str) -> bytes:
+def _w8enc3(txt: str) -> bytes:
     """encodes wtf8 to filesystem-bytes"""
-    if PY2:
-        return surrogateescape.encodefilename(txt)
-
     return txt.encode(FS_ENCODING, "surrogateescape")
 
 
+w8dec = _w8dec3 if not PY2 else _w8dec2
+w8enc = _w8enc3 if not PY2 else _w8enc2
+
+
 def w8b64dec(txt: str) -> str:
     """decodes base64(filesystem-bytes) to wtf8"""
     return w8dec(base64.urlsafe_b64decode(txt.encode("ascii")))
@@ -1593,17 +1806,20 @@ def w8b64enc(txt: str) -> str:
     return base64.urlsafe_b64encode(w8enc(txt)).decode("ascii")
 
 
-if PY2 and WINDOWS:
-    # moonrunes become \x3f with bytestrings,
-    # losing mojibake support is worth
-    def _not_actually_mbcs(txt):
-        return txt
-
-    fsenc = _not_actually_mbcs
-    fsdec = _not_actually_mbcs
-else:
+if not PY2 or not WINDOWS:
     fsenc = w8enc
     fsdec = w8dec
+else:
+    # moonrunes become \x3f with bytestrings,
+    # losing mojibake support is worth
+    def _not_actually_mbcs_enc(txt: str) -> bytes:
+        return txt
+
+    def _not_actually_mbcs_dec(txt: bytes) -> str:
+        return txt
+
+    fsenc = _not_actually_mbcs_enc
+    fsdec = _not_actually_mbcs_dec
 
 
 def s3enc(mem_cur: "sqlite3.Cursor", rd: str, fn: str) -> tuple[str, str]:
@@ -1630,10 +1846,7 @@ def db_ex_chk(log: "NamedLogger", ex: Exception, db_path: str) -> bool:
     if str(ex) != "database is locked":
         return False
 
-    thr = threading.Thread(target=lsof, args=(log, db_path), name="dbex")
-    thr.daemon = True
-    thr.start()
-
+    Daemon(lsof, "dbex", (log, db_path))
     return True
 
 
@@ -1720,7 +1933,7 @@ def shut_socket(log: "NamedLogger", sck: socket.socket, timeout: int = 3) -> Non
     finally:
         td = time.time() - t0
         if td >= 1:
-            log("shut({}) in {:.3f} sec".format(fd, td), "1;30")
+            log("shut({}) in {:.3f} sec".format(fd, td), "90")
 
         sck.close()
 
@@ -1877,9 +2090,9 @@ def sendfile_kern(
             n = os.sendfile(out_fd, in_fd, ofs, req)
             stuck = 0
         except OSError as ex:
+            # client stopped reading; do another select
             d = time.time() - stuck
-            log("sendfile stuck for {:.3f} sec: {!r}".format(d, ex))
-            if d < 3600 and ex.errno == 11:  # eagain
+            if d < 3600 and ex.errno == errno.EWOULDBLOCK:
                 continue
 
             n = 0
@@ -1906,6 +2119,7 @@ def statdir(
     if lstat and (PY2 or os.stat not in os.supports_follow_symlinks):
         scandir = False
 
+    src = "statdir"
     try:
         btop = fsenc(top)
         if scandir and hasattr(os, "scandir"):
@@ -2083,7 +2297,7 @@ def killtree(root: int) -> None:
             os.kill(pid, signal.SIGTERM)
     else:
         # windows gets minimal effort sorry
-        os.kill(pid, signal.SIGTERM)
+        os.kill(root, signal.SIGTERM)
         return
 
     for n in range(10):
@@ -2105,19 +2319,21 @@ def runcmd(
     kill = ka.pop("kill", "t")  # [t]ree [m]ain [n]one
     capture = ka.pop("capture", 3)  # 0=none 1=stdout 2=stderr 3=both
 
-    sin = ka.pop("sin", None)
+    sin: Optional[bytes] = ka.pop("sin", None)
     if sin:
         ka["stdin"] = sp.PIPE
 
     cout = sp.PIPE if capture in [1, 3] else None
     cerr = sp.PIPE if capture in [2, 3] else None
+    bout: bytes
+    berr: bytes
 
     p = sp.Popen(argv, stdout=cout, stderr=cerr, **ka)
     if not timeout or PY2:
-        stdout, stderr = p.communicate(sin)
+        bout, berr = p.communicate(sin)
     else:
         try:
-            stdout, stderr = p.communicate(sin, timeout=timeout)
+            bout, berr = p.communicate(sin, timeout=timeout)
         except sp.TimeoutExpired:
             if kill == "n":
                 return -18, "", ""  # SIGCONT; leave it be
@@ -2127,15 +2343,15 @@ def runcmd(
                 killtree(p.pid)
 
             try:
-                stdout, stderr = p.communicate(timeout=1)
+                bout, berr = p.communicate(timeout=1)
             except:
-                stdout = b""
-                stderr = b""
+                bout = b""
+                berr = b""
 
-    stdout = stdout.decode("utf-8", "replace") if cout else b""
-    stderr = stderr.decode("utf-8", "replace") if cerr else b""
+    stdout = bout.decode("utf-8", "replace") if cout else ""
+    stderr = berr.decode("utf-8", "replace") if cerr else ""
 
-    rc = p.returncode
+    rc: int = p.returncode
     if rc is None:
         rc = -14  # SIGALRM; failed to kill
 

copyparty/web/a/partyfuse.py

@@ -0,0 +1 @@
+../../../bin/partyfuse.py

copyparty/web/a/up2k.py

@@ -0,0 +1 @@
+../../../bin/up2k.py

copyparty/web/a/webdav-cfg.bat

@@ -0,0 +1 @@
+../../../contrib/webdav-cfg.bat

copyparty/web/baguettebox.js

@@ -28,7 +28,7 @@ window.baguetteBox = (function () {
         touch = {},  // start-pos
         touchFlag = false,  // busy
         re_i = /.+\.(gif|jpe?g|png|webp)(\?|$)/i,
-        re_v = /.+\.(webm|mp4)(\?|$)/i,
+        re_v = /.+\.(webm|mkv|mp4)(\?|$)/i,
         anims = ['slideIn', 'fadeIn', 'none'],
         data = {},  // all galleries
         imagesElements = [],
@@ -246,7 +246,13 @@ window.baguetteBox = (function () {
     }
 
     function keyDownHandler(e) {
-        if (anymod(e, true) || modal.busy)
+        if (modal.busy)
+            return;
+
+        if (e.key == '?')
+            return halp();
+
+        if (anymod(e, true))
             return;
 
         var k = e.code + '', v = vid(), pos = -1;

copyparty/web/browser.css

@@ -857,6 +857,12 @@ html.y #path a:hover {
 	color: var(--srv-3);
 	border-bottom: 1px solid var(--srv-3b);
 }
+#goh+span {
+	color: var(--bg-u5);
+	padding-left: .5em;
+	margin-left: .5em;
+	border-left: .2em solid var(--bg-u5);
+}
 #repl {
 	padding: .33em;
 }
@@ -1095,7 +1101,6 @@ html.y #widget.open {
 #wtoggle {
 	position: absolute;
 	white-space: nowrap;
-	font-size: .8em;
 	top: -1em;
 	right: 0;
 	height: 1em;
@@ -1177,7 +1182,7 @@ html.y #widget.open {
 	font-size: .4em;
 	margin: -.3em .1em;
 }
-#wtoggle.sel #wzip #selzip {
+#wtoggle.sel .l1 {
 	top: -.6em;
 	padding: .4em .3em;
 }
@@ -1221,6 +1226,40 @@ html.y #widget.open {
 	width: calc(100% - 10.5em);
 	background: rgba(0,0,0,0.2);
 }
+#widget.cmp {
+	height: 1.6em;
+	bottom: -1.6em;
+}
+#widget.cmp.open {
+	bottom: 0;
+}
+#widget.cmp #wtoggle {
+	font-size: 1.2em;
+}
+#widget.cmp #wtgrid {
+	display: none;
+}
+#widget.cmp #pctl {
+	top: 0;
+	left: 0;
+	font-size: .75em;
+}
+#widget.cmp #pctl a {
+	margin: 0;
+}
+#widget.cmp #barpos,
+#widget.cmp #barbuf {
+	width: calc(100% - 11em);
+	border-radius: 0;
+	left: 5em;
+	top: 0;
+}
+#widget.cmp #pvol {
+	top: 0;
+	right: 0;
+	max-width: 5.8em;
+	border-radius: 0;
+}
 .opview {
 	display: none;
 }
@@ -1344,8 +1383,12 @@ input.eq_gain {
 	padding-right: .2em;
 	text-align: right;
 }
+#srch_form:not(.tags) #tsrch_tags,
+#srch_form:not(.tags) #tsrch_adv {
+	display: none;
+}
 #op_search input {
-	margin: 0;
+	margin: .1em 0 0 0;
 }
 #srch_q {
 	white-space: pre;
@@ -1805,6 +1848,36 @@ a.btn,
 	-ms-user-select: none;
 	user-select: none;
 }
+#hkhelp {
+	background: var(--bg);
+}
+#hkhelp table {
+	margin: 2em 2em 0 2em;
+	float: left;
+}
+#hkhelp th {
+	border-bottom: 1px solid var(--bg-u5);
+	background: var(--bg-u1);
+	font-weight: bold;
+	text-align: right;
+}
+#hkhelp tr+tr th {
+	border-top: 1.5em solid var(--bg);
+}
+#hkhelp td {
+	padding: .2em .3em;
+}
+#hkhelp td:first-child {
+	font-family: 'scp', monospace, monospace;
+}
+html.noscroll,
+html.noscroll .sbar {
+	scrollbar-width: none;
+}
+html.noscroll::-webkit-scrollbar,
+html.noscroll .sbar::-webkit-scrollbar {
+	display: none;
+}
 
 
 

copyparty/web/browser.html

@@ -6,6 +6,7 @@
 	<title>{{ title }}</title>
 	<meta http-equiv="X-UA-Compatible" content="IE=edge">
 	<meta name="viewport" content="width=device-width, initial-scale=0.8, minimum-scale=0.6">
+	<meta name="theme-color" content="#333">
 {{ html_head }}
 	<link rel="stylesheet" media="screen" href="/.cpr/ui.css?_={{ ts }}">
 	<link rel="stylesheet" media="screen" href="/.cpr/browser.css?_={{ ts }}">
@@ -41,7 +42,7 @@
 	<div id="op_mkdir" class="opview opbox act">
 		<form method="post" enctype="multipart/form-data" accept-charset="utf-8" action="{{ url_suf }}">
 			<input type="hidden" name="act" value="mkdir" />
-			📂<input type="text" name="name" class="i">
+			📂<input type="text" name="name" class="i" placeholder="awesome mix vol.1">
 			<input type="submit" value="make directory">
 		</form>
 	</div>
@@ -49,14 +50,14 @@
 	<div id="op_new_md" class="opview opbox">
 		<form method="post" enctype="multipart/form-data" accept-charset="utf-8" action="{{ url_suf }}">
 			<input type="hidden" name="act" value="new_md" />
-			📝<input type="text" name="name" class="i">
+			📝<input type="text" name="name" class="i" placeholder="weekend-plans">
 			<input type="submit" value="new markdown doc">
 		</form>
 	</div>
 
 	<div id="op_msg" class="opview opbox act">
 		<form method="post" enctype="application/x-www-form-urlencoded" accept-charset="utf-8" action="{{ url_suf }}">
-			📟<input type="text" name="msg" class="i">
+			📟<input type="text" name="msg" class="i" placeholder="lorem ipsum dolor sit amet">
 			<input type="submit" value="send msg to srv log">
 		</form>
 	</div>

copyparty/web/md.css

@@ -4,6 +4,12 @@ html, body {
 	font-family: sans-serif;
 	line-height: 1.5em;
 }
+html.y #helpbox a {
+	color: #079;
+}
+html.z #helpbox a {
+	color: #fc5;
+}
 #repl {
 	position: absolute;
 	top: 0;

copyparty/web/md.html

@@ -3,6 +3,7 @@
 	<title>📝 {{ title }}</title>
 	<meta http-equiv="X-UA-Compatible" content="IE=edge">
 	<meta name="viewport" content="width=device-width, initial-scale=0.7">
+	<meta name="theme-color" content="#333">
 {{ html_head }}
 	<link rel="stylesheet" href="/.cpr/ui.css?_={{ ts }}">
 	<link rel="stylesheet" href="/.cpr/md.css?_={{ ts }}">
@@ -31,7 +32,7 @@
 		{%- else %}
 			<a href="{{ arg_base }}edit" tt="good: higher performance$Ngood: same document width as viewer$Nbad: assumes you know markdown">edit (basic)</a>
 			<a href="{{ arg_base }}edit2" tt="not in-house so probably less buggy">edit (fancy)</a>
-			<a href="{{ arg_base }}raw">view raw</a>
+			<a href="{{ arg_base }}">view raw</a>
 		{%- endif %}
 	</div>
 	<div id="toc"></div>

copyparty/web/md.js

@@ -1,12 +1,13 @@
 "use strict";
 
-var dom_toc = ebi('toc');
-var dom_wrap = ebi('mw');
-var dom_hbar = ebi('mh');
-var dom_nav = ebi('mn');
-var dom_pre = ebi('mp');
-var dom_src = ebi('mt');
-var dom_navtgl = ebi('navtoggle');
+var dom_toc = ebi('toc'),
+    dom_wrap = ebi('mw'),
+    dom_hbar = ebi('mh'),
+    dom_nav = ebi('mn'),
+    dom_pre = ebi('mp'),
+    dom_src = ebi('mt'),
+    dom_navtgl = ebi('navtoggle'),
+    hash0 = location.hash;
 
 
 // chrome 49 needs this
@@ -35,12 +36,12 @@ var dbg = function () { };
 
 // add navbar
 (function () {
-    var parts = get_evpath().split('/'), link = '', o;
-    for (var a = 0, aa = parts.length - 2; a <= aa; a++) {
+    var parts = (get_evpath().slice(0, -1).split('?')[0] + '?v').split('/'), link = '', o;
+    for (var a = 0, aa = parts.length - 1; a <= aa; a++) {
         link += parts[a] + (a < aa ? '/' : '');
         o = mknod('a');
         o.setAttribute('href', link);
-        o.textContent = uricom_dec(parts[a]) || 'top';
+        o.textContent = uricom_dec(parts[a].split('?')[0]) || 'top';
         dom_nav.appendChild(o);
     }
 })();
@@ -256,7 +257,7 @@ function convert_markdown(md_text, dest_dom) {
         var html = dom_li.innerHTML;
         dom_li.innerHTML =
             '<span class="todo_' + clas + '">' + char + '</span>' +
-            html.substr(html.indexOf('>') + 1);
+            html.slice(html.indexOf('>') + 1);
     }
 
     // separate <code> for each line in <pre>
@@ -328,6 +329,15 @@ function convert_markdown(md_text, dest_dom) {
         catch (ex) {
             md_plug_err(ex, ext[1]);
         }
+
+    if (hash0)
+        setTimeout(function () {
+            try {
+                QS(hash0).scrollIntoView();
+                hash0 = '';
+            }
+            catch (ex) { }
+        }, 1);
 }
 
 

copyparty/web/md2.js

@@ -107,7 +107,8 @@ var draw_md = (function () {
         map_src = genmap(dom_ref, map_src);
         map_pre = genmap(dom_pre, map_pre);
 
-        clmod(ebi('save'), 'disabled', src == server_md);
+        clmod(ebi('save'), 'disabled',
+            src.replace(/\r/g, "") == server_md.replace(/\r/g, ""));
 
         var t1 = Date.now();
         delay = t1 - t0 > 100 ? 25 : 1;
@@ -230,7 +231,8 @@ redraw = (function () {
 // modification checker
 function Modpoll() {
     var r = {
-        skip_one: true,
+        initial: true,
+        skip_one: false,
         disabled: false
     };
 
@@ -253,7 +255,7 @@ function Modpoll() {
         }
 
         console.log('modpoll...');
-        var url = (document.location + '').split('?')[0] + '?raw&_=' + Date.now();
+        var url = (document.location + '').split('?')[0] + '?_=' + Date.now();
         var xhr = new XHR();
         xhr.open('GET', url, true);
         xhr.responseType = 'text';
@@ -275,8 +277,18 @@ function Modpoll() {
         if (!this.responseText)
             return;
 
-        var server_ref = server_md.replace(/\r/g, '');
-        var server_now = this.responseText.replace(/\r/g, '');
+        var new_md = this.responseText,
+            server_ref = server_md.replace(/\r/g, ''),
+            server_now = new_md.replace(/\r/g, '');
+
+        // firefox bug: sometimes get stale text even if copyparty sent a 200
+        if (r.initial && server_ref != server_now)
+            return modal.confirm('Your browser decided to show an outdated copy of the document!\n\nDo you want to load the latest version from the server instead?', function () {
+                dom_src.value = server_md = new_md;
+                draw_md();
+            }, null);
+
+        r.initial = false;
 
         if (server_ref != server_now) {
             console.log("modpoll diff |" + server_ref.length + "|, |" + server_now.length + "|");
@@ -296,6 +308,7 @@ function Modpoll() {
         console.log('modpoll eq');
     };
 
+    setTimeout(r.periodic, 300);
     if (md_opt.modpoll_freq > 0)
         setInterval(r.periodic, 1000 * md_opt.modpoll_freq);
 
@@ -389,7 +402,7 @@ function save_cb() {
 
 function run_savechk(lastmod, txt, btn, ntry) {
     // download the saved doc from the server and compare
-    var url = (document.location + '').split('?')[0] + '?raw&_=' + Date.now();
+    var url = (document.location + '').split('?')[0] + '?_=' + Date.now();
     var xhr = new XHR();
     xhr.open('GET', url, true);
     xhr.responseType = 'text';

copyparty/web/mde.html

@@ -3,6 +3,7 @@
 	<title>📝 {{ title }}</title>
 	<meta http-equiv="X-UA-Compatible" content="IE=edge">
 	<meta name="viewport" content="width=device-width, initial-scale=0.7">
+	<meta name="theme-color" content="#333">
 {{ html_head }}
 	<link rel="stylesheet" href="/.cpr/ui.css?_={{ ts }}">
 	<link rel="stylesheet" href="/.cpr/mde.css?_={{ ts }}">

copyparty/web/mde.js

@@ -7,7 +7,7 @@ var dom_md = ebi('mt');
 
 (function () {
     var n = document.location + '';
-    n = n.substr(n.indexOf('//') + 2).split('?')[0].split('/');
+    n = (n.slice(n.indexOf('//') + 2).split('?')[0] + '?v').split('/');
     n[0] = 'top';
     var loc = [];
     var nav = [];
@@ -15,7 +15,7 @@ var dom_md = ebi('mt');
         if (a > 0)
             loc.push(n[a]);
 
-        var dec = uricom_dec(n[a]).replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
+        var dec = uricom_dec(n[a].split('?')[0]).replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
 
         nav.push('<a href="/' + loc.join('/') + '">' + dec + '</a>');
     }
@@ -166,7 +166,7 @@ function save_cb() {
     //alert('save OK -- wrote ' + r.size + ' bytes.\n\nsha512: ' + r.sha512);
 
     // download the saved doc from the server and compare
-    var url = (document.location + '').split('?')[0] + '?raw';
+    var url = (document.location + '').split('?')[0] + '?_=' + Date.now();
     var xhr = new XHR();
     xhr.open('GET', url, true);
     xhr.responseType = 'text';

copyparty/web/msg.html

@@ -6,7 +6,8 @@
 	<title>{{ svcname }}</title>
 	<meta http-equiv="X-UA-Compatible" content="IE=edge">
 	<meta name="viewport" content="width=device-width, initial-scale=0.8">
-	{{ html_head }}
+	<meta name="theme-color" content="#333">
+{{ html_head }}
 	<link rel="stylesheet" media="screen" href="/.cpr/msg.css?_={{ ts }}">
 </head>
 

copyparty/web/splash.css

@@ -10,6 +10,9 @@ html {
 	padding: 0 1em 3em 1em;
 	line-height: 1.3em;
 }
+#wrap.w {
+	max-width: 96%;
+}
 h1 {
 	border-bottom: 1px solid #ccc;
 	margin: 2em 0 .4em 0;
@@ -25,21 +28,27 @@ a {
 	text-decoration: none;
 	border-bottom: 1px solid #8ab;
 	border-radius: .2em;
-	padding: .2em .8em;
+	padding: .2em .6em;
+	margin: 0 .3em;
 }
-a+a {
-	margin-left: .5em;
+td a {
+	margin: 0;
 }
-.refresh,
+.af,
 .logout {
 	float: right;
-	margin: -.2em 0 0 .5em;
+	margin: -.2em 0 0 .8em;
 }
 .logout,
 a.r {
 	color: #c04;
 	border-color: #c7a;
 }
+a.g {
+	color: #2b0;
+	border-color: #3a0;
+	box-shadow: 0 .3em 1em #4c0;
+}
 #repl {
 	border: none;
 	background: none;
@@ -64,9 +73,15 @@ table {
 .num td:first-child {
 	text-align: right;
 }
+.cn {
+	text-align: center;
+}
 .btns {
 	margin: 1em 0;
 }
+.btns>a:first-child {
+	margin-left: 0;
+}
 #msg {
 	margin: 3em 0;
 }
@@ -83,6 +98,39 @@ blockquote {
 	border-left: .3em solid rgba(128,128,128,0.5);
 	border-radius: 0 0 0 .25em;
 }
+pre, code {
+	color: #480;
+	background: #fff;
+	font-family: 'scp', monospace, monospace;
+	border: 1px solid rgba(128,128,128,0.3);
+	border-radius: .2em;
+	padding: .15em .2em;
+}
+html.z pre,
+html.z code {
+	color: #9e0;
+	background: #000;
+	background: rgba(0,16,0,0.2);
+}
+.os {
+	line-height: 1.5em;
+}
+.sph {
+	margin-top: 4em;
+}
+.sph code {
+	margin-left: .3em;
+}
+pre b,
+code b {
+	color: #000;
+	font-weight: normal;
+	text-shadow: 0 0 .2em #0f0;
+}
+html.z pre b,
+html.z code b {
+	color: #fff;
+}
 
 
 html.z {
@@ -102,6 +150,11 @@ html.z a.r {
 	background: #804;
 	border-color: #c28;
 }
+html.z a.g {
+	background: #470;
+	border-color: #af4;
+	box-shadow: 0 .3em 1em #7d0;
+}
 html.z input {
 	color: #fff;
 	background: #626;

copyparty/web/splash.html

@@ -6,6 +6,7 @@
 	<title>{{ svcname }}</title>
 	<meta http-equiv="X-UA-Compatible" content="IE=edge">
 	<meta name="viewport" content="width=device-width, initial-scale=0.8">
+	<meta name="theme-color" content="#333">
 {{ html_head }}
 	<link rel="stylesheet" media="screen" href="/.cpr/splash.css?_={{ ts }}">
 	<link rel="stylesheet" media="screen" href="/.cpr/ui.css?_={{ ts }}">
@@ -13,7 +14,8 @@
 
 <body>
 	<div id="wrap">
-		<a id="a" href="/?h" class="refresh">refresh</a>
+		<a id="a" href="/?h" class="af">refresh</a>
+		<a id="v" href="/?hc" class="af">connect</a>
 
 		{%- if this.uname == '*' %}
 			<p id="b">howdy stranger &nbsp; <small>(you're not logged in)</small></p>

copyparty/web/splash.js

@@ -12,7 +12,7 @@ var Ls = {
 		"cc1": "klient-konfigurasjon",
 		"h1": "skru av k304",
 		"i1": "skru på k304",
-		"j1": "k304 bryter tilkoplingen for hver HTTP 304. Dette hjelper visse mellomtjenere som kan sette seg fast / plutselig slutter å laste sider, men det reduserer også ytelsen betydelig",
+		"j1": "k304 bryter tilkoplingen for hver HTTP 304. Dette hjelper mot visse mellomtjenere som kan sette seg fast / plutselig slutter å laste sider, men det reduserer også ytelsen betydelig",
 		"k1": "nullstill innstillinger",
 		"l1": "logg inn:",
 		"m1": "velkommen tilbake,",
@@ -24,11 +24,14 @@ var Ls = {
 		".s1": "kartlegg",
 		"t1": "handling",
 		"u2": "tid siden noen sist skrev til serveren$N( opplastning / navneendring / ... )$N$N17d = 17 dager$N1h23 = 1 time 23 minutter$N4m56 = 4 minuter 56 sekunder",
+		"v1": "koble til",
+		"v2": "bruk denne serveren som en lokal harddisk$N$NADVARSEL: kommer til å vise passordet ditt!"
 	},
 	"eng": {
 		"d2": "shows the state of all active threads",
 		"e2": "reload config files (accounts/volumes/volflags),$Nand rescan all e2ds volumes",
 		"u2": "time since the last server write$N( upload / rename / ... )$N$N17d = 17 days$N1h23 = 1 hour 23 minutes$N4m56 = 4 minutes 56 seconds",
+		"v2": "use this server as a local HDD$N$NWARNING: this will show your password!",
 	}
 },
 	d = Ls[sread("lang") || lang];

copyparty/web/svcs.html

@@ -0,0 +1,200 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+	<meta charset="utf-8">
+	<title>{{ args.doctitle }} @ {{ args.name }}</title>
+	<meta http-equiv="X-UA-Compatible" content="IE=edge">
+	<meta name="viewport" content="width=device-width, initial-scale=0.8">
+	<meta name="theme-color" content="#333">
+{{ html_head }}
+	<link rel="stylesheet" media="screen" href="/.cpr/splash.css?_={{ ts }}">
+	<link rel="stylesheet" media="screen" href="/.cpr/ui.css?_={{ ts }}">
+</head>
+
+<body>
+	<div id="wrap" class="w">
+        <div class="cn">
+            <p class="btns"><a href="/">browse files</a> // <a href="/?h">control panel</a></p>
+            <p>or choose your OS for cooler alternatives:</p>
+            <div class="ossel">
+                <a id="swin" href="#">Windows</a>
+                <a id="slin" href="#">Linux</a>
+                <a id="smac" href="#">macOS</a>
+            </div>
+        </div>
+
+        <p class="sph">
+            make this server appear on your computer as a regular HDD!<br />
+            pick your favorite below (sorted by performance, best first) and lets 🎉<br />
+            <br />
+            placeholders:
+            <span class="os win">
+                {% if accs %}<code><b>{{ pw }}</b></code>=password, {% endif %}<code><b>W:</b></code>=mountpoint
+            </span>
+            <span class="os lin mac">
+                {% if accs %}<code><b>{{ pw }}</b></code>=password, {% endif %}<code><b>mp</b></code>=mountpoint
+            </span>
+        </p>
+
+
+
+        {% if not args.no_dav %}
+        <h1>WebDAV</h1>
+
+        <div class="os win">
+            <p><em>note: rclone-FTP is a bit faster, so {% if args.ftp or args.ftps %}try that first{% else %}consider enabling FTP in server settings{% endif %}</em></p>
+            <p>if you can, install <a href="https://winfsp.dev/rel/">winfsp</a>+<a href="https://downloads.rclone.org/rclone-current-windows-amd64.zip">rclone</a> and then paste this in cmd:</p>
+            <pre>
+                rclone config create {{ aname }}-dav webdav url=http{{ s }}://{{ rip }}{{ hport }} vendor=other{% if accs %} user=k pass=<b>{{ pw }}</b>{% endif %}
+                rclone mount --vfs-cache-mode writes --dir-cache-time 5s {{ aname }}-dav:{{ vp }} <b>W:</b>
+            </pre>
+            {% if s %}
+            <p><em>note: if you are on LAN (or just dont have valid certificates), add <code>--no-check-certificate</code> to the mount command</em><br />---</p>
+            {% endif %}
+            
+            <p>if you want to use the native WebDAV client in windows instead (slow and buggy), first run <a href="/.cpr/a/webdav-cfg.bat">webdav-cfg.bat</a> to remove the 47 MiB filesize limit (also fixes latency and password login), then connect:</p>
+            <pre>
+                net use <b>w:</b> http{{ s }}://{{ ep }}/{{ vp }}{% if accs %} k /user:<b>{{ pw }}</b>{% endif %}
+            </pre>
+        </div>
+
+        <div class="os lin">
+            <pre>
+                yum install davfs2
+                {% if accs %}printf '%s\n' <b>{{ pw }}</b> k | {% endif %}mount -t davfs -ouid=1000 http{{ s }}://{{ ep }}/{{ vp }} <b>mp</b>
+            </pre>
+            <p>or you can use rclone instead, which is much slower but doesn't require root:</p>
+            <pre>
+                rclone config create {{ aname }}-dav webdav url=http{{ s }}://{{ rip }}{{ hport }} vendor=other{% if accs %} user=k pass=<b>{{ pw }}</b>{% endif %}
+                rclone mount --vfs-cache-mode writes --dir-cache-time 5s {{ aname }}-dav:{{ vp }} <b>mp</b>
+            </pre>
+            {% if s %}
+            <p><em>note: if you are on LAN (or just dont have valid certificates), add <code>--no-check-certificate</code> to the mount command</em><br />---</p>
+            {% endif %}
+            
+            <p>or the emergency alternative (gnome/gui-only):</p>
+            <!-- gnome-bug: ignores vp -->
+            <pre>
+                {%- if accs %}
+                echo <b>{{ pw }}</b> | gio mount dav{{ s }}://k@{{ ep }}/{{ vp }}
+                {%- else %}
+                gio mount -a dav{{ s }}://{{ ep }}/{{ vp }}
+                {%- endif %}
+            </pre>
+        </div>
+
+        <div class="os mac">
+            <pre>
+                osascript -e ' mount volume "http{{ s }}://k:<b>{{ pw }}</b>@{{ ep }}/{{ vp }}" '
+            </pre>
+            <p>or you can open up a Finder, press command-K and paste this instead:</p>
+            <pre>
+                http{{ s }}://k:<b>{{ pw }}</b>@{{ ep }}/{{ vp }}
+            </pre>
+            
+            {% if s %}
+            <p><em>replace <code>https</code> with <code>http</code> if it doesn't work</em></p>
+            {% endif %}
+        </div>
+        {% endif %}
+
+
+
+        {% if args.ftp or args.ftps %}
+        <h1>FTP</h1>
+
+        <div class="os win">
+            <p>if you can, install <a href="https://winfsp.dev/rel/">winfsp</a>+<a href="https://downloads.rclone.org/rclone-current-windows-amd64.zip">rclone</a> and then paste this in cmd:</p>
+            <pre>
+                rclone config create {{ aname }}-ftp ftp host={{ rip }} port={{ args.ftp or args.ftps }} pass=k user={% if accs %}<b>{{ pw }}</b>{% else %}anonymous{% endif %} tls={{ "false" if args.ftp else "true" }}
+                rclone mount --vfs-cache-mode writes --dir-cache-time 5s {{ aname }}-ftp:{{ vp }} <b>W:</b>
+            </pre>
+            <p>if you want to use the native FTP client in windows instead (please dont), press <code>win+R</code> and run this command:</p>
+            <pre>
+                explorer {{ "ftp" if args.ftp else "ftps" }}://{% if accs %}<b>{{ pw }}</b>:k@{% endif %}{{ host }}:{{ args.ftp or args.ftps }}/{{ vp }}
+            </pre>
+        </div>
+
+        <div class="os lin">
+            <pre>
+                rclone config create {{ aname }}-ftp ftp host={{ rip }} port={{ args.ftp or args.ftps }} pass=k user={% if accs %}<b>{{ pw }}</b>{% else %}anonymous{% endif %} tls={{ "false" if args.ftp else "true" }}
+                rclone mount --vfs-cache-mode writes --dir-cache-time 5s {{ aname }}-ftp:{{ vp }} <b>mp</b>
+            </pre>
+            <p>emergency alternative (gnome/gui-only):</p>
+            <!-- gnome-bug: ignores vp -->
+            <pre>
+                {%- if accs %}
+                echo <b>{{ pw }}</b> | gio mount ftp{{ "" if args.ftp else "s" }}://k@{{ host }}:{{ args.ftp or args.ftps }}/{{ vp }}
+                {%- else %}
+                gio mount -a ftp{{ "" if args.ftp else "s" }}://{{ host }}:{{ args.ftp or args.ftps }}/{{ vp }}
+                {%- endif %}
+            </pre>
+        </div>
+
+        <div class="os mac">
+            <p>note: FTP is read-only on macos; please use WebDAV instead</p>
+            <pre>
+                open {{ "ftp" if args.ftp else "ftps" }}://{% if accs %}k:<b>{{ pw }}</b>@{% else %}anonymous:@{% endif %}{{ host }}:{{ args.ftp or args.ftps }}/{{ vp }}
+            </pre>
+        </div>
+        {% endif %}
+
+
+
+        <h1>partyfuse</h1>
+        <p>
+            <a href="/.cpr/a/partyfuse.py">partyfuse.py</a> -- fast, read-only,
+            <span class="os win">needs <a href="https://winfsp.dev/rel/">winfsp</a></span>
+            <span class="os lin">doesn't need root</span>
+        </p>
+        <pre>
+            partyfuse.py{% if accs %} -a <b>{{ pw }}</b>{% endif %} http{{ s }}://{{ ep }}/{{ vp }} <b><span class="os win">W:</span><span class="os lin mac">mp</span></b>
+        </pre>
+        {% if s %}
+        <p><em>note: if you are on LAN (or just dont have valid certificates), add <code>-td</code></em></p>
+        {% endif %}
+        <p>
+            you can use <a href="/.cpr/a/up2k.py">up2k.py</a> to upload (sometimes faster than web-browsers)
+        </p>
+
+
+        {% if args.smb %}
+        <h1>SMB / CIFS</h1>
+        <em><a href="https://github.com/SecureAuthCorp/impacket/issues/1433">bug:</a> max ~300 files in each folder</em>
+
+        <div class="os win">
+            <pre>
+                net use <b>w:</b> \\{{ host }}\a{% if accs %} k /user:<b>{{ pw }}</b>{% endif %}
+            </pre>
+            <!-- rclone fails due to copyparty-smb bugs -->
+        </div>
+
+        <div class="os lin">
+            <pre>
+                mount -t cifs -o{% if accs %}user=<b>{{ pw }}</b>,pass=k,{% endif %}vers={{ 1 if args.smb1 else 2 }}.0,port={{ args.smb_port }},uid=1000 //{{ host }}/a/ <b>mp</b>
+            </pre>
+            <!-- p>or the emergency alternative (gnome/gui-only):</p nevermind, only works through mdns -->
+        </div>
+
+        <pre class="os mac">
+            open 'smb://<b>{{ pw }}</b>:k@{{ host }}/a'
+        </pre>
+        {% endif %}
+
+
+
+    </div>
+	<a href="#" id="repl">π</a>
+	<script>
+
+var lang="{{ lang }}",
+	dfavico="{{ favico }}";
+
+document.documentElement.className=localStorage.theme||"{{ args.theme }}";
+
+</script>
+<script src="/.cpr/util.js?_={{ ts }}"></script>
+<script src="/.cpr/svcs.js?_={{ ts }}"></script>
+</body>
+</html>

copyparty/web/svcs.js

@@ -0,0 +1,42 @@
+function QSA(x) {
+    return document.querySelectorAll(x);
+}
+var LINUX = /Linux/.test(navigator.userAgent),
+    MACOS = /[^a-z]mac ?os/i.test(navigator.userAgent),
+    WINDOWS = /Windows/.test(navigator.userAgent);
+
+
+var oa = QSA('pre');
+for (var a = 0; a < oa.length; a++) {
+    var html = oa[a].innerHTML,
+        nd = /^ +/.exec(html)[0].length,
+        rd = new RegExp('(^|\r?\n) {' + nd + '}', 'g');
+
+    oa[a].innerHTML = html.replace(rd, '$1').replace(/[ \r\n]+$/, '').replace(/\r?\n/g, '<br />');
+}
+
+
+oa = QSA('.ossel a');
+for (var a = 0; a < oa.length; a++)
+    oa[a].onclick = esetos;
+
+function esetos(e) {
+    ev(e);
+    setos(((e && e.target) || (window.event && window.event.srcElement)).id.slice(1));
+}
+
+function setos(os) {
+    var oa = QSA('.os');
+    for (var a = 0; a < oa.length; a++)
+        oa[a].style.display = 'none';
+
+    var oa = QSA('.' + os);
+    for (var a = 0; a < oa.length; a++)
+        oa[a].style.display = '';
+
+    oa = QSA('.ossel a');
+    for (var a = 0; a < oa.length; a++)
+        clmod(oa[a], 'g', oa[a].id.slice(1) == os);
+}
+
+setos(WINDOWS ? 'win' : LINUX ? 'lin' : MACOS ? 'mac' : '');

copyparty/web/ui.css

@@ -202,6 +202,7 @@ html.y #tth {
 	border: .4em solid var(--fg);
 	box-shadow: 0 2em 4em 1em var(--bg-max);
 }
+#hkhelp,
 #modal {
 	position: fixed;
     overflow: auto;

copyparty/web/up2k.js

@@ -865,7 +865,7 @@ function up2k_init(subtle) {
     bcfg_bind(uc, 'turbo', 'u2turbo', turbolvl > 1, draw_turbo);
     bcfg_bind(uc, 'datechk', 'u2tdate', turbolvl < 3, null);
     bcfg_bind(uc, 'az', 'u2sort', u2sort.indexOf('n') + 1, set_u2sort);
-    bcfg_bind(uc, 'hashw', 'hashw', !!window.WebAssembly && (!subtle || !CHROME || MOBILE), set_hashw);
+    bcfg_bind(uc, 'hashw', 'hashw', !!window.WebAssembly && (!subtle || !CHROME || MOBILE || VCHROME >= 107), set_hashw);
     bcfg_bind(uc, 'upnag', 'upnag', false, set_upnag);
     bcfg_bind(uc, 'upsfx', 'upsfx', false);
 
@@ -1124,7 +1124,7 @@ function up2k_init(subtle) {
                     continue;
 
                 try {
-                    var wi = fobj.webkitGetAsEntry();
+                    var wi = fobj.getAsEntry ? fobj.getAsEntry() : fobj.webkitGetAsEntry();
                     if (wi.isDirectory) {
                         dirs.push(wi);
                         continue;
@@ -1206,7 +1206,7 @@ function up2k_init(subtle) {
                 }
                 else {
                     var name = dn.fullPath;
-                    if (name.indexOf('/') === 0)
+                    if (name.startsWith('/'))
                         name = name.slice(1);
 
                     pf.push(name);
@@ -1229,7 +1229,16 @@ function up2k_init(subtle) {
                 dirs.shift();
                 rd = null;
             }
-            return read_dirs(rd, pf, dirs, good, nil, bad, spins);
+            read_dirs(rd, pf, dirs, good, nil, bad, spins);
+        }, function () {
+            var dn = dirs[0],
+                name = dn.fullPath;
+
+            if (name.startsWith('/'))
+                name = name.slice(1);
+
+            bad.push([dn, name + '/']);
+            read_dirs(null, pf, dirs.slice(1), good, nil, bad, spins);
         });
     }
 
@@ -1499,7 +1508,7 @@ function up2k_init(subtle) {
 
         st.oserr = true;
         var msg = HTTPS ? L.u_emtleak3 : L.u_emtleak2.format((window.location + '').replace(':', 's:'));
-        modal.alert(L.u_emtleak1 + msg + L.u_emtleak4 + (CHROME ? L.u_emtleakc : FIREFOX ? L.u_emtleakf : ''));
+        modal.alert(L.u_emtleak1 + msg + (CHROME ? L.u_emtleakc : FIREFOX ? L.u_emtleakf : ''));
     }
 
     /////
@@ -1804,7 +1813,7 @@ function up2k_init(subtle) {
         while (true) {
             for (var mul = 1; mul <= 2; mul++) {
                 var nchunks = Math.ceil(filesize / chunksize);
-                if (nchunks <= 256 || chunksize >= 32 * 1024 * 1024)
+                if (nchunks <= 256 || (chunksize >= 32 * 1024 * 1024 && nchunks <= 4096))
                     return chunksize;
 
                 chunksize += stepsize;
@@ -2092,7 +2101,7 @@ function up2k_init(subtle) {
             try { orz(e); } catch (ex) { vis_exh(ex + '', 'up2k.js', '', '', ex); }
         };
 
-        xhr.open('HEAD', t.purl + uricom_enc(t.name) + '?raw', true);
+        xhr.open('HEAD', t.purl + uricom_enc(t.name), true);
         xhr.send();
     }
 
@@ -2288,8 +2297,8 @@ function up2k_init(subtle) {
                     return;
                 }
 
-                var err_pend = rsp.indexOf('partial upload exists') + 1,
-                    err_dupe = rsp.indexOf('file already exists') + 1;
+                var err_pend = rsp.indexOf('partial upload exists at a different') + 1,
+                    err_dupe = rsp.indexOf('upload rejected, file already exists') + 1;
 
                 if (err_pend || err_dupe) {
                     err = rsp;

copyparty/web/util.js

@@ -14,9 +14,12 @@ var wah = '',
     TOUCH = 'ontouchstart' in window,
     MOBILE = TOUCH,
     CHROME = !!window.chrome,
+    VCHROME = CHROME ? 1 : 0,
     FIREFOX = ('netscape' in window) && / rv:/.test(navigator.userAgent),
     IPHONE = TOUCH && /iPhone|iPad|iPod/i.test(navigator.userAgent),
-    WINDOWS = navigator.platform ? navigator.platform == 'Win32' : /Windows/.test(navigator.userAgent);
+    LINUX = /Linux/.test(navigator.userAgent),
+    MACOS = /[^a-z]mac ?os/i.test(navigator.userAgent),
+    WINDOWS = /Windows/.test(navigator.userAgent);
 
 if (!window.WebAssembly || !WebAssembly.Memory)
     window.WebAssembly = false;
@@ -36,8 +39,13 @@ try {
     if (navigator.userAgentData.platform == 'Windows')
         WINDOWS = true;
 
-    if (navigator.userAgentData.brands.some(function (d) { return d.brand == 'Chromium' }))
-        CHROME = true;
+    CHROME = navigator.userAgentData.brands.find(function (d) { return d.brand == 'Chromium' });
+    if (CHROME)
+        VCHROME = CHROME.version;
+    else
+        VCHROME = 0;
+
+    CHROME = !!CHROME;
 }
 catch (ex) { }
 
@@ -48,11 +56,15 @@ var ebi = document.getElementById.bind(document),
     XHR = XMLHttpRequest;
 
 
-function mknod(et, eid) {
+function mknod(et, eid, html) {
     var ret = document.createElement(et);
+
     if (eid)
         ret.id = eid;
 
+    if (html)
+        ret.innerHTML = html;
+
     return ret;
 }
 
@@ -143,7 +155,7 @@ function vis_exh(msg, url, lineNo, columnNo, error) {
     window.onerror = undefined;
     var html = [
         '<h1>you hit a bug!</h1>',
-        '<p style="font-size:1.3em;margin:0">try to <a href="#" onclick="localStorage.clear();location.reload();">reset copyparty settings</a> if you are stuck here, or <a href="#" onclick="ignex();">ignore this</a> / <a href="#" onclick="ignex(true);">ignore all</a></p>',
+        '<p style="font-size:1.3em;margin:0">try to <a href="#" onclick="localStorage.clear();location.reload();">reset copyparty settings</a> if you are stuck here, or <a href="#" onclick="ignex();">ignore this</a> / <a href="#" onclick="ignex(true);">ignore all</a> / <a href="?b=u">basic</a></p>',
         '<p style="color:#fff">please send me a screenshot arigathanks gozaimuch: <a href="<ghi>" target="_blank">github issue</a> or <code>ed#2644</code></p>',
         '<p class="b">' + esc(url + ' @' + lineNo + ':' + columnNo), '<br />' + esc(String(msg)).replace(/\n/g, '<br />') + '</p>',
         '<p><b>UA:</b> ' + esc(navigator.userAgent + '')
@@ -1622,6 +1634,25 @@ var favico = (function () {
 })();
 
 
+function cprop(name) {
+    return getComputedStyle(document.documentElement).getPropertyValue(name);
+}
+
+
+function bchrome() {
+    console.log(document.documentElement.className);
+    var v, o = QS('meta[name=theme-color]');
+    if (!o)
+        return;
+
+    try {
+        v = cprop('--bg-u3');
+    }
+    catch (ex) { }
+    o.setAttribute('content', v ? v : document.documentElement.className.indexOf('y') + 1 ? '#eee' : '#333');
+}
+bchrome();
+
 var cf_cha_t = 0;
 function xhrchk(xhr, prefix, e404, lvl, tag) {
     if (xhr.status < 400 && xhr.status >= 200)

docs/changelog.md

@@ -1,3 +1,63 @@
+▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀  
+# 2022-1013-1937  `v1.4.6`  wav2opus
+
+* read-only demo server at https://a.ocv.me/pub/demo/
+* latest gzip edition of the sfx: *This version*
+
+## bugfixes
+* the option to transcode flac to opus while playing audio in the browser was supposed to transcode wav-files as well, instead of being extremely hazardous to mobile data plans (sorry)
+* `--license` didn't work if copyparty was installed from `pip`
+
+
+
+▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀  
+# 2022-1009-0919  `v1.4.5`  qr-code
+
+* read-only demo server at https://a.ocv.me/pub/demo/
+* latest gzip edition of the sfx: [v1.0.14](https://github.com/9001/copyparty/releases/tag/v1.0.14#:~:text=release-specific%20notes)
+
+## new features
+* display a server [qr-code](https://github.com/9001/copyparty#qr-code) [(screenshot)](https://user-images.githubusercontent.com/241032/194728533-6f00849b-c6ac-43c6-9359-83e454d11e00.png) on startup
+  * primarily for running copyparty on a phone and accessing it from another
+  * optionally specify a path or password with `--qrl lootbox/?pw=hunter2`
+  * uses the server's exteral ip (default route) unless `--qri` specifies a domain / ip-prefix
+  * classic cp437 `▄` `▀` for space efficiency; some misbehaving terminals / fonts need `--qrz 2`
+* new permission `G` returns the filekey of uploaded files for users without read-access
+  * when combined with permission `w` and volflag `fk`, uploaded files will not be accessible unless the filekey is provided in the url, and `G` provides the filekey to the uploader unlike `g`
+* filekeys are added to the unpost listing
+
+## bugfixes
+* renaming / moving folders is now **at least 120x faster**
+  * and that's on nvme drives, so probably like 2000x on HDDs
+* uploads to volumes with lifetimes could get instapurged depending on browser and browser settings
+* ux fixes
+  * FINALLY fixed messageboxes appearing offscreen on phones (and some other layout issues)
+  * stop asking about folder-uploads on phones because they dont support it
+  * on android-firefox, default to truncating huge folders with the load-more button due to ff onscroll being buggy
+  * audioplayer looking funky if ffmpeg unavailable
+* waveform-seekbar cache expiration (the thumbcleaner complaining about png files)
+* ie11 panic when opening a folder which contains a file named `up2k`
+  * turns out `<a name=foo>` becomes `window.foo` unless that's already declared somewhere in js -- luckily other browsers "only" do that with IDs
+
+
+
+▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀  
+# 2022-0926-2037  `v1.4.3`  signal in the noise
+
+* read-only demo server at https://a.ocv.me/pub/demo/
+* latest gzip edition of the sfx: [v1.0.14](https://github.com/9001/copyparty/releases/tag/v1.0.14#:~:text=release-specific%20notes)
+
+## new features
+* `--bak-flips` saves a copy of corrupted / bitflipped up2k uploads
+  * comparing against a good copy can help pinpoint the culprit
+  * also see [tracking bitflips](https://github.com/9001/copyparty/blob/hovudstraum/docs/notes.sh#:~:text=tracking%20bitflips)
+
+## bugfixes
+* some edgecases where deleted files didn't get dropped from the db
+  * can reduce performance over time, hitting the filesystem more than necessary
+
+
+
 ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀  
 # 2022-0925-1236  `v1.4.2`  fuhgeddaboudit
 

docs/copyparty.d/foo/another.conf

@@ -0,0 +1,5 @@
+# this file gets included twice from ../some.conf,
+# setting user permissions for a volume
+rw usr1
+r usr2
+% sibling.conf

docs/copyparty.d/foo/sibling.conf

@@ -0,0 +1,3 @@
+# and this config file gets included from ./another.conf,
+# adding a final permission for each of the two volumes in ../some.conf
+m usr1 usr2

docs/copyparty.d/some.conf

@@ -0,0 +1,26 @@
+# lets make two volumes with the same accounts/permissions for both;
+# first declare the accounts just once:
+u usr1:passw0rd
+u usr2:letmein
+
+# and listen on 127.0.0.1 only, port 2434
+-i 127.0.0.1
+-p 2434
+
+# share /usr/share/games from the server filesystem
+/usr/share/games
+/vidya
+# include config file with volume permissions
+% foo/another.conf
+
+# and share your ~/Music folder too
+~/Music
+/bangers
+% foo/another.conf
+
+# which should result in each of the volumes getting the following permissions:
+# usr1 read/write/move
+# usr2 read/move
+#
+# because another.conf sets the read/write permissions before it
+# includes sibling.conf which adds the move permission

docs/devnotes.md

@@ -0,0 +1,280 @@
+## devnotes toc
+
+* top
+* [future plans](#future-plans) - some improvement ideas
+* [design](#design)
+    * [why chunk-hashes](#why-chunk-hashes) - a single sha512 would be better, right?
+* [assumptions](#assumptions)
+    * [mdns](#mdns)
+* [sfx repack](#sfx-repack) - reduce the size of an sfx by removing features
+* [building](#building)
+    * [dev env setup](#dev-env-setup)
+    * [just the sfx](#just-the-sfx)
+    * [complete release](#complete-release)
+* [todo](#todo) - roughly sorted by priority
+    * [discarded ideas](#discarded-ideas)
+
+
+# future plans
+
+some improvement ideas
+
+* the JS is a mess -- a preact rewrite would be nice
+  * preferably without build dependencies like webpack/babel/node.js, maybe a python thing to assemble js files into main.js
+  * good excuse to look at using virtual lists (browsers start to struggle when folders contain over 5000 files)
+* the UX is a mess -- a proper design would be nice
+  * very organic (much like the python/js), everything was an afterthought
+  * true for both the layout and the visual flair
+  * something like the tron board-room ui (or most other hollywood ones, like ironman) would be :100:
+* some of the python files are way too big
+  * `up2k.py` ended up doing all the file indexing / db management
+  * `httpcli.py` should be separated into modules in general
+
+
+# design
+
+## up2k
+
+quick outline of the up2k protocol, see [uploading](#uploading) for the web-client
+* the up2k client splits a file into an "optimal" number of chunks
+  * 1 MiB each, unless that becomes more than 256 chunks
+  * tries 1.5M, 2M, 3, 4, 6, ... until <= 256 chunks or size >= 32M
+* client posts the list of hashes, filename, size, last-modified
+* server creates the `wark`, an identifier for this upload
+  * `sha512( salt + filesize + chunk_hashes )`
+  * and a sparse file is created for the chunks to drop into
+* client uploads each chunk
+  * header entries for the chunk-hash and wark
+  * server writes chunks into place based on the hash
+* client does another handshake with the hashlist; server replies with OK or a list of chunks to reupload
+
+up2k has saved a few uploads from becoming corrupted in-transfer already;
+* caught an android phone on wifi redhanded in wireshark with a bitflip, however bup with https would *probably* have noticed as well (thanks to tls also functioning as an integrity check)
+* also stopped someone from uploading because their ram was bad
+
+regarding the frequent server log message during uploads;  
+`6.0M 106M/s 2.77G 102.9M/s n948 thank 4/0/3/1 10042/7198 00:01:09`
+* this chunk was `6 MiB`, uploaded at `106 MiB/s`
+* on this http connection, `2.77 GiB` transferred, `102.9 MiB/s` average, `948` chunks handled
+* client says `4` uploads OK, `0` failed, `3` busy, `1` queued, `10042 MiB` total size, `7198 MiB` and `00:01:09` left
+
+## why chunk-hashes
+
+a single sha512 would be better, right?
+
+this was due to `crypto.subtle` [not yet](https://github.com/w3c/webcrypto/issues/73) providing a streaming api (or the option to seed the sha512 hasher with a starting hash)
+
+as a result, the hashes are much less useful than they could have been (search the server by sha512, provide the sha512 in the response http headers, ...)
+
+however it allows for hashing multiple chunks in parallel, greatly increasing upload speed from fast storage (NVMe, raid-0 and such)
+
+* both the [browser uploader](https://github.com/9001/copyparty#uploading) and the [commandline one](https://github.com/9001/copyparty/blob/hovudstraum/bin/up2k.py) does this now, allowing for fast uploading even from plaintext http
+
+hashwasm would solve the streaming issue but reduces hashing speed for sha512 (xxh128 does 6 GiB/s), and it would make old browsers and [iphones](https://bugs.webkit.org/show_bug.cgi?id=228552) unsupported
+
+* blake2 might be a better choice since xxh is non-cryptographic, but that gets ~15 MiB/s on slower androids
+
+
+# http api
+
+* table-column `params` = URL parameters; `?foo=bar&qux=...`
+* table-column `body` = POST payload
+* method `jPOST` = json post
+* method `mPOST` = multipart post
+* method `uPOST` = url-encoded post
+* `FILE` = conventional HTTP file upload entry (rfc1867 et al, filename in `Content-Disposition`)
+
+authenticate using header `Cookie: cppwd=foo` or url param `&pw=foo`
+
+## read
+
+| method | params | result |
+|--|--|--|
+| GET | `?ls` | list files/folders at URL as JSON |
+| GET | `?ls&dots` | list files/folders at URL as JSON, including dotfiles |
+| GET | `?ls=t` | list files/folders at URL as plaintext |
+| GET | `?ls=v` | list files/folders at URL, terminal-formatted |
+| GET | `?b` | list files/folders at URL as simplified HTML |
+| GET | `?tree=.` | list one level of subdirectories inside URL |
+| GET | `?tree` | list one level of subdirectories for each level until URL |
+| GET | `?tar` | download everything below URL as a tar file |
+| GET | `?zip=utf-8` | download everything below URL as a zip file |
+| GET | `?ups` | show recent uploads from your IP |
+| GET | `?ups&filter=f` | ...where URL contains `f` |
+| GET | `?mime=foo` | specify return mimetype `foo` |
+| GET | `?v` | render markdown file at URL |
+| GET | `?txt` | get file at URL as plaintext |
+| GET | `?txt=iso-8859-1` | ...with specific charset |
+| GET | `?th` | get image/video at URL as thumbnail |
+| GET | `?th=opus` | convert audio file to 128kbps opus |
+| GET | `?th=caf` | ...in the iOS-proprietary container |
+
+| method | body | result |
+|--|--|--|
+| jPOST | `{"q":"foo"}` | do a server-wide search; see the `[🔎]` search tab `raw` field for syntax |
+
+| method | params | body | result |
+|--|--|--|--|
+| jPOST | `?tar` | `["foo","bar"]` | download folders `foo` and `bar` inside URL as a tar file |
+
+## write
+
+| method | params | result |
+|--|--|--|
+| GET | `?move=/foo/bar` | move/rename the file/folder at URL to /foo/bar |
+
+| method | params | body | result |
+|--|--|--|--|
+| PUT | | (binary data) | upload into file at URL |
+| PUT | `?gz` | (binary data) | compress with gzip and write into file at URL |
+| PUT | `?xz` | (binary data) | compress with xz and write into file at URL |
+| mPOST | | `act=bput`, `f=FILE` | upload `FILE` into the folder at URL |
+| mPOST | `?j` | `act=bput`, `f=FILE` | ...and reply with json |
+| mPOST | | `act=mkdir`, `name=foo` | create directory `foo` at URL |
+| GET | `?delete` | | delete URL recursively |
+| jPOST | `?delete` | `["/foo","/bar"]` | delete `/foo` and `/bar` recursively |
+| uPOST | | `msg=foo` | send message `foo` into server log |
+| mPOST | | `act=tput`, `body=TEXT` | overwrite markdown document at URL |
+
+upload modifiers:
+
+| http-header | url-param | effect |
+|--|--|--|
+| `Accept: url` | `want=url` | return just the file URL |
+| `Rand: 4` | `rand=4` | generate random filename with 4 characters |
+| `Life: 30` | `life=30` | delete file after 30 seconds |
+
+* `life` only has an effect if the volume has a lifetime, and the volume lifetime must be greater than the file's
+
+* server behavior of `msg` can be reconfigured with `--urlform`
+
+## admin
+
+| method | params | result |
+|--|--|--|
+| GET | `?reload=cfg` | reload config files and rescan volumes |
+| GET | `?scan` | initiate a rescan of the volume which provides URL |
+| GET | `?stack` | show a stacktrace of all threads |
+
+## general
+
+| method | params | result |
+|--|--|--|
+| GET | `?pw=x` | logout |
+
+
+# assumptions
+
+## mdns
+
+* outgoing replies will always fit in one packet
+* if a client mentions any of our services, assume it's not missing any
+* always answer with all services, even if the client only asked for a few
+* not-impl: probe tiebreaking (too complicated)
+* not-impl: unicast listen (assume avahi took it)
+
+
+# sfx repack
+
+reduce the size of an sfx by removing features
+
+if you don't need all the features, you can repack the sfx and save a bunch of space; all you need is an sfx and a copy of this repo (nothing else to download or build, except if you're on windows then you need msys2 or WSL)
+* `393k` size of original sfx.py as of v1.1.3
+* `310k` after `./scripts/make-sfx.sh re no-cm`
+* `269k` after `./scripts/make-sfx.sh re no-cm no-hl`
+
+the features you can opt to drop are
+* `cm`/easymde, the "fancy" markdown editor, saves ~82k
+* `hl`, prism, the syntax hilighter, saves ~41k
+* `fnt`, source-code-pro, the monospace font, saves ~9k
+* `dd`, the custom mouse cursor for the media player tray tab, saves ~2k
+
+for the `re`pack to work, first run one of the sfx'es once to unpack it
+
+**note:** you can also just download and run [/scripts/copyparty-repack.sh](https://github.com/9001/copyparty/blob/hovudstraum/scripts/copyparty-repack.sh) -- this will grab the latest copyparty release from github and do a few repacks; works on linux/macos (and windows with msys2 or WSL)
+
+
+# building
+
+## dev env setup
+
+you need python 3.9 or newer due to type hints
+
+the rest is mostly optional; if you need a working env for vscode or similar
+
+```sh
+python3 -m venv .venv
+. .venv/bin/activate
+pip install jinja2 strip_hints  # MANDATORY
+pip install mutagen  # audio metadata
+pip install pyftpdlib  # ftp server
+pip install impacket  # smb server -- disable Windows Defender if you REALLY need this on windows
+pip install Pillow pyheif-pillow-opener pillow-avif-plugin  # thumbnails
+pip install black==21.12b0 click==8.0.2 bandit pylint flake8 isort mypy  # vscode tooling
+```
+
+
+## just the sfx
+
+first grab the web-dependencies from a previous sfx (assuming you don't need to modify something in those):
+
+```sh
+rm -rf copyparty/web/deps
+curl -L https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.py >x.py
+python3 x.py --version
+rm x.py
+mv /tmp/pe-copyparty/copyparty/web/deps/ copyparty/web/deps/
+```
+
+then build the sfx using any of the following examples:
+
+```sh
+./scripts/make-sfx.sh           # regular edition
+./scripts/make-sfx.sh gz no-cm  # gzip-compressed + no fancy markdown editor
+```
+
+
+## complete release
+
+also builds the sfx so skip the sfx section above
+
+in the `scripts` folder:
+
+* run `make -C deps-docker` to build all dependencies
+* run `./rls.sh 1.2.3` which uploads to pypi + creates github release + sfx
+
+
+# todo
+
+roughly sorted by priority
+
+* nothing! currently
+
+
+## discarded ideas
+
+* reduce up2k roundtrips
+  * start from a chunk index and just go
+  * terminate client on bad data
+    * not worth the effort, just throw enough conncetions at it
+* single sha512 across all up2k chunks?
+  * crypto.subtle cannot into streaming, would have to use hashwasm, expensive
+* separate sqlite table per tag
+  * performance fixed by skipping some indexes (`+mt.k`)
+* audio fingerprinting
+  * only makes sense if there can be a wasm client and that doesn't exist yet (except for olaf which is agpl hence counts as not existing)
+* `os.copy_file_range` for up2k cloning
+  * almost never hit this path anyways
+* up2k partials ui
+  * feels like there isn't much point
+* cache sha512 chunks on client
+  * too dangerous -- overtaken by turbo mode
+* comment field
+  * nah
+* look into android thumbnail cache file format
+  * absolutely not
+* indexedDB for hashes, cfg enable/clear/sz, 2gb avail, ~9k for 1g, ~4k for 100m, 500k items before autoeviction
+  * blank hashlist when up-ok to skip handshake
+    * too many confusing side-effects
+* hls framework for Someone Else to drop code into :^)
+  * probably not, too much stuff to consider -- seeking, start at offset, task stitching (probably np-hard), conditional passthru, rate-control (especially multi-consumer), session keepalive, cache mgmt...

docs/example2.conf

@@ -0,0 +1,13 @@
+# you can include additional config like this
+# (the space after the % is important)
+#
+# since copyparty.d is a folder, it'll include all *.conf
+# files inside (not recursively) in alphabetical order
+# (not necessarily same as numerical/natural order)
+#
+# paths are relative from the location of each included file
+# unless the path is absolute, for example % /etc/copyparty.d
+#
+# max include depth is 64
+
+% copyparty.d

docs/lics.txt

@@ -6,17 +6,25 @@ L: MIT
 
 https://github.com/pallets/jinja/
 C: 2007 Pallets
-L: BSD 3-Clause 
+L: BSD 3-Clause
 
 https://github.com/pallets/markupsafe/
 C: 2010 Pallets
-L: BSD 3-Clause 
+L: BSD 3-Clause
+
+https://github.com/paulc/dnslib/
+C: 2010-2017 Paul Chakravarti
+L: BSD 2-Clause
+
+https://github.com/pydron/ifaddr/
+C: 2014 Stefan C. Mueller
+L: BSD-2-Clause
 
 https://github.com/giampaolo/pyftpdlib/
-C: 2007 Giampaolo Rodola'
+C: 2007 Giampaolo Rodola
 L: MIT
 
-https://github.com/nayuki/QR-Code-generator
+https://github.com/nayuki/QR-Code-generator/
 C: Project Nayuki
 L: MIT
 

docs/notes.sh

@@ -67,6 +67,7 @@ mkdir -p "${dirs[@]}"
 for dir in "${dirs[@]}"; do for fn in ふが "$(printf \\xed\\x93)" 'qw,er;ty%20as df?gh+jkl%zxc&vbn <qwe>"rty'"'"'uio&asd&nbsp;fgh'; do echo "$dir" > "$dir/$fn.html"; done; done
 # qw er+ty%20ui%%20op<as>df&gh&amp;jk#zx'cv"bn`m=qw*er^ty?ui@op,as.df-gh_jk
 
+
 ##
 ## upload mojibake
 
@@ -143,6 +144,17 @@ sqlite3 -readonly up2k.db.key-full 'select w, v from mt where k = "key" order by
 sqlite3 -readonly up2k.db.key-full 'select w, v from mt where k = "key" order by w' > k1; sqlite3 -readonly up2k.db 'select mt.w, mt.v, up.rd, up.fn from mt inner join up on mt.w = substr(up.w,1,16) where mt.k = "key" order by up.rd, up.fn' > k2; ok=0; ng=0; while IFS='|' read w k2 path; do k1="$(grep -E "^$w" k1 | sed -r 's/.*\|//')"; [ "$k1" = "$k2" ] && ok=$((ok+1)) || { ng=$((ng+1)); printf '%3s %3s  %s\n' "$k1" "$k2" "$path"; }; done < <(cat k2); echo "match $ok   diff $ng"
 
 
+##
+## scanning for exceptions
+
+cd /dev/shm
+journalctl -aS '720 hour ago' -t python3 -o with-unit --utc | cut -d\  -f2,6- > cpp.log
+tac cpp.log | awk '/RuntimeError: generator ignored GeneratorExit/{n=1} n{n--;if(n==0)print} 1' | grep 'generator ignored GeneratorExit' -C7 | head -n 100
+awk '/Exception ignored in: <generator object StreamZip.gen/{s=1;next} /could not create thumbnail/{s=3;next} s{s--;next} 1' <cpp.log | less -R
+less-search:
+  >: |Exception|Traceback
+
+
 ##
 ## tracking bitflips
 
@@ -168,6 +180,7 @@ printf '  %s [%s]\n' $h2 "$(grep -F $h2 <handshakes | head -n 1)"
 # BUT the clients will immediately re-handshake the upload with the same bitflipped hashes, so the uploaders have to refresh their browsers before you do that,
 # so maybe just ask them to refresh and do nothing for 6 hours so the timeout kicks in, which deletes the placeholders/name-reservations and you can then manually delete the .PARTIALs at some point later
 
+
 ##
 ## media
 
@@ -214,9 +227,6 @@ for d in /usr /var; do find $d -type f -size +30M 2>/dev/null; done | while IFS=
 brew install python@2
 pip install virtualenv
 
-# readme toc
-cat README.md | awk 'function pr() { if (!h) {return}; if (/^ *[*!#|]/||!s) {printf "%s\n",h;h=0;return}; if (/.../) {printf "%s - %s\n",h,$0;h=0}; };    /^#/{s=1;pr()} /^#* *(install on android|dev env setup|just the sfx|complete release|optional gpl stuff)|`$/{s=0} /^#/{lv=length($1);sub(/[^ ]+ /,"");bab=$0;gsub(/ /,"-",bab);gsub(/\./,"",bab); h=sprintf("%" ((lv-1)*4+1) "s [%s](#%s)", "*",$0,bab);next} !h{next} {sub(/  .*/,"");sub(/[:;,]$/,"")} {pr()}' > toc; grep -E '^## readme toc' -B1000 -A2 <README.md >p1; grep -E '^## quickstart' -B2 -A999999 <README.md >p2; (cat p1; grep quickstart -A1000 <toc; cat p2) >README.md; rm p1 p2 toc
-
 # fix firefox phantom breakpoints,
 # suggestions from bugtracker, doesnt work (debugger is not attachable)
 devtools settings >> advanced >> enable browser chrome debugging + enable remote debugging

docs/rclone.md

@@ -4,25 +4,32 @@ speed estimates with server and client on the same win10 machine:
 * `1070 MiB/s` with rclone as both server and client
 * `570 MiB/s` with rclone-client and `copyparty -ed -j16` as server
 * `220 MiB/s` with rclone-client and `copyparty -ed` as server
-* `100 MiB/s` with [../bin/copyparty-fuse.py](../bin/copyparty-fuse.py) as client
+* `100 MiB/s` with [../bin/partyfuse.py](../bin/partyfuse.py) as client
 
 when server is on another machine (1gbit LAN),
-* `75 MiB/s` with [../bin/copyparty-fuse.py](../bin/copyparty-fuse.py) as client
+* `75 MiB/s` with [../bin/partyfuse.py](../bin/partyfuse.py) as client
 * `92 MiB/s` with rclone-client and `copyparty -ed` as server
 * `103 MiB/s` (connection max) with `copyparty -ed -j16` and all the others
 
 
 # creating the config file
 
-if you want to use password auth, add `headers = Cookie,cppwd=fgsfds` below
+replace `hunter2` with your password, or remove the `hunter2` lines if you allow anonymous access
 
 
 ### on windows clients:
 ```
 (
-echo [cpp]
+echo [cpp-rw]
+echo type = webdav
+echo vendor = other
+echo url = http://127.0.0.1:3923/
+echo headers = Cookie,cppwd=hunter2
+echo(
+echo [cpp-ro]
 echo type = http
 echo url = http://127.0.0.1:3923/
+echo headers = Cookie,cppwd=hunter2
 ) > %userprofile%\.config\rclone\rclone.conf
 ```
 
@@ -32,16 +39,26 @@ also install the windows dependencies: [winfsp](https://github.com/billziss-gh/w
 ### on unix clients:
 ```
 cat > ~/.config/rclone/rclone.conf <<'EOF'
-[cpp]
+[cpp-rw]
+type = webdav
+vendor = other
+url = http://127.0.0.1:3923/
+headers = Cookie,cppwd=hunter2
+
+[cpp-ro]
 type = http
 url = http://127.0.0.1:3923/
+headers = Cookie,cppwd=hunter2
 EOF
 ```
 
 
 # mounting the copyparty server locally
+
+connect to `cpp-rw:` for read-write, or `cpp-ro:` for read-only (twice as fast):
+
 ```
-rclone.exe mount --vfs-cache-max-age 5s --attr-timeout 5s --dir-cache-time 5s cpp: Z:
+rclone.exe mount --vfs-cache-mode writes --vfs-cache-max-age 5s --attr-timeout 5s --dir-cache-time 5s cpp-rw: W:
 ```
 
 
@@ -51,12 +68,5 @@ feels out of place but is too good not to mention
 
 ```
 rclone.exe serve http --read-only .
+rclone.exe serve webdav .
 ```
-
-* `webdav` gives write-access but `http` is twice as fast
-* `ftp` is buggy, avoid
-
-
-# bugs
-
-* rclone-client throws an exception if you try to read an empty file (should return zero bytes)

scripts/copyparty-repack.sh

@@ -37,7 +37,7 @@ set -e
 #  23663  copyparty-extras/up2k.py
 #           `- standalone utility to upload or search for files
 #
-#  32280  copyparty-extras/copyparty-fuse.py
+#  32280  copyparty-extras/partyfuse.py
 #           `- standalone to mount a URL as a local read-only filesystem
 #
 # 270004  copyparty
@@ -119,7 +119,7 @@ chmod 755 \
 
 # extract the sfx
 ( cd copyparty-extras/sfx-full/
-./copyparty-sfx.py -h
+./copyparty-sfx.py --version
 )
 
 
@@ -148,7 +148,7 @@ repack sfx-lite "re no-dd no-cm no-hl gz"
 # delete extracted source code
 ( cd copyparty-extras/
 mv copyparty-*/bin/up2k.py .
-mv copyparty-*/bin/copyparty-fuse.py .
+mv copyparty-*/bin/partyfuse.py .
 cp -pv sfx-lite/copyparty-sfx.py ../copyparty
 rm -rf copyparty-{0..9}*.*.*{0..9}
 )

scripts/deps-docker/Dockerfile

@@ -1,10 +1,11 @@
-FROM    alpine:3
+# TODO easymde embeds codemirror on 3.17 due to new npm probably
+FROM    alpine:3.16
 WORKDIR /z
 ENV     ver_asmcrypto=c72492f4a66e17a0e5dd8ad7874de354f3ccdaa5 \
         ver_hashwasm=4.9.0 \
-        ver_marked=4.0.18 \
+        ver_marked=4.2.3 \
         ver_mde=2.18.0 \
-        ver_codemirror=5.65.9 \
+        ver_codemirror=5.65.10 \
         ver_fontawesome=5.13.0 \
         ver_zopfli=1.0.3
 

scripts/deps-docker/marked-ln.patch

@@ -1,5 +1,5 @@
 diff --git a/src/Lexer.js b/src/Lexer.js
-adds linetracking to marked.js v4.0.17;
+adds linetracking to marked.js v4.2.3;
 add data-ln="%d" to most tags, %d is the source markdown line
 --- a/src/Lexer.js
 +++ b/src/Lexer.js
@@ -123,20 +123,20 @@ add data-ln="%d" to most tags, %d is the source markdown line
 +        this.ln++;
          lastToken = tokens[tokens.length - 1];
          if (lastToken && lastToken.type === 'text') {
-@@ -365,4 +396,5 @@ export class Lexer {
+@@ -367,4 +398,5 @@ export class Lexer {
            if (token = extTokenizer.call({ lexer: this }, src, tokens)) {
              src = src.substring(token.raw.length);
 +            this.ln = token.ln || this.ln;
              tokens.push(token);
              return true;
-@@ -430,4 +462,6 @@ export class Lexer {
+@@ -432,4 +464,6 @@ export class Lexer {
        if (token = this.tokenizer.br(src)) {
          src = src.substring(token.raw.length);
 +        // no need to reset (no more blockTokens anyways)
 +        token.ln = this.ln++;
          tokens.push(token);
          continue;
-@@ -472,4 +506,5 @@ export class Lexer {
+@@ -474,4 +508,5 @@ export class Lexer {
        if (token = this.tokenizer.inlineText(cutSrc, smartypants)) {
          src = src.substring(token.raw.length);
 +        this.ln = token.ln || this.ln;
@@ -234,7 +234,7 @@ index 7c36a75..aa1a53a 100644
 -    return '<pre><code class="'
 +    return '<pre' + this.ln + '><code class="'
        + this.options.langPrefix
-       + escape(lang, true)
+       + escape(lang)
 @@ -43,5 +49,5 @@ export class Renderer {
     */
    blockquote(quote) {
@@ -293,7 +293,7 @@ diff --git a/src/Tokenizer.js b/src/Tokenizer.js
 index e8a69b6..2cc772b 100644
 --- a/src/Tokenizer.js
 +++ b/src/Tokenizer.js
-@@ -302,4 +302,7 @@ export class Tokenizer {
+@@ -312,4 +312,7 @@ export class Tokenizer {
        const l = list.items.length;
  
 +      // each nested list gets +1 ahead; this hack makes every listgroup -1 but atleast it doesn't get infinitely bad

scripts/deps-docker/marked.patch

@@ -1,35 +1,35 @@
 diff --git a/src/Lexer.js b/src/Lexer.js
 --- a/src/Lexer.js
 +++ b/src/Lexer.js
-@@ -6,5 +6,5 @@ import { repeatString } from './helpers.js';
- /**
+@@ -7,5 +7,5 @@ import { repeatString } from './helpers.js';
   * smartypants text replacement
+  * @param {string} text
 - */
 + *
  function smartypants(text) {
    return text
-@@ -27,5 +27,5 @@ function smartypants(text) {
- /**
+@@ -29,5 +29,5 @@ function smartypants(text) {
   * mangle email addresses
+  * @param {string} text
 - */
 + *
  function mangle(text) {
    let out = '',
-@@ -466,5 +466,5 @@ export class Lexer {
+@@ -478,5 +478,5 @@ export class Lexer {
  
        // autolink
 -      if (token = this.tokenizer.autolink(src, mangle)) {
 +      if (token = this.tokenizer.autolink(src)) {
          src = src.substring(token.raw.length);
          tokens.push(token);
-@@ -473,5 +473,5 @@ export class Lexer {
+@@ -485,5 +485,5 @@ export class Lexer {
  
        // url (gfm)
 -      if (!this.state.inLink && (token = this.tokenizer.url(src, mangle))) {
 +      if (!this.state.inLink && (token = this.tokenizer.url(src))) {
          src = src.substring(token.raw.length);
          tokens.push(token);
-@@ -494,5 +494,5 @@ export class Lexer {
+@@ -506,5 +506,5 @@ export class Lexer {
          }
        }
 -      if (token = this.tokenizer.inlineText(cutSrc, smartypants)) {
@@ -39,15 +39,15 @@ diff --git a/src/Lexer.js b/src/Lexer.js
 diff --git a/src/Renderer.js b/src/Renderer.js
 --- a/src/Renderer.js
 +++ b/src/Renderer.js
-@@ -142,5 +142,5 @@ export class Renderer {
- 
+@@ -173,5 +173,5 @@ export class Renderer {
+    */
    link(href, title, text) {
 -    href = cleanUrl(this.options.sanitize, this.options.baseUrl, href);
 +    href = cleanUrl(this.options.baseUrl, href);
      if (href === null) {
        return text;
-@@ -155,5 +155,5 @@ export class Renderer {
- 
+@@ -191,5 +191,5 @@ export class Renderer {
+    */
    image(href, title, text) {
 -    href = cleanUrl(this.options.sanitize, this.options.baseUrl, href);
 +    href = cleanUrl(this.options.baseUrl, href);
@@ -56,7 +56,7 @@ diff --git a/src/Renderer.js b/src/Renderer.js
 diff --git a/src/Tokenizer.js b/src/Tokenizer.js
 --- a/src/Tokenizer.js
 +++ b/src/Tokenizer.js
-@@ -320,14 +320,7 @@ export class Tokenizer {
+@@ -352,14 +352,7 @@ export class Tokenizer {
          type: 'html',
          raw: cap[0],
 -        pre: !this.options.sanitizer
@@ -65,14 +65,14 @@ diff --git a/src/Tokenizer.js b/src/Tokenizer.js
          text: cap[0]
        };
 -      if (this.options.sanitize) {
+-        const text = this.options.sanitizer ? this.options.sanitizer(cap[0]) : escape(cap[0]);
 -        token.type = 'paragraph';
--        token.text = this.options.sanitizer ? this.options.sanitizer(cap[0]) : escape(cap[0]);
--        token.tokens = [];
--        this.lexer.inline(token.text, token.tokens);
+-        token.text = text;
+-        token.tokens = this.lexer.inline(text);
 -      }
        return token;
      }
-@@ -476,15 +469,9 @@ export class Tokenizer {
+@@ -502,15 +495,9 @@ export class Tokenizer {
  
        return {
 -        type: this.options.sanitize
@@ -90,7 +90,7 @@ diff --git a/src/Tokenizer.js b/src/Tokenizer.js
 +        text: cap[0]
        };
      }
-@@ -671,10 +658,10 @@ export class Tokenizer {
+@@ -699,10 +686,10 @@ export class Tokenizer {
    }
  
 -  autolink(src, mangle) {
@@ -103,7 +103,7 @@ diff --git a/src/Tokenizer.js b/src/Tokenizer.js
 +        text = escape(cap[1]);
          href = 'mailto:' + text;
        } else {
-@@ -699,10 +686,10 @@ export class Tokenizer {
+@@ -727,10 +714,10 @@ export class Tokenizer {
    }
  
 -  url(src, mangle) {
@@ -116,7 +116,7 @@ diff --git a/src/Tokenizer.js b/src/Tokenizer.js
 +        text = escape(cap[0]);
          href = 'mailto:' + text;
        } else {
-@@ -736,12 +723,12 @@ export class Tokenizer {
+@@ -764,12 +751,12 @@ export class Tokenizer {
    }
  
 -  inlineText(src, smartypants) {
@@ -135,7 +135,7 @@ diff --git a/src/Tokenizer.js b/src/Tokenizer.js
 diff --git a/src/defaults.js b/src/defaults.js
 --- a/src/defaults.js
 +++ b/src/defaults.js
-@@ -9,12 +9,8 @@ export function getDefaults() {
+@@ -10,11 +10,7 @@ export function getDefaults() {
      highlight: null,
      langPrefix: 'language-',
 -    mangle: true,
@@ -144,16 +144,15 @@ diff --git a/src/defaults.js b/src/defaults.js
 -    sanitize: false,
 -    sanitizer: null,
      silent: false,
-     smartLists: false,
 -    smartypants: false,
      tokenizer: null,
      walkTokens: null,
 diff --git a/src/helpers.js b/src/helpers.js
 --- a/src/helpers.js
 +++ b/src/helpers.js
-@@ -64,18 +64,5 @@ export function edit(regex, opt) {
- const nonWordAndColonTest = /[^\w:]/g;
- const originIndependentUrl = /^$|^[a-z][a-z0-9+.-]*:|^[?#]/i;
+@@ -78,18 +78,5 @@ const originIndependentUrl = /^$|^[a-z][a-z0-9+.-]*:|^[?#]/i;
+  * @param {string} href
+  */
 -export function cleanUrl(sanitize, base, href) {
 -  if (sanitize) {
 -    let prot;
@@ -171,7 +170,7 @@ diff --git a/src/helpers.js b/src/helpers.js
 +export function cleanUrl(base, href) {
    if (base && !originIndependentUrl.test(href)) {
      href = resolveUrl(base, href);
-@@ -227,10 +214,4 @@ export function findClosingBracket(str, b) {
+@@ -250,10 +237,4 @@ export function findClosingBracket(str, b) {
  }
  
 -export function checkSanitizeDeprecation(opt) {
@@ -181,7 +180,7 @@ diff --git a/src/helpers.js b/src/helpers.js
 -}
 -
  // copied from https://stackoverflow.com/a/5450113/806777
- export function repeatString(pattern, count) {
+ /**
 diff --git a/src/marked.js b/src/marked.js
 --- a/src/marked.js
 +++ b/src/marked.js
@@ -197,13 +196,13 @@ diff --git a/src/marked.js b/src/marked.js
 -  checkSanitizeDeprecation(opt);
  
    if (callback) {
-@@ -302,5 +300,4 @@ marked.parseInline = function(src, opt) {
+@@ -318,5 +316,4 @@ marked.parseInline = function(src, opt) {
  
    opt = merge({}, marked.defaults, opt || {});
 -  checkSanitizeDeprecation(opt);
  
    try {
-@@ -311,5 +308,5 @@ marked.parseInline = function(src, opt) {
+@@ -327,5 +324,5 @@ marked.parseInline = function(src, opt) {
      return Parser.parseInline(tokens, opt);
    } catch (e) {
 -    e.message += '\nPlease report this to https://github.com/markedjs/marked.';
@@ -213,42 +212,24 @@ diff --git a/src/marked.js b/src/marked.js
 diff --git a/test/bench.js b/test/bench.js
 --- a/test/bench.js
 +++ b/test/bench.js
-@@ -37,5 +37,4 @@ export async function runBench(options) {
+@@ -39,5 +39,4 @@ export async function runBench(options) {
      breaks: false,
      pedantic: false,
--    sanitize: false,
-     smartLists: false
+-    sanitize: false
    });
-@@ -49,5 +48,4 @@ export async function runBench(options) {
+   if (options.marked) {
+@@ -50,5 +49,4 @@ export async function runBench(options) {
      breaks: false,
      pedantic: false,
--    sanitize: false,
-     smartLists: false
-   });
-@@ -62,5 +60,4 @@ export async function runBench(options) {
-     breaks: false,
-     pedantic: false,
--    sanitize: false,
-     smartLists: false
-   });
-@@ -74,5 +71,4 @@ export async function runBench(options) {
-     breaks: false,
-     pedantic: false,
--    sanitize: false,
-     smartLists: false
-   });
-@@ -87,5 +83,4 @@ export async function runBench(options) {
-     breaks: false,
-     pedantic: true,
--    sanitize: false,
-     smartLists: false
-   });
-@@ -99,5 +94,4 @@ export async function runBench(options) {
-     breaks: false,
-     pedantic: true,
--    sanitize: false,
-     smartLists: false
+-    sanitize: false
    });
+   if (options.marked) {
+@@ -61,5 +59,4 @@ export async function runBench(options) {
+   //   breaks: false,
+   //   pedantic: false,
+-  //   sanitize: false
+   // });
+   // if (options.marked) {
 diff --git a/test/specs/run-spec.js b/test/specs/run-spec.js
 --- a/test/specs/run-spec.js
 +++ b/test/specs/run-spec.js
@@ -269,70 +250,70 @@ diff --git a/test/specs/run-spec.js b/test/specs/run-spec.js
 diff --git a/test/unit/Lexer-spec.js b/test/unit/Lexer-spec.js
 --- a/test/unit/Lexer-spec.js
 +++ b/test/unit/Lexer-spec.js
-@@ -635,5 +635,5 @@ paragraph
+@@ -712,5 +712,5 @@ paragraph
      });
  
 -    it('sanitize', () => {
 +    /*it('sanitize', () => {
        expectTokens({
          md: '<div>html</div>',
-@@ -653,5 +653,5 @@ paragraph
+@@ -730,5 +730,5 @@ paragraph
          ]
        });
 -    });
 +    });*/
    });
  
-@@ -698,5 +698,5 @@ paragraph
+@@ -810,5 +810,5 @@ paragraph
        });
  
 -      it('html sanitize', () => {
 +      /*it('html sanitize', () => {
          expectInlineTokens({
            md: '<div>html</div>',
-@@ -706,5 +706,5 @@ paragraph
+@@ -818,5 +818,5 @@ paragraph
            ]
          });
 -      });
 +      });*/
  
        it('link', () => {
-@@ -1017,5 +1017,5 @@ paragraph
+@@ -1129,5 +1129,5 @@ paragraph
          });
  
 -        it('autolink mangle email', () => {
 +        /*it('autolink mangle email', () => {
            expectInlineTokens({
              md: '<test@example.com>',
-@@ -1037,5 +1037,5 @@ paragraph
+@@ -1149,5 +1149,5 @@ paragraph
              ]
            });
 -        });
 +        });*/
  
          it('url', () => {
-@@ -1074,5 +1074,5 @@ paragraph
+@@ -1186,5 +1186,5 @@ paragraph
          });
  
 -        it('url mangle email', () => {
 +        /*it('url mangle email', () => {
            expectInlineTokens({
              md: 'test@example.com',
-@@ -1094,5 +1094,5 @@ paragraph
+@@ -1206,5 +1206,5 @@ paragraph
              ]
            });
 -        });
 +        });*/
        });
  
-@@ -1110,5 +1110,5 @@ paragraph
+@@ -1222,5 +1222,5 @@ paragraph
        });
  
 -      describe('smartypants', () => {
 +      /*describe('smartypants', () => {
          it('single quotes', () => {
            expectInlineTokens({
-@@ -1180,5 +1180,5 @@ paragraph
+@@ -1292,5 +1292,5 @@ paragraph
            });
          });
 -      });

scripts/fusefuzz.py

@@ -6,7 +6,7 @@ import time
 """
 td=/dev/shm/; [ -e $td ] || td=$HOME; mkdir -p $td/fusefuzz/{r,v}
 PYTHONPATH=.. python3 -m copyparty -v $td/fusefuzz/r::r -i 127.0.0.1
-../bin/copyparty-fuse.py http://127.0.0.1:3923/ $td/fusefuzz/v -cf 2 -cd 0.5
+../bin/partyfuse.py http://127.0.0.1:3923/ $td/fusefuzz/v -cf 2 -cd 0.5
 (d="$PWD"; cd $td/fusefuzz && "$d"/fusefuzz.py)
 """
 

scripts/genlic.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+set -e
+
+outfile="$($(command -v realpath || command -v grealpath) "$1")"
+
+[ -e genlic.sh ] || cd scripts
+[ -e genlic.sh ]
+
+f=../build/mit.txt
+[ -e $f ] ||
+	curl https://opensource.org/licenses/MIT |
+	awk '/div>/{o=0}o>1;o{o++}/;COPYRIGHT HOLDER/{o=1}' |
+	awk '{gsub(/<[^>]+>/,"")};1' >$f
+
+f=../build/isc.txt
+[ -e $f ] ||
+	curl https://opensource.org/licenses/ISC |
+	awk '/div>/{o=0}o>2;o{o++}/;OWNER/{o=1}' |
+	awk '{gsub(/<[^>]+>/,"")};/./{b=0}!/./{b++}b>1{next}1' >$f
+
+f=../build/2bsd.txt
+[ -e $f ] ||
+	curl https://opensource.org/licenses/BSD-2-Clause |
+	awk '/div>/{o=0}o>1;o{o++}/HOLDER/{o=1}' |
+	awk '{gsub(/<[^>]+>/,"")};1' >$f
+
+f=../build/3bsd.txt
+[ -e $f ] ||
+	curl https://opensource.org/licenses/BSD-3-Clause |
+	awk '/div>/{o=0}o>1;o{o++}/HOLDER/{o=1}' |
+	awk '{gsub(/<[^>]+>/,"")};1' >$f
+
+f=../build/ofl.txt
+[ -e $f ] ||
+	curl https://opensource.org/licenses/OFL-1.1 |
+	awk '/PREAMBLE/{o=1}/sil\.org/{o=0}!o{next}/./{printf "%s ",$0;next}{print"\n"}' |
+	awk '{gsub(/<[^>]+>/,"");gsub(/^\s+/,"");gsub(/&amp;/,"\\&")}/./{b=0}!/./{b++}b>1{next}1' >$f
+
+(sed -r 's/^L: /License: /;s/^C: /Copyright (c) /' <../docs/lics.txt
+printf '\n\n--- MIT License ---\n\n'; cat ../build/mit.txt
+printf '\n\n--- ISC License ---\n\n'; cat ../build/isc.txt
+printf '\n\n--- BSD 2-Clause License ---\n\n'; cat ../build/2bsd.txt
+printf '\n\n--- BSD 3-Clause License ---\n\n'; cat ../build/3bsd.txt
+printf '\n\n--- SIL Open Font License v1.1 ---\n\n'; cat ../build/ofl.txt
+) |
+while IFS= read -r x; do
+	[ "${x:0:4}" = "--- " ] || {
+		printf '%s\n' "$x"
+		continue
+	}
+	n=${#x}
+	p=$(( (80-n)/2 ))
+	printf "%${p}s\033[07m%s\033[0m\n" "" "$x"
+done > "$outfile"

scripts/make-pypi-release.bat

@@ -1,5 +0,0 @@
-cd %~dp0\..
-python setup.py clean2
-python setup.py rstconv
-python setup.py sdist bdist_wheel --universal
-REM python setup.py sdist upload -r pypi

scripts/make-pypi-release.sh

@@ -55,14 +55,6 @@ EOF
 	# set pypi password
 	chmod 600 ~/.pypirc
 	sed -ri 's/qwer/username/;s/asdf/password/' ~/.pypirc
-
-	# if PY2: create build env
-	cd ~/dev/copyparty && virtualenv buildenv
-	(. buildenv/bin/activate && pip install twine)
-
-	# if PY3: create build env
-	cd ~/dev/copyparty && python3 -m venv buildenv
-	(. buildenv/bin/activate && pip install twine wheel)
 }
 
 
@@ -82,17 +74,33 @@ function have() {
 	python -c "import $1; $1; $1.__version__"
 }
 
-. buildenv/bin/activate
-have setuptools
-have wheel
-have twine
+function load_env() {
+	. buildenv/bin/activate
+	have setuptools
+	have wheel
+	have twine
+}
+
+load_env || {
+	echo creating buildenv
+	deactivate || true
+	rm -rf buildenv
+	python3 -m venv buildenv
+	(. buildenv/bin/activate && pip install twine wheel)
+	load_env
+}
 
 # remove type hints to support python < 3.9
 rm -rf build/pypi
 mkdir -p build/pypi
 cp -pR setup.py README.md LICENSE copyparty tests bin scripts/strip_hints build/pypi/
+tar -c docs/lics.txt scripts/genlic.sh build/*.txt | tar -xC build/pypi/
 cd build/pypi
-tar --strip-components=2 -xf ../strip-hints-0.1.10.tar.gz strip-hints-0.1.10/src/strip_hints
+f=../strip-hints-0.1.10.tar.gz
+[ -e $f ] || 
+	(url=https://files.pythonhosted.org/packages/9c/d4/312ddce71ee10f7e0ab762afc027e07a918f1c0e1be5b0069db5b0e7542d/strip-hints-0.1.10.tar.gz;
+	wget -O$f "$url" || curl -L "$url" >$f)
+tar --strip-components=2 -xf $f strip-hints-0.1.10/src/strip_hints
 python3 -c 'from strip_hints.a import uh; uh("copyparty")'
 
 ./setup.py clean2