v1.3.14

.gitignore

@@ -5,13 +5,16 @@ __pycache__/
 MANIFEST.in
 MANIFEST
 copyparty.egg-info/
-buildenv/
-build/
-dist/
-sfx/
-py2/
 .venv/
 
+/buildenv/
+/build/
+/dist/
+/py2/
+/sfx/
+/unt/
+/log/
+
 # ide
 *.sublime-workspace
 

README.md

@@ -56,8 +56,11 @@ try the **[read-only demo server](https://a.ocv.me/pub/demo/)** 👀 running fro
     * [searching](#searching) - search by size, date, path/name, mp3-tags, ...
 * [server config](#server-config) - using arguments or config files, or a mix of both
     * [ftp-server](#ftp-server) - an FTP server can be started using `--ftp 3921`
-    * [file indexing](#file-indexing)
-    * [upload rules](#upload-rules) - set upload rules using volume flags
+    * [file indexing](#file-indexing) - enables dedup and music search ++
+        * [exclude-patterns](#exclude-patterns) - to save some time
+        * [filesystem guards](#filesystem-guards) - avoid traversing into other filesystems
+        * [periodic rescan](#periodic-rescan) - filesystem monitoring
+    * [upload rules](#upload-rules) - set upload rules using volflags
     * [compress uploads](#compress-uploads) - files can be autocompressed on upload
     * [database location](#database-location) - in-volume (`.hist/up2k.db`, default) or somewhere else
     * [metadata from audio files](#metadata-from-audio-files) - set `-e2t` to index tags on upload
@@ -246,12 +249,18 @@ some improvement ideas
 * Windows: if the `up2k.db` (filesystem index) is on a samba-share or network disk, you'll get unpredictable behavior if the share is disconnected for a bit
   * use `--hist` or the `hist` volflag (`-v [...]:c,hist=/tmp/foo`) to place the db on a local disk instead
 * all volumes must exist / be available on startup; up2k (mtp especially) gets funky otherwise
+* [the database can get stuck](https://github.com/9001/copyparty/issues/10)
+  * has only happened once but that is once too many
+  * luckily not dangerous for file integrity and doesn't really stop uploads or anything like that
+  * but would really appreciate some logs if anyone ever runs into it again
 * probably more, pls let me know
 
 ## not my bugs
 
 * [Chrome issue 1317069](https://bugs.chromium.org/p/chromium/issues/detail?id=1317069) -- if you try to upload a folder which contains symlinks by dragging it into the browser, the symlinked files will not get uploaded
 
+* [Chrome issue 1352210](https://bugs.chromium.org/p/chromium/issues/detail?id=1352210) -- plaintext http may be faster at filehashing than https (but also extremely CPU-intensive)
+
 * iPhones: the volume control doesn't work because [apple doesn't want it to](https://developer.apple.com/library/archive/documentation/AudioVideo/Conceptual/Using_HTML5_Audio_Video/Device-SpecificConsiderations/Device-SpecificConsiderations.html#//apple_ref/doc/uid/TP40009523-CH5-SW11)
   * *future workaround:* enable the equalizer, make it all-zero, and set a negative boost to reduce the volume
     * "future" because `AudioContext` is broken in the current iOS version (15.1), maybe one day...
@@ -309,7 +318,7 @@ examples:
   * `u1` can open the `inc` folder, but cannot see the contents, only upload new files to it
   * `u2` can browse it and move files *from* `/inc` into any folder where `u2` has write-access
 * make folder `/mnt/ss` available at `/i`, read-write for u1, get-only for everyone else, and enable accesskeys: `-v /mnt/ss:i:rw,u1:g:c,fk=4`
-  * `c,fk=4` sets the `fk` volume-flag to 4, meaning each file gets a 4-character accesskey
+  * `c,fk=4` sets the `fk` volflag to 4, meaning each file gets a 4-character accesskey
   * `u1` can upload files, browse the folder, and see the generated accesskeys
   * other users cannot browse the folder, but can access the files if they have the full file URL with the accesskey
 
@@ -373,6 +382,7 @@ the browser has the following hotkeys  (always qwerty)
   * `Esc` close viewer
   * videos:
     * `U/O` skip 10sec back/forward
+    * `0..9` jump to 0%..90%
     * `P/K/Space` play/pause
     * `M` mute
     * `C` continue playing next video
@@ -481,6 +491,7 @@ see [up2k](#up2k) for details on how it works, or watch a [demo video](https://a
 the up2k UI is the epitome of polished inutitive experiences:
 * "parallel uploads" specifies how many chunks to upload at the same time
 * `[🏃]` analysis of other files should continue while one is uploading
+* `[🥔]` shows a simpler UI for faster uploads from slow devices
 * `[💭]` ask for confirmation before files are added to the queue
 * `[🔎]` switch between upload and [file-search](#file-search) mode
   * ignore `[🔎]` if you add files by dragging them into the browser
@@ -654,7 +665,9 @@ an FTP server can be started using `--ftp 3921`,  and/or `--ftps` for explicit T
 
 ## file indexing
 
-file indexing relies on two database tables, the up2k filetree (`-e2d`) and the metadata tags (`-e2t`), stored in `.hist/up2k.db`. Configuration can be done through arguments, volume flags, or a mix of both.
+enables dedup and music search ++
+
+file indexing relies on two database tables, the up2k filetree (`-e2d`) and the metadata tags (`-e2t`), stored in `.hist/up2k.db`. Configuration can be done through arguments, volflags, or a mix of both.
 
 through arguments:
 * `-e2d` enables file indexing on upload
@@ -663,8 +676,11 @@ through arguments:
 * `-e2t` enables metadata indexing on upload
 * `-e2ts` also scans for tags in all files that don't have tags yet
 * `-e2tsr` also deletes all existing tags, doing a full reindex
+* `-e2v` verfies file integrity at startup, comparing hashes from the db
+* `-e2vu` patches the database with the new hashes from the filesystem
+* `-e2vp` panics and kills copyparty instead
 
-the same arguments can be set as volume flags, in addition to `d2d`, `d2ds`, `d2t`, `d2ts` for disabling:
+the same arguments can be set as volflags, in addition to `d2d`, `d2ds`, `d2t`, `d2ts`, `d2v` for disabling:
 * `-v ~/music::r:c,e2dsa,e2tsr` does a full reindex of everything on startup
 * `-v ~/music::r:c,d2d` disables **all** indexing, even if any `-e2*` are on
 * `-v ~/music::r:c,d2t` disables all `-e2t*` (tags), does not affect `-e2d*`
@@ -676,7 +692,9 @@ note:
 * `e2tsr` is probably always overkill, since `e2ds`/`e2dsa` would pick up any file modifications and `e2ts` would then reindex those, unless there is a new copyparty version with new parsers and the release note says otherwise
 * the rescan button in the admin panel has no effect unless the volume has `-e2ds` or higher
 
-to save some time, you can provide a regex pattern for filepaths to only index by filename/path/size/last-modified (and not the hash of the file contents) by setting `--no-hash \.iso$` or the volume-flag `:c,nohash=\.iso$`, this has the following consequences:
+### exclude-patterns
+
+to save some time,  you can provide a regex pattern for filepaths to only index by filename/path/size/last-modified (and not the hash of the file contents) by setting `--no-hash \.iso$` or the volflag `:c,nohash=\.iso$`, this has the following consequences:
 * initial indexing is way faster, especially when the volume is on a network disk
 * makes it impossible to [file-search](#file-search)
 * if someone uploads the same file contents, the upload will not be detected as a dupe, so it will not get symlinked or rejected
@@ -685,12 +703,29 @@ similarly, you can fully ignore files/folders using `--no-idx [...]` and `:c,noi
 
 if you set `--no-hash [...]` globally, you can enable hashing for specific volumes using flag `:c,nohash=`
 
+### filesystem guards
+
+avoid traversing into other filesystems  using `--xdev` / volflag `:c,xdev`, skipping any symlinks or bind-mounts to another HDD for example
+
+and/or you can `--xvol` / `:c,xvol` to ignore all symlinks leaving the volume's top directory, but still allow bind-mounts pointing elsewhere
+
+**NB: only affects the indexer** -- users can still access anything inside a volume, unless shadowed by another volume
+
+### periodic rescan
+
+filesystem monitoring;  if copyparty is not the only software doing stuff on your filesystem, you may want to enable periodic rescans to keep the index up to date
+
+argument `--re-maxage 60` will rescan all volumes every 60 sec, same as volflag `:c,scan=60` to specify it per-volume
+
+uploads are disabled while a rescan is happening, so rescans will be delayed by `--db-act` (default 10 sec) when there is write-activity going on (uploads, renames, ...)
+
 
 ## upload rules
 
-set upload rules using volume flags,  some examples:
+set upload rules using volflags,  some examples:
 
 * `:c,sz=1k-3m` sets allowed filesize between 1 KiB and 3 MiB inclusive (suffixes: `b`, `k`, `m`, `g`)
+* `:c,df=4g` block uploads if there would be less than 4 GiB free disk space afterwards
 * `:c,nosub` disallow uploading into subdirectories; goes well with `rotn` and `rotf`:
 * `:c,rotn=1000,2` moves uploads into subfolders, up to 1000 files in each folder before making a new one, two levels deep (must be at least 1)
 * `:c,rotf=%Y/%m/%d/%H` enforces files to be uploaded into a structure of subfolders according to that date format
@@ -709,16 +744,16 @@ you can also set transaction limits which apply per-IP and per-volume, but these
 
 files can be autocompressed on upload,  either on user-request (if config allows) or forced by server-config
 
-* volume flag `gz` allows gz compression
-* volume flag `xz` allows lzma compression
-* volume flag `pk` **forces** compression on all files
+* volflag `gz` allows gz compression
+* volflag `xz` allows lzma compression
+* volflag `pk` **forces** compression on all files
 * url parameter `pk` requests compression with server-default algorithm
 * url parameter `gz` or `xz` requests compression with a specific algorithm
 * url parameter `xz` requests xz compression
 
 things to note,
 * the `gz` and `xz` arguments take a single optional argument, the compression level (range 0 to 9)
-* the `pk` volume flag takes the optional argument `ALGORITHM,LEVEL` which will then be forced for all uploads, for example `gz,9` or `xz,0`
+* the `pk` volflag takes the optional argument `ALGORITHM,LEVEL` which will then be forced for all uploads, for example `gz,9` or `xz,0`
 * default compression is gzip level 9
 * all upload methods except up2k are supported
 * the files will be indexed after compression, so dupe-detection and file-search will not work as expected
@@ -738,7 +773,7 @@ in-volume (`.hist/up2k.db`, default) or somewhere else
 
 copyparty creates a subfolder named `.hist` inside each volume where it stores the database, thumbnails, and some other stuff
 
-this can instead be kept in a single place using the `--hist` argument, or the `hist=` volume flag, or a mix of both:
+this can instead be kept in a single place using the `--hist` argument, or the `hist=` volflag, or a mix of both:
 * `--hist ~/.cache/copyparty -v ~/music::r:c,hist=-` sets `~/.cache/copyparty` as the default place to put volume info, but `~/music` gets the regular `.hist` subfolder (`-` restores default behavior)
 
 note:
@@ -776,7 +811,7 @@ see the beautiful mess of a dictionary in [mtag.py](https://github.com/9001/copy
 
 provide custom parsers to index additional tags, also see [./bin/mtag/README.md](./bin/mtag/README.md)
 
-copyparty can invoke external programs to collect additional metadata for files using `mtp` (either as argument or volume flag), there is a default timeout of 30sec, and only files which contain audio get analyzed by default (see ay/an/ad below)
+copyparty can invoke external programs to collect additional metadata for files using `mtp` (either as argument or volflag), there is a default timeout of 30sec, and only files which contain audio get analyzed by default (see ay/an/ad below)
 
 * `-mtp .bpm=~/bin/audio-bpm.py` will execute `~/bin/audio-bpm.py` with the audio file as argument 1 to provide the `.bpm` tag, if that does not exist in the audio metadata
 * `-mtp key=f,t5,~/bin/audio-key.py` uses `~/bin/audio-key.py` to get the `key` tag, replacing any existing metadata tag (`f,`), aborting if it takes longer than 5sec (`t5,`)
@@ -817,8 +852,8 @@ if this becomes popular maybe there should be a less janky way to do it actually
 tell search engines you dont wanna be indexed,  either using the good old [robots.txt](https://www.robotstxt.org/robotstxt.html) or through copyparty settings:
 
 * `--no-robots` adds HTTP (`X-Robots-Tag`) and HTML (`<meta>`) headers with `noindex, nofollow` globally
-* volume-flag `[...]:c,norobots` does the same thing for that single volume
-* volume-flag `[...]:c,robots` ALLOWS search-engine crawling for that volume, even if `--no-robots` is set globally
+* volflag `[...]:c,norobots` does the same thing for that single volume
+* volflag `[...]:c,robots` ALLOWS search-engine crawling for that volume, even if `--no-robots` is set globally
 
 also, `--force-js` disables the plain HTML folder listing, making things harder to parse for search engines
 
@@ -964,6 +999,12 @@ quick outline of the up2k protocol, see [uploading](#uploading) for the web-clie
 
 up2k has saved a few uploads from becoming corrupted in-transfer already; caught an android phone on wifi redhanded in wireshark with a bitflip, however bup with https would *probably* have noticed as well (thanks to tls also functioning as an integrity check)
 
+regarding the frequent server log message during uploads;  
+`6.0M 106M/s 2.77G 102.9M/s n948 thank 4/0/3/1 10042/7198 00:01:09`
+* this chunk was `6 MiB`, uploaded at `106 MiB/s`
+* on this http connection, `2.77 GiB` transferred, `102.9 MiB/s` average, `948` chunks handled
+* client says `4` uploads OK, `0` failed, `3` busy, `1` queued, `10042 MiB` total size, `7198 MiB` and `00:01:09` left
+
 
 ## why chunk-hashes
 
@@ -973,6 +1014,10 @@ this is due to `crypto.subtle` [not yet](https://github.com/w3c/webcrypto/issues
 
 as a result, the hashes are much less useful than they could have been (search the server by sha512, provide the sha512 in the response http headers, ...)
 
+however it allows for hashing multiple chunks in parallel, greatly increasing upload speed from fast storage (NVMe, raid-0 and such)
+
+* both the [browser uploader](#uploading) and the [commandline one](https://github.com/9001/copyparty/blob/hovudstraum/bin/up2k.py) does this now, allowing for fast uploading even from plaintext http
+
 hashwasm would solve the streaming issue but reduces hashing speed for sha512 (xxh128 does 6 GiB/s), and it would make old browsers and [iphones](https://bugs.webkit.org/show_bug.cgi?id=228552) unsupported
 
 * blake2 might be a better choice since xxh is non-cryptographic, but that gets ~15 MiB/s on slower androids
@@ -1006,6 +1051,7 @@ when uploading files,
 
 * if you're cpu-bottlenecked, or the browser is maxing a cpu core:
   * up to 30% faster uploads if you hide the upload status list by switching away from the `[🚀]` up2k ui-tab (or closing it)
+    * optionally you can switch to the lightweight potato ui by clicking the `[🥔]`
     * switching to another browser-tab also works, the favicon will update every 10 seconds in that case
   * unlikely to be a problem, but can happen when uploding many small files, or your internet is too fast, or PC too slow
 
@@ -1035,7 +1081,7 @@ some notes on hardening
 other misc notes:
 
 * you can disable directory listings by giving permission `g` instead of `r`, only accepting direct URLs to files
-  * combine this with volume-flag `c,fk` to generate per-file accesskeys; users which have full read-access will then see URLs with `?k=...` appended to the end, and `g` users must provide that URL including the correct key to avoid a 404
+  * combine this with volflag `c,fk` to generate per-file accesskeys; users which have full read-access will then see URLs with `?k=...` appended to the end, and `g` users must provide that URL including the correct key to avoid a 404
 
 
 ## gotchas
@@ -1228,11 +1274,15 @@ if you want thumbnails, `apt -y install ffmpeg`
 
 ideas for context to include in bug reports
 
+in general, commandline arguments (and config file if any)
+
 if something broke during an upload (replacing FILENAME with a part of the filename that broke):
 ```
 journalctl -aS '48 hour ago' -u copyparty | grep -C10 FILENAME | tee bug.log
 ```
 
+if there's a wall of base64 in the log (thread stacks) then please include that, especially if you run into something freezing up or getting stuck, for example `OperationalError('database is locked')` -- alternatively you can visit `/?stack` to see the stacks live, so http://127.0.0.1:3923/?stack for example
+
 
 # building
 

bin/mtag/README.md

@@ -42,7 +42,7 @@ run [`install-deps.sh`](install-deps.sh) to build/install most dependencies requ
 * `mtp` modules will not run if a file has existing tags in the db, so clear out the tags with `-e2tsr` the first time you launch with new `mtp` options
 
 
-## usage with volume-flags
+## usage with volflags
 
 instead of affecting all volumes, you can set the options for just one volume like so:
 

bin/mtag/image-noexif.py

@@ -89,4 +89,7 @@ def main():
 
 
 if __name__ == "__main__":
-    main()
+    try:
+        main()
+    except:
+        pass

bin/mtag/mousepad.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+import subprocess as sp
+
+
+"""
+mtp test -- opens a texteditor
+
+usage:
+  -vsrv/v1:v1:r:c,mte=+x1:c,mtp=x1=ad,p,bin/mtag/mousepad.py
+
+explained:
+  c,mte: list of tags to index in this volume
+  c,mtp: add new tag provider
+     x1: dummy tag to provide
+     ad: dontcare if audio or not
+      p: priority 1 (run after initial tag-scan with ffprobe or mutagen)
+"""
+
+
+def main():
+    env = os.environ.copy()
+    env["DISPLAY"] = ":0.0"
+
+    if False:
+        # open the uploaded file
+        fp = sys.argv[-1]
+    else:
+        # display stdin contents (`oth_tags`)
+        fp = "/dev/stdin"
+
+    p = sp.Popen(["/usr/bin/mousepad", fp])
+    p.communicate()
+
+
+main()

bin/mtag/rclone-upload.py

@@ -0,0 +1,76 @@
+#!/usr/bin/env python
+
+import json
+import os
+import subprocess as sp
+import sys
+import time
+
+try:
+    from copyparty.util import fsenc
+except:
+
+    def fsenc(p):
+        return p.encode("utf-8")
+
+
+_ = r"""
+first checks the tag "vidchk" which must be "ok" to continue,
+then uploads all files to some cloud storage (RCLONE_REMOTE)
+and DELETES THE ORIGINAL FILES if rclone returns 0 ("success")
+
+deps:
+  rclone
+
+usage:
+  -mtp x2=t43200,ay,p2,bin/mtag/rclone-upload.py
+
+explained:
+t43200: timeout 12h
+    ay: only process files which contain audio (including video with audio)
+    p2: set priority 2 (after vidchk's suggested priority of 1),
+          so the output of vidchk will be passed in here
+
+complete usage example as vflags along with vidchk:
+  -vsrv/vidchk:vidchk:r:rw,ed:c,e2dsa,e2ts,mtp=vidchk=t600,p,bin/mtag/vidchk.py:c,mtp=rupload=t43200,ay,p2,bin/mtag/rclone-upload.py:c,mte=+vidchk,rupload
+
+setup: see https://rclone.org/drive/
+
+if you wanna use this script standalone / separately from copyparty,
+either set CONDITIONAL_UPLOAD False or provide the following stdin:
+  {"vidchk":"ok"}
+"""
+
+
+RCLONE_REMOTE = "notmybox"
+CONDITIONAL_UPLOAD = True
+
+
+def main():
+    fp = sys.argv[1]
+    if CONDITIONAL_UPLOAD:
+        zb = sys.stdin.buffer.read()
+        zs = zb.decode("utf-8", "replace")
+        md = json.loads(zs)
+
+        chk = md.get("vidchk", None)
+        if chk != "ok":
+            print(f"vidchk={chk}", file=sys.stderr)
+            sys.exit(1)
+
+    dst = f"{RCLONE_REMOTE}:".encode("utf-8")
+    cmd = [b"rclone", b"copy", b"--", fsenc(fp), dst]
+
+    t0 = time.time()
+    try:
+        sp.check_call(cmd)
+    except:
+        print("rclone failed", file=sys.stderr)
+        sys.exit(1)
+
+    print(f"{time.time() - t0:.1f} sec")
+    os.unlink(fsenc(fp))
+
+
+if __name__ == "__main__":
+    main()

bin/mtag/vidchk.py

@@ -1,15 +1,33 @@
 #!/usr/bin/env python3
 
+import json
+import re
 import sys
 import subprocess as sp
 
-from copyparty.util import fsenc
-from copyparty.mtag import ffprobe
+try:
+    from copyparty.util import fsenc
+except:
+
+    def fsenc(p):
+        return p.encode("utf-8")
 
 
-"""
+_ = r"""
 inspects video files for errors and such
-usage: -mtp vidchk=t600,ay,bin/mtag/vidchk.py
+plus stores a bunch of metadata to filename.ff.json
+
+usage:
+  -mtp vidchk=t600,ay,p,bin/mtag/vidchk.py
+
+explained:
+t600: timeout 10min
+  ay: only process files which contain audio (including video with audio)
+   p: set priority 1 (lowest priority after initial ffprobe/mutagen for base tags),
+       makes copyparty feed base tags into this script as json
+
+if you wanna use this script standalone / separately from copyparty,
+provide the video resolution on stdin as json:  {"res":"1920x1080"}
 """
 
 
@@ -17,25 +35,69 @@ FAST = True  # parse entire file at container level
 # FAST = False  # fully decode audio and video streams
 
 
+# warnings to ignore
+harmless = re.compile("^Unsupported codec with id ")
+
+
+def wfilter(lines):
+    return [x for x in lines if not harmless.search(x)]
+
+
+def errchk(so, se, rc):
+    if rc:
+        err = (so + se).decode("utf-8", "replace").split("\n", 1)
+        err = wfilter(err) or err
+        return f"ERROR {rc}: {err[0]}"
+
+    if se:
+        err = se.decode("utf-8", "replace").split("\n", 1)
+        err = wfilter(err)
+        if err:
+            return f"Warning: {err[0]}"
+
+    return None
+
+
 def main():
     fp = sys.argv[1]
-    md, _ = ffprobe(fp)
+    zb = sys.stdin.buffer.read()
+    zs = zb.decode("utf-8", "replace")
+    md = json.loads(zs)
 
     try:
-        w = int(md[".resw"][1])
-        h = int(md[".resh"][1])
+        w, h = [int(x) for x in md["res"].split("x")]
         if not w + h:
             raise Exception()
     except:
         return "could not determine resolution"
 
-    if min(w, h) < 720:
+    # grab streams/format metadata + 2 seconds of frames at the start and end
+    zs = "ffprobe -hide_banner -v warning -of json -show_streams -show_format -show_packets -show_data_hash crc32 -read_intervals %+2,999999%+2"
+    cmd = zs.encode("ascii").split(b" ") + [fsenc(fp)]
+    p = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE)
+    so, se = p.communicate()
+
+    # spaces to tabs, drops filesize from 69k to 48k
+    so = b"\n".join(
+        [
+            b"\t" * int((len(x) - len(x.lstrip())) / 4) + x.lstrip()
+            for x in (so or b"").split(b"\n")
+        ]
+    )
+    with open(fsenc(f"{fp}.ff.json"), "wb") as f:
+        f.write(so)
+
+    err = errchk(so, se, p.returncode)
+    if err:
+        return err
+
+    if min(w, h) < 1080:
         return "resolution too small"
 
     zs = (
         "ffmpeg -y -hide_banner -nostdin -v warning"
         + " -err_detect +crccheck+bitstream+buffer+careful+compliant+aggressive+explode"
-        " -xerror -i"
+        + " -xerror -i"
     )
 
     cmd = zs.encode("ascii").split(b" ") + [fsenc(fp)]
@@ -49,16 +111,7 @@ def main():
 
     p = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE)
     so, se = p.communicate()
-    rc = p.returncode
-    if rc:
-        err = (so + se).decode("utf-8", "replace").split("\n", 1)[0]
-        return f"ERROR {rc}: {err}"
-
-    if se:
-        err = se.decode("utf-8", "replace").split("\n", 1)[0]
-        return f"Warning: {err}"
-
-    return None
+    return errchk(so, se, p.returncode)
 
 
 if __name__ == "__main__":

bin/prisonparty.sh

@@ -11,13 +11,13 @@ sysdirs=( /bin /lib /lib32 /lib64 /sbin /usr )
 help() { cat <<'EOF'
 
 usage:
-  ./prisonparty.sh <ROOTDIR> <UID> <GID> [VOLDIR [VOLDIR...]] -- python3 copyparty-sfx.py [...]"
+  ./prisonparty.sh <ROOTDIR> <UID> <GID> [VOLDIR [VOLDIR...]] -- python3 copyparty-sfx.py [...]
 
 example:
-  ./prisonparty.sh /var/lib/copyparty-jail 1000 1000 /mnt/nas/music -- python3 copyparty-sfx.py -v /mnt/nas/music::rwmd"
+  ./prisonparty.sh /var/lib/copyparty-jail 1000 1000 /mnt/nas/music -- python3 copyparty-sfx.py -v /mnt/nas/music::rwmd
 
 example for running straight from source (instead of using an sfx):
-  PYTHONPATH=$PWD ./prisonparty.sh /var/lib/copyparty-jail 1000 1000 /mnt/nas/music -- python3 -um copyparty -v /mnt/nas/music::rwmd"
+  PYTHONPATH=$PWD ./prisonparty.sh /var/lib/copyparty-jail 1000 1000 /mnt/nas/music -- python3 -um copyparty -v /mnt/nas/music::rwmd
 
 note that if you have python modules installed as --user (such as bpm/key detectors),
   you should add /home/foo/.local as a VOLDIR

bin/up2k.py

@@ -3,7 +3,7 @@ from __future__ import print_function, unicode_literals
 
 """
 up2k.py: upload to copyparty
-2022-06-16, v0.15, ed <irc.rizon.net>, MIT-Licensed
+2022-08-13, v0.18, ed <irc.rizon.net>, MIT-Licensed
 https://github.com/9001/copyparty/blob/hovudstraum/bin/up2k.py
 
 - dependencies: requests
@@ -22,12 +22,29 @@ import atexit
 import signal
 import base64
 import hashlib
-import argparse
 import platform
 import threading
 import datetime
 
-import requests
+try:
+    import argparse
+except:
+    m = "\n  ERROR: need 'argparse'; download it here:\n   https://github.com/ThomasWaldmann/argparse/raw/master/argparse.py\n"
+    print(m)
+    raise
+
+try:
+    import requests
+except:
+    if sys.version_info > (2, 7):
+        m = "\n  ERROR: need 'requests'; run this:\n   python -m pip install --user requests\n"
+    else:
+        m = "requests/2.18.4 urllib3/1.23 chardet/3.0.4 certifi/2020.4.5.1 idna/2.7"
+        m = ["   https://pypi.org/project/" + x + "/#files" for x in m.split()]
+        m = "\n  ERROR: need these:\n" + "\n".join(m) + "\n"
+
+    print(m)
+    raise
 
 
 # from copyparty/__init__.py
@@ -126,6 +143,89 @@ class FileSlice(object):
         return ret
 
 
+class MTHash(object):
+    def __init__(self, cores):
+        self.f = None
+        self.sz = 0
+        self.csz = 0
+        self.omutex = threading.Lock()
+        self.imutex = threading.Lock()
+        self.work_q = Queue()
+        self.done_q = Queue()
+        self.thrs = []
+        for _ in range(cores):
+            t = threading.Thread(target=self.worker)
+            t.daemon = True
+            t.start()
+            self.thrs.append(t)
+
+    def hash(self, f, fsz, chunksz, pcb=None, pcb_opaque=None):
+        with self.omutex:
+            self.f = f
+            self.sz = fsz
+            self.csz = chunksz
+
+            chunks = {}
+            nchunks = int(math.ceil(fsz / chunksz))
+            for nch in range(nchunks):
+                self.work_q.put(nch)
+
+            ex = ""
+            for nch in range(nchunks):
+                qe = self.done_q.get()
+                try:
+                    nch, dig, ofs, csz = qe
+                    chunks[nch] = [dig, ofs, csz]
+                except:
+                    ex = ex or qe
+
+                if pcb:
+                    pcb(pcb_opaque, chunksz * nch)
+
+            if ex:
+                raise Exception(ex)
+
+            ret = []
+            for n in range(nchunks):
+                ret.append(chunks[n])
+
+            self.f = None
+            self.csz = 0
+            self.sz = 0
+            return ret
+
+    def worker(self):
+        while True:
+            ofs = self.work_q.get()
+            try:
+                v = self.hash_at(ofs)
+            except Exception as ex:
+                v = str(ex)
+
+            self.done_q.put(v)
+
+    def hash_at(self, nch):
+        f = self.f
+        ofs = ofs0 = nch * self.csz
+        hashobj = hashlib.sha512()
+        chunk_sz = chunk_rem = min(self.csz, self.sz - ofs)
+        while chunk_rem > 0:
+            with self.imutex:
+                f.seek(ofs)
+                buf = f.read(min(chunk_rem, 1024 * 1024 * 12))
+
+            if not buf:
+                raise Exception("EOF at " + str(ofs))
+
+            hashobj.update(buf)
+            chunk_rem -= len(buf)
+            ofs += len(buf)
+
+        digest = hashobj.digest()[:33]
+        digest = base64.urlsafe_b64encode(digest).decode("utf-8")
+        return nch, digest, ofs0, chunk_sz
+
+
 _print = print
 
 
@@ -230,8 +330,8 @@ def _scd(err, top):
             abspath = os.path.join(top, fh.name)
             try:
                 yield [abspath, fh.stat()]
-            except:
-                err.append(abspath)
+            except Exception as ex:
+                err.append((abspath, str(ex)))
 
 
 def _lsd(err, top):
@@ -240,8 +340,8 @@ def _lsd(err, top):
         abspath = os.path.join(top, name)
         try:
             yield [abspath, os.stat(abspath)]
-        except:
-            err.append(abspath)
+        except Exception as ex:
+            err.append((abspath, str(ex)))
 
 
 if hasattr(os, "scandir"):
@@ -250,15 +350,21 @@ else:
     statdir = _lsd
 
 
-def walkdir(err, top):
+def walkdir(err, top, seen):
     """recursive statdir"""
+    atop = os.path.abspath(os.path.realpath(top))
+    if atop in seen:
+        err.append((top, "recursive-symlink"))
+        return
+
+    seen = seen[:] + [atop]
     for ap, inf in sorted(statdir(err, top)):
         if stat.S_ISDIR(inf.st_mode):
             try:
-                for x in walkdir(err, ap):
+                for x in walkdir(err, ap, seen):
                     yield x
-            except:
-                err.append(ap)
+            except Exception as ex:
+                err.append((ap, str(ex)))
         else:
             yield ap, inf
 
@@ -273,7 +379,7 @@ def walkdirs(err, tops):
             stop = os.path.dirname(top)
 
         if os.path.isdir(top):
-            for ap, inf in walkdir(err, top):
+            for ap, inf in walkdir(err, top, []):
                 yield stop, ap[len(stop) :].lstrip(sep), inf
         else:
             d, n = top.rsplit(sep, 1)
@@ -322,8 +428,8 @@ def up2k_chunksize(filesize):
 
 
 # mostly from copyparty/up2k.py
-def get_hashlist(file, pcb):
-    # type: (File, any) -> None
+def get_hashlist(file, pcb, mth):
+    # type: (File, any, any) -> None
     """generates the up2k hashlist from file contents, inserts it into `file`"""
 
     chunk_sz = up2k_chunksize(file.size)
@@ -331,7 +437,12 @@ def get_hashlist(file, pcb):
     file_ofs = 0
     ret = []
     with open(file.abs, "rb", 512 * 1024) as f:
+        if mth and file.size >= 1024 * 512:
+            ret = mth.hash(f, file.size, chunk_sz, pcb, file)
+            file_rem = 0
+
         while file_rem > 0:
+            # same as `hash_at` except for `imutex` / bufsz
             hashobj = hashlib.sha512()
             chunk_sz = chunk_rem = min(chunk_sz, file_rem)
             while chunk_rem > 0:
@@ -388,8 +499,9 @@ def handshake(req_ses, url, file, pw, search):
         try:
             r = req_ses.post(url, headers=headers, json=req)
             break
-        except:
-            eprint("handshake failed, retrying: {0}\n".format(file.name))
+        except Exception as ex:
+            em = str(ex).split("SSLError(")[-1]
+            eprint("handshake failed, retrying: {0}\n  {1}\n\n".format(file.name, em))
             time.sleep(1)
 
     try:
@@ -398,7 +510,7 @@ def handshake(req_ses, url, file, pw, search):
         raise Exception(r.text)
 
     if search:
-        return r["hits"]
+        return r["hits"], False
 
     try:
         pre, url = url.split("://")
@@ -470,12 +582,19 @@ class Ctl(object):
 
         if err:
             eprint("\n# failed to access {0} paths:\n".format(len(err)))
-            for x in err:
-                eprint(x.decode("utf-8", "replace") + "\n")
+            for ap, msg in err:
+                if ar.v:
+                    eprint("{0}\n `-{1}\n\n".format(ap.decode("utf-8", "replace"), msg))
+                else:
+                    eprint(ap.decode("utf-8", "replace") + "\n")
 
             eprint("^ failed to access those {0} paths ^\n\n".format(len(err)))
+
+            if not ar.v:
+                eprint("hint: set -v for detailed error messages\n")
+
             if not ar.ok:
-                eprint("aborting because --ok is not set\n")
+                eprint("hint: aborting because --ok is not set\n")
                 return
 
         eprint("found {0} files, {1}\n\n".format(nfiles, humansize(nbytes)))
@@ -516,6 +635,8 @@ class Ctl(object):
             self.st_hash = [None, "(idle, starting...)"]  # type: tuple[File, int]
             self.st_up = [None, "(idle, starting...)"]  # type: tuple[File, int]
 
+            self.mth = MTHash(ar.J) if ar.J > 1 else None
+
             self._fancy()
 
     def _safe(self):
@@ -526,7 +647,7 @@ class Ctl(object):
             upath = file.abs.decode("utf-8", "replace")
 
             print("{0} {1}\n  hash...".format(self.nfiles - nf, upath))
-            get_hashlist(file, None)
+            get_hashlist(file, None, None)
 
             burl = self.ar.url[:12] + self.ar.url[8:].split("/")[0] + "/"
             while True:
@@ -679,7 +800,7 @@ class Ctl(object):
 
                 time.sleep(0.05)
 
-            get_hashlist(file, self.cb_hasher)
+            get_hashlist(file, self.cb_hasher, self.mth)
             with self.mutex:
                 self.hash_f += 1
                 self.hash_c += len(file.cids)
@@ -808,6 +929,9 @@ def main():
     if not VT100:
         os.system("rem")  # enables colors
 
+    cores = os.cpu_count() if hasattr(os, "cpu_count") else 4
+    hcores = min(cores, 3)  # 4% faster than 4+ on py3.9 @ r5-4500U
+
     # fmt: off
     ap = app = argparse.ArgumentParser(formatter_class=APF, epilog="""
 NOTE:
@@ -818,11 +942,13 @@ source file/folder selection uses rsync syntax, meaning that:
 
     ap.add_argument("url", type=unicode, help="server url, including destination folder")
     ap.add_argument("files", type=unicode, nargs="+", help="files and/or folders to process")
+    ap.add_argument("-v", action="store_true", help="verbose")
     ap.add_argument("-a", metavar="PASSWORD", help="password")
     ap.add_argument("-s", action="store_true", help="file-search (disables upload)")
     ap.add_argument("--ok", action="store_true", help="continue even if some local files are inaccessible")
     ap = app.add_argument_group("performance tweaks")
     ap.add_argument("-j", type=int, metavar="THREADS", default=4, help="parallel connections")
+    ap.add_argument("-J", type=int, metavar="THREADS", default=hcores, help="num cpu-cores to use for hashing; set 0 or 1 for single-core hashing")
     ap.add_argument("-nh", action="store_true", help="disable hashing while uploading")
     ap.add_argument("--safe", action="store_true", help="use simple fallback approach")
     ap.add_argument("-z", action="store_true", help="ZOOMIN' (skip uploading files if they exist at the destination with the ~same last-modified timestamp, so same as yolo / turbo with date-chk but even faster)")

contrib/plugins/README.md

@@ -15,6 +15,8 @@ save one of these as `.epilogue.html` inside a folder to customize it:
 point `--js-browser` to one of these by URL:
 
 * [`minimal-up2k.js`](minimal-up2k.js) is similar to the above `minimal-up2k.html` except it applies globally to all write-only folders
+* [`up2k-hooks.js`](up2k-hooks.js) lets you specify a ruleset for files to skip uploading
+  * [`up2k-hook-ytid.js`](up2k-hook-ytid.js) is a more specific example checking youtube-IDs against some API
 
 
 

contrib/plugins/minimal-up2k.js

@@ -18,7 +18,7 @@ var u2min = `
 <style>
 
 #ops, #path, #tree, #files, #epi+div+h2,
-#u2conf td.c+.c, #u2cards, #u2foot, #srch_dz, #srch_zd {
+#u2conf td.c+.c, #u2cards, #srch_dz, #srch_zd {
   display: none !important;
 }
 #u2conf {margin:5em auto 0 auto !important}

contrib/plugins/up2k-hook-ytid.js

@@ -0,0 +1,220 @@
+// way more specific example --
+// assumes all files dropped into the uploader have a youtube-id somewhere in the filename,
+// locates the youtube-ids and passes them to an API which returns a list of IDs which should be uploaded
+//
+// also tries to find the youtube-id in the embedded metadata
+//
+// assumes copyparty is behind nginx as /ytq is a standalone service which must be rproxied in place
+
+function up2k_namefilter(good_files, nil_files, bad_files, hooks) {
+    var passthru = up2k.uc.fsearch;
+    if (passthru)
+        return hooks[0](good_files, nil_files, bad_files, hooks.slice(1));
+
+    a_up2k_namefilter(good_files, nil_files, bad_files, hooks).then(() => { });
+}
+
+function bstrpos(buf, ptn) {
+    var ofs = 0,
+        ch0 = ptn[0],
+        sz = buf.byteLength;
+
+    while (true) {
+        ofs = buf.indexOf(ch0, ofs);
+        if (ofs < 0 || ofs >= sz)
+            return -1;
+
+        for (var a = 1; a < ptn.length; a++)
+            if (buf[ofs + a] !== ptn[a])
+                break;
+
+        if (a === ptn.length)
+            return ofs;
+
+        ++ofs;
+    }
+}
+
+async function a_up2k_namefilter(good_files, nil_files, bad_files, hooks) {
+    var t0 = Date.now(),
+        yt_ids = new Set(),
+        textdec = new TextDecoder('latin1'),
+        md_ptn = new TextEncoder().encode('youtube.com/watch?v='),
+        file_ids = [],  // all IDs found for each good_files
+        md_only = [],  // `${id} ${fn}` where ID was only found in metadata
+        mofs = 0,
+        mnchk = 0,
+        mfile = '';
+
+    for (var a = 0; a < good_files.length; a++) {
+        var [fobj, name] = good_files[a],
+            cname = name,  // will clobber
+            sz = fobj.size,
+            ids = [],
+            id_ok = false,
+            m;
+
+        // all IDs found in this file
+        file_ids.push(ids);
+
+        // look for ID in filename; reduce the
+        // metadata-scan intensity if the id looks safe
+        m = /[\[(-]([\w-]{11})[\])]?\.(?:mp4|webm|mkv|flv|opus|ogg|mp3|m4a|aac)$/i.exec(name);
+        id_ok = !!m;
+
+        while (true) {
+            // fuzzy catch-all;
+            // some ytdl fork did %(title)-%(id).%(ext) ...
+            m = /(?:^|[^\w])([\w-]{11})(?:$|[^\w-])/.exec(cname);
+            if (!m)
+                break;
+
+            cname = cname.replace(m[1], '');
+            yt_ids.add(m[1]);
+            ids.push(m[1]);
+        }
+
+        // look for IDs in video metadata,
+        if (/\.(mp4|webm|mkv|flv|opus|ogg|mp3|m4a|aac)$/i.exec(name)) {
+            toast.show('inf r', 0, `analyzing file ${a + 1} / ${good_files.length} :\n${name}\n\nhave analysed ${++mnchk} files in ${(Date.now() - t0) / 1000} seconds, ${humantime((good_files.length - (a + 1)) * (((Date.now() - t0) / 1000) / mnchk))} remaining,\n\nbiggest offset so far is ${mofs}, in this file:\n\n${mfile}`);
+
+            // check first and last 128 MiB;
+            // pWxOroN5WCo.mkv @  6edb98 (6.92M)
+            // Nf-nN1wF5Xo.mp4 @ 4a98034 (74.6M)
+            var chunksz = 1024 * 1024 * 2,  // byte
+                aspan = id_ok ? 128 : 512;  // MiB
+
+            aspan = parseInt(Math.min(sz / 2, aspan * 1024 * 1024) / chunksz) * chunksz;
+
+            for (var side = 0; side < 2; side++) {
+                var ofs = side ? Math.max(0, sz - aspan) : 0,
+                    nchunks = aspan / chunksz;
+
+                for (var chunk = 0; chunk < nchunks; chunk++) {
+                    var bchunk = await fobj.slice(ofs, ofs + chunksz + 16).arrayBuffer(),
+                        uchunk = new Uint8Array(bchunk, 0, bchunk.byteLength),
+                        bofs = bstrpos(uchunk, md_ptn),
+                        absofs = Math.min(ofs + bofs, (sz - ofs) + bofs),
+                        txt = bofs < 0 ? '' : textdec.decode(uchunk.subarray(bofs)),
+                        m;
+
+                    //console.log(`side ${ side }, chunk ${ chunk }, ofs ${ ofs }, bchunk ${ bchunk.byteLength }, txt ${ txt.length }`);
+                    while (true) {
+                        // mkv/webm have [a-z] immediately after url
+                        m = /(youtube\.com\/watch\?v=[\w-]{11})/.exec(txt);
+                        if (!m)
+                            break;
+
+                        txt = txt.replace(m[1], '');
+                        m = m[1].slice(-11);
+
+                        console.log(`found ${m} @${bofs}, ${name} `);
+                        yt_ids.add(m);
+                        if (!has(ids, m)) {
+                            ids.push(m);
+                            md_only.push(`${m} ${name}`);
+                        }
+
+                        // bail after next iteration
+                        chunk = nchunks - 1;
+                        side = 9;
+
+                        if (mofs < absofs) {
+                            mofs = absofs;
+                            mfile = name;
+                        }
+                    }
+                    ofs += chunksz;
+                    if (ofs >= sz)
+                        break;
+                }
+            }
+        }
+    }
+
+    if (md_only.length)
+        console.log('recovered the following youtube-IDs by inspecting metadata:\n\n' + md_only.join('\n'));
+    else if (yt_ids.size)
+        console.log('did not discover any additional youtube-IDs by inspecting metadata; all the IDs also existed in the filenames');
+    else
+        console.log('failed to find any youtube-IDs at all, sorry');
+
+    if (false) {
+        var msg = `finished analysing ${mnchk} files in ${(Date.now() - t0) / 1000} seconds,\n\nbiggest offset was ${mofs} in this file:\n\n${mfile}`,
+            mfun = function () { toast.ok(0, msg); };
+
+        mfun();
+        setTimeout(mfun, 200);
+
+        return hooks[0]([], [], [], hooks.slice(1));
+    }
+
+    toast.inf(5, `running query for ${yt_ids.size} youtube-IDs...`);
+
+    var xhr = new XHR();
+    xhr.open('POST', '/ytq', true);
+    xhr.setRequestHeader('Content-Type', 'text/plain');
+    xhr.onload = xhr.onerror = function () {
+        if (this.status != 200)
+            return toast.err(0, `sorry, database query failed ;_;\n\nplease let us know so we can look at it, thx!!\n\nerror ${this.status}: ${(this.response && this.response.err) || this.responseText}`);
+
+        process_id_list(this.responseText);
+    };
+    xhr.send(Array.from(yt_ids).join('\n'));
+
+    function process_id_list(txt) {
+        var wanted_ids = new Set(txt.trim().split('\n')),
+            wanted_names = new Set(),  // basenames with a wanted ID
+            wanted_files = new Set();  // filedrops
+
+        for (var a = 0; a < good_files.length; a++) {
+            var name = good_files[a][1];
+            for (var b = 0; b < file_ids[a].length; b++)
+                if (wanted_ids.has(file_ids[a][b])) {
+                    wanted_files.add(good_files[a]);
+
+                    var m = /(.*)\.(mp4|webm|mkv|flv|opus|ogg|mp3|m4a|aac)$/i.exec(name);
+                    if (m)
+                        wanted_names.add(m[1]);
+
+                    break;
+                }
+        }
+
+        // add all files with the same basename as each explicitly wanted file
+        // (infojson/chatlog/etc when ID was discovered from metadata)
+        for (var a = 0; a < good_files.length; a++) {
+            var name = good_files[a][1];
+            for (var b = 0; b < 3; b++) {
+                name = name.replace(/\.[^\.]+$/, '');
+                if (wanted_names.has(name)) {
+                    wanted_files.add(good_files[a]);
+                    break;
+                }
+            }
+        }
+
+        function upload_filtered() {
+            if (!wanted_files.size)
+                return modal.alert('Good news -- turns out we already have all those.\n\nBut thank you for checking in!');
+
+            hooks[0](Array.from(wanted_files), nil_files, bad_files, hooks.slice(1));
+        }
+
+        function upload_all() {
+            hooks[0](good_files, nil_files, bad_files, hooks.slice(1));
+        }
+
+        var n_skip = good_files.length - wanted_files.size,
+            msg = `you added ${good_files.length} files; ${good_files.length == n_skip ? 'all' : n_skip} of them were skipped --\neither because we already have them,\nor because there is no youtube-ID in your filenames.\n\n<code>OK</code> / <code>Enter</code> = continue uploading just the ${wanted_files.size} files we definitely need\n\n<code>Cancel</code> / <code>ESC</code> = override the filter; upload ALL the files you added`;
+
+        if (!n_skip)
+            upload_filtered();
+        else
+            modal.confirm(msg, upload_filtered, upload_all);
+    };
+}
+
+up2k_hooks.push(function () {
+    up2k.gotallfiles.unshift(up2k_namefilter);
+});

contrib/plugins/up2k-hooks.js

@@ -0,0 +1,45 @@
+// hooks into up2k
+
+function up2k_namefilter(good_files, nil_files, bad_files, hooks) {
+    // is called when stuff is dropped into the browser,
+    // after iterating through the directory tree and discovering all files,
+    // before the upload confirmation dialogue is shown
+
+    // good_files will successfully upload
+    // nil_files are empty files and will show an alert in the final hook
+    // bad_files are unreadable and cannot be uploaded
+    var file_lists = [good_files, nil_files, bad_files];
+
+    // build a list of filenames
+    var filenames = [];
+    for (var lst of file_lists)
+        for (var ent of lst)
+            filenames.push(ent[1]);
+
+    toast.inf(5, "running database query...");
+
+    // simulate delay while passing the list to some api for checking
+    setTimeout(function () {
+
+        // only keep webm files as an example
+        var new_lists = [];
+        for (var lst of file_lists) {
+            var keep = [];
+            new_lists.push(keep);
+
+            for (var ent of lst)
+                if (/\.webm$/.test(ent[1]))
+                    keep.push(ent);
+        }
+
+        // finally, call the next hook in the chain
+        [good_files, nil_files, bad_files] = new_lists;
+        hooks[0](good_files, nil_files, bad_files, hooks.slice(1));
+
+    }, 1000);
+}
+
+// register
+up2k_hooks.push(function () {
+    up2k.gotallfiles.unshift(up2k_namefilter);
+});

copyparty/__main__.py

@@ -24,7 +24,18 @@ from .__init__ import ANYWIN, PY2, VT100, WINDOWS, E, unicode
 from .__version__ import CODENAME, S_BUILD_DT, S_VERSION
 from .authsrv import re_vol
 from .svchub import SvcHub
-from .util import IMPLICATIONS, align_tab, ansi_re, min_ex, py_desc, termsize, wrap
+from .util import (
+    IMPLICATIONS,
+    JINJA_VER,
+    PYFTPD_VER,
+    SQLITE_VER,
+    align_tab,
+    ansi_re,
+    min_ex,
+    py_desc,
+    termsize,
+    wrap,
+)
 
 try:
     from types import FrameType
@@ -107,12 +118,13 @@ class BasicDodge11874(
 
 
 def lprint(*a: Any, **ka: Any) -> None:
-    txt: str = " ".join(unicode(x) for x in a) + ka.get("end", "\n")
+    eol = ka.pop("end", "\n")
+    txt: str = " ".join(unicode(x) for x in a) + eol
     printed.append(txt)
     if not VT100:
         txt = ansi_re.sub("", txt)
 
-    print(txt, **ka)
+    print(txt, end="", **ka)
 
 
 def warn(msg: str) -> None:
@@ -127,7 +139,7 @@ def ensure_locale() -> None:
     ]:
         try:
             locale.setlocale(locale.LC_ALL, x)
-            lprint("Locale:", x)
+            lprint("Locale: {}\n".format(x))
             break
         except:
             continue
@@ -275,7 +287,7 @@ def disable_quickedit() -> None:
                 raise ctypes.WinError(err)  # type: ignore
         return args
 
-    k32 = ctypes.WinDLL("kernel32", use_last_error=True)  # type: ignore
+    k32 = ctypes.WinDLL(str("kernel32"), use_last_error=True)  # type: ignore
     if PY2:
         wintypes.LPDWORD = ctypes.POINTER(wintypes.DWORD)
 
@@ -323,7 +335,8 @@ def run_argparse(argv: list[str], formatter: Any, retry: bool) -> argparse.Names
     except:
         fk_salt = "hunter2"
 
-    cores = os.cpu_count() if hasattr(os, "cpu_count") else 4
+    cores = (os.cpu_count() if hasattr(os, "cpu_count") else 0) or 4
+    hcores = min(cores, 3)  # 4% faster than 4+ on py3.9 @ r5-4500U
 
     sects = [
         [
@@ -382,6 +395,7 @@ def run_argparse(argv: list[str], formatter: Any, retry: bool) -> argparse.Names
               \033[36mmaxn=250,600\033[35m max 250 uploads over 15min
               \033[36mmaxb=1g,300\033[35m max 1 GiB over 5min (suffixes: b, k, m, g)
               \033[36msz=1k-3m\033[35m allow filesizes between 1 KiB and 3MiB
+              \033[36mdf=1g\033[35m ensure 1 GiB free disk space
 
             \033[0mupload rotation:
             (moves all uploads into the specified folder structure)
@@ -394,11 +408,14 @@ def run_argparse(argv: list[str], formatter: Any, retry: bool) -> argparse.Names
               \033[36md2ts\033[35m disables metadata collection for existing files
               \033[36md2ds\033[35m disables onboot indexing, overrides -e2ds*
               \033[36md2t\033[35m disables metadata collection, overrides -e2t*
+              \033[36md2v\033[35m disables file verification, overrides -e2v*
               \033[36md2d\033[35m disables all database stuff, overrides -e2*
-              \033[36mnohash=\\.iso$\033[35m skips hashing file contents if path matches *.iso
-              \033[36mnoidx=\\.iso$\033[35m fully ignores the contents at paths matching *.iso
               \033[36mhist=/tmp/cdb\033[35m puts thumbnails and indexes at that location
               \033[36mscan=60\033[35m scan for new files every 60sec, same as --re-maxage
+              \033[36mnohash=\\.iso$\033[35m skips hashing file contents if path matches *.iso
+              \033[36mnoidx=\\.iso$\033[35m fully ignores the contents at paths matching *.iso
+              \033[36mxdev\033[35m do not descend into other filesystems
+              \033[36mxvol\033[35m skip symlinks leaving the volume root
 
             \033[0mdatabase, audio tags:
             "mte", "mth", "mtp", "mtm" all work the same as -mte, -mth, ...
@@ -481,8 +498,10 @@ def run_argparse(argv: list[str], formatter: Any, retry: bool) -> argparse.Names
     ap2.add_argument("--hardlink", action="store_true", help="prefer hardlinks instead of symlinks when possible (within same filesystem)")
     ap2.add_argument("--never-symlink", action="store_true", help="do not fallback to symlinks when a hardlink cannot be made")
     ap2.add_argument("--no-dedup", action="store_true", help="disable symlink/hardlink creation; copy file contents instead")
+    ap2.add_argument("--df", metavar="GiB", type=float, default=0, help="ensure GiB free disk space by rejecting upload requests")
     ap2.add_argument("--sparse", metavar="MiB", type=int, default=4, help="windows-only: minimum size of incoming uploads through up2k before they are made into sparse files")
     ap2.add_argument("--turbo", metavar="LVL", type=int, default=0, help="configure turbo-mode in up2k client; 0 = off and warn if enabled, 1 = off, 2 = on, 3 = on and disable datecheck")
+    ap2.add_argument("--u2sort", metavar="TXT", type=u, default="s", help="upload order; s=smallest-first, n=alphabetical, fs=force-s, fn=force-n -- alphabetical is a bit slower on fiber/LAN but makes it easier to eyeball if everything went fine")
 
     ap2 = ap.add_argument_group('network options')
     ap2.add_argument("-i", metavar="IP", type=u, default="0.0.0.0", help="ip to bind (comma-sep.)")
@@ -533,9 +552,10 @@ def run_argparse(argv: list[str], formatter: Any, retry: bool) -> argparse.Names
     ap2.add_argument("--no-robots", action="store_true", help="adds http and html headers asking search engines to not index anything")
     ap2.add_argument("--logout", metavar="H", type=float, default="8086", help="logout clients after H hours of inactivity (0.0028=10sec, 0.1=6min, 24=day, 168=week, 720=month, 8760=year)")
 
-    ap2 = ap.add_argument_group('yolo options')
+    ap2 = ap.add_argument_group('shutdown options')
     ap2.add_argument("--ign-ebind", action="store_true", help="continue running even if it's impossible to listen on some of the requested endpoints")
     ap2.add_argument("--ign-ebind-all", action="store_true", help="continue running even if it's impossible to receive connections at all")
+    ap2.add_argument("--exit", metavar="WHEN", type=u, default="", help="shutdown after WHEN has finished; for example 'idx' will do volume indexing + metadata analysis")
 
     ap2 = ap.add_argument_group('logging options')
     ap2.add_argument("-q", action="store_true", help="quiet")
@@ -585,10 +605,18 @@ def run_argparse(argv: list[str], formatter: Any, retry: bool) -> argparse.Names
     ap2.add_argument("-e2d", action="store_true", help="enable up2k database, making files searchable + enables upload deduplocation")
     ap2.add_argument("-e2ds", action="store_true", help="scan writable folders for new files on startup; sets -e2d")
     ap2.add_argument("-e2dsa", action="store_true", help="scans all folders on startup; sets -e2ds")
+    ap2.add_argument("-e2v", action="store_true", help="verify file integrity; rehash all files and compare with db")
+    ap2.add_argument("-e2vu", action="store_true", help="on hash mismatch: update the database with the new hash")
+    ap2.add_argument("-e2vp", action="store_true", help="on hash mismatch: panic and quit copyparty")
     ap2.add_argument("--hist", metavar="PATH", type=u, help="where to store volume data (db, thumbs)")
     ap2.add_argument("--no-hash", metavar="PTN", type=u, help="regex: disable hashing of matching paths during e2ds folder scans")
     ap2.add_argument("--no-idx", metavar="PTN", type=u, help="regex: disable indexing of matching paths during e2ds folder scans")
+    ap2.add_argument("--no-dhash", action="store_true", help="disable rescan acceleration; do full database integrity check -- makes the db ~5%% smaller and bootup/rescans 3~10x slower")
+    ap2.add_argument("--xdev", action="store_true", help="do not descend into other filesystems (symlink or bind-mount to another HDD, ...)")
+    ap2.add_argument("--xvol", action="store_true", help="skip symlinks leaving the volume root")
+    ap2.add_argument("--hash-mt", metavar="CORES", type=int, default=hcores, help="num cpu cores to use for file hashing; set 0 or 1 for single-core hashing")
     ap2.add_argument("--re-maxage", metavar="SEC", type=int, default=0, help="disk rescan volume interval, 0=off, can be set per-volume with the 'scan' volflag")
+    ap2.add_argument("--db-act", metavar="SEC", type=float, default=10, help="defer any scheduled volume reindexing until SEC seconds after last db write (uploads, renames, ...)")
     ap2.add_argument("--srch-time", metavar="SEC", type=int, default=30, help="search deadline -- terminate searches running for more than SEC seconds")
     ap2.add_argument("--srch-hits", metavar="N", type=int, default=7999, help="max search results to allow clients to fetch; 125 results will be shown initially")
 
@@ -602,9 +630,9 @@ def run_argparse(argv: list[str], formatter: Any, retry: bool) -> argparse.Names
     ap2.add_argument("--mtag-v", action="store_true", help="verbose tag scanning; print errors from mtp subprocesses and such")
     ap2.add_argument("-mtm", metavar="M=t,t,t", type=u, action="append", help="add/replace metadata mapping")
     ap2.add_argument("-mte", metavar="M,M,M", type=u, help="tags to index/display (comma-sep.)",
-        default="circle,album,.tn,artist,title,.bpm,key,.dur,.q,.vq,.aq,vc,ac,res,.fps,ahash,vhash")
+        default="circle,album,.tn,artist,title,.bpm,key,.dur,.q,.vq,.aq,vc,ac,fmt,res,.fps,ahash,vhash")
     ap2.add_argument("-mth", metavar="M,M,M", type=u, help="tags to hide by default (comma-sep.)",
-        default=".vq,.aq,vc,ac,res,.fps")
+        default=".vq,.aq,vc,ac,fmt,res,.fps")
     ap2.add_argument("-mtp", metavar="M=[f,]BIN", type=u, action="append", help="read tag M using program BIN to parse the file")
 
     ap2 = ap.add_argument_group('ui options')
@@ -626,6 +654,7 @@ def run_argparse(argv: list[str], formatter: Any, retry: bool) -> argparse.Names
     ap2.add_argument("--no-htp", action="store_true", help="disable httpserver threadpool, create threads as-needed instead")
     ap2.add_argument("--stackmon", metavar="P,S", type=u, help="write stacktrace to Path every S second")
     ap2.add_argument("--log-thrs", metavar="SEC", type=float, help="list active threads every SEC")
+    ap2.add_argument("--log-fk", metavar="REGEX", type=u, default="", help="log filekey params for files where path matches REGEX; '.' (a single dot) = all files")
     # fmt: on
 
     ap2 = ap.add_argument_group("help sections")
@@ -651,10 +680,17 @@ def main(argv: Optional[list[str]] = None) -> None:
     if argv is None:
         argv = sys.argv
 
-    desc = py_desc().replace("[", "\033[1;30m[")
-
-    f = '\033[36mcopyparty v{} "\033[35m{}\033[36m" ({})\n{}\033[0m\n'
-    lprint(f.format(S_VERSION, CODENAME, S_BUILD_DT, desc))
+    f = '\033[36mcopyparty v{} "\033[35m{}\033[36m" ({})\n{}\033[0;36m\n   sqlite v{} | jinja2 v{} | pyftpd v{}\n\033[0m'
+    f = f.format(
+        S_VERSION,
+        CODENAME,
+        S_BUILD_DT,
+        py_desc().replace("[", "\033[1;30m["),
+        SQLITE_VER,
+        JINJA_VER,
+        PYFTPD_VER,
+    )
+    lprint(f)
 
     ensure_locale()
     if HAVE_SSL:
@@ -755,6 +791,12 @@ def main(argv: Optional[list[str]] = None) -> None:
     except:
         raise Exception("invalid value for -p")
 
+    for arg, kname, okays in [["--u2sort", "u2sort", "s n fs fn"]]:
+        val = unicode(getattr(al, kname))
+        if val not in okays.split():
+            zs = "argument {} cannot be '{}'; try one of these: {}"
+            raise Exception(zs.format(arg, val, okays))
+
     if HAVE_SSL:
         if al.ssl_ver:
             configure_ssl_ver(al)

copyparty/__version__.py

@@ -1,8 +1,8 @@
 # coding: utf-8
 
-VERSION = (1, 3, 4)
+VERSION = (1, 3, 14)
 CODENAME = "god dag"
-BUILD_DT = (2022, 7, 6)
+BUILD_DT = (2022, 8, 15)
 
 S_VERSION = ".".join(map(str, VERSION))
 S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT)

copyparty/authsrv.py

@@ -17,9 +17,12 @@ from .bos import bos
 from .util import (
     IMPLICATIONS,
     META_NOBOTS,
+    SQLITE_VER,
     Pebkac,
     absreal,
     fsenc,
+    get_df,
+    humansize,
     relchk,
     statdir,
     uncyg,
@@ -72,15 +75,23 @@ class AXS(object):
 
 
 class Lim(object):
-    def __init__(self) -> None:
+    def __init__(self, log_func: Optional["RootLogger"]) -> None:
+        self.log_func = log_func
+
+        self.reg: Optional[dict[str, dict[str, Any]]] = None  # up2k registry
+
         self.nups: dict[str, list[float]] = {}  # num tracker
         self.bups: dict[str, list[tuple[float, int]]] = {}  # byte tracker list
         self.bupc: dict[str, int] = {}  # byte tracker cache
 
         self.nosub = False  # disallow subdirectories
 
-        self.smin = -1  # filesize min
-        self.smax = -1  # filesize max
+        self.dfl = 0  # free disk space limit
+        self.dft = 0  # last-measured time
+        self.dfv = 0  # currently free
+
+        self.smin = 0  # filesize min
+        self.smax = 0  # filesize max
 
         self.bwin = 0  # bytes window
         self.bmax = 0  # bytes max
@@ -92,18 +103,34 @@ class Lim(object):
         self.rotf = ""  # rot datefmt
         self.rot_re = re.compile("")  # rotf check
 
+    def log(self, msg: str, c: Union[int, str] = 0) -> None:
+        if self.log_func:
+            self.log_func("up-lim", msg, c)
+
     def set_rotf(self, fmt: str) -> None:
         self.rotf = fmt
         r = re.escape(fmt).replace("%Y", "[0-9]{4}").replace("%j", "[0-9]{3}")
         r = re.sub("%[mdHMSWU]", "[0-9]{2}", r)
         self.rot_re = re.compile("(^|/)" + r + "$")
 
-    def all(self, ip: str, rem: str, sz: float, abspath: str) -> tuple[str, str]:
+    def all(
+        self,
+        ip: str,
+        rem: str,
+        sz: int,
+        abspath: str,
+        reg: Optional[dict[str, dict[str, Any]]] = None,
+    ) -> tuple[str, str]:
+        if reg is not None and self.reg is None:
+            self.reg = reg
+            self.dft = 0
+
         self.chk_nup(ip)
         self.chk_bup(ip)
         self.chk_rem(rem)
         if sz != -1:
             self.chk_sz(sz)
+            self.chk_df(abspath, sz)  # side effects; keep last-ish
 
         ap2, vp2 = self.rot(abspath)
         if abspath == ap2:
@@ -111,13 +138,33 @@ class Lim(object):
 
         return ap2, ("{}/{}".format(rem, vp2) if rem else vp2)
 
-    def chk_sz(self, sz: float) -> None:
-        if self.smin != -1 and sz < self.smin:
+    def chk_sz(self, sz: int) -> None:
+        if sz < self.smin:
             raise Pebkac(400, "file too small")
 
-        if self.smax != -1 and sz > self.smax:
+        if self.smax and sz > self.smax:
             raise Pebkac(400, "file too big")
 
+    def chk_df(self, abspath: str, sz: int, already_written: bool = False) -> None:
+        if not self.dfl:
+            return
+
+        if self.dft < time.time():
+            self.dft = int(time.time()) + 300
+            self.dfv = get_df(abspath)[0] or 0
+            for j in list(self.reg.values()) if self.reg else []:
+                self.dfv -= int(j["size"] / len(j["hash"]) * len(j["need"]))
+
+            if already_written:
+                sz = 0
+
+        if self.dfv - sz < self.dfl:
+            self.dft = min(self.dft, int(time.time()) + 10)
+            t = "server HDD is full; {} free, need {}"
+            raise Pebkac(500, t.format(humansize(self.dfv - self.dfl), humansize(sz)))
+
+        self.dfv -= int(sz)
+
     def chk_rem(self, rem: str) -> None:
         if self.nosub and rem:
             raise Pebkac(500, "no subdirectories allowed")
@@ -226,7 +273,7 @@ class VFS(object):
 
     def __init__(
         self,
-        log: Optional[RootLogger],
+        log: Optional["RootLogger"],
         realpath: str,
         vpath: str,
         axs: AXS,
@@ -569,7 +616,7 @@ class AuthSrv(object):
     def __init__(
         self,
         args: argparse.Namespace,
-        log_func: Optional[RootLogger],
+        log_func: Optional["RootLogger"],
         warn_anonwrite: bool = True,
     ) -> None:
         self.args = args
@@ -661,7 +708,7 @@ class AuthSrv(object):
                     raise Exception('invalid mountpoint "{}"'.format(vol_dst))
 
                 # cfg files override arguments and previous files
-                vol_src = bos.path.abspath(vol_src)
+                vol_src = absreal(vol_src)
                 vol_dst = vol_dst.strip("/")
                 self._map_volume(vol_src, vol_dst, mount, daxs, mflags)
                 continue
@@ -682,12 +729,12 @@ class AuthSrv(object):
         self, lvl: str, uname: str, axs: AXS, flags: dict[str, Any]
     ) -> None:
         if lvl.strip("crwmdg"):
-            raise Exception("invalid volume flag: {},{}".format(lvl, uname))
+            raise Exception("invalid volflag: {},{}".format(lvl, uname))
 
         if lvl == "c":
             cval: Union[bool, str] = True
             try:
-                # volume flag with arguments, possibly with a preceding list of bools
+                # volflag with arguments, possibly with a preceding list of bools
                 uname, cval = uname.split("=", 1)
             except:
                 # just one or more bools
@@ -772,7 +819,7 @@ class AuthSrv(object):
                     src = uncyg(src)
 
                 # print("\n".join([src, dst, perms]))
-                src = bos.path.abspath(src)
+                src = absreal(src)
                 dst = dst.strip("/")
                 self._map_volume(src, dst, mount, daxs, mflags)
 
@@ -801,7 +848,7 @@ class AuthSrv(object):
         if not mount:
             # -h says our defaults are CWD at root and read/write for everyone
             axs = AXS(["*"], ["*"], None, None)
-            vfs = VFS(self.log_func, bos.path.abspath("."), "", axs, {})
+            vfs = VFS(self.log_func, absreal("."), "", axs, {})
         elif "" not in mount:
             # there's volumes but no root; make root inaccessible
             vfs = VFS(self.log_func, "", "", AXS(), {})
@@ -917,13 +964,20 @@ class AuthSrv(object):
         vfs.histtab = {zv.realpath: zv.histpath for zv in vfs.all_vols.values()}
 
         for vol in vfs.all_vols.values():
-            lim = Lim()
+            lim = Lim(self.log_func)
             use = False
 
             if vol.flags.get("nosub"):
                 use = True
                 lim.nosub = True
 
+            zs = vol.flags.get("df") or (
+                "{}g".format(self.args.df) if self.args.df else ""
+            )
+            if zs:
+                use = True
+                lim.dfl = unhumanize(zs)
+
             zs = vol.flags.get("sz")
             if zs:
                 use = True
@@ -976,10 +1030,15 @@ class AuthSrv(object):
                 vol.flags["dathumb"] = True
                 vol.flags["dithumb"] = True
 
+        have_fk = False
         for vol in vfs.all_vols.values():
             fk = vol.flags.get("fk")
             if fk:
                 vol.flags["fk"] = int(fk) if fk is not True else 8
+                have_fk = True
+
+        if have_fk and re.match(r"^[0-9\.]+$", self.args.fk_salt):
+            self.log("filekey salt: {}".format(self.args.fk_salt))
 
         for vol in vfs.all_vols.values():
             if "pk" in vol.flags and "gz" not in vol.flags and "xz" not in vol.flags:
@@ -1008,7 +1067,7 @@ class AuthSrv(object):
                 if ptn:
                     vol.flags[vf] = re.compile(ptn)
 
-            for k in ["e2t", "e2ts", "e2tsr"]:
+            for k in ["e2t", "e2ts", "e2tsr", "e2v", "e2vu", "e2vp", "xdev", "xvol"]:
                 if getattr(self.args, k):
                     vol.flags[k] = True
 
@@ -1026,11 +1085,11 @@ class AuthSrv(object):
             if "mth" not in vol.flags:
                 vol.flags["mth"] = self.args.mth
 
-            # append parsers from argv to volume-flags
+            # append parsers from argv to volflags
             self._read_volflag(vol.flags, "mtp", self.args.mtp, True)
 
             # d2d drops all database features for a volume
-            for grp, rm in [["d2d", "e2d"], ["d2t", "e2t"]]:
+            for grp, rm in [["d2d", "e2d"], ["d2t", "e2t"], ["d2d", "e2v"]]:
                 if not vol.flags.get(grp, False):
                     continue
 
@@ -1052,6 +1111,12 @@ class AuthSrv(object):
 
                 vol.flags = {k: v for k, v in vol.flags.items() if not k.startswith(rm)}
 
+            for grp, rm in [["d2v", "e2v"]]:
+                if not vol.flags.get(grp, False):
+                    continue
+
+                vol.flags = {k: v for k, v in vol.flags.items() if not k.startswith(rm)}
+
             # verify tags mentioned by -mt[mp] are used by -mte
             local_mtp = {}
             local_only_mtp = {}
@@ -1083,7 +1148,7 @@ class AuthSrv(object):
 
             for mtp in local_only_mtp:
                 if mtp not in local_mte:
-                    t = 'volume "/{}" defines metadata tag "{}", but doesnt use it in "-mte" (or with "cmte" in its volume-flags)'
+                    t = 'volume "/{}" defines metadata tag "{}", but doesnt use it in "-mte" (or with "cmte" in its volflags)'
                     self.log(t.format(vol.vpath, mtp), 1)
                     errors = True
 
@@ -1092,7 +1157,7 @@ class AuthSrv(object):
         tags = [y for x in tags for y in x.split(",")]
         for mtp in tags:
             if mtp not in all_mte:
-                t = 'metadata tag "{}" is defined by "-mtm" or "-mtp", but is not used by "-mte" (or by any "cmte" volume-flag)'
+                t = 'metadata tag "{}" is defined by "-mtm" or "-mtp", but is not used by "-mte" (or by any "cmte" volflag)'
                 self.log(t.format(mtp), 1)
                 errors = True
 
@@ -1101,6 +1166,7 @@ class AuthSrv(object):
 
         vfs.bubble_flags()
 
+        have_e2d = False
         t = "volumes and permissions:\n"
         for zv in vfs.all_vols.values():
             if not self.warn_anonwrite:
@@ -1118,17 +1184,28 @@ class AuthSrv(object):
                 u = ", ".join("\033[35meverybody\033[0m" if x == "*" else x for x in u)
                 u = u if u else "\033[36m--none--\033[0m"
                 t += "\n|  {}:  {}".format(txt, u)
+
+            if "e2d" in zv.flags:
+                have_e2d = True
+
             t += "\n"
 
-        if self.warn_anonwrite and not self.args.no_voldump:
-            self.log(t)
+        if self.warn_anonwrite:
+            if not self.args.no_voldump:
+                self.log(t)
+
+            if have_e2d:
+                t = self.chk_sqlite_threadsafe()
+                if t:
+                    self.log("\n\033[{}\033[0m\n".format(t))
 
         try:
             zv, _ = vfs.get("/", "*", False, True)
             if self.warn_anonwrite and os.getcwd() == zv.realpath:
-                self.warn_anonwrite = False
-                t = "anyone can read/write the current directory: {}\n"
+                t = "anyone can write to the current directory: {}\n"
                 self.log(t.format(zv.realpath), c=1)
+
+            self.warn_anonwrite = False
         except Pebkac:
             self.warn_anonwrite = True
 
@@ -1142,6 +1219,23 @@ class AuthSrv(object):
             if pwds:
                 self.re_pwd = re.compile("=(" + "|".join(pwds) + ")([]&; ]|$)")
 
+    def chk_sqlite_threadsafe(self) -> str:
+        v = SQLITE_VER[-1:]
+
+        if v == "1":
+            # threadsafe (linux, windows)
+            return ""
+
+        if v == "2":
+            # module safe, connections unsafe (macos)
+            return "33m  your sqlite3 was compiled with reduced thread-safety;\n   database features (-e2d, -e2t) SHOULD be fine\n    but MAY cause database-corruption and crashes"
+
+        if v == "0":
+            # everything unsafe
+            return "31m  your sqlite3 was compiled WITHOUT thread-safety!\n   database features (-e2d, -e2t) will PROBABLY cause crashes!"
+
+        return "36m  cannot verify sqlite3 thread-safety; strange but probably fine"
+
     def dbg_ls(self) -> None:
         users = self.args.ls
         vol = "*"

copyparty/broker_util.py

@@ -42,7 +42,7 @@ class BrokerCli(object):
     """
 
     def __init__(self) -> None:
-        self.log: RootLogger = None
+        self.log: "RootLogger" = None
         self.args: argparse.Namespace = None
         self.asrv: AuthSrv = None
         self.httpsrv: "HttpSrv" = None

copyparty/fsutil.py

@@ -1,16 +1,17 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
-import ctypes
+import os
 import re
 import time
 
 from .__init__ import ANYWIN, MACOS
 from .authsrv import AXS, VFS
+from .bos import bos
 from .util import chkcmd, min_ex
 
 try:
-    from typing import Any, Optional, Union
+    from typing import Optional, Union
 
     from .util import RootLogger
 except:
@@ -18,9 +19,10 @@ except:
 
 
 class Fstab(object):
-    def __init__(self, log: RootLogger):
+    def __init__(self, log: "RootLogger"):
         self.log_func = log
 
+        self.trusted = False
         self.tab: Optional[VFS] = None
         self.cache: dict[str, str] = {}
         self.age = 0.0
@@ -38,17 +40,14 @@ class Fstab(object):
         msg = "failed to determine filesystem at [{}]; assuming {}\n{}"
 
         if ANYWIN:
-            fs = "vfat"  # can smb do sparse files? gonna guess no
+            fs = "vfat"
             try:
-                # good enough
-                disk = path.split(":", 1)[0]
-                disk = "{}:\\".format(disk).lower()
-                assert len(disk) == 3
-                path = disk
+                path = self._winpath(path)
             except:
                 self.log(msg.format(path, fs, min_ex()), 3)
                 return fs
 
+        path = path.lstrip("/")
         try:
             return self.cache[path]
         except:
@@ -64,6 +63,19 @@ class Fstab(object):
         self.log("found {} at {}".format(fs, path))
         return fs
 
+    def _winpath(self, path: str) -> str:
+        # try to combine volume-label + st_dev (vsn)
+        path = path.replace("/", "\\")
+        vid = path.split(":", 1)[0].strip("\\").split("\\", 1)[0]
+        try:
+            return "{}*{}".format(vid, bos.stat(path).st_dev)
+        except:
+            return vid
+
+    def build_fallback(self) -> None:
+        self.tab = VFS(self.log_func, "idk", "/", AXS(), {})
+        self.trusted = False
+
     def build_tab(self) -> None:
         self.log("building tab")
 
@@ -92,60 +104,51 @@ class Fstab(object):
 
     def relabel(self, path: str, nval: str) -> None:
         assert self.tab
+        self.cache = {}
+        if ANYWIN:
+            path = self._winpath(path)
+
+        path = path.lstrip("/")
         ptn = re.compile(r"^[^\\/]*")
-        vn, _ = self.tab._find(path)
+        vn, rem = self.tab._find(path)
+        if not self.trusted:
+            # no mtab access; have to build as we go
+            if "/" in rem:
+                self.tab.add("idk", os.path.join(vn.vpath, rem.split("/")[0]))
+            if rem:
+                self.tab.add(nval, path)
+            else:
+                vn.realpath = nval
+
+            return
+
         visit = [vn]
         while visit:
             vn = visit.pop()
             vn.realpath = ptn.sub(nval, vn.realpath)
             visit.extend(list(vn.nodes.values()))
-        self.cache = {}
 
     def get_unix(self, path: str) -> str:
         if not self.tab:
-            self.build_tab()
+            try:
+                self.build_tab()
+                self.trusted = True
+            except:
+                # prisonparty or other restrictive environment
+                self.log("failed to build tab:\n{}".format(min_ex()), 3)
+                self.build_fallback()
 
         assert self.tab
-        return self.tab._find(path)[0].realpath.split("/")[0]
+        ret = self.tab._find(path)[0]
+        if self.trusted or path == ret.vpath:
+            return ret.realpath.split("/")[0]
+        else:
+            return "idk"
 
     def get_w32(self, path: str) -> str:
-        # list mountpoints: fsutil fsinfo drives
+        if not self.tab:
+            self.build_fallback()
 
-        from ctypes.wintypes import BOOL, DWORD, LPCWSTR, LPDWORD, LPWSTR, MAX_PATH
-
-        def echk(rc: int, fun: Any, args: Any) -> None:
-            if not rc:
-                raise ctypes.WinError(ctypes.get_last_error())
-            return None
-
-        k32 = ctypes.WinDLL("kernel32", use_last_error=True)
-        k32.GetVolumeInformationW.errcheck = echk
-        k32.GetVolumeInformationW.restype = BOOL
-        k32.GetVolumeInformationW.argtypes = (
-            LPCWSTR,
-            LPWSTR,
-            DWORD,
-            LPDWORD,
-            LPDWORD,
-            LPDWORD,
-            LPWSTR,
-            DWORD,
-        )
-
-        bvolname = ctypes.create_unicode_buffer(MAX_PATH + 1)
-        bfstype = ctypes.create_unicode_buffer(MAX_PATH + 1)
-        serial = DWORD()
-        max_name_len = DWORD()
-        fs_flags = DWORD()
-
-        k32.GetVolumeInformationW(
-            path,
-            bvolname,
-            ctypes.sizeof(bvolname),
-            ctypes.byref(serial),
-            ctypes.byref(max_name_len),
-            ctypes.byref(fs_flags),
-            bfstype,
-            ctypes.sizeof(bfstype),
-        )
-        return bfstype.value
+        assert self.tab
+        ret = self.tab._find(path)[0]
+        return ret.realpath

copyparty/ftpd.py

@@ -391,7 +391,7 @@ class Ftpd(object):
             for h, lp in hs:
                 FTPServer((ip, int(lp)), h, ioloop)
 
-        thr = threading.Thread(target=ioloop.loop)
+        thr = threading.Thread(target=ioloop.loop, name="ftp")
         thr.daemon = True
         thr.start()
 

copyparty/httpcli.py

@@ -24,12 +24,7 @@ try:
 except:
     pass
 
-try:
-    import ctypes
-except:
-    pass
-
-from .__init__ import ANYWIN, PY2, TYPE_CHECKING, WINDOWS, E, unicode
+from .__init__ import ANYWIN, PY2, TYPE_CHECKING, E, unicode
 from .authsrv import VFS  # typechk
 from .bos import bos
 from .star import StreamTar
@@ -47,7 +42,9 @@ from .util import (
     exclude_dotfiles,
     fsenc,
     gen_filekey,
+    gen_filekey_dbg,
     gencookie,
+    get_df,
     get_spd,
     guess_mime,
     gzip_orig_sz,
@@ -112,6 +109,7 @@ class HttpCli(object):
         self.u2fh = conn.u2fh  # mypy404
         self.log_func = conn.log_func  # mypy404
         self.log_src = conn.log_src  # mypy404
+        self.gen_fk = self._gen_fk if self.args.log_fk else gen_filekey
         self.tls: bool = hasattr(self.s, "cipher")
 
         # placeholders; assigned by run()
@@ -181,6 +179,9 @@ class HttpCli(object):
         if rem.startswith("/") or rem.startswith("../") or "/../" in rem:
             raise Exception("that was close")
 
+    def _gen_fk(self, salt: str, fspath: str, fsize: int, inode: int) -> str:
+        return gen_filekey_dbg(salt, fspath, fsize, inode, self.log, self.args.log_fk)
+
     def j2s(self, name: str, **ka: Any) -> str:
         tpl = self.conn.hsrv.j2[name]
         ka["ts"] = self.conn.hsrv.cachebuster()
@@ -380,13 +381,21 @@ class HttpCli(object):
                 if not self._check_nonfatal(pex, post):
                     self.keepalive = False
 
-                msg = str(ex) if pex == ex else min_ex()
-                self.log("{}\033[0m, {}".format(msg, self.vpath), 3)
+                em = str(ex)
+                msg = em if pex == ex else min_ex()
+                self.log(
+                    "{}\033[0m, {}".format(msg, self.vpath),
+                    6 if em.startswith("client d/c ") else 3,
+                )
 
-                msg = "{}\r\nURL: {}\r\n".format(str(ex), self.vpath)
+                msg = "{}\r\nURL: {}\r\n".format(em, self.vpath)
                 if self.hint:
                     msg += "hint: {}\r\n".format(self.hint)
 
+                if "database is locked" in em:
+                    self.conn.hsrv.broker.say("log_stacks")
+                    msg += "hint: important info in the server log\r\n"
+
                 msg = "<pre>" + html_escape(msg)
                 self.reply(msg.encode("utf-8", "replace"), status=pex.code, volsan=True)
                 return self.keepalive
@@ -707,7 +716,7 @@ class HttpCli(object):
         reader, remains = self.get_body_reader()
         vfs, rem = self.asrv.vfs.get(self.vpath, self.uname, False, True)
         lim = vfs.get_dbv(rem)[0].lim
-        fdir = os.path.join(vfs.realpath, rem)
+        fdir = vfs.canonical(rem)
         if lim:
             fdir, rem = lim.all(self.ip, rem, remains, fdir)
 
@@ -809,7 +818,7 @@ class HttpCli(object):
 
         vsuf = ""
         if self.can_read and "fk" in vfs.flags:
-            vsuf = "?k=" + gen_filekey(
+            vsuf = "?k=" + self.gen_fk(
                 self.args.fk_salt,
                 path,
                 post_sz,
@@ -917,7 +926,7 @@ class HttpCli(object):
         except:
             raise Pebkac(422, "you POSTed invalid json")
 
-        # self.reply(b" DD" + b"oS Protection ", 503)
+        # self.reply(b"cloudflare", 503)
         # return True
 
         if "srch" in self.uparam or "srch" in body:
@@ -946,7 +955,7 @@ class HttpCli(object):
 
         if rem:
             try:
-                dst = os.path.join(vfs.realpath, rem)
+                dst = vfs.canonical(rem)
                 if not bos.path.isdir(dst):
                     bos.makedirs(dst)
             except OSError as ex:
@@ -1127,8 +1136,10 @@ class HttpCli(object):
             except:
                 self.log("failed to utime ({}, {})".format(fin_path, times))
 
+        cinf = self.headers.get("x-up2k-stat", "")
+
         spd = self._spd(post_sz)
-        self.log("{} thank".format(spd))
+        self.log("{:70} thank {}".format(spd, cinf))
         self.reply(b"thank")
         return True
 
@@ -1179,7 +1190,7 @@ class HttpCli(object):
         sanitized = sanitize_fn(new_dir, "", [])
 
         if not nullwrite:
-            fdir = os.path.join(vfs.realpath, rem)
+            fdir = vfs.canonical(rem)
             fn = os.path.join(fdir, sanitized)
 
             if not bos.path.isdir(fdir):
@@ -1218,7 +1229,7 @@ class HttpCli(object):
         sanitized = sanitize_fn(new_file, "", [])
 
         if not nullwrite:
-            fdir = os.path.join(vfs.realpath, rem)
+            fdir = vfs.canonical(rem)
             fn = os.path.join(fdir, sanitized)
 
             if bos.path.exists(fn):
@@ -1239,7 +1250,7 @@ class HttpCli(object):
 
         upload_vpath = self.vpath
         lim = vfs.get_dbv(rem)[0].lim
-        fdir_base = os.path.join(vfs.realpath, rem)
+        fdir_base = vfs.canonical(rem)
         if lim:
             fdir_base, rem = lim.all(self.ip, rem, -1, fdir_base)
             upload_vpath = "{}/{}".format(vfs.vpath, rem).strip("/")
@@ -1280,14 +1291,19 @@ class HttpCli(object):
                 else:
                     open_args = {}
                     tnam = fname = os.devnull
-                    fdir = ""
+                    fdir = abspath = ""
 
                 if lim:
                     lim.chk_bup(self.ip)
                     lim.chk_nup(self.ip)
 
                 try:
-                    max_sz = lim.smax if lim else 0
+                    max_sz = 0
+                    if lim:
+                        v1 = lim.smax
+                        v2 = lim.dfv - lim.dfl
+                        max_sz = min(v1, v2) if v1 and v2 else v1 or v2
+
                     with ren_open(tnam, "wb", 512 * 1024, **open_args) as zfw:
                         f, tnam = zfw["orz"]
                         tabspath = os.path.join(fdir, tnam)
@@ -1302,16 +1318,20 @@ class HttpCli(object):
                         lim.nup(self.ip)
                         lim.bup(self.ip, sz)
                         try:
+                            lim.chk_df(tabspath, sz, True)
                             lim.chk_sz(sz)
                             lim.chk_bup(self.ip)
                             lim.chk_nup(self.ip)
                         except:
-                            bos.unlink(tabspath)
-                            bos.unlink(abspath)
+                            if not nullwrite:
+                                bos.unlink(tabspath)
+                                bos.unlink(abspath)
                             fname = os.devnull
                             raise
 
-                    atomic_move(tabspath, abspath)
+                    if not nullwrite:
+                        atomic_move(tabspath, abspath)
+
                     files.append(
                         (sz, sha_hex, sha_b64, p_file or "(discarded)", fname, abspath)
                     )
@@ -1361,9 +1381,9 @@ class HttpCli(object):
         for sz, sha_hex, sha_b64, ofn, lfn, ap in files:
             vsuf = ""
             if self.can_read and "fk" in vfs.flags:
-                vsuf = "?k=" + gen_filekey(
+                vsuf = "?k=" + self.gen_fk(
                     self.args.fk_salt,
-                    abspath,
+                    ap,
                     sz,
                     0 if ANYWIN or not ap else bos.stat(ap).st_ino,
                 )[: vfs.flags["fk"]]
@@ -1441,7 +1461,7 @@ class HttpCli(object):
             raise Pebkac(411)
 
         rp, fn = vsplit(rem)
-        fp = os.path.join(vfs.realpath, rp)
+        fp = vfs.canonical(rp)
         lim = vfs.get_dbv(rem)[0].lim
         if lim:
             fp, rp = lim.all(self.ip, rp, clen, fp)
@@ -1915,7 +1935,13 @@ class HttpCli(object):
             vstate = {("/" + k).rstrip("/") + "/": v for k, v in vs["volstate"].items()}
         else:
             vstate = {}
-            vs = {"scanning": None, "hashq": None, "tagq": None, "mtpq": None}
+            vs = {
+                "scanning": None,
+                "hashq": None,
+                "tagq": None,
+                "mtpq": None,
+                "dbwt": None,
+            }
 
         if self.uparam.get("ls") in ["v", "t", "txt"]:
             if self.uname == "*":
@@ -1925,7 +1951,7 @@ class HttpCli(object):
 
             if vstate:
                 txt += "\nstatus:"
-                for k in ["scanning", "hashq", "tagq", "mtpq"]:
+                for k in ["scanning", "hashq", "tagq", "mtpq", "dbwt"]:
                     txt += " {}({})".format(k, vs[k])
 
             if rvol:
@@ -1954,6 +1980,7 @@ class HttpCli(object):
             hashq=vs["hashq"],
             tagq=vs["tagq"],
             mtpq=vs["mtpq"],
+            dbwt=vs["dbwt"],
             url_suf=suf,
             k304=self.k304(),
         )
@@ -2291,7 +2318,7 @@ class HttpCli(object):
 
         if not is_dir and (self.can_read or self.can_get):
             if not self.can_read and "fk" in vn.flags:
-                correct = gen_filekey(
+                correct = self.gen_fk(
                     self.args.fk_salt, abspath, st.st_size, 0 if ANYWIN else st.st_ino
                 )[: vn.flags["fk"]]
                 got = self.uparam.get("k")
@@ -2315,26 +2342,14 @@ class HttpCli(object):
         except:
             self.log("#wow #whoa")
 
-        try:
-            # some fuses misbehave
-            if not self.args.nid:
-                if WINDOWS:
-                    try:
-                        bfree = ctypes.c_ulonglong(0)
-                        ctypes.windll.kernel32.GetDiskFreeSpaceExW(  # type: ignore
-                            ctypes.c_wchar_p(abspath), None, None, ctypes.pointer(bfree)
-                        )
-                        srv_info.append(humansize(bfree.value) + " free")
-                    except:
-                        pass
-                else:
-                    sv = os.statvfs(fsenc(abspath))
-                    free = humansize(sv.f_frsize * sv.f_bfree, True)
-                    total = humansize(sv.f_frsize * sv.f_blocks, True)
-
-                    srv_info.append("{} free of {}".format(free, total))
-        except:
-            pass
+        if not self.args.nid:
+            free, total = get_df(abspath)
+            if total is not None:
+                h1 = humansize(free or 0)
+                h2 = humansize(total)
+                srv_info.append("{} free of {}".format(h1, h2))
+            elif free is not None:
+                srv_info.append(humansize(free, True) + " free")
 
         srv_infot = "</span> // <span>".join(srv_info)
 
@@ -2413,6 +2428,7 @@ class HttpCli(object):
             "dtheme": self.args.theme,
             "themes": self.args.themes,
             "turbolvl": self.args.turbo,
+            "u2sort": self.args.u2sort,
         }
 
         if self.args.js_browser:
@@ -2526,7 +2542,7 @@ class HttpCli(object):
             if add_fk:
                 href = "{}?k={}".format(
                     quotep(href),
-                    gen_filekey(
+                    self.gen_fk(
                         self.args.fk_salt, fspath, sz, 0 if ANYWIN else inf.st_ino
                     )[:add_fk],
                 )

copyparty/httpconn.py

@@ -62,7 +62,7 @@ class HttpConn(object):
         self.nreq: int = 0  # mypy404
         self.nbyte: int = 0  # mypy404
         self.u2idx: Optional[U2idx] = None
-        self.log_func: Util.RootLogger = hsrv.log  # mypy404
+        self.log_func: "Util.RootLogger" = hsrv.log  # mypy404
         self.log_src: str = "httpconn"  # mypy404
         self.lf_url: Optional[Pattern[str]] = (
             re.compile(self.args.lf_url) if self.args.lf_url else None

copyparty/httpsrv.py

@@ -102,7 +102,7 @@ class HttpSrv(object):
                 start_log_thrs(self.log, self.args.log_thrs, nid)
 
         self.th_cfg: dict[str, Any] = {}
-        t = threading.Thread(target=self.post_init)
+        t = threading.Thread(target=self.post_init, name="hsrv-init2")
         t.daemon = True
         t.start()
 
@@ -165,13 +165,13 @@ class HttpSrv(object):
         """listens on a shared tcp server"""
         ip, port = srv_sck.getsockname()
         fno = srv_sck.fileno()
-        msg = "subscribed @ {}:{}  f{}".format(ip, port, fno)
+        msg = "subscribed @ {}:{}  f{} p{}".format(ip, port, fno, os.getpid())
         self.log(self.name, msg)
 
         def fun() -> None:
             self.broker.say("cb_httpsrv_up")
 
-        threading.Thread(target=fun).start()
+        threading.Thread(target=fun, name="sig-hsrv-up1").start()
 
         while not self.stopping:
             if self.args.log_conn:
@@ -261,8 +261,11 @@ class HttpSrv(object):
                 )
                 self.thr_client(sck, addr)
                 me.name = self.name + "-poolw"
-            except:
-                self.log(self.name, "thr_client: " + min_ex(), 3)
+            except Exception as ex:
+                if str(ex).startswith("client d/c "):
+                    self.log(self.name, "thr_client: " + str(ex), 6)
+                else:
+                    self.log(self.name, "thr_client: " + min_ex(), 3)
 
     def shutdown(self) -> None:
         self.stopping = True

copyparty/ico.py

@@ -15,7 +15,7 @@ class Ico(object):
     def get(self, ext: str, as_thumb: bool) -> tuple[str, bytes]:
         """placeholder to make thumbnails not break"""
 
-        zb = hashlib.md5(ext.encode("utf-8")).digest()[:2]
+        zb = hashlib.sha1(ext.encode("utf-8")).digest()[2:4]
         if PY2:
             zb = [ord(x) for x in zb]
 

copyparty/mtag.py

@@ -46,6 +46,7 @@ class MParser(object):
         self.force = False
         self.kill = "t"  # tree; all children recursively
         self.audio = "y"
+        self.pri = 0  # priority; higher = later
         self.ext = []
 
         while True:
@@ -83,6 +84,10 @@ class MParser(object):
                 self.ext.append(arg[1:])
                 continue
 
+            if arg.startswith("p"):
+                self.pri = int(arg[1:] or "1")
+                continue
+
             raise Exception()
 
 
@@ -173,7 +178,7 @@ def parse_ffprobe(txt: str) -> tuple[dict[str, tuple[int, Any]], dict[str, list[
             ]
 
         if typ == "format":
-            kvm = [["duration", ".dur"], ["bit_rate", ".q"]]
+            kvm = [["duration", ".dur"], ["bit_rate", ".q"], ["format_name", "fmt"]]
 
         for sk, rk in kvm:
             v1 = strm.get(sk)
@@ -234,6 +239,9 @@ def parse_ffprobe(txt: str) -> tuple[dict[str, tuple[int, Any]], dict[str, list[
             if ".q" in ret:
                 del ret[".q"]
 
+    if "fmt" in ret:
+        ret["fmt"] = ret["fmt"].split(",")[0]
+
     if ".resw" in ret and ".resh" in ret:
         ret["res"] = "{}x{}".format(ret[".resw"], ret[".resh"])
 
@@ -243,7 +251,7 @@ def parse_ffprobe(txt: str) -> tuple[dict[str, tuple[int, Any]], dict[str, list[
 
 
 class MTag(object):
-    def __init__(self, log_func: RootLogger, args: argparse.Namespace) -> None:
+    def __init__(self, log_func: "RootLogger", args: argparse.Namespace) -> None:
         self.log_func = log_func
         self.args = args
         self.usable = True
@@ -432,6 +440,8 @@ class MTag(object):
         return r1
 
     def get_mutagen(self, abspath: str) -> dict[str, Union[str, float]]:
+        ret: dict[str, tuple[int, Any]] = {}
+
         if not bos.path.isfile(abspath):
             return {}
 
@@ -445,7 +455,10 @@ class MTag(object):
             return self.get_ffprobe(abspath) if self.can_ffprobe else {}
 
         sz = bos.path.getsize(abspath)
-        ret = {".q": (0, int((sz / md.info.length) / 128))}
+        try:
+            ret[".q"] = (0, int((sz / md.info.length) / 128))
+        except:
+            pass
 
         for attr, k, norm in [
             ["codec", "ac", unicode],
@@ -487,7 +500,9 @@ class MTag(object):
         ret, md = ffprobe(abspath)
         return self.normalize_tags(ret, md)
 
-    def get_bin(self, parsers: dict[str, MParser], abspath: str) -> dict[str, Any]:
+    def get_bin(
+        self, parsers: dict[str, MParser], abspath: str, oth_tags: dict[str, Any]
+    ) -> dict[str, Any]:
         if not bos.path.isfile(abspath):
             return {}
 
@@ -497,8 +512,8 @@ class MTag(object):
         env = os.environ.copy()
         env["PYTHONPATH"] = pypath
 
-        ret = {}
-        for tagname, parser in parsers.items():
+        ret: dict[str, Any] = {}
+        for tagname, parser in sorted(parsers.items(), key=lambda x: (x[1].pri, x[0])):
             try:
                 cmd = [parser.bin, abspath]
                 if parser.bin.endswith(".py"):
@@ -506,6 +521,11 @@ class MTag(object):
 
                 args = {"env": env, "timeout": parser.timeout, "kill": parser.kill}
 
+                if parser.pri:
+                    zd = oth_tags.copy()
+                    zd.update(ret)
+                    args["sin"] = json.dumps(zd).encode("utf-8", "replace")
+
                 if WINDOWS:
                     args["creationflags"] = 0x4000
                 else:

copyparty/star.py

@@ -44,7 +44,7 @@ class StreamTar(StreamArc):
 
     def __init__(
         self,
-        log: NamedLogger,
+        log: "NamedLogger",
         fgen: Generator[dict[str, Any], None, None],
         **kwargs: Any
     ):
@@ -65,17 +65,19 @@ class StreamTar(StreamArc):
         w.start()
 
     def gen(self) -> Generator[Optional[bytes], None, None]:
-        while True:
-            buf = self.qfile.q.get()
-            if not buf:
-                break
+        try:
+            while True:
+                buf = self.qfile.q.get()
+                if not buf:
+                    break
 
-            self.co += len(buf)
-            yield buf
+                self.co += len(buf)
+                yield buf
 
-        yield None
-        if self.errf:
-            bos.unlink(self.errf["ap"])
+            yield None
+        finally:
+            if self.errf:
+                bos.unlink(self.errf["ap"])
 
     def ser(self, f: dict[str, Any]) -> None:
         name = f["vp"]

copyparty/sutil.py

@@ -17,7 +17,7 @@ except:
 class StreamArc(object):
     def __init__(
         self,
-        log: NamedLogger,
+        log: "NamedLogger",
         fgen: Generator[dict[str, Any], None, None],
         **kwargs: Any
     ):

copyparty/svchub.py

@@ -2,8 +2,11 @@
 from __future__ import print_function, unicode_literals
 
 import argparse
+import base64
 import calendar
+import gzip
 import os
+import re
 import shlex
 import signal
 import socket
@@ -17,7 +20,7 @@ try:
     from types import FrameType
 
     import typing
-    from typing import Optional, Union
+    from typing import Any, Optional, Union
 except:
     pass
 
@@ -27,7 +30,15 @@ from .mtag import HAVE_FFMPEG, HAVE_FFPROBE
 from .tcpsrv import TcpSrv
 from .th_srv import HAVE_PIL, HAVE_VIPS, HAVE_WEBP, ThumbSrv
 from .up2k import Up2k
-from .util import ansi_re, min_ex, mp, start_log_thrs, start_stackmon
+from .util import (
+    VERSIONS,
+    alltrace,
+    ansi_re,
+    min_ex,
+    mp,
+    start_log_thrs,
+    start_stackmon,
+)
 
 
 class SvcHub(object):
@@ -47,15 +58,18 @@ class SvcHub(object):
         self.logf: Optional[typing.TextIO] = None
         self.logf_base_fn = ""
         self.stop_req = False
-        self.reload_req = False
         self.stopping = False
+        self.stopped = False
+        self.reload_req = False
         self.reloading = False
         self.stop_cond = threading.Condition()
+        self.nsigs = 3
         self.retcode = 0
         self.httpsrv_up = 0
 
         self.log_mutex = threading.Lock()
         self.next_day = 0
+        self.tstack = 0.0
 
         if args.sss or args.s >= 3:
             args.ss = True
@@ -110,6 +124,9 @@ class SvcHub(object):
         if not args.hardlink and args.never_symlink:
             args.no_dedup = True
 
+        if args.log_fk:
+            args.log_fk = re.compile(args.log_fk)
+
         # initiate all services to manage
         self.asrv = AuthSrv(self.args, self.log)
         if args.ls:
@@ -129,8 +146,8 @@ class SvcHub(object):
         self.args.th_dec = list(decs.keys())
         self.thumbsrv = None
         if not args.no_thumb:
-            t = "decoder preference: {}".format(", ".join(self.args.th_dec))
-            self.log("thumb", t)
+            t = ", ".join(self.args.th_dec) or "(None available)"
+            self.log("thumb", "decoder preference: {}".format(t))
 
             if "pil" in self.args.th_dec and not HAVE_WEBP:
                 msg = "disabling webp thumbnails because either libwebp is not available or your Pillow is too old"
@@ -189,6 +206,9 @@ class SvcHub(object):
         self.log("root", t, 1)
 
         self.retcode = 1
+        self.sigterm()
+
+    def sigterm(self) -> None:
         os.kill(os.getpid(), signal.SIGTERM)
 
     def cb_httpsrv_up(self) -> None:
@@ -252,7 +272,7 @@ class SvcHub(object):
     def run(self) -> None:
         self.tcpsrv.run()
 
-        thr = threading.Thread(target=self.thr_httpsrv_up)
+        thr = threading.Thread(target=self.thr_httpsrv_up, name="sig-hsrv-up2")
         thr.daemon = True
         thr.start()
 
@@ -280,7 +300,9 @@ class SvcHub(object):
                 pass
 
             self.shutdown()
-            thr.join()
+            # cant join; eats signals on win10
+            while not self.stopped:
+                time.sleep(0.1)
         else:
             self.stop_thr()
 
@@ -289,7 +311,7 @@ class SvcHub(object):
             return "cannot reload; already in progress"
 
         self.reloading = True
-        t = threading.Thread(target=self._reload)
+        t = threading.Thread(target=self._reload, name="reloading")
         t.daemon = True
         t.start()
         return "reload initiated"
@@ -316,9 +338,22 @@ class SvcHub(object):
 
     def signal_handler(self, sig: int, frame: Optional[FrameType]) -> None:
         if self.stopping:
-            return
+            if self.nsigs <= 0:
+                try:
+                    threading.Thread(target=self.pr, args=("OMBO BREAKER",)).start()
+                    time.sleep(0.1)
+                except:
+                    pass
 
-        if sig == signal.SIGUSR1:
+                if ANYWIN:
+                    os.system("taskkill /f /pid {}".format(os.getpid()))
+                else:
+                    os.kill(os.getpid(), signal.SIGKILL)
+            else:
+                self.nsigs -= 1
+                return
+
+        if not ANYWIN and sig == signal.SIGUSR1:
             self.reload_req = True
         else:
             self.stop_req = True
@@ -339,9 +374,7 @@ class SvcHub(object):
 
         ret = 1
         try:
-            with self.log_mutex:
-                print("OPYTHAT")
-
+            self.pr("OPYTHAT")
             self.tcpsrv.shutdown()
             self.broker.shutdown()
             self.up2k.shutdown()
@@ -354,19 +387,23 @@ class SvcHub(object):
                         break
 
                     if n == 3:
-                        print("waiting for thumbsrv (10sec)...")
+                        self.pr("waiting for thumbsrv (10sec)...")
 
-            print("nailed it", end="")
+            self.pr("nailed it", end="")
             ret = self.retcode
+        except:
+            self.pr("\033[31m[ error during shutdown ]\n{}\033[0m".format(min_ex()))
+            raise
         finally:
             if self.args.wintitle:
                 print("\033]0;\033\\", file=sys.stderr, end="")
                 sys.stderr.flush()
 
-            print("\033[0m")
+            self.pr("\033[0m")
             if self.logf:
                 self.logf.close()
 
+            self.stopped = True
             sys.exit(ret)
 
     def _log_disabled(self, src: str, msg: str, c: Union[int, str] = 0) -> None:
@@ -433,6 +470,10 @@ class SvcHub(object):
             if self.logf:
                 self.logf.write(msg)
 
+    def pr(self, *a: Any, **ka: Any) -> None:
+        with self.log_mutex:
+            print(*a, **ka)
+
     def check_mp_support(self) -> str:
         vmin = sys.version_info[1]
         if WINDOWS:
@@ -497,3 +538,16 @@ class SvcHub(object):
             sck.sendall(b"READY=1")
         except:
             self.log("sd_notify", min_ex())
+
+    def log_stacks(self) -> None:
+        td = time.time() - self.tstack
+        if td < 300:
+            self.log("stacks", "cooldown {}".format(td))
+            return
+
+        self.tstack = time.time()
+        zs = "{}\n{}".format(VERSIONS, alltrace())
+        zb = zs.encode("utf-8", "replace")
+        zb = gzip.compress(zb)
+        zs = base64.b64encode(zb).decode("ascii")
+        self.log("stacks", zs)

copyparty/szip.py

@@ -218,7 +218,7 @@ def gen_ecdr64_loc(ecdr64_pos: int) -> bytes:
 class StreamZip(StreamArc):
     def __init__(
         self,
-        log: NamedLogger,
+        log: "NamedLogger",
         fgen: Generator[dict[str, Any], None, None],
         utf8: bool = False,
         pre_crc: bool = False,
@@ -272,41 +272,44 @@ class StreamZip(StreamArc):
 
     def gen(self) -> Generator[bytes, None, None]:
         errors = []
-        for f in self.fgen:
-            if "err" in f:
-                errors.append((f["vp"], f["err"]))
-                continue
+        try:
+            for f in self.fgen:
+                if "err" in f:
+                    errors.append((f["vp"], f["err"]))
+                    continue
 
-            try:
-                for x in self.ser(f):
+                try:
+                    for x in self.ser(f):
+                        yield x
+                except GeneratorExit:
+                    raise
+                except:
+                    ex = min_ex(5, True).replace("\n", "\n-- ")
+                    errors.append((f["vp"], ex))
+
+            if errors:
+                errf, txt = errdesc(errors)
+                self.log("\n".join(([repr(errf)] + txt[1:])))
+                for x in self.ser(errf):
                     yield x
-            except:
-                ex = min_ex(5, True).replace("\n", "\n-- ")
-                errors.append((f["vp"], ex))
 
-        if errors:
-            errf, txt = errdesc(errors)
-            self.log("\n".join(([repr(errf)] + txt[1:])))
-            for x in self.ser(errf):
-                yield x
+            cdir_pos = self.pos
+            for name, sz, ts, crc, h_pos in self.items:
+                buf = gen_hdr(h_pos, name, sz, ts, self.utf8, crc, self.pre_crc)
+                yield self._ct(buf)
+            cdir_end = self.pos
 
-        cdir_pos = self.pos
-        for name, sz, ts, crc, h_pos in self.items:
-            buf = gen_hdr(h_pos, name, sz, ts, self.utf8, crc, self.pre_crc)
-            yield self._ct(buf)
-        cdir_end = self.pos
+            _, need_64 = gen_ecdr(self.items, cdir_pos, cdir_end)
+            if need_64:
+                ecdir64_pos = self.pos
+                buf = gen_ecdr64(self.items, cdir_pos, cdir_end)
+                yield self._ct(buf)
 
-        _, need_64 = gen_ecdr(self.items, cdir_pos, cdir_end)
-        if need_64:
-            ecdir64_pos = self.pos
-            buf = gen_ecdr64(self.items, cdir_pos, cdir_end)
-            yield self._ct(buf)
+                buf = gen_ecdr64_loc(ecdir64_pos)
+                yield self._ct(buf)
 
-            buf = gen_ecdr64_loc(ecdir64_pos)
-            yield self._ct(buf)
-
-        ecdr, _ = gen_ecdr(self.items, cdir_pos, cdir_end)
-        yield self._ct(ecdr)
-
-        if errors:
-            bos.unlink(errf["ap"])
+            ecdr, _ = gen_ecdr(self.items, cdir_pos, cdir_end)
+            yield self._ct(ecdr)
+        finally:
+            if errors:
+                bos.unlink(errf["ap"])

copyparty/tcpsrv.py

@@ -1,6 +1,7 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
+import os
 import re
 import socket
 import sys
@@ -128,7 +129,7 @@ class TcpSrv(object):
             srv.listen(self.args.nc)
             ip, port = srv.getsockname()
             fno = srv.fileno()
-            msg = "listening @ {}:{}  f{}".format(ip, port, fno)
+            msg = "listening @ {}:{}  f{} p{}".format(ip, port, fno, os.getpid())
             self.log("tcpsrv", msg)
             if self.args.q:
                 print(msg)

copyparty/th_srv.py

@@ -559,14 +559,15 @@ class ThumbSrv(object):
     def clean(self, histpath: str) -> int:
         ret = 0
         for cat in ["th", "ac"]:
-            ret += self._clean(histpath, cat, "")
+            top = os.path.join(histpath, cat)
+            if not bos.path.isdir(top):
+                continue
+
+            ret += self._clean(cat, top)
 
         return ret
 
-    def _clean(self, histpath: str, cat: str, thumbpath: str) -> int:
-        if not thumbpath:
-            thumbpath = os.path.join(histpath, cat)
-
+    def _clean(self, cat: str, thumbpath: str) -> int:
         # self.log("cln {}".format(thumbpath))
         exts = ["jpg", "webp"] if cat == "th" else ["opus", "caf"]
         maxage = getattr(self.args, cat + "_maxage")
@@ -600,7 +601,7 @@ class ThumbSrv(object):
                             self.log("rm -rf [{}]".format(fp))
                             shutil.rmtree(fp, ignore_errors=True)
                 else:
-                    self._clean(histpath, cat, fp)
+                    ndirs += self._clean(cat, fp)
 
                 continue
 

copyparty/util.py

@@ -4,6 +4,7 @@ from __future__ import print_function, unicode_literals
 import base64
 import contextlib
 import hashlib
+import math
 import mimetypes
 import os
 import platform
@@ -21,9 +22,17 @@ import traceback
 from collections import Counter
 from datetime import datetime
 
+from queue import Queue
+
 from .__init__ import ANYWIN, PY2, TYPE_CHECKING, VT100, WINDOWS
+from .__version__ import S_BUILD_DT, S_VERSION
 from .stolen import surrogateescape
 
+try:
+    import ctypes
+except:
+    pass
+
 try:
     HAVE_SQLITE3 = True
     import sqlite3  # pylint: disable=unused-import  # typechk
@@ -41,7 +50,7 @@ try:
     from collections.abc import Callable, Iterable
 
     import typing
-    from typing import Any, Generator, Optional, Protocol, Union
+    from typing import Any, Generator, Optional, Pattern, Protocol, Union
 
     class RootLogger(Protocol):
         def __call__(self, src: str, msg: str, c: Union[int, str] = 0) -> None:
@@ -78,8 +87,6 @@ else:
     from urllib import quote  # pylint: disable=no-name-in-module
     from urllib import unquote  # pylint: disable=no-name-in-module
 
-_: Any = (mp, BytesIO, quote, unquote)
-__all__ = ["mp", "BytesIO", "quote", "unquote"]
 
 try:
     struct.unpack(b">i", b"idgi")
@@ -137,6 +144,9 @@ IMPLICATIONS = [
     ["e2tsr", "e2ts"],
     ["e2ts", "e2t"],
     ["e2t", "e2d"],
+    ["e2vu", "e2v"],
+    ["e2vp", "e2v"],
+    ["e2v", "e2d"],
 ]
 
 
@@ -205,6 +215,72 @@ REKOBO_KEY = {
 REKOBO_LKEY = {k.lower(): v for k, v in REKOBO_KEY.items()}
 
 
+def py_desc() -> str:
+    interp = platform.python_implementation()
+    py_ver = ".".join([str(x) for x in sys.version_info])
+    ofs = py_ver.find(".final.")
+    if ofs > 0:
+        py_ver = py_ver[:ofs]
+
+    try:
+        bitness = struct.calcsize(b"P") * 8
+    except:
+        bitness = struct.calcsize("P") * 8
+
+    host_os = platform.system()
+    compiler = platform.python_compiler()
+
+    m = re.search(r"([0-9]+\.[0-9\.]+)", platform.version())
+    os_ver = m.group(1) if m else ""
+
+    return "{:>9} v{} on {}{} {} [{}]".format(
+        interp, py_ver, host_os, bitness, os_ver, compiler
+    )
+
+
+def _sqlite_ver() -> str:
+    try:
+        co = sqlite3.connect(":memory:")
+        cur = co.cursor()
+        try:
+            vs = cur.execute("select * from pragma_compile_options").fetchall()
+        except:
+            vs = cur.execute("pragma compile_options").fetchall()
+
+        v = next(x[0].split("=")[1] for x in vs if x[0].startswith("THREADSAFE="))
+        cur.close()
+        co.close()
+    except:
+        v = "W"
+
+    return "{}*{}".format(sqlite3.sqlite_version, v)
+
+
+try:
+    SQLITE_VER = _sqlite_ver()
+except:
+    SQLITE_VER = "(None)"
+
+try:
+    from jinja2 import __version__ as JINJA_VER
+except:
+    JINJA_VER = "(None)"
+
+try:
+    from pyftpdlib.__init__ import __ver__ as PYFTPD_VER
+except:
+    PYFTPD_VER = "(None)"
+
+
+VERSIONS = "copyparty v{} ({})\n{}\n   sqlite v{} | jinja v{} | pyftpd v{}".format(
+    S_VERSION, S_BUILD_DT, py_desc(), SQLITE_VER, JINJA_VER, PYFTPD_VER
+)
+
+
+_: Any = (mp, BytesIO, quote, unquote, SQLITE_VER, JINJA_VER, PYFTPD_VER)
+__all__ = ["mp", "BytesIO", "quote", "unquote", "SQLITE_VER", "JINJA_VER", "PYFTPD_VER"]
+
+
 class Cooldown(object):
     def __init__(self, maxage: float) -> None:
         self.maxage = maxage
@@ -240,7 +316,7 @@ class _Unrecv(object):
     undo any number of socket recv ops
     """
 
-    def __init__(self, s: socket.socket, log: Optional[NamedLogger]) -> None:
+    def __init__(self, s: socket.socket, log: Optional["NamedLogger"]) -> None:
         self.s = s
         self.log = log
         self.buf: bytes = b""
@@ -284,7 +360,7 @@ class _LUnrecv(object):
     with expensive debug logging
     """
 
-    def __init__(self, s: socket.socket, log: Optional[NamedLogger]) -> None:
+    def __init__(self, s: socket.socket, log: Optional["NamedLogger"]) -> None:
         self.s = s
         self.log = log
         self.buf = b""
@@ -421,6 +497,104 @@ class ProgressPrinter(threading.Thread):
         sys.stdout.flush()  # necessary on win10 even w/ stderr btw
 
 
+class MTHash(object):
+    def __init__(self, cores: int):
+        self.pp: Optional[ProgressPrinter] = None
+        self.f: Optional[typing.BinaryIO] = None
+        self.sz = 0
+        self.csz = 0
+        self.stop = False
+        self.omutex = threading.Lock()
+        self.imutex = threading.Lock()
+        self.work_q: Queue[int] = Queue()
+        self.done_q: Queue[tuple[int, str, int, int]] = Queue()
+        self.thrs = []
+        for n in range(cores):
+            t = threading.Thread(target=self.worker, name="mth-" + str(n))
+            t.daemon = True
+            t.start()
+            self.thrs.append(t)
+
+    def hash(
+        self,
+        f: typing.BinaryIO,
+        fsz: int,
+        chunksz: int,
+        pp: Optional[ProgressPrinter] = None,
+        prefix: str = "",
+        suffix: str = "",
+    ) -> list[tuple[str, int, int]]:
+        with self.omutex:
+            self.f = f
+            self.sz = fsz
+            self.csz = chunksz
+
+            chunks: dict[int, tuple[str, int, int]] = {}
+            nchunks = int(math.ceil(fsz / chunksz))
+            for nch in range(nchunks):
+                self.work_q.put(nch)
+
+            ex = ""
+            for nch in range(nchunks):
+                qe = self.done_q.get()
+                try:
+                    nch, dig, ofs, csz = qe
+                    chunks[nch] = (dig, ofs, csz)
+                except:
+                    ex = ex or str(qe)
+
+                if pp:
+                    mb = int((fsz - nch * chunksz) / 1024 / 1024)
+                    pp.msg = prefix + str(mb) + suffix
+
+            if ex:
+                raise Exception(ex)
+
+            ret = []
+            for n in range(nchunks):
+                ret.append(chunks[n])
+
+            self.f = None
+            self.csz = 0
+            self.sz = 0
+            return ret
+
+    def worker(self) -> None:
+        while True:
+            ofs = self.work_q.get()
+            try:
+                v = self.hash_at(ofs)
+            except Exception as ex:
+                v = str(ex)  # type: ignore
+
+            self.done_q.put(v)
+
+    def hash_at(self, nch: int) -> tuple[int, str, int, int]:
+        f = self.f
+        ofs = ofs0 = nch * self.csz
+        chunk_sz = chunk_rem = min(self.csz, self.sz - ofs)
+        if self.stop:
+            return nch, "", ofs0, chunk_sz
+
+        assert f
+        hashobj = hashlib.sha512()
+        while chunk_rem > 0:
+            with self.imutex:
+                f.seek(ofs)
+                buf = f.read(min(chunk_rem, 1024 * 1024 * 12))
+
+            if not buf:
+                raise Exception("EOF at " + str(ofs))
+
+            hashobj.update(buf)
+            chunk_rem -= len(buf)
+            ofs += len(buf)
+
+        bdig = hashobj.digest()[:33]
+        udig = base64.urlsafe_b64encode(bdig).decode("utf-8")
+        return nch, udig, ofs0, chunk_sz
+
+
 def uprint(msg: str) -> None:
     try:
         print(msg, end="")
@@ -659,7 +833,9 @@ def ren_open(
 
 
 class MultipartParser(object):
-    def __init__(self, log_func: NamedLogger, sr: Unrecv, http_headers: dict[str, str]):
+    def __init__(
+        self, log_func: "NamedLogger", sr: Unrecv, http_headers: dict[str, str]
+    ):
         self.sr = sr
         self.log = log_func
         self.headers = http_headers
@@ -922,6 +1098,24 @@ def gen_filekey(salt: str, fspath: str, fsize: int, inode: int) -> str:
     ).decode("ascii")
 
 
+def gen_filekey_dbg(
+    salt: str,
+    fspath: str,
+    fsize: int,
+    inode: int,
+    log: "NamedLogger",
+    log_ptn: Optional[Pattern[str]],
+) -> str:
+    ret = gen_filekey(salt, fspath, fsize, inode)
+
+    assert log_ptn
+    if log_ptn.search(fspath):
+        t = "fk({}) salt({}) size({}) inode({}) fspath({})"
+        log(t.format(ret[:8], salt, fsize, inode, fspath))
+
+    return ret
+
+
 def gencookie(k: str, v: str, dur: Optional[int]) -> str:
     v = v.replace(";", "")
     if dur:
@@ -979,6 +1173,11 @@ def s2hms(s: float, optional_h: bool = False) -> str:
     return "{}:{:02}:{:02}".format(h, m, s)
 
 
+def djoin(*paths: str) -> str:
+    """joins without adding a trailing slash on blank args"""
+    return os.path.join(*[x for x in paths if x])
+
+
 def uncyg(path: str) -> str:
     if len(path) < 2 or not path.startswith("/"):
         return path
@@ -1128,6 +1327,10 @@ def vsplit(vpath: str) -> tuple[str, str]:
     return vpath.rsplit("/", 1)  # type: ignore
 
 
+def vjoin(rd: str, fn: str) -> str:
+    return rd + "/" + fn if rd else fn
+
+
 def w8dec(txt: bytes) -> str:
     """decodes filesystem-bytes to wtf8"""
     if PY2:
@@ -1181,15 +1384,30 @@ def s3enc(mem_cur: "sqlite3.Cursor", rd: str, fn: str) -> tuple[str, str]:
 
 
 def s3dec(rd: str, fn: str) -> tuple[str, str]:
-    ret = []
-    for v in [rd, fn]:
-        if v.startswith("//"):
-            ret.append(w8b64dec(v[2:]))
-            # self.log("mojide [{}] {}".format(ret[-1], v[2:]))
-        else:
-            ret.append(v)
+    return (
+        w8b64dec(rd[2:]) if rd.startswith("//") else rd,
+        w8b64dec(fn[2:]) if fn.startswith("//") else fn,
+    )
 
-    return ret[0], ret[1]
+
+def db_ex_chk(log: "NamedLogger", ex: Exception, db_path: str) -> bool:
+    if str(ex) != "database is locked":
+        return False
+
+    thr = threading.Thread(target=lsof, args=(log, db_path), name="dbex")
+    thr.daemon = True
+    thr.start()
+
+    return True
+
+
+def lsof(log: "NamedLogger", abspath: str) -> None:
+    try:
+        rc, so, se = runcmd([b"lsof", b"-R", fsenc(abspath)], timeout=5)
+        zs = (so.strip() + "\n" + se.strip()).strip()
+        log("lsof {} = {}\n{}".format(abspath, rc, zs), 3)
+    except:
+        log("lsof failed; " + min_ex(), 3)
 
 
 def atomic_move(usrc: str, udst: str) -> None:
@@ -1204,6 +1422,24 @@ def atomic_move(usrc: str, udst: str) -> None:
         os.rename(src, dst)
 
 
+def get_df(abspath: str) -> tuple[Optional[int], Optional[int]]:
+    try:
+        # some fuses misbehave
+        if ANYWIN:
+            bfree = ctypes.c_ulonglong(0)
+            ctypes.windll.kernel32.GetDiskFreeSpaceExW(  # type: ignore
+                ctypes.c_wchar_p(abspath), None, None, ctypes.pointer(bfree)
+            )
+            return (bfree.value, None)
+        else:
+            sv = os.statvfs(fsenc(abspath))
+            free = sv.f_frsize * sv.f_bfree
+            total = sv.f_frsize * sv.f_blocks
+            return (free, total)
+    except:
+        return (None, None)
+
+
 def read_socket(sr: Unrecv, total_size: int) -> Generator[bytes, None, None]:
     remains = total_size
     while remains > 0:
@@ -1230,7 +1466,7 @@ def read_socket_unbounded(sr: Unrecv) -> Generator[bytes, None, None]:
 
 
 def read_socket_chunked(
-    sr: Unrecv, log: Optional[NamedLogger] = None
+    sr: Unrecv, log: Optional["NamedLogger"] = None
 ) -> Generator[bytes, None, None]:
     err = "upload aborted: expected chunk length, got [{}] |{}| instead"
     while True:
@@ -1308,7 +1544,7 @@ def hashcopy(
 
 
 def sendfile_py(
-    log: NamedLogger,
+    log: "NamedLogger",
     lower: int,
     upper: int,
     f: typing.BinaryIO,
@@ -1336,7 +1572,7 @@ def sendfile_py(
 
 
 def sendfile_kern(
-    log: NamedLogger,
+    log: "NamedLogger",
     lower: int,
     upper: int,
     f: typing.BinaryIO,
@@ -1377,7 +1613,7 @@ def sendfile_kern(
 
 
 def statdir(
-    logger: Optional[RootLogger], scandir: bool, lstat: bool, top: str
+    logger: Optional["RootLogger"], scandir: bool, lstat: bool, top: str
 ) -> Generator[tuple[str, os.stat_result], None, None]:
     if lstat and ANYWIN:
         lstat = False
@@ -1420,9 +1656,10 @@ def statdir(
 
 
 def rmdirs(
-    logger: RootLogger, scandir: bool, lstat: bool, top: str, depth: int
+    logger: "RootLogger", scandir: bool, lstat: bool, top: str, depth: int
 ) -> tuple[list[str], list[str]]:
-    if not os.path.exists(fsenc(top)) or not os.path.isdir(fsenc(top)):
+    """rmdir all descendants, then self"""
+    if not os.path.isdir(fsenc(top)):
         top = os.path.dirname(top)
         depth -= 1
 
@@ -1446,6 +1683,21 @@ def rmdirs(
     return ok, ng
 
 
+def rmdirs_up(top: str) -> tuple[list[str], list[str]]:
+    """rmdir on self, then all parents"""
+    try:
+        os.rmdir(fsenc(top))
+    except:
+        return [], [top]
+
+    par = os.path.dirname(top)
+    if not par:
+        return [top], []
+
+    ok, ng = rmdirs_up(par)
+    return [top] + ok, ng
+
+
 def unescape_cookie(orig: str) -> str:
     # mw=idk; doot=qwe%2Crty%3Basd+fgh%2Bjkl%25zxc%26vbn  # qwe,rty;asd fgh+jkl%zxc&vbn
     ret = ""
@@ -1566,12 +1818,17 @@ def runcmd(
     argv: Union[list[bytes], list[str]], timeout: Optional[int] = None, **ka: Any
 ) -> tuple[int, str, str]:
     kill = ka.pop("kill", "t")  # [t]ree [m]ain [n]one
+
+    sin = ka.pop("sin", None)
+    if sin:
+        ka["stdin"] = sp.PIPE
+
     p = sp.Popen(argv, stdout=sp.PIPE, stderr=sp.PIPE, **ka)
     if not timeout or PY2:
-        stdout, stderr = p.communicate()
+        stdout, stderr = p.communicate(sin)
     else:
         try:
-            stdout, stderr = p.communicate(timeout=timeout)
+            stdout, stderr = p.communicate(sin, timeout=timeout)
         except sp.TimeoutExpired:
             if kill == "n":
                 return -18, "", ""  # SIGCONT; leave it be
@@ -1620,7 +1877,7 @@ def retchk(
     rc: int,
     cmd: Union[list[bytes], list[str]],
     serr: str,
-    logger: Optional[NamedLogger] = None,
+    logger: Optional["NamedLogger"] = None,
     color: Union[int, str] = 0,
     verbose: bool = False,
 ) -> None:
@@ -1672,29 +1929,6 @@ def gzip_orig_sz(fn: str) -> int:
         return sunpack(b"I", rv)[0]  # type: ignore
 
 
-def py_desc() -> str:
-    interp = platform.python_implementation()
-    py_ver = ".".join([str(x) for x in sys.version_info])
-    ofs = py_ver.find(".final.")
-    if ofs > 0:
-        py_ver = py_ver[:ofs]
-
-    try:
-        bitness = struct.calcsize(b"P") * 8
-    except:
-        bitness = struct.calcsize("P") * 8
-
-    host_os = platform.system()
-    compiler = platform.python_compiler()
-
-    m = re.search(r"([0-9]+\.[0-9\.]+)", platform.version())
-    os_ver = m.group(1) if m else ""
-
-    return "{:>9} v{} on {}{} {} [{}]".format(
-        interp, py_ver, host_os, bitness, os_ver, compiler
-    )
-
-
 def align_tab(lines: list[str]) -> list[str]:
     rows = []
     ncols = 0
@@ -1797,7 +2031,6 @@ def termsize() -> tuple[int, int]:
     def ioctl_GWINSZ(fd: int) -> Optional[tuple[int, int]]:
         try:
             import fcntl
-            import struct
             import termios
 
             cr = struct.unpack("hh", fcntl.ioctl(fd, termios.TIOCGWINSZ, b"1234"))

copyparty/web/baguettebox.js

@@ -224,12 +224,13 @@ window.baguetteBox = (function () {
             ['space, P, K', 'video: play / pause'],
             ['U', 'video: seek 10sec back'],
             ['P', 'video: seek 10sec ahead'],
+            ['0..9', 'video: seek 0%..90%'],
             ['M', 'video: toggle mute'],
             ['V', 'video: toggle loop'],
             ['C', 'video: toggle auto-next'],
             ['<code>[</code>, <code>]</code>', 'video: loop start / end'],
         ],
-            d = mknod('table'),
+            d = mknod('table', 'bbox-halp'),
             html = ['<tbody>'];
 
         for (var a = 0; a < list.length; a++)
@@ -238,7 +239,6 @@ window.baguetteBox = (function () {
         html.push('<tr><td colspan="2">tap middle of img to hide btns</td></tr>');
         html.push('<tr><td colspan="2">tap left/right sides for prev/next</td></tr>');
         d.innerHTML = html.join('\n') + '</tbody>';
-        d.setAttribute('id', 'bbox-halp');
         d.onclick = function () {
             overlay.removeChild(d);
         };
@@ -249,7 +249,7 @@ window.baguetteBox = (function () {
         if (e.ctrlKey || e.altKey || e.metaKey || e.isComposing || modal.busy)
             return;
 
-        var k = e.code + '', v = vid();
+        var k = e.code + '', v = vid(), pos = -1;
 
         if (k == "ArrowLeft" || k == "KeyJ")
             showPreviousImage();
@@ -265,6 +265,8 @@ window.baguetteBox = (function () {
             playpause();
         else if (k == "KeyU" || k == "KeyO")
             relseek(k == "KeyU" ? -10 : 10);
+        else if (k.indexOf('Digit') === 0)
+            vid().currentTime = vid().duration * parseInt(k.slice(-1)) * 0.1;
         else if (k == "KeyM" && v) {
             v.muted = vmute = !vmute;
             mp_ctl();
@@ -487,9 +489,8 @@ window.baguetteBox = (function () {
         var imagesFiguresIds = [];
         var imagesCaptionsIds = [];
         for (var i = 0, fullImage; i < gallery.length; i++) {
-            fullImage = mknod('div');
+            fullImage = mknod('div', 'baguette-img-' + i);
             fullImage.className = 'full-image';
-            fullImage.id = 'baguette-img-' + i;
             imagesElements.push(fullImage);
 
             imagesFiguresIds.push('bbox-figure-' + i);
@@ -631,16 +632,14 @@ window.baguetteBox = (function () {
         if (is_vid && index != currentIndex)
             return;  // no preload
 
-        var figure = mknod('figure');
-        figure.id = 'bbox-figure-' + index;
+        var figure = mknod('figure', 'bbox-figure-' + index);
         figure.innerHTML = '<div class="bbox-spinner">' +
             '<div class="bbox-double-bounce1"></div>' +
             '<div class="bbox-double-bounce2"></div>' +
             '</div>';
 
         if (options.captions && imageCaption) {
-            var figcaption = mknod('figcaption');
-            figcaption.id = 'bbox-figcaption-' + index;
+            var figcaption = mknod('figcaption', 'bbox-figcaption-' + index);
             figcaption.innerHTML = imageCaption;
             figure.appendChild(figcaption);
         }
@@ -700,18 +699,12 @@ window.baguetteBox = (function () {
             showOverlay(index);
             return true;
         }
-        if (index < 0) {
-            if (options.animation)
-                bounceAnimation('left');
 
-            return false;
-        }
-        if (index >= imagesElements.length) {
-            if (options.animation)
-                bounceAnimation('right');
+        if (index < 0)
+            return bounceAnimation('left');
 
-            return false;
-        }
+        if (index >= imagesElements.length)
+            return bounceAnimation('right');
 
         var v = vid();
         if (v) {
@@ -894,10 +887,11 @@ window.baguetteBox = (function () {
     }
 
     function bounceAnimation(direction) {
-        slider.className = 'bounce-from-' + direction;
+        slider.className = options.animation == 'slideIn' ? 'bounce-from-' + direction : 'eog';
         setTimeout(function () {
             slider.className = '';
-        }, 400);
+        }, 300);
+        return false;
     }
 
     function updateOffset() {
@@ -943,7 +937,8 @@ window.baguetteBox = (function () {
         else
             timer.rm(rotn);
 
-        el.onclick = function (e) {
+        var ctime = 0;
+        el.onclick = v ? null : function (e) {
             var rc = e.target.getBoundingClientRect(),
                 x = e.clientX - rc.left,
                 fx = x / (rc.right - rc.left);
@@ -957,6 +952,11 @@ window.baguetteBox = (function () {
             clmod(ebi('bbox-btns'), 'off', 't');
             clmod(btnPrev, 'off', 't');
             clmod(btnNext, 'off', 't');
+
+            if (Date.now() - ctime <= 500)
+                tglfull();
+
+            ctime = Date.now();
         };
 
         var prev = QS('.full-image.vis');

copyparty/web/browser.css

@@ -259,7 +259,7 @@ html.bz {
 	--bg-d2: #34384e;
 	--bg-d3: #34384e;
 
-	--row-alt: rgba(139, 150, 205, 0.06);
+	--row-alt: #181a27;
 
 	--btn-bg: #202231;
 	--btn-h-bg: #2d2f45;
@@ -309,7 +309,7 @@ html.c {
 	--a-gray: #0ae;
 
 	--tab-alt: #6ef;
-	--row-alt: rgba(180,0,255,0.3);
+	--row-alt: #47237d;
 	--scroll: #ff0;
 
 	--btn-fg: #fff;
@@ -504,7 +504,7 @@ html.dy {
 	--a: #000;
 	--a-b: #000;
 	--a-hil: #000;
-	--a-gray: #000;
+	--a-gray: #bbb;
 	--a-dark: #000;
 
 	--btn-fg: #000;
@@ -527,6 +527,11 @@ html.dy {
 	--u2-tab-1-bg: a;
 	--u2-b1-bg: #000;
 	--u2-b2-bg: #000;
+	--u2-o-h-bg: #999;
+	--u2-o-1h-bg: #999;
+	--u2-o-bg: #eee;
+	--u2-o-1-bg: #000;
+
 	--ud-b1: a;
 
 	--sort-1: a;
@@ -539,6 +544,9 @@ html.dy {
 
 	--tree-bg: #fff;
 
+	--g-sel-bg: #000;
+	--g-fsel-bg: #444;
+	--g-fsel-ts: #000;
 	--g-fg: a;
 	--g-bg: a;
 	--g-b1: a;
@@ -702,6 +710,7 @@ html.y #files thead th {
 #files td {
 	margin: 0;
 	padding: .3em .5em;
+	background: var(--bg);
 }
 #files tr:nth-child(2n) td {
 	background: var(--row-alt);
@@ -1590,9 +1599,6 @@ html.y #tree.nowrap .ntree a+a:hover {
 	margin: .7em 0 .7em .5em;
 	padding-left: .5em;
 }
-.opwide>div.fill {
-	display: block;
-}
 .opwide>div>div>a {
 	line-height: 2em;
 }
@@ -1903,10 +1909,13 @@ html.y #bbox-overlay figcaption a {
 	transition: left .2s ease, transform .2s ease;
 }
 .bounce-from-right {
-	animation: bounceFromRight .4s ease-out;
+	animation: bounceFromRight .3s ease-out;
 }
 .bounce-from-left {
-	animation: bounceFromLeft .4s ease-out;
+	animation: bounceFromLeft .3s ease-out;
+}
+.eog {
+	animation: eog .2s;
 }
 @keyframes bounceFromRight {
 	0% {margin-left: 0}
@@ -1918,6 +1927,9 @@ html.y #bbox-overlay figcaption a {
 	50% {margin-left: 30px}
 	100% {margin-left: 0}
 }
+@keyframes eog {
+	0% {filter: brightness(1.5)}
+}
 #bbox-next,
 #bbox-prev {
 	top: 50%;
@@ -2216,6 +2228,10 @@ html.y #bbox-overlay figcaption a {
 #u2notbtn * {
 	line-height: 1.3em;
 }
+#u2mu div {
+	height: 1.2em;
+	overflow: hidden;
+}
 #u2tabw {
 	min-height: 0;
 	transition: min-height .2s;
@@ -2225,6 +2241,7 @@ html.y #bbox-overlay figcaption a {
 	display: none;
 }
 #u2tab {
+	table-layout: fixed;
 	border-collapse: collapse;
 	width: calc(100% - 2em);
 	max-width: 100em;
@@ -2234,6 +2251,7 @@ html.y #bbox-overlay figcaption a {
 	max-width: none;
 }
 #u2tab td {
+	word-wrap: break-word;
 	border: 1px solid rgba(128,128,128,0.8);
 	border-width: 0 0px 1px 0;
 	padding: .2em .3em;
@@ -2248,7 +2266,19 @@ html.y #bbox-overlay figcaption a {
 #u2tab.up.ok td:nth-child(3),
 #u2tab.up.bz td:nth-child(3),
 #u2tab.up.q  td:nth-child(3) {
-	width: 19em;
+	width: 18em;
+}
+@media (max-width: 65em) {
+	#u2tab {
+		font-size: .9em;
+	}
+}
+@media (max-width: 50em) {
+	#u2tab.up.ok td:nth-child(3),
+	#u2tab.up.bz td:nth-child(3),
+	#u2tab.up.q  td:nth-child(3) {
+		width: 16em;
+	}
 }
 #op_up2k.srch td.prog {
 	font-family: sans-serif;
@@ -2348,7 +2378,7 @@ html.y #bbox-overlay figcaption a {
 	width: 48em;
 }
 #u2conf.ww {
-	width: 74em;
+	width: 78em;
 }
 #u2conf.ww #u2c3w {
 	width: 29em;
@@ -2449,11 +2479,11 @@ html.b #u2conf a.b:hover {
 	color: var(--fg-max);
 	font-style: italic;
 	text-align: center;
-	font-size: .9em;
-	margin: 1em 0;
+	font-size: 1.2em;
+	margin: .8em 0;
 }
 #u2foot .warn {
-	font-size: 1.3em;
+	font-size: 1.2em;
 	padding: .5em .8em;
 	margin: 1em -.6em;
 	border-width: .1em 0;
@@ -2467,6 +2497,9 @@ html.b #u2conf a.b:hover {
 	font-size: .9em;
 	font-weight: normal;
 }
+#u2foot>*+* {
+	margin-top: 1.5em;
+}
 .prog {
 	font-family: 'scp', monospace, monospace;
 }

copyparty/web/browser.html

@@ -148,7 +148,8 @@
 			have_del = {{ have_del|tojson }},
 			have_unpost = {{ have_unpost|tojson }},
 			have_zip = {{ have_zip|tojson }},
-			turbolvl = {{ turbolvl|tojson }},
+			turbolvl = {{ turbolvl }},
+			u2sort = "{{ u2sort }}",
 			have_emp = {{ have_emp|tojson }},
 			txt_ext = "{{ txt_ext }}",
 			{% if no_prism %}no_prism = 1,{% endif %}

copyparty/web/browser.js

@@ -11,6 +11,7 @@ var Ls = {
 			"q": "quality / bitrate",
 			"Ac": "audio codec",
 			"Vc": "video codec",
+			"Fmt": "format / container",
 			"Ahash": "audio checksum",
 			"Vhash": "video checksum",
 			"Res": "resolution",
@@ -63,6 +64,7 @@ var Ls = {
 		"ul_par": "parallel uploads:",
 		"ut_mt": "continue hashing other files while uploading$N$Nmaybe disable if your CPU or HDD is a bottleneck",
 		"ut_ask": "ask for confirmation before upload starts",
+		"ut_pot": "improve upload speed on slow devices$Nby making the UI less complex",
 		"ut_srch": "don't actually upload, instead check if the files already $N exist on the server (will scan all folders you can read)",
 		"ut_par": "pause uploads by setting it to 0$N$Nincrease if your connection is slow / high latency$N$Nkeep it 1 on LAN or if the server HDD is a bottleneck",
 		"ul_btn": "drop files / folders<br>here (or click me)",
@@ -105,6 +107,7 @@ var Ls = {
 
 		"ct_thumb": "in icon view, toggle icons or thumbnails$NHotkey: T",
 		"ct_dots": "show hidden files (if server permits)",
+		"ct_dir1st": "sort folders before files",
 		"ct_readme": "show README.md in folder listings",
 
 		"cut_turbo": "the yolo button, you probably DO NOT want to enable this:$N$Nuse this if you were uploading a huge amount of files and had to restart for some reason, and want to continue the upload ASAP$N$Nthis replaces the hash-check with a simple <em>&quot;does this have the same filesize on the server?&quot;</em> so if the file contents are different it will NOT be uploaded$N$Nyou should turn this off when the upload is done, and then &quot;upload&quot; the same files again to let the client verify them",
@@ -113,6 +116,10 @@ var Ls = {
 
 		"cut_flag": "ensure only one tab is uploading at a time $N -- other tabs must have this enabled too $N -- only affects tabs on the same domain",
 
+		"cut_az": "upload files in alphabetical order, rather than smallest-file-first$N$Nalphabetical order can make it easier to eyeball if something went wrong on the server, but it makes uploading slightly slower on fiber / LAN",
+
+		"cut_mt": "use multithreading to accelerate file hashing$N$Nthis uses web-workers and requires$Nmore RAM (up to 512 MiB extra)$N$N30% faster https, 4.5x faster http,$Nand 5.3x faster on android phones",
+
 		"cft_text": "favicon text (blank and refresh to disable)",
 		"cft_fg": "foreground color",
 		"cft_bg": "background color",
@@ -143,7 +150,7 @@ var Ls = {
 		"mt_caac": "convert aac / m4a to opus\">aac",
 		"mt_coth": "convert all others (not mp3) to opus\">oth",
 		"mt_tint": "background level (0-100) on the seekbar$Nto make buffering less distracting",
-		"mt_eq": "enables the equalizer and gain control;$Nboost 0 = unmodified 100% volume$N$Nenabling the equalizer makes gapless albums fully gapless, so leave it on with all the values at zero if you care about that",
+		"mt_eq": "enables the equalizer and gain control;$N$Nboost &lt;code&gt;0&lt;/code&gt; = standard 100% volume (unmodified)$N$Nwidth &lt;code&gt;1 &nbsp;&lt;/code&gt; = standard stereo (unmodified)$Nwidth &lt;code&gt;0.5&lt;/code&gt; = 50% left-right crossfeed$Nwidth &lt;code&gt;0 &nbsp;&lt;/code&gt; = mono$N$Nboost &lt;code&gt;-0.8&lt;/code&gt; &amp; width &lt;code&gt;10&lt;/code&gt; = vocal removal :^)$N$Nenabling the equalizer makes gapless albums fully gapless, so leave it on with all the values at zero (except width = 1) if you care about that",
 
 		"mb_play": "play",
 		"mm_hashplay": "play this audio file?",
@@ -283,8 +290,12 @@ var Ls = {
 
 		"u_https1": "you should",
 		"u_https2": "switch to https",
-		"u_https3": "for much better performance",
+		"u_https3": "for better performance",
 		"u_ancient": 'your browser is impressively ancient -- maybe you should <a href="#" onclick="goto(\'bup\')">use bup instead</a>',
+		"u_nowork": "need firefox 53+ or chrome 57+ or iOS 11+",
+		"u_enpot": 'switch to <a href="#">potato UI</a> (may improve upload speed)',
+		"u_depot": 'switch to <a href="#">fancy UI</a> (may reduce upload speed)',
+		"u_gotpot": 'switching to the potato UI for improved upload speed,\n\nfeel free to disagree and switch back!',
 		"u_ever": "this is the basic uploader; up2k needs at least<br>chrome 21 // firefox 13 // edge 12 // opera 12 // safari 5.1",
 		"u_su2k": 'this is the basic uploader; <a href="#" id="u2yea">up2k</a> is better',
 		"u_ewrite": 'you do not have write-access to this folder',
@@ -302,9 +313,11 @@ var Ls = {
 		"u_upping": 'uploading',
 		"u_cuerr": "failed to upload chunk {0} of {1};\nprobably harmless, continuing\n\nfile: {2}",
 		"u_cuerr2": "server rejected upload (chunk {0} of {1});\n\nfile: {2}\n\nerror ",
+		"u_ehstmp": "will retry; see bottom-right",
 		"u_ehsfin": "server rejected the request to finalize upload",
 		"u_ehssrch": "server rejected the request to perform search",
 		"u_ehsinit": "server rejected the request to initiate upload",
+		"u_ehsdf": "server ran out of disk space!\n\nwill keep retrying, in case someone\nfrees up enough space to continue",
 		"u_s404": "not found on server",
 		"u_expl": "explain",
 		"u_tu": '<p class="warn">WARNING: turbo enabled, <span>&nbsp;client may not detect and resume incomplete uploads; see turbo-button tooltip</span></p>',
@@ -336,6 +349,7 @@ var Ls = {
 			"q": "kvalitet / bitrate",
 			"Ac": "lyd-format",
 			"Vc": "video-format",
+			"Fmt": "format / innpakning",
 			"Ahash": "lyd-kontrollsum",
 			"Vhash": "video-kontrollsum",
 			"Res": "oppløsning",
@@ -388,6 +402,7 @@ var Ls = {
 		"ul_par": "samtidige handl.:",
 		"ut_mt": "fortsett å befare køen mens opplastning foregår$N$Nskru denne av dersom du har en$Ntreg prosessor eller harddisk",
 		"ut_ask": "bekreft filutvalg før opplastning starter",
+		"ut_pot": "forbedre ytelsen på trege enheter ved å$Nforenkle brukergrensesnittet",
 		"ut_srch": "utfør søk istedenfor å laste opp --$Nleter igjennom alle mappene du har lov til å se",
 		"ut_par": "sett til 0 for å midlertidig stanse opplastning$N$Nhøye verdier (4 eller 8) kan gi bedre ytelse,$Nspesielt på trege internettlinjer$N$Nbør ikke være høyere enn 1 på LAN$Neller hvis serveren sin harddisk er treg",
 		"ul_btn": "slipp filer / mapper<br>her (eller klikk meg)",
@@ -430,6 +445,7 @@ var Ls = {
 
 		"ct_thumb": "vis miniatyrbilder istedenfor ikoner$NSnarvei: T",
 		"ct_dots": "vis skjulte filer (gitt at serveren tillater det)",
+		"ct_dir1st": "sorter slik at mapper kommer foran filer",
 		"ct_readme": "vis README.md nedenfor filene",
 
 		"cut_turbo": "forenklet befaring ved opplastning; bør sannsynlig <em>ikke</em> skrus på:$N$Nnyttig dersom du var midt i en svær opplastning som måtte restartes av en eller annen grunn, og du vil komme igang igjen så raskt som overhodet mulig.$N$Nnår denne er skrudd på så forenkles befaringen kraftig; istedenfor å utføre en trygg sjekk på om filene finnes på serveren i god stand, så sjekkes kun om <em>filstørrelsen</em> stemmer. Så dersom en korrupt fil skulle befinne seg på serveren allerede, på samme sted med samme størrelse og navn, så blir det <em>ikke oppdaget</em>.$N$Ndet anbefales å kun benytte denne funksjonen for å komme seg raskt igjennom selve opplastningen, for så å skru den av, og til slutt &quot;laste opp&quot; de samme filene én gang til -- slik at integriteten kan verifiseres",
@@ -438,6 +454,10 @@ var Ls = {
 
 		"cut_flag": "samkjører nettleserfaner slik at bare én $N kan holde på med befaring / opplastning $N -- andre faner må også ha denne skrudd på $N -- fungerer kun innenfor samme domene",
 
+		"cut_az": "last opp filer i alfabetisk rekkefølge, istedenfor minste-fil-først$N$Nalfabetisk kan gjøre det lettere å anslå om alt gikk bra, men er bittelitt tregere på fiber / LAN",
+
+		"cut_mt": "raskere befaring ved å bruke hele CPU'en$N$Ndenne funksjonen anvender web-workers$Nog krever mer RAM (opptil 512 MiB ekstra)$N$N30% raskere https, 4.5x raskere http,$Nog 5.3x raskere på android-telefoner",
+
 		"cft_text": "ikontekst (blank ut og last siden på nytt for å deaktivere)",
 		"cft_fg": "farge",
 		"cft_bg": "bakgrunnsfarge",
@@ -468,7 +488,7 @@ var Ls = {
 		"mt_caac": "konverter aac / m4a-filer til to opus\">aac",
 		"mt_coth": "konverter alt annet (men ikke mp3) til opus\">andre",
 		"mt_tint": "nivå av bakgrunnsfarge på søkestripa (0-100),$Ngjør oppdateringer mindre distraherende",
-		"mt_eq": "aktiver tonekontroll og forsterker;$Nboost 0 = normal volumskala$N$Nreduserer også dødtid imellom sangfiler",
+		"mt_eq": "aktiver tonekontroll og forsterker;$N$Nboost &lt;code&gt;0&lt;/code&gt; = normal volumskala$N$Nwidth &lt;code&gt;1 &nbsp;&lt;/code&gt; = normal stereo$Nwidth &lt;code&gt;0.5&lt;/code&gt; = 50% blanding venstre-høyre$Nwidth &lt;code&gt;0 &nbsp;&lt;/code&gt; = mono$N$Nboost &lt;code&gt;-0.8&lt;/code&gt; &amp; width &lt;code&gt;10&lt;/code&gt; = instrumental :^)$N$Nreduserer også dødtid imellom sangfiler",
 
 		"mb_play": "lytt",
 		"mm_hashplay": "spill denne sangen?",
@@ -608,8 +628,12 @@ var Ls = {
 
 		"u_https1": "du burde",
 		"u_https2": "bytte til https",
-		"u_https3": "for mye høyere hastighet",
+		"u_https3": "for høyere hastighet",
 		"u_ancient": 'nettleseren din er prehistorisk -- mulig du burde <a href="#" onclick="goto(\'bup\')">bruke bup istedenfor</a>',
+		"u_nowork": "krever firefox 53+, chrome 57+, eller iOS 11+",
+		"u_enpot": 'bytt til <a href="#">enkelt UI</a> (gir sannsynlig raskere opplastning)',
+		"u_depot": 'bytt til <a href="#">snæsent UI</a> (gir sannsynlig tregere opplastning)',
+		"u_gotpot": 'byttet til et enklere UI for å laste opp raskere,\n\ndu kan gjerne bytte tilbake altså!',
 		"u_ever": "dette er den primitive opplasteren; up2k krever minst:<br>chrome 21 // firefox 13 // edge 12 // opera 12 // safari 5.1",
 		"u_su2k": 'dette er den primitive opplasteren; <a href="#" id="u2yea">up2k</a> er bedre',
 		"u_ewrite": 'du har ikke skrivetilgang i denne mappen',
@@ -627,9 +651,11 @@ var Ls = {
 		"u_upping": 'sender',
 		"u_cuerr": "kunne ikke laste opp del {0} av {1};\nsikkert harmløst, fortsetter\n\nfil: {2}",
 		"u_cuerr2": "server nektet opplastningen (del {0} av {1});\n\nfile: {2}\n\nerror ",
+		"u_ehstmp": "prøver igjen; se mld nederst",
 		"u_ehsfin": "server nektet forespørselen om å ferdigstille filen",
 		"u_ehssrch": "server nektet forespørselen om å utføre søk",
 		"u_ehsinit": "server nektet forespørselen om å begynne en ny opplastning",
+		"u_ehsdf": "serveren er full!\n\nprøver igjen regelmessig,\ni tilfelle noen rydder litt...",
 		"u_s404": "ikke funnet på serveren",
 		"u_expl": "forklar",
 		"u_tu": '<p class="warn">ADVARSEL: turbo er på, <span>&nbsp;avbrutte opplastninger vil muligens ikke oppdages og gjenopptas; hold musepekeren over turbo-knappen for mer info</span></p>',
@@ -717,6 +743,10 @@ ebi('op_up2k').innerHTML = (
 	'			<label for="multitask" tt="' + L.ut_mt + '">🏃</label>\n' +
 	'		</td>\n' +
 	'		<td class="c" rowspan="2">\n' +
+	'			<input type="checkbox" id="potato" />\n' +
+	'			<label for="potato" tt="' + L.ut_pot + '">🥔</label>\n' +
+	'		</td>\n' +
+	'		<td class="c" rowspan="2">\n' +
 	'			<input type="checkbox" id="ask_up" />\n' +
 	'			<label for="ask_up" tt="' + L.ut_ask + '">💭</label>\n' +
 	'		</td>\n' +
@@ -772,10 +802,10 @@ ebi('op_up2k').innerHTML = (
 	'		</tr>\n' +
 	'	</thead>\n' +
 	'	<tbody></tbody>\n' +
-	'</table></div>\n' +
+	'</table><div id="u2mu"></div></div>\n' +
 
 	'<p id="u2flagblock"><b>' + L.ul_flagblk + '</p>\n' +
-	'<p id="u2foot"></p>'
+	'<div id="u2foot"></div>'
 );
 
 
@@ -802,6 +832,7 @@ ebi('op_cfg').innerHTML = (
 	'		<a id="griden" class="tgl btn" href="#" tt="' + L.wt_grid + '">田 the grid</a>\n' +
 	'		<a id="thumbs" class="tgl btn" href="#" tt="' + L.ct_thumb + '">🖼️ thumbs</a>\n' +
 	'		<a id="dotfiles" class="tgl btn" href="#" tt="' + L.ct_dots + '">dotfiles</a>\n' +
+	'		<a id="dir1st" class="tgl btn" href="#" tt="' + L.ct_dir1st + '">📁 first</a>\n' +
 	'		<a id="ireadme" class="tgl btn" href="#" tt="' + L.ct_readme + '">📜 readme</a>\n' +
 	'	</div>\n' +
 	'</div>\n' +
@@ -821,9 +852,11 @@ ebi('op_cfg').innerHTML = (
 	'<div>\n' +
 	'	<h3>' + L.cl_uopts + '</h3>\n' +
 	'	<div>\n' +
+	'		<a id="hashw" class="tgl btn" href="#" tt="' + L.cut_mt + '">mt</a>\n' +
 	'		<a id="u2turbo" class="tgl btn ttb" href="#" tt="' + L.cut_turbo + '">turbo</a>\n' +
 	'		<a id="u2tdate" class="tgl btn ttb" href="#" tt="' + L.cut_datechk + '">date-chk</a>\n' +
 	'		<a id="flag_en" class="tgl btn" href="#" tt="' + L.cut_flag + '">💤</a>\n' +
+	'		<a id="u2sort" class="tgl btn" href="#" tt="' + L.cut_az + '">az</a>\n' +
 	'		</td>\n' +
 	'	</div>\n' +
 	'</div>\n' +
@@ -837,7 +870,7 @@ ebi('op_cfg').innerHTML = (
 	'	</div>\n' +
 	'</div>\n' +
 	'<div><h3>' + L.cl_keytype + '</h3><div id="key_notation"></div></div>\n' +
-	'<div class="fill"><h3>' + L.cl_hiddenc + ' <a href="#" id="hcolsr">' + L.cl_reset + '</h3><div id="hcols"></div></div>'
+	'<div><h3>' + L.cl_hiddenc + ' <a href="#" id="hcolsr">' + L.cl_reset + '</h3><div id="hcols"></div></div>'
 );
 
 
@@ -890,7 +923,7 @@ function opclick(e) {
 	goto(dest);
 
 	var input = QS('.opview.act input:not([type="hidden"])')
-	if (input && !is_touch) {
+	if (input && !TOUCH) {
 		tt.skip = true;
 		input.focus();
 	}
@@ -1660,7 +1693,7 @@ var vbar = (function () {
 		if (e.button === 0)
 			can.onmousemove = null;
 	};
-	if (is_touch) {
+	if (TOUCH) {
 		can.ontouchstart = mousedown;
 		can.ontouchmove = mousemove;
 	}
@@ -1765,7 +1798,7 @@ function playpause(e) {
 		seek_au_mul(x * 1.0 / rect.width);
 	};
 
-	if (!is_touch)
+	if (!TOUCH)
 		bar.onwheel = function (e) {
 			var dist = Math.sign(e.deltaY) * 10;
 			if (Math.abs(e.deltaY) < 30 && !e.deltaMode)
@@ -1805,7 +1838,7 @@ var mpui = (function () {
 		if (++nth > 69) {
 			// android-chrome breaks aspect ratio with unannounced viewport changes
 			nth = 0;
-			if (is_touch) {
+			if (MOBILE) {
 				nth = 1;
 				pbar.onresize();
 				vbar.onresize();
@@ -1873,6 +1906,7 @@ var audio_eq = (function () {
 		"gains": [4, 3, 2, 1, 0, 0, 1, 2, 3, 4],
 		"filters": [],
 		"amp": 0,
+		"chw": 1,
 		"last_au": null,
 		"acst": {}
 	};
@@ -1924,6 +1958,7 @@ var audio_eq = (function () {
 
 	try {
 		r.amp = fcfg_get('au_eq_amp', r.amp);
+		r.chw = fcfg_get('au_eq_chw', r.chw);
 		var gains = jread('au_eq_gain', r.gains);
 		if (r.gains.length == gains.length)
 			r.gains = gains;
@@ -1933,12 +1968,14 @@ var audio_eq = (function () {
 	r.draw = function () {
 		jwrite('au_eq_gain', r.gains);
 		swrite('au_eq_amp', r.amp);
+		swrite('au_eq_chw', r.chw);
 
 		var txt = QSA('input.eq_gain');
 		for (var a = 0; a < r.bands.length; a++)
 			txt[a].value = r.gains[a];
 
 		QS('input.eq_gain[band="amp"]').value = r.amp;
+		QS('input.eq_gain[band="chw"]').value = r.chw;
 	};
 
 	r.stop = function () {
@@ -2008,16 +2045,47 @@ var audio_eq = (function () {
 		for (var a = r.filters.length - 1; a >= 0; a--)
 			r.filters[a].connect(a > 0 ? r.filters[a - 1] : actx.destination);
 
+		if (Math.round(r.chw * 25) != 25) {
+			var split = actx.createChannelSplitter(2),
+				merge = actx.createChannelMerger(2),
+				lg1 = actx.createGain(),
+				lg2 = actx.createGain(),
+				rg1 = actx.createGain(),
+				rg2 = actx.createGain(),
+				vg1 = 1 - (1 - r.chw) / 2,
+				vg2 = 1 - vg1;
+
+			console.log('chw', vg1, vg2);
+
+			merge.connect(r.filters[r.filters.length - 1]);
+			lg1.gain.value = rg2.gain.value = vg1;
+			lg2.gain.value = rg1.gain.value = vg2;
+			lg1.connect(merge, 0, 0);
+			rg1.connect(merge, 0, 0);
+			lg2.connect(merge, 0, 1);
+			rg2.connect(merge, 0, 1);
+
+			split.connect(lg1, 0);
+			split.connect(lg2, 0);
+			split.connect(rg1, 1);
+			split.connect(rg2, 1);
+			r.filters.push(split);
+			mp.acs.channelCountMode = 'explicit';
+		}
+
 		mp.acs.connect(r.filters[r.filters.length - 1]);
 	}
 
 	function eq_step(e) {
 		ev(e);
-		var band = parseInt(this.getAttribute('band')),
+		var sb = this.getAttribute('band'),
+			band = parseInt(sb),
 			step = parseFloat(this.getAttribute('step'));
 
-		if (isNaN(band))
+		if (sb == 'amp')
 			r.amp = Math.round((r.amp + step * 0.2) * 100) / 100;
+		else if (sb == 'chw')
+			r.chw = Math.round((r.chw + step * 0.2) * 100) / 100;
 		else
 			r.gains[band] += step;
 
@@ -2027,15 +2095,18 @@ var audio_eq = (function () {
 	function adj_band(that, step) {
 		var err = false;
 		try {
-			var band = parseInt(that.getAttribute('band')),
+			var sb = that.getAttribute('band'),
+				band = parseInt(sb),
 				vs = that.value,
 				v = parseFloat(vs);
 
 			if (isNaN(v) || v + '' != vs)
 				throw new Error('inval band');
 
-			if (isNaN(band))
+			if (sb == 'amp')
 				r.amp = Math.round((v + step * 0.2) * 100) / 100;
+			else if (sb == 'chw')
+				r.chw = Math.round((v + step * 0.2) * 100) / 100;
 			else
 				r.gains[band] = v + step;
 
@@ -2072,6 +2143,7 @@ var audio_eq = (function () {
 		vs.push([a, hz, r.gains[a]]);
 	}
 	vs.push(["amp", "boost", r.amp]);
+	vs.push(["chw", "width", r.chw]);
 
 	for (var a = 0; a < vs.length; a++) {
 		var b = vs[a][0];
@@ -2194,6 +2266,12 @@ function play(tid, is_ev, seek) {
 	if (window.thegrid)
 		thegrid.loadsel();
 
+	try {
+		if (actx.state == 'suspended')
+			actx.resume();
+	}
+	catch (ex) { }
+
 	try {
 		mp.au.play();
 		if (mp.au.paused)
@@ -2400,12 +2478,11 @@ function eval_hash() {
 				if (a)
 					QS(treectl.hidden ? '#path a:nth-last-child(2)' : '#treeul a.hl').focus();
 				else
-					QS(thegrid.en ? '#ggrid a' : '#files tbody a').focus();
+					QS(thegrid.en ? '#ggrid a' : '#files tbody tr[tabindex]').focus();
 			};
 		})(a);
 
-	var d = mknod('div');
-	d.setAttribute('id', 'acc_info');
+	var d = mknod('div', 'acc_info');
 	document.body.insertBefore(d, ebi('ops'));
 })();
 
@@ -2414,7 +2491,8 @@ function sortfiles(nodes) {
 	if (!nodes.length)
 		return nodes;
 
-	var sopts = jread('fsort', [["href", 1, ""]]);
+	var sopts = jread('fsort', [["href", 1, ""]]),
+		dir1st = sread('dir1st') !== '0';
 
 	try {
 		var is_srch = false;
@@ -2445,14 +2523,10 @@ function sortfiles(nodes) {
 
 					if ((v + '').indexOf('<a ') === 0)
 						v = v.split('>')[1];
-					else if (name == "href" && v) {
-						if (v.split('?')[0].slice(-1) == '/')
-							v = '\t' + v;
-
+					else if (name == "href" && v)
 						v = uricom_dec(v)[0];
-					}
 
-					nodes[b]._sv = v;
+					nodes[b]._sv = v
 				}
 			}
 
@@ -2481,6 +2555,13 @@ function sortfiles(nodes) {
 			if (is_srch)
 				delete nodes[b].ext;
 		}
+		if (dir1st) {
+			var r1 = [], r2 = [];
+			for (var b = 0, bb = nodes.length; b < bb; b++)
+				(nodes[b].href.split('?')[0].slice(-1) == '/' ? r1 : r2).push(nodes[b]);
+
+			nodes = r1.concat(r2);
+		}
 	}
 	catch (ex) {
 		console.log("failed to apply sort config: " + ex);
@@ -2670,8 +2751,7 @@ var fileman = (function () {
 
 		var rui = ebi('rui');
 		if (!rui) {
-			rui = mknod('div');
-			rui.setAttribute('id', 'rui');
+			rui = mknod('div', 'rui');
 			document.body.appendChild(rui);
 		}
 
@@ -3154,10 +3234,9 @@ var showfile = (function () {
 			return;
 
 		qsr('#prism_css');
-		var el = mknod('link');
+		var el = mknod('link', 'prism_css');
 		el.rel = 'stylesheet';
 		el.href = '/.cpr/deps/prism' + (light ? '' : 'd') + '.css';
-		el.setAttribute('id', 'prism_css');
 		document.head.appendChild(el);
 	};
 
@@ -3271,8 +3350,7 @@ var showfile = (function () {
 			fun = function (el) { };
 
 		qsr('#doc');
-		var el = mknod('pre');
-		el.setAttribute('id', 'doc');
+		var el = mknod('pre', 'doc');
 		el.setAttribute('tabindex', '0');
 		clmod(ebi('wrap'), 'doc', !is_md);
 		if (is_md) {
@@ -3300,9 +3378,8 @@ var showfile = (function () {
 		hfun(get_evpath() + '?doc=' + url.split('/').pop());
 
 		qsr('#docname');
-		el = mknod('span');
+		el = mknod('span', 'docname');
 		el.textContent = tname;
-		el.setAttribute('id', 'docname');
 		ebi('path').appendChild(el);
 
 		r.updtree();
@@ -3437,9 +3514,8 @@ var showfile = (function () {
 
 var thegrid = (function () {
 	var lfiles = ebi('files'),
-		gfiles = mknod('div');
+		gfiles = mknod('div', 'gfiles');
 
-	gfiles.setAttribute('id', 'gfiles');
 	gfiles.style.display = 'none';
 	gfiles.innerHTML = (
 		'<div id="ghead" class="ghead">' +
@@ -3938,6 +4014,9 @@ document.onkeydown = function (e) {
 		}
 	}
 
+	if (k == 'Enter' && ae && (ae.onclick || ae.hasAttribute('tabIndex')))
+		return ev(e) && ae.click() || true;
+
 	if (aet && aet != 'a' && aet != 'tr' && aet != 'pre')
 		return;
 
@@ -4137,7 +4216,7 @@ document.onkeydown = function (e) {
 			clearTimeout(defer_timeout);
 			clearTimeout(search_timeout);
 			search_timeout = setTimeout(do_search,
-				v && v.length < (is_touch ? 4 : 3) ? 1000 : 500);
+				v && v.length < (MOBILE ? 4 : 3) ? 1000 : 500);
 		}
 	}
 
@@ -4376,6 +4455,9 @@ var treectl = (function () {
 	bcfg_bind(r, 'dots', 'dotfiles', false, function (v) {
 		r.goto(get_evpath());
 	});
+	bcfg_bind(r, 'dir1st', 'dir1st', true, function (v) {
+		treectl.gentab(get_evpath(), treectl.lsc);
+	});
 	setwrap(bcfg_bind(r, 'wtree', 'wraptree', true, setwrap));
 	setwrap(bcfg_bind(r, 'parpane', 'parpane', true, onscroll));
 	bcfg_bind(r, 'htree', 'hovertree', false, reload_tree);
@@ -4572,9 +4654,9 @@ var treectl = (function () {
 				return ta[a];
 	};
 
-	r.goto = function (url, push) {
+	r.goto = function (url, push, back) {
 		get_tree("", url, true);
-		r.reqls(url, push, true);
+		r.reqls(url, push, true, back);
 	};
 
 	function get_tree(top, dst, rst) {
@@ -4743,9 +4825,10 @@ var treectl = (function () {
 		thegrid.setvis(true);
 	}
 
-	r.reqls = function (url, hpush, no_tree) {
+	r.reqls = function (url, hpush, no_tree, back) {
 		var xhr = new XHR();
 		xhr.top = url;
+		xhr.back = back
 		xhr.hpush = hpush;
 		xhr.ts = Date.now();
 		xhr.open('GET', xhr.top + '?ls' + (r.dots ? '&dots' : ''), true);
@@ -4813,6 +4896,12 @@ var treectl = (function () {
 		if (res.readme)
 			show_readme(res.readme);
 
+		if (this.hpush && !this.back) {
+			var ofs = ebi('wrap').offsetTop;
+			if (document.documentElement.scrollTop > ofs)
+				document.documentElement.scrollTop = ofs;
+		}
+
 		wintitle();
 		var fun = r.ls_cb;
 		if (fun) {
@@ -4822,6 +4911,7 @@ var treectl = (function () {
 	}
 
 	r.gentab = function (top, res) {
+		r.lsc = res;
 		var nodes = res.dirs.concat(res.files),
 			html = mk_files_header(res.taglist),
 			seen = {};
@@ -4834,7 +4924,6 @@ var treectl = (function () {
 				bhref = tn.href.split('?')[0],
 				fname = uricom_dec(bhref)[0],
 				hname = esc(fname),
-				sortv = (bhref.slice(-1) == '/' ? '\t' : '') + hname,
 				id = 'f-' + ('00000000' + crc32(fname)).slice(-8),
 				lang = showfile.getlang(fname);
 
@@ -4849,8 +4938,8 @@ var treectl = (function () {
 				tn.lead = '<a href="?doc=' + tn.href + '" class="doc' + (lang ? ' bri' : '') +
 					'" hl="' + id + '" name="' + hname + '">-txt-</a>';
 
-			var ln = ['<tr><td>' + tn.lead + '</td><td sortv="' + sortv +
-				'"><a href="' + top + tn.href + '" id="' + id + '">' + hname + '</a>', tn.sz];
+			var ln = ['<tr><td>' + tn.lead + '</td><td><a href="' +
+				top + tn.href + '" id="' + id + '">' + hname + '</a>', tn.sz];
 
 			for (var b = 0; b < res.taglist.length; b++) {
 				var k = res.taglist[b],
@@ -4988,7 +5077,7 @@ var treectl = (function () {
 		if (url.search.indexOf('doc=') + 1 && hbase == cbase)
 			return showfile.show(hbase + showfile.sname(url.search), true);
 
-		r.goto(url.pathname);
+		r.goto(url.pathname, false, true);
 	};
 
 	hist_replace(get_evpath() + window.location.hash);

copyparty/web/dbg-audio.js

@@ -13,8 +13,7 @@ audio_eq.apply = function () {
 
     var can = ebi('fft_can');
     if (!can) {
-        can = mknod('canvas');
-        can.setAttribute('id', 'fft_can');
+        can = mknod('canvas', 'fft_can');
         can.style.cssText = 'position:absolute;left:0;bottom:5em;width:' + w + 'px;height:' + h + 'px;z-index:9001';
         document.body.appendChild(can);
         can.width = w;

copyparty/web/md.js

@@ -173,8 +173,7 @@ md_plug_err = function (ex, js) {
             o.textContent = lns[ln - 1];
         }
     }
-    var errbox = mknod('div');
-    errbox.setAttribute('id', 'md_errbox');
+    var errbox = mknod('div', 'md_errbox');
     errbox.style.cssText = 'position:absolute;top:0;left:0;padding:1em .5em;background:#2b2b2b;color:#fc5'
     errbox.textContent = msg;
     errbox.onclick = function () {

copyparty/web/md2.css

@@ -36,6 +36,11 @@
 	width: 55em;
 	width: min(55em, calc(100% - 2em));
 }
+#mtw.single.editor,
+#mw.single.editor {
+	width: calc(100% - 1em);
+	left: .5em;
+}
 
 
 #mp {

copyparty/web/md2.js

@@ -16,8 +16,7 @@ var dom_sbs = ebi('sbs');
 var dom_nsbs = ebi('nsbs');
 var dom_tbox = ebi('toolsbox');
 var dom_ref = (function () {
-    var d = mknod('div');
-    d.setAttribute('id', 'mtr');
+    var d = mknod('div', 'mtr');
     dom_swrap.appendChild(d);
     d = ebi('mtr');
     // hide behind the textarea (offsetTop is not computed if display:none)
@@ -509,6 +508,20 @@ function setsel(s) {
 }
 
 
+// cut/copy current line
+function md_cut(cut) {
+    var s = linebounds();
+    if (s.car != s.cdr)
+        return;
+
+    dom_src.setSelectionRange(s.n1, s.n2 + 1, 'forward');
+    setTimeout(function () {
+        var i = cut ? s.n1 : s.car;
+        dom_src.setSelectionRange(i, i, 'forward');
+    }, 1);
+}
+
+
 // indent/dedent
 function md_indent(dedent) {
     var s = getsel(),
@@ -955,6 +968,10 @@ var set_lno = (function () {
                 md_p_jump(dn);
                 return false;
             }
+            if (ev.code == "KeyX" || ev.code == "KeyC") {
+                md_cut(ev.code == "KeyX");
+                return true; //sic
+            }
         }
         else {
             if (ev.code == "Tab" || kc == 9) {

copyparty/web/splash.html

@@ -36,6 +36,7 @@
 				<tr><td>hash-q</td><td>{{ hashq }}</td></tr>
 				<tr><td>tag-q</td><td>{{ tagq }}</td></tr>
 				<tr><td>mtp-q</td><td>{{ mtpq }}</td></tr>
+				<tr><td>db-act</td><td id="u">{{ dbwt }}</td></tr>
 			</table>
 		</td><td>
 			<table class="vols">
@@ -50,8 +51,8 @@
 			</table>
 		</td></tr></table>
 		<div class="btns">
-			<a id="d" href="/?stack" tt="shows the state of all active threads">dump stack</a>
-			<a id="e" href="/?reload=cfg" tt="reload config files (accounts/volumes/volflags),$Nand rescan all e2ds volumes">reload cfg</a>
+			<a id="d" href="/?stack">dump stack</a>
+			<a id="e" href="/?reload=cfg">reload cfg</a>
 		</div>
 		{%- endif %}
 

copyparty/web/splash.js

@@ -23,6 +23,12 @@ var Ls = {
 		"r1": "gå hjem",
 		".s1": "kartlegg",
 		"t1": "handling",
+		"u2": "tid siden noen sist skrev til serveren$N( opplastning / navneendring / ... )$N$N17d = 17 dager$N1h23 = 1 time 23 minutter$N4m56 = 4 minuter 56 sekunder",
+	},
+	"eng": {
+		"d2": "shows the state of all active threads",
+		"e2": "reload config files (accounts/volumes/volflags),$Nand rescan all e2ds volumes",
+		"u2": "time since the last server write$N( upload / rename / ... )$N$N17d = 17 days$N1h23 = 1 hour 23 minutes$N4m56 = 4 minutes 56 seconds",
 	}
 },
 	d = Ls[sread("lang") || lang];
@@ -40,5 +46,10 @@ for (var k in (d || {})) {
 }
 
 tt.init();
-if (!ebi('c'))
-	QS('input[name="cppwd"]').focus();
+var o = QS('input[name="cppwd"]');
+if (!ebi('c') && o.offsetTop + o.offsetHeight < window.innerHeight)
+	o.focus();
+
+o = ebi('u');
+if (o && /[0-9]+$/.exec(o.innerHTML))
+	o.innerHTML = shumantime(o.innerHTML);

copyparty/web/util.js

@@ -7,19 +7,43 @@ if (!window['console'])
 
 
 var wah = '',
-    is_touch = 'ontouchstart' in window,
-    is_https = (window.location + '').indexOf('https:') === 0,
-    IPHONE = is_touch && /iPhone|iPad|iPod/i.test(navigator.userAgent),
+    HALFMAX = 8192 * 8192 * 8192 * 8192,
+    HTTPS = (window.location + '').indexOf('https:') === 0,
+    TOUCH = 'ontouchstart' in window,
+    MOBILE = TOUCH,
+    CHROME = !!window.chrome,
+    IPHONE = TOUCH && /iPhone|iPad|iPod/i.test(navigator.userAgent),
     WINDOWS = navigator.platform ? navigator.platform == 'Win32' : /Windows/.test(navigator.userAgent);
 
 
+try {
+    if (navigator.userAgentData.mobile)
+        MOBILE = true;
+
+    if (navigator.userAgentData.platform == 'Windows')
+        WINDOWS = true;
+
+    if (navigator.userAgentData.brands.some(function (d) { return d.brand == 'Chromium' }))
+        CHROME = true;
+}
+catch (ex) { }
+
+
 var ebi = document.getElementById.bind(document),
     QS = document.querySelector.bind(document),
     QSA = document.querySelectorAll.bind(document),
-    mknod = document.createElement.bind(document),
     XHR = XMLHttpRequest;
 
 
+function mknod(et, eid) {
+    var ret = document.createElement(et);
+    if (eid)
+        ret.id = eid;
+
+    return ret;
+}
+
+
 function qsr(sel) {
     var el = QS(sel);
     if (el)
@@ -160,8 +184,7 @@ function vis_exh(msg, url, lineNo, columnNo, error) {
     try {
         var exbox = ebi('exbox');
         if (!exbox) {
-            exbox = mknod('div');
-            exbox.setAttribute('id', 'exbox');
+            exbox = mknod('div', 'exbox');
             document.body.appendChild(exbox);
 
             var s = mknod('style');
@@ -222,6 +245,11 @@ function ev(e) {
 }
 
 
+function noope(e) {
+    ev(e);
+}
+
+
 // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/endsWith
 if (!String.prototype.endsWith)
     String.prototype.endsWith = function (search, this_len) {
@@ -447,6 +475,16 @@ function sortTable(table, col, cb) {
         }
         return reverse * (a.localeCompare(b));
     });
+    if (sread('dir1st') !== '0') {
+        var r1 = [], r2 = [];
+        for (var i = 0; i < tr.length; i++) {
+            var cell = tr[vl[i][1]].cells[1],
+                href = cell.getAttribute('sortv') || cell.textContent.trim();
+
+            (href.split('?')[0].slice(-1) == '/' ? r1 : r2).push(vl[i]);
+        }
+        vl = r1.concat(r2);
+    }
     for (i = 0; i < tr.length; ++i) tb.appendChild(tr[vl[i][1]]);
     if (cb) cb();
 }
@@ -630,7 +668,7 @@ function humansize(b, terse) {
 
 function humantime(v) {
     if (v >= 60 * 60 * 24)
-        return v;
+        return shumantime(v);
 
     try {
         return /.*(..:..:..).*/.exec(new Date(v * 1000).toUTCString())[1];
@@ -641,12 +679,39 @@ function humantime(v) {
 }
 
 
+function shumantime(v) {
+    if (v < 10)
+        return f2f(v, 2) + 's';
+    if (v < 60)
+        return f2f(v, 1) + 's';
+
+    v = parseInt(v);
+    var st = [[60 * 60 * 24, 60 * 60, 'd'], [60 * 60, 60, 'h'], [60, 1, 'm']];
+
+    for (var a = 0; a < st.length; a++) {
+        var m1 = st[a][0],
+            m2 = st[a][1],
+            ch = st[a][2];
+
+        if (v < m1)
+            continue;
+
+        var v1 = parseInt(v / m1),
+            v2 = ('0' + parseInt((v % m1) / m2)).slice(-2);
+
+        return v1 + ch + (v1 >= 10 ? '' : v2);
+    }
+}
+
+
 function clamp(v, a, b) {
     return Math.min(Math.max(v, a), b);
 }
 
 
 function has(haystack, needle) {
+    try { return haystack.includes(needle); } catch (ex) { }
+
     for (var a = 0; a < haystack.length; a++)
         if (haystack[a] == needle)
             return true;
@@ -866,8 +931,8 @@ var timer = (function () {
 
 var tt = (function () {
     var r = {
-        "tt": mknod("div"),
-        "th": mknod("div"),
+        "tt": mknod("div", 'tt'),
+        "th": mknod("div", 'tth'),
         "en": true,
         "el": null,
         "skip": false,
@@ -875,8 +940,6 @@ var tt = (function () {
     };
 
     r.th.innerHTML = '?';
-    r.tt.setAttribute('id', 'tt');
-    r.th.setAttribute('id', 'tth');
     document.body.appendChild(r.tt);
     document.body.appendChild(r.th);
 
@@ -898,7 +961,7 @@ var tt = (function () {
             return r.show.bind(this)();
 
         tev = setTimeout(r.show.bind(this), 800);
-        if (is_touch)
+        if (TOUCH)
             return;
 
         this.addEventListener('mousemove', r.move);
@@ -1045,9 +1108,8 @@ var toast = (function () {
     var r = {},
         te = null,
         scrolling = false,
-        obj = mknod('div');
+        obj = mknod('div', 'toast');
 
-    obj.setAttribute('id', 'toast');
     document.body.appendChild(obj);
     r.visible = false;
     r.txt = null;
@@ -1130,8 +1192,7 @@ var modal = (function () {
     r.busy = false;
 
     r.show = function (html) {
-        o = mknod('div');
-        o.setAttribute('id', 'modal');
+        o = mknod('div', 'modal');
         o.innerHTML = '<table><tr><td><div id="modalc">' + html + '</div></td></tr></table>';
         document.body.appendChild(o);
         document.addEventListener('keydown', onkey);
@@ -1186,7 +1247,8 @@ var modal = (function () {
             return;
 
         setTimeout(function () {
-            ebi('modal-ok').focus();
+            if (ctr = ebi('modal-ok'))
+                ctr.focus();
         }, 20);
         ev(e);
     }
@@ -1486,19 +1548,18 @@ function xhrchk(xhr, prefix, e404) {
     var errtxt = (xhr.response && xhr.response.err) || xhr.responseText,
         fun = toast.err;
 
-    if (xhr.status == 503 && /\bDD(?:wah){0}[o]S [Pp]rote[c]tion|>Just a mo[m]ent|#cf-b[u]bbles|Chec[k]ing your br[o]wser/.test(errtxt)) {
+    if (xhr.status == 503 && /[Cc]loud[f]lare|>Just a mo[m]ent|#cf-b[u]bbles|Chec[k]ing your br[o]wser/.test(errtxt)) {
         var now = Date.now(), td = now - cf_cha_t;
         if (td < 15000)
             return;
 
         cf_cha_t = now;
-        errtxt = 'Cloudflare DD' + wah + 'oS protection kicked in\n\n<strong>trying to fix it...</strong>';
+        errtxt = 'Clou' + wah + 'dflare protection kicked in\n\n<strong>trying to fix it...</strong>';
         fun = toast.warn;
 
         qsr('#cf_frame');
-        var fr = mknod('iframe');
+        var fr = mknod('iframe', 'cf_frame');
         fr.src = '/?cf_challenge';
-        fr.setAttribute('id', 'cf_frame');
         document.body.appendChild(fr);
     }
 

copyparty/web/w.hash.js

@@ -0,0 +1,77 @@
+"use strict";
+
+
+function hex2u8(txt) {
+    return new Uint8Array(txt.match(/.{2}/g).map(function (b) { return parseInt(b, 16); }));
+}
+
+
+var subtle = null;
+try {
+    subtle = crypto.subtle || crypto.webkitSubtle;
+    subtle.digest('SHA-512', new Uint8Array(1)).then(
+        function (x) { },
+        function (x) { load_fb(); }
+    );
+}
+catch (ex) {
+    load_fb();
+}
+function load_fb() {
+    subtle = null;
+    importScripts('/.cpr/deps/sha512.hw.js');
+}
+
+
+onmessage = (d) => {
+    var [nchunk, fobj, car, cdr] = d.data,
+        t0 = Date.now(),
+        reader = new FileReader();
+
+    reader.onload = function (e) {
+        try {
+            //console.log('[ w] %d HASH bgin', nchunk);
+            postMessage(["read", nchunk, cdr - car, Date.now() - t0]);
+            hash_calc(e.target.result);
+        }
+        catch (ex) {
+            postMessage(["panic", ex + '']);
+        }
+    };
+    reader.onerror = function () {
+        var err = reader.error + '';
+
+        if (err.indexOf('NotReadableError') !== -1 || // win10-chrome defender
+            err.indexOf('NotFoundError') !== -1  // macos-firefox permissions
+        )
+            return postMessage(["fail", 'OS-error', err + ' @ ' + car]);
+
+        postMessage(["ferr", err]);
+    };
+    //console.log('[ w] %d read bgin', nchunk);
+    reader.readAsArrayBuffer(
+        File.prototype.slice.call(fobj, car, cdr));
+
+
+    var hash_calc = function (buf) {
+        var hash_done = function (hashbuf) {
+            try {
+                var hslice = new Uint8Array(hashbuf).subarray(0, 33);
+                //console.log('[ w] %d HASH DONE', nchunk);
+                postMessage(["done", nchunk, hslice, cdr - car]);
+            }
+            catch (ex) {
+                postMessage(["panic", ex + '']);
+            }
+        };
+
+        if (subtle)
+            subtle.digest('SHA-512', buf).then(hash_done);
+        else {
+            var u8buf = new Uint8Array(buf);
+            hashwasm.sha512(u8buf).then(function (v) {
+                hash_done(hex2u8(v))
+            });
+        }
+    };
+}

docs/changelog.md

@@ -1,3 +1,217 @@
+▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀  
+# 2022-0812-2258  `v1.3.12`  quickboot
+
+* read-only demo server at https://a.ocv.me/pub/demo/
+* latest gzip edition of the sfx: [v1.0.14](https://github.com/9001/copyparty/releases/tag/v1.0.14#:~:text=release-specific%20notes)
+
+## new features
+*but wait, there's more!*   not only do you get the [multithreaded file hashing](https://github.com/9001/copyparty/releases/tag/v1.3.11) but also --
+* faster bootup and volume reindexing when `-e2ds` (file indexing) is enabled
+  * `3x` faster is probably the average on most instances; more files per folder = faster
+  * `9x` faster on a 36 TiB zfs music/media nas with `-e2ts` (metadata indexing), dropping from 46sec to 5sec
+  * and `34x` on another zfs box, 63sec -> 1.8sec
+  * new arg `--no-dhash` disables the speedhax in case it's buggy (skipping files or audio tags)
+* add option `--exit idx` to abort and shutdown after volume indexing has finished
+
+## bugfixes
+* [u2cli](https://github.com/9001/copyparty/tree/hovudstraum/bin#up2kpy): detect and skip uploading from recursive symlinks
+* stop reindexing empty files on startup
+* support fips-compliant cpython builds
+  * replaces md5 with sha1, changing the filetype-associated colors in the gallery view
+
+
+
+▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀  
+# 2022-0810-2135  `v1.3.11`  webworkers
+
+* read-only demo server at https://a.ocv.me/pub/demo/
+* latest gzip edition of the sfx: [v1.0.14](https://github.com/9001/copyparty/releases/tag/v1.0.14#:~:text=release-specific%20notes)
+
+## new features
+* multithreaded file hashing! **300%** average speed increase
+  * when uploading files through the browser client, based on web-workers
+    * `4.5x` faster on http from a laptop -- `146` -> `670` MiB/s
+    * ` 30%` faster on https from a laptop -- `552` -> `716` MiB/s
+    * `4.2x` faster on http from android -- `13.5` -> `57.1` MiB/s
+    * `5.3x` faster on https from android -- `13.8` -> `73.3` MiB/s
+    * can be disabled using the `mt` togglebtn in the settings pane, for example if your phone runs out of memory (it eats ~250 MiB extra RAM)
+  * `2.3x` faster [u2cli](https://github.com/9001/copyparty/tree/hovudstraum/bin#up2kpy) (cmd-line client) -- `398` -> `930` MiB/s
+  * `2.4x` faster filesystem indexing on the server
+  * thx to @kipukun for the webworker suggestion!
+
+## bugfixes
+* ux: reset scroll when navigating into a new folder
+* u2cli: better errormsg if the server's tls certificate got rejected
+* js: more futureproof cloudflare-challenge detection (they got a new one recently)
+
+## other changes
+* print warning if the python interpreter was built with an unsafe sqlite
+* u2cli: add helpful messages on how to make it run on python 2.6
+
+**trivia:** due to a [chrome bug](https://bugs.chromium.org/p/chromium/issues/detail?id=1352210), http can sometimes be faster than https now ¯\\\_(ツ)\_/¯
+
+
+
+▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀  
+# 2022-0803-2340  `v1.3.10`  folders first
+
+* read-only demo server at https://a.ocv.me/pub/demo/
+* latest gzip edition of the sfx: [v1.0.14](https://github.com/9001/copyparty/releases/tag/v1.0.14#:~:text=release-specific%20notes)
+
+## new features
+* faster
+  * tag scanner
+  * on windows: uploading to fat32 or smb
+* toggle-button to sort folders before files (default-on)
+  * almost the same as before, but now also when sorting by size / date
+* repeatedly hit `ctrl-c` to force-quit if everything dies
+* new file-indexing guards
+  * `--xdev` / volflag `:c,xdev` stops if it hits another filesystem (bindmount/symlink)
+  * `--xvol` / volflag `:c,xvol` does not follow symlinks pointing outside the volume
+  * only affects file indexing -- does NOT prevent access!
+
+## bugfixes
+* forget uploads that failed to initialize (allows retry in another folder)
+* wrong filekeys in upload response if volume path contained a symlink
+* faster shutdown on `ctrl-c` while hashing huge files
+* ux: fix navpane covering files on horizontal scroll
+
+## other changes
+* include version info in the base64 crash-message
+* ux: make upload errors more visible on mobile
+
+
+
+▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀  
+# 2022-0727-1407  `v1.3.8`  more async
+
+* read-only demo server at https://a.ocv.me/pub/demo/
+* latest gzip edition of the sfx: [v1.0.14](https://github.com/9001/copyparty/releases/tag/v1.0.14#:~:text=release-specific%20notes)
+
+## new features
+* new arg `--df 4` and volflag `:c,df=4g` to guarantee 4 GiB free disk space by rejecting uploads
+* some features no longer block new uploads while they're processing
+  * `-e2v` file integrity checker
+  * `-e2ts` initial tag scanner
+  * hopefully fixes a [deadlock](https://www.youtube.com/watch?v=DkKoMveT_jo&t=3s) someone ran into (but probably doesn't)
+    * (the "deadlock" link is an addictive demoscene banger -- the actual issue is #10)
+* reduced the impact of some features which still do
+  * defer `--re-maxage` reindexing if there was a write (upload/rename/...) recently
+    * `--db-act` sets minimum idle period before reindex can start (default 10sec)
+* bbox / image-viewer: add video hotkeys 0..9 to seek 0%..90%
+* audio-player: add audio crossfeed (left-right channel mixer / vocal isolation)
+* splashpage (`/?h`) shows time since the most recent write
+
+## bugfixes
+* a11y:
+  * enter-key should always trigger onclick
+  * only focus password box if in-bounds
+  * improve skip-to-files
+* prisonparty: volume labeling in root folders
+* other minor stuff
+  * forget deleted shadowed files from the db
+  * be less noisy if a client disconnects mid-reply
+  * up2k.js less eager to thrash slow server HDDs
+
+## other changes
+* show client's upload ETA in server log
+* dump stacks and issue `lsof` on the db if a transaction is stuck
+  * will hopefully help if there's any more deadlocks
+* [up2k-hook-ytid](https://github.com/9001/copyparty/blob/hovudstraum/contrib/plugins/up2k-hook-ytid.js) (the overengineered up2k.js plugin example) now has an mp4/webm/mkv metadata parser
+
+
+
+▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀  
+# 2022-0716-1848  `v1.3.7`  faster
+
+* read-only demo server at https://a.ocv.me/pub/demo/
+* latest gzip edition of the sfx: [v1.0.14](https://github.com/9001/copyparty/releases/tag/v1.0.14#:~:text=release-specific%20notes)
+
+## new features
+* `up2k.js`: **improved upload speeds!**
+  * **...when there's many small files** (or the browser is slow)
+    * add [potato mode](https://user-images.githubusercontent.com/241032/179336639-8ecc01ea-2662-4cb6-8048-5be3ad599f33.png) -- lightweight UI for faster uploads from slow boxes
+    * enables automatically if it detects a cpu bottleneck (not very accurate)
+  * **...on really fast connections (LAN / fiber)**
+    * batch progress updates to reduce repaints
+  * **...when there is a mix of big and small files**
+    * sort the uploads by size, smallest first, for optimal cpu/network usage
+      * can be overridden to alphabetical order in the settings tab
+      * new arg `--u2sort` changes the default + overrides the override button
+    * improve upload pacing when alphabetical order is enabled
+      * mainly affecting single files that are 300 GiB + 
+* `up2k.js`: add [up2k hooks](https://github.com/9001/copyparty/blob/hovudstraum/contrib/plugins/up2k-hooks.js)
+  * specify *client-side* rules to reject files as they are dropped into the browser
+  * not a hard-reject since people can use [up2k.py](https://github.com/9001/copyparty/blob/hovudstraum/bin/up2k.py) and whatnot, more like a hint
+* `up2k.py`: add file integrity checker
+  * new arg `-e2v` to scan volumes and verify file checksums on startup
+  * `-e2vu` updates the db on mismatch, `-e2vp` panics
+  * uploads are blocked while the scan is running -- might get fixed at some point
+    * for now it prints a warning
+* bbox / image-viewer: doubletap a picture to enter fullscreen mode
+* md-editor: `ctrl-c/x` affects current line if no selection, and `ctrl-e` is fullscreen
+* tag-parser plugins:
+  * add support for passing metadata from one mtp to another (parser dependencies)
+    * the `p` flag in [vidchk](https://github.com/9001/copyparty/blob/hovudstraum/bin/mtag/vidchk.py) usage makes it run after the base parser, eating its output
+  * add [rclone uploader](https://github.com/9001/copyparty/blob/hovudstraum/bin/mtag/rclone-upload.py) which optionally and by default depends on vidchk
+
+## bugfixes
+* sfx would crash if it got the same PID as recently (for example across two reboots)
+* audio equalizer on recent chromes
+  * still can't figure out why chrome sometimes drops the mediasession
+* bbox: don't attach click events to videos
+* up2k.py:
+  * more sensible behavior w/ blank files
+  * avoid some extra directory scans when deleting files
+  * faster shutdown on `ctrl-c` during volume indexing
+* warning from the thumbnail cleaner if the volume has no thumbnails
+* `>fixing py2 support` `>2022`
+
+## other changes
+* up2k.js:
+  * sends a summary of the upload queue to [the server log](https://github.com/9001/copyparty#up2k)
+  * shows a toast while loading huge filedrops to indicate it's still alive
+* sfx: disable guru meditation unless running on windows
+  * avoids hanging systemd on certain crashes
+* logs the state of all threads if sqlite hits a timeout
+
+
+
+▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀  
+# 2022-0706-0029  `v1.3.5`  sup cloudflare
+
+* read-only demo server at https://a.ocv.me/pub/demo/
+* latest gzip edition of the sfx: [v1.0.14](https://github.com/9001/copyparty/releases/tag/v1.0.14#:~:text=release-specific%20notes)
+
+## new features
+* detect + recover from cloudflare ddos-protection memes during upload
+  * while carefully avoiding any mention of "DDoS" in the JS because enterprise firewalls do not enjoy that
+* new option `--favico` to specify a default favicon
+  * set to `🎉` by default, which also enables the fancy upload progress donut 👌
+* baguettebox (image/video viewer):
+  * toolbar button `⛶` to enter fullscreen mode (same as hotkey `F`)
+  * tap middle of screen to show/hide toolbar
+  * tap left/right-side of pics to navigate prev/next
+  * hotkeys `[` and `]` to set A-B loop in videos
+    * and [URL parameters](https://a.ocv.me/pub/demo/pics-vids/#gf-e2e482ae&t=4.2-6) for that + [initial seekpoint](https://a.ocv.me/pub/demo/pics-vids/#gf-c04bb0f6&t=26s) (same as the audio player)
+
+## bugfixes
+* when a tag-parser hits the timeout, `pkill` all its descendants too
+  * and a [new mtp flag](https://github.com/9001/copyparty/#file-parser-plugins) to override that; `kt` (kill tree, default), `km` (kill main, old default), `kn` (kill none)
+* cpu-wasting spin while waiting for the final handful of files to finish tag-scraping
+* detection of sparse-files support inside [prisonparty](https://github.com/9001/copyparty/tree/hovudstraum/bin#prisonpartysh) and other strict jails
+* baguettebox (image/video viewer):
+  * crash on swipe during close
+* didn't reset terminal color at the end of `?ls=v`
+* don't try to thumbnail empty files (harmless but dumb)
+
+## other changes
+* ux improvements
+  * hide the uploads table until something happens
+* bump codemirror to 5.65.6
+
+
+
 ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀  
 # 2022-0627-2057  `v1.3.3`  sdcardfs
 

docs/notes.md

@@ -0,0 +1,10 @@
+# up2k.js
+
+## potato detection
+
+* tsk 0.25/8.4/31.5 bzw 1.27/22.9/18 = 77% (38.4s, 49.7s)
+  * 4c locale #1313, ff-102,deb-11 @ ryzen4500u wifi -> win10
+  * profiling shows 2sec heavy gc every 2sec
+
+* tsk 0.41/4.1/10 bzw 1.41/9.9/7 = 73% (13.3s, 18.2s)
+  * 4c locale #1313, ch-103,deb-11 @ ryzen4500u wifi -> win10

docs/notes.sh

@@ -48,7 +48,10 @@ avg() { awk 'function pr(ncsz) {if (nsmp>0) {printf "%3s %s\n", csz, sum/nsmp} c
 ## time between first and last upload
 
 python3 -um copyparty -nw -v srv::rw -i 127.0.0.1 2>&1 | tee log 
-cat log | awk '!/"purl"/{next} {s=$1;sub(/[^m]+m/,"");gsub(/:/," ");t=60*(60*$1+$2)+$3} !a{a=t;sa=s} {b=t;sb=s} END {print b-a,sa,sb}'
+cat log | awk '!/"purl"/{next} {s=$1;sub(/[^m]+m/,"");gsub(/:/," ");t=60*(60*$1+$2)+$3} t<p{t+=86400} !a{a=t;sa=s} {b=t;sb=s} END {print b-a,sa,sb}'
+
+# or if the client youre measuring dies for ~15sec every once ina while and you wanna filter those out,
+cat log | awk '!/"purl"/{next} {s=$1;sub(/[^m]+m/,"");gsub(/:/," ");t=60*(60*$1+$2)+$3} t<p{t+=86400} !p{a=t;p=t;r=0;next} t-p>1{printf "%.3f += %.3f - %.3f  (%.3f)  # %.3f -> %.3f\n",r,p,a,p-a,p,t;r+=p-a;a=t} {p=t} END {print r+p-a}'
 
 
 ##
@@ -182,7 +185,7 @@ brew install python@2
 pip install virtualenv
 
 # readme toc
-cat README.md | awk 'function pr() { if (!h) {return}; if (/^ *[*!#|]/||!s) {printf "%s\n",h;h=0;return}; if (/.../) {printf "%s - %s\n",h,$0;h=0}; };    /^#/{s=1;pr()} /^#* *(file indexing|install on android|dev env setup|just the sfx|complete release|optional gpl stuff)|`$/{s=0} /^#/{lv=length($1);sub(/[^ ]+ /,"");bab=$0;gsub(/ /,"-",bab); h=sprintf("%" ((lv-1)*4+1) "s [%s](#%s)", "*",$0,bab);next} !h{next} {sub(/  .*/,"");sub(/[:,]$/,"")} {pr()}' > toc; grep -E '^## readme toc' -B1000 -A2 <README.md >p1; grep -E '^## quickstart' -B2 -A999999 <README.md >p2; (cat p1; grep quickstart -A1000 <toc; cat p2) >README.md; rm p1 p2 toc
+cat README.md | awk 'function pr() { if (!h) {return}; if (/^ *[*!#|]/||!s) {printf "%s\n",h;h=0;return}; if (/.../) {printf "%s - %s\n",h,$0;h=0}; };    /^#/{s=1;pr()} /^#* *(install on android|dev env setup|just the sfx|complete release|optional gpl stuff)|`$/{s=0} /^#/{lv=length($1);sub(/[^ ]+ /,"");bab=$0;gsub(/ /,"-",bab); h=sprintf("%" ((lv-1)*4+1) "s [%s](#%s)", "*",$0,bab);next} !h{next} {sub(/  .*/,"");sub(/[:;,]$/,"")} {pr()}' > toc; grep -E '^## readme toc' -B1000 -A2 <README.md >p1; grep -E '^## quickstart' -B2 -A999999 <README.md >p2; (cat p1; grep quickstart -A1000 <toc; cat p2) >README.md; rm p1 p2 toc
 
 # fix firefox phantom breakpoints,
 # suggestions from bugtracker, doesnt work (debugger is not attachable)

scripts/deps-docker/Dockerfile

@@ -2,9 +2,9 @@ FROM    alpine:3.16
 WORKDIR /z
 ENV     ver_asmcrypto=5b994303a9d3e27e0915f72a10b6c2c51535a4dc \
         ver_hashwasm=4.9.0 \
-        ver_marked=4.0.17 \
+        ver_marked=4.0.18 \
         ver_mde=2.16.1 \
-        ver_codemirror=5.65.6 \
+        ver_codemirror=5.65.7 \
         ver_fontawesome=5.13.0 \
         ver_zopfli=1.0.3
 

scripts/make-pypi-release.sh

@@ -14,10 +14,6 @@ gtar=$(command -v gtar || command -v gnutar) || true
 		realpath() { grealpath "$@"; }
 }
 
-which md5sum 2>/dev/null >/dev/null &&
-	md5sum=md5sum ||
-	md5sum="md5 -r"
-
 mode="$1"
 
 [ -z "$mode" ] &&

scripts/make-sfx.sh

@@ -26,6 +26,11 @@ help() { exec cat <<'EOF'
 #   (browsers will try to use 'Consolas' instead)
 #
 # `no-dd` saves ~2k by removing the mouse cursor
+#
+# ---------------------------------------------------------------------
+#
+# if you are on windows, you can use msys2:
+#   PATH=/c/Users/$USER/AppData/Local/Programs/Python/Python310:"$PATH" ./make-sfx.sh fast
 
 EOF
 }
@@ -64,6 +69,9 @@ pybin=$(command -v python3 || command -v python) || {
 	exit 1
 }
 
+[ $CSN ] ||
+	CSN=sfx
+
 langs=
 use_gz=
 zopf=2560
@@ -94,9 +102,9 @@ stamp=$(
 	done | sort | tail -n 1 | sha1sum | cut -c-16
 )
 
-rm -rf sfx/*
-mkdir -p sfx build
-cd sfx
+rm -rf $CSN/*
+mkdir -p $CSN build
+cd $CSN
 
 tmpdir="$(
 	printf '%s\n' "$TMPDIR" /tmp |
@@ -106,7 +114,7 @@ tmpdir="$(
 [ $repack ] && {
 	old="$tmpdir/pe-copyparty"
 	echo "repack of files in $old"
-	cp -pR "$old/"*{j2,ftp,copyparty} .
+	cp -pR "$old/"*{py2,j2,ftp,copyparty} .
 }
 
 [ $repack ] || {
@@ -190,7 +198,7 @@ tmpdir="$(
 	done
 
 	# remove type hints before build instead
-	(cd copyparty; python3 ../../scripts/strip_hints/a.py; rm uh)
+	(cd copyparty; "$pybin" ../../scripts/strip_hints/a.py; rm uh)
 }
 
 ver=
@@ -232,7 +240,7 @@ ts=$(date -u +%s)
 hts=$(date -u +%Y-%m%d-%H%M%S) # --date=@$ts (thx osx)
 
 mkdir -p ../dist
-sfx_out=../dist/copyparty-sfx
+sfx_out=../dist/copyparty-$CSN
 
 echo cleanup
 find -name '*.pyc' -delete
@@ -366,7 +374,7 @@ gzres() {
 }
 
 
-zdir="$tmpdir/cpp-mksfx"
+zdir="$tmpdir/cpp-mk$CSN"
 [ -e "$zdir/$stamp" ] || rm -rf "$zdir"
 mkdir -p "$zdir"
 echo a > "$zdir/$stamp"
@@ -397,8 +405,8 @@ sed -r 's/(.*)\.(.*)/\2 \1/' | LC_ALL=C sort |
 sed -r 's/([^ ]*) (.*)/\2.\1/' | grep -vE '/list1?$' > list1
 
 for n in {1..50}; do
-	(grep -vE '\.(gz|br)$' list1; grep -E '\.(gz|br)$' list1 | shuf) >list || true
-	s=$(md5sum list | cut -c-16)
+	(grep -vE '\.(gz|br)$' list1; grep -E '\.(gz|br)$' list1 | (shuf||gshuf) ) >list || true
+	s=$( (sha1sum||shasum) < list | cut -c-16)
 	grep -q $s "$zdir/h" && continue
 	echo $s >> "$zdir/h"
 	break
@@ -418,7 +426,7 @@ pe=bz2
 
 echo compressing tar
 # detect best level; bzip2 -7 is usually better than -9
-for n in {2..9}; do cp tar t.$n; $pc  -$n t.$n & done; wait; mv -v $(ls -1S t.*.$pe | tail -n 1) tar.bz2
+for n in {2..9}; do cp tar t.$n; nice $pc  -$n t.$n & done; wait; mv -v $(ls -1S t.*.$pe | tail -n 1) tar.bz2
 rm t.* || true
 exts=()
 

scripts/profile.py

@@ -8,7 +8,7 @@ cmd = sys.argv[1]
 if cmd == "cpp":
     from copyparty.__main__ import main
 
-    argv = ["__main__", "-v", "srv::r", "-v", "../../yt:yt:r"]
+    argv = ["__main__", "-vsrv::r:c,e2ds,e2ts"]
     main(argv=argv)
 
 elif cmd == "test":
@@ -29,6 +29,6 @@ else:
 #
 # python -m vmprof -o prof --lines ./scripts/profile.py test
 
-# linux: ~/.local/bin/vmprofshow prof tree | grep -vF '[1m  0.'
-# macos: ~/Library/Python/3.9/bin/vmprofshow prof tree | grep -vF '[1m  0.'
+# linux: ~/.local/bin/vmprofshow prof tree | awk '$2>1{n=5} !n{next} 1;{n--} !n{print""}'
+# macos: ~/Library/Python/3.9/bin/vmprofshow prof tree
 #   win: %appdata%\..\Roaming\Python\Python39\Scripts\vmprofshow.exe prof tree

scripts/py2/queue/__init__.py

@@ -0,0 +1,4 @@
+# coding: utf-8
+from __future__ import print_function, unicode_literals
+
+from Queue import Queue, LifoQueue, PriorityQueue, Empty, Full

scripts/rls.sh

@@ -1,6 +1,8 @@
 #!/bin/bash
 set -e
 
+parallel=2
+
 cd ~/dev/copyparty/scripts
 
 v=$1
@@ -21,14 +23,32 @@ v=$1
     ./make-tgz-release.sh $v
 }
 
-rm -f ../dist/copyparty-sfx.*
-f=../dist/copyparty-sfx.py
-./make-sfx.sh
-$f -h
+rm -f ../dist/copyparty-sfx*
+shift
+./make-sfx.sh "$@"
+f=../dist/copyparty-sfx
+[ -e $f.py ] || 
+    f=../dist/copyparty-sfx-gz
+
+$f.py -h >/dev/null
+
+[ $parallel -gt 1 ] && {
+    printf '\033[%s' s 2r H "0;1;37;44mbruteforcing sfx size -- press enter to terminate" K u "7m $* " K $'27m\n'
+    trap "rm -f .sfx-run; printf '\033[%s' s r u" INT TERM EXIT
+    touch .sfx-run
+    for ((a=0; a<$parallel; a++)); do
+        while [ -e .sfx-run ]; do
+            CSN=sfx$a ./make-sfx.sh re "$@"
+            mv $f$a.py $f.$(wc -c <$f$a.py | awk '{print$1}').py
+        done &
+    done
+    read
+    exit
+}
 
 while true; do
-    mv $f $f.$(wc -c <$f | awk '{print$1}')
-    ./make-sfx.sh re $ar
+    mv $f.py $f.$(wc -c <$f.py | awk '{print$1}').py
+    ./make-sfx.sh re "$@"
 done
 
 # git tag -d v$v; git push --delete origin v$v

scripts/sfx.ls

@@ -77,3 +77,4 @@ copyparty/web/splash.js,
 copyparty/web/ui.css,
 copyparty/web/up2k.js,
 copyparty/web/util.js,
+copyparty/web/w.hash.js,

scripts/sfx.py

@@ -213,22 +213,26 @@ def yieldfile(fn):
 
 
 def hashfile(fn):
-    h = hashlib.md5()
+    h = hashlib.sha1()
     for block in yieldfile(fn):
         h.update(block)
 
-    return h.hexdigest()
+    return h.hexdigest()[:24]
 
 
 def unpack():
     """unpacks the tar yielded by `data`"""
     name = "pe-copyparty"
     tag = "v" + str(STAMP)
-    withpid = "{}.{}".format(name, os.getpid())
     top = tempfile.gettempdir()
     opj = os.path.join
     final = opj(top, name)
-    mine = opj(top, withpid)
+    for suf in range(0, 9001):
+        withpid = "{}.{}.{}".format(name, os.getpid(), suf)
+        mine = opj(top, withpid)
+        if not os.path.exists(mine):
+            break
+
     tar = opj(mine, "tar")
 
     try:
@@ -360,11 +364,12 @@ def utime(top):
 def confirm(rv):
     msg()
     msg("retcode", rv if rv else traceback.format_exc())
-    msg("*** hit enter to exit ***")
-    try:
-        raw_input() if PY2 else input()
-    except:
-        pass
+    if WINDOWS:
+        msg("*** hit enter to exit ***")
+        try:
+            raw_input() if PY2 else input()
+        except:
+            pass
 
     sys.exit(rv or 1)
 

tests/test_httpcli.py

@@ -10,9 +10,10 @@ import pprint
 import tarfile
 import tempfile
 import unittest
-from argparse import Namespace
 
 from tests import util as tu
+from tests.util import Cfg
+
 from copyparty.authsrv import AuthSrv
 from copyparty.httpcli import HttpCli
 
@@ -22,56 +23,6 @@ def hdr(query):
     return h.format(query).encode("utf-8")
 
 
-class Cfg(Namespace):
-    def __init__(self, a=None, v=None, c=None):
-        super(Cfg, self).__init__(
-            a=a or [],
-            v=v or [],
-            c=c,
-            rproxy=0,
-            rsp_slp=0,
-            s_wr_slp=0,
-            s_wr_sz=512 * 1024,
-            ed=False,
-            nw=False,
-            unpost=600,
-            no_mv=False,
-            no_del=False,
-            no_zip=False,
-            no_thumb=False,
-            no_athumb=False,
-            no_vthumb=False,
-            no_voldump=True,
-            no_scandir=False,
-            no_sendfile=True,
-            no_rescan=True,
-            no_logues=False,
-            no_readme=False,
-            re_maxage=0,
-            ihead=False,
-            nih=True,
-            mtp=[],
-            mte="a",
-            mth="",
-            textfiles="",
-            doctitle="",
-            html_head="",
-            lang="eng",
-            theme=0,
-            themes=0,
-            turbo=0,
-            logout=573,
-            hist=None,
-            no_idx=None,
-            no_hash=None,
-            force_js=False,
-            no_robots=False,
-            js_browser=None,
-            css_browser=None,
-            **{k: False for k in "e2d e2ds e2dsa e2t e2ts e2tsr no_acode".split()}
-        )
-
-
 class TestHttpCli(unittest.TestCase):
     def setUp(self):
         self.td = tu.get_ramdisk()

tests/test_vfs.py

@@ -8,44 +8,14 @@ import shutil
 import tempfile
 import unittest
 from textwrap import dedent
-from argparse import Namespace
 
 from tests import util as tu
+from tests.util import Cfg
+
 from copyparty.authsrv import AuthSrv, VFS
 from copyparty import util
 
 
-class Cfg(Namespace):
-    def __init__(self, a=None, v=None, c=None):
-        ex = "nw e2d e2ds e2dsa e2t e2ts e2tsr no_logues no_readme no_acode force_js no_robots no_thumb no_athumb no_vthumb"
-        ex = {k: False for k in ex.split()}
-        ex2 = {
-            "mtp": [],
-            "mte": "a",
-            "mth": "",
-            "doctitle": "",
-            "html_head": "",
-            "hist": None,
-            "no_idx": None,
-            "no_hash": None,
-            "js_browser": None,
-            "css_browser": None,
-            "no_voldump": True,
-            "re_maxage": 0,
-            "rproxy": 0,
-            "rsp_slp": 0,
-            "s_wr_slp": 0,
-            "s_wr_sz": 512 * 1024,
-            "lang": "eng",
-            "theme": 0,
-            "themes": 0,
-            "turbo": 0,
-            "logout": 573,
-        }
-        ex.update(ex2)
-        super(Cfg, self).__init__(a=a or [], v=v or [], c=c, **ex)
-
-
 class TestVFS(unittest.TestCase):
     def setUp(self):
         self.td = tu.get_ramdisk()

tests/util.py

@@ -7,6 +7,7 @@ import threading
 import tempfile
 import platform
 import subprocess as sp
+from argparse import Namespace
 
 
 WINDOWS = platform.system() == "Windows"
@@ -89,6 +90,40 @@ def get_ramdisk():
         return subdir(ret)
 
 
+class Cfg(Namespace):
+    def __init__(self, a=None, v=None, c=None):
+        ka = {}
+
+        ex = "e2d e2ds e2dsa e2t e2ts e2tsr e2v e2vu e2vp xdev xvol ed emp force_js ihead no_acode no_athumb no_del no_logues no_mv no_readme no_robots no_scandir no_thumb no_vthumb no_zip nid nih nw"
+        ka.update(**{k: False for k in ex.split()})
+
+        ex = "no_rescan no_sendfile no_voldump"
+        ka.update(**{k: True for k in ex.split()})
+
+        ex = "css_browser hist js_browser no_hash no_idx"
+        ka.update(**{k: None for k in ex.split()})
+
+        ex = "re_maxage rproxy rsp_slp s_wr_slp theme themes turbo df"
+        ka.update(**{k: 0 for k in ex.split()})
+
+        ex = "doctitle favico html_head mth textfiles log_fk"
+        ka.update(**{k: "" for k in ex.split()})
+
+        super(Cfg, self).__init__(
+            a=a or [],
+            v=v or [],
+            c=c,
+            s_wr_sz=512 * 1024,
+            unpost=600,
+            u2sort="s",
+            mtp=[],
+            mte="a",
+            lang="eng",
+            logout=573,
+            **ka
+        )
+
+
 class NullBroker(object):
     def say(*args):
         pass