v1.15.0

dedup is still encouraged and fully supported, but
being default-enabled has caused too many surprises

enabling `--dedup` restores the previous default behavior

also renames `--never-symlink` to `--hardlink-only`

.gitignore

@@ -12,6 +12,7 @@ copyparty.egg-info/
 /dist/
 /py2/
 /sfx*
+/pyz/
 /unt/
 /log/
 

.vscode/launch.json

@@ -9,14 +9,17 @@
             "console": "integratedTerminal",
             "cwd": "${workspaceFolder}",
             "justMyCode": false,
+            "env": {
+                "PYDEVD_DISABLE_FILE_VALIDATION": "1",
+                "PYTHONWARNINGS": "always", //error
+            },
             "args": [
                 //"-nw",
                 "-ed",
                 "-emp",
                 "-e2dsa",
                 "-e2ts",
-                "-mtp",
-                ".bpm=f,bin/mtag/audio-bpm.py",
+                "-mtp=.bpm=f,bin/mtag/audio-bpm.py",
                 "-aed:wark",
                 "-vsrv::r:rw,ed:c,dupe",
                 "-vdist:dist:r"

.vscode/launch.py

@@ -41,7 +41,7 @@ if sfx:
     argv = [sys.executable, sfx] + argv
     sp.check_call(argv)
 elif re.search(" -j ?[0-9]", " ".join(argv)):
-    argv = [sys.executable, "-m", "copyparty"] + argv
+    argv = [sys.executable, "-Wa", "-m", "copyparty"] + argv
     sp.check_call(argv)
 else:
     sys.path.insert(0, os.getcwd())

.vscode/settings.json

@@ -22,6 +22,9 @@
         "terminal.ansiBrightCyan": "#9cf0ed",
         "terminal.ansiBrightWhite": "#ffffff",
     },
+    "python.terminal.activateEnvironment": false,
+    "python.analysis.enablePytestSupport": false,
+    "python.analysis.typeCheckingMode": "standard",
     "python.testing.pytestEnabled": false,
     "python.testing.unittestEnabled": true,
     "python.testing.unittestArgs": [
@@ -31,23 +34,8 @@
         "-p",
         "test_*.py"
     ],
-    "python.linting.pylintEnabled": true,
-    "python.linting.flake8Enabled": true,
-    "python.linting.banditEnabled": true,
-    "python.linting.mypyEnabled": true,
-    "python.linting.flake8Args": [
-        "--max-line-length=120",
-        "--ignore=E722,F405,E203,W503,W293,E402,E501,E128,E226",
-    ],
-    "python.linting.banditArgs": [
-        "--ignore=B104,B110,B112"
-    ],
-    // python3 -m isort --py=27 --profile=black copyparty/
-    "python.formatting.provider": "none",
-    "[python]": {
-        "editor.defaultFormatter": "ms-python.black-formatter"
-    },
-    "editor.formatOnSave": true,
+    // python3 -m isort --py=27 --profile=black ~/dev/copyparty/{copyparty,tests}/*.py && python3 -m black -t py27 ~/dev/copyparty/{copyparty,tests,bin}/*.py $(find ~/dev/copyparty/copyparty/stolen -iname '*.py')
+    "editor.formatOnSave": false,
     "[html]": {
         "editor.formatOnSave": false,
         "editor.autoIndent": "keep",
@@ -58,6 +46,4 @@
     "files.associations": {
         "*.makefile": "makefile"
     },
-    "python.linting.enabled": true,
-    "python.pythonPath": "/usr/bin/python3"
 }

.vscode/tasks.json

@@ -11,6 +11,7 @@
             "type": "shell",
             "command": "${config:python.pythonPath}",
             "args": [
+                "-Wa", //-We
                 ".vscode/launch.py"
             ]
         }

CONTRIBUTING.md

@@ -1,3 +1,43 @@
 * do something cool
 
-really tho, send a PR or an issue or whatever, all appreciated, anything goes, just behave aight
+really tho, send a PR or an issue or whatever, all appreciated, anything goes, just behave aight 👍👍
+
+but to be more specific,
+
+
+# contribution ideas
+
+
+## documentation
+
+I think we can agree that the documentation leaves a LOT to be desired. I've realized I'm not exactly qualified for this 😅 but maybe the [soon-to-come setup GUI](https://github.com/9001/copyparty/issues/57) will make this more manageable. The best documentation is the one that never had to be written, right? :> so I suppose we can give this a wait-and-see approach for a bit longer.
+
+
+## crazy ideas & features
+
+assuming they won't cause too much problems or side-effects :> 
+
+i think someone was working on a way to list directories over DNS for example...
+
+if you wanna have a go at coding it up yourself then maybe mention the idea on discord before you get too far, otherwise just go nuts 👍
+
+
+## others
+
+aside from documentation and ideas, some other things that would be cool to have some help with is:
+
+* **translations** -- the copyparty web-UI has translations for english and norwegian at the top of [browser.js](https://github.com/9001/copyparty/blob/hovudstraum/copyparty/web/browser.js); if you'd like to add a translation for another language then that'd be welcome! and if that language has a grammar that doesn't fit into the way the strings are assembled, then we'll fix that as we go :>
+
+* **UI ideas** -- at some point I was thinking of rewriting the UI in react/preact/something-not-vanilla-javascript, but I'll admit the comfiness of not having any build stage combined with raw performance has kinda convinced me otherwise :p but I'd be very open to ideas on how the UI could be improved, or be more intuitive.
+
+* **docker improvements** -- I don't really know what I'm doing when it comes to containers, so I'm sure there's a *huge* room for improvement here, mainly regarding how you're supposed to use the container with kubernetes / docker-compose / any of the other popular ways to do things. At some point I swear I'll start learning about docker so I can pick up clach04's [docker-compose draft](https://github.com/9001/copyparty/issues/38) and learn how that stuff ticks, unless someone beats me to it!
+
+* **packaging** for various linux distributions -- this could either be as simple as just plopping the sfx.py in the right place and calling that from systemd (the archlinux package [originally did this](https://github.com/9001/copyparty/pull/18)); maybe with a small config-file which would cause copyparty to load settings from `/etc/copyparty.d` (like the [archlinux package](https://github.com/9001/copyparty/tree/hovudstraum/contrib/package/arch) does with `copyparty.conf`), or it could be a proper installation of the copyparty python package into /usr/lib or similar (the archlinux package [eventually went for this approach](https://github.com/9001/copyparty/pull/26))
+
+  * [fpm](https://github.com/jordansissel/fpm) can probably help with the technical part of it, but someone needs to handle distro relations :-)
+
+* **software integration** -- I'm sure there's a lot of usecases where copyparty could complement something else, or the other way around, so any ideas or any work in this regard would be dope. This doesn't necessarily have to be code inside copyparty itself;
+
+  * [hooks](https://github.com/9001/copyparty/tree/hovudstraum/bin/hooks) -- these are small programs which are called by copyparty when certain things happen (files are uploaded, someone hits a 404, etc.), and could be a fun way to add support for more usecases
+
+  * [parser plugins](https://github.com/9001/copyparty/tree/hovudstraum/bin/mtag) -- if you want to have copyparty analyze and index metadata for some oddball file-formats, then additional plugins would be neat :>

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2019 ed
+Copyright (c) 2019 ed <oss@ocv.me>
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

bin/dbtool.py

@@ -207,7 +207,7 @@ def examples():
 
 
 def main():
-    global NC, BY_PATH
+    global NC, BY_PATH  # pylint: disable=global-statement
     os.system("")
     print()
 
@@ -282,7 +282,8 @@ def main():
         if ver == "corrupt":
             die("{} database appears to be corrupt, sorry")
 
-        if ver < DB_VER1 or ver > DB_VER2:
+        iver = int(ver)
+        if iver < DB_VER1 or iver > DB_VER2:
             m = f"{n} db is version {ver}, this tool only supports versions between {DB_VER1} and {DB_VER2}, please upgrade it with copyparty first"
             die(m)
 

bin/hooks/README.md

@@ -2,7 +2,7 @@ standalone programs which are executed by copyparty when an event happens (uploa
 
 these programs either take zero arguments, or a filepath (the affected file), or a json message with filepath + additional info
 
-run copyparty with `--help-hooks` for usage details / hook type explanations (xbu/xau/xiu/xbr/xar/xbd/xad)
+run copyparty with `--help-hooks` for usage details / hook type explanations (xm/xbu/xau/xiu/xbr/xar/xbd/xad/xban)
 
 > **note:** in addition to event hooks (the stuff described here), copyparty has another api to run your programs/scripts while providing way more information such as audio tags / video codecs / etc and optionally daisychaining data between scripts in a processing pipeline; if that's what you want then see [mtp plugins](../mtag/) instead
 
@@ -13,6 +13,7 @@ run copyparty with `--help-hooks` for usage details / hook type explanations (xb
 * [image-noexif.py](image-noexif.py) removes image exif by overwriting / directly editing the uploaded file
 * [discord-announce.py](discord-announce.py) announces new uploads on discord using webhooks ([example](https://user-images.githubusercontent.com/241032/215304439-1c1cb3c8-ec6f-4c17-9f27-81f969b1811a.png))
 * [reject-mimetype.py](reject-mimetype.py) rejects uploads unless the mimetype is acceptable
+* [into-the-cache-it-goes.py](into-the-cache-it-goes.py) avoids bugs in caching proxies by immediately downloading each file that is uploaded
 
 
 # upload batches
@@ -23,7 +24,10 @@ these are `--xiu` hooks; unlike `xbu` and `xau` (which get executed on every sin
 
 # before upload
 * [reject-extension.py](reject-extension.py) rejects uploads if they match a list of file extensions
+* [reloc-by-ext.py](reloc-by-ext.py) redirects an upload to another destination based on the file extension
 
 
 # on message
 * [wget.py](wget.py) lets you download files by POSTing URLs to copyparty
+* [qbittorrent-magnet.py](qbittorrent-magnet.py) starts downloading a torrent if you post a magnet url
+* [msg-log.py](msg-log.py) is a guestbook; logs messages to a doc in the same folder

bin/hooks/discord-announce.py

@@ -12,19 +12,28 @@ announces a new upload on discord
 example usage as global config:
     --xau f,t5,j,bin/hooks/discord-announce.py
 
+parameters explained,
+    xau = execute after upload
+    f  = fork; don't delay other hooks while this is running
+    t5 = timeout if it's still running after 5 sec
+    j  = this hook needs upload information as json (not just the filename)
+
 example usage as a volflag (per-volume config):
     -v srv/inc:inc:r:rw,ed:c,xau=f,t5,j,bin/hooks/discord-announce.py
                            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
     (share filesystem-path srv/inc as volume /inc,
      readable by everyone, read-write for user 'ed',
-     running this plugin on all uploads with the params listed below)
+     running this plugin on all uploads with the params explained above)
 
-parameters explained,
-    xbu = execute after upload
-    f  = fork; don't wait for it to finish
-    t5 = timeout if it's still running after 5 sec
-    j  = provide upload information as json; not just the filename
+example usage as a volflag in a copyparty config file:
+    [/inc]
+      srv/inc
+      accs:
+        r: *
+        rw: ed
+      flags:
+        xau: f,t5,j,bin/hooks/discord-announce.py
 
 replace "xau" with "xbu" to announce Before upload starts instead of After completion
 

bin/hooks/into-the-cache-it-goes.py

@@ -0,0 +1,140 @@
+#!/usr/bin/env python3
+
+import sys
+import json
+import shutil
+import platform
+import subprocess as sp
+from urllib.parse import quote
+
+
+_ = r"""
+try to avoid race conditions in caching proxies
+(primarily cloudflare, but probably others too)
+by means of the most obvious solution possible:
+
+just as each file has finished uploading, use
+the server's external URL to download the file
+so that it ends up in the cache, warm and snug
+
+this intentionally delays the upload response
+as it waits for the file to finish downloading
+before copyparty is allowed to return the URL
+
+NOTE: you must edit this script before use,
+  replacing https://example.com with your URL
+
+NOTE: if the files are only accessible with a
+  password and/or filekey, you must also add
+  a cromulent password in the PASSWORD field
+
+NOTE: needs either wget, curl, or "requests":
+  python3 -m pip install --user -U requests
+
+
+example usage as global config:
+    --xau j,t10,bin/hooks/into-the-cache-it-goes.py
+
+parameters explained,
+    xau = execute after upload
+    j   = this hook needs upload information as json (not just the filename)
+    t10 = abort download and continue if it takes longer than 10sec
+
+example usage as a volflag (per-volume config):
+    -v srv/inc:inc:r:rw,ed:c,xau=j,t10,bin/hooks/into-the-cache-it-goes.py
+                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+    (share filesystem-path srv/inc as volume /inc,
+     readable by everyone, read-write for user 'ed',
+     running this plugin on all uploads with params explained above)
+
+example usage as a volflag in a copyparty config file:
+    [/inc]
+      srv/inc
+      accs:
+        r: *
+        rw: ed
+      flags:
+        xau: j,t10,bin/hooks/into-the-cache-it-goes.py
+"""
+
+
+# replace this with your site's external URL
+# (including the :portnumber if necessary)
+SITE_URL = "https://example.com"
+
+# if downloading is protected by passwords or filekeys,
+# specify a valid password between the quotes below:
+PASSWORD = ""
+
+# if file is larger than this, skip download
+MAX_MEGABYTES = 8
+
+# =============== END OF CONFIG ===============
+
+
+WINDOWS = platform.system() == "Windows"
+
+
+def main():
+    fun = download_with_python
+    if shutil.which("curl"):
+        fun = download_with_curl
+    elif shutil.which("wget"):
+        fun = download_with_wget
+
+    inf = json.loads(sys.argv[1])
+
+    if inf["sz"] > 1024 * 1024 * MAX_MEGABYTES:
+        print("[into-the-cache] file is too large; will not download")
+        return
+
+    file_url = "/"
+    if inf["vp"]:
+        file_url += inf["vp"] + "/"
+    file_url += inf["ap"].replace("\\", "/").split("/")[-1]
+    file_url = SITE_URL.rstrip("/") + quote(file_url, safe=b"/")
+
+    print("[into-the-cache] %s(%s)" % (fun.__name__, file_url))
+    fun(file_url, PASSWORD.strip())
+
+    print("[into-the-cache] Download OK")
+
+
+def download_with_curl(url, pw):
+    cmd = ["curl"]
+
+    if pw:
+        cmd += ["-HPW:%s" % (pw,)]
+
+    nah = sp.DEVNULL
+    sp.check_call(cmd + [url], stdout=nah, stderr=nah)
+
+
+def download_with_wget(url, pw):
+    cmd = ["wget", "-O"]
+
+    cmd += ["nul" if WINDOWS else "/dev/null"]
+
+    if pw:
+        cmd += ["--header=PW:%s" % (pw,)]
+
+    nah = sp.DEVNULL
+    sp.check_call(cmd + [url], stdout=nah, stderr=nah)
+
+
+def download_with_python(url, pw):
+    import requests
+
+    headers = {}
+    if pw:
+        headers["PW"] = pw
+
+    with requests.get(url, headers=headers, stream=True) as r:
+        r.raise_for_status()
+        for _ in r.iter_content(chunk_size=1024 * 256):
+            pass
+
+
+if __name__ == "__main__":
+    main()

bin/hooks/msg-log.py

@@ -0,0 +1,136 @@
+#!/usr/bin/env python
+# coding: utf-8
+# vim:ts=4:sw=4:softtabstop=4:smarttab:expandtab
+from __future__ import print_function, unicode_literals
+
+import json
+import os
+import sys
+import time
+
+try:
+    from datetime import datetime, timezone
+except:
+    from datetime import datetime
+
+
+_ = r"""
+use copyparty as a dumb messaging server / guestbook thing;
+accepts guestbook entries from 📟 (message-to-server-log) in the web-ui
+initially contributed by @clach04 in https://github.com/9001/copyparty/issues/35 (thanks!)
+
+example usage as global config:
+    python copyparty-sfx.py --xm j,bin/hooks/msg-log.py
+
+parameters explained,
+    xm = execute on message (📟)
+    j  = this hook needs message information as json (not just the message-text)
+
+example usage as a volflag (per-volume config):
+    python copyparty-sfx.py -v srv/log:log:r:c,xm=j,bin/hooks/msg-log.py
+                                             ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+    (share filesystem-path srv/log as volume /log, readable by everyone,
+     running this plugin on all messages with the params explained above)
+
+example usage as a volflag in a copyparty config file:
+    [/log]
+      srv/log
+      accs:
+        r: *
+      flags:
+        xm: j,bin/hooks/msg-log.py
+"""
+
+
+# output filename
+FILENAME = os.environ.get("COPYPARTY_MESSAGE_FILENAME", "") or "README.md"
+
+# set True to write in descending order (newest message at top of file);
+# note that this becomes very slow/expensive as the file gets bigger
+DESCENDING = True
+
+# the message template; the following parameters are provided by copyparty and can be referenced below:
+# 'ap' = absolute filesystem path where the message was posted
+# 'vp' = virtual path (URL 'path') where the message was posted
+# 'mt' = 'at' = unix-timestamp when the message was posted
+# 'datetime' = ISO-8601 time when the message was posted
+# 'sz' = message size in bytes
+# 'host' = the server hostname which the user was accessing (URL 'host')
+# 'user' = username (if logged in), otherwise '*'
+# 'txt' = the message text itself
+# (uncomment the print(msg_info) to see if additional information has been introduced by copyparty since this was written)
+TEMPLATE = """
+🕒 %(datetime)s, 👤 %(user)s @ %(ip)s
+%(txt)s
+"""
+
+
+def write_ascending(filepath, msg_text):
+    with open(filepath, "a", encoding="utf-8", errors="replace") as outfile:
+        outfile.write(msg_text)
+
+
+def write_descending(filepath, msg_text):
+    lockpath = filepath + ".lock"
+    got_it = False
+    for _ in range(16):
+        try:
+            os.mkdir(lockpath)
+            got_it = True
+            break
+        except:
+            time.sleep(0.1)
+            continue
+
+    if not got_it:
+        return sys.exit(1)
+
+    try:
+        oldpath = filepath + ".old"
+        os.rename(filepath, oldpath)
+        with open(oldpath, "r", encoding="utf-8", errors="replace") as infile, open(
+            filepath, "w", encoding="utf-8", errors="replace"
+        ) as outfile:
+            outfile.write(msg_text)
+            while True:
+                buf = infile.read(4096)
+                if not buf:
+                    break
+                outfile.write(buf)
+    finally:
+        try:
+            os.unlink(oldpath)
+        except:
+            pass
+        os.rmdir(lockpath)
+
+
+def main(argv=None):
+    if argv is None:
+        argv = sys.argv
+
+    msg_info = json.loads(sys.argv[1])
+    # print(msg_info)
+
+    try:
+        dt = datetime.fromtimestamp(msg_info["at"], timezone.utc)
+    except:
+        dt = datetime.utcfromtimestamp(msg_info["at"])
+
+    msg_info["datetime"] = dt.strftime("%Y-%m-%d, %H:%M:%S")
+
+    msg_text = TEMPLATE % msg_info
+
+    filepath = os.path.join(msg_info["ap"], FILENAME)
+
+    if DESCENDING and os.path.exists(filepath):
+        write_descending(filepath, msg_text)
+    else:
+        write_ascending(filepath, msg_text)
+
+    print(msg_text)
+
+
+if __name__ == "__main__":
+    main()

bin/hooks/notify2.py

@@ -4,7 +4,7 @@ import json
 import os
 import sys
 import subprocess as sp
-from datetime import datetime
+from datetime import datetime, timezone
 from plyer import notification
 
 
@@ -43,7 +43,8 @@ def main():
     fp = inf["ap"]
     sz = humansize(inf["sz"])
     dp, fn = os.path.split(fp)
-    mt = datetime.utcfromtimestamp(inf["mt"]).strftime("%Y-%m-%d %H:%M:%S")
+    dt = datetime.fromtimestamp(inf["mt"], timezone.utc)
+    mt = dt.strftime("%Y-%m-%d %H:%M:%S")
 
     msg = f"{fn} ({sz})\n📁 {dp}"
     title = "File received"

bin/hooks/qbittorrent-magnet.py

@@ -0,0 +1,128 @@
+#!/usr/bin/env python3
+# coding: utf-8
+
+import os
+import sys
+import json
+import shutil
+import subprocess as sp
+
+
+_ = r"""
+start downloading a torrent by POSTing a magnet URL to copyparty,
+for example using 📟 (message-to-server-log) in the web-ui
+
+by default it will download the torrent to the folder you were in
+when you pasted the magnet into the message-to-server-log field
+
+you can optionally specify another location by adding a whitespace
+after the magnet URL followed by the name of the subfolder to DL into,
+or for example "anime/airing" would download to /srv/media/anime/airing
+because the keyword "anime" is in the DESTS config below
+
+needs python3
+
+example usage as global config (not a good idea):
+    python copyparty-sfx.py --xm aw,f,j,t60,bin/hooks/qbittorrent-magnet.py
+
+parameters explained,
+    xm = execute on message (📟)
+    aw = only users with write-access can use this
+    f = fork; don't delay other hooks while this is running
+    j = provide message information as json (not just the text)
+    t60 = abort if qbittorrent has to think about it for more than 1 min
+
+example usage as a volflag (per-volume config, much better):
+    -v srv/qb:qb:A,ed:c,xm=aw,f,j,t60,bin/hooks/qbittorrent-magnet.py
+                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+    (share filesystem-path srv/qb as volume /qb with Admin for user 'ed',
+     running this plugin on all messages with the params explained above)
+
+example usage as a volflag in a copyparty config file:
+    [/qb]
+      srv/qb
+      accs:
+        A: ed
+      flags:
+        xm: aw,f,j,t60,bin/hooks/qbittorrent-magnet.py
+
+the volflag examples only kicks in if you send the torrent magnet
+while you're in the /qb folder (or any folder below there)
+"""
+
+
+# list of usernames to allow
+ALLOWLIST = [ "ed", "morpheus" ]
+
+
+# list of destination aliases to translate into full filesystem
+# paths; takes effect if the first folder component in the
+# custom download location matches anything in this dict
+DESTS = {
+    "iso": "/srv/pub/linux-isos",
+    "anime": "/srv/media/anime",
+}
+
+
+def main():
+    inf = json.loads(sys.argv[1])
+    url = inf["txt"]
+    if not url.lower().startswith("magnet:?"):
+        # not a magnet, abort
+        return
+
+    if inf["user"] not in ALLOWLIST:
+        print("🧲 denied for user", inf["user"])
+        return
+
+    # might as well run the command inside the filesystem folder
+    # which matches the URL that the magnet message was sent to
+    os.chdir(inf["ap"])
+
+    # is there is a custom download location in the url?
+    dst = ""
+    if " " in url:
+        url, dst = url.split(" ", 1)
+
+        # is the location in the predefined list of locations?
+        parts = dst.replace("\\", "/").split("/")
+        if parts[0] in DESTS:
+            dst = os.path.join(DESTS[parts[0]], *(parts[1:]))
+
+    else:
+        # nope, so download to the current folder instead;
+        # comment the dst line below to instead use the default
+        # download location from your qbittorrent settings
+        dst = inf["ap"]
+        pass
+
+    # archlinux has a -nox suffix for qbittorrent if headless
+    # so check if we should be using that
+    if shutil.which("qbittorrent-nox"):
+        torrent_bin = "qbittorrent-nox"
+    else:
+        torrent_bin = "qbittorrent"
+
+    # the command to add a new torrent, adjust if necessary
+    cmd = [torrent_bin, url]
+    if dst:
+        cmd += ["--save-path=%s" % (dst,)]
+
+    # if copyparty and qbittorrent are running as different users
+    # you may have to do something like the following
+    # (assuming qbittorrent* is nopasswd-allowed in sudoers):
+    #
+    # cmd = ["sudo", "-u", "qbitter"] + cmd
+
+    print("🧲", cmd)
+
+    try:
+        sp.check_call(cmd)
+    except:
+        print("🧲 FAILED TO ADD", url)
+
+
+if __name__ == "__main__":
+    main()
+

bin/hooks/reloc-by-ext.py

@@ -0,0 +1,127 @@
+#!/usr/bin/env python3
+
+import json
+import os
+import re
+import sys
+
+
+_ = r"""
+relocate/redirect incoming uploads according to file extension or name
+
+example usage as global config:
+    --xbu j,c1,bin/hooks/reloc-by-ext.py
+
+parameters explained,
+    xbu = execute before upload
+    j   = this hook needs upload information as json (not just the filename)
+    c1  = this hook returns json on stdout, so tell copyparty to read that
+
+example usage as a volflag (per-volume config):
+    -v srv/inc:inc:r:rw,ed:c,xbu=j,c1,bin/hooks/reloc-by-ext.py
+                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+    (share filesystem-path srv/inc as volume /inc,
+     readable by everyone, read-write for user 'ed',
+     running this plugin on all uploads with the params explained above)
+
+example usage as a volflag in a copyparty config file:
+    [/inc]
+      srv/inc
+      accs:
+        r: *
+        rw: ed
+      flags:
+        xbu: j,c1,bin/hooks/reloc-by-ext.py
+
+note: this could also work as an xau hook (after-upload), but
+  because it doesn't need to read the file contents its better
+  as xbu (before-upload) since that's safer / less buggy,
+  and only xbu works with up2k (dragdrop into browser)
+"""
+
+
+PICS = "avif bmp gif heic heif jpeg jpg jxl png psd qoi tga tif tiff webp"
+VIDS = "3gp asf avi flv mkv mov mp4 mpeg mpeg2 mpegts mpg mpg2 nut ogm ogv rm ts vob webm wmv"
+MUSIC = "aac aif aiff alac amr ape dfpwm flac m4a mp3 ogg opus ra tak tta wav wma wv"
+
+
+def main():
+    inf = json.loads(sys.argv[1])
+    vdir, fn = os.path.split(inf["vp"])
+
+    try:
+        fn, ext = fn.rsplit(".", 1)
+    except:
+        # no file extension; pretend it's "bin"
+        ext = "bin"
+
+    ext = ext.lower()
+
+    # this function must end by printing the action to perform;
+    # that's handled by the print(json.dumps(... at the bottom
+    #
+    # the action can contain the following keys:
+    # "vp" is the folder URL to move the upload to,
+    # "ap" is the filesystem-path to move it to (but "vp" is safer),
+    # "fn" overrides the final filename to use
+
+    ##
+    ## some example actions to take; pick one by
+    ## selecting it inside the print at the end:
+    ##
+
+    # create a subfolder named after the filetype and move it into there
+    into_subfolder = {"vp": ext}
+
+    # move it into a toplevel folder named after the filetype
+    into_toplevel = {"vp": "/" + ext}
+
+    # move it into a filetype-named folder next to the target folder
+    into_sibling = {"vp": "../" + ext}
+
+    # move images into "/just/pics", vids into "/just/vids",
+    # music into "/just/tunes", and anything else as-is
+    if ext in PICS.split():
+        by_category = {"vp": "/just/pics"}
+    elif ext in VIDS.split():
+        by_category = {"vp": "/just/vids"}
+    elif ext in MUSIC.split():
+        by_category = {"vp": "/just/tunes"}
+    else:
+        by_category = {}  # no action
+
+    # now choose the default effect to apply; can be any of these:
+    # into_subfolder  into_toplevel  into_sibling  by_category
+    effect = {"vp": "/junk"}
+
+    ##
+    ## but we can keep going, adding more speicifc rules
+    ## which can take precedence, replacing the fallback
+    ## effect we just specified:
+    ##
+
+    fn = fn.lower()  # lowercase filename to make this easier
+
+    if "screenshot" in fn:
+        effect = {"vp": "/ss"}
+    if "mpv_" in fn:
+        effect = {"vp": "/anishots"}
+    elif "debian" in fn or "biebian" in fn:
+        effect = {"vp": "/linux-ISOs"}
+    elif re.search(r"ep(isode |\.)?[0-9]", fn):
+        effect = {"vp": "/podcasts"}
+
+    # regex lets you grab a part of the matching
+    # text and use that in the upload path:
+    m = re.search(r"\b(op|ed)([^a-z]|$)", fn)
+    if m:
+        # the regex matched; use "anime-op" or "anime-ed"
+        effect = {"vp": "/anime-" + m[1]}
+
+    # aaand DO IT
+    print(json.dumps({"reloc": effect}))
+
+
+if __name__ == "__main__":
+    main()

bin/hooks/wget.py

@@ -9,25 +9,38 @@ import subprocess as sp
 _ = r"""
 use copyparty as a file downloader by POSTing URLs as
 application/x-www-form-urlencoded (for example using the
-message/pager function on the website)
+📟 message-to-server-log in the web-ui)
 
 example usage as global config:
-    --xm f,j,t3600,bin/hooks/wget.py
-
-example usage as a volflag (per-volume config):
-    -v srv/inc:inc:r:rw,ed:c,xm=f,j,t3600,bin/hooks/wget.py
-                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-    (share filesystem-path srv/inc as volume /inc,
-     readable by everyone, read-write for user 'ed',
-     running this plugin on all messages with the params listed below)
+    --xm aw,f,j,t3600,bin/hooks/wget.py
 
 parameters explained,
     xm = execute on message-to-server-log
-    f = fork so it doesn't block uploads
-    j = provide message information as json; not just the text
+    aw = only users with write-access can use this
+    f = fork; don't delay other hooks while this is running
+    j = provide message information as json (not just the text)
     c3 = mute all output
-    t3600 = timeout and kill download after 1 hour
+    t3600 = timeout and abort download after 1 hour
+
+example usage as a volflag (per-volume config):
+    -v srv/inc:inc:r:rw,ed:c,xm=aw,f,j,t3600,bin/hooks/wget.py
+                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+    (share filesystem-path srv/inc as volume /inc,
+     readable by everyone, read-write for user 'ed',
+     running this plugin on all messages with the params explained above)
+
+example usage as a volflag in a copyparty config file:
+    [/inc]
+      srv/inc
+      accs:
+        r: *
+        rw: ed
+      flags:
+        xm: aw,f,j,t3600,bin/hooks/wget.py
+
+the volflag examples only kicks in if you send the message
+while you're in the /inc folder (or any folder below there)
 """
 
 

bin/hooks/xiu-sha.py

@@ -3,7 +3,7 @@
 import hashlib
 import json
 import sys
-from datetime import datetime
+from datetime import datetime, timezone
 
 
 _ = r"""
@@ -43,8 +43,11 @@ except:
         return p
 
 
+UTC = timezone.utc
+
+
 def humantime(ts):
-    return datetime.utcfromtimestamp(ts).strftime("%Y-%m-%d %H:%M:%S")
+    return datetime.fromtimestamp(ts, UTC).strftime("%Y-%m-%d %H:%M:%S")
 
 
 def find_files_root(inf):
@@ -96,7 +99,7 @@ def main():
 
     ret.append("# {} files, {} bytes total".format(len(inf), total_sz))
     ret.append("")
-    ftime = datetime.utcnow().strftime("%Y-%m%d-%H%M%S.%f")
+    ftime = datetime.now(UTC).strftime("%Y-%m%d-%H%M%S.%f")
     fp = "{}xfer-{}.sha512".format(inf[0]["ap"][:di], ftime)
     with open(fsenc(fp), "wb") as f:
         f.write("\n".join(ret).encode("utf-8", "replace"))

bin/mtag/install-deps.sh

@@ -7,6 +7,7 @@ set -e
 # linux/alpine: requires gcc g++ make cmake patchelf {python3,ffmpeg,fftw,libsndfile}-dev py3-{wheel,pip} py3-numpy{,-dev}
 # linux/debian: requires libav{codec,device,filter,format,resample,util}-dev {libfftw3,python3,libsndfile1}-dev python3-{numpy,pip} vamp-{plugin-sdk,examples} patchelf cmake
 # linux/fedora: requires gcc gcc-c++ make cmake patchelf {python3,ffmpeg,fftw,libsndfile}-devel python3-numpy vamp-plugin-sdk qm-vamp-plugins
+# linux/arch:   requires gcc make cmake patchelf python3 ffmpeg fftw libsndfile python-{numpy,wheel,pip,setuptools}
 # win64: requires msys2-mingw64 environment
 # macos: requires macports
 #
@@ -222,27 +223,31 @@ install_vamp() {
 	#   use msys2 in mingw-w64 mode
 	#   pacman -S --needed mingw-w64-x86_64-{ffmpeg,python,python-pip,vamp-plugin-sdk}
 	
-	$pybin -m pip install --user vamp
+	$pybin -m pip install --user vamp || {
+		printf '\n\033[7malright, trying something else...\033[0m\n'
+		$pybin -m pip install --user --no-build-isolation vamp
+	}
 
 	cd "$td"
 	echo '#include <vamp-sdk/Plugin.h>' | g++ -x c++ -c -o /dev/null - || [ -e ~/pe/vamp-sdk ] || {
 		printf '\033[33mcould not find the vamp-sdk, building from source\033[0m\n'
-		(dl_files yolo https://code.soundsoftware.ac.uk/attachments/download/2588/vamp-plugin-sdk-2.9.0.tar.gz)
+		(dl_files yolo https://ocv.me/mirror/vamp-plugin-sdk-2.10.0.tar.gz)
 		sha512sum -c <(
-			echo "7ef7f837d19a08048b059e0da408373a7964ced452b290fae40b85d6d70ca9000bcfb3302cd0b4dc76cf2a848528456f78c1ce1ee0c402228d812bd347b6983b  -"
-		) <vamp-plugin-sdk-2.9.0.tar.gz
-		tar -xf vamp-plugin-sdk-2.9.0.tar.gz
+			echo "153b7f2fa01b77c65ad393ca0689742d66421017fd5931d216caa0fcf6909355fff74706fabbc062a3a04588a619c9b515a1dae00f21a57afd97902a355c48ed  -"
+		) <vamp-plugin-sdk-2.10.0.tar.gz
+		tar -xf vamp-plugin-sdk-2.10.0.tar.gz
 		rm -- *.tar.gz
 		ls -al
 		cd vamp-plugin-sdk-*
-		./configure --prefix=$HOME/pe/vamp-sdk
+		printf '%s\n' "int main(int argc, char **argv) { return 0; }" > host/vamp-simple-host.cpp
+		./configure --disable-programs --prefix=$HOME/pe/vamp-sdk
 		make -j1 install
 	}
 
 	cd "$td"
 	have_beatroot || {
 		printf '\033[33mcould not find the vamp beatroot plugin, building from source\033[0m\n'
-		(dl_files yolo https://code.soundsoftware.ac.uk/attachments/download/885/beatroot-vamp-v1.0.tar.gz)
+		(dl_files yolo https://ocv.me/mirror/beatroot-vamp-v1.0.tar.gz)
 		sha512sum -c <(
 			echo "1f444d1d58ccf565c0adfe99f1a1aa62789e19f5071e46857e2adfbc9d453037bc1c4dcb039b02c16240e9b97f444aaff3afb625c86aa2470233e711f55b6874  -"
 		) <beatroot-vamp-v1.0.tar.gz
@@ -250,8 +255,9 @@ install_vamp() {
 		rm -- *.tar.gz
 		cd beatroot-vamp-v1.0
 		[ -e ~/pe/vamp-sdk ] &&
-			sed -ri 's`^(CFLAGS :=.*)`\1 -I'$HOME'/pe/vamp-sdk/include`' Makefile.linux
-		make -f Makefile.linux -j4 LDFLAGS=-L$HOME/pe/vamp-sdk/lib
+			sed -ri 's`^(CFLAGS :=.*)`\1 -I'$HOME'/pe/vamp-sdk/include`' Makefile.linux ||
+			sed -ri 's`^(CFLAGS :=.*)`\1 -I/usr/include/vamp-sdk`' Makefile.linux
+		make -f Makefile.linux -j4 LDFLAGS="-L$HOME/pe/vamp-sdk/lib -L/usr/lib64"
 		# /home/ed/vamp /home/ed/.vamp /usr/local/lib/vamp
 		mkdir ~/vamp
 		cp -pv beatroot-vamp.* ~/vamp/

bin/partyfuse.py

@@ -46,13 +46,20 @@ import traceback
 import http.client  # py2: httplib
 import urllib.parse
 import calendar
-from datetime import datetime
+from datetime import datetime, timezone
 from urllib.parse import quote_from_bytes as quote
 from urllib.parse import unquote_to_bytes as unquote
 
 WINDOWS = sys.platform == "win32"
 MACOS = platform.system() == "Darwin"
-info = log = dbg = None
+UTC = timezone.utc
+
+
+def print(*args, **kwargs):
+    try:
+        builtins.print(*list(args), **kwargs)
+    except:
+        builtins.print(termsafe(" ".join(str(x) for x in args)), **kwargs)
 
 
 print(
@@ -64,6 +71,13 @@ print(
 )
 
 
+def null_log(msg):
+    pass
+
+
+info = log = dbg = null_log
+
+
 try:
     from fuse import FUSE, FuseOSError, Operations
 except:
@@ -83,13 +97,6 @@ except:
     raise
 
 
-def print(*args, **kwargs):
-    try:
-        builtins.print(*list(args), **kwargs)
-    except:
-        builtins.print(termsafe(" ".join(str(x) for x in args)), **kwargs)
-
-
 def termsafe(txt):
     try:
         return txt.encode(sys.stdout.encoding, "backslashreplace").decode(
@@ -118,10 +125,6 @@ def fancy_log(msg):
     print("{:10.6f} {} {}\n".format(time.time() % 900, rice_tid(), msg), end="")
 
 
-def null_log(msg):
-    pass
-
-
 def hexler(binary):
     return binary.replace("\r", "\\r").replace("\n", "\\n")
     return " ".join(["{}\033[36m{:02x}\033[0m".format(b, ord(b)) for b in binary])
@@ -176,7 +179,7 @@ class RecentLog(object):
     def put(self, msg):
         msg = "{:10.6f} {} {}\n".format(time.time() % 900, rice_tid(), msg)
         if self.f:
-            fmsg = " ".join([datetime.utcnow().strftime("%H%M%S.%f"), str(msg)])
+            fmsg = " ".join([datetime.now(UTC).strftime("%H%M%S.%f"), str(msg)])
             self.f.write(fmsg.encode("utf-8"))
 
         with self.mtx:

bin/partyjournal.py

@@ -20,12 +20,13 @@ import sys
 import base64
 import sqlite3
 import argparse
-from datetime import datetime
+from datetime import datetime, timezone
 from urllib.parse import quote_from_bytes as quote
 from urllib.parse import unquote_to_bytes as unquote
 
 
 FS_ENCODING = sys.getfilesystemencoding()
+UTC = timezone.utc
 
 
 class APF(argparse.ArgumentDefaultsHelpFormatter, argparse.RawDescriptionHelpFormatter):
@@ -155,11 +156,10 @@ th {
                 link = txt.decode("utf-8")[4:]
 
         sz = "{:,}".format(sz)
+        dt = datetime.fromtimestamp(at if at > 0 else mt, UTC)
         v = [
             w[:16],
-            datetime.utcfromtimestamp(at if at > 0 else mt).strftime(
-                "%Y-%m-%d %H:%M:%S"
-            ),
+            dt.strftime("%Y-%m-%d %H:%M:%S"),
             sz,
             imap.get(ip, ip),
         ]

bin/prisonparty.sh

@@ -12,13 +12,13 @@ done
 help() { cat <<'EOF'
 
 usage:
-  ./prisonparty.sh <ROOTDIR> <UID> <GID> [VOLDIR [VOLDIR...]] -- python3 copyparty-sfx.py [...]
+  ./prisonparty.sh <ROOTDIR> <USER|UID> <GROUP|GID> [VOLDIR [VOLDIR...]] -- python3 copyparty-sfx.py [...]
 
 example:
-  ./prisonparty.sh /var/lib/copyparty-jail 1000 1000 /mnt/nas/music -- python3 copyparty-sfx.py -v /mnt/nas/music::rwmd
+  ./prisonparty.sh /var/lib/copyparty-jail cpp cpp /mnt/nas/music -- python3 copyparty-sfx.py -v /mnt/nas/music::rwmd
 
 example for running straight from source (instead of using an sfx):
-  PYTHONPATH=$PWD ./prisonparty.sh /var/lib/copyparty-jail 1000 1000 /mnt/nas/music -- python3 -um copyparty -v /mnt/nas/music::rwmd
+  PYTHONPATH=$PWD ./prisonparty.sh /var/lib/copyparty-jail cpp cpp /mnt/nas/music -- python3 -um copyparty -v /mnt/nas/music::rwmd
 
 note that if you have python modules installed as --user (such as bpm/key detectors),
   you should add /home/foo/.local as a VOLDIR
@@ -28,6 +28,16 @@ exit 1
 }
 
 
+errs=
+for c in awk chroot dirname getent lsof mknod mount realpath sed sort stat uniq; do
+	command -v $c >/dev/null || {
+		echo ERROR: command not found: $c
+		errs=1
+	}
+done
+[ $errs ] && exit 1
+
+
 # read arguments
 trap help EXIT
 jail="$(realpath "$1")"; shift
@@ -58,11 +68,18 @@ cpp="$1"; shift
 }
 trap - EXIT
 
+usr="$(getent passwd $uid | cut -d: -f1)"
+[ "$usr" ] || { echo "ERROR invalid username/uid $uid"; exit 1; }
+uid="$(getent passwd $uid | cut -d: -f3)"
+
+grp="$(getent group $gid | cut -d: -f1)"
+[ "$grp" ] || { echo "ERROR invalid groupname/gid $gid"; exit 1; }
+gid="$(getent group $gid | cut -d: -f3)"
 
 # debug/vis
 echo
 echo "chroot-dir = $jail"
-echo "user:group = $uid:$gid"
+echo "user:group = $uid:$gid ($usr:$grp)"
 echo " copyparty = $cpp"
 echo
 printf '\033[33m%s\033[0m\n' "copyparty can access these folders and all their subdirectories:"
@@ -80,34 +97,39 @@ jail="${jail%/}"
 
 
 # bind-mount system directories and volumes
+for a in {1..30}; do mkdir "$jail/.prisonlock" && break; sleep 0.1; done
 printf '%s\n' "${sysdirs[@]}" "${vols[@]}" | sed -r 's`/$``' | LC_ALL=C sort | uniq |
 while IFS= read -r v; do
 	[ -e "$v" ] || {
 		printf '\033[1;31mfolder does not exist:\033[0m %s\n' "$v"
 		continue
 	}
-	i1=$(stat -c%D.%i "$v"      2>/dev/null || echo a)
-	i2=$(stat -c%D.%i "$jail$v" 2>/dev/null || echo b)
-	# echo "v [$v] i1 [$i1] i2 [$i2]"
+	i1=$(stat -c%D.%i "$v/"      2>/dev/null || echo a)
+	i2=$(stat -c%D.%i "$jail$v/" 2>/dev/null || echo b)
 	[ $i1 = $i2 ] && continue
-	
+	mount | grep -qF " $jail$v " && echo wtf $i1 $i2 $v && continue
 	mkdir -p "$jail$v"
 	mount --bind "$v" "$jail$v"
 done
+rmdir "$jail/.prisonlock" || true
 
 
 cln() {
-	rv=$?
-	wait -f -p rv $p || true
+	trap - EXIT
+	wait -f -n $p && rv=0 || rv=$?
 	cd /
 	echo "stopping chroot..."
-	lsof "$jail" | grep -F "$jail" &&
+	for a in {1..30}; do mkdir "$jail/.prisonlock" && break; sleep 0.1; done
+	lsof "$jail" 2>/dev/null | grep -F "$jail" &&
 		echo "chroot is in use; will not unmount" ||
 	{
 		mount | grep -F " on $jail" |
 		awk '{sub(/ type .*/,"");sub(/.* on /,"");print}' |
-		LC_ALL=C sort -r  | tee /dev/stderr | tr '\n' '\0' | xargs -r0 umount
+		LC_ALL=C sort -r | while IFS= read -r v; do
+			umount "$v" && echo "umount OK: $v"
+		done
 	}
+	rmdir "$jail/.prisonlock" || true
 	exit $rv
 }
 trap cln EXIT
@@ -128,8 +150,8 @@ chmod 777 "$jail/tmp"
 
 
 # run copyparty
-export HOME=$(getent passwd $uid | cut -d: -f6)
-export USER=$(getent passwd $uid | cut -d: -f1)
+export HOME="$(getent passwd $uid | cut -d: -f6)"
+export USER="$usr"
 export LOGNAME="$USER"
 #echo "pybin [$pybin]"
 #echo "pyarg [$pyarg]"
@@ -137,5 +159,5 @@ export LOGNAME="$USER"
 chroot --userspec=$uid:$gid "$jail" "$pybin" $pyarg "$cpp" "$@" &
 p=$!
 trap 'kill -USR1 $p' USR1
-trap 'kill $p' INT TERM
+trap 'trap - INT TERM; kill $p' INT TERM
 wait

bin/u2c.py

@@ -1,8 +1,8 @@
 #!/usr/bin/env python3
 from __future__ import print_function, unicode_literals
 
-S_VERSION = "1.10"
-S_BUILD_DT = "2023-08-15"
+S_VERSION = "1.24"
+S_BUILD_DT = "2024-09-05"
 
 """
 u2c.py: upload to copyparty
@@ -20,6 +20,7 @@ import sys
 import stat
 import math
 import time
+import json
 import atexit
 import signal
 import socket
@@ -29,7 +30,7 @@ import platform
 import threading
 import datetime
 
-EXE = sys.executable.endswith("exe")
+EXE = bool(getattr(sys, "frozen", False))
 
 try:
     import argparse
@@ -40,19 +41,25 @@ except:
 
 try:
     import requests
+
+    req_ses = requests.Session()
 except ImportError as ex:
-    if EXE:
+    if "-" in sys.argv or "-h" in sys.argv:
+        m = ""
+    elif EXE:
         raise
     elif sys.version_info > (2, 7):
-        m = "\nERROR: need 'requests'; please run this command:\n {0} -m pip install --user requests\n"
+        m = "\nERROR: need 'requests'{0}; please run this command:\n {1} -m pip install --user requests\n"
     else:
         m = "requests/2.18.4 urllib3/1.23 chardet/3.0.4 certifi/2020.4.5.1 idna/2.7"
         m = ["   https://pypi.org/project/" + x + "/#files" for x in m.split()]
-        m = "\n  ERROR: need these:\n" + "\n".join(m) + "\n"
+        m = "\n  ERROR: need these{0}:\n" + "\n".join(m) + "\n"
         m += "\n  for f in *.whl; do unzip $f; done; rm -r *.dist-info\n"
 
-    print(m.format(sys.executable), "\nspecifically,", ex)
-    sys.exit(1)
+    if m:
+        t = " when not running with '-h' or url '-'"
+        print(m.format(t, sys.executable), "\nspecifically,", ex)
+        sys.exit(1)
 
 
 # from copyparty/__init__.py
@@ -75,16 +82,40 @@ else:
 VT100 = platform.system() != "Windows"
 
 
-req_ses = requests.Session()
+try:
+    UTC = datetime.timezone.utc
+except:
+    TD_ZERO = datetime.timedelta(0)
+
+    class _UTC(datetime.tzinfo):
+        def utcoffset(self, dt):
+            return TD_ZERO
+
+        def tzname(self, dt):
+            return "UTC"
+
+        def dst(self, dt):
+            return TD_ZERO
+
+    UTC = _UTC()
 
 
 class Daemon(threading.Thread):
     def __init__(self, target, name=None, a=None):
-        # type: (Any, Any, Any) -> None
-        threading.Thread.__init__(self, target=target, args=a or (), name=name)
+        threading.Thread.__init__(self, name=name)
+        self.a = a or ()
+        self.fun = target
         self.daemon = True
         self.start()
 
+    def run(self):
+        try:
+            signal.pthread_sigmask(signal.SIG_BLOCK, [signal.SIGINT, signal.SIGTERM])
+        except:
+            pass
+
+        self.fun(*self.a)
+
 
 class File(object):
     """an up2k upload task; represents a single file"""
@@ -101,16 +132,22 @@ class File(object):
         # set by get_hashlist
         self.cids = []  # type: list[tuple[str, int, int]]  # [ hash, ofs, sz ]
         self.kchunks = {}  # type: dict[str, tuple[int, int]]  # hash: [ ofs, sz ]
+        self.t_hash = 0.0  # type: float
 
         # set by handshake
         self.recheck = False  # duplicate; redo handshake after all files done
         self.ucids = []  # type: list[str]  # chunks which need to be uploaded
-        self.wark = None  # type: str
-        self.url = None  # type: str
+        self.wark = ""  # type: str
+        self.url = ""  # type: str
+        self.nhs = 0  # type: int
 
         # set by upload
+        self.t0_up = 0.0  # type: float
+        self.t1_up = 0.0  # type: float
+        self.nojoin = 0  # type: int
         self.up_b = 0  # type: int
         self.up_c = 0  # type: int
+        self.cd = 0  # type: int
 
         # t = "size({}) lmod({}) top({}) rel({}) abs({}) name({})\n"
         # eprint(t.format(self.size, self.lmod, self.top, self.rel, self.abs, self.name))
@@ -119,10 +156,20 @@ class File(object):
 class FileSlice(object):
     """file-like object providing a fixed window into a file"""
 
-    def __init__(self, file, cid):
+    def __init__(self, file, cids):
         # type: (File, str) -> None
 
-        self.car, self.len = file.kchunks[cid]
+        self.file = file
+        self.cids = cids
+
+        self.car, tlen = file.kchunks[cids[0]]
+        for cid in cids[1:]:
+            ofs, clen = file.kchunks[cid]
+            if ofs != self.car + tlen:
+                raise Exception(9)
+            tlen += clen
+
+        self.len = tlen
         self.cdr = self.car + self.len
         self.ofs = 0  # type: int
         self.f = open(file.abs, "rb", 512 * 1024)
@@ -221,6 +268,7 @@ class MTHash(object):
 
     def hash_at(self, nch):
         f = self.f
+        assert f
         ofs = ofs0 = nch * self.csz
         hashobj = hashlib.sha512()
         chunk_sz = chunk_rem = min(self.csz, self.sz - ofs)
@@ -244,6 +292,12 @@ class MTHash(object):
 _print = print
 
 
+def safe_print(*a, **ka):
+    ka["end"] = ""
+    zs = " ".join([unicode(x) for x in a])
+    _print(zs + "\n", **ka)
+
+
 def eprint(*a, **ka):
     ka["file"] = sys.stderr
     ka["end"] = ""
@@ -257,18 +311,17 @@ def eprint(*a, **ka):
 
 def flushing_print(*a, **ka):
     try:
-        _print(*a, **ka)
+        safe_print(*a, **ka)
     except:
         v = " ".join(str(x) for x in a)
         v = v.encode("ascii", "replace").decode("ascii")
-        _print(v, **ka)
+        safe_print(v, **ka)
 
     if "flush" not in ka:
         sys.stdout.flush()
 
 
-if not VT100:
-    print = flushing_print
+print = safe_print if VT100 else flushing_print
 
 
 def termsize():
@@ -345,7 +398,7 @@ def undns(url):
     usp = urlsplit(url)
     hn = usp.hostname
     gai = None
-    eprint("resolving host [{0}] ...".format(hn), end="")
+    eprint("resolving host [%s] ..." % (hn,))
     try:
         gai = socket.getaddrinfo(hn, None)
         hn = gai[0][4][0]
@@ -363,7 +416,7 @@ def undns(url):
 
     usp = usp._replace(netloc=hn)
     url = urlunsplit(usp)
-    eprint(" {0}".format(url))
+    eprint(" %s\n" % (url,))
     return url
 
 
@@ -433,7 +486,7 @@ def walkdirs(err, tops, excl):
         za = [x.replace(b"/", b"\\") for x in za]
         tops = za
 
-    ptn = re.compile(excl.encode("utf-8") or b"\n")
+    ptn = re.compile(excl.encode("utf-8") or b"\n", re.I)
 
     for top in tops:
         isdir = os.path.isdir(top)
@@ -461,7 +514,7 @@ def quotep(btxt):
     if not PY2:
         quot1 = quot1.encode("ascii")
 
-    return quot1.replace(b" ", b"+")
+    return quot1.replace(b" ", b"+")  # type: ignore
 
 
 # from copyparty/util.py
@@ -498,7 +551,7 @@ def up2k_chunksize(filesize):
 
 # mostly from copyparty/up2k.py
 def get_hashlist(file, pcb, mth):
-    # type: (File, any, any) -> None
+    # type: (File, Any, Any) -> None
     """generates the up2k hashlist from file contents, inserts it into `file`"""
 
     chunk_sz = up2k_chunksize(file.size)
@@ -506,6 +559,8 @@ def get_hashlist(file, pcb, mth):
     file_ofs = 0
     ret = []
     with open(file.abs, "rb", 512 * 1024) as f:
+        t0 = time.time()
+
         if mth and file.size >= 1024 * 512:
             ret = mth.hash(f, file.size, chunk_sz, pcb, file)
             file_rem = 0
@@ -532,10 +587,12 @@ def get_hashlist(file, pcb, mth):
             if pcb:
                 pcb(file, file_ofs)
 
+    file.t_hash = time.time() - t0
     file.cids = ret
     file.kchunks = {}
     for k, v1, v2 in ret:
-        file.kchunks[k] = [v1, v2]
+        if k not in file.kchunks:
+            file.kchunks[k] = [v1, v2]
 
 
 def handshake(ar, file, search):
@@ -557,8 +614,11 @@ def handshake(ar, file, search):
     }
     if search:
         req["srch"] = 1
-    elif ar.dr:
-        req["replace"] = True
+    else:
+        if ar.touch:
+            req["umod"] = True
+        if ar.ow:
+            req["replace"] = True
 
     headers = {"Content-Type": "text/plain"}  # <=1.5.1 compat
     if pw:
@@ -574,7 +634,8 @@ def handshake(ar, file, search):
         sc = 600
         txt = ""
         try:
-            r = req_ses.post(url, headers=headers, json=req)
+            zs = json.dumps(req, separators=(",\n", ": "))
+            r = req_ses.post(url, headers=headers, data=zs)
             sc = r.status_code
             txt = r.text
             if sc < 400:
@@ -598,7 +659,7 @@ def handshake(ar, file, search):
                 raise
 
             eprint("handshake failed, retrying: {0}\n  {1}\n\n".format(file.name, em))
-            time.sleep(1)
+            time.sleep(ar.cd)
 
     try:
         r = r.json()
@@ -621,13 +682,20 @@ def handshake(ar, file, search):
     return r["hash"], r["sprs"]
 
 
-def upload(file, cid, pw, stats):
-    # type: (File, str, str, str) -> None
-    """upload one specific chunk, `cid` (a chunk-hash)"""
+def upload(fsl, pw, stats):
+    # type: (FileSlice, str, str) -> None
+    """upload a range of file data, defined by one or more `cid` (chunk-hash)"""
+
+    ctxt = fsl.cids[0]
+    if len(fsl.cids) > 1:
+        n = 192 // len(fsl.cids)
+        n = 9 if n > 9 else 2 if n < 2 else n
+        zsl = [zs[:n] for zs in fsl.cids[1:]]
+        ctxt += ",%d,%s" % (n, "".join(zsl))
 
     headers = {
-        "X-Up2k-Hash": cid,
-        "X-Up2k-Wark": file.wark,
+        "X-Up2k-Hash": ctxt,
+        "X-Up2k-Wark": fsl.file.wark,
         "Content-Type": "application/octet-stream",
     }
 
@@ -637,15 +705,24 @@ def upload(file, cid, pw, stats):
     if pw:
         headers["Cookie"] = "=".join(["cppwd", pw])
 
-    f = FileSlice(file, cid)
     try:
-        r = req_ses.post(file.url, headers=headers, data=f)
+        r = req_ses.post(fsl.file.url, headers=headers, data=fsl)
+
+        if r.status_code == 400:
+            txt = r.text
+            if (
+                "already being written" in txt
+                or "already got that" in txt
+                or "only sibling chunks" in txt
+            ):
+                fsl.file.nojoin = 1
+
         if not r:
             raise Exception(repr(r))
 
         _ = r.content
     finally:
-        f.f.close()
+        fsl.f.close()
 
 
 class Ctl(object):
@@ -689,6 +766,7 @@ class Ctl(object):
 
     def __init__(self, ar, stats=None):
         self.ok = False
+        self.errs = 0
         self.ar = ar
         self.stats = stats or self._scan()
         if not self.stats:
@@ -708,6 +786,9 @@ class Ctl(object):
         if ar.safe:
             self._safe()
         else:
+            self.at_hash = 0.0
+            self.at_up = 0.0
+            self.at_upr = 0.0
             self.hash_f = 0
             self.hash_c = 0
             self.hash_b = 0
@@ -715,8 +796,6 @@ class Ctl(object):
             self.up_c = 0
             self.up_b = 0
             self.up_br = 0
-            self.hasher_busy = 1
-            self.handshaker_busy = 0
             self.uploader_busy = 0
             self.serialized = False
 
@@ -726,8 +805,11 @@ class Ctl(object):
             self.eta = "99:99:99"
 
             self.mutex = threading.Lock()
+            self.exit_cond = threading.Condition()
+            self.uploader_alive = ar.j
+            self.handshaker_alive = ar.j
             self.q_handshake = Queue()  # type: Queue[File]
-            self.q_upload = Queue()  # type: Queue[tuple[File, str]]
+            self.q_upload = Queue()  # type: Queue[FileSlice]
 
             self.st_hash = [None, "(idle, starting...)"]  # type: tuple[File, int]
             self.st_up = [None, "(idle, starting...)"]  # type: tuple[File, int]
@@ -736,7 +818,7 @@ class Ctl(object):
 
             self._fancy()
 
-        self.ok = True
+        self.ok = not self.errs
 
     def _safe(self):
         """minimal basic slow boring fallback codepath"""
@@ -772,7 +854,8 @@ class Ctl(object):
                 for nc, cid in enumerate(hs):
                     print("  {0} up {1}".format(ncs - nc, cid))
                     stats = "{0}/0/0/{1}".format(nf, self.nfiles - nf)
-                    upload(file, cid, self.ar.a, stats)
+                    fslice = FileSlice(file, [cid])
+                    upload(fslice, self.ar.a, stats)
 
             print("  ok!")
             if file.recheck:
@@ -781,7 +864,7 @@ class Ctl(object):
         if not self.recheck:
             return
 
-        eprint("finalizing {0} duplicate files".format(len(self.recheck)))
+        eprint("finalizing %d duplicate files\n" % (len(self.recheck),))
         for file in self.recheck:
             handshake(self.ar, file, search)
 
@@ -795,27 +878,21 @@ class Ctl(object):
             Daemon(self.handshaker)
             Daemon(self.uploader)
 
-        idles = 0
-        while idles < 3:
-            time.sleep(0.07)
+        while True:
+            with self.exit_cond:
+                self.exit_cond.wait(0.07)
             with self.mutex:
-                if (
-                    self.q_handshake.empty()
-                    and self.q_upload.empty()
-                    and not self.hasher_busy
-                    and not self.handshaker_busy
-                    and not self.uploader_busy
-                ):
-                    idles += 1
-                else:
-                    idles = 0
+                if not self.handshaker_alive and not self.uploader_alive:
+                    break
+                st_hash = self.st_hash[:]
+                st_up = self.st_up[:]
 
             if VT100 and not self.ar.ns:
                 maxlen = ss.w - len(str(self.nfiles)) - 14
                 txt = "\033[s\033[{0}H".format(ss.g)
                 for y, k, st, f in [
-                    [0, "hash", self.st_hash, self.hash_f],
-                    [1, "send", self.st_up, self.up_f],
+                    [0, "hash", st_hash, self.hash_f],
+                    [1, "send", st_up, self.up_f],
                 ]:
                     txt += "\033[{0}H{1}:".format(ss.g + y, k)
                     file, arg = st
@@ -839,12 +916,12 @@ class Ctl(object):
                 txt = " "
 
             if not self.up_br:
-                spd = self.hash_b / (time.time() - self.t0)
-                eta = (self.nbytes - self.hash_b) / (spd + 1)
+                spd = self.hash_b / ((time.time() - self.t0) or 1)
+                eta = (self.nbytes - self.hash_b) / (spd or 1)
             else:
-                spd = self.up_br / (time.time() - self.t0_up)
+                spd = self.up_br / ((time.time() - self.t0_up) or 1)
                 spd = self.spd = (self.spd or spd) * 0.9 + spd * 0.1
-                eta = (self.nbytes - self.up_b) / (spd + 1)
+                eta = (self.nbytes - self.up_b) / (spd or 1)
 
             spd = humansize(spd)
             self.eta = str(datetime.timedelta(seconds=int(eta)))
@@ -855,10 +932,17 @@ class Ctl(object):
             t = "{0} eta @ {1}/s, {2}, {3}# left".format(self.eta, spd, sleft, nleft)
             eprint(txt + "\033]0;{0}\033\\\r{0}{1}".format(t, tail))
 
+        if self.hash_b and self.at_hash:
+            spd = humansize(self.hash_b / self.at_hash)
+            eprint("\nhasher: %.2f sec, %s/s\n" % (self.at_hash, spd))
+        if self.up_b and self.at_up:
+            spd = humansize(self.up_b / self.at_up)
+            eprint("upload: %.2f sec, %s/s\n" % (self.at_up, spd))
+
         if not self.recheck:
             return
 
-        eprint("finalizing {0} duplicate files".format(len(self.recheck)))
+        eprint("finalizing %d duplicate files\n" % (len(self.recheck),))
         for file in self.recheck:
             handshake(self.ar, file, False)
 
@@ -870,6 +954,8 @@ class Ctl(object):
         self.st_hash = [file, ofs]
 
     def hasher(self):
+        ptn = re.compile(self.ar.x.encode("utf-8"), re.I) if self.ar.x else None
+        sep = "{0}".format(os.sep).encode("ascii")
         prd = None
         ls = {}
         for top, rel, inf in self.filegen:
@@ -902,13 +988,29 @@ class Ctl(object):
                     if self.ar.drd:
                         dp = os.path.join(top, rd)
                         lnodes = set(os.listdir(dp))
-                        bnames = [x for x in ls if x not in lnodes]
-                        if bnames:
-                            vpath = self.ar.url.split("://")[-1].split("/", 1)[-1]
-                            names = [x.decode("utf-8", "replace") for x in bnames]
-                            locs = [vpath + srd + "/" + x for x in names]
-                            print("DELETING ~{0}/#{1}".format(srd, len(names)))
-                            req_ses.post(self.ar.url + "?delete", json=locs)
+                        if ptn:
+                            zs = dp.replace(sep, b"/").rstrip(b"/") + b"/"
+                            zls = [zs + x for x in lnodes]
+                            zls = [x for x in zls if not ptn.match(x)]
+                            lnodes = [x.split(b"/")[-1] for x in zls]
+                        bnames = [x for x in ls if x not in lnodes and x != b".hist"]
+                        vpath = self.ar.url.split("://")[-1].split("/", 1)[-1]
+                        names = [x.decode("utf-8", "replace") for x in bnames]
+                        locs = [vpath + srd + "/" + x for x in names]
+                        while locs:
+                            req = locs
+                            while req:
+                                print("DELETING ~%s/#%s" % (srd, len(req)))
+                                r = req_ses.post(self.ar.url + "?delete", json=req)
+                                if r.status_code == 413 and "json 2big" in r.text:
+                                    print(" (delete request too big; slicing...)")
+                                    req = req[: len(req) // 2]
+                                    continue
+                                elif not r:
+                                    t = "delete request failed: %r %s"
+                                    raise Exception(t % (r, r.text))
+                                break
+                            locs = locs[len(req) :]
 
             if isdir:
                 continue
@@ -946,11 +1048,42 @@ class Ctl(object):
                 self.hash_f += 1
                 self.hash_c += len(file.cids)
                 self.hash_b += file.size
+                if self.ar.wlist:
+                    self.up_f = self.hash_f
+                    self.up_c = self.hash_c
+                    self.up_b = self.hash_b
+
+            if self.ar.wlist:
+                zsl = [self.ar.wsalt, str(file.size)] + [x[0] for x in file.kchunks]
+                zb = hashlib.sha512("\n".join(zsl).encode("utf-8")).digest()[:33]
+                wark = base64.urlsafe_b64encode(zb).decode("utf-8")
+                vp = file.rel.decode("utf-8")
+                if self.ar.jw:
+                    print("%s  %s" % (wark, vp))
+                else:
+                    zd = datetime.datetime.fromtimestamp(file.lmod, UTC)
+                    dt = "%04d-%02d-%02d %02d:%02d:%02d" % (
+                        zd.year,
+                        zd.month,
+                        zd.day,
+                        zd.hour,
+                        zd.minute,
+                        zd.second,
+                    )
+                    print("%s %12d %s %s" % (dt, file.size, wark, vp))
+                continue
 
             self.q_handshake.put(file)
 
-        self.hasher_busy = 0
         self.st_hash = [None, "(finished)"]
+        self._check_if_done()
+
+    def _check_if_done(self):
+        with self.mutex:
+            if self.nfiles - self.up_f:
+                return
+        for _ in range(self.ar.j):
+            self.q_handshake.put(None)
 
     def handshaker(self):
         search = self.ar.s
@@ -958,31 +1091,39 @@ class Ctl(object):
         while True:
             file = self.q_handshake.get()
             if not file:
+                with self.mutex:
+                    self.handshaker_alive -= 1
                 self.q_upload.put(None)
-                break
-
-            with self.mutex:
-                self.handshaker_busy += 1
+                return
 
             upath = file.abs.decode("utf-8", "replace")
             if not VT100:
                 upath = upath.lstrip("\\?")
 
+            file.nhs += 1
+            if file.nhs > 32:
+                print("ERROR: giving up on file %s" % (upath))
+                self.errs += 1
+                continue
+
+            while time.time() < file.cd:
+                time.sleep(0.1)
+
             hs, sprs = handshake(self.ar, file, search)
             if search:
                 if hs:
                     for hit in hs:
-                        t = "found: {0}\n  {1}{2}\n"
-                        print(t.format(upath, burl, hit["rp"]), end="")
+                        t = "found: {0}\n  {1}{2}"
+                        print(t.format(upath, burl, hit["rp"]))
                 else:
-                    print("NOT found: {0}\n".format(upath), end="")
+                    print("NOT found: {0}".format(upath))
 
                 with self.mutex:
                     self.up_f += 1
                     self.up_c += len(file.cids)
                     self.up_b += file.size
-                    self.handshaker_busy -= 1
 
+                self._check_if_done()
                 continue
 
             if file.recheck:
@@ -1014,57 +1155,110 @@ class Ctl(object):
                         file.up_b -= sz
 
                 file.ucids = hs
-                self.handshaker_busy -= 1
 
             if not hs:
-                kw = "uploaded" if file.up_b else "   found"
-                print("{0} {1}".format(kw, upath))
-            for cid in hs:
-                self.q_upload.put([file, cid])
+                self.at_hash += file.t_hash
+
+                if self.ar.spd:
+                    if VT100:
+                        c1 = "\033[36m"
+                        c2 = "\033[0m"
+                    else:
+                        c1 = c2 = ""
+
+                    spd_h = humansize(file.size / file.t_hash, True)
+                    if file.up_b:
+                        t_up = file.t1_up - file.t0_up
+                        spd_u = humansize(file.size / t_up, True)
+
+                        t = "uploaded %s %s(h:%.2fs,%s/s,up:%.2fs,%s/s)%s"
+                        print(t % (upath, c1, file.t_hash, spd_h, t_up, spd_u, c2))
+                    else:
+                        t = "   found %s %s(%.2fs,%s/s)%s"
+                        print(t % (upath, c1, file.t_hash, spd_h, c2))
+                else:
+                    kw = "uploaded" if file.up_b else "   found"
+                    print("{0} {1}".format(kw, upath))
+
+                self._check_if_done()
+                continue
+
+            chunksz = up2k_chunksize(file.size)
+            njoin = (self.ar.sz * 1024 * 1024) // chunksz
+            cs = hs[:]
+            while cs:
+                fsl = FileSlice(file, cs[:1])
+                try:
+                    if file.nojoin:
+                        raise Exception()
+                    for n in range(2, min(len(cs), njoin + 1)):
+                        fsl = FileSlice(file, cs[:n])
+                except:
+                    pass
+                cs = cs[len(fsl.cids) :]
+                self.q_upload.put(fsl)
 
     def uploader(self):
         while True:
-            task = self.q_upload.get()
-            if not task:
-                self.st_up = [None, "(finished)"]
-                break
+            fsl = self.q_upload.get()
+            if not fsl:
+                done = False
+                with self.mutex:
+                    self.uploader_alive -= 1
+                    if not self.uploader_alive:
+                        done = not self.handshaker_alive
+                        self.st_up = [None, "(finished)"]
+                if done:
+                    with self.exit_cond:
+                        self.exit_cond.notify_all()
+                return
+
+            file = fsl.file
+            cids = fsl.cids
 
             with self.mutex:
+                if not self.uploader_busy:
+                    self.at_upr = time.time()
                 self.uploader_busy += 1
-                self.t0_up = self.t0_up or time.time()
+                if not file.t0_up:
+                    file.t0_up = time.time()
+                    if not self.t0_up:
+                        self.t0_up = file.t0_up
 
-            zs = "{0}/{1}/{2}/{3} {4}/{5} {6}"
-            stats = zs.format(
+            stats = "%d/%d/%d/%d %d/%d %s" % (
                 self.up_f,
                 len(self.recheck),
                 self.uploader_busy,
                 self.nfiles - self.up_f,
-                int(self.nbytes / (1024 * 1024)),
-                int((self.nbytes - self.up_b) / (1024 * 1024)),
+                self.nbytes // (1024 * 1024),
+                (self.nbytes - self.up_b) // (1024 * 1024),
                 self.eta,
             )
 
-            file, cid = task
             try:
-                upload(file, cid, self.ar.a, stats)
+                upload(fsl, self.ar.a, stats)
             except Exception as ex:
-                t = "upload failed, retrying: {0} #{1} ({2})\n"
-                eprint(t.format(file.name, cid[:8], ex))
+                t = "upload failed, retrying: %s #%s+%d (%s)\n"
+                eprint(t % (file.name, cids[0][:8], len(cids) - 1, ex))
+                file.cd = time.time() + self.ar.cd
                 # handshake will fix it
 
             with self.mutex:
-                sz = file.kchunks[cid][1]
-                file.ucids = [x for x in file.ucids if x != cid]
+                sz = fsl.len
+                file.ucids = [x for x in file.ucids if x not in cids]
                 if not file.ucids:
+                    file.t1_up = time.time()
                     self.q_handshake.put(file)
 
-                self.st_up = [file, cid]
+                self.st_up = [file, cids[0]]
                 file.up_b += sz
                 self.up_b += sz
                 self.up_br += sz
                 file.up_c += 1
                 self.up_c += 1
                 self.uploader_busy -= 1
+                if not self.uploader_busy:
+                    self.at_up += time.time() - self.at_upr
 
     def up_done(self, file):
         if self.ar.dl:
@@ -1101,10 +1295,13 @@ source file/folder selection uses rsync syntax, meaning that:
     ap.add_argument("url", type=unicode, help="server url, including destination folder")
     ap.add_argument("files", type=unicode, nargs="+", help="files and/or folders to process")
     ap.add_argument("-v", action="store_true", help="verbose")
-    ap.add_argument("-a", metavar="PASSWORD", help="password or $filepath")
+    ap.add_argument("-a", metavar="PASSWD", help="password or $filepath")
     ap.add_argument("-s", action="store_true", help="file-search (disables upload)")
-    ap.add_argument("-x", type=unicode, metavar="REGEX", default="", help="skip file if filesystem-abspath matches REGEX, example: '.*/\.hist/.*'")
+    ap.add_argument("-x", type=unicode, metavar="REGEX", action="append", help="skip file if filesystem-abspath matches REGEX (option can be repeated), example: '.*/\\.hist/.*'")
     ap.add_argument("--ok", action="store_true", help="continue even if some local files are inaccessible")
+    ap.add_argument("--touch", action="store_true", help="if last-modified timestamps differ, push local to server (need write+delete perms)")
+    ap.add_argument("--ow", action="store_true", help="overwrite existing files instead of autorenaming")
+    ap.add_argument("--spd", action="store_true", help="print speeds for each file")
     ap.add_argument("--version", action="store_true", help="show version and exit")
 
     ap = app.add_argument_group("compatibility")
@@ -1113,19 +1310,25 @@ source file/folder selection uses rsync syntax, meaning that:
 
     ap = app.add_argument_group("folder sync")
     ap.add_argument("--dl", action="store_true", help="delete local files after uploading")
-    ap.add_argument("--dr", action="store_true", help="delete remote files which don't exist locally")
+    ap.add_argument("--dr", action="store_true", help="delete remote files which don't exist locally (implies --ow)")
     ap.add_argument("--drd", action="store_true", help="delete remote files during upload instead of afterwards; reduces peak disk space usage, but will reupload instead of detecting renames")
 
+    ap = app.add_argument_group("file-ID calculator; enable with url '-' to list warks (file identifiers) instead of upload/search")
+    ap.add_argument("--wsalt", type=unicode, metavar="S", default="hunter2", help="salt to use when creating warks; must match server config")
+    ap.add_argument("--jw", action="store_true", help="just identifier+filepath, not mtime/size too")
+
     ap = app.add_argument_group("performance tweaks")
-    ap.add_argument("-j", type=int, metavar="THREADS", default=4, help="parallel connections")
-    ap.add_argument("-J", type=int, metavar="THREADS", default=hcores, help="num cpu-cores to use for hashing; set 0 or 1 for single-core hashing")
+    ap.add_argument("-j", type=int, metavar="CONNS", default=2, help="parallel connections")
+    ap.add_argument("-J", type=int, metavar="CORES", default=hcores, help="num cpu-cores to use for hashing; set 0 or 1 for single-core hashing")
+    ap.add_argument("--sz", type=int, metavar="MiB", default=64, help="try to make each POST this big")
     ap.add_argument("-nh", action="store_true", help="disable hashing while uploading")
     ap.add_argument("-ns", action="store_true", help="no status panel (for slow consoles and macos)")
+    ap.add_argument("--cd", type=float, metavar="SEC", default=5, help="delay before reattempting a failed handshake/upload")
     ap.add_argument("--safe", action="store_true", help="use simple fallback approach")
     ap.add_argument("-z", action="store_true", help="ZOOMIN' (skip uploading files if they exist at the destination with the ~same last-modified timestamp, so same as yolo / turbo with date-chk but even faster)")
 
     ap = app.add_argument_group("tls")
-    ap.add_argument("-te", metavar="PEM_FILE", help="certificate to expect/verify")
+    ap.add_argument("-te", metavar="PATH", help="path to ca.pem or cert.pem to expect/verify")
     ap.add_argument("-td", action="store_true", help="disable certificate check")
     # fmt: on
 
@@ -1142,7 +1345,14 @@ source file/folder selection uses rsync syntax, meaning that:
     if ar.drd:
         ar.dr = True
 
-    for k in "dl dr drd".split():
+    if ar.dr:
+        ar.ow = True
+
+    ar.x = "|".join(ar.x or [])
+
+    setattr(ar, "wlist", ar.url == "-")
+
+    for k in "dl dr drd wlist".split():
         errs = []
         if ar.safe and getattr(ar, k):
             errs.append(k)
@@ -1160,6 +1370,14 @@ source file/folder selection uses rsync syntax, meaning that:
     if "://" not in ar.url:
         ar.url = "http://" + ar.url
 
+    if "https://" in ar.url.lower():
+        try:
+            import ssl, zipfile
+        except:
+            t = "ERROR: https is not available for some reason; please use http"
+            print("\n\n   %s\n\n" % (t,))
+            raise
+
     if ar.a and ar.a.startswith("$"):
         fn = ar.a[1:]
         print("reading password from file [{0}]".format(fn))
@@ -1177,7 +1395,7 @@ source file/folder selection uses rsync syntax, meaning that:
                 raise
 
     if ar.cls:
-        eprint("\x1b\x5b\x48\x1b\x5b\x32\x4a\x1b\x5b\x33\x4a", end="")
+        eprint("\033[H\033[2J\033[3J", end="")
 
     ctl = Ctl(ar)
 
@@ -1187,6 +1405,9 @@ source file/folder selection uses rsync syntax, meaning that:
         ar.z = True
         ctl = Ctl(ar, ctl.stats)
 
+    if ctl.errs:
+        print("WARNING: %d errors" % (ctl.errs))
+
     sys.exit(0 if ctl.ok else 1)
 
 

bin/unforget.py

@@ -66,7 +66,7 @@ def main():
                 ofs = ln.find("{")
                 j = json.loads(ln[ofs:])
             except:
-                pass
+                continue
 
             w = j["wark"]
             if db.execute("select w from up where w = ?", (w,)).fetchone():

contrib/README.md

@@ -16,11 +16,13 @@
 * sharex config file to upload screenshots and grab the URL
 * `RequestURL`: full URL to the target folder
 * `pw`: password (remove the `pw` line if anon-write)
+* the `act:bput` thing is optional since copyparty v1.9.29
+* using an older sharex version, maybe sharex v12.1.1 for example? dw fam i got your back 👉😎👉 [`sharex12.sxcu`](sharex12.sxcu)
 
-however if your copyparty is behind a reverse-proxy, you may want to use [`sharex-html.sxcu`](sharex-html.sxcu) instead:
-* `RequestURL`: full URL to the target folder
-* `URL`: full URL to the root folder (with trailing slash) followed by `$regex:1|1$`
-* `pw`: password (remove `Parameters` if anon-write)
+### [`send-to-cpp.contextlet.json`](send-to-cpp.contextlet.json)
+* browser integration, kind of? custom rightclick actions and stuff
+* rightclick a pic and send it to copyparty straight from your browser
+* for the [contextlet](https://addons.mozilla.org/en-US/firefox/addon/contextlets/) firefox extension
 
 ### [`media-osd-bgone.ps1`](media-osd-bgone.ps1)
 * disables the [windows OSD popup](https://user-images.githubusercontent.com/241032/122821375-0e08df80-d2dd-11eb-9fd9-184e8aacf1d0.png) (the thing on the left) which appears every time you hit media hotkeys to adjust volume or change song while playing music with the copyparty web-ui, or most other audio players really

contrib/index.html

@@ -26,8 +26,8 @@ a {
 	<script>
 
 var a = document.getElementById('redir'),
-	proto = window.location.protocol.indexOf('https') === 0 ? 'https' : 'http',
-	loc = window.location.hostname || '127.0.0.1',
+	proto = location.protocol.indexOf('https') === 0 ? 'https' : 'http',
+	loc = location.hostname || '127.0.0.1',
 	port = a.getAttribute('href').split(':').pop().split('/')[0],
 	url = proto + '://' + loc + ':' + port + '/';
 
@@ -35,7 +35,7 @@ a.setAttribute('href', url);
 document.getElementById('desc').innerHTML = 'redirecting to';
 
 setTimeout(function() {
-	window.location.href = url;
+	location.href = url;
 }, 500);
 
 </script>

contrib/nginx/copyparty.conf

@@ -11,9 +11,17 @@
 # (5'000 requests per second, or 20gbps upload/download in parallel)
 #
 # on fedora/rhel, remember to setsebool -P httpd_can_network_connect 1
+#
+# if you are behind cloudflare (or another protection service),
+# remember to reject all connections which are not coming from your
+# protection service -- for cloudflare in particular, you can
+# generate the list of permitted IP ranges like so:
+#   (curl -s https://www.cloudflare.com/ips-v{4,6} | sed 's/^/allow /; s/$/;/'; echo; echo "deny all;") > /etc/nginx/cloudflare-only.conf
+#
+# and then enable it below by uncomenting the cloudflare-only.conf line
 
 upstream cpp {
-	server 127.0.0.1:3923;
+	server 127.0.0.1:3923 fail_timeout=1s;
 	keepalive 1;
 }
 server {
@@ -21,7 +29,10 @@ server {
 	listen [::]:443 ssl;
 
 	server_name fs.example.com;
-	
+
+	# uncomment the following line to reject non-cloudflare connections, ensuring client IPs cannot be spoofed:
+	#include /etc/nginx/cloudflare-only.conf;
+
 	location / {
 		proxy_pass http://cpp;
 		proxy_redirect off;

contrib/nixos/modules/copyparty.nix

@@ -138,7 +138,8 @@ in {
                 "d" (delete): permanently delete files and folders
                 "g" (get):    download files, but cannot see folder contents
                 "G" (upget):  "get", but can see filekeys of their own uploads
-                "a" (upget):  can see uploader IPs, config-reload
+                "h" (html):   "get", but folders return their index.html
+                "a" (admin):  can see uploader IPs, config-reload
 
               For example: "rwmd"
 

contrib/package/arch/PKGBUILD

@@ -1,14 +1,15 @@
 # Maintainer: icxes <dev.null@need.moe>
 pkgname=copyparty
-pkgver="1.9.3"
+pkgver="1.14.4"
 pkgrel=1
-pkgdesc="Portable file sharing hub"
+pkgdesc="File server with accelerated resumable uploads, dedup, WebDAV, FTP, TFTP, zeroconf, media indexer, thumbnails++"
 arch=("any")
 url="https://github.com/9001/${pkgname}"
 license=('MIT')
 depends=("python" "lsof" "python-jinja")
 makedepends=("python-wheel" "python-setuptools" "python-build" "python-installer" "make" "pigz")
 optdepends=("ffmpeg: thumbnails for videos, images (slower) and audio, music tags"
+            "cfssl: generate TLS certificates on startup (pointless when reverse-proxied)"
             "python-mutagen: music tags (alternative)" 
             "python-pillow: thumbnails for images" 
             "python-pyvips: thumbnails for images (higher quality, faster, uses more ram)" 
@@ -20,7 +21,7 @@ optdepends=("ffmpeg: thumbnails for videos, images (slower) and audio, music tag
 )
 source=("https://github.com/9001/${pkgname}/releases/download/v${pkgver}/${pkgname}-${pkgver}.tar.gz")
 backup=("etc/${pkgname}.d/init" )
-sha256sums=("87db55a57adf14b3b875c72d94b5df67560abc6dbfc104104e0c76d7f02848b6")
+sha256sums=("1e8004e4369e59487c47a0a9949668de704b1884beda0421887e342edcff0961")
 
 build() {
     cd "${srcdir}/${pkgname}-${pkgver}"

contrib/package/arch/prisonparty.service

@@ -1,11 +1,11 @@
 # this will start `/usr/bin/copyparty-sfx.py`
-# in a chroot, preventing accidental access elsewhere
-# and read config from `/etc/copyparty.d/*.conf`
+# in a chroot, preventing accidental access elsewhere,
+# and read copyparty config from `/etc/copyparty.d/*.conf`
 #
 # expose additional filesystem locations to copyparty
-#   by listing them between the last `1000` and `--`
+#   by listing them between the last `cpp` and `--`
 #
-# `1000 1000` = what user to run copyparty as
+# `cpp cpp` = user/group to run copyparty as; can be IDs (1000 1000)
 #
 # unless you add -q to disable logging, you may want to remove the
 #   following line to allow buffering (slightly better performance):
@@ -24,7 +24,9 @@ ExecReload=/bin/kill -s USR1 $MAINPID
 ExecStartPre=+/bin/bash -c 'mkdir -p /run/tmpfiles.d/ && echo "x /tmp/pe-copyparty*" > /run/tmpfiles.d/copyparty.conf'
 
 # run copyparty
-ExecStart=/bin/bash /usr/bin/prisonparty /var/lib/copyparty-jail 1000 1000 /etc/copyparty.d -- \
+ExecStart=/bin/bash /usr/bin/prisonparty /var/lib/copyparty-jail cpp cpp \
+  /etc/copyparty.d \
+  -- \
   /usr/bin/python3 /usr/bin/copyparty -c /etc/copyparty.d/init
 
 [Install]

contrib/package/nix/copyparty/default.nix

@@ -3,6 +3,9 @@
 # use argon2id-hashed passwords in config files (sha2 is always available)
 withHashedPasswords ? true,
 
+# generate TLS certificates on startup (pointless when reverse-proxied)
+withCertgen ? false,
+
 # create thumbnails with Pillow; faster than FFmpeg / MediaProcessing
 withThumbnails ? true,
 
@@ -34,6 +37,7 @@ let
     ]
     ++ lib.optional withSMB impacket
     ++ lib.optional withFTPS pyopenssl
+    ++ lib.optional withCertgen cfssl
     ++ lib.optional withThumbnails pillow
     ++ lib.optional withFastThumbnails pyvips
     ++ lib.optional withMediaProcessing ffmpeg

contrib/package/nix/copyparty/pin.json

@@ -1,5 +1,5 @@
 {
-    "url": "https://github.com/9001/copyparty/releases/download/v1.9.3/copyparty-sfx.py",
-    "version": "1.9.3",
-    "hash": "sha256-ufT7WARaj6nKaLX/r3X/ex/hMLMh1rtG0lkZHCm4Gu4="
+    "url": "https://github.com/9001/copyparty/releases/download/v1.14.4/copyparty-sfx.py",
+    "version": "1.14.4",
+    "hash": "sha256-nfcSXddM0jeHr7nGwoguOv/eq750bpHrImKmmODxK6E="
 }

contrib/plugins/README.md

@@ -20,6 +20,13 @@ point `--js-browser` to one of these by URL:
 
 
 
+## example any-js
+point `--js-browser` and/or `--js-other` to one of these by URL:
+
+* [`banner.js`](banner.js) shows a very enterprise [legal-banner](https://github.com/user-attachments/assets/8ae8e087-b209-449c-b08d-74e040f0284b)
+
+
+
 ## example browser-css
 point `--css-browser` to one of these by URL:
 

contrib/plugins/banner.js

@@ -0,0 +1,93 @@
+(function() {
+
+// usage: copy this to '.banner.js' in your webroot,
+// and run copyparty with the following arguments:
+// --js-browser /.banner.js  --js-other /.banner.js
+
+
+
+// had to pick the most chuuni one as the default
+var bannertext = '' +
+'<h3>You are accessing a U.S. Government (USG) Information System (IS) that is provided for USG-authorized use only.</h3>' +
+'<p>By using this IS (which includes any device attached to this IS), you consent to the following conditions:</p>' +
+'<ul>' +
+'<li>The USG routinely intercepts and monitors communications on this IS for purposes including, but not limited to, penetration testing, COMSEC monitoring, network operations and defense, personnel misconduct (PM), law enforcement (LE), and counterintelligence (CI) investigations.</li>' +
+'<li>At any time, the USG may inspect and seize data stored on this IS.</li>' +
+'<li>Communications using, or data stored on, this IS are not private, are subject to routine monitoring, interception, and search, and may be disclosed or used for any USG-authorized purpose.</li>' +
+'<li>This IS includes security measures (e.g., authentication and access controls) to protect USG interests -- not for your personal benefit or privacy.</li>' +
+'<li>Notwithstanding the above, using this IS does not constitute consent to PM, LE or CI investigative searching or monitoring of the content of privileged communications, or work product, related to personal representation or services by attorneys, psychotherapists, or clergy, and their assistants. Such communications and work product are private and confidential. See User Agreement for details.</li>' +
+'</ul>';
+
+
+
+// fancy div to insert into pages
+function bannerdiv(border) {
+    var ret = mknod('div', null, bannertext);
+    if (border)
+        ret.setAttribute("style", "border:1em solid var(--fg); border-width:.3em 0; margin:3em 0");
+    return ret;
+}
+
+
+
+// keep all of these false and then selectively enable them in the if-blocks below
+var show_msgbox = false,
+    login_top = false,
+    top = false,
+    bottom = false,
+    top_bordered = false,
+    bottom_bordered = false;
+
+if (QS("h1#cc") && QS("a#k")) {
+    // this is the controlpanel
+    // (you probably want to keep just one of these enabled)
+    show_msgbox = true;
+    login_top = true;
+    bottom = true;
+}
+else if (ebi("swin") && ebi("smac")) {
+    // this is the connect-page, same deal here
+    show_msgbox = true;
+    top_bordered = true;
+    bottom_bordered = true;
+}
+else if (ebi("op_cfg") || ebi("div#mw") ) {
+    // we're running in the main filebrowser (op_cfg) or markdown-viewer/editor (div#mw),
+    // fragile pages which break if you do something too fancy
+    show_msgbox = true;
+}
+
+
+
+// shows a fullscreen messagebox; works on all pages
+if (show_msgbox) {
+    var now = Math.floor(Date.now() / 1000),
+        last_shown = sread("bannerts") || 0;
+
+    // 60 * 60 * 17 = 17 hour cooldown
+    if (now - last_shown > 60 * 60 * 17) {
+        swrite("bannerts", now);
+        modal.confirm(bannertext, null, function () {
+            location = 'https://this-page-intentionally-left-blank.org/';
+        });
+    }
+}
+
+// show a message on the page footer; only works on the connect-page
+if (top || top_bordered) {
+    var dst = ebi('wrap');
+    dst.insertBefore(bannerdiv(top_bordered), dst.firstChild);
+}
+
+// show a message on the page footer; only works on the controlpanel and connect-page
+if (bottom || bottom_bordered) {
+    ebi('wrap').appendChild(bannerdiv(bottom_bordered));
+}
+
+// show a message on the top of the page; only works on the controlpanel
+if (login_top) {
+    var dst = QS('h1');
+    dst.parentNode.insertBefore(bannerdiv(false), dst);
+}
+
+})();

contrib/rc/copyparty

@@ -10,7 +10,7 @@ name="copyparty"
 rcvar="copyparty_enable"
 copyparty_user="copyparty"
 copyparty_args="-e2dsa -v /storage:/storage:r" # change as you see fit
-copyparty_command="/usr/local/bin/python3.8 /usr/local/copyparty/copyparty-sfx.py ${copyparty_args}"
+copyparty_command="/usr/local/bin/python3.9 /usr/local/copyparty/copyparty-sfx.py ${copyparty_args}"
 pidfile="/var/run/copyparty/${name}.pid"
 command="/usr/sbin/daemon"
 command_args="-P ${pidfile} -r -f ${copyparty_command}"

contrib/send-to-cpp.contextlet.json

@@ -0,0 +1,11 @@
+{
+    "code": "// https://addons.mozilla.org/en-US/firefox/addon/contextlets/\n// https://github.com/davidmhammond/contextlets\n\nvar url = 'http://partybox.local:3923/';\nvar pw = 'wark';\n\nvar xhr = new XMLHttpRequest();\nxhr.msg = this.info.linkUrl || this.info.srcUrl;\nxhr.open('POST', url, true);\nxhr.setRequestHeader('Content-Type', 'application/x-www-form-urlencoded;charset=UTF-8');\nxhr.setRequestHeader('PW', pw);\nxhr.send('msg=' + xhr.msg);\n",
+    "contexts": [
+        "link"
+    ],
+    "icons": null,
+    "patterns": "",
+    "scope": "background",
+    "title": "send to cpp",
+    "type": "normal"
+}

contrib/sharex-html.sxcu

@@ -1,19 +0,0 @@
-{
-  "Version": "13.5.0",
-  "Name": "copyparty-html",
-  "DestinationType": "ImageUploader",
-  "RequestMethod": "POST",
-  "RequestURL": "http://127.0.0.1:3923/sharex",
-  "Parameters": {
-    "pw": "wark"
-  },
-  "Body": "MultipartFormData",
-  "Arguments": {
-    "act": "bput"
-  },
-  "FileFormName": "f",
-  "RegexList": [
-    "bytes // <a href=\"/([^\"]+)\""
-  ],
-  "URL": "http://127.0.0.1:3923/$regex:1|1$"
-}

contrib/sharex.sxcu

@@ -1,17 +1,19 @@
 {
-  "Version": "13.5.0",
+  "Version": "15.0.0",
   "Name": "copyparty",
   "DestinationType": "ImageUploader",
   "RequestMethod": "POST",
   "RequestURL": "http://127.0.0.1:3923/sharex",
   "Parameters": {
-    "pw": "wark",
     "j": null
   },
+  "Headers": {
+    "pw": "PUT_YOUR_PASSWORD_HERE_MY_DUDE"
+  },
   "Body": "MultipartFormData",
   "Arguments": {
     "act": "bput"
   },
   "FileFormName": "f",
-  "URL": "$json:files[0].url$"
+  "URL": "{json:files[0].url}"
 }

contrib/sharex12.sxcu

@@ -0,0 +1,13 @@
+{
+  "Name": "copyparty",
+  "DestinationType": "ImageUploader, TextUploader, FileUploader",
+  "RequestURL": "http://127.0.0.1:3923/sharex",
+  "FileFormName": "f",
+  "Arguments": {
+    "act": "bput"
+  },
+  "Headers": {
+    "accept": "url",
+    "pw": "PUT_YOUR_PASSWORD_HERE_MY_DUDE"
+  }
+}

contrib/systemd/copyparty.conf

@@ -0,0 +1,42 @@
+# not actually YAML but lets pretend:
+# -*- mode: yaml -*-
+# vim: ft=yaml:
+
+
+# put this file in /etc/
+
+
+[global]
+  e2dsa  # enable file indexing and filesystem scanning
+  e2ts   # and enable multimedia indexing
+  ansi   # and colors in log messages
+
+  # disable logging to stdout/journalctl and log to a file instead;
+  # $LOGS_DIRECTORY is usually /var/log/copyparty (comes from systemd)
+  # and copyparty replaces %Y-%m%d with Year-MonthDay, so the
+  # full path will be something like /var/log/copyparty/2023-1130.txt
+  # (note: enable compression by adding .xz at the end)
+  q, lo: $LOGS_DIRECTORY/%Y-%m%d.log
+
+  # p: 80,443,3923   # listen on 80/443 as well (requires CAP_NET_BIND_SERVICE)
+  # i: 127.0.0.1     # only allow connections from localhost (reverse-proxies)
+  # ftp: 3921        # enable ftp server on port 3921
+  # p: 3939          # listen on another port
+  # df: 16           # stop accepting uploads if less than 16 GB free disk space
+  # ver              # show copyparty version in the controlpanel
+  # grid             # show thumbnails/grid-view by default
+  # theme: 2         # monokai
+  # name: datasaver  # change the server-name that's displayed in the browser
+  # stats, nos-dup   # enable the prometheus endpoint, but disable the dupes counter (too slow)
+  # no-robots, force-js  # make it harder for search engines to read your server
+
+
+[accounts]
+  ed: wark  # username: password
+
+
+[/]            # create a volume at "/" (the webroot), which will
+  /mnt         # share the contents of the "/mnt" folder
+  accs:
+    rw: *      # everyone gets read-write access, but
+    rwmda: ed  # the user "ed" gets read-write-move-delete-admin

contrib/systemd/copyparty.service

@@ -1,28 +1,34 @@
-# this will start `/usr/local/bin/copyparty-sfx.py`
-# and share '/mnt' with anonymous read+write
+# this will start `/usr/local/bin/copyparty-sfx.py` and
+# read copyparty config from `/etc/copyparty.conf`, for example:
+# https://github.com/9001/copyparty/blob/hovudstraum/contrib/systemd/copyparty.conf
 #
 # installation:
 #   wget https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.py -O /usr/local/bin/copyparty-sfx.py
-#   cp -pv copyparty.service /etc/systemd/system/
-#   restorecon -vr /etc/systemd/system/copyparty.service  # on fedora/rhel
-#   firewall-cmd --permanent --add-port={80,443,3923}/tcp  # --zone=libvirt
+#   useradd -r -s /sbin/nologin -m -d /var/lib/copyparty copyparty
+#   firewall-cmd --permanent --add-port=3923/tcp  # --zone=libvirt
 #   firewall-cmd --reload
+#   cp -pv copyparty.service /etc/systemd/system/
+#   cp -pv copyparty.conf /etc/
+#   restorecon -vr /etc/systemd/system/copyparty.service  # on fedora/rhel
 #   systemctl daemon-reload && systemctl enable --now copyparty
 #
+# every time you edit this file, you must "systemctl daemon-reload"
+# for the changes to take effect and then "systemctl restart copyparty"
+#
 # if it fails to start, first check this: systemctl status copyparty
-# then try starting it while viewing logs: journalctl -fan 100
+# then try starting it while viewing logs:
+#   journalctl -fan 100
+#   tail -Fn 100 /var/log/copyparty/$(date +%Y-%m%d.log)
+#
+# if you run into any issues, for example thumbnails not working,
+# try removing the "some quick hardening" section and then please
+# let me know if that actually helped so we can look into it
 #
 # you may want to:
-#   change "User=cpp" and "/home/cpp/" to another user
-#   remove the nft lines to only listen on port 3923
+#  - change "User=copyparty" and "/var/lib/copyparty/" to another user
+#  - edit /etc/copyparty.conf to configure copyparty
 # and in the ExecStart= line:
-#   change '/usr/bin/python3' to another interpreter
-#   change '/mnt::rw' to another location or permission-set
-#   add '-q' to disable logging on busy servers
-#   add '-i 127.0.0.1' to only allow local connections
-#   add '-e2dsa' to enable filesystem scanning + indexing
-#   add '-e2ts' to enable metadata indexing
-#   remove '--ansi' to disable colored logs
+#  - change '/usr/bin/python3' to another interpreter
 #
 # with `Type=notify`, copyparty will signal systemd when it is ready to
 #   accept connections; correctly delaying units depending on copyparty.
@@ -30,11 +36,9 @@
 #   python disabling line-buffering, so messages are out-of-order:
 #   https://user-images.githubusercontent.com/241032/126040249-cb535cc7-c599-4931-a796-a5d9af691bad.png
 #
-# unless you add -q to disable logging, you may want to remove the
-#   following line to allow buffering (slightly better performance):
-#   Environment=PYTHONUNBUFFERED=x
-#
-# keep ExecStartPre before ExecStart, at least on rhel8
+########################################################################
+########################################################################
+
 
 [Unit]
 Description=copyparty file server
@@ -44,23 +48,52 @@ Type=notify
 SyslogIdentifier=copyparty
 Environment=PYTHONUNBUFFERED=x
 ExecReload=/bin/kill -s USR1 $MAINPID
+PermissionsStartOnly=true
 
-# user to run as + where the TLS certificate is (if any)
-User=cpp
-Environment=XDG_CONFIG_HOME=/home/cpp/.config
+## user to run as + where the TLS certificate is (if any)
+##
+User=copyparty
+Group=copyparty
+WorkingDirectory=/var/lib/copyparty
+Environment=XDG_CONFIG_HOME=/var/lib/copyparty/.config
 
-# OPTIONAL: setup forwarding from ports 80 and 443 to port 3923
-ExecStartPre=+/bin/bash -c 'nft -n -a list table nat | awk "/ to :3923 /{print\$NF}" | xargs -rL1 nft delete rule nat prerouting handle; true'
-ExecStartPre=+nft add table ip nat
-ExecStartPre=+nft -- add chain ip nat prerouting { type nat hook prerouting priority -100 \; }
-ExecStartPre=+nft add rule ip nat prerouting tcp dport 80 redirect to :3923
-ExecStartPre=+nft add rule ip nat prerouting tcp dport 443 redirect to :3923
+## OPTIONAL: allow copyparty to listen on low ports (like 80/443);
+##   you need to uncomment the "p: 80,443,3923" in the config too
+##   ------------------------------------------------------------
+##   a slightly safer alternative is to enable partyalone.service
+##   which does portforwarding with nftables instead, but an even
+##   better option is to use a reverse-proxy (nginx/caddy/...)
+##
+AmbientCapabilities=CAP_NET_BIND_SERVICE
 
-# stop systemd-tmpfiles-clean.timer from deleting copyparty while it's running
-ExecStartPre=+/bin/bash -c 'mkdir -p /run/tmpfiles.d/ && echo "x /tmp/pe-copyparty*" > /run/tmpfiles.d/copyparty.conf'
+## some quick hardening; TODO port more from the nixos package
+##
+MemoryMax=50%
+MemorySwapMax=50%
+ProtectClock=true
+ProtectControlGroups=true
+ProtectHostname=true
+ProtectKernelLogs=true
+ProtectKernelModules=true
+ProtectKernelTunables=true
+ProtectProc=invisible
+RemoveIPC=true
+RestrictNamespaces=true
+RestrictRealtime=true
+RestrictSUIDSGID=true
 
-# copyparty settings
-ExecStart=/usr/bin/python3 /usr/local/bin/copyparty-sfx.py --ansi -e2d -v /mnt::rw
+## create a directory for logfiles;
+##   this defines $LOGS_DIRECTORY which is used in copyparty.conf
+##
+LogsDirectory=copyparty
+
+## finally, start copyparty and give it the config file:
+##
+ExecStart=/usr/bin/python3 /usr/local/bin/copyparty-sfx.py -c /etc/copyparty.conf
+
+# NOTE: if you installed copyparty from an OS package repo (nice)
+#   then you probably want something like this instead:
+#ExecStart=/usr/bin/copyparty -c /etc/copyparty.conf
 
 [Install]
 WantedBy=multi-user.target

contrib/systemd/prisonparty.service

@@ -1,5 +1,5 @@
 # this will start `/usr/local/bin/copyparty-sfx.py`
-# in a chroot, preventing accidental access elsewhere
+# in a chroot, preventing accidental access elsewhere,
 # and share '/mnt' with anonymous read+write
 #
 # installation:
@@ -7,9 +7,9 @@
 #   2) cp -pv prisonparty.service /etc/systemd/system && systemctl enable --now prisonparty
 #
 # expose additional filesystem locations to copyparty
-#   by listing them between the last `1000` and `--`
+#   by listing them between the last `cpp` and `--`
 #
-# `1000 1000` = what user to run copyparty as
+# `cpp cpp` = user/group to run copyparty as; can be IDs (1000 1000)
 #
 # you may want to:
 #   change '/mnt::rw' to another location or permission-set
@@ -32,7 +32,9 @@ ExecReload=/bin/kill -s USR1 $MAINPID
 ExecStartPre=+/bin/bash -c 'mkdir -p /run/tmpfiles.d/ && echo "x /tmp/pe-copyparty*" > /run/tmpfiles.d/copyparty.conf'
 
 # run copyparty
-ExecStart=/bin/bash /usr/local/bin/prisonparty.sh /var/lib/copyparty-jail 1000 1000 /mnt -- \
+ExecStart=/bin/bash /usr/local/bin/prisonparty.sh /var/lib/copyparty-jail cpp cpp \
+  /mnt \
+  -- \
   /usr/bin/python3 /usr/local/bin/copyparty-sfx.py -q -v /mnt::rw
 
 [Install]

contrib/themes/bsod.css

@@ -0,0 +1,116 @@
+/* copy bsod.* into a folder named ".themes" in your webroot and then
+     --themes=10 --theme=9 --css-browser=/.themes/bsod.css
+*/
+
+html.ey {
+    --w2: #3d7bbc;
+    --w3: #5fcbec;
+
+    --fg: #fff;
+    --fg-max: #fff;
+    --fg-weak: var(--w3);
+
+    --bg: #2067b2;
+    --bg-d3: var(--bg);
+    --bg-d2: var(--w2);
+    --bg-d1: var(--fg-weak);
+    --bg-u2: var(--bg);
+    --bg-u3: var(--bg);
+    --bg-u5: var(--w2);
+
+    --tab-alt: var(--fg-weak);
+	--row-alt: var(--w2);
+
+    --scroll: var(--w3);
+
+    --a: #fff;
+    --a-b: #fff;
+    --a-hil: #fff;
+    --a-h-bg: var(--fg-weak);
+    --a-dark: var(--a);
+    --a-gray: var(--fg-weak);
+
+	--btn-fg: var(--a);
+	--btn-bg: var(--w2);
+    --btn-h-fg: var(--w2);
+	--btn-1-fg: var(--bg);
+	--btn-1-bg: var(--a);
+    --txt-sh: a;
+    --txt-bg: var(--w2);
+
+	--u2-b1-bg: var(--w2);
+	--u2-b2-bg: var(--w2);
+    --u2-txt-bg: var(--w2);
+	--u2-tab-bg: a;
+	--u2-tab-1-bg: var(--w2);
+
+    --sort-1: var(--a);
+    --sort-1: var(--fg-weak);
+
+    --tree-bg: var(--bg);
+
+	--g-b1: a;
+	--g-b2: a;
+    --g-f-bg: var(--w2);
+
+    --f-sh1: 0.1;
+    --f-sh2: 0.02;
+    --f-sh3: 0.1;
+    --f-h-b1: a;
+
+    --srv-1: var(--a);
+    --srv-3: var(--a);
+
+    --mp-sh: a;
+}
+
+html.ey {
+    background: url('bsod.png') top 5em right 4.5em no-repeat fixed var(--bg);
+}
+html.ey body#b {
+    background: var(--bg);  /*sandbox*/
+}
+html.ey #ops {
+    margin: 1.7em 1.5em 0 1.5em;
+	border-radius: .3em;
+	border-width: 1px 0;
+}
+html.ey #ops a {
+    text-shadow: 1px 1px 0 rgba(0,0,0,0.5);
+}
+html.ey .opbox {
+	margin: 1.5em 0 0 0;
+}
+html.ey #tree {
+    box-shadow: none;
+}
+html.ey #tt {
+    border-color: var(--w2);
+    background: var(--w2);
+}
+html.ey .mdo a {
+    background: none;
+    text-decoration: underline;
+}
+html.ey .mdo pre,
+html.ey .mdo code {
+    color: #fff;
+    background: var(--w2);
+    border: none;
+}
+html.ey .mdo h1,
+html.ey .mdo h2 {
+    background: none;
+    border-color: var(--w2);
+}
+html.ey .mdo ul ul,
+html.ey .mdo ul ol,
+html.ey .mdo ol ul,
+html.ey .mdo ol ol {
+	border-color: var(--w2);
+}
+html.ey .mdo p>em,
+html.ey .mdo li>em,
+html.ey .mdo td>em {
+	color: #fd0;
+}

contrib/windows/copyparty-ctmp.bat

@@ -0,0 +1,2 @@
+rem run copyparty.exe on machines with busted environment variables
+cmd /v /c "set TMP=\tmp && copyparty.exe"

copyparty/__init__.py

@@ -23,7 +23,7 @@ if not PY2:
     unicode: Callable[[Any], str] = str
 else:
     sys.dont_write_bytecode = True
-    unicode = unicode  # noqa: F821  # pylint: disable=undefined-variable,self-assigning-variable
+    unicode = unicode  # type: ignore
 
 WINDOWS: Any = (
     [int(x) for x in platform.version().split(".")]
@@ -56,7 +56,6 @@ class EnvParams(object):
         self.t0 = time.time()
         self.mod = ""
         self.cfg = ""
-        self.ox = getattr(sys, "oxidized", None)
 
 
 E = EnvParams()

copyparty/__version__.py

@@ -1,8 +1,8 @@
 # coding: utf-8
 
-VERSION = (1, 9, 4)
-CODENAME = "prometheable"
-BUILD_DT = (2023, 9, 2)
+VERSION = (1, 15, 0)
+CODENAME = "fill the drives"
+BUILD_DT = (2024, 9, 8)
 
 S_VERSION = ".".join(map(str, VERSION))
 S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT)

copyparty/bos/bos.py

@@ -43,6 +43,10 @@ def open(p: str, *a, **ka) -> int:
     return os.open(fsenc(p), *a, **ka)
 
 
+def readlink(p: str) -> str:
+    return fsdec(os.readlink(fsenc(p)))
+
+
 def rename(src: str, dst: str) -> None:
     return os.rename(fsenc(src), fsenc(dst))
 

copyparty/broker_mp.py

@@ -46,8 +46,8 @@ class BrokerMp(object):
         self.num_workers = self.args.j or CORES
         self.log("broker", "booting {} subprocesses".format(self.num_workers))
         for n in range(1, self.num_workers + 1):
-            q_pend: queue.Queue[tuple[int, str, list[Any]]] = mp.Queue(1)
-            q_yield: queue.Queue[tuple[int, str, list[Any]]] = mp.Queue(64)
+            q_pend: queue.Queue[tuple[int, str, list[Any]]] = mp.Queue(1)  # type: ignore
+            q_yield: queue.Queue[tuple[int, str, list[Any]]] = mp.Queue(64)  # type: ignore
 
             proc = MProcess(q_pend, q_yield, MpWorker, (q_pend, q_yield, self.args, n))
             Daemon(self.collector, "mp-sink-{}".format(n), (proc,))
@@ -57,11 +57,8 @@ class BrokerMp(object):
     def shutdown(self) -> None:
         self.log("broker", "shutting down")
         for n, proc in enumerate(self.procs):
-            thr = threading.Thread(
-                target=proc.q_pend.put((0, "shutdown", [])),
-                name="mp-shutdown-{}-{}".format(n, len(self.procs)),
-            )
-            thr.start()
+            name = "mp-shut-%d-%d" % (n, len(self.procs))
+            Daemon(proc.q_pend.put, name, ((0, "shutdown", []),))
 
         with self.mutex:
             procs = self.procs
@@ -69,7 +66,7 @@ class BrokerMp(object):
 
         while procs:
             if procs[-1].is_alive():
-                time.sleep(0.1)
+                time.sleep(0.05)
                 continue
 
             procs.pop()

copyparty/broker_mpw.py

@@ -76,7 +76,7 @@ class MpWorker(BrokerCli):
         pass
 
     def logw(self, msg: str, c: Union[int, str] = 0) -> None:
-        self.log("mp{}".format(self.n), msg, c)
+        self.log("mp%d" % (self.n,), msg, c)
 
     def main(self) -> None:
         while True:

copyparty/broker_util.py

@@ -28,7 +28,7 @@ class ExceptionalQueue(Queue, object):
                 if rv[1] == "pebkac":
                     raise Pebkac(*rv[2:])
                 else:
-                    raise Exception(rv[2])
+                    raise rv[2]
 
         return rv
 
@@ -65,8 +65,8 @@ def try_exec(want_retval: Union[bool, int], func: Any, *args: list[Any]) -> Any:
 
         return ["exception", "pebkac", ex.code, str(ex)]
 
-    except:
+    except Exception as ex:
         if not want_retval:
             raise
 
-        return ["exception", "stack", traceback.format_exc()]
+        return ["exception", "stack", ex]

copyparty/cert.py

@@ -6,12 +6,19 @@ import os
 import shutil
 import time
 
-from .util import Netdev, runcmd
+from .__init__ import ANYWIN
+from .util import Netdev, runcmd, wrename, wunlink
 
-HAVE_CFSSL = True
+HAVE_CFSSL = not os.environ.get("PRTY_NO_CFSSL")
 
 if True:  # pylint: disable=using-constant-test
-    from .util import RootLogger
+    from .util import NamedLogger, RootLogger
+
+
+if ANYWIN:
+    VF = {"mv_re_t": 5, "rm_re_t": 5, "mv_re_r": 0.1, "rm_re_r": 0.1}
+else:
+    VF = {"mv_re_t": 0, "rm_re_t": 0}
 
 
 def ensure_cert(log: "RootLogger", args) -> None:
@@ -76,6 +83,8 @@ def _read_crt(args, fn):
 
 
 def _gen_ca(log: "RootLogger", args):
+    nlog: "NamedLogger" = lambda msg, c=0: log("cert-gen-ca", msg, c)
+
     expiry = _read_crt(args, "ca.pem")[0]
     if time.time() + args.crt_cdays * 60 * 60 * 24 * 0.1 < expiry:
         return
@@ -105,13 +114,19 @@ def _gen_ca(log: "RootLogger", args):
         raise Exception("failed to translate ca-cert: {}, {}".format(rc, se), 3)
 
     bname = os.path.join(args.crt_dir, "ca")
-    os.rename(bname + "-key.pem", bname + ".key")
-    os.unlink(bname + ".csr")
+    try:
+        wunlink(nlog, bname + ".key", VF)
+    except:
+        pass
+    wrename(nlog, bname + "-key.pem", bname + ".key", VF)
+    wunlink(nlog, bname + ".csr", VF)
 
     log("cert", "new ca OK", 2)
 
 
 def _gen_srv(log: "RootLogger", args, netdevs: dict[str, Netdev]):
+    nlog: "NamedLogger" = lambda msg, c=0: log("cert-gen-srv", msg, c)
+
     names = args.crt_ns.split(",") if args.crt_ns else []
     if not args.crt_exact:
         for n in names[:]:
@@ -132,7 +147,10 @@ def _gen_srv(log: "RootLogger", args, netdevs: dict[str, Netdev]):
 
     try:
         expiry, inf = _read_crt(args, "srv.pem")
-        expired = time.time() + args.crt_sdays * 60 * 60 * 24 * 0.1 > expiry
+        if "sans" not in inf:
+            raise Exception("no useable cert found")
+
+        expired = time.time() + args.crt_sdays * 60 * 60 * 24 * 0.5 > expiry
         cert_insec = os.path.join(args.E.mod, "res/insecure.pem")
         for n in names:
             if n not in inf["sans"]:
@@ -182,11 +200,11 @@ def _gen_srv(log: "RootLogger", args, netdevs: dict[str, Netdev]):
 
     bname = os.path.join(args.crt_dir, "srv")
     try:
-        os.unlink(bname + ".key")
+        wunlink(nlog, bname + ".key", VF)
     except:
         pass
-    os.rename(bname + "-key.pem", bname + ".key")
-    os.unlink(bname + ".csr")
+    wrename(nlog, bname + "-key.pem", bname + ".key", VF)
+    wunlink(nlog, bname + ".csr", VF)
 
     with open(os.path.join(args.crt_dir, "ca.pem"), "rb") as f:
         ca = f.read()

copyparty/cfg.py

@@ -9,27 +9,40 @@ onedash = set(zs.split())
 def vf_bmap() -> dict[str, str]:
     """argv-to-volflag: simple bools"""
     ret = {
-        "never_symlink": "neversymlink",
-        "no_dedup": "copydupes",
-        "no_dupe": "nodupe",
-        "no_forget": "noforget",
-        "th_no_crop": "nocrop",
         "dav_auth": "davauth",
         "dav_rt": "davrt",
+        "ed": "dots",
+        "hardlink_only": "hardlinkonly",
+        "no_dupe": "nodupe",
+        "no_forget": "noforget",
+        "no_pipe": "nopipe",
+        "no_robots": "norobots",
+        "no_thumb": "dthumb",
+        "no_vthumb": "dvthumb",
+        "no_athumb": "dathumb",
     }
     for k in (
+        "dedup",
         "dotsrch",
+        "e2d",
+        "e2ds",
+        "e2dsa",
         "e2t",
         "e2ts",
         "e2tsr",
         "e2v",
         "e2vu",
         "e2vp",
+        "exp",
         "grid",
+        "gsel",
         "hardlink",
         "magic",
         "no_sb_md",
         "no_sb_lg",
+        "og",
+        "og_no_head",
+        "og_s_title",
         "rand",
         "xdev",
         "xlink",
@@ -41,8 +54,39 @@ def vf_bmap() -> dict[str, str]:
 
 def vf_vmap() -> dict[str, str]:
     """argv-to-volflag: simple values"""
-    ret = {"th_convt": "convt", "th_size": "thsize"}
-    for k in ("dbd", "lg_sbf", "md_sbf", "nrand", "unlist"):
+    ret = {
+        "no_hash": "nohash",
+        "no_idx": "noidx",
+        "re_maxage": "scan",
+        "safe_dedup": "safededup",
+        "th_convt": "convt",
+        "th_size": "thsize",
+        "th_crop": "crop",
+        "th_x3": "th3x",
+    }
+    for k in (
+        "dbd",
+        "html_head",
+        "lg_sbf",
+        "md_sbf",
+        "nrand",
+        "og_desc",
+        "og_site",
+        "og_th",
+        "og_title",
+        "og_title_a",
+        "og_title_v",
+        "og_title_i",
+        "og_tpl",
+        "og_ua",
+        "mv_retry",
+        "rm_retry",
+        "sort",
+        "tcolor",
+        "unlist",
+        "u2abort",
+        "u2ts",
+    ):
         ret[k] = k
     return ret
 
@@ -50,7 +94,22 @@ def vf_vmap() -> dict[str, str]:
 def vf_cmap() -> dict[str, str]:
     """argv-to-volflag: complex/lists"""
     ret = {}
-    for k in ("html_head", "mte", "mth"):
+    for k in (
+        "exp_lg",
+        "exp_md",
+        "mte",
+        "mth",
+        "mtp",
+        "xad",
+        "xar",
+        "xau",
+        "xban",
+        "xbd",
+        "xbr",
+        "xbu",
+        "xiu",
+        "xm",
+    ):
         ret[k] = k
     return ret
 
@@ -60,17 +119,23 @@ permdescs = {
     "w": 'write; upload files; need "r" to see the uploads',
     "m": 'move; move files and folders; need "w" at destination',
     "d": "delete; permanently delete files and folders",
+    ".": "dots; user can ask to show dotfiles in listings",
     "g": "get; download files, but cannot see folder contents",
     "G": 'upget; same as "g" but can see filekeys of their own uploads',
+    "h": 'html; same as "g" but folders return their index.html',
+    "a": "admin; can see uploader IPs, config-reload",
+    "A": "all; same as 'rwmda.' (read/write/move/delete/dotfiles)",
 }
 
 
 flagcats = {
     "uploads, general": {
+        "dedup": "enable symlink-based file deduplication",
+        "hardlink": "enable hardlink-based file deduplication,\nwith fallback on symlinks when that is impossible",
+        "hardlinkonly": "dedup with hardlink only, never symlink;\nmake a full copy if hardlink is impossible",
+        "safededup": "verify on-disk data before using it for dedup",
         "nodupe": "rejects existing files (instead of symlinking them)",
-        "hardlink": "does dedup with hardlinks instead of symlinks",
-        "neversymlink": "disables symlink fallback; full copy instead",
-        "copydupes": "disables dedup, always saves full copies of dupes",
+        "sparse": "force use of sparse files, mainly for s3-backed storage",
         "daw": "enable full WebDAV write support (dangerous);\nPUT-operations will now \033[1;31mOVERWRITE\033[0;35m existing files",
         "nosub": "forces all uploads into the top folder of the vfs",
         "magic": "enables filetype detection for nameless uploads",
@@ -82,8 +147,11 @@ flagcats = {
         "maxb=1g,300": "max 1 GiB over 5min (suffixes: b, k, m, g, t)",
         "vmaxb=1g": "total volume size max 1 GiB (suffixes: b, k, m, g, t)",
         "vmaxn=4k": "max 4096 files in volume (suffixes: b, k, m, g, t)",
+        "medialinks": "return medialinks for non-up2k uploads (not hotlinks)",
         "rand": "force randomized filenames, 9 chars long by default",
         "nrand=N": "randomized filenames are N chars long",
+        "u2ts=fc": "[f]orce [c]lient-last-modified or [u]pload-time",
+        "u2abort=1": "allow aborting unfinished uploads? 0=no 1=strict 2=ip-chk 3=acct-chk",
         "sz=1k-3m": "allow filesizes between 1 KiB and 3MiB",
         "df=1g": "ensure 1 GiB free disk space",
     },
@@ -93,7 +161,7 @@ flagcats = {
         "lifetime=3600": "uploads are deleted after 1 hour",
     },
     "database, general": {
-        "e2d": "enable database; makes files searchable + enables upload dedup",
+        "e2d": "enable database; makes files searchable + enables upload-undo",
         "e2ds": "scan writable folders for new files on startup; also sets -e2d",
         "e2dsa": "scans all folders for new files on startup; also sets -e2d",
         "e2t": "enable multimedia indexing; makes it possible to search for tags",
@@ -111,7 +179,7 @@ flagcats = {
         "noforget": "don't forget files when deleted from disk",
         "fat32": "avoid excessive reindexing on android sdcardfs",
         "dbd=[acid|swal|wal|yolo]": "database speed-durability tradeoff",
-        "xlink": "cross-volume dupe detection / linking",
+        "xlink": "cross-volume dupe detection / linking (dangerous)",
         "xdev": "do not descend into other filesystems",
         "xvol": "do not follow symlinks leaving the volume root",
         "dotsrch": "show dotfiles in search results",
@@ -126,8 +194,10 @@ flagcats = {
         "dvthumb": "disables video thumbnails",
         "dathumb": "disables audio thumbnails (spectrograms)",
         "dithumb": "disables image thumbnails",
+        "pngquant": "compress audio waveforms 33% better",
         "thsize": "thumbnail res; WxH",
-        "nocrop": "disable center-cropping",
+        "crop": "center-cropping (y/n/fy/fn)",
+        "th3x": "3x resolution (y/n/fy/fn)",
         "convt": "conversion timeout in seconds",
     },
     "handlers\n(better explained in --help-handlers)": {
@@ -147,8 +217,10 @@ flagcats = {
     },
     "client and ux": {
         "grid": "show grid/thumbnails by default",
+        "gsel": "select files in grid by ctrl-click",
+        "sort": "default sort order",
         "unlist": "dont list files matching REGEX",
-        "html_head=TXT": "includes TXT in the <head>",
+        "html_head=TXT": "includes TXT in the <head>, or @PATH for file at PATH",
         "robots": "allows indexing by search engines (default)",
         "norobots": "kindly asks search engines to leave",
         "no_sb_md": "disable js sandbox for markdown files",
@@ -160,7 +232,11 @@ flagcats = {
         "nohtml": "return html and markdown as text/html",
     },
     "others": {
-        "fk=8": 'generates per-file accesskeys,\nwhich will then be required at the "g" permission',
+        "dots": "allow all users with read-access to\nenable the option to show dotfiles in listings",
+        "fk=8": 'generates per-file accesskeys,\nwhich are then required at the "g" permission;\nkeys are invalidated if filesize or inode changes',
+        "fka=8": 'generates slightly weaker per-file accesskeys,\nwhich are then required at the "g" permission;\nnot affected by filesize or inode numbers',
+        "mv_retry": "ms-windows: timeout for renaming busy files",
+        "rm_retry": "ms-windows: timeout for deleting busy files",
         "davauth": "ask webdav clients to login for all folders",
         "davrt": "show lastmod time of symlink destination, not the link itself\n(note: this option is always enabled for recursive listings)",
     },

copyparty/fsutil.py

@@ -1,6 +1,7 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
+import argparse
 import os
 import re
 import time
@@ -8,29 +9,35 @@ import time
 from .__init__ import ANYWIN, MACOS
 from .authsrv import AXS, VFS
 from .bos import bos
-from .util import chkcmd, min_ex
+from .util import chkcmd, min_ex, undot
 
 if True:  # pylint: disable=using-constant-test
     from typing import Optional, Union
 
-    from .util import RootLogger
+    from .util import RootLogger, undot
 
 
 class Fstab(object):
-    def __init__(self, log: "RootLogger"):
+    def __init__(self, log: "RootLogger", args: argparse.Namespace):
         self.log_func = log
 
+        self.warned = False
         self.trusted = False
         self.tab: Optional[VFS] = None
+        self.oldtab: Optional[VFS] = None
+        self.srctab = "a"
         self.cache: dict[str, str] = {}
         self.age = 0.0
+        self.maxage = args.mtab_age
 
     def log(self, msg: str, c: Union[int, str] = 0) -> None:
         self.log_func("fstab", msg, c)
 
     def get(self, path: str) -> str:
-        if len(self.cache) > 9000:
-            self.age = time.time()
+        now = time.time()
+        if now - self.age > self.maxage or len(self.cache) > 9000:
+            self.age = now
+            self.oldtab = self.tab or self.oldtab
             self.tab = None
             self.cache = {}
 
@@ -45,7 +52,7 @@ class Fstab(object):
                 self.log(msg.format(path, fs, min_ex()), 3)
                 return fs
 
-        path = path.lstrip("/")
+        path = undot(path)
         try:
             return self.cache[path]
         except:
@@ -75,7 +82,7 @@ class Fstab(object):
         self.trusted = False
 
     def build_tab(self) -> None:
-        self.log("building tab")
+        self.log("inspecting mtab for changes")
 
         sptn = r"^.*? on (.*) type ([^ ]+) \(.*"
         if MACOS:
@@ -84,6 +91,7 @@ class Fstab(object):
         ptn = re.compile(sptn)
         so, _ = chkcmd(["mount"])
         tab1: list[tuple[str, str]] = []
+        atab = []
         for ln in so.split("\n"):
             m = ptn.match(ln)
             if not m:
@@ -91,6 +99,15 @@ class Fstab(object):
 
             zs1, zs2 = m.groups()
             tab1.append((str(zs1), str(zs2)))
+            atab.append(ln)
+
+        # keep empirically-correct values if mounttab unchanged
+        srctab = "\n".join(sorted(atab))
+        if srctab == self.srctab:
+            self.tab = self.oldtab
+            return
+
+        self.log("mtab has changed; reevaluating support for sparse files")
 
         tab1.sort(key=lambda x: (len(x[0]), x[0]))
         path1, fs1 = tab1[0]
@@ -99,6 +116,7 @@ class Fstab(object):
             tab.add(fs, path.lstrip("/"))
 
         self.tab = tab
+        self.srctab = srctab
 
     def relabel(self, path: str, nval: str) -> None:
         assert self.tab
@@ -106,7 +124,7 @@ class Fstab(object):
         if ANYWIN:
             path = self._winpath(path)
 
-        path = path.lstrip("/")
+        path = undot(path)
         ptn = re.compile(r"^[^\\/]*")
         vn, rem = self.tab._find(path)
         if not self.trusted:
@@ -133,7 +151,9 @@ class Fstab(object):
                 self.trusted = True
             except:
                 # prisonparty or other restrictive environment
-                self.log("failed to build tab:\n{}".format(min_ex()), 3)
+                if not self.warned:
+                    self.warned = True
+                    self.log("failed to build tab:\n{}".format(min_ex()), 3)
                 self.build_fallback()
 
         assert self.tab

copyparty/ftpd.py

@@ -9,23 +9,19 @@ import stat
 import sys
 import time
 
-from .__init__ import ANYWIN, PY2, TYPE_CHECKING, E
-
-try:
-    import asynchat
-except:
-    sys.path.append(os.path.join(E.mod, "vend"))
-
 from pyftpdlib.authorizers import AuthenticationFailed, DummyAuthorizer
 from pyftpdlib.filesystems import AbstractedFS, FilesystemError
 from pyftpdlib.handlers import FTPHandler
 from pyftpdlib.ioloop import IOLoop
 from pyftpdlib.servers import FTPServer
 
+from .__init__ import PY2, TYPE_CHECKING
 from .authsrv import VFS
 from .bos import bos
 from .util import (
+    VF_CAREFUL,
     Daemon,
+    ODict,
     Pebkac,
     exclude_dotfiles,
     fsenc,
@@ -35,6 +31,7 @@ from .util import (
     runhook,
     sanitize_fn,
     vjoin,
+    wunlink,
 )
 
 if TYPE_CHECKING:
@@ -42,7 +39,10 @@ if TYPE_CHECKING:
 
 if True:  # pylint: disable=using-constant-test
     import typing
-    from typing import Any, Optional
+    from typing import Any, Optional, Union
+
+if PY2:
+    range = xrange  # type: ignore
 
 
 class FSE(FilesystemError):
@@ -79,6 +79,7 @@ class FtpAuth(DummyAuthorizer):
         asrv = self.hub.asrv
         uname = "*"
         if username != "anonymous":
+            uname = ""
             for zs in (password, username):
                 zs = asrv.iacct.get(asrv.ah.hash(zs), "")
                 if zs:
@@ -92,6 +93,12 @@ class FtpAuth(DummyAuthorizer):
                 if bonk:
                     logging.warning("client banned: invalid passwords")
                     bans[ip] = bonk
+                    try:
+                        # only possible if multiprocessing disabled
+                        self.hub.broker.httpsrv.bans[ip] = bonk  # type: ignore
+                        self.hub.broker.httpsrv.nban += 1  # type: ignore
+                    except:
+                        pass
 
             raise AuthenticationFailed("Authentication failed.")
 
@@ -132,11 +139,14 @@ class FtpFs(AbstractedFS):
 
         self.can_read = self.can_write = self.can_move = False
         self.can_delete = self.can_get = self.can_upget = False
-        self.can_admin = False
+        self.can_admin = self.can_dot = False
 
         self.listdirinfo = self.listdir
         self.chdir(".")
 
+    def log(self, msg: str, c: Union[int, str] = 0) -> None:
+        self.hub.log("ftpd", msg, c)
+
     def v2a(
         self,
         vpath: str,
@@ -148,7 +158,7 @@ class FtpFs(AbstractedFS):
         try:
             vpath = vpath.replace("\\", "/").strip("/")
             rd, fn = os.path.split(vpath)
-            if ANYWIN and relchk(rd):
+            if relchk(rd):
                 logging.warning("malicious vpath: %s", vpath)
                 t = "Unsupported characters in [{}]"
                 raise FSE(t.format(vpath), 1)
@@ -167,7 +177,7 @@ class FtpFs(AbstractedFS):
                 if not avfs:
                     raise FSE(t.format(vpath), 1)
 
-                cr, cw, cm, cd, _, _, _ = avfs.can_access("", self.h.uname)
+                cr, cw, cm, cd, _, _, _, _ = avfs.can_access("", self.h.uname)
                 if r and not cr or w and not cw or m and not cm or d and not cd:
                     raise FSE(t.format(vpath), 1)
 
@@ -205,18 +215,38 @@ class FtpFs(AbstractedFS):
         w = "w" in mode or "a" in mode or "+" in mode
 
         ap = self.rv2a(filename, r, w)[0]
+        self.validpath(ap)
         if w:
             try:
                 st = bos.stat(ap)
                 td = time.time() - st.st_mtime
+                need_unlink = True
             except:
+                need_unlink = False
                 td = 0
 
-            if td < -1 or td > self.args.ftp_wt:
-                raise FSE("Cannot open existing file for writing")
+        if w and need_unlink:
+            if td >= -1 and td <= self.args.ftp_wt:
+                # within permitted timeframe; unlink and accept
+                do_it = True
+            elif self.args.no_del or self.args.ftp_no_ow:
+                # file too old, or overwrite not allowed; reject
+                do_it = False
+            else:
+                # allow overwrite if user has delete permission
+                # (avoids win2000 freaking out and deleting the server copy without uploading its own)
+                try:
+                    self.rv2a(filename, False, True, False, True)
+                    do_it = True
+                except:
+                    do_it = False
 
-        self.validpath(ap)
-        return open(fsenc(ap), mode)
+            if not do_it:
+                raise FSE("File already exists")
+
+            wunlink(self.log, ap, VF_CAREFUL)
+
+        return open(fsenc(ap), mode, self.args.iobuf)
 
     def chdir(self, path: str) -> None:
         nwd = join(self.cwd, path)
@@ -243,6 +273,7 @@ class FtpFs(AbstractedFS):
             self.can_get,
             self.can_upget,
             self.can_admin,
+            self.can_dot,
         ) = avfs.can_access("", self.h.uname)
 
     def mkdir(self, path: str) -> None:
@@ -265,7 +296,7 @@ class FtpFs(AbstractedFS):
             vfs_ls = [x[0] for x in vfs_ls1]
             vfs_ls.extend(vfs_virt.keys())
 
-            if not self.args.ed:
+            if not self.can_dot:
                 vfs_ls = exclude_dotfiles(vfs_ls)
 
             vfs_ls.sort()
@@ -279,9 +310,20 @@ class FtpFs(AbstractedFS):
                 # display write-only folders as empty
                 return []
 
-            # return list of volumes
-            r = {x.split("/")[0]: 1 for x in self.hub.asrv.vfs.all_vols.keys()}
-            return list(sorted(list(r.keys())))
+            # return list of accessible volumes
+            ret = []
+            for vn in self.hub.asrv.vfs.all_vols.values():
+                if "/" in vn.vpath or not vn.vpath:
+                    continue  # only include toplevel-mounted vols
+
+                try:
+                    self.hub.asrv.vfs.get(vn.vpath, self.uname, True, False)
+                    ret.append(vn.vpath)
+                except:
+                    pass
+
+            ret.sort()
+            return ret
 
     def rmdir(self, path: str) -> None:
         ap = self.rv2a(path, d=True)[0]
@@ -297,7 +339,7 @@ class FtpFs(AbstractedFS):
 
         vp = join(self.cwd, path).lstrip("/")
         try:
-            self.hub.up2k.handle_rm(self.uname, self.h.cli_ip, [vp], [], False)
+            self.hub.up2k.handle_rm(self.uname, self.h.cli_ip, [vp], [], False, False)
         except Exception as ex:
             raise FSE(str(ex))
 
@@ -311,7 +353,7 @@ class FtpFs(AbstractedFS):
         svp = join(self.cwd, src).lstrip("/")
         dvp = join(self.cwd, dst).lstrip("/")
         try:
-            self.hub.up2k.handle_mv(self.uname, svp, dvp)
+            self.hub.up2k.handle_mv(self.uname, self.h.cli_ip, svp, dvp)
         except Exception as ex:
             raise FSE(str(ex))
 
@@ -404,7 +446,16 @@ class FtpHandler(FTPHandler):
             super(FtpHandler, self).__init__(conn, server, ioloop)
 
         cip = self.remote_ip
-        self.cli_ip = cip[7:] if cip.startswith("::ffff:") else cip
+        if cip.startswith("::ffff:"):
+            cip = cip[7:]
+
+        if self.args.ftp_ipa_nm and not self.args.ftp_ipa_nm.map(cip):
+            logging.warning("client rejected (--ftp-ipa): %s", cip)
+            self.connected = False
+            conn.close()
+            return
+
+        self.cli_ip = cip
 
         # abspath->vpath mapping to resolve log_transfer paths
         self.vfs_map: dict[str, str] = {}
@@ -420,15 +471,19 @@ class FtpHandler(FTPHandler):
         xbu = vfs.flags.get("xbu")
         if xbu and not runhook(
             None,
+            None,
+            self.hub.up2k,
+            "xbu.ftpd",
             xbu,
             ap,
-            vfs.canonical(rem),
+            vp,
             "",
             self.uname,
+            self.hub.asrv.vfs.get_perms(vp, self.uname),
             0,
             0,
             self.cli_ip,
-            0,
+            time.time(),
             "",
         ):
             raise FSE("Upload blocked by xbu server config")
@@ -531,9 +586,17 @@ class Ftpd(object):
         if "::" in ips:
             ips.append("0.0.0.0")
 
+        ips = [x for x in ips if "unix:" not in x]
+
         if self.args.ftp4:
             ips = [x for x in ips if ":" not in x]
 
+        if not ips:
+            lgr.fatal("cannot start ftp-server; no compatible IPs in -i")
+            return
+
+        ips = list(ODict.fromkeys(ips))  # dedup
+
         ioloop = IOLoop()
         for ip in ips:
             for h, lp in hs:

copyparty/httpconn.py

@@ -9,6 +9,9 @@ import threading  # typechk
 import time
 
 try:
+    if os.environ.get("PRTY_NO_TLS"):
+        raise Exception()
+
     HAVE_SSL = True
     import ssl
 except:
@@ -23,7 +26,7 @@ from .mtag import HAVE_FFMPEG
 from .th_cli import ThumbCli
 from .th_srv import HAVE_PIL, HAVE_VIPS
 from .u2idx import U2idx
-from .util import HMaccas, shut_socket
+from .util import HMaccas, NetMap, shut_socket
 
 if True:  # pylint: disable=using-constant-test
     from typing import Optional, Pattern, Union
@@ -50,11 +53,15 @@ class HttpConn(object):
         self.addr = addr
         self.hsrv = hsrv
 
-        self.mutex: threading.Lock = hsrv.mutex  # mypy404
+        self.u2mutex: threading.Lock = hsrv.u2mutex  # mypy404
         self.args: argparse.Namespace = hsrv.args  # mypy404
         self.E: EnvParams = self.args.E
         self.asrv: AuthSrv = hsrv.asrv  # mypy404
         self.u2fh: Util.FHC = hsrv.u2fh  # mypy404
+        self.pipes: Util.CachedDict = hsrv.pipes  # mypy404
+        self.ipa_nm: Optional[NetMap] = hsrv.ipa_nm
+        self.xff_nm: Optional[NetMap] = hsrv.xff_nm
+        self.xff_lan: NetMap = hsrv.xff_lan  # type: ignore
         self.iphash: HMaccas = hsrv.broker.iphash
         self.bans: dict[str, int] = hsrv.bans
         self.aclose: dict[str, int] = hsrv.aclose
@@ -93,7 +100,7 @@ class HttpConn(object):
             self.rproxy = ip
 
         self.ip = ip
-        self.log_src = "{} \033[{}m{}".format(ip, color, self.addr[1]).ljust(26)
+        self.log_src = ("%s \033[%dm%d" % (ip, color, self.addr[1])).ljust(26)
         return self.log_src
 
     def respath(self, res_name: str) -> str:
@@ -112,32 +119,30 @@ class HttpConn(object):
         return self.u2idx
 
     def _detect_https(self) -> bool:
-        method = None
-        if True:
-            try:
-                method = self.s.recv(4, socket.MSG_PEEK)
-            except socket.timeout:
-                return False
-            except AttributeError:
-                # jython does not support msg_peek; forget about https
-                method = self.s.recv(4)
-                self.sr = Util.Unrecv(self.s, self.log)
-                self.sr.buf = method
+        try:
+            method = self.s.recv(4, socket.MSG_PEEK)
+        except socket.timeout:
+            return False
+        except AttributeError:
+            # jython does not support msg_peek; forget about https
+            method = self.s.recv(4)
+            self.sr = Util.Unrecv(self.s, self.log)
+            self.sr.buf = method
 
-                # jython used to do this, they stopped since it's broken
-                # but reimplementing sendall is out of scope for now
-                if not getattr(self.s, "sendall", None):
-                    self.s.sendall = self.s.send  # type: ignore
+            # jython used to do this, they stopped since it's broken
+            # but reimplementing sendall is out of scope for now
+            if not getattr(self.s, "sendall", None):
+                self.s.sendall = self.s.send  # type: ignore
 
-            if len(method) != 4:
-                err = "need at least 4 bytes in the first packet; got {}".format(
-                    len(method)
-                )
-                if method:
-                    self.log(err)
+        if len(method) != 4:
+            err = "need at least 4 bytes in the first packet; got {}".format(
+                len(method)
+            )
+            if method:
+                self.log(err)
 
-                self.s.send(b"HTTP/1.1 400 Bad Request\r\n\r\n" + err.encode("utf-8"))
-                return False
+            self.s.send(b"HTTP/1.1 400 Bad Request\r\n\r\n" + err.encode("utf-8"))
+            return False
 
         return not method or not bool(PTN_HTTP.match(method))
 
@@ -178,7 +183,7 @@ class HttpConn(object):
 
                 self.s = ctx.wrap_socket(self.s, server_side=True)
                 msg = [
-                    "\033[1;3{:d}m{}".format(c, s)
+                    "\033[1;3%dm%s" % (c, s)
                     for c, s in zip([0, 5, 0], self.s.cipher())  # type: ignore
                 ]
                 self.log(" ".join(msg) + "\033[0m")

copyparty/httpsrv.py

@@ -12,7 +12,7 @@ import time
 
 import queue
 
-from .__init__ import ANYWIN, CORES, EXE, MACOS, TYPE_CHECKING, EnvParams
+from .__init__ import ANYWIN, CORES, EXE, MACOS, PY2, TYPE_CHECKING, EnvParams, unicode
 
 try:
     MNFE = ModuleNotFoundError
@@ -61,12 +61,14 @@ from .u2idx import U2idx
 from .util import (
     E_SCK,
     FHC,
+    CachedDict,
     Daemon,
     Garda,
     Magician,
     Netdev,
     NetMap,
     absreal,
+    build_netmap,
     ipnorm,
     min_ex,
     shut_socket,
@@ -82,6 +84,12 @@ if TYPE_CHECKING:
 if True:  # pylint: disable=using-constant-test
     from typing import Any, Optional
 
+if PY2:
+    range = xrange  # type: ignore
+
+if not hasattr(socket, "AF_UNIX"):
+    setattr(socket, "AF_UNIX", -9001)
+
 
 class HttpSrv(object):
     """
@@ -103,12 +111,13 @@ class HttpSrv(object):
         self.t0 = time.time()
         nsuf = "-n{}-i{:x}".format(nid, os.getpid()) if nid else ""
         self.magician = Magician()
-        self.nm = NetMap([], {})
+        self.nm = NetMap([], [])
         self.ssdp: Optional["SSDPr"] = None
         self.gpwd = Garda(self.args.ban_pw)
         self.g404 = Garda(self.args.ban_404)
         self.g403 = Garda(self.args.ban_403)
         self.g422 = Garda(self.args.ban_422, False)
+        self.gmal = Garda(self.args.ban_422)
         self.gurl = Garda(self.args.ban_url)
         self.bans: dict[str, int] = {}
         self.aclose: dict[str, int] = {}
@@ -116,6 +125,7 @@ class HttpSrv(object):
         self.bound: set[tuple[str, int]] = set()
         self.name = "hsrv" + nsuf
         self.mutex = threading.Lock()
+        self.u2mutex = threading.Lock()
         self.stopping = False
 
         self.tp_nthr = 0  # actual
@@ -127,7 +137,11 @@ class HttpSrv(object):
         self.t_periodic: Optional[threading.Thread] = None
 
         self.u2fh = FHC()
+        self.pipes = CachedDict(0.2)
         self.metrics = Metrics(self)
+        self.nreq = 0
+        self.nsus = 0
+        self.nban = 0
         self.srvs: list[socket.socket] = []
         self.ncli = 0  # exact
         self.clients: set[HttpConn] = set()  # laggy
@@ -140,15 +154,30 @@ class HttpSrv(object):
 
         env = jinja2.Environment()
         env.loader = jinja2.FileSystemLoader(os.path.join(self.E.mod, "web"))
-        jn = ["splash", "svcs", "browser", "browser2", "msg", "md", "mde", "cf"]
+        jn = [
+            "splash",
+            "shares",
+            "svcs",
+            "browser",
+            "browser2",
+            "msg",
+            "md",
+            "mde",
+            "cf",
+        ]
         self.j2 = {x: env.get_template(x + ".html") for x in jn}
         zs = os.path.join(self.E.mod, "web", "deps", "prism.js.gz")
         self.prism = os.path.exists(zs)
 
+        self.ipa_nm = build_netmap(self.args.ipa)
+        self.xff_nm = build_netmap(self.args.xff_src)
+        self.xff_lan = build_netmap("lan")
+
         self.statics: set[str] = set()
         self._build_statics()
 
         self.ptn_cc = re.compile(r"[\x00-\x1f]")
+        self.ptn_hsafe = re.compile(r"[\x00-\x1f<>\"'&]")
 
         self.mallow = "GET HEAD POST PUT DELETE OPTIONS".split()
         if not self.args.no_dav:
@@ -170,7 +199,7 @@ class HttpSrv(object):
             if self.args.log_thrs:
                 start_log_thrs(self.log, self.args.log_thrs, nid)
 
-        self.th_cfg: dict[str, Any] = {}
+        self.th_cfg: dict[str, set[str]] = {}
         Daemon(self.post_init, "hsrv-init2")
 
     def post_init(self) -> None:
@@ -185,7 +214,7 @@ class HttpSrv(object):
             for fn in df:
                 ap = absreal(os.path.join(dp, fn))
                 self.statics.add(ap)
-                if ap.endswith(".gz") or ap.endswith(".br"):
+                if ap.endswith(".gz"):
                     self.statics.add(ap[:-3])
 
     def set_netdevs(self, netdevs: dict[str, Netdev]) -> None:
@@ -193,7 +222,7 @@ class HttpSrv(object):
         for ip, _ in self.bound:
             ips.add(ip)
 
-        self.nm = NetMap(list(ips), netdevs)
+        self.nm = NetMap(list(ips), list(netdevs))
 
     def start_threads(self, n: int) -> None:
         self.tp_nthr += n
@@ -215,7 +244,7 @@ class HttpSrv(object):
     def periodic(self) -> None:
         while True:
             time.sleep(2 if self.tp_ncli or self.ncli else 10)
-            with self.mutex:
+            with self.u2mutex, self.mutex:
                 self.u2fh.clean()
                 if self.tp_q:
                     self.tp_ncli = max(self.ncli, self.tp_ncli - 2)
@@ -227,15 +256,24 @@ class HttpSrv(object):
                     return
 
     def listen(self, sck: socket.socket, nlisteners: int) -> None:
+        tcp = sck.family != socket.AF_UNIX
+
         if self.args.j != 1:
             # lost in the pickle; redefine
             if not ANYWIN or self.args.reuseaddr:
                 sck.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
 
-            sck.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
+            if tcp:
+                sck.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
+
             sck.settimeout(None)  # < does not inherit, ^ opts above do
 
-        ip, port = sck.getsockname()[:2]
+        if tcp:
+            ip, port = sck.getsockname()[:2]
+        else:
+            ip = re.sub(r"\.[0-9]+$", "", sck.getsockname().split("/")[-1])
+            port = 0
+
         self.srvs.append(sck)
         self.bound.add((ip, port))
         self.nclimax = math.ceil(self.args.nc * 1.0 / nlisteners)
@@ -247,16 +285,22 @@ class HttpSrv(object):
 
     def thr_listen(self, srv_sck: socket.socket) -> None:
         """listens on a shared tcp server"""
-        ip, port = srv_sck.getsockname()[:2]
         fno = srv_sck.fileno()
-        hip = "[{}]".format(ip) if ":" in ip else ip
-        msg = "subscribed @ {}:{}  f{} p{}".format(hip, port, fno, os.getpid())
+        if srv_sck.family == socket.AF_UNIX:
+            ip = re.sub(r"\.[0-9]+$", "", srv_sck.getsockname())
+            msg = "subscribed @ %s  f%d p%d" % (ip, fno, os.getpid())
+            ip = ip.split("/")[-1]
+            port = 0
+            tcp = False
+        else:
+            tcp = True
+            ip, port = srv_sck.getsockname()[:2]
+            hip = "[%s]" % (ip,) if ":" in ip else ip
+            msg = "subscribed @ %s:%d  f%d p%d" % (hip, port, fno, os.getpid())
+
         self.log(self.name, msg)
 
-        def fun() -> None:
-            self.broker.say("cb_httpsrv_up")
-
-        threading.Thread(target=fun, name="sig-hsrv-up1").start()
+        Daemon(self.broker.say, "sig-hsrv-up1", ("cb_httpsrv_up",))
 
         while not self.stopping:
             if self.args.log_conn:
@@ -325,11 +369,13 @@ class HttpSrv(object):
 
             try:
                 sck, saddr = srv_sck.accept()
-                cip, cport = saddr[:2]
-                if cip.startswith("::ffff:"):
-                    cip = cip[7:]
-
-                addr = (cip, cport)
+                if tcp:
+                    cip = unicode(saddr[0])
+                    if cip.startswith("::ffff:"):
+                        cip = cip[7:]
+                    addr = (cip, saddr[1])
+                else:
+                    addr = ("127.8.3.7", sck.fileno())
             except (OSError, socket.error) as ex:
                 if self.stopping:
                     break
@@ -361,7 +407,7 @@ class HttpSrv(object):
             if not self.t_periodic:
                 name = "hsrv-pt"
                 if self.nid:
-                    name += "-{}".format(self.nid)
+                    name += "-%d" % (self.nid,)
 
                 self.t_periodic = Daemon(self.periodic, name)
 
@@ -380,7 +426,7 @@ class HttpSrv(object):
 
         Daemon(
             self.thr_client,
-            "httpconn-{}-{}".format(addr[0].split(".", 2)[-1][-6:], addr[1]),
+            "httpconn-%s-%d" % (addr[0].split(".", 2)[-1][-6:], addr[1]),
             (sck, addr),
         )
 
@@ -397,9 +443,7 @@ class HttpSrv(object):
             try:
                 sck, addr = task
                 me = threading.current_thread()
-                me.name = "httpconn-{}-{}".format(
-                    addr[0].split(".", 2)[-1][-6:], addr[1]
-                )
+                me.name = "httpconn-%s-%d" % (addr[0].split(".", 2)[-1][-6:], addr[1])
                 self.thr_client(sck, addr)
                 me.name = self.name + "-poolw"
             except Exception as ex:

copyparty/ico.py

@@ -4,10 +4,11 @@ from __future__ import print_function, unicode_literals
 import argparse  # typechk
 import colorsys
 import hashlib
+import re
 
 from .__init__ import PY2
-from .th_srv import HAVE_PIL
-from .util import BytesIO
+from .th_srv import HAVE_PIL, HAVE_PILF
+from .util import BytesIO, html_escape  # type: ignore
 
 
 class Ico(object):
@@ -21,34 +22,59 @@ class Ico(object):
         ext = bext.decode("utf-8")
         zb = hashlib.sha1(bext).digest()[2:4]
         if PY2:
-            zb = [ord(x) for x in zb]
+            zb = [ord(x) for x in zb]  # type: ignore
 
         c1 = colorsys.hsv_to_rgb(zb[0] / 256.0, 1, 0.3)
-        c2 = colorsys.hsv_to_rgb(zb[0] / 256.0, 1, 1)
+        c2 = colorsys.hsv_to_rgb(zb[0] / 256.0, 0.8 if HAVE_PILF else 1, 1)
         ci = [int(x * 255) for x in list(c1) + list(c2)]
-        c = "".join(["{:02x}".format(x) for x in ci])
+        c = "".join(["%02x" % (x,) for x in ci])
 
         w = 100
         h = 30
-        if not self.args.th_no_crop and as_thumb:
+        if as_thumb:
             sw, sh = self.args.th_size.split("x")
-            h = int(100 / (float(sw) / float(sh)))
-            w = 100
+            h = int(100.0 / (float(sw) / float(sh)))
 
         if chrome:
             # cannot handle more than ~2000 unique SVGs
+            if HAVE_PILF:
+                # pillow 10.1 made this the default font;
+                # svg: 3.7s, this: 36s
+                try:
+                    from PIL import Image, ImageDraw
+
+                    # [.lt] are hard to see lowercase / unspaced
+                    ext2 = re.sub("(.)", "\\1 ", ext).upper()
+
+                    h = int(128.0 * h / w)
+                    w = 128
+                    img = Image.new("RGB", (w, h), "#" + c[:6])
+                    pb = ImageDraw.Draw(img)
+                    _, _, tw, th = pb.textbbox((0, 0), ext2, font_size=16)
+                    xy = (int((w - tw) / 2), int((h - th) / 2))
+                    pb.text(xy, ext2, fill="#" + c[6:], font_size=16)
+
+                    img = img.resize((w * 2, h * 2), Image.NEAREST)
+
+                    buf = BytesIO()
+                    img.save(buf, format="PNG", compress_level=1)
+                    return "image/png", buf.getvalue()
+
+                except:
+                    pass
+
             if HAVE_PIL:
                 # svg: 3s, cache: 6s, this: 8s
                 from PIL import Image, ImageDraw
 
-                h = int(64 * h / w)
+                h = int(64.0 * h / w)
                 w = 64
                 img = Image.new("RGB", (w, h), "#" + c[:6])
                 pb = ImageDraw.Draw(img)
                 try:
                     _, _, tw, th = pb.textbbox((0, 0), ext)
                 except:
-                    tw, th = pb.textsize(ext)
+                    tw, th = pb.textsize(ext)  # type: ignore
 
                 tw += len(ext)
                 cw = tw // len(ext)
@@ -64,20 +90,6 @@ class Ico(object):
                 img.save(buf, format="PNG", compress_level=1)
                 return "image/png", buf.getvalue()
 
-            elif False:
-                # 48s, too slow
-                import pyvips
-
-                h = int(192 * h / w)
-                w = 192
-                img = pyvips.Image.text(
-                    ext, width=w, height=h, dpi=192, align=pyvips.Align.CENTRE
-                )
-                img = img.ifthenelse(ci[3:], ci[:3], blend=True)
-                # i = i.resize(3, kernel=pyvips.Kernel.NEAREST)
-                buf = img.write_to_buffer(".png[compression=1]")
-                return "image/png", buf
-
         svg = """\
 <?xml version="1.0" encoding="UTF-8"?>
 <svg version="1.1" viewBox="0 0 100 {}" xmlns="http://www.w3.org/2000/svg"><g>
@@ -86,6 +98,6 @@ class Ico(object):
   fill="#{}" font-family="monospace" font-size="14px" style="letter-spacing:.5px">{}</text>
 </g></svg>
 """
-        svg = svg.format(h, c[:6], c[6:], ext)
+        svg = svg.format(h, c[:6], c[6:], html_escape(ext, True))
 
         return "image/svg+xml", svg.encode("utf-8")

copyparty/mdns.py

@@ -292,6 +292,22 @@ class MDNS(MCast):
     def run2(self) -> None:
         last_hop = time.time()
         ihop = self.args.mc_hop
+
+        try:
+            if self.args.no_poll:
+                raise Exception()
+            fd2sck = {}
+            srvpoll = select.poll()
+            for sck in self.srv:
+                fd = sck.fileno()
+                fd2sck[fd] = sck
+                srvpoll.register(fd, select.POLLIN)
+        except Exception as ex:
+            srvpoll = None
+            if not self.args.no_poll:
+                t = "WARNING: failed to poll(), will use select() instead: %r"
+                self.log(t % (ex,), 3)
+
         while self.running:
             timeout = (
                 0.02 + random.random() * 0.07
@@ -300,8 +316,13 @@ class MDNS(MCast):
                 if self.unsolicited
                 else (last_hop + ihop if ihop else 180)
             )
-            rdy = select.select(self.srv, [], [], timeout)
-            rx: list[socket.socket] = rdy[0]  # type: ignore
+            if srvpoll:
+                pr = srvpoll.poll(timeout * 1000)
+                rx = [fd2sck[x[0]] for x in pr if x[1] & select.POLLIN]
+            else:
+                rdy = select.select(self.srv, [], [], timeout)
+                rx: list[socket.socket] = rdy[0]  # type: ignore
+
             self.rx4.cln()
             self.rx6.cln()
             buf = b""
@@ -340,7 +361,7 @@ class MDNS(MCast):
             except:
                 pass
 
-        self.srv = {}
+        self.srv.clear()
 
     def eat(self, buf: bytes, addr: tuple[str, int], sck: socket.socket) -> None:
         cip = addr[0]

copyparty/metrics.py

@@ -34,14 +34,23 @@ class Metrics(object):
 
         ret: list[str] = []
 
-        def addc(k: str, unit: str, v: str, desc: str) -> None:
-            if unit:
-                k += "_" + unit
-                zs = "# TYPE %s counter\n# UNIT %s %s\n# HELP %s %s\n%s_created %s\n%s_total %s"
-                ret.append(zs % (k, k, unit, k, desc, k, int(self.hsrv.t0), k, v))
-            else:
-                zs = "# TYPE %s counter\n# HELP %s %s\n%s_created %s\n%s_total %s"
-                ret.append(zs % (k, k, desc, k, int(self.hsrv.t0), k, v))
+        def addc(k: str, v: str, desc: str) -> None:
+            zs = "# TYPE %s counter\n# HELP %s %s\n%s_created %s\n%s_total %s"
+            ret.append(zs % (k, k, desc, k, int(self.hsrv.t0), k, v))
+
+        def adduc(k: str, unit: str, v: str, desc: str) -> None:
+            k += "_" + unit
+            zs = "# TYPE %s counter\n# UNIT %s %s\n# HELP %s %s\n%s_created %s\n%s_total %s"
+            ret.append(zs % (k, k, unit, k, desc, k, int(self.hsrv.t0), k, v))
+
+        def addg(k: str, v: str, desc: str) -> None:
+            zs = "# TYPE %s gauge\n# HELP %s %s\n%s %s"
+            ret.append(zs % (k, k, desc, k, v))
+
+        def addug(k: str, unit: str, v: str, desc: str) -> None:
+            k += "_" + unit
+            zs = "# TYPE %s gauge\n# UNIT %s %s\n# HELP %s %s\n%s %s"
+            ret.append(zs % (k, k, unit, k, desc, k, v))
 
         def addh(k: str, typ: str, desc: str) -> None:
             zs = "# TYPE %s %s\n# HELP %s %s"
@@ -54,17 +63,75 @@ class Metrics(object):
         def addv(k: str, v: str) -> None:
             ret.append("%s %s" % (k, v))
 
+        t = "time since last copyparty restart"
         v = "{:.3f}".format(time.time() - self.hsrv.t0)
-        addc("cpp_uptime", "seconds", v, "time since last server restart")
+        addug("cpp_uptime", "seconds", v, t)
+
+        # timestamps are gauges because initial value is not zero
+        t = "unixtime of last copyparty restart"
+        v = "{:.3f}".format(self.hsrv.t0)
+        addug("cpp_boot_unixtime", "seconds", v, t)
+
+        t = "number of open http(s) client connections"
+        addg("cpp_http_conns", str(self.hsrv.ncli), t)
+
+        t = "number of http(s) requests since last restart"
+        addc("cpp_http_reqs", str(self.hsrv.nreq), t)
+
+        t = "number of 403/422/malicious reqs since restart"
+        addc("cpp_sus_reqs", str(self.hsrv.nsus), t)
 
         v = str(len(conn.bans or []))
-        addc("cpp_bans", "", v, "number of banned IPs")
+        addg("cpp_active_bans", v, "number of currently banned IPs")
+
+        t = "number of IPs banned since last restart"
+        addg("cpp_total_bans", str(self.hsrv.nban), t)
+
+        if not args.nos_vst:
+            x = self.hsrv.broker.ask("up2k.get_state")
+            vs = json.loads(x.get())
+
+            nvidle = 0
+            nvbusy = 0
+            nvoffline = 0
+            for v in vs["volstate"].values():
+                if v == "online, idle":
+                    nvidle += 1
+                elif "OFFLINE" in v:
+                    nvoffline += 1
+                else:
+                    nvbusy += 1
+
+            addg("cpp_idle_vols", str(nvidle), "number of idle/ready volumes")
+            addg("cpp_busy_vols", str(nvbusy), "number of busy/indexing volumes")
+            addg("cpp_offline_vols", str(nvoffline), "number of offline volumes")
+
+            t = "time since last database activity (upload/rename/delete)"
+            addug("cpp_db_idle", "seconds", str(vs["dbwt"]), t)
+
+            t = "unixtime of last database activity (upload/rename/delete)"
+            addug("cpp_db_act", "seconds", str(vs["dbwu"]), t)
+
+            t = "number of files queued for hashing/indexing"
+            addg("cpp_hashing_files", str(vs["hashq"]), t)
+
+            t = "number of files queued for metadata scanning"
+            addg("cpp_tagq_files", str(vs["tagq"]), t)
+
+            try:
+                t = "number of files queued for plugin-based analysis"
+                addg("cpp_mtpq_files", str(int(vs["mtpq"])), t)
+            except:
+                pass
 
         if not args.nos_hdd:
             addbh("cpp_disk_size_bytes", "total HDD size of volume")
             addbh("cpp_disk_free_bytes", "free HDD space in volume")
             for vpath, vol in allvols:
                 free, total = get_df(vol.realpath)
+                if free is None or total is None:
+                    continue
+
                 addv('cpp_disk_size_bytes{vol="/%s"}' % (vpath), str(total))
                 addv('cpp_disk_free_bytes{vol="/%s"}' % (vpath), str(free))
 
@@ -112,7 +179,7 @@ class Metrics(object):
             tnbytes = 0
             tnfiles = 0
             for vpath, vol in allvols:
-                cur = idx.get_cur(vol.realpath)
+                cur = idx.get_cur(vol)
                 if not cur:
                     continue
 
@@ -139,6 +206,9 @@ class Metrics(object):
             try:
                 x = self.hsrv.broker.ask("up2k.get_unfinished")
                 xs = x.get()
+                if not xs:
+                    raise Exception("up2k mutex acquisition timed out")
+
                 xj = json.loads(xs)
                 for ptop, (nbytes, nfiles) in xj.items():
                     tnbytes += nbytes
@@ -161,5 +231,6 @@ class Metrics(object):
         ret.append("# EOF")
 
         mime = "application/openmetrics-text; version=1.0.0; charset=utf-8"
+        mime = cli.uparam.get("mime") or mime
         cli.reply("\n".join(ret).encode("utf-8"), mime=mime)
         return True

copyparty/mtag.py

@@ -7,12 +7,15 @@ import os
 import shutil
 import subprocess as sp
 import sys
+import tempfile
 
-from .__init__ import EXE, PY2, WINDOWS, E, unicode
+from .__init__ import ANYWIN, EXE, PY2, WINDOWS, E, unicode
+from .authsrv import VFS
 from .bos import bos
 from .util import (
     FFMPEG_URL,
     REKOBO_LKEY,
+    VF_CAREFUL,
     fsenc,
     min_ex,
     pybin,
@@ -20,15 +23,30 @@ from .util import (
     runcmd,
     sfsenc,
     uncyg,
+    wunlink,
 )
 
 if True:  # pylint: disable=using-constant-test
-    from typing import Any, Union
+    from typing import Any, Optional, Union
 
-    from .util import RootLogger
+    from .util import NamedLogger, RootLogger
+
+
+try:
+    if os.environ.get("PRTY_NO_MUTAGEN"):
+        raise Exception()
+
+    from mutagen import version  # noqa: F401
+
+    HAVE_MUTAGEN = True
+except:
+    HAVE_MUTAGEN = False
 
 
 def have_ff(scmd: str) -> bool:
+    if ANYWIN:
+        scmd += ".exe"
+
     if PY2:
         print("# checking {}".format(scmd))
         acmd = (scmd + " -version").encode("ascii").split(b" ")
@@ -41,8 +59,8 @@ def have_ff(scmd: str) -> bool:
         return bool(shutil.which(scmd))
 
 
-HAVE_FFMPEG = have_ff("ffmpeg")
-HAVE_FFPROBE = have_ff("ffprobe")
+HAVE_FFMPEG = not os.environ.get("PRTY_NO_FFMPEG") and have_ff("ffmpeg")
+HAVE_FFPROBE = not os.environ.get("PRTY_NO_FFPROBE") and have_ff("ffprobe")
 
 
 class MParser(object):
@@ -104,6 +122,56 @@ class MParser(object):
             raise Exception()
 
 
+def au_unpk(
+    log: "NamedLogger", fmt_map: dict[str, str], abspath: str, vn: Optional[VFS] = None
+) -> str:
+    ret = ""
+    try:
+        ext = abspath.split(".")[-1].lower()
+        au, pk = fmt_map[ext].split(".")
+
+        fd, ret = tempfile.mkstemp("." + au)
+
+        if pk == "gz":
+            import gzip
+
+            fi = gzip.GzipFile(abspath, mode="rb")
+
+        elif pk == "xz":
+            import lzma
+
+            fi = lzma.open(abspath, "rb")
+
+        elif pk == "zip":
+            import zipfile
+
+            zf = zipfile.ZipFile(abspath, "r")
+            zil = zf.infolist()
+            zil = [x for x in zil if x.filename.lower().split(".")[-1] == au]
+            fi = zf.open(zil[0])
+
+        else:
+            raise Exception("unknown compression %s" % (pk,))
+
+        with os.fdopen(fd, "wb") as fo:
+            while True:
+                buf = fi.read(32768)
+                if not buf:
+                    break
+
+                fo.write(buf)
+
+        return ret
+
+    except Exception as ex:
+        if ret:
+            t = "failed to decompress audio file [%s]: %r"
+            log(t % (abspath, ex))
+            wunlink(log, ret, vn.flags if vn else VF_CAREFUL)
+
+        return abspath
+
+
 def ffprobe(
     abspath: str, timeout: int = 60
 ) -> tuple[dict[str, tuple[int, Any]], dict[str, list[Any]]]:
@@ -115,7 +183,7 @@ def ffprobe(
         b"--",
         fsenc(abspath),
     ]
-    rc, so, se = runcmd(cmd, timeout=timeout)
+    rc, so, se = runcmd(cmd, timeout=timeout, nice=True, oom=200)
     retchk(rc, cmd, se)
     return parse_ffprobe(so)
 
@@ -237,7 +305,7 @@ def parse_ffprobe(txt: str) -> tuple[dict[str, tuple[int, Any]], dict[str, list[
         if "/" in fps:
             fa, fb = fps.split("/")
             try:
-                fps = int(fa) * 1.0 / int(fb)
+                fps = float(fa) / float(fb)
             except:
                 fps = 9001
 
@@ -258,7 +326,8 @@ def parse_ffprobe(txt: str) -> tuple[dict[str, tuple[int, Any]], dict[str, list[
     if ".resw" in ret and ".resh" in ret:
         ret["res"] = "{}x{}".format(ret[".resw"], ret[".resh"])
 
-    zd = {k: (0, v) for k, v in ret.items()}
+    zero = int("0")
+    zd = {k: (zero, v) for k, v in ret.items()}
 
     return zd, md
 
@@ -277,16 +346,14 @@ class MTag(object):
         or_ffprobe = " or FFprobe"
 
         if self.backend == "mutagen":
-            self.get = self.get_mutagen
-            try:
-                from mutagen import version  # noqa: F401
-            except:
+            self._get = self.get_mutagen
+            if not HAVE_MUTAGEN:
                 self.log("could not load Mutagen, trying FFprobe instead", c=3)
                 self.backend = "ffprobe"
 
         if self.backend == "ffprobe":
             self.usable = self.can_ffprobe
-            self.get = self.get_ffprobe
+            self._get = self.get_ffprobe
             self.prefer_mt = True
 
             if not HAVE_FFPROBE:
@@ -456,6 +523,17 @@ class MTag(object):
 
         return r1
 
+    def get(self, abspath: str) -> dict[str, Union[str, float]]:
+        ext = abspath.split(".")[-1].lower()
+        if ext not in self.args.au_unpk:
+            return self._get(abspath)
+
+        ap = au_unpk(self.log, self.args.au_unpk, abspath)
+        ret = self._get(ap)
+        if ap != abspath:
+            wunlink(self.log, ap, VF_CAREFUL)
+        return ret
+
     def get_mutagen(self, abspath: str) -> dict[str, Union[str, float]]:
         ret: dict[str, tuple[int, Any]] = {}
 
@@ -509,7 +587,7 @@ class MTag(object):
                 continue
 
             if k == ".aq":
-                v /= 1000
+                v /= 1000  # type: ignore
 
             if k == "ac" and v.startswith("mp4a.40."):
                 v = "aac"
@@ -547,18 +625,25 @@ class MTag(object):
             pypath = str(os.pathsep.join(zsl))
             env["PYTHONPATH"] = pypath
         except:
-            if not E.ox and not EXE:
-                raise
+            raise  # might be expected outside cpython
+
+        ext = abspath.split(".")[-1].lower()
+        if ext in self.args.au_unpk:
+            ap = au_unpk(self.log, self.args.au_unpk, abspath)
+        else:
+            ap = abspath
 
         ret: dict[str, Any] = {}
         for tagname, parser in sorted(parsers.items(), key=lambda x: (x[1].pri, x[0])):
             try:
-                cmd = [parser.bin, abspath]
+                cmd = [parser.bin, ap]
                 if parser.bin.endswith(".py"):
                     cmd = [pybin] + cmd
 
                 args = {
                     "env": env,
+                    "nice": True,
+                    "oom": 300,
                     "timeout": parser.timeout,
                     "kill": parser.kill,
                     "capture": parser.capture,
@@ -569,11 +654,6 @@ class MTag(object):
                     zd.update(ret)
                     args["sin"] = json.dumps(zd).encode("utf-8", "replace")
 
-                if WINDOWS:
-                    args["creationflags"] = 0x4000
-                else:
-                    cmd = ["nice"] + cmd
-
                 bcmd = [sfsenc(x) for x in cmd[:-1]] + [fsenc(cmd[-1])]
                 rc, v, err = runcmd(bcmd, **args)  # type: ignore
                 retchk(rc, bcmd, err, self.log, 5, self.args.mtag_v)
@@ -593,4 +673,7 @@ class MTag(object):
                     t = "mtag error: tagname {}, parser {}, file {} => {}"
                     self.log(t.format(tagname, parser.bin, abspath, min_ex()))
 
+        if ap != abspath:
+            wunlink(self.log, ap, VF_CAREFUL)
+
         return ret

copyparty/multicast.py

@@ -110,7 +110,7 @@ class MCast(object):
             )
 
         ips = [x for x in ips if x not in ("::1", "127.0.0.1")]
-        ips = find_prefix(ips, netdevs)
+        ips = find_prefix(ips, list(netdevs))
 
         on = self.on[:]
         off = self.off[:]
@@ -206,6 +206,7 @@ class MCast(object):
             except:
                 t = "announce failed on {} [{}]:\n{}"
                 self.log(t.format(netdev, ip, min_ex()), 3)
+                sck.close()
 
         if self.args.zm_msub:
             for s1 in self.srv.values():

copyparty/pwhash.py

@@ -4,11 +4,21 @@ from __future__ import print_function, unicode_literals
 import argparse
 import base64
 import hashlib
+import os
 import sys
 import threading
 
 from .__init__ import unicode
 
+try:
+    if os.environ.get("PRTY_NO_ARGON2"):
+        raise Exception()
+
+    HAVE_ARGON2 = True
+    from argon2 import __version__ as argon2ver
+except:
+    HAVE_ARGON2 = False
+
 
 class PWHash(object):
     def __init__(self, args: argparse.Namespace):
@@ -136,8 +146,12 @@ class PWHash(object):
         import getpass
 
         while True:
-            p1 = getpass.getpass("password> ")
-            p2 = getpass.getpass("again or just hit ENTER> ")
+            try:
+                p1 = getpass.getpass("password> ")
+                p2 = getpass.getpass("again or just hit ENTER> ")
+            except EOFError:
+                return
+
             if p2 and p1 != p2:
                 print("\033[31minputs don't match; try again\033[0m", file=sys.stderr)
                 continue

copyparty/smbd.py

@@ -32,6 +32,8 @@ class SMB(object):
         self.asrv = hub.asrv
         self.log = hub.log
         self.files: dict[int, tuple[float, str]] = {}
+        self.noacc = self.args.smba
+        self.accs = not self.args.smba
 
         lg.setLevel(logging.DEBUG if self.args.smbvvv else logging.INFO)
         for x in ["impacket", "impacket.smbserver"]:
@@ -94,6 +96,14 @@ class SMB(object):
 
         port = int(self.args.smb_port)
         srv = smbserver.SimpleSMBServer(listenAddress=ip, listenPort=port)
+        try:
+            if self.accs:
+                srv.setAuthCallback(self._auth_cb)
+        except:
+            self.accs = False
+            self.noacc = True
+            t = "impacket too old; access permissions will not work! all accounts are admin!"
+            self.log("smb", t, 1)
 
         ro = "no" if self.args.smbw else "yes"  # (does nothing)
         srv.addShare("A", "/", readOnly=ro)
@@ -117,26 +127,80 @@ class SMB(object):
         self.log("smb", msg, c)
 
     def start(self) -> None:
-        Daemon(self.srv.start)
+        Daemon(self.srv.start, "smbd")
 
-    def _v2a(self, caller: str, vpath: str, *a: Any) -> tuple[VFS, str]:
+    def _auth_cb(self, *a, **ka):
+        debug("auth-result: %s %s", a, ka)
+        conndata = ka["connData"]
+        auth_ok = conndata["Authenticated"]
+        uname = ka["user_name"] if auth_ok else "*"
+        uname = self.asrv.iacct.get(uname, uname) or "*"
+        oldname = conndata.get("partygoer", "*") or "*"
+        cli_ip = conndata["ClientIP"]
+        cli_hn = ka["host_name"]
+        if uname != "*":
+            conndata["partygoer"] = uname
+            info("client %s [%s] authed as %s", cli_ip, cli_hn, uname)
+        elif oldname != "*":
+            info("client %s [%s] keeping old auth as %s", cli_ip, cli_hn, oldname)
+        elif auth_ok:
+            info("client %s [%s] authed as [*] (anon)", cli_ip, cli_hn)
+        else:
+            info("client %s [%s] rejected", cli_ip, cli_hn)
+
+    def _uname(self) -> str:
+        if self.noacc:
+            return LEELOO_DALLAS
+
+        try:
+            # you found it! my single worst bit of code so far
+            # (if you can think of a better way to track users through impacket i'm all ears)
+            cf0 = inspect.currentframe().f_back.f_back
+            cf = cf0.f_back
+            for n in range(3):
+                cl = cf.f_locals
+                if "connData" in cl:
+                    return cl["connData"]["partygoer"]
+                cf = cf.f_back
+            raise Exception()
+        except:
+            warning(
+                "nyoron... %s <<-- %s <<-- %s <<-- %s",
+                cf0.f_code.co_name,
+                cf0.f_back.f_code.co_name,
+                cf0.f_back.f_back.f_code.co_name,
+                cf0.f_back.f_back.f_back.f_code.co_name,
+            )
+            return "*"
+
+    def _v2a(
+        self, caller: str, vpath: str, *a: Any, uname="", perms=None
+    ) -> tuple[VFS, str]:
         vpath = vpath.replace("\\", "/").lstrip("/")
         # cf = inspect.currentframe().f_back
         # c1 = cf.f_back.f_code.co_name
         # c2 = cf.f_code.co_name
-        debug('%s("%s", %s)\033[K\033[0m', caller, vpath, str(a))
+        if not uname:
+            uname = self._uname()
+        if not perms:
+            perms = [True, True]
 
-        # TODO find a way to grab `identity` in smbComSessionSetupAndX and smb2SessionSetup
-        vfs, rem = self.asrv.vfs.get(vpath, LEELOO_DALLAS, True, True)
+        debug('%s("%s", %s) %s @%s\033[K\033[0m', caller, vpath, str(a), perms, uname)
+        vfs, rem = self.asrv.vfs.get(vpath, uname, *perms)
+        if not vfs.realpath:
+            raise Exception("unmapped vfs")
         return vfs, vfs.canonical(rem)
 
     def _listdir(self, vpath: str, *a: Any, **ka: Any) -> list[str]:
         vpath = vpath.replace("\\", "/").lstrip("/")
         # caller = inspect.currentframe().f_back.f_code.co_name
-        debug('listdir("%s", %s)\033[K\033[0m', vpath, str(a))
-        vfs, rem = self.asrv.vfs.get(vpath, LEELOO_DALLAS, False, False)
+        uname = self._uname()
+        # debug('listdir("%s", %s) @%s\033[K\033[0m', vpath, str(a), uname)
+        vfs, rem = self.asrv.vfs.get(vpath, uname, False, False)
+        if not vfs.realpath:
+            raise Exception("unmapped vfs")
         _, vfs_ls, vfs_virt = vfs.ls(
-            rem, LEELOO_DALLAS, not self.args.no_scandir, [[False, False]]
+            rem, uname, not self.args.no_scandir, [[False, False]]
         )
         dirs = [x[0] for x in vfs_ls if stat.S_ISDIR(x[1].st_mode)]
         fils = [x[0] for x in vfs_ls if x[0] not in dirs]
@@ -149,8 +213,8 @@ class SMB(object):
         sz = 112 * 2  # ['.', '..']
         for n, fn in enumerate(ls):
             if sz >= 64000:
-                t = "listing only %d of %d files (%d byte); see impacket#1433"
-                warning(t, n, len(ls), sz)
+                t = "listing only %d of %d files (%d byte) in /%s; see impacket#1433"
+                warning(t, n, len(ls), sz, vpath)
                 break
 
             nsz = len(fn.encode("utf-16", "replace"))
@@ -171,14 +235,30 @@ class SMB(object):
         if wr and not self.args.smbw:
             yeet("blocked write (no --smbw): " + vpath)
 
-        vfs, ap = self._v2a("open", vpath, *a)
+        uname = self._uname()
+        vfs, ap = self._v2a("open", vpath, *a, uname=uname, perms=[True, wr])
         if wr:
             if not vfs.axs.uwrite:
-                yeet("blocked write (no-write-acc): " + vpath)
+                t = "blocked write (no-write-acc %s): /%s @%s"
+                yeet(t % (vfs.axs.uwrite, vpath, uname))
 
             xbu = vfs.flags.get("xbu")
             if xbu and not runhook(
-                self.nlog, xbu, ap, vpath, "", "", 0, 0, "1.7.6.2", 0, ""
+                self.nlog,
+                None,
+                self.hub.up2k,
+                "xbu.smb",
+                xbu,
+                ap,
+                vpath,
+                "",
+                "",
+                "",
+                0,
+                0,
+                "1.7.6.2",
+                time.time(),
+                "",
             ):
                 yeet("blocked by xbu server config: " + vpath)
 
@@ -204,7 +284,7 @@ class SMB(object):
 
         _, vp = self.files.pop(fd)
         vp, fn = os.path.split(vp)
-        vfs, rem = self.hub.asrv.vfs.get(vp, LEELOO_DALLAS, False, True)
+        vfs, rem = self.hub.asrv.vfs.get(vp, self._uname(), False, True)
         vfs, rem = vfs.get_dbv(rem)
         self.hub.up2k.hash_file(
             vfs.realpath,
@@ -224,15 +304,18 @@ class SMB(object):
         vp1 = vp1.lstrip("/")
         vp2 = vp2.lstrip("/")
 
-        vfs2, ap2 = self._v2a("rename", vp2, vp1)
+        uname = self._uname()
+        vfs2, ap2 = self._v2a("rename", vp2, vp1, uname=uname)
         if not vfs2.axs.uwrite:
-            yeet("blocked rename (no-write-acc): " + vp2)
+            t = "blocked write (no-write-acc %s): /%s @%s"
+            yeet(t % (vfs2.axs.uwrite, vp2, uname))
 
-        vfs1, _ = self.asrv.vfs.get(vp1, LEELOO_DALLAS, True, True)
+        vfs1, _ = self.asrv.vfs.get(vp1, uname, True, True, True)
         if not vfs1.axs.umove:
-            yeet("blocked rename (no-move-acc): " + vp1)
+            t = "blocked rename (no-move-acc %s): /%s @%s"
+            yeet(t % (vfs1.axs.umove, vp1, uname))
 
-        self.hub.up2k.handle_mv(LEELOO_DALLAS, vp1, vp2)
+        self.hub.up2k.handle_mv(uname, "1.7.6.2", vp1, vp2)
         try:
             bos.makedirs(ap2)
         except:
@@ -242,52 +325,74 @@ class SMB(object):
         if not self.args.smbw:
             yeet("blocked mkdir (no --smbw): " + vpath)
 
-        vfs, ap = self._v2a("mkdir", vpath)
+        uname = self._uname()
+        vfs, ap = self._v2a("mkdir", vpath, uname=uname)
         if not vfs.axs.uwrite:
-            yeet("blocked mkdir (no-write-acc): " + vpath)
+            t = "blocked mkdir (no-write-acc %s): /%s @%s"
+            yeet(t % (vfs.axs.uwrite, vpath, uname))
 
         return bos.mkdir(ap)
 
     def _stat(self, vpath: str, *a: Any, **ka: Any) -> os.stat_result:
-        return bos.stat(self._v2a("stat", vpath, *a)[1], *a, **ka)
+        try:
+            ap = self._v2a("stat", vpath, *a, perms=[True, False])[1]
+            ret = bos.stat(ap, *a, **ka)
+            # debug(" `-stat:ok")
+            return ret
+        except:
+            # white lie: windows freaks out if we raise due to an offline volume
+            # debug(" `-stat:NOPE (faking a directory)")
+            ts = int(time.time())
+            return os.stat_result((16877, -1, -1, 1, 1000, 1000, 8, ts, ts, ts))
 
     def _unlink(self, vpath: str) -> None:
         if not self.args.smbw:
             yeet("blocked delete (no --smbw): " + vpath)
 
         # return bos.unlink(self._v2a("stat", vpath, *a)[1])
-        vfs, ap = self._v2a("delete", vpath)
+        uname = self._uname()
+        vfs, ap = self._v2a(
+            "delete", vpath, uname=uname, perms=[True, False, False, True]
+        )
         if not vfs.axs.udel:
             yeet("blocked delete (no-del-acc): " + vpath)
 
         vpath = vpath.replace("\\", "/").lstrip("/")
-        self.hub.up2k.handle_rm(LEELOO_DALLAS, "1.7.6.2", [vpath], [], False)
+        self.hub.up2k.handle_rm(uname, "1.7.6.2", [vpath], [], False, False)
 
     def _utime(self, vpath: str, times: tuple[float, float]) -> None:
         if not self.args.smbw:
             yeet("blocked utime (no --smbw): " + vpath)
 
-        vfs, ap = self._v2a("utime", vpath)
+        uname = self._uname()
+        vfs, ap = self._v2a("utime", vpath, uname=uname)
         if not vfs.axs.uwrite:
-            yeet("blocked utime (no-write-acc): " + vpath)
+            t = "blocked utime (no-write-acc %s): /%s @%s"
+            yeet(t % (vfs.axs.uwrite, vpath, uname))
 
         return bos.utime(ap, times)
 
     def _p_exists(self, vpath: str) -> bool:
+        # ap = "?"
         try:
-            bos.stat(self._v2a("p.exists", vpath)[1])
+            ap = self._v2a("p.exists", vpath, perms=[True, False])[1]
+            bos.stat(ap)
+            # debug(" `-exists((%s)->(%s)):ok", vpath, ap)
             return True
         except:
+            # debug(" `-exists((%s)->(%s)):NOPE", vpath, ap)
             return False
 
     def _p_getsize(self, vpath: str) -> int:
-        st = bos.stat(self._v2a("p.getsize", vpath)[1])
+        st = bos.stat(self._v2a("p.getsize", vpath, perms=[True, False])[1])
         return st.st_size
 
     def _p_isdir(self, vpath: str) -> bool:
         try:
-            st = bos.stat(self._v2a("p.isdir", vpath)[1])
-            return stat.S_ISDIR(st.st_mode)
+            st = bos.stat(self._v2a("p.isdir", vpath, perms=[True, False])[1])
+            ret = stat.S_ISDIR(st.st_mode)
+            # debug(" `-isdir:%s:%s", st.st_mode, ret)
+            return ret
         except:
             return False
 
@@ -319,6 +424,7 @@ class SMB(object):
 
         smbserver.os.path.abspath = self._hook
         smbserver.os.path.expanduser = self._hook
+        smbserver.os.path.expandvars = self._hook
         smbserver.os.path.getatime = self._hook
         smbserver.os.path.getctime = self._hook
         smbserver.os.path.getmtime = self._hook

copyparty/ssdp.py

@@ -5,11 +5,11 @@ import errno
 import re
 import select
 import socket
-from email.utils import formatdate
+import time
 
 from .__init__ import TYPE_CHECKING
 from .multicast import MC_Sck, MCast
-from .util import CachedSet, html_escape, min_ex
+from .util import CachedSet, formatdate, html_escape, min_ex
 
 if TYPE_CHECKING:
     from .broker_util import BrokerCli
@@ -141,9 +141,29 @@ class SSDPd(MCast):
         self.log("stopped", 2)
 
     def run2(self) -> None:
+        try:
+            if self.args.no_poll:
+                raise Exception()
+            fd2sck = {}
+            srvpoll = select.poll()
+            for sck in self.srv:
+                fd = sck.fileno()
+                fd2sck[fd] = sck
+                srvpoll.register(fd, select.POLLIN)
+        except Exception as ex:
+            srvpoll = None
+            if not self.args.no_poll:
+                t = "WARNING: failed to poll(), will use select() instead: %r"
+                self.log(t % (ex,), 3)
+
         while self.running:
-            rdy = select.select(self.srv, [], [], self.args.z_chk or 180)
-            rx: list[socket.socket] = rdy[0]  # type: ignore
+            if srvpoll:
+                pr = srvpoll.poll((self.args.z_chk or 180) * 1000)
+                rx = [fd2sck[x[0]] for x in pr if x[1] & select.POLLIN]
+            else:
+                rdy = select.select(self.srv, [], [], self.args.z_chk or 180)
+                rx: list[socket.socket] = rdy[0]  # type: ignore
+
             self.rxc.cln()
             buf = b""
             addr = ("0", 0)
@@ -168,7 +188,7 @@ class SSDPd(MCast):
             except:
                 pass
 
-        self.srv = {}
+        self.srv.clear()
 
     def eat(self, buf: bytes, addr: tuple[str, int]) -> None:
         cip = addr[0]
@@ -209,12 +229,12 @@ CONFIGID.UPNP.ORG: 1
 
 """
         v4 = srv.ip.replace("::ffff:", "")
-        zs = zs.format(formatdate(usegmt=True), v4, srv.hport, self.args.zsid)
+        zs = zs.format(formatdate(), v4, srv.hport, self.args.zsid)
         zb = zs[1:].replace("\n", "\r\n").encode("utf-8", "replace")
         srv.sck.sendto(zb, addr[:2])
 
         if cip not in self.txc.c:
-            self.log("{} [{}] --> {}".format(srv.name, srv.ip, cip), "6")
+            self.log("{} [{}] --> {}".format(srv.name, srv.ip, cip), 6)
 
         self.txc.add(cip)
         self.txc.cln()

copyparty/star.py

@@ -1,11 +1,13 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
+import re
 import stat
 import tarfile
 
 from queue import Queue
 
+from .authsrv import AuthSrv
 from .bos import bos
 from .sutil import StreamArc, errdesc
 from .util import Daemon, fsenc, min_ex
@@ -43,37 +45,43 @@ class StreamTar(StreamArc):
     def __init__(
         self,
         log: "NamedLogger",
+        asrv: AuthSrv,
         fgen: Generator[dict[str, Any], None, None],
         cmp: str = "",
         **kwargs: Any
     ):
-        super(StreamTar, self).__init__(log, fgen)
+        super(StreamTar, self).__init__(log, asrv, fgen)
 
         self.ci = 0
         self.co = 0
         self.qfile = QFile()
         self.errf: dict[str, Any] = {}
 
-        try:
-            cmp, lv = cmp.replace(":", ",").split(",")
-            lv = int(lv)
-        except:
-            lv = None
-
-        # python 3.8 changed to PAX_FORMAT as default,
-        # waste of space and don't care about the new features
+        # python 3.8 changed to PAX_FORMAT as default;
+        # slower, bigger, and no particular advantage
         fmt = tarfile.GNU_FORMAT
+        if "pax" in cmp:
+            # unless a client asks for it (currently
+            # gnu-tar has wider support than pax-tar)
+            fmt = tarfile.PAX_FORMAT
+            cmp = re.sub(r"[^a-z0-9]*pax[^a-z0-9]*", "", cmp)
+
+        try:
+            cmp, zs = cmp.replace(":", ",").split(",")
+            lv = int(zs)
+        except:
+            lv = -1
 
         arg = {"name": None, "fileobj": self.qfile, "mode": "w", "format": fmt}
         if cmp == "gz":
             fun = tarfile.TarFile.gzopen
-            arg["compresslevel"] = lv if lv is not None else 3
+            arg["compresslevel"] = lv if lv >= 0 else 3
         elif cmp == "bz2":
             fun = tarfile.TarFile.bz2open
-            arg["compresslevel"] = lv if lv is not None else 2
+            arg["compresslevel"] = lv if lv >= 0 else 2
         elif cmp == "xz":
             fun = tarfile.TarFile.xzopen
-            arg["preset"] = lv if lv is not None else 1
+            arg["preset"] = lv if lv >= 0 else 1
         else:
             fun = tarfile.open
             arg["mode"] = "w|"
@@ -120,7 +128,7 @@ class StreamTar(StreamArc):
         inf.gid = 0
 
         self.ci += inf.size
-        with open(fsenc(src), "rb", 512 * 1024) as fo:
+        with open(fsenc(src), "rb", self.args.iobuf) as fo:
             self.tar.addfile(inf, fo)
 
     def _gen(self) -> None:
@@ -140,7 +148,7 @@ class StreamTar(StreamArc):
                 errors.append((f["vp"], ex))
 
         if errors:
-            self.errf, txt = errdesc(errors)
+            self.errf, txt = errdesc(self.asrv.vfs, errors)
             self.log("\n".join(([repr(self.errf)] + txt[1:])))
             self.ser(self.errf)
 

copyparty/stolen/dnslib/dns.py

@@ -12,6 +12,12 @@ from .label import DNSBuffer, DNSLabel
 from .ranges import IP4, IP6, H, I, check_bytes
 
 
+try:
+    range = xrange
+except:
+    pass
+
+
 class DNSError(Exception):
     pass
 

copyparty/stolen/ifaddr/__init__.py

@@ -11,7 +11,21 @@ import os
 
 from ._shared import IP, Adapter
 
-if os.name == "nt":
+
+def nope(include_unconfigured=False):
+    return []
+
+
+try:
+    S390X = os.uname().machine == "s390x"
+except:
+    S390X = False
+
+
+if os.environ.get("PRTY_NO_IFADDR") or S390X:
+    # s390x deadlocks at libc.getifaddrs
+    get_adapters = nope
+elif os.name == "nt":
     from ._win32 import get_adapters
 elif os.name == "posix":
     from ._posix import get_adapters

copyparty/stolen/ifaddr/_shared.py

@@ -17,6 +17,7 @@ if not PY2:
     U: Callable[[str], str] = str
 else:
     U = unicode  # noqa: F821  # pylint: disable=undefined-variable,self-assigning-variable
+    range = xrange  # noqa: F821  # pylint: disable=undefined-variable,self-assigning-variable
 
 
 class Adapter(object):
@@ -61,7 +62,7 @@ class Adapter(object):
         )
 
 
-if True:
+if True:  # pylint: disable=using-constant-test
     # Type of an IPv4 address (a string in "xxx.xxx.xxx.xxx" format)
     _IPv4Address = str
 

copyparty/stolen/qrcodegen.py

@@ -16,6 +16,11 @@ if True:  # pylint: disable=using-constant-test
 
     from typing import Callable, List, Optional, Tuple, Union
 
+try:
+    range = xrange
+except:
+    pass
+
 
 def num_char_count_bits(ver: int) -> int:
     return 16 if (ver + 7) // 17 else 8

copyparty/sutil.py

@@ -6,9 +6,10 @@ import tempfile
 from datetime import datetime
 
 from .__init__ import CORES
+from .authsrv import VFS, AuthSrv
 from .bos import bos
 from .th_cli import ThumbCli
-from .util import vjoin
+from .util import UTC, vjoin, vol_san
 
 if True:  # pylint: disable=using-constant-test
     from typing import Any, Generator, Optional
@@ -20,10 +21,13 @@ class StreamArc(object):
     def __init__(
         self,
         log: "NamedLogger",
+        asrv: AuthSrv,
         fgen: Generator[dict[str, Any], None, None],
         **kwargs: Any
     ):
         self.log = log
+        self.asrv = asrv
+        self.args = asrv.args
         self.fgen = fgen
         self.stopped = False
 
@@ -78,7 +82,9 @@ def enthumb(
 ) -> dict[str, Any]:
     rem = f["vp"]
     ext = rem.rsplit(".", 1)[-1].lower()
-    if fmt == "opus" and ext in "aac|m4a|mp3|ogg|opus|wma".split("|"):
+    if (fmt == "mp3" and ext == "mp3") or (
+        fmt == "opus" and ext in "aac|m4a|mp3|ogg|opus|wma".split("|")
+    ):
         raise Exception()
 
     vp = vjoin(vtop, rem.split("/", 1)[1])
@@ -98,17 +104,22 @@ def enthumb(
     return f
 
 
-def errdesc(errors: list[tuple[str, str]]) -> tuple[dict[str, Any], list[str]]:
+def errdesc(
+    vfs: VFS, errors: list[tuple[str, str]]
+) -> tuple[dict[str, Any], list[str]]:
     report = ["copyparty failed to add the following files to the archive:", ""]
 
     for fn, err in errors:
         report.extend([" file: {}".format(fn), "error: {}".format(err), ""])
 
+    btxt = "\r\n".join(report).encode("utf-8", "replace")
+    btxt = vol_san(list(vfs.all_vols.values()), btxt)
+
     with tempfile.NamedTemporaryFile(prefix="copyparty-", delete=False) as tf:
         tf_path = tf.name
-        tf.write("\r\n".join(report).encode("utf-8", "replace"))
+        tf.write(btxt)
 
-    dt = datetime.utcnow().strftime("%Y-%m%d-%H%M%S")
+    dt = datetime.now(UTC).strftime("%Y-%m%d-%H%M%S")
 
     bos.chmod(tf_path, 0o444)
     return {

copyparty/svchub.py

@@ -3,7 +3,6 @@ from __future__ import print_function, unicode_literals
 
 import argparse
 import base64
-import calendar
 import errno
 import gzip
 import logging
@@ -16,7 +15,7 @@ import string
 import sys
 import threading
 import time
-from datetime import datetime, timedelta
+from datetime import datetime
 
 # from inspect import currentframe
 # print(currentframe().f_lineno)
@@ -28,24 +27,43 @@ if True:  # pylint: disable=using-constant-test
     import typing
     from typing import Any, Optional, Union
 
-from .__init__ import ANYWIN, EXE, MACOS, TYPE_CHECKING, EnvParams, unicode
+from .__init__ import ANYWIN, EXE, MACOS, PY2, TYPE_CHECKING, E, EnvParams, unicode
 from .authsrv import BAD_CFG, AuthSrv
 from .cert import ensure_cert
-from .mtag import HAVE_FFMPEG, HAVE_FFPROBE
+from .mtag import HAVE_FFMPEG, HAVE_FFPROBE, HAVE_MUTAGEN
+from .pwhash import HAVE_ARGON2
 from .tcpsrv import TcpSrv
-from .th_srv import HAVE_PIL, HAVE_VIPS, HAVE_WEBP, ThumbSrv
+from .th_srv import (
+    HAVE_AVIF,
+    HAVE_FFMPEG,
+    HAVE_FFPROBE,
+    HAVE_HEIF,
+    HAVE_PIL,
+    HAVE_VIPS,
+    HAVE_WEBP,
+    ThumbSrv,
+)
 from .up2k import Up2k
 from .util import (
+    DEF_EXP,
+    DEF_MTE,
+    DEF_MTH,
     FFMPEG_URL,
+    HAVE_PSUTIL,
+    HAVE_SQLITE3,
+    UTC,
     VERSIONS,
     Daemon,
     Garda,
     HLog,
     HMaccas,
+    ODict,
     alltrace,
     ansi_re,
+    build_netmap,
     min_ex,
     mp,
+    odfusion,
     pybin,
     start_log_thrs,
     start_stackmon,
@@ -58,6 +76,9 @@ if TYPE_CHECKING:
     except:
         pass
 
+if PY2:
+    range = xrange  # type: ignore
+
 
 class SvcHub(object):
     """
@@ -82,20 +103,23 @@ class SvcHub(object):
         self.argv = argv
         self.E: EnvParams = args.E
         self.no_ansi = args.no_ansi
+        self.tz = UTC if args.log_utc else None
         self.logf: Optional[typing.TextIO] = None
         self.logf_base_fn = ""
+        self.is_dut = False  # running in unittest; always False
         self.stop_req = False
         self.stopping = False
         self.stopped = False
         self.reload_req = False
-        self.reloading = False
+        self.reloading = 0
         self.stop_cond = threading.Condition()
         self.nsigs = 3
         self.retcode = 0
         self.httpsrv_up = 0
 
         self.log_mutex = threading.Lock()
-        self.next_day = 0
+        self.cday = 0
+        self.cmon = 0
         self.tstack = 0.0
 
         self.iphash = HMaccas(os.path.join(self.E.cfg, "iphash"), 8)
@@ -115,8 +139,6 @@ class SvcHub(object):
             args.no_mv = True
             args.hardlink = True
             args.vague_403 = True
-            args.ban_404 = "50,60,1440"
-            args.turbo = -1
             args.nih = True
 
         if args.s:
@@ -129,12 +151,13 @@ class SvcHub(object):
         if not self._process_config():
             raise Exception(BAD_CFG)
 
-        # for non-http clients (ftp)
+        # for non-http clients (ftp, tftp)
         self.bans: dict[str, int] = {}
         self.gpwd = Garda(self.args.ban_pw)
         self.g404 = Garda(self.args.ban_404)
         self.g403 = Garda(self.args.ban_403)
-        self.g422 = Garda(self.args.ban_422)
+        self.g422 = Garda(self.args.ban_422, False)
+        self.gmal = Garda(self.args.ban_422)
         self.gurl = Garda(self.args.ban_url)
 
         self.log_div = 10 ** (6 - args.log_tdec)
@@ -149,6 +172,8 @@ class SvcHub(object):
         lg.handlers = [lh]
         lg.setLevel(logging.DEBUG)
 
+        self._check_env()
+
         if args.stackmon:
             start_stackmon(args.stackmon, 0)
 
@@ -165,6 +190,40 @@ class SvcHub(object):
             self.log("root", t.format(args.j), c=3)
             args.no_fpool = True
 
+        for name, arg in (
+            ("iobuf", "iobuf"),
+            ("s-rd-sz", "s_rd_sz"),
+            ("s-wr-sz", "s_wr_sz"),
+        ):
+            zi = getattr(args, arg)
+            if zi < 32768:
+                t = "WARNING: expect very poor performance because you specified a very low value (%d) for --%s"
+                self.log("root", t % (zi, name), 3)
+                zi = 2
+            zi2 = 2 ** (zi - 1).bit_length()
+            if zi != zi2:
+                zi3 = 2 ** ((zi - 1).bit_length() - 1)
+                t = "WARNING: expect poor performance because --%s is not a power-of-two; consider using %d or %d instead of %d"
+                self.log("root", t % (name, zi2, zi3, zi), 3)
+
+        if args.s_rd_sz > args.iobuf:
+            t = "WARNING: --s-rd-sz (%d) is larger than --iobuf (%d); this may lead to reduced performance"
+            self.log("root", t % (args.s_rd_sz, args.iobuf), 3)
+
+        if args.chpw and args.idp_h_usr:
+            t = "ERROR: user-changeable passwords is incompatible with IdP/identity-providers; you must disable either --chpw or --idp-h-usr"
+            self.log("root", t, 1)
+            raise Exception(t)
+
+        noch = set()
+        for zs in args.chpw_no or []:
+            zsl = [x.strip() for x in zs.split(",")]
+            noch.update([x for x in zsl if x])
+        args.chpw_no = noch
+
+        if args.shr:
+            self.setup_share_db()
+
         bri = "zy"[args.theme % 2 :][:1]
         ch = "abcdefghijklmnopqrstuvwx"[int(args.theme / 2)]
         args.theme = "{0}{1} {0} {1}".format(ch, bri)
@@ -198,8 +257,14 @@ class SvcHub(object):
         self.log("root", "max clients: {}".format(self.args.nc))
 
         self.tcpsrv = TcpSrv(self)
+
+        if not self.tcpsrv.srv and self.args.ign_ebind_all:
+            self.args.no_fastboot = True
+
         self.up2k = Up2k(self)
 
+        self._feature_test()
+
         decs = {k: 1 for k in self.args.th_dec.split(",")}
         if not HAVE_VIPS:
             decs.pop("vips", None)
@@ -208,6 +273,10 @@ class SvcHub(object):
         if not HAVE_FFMPEG or not HAVE_FFPROBE:
             decs.pop("ff", None)
 
+        # compressed formats; "s3z=s3m.zip, s3gz=s3m.gz, ..."
+        zlss = [x.strip().lower().split("=", 1) for x in args.au_unpk.split(",")]
+        args.au_unpk = {x[0]: x[1] for x in zlss}
+
         self.args.th_dec = list(decs.keys())
         self.thumbsrv = None
         want_ff = False
@@ -244,6 +313,13 @@ class SvcHub(object):
         if want_ff and ANYWIN:
             self.log("thumb", "download FFmpeg to fix it:\033[0m " + FFMPEG_URL, 3)
 
+        if not args.no_acode:
+            if not re.match("^(0|[qv][0-9]|[0-9]{2,3}k)$", args.q_mp3.lower()):
+                t = "invalid mp3 transcoding quality [%s] specified; only supports [0] to disable, a CBR value such as [192k], or a CQ/CRF value such as [v2]"
+                raise Exception(t % (args.q_mp3,))
+        else:
+            args.au_unpk = {}
+
         args.th_poke = min(args.th_poke, args.th_maxage, args.ac_maxage)
 
         zms = ""
@@ -256,9 +332,16 @@ class SvcHub(object):
             from .ftpd import Ftpd
 
             self.ftpd: Optional[Ftpd] = None
-            Daemon(self.start_ftpd, "start_ftpd")
             zms += "f" if args.ftp else "F"
 
+        if args.tftp:
+            from .tftpd import Tftpd
+
+            self.tftpd: Optional[Tftpd] = None
+
+        if args.ftp or args.ftps or args.tftp:
+            Daemon(self.start_ftpd, "start_tftpd")
+
         if args.smb:
             # impacket.dcerpc is noisy about listen timeouts
             sto = socket.getdefaulttimeout()
@@ -286,12 +369,101 @@ class SvcHub(object):
 
         self.broker = Broker(self)
 
-    def start_ftpd(self) -> None:
-        time.sleep(30)
-        if self.ftpd:
+    def setup_share_db(self) -> None:
+        al = self.args
+        if not HAVE_SQLITE3:
+            self.log("root", "sqlite3 not available; disabling --shr", 1)
+            al.shr = ""
             return
 
-        self.restart_ftpd()
+        import sqlite3
+
+        al.shr = al.shr.strip("/")
+        if "/" in al.shr or not al.shr:
+            t = "config error: --shr must be the name of a virtual toplevel directory to put shares inside"
+            self.log("root", t, 1)
+            raise Exception(t)
+
+        al.shr = "/%s/" % (al.shr,)
+
+        create = True
+        modified = False
+        db_path = self.args.shr_db
+        self.log("root", "opening shares-db %s" % (db_path,))
+        for n in range(2):
+            try:
+                db = sqlite3.connect(db_path)
+                cur = db.cursor()
+                try:
+                    cur.execute("select count(*) from sh").fetchone()
+                    create = False
+                    break
+                except:
+                    pass
+            except Exception as ex:
+                if n:
+                    raise
+                t = "shares-db corrupt; deleting and recreating: %r"
+                self.log("root", t % (ex,), 3)
+                try:
+                    cur.close()  # type: ignore
+                except:
+                    pass
+                try:
+                    db.close()  # type: ignore
+                except:
+                    pass
+                os.unlink(db_path)
+
+        sch1 = [
+            r"create table kv (k text, v int)",
+            r"create table sh (k text, pw text, vp text, pr text, st int, un text, t0 int, t1 int)",
+            # sharekey, password, src, perms, numFiles, owner, created, expires
+        ]
+        sch2 = [
+            r"create table sf (k text, vp text)",
+            r"create index sf_k on sf(k)",
+            r"create index sh_k on sh(k)",
+            r"create index sh_t1 on sh(t1)",
+        ]
+
+        assert db  # type: ignore
+        assert cur  # type: ignore
+        if create:
+            dver = 2
+            modified = True
+            for cmd in sch1 + sch2:
+                cur.execute(cmd)
+            self.log("root", "created new shares-db")
+        else:
+            (dver,) = cur.execute("select v from kv where k = 'sver'").fetchall()[0]
+
+        if dver == 1:
+            modified = True
+            for cmd in sch2:
+                cur.execute(cmd)
+            cur.execute("update sh set st = 0")
+            self.log("root", "shares-db schema upgrade ok")
+
+        if modified:
+            for cmd in [
+                r"delete from kv where k = 'sver'",
+                r"insert into kv values ('sver', %d)" % (2,),
+            ]:
+                cur.execute(cmd)
+            db.commit()
+
+        cur.close()
+        db.close()
+
+    def start_ftpd(self) -> None:
+        time.sleep(30)
+
+        if hasattr(self, "ftpd") and not self.ftpd:
+            self.restart_ftpd()
+
+        if hasattr(self, "tftpd") and not self.tftpd:
+            self.restart_tftpd()
 
     def restart_ftpd(self) -> None:
         if not hasattr(self, "ftpd"):
@@ -308,6 +480,17 @@ class SvcHub(object):
         self.ftpd = Ftpd(self)
         self.log("root", "started FTPd")
 
+    def restart_tftpd(self) -> None:
+        if not hasattr(self, "tftpd"):
+            return
+
+        from .tftpd import Tftpd
+
+        if self.tftpd:
+            return  # todo
+
+        self.tftpd = Tftpd(self)
+
     def thr_httpsrv_up(self) -> None:
         time.sleep(1 if self.args.ign_ebind_all else 5)
         expected = self.broker.num_workers * self.tcpsrv.nsrv
@@ -332,23 +515,100 @@ class SvcHub(object):
         self.sigterm()
 
     def sigterm(self) -> None:
-        os.kill(os.getpid(), signal.SIGTERM)
+        self.signal_handler(signal.SIGTERM, None)
 
     def cb_httpsrv_up(self) -> None:
         self.httpsrv_up += 1
         if self.httpsrv_up != self.broker.num_workers:
             return
 
-        time.sleep(0.1)  # purely cosmetic dw
+        ar = self.args
+        for _ in range(10 if ar.ftp or ar.ftps else 0):
+            time.sleep(0.03)
+            if self.ftpd:
+                break
+
         if self.tcpsrv.qr:
             self.log("qr-code", self.tcpsrv.qr)
         else:
             self.log("root", "workers OK\n")
 
+        self.after_httpsrv_up()
+
+    def after_httpsrv_up(self) -> None:
         self.up2k.init_vols()
 
         Daemon(self.sd_notify, "sd-notify")
 
+    def _feature_test(self) -> None:
+        fok = []
+        fng = []
+        t_ff = "transcode audio, create spectrograms, video thumbnails"
+        to_check = [
+            (HAVE_SQLITE3, "sqlite", "file and media indexing"),
+            (HAVE_PIL, "pillow", "image thumbnails (plenty fast)"),
+            (HAVE_VIPS, "vips", "image thumbnails (faster, eats more ram)"),
+            (HAVE_WEBP, "pillow-webp", "create thumbnails as webp files"),
+            (HAVE_FFMPEG, "ffmpeg", t_ff + ", good-but-slow image thumbnails"),
+            (HAVE_FFPROBE, "ffprobe", t_ff + ", read audio/media tags"),
+            (HAVE_MUTAGEN, "mutagen", "read audio tags (ffprobe is better but slower)"),
+            (HAVE_ARGON2, "argon2", "secure password hashing (advanced users only)"),
+            (HAVE_HEIF, "pillow-heif", "read .heif images with pillow (rarely useful)"),
+            (HAVE_AVIF, "pillow-avif", "read .avif images with pillow (rarely useful)"),
+        ]
+        if ANYWIN:
+            to_check += [
+                (HAVE_PSUTIL, "psutil", "improved plugin cleanup  (rarely useful)")
+            ]
+
+        verbose = self.args.deps
+        if verbose:
+            self.log("dependencies", "")
+
+        for have, feat, what in to_check:
+            lst = fok if have else fng
+            lst.append((feat, what))
+            if verbose:
+                zi = 2 if have else 5
+                sgot = "found" if have else "missing"
+                t = "%7s: %s \033[36m(%s)"
+                self.log("dependencies", t % (sgot, feat, what), zi)
+
+        if verbose:
+            self.log("dependencies", "")
+            return
+
+        sok = ", ".join(x[0] for x in fok)
+        sng = ", ".join(x[0] for x in fng)
+
+        t = ""
+        if sok:
+            t += "OK: \033[32m" + sok
+        if sng:
+            if t:
+                t += ", "
+            t += "\033[0mNG: \033[35m" + sng
+
+        t += "\033[0m, see --deps"
+        self.log("dependencies", t, 6)
+
+    def _check_env(self) -> None:
+        try:
+            files = os.listdir(E.cfg)
+        except:
+            files = []
+
+        hits = [x for x in files if x.lower().endswith(".conf")]
+        if hits:
+            t = "WARNING: found config files in [%s]: %s\n  config files are not expected here, and will NOT be loaded (unless your setup is intentionally hella funky)"
+            self.log("root", t % (E.cfg, ", ".join(hits)), 3)
+
+        if self.args.no_bauth:
+            t = "WARNING: --no-bauth disables support for the Android app; you may want to use --bauth-last instead"
+            self.log("root", t, 3)
+            if self.args.bauth_last:
+                self.log("root", "WARNING: ignoring --bauth-last due to --no-bauth", 3)
+
     def _process_config(self) -> bool:
         al = self.args
 
@@ -388,20 +648,27 @@ class SvcHub(object):
         if al.rsp_jtr:
             al.rsp_slp = 0.000001
 
-        al.th_covers = set(al.th_covers.split(","))
+        zsl = al.th_covers.split(",")
+        zsl = [x.strip() for x in zsl]
+        zsl = [x for x in zsl if x]
+        al.th_covers = zsl
+        al.th_coversd = zsl + ["." + x for x in zsl]
+        al.th_covers_set = set(al.th_covers)
+        al.th_coversd_set = set(al.th_coversd)
 
         for k in "c".split(" "):
             vl = getattr(al, k)
             if not vl:
                 continue
 
-            vl = [os.path.expanduser(x) if x.startswith("~") else x for x in vl]
+            vl = [os.path.expandvars(os.path.expanduser(x)) for x in vl]
             setattr(al, k, vl)
 
         for k in "lo hist ssl_log".split(" "):
             vs = getattr(al, k)
-            if vs and vs.startswith("~"):
-                setattr(al, k, os.path.expanduser(vs))
+            if vs:
+                vs = os.path.expandvars(os.path.expanduser(vs))
+                setattr(al, k, vs)
 
         for k in "sus_urls nonsus_urls".split(" "):
             vs = getattr(al, k)
@@ -410,19 +677,86 @@ class SvcHub(object):
             else:
                 setattr(al, k, re.compile(vs))
 
+        for k in "tftp_lsf".split(" "):
+            vs = getattr(al, k)
+            if not vs or vs == "no":
+                setattr(al, k, None)
+            else:
+                setattr(al, k, re.compile("^" + vs + "$"))
+
         if not al.sus_urls:
             al.ban_url = "no"
         elif al.ban_url == "no":
             al.sus_urls = None
 
+        al.xff_hdr = al.xff_hdr.lower()
+        al.idp_h_usr = al.idp_h_usr.lower()
+        al.idp_h_grp = al.idp_h_grp.lower()
+        al.idp_h_key = al.idp_h_key.lower()
+
+        al.ftp_ipa_nm = build_netmap(al.ftp_ipa or al.ipa)
+        al.tftp_ipa_nm = build_netmap(al.tftp_ipa or al.ipa)
+
+        mte = ODict.fromkeys(DEF_MTE.split(","), True)
+        al.mte = odfusion(mte, al.mte)
+
+        mth = ODict.fromkeys(DEF_MTH.split(","), True)
+        al.mth = odfusion(mth, al.mth)
+
+        exp = ODict.fromkeys(DEF_EXP.split(" "), True)
+        al.exp_md = odfusion(exp, al.exp_md.replace(" ", ","))
+        al.exp_lg = odfusion(exp, al.exp_lg.replace(" ", ","))
+
+        for k in ["no_hash", "no_idx", "og_ua"]:
+            ptn = getattr(self.args, k)
+            if ptn:
+                setattr(self.args, k, re.compile(ptn))
+
+        for k in ["idp_gsep"]:
+            ptn = getattr(self.args, k)
+            if "]" in ptn:
+                ptn = "]" + ptn.replace("]", "")
+            if "[" in ptn:
+                ptn = ptn.replace("[", "") + "["
+            if "-" in ptn:
+                ptn = ptn.replace("-", "") + "-"
+
+            ptn = ptn.replace("\\", "\\\\").replace("^", "\\^")
+            setattr(self.args, k, re.compile("[%s]" % (ptn,)))
+
+        try:
+            zf1, zf2 = self.args.rm_retry.split("/")
+            self.args.rm_re_t = float(zf1)
+            self.args.rm_re_r = float(zf2)
+        except:
+            raise Exception("invalid --rm-retry [%s]" % (self.args.rm_retry,))
+
+        try:
+            zf1, zf2 = self.args.mv_retry.split("/")
+            self.args.mv_re_t = float(zf1)
+            self.args.mv_re_r = float(zf2)
+        except:
+            raise Exception("invalid --mv-retry [%s]" % (self.args.mv_retry,))
+
+        al.tcolor = al.tcolor.lstrip("#")
+        if len(al.tcolor) == 3:  # fc5 => ffcc55
+            al.tcolor = "".join([x * 2 for x in al.tcolor])
+
         return True
 
+    def _ipa2re(self, txt) -> Optional[re.Pattern]:
+        if txt in ("any", "0", ""):
+            return None
+
+        zs = txt.replace(" ", "").replace(".", "\\.").replace(",", "|")
+        return re.compile("^(?:" + zs + ")")
+
     def _setlimits(self) -> None:
         try:
             import resource
 
             soft, hard = [
-                x if x > 0 else 1024 * 1024
+                int(x) if x > 0 else 1024 * 1024
                 for x in list(resource.getrlimit(resource.RLIMIT_NOFILE))
             ]
         except:
@@ -458,7 +792,7 @@ class SvcHub(object):
             self.args.nc = min(self.args.nc, soft // 2)
 
     def _logname(self) -> str:
-        dt = datetime.utcnow()
+        dt = datetime.now(self.tz)
         fn = str(self.args.lo)
         for fs in "YmdHMS":
             fs = "%" + fs
@@ -479,12 +813,17 @@ class SvcHub(object):
                 sel_fn = "{}.{}".format(fn, ctr)
 
         fn = sel_fn
+        try:
+            os.makedirs(os.path.dirname(fn))
+        except:
+            pass
 
         try:
             if do_xz:
                 import lzma
 
                 lh = lzma.open(fn, "wt", encoding="utf-8", errors="replace", preset=0)
+                self.args.no_logflush = True
             else:
                 lh = open(fn, "wt", encoding="utf-8", errors="replace")
         except:
@@ -571,21 +910,40 @@ class SvcHub(object):
                 self.log("root", "ssdp startup failed;\n" + min_ex(), 3)
 
     def reload(self) -> str:
-        if self.reloading:
-            return "cannot reload; already in progress"
+        with self.up2k.mutex:
+            if self.reloading:
+                return "cannot reload; already in progress"
+            self.reloading = 1
 
-        self.reloading = True
         Daemon(self._reload, "reloading")
         return "reload initiated"
 
-    def _reload(self) -> None:
-        self.log("root", "reload scheduled")
+    def _reload(self, rescan_all_vols: bool = True, up2k: bool = True) -> None:
         with self.up2k.mutex:
-            self.asrv.reload()
-            self.up2k.reload()
+            if self.reloading != 1:
+                return
+            self.reloading = 2
+            self.log("root", "reloading config")
+            self.asrv.reload(9 if up2k else 4)
+            if up2k:
+                self.up2k.reload(rescan_all_vols)
+            else:
+                self.log("root", "reload done")
             self.broker.reload()
+            self.reloading = 0
 
-        self.reloading = False
+    def _reload_blocking(self, rescan_all_vols: bool = True, up2k: bool = True) -> None:
+        while True:
+            with self.up2k.mutex:
+                if self.reloading < 2:
+                    self.reloading = 1
+                    break
+            time.sleep(0.05)
+
+        # try to handle multiple pending IdP reloads at once:
+        time.sleep(0.2)
+
+        self._reload(rescan_all_vols=rescan_all_vols, up2k=up2k)
 
     def stop_thr(self) -> None:
         while not self.stop_req:
@@ -645,19 +1003,25 @@ class SvcHub(object):
         ret = 1
         try:
             self.pr("OPYTHAT")
+            tasks = []
             slp = 0.0
 
             if self.mdns:
-                Daemon(self.mdns.stop)
+                tasks.append(Daemon(self.mdns.stop, "mdns"))
                 slp = time.time() + 0.5
 
             if self.ssdp:
-                Daemon(self.ssdp.stop)
+                tasks.append(Daemon(self.ssdp.stop, "ssdp"))
                 slp = time.time() + 0.5
 
             self.broker.shutdown()
             self.tcpsrv.shutdown()
             self.up2k.shutdown()
+
+            if hasattr(self, "smbd"):
+                slp = max(slp, time.time() + 0.5)
+                tasks.append(Daemon(self.smbd.stop, "smbd"))
+
             if self.thumbsrv:
                 self.thumbsrv.shutdown()
 
@@ -667,17 +1031,19 @@ class SvcHub(object):
                         break
 
                     if n == 3:
-                        self.pr("waiting for thumbsrv (10sec)...")
+                        self.log("root", "waiting for thumbsrv (10sec)...")
 
             if hasattr(self, "smbd"):
-                slp = max(slp, time.time() + 0.5)
-                Daemon(self.kill9, a=(1,))
-                Daemon(self.smbd.stop)
+                zf = max(time.time() - slp, 0)
+                Daemon(self.kill9, a=(zf + 0.5,))
 
             while time.time() < slp:
-                time.sleep(0.1)
+                if not next((x for x in tasks if x.is_alive), None):
+                    break
 
-            self.pr("nailed it", end="")
+                time.sleep(0.05)
+
+            self.log("root", "nailed it")
             ret = self.retcode
         except:
             self.pr("\033[31m[ error during shutdown ]\n{}\033[0m".format(min_ex()))
@@ -687,7 +1053,7 @@ class SvcHub(object):
                 print("\033]0;\033\\", file=sys.stderr, end="")
                 sys.stderr.flush()
 
-            self.pr("\033[0m")
+            self.pr("\033[0m", end="")
             if self.logf:
                 self.logf.close()
 
@@ -699,44 +1065,52 @@ class SvcHub(object):
             return
 
         with self.log_mutex:
-            zd = datetime.utcnow()
+            dt = datetime.now(self.tz)
             ts = self.log_dfmt % (
-                zd.year,
-                zd.month * 100 + zd.day,
-                (zd.hour * 100 + zd.minute) * 100 + zd.second,
-                zd.microsecond // self.log_div,
+                dt.year,
+                dt.month * 100 + dt.day,
+                (dt.hour * 100 + dt.minute) * 100 + dt.second,
+                dt.microsecond // self.log_div,
             )
-            self.logf.write("@%s [%s\033[0m] %s\n" % (ts, src, msg))
 
-            now = time.time()
-            if now >= self.next_day:
-                self._set_next_day()
+            if c and not self.args.no_ansi:
+                if isinstance(c, int):
+                    msg = "\033[3%sm%s\033[0m" % (c, msg)
+                elif "\033" not in c:
+                    msg = "\033[%sm%s\033[0m" % (c, msg)
+                else:
+                    msg = "%s%s\033[0m" % (c, msg)
 
-    def _set_next_day(self) -> None:
-        if self.next_day and self.logf and self.logf_base_fn != self._logname():
+            if "\033" in src:
+                src += "\033[0m"
+
+            if "\033" in msg:
+                msg += "\033[0m"
+
+            self.logf.write("@%s [%-21s] %s\n" % (ts, src, msg))
+            if not self.args.no_logflush:
+                self.logf.flush()
+
+            if dt.day != self.cday or dt.month != self.cmon:
+                self._set_next_day(dt)
+
+    def _set_next_day(self, dt: datetime) -> None:
+        if self.cday and self.logf and self.logf_base_fn != self._logname():
             self.logf.close()
             self._setup_logfile("")
 
-        dt = datetime.utcnow()
-
-        # unix timestamp of next 00:00:00 (leap-seconds safe)
-        day_now = dt.day
-        while dt.day == day_now:
-            dt += timedelta(hours=12)
-
-        dt = dt.replace(hour=0, minute=0, second=0)
-        self.next_day = calendar.timegm(dt.utctimetuple())
+        self.cday = dt.day
+        self.cmon = dt.month
 
     def _log_enabled(self, src: str, msg: str, c: Union[int, str] = 0) -> None:
         """handles logging from all components"""
         with self.log_mutex:
-            now = time.time()
-            if now >= self.next_day:
-                dt = datetime.utcfromtimestamp(now)
+            dt = datetime.now(self.tz)
+            if dt.day != self.cday or dt.month != self.cmon:
                 zs = "{}\n" if self.no_ansi else "\033[36m{}\033[0m\n"
                 zs = zs.format(dt.strftime("%Y-%m-%d"))
                 print(zs, end="")
-                self._set_next_day()
+                self._set_next_day(dt)
                 if self.logf:
                     self.logf.write(zs)
 
@@ -755,12 +1129,11 @@ class SvcHub(object):
                 else:
                     msg = "%s%s\033[0m" % (c, msg)
 
-            zd = datetime.utcfromtimestamp(now)
             ts = self.log_efmt % (
-                zd.hour,
-                zd.minute,
-                zd.second,
-                zd.microsecond // self.log_div,
+                dt.hour,
+                dt.minute,
+                dt.second,
+                dt.microsecond // self.log_div,
             )
             msg = fmt % (ts, src, msg)
             try:
@@ -776,6 +1149,8 @@ class SvcHub(object):
 
             if self.logf:
                 self.logf.write(msg)
+                if not self.args.no_logflush:
+                    self.logf.flush()
 
     def pr(self, *a: Any, **ka: Any) -> None:
         try:

copyparty/szip.py

@@ -6,6 +6,7 @@ import stat
 import time
 import zlib
 
+from .authsrv import AuthSrv
 from .bos import bos
 from .sutil import StreamArc, errdesc
 from .util import min_ex, sanitize_fn, spack, sunpack, yieldfile
@@ -36,9 +37,7 @@ def dostime2unix(buf: bytes) -> int:
 
 
 def unixtime2dos(ts: int) -> bytes:
-    tt = time.gmtime(ts + 1)
-    dy, dm, dd, th, tm, ts = list(tt)[:6]
-
+    dy, dm, dd, th, tm, ts, _, _, _ = time.gmtime(ts + 1)
     bd = ((dy - 1980) << 9) + (dm << 5) + dd
     bt = (th << 11) + (tm << 5) + ts // 2
     try:
@@ -218,12 +217,13 @@ class StreamZip(StreamArc):
     def __init__(
         self,
         log: "NamedLogger",
+        asrv: AuthSrv,
         fgen: Generator[dict[str, Any], None, None],
         utf8: bool = False,
         pre_crc: bool = False,
         **kwargs: Any
     ) -> None:
-        super(StreamZip, self).__init__(log, fgen)
+        super(StreamZip, self).__init__(log, asrv, fgen)
 
         self.utf8 = utf8
         self.pre_crc = pre_crc
@@ -248,7 +248,7 @@ class StreamZip(StreamArc):
 
         crc = 0
         if self.pre_crc:
-            for buf in yieldfile(src):
+            for buf in yieldfile(src, self.args.iobuf):
                 crc = zlib.crc32(buf, crc)
 
             crc &= 0xFFFFFFFF
@@ -257,7 +257,7 @@ class StreamZip(StreamArc):
         buf = gen_hdr(None, name, sz, ts, self.utf8, crc, self.pre_crc)
         yield self._ct(buf)
 
-        for buf in yieldfile(src):
+        for buf in yieldfile(src, self.args.iobuf):
             if not self.pre_crc:
                 crc = zlib.crc32(buf, crc)
 
@@ -276,6 +276,7 @@ class StreamZip(StreamArc):
     def gen(self) -> Generator[bytes, None, None]:
         errf: dict[str, Any] = {}
         errors = []
+        mbuf = b""
         try:
             for f in self.fgen:
                 if "err" in f:
@@ -284,15 +285,22 @@ class StreamZip(StreamArc):
 
                 try:
                     for x in self.ser(f):
-                        yield x
+                        mbuf += x
+                        if len(mbuf) >= 16384:
+                            yield mbuf
+                            mbuf = b""
                 except GeneratorExit:
                     raise
                 except:
                     ex = min_ex(5, True).replace("\n", "\n-- ")
                     errors.append((f["vp"], ex))
 
+            if mbuf:
+                yield mbuf
+                mbuf = b""
+
             if errors:
-                errf, txt = errdesc(errors)
+                errf, txt = errdesc(self.asrv.vfs, errors)
                 self.log("\n".join(([repr(errf)] + txt[1:])))
                 for x in self.ser(errf):
                     yield x
@@ -300,20 +308,23 @@ class StreamZip(StreamArc):
             cdir_pos = self.pos
             for name, sz, ts, crc, h_pos in self.items:
                 buf = gen_hdr(h_pos, name, sz, ts, self.utf8, crc, self.pre_crc)
-                yield self._ct(buf)
+                mbuf += self._ct(buf)
+                if len(mbuf) >= 16384:
+                    yield mbuf
+                    mbuf = b""
             cdir_end = self.pos
 
             _, need_64 = gen_ecdr(self.items, cdir_pos, cdir_end)
             if need_64:
                 ecdir64_pos = self.pos
                 buf = gen_ecdr64(self.items, cdir_pos, cdir_end)
-                yield self._ct(buf)
+                mbuf += self._ct(buf)
 
                 buf = gen_ecdr64_loc(ecdir64_pos)
-                yield self._ct(buf)
+                mbuf += self._ct(buf)
 
             ecdr, _ = gen_ecdr(self.items, cdir_pos, cdir_end)
-            yield self._ct(ecdr)
+            yield mbuf + self._ct(ecdr)
         finally:
             if errf:
                 bos.unlink(errf["ap"])

copyparty/tcpsrv.py

@@ -15,18 +15,25 @@ from .util import (
     E_ADDR_IN_USE,
     E_ADDR_NOT_AVAIL,
     E_UNREACH,
+    HAVE_IPV6,
+    IP6ALL,
+    VF_CAREFUL,
     Netdev,
+    atomic_move,
     min_ex,
     sunpack,
     termsize,
 )
 
 if True:
-    from typing import Generator
+    from typing import Generator, Union
 
 if TYPE_CHECKING:
     from .svchub import SvcHub
 
+if not hasattr(socket, "AF_UNIX"):
+    setattr(socket, "AF_UNIX", -9001)
+
 if not hasattr(socket, "IPPROTO_IPV6"):
     setattr(socket, "IPPROTO_IPV6", 41)
 
@@ -110,8 +117,10 @@ class TcpSrv(object):
 
         eps = {
             "127.0.0.1": Netdev("127.0.0.1", 0, "", "local only"),
-            "::1": Netdev("::1", 0, "", "local only"),
         }
+        if HAVE_IPV6:
+            eps["::1"] = Netdev("::1", 0, "", "local only")
+
         nonlocals = [x for x in self.args.i if x not in [k.split("/")[0] for k in eps]]
         if nonlocals:
             try:
@@ -213,14 +222,41 @@ class TcpSrv(object):
         if self.args.qr or self.args.qrs:
             self.qr = self._qr(qr1, qr2)
 
+    def nlog(self, msg: str, c: Union[int, str] = 0) -> None:
+        self.log("tcpsrv", msg, c)
+
     def _listen(self, ip: str, port: int) -> None:
-        ipv = socket.AF_INET6 if ":" in ip else socket.AF_INET
+        uds_perm = uds_gid = -1
+        if "unix:" in ip:
+            tcp = False
+            ipv = socket.AF_UNIX
+            uds = ip.split(":")
+            ip = uds[-1]
+            if len(uds) > 2:
+                uds_perm = int(uds[1], 8)
+            if len(uds) > 3:
+                try:
+                    uds_gid = int(uds[2])
+                except:
+                    import grp
+
+                    uds_gid = grp.getgrnam(uds[2]).gr_gid
+
+        elif ":" in ip:
+            tcp = True
+            ipv = socket.AF_INET6
+        else:
+            tcp = True
+            ipv = socket.AF_INET
+
         srv = socket.socket(ipv, socket.SOCK_STREAM)
 
         if not ANYWIN or self.args.reuseaddr:
             srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
 
-        srv.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
+        if tcp:
+            srv.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
+
         srv.settimeout(None)  # < does not inherit, ^ opts above do
 
         try:
@@ -232,20 +268,53 @@ class TcpSrv(object):
             srv.setsockopt(socket.SOL_IP, socket.IP_FREEBIND, 1)
 
         try:
-            srv.bind((ip, port))
-            sport = srv.getsockname()[1]
+            if tcp:
+                srv.bind((ip, port))
+            else:
+                if ANYWIN or self.args.rm_sck:
+                    if os.path.exists(ip):
+                        os.unlink(ip)
+                    srv.bind(ip)
+                else:
+                    tf = "%s.%d" % (ip, os.getpid())
+                    if os.path.exists(tf):
+                        os.unlink(tf)
+                    srv.bind(tf)
+                    if uds_gid != -1:
+                        os.chown(tf, -1, uds_gid)
+                    if uds_perm != -1:
+                        os.chmod(tf, uds_perm)
+                    atomic_move(self.nlog, tf, ip, VF_CAREFUL)
+
+            sport = srv.getsockname()[1] if tcp else port
             if port != sport:
                 # linux 6.0.16 lets you bind a port which is in use
                 # except it just gives you a random port instead
                 raise OSError(E_ADDR_IN_USE[0], "")
             self.srv.append(srv)
         except (OSError, socket.error) as ex:
+            try:
+                srv.close()
+            except:
+                pass
+
+            e = ""
             if ex.errno in E_ADDR_IN_USE:
                 e = "\033[1;31mport {} is busy on interface {}\033[0m".format(port, ip)
+                if not tcp:
+                    e = "\033[1;31munix-socket {} is busy\033[0m".format(ip)
             elif ex.errno in E_ADDR_NOT_AVAIL:
                 e = "\033[1;31minterface {} does not exist\033[0m".format(ip)
-            else:
+
+            if not e:
+                if not tcp:
+                    t = "\n\n\n  NOTE: this crash may be due to a unix-socket bug; try --rm-sck\n"
+                    self.log("tcpsrv", t, 2)
                 raise
+
+            if not tcp and not self.args.rm_sck:
+                e += "; maybe this is a bug? try --rm-sck"
+
             raise Exception(e)
 
     def run(self) -> None:
@@ -253,7 +322,17 @@ class TcpSrv(object):
         bound: list[tuple[str, int]] = []
         srvs: list[socket.socket] = []
         for srv in self.srv:
-            ip, port = srv.getsockname()[:2]
+            if srv.family == socket.AF_UNIX:
+                tcp = False
+                ip = re.sub(r"\.[0-9]+$", "", srv.getsockname())
+                port = 0
+            else:
+                tcp = True
+                ip, port = srv.getsockname()[:2]
+
+            if ip == IP6ALL:
+                ip = "::"  # jython
+
             try:
                 srv.listen(self.args.nc)
                 try:
@@ -275,13 +354,19 @@ class TcpSrv(object):
                     srv.close()
                     continue
 
+                t = "\n\nERROR: could not open listening socket, probably because one of the server ports ({}) is busy on one of the requested interfaces ({}); avoid this issue by specifying a different port (-p 3939) and/or a specific interface to listen on (-i 192.168.56.1)\n"
+                self.log("tcpsrv", t.format(port, ip), 1)
                 raise
 
             bound.append((ip, port))
             srvs.append(srv)
             fno = srv.fileno()
-            hip = "[{}]".format(ip) if ":" in ip else ip
-            msg = "listening @ {}:{}  f{} p{}".format(hip, port, fno, os.getpid())
+            if tcp:
+                hip = "[{}]".format(ip) if ":" in ip else ip
+                msg = "listening @ {}:{}  f{} p{}".format(hip, port, fno, os.getpid())
+            else:
+                msg = "listening @ {}  f{} p{}".format(ip, fno, os.getpid())
+
             self.log("tcpsrv", msg)
             if self.args.q:
                 print(msg)
@@ -298,6 +383,7 @@ class TcpSrv(object):
         self.hub.start_zeroconf()
         gencert(self.log, self.args, self.netdevs)
         self.hub.restart_ftpd()
+        self.hub.restart_tftpd()
 
     def shutdown(self) -> None:
         self.stopping = True
@@ -333,6 +419,8 @@ class TcpSrv(object):
     def detect_interfaces(self, listen_ips: list[str]) -> dict[str, Netdev]:
         from .stolen.ifaddr import get_adapters
 
+        listen_ips = [x for x in listen_ips if "unix:" not in x]
+
         nics = get_adapters(True)
         eps: dict[str, Netdev] = {}
         for nic in nics:
@@ -451,6 +539,12 @@ class TcpSrv(object):
         sys.stderr.flush()
 
     def _qr(self, t1: dict[str, list[int]], t2: dict[str, list[int]]) -> str:
+        t2c = {zs: zli for zs, zli in t2.items() if zs in ("127.0.0.1", "::1")}
+        t2b = {zs: zli for zs, zli in t2.items() if ":" in zs and zs not in t2c}
+        t2 = {zs: zli for zs, zli in t2.items() if zs not in t2b and zs not in t2c}
+        t2.update(t2b)  # first ipv4, then ipv6...
+        t2.update(t2c)  # ...and finally localhost
+
         ip = None
         ips = list(t1) + list(t2)
         qri = self.args.qri

copyparty/tftpd.py

@@ -0,0 +1,455 @@
+# coding: utf-8
+from __future__ import print_function, unicode_literals
+
+try:
+    from types import SimpleNamespace
+except:
+
+    class SimpleNamespace(object):
+        def __init__(self, **attr):
+            self.__dict__.update(attr)
+
+
+import logging
+import os
+import re
+import socket
+import stat
+import threading
+import time
+from datetime import datetime
+
+try:
+    import inspect
+except:
+    pass
+
+from partftpy import (
+    TftpContexts,
+    TftpPacketFactory,
+    TftpPacketTypes,
+    TftpServer,
+    TftpStates,
+)
+from partftpy.TftpShared import TftpException
+
+from .__init__ import EXE, PY2, TYPE_CHECKING
+from .authsrv import VFS
+from .bos import bos
+from .util import UTC, BytesIO, Daemon, ODict, exclude_dotfiles, min_ex, runhook, undot
+
+if True:  # pylint: disable=using-constant-test
+    from typing import Any, Union
+
+if TYPE_CHECKING:
+    from .svchub import SvcHub
+
+if PY2:
+    range = xrange  # type: ignore
+
+
+lg = logging.getLogger("tftp")
+debug, info, warning, error = (lg.debug, lg.info, lg.warning, lg.error)
+
+
+def noop(*a, **ka) -> None:
+    pass
+
+
+def _serverInitial(self, pkt: Any, raddress: str, rport: int) -> bool:
+    info("connection from %s:%s", raddress, rport)
+    ret = _sinitial[0](self, pkt, raddress, rport)
+    nm = _hub[0].args.tftp_ipa_nm
+    if nm and not nm.map(raddress):
+        yeet("client rejected (--tftp-ipa): %s" % (raddress,))
+    return ret
+
+
+# patch ipa-check into partftpd (part 1/2)
+_hub: list["SvcHub"] = []
+_sinitial: list[Any] = []
+
+
+class Tftpd(object):
+    def __init__(self, hub: "SvcHub") -> None:
+        self.hub = hub
+        self.args = hub.args
+        self.asrv = hub.asrv
+        self.log = hub.log
+        self.mutex = threading.Lock()
+
+        _hub[:] = []
+        _hub.append(hub)
+
+        lg.setLevel(logging.DEBUG if self.args.tftpv else logging.INFO)
+        for x in ["partftpy", "partftpy.TftpStates", "partftpy.TftpServer"]:
+            lgr = logging.getLogger(x)
+            lgr.setLevel(logging.DEBUG if self.args.tftpv else logging.INFO)
+
+        if not self.args.tftpv and not self.args.tftpvv:
+            # contexts -> states -> packettypes -> shared
+            # contexts -> packetfactory
+            # packetfactory -> packettypes
+            Cs = [
+                TftpPacketTypes,
+                TftpPacketFactory,
+                TftpStates,
+                TftpContexts,
+                TftpServer,
+            ]
+            cbak = []
+            if not self.args.tftp_no_fast and not EXE and not PY2:
+                try:
+                    ptn = re.compile(r"(^\s*)log\.debug\(.*\)$")
+                    for C in Cs:
+                        cbak.append(C.__dict__)
+                        src1 = inspect.getsource(C).split("\n")
+                        src2 = "\n".join([ptn.sub("\\1pass", ln) for ln in src1])
+                        cfn = C.__spec__.origin
+                        exec (compile(src2, filename=cfn, mode="exec"), C.__dict__)
+                except Exception:
+                    t = "failed to optimize tftp code; run with --tftp-no-fast if there are issues:\n"
+                    self.log("tftp", t + min_ex(), 3)
+                    for n, zd in enumerate(cbak):
+                        Cs[n].__dict__ = zd
+
+            for C in Cs:
+                C.log.debug = noop
+
+        # patch ipa-check into partftpd (part 2/2)
+        _sinitial[:] = []
+        _sinitial.append(TftpStates.TftpServerState.serverInitial)
+        TftpStates.TftpServerState.serverInitial = _serverInitial
+
+        # patch vfs into partftpy
+        TftpContexts.open = self._open
+        TftpStates.open = self._open
+
+        fos = SimpleNamespace()
+        for k in os.__dict__:
+            try:
+                setattr(fos, k, getattr(os, k))
+            except:
+                pass
+        fos.access = self._access
+        fos.mkdir = self._mkdir
+        fos.unlink = self._unlink
+        fos.sep = "/"
+        TftpContexts.os = fos
+        TftpServer.os = fos
+        TftpStates.os = fos
+
+        fop = SimpleNamespace()
+        for k in os.path.__dict__:
+            try:
+                setattr(fop, k, getattr(os.path, k))
+            except:
+                pass
+        fop.abspath = self._p_abspath
+        fop.exists = self._p_exists
+        fop.isdir = self._p_isdir
+        fop.normpath = self._p_normpath
+        fos.path = fop
+
+        self._disarm(fos)
+
+        self.port = int(self.args.tftp)
+        self.srv = []
+        self.ips = []
+
+        ports = []
+        if self.args.tftp_pr:
+            p1, p2 = [int(x) for x in self.args.tftp_pr.split("-")]
+            ports = list(range(p1, p2 + 1))
+
+        ips = self.args.i
+        if "::" in ips:
+            ips.append("0.0.0.0")
+
+        ips = [x for x in ips if "unix:" not in x]
+
+        if self.args.tftp4:
+            ips = [x for x in ips if ":" not in x]
+
+        if not ips:
+            t = "cannot start tftp-server; no compatible IPs in -i"
+            self.nlog(t, 1)
+            return
+
+        ips = list(ODict.fromkeys(ips))  # dedup
+
+        for ip in ips:
+            name = "tftp_%s" % (ip,)
+            Daemon(self._start, name, [ip, ports])
+            time.sleep(0.2)  # give dualstack a chance
+
+    def nlog(self, msg: str, c: Union[int, str] = 0) -> None:
+        self.log("tftp", msg, c)
+
+    def _start(self, ip, ports):
+        fam = socket.AF_INET6 if ":" in ip else socket.AF_INET
+        have_been_alive = False
+        while True:
+            srv = TftpServer.TftpServer("/", self._ls)
+            with self.mutex:
+                self.srv.append(srv)
+                self.ips.append(ip)
+
+            try:
+                # this is the listen loop; it should block forever
+                srv.listen(ip, self.port, af_family=fam, ports=ports)
+            except:
+                with self.mutex:
+                    self.srv.remove(srv)
+                    self.ips.remove(ip)
+
+                try:
+                    srv.sock.close()
+                except:
+                    pass
+
+                try:
+                    bound = bool(srv.listenport)
+                except:
+                    bound = False
+
+                if bound:
+                    # this instance has managed to bind at least once
+                    have_been_alive = True
+
+                if have_been_alive:
+                    t = "tftp server [%s]:%d crashed; restarting in 3 sec:\n%s"
+                    error(t, ip, self.port, min_ex())
+                    time.sleep(3)
+                    continue
+
+                # server failed to start; could be due to dualstack (ipv6 managed to bind and this is ipv4)
+                if ip != "0.0.0.0" or "::" not in self.ips:
+                    # nope, it's fatal
+                    t = "tftp server [%s]:%d failed to start:\n%s"
+                    error(t, ip, self.port, min_ex())
+
+                # yep; ignore
+                # (TODO: move the "listening @ ..." infolog in partftpy to
+                #   after the bind attempt so it doesn't print twice)
+                return
+
+            info("tftp server [%s]:%d terminated", ip, self.port)
+            break
+
+    def stop(self):
+        with self.mutex:
+            srvs = self.srv[:]
+
+        for srv in srvs:
+            srv.stop()
+
+    def _v2a(self, caller: str, vpath: str, perms: list, *a: Any) -> tuple[VFS, str]:
+        vpath = vpath.replace("\\", "/").lstrip("/")
+        if not perms:
+            perms = [True, True]
+
+        debug('%s("%s", %s) %s\033[K\033[0m', caller, vpath, str(a), perms)
+        vfs, rem = self.asrv.vfs.get(vpath, "*", *perms)
+        if not vfs.realpath:
+            raise Exception("unmapped vfs")
+        return vfs, vfs.canonical(rem)
+
+    def _ls(self, vpath: str, raddress: str, rport: int, force=False) -> Any:
+        # generate file listing if vpath is dir.txt and return as file object
+        if not force:
+            vpath, fn = os.path.split(vpath.replace("\\", "/"))
+            ptn = self.args.tftp_lsf
+            if not ptn or not ptn.match(fn.lower()):
+                return None
+
+        vn, rem = self.asrv.vfs.get(vpath, "*", True, False)
+        fsroot, vfs_ls, vfs_virt = vn.ls(
+            rem,
+            "*",
+            not self.args.no_scandir,
+            [[True, False]],
+        )
+        dnames = set([x[0] for x in vfs_ls if stat.S_ISDIR(x[1].st_mode)])
+        dirs1 = [(v.st_mtime, v.st_size, k + "/") for k, v in vfs_ls if k in dnames]
+        fils1 = [(v.st_mtime, v.st_size, k) for k, v in vfs_ls if k not in dnames]
+        real1 = dirs1 + fils1
+        realt = [(datetime.fromtimestamp(mt, UTC), sz, fn) for mt, sz, fn in real1]
+        reals = [
+            (
+                "%04d-%02d-%02d %02d:%02d:%02d"
+                % (
+                    zd.year,
+                    zd.month,
+                    zd.day,
+                    zd.hour,
+                    zd.minute,
+                    zd.second,
+                ),
+                sz,
+                fn,
+            )
+            for zd, sz, fn in realt
+        ]
+        virs = [("????-??-?? ??:??:??", 0, k + "/") for k in vfs_virt.keys()]
+        ls = virs + reals
+
+        if "*" not in vn.axs.udot:
+            names = set(exclude_dotfiles([x[2] for x in ls]))
+            ls = [x for x in ls if x[2] in names]
+
+        try:
+            biggest = max([x[1] for x in ls])
+        except:
+            biggest = 0
+
+        perms = []
+        if "*" in vn.axs.uread:
+            perms.append("read")
+        if "*" in vn.axs.udot:
+            perms.append("hidden")
+        if "*" in vn.axs.uwrite:
+            if "*" in vn.axs.udel:
+                perms.append("overwrite")
+            else:
+                perms.append("write")
+
+        fmt = "{{}}  {{:{},}}  {{}}"
+        fmt = fmt.format(len("{:,}".format(biggest)))
+        retl = ["# permissions: %s" % (", ".join(perms),)]
+        retl += [fmt.format(*x) for x in ls]
+        ret = "\n".join(retl).encode("utf-8", "replace")
+        return BytesIO(ret + b"\n")
+
+    def _open(self, vpath: str, mode: str, *a: Any, **ka: Any) -> Any:
+        rd = wr = False
+        if mode == "rb":
+            rd = True
+        elif mode == "wb":
+            wr = True
+        else:
+            raise Exception("bad mode %s" % (mode,))
+
+        vfs, ap = self._v2a("open", vpath, [rd, wr])
+        if wr:
+            if "*" not in vfs.axs.uwrite:
+                yeet("blocked write; folder not world-writable: /%s" % (vpath,))
+
+            if bos.path.exists(ap) and "*" not in vfs.axs.udel:
+                yeet("blocked write; folder not world-deletable: /%s" % (vpath,))
+
+            xbu = vfs.flags.get("xbu")
+            if xbu and not runhook(
+                self.nlog,
+                None,
+                self.hub.up2k,
+                "xbu.tftpd",
+                xbu,
+                ap,
+                vpath,
+                "",
+                "",
+                "",
+                0,
+                0,
+                "8.3.8.7",
+                time.time(),
+                "",
+            ):
+                yeet("blocked by xbu server config: " + vpath)
+
+        if not self.args.tftp_nols and bos.path.isdir(ap):
+            return self._ls(vpath, "", 0, True)
+
+        if not a:
+            a = (self.args.iobuf,)
+
+        return open(ap, mode, *a, **ka)
+
+    def _mkdir(self, vpath: str, *a) -> None:
+        vfs, ap = self._v2a("mkdir", vpath, [])
+        if "*" not in vfs.axs.uwrite:
+            yeet("blocked mkdir; folder not world-writable: /%s" % (vpath,))
+
+        return bos.mkdir(ap)
+
+    def _unlink(self, vpath: str) -> None:
+        # return bos.unlink(self._v2a("stat", vpath, *a)[1])
+        vfs, ap = self._v2a("delete", vpath, [True, False, False, True])
+
+        try:
+            inf = bos.stat(ap)
+        except:
+            return
+
+        if not stat.S_ISREG(inf.st_mode) or inf.st_size:
+            yeet("attempted delete of non-empty file")
+
+        vpath = vpath.replace("\\", "/").lstrip("/")
+        self.hub.up2k.handle_rm("*", "8.3.8.7", [vpath], [], False, False)
+
+    def _access(self, *a: Any) -> bool:
+        return True
+
+    def _p_abspath(self, vpath: str) -> str:
+        return "/" + undot(vpath)
+
+    def _p_normpath(self, *a: Any) -> str:
+        return ""
+
+    def _p_exists(self, vpath: str) -> bool:
+        try:
+            ap = self._v2a("p.exists", vpath, [False, False])[1]
+            bos.stat(ap)
+            return True
+        except:
+            return vpath == "/"
+
+    def _p_isdir(self, vpath: str) -> bool:
+        try:
+            st = bos.stat(self._v2a("p.isdir", vpath, [False, False])[1])
+            ret = stat.S_ISDIR(st.st_mode)
+            return ret
+        except:
+            return vpath == "/"
+
+    def _hook(self, *a: Any, **ka: Any) -> None:
+        src = inspect.currentframe().f_back.f_code.co_name
+        error("\033[31m%s:hook(%s)\033[0m", src, a)
+        raise Exception("nope")
+
+    def _disarm(self, fos: SimpleNamespace) -> None:
+        fos.chmod = self._hook
+        fos.chown = self._hook
+        fos.close = self._hook
+        fos.ftruncate = self._hook
+        fos.lchown = self._hook
+        fos.link = self._hook
+        fos.listdir = self._hook
+        fos.lstat = self._hook
+        fos.open = self._hook
+        fos.remove = self._hook
+        fos.rename = self._hook
+        fos.replace = self._hook
+        fos.scandir = self._hook
+        fos.stat = self._hook
+        fos.symlink = self._hook
+        fos.truncate = self._hook
+        fos.utime = self._hook
+        fos.walk = self._hook
+
+        fos.path.expanduser = self._hook
+        fos.path.expandvars = self._hook
+        fos.path.getatime = self._hook
+        fos.path.getctime = self._hook
+        fos.path.getmtime = self._hook
+        fos.path.getsize = self._hook
+        fos.path.isabs = self._hook
+        fos.path.isfile = self._hook
+        fos.path.islink = self._hook
+        fos.path.realpath = self._hook
+
+
+def yeet(msg: str) -> None:
+    warning(msg)
+    raise TftpException(msg)

copyparty/th_cli.py

@@ -31,7 +31,7 @@ class ThumbCli(object):
             if not c:
                 raise Exception()
         except:
-            c = {k: {} for k in ["thumbable", "pil", "vips", "ffi", "ffv", "ffa"]}
+            c = {k: set() for k in ["thumbable", "pil", "vips", "ffi", "ffv", "ffa"]}
 
         self.thumbable = c["thumbable"]
         self.fmt_pil = c["pil"]
@@ -57,9 +57,10 @@ class ThumbCli(object):
         if is_vid and "dvthumb" in dbv.flags:
             return None
 
-        want_opus = fmt in ("opus", "caf")
+        want_opus = fmt in ("opus", "caf", "mp3")
         is_au = ext in self.fmt_ffa
-        if is_au:
+        is_vau = want_opus and ext in self.fmt_ffv
+        if is_au or is_vau:
             if want_opus:
                 if self.args.no_acode:
                     return None
@@ -78,23 +79,46 @@ class ThumbCli(object):
         if rem.startswith(".hist/th/") and rem.split(".")[-1] in ["webp", "jpg", "png"]:
             return os.path.join(ptop, rem)
 
-        if fmt == "j" and self.args.th_no_jpg:
-            fmt = "w"
+        if fmt[:1] in "jw":
+            sfmt = fmt[:1]
 
-        if fmt == "w":
-            if (
-                self.args.th_no_webp
-                or (is_img and not self.can_webp)
-                or (self.args.th_ff_jpg and (not is_img or preferred == "ff"))
-            ):
-                fmt = "j"
+            if sfmt == "j" and self.args.th_no_jpg:
+                sfmt = "w"
+
+            if sfmt == "w":
+                if (
+                    self.args.th_no_webp
+                    or (is_img and not self.can_webp)
+                    or (self.args.th_ff_jpg and (not is_img or preferred == "ff"))
+                ):
+                    sfmt = "j"
+
+            vf_crop = dbv.flags["crop"]
+            vf_th3x = dbv.flags["th3x"]
+
+            if "f" in vf_crop:
+                sfmt += "f" if "n" in vf_crop else ""
+            else:
+                sfmt += "f" if "f" in fmt else ""
+
+            if "f" in vf_th3x:
+                sfmt += "3" if "y" in vf_th3x else ""
+            else:
+                sfmt += "3" if "3" in fmt else ""
+
+            fmt = sfmt
+
+        elif fmt[:1] == "p" and not is_au and not is_vid:
+            t = "cannot thumbnail [%s]: png only allowed for waveforms"
+            self.log(t % (rem), 6)
+            return None
 
         histpath = self.asrv.vfs.histtab.get(ptop)
         if not histpath:
             self.log("no histpath for [{}]".format(ptop))
             return None
 
-        tpath = thumb_path(histpath, rem, mtime, fmt)
+        tpath = thumb_path(histpath, rem, mtime, fmt, self.fmt_ffa)
         tpaths = [tpath]
         if fmt == "w":
             # also check for jpg (maybe webp is unavailable)

copyparty/th_srv.py

@@ -12,13 +12,13 @@ import time
 
 from queue import Queue
 
-from .__init__ import ANYWIN, TYPE_CHECKING
+from .__init__ import ANYWIN, PY2, TYPE_CHECKING
 from .authsrv import VFS
 from .bos import bos
-from .mtag import HAVE_FFMPEG, HAVE_FFPROBE, ffprobe
+from .mtag import HAVE_FFMPEG, HAVE_FFPROBE, au_unpk, ffprobe
+from .util import BytesIO  # type: ignore
 from .util import (
     FFMPEG_URL,
-    BytesIO,
     Cooldown,
     Daemon,
     Pebkac,
@@ -28,6 +28,8 @@ from .util import (
     runcmd,
     statdir,
     vsplit,
+    wrename,
+    wunlink,
 )
 
 if True:  # pylint: disable=using-constant-test
@@ -36,22 +38,44 @@ if True:  # pylint: disable=using-constant-test
 if TYPE_CHECKING:
     from .svchub import SvcHub
 
+if PY2:
+    range = xrange  # type: ignore
+
 HAVE_PIL = False
+HAVE_PILF = False
 HAVE_HEIF = False
 HAVE_AVIF = False
 HAVE_WEBP = False
 
 try:
-    from PIL import ExifTags, Image, ImageOps
+    if os.environ.get("PRTY_NO_PIL"):
+        raise Exception()
+
+    from PIL import ExifTags, Image, ImageFont, ImageOps
 
     HAVE_PIL = True
     try:
+        if os.environ.get("PRTY_NO_PILF"):
+            raise Exception()
+
+        ImageFont.load_default(size=16)
+        HAVE_PILF = True
+    except:
+        pass
+
+    try:
+        if os.environ.get("PRTY_NO_PIL_WEBP"):
+            raise Exception()
+
         Image.new("RGB", (2, 2)).save(BytesIO(), format="webp")
         HAVE_WEBP = True
     except:
         pass
 
     try:
+        if os.environ.get("PRTY_NO_PIL_HEIF"):
+            raise Exception()
+
         from pyheif_pillow_opener import register_heif_opener
 
         register_heif_opener()
@@ -60,6 +84,9 @@ try:
         pass
 
     try:
+        if os.environ.get("PRTY_NO_PIL_AVIF"):
+            raise Exception()
+
         import pillow_avif  # noqa: F401  # pylint: disable=unused-import
 
         HAVE_AVIF = True
@@ -71,6 +98,9 @@ except:
     pass
 
 try:
+    if os.environ.get("PRTY_NO_VIPS"):
+        raise Exception()
+
     HAVE_VIPS = True
     import pyvips
 
@@ -79,29 +109,36 @@ except:
     HAVE_VIPS = False
 
 
-def thumb_path(histpath: str, rem: str, mtime: float, fmt: str) -> str:
+def thumb_path(histpath: str, rem: str, mtime: float, fmt: str, ffa: set[str]) -> str:
     # base16 = 16 = 256
     # b64-lc = 38 = 1444
     # base64 = 64 = 4096
     rd, fn = vsplit(rem)
-    if rd:
-        h = hashlib.sha512(afsenc(rd)).digest()
-        b64 = base64.urlsafe_b64encode(h).decode("ascii")[:24]
-        rd = "{}/{}/".format(b64[:2], b64[2:4]).lower() + b64
-    else:
-        rd = "top"
+    if not rd:
+        rd = "\ntop"
+
+    # spectrograms are never cropped; strip fullsize flag
+    ext = rem.split(".")[-1].lower()
+    if ext in ffa and fmt[:2] in ("wf", "jf"):
+        fmt = fmt.replace("f", "")
+
+    rd += "\n" + fmt
+    h = hashlib.sha512(afsenc(rd)).digest()
+    b64 = base64.urlsafe_b64encode(h).decode("ascii")[:24]
+    rd = ("%s/%s/" % (b64[:2], b64[2:4])).lower() + b64
 
     # could keep original filenames but this is safer re pathlen
     h = hashlib.sha512(afsenc(fn)).digest()
     fn = base64.urlsafe_b64encode(h).decode("ascii")[:24]
 
-    if fmt in ("opus", "caf"):
+    if fmt in ("opus", "caf", "mp3"):
         cat = "ac"
     else:
-        fmt = "webp" if fmt == "w" else "png" if fmt == "p" else "jpg"
+        fc = fmt[:1]
+        fmt = "webp" if fc == "w" else "png" if fc == "p" else "jpg"
         cat = "th"
 
-    return "{}/{}/{}/{}.{:x}.{}".format(histpath, cat, rd, fn, int(mtime), fmt)
+    return "%s/%s/%s/%s.%x.%s" % (histpath, cat, rd, fn, int(mtime), fmt)
 
 
 class ThumbSrv(object):
@@ -115,10 +152,12 @@ class ThumbSrv(object):
 
         self.mutex = threading.Lock()
         self.busy: dict[str, list[threading.Condition]] = {}
+        self.ram: dict[str, float] = {}
+        self.memcond = threading.Condition(self.mutex)
         self.stopping = False
         self.nthr = max(1, self.args.th_mt)
 
-        self.q: Queue[Optional[tuple[str, str, VFS]]] = Queue(self.nthr * 4)
+        self.q: Queue[Optional[tuple[str, str, str, VFS]]] = Queue(self.nthr * 4)
         for n in range(self.nthr):
             Daemon(self.worker, "thumb-{}-{}".format(n, self.nthr))
 
@@ -183,9 +222,10 @@ class ThumbSrv(object):
         with self.mutex:
             return not self.nthr
 
-    def getres(self, vn: VFS) -> tuple[int, int]:
+    def getres(self, vn: VFS, fmt: str) -> tuple[int, int]:
+        mul = 3 if "3" in fmt else 1
         w, h = vn.flags["thsize"].split("x")
-        return int(w), int(h)
+        return int(w) * mul, int(h) * mul
 
     def get(self, ptop: str, rem: str, mtime: float, fmt: str) -> Optional[str]:
         histpath = self.asrv.vfs.histtab.get(ptop)
@@ -193,14 +233,14 @@ class ThumbSrv(object):
             self.log("no histpath for [{}]".format(ptop))
             return None
 
-        tpath = thumb_path(histpath, rem, mtime, fmt)
+        tpath = thumb_path(histpath, rem, mtime, fmt, self.fmt_ffa)
         abspath = os.path.join(ptop, rem)
         cond = threading.Condition(self.mutex)
         do_conv = False
         with self.mutex:
             try:
                 self.busy[tpath].append(cond)
-                self.log("wait {}".format(tpath))
+                self.log("joined waiting room for %s" % (tpath,))
             except:
                 thdir = os.path.dirname(tpath)
                 bos.makedirs(os.path.join(thdir, "w"))
@@ -220,8 +260,8 @@ class ThumbSrv(object):
                 self.log("ptop [{}] not in {}".format(ptop, allvols), 3)
                 vn = self.asrv.vfs.all_aps[0][1]
 
-            self.q.put((abspath, tpath, vn))
-            self.log("conv {} \033[0m{}".format(tpath, abspath), c=6)
+            self.q.put((abspath, tpath, fmt, vn))
+            self.log("conv {} :{} \033[0m{}".format(tpath, fmt, abspath), c=6)
 
         while not self.stopping:
             with self.mutex:
@@ -251,46 +291,79 @@ class ThumbSrv(object):
             "ffa": self.fmt_ffa,
         }
 
+    def wait4ram(self, need: float, ttpath: str) -> None:
+        ram = self.args.th_ram_max
+        if need > ram * 0.99:
+            t = "file too big; need %.2f GiB RAM, but --th-ram-max is only %.1f"
+            raise Exception(t % (need, ram))
+
+        while True:
+            with self.mutex:
+                used = sum([v for k, v in self.ram.items() if k != ttpath]) + need
+                if used < ram:
+                    # self.log("XXX self.ram: %s" % (self.ram,), 5)
+                    self.ram[ttpath] = need
+                    return
+            with self.memcond:
+                # self.log("at RAM limit; used %.2f GiB, need %.2f more" % (used-need, need), 1)
+                self.memcond.wait(3)
+
     def worker(self) -> None:
         while not self.stopping:
             task = self.q.get()
             if not task:
                 break
 
-            abspath, tpath, vn = task
+            abspath, tpath, fmt, vn = task
             ext = abspath.split(".")[-1].lower()
             png_ok = False
             funs = []
+
+            if ext in self.args.au_unpk:
+                ap_unpk = au_unpk(self.log, self.args.au_unpk, abspath, vn)
+            else:
+                ap_unpk = abspath
+
             if not bos.path.exists(tpath):
+                want_mp3 = tpath.endswith(".mp3")
+                want_opus = tpath.endswith(".opus") or tpath.endswith(".caf")
+                want_png = tpath.endswith(".png")
+                want_au = want_mp3 or want_opus
                 for lib in self.args.th_dec:
+                    can_au = lib == "ff" and (
+                        ext in self.fmt_ffa or ext in self.fmt_ffv
+                    )
+
                     if lib == "pil" and ext in self.fmt_pil:
                         funs.append(self.conv_pil)
                     elif lib == "vips" and ext in self.fmt_vips:
                         funs.append(self.conv_vips)
-                    elif lib == "ff" and ext in self.fmt_ffi or ext in self.fmt_ffv:
-                        funs.append(self.conv_ffmpeg)
-                    elif lib == "ff" and ext in self.fmt_ffa:
-                        if tpath.endswith(".opus") or tpath.endswith(".caf"):
+                    elif can_au and (want_png or want_au):
+                        if want_opus:
                             funs.append(self.conv_opus)
-                        elif tpath.endswith(".png"):
+                        elif want_mp3:
+                            funs.append(self.conv_mp3)
+                        elif want_png:
                             funs.append(self.conv_waves)
                             png_ok = True
-                        else:
-                            funs.append(self.conv_spec)
-
-            if not png_ok and tpath.endswith(".png"):
-                raise Pebkac(400, "png only allowed for waveforms")
+                    elif lib == "ff" and (ext in self.fmt_ffi or ext in self.fmt_ffv):
+                        funs.append(self.conv_ffmpeg)
+                    elif lib == "ff" and ext in self.fmt_ffa and not want_au:
+                        funs.append(self.conv_spec)
 
             tdir, tfn = os.path.split(tpath)
             ttpath = os.path.join(tdir, "w", tfn)
             try:
-                bos.unlink(ttpath)
+                wunlink(self.log, ttpath, vn.flags)
             except:
                 pass
 
             for fun in funs:
                 try:
-                    fun(abspath, ttpath, vn)
+                    if not png_ok and tpath.endswith(".png"):
+                        raise Exception("png only allowed for waveforms")
+
+                    fun(ap_unpk, ttpath, fmt, vn)
                     break
                 except Exception as ex:
                     msg = "{} could not create thumbnail of {}\n{}"
@@ -304,29 +377,36 @@ class ThumbSrv(object):
                     else:
                         # ffmpeg may spawn empty files on windows
                         try:
-                            os.unlink(ttpath)
+                            wunlink(self.log, ttpath, vn.flags)
                         except:
                             pass
 
+            if abspath != ap_unpk:
+                wunlink(self.log, ap_unpk, vn.flags)
+
             try:
-                bos.rename(ttpath, tpath)
+                wrename(self.log, ttpath, tpath, vn.flags)
             except:
                 pass
 
             with self.mutex:
                 subs = self.busy[tpath]
                 del self.busy[tpath]
+                self.ram.pop(ttpath, None)
 
             for x in subs:
                 with x:
                     x.notify_all()
 
+            with self.memcond:
+                self.memcond.notify_all()
+
         with self.mutex:
             self.nthr -= 1
 
-    def fancy_pillow(self, im: "Image.Image", vn: VFS) -> "Image.Image":
+    def fancy_pillow(self, im: "Image.Image", fmt: str, vn: VFS) -> "Image.Image":
         # exif_transpose is expensive (loads full image + unconditional copy)
-        res = self.getres(vn)
+        res = self.getres(vn, fmt)
         r = max(*res) * 2
         im.thumbnail((r, r), resample=Image.LANCZOS)
         try:
@@ -341,7 +421,7 @@ class ThumbSrv(object):
         if rot in rots:
             im = im.transpose(rots[rot])
 
-        if "nocrop" in vn.flags:
+        if "f" in fmt:
             im.thumbnail(res, resample=Image.LANCZOS)
         else:
             iw, ih = im.size
@@ -351,13 +431,14 @@ class ThumbSrv(object):
 
         return im
 
-    def conv_pil(self, abspath: str, tpath: str, vn: VFS) -> None:
+    def conv_pil(self, abspath: str, tpath: str, fmt: str, vn: VFS) -> None:
+        self.wait4ram(0.2, tpath)
         with Image.open(fsenc(abspath)) as im:
             try:
-                im = self.fancy_pillow(im, vn)
+                im = self.fancy_pillow(im, fmt, vn)
             except Exception as ex:
                 self.log("fancy_pillow {}".format(ex), "90")
-                im.thumbnail(self.getres(vn))
+                im.thumbnail(self.getres(vn, fmt))
 
             fmts = ["RGB", "L"]
             args = {"quality": 40}
@@ -368,7 +449,7 @@ class ThumbSrv(object):
                 # method 0 = pillow-default, fast
                 # method 4 = ffmpeg-default
                 # method 6 = max, slow
-                fmts += ["RGBA", "LA"]
+                fmts.extend(("RGBA", "LA"))
                 args["method"] = 6
             else:
                 # default q = 75
@@ -380,12 +461,13 @@ class ThumbSrv(object):
 
             im.save(tpath, **args)
 
-    def conv_vips(self, abspath: str, tpath: str, vn: VFS) -> None:
+    def conv_vips(self, abspath: str, tpath: str, fmt: str, vn: VFS) -> None:
+        self.wait4ram(0.2, tpath)
         crops = ["centre", "none"]
-        if "nocrop" in vn.flags:
+        if "f" in fmt:
             crops = ["none"]
 
-        w, h = self.getres(vn)
+        w, h = self.getres(vn, fmt)
         kw = {"height": h, "size": "down", "intent": "relative"}
 
         for c in crops:
@@ -397,9 +479,11 @@ class ThumbSrv(object):
                 if c == crops[-1]:
                     raise
 
+        assert img  # type: ignore
         img.write_to_file(tpath, Q=40)
 
-    def conv_ffmpeg(self, abspath: str, tpath: str, vn: VFS) -> None:
+    def conv_ffmpeg(self, abspath: str, tpath: str, fmt: str, vn: VFS) -> None:
+        self.wait4ram(0.2, tpath)
         ret, _ = ffprobe(abspath, int(vn.flags["convt"] / 2))
         if not ret:
             return
@@ -412,12 +496,12 @@ class ThumbSrv(object):
             seek = [b"-ss", "{:.0f}".format(dur / 3).encode("utf-8")]
 
         scale = "scale={0}:{1}:force_original_aspect_ratio="
-        if "nocrop" in vn.flags:
+        if "f" in fmt:
             scale += "decrease,setsar=1:1"
         else:
             scale += "increase,crop={0}:{1},setsar=1:1"
 
-        res = self.getres(vn)
+        res = self.getres(vn, fmt)
         bscale = scale.format(*list(res)).encode("utf-8")
         # fmt: off
         cmd = [
@@ -452,9 +536,9 @@ class ThumbSrv(object):
         cmd += [fsenc(tpath)]
         self._run_ff(cmd, vn)
 
-    def _run_ff(self, cmd: list[bytes], vn: VFS) -> None:
+    def _run_ff(self, cmd: list[bytes], vn: VFS, oom: int = 400) -> None:
         # self.log((b" ".join(cmd)).decode("utf-8"))
-        ret, _, serr = runcmd(cmd, timeout=vn.flags["convt"])
+        ret, _, serr = runcmd(cmd, timeout=vn.flags["convt"], nice=True, oom=oom)
         if not ret:
             return
 
@@ -497,13 +581,26 @@ class ThumbSrv(object):
         self.log(t + txt, c=c)
         raise sp.CalledProcessError(ret, (cmd[0], b"...", cmd[-1]))
 
-    def conv_waves(self, abspath: str, tpath: str, vn: VFS) -> None:
+    def conv_waves(self, abspath: str, tpath: str, fmt: str, vn: VFS) -> None:
         ret, _ = ffprobe(abspath, int(vn.flags["convt"] / 2))
         if "ac" not in ret:
             raise Exception("not audio")
 
-        flt = (
-            b"[0:a:0]"
+        # jt_versi.xm: 405M/839s
+        dur = ret[".dur"][1] if ".dur" in ret else 300
+        need = 0.2 + dur / 3000
+        speedup = b""
+        if need > self.args.th_ram_max * 0.7:
+            self.log("waves too big (need %.2f GiB); trying to optimize" % (need,))
+            need = 0.2 + dur / 4200  # only helps about this much...
+            speedup = b"aresample=8000,"
+        if need > self.args.th_ram_max * 0.96:
+            raise Exception("file too big; cannot waves")
+
+        self.wait4ram(need, tpath)
+
+        flt = b"[0:a:0]" + speedup
+        flt += (
             b"compand=.3|.3:1|1:-90/-60|-60/-40|-40/-30|-20/-20:6:0:-90:0.2"
             b",volume=2"
             b",showwavespic=s=2048x64:colors=white"
@@ -525,12 +622,44 @@ class ThumbSrv(object):
         cmd += [fsenc(tpath)]
         self._run_ff(cmd, vn)
 
-    def conv_spec(self, abspath: str, tpath: str, vn: VFS) -> None:
+        if "pngquant" in vn.flags:
+            wtpath = tpath + ".png"
+            cmd = [
+                b"pngquant",
+                b"--strip",
+                b"--nofs",
+                b"--output",
+                fsenc(wtpath),
+                fsenc(tpath),
+            ]
+            ret = runcmd(cmd, timeout=vn.flags["convt"], nice=True, oom=400)[0]
+            if ret:
+                try:
+                    wunlink(self.log, wtpath, vn.flags)
+                except:
+                    pass
+            else:
+                wrename(self.log, wtpath, tpath, vn.flags)
+
+    def conv_spec(self, abspath: str, tpath: str, fmt: str, vn: VFS) -> None:
         ret, _ = ffprobe(abspath, int(vn.flags["convt"] / 2))
         if "ac" not in ret:
             raise Exception("not audio")
 
-        fc = "[0:a:0]aresample=48000{},showspectrumpic=s=640x512,crop=780:544:70:50[o]"
+        # https://trac.ffmpeg.org/ticket/10797
+        # expect 1 GiB every 600 seconds when duration is tricky;
+        # simple filetypes are generally safer so let's special-case those
+        safe = ("flac", "wav", "aif", "aiff", "opus")
+        coeff = 1800 if abspath.split(".")[-1].lower() in safe else 600
+        dur = ret[".dur"][1] if ".dur" in ret else 300
+        need = 0.2 + dur / coeff
+        self.wait4ram(need, tpath)
+
+        fc = "[0:a:0]aresample=48000{},showspectrumpic=s="
+        if "3" in fmt:
+            fc += "1280x1024,crop=1420:1056:70:48[o]"
+        else:
+            fc += "640x512,crop=780:544:70:48[o]"
 
         if self.args.th_ff_swr:
             fco = ":filter_size=128:cutoff=0.877"
@@ -568,30 +697,71 @@ class ThumbSrv(object):
         cmd += [fsenc(tpath)]
         self._run_ff(cmd, vn)
 
-    def conv_opus(self, abspath: str, tpath: str, vn: VFS) -> None:
-        if self.args.no_acode:
+    def conv_mp3(self, abspath: str, tpath: str, fmt: str, vn: VFS) -> None:
+        quality = self.args.q_mp3.lower()
+        if self.args.no_acode or not quality:
             raise Exception("disabled in server config")
 
-        ret, _ = ffprobe(abspath, int(vn.flags["convt"] / 2))
-        if "ac" not in ret:
+        self.wait4ram(0.2, tpath)
+        tags, rawtags = ffprobe(abspath, int(vn.flags["convt"] / 2))
+        if "ac" not in tags:
+            raise Exception("not audio")
+
+        if quality.endswith("k"):
+            qk = b"-b:a"
+            qv = quality.encode("ascii")
+        else:
+            qk = b"-q:a"
+            qv = quality[1:].encode("ascii")
+
+        # extremely conservative choices for output format
+        # (always 2ch 44k1) because if a device is old enough
+        # to not support opus then it's probably also super picky
+
+        # fmt: off
+        cmd = [
+            b"ffmpeg",
+            b"-nostdin",
+            b"-v", b"error",
+            b"-hide_banner",
+            b"-i", fsenc(abspath),
+        ] + self.big_tags(rawtags) + [
+            b"-map", b"0:a:0",
+            b"-ar", b"44100",
+            b"-ac", b"2",
+            b"-c:a", b"libmp3lame",
+            qk, qv,
+            fsenc(tpath)
+        ]
+        # fmt: on
+        self._run_ff(cmd, vn, oom=300)
+
+    def conv_opus(self, abspath: str, tpath: str, fmt: str, vn: VFS) -> None:
+        if self.args.no_acode or not self.args.q_opus:
+            raise Exception("disabled in server config")
+
+        self.wait4ram(0.2, tpath)
+        tags, rawtags = ffprobe(abspath, int(vn.flags["convt"] / 2))
+        if "ac" not in tags:
             raise Exception("not audio")
 
         try:
-            dur = ret[".dur"][1]
+            dur = tags[".dur"][1]
         except:
             dur = 0
 
-        src_opus = abspath.lower().endswith(".opus") or ret["ac"][1] == "opus"
+        src_opus = abspath.lower().endswith(".opus") or tags["ac"][1] == "opus"
         want_caf = tpath.endswith(".caf")
         tmp_opus = tpath
         if want_caf:
             tmp_opus = tpath + ".opus"
             try:
-                bos.unlink(tmp_opus)
+                wunlink(self.log, tmp_opus, vn.flags)
             except:
                 pass
 
         caf_src = abspath if src_opus else tmp_opus
+        bq = ("%dk" % (self.args.q_opus,)).encode("ascii")
 
         if not want_caf or not src_opus:
             # fmt: off
@@ -601,14 +771,14 @@ class ThumbSrv(object):
                 b"-v", b"error",
                 b"-hide_banner",
                 b"-i", fsenc(abspath),
-                b"-map_metadata", b"-1",
+            ] + self.big_tags(rawtags) + [
                 b"-map", b"0:a:0",
                 b"-c:a", b"libopus",
-                b"-b:a", b"128k",
+                b"-b:a", bq,
                 fsenc(tmp_opus)
             ]
             # fmt: on
-            self._run_ff(cmd, vn)
+            self._run_ff(cmd, vn, oom=300)
 
         # iOS fails to play some "insufficiently complex" files
         # (average file shorter than 8 seconds), so of course we
@@ -627,12 +797,12 @@ class ThumbSrv(object):
                 b"-map_metadata", b"-1",
                 b"-ac", b"2",
                 b"-c:a", b"libopus",
-                b"-b:a", b"128k",
+                b"-b:a", bq,
                 b"-f", b"caf",
                 fsenc(tpath)
             ]
             # fmt: on
-            self._run_ff(cmd, vn)
+            self._run_ff(cmd, vn, oom=300)
 
         elif want_caf:
             # simple remux should be safe
@@ -650,14 +820,24 @@ class ThumbSrv(object):
                 fsenc(tpath)
             ]
             # fmt: on
-            self._run_ff(cmd, vn)
+            self._run_ff(cmd, vn, oom=300)
 
         if tmp_opus != tpath:
             try:
-                bos.unlink(tmp_opus)
+                wunlink(self.log, tmp_opus, vn.flags)
             except:
                 pass
 
+    def big_tags(self, raw_tags: dict[str, list[str]]) -> list[bytes]:
+        ret = []
+        for k, vs in raw_tags.items():
+            for v in vs:
+                if len(str(v)) >= 1024:
+                    bv = k.encode("utf-8", "replace")
+                    ret += [b"-metadata", bv + b"="]
+                    break
+        return ret
+
     def poke(self, tdir: str) -> None:
         if not self.poke_cd.poke(tdir):
             return
@@ -681,7 +861,10 @@ class ThumbSrv(object):
                 else:
                     self.log("\033[Jcln {} ({})/\033[A".format(histpath, vol))
 
-                ndirs += self.clean(histpath)
+                try:
+                    ndirs += self.clean(histpath)
+                except Exception as ex:
+                    self.log("\033[Jcln err in %s: %r" % (histpath, ex), 3)
 
             self.log("\033[Jcln ok; rm {} dirs".format(ndirs))
 
@@ -698,7 +881,7 @@ class ThumbSrv(object):
 
     def _clean(self, cat: str, thumbpath: str) -> int:
         # self.log("cln {}".format(thumbpath))
-        exts = ["jpg", "webp", "png"] if cat == "th" else ["opus", "caf"]
+        exts = ["jpg", "webp", "png"] if cat == "th" else ["opus", "caf", "mp3"]
         maxage = getattr(self.args, cat + "_maxage")
         now = time.time()
         prev_b64 = None

copyparty/u2idx.py

@@ -8,7 +8,8 @@ import threading
 import time
 from operator import itemgetter
 
-from .__init__ import ANYWIN, TYPE_CHECKING, unicode
+from .__init__ import ANYWIN, PY2, TYPE_CHECKING, unicode
+from .authsrv import LEELOO_DALLAS, VFS
 from .bos import bos
 from .up2k import up2k_wark_from_hashlist
 from .util import (
@@ -20,6 +21,7 @@ from .util import (
     min_ex,
     quotep,
     s3dec,
+    vjoin,
 )
 
 if HAVE_SQLITE3:
@@ -36,6 +38,9 @@ if True:  # pylint: disable=using-constant-test
 if TYPE_CHECKING:
     from .httpsrv import HttpSrv
 
+if PY2:
+    range = xrange  # type: ignore
+
 
 class U2idx(object):
     def __init__(self, hsrv: "HttpSrv") -> None:
@@ -54,14 +59,27 @@ class U2idx(object):
         self.mem_cur = sqlite3.connect(":memory:", check_same_thread=False).cursor()
         self.mem_cur.execute(r"create table a (b text)")
 
+        self.sh_cur: Optional["sqlite3.Cursor"] = None
+
         self.p_end = 0.0
         self.p_dur = 0.0
 
     def log(self, msg: str, c: Union[int, str] = 0) -> None:
         self.log_func("u2idx", msg, c)
 
+    def shutdown(self) -> None:
+        for cur in self.cur.values():
+            db = cur.connection
+            try:
+                db.interrupt()
+            except:
+                pass
+
+            cur.close()
+            db.close()
+
     def fsearch(
-        self, vols: list[tuple[str, str, dict[str, Any]]], body: dict[str, Any]
+        self, uname: str, vols: list[VFS], body: dict[str, Any]
     ) -> list[dict[str, Any]]:
         """search by up2k hashlist"""
         if not HAVE_SQLITE3:
@@ -75,18 +93,36 @@ class U2idx(object):
         uv: list[Union[str, int]] = [wark[:16], wark]
 
         try:
-            return self.run_query(vols, uq, uv, True, False, 99999)[0]
+            return self.run_query(uname, vols, uq, uv, False, 99999)[0]
         except:
             raise Pebkac(500, min_ex())
 
-    def get_cur(self, ptop: str) -> Optional["sqlite3.Cursor"]:
-        if not HAVE_SQLITE3:
+    def get_shr(self) -> Optional["sqlite3.Cursor"]:
+        if self.sh_cur:
+            return self.sh_cur
+
+        if not HAVE_SQLITE3 or not self.args.shr:
             return None
 
-        cur = self.cur.get(ptop)
+        assert sqlite3  # type: ignore
+
+        db = sqlite3.connect(self.args.shr_db, timeout=2, check_same_thread=False)
+        cur = db.cursor()
+        cur.execute('pragma table_info("sh")').fetchall()
+        self.sh_cur = cur
+        return cur
+
+    def get_cur(self, vn: VFS) -> Optional["sqlite3.Cursor"]:
+        cur = self.cur.get(vn.realpath)
         if cur:
             return cur
 
+        if not HAVE_SQLITE3 or "e2d" not in vn.flags:
+            return None
+
+        assert sqlite3  # type: ignore
+
+        ptop = vn.realpath
         histpath = self.asrv.vfs.histtab.get(ptop)
         if not histpath:
             self.log("no histpath for [{}]".format(ptop))
@@ -101,7 +137,7 @@ class U2idx(object):
             uri = ""
             try:
                 uri = "{}?mode=ro&nolock=1".format(Path(db_path).as_uri())
-                db = sqlite3.connect(uri, 2, uri=True, check_same_thread=False)
+                db = sqlite3.connect(uri, timeout=2, uri=True, check_same_thread=False)
                 cur = db.cursor()
                 cur.execute('pragma table_info("up")').fetchone()
                 self.log("ro: {}".format(db_path))
@@ -113,14 +149,14 @@ class U2idx(object):
         if not cur:
             # on windows, this steals the write-lock from up2k.deferred_init --
             # seen on win 10.0.17763.2686, py 3.10.4, sqlite 3.37.2
-            cur = sqlite3.connect(db_path, 2, check_same_thread=False).cursor()
+            cur = sqlite3.connect(db_path, timeout=2, check_same_thread=False).cursor()
             self.log("opened {}".format(db_path))
 
         self.cur[ptop] = cur
         return cur
 
     def search(
-        self, vols: list[tuple[str, str, dict[str, Any]]], uq: str, lim: int
+        self, uname: str, vols: list[VFS], uq: str, lim: int
     ) -> tuple[list[dict[str, Any]], list[str], bool]:
         """search by query params"""
         if not HAVE_SQLITE3:
@@ -129,7 +165,6 @@ class U2idx(object):
         q = ""
         v: Union[str, int] = ""
         va: list[Union[str, int]] = []
-        have_up = False  # query has up.* operands
         have_mt = False
         is_key = True
         is_size = False
@@ -174,21 +209,21 @@ class U2idx(object):
                 if v == "size":
                     v = "up.sz"
                     is_size = True
-                    have_up = True
 
                 elif v == "date":
                     v = "up.mt"
                     is_date = True
-                    have_up = True
+
+                elif v == "up_at":
+                    v = "up.at"
+                    is_date = True
 
                 elif v == "path":
                     v = "trim(?||up.rd,'/')"
                     va.append("\nrd")
-                    have_up = True
 
                 elif v == "name":
                     v = "up.fn"
-                    have_up = True
 
                 elif v == "tags" or ptn_mt.match(v):
                     have_mt = True
@@ -264,19 +299,24 @@ class U2idx(object):
                 q += " lower({}) {} ? ) ".format(field, oper)
 
         try:
-            return self.run_query(vols, q, va, have_up, have_mt, lim)
+            return self.run_query(uname, vols, q, va, have_mt, lim)
         except Exception as ex:
             raise Pebkac(500, repr(ex))
 
     def run_query(
         self,
-        vols: list[tuple[str, str, dict[str, Any]]],
+        uname: str,
+        vols: list[VFS],
         uq: str,
         uv: list[Union[str, int]],
-        have_up: bool,
         have_mt: bool,
         lim: int,
     ) -> tuple[list[dict[str, Any]], list[str], bool]:
+        if self.args.srch_dbg:
+            t = "searching across all %s volumes in which the user has 'r' (full read access):\n  %s"
+            zs = "\n  ".join(["/%s = %s" % (x.vpath, x.realpath) for x in vols])
+            self.log(t % (len(vols), zs), 5)
+
         done_flag: list[bool] = []
         self.active_id = "{:.6f}_{}".format(
             time.time(), threading.current_thread().ident
@@ -295,13 +335,35 @@ class U2idx(object):
 
         ret = []
         seen_rps: set[str] = set()
-        lim = min(lim, int(self.args.srch_hits))
+        clamp = int(self.args.srch_hits)
+        if lim >= clamp:
+            lim = clamp
+            clamped = True
+        else:
+            clamped = False
+
         taglist = {}
-        for (vtop, ptop, flags) in vols:
-            cur = self.get_cur(ptop)
+        for vol in vols:
+            if lim < 0:
+                break
+
+            vtop = vol.vpath
+            ptop = vol.realpath
+            flags = vol.flags
+
+            cur = self.get_cur(vol)
             if not cur:
                 continue
 
+            excl = []
+            for vp2 in self.asrv.vfs.all_vols.keys():
+                if vp2.startswith((vtop + "/").lstrip("/")) and vtop != vp2:
+                    excl.append(vp2[len(vtop) :].lstrip("/"))
+
+            if self.args.srch_dbg:
+                t = "searching in volume /%s (%s), excludelist %s"
+                self.log(t % (vtop, ptop, excl), 5)
+
             self.active_cur = cur
 
             vuv = []
@@ -313,7 +375,8 @@ class U2idx(object):
 
             sret = []
             fk = flags.get("fk")
-            dots = flags.get("dotsrch")
+            dots = flags.get("dotsrch") and uname in vol.axs.udot
+            fk_alg = 2 if "fka" in flags else 1
             c = cur.execute(uq, tuple(vuv))
             for hit in c:
                 w, ts, sz, rd, fn, ip, at = hit[:7]
@@ -321,6 +384,13 @@ class U2idx(object):
                 if rd.startswith("//") or fn.startswith("//"):
                     rd, fn = s3dec(rd, fn)
 
+                if rd in excl or any([x for x in excl if rd.startswith(x + "/")]):
+                    if self.args.srch_dbg:
+                        zs = vjoin(vjoin(vtop, rd), fn)
+                        t = "database inconsistency in volume '/%s'; ignoring: %s"
+                        self.log(t % (vtop, zs), 1)
+                    continue
+
                 rp = quotep("/".join([x for x in [vtop, rd, fn] if x]))
                 if not dots and "/." in ("/" + rp):
                     continue
@@ -333,21 +403,35 @@ class U2idx(object):
                 else:
                     try:
                         ap = absreal(os.path.join(ptop, rd, fn))
-                        inf = bos.stat(ap)
+                        ino = 0 if ANYWIN or fk_alg == 2 else bos.stat(ap).st_ino
                     except:
                         continue
 
-                    suf = (
-                        "?k="
-                        + gen_filekey(
-                            self.args.fk_salt, ap, sz, 0 if ANYWIN else inf.st_ino
-                        )[:fk]
-                    )
+                    suf = "?k=" + gen_filekey(
+                        fk_alg,
+                        self.args.fk_salt,
+                        ap,
+                        sz,
+                        ino,
+                    )[:fk]
 
                 lim -= 1
                 if lim < 0:
                     break
 
+                if self.args.srch_dbg:
+                    t = "in volume '/%s': hit: %s"
+                    self.log(t % (vtop, rp), 5)
+
+                    zs = vjoin(vtop, rp)
+                    chk_vn, _ = self.asrv.vfs.get(zs, LEELOO_DALLAS, True, False)
+                    chk_vn = chk_vn.dbv or chk_vn
+                    if chk_vn.vpath != vtop:
+                        raise Exception(
+                            "database inconsistency! in volume '/%s' (%s), found file [%s] which belongs to volume '/%s' (%s)"
+                            % (vtop, ptop, zs, chk_vn.vpath, chk_vn.realpath)
+                        )
+
                 seen_rps.add(rp)
                 sret.append({"ts": int(ts), "sz": sz, "rp": rp + suf, "w": w[:16]})
 
@@ -365,12 +449,16 @@ class U2idx(object):
             ret.extend(sret)
             # print("[{}] {}".format(ptop, sret))
 
+            if self.args.srch_dbg:
+                t = "in volume '/%s': got %d hits, %d total so far"
+                self.log(t % (vtop, len(sret), len(ret)), 5)
+
         done_flag.append(True)
         self.active_id = ""
 
         ret.sort(key=itemgetter("rp"))
 
-        return ret, list(taglist.keys()), lim < 0
+        return ret, list(taglist.keys()), lim < 0 and not clamped
 
     def terminator(self, identifier: str, done_flag: list[bool]) -> None:
         for _ in range(self.timeout):

copyparty/web/baguettebox.js

@@ -17,8 +17,10 @@ window.baguetteBox = (function () {
             titleTag: false,
             async: false,
             preload: 2,
+            refocus: true,
             afterShow: null,
             afterHide: null,
+            duringHide: null,
             onChange: null,
         },
         overlay, slider, btnPrev, btnNext, btnHelp, btnAnim, btnRotL, btnRotR, btnSel, btnFull, btnVmode, btnClose,
@@ -27,6 +29,8 @@ window.baguetteBox = (function () {
         isOverlayVisible = false,
         touch = {},  // start-pos
         touchFlag = false,  // busy
+        scrollCSS = ['', ''],
+        scrollTimer = 0,
         re_i = /^[^?]+\.(a?png|avif|bmp|gif|heif|jpe?g|jfif|svg|webp)(\?|$)/i,
         re_v = /^[^?]+\.(webm|mkv|mp4)(\?|$)/i,
         anims = ['slideIn', 'fadeIn', 'none'],
@@ -89,6 +93,30 @@ window.baguetteBox = (function () {
         touchendHandler();
     };
 
+    var overlayWheelHandler = function (e) {
+        if (!options.noScrollbars || anymod(e))
+            return;
+
+        ev(e);
+
+        var x = e.deltaX,
+            y = e.deltaY,
+            d = Math.abs(x) > Math.abs(y) ? x : y;
+
+        if (e.deltaMode)
+            d *= 10;
+
+        if (Date.now() - scrollTimer < (Math.abs(d) > 20 ? 100 : 300))
+            return;
+
+        scrollTimer = Date.now();
+
+        if (d > 0)
+            showNextImage();
+        else
+            showPreviousImage();
+    };
+
     var trapFocusInsideOverlay = function (e) {
         if (overlay.style.display === 'block' && (overlay.contains && !overlay.contains(e.target))) {
             e.stopPropagation();
@@ -144,7 +172,7 @@ window.baguetteBox = (function () {
             selectorData.galleries.push(gallery);
         });
 
-        return selectorData.galleries;
+        return [selectorData.galleries, options];
     }
 
     function clearCachedData() {
@@ -255,19 +283,19 @@ window.baguetteBox = (function () {
         if (anymod(e, true))
             return;
 
-        var k = e.code + '', v = vid(), pos = -1;
+        var k = (e.code || e.key) + '', v = vid(), pos = -1;
 
         if (k == "BracketLeft")
             setloop(1);
         else if (k == "BracketRight")
             setloop(2);
-        else if (e.shiftKey)
+        else if (e.shiftKey && k != "KeyR" && k != "R")
             return;
-        else if (k == "ArrowLeft" || k == "KeyJ")
+        else if (k == "ArrowLeft" || k == "KeyJ" || k == "Left" || k == "j")
             showPreviousImage();
-        else if (k == "ArrowRight" || k == "KeyL")
+        else if (k == "ArrowRight" || k == "KeyL" || k == "Right" || k == "l")
             showNextImage();
-        else if (k == "Escape")
+        else if (k == "Escape" || k == "Esc")
             hideOverlay();
         else if (k == "Home")
             showFirstImage(e);
@@ -295,9 +323,9 @@ window.baguetteBox = (function () {
         }
         else if (k == "KeyF")
             tglfull();
-        else if (k == "KeyS")
+        else if (k == "KeyS" || k == "s")
             tglsel();
-        else if (k == "KeyR")
+        else if (k == "KeyR" || k == "r" || k == "R")
             rotn(e.shiftKey ? -1 : 1);
         else if (k == "KeyY")
             dlpic();
@@ -392,8 +420,7 @@ window.baguetteBox = (function () {
     }
 
     function dlpic() {
-        var url = findfile()[3].href;
-        url += (url.indexOf('?') < 0 ? '?' : '&') + 'cache';
+        var url = addq(findfile()[3].href, 'cache');
         dl_file(url);
     }
 
@@ -450,6 +477,7 @@ window.baguetteBox = (function () {
         bind(document, 'keyup', keyUpHandler);
         bind(document, 'fullscreenchange', onFSC);
         bind(overlay, 'click', overlayClickHandler);
+        bind(overlay, 'wheel', overlayWheelHandler);
         bind(btnPrev, 'click', showPreviousImage);
         bind(btnNext, 'click', showNextImage);
         bind(btnClose, 'click', hideOverlay);
@@ -472,6 +500,7 @@ window.baguetteBox = (function () {
         unbind(document, 'keyup', keyUpHandler);
         unbind(document, 'fullscreenchange', onFSC);
         unbind(overlay, 'click', overlayClickHandler);
+        unbind(overlay, 'wheel', overlayWheelHandler);
         unbind(btnPrev, 'click', showPreviousImage);
         unbind(btnNext, 'click', showNextImage);
         unbind(btnClose, 'click', hideOverlay);
@@ -524,7 +553,7 @@ window.baguetteBox = (function () {
                 options[item] = newOptions[item];
         }
 
-        var an = options.animation = sread('ganim') || anims[ANIM ? 0 : 2];
+        var an = options.animation = sread('ganim', anims) || anims[ANIM ? 0 : 2];
         btnAnim.textContent = ['⇄', '⮺', '⚡'][anims.indexOf(an)];
         btnAnim.setAttribute('tt', 'animation: ' + an);
 
@@ -539,6 +568,12 @@ window.baguetteBox = (function () {
 
     function showOverlay(chosenImageIndex) {
         if (options.noScrollbars) {
+            var a = document.documentElement.style.overflowY,
+                b = document.body.style.overflowY;
+
+            if (a != 'hidden' || b != 'scroll')
+                scrollCSS = [a, b];
+
             document.documentElement.style.overflowY = 'hidden';
             document.body.style.overflowY = 'scroll';
         }
@@ -582,24 +617,30 @@ window.baguetteBox = (function () {
         isOverlayVisible = true;
     }
 
-    function hideOverlay(e) {
+    function hideOverlay(e, dtor) {
         ev(e);
         playvid(false);
         removeFromCache('#files');
         if (options.noScrollbars) {
-            document.documentElement.style.overflowY = 'auto';
-            document.body.style.overflowY = 'auto';
+            document.documentElement.style.overflowY = scrollCSS[0];
+            document.body.style.overflowY = scrollCSS[1];
         }
-        if (overlay.style.display === 'none')
+
+        try {
+            if (document.fullscreenElement)
+                document.exitFullscreen();
+        }
+        catch (ex) { }
+        isFullscreen = false;
+
+        if (dtor || overlay.style.display === 'none')
             return;
 
+        if (options.duringHide)
+            options.duringHide();
+
         sethash('');
         unbindEvents();
-        try {
-            document.exitFullscreen();
-            isFullscreen = false;
-        }
-        catch (ex) { }
 
         // Fade out and hide the overlay
         overlay.className = '';
@@ -613,9 +654,45 @@ window.baguetteBox = (function () {
             if (options.afterHide)
                 options.afterHide();
 
-            documentLastFocus && documentLastFocus.focus();
+            options.refocus && documentLastFocus && documentLastFocus.focus();
             isOverlayVisible = false;
-        }, 500);
+            unvid();
+            unfig();
+        }, 250);
+    }
+
+    function unvid(keep) {
+        var vids = QSA('#bbox-overlay video');
+        for (var a = vids.length - 1; a >= 0; a--) {
+            var v = vids[a];
+            if (v == keep)
+                continue;
+
+            v.src = '';
+            v.load();
+
+            var p = v.parentNode;
+            p.removeChild(v);
+            p.parentNode.removeChild(p);
+        }
+    }
+
+    function unfig(keep) {
+        var figs = QSA('#bbox-overlay figure'),
+            npre = options.preload || 0,
+            k = [];
+
+        if (keep === undefined)
+            keep = -9;
+
+        for (var a = keep - npre; a <= keep + npre; a++)
+            k.push('bbox-figure-' + a);
+
+        for (var a = figs.length - 1; a >= 0; a--) {
+            var f = figs[a];
+            if (!has(k, f.getAttribute('id')))
+                f.parentNode.removeChild(f);
+        }
     }
 
     function loadImage(index, callback) {
@@ -641,7 +718,7 @@ window.baguetteBox = (function () {
                 options.captions.call(currentGallery, imageElement) :
                 imageElement.getAttribute('data-caption') || imageElement.title;
 
-        imageSrc += imageSrc.indexOf('?') < 0 ? '?cache' : '&cache';
+        imageSrc = addq(imageSrc, 'cache');
 
         if (is_vid && index != currentIndex)
             return;  // no preload
@@ -670,8 +747,11 @@ window.baguetteBox = (function () {
         });
         image.setAttribute('src', imageSrc);
         if (is_vid) {
+            image.volume = clamp(fcfg_get('vol', dvol / 100), 0, 1);
             image.setAttribute('controls', 'controls');
             image.onended = vidEnd;
+            image.onplay = function () { show_buttons(1); };
+            image.onpause = function () { show_buttons(); };
         }
         image.alt = thumbnailElement ? thumbnailElement.alt || '' : '';
         if (options.titleTag && imageCaption)
@@ -679,6 +759,9 @@ window.baguetteBox = (function () {
 
         figure.appendChild(image);
 
+        if (is_vid && window.afilt)
+            afilt.apply(undefined, image);
+
         if (options.async && callback)
             callback();
     }
@@ -708,6 +791,7 @@ window.baguetteBox = (function () {
     }
 
     function show(index, gallery) {
+        gallery = gallery || currentGallery;
         if (!isOverlayVisible && index >= 0 && index < gallery.length) {
             prepareOverlay(gallery, options);
             showOverlay(index);
@@ -720,12 +804,10 @@ window.baguetteBox = (function () {
         if (index >= imagesElements.length)
             return bounceAnimation('right');
 
-        var v = vid();
-        if (v) {
-            v.src = '';
-            v.load();
-            v.parentNode.removeChild(v);
+        try {
+            vid().pause();
         }
+        catch (ex) { }
 
         currentIndex = index;
         loadImage(currentIndex, function () {
@@ -734,6 +816,15 @@ window.baguetteBox = (function () {
         });
         updateOffset();
 
+        if (options.animation == 'none')
+            unvid(vid());
+        else
+            setTimeout(function () {
+                unvid(vid());
+            }, 100);
+
+        unfig(index);
+
         if (options.onChange)
             options.onChange(currentIndex, imagesElements.length);
 
@@ -875,7 +966,7 @@ window.baguetteBox = (function () {
 
         if (loopB !== null) {
             timer.add(loopchk);
-            sethash(window.location.hash.slice(1).split('&')[0] + '&t=' + (loopA || 0) + '-' + loopB);
+            sethash(location.hash.slice(1).split('&')[0] + '&t=' + (loopA || 0) + '-' + loopB);
         }
     }
 
@@ -906,6 +997,12 @@ window.baguetteBox = (function () {
         }
     }
 
+    function show_buttons(v) {
+        clmod(ebi('bbox-btns'), 'off', v);
+        clmod(btnPrev, 'off', v);
+        clmod(btnNext, 'off', v);
+    }
+
     function bounceAnimation(direction) {
         slider.className = options.animation == 'slideIn' ? 'bounce-from-' + direction : 'eog';
         setTimeout(function () {
@@ -969,9 +1066,7 @@ window.baguetteBox = (function () {
             if (fx > 0.7)
                 return showNextImage();
 
-            clmod(ebi('bbox-btns'), 'off', 't');
-            clmod(btnPrev, 'off', 't');
-            clmod(btnNext, 'off', 't');
+            show_buttons('t');
 
             if (Date.now() - ctime <= 500 && !IPHONE)
                 tglfull();
@@ -1013,6 +1108,7 @@ window.baguetteBox = (function () {
     }
 
     function destroyPlugin() {
+        hideOverlay(undefined, true);
         unbindEvents();
         clearCachedData();
         document.getElementsByTagName('body')[0].removeChild(ebi('bbox-overlay'));

copyparty/web/browser.css

@@ -10,7 +10,6 @@
 	--fg2-max: #fff;
 	--fg-weak: #bbb;
 
-	--bg-u7: #555;
 	--bg-u6: #4c4c4c;
 	--bg-u5: #444;
 	--bg-u4: #383838;
@@ -28,6 +27,8 @@
 	--row-alt: #282828;
 
 	--scroll: #eb0;
+	--sel-fg: var(--bg-d1);
+	--sel-bg: var(--fg);
 
 	--a: #fc5;
 	--a-b: #c90;
@@ -41,8 +42,14 @@
 	--btn-h-bg: #805;
 	--btn-1-fg: #400;
 	--btn-1-bg: var(--a);
+	--btn-h-bs: var(--btn-bs);
+	--btn-h-bb: var(--btn-bb);
+	--btn-1-bs: var(--btn-bs);
+	--btn-1-bb: var(--btn-bb);
 	--btn-1h-fg: var(--btn-1-fg);
 	--btn-1h-bg: #fe8;
+	--btn-1h-bs: var(--btn-1-bs);
+	--btn-1h-bb: var(--btn-1-bb);
 	--chk-fg: var(--tab-alt);
 	--txt-sh: var(--bg-d2);
 	--txt-bg: var(--btn-bg);
@@ -57,7 +64,7 @@
 	--u2-tab-bg: linear-gradient(to bottom, var(--bg), var(--bg-u1));
 	--u2-tab-b1: rgba(128,128,128,0.8);
 	--u2-tab-1-fg: #fd7;
-	--u2-tab-1-bg: linear-gradient(to bottom, var(#353), var(--bg) 80%);
+	--u2-tab-1-bg: linear-gradient(to bottom, #353, var(--bg) 80%);
 	--u2-tab-1-b1: #7c5;
 	--u2-tab-1-b2: #583;
 	--u2-tab-1-sh: #280;
@@ -210,22 +217,19 @@ html.y {
 html.a {
 	--op-aa-sh: 0 0 .2em var(--bg-d3) inset;
 
-	--u2-o-bg: #603;
-	--u2-o-b1: #a16;
-	--u2-o-sh: #a00;
-	--u2-o-h-bg: var(--u2-o-bg);
-	--u2-o-h-b1: #fb0;
-	--u2-o-h-sh: #fb0;
-	--u2-o-1-bg: #6a1;
-	--u2-o-1-b1: #efa;
-	--u2-o-1-sh: #0c0;
-	--u2-o-1h-bg: var(--u2-o-1-bg);
+	--btn-bs: 0 0 .2em var(--bg-d3);
+}
+html.az {
+	--btn-1-bs: 0 0 .1em var(--fg) inset;
 }
 html.ay {
 	--op-aa-sh: 0 .1em .2em #ccc;
 	--op-aa-bg: var(--bg-max);
 }
 html.b {
+	--btn-bs: 0 .05em 0 var(--bg-d3) inset;
+	--btn-1-bs: 0 .05em 0 var(--btn-1h-bg) inset;
+
 	--tree-bg: var(--bg);
 
 	--g-bg: var(--bg);
@@ -242,17 +246,13 @@ html.b {
 	--u2-b1-bg: rgba(128,128,128,0.15);
 	--u2-b2-bg: var(--u2-b1-bg);
 
-	--u2-o-bg: var(--btn-bg);
-	--u2-o-h-bg: var(--btn-h-bg);
-	--u2-o-1-bg: var(--a);
-	--u2-o-1h-bg: var(--a-hil);
-
 	--f-sh1: 0.1;
 	--mp-b-bg: transparent;
 }
 html.bz {
 	--fg: #cce;
 	--fg-weak: #bbd;
+
 	--bg-u5: #3b3f58;
 	--bg-u4: #1e2130;
 	--bg-u3: #1e2130;
@@ -264,11 +264,14 @@ html.bz {
 
 	--row-alt: #181a27;
 
+	--a-b: #fb4;
+
 	--btn-bg: #202231;
 	--btn-h-bg: #2d2f45;
-	--btn-1-bg: #ba2959;
-	--btn-1-fg: #fff;
+	--btn-1-bg: #eb6;
+	--btn-1-fg: #000;
 	--btn-1h-fg: #000;
+	--btn-1h-bg: #ff9;
 	--txt-sh: a;
 
 	--u2-tab-b1: var(--bg-u5);
@@ -303,6 +306,7 @@ html.by {
 }
 html.c {
 	font-weight: bold;
+
 	--fg: #fff;
 	--fg-weak: #cef;
 	--bg-u5: #409;
@@ -323,15 +327,25 @@ html.c {
 	--chk-fg: #d90;
 
 	--op-aa-bg: #f9dd22;
-	--u2-o-1-bg: #4cf;
 
 	--srv-1: #ea0;
 	--mp-b-bg: transparent;
 }
 html.cz {
 	--bgg: var(--bg-u2);
+
+	--sel-bg: var(--bg-u5);
+	--sel-fg: var(--fg);
+
+	--btn-bb: .2em solid #709;
+	--btn-bs: 0 .1em .6em rgba(255,0,185,0.5);
+	--btn-1-bb: .2em solid #e90;
+	--btn-1-bs: 0 .1em .8em rgba(255,205,0,0.9);
+
 	--srv-3: #fff;
+
 	--u2-tab-b1: var(--bg-d3);
+	--u2-tab-1-bg: a;
 }
 html.cy {
 	--fg: #fff;
@@ -343,6 +357,8 @@ html.cy {
 	--bg-d3: #f77;
 	--bg-d2: #ff0;
 
+	--sel-bg: #f77;
+
 	--a: #fff;
 	--a-hil: #fff;
 	--a-h-bg: #000;
@@ -356,16 +372,20 @@ html.cy {
 	--btn-h-fg: #fff;
 	--btn-1-bg: #ff0;
 	--btn-1-fg: #000;
+	--btn-bs: 0 .25em 0 #f00;
 	--chk-fg: #fd0;
 
+	--txt-bg: #000;
 	--srv-1: #f00;
 	--srv-3: #fff;
 	--op-aa-bg: #fff;
 
 	--u2-b1-bg: #f00;
 	--u2-b2-bg: #f00;
-	--u2-o-bg: #ff0;
-	--u2-o-1-bg: #f00;
+
+	--g-sel-fg: #fff;
+	--g-sel-bg: #aaa;
+	--g-fsel-bg: #aaa;
 }
 html.dz {
 	--fg: #4d4;
@@ -373,7 +393,6 @@ html.dz {
 	--fg2-max: #fff;
 	--fg-weak: #2a2;
 
-	--bg-u7: #020;
 	--bg-u6: #020;
 	--bg-u5: #050;
 	--bg-u4: #020;
@@ -406,6 +425,9 @@ html.dz {
 	--btn-1-bg: #4f4;
 	--btn-1h-fg: var(--btn-1-fg);
 	--btn-1h-bg: #3f3;
+	--btn-bs: 0 0 0 .1em #080 inset;
+	--btn-1-bs: a;
+
 	--chk-fg: var(--tab-alt);
 	--txt-sh: var(--bg-d2);
 	--txt-bg: var(--btn-bg);
@@ -420,19 +442,13 @@ html.dz {
 	--u2-tab-bg: linear-gradient(to bottom, var(--bg), var(--bg-u1));
 	--u2-tab-b1: var(--fg-weak);
 	--u2-tab-1-fg: #fff;
-	--u2-tab-1-bg: linear-gradient(to bottom, var(#353), var(--bg) 80%);
+	--u2-tab-1-bg: linear-gradient(to bottom, #151, var(--bg) 80%);
 	--u2-tab-1-b1: #7c5;
 	--u2-tab-1-b2: #583;
 	--u2-tab-1-sh: #280;
 	--u2-b-fg: #fff;
 	--u2-b1-bg: #3a3;
 	--u2-b2-bg: #3a3;
-	--u2-o-bg: var(--btn-bg);
-	--u2-o-b1: var(--bg-u5);
-	--u2-o-h-bg: var(--fg-weak);
-	--u2-o-1-bg: var(--fg-weak);
-	--u2-o-1-b1: var(--a);
-	--u2-o-1h-bg: var(--a);
 	--u2-inf-bg: #07a;
 	--u2-inf-b1: #0be;
 	--u2-ok-bg: #380;
@@ -493,6 +509,8 @@ html.dz {
 	--err-ts: #500;
 
 	text-shadow: none;
+	font-family: 'scp', monospace, monospace;
+	font-family: var(--font-mono), 'scp', monospace, monospace;
 }
 html.dy {
 	--fg: #000;
@@ -542,10 +560,6 @@ html.dy {
 	--u2-tab-1-bg: a;
 	--u2-b1-bg: #000;
 	--u2-b2-bg: #000;
-	--u2-o-h-bg: #999;
-	--u2-o-1h-bg: #999;
-	--u2-o-bg: #eee;
-	--u2-o-1-bg: #000;
 
 	--ud-b1: a;
 
@@ -586,8 +600,8 @@ html.dy {
 	line-height: 1.2em;
 }
 ::selection {
-	color: var(--bg-d1);
-	background: var(--fg);
+	color: var(--sel-fg);
+	background: var(--sel-bg);
 	text-shadow: none;
 }
 html,body,tr,th,td,#files,a {
@@ -602,6 +616,7 @@ html {
 	color: var(--fg);
 	background: var(--bgg);
 	font-family: sans-serif;
+	font-family: var(--font-main), sans-serif;
 	text-shadow: 1px 1px 0px var(--bg-max);
 }
 html, body {
@@ -610,12 +625,14 @@ html, body {
 }
 pre, code, tt, #doc, #doc>code {
 	font-family: 'scp', monospace, monospace;
+	font-family: var(--font-mono), 'scp', monospace, monospace;
 }
 .ayjump {
 	position: fixed;
 	overflow: hidden;
 	width: 0;
 	height: 0;
+	color: var(--bg);
 }
 html .ayjump:focus {
 	z-index: 80386;
@@ -695,12 +712,12 @@ a:hover {
 .s0:after,
 .s1:after {
 	content: '⌄';
-	margin-left: -.1em;
+	margin-left: -.15em;
 }
 .s0r:after,
 .s1r:after {
 	content: '⌃';
-	margin-left: -.1em;
+	margin-left: -.15em;
 }
 .s0:after,
 .s0r:after {
@@ -711,7 +728,7 @@ a:hover {
 	color: var(--sort-2);
 }
 #files thead th:after {
-	margin-right: -.7em;
+	margin-right: -.5em;
 }
 #files tbody tr:hover td,
 #files tbody tr:hover td+td {
@@ -727,6 +744,11 @@ a:hover {
 html.y #files thead th {
 	box-shadow: 0 1px 0 rgba(0,0,0,0.12);
 }
+html #files.hhpick thead th {
+	color: #f7d;
+	background: #000;
+	box-shadow: .1em .2em 0 #f6c inset, -.1em -.1em 0 #f6c inset;
+}
 #files td {
 	margin: 0;
 	padding: .3em .5em;
@@ -735,6 +757,15 @@ html.y #files thead th {
 	word-wrap: break-word;
 	overflow: hidden;
 }
+#files tr.fade a {
+	color: #999;
+	color: rgba(255, 255, 255, 0.4);
+	font-style: italic;
+}
+html.y #files tr.fade a {
+	color: #999;
+	color: rgba(0, 0, 0, 0.4);
+}
 #files tr:nth-child(2n) td {
 	background: var(--row-alt);
 }
@@ -753,6 +784,7 @@ html.y #files thead th {
 }
 #files tbody td:nth-child(3) {
 	font-family: 'scp', monospace, monospace;
+	font-family: var(--font-mono), 'scp', monospace, monospace;
 	text-align: right;
 	padding-right: 1em;
 	white-space: nowrap;
@@ -812,6 +844,11 @@ html.y #path a:hover {
 .logue:empty {
 	display: none;
 }
+.logue.raw {
+	white-space: pre;
+	font-family: 'scp', 'consolas', monospace;
+	font-family: var(--font-mono), 'scp', 'consolas', monospace;
+}
 #doc>iframe,
 .logue>iframe {
 	background: var(--bgg);
@@ -932,6 +969,8 @@ html.y #path a:hover {
 #files tbody tr.play a:hover {
 	color: var(--btn-1h-fg);
 	background: var(--btn-1h-bg);
+	box-shadow: var(--btn-1h-bs);
+	border-bottom: var(--btn-1h-bb);
 }
 #ggrid {
 	margin: -.2em -.5em;
@@ -940,6 +979,7 @@ html.y #path a:hover {
 	overflow: hidden;
 	display: block;
 	display: -webkit-box;
+	line-clamp: var(--grid-ln);
 	-webkit-line-clamp: var(--grid-ln);
 	-webkit-box-orient: vertical;
 	padding-top: .3em;
@@ -975,6 +1015,10 @@ html.y #path a:hover {
 	margin: 0 auto;
 	display: block;
 }
+#ggrid.nocrop>a img {
+	max-height: 20em;
+	max-height: calc(var(--grid-sz)*2);
+}
 #ggrid>a.dir:before {
 	content: '📂';
 }
@@ -982,9 +1026,6 @@ html.y #path a:hover {
 	color: var(--g-dfg);
 }
 #ggrid>a.au:before {
-	content: '💾';
-}
-html.np_open #ggrid>a.au:before {
 	content: '▶';
 }
 #ggrid>a:before {
@@ -1113,6 +1154,7 @@ html.y #widget.open {
 	width: 100%;
 	height: 100%;
 }
+#fshr,
 #wtgrid,
 #wtico {
 	position: relative;
@@ -1141,9 +1183,6 @@ html.y #widget.open {
 @keyframes spin {
 	100% {transform: rotate(360deg)}
 }
-@media (prefers-reduced-motion) {
-	@keyframes spin { }
-}
 @keyframes fadein {
 	0% {opacity: 0}
 	100% {opacity: 1}
@@ -1237,6 +1276,13 @@ html.y #widget.open {
 	0% {opacity:0}
 	100% {opacity:1}
 }
+#ggrid>a.glow {
+	animation: gexit .6s ease-out;
+}
+@keyframes gexit {
+	0% {box-shadow: 0 0 0 2em var(--a)}
+	100% {box-shadow: 0 0 0em 0em var(--a)}
+}
 #wzip a {
 	font-size: .4em;
 	margin: -.3em .1em;
@@ -1295,6 +1341,7 @@ html.y #widget.open {
 #widget.cmp #wtoggle {
 	font-size: 1.2em;
 }
+#widget.cmp #fshr,
 #widget.cmp #wtgrid {
 	display: none;
 }
@@ -1395,7 +1442,15 @@ input[type="checkbox"]+label {
 input[type="radio"]:checked+label,
 input[type="checkbox"]:checked+label {
 	color: #0e0;
-	color: var(--a);
+	color: var(--btn-1-bg);
+}
+input[type="checkbox"]:checked+label {
+	box-shadow: var(--btn-1-bs);
+	border-bottom: var(--btn-1-bb);
+}
+html.dz input {
+	font-family: 'scp', monospace, monospace;
+	font-family: var(--font-mono), 'scp', monospace, monospace;
 }
 .opwide div>span>input+label {
 	padding: .3em 0 .3em .3em;
@@ -1405,14 +1460,17 @@ input[type="checkbox"]:checked+label {
 .opview input.i {
 	width: calc(100% - 16.2em);
 }
+input.drc_v,
 input.eq_gain {
 	width: 3em;
 	text-align: center;
 	margin: 0 .6em;
 }
+#audio_drc table,
 #audio_eq table {
 	border-collapse: collapse;
 }
+#audio_drc td,
 #audio_eq td {
 	text-align: center;
 }
@@ -1421,11 +1479,15 @@ input.eq_gain {
 	display: block;
 	padding: 0;
 }
+#au_drc,
 #au_eq {
 	display: block;
 	margin-top: .5em;
 	padding: 1.3em .3em;
 }
+#au_drc {
+	padding: .4em .3em;
+}
 #ico1 {
 	cursor: pointer;
 }
@@ -1466,6 +1528,8 @@ input.eq_gain {
 	width: calc(100% - 2em);
 	margin: .3em 0 0 1.4em;
 }
+@media (max-width: 130em) { #srch_form.tags #tq_raw { width: calc(100% - 34em) } }
+@media (max-width: 95em) { #srch_form.tags #tq_raw { width: calc(100% - 2em) } }
 #tq_raw td+td {
 	width: 100%;
 }
@@ -1560,10 +1624,13 @@ html {
 	color: var(--btn-fg);
 	background: #eee;
 	background: var(--btn-bg);
+	box-shadow: var(--btn-bs);
+	border-bottom: var(--btn-bb);
 	border-radius: .3em;
 	padding: .2em .4em;
 	font-size: 1.2em;
 	margin: .2em;
+	display: inline-block;
 	white-space: pre;
 	position: relative;
 	top: -.12em;
@@ -1572,19 +1639,14 @@ html.c .btn,
 html.a .btn {
 	border-radius: .2em;
 }
-html.cz .btn {
-	box-shadow: 0 .1em .6em rgba(255,0,185,0.5);
-	border-bottom: .2em solid #709;
-}
 html.dz .btn {
-	box-shadow: 0 0 0 .1em #080 inset;
-}
-html.dz .tgl.btn.on {
-	box-shadow: 0 0 0 .1em var(--btn-1-bg) inset;
+	font-size: 1em;
 }
 .btn:hover {
 	color: var(--btn-h-fg);
 	background: var(--btn-h-bg);
+	box-shadow: var(--btn-h-bs);
+	border-bottom: var(--btn-h-bb);
 }
 .tgl.btn.on {
 	background: #000;
@@ -1592,14 +1654,14 @@ html.dz .tgl.btn.on {
 	color: #fff;
 	color: var(--btn-1-fg);
 	text-shadow: none;
-}
-html.cz .tgl.btn.on {
-	box-shadow: 0 .1em .8em rgba(255,205,0,0.9);
-	border-bottom: .2em solid #e90;
+	box-shadow: var(--btn-1-bs);
+	border-bottom: var(--btn-1-bb);
 }
 .tgl.btn.on:hover {
-	background: var(--btn-1h-bg);
 	color: var(--btn-1h-fg);
+	background: var(--btn-1h-bg);
+	box-shadow: var(--btn-1h-bs);
+	border-bottom: var(--btn-1h-bb);
 }
 #detree {
 	padding: .3em .5em;
@@ -1634,7 +1696,9 @@ html.cz .tgl.btn.on {
 	color: var(--fg-max);
 }
 #tree ul a.hl {
+	color: #fff;
 	color: var(--btn-1-fg);
+	background: #000;
 	background: var(--btn-1-bg);
 	text-shadow: none;
 }
@@ -1669,6 +1733,7 @@ html.y #tree.nowrap .ntree a+a:hover {
 }
 .ntree a:first-child {
 	font-family: 'scp', monospace, monospace;
+	font-family: var(--font-mono), 'scp', monospace, monospace;
 	font-size: 1.2em;
 	line-height: 0;
 }
@@ -1696,6 +1761,7 @@ html.y #tree.nowrap .ntree a+a:hover {
 }
 #files th span {
 	position: relative;
+	white-space: nowrap;
 }
 #files>thead>tr>th.min,
 #files td.min {
@@ -1733,9 +1799,6 @@ html.y #tree.nowrap .ntree a+a:hover {
 	margin: .7em 0 .7em .5em;
 	padding-left: .5em;
 }
-.opwide>div>div>a {
-	line-height: 2em;
-}
 .opwide>div>h3 {
 	color: var(--fg-weak);
 	margin: 0 .4em;
@@ -1750,6 +1813,7 @@ html.y #tree.nowrap .ntree a+a:hover {
 	padding: 0;
 }
 #thumbs,
+#au_prescan,
 #au_fullpre,
 #au_os_seek,
 #au_osd_cv,
@@ -1757,7 +1821,8 @@ html.y #tree.nowrap .ntree a+a:hover {
 	opacity: .3;
 }
 #griden.on+#thumbs,
-#au_preload.on+#au_fullpre,
+#au_preload.on+#au_prescan,
+#au_preload.on+#au_prescan+#au_fullpre,
 #au_os_ctl.on+#au_os_seek,
 #au_os_ctl.on+#au_os_seek+#au_osd_cv,
 #u2turbo.on+#u2tdate {
@@ -1767,6 +1832,7 @@ html.y #tree.nowrap .ntree a+a:hover {
 	display: none;
 }
 .ghead {
+	background: #fff;
 	background: var(--bg-u2);
 	border-radius: .3em;
 	padding: .2em .5em;
@@ -1796,7 +1862,13 @@ html.y #tree.nowrap .ntree a+a:hover {
 	margin: 0;
 	padding: 0;
 }
+#unpost td:nth-child(3),
+#unpost td:nth-child(4) {
+	text-align: right;
+}
+#shui,
 #rui {
+	background: #fff;
 	background: var(--bg);
 	position: fixed;
 	top: 0;
@@ -1810,23 +1882,41 @@ html.y #tree.nowrap .ntree a+a:hover {
 	padding: 1em;
 	z-index: 765;
 }
+#shui div+div,
 #rui div+div {
 	margin-top: 1em;
 }
+#shui table,
 #rui table {
 	width: 100%;
 	border-collapse: collapse;
 }
+#shui button {
+	margin: 0 1em 0 0;
+}
+#shui .btn {
+	font-size: 1em;
+}
+#shui td {
+	padding: .8em 0;
+}
+#shui td+td,
 #rui td+td {
 	padding: .2em 0 .2em .5em;
 }
 #rn_vadv input {
 	font-family: 'scp', monospace, monospace;
+	font-family: var(--font-mono), 'scp', monospace, monospace;
 }
+#shui td+td,
 #rui td+td,
+#shui td input[type="text"],
 #rui td input[type="text"] {
 	width: 100%;
 }
+#shui td.exs input[type="text"] {
+	width: 3em;
+}
 #rn_f.m td:first-child {
 	white-space: nowrap;
 }
@@ -1853,6 +1943,7 @@ html.y #tree.nowrap .ntree a+a:hover {
 }
 #doc {
 	overflow: visible;
+	background: #fff;
 	background: var(--bg);
 	margin: -1em 0 .5em 0;
 	padding: 1em 0 1em 0;
@@ -1869,6 +1960,10 @@ html.y #doc {
 	text-align: center;
 	padding: .5em;
 }
+#docul li.bn span {
+	font-weight: bold;
+	color: var(--fg-max);
+}
 #doc.prism {
 	padding-left: 3em;
 }
@@ -1880,6 +1975,7 @@ html.y #doc {
 #doc.mdo {
 	white-space: normal;
 	font-family: sans-serif;
+	font-family: var(--font-main), sans-serif;
 }
 #doc.prism * {
 	line-height: 1.5em;
@@ -1939,6 +2035,7 @@ a.btn,
 }
 #hkhelp td:first-child {
 	font-family: 'scp', monospace, monospace;
+	font-family: var(--font-mono), 'scp', monospace, monospace;
 }
 html.noscroll,
 html.noscroll .sbar {
@@ -2107,12 +2204,12 @@ html.y #bbox-overlay figcaption a {
 }
 .bbox-btn,
 #bbox-btns {
-    opacity: 1;
+	opacity: 1;
 	animation: opacity .2s infinite ease-in-out;
 }
 .bbox-btn.off,
 #bbox-btns.off {
-    opacity: 0;
+	opacity: 0;
 }
 #bbox-overlay button {
 	cursor: pointer;
@@ -2148,6 +2245,7 @@ html.y #bbox-overlay figcaption a {
 }
 #bbox-halp {
 	color: var(--fg-max);
+	background: #fff;
 	background: var(--bg);
 	position: absolute;
 	top: 0;
@@ -2382,7 +2480,7 @@ html.y #bbox-overlay figcaption a {
 	display: block;
 }
 #u2bm sup {
-    font-weight: bold;
+	font-weight: bold;
 }
 #u2notbtn {
 	display: none;
@@ -2447,6 +2545,7 @@ html.y #bbox-overlay figcaption a {
 }
 #op_up2k.srch td.prog {
 	font-family: sans-serif;
+	font-family: var(--font-main), sans-serif;
 	font-size: 1em;
 	width: auto;
 }
@@ -2461,6 +2560,7 @@ html.y #bbox-overlay figcaption a {
 	white-space: nowrap;
 	display: inline-block;
 	font-family: 'scp', monospace, monospace;
+	font-family: var(--font-mono), 'scp', monospace, monospace;
 }
 #u2etas.o {
 	width: 20em;
@@ -2489,14 +2589,14 @@ html.y #bbox-overlay figcaption a {
 	min-width: 24em;
 }
 #u2cards.w {
-	width: 44em;
+	width: 48em;
 	text-align: left;
 }
 #u2cards.ww {
 	display: inline-block;
 }
 #u2etaw.w {
-	width: 52em;
+	width: 55em;
 	text-align: right;
 	margin: 2em auto -2.7em auto;
 }
@@ -2530,6 +2630,7 @@ html.y #bbox-overlay figcaption a {
 #u2cards span {
 	color: var(--fg-max);
 	font-family: 'scp', monospace;
+	font-family: var(--font-mono), 'scp', monospace;
 }
 #u2cards > a:nth-child(4) > span {
 	display: inline-block;
@@ -2541,10 +2642,10 @@ html.y #bbox-overlay figcaption a {
 	width: 30em;
 }
 #u2conf.w {
-	width: 48em;
+	width: 51em;
 }
 #u2conf.ww {
-	width: 78em;
+	width: 82em;
 }
 #u2conf.ww #u2c3w {
 	width: 29em;
@@ -2610,23 +2711,25 @@ html.b #u2conf a.b:hover {
 #u2conf input[type="checkbox"]:checked+label {
 	position: relative;
 	cursor: pointer;
-	background: var(--u2-o-bg);
-	border-bottom: .2em solid var(--u2-o-b1);
-	box-shadow: 0 .1em .3em var(--u2-o-sh) inset;
+	background: var(--btn-bg);
+	box-shadow: var(--btn-bs);
+	border-bottom: var(--btn-bb);
 	text-shadow: 1px 1px 1px #000, 1px -1px 1px #000, -1px -1px 1px #000, -1px 1px 1px #000;
 }
 #u2conf input[type="checkbox"]:checked+label {
-	background: var(--u2-o-1-bg);
-	border-bottom: .2em solid var(--u2-o-1-b1);
-	box-shadow: 0 .1em .5em var(--u2-o-1-sh);
+	background: var(--btn-1-bg);
+	box-shadow: var(--btn-1-bs);
+	border-bottom: var(--btn-1-bb);
 }
 #u2conf input[type="checkbox"]+label:hover {
-	box-shadow: 0 .1em .3em var(--u2-o-h-sh);
-	border-color: var(--u2-o-h-b1);
-	background: var(--u2-o-h-bg);
+	background: var(--btn-h-bg);
+	box-shadow: var(--btn-h-bs);
+	border-bottom: var(--btn-h-bb);
 }
 #u2conf input[type="checkbox"]:checked+label:hover {
-	background: var(--u2-o-1h-bg);
+	background: var(--btn-1h-bg);
+	box-shadow: var(--btn-1h-bs);
+	border-bottom: var(--btn-1h-bb);
 }
 #op_up2k.srch #u2conf td:nth-child(2)>*,
 #op_up2k.srch #u2conf td:nth-child(3)>* {
@@ -2695,6 +2798,7 @@ html.b #u2conf a.b:hover {
 }
 .prog {
 	font-family: 'scp', monospace, monospace;
+	font-family: var(--font-mono), 'scp', monospace, monospace;
 }
 #u2tab span.inf,
 #u2tab span.ok,
@@ -2766,6 +2870,9 @@ html.c .opbox,
 html.a .opbox {
 	margin: 1.5em 0 0 0;
 }
+html.dz .opview input.i {
+	width: calc(100% - 18em);
+}
 html.c #tree,
 html.c #treeh,
 html.a #tree,
@@ -2818,6 +2925,9 @@ html.a #u2btn {
 html.ay #u2btn {
 	box-shadow: .4em .4em 0 #ccc;
 }
+html.dz #u2btn {
+	letter-spacing: -.033em;
+}
 html.c #u2conf.ww #u2btn,
 html.a #u2conf.ww #u2btn {
 	margin: -2em .5em -3em 0;
@@ -2980,13 +3090,13 @@ html.b .btn {
 	top: -.1em;
 }
 html.b #op_up2k.srch sup {
-    color: #fc0;
+	color: #fc0;
 }
 html.by #u2btn sup {
-    color: #06b;
+	color: #06b;
 }
 html.by #op_up2k.srch sup {
-    color: #b70;
+	color: #b70;
 }
 html.bz #u2cards a.act {
 	box-shadow: 0 -.1em .2em var(--bg-d2);
@@ -3008,18 +3118,30 @@ html.by #u2cards a.act {
 
 
 
-html.cy #wrap {
-	color: #000;
-}
 html.cy .mdo a {
 	background: #f00;
 }
+html.cy #wrap,
+html.cy #acc_info a,
 html.cy #op_up2k,
 html.cy #files,
 html.cy #files a,
 html.cy #files tbody div a:last-child {
 	color: #000;
 }
+html.cy #u2tab a,
+html.cy #u2cards a {
+	color: #f00;
+}
+html.cy #unpost a {
+	color: #ff0;
+}
+html.cy #barbuf {
+	filter: hue-rotate(267deg) brightness(0.8) contrast(4);
+}
+html.cy #pvol {
+	filter: hue-rotate(4deg) contrast(2.2);
+}
 
 
 
@@ -3035,6 +3157,16 @@ html.d #treepar {
 
 
 
+@media (max-width: 32em) {
+	#u2conf {
+		font-size: .9em;
+	}
+}
+@media (max-width: 28em) {
+	#u2conf {
+		font-size: .8em;
+	}
+}
 @media (min-width: 70em) {
 	#barpos,
 	#barbuf {
@@ -3087,7 +3219,7 @@ html.d #treepar {
 		margin-top: 1.7em;
 	}
 }
-@supports (display: grid) {
+@supports (display: grid) and (gap: 1em) {
 	#ggrid {
 		display: grid;
 		margin: 0em 0.25em;
@@ -3112,3 +3244,24 @@ html.d #treepar {
 		padding: 0.2em;
 	}
 }
+
+
+
+
+
+@media (prefers-reduced-motion) {
+	@keyframes spin { }
+	@keyframes gexit { }
+	@keyframes bounce { }
+	@keyframes bounceFromLeft { }
+	@keyframes bounceFromRight { }
+
+	#ggrid>a:before,
+	#widget.anim,
+	#u2tabw,
+	.dropdesc,
+	.dropdesc b,
+	.dropdesc>div>div {
+		transition: none;
+	}
+}

copyparty/web/browser.html

@@ -6,12 +6,12 @@
 	<title>{{ title }}</title>
 	<meta http-equiv="X-UA-Compatible" content="IE=edge">
 	<meta name="viewport" content="width=device-width, initial-scale=0.8, minimum-scale=0.6">
-	<meta name="theme-color" content="#333">
-{{ html_head }}
+	<meta name="theme-color" content="#{{ tcolor }}">
 	<link rel="stylesheet" media="screen" href="{{ r }}/.cpr/ui.css?_={{ ts }}">
 	<link rel="stylesheet" media="screen" href="{{ r }}/.cpr/browser.css?_={{ ts }}">
+{{ html_head }}
 	{%- if css %}
-	<link rel="stylesheet" media="screen" href="{{ css }}?_={{ ts }}">
+	<link rel="stylesheet" media="screen" href="{{ css }}_={{ ts }}">
 	{%- endif %}
 </head>
 
@@ -67,14 +67,14 @@
 	<div id="op_up2k" class="opview"></div>
 
 	<div id="op_cfg" class="opview opbox opwide"></div>
-	
+
 	<h1 id="path">
 		<a href="#" id="entree">🌲</a>
 		{%- for n in vpnodes %}
 		<a href="{{ r }}/{{ n[0] }}">{{ n[1] }}</a>
 		{%- endfor %}
 	</h1>
-	
+
 	<div id="tree"></div>
 
 <div id="wrap">
@@ -118,11 +118,11 @@
 
 		</tbody>
 	</table>
-	
+
 	<div id="epi" class="logue">{{ "" if sb_lg else logues[1] }}</div>
 
 	<h2 id="wfp"><a href="{{ r }}/?h" id="goh">control-panel</a></h2>
-	
+
 	<a href="#" id="repl">π</a>
 
 </div>
@@ -135,46 +135,30 @@
 
 	<script>
 		var SR = {{ r|tojson }},
+			CGV = {{ cgv|tojson }},
 			TS = "{{ ts }}",
-			acct = "{{ acct }}",
-			perms = {{ perms }},
-			dgrid = {{ dgrid|tojson }},
-			themes = {{ themes }},
 			dtheme = "{{ dtheme }}",
 			srvinf = "{{ srv_info }}",
 			s_name = "{{ s_name }}",
 			lang = "{{ lang }}",
 			dfavico = "{{ favico }}",
-			def_hcols = {{ def_hcols|tojson }},
-			have_up2k_idx = {{ have_up2k_idx|tojson }},
 			have_tags_idx = {{ have_tags_idx|tojson }},
-			have_acode = {{ have_acode|tojson }},
-			have_mv = {{ have_mv|tojson }},
-			have_del = {{ have_del|tojson }},
-			have_unpost = {{ have_unpost }},
-			have_zip = {{ have_zip|tojson }},
-			sb_md = "{{ sb_md }}",
 			sb_lg = "{{ sb_lg }}",
-			lifetime = {{ lifetime }},
-			turbolvl = {{ turbolvl }},
-			idxh = {{ idxh }},
-			frand = {{ frand|tojson }},
-			u2sort = "{{ u2sort }}",
-			have_emp = {{ have_emp|tojson }},
 			txt_ext = "{{ txt_ext }}",
 			logues = {{ logues|tojson if sb_lg else "[]" }},
-			readme = {{ readme|tojson }},
 			ls0 = {{ ls0|tojson }};
 
-		document.documentElement.className = localStorage.theme || dtheme;
+		var STG = window.localStorage;
+		document.documentElement.className = (STG && STG.cpp_thm) || dtheme;
 	</script>
 	<script src="{{ r }}/.cpr/util.js?_={{ ts }}"></script>
 	<script src="{{ r }}/.cpr/baguettebox.js?_={{ ts }}"></script>
 	<script src="{{ r }}/.cpr/browser.js?_={{ ts }}"></script>
 	<script src="{{ r }}/.cpr/up2k.js?_={{ ts }}"></script>
 	{%- if js %}
-	<script src="{{ js }}?_={{ ts }}"></script>
+	<script src="{{ js }}_={{ ts }}"></script>
 	{%- endif %}
 </body>
 
 </html>
+

copyparty/web/browser2.html

@@ -6,7 +6,6 @@
 	<title>{{ title }}</title>
 	<meta http-equiv="X-UA-Compatible" content="IE=edge">
 	<meta name="viewport" content="width=device-width, initial-scale=0.8">
-{{ html_head }}
 	<style>
 		html{font-family:sans-serif}
 		td{border:1px solid #999;border-width:1px 1px 0 0;padding:0 5px}
@@ -52,12 +51,13 @@
 
 		</tbody>
 	</table>
-	
+
 	{%- if logues[1] %}
 	<div>{{ logues[1] }}</div><br />
 	{%- endif %}
-	
+
 	<h2><a href="{{ r }}/{{ url_suf }}{{ url_suf and '&amp;' or '?' }}h">control-panel</a></h2>
 
 </body>
 </html>
+

copyparty/web/cf.html

@@ -25,3 +25,4 @@
 </body>
 
 </html>
+

copyparty/web/md.css

@@ -2,6 +2,7 @@ html, body {
 	color: #333;
 	background: #eee;
 	font-family: sans-serif;
+	font-family: var(--font-main), sans-serif;
 	line-height: 1.5em;
 }
 html.y #helpbox a {
@@ -67,6 +68,7 @@ a {
 	position: relative;
 	display: inline-block;
 	font-family: 'scp', monospace, monospace;
+	font-family: var(--font-mono), 'scp', monospace, monospace;
 	font-weight: bold;
 	font-size: 1.3em;
 	line-height: .1em;

copyparty/web/md.html

@@ -3,13 +3,13 @@
 	<title>📝 {{ title }}</title>
 	<meta http-equiv="X-UA-Compatible" content="IE=edge">
 	<meta name="viewport" content="width=device-width, initial-scale=0.7">
-	<meta name="theme-color" content="#333">
-{{ html_head }}
+	<meta name="theme-color" content="#{{ tcolor }}">
 	<link rel="stylesheet" href="{{ r }}/.cpr/ui.css?_={{ ts }}">
 	<link rel="stylesheet" href="{{ r }}/.cpr/md.css?_={{ ts }}">
 	{%- if edit %}
 	<link rel="stylesheet" href="{{ r }}/.cpr/md2.css?_={{ ts }}">
 	{%- endif %}
+{{ html_head }}
 </head>
 <body>
 	<div id="mn"></div>
@@ -31,7 +31,7 @@
 			<span id="lno">L#</span>
 		{%- else %}
 			<a href="{{ arg_base }}edit" tt="good: higher performance$Ngood: same document width as viewer$Nbad: assumes you know markdown">edit (basic)</a>
-			<a href="{{ arg_base }}edit2" tt="not in-house so probably less buggy">edit (fancy)</a>
+			<a href="{{ arg_base }}edit2" id="edit2" tt="not in-house so probably less buggy">edit (fancy)</a>
 			<a href="{{ arg_base }}">view raw</a>
 		{%- endif %}
 	</div>
@@ -49,7 +49,7 @@
 		<div id="mp" class="mdo"></div>
 	</div>
 	<a href="#" id="repl">π</a>
-	
+
 	{%- if edit %}
 	<div id="helpbox">
 		<textarea autocomplete="off">
@@ -125,7 +125,7 @@ write markdown (most html is 🙆 too)
 		</textarea>
 	</div>
 	{%- endif %}
-	
+
 	<script>
 
 var SR = {{ r|tojson }},
@@ -139,16 +139,15 @@ var md_opt = {
 };
 
 (function () {
-	var l = localStorage,
-		drk = l.light != 1,
+	var l = window.localStorage,
+		drk = (l && l.light) != 1,
 		btn = document.getElementById("lightswitch"),
 		f = function (e) {
 if (e) { e.preventDefault(); drk = !drk; }
 document.documentElement.className = drk? "z":"y";
 btn.innerHTML = "go " + (drk ? "light":"dark");
-l.light = drk? 0:1;
+try { l.light = drk? 0:1; } catch (ex) { }
 		};
-	
 	btn.onclick = f;
 	f();
 })();
@@ -160,4 +159,8 @@ l.light = drk? 0:1;
 	{%- if edit %}
 	<script src="{{ r }}/.cpr/md2.js?_={{ ts }}"></script>
 	{%- endif %}
+	{%- if js %}
+	<script src="{{ js }}_={{ ts }}"></script>
+	{%- endif %}
 </body></html>
+

copyparty/web/md.js

@@ -52,7 +52,7 @@ var img_load = (function () {
     var r = {};
     r.callbacks = [];
 
-    function fire() {
+    var fire = function () {
         for (var a = 0; a < r.callbacks.length; a++)
             r.callbacks[a]();
     }
@@ -216,6 +216,11 @@ function convert_markdown(md_text, dest_dom) {
             md_html = DOMPurify.sanitize(md_html);
     }
     catch (ex) {
+        if (IE) {
+            dest_dom.innerHTML = 'IE cannot into markdown ;_;';
+            return;
+        }
+
         if (ext)
             md_plug_err(ex, ext[1]);
 
@@ -472,7 +477,7 @@ img_load.callbacks = [toc.refresh];
 // scroll handler
 var redraw = (function () {
     var sbs = true;
-    function onresize() {
+    var onresize = function () {
         if (window.matchMedia)
             sbs = window.matchMedia('(min-width: 64em)').matches;
 
@@ -485,7 +490,7 @@ var redraw = (function () {
         onscroll();
     }
 
-    function onscroll() {
+    var onscroll = function () {
         toc.refresh();
     }
 

copyparty/web/md2.css

@@ -9,7 +9,7 @@
 	width: calc(100% - 56em);
 }
 #mw {
-	left: calc(100% - 55em);
+	left: max(0em, calc(100% - 55em));
 	overflow-y: auto;
 	position: fixed;
 	bottom: 0;
@@ -56,6 +56,7 @@
 	padding: 0;
 	margin: 0;
 	font-family: 'scp', monospace, monospace;
+	font-family: var(--font-mono), 'scp', monospace, monospace;
 	white-space: pre-wrap;
 	word-break: break-word;
 	overflow-wrap: break-word;

copyparty/web/md2.js

@@ -92,7 +92,7 @@ var action_stack = null;
 var nlines = 0;
 var draw_md = (function () {
     var delay = 1;
-    function draw_md() {
+    var draw_md = function () {
         var t0 = Date.now();
         var src = dom_src.value;
         convert_markdown(src, dom_pre);
@@ -135,7 +135,7 @@ img_load.callbacks = [function () {
 
 // resize handler
 redraw = (function () {
-    function onresize() {
+    var onresize = function () {
         var y = (dom_hbar.offsetTop + dom_hbar.offsetHeight) + 'px';
         dom_wrap.style.top = y;
         dom_swrap.style.top = y;
@@ -143,12 +143,12 @@ redraw = (function () {
         map_src = genmap(dom_ref, map_src);
         map_pre = genmap(dom_pre, map_pre);
     }
-    function setsbs() {
+    var setsbs = function () {
         dom_wrap.className = '';
         dom_swrap.className = '';
         onresize();
     }
-    function modetoggle() {
+    var modetoggle = function () {
         var mode = dom_nsbs.innerHTML;
         dom_nsbs.innerHTML = mode == 'editor' ? 'preview' : 'editor';
         mode += ' single';
@@ -163,7 +163,7 @@ redraw = (function () {
     dom_sbs.onclick = setsbs;
     dom_nsbs.onclick = modetoggle;
 
-    onresize();
+    (IE ? modetoggle : onresize)();
     return onresize;
 })();
 
@@ -172,7 +172,7 @@ redraw = (function () {
 (function () {
     var skip_src = false, skip_pre = false;
 
-    function scroll(src, srcmap, dst, dstmap) {
+    var scroll = function (src, srcmap, dst, dstmap) {
         var y = src.scrollTop;
         if (y < 8) {
             dst.scrollTop = 0;
@@ -368,14 +368,14 @@ function save(e) {
 
 function save_cb() {
     if (this.status !== 200)
-        return toast.err(0, 'Error!  The file was NOT saved.\n\n' + this.status + ": " + (this.responseText + '').replace(/^<pre>/, ""));
+        return toast.err(0, 'Error!  The file was NOT saved.\n\nError ' + this.status + ":\n" + unpre(this.responseText));
 
     var r;
     try {
         r = JSON.parse(this.responseText);
     }
     catch (ex) {
-        return toast.err(0, 'Failed to parse reply from server:\n\n' + this.responseText);
+        return toast.err(0, 'Error!  The file was likely NOT saved.\n\nFailed to parse reply from server:\n\n' + unpre(this.responseText));
     }
 
     if (!r.ok) {
@@ -418,7 +418,7 @@ function run_savechk(lastmod, txt, btn, ntry) {
 
 function savechk_cb() {
     if (this.status !== 200)
-        return toast.err(0, 'Error!  The file was NOT saved.\n\n' + this.status + ": " + (this.responseText + '').replace(/^<pre>/, ""));
+        return toast.err(0, 'Error!  The file was NOT saved.\n\nError ' + this.status + ":\n" + unpre(this.responseText));
 
     var doc1 = this.txt.replace(/\r\n/g, "\n");
     var doc2 = this.responseText.replace(/\r\n/g, "\n");
@@ -607,10 +607,10 @@ function md_newline() {
     var s = linebounds(true),
         ln = s.md.substring(s.n1, s.n2),
         m1 = /^( *)([0-9]+)(\. +)/.exec(ln),
-        m2 = /^[ \t>+-]*(\* )?/.exec(ln),
+        m2 = /^[ \t]*[>+*-]{0,2}[ \t]/.exec(ln),
         drop = dom_src.selectionEnd - dom_src.selectionStart;
 
-    var pre = m2[0];
+    var pre = m2 ? m2[0] : '';
     if (m1 !== null)
         pre = m1[1] + (parseInt(m1[2]) + 1) + m1[3];
 
@@ -900,12 +900,12 @@ var set_lno = (function () {
         pv = null,
         lno = ebi('lno');
 
-    function poke() {
+    var poke = function () {
         clearTimeout(t);
         t = setTimeout(fire, 20);
     }
 
-    function fire() {
+    var fire = function () {
         try {
             clearTimeout(t);
 
@@ -930,8 +930,14 @@ var set_lno = (function () {
 
 // hotkeys / toolbar
 (function () {
-    function keydown(ev) {
-        ev = ev || window.event;
+    var keydown = function (ev) {
+        if (!ev && window.event) {
+            ev = window.event;
+            if (dev_fbw == 1) {
+                toast.warn(10, 'hello from fallback code ;_;\ncheck console trace');
+                console.error('using window.event');
+            }
+        }
         var kc = ev.code || ev.keyCode || ev.which,
             editing = document.activeElement == dom_src;
 
@@ -1003,7 +1009,7 @@ var set_lno = (function () {
                 md_home(ev.shiftKey);
                 return false;
             }
-            if (!ev.shiftKey && (ev.code == "Enter" || kc == 13)) {
+            if (!ev.shiftKey && ((ev.code + '').endsWith("Enter") || kc == 13)) {
                 return md_newline();
             }
             if (!ev.shiftKey && kc == 8) {
@@ -1058,7 +1064,7 @@ action_stack = (function () {
     var ignore = false;
     var ref = dom_src.value;
 
-    function diff(from, to, cpos) {
+    var diff = function (from, to, cpos) {
         if (from === to)
             return null;
 
@@ -1089,14 +1095,14 @@ action_stack = (function () {
         };
     }
 
-    function undiff(from, change) {
+    var undiff = function (from, change) {
         return {
             txt: from.substring(0, change.car) + change.txt + from.substring(change.cdr),
             cpos: change.cpos
         };
     }
 
-    function apply(src, dst) {
+    var apply = function (src, dst) {
         dbg('undos(%d) redos(%d)', hist.un.length, hist.re.length);
 
         if (src.length === 0)
@@ -1120,7 +1126,7 @@ action_stack = (function () {
         return true;
     }
 
-    function schedule_push() {
+    var schedule_push = function () {
         if (ignore) {
             ignore = false;
             return;
@@ -1131,7 +1137,7 @@ action_stack = (function () {
         sched_timer = setTimeout(push, 500);
     }
 
-    function undo() {
+    var undo = function () {
         if (hist.re.length == 0) {
             clearTimeout(sched_timer);
             push();
@@ -1139,11 +1145,11 @@ action_stack = (function () {
         return apply(hist.un, hist.re);
     }
 
-    function redo() {
+    var redo = function () {
         return apply(hist.re, hist.un);
     }
 
-    function push() {
+    var push = function () {
         var newtxt = dom_src.value;
         var change = diff(ref, newtxt, sched_cpos);
         if (change !== null)

copyparty/web/mde.css

@@ -17,6 +17,7 @@ html, body {
 	padding: 0;
 	min-height: 100%;
 	font-family: sans-serif;
+	font-family: var(--font-main), sans-serif;
 	background: #f7f7f7;
 	color: #333;
 }

copyparty/web/mde.html

@@ -3,12 +3,12 @@
 	<title>📝 {{ title }}</title>
 	<meta http-equiv="X-UA-Compatible" content="IE=edge">
 	<meta name="viewport" content="width=device-width, initial-scale=0.7">
-	<meta name="theme-color" content="#333">
-{{ html_head }}
+	<meta name="theme-color" content="#{{ tcolor }}">
 	<link rel="stylesheet" href="{{ r }}/.cpr/ui.css?_={{ ts }}">
 	<link rel="stylesheet" href="{{ r }}/.cpr/mde.css?_={{ ts }}">
 	<link rel="stylesheet" href="{{ r }}/.cpr/deps/mini-fa.css?_={{ ts }}">
 	<link rel="stylesheet" href="{{ r }}/.cpr/deps/easymde.css?_={{ ts }}">
+{{ html_head }}
 </head>
 <body>
 	<div id="mw">
@@ -37,12 +37,12 @@ var md_opt = {
 };
 
 var lightswitch = (function () {
-	var l = localStorage,
-		drk = l.light != 1,
+	var l = window.localStorage,
+		drk = (l && l.light) != 1,
 		f = function (e) {
 if (e) drk = !drk;
 document.documentElement.className = drk? "z":"y";
-l.light = drk? 0:1;
+try { l.light = drk? 0:1; } catch (ex) { }
 		};
 	f();
 	return f;
@@ -53,4 +53,8 @@ l.light = drk? 0:1;
 	<script src="{{ r }}/.cpr/deps/marked.js?_={{ ts }}"></script>
 	<script src="{{ r }}/.cpr/deps/easymde.js?_={{ ts }}"></script>
 	<script src="{{ r }}/.cpr/mde.js?_={{ ts }}"></script>
+	{%- if js %}
+	<script src="{{ js }}_={{ ts }}"></script>
+	{%- endif %}
 </body></html>
+

copyparty/web/mde.js

@@ -134,14 +134,14 @@ function save(mde) {
 
 function save_cb() {
     if (this.status !== 200)
-        return toast.err(0, 'Error!  The file was NOT saved.\n\n' + this.status + ": " + (this.responseText + '').replace(/^<pre>/, ""));
+        return toast.err(0, 'Error!  The file was NOT saved.\n\nError ' + this.status + ":\n" + unpre(this.responseText));
 
     var r;
     try {
         r = JSON.parse(this.responseText);
     }
     catch (ex) {
-        return toast.err(0, 'Failed to parse reply from server:\n\n' + this.responseText);
+        return toast.err(0, 'Error!  The file was likely NOT saved.\n\nFailed to parse reply from server:\n\n' + unpre(this.responseText));
     }
 
     if (!r.ok) {
@@ -180,7 +180,7 @@ function save_cb() {
 
 function save_chk() {
     if (this.status !== 200)
-        return toast.err(0, 'Error!  The file was NOT saved.\n\n' + this.status + ": " + (this.responseText + '').replace(/^<pre>/, ""));
+        return toast.err(0, 'Error!  The file was NOT saved.\n\nError ' + this.status + ":\n" + unpre(this.responseText));
 
     var doc1 = this.txt.replace(/\r\n/g, "\n");
     var doc2 = this.responseText.replace(/\r\n/g, "\n");

copyparty/web/msg.css

@@ -1,3 +1,8 @@
+:root {
+	--font-main: sans-serif;
+	--font-serif: serif;
+	--font-mono: 'scp';
+}
 html,body,tr,th,td,#files,a {
 	color: inherit;
 	background: none;
@@ -10,6 +15,7 @@ html {
 	color: #ccc;
 	background: #333;
 	font-family: sans-serif;
+	font-family: var(--font-main), sans-serif;
 	text-shadow: 1px 1px 0px #000;
 	touch-action: manipulation;
 }
@@ -23,6 +29,7 @@ html, body {
 }
 pre {
 	font-family: monospace, monospace;
+	font-family: var(--font-mono), monospace, monospace;
 }
 a {
 	color: #fc5;

copyparty/web/msg.html

@@ -6,9 +6,9 @@
 	<title>{{ s_doctitle }}</title>
 	<meta http-equiv="X-UA-Compatible" content="IE=edge">
 	<meta name="viewport" content="width=device-width, initial-scale=0.8">
-	<meta name="theme-color" content="#333">
-{{ html_head }}
+	<meta name="theme-color" content="#{{ tcolor }}">
 	<link rel="stylesheet" media="screen" href="{{ r }}/.cpr/msg.css?_={{ ts }}">
+{{ html_head }}
 </head>
 
 <body>
@@ -42,10 +42,14 @@
 	{%- if redir %}
 	<script>
 		setTimeout(function() {
-			window.location.replace("{{ redir }}");
+			location.replace("{{ redir }}");
 		}, 1000);
 	</script>
 	{%- endif %}
+	{%- if js %}
+	<script src="{{ js }}_={{ ts }}"></script>
+	{%- endif %}
 </body>
 
-</html>
+</html>
+

copyparty/web/shares.css

@@ -0,0 +1,82 @@
+html {
+	color: #333;
+	background: #f7f7f7;
+	font-family: sans-serif;
+	font-family: var(--font-main), sans-serif;
+	touch-action: manipulation;
+}
+#wrap {
+	margin: 2em auto;
+	padding: 0 1em 3em 1em;
+	line-height: 2.3em;
+}
+#wrap>span {
+    margin: 0 0 0 1em;
+    border-bottom: 1px solid #999;
+}
+li {
+	margin: 1em 0;
+}
+a {
+	color: #047;
+	background: #fff;
+	text-decoration: none;
+	white-space: nowrap;
+	border-bottom: 1px solid #8ab;
+	border-radius: .2em;
+	padding: .2em .6em;
+	margin: 0 .3em;
+}
+td a {
+	margin: 0;
+}
+#w {
+	color: #fff;
+	background: #940;
+	border-color: #b70;
+}
+#repl {
+	border: none;
+	background: none;
+	color: inherit;
+	padding: 0;
+	position: fixed;
+	bottom: .25em;
+	left: .2em;
+}
+table {
+	border-collapse: collapse;
+    position: relative;
+}
+th {
+	top: -1px;
+	position: sticky;
+    background: #f7f7f7;
+}
+td, th {
+	padding: .3em .6em;
+	text-align: left;
+	white-space: nowrap;
+}
+td+td+td+td+td+td+td+td {
+	font-family: var(--font-mono), monospace, monospace;
+}
+
+
+
+html.z {
+	background: #222;
+	color: #ccc;
+}
+html.z a {
+	color: #fff;
+	background: #057;
+	border-color: #37a;
+}
+html.z th {
+    background: #222;
+}
+html.bz {
+	color: #bbd;
+	background: #11121d;
+}