v0.11.16

.vscode/launch.json

@@ -16,12 +16,9 @@
                 "-e2ts",
                 "-mtp",
                 ".bpm=f,bin/mtag/audio-bpm.py",
-                "-a",
+                "-aed:wark",
-                "ed:wark",
+                "-vsrv::r:aed:cnodupe",
-                "-v",
+                "-vdist:dist:r"
-                "srv::r:aed:cnodupe",
-                "-v",
-                "dist:dist:r"
             ]
         },
         {
@@ -43,5 +40,13 @@
                 "${file}"
             ]
         },
+        {
+            "name": "Python: Current File",
+            "type": "python",
+            "request": "launch",
+            "program": "${file}",
+            "console": "integratedTerminal",
+            "justMyCode": false
+        },
     ]
 }

.vscode/launch.py

@@ -3,14 +3,16 @@
 # launches 10x faster than mspython debugpy
 # and is stoppable with ^C
 
+import re
 import os
 import sys
+
+print(sys.executable)
+
 import shlex
-
-sys.path.insert(0, os.getcwd())
-
 import jstyleson
-from copyparty.__main__ import main as copyparty
+import subprocess as sp
+
 
 with open(".vscode/launch.json", "r", encoding="utf-8") as f:
     tj = f.read()
@@ -25,11 +27,19 @@ except:
     pass
 
 argv = [os.path.expanduser(x) if x.startswith("~") else x for x in argv]
-try:
+
-    copyparty(["a"] + argv)
+if re.search(" -j ?[0-9]", " ".join(argv)):
-except SystemExit as ex:
+    argv = [sys.executable, "-m", "copyparty"] + argv
-    if ex.code:
+    sp.check_call(argv)
-        raise
+else:
+    sys.path.insert(0, os.getcwd())
+    from copyparty.__main__ import main as copyparty
+
+    try:
+        copyparty(["a"] + argv)
+    except SystemExit as ex:
+        if ex.code:
+            raise
 
 print("\n\033[32mokke\033[0m")
 sys.exit(1)

README.md

@@ -37,6 +37,7 @@ turn your phone or raspi into a portable file server with resumable uploads/down
     * [other tricks](#other-tricks)
 * [searching](#searching)
     * [search configuration](#search-configuration)
+    * [database location](#database-location)
     * [metadata from audio files](#metadata-from-audio-files)
     * [file parser plugins](#file-parser-plugins)
     * [complete examples](#complete-examples)
@@ -105,6 +106,7 @@ summary: all planned features work! now please enjoy the bloatening
     * ☑ images using Pillow
     * ☑ videos using FFmpeg
     * ☑ cache eviction (max-age; maybe max-size eventually)
+  * ☑ image gallery
   * ☑ SPA (browse while uploading)
     * if you use the file-tree on the left only, not folders in the file list
 * server indexing
@@ -121,12 +123,12 @@ summary: all planned features work! now please enjoy the bloatening
 * Windows: python 3.7 and older cannot read tags with ffprobe, so use mutagen or upgrade
 * Windows: python 2.7 cannot index non-ascii filenames with `-e2d`
 * Windows: python 2.7 cannot handle filenames with mojibake
+* MacOS: `--th-ff-jpg` may fix thumbnails using macports-FFmpeg
 
 ## general bugs
 
 * all volumes must exist / be available on startup; up2k (mtp especially) gets funky otherwise
 * cannot mount something at `/d1/d2/d3` unless `d2` exists inside `d1`
-* hiding the contents at url `/d1/d2/d3` using `-v :d1/d2/d3:cd2d` has the side-effect of creating databases (for files/tags) inside folders d1 and d2, and those databases take precedence over the main db at the top of the vfs - this means all files in d2 and below will be reindexed unless you already had a vfs entry at or below d2
 * probably more, pls let me know
 
 ## not my bugs
@@ -180,6 +182,8 @@ click `[-]` and `[+]` to adjust the size, and the `[a]` toggles if the tree shou
 
 it does static images with Pillow and uses FFmpeg for video files, so you may want to `--no-thumb` or maybe just `--no-vthumb` depending on how destructive your users are
 
+images named `folder.jpg` and `folder.png` become the thumbnail of the folder they're in
+
 
 ## zip downloads
 
@@ -296,9 +300,29 @@ the same arguments can be set as volume flags, in addition to `d2d` and `d2t` fo
 * `-v ~/music::r:cd2d` disables **all** indexing, even if any `-e2*` are on
 * `-v ~/music::r:cd2t` disables all `-e2t*` (tags), does not affect `-e2d*`
 
-`e2tsr` is probably always overkill, since `e2ds`/`e2dsa` would pick up any file modifications and cause `e2ts` to reindex those
+note:
+* `e2tsr` is probably always overkill, since `e2ds`/`e2dsa` would pick up any file modifications and cause `e2ts` to reindex those
+* the rescan button in the admin panel has no effect unless the volume has `-e2ds` or higher
 
-the rescan button in the admin panel has no effect unless the volume has `-e2ds` or higher
+you can choose to only index filename/path/size/last-modified (and not the hash of the file contents) by setting `--no-hash` or the volume-flag `cdhash`, this has the following consequences:
+* initial indexing is way faster, especially when the volume is on a networked disk
+* makes it impossible to [file-search](#file-search)
+* if someone uploads the same file contents, the upload will not be detected as a dupe, so it will not get symlinked or rejected
+
+if you set `--no-hash`, you can enable hashing for specific volumes using flag `cehash`
+
+
+## database location
+
+copyparty creates a subfolder named `.hist` inside each volume where it stores the database, thumbnails, and some other stuff
+
+this can instead be kept in a single place using the `--hist` argument, or the `hist=` volume flag, or a mix of both:
+* `--hist ~/.cache/copyparty -v ~/music::r:chist=-` sets `~/.cache/copyparty` as the default place to put volume info, but `~/music` gets the regular `.hist` subfolder (`-` restores default behavior)
+
+note:
+* markdown edits are always stored in a local `.hist` subdirectory
+* on windows the volflag path is cyglike, so `/c/temp` means `C:\temp` but use regular paths for `--hist`
+  * you can use cygpaths for volumes too, `-v C:\Users::r` and `-v /c/users::r` both work
 
 
 ## metadata from audio files

bin/copyparty-fuse.py

@@ -54,6 +54,12 @@ MACOS = platform.system() == "Darwin"
 info = log = dbg = None
 
 
+print("{} v{} @ {}".format(
+    platform.python_implementation(),
+    ".".join([str(x) for x in sys.version_info]),
+    sys.executable))
+
+
 try:
     from fuse import FUSE, FuseOSError, Operations
 except:

contrib/nginx/copyparty.conf

@@ -1,3 +1,8 @@
+# when running copyparty behind a reverse-proxy,
+# make sure that copyparty allows at least as many clients as the proxy does,
+# so run copyparty with -nc 512 if your nginx has the default limits
+# (worker_processes 1, worker_connections 512)
+
 upstream cpp {
 	server 127.0.0.1:3923;
 	keepalive 120;

copyparty/__main__.py

@@ -23,7 +23,7 @@ from textwrap import dedent
 from .__init__ import E, WINDOWS, VT100, PY2
 from .__version__ import S_VERSION, S_BUILD_DT, CODENAME
 from .svchub import SvcHub
-from .util import py_desc, align_tab, IMPLICATIONS
+from .util import py_desc, align_tab, IMPLICATIONS, alltrace
 
 HAVE_SSL = True
 try:
@@ -182,6 +182,16 @@ def sighandler(sig=None, frame=None):
     print("\n".join(msg))
 
 
+def stackmon(fp, ival):
+    ctr = 0
+    while True:
+        ctr += 1
+        time.sleep(ival)
+        st = "{}, {}\n{}".format(ctr, time.time(), alltrace())
+        with open(fp, "wb") as f:
+            f.write(st.encode("utf-8", "replace"))
+
+
 def run_argparse(argv, formatter):
     ap = argparse.ArgumentParser(
         formatter_class=formatter,
@@ -222,10 +232,6 @@ def run_argparse(argv, formatter):
               "print,get" prints the data in the log and returns GET
               (leave out the ",get" to return an error instead)
 
-            --ciphers help = available ssl/tls ciphers,
-            --ssl-ver help = available ssl/tls versions,
-              default is what python considers safe, usually >= TLS1
-            
             values for --ls:
               "USR" is a user to browse as; * is anonymous, ** is all users
               "VOL" is a single volume to scan, default is * (all vols)
@@ -244,24 +250,45 @@ def run_argparse(argv, formatter):
     )
     # fmt: off
     ap.add_argument("-c", metavar="PATH", type=str, action="append", help="add config file")
-    ap.add_argument("-i", metavar="IP", type=str, default="0.0.0.0", help="ip to bind (comma-sep.)")
-    ap.add_argument("-p", metavar="PORT", type=str, default="3923", help="ports to bind (comma/range)")
     ap.add_argument("-nc", metavar="NUM", type=int, default=64, help="max num clients")
     ap.add_argument("-j", metavar="CORES", type=int, default=1, help="max num cpu cores")
-    ap.add_argument("-a", metavar="ACCT", type=str, action="append", help="add account")
+    ap.add_argument("-a", metavar="ACCT", type=str, action="append", help="add account, USER:PASS; example [ed:wark")
-    ap.add_argument("-v", metavar="VOL", type=str, action="append", help="add volume")
+    ap.add_argument("-v", metavar="VOL", type=str, action="append", help="add volume, SRC:DST:FLAG; example [.::r], [/mnt/nas/music:/music:r:aed")
-    ap.add_argument("-q", action="store_true", help="quiet")
     ap.add_argument("-ed", action="store_true", help="enable ?dots")
     ap.add_argument("-emp", action="store_true", help="enable markdown plugins")
     ap.add_argument("-mcr", metavar="SEC", type=int, default=60, help="md-editor mod-chk rate")
-    ap.add_argument("-nw", action="store_true", help="disable writes (benchmark)")
-    ap.add_argument("-nih", action="store_true", help="no info hostname")
-    ap.add_argument("-nid", action="store_true", help="no info disk-usage")
     ap.add_argument("--dotpart", action="store_true", help="dotfile incomplete uploads")
-    ap.add_argument("--no-zip", action="store_true", help="disable download as zip/tar")
     ap.add_argument("--sparse", metavar="MiB", type=int, default=4, help="up2k min.size threshold (mswin-only)")
-    ap.add_argument("--urlform", metavar="MODE", type=str, default="print,get", help="how to handle url-forms")
+    ap.add_argument("--urlform", metavar="MODE", type=str, default="print,get", help="how to handle url-forms; examples: [stash], [save,get]")
-    ap.add_argument("--salt", type=str, default="hunter2", help="up2k file-hash salt")
+
+    ap2 = ap.add_argument_group('network options')
+    ap2.add_argument("-i", metavar="IP", type=str, default="0.0.0.0", help="ip to bind (comma-sep.)")
+    ap2.add_argument("-p", metavar="PORT", type=str, default="3923", help="ports to bind (comma/range)")
+    ap2.add_argument("--rproxy", metavar="DEPTH", type=int, default=1, help="which ip to keep; 0 = tcp, 1 = origin (first x-fwd), 2 = cloudflare, 3 = nginx, -1 = closest proxy")
+    
+    ap2 = ap.add_argument_group('SSL/TLS options')
+    ap2.add_argument("--http-only", action="store_true", help="disable ssl/tls")
+    ap2.add_argument("--https-only", action="store_true", help="disable plaintext")
+    ap2.add_argument("--ssl-ver", metavar="LIST", type=str, help="set allowed ssl/tls versions; [help] shows available versions; default is what your python version considers safe")
+    ap2.add_argument("--ciphers", metavar="LIST", help="set allowed ssl/tls ciphers; [help] shows available ciphers")
+    ap2.add_argument("--ssl-dbg", action="store_true", help="dump some tls info")
+    ap2.add_argument("--ssl-log", metavar="PATH", help="log master secrets")
+
+    ap2 = ap.add_argument_group('opt-outs')
+    ap2.add_argument("-nw", action="store_true", help="disable writes (benchmark)")
+    ap2.add_argument("-nih", action="store_true", help="no info hostname")
+    ap2.add_argument("-nid", action="store_true", help="no info disk-usage")
+    ap2.add_argument("--no-zip", action="store_true", help="disable download as zip/tar")
+
+    ap2 = ap.add_argument_group('safety options')
+    ap2.add_argument("--ls", metavar="U[,V[,F]]", help="scan all volumes; arguments USER,VOL,FLAGS; example [**,*,ln,p,r]")
+    ap2.add_argument("--salt", type=str, default="hunter2", help="up2k file-hash salt")
+
+    ap2 = ap.add_argument_group('logging options')
+    ap2.add_argument("-q", action="store_true", help="quiet")
+    ap2.add_argument("--log-conn", action="store_true", help="print tcp-server msgs")
+    ap2.add_argument("--ihead", metavar="HEADER", action='append', help="dump incoming header")
+    ap2.add_argument("--lf-url", metavar="RE", type=str, default=r"^/\.cpr/|\?th=[wj]$", help="dont log URLs matching")
 
     ap2 = ap.add_argument_group('admin panel options')
     ap2.add_argument("--no-rescan", action="store_true", help="disable ?scan (volume reindexing)")
@@ -274,6 +301,7 @@ def run_argparse(argv, formatter):
     ap2.add_argument("--th-no-crop", action="store_true", help="dynamic height; show full image")
     ap2.add_argument("--th-no-jpg", action="store_true", help="disable jpg output")
     ap2.add_argument("--th-no-webp", action="store_true", help="disable webp output")
+    ap2.add_argument("--th-ff-jpg", action="store_true", help="force jpg for video thumbs")
     ap2.add_argument("--th-poke", metavar="SEC", type=int, default=300, help="activity labeling cooldown")
     ap2.add_argument("--th-clean", metavar="SEC", type=int, default=43200, help="cleanup interval")
     ap2.add_argument("--th-maxage", metavar="SEC", type=int, default=604800, help="max folder age")
@@ -285,6 +313,8 @@ def run_argparse(argv, formatter):
     ap2.add_argument("-e2t", action="store_true", help="enable metadata indexing")
     ap2.add_argument("-e2ts", action="store_true", help="enable metadata scanner, sets -e2t")
     ap2.add_argument("-e2tsr", action="store_true", help="rescan all metadata, sets -e2ts")
+    ap2.add_argument("--hist", metavar="PATH", type=str, help="where to store volume state")
+    ap2.add_argument("--no-hash", action="store_true", help="disable hashing during e2ds folder scans")
     ap2.add_argument("--no-mutagen", action="store_true", help="use ffprobe for tags instead")
     ap2.add_argument("--no-mtag-mt", action="store_true", help="disable tag-read parallelism")
     ap2.add_argument("-mtm", metavar="M=t,t,t", action="append", type=str, help="add/replace metadata mapping")
@@ -293,22 +323,14 @@ def run_argparse(argv, formatter):
     ap2.add_argument("-mtp", metavar="M=[f,]bin", action="append", type=str, help="read tag M using bin")
     ap2.add_argument("--srch-time", metavar="SEC", type=int, default=30, help="search deadline")
 
-    ap2 = ap.add_argument_group('SSL/TLS options')
+    ap2 = ap.add_argument_group('appearance options')
-    ap2.add_argument("--http-only", action="store_true", help="disable ssl/tls")
+    ap2.add_argument("--css-browser", metavar="L", help="URL to additional CSS to include")
-    ap2.add_argument("--https-only", action="store_true", help="disable plaintext")
-    ap2.add_argument("--ssl-ver", metavar="LIST", type=str, help="ssl/tls versions to allow")
-    ap2.add_argument("--ciphers", metavar="LIST", help="set allowed ciphers")
-    ap2.add_argument("--ssl-dbg", action="store_true", help="dump some tls info")
-    ap2.add_argument("--ssl-log", metavar="PATH", help="log master secrets")
 
     ap2 = ap.add_argument_group('debug options')
-    ap2.add_argument("--ls", metavar="U[,V[,F]]", help="scan all volumes")
-    ap2.add_argument("--log-conn", action="store_true", help="print tcp-server msgs")
     ap2.add_argument("--no-sendfile", action="store_true", help="disable sendfile")
     ap2.add_argument("--no-scandir", action="store_true", help="disable scandir")
     ap2.add_argument("--no-fastboot", action="store_true", help="wait for up2k indexing")
-    ap2.add_argument("--ihead", metavar="HEADER", action='append', help="dump incoming header")
+    ap2.add_argument("--stackmon", metavar="P,S", help="write stacktrace to Path every S second")
-    ap2.add_argument("--lf-url", metavar="RE", type=str, default=r"^/\.cpr/|\?th=[wj]$", help="dont log URLs matching")
     
     return ap.parse_args(args=argv[1:])
     # fmt: on
@@ -348,6 +370,16 @@ def main(argv=None):
     except AssertionError:
         al = run_argparse(argv, Dodge11874)
 
+    if al.stackmon:
+        fp, f = al.stackmon.rsplit(",", 1)
+        f = int(f)
+        t = threading.Thread(
+            target=stackmon,
+            args=(fp, f),
+        )
+        t.daemon = True
+        t.start()
+
     # propagate implications
     for k1, k2 in IMPLICATIONS:
         if getattr(al, k1):

copyparty/__version__.py

@@ -1,8 +1,8 @@
 # coding: utf-8
 
-VERSION = (0, 11, 9)
+VERSION = (0, 11, 16)
 CODENAME = "the grid"
-BUILD_DT = (2021, 6, 7)
+BUILD_DT = (2021, 6, 16)
 
 S_VERSION = ".".join(map(str, VERSION))
 S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT)

copyparty/authsrv.py

@@ -5,6 +5,8 @@ import re
 import os
 import sys
 import stat
+import base64
+import hashlib
 import threading
 
 from .__init__ import WINDOWS
@@ -22,7 +24,15 @@ class VFS(object):
         self.uadm = uadm  # users who are regular admins
         self.flags = flags  # config switches
         self.nodes = {}  # child nodes
-        self.all_vols = {vpath: self}  # flattened recursive
+        self.histtab = None  # all realpath->histpath
+        self.dbv = None  # closest full/non-jump parent
+
+        if realpath:
+            self.histpath = os.path.join(realpath, ".hist")  # db / thumbcache
+            self.all_vols = {vpath: self}  # flattened recursive
+        else:
+            self.histpath = None
+            self.all_vols = None
 
     def __repr__(self):
         return "VFS({})".format(
@@ -32,9 +42,12 @@ class VFS(object):
             )
         )
 
-    def _trk(self, vol):
+    def get_all_vols(self, outdict):
-        self.all_vols[vol.vpath] = vol
+        if self.realpath:
-        return vol
+            outdict[self.vpath] = self
+
+        for v in self.nodes.values():
+            v.get_all_vols(outdict)
 
     def add(self, src, dst):
         """get existing, or add new path to the vfs"""
@@ -46,19 +59,19 @@ class VFS(object):
             name, dst = dst.split("/", 1)
             if name in self.nodes:
                 # exists; do not manipulate permissions
-                return self._trk(self.nodes[name].add(src, dst))
+                return self.nodes[name].add(src, dst)
 
             vn = VFS(
-                "{}/{}".format(self.realpath, name),
+                os.path.join(self.realpath, name) if self.realpath else None,
                 "{}/{}".format(self.vpath, name).lstrip("/"),
                 self.uread,
                 self.uwrite,
                 self.uadm,
-                self.flags,
+                self._copy_flags(name),
             )
-            self._trk(vn)
+            vn.dbv = self.dbv or self
             self.nodes[name] = vn
-            return self._trk(vn.add(src, dst))
+            return vn.add(src, dst)
 
         if dst in self.nodes:
             # leaf exists; return as-is
@@ -67,8 +80,26 @@ class VFS(object):
         # leaf does not exist; create and keep permissions blank
         vp = "{}/{}".format(self.vpath, dst).lstrip("/")
         vn = VFS(src, vp)
+        vn.dbv = self.dbv or self
         self.nodes[dst] = vn
-        return self._trk(vn)
+        return vn
+
+    def _copy_flags(self, name):
+        flags = {k: v for k, v in self.flags.items()}
+        hist = flags.get("hist")
+        if hist and hist != "-":
+            flags["hist"] = "{}/{}".format(hist.rstrip("/"), name)
+
+        return flags
+
+    def bubble_flags(self):
+        if self.dbv:
+            for k, v in self.dbv.flags.items():
+                if k not in ["hist"]:
+                    self.flags[k] = v
+
+        for v in self.nodes.values():
+            v.bubble_flags()
 
     def _find(self, vpath):
         """return [vfs,remainder]"""
@@ -108,6 +139,15 @@ class VFS(object):
 
         return vn, rem
 
+    def get_dbv(self, vrem):
+        dbv = self.dbv
+        if not dbv:
+            return self, vrem
+
+        vrem = [self.vpath[len(dbv.vpath) + 1 :], vrem]
+        vrem = "/".join([x for x in vrem if x])
+        return dbv, vrem
+
     def canonical(self, rem):
         """returns the canonical path (fully-resolved absolute fs path)"""
         rp = self.realpath
@@ -273,7 +313,8 @@ class AuthSrv(object):
         self.reload()
 
     def log(self, msg, c=0):
-        self.log_func("auth", msg, c)
+        if self.log_func:
+            self.log_func("auth", msg, c)
 
     def laggy_iter(self, iterable):
         """returns [value,isFinalValue]"""
@@ -398,6 +439,9 @@ class AuthSrv(object):
                     raise Exception("invalid -v argument: [{}]".format(v_str))
 
                 src, dst, perms = m.groups()
+                if WINDOWS and src.startswith("/"):
+                    src = "{}:\\{}".format(src[1], src[3:])
+
                 # print("\n".join([src, dst, perms]))
                 src = fsdec(os.path.abspath(fsenc(src)))
                 dst = dst.strip("/")
@@ -430,7 +474,7 @@ class AuthSrv(object):
             vfs = VFS(os.path.abspath("."), "", ["*"], ["*"])
         elif "" not in mount:
             # there's volumes but no root; make root inaccessible
-            vfs = VFS(os.path.abspath("."), "")
+            vfs = VFS(None, "")
             vfs.flags["d2d"] = True
 
         maxdepth = 0
@@ -451,6 +495,10 @@ class AuthSrv(object):
             v.uwrite = mwrite[dst]
             v.uadm = madm[dst]
             v.flags = mflags[dst]
+            v.dbv = None
+
+        vfs.all_vols = {}
+        vfs.get_all_vols(vfs.all_vols)
 
         missing_users = {}
         for d in [mread, mwrite]:
@@ -467,6 +515,69 @@ class AuthSrv(object):
             )
             raise Exception("invalid config")
 
+        promote = []
+        demote = []
+        for vol in vfs.all_vols.values():
+            hid = hashlib.sha512(fsenc(vol.realpath)).digest()
+            hid = base64.b32encode(hid).decode("ascii").lower()
+            vflag = vol.flags.get("hist")
+            if vflag == "-":
+                pass
+            elif vflag:
+                if WINDOWS and vflag.startswith("/"):
+                    vflag = "{}:\\{}".format(vflag[1], vflag[3:])
+                vol.histpath = vflag
+            elif self.args.hist:
+                for nch in range(len(hid)):
+                    hpath = os.path.join(self.args.hist, hid[: nch + 1])
+                    try:
+                        os.makedirs(hpath)
+                    except:
+                        pass
+
+                    powner = os.path.join(hpath, "owner.txt")
+                    try:
+                        with open(powner, "rb") as f:
+                            owner = f.read().rstrip()
+                    except:
+                        owner = None
+
+                    me = fsenc(vol.realpath).rstrip()
+                    if owner not in [None, me]:
+                        continue
+
+                    if owner is None:
+                        with open(powner, "wb") as f:
+                            f.write(me)
+
+                    vol.histpath = hpath
+                    break
+
+            vol.histpath = os.path.realpath(vol.histpath)
+            if vol.dbv:
+                if os.path.exists(os.path.join(vol.histpath, "up2k.db")):
+                    promote.append(vol)
+                    vol.dbv = None
+                else:
+                    demote.append(vol)
+
+        # discard jump-vols
+        for v in demote:
+            vfs.all_vols.pop(v.vpath)
+
+        if promote:
+            msg = [
+                "\n  the following jump-volumes were generated to assist the vfs.\n  As they contain a database (probably from v0.11.11 or older),\n  they are promoted to full volumes:"
+            ]
+            for vol in promote:
+                msg.append(
+                    "  /{}  ({})  ({})".format(vol.vpath, vol.realpath, vol.histpath)
+                )
+
+            self.log("\n\n".join(msg) + "\n", c=3)
+
+        vfs.histtab = {v.realpath: v.histpath for v in vfs.all_vols.values()}
+
         all_mte = {}
         errors = False
         for vol in vfs.all_vols.values():
@@ -476,6 +587,10 @@ class AuthSrv(object):
             if self.args.e2d or "e2ds" in vol.flags:
                 vol.flags["e2d"] = True
 
+            if self.args.no_hash:
+                if "ehash" not in vol.flags:
+                    vol.flags["dhash"] = True
+
             for k in ["e2t", "e2ts", "e2tsr"]:
                 if getattr(self.args, k):
                     vol.flags[k] = True
@@ -553,6 +668,8 @@ class AuthSrv(object):
         if errors:
             sys.exit(1)
 
+        vfs.bubble_flags()
+
         try:
             v, _ = vfs.get("/", "*", False, True)
             if self.warn_anonwrite and os.getcwd() == v.realpath:

copyparty/broker_mp.py

@@ -44,7 +44,9 @@ class BrokerMp(object):
             proc.clients = {}
             proc.workload = 0
 
-            thr = threading.Thread(target=self.collector, args=(proc,))
+            thr = threading.Thread(
+                target=self.collector, args=(proc,), name="mp-collector"
+            )
             thr.daemon = True
             thr.start()
 
@@ -52,14 +54,19 @@ class BrokerMp(object):
             proc.start()
 
         if not self.args.q:
-            thr = threading.Thread(target=self.debug_load_balancer)
+            thr = threading.Thread(
+                target=self.debug_load_balancer, name="mp-dbg-loadbalancer"
+            )
             thr.daemon = True
             thr.start()
 
     def shutdown(self):
         self.log("broker", "shutting down")
-        for proc in self.procs:
+        for n, proc in enumerate(self.procs):
-            thr = threading.Thread(target=proc.q_pend.put([0, "shutdown", []]))
+            thr = threading.Thread(
+                target=proc.q_pend.put([0, "shutdown", []]),
+                name="mp-shutdown-{}-{}".format(n, len(self.procs)),
+            )
             thr.start()
 
         with self.mutex:

copyparty/broker_mpw.py

@@ -1,5 +1,6 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
+from copyparty.authsrv import AuthSrv
 
 import sys
 import time
@@ -27,20 +28,23 @@ class MpWorker(object):
         self.retpend = {}
         self.retpend_mutex = threading.Lock()
         self.mutex = threading.Lock()
-        self.workload_thr_active = False
+        self.workload_thr_alive = False
 
         # we inherited signal_handler from parent,
         # replace it with something harmless
         if not FAKE_MP:
             signal.signal(signal.SIGINT, self.signal_handler)
 
+        # starting to look like a good idea
+        self.asrv = AuthSrv(args, None, False)
+
         # instantiate all services here (TODO: inheritance?)
-        self.httpsrv = HttpSrv(self)
+        self.httpsrv = HttpSrv(self, True)
         self.httpsrv.disconnect_func = self.httpdrop
 
         # on winxp and some other platforms,
         # use thr.join() to block all signals
-        thr = threading.Thread(target=self.main)
+        thr = threading.Thread(target=self.main, name="mpw-main")
         thr.daemon = True
         thr.start()
         thr.join()
@@ -79,9 +83,11 @@ class MpWorker(object):
                 self.httpsrv.accept(sck, addr)
 
                 with self.mutex:
-                    if not self.workload_thr_active:
+                    if not self.workload_thr_alive:
                         self.workload_thr_alive = True
-                        thr = threading.Thread(target=self.thr_workload)
+                        thr = threading.Thread(
+                            target=self.thr_workload, name="mpw-workload"
+                        )
                         thr.daemon = True
                         thr.start()
 

copyparty/broker_thr.py

@@ -3,6 +3,7 @@ from __future__ import print_function, unicode_literals
 
 import threading
 
+from .authsrv import AuthSrv
 from .httpsrv import HttpSrv
 from .broker_util import ExceptionalQueue, try_exec
 
@@ -14,6 +15,7 @@ class BrokerThr(object):
         self.hub = hub
         self.log = hub.log
         self.args = hub.args
+        self.asrv = hub.asrv
 
         self.mutex = threading.Lock()
 

copyparty/httpcli.py

@@ -41,7 +41,8 @@ class HttpCli(object):
         self.ip = conn.addr[0]
         self.addr = conn.addr  # type: tuple[str, int]
         self.args = conn.args
-        self.auth = conn.auth  # type: AuthSrv
+        self.is_mp = conn.is_mp
+        self.asrv = conn.asrv  # type: AuthSrv
         self.ico = conn.ico
         self.thumbcli = conn.thumbcli
         self.log_func = conn.log_func
@@ -103,10 +104,21 @@ class HttpCli(object):
         v = self.headers.get("connection", "").lower()
         self.keepalive = not v.startswith("close") and self.http_ver != "HTTP/1.0"
 
-        v = self.headers.get("x-forwarded-for", None)
+        n = self.args.rproxy
-        if v is not None and self.conn.addr[0] in ["127.0.0.1", "::1"]:
+        if n:
-            self.ip = v.split(",")[0]
+            v = self.headers.get("x-forwarded-for")
-            self.log_src = self.conn.set_rproxy(self.ip)
+            if v and self.conn.addr[0] in ["127.0.0.1", "::1"]:
+                if n > 0:
+                    n -= 1
+
+                vs = v.split(",")
+                try:
+                    self.ip = vs[n].strip()
+                except:
+                    self.ip = vs[-1].strip()
+                    self.log("rproxy={} oob x-fwd {}".format(self.args.rproxy, v), c=3)
+
+                self.log_src = self.conn.set_rproxy(self.ip)
 
         if self.args.ihead:
             keys = self.args.ihead
@@ -153,9 +165,9 @@ class HttpCli(object):
         self.vpath = unquotep(vpath)
 
         pwd = uparam.get("pw")
-        self.uname = self.auth.iuser.get(pwd, "*")
+        self.uname = self.asrv.iuser.get(pwd, "*")
         self.rvol, self.wvol, self.avol = [[], [], []]
-        self.auth.vfs.user_tree(self.uname, self.rvol, self.wvol, self.avol)
+        self.asrv.vfs.user_tree(self.uname, self.rvol, self.wvol, self.avol)
 
         ua = self.headers.get("user-agent", "")
         self.is_rclone = ua.startswith("rclone/")
@@ -257,7 +269,14 @@ class HttpCli(object):
         return "?" + "&".join(r)
 
     def redirect(
-        self, vpath, suf="", msg="aight", flavor="go to", click=True, use302=False
+        self,
+        vpath,
+        suf="",
+        msg="aight",
+        flavor="go to",
+        click=True,
+        status=200,
+        use302=False,
     ):
         html = self.j2(
             "msg",
@@ -272,7 +291,7 @@ class HttpCli(object):
             h = {"Location": "/" + vpath, "Cache-Control": "no-cache"}
             self.reply(html, status=302, headers=h)
         else:
-            self.reply(html)
+            self.reply(html, status=status)
 
     def handle_get(self):
         if self.do_log:
@@ -313,9 +332,7 @@ class HttpCli(object):
                     self.redirect(vpath, flavor="redirecting to", use302=True)
                     return True
 
-        self.readable, self.writable = self.conn.auth.vfs.can_access(
+        self.readable, self.writable = self.asrv.vfs.can_access(self.vpath, self.uname)
-            self.vpath, self.uname
-        )
         if not self.readable and not self.writable:
             if self.vpath:
                 self.log("inaccessible: [{}]".format(self.vpath))
@@ -432,7 +449,7 @@ class HttpCli(object):
 
     def dump_to_file(self):
         reader, remains = self.get_body_reader()
-        vfs, rem = self.conn.auth.vfs.get(self.vpath, self.uname, False, True)
+        vfs, rem = self.asrv.vfs.get(self.vpath, self.uname, False, True)
         fdir = os.path.join(vfs.realpath, rem)
 
         addr = self.ip.replace(":", ".")
@@ -442,8 +459,10 @@ class HttpCli(object):
         with open(fsenc(path), "wb", 512 * 1024) as f:
             post_sz, _, sha_b64 = hashcopy(self.conn, reader, f)
 
+        vfs, vrem = vfs.get_dbv(rem)
+
         self.conn.hsrv.broker.put(
-            False, "up2k.hash_file", vfs.realpath, vfs.flags, rem, fn
+            False, "up2k.hash_file", vfs.realpath, vfs.flags, vrem, fn
         )
 
         return post_sz, sha_b64, remains, path
@@ -499,7 +518,7 @@ class HttpCli(object):
         if v is None:
             raise Pebkac(422, "need zip or tar keyword")
 
-        vn, rem = self.auth.vfs.get(self.vpath, self.uname, True, False)
+        vn, rem = self.asrv.vfs.get(self.vpath, self.uname, True, False)
         items = self.parser.require("files", 1024 * 1024)
         if not items:
             raise Pebkac(422, "need files list")
@@ -507,6 +526,7 @@ class HttpCli(object):
         items = items.replace("\r", "").split("\n")
         items = [unquotep(x) for x in items if items]
 
+        self.parser.drop()
         return self.tx_zip(k, v, vn, rem, items, self.args.ed)
 
     def handle_post_json(self):
@@ -548,13 +568,14 @@ class HttpCli(object):
             self.vpath = "/".join([self.vpath, sub]).strip("/")
             body["name"] = name
 
-        vfs, rem = self.conn.auth.vfs.get(self.vpath, self.uname, False, True)
+        vfs, rem = self.asrv.vfs.get(self.vpath, self.uname, False, True)
+        dbv, vrem = vfs.get_dbv(rem)
 
-        body["vtop"] = vfs.vpath
+        body["vtop"] = dbv.vpath
-        body["ptop"] = vfs.realpath
+        body["ptop"] = dbv.realpath
-        body["prel"] = rem
+        body["prel"] = vrem
         body["addr"] = self.ip
-        body["vcfg"] = vfs.flags
+        body["vcfg"] = dbv.flags
 
         if sub:
             try:
@@ -576,8 +597,14 @@ class HttpCli(object):
 
     def handle_search(self, body):
         vols = []
+        seen = {}
         for vtop in self.rvol:
-            vfs, _ = self.conn.auth.vfs.get(vtop, self.uname, True, False)
+            vfs, _ = self.asrv.vfs.get(vtop, self.uname, True, False)
+            vfs = vfs.dbv or vfs
+            if vfs in seen:
+                continue
+
+            seen[vfs] = True
             vols.append([vfs.vpath, vfs.realpath, vfs.flags])
 
         idx = self.conn.get_u2idx()
@@ -633,8 +660,8 @@ class HttpCli(object):
         except KeyError:
             raise Pebkac(400, "need hash and wark headers for binary POST")
 
-        vfs, _ = self.conn.auth.vfs.get(self.vpath, self.uname, False, True)
+        vfs, _ = self.asrv.vfs.get(self.vpath, self.uname, False, True)
-        ptop = vfs.realpath
+        ptop = (vfs.dbv or vfs).realpath
 
         x = self.conn.hsrv.broker.put(True, "up2k.handle_chunk", ptop, wark, chash)
         response = x.get()
@@ -706,7 +733,7 @@ class HttpCli(object):
         pwd = self.parser.require("cppwd", 64)
         self.parser.drop()
 
-        if pwd in self.auth.iuser:
+        if pwd in self.asrv.iuser:
             msg = "login ok"
             dt = datetime.utcfromtimestamp(time.time() + 60 * 60 * 24 * 365)
             exp = dt.strftime("%a, %d %b %Y %H:%M:%S GMT")
@@ -725,7 +752,7 @@ class HttpCli(object):
         self.parser.drop()
 
         nullwrite = self.args.nw
-        vfs, rem = self.conn.auth.vfs.get(self.vpath, self.uname, False, True)
+        vfs, rem = self.asrv.vfs.get(self.vpath, self.uname, False, True)
         self._assert_safe_rem(rem)
 
         sanitized = sanitize_fn(new_dir)
@@ -754,7 +781,7 @@ class HttpCli(object):
         self.parser.drop()
 
         nullwrite = self.args.nw
-        vfs, rem = self.conn.auth.vfs.get(self.vpath, self.uname, False, True)
+        vfs, rem = self.asrv.vfs.get(self.vpath, self.uname, False, True)
         self._assert_safe_rem(rem)
 
         if not new_file.endswith(".md"):
@@ -778,7 +805,7 @@ class HttpCli(object):
 
     def handle_plain_upload(self):
         nullwrite = self.args.nw
-        vfs, rem = self.conn.auth.vfs.get(self.vpath, self.uname, False, True)
+        vfs, rem = self.asrv.vfs.get(self.vpath, self.uname, False, True)
         self._assert_safe_rem(rem)
 
         files = []
@@ -815,8 +842,14 @@ class HttpCli(object):
                             raise Pebkac(400, "empty files in post")
 
                         files.append([sz, sha512_hex, p_file, fname])
+                        dbv, vrem = vfs.get_dbv(rem)
                         self.conn.hsrv.broker.put(
-                            False, "up2k.hash_file", vfs.realpath, vfs.flags, rem, fname
+                            False,
+                            "up2k.hash_file",
+                            dbv.realpath,
+                            dbv.flags,
+                            vrem,
+                            fname,
                         )
                         self.conn.nbyte += sz
 
@@ -846,12 +879,16 @@ class HttpCli(object):
         status = "OK"
         if errmsg:
             self.log(errmsg)
-            errmsg = "ERROR: " + errmsg
             status = "ERROR"
 
         msg = "{} // {} bytes // {:.3f} MiB/s\n".format(status, sz_total, spd)
         jmsg = {"status": status, "sz": sz_total, "mbps": round(spd, 3), "files": []}
 
+        if errmsg:
+            msg += errmsg + "\n"
+            jmsg["error"] = errmsg
+            errmsg = "ERROR: " + errmsg
+
         for sz, sha512, ofn, lfn in files:
             vpath = (self.vpath + "/" if self.vpath else "") + lfn
             msg += 'sha512: {} // {} bytes // <a href="/{}">{}</a>\n'.format(
@@ -883,11 +920,21 @@ class HttpCli(object):
                 ft = "{}\n{}\n{}\n".format(ft, msg.rstrip(), errmsg)
                 f.write(ft.encode("utf-8"))
 
+        status = 400 if errmsg else 200
         if "j" in self.uparam:
-            jtxt = json.dumps(jmsg, indent=2, sort_keys=True)
+            jtxt = json.dumps(jmsg, indent=2, sort_keys=True).encode("utf-8", "replace")
-            self.reply(jtxt.encode("utf-8", "replace"), mime="application/json")
+            self.reply(jtxt, mime="application/json", status=status)
         else:
-            self.redirect(self.vpath, msg=msg, flavor="return to", click=False)
+            self.redirect(
+                self.vpath,
+                msg=msg,
+                flavor="return to",
+                click=False,
+                status=status,
+            )
+
+        if errmsg:
+            return False
 
         self.parser.drop()
         return True
@@ -899,7 +946,7 @@ class HttpCli(object):
             raise Pebkac(400, "could not read lastmod from request")
 
         nullwrite = self.args.nw
-        vfs, rem = self.conn.auth.vfs.get(self.vpath, self.uname, False, True)
+        vfs, rem = self.asrv.vfs.get(self.vpath, self.uname, False, True)
         self._assert_safe_rem(rem)
 
         # TODO:
@@ -992,6 +1039,8 @@ class HttpCli(object):
         cli_lastmod = self.headers.get("if-modified-since")
         if cli_lastmod:
             try:
+                # some browser append "; length=573"
+                cli_lastmod = cli_lastmod.split(";")[0].strip()
                 cli_dt = time.strptime(cli_lastmod, HTTP_TS_FMT)
                 cli_ts = calendar.timegm(cli_dt)
                 return file_lastmod, int(file_ts) > int(cli_ts)
@@ -1161,7 +1210,8 @@ class HttpCli(object):
             if use_sendfile:
                 remains = sendfile_kern(lower, upper, f, self.s)
             else:
-                remains = sendfile_py(lower, upper, f, self.s)
+                actor = self.conn if self.is_mp else None
+                remains = sendfile_py(lower, upper, f, self.s, actor)
 
         if remains > 0:
             logmsg += " \033[31m" + unicode(upper - remains) + "\033[0m"
@@ -1334,11 +1384,13 @@ class HttpCli(object):
             for y in [self.rvol, self.wvol, self.avol]
         ]
 
-        vstate = {}
         if self.avol and not self.args.no_rescan:
-            x = self.conn.hsrv.broker.put(True, "up2k.get_volstate")
+            x = self.conn.hsrv.broker.put(True, "up2k.get_state")
-            vstate = json.loads(x.get())
+            vs = json.loads(x.get())
-            vstate = {("/" + k).rstrip("/") + "/": v for k, v in vstate.items()}
+            vstate = {("/" + k).rstrip("/") + "/": v for k, v in vs["volstate"].items()}
+        else:
+            vstate = {}
+            vs = {"scanning": None, "hashq": None, "tagq": None, "mtpq": None}
 
         html = self.j2(
             "splash",
@@ -1347,6 +1399,10 @@ class HttpCli(object):
             wvol=wvol,
             avol=avol,
             vstate=vstate,
+            scanning=vs["scanning"],
+            hashq=vs["hashq"],
+            tagq=vs["tagq"],
+            mtpq=vs["mtpq"],
             url_suf=suf,
         )
         self.reply(html.encode("utf-8"), headers=NO_STORE)
@@ -1359,9 +1415,10 @@ class HttpCli(object):
         if self.args.no_rescan:
             raise Pebkac(403, "disabled by argv")
 
-        vn, _ = self.auth.vfs.get(self.vpath, self.uname, True, True)
+        vn, _ = self.asrv.vfs.get(self.vpath, self.uname, True, True)
+
+        args = [self.asrv.vfs.all_vols, [vn.vpath]]
 
-        args = [self.auth.vfs.all_vols, [vn.vpath]]
         x = self.conn.hsrv.broker.put(True, "up2k.rescan", *args)
         x = x.get()
         if not x:
@@ -1377,17 +1434,8 @@ class HttpCli(object):
         if self.args.no_stack:
             raise Pebkac(403, "disabled by argv")
 
-        ret = []
+        ret = "<pre>{}\n{}".format(time.time(), alltrace())
-        names = dict([(t.ident, t.name) for t in threading.enumerate()])
+        self.reply(ret.encode("utf-8"))
-        for tid, stack in sys._current_frames().items():
-            ret.append("\n\n# {} ({:x})".format(names.get(tid), tid))
-            for fn, lno, name, line in traceback.extract_stack(stack):
-                ret.append('File: "{}", line {}, in {}'.format(fn, lno, name))
-                if line:
-                    ret.append("  " + str(line.strip()))
-
-        ret = ("<pre>" + "\n".join(ret)).encode("utf-8")
-        self.reply(ret)
 
     def tx_tree(self):
         top = self.uparam["tree"] or ""
@@ -1417,7 +1465,7 @@ class HttpCli(object):
             ret["k" + quotep(excl)] = sub
 
         try:
-            vn, rem = self.auth.vfs.get(top, self.uname, True, False)
+            vn, rem = self.asrv.vfs.get(top, self.uname, True, False)
             fsroot, vfs_ls, vfs_virt = vn.ls(
                 rem, self.uname, not self.args.no_scandir, incl_wo=True
             )
@@ -1458,35 +1506,51 @@ class HttpCli(object):
 
                 vpnodes.append([quotep(vpath) + "/", html_escape(node, crlf=True)])
 
-        vn, rem = self.auth.vfs.get(
+        vn, rem = self.asrv.vfs.get(
             self.vpath, self.uname, self.readable, self.writable
         )
         abspath = vn.canonical(rem)
+        dbv, vrem = vn.get_dbv(rem)
 
         try:
             st = os.stat(fsenc(abspath))
         except:
             raise Pebkac(404)
 
-        if self.readable and not stat.S_ISDIR(st.st_mode):
+        if self.readable:
             if rem.startswith(".hist/up2k."):
                 raise Pebkac(403)
 
+            is_dir = stat.S_ISDIR(st.st_mode)
             th_fmt = self.uparam.get("th")
             if th_fmt is not None:
+                if is_dir:
+                    for fn in ["folder.png", "folder.jpg"]:
+                        fp = os.path.join(abspath, fn)
+                        if os.path.exists(fp):
+                            vrem = "{}/{}".format(vrem.rstrip("/"), fn)
+                            is_dir = False
+                            break
+
+                    if is_dir:
+                        return self.tx_ico("a.folder")
+
                 thp = None
                 if self.thumbcli:
-                    thp = self.thumbcli.get(vn.realpath, rem, int(st.st_mtime), th_fmt)
+                    thp = self.thumbcli.get(
+                        dbv.realpath, vrem, int(st.st_mtime), th_fmt
+                    )
 
                 if thp:
                     return self.tx_file(thp)
 
                 return self.tx_ico(rem)
 
-            if abspath.endswith(".md") and "raw" not in self.uparam:
+            if not is_dir:
-                return self.tx_md(abspath)
+                if abspath.endswith(".md") and "raw" not in self.uparam:
+                    return self.tx_md(abspath)
 
-            return self.tx_file(abspath)
+                return self.tx_file(abspath)
 
         srv_info = []
 
@@ -1619,7 +1683,7 @@ class HttpCli(object):
         icur = None
         if "e2t" in vn.flags:
             idx = self.conn.get_u2idx()
-            icur = idx.get_cur(vn.realpath)
+            icur = idx.get_cur(dbv.realpath)
 
         dirs = []
         files = []
@@ -1687,6 +1751,9 @@ class HttpCli(object):
             rd = f["rd"]
             del f["rd"]
             if icur:
+                if vn != dbv:
+                    _, rd = vn.get_dbv(rd)
+
                 q = "select w from up where rd = ? and fn = ?"
                 try:
                     r = icur.execute(q, (rd, fn)).fetchone()
@@ -1727,9 +1794,13 @@ class HttpCli(object):
         j2a["files"] = dirs + files
         j2a["logues"] = logues
         j2a["taglist"] = taglist
+
         if "mte" in vn.flags:
             j2a["tag_order"] = json.dumps(vn.flags["mte"].split(","))
 
+        if self.args.css_browser:
+            j2a["css"] = self.args.css_browser
+
         html = self.j2(tpl, **j2a)
         self.reply(html.encode("utf-8", "replace"), headers=NO_STORE)
         return True

copyparty/httpconn.py

@@ -34,7 +34,8 @@ class HttpConn(object):
         self.hsrv = hsrv
 
         self.args = hsrv.args
-        self.auth = hsrv.auth
+        self.asrv = hsrv.asrv
+        self.is_mp = hsrv.is_mp
         self.cert_path = hsrv.cert_path
 
         enth = HAVE_PIL and not self.args.no_thumb
@@ -70,7 +71,7 @@ class HttpConn(object):
 
     def get_u2idx(self):
         if not self.u2idx:
-            self.u2idx = U2idx(self.args, self.log_func)
+            self.u2idx = U2idx(self)
 
         return self.u2idx
 
@@ -174,6 +175,11 @@ class HttpConn(object):
             self.sr = Unrecv(self.s)
 
         while True:
+            if self.is_mp:
+                self.workload += 50
+                if self.workload >= 2 ** 31:
+                    self.workload = 100
+
             cli = HttpCli(self)
             if not cli.run():
                 return

copyparty/httpsrv.py

@@ -35,10 +35,12 @@ class HttpSrv(object):
     relying on MpSrv for performance (HttpSrv is just plain threads)
     """
 
-    def __init__(self, broker):
+    def __init__(self, broker, is_mp=False):
         self.broker = broker
+        self.is_mp = is_mp
         self.args = broker.args
         self.log = broker.log
+        self.asrv = broker.asrv
 
         self.disconnect_func = None
         self.mutex = threading.Lock()
@@ -46,7 +48,6 @@ class HttpSrv(object):
         self.clients = {}
         self.workload = 0
         self.workload_thr_alive = False
-        self.auth = AuthSrv(self.args, self.log)
 
         env = jinja2.Environment()
         env.loader = jinja2.FileSystemLoader(os.path.join(E.mod, "web"))
@@ -66,7 +67,11 @@ class HttpSrv(object):
         if self.args.log_conn:
             self.log("%s %s" % addr, "|%sC-cthr" % ("-" * 5,), c="1;30")
 
-        thr = threading.Thread(target=self.thr_client, args=(sck, addr))
+        thr = threading.Thread(
+            target=self.thr_client,
+            args=(sck, addr),
+            name="httpsrv-{}-{}".format(addr[0].split(".", 2)[-1][-6:], addr[1]),
+        )
         thr.daemon = True
         thr.start()
 
@@ -84,13 +89,16 @@ class HttpSrv(object):
         cli = HttpConn(sck, addr, self)
         with self.mutex:
             self.clients[cli] = 0
-            self.workload += 50
 
-            if not self.workload_thr_alive:
+            if self.is_mp:
-                self.workload_thr_alive = True
+                self.workload += 50
-                thr = threading.Thread(target=self.thr_workload)
+                if not self.workload_thr_alive:
-                thr.daemon = True
+                    self.workload_thr_alive = True
-                thr.start()
+                    thr = threading.Thread(
+                        target=self.thr_workload, name="httpsrv-workload"
+                    )
+                    thr.daemon = True
+                    thr.start()
 
         try:
             if self.args.log_conn:
@@ -99,6 +107,7 @@ class HttpSrv(object):
             cli.run()
 
         finally:
+            sck = cli.s
             if self.args.log_conn:
                 self.log("%s %s" % addr, "|%sC-cdone" % ("-" * 7,), c="1;30")
 

copyparty/ico.py

@@ -1,3 +1,6 @@
+# coding: utf-8
+from __future__ import print_function, unicode_literals
+
 import hashlib
 import colorsys
 

copyparty/mtag.py

@@ -1,7 +1,6 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
-import re
 import os
 import sys
 import json

copyparty/star.py

@@ -1,3 +1,6 @@
+# coding: utf-8
+from __future__ import print_function, unicode_literals
+
 import os
 import tarfile
 import threading
@@ -42,7 +45,7 @@ class StreamTar(object):
         fmt = tarfile.GNU_FORMAT
         self.tar = tarfile.open(fileobj=self.qfile, mode="w|", format=fmt)
 
-        w = threading.Thread(target=self._gen)
+        w = threading.Thread(target=self._gen, name="star-gen")
         w.daemon = True
         w.start()
 

copyparty/sutil.py

@@ -1,3 +1,6 @@
+# coding: utf-8
+from __future__ import print_function, unicode_literals
+
 import os
 import time
 import tempfile

copyparty/svchub.py

@@ -37,14 +37,13 @@ class SvcHub(object):
 
         self.log = self._log_disabled if args.q else self._log_enabled
 
-        # jank goes here
-        auth = AuthSrv(self.args, self.log, False)
-        if args.ls:
-            auth.dbg_ls()
-
         # initiate all services to manage
+        self.asrv = AuthSrv(self.args, self.log, False)
+        if args.ls:
+            self.asrv.dbg_ls()
+
         self.tcpsrv = TcpSrv(self)
-        self.up2k = Up2k(self, auth.vfs.all_vols)
+        self.up2k = Up2k(self)
 
         self.thumbsrv = None
         if not args.no_thumb:
@@ -54,7 +53,7 @@ class SvcHub(object):
                     msg = "setting --th-no-webp because either libwebp is not available or your Pillow is too old"
                     self.log("thumb", msg, c=3)
 
-                self.thumbsrv = ThumbSrv(self, auth.vfs.all_vols)
+                self.thumbsrv = ThumbSrv(self)
             else:
                 msg = "need Pillow to create thumbnails; for example:\n{}{} -m pip install --user Pillow\n"
                 self.log(
@@ -71,7 +70,7 @@ class SvcHub(object):
         self.broker = Broker(self)
 
     def run(self):
-        thr = threading.Thread(target=self.tcpsrv.run)
+        thr = threading.Thread(target=self.tcpsrv.run, name="svchub-main")
         thr.daemon = True
         thr.start()
 

copyparty/szip.py

@@ -1,3 +1,6 @@
+# coding: utf-8
+from __future__ import print_function, unicode_literals
+
 import os
 import time
 import zlib

copyparty/th_cli.py

@@ -1,5 +1,7 @@
+# coding: utf-8
+from __future__ import print_function, unicode_literals
+
 import os
-import time
 
 from .util import Cooldown
 from .th_srv import thumb_path, THUMBABLE, FMT_FF
@@ -9,6 +11,7 @@ class ThumbCli(object):
     def __init__(self, broker):
         self.broker = broker
         self.args = broker.args
+        self.asrv = broker.asrv
 
         # cache on both sides for less broker spam
         self.cooldown = Cooldown(self.args.th_poke)
@@ -18,16 +21,19 @@ class ThumbCli(object):
         if ext not in THUMBABLE:
             return None
 
-        if self.args.no_vthumb and ext in FMT_FF:
+        is_vid = ext in FMT_FF
+        if is_vid and self.args.no_vthumb:
             return None
 
         if fmt == "j" and self.args.th_no_jpg:
             fmt = "w"
 
-        if fmt == "w" and self.args.th_no_webp:
+        if fmt == "w":
-            fmt = "j"
+            if self.args.th_no_webp or (is_vid and self.args.th_ff_jpg):
+                fmt = "j"
 
-        tpath = thumb_path(ptop, rem, mtime, fmt)
+        histpath = self.asrv.vfs.histtab[ptop]
+        tpath = thumb_path(histpath, rem, mtime, fmt)
         ret = None
         try:
             st = os.stat(tpath)

copyparty/th_srv.py

@@ -1,5 +1,7 @@
+# coding: utf-8
+from __future__ import print_function, unicode_literals
+
 import os
-import sys
 import time
 import shutil
 import base64
@@ -8,7 +10,7 @@ import threading
 import subprocess as sp
 
 from .__init__ import PY2
-from .util import fsenc, mchkcmd, Queue, Cooldown, BytesIO
+from .util import fsenc, runcmd, Queue, Cooldown, BytesIO, min_ex
 from .mtag import HAVE_FFMPEG, HAVE_FFPROBE, ffprobe
 
 
@@ -51,7 +53,7 @@ except:
 # https://pillow.readthedocs.io/en/stable/handbook/image-file-formats.html
 # ffmpeg -formats
 FMT_PIL = "bmp dib gif icns ico jpg jpeg jp2 jpx pcx png pbm pgm ppm pnm sgi tga tif tiff webp xbm dds xpm"
-FMT_FF = "av1 asf avi flv m4v mkv mjpeg mjpg mpg mpeg mpg2 mpeg2 mov 3gp mp4 ts mpegts nut ogv ogm rm vob webm wmv"
+FMT_FF = "av1 asf avi flv m4v mkv mjpeg mjpg mpg mpeg mpg2 mpeg2 h264 avc h265 hevc mov 3gp mp4 ts mpegts nut ogv ogm rm vob webm wmv"
 
 if HAVE_HEIF:
     FMT_PIL += " heif heifs heic heics"
@@ -71,7 +73,7 @@ if HAVE_FFMPEG and HAVE_FFPROBE:
     THUMBABLE.update(FMT_FF)
 
 
-def thumb_path(ptop, rem, mtime, fmt):
+def thumb_path(histpath, rem, mtime, fmt):
     # base16 = 16 = 256
     # b64-lc = 38 = 1444
     # base64 = 64 = 4096
@@ -92,16 +94,15 @@ def thumb_path(ptop, rem, mtime, fmt):
     h = hashlib.sha512(fsenc(fn)).digest()[:24]
     fn = base64.urlsafe_b64encode(h).decode("ascii")[:24]
 
-    return "{}/.hist/th/{}/{}.{:x}.{}".format(
+    return "{}/th/{}/{}.{:x}.{}".format(
-        ptop, rd, fn, int(mtime), "webp" if fmt == "w" else "jpg"
+        histpath, rd, fn, int(mtime), "webp" if fmt == "w" else "jpg"
     )
 
 
 class ThumbSrv(object):
-    def __init__(self, hub, vols):
+    def __init__(self, hub):
         self.hub = hub
-        self.vols = [v.realpath for v in vols.values()]
+        self.asrv = hub.asrv
-
         self.args = hub.args
         self.log_func = hub.log
 
@@ -114,8 +115,10 @@ class ThumbSrv(object):
         self.stopping = False
         self.nthr = os.cpu_count() if hasattr(os, "cpu_count") else 4
         self.q = Queue(self.nthr * 4)
-        for _ in range(self.nthr):
+        for n in range(self.nthr):
-            t = threading.Thread(target=self.worker)
+            t = threading.Thread(
+                target=self.worker, name="thumb-{}-{}".format(n, self.nthr)
+            )
             t.daemon = True
             t.start()
 
@@ -131,7 +134,7 @@ class ThumbSrv(object):
             msg += ", ".join(missing)
             self.log(msg, c=3)
 
-        t = threading.Thread(target=self.cleaner)
+        t = threading.Thread(target=self.cleaner, name="thumb-cleaner")
         t.daemon = True
         t.start()
 
@@ -148,9 +151,11 @@ class ThumbSrv(object):
             return not self.nthr
 
     def get(self, ptop, rem, mtime, fmt):
-        tpath = thumb_path(ptop, rem, mtime, fmt)
+        histpath = self.asrv.vfs.histtab[ptop]
+        tpath = thumb_path(histpath, rem, mtime, fmt)
         abspath = os.path.join(ptop, rem)
-        cond = threading.Condition()
+        cond = threading.Condition(self.mutex)
+        do_conv = False
         with self.mutex:
             try:
                 self.busy[tpath].append(cond)
@@ -168,8 +173,11 @@ class ThumbSrv(object):
                         f.write(fsenc(os.path.dirname(abspath)))
 
                 self.busy[tpath] = [cond]
-                self.q.put([abspath, tpath])
+                do_conv = True
-                self.log("conv {} \033[0m{}".format(tpath, abspath), c=6)
+
+        if do_conv:
+            self.q.put([abspath, tpath])
+            self.log("conv {} \033[0m{}".format(tpath, abspath), c=6)
 
         while not self.stopping:
             with self.mutex:
@@ -177,7 +185,7 @@ class ThumbSrv(object):
                     break
 
             with cond:
-                cond.wait()
+                cond.wait(3)
 
         try:
             st = os.stat(tpath)
@@ -206,9 +214,9 @@ class ThumbSrv(object):
             if fun:
                 try:
                     fun(abspath, tpath)
-                except Exception as ex:
+                except:
-                    msg = "{} failed on {}\n  {!r}"
+                    msg = "{} failed on {}\n{}"
-                    self.log(msg.format(fun.__name__, abspath, ex), 3)
+                    self.log(msg.format(fun.__name__, abspath, min_ex()), 3)
                     with open(tpath, "wb") as _:
                         pass
 
@@ -240,8 +248,8 @@ class ThumbSrv(object):
             except:
                 im.thumbnail(self.res)
 
-            if im.mode not in ("RGB", "L"):
+            fmts = ["RGB", "L"]
-                im = im.convert("RGB")
+            args = {"quality": 40}
 
             if tpath.endswith(".webp"):
                 # quality 80 = pillow-default
@@ -249,15 +257,27 @@ class ThumbSrv(object):
                 # method 0 = pillow-default, fast
                 # method 4 = ffmpeg-default
                 # method 6 = max, slow
-                im.save(tpath, quality=40, method=6)
+                fmts += ["RGBA", "LA"]
+                args["method"] = 6
             else:
-                im.save(tpath, quality=40)  # default=75
+                pass  # default q = 75
+
+            if im.mode not in fmts:
+                print("conv {}".format(im.mode))
+                im = im.convert("RGB")
+
+            im.save(tpath, quality=40, method=6)
 
     def conv_ffmpeg(self, abspath, tpath):
         ret, _ = ffprobe(abspath)
 
-        dur = ret[".dur"][1] if ".dur" in ret else 4
+        ext = abspath.rsplit(".")[-1]
-        seek = "{:.0f}".format(dur / 3)
+        if ext in ["h264", "h265"]:
+            seek = []
+        else:
+            dur = ret[".dur"][1] if ".dur" in ret else 4
+            seek = "{:.0f}".format(dur / 3)
+            seek = [b"-ss", seek.encode("utf-8")]
 
         scale = "scale={0}:{1}:force_original_aspect_ratio="
         if self.args.th_no_crop:
@@ -266,19 +286,20 @@ class ThumbSrv(object):
             scale += "increase,crop={0}:{1},setsar=1:1"
 
         scale = scale.format(*list(self.res)).encode("utf-8")
+        # fmt: off
         cmd = [
             b"ffmpeg",
             b"-nostdin",
-            b"-hide_banner",
+            b"-v", b"error",
-            b"-ss",
+            b"-hide_banner"
-            seek,
-            b"-i",
-            fsenc(abspath),
-            b"-vf",
-            scale,
-            b"-vframes",
-            b"1",
         ]
+        cmd += seek
+        cmd += [
+            b"-i", fsenc(abspath),
+            b"-vf", scale,
+            b"-vframes", b"1",
+        ]
+        # fmt: on
 
         if tpath.endswith(".jpg"):
             cmd += [
@@ -295,7 +316,11 @@ class ThumbSrv(object):
 
         cmd += [fsenc(tpath)]
 
-        mchkcmd(cmd)
+        ret, sout, serr = runcmd(*cmd)
+        if ret != 0:
+            msg = ["ff: {}".format(x) for x in serr.split("\n")]
+            self.log("FFmpeg failed:\n" + "\n".join(msg), c="1;30")
+            raise sp.CalledProcessError(ret, (cmd[0], b"...", cmd[-1]))
 
     def poke(self, tdir):
         if not self.poke_cd.poke(tdir):
@@ -314,26 +339,29 @@ class ThumbSrv(object):
         interval = self.args.th_clean
         while True:
             time.sleep(interval)
-            for vol in self.vols:
+            for vol, histpath in self.asrv.vfs.histtab.items():
-                vol += "/.hist/th"
+                if histpath.startswith(vol):
-                self.log("\033[Jcln {}/\033[A".format(vol))
+                    self.log("\033[Jcln {}/\033[A".format(histpath))
-                self.clean(vol)
+                else:
+                    self.log("\033[Jcln {} ({})/\033[A".format(histpath, vol))
+
+                self.clean(histpath)
 
             self.log("\033[Jcln ok")
 
-    def clean(self, vol):
+    def clean(self, histpath):
-        # self.log("cln {}".format(vol))
+        # self.log("cln {}".format(histpath))
         maxage = self.args.th_maxage
         now = time.time()
         prev_b64 = None
         prev_fp = None
         try:
-            ents = os.listdir(vol)
+            ents = os.listdir(histpath)
         except:
             return
 
         for f in sorted(ents):
-            fp = os.path.join(vol, f)
+            fp = os.path.join(histpath, f)
             cmp = fp.lower().replace("\\", "/")
 
             # "top" or b64 prefix/full (a folder)

copyparty/u2idx.py

@@ -7,7 +7,7 @@ import time
 import threading
 from datetime import datetime
 
-from .util import u8safe, s3dec, html_escape, Pebkac
+from .util import s3dec, Pebkac, min_ex
 from .up2k import up2k_wark_from_hashlist
 
 
@@ -19,10 +19,11 @@ except:
 
 
 class U2idx(object):
-    def __init__(self, args, log_func):
+    def __init__(self, conn):
-        self.args = args
+        self.log_func = conn.log_func
-        self.log_func = log_func
+        self.asrv = conn.asrv
-        self.timeout = args.srch_time
+        self.args = conn.args
+        self.timeout = self.args.srch_time
 
         if not HAVE_SQLITE3:
             self.log("could not load sqlite3; searchign wqill be disabled")
@@ -52,18 +53,20 @@ class U2idx(object):
 
         try:
             return self.run_query(vols, uq, uv)[0]
-        except Exception as ex:
+        except:
-            raise Pebkac(500, repr(ex))
+            raise Pebkac(500, min_ex())
 
     def get_cur(self, ptop):
         cur = self.cur.get(ptop)
         if cur:
             return cur
 
-        cur = _open(ptop)
+        histpath = self.asrv.vfs.histtab[ptop]
-        if not cur:
+        db_path = os.path.join(histpath, "up2k.db")
+        if not os.path.exists(db_path):
             return None
 
+        cur = sqlite3.connect(db_path).cursor()
         self.cur[ptop] = cur
         return cur
 
@@ -192,6 +195,7 @@ class U2idx(object):
                 self.active_id,
                 done_flag,
             ),
+            name="u2idx-terminator",
         )
         thr.daemon = True
         thr.start()
@@ -241,6 +245,7 @@ class U2idx(object):
                 hit["tags"] = tags
 
             ret.extend(sret)
+            # print("[{}] {}".format(ptop, sret))
 
         done_flag.append(True)
         self.active_id = None
@@ -261,9 +266,3 @@ class U2idx(object):
 
         if identifier == self.active_id:
             self.active_cur.connection.interrupt()
-
-
-def _open(ptop):
-    db_path = os.path.join(ptop, ".hist", "up2k.db")
-    if os.path.exists(db_path):
-        return sqlite3.connect(db_path).cursor()

copyparty/up2k.py

@@ -48,8 +48,9 @@ class Up2k(object):
         * ~/.config flatfiles for active jobs
     """
 
-    def __init__(self, hub, all_vols):
+    def __init__(self, hub):
         self.hub = hub
+        self.asrv = hub.asrv
         self.args = hub.args
         self.log_func = hub.log
 
@@ -60,6 +61,8 @@ class Up2k(object):
         self.mutex = threading.Lock()
         self.hashq = Queue()
         self.tagq = Queue()
+        self.n_hashq = 0
+        self.n_tagq = 0
         self.volstate = {}
         self.registry = {}
         self.entags = {}
@@ -83,7 +86,7 @@ class Up2k(object):
         if ANYWIN:
             # usually fails to set lastmod too quickly
             self.lastmod_q = Queue()
-            thr = threading.Thread(target=self._lastmodder)
+            thr = threading.Thread(target=self._lastmodder, name="up2k-lastmod")
             thr.daemon = True
             thr.start()
 
@@ -94,45 +97,73 @@ class Up2k(object):
             self.log("could not initialize sqlite3, will use in-memory registry only")
 
         if self.args.no_fastboot:
-            self.deferred_init(all_vols)
+            self.deferred_init()
         else:
-            t = threading.Thread(target=self.deferred_init, args=(all_vols,))
+            t = threading.Thread(
+                target=self.deferred_init,
+                name="up2k-deferred-init",
+            )
             t.daemon = True
             t.start()
 
-    def deferred_init(self, all_vols):
+    def deferred_init(self):
+        all_vols = self.asrv.vfs.all_vols
         have_e2d = self.init_indexes(all_vols)
 
         if have_e2d:
-            thr = threading.Thread(target=self._snapshot)
+            thr = threading.Thread(target=self._snapshot, name="up2k-snapshot")
             thr.daemon = True
             thr.start()
 
-            thr = threading.Thread(target=self._hasher)
+            thr = threading.Thread(target=self._hasher, name="up2k-hasher")
             thr.daemon = True
             thr.start()
 
             if self.mtag:
-                thr = threading.Thread(target=self._tagger)
+                thr = threading.Thread(target=self._tagger, name="up2k-tagger")
                 thr.daemon = True
                 thr.start()
 
-                thr = threading.Thread(target=self._run_all_mtp)
+                thr = threading.Thread(target=self._run_all_mtp, name="up2k-mtp-init")
                 thr.daemon = True
                 thr.start()
 
     def log(self, msg, c=0):
         self.log_func("up2k", msg + "\033[K", c)
 
-    def get_volstate(self):
+    def get_state(self):
-        return json.dumps(self.volstate, indent=4)
+        mtpq = 0
+        q = "select count(w) from mt where k = 't:mtp'"
+        got_lock = self.mutex.acquire(timeout=0.5)
+        if got_lock:
+            for cur in self.cur.values():
+                try:
+                    mtpq += cur.execute(q).fetchone()[0]
+                except:
+                    pass
+            self.mutex.release()
+        else:
+            mtpq = "?"
+
+        ret = {
+            "volstate": self.volstate,
+            "scanning": hasattr(self, "pp"),
+            "hashq": self.n_hashq,
+            "tagq": self.n_tagq,
+            "mtpq": mtpq,
+        }
+        return json.dumps(ret, indent=4)
 
     def rescan(self, all_vols, scan_vols):
         if hasattr(self, "pp"):
             return "cannot initiate; scan is already in progress"
 
         args = (all_vols, scan_vols)
-        t = threading.Thread(target=self.init_indexes, args=args)
+        t = threading.Thread(
+            target=self.init_indexes,
+            args=args,
+            name="up2k-rescan-{}".format(scan_vols[0]),
+        )
         t.daemon = True
         t.start()
         return None
@@ -178,23 +209,27 @@ class Up2k(object):
                 self.log(msg, c=3)
 
         live_vols = []
-        for vol in vols:
+        with self.mutex:
-            try:
+            # only need to protect register_vpath but all in one go feels right
-                os.listdir(vol.realpath)
+            for vol in vols:
-            except:
+                try:
-                self.volstate[vol.vpath] = "OFFLINE (cannot access folder)"
+                    os.listdir(vol.realpath)
-                self.log("cannot access " + vol.realpath, c=1)
+                except:
-                continue
+                    self.volstate[vol.vpath] = "OFFLINE (cannot access folder)"
+                    self.log("cannot access " + vol.realpath, c=1)
+                    continue
 
-            if not self.register_vpath(vol.realpath, vol.flags):
+                if scan_vols and vol.vpath not in scan_vols:
-                # self.log("db not enabled for {}".format(m, vol.realpath))
+                    continue
-                continue
+
+                if not self.register_vpath(vol.realpath, vol.flags):
+                    # self.log("db not enable for {}".format(m, vol.realpath))
+                    continue
 
-            if vol.vpath in scan_vols or not scan_vols:
                 live_vols.append(vol)
 
-            if vol.vpath not in self.volstate:
+                if vol.vpath not in self.volstate:
-                self.volstate[vol.vpath] = "OFFLINE (pending initialization)"
+                    self.volstate[vol.vpath] = "OFFLINE (pending initialization)"
 
         vols = live_vols
         need_vac = {}
@@ -271,7 +306,7 @@ class Up2k(object):
         if self.mtag:
             m = "online (running mtp)"
             if scan_vols:
-                thr = threading.Thread(target=self._run_all_mtp)
+                thr = threading.Thread(target=self._run_all_mtp, name="up2k-mtp-scan")
                 thr.daemon = True
         else:
             del self.pp
@@ -286,9 +321,13 @@ class Up2k(object):
         return have_e2d
 
     def register_vpath(self, ptop, flags):
-        db_path = os.path.join(ptop, ".hist", "up2k.db")
+        histpath = self.asrv.vfs.histtab[ptop]
+        db_path = os.path.join(histpath, "up2k.db")
         if ptop in self.registry:
-            return [self.cur[ptop], db_path]
+            try:
+                return [self.cur[ptop], db_path]
+            except:
+                return None
 
         _, flags = self._expr_idx_filter(flags)
 
@@ -303,7 +342,7 @@ class Up2k(object):
             self.log(" ".join(sorted(a)) + "\033[0m")
 
         reg = {}
-        path = os.path.join(ptop, ".hist", "up2k.snap")
+        path = os.path.join(histpath, "up2k.snap")
         if "e2d" in flags and os.path.exists(path):
             with gzip.GzipFile(path, "rb") as f:
                 j = f.read().decode("utf-8")
@@ -327,7 +366,7 @@ class Up2k(object):
             return None
 
         try:
-            os.mkdir(os.path.join(ptop, ".hist"))
+            os.makedirs(histpath)
         except:
             pass
 
@@ -344,6 +383,7 @@ class Up2k(object):
     def _build_file_index(self, vol, all_vols):
         do_vac = False
         top = vol.realpath
+        nohash = "dhash" in vol.flags
         with self.mutex:
             cur, _ = self.register_vpath(top, vol.flags)
 
@@ -358,7 +398,7 @@ class Up2k(object):
             if WINDOWS:
                 excl = [x.replace("/", "\\") for x in excl]
 
-            n_add = self._build_dir(dbw, top, set(excl), top)
+            n_add = self._build_dir(dbw, top, set(excl), top, nohash)
             n_rm = self._drop_lost(dbw[0], top)
             if dbw[1]:
                 self.log("commit {} new files".format(dbw[1]))
@@ -366,23 +406,25 @@ class Up2k(object):
 
             return True, n_add or n_rm or do_vac
 
-    def _build_dir(self, dbw, top, excl, cdir):
+    def _build_dir(self, dbw, top, excl, cdir, nohash):
         self.pp.msg = "a{} {}".format(self.pp.n, cdir)
-        histdir = os.path.join(top, ".hist")
+        histpath = self.asrv.vfs.histtab[top]
         ret = 0
-        for iname, inf in statdir(self.log, not self.args.no_scandir, False, cdir):
+        g = statdir(self.log, not self.args.no_scandir, False, cdir)
+        for iname, inf in sorted(g):
             abspath = os.path.join(cdir, iname)
             lmod = int(inf.st_mtime)
+            sz = inf.st_size
             if stat.S_ISDIR(inf.st_mode):
-                if abspath in excl or abspath == histdir:
+                if abspath in excl or abspath == histpath:
                     continue
                 # self.log(" dir: {}".format(abspath))
-                ret += self._build_dir(dbw, top, excl, abspath)
+                ret += self._build_dir(dbw, top, excl, abspath, nohash)
             else:
                 # self.log("file: {}".format(abspath))
                 rp = abspath[len(top) :].replace("\\", "/").strip("/")
                 rd, fn = rp.rsplit("/", 1) if "/" in rp else ["", rp]
-                sql = "select * from up where rd = ? and fn = ?"
+                sql = "select w, mt, sz from up where rd = ? and fn = ?"
                 try:
                     c = dbw[0].execute(sql, (rd, fn))
                 except:
@@ -391,18 +433,18 @@ class Up2k(object):
                 in_db = list(c.fetchall())
                 if in_db:
                     self.pp.n -= 1
-                    _, dts, dsz, _, _ = in_db[0]
+                    dw, dts, dsz = in_db[0]
                     if len(in_db) > 1:
                         m = "WARN: multiple entries: [{}] => [{}] |{}|\n{}"
                         rep_db = "\n".join([repr(x) for x in in_db])
                         self.log(m.format(top, rp, len(in_db), rep_db))
                         dts = -1
 
-                    if dts == lmod and dsz == inf.st_size:
+                    if dts == lmod and dsz == sz and (nohash or dw[0] != "#"):
                         continue
 
                     m = "reindex [{}] => [{}] ({}/{}) ({}/{})".format(
-                        top, rp, dts, lmod, dsz, inf.st_size
+                        top, rp, dts, lmod, dsz, sz
                     )
                     self.log(m)
                     self.db_rm(dbw[0], rd, fn)
@@ -411,17 +453,22 @@ class Up2k(object):
                     in_db = None
 
                 self.pp.msg = "a{} {}".format(self.pp.n, abspath)
-                if inf.st_size > 1024 * 1024:
-                    self.log("file: {}".format(abspath))
 
-                try:
+                if nohash:
-                    hashes = self._hashlist_from_file(abspath)
+                    wark = up2k_wark_from_metadata(self.salt, sz, lmod, rd, fn)
-                except Exception as ex:
+                else:
-                    self.log("hash: {} @ [{}]".format(repr(ex), abspath))
+                    if sz > 1024 * 1024:
-                    continue
+                        self.log("file: {}".format(abspath))
 
-                wark = up2k_wark_from_hashlist(self.salt, inf.st_size, hashes)
+                    try:
-                self.db_add(dbw[0], wark, rd, fn, lmod, inf.st_size)
+                        hashes = self._hashlist_from_file(abspath)
+                    except Exception as ex:
+                        self.log("hash: {} @ [{}]".format(repr(ex), abspath))
+                        continue
+
+                    wark = up2k_wark_from_hashlist(self.salt, sz, hashes)
+
+                self.db_add(dbw[0], wark, rd, fn, lmod, sz)
                 dbw[1] += 1
                 ret += 1
                 td = time.time() - dbw[2]
@@ -753,7 +800,9 @@ class Up2k(object):
 
         mpool = Queue(nw)
         for _ in range(nw):
-            thr = threading.Thread(target=self._tag_thr, args=(mpool,))
+            thr = threading.Thread(
+                target=self._tag_thr, args=(mpool,), name="up2k-mpool"
+            )
             thr.daemon = True
             thr.start()
 
@@ -914,7 +963,7 @@ class Up2k(object):
     def _create_v3(self, cur):
         """
         collision in 2^(n/2) files where n = bits (6 bits/ch)
-          10*6/2 = 2^30 =       1'073'741'824, 24.1mb idx
+          10*6/2 = 2^30 =       1'073'741'824, 24.1mb idx  1<<(3*10)
           12*6/2 = 2^36 =      68'719'476'736, 24.8mb idx
           16*6/2 = 2^48 = 281'474'976'710'656, 26.1mb idx
         """
@@ -962,9 +1011,10 @@ class Up2k(object):
         return self._orz(db_path)
 
     def handle_json(self, cj):
-        if not self.register_vpath(cj["ptop"], cj["vcfg"]):
+        with self.mutex:
-            if cj["ptop"] not in self.registry:
+            if not self.register_vpath(cj["ptop"], cj["vcfg"]):
-                raise Pebkac(410, "location unavailable")
+                if cj["ptop"] not in self.registry:
+                    raise Pebkac(410, "location unavailable")
 
         cj["name"] = sanitize_fn(cj["name"], bad=[".prologue.html", ".epilogue.html"])
         cj["poke"] = time.time()
@@ -972,7 +1022,7 @@ class Up2k(object):
         now = time.time()
         job = None
         with self.mutex:
-            cur = self.cur.get(cj["ptop"], None)
+            cur = self.cur.get(cj["ptop"])
             reg = self.registry[cj["ptop"]]
             if cur:
                 if self.no_expr_idx:
@@ -1130,7 +1180,7 @@ class Up2k(object):
 
     def handle_chunk(self, ptop, wark, chash):
         with self.mutex:
-            job = self.registry[ptop].get(wark, None)
+            job = self.registry[ptop].get(wark)
             if not job:
                 known = " ".join([x for x in self.registry[ptop].keys()])
                 self.log("unknown wark [{}], known: {}".format(wark, known))
@@ -1195,7 +1245,7 @@ class Up2k(object):
         return ret, dst
 
     def idx_wark(self, ptop, wark, rd, fn, lmod, sz):
-        cur = self.cur.get(ptop, None)
+        cur = self.cur.get(ptop)
         if not cur:
             return False
 
@@ -1205,6 +1255,7 @@ class Up2k(object):
 
         if "e2t" in self.flags[ptop]:
             self.tagq.put([ptop, wark, rd, fn])
+            self.n_tagq += 1
 
         return True
 
@@ -1330,11 +1381,12 @@ class Up2k(object):
                 for k, reg in self.registry.items():
                     self._snap_reg(prev, k, reg, discard_interval)
 
-    def _snap_reg(self, prev, k, reg, discard_interval):
+    def _snap_reg(self, prev, ptop, reg, discard_interval):
         now = time.time()
+        histpath = self.asrv.vfs.histtab[ptop]
         rm = [x for x in reg.values() if now - x["poke"] > discard_interval]
         if rm:
-            m = "dropping {} abandoned uploads in {}".format(len(rm), k)
+            m = "dropping {} abandoned uploads in {}".format(len(rm), ptop)
             vis = [self._vis_job_progress(x) for x in rm]
             self.log("\n".join([m] + vis))
             for job in rm:
@@ -1352,21 +1404,21 @@ class Up2k(object):
                 except:
                     pass
 
-        path = os.path.join(k, ".hist", "up2k.snap")
+        path = os.path.join(histpath, "up2k.snap")
         if not reg:
-            if k not in prev or prev[k] is not None:
+            if ptop not in prev or prev[ptop] is not None:
-                prev[k] = None
+                prev[ptop] = None
                 if os.path.exists(fsenc(path)):
                     os.unlink(fsenc(path))
             return
 
         newest = max(x["poke"] for _, x in reg.items()) if reg else 0
         etag = [len(reg), newest]
-        if etag == prev.get(k, None):
+        if etag == prev.get(ptop):
             return
 
         try:
-            os.mkdir(os.path.join(k, ".hist"))
+            os.makedirs(histpath)
         except:
             pass
 
@@ -1378,14 +1430,21 @@ class Up2k(object):
         atomic_move(path2, path)
 
         self.log("snap: {} |{}|".format(path, len(reg.keys())))
-        prev[k] = etag
+        prev[ptop] = etag
 
     def _tagger(self):
+        with self.mutex:
+            self.n_tagq += 1
+
         while True:
+            with self.mutex:
+                self.n_tagq -= 1
+
             ptop, wark, rd, fn = self.tagq.get()
             if "e2t" not in self.flags[ptop]:
                 continue
 
+            # self.log("\n  " + repr([ptop, rd, fn]))
             abspath = os.path.join(ptop, rd, fn)
             tags = self.mtag.get(abspath)
             ntags1 = len(tags)
@@ -1411,8 +1470,16 @@ class Up2k(object):
             self.log("tagged {} ({}+{})".format(abspath, ntags1, len(tags) - ntags1))
 
     def _hasher(self):
+        with self.mutex:
+            self.n_hashq += 1
+
         while True:
+            with self.mutex:
+                self.n_hashq -= 1
+            # self.log("hashq {}".format(self.n_hashq))
+
             ptop, rd, fn = self.hashq.get()
+            # self.log("hashq {} pop {}/{}/{}".format(self.n_hashq, ptop, rd, fn))
             if "e2d" not in self.flags[ptop]:
                 continue
 
@@ -1425,8 +1492,11 @@ class Up2k(object):
                 self.idx_wark(ptop, wark, rd, fn, inf.st_mtime, inf.st_size)
 
     def hash_file(self, ptop, flags, rd, fn):
-        self.register_vpath(ptop, flags)
+        with self.mutex:
-        self.hashq.put([ptop, rd, fn])
+            self.register_vpath(ptop, flags)
+            self.hashq.put([ptop, rd, fn])
+            self.n_hashq += 1
+        # self.log("hashq {} push {}/{}/{}".format(self.n_hashq, ptop, rd, fn))
 
 
 def up2k_chunksize(filesize):
@@ -1448,9 +1518,12 @@ def up2k_wark_from_hashlist(salt, filesize, hashes):
     ident.extend(hashes)
     ident = "\n".join(ident)
 
-    hasher = hashlib.sha512()
+    wark = hashlib.sha512(ident.encode("utf-8")).digest()
-    hasher.update(ident.encode("utf-8"))
+    wark = base64.urlsafe_b64encode(wark)
-    digest = hasher.digest()[:32]
+    return wark.decode("ascii")[:43]
 
-    wark = base64.urlsafe_b64encode(digest)
+
-    return wark.decode("utf-8").rstrip("=")
+def up2k_wark_from_metadata(salt, sz, lastmod, rd, fn):
+    ret = fsenc("{}\n{}\n{}\n{}\n{}".format(salt, lastmod, sz, rd, fn))
+    ret = base64.urlsafe_b64encode(hashlib.sha512(ret).digest())
+    return "#{}".format(ret[:42].decode("ascii"))

copyparty/util.py

@@ -193,7 +193,7 @@ class ProgressPrinter(threading.Thread):
     """
 
     def __init__(self):
-        threading.Thread.__init__(self)
+        threading.Thread.__init__(self, name="pp")
         self.daemon = True
         self.msg = None
         self.end = False
@@ -208,6 +208,8 @@ class ProgressPrinter(threading.Thread):
 
             msg = self.msg
             uprint(" {}\033[K\r".format(msg))
+            if PY2:
+                sys.stdout.flush()
 
         print("\033[K", end="")
         sys.stdout.flush()  # necessary on win10 even w/ stderr btw
@@ -252,6 +254,45 @@ def trace(*args, **kwargs):
     nuprint(msg)
 
 
+def alltrace():
+    threads = {}
+    names = dict([(t.ident, t.name) for t in threading.enumerate()])
+    for tid, stack in sys._current_frames().items():
+        name = "{} ({:x})".format(names.get(tid), tid)
+        threads[name] = stack
+
+    rret = []
+    bret = []
+    for name, stack in sorted(threads.items()):
+        ret = ["\n\n# {}".format(name)]
+        pad = None
+        for fn, lno, name, line in traceback.extract_stack(stack):
+            fn = os.sep.join(fn.split(os.sep)[-3:])
+            ret.append('File: "{}", line {}, in {}'.format(fn, lno, name))
+            if line:
+                ret.append("  " + str(line.strip()))
+                if "self.not_empty.wait()" in line:
+                    pad = " " * 4
+
+        if pad:
+            bret += [ret[0]] + [pad + x for x in ret[1:]]
+        else:
+            rret += ret
+
+    return "\n".join(rret + bret)
+
+
+def min_ex():
+    et, ev, tb = sys.exc_info()
+    tb = traceback.extract_tb(tb, 2)
+    ex = [
+        "{} @ {} <{}>: {}".format(fp.split(os.sep)[-1], ln, fun, txt)
+        for fp, ln, fun, txt in tb
+    ]
+    ex.append("{}: {}".format(et.__name__, ev))
+    return "\n".join(ex)
+
+
 @contextlib.contextmanager
 def ren_open(fname, *args, **kwargs):
     fdir = kwargs.pop("fdir", None)
@@ -566,8 +607,10 @@ def read_header(sr):
             else:
                 continue
 
-        sr.unrecv(ret[ofs + 4 :])
+        if len(ret) > ofs + 4:
-        return ret[:ofs].decode("utf-8", "surrogateescape").split("\r\n")
+            sr.unrecv(ret[ofs + 4 :])
+
+        return ret[:ofs].decode("utf-8", "surrogateescape").lstrip("\r\n").split("\r\n")
 
 
 def humansize(sz, terse=False):
@@ -852,13 +895,14 @@ def yieldfile(fn):
 
 
 def hashcopy(actor, fin, fout):
-    u32_lim = int((2 ** 31) * 0.9)
+    is_mp = actor.is_mp
     hashobj = hashlib.sha512()
     tlen = 0
     for buf in fin:
-        actor.workload += 1
+        if is_mp:
-        if actor.workload > u32_lim:
+            actor.workload += 1
-            actor.workload = 100  # prevent overflow
+            if actor.workload > 2 ** 31:
+                actor.workload = 100
 
         tlen += len(buf)
         hashobj.update(buf)
@@ -870,12 +914,17 @@ def hashcopy(actor, fin, fout):
     return tlen, hashobj.hexdigest(), digest_b64
 
 
-def sendfile_py(lower, upper, f, s):
+def sendfile_py(lower, upper, f, s, actor=None):
     remains = upper - lower
     f.seek(lower)
     while remains > 0:
+        if actor:
+            actor.workload += 1
+            if actor.workload > 2 ** 31:
+                actor.workload = 100
+
         # time.sleep(0.01)
-        buf = f.read(min(4096, remains))
+        buf = f.read(min(1024 * 32, remains))
         if not buf:
             return remains
 
@@ -977,8 +1026,8 @@ def guess_mime(url, fallback="application/octet-stream"):
 def runcmd(*argv):
     p = sp.Popen(argv, stdout=sp.PIPE, stderr=sp.PIPE)
     stdout, stderr = p.communicate()
-    stdout = stdout.decode("utf-8")
+    stdout = stdout.decode("utf-8", "replace")
-    stderr = stderr.decode("utf-8")
+    stderr = stderr.decode("utf-8", "replace")
     return [p.returncode, stdout, stderr]
 
 

copyparty/web/baguettebox.js

@@ -0,0 +1,583 @@
+/*!
+ * baguetteBox.js
+ * @author  feimosi
+ * @version 1.11.1-mod
+ * @url https://github.com/feimosi/baguetteBox.js
+ */
+
+window.baguetteBox = (function () {
+    'use strict';
+
+    var options = {},
+        defaults = {
+            captions: true,
+            buttons: 'auto',
+            noScrollbars: false,
+            bodyClass: 'baguetteBox-open',
+            titleTag: false,
+            async: false,
+            preload: 2,
+            animation: 'slideIn',
+            afterShow: null,
+            afterHide: null,
+            onChange: null,
+        },
+        overlay, slider, previousButton, nextButton, closeButton,
+        currentGallery = [],
+        currentIndex = 0,
+        isOverlayVisible = false,
+        touch = {},  // start-pos
+        touchFlag = false,  // busy
+        regex = /.+\.(gif|jpe?g|png|webp)/i,
+        data = {},  // all galleries
+        imagesElements = [],
+        documentLastFocus = null;
+
+    var overlayClickHandler = function (event) {
+        if (event.target.id.indexOf('baguette-img') !== -1) {
+            hideOverlay();
+        }
+    };
+
+    var touchstartHandler = function (event) {
+        touch.count++;
+        if (touch.count > 1) {
+            touch.multitouch = true;
+        }
+        touch.startX = event.changedTouches[0].pageX;
+        touch.startY = event.changedTouches[0].pageY;
+    };
+    var touchmoveHandler = function (event) {
+        if (touchFlag || touch.multitouch) {
+            return;
+        }
+        event.preventDefault ? event.preventDefault() : event.returnValue = false;
+        var touchEvent = event.touches[0] || event.changedTouches[0];
+        if (touchEvent.pageX - touch.startX > 40) {
+            touchFlag = true;
+            showPreviousImage();
+        } else if (touchEvent.pageX - touch.startX < -40) {
+            touchFlag = true;
+            showNextImage();
+        } else if (touch.startY - touchEvent.pageY > 100) {
+            hideOverlay();
+        }
+    };
+    var touchendHandler = function () {
+        touch.count--;
+        if (touch.count <= 0) {
+            touch.multitouch = false;
+        }
+        touchFlag = false;
+    };
+    var contextmenuHandler = function () {
+        touchendHandler();
+    };
+
+    var trapFocusInsideOverlay = function (event) {
+        if (overlay.style.display === 'block' && (overlay.contains && !overlay.contains(event.target))) {
+            event.stopPropagation();
+            initFocus();
+        }
+    };
+
+    function run(selector, userOptions) {
+        buildOverlay();
+        removeFromCache(selector);
+        return bindImageClickListeners(selector, userOptions);
+    }
+
+    function bindImageClickListeners(selector, userOptions) {
+        var galleryNodeList = document.querySelectorAll(selector);
+        var selectorData = {
+            galleries: [],
+            nodeList: galleryNodeList
+        };
+        data[selector] = selectorData;
+
+        [].forEach.call(galleryNodeList, function (galleryElement) {
+            if (userOptions && userOptions.filter) {
+                regex = userOptions.filter;
+            }
+
+            var tagsNodeList = [];
+            if (galleryElement.tagName === 'A') {
+                tagsNodeList = [galleryElement];
+            } else {
+                tagsNodeList = galleryElement.getElementsByTagName('a');
+            }
+
+            tagsNodeList = [].filter.call(tagsNodeList, function (element) {
+                if (element.className.indexOf(userOptions && userOptions.ignoreClass) === -1) {
+                    return regex.test(element.href);
+                }
+            });
+            if (tagsNodeList.length === 0) {
+                return;
+            }
+
+            var gallery = [];
+            [].forEach.call(tagsNodeList, function (imageElement, imageIndex) {
+                var imageElementClickHandler = function (event) {
+                    if (event && event.ctrlKey)
+                        return true;
+
+                    event.preventDefault ? event.preventDefault() : event.returnValue = false;
+                    prepareOverlay(gallery, userOptions);
+                    showOverlay(imageIndex);
+                };
+                var imageItem = {
+                    eventHandler: imageElementClickHandler,
+                    imageElement: imageElement
+                };
+                bind(imageElement, 'click', imageElementClickHandler);
+                gallery.push(imageItem);
+            });
+            selectorData.galleries.push(gallery);
+        });
+
+        return selectorData.galleries;
+    }
+
+    function clearCachedData() {
+        for (var selector in data) {
+            if (data.hasOwnProperty(selector)) {
+                removeFromCache(selector);
+            }
+        }
+    }
+
+    function removeFromCache(selector) {
+        if (!data.hasOwnProperty(selector)) {
+            return;
+        }
+        var galleries = data[selector].galleries;
+        [].forEach.call(galleries, function (gallery) {
+            [].forEach.call(gallery, function (imageItem) {
+                unbind(imageItem.imageElement, 'click', imageItem.eventHandler);
+            });
+
+            if (currentGallery === gallery) {
+                currentGallery = [];
+            }
+        });
+
+        delete data[selector];
+    }
+
+    function buildOverlay() {
+        overlay = ebi('baguetteBox-overlay');
+        if (overlay) {
+            slider = ebi('baguetteBox-slider');
+            previousButton = ebi('previous-button');
+            nextButton = ebi('next-button');
+            closeButton = ebi('close-button');
+            return;
+        }
+        overlay = mknod('div');
+        overlay.setAttribute('role', 'dialog');
+        overlay.id = 'baguetteBox-overlay';
+        document.getElementsByTagName('body')[0].appendChild(overlay);
+
+        slider = mknod('div');
+        slider.id = 'baguetteBox-slider';
+        overlay.appendChild(slider);
+
+        previousButton = mknod('button');
+        previousButton.setAttribute('type', 'button');
+        previousButton.id = 'previous-button';
+        previousButton.setAttribute('aria-label', 'Previous');
+        previousButton.innerHTML = '&lt;';
+        overlay.appendChild(previousButton);
+
+        nextButton = mknod('button');
+        nextButton.setAttribute('type', 'button');
+        nextButton.id = 'next-button';
+        nextButton.setAttribute('aria-label', 'Next');
+        nextButton.innerHTML = '&gt;';
+        overlay.appendChild(nextButton);
+
+        closeButton = mknod('button');
+        closeButton.setAttribute('type', 'button');
+        closeButton.id = 'close-button';
+        closeButton.setAttribute('aria-label', 'Close');
+        closeButton.innerHTML = '&times;';
+        overlay.appendChild(closeButton);
+
+        previousButton.className = nextButton.className = closeButton.className = 'baguetteBox-button';
+
+        bindEvents();
+    }
+
+    function keyDownHandler(event) {
+        switch (event.keyCode) {
+            case 37: // Left
+                showPreviousImage();
+                break;
+            case 39: // Right
+                showNextImage();
+                break;
+            case 27: // Esc
+                hideOverlay();
+                break;
+            case 36: // Home
+                showFirstImage(event);
+                break;
+            case 35: // End
+                showLastImage(event);
+                break;
+        }
+    }
+
+    var passiveSupp = false;
+    try {
+        var opts = {
+            get passive() {
+                passiveSupp = true;
+                return false;
+            }
+        };
+        window.addEventListener('test', null, opts);
+        window.removeEventListener('test', null, opts);
+    }
+    catch (ex) {
+        passiveSupp = false;
+    }
+    var passiveEvent = passiveSupp ? { passive: false } : null;
+    var nonPassiveEvent = passiveSupp ? { passive: true } : null;
+
+    function bindEvents() {
+        bind(overlay, 'click', overlayClickHandler);
+        bind(previousButton, 'click', showPreviousImage);
+        bind(nextButton, 'click', showNextImage);
+        bind(closeButton, 'click', hideOverlay);
+        bind(slider, 'contextmenu', contextmenuHandler);
+        bind(overlay, 'touchstart', touchstartHandler, nonPassiveEvent);
+        bind(overlay, 'touchmove', touchmoveHandler, passiveEvent);
+        bind(overlay, 'touchend', touchendHandler);
+        bind(document, 'focus', trapFocusInsideOverlay, true);
+    }
+
+    function unbindEvents() {
+        unbind(overlay, 'click', overlayClickHandler);
+        unbind(previousButton, 'click', showPreviousImage);
+        unbind(nextButton, 'click', showNextImage);
+        unbind(closeButton, 'click', hideOverlay);
+        unbind(slider, 'contextmenu', contextmenuHandler);
+        unbind(overlay, 'touchstart', touchstartHandler, nonPassiveEvent);
+        unbind(overlay, 'touchmove', touchmoveHandler, passiveEvent);
+        unbind(overlay, 'touchend', touchendHandler);
+        unbind(document, 'focus', trapFocusInsideOverlay, true);
+    }
+
+    function prepareOverlay(gallery, userOptions) {
+        if (currentGallery === gallery) {
+            return;
+        }
+        currentGallery = gallery;
+        setOptions(userOptions);
+        slider.innerHTML = '';
+        imagesElements.length = 0;
+
+        var imagesFiguresIds = [];
+        var imagesCaptionsIds = [];
+        for (var i = 0, fullImage; i < gallery.length; i++) {
+            fullImage = mknod('div');
+            fullImage.className = 'full-image';
+            fullImage.id = 'baguette-img-' + i;
+            imagesElements.push(fullImage);
+
+            imagesFiguresIds.push('baguetteBox-figure-' + i);
+            imagesCaptionsIds.push('baguetteBox-figcaption-' + i);
+            slider.appendChild(imagesElements[i]);
+        }
+        overlay.setAttribute('aria-labelledby', imagesFiguresIds.join(' '));
+        overlay.setAttribute('aria-describedby', imagesCaptionsIds.join(' '));
+    }
+
+    function setOptions(newOptions) {
+        if (!newOptions) {
+            newOptions = {};
+        }
+        for (var item in defaults) {
+            options[item] = defaults[item];
+            if (typeof newOptions[item] !== 'undefined') {
+                options[item] = newOptions[item];
+            }
+        }
+        slider.style.transition = (options.animation === 'fadeIn' ? 'opacity .4s ease' :
+            options.animation === 'slideIn' ? '' : 'none');
+
+        if (options.buttons === 'auto' && ('ontouchstart' in window || currentGallery.length === 1)) {
+            options.buttons = false;
+        }
+
+        previousButton.style.display = nextButton.style.display = (options.buttons ? '' : 'none');
+    }
+
+    function showOverlay(chosenImageIndex) {
+        if (options.noScrollbars) {
+            document.documentElement.style.overflowY = 'hidden';
+            document.body.style.overflowY = 'scroll';
+        }
+        if (overlay.style.display === 'block') {
+            return;
+        }
+
+        bind(document, 'keydown', keyDownHandler);
+        currentIndex = chosenImageIndex;
+        touch = {
+            count: 0,
+            startX: null,
+            startY: null
+        };
+        loadImage(currentIndex, function () {
+            preloadNext(currentIndex);
+            preloadPrev(currentIndex);
+        });
+
+        updateOffset();
+        overlay.style.display = 'block';
+        // Fade in overlay
+        setTimeout(function () {
+            overlay.className = 'visible';
+            if (options.bodyClass && document.body.classList) {
+                document.body.classList.add(options.bodyClass);
+            }
+            if (options.afterShow) {
+                options.afterShow();
+            }
+        }, 50);
+        if (options.onChange) {
+            options.onChange(currentIndex, imagesElements.length);
+        }
+        documentLastFocus = document.activeElement;
+        initFocus();
+        isOverlayVisible = true;
+    }
+
+    function initFocus() {
+        if (options.buttons) {
+            previousButton.focus();
+        } else {
+            closeButton.focus();
+        }
+    }
+
+    function hideOverlay(e) {
+        ev(e);
+        if (options.noScrollbars) {
+            document.documentElement.style.overflowY = 'auto';
+            document.body.style.overflowY = 'auto';
+        }
+        if (overlay.style.display === 'none') {
+            return;
+        }
+
+        unbind(document, 'keydown', keyDownHandler);
+        // Fade out and hide the overlay
+        overlay.className = '';
+        setTimeout(function () {
+            overlay.style.display = 'none';
+            if (options.bodyClass && document.body.classList) {
+                document.body.classList.remove(options.bodyClass);
+            }
+            if (options.afterHide) {
+                options.afterHide();
+            }
+            documentLastFocus && documentLastFocus.focus();
+            isOverlayVisible = false;
+        }, 500);
+    }
+
+    function loadImage(index, callback) {
+        var imageContainer = imagesElements[index];
+        var galleryItem = currentGallery[index];
+
+        if (typeof imageContainer === 'undefined' || typeof galleryItem === 'undefined') {
+            return;  // out-of-bounds or gallery dirty
+        }
+
+        if (imageContainer.getElementsByTagName('img')[0]) {
+            // image is loaded, cb and bail
+            if (callback) {
+                callback();
+            }
+            return;
+        }
+
+        var imageElement = galleryItem.imageElement,
+            imageSrc = imageElement.href,
+            thumbnailElement = imageElement.getElementsByTagName('img')[0],
+            imageCaption = typeof options.captions === 'function' ?
+                options.captions.call(currentGallery, imageElement) :
+                imageElement.getAttribute('data-caption') || imageElement.title;
+
+        var figure = mknod('figure');
+        figure.id = 'baguetteBox-figure-' + index;
+        figure.innerHTML = '<div class="baguetteBox-spinner">' +
+            '<div class="baguetteBox-double-bounce1"></div>' +
+            '<div class="baguetteBox-double-bounce2"></div>' +
+            '</div>';
+
+        if (options.captions && imageCaption) {
+            var figcaption = mknod('figcaption');
+            figcaption.id = 'baguetteBox-figcaption-' + index;
+            figcaption.innerHTML = imageCaption;
+            figure.appendChild(figcaption);
+        }
+        imageContainer.appendChild(figure);
+
+        var image = mknod('img');
+        image.onload = function () {
+            // Remove loader element
+            var spinner = document.querySelector('#baguette-img-' + index + ' .baguetteBox-spinner');
+            figure.removeChild(spinner);
+            if (!options.async && callback) {
+                callback();
+            }
+        };
+        image.setAttribute('src', imageSrc);
+        image.alt = thumbnailElement ? thumbnailElement.alt || '' : '';
+        if (options.titleTag && imageCaption) {
+            image.title = imageCaption;
+        }
+        figure.appendChild(image);
+
+        if (options.async && callback) {
+            callback();
+        }
+    }
+
+    function showNextImage(e) {
+        ev(e);
+        return show(currentIndex + 1);
+    }
+
+    function showPreviousImage(e) {
+        ev(e);
+        return show(currentIndex - 1);
+    }
+
+    function showFirstImage(event) {
+        if (event) {
+            event.preventDefault();
+        }
+        return show(0);
+    }
+
+    function showLastImage(event) {
+        if (event) {
+            event.preventDefault();
+        }
+        return show(currentGallery.length - 1);
+    }
+
+    /**
+     * Move the gallery to a specific index
+     * @param `index` {number} - the position of the image
+     * @param `gallery` {array} - gallery which should be opened, if omitted assumes the currently opened one
+     * @return {boolean} - true on success or false if the index is invalid
+     */
+    function show(index, gallery) {
+        if (!isOverlayVisible && index >= 0 && index < gallery.length) {
+            prepareOverlay(gallery, options);
+            showOverlay(index);
+            return true;
+        }
+        if (index < 0) {
+            if (options.animation) {
+                bounceAnimation('left');
+            }
+            return false;
+        }
+        if (index >= imagesElements.length) {
+            if (options.animation) {
+                bounceAnimation('right');
+            }
+            return false;
+        }
+
+        currentIndex = index;
+        loadImage(currentIndex, function () {
+            preloadNext(currentIndex);
+            preloadPrev(currentIndex);
+        });
+        updateOffset();
+
+        if (options.onChange) {
+            options.onChange(currentIndex, imagesElements.length);
+        }
+
+        return true;
+    }
+
+    /**
+     * Triggers the bounce animation
+     * @param {('left'|'right')} direction - Direction of the movement
+     */
+    function bounceAnimation(direction) {
+        slider.className = 'bounce-from-' + direction;
+        setTimeout(function () {
+            slider.className = '';
+        }, 400);
+    }
+
+    function updateOffset() {
+        var offset = -currentIndex * 100 + '%';
+        if (options.animation === 'fadeIn') {
+            slider.style.opacity = 0;
+            setTimeout(function () {
+                slider.style.transform = 'translate3d(' + offset + ',0,0)';
+                slider.style.opacity = 1;
+            }, 400);
+        } else {
+            slider.style.transform = 'translate3d(' + offset + ',0,0)';
+        }
+    }
+
+    function preloadNext(index) {
+        if (index - currentIndex >= options.preload) {
+            return;
+        }
+        loadImage(index + 1, function () {
+            preloadNext(index + 1);
+        });
+    }
+
+    function preloadPrev(index) {
+        if (currentIndex - index >= options.preload) {
+            return;
+        }
+        loadImage(index - 1, function () {
+            preloadPrev(index - 1);
+        });
+    }
+
+    function bind(element, event, callback, options) {
+        element.addEventListener(event, callback, options);
+    }
+
+    function unbind(element, event, callback, options) {
+        element.removeEventListener(event, callback, options);
+    }
+
+    function destroyPlugin() {
+        unbindEvents();
+        clearCachedData();
+        unbind(document, 'keydown', keyDownHandler);
+        document.getElementsByTagName('body')[0].removeChild(ebi('baguetteBox-overlay'));
+        data = {};
+        currentGallery = [];
+        currentIndex = 0;
+    }
+
+    return {
+        run: run,
+        show: show,
+        showNext: showNextImage,
+        showPrevious: showPreviousImage,
+        hide: hideOverlay,
+        destroy: destroyPlugin
+    };
+})();

copyparty/web/browser.css

@@ -497,6 +497,27 @@ input[type="checkbox"]+label {
 input[type="checkbox"]:checked+label {
 	color: #fc5;
 }
+input.eq_gain {
+	width: 3em;
+	text-align: center;
+	margin: 0 .6em;
+}
+#audio_eq table {
+	border-collapse: collapse;
+}
+#audio_eq td {
+	text-align: center;
+}
+#audio_eq a.eq_step {
+	font-size: 1.5em;
+	display: block;
+	padding: 0;
+}
+#au_eq {
+	display: block;
+	margin-top: .5em;
+	padding: 1.3em .3em;
+}
 
 
 
@@ -563,6 +584,7 @@ input[type="checkbox"]:checked+label {
 }
 #wrap {
 	margin-top: 2em;
+	min-height: 90vh;
 }
 #tree {
 	display: none;
@@ -575,6 +597,12 @@ input[type="checkbox"]:checked+label {
 	overscroll-behavior-y: none;
 	scrollbar-color: #eb0 #333;
 }
+#treeh {
+	background: #333;
+	position: sticky;
+	z-index: 1;
+	top: 0;
+}
 #thx_ff {
 	padding: 5em 0;
 }
@@ -600,6 +628,7 @@ input[type="checkbox"]:checked+label {
 	box-shadow: 0 .1em .2em #222 inset;
 	border-radius: .3em;
 	margin: .2em;
+	white-space: pre;
 	position: relative;
 	top: -.2em;
 }
@@ -644,7 +673,6 @@ input[type="checkbox"]:checked+label {
 }
 #treeul a+a {
 	width: calc(100% - 2em);
-	background: #333;
 	line-height: 1em;
 }
 #treeul a+a:hover {
@@ -668,34 +696,20 @@ input[type="checkbox"]:checked+label {
 	font-size: 2em;
 	white-space: nowrap;
 }
-#files th:hover .cfg,
+#files th:hover .cfg {
-#files th.min .cfg {
 	display: block;
 	width: 1em;
 	border-radius: .2em;
 	margin: -1.3em auto 0 auto;
 	background: #444;
 }
-#files th.min .cfg {
+#files>thead>tr>th.min,
-	margin: -.6em;
+#files td.min {
-}
+		display: none;
-#files>thead>tr>th.min span {
-	position: absolute;
-	transform: rotate(270deg);
-	background: linear-gradient(90deg, rgba(68,68,68,0), rgba(68,68,68,0.5) 70%, #444);
-	margin-left: -4.6em;
-	padding: .4em;
-	top: 5.4em;
-	width: 8em;
-	text-align: right;
-	letter-spacing: .04em;
 }
 #files td:nth-child(2n) {
 	color: #f5a;
 }
-#files td.min a {
-	display: none;
-}
 #files tr.play td,
 #files tr.play div a {
 	background: #fc4;
@@ -710,18 +724,18 @@ input[type="checkbox"]:checked+label {
 	color: #300;
 	background: #fea;
 }
-#op_cfg {
+.opwide {
 	max-width: none;
 	margin-right: 1.5em;
 }
-#op_cfg>div>a {
+.opwide>div>a {
 	line-height: 2em;
 }
 #op_cfg>div>span {
 	display: inline-block;
 	padding: .2em .4em;
 }
-#op_cfg h3 {
+.opbox h3 {
 	margin: .8em 0 0 .6em;
 	padding: 0;
 	border-bottom: 1px solid #555;
@@ -751,9 +765,12 @@ input[type="checkbox"]:checked+label {
 	font-family: monospace, monospace;
 	line-height: 2em;
 }
-#griden.on+#thumbs {
+#thumbs {
 	opacity: .3;
 }
+#griden.on+#thumbs {
+	opacity: 1;
+}
 #ghead {
 	background: #3c3c3c;
 	border: 1px solid #444;
@@ -798,6 +815,12 @@ html.light #ghead {
 	padding: .2em .3em;
 	display: block;
 }
+#ggrid span.dir:before {
+	content: '📂';
+	line-height: 0;
+	font-size: 2em;
+	margin: -.7em .1em -.5em -.3em;
+}
 #ggrid a:hover {
 	background: #444;
 	border-color: #555;
@@ -910,6 +933,7 @@ html.light #files {
 }
 html.light #files thead th {
 	background: #eee;
+	border-radius: 0;
 }
 html.light #files tr td {
 	border-top: 1px solid #ddd;
@@ -935,13 +959,9 @@ html.light tr.play td {
 html.light tr.play a {
 	color: #406;
 }
-html.light #files th:hover .cfg,
+html.light #files th:hover .cfg {
-html.light #files th.min .cfg {
 	background: #ccc;
 }
-html.light #files > thead > tr > th.min span {
-	background: linear-gradient(90deg, rgba(204,204,204,0), rgba(204,204,204,0.5) 70%, #ccc);
-}
 html.light #blocked {
 	background: #eee;
 }
@@ -1012,6 +1032,9 @@ html.light #files tr.sel a:hover {
 	color: #000;
 	background: #fff;
 }
+html.light #treeh {
+	background: #eee;
+}
 html.light #tree {
 	scrollbar-color: #a70 #ddd;
 }
@@ -1022,3 +1045,160 @@ html.light #tree::-webkit-scrollbar {
 #tree::-webkit-scrollbar-thumb {
 	background: #da0;
 }
+
+
+
+#baguetteBox-overlay {
+	display: none;
+	opacity: 0;
+	position: fixed;
+	overflow: hidden;
+	top: 0;
+	left: 0;
+	width: 100%;
+	height: 100%;
+	z-index: 1000000;
+	background: rgba(0, 0, 0, 0.8);
+	transition: opacity .3s ease;
+}
+#baguetteBox-overlay.visible {
+	opacity: 1;
+}
+#baguetteBox-overlay .full-image {
+    display: inline-block;
+    position: relative;
+    width: 100%;
+    height: 100%;
+    text-align: center;
+}
+#baguetteBox-overlay .full-image figure {
+	display: inline;
+	margin: 0;
+	height: 100%;
+}
+#baguetteBox-overlay .full-image img {
+	display: inline-block;
+	width: auto;
+	height: auto;
+	max-height: 100%;
+	max-width: 100%;
+	vertical-align: middle;
+	box-shadow: 0 0 8px rgba(0, 0, 0, 0.6);
+}
+#baguetteBox-overlay .full-image figcaption {
+	display: block;
+	position: absolute;
+	bottom: 0;
+	width: 100%;
+	text-align: center;
+	line-height: 1.8;
+	white-space: normal;
+	color: #ccc;
+}
+#baguetteBox-overlay figcaption a {
+	background: rgba(0, 0, 0, 0.6);
+	border-radius: .4em;
+	padding: .3em .6em;
+}
+#baguetteBox-overlay .full-image:before {
+	content: "";
+	display: inline-block;
+	height: 50%;
+	width: 1px;
+	margin-right: -1px;
+}
+#baguetteBox-slider {
+	position: absolute;
+	left: 0;
+	top: 0;
+	height: 100%;
+	width: 100%;
+	white-space: nowrap;
+	transition: left .2s ease, transform .2s ease;
+}
+#baguetteBox-slider.bounce-from-right {
+	animation: bounceFromRight .4s ease-out;
+}
+#baguetteBox-slider.bounce-from-left {
+	animation: bounceFromLeft .4s ease-out;
+}
+@keyframes bounceFromRight {
+	0% {margin-left: 0}
+	50% {margin-left: -30px}
+	100% {margin-left: 0}
+}
+@keyframes bounceFromLeft {
+	0% {margin-left: 0}
+	50% {margin-left: 30px}
+	100% {margin-left: 0}
+}
+.baguetteBox-button#next-button,
+.baguetteBox-button#previous-button {
+	top: 50%;
+	top: calc(50% - 30px);
+	width: 44px;
+	height: 60px;
+} 
+.baguetteBox-button {
+	position: absolute;
+	cursor: pointer;
+	outline: none;
+	padding: 0;
+	margin: 0;
+	border: 0;
+	border-radius: 15%;
+	background: rgba(50, 50, 50, 0.5);
+	color: #ddd;
+	font: 1.6em sans-serif;
+	transition: background-color .3s ease;
+}
+.baguetteBox-button:focus,
+.baguetteBox-button:hover {
+	background: rgba(50, 50, 50, 0.9);
+}
+#next-button {
+	right: 2%;
+}
+#previous-button {
+	left: 2%;
+}
+#close-button {
+	top: 20px;
+	right: 2%;
+	width: 30px;
+	height: 30px;
+}
+.baguetteBox-button svg {
+	position: absolute;
+	left: 0;
+	top: 0;
+}
+.baguetteBox-spinner {
+	width: 40px;
+	height: 40px;
+	display: inline-block;
+	position: absolute;
+	top: 50%;
+	left: 50%;
+	margin-top: -20px;
+	margin-left: -20px;
+}
+.baguetteBox-double-bounce1,
+.baguetteBox-double-bounce2 {
+	width: 100%;
+	height: 100%;
+	border-radius: 50%;
+	background-color: #fff;
+	opacity: .6;
+	position: absolute;
+	top: 0;
+	left: 0;
+	animation: bounce 2s infinite ease-in-out;
+}
+.baguetteBox-double-bounce2 {
+	animation-delay: -1s;
+}
+@keyframes bounce {
+	0%, 100% {transform: scale(0)}
+	50% {transform: scale(1)}
+}

copyparty/web/browser.html

@@ -8,6 +8,9 @@
     <meta name="viewport" content="width=device-width, initial-scale=0.8">
     <link rel="stylesheet" type="text/css" media="screen" href="/.cpr/browser.css{{ ts }}">
     <link rel="stylesheet" type="text/css" media="screen" href="/.cpr/upload.css{{ ts }}">
+    {%- if css %}
+    <link rel="stylesheet" type="text/css" media="screen" href="{{ css }}{{ ts }}">
+    {%- endif %}
 </head>
 
 <body>
@@ -23,6 +26,7 @@
         <a href="#" data-perm="write" data-dest="mkdir" data-desc="mkdir: create a new directory">📂</a>
         <a href="#" data-perm="read write" data-dest="new_md" data-desc="new-md: create a new markdown document">📝</a>
         <a href="#" data-perm="write" data-dest="msg" data-desc="msg: send a message to the server log">📟</a>
+        <a href="#" data-dest="player" data-desc="media player options">🎺</a>
         <a href="#" data-dest="cfg" data-desc="configuration options">⚙️</a>
         <div id="opdesc"></div>
     </div>
@@ -36,22 +40,23 @@
         <div id="srch_q"></div>
     </div>
 
+    <div id="op_player" class="opview opbox opwide"></div>
+
     {%- include 'upload.html' %}
 
-    <div id="op_cfg" class="opview opbox">
+    <div id="op_cfg" class="opview opbox opwide">
         <h3>switches</h3>
         <div>
-            <a id="tooltips" class="tgl btn" href="#">tooltips</a>
+            <a id="tooltips" class="tgl btn" href="#">ℹ️ tooltips</a>
-            <a id="lightmode" class="tgl btn" href="#">lightmode</a>
+            <a id="lightmode" class="tgl btn" href="#">☀️ lightmode</a>
-            <a id="griden" class="tgl btn" href="#">the grid</a>
+            <a id="griden" class="tgl btn" href="#">田 the grid</a>
-            <a id="thumbs" class="tgl btn" href="#">thumbs</a>
+            <a id="thumbs" class="tgl btn" href="#">🖼️ thumbs</a>
         </div>
         {%- if have_zip %}
-        <h3>folder download</h3>
+        <h3>folder download</h3><div id="arc_fmt"></div>
-        <div id="arc_fmt"></div>
         {%- endif %}
-        <h3>key notation</h3>
+        <h3>key notation</h3><div id="key_notation"></div>
-        <div id="key_notation"></div>
+        <h3>hidden columns</h3><div id="hcols"></div>
     </div>
     
     <h1 id="path">
@@ -62,10 +67,12 @@
     </h1>
     
     <div id="tree">
-        <a href="#" id="detree">🍞...</a>
+        <div id="treeh">
-        <a href="#" class="btn" step="2" id="twobytwo">+</a>
+            <a href="#" id="detree">🍞...</a>
-        <a href="#" class="btn" step="-2" id="twig">&ndash;</a>
+            <a href="#" class="btn" step="2" id="twobytwo">+</a>
-        <a href="#" class="tgl btn" id="dyntree">a</a>
+            <a href="#" class="btn" step="-2" id="twig">&ndash;</a>
+            <a href="#" class="tgl btn" id="dyntree">a</a>
+        </div>
         <ul id="treeul"></ul>
         <div id="thx_ff">&nbsp;</div>
     </div>

copyparty/web/browser.js

@@ -38,7 +38,40 @@ var have_webp = null;
 	img.onerror = function () {
 		have_webp = false;
 	};
-	img.src = "data:image/webp;base64,UklGRiIAAABXRUJQVlA4IBYAAAAwAQCdASoBAAEADsD+JaQAA3AAAAAA";
+	img.src = "data:image/webp;base64,UklGRhoAAABXRUJQVlA4TA0AAAAvAAAAEAcQERGIiP4HAA==";
+})();
+
+
+var mpl = (function () {
+	ebi('op_player').innerHTML = (
+		'<h3>audio equalizer</h3><div id="audio_eq"></div>' +
+
+		'<h3>playback mode</h3><div id="pb_mode">' +
+		'<a href="#" class="tgl btn">🔁 loop-folder</a>' +
+		'<a href="#" class="tgl btn">📂 next-folder</a>' +
+		'</div>');
+
+	var r = {
+		"pb_mode": sread('pb_mode') || 'loop-folder'
+	};
+
+	function draw_pb_mode() {
+		var btns = QSA('#pb_mode>a');
+		for (var a = 0, aa = btns.length; a < aa; a++) {
+			clmod(btns[a], 'on', btns[a].textContent.indexOf(r.pb_mode) != -1);
+			btns[a].onclick = set_pb_mode;
+		}
+	}
+	draw_pb_mode();
+
+	function set_pb_mode(e) {
+		ev(e);
+		r.pb_mode = this.textContent.split(' ').slice(-1)[0];
+		swrite('pb_mode', r.pb_mode);
+		draw_pb_mode();
+	}
+
+	return r;
 })();
 
 
@@ -48,7 +81,6 @@ function MPlayer() {
 	this.au = null;
 	this.au_native = null;
 	this.au_ogvjs = null;
-	this.cover_url = '';
 	this.tracks = {};
 	this.order = [];
 
@@ -163,8 +195,9 @@ var widget = (function () {
 			m = ck + 'np: ';
 
 		for (var a = 1, aa = th.length; a < aa; a++) {
-			var tk = a == 1 ? '' : th[a].getAttribute('name').split('/').slice(-1)[0];
+			var tv = tr[a].textContent,
-			var tv = tr[a].getAttribute('html') || tr[a].textContent;
+				tk = a == 1 ? '' : th[a].getAttribute('name').split('/').slice(-1)[0];
+
 			m += tk + '(' + cv + tv + ck + ') // ';
 		}
 
@@ -408,7 +441,7 @@ function song_skip(n) {
 	if (tid !== null)
 		play(mp.order.indexOf(tid) + n);
 	else
-		play(mp.order[0]);
+		play(mp.order[n == -1 ? mp.order.length - 1 : 0]);
 }
 
 
@@ -509,6 +542,194 @@ try {
 catch (ex) { }
 
 
+var audio_eq = (function () {
+	var r = {
+		"en": false,
+		"bands": [31.25, 62.5, 125, 250, 500, 1000, 2000, 4000, 8000, 16000],
+		"gains": [4, 3, 2, 1, 0, 0, 1, 2, 3, 4],
+		"filters": [],
+		"last_au": null
+	};
+
+	var cfg = [ // hz, q, g
+		[31.25 * 0.88, 0, 1.4],  // shelf
+		[31.25 * 1.04, 0.7, 0.96],  // peak
+		[62.5, 0.7, 1],
+		[125, 0.8, 1],
+		[250, 0.9, 1.03],
+		[500, 0.9, 1.1],
+		[1000, 0.9, 1.1],
+		[2000, 0.9, 1.105],
+		[4000, 0.88, 1.05],
+		[8000 * 1.006, 0.73, 1.24],
+		[16000 * 0.89, 0.7, 1.26],  // peak
+		[16000 * 1.13, 0.82, 1.09],  // peak
+		[16000 * 1.205, 0, 1.9]  // shelf
+	];
+
+	try {
+		var gains = jread('au_eq_gain', r.gains);
+		if (r.gains.length == gains.length)
+			r.gains = gains;
+	}
+	catch (ex) { }
+
+	r.draw = function () {
+		jwrite('au_eq_gain', r.gains);
+
+		var txt = QSA('input.eq_gain');
+		for (var a = 0; a < r.bands.length; a++)
+			txt[a].value = r.gains[a];
+	};
+
+	r.apply = function () {
+		r.draw();
+
+		var Ctx = window.AudioContext || window.webkitAudioContext;
+		if (!Ctx)
+			bcfg_set('au_eq', false);
+
+		if (!Ctx || !mp.au)
+			return;
+
+		if (!r.en && !mp.ac)
+			return;
+
+		if (mp.ac) {
+			for (var a = 0; a < r.filters.length; a++)
+				r.filters[a].disconnect();
+
+			mp.acs.disconnect();
+		}
+
+		if (!mp.ac || mp.au != r.last_au) {
+			if (mp.ac)
+				mp.ac.close();
+
+			r.last_au = mp.au;
+			mp.ac = new Ctx();
+			mp.acs = mp.ac.createMediaElementSource(mp.au);
+		}
+
+		r.filters = [];
+
+		if (!r.en) {
+			mp.acs.connect(mp.ac.destination);
+			return;
+		}
+
+		var max = 0;
+		for (var a = 0; a < r.gains.length; a++)
+			if (max < r.gains[a])
+				max = r.gains[a];
+
+		var gains = []
+		for (var a = 0; a < r.gains.length; a++)
+			gains.push(r.gains[a] - max);
+
+		var t = gains[gains.length - 1];
+		gains.push(t);
+		gains.push(t);
+		gains.unshift(gains[0]);
+
+		for (var a = 0; a < cfg.length; a++) {
+			var fi = mp.ac.createBiquadFilter();
+			fi.frequency.value = cfg[a][0];
+			fi.gain.value = cfg[a][2] * gains[a];
+			fi.Q.value = cfg[a][1];
+			fi.type = a == 0 ? 'lowshelf' : a == cfg.length - 1 ? 'highshelf' : 'peaking';
+			r.filters.push(fi);
+		}
+		for (var a = r.filters.length - 1; a >= 0; a--) {
+			r.filters[a].connect(a > 0 ? r.filters[a - 1] : mp.ac.destination);
+		}
+		fi = mp.ac.createGain();
+		fi.gain.value = '0.94';  // +.137 dB measured; now -.25 dB and almost bitperfect
+		mp.acs.connect(fi);
+		fi.connect(r.filters[r.filters.length - 1]);
+		r.filters.push(fi);
+	}
+
+	function eq_step(e) {
+		ev(e);
+		var band = parseInt(this.getAttribute('band')),
+			step = parseFloat(this.getAttribute('step'));
+
+		r.gains[band] += step;
+		r.apply();
+	}
+
+	function adj_band(that, step) {
+		try {
+			var band = parseInt(that.getAttribute('band')),
+				v = parseFloat(that.value);
+
+			if (isNaN(v))
+				throw 42;
+
+			r.gains[band] = v + step;
+		}
+		catch (ex) {
+			return;
+		}
+		r.apply();
+	}
+
+	function eq_mod(e) {
+		ev(e);
+		adj_band(this, 0);
+	}
+
+	function eq_keydown(e) {
+		var step = e.key == 'ArrowUp' ? 0.25 : e.key == 'ArrowDown' ? -0.25 : 0;
+		if (step != 0)
+			adj_band(this, step);
+	}
+
+	var html = ['<table><tr><td rowspan="4">',
+		'<a id="au_eq" class="tgl btn" href="#">enable</a></td>'],
+		h2 = [], h3 = [], h4 = [];
+
+	for (var a = 0; a < r.bands.length; a++) {
+		var hz = r.bands[a];
+		if (hz >= 1000)
+			hz = (hz / 1000) + 'k';
+
+		hz = (hz + '').split('.')[0];
+		html.push('<td><a href="#" class="eq_step" step="0.5" band="' + a + '">+</a></td>');
+		h2.push('<td>' + hz + '</td>');
+		h4.push('<td><a href="#" class="eq_step" step="-0.5" band="' + a + '">&ndash;</a></td>');
+		h3.push('<td><input type="text" class="eq_gain" band="' + a + '" value="' + r.gains[a] + '" /></td>');
+	}
+	html = html.join('\n') + '</tr><tr>';
+	html += h2.join('\n') + '</tr><tr>';
+	html += h3.join('\n') + '</tr><tr>';
+	html += h4.join('\n') + '</tr><table>';
+	ebi('audio_eq').innerHTML = html;
+
+	var stp = QSA('a.eq_step');
+	for (var a = 0, aa = stp.length; a < aa; a++)
+		stp[a].onclick = eq_step;
+
+	var txt = QSA('input.eq_gain');
+	for (var a = 0; a < r.gains.length; a++) {
+		txt[a].oninput = eq_mod;
+		txt[a].onkeydown = eq_keydown;
+	}
+
+	r.en = bcfg_get('au_eq', false);
+	ebi('au_eq').onclick = function (e) {
+		ev(e);
+		r.en = !r.en;
+		bcfg_set('au_eq', r.en);
+		r.apply();
+	};
+
+	r.draw();
+	return r;
+})();
+
+
 // plays the tid'th audio file on the page
 function play(tid, seek, call_depth) {
 	if (mp.order.length == 0)
@@ -518,11 +739,25 @@ function play(tid, seek, call_depth) {
 	if ((tn + '').indexOf('f-') === 0)
 		tn = mp.order.indexOf(tn);
 
-	while (tn >= mp.order.length)
+	if (tn >= mp.order.length) {
-		tn -= mp.order.length;
+		if (mpl.pb_mode == 'loop-folder') {
+			tn = 0;
+		}
+		else if (mpl.pb_mode == 'next-folder') {
+			treectl.ls_cb = function () { song_skip(1); };
+			return tree_neigh(1);
+		}
+	}
 
-	while (tn < 0)
+	if (tn < 0) {
-		tn += mp.order.length;
+		if (mpl.pb_mode == 'loop-folder') {
+			tn = mp.order.length - 1;
+		}
+		else if (mpl.pb_mode == 'next-folder') {
+			treectl.ls_cb = function () { song_skip(-1); };
+			return tree_neigh(-1);
+		}
+	}
 
 	tid = mp.order[tn];
 
@@ -569,6 +804,8 @@ function play(tid, seek, call_depth) {
 		mp.au = mp.au_native;
 	}
 
+	audio_eq.apply();
+
 	mp.au.tid = tid;
 	mp.au.src = url;
 	mp.au.volume = mp.expvol();
@@ -710,8 +947,9 @@ function autoplay_blocked(seek) {
 
 
 var thegrid = (function () {
-	var lfiles = ebi('files');
+	var lfiles = ebi('files'),
-	var gfiles = document.createElement('div');
+		gfiles = document.createElement('div');
+
 	gfiles.setAttribute('id', 'gfiles');
 	gfiles.style.display = 'none';
 	gfiles.innerHTML = (
@@ -733,7 +971,8 @@ var thegrid = (function () {
 		'en': bcfg_get('griden', false),
 		'sel': bcfg_get('gridsel', false),
 		'sz': fcfg_get('gridsz', 10),
-		'isdirty': true
+		'isdirty': true,
+		'bbox': null
 	};
 
 	ebi('thumbs').onclick = function (e) {
@@ -858,14 +1097,15 @@ var thegrid = (function () {
 				href = esc(ao.getAttribute('href')),
 				ref = ao.getAttribute('id'),
 				isdir = href.split('?')[0].slice(-1)[0] == '/',
+				ac = isdir ? ' class="dir"' : '',
 				ihref = href;
 
-			if (isdir) {
+			if (r.thumbs) {
-				ihref = '/.cpr/ico/folder'
-			}
-			else if (r.thumbs) {
 				ihref += (ihref.indexOf('?') === -1 ? '?' : '&') + 'th=' + (have_webp ? 'w' : 'j');
 			}
+			else if (isdir) {
+				ihref = '/.cpr/ico/folder';
+			}
 			else {
 				var ar = href.split('?')[0].split('.');
 				if (ar.length > 1)
@@ -886,14 +1126,42 @@ var thegrid = (function () {
 			}
 
 			html.push('<a href="' + href + '" ref="' + ref + '"><img src="' +
-				ihref + '" /><span>' + ao.innerHTML + '</span></a>');
+				ihref + '" /><span' + ac + '>' + ao.innerHTML + '</span></a>');
 		}
 		lfiles.style.display = 'none';
 		gfiles.style.display = 'block';
 		ebi('ggrid').innerHTML = html.join('\n');
+		r.bagit();
 		r.loadsel();
 	}
 
+	r.bagit = function () {
+		if (!window.baguetteBox)
+			return;
+
+		if (r.bbox)
+			baguetteBox.destroy();
+
+		r.bbox = baguetteBox.run('#ggrid', {
+			captions: function (g) {
+				var idx = -1,
+					h = '' + g;
+
+				for (var a = 0; a < r.bbox.length; a++)
+					if (r.bbox[a].imageElement == g)
+						idx = a;
+
+				return '<a download href="' + h +
+					'">' + (idx + 1) + ' / ' + r.bbox.length + ' -- ' +
+					esc(uricom_dec(h.split('/').slice(-1)[0])[0]) + '</a>';
+			}
+		})[0];
+	};
+
+	setTimeout(function () {
+		import_js('/.cpr/baguettebox.js', r.bagit);
+	}, 1);
+
 	if (r.en) {
 		loadgrid();
 	}
@@ -1246,7 +1514,8 @@ document.onkeydown = function (e) {
 
 var treectl = (function () {
 	var treectl = {
-		"hidden": false
+		"hidden": false,
+		"ls_cb": null
 	},
 		entreed = false,
 		fixedpos = false,
@@ -1346,6 +1615,11 @@ var treectl = (function () {
 		onscroll();
 	}
 
+	treectl.goto = function (url, push) {
+		get_tree("", url, true);
+		reqls(url, push);
+	}
+
 	function get_tree(top, dst, rst) {
 		var xhr = new XMLHttpRequest();
 		xhr.top = top;
@@ -1540,6 +1814,12 @@ var treectl = (function () {
 		msel.render();
 		reload_tree();
 		reload_browser();
+
+		var fun = treectl.ls_cb;
+		if (fun) {
+			treectl.ls_cb = null;
+			fun();
+		}
 	}
 
 	function parsetree(res, top) {
@@ -1604,9 +1884,7 @@ var treectl = (function () {
 			return;
 
 		var url = new URL(e.state, "https://" + document.location.host);
-		url = url.pathname;
+		treectl.goto(url.pathname);
-		get_tree("", url, true);
-		reqls(url);
 	};
 
 	if (window.history && history.pushState) {
@@ -1731,17 +2009,34 @@ var filecols = (function () {
 	var add_btns = function () {
 		var ths = QSA('#files th>span');
 		for (var a = 0, aa = ths.length; a < aa; a++) {
-			var th = ths[a].parentElement,
+			var th = ths[a].parentElement;
-				is_hidden = has(hidden, ths[a].textContent);
+			th.innerHTML = '<div class="cfg"><a href="#">-</a></div>' + ths[a].outerHTML;
-
-			th.innerHTML = '<div class="cfg"><a href="#">' +
-				(is_hidden ? '+' : '-') + '</a></div>' + ths[a].outerHTML;
-
 			th.getElementsByTagName('a')[0].onclick = ev_row_tgl;
 		}
 	};
 
+	function hcols_click(e) {
+		ev(e);
+		var t = e.target;
+		if (t.tagName != 'A')
+			return;
+
+		toggle(t.textContent);
+	}
+
 	var set_style = function () {
+		hidden.sort();
+
+		var html = [],
+			hcols = ebi('hcols');
+
+		for (var a = 0; a < hidden.length; a++) {
+			html.push('<a href="#" class="btn">' + esc(hidden[a]) + '</a>');
+		}
+		hcols.previousSibling.style.display = html.length ? 'block' : 'none';
+		hcols.innerHTML = html.join('\n');
+		hcols.onclick = hcols_click;
+
 		add_btns();
 
 		var ohidden = [],
@@ -1766,22 +2061,8 @@ var filecols = (function () {
 			var cls = has(ohidden, a) ? 'min' : '',
 				tds = QSA('#files>tbody>tr>td:nth-child(' + (a + 1) + ')');
 
-			for (var b = 0, bb = tds.length; b < bb; b++) {
+			for (var b = 0, bb = tds.length; b < bb; b++)
 				tds[b].setAttribute('class', cls);
-				if (a < 2)
-					continue;
-
-				if (cls) {
-					if (!tds[b].hasAttribute('html')) {
-						tds[b].setAttribute('html', tds[b].innerHTML);
-						tds[b].innerHTML = '...';
-					}
-				}
-				else if (tds[b].hasAttribute('html')) {
-					tds[b].innerHTML = tds[b].getAttribute('html');
-					tds[b].removeAttribute('html');
-				}
-			}
 		}
 	};
 	set_style();
@@ -1800,15 +2081,13 @@ var filecols = (function () {
 	try {
 		var ci = find_file_col('dur'),
 			i = ci[0],
-			min = ci[1],
 			rows = ebi('files').tBodies[0].rows;
 
-		if (!min)
+		for (var a = 0, aa = rows.length; a < aa; a++) {
-			for (var a = 0, aa = rows.length; a < aa; a++) {
+			var c = rows[a].cells[i];
-				var c = rows[a].cells[i];
+			if (c && c.textContent)
-				if (c && c.textContent)
+				c.textContent = s2ms(c.textContent);
-					c.textContent = s2ms(c.textContent);
+		}
-			}
 	}
 	catch (ex) { }
 

copyparty/web/dbg-audio.js

@@ -0,0 +1,61 @@
+var ofun = audio_eq.apply.bind(audio_eq);
+audio_eq.apply = function () {
+    var ac1 = mp.ac;
+    ofun();
+    var ac = mp.ac,
+        w = 2048,
+        h = 256;
+
+    if (!audio_eq.filters.length) {
+        audio_eq.ana = null;
+        return;
+    }
+
+    var can = ebi('fft_can');
+    if (!can) {
+        can = mknod('canvas');
+        can.setAttribute('id', 'fft_can');
+        can.style.cssText = 'position:absolute;left:0;bottom:5em;width:' + w + 'px;height:' + h + 'px;z-index:9001';
+        document.body.appendChild(can);
+        can.width = w;
+        can.height = h;
+    }
+    var cc = can.getContext('2d');
+    if (!ac)
+        return;
+
+    var ana = ac.createAnalyser();
+    ana.smoothingTimeConstant = 0;
+    ana.fftSize = 8192;
+
+    audio_eq.filters[0].connect(ana);
+    audio_eq.ana = ana;
+
+    var buf = new Uint8Array(ana.frequencyBinCount),
+        colw = can.width / buf.length;
+
+    cc.fillStyle = '#fc0';
+    function draw() {
+        if (ana == audio_eq.ana)
+            requestAnimationFrame(draw);
+
+        ana.getByteFrequencyData(buf);
+
+        cc.clearRect(0, 0, can.width, can.height);
+
+        /*var x = 0, w = 1;
+        for (var a = 0; a < buf.length; a++) {
+            cc.fillRect(x, h - buf[a], w, h);
+            x += w;
+        }*/
+        var mul = Math.pow(w, 4) / buf.length;
+        for (var x = 0; x < w; x++) {
+            var a = Math.floor(Math.pow(x, 4) / mul),
+                v = buf[a];
+
+            cc.fillRect(x, h - v, 1, v);
+        }
+    }
+    draw();
+};
+audio_eq.apply();

copyparty/web/splash.css

@@ -26,10 +26,23 @@ a {
 	border-radius: .2em;
 	padding: .2em .8em;
 }
-td, th {
+table {
+	border-collapse: collapse;
+}
+.vols td,
+.vols th {
 	padding: .3em .6em;
 	text-align: left;
 }
+.num {
+	border-right: 1px solid #bbb;
+}
+.num td {
+	padding: .1em .7em .1em 0;
+}
+.num td:first-child {
+	text-align: right;
+}
 .btns {
 	margin: 1em 0;
 }
@@ -58,3 +71,6 @@ html.dark input {
 	padding: .5em .7em;
 	margin: 0 .5em 0 0;
 }
+html.dark .num {
+	border-color: #777;
+}

copyparty/web/splash.html

@@ -15,16 +15,25 @@
 
         {%- if avol %}
         <h1>admin panel:</h1>
-        <table>
+        <table><tr><td> <!-- hehehe -->
-            <thead><tr><th>vol</th><th>action</th><th>status</th></tr></thead>
+            <table class="num">
-            <tbody>
+                <tr><td>scanning</td><td>{{ scanning }}</td></tr>
-                {% for mp in avol %}
+                <tr><td>hash-q</td><td>{{ hashq }}</td></tr>
-                {%- if mp in vstate and vstate[mp] %}
+                <tr><td>tag-q</td><td>{{ tagq }}</td></tr>
-                <tr><td><a href="{{ mp }}{{ url_suf }}">{{ mp }}</a></td><td><a href="{{ mp }}?scan">rescan</a></td><td>{{ vstate[mp] }}</td></tr>
+                <tr><td>mtp-q</td><td>{{ mtpq }}</td></tr>
-                {%- endif %}
+            </table>
-                {% endfor %}
+        </td><td>
-            </tbody>
+            <table class="vols">
-        </table>
+                <thead><tr><th>vol</th><th>action</th><th>status</th></tr></thead>
+                <tbody>
+                    {% for mp in avol %}
+                    {%- if mp in vstate and vstate[mp] %}
+                    <tr><td><a href="{{ mp }}{{ url_suf }}">{{ mp }}</a></td><td><a href="{{ mp }}?scan">rescan</a></td><td>{{ vstate[mp] }}</td></tr>
+                    {%- endif %}
+                    {% endfor %}
+                </tbody>
+            </table>
+        </td></tr></table>
         <div class="btns">
             <a href="{{ avol[0] }}?stack">dump stack</a>
         </div>
@@ -50,7 +59,7 @@
 
         <h1>login for more:</h1>
         <ul>
-            <form method="post" enctype="multipart/form-data" action="/{{ url_suf }}">
+            <form method="post" enctype="multipart/form-data" action="/">
                 <input type="hidden" name="act" value="login" />
                 <input type="password" name="cppwd" />
                 <input type="submit" value="Login" />

copyparty/web/up2k.js

@@ -804,6 +804,14 @@ function up2k_init(subtle) {
 
                 var mou_ikkai = false;
 
+                if (st.busy.handshake.length > 0 &&
+                    st.busy.handshake[0].busied < Date.now() - 30 * 1000
+                ) {
+                    console.log("retrying stuck handshake");
+                    var t = st.busy.handshake.shift();
+                    st.todo.handshake.unshift(t);
+                }
+
                 if (st.todo.handshake.length > 0 &&
                     st.busy.handshake.length == 0 && (
                         st.todo.handshake[0].t4 || (
@@ -1019,11 +1027,27 @@ function up2k_init(subtle) {
     //
 
     function exec_handshake() {
-        var t = st.todo.handshake.shift();
+        var t = st.todo.handshake.shift(),
+            me = Date.now();
+
         st.busy.handshake.push(t);
+        t.busied = me;
 
         var xhr = new XMLHttpRequest();
+        xhr.onerror = function () {
+            if (t.busied != me) {
+                console.log('zombie handshake onerror,', t);
+                return;
+            }
+            console.log('handshake onerror, retrying');
+            st.busy.handshake.splice(st.busy.handshake.indexOf(t), 1);
+            st.todo.handshake.unshift(t);
+        };
         xhr.onload = function (e) {
+            if (t.busied != me) {
+                console.log('zombie handshake onload,', t);
+                return;
+            }
             if (xhr.status == 200) {
                 var response = JSON.parse(xhr.responseText);
 

docs/README.md

@@ -0,0 +1,22 @@
+# example `.epilogue.html`
+save one of these as `.epilogue.html` inside a folder to customize it:
+
+* [`minimal-up2k.html`](minimal-up2k.html) will [simplify the upload ui](https://user-images.githubusercontent.com/241032/118311195-dd6ca380-b4ef-11eb-86f3-75a3ff2e1332.png)
+
+
+
+# example browser-css
+point `--css-browser` to one of these by URL:
+
+* [`browser.css`](browser.css) changes the background
+* [`browser-icons.css`](browser-icons.css) adds filetype icons
+
+
+
+# other stuff
+
+## [`rclone.md`](rclone.md)
+* notes on using rclone as a fuse client/server
+
+## [`example.conf`](example.conf)
+* example config file for `-c` which never really happened

docs/biquad.html

@@ -0,0 +1,95 @@
+<!DOCTYPE html><html><head></head><body><script>
+
+setTimeout(location.reload.bind(location), 700);
+document.documentElement.scrollLeft = 0;
+
+var can = document.createElement('canvas'),
+    cc = can.getContext('2d'),
+    w = 2048,
+    h = 1024;
+
+w = 2048;
+
+can.width = w;
+can.height = h;
+document.body.appendChild(can);
+can.style.cssText = 'width:' + w + 'px;height:' + h + 'px';
+
+cc.fillStyle = '#000';
+cc.fillRect(0, 0, w, h);
+
+var cfg = [ // hz, q, g
+    [31.25 * 0.88, 0, 1.4],  // shelf
+    [31.25 * 1.04, 0.7, 0.96],  // peak
+    [62.5, 0.7, 1],
+    [125, 0.8, 1],
+    [250, 0.9, 1.03],
+    [500, 0.9, 1.1],
+    [1000, 0.9, 1.1],
+    [2000, 0.9, 1.105],
+    [4000, 0.88, 1.05],
+    [8000 * 1.006, 0.73, 1.24],
+    //[16000 * 1.00, 0.5, 1.75],  // peak.v1
+    //[16000 * 1.19, 0, 1.8]  // shelf.v1
+    [16000 * 0.89, 0.7, 1.26],  // peak
+    [16000 * 1.13, 0.82, 1.09],  // peak
+    [16000 * 1.205, 0, 1.9]  // shelf
+];
+
+var freqs = new Float32Array(22000),
+    sum = new Float32Array(freqs.length),
+    ac = new AudioContext(),
+    step = w / freqs.length,
+    colors = [
+        'rgba(255, 0, 0, 0.7)',
+        'rgba(0, 224, 0, 0.7)',
+        'rgba(0, 64, 255, 0.7)'
+    ];
+
+var order = [];
+
+for (var a = 0; a < cfg.length; a += 2)
+    order.push(a);
+
+for (var a = 1; a < cfg.length; a += 2)
+    order.push(a);
+
+for (var ia = 0; ia < order.length; ia++) {
+    var a = order[ia],
+        fi = ac.createBiquadFilter(),
+        mag = new Float32Array(freqs.length),
+        phase = new Float32Array(freqs.length);
+
+    for (var b = 0; b < freqs.length; b++)
+        freqs[b] = b;
+
+    fi.type = a == 0 ? 'lowshelf' : a == cfg.length - 1 ? 'highshelf' : 'peaking';
+    fi.frequency.value = cfg[a][0];
+    fi.Q.value = cfg[a][1];
+    fi.gain.value = 1;
+
+    fi.getFrequencyResponse(freqs, mag, phase);
+    cc.fillStyle = colors[a % colors.length];
+    for (var b = 0; b < sum.length; b++) {
+        mag[b] -= 1;
+        sum[b] += mag[b] * cfg[a][2];
+        var y = h - (mag[b] * h * 3);
+        cc.fillRect(b * step, y, step, h - y);
+        cc.fillRect(b * step - 1, y - 1, 3, 3);
+    }
+}
+
+var min = 999999, max = 0;
+for (var a = 0; a < sum.length; a++) {
+    min = Math.min(min, sum[a]);
+    max = Math.max(max, sum[a]);
+}
+cc.fillStyle = 'rgba(255,255,255,1)';
+for (var a = 0; a < sum.length; a++) {
+    var v = (sum[a] - min) / (max - min);
+    cc.fillRect(a * step, 0, step, v * h / 2);
+}
+
+cc.fillRect(0, 460, w, 1);
+
+</script></body></html>

docs/browser-icons.css

@@ -0,0 +1,68 @@
+/* put filetype icons inline with text
+#ggrid>a>span:before,
+#ggrid>a>span.dir:before {
+	display: inline;
+	line-height: 0;
+	font-size: 1.7em;
+	margin: -.7em .1em -.5em -.6em;
+}
+*/
+
+
+/* move folder icons top-left */
+#ggrid>a>span.dir:before {
+	content: initial;
+}
+#ggrid>a[href$="/"]:before {
+	content: '📂';
+    display: block;
+    position: absolute;
+    margin: -.1em -.4em;
+    text-shadow: 0 0 .1em #000;
+    font-size: 2em;
+}
+
+
+/* put filetype icons top-left */
+#ggrid>a:before {
+    display: block;
+    position: absolute;
+    margin: -.1em -.4em;
+    text-shadow: 0 0 .1em #000;
+    font-size: 2em;
+}
+
+
+/* video */
+#ggrid>a:is(
+[href$=".mkv"i],
+[href$=".mp4"i],
+[href$=".webm"i],
+):before {
+    content: '📺';
+}
+
+
+/* audio */
+#ggrid>a:is(
+[href$=".mp3"i],
+[href$=".ogg"i],
+[href$=".opus"i],
+[href$=".flac"i],
+[href$=".m4a"i],
+[href$=".aac"i],
+):before {
+    content: '🎵';
+}
+
+
+/* image */
+#ggrid>a:is(
+[href$=".jpg"i],
+[href$=".jpeg"i],
+[href$=".png"i],
+[href$=".gif"i],
+[href$=".webp"i],
+):before {
+    content: '🎨';
+}

docs/browser.css

@@ -0,0 +1,29 @@
+html {
+    background: #333 url('/wp/wallhaven-mdjrqy.jpg') center / cover no-repeat fixed;
+}
+#files th {
+    background: rgba(32, 32, 32, 0.9) !important;
+}
+#ops,
+#treeul,
+#files td {
+    background: rgba(32, 32, 32, 0.3) !important;
+}
+
+
+html.light {
+    background: #eee url('/wp/wallhaven-dpxl6l.png') center / cover no-repeat fixed;
+}
+html.light #files th {
+    background: rgba(255, 255, 255, 0.9) !important;
+}
+html.light #ops,
+html.light #treeul,
+html.light #files td {
+    background: rgba(248, 248, 248, 0.8) !important;
+}
+
+
+#files * {
+    background: transparent !important;
+}

docs/notes.sh

@@ -86,6 +86,9 @@ var t=[]; var b=document.location.href.split('#')[0].slice(0, -1); document.quer
 # get the size and video-id of all youtube vids in folder, assuming filename ends with -id.ext, and create a copyparty search query
 find -maxdepth 1 -printf '%s %p\n' | sort -n | awk '!/-([0-9a-zA-Z_-]{11})\.(mkv|mp4|webm)$/{next} {sub(/\.[^\.]+$/,"");n=length($0);v=substr($0,n-10);print $1, v}' | tee /dev/stderr | awk 'BEGIN {p="("} {printf("%s name like *-%s.* ",p,$2);p="or"} END {print ")\n"}' | cat >&2
 
+# unique stacks in a stackdump
+f=a; rm -rf stacks; mkdir stacks; grep -E '^#' $f | while IFS= read -r n; do awk -v n="$n" '!$0{o=0} o; $0==n{o=1}' <$f >stacks/f; h=$(sha1sum <stacks/f | cut -c-16); mv stacks/f stacks/$h-"$n"; done ; find stacks/ | sort | uniq -cw24
+
 
 ##
 ## sqlite3 stuff
@@ -153,6 +156,9 @@ dbg.asyncStore.pendingBreakpoints = {}
 # fix firefox phantom breakpoints
 about:config >> devtools.debugger.prefs-schema-version = -1
 
+# determine server version
+git reset --hard origin/HEAD && git log --format=format:"%H %ai %d" --decorate=full > /dev/shm/revs && cat /dev/shm/revs | while read -r rev extra; do (git reset --hard $rev >/dev/null 2>/dev/null && dsz=$(cat copyparty/web/{util,browser,up2k}.js 2>/dev/null | diff -wNarU0 - <(cat /mnt/Users/ed/Downloads/ref/{util,browser,up2k}.js) | wc -c) && printf '%s %6s %s\n' "$rev" $dsz "$extra") </dev/null; done                
+
 
 ##
 ## http 206

docs/tcp-debug.sh

@@ -0,0 +1,32 @@
+(cd ~/dev/copyparty && strace -Tttyyvfs 256 -o strace.strace python3 -um copyparty -i 127.0.0.1 --http-only --stackmon /dev/shm/cpps,10 ) 2>&1 | tee /dev/stderr > ~/log-copyparty-$(date +%Y-%m%d-%H%M%S).txt
+
+14/Jun/2021:16:34:02 1623688447.212405 death
+14/Jun/2021:16:35:02 1623688502.420860 back
+
+tcpdump -nni lo -w /home/ed/lo.pcap
+
+# 16:35:25.324662 IP 127.0.0.1.48632 > 127.0.0.1.3920: Flags [F.], seq 849, ack 544, win 359, options [nop,nop,TS val 809396796 ecr 809396796], length 0
+
+tcpdump -nnr /home/ed/lo.pcap | awk '/ > 127.0.0.1.3920: /{sub(/ > .*/,"");sub(/.*\./,"");print}' | sort -n | uniq | while IFS= read -r port; do echo; tcpdump -nnr /home/ed/lo.pcap 2>/dev/null | grep -E "\.$port( > |: F)" | sed -r 's/ > .*, /, /'; done | grep -E '^16:35:0.*length [^0]' -C50
+
+16:34:02.441732 IP 127.0.0.1.48638, length 0
+16:34:02.441738 IP 127.0.0.1.3920, length 0
+16:34:02.441744 IP 127.0.0.1.48638, length 0
+16:34:02.441756 IP 127.0.0.1.48638, length 791
+16:34:02.441759 IP 127.0.0.1.3920, length 0
+16:35:02.445529 IP 127.0.0.1.48638, length 0
+16:35:02.489194 IP 127.0.0.1.3920, length 0
+16:35:02.515595 IP 127.0.0.1.3920, length 216
+16:35:02.515600 IP 127.0.0.1.48638, length 0
+
+grep 48638 "$(find ~ -maxdepth 1 -name log-copyparty-\*.txt | sort | tail -n 1)"
+
+1623688502.510380 48638 rh
+1623688502.511291 48638 Unrecv direct ...
+1623688502.511827 48638 rh = 791
+16:35:02.518 127.0.0.1 48638       shut(8): [Errno 107] Socket not connected
+Exception in thread httpsrv-0.1-48638:
+
+grep 48638 ~/dev/copyparty/strace.strace
+14561 16:35:02.506310 <... accept4 resumed> {sa_family=AF_INET, sin_port=htons(48638), sin_addr=inet_addr("127.0.0.1")}, [16], SOCK_CLOEXEC) = 8<TCP:[127.0.0.1:3920->127.0.0.1:48638]> <0.000012>
+15230 16:35:02.510725 write(1<pipe:[256639555]>, "1623688502.510380 48638 rh\n", 27 <unfinished ...>

scripts/make-sfx.sh

@@ -32,6 +32,10 @@ gtar=$(command -v gtar || command -v gnutar) || true
 	[ -e /opt/local/bin/bzip2 ] &&
 		bzip2() { /opt/local/bin/bzip2 "$@"; }
 }
+
+gawk=$(command -v gawk || command -v gnuawk || command -v awk)
+awk() { $gawk "$@"; }
+
 pybin=$(command -v python3 || command -v python) || {
 	echo need python
 	exit 1
@@ -163,7 +167,7 @@ find .. -type f \( -name .DS_Store -or -name ._.DS_Store \) -delete
 find .. -type f -name ._\* | while IFS= read -r f; do cmp <(printf '\x00\x05\x16') <(head -c 3 -- "$f") && rm -f -- "$f"; done
 
 echo use smol web deps
-rm -f copyparty/web/deps/*.full.* copyparty/web/Makefile
+rm -f copyparty/web/deps/*.full.* copyparty/web/dbg-* copyparty/web/Makefile
 
 # it's fine dw
 grep -lE '\.full\.(js|css)' copyparty/web/* |
@@ -194,11 +198,40 @@ tmv "$f"
 
 # up2k goes from 28k to 22k laff
 echo entabbening
-find | grep -E '\.(js|css|html)$' | while IFS= read -r f; do
+find | grep -E '\.css$' | while IFS= read -r f; do
+	awk '{
+		sub(/^[ \t]+/,"");
+		sub(/[ \t]+$/,"");
+		$0=gensub(/^([a-z-]+) *: *(.*[^ ]) *;$/,"\\1:\\2;","1");
+		sub(/ +\{$/,"{");
+		gsub(/, /,",")
+	}
+	!/\}$/ {printf "%s",$0;next}
+	1
+	' <$f | sed 's/;\}$/}/' >t
+	tmv "$f"
+done
+find | grep -E '\.(js|html)$' | while IFS= read -r f; do
 	unexpand -t 4 --first-only <"$f" >t
 	tmv "$f"
 done
 
+
+gzres() {
+command -v pigz &&
+	pk='pigz -11 -J 34 -I 100' ||
+	pk='gzip'
+
+echo "$pk"
+find | grep -E '\.(js|css)$' | grep -vF /deps/ | while IFS= read -r f; do
+	echo -n .
+	$pk "$f"
+done
+echo
+}
+gzres
+
+
 echo gen tarlist
 for d in copyparty dep-j2; do find $d -type f; done |
 sed -r 's/(.*)\.(.*)/\2 \1/' | LC_ALL=C sort |

scripts/run-tests.sh

@@ -3,10 +3,13 @@ set -ex
 
 pids=()
 for py in python{2,3}; do
-    $py -m unittest discover -s tests >/dev/null &
+    nice $py -m unittest discover -s tests >/dev/null &
     pids+=($!)
 done
 
+python3 scripts/test/smoketest.py &
+pids+=($!)
+
 for pid in ${pids[@]}; do
     wait $pid
 done

scripts/test/smoketest.py

@@ -0,0 +1,209 @@
+import os
+import sys
+import time
+import shlex
+import shutil
+import signal
+import tempfile
+import requests
+import threading
+import subprocess as sp
+
+
+CPP = []
+
+
+class Cpp(object):
+    def __init__(self, args):
+        args = [sys.executable, "-m", "copyparty"] + args
+        print(" ".join([shlex.quote(x) for x in args]))
+
+        self.ls_pre = set(list(os.listdir()))
+        self.p = sp.Popen(args)
+        # , stdout=sp.PIPE, stderr=sp.PIPE)
+
+        self.t = threading.Thread(target=self._run)
+        self.t.daemon = True
+        self.t.start()
+
+    def _run(self):
+        self.so, self.se = self.p.communicate()
+
+    def stop(self, wait):
+        if wait:
+            os.kill(self.p.pid, signal.SIGINT)
+            self.t.join(timeout=2)
+        else:
+            self.p.kill()  # macos py3.8
+
+    def clean(self):
+        t = os.listdir()
+        for f in t:
+            if f not in self.ls_pre and f.startswith("up."):
+                os.unlink(f)
+
+    def await_idle(self, ub, timeout):
+        req = ["scanning</td><td>False", "hash-q</td><td>0", "tag-q</td><td>0"]
+        lim = int(timeout * 10)
+        u = ub + "?h"
+        for n in range(lim):
+            try:
+                time.sleep(0.1)
+                r = requests.get(u, timeout=0.1)
+                for x in req:
+                    if x not in r.text:
+                        print("ST: {}/{} miss {}".format(n, lim, x))
+                        raise Exception()
+                print("ST: idle")
+                return
+            except:
+                pass
+
+
+def tc1():
+    ub = "http://127.0.0.1:4321/"
+    td = os.path.join("srv", "smoketest")
+    try:
+        shutil.rmtree(td)
+    except:
+        if os.path.exists(td):
+            raise
+
+    for _ in range(10):
+        try:
+            os.mkdir(td)
+        except:
+            time.sleep(0.1)  # win10
+
+    assert os.path.exists(td)
+
+    vidp = os.path.join(tempfile.gettempdir(), "smoketest.h264")
+    if not os.path.exists(vidp):
+        cmd = "ffmpeg -f lavfi -i testsrc=48x32:3 -t 1 -c:v libx264 -tune animation -preset veryslow -crf 69"
+        sp.check_call(cmd.split(" ") + [vidp])
+
+    with open(vidp, "rb") as f:
+        ovid = f.read()
+
+    args = [
+        "-p4321",
+        "-e2dsa",
+        "-e2tsr",
+        "--no-mutagen",
+        "--th-ff-jpg",
+        "--hist",
+        os.path.join(td, "dbm"),
+    ]
+    pdirs = []
+    hpaths = {}
+
+    for d1 in ["r", "w", "a"]:
+        pdirs.append("{}/{}".format(td, d1))
+        pdirs.append("{}/{}/j".format(td, d1))
+        for d2 in ["r", "w", "a"]:
+            d = os.path.join(td, d1, "j", d2)
+            pdirs.append(d)
+            os.makedirs(d)
+
+    pdirs = [x.replace("\\", "/") for x in pdirs]
+    udirs = [x.split("/", 2)[2] for x in pdirs]
+    perms = [x.rstrip("j/")[-1] for x in pdirs]
+    for pd, ud, p in zip(pdirs, udirs, perms):
+        if ud[-1] == "j":
+            continue
+
+        hp = None
+        if pd.endswith("st/a"):
+            hp = hpaths[ud] = os.path.join(td, "db1")
+        elif pd[:-1].endswith("a/j/"):
+            hpaths[ud] = os.path.join(td, "dbm")
+            hp = None
+        else:
+            hp = "-"
+            hpaths[ud] = os.path.join(pd, ".hist")
+
+        arg = "{}:{}:{}".format(pd, ud, p, hp)
+        if hp:
+            arg += ":chist=" + hp
+
+        args += ["-v", arg]
+
+    # return
+    cpp = Cpp(args)
+    CPP.append(cpp)
+    cpp.await_idle(ub, 3)
+
+    for d in udirs:
+        vid = ovid + "\n{}".format(d).encode("utf-8")
+        try:
+            requests.post(ub + d, data={"act": "bput"}, files={"f": ("a.h264", vid)})
+        except:
+            pass
+
+    cpp.clean()
+
+    # GET permission
+    for d, p in zip(udirs, perms):
+        u = "{}{}/a.h264".format(ub, d)
+        r = requests.get(u)
+        ok = bool(r)
+        if ok != (p in ["a"]):
+            raise Exception("get {} with perm {} at {}".format(ok, p, u))
+
+    # stat filesystem
+    for d, p in zip(pdirs, perms):
+        u = "{}/a.h264".format(d)
+        ok = os.path.exists(u)
+        if ok != (p in ["a", "w"]):
+            raise Exception("stat {} with perm {} at {}".format(ok, p, u))
+
+    # GET thumbnail, vreify contents
+    for d, p in zip(udirs, perms):
+        u = "{}{}/a.h264?th=j".format(ub, d)
+        r = requests.get(u)
+        ok = bool(r and r.content[:3] == b"\xff\xd8\xff")
+        if ok != (p in ["a"]):
+            raise Exception("thumb {} with perm {} at {}".format(ok, p, u))
+
+    # check tags
+    cpp.await_idle(ub, 5)
+    for d, p in zip(udirs, perms):
+        u = "{}{}?ls".format(ub, d)
+        r = requests.get(u)
+        j = r.json() if r else False
+        tag = None
+        if j:
+            for f in j["files"]:
+                tag = tag or f["tags"].get("res")
+
+        r_ok = bool(j)
+        w_ok = bool(r_ok and j.get("files"))
+
+        if not r_ok or w_ok != (p in ["a"]):
+            raise Exception("ls {} with perm {} at {}".format(ok, p, u))
+
+        if (tag and p != "a") or (not tag and p == "a"):
+            raise Exception("tag {} with perm {} at {}".format(tag, p, u))
+
+        if tag is not None and tag != "48x32":
+            raise Exception("tag [{}] at {}".format(tag, u))
+
+    cpp.stop(True)
+
+
+def run(tc):
+    try:
+        tc()
+    finally:
+        try:
+            CPP[0].stop(False)
+        except:
+            pass
+
+
+def main():
+    run(tc1)
+
+
+if __name__ == "__main__":
+    main()

tests/test_httpcli.py

@@ -28,6 +28,7 @@ class Cfg(Namespace):
             a=a,
             v=v,
             c=c,
+            rproxy=0,
             ed=False,
             no_zip=False,
             no_scandir=False,
@@ -37,6 +38,9 @@ class Cfg(Namespace):
             nih=True,
             mtp=[],
             mte="a",
+            hist=None,
+            no_hash=False,
+            css_browser=None,
             **{k: False for k in "e2d e2ds e2dsa e2t e2ts e2tsr".split()}
         )
 
@@ -99,7 +103,7 @@ class TestHttpCli(unittest.TestCase):
             pprint.pprint(vcfg)
 
             self.args = Cfg(v=vcfg, a=["o:o", "x:x"])
-            self.auth = AuthSrv(self.args, self.log)
+            self.asrv = AuthSrv(self.args, self.log)
             vfiles = [x for x in allfiles if x.startswith(top)]
             for fp in vfiles:
                 rok, wok = self.can_rw(fp)
@@ -188,12 +192,12 @@ class TestHttpCli(unittest.TestCase):
     def put(self, url):
         buf = "PUT /{0} HTTP/1.1\r\nCookie: cppwd=o\r\nConnection: close\r\nContent-Length: {1}\r\n\r\nok {0}\n"
         buf = buf.format(url, len(url) + 4).encode("utf-8")
-        conn = tu.VHttpConn(self.args, self.auth, self.log, buf)
+        conn = tu.VHttpConn(self.args, self.asrv, self.log, buf)
         HttpCli(conn).run()
         return conn.s._reply.decode("utf-8").split("\r\n\r\n", 1)
 
     def curl(self, url, binary=False):
-        conn = tu.VHttpConn(self.args, self.auth, self.log, hdr(url))
+        conn = tu.VHttpConn(self.args, self.asrv, self.log, hdr(url))
         HttpCli(conn).run()
         if binary:
             h, b = conn.s._reply.split(b"\r\n\r\n", 1)

tests/test_vfs.py

@@ -18,8 +18,15 @@ from copyparty import util
 class Cfg(Namespace):
     def __init__(self, a=[], v=[], c=None):
         ex = {k: False for k in "e2d e2ds e2dsa e2t e2ts e2tsr".split()}
-        ex["mtp"] = []
+        ex2 = {
-        ex["mte"] = "a"
+            "mtp": [],
+            "mte": "a",
+            "hist": None,
+            "no_hash": False,
+            "css_browser": None,
+            "rproxy": 0,
+        }
+        ex.update(ex2)
         super(Cfg, self).__init__(a=a, v=v, c=c, **ex)
 
 
@@ -113,13 +120,13 @@ class TestVFS(unittest.TestCase):
         n = vfs.nodes["a"]
         self.assertEqual(len(vfs.nodes), 1)
         self.assertEqual(n.vpath, "a")
-        self.assertEqual(n.realpath, td + "/a")
+        self.assertEqual(n.realpath, os.path.join(td, "a"))
         self.assertEqual(n.uread, ["*", "k"])
         self.assertEqual(n.uwrite, ["k"])
         n = n.nodes["ac"]
         self.assertEqual(len(vfs.nodes), 1)
         self.assertEqual(n.vpath, "a/ac")
-        self.assertEqual(n.realpath, td + "/a/ac")
+        self.assertEqual(n.realpath, os.path.join(td, "a", "ac"))
         self.assertEqual(n.uread, ["*", "k"])
         self.assertEqual(n.uwrite, ["k"])
         n = n.nodes["acb"]
@@ -251,7 +258,7 @@ class TestVFS(unittest.TestCase):
         n = au.vfs
         # root was not defined, so PWD with no access to anyone
         self.assertEqual(n.vpath, "")
-        self.assertEqual(n.realpath, td)
+        self.assertEqual(n.realpath, None)
         self.assertEqual(n.uread, [])
         self.assertEqual(n.uwrite, [])
         self.assertEqual(len(n.nodes), 1)

tests/util.py

@@ -60,7 +60,7 @@ def get_ramdisk():
 
     if os.path.exists("/Volumes"):
         # hdiutil eject /Volumes/cptd/
-        devname, _ = chkcmd("hdiutil", "attach", "-nomount", "ram://65536")
+        devname, _ = chkcmd("hdiutil", "attach", "-nomount", "ram://131072")
         devname = devname.strip()
         print("devname: [{}]".format(devname))
         for _ in range(10):
@@ -110,12 +110,13 @@ class VHttpSrv(object):
 
 
 class VHttpConn(object):
-    def __init__(self, args, auth, log, buf):
+    def __init__(self, args, asrv, log, buf):
         self.s = VSock(buf)
         self.sr = Unrecv(self.s)
         self.addr = ("127.0.0.1", "42069")
         self.args = args
-        self.auth = auth
+        self.asrv = asrv
+        self.is_mp = False
         self.log_func = log
         self.log_src = "a"
         self.lf_url = None