v0.11.40

README.md

@@ -46,6 +46,7 @@ turn your phone or raspi into a portable file server with resumable uploads/down
 * [browser support](#browser-support)
 * [client examples](#client-examples)
 * [up2k](#up2k)
+* [performance](#performance)
 * [dependencies](#dependencies)
     * [optional dependencies](#optional-dependencies)
     * [install recommended deps](#install-recommended-deps)
@@ -199,10 +200,21 @@ the browser has the following hotkeys
 * `G` toggle list / grid view
 * `T` toggle thumbnails / icons
 * when playing audio:
-  * `0..9` jump to 10%..90%
-  * `U/O` skip 10sec back/forward
   * `J/L` prev/next song
+  * `U/O` skip 10sec back/forward
+  * `0..9` jump to 0%..90%
   * `P` play/pause (also starts playing the folder)
+* when viewing images / playing videos:
+  * `J/L, Left/Right` prev/next file
+  * `Home/End` first/last file
+  * `Esc` close viewer
+  * videos:
+    * `U/O` skip 10sec back/forward
+    * `P/K/Space` play/pause
+    * `F` fullscreen
+    * `C` continue playing next video
+    * `R` loop
+    * `M` mute
 * when tree-sidebar is open:
   * `A/D` adjust tree width
 * in the grid view:
@@ -223,7 +235,7 @@ click `[-]` and `[+]` (or hotkeys `A`/`D`) to adjust the size, and the `[a]` tog
 
 it does static images with Pillow and uses FFmpeg for video files, so you may want to `--no-thumb` or maybe just `--no-vthumb` depending on how destructive your users are
 
-images named `folder.jpg` and `folder.png` become the thumbnail of the folder they're in
+images with the following names (see `--th-covers`) become the thumbnail of the folder they're in: `folder.png`, `folder.jpg`, `cover.png`, `cover.jpg`
 
 in the grid/thumbnail view, if the audio player panel is open, songs will start playing when clicked
 
@@ -494,6 +506,23 @@ quick outline of the up2k protocol, see [uploading](#uploading) for the web-clie
 * client does another handshake with the hashlist; server replies with OK or a list of chunks to reupload
 
 
+# performance
+
+defaults are good for most cases, don't mind the `cannot efficiently use multiple CPU cores` message, it's very unlikely to be a problem
+
+below are some tweaks roughly ordered by usefulness:
+
+* `-q` disables logging and can help a bunch, even when combined with `-lo` to redirect logs to file
+* `--http-only` or `--https-only` (unless you want to support both protocols) will reduce the delay before a new connection is established
+* `--hist` pointing to a fast location (ssd) will make directory listings and searches faster when `-e2d` or `-e2t` is set
+* `--no-hash` when indexing a networked disk if you don't care about the actual filehashes and only want the names/tags searchable
+* `-j` enables multiprocessing (actual multithreading) and can make copyparty perform better in cpu-intensive workloads, for example:
+  * huge amount of short-lived connections
+  * really heavy traffic (downloads/uploads)
+  
+  ...however it adds an overhead to internal communication so it might be a net loss, see if it works 4 u
+
+
 # dependencies
 
 * `jinja2` (is built into the SFX)

bin/copyparty-fuse-streaming.py

@@ -345,7 +345,7 @@ class Gateway(object):
         except:
             pass
 
-    def sendreq(self, *args, headers={}, **kwargs):
+    def sendreq(self, meth, path, headers, **kwargs):
         if self.password:
             headers["Cookie"] = "=".join(["cppwd", self.password])
 
@@ -354,21 +354,21 @@ class Gateway(object):
             if c.rx_path:
                 raise Exception()
 
-            c.request(*list(args), headers=headers, **kwargs)
+            c.request(meth, path, headers=headers, **kwargs)
             c.rx = c.getresponse()
             return c
         except:
             tid = threading.current_thread().ident
             dbg(
-                "\033[1;37;44mbad conn {:x}\n  {}\n  {}\033[0m".format(
+                "\033[1;37;44mbad conn {:x}\n  {} {}\n  {}\033[0m".format(
-                    tid, " ".join(str(x) for x in args), c.rx_path if c else "(null)"
+                    tid, meth, path, c.rx_path if c else "(null)"
                 )
             )
 
         self.closeconn(c)
         c = self.getconn()
         try:
-            c.request(*list(args), headers=headers, **kwargs)
+            c.request(meth, path, headers=headers, **kwargs)
             c.rx = c.getresponse()
             return c
         except:
@@ -386,7 +386,7 @@ class Gateway(object):
             path = dewin(path)
 
         web_path = self.quotep("/" + "/".join([self.web_root, path])) + "?dots"
-        c = self.sendreq("GET", web_path)
+        c = self.sendreq("GET", web_path, {})
         if c.rx.status != 200:
             self.closeconn(c)
             log(
@@ -440,7 +440,7 @@ class Gateway(object):
             )
         )
 
-        c = self.sendreq("GET", web_path, headers={"Range": hdr_range})
+        c = self.sendreq("GET", web_path, {"Range": hdr_range})
         if c.rx.status != http.client.PARTIAL_CONTENT:
             self.closeconn(c)
             raise Exception(

bin/copyparty-fuse.py

@@ -54,10 +54,13 @@ MACOS = platform.system() == "Darwin"
 info = log = dbg = None
 
 
-print("{} v{} @ {}".format(
+print(
-    platform.python_implementation(),
+    "{} v{} @ {}".format(
-    ".".join([str(x) for x in sys.version_info]),
+        platform.python_implementation(),
-    sys.executable))
+        ".".join([str(x) for x in sys.version_info]),
+        sys.executable,
+    )
+)
 
 
 try:
@@ -299,14 +302,14 @@ class Gateway(object):
         except:
             pass
 
-    def sendreq(self, *args, headers={}, **kwargs):
+    def sendreq(self, meth, path, headers, **kwargs):
         tid = get_tid()
         if self.password:
             headers["Cookie"] = "=".join(["cppwd", self.password])
 
         try:
             c = self.getconn(tid)
-            c.request(*list(args), headers=headers, **kwargs)
+            c.request(meth, path, headers=headers, **kwargs)
             return c.getresponse()
         except:
             dbg("bad conn")
@@ -314,7 +317,7 @@ class Gateway(object):
         self.closeconn(tid)
         try:
             c = self.getconn(tid)
-            c.request(*list(args), headers=headers, **kwargs)
+            c.request(meth, path, headers=headers, **kwargs)
             return c.getresponse()
         except:
             info("http connection failed:\n" + traceback.format_exc())
@@ -331,7 +334,7 @@ class Gateway(object):
             path = dewin(path)
 
         web_path = self.quotep("/" + "/".join([self.web_root, path])) + "?dots&ls"
-        r = self.sendreq("GET", web_path)
+        r = self.sendreq("GET", web_path, {})
         if r.status != 200:
             self.closeconn()
             log(
@@ -368,7 +371,7 @@ class Gateway(object):
             )
         )
 
-        r = self.sendreq("GET", web_path, headers={"Range": hdr_range})
+        r = self.sendreq("GET", web_path, {"Range": hdr_range})
         if r.status != http.client.PARTIAL_CONTENT:
             self.closeconn()
             raise Exception(

contrib/nginx/copyparty.conf

@@ -1,7 +1,15 @@
-# when running copyparty behind a reverse-proxy,
+# when running copyparty behind a reverse proxy,
-# make sure that copyparty allows at least as many clients as the proxy does,
+# the following arguments are recommended:
-# so run copyparty with -nc 512 if your nginx has the default limits
+#
-# (worker_processes 1, worker_connections 512)
+#   -nc 512         important, see next paragraph
+#   --http-only     lower latency on initial connection
+#   -i 127.0.0.1    only accept connections from nginx
+#
+# -nc must match or exceed the webserver's max number of concurrent clients;
+# nginx default is 512  (worker_processes 1, worker_connections 512)
+#
+# you may also consider adding -j0 for CPU-intensive configurations
+# (not that i can really think of any good examples)
 
 upstream cpp {
 	server 127.0.0.1:3923;

copyparty/__init__.py

@@ -9,6 +9,9 @@ import os
 PY2 = sys.version_info[0] == 2
 if PY2:
     sys.dont_write_bytecode = True
+    unicode = unicode
+else:
+    unicode = str
 
 WINDOWS = False
 if platform.system() == "Windows":

copyparty/__main__.py

@@ -20,10 +20,10 @@ import threading
 import traceback
 from textwrap import dedent
 
-from .__init__ import E, WINDOWS, VT100, PY2
+from .__init__ import E, WINDOWS, VT100, PY2, unicode
 from .__version__ import S_VERSION, S_BUILD_DT, CODENAME
 from .svchub import SvcHub
-from .util import py_desc, align_tab, IMPLICATIONS, alltrace
+from .util import py_desc, align_tab, IMPLICATIONS
 
 HAVE_SSL = True
 try:
@@ -31,6 +31,8 @@ try:
 except:
     HAVE_SSL = False
 
+printed = ""
+
 
 class RiceFormatter(argparse.HelpFormatter):
     def _get_help_string(self, action):
@@ -61,8 +63,15 @@ class Dodge11874(RiceFormatter):
         super(Dodge11874, self).__init__(*args, **kwargs)
 
 
+def lprint(*a, **ka):
+    global printed
+
+    printed += " ".join(unicode(x) for x in a) + ka.get("end", "\n")
+    print(*a, **ka)
+
+
 def warn(msg):
-    print("\033[1mwarning:\033[0;33m {}\033[0m\n".format(msg))
+    lprint("\033[1mwarning:\033[0;33m {}\033[0m\n".format(msg))
 
 
 def ensure_locale():
@@ -73,7 +82,7 @@ def ensure_locale():
     ]:
         try:
             locale.setlocale(locale.LC_ALL, x)
-            print("Locale:", x)
+            lprint("Locale:", x)
             break
         except:
             continue
@@ -94,7 +103,7 @@ def ensure_cert():
 
     try:
         if filecmp.cmp(cert_cfg, cert_insec):
-            print(
+            lprint(
                 "\033[33m  using default TLS certificate; https will be insecure."
                 + "\033[36m\n  certificate location: {}\033[0m\n".format(cert_cfg)
             )
@@ -123,7 +132,7 @@ def configure_ssl_ver(al):
     if "help" in sslver:
         avail = [terse_sslver(x[6:]) for x in flags]
         avail = " ".join(sorted(avail) + ["all"])
-        print("\navailable ssl/tls versions:\n  " + avail)
+        lprint("\navailable ssl/tls versions:\n  " + avail)
         sys.exit(0)
 
     al.ssl_flags_en = 0
@@ -143,7 +152,7 @@ def configure_ssl_ver(al):
 
     for k in ["ssl_flags_en", "ssl_flags_de"]:
         num = getattr(al, k)
-        print("{}: {:8x} ({})".format(k, num, num))
+        lprint("{}: {:8x} ({})".format(k, num, num))
 
     # think i need that beer now
 
@@ -160,13 +169,13 @@ def configure_ssl_ciphers(al):
         try:
             ctx.set_ciphers(al.ciphers)
         except:
-            print("\n\033[1;31mfailed to set ciphers\033[0m\n")
+            lprint("\n\033[1;31mfailed to set ciphers\033[0m\n")
 
     if not hasattr(ctx, "get_ciphers"):
-        print("cannot read cipher list: openssl or python too old")
+        lprint("cannot read cipher list: openssl or python too old")
     else:
         ciphers = [x["description"] for x in ctx.get_ciphers()]
-        print("\n  ".join(["\nenabled ciphers:"] + align_tab(ciphers) + [""]))
+        lprint("\n  ".join(["\nenabled ciphers:"] + align_tab(ciphers) + [""]))
 
     if is_help:
         sys.exit(0)
@@ -182,16 +191,6 @@ def sighandler(sig=None, frame=None):
     print("\n".join(msg))
 
 
-def stackmon(fp, ival):
-    ctr = 0
-    while True:
-        ctr += 1
-        time.sleep(ival)
-        st = "{}, {}\n{}".format(ctr, time.time(), alltrace())
-        with open(fp, "wb") as f:
-            f.write(st.encode("utf-8", "replace"))
-
-
 def run_argparse(argv, formatter):
     ap = argparse.ArgumentParser(
         formatter_class=formatter,
@@ -249,30 +248,32 @@ def run_argparse(argv, formatter):
         ),
     )
     # fmt: off
-    ap.add_argument("-c", metavar="PATH", type=str, action="append", help="add config file")
+    u = unicode
-    ap.add_argument("-nc", metavar="NUM", type=int, default=64, help="max num clients")
+    ap2 = ap.add_argument_group('general options')
-    ap.add_argument("-j", metavar="CORES", type=int, default=1, help="max num cpu cores")
+    ap2.add_argument("-c", metavar="PATH", type=u, action="append", help="add config file")
-    ap.add_argument("-a", metavar="ACCT", type=str, action="append", help="add account, USER:PASS; example [ed:wark")
+    ap2.add_argument("-nc", metavar="NUM", type=int, default=64, help="max num clients")
-    ap.add_argument("-v", metavar="VOL", type=str, action="append", help="add volume, SRC:DST:FLAG; example [.::r], [/mnt/nas/music:/music:r:aed")
+    ap2.add_argument("-j", metavar="CORES", type=int, default=1, help="max num cpu cores")
-    ap.add_argument("-ed", action="store_true", help="enable ?dots")
+    ap2.add_argument("-a", metavar="ACCT", type=u, action="append", help="add account, USER:PASS; example [ed:wark")
-    ap.add_argument("-emp", action="store_true", help="enable markdown plugins")
+    ap2.add_argument("-v", metavar="VOL", type=u, action="append", help="add volume, SRC:DST:FLAG; example [.::r], [/mnt/nas/music:/music:r:aed")
-    ap.add_argument("-mcr", metavar="SEC", type=int, default=60, help="md-editor mod-chk rate")
+    ap2.add_argument("-ed", action="store_true", help="enable ?dots")
-    ap.add_argument("--dotpart", action="store_true", help="dotfile incomplete uploads")
+    ap2.add_argument("-emp", action="store_true", help="enable markdown plugins")
-    ap.add_argument("--sparse", metavar="MiB", type=int, default=4, help="up2k min.size threshold (mswin-only)")
+    ap2.add_argument("-mcr", metavar="SEC", type=int, default=60, help="md-editor mod-chk rate")
-    ap.add_argument("--urlform", metavar="MODE", type=str, default="print,get", help="how to handle url-forms; examples: [stash], [save,get]")
+    ap2.add_argument("--dotpart", action="store_true", help="dotfile incomplete uploads")
+    ap2.add_argument("--sparse", metavar="MiB", type=int, default=4, help="up2k min.size threshold (mswin-only)")
+    ap2.add_argument("--urlform", metavar="MODE", type=u, default="print,get", help="how to handle url-forms; examples: [stash], [save,get]")
 
     ap2 = ap.add_argument_group('network options')
-    ap2.add_argument("-i", metavar="IP", type=str, default="0.0.0.0", help="ip to bind (comma-sep.)")
+    ap2.add_argument("-i", metavar="IP", type=u, default="0.0.0.0", help="ip to bind (comma-sep.)")
-    ap2.add_argument("-p", metavar="PORT", type=str, default="3923", help="ports to bind (comma/range)")
+    ap2.add_argument("-p", metavar="PORT", type=u, default="3923", help="ports to bind (comma/range)")
     ap2.add_argument("--rproxy", metavar="DEPTH", type=int, default=1, help="which ip to keep; 0 = tcp, 1 = origin (first x-fwd), 2 = cloudflare, 3 = nginx, -1 = closest proxy")
     
     ap2 = ap.add_argument_group('SSL/TLS options')
     ap2.add_argument("--http-only", action="store_true", help="disable ssl/tls")
     ap2.add_argument("--https-only", action="store_true", help="disable plaintext")
-    ap2.add_argument("--ssl-ver", metavar="LIST", type=str, help="set allowed ssl/tls versions; [help] shows available versions; default is what your python version considers safe")
+    ap2.add_argument("--ssl-ver", metavar="LIST", type=u, help="set allowed ssl/tls versions; [help] shows available versions; default is what your python version considers safe")
-    ap2.add_argument("--ciphers", metavar="LIST", help="set allowed ssl/tls ciphers; [help] shows available ciphers")
+    ap2.add_argument("--ciphers", metavar="LIST", type=u, help="set allowed ssl/tls ciphers; [help] shows available ciphers")
     ap2.add_argument("--ssl-dbg", action="store_true", help="dump some tls info")
-    ap2.add_argument("--ssl-log", metavar="PATH", help="log master secrets")
+    ap2.add_argument("--ssl-log", metavar="PATH", type=u, help="log master secrets")
 
     ap2 = ap.add_argument_group('opt-outs')
     ap2.add_argument("-nw", action="store_true", help="disable writes (benchmark)")
@@ -281,14 +282,16 @@ def run_argparse(argv, formatter):
     ap2.add_argument("--no-zip", action="store_true", help="disable download as zip/tar")
 
     ap2 = ap.add_argument_group('safety options')
-    ap2.add_argument("--ls", metavar="U[,V[,F]]", help="scan all volumes; arguments USER,VOL,FLAGS; example [**,*,ln,p,r]")
+    ap2.add_argument("--ls", metavar="U[,V[,F]]", type=u, help="scan all volumes; arguments USER,VOL,FLAGS; example [**,*,ln,p,r]")
-    ap2.add_argument("--salt", type=str, default="hunter2", help="up2k file-hash salt")
+    ap2.add_argument("--salt", type=u, default="hunter2", help="up2k file-hash salt")
 
     ap2 = ap.add_argument_group('logging options')
     ap2.add_argument("-q", action="store_true", help="quiet")
+    ap2.add_argument("-lo", metavar="PATH", type=u, help="logfile, example: cpp-%%Y-%%m%%d-%%H%%M%%S.txt.xz")
     ap2.add_argument("--log-conn", action="store_true", help="print tcp-server msgs")
-    ap2.add_argument("--ihead", metavar="HEADER", action='append', help="dump incoming header")
+    ap2.add_argument("--log-htp", action="store_true", help="print http-server threadpool scaling")
-    ap2.add_argument("--lf-url", metavar="RE", type=str, default=r"^/\.cpr/|\?th=[wj]$", help="dont log URLs matching")
+    ap2.add_argument("--ihead", metavar="HEADER", type=u, action='append', help="dump incoming header")
+    ap2.add_argument("--lf-url", metavar="RE", type=u, default=r"^/\.cpr/|\?th=[wj]$", help="dont log URLs matching")
 
     ap2 = ap.add_argument_group('admin panel options')
     ap2.add_argument("--no-rescan", action="store_true", help="disable ?scan (volume reindexing)")
@@ -303,9 +306,9 @@ def run_argparse(argv, formatter):
     ap2.add_argument("--th-no-webp", action="store_true", help="disable webp output")
     ap2.add_argument("--th-ff-jpg", action="store_true", help="force jpg for video thumbs")
     ap2.add_argument("--th-poke", metavar="SEC", type=int, default=300, help="activity labeling cooldown")
-    ap2.add_argument("--th-clean", metavar="SEC", type=int, default=43200, help="cleanup interval")
+    ap2.add_argument("--th-clean", metavar="SEC", type=int, default=43200, help="cleanup interval; 0=disabled")
     ap2.add_argument("--th-maxage", metavar="SEC", type=int, default=604800, help="max folder age")
-    ap2.add_argument("--th-covers", metavar="N,N", type=str, default="folder.png,folder.jpg,cover.png,cover.jpg", help="folder thumbnails to stat for")
+    ap2.add_argument("--th-covers", metavar="N,N", type=u, default="folder.png,folder.jpg,cover.png,cover.jpg", help="folder thumbnails to stat for")
 
     ap2 = ap.add_argument_group('database options')
     ap2.add_argument("-e2d", action="store_true", help="enable up2k database")
@@ -314,24 +317,26 @@ def run_argparse(argv, formatter):
     ap2.add_argument("-e2t", action="store_true", help="enable metadata indexing")
     ap2.add_argument("-e2ts", action="store_true", help="enable metadata scanner, sets -e2t")
     ap2.add_argument("-e2tsr", action="store_true", help="rescan all metadata, sets -e2ts")
-    ap2.add_argument("--hist", metavar="PATH", type=str, help="where to store volume state")
+    ap2.add_argument("--hist", metavar="PATH", type=u, help="where to store volume state")
     ap2.add_argument("--no-hash", action="store_true", help="disable hashing during e2ds folder scans")
     ap2.add_argument("--no-mutagen", action="store_true", help="use ffprobe for tags instead")
     ap2.add_argument("--no-mtag-mt", action="store_true", help="disable tag-read parallelism")
-    ap2.add_argument("-mtm", metavar="M=t,t,t", action="append", type=str, help="add/replace metadata mapping")
+    ap2.add_argument("-mtm", metavar="M=t,t,t", type=u, action="append", help="add/replace metadata mapping")
-    ap2.add_argument("-mte", metavar="M,M,M", type=str, help="tags to index/display (comma-sep.)",
+    ap2.add_argument("-mte", metavar="M,M,M", type=u, help="tags to index/display (comma-sep.)",
         default="circle,album,.tn,artist,title,.bpm,key,.dur,.q,.vq,.aq,ac,vc,res,.fps")
-    ap2.add_argument("-mtp", metavar="M=[f,]bin", action="append", type=str, help="read tag M using bin")
+    ap2.add_argument("-mtp", metavar="M=[f,]bin", type=u, action="append", help="read tag M using bin")
     ap2.add_argument("--srch-time", metavar="SEC", type=int, default=30, help="search deadline")
 
     ap2 = ap.add_argument_group('appearance options')
-    ap2.add_argument("--css-browser", metavar="L", help="URL to additional CSS to include")
+    ap2.add_argument("--css-browser", metavar="L", type=u, help="URL to additional CSS to include")
 
     ap2 = ap.add_argument_group('debug options')
     ap2.add_argument("--no-sendfile", action="store_true", help="disable sendfile")
     ap2.add_argument("--no-scandir", action="store_true", help="disable scandir")
     ap2.add_argument("--no-fastboot", action="store_true", help="wait for up2k indexing")
-    ap2.add_argument("--stackmon", metavar="P,S", help="write stacktrace to Path every S second")
+    ap2.add_argument("--no-htp", action="store_true", help="disable httpserver threadpool, create threads as-needed instead")
+    ap2.add_argument("--stackmon", metavar="P,S", type=u, help="write stacktrace to Path every S second")
+    ap2.add_argument("--log-thrs", metavar="SEC", type=float, help="list active threads every SEC")
     
     return ap.parse_args(args=argv[1:])
     # fmt: on
@@ -348,7 +353,7 @@ def main(argv=None):
     desc = py_desc().replace("[", "\033[1;30m[")
 
     f = '\033[36mcopyparty v{} "\033[35m{}\033[36m" ({})\n{}\033[0m\n'
-    print(f.format(S_VERSION, CODENAME, S_BUILD_DT, desc))
+    lprint(f.format(S_VERSION, CODENAME, S_BUILD_DT, desc))
 
     ensure_locale()
     if HAVE_SSL:
@@ -362,7 +367,7 @@ def main(argv=None):
             continue
 
         msg = "\033[1;31mWARNING:\033[0;1m\n  {} \033[0;33mwas replaced with\033[0;1m {} \033[0;33mand will be removed\n\033[0m"
-        print(msg.format(dk, nk))
+        lprint(msg.format(dk, nk))
         argv[idx] = nk
         time.sleep(2)
 
@@ -371,16 +376,6 @@ def main(argv=None):
     except AssertionError:
         al = run_argparse(argv, Dodge11874)
 
-    if al.stackmon:
-        fp, f = al.stackmon.rsplit(",", 1)
-        f = int(f)
-        t = threading.Thread(
-            target=stackmon,
-            args=(fp, f),
-        )
-        t.daemon = True
-        t.start()
-
     # propagate implications
     for k1, k2 in IMPLICATIONS:
         if getattr(al, k1):
@@ -416,7 +411,7 @@ def main(argv=None):
 
     # signal.signal(signal.SIGINT, sighandler)
 
-    SvcHub(al).run()
+    SvcHub(al, argv, printed).run()
 
 
 if __name__ == "__main__":

copyparty/__version__.py

@@ -1,8 +1,8 @@
 # coding: utf-8
 
-VERSION = (0, 11, 31)
+VERSION = (0, 11, 40)
 CODENAME = "the grid"
-BUILD_DT = (2021, 7, 4)
+BUILD_DT = (2021, 7, 15)
 
 S_VERSION = ".".join(map(str, VERSION))
 S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT)

copyparty/authsrv.py

@@ -10,13 +10,14 @@ import hashlib
 import threading
 
 from .__init__ import WINDOWS
-from .util import IMPLICATIONS, uncyg, undot, Pebkac, fsdec, fsenc, statdir, nuprint
+from .util import IMPLICATIONS, uncyg, undot, Pebkac, fsdec, fsenc, statdir
 
 
 class VFS(object):
     """single level in the virtual fs"""
 
-    def __init__(self, realpath, vpath, uread=[], uwrite=[], uadm=[], flags={}):
+    def __init__(self, log, realpath, vpath, uread, uwrite, uadm, flags):
+        self.log = log
         self.realpath = realpath  # absolute path on host filesystem
         self.vpath = vpath  # absolute path in the virtual filesystem
         self.uread = uread  # users who can read this
@@ -62,6 +63,7 @@ class VFS(object):
                 return self.nodes[name].add(src, dst)
 
             vn = VFS(
+                self.log,
                 os.path.join(self.realpath, name) if self.realpath else None,
                 "{}/{}".format(self.vpath, name).lstrip("/"),
                 self.uread,
@@ -79,7 +81,7 @@ class VFS(object):
 
         # leaf does not exist; create and keep permissions blank
         vp = "{}/{}".format(self.vpath, dst).lstrip("/")
-        vn = VFS(src, vp)
+        vn = VFS(self.log, src, vp, [], [], [], {})
         vn.dbv = self.dbv or self
         self.nodes[dst] = vn
         return vn
@@ -181,7 +183,7 @@ class VFS(object):
         """return user-readable [fsdir,real,virt] items at vpath"""
         virt_vis = {}  # nodes readable by user
         abspath = self.canonical(rem)
-        real = list(statdir(nuprint, scandir, lstat, abspath))
+        real = list(statdir(self.log, scandir, lstat, abspath))
         real.sort()
         if not rem:
             for name, vn2 in sorted(self.nodes.items()):
@@ -208,8 +210,13 @@ class VFS(object):
             rem, uname, scandir, incl_wo=False, lstat=lstat
         )
 
-        if seen and not fsroot.startswith(seen[-1]) and fsroot in seen:
+        if (
-            print("bailing from symlink loop,\n  {}\n  {}".format(seen[-1], fsroot))
+            seen
+            and (not fsroot.startswith(seen[-1]) or fsroot == seen[-1])
+            and fsroot in seen
+        ):
+            m = "bailing from symlink loop,\n  prev: {}\n  curr: {}\n  from: {}/{}"
+            self.log("vfs.walk", m.format(seen[-1], fsroot, self.vpath, rem), 3)
             return
 
         seen = seen[:] + [fsroot]
@@ -242,6 +249,10 @@ class VFS(object):
         if flt:
             flt = {k: True for k in flt}
 
+        f1 = "{0}.hist{0}up2k.".format(os.sep)
+        f2a = os.sep + "dir.txt"
+        f2b = "{0}.hist{0}".format(os.sep)
+
         for vpath, apath, files, rd, vd in self.walk(
             "", vrem, [], uname, dots, scandir, False
         ):
@@ -275,7 +286,11 @@ class VFS(object):
                     del vd[x]
 
             # up2k filetring based on actual abspath
-            files = [x for x in files if "{0}.hist{0}up2k.".format(os.sep) not in x[1]]
+            files = [
+                x
+                for x in files
+                if f1 not in x[1] and (not x[1].endswith(f2a) or f2b not in x[1])
+            ]
 
             for f in [{"vp": v, "ap": a, "st": n[1]} for v, a, n in files]:
                 yield f
@@ -466,7 +481,7 @@ class AuthSrv(object):
                         )
                     except:
                         m = "\n\033[1;31m\nerror in config file {} on line {}:\n\033[0m"
-                        print(m.format(cfg_fn, self.line_ctr))
+                        self.log(m.format(cfg_fn, self.line_ctr), 1)
                         raise
 
         # case-insensitive; normalize
@@ -482,10 +497,10 @@ class AuthSrv(object):
 
         if not mount:
             # -h says our defaults are CWD at root and read/write for everyone
-            vfs = VFS(os.path.abspath("."), "", ["*"], ["*"])
+            vfs = VFS(self.log_func, os.path.abspath("."), "", ["*"], ["*"], ["*"], {})
         elif "" not in mount:
             # there's volumes but no root; make root inaccessible
-            vfs = VFS(None, "")
+            vfs = VFS(self.log_func, None, "", [], [], [], {})
             vfs.flags["d2d"] = True
 
         maxdepth = 0
@@ -497,7 +512,13 @@ class AuthSrv(object):
             if dst == "":
                 # rootfs was mapped; fully replaces the default CWD vfs
                 vfs = VFS(
-                    mount[dst], dst, mread[dst], mwrite[dst], madm[dst], mflags[dst]
+                    self.log_func,
+                    mount[dst],
+                    dst,
+                    mread[dst],
+                    mwrite[dst],
+                    madm[dst],
+                    mflags[dst],
                 )
                 continue
 
@@ -780,7 +801,7 @@ class AuthSrv(object):
                         msg = [x[1] for x in files]
 
                     if msg:
-                        nuprint("\n".join(msg))
+                        self.log("\n" + "\n".join(msg))
 
                 if n_bads and flag_p:
                     raise Exception("found symlink leaving volume, and strict is set")

copyparty/broker_mp.py

@@ -4,17 +4,11 @@ from __future__ import print_function, unicode_literals
 import time
 import threading
 
-from .__init__ import PY2, WINDOWS, VT100
 from .broker_util import try_exec
 from .broker_mpw import MpWorker
 from .util import mp
 
 
-if PY2 and not WINDOWS:
-    from multiprocessing.reduction import ForkingPickler
-    from StringIO import StringIO as MemesIO  # pylint: disable=import-error
-
-
 class BrokerMp(object):
     """external api; manages MpWorkers"""
 
@@ -33,19 +27,17 @@ class BrokerMp(object):
             cores = mp.cpu_count()
 
         self.log("broker", "booting {} subprocesses".format(cores))
-        for n in range(cores):
+        for n in range(1, cores + 1):
             q_pend = mp.Queue(1)
             q_yield = mp.Queue(64)
 
             proc = mp.Process(target=MpWorker, args=(q_pend, q_yield, self.args, n))
             proc.q_pend = q_pend
             proc.q_yield = q_yield
-            proc.nid = n
             proc.clients = {}
-            proc.workload = 0
 
             thr = threading.Thread(
-                target=self.collector, args=(proc,), name="mp-collector"
+                target=self.collector, args=(proc,), name="mp-sink-{}".format(n)
             )
             thr.daemon = True
             thr.start()
@@ -53,13 +45,6 @@ class BrokerMp(object):
             self.procs.append(proc)
             proc.start()
 
-        if not self.args.q:
-            thr = threading.Thread(
-                target=self.debug_load_balancer, name="mp-dbg-loadbalancer"
-            )
-            thr.daemon = True
-            thr.start()
-
     def shutdown(self):
         self.log("broker", "shutting down")
         for n, proc in enumerate(self.procs):
@@ -89,20 +74,6 @@ class BrokerMp(object):
             if dest == "log":
                 self.log(*args)
 
-            elif dest == "workload":
-                with self.mutex:
-                    proc.workload = args[0]
-
-            elif dest == "httpdrop":
-                addr = args[0]
-
-                with self.mutex:
-                    del proc.clients[addr]
-                    if not proc.clients:
-                        proc.workload = 0
-
-                self.hub.tcpsrv.num_clients.add(-1)
-
             elif dest == "retq":
                 # response from previous ipc call
                 with self.retpend_mutex:
@@ -128,38 +99,9 @@ class BrokerMp(object):
         returns a Queue object which eventually contains the response if want_retval
         (not-impl here since nothing uses it yet)
         """
-        if dest == "httpconn":
+        if dest == "listen":
-            sck, addr = args
+            for p in self.procs:
-            sck2 = sck
+                p.q_pend.put([0, dest, [args[0], len(self.procs)]])
-            if PY2:
-                buf = MemesIO()
-                ForkingPickler(buf).dump(sck)
-                sck2 = buf.getvalue()
-
-            proc = sorted(self.procs, key=lambda x: x.workload)[0]
-            proc.q_pend.put([0, dest, [sck2, addr]])
-
-            with self.mutex:
-                proc.clients[addr] = 50
-                proc.workload += 50
 
         else:
             raise Exception("what is " + str(dest))
-
-    def debug_load_balancer(self):
-        fmt = "\033[1m{}\033[0;36m{:4}\033[0m "
-        if not VT100:
-            fmt = "({}{:4})"
-
-        last = ""
-        while self.procs:
-            msg = ""
-            for proc in self.procs:
-                msg += fmt.format(len(proc.clients), proc.workload)
-
-            if msg != last:
-                last = msg
-                with self.hub.log_mutex:
-                    print(msg)
-
-            time.sleep(0.1)

copyparty/broker_mpw.py

@@ -3,18 +3,13 @@ from __future__ import print_function, unicode_literals
 from copyparty.authsrv import AuthSrv
 
 import sys
-import time
 import signal
 import threading
 
-from .__init__ import PY2, WINDOWS
 from .broker_util import ExceptionalQueue
 from .httpsrv import HttpSrv
 from .util import FAKE_MP
 
-if PY2 and not WINDOWS:
-    import pickle  # nosec
-
 
 class MpWorker(object):
     """one single mp instance"""
@@ -25,10 +20,11 @@ class MpWorker(object):
         self.args = args
         self.n = n
 
+        self.log = self._log_disabled if args.q and not args.lo else self._log_enabled
+
         self.retpend = {}
         self.retpend_mutex = threading.Lock()
         self.mutex = threading.Lock()
-        self.workload_thr_alive = False
 
         # we inherited signal_handler from parent,
         # replace it with something harmless
@@ -39,8 +35,7 @@ class MpWorker(object):
         self.asrv = AuthSrv(args, None, False)
 
         # instantiate all services here (TODO: inheritance?)
-        self.httpsrv = HttpSrv(self, True)
+        self.httpsrv = HttpSrv(self, n)
-        self.httpsrv.disconnect_func = self.httpdrop
 
         # on winxp and some other platforms,
         # use thr.join() to block all signals
@@ -53,15 +48,15 @@ class MpWorker(object):
         # print('k')
         pass
 
-    def log(self, src, msg, c=0):
+    def _log_enabled(self, src, msg, c=0):
         self.q_yield.put([0, "log", [src, msg, c]])
 
+    def _log_disabled(self, src, msg, c=0):
+        pass
+
     def logw(self, msg, c=0):
         self.log("mp{}".format(self.n), msg, c)
 
-    def httpdrop(self, addr):
-        self.q_yield.put([0, "httpdrop", [addr]])
-
     def main(self):
         while True:
             retq_id, dest, args = self.q_pend.get()
@@ -73,24 +68,8 @@ class MpWorker(object):
                 sys.exit(0)
                 return
 
-            elif dest == "httpconn":
+            elif dest == "listen":
-                sck, addr = args
+                self.httpsrv.listen(args[0], args[1])
-                if PY2:
-                    sck = pickle.loads(sck)  # nosec
-
-                if self.args.log_conn:
-                    self.log("%s %s" % addr, "|%sC-qpop" % ("-" * 4,), c="1;30")
-
-                self.httpsrv.accept(sck, addr)
-
-                with self.mutex:
-                    if not self.workload_thr_alive:
-                        self.workload_thr_alive = True
-                        thr = threading.Thread(
-                            target=self.thr_workload, name="mpw-workload"
-                        )
-                        thr.daemon = True
-                        thr.start()
 
             elif dest == "retq":
                 # response from previous ipc call
@@ -114,16 +93,3 @@ class MpWorker(object):
 
         self.q_yield.put([retq_id, dest, args])
         return retq
-
-    def thr_workload(self):
-        """announce workloads to MpSrv (the mp controller / loadbalancer)"""
-        # avoid locking in extract_filedata by tracking difference here
-        while True:
-            time.sleep(0.2)
-            with self.mutex:
-                if self.httpsrv.num_clients() == 0:
-                    # no clients rn, termiante thread
-                    self.workload_thr_alive = False
-                    return
-
-            self.q_yield.put([0, "workload", [self.httpsrv.workload]])

copyparty/broker_thr.py

@@ -3,7 +3,6 @@ from __future__ import print_function, unicode_literals
 
 import threading
 
-from .authsrv import AuthSrv
 from .httpsrv import HttpSrv
 from .broker_util import ExceptionalQueue, try_exec
 
@@ -20,8 +19,7 @@ class BrokerThr(object):
         self.mutex = threading.Lock()
 
         # instantiate all services here (TODO: inheritance?)
-        self.httpsrv = HttpSrv(self)
+        self.httpsrv = HttpSrv(self, None)
-        self.httpsrv.disconnect_func = self.httpdrop
 
     def shutdown(self):
         # self.log("broker", "shutting down")
@@ -29,12 +27,8 @@ class BrokerThr(object):
         pass
 
     def put(self, want_retval, dest, *args):
-        if dest == "httpconn":
+        if dest == "listen":
-            sck, addr = args
+            self.httpsrv.listen(args[0], 1)
-            if self.args.log_conn:
-                self.log("%s %s" % addr, "|%sC-qpop" % ("-" * 4,), c="1;30")
-
-            self.httpsrv.accept(sck, addr)
 
         else:
             # new ipc invoking managed service in hub
@@ -51,6 +45,3 @@ class BrokerThr(object):
             retq = ExceptionalQueue(1)
             retq.put(rv)
             return retq
-
-    def httpdrop(self, addr):
-        self.hub.tcpsrv.num_clients.add(-1)

copyparty/httpcli.py

@@ -13,15 +13,12 @@ import ctypes
 from datetime import datetime
 import calendar
 
-from .__init__ import E, PY2, WINDOWS, ANYWIN
+from .__init__ import E, PY2, WINDOWS, ANYWIN, unicode
 from .util import *  # noqa  # pylint: disable=unused-wildcard-import
 from .authsrv import AuthSrv
 from .szip import StreamZip
 from .star import StreamTar
 
-if not PY2:
-    unicode = str
-
 
 NO_CACHE = {"Cache-Control": "no-cache"}
 NO_STORE = {"Cache-Control": "no-store; max-age=0"}
@@ -40,7 +37,6 @@ class HttpCli(object):
         self.ip = conn.addr[0]
         self.addr = conn.addr  # type: tuple[str, int]
         self.args = conn.args
-        self.is_mp = conn.is_mp
         self.asrv = conn.asrv  # type: AuthSrv
         self.ico = conn.ico
         self.thumbcli = conn.thumbcli
@@ -98,9 +94,13 @@ class HttpCli(object):
             try:
                 self.mode, self.req, self.http_ver = headerlines[0].split(" ")
             except:
-                raise Pebkac(400, "bad headers:\n" + "\n".join(headerlines))
+                msg = " ]\n#[ ".join(headerlines)
+                raise Pebkac(400, "bad headers:\n#[ " + msg + " ]")
 
         except Pebkac as ex:
+            self.mode = "GET"
+            self.req = "[junk]"
+            self.http_ver = "HTTP/1.1"
             # self.log("pebkac at httpcli.run #1: " + repr(ex))
             self.keepalive = self._check_nonfatal(ex)
             self.loud_reply(unicode(ex), status=ex.code)
@@ -226,7 +226,7 @@ class HttpCli(object):
             except Pebkac:
                 return False
 
-    def send_headers(self, length, status=200, mime=None, headers={}):
+    def send_headers(self, length, status=200, mime=None, headers=None):
         response = ["{} {} {}".format(self.http_ver, status, HTTPCODE[status])]
 
         if length is not None:
@@ -236,7 +236,8 @@ class HttpCli(object):
         response.append("Connection: " + ("Keep-Alive" if self.keepalive else "Close"))
 
         # headers{} overrides anything set previously
-        self.out_headers.update(headers)
+        if headers:
+            self.out_headers.update(headers)
 
         # default to utf8 html if no content-type is set
         if not mime:
@@ -253,7 +254,7 @@ class HttpCli(object):
         except:
             raise Pebkac(400, "client d/c while replying headers")
 
-    def reply(self, body, status=200, mime=None, headers={}):
+    def reply(self, body, status=200, mime=None, headers=None):
         # TODO something to reply with user-supplied values safely
         self.send_headers(len(body), status, mime, headers)
 
@@ -269,7 +270,7 @@ class HttpCli(object):
         self.log(body.rstrip())
         self.reply(b"<pre>" + body.encode("utf-8") + b"\r\n", *list(args), **kwargs)
 
-    def urlq(self, add={}, rm=[]):
+    def urlq(self, add, rm):
         """
         generates url query based on uparam (b, pw, all others)
         removing anything in rm, adding pairs in add
@@ -341,6 +342,9 @@ class HttpCli(object):
         if "tree" in self.uparam:
             return self.tx_tree()
 
+        if "stack" in self.uparam:
+            return self.tx_stack()
+
         # conditional redirect to single volumes
         if self.vpath == "" and not self.ouparam:
             nread = len(self.rvol)
@@ -370,9 +374,6 @@ class HttpCli(object):
         if "scan" in self.uparam:
             return self.scanvol()
 
-        if "stack" in self.uparam:
-            return self.tx_stack()
-
         return self.tx_browser()
 
     def handle_options(self):
@@ -478,15 +479,17 @@ class HttpCli(object):
         addr = self.ip.replace(":", ".")
         fn = "put-{:.6f}-{}.bin".format(time.time(), addr)
         path = os.path.join(fdir, fn)
+        if self.args.nw:
+            path = os.devnull
 
         with open(fsenc(path), "wb", 512 * 1024) as f:
-            post_sz, _, sha_b64 = hashcopy(self.conn, reader, f)
+            post_sz, _, sha_b64 = hashcopy(reader, f)
 
-        vfs, vrem = vfs.get_dbv(rem)
+        if not self.args.nw:
-
+            vfs, vrem = vfs.get_dbv(rem)
-        self.conn.hsrv.broker.put(
+            self.conn.hsrv.broker.put(
-            False, "up2k.hash_file", vfs.realpath, vfs.flags, vrem, fn
+                False, "up2k.hash_file", vfs.realpath, vfs.flags, vrem, fn
-        )
+            )
 
         return post_sz, sha_b64, remains, path
 
@@ -712,7 +715,7 @@ class HttpCli(object):
 
         with open(fsenc(path), "rb+", 512 * 1024) as f:
             f.seek(cstart[0])
-            post_sz, _, sha_b64 = hashcopy(self.conn, reader, f)
+            post_sz, _, sha_b64 = hashcopy(reader, f)
 
             if sha_b64 != chash:
                 raise Pebkac(
@@ -792,7 +795,7 @@ class HttpCli(object):
         vfs, rem = self.asrv.vfs.get(self.vpath, self.uname, False, True)
         self._assert_safe_rem(rem)
 
-        sanitized = sanitize_fn(new_dir)
+        sanitized = sanitize_fn(new_dir, "", [])
 
         if not nullwrite:
             fdir = os.path.join(vfs.realpath, rem)
@@ -829,7 +832,7 @@ class HttpCli(object):
         if not new_file.endswith(".md"):
             new_file += ".md"
 
-        sanitized = sanitize_fn(new_file)
+        sanitized = sanitize_fn(new_file, "", [])
 
         if not nullwrite:
             fdir = os.path.join(vfs.realpath, rem)
@@ -862,7 +865,7 @@ class HttpCli(object):
                 if p_file and not nullwrite:
                     fdir = os.path.join(vfs.realpath, rem)
                     fname = sanitize_fn(
-                        p_file, bad=[".prologue.html", ".epilogue.html"]
+                        p_file, "", [".prologue.html", ".epilogue.html"]
                     )
 
                     if not os.path.isdir(fsenc(fdir)):
@@ -879,7 +882,7 @@ class HttpCli(object):
                     with ren_open(fname, "wb", 512 * 1024, **open_args) as f:
                         f, fname = f["orz"]
                         self.log("writing to {}/{}".format(fdir, fname))
-                        sz, sha512_hex, _ = hashcopy(self.conn, p_data, f)
+                        sz, sha512_hex, _ = hashcopy(p_data, f)
                         if sz == 0:
                             raise Pebkac(400, "empty files in post")
 
@@ -1062,7 +1065,7 @@ class HttpCli(object):
             raise Pebkac(400, "expected body, got {}".format(p_field))
 
         with open(fsenc(fp), "wb", 512 * 1024) as f:
-            sz, sha512, _ = hashcopy(self.conn, p_data, f)
+            sz, sha512, _ = hashcopy(p_data, f)
 
         new_lastmod = os.stat(fsenc(fp)).st_mtime
         new_lastmod3 = int(new_lastmod * 1000)
@@ -1252,8 +1255,7 @@ class HttpCli(object):
             if use_sendfile:
                 remains = sendfile_kern(lower, upper, f, self.s)
             else:
-                actor = self.conn if self.is_mp else None
+                remains = sendfile_py(lower, upper, f, self.s)
-                remains = sendfile_py(lower, upper, f, self.s, actor)
 
         if remains > 0:
             logmsg += " \033[31m" + unicode(upper - remains) + "\033[0m"
@@ -1310,7 +1312,7 @@ class HttpCli(object):
 
         fgen = vn.zipgen(rem, items, self.uname, dots, not self.args.no_scandir)
         # for f in fgen: print(repr({k: f[k] for k in ["vp", "ap"]}))
-        bgen = packer(fgen, utf8="utf" in uarg, pre_crc="crc" in uarg)
+        bgen = packer(self.log, fgen, utf8="utf" in uarg, pre_crc="crc" in uarg)
         bsent = 0
         for buf in bgen.gen():
             if not buf:
@@ -1421,7 +1423,7 @@ class HttpCli(object):
         return True
 
     def tx_mounts(self):
-        suf = self.urlq(rm=["h"])
+        suf = self.urlq({}, ["h"])
         rvol, wvol, avol = [
             [("/" + x).rstrip("/") + "/" for x in y]
             for y in [self.rvol, self.wvol, self.avol]
@@ -1471,7 +1473,7 @@ class HttpCli(object):
         raise Pebkac(500, x)
 
     def tx_stack(self):
-        if not self.readable or not self.writable:
+        if not self.avol:
             raise Pebkac(403, "not admin")
 
         if self.args.no_stack:
@@ -1561,7 +1563,9 @@ class HttpCli(object):
             raise Pebkac(404)
 
         if self.readable:
-            if rem.startswith(".hist/up2k."):
+            if rem.startswith(".hist/up2k.") or (
+                rem.endswith("/dir.txt") and rem.startswith(".hist/th/")
+            ):
                 raise Pebkac(403)
 
             is_dir = stat.S_ISDIR(st.st_mode)
@@ -1630,7 +1634,7 @@ class HttpCli(object):
         if self.writable:
             perms.append("write")
 
-        url_suf = self.urlq()
+        url_suf = self.urlq({}, [])
         is_ls = "ls" in self.uparam
 
         tpl = "browser"

copyparty/httpconn.py

@@ -34,7 +34,6 @@ class HttpConn(object):
 
         self.args = hsrv.args
         self.asrv = hsrv.asrv
-        self.is_mp = hsrv.is_mp
         self.cert_path = hsrv.cert_path
 
         enth = HAVE_PIL and not self.args.no_thumb
@@ -45,7 +44,6 @@ class HttpConn(object):
         self.stopping = False
         self.nreq = 0
         self.nbyte = 0
-        self.workload = 0
         self.u2idx = None
         self.log_func = hsrv.log
         self.lf_url = re.compile(self.args.lf_url) if self.args.lf_url else None
@@ -184,11 +182,6 @@ class HttpConn(object):
             self.sr = Unrecv(self.s)
 
         while not self.stopping:
-            if self.is_mp:
-                self.workload += 50
-                if self.workload >= 2 ** 31:
-                    self.workload = 100
-
             self.nreq += 1
             cli = HttpCli(self)
             if not cli.run():

copyparty/httpsrv.py

@@ -4,8 +4,8 @@ from __future__ import print_function, unicode_literals
 import os
 import sys
 import time
+import math
 import base64
-import struct
 import socket
 import threading
 
@@ -26,9 +26,15 @@ except ImportError:
     )
     sys.exit(1)
 
-from .__init__ import E, MACOS
+from .__init__ import E, PY2, MACOS
+from .util import spack, min_ex, start_stackmon, start_log_thrs
 from .httpconn import HttpConn
 
+if PY2:
+    import Queue as queue
+else:
+    import queue
+
 
 class HttpSrv(object):
     """
@@ -36,19 +42,26 @@ class HttpSrv(object):
     relying on MpSrv for performance (HttpSrv is just plain threads)
     """
 
-    def __init__(self, broker, is_mp=False):
+    def __init__(self, broker, nid):
         self.broker = broker
-        self.is_mp = is_mp
+        self.nid = nid
         self.args = broker.args
         self.log = broker.log
         self.asrv = broker.asrv
 
-        self.disconnect_func = None
+        self.name = "httpsrv" + ("-n{}-i{:x}".format(nid, os.getpid()) if nid else "")
         self.mutex = threading.Lock()
+        self.stopping = False
 
-        self.clients = {}
+        self.tp_nthr = 0  # actual
-        self.workload = 0
+        self.tp_ncli = 0  # fading
-        self.workload_thr_alive = False
+        self.tp_time = None  # latest worker collect
+        self.tp_q = None if self.args.no_htp else queue.LifoQueue()
+
+        self.srvs = []
+        self.ncli = 0  # exact
+        self.clients = {}  # laggy
+        self.nclimax = 0
         self.cb_ts = 0
         self.cb_v = 0
 
@@ -65,24 +78,155 @@ class HttpSrv(object):
         else:
             self.cert_path = None
 
+        if self.tp_q:
+            self.start_threads(4)
+
+            name = "httpsrv-scaler" + ("-{}".format(nid) if nid else "")
+            t = threading.Thread(target=self.thr_scaler, name=name)
+            t.daemon = True
+            t.start()
+
+        if nid:
+            if self.args.stackmon:
+                start_stackmon(self.args.stackmon, nid)
+
+            if self.args.log_thrs:
+                start_log_thrs(self.log, self.args.log_thrs, nid)
+
+    def start_threads(self, n):
+        self.tp_nthr += n
+        if self.args.log_htp:
+            self.log(self.name, "workers += {} = {}".format(n, self.tp_nthr), 6)
+
+        for _ in range(n):
+            thr = threading.Thread(
+                target=self.thr_poolw,
+                name=self.name + "-poolw",
+            )
+            thr.daemon = True
+            thr.start()
+
+    def stop_threads(self, n):
+        self.tp_nthr -= n
+        if self.args.log_htp:
+            self.log(self.name, "workers -= {} = {}".format(n, self.tp_nthr), 6)
+
+        for _ in range(n):
+            self.tp_q.put(None)
+
+    def thr_scaler(self):
+        while True:
+            time.sleep(2 if self.tp_ncli else 30)
+            with self.mutex:
+                self.tp_ncli = max(self.ncli, self.tp_ncli - 2)
+                if self.tp_nthr > self.tp_ncli + 8:
+                    self.stop_threads(4)
+
+    def listen(self, sck, nlisteners):
+        ip, port = sck.getsockname()
+        self.srvs.append(sck)
+        self.nclimax = math.ceil(self.args.nc * 1.0 / nlisteners)
+        t = threading.Thread(
+            target=self.thr_listen,
+            args=(sck,),
+            name="httpsrv-n{}-listen-{}-{}".format(self.nid or "0", ip, port),
+        )
+        t.daemon = True
+        t.start()
+
+    def thr_listen(self, srv_sck):
+        """listens on a shared tcp server"""
+        ip, port = srv_sck.getsockname()
+        fno = srv_sck.fileno()
+        msg = "subscribed @ {}:{}  f{}".format(ip, port, fno)
+        self.log(self.name, msg)
+        while not self.stopping:
+            if self.args.log_conn:
+                self.log(self.name, "|%sC-ncli" % ("-" * 1,), c="1;30")
+
+            if self.ncli >= self.nclimax:
+                self.log(self.name, "at connection limit; waiting", 3)
+                while self.ncli >= self.nclimax:
+                    time.sleep(0.1)
+
+            if self.args.log_conn:
+                self.log(self.name, "|%sC-acc1" % ("-" * 2,), c="1;30")
+
+            try:
+                sck, addr = srv_sck.accept()
+            except (OSError, socket.error) as ex:
+                self.log(self.name, "accept({}): {}".format(fno, ex), c=6)
+                time.sleep(0.02)
+                continue
+
+            if self.args.log_conn:
+                m = "|{}C-acc2 \033[0;36m{} \033[3{}m{}".format(
+                    "-" * 3, ip, port % 8, port
+                )
+                self.log("%s %s" % addr, m, c="1;30")
+
+            self.accept(sck, addr)
+
     def accept(self, sck, addr):
         """takes an incoming tcp connection and creates a thread to handle it"""
-        if self.args.log_conn:
+        now = time.time()
-            self.log("%s %s" % addr, "|%sC-cthr" % ("-" * 5,), c="1;30")
+
+        if now - (self.tp_time or now) > 300:
+            self.tp_q = None
+
+        if self.tp_q:
+            self.tp_q.put((sck, addr))
+            with self.mutex:
+                self.ncli += 1
+                self.tp_time = self.tp_time or now
+                self.tp_ncli = max(self.tp_ncli, self.ncli + 1)
+                if self.tp_nthr < self.ncli + 4:
+                    self.start_threads(8)
+            return
+
+        if not self.args.no_htp:
+            m = "looks like the httpserver threadpool died; please make an issue on github and tell me the story of how you pulled that off, thanks and dog bless\n"
+            self.log(self.name, m, 1)
+
+        with self.mutex:
+            self.ncli += 1
 
         thr = threading.Thread(
             target=self.thr_client,
             args=(sck, addr),
-            name="httpsrv-{}-{}".format(addr[0].split(".", 2)[-1][-6:], addr[1]),
+            name="httpconn-{}-{}".format(addr[0].split(".", 2)[-1][-6:], addr[1]),
         )
         thr.daemon = True
         thr.start()
 
-    def num_clients(self):
+    def thr_poolw(self):
-        with self.mutex:
+        while True:
-            return len(self.clients)
+            task = self.tp_q.get()
+            if not task:
+                break
+
+            with self.mutex:
+                self.tp_time = None
+
+            try:
+                sck, addr = task
+                me = threading.current_thread()
+                me.name = "httpconn-{}-{}".format(
+                    addr[0].split(".", 2)[-1][-6:], addr[1]
+                )
+                self.thr_client(sck, addr)
+                me.name = self.name + "-poolw"
+            except:
+                self.log(self.name, "thr_client: " + min_ex(), 3)
 
     def shutdown(self):
+        self.stopping = True
+        for srv in self.srvs:
+            try:
+                srv.close()
+            except:
+                pass
+
         clients = list(self.clients.keys())
         for cli in clients:
             try:
@@ -90,7 +234,14 @@ class HttpSrv(object):
             except:
                 pass
 
-        self.log("httpsrv-n", "ok bye")
+        if self.tp_q:
+            self.stop_threads(self.tp_nthr)
+            for _ in range(10):
+                time.sleep(0.05)
+                if self.tp_q.empty():
+                    break
+
+        self.log(self.name, "ok bye")
 
     def thr_client(self, sck, addr):
         """thread managing one tcp client"""
@@ -100,25 +251,15 @@ class HttpSrv(object):
         with self.mutex:
             self.clients[cli] = 0
 
-            if self.is_mp:
-                self.workload += 50
-                if not self.workload_thr_alive:
-                    self.workload_thr_alive = True
-                    thr = threading.Thread(
-                        target=self.thr_workload, name="httpsrv-workload"
-                    )
-                    thr.daemon = True
-                    thr.start()
-
         fno = sck.fileno()
         try:
             if self.args.log_conn:
-                self.log("%s %s" % addr, "|%sC-crun" % ("-" * 6,), c="1;30")
+                self.log("%s %s" % addr, "|%sC-crun" % ("-" * 4,), c="1;30")
 
             cli.run()
 
         except (OSError, socket.error) as ex:
-            if ex.errno not in [10038, 10054, 107, 57, 9]:
+            if ex.errno not in [10038, 10054, 107, 57, 49, 9]:
                 self.log(
                     "%s %s" % addr,
                     "run({}): {}".format(fno, ex),
@@ -128,7 +269,7 @@ class HttpSrv(object):
         finally:
             sck = cli.s
             if self.args.log_conn:
-                self.log("%s %s" % addr, "|%sC-cdone" % ("-" * 7,), c="1;30")
+                self.log("%s %s" % addr, "|%sC-cdone" % ("-" * 5,), c="1;30")
 
             try:
                 fno = sck.fileno()
@@ -152,35 +293,7 @@ class HttpSrv(object):
             finally:
                 with self.mutex:
                     del self.clients[cli]
-
+                    self.ncli -= 1
-                if self.disconnect_func:
-                    self.disconnect_func(addr)  # pylint: disable=not-callable
-
-    def thr_workload(self):
-        """indicates the python interpreter workload caused by this HttpSrv"""
-        # avoid locking in extract_filedata by tracking difference here
-        while True:
-            time.sleep(0.2)
-            with self.mutex:
-                if not self.clients:
-                    # no clients rn, termiante thread
-                    self.workload_thr_alive = False
-                    self.workload = 0
-                    return
-
-            total = 0
-            with self.mutex:
-                for cli in self.clients.keys():
-                    now = cli.workload
-                    delta = now - self.clients[cli]
-                    if delta < 0:
-                        # was reset in HttpCli to prevent overflow
-                        delta = now
-
-                    total += delta
-                    self.clients[cli] = now
-
-            self.workload = total
 
     def cachebuster(self):
         if time.time() - self.cb_ts < 1:
@@ -199,7 +312,7 @@ class HttpSrv(object):
             except:
                 pass
 
-            v = base64.urlsafe_b64encode(struct.pack(">xxL", int(v)))
+            v = base64.urlsafe_b64encode(spack(b">xxL", int(v)))
             self.cb_v = v.decode("ascii")[-4:]
             self.cb_ts = time.time()
             return self.cb_v

copyparty/mtag.py

@@ -7,12 +7,9 @@ import json
 import shutil
 import subprocess as sp
 
-from .__init__ import PY2, WINDOWS
+from .__init__ import PY2, WINDOWS, unicode
 from .util import fsenc, fsdec, uncyg, REKOBO_LKEY
 
-if not PY2:
-    unicode = str
-
 
 def have_ff(cmd):
     if PY2:

copyparty/star.py

@@ -33,10 +33,11 @@ class QFile(object):
 class StreamTar(object):
     """construct in-memory tar file from the given path"""
 
-    def __init__(self, fgen, **kwargs):
+    def __init__(self, log, fgen, **kwargs):
         self.ci = 0
         self.co = 0
         self.qfile = QFile()
+        self.log = log
         self.fgen = fgen
         self.errf = None
 
@@ -91,7 +92,8 @@ class StreamTar(object):
                 errors.append([f["vp"], repr(ex)])
 
         if errors:
-            self.errf = errdesc(errors)
+            self.errf, txt = errdesc(errors)
+            self.log("\n".join(([repr(self.errf)] + txt[1:])))
             self.ser(self.errf)
 
         self.tar.close()

copyparty/sutil.py

@@ -25,4 +25,4 @@ def errdesc(errors):
         "vp": "archive-errors-{}.txt".format(dt),
         "ap": tf_path,
         "st": os.stat(tf_path),
-    }
+    }, report

copyparty/svchub.py

@@ -5,12 +5,13 @@ import re
 import os
 import sys
 import time
+import shlex
 import threading
 from datetime import datetime, timedelta
 import calendar
 
-from .__init__ import PY2, WINDOWS, MACOS, VT100
+from .__init__ import E, PY2, WINDOWS, MACOS, VT100
-from .util import mp
+from .util import mp, start_log_thrs, start_stackmon
 from .authsrv import AuthSrv
 from .tcpsrv import TcpSrv
 from .up2k import Up2k
@@ -28,14 +29,24 @@ class SvcHub(object):
     put() can return a queue (if want_reply=True) which has a blocking get() with the response.
     """
 
-    def __init__(self, args):
+    def __init__(self, args, argv, printed):
         self.args = args
+        self.argv = argv
+        self.logf = None
 
         self.ansi_re = re.compile("\033\\[[^m]*m")
         self.log_mutex = threading.Lock()
         self.next_day = 0
 
         self.log = self._log_disabled if args.q else self._log_enabled
+        if args.lo:
+            self._setup_logfile(printed)
+
+        if args.stackmon:
+            start_stackmon(args.stackmon, 0)
+
+        if args.log_thrs:
+            start_log_thrs(self.log, args.log_thrs, 0)
 
         # initiate all services to manage
         self.asrv = AuthSrv(self.args, self.log, False)
@@ -69,6 +80,52 @@ class SvcHub(object):
 
         self.broker = Broker(self)
 
+    def _logname(self):
+        dt = datetime.utcfromtimestamp(time.time())
+        fn = self.args.lo
+        for fs in "YmdHMS":
+            fs = "%" + fs
+            if fs in fn:
+                fn = fn.replace(fs, dt.strftime(fs))
+
+        return fn
+
+    def _setup_logfile(self, printed):
+        base_fn = fn = sel_fn = self._logname()
+        if fn != self.args.lo:
+            ctr = 0
+            # yup this is a race; if started sufficiently concurrently, two
+            # copyparties can grab the same logfile (considered and ignored)
+            while os.path.exists(sel_fn):
+                ctr += 1
+                sel_fn = "{}.{}".format(fn, ctr)
+
+        fn = sel_fn
+
+        try:
+            import lzma
+
+            lh = lzma.open(fn, "wt", encoding="utf-8", errors="replace", preset=0)
+
+        except:
+            import codecs
+
+            lh = codecs.open(fn, "w", encoding="utf-8", errors="replace")
+
+        lh.base_fn = base_fn
+
+        argv = [sys.executable] + self.argv
+        if hasattr(shlex, "quote"):
+            argv = [shlex.quote(x) for x in argv]
+        else:
+            argv = ['"{}"'.format(x) for x in argv]
+
+        msg = "[+] opened logfile [{}]\n".format(fn)
+        printed += msg
+        lh.write("t0: {:.3f}\nargv: {}\n\n{}".format(E.t0, " ".join(argv), printed))
+        self.logf = lh
+        print(msg, end="")
+
     def run(self):
         thr = threading.Thread(target=self.tcpsrv.run, name="svchub-main")
         thr.daemon = True
@@ -99,9 +156,36 @@ class SvcHub(object):
             print("nailed it", end="")
         finally:
             print("\033[0m")
+            if self.logf:
+                self.logf.close()
 
     def _log_disabled(self, src, msg, c=0):
-        pass
+        if not self.logf:
+            return
+
+        with self.log_mutex:
+            ts = datetime.utcfromtimestamp(time.time())
+            ts = ts.strftime("%Y-%m%d-%H%M%S.%f")[:-3]
+            self.logf.write("@{} [{}] {}\n".format(ts, src, msg))
+
+            now = time.time()
+            if now >= self.next_day:
+                self._set_next_day()
+
+    def _set_next_day(self):
+        if self.next_day and self.logf and self.logf.base_fn != self._logname():
+            self.logf.close()
+            self._setup_logfile("")
+
+        dt = datetime.utcfromtimestamp(time.time())
+
+        # unix timestamp of next 00:00:00 (leap-seconds safe)
+        day_now = dt.day
+        while dt.day == day_now:
+            dt += timedelta(hours=12)
+
+        dt = dt.replace(hour=0, minute=0, second=0)
+        self.next_day = calendar.timegm(dt.utctimetuple())
 
     def _log_enabled(self, src, msg, c=0):
         """handles logging from all components"""
@@ -110,14 +194,7 @@ class SvcHub(object):
             if now >= self.next_day:
                 dt = datetime.utcfromtimestamp(now)
                 print("\033[36m{}\033[0m\n".format(dt.strftime("%Y-%m-%d")), end="")
-
+                self._set_next_day()
-                # unix timestamp of next 00:00:00 (leap-seconds safe)
-                day_now = dt.day
-                while dt.day == day_now:
-                    dt += timedelta(hours=12)
-
-                dt = dt.replace(hour=0, minute=0, second=0)
-                self.next_day = calendar.timegm(dt.utctimetuple())
 
             fmt = "\033[36m{} \033[33m{:21} \033[0m{}\n"
             if not VT100:
@@ -144,20 +221,20 @@ class SvcHub(object):
                 except:
                     print(msg.encode("ascii", "replace").decode(), end="")
 
+            if self.logf:
+                self.logf.write(msg)
+
     def check_mp_support(self):
         vmin = sys.version_info[1]
         if WINDOWS:
             msg = "need python 3.3 or newer for multiprocessing;"
-            if PY2:
+            if PY2 or vmin < 3:
-                # py2 pickler doesn't support winsock
-                return msg
-            elif vmin < 3:
                 return msg
         elif MACOS:
             return "multiprocessing is wonky on mac osx;"
         else:
-            msg = "need python 2.7 or 3.3+ for multiprocessing;"
+            msg = "need python 3.3+ for multiprocessing;"
-            if not PY2 and vmin < 3:
+            if PY2 or vmin < 3:
                 return msg
 
         try:
@@ -189,5 +266,5 @@ class SvcHub(object):
         if not err:
             return True
         else:
-            self.log("root", err)
+            self.log("svchub", err)
             return False

copyparty/szip.py

@@ -4,15 +4,14 @@ from __future__ import print_function, unicode_literals
 import os
 import time
 import zlib
-import struct
 from datetime import datetime
 
 from .sutil import errdesc
-from .util import yieldfile, sanitize_fn
+from .util import yieldfile, sanitize_fn, spack, sunpack
 
 
 def dostime2unix(buf):
-    t, d = struct.unpack("<HH", buf)
+    t, d = sunpack(b"<HH", buf)
 
     ts = (t & 0x1F) * 2
     tm = (t >> 5) & 0x3F
@@ -36,13 +35,13 @@ def unixtime2dos(ts):
 
     bd = ((dy - 1980) << 9) + (dm << 5) + dd
     bt = (th << 11) + (tm << 5) + ts // 2
-    return struct.pack("<HH", bt, bd)
+    return spack(b"<HH", bt, bd)
 
 
 def gen_fdesc(sz, crc32, z64):
     ret = b"\x50\x4b\x07\x08"
-    fmt = "<LQQ" if z64 else "<LLL"
+    fmt = b"<LQQ" if z64 else b"<LLL"
-    ret += struct.pack(fmt, crc32, sz, sz)
+    ret += spack(fmt, crc32, sz, sz)
     return ret
 
 
@@ -66,7 +65,7 @@ def gen_hdr(h_pos, fn, sz, lastmod, utf8, crc32, pre_crc):
     req_ver = b"\x2d\x00" if z64 else b"\x0a\x00"
 
     if crc32:
-        crc32 = struct.pack("<L", crc32)
+        crc32 = spack(b"<L", crc32)
     else:
         crc32 = b"\x00" * 4
 
@@ -87,14 +86,14 @@ def gen_hdr(h_pos, fn, sz, lastmod, utf8, crc32, pre_crc):
     # however infozip does actual sz and it even works on winxp
     # (same reasning for z64 extradata later)
     vsz = 0xFFFFFFFF if z64 else sz
-    ret += struct.pack("<LL", vsz, vsz)
+    ret += spack(b"<LL", vsz, vsz)
 
     # windows support (the "?" replace below too)
-    fn = sanitize_fn(fn, ok="/")
+    fn = sanitize_fn(fn, "/", [])
     bfn = fn.encode("utf-8" if utf8 else "cp437", "replace").replace(b"?", b"_")
 
     z64_len = len(z64v) * 8 + 4 if z64v else 0
-    ret += struct.pack("<HH", len(bfn), z64_len)
+    ret += spack(b"<HH", len(bfn), z64_len)
 
     if h_pos is not None:
         # 2b comment, 2b diskno
@@ -106,12 +105,12 @@ def gen_hdr(h_pos, fn, sz, lastmod, utf8, crc32, pre_crc):
         ret += b"\x01\x00\x00\x00\xa4\x81"
 
         # 4b local-header-ofs
-        ret += struct.pack("<L", min(h_pos, 0xFFFFFFFF))
+        ret += spack(b"<L", min(h_pos, 0xFFFFFFFF))
 
     ret += bfn
 
     if z64v:
-        ret += struct.pack("<HH" + "Q" * len(z64v), 1, len(z64v) * 8, *z64v)
+        ret += spack(b"<HH" + b"Q" * len(z64v), 1, len(z64v) * 8, *z64v)
 
     return ret
 
@@ -136,7 +135,7 @@ def gen_ecdr(items, cdir_pos, cdir_end):
     need_64 = nitems == 0xFFFF or 0xFFFFFFFF in [csz, cpos]
 
     # 2b tnfiles, 2b dnfiles, 4b dir sz, 4b dir pos
-    ret += struct.pack("<HHLL", nitems, nitems, csz, cpos)
+    ret += spack(b"<HHLL", nitems, nitems, csz, cpos)
 
     # 2b comment length
     ret += b"\x00\x00"
@@ -163,7 +162,7 @@ def gen_ecdr64(items, cdir_pos, cdir_end):
 
     # 8b tnfiles, 8b dnfiles, 8b dir sz, 8b dir pos
     cdir_sz = cdir_end - cdir_pos
-    ret += struct.pack("<QQQQ", len(items), len(items), cdir_sz, cdir_pos)
+    ret += spack(b"<QQQQ", len(items), len(items), cdir_sz, cdir_pos)
 
     return ret
 
@@ -178,13 +177,14 @@ def gen_ecdr64_loc(ecdr64_pos):
     ret = b"\x50\x4b\x06\x07"
 
     # 4b cdisk, 8b start of ecdr64, 4b ndisks
-    ret += struct.pack("<LQL", 0, ecdr64_pos, 1)
+    ret += spack(b"<LQL", 0, ecdr64_pos, 1)
 
     return ret
 
 
 class StreamZip(object):
-    def __init__(self, fgen, utf8=False, pre_crc=False):
+    def __init__(self, log, fgen, utf8=False, pre_crc=False):
+        self.log = log
         self.fgen = fgen
         self.utf8 = utf8
         self.pre_crc = pre_crc
@@ -247,8 +247,8 @@ class StreamZip(object):
                 errors.append([f["vp"], repr(ex)])
 
         if errors:
-            errf = errdesc(errors)
+            errf, txt = errdesc(errors)
-            print(repr(errf))
+            self.log("\n".join(([repr(errf)] + txt[1:])))
             for x in self.ser(errf):
                 yield x
 

copyparty/tcpsrv.py

@@ -2,11 +2,9 @@
 from __future__ import print_function, unicode_literals
 
 import re
-import time
 import socket
-import select
 
-from .util import chkcmd, Counter
+from .util import chkcmd
 
 
 class TcpSrv(object):
@@ -20,7 +18,6 @@ class TcpSrv(object):
         self.args = hub.args
         self.log = hub.log
 
-        self.num_clients = Counter()
         self.stopping = False
 
         ip = "127.0.0.1"
@@ -66,44 +63,13 @@ class TcpSrv(object):
         for srv in self.srv:
             srv.listen(self.args.nc)
             ip, port = srv.getsockname()
-            self.log("tcpsrv", "listening @ {0}:{1}".format(ip, port))
+            fno = srv.fileno()
+            msg = "listening @ {}:{}  f{}".format(ip, port, fno)
+            self.log("tcpsrv", msg)
+            if self.args.q:
+                print(msg)
 
-        while not self.stopping:
+            self.hub.broker.put(False, "listen", srv)
-            if self.args.log_conn:
-                self.log("tcpsrv", "|%sC-ncli" % ("-" * 1,), c="1;30")
-
-            if self.num_clients.v >= self.args.nc:
-                time.sleep(0.1)
-                continue
-
-            if self.args.log_conn:
-                self.log("tcpsrv", "|%sC-acc1" % ("-" * 2,), c="1;30")
-
-            try:
-                # macos throws bad-fd
-                ready, _, _ = select.select(self.srv, [], [])
-            except:
-                ready = []
-                if not self.stopping:
-                    raise
-
-            for srv in ready:
-                if self.stopping:
-                    break
-
-                sck, addr = srv.accept()
-                sip, sport = srv.getsockname()
-                if self.args.log_conn:
-                    self.log(
-                        "%s %s" % addr,
-                        "|{}C-acc2 \033[0;36m{} \033[3{}m{}".format(
-                            "-" * 3, sip, sport % 8, sport
-                        ),
-                        c="1;30",
-                    )
-
-                self.num_clients.add()
-                self.hub.broker.put(False, "httpconn", sck, addr)
 
     def shutdown(self):
         self.stopping = True

copyparty/th_srv.py

@@ -9,15 +9,11 @@ import hashlib
 import threading
 import subprocess as sp
 
-from .__init__ import PY2
+from .__init__ import PY2, unicode
 from .util import fsenc, runcmd, Queue, Cooldown, BytesIO, min_ex
 from .mtag import HAVE_FFMPEG, HAVE_FFPROBE, ffprobe
 
 
-if not PY2:
-    unicode = str
-
-
 HAVE_PIL = False
 HAVE_HEIF = False
 HAVE_AVIF = False
@@ -53,7 +49,7 @@ except:
 # https://pillow.readthedocs.io/en/stable/handbook/image-file-formats.html
 # ffmpeg -formats
 FMT_PIL = "bmp dib gif icns ico jpg jpeg jp2 jpx pcx png pbm pgm ppm pnm sgi tga tif tiff webp xbm dds xpm"
-FMT_FF = "av1 asf avi flv m4v mkv mjpeg mjpg mpg mpeg mpg2 mpeg2 h264 avc h265 hevc mov 3gp mp4 ts mpegts nut ogv ogm rm vob webm wmv"
+FMT_FF = "av1 asf avi flv m4v mkv mjpeg mjpg mpg mpeg mpg2 mpeg2 h264 avc mts h265 hevc mov 3gp mp4 ts mpegts nut ogv ogm rm vob webm wmv"
 
 if HAVE_HEIF:
     FMT_PIL += " heif heifs heic heics"
@@ -134,9 +130,10 @@ class ThumbSrv(object):
             msg += ", ".join(missing)
             self.log(msg, c=3)
 
-        t = threading.Thread(target=self.cleaner, name="thumb-cleaner")
+        if self.args.th_clean:
-        t.daemon = True
+            t = threading.Thread(target=self.cleaner, name="thumb-cleaner")
-        t.start()
+            t.daemon = True
+            t.start()
 
     def log(self, msg, c=0):
         self.log_func("thumb", msg, c)
@@ -263,7 +260,7 @@ class ThumbSrv(object):
                 pass  # default q = 75
 
             if im.mode not in fmts:
-                print("conv {}".format(im.mode))
+                # print("conv {}".format(im.mode))
                 im = im.convert("RGB")
 
             im.save(tpath, quality=40, method=6)

copyparty/up2k.py

@@ -103,13 +103,15 @@ class Up2k(object):
             self.deferred_init()
         else:
             t = threading.Thread(
-                target=self.deferred_init,
+                target=self.deferred_init, name="up2k-deferred-init", args=(0.5,)
-                name="up2k-deferred-init",
             )
             t.daemon = True
             t.start()
 
-    def deferred_init(self):
+    def deferred_init(self, wait=0):
+        if wait:
+            time.sleep(wait)
+
         all_vols = self.asrv.vfs.all_vols
         have_e2d = self.init_indexes(all_vols)
 
@@ -193,7 +195,7 @@ class Up2k(object):
 
         return True, ret
 
-    def init_indexes(self, all_vols, scan_vols=[]):
+    def init_indexes(self, all_vols, scan_vols=None):
         self.pp = ProgressPrinter()
         vols = all_vols.values()
         t0 = time.time()
@@ -342,7 +344,15 @@ class Up2k(object):
             for k, v in flags.items()
         ]
         if a:
-            self.log(" ".join(sorted(a)) + "\033[0m")
+            vpath = "?"
+            for k, v in self.asrv.vfs.all_vols.items():
+                if v.realpath == ptop:
+                    vpath = k
+
+            if vpath:
+                vpath += "/"
+
+            self.log("/{} {}".format(vpath, " ".join(sorted(a))), "35")
 
         reg = {}
         path = os.path.join(histpath, "up2k.snap")
@@ -401,7 +411,7 @@ class Up2k(object):
             if WINDOWS:
                 excl = [x.replace("/", "\\") for x in excl]
 
-            n_add = self._build_dir(dbw, top, set(excl), top, nohash)
+            n_add = self._build_dir(dbw, top, set(excl), top, nohash, [])
             n_rm = self._drop_lost(dbw[0], top)
             if dbw[1]:
                 self.log("commit {} new files".format(dbw[1]))
@@ -409,11 +419,25 @@ class Up2k(object):
 
             return True, n_add or n_rm or do_vac
 
-    def _build_dir(self, dbw, top, excl, cdir, nohash):
+    def _build_dir(self, dbw, top, excl, cdir, nohash, seen):
+        rcdir = cdir
+        if not ANYWIN:
+            try:
+                # a bit expensive but worth
+                rcdir = os.path.realpath(cdir)
+            except:
+                pass
+
+        if rcdir in seen:
+            m = "bailing from symlink loop,\n  prev: {}\n  curr: {}\n  from: {}"
+            self.log(m.format(seen[-1], rcdir, cdir), 3)
+            return 0
+
+        seen = seen + [cdir]
         self.pp.msg = "a{} {}".format(self.pp.n, cdir)
         histpath = self.asrv.vfs.histtab[top]
         ret = 0
-        g = statdir(self.log, not self.args.no_scandir, False, cdir)
+        g = statdir(self.log_func, not self.args.no_scandir, False, cdir)
         for iname, inf in sorted(g):
             abspath = os.path.join(cdir, iname)
             lmod = int(inf.st_mtime)
@@ -422,7 +446,7 @@ class Up2k(object):
                 if abspath in excl or abspath == histpath:
                     continue
                 # self.log(" dir: {}".format(abspath))
-                ret += self._build_dir(dbw, top, excl, abspath, nohash)
+                ret += self._build_dir(dbw, top, excl, abspath, nohash, seen)
             else:
                 # self.log("file: {}".format(abspath))
                 rp = abspath[len(top) + 1 :]
@@ -967,7 +991,7 @@ class Up2k(object):
                 if cj["ptop"] not in self.registry:
                     raise Pebkac(410, "location unavailable")
 
-        cj["name"] = sanitize_fn(cj["name"], bad=[".prologue.html", ".epilogue.html"])
+        cj["name"] = sanitize_fn(cj["name"], "", [".prologue.html", ".epilogue.html"])
         cj["poke"] = time.time()
         wark = self._get_wark(cj)
         now = time.time()

copyparty/util.py

@@ -16,6 +16,7 @@ import mimetypes
 import contextlib
 import subprocess as sp  # nosec
 from datetime import datetime
+from collections import Counter
 
 from .__init__ import PY2, WINDOWS, ANYWIN
 from .stolen import surrogateescape
@@ -42,6 +43,20 @@ else:
     from Queue import Queue  # pylint: disable=import-error,no-name-in-module
     from StringIO import StringIO as BytesIO
 
+
+try:
+    struct.unpack(b">i", b"idgi")
+    spack = struct.pack
+    sunpack = struct.unpack
+except:
+
+    def spack(f, *a, **ka):
+        return struct.pack(f.decode("ascii"), *a, **ka)
+
+    def sunpack(f, *a, **ka):
+        return struct.unpack(f.decode("ascii"), *a, **ka)
+
+
 surrogateescape.register_surrogateescape()
 FS_ENCODING = sys.getfilesystemencoding()
 if WINDOWS and PY2:
@@ -123,20 +138,6 @@ REKOBO_KEY = {
 REKOBO_LKEY = {k.lower(): v for k, v in REKOBO_KEY.items()}
 
 
-class Counter(object):
-    def __init__(self, v=0):
-        self.v = v
-        self.mutex = threading.Lock()
-
-    def add(self, delta=1):
-        with self.mutex:
-            self.v += delta
-
-    def set(self, absval):
-        with self.mutex:
-            self.v = absval
-
-
 class Cooldown(object):
     def __init__(self, maxage):
         self.maxage = maxage
@@ -231,7 +232,7 @@ def nuprint(msg):
 
 def rice_tid():
     tid = threading.current_thread().ident
-    c = struct.unpack(b"B" * 5, struct.pack(b">Q", tid)[-5:])
+    c = sunpack(b"B" * 5, spack(b">Q", tid)[-5:])
     return "".join("\033[1;37;48;5;{}m{:02x}".format(x, x) for x in c) + "\033[0m"
 
 
@@ -282,15 +283,69 @@ def alltrace():
     return "\n".join(rret + bret)
 
 
+def start_stackmon(arg_str, nid):
+    suffix = "-{}".format(nid) if nid else ""
+    fp, f = arg_str.rsplit(",", 1)
+    f = int(f)
+    t = threading.Thread(
+        target=stackmon,
+        args=(fp, f, suffix),
+        name="stackmon" + suffix,
+    )
+    t.daemon = True
+    t.start()
+
+
+def stackmon(fp, ival, suffix):
+    ctr = 0
+    while True:
+        ctr += 1
+        time.sleep(ival)
+        st = "{}, {}\n{}".format(ctr, time.time(), alltrace())
+        with open(fp + suffix, "wb") as f:
+            f.write(st.encode("utf-8", "replace"))
+
+
+def start_log_thrs(logger, ival, nid):
+    ival = int(ival)
+    tname = lname = "log-thrs"
+    if nid:
+        tname = "logthr-n{}-i{:x}".format(nid, os.getpid())
+        lname = tname[3:]
+
+    t = threading.Thread(
+        target=log_thrs,
+        args=(logger, ival, lname),
+        name=tname,
+    )
+    t.daemon = True
+    t.start()
+
+
+def log_thrs(log, ival, name):
+    while True:
+        time.sleep(ival)
+        tv = [x.name for x in threading.enumerate()]
+        tv = [
+            x.split("-")[0]
+            if x.startswith("httpconn-") or x.startswith("thumb-")
+            else "listen"
+            if "-listen-" in x
+            else x
+            for x in tv
+            if not x.startswith("pydevd.")
+        ]
+        tv = ["{}\033[36m{}".format(v, k) for k, v in sorted(Counter(tv).items())]
+        log(name, "\033[0m \033[33m".join(tv), 3)
+
+
 def min_ex():
     et, ev, tb = sys.exc_info()
-    tb = traceback.extract_tb(tb, 2)
+    tb = traceback.extract_tb(tb)
-    ex = [
+    fmt = "{} @ {} <{}>: {}"
-        "{} @ {} <{}>: {}".format(fp.split(os.sep)[-1], ln, fun, txt)
+    ex = [fmt.format(fp.split(os.sep)[-1], ln, fun, txt) for fp, ln, fun, txt in tb]
-        for fp, ln, fun, txt in tb
+    ex.append("[{}] {}".format(et.__name__, ev))
-    ]
+    return "\n".join(ex[-8:])
-    ex.append("{}: {}".format(et.__name__, ev))
-    return "\n".join(ex)
 
 
 @contextlib.contextmanager
@@ -674,7 +729,7 @@ def undot(path):
     return "/".join(ret)
 
 
-def sanitize_fn(fn, ok="", bad=[]):
+def sanitize_fn(fn, ok, bad):
     if "/" not in ok:
         fn = fn.replace("\\", "/").split("/")[-1]
 
@@ -904,16 +959,10 @@ def yieldfile(fn):
             yield buf
 
 
-def hashcopy(actor, fin, fout):
+def hashcopy(fin, fout):
-    is_mp = actor.is_mp
     hashobj = hashlib.sha512()
     tlen = 0
     for buf in fin:
-        if is_mp:
-            actor.workload += 1
-            if actor.workload > 2 ** 31:
-                actor.workload = 100
-
         tlen += len(buf)
         hashobj.update(buf)
         fout.write(buf)
@@ -924,15 +973,10 @@ def hashcopy(actor, fin, fout):
     return tlen, hashobj.hexdigest(), digest_b64
 
 
-def sendfile_py(lower, upper, f, s, actor=None):
+def sendfile_py(lower, upper, f, s):
     remains = upper - lower
     f.seek(lower)
     while remains > 0:
-        if actor:
-            actor.workload += 1
-            if actor.workload > 2 ** 31:
-                actor.workload = 100
-
         # time.sleep(0.01)
         buf = f.read(min(1024 * 32, remains))
         if not buf:
@@ -979,8 +1023,7 @@ def statdir(logger, scandir, lstat, top):
                     try:
                         yield [fsdec(fh.name), fh.stat(follow_symlinks=not lstat)]
                     except Exception as ex:
-                        msg = "scan-stat: \033[36m{} @ {}"
+                        logger(src, "[s] {} @ {}".format(repr(ex), fsdec(fh.path)), 6)
-                        logger(msg.format(repr(ex), fsdec(fh.path)))
         else:
             src = "listdir"
             fun = os.lstat if lstat else os.stat
@@ -989,11 +1032,10 @@ def statdir(logger, scandir, lstat, top):
                 try:
                     yield [fsdec(name), fun(abspath)]
                 except Exception as ex:
-                    msg = "list-stat: \033[36m{} @ {}"
+                    logger(src, "[s] {} @ {}".format(repr(ex), fsdec(abspath)), 6)
-                    logger(msg.format(repr(ex), fsdec(abspath)))
 
     except Exception as ex:
-        logger("{}: \033[31m{} @ {}".format(src, repr(ex), top))
+        logger(src, "{} @ {}".format(repr(ex), top), 1)
 
 
 def unescape_cookie(orig):
@@ -1035,7 +1077,7 @@ def guess_mime(url, fallback="application/octet-stream"):
     if ";" not in ret:
         if ret.startswith("text/") or ret.endswith("/javascript"):
             ret += "; charset=UTF-8"
-    
+
     return ret
 
 
@@ -1070,10 +1112,7 @@ def gzip_orig_sz(fn):
     with open(fsenc(fn), "rb") as f:
         f.seek(-4, 2)
         rv = f.read(4)
-        try:
+        return sunpack(b"I", rv)[0]
-            return struct.unpack(b"I", rv)[0]
-        except:
-            return struct.unpack("I", rv)[0]
 
 
 def py_desc():

copyparty/web/baguettebox.js

@@ -13,7 +13,7 @@ window.baguetteBox = (function () {
             captions: true,
             buttons: 'auto',
             noScrollbars: false,
-            bodyClass: 'baguetteBox-open',
+            bodyClass: 'bbox-open',
             titleTag: false,
             async: false,
             preload: 2,
@@ -22,37 +22,46 @@ window.baguetteBox = (function () {
             afterHide: null,
             onChange: null,
         },
-        overlay, slider, previousButton, nextButton, closeButton,
+        overlay, slider, btnPrev, btnNext, btnVmode, btnClose,
         currentGallery = [],
         currentIndex = 0,
         isOverlayVisible = false,
         touch = {},  // start-pos
         touchFlag = false,  // busy
-        regex = /.+\.(gif|jpe?g|png|webp)/i,
+        re_i = /.+\.(gif|jpe?g|png|webp)(\?|$)/i,
+        re_v = /.+\.(webm|mp4)(\?|$)/i,
         data = {},  // all galleries
         imagesElements = [],
-        documentLastFocus = null;
+        documentLastFocus = null,
+        isFullscreen = false,
+        vmute = false,
+        vloop = false,
+        vnext = false,
+        resume_mp = false;
 
-    var overlayClickHandler = function (event) {
+    var onFSC = function (e) {
-        if (event.target.id.indexOf('baguette-img') !== -1) {
+        isFullscreen = !!document.fullscreenElement;
+    };
+
+    var overlayClickHandler = function (e) {
+        if (e.target.id.indexOf('baguette-img') !== -1)
             hideOverlay();
-        }
     };
 
-    var touchstartHandler = function (event) {
+    var touchstartHandler = function (e) {
         touch.count++;
-        if (touch.count > 1) {
+        if (touch.count > 1)
             touch.multitouch = true;
-        }
+
-        touch.startX = event.changedTouches[0].pageX;
+        touch.startX = e.changedTouches[0].pageX;
-        touch.startY = event.changedTouches[0].pageY;
+        touch.startY = e.changedTouches[0].pageY;
     };
-    var touchmoveHandler = function (event) {
+    var touchmoveHandler = function (e) {
-        if (touchFlag || touch.multitouch) {
+        if (touchFlag || touch.multitouch)
             return;
-        }
+
-        event.preventDefault ? event.preventDefault() : event.returnValue = false;
+        e.preventDefault ? e.preventDefault() : e.returnValue = false;
-        var touchEvent = event.touches[0] || event.changedTouches[0];
+        var touchEvent = e.touches[0] || e.changedTouches[0];
         if (touchEvent.pageX - touch.startX > 40) {
             touchFlag = true;
             showPreviousImage();
@@ -65,19 +74,19 @@ window.baguetteBox = (function () {
     };
     var touchendHandler = function () {
         touch.count--;
-        if (touch.count <= 0) {
+        if (touch.count <= 0)
             touch.multitouch = false;
-        }
+
         touchFlag = false;
     };
     var contextmenuHandler = function () {
         touchendHandler();
     };
 
-    var trapFocusInsideOverlay = function (event) {
+    var trapFocusInsideOverlay = function (e) {
-        if (overlay.style.display === 'block' && (overlay.contains && !overlay.contains(event.target))) {
+        if (overlay.style.display === 'block' && (overlay.contains && !overlay.contains(e.target))) {
-            event.stopPropagation();
+            e.stopPropagation();
-            initFocus();
+            btnClose.focus();
         }
     };
 
@@ -96,33 +105,26 @@ window.baguetteBox = (function () {
         data[selector] = selectorData;
 
         [].forEach.call(galleryNodeList, function (galleryElement) {
-            if (userOptions && userOptions.filter) {
-                regex = userOptions.filter;
-            }
-
             var tagsNodeList = [];
-            if (galleryElement.tagName === 'A') {
+            if (galleryElement.tagName === 'A')
                 tagsNodeList = [galleryElement];
-            } else {
+            else
                 tagsNodeList = galleryElement.getElementsByTagName('a');
-            }
 
             tagsNodeList = [].filter.call(tagsNodeList, function (element) {
-                if (element.className.indexOf(userOptions && userOptions.ignoreClass) === -1) {
+                if (element.className.indexOf(userOptions && userOptions.ignoreClass) === -1)
-                    return regex.test(element.href);
+                    return re_i.test(element.href) || re_v.test(element.href);
-                }
             });
-            if (tagsNodeList.length === 0) {
+            if (!tagsNodeList.length)
                 return;
-            }
 
             var gallery = [];
             [].forEach.call(tagsNodeList, function (imageElement, imageIndex) {
-                var imageElementClickHandler = function (event) {
+                var imageElementClickHandler = function (e) {
-                    if (event && (event.ctrlKey || event.metaKey))
+                    if (ctrl(e))
                         return true;
 
-                    event.preventDefault ? event.preventDefault() : event.returnValue = false;
+                    e.preventDefault ? e.preventDefault() : e.returnValue = false;
                     prepareOverlay(gallery, userOptions);
                     showOverlay(imageIndex);
                 };
@@ -140,93 +142,149 @@ window.baguetteBox = (function () {
     }
 
     function clearCachedData() {
-        for (var selector in data) {
+        for (var selector in data)
-            if (data.hasOwnProperty(selector)) {
+            if (data.hasOwnProperty(selector))
                 removeFromCache(selector);
-            }
-        }
     }
 
     function removeFromCache(selector) {
-        if (!data.hasOwnProperty(selector)) {
+        if (!data.hasOwnProperty(selector))
             return;
-        }
+
         var galleries = data[selector].galleries;
         [].forEach.call(galleries, function (gallery) {
             [].forEach.call(gallery, function (imageItem) {
                 unbind(imageItem.imageElement, 'click', imageItem.eventHandler);
             });
 
-            if (currentGallery === gallery) {
+            if (currentGallery === gallery)
                 currentGallery = [];
-            }
         });
 
         delete data[selector];
     }
 
     function buildOverlay() {
-        overlay = ebi('baguetteBox-overlay');
+        overlay = ebi('bbox-overlay');
-        if (overlay) {
+        if (!overlay) {
-            slider = ebi('baguetteBox-slider');
+            var ctr = mknod('div');
-            previousButton = ebi('previous-button');
+            ctr.innerHTML = (
-            nextButton = ebi('next-button');
+                '<div id="bbox-overlay" role="dialog">' +
-            closeButton = ebi('close-button');
+                '<div id="bbox-slider"></div>' +
-            return;
+                '<button id="bbox-prev" class="bbox-btn" type="button" aria-label="Previous">&lt;</button>' +
+                '<button id="bbox-next" class="bbox-btn" type="button" aria-label="Next">&gt;</button>' +
+                '<div id="bbox-btns">' +
+                '<button id="bbox-vmode" type="button" tt="a"></button>' +
+                '<button id="bbox-close" type="button" aria-label="Close">&times;</button>' +
+                '</div></div>'
+            );
+            overlay = ctr.firstChild;
+            QS('body').appendChild(overlay);
+            tt.init();
         }
-        overlay = mknod('div');
+        slider = ebi('bbox-slider');
-        overlay.setAttribute('role', 'dialog');
+        btnPrev = ebi('bbox-prev');
-        overlay.id = 'baguetteBox-overlay';
+        btnNext = ebi('bbox-next');
-        document.getElementsByTagName('body')[0].appendChild(overlay);
+        btnVmode = ebi('bbox-vmode');
-
+        btnClose = ebi('bbox-close');
-        slider = mknod('div');
-        slider.id = 'baguetteBox-slider';
-        overlay.appendChild(slider);
-
-        previousButton = mknod('button');
-        previousButton.setAttribute('type', 'button');
-        previousButton.id = 'previous-button';
-        previousButton.setAttribute('aria-label', 'Previous');
-        previousButton.innerHTML = '&lt;';
-        overlay.appendChild(previousButton);
-
-        nextButton = mknod('button');
-        nextButton.setAttribute('type', 'button');
-        nextButton.id = 'next-button';
-        nextButton.setAttribute('aria-label', 'Next');
-        nextButton.innerHTML = '&gt;';
-        overlay.appendChild(nextButton);
-
-        closeButton = mknod('button');
-        closeButton.setAttribute('type', 'button');
-        closeButton.id = 'close-button';
-        closeButton.setAttribute('aria-label', 'Close');
-        closeButton.innerHTML = '&times;';
-        overlay.appendChild(closeButton);
-
-        previousButton.className = nextButton.className = closeButton.className = 'baguetteBox-button';
-
         bindEvents();
     }
 
-    function keyDownHandler(event) {
+    function keyDownHandler(e) {
-        switch (event.keyCode) {
+        if (e.ctrlKey || e.altKey || e.metaKey || e.isComposing)
-            case 37: // Left
+            return;
-                showPreviousImage();
+
-                break;
+        var k = e.code + '', v = vid();
-            case 39: // Right
+
-                showNextImage();
+        if (k == "ArrowLeft" || k == "KeyJ")
-                break;
+            showPreviousImage();
-            case 27: // Esc
+        else if (k == "ArrowRight" || k == "KeyL")
-                hideOverlay();
+            showNextImage();
-                break;
+        else if (k == "Escape")
-            case 36: // Home
+            hideOverlay();
-                showFirstImage(event);
+        else if (k == "Home")
-                break;
+            showFirstImage(e);
-            case 35: // End
+        else if (k == "End")
-                showLastImage(event);
+            showLastImage(e);
-                break;
+        else if (k == "Space" || k == "KeyP" || k == "KeyK")
+            playpause();
+        else if (k == "KeyU" || k == "KeyO")
+            relseek(k == "KeyU" ? -10 : 10);
+        else if (k == "KeyM" && v) {
+            v.muted = vmute = !vmute;
+            mp_ctl();
         }
+        else if (k == "KeyR" && v) {
+            vloop = !vloop;
+            vnext = vnext && !vloop;
+            setVmode();
+        }
+        else if (k == "KeyC" && v) {
+            vnext = !vnext;
+            vloop = vloop && !vnext;
+            setVmode();
+        }
+        else if (k == "KeyF")
+            try {
+                if (isFullscreen)
+                    document.exitFullscreen();
+                else
+                    v.requestFullscreen();
+            }
+            catch (ex) { }
+    }
+
+    function setVmode() {
+        var v = vid();
+        ebi('bbox-vmode').style.display = v ? '' : 'none';
+        if (!v)
+            return;
+
+        var msg = 'When video ends, ', lbl;
+        if (vloop) {
+            lbl = 'Loop';
+            msg += 'repeat it';
+        }
+        else if (vnext) {
+            lbl = 'Cont';
+            msg += 'continue to next';
+        }
+        else {
+            lbl = 'Stop';
+            msg += 'just stop'
+        }
+        btnVmode.setAttribute('aria-label', msg);
+        btnVmode.setAttribute('tt', msg);
+        btnVmode.textContent = lbl;
+
+        v.loop = vloop
+        if (vloop && v.paused)
+            v.play();
+    }
+
+    function tglVmode() {
+        if (vloop) {
+            vnext = true;
+            vloop = false;
+        }
+        else if (vnext)
+            vnext = false;
+        else
+            vloop = true;
+
+        setVmode();
+        if (tt.en)
+            tt.show.bind(this)();
+    }
+
+    function keyUpHandler(e) {
+        if (e.ctrlKey || e.altKey || e.metaKey || e.isComposing)
+            return;
+
+        var k = e.code + '';
+
+        if (k == "Space")
+            ev(e);
     }
 
     var passiveSupp = false;
@@ -248,9 +306,10 @@ window.baguetteBox = (function () {
 
     function bindEvents() {
         bind(overlay, 'click', overlayClickHandler);
-        bind(previousButton, 'click', showPreviousImage);
+        bind(btnPrev, 'click', showPreviousImage);
-        bind(nextButton, 'click', showNextImage);
+        bind(btnNext, 'click', showNextImage);
-        bind(closeButton, 'click', hideOverlay);
+        bind(btnClose, 'click', hideOverlay);
+        bind(btnVmode, 'click', tglVmode);
         bind(slider, 'contextmenu', contextmenuHandler);
         bind(overlay, 'touchstart', touchstartHandler, nonPassiveEvent);
         bind(overlay, 'touchmove', touchmoveHandler, passiveEvent);
@@ -260,9 +319,10 @@ window.baguetteBox = (function () {
 
     function unbindEvents() {
         unbind(overlay, 'click', overlayClickHandler);
-        unbind(previousButton, 'click', showPreviousImage);
+        unbind(btnPrev, 'click', showPreviousImage);
-        unbind(nextButton, 'click', showNextImage);
+        unbind(btnNext, 'click', showNextImage);
-        unbind(closeButton, 'click', hideOverlay);
+        unbind(btnClose, 'click', hideOverlay);
+        unbind(btnVmode, 'click', tglVmode);
         unbind(slider, 'contextmenu', contextmenuHandler);
         unbind(overlay, 'touchstart', touchstartHandler, nonPassiveEvent);
         unbind(overlay, 'touchmove', touchmoveHandler, passiveEvent);
@@ -271,9 +331,9 @@ window.baguetteBox = (function () {
     }
 
     function prepareOverlay(gallery, userOptions) {
-        if (currentGallery === gallery) {
+        if (currentGallery === gallery)
             return;
-        }
+
         currentGallery = gallery;
         setOptions(userOptions);
         slider.innerHTML = '';
@@ -287,8 +347,8 @@ window.baguetteBox = (function () {
             fullImage.id = 'baguette-img-' + i;
             imagesElements.push(fullImage);
 
-            imagesFiguresIds.push('baguetteBox-figure-' + i);
+            imagesFiguresIds.push('bbox-figure-' + i);
-            imagesCaptionsIds.push('baguetteBox-figcaption-' + i);
+            imagesCaptionsIds.push('bbox-figcaption-' + i);
             slider.appendChild(imagesElements[i]);
         }
         overlay.setAttribute('aria-labelledby', imagesFiguresIds.join(' '));
@@ -296,23 +356,21 @@ window.baguetteBox = (function () {
     }
 
     function setOptions(newOptions) {
-        if (!newOptions) {
+        if (!newOptions)
             newOptions = {};
-        }
+
         for (var item in defaults) {
             options[item] = defaults[item];
-            if (typeof newOptions[item] !== 'undefined') {
+            if (typeof newOptions[item] !== 'undefined')
                 options[item] = newOptions[item];
-            }
         }
         slider.style.transition = (options.animation === 'fadeIn' ? 'opacity .4s ease' :
             options.animation === 'slideIn' ? '' : 'none');
 
-        if (options.buttons === 'auto' && ('ontouchstart' in window || currentGallery.length === 1)) {
+        if (options.buttons === 'auto' && ('ontouchstart' in window || currentGallery.length === 1))
             options.buttons = false;
-        }
 
-        previousButton.style.display = nextButton.style.display = (options.buttons ? '' : 'none');
+        btnPrev.style.display = btnNext.style.display = (options.buttons ? '' : 'none');
     }
 
     function showOverlay(chosenImageIndex) {
@@ -320,11 +378,12 @@ window.baguetteBox = (function () {
             document.documentElement.style.overflowY = 'hidden';
             document.body.style.overflowY = 'scroll';
         }
-        if (overlay.style.display === 'block') {
+        if (overlay.style.display === 'block')
             return;
-        }
 
         bind(document, 'keydown', keyDownHandler);
+        bind(document, 'keyup', keyUpHandler);
+        bind(document, 'fullscreenchange', onFSC);
         currentIndex = chosenImageIndex;
         touch = {
             count: 0,
@@ -341,50 +400,44 @@ window.baguetteBox = (function () {
         // Fade in overlay
         setTimeout(function () {
             overlay.className = 'visible';
-            if (options.bodyClass && document.body.classList) {
+            if (options.bodyClass && document.body.classList)
                 document.body.classList.add(options.bodyClass);
-            }
-            if (options.afterShow) {
-                options.afterShow();
-            }
-        }, 50);
-        if (options.onChange) {
-            options.onChange(currentIndex, imagesElements.length);
-        }
-        documentLastFocus = document.activeElement;
-        initFocus();
-        isOverlayVisible = true;
-    }
 
-    function initFocus() {
+            if (options.afterShow)
-        if (options.buttons) {
+                options.afterShow();
-            previousButton.focus();
+        }, 50);
-        } else {
+
-            closeButton.focus();
+        if (options.onChange)
-        }
+            options.onChange(currentIndex, imagesElements.length);
+
+        documentLastFocus = document.activeElement;
+        btnClose.focus();
+        isOverlayVisible = true;
     }
 
     function hideOverlay(e) {
         ev(e);
+        playvid(false);
         if (options.noScrollbars) {
             document.documentElement.style.overflowY = 'auto';
             document.body.style.overflowY = 'auto';
         }
-        if (overlay.style.display === 'none') {
+        if (overlay.style.display === 'none')
             return;
-        }
 
         unbind(document, 'keydown', keyDownHandler);
+        unbind(document, 'keyup', keyUpHandler);
+        unbind(document, 'fullscreenchange', onFSC);
         // Fade out and hide the overlay
         overlay.className = '';
         setTimeout(function () {
             overlay.style.display = 'none';
-            if (options.bodyClass && document.body.classList) {
+            if (options.bodyClass && document.body.classList)
                 document.body.classList.remove(options.bodyClass);
-            }
+
-            if (options.afterHide) {
+            if (options.afterHide)
                 options.afterHide();
-            }
+
             documentLastFocus && documentLastFocus.focus();
             isOverlayVisible = false;
         }, 500);
@@ -394,59 +447,68 @@ window.baguetteBox = (function () {
         var imageContainer = imagesElements[index];
         var galleryItem = currentGallery[index];
 
-        if (typeof imageContainer === 'undefined' || typeof galleryItem === 'undefined') {
+        if (typeof imageContainer === 'undefined' || typeof galleryItem === 'undefined')
             return;  // out-of-bounds or gallery dirty
-        }
 
-        if (imageContainer.getElementsByTagName('img')[0]) {
+        if (imageContainer.querySelector('img, video'))
-            // image is loaded, cb and bail
+            // was loaded, cb and bail
-            if (callback) {
+            return callback ? callback() : null;
-                callback();
+
-            }
+        // maybe unloaded video
-            return;
+        while (imageContainer.firstChild)
-        }
+            imageContainer.removeChild(imageContainer.firstChild);
 
         var imageElement = galleryItem.imageElement,
             imageSrc = imageElement.href,
-            thumbnailElement = imageElement.getElementsByTagName('img')[0],
+            is_vid = re_v.test(imageSrc),
+            thumbnailElement = imageElement.querySelector('img, video'),
             imageCaption = typeof options.captions === 'function' ?
                 options.captions.call(currentGallery, imageElement) :
                 imageElement.getAttribute('data-caption') || imageElement.title;
 
+        imageSrc += imageSrc.indexOf('?') < 0 ? '?cache' : '&cache';
+
+        if (is_vid && index != currentIndex)
+            return;  // no preload
+
         var figure = mknod('figure');
-        figure.id = 'baguetteBox-figure-' + index;
+        figure.id = 'bbox-figure-' + index;
-        figure.innerHTML = '<div class="baguetteBox-spinner">' +
+        figure.innerHTML = '<div class="bbox-spinner">' +
-            '<div class="baguetteBox-double-bounce1"></div>' +
+            '<div class="bbox-double-bounce1"></div>' +
-            '<div class="baguetteBox-double-bounce2"></div>' +
+            '<div class="bbox-double-bounce2"></div>' +
             '</div>';
 
         if (options.captions && imageCaption) {
             var figcaption = mknod('figcaption');
-            figcaption.id = 'baguetteBox-figcaption-' + index;
+            figcaption.id = 'bbox-figcaption-' + index;
             figcaption.innerHTML = imageCaption;
             figure.appendChild(figcaption);
         }
         imageContainer.appendChild(figure);
 
-        var image = mknod('img');
+        var image = mknod(is_vid ? 'video' : 'img');
-        image.onload = function () {
+        clmod(imageContainer, 'vid', is_vid);
+
+        image.addEventListener(is_vid ? 'loadedmetadata' : 'load', function () {
             // Remove loader element
-            var spinner = document.querySelector('#baguette-img-' + index + ' .baguetteBox-spinner');
+            var spinner = document.querySelector('#baguette-img-' + index + ' .bbox-spinner');
             figure.removeChild(spinner);
-            if (!options.async && callback) {
+            if (!options.async && callback)
                 callback();
-            }
+        });
-        };
         image.setAttribute('src', imageSrc);
-        image.alt = thumbnailElement ? thumbnailElement.alt || '' : '';
+        if (is_vid) {
-        if (options.titleTag && imageCaption) {
+            image.setAttribute('controls', 'controls');
-            image.title = imageCaption;
+            image.onended = vidEnd;
         }
+        image.alt = thumbnailElement ? thumbnailElement.alt || '' : '';
+        if (options.titleTag && imageCaption)
+            image.title = imageCaption;
+
         figure.appendChild(image);
 
-        if (options.async && callback) {
+        if (options.async && callback)
             callback();
-        }
     }
 
     function showNextImage(e) {
@@ -459,26 +521,20 @@ window.baguetteBox = (function () {
         return show(currentIndex - 1);
     }
 
-    function showFirstImage(event) {
+    function showFirstImage(e) {
-        if (event) {
+        if (e)
-            event.preventDefault();
+            e.preventDefault();
-        }
+
         return show(0);
     }
 
-    function showLastImage(event) {
+    function showLastImage(e) {
-        if (event) {
+        if (e)
-            event.preventDefault();
+            e.preventDefault();
-        }
+
         return show(currentGallery.length - 1);
     }
 
-    /**
-     * Move the gallery to a specific index
-     * @param `index` {number} - the position of the image
-     * @param `gallery` {array} - gallery which should be opened, if omitted assumes the currently opened one
-     * @return {boolean} - true on success or false if the index is invalid
-     */
     function show(index, gallery) {
         if (!isOverlayVisible && index >= 0 && index < gallery.length) {
             prepareOverlay(gallery, options);
@@ -486,18 +542,25 @@ window.baguetteBox = (function () {
             return true;
         }
         if (index < 0) {
-            if (options.animation) {
+            if (options.animation)
                 bounceAnimation('left');
-            }
+
             return false;
         }
         if (index >= imagesElements.length) {
-            if (options.animation) {
+            if (options.animation)
                 bounceAnimation('right');
-            }
+
             return false;
         }
 
+        var v = vid();
+        if (v) {
+            v.src = '';
+            v.load();
+            v.parentNode.removeChild(v);
+        }
+
         currentIndex = index;
         loadImage(currentIndex, function () {
             preloadNext(currentIndex);
@@ -505,17 +568,49 @@ window.baguetteBox = (function () {
         });
         updateOffset();
 
-        if (options.onChange) {
+        if (options.onChange)
             options.onChange(currentIndex, imagesElements.length);
-        }
 
         return true;
     }
 
-    /**
+    function vid() {
-     * Triggers the bounce animation
+        return imagesElements[currentIndex].querySelector('video');
-     * @param {('left'|'right')} direction - Direction of the movement
+    }
-     */
+
+    function playvid(play) {
+        if (vid())
+            vid()[play ? 'play' : 'pause']();
+    }
+
+    function playpause() {
+        var v = vid();
+        if (v)
+            v[v.paused ? "play" : "pause"]();
+    }
+
+    function relseek(sec) {
+        if (vid())
+            vid().currentTime += sec;
+    }
+
+    function vidEnd() {
+        if (this == vid() && vnext)
+            showNextImage();
+    }
+
+    function mp_ctl() {
+        var v = vid();
+        if (!vmute && v && mp.au && !mp.au.paused) {
+            mp.fade_out();
+            resume_mp = true;
+        }
+        else if (resume_mp && (vmute || !v) && mp.au && mp.au.paused) {
+            mp.fade_in();
+            resume_mp = false;
+        }
+    }
+
     function bounceAnimation(direction) {
         slider.className = 'bounce-from-' + direction;
         setTimeout(function () {
@@ -534,21 +629,30 @@ window.baguetteBox = (function () {
         } else {
             slider.style.transform = 'translate3d(' + offset + ',0,0)';
         }
+        playvid(false);
+        var v = vid();
+        if (v) {
+            playvid(true);
+            v.muted = vmute;
+            v.loop = vloop;
+        }
+        mp_ctl();
+        setVmode();
     }
 
     function preloadNext(index) {
-        if (index - currentIndex >= options.preload) {
+        if (index - currentIndex >= options.preload)
             return;
-        }
+
         loadImage(index + 1, function () {
             preloadNext(index + 1);
         });
     }
 
     function preloadPrev(index) {
-        if (currentIndex - index >= options.preload) {
+        if (currentIndex - index >= options.preload)
             return;
-        }
+
         loadImage(index - 1, function () {
             preloadPrev(index - 1);
         });
@@ -566,7 +670,8 @@ window.baguetteBox = (function () {
         unbindEvents();
         clearCachedData();
         unbind(document, 'keydown', keyDownHandler);
-        document.getElementsByTagName('body')[0].removeChild(ebi('baguetteBox-overlay'));
+        unbind(document, 'keyup', keyUpHandler);
+        document.getElementsByTagName('body')[0].removeChild(ebi('bbox-overlay'));
         data = {};
         currentGallery = [];
         currentIndex = 0;
@@ -577,6 +682,8 @@ window.baguetteBox = (function () {
         show: show,
         showNext: showNextImage,
         showPrevious: showPreviousImage,
+        relseek: relseek,
+        playpause: playpause,
         hide: hideOverlay,
         destroy: destroyPlugin
     };

copyparty/web/browser.css

@@ -29,10 +29,10 @@ body {
 	position: fixed;
 	max-width: 34em;
 	background: #222;
-	border: 0 solid #555;
+	border: 0 solid #777;
 	overflow: hidden;
 	margin-top: 1em;
-	padding: 0 1em;
+	padding: 0 1.3em;
 	height: 0;
 	opacity: .1;
 	transition: opacity 0.14s, height 0.14s, padding 0.14s;
@@ -40,19 +40,31 @@ body {
 	border-radius: .4em;
 	z-index: 9001;
 }
+#tt.b {
+	padding: 0 2em;
+	border-radius: .5em;
+	box-shadow: 0 .2em 1em #000;
+}
 #tt.show {
-	padding: 1em;
+	padding: 1em 1.3em;
+	border-width: .4em 0;
 	height: auto;
-	border-width: .2em 0;
 	opacity: 1;
 }
+#tt.show.b {
+	padding: 1.5em 2em;
+	border-width: .5em 0;
+}
 #tt code {
 	background: #3c3c3c;
-	padding: .2em .3em;
+	padding: .1em .3em;
 	border-top: 1px solid #777;
 	border-radius: .3em;
 	font-family: monospace, monospace;
-	line-height: 2em;
+	line-height: 1.7em;
+}
+#tt em {
+	color: #f6a;
 }
 #path,
 #path * {
@@ -812,11 +824,13 @@ input.eq_gain {
 	border-bottom: 1px solid #555;
 }
 #thumbs,
-#au_osd_cv {
+#au_osd_cv,
+#u2tdate {
 	opacity: .3;
 }
 #griden.on+#thumbs,
-#au_os_ctl.on+#au_osd_cv {
+#au_os_ctl.on+#au_osd_cv,
+#u2turbo.on+#u2tdate {
 	opacity: 1;
 }
 #ghead {
@@ -888,7 +902,8 @@ html.light #ggrid a:hover {
 #pvol,
 #barbuf,
 #barpos,
-#u2conf label {
+#u2conf label,
+#ops {
 	-webkit-user-select: none;
 	-moz-user-select: none;
 	-ms-user-select: none;
@@ -921,13 +936,16 @@ html.light {
 }
 html.light #tt {
 	background: #fff;
-	border-color: #888;
+	border-color: #888 #000 #777 #000;
 	box-shadow: 0 .3em 1em rgba(0,0,0,0.4);
 }
 html.light #tt code {
 	background: #060;
 	color: #fff;
 }
+html.light #tt em {
+	color: #d38;
+}
 html.light #ops,
 html.light .opbox,
 html.light #srch_form {
@@ -1128,7 +1146,7 @@ html.light #tree::-webkit-scrollbar {
 
 
 
-#baguetteBox-overlay {
+#bbox-overlay {
 	display: none;
 	opacity: 0;
 	position: fixed;
@@ -1138,58 +1156,66 @@ html.light #tree::-webkit-scrollbar {
 	left: 0;
 	width: 100%;
 	height: 100%;
-	z-index: 1000000;
+	z-index: 10;
 	background: rgba(0, 0, 0, 0.8);
 	transition: opacity .3s ease;
 }
-#baguetteBox-overlay.visible {
+#bbox-overlay.visible {
 	opacity: 1;
 }
-#baguetteBox-overlay .full-image {
+.full-image {
     display: inline-block;
     position: relative;
     width: 100%;
     height: 100%;
     text-align: center;
 }
-#baguetteBox-overlay .full-image figure {
+.full-image figure {
 	display: inline;
 	margin: 0;
 	height: 100%;
 }
-#baguetteBox-overlay .full-image img {
+.full-image img,
+.full-image video {
 	display: inline-block;
 	width: auto;
 	height: auto;
-	max-height: 100%;
 	max-width: 100%;
+	max-height: 100%;
+	max-height: calc(100% - 1.4em);
+	margin-bottom: 1.4em;
 	vertical-align: middle;
 	box-shadow: 0 0 8px rgba(0, 0, 0, 0.6);
 }
-#baguetteBox-overlay .full-image figcaption {
+.full-image video {
+	background: #333;
+}
+.full-image figcaption {
 	display: block;
-	position: absolute;
+	position: fixed;
-	bottom: 0;
+	bottom: .1em;
 	width: 100%;
 	text-align: center;
-	line-height: 1.8;
 	white-space: normal;
 	color: #ccc;
 }
-#baguetteBox-overlay figcaption a {
+#bbox-overlay figcaption a {
 	background: rgba(0, 0, 0, 0.6);
 	border-radius: .4em;
 	padding: .3em .6em;
 }
-#baguetteBox-overlay .full-image:before {
+html.light #bbox-overlay figcaption a {
+	color: #0bf;
+}
+.full-image:before {
 	content: "";
 	display: inline-block;
 	height: 50%;
 	width: 1px;
 	margin-right: -1px;
 }
-#baguetteBox-slider {
+#bbox-slider {
-	position: absolute;
+	position: fixed;
 	left: 0;
 	top: 0;
 	height: 100%;
@@ -1197,10 +1223,10 @@ html.light #tree::-webkit-scrollbar {
 	white-space: nowrap;
 	transition: left .2s ease, transform .2s ease;
 }
-#baguetteBox-slider.bounce-from-right {
+.bounce-from-right {
 	animation: bounceFromRight .4s ease-out;
 }
-#baguetteBox-slider.bounce-from-left {
+.bounce-from-left {
 	animation: bounceFromLeft .4s ease-out;
 }
 @keyframes bounceFromRight {
@@ -1213,48 +1239,51 @@ html.light #tree::-webkit-scrollbar {
 	50% {margin-left: 30px}
 	100% {margin-left: 0}
 }
-.baguetteBox-button#next-button,
+#bbox-next,
-.baguetteBox-button#previous-button {
+#bbox-prev {
 	top: 50%;
 	top: calc(50% - 30px);
 	width: 44px;
 	height: 60px;
 } 
-.baguetteBox-button {
+.bbox-btn {
-	position: absolute;
+	position: fixed;
+}
+.bbox-btn,
+#bbox-btns>button {
 	cursor: pointer;
 	outline: none;
-	padding: 0;
+	padding: 0 .3em;
-	margin: 0;
+	margin: 0 .4em;
 	border: 0;
 	border-radius: 15%;
 	background: rgba(50, 50, 50, 0.5);
 	color: #ddd;
 	font: 1.6em sans-serif;
 	transition: background-color .3s ease;
+	line-height: 1em;
+	vertical-align: top;
 }
-.baguetteBox-button:focus,
+.bbox-btn:focus,
-.baguetteBox-button:hover {
+.bbox-btn:hover {
 	background: rgba(50, 50, 50, 0.9);
 }
-#next-button {
+button#bbox-vmode {
+	font-size: 1em;
+	line-height: 1.6em;
+}
+#bbox-next {
+	right: 1%;
+}
+#bbox-prev {
+	left: 1%;
+}
+#bbox-btns {
+	top: .5em;
 	right: 2%;
+	position: fixed;
 }
-#previous-button {
+.bbox-spinner {
-	left: 2%;
-}
-#close-button {
-	top: 20px;
-	right: 2%;
-	width: 30px;
-	height: 30px;
-}
-.baguetteBox-button svg {
-	position: absolute;
-	left: 0;
-	top: 0;
-}
-.baguetteBox-spinner {
 	width: 40px;
 	height: 40px;
 	display: inline-block;
@@ -1264,8 +1293,8 @@ html.light #tree::-webkit-scrollbar {
 	margin-top: -20px;
 	margin-left: -20px;
 }
-.baguetteBox-double-bounce1,
+.bbox-double-bounce1,
-.baguetteBox-double-bounce2 {
+.bbox-double-bounce2 {
 	width: 100%;
 	height: 100%;
 	border-radius: 50%;
@@ -1276,7 +1305,7 @@ html.light #tree::-webkit-scrollbar {
 	left: 0;
 	animation: bounce 2s infinite ease-in-out;
 }
-.baguetteBox-double-bounce2 {
+.bbox-double-bounce2 {
 	animation-delay: -1s;
 }
 @keyframes bounce {

copyparty/web/browser.js

@@ -133,6 +133,13 @@ ebi('op_cfg').innerHTML = (
 	(have_zip ? (
 		'<div><h3>folder download</h3><div id="arc_fmt"></div></div>\n'
 	) : '') +
+	'<div>\n' +
+	'	<h3>up2k switches</h3>\n' +
+	'	<div>\n' +
+	'		<a id="u2turbo" class="tgl btn ttb" href="#" tt="the yolo button, you probably DO NOT want to enable this:$N$Nuse this if you were uploading a huge amount of files and had to restart for some reason, and want to continue the upload ASAP$N$Nthis replaces the hash-check with a simple <em>&quot;does this have the same filesize on the server?&quot;</em> so if the file contents are different it will NOT be uploaded$N$Nyou should turn this off when the upload is done, and then &quot;upload&quot; the same files again to let the client verify them">turbo</a>\n' +
+	'		<a id="u2tdate" class="tgl btn ttb" href="#" tt="has no effect unless the turbo button is enabled$N$Nreduces the yolo factor by a tiny amount; checks whether the file timestamps on the server matches yours$N$Nshould <em>theoretically</em> catch most unfinished/corrupted uploads, but is not a substitute for doing a verification pass with turbo disabled afterwards">date-chk</a>\n' +
+	'	</div>\n' +
+	'</div>\n' +
 	'<div><h3>key notation</h3><div id="key_notation"></div></div>\n' +
 	'<div class="fill"><h3>hidden columns</h3><div id="hcols"></div></div>'
 );
@@ -160,12 +167,13 @@ ebi('tree').innerHTML = (
 
 
 function opclick(e) {
-	ev(e);
-
 	var dest = this.getAttribute('data-dest');
-	goto(dest);
-
 	swrite('opmode', dest || null);
+	if (ctrl(e))
+		return;
+
+	ev(e);
+	goto(dest);
 
 	var input = QS('.opview.act input:not([type="hidden"])')
 	if (input && !is_touch)
@@ -1749,7 +1757,10 @@ document.onkeydown = function (e) {
 	if (e.ctrlKey || e.altKey || e.metaKey || e.isComposing)
 		return;
 
-	var k = (e.code + ''), pos = -1, n;
+	if (QS('#bbox-overlay.visible'))
+		return;
+
+	var k = e.code + '', pos = -1, n;
 
 	if (e.shiftKey && k != 'KeyA' && k != 'KeyD')
 		return;

copyparty/web/splash.html

@@ -35,7 +35,7 @@
             </table>
         </td></tr></table>
         <div class="btns">
-            <a href="{{ avol[0] }}?stack">dump stack</a>
+            <a href="/?stack">dump stack</a>
         </div>
         {%- endif %}
 

copyparty/web/up2k.js

@@ -308,6 +308,12 @@ function U2pvis(act, btns) {
             throw 42;
         }
 
+        //console.log("oldcat %s %d, newcat %s %d, head=%d, tail=%d, file=%d, act.old=%s, act.new=%s, bz_act=%s",
+        //    oldcat, this.ctr[oldcat],
+        //    newcat, this.ctr[newcat],
+        //    this.head, this.tail, nfile,
+        //    this.is_act(oldcat), this.is_act(newcat), bz_act);
+
         fo.in = newcat;
         this.ctr[oldcat]--;
         this.ctr[newcat]++;
@@ -319,7 +325,7 @@ function U2pvis(act, btns) {
                 this.addrow(nfile);
         }
         else if (this.is_act(oldcat)) {
-            while (this.head < Math.min(this.tab.length, this.tail) && (this.head == nfile || !this.is_act(this.tab[this.head].in)))
+            while (this.head < Math.min(this.tab.length, this.tail) && this.precard[this.tab[this.head].in])
                 this.head++;
 
             if (!bz_act) {
@@ -327,9 +333,10 @@ function U2pvis(act, btns) {
                 tr.parentNode.removeChild(tr);
             }
         }
-        if (bz_act) {
+        else return;
+
+        if (bz_act)
             this.bzw();
-        }
     };
 
     this.bzw = function () {
@@ -343,7 +350,8 @@ function U2pvis(act, btns) {
 
         while (this.head - first > this.wsz) {
             var obj = ebi('f' + (first++));
-            obj.parentNode.removeChild(obj);
+            if (obj)
+                obj.parentNode.removeChild(obj);
         }
         while (last - this.tail < this.wsz && last < this.tab.length - 2) {
             var obj = ebi('f' + (++last));
@@ -376,6 +384,8 @@ function U2pvis(act, btns) {
 
     this.changecard = function (card) {
         this.act = card;
+        this.precard = has(["ok", "ng", "done"], this.act) ? {} : this.act == "bz" ? { "ok": 1, "ng": 1 } : { "ok": 1, "ng": 1, "bz": 1 };
+        this.postcard = has(["ok", "ng", "done"], this.act) ? { "bz": 1, "q": 1 } : this.act == "bz" ? { "q": 1 } : {};
         this.head = -1;
         this.tail = -1;
         var html = [];
@@ -390,22 +400,23 @@ function U2pvis(act, btns) {
             }
         }
         if (this.head == -1) {
-            var precard = has(["ok", "ng", "done"], this.act) ? {} : this.act == "bz" ? { "ok": 1, "ng": 1 } : { "ok": 1, "ng": 1, "bz": 1 },
-                postcard = has(["ok", "ng", "done"], this.act) ? { "bz": 1, "q": 1 } : this.act == "bz" ? { "q": 1 } : {};
-
             for (var a = 0; a < this.tab.length; a++) {
                 var rt = this.tab[a].in;
-                if (precard[rt]) {
+                if (this.precard[rt]) {
                     this.head = a + 1;
                     this.tail = a;
                 }
-                else if (postcard[rt]) {
+                else if (this.postcard[rt]) {
                     this.head = a;
                     this.tail = a - 1;
                     break;
                 }
             }
         }
+
+        if (this.head < 0)
+            this.head = 0;
+
         if (card == "bz") {
             for (var a = this.head - 1; a >= this.head - this.wsz && a >= 0; a--) {
                 html.unshift(this.genrow(a, true).replace(/><td>/, "><td>a "));
@@ -452,6 +463,8 @@ function U2pvis(act, btns) {
             that.changecard(newtab);
         };
     }
+
+    this.changecard(this.act);
 }
 
 
@@ -548,17 +561,21 @@ function up2k_init(subtle) {
         ask_up = bcfg_get('ask_up', true),
         flag_en = bcfg_get('flag_en', false),
         fsearch = bcfg_get('fsearch', false),
+        turbo = bcfg_get('u2turbo', false),
+        datechk = bcfg_get('u2tdate', true),
         fdom_ctr = 0,
         min_filebuf = 0;
 
     var st = {
         "files": [],
         "todo": {
+            "head": [],
             "hash": [],
             "handshake": [],
             "upload": []
         },
         "busy": {
+            "head": [],
             "hash": [],
             "handshake": [],
             "upload": []
@@ -569,6 +586,15 @@ function up2k_init(subtle) {
         }
     };
 
+    function push_t(arr, t) {
+        var sort = arr.length && arr[arr.length - 1].n > t.n;
+        arr.push(t);
+        if (sort)
+            arr.sort(function (a, b) {
+                return a.n < b.n ? -1 : 1;
+            });
+    }
+
     var pvis = new U2pvis("bz", '#u2cards');
 
     var bobslice = null;
@@ -796,7 +822,10 @@ function up2k_init(subtle) {
             ], fobj.size, draw_each);
 
             st.files.push(entry);
-            st.todo.hash.push(entry);
+            if (turbo)
+                push_t(st.todo.head, entry);
+            else
+                push_t(st.todo.hash, entry);
         }
         if (!draw_each) {
             pvis.drawcard("q");
@@ -839,20 +868,27 @@ function up2k_init(subtle) {
         if (!st.todo.handshake.length)
             return true;
 
-        var cd = st.todo.handshake[0].cooldown;
+        var t = st.todo.handshake[0],
+            cd = t.cooldown;
+
         if (cd && cd - Date.now() > 0)
             return false;
 
         // keepalive or verify
-        if (st.todo.handshake[0].keepalive ||
+        if (t.keepalive ||
-            st.todo.handshake[0].t_uploaded)
+            t.t_uploaded)
             return true;
 
         if (parallel_uploads <
             st.busy.handshake.length)
             return false;
 
-        if ((multitask ? parallel_uploads : 0) <
+        if (st.busy.handshake.length)
+            for (var n = t.n - 1; n >= t.n - parallel_uploads && n >= 0; n--)
+                if (st.files[n].t_uploading)
+                    return false;
+
+        if ((multitask ? 1 : 0) <
             st.todo.upload.length +
             st.busy.upload.length)
             return false;
@@ -891,9 +927,11 @@ function up2k_init(subtle) {
             while (window['vis_exh']) {
                 var now = Date.now(),
                     is_busy = 0 !=
+                        st.todo.head.length +
                         st.todo.hash.length +
                         st.todo.handshake.length +
                         st.todo.upload.length +
+                        st.busy.head.length +
                         st.busy.hash.length +
                         st.busy.handshake.length +
                         st.busy.upload.length;
@@ -943,6 +981,12 @@ function up2k_init(subtle) {
                     }
                 }
 
+                if (st.todo.head.length &&
+                    st.busy.head.length < parallel_uploads) {
+                    exec_head();
+                    mou_ikkai = true;
+                }
+
                 if (handshakes_permitted() &&
                     st.todo.handshake.length) {
                     exec_handshake();
@@ -1160,6 +1204,53 @@ function up2k_init(subtle) {
         segm_next();
     }
 
+    /////
+    ////
+    ///   head
+    //
+
+    function exec_head() {
+        var t = st.todo.head.shift();
+        st.busy.head.push(t);
+
+        var xhr = new XMLHttpRequest();
+        xhr.onerror = function () {
+            console.log('head onerror, retrying', t);
+            apop(st.busy.head, t);
+            st.todo.head.unshift(t);
+            tasker();
+        };
+        function orz(e) {
+            var ok = false;
+            if (xhr.status == 200) {
+                var srv_sz = xhr.getResponseHeader('Content-Length'),
+                    srv_ts = xhr.getResponseHeader('Last-Modified');
+
+                ok = t.size == srv_sz;
+                if (ok && datechk) {
+                    srv_ts = new Date(srv_ts) / 1000;
+                    ok = Math.abs(srv_ts - t.lmod) < 2;
+                }
+            }
+            apop(st.busy.head, t);
+            if (!ok)
+                return push_t(st.todo.hash, t);
+
+            t.done = true;
+            st.bytes.hashed += t.size;
+            st.bytes.uploaded += t.size;
+            pvis.seth(t.n, 1, 'YOLO');
+            pvis.seth(t.n, 2, "turbo'd");
+            pvis.move(t.n, 'ok');
+        };
+        xhr.onload = function (e) {
+            try { orz(e); } catch (ex) { vis_exh(ex + '', '', '', '', ex); }
+        };
+
+        xhr.open('HEAD', t.purl + t.name, true);
+        xhr.send();
+    }
+
     /////
     ////
     ///   handshake
@@ -1268,14 +1359,24 @@ function up2k_init(subtle) {
                     msg = '&#x1f3b7;&#x1f41b;';
 
                 if (t.postlist.length) {
+                    var arr = st.todo.upload,
+                        sort = arr.length && arr[arr.length - 1].nfile > t.n;
+
                     for (var a = 0; a < t.postlist.length; a++)
-                        st.todo.upload.push({
+                        arr.push({
                             'nfile': t.n,
                             'npart': t.postlist[a]
                         });
 
                     msg = 'uploading';
                     done = false;
+
+                    if (sort)
+                        arr.sort(function (a, b) {
+                            return a.nfile < b.nfile ? -1 :
+                            /*  */ a.nfile > b.nfile ? 1 :
+                                    a.npart < b.npart ? -1 : 1;
+                        });
                 }
                 pvis.seth(t.n, 1, msg);
                 apop(st.busy.handshake, t);
@@ -1527,6 +1628,35 @@ function up2k_init(subtle) {
         set_fsearch(!fsearch);
     }
 
+    function tgl_turbo() {
+        turbo = !turbo;
+        bcfg_set('u2turbo', turbo);
+        draw_turbo();
+    }
+
+    function tgl_datechk() {
+        datechk = !datechk;
+        bcfg_set('u2tdate', datechk);
+    }
+
+    function draw_turbo() {
+        var msgu = '<p class="warn">WARNING: turbo enabled, <span>&nbsp;client may not detect and resume incomplete uploads; see turbo-button tooltip</span></p>',
+            msgs = '<p class="warn">WARNING: turbo enabled, <span>&nbsp;search may give false-positives; see turbo-button tooltip</span></p>',
+            msg = fsearch ? msgs : msgu,
+            omsg = fsearch ? msgu : msgs,
+            html = ebi('u2foot').innerHTML,
+            ohtml = html;
+
+        if (turbo && html.indexOf(msg) === -1)
+            html = html.replace(omsg, '') + msg;
+        else if (!turbo)
+            html = html.replace(msgu, '').replace(msgs, '');
+
+        if (html !== ohtml)
+            ebi('u2foot').innerHTML = html;
+    }
+    draw_turbo();
+
     function set_fsearch(new_state) {
         var fixed = false;
 
@@ -1564,6 +1694,7 @@ function up2k_init(subtle) {
         }
         catch (ex) { }
 
+        draw_turbo();
         onresize();
     }
 
@@ -1608,6 +1739,8 @@ function up2k_init(subtle) {
     ebi('multitask').addEventListener('click', tgl_multitask, false);
     ebi('ask_up').addEventListener('click', tgl_ask_up, false);
     ebi('flag_en').addEventListener('click', tgl_flag_en, false);
+    ebi('u2turbo').addEventListener('click', tgl_turbo, false);
+    ebi('u2tdate').addEventListener('click', tgl_datechk, false);
     var o = ebi('fsearch');
     if (o)
         o.addEventListener('click', tgl_fsearch, false);

copyparty/web/upload.css

@@ -215,9 +215,31 @@
 	color: #fff;
 	font-style: italic;
 }
+#u2foot .warn {
+	font-size: 1.3em;
+	padding: .5em .8em;
+	margin: 1em -.6em;
+	color: #f74;
+	background: #322;
+	border: 1px solid #633;
+	border-width: .1em 0;
+	text-align: center;
+}
+#u2foot .warn span {
+	color: #f86;
+}
+html.light #u2foot .warn {
+	color: #b00;
+	background: #fca;
+	border-color: #f70;
+}
+html.light #u2foot .warn span {
+	color: #930;
+}
 #u2foot span {
 	color: #999;
 	font-size: .9em;
+	font-weight: normal;
 }
 #u2footfoot {
 	margin-bottom: -1em;

copyparty/web/util.js

@@ -27,20 +27,20 @@ function vis_exh(msg, url, lineNo, columnNo, error) {
 
     window.onerror = undefined;
     window['vis_exh'] = null;
-    var html = ['<h1>you hit a bug!</h1><p>please send me a screenshot arigathanks gozaimuch: <code>ed/irc.rizon.net</code> or <code>ed#2644</code><br />&nbsp; (and if you can, press F12 and include the "Console" tab in the screenshot too)</p><p>',
+    var html = ['<h1>you hit a bug!</h1><p style="font-size:1.3em;margin:0">try to <a href="#" onclick="localStorage.clear();location.reload();" style="text-decoration:underline;color:#fc0">reset copyparty settings</a> if you are stuck here</p><p>please send me a screenshot arigathanks gozaimuch: <code>ed/irc.rizon.net</code> or <code>ed#2644</code><br />&nbsp; (and if you can, press F12 and include the "Console" tab in the screenshot too)</p><p>',
-        esc(String(msg)), '</p><p>', esc(url + ' @' + lineNo + ':' + columnNo), '</p>'];
+        esc(url + ' @' + lineNo + ':' + columnNo), '<br />' + esc(String(msg)) + '</p>'];
 
     if (error) {
         var find = ['desc', 'stack', 'trace'];
         for (var a = 0; a < find.length; a++)
             if (String(error[find[a]]) !== 'undefined')
-                html.push('<h2>' + find[a] + '</h2>' +
+                html.push('<h3>' + find[a] + '</h3>' +
                     esc(String(error[find[a]])).replace(/\n/g, '<br />\n'));
     }
     document.body.innerHTML = html.join('\n');
 
     var s = mknod('style');
-    s.innerHTML = 'body{background:#333;color:#ddd;font-family:sans-serif;font-size:0.8em;padding:0 1em 1em 1em} code{color:#bf7;background:#222;padding:.1em;margin:.2em;font-size:1.1em;font-family:monospace,monospace} *{line-height:1.5em}';
+    s.innerHTML = 'body{background:#333;color:#ddd;font-family:sans-serif;font-size:0.8em;padding:0 1em 1em 1em} h1{margin:.5em 1em 0 0;padding:0} h3{border-top:1px solid #999;margin:0} code{color:#bf7;background:#222;padding:.1em;margin:.2em;font-size:1.1em;font-family:monospace,monospace} *{line-height:1.5em}';
     document.head.appendChild(s);
 
     throw 'fatal_err';
@@ -53,6 +53,11 @@ var ebi = document.getElementById.bind(document),
     mknod = document.createElement.bind(document);
 
 
+function ctrl(e) {
+    return e && (e.ctrlKey || e.metaKey);
+}
+
+
 function ev(e) {
     e = e || window.event;
     if (!e)
@@ -503,7 +508,7 @@ var tt = (function () {
     r.tt.setAttribute('id', 'tt');
     document.body.appendChild(r.tt);
 
-    function show() {
+    r.show = function () {
         var cfg = sread('tooltips');
         if (cfg !== null && cfg != '1')
             return;
@@ -514,8 +519,10 @@ var tt = (function () {
 
         var pos = this.getBoundingClientRect(),
             left = pos.left < window.innerWidth / 2,
-            top = pos.top < window.innerHeight / 2;
+            top = pos.top < window.innerHeight / 2,
+            big = this.className.indexOf(' ttb') !== -1;
 
+        clmod(r.tt, 'b', big);
         r.tt.style.top = top ? pos.bottom + 'px' : 'auto';
         r.tt.style.bottom = top ? 'auto' : (window.innerHeight - pos.top) + 'px';
         r.tt.style.left = left ? pos.left + 'px' : 'auto';
@@ -523,11 +530,11 @@ var tt = (function () {
 
         r.tt.innerHTML = msg.replace(/\$N/g, "<br />");
         clmod(r.tt, 'show', 1);
-    }
+    };
 
-    function hide() {
+    r.hide = function () {
         clmod(r.tt, 'show');
-    }
+    };
 
     r.init = function () {
         var ttb = ebi('tooltips');
@@ -541,8 +548,8 @@ var tt = (function () {
             r.en = bcfg_get('tooltips', true)
         }
 
-        var _show = r.en ? show : null,
+        var _show = r.en ? r.show : null,
-            _hide = r.en ? hide : null;
+            _hide = r.en ? r.hide : null;
 
         var o = QSA('*[tt]');
         for (var a = o.length - 1; a >= 0; a--) {
@@ -551,7 +558,7 @@ var tt = (function () {
             o[a].onmouseenter = _show;
             o[a].onmouseleave = _hide;
         }
-        hide();
+        r.hide();
     };
 
     return r;

scripts/sfx.py

@@ -6,8 +6,8 @@ import re, os, sys, time, shutil, signal, threading, tarfile, hashlib, platform,
 import subprocess as sp
 
 """
-pls don't edit this file with a text editor,
+to edit this file, use HxD or "vim -b"
-  it breaks the compressed stuff at the end
+  (there is compressed stuff at the end)
 
 run me with any version of python, i will unpack and run copyparty
 

tests/test_httpcli.py

@@ -23,13 +23,14 @@ def hdr(query):
 
 
 class Cfg(Namespace):
-    def __init__(self, a=[], v=[], c=None):
+    def __init__(self, a=None, v=None, c=None):
         super(Cfg, self).__init__(
-            a=a,
+            a=a or [],
-            v=v,
+            v=v or [],
             c=c,
             rproxy=0,
             ed=False,
+            nw=False,
             no_zip=False,
             no_scandir=False,
             no_sendfile=True,

tests/test_vfs.py

@@ -16,8 +16,8 @@ from copyparty import util
 
 
 class Cfg(Namespace):
-    def __init__(self, a=[], v=[], c=None):
+    def __init__(self, a=None, v=None, c=None):
-        ex = {k: False for k in "e2d e2ds e2dsa e2t e2ts e2tsr".split()}
+        ex = {k: False for k in "nw e2d e2ds e2dsa e2t e2ts e2tsr".split()}
         ex2 = {
             "mtp": [],
             "mte": "a",
@@ -27,7 +27,7 @@ class Cfg(Namespace):
             "rproxy": 0,
         }
         ex.update(ex2)
-        super(Cfg, self).__init__(a=a, v=v, c=c, **ex)
+        super(Cfg, self).__init__(a=a or [], v=v or [], c=c, **ex)
 
 
 class TestVFS(unittest.TestCase):

tests/util.py

@@ -126,7 +126,6 @@ class VHttpConn(object):
         self.hsrv = VHttpSrv()
         self.nreq = 0
         self.nbyte = 0
-        self.workload = 0
         self.ico = None
         self.thumbcli = None
         self.t0 = time.time()