v0.11.32

README.md

@@ -23,6 +23,7 @@ turn your phone or raspi into a portable file server with resumable uploads/down
         * [on debian](#on-debian)
     * [notes](#notes)
     * [status](#status)
+    * [testimonials](#testimonials)
 * [bugs](#bugs)
     * [general bugs](#general-bugs)
     * [not my bugs](#not-my-bugs)
@@ -143,6 +144,13 @@ summary: all planned features work! now please enjoy the bloatening
   * ☑ editor (sure why not)
 
 
+## testimonials
+
+small collection of user feedback
+
+`good enough`, `surprisingly correct`, `certified good software`, `just works`, `why`
+
+
 # bugs
 
 * Windows: python 3.7 and older cannot read tags with ffprobe, so use mutagen or upgrade
@@ -618,6 +626,7 @@ roughly sorted by priority
 * reduce up2k roundtrips
   * start from a chunk index and just go
   * terminate client on bad data
+* logging to file
 
 discarded ideas
 
@@ -637,3 +646,6 @@ discarded ideas
   * nah
 * look into android thumbnail cache file format
   * absolutely not
+* indexedDB for hashes, cfg enable/clear/sz, 2gb avail, ~9k for 1g, ~4k for 100m, 500k items before autoeviction
+  * blank hashlist when up-ok to skip handshake
+    * too many confusing side-effects

copyparty/__init__.py

@@ -9,6 +9,9 @@ import os
 PY2 = sys.version_info[0] == 2
 if PY2:
     sys.dont_write_bytecode = True
+    unicode = unicode
+else:
+    unicode = str
 
 WINDOWS = False
 if platform.system() == "Windows":

copyparty/__main__.py

@@ -20,7 +20,7 @@ import threading
 import traceback
 from textwrap import dedent
 
-from .__init__ import E, WINDOWS, VT100, PY2
+from .__init__ import E, WINDOWS, VT100, PY2, unicode
 from .__version__ import S_VERSION, S_BUILD_DT, CODENAME
 from .svchub import SvcHub
 from .util import py_desc, align_tab, IMPLICATIONS, alltrace
@@ -31,6 +31,8 @@ try:
 except:
     HAVE_SSL = False
 
+printed = ""
+
 
 class RiceFormatter(argparse.HelpFormatter):
     def _get_help_string(self, action):
@@ -61,8 +63,15 @@ class Dodge11874(RiceFormatter):
         super(Dodge11874, self).__init__(*args, **kwargs)
 
 
+def lprint(*a, **ka):
+    global printed
+
+    printed += " ".join(unicode(x) for x in a) + ka.get("end", "\n")
+    print(*a, **ka)
+
+
 def warn(msg):
-    print("\033[1mwarning:\033[0;33m {}\033[0m\n".format(msg))
+    lprint("\033[1mwarning:\033[0;33m {}\033[0m\n".format(msg))
 
 
 def ensure_locale():
@@ -73,7 +82,7 @@ def ensure_locale():
     ]:
         try:
             locale.setlocale(locale.LC_ALL, x)
-            print("Locale:", x)
+            lprint("Locale:", x)
             break
         except:
             continue
@@ -94,7 +103,7 @@ def ensure_cert():
 
     try:
         if filecmp.cmp(cert_cfg, cert_insec):
-            print(
+            lprint(
                 "\033[33m  using default TLS certificate; https will be insecure."
                 + "\033[36m\n  certificate location: {}\033[0m\n".format(cert_cfg)
             )
@@ -123,7 +132,7 @@ def configure_ssl_ver(al):
     if "help" in sslver:
         avail = [terse_sslver(x[6:]) for x in flags]
         avail = " ".join(sorted(avail) + ["all"])
-        print("\navailable ssl/tls versions:\n  " + avail)
+        lprint("\navailable ssl/tls versions:\n  " + avail)
         sys.exit(0)
 
     al.ssl_flags_en = 0
@@ -143,7 +152,7 @@ def configure_ssl_ver(al):
 
     for k in ["ssl_flags_en", "ssl_flags_de"]:
         num = getattr(al, k)
-        print("{}: {:8x} ({})".format(k, num, num))
+        lprint("{}: {:8x} ({})".format(k, num, num))
 
     # think i need that beer now
 
@@ -160,13 +169,13 @@ def configure_ssl_ciphers(al):
         try:
             ctx.set_ciphers(al.ciphers)
         except:
-            print("\n\033[1;31mfailed to set ciphers\033[0m\n")
+            lprint("\n\033[1;31mfailed to set ciphers\033[0m\n")
 
     if not hasattr(ctx, "get_ciphers"):
-        print("cannot read cipher list: openssl or python too old")
+        lprint("cannot read cipher list: openssl or python too old")
     else:
         ciphers = [x["description"] for x in ctx.get_ciphers()]
-        print("\n  ".join(["\nenabled ciphers:"] + align_tab(ciphers) + [""]))
+        lprint("\n  ".join(["\nenabled ciphers:"] + align_tab(ciphers) + [""]))
 
     if is_help:
         sys.exit(0)
@@ -249,17 +258,18 @@ def run_argparse(argv, formatter):
         ),
     )
     # fmt: off
-    ap.add_argument("-c", metavar="PATH", type=str, action="append", help="add config file")
-    ap.add_argument("-nc", metavar="NUM", type=int, default=64, help="max num clients")
-    ap.add_argument("-j", metavar="CORES", type=int, default=1, help="max num cpu cores")
-    ap.add_argument("-a", metavar="ACCT", type=str, action="append", help="add account, USER:PASS; example [ed:wark")
-    ap.add_argument("-v", metavar="VOL", type=str, action="append", help="add volume, SRC:DST:FLAG; example [.::r], [/mnt/nas/music:/music:r:aed")
-    ap.add_argument("-ed", action="store_true", help="enable ?dots")
-    ap.add_argument("-emp", action="store_true", help="enable markdown plugins")
-    ap.add_argument("-mcr", metavar="SEC", type=int, default=60, help="md-editor mod-chk rate")
-    ap.add_argument("--dotpart", action="store_true", help="dotfile incomplete uploads")
-    ap.add_argument("--sparse", metavar="MiB", type=int, default=4, help="up2k min.size threshold (mswin-only)")
-    ap.add_argument("--urlform", metavar="MODE", type=str, default="print,get", help="how to handle url-forms; examples: [stash], [save,get]")
+    ap2 = ap.add_argument_group('general options')
+    ap2.add_argument("-c", metavar="PATH", type=str, action="append", help="add config file")
+    ap2.add_argument("-nc", metavar="NUM", type=int, default=64, help="max num clients")
+    ap2.add_argument("-j", metavar="CORES", type=int, default=1, help="max num cpu cores")
+    ap2.add_argument("-a", metavar="ACCT", type=str, action="append", help="add account, USER:PASS; example [ed:wark")
+    ap2.add_argument("-v", metavar="VOL", type=str, action="append", help="add volume, SRC:DST:FLAG; example [.::r], [/mnt/nas/music:/music:r:aed")
+    ap2.add_argument("-ed", action="store_true", help="enable ?dots")
+    ap2.add_argument("-emp", action="store_true", help="enable markdown plugins")
+    ap2.add_argument("-mcr", metavar="SEC", type=int, default=60, help="md-editor mod-chk rate")
+    ap2.add_argument("--dotpart", action="store_true", help="dotfile incomplete uploads")
+    ap2.add_argument("--sparse", metavar="MiB", type=int, default=4, help="up2k min.size threshold (mswin-only)")
+    ap2.add_argument("--urlform", metavar="MODE", type=str, default="print,get", help="how to handle url-forms; examples: [stash], [save,get]")
 
     ap2 = ap.add_argument_group('network options')
     ap2.add_argument("-i", metavar="IP", type=str, default="0.0.0.0", help="ip to bind (comma-sep.)")
@@ -286,6 +296,7 @@ def run_argparse(argv, formatter):
 
     ap2 = ap.add_argument_group('logging options')
     ap2.add_argument("-q", action="store_true", help="quiet")
+    ap2.add_argument("-lo", metavar="PATH", type=str, help="logfile, example: cpp-%%Y-%%m%%d-%%H%%M%%S.txt.xz")
     ap2.add_argument("--log-conn", action="store_true", help="print tcp-server msgs")
     ap2.add_argument("--ihead", metavar="HEADER", action='append', help="dump incoming header")
     ap2.add_argument("--lf-url", metavar="RE", type=str, default=r"^/\.cpr/|\?th=[wj]$", help="dont log URLs matching")
@@ -348,7 +359,7 @@ def main(argv=None):
     desc = py_desc().replace("[", "\033[1;30m[")
 
     f = '\033[36mcopyparty v{} "\033[35m{}\033[36m" ({})\n{}\033[0m\n'
-    print(f.format(S_VERSION, CODENAME, S_BUILD_DT, desc))
+    lprint(f.format(S_VERSION, CODENAME, S_BUILD_DT, desc))
 
     ensure_locale()
     if HAVE_SSL:
@@ -362,7 +373,7 @@ def main(argv=None):
             continue
 
         msg = "\033[1;31mWARNING:\033[0;1m\n  {} \033[0;33mwas replaced with\033[0;1m {} \033[0;33mand will be removed\n\033[0m"
-        print(msg.format(dk, nk))
+        lprint(msg.format(dk, nk))
         argv[idx] = nk
         time.sleep(2)
 
@@ -416,7 +427,7 @@ def main(argv=None):
 
     # signal.signal(signal.SIGINT, sighandler)
 
-    SvcHub(al).run()
+    SvcHub(al, argv, printed).run()
 
 
 if __name__ == "__main__":

copyparty/__version__.py

@@ -1,8 +1,8 @@
 # coding: utf-8
 
-VERSION = (0, 11, 29)
+VERSION = (0, 11, 32)
 CODENAME = "the grid"
-BUILD_DT = (2021, 6, 30)
+BUILD_DT = (2021, 7, 6)
 
 S_VERSION = ".".join(map(str, VERSION))
 S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT)

copyparty/authsrv.py

@@ -10,13 +10,14 @@ import hashlib
 import threading
 
 from .__init__ import WINDOWS
-from .util import IMPLICATIONS, uncyg, undot, Pebkac, fsdec, fsenc, statdir, nuprint
+from .util import IMPLICATIONS, uncyg, undot, Pebkac, fsdec, fsenc, statdir
 
 
 class VFS(object):
     """single level in the virtual fs"""
 
-    def __init__(self, realpath, vpath, uread=[], uwrite=[], uadm=[], flags={}):
+    def __init__(self, log, realpath, vpath, uread=[], uwrite=[], uadm=[], flags={}):
+        self.log = log
         self.realpath = realpath  # absolute path on host filesystem
         self.vpath = vpath  # absolute path in the virtual filesystem
         self.uread = uread  # users who can read this
@@ -62,6 +63,7 @@ class VFS(object):
                 return self.nodes[name].add(src, dst)
 
             vn = VFS(
+                self.log,
                 os.path.join(self.realpath, name) if self.realpath else None,
                 "{}/{}".format(self.vpath, name).lstrip("/"),
                 self.uread,
@@ -79,7 +81,7 @@ class VFS(object):
 
         # leaf does not exist; create and keep permissions blank
         vp = "{}/{}".format(self.vpath, dst).lstrip("/")
-        vn = VFS(src, vp)
+        vn = VFS(self.log, src, vp)
         vn.dbv = self.dbv or self
         self.nodes[dst] = vn
         return vn
@@ -181,7 +183,7 @@ class VFS(object):
         """return user-readable [fsdir,real,virt] items at vpath"""
         virt_vis = {}  # nodes readable by user
         abspath = self.canonical(rem)
-        real = list(statdir(nuprint, scandir, lstat, abspath))
+        real = list(statdir(self.log, scandir, lstat, abspath))
         real.sort()
         if not rem:
             for name, vn2 in sorted(self.nodes.items()):
@@ -208,8 +210,13 @@ class VFS(object):
             rem, uname, scandir, incl_wo=False, lstat=lstat
         )
 
-        if seen and not fsroot.startswith(seen[-1]) and fsroot in seen:
-            print("bailing from symlink loop,\n  {}\n  {}".format(seen[-1], fsroot))
+        if (
+            seen
+            and (not fsroot.startswith(seen[-1]) or fsroot == seen[-1])
+            and fsroot in seen
+        ):
+            m = "bailing from symlink loop,\n  prev: {}\n  curr: {}\n  from: {}/{}"
+            self.log("vfs.walk", m.format(seen[-1], fsroot, self.vpath, rem), 3)
             return
 
         seen = seen[:] + [fsroot]
@@ -242,6 +249,10 @@ class VFS(object):
         if flt:
             flt = {k: True for k in flt}
 
+        f1 = "{0}.hist{0}up2k.".format(os.sep)
+        f2a = os.sep + "dir.txt"
+        f2b = "{0}.hist{0}".format(os.sep)
+
         for vpath, apath, files, rd, vd in self.walk(
             "", vrem, [], uname, dots, scandir, False
         ):
@@ -275,7 +286,11 @@ class VFS(object):
                     del vd[x]
 
             # up2k filetring based on actual abspath
-            files = [x for x in files if "{0}.hist{0}up2k.".format(os.sep) not in x[1]]
+            files = [
+                x
+                for x in files
+                if f1 not in x[1] and (not x[1].endswith(f2a) or f2b not in x[1])
+            ]
 
             for f in [{"vp": v, "ap": a, "st": n[1]} for v, a, n in files]:
                 yield f
@@ -466,7 +481,7 @@ class AuthSrv(object):
                         )
                     except:
                         m = "\n\033[1;31m\nerror in config file {} on line {}:\n\033[0m"
-                        print(m.format(cfg_fn, self.line_ctr))
+                        self.log(m.format(cfg_fn, self.line_ctr), 1)
                         raise
 
         # case-insensitive; normalize
@@ -482,10 +497,10 @@ class AuthSrv(object):
 
         if not mount:
             # -h says our defaults are CWD at root and read/write for everyone
-            vfs = VFS(os.path.abspath("."), "", ["*"], ["*"])
+            vfs = VFS(self.log_func, os.path.abspath("."), "", ["*"], ["*"])
         elif "" not in mount:
             # there's volumes but no root; make root inaccessible
-            vfs = VFS(None, "")
+            vfs = VFS(self.log_func, None, "")
             vfs.flags["d2d"] = True
 
         maxdepth = 0
@@ -497,7 +512,13 @@ class AuthSrv(object):
             if dst == "":
                 # rootfs was mapped; fully replaces the default CWD vfs
                 vfs = VFS(
-                    mount[dst], dst, mread[dst], mwrite[dst], madm[dst], mflags[dst]
+                    self.log_func,
+                    mount[dst],
+                    dst,
+                    mread[dst],
+                    mwrite[dst],
+                    madm[dst],
+                    mflags[dst],
                 )
                 continue
 
@@ -780,7 +801,7 @@ class AuthSrv(object):
                         msg = [x[1] for x in files]
 
                     if msg:
-                        nuprint("\n".join(msg))
+                        self.log("\n" + "\n".join(msg))
 
                 if n_bads and flag_p:
                     raise Exception("found symlink leaving volume, and strict is set")

copyparty/httpcli.py

@@ -13,15 +13,12 @@ import ctypes
 from datetime import datetime
 import calendar
 
-from .__init__ import E, PY2, WINDOWS, ANYWIN
+from .__init__ import E, PY2, WINDOWS, ANYWIN, unicode
 from .util import *  # noqa  # pylint: disable=unused-wildcard-import
 from .authsrv import AuthSrv
 from .szip import StreamZip
 from .star import StreamTar
 
-if not PY2:
-    unicode = str
-
 
 NO_CACHE = {"Cache-Control": "no-cache"}
 NO_STORE = {"Cache-Control": "no-store; max-age=0"}
@@ -98,9 +95,13 @@ class HttpCli(object):
             try:
                 self.mode, self.req, self.http_ver = headerlines[0].split(" ")
             except:
-                raise Pebkac(400, "bad headers:\n" + "\n".join(headerlines))
+                msg = " ]\n#[ ".join(headerlines)
+                raise Pebkac(400, "bad headers:\n#[ " + msg + " ]")
 
         except Pebkac as ex:
+            self.mode = "GET"
+            self.req = "[junk]"
+            self.http_ver = "HTTP/1.1"
             # self.log("pebkac at httpcli.run #1: " + repr(ex))
             self.keepalive = self._check_nonfatal(ex)
             self.loud_reply(unicode(ex), status=ex.code)
@@ -478,15 +479,17 @@ class HttpCli(object):
         addr = self.ip.replace(":", ".")
         fn = "put-{:.6f}-{}.bin".format(time.time(), addr)
         path = os.path.join(fdir, fn)
+        if self.args.nw:
+            path = os.devnull
 
         with open(fsenc(path), "wb", 512 * 1024) as f:
             post_sz, _, sha_b64 = hashcopy(self.conn, reader, f)
 
-        vfs, vrem = vfs.get_dbv(rem)
-
-        self.conn.hsrv.broker.put(
-            False, "up2k.hash_file", vfs.realpath, vfs.flags, vrem, fn
-        )
+        if not self.args.nw:
+            vfs, vrem = vfs.get_dbv(rem)
+            self.conn.hsrv.broker.put(
+                False, "up2k.hash_file", vfs.realpath, vfs.flags, vrem, fn
+            )
 
         return post_sz, sha_b64, remains, path
 
@@ -607,13 +610,14 @@ class HttpCli(object):
                     os.makedirs(fsenc(dst))
             except OSError as ex:
                 self.log("makedirs failed [{}]".format(dst))
-                if ex.errno == 13:
-                    raise Pebkac(500, "the server OS denied write-access")
+                if not os.path.isdir(fsenc(dst)):
+                    if ex.errno == 13:
+                        raise Pebkac(500, "the server OS denied write-access")
 
-                if ex.errno == 17:
-                    raise Pebkac(400, "some file got your folder name")
+                    if ex.errno == 17:
+                        raise Pebkac(400, "some file got your folder name")
 
-                raise Pebkac(500, min_ex())
+                    raise Pebkac(500, min_ex())
             except:
                 raise Pebkac(500, min_ex())
 
@@ -1560,7 +1564,9 @@ class HttpCli(object):
             raise Pebkac(404)
 
         if self.readable:
-            if rem.startswith(".hist/up2k."):
+            if rem.startswith(".hist/up2k.") or (
+                rem.endswith("/dir.txt") and rem.startswith(".hist/th/")
+            ):
                 raise Pebkac(403)
 
             is_dir = stat.S_ISDIR(st.st_mode)

copyparty/mtag.py

@@ -7,12 +7,9 @@ import json
 import shutil
 import subprocess as sp
 
-from .__init__ import PY2, WINDOWS
+from .__init__ import PY2, WINDOWS, unicode
 from .util import fsenc, fsdec, uncyg, REKOBO_LKEY
 
-if not PY2:
-    unicode = str
-
 
 def have_ff(cmd):
     if PY2:

copyparty/svchub.py

@@ -5,11 +5,12 @@ import re
 import os
 import sys
 import time
+import shlex
 import threading
 from datetime import datetime, timedelta
 import calendar
 
-from .__init__ import PY2, WINDOWS, MACOS, VT100
+from .__init__ import E, PY2, WINDOWS, MACOS, VT100
 from .util import mp
 from .authsrv import AuthSrv
 from .tcpsrv import TcpSrv
@@ -28,14 +29,18 @@ class SvcHub(object):
     put() can return a queue (if want_reply=True) which has a blocking get() with the response.
     """
 
-    def __init__(self, args):
+    def __init__(self, args, argv, printed):
         self.args = args
+        self.argv = argv
+        self.logf = None
 
         self.ansi_re = re.compile("\033\\[[^m]*m")
         self.log_mutex = threading.Lock()
         self.next_day = 0
 
         self.log = self._log_disabled if args.q else self._log_enabled
+        if args.lo:
+            self._setup_logfile(printed)
 
         # initiate all services to manage
         self.asrv = AuthSrv(self.args, self.log, False)
@@ -69,6 +74,52 @@ class SvcHub(object):
 
         self.broker = Broker(self)
 
+    def _logname(self):
+        dt = datetime.utcfromtimestamp(time.time())
+        fn = self.args.lo
+        for fs in "YmdHMS":
+            fs = "%" + fs
+            if fs in fn:
+                fn = fn.replace(fs, dt.strftime(fs))
+
+        return fn
+
+    def _setup_logfile(self, printed):
+        base_fn = fn = sel_fn = self._logname()
+        if fn != self.args.lo:
+            ctr = 0
+            # yup this is a race; if started sufficiently concurrently, two
+            # copyparties can grab the same logfile (considered and ignored)
+            while os.path.exists(sel_fn):
+                ctr += 1
+                sel_fn = "{}.{}".format(fn, ctr)
+
+        fn = sel_fn
+
+        try:
+            import lzma
+
+            lh = lzma.open(fn, "wt", encoding="utf-8", errors="replace", preset=0)
+
+        except:
+            import codecs
+
+            lh = codecs.open(fn, "w", encoding="utf-8", errors="replace")
+
+        lh.base_fn = base_fn
+
+        argv = [sys.executable] + self.argv
+        if hasattr(shlex, "quote"):
+            argv = [shlex.quote(x) for x in argv]
+        else:
+            argv = ['"{}"'.format(x) for x in argv]
+
+        msg = "[+] opened logfile [{}]\n".format(fn)
+        printed += msg
+        lh.write("t0: {:.3f}\nargv: {}\n\n{}".format(E.t0, " ".join(argv), printed))
+        self.logf = lh
+        print(msg, end="")
+
     def run(self):
         thr = threading.Thread(target=self.tcpsrv.run, name="svchub-main")
         thr.daemon = True
@@ -99,9 +150,36 @@ class SvcHub(object):
             print("nailed it", end="")
         finally:
             print("\033[0m")
+            if self.logf:
+                self.logf.close()
 
     def _log_disabled(self, src, msg, c=0):
-        pass
+        if not self.logf:
+            return
+
+        with self.log_mutex:
+            ts = datetime.utcfromtimestamp(time.time())
+            ts = ts.strftime("%Y-%m%d-%H%M%S.%f")[:-3]
+            self.logf.write("@{} [{}] {}\n".format(ts, src, msg))
+
+            now = time.time()
+            if now >= self.next_day:
+                self._set_next_day()
+
+    def _set_next_day(self):
+        if self.next_day and self.logf and self.logf.base_fn != self._logname():
+            self.logf.close()
+            self._setup_logfile("")
+
+        dt = datetime.utcfromtimestamp(time.time())
+
+        # unix timestamp of next 00:00:00 (leap-seconds safe)
+        day_now = dt.day
+        while dt.day == day_now:
+            dt += timedelta(hours=12)
+
+        dt = dt.replace(hour=0, minute=0, second=0)
+        self.next_day = calendar.timegm(dt.utctimetuple())
 
     def _log_enabled(self, src, msg, c=0):
         """handles logging from all components"""
@@ -110,14 +188,7 @@ class SvcHub(object):
             if now >= self.next_day:
                 dt = datetime.utcfromtimestamp(now)
                 print("\033[36m{}\033[0m\n".format(dt.strftime("%Y-%m-%d")), end="")
-
-                # unix timestamp of next 00:00:00 (leap-seconds safe)
-                day_now = dt.day
-                while dt.day == day_now:
-                    dt += timedelta(hours=12)
-
-                dt = dt.replace(hour=0, minute=0, second=0)
-                self.next_day = calendar.timegm(dt.utctimetuple())
+                self._set_next_day()
 
             fmt = "\033[36m{} \033[33m{:21} \033[0m{}\n"
             if not VT100:
@@ -144,6 +215,9 @@ class SvcHub(object):
                 except:
                     print(msg.encode("ascii", "replace").decode(), end="")
 
+            if self.logf:
+                self.logf.write(msg)
+
     def check_mp_support(self):
         vmin = sys.version_info[1]
         if WINDOWS:
@@ -189,5 +263,5 @@ class SvcHub(object):
         if not err:
             return True
         else:
-            self.log("root", err)
+            self.log("svchub", err)
             return False

copyparty/tcpsrv.py

@@ -66,7 +66,10 @@ class TcpSrv(object):
         for srv in self.srv:
             srv.listen(self.args.nc)
             ip, port = srv.getsockname()
-            self.log("tcpsrv", "listening @ {0}:{1}".format(ip, port))
+            msg = "listening @ {0}:{1}".format(ip, port)
+            self.log("tcpsrv", msg)
+            if self.args.q:
+                print(msg)
 
         while not self.stopping:
             if self.args.log_conn:

copyparty/th_srv.py

@@ -9,15 +9,11 @@ import hashlib
 import threading
 import subprocess as sp
 
-from .__init__ import PY2
+from .__init__ import PY2, unicode
 from .util import fsenc, runcmd, Queue, Cooldown, BytesIO, min_ex
 from .mtag import HAVE_FFMPEG, HAVE_FFPROBE, ffprobe
 
 
-if not PY2:
-    unicode = str
-
-
 HAVE_PIL = False
 HAVE_HEIF = False
 HAVE_AVIF = False

copyparty/up2k.py

@@ -342,7 +342,15 @@ class Up2k(object):
             for k, v in flags.items()
         ]
         if a:
-            self.log(" ".join(sorted(a)) + "\033[0m")
+            vpath = "?"
+            for k, v in self.asrv.vfs.all_vols.items():
+                if v.realpath == ptop:
+                    vpath = k
+
+            if vpath:
+                vpath += "/"
+
+            self.log("/{} {}".format(vpath, " ".join(sorted(a))), "35")
 
         reg = {}
         path = os.path.join(histpath, "up2k.snap")
@@ -401,7 +409,7 @@ class Up2k(object):
             if WINDOWS:
                 excl = [x.replace("/", "\\") for x in excl]
 
-            n_add = self._build_dir(dbw, top, set(excl), top, nohash)
+            n_add = self._build_dir(dbw, top, set(excl), top, nohash, [])
             n_rm = self._drop_lost(dbw[0], top)
             if dbw[1]:
                 self.log("commit {} new files".format(dbw[1]))
@@ -409,11 +417,25 @@ class Up2k(object):
 
             return True, n_add or n_rm or do_vac
 
-    def _build_dir(self, dbw, top, excl, cdir, nohash):
+    def _build_dir(self, dbw, top, excl, cdir, nohash, seen):
+        rcdir = cdir
+        if not ANYWIN:
+            try:
+                # a bit expensive but worth
+                rcdir = os.path.realpath(cdir)
+            except:
+                pass
+
+        if rcdir in seen:
+            m = "bailing from symlink loop,\n  prev: {}\n  curr: {}\n  from: {}"
+            self.log(m.format(seen[-1], rcdir, cdir), 3)
+            return 0
+
+        seen = seen + [cdir]
         self.pp.msg = "a{} {}".format(self.pp.n, cdir)
         histpath = self.asrv.vfs.histtab[top]
         ret = 0
-        g = statdir(self.log, not self.args.no_scandir, False, cdir)
+        g = statdir(self.log_func, not self.args.no_scandir, False, cdir)
         for iname, inf in sorted(g):
             abspath = os.path.join(cdir, iname)
             lmod = int(inf.st_mtime)
@@ -422,7 +444,7 @@ class Up2k(object):
                 if abspath in excl or abspath == histpath:
                     continue
                 # self.log(" dir: {}".format(abspath))
-                ret += self._build_dir(dbw, top, excl, abspath, nohash)
+                ret += self._build_dir(dbw, top, excl, abspath, nohash, seen)
             else:
                 # self.log("file: {}".format(abspath))
                 rp = abspath[len(top) + 1 :]
@@ -1047,8 +1069,9 @@ class Up2k(object):
                         pdir = os.path.join(cj["ptop"], cj["prel"])
                         job["name"] = self._untaken(pdir, cj["name"], now, cj["addr"])
                         dst = os.path.join(job["ptop"], job["prel"], job["name"])
-                        os.unlink(fsenc(dst))  # TODO ed pls
-                        self._symlink(src, dst)
+                        if not self.args.nw:
+                            os.unlink(fsenc(dst))  # TODO ed pls
+                            self._symlink(src, dst)
 
             if not job:
                 job = {

copyparty/util.py

@@ -979,8 +979,7 @@ def statdir(logger, scandir, lstat, top):
                     try:
                         yield [fsdec(fh.name), fh.stat(follow_symlinks=not lstat)]
                     except Exception as ex:
-                        msg = "scan-stat: \033[36m{} @ {}"
-                        logger(msg.format(repr(ex), fsdec(fh.path)))
+                        logger(src, "[s] {} @ {}".format(repr(ex), fsdec(fh.path)), 6)
         else:
             src = "listdir"
             fun = os.lstat if lstat else os.stat
@@ -989,11 +988,10 @@ def statdir(logger, scandir, lstat, top):
                 try:
                     yield [fsdec(name), fun(abspath)]
                 except Exception as ex:
-                    msg = "list-stat: \033[36m{} @ {}"
-                    logger(msg.format(repr(ex), fsdec(abspath)))
+                    logger(src, "[s] {} @ {}".format(repr(ex), fsdec(abspath)), 6)
 
     except Exception as ex:
-        logger("{}: \033[31m{} @ {}".format(src, repr(ex), top))
+        logger(src, "{} @ {}".format(repr(ex), top), 1)
 
 
 def unescape_cookie(orig):
@@ -1035,7 +1033,7 @@ def guess_mime(url, fallback="application/octet-stream"):
     if ";" not in ret:
         if ret.startswith("text/") or ret.endswith("/javascript"):
             ret += "; charset=UTF-8"
-    
+
     return ret
 
 

copyparty/web/browser.css

@@ -29,10 +29,10 @@ body {
 	position: fixed;
 	max-width: 34em;
 	background: #222;
-	border: 0 solid #555;
+	border: 0 solid #777;
 	overflow: hidden;
 	margin-top: 1em;
-	padding: 0 1em;
+	padding: 0 1.3em;
 	height: 0;
 	opacity: .1;
 	transition: opacity 0.14s, height 0.14s, padding 0.14s;
@@ -40,19 +40,31 @@ body {
 	border-radius: .4em;
 	z-index: 9001;
 }
+#tt.b {
+	padding: 0 2em;
+	border-radius: .5em;
+	box-shadow: 0 .2em 1em #000;
+}
 #tt.show {
-	padding: 1em;
+	padding: 1em 1.3em;
+	border-width: .4em 0;
 	height: auto;
-	border-width: .2em 0;
 	opacity: 1;
 }
+#tt.show.b {
+	padding: 1.5em 2em;
+	border-width: .5em 0;
+}
 #tt code {
 	background: #3c3c3c;
-	padding: .2em .3em;
+	padding: .1em .3em;
 	border-top: 1px solid #777;
 	border-radius: .3em;
 	font-family: monospace, monospace;
-	line-height: 2em;
+	line-height: 1.7em;
+}
+#tt em {
+	color: #f6a;
 }
 #path,
 #path * {
@@ -812,11 +824,13 @@ input.eq_gain {
 	border-bottom: 1px solid #555;
 }
 #thumbs,
-#au_osd_cv {
+#au_osd_cv,
+#u2tdate {
 	opacity: .3;
 }
 #griden.on+#thumbs,
-#au_os_ctl.on+#au_osd_cv {
+#au_os_ctl.on+#au_osd_cv,
+#u2turbo.on+#u2tdate {
 	opacity: 1;
 }
 #ghead {
@@ -921,13 +935,16 @@ html.light {
 }
 html.light #tt {
 	background: #fff;
-	border-color: #888;
+	border-color: #888 #000 #777 #000;
 	box-shadow: 0 .3em 1em rgba(0,0,0,0.4);
 }
 html.light #tt code {
 	background: #060;
 	color: #fff;
 }
+html.light #tt em {
+	color: #d38;
+}
 html.light #ops,
 html.light .opbox,
 html.light #srch_form {

copyparty/web/browser.js

@@ -133,6 +133,13 @@ ebi('op_cfg').innerHTML = (
 	(have_zip ? (
 		'<div><h3>folder download</h3><div id="arc_fmt"></div></div>\n'
 	) : '') +
+	'<div>\n' +
+	'	<h3>up2k switches</h3>\n' +
+	'	<div>\n' +
+	'		<a id="u2turbo" class="tgl btn ttb" href="#" tt="the yolo button, you probably DO NOT want to enable this:$N$Nuse this if you were uploading a huge amount of files and had to restart for some reason, and want to continue the upload ASAP$N$Nthis replaces the hash-check with a simple <em>&quot;does this have the same filesize on the server?&quot;</em> so if the file contents are different it will NOT be uploaded$N$Nyou should turn this off when the upload is done, and then &quot;upload&quot; the same files again to let the client verify them">turbo</a>\n' +
+	'		<a id="u2tdate" class="tgl btn ttb" href="#" tt="has no effect unless the turbo button is enabled$N$Nreduces the yolo factor by a tiny amount; checks whether the file timestamps on the server matches yours$N$Nshould <em>theoretically</em> catch most unfinished/corrupted uploads, but is not a substitute for doing a verification pass with turbo disabled afterwards">date-chk</a>\n' +
+	'	</div>\n' +
+	'</div>\n' +
 	'<div><h3>key notation</h3><div id="key_notation"></div></div>\n' +
 	'<div class="fill"><h3>hidden columns</h3><div id="hcols"></div></div>'
 );

copyparty/web/up2k.js

@@ -225,7 +225,7 @@ function U2pvis(act, btns) {
     this.hashed = function (fobj) {
         var fo = this.tab[fobj.n],
             nb = fo.bt * (++fo.nh / fo.cb.length),
-            p = this.perc(nb, 0, fobj.size, fobj.t1);
+            p = this.perc(nb, 0, fobj.size, fobj.t_hashing);
 
         fo.hp = '{0}%, {1}, {2} MB/s'.format(
             p[0].toFixed(2), p[1], p[2].toFixed(2)
@@ -248,7 +248,7 @@ function U2pvis(act, btns) {
         fo.cb[nchunk] = cbd;
         fo.bd += delta;
 
-        var p = this.perc(fo.bd, fo.bd0, fo.bt, fobj.t3);
+        var p = this.perc(fo.bd, fo.bd0, fo.bt, fobj.t_uploading);
         fo.hp = '{0}%, {1}, {2} MB/s'.format(
             p[0].toFixed(2), p[1], p[2].toFixed(2)
         );
@@ -308,6 +308,12 @@ function U2pvis(act, btns) {
             throw 42;
         }
 
+        //console.log("oldcat %s %d, newcat %s %d, head=%d, tail=%d, file=%d, act.old=%s, act.new=%s, bz_act=%s",
+        //    oldcat, this.ctr[oldcat],
+        //    newcat, this.ctr[newcat],
+        //    this.head, this.tail, nfile,
+        //    this.is_act(oldcat), this.is_act(newcat), bz_act);
+
         fo.in = newcat;
         this.ctr[oldcat]--;
         this.ctr[newcat]++;
@@ -319,7 +325,7 @@ function U2pvis(act, btns) {
                 this.addrow(nfile);
         }
         else if (this.is_act(oldcat)) {
-            while (this.head < Math.min(this.tab.length, this.tail) && (this.head == nfile || !this.is_act(this.tab[this.head].in)))
+            while (this.head < Math.min(this.tab.length, this.tail) && this.precard[this.tab[this.head].in])
                 this.head++;
 
             if (!bz_act) {
@@ -327,9 +333,10 @@ function U2pvis(act, btns) {
                 tr.parentNode.removeChild(tr);
             }
         }
-        if (bz_act) {
+        else return;
+
+        if (bz_act)
             this.bzw();
-        }
     };
 
     this.bzw = function () {
@@ -343,7 +350,8 @@ function U2pvis(act, btns) {
 
         while (this.head - first > this.wsz) {
             var obj = ebi('f' + (first++));
-            obj.parentNode.removeChild(obj);
+            if (obj)
+                obj.parentNode.removeChild(obj);
         }
         while (last - this.tail < this.wsz && last < this.tab.length - 2) {
             var obj = ebi('f' + (++last));
@@ -376,6 +384,8 @@ function U2pvis(act, btns) {
 
     this.changecard = function (card) {
         this.act = card;
+        this.precard = has(["ok", "ng", "done"], this.act) ? {} : this.act == "bz" ? { "ok": 1, "ng": 1 } : { "ok": 1, "ng": 1, "bz": 1 };
+        this.postcard = has(["ok", "ng", "done"], this.act) ? { "bz": 1, "q": 1 } : this.act == "bz" ? { "q": 1 } : {};
         this.head = -1;
         this.tail = -1;
         var html = [];
@@ -390,16 +400,13 @@ function U2pvis(act, btns) {
             }
         }
         if (this.head == -1) {
-            var precard = has(["ok", "ng", "done"], this.act) ? {} : this.act == "bz" ? { "ok": 1, "ng": 1 } : { "ok": 1, "ng": 1, "bz": 1 },
-                postcard = has(["ok", "ng", "done"], this.act) ? { "bz": 1, "q": 1 } : this.act == "bz" ? { "q": 1 } : {};
-
             for (var a = 0; a < this.tab.length; a++) {
                 var rt = this.tab[a].in;
-                if (precard[rt]) {
+                if (this.precard[rt]) {
                     this.head = a + 1;
                     this.tail = a;
                 }
-                else if (postcard[rt]) {
+                else if (this.postcard[rt]) {
                     this.head = a;
                     this.tail = a - 1;
                     break;
@@ -452,6 +459,8 @@ function U2pvis(act, btns) {
             that.changecard(newtab);
         };
     }
+
+    this.changecard(this.act);
 }
 
 
@@ -548,17 +557,21 @@ function up2k_init(subtle) {
         ask_up = bcfg_get('ask_up', true),
         flag_en = bcfg_get('flag_en', false),
         fsearch = bcfg_get('fsearch', false),
+        turbo = bcfg_get('u2turbo', false),
+        datechk = bcfg_get('u2tdate', true),
         fdom_ctr = 0,
         min_filebuf = 0;
 
     var st = {
         "files": [],
         "todo": {
+            "head": [],
             "hash": [],
             "handshake": [],
             "upload": []
         },
         "busy": {
+            "head": [],
             "hash": [],
             "handshake": [],
             "upload": []
@@ -569,6 +582,15 @@ function up2k_init(subtle) {
         }
     };
 
+    function push_t(arr, t) {
+        var sort = arr.length && arr[arr.length - 1].n > t.n;
+        arr.push(t);
+        if (sort)
+            arr.sort(function (a, b) {
+                return a.n < b.n ? -1 : 1;
+            });
+    }
+
     var pvis = new U2pvis("bz", '#u2cards');
 
     var bobslice = null;
@@ -612,7 +634,7 @@ function up2k_init(subtle) {
         }
         else files = e.target.files;
 
-        if (!files || files.length == 0)
+        if (!files || !files.length)
             return alert('no files selected??');
 
         more_one_file();
@@ -715,8 +737,7 @@ function up2k_init(subtle) {
 
                     pf.push(name);
                     dn.file(function (fobj) {
-                        var idx = pf.indexOf(name);
-                        pf.splice(idx, 1);
+                        apop(pf, name);
                         try {
                             if (fobj.size > 0) {
                                 good.push([fobj, name]);
@@ -739,7 +760,7 @@ function up2k_init(subtle) {
     }
 
     function gotallfiles(good_files, bad_files) {
-        if (bad_files.length > 0) {
+        if (bad_files.length) {
             var ntot = bad_files.length + good_files.length,
                 msg = 'These {0} files (of {1} total) were skipped because they are empty:\n'.format(bad_files.length, ntot);
 
@@ -797,7 +818,10 @@ function up2k_init(subtle) {
             ], fobj.size, draw_each);
 
             st.files.push(entry);
-            st.todo.hash.push(entry);
+            if (turbo)
+                push_t(st.todo.head, entry);
+            else
+                push_t(st.todo.hash, entry);
         }
         if (!draw_each) {
             pvis.drawcard("q");
@@ -837,15 +861,25 @@ function up2k_init(subtle) {
     //
 
     function handshakes_permitted() {
-        var lim = multitask ? 1 : 0;
+        if (!st.todo.handshake.length)
+            return true;
 
-        if (lim <
-            st.todo.upload.length +
-            st.busy.upload.length)
+        var cd = st.todo.handshake[0].cooldown;
+        if (cd && cd - Date.now() > 0)
             return false;
 
-        var cd = st.todo.handshake.length ? st.todo.handshake[0].cooldown : 0;
-        if (cd && cd - Date.now() > 0)
+        // keepalive or verify
+        if (st.todo.handshake[0].keepalive ||
+            st.todo.handshake[0].t_uploaded)
+            return true;
+
+        if (parallel_uploads <
+            st.busy.handshake.length)
+            return false;
+
+        if ((multitask ? parallel_uploads : 0) <
+            st.todo.upload.length +
+            st.busy.upload.length)
             return false;
 
         return true;
@@ -880,13 +914,16 @@ function up2k_init(subtle) {
             clearTimeout(tto);
             running = true;
             while (window['vis_exh']) {
-                var is_busy = 0 !=
-                    st.todo.hash.length +
-                    st.todo.handshake.length +
-                    st.todo.upload.length +
-                    st.busy.hash.length +
-                    st.busy.handshake.length +
-                    st.busy.upload.length;
+                var now = Date.now(),
+                    is_busy = 0 !=
+                        st.todo.head.length +
+                        st.todo.hash.length +
+                        st.todo.handshake.length +
+                        st.todo.upload.length +
+                        st.busy.head.length +
+                        st.busy.hash.length +
+                        st.busy.handshake.length +
+                        st.busy.upload.length;
 
                 if (was_busy != is_busy) {
                     was_busy = is_busy;
@@ -897,7 +934,6 @@ function up2k_init(subtle) {
 
                 if (flag) {
                     if (is_busy) {
-                        var now = Date.now();
                         flag.take(now);
                         if (!flag.ours)
                             return defer();
@@ -909,43 +945,52 @@ function up2k_init(subtle) {
 
                 var mou_ikkai = false;
 
-                if (st.busy.handshake.length > 0 &&
-                    st.busy.handshake[0].busied < Date.now() - 30 * 1000
+                if (st.busy.handshake.length &&
+                    st.busy.handshake[0].t_busied < now - 30 * 1000
                 ) {
                     console.log("retrying stuck handshake");
                     var t = st.busy.handshake.shift();
                     st.todo.handshake.unshift(t);
                 }
 
-                if (st.todo.handshake.length > 0 &&
-                    st.busy.handshake.length == 0 && (
-                        st.todo.handshake[0].t4 || (
-                            handshakes_permitted() &&
-                            st.busy.upload.length < parallel_uploads
-                        )
-                    )
-                ) {
-                    exec_handshake();
+                var nprev = -1;
+                for (var a = 0; a < st.todo.upload.length; a++) {
+                    var nf = st.todo.upload[a].nfile;
+                    if (nprev == nf)
+                        continue;
+
+                    nprev = nf;
+                    var t = st.files[nf];
+                    if (now - t.t_busied > 1000 * 30 &&
+                        now - t.t_handshake > 1000 * (21600 - 1800)
+                    ) {
+                        apop(st.todo.handshake, t);
+                        st.todo.handshake.unshift(t);
+                        t.keepalive = true;
+                    }
+                }
+
+                if (st.todo.head.length &&
+                    st.busy.head.length < parallel_uploads) {
+                    exec_head();
                     mou_ikkai = true;
                 }
 
                 if (handshakes_permitted() &&
-                    st.todo.handshake.length > 0 &&
-                    st.busy.handshake.length == 0 &&
-                    st.busy.upload.length < parallel_uploads) {
+                    st.todo.handshake.length) {
                     exec_handshake();
                     mou_ikkai = true;
                 }
 
-                if (st.todo.upload.length > 0 &&
+                if (st.todo.upload.length &&
                     st.busy.upload.length < parallel_uploads) {
                     exec_upload();
                     mou_ikkai = true;
                 }
 
                 if (hashing_permitted() &&
-                    st.todo.hash.length > 0 &&
-                    st.busy.hash.length == 0) {
+                    st.todo.hash.length &&
+                    !st.busy.hash.length) {
                     exec_hash();
                     mou_ikkai = true;
                 }
@@ -1080,7 +1125,7 @@ function up2k_init(subtle) {
 
                 if (handled) {
                     pvis.move(t.n, 'ng');
-                    st.busy.hash.splice(st.busy.hash.indexOf(t), 1);
+                    apop(st.busy.hash, t);
                     st.bytes.uploaded += t.size;
                     return tasker();
                 }
@@ -1113,15 +1158,15 @@ function up2k_init(subtle) {
                     t.hash.push(hashtab[a]);
                 }
 
-                t.t2 = Date.now();
+                t.t_hashed = Date.now();
                 if (t.n == 0 && window.location.hash == '#dbg') {
-                    var spd = (t.size / ((t.t2 - t.t1) / 1000.)) / (1024 * 1024.);
-                    alert('{0} ms, {1} MB/s\n'.format(t.t2 - t.t1, spd.toFixed(3)) + t.hash.join('\n'));
+                    var spd = (t.size / ((t.t_hashed - t.t_hashing) / 1000.)) / (1024 * 1024.);
+                    alert('{0} ms, {1} MB/s\n'.format(t.t_hashed - t.t_hashing, spd.toFixed(3)) + t.hash.join('\n'));
                 }
 
                 pvis.seth(t.n, 2, 'hashing done');
                 pvis.seth(t.n, 1, '📦 wait');
-                st.busy.hash.splice(st.busy.hash.indexOf(t), 1);
+                apop(st.busy.hash, t);
                 st.todo.handshake.push(t);
                 tasker();
             };
@@ -1144,10 +1189,57 @@ function up2k_init(subtle) {
             }, 1);
         };
 
-        t.t1 = Date.now();
+        t.t_hashing = Date.now();
         segm_next();
     }
 
+    /////
+    ////
+    ///   head
+    //
+
+    function exec_head() {
+        var t = st.todo.head.shift();
+        st.busy.head.push(t);
+
+        var xhr = new XMLHttpRequest();
+        xhr.onerror = function () {
+            console.log('head onerror, retrying', t);
+            apop(st.busy.head, t);
+            st.todo.head.unshift(t);
+            tasker();
+        };
+        function orz(e) {
+            var ok = false;
+            if (xhr.status == 200) {
+                var srv_sz = xhr.getResponseHeader('Content-Length'),
+                    srv_ts = xhr.getResponseHeader('Last-Modified');
+
+                ok = t.size == srv_sz;
+                if (ok && datechk) {
+                    srv_ts = new Date(srv_ts) / 1000;
+                    ok = Math.abs(srv_ts - t.lmod) < 2;
+                }
+            }
+            apop(st.busy.head, t);
+            if (!ok)
+                return push_t(st.todo.hash, t);
+
+            t.done = true;
+            st.bytes.hashed += t.size;
+            st.bytes.uploaded += t.size;
+            pvis.seth(t.n, 1, 'YOLO');
+            pvis.seth(t.n, 2, "turbo'd");
+            pvis.move(t.n, 'ok');
+        };
+        xhr.onload = function (e) {
+            try { orz(e); } catch (ex) { vis_exh(ex + '', '', '', '', ex); }
+        };
+
+        xhr.open('HEAD', t.purl + t.name, true);
+        xhr.send();
+    }
+
     /////
     ////
     ///   handshake
@@ -1155,30 +1247,41 @@ function up2k_init(subtle) {
 
     function exec_handshake() {
         var t = st.todo.handshake.shift(),
+            keepalive = t.keepalive,
             me = Date.now();
 
         st.busy.handshake.push(t);
-        t.busied = me;
+        t.keepalive = undefined;
+        t.t_busied = me;
+
+        if (keepalive)
+            console.log("sending keepalive handshake", t);
 
         var xhr = new XMLHttpRequest();
         xhr.onerror = function () {
-            if (t.busied != me) {
+            if (t.t_busied != me) {
                 console.log('zombie handshake onerror,', t);
                 return;
             }
             console.log('handshake onerror, retrying', t);
-            st.busy.handshake.splice(st.busy.handshake.indexOf(t), 1);
+            apop(st.busy.handshake, t);
             st.todo.handshake.unshift(t);
+            t.keepalive = keepalive;
             tasker();
         };
         function orz(e) {
-            if (t.busied != me) {
+            if (t.t_busied != me) {
                 console.log('zombie handshake onload,', t);
                 return;
             }
             if (xhr.status == 200) {
-                var response = JSON.parse(xhr.responseText);
+                t.t_handshake = Date.now();
+                if (keepalive) {
+                    apop(st.busy.handshake, t);
+                    return;
+                }
 
+                var response = JSON.parse(xhr.responseText);
                 if (!response.name) {
                     var msg = '',
                         smsg = '';
@@ -1202,7 +1305,7 @@ function up2k_init(subtle) {
                     pvis.seth(t.n, 2, msg);
                     pvis.seth(t.n, 1, smsg);
                     pvis.move(t.n, smsg == '404' ? 'ng' : 'ok');
-                    st.busy.handshake.splice(st.busy.handshake.indexOf(t), 1);
+                    apop(st.busy.handshake, t);
                     st.bytes.uploaded += t.size;
                     t.done = true;
                     tasker();
@@ -1244,31 +1347,41 @@ function up2k_init(subtle) {
                 var done = true,
                     msg = '&#x1f3b7;&#x1f41b;';
 
-                if (t.postlist.length > 0) {
+                if (t.postlist.length) {
+                    var arr = st.todo.upload,
+                        sort = arr.length && arr[arr.length - 1].nfile > t.n;
+
                     for (var a = 0; a < t.postlist.length; a++)
-                        st.todo.upload.push({
+                        arr.push({
                             'nfile': t.n,
                             'npart': t.postlist[a]
                         });
 
                     msg = 'uploading';
                     done = false;
+
+                    if (sort)
+                        arr.sort(function (a, b) {
+                            return a.nfile < b.nfile ? -1 :
+                            /*  */ a.nfile > b.nfile ? 1 :
+                                    a.npart < b.npart ? -1 : 1;
+                        });
                 }
                 pvis.seth(t.n, 1, msg);
-                st.busy.handshake.splice(st.busy.handshake.indexOf(t), 1);
+                apop(st.busy.handshake, t);
 
                 if (done) {
                     t.done = true;
                     st.bytes.uploaded += t.size - t.bytes_uploaded;
-                    var spd1 = (t.size / ((t.t2 - t.t1) / 1000.)) / (1024 * 1024.),
-                        spd2 = (t.size / ((t.t4 - t.t3) / 1000.)) / (1024 * 1024.);
+                    var spd1 = (t.size / ((t.t_hashed - t.t_hashing) / 1000.)) / (1024 * 1024.),
+                        spd2 = (t.size / ((t.t_uploaded - t.t_uploading) / 1000.)) / (1024 * 1024.);
 
                     pvis.seth(t.n, 2, 'hash {0}, up {1} MB/s'.format(
                         spd1.toFixed(2), spd2.toFixed(2)));
 
                     pvis.move(t.n, 'ok');
                 }
-                else t.t4 = undefined;
+                else t.t_uploaded = undefined;
 
                 tasker();
             }
@@ -1287,7 +1400,7 @@ function up2k_init(subtle) {
                     var penalty = rsp.replace(/.*rate-limit /, "").split(' ')[0];
                     console.log("rate-limit: " + penalty);
                     t.cooldown = Date.now() + parseFloat(penalty) * 1000;
-                    st.busy.handshake.splice(st.busy.handshake.indexOf(t), 1);
+                    apop(st.busy.handshake, t);
                     st.todo.handshake.unshift(t);
                     return;
                 }
@@ -1306,7 +1419,7 @@ function up2k_init(subtle) {
                     pvis.seth(t.n, 2, err);
                     pvis.move(t.n, 'ng');
 
-                    st.busy.handshake.splice(st.busy.handshake.indexOf(t), 1);
+                    apop(st.busy.handshake, t);
                     tasker();
                     return;
                 }
@@ -1347,8 +1460,8 @@ function up2k_init(subtle) {
         var npart = upt.npart,
             t = st.files[upt.nfile];
 
-        if (!t.t3)
-            t.t3 = Date.now();
+        if (!t.t_uploading)
+            t.t_uploading = Date.now();
 
         pvis.seth(t.n, 1, "🚀 send");
 
@@ -1367,17 +1480,17 @@ function up2k_init(subtle) {
                 t.bytes_uploaded += cdr - car;
             }
             else if (txt.indexOf('already got that') !== -1) {
-                console.log("ignoring dupe-segment error");
+                console.log("ignoring dupe-segment error", t);
             }
             else {
                 alert("server broke; cu-err {0} on file [{1}]:\n".format(
                     xhr.status, t.name) + (txt || "no further information"));
                 return;
             }
-            st.busy.upload.splice(st.busy.upload.indexOf(upt), 1);
-            t.postlist.splice(t.postlist.indexOf(npart), 1);
-            if (t.postlist.length == 0) {
-                t.t4 = Date.now();
+            apop(st.busy.upload, upt);
+            apop(t.postlist, npart);
+            if (!t.postlist.length) {
+                t.t_uploaded = Date.now();
                 pvis.seth(t.n, 1, 'verifying');
                 st.todo.handshake.unshift(t);
             }
@@ -1504,6 +1617,30 @@ function up2k_init(subtle) {
         set_fsearch(!fsearch);
     }
 
+    function tgl_turbo() {
+        turbo = !turbo;
+        bcfg_set('u2turbo', turbo);
+        draw_turbo();
+    }
+
+    function tgl_datechk() {
+        datechk = !datechk;
+        bcfg_set('u2tdate', datechk);
+    }
+
+    function draw_turbo() {
+        var msg = '<p class="warn">WARNING: turbo enabled, <span>&nbsp;client may not detect and resume incomplete uploads; see turbo-button tooltip</span></p>',
+            html = ebi('u2foot').innerHTML;
+
+        if (turbo && html.indexOf(msg) === -1)
+            html += msg;
+        else if (!turbo)
+            html = html.replace(msg, '');
+
+        ebi('u2foot').innerHTML = html;
+    }
+    draw_turbo();
+
     function set_fsearch(new_state) {
         var fixed = false;
 
@@ -1585,6 +1722,8 @@ function up2k_init(subtle) {
     ebi('multitask').addEventListener('click', tgl_multitask, false);
     ebi('ask_up').addEventListener('click', tgl_ask_up, false);
     ebi('flag_en').addEventListener('click', tgl_flag_en, false);
+    ebi('u2turbo').addEventListener('click', tgl_turbo, false);
+    ebi('u2tdate').addEventListener('click', tgl_datechk, false);
     var o = ebi('fsearch');
     if (o)
         o.addEventListener('click', tgl_fsearch, false);

copyparty/web/upload.css

@@ -215,9 +215,31 @@
 	color: #fff;
 	font-style: italic;
 }
+#u2foot .warn {
+	font-size: 1.3em;
+	padding: .5em .8em;
+	margin: 1em -.6em;
+	color: #f74;
+	background: #322;
+	border: 1px solid #633;
+	border-width: .1em 0;
+	text-align: center;
+}
+#u2foot .warn span {
+	color: #f86;
+}
+html.light #u2foot .warn {
+	color: #b00;
+	background: #fca;
+	border-color: #f70;
+}
+html.light #u2foot .warn span {
+	color: #930;
+}
 #u2foot span {
 	color: #999;
 	font-size: .9em;
+	font-weight: normal;
 }
 #u2footfoot {
 	margin-bottom: -1em;

copyparty/web/util.js

@@ -389,6 +389,13 @@ function has(haystack, needle) {
 }
 
 
+function apop(arr, v) {
+    var ofs = arr.indexOf(v);
+    if (ofs !== -1)
+        arr.splice(ofs, 1);
+}
+
+
 function jcp(obj) {
     return JSON.parse(JSON.stringify(obj));
 }
@@ -507,8 +514,10 @@ var tt = (function () {
 
         var pos = this.getBoundingClientRect(),
             left = pos.left < window.innerWidth / 2,
-            top = pos.top < window.innerHeight / 2;
+            top = pos.top < window.innerHeight / 2,
+            big = this.className.indexOf(' ttb') !== -1;
 
+        clmod(r.tt, 'b', big);
         r.tt.style.top = top ? pos.bottom + 'px' : 'auto';
         r.tt.style.bottom = top ? 'auto' : (window.innerHeight - pos.top) + 'px';
         r.tt.style.left = left ? pos.left + 'px' : 'auto';

docs/hls.html

@@ -0,0 +1,51 @@
+<!DOCTYPE html><html lang="en"><head>
+	<meta charset="utf-8">
+	<title>hls-test</title>
+	<meta http-equiv="X-UA-Compatible" content="IE=edge">
+</head><body>
+
+<video id="vid" controls></video>
+<script src="hls.light.js"></script>
+<script>
+
+var video = document.getElementById('vid');
+var hls = new Hls({
+	debug: true,
+	autoStartLoad: false
+});
+hls.loadSource('live/v.m3u8');
+hls.attachMedia(video);
+hls.on(Hls.Events.MANIFEST_PARSED, function() {
+	hls.startLoad(0);
+});
+hls.on(Hls.Events.MEDIA_ATTACHED, function() {
+	video.muted = true;
+	video.play();
+});
+
+/*
+general good news:
+- doesn't need fixed-length segments; ok to let x264 pick optimal keyframes and slice on those
+- hls.js polls the m3u8 for new segments, scales the duration accordingly, seeking works great
+- the sfx will grow by 66 KiB since that's how small hls.js can get, wait thats not good
+
+# vod, creates m3u8 at the end, fixed keyframes, v bad
+ffmpeg -hide_banner -threads 0 -flags -global_header -i ..\CowboyBebopMovie-OP1.webm -vf scale=1280:-4,format=yuv420p -ac 2 -c:a libopus -b:a 128k -c:v libx264 -preset slow -crf 24 -maxrate:v 5M -bufsize:v 10M -g 120 -keyint_min 120 -sc_threshold 0 -hls_time 4 -hls_playlist_type vod -hls_segment_filename v%05d.ts v.m3u8
+
+# live, updates m3u8 as it goes, dynamic keyframes, streamable with hls.js
+ffmpeg -hide_banner -threads 0 -flags -global_header -i ..\..\CowboyBebopMovie-OP1.webm -vf scale=1280:-4,format=yuv420p -ac 2 -c:a libopus -b:a 128k -c:v libx264 -preset slow -crf 24 -maxrate:v 5M -bufsize:v 10M -f segment -segment_list v.m3u8 -segment_format mpegts -segment_list_flags live v%05d.ts
+
+# fmp4 (fragmented mp4), doesn't work with hls.js, gets duratoin 149:07:51 (536871s), probably the tkhd/mdhd 0xffffffff (timebase 8000? ok)
+ffmpeg -re -hide_banner -threads 0 -flags +cgop -i ..\..\CowboyBebopMovie-OP1.webm -vf scale=1280:-4,format=yuv420p -ac 2 -c:a libopus -b:a 128k -c:v libx264 -preset slow -crf 24 -maxrate:v 5M -bufsize:v 10M -f segment -segment_list v.m3u8 -segment_format fmp4 -segment_list_flags live v%05d.mp4
+
+# try 2, works, uses tempfiles for m3u8 updates, good, 6% smaller
+ffmpeg -re -hide_banner -threads 0 -flags +cgop -i ..\..\CowboyBebopMovie-OP1.webm -vf scale=1280:-4,format=yuv420p -ac 2 -c:a libopus -b:a 128k -c:v libx264 -preset slow -crf 24 -maxrate:v 5M -bufsize:v 10M -f hls -hls_segment_type fmp4 -hls_list_size 0 -hls_segment_filename v%05d.mp4 v.m3u8
+
+more notes
+- adding -hls_flags single_file makes duration wack during playback (for both fmp4 and ts), ok once finalized and refreshed, gives no size reduction anyways
+- bebop op has good keyframe spacing for testing hls.js, in particular it hops one seg back and immediately resumes if it hits eof with the explicit hls.startLoad(0); otherwise it jumps into the middle of a seg and becomes art
+- can probably -c:v copy most of the time, is there a way to check for cgop? todo
+
+*/
+</script>
+</body></html>

tests/test_httpcli.py

@@ -30,6 +30,7 @@ class Cfg(Namespace):
             c=c,
             rproxy=0,
             ed=False,
+            nw=False,
             no_zip=False,
             no_scandir=False,
             no_sendfile=True,

tests/test_vfs.py

@@ -17,7 +17,7 @@ from copyparty import util
 
 class Cfg(Namespace):
     def __init__(self, a=[], v=[], c=None):
-        ex = {k: False for k in "e2d e2ds e2dsa e2t e2ts e2tsr".split()}
+        ex = {k: False for k in "nw e2d e2ds e2dsa e2t e2ts e2tsr".split()}
         ex2 = {
             "mtp": [],
             "mte": "a",