v1.3.7

README.md

@@ -481,6 +481,7 @@ see [up2k](#up2k) for details on how it works, or watch a [demo video](https://a
 the up2k UI is the epitome of polished inutitive experiences:
 * "parallel uploads" specifies how many chunks to upload at the same time
 * `[🏃]` analysis of other files should continue while one is uploading
+* `[🥔]` shows a simpler UI for faster uploads from slow devices
 * `[💭]` ask for confirmation before files are added to the queue
 * `[🔎]` switch between upload and [file-search](#file-search) mode
   * ignore `[🔎]` if you add files by dragging them into the browser
@@ -663,8 +664,11 @@ through arguments:
 * `-e2t` enables metadata indexing on upload
 * `-e2ts` also scans for tags in all files that don't have tags yet
 * `-e2tsr` also deletes all existing tags, doing a full reindex
+* `-e2v` verfies file integrity at startup, comparing hashes from the db
+* `-e2vu` patches the database with the new hashes from the filesystem
+* `-e2vp` panics and kills copyparty instead
 
-the same arguments can be set as volume flags, in addition to `d2d`, `d2ds`, `d2t`, `d2ts` for disabling:
+the same arguments can be set as volume flags, in addition to `d2d`, `d2ds`, `d2t`, `d2ts`, `d2v` for disabling:
 * `-v ~/music::r:c,e2dsa,e2tsr` does a full reindex of everything on startup
 * `-v ~/music::r:c,d2d` disables **all** indexing, even if any `-e2*` are on
 * `-v ~/music::r:c,d2t` disables all `-e2t*` (tags), does not affect `-e2d*`
@@ -964,6 +968,12 @@ quick outline of the up2k protocol, see [uploading](#uploading) for the web-clie
 
 up2k has saved a few uploads from becoming corrupted in-transfer already; caught an android phone on wifi redhanded in wireshark with a bitflip, however bup with https would *probably* have noticed as well (thanks to tls also functioning as an integrity check)
 
+regarding the frequent server log message during uploads;  
+`6.0M 106M/s 2.77G 102.9M/s n948 thank 4/0/3/1 10042/7198`
+* this chunk was `6 MiB`, uploaded at `106 MiB/s`
+* on this http connection, `2.77 GiB` transferred, `102.9 MiB/s` average, `948` chunks handled
+* client says `4` uploads OK, `0` failed, `3` busy, `1` queued, `10042 MiB` total size, `7198 MiB` left
+
 
 ## why chunk-hashes
 
@@ -1228,11 +1238,15 @@ if you want thumbnails, `apt -y install ffmpeg`
 
 ideas for context to include in bug reports
 
+in general, commandline arguments (and config file if any)
+
 if something broke during an upload (replacing FILENAME with a part of the filename that broke):
 ```
 journalctl -aS '48 hour ago' -u copyparty | grep -C10 FILENAME | tee bug.log
 ```
 
+if there's a wall of base64 in the log (thread stacks) then please include that, especially if you run into something freezing up or getting stuck, for example `OperationalError('database is locked')` -- alternatively you can visit `/?stack` to see the stacks live, so http://127.0.0.1:3923/?stack for example
+
 
 # building
 

bin/mtag/image-noexif.py

@@ -89,4 +89,7 @@ def main():
 
 
 if __name__ == "__main__":
-    main()
+    try:
+        main()
+    except:
+        pass

bin/mtag/rclone-upload.py

@@ -0,0 +1,76 @@
+#!/usr/bin/env python
+
+import json
+import os
+import subprocess as sp
+import sys
+import time
+
+try:
+    from copyparty.util import fsenc
+except:
+
+    def fsenc(p):
+        return p.encode("utf-8")
+
+
+_ = r"""
+first checks the tag "vidchk" which must be "ok" to continue,
+then uploads all files to some cloud storage (RCLONE_REMOTE)
+and DELETES THE ORIGINAL FILES if rclone returns 0 ("success")
+
+deps:
+  rclone
+
+usage:
+  -mtp x2=t43200,ay,p2,bin/mtag/rclone-upload.py
+
+explained:
+t43200: timeout 12h
+    ay: only process files which contain audio (including video with audio)
+    p2: set priority 2 (after vidchk's suggested priority of 1),
+          so the output of vidchk will be passed in here
+
+complete usage example as vflags along with vidchk:
+  -vsrv/vidchk:vidchk:r:rw,ed:c,e2dsa,e2ts,mtp=vidchk=t600,p,bin/mtag/vidchk.py:c,mtp=rupload=t43200,ay,p2,bin/mtag/rclone-upload.py:c,mte=+vidchk,rupload
+
+setup: see https://rclone.org/drive/
+
+if you wanna use this script standalone / separately from copyparty,
+either set CONDITIONAL_UPLOAD False or provide the following stdin:
+  {"vidchk":"ok"}
+"""
+
+
+RCLONE_REMOTE = "notmybox"
+CONDITIONAL_UPLOAD = True
+
+
+def main():
+    if CONDITIONAL_UPLOAD:
+        fp = sys.argv[1]
+        zb = sys.stdin.buffer.read()
+        zs = zb.decode("utf-8", "replace")
+        md = json.loads(zs)
+
+        chk = md.get("vidchk", None)
+        if chk != "ok":
+            print(f"vidchk={chk}", file=sys.stderr)
+            sys.exit(1)
+
+    dst = f"{RCLONE_REMOTE}:".encode("utf-8")
+    cmd = [b"rclone", b"copy", b"--", fsenc(fp), dst]
+
+    t0 = time.time()
+    try:
+        sp.check_call(cmd)
+    except:
+        print("rclone failed", file=sys.stderr)
+        sys.exit(1)
+
+    print(f"{time.time() - t0:.1f} sec")
+    os.unlink(fsenc(fp))
+
+
+if __name__ == "__main__":
+    main()

bin/mtag/vidchk.py

@@ -1,15 +1,33 @@
 #!/usr/bin/env python3
 
+import json
+import re
 import sys
 import subprocess as sp
 
-from copyparty.util import fsenc
-from copyparty.mtag import ffprobe
+try:
+    from copyparty.util import fsenc
+except:
+
+    def fsenc(p):
+        return p.encode("utf-8")
 
 
-"""
+_ = r"""
 inspects video files for errors and such
-usage: -mtp vidchk=t600,ay,bin/mtag/vidchk.py
+plus stores a bunch of metadata to filename.ff.json
+
+usage:
+  -mtp vidchk=t600,ay,p,bin/mtag/vidchk.py
+
+explained:
+t600: timeout 10min
+  ay: only process files which contain audio (including video with audio)
+   p: set priority 1 (lowest priority after initial ffprobe/mutagen for base tags),
+       makes copyparty feed base tags into this script as json
+
+if you wanna use this script standalone / separately from copyparty,
+provide the video resolution on stdin as json:  {"res":"1920x1080"}
 """
 
 
@@ -17,19 +35,63 @@ FAST = True  # parse entire file at container level
 # FAST = False  # fully decode audio and video streams
 
 
+# warnings to ignore
+harmless = re.compile("^Unsupported codec with id ")
+
+
+def wfilter(lines):
+    return [x for x in lines if not harmless.search(x)]
+
+
+def errchk(so, se, rc):
+    if rc:
+        err = (so + se).decode("utf-8", "replace").split("\n", 1)
+        err = wfilter(err) or err
+        return f"ERROR {rc}: {err[0]}"
+
+    if se:
+        err = se.decode("utf-8", "replace").split("\n", 1)
+        err = wfilter(err)
+        if err:
+            return f"Warning: {err[0]}"
+
+    return None
+
+
 def main():
     fp = sys.argv[1]
-    md, _ = ffprobe(fp)
+    zb = sys.stdin.buffer.read()
+    zs = zb.decode("utf-8", "replace")
+    md = json.loads(zs)
 
     try:
-        w = int(md[".resw"][1])
-        h = int(md[".resh"][1])
+        w, h = [int(x) for x in md["res"].split("x")]
         if not w + h:
             raise Exception()
     except:
         return "could not determine resolution"
 
-    if min(w, h) < 720:
+    # grab streams/format metadata + 2 seconds of frames at the start and end
+    zs = "ffprobe -hide_banner -v warning -of json -show_streams -show_format -show_packets -show_data_hash crc32 -read_intervals %+2,999999%+2"
+    cmd = zs.encode("ascii").split(b" ") + [fsenc(fp)]
+    p = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE)
+    so, se = p.communicate()
+
+    # spaces to tabs, drops filesize from 69k to 48k
+    so = b"\n".join(
+        [
+            b"\t" * int((len(x) - len(x.lstrip())) / 4) + x.lstrip()
+            for x in (so or b"").split(b"\n")
+        ]
+    )
+    with open(fsenc(f"{fp}.ff.json"), "wb") as f:
+        f.write(so)
+
+    err = errchk(so, se, p.returncode)
+    if err:
+        return err
+
+    if min(w, h) < 1080:
         return "resolution too small"
 
     zs = (
@@ -49,16 +111,7 @@ def main():
 
     p = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE)
     so, se = p.communicate()
-    rc = p.returncode
-    if rc:
-        err = (so + se).decode("utf-8", "replace").split("\n", 1)[0]
-        return f"ERROR {rc}: {err}"
-
-    if se:
-        err = se.decode("utf-8", "replace").split("\n", 1)[0]
-        return f"Warning: {err}"
-
-    return None
+    return errchk(so, se, p.returncode)
 
 
 if __name__ == "__main__":

contrib/plugins/README.md

@@ -15,6 +15,8 @@ save one of these as `.epilogue.html` inside a folder to customize it:
 point `--js-browser` to one of these by URL:
 
 * [`minimal-up2k.js`](minimal-up2k.js) is similar to the above `minimal-up2k.html` except it applies globally to all write-only folders
+* [`up2k-hooks.js`](up2k-hooks.js) lets you specify a ruleset for files to skip uploading
+  * [`up2k-hook-ytid.js`](up2k-hook-ytid.js) is a more specific example checking youtube-IDs against some API
 
 
 

contrib/plugins/minimal-up2k.js

@@ -18,7 +18,7 @@ var u2min = `
 <style>
 
 #ops, #path, #tree, #files, #epi+div+h2,
-#u2conf td.c+.c, #u2cards, #u2foot, #srch_dz, #srch_zd {
+#u2conf td.c+.c, #u2cards, #srch_dz, #srch_zd {
   display: none !important;
 }
 #u2conf {margin:5em auto 0 auto !important}

contrib/plugins/up2k-hook-ytid.js

@@ -0,0 +1,71 @@
+// way more specific example --
+// assumes all files dropped into the uploader have a youtube-id somewhere in the filename,
+// locates the youtube-ids and passes them to an API which returns a list of IDs which should be uploaded
+//
+// assumes copyparty is behind nginx as /ytq is a standalone service which must be rproxied in place
+
+function up2k_namefilter(good_files, nil_files, bad_files, hooks) {
+    var filenames = [],
+        file_lists = [good_files, nil_files, bad_files];
+
+    for (var lst of file_lists)
+        for (var ent of lst)
+            filenames.push(ent[1]);
+
+    var yt_ids = new Set();
+    for (var lst of file_lists)
+        for (var ent of lst) {
+            var m, name = ent[1];
+            while (true) {
+                // some ytdl fork did %(title)-%(id).%(ext) ...
+                m = /(?:^|[^\w])([\w-]{11})(?:$|[^\w-])/.exec(name);
+                if (!m)
+                    break;
+
+                yt_ids.add(m[1]);
+                name = name.replace(m[1], '');
+            }
+        }
+
+    toast.inf(5, `running query for ${yt_ids.size} videos...`);
+
+    var xhr = new XHR();
+    xhr.open('POST', '/ytq', true);
+    xhr.setRequestHeader('Content-Type', 'text/plain');
+    xhr.onload = xhr.onerror = function () {
+        if (this.status != 200)
+            return toast.err(0, `sorry, database query failed ;_;\n\nplease let us know so we can look at it, thx!!\n\nerror ${this.status}: ${(this.response && this.response.err) || this.responseText}`);
+
+        var new_lists = [],
+            ptn = new RegExp(this.responseText.trim().split('\n').join('|') || '\n'),
+            nothing_to_do = true,
+            n_skip = 0;
+
+        for (var lst of file_lists) {
+            var keep = [];
+            new_lists.push(keep);
+
+            for (var ent of lst)
+                if (ptn.exec(ent[1]))
+                    keep.push(ent);
+                else
+                    n_skip++;
+
+            if (keep.length)
+                nothing_to_do = false;
+        }
+
+        if (nothing_to_do)
+            return modal.alert('Good news -- turns out we already have all those videos.\n\nBut thank you for checking in!');
+        else if (n_skip)
+            toast.inf(0, `skipped ${n_skip} files which already exist on the server`);
+
+        [good_files, nil_files, bad_files] = new_lists;
+        hooks[0](good_files, nil_files, bad_files, hooks.slice(1));
+    };
+    xhr.send(Array.from(yt_ids).join('\n'));
+}
+
+up2k_hooks.push(function () {
+    up2k.gotallfiles.unshift(up2k_namefilter);
+});

contrib/plugins/up2k-hooks.js

@@ -0,0 +1,45 @@
+// hooks into up2k
+
+function up2k_namefilter(good_files, nil_files, bad_files, hooks) {
+    // is called when stuff is dropped into the browser,
+    // after iterating through the directory tree and discovering all files,
+    // before the upload confirmation dialogue is shown
+
+    // good_files will successfully upload
+    // nil_files are empty files and will show an alert in the final hook
+    // bad_files are unreadable and cannot be uploaded
+    var file_lists = [good_files, nil_files, bad_files];
+
+    // build a list of filenames
+    var filenames = [];
+    for (var lst of file_lists)
+        for (var ent of lst)
+            filenames.push(ent[1]);
+
+    toast.inf(5, "running database query...");
+
+    // simulate delay while passing the list to some api for checking
+    setTimeout(function () {
+
+        // only keep webm files as an example
+        var new_lists = [];
+        for (var lst of file_lists) {
+            var keep = [];
+            new_lists.push(keep);
+
+            for (var ent of lst)
+                if (/\.webm$/.test(ent[1]))
+                    keep.push(ent);
+        }
+
+        // finally, call the next hook in the chain
+        [good_files, nil_files, bad_files] = new_lists;
+        hooks[0](good_files, nil_files, bad_files, hooks.slice(1));
+
+    }, 1000);
+}
+
+// register
+up2k_hooks.push(function () {
+    up2k.gotallfiles.unshift(up2k_namefilter);
+});

copyparty/__main__.py

@@ -275,7 +275,7 @@ def disable_quickedit() -> None:
                 raise ctypes.WinError(err)  # type: ignore
         return args
 
-    k32 = ctypes.WinDLL("kernel32", use_last_error=True)  # type: ignore
+    k32 = ctypes.WinDLL(str("kernel32"), use_last_error=True)  # type: ignore
     if PY2:
         wintypes.LPDWORD = ctypes.POINTER(wintypes.DWORD)
 
@@ -394,6 +394,7 @@ def run_argparse(argv: list[str], formatter: Any, retry: bool) -> argparse.Names
               \033[36md2ts\033[35m disables metadata collection for existing files
               \033[36md2ds\033[35m disables onboot indexing, overrides -e2ds*
               \033[36md2t\033[35m disables metadata collection, overrides -e2t*
+              \033[36md2v\033[35m disables file verification, overrides -e2v*
               \033[36md2d\033[35m disables all database stuff, overrides -e2*
               \033[36mnohash=\\.iso$\033[35m skips hashing file contents if path matches *.iso
               \033[36mnoidx=\\.iso$\033[35m fully ignores the contents at paths matching *.iso
@@ -483,6 +484,7 @@ def run_argparse(argv: list[str], formatter: Any, retry: bool) -> argparse.Names
     ap2.add_argument("--no-dedup", action="store_true", help="disable symlink/hardlink creation; copy file contents instead")
     ap2.add_argument("--sparse", metavar="MiB", type=int, default=4, help="windows-only: minimum size of incoming uploads through up2k before they are made into sparse files")
     ap2.add_argument("--turbo", metavar="LVL", type=int, default=0, help="configure turbo-mode in up2k client; 0 = off and warn if enabled, 1 = off, 2 = on, 3 = on and disable datecheck")
+    ap2.add_argument("--u2sort", metavar="TXT", type=u, default="s", help="upload order; s=smallest-first, n=alphabetical, fs=force-s, fn=force-n -- alphabetical is a bit slower on fiber/LAN but makes it easier to eyeball if everything went fine")
 
     ap2 = ap.add_argument_group('network options')
     ap2.add_argument("-i", metavar="IP", type=u, default="0.0.0.0", help="ip to bind (comma-sep.)")
@@ -585,6 +587,9 @@ def run_argparse(argv: list[str], formatter: Any, retry: bool) -> argparse.Names
     ap2.add_argument("-e2d", action="store_true", help="enable up2k database, making files searchable + enables upload deduplocation")
     ap2.add_argument("-e2ds", action="store_true", help="scan writable folders for new files on startup; sets -e2d")
     ap2.add_argument("-e2dsa", action="store_true", help="scans all folders on startup; sets -e2ds")
+    ap2.add_argument("-e2v", action="store_true", help="verify file integrity; rehash all files and compare with db")
+    ap2.add_argument("-e2vu", action="store_true", help="on hash mismatch: update the database with the new hash")
+    ap2.add_argument("-e2vp", action="store_true", help="on hash mismatch: panic and quit copyparty")
     ap2.add_argument("--hist", metavar="PATH", type=u, help="where to store volume data (db, thumbs)")
     ap2.add_argument("--no-hash", metavar="PTN", type=u, help="regex: disable hashing of matching paths during e2ds folder scans")
     ap2.add_argument("--no-idx", metavar="PTN", type=u, help="regex: disable indexing of matching paths during e2ds folder scans")
@@ -755,6 +760,12 @@ def main(argv: Optional[list[str]] = None) -> None:
     except:
         raise Exception("invalid value for -p")
 
+    for arg, kname, okays in [["--u2sort", "u2sort", "s n fs fn"]]:
+        val = unicode(getattr(al, kname))
+        if val not in okays.split():
+            zs = "argument {} cannot be '{}'; try one of these: {}"
+            raise Exception(zs.format(arg, val, okays))
+
     if HAVE_SSL:
         if al.ssl_ver:
             configure_ssl_ver(al)

copyparty/__version__.py

@@ -1,8 +1,8 @@
 # coding: utf-8
 
-VERSION = (1, 3, 4)
+VERSION = (1, 3, 7)
 CODENAME = "god dag"
-BUILD_DT = (2022, 7, 6)
+BUILD_DT = (2022, 7, 16)
 
 S_VERSION = ".".join(map(str, VERSION))
 S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT)

copyparty/authsrv.py

@@ -1008,7 +1008,7 @@ class AuthSrv(object):
                 if ptn:
                     vol.flags[vf] = re.compile(ptn)
 
-            for k in ["e2t", "e2ts", "e2tsr"]:
+            for k in ["e2t", "e2ts", "e2tsr", "e2v", "e2vu", "e2vp"]:
                 if getattr(self.args, k):
                     vol.flags[k] = True
 
@@ -1030,7 +1030,7 @@ class AuthSrv(object):
             self._read_volflag(vol.flags, "mtp", self.args.mtp, True)
 
             # d2d drops all database features for a volume
-            for grp, rm in [["d2d", "e2d"], ["d2t", "e2t"]]:
+            for grp, rm in [["d2d", "e2d"], ["d2t", "e2t"], ["d2d", "e2v"]]:
                 if not vol.flags.get(grp, False):
                     continue
 
@@ -1052,6 +1052,12 @@ class AuthSrv(object):
 
                 vol.flags = {k: v for k, v in vol.flags.items() if not k.startswith(rm)}
 
+            for grp, rm in [["d2v", "e2v"]]:
+                if not vol.flags.get(grp, False):
+                    continue
+
+                vol.flags = {k: v for k, v in vol.flags.items() if not k.startswith(rm)}
+
             # verify tags mentioned by -mt[mp] are used by -mte
             local_mtp = {}
             local_only_mtp = {}
@@ -1101,6 +1107,7 @@ class AuthSrv(object):
 
         vfs.bubble_flags()
 
+        e2vs = []
         t = "volumes and permissions:\n"
         for zv in vfs.all_vols.values():
             if not self.warn_anonwrite:
@@ -1118,8 +1125,16 @@ class AuthSrv(object):
                 u = ", ".join("\033[35meverybody\033[0m" if x == "*" else x for x in u)
                 u = u if u else "\033[36m--none--\033[0m"
                 t += "\n|  {}:  {}".format(txt, u)
+
+            if "e2v" in zv.flags and zv.axs.uwrite:
+                e2vs.append(zv.vpath or "/")
+
             t += "\n"
 
+        if e2vs:
+            t += "\n\033[33me2v enabled for the following volumes;\nuploads will be blocked until scan has finished:\n  \033[0m"
+            t += "  ".join(e2vs) + "\n"
+
         if self.warn_anonwrite and not self.args.no_voldump:
             self.log(t)
 
@@ -1127,7 +1142,7 @@ class AuthSrv(object):
             zv, _ = vfs.get("/", "*", False, True)
             if self.warn_anonwrite and os.getcwd() == zv.realpath:
                 self.warn_anonwrite = False
-                t = "anyone can read/write the current directory: {}\n"
+                t = "anyone can write to the current directory: {}\n"
                 self.log(t.format(zv.realpath), c=1)
         except Pebkac:
             self.warn_anonwrite = True

copyparty/fsutil.py

@@ -2,6 +2,7 @@
 from __future__ import print_function, unicode_literals
 
 import ctypes
+import os
 import re
 import time
 
@@ -21,6 +22,7 @@ class Fstab(object):
     def __init__(self, log: RootLogger):
         self.log_func = log
 
+        self.trusted = False
         self.tab: Optional[VFS] = None
         self.cache: dict[str, str] = {}
         self.age = 0.0
@@ -49,6 +51,7 @@ class Fstab(object):
                 self.log(msg.format(path, fs, min_ex()), 3)
                 return fs
 
+        path = path.lstrip("/")
         try:
             return self.cache[path]
         except:
@@ -92,21 +95,44 @@ class Fstab(object):
 
     def relabel(self, path: str, nval: str) -> None:
         assert self.tab
+        self.cache = {}
+        path = path.lstrip("/")
         ptn = re.compile(r"^[^\\/]*")
-        vn, _ = self.tab._find(path)
+        vn, rem = self.tab._find(path)
+        if not self.trusted:
+            # no mtab access; have to build as we go
+            if "/" in rem:
+                self.tab.add("idk", os.path.join(vn.vpath, rem.split("/")[0]))
+            if rem:
+                self.tab.add(nval, path)
+            else:
+                vn.realpath = nval
+
+            return
+
         visit = [vn]
         while visit:
             vn = visit.pop()
             vn.realpath = ptn.sub(nval, vn.realpath)
             visit.extend(list(vn.nodes.values()))
-        self.cache = {}
 
     def get_unix(self, path: str) -> str:
         if not self.tab:
-            self.build_tab()
+            try:
+                self.build_tab()
+                self.trusted = True
+            except:
+                # prisonparty or other restrictive environment
+                self.log("failed to build tab:\n{}".format(min_ex()), 3)
+                self.tab = VFS(self.log_func, "idk", "/", AXS(), {})
+                self.trusted = False
 
         assert self.tab
-        return self.tab._find(path)[0].realpath.split("/")[0]
+        ret = self.tab._find(path)[0]
+        if self.trusted or path == ret.vpath:
+            return ret.realpath.split("/")[0]
+        else:
+            return "idk"
 
     def get_w32(self, path: str) -> str:
         # list mountpoints: fsutil fsinfo drives

copyparty/httpcli.py

@@ -380,13 +380,18 @@ class HttpCli(object):
                 if not self._check_nonfatal(pex, post):
                     self.keepalive = False
 
-                msg = str(ex) if pex == ex else min_ex()
+                em = str(ex)
+                msg = em if pex == ex else min_ex()
                 self.log("{}\033[0m, {}".format(msg, self.vpath), 3)
 
-                msg = "{}\r\nURL: {}\r\n".format(str(ex), self.vpath)
+                msg = "{}\r\nURL: {}\r\n".format(em, self.vpath)
                 if self.hint:
                     msg += "hint: {}\r\n".format(self.hint)
 
+                if "database is locked" in em:
+                    self.conn.hsrv.broker.say("log_stacks")
+                    msg += "hint: important info in the server log\r\n"
+
                 msg = "<pre>" + html_escape(msg)
                 self.reply(msg.encode("utf-8", "replace"), status=pex.code, volsan=True)
                 return self.keepalive
@@ -1127,8 +1132,10 @@ class HttpCli(object):
             except:
                 self.log("failed to utime ({}, {})".format(fin_path, times))
 
+        cinf = self.headers.get("x-up2k-stat", "")
+
         spd = self._spd(post_sz)
-        self.log("{} thank".format(spd))
+        self.log("{:70} thank {}".format(spd, cinf))
         self.reply(b"thank")
         return True
 
@@ -2413,6 +2420,7 @@ class HttpCli(object):
             "dtheme": self.args.theme,
             "themes": self.args.themes,
             "turbolvl": self.args.turbo,
+            "u2sort": self.args.u2sort,
         }
 
         if self.args.js_browser:

copyparty/mtag.py

@@ -46,6 +46,7 @@ class MParser(object):
         self.force = False
         self.kill = "t"  # tree; all children recursively
         self.audio = "y"
+        self.pri = 0  # priority; higher = later
         self.ext = []
 
         while True:
@@ -83,6 +84,10 @@ class MParser(object):
                 self.ext.append(arg[1:])
                 continue
 
+            if arg.startswith("p"):
+                self.pri = int(arg[1:] or "1")
+                continue
+
             raise Exception()
 
 
@@ -487,7 +492,9 @@ class MTag(object):
         ret, md = ffprobe(abspath)
         return self.normalize_tags(ret, md)
 
-    def get_bin(self, parsers: dict[str, MParser], abspath: str) -> dict[str, Any]:
+    def get_bin(
+        self, parsers: dict[str, MParser], abspath: str, oth_tags: dict[str, Any]
+    ) -> dict[str, Any]:
         if not bos.path.isfile(abspath):
             return {}
 
@@ -497,8 +504,8 @@ class MTag(object):
         env = os.environ.copy()
         env["PYTHONPATH"] = pypath
 
-        ret = {}
-        for tagname, parser in parsers.items():
+        ret: dict[str, Any] = {}
+        for tagname, parser in sorted(parsers.items(), key=lambda x: (x[1].pri, x[0])):
             try:
                 cmd = [parser.bin, abspath]
                 if parser.bin.endswith(".py"):
@@ -506,6 +513,11 @@ class MTag(object):
 
                 args = {"env": env, "timeout": parser.timeout, "kill": parser.kill}
 
+                if parser.pri:
+                    zd = oth_tags.copy()
+                    zd.update(ret)
+                    args["sin"] = json.dumps(zd).encode("utf-8", "replace")
+
                 if WINDOWS:
                     args["creationflags"] = 0x4000
                 else:

copyparty/svchub.py

@@ -2,7 +2,9 @@
 from __future__ import print_function, unicode_literals
 
 import argparse
+import base64
 import calendar
+import gzip
 import os
 import shlex
 import signal
@@ -27,7 +29,7 @@ from .mtag import HAVE_FFMPEG, HAVE_FFPROBE
 from .tcpsrv import TcpSrv
 from .th_srv import HAVE_PIL, HAVE_VIPS, HAVE_WEBP, ThumbSrv
 from .up2k import Up2k
-from .util import ansi_re, min_ex, mp, start_log_thrs, start_stackmon
+from .util import ansi_re, min_ex, mp, start_log_thrs, start_stackmon, alltrace
 
 
 class SvcHub(object):
@@ -56,6 +58,7 @@ class SvcHub(object):
 
         self.log_mutex = threading.Lock()
         self.next_day = 0
+        self.tstack = 0.0
 
         if args.sss or args.s >= 3:
             args.ss = True
@@ -358,6 +361,9 @@ class SvcHub(object):
 
             print("nailed it", end="")
             ret = self.retcode
+        except:
+            print("\033[31m[ error during shutdown ]\n{}\033[0m".format(min_ex()))
+            raise
         finally:
             if self.args.wintitle:
                 print("\033]0;\033\\", file=sys.stderr, end="")
@@ -497,3 +503,15 @@ class SvcHub(object):
             sck.sendall(b"READY=1")
         except:
             self.log("sd_notify", min_ex())
+
+    def log_stacks(self) -> None:
+        td = time.time() - self.tstack
+        if td < 300:
+            self.log("stacks", "cooldown {}".format(td))
+            return
+
+        self.tstack = time.time()
+        zb = alltrace().encode("utf-8", "replace")
+        zb = gzip.compress(zb)
+        zs = base64.b64encode(zb).decode("ascii")
+        self.log("stacks", zs)

copyparty/th_srv.py

@@ -559,14 +559,15 @@ class ThumbSrv(object):
     def clean(self, histpath: str) -> int:
         ret = 0
         for cat in ["th", "ac"]:
-            ret += self._clean(histpath, cat, "")
+            top = os.path.join(histpath, cat)
+            if not bos.path.isdir(top):
+                continue
+
+            ret += self._clean(cat, top)
 
         return ret
 
-    def _clean(self, histpath: str, cat: str, thumbpath: str) -> int:
-        if not thumbpath:
-            thumbpath = os.path.join(histpath, cat)
-
+    def _clean(self, cat: str, thumbpath: str) -> int:
         # self.log("cln {}".format(thumbpath))
         exts = ["jpg", "webp"] if cat == "th" else ["opus", "caf"]
         maxage = getattr(self.args, cat + "_maxage")
@@ -600,7 +601,7 @@ class ThumbSrv(object):
                             self.log("rm -rf [{}]".format(fp))
                             shutil.rmtree(fp, ignore_errors=True)
                 else:
-                    self._clean(histpath, cat, fp)
+                    ndirs += self._clean(cat, fp)
 
                 continue
 

copyparty/up2k.py

@@ -9,6 +9,7 @@ import math
 import os
 import re
 import shutil
+import signal
 import stat
 import subprocess as sp
 import threading
@@ -35,6 +36,7 @@ from .util import (
     quotep,
     ren_open,
     rmdirs,
+    rmdirs_up,
     s2hms,
     s3dec,
     s3enc,
@@ -67,12 +69,20 @@ class Dbw(object):
 
 
 class Mpqe(object):
-    def __init__(self, mtp: dict[str, MParser], entags: set[str], w: str, abspath: str):
+    def __init__(
+        self,
+        mtp: dict[str, MParser],
+        entags: set[str],
+        w: str,
+        abspath: str,
+        oth_tags: dict[str, Any],
+    ):
         # mtp empty = mtag
         self.mtp = mtp
         self.entags = entags
         self.w = w
         self.abspath = abspath
+        self.oth_tags = oth_tags
 
 
 class Up2k(object):
@@ -86,7 +96,9 @@ class Up2k(object):
         self.r_hash = re.compile("^[0-9a-zA-Z_-]{44}$")
 
         self.gid = 0
+        self.stop = False
         self.mutex = threading.Lock()
+        self.blocked: Optional[str] = None
         self.pp: Optional[ProgressPrinter] = None
         self.rescan_cond = threading.Condition()
         self.need_rescan: set[str] = set()
@@ -182,6 +194,14 @@ class Up2k(object):
     def log(self, msg: str, c: Union[int, str] = 0) -> None:
         self.log_func("up2k", msg + "\033[K", c)
 
+    def _block(self, why: str) -> None:
+        self.blocked = why
+        self.log("uploads are temporarily blocked due to " + why, 3)
+
+    def _unblock(self) -> None:
+        self.blocked = None
+        self.log("uploads are now possible", 2)
+
     def get_state(self) -> str:
         mtpq: Union[int, str] = 0
         q = "select count(w) from mt where k = 't:mtp'"
@@ -405,7 +425,13 @@ class Up2k(object):
                 self.mtag = None
 
         # e2ds(a) volumes first
+        if next((zv for zv in vols if "e2ds" in zv.flags), None):
+            self._block("indexing")
+
         for vol in vols:
+            if self.stop:
+                break
+
             en: set[str] = set()
             if "mte" in vol.flags:
                 en = set(vol.flags["mte"].split(","))
@@ -421,16 +447,51 @@ class Up2k(object):
                 if vac:
                     need_vac[vol] = True
 
-            if "e2ts" not in vol.flags:
-                t = "online, idle"
-            else:
+            if "e2v" in vol.flags:
+                t = "online (integrity-check pending)"
+            elif "e2ts" in vol.flags:
                 t = "online (tags pending)"
+            else:
+                t = "online, idle"
 
             self.volstate[vol.vpath] = t
 
+        # file contents verification
+        if next((zv for zv in vols if "e2v" in zv.flags), None):
+            self._block("integrity verification")
+
+        for vol in vols:
+            if self.stop:
+                break
+
+            if "e2v" not in vol.flags:
+                continue
+
+            t = "online (verifying integrity)"
+            self.volstate[vol.vpath] = t
+            self.log("{} [{}]".format(t, vol.realpath))
+
+            nmod = self._verify_integrity(vol)
+            if nmod:
+                self.log("modified {} entries in the db".format(nmod), 3)
+                need_vac[vol] = True
+
+            if "e2ts" in vol.flags:
+                t = "online (tags pending)"
+            else:
+                t = "online, idle"
+
+            self.volstate[vol.vpath] = t
+
+        if self.blocked:
+            self._unblock()
+
         # open the rest + do any e2ts(a)
         needed_mutagen = False
         for vol in vols:
+            if self.stop:
+                break
+
             if "e2ts" not in vol.flags:
                 continue
 
@@ -455,6 +516,9 @@ class Up2k(object):
                 cur.connection.commit()
                 cur.execute("vacuum")
 
+        if self.stop:
+            return False
+
         self.pp.end = True
 
         msg = "{} volumes in {:.2f} sec"
@@ -517,7 +581,9 @@ class Up2k(object):
             if vpath:
                 vpath += "/"
 
-            self.log("/{} {}".format(vpath, " ".join(sorted(a))), "35")
+            zs = " ".join(sorted(a))
+            zs = zs.replace("30mre.compile(", "30m(")  # nohash
+            self.log("/{} {}".format(vpath, zs), "35")
 
         reg = {}
         drp = None
@@ -634,6 +700,9 @@ class Up2k(object):
         gl = sorted(g)
         inames = {x[0]: 1 for x in gl}
         for iname, inf in gl:
+            if self.stop:
+                return -1
+
             abspath = os.path.join(cdir, iname)
             if rei and rei.search(abspath):
                 continue
@@ -712,7 +781,9 @@ class Up2k(object):
                         self.log("file: {}".format(abspath))
 
                     try:
-                        hashes = self._hashlist_from_file(abspath)
+                        hashes = self._hashlist_from_file(
+                            abspath, "a{}, ".format(self.pp.n)
+                        )
                     except Exception as ex:
                         self.log("hash: {} @ [{}]".format(repr(ex), abspath))
                         continue
@@ -729,6 +800,9 @@ class Up2k(object):
                     db.n = 0
                     db.t = time.time()
 
+        if self.stop:
+            return -1
+
         # drop missing files
         rd = cdir[len(top) + 1 :].strip("/")
         if WINDOWS:
@@ -789,6 +863,106 @@ class Up2k(object):
 
         return n_rm
 
+    def _verify_integrity(self, vol: VFS) -> int:
+        """expensive; blocks database access until finished"""
+        ptop = vol.realpath
+        assert self.pp and self.mtag
+
+        cur = self.cur[ptop]
+        rei = vol.flags.get("noidx")
+        reh = vol.flags.get("nohash")
+        e2vu = "e2vu" in vol.flags
+        e2vp = "e2vp" in vol.flags
+
+        excl = [
+            d[len(vol.vpath) :].lstrip("/")
+            for d in self.asrv.vfs.all_vols
+            if d != vol.vpath and (d.startswith(vol.vpath + "/") or not vol.vpath)
+        ]
+        qexa: list[str] = []
+        pexa: list[str] = []
+        for vpath in excl:
+            qexa.append("up.rd != ? and not up.rd like ?||'%'")
+            pexa.extend([vpath, vpath])
+
+        pex = tuple(pexa)
+        qex = " and ".join(qexa)
+        if qex:
+            qex = " where " + qex
+
+        rewark: list[tuple[str, str, str, int, int]] = []
+
+        with self.mutex:
+            b_left = 0
+            n_left = 0
+            q = "select sz from up" + qex
+            for (sz,) in cur.execute(q, pex):
+                b_left += sz  # sum() can overflow according to docs
+                n_left += 1
+
+            q = "select w, mt, sz, rd, fn from up" + qex
+            for w, mt, sz, drd, dfn in cur.execute(q, pex):
+                if self.stop:
+                    return -1
+
+                n_left -= 1
+                b_left -= sz
+                if drd.startswith("//") or dfn.startswith("//"):
+                    rd, fn = s3dec(drd, dfn)
+                else:
+                    rd = drd
+                    fn = dfn
+
+                abspath = os.path.join(ptop, rd, fn)
+                if rei and rei.search(abspath):
+                    continue
+
+                nohash = reh.search(abspath) if reh else False
+
+                pf = "v{}, {:.0f}+".format(n_left, b_left / 1024 / 1024)
+                self.pp.msg = pf + abspath
+
+                st = bos.stat(abspath)
+                sz2 = st.st_size
+                mt2 = int(st.st_mtime)
+
+                if nohash:
+                    w2 = up2k_wark_from_metadata(self.salt, sz2, mt2, rd, fn)
+                else:
+                    if sz2 > 1024 * 1024 * 32:
+                        self.log("file: {}".format(abspath))
+
+                    try:
+                        hashes = self._hashlist_from_file(abspath, pf)
+                    except Exception as ex:
+                        self.log("hash: {} @ [{}]".format(repr(ex), abspath))
+                        continue
+
+                    w2 = up2k_wark_from_hashlist(self.salt, sz2, hashes)
+
+                if w == w2:
+                    continue
+
+                rewark.append((drd, dfn, w2, sz2, mt2))
+
+                t = "hash mismatch: {}\n  db: {} ({} byte, {})\n  fs: {} ({} byte, {})"
+                t = t.format(abspath, w, sz, mt, w2, sz2, mt2)
+                self.log(t, 1)
+
+            if e2vp and rewark:
+                self.hub.retcode = 1
+                os.kill(os.getpid(), signal.SIGTERM)
+                raise Exception("{} files have incorrect hashes".format(len(rewark)))
+
+            if not e2vu:
+                return 0
+
+            for rd, fn, w, sz, mt in rewark:
+                q = "update up set w = ?, sz = ?, mt = ? where rd = ? and fn = ? limit 1"
+                cur.execute(q, (w, sz, int(mt), rd, fn))
+
+        return len(rewark)
+
     def _build_tags_index(self, vol: VFS) -> tuple[int, int, bool]:
         ptop = vol.realpath
         with self.mutex:
@@ -855,6 +1029,9 @@ class Up2k(object):
             c3 = conn.cursor()
             n_left = cur.execute("select count(w) from up").fetchone()[0]
             for w, rd, fn in cur.execute("select w, rd, fn from up order by rd, fn"):
+                if self.stop:
+                    return -1, -1, False
+
                 n_left -= 1
                 q = "select w from mt where w = ?"
                 if c2.execute(q, (w[:16],)).fetchone():
@@ -872,7 +1049,7 @@ class Up2k(object):
                 if not mpool:
                     n_tags = self._tag_file(c3, entags, w, abspath)
                 else:
-                    mpool.put(Mpqe({}, entags, w, abspath))
+                    mpool.put(Mpqe({}, entags, w, abspath, {}))
                     # not registry cursor; do not self.mutex:
                     n_tags = len(self._flush_mpool(c3))
 
@@ -978,8 +1155,8 @@ class Up2k(object):
                     abspath = os.path.join(ptop, rd, fn)
 
                     q = "select k from mt where w = ?"
-                    zq2 = cur.execute(q, (w,)).fetchall()
-                    have: dict[str, Union[str, float]] = {x[0]: 1 for x in zq2}
+                    zq = cur.execute(q, (w,)).fetchall()
+                    have: dict[str, Union[str, float]] = {x[0]: 1 for x in zq}
 
                     did_nothing = False
                     parsers = self._get_parsers(ptop, have, abspath)
@@ -988,7 +1165,14 @@ class Up2k(object):
                         n_left -= 1
                         continue
 
-                    jobs.append(Mpqe(parsers, set(), w, abspath))
+                    if next((x for x in parsers.values() if x.pri), None):
+                        q = "select k, v from mt where w = ?"
+                        zq2 = cur.execute(q, (w,)).fetchall()
+                        oth_tags = {str(k): v for k, v in zq2}
+                    else:
+                        oth_tags = {}
+
+                    jobs.append(Mpqe(parsers, set(), w, abspath, oth_tags))
                     in_progress[w] = True
 
             with self.mutex:
@@ -1112,7 +1296,7 @@ class Up2k(object):
             return
 
         for _ in range(mpool.maxsize):
-            mpool.put(Mpqe({}, set(), "", ""))
+            mpool.put(Mpqe({}, set(), "", "", {}))
 
         mpool.join()
 
@@ -1128,7 +1312,7 @@ class Up2k(object):
                 if not qe.mtp:
                     tags = self.mtag.get(qe.abspath)
                 else:
-                    tags = self.mtag.get_bin(qe.mtp, qe.abspath)
+                    tags = self.mtag.get_bin(qe.mtp, qe.abspath, qe.oth_tags)
                     vtags = [
                         "\033[36m{} \033[33m{}".format(k, v) for k, v in tags.items()
                     ]
@@ -1319,11 +1503,24 @@ class Up2k(object):
 
         cur.connection.commit()
 
+    def _job_volchk(self, cj: dict[str, Any]) -> None:
+        if not self.register_vpath(cj["ptop"], cj["vcfg"]):
+            if cj["ptop"] not in self.registry:
+                raise Pebkac(410, "location unavailable")
+
     def handle_json(self, cj: dict[str, Any]) -> dict[str, Any]:
-        with self.mutex:
-            if not self.register_vpath(cj["ptop"], cj["vcfg"]):
-                if cj["ptop"] not in self.registry:
-                    raise Pebkac(410, "location unavailable")
+        try:
+            # bit expensive; 3.9=10x 3.11=2x
+            if self.mutex.acquire(timeout=10):
+                self._job_volchk(cj)
+                self.mutex.release()
+            else:
+                t = "cannot receive uploads right now;\nserver busy with {}.\nPlease wait; the client will retry..."
+                raise Pebkac(503, t.format(self.blocked or "[unknown]"))
+        except TypeError:
+            # py2
+            with self.mutex:
+                self._job_volchk(cj)
 
         cj["name"] = sanitize_fn(cj["name"], "", [".prologue.html", ".epilogue.html"])
         cj["poke"] = time.time()
@@ -1842,11 +2039,12 @@ class Up2k(object):
         adir, fn = os.path.split(atop)
         try:
             st = bos.lstat(atop)
+            is_dir = stat.S_ISDIR(st.st_mode)
         except:
             raise Pebkac(400, "file not found on disk (already deleted?)")
 
         scandir = not self.args.no_scandir
-        if stat.S_ISDIR(st.st_mode):
+        if is_dir:
             g = vn.walk("", rem, [], uname, permsets, True, scandir, True)
             if unpost:
                 raise Pebkac(400, "cannot unpost folders")
@@ -1881,8 +2079,14 @@ class Up2k(object):
 
                 bos.unlink(abspath)
 
-        rm = rmdirs(self.log_func, scandir, True, atop, 1)
-        return n_files, rm[0], rm[1]
+        ok: list[str] = []
+        ng: list[str] = []
+        if is_dir:
+            ok, ng = rmdirs(self.log_func, scandir, True, atop, 1)
+
+        ok2, ng2 = rmdirs_up(os.path.dirname(atop))
+
+        return n_files, ok + ok2, ng + ng2
 
     def handle_mv(self, uname: str, svp: str, dvp: str) -> str:
         svn, srem = self.asrv.vfs.get(svp, uname, True, False, True)
@@ -1924,6 +2128,7 @@ class Up2k(object):
                     self._mv_file(uname, svpf, dvpf)
 
         rmdirs(self.log_func, scandir, True, sabs, 1)
+        rmdirs_up(os.path.dirname(sabs))
         return "k"
 
     def _mv_file(self, uname: str, svp: str, dvp: str) -> str:
@@ -2180,14 +2385,15 @@ class Up2k(object):
 
         return wark
 
-    def _hashlist_from_file(self, path: str) -> list[str]:
+    def _hashlist_from_file(self, path: str, prefix: str = "") -> list[str]:
         fsz = bos.path.getsize(path)
         csz = up2k_chunksize(fsz)
         ret = []
         with open(fsenc(path), "rb", 512 * 1024) as f:
             while fsz > 0:
                 if self.pp:
-                    self.pp.msg = "{} MB, {}".format(int(fsz / 1024 / 1024), path)
+                    mb = int(fsz / 1024 / 1024)
+                    self.pp.msg = "{}{} MB, {}".format(prefix, mb, path)
 
                 hashobj = hashlib.sha512()
                 rem = min(csz, fsz)
@@ -2207,8 +2413,11 @@ class Up2k(object):
         return ret
 
     def _new_upload(self, job: dict[str, Any]) -> None:
-        self.registry[job["ptop"]][job["wark"]] = job
         pdir = os.path.join(job["ptop"], job["prel"])
+        if not job["size"] and bos.path.isfile(os.path.join(pdir, job["name"])):
+            return
+
+        self.registry[job["ptop"]][job["wark"]] = job
         job["name"] = self._untaken(pdir, job["name"], job["t0"], job["addr"])
         # if len(job["name"].split(".")) > 8:
         #    raise Exception("aaa")
@@ -2337,6 +2546,9 @@ class Up2k(object):
                 except:
                     pass
 
+        if self.args.nw:
+            return
+
         path = os.path.join(histpath, "up2k.snap")
         if not reg:
             if ptop not in self.snap_prev or self.snap_prev[ptop] is not None:
@@ -2383,7 +2595,7 @@ class Up2k(object):
                 ntags1 = len(tags)
                 parsers = self._get_parsers(ptop, tags, abspath)
                 if parsers:
-                    tags.update(self.mtag.get_bin(parsers, abspath))
+                    tags.update(self.mtag.get_bin(parsers, abspath, tags))
             except Exception as ex:
                 self._log_tag_err("", abspath, ex)
                 continue
@@ -2437,6 +2649,7 @@ class Up2k(object):
         # self.log("hashq {} push {}/{}/{}".format(self.n_hashq, ptop, rd, fn))
 
     def shutdown(self) -> None:
+        self.stop = True
         self.log("writing snapshot")
         self.do_snapshot()
 

copyparty/util.py

@@ -137,6 +137,9 @@ IMPLICATIONS = [
     ["e2tsr", "e2ts"],
     ["e2ts", "e2t"],
     ["e2t", "e2d"],
+    ["e2vu", "e2v"],
+    ["e2vp", "e2v"],
+    ["e2v", "e2d"],
 ]
 
 
@@ -1422,7 +1425,8 @@ def statdir(
 def rmdirs(
     logger: RootLogger, scandir: bool, lstat: bool, top: str, depth: int
 ) -> tuple[list[str], list[str]]:
-    if not os.path.exists(fsenc(top)) or not os.path.isdir(fsenc(top)):
+    """rmdir all descendants, then self"""
+    if not os.path.isdir(fsenc(top)):
         top = os.path.dirname(top)
         depth -= 1
 
@@ -1446,6 +1450,21 @@ def rmdirs(
     return ok, ng
 
 
+def rmdirs_up(top: str) -> tuple[list[str], list[str]]:
+    """rmdir on self, then all parents"""
+    try:
+        os.rmdir(fsenc(top))
+    except:
+        return [], [top]
+
+    par = os.path.dirname(top)
+    if not par:
+        return [top], []
+
+    ok, ng = rmdirs_up(par)
+    return [top] + ok, ng
+
+
 def unescape_cookie(orig: str) -> str:
     # mw=idk; doot=qwe%2Crty%3Basd+fgh%2Bjkl%25zxc%26vbn  # qwe,rty;asd fgh+jkl%zxc&vbn
     ret = ""
@@ -1566,12 +1585,17 @@ def runcmd(
     argv: Union[list[bytes], list[str]], timeout: Optional[int] = None, **ka: Any
 ) -> tuple[int, str, str]:
     kill = ka.pop("kill", "t")  # [t]ree [m]ain [n]one
+
+    sin = ka.pop("sin", None)
+    if sin:
+        ka["stdin"] = sp.PIPE
+
     p = sp.Popen(argv, stdout=sp.PIPE, stderr=sp.PIPE, **ka)
     if not timeout or PY2:
-        stdout, stderr = p.communicate()
+        stdout, stderr = p.communicate(sin)
     else:
         try:
-            stdout, stderr = p.communicate(timeout=timeout)
+            stdout, stderr = p.communicate(sin, timeout=timeout)
         except sp.TimeoutExpired:
             if kill == "n":
                 return -18, "", ""  # SIGCONT; leave it be
@@ -1797,7 +1821,6 @@ def termsize() -> tuple[int, int]:
     def ioctl_GWINSZ(fd: int) -> Optional[tuple[int, int]]:
         try:
             import fcntl
-            import struct
             import termios
 
             cr = struct.unpack("hh", fcntl.ioctl(fd, termios.TIOCGWINSZ, b"1234"))

copyparty/web/baguettebox.js

@@ -229,7 +229,7 @@ window.baguetteBox = (function () {
             ['C', 'video: toggle auto-next'],
             ['<code>[</code>, <code>]</code>', 'video: loop start / end'],
         ],
-            d = mknod('table'),
+            d = mknod('table', 'bbox-halp'),
             html = ['<tbody>'];
 
         for (var a = 0; a < list.length; a++)
@@ -238,7 +238,6 @@ window.baguetteBox = (function () {
         html.push('<tr><td colspan="2">tap middle of img to hide btns</td></tr>');
         html.push('<tr><td colspan="2">tap left/right sides for prev/next</td></tr>');
         d.innerHTML = html.join('\n') + '</tbody>';
-        d.setAttribute('id', 'bbox-halp');
         d.onclick = function () {
             overlay.removeChild(d);
         };
@@ -487,9 +486,8 @@ window.baguetteBox = (function () {
         var imagesFiguresIds = [];
         var imagesCaptionsIds = [];
         for (var i = 0, fullImage; i < gallery.length; i++) {
-            fullImage = mknod('div');
+            fullImage = mknod('div', 'baguette-img-' + i);
             fullImage.className = 'full-image';
-            fullImage.id = 'baguette-img-' + i;
             imagesElements.push(fullImage);
 
             imagesFiguresIds.push('bbox-figure-' + i);
@@ -631,16 +629,14 @@ window.baguetteBox = (function () {
         if (is_vid && index != currentIndex)
             return;  // no preload
 
-        var figure = mknod('figure');
-        figure.id = 'bbox-figure-' + index;
+        var figure = mknod('figure', 'bbox-figure-' + index);
         figure.innerHTML = '<div class="bbox-spinner">' +
             '<div class="bbox-double-bounce1"></div>' +
             '<div class="bbox-double-bounce2"></div>' +
             '</div>';
 
         if (options.captions && imageCaption) {
-            var figcaption = mknod('figcaption');
-            figcaption.id = 'bbox-figcaption-' + index;
+            var figcaption = mknod('figcaption', 'bbox-figcaption-' + index);
             figcaption.innerHTML = imageCaption;
             figure.appendChild(figcaption);
         }
@@ -943,7 +939,8 @@ window.baguetteBox = (function () {
         else
             timer.rm(rotn);
 
-        el.onclick = function (e) {
+        var ctime = 0;
+        el.onclick = v ? null : function (e) {
             var rc = e.target.getBoundingClientRect(),
                 x = e.clientX - rc.left,
                 fx = x / (rc.right - rc.left);
@@ -957,6 +954,11 @@ window.baguetteBox = (function () {
             clmod(ebi('bbox-btns'), 'off', 't');
             clmod(btnPrev, 'off', 't');
             clmod(btnNext, 'off', 't');
+
+            if (Date.now() - ctime <= 500)
+                tglfull();
+
+            ctime = Date.now();
         };
 
         var prev = QS('.full-image.vis');

copyparty/web/browser.css

@@ -527,6 +527,11 @@ html.dy {
 	--u2-tab-1-bg: a;
 	--u2-b1-bg: #000;
 	--u2-b2-bg: #000;
+	--u2-o-h-bg: #999;
+	--u2-o-1h-bg: #999;
+	--u2-o-bg: #eee;
+	--u2-o-1-bg: #000;
+
 	--ud-b1: a;
 
 	--sort-1: a;
@@ -2216,6 +2221,10 @@ html.y #bbox-overlay figcaption a {
 #u2notbtn * {
 	line-height: 1.3em;
 }
+#u2mu div {
+	height: 1.2em;
+	overflow: hidden;
+}
 #u2tabw {
 	min-height: 0;
 	transition: min-height .2s;
@@ -2225,6 +2234,7 @@ html.y #bbox-overlay figcaption a {
 	display: none;
 }
 #u2tab {
+	table-layout: fixed;
 	border-collapse: collapse;
 	width: calc(100% - 2em);
 	max-width: 100em;
@@ -2234,6 +2244,7 @@ html.y #bbox-overlay figcaption a {
 	max-width: none;
 }
 #u2tab td {
+	word-wrap: break-word;
 	border: 1px solid rgba(128,128,128,0.8);
 	border-width: 0 0px 1px 0;
 	padding: .2em .3em;
@@ -2248,7 +2259,19 @@ html.y #bbox-overlay figcaption a {
 #u2tab.up.ok td:nth-child(3),
 #u2tab.up.bz td:nth-child(3),
 #u2tab.up.q  td:nth-child(3) {
-	width: 19em;
+	width: 18em;
+}
+@media (max-width: 65em) {
+	#u2tab {
+		font-size: .9em;
+	}
+}
+@media (max-width: 50em) {
+	#u2tab.up.ok td:nth-child(3),
+	#u2tab.up.bz td:nth-child(3),
+	#u2tab.up.q  td:nth-child(3) {
+		width: 16em;
+	}
 }
 #op_up2k.srch td.prog {
 	font-family: sans-serif;
@@ -2348,7 +2371,7 @@ html.y #bbox-overlay figcaption a {
 	width: 48em;
 }
 #u2conf.ww {
-	width: 74em;
+	width: 78em;
 }
 #u2conf.ww #u2c3w {
 	width: 29em;
@@ -2449,11 +2472,11 @@ html.b #u2conf a.b:hover {
 	color: var(--fg-max);
 	font-style: italic;
 	text-align: center;
-	font-size: .9em;
-	margin: 1em 0;
+	font-size: 1.2em;
+	margin: .8em 0;
 }
 #u2foot .warn {
-	font-size: 1.3em;
+	font-size: 1.2em;
 	padding: .5em .8em;
 	margin: 1em -.6em;
 	border-width: .1em 0;
@@ -2467,6 +2490,9 @@ html.b #u2conf a.b:hover {
 	font-size: .9em;
 	font-weight: normal;
 }
+#u2foot>*+* {
+	margin-top: 1.5em;
+}
 .prog {
 	font-family: 'scp', monospace, monospace;
 }

copyparty/web/browser.html

@@ -148,7 +148,8 @@
 			have_del = {{ have_del|tojson }},
 			have_unpost = {{ have_unpost|tojson }},
 			have_zip = {{ have_zip|tojson }},
-			turbolvl = {{ turbolvl|tojson }},
+			turbolvl = {{ turbolvl }},
+			u2sort = "{{ u2sort }}",
 			have_emp = {{ have_emp|tojson }},
 			txt_ext = "{{ txt_ext }}",
 			{% if no_prism %}no_prism = 1,{% endif %}

copyparty/web/browser.js

@@ -63,6 +63,7 @@ var Ls = {
 		"ul_par": "parallel uploads:",
 		"ut_mt": "continue hashing other files while uploading$N$Nmaybe disable if your CPU or HDD is a bottleneck",
 		"ut_ask": "ask for confirmation before upload starts",
+		"ut_pot": "improve upload speed on slow devices$Nby making the UI less complex",
 		"ut_srch": "don't actually upload, instead check if the files already $N exist on the server (will scan all folders you can read)",
 		"ut_par": "pause uploads by setting it to 0$N$Nincrease if your connection is slow / high latency$N$Nkeep it 1 on LAN or if the server HDD is a bottleneck",
 		"ul_btn": "drop files / folders<br>here (or click me)",
@@ -113,6 +114,8 @@ var Ls = {
 
 		"cut_flag": "ensure only one tab is uploading at a time $N -- other tabs must have this enabled too $N -- only affects tabs on the same domain",
 
+		"cut_az": "upload files in alphabetical order, rather than smallest-file-first$N$Nalphabetical order can make it easier to eyeball if something went wrong on the server, but it makes uploading slightly slower on fiber / LAN",
+
 		"cft_text": "favicon text (blank and refresh to disable)",
 		"cft_fg": "foreground color",
 		"cft_bg": "background color",
@@ -285,6 +288,9 @@ var Ls = {
 		"u_https2": "switch to https",
 		"u_https3": "for much better performance",
 		"u_ancient": 'your browser is impressively ancient -- maybe you should <a href="#" onclick="goto(\'bup\')">use bup instead</a>',
+		"u_enpot": 'switch to <a href="#">potato UI</a> (may improve upload speed)',
+		"u_depot": 'switch to <a href="#">fancy UI</a> (may reduce upload speed)',
+		"u_gotpot": 'switching to the potato UI for improved upload speed,\n\nfeel free to disagree and switch back!',
 		"u_ever": "this is the basic uploader; up2k needs at least<br>chrome 21 // firefox 13 // edge 12 // opera 12 // safari 5.1",
 		"u_su2k": 'this is the basic uploader; <a href="#" id="u2yea">up2k</a> is better',
 		"u_ewrite": 'you do not have write-access to this folder',
@@ -388,6 +394,7 @@ var Ls = {
 		"ul_par": "samtidige handl.:",
 		"ut_mt": "fortsett å befare køen mens opplastning foregår$N$Nskru denne av dersom du har en$Ntreg prosessor eller harddisk",
 		"ut_ask": "bekreft filutvalg før opplastning starter",
+		"ut_pot": "forbedre ytelsen på trege enheter ved å$Nforenkle brukergrensesnittet",
 		"ut_srch": "utfør søk istedenfor å laste opp --$Nleter igjennom alle mappene du har lov til å se",
 		"ut_par": "sett til 0 for å midlertidig stanse opplastning$N$Nhøye verdier (4 eller 8) kan gi bedre ytelse,$Nspesielt på trege internettlinjer$N$Nbør ikke være høyere enn 1 på LAN$Neller hvis serveren sin harddisk er treg",
 		"ul_btn": "slipp filer / mapper<br>her (eller klikk meg)",
@@ -438,6 +445,8 @@ var Ls = {
 
 		"cut_flag": "samkjører nettleserfaner slik at bare én $N kan holde på med befaring / opplastning $N -- andre faner må også ha denne skrudd på $N -- fungerer kun innenfor samme domene",
 
+		"cut_az": "last opp filer i alfabetisk rekkefølge, istedenfor minste-fil-først$N$Nalfabetisk kan gjøre det lettere å anslå om alt gikk bra, men er bittelitt tregere på fiber / LAN",
+
 		"cft_text": "ikontekst (blank ut og last siden på nytt for å deaktivere)",
 		"cft_fg": "farge",
 		"cft_bg": "bakgrunnsfarge",
@@ -610,6 +619,9 @@ var Ls = {
 		"u_https2": "bytte til https",
 		"u_https3": "for mye høyere hastighet",
 		"u_ancient": 'nettleseren din er prehistorisk -- mulig du burde <a href="#" onclick="goto(\'bup\')">bruke bup istedenfor</a>',
+		"u_enpot": 'bytt til <a href="#">enkelt UI</a> (gir sannsynlig raskere opplastning)',
+		"u_depot": 'bytt til <a href="#">snæsent UI</a> (gir sannsynlig tregere opplastning)',
+		"u_gotpot": 'byttet til et enklere UI for å laste opp raskere,\n\ndu kan gjerne bytte tilbake altså!',
 		"u_ever": "dette er den primitive opplasteren; up2k krever minst:<br>chrome 21 // firefox 13 // edge 12 // opera 12 // safari 5.1",
 		"u_su2k": 'dette er den primitive opplasteren; <a href="#" id="u2yea">up2k</a> er bedre',
 		"u_ewrite": 'du har ikke skrivetilgang i denne mappen',
@@ -717,6 +729,10 @@ ebi('op_up2k').innerHTML = (
 	'			<label for="multitask" tt="' + L.ut_mt + '">🏃</label>\n' +
 	'		</td>\n' +
 	'		<td class="c" rowspan="2">\n' +
+	'			<input type="checkbox" id="potato" />\n' +
+	'			<label for="potato" tt="' + L.ut_pot + '">🥔</label>\n' +
+	'		</td>\n' +
+	'		<td class="c" rowspan="2">\n' +
 	'			<input type="checkbox" id="ask_up" />\n' +
 	'			<label for="ask_up" tt="' + L.ut_ask + '">💭</label>\n' +
 	'		</td>\n' +
@@ -772,10 +788,10 @@ ebi('op_up2k').innerHTML = (
 	'		</tr>\n' +
 	'	</thead>\n' +
 	'	<tbody></tbody>\n' +
-	'</table></div>\n' +
+	'</table><div id="u2mu"></div></div>\n' +
 
 	'<p id="u2flagblock"><b>' + L.ul_flagblk + '</p>\n' +
-	'<p id="u2foot"></p>'
+	'<div id="u2foot"></div>'
 );
 
 
@@ -824,6 +840,7 @@ ebi('op_cfg').innerHTML = (
 	'		<a id="u2turbo" class="tgl btn ttb" href="#" tt="' + L.cut_turbo + '">turbo</a>\n' +
 	'		<a id="u2tdate" class="tgl btn ttb" href="#" tt="' + L.cut_datechk + '">date-chk</a>\n' +
 	'		<a id="flag_en" class="tgl btn" href="#" tt="' + L.cut_flag + '">💤</a>\n' +
+	'		<a id="u2sort" class="tgl btn" href="#" tt="' + L.cut_az + '">az</a>\n' +
 	'		</td>\n' +
 	'	</div>\n' +
 	'</div>\n' +
@@ -2194,6 +2211,12 @@ function play(tid, is_ev, seek) {
 	if (window.thegrid)
 		thegrid.loadsel();
 
+	try {
+		if (actx.state == 'suspended')
+			actx.resume();
+	}
+	catch (ex) { }
+
 	try {
 		mp.au.play();
 		if (mp.au.paused)
@@ -2404,8 +2427,7 @@ function eval_hash() {
 			};
 		})(a);
 
-	var d = mknod('div');
-	d.setAttribute('id', 'acc_info');
+	var d = mknod('div', 'acc_info');
 	document.body.insertBefore(d, ebi('ops'));
 })();
 
@@ -2670,8 +2692,7 @@ var fileman = (function () {
 
 		var rui = ebi('rui');
 		if (!rui) {
-			rui = mknod('div');
-			rui.setAttribute('id', 'rui');
+			rui = mknod('div', 'rui');
 			document.body.appendChild(rui);
 		}
 
@@ -3154,10 +3175,9 @@ var showfile = (function () {
 			return;
 
 		qsr('#prism_css');
-		var el = mknod('link');
+		var el = mknod('link', 'prism_css');
 		el.rel = 'stylesheet';
 		el.href = '/.cpr/deps/prism' + (light ? '' : 'd') + '.css';
-		el.setAttribute('id', 'prism_css');
 		document.head.appendChild(el);
 	};
 
@@ -3271,8 +3291,7 @@ var showfile = (function () {
 			fun = function (el) { };
 
 		qsr('#doc');
-		var el = mknod('pre');
-		el.setAttribute('id', 'doc');
+		var el = mknod('pre', 'doc');
 		el.setAttribute('tabindex', '0');
 		clmod(ebi('wrap'), 'doc', !is_md);
 		if (is_md) {
@@ -3300,9 +3319,8 @@ var showfile = (function () {
 		hfun(get_evpath() + '?doc=' + url.split('/').pop());
 
 		qsr('#docname');
-		el = mknod('span');
+		el = mknod('span', 'docname');
 		el.textContent = tname;
-		el.setAttribute('id', 'docname');
 		ebi('path').appendChild(el);
 
 		r.updtree();
@@ -3437,9 +3455,8 @@ var showfile = (function () {
 
 var thegrid = (function () {
 	var lfiles = ebi('files'),
-		gfiles = mknod('div');
+		gfiles = mknod('div', 'gfiles');
 
-	gfiles.setAttribute('id', 'gfiles');
 	gfiles.style.display = 'none';
 	gfiles.innerHTML = (
 		'<div id="ghead" class="ghead">' +

copyparty/web/dbg-audio.js

@@ -13,8 +13,7 @@ audio_eq.apply = function () {
 
     var can = ebi('fft_can');
     if (!can) {
-        can = mknod('canvas');
-        can.setAttribute('id', 'fft_can');
+        can = mknod('canvas', 'fft_can');
         can.style.cssText = 'position:absolute;left:0;bottom:5em;width:' + w + 'px;height:' + h + 'px;z-index:9001';
         document.body.appendChild(can);
         can.width = w;

copyparty/web/md.js

@@ -173,8 +173,7 @@ md_plug_err = function (ex, js) {
             o.textContent = lns[ln - 1];
         }
     }
-    var errbox = mknod('div');
-    errbox.setAttribute('id', 'md_errbox');
+    var errbox = mknod('div', 'md_errbox');
     errbox.style.cssText = 'position:absolute;top:0;left:0;padding:1em .5em;background:#2b2b2b;color:#fc5'
     errbox.textContent = msg;
     errbox.onclick = function () {

copyparty/web/md2.css

@@ -36,6 +36,11 @@
 	width: 55em;
 	width: min(55em, calc(100% - 2em));
 }
+#mtw.single.editor,
+#mw.single.editor {
+	width: calc(100% - 1em);
+	left: .5em;
+}
 
 
 #mp {

copyparty/web/md2.js

@@ -16,8 +16,7 @@ var dom_sbs = ebi('sbs');
 var dom_nsbs = ebi('nsbs');
 var dom_tbox = ebi('toolsbox');
 var dom_ref = (function () {
-    var d = mknod('div');
-    d.setAttribute('id', 'mtr');
+    var d = mknod('div', 'mtr');
     dom_swrap.appendChild(d);
     d = ebi('mtr');
     // hide behind the textarea (offsetTop is not computed if display:none)
@@ -509,6 +508,20 @@ function setsel(s) {
 }
 
 
+// cut/copy current line
+function md_cut(cut) {
+    var s = linebounds();
+    if (s.car != s.cdr)
+        return;
+
+    dom_src.setSelectionRange(s.n1, s.n2 + 1, 'forward');
+    setTimeout(function () {
+        var i = cut ? s.n1 : s.car;
+        dom_src.setSelectionRange(i, i, 'forward');
+    }, 1);
+}
+
+
 // indent/dedent
 function md_indent(dedent) {
     var s = getsel(),
@@ -955,6 +968,10 @@ var set_lno = (function () {
                 md_p_jump(dn);
                 return false;
             }
+            if (ev.code == "KeyX" || ev.code == "KeyC") {
+                md_cut(ev.code == "KeyX");
+                return true; //sic
+            }
         }
         else {
             if (ev.code == "Tab" || kc == 9) {

copyparty/web/up2k.js

@@ -15,6 +15,7 @@ function goto_up2k() {
 // chrome requires https to use crypto.subtle,
 // usually it's undefined but some chromes throw on invoke
 var up2k = null,
+    up2k_hooks = [],
     sha_js = window.WebAssembly ? 'hw' : 'ac',  // ff53,c57,sa11
     m = 'will use ' + sha_js + ' instead of native sha512 due to';
 
@@ -135,14 +136,19 @@ function up2k_flagbus() {
 }
 
 
-function U2pvis(act, btns, uc) {
+function U2pvis(act, btns, uc, st) {
     var r = this;
     r.act = act;
     r.ctr = { "ok": 0, "ng": 0, "bz": 0, "q": 0 };
     r.tab = [];
+    r.hq = {};
     r.head = 0;
     r.tail = -1;
     r.wsz = 3;
+    r.npotato = 99;
+    r.modn = 0;
+    r.modv = 0;
+    r.mod0 = null;
 
     var markup = {
         '404': '<span class="err">404</span>',
@@ -180,6 +186,9 @@ function U2pvis(act, btns, uc) {
     };
 
     r.is_act = function (card) {
+        if (uc.potato && !uc.fsearch)
+            return false;
+
         if (r.act == "done")
             return card == "ok" || card == "ng";
 
@@ -196,11 +205,14 @@ function U2pvis(act, btns, uc) {
         if (!r.is_act(fo.in))
             return;
 
-        var obj = ebi('f{0}{1}'.format(nfile, field.slice(1)));
+        var k = 'f{0}{1}'.format(nfile, field.slice(1)),
+            obj = ebi(k);
+
         obj.innerHTML = field == 'ht' ? (markup[html] || html) : html;
         if (field == 'hp') {
             obj.style.color = '';
             obj.style.background = '';
+            delete r.hq[nfile];
         }
     };
 
@@ -244,12 +256,9 @@ function U2pvis(act, btns, uc) {
         if (!r.is_act(fo.in))
             return;
 
-        var obj = ebi('f{0}p'.format(fobj.n)),
-            o1 = p[0] - 2, o2 = p[0] - 0.1, o3 = p[0];
+        var o1 = p[0] - 2, o2 = p[0] - 0.1, o3 = p[0];
 
-        obj.innerHTML = fo.hp;
-        obj.style.color = '#fff';
-        obj.style.background = 'linear-gradient(90deg, #025, #06a ' + o1 + '%, #09d ' + o2 + '%, #222 ' + o3 + '%, #222 99%, #555)';
+        r.hq[fobj.n] = [fo.hp, '#fff', 'linear-gradient(90deg, #025, #06a ' + o1 + '%, #09d ' + o2 + '%, #222 ' + o3 + '%, #222 99%, #555)'];
     };
 
     r.prog = function (fobj, nchunk, cbd) {
@@ -304,9 +313,7 @@ function U2pvis(act, btns, uc) {
             throw new Error('see console');
         }
 
-        obj.innerHTML = fo.hp;
-        obj.style.color = '#fff';
-        obj.style.background = 'linear-gradient(90deg, #050, #270 ' + o1 + '%, #4b0 ' + o2 + '%, #222 ' + o3 + '%, #222 99%, #555)';
+        r.hq[fobj.n] = [fo.hp, '#fff', 'linear-gradient(90deg, #050, #270 ' + o1 + '%, #4b0 ' + o2 + '%, #222 ' + o3 + '%, #222 99%, #555)'];
     };
 
     r.move = function (nfile, newcat) {
@@ -320,6 +327,10 @@ function U2pvis(act, btns, uc) {
         fo.in = newcat;
         r.ctr[oldcat]--;
         r.ctr[newcat]++;
+
+        while (st.car < r.tab.length && has(['ok', 'ng'], r.tab[st.car].in))
+            st.car++;
+
         r.drawcard(oldcat);
         r.drawcard(newcat);
         if (r.is_act(newcat)) {
@@ -342,7 +353,10 @@ function U2pvis(act, btns, uc) {
     };
 
     r.bzw = function () {
-        var first = QS('#u2tab>tbody>tr:first-child');
+        var mod = 0,
+            t0 = Date.now(),
+            first = QS('#u2tab>tbody>tr:first-child');
+
         if (!first)
             return;
 
@@ -352,14 +366,131 @@ function U2pvis(act, btns, uc) {
 
         while (r.head - first > r.wsz) {
             qsr('#f' + (first++));
+            mod++;
         }
-        while (last - r.tail < r.wsz && last < r.tab.length - 2) {
+        while (last - r.tail < r.wsz && last < r.tab.length - 1) {
             var obj = ebi('f' + (++last));
-            if (!obj)
+            if (!obj) {
                 r.addrow(last);
+                mod++;
+            }
+        }
+        if (mod && r.modn < 200 && ebi('repl').offsetTop) {
+            if (++r.modn >= 10) {
+                if (r.modn == 10)
+                    r.mod0 = Date.now();
+
+                r.modv += Date.now() - t0;
+            }
+
+            if (r.modn >= 200) {
+                var n = r.modn - 10,
+                    ipu = r.modv / n,
+                    spu = (Date.now() - r.mod0) / n,
+                    ir = spu / ipu;
+
+                console.log('bzw:', f2f(ipu, 2), ' spu:', f2f(spu, 2), ' ir:', f2f(ir, 2), ' tab:', r.tab.length);
+                // efficiency estimates;
+                // ir: 5=16% 4=50%,30% 27=100%
+                // ipu: 2.7=16% 2=30% 1.6=50% 1.8=100% (ng for big files)
+                if (ipu >= 1.5 && ir <= 9 && r.tab.length >= 1000 && r.tab[Math.floor(r.tab.length / 3)].bt <= 1024 * 1024 * 4)
+                    r.go_potato();
+            }
         }
     };
 
+    r.potatolabels = function () {
+        var ode = ebi('u2depotato'),
+            oen = ebi('u2enpotato');
+
+        if (!ode)
+            return;
+
+        ode.style.display = uc.potato ? '' : 'none';
+        oen.style.display = uc.potato ? 'none' : '';
+    }
+
+    r.potato = function () {
+        ebi('u2tabw').style.minHeight = '';
+        QS('#u2cards a[act="bz"]').click();
+        timer[uc.potato ? "add" : "rm"](draw_potato);
+        timer[uc.potato ? "rm" : "add"](apply_html);
+        r.potatolabels();
+    };
+
+    r.go_potato = function () {
+        r.go_potato = noop;
+        var ode = mknod('div', 'u2depotato'),
+            oen = mknod('div', 'u2enpotato'),
+            u2f = ebi('u2foot'),
+            btn = ebi('potato');
+
+        ode.innerHTML = L.u_depot;
+        oen.innerHTML = L.u_enpot;
+
+        if (sread('potato') === null) {
+            btn.click();
+            toast.inf(30, L.u_gotpot);
+            localStorage.removeItem('potato');
+        }
+
+        u2f.appendChild(ode);
+        u2f.appendChild(oen);
+        ode.onclick = oen.onclick = btn.onclick;
+        r.potatolabels();
+    };
+
+    function draw_potato() {
+        if (++r.npotato < 2)
+            return;
+
+        r.npotato = 0;
+        var html = [
+            "<p>files: &nbsp; <b>{0}</b> finished, &nbsp; <b>{1}</b> failed, &nbsp; <b>{2}</b> busy, &nbsp; <b>{3}</b> queued</p>".format(r.ctr.ok, r.ctr.ng, r.ctr.bz, r.ctr.q),
+        ];
+
+        while (r.head < r.tab.length && has(["ok", "ng"], r.tab[r.head].in))
+            r.head++;
+
+        var act = null;
+        if (r.head < r.tab.length)
+            act = r.tab[r.head];
+
+        if (act)
+            html.push("<p>file {0} of {1} : &nbsp; {2} &nbsp; <code>{3}</code></p>\n<div>{4}</div>".format(r.head + 1, r.tab.length, act.ht, act.hp, act.hn));
+
+        html = html.join('\n');
+        if (r.hpotato == html)
+            return;
+
+        r.hpotato = html;
+        ebi('u2mu').innerHTML = html;
+    }
+
+    function apply_html() {
+        var oq = {}, n = 0;
+        for (var k in r.hq) {
+            var o = ebi('f{0}p'.format(k));
+            if (!o)
+                continue;
+
+            oq[k] = o;
+            n++;
+        }
+        if (!n)
+            return;
+
+        for (var k in oq) {
+            var o = oq[k],
+                v = r.hq[k];
+
+            o.innerHTML = v[0];
+            o.style.color = v[1];
+            o.style.background = v[2];
+        }
+        r.hq = {};
+    }
+
     r.drawcard = function (cat) {
         var cards = QSA('#u2cards>a>span');
 
@@ -441,8 +572,7 @@ function U2pvis(act, btns, uc) {
         if (as_html)
             return '<tr id="f' + nfile + '">' + ret + '</tr>';
 
-        var obj = mknod('tr');
-        obj.setAttribute('id', 'f' + nfile);
+        var obj = mknod('tr', 'f' + nfile);
         obj.innerHTML = ret;
         return obj;
     };
@@ -476,6 +606,7 @@ function U2pvis(act, btns, uc) {
     }
 
     r.changecard(r.act);
+    r.potato();
 }
 
 
@@ -527,7 +658,7 @@ function Donut(uc, st) {
     }
 
     r.on = function (ya) {
-        r.fc = r.tc = 99;
+        r.fc = r.tc = r.dc = 99;
         r.eta = null;
         r.base = pos();
         optab.innerHTML = ya ? svg() : optab.getAttribute('ico');
@@ -543,10 +674,16 @@ function Donut(uc, st) {
 
         var t = st.bytes.total - r.base,
             v = pos() - r.base,
-            ofs = el.style.strokeDashoffset = o - o * v / t;
+            ofs = o - o * v / t;
+
+        if (!uc.potato || ++r.dc >= 4) {
+            el.style.strokeDashoffset = ofs;
+            r.dc = 0;
+        }
 
         if (++r.tc >= 10) {
-            wintitle(f2f(v * 100 / t, 1) + '%, ' + r.eta + 's, ', true);
+            wintitle("{0}%, {1}s, #{2}, ".format(
+                f2f(v * 100 / t, 1), r.eta, st.files.length - st.nfile.upload), true);
             r.tc = 0;
         }
 
@@ -578,6 +715,12 @@ function fsearch_explain(n) {
 
 
 function up2k_init(subtle) {
+    var r = {
+        "init_deps": init_deps,
+        "set_fsearch": set_fsearch,
+        "gotallfiles": [gotallfiles]  // hooks
+    };
+
     function showmodal(msg) {
         ebi('u2notbtn').innerHTML = msg;
         ebi('u2btn').style.display = 'none';
@@ -612,7 +755,10 @@ function up2k_init(subtle) {
                 m = L.u_ancient;
                 setmsg('');
             }
-            ebi('u2foot').innerHTML = '<big>' + m + '</big>';
+            qsr('#u2depmsg');
+            var o = mknod('div', 'u2depmsg');
+            o.innerHTML = m;
+            ebi('u2foot').appendChild(o);
         }
         loading_deps = true;
     }
@@ -647,17 +793,24 @@ function up2k_init(subtle) {
     var parallel_uploads = icfg_get('nthread'),
         uc = {},
         fdom_ctr = 0,
-        min_filebuf = 0;
+        min_filebuf = 0,
+        biggest_file = 0;
 
     bcfg_bind(uc, 'multitask', 'multitask', true, null, false);
+    bcfg_bind(uc, 'potato', 'potato', false, set_potato, false);
     bcfg_bind(uc, 'ask_up', 'ask_up', true, null, false);
     bcfg_bind(uc, 'flag_en', 'flag_en', false, apply_flag_cfg);
     bcfg_bind(uc, 'fsearch', 'fsearch', false, set_fsearch, false);
     bcfg_bind(uc, 'turbo', 'u2turbo', turbolvl > 1, draw_turbo, false);
     bcfg_bind(uc, 'datechk', 'u2tdate', turbolvl < 3, null, false);
+    bcfg_bind(uc, 'az', 'u2sort', u2sort.indexOf('n') + 1, set_u2sort, false);
 
     var st = {
         "files": [],
+        "nfile": {
+            "hash": 0,
+            "upload": 0
+        },
         "seen": {},
         "todo": {
             "head": [],
@@ -681,7 +834,11 @@ function up2k_init(subtle) {
             "hashing": 0,
             "uploading": 0,
             "busy": 0
-        }
+        },
+        "car": 0,
+        "modn": 0,
+        "modv": 0,
+        "mod0": null
     };
 
     function push_t(arr, t) {
@@ -693,9 +850,13 @@ function up2k_init(subtle) {
             });
     }
 
-    var pvis = new U2pvis("bz", '#u2cards', uc),
+    var pvis = new U2pvis("bz", '#u2cards', uc, st),
         donut = new Donut(uc, st);
 
+    r.ui = pvis;
+    r.st = st;
+    r.uc = uc;
+
     var bobslice = null;
     if (window.File)
         bobslice = File.prototype.slice || File.prototype.mozSlice || File.prototype.webkitSlice;
@@ -808,6 +969,8 @@ function up2k_init(subtle) {
         if (err)
             return modal.alert('sorry, ' + err);
 
+        toast.inf(0, 'Scanning files...');
+
         if ((dz == 'up_dz' && uc.fsearch) || (dz == 'srch_dz' && !uc.fsearch))
             tgl_fsearch();
 
@@ -906,7 +1069,8 @@ function up2k_init(subtle) {
 
         if (!dirs.length) {
             if (!pf.length)
-                return gotallfiles(good, nil, bad);
+                // call first hook, pass list of remaining hooks to call
+                return r.gotallfiles[0](good, nil, bad, r.gotallfiles.slice(1));
 
             console.log("retry pf, " + pf.length);
             setTimeout(function () {
@@ -954,6 +1118,12 @@ function up2k_init(subtle) {
     }
 
     function gotallfiles(good_files, nil_files, bad_files) {
+        if (toast.txt == 'Scanning files...')
+            toast.hide();
+
+        if (uc.fsearch && !uc.turbo)
+            nil_files = [];
+
         var ntot = good_files.concat(nil_files, bad_files).length;
         if (bad_files.length) {
             var msg = L.u_badf.format(bad_files.length, ntot);
@@ -1002,6 +1172,13 @@ function up2k_init(subtle) {
         var evpath = get_evpath(),
             draw_each = good_files.length < 50;
 
+        if (!uc.az)
+            good_files.sort(function (a, b) {
+                a = a[0].size;
+                b = b[0].size;
+                return a < b ? -1 : a > b ? 1 : 0;
+            });
+
         for (var a = 0; a < good_files.length; a++) {
             var fobj = good_files[a][0],
                 name = good_files[a][1],
@@ -1032,6 +1209,9 @@ function up2k_init(subtle) {
             if (uc.fsearch)
                 entry.srch = 1;
 
+            if (biggest_file < entry.size)
+                biggest_file = entry.size;
+
             try {
                 if (st.seen[fdir][key])
                     continue;
@@ -1047,9 +1227,9 @@ function up2k_init(subtle) {
                     entry.purl + uricom_enc(entry.name)).join(' '),
                 '📐 ' + L.u_hashing,
                 ''
-            ], fobj.size, draw_each);
+            ], entry.size, draw_each);
 
-            st.bytes.total += fobj.size;
+            st.bytes.total += entry.size;
             st.files.push(entry);
             if (!entry.size)
                 push_t(st.todo.handshake, entry);
@@ -1074,7 +1254,7 @@ function up2k_init(subtle) {
     }
     more_one_file();
 
-    var etaref = 0, etaskip = 0, utw_minh = 0;
+    var etaref = 0, etaskip = 0, utw_minh = 0, utw_read = 0;
     function etafun() {
         var nhash = st.busy.head.length + st.busy.hash.length + st.todo.head.length + st.todo.hash.length,
             nsend = st.busy.upload.length + st.todo.upload.length,
@@ -1087,6 +1267,11 @@ function up2k_init(subtle) {
 
         //ebi('acc_info').innerHTML = humantime(st.time.busy) + ' ' + f2f(now / 1000, 1);
 
+        if (++utw_read >= 20) {
+            utw_read = 0;
+            utw_minh = parseInt(ebi('u2tabw').style.minHeight || '0');
+        }
+
         var minh = QS('#op_up2k.act') && st.is_busy ? Math.max(utw_minh, ebi('u2tab').offsetHeight + 32) : 0;
         if (utw_minh < minh || !utw_minh) {
             utw_minh = minh;
@@ -1094,11 +1279,11 @@ function up2k_init(subtle) {
         }
 
         if (!nhash)
-            ebi('u2etah').innerHTML = L.u_etadone.format(humansize(st.bytes.hashed), pvis.ctr["ok"] + pvis.ctr["ng"]);
+            ebi('u2etah').innerHTML = L.u_etadone.format(humansize(st.bytes.hashed), pvis.ctr.ok + pvis.ctr.ng);
 
         if (!nsend && !nhash)
             ebi('u2etau').innerHTML = ebi('u2etat').innerHTML = (
-                L.u_etadone.format(humansize(st.bytes.uploaded), pvis.ctr["ok"] + pvis.ctr["ng"]));
+                L.u_etadone.format(humansize(st.bytes.uploaded), pvis.ctr.ok + pvis.ctr.ng));
 
         if (!st.busy.hash.length && !hashing_permitted())
             nhash = 0;
@@ -1184,9 +1369,14 @@ function up2k_init(subtle) {
             return false;
 
         if (uc.multitask) {
-            var ahead = st.bytes.hashed - st.bytes.finished;
-            return ahead < 1024 * 1024 * 1024 * 4 &&
-                st.todo.handshake.length + st.busy.handshake.length < 16;
+            if (!uc.az)
+                return st.todo.handshake.length + st.busy.handshake.length < 2;
+
+            var ahead = st.bytes.hashed - st.bytes.finished,
+                nmax = ahead < biggest_file / 8 ? 32 : 16;
+
+            return ahead < biggest_file &&
+                st.todo.handshake.length + st.busy.handshake.length < nmax;
         }
         return handshakes_permitted() && 0 ==
             st.todo.handshake.length +
@@ -1213,16 +1403,7 @@ function up2k_init(subtle) {
             running = true;
             while (true) {
                 var now = Date.now(),
-                    oldest_active = Math.min(  // gzip take the wheel
-                        st.todo.head.length ? st.todo.head[0].n : st.files.length,
-                        st.todo.hash.length ? st.todo.hash[0].n : st.files.length,
-                        st.todo.upload.length ? st.todo.upload[0].nfile : st.files.length,
-                        st.todo.handshake.length ? st.todo.handshake[0].n : st.files.length,
-                        st.busy.head.length ? st.busy.head[0].n : st.files.length,
-                        st.busy.hash.length ? st.busy.hash[0].n : st.files.length,
-                        st.busy.upload.length ? st.busy.upload[0].nfile : st.files.length,
-                        st.busy.handshake.length ? st.busy.handshake[0].n : st.files.length),
-                    is_busy = oldest_active < st.files.length;
+                    is_busy = st.car < st.files.length;
 
                 if (was_busy && !is_busy) {
                     for (var a = 0; a < st.files.length; a++) {
@@ -1250,25 +1431,7 @@ function up2k_init(subtle) {
                     donut.on(is_busy);
 
                     if (!is_busy) {
-                        var sr = uc.fsearch,
-                            ok = pvis.ctr["ok"],
-                            ng = pvis.ctr["ng"],
-                            t = uc.ask_up ? 0 : 10;
-
-                        if (ok && ng)
-                            toast.warn(t, (sr ? L.ur_sm : L.ur_um).format(ok, ng));
-                        else if (ok > 1)
-                            toast.ok(t, (sr ? L.ur_aso : L.ur_auo).format(ok));
-                        else if (ok)
-                            toast.ok(t, sr ? L.ur_1so : L.ur_1uo);
-                        else if (ng > 1)
-                            toast.err(t, (sr ? L.ur_asn : L.ur_aun).format(ng));
-                        else if (ng)
-                            toast.err(t, sr ? L.ur_1sn : L.ur_1un);
-
-                        timer.rm(etafun);
-                        timer.rm(donut.do);
-                        utw_minh = 0;
+                        uptoast();
                     }
                     else {
                         timer.add(donut.do);
@@ -1321,7 +1484,7 @@ function up2k_init(subtle) {
 
                 if (st.todo.head.length &&
                     st.busy.head.length < parallel_uploads &&
-                    (!is_busy || st.todo.head[0].n - oldest_active < parallel_uploads * 2)) {
+                    (!is_busy || st.todo.head[0].n - st.car < parallel_uploads * 2)) {
                     exec_head();
                     mou_ikkai = true;
                 }
@@ -1346,6 +1509,31 @@ function up2k_init(subtle) {
                     mou_ikkai = true;
                 }
 
+                if (is_busy && st.modn < 100) {
+                    var t0 = Date.now() + (ebi('repl').offsetTop ? 0 : 0);
+
+                    if (++st.modn >= 10) {
+                        if (st.modn == 10)
+                            st.mod0 = Date.now();
+
+                        st.modv += Date.now() - t0;
+                    }
+
+                    if (st.modn >= 100) {
+                        var n = st.modn - 10,
+                            ipu = st.modv / n,
+                            spu = (Date.now() - st.mod0) / n,
+                            ir = spu / ipu;
+
+                        console.log('tsk:', f2f(ipu, 2), ' spu:', f2f(spu, 2), ' ir:', f2f(ir, 2));
+                        // efficiency estimates;
+                        // ir: 8=16% 11=60% 16=90% 24=100%
+                        // ipu: 1=40% .8=60% .3=100%
+                        if (ipu >= 0.5 && ir <= 15)
+                            pvis.go_potato();
+                    }
+                }
+
                 if (!mou_ikkai || crashed)
                     return defer();
             }
@@ -1354,6 +1542,30 @@ function up2k_init(subtle) {
         return taskerd;
     })();
 
+    function uptoast() {
+        var sr = uc.fsearch,
+            ok = pvis.ctr.ok,
+            ng = pvis.ctr.ng,
+            t = uc.ask_up ? 0 : 10;
+
+        console.log('toast', ok, ng);
+
+        if (ok && ng)
+            toast.warn(t, (sr ? L.ur_sm : L.ur_um).format(ok, ng));
+        else if (ok > 1)
+            toast.ok(t, (sr ? L.ur_aso : L.ur_auo).format(ok));
+        else if (ok)
+            toast.ok(t, sr ? L.ur_1so : L.ur_1uo);
+        else if (ng > 1)
+            toast.err(t, (sr ? L.ur_asn : L.ur_aun).format(ng));
+        else if (ng)
+            toast.err(t, sr ? L.ur_1sn : L.ur_1un);
+
+        timer.rm(etafun);
+        timer.rm(donut.do);
+        utw_minh = 0;
+    }
+
     function chill(t) {
         var now = Date.now();
         if ((t.coolmul || 0) < 2 || now - t.cooldown < t.coolmul * 700)
@@ -1429,6 +1641,7 @@ function up2k_init(subtle) {
     function exec_hash() {
         var t = st.todo.hash.shift();
         st.busy.hash.push(t);
+        st.nfile.hash = t.n;
 
         var bpend = 0,
             nchunk = 0,
@@ -1466,7 +1679,7 @@ function up2k_init(subtle) {
                 hash_calc(nch, e.target.result);
             }
             reader.onload = function (e) {
-                try { orz(e); } catch (ex) { vis_exh(ex + '', '', '', '', ex); }
+                try { orz(e); } catch (ex) { vis_exh(ex + '', 'up2k.js', '', '', ex); }
             };
             reader.onerror = function () {
                 var err = reader.error + '';
@@ -1499,7 +1712,7 @@ function up2k_init(subtle) {
         var hash_calc = function (nch, buf) {
             while (segm_next());
 
-            var hash_done = function (hashbuf) {
+            var orz = function (hashbuf) {
                 var hslice = new Uint8Array(hashbuf).subarray(0, 33),
                     b64str = buf2b64(hslice);
 
@@ -1525,6 +1738,10 @@ function up2k_init(subtle) {
                 tasker();
             };
 
+            var hash_done = function (hashbuf) {
+                try { orz(hashbuf); } catch (ex) { vis_exh(ex + '', 'up2k.js', '', '', ex); }
+            };
+
             if (subtle)
                 subtle.digest('SHA-512', buf).then(hash_done);
             else setTimeout(function () {
@@ -1592,7 +1809,7 @@ function up2k_init(subtle) {
             tasker();
         };
         xhr.onload = function (e) {
-            try { orz(e); } catch (ex) { vis_exh(ex + '', '', '', '', ex); }
+            try { orz(e); } catch (ex) { vis_exh(ex + '', 'up2k.js', '', '', ex); }
         };
 
         xhr.open('HEAD', t.purl + uricom_enc(t.name) + '?raw', true);
@@ -1748,6 +1965,8 @@ function up2k_init(subtle) {
                         f2f(spd1, 2), isNaN(spd2) ? '--' : f2f(spd2, 2)));
 
                     pvis.move(t.n, 'ok');
+                    if (!pvis.ctr.bz && !pvis.ctr.q)
+                        uptoast();
                 }
                 else {
                     if (t.t_uploaded)
@@ -1806,7 +2025,7 @@ function up2k_init(subtle) {
             }
         }
         xhr.onload = function (e) {
-            try { orz(e); } catch (ex) { vis_exh(ex + '', '', '', '', ex); }
+            try { orz(e); } catch (ex) { vis_exh(ex + '', 'up2k.js', '', '', ex); }
         };
 
         var req = {
@@ -1845,6 +2064,7 @@ function up2k_init(subtle) {
     function exec_upload() {
         var upt = st.todo.upload.shift();
         st.busy.upload.push(upt);
+        st.nfile.upload = upt.nfile;
 
         var npart = upt.npart,
             t = st.files[upt.nfile],
@@ -1892,12 +2112,15 @@ function up2k_init(subtle) {
             tasker();
         }
         function do_send() {
-            var xhr = new XMLHttpRequest();
+            var xhr = new XMLHttpRequest(),
+                bfin = Math.floor(st.bytes.finished / 1024 / 1024),
+                btot = Math.floor(st.bytes.total / 1024 / 1024);
+
             xhr.upload.onprogress = function (xev) {
                 pvis.prog(t, npart, xev.loaded);
             };
             xhr.onload = function (xev) {
-                try { orz(xhr); } catch (ex) { vis_exh(ex + '', '', '', '', ex); }
+                try { orz(xhr); } catch (ex) { vis_exh(ex + '', 'up2k.js', '', '', ex); }
             };
             xhr.onerror = function (xev) {
                 if (crashed)
@@ -1912,6 +2135,8 @@ function up2k_init(subtle) {
             xhr.open('POST', t.purl, true);
             xhr.setRequestHeader("X-Up2k-Hash", t.hash[npart]);
             xhr.setRequestHeader("X-Up2k-Wark", t.wark);
+            xhr.setRequestHeader("X-Up2k-Stat", "{0}/{1}/{2}/{3} {4}/{5}".format(
+                pvis.ctr.ok, pvis.ctr.ng, pvis.ctr.bz, pvis.ctr.q, btot, btot - bfin));
             xhr.setRequestHeader('Content-Type', 'application/octet-stream');
             if (xhr.overrideMimeType)
                 xhr.overrideMimeType('Content-Type', 'application/octet-stream');
@@ -1943,7 +2168,7 @@ function up2k_init(subtle) {
             ebi('u2conf').className = ebi('u2cards').className = ebi('u2etaw').className = wide;
         }
 
-        wide = write && wem > 78 ? 'ww' : wide;
+        wide = write && wem > 82 ? 'ww' : wide;
         parent = ebi(wide == 'ww' && write ? 'u2c3w' : 'u2c3t');
         var its = [ebi('u2etaw'), ebi('u2cards')];
         if (its[0].parentNode !== parent) {
@@ -2017,24 +2242,27 @@ function up2k_init(subtle) {
     }
 
     function draw_turbo() {
-        var msg = uc.fsearch ? L.u_ts : L.u_tu,
-            omsg = uc.fsearch ? L.u_tu : L.u_ts,
-            html = ebi('u2foot').innerHTML,
-            ohtml = html;
+        var msg = (turbolvl || !uc.turbo) ? null : uc.fsearch ? L.u_ts : L.u_tu,
+            html = ebi('u2foot').innerHTML;
 
-        if (turbolvl || !uc.turbo)
-            msg = null;
+        if (msg && html.indexOf(msg) + 1)
+            return;
 
-        if (msg && html.indexOf(msg) === -1)
-            html = html.replace(omsg, '') + msg;
-        else if (!msg)
-            html = html.replace(L.u_tu, '').replace(L.u_ts, '');
+        qsr('#u2turbomsg');
+        if (!msg)
+            return;
 
-        if (html !== ohtml)
-            ebi('u2foot').innerHTML = html;
+        var o = mknod('div', 'u2turbomsg');
+        o.innerHTML = msg;
+        ebi('u2foot').appendChild(o);
     }
     draw_turbo();
 
+    function set_potato() {
+        pvis.potato();
+        set_fsearch();
+    }
+
     function set_fsearch(new_state) {
         var fixed = false;
 
@@ -2073,6 +2301,10 @@ function up2k_init(subtle) {
 
         ebi('u2tab').className = (uc.fsearch ? 'srch ' : 'up ') + pvis.act;
 
+        var potato = uc.potato && !uc.fsearch;
+        ebi('u2cards').style.display = ebi('u2tab').style.display = potato ? 'none' : '';
+        ebi('u2mu').style.display = potato ? '' : 'none';
+
         draw_turbo();
         onresize();
     }
@@ -2096,9 +2328,12 @@ function up2k_init(subtle) {
         }
     }
 
-    function nop(e) {
-        ev(e);
-        this.click();
+    function set_u2sort() {
+        if (u2sort.indexOf('f') < 0)
+            return;
+
+        bcfg_set('u2sort', uc.az = u2sort.indexOf('n') + 1);
+        localStorage.removeItem('u2sort');
     }
 
     ebi('nthread_add').onclick = function (e) {
@@ -2123,7 +2358,12 @@ function up2k_init(subtle) {
     if (parallel_uploads < 1)
         bumpthread(1);
 
-    return { "init_deps": init_deps, "set_fsearch": set_fsearch, "ui": pvis, "st": st, "uc": uc }
+    setTimeout(function () {
+        for (var a = 0; a < up2k_hooks.length; a++)
+            up2k_hooks[a]();
+    }, 1);
+
+    return r;
 }
 
 

copyparty/web/util.js

@@ -16,10 +16,18 @@ var wah = '',
 var ebi = document.getElementById.bind(document),
     QS = document.querySelector.bind(document),
     QSA = document.querySelectorAll.bind(document),
-    mknod = document.createElement.bind(document),
     XHR = XMLHttpRequest;
 
 
+function mknod(et, eid) {
+    var ret = document.createElement(et);
+    if (eid)
+        ret.id = eid;
+
+    return ret;
+}
+
+
 function qsr(sel) {
     var el = QS(sel);
     if (el)
@@ -160,8 +168,7 @@ function vis_exh(msg, url, lineNo, columnNo, error) {
     try {
         var exbox = ebi('exbox');
         if (!exbox) {
-            exbox = mknod('div');
-            exbox.setAttribute('id', 'exbox');
+            exbox = mknod('div', 'exbox');
             document.body.appendChild(exbox);
 
             var s = mknod('style');
@@ -222,6 +229,11 @@ function ev(e) {
 }
 
 
+function noope(e) {
+    ev(e);
+}
+
+
 // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/endsWith
 if (!String.prototype.endsWith)
     String.prototype.endsWith = function (search, this_len) {
@@ -866,8 +878,8 @@ var timer = (function () {
 
 var tt = (function () {
     var r = {
-        "tt": mknod("div"),
-        "th": mknod("div"),
+        "tt": mknod("div", 'tt'),
+        "th": mknod("div", 'tth'),
         "en": true,
         "el": null,
         "skip": false,
@@ -875,8 +887,6 @@ var tt = (function () {
     };
 
     r.th.innerHTML = '?';
-    r.tt.setAttribute('id', 'tt');
-    r.th.setAttribute('id', 'tth');
     document.body.appendChild(r.tt);
     document.body.appendChild(r.th);
 
@@ -1045,9 +1055,8 @@ var toast = (function () {
     var r = {},
         te = null,
         scrolling = false,
-        obj = mknod('div');
+        obj = mknod('div', 'toast');
 
-    obj.setAttribute('id', 'toast');
     document.body.appendChild(obj);
     r.visible = false;
     r.txt = null;
@@ -1130,8 +1139,7 @@ var modal = (function () {
     r.busy = false;
 
     r.show = function (html) {
-        o = mknod('div');
-        o.setAttribute('id', 'modal');
+        o = mknod('div', 'modal');
         o.innerHTML = '<table><tr><td><div id="modalc">' + html + '</div></td></tr></table>';
         document.body.appendChild(o);
         document.addEventListener('keydown', onkey);
@@ -1186,7 +1194,8 @@ var modal = (function () {
             return;
 
         setTimeout(function () {
-            ebi('modal-ok').focus();
+            if (ctr = ebi('modal-ok'))
+                ctr.focus();
         }, 20);
         ev(e);
     }
@@ -1496,9 +1505,8 @@ function xhrchk(xhr, prefix, e404) {
         fun = toast.warn;
 
         qsr('#cf_frame');
-        var fr = mknod('iframe');
+        var fr = mknod('iframe', 'cf_frame');
         fr.src = '/?cf_challenge';
-        fr.setAttribute('id', 'cf_frame');
         document.body.appendChild(fr);
     }
 

docs/notes.sh

@@ -48,7 +48,10 @@ avg() { awk 'function pr(ncsz) {if (nsmp>0) {printf "%3s %s\n", csz, sum/nsmp} c
 ## time between first and last upload
 
 python3 -um copyparty -nw -v srv::rw -i 127.0.0.1 2>&1 | tee log 
-cat log | awk '!/"purl"/{next} {s=$1;sub(/[^m]+m/,"");gsub(/:/," ");t=60*(60*$1+$2)+$3} !a{a=t;sa=s} {b=t;sb=s} END {print b-a,sa,sb}'
+cat log | awk '!/"purl"/{next} {s=$1;sub(/[^m]+m/,"");gsub(/:/," ");t=60*(60*$1+$2)+$3} t<p{t+=86400} !a{a=t;sa=s} {b=t;sb=s} END {print b-a,sa,sb}'
+
+# or if the client youre measuring dies for ~15sec every once ina while and you wanna filter those out,
+cat log | awk '!/"purl"/{next} {s=$1;sub(/[^m]+m/,"");gsub(/:/," ");t=60*(60*$1+$2)+$3} t<p{t+=86400} !p{a=t;p=t;r=0;next} t-p>1{printf "%.3f += %.3f - %.3f  (%.3f)  # %.3f -> %.3f\n",r,p,a,p-a,p,t;r+=p-a;a=t} {p=t} END {print r+p-a}'
 
 
 ##

scripts/make-sfx.sh

@@ -106,7 +106,7 @@ tmpdir="$(
 [ $repack ] && {
 	old="$tmpdir/pe-copyparty"
 	echo "repack of files in $old"
-	cp -pR "$old/"*{j2,ftp,copyparty} .
+	cp -pR "$old/"*{py2,j2,ftp,copyparty} .
 }
 
 [ $repack ] || {

scripts/rls.sh

@@ -22,13 +22,16 @@ v=$1
 }
 
 rm -f ../dist/copyparty-sfx.*
+shift
+./make-sfx.sh "$@"
 f=../dist/copyparty-sfx.py
-./make-sfx.sh
-$f -h
+[ -e $f ] || 
+    f=../dist/copyparty-sfx-gz.py
 
+$f -h
 while true; do
     mv $f $f.$(wc -c <$f | awk '{print$1}')
-    ./make-sfx.sh re $ar
+    ./make-sfx.sh re "$@"
 done
 
 # git tag -d v$v; git push --delete origin v$v

scripts/sfx.py

@@ -224,11 +224,15 @@ def unpack():
     """unpacks the tar yielded by `data`"""
     name = "pe-copyparty"
     tag = "v" + str(STAMP)
-    withpid = "{}.{}".format(name, os.getpid())
     top = tempfile.gettempdir()
     opj = os.path.join
     final = opj(top, name)
-    mine = opj(top, withpid)
+    for suf in range(0, 9001):
+        withpid = "{}.{}.{}".format(name, os.getpid(), suf)
+        mine = opj(top, withpid)
+        if not os.path.exists(mine):
+            break
+
     tar = opj(mine, "tar")
 
     try:
@@ -360,11 +364,12 @@ def utime(top):
 def confirm(rv):
     msg()
     msg("retcode", rv if rv else traceback.format_exc())
-    msg("*** hit enter to exit ***")
-    try:
-        raw_input() if PY2 else input()
-    except:
-        pass
+    if WINDOWS:
+        msg("*** hit enter to exit ***")
+        try:
+            raw_input() if PY2 else input()
+        except:
+            pass
 
     sys.exit(rv or 1)
 

tests/test_httpcli.py

@@ -24,51 +24,34 @@ def hdr(query):
 
 class Cfg(Namespace):
     def __init__(self, a=None, v=None, c=None):
+        ka = {}
+
+        ex = "e2d e2ds e2dsa e2t e2ts e2tsr ed emp force_js ihead no_acode no_athumb no_del no_logues no_mv no_readme no_robots no_scandir no_thumb no_vthumb no_zip nw"
+        ka.update(**{k: False for k in ex.split()})
+
+        ex = "nih no_rescan no_sendfile no_voldump"
+        ka.update(**{k: True for k in ex.split()})
+
+        ex = "css_browser hist js_browser no_hash no_idx"
+        ka.update(**{k: None for k in ex.split()})
+
+        ex = "re_maxage rproxy rsp_slp s_wr_slp theme themes turbo"
+        ka.update(**{k: 0 for k in ex.split()})
+
+        ex = "doctitle favico html_head mth textfiles"
+        ka.update(**{k: "" for k in ex.split()})
+
         super(Cfg, self).__init__(
             a=a or [],
             v=v or [],
             c=c,
-            rproxy=0,
-            rsp_slp=0,
-            s_wr_slp=0,
             s_wr_sz=512 * 1024,
-            ed=False,
-            nw=False,
             unpost=600,
-            no_mv=False,
-            no_del=False,
-            no_zip=False,
-            no_thumb=False,
-            no_athumb=False,
-            no_vthumb=False,
-            no_voldump=True,
-            no_scandir=False,
-            no_sendfile=True,
-            no_rescan=True,
-            no_logues=False,
-            no_readme=False,
-            re_maxage=0,
-            ihead=False,
-            nih=True,
             mtp=[],
             mte="a",
-            mth="",
-            textfiles="",
-            doctitle="",
-            html_head="",
             lang="eng",
-            theme=0,
-            themes=0,
-            turbo=0,
             logout=573,
-            hist=None,
-            no_idx=None,
-            no_hash=None,
-            force_js=False,
-            no_robots=False,
-            js_browser=None,
-            css_browser=None,
-            **{k: False for k in "e2d e2ds e2dsa e2t e2ts e2tsr no_acode".split()}
+            **ka
         )