v1.0.8

.gitignore

@@ -20,3 +20,7 @@ sfx/
 # derived
 copyparty/web/deps/
 srv/
+
+# state/logs
+up.*.txt
+.hist/

README.md

@@ -19,7 +19,7 @@ turn your phone or raspi into a portable file server with resumable uploads/down
 ## readme toc
 
 * top
-    * **[quickstart](#quickstart)** - download **[copyparty-sfx.py](https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.py)** and you're all set!
+    * [quickstart](#quickstart) - download **[copyparty-sfx.py](https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.py)** and you're all set!
         * [on servers](#on-servers) - you may also want these, especially on servers
         * [on debian](#on-debian) - recommended additional steps on debian
     * [notes](#notes) - general notes
@@ -61,6 +61,9 @@ turn your phone or raspi into a portable file server with resumable uploads/down
 * [performance](#performance) - defaults are usually fine - expect `8 GiB/s` download, `1 GiB/s` upload
 * [security](#security) - some notes on hardening
     * [gotchas](#gotchas) - behavior that might be unexpected
+* [recovering from crashes](#recovering-from-crashes)
+    * [client crashes](#client-crashes)
+        * [frefox wsod](#frefox-wsod) - firefox 87 can crash during uploads
 * [dependencies](#dependencies) - mandatory deps
     * [optional dependencies](#optional-dependencies) - install these to enable bonus features
     * [install recommended deps](#install-recommended-deps)
@@ -433,7 +436,7 @@ and then theres the tabs below it,
   * plus up to 3 entries each from `[done]` and `[que]` for context
 * `[que]` is all the files that are still queued
 
-note that since up2k has to read each file twice, `[🎈 bup]` can *theoretically* be up to 2x faster in some extreme cases (files bigger than your ram, combined with an internet connection faster than the read-speed of your HDD)
+note that since up2k has to read each file twice, `[🎈 bup]` can *theoretically* be up to 2x faster in some extreme cases (files bigger than your ram, combined with an internet connection faster than the read-speed of your HDD, or if you're uploading from a cuo2duo)
 
 if you are resuming a massive upload and want to skip hashing the files which already finished, you can enable `turbo` in the `[⚙️] config` tab, but please read the tooltip on that button
 
@@ -766,6 +769,14 @@ interact with copyparty using non-browser clients
   * `chunk(){ curl -b cppwd=wark -T- http://127.0.0.1:3923/;}`  
     `chunk <movie.mkv`
 
+* bash: when curl and wget is not available or too boring
+  * `(printf 'PUT /junk?pw=wark HTTP/1.1\r\n\r\n'; cat movie.mkv) | nc 127.0.0.1 3923`
+  * `(printf 'PUT / HTTP/1.1\r\n\r\n'; cat movie.mkv) >/dev/tcp/127.0.0.1/3923`
+
+* python: [up2k.py](https://github.com/9001/copyparty/blob/hovudstraum/bin/up2k.py) is a command-line up2k client [(webm)](https://ocv.me/stuff/u2cli.webm)
+  * file uploads, file-search, autoresume of aborted/broken uploads
+  * see [./bin/README.md#up2kpy](bin/README.md#up2kpy)
+
 * FUSE: mount a copyparty server as a local filesystem
   * cross-platform python client available in [./bin/](bin/)
   * [rclone](https://rclone.org/) as client can give ~5x performance, see [./docs/rclone.md](docs/rclone.md)
@@ -851,6 +862,26 @@ behavior that might be unexpected
 * users without read-access to a folder can still see the `.prologue.html` / `.epilogue.html` / `README.md` contents, for the purpose of showing a description on how to use the uploader for example
 
 
+# recovering from crashes
+
+## client crashes
+
+### frefox wsod
+
+firefox 87 can crash during uploads  -- the entire browser goes, including all other browser tabs, everything turns white
+
+however you can hit `F12` in the up2k tab and use the devtools to see how far you got in the uploads:
+
+* get a complete list of all uploads, organized by statuts (ok / no-good / busy / queued):  
+  `var tabs = { ok:[], ng:[], bz:[], q:[] }; for (var a of up2k.ui.tab) tabs[a.in].push(a); tabs`
+
+* list of filenames which failed:  
+  `​var ng = []; for (var a of up2k.ui.tab) if (a.in != 'ok') ng.push(a.hn.split('<a href=\"').slice(-1)[0].split('\">')[0]); ng`
+
+* send the list of filenames to copyparty for safekeeping:  
+  `await fetch('/inc', {method:'PUT', body:JSON.stringify(ng,null,1)})`
+
+
 # dependencies
 
 mandatory deps:

bin/README.md

@@ -1,3 +1,11 @@
+# [`up2k.py`](up2k.py)
+* command-line up2k client [(webm)](https://ocv.me/stuff/u2cli.webm)
+* file uploads, file-search, autoresume of aborted/broken uploads
+* faster than browsers
+* early beta, if something breaks just restart it
+
+
+
 # [`copyparty-fuse.py`](copyparty-fuse.py)
 * mount a copyparty server as a local filesystem (read-only)
 * **supports Windows!** -- expect `194 MiB/s` sequential read
@@ -47,6 +55,7 @@ you could replace winfsp with [dokan](https://github.com/dokan-dev/dokany/releas
 * copyparty can Popen programs like these during file indexing to collect additional metadata
 
 
+
 # [`dbtool.py`](dbtool.py)
 upgrade utility which can show db info and help transfer data between databases, for example when a new version of copyparty is incompatible with the old DB and automatically rebuilds the DB from scratch, but you have some really expensive `-mtp` parsers and want to copy over the tags from the old db
 
@@ -63,6 +72,7 @@ cd /mnt/nas/music/.hist
 ```
 
 
+
 # [`prisonparty.sh`](prisonparty.sh)
 * run copyparty in a chroot, preventing any accidental file access
 * creates bindmounts for /bin, /lib, and so on, see `sysdirs=`

bin/copyparty-fuse.py

@@ -71,7 +71,7 @@ except:
     elif MACOS:
         libfuse = "install https://osxfuse.github.io/"
     else:
-        libfuse = "apt install libfuse\n    modprobe fuse"
+        libfuse = "apt install libfuse3-3\n    modprobe fuse"
 
     print(
         "\n  could not import fuse; these may help:"
@@ -393,15 +393,16 @@ class Gateway(object):
 
         rsp = json.loads(rsp.decode("utf-8"))
         ret = []
-        for is_dir, nodes in [[True, rsp["dirs"]], [False, rsp["files"]]]:
+        for statfun, nodes in [
+            [self.stat_dir, rsp["dirs"]],
+            [self.stat_file, rsp["files"]],
+        ]:
             for n in nodes:
-                fname = unquote(n["href"]).rstrip(b"/")
-                fname = fname.decode("wtf-8")
+                fname = unquote(n["href"].split("?")[0]).rstrip(b"/").decode("wtf-8")
                 if bad_good:
                     fname = enwin(fname)
 
-                fun = self.stat_dir if is_dir else self.stat_file
-                ret.append([fname, fun(n["ts"], n["sz"]), 0])
+                ret.append([fname, statfun(n["ts"], n["sz"]), 0])
 
         return ret
 

bin/mtag/README.md

@@ -10,6 +10,7 @@ some of these rely on libraries which are not MIT-compatible
 
 these do not have any problematic dependencies:
 
+* [cksum.py](./cksum.py) computes various checksums
 * [exe.py](./exe.py) grabs metadata from .exe and .dll files (example for retrieving multiple tags with one parser)
 * [wget.py](./wget.py) lets you download files by POSTing URLs to copyparty
 

bin/mtag/cksum.py

@@ -0,0 +1,89 @@
+#!/usr/bin/env python3
+
+import sys
+import json
+import zlib
+import struct
+import base64
+import hashlib
+
+try:
+    from copyparty.util import fsenc
+except:
+
+    def fsenc(p):
+        return p
+
+
+"""
+calculates various checksums for uploads,
+usage: -mtp crc32,md5,sha1,sha256b=bin/mtag/cksum.py
+"""
+
+
+def main():
+    config = "crc32 md5 md5b sha1 sha1b sha256 sha256b sha512/240 sha512b/240"
+    # b suffix = base64 encoded
+    # slash = truncate to n bits
+
+    known = {
+        "md5": hashlib.md5,
+        "sha1": hashlib.sha1,
+        "sha256": hashlib.sha256,
+        "sha512": hashlib.sha512,
+    }
+    config = config.split()
+    hashers = {
+        k: v()
+        for k, v in known.items()
+        if k in [x.split("/")[0].rstrip("b") for x in known]
+    }
+    crc32 = 0 if "crc32" in config else None
+
+    with open(fsenc(sys.argv[1]), "rb", 512 * 1024) as f:
+        while True:
+            buf = f.read(64 * 1024)
+            if not buf:
+                break
+
+            for x in hashers.values():
+                x.update(buf)
+
+            if crc32 is not None:
+                crc32 = zlib.crc32(buf, crc32)
+
+    ret = {}
+    for s in config:
+        alg = s.split("/")[0]
+        b64 = alg.endswith("b")
+        alg = alg.rstrip("b")
+        if alg in hashers:
+            v = hashers[alg].digest()
+        elif alg == "crc32":
+            v = crc32
+            if v < 0:
+                v &= 2 ** 32 - 1
+            v = struct.pack(">L", v)
+        else:
+            raise Exception("what is {}".format(s))
+
+        if "/" in s:
+            v = v[: int(int(s.split("/")[1]) / 8)]
+
+        if b64:
+            v = base64.b64encode(v).decode("ascii").rstrip("=")
+        else:
+            try:
+                v = v.hex()
+            except:
+                import binascii
+
+                v = binascii.hexlify(v)
+
+        ret[s] = v
+
+    print(json.dumps(ret, indent=4))
+
+
+if __name__ == "__main__":
+    main()

bin/up2k.py

@@ -0,0 +1,723 @@
+#!/usr/bin/env python3
+from __future__ import print_function, unicode_literals
+
+"""
+up2k.py: upload to copyparty
+2021-10-04, v0.7, ed <irc.rizon.net>, MIT-Licensed
+https://github.com/9001/copyparty/blob/hovudstraum/bin/up2k.py
+
+- dependencies: requests
+- supports python 2.6, 2.7, and 3.3 through 3.10
+
+- almost zero error-handling
+- but if something breaks just try again and it'll autoresume
+"""
+
+import os
+import sys
+import stat
+import math
+import time
+import atexit
+import signal
+import base64
+import hashlib
+import argparse
+import platform
+import threading
+import requests
+import datetime
+
+
+# from copyparty/__init__.py
+PY2 = sys.version_info[0] == 2
+if PY2:
+    from Queue import Queue
+
+    sys.dont_write_bytecode = True
+    bytes = str
+else:
+    from queue import Queue
+
+    unicode = str
+
+VT100 = platform.system() != "Windows"
+
+
+req_ses = requests.Session()
+
+
+class File(object):
+    """an up2k upload task; represents a single file"""
+
+    def __init__(self, top, rel, size, lmod):
+        self.top = top  # type: bytes
+        self.rel = rel.replace(b"\\", b"/")  # type: bytes
+        self.size = size  # type: int
+        self.lmod = lmod  # type: float
+
+        self.abs = os.path.join(top, rel)  # type: bytes
+        self.name = self.rel.split(b"/")[-1].decode("utf-8", "replace")  # type: str
+
+        # set by get_hashlist
+        self.cids = []  # type: list[tuple[str, int, int]]  # [ hash, ofs, sz ]
+        self.kchunks = {}  # type: dict[str, tuple[int, int]]  # hash: [ ofs, sz ]
+
+        # set by handshake
+        self.ucids = []  # type: list[str]  # chunks which need to be uploaded
+        self.wark = None  # type: str
+        self.url = None  # type: str
+
+        # set by upload
+        self.up_b = 0  # type: int
+        self.up_c = 0  # type: int
+
+        # m = "size({}) lmod({}) top({}) rel({}) abs({}) name({})\n"
+        # eprint(m.format(self.size, self.lmod, self.top, self.rel, self.abs, self.name))
+
+
+class FileSlice(object):
+    """file-like object providing a fixed window into a file"""
+
+    def __init__(self, file, cid):
+        # type: (File, str) -> FileSlice
+
+        self.car, self.len = file.kchunks[cid]
+        self.cdr = self.car + self.len
+        self.ofs = 0  # type: int
+        self.f = open(file.abs, "rb", 512 * 1024)
+        self.f.seek(self.car)
+
+        # https://stackoverflow.com/questions/4359495/what-is-exactly-a-file-like-object-in-python
+        # IOBase, RawIOBase, BufferedIOBase
+        funs = "close closed __enter__ __exit__ __iter__ isatty __next__ readable seekable writable"
+        try:
+            for fun in funs.split():
+                setattr(self, fun, getattr(self.f, fun))
+        except:
+            pass  # py27 probably
+
+    def tell(self):
+        return self.ofs
+
+    def seek(self, ofs, wh=0):
+        if wh == 1:
+            ofs = self.ofs + ofs
+        elif wh == 2:
+            ofs = self.len + ofs  # provided ofs is negative
+
+        if ofs < 0:
+            ofs = 0
+        elif ofs >= self.len:
+            ofs = self.len - 1
+
+        self.ofs = ofs
+        self.f.seek(self.car + ofs)
+
+    def read(self, sz):
+        sz = min(sz, self.len - self.ofs)
+        ret = self.f.read(sz)
+        self.ofs += len(ret)
+        return ret
+
+
+def eprint(*a, **ka):
+    ka["file"] = sys.stderr
+    ka["end"] = ""
+    if not PY2:
+        ka["flush"] = True
+
+    print(*a, **ka)
+    if PY2:
+        sys.stderr.flush()
+
+
+def termsize():
+    import os
+
+    env = os.environ
+
+    def ioctl_GWINSZ(fd):
+        try:
+            import fcntl, termios, struct, os
+
+            cr = struct.unpack("hh", fcntl.ioctl(fd, termios.TIOCGWINSZ, "1234"))
+        except:
+            return
+        return cr
+
+    cr = ioctl_GWINSZ(0) or ioctl_GWINSZ(1) or ioctl_GWINSZ(2)
+    if not cr:
+        try:
+            fd = os.open(os.ctermid(), os.O_RDONLY)
+            cr = ioctl_GWINSZ(fd)
+            os.close(fd)
+        except:
+            pass
+    if not cr:
+        try:
+            cr = (env["LINES"], env["COLUMNS"])
+        except:
+            cr = (25, 80)
+    return int(cr[1]), int(cr[0])
+
+
+class CTermsize(object):
+    def __init__(self):
+        self.ev = False
+        self.margin = None
+        self.g = None
+        self.w, self.h = termsize()
+
+        try:
+            signal.signal(signal.SIGWINCH, self.ev_sig)
+        except:
+            return
+
+        thr = threading.Thread(target=self.worker)
+        thr.daemon = True
+        thr.start()
+
+    def worker(self):
+        while True:
+            time.sleep(0.5)
+            if not self.ev:
+                continue
+
+            self.ev = False
+            self.w, self.h = termsize()
+
+            if self.margin is not None:
+                self.scroll_region(self.margin)
+
+    def ev_sig(self, *a, **ka):
+        self.ev = True
+
+    def scroll_region(self, margin):
+        self.margin = margin
+        if margin is None:
+            self.g = None
+            eprint("\033[s\033[r\033[u")
+        else:
+            self.g = 1 + self.h - margin
+            m = "{0}\033[{1}A".format("\n" * margin, margin)
+            eprint("{0}\033[s\033[1;{1}r\033[u".format(m, self.g - 1))
+
+
+ss = CTermsize()
+
+
+def statdir(top):
+    """non-recursive listing of directory contents, along with stat() info"""
+    if hasattr(os, "scandir"):
+        with os.scandir(top) as dh:
+            for fh in dh:
+                yield [os.path.join(top, fh.name), fh.stat()]
+    else:
+        for name in os.listdir(top):
+            abspath = os.path.join(top, name)
+            yield [abspath, os.stat(abspath)]
+
+
+def walkdir(top):
+    """recursive statdir"""
+    for ap, inf in sorted(statdir(top)):
+        if stat.S_ISDIR(inf.st_mode):
+            for x in walkdir(ap):
+                yield x
+        else:
+            yield ap, inf
+
+
+def walkdirs(tops):
+    """recursive statdir for a list of tops, yields [top, relpath, stat]"""
+    for top in tops:
+        if os.path.isdir(top):
+            for ap, inf in walkdir(top):
+                yield top, ap[len(top) + 1 :], inf
+        else:
+            sep = "{0}".format(os.sep).encode("ascii")
+            d, n = top.rsplit(sep, 1)
+            yield d, n, os.stat(top)
+
+
+# from copyparty/util.py
+def humansize(sz, terse=False):
+    """picks a sensible unit for the given extent"""
+    for unit in ["B", "KiB", "MiB", "GiB", "TiB"]:
+        if sz < 1024:
+            break
+
+        sz /= 1024.0
+
+    ret = " ".join([str(sz)[:4].rstrip("."), unit])
+
+    if not terse:
+        return ret
+
+    return ret.replace("iB", "").replace(" ", "")
+
+
+# from copyparty/up2k.py
+def up2k_chunksize(filesize):
+    """gives The correct chunksize for up2k hashing"""
+    chunksize = 1024 * 1024
+    stepsize = 512 * 1024
+    while True:
+        for mul in [1, 2]:
+            nchunks = math.ceil(filesize * 1.0 / chunksize)
+            if nchunks <= 256 or chunksize >= 32 * 1024 * 1024:
+                return chunksize
+
+            chunksize += stepsize
+            stepsize *= mul
+
+
+# mostly from copyparty/up2k.py
+def get_hashlist(file, pcb):
+    # type: (File, any) -> None
+    """generates the up2k hashlist from file contents, inserts it into `file`"""
+
+    chunk_sz = up2k_chunksize(file.size)
+    file_rem = file.size
+    file_ofs = 0
+    ret = []
+    with open(file.abs, "rb", 512 * 1024) as f:
+        while file_rem > 0:
+            hashobj = hashlib.sha512()
+            chunk_sz = chunk_rem = min(chunk_sz, file_rem)
+            while chunk_rem > 0:
+                buf = f.read(min(chunk_rem, 64 * 1024))
+                if not buf:
+                    raise Exception("EOF at " + str(f.tell()))
+
+                hashobj.update(buf)
+                chunk_rem -= len(buf)
+
+            digest = hashobj.digest()[:33]
+            digest = base64.urlsafe_b64encode(digest).decode("utf-8")
+
+            ret.append([digest, file_ofs, chunk_sz])
+            file_ofs += chunk_sz
+            file_rem -= chunk_sz
+
+            if pcb:
+                pcb(file, file_ofs)
+
+    file.cids = ret
+    file.kchunks = {}
+    for k, v1, v2 in ret:
+        file.kchunks[k] = [v1, v2]
+
+
+def handshake(req_ses, url, file, pw, search):
+    # type: (requests.Session, str, File, any, bool) -> List[str]
+    """
+    performs a handshake with the server; reply is:
+      if search, a list of search results
+      otherwise, a list of chunks to upload
+    """
+
+    req = {
+        "hash": [x[0] for x in file.cids],
+        "name": file.name,
+        "lmod": file.lmod,
+        "size": file.size,
+    }
+    if search:
+        req["srch"] = 1
+
+    headers = {"Content-Type": "text/plain"}  # wtf ed
+    if pw:
+        headers["Cookie"] = "=".join(["cppwd", pw])
+
+    if file.url:
+        url = file.url
+    elif b"/" in file.rel:
+        url += file.rel.rsplit(b"/", 1)[0].decode("utf-8", "replace")
+
+    while True:
+        try:
+            r = req_ses.post(url, headers=headers, json=req)
+            break
+        except:
+            eprint("handshake failed, retry...\n")
+            time.sleep(1)
+
+    try:
+        r = r.json()
+    except:
+        raise Exception(r.text)
+
+    if search:
+        return r["hits"]
+
+    try:
+        pre, url = url.split("://")
+        pre += "://"
+    except:
+        pre = ""
+
+    file.url = pre + url.split("/")[0] + r["purl"]
+    file.name = r["name"]
+    file.wark = r["wark"]
+
+    return r["hash"]
+
+
+def upload(req_ses, file, cid, pw):
+    # type: (requests.Session, File, str, any) -> None
+    """upload one specific chunk, `cid` (a chunk-hash)"""
+
+    headers = {
+        "X-Up2k-Hash": cid,
+        "X-Up2k-Wark": file.wark,
+        "Content-Type": "application/octet-stream",
+    }
+    if pw:
+        headers["Cookie"] = "=".join(["cppwd", pw])
+
+    f = FileSlice(file, cid)
+    try:
+        r = req_ses.post(file.url, headers=headers, data=f)
+        if not r:
+            raise Exception(repr(r))
+
+        _ = r.content
+    finally:
+        f.f.close()
+
+
+class Daemon(threading.Thread):
+    def __init__(self, *a, **ka):
+        threading.Thread.__init__(self, *a, **ka)
+        self.daemon = True
+
+
+class Ctl(object):
+    """
+    this will be the coordinator which runs everything in parallel
+    (hashing, handshakes, uploads)  but right now it's p dumb
+    """
+
+    def __init__(self, ar):
+        self.ar = ar
+        ar.files = [
+            os.path.abspath(os.path.realpath(x.encode("utf-8"))) for x in ar.files
+        ]
+        ar.url = ar.url.rstrip("/") + "/"
+        if "://" not in ar.url:
+            ar.url = "http://" + ar.url
+
+        eprint("\nscanning {0} locations\n".format(len(ar.files)))
+
+        nfiles = 0
+        nbytes = 0
+        for _, _, inf in walkdirs(ar.files):
+            nfiles += 1
+            nbytes += inf.st_size
+
+        eprint("found {0} files, {1}\n\n".format(nfiles, humansize(nbytes)))
+        self.nfiles = nfiles
+        self.nbytes = nbytes
+
+        if ar.td:
+            req_ses.verify = False
+        if ar.te:
+            req_ses.verify = ar.te
+
+        self.filegen = walkdirs(ar.files)
+        if ar.safe:
+            self.safe()
+        else:
+            self.fancy()
+
+    def safe(self):
+        """minimal basic slow boring fallback codepath"""
+        search = self.ar.s
+        for nf, (top, rel, inf) in enumerate(self.filegen):
+            file = File(top, rel, inf.st_size, inf.st_mtime)
+            upath = file.abs.decode("utf-8", "replace")
+
+            print("{0} {1}\n  hash...".format(self.nfiles - nf, upath))
+            get_hashlist(file, None)
+
+            while True:
+                print("  hs...")
+                hs = handshake(req_ses, self.ar.url, file, self.ar.a, search)
+                if search:
+                    if hs:
+                        for hit in hs:
+                            print("  found: {0}{1}".format(self.ar.url, hit["rp"]))
+                    else:
+                        print("  NOT found")
+                    break
+
+                file.ucids = hs
+                if not hs:
+                    break
+
+                print("{0} {1}".format(self.nfiles - nf, upath))
+                ncs = len(hs)
+                for nc, cid in enumerate(hs):
+                    print("  {0} up {1}".format(ncs - nc, cid))
+                    upload(req_ses, file, cid, self.ar.a)
+
+            print("  ok!")
+
+    def fancy(self):
+        self.hash_f = 0
+        self.hash_c = 0
+        self.hash_b = 0
+        self.up_f = 0
+        self.up_c = 0
+        self.up_b = 0
+        self.up_br = 0
+        self.hasher_busy = 1
+        self.handshaker_busy = 0
+        self.uploader_busy = 0
+
+        self.t0 = time.time()
+        self.t0_up = None
+        self.spd = None
+
+        self.mutex = threading.Lock()
+        self.q_handshake = Queue()  # type: Queue[File]
+        self.q_recheck = Queue()  # type: Queue[File]  # partial upload exists [...]
+        self.q_upload = Queue()  # type: Queue[tuple[File, str]]
+
+        self.st_hash = [None, "(idle, starting...)"]  # type: tuple[File, int]
+        self.st_up = [None, "(idle, starting...)"]  # type: tuple[File, int]
+        if VT100:
+            atexit.register(self.cleanup_vt100)
+            ss.scroll_region(3)
+
+        Daemon(target=self.hasher).start()
+        for _ in range(self.ar.j):
+            Daemon(target=self.handshaker).start()
+            Daemon(target=self.uploader).start()
+
+        idles = 0
+        while idles < 3:
+            time.sleep(0.07)
+            with self.mutex:
+                if (
+                    self.q_handshake.empty()
+                    and self.q_upload.empty()
+                    and not self.hasher_busy
+                    and not self.handshaker_busy
+                    and not self.uploader_busy
+                ):
+                    idles += 1
+                else:
+                    idles = 0
+
+            if VT100:
+                maxlen = ss.w - len(str(self.nfiles)) - 14
+                txt = "\033[s\033[{0}H".format(ss.g)
+                for y, k, st, f in [
+                    [0, "hash", self.st_hash, self.hash_f],
+                    [1, "send", self.st_up, self.up_f],
+                ]:
+                    txt += "\033[{0}H{1}:".format(ss.g + y, k)
+                    file, arg = st
+                    if not file:
+                        txt += " {0}\033[K".format(arg)
+                    else:
+                        if y:
+                            p = 100 * file.up_b / file.size
+                        else:
+                            p = 100 * arg / file.size
+
+                        name = file.abs.decode("utf-8", "replace")[-maxlen:]
+                        if "/" in name:
+                            name = "\033[36m{0}\033[0m/{1}".format(*name.rsplit("/", 1))
+
+                        m = "{0:6.1f}% {1} {2}\033[K"
+                        txt += m.format(p, self.nfiles - f, name)
+
+                txt += "\033[{0}H ".format(ss.g + 2)
+            else:
+                txt = " "
+
+            if not self.up_br:
+                spd = self.hash_b / (time.time() - self.t0)
+                eta = (self.nbytes - self.hash_b) / (spd + 1)
+            else:
+                spd = self.up_br / (time.time() - self.t0_up)
+                spd = self.spd = (self.spd or spd) * 0.9 + spd * 0.1
+                eta = (self.nbytes - self.up_b) / (spd + 1)
+
+            spd = humansize(spd)
+            eta = str(datetime.timedelta(seconds=int(eta)))
+            left = humansize(self.nbytes - self.up_b)
+            tail = "\033[K\033[u" if VT100 else "\r"
+
+            m = "eta: {0} @ {1}/s, {2} left".format(eta, spd, left)
+            eprint(txt + "\033]0;{0}\033\\\r{1}{2}".format(m, m, tail))
+
+    def cleanup_vt100(self):
+        ss.scroll_region(None)
+        eprint("\033[J\033]0;\033\\")
+
+    def cb_hasher(self, file, ofs):
+        self.st_hash = [file, ofs]
+
+    def hasher(self):
+        for top, rel, inf in self.filegen:
+            file = File(top, rel, inf.st_size, inf.st_mtime)
+            while True:
+                with self.mutex:
+                    if (
+                        self.hash_b - self.up_b < 1024 * 1024 * 128
+                        and self.hash_c - self.up_c < 64
+                        and (
+                            not self.ar.nh
+                            or (
+                                self.q_upload.empty()
+                                and self.q_handshake.empty()
+                                and not self.uploader_busy
+                            )
+                        )
+                    ):
+                        break
+
+                time.sleep(0.05)
+
+            get_hashlist(file, self.cb_hasher)
+            with self.mutex:
+                self.hash_f += 1
+                self.hash_c += len(file.cids)
+                self.hash_b += file.size
+
+            self.q_handshake.put(file)
+
+        self.hasher_busy = 0
+        self.st_hash = [None, "(finished)"]
+
+    def handshaker(self):
+        search = self.ar.s
+        q = self.q_handshake
+        while True:
+            file = q.get()
+            if not file:
+                if q == self.q_handshake:
+                    q = self.q_recheck
+                    q.put(None)
+                    continue
+
+                self.q_upload.put(None)
+                break
+
+            with self.mutex:
+                self.handshaker_busy += 1
+
+            upath = file.abs.decode("utf-8", "replace")
+
+            try:
+                hs = handshake(req_ses, self.ar.url, file, self.ar.a, search)
+            except Exception as ex:
+                if q == self.q_handshake and "<pre>partial upload exists" in str(ex):
+                    self.q_recheck.put(file)
+                    hs = []
+                else:
+                    raise
+
+            if search:
+                if hs:
+                    for hit in hs:
+                        m = "found: {0}\n  {1}{2}\n"
+                        print(m.format(upath, self.ar.url, hit["rp"]), end="")
+                else:
+                    print("NOT found: {0}\n".format(upath), end="")
+
+                with self.mutex:
+                    self.up_f += 1
+                    self.up_c += len(file.cids)
+                    self.up_b += file.size
+                    self.handshaker_busy -= 1
+
+                continue
+
+            with self.mutex:
+                if not hs:
+                    # all chunks done
+                    self.up_f += 1
+                    self.up_c += len(file.cids) - file.up_c
+                    self.up_b += file.size - file.up_b
+
+                if hs and file.up_c:
+                    # some chunks failed
+                    self.up_c -= len(hs)
+                    file.up_c -= len(hs)
+                    for cid in hs:
+                        sz = file.kchunks[cid][1]
+                        self.up_b -= sz
+                        file.up_b -= sz
+
+                file.ucids = hs
+                self.handshaker_busy -= 1
+
+            if not hs:
+                print("uploaded {0}".format(upath))
+            for cid in hs:
+                self.q_upload.put([file, cid])
+
+    def uploader(self):
+        while True:
+            task = self.q_upload.get()
+            if not task:
+                self.st_up = [None, "(finished)"]
+                break
+
+            with self.mutex:
+                self.uploader_busy += 1
+                self.t0_up = self.t0_up or time.time()
+
+            file, cid = task
+            try:
+                upload(req_ses, file, cid, self.ar.a)
+            except:
+                eprint("upload failed, retry...\n")
+                pass  # handshake will fix it
+
+            with self.mutex:
+                sz = file.kchunks[cid][1]
+                file.ucids = [x for x in file.ucids if x != cid]
+                if not file.ucids:
+                    self.q_handshake.put(file)
+
+                self.st_up = [file, cid]
+                file.up_b += sz
+                self.up_b += sz
+                self.up_br += sz
+                file.up_c += 1
+                self.up_c += 1
+                self.uploader_busy -= 1
+
+
+def main():
+    time.strptime("19970815", "%Y%m%d")  # python#7980
+    if not VT100:
+        os.system("rem")  # enables colors
+
+    # fmt: off
+    ap = app = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    ap.add_argument("url", type=unicode, help="server url, including destination folder")
+    ap.add_argument("files", type=unicode, nargs="+", help="files and/or folders to process")
+    ap.add_argument("-a", metavar="PASSWORD", help="password")
+    ap.add_argument("-s", action="store_true", help="file-search (disables upload)")
+    ap = app.add_argument_group("performance tweaks")
+    ap.add_argument("-j", type=int, metavar="THREADS", default=4, help="parallel connections")
+    ap.add_argument("-nh", action="store_true", help="disable hashing while uploading")
+    ap.add_argument("--safe", action="store_true", help="use simple fallback approach")
+    ap = app.add_argument_group("tls")
+    ap.add_argument("-te", metavar="PEM_FILE", help="certificate to expect/verify")
+    ap.add_argument("-td", action="store_true", help="disable certificate check")
+    # fmt: on
+
+    Ctl(app.parse_args())
+
+
+if __name__ == "__main__":
+    main()

bin/up2k.sh

@@ -8,7 +8,7 @@ set -e
 ##
 ## config
 
-datalen=$((2*1024*1024*1024))
+datalen=$((128*1024*1024))
 target=127.0.0.1
 posturl=/inc
 passwd=wark
@@ -37,10 +37,10 @@ gendata() {
 # pipe a chunk, get the base64 checksum
 gethash() {
     printf $(
-        sha512sum | cut -c-64 |
+        sha512sum | cut -c-66 |
         sed -r 's/ .*//;s/(..)/\\x\1/g'
     ) |
-    base64 -w0 | cut -c-43 |
+    base64 -w0 | cut -c-44 |
     tr '+/' '-_'
 }
 
@@ -123,7 +123,7 @@ printf '\033[36m'
 {
     {
         cat <<EOF
-POST $posturl/handshake.php HTTP/1.1
+POST $posturl/ HTTP/1.1
 Connection: Close
 Cookie: cppwd=$passwd
 Content-Type: text/plain;charset=UTF-8
@@ -145,14 +145,16 @@ printf '\033[0m\nwark: %s\n' $wark
 ##
 ## wait for signal to continue
 
-w8=/dev/shm/$salt.w8
-touch $w8
+true || {
+    w8=/dev/shm/$salt.w8
+    touch $w8
 
-echo "ready;  rm -f $w8"
+    echo "ready;  rm -f $w8"
 
-while [ -e $w8 ]; do
-    sleep 0.2
-done
+    while [ -e $w8 ]; do
+        sleep 0.2
+    done
+}
 
 
 ##
@@ -175,7 +177,7 @@ while [ $remains -gt 0 ]; do
     
     {
         cat <<EOF
-POST $posturl/chunkpit.php HTTP/1.1
+POST $posturl/ HTTP/1.1
 Connection: Keep-Alive
 Cookie: cppwd=$passwd
 Content-Type: application/octet-stream

copyparty/__main__.py

@@ -344,6 +344,9 @@ def run_argparse(argv, formatter):
     ap2.add_argument("--dotpart", action="store_true", help="dotfile incomplete uploads")
     ap2.add_argument("--sparse", metavar="MiB", type=int, default=4, help="up2k min.size threshold (mswin-only)")
     ap2.add_argument("--unpost", metavar="SEC", type=int, default=3600*12, help="grace period where uploads can be deleted by the uploader, even without delete permissions; 0=disabled")
+    ap2.add_argument("--no-fpool", action="store_true", help="disable file-handle pooling -- instead, repeatedly close and reopen files during upload")
+    ap2.add_argument("--use-fpool", action="store_true", help="force file-handle pooling, even if copyparty thinks you're better off without")
+    ap2.add_argument("--no-symlink", action="store_true", help="duplicate file contents instead")
 
     ap2 = ap.add_argument_group('network options')
     ap2.add_argument("-i", metavar="IP", type=u, default="0.0.0.0", help="ip to bind (comma-sep.)")
@@ -375,6 +378,7 @@ def run_argparse(argv, formatter):
     ap2.add_argument("--no-dot-ren", action="store_true", help="disallow renaming dotfiles; makes it impossible to make something a dotfile")
     ap2.add_argument("--no-logues", action="store_true", help="disable rendering .prologue/.epilogue.html into directory listings")
     ap2.add_argument("--no-readme", action="store_true", help="disable rendering readme.md into directory listings")
+    ap2.add_argument("--vague-403", action="store_true", help="send 404 instead of 403 (security through ambiguity, very enterprise)")
 
     ap2 = ap.add_argument_group('logging options')
     ap2.add_argument("-q", action="store_true", help="quiet")

copyparty/__version__.py

@@ -1,8 +1,8 @@
 # coding: utf-8
 
-VERSION = (1, 0, 5)
+VERSION = (1, 0, 8)
 CODENAME = "sufficient"
-BUILD_DT = (2021, 9, 19)
+BUILD_DT = (2021, 10, 4)
 
 S_VERSION = ".".join(map(str, VERSION))
 S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT)

copyparty/authsrv.py

@@ -356,7 +356,7 @@ class VFS(object):
         if not dbv:
             return self, vrem
 
-        vrem = [self.vpath[len(dbv.vpath) + 1 :], vrem]
+        vrem = [self.vpath[len(dbv.vpath) :].lstrip("/"), vrem]
         vrem = "/".join([x for x in vrem if x])
         return dbv, vrem
 
@@ -880,6 +880,10 @@ class AuthSrv(object):
             # default tag cfgs if unset
             if "mte" not in vol.flags:
                 vol.flags["mte"] = self.args.mte
+            elif vol.flags["mte"].startswith("+"):
+                vol.flags["mte"] = ",".join(
+                    x for x in [self.args.mte, vol.flags["mte"][1:]] if x
+                )
             if "mth" not in vol.flags:
                 vol.flags["mth"] = self.args.mth
 

copyparty/httpcli.py

@@ -39,6 +39,7 @@ class HttpCli(object):
     def __init__(self, conn):
         self.t0 = time.time()
         self.conn = conn
+        self.mutex = conn.mutex
         self.s = conn.s  # type: socket
         self.sr = conn.sr  # type: Unrecv
         self.ip = conn.addr[0]
@@ -47,6 +48,7 @@ class HttpCli(object):
         self.asrv = conn.asrv  # type: AuthSrv
         self.ico = conn.ico
         self.thumbcli = conn.thumbcli
+        self.u2fh = conn.u2fh
         self.log_func = conn.log_func
         self.log_src = conn.log_src
         self.tls = hasattr(self.s, "cipher")
@@ -387,7 +389,7 @@ class HttpCli(object):
         if not self.can_read and not self.can_write and not self.can_get:
             if self.vpath:
                 self.log("inaccessible: [{}]".format(self.vpath))
-                return self.tx_404()
+                return self.tx_404(True)
 
             self.uparam["h"] = False
 
@@ -835,7 +837,18 @@ class HttpCli(object):
 
         reader = read_socket(self.sr, remains)
 
-        with open(fsenc(path), "rb+", 512 * 1024) as f:
+        f = None
+        fpool = not self.args.no_fpool
+        if fpool:
+            with self.mutex:
+                try:
+                    f = self.u2fh.pop(path)
+                except:
+                    pass
+
+        f = f or open(fsenc(path), "rb+", 512 * 1024)
+
+        try:
             f.seek(cstart[0])
             post_sz, _, sha_b64 = hashcopy(reader, f)
 
@@ -865,22 +878,36 @@ class HttpCli(object):
                     ofs += len(buf)
 
                 self.log("clone {} done".format(cstart[0]))
+        finally:
+            if not fpool:
+                f.close()
+            else:
+                with self.mutex:
+                    self.u2fh.put(path, f)
 
         x = self.conn.hsrv.broker.put(True, "up2k.confirm_chunk", ptop, wark, chash)
         x = x.get()
         try:
-            num_left, path = x
+            num_left, fin_path = x
         except:
             self.loud_reply(x, status=500)
             return False
 
-        if not ANYWIN and num_left == 0:
+        if not num_left and fpool:
+            with self.mutex:
+                self.u2fh.close(path)
+
+        # windows cant rename open files
+        if ANYWIN and path != fin_path and not self.args.nw:
+            self.conn.hsrv.broker.put(True, "up2k.finish_upload", ptop, wark).get()
+
+        if not ANYWIN and not num_left:
             times = (int(time.time()), int(lastmod))
             self.log("no more chunks, setting times {}".format(times))
             try:
-                bos.utime(path, times)
+                bos.utime(fin_path, times)
             except:
-                self.log("failed to utime ({}, {})".format(path, times))
+                self.log("failed to utime ({}, {})".format(fin_path, times))
 
         spd = self._spd(post_sz)
         self.log("{} thank".format(spd))
@@ -1538,7 +1565,7 @@ class HttpCli(object):
 
         if not self.can_write:
             if "edit" in self.uparam or "edit2" in self.uparam:
-                return self.tx_404()
+                return self.tx_404(True)
 
         tpl = "mde" if "edit2" in self.uparam else "md"
         html_path = os.path.join(E.mod, "web", "{}.html".format(tpl))
@@ -1640,8 +1667,14 @@ class HttpCli(object):
         self.reply(html.encode("utf-8"))
         return True
 
-    def tx_404(self):
-        m = '<h1>404 not found &nbsp;┐( ´ -`)┌</h1><p>or maybe you don\'t have access -- try logging in or <a href="/?h">go home</a></p>'
+    def tx_404(self, is_403=False):
+        if self.args.vague_403:
+            m = '<h1>404 not found &nbsp;┐( ´ -`)┌</h1><p>or maybe you don\'t have access -- try logging in or <a href="/?h">go home</a></p>'
+        elif is_403:
+            m = '<h1>403 forbiddena &nbsp;~┻━┻</h1><p>you\'ll have to log in or <a href="/?h">go home</a></p>'
+        else:
+            m = '<h1>404 not found &nbsp;┐( ´ -`)┌</h1><p><a href="/?h">go home</a></p>'
+
         html = self.j2("splash", this=self, qvpath=quotep(self.vpath), msg=m)
         self.reply(html.encode("utf-8"), status=404)
         return True
@@ -1755,7 +1788,7 @@ class HttpCli(object):
                 if filt and filt not in vp:
                     continue
 
-                ret.append({"vp": vp, "sz": sz, "at": at})
+                ret.append({"vp": quotep(vp), "sz": sz, "at": at})
                 if len(ret) > 3000:
                     ret.sort(key=lambda x: x["at"], reverse=True)
                     ret = ret[:2000]
@@ -1868,7 +1901,7 @@ class HttpCli(object):
             return self.tx_file(abspath)
 
         elif is_dir and not self.can_read and not self.can_write:
-            return self.tx_404()
+            return self.tx_404(True)
 
         srv_info = []
 
@@ -1973,7 +2006,7 @@ class HttpCli(object):
                 return True
 
             if not stat.S_ISDIR(st.st_mode):
-                return self.tx_404()
+                return self.tx_404(True)
 
             if "zip" in self.uparam or "tar" in self.uparam:
                 raise Pebkac(403)

copyparty/httpconn.py

@@ -32,9 +32,11 @@ class HttpConn(object):
         self.addr = addr
         self.hsrv = hsrv
 
+        self.mutex = hsrv.mutex
         self.args = hsrv.args
         self.asrv = hsrv.asrv
         self.cert_path = hsrv.cert_path
+        self.u2fh = hsrv.u2fh
 
         enth = HAVE_PIL and not self.args.no_thumb
         self.thumbcli = ThumbCli(hsrv.broker) if enth else None

copyparty/httpsrv.py

@@ -27,7 +27,7 @@ except ImportError:
     sys.exit(1)
 
 from .__init__ import E, PY2, MACOS
-from .util import spack, min_ex, start_stackmon, start_log_thrs
+from .util import FHC, spack, min_ex, start_stackmon, start_log_thrs
 from .bos import bos
 from .httpconn import HttpConn
 
@@ -50,7 +50,10 @@ class HttpSrv(object):
         self.log = broker.log
         self.asrv = broker.asrv
 
-        self.name = "httpsrv" + ("-n{}-i{:x}".format(nid, os.getpid()) if nid else "")
+        nsuf = "-{}".format(nid) if nid else ""
+        nsuf2 = "-n{}-i{:x}".format(nid, os.getpid()) if nid else ""
+
+        self.name = "hsrv" + nsuf2
         self.mutex = threading.Lock()
         self.stopping = False
 
@@ -59,6 +62,7 @@ class HttpSrv(object):
         self.tp_time = None  # latest worker collect
         self.tp_q = None if self.args.no_htp else queue.LifoQueue()
 
+        self.u2fh = FHC()
         self.srvs = []
         self.ncli = 0  # exact
         self.clients = {}  # laggy
@@ -82,11 +86,6 @@ class HttpSrv(object):
         if self.tp_q:
             self.start_threads(4)
 
-            name = "httpsrv-scaler" + ("-{}".format(nid) if nid else "")
-            t = threading.Thread(target=self.thr_scaler, name=name)
-            t.daemon = True
-            t.start()
-
         if nid:
             if self.args.stackmon:
                 start_stackmon(self.args.stackmon, nid)
@@ -94,6 +93,10 @@ class HttpSrv(object):
             if self.args.log_thrs:
                 start_log_thrs(self.log, self.args.log_thrs, nid)
 
+        t = threading.Thread(target=self.periodic, name="hsrv-pt" + nsuf)
+        t.daemon = True
+        t.start()
+
     def start_threads(self, n):
         self.tp_nthr += n
         if self.args.log_htp:
@@ -115,13 +118,15 @@ class HttpSrv(object):
         for _ in range(n):
             self.tp_q.put(None)
 
-    def thr_scaler(self):
+    def periodic(self):
         while True:
-            time.sleep(2 if self.tp_ncli else 30)
+            time.sleep(2 if self.tp_ncli else 10)
             with self.mutex:
-                self.tp_ncli = max(self.ncli, self.tp_ncli - 2)
-                if self.tp_nthr > self.tp_ncli + 8:
-                    self.stop_threads(4)
+                self.u2fh.clean()
+                if self.tp_q:
+                    self.tp_ncli = max(self.ncli, self.tp_ncli - 2)
+                    if self.tp_nthr > self.tp_ncli + 8:
+                        self.stop_threads(4)
 
     def listen(self, sck, nlisteners):
         ip, port = sck.getsockname()

copyparty/svchub.py

@@ -53,6 +53,17 @@ class SvcHub(object):
         if args.log_thrs:
             start_log_thrs(self.log, args.log_thrs, 0)
 
+        if not ANYWIN and not args.use_fpool:
+            args.no_fpool = True
+
+        if not args.no_fpool and args.j != 1:
+            m = "WARNING: --use-fpool combined with multithreading is untested and can probably cause undefined behavior"
+            if ANYWIN:
+                m = "windows cannot do multithreading without --no-fpool, so enabling that -- note that upload performance will suffer if you have microsoft defender \"real-time protection\" enabled, so you probably want to use -j 1 instead"
+                args.no_fpool = True
+            
+            self.log("root", m, c=3)
+
         # initiate all services to manage
         self.asrv = AuthSrv(self.args, self.log)
         if args.ls:

copyparty/u2idx.py

@@ -6,9 +6,10 @@ import os
 import time
 import threading
 from datetime import datetime
+from operator import itemgetter
 
 from .__init__ import ANYWIN, unicode
-from .util import absreal, s3dec, Pebkac, min_ex, gen_filekey
+from .util import absreal, s3dec, Pebkac, min_ex, gen_filekey, quotep
 from .bos import bos
 from .up2k import up2k_wark_from_hashlist
 
@@ -253,21 +254,23 @@ class U2idx(object):
                 if rd.startswith("//") or fn.startswith("//"):
                     rd, fn = s3dec(rd, fn)
 
-                if fk:
+                if not fk:
+                    suf = ""
+                else:
                     try:
                         ap = absreal(os.path.join(ptop, rd, fn))
                         inf = bos.stat(ap)
                     except:
                         continue
 
-                    fn += (
+                    suf = (
                         "?k="
                         + gen_filekey(
                             self.args.fk_salt, ap, sz, 0 if ANYWIN else inf.st_ino
                         )[:fk]
                     )
 
-                rp = "/".join([x for x in [vtop, rd, fn] if x])
+                rp = quotep("/".join([x for x in [vtop, rd, fn] if x])) + suf
                 sret.append({"ts": int(ts), "sz": sz, "rp": rp, "w": w[:16]})
 
             for hit in sret:
@@ -290,9 +293,13 @@ class U2idx(object):
         # undupe hits from multiple metadata keys
         if len(ret) > 1:
             ret = [ret[0]] + [
-                y for x, y in zip(ret[:-1], ret[1:]) if x["rp"] != y["rp"]
+                y
+                for x, y in zip(ret[:-1], ret[1:])
+                if x["rp"].split("?")[0] != y["rp"].split("?")[0]
             ]
 
+        ret.sort(key=itemgetter("rp"))
+
         return ret, list(taglist.keys())
 
     def terminator(self, identifier, done_flag):

copyparty/up2k.py

@@ -27,6 +27,7 @@ from .util import (
     sanitize_fn,
     ren_open,
     atomic_move,
+    quotep,
     vsplit,
     s3enc,
     s3dec,
@@ -66,6 +67,7 @@ class Up2k(object):
         self.n_tagq = 0
         self.volstate = {}
         self.need_rescan = {}
+        self.dupesched = {}
         self.registry = {}
         self.entags = {}
         self.flags = {}
@@ -940,7 +942,12 @@ class Up2k(object):
 
     def _tag_file(self, write_cur, entags, wark, abspath, tags=None):
         if tags is None:
-            tags = self.mtag.get(abspath)
+            try:
+                tags = self.mtag.get(abspath)
+            except Exception as ex:
+                msg = "failed to read tags from {}:\n{}"
+                self.log(msg.format(abspath, ex), c=3)
+                return
 
         if entags:
             tags = {k: v for k, v in tags.items() if k in entags}
@@ -1112,9 +1119,18 @@ class Up2k(object):
                     if dp_dir.startswith("//") or dp_fn.startswith("//"):
                         dp_dir, dp_fn = s3dec(dp_dir, dp_fn)
 
+                    if job and (dp_dir != cj["prel"] or dp_fn != cj["name"]):
+                        continue
+
                     dp_abs = "/".join([cj["ptop"], dp_dir, dp_fn])
-                    # relying on path.exists to return false on broken symlinks
-                    if bos.path.exists(dp_abs):
+                    # relying on this to fail on broken symlinks
+                    try:
+                        sz = bos.path.getsize(dp_abs)
+                    except:
+                        sz = 0
+
+                    if sz:
+                        # self.log("--- " + wark + "  " + dp_abs + " found file", 4)
                         job = {
                             "name": dp_fn,
                             "prel": dp_dir,
@@ -1127,9 +1143,9 @@ class Up2k(object):
                             "hash": [],
                             "need": [],
                         }
-                        break
 
                 if job and wark in reg:
+                    # self.log("pop " + wark + "  " + job["name"] + " handle_json db", 4)
                     del reg[wark]
 
             if job or wark in reg:
@@ -1157,11 +1173,20 @@ class Up2k(object):
                     if job["need"]:
                         self.log("unfinished:\n  {0}\n  {1}".format(src, dst))
                         err = "partial upload exists at a different location; please resume uploading here instead:\n"
-                        err += "/" + vsrc + " "
+                        err += "/" + quotep(vsrc) + " "
+
+                        dupe = [cj["prel"], cj["name"]]
+                        try:
+                            self.dupesched[src].append(dupe)
+                        except:
+                            self.dupesched[src] = [dupe]
+
                         raise Pebkac(400, err)
+
                     elif "nodupe" in self.flags[job["ptop"]]:
                         self.log("dupe-reject:\n  {0}\n  {1}".format(src, dst))
-                        err = "upload rejected, file already exists:\n/" + vsrc + " "
+                        err = "upload rejected, file already exists:\n"
+                        err += "/" + quotep(vsrc) + " "
                         raise Pebkac(400, err)
                     else:
                         # symlink to the client-provided name,
@@ -1254,6 +1279,9 @@ class Up2k(object):
             return
 
         try:
+            if self.args.no_symlink:
+                raise Exception("disabled in config")
+
             lsrc = src
             ldst = dst
             fs1 = bos.stat(os.path.dirname(src)).st_dev
@@ -1334,20 +1362,57 @@ class Up2k(object):
                 # del self.registry[ptop][wark]
                 return ret, dst
 
-            atomic_move(src, dst)
-
-            if ANYWIN:
-                a = [dst, job["size"], (int(time.time()), int(job["lmod"]))]
-                self.lastmod_q.put(a)
-
-            a = [job[x] for x in "ptop wark prel name lmod size addr".split()]
-            a += [job.get("at") or time.time()]
-            if self.idx_wark(*a):
-                del self.registry[ptop][wark]
-                # in-memory registry is reserved for unfinished uploads
+            # windows cant rename open files
+            if not ANYWIN or src == dst:
+                self._finish_upload(ptop, wark)
 
         return ret, dst
 
+    def finish_upload(self, ptop, wark):
+        with self.mutex:
+            self._finish_upload(ptop, wark)
+
+    def _finish_upload(self, ptop, wark):
+        try:
+            job = self.registry[ptop][wark]
+            pdir = os.path.join(job["ptop"], job["prel"])
+            src = os.path.join(pdir, job["tnam"])
+            dst = os.path.join(pdir, job["name"])
+        except Exception as ex:
+            return "finish_upload, wark, " + repr(ex)
+
+        # self.log("--- " + wark + "  " + dst + " finish_upload atomic " + dst, 4)
+        atomic_move(src, dst)
+
+        if ANYWIN:
+            a = [dst, job["size"], (int(time.time()), int(job["lmod"]))]
+            self.lastmod_q.put(a)
+
+        a = [job[x] for x in "ptop wark prel name lmod size addr".split()]
+        a += [job.get("at") or time.time()]
+        if self.idx_wark(*a):
+            # self.log("pop " + wark + "  " + dst + " finish_upload idx_wark", 4)
+            del self.registry[ptop][wark]
+            # in-memory registry is reserved for unfinished uploads
+
+        dupes = self.dupesched.pop(dst, [])
+        if not dupes:
+            return
+
+        cur = self.cur.get(ptop)
+        for rd, fn in dupes:
+            d2 = os.path.join(ptop, rd, fn)
+            if os.path.exists(d2):
+                continue
+
+            self._symlink(dst, d2)
+            if cur:
+                self.db_rm(cur, rd, fn)
+                self.db_add(cur, wark, rd, fn, *a[-4:])
+
+        if cur:
+            cur.connection.commit()
+
     def idx_wark(self, ptop, wark, rd, fn, lmod, sz, ip, at):
         cur = self.cur.get(ptop)
         if not cur:
@@ -1623,7 +1688,7 @@ class Up2k(object):
                 wark = [
                     x
                     for x, y in reg.items()
-                    if fn in [y["name"], y.get("tnam")] and y["prel"] == vrem
+                    if sfn in [y["name"], y.get("tnam")] and y["prel"] == vrem
                 ]
 
             if wark and wark in reg:
@@ -1706,7 +1771,13 @@ class Up2k(object):
         except:
             cj["lmod"] = int(time.time())
 
-        wark = up2k_wark_from_hashlist(self.salt, cj["size"], cj["hash"])
+        if cj["hash"]:
+            wark = up2k_wark_from_hashlist(self.salt, cj["size"], cj["hash"])
+        else:
+            wark = up2k_wark_from_metadata(
+                self.salt, cj["size"], cj["lmod"], cj["prel"], cj["name"]
+            )
+
         return wark
 
     def _hashlist_from_file(self, path):
@@ -1749,6 +1820,8 @@ class Up2k(object):
 
         if self.args.nw:
             job["tnam"] = tnam
+            if not job["hash"]:
+                del self.registry[job["ptop"]][job["wark"]]
             return
 
         suffix = ".{:.6f}-{}".format(job["t0"], job["addr"])
@@ -1765,8 +1838,12 @@ class Up2k(object):
                 except:
                     self.log("could not sparse [{}]".format(fp), 3)
 
-            f.seek(job["size"] - 1)
-            f.write(b"e")
+            if job["hash"]:
+                f.seek(job["size"] - 1)
+                f.write(b"e")
+
+        if not job["hash"]:
+            self._finish_upload(job["ptop"], job["wark"])
 
     def _lastmodder(self):
         while True:
@@ -1864,11 +1941,16 @@ class Up2k(object):
 
             # self.log("\n  " + repr([ptop, rd, fn]))
             abspath = os.path.join(ptop, rd, fn)
-            tags = self.mtag.get(abspath)
-            ntags1 = len(tags)
-            parsers = self._get_parsers(ptop, tags, abspath)
-            if parsers:
-                tags.update(self.mtag.get_bin(parsers, abspath))
+            try:
+                tags = self.mtag.get(abspath)
+                ntags1 = len(tags)
+                parsers = self._get_parsers(ptop, tags, abspath)
+                if parsers:
+                    tags.update(self.mtag.get_bin(parsers, abspath))
+            except Exception as ex:
+                msg = "failed to read tags from {}:\n{}"
+                self.log(msg.format(abspath, ex), c=3)
+                continue
 
             with self.mutex:
                 cur = self.cur[ptop]

copyparty/util.py

@@ -251,6 +251,55 @@ class _LUnrecv(object):
 Unrecv = _Unrecv
 
 
+class FHC(object):
+    class CE(object):
+        def __init__(self, fh):
+            self.ts = 0
+            self.fhs = [fh]
+
+    def __init__(self):
+        self.cache = {}
+
+    def close(self, path):
+        try:
+            ce = self.cache[path]
+        except:
+            return
+
+        for fh in ce.fhs:
+            fh.close()
+
+        del self.cache[path]
+
+    def clean(self):
+        if not self.cache:
+            return
+
+        keep = {}
+        now = time.time()
+        for path, ce in self.cache.items():
+            if now < ce.ts + 5:
+                keep[path] = ce
+            else:
+                for fh in ce.fhs:
+                    fh.close()
+
+        self.cache = keep
+
+    def pop(self, path):
+        return self.cache[path].fhs.pop()
+
+    def put(self, path, fh):
+        try:
+            ce = self.cache[path]
+            ce.fhs.append(fh)
+        except:
+            ce = self.CE(fh)
+            self.cache[path] = ce
+
+        ce.ts = time.time()
+
+
 class ProgressPrinter(threading.Thread):
     """
     periodically print progress info without linefeeds
@@ -410,6 +459,10 @@ def log_thrs(log, ival, name):
 def vol_san(vols, txt):
     for vol in vols:
         txt = txt.replace(vol.realpath.encode("utf-8"), vol.vpath.encode("utf-8"))
+        txt = txt.replace(
+            vol.realpath.encode("utf-8").replace(b"\\", b"\\\\"),
+            vol.vpath.encode("utf-8"),
+        )
 
     return txt
 
@@ -508,8 +561,8 @@ class MultipartParser(object):
         self.log = log_func
         self.headers = http_headers
 
-        self.re_ctype = re.compile(r"^content-type: *([^;]+)", re.IGNORECASE)
-        self.re_cdisp = re.compile(r"^content-disposition: *([^;]+)", re.IGNORECASE)
+        self.re_ctype = re.compile(r"^content-type: *([^; ]+)", re.IGNORECASE)
+        self.re_cdisp = re.compile(r"^content-disposition: *([^; ]+)", re.IGNORECASE)
         self.re_cdisp_field = re.compile(
             r'^content-disposition:(?: *|.*; *)name="([^"]+)"', re.IGNORECASE
         )
@@ -708,7 +761,7 @@ class MultipartParser(object):
 def get_boundary(headers):
     # boundaries contain a-z A-Z 0-9 ' ( ) + _ , - . / : = ?
     # (whitespace allowed except as the last char)
-    ptn = r"^multipart/form-data; *(.*; *)?boundary=([^;]+)"
+    ptn = r"^multipart/form-data *; *(.*; *)?boundary=([^;]+)"
     ct = headers["content-type"]
     m = re.match(ptn, ct, re.IGNORECASE)
     if not m:

copyparty/web/browser.css

@@ -165,6 +165,7 @@ a, #files tbody div a:last-child {
 .logue {
 	padding: .2em 1.5em;
 }
+.logue.hidden,
 .logue:empty {
 	display: none;
 }
@@ -1602,7 +1603,7 @@ html.light #bbox-overlay figcaption a {
 	border-radius: .5em;
 	border-width: 1vw;
 	color: #fff;
-	transition: all 0.2s;
+	transition: all 0.12s;
 }
 #drops .dropdesc.hl.ok {
 	border-color: #fff;
@@ -1623,6 +1624,16 @@ html.light #bbox-overlay figcaption a {
 	vertical-align: middle;
 	text-align: center;
 }
+#drops .dropdesc>div>div {
+	position: absolute;
+	top: 40%;
+	top: calc(50% - .5em);
+	left: -.8em;
+}
+#drops .dropdesc>div>div+div {
+	left: auto;
+	right: -.8em;
+}
 #drops .dropzone {
 	z-index: 80386;
 	height: 50%;
@@ -1964,7 +1975,8 @@ html.light #u2foot .warn span {
 	background: #900;
 	border-color: #d06;
 }
-#u2tab a>span {
+#u2tab a>span,
+#unpost a>span {
 	font-weight: bold;
 	font-style: italic;
 	color: #fff;

copyparty/web/browser.js

@@ -70,10 +70,6 @@ ebi('op_up2k').innerHTML = (
 	'			<input type="checkbox" id="ask_up" />\n' +
 	'			<label for="ask_up" tt="ask for confirmation before upload starts">💭</label>\n' +
 	'		</td>\n' +
-	'		<td rowspan="2">\n' +
-	'			<input type="checkbox" id="flag_en" />\n' +
-	'			<label for="flag_en" tt="ensure only one tab is uploading at a time $N (other tabs must have this enabled too)">💤</label>\n' +
-	'		</td>\n' +
 	(have_up2k_idx ? (
 		'		<td data-perm="read" rowspan="2">\n' +
 		'			<input type="checkbox" id="fsearch" />\n' +
@@ -137,8 +133,8 @@ ebi('op_up2k').innerHTML = (
 	var o = mknod('div');
 	o.innerHTML = (
 		'<div id="drops">\n' +
-		'	<div class="dropdesc" id="up_zd"><div>🚀 Upload<br /><span></span></div></div>\n' +
-		'	<div class="dropdesc" id="srch_zd"><div>🔎 Search<br /><span></span></div></div>\n' +
+		'	<div class="dropdesc" id="up_zd"><div>🚀 Upload<br /><span></span><div>🚀</div><div>🚀</div></div></div>\n' +
+		'	<div class="dropdesc" id="srch_zd"><div>🔎 Search<br /><span></span><div>🔎</div><div>🔎</div></div></div>\n' +
 		'	<div class="dropzone" id="up_dz" v="up_zd"></div>\n' +
 		'	<div class="dropzone" id="srch_dz" v="srch_zd"></div>\n' +
 		'</div>'
@@ -168,6 +164,8 @@ ebi('op_cfg').innerHTML = (
 	'	<div>\n' +
 	'		<a id="u2turbo" class="tgl btn ttb" href="#" tt="the yolo button, you probably DO NOT want to enable this:$N$Nuse this if you were uploading a huge amount of files and had to restart for some reason, and want to continue the upload ASAP$N$Nthis replaces the hash-check with a simple <em>&quot;does this have the same filesize on the server?&quot;</em> so if the file contents are different it will NOT be uploaded$N$Nyou should turn this off when the upload is done, and then &quot;upload&quot; the same files again to let the client verify them">turbo</a>\n' +
 	'		<a id="u2tdate" class="tgl btn ttb" href="#" tt="has no effect unless the turbo button is enabled$N$Nreduces the yolo factor by a tiny amount; checks whether the file timestamps on the server matches yours$N$Nshould <em>theoretically</em> catch most unfinished/corrupted uploads, but is not a substitute for doing a verification pass with turbo disabled afterwards">date-chk</a>\n' +
+	'		<a id="flag_en" class="tgl btn" href="#" tt="ensure only one tab is uploading at a time $N (other tabs must have this enabled too)">💤</a>\n' +
+	'		</td>\n' +
 	'	</div>\n' +
 	'</div>\n' +
 	'<div><h3>key notation</h3><div id="key_notation"></div></div>\n' +
@@ -733,6 +731,12 @@ var pbar = (function () {
 		for (var p = 1, mins = adur / 60; p <= mins; p++)
 			pctx.fillRect(Math.floor(sm * p * 60), 0, 2, pc.h);
 
+		pctx.font = '.5em sans-serif';
+		pctx.fillStyle = light ? 'rgba(0,64,0,0.9)' : 'rgba(192,255,96,1)';
+		for (var p = 1, mins = adur / 60; p <= mins; p++) {
+			pctx.fillText(p, Math.floor(sm * p * 60 + 3), pc.h / 3);
+		}
+
 		pctx.fillStyle = light ? 'rgba(0,0,0,1)' : 'rgba(255,255,255,1)';
 		for (var p = 1, mins = adur / 600; p <= mins; p++)
 			pctx.fillRect(Math.floor(sm * p * 600), 0, 2, pc.h);
@@ -1349,7 +1353,7 @@ function play(tid, is_ev, seek, call_depth) {
 				mp.au = mp.au_ogvjs = new OGVPlayer();
 			}
 			catch (ex) {
-				return toast.err(30, 'your browser cannot play ogg/vorbis/opus\n\n' + ex +
+				return toast.err(30, 'your browser cannot play ogg/vorbis/opus\n\n' + basenames(ex) +
 					'\n\n<a href="#" onclick="new OGVPlayer();">click here</a> for a full crash report');
 			}
 			attempt_play = is_ev;
@@ -1441,7 +1445,7 @@ function play(tid, is_ev, seek, call_depth) {
 		return true;
 	}
 	catch (ex) {
-		toast.err(0, esc('playback failed: ' + ex));
+		toast.err(0, esc('playback failed: ' + basenames(ex)));
 	}
 	setclass(oid, 'play');
 	setTimeout(next_song, 500);
@@ -1475,7 +1479,7 @@ function evau_error(e) {
 
 	err += '\n\nFile: «' + uricom_dec(eplaya.src.split('/').slice(-1)[0])[0] + '»';
 
-	toast.warn(15, esc(err + ''));
+	toast.warn(15, esc(basenames(err)));
 }
 
 
@@ -1561,6 +1565,9 @@ function play_linked() {
 
 
 function sortfiles(nodes) {
+	if (!nodes.length)
+		return nodes;
+
 	var sopts = jread('fsort', [["href", 1, ""]]);
 
 	try {
@@ -2951,7 +2958,7 @@ document.onkeydown = function (e) {
 			var r = res.hits[a],
 				ts = parseInt(r.ts),
 				sz = esc(r.sz + ''),
-				rp = esc(r.rp + ''),
+				rp = esc(uricom_dec(r.rp + '')[0]),
 				ext = rp.lastIndexOf('.') > 0 ? rp.split('.').slice(-1)[0] : '%',
 				links = linksplit(r.rp + '');
 
@@ -3227,12 +3234,12 @@ var treectl = (function () {
 	}
 
 	function reload_tree() {
-		var cdir = get_evpath(),
+		var cdir = get_vpath(),
 			links = QSA('#treeul a+a'),
 			nowrap = QS('#tree.nowrap') && QS('#hovertree.on');
 
 		for (var a = 0, aa = links.length; a < aa; a++) {
-			var href = links[a].getAttribute('href');
+			var href = uricom_dec(links[a].getAttribute('href'))[0];
 			links[a].setAttribute('class', href == cdir ? 'hl' : '');
 			links[a].onclick = treego;
 			links[a].onmouseenter = nowrap ? menter : null;
@@ -3612,7 +3619,7 @@ var filecols = (function () {
 			"pixfmt": "subsampling / pixel structure",
 			"resw": "horizontal resolution",
 			"resh": "veritcal resolution",
-			"acs": "audio channels",
+			"chs": "audio channels",
 			"hz": "sample rate"
 		};
 
@@ -4078,6 +4085,106 @@ var msel = (function () {
 })();
 
 
+(function () {
+	if (!window.FormData)
+		return;
+
+	var form = QS('#op_mkdir>form'),
+		tb = QS('#op_mkdir input[name="name"]'),
+		sf = mknod('div');
+
+	clmod(sf, 'msg', 1);
+	form.parentNode.appendChild(sf);
+
+	form.onsubmit = function (e) {
+		ev(e);
+		clmod(sf, 'vis', 1);
+		sf.textContent = 'creating "' + tb.value + '"...';
+
+		var fd = new FormData();
+		fd.append("act", "mkdir");
+		fd.append("name", tb.value);
+
+		var xhr = new XMLHttpRequest();
+		xhr.vp = get_evpath();
+		xhr.dn = tb.value;
+		xhr.open('POST', xhr.vp, true);
+		xhr.onreadystatechange = cb;
+		xhr.responseType = 'text';
+		xhr.send(fd);
+
+		return false;
+	};
+
+	function cb() {
+		if (this.readyState != XMLHttpRequest.DONE)
+			return;
+
+		if (this.vp !== get_evpath()) {
+			sf.textContent = 'aborted due to location change';
+			return;
+		}
+
+		if (this.status !== 200) {
+			sf.textContent = 'error: ' + this.responseText;
+			return;
+		}
+
+		tb.value = '';
+		clmod(sf, 'vis');
+		sf.textContent = '';
+		treectl.goto(this.vp + uricom_enc(this.dn) + '/', true);
+	}
+})();
+
+
+(function () {
+	var form = QS('#op_msg>form'),
+		tb = QS('#op_msg input[name="msg"]'),
+		sf = mknod('div');
+
+	clmod(sf, 'msg', 1);
+	form.parentNode.appendChild(sf);
+
+	form.onsubmit = function (e) {
+		ev(e);
+		clmod(sf, 'vis', 1);
+		sf.textContent = 'sending...';
+
+		var xhr = new XMLHttpRequest(),
+			ct = 'application/x-www-form-urlencoded;charset=UTF-8';
+
+		xhr.msg = tb.value;
+		xhr.open('POST', get_evpath(), true);
+		xhr.responseType = 'text';
+		xhr.onreadystatechange = cb;
+		xhr.setRequestHeader('Content-Type', ct);
+		if (xhr.overrideMimeType)
+			xhr.overrideMimeType('Content-Type', ct);
+
+		xhr.send('msg=' + uricom_enc(xhr.msg));
+		return false;
+	};
+
+	function cb() {
+		if (this.readyState != XMLHttpRequest.DONE)
+			return;
+
+		if (this.status !== 200) {
+			sf.textContent = 'error: ' + this.responseText;
+			return;
+		}
+
+		tb.value = '';
+		clmod(sf, 'vis');
+		sf.textContent = 'sent: "' + this.msg + '"';
+		setTimeout(function () {
+			treectl.goto(get_evpath());
+		}, 100);
+	}
+})();
+
+
 function show_readme(md, url, depth) {
 	if (!treectl.ireadme)
 		return;
@@ -4245,7 +4352,6 @@ var unpost = (function () {
 	}
 
 	ct.onclick = function (e) {
-		ev(e);
 		var tgt = e.target.closest('a[me]');
 		if (!tgt)
 			return;
@@ -4253,6 +4359,7 @@ var unpost = (function () {
 		if (!tgt.getAttribute('href'))
 			return;
 
+		ev(e);
 		var ame = tgt.getAttribute('me');
 		if (ame != r.me)
 			return toast.err(0, 'something broke, please try a refresh');
@@ -4263,7 +4370,7 @@ var unpost = (function () {
 
 		for (var a = n; a < n2; a++)
 			if (QS('#op_unpost a.n' + a))
-				req.push(r.files[a].vp);
+				req.push(uricom_dec(r.files[a].vp)[0]);
 
 		var links = QSA('#op_unpost a.n' + n);
 		for (var a = 0, aa = links.length; a < aa; a++) {
@@ -4351,6 +4458,9 @@ function reload_browser(not_mp) {
 		makeSortable(ebi('files'), mp.read_order.bind(mp));
 	}
 
+	for (var a = 0; a < 2; a++)
+		clmod(ebi(a ? 'pro' : 'epi'), 'hidden', ebi('unsearch'));
+
 	if (window['up2k'])
 		up2k.set_fsearch();
 

copyparty/web/ui.css

@@ -258,6 +258,16 @@ html.light #pctl *:focus,
 html.light .btn:focus {
 	box-shadow: 0 .1em .2em #037 inset;
 }
+input[type="text"]:focus,
+input:not([type]):focus,
+textarea:focus {
+	box-shadow: 0 .1em .3em #fc0, 0 -.1em .3em #fc0;
+}
+html.light input[type="text"]:focus,
+html.light input:not([type]):focus,
+html.light textarea:focus {
+	box-shadow: 0 .1em .3em #037, 0 -.1em .3em #037;
+}
 
 
 

copyparty/web/up2k.js

@@ -512,9 +512,13 @@ function up2k_init(subtle) {
         // chrome<37 firefox<34 edge<12 opera<24 safari<7
         shame = 'your browser is impressively ancient';
 
-    var got_deps = false;
+    function got_deps() {
+        return subtle || window.asmCrypto || window.hashwasm;
+    }
+
+    var loading_deps = false;
     function init_deps() {
-        if (!got_deps && !subtle && !window.asmCrypto) {
+        if (!loading_deps && !got_deps()) {
             var fn = 'sha512.' + sha_js + '.js';
             showmodal('<h1>loading ' + fn + '</h1><h2>since ' + shame + '</h2><h4>thanks chrome</h4>');
             import_js('/.cpr/deps/' + fn, unmodal);
@@ -525,7 +529,7 @@ function up2k_init(subtle) {
                 ebi('u2foot').innerHTML = 'seems like ' + shame + ' so do that if you want more performance <span style="color:#' +
                     (sha_js == 'ac' ? 'c84">(expecting 20' : '8a5">(but dont worry too much, expect 100') + ' MiB/s)</span>';
         }
-        got_deps = true;
+        loading_deps = true;
     }
 
     if (perms.length && !has(perms, 'read') && has(perms, 'write'))
@@ -578,7 +582,7 @@ function up2k_init(subtle) {
 
     bcfg_bind(uc, 'multitask', 'multitask', true, null, false);
     bcfg_bind(uc, 'ask_up', 'ask_up', true, null, false);
-    bcfg_bind(uc, 'flag_en', 'flag_en', false, apply_flag_cfg, false);
+    bcfg_bind(uc, 'flag_en', 'flag_en', false, apply_flag_cfg);
     bcfg_bind(uc, 'fsearch', 'fsearch', false, set_fsearch, false);
     bcfg_bind(uc, 'turbo', 'u2turbo', false, draw_turbo, false);
     bcfg_bind(uc, 'datechk', 'u2tdate', true, null, false);
@@ -744,11 +748,14 @@ function up2k_init(subtle) {
 
         more_one_file();
         var bad_files = [],
+            nil_files = [],
             good_files = [],
             dirs = [];
 
         for (var a = 0; a < files.length; a++) {
-            var fobj = files[a];
+            var fobj = files[a],
+                dst = good_files;
+
             if (is_itemlist) {
                 if (fobj.kind !== 'file')
                     continue;
@@ -765,16 +772,15 @@ function up2k_init(subtle) {
             }
             try {
                 if (fobj.size < 1)
-                    throw 1;
+                    dst = nil_files;
             }
             catch (ex) {
-                bad_files.push(fobj.name);
-                continue;
+                dst = bad_files;
             }
-            good_files.push([fobj, fobj.name]);
+            dst.push([fobj, fobj.name]);
         }
         if (dirs) {
-            return read_dirs(null, [], dirs, good_files, bad_files);
+            return read_dirs(null, [], dirs, good_files, nil_files, bad_files);
         }
     }
 
@@ -788,7 +794,7 @@ function up2k_init(subtle) {
     }
 
     var rd_missing_ref = [];
-    function read_dirs(rd, pf, dirs, good, bad, spins) {
+    function read_dirs(rd, pf, dirs, good, nil, bad, spins) {
         spins = spins || 0;
         if (++spins == 5)
             rd_missing_ref = rd_flatten(pf, dirs);
@@ -809,7 +815,7 @@ function up2k_init(subtle) {
                     msg.push('<li>' + esc(missing[a]) + '</li>');
 
                 return modal.alert(msg.join('') + '</ul>', function () {
-                    read_dirs(rd, [], [], good, bad, spins);
+                    read_dirs(rd, [], [], good, nil, bad, spins);
                 });
             }
             spins = 0;
@@ -817,11 +823,11 @@ function up2k_init(subtle) {
 
         if (!dirs.length) {
             if (!pf.length)
-                return gotallfiles(good, bad);
+                return gotallfiles(good, nil, bad);
 
             console.log("retry pf, " + pf.length);
             setTimeout(function () {
-                read_dirs(rd, pf, dirs, good, bad, spins);
+                read_dirs(rd, pf, dirs, good, nil, bad, spins);
             }, 50);
             return;
         }
@@ -843,14 +849,15 @@ function up2k_init(subtle) {
                     pf.push(name);
                     dn.file(function (fobj) {
                         apop(pf, name);
+                        var dst = good;
                         try {
-                            if (fobj.size > 0) {
-                                good.push([fobj, name]);
-                                return;
-                            }
+                            if (fobj.size < 1)
+                                dst = nil;
                         }
-                        catch (ex) { }
-                        bad.push(name);
+                        catch (ex) {
+                            dst = bad;
+                        }
+                        dst.push([fobj, name]);
                     });
                 }
                 ngot += 1;
@@ -859,23 +866,33 @@ function up2k_init(subtle) {
                 dirs.shift();
                 rd = null;
             }
-            return read_dirs(rd, pf, dirs, good, bad, spins);
+            return read_dirs(rd, pf, dirs, good, nil, bad, spins);
         });
     }
 
-    function gotallfiles(good_files, bad_files) {
+    function gotallfiles(good_files, nil_files, bad_files) {
+        var ntot = good_files.concat(nil_files, bad_files).length;
         if (bad_files.length) {
-            var ntot = bad_files.length + good_files.length,
-                msg = 'These {0} files (of {1} total) were skipped because they are empty:\n'.format(bad_files.length, ntot);
-
+            var msg = 'These {0} files (of {1} total) were skipped, possibly due to filesystem permissions:\n'.format(bad_files.length, ntot);
             for (var a = 0, aa = Math.min(20, bad_files.length); a < aa; a++)
-                msg += '-- ' + bad_files[a] + '\n';
-
-            if (good_files.length - bad_files.length <= 1 && ANDROID)
-                msg += '\nFirefox-Android has a bug which prevents selecting multiple files. Try selecting one file at a time. For more info, see firefox bug 1456557';
+                msg += '-- ' + bad_files[a][1] + '\n';
 
+            msg += '\nMaybe it works better if you select just one file';
             return modal.alert(msg, function () {
-                gotallfiles(good_files, []);
+                gotallfiles(good_files, nil_files, []);
+            });
+        }
+
+        if (nil_files.length) {
+            var msg = 'These {0} files (of {1} total) are blank/empty; upload them anyways?\n'.format(nil_files.length, ntot);
+            for (var a = 0, aa = Math.min(20, nil_files.length); a < aa; a++)
+                msg += '-- ' + nil_files[a][1] + '\n';
+
+            msg += '\nMaybe it works better if you select just one file';
+            return modal.confirm(msg, function () {
+                gotallfiles(good_files.concat(nil_files), [], []);
+            }, function () {
+                gotallfiles(good_files, [], []);
             });
         }
 
@@ -921,7 +938,7 @@ function up2k_init(subtle) {
                 "t0": now,
                 "fobj": fobj,
                 "name": name,
-                "size": fobj.size,
+                "size": fobj.size || 0,
                 "lmod": lmod / 1000,
                 "purl": fdir,
                 "done": false,
@@ -939,14 +956,16 @@ function up2k_init(subtle) {
 
             pvis.addfile([
                 uc.fsearch ? esc(entry.name) : linksplit(
-                    uricom_dec(entry.purl)[0] + entry.name).join(' '),
+                    entry.purl + uricom_enc(entry.name)).join(' '),
                 '📐 hash',
                 ''
             ], fobj.size, draw_each);
 
             st.bytes.total += fobj.size;
             st.files.push(entry);
-            if (uc.turbo)
+            if (!entry.size)
+                push_t(st.todo.handshake, entry);
+            else if (uc.turbo)
                 push_t(st.todo.head, entry);
             else
                 push_t(st.todo.hash, entry);
@@ -1081,11 +1100,6 @@ function up2k_init(subtle) {
             st.busy.handshake.length)
             return false;
 
-        if (st.busy.handshake.length)
-            for (var n = t.n - 1; n >= t.n - parallel_uploads && n >= 0; n--)
-                if (st.files[n].t_uploading)
-                    return false;
-
         if ((uc.multitask ? 1 : 0) <
             st.todo.upload.length +
             st.busy.upload.length)
@@ -1122,7 +1136,7 @@ function up2k_init(subtle) {
             if (running)
                 return;
 
-            if (crashed)
+            if (crashed || !got_deps())
                 return defer();
 
             running = true;
@@ -1138,6 +1152,18 @@ function up2k_init(subtle) {
                         st.busy.handshake.length +
                         st.busy.upload.length;
 
+                if (was_busy && !is_busy) {
+                    for (var a = 0; a < st.files.length; a++) {
+                        var t = st.files[a];
+                        if (t.want_recheck) {
+                            t.rechecks++;
+                            t.want_recheck = false;
+                            push_t(st.todo.handshake, t);
+                        }
+                    }
+                    is_busy = st.todo.handshake.length;
+                }
+
                 if (was_busy != is_busy) {
                     was_busy = is_busy;
 
@@ -1172,6 +1198,8 @@ function up2k_init(subtle) {
                         ebi('u2etas').style.textAlign = 'left';
                     }
                     etafun();
+                    if (pvis.act == 'bz')
+                        pvis.changecard('bz');
                 }
 
                 if (flag) {
@@ -1370,7 +1398,7 @@ function up2k_init(subtle) {
                     pvis.move(t.n, 'ng');
                     apop(st.busy.hash, t);
                     st.bytes.finished += t.size;
-                    return tasker();
+                    return;
                 }
 
                 toast.err(0, 'y o u   b r o k e    i t\nfile: ' + esc(t.name + '') + '\nerror: ' + err);
@@ -1446,7 +1474,6 @@ function up2k_init(subtle) {
             console.log('head onerror, retrying', t);
             apop(st.busy.head, t);
             st.todo.head.unshift(t);
-            tasker();
         };
         function orz(e) {
             var ok = false;
@@ -1468,6 +1495,7 @@ function up2k_init(subtle) {
             }
 
             t.done = true;
+            t.fobj = null;
             st.bytes.hashed += t.size;
             st.bytes.finished += t.size;
             pvis.move(t.n, 'bz');
@@ -1511,7 +1539,6 @@ function up2k_init(subtle) {
             apop(st.busy.handshake, t);
             st.todo.handshake.unshift(t);
             t.keepalive = keepalive;
-            tasker();
         };
         function orz(e) {
             if (t.t_busied != me) {
@@ -1537,15 +1564,18 @@ function up2k_init(subtle) {
                     }
                     else {
                         smsg = 'found';
-                        var hit = response.hits[0],
-                            msg = linksplit(hit.rp).join(''),
-                            tr = unix2iso(hit.ts),
-                            tu = unix2iso(t.lmod),
-                            diff = parseInt(t.lmod) - parseInt(hit.ts),
-                            cdiff = (Math.abs(diff) <= 2) ? '3c0' : 'f0b',
-                            sdiff = '<span style="color:#' + cdiff + '">diff ' + diff;
+                        var msg = [];
+                        for (var a = 0, aa = Math.min(20, response.hits.length); a < aa; a++) {
+                            var hit = response.hits[a],
+                                tr = unix2iso(hit.ts),
+                                tu = unix2iso(t.lmod),
+                                diff = parseInt(t.lmod) - parseInt(hit.ts),
+                                cdiff = (Math.abs(diff) <= 2) ? '3c0' : 'f0b',
+                                sdiff = '<span style="color:#' + cdiff + '">diff ' + diff;
 
-                        msg += '<br /><small>' + tr + ' (srv), ' + tu + ' (You), ' + sdiff + '</span></span>';
+                            msg.push(linksplit(hit.rp).join('') + '<br /><small>' + tr + ' (srv), ' + tu + ' (You), ' + sdiff + '</small></span>');
+                        }
+                        msg = msg.join('<br />\n');
                     }
                     pvis.seth(t.n, 2, msg);
                     pvis.seth(t.n, 1, smsg);
@@ -1553,6 +1583,7 @@ function up2k_init(subtle) {
                     apop(st.busy.handshake, t);
                     st.bytes.finished += t.size;
                     t.done = true;
+                    t.fobj = null;
                     tasker();
                     return;
                 }
@@ -1563,7 +1594,7 @@ function up2k_init(subtle) {
                     console.log("server-rename [" + t.purl + "] [" + t.name + "] to [" + rsp_purl + "] [" + response.name + "]");
                     t.purl = rsp_purl;
                     t.name = response.name;
-                    pvis.seth(t.n, 0, linksplit(uricom_dec(t.purl)[0] + t.name).join(' '));
+                    pvis.seth(t.n, 0, linksplit(t.purl + uricom_enc(t.name)).join(' '));
                 }
 
                 var chunksize = get_chunksize(t.size),
@@ -1619,6 +1650,7 @@ function up2k_init(subtle) {
 
                 if (done) {
                     t.done = true;
+                    t.fobj = null;
                     st.bytes.finished += t.size - t.bytes_uploaded;
                     var spd1 = (t.size / ((t.t_hashed - t.t_hashing) / 1000.)) / (1024 * 1024.),
                         spd2 = (t.size / ((t.t_uploaded - t.t_uploading) / 1000.)) / (1024 * 1024.);
@@ -1653,13 +1685,19 @@ function up2k_init(subtle) {
                 }
 
                 st.bytes.finished += t.size;
-                if (rsp.indexOf('partial upload exists') !== -1 ||
-                    rsp.indexOf('file already exists') !== -1) {
+                var err_pend = rsp.indexOf('partial upload exists') + 1,
+                    err_dupe = rsp.indexOf('file already exists') + 1;
+
+                if (err_pend || err_dupe) {
                     err = rsp;
                     ofs = err.indexOf('\n/');
                     if (ofs !== -1) {
                         err = err.slice(0, ofs + 1) + linksplit(err.slice(ofs + 2).trimEnd()).join(' ');
                     }
+                    if (!t.rechecks && err_pend) {
+                        t.rechecks = 0;
+                        t.want_recheck = true;
+                    }
                 }
                 if (err != "") {
                     pvis.seth(t.n, 1, "ERROR");
@@ -1705,7 +1743,8 @@ function up2k_init(subtle) {
         st.busy.upload.push(upt);
 
         var npart = upt.npart,
-            t = st.files[upt.nfile];
+            t = st.files[upt.nfile],
+            tries = 0;
 
         if (!t.t_uploading)
             t.t_uploading = Date.now();
@@ -1756,8 +1795,9 @@ function up2k_init(subtle) {
                 if (crashed)
                     return;
 
-                console.log('chunkpit onerror, retrying', t);
-                do_send();
+                toast.err(9.98, "failed to upload a chunk,\n" + tries + " retries so far -- retrying in 10sec\n\n" + t.name);
+                console.log('chunkpit onerror,', ++tries, t);
+                setTimeout(do_send, 10 * 1000);
             };
             xhr.open('POST', t.purl, true);
             xhr.setRequestHeader("X-Up2k-Hash", t.hash[npart]);
@@ -1920,8 +1960,8 @@ function up2k_init(subtle) {
                 flag = up2k_flagbus();
             }
             catch (ex) {
-                toast.err(5, "not supported on your browser:\n" + ex);
-                tgl_flag_en();
+                toast.err(5, "not supported on your browser:\n" + esc(basenames(ex)));
+                bcfg_set('flag_en', false);
             }
         }
         else if (!uc.flag_en && flag) {

copyparty/web/util.js

@@ -29,18 +29,24 @@ function esc(txt) {
         }[c];
     });
 }
-window.onunhandledrejection = function (e) {
-    var err = e.reason;
-    try {
-        err += '\n' + e.reason.stack;
-    }
-    catch (e) { }
-    console.log("REJ: " + err);
-    try {
-        toast.warn(30, err);
-    }
-    catch (e) { }
-};
+function basenames(txt) {
+    return (txt + '').replace(/https?:\/\/[^ \/]+\//g, '/').replace(/js\?_=[a-zA-Z]{4}/g, 'js');
+}
+if ((document.location + '').indexOf(',rej,') + 1)
+    window.onunhandledrejection = function (e) {
+        var err = e.reason;
+        try {
+            err += '\n' + e.reason.stack;
+        }
+        catch (e) { }
+        err = basenames(err);
+        console.log("REJ: " + err);
+        try {
+            toast.warn(30, err);
+        }
+        catch (e) { }
+    };
+
 try {
     console.hist = [];
     var hook = function (t) {
@@ -151,7 +157,7 @@ function vis_exh(msg, url, lineNo, columnNo, error) {
             );
             document.head.appendChild(s);
         }
-        exbox.innerHTML = html.join('\n').replace(/https?:\/\/[^ \/]+\//g, '/').replace(/js\?_=[a-zA-Z]{4}/g, 'js').replace(/<ghi>/, 'https://github.com/9001/copyparty/issues/new?labels=bug&template=bug_report.md');
+        exbox.innerHTML = basenames(html.join('\n')).replace(/<ghi>/, 'https://github.com/9001/copyparty/issues/new?labels=bug&template=bug_report.md');
         exbox.style.display = 'block';
     }
     catch (e) {
@@ -241,7 +247,9 @@ function import_js(url, cb) {
     script.src = url;
     script.onload = cb;
     script.onerror = function () {
-        toast.err(0, 'Failed to load module:\n' + url);
+        var m = 'Failed to load module:\n' + url;
+        console.log(m);
+        toast.err(0, m);
     };
     head.appendChild(script);
 }
@@ -400,19 +408,17 @@ function linksplit(rp) {
             link = rp.slice(0, ofs + 1);
             rp = rp.slice(ofs + 1);
         }
-        var vlink = esc(link),
-            elink = uricom_enc(link);
+        var vlink = esc(uricom_dec(link)[0]);
 
         if (link.indexOf('/') !== -1) {
             vlink = vlink.slice(0, -1) + '<span>/</span>';
-            elink = elink.slice(0, -3) + '/';
         }
 
         if (!rp && q)
-            elink += q;
+            link += q;
 
-        ret.push('<a href="' + apath + elink + '">' + vlink + '</a>');
-        apath += elink;
+        ret.push('<a href="' + apath + link + '">' + vlink + '</a>');
+        apath += link;
     }
     return ret;
 }
@@ -907,6 +913,9 @@ var toast = (function () {
         if (sec)
             te = setTimeout(r.hide, sec * 1000);
 
+        if (txt.indexOf('<body>') + 1)
+            txt = txt.slice(0, txt.indexOf('<')) + ' [...]';
+
         obj.innerHTML = '<a href="#" id="toastc">x</a><div id="toastb">' + lf2br(txt) + '</div>';
         obj.className = cl;
         sec += obj.offsetWidth;
@@ -1048,7 +1057,7 @@ var modal = (function () {
     }
     function _confirm(html, cok, cng, fun) {
         cb_ok = cok;
-        cb_ng = cng === undefined ? cok : null;
+        cb_ng = cng === undefined ? cok : cng;
         cb_up = fun;
         html += '<div id="modalb">' + ok_cancel + '</div>';
         r.show(html);

docs/notes.sh

@@ -162,7 +162,7 @@ brew install python@2
 pip install virtualenv
 
 # readme toc
-cat README.md | awk 'function pr() { if (!h) {return}; if (/^ *[*!#]/||!s) {printf "%s\n",h;h=0;return}; if (/.../) {printf "%s - %s\n",h,$0;h=0}; };    /^#/{s=1;pr()} /^#* *(file indexing|install on android|dev env setup|just the sfx|complete release|optional gpl stuff)|`$/{s=0} /^#/{lv=length($1);sub(/[^ ]+ /,"");bab=$0;gsub(/ /,"-",bab); h=sprintf("%" ((lv-1)*4+1) "s [%s](#%s)", "*",$0,bab);next} !h{next} {sub(/  .*/,"");sub(/[:,]$/,"")} {pr()}' > toc; grep -E '^## readme toc' -B1000 -A2 <README.md >p1; grep -E '^## quickstart' -B2 -A999999 <README.md >p2; (cat p1; grep quickstart -A1000 <toc; cat p2) >README.md
+cat README.md | awk 'function pr() { if (!h) {return}; if (/^ *[*!#]/||!s) {printf "%s\n",h;h=0;return}; if (/.../) {printf "%s - %s\n",h,$0;h=0}; };    /^#/{s=1;pr()} /^#* *(file indexing|install on android|dev env setup|just the sfx|complete release|optional gpl stuff)|`$/{s=0} /^#/{lv=length($1);sub(/[^ ]+ /,"");bab=$0;gsub(/ /,"-",bab); h=sprintf("%" ((lv-1)*4+1) "s [%s](#%s)", "*",$0,bab);next} !h{next} {sub(/  .*/,"");sub(/[:,]$/,"")} {pr()}' > toc; grep -E '^## readme toc' -B1000 -A2 <README.md >p1; grep -E '^## quickstart' -B2 -A999999 <README.md >p2; (cat p1; grep quickstart -A1000 <toc; cat p2) >README.md; rm p1 p2 toc
 
 # fix firefox phantom breakpoints,
 # suggestions from bugtracker, doesnt work (debugger is not attachable)

setup.py

@@ -114,7 +114,7 @@ args = {
     "install_requires": ["jinja2"],
     "extras_require": {"thumbnails": ["Pillow"], "audiotags": ["mutagen"]},
     "entry_points": {"console_scripts": ["copyparty = copyparty.__main__:main"]},
-    "scripts": ["bin/copyparty-fuse.py"],
+    "scripts": ["bin/copyparty-fuse.py", "bin/up2k.py"],
     "cmdclass": {"clean2": clean2},
 }
 

tests/util.py

@@ -3,6 +3,7 @@ import sys
 import time
 import shutil
 import jinja2
+import threading
 import tempfile
 import platform
 import subprocess as sp
@@ -28,7 +29,7 @@ if MACOS:
     # 25% faster; until any tests do symlink stuff
 
 
-from copyparty.util import Unrecv
+from copyparty.util import Unrecv, FHC
 
 
 def runcmd(argv):
@@ -132,8 +133,10 @@ class VHttpConn(object):
         self.log_src = "a"
         self.lf_url = None
         self.hsrv = VHttpSrv()
+        self.u2fh = FHC()
+        self.mutex = threading.Lock()
         self.nreq = 0
         self.nbyte = 0
         self.ico = None
         self.thumbcli = None
-        self.t0 = time.time()
+        self.t0 = time.time()