v1.0.6

README.md

@@ -19,7 +19,7 @@ turn your phone or raspi into a portable file server with resumable uploads/down
 ## readme toc
 
 * top
-    * **[quickstart](#quickstart)** - download **[copyparty-sfx.py](https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.py)** and you're all set!
+    * [quickstart](#quickstart) - download **[copyparty-sfx.py](https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.py)** and you're all set!
         * [on servers](#on-servers) - you may also want these, especially on servers
         * [on debian](#on-debian) - recommended additional steps on debian
     * [notes](#notes) - general notes
@@ -61,6 +61,9 @@ turn your phone or raspi into a portable file server with resumable uploads/down
 * [performance](#performance) - defaults are usually fine - expect `8 GiB/s` download, `1 GiB/s` upload
 * [security](#security) - some notes on hardening
     * [gotchas](#gotchas) - behavior that might be unexpected
+* [recovering from crashes](#recovering-from-crashes)
+    * [client crashes](#client-crashes)
+        * [frefox wsod](#frefox-wsod) - firefox 87 can crash during uploads
 * [dependencies](#dependencies) - mandatory deps
     * [optional dependencies](#optional-dependencies) - install these to enable bonus features
     * [install recommended deps](#install-recommended-deps)
@@ -433,7 +436,7 @@ and then theres the tabs below it,
   * plus up to 3 entries each from `[done]` and `[que]` for context
 * `[que]` is all the files that are still queued
 
-note that since up2k has to read each file twice, `[🎈 bup]` can *theoretically* be up to 2x faster in some extreme cases (files bigger than your ram, combined with an internet connection faster than the read-speed of your HDD)
+note that since up2k has to read each file twice, `[🎈 bup]` can *theoretically* be up to 2x faster in some extreme cases (files bigger than your ram, combined with an internet connection faster than the read-speed of your HDD, or if you're uploading from a cuo2duo)
 
 if you are resuming a massive upload and want to skip hashing the files which already finished, you can enable `turbo` in the `[⚙️] config` tab, but please read the tooltip on that button
 
@@ -851,6 +854,26 @@ behavior that might be unexpected
 * users without read-access to a folder can still see the `.prologue.html` / `.epilogue.html` / `README.md` contents, for the purpose of showing a description on how to use the uploader for example
 
 
+# recovering from crashes
+
+## client crashes
+
+### frefox wsod
+
+firefox 87 can crash during uploads  -- the entire browser goes, including all other browser tabs, everything turns white
+
+however you can hit `F12` in the up2k tab and use the devtools to see how far you got in the uploads:
+
+* get a complete list of all uploads, organized by statuts (ok / no-good / busy / queued):  
+  `var tabs = { ok:[], ng:[], bz:[], q:[] }; for (var a of up2k.ui.tab) tabs[a.in].push(a); tabs`
+
+* list of filenames which failed:  
+  `​var ng = []; for (var a of up2k.ui.tab) if (a.in != 'ok') ng.push(a.hn.split('<a href=\"').slice(-1)[0].split('\">')[0]); ng`
+
+* send the list of filenames to copyparty for safekeeping:  
+  `await fetch('/inc', {method:'PUT', body:JSON.stringify(ng,null,1)})`
+
+
 # dependencies
 
 mandatory deps:

bin/copyparty-fuse.py

@@ -71,7 +71,7 @@ except:
     elif MACOS:
         libfuse = "install https://osxfuse.github.io/"
     else:
-        libfuse = "apt install libfuse\n    modprobe fuse"
+        libfuse = "apt install libfuse3-3\n    modprobe fuse"
 
     print(
         "\n  could not import fuse; these may help:"
@@ -393,15 +393,16 @@ class Gateway(object):
 
         rsp = json.loads(rsp.decode("utf-8"))
         ret = []
-        for is_dir, nodes in [[True, rsp["dirs"]], [False, rsp["files"]]]:
+        for statfun, nodes in [
+            [self.stat_dir, rsp["dirs"]],
+            [self.stat_file, rsp["files"]],
+        ]:
             for n in nodes:
-                fname = unquote(n["href"]).rstrip(b"/")
-                fname = fname.decode("wtf-8")
+                fname = unquote(n["href"].split("?")[0]).rstrip(b"/").decode("wtf-8")
                 if bad_good:
                     fname = enwin(fname)
 
-                fun = self.stat_dir if is_dir else self.stat_file
-                ret.append([fname, fun(n["ts"], n["sz"]), 0])
+                ret.append([fname, statfun(n["ts"], n["sz"]), 0])
 
         return ret
 

bin/up2k.sh

@@ -8,7 +8,7 @@ set -e
 ##
 ## config
 
-datalen=$((2*1024*1024*1024))
+datalen=$((128*1024*1024))
 target=127.0.0.1
 posturl=/inc
 passwd=wark
@@ -37,10 +37,10 @@ gendata() {
 # pipe a chunk, get the base64 checksum
 gethash() {
     printf $(
-        sha512sum | cut -c-64 |
+        sha512sum | cut -c-66 |
         sed -r 's/ .*//;s/(..)/\\x\1/g'
     ) |
-    base64 -w0 | cut -c-43 |
+    base64 -w0 | cut -c-44 |
     tr '+/' '-_'
 }
 
@@ -123,7 +123,7 @@ printf '\033[36m'
 {
     {
         cat <<EOF
-POST $posturl/handshake.php HTTP/1.1
+POST $posturl/ HTTP/1.1
 Connection: Close
 Cookie: cppwd=$passwd
 Content-Type: text/plain;charset=UTF-8
@@ -145,14 +145,16 @@ printf '\033[0m\nwark: %s\n' $wark
 ##
 ## wait for signal to continue
 
-w8=/dev/shm/$salt.w8
-touch $w8
+true || {
+    w8=/dev/shm/$salt.w8
+    touch $w8
 
-echo "ready;  rm -f $w8"
+    echo "ready;  rm -f $w8"
 
-while [ -e $w8 ]; do
-    sleep 0.2
-done
+    while [ -e $w8 ]; do
+        sleep 0.2
+    done
+}
 
 
 ##
@@ -175,7 +177,7 @@ while [ $remains -gt 0 ]; do
     
     {
         cat <<EOF
-POST $posturl/chunkpit.php HTTP/1.1
+POST $posturl/ HTTP/1.1
 Connection: Keep-Alive
 Cookie: cppwd=$passwd
 Content-Type: application/octet-stream

copyparty/__main__.py

@@ -344,6 +344,9 @@ def run_argparse(argv, formatter):
     ap2.add_argument("--dotpart", action="store_true", help="dotfile incomplete uploads")
     ap2.add_argument("--sparse", metavar="MiB", type=int, default=4, help="up2k min.size threshold (mswin-only)")
     ap2.add_argument("--unpost", metavar="SEC", type=int, default=3600*12, help="grace period where uploads can be deleted by the uploader, even without delete permissions; 0=disabled")
+    ap2.add_argument("--no-fpool", action="store_true", help="disable file-handle pooling -- instead, repeatedly close and reopen files during upload")
+    ap2.add_argument("--use-fpool", action="store_true", help="force file-handle pooling, even if copyparty thinks you're better off without")
+    ap2.add_argument("--no-symlink", action="store_true", help="duplicate file contents instead")
 
     ap2 = ap.add_argument_group('network options')
     ap2.add_argument("-i", metavar="IP", type=u, default="0.0.0.0", help="ip to bind (comma-sep.)")

copyparty/__version__.py

@@ -1,8 +1,8 @@
 # coding: utf-8
 
-VERSION = (1, 0, 4)
+VERSION = (1, 0, 6)
 CODENAME = "sufficient"
-BUILD_DT = (2021, 9, 19)
+BUILD_DT = (2021, 9, 26)
 
 S_VERSION = ".".join(map(str, VERSION))
 S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT)

copyparty/httpcli.py

@@ -39,6 +39,7 @@ class HttpCli(object):
     def __init__(self, conn):
         self.t0 = time.time()
         self.conn = conn
+        self.mutex = conn.mutex
         self.s = conn.s  # type: socket
         self.sr = conn.sr  # type: Unrecv
         self.ip = conn.addr[0]
@@ -47,6 +48,7 @@ class HttpCli(object):
         self.asrv = conn.asrv  # type: AuthSrv
         self.ico = conn.ico
         self.thumbcli = conn.thumbcli
+        self.u2fh = conn.u2fh
         self.log_func = conn.log_func
         self.log_src = conn.log_src
         self.tls = hasattr(self.s, "cipher")
@@ -835,7 +837,18 @@ class HttpCli(object):
 
         reader = read_socket(self.sr, remains)
 
-        with open(fsenc(path), "rb+", 512 * 1024) as f:
+        f = None
+        fpool = not self.args.no_fpool
+        if fpool:
+            with self.mutex:
+                try:
+                    f = self.u2fh.pop(path)
+                except:
+                    pass
+
+        f = f or open(fsenc(path), "rb+", 512 * 1024)
+
+        try:
             f.seek(cstart[0])
             post_sz, _, sha_b64 = hashcopy(reader, f)
 
@@ -865,22 +878,36 @@ class HttpCli(object):
                     ofs += len(buf)
 
                 self.log("clone {} done".format(cstart[0]))
+        finally:
+            if not fpool:
+                f.close()
+            else:
+                with self.mutex:
+                    self.u2fh.put(path, f)
 
         x = self.conn.hsrv.broker.put(True, "up2k.confirm_chunk", ptop, wark, chash)
         x = x.get()
         try:
-            num_left, path = x
+            num_left, fin_path = x
         except:
             self.loud_reply(x, status=500)
             return False
 
-        if not ANYWIN and num_left == 0:
+        if not num_left and fpool:
+            with self.mutex:
+                self.u2fh.close(path)
+
+        # windows cant rename open files
+        if ANYWIN and path != fin_path and not self.args.nw:
+            self.conn.hsrv.broker.put(True, "up2k.finish_upload", ptop, wark).get()
+
+        if not ANYWIN and not num_left:
             times = (int(time.time()), int(lastmod))
             self.log("no more chunks, setting times {}".format(times))
             try:
-                bos.utime(path, times)
+                bos.utime(fin_path, times)
             except:
-                self.log("failed to utime ({}, {})".format(path, times))
+                self.log("failed to utime ({}, {})".format(fin_path, times))
 
         spd = self._spd(post_sz)
         self.log("{} thank".format(spd))
@@ -1032,7 +1059,7 @@ class HttpCli(object):
                             bos.unlink(abspath)
                             raise
 
-                    files.append([sz, sha512_hex, p_file, fname])
+                    files.append([sz, sha512_hex, p_file, fname, abspath])
                     dbv, vrem = vfs.get_dbv(rem)
                     self.conn.hsrv.broker.put(
                         False,
@@ -1084,14 +1111,14 @@ class HttpCli(object):
             jmsg["error"] = errmsg
             errmsg = "ERROR: " + errmsg
 
-        for sz, sha512, ofn, lfn in files:
+        for sz, sha512, ofn, lfn, ap in files:
             vsuf = ""
             if self.can_read and "fk" in vfs.flags:
                 vsuf = "?k=" + gen_filekey(
                     self.args.fk_salt,
                     abspath,
                     sz,
-                    0 if ANYWIN else bos.stat(os.path.join(vfs.realpath, lfn)).st_ino,
+                    0 if ANYWIN or not ap else bos.stat(ap).st_ino,
                 )[: vfs.flags["fk"]]
 
             vpath = "{}/{}".format(upload_vpath, lfn).strip("/")
@@ -1755,7 +1782,7 @@ class HttpCli(object):
                 if filt and filt not in vp:
                     continue
 
-                ret.append({"vp": vp, "sz": sz, "at": at})
+                ret.append({"vp": quotep(vp), "sz": sz, "at": at})
                 if len(ret) > 3000:
                     ret.sort(key=lambda x: x["at"], reverse=True)
                     ret = ret[:2000]

copyparty/httpconn.py

@@ -32,9 +32,11 @@ class HttpConn(object):
         self.addr = addr
         self.hsrv = hsrv
 
+        self.mutex = hsrv.mutex
         self.args = hsrv.args
         self.asrv = hsrv.asrv
         self.cert_path = hsrv.cert_path
+        self.u2fh = hsrv.u2fh
 
         enth = HAVE_PIL and not self.args.no_thumb
         self.thumbcli = ThumbCli(hsrv.broker) if enth else None

copyparty/httpsrv.py

@@ -27,7 +27,7 @@ except ImportError:
     sys.exit(1)
 
 from .__init__ import E, PY2, MACOS
-from .util import spack, min_ex, start_stackmon, start_log_thrs
+from .util import FHC, spack, min_ex, start_stackmon, start_log_thrs
 from .bos import bos
 from .httpconn import HttpConn
 
@@ -50,7 +50,10 @@ class HttpSrv(object):
         self.log = broker.log
         self.asrv = broker.asrv
 
-        self.name = "httpsrv" + ("-n{}-i{:x}".format(nid, os.getpid()) if nid else "")
+        nsuf = "-{}".format(nid) if nid else ""
+        nsuf2 = "-n{}-i{:x}".format(nid, os.getpid()) if nid else ""
+
+        self.name = "hsrv" + nsuf2
         self.mutex = threading.Lock()
         self.stopping = False
 
@@ -59,6 +62,7 @@ class HttpSrv(object):
         self.tp_time = None  # latest worker collect
         self.tp_q = None if self.args.no_htp else queue.LifoQueue()
 
+        self.u2fh = FHC()
         self.srvs = []
         self.ncli = 0  # exact
         self.clients = {}  # laggy
@@ -82,11 +86,6 @@ class HttpSrv(object):
         if self.tp_q:
             self.start_threads(4)
 
-            name = "httpsrv-scaler" + ("-{}".format(nid) if nid else "")
-            t = threading.Thread(target=self.thr_scaler, name=name)
-            t.daemon = True
-            t.start()
-
         if nid:
             if self.args.stackmon:
                 start_stackmon(self.args.stackmon, nid)
@@ -94,6 +93,10 @@ class HttpSrv(object):
             if self.args.log_thrs:
                 start_log_thrs(self.log, self.args.log_thrs, nid)
 
+        t = threading.Thread(target=self.periodic, name="hsrv-pt" + nsuf)
+        t.daemon = True
+        t.start()
+
     def start_threads(self, n):
         self.tp_nthr += n
         if self.args.log_htp:
@@ -115,13 +118,15 @@ class HttpSrv(object):
         for _ in range(n):
             self.tp_q.put(None)
 
-    def thr_scaler(self):
+    def periodic(self):
         while True:
-            time.sleep(2 if self.tp_ncli else 30)
+            time.sleep(2 if self.tp_ncli else 10)
             with self.mutex:
-                self.tp_ncli = max(self.ncli, self.tp_ncli - 2)
-                if self.tp_nthr > self.tp_ncli + 8:
-                    self.stop_threads(4)
+                self.u2fh.clean()
+                if self.tp_q:
+                    self.tp_ncli = max(self.ncli, self.tp_ncli - 2)
+                    if self.tp_nthr > self.tp_ncli + 8:
+                        self.stop_threads(4)
 
     def listen(self, sck, nlisteners):
         ip, port = sck.getsockname()

copyparty/svchub.py

@@ -1,7 +1,6 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
-import re
 import os
 import sys
 import time
@@ -54,6 +53,17 @@ class SvcHub(object):
         if args.log_thrs:
             start_log_thrs(self.log, args.log_thrs, 0)
 
+        if not ANYWIN and not args.use_fpool:
+            args.no_fpool = True
+
+        if not args.no_fpool and args.j != 1:
+            m = "WARNING: --use-fpool combined with multithreading is untested and can probably cause undefined behavior"
+            if ANYWIN:
+                m = "windows cannot do multithreading without --no-fpool, so enabling that -- note that upload performance will suffer if you have microsoft defender \"real-time protection\" enabled, so you probably want to use -j 1 instead"
+                args.no_fpool = True
+            
+            self.log("root", m, c=3)
+
         # initiate all services to manage
         self.asrv = AuthSrv(self.args, self.log)
         if args.ls:
@@ -205,6 +215,8 @@ class SvcHub(object):
         if self.stopping:
             return
 
+        # start_log_thrs(print, 0.1, 1)
+
         self.stopping = True
         self.stop_req = True
         with self.stop_cond:

copyparty/u2idx.py

@@ -8,7 +8,7 @@ import threading
 from datetime import datetime
 
 from .__init__ import ANYWIN, unicode
-from .util import absreal, s3dec, Pebkac, min_ex, gen_filekey
+from .util import absreal, s3dec, Pebkac, min_ex, gen_filekey, quotep
 from .bos import bos
 from .up2k import up2k_wark_from_hashlist
 
@@ -253,21 +253,23 @@ class U2idx(object):
                 if rd.startswith("//") or fn.startswith("//"):
                     rd, fn = s3dec(rd, fn)
 
-                if fk:
+                if not fk:
+                    suf = ""
+                else:
                     try:
                         ap = absreal(os.path.join(ptop, rd, fn))
                         inf = bos.stat(ap)
                     except:
                         continue
 
-                    fn += (
+                    suf = (
                         "?k="
                         + gen_filekey(
                             self.args.fk_salt, ap, sz, 0 if ANYWIN else inf.st_ino
                         )[:fk]
                     )
 
-                rp = "/".join([x for x in [vtop, rd, fn] if x])
+                rp = quotep("/".join([x for x in [vtop, rd, fn] if x])) + suf
                 sret.append({"ts": int(ts), "sz": sz, "rp": rp, "w": w[:16]})
 
             for hit in sret:

copyparty/up2k.py

@@ -27,6 +27,7 @@ from .util import (
     sanitize_fn,
     ren_open,
     atomic_move,
+    quotep,
     vsplit,
     s3enc,
     s3dec,
@@ -66,6 +67,7 @@ class Up2k(object):
         self.n_tagq = 0
         self.volstate = {}
         self.need_rescan = {}
+        self.dupesched = {}
         self.registry = {}
         self.entags = {}
         self.flags = {}
@@ -940,7 +942,12 @@ class Up2k(object):
 
     def _tag_file(self, write_cur, entags, wark, abspath, tags=None):
         if tags is None:
-            tags = self.mtag.get(abspath)
+            try:
+                tags = self.mtag.get(abspath)
+            except Exception as ex:
+                msg = "failed to read tags from {}:\n{}"
+                self.log(msg.format(abspath, ex), c=3)
+                return
 
         if entags:
             tags = {k: v for k, v in tags.items() if k in entags}
@@ -1112,9 +1119,18 @@ class Up2k(object):
                     if dp_dir.startswith("//") or dp_fn.startswith("//"):
                         dp_dir, dp_fn = s3dec(dp_dir, dp_fn)
 
+                    if job and (dp_dir != cj["prel"] or dp_fn != cj["name"]):
+                        continue
+
                     dp_abs = "/".join([cj["ptop"], dp_dir, dp_fn])
-                    # relying on path.exists to return false on broken symlinks
-                    if bos.path.exists(dp_abs):
+                    # relying on this to fail on broken symlinks
+                    try:
+                        sz = bos.path.getsize(dp_abs)
+                    except:
+                        sz = 0
+
+                    if sz:
+                        # self.log("--- " + wark + "  " + dp_abs + " found file", 4)
                         job = {
                             "name": dp_fn,
                             "prel": dp_dir,
@@ -1127,9 +1143,9 @@ class Up2k(object):
                             "hash": [],
                             "need": [],
                         }
-                        break
 
                 if job and wark in reg:
+                    # self.log("pop " + wark + "  " + job["name"] + " handle_json db", 4)
                     del reg[wark]
 
             if job or wark in reg:
@@ -1157,11 +1173,20 @@ class Up2k(object):
                     if job["need"]:
                         self.log("unfinished:\n  {0}\n  {1}".format(src, dst))
                         err = "partial upload exists at a different location; please resume uploading here instead:\n"
-                        err += "/" + vsrc + " "
+                        err += "/" + quotep(vsrc) + " "
+
+                        dupe = [cj["prel"], cj["name"]]
+                        try:
+                            self.dupesched[src].append(dupe)
+                        except:
+                            self.dupesched[src] = [dupe]
+
                         raise Pebkac(400, err)
+
                     elif "nodupe" in self.flags[job["ptop"]]:
                         self.log("dupe-reject:\n  {0}\n  {1}".format(src, dst))
-                        err = "upload rejected, file already exists:\n/" + vsrc + " "
+                        err = "upload rejected, file already exists:\n"
+                        err += "/" + quotep(vsrc) + " "
                         raise Pebkac(400, err)
                     else:
                         # symlink to the client-provided name,
@@ -1254,6 +1279,9 @@ class Up2k(object):
             return
 
         try:
+            if self.args.no_symlink:
+                raise Exception("disabled in config")
+
             lsrc = src
             ldst = dst
             fs1 = bos.stat(os.path.dirname(src)).st_dev
@@ -1334,6 +1362,23 @@ class Up2k(object):
                 # del self.registry[ptop][wark]
                 return ret, dst
 
+        # windows cant rename open files
+        if not ANYWIN or src == dst:
+            self.finish_upload(ptop, wark)
+
+        return ret, dst
+
+    def finish_upload(self, ptop, wark):
+        with self.mutex:
+            try:
+                job = self.registry[ptop][wark]
+                pdir = os.path.join(job["ptop"], job["prel"])
+                src = os.path.join(pdir, job["tnam"])
+                dst = os.path.join(pdir, job["name"])
+            except Exception as ex:
+                return "finish_upload, wark, " + repr(ex)
+
+            # self.log("--- " + wark + "  " + dst + " finish_upload atomic " + dst, 4)
             atomic_move(src, dst)
 
             if ANYWIN:
@@ -1343,10 +1388,27 @@ class Up2k(object):
             a = [job[x] for x in "ptop wark prel name lmod size addr".split()]
             a += [job.get("at") or time.time()]
             if self.idx_wark(*a):
+                # self.log("pop " + wark + "  " + dst + " finish_upload idx_wark", 4)
                 del self.registry[ptop][wark]
                 # in-memory registry is reserved for unfinished uploads
 
-        return ret, dst
+            dupes = self.dupesched.pop(dst, [])
+            if not dupes:
+                return
+
+            cur = self.cur.get(ptop)
+            for rd, fn in dupes:
+                d2 = os.path.join(ptop, rd, fn)
+                if os.path.exists(d2):
+                    continue
+
+                self._symlink(dst, d2)
+                if cur:
+                    self.db_rm(cur, rd, fn)
+                    self.db_add(cur, wark, rd, fn, *a[-4:])
+
+            if cur:
+                cur.connection.commit()
 
     def idx_wark(self, ptop, wark, rd, fn, lmod, sz, ip, at):
         cur = self.cur.get(ptop)
@@ -1623,7 +1685,7 @@ class Up2k(object):
                 wark = [
                     x
                     for x, y in reg.items()
-                    if fn in [y["name"], y.get("tnam")] and y["prel"] == vrem
+                    if sfn in [y["name"], y.get("tnam")] and y["prel"] == vrem
                 ]
 
             if wark and wark in reg:
@@ -1864,11 +1926,16 @@ class Up2k(object):
 
             # self.log("\n  " + repr([ptop, rd, fn]))
             abspath = os.path.join(ptop, rd, fn)
-            tags = self.mtag.get(abspath)
-            ntags1 = len(tags)
-            parsers = self._get_parsers(ptop, tags, abspath)
-            if parsers:
-                tags.update(self.mtag.get_bin(parsers, abspath))
+            try:
+                tags = self.mtag.get(abspath)
+                ntags1 = len(tags)
+                parsers = self._get_parsers(ptop, tags, abspath)
+                if parsers:
+                    tags.update(self.mtag.get_bin(parsers, abspath))
+            except Exception as ex:
+                msg = "failed to read tags from {}:\n{}"
+                self.log(msg.format(abspath, ex), c=3)
+                continue
 
             with self.mutex:
                 cur = self.cur[ptop]

copyparty/util.py

@@ -251,6 +251,55 @@ class _LUnrecv(object):
 Unrecv = _Unrecv
 
 
+class FHC(object):
+    class CE(object):
+        def __init__(self, fh):
+            self.ts = 0
+            self.fhs = [fh]
+
+    def __init__(self):
+        self.cache = {}
+
+    def close(self, path):
+        try:
+            ce = self.cache[path]
+        except:
+            return
+
+        for fh in ce.fhs:
+            fh.close()
+
+        del self.cache[path]
+
+    def clean(self):
+        if not self.cache:
+            return
+
+        keep = {}
+        now = time.time()
+        for path, ce in self.cache.items():
+            if now < ce.ts + 5:
+                keep[path] = ce
+            else:
+                for fh in ce.fhs:
+                    fh.close()
+
+        self.cache = keep
+
+    def pop(self, path):
+        return self.cache[path].fhs.pop()
+
+    def put(self, path, fh):
+        try:
+            ce = self.cache[path]
+            ce.fhs.append(fh)
+        except:
+            ce = self.CE(fh)
+            self.cache[path] = ce
+
+        ce.ts = time.time()
+
+
 class ProgressPrinter(threading.Thread):
     """
     periodically print progress info without linefeeds
@@ -375,7 +424,7 @@ def stackmon(fp, ival, suffix):
 
 
 def start_log_thrs(logger, ival, nid):
-    ival = int(ival)
+    ival = float(ival)
     tname = lname = "log-thrs"
     if nid:
         tname = "logthr-n{}-i{:x}".format(nid, os.getpid())
@@ -410,6 +459,10 @@ def log_thrs(log, ival, name):
 def vol_san(vols, txt):
     for vol in vols:
         txt = txt.replace(vol.realpath.encode("utf-8"), vol.vpath.encode("utf-8"))
+        txt = txt.replace(
+            vol.realpath.encode("utf-8").replace(b"\\", b"\\\\"),
+            vol.vpath.encode("utf-8"),
+        )
 
     return txt
 
@@ -508,8 +561,8 @@ class MultipartParser(object):
         self.log = log_func
         self.headers = http_headers
 
-        self.re_ctype = re.compile(r"^content-type: *([^;]+)", re.IGNORECASE)
-        self.re_cdisp = re.compile(r"^content-disposition: *([^;]+)", re.IGNORECASE)
+        self.re_ctype = re.compile(r"^content-type: *([^; ]+)", re.IGNORECASE)
+        self.re_cdisp = re.compile(r"^content-disposition: *([^; ]+)", re.IGNORECASE)
         self.re_cdisp_field = re.compile(
             r'^content-disposition:(?: *|.*; *)name="([^"]+)"', re.IGNORECASE
         )
@@ -708,7 +761,7 @@ class MultipartParser(object):
 def get_boundary(headers):
     # boundaries contain a-z A-Z 0-9 ' ( ) + _ , - . / : = ?
     # (whitespace allowed except as the last char)
-    ptn = r"^multipart/form-data; *(.*; *)?boundary=([^;]+)"
+    ptn = r"^multipart/form-data *; *(.*; *)?boundary=([^;]+)"
     ct = headers["content-type"]
     m = re.match(ptn, ct, re.IGNORECASE)
     if not m:

copyparty/web/browser.css

@@ -1964,7 +1964,8 @@ html.light #u2foot .warn span {
 	background: #900;
 	border-color: #d06;
 }
-#u2tab a>span {
+#u2tab a>span,
+#unpost a>span {
 	font-weight: bold;
 	font-style: italic;
 	color: #fff;

copyparty/web/browser.js

@@ -70,10 +70,6 @@ ebi('op_up2k').innerHTML = (
 	'			<input type="checkbox" id="ask_up" />\n' +
 	'			<label for="ask_up" tt="ask for confirmation before upload starts">💭</label>\n' +
 	'		</td>\n' +
-	'		<td rowspan="2">\n' +
-	'			<input type="checkbox" id="flag_en" />\n' +
-	'			<label for="flag_en" tt="ensure only one tab is uploading at a time $N (other tabs must have this enabled too)">💤</label>\n' +
-	'		</td>\n' +
 	(have_up2k_idx ? (
 		'		<td data-perm="read" rowspan="2">\n' +
 		'			<input type="checkbox" id="fsearch" />\n' +
@@ -168,6 +164,8 @@ ebi('op_cfg').innerHTML = (
 	'	<div>\n' +
 	'		<a id="u2turbo" class="tgl btn ttb" href="#" tt="the yolo button, you probably DO NOT want to enable this:$N$Nuse this if you were uploading a huge amount of files and had to restart for some reason, and want to continue the upload ASAP$N$Nthis replaces the hash-check with a simple <em>&quot;does this have the same filesize on the server?&quot;</em> so if the file contents are different it will NOT be uploaded$N$Nyou should turn this off when the upload is done, and then &quot;upload&quot; the same files again to let the client verify them">turbo</a>\n' +
 	'		<a id="u2tdate" class="tgl btn ttb" href="#" tt="has no effect unless the turbo button is enabled$N$Nreduces the yolo factor by a tiny amount; checks whether the file timestamps on the server matches yours$N$Nshould <em>theoretically</em> catch most unfinished/corrupted uploads, but is not a substitute for doing a verification pass with turbo disabled afterwards">date-chk</a>\n' +
+	'		<a id="flag_en" class="tgl btn" href="#" tt="ensure only one tab is uploading at a time $N (other tabs must have this enabled too)">💤</a>\n' +
+	'		</td>\n' +
 	'	</div>\n' +
 	'</div>\n' +
 	'<div><h3>key notation</h3><div id="key_notation"></div></div>\n' +
@@ -1561,6 +1559,9 @@ function play_linked() {
 
 
 function sortfiles(nodes) {
+	if (!nodes.length)
+		return nodes;
+
 	var sopts = jread('fsort', [["href", 1, ""]]);
 
 	try {
@@ -2951,7 +2952,7 @@ document.onkeydown = function (e) {
 			var r = res.hits[a],
 				ts = parseInt(r.ts),
 				sz = esc(r.sz + ''),
-				rp = esc(r.rp + ''),
+				rp = esc(uricom_dec(r.rp + '')[0]),
 				ext = rp.lastIndexOf('.') > 0 ? rp.split('.').slice(-1)[0] : '%',
 				links = linksplit(r.rp + '');
 
@@ -3612,7 +3613,7 @@ var filecols = (function () {
 			"pixfmt": "subsampling / pixel structure",
 			"resw": "horizontal resolution",
 			"resh": "veritcal resolution",
-			"acs": "audio channels",
+			"chs": "audio channels",
 			"hz": "sample rate"
 		};
 
@@ -4078,6 +4079,106 @@ var msel = (function () {
 })();
 
 
+(function () {
+	if (!window.FormData)
+		return;
+
+	var form = QS('#op_mkdir>form'),
+		tb = QS('#op_mkdir input[name="name"]'),
+		sf = mknod('div');
+
+	clmod(sf, 'msg', 1);
+	form.parentNode.appendChild(sf);
+
+	form.onsubmit = function (e) {
+		ev(e);
+		clmod(sf, 'vis', 1);
+		sf.textContent = 'creating "' + tb.value + '"...';
+
+		var fd = new FormData();
+		fd.append("act", "mkdir");
+		fd.append("name", tb.value);
+
+		var xhr = new XMLHttpRequest();
+		xhr.vp = get_evpath();
+		xhr.dn = tb.value;
+		xhr.open('POST', xhr.vp, true);
+		xhr.onreadystatechange = cb;
+		xhr.responseType = 'text';
+		xhr.send(fd);
+
+		return false;
+	};
+
+	function cb() {
+		if (this.readyState != XMLHttpRequest.DONE)
+			return;
+
+		if (this.vp !== get_evpath()) {
+			sf.textContent = 'aborted due to location change';
+			return;
+		}
+
+		if (this.status !== 200) {
+			sf.textContent = 'error: ' + this.responseText;
+			return;
+		}
+
+		tb.value = '';
+		clmod(sf, 'vis');
+		sf.textContent = '';
+		treectl.goto(this.vp + uricom_enc(this.dn) + '/', true);
+	}
+})();
+
+
+(function () {
+	var form = QS('#op_msg>form'),
+		tb = QS('#op_msg input[name="msg"]'),
+		sf = mknod('div');
+
+	clmod(sf, 'msg', 1);
+	form.parentNode.appendChild(sf);
+
+	form.onsubmit = function (e) {
+		ev(e);
+		clmod(sf, 'vis', 1);
+		sf.textContent = 'sending...';
+
+		var xhr = new XMLHttpRequest(),
+			ct = 'application/x-www-form-urlencoded;charset=UTF-8';
+
+		xhr.msg = tb.value;
+		xhr.open('POST', get_evpath(), true);
+		xhr.responseType = 'text';
+		xhr.onreadystatechange = cb;
+		xhr.setRequestHeader('Content-Type', ct);
+		if (xhr.overrideMimeType)
+			xhr.overrideMimeType('Content-Type', ct);
+
+		xhr.send('msg=' + uricom_enc(xhr.msg));
+		return false;
+	};
+
+	function cb() {
+		if (this.readyState != XMLHttpRequest.DONE)
+			return;
+
+		if (this.status !== 200) {
+			sf.textContent = 'error: ' + this.responseText;
+			return;
+		}
+
+		tb.value = '';
+		clmod(sf, 'vis');
+		sf.textContent = 'sent: "' + this.msg + '"';
+		setTimeout(function () {
+			treectl.goto(get_evpath());
+		}, 100);
+	}
+})();
+
+
 function show_readme(md, url, depth) {
 	if (!treectl.ireadme)
 		return;
@@ -4245,7 +4346,6 @@ var unpost = (function () {
 	}
 
 	ct.onclick = function (e) {
-		ev(e);
 		var tgt = e.target.closest('a[me]');
 		if (!tgt)
 			return;
@@ -4253,6 +4353,7 @@ var unpost = (function () {
 		if (!tgt.getAttribute('href'))
 			return;
 
+		ev(e);
 		var ame = tgt.getAttribute('me');
 		if (ame != r.me)
 			return toast.err(0, 'something broke, please try a refresh');

copyparty/web/up2k.js

@@ -578,7 +578,7 @@ function up2k_init(subtle) {
 
     bcfg_bind(uc, 'multitask', 'multitask', true, null, false);
     bcfg_bind(uc, 'ask_up', 'ask_up', true, null, false);
-    bcfg_bind(uc, 'flag_en', 'flag_en', false, apply_flag_cfg, false);
+    bcfg_bind(uc, 'flag_en', 'flag_en', false, apply_flag_cfg);
     bcfg_bind(uc, 'fsearch', 'fsearch', false, set_fsearch, false);
     bcfg_bind(uc, 'turbo', 'u2turbo', false, draw_turbo, false);
     bcfg_bind(uc, 'datechk', 'u2tdate', true, null, false);
@@ -939,7 +939,7 @@ function up2k_init(subtle) {
 
             pvis.addfile([
                 uc.fsearch ? esc(entry.name) : linksplit(
-                    uricom_dec(entry.purl)[0] + entry.name).join(' '),
+                    entry.purl + uricom_enc(entry.name)).join(' '),
                 '📐 hash',
                 ''
             ], fobj.size, draw_each);
@@ -1081,11 +1081,6 @@ function up2k_init(subtle) {
             st.busy.handshake.length)
             return false;
 
-        if (st.busy.handshake.length)
-            for (var n = t.n - 1; n >= t.n - parallel_uploads && n >= 0; n--)
-                if (st.files[n].t_uploading)
-                    return false;
-
         if ((uc.multitask ? 1 : 0) <
             st.todo.upload.length +
             st.busy.upload.length)
@@ -1138,6 +1133,18 @@ function up2k_init(subtle) {
                         st.busy.handshake.length +
                         st.busy.upload.length;
 
+                if (was_busy && !is_busy) {
+                    for (var a = 0; a < st.files.length; a++) {
+                        var t = st.files[a];
+                        if (t.want_recheck) {
+                            t.rechecks++;
+                            t.want_recheck = false;
+                            push_t(st.todo.handshake, t);
+                        }
+                    }
+                    is_busy = st.todo.handshake.length;
+                }
+
                 if (was_busy != is_busy) {
                     was_busy = is_busy;
 
@@ -1172,6 +1179,8 @@ function up2k_init(subtle) {
                         ebi('u2etas').style.textAlign = 'left';
                     }
                     etafun();
+                    if (pvis.act == 'bz')
+                        pvis.changecard('bz');
                 }
 
                 if (flag) {
@@ -1370,7 +1379,7 @@ function up2k_init(subtle) {
                     pvis.move(t.n, 'ng');
                     apop(st.busy.hash, t);
                     st.bytes.finished += t.size;
-                    return tasker();
+                    return;
                 }
 
                 toast.err(0, 'y o u   b r o k e    i t\nfile: ' + esc(t.name + '') + '\nerror: ' + err);
@@ -1446,7 +1455,6 @@ function up2k_init(subtle) {
             console.log('head onerror, retrying', t);
             apop(st.busy.head, t);
             st.todo.head.unshift(t);
-            tasker();
         };
         function orz(e) {
             var ok = false;
@@ -1468,6 +1476,7 @@ function up2k_init(subtle) {
             }
 
             t.done = true;
+            t.fobj = null;
             st.bytes.hashed += t.size;
             st.bytes.finished += t.size;
             pvis.move(t.n, 'bz');
@@ -1511,7 +1520,6 @@ function up2k_init(subtle) {
             apop(st.busy.handshake, t);
             st.todo.handshake.unshift(t);
             t.keepalive = keepalive;
-            tasker();
         };
         function orz(e) {
             if (t.t_busied != me) {
@@ -1553,6 +1561,7 @@ function up2k_init(subtle) {
                     apop(st.busy.handshake, t);
                     st.bytes.finished += t.size;
                     t.done = true;
+                    t.fobj = null;
                     tasker();
                     return;
                 }
@@ -1563,7 +1572,7 @@ function up2k_init(subtle) {
                     console.log("server-rename [" + t.purl + "] [" + t.name + "] to [" + rsp_purl + "] [" + response.name + "]");
                     t.purl = rsp_purl;
                     t.name = response.name;
-                    pvis.seth(t.n, 0, linksplit(uricom_dec(t.purl)[0] + t.name).join(' '));
+                    pvis.seth(t.n, 0, linksplit(t.purl + uricom_enc(t.name)).join(' '));
                 }
 
                 var chunksize = get_chunksize(t.size),
@@ -1619,6 +1628,7 @@ function up2k_init(subtle) {
 
                 if (done) {
                     t.done = true;
+                    t.fobj = null;
                     st.bytes.finished += t.size - t.bytes_uploaded;
                     var spd1 = (t.size / ((t.t_hashed - t.t_hashing) / 1000.)) / (1024 * 1024.),
                         spd2 = (t.size / ((t.t_uploaded - t.t_uploading) / 1000.)) / (1024 * 1024.);
@@ -1653,13 +1663,19 @@ function up2k_init(subtle) {
                 }
 
                 st.bytes.finished += t.size;
-                if (rsp.indexOf('partial upload exists') !== -1 ||
-                    rsp.indexOf('file already exists') !== -1) {
+                var err_pend = rsp.indexOf('partial upload exists') + 1,
+                    err_dupe = rsp.indexOf('file already exists') + 1;
+
+                if (err_pend || err_dupe) {
                     err = rsp;
                     ofs = err.indexOf('\n/');
                     if (ofs !== -1) {
                         err = err.slice(0, ofs + 1) + linksplit(err.slice(ofs + 2).trimEnd()).join(' ');
                     }
+                    if (!t.rechecks && err_pend) {
+                        t.rechecks = 0;
+                        t.want_recheck = true;
+                    }
                 }
                 if (err != "") {
                     pvis.seth(t.n, 1, "ERROR");
@@ -1705,7 +1721,8 @@ function up2k_init(subtle) {
         st.busy.upload.push(upt);
 
         var npart = upt.npart,
-            t = st.files[upt.nfile];
+            t = st.files[upt.nfile],
+            tries = 0;
 
         if (!t.t_uploading)
             t.t_uploading = Date.now();
@@ -1756,8 +1773,9 @@ function up2k_init(subtle) {
                 if (crashed)
                     return;
 
-                console.log('chunkpit onerror, retrying', t);
-                do_send();
+                toast.err(9.98, "failed to upload a chunk,\n" + tries + " retries so far -- retrying in 10sec\n\n" + t.name);
+                console.log('chunkpit onerror,', ++tries, t);
+                setTimeout(do_send, 10 * 1000);
             };
             xhr.open('POST', t.purl, true);
             xhr.setRequestHeader("X-Up2k-Hash", t.hash[npart]);
@@ -1921,7 +1939,7 @@ function up2k_init(subtle) {
             }
             catch (ex) {
                 toast.err(5, "not supported on your browser:\n" + ex);
-                tgl_flag_en();
+                bcfg_set('flag_en', false);
             }
         }
         else if (!uc.flag_en && flag) {

copyparty/web/util.js

@@ -400,19 +400,17 @@ function linksplit(rp) {
             link = rp.slice(0, ofs + 1);
             rp = rp.slice(ofs + 1);
         }
-        var vlink = esc(link),
-            elink = uricom_enc(link);
+        var vlink = esc(uricom_dec(link)[0]);
 
         if (link.indexOf('/') !== -1) {
             vlink = vlink.slice(0, -1) + '<span>/</span>';
-            elink = elink.slice(0, -3) + '/';
         }
 
         if (!rp && q)
-            elink += q;
+            link += q;
 
-        ret.push('<a href="' + apath + elink + '">' + vlink + '</a>');
-        apath += elink;
+        ret.push('<a href="' + apath + link + '">' + vlink + '</a>');
+        apath += link;
     }
     return ret;
 }

scripts/test/smoketest.py

@@ -60,7 +60,7 @@ class Cpp(object):
                 pass
 
 
-def tc1():
+def tc1(vflags):
     ub = "http://127.0.0.1:4321/"
     td = os.path.join("srv", "smoketest")
     try:
@@ -100,17 +100,17 @@ def tc1():
     for d1 in ["r", "w", "a"]:
         pdirs.append("{}/{}".format(td, d1))
         pdirs.append("{}/{}/j".format(td, d1))
-        for d2 in ["r", "w", "a"]:
+        for d2 in ["r", "w", "a", "c"]:
             d = os.path.join(td, d1, "j", d2)
             pdirs.append(d)
             os.makedirs(d)
 
     pdirs = [x.replace("\\", "/") for x in pdirs]
     udirs = [x.split("/", 2)[2] for x in pdirs]
-    perms = [x.rstrip("j/")[-1] for x in pdirs]
+    perms = [x.rstrip("cj/")[-1] for x in pdirs]
     perms = ["rw" if x == "a" else x for x in perms]
     for pd, ud, p in zip(pdirs, udirs, perms):
-        if ud[-1] == "j":
+        if ud[-1] == "j" or ud[-1] == "c":
             continue
 
         hp = None
@@ -123,29 +123,37 @@ def tc1():
             hp = "-"
             hpaths[ud] = os.path.join(pd, ".hist")
 
-        arg = "{}:{}:{}".format(pd, ud, p, hp)
+        arg = "{}:{}:{}".format(pd, ud, p)
         if hp:
             arg += ":c,hist=" + hp
 
-        args += ["-v", arg]
+        args += ["-v", arg + vflags]
 
     # return
     cpp = Cpp(args)
     CPP.append(cpp)
     cpp.await_idle(ub, 3)
 
-    for d in udirs:
+    for d, p in zip(udirs, perms):
         vid = ovid + "\n{}".format(d).encode("utf-8")
-        try:
-            requests.post(ub + d, data={"act": "bput"}, files={"f": ("a.h264", vid)})
-        except:
-            pass
+        r = requests.post(
+            ub + d,
+            data={"act": "bput"},
+            files={"f": (d.replace("/", "") + ".h264", vid)},
+        )
+        c = r.status_code
+        if c == 200 and p not in ["w", "rw"]:
+            raise Exception("post {} with perm {} at {}".format(c, p, d))
+        elif c == 403 and p not in ["r"]:
+            raise Exception("post {} with perm {} at {}".format(c, p, d))
+        elif c not in [200, 403]:
+            raise Exception("post {} with perm {} at {}".format(c, p, d))
 
     cpp.clean()
 
     # GET permission
     for d, p in zip(udirs, perms):
-        u = "{}{}/a.h264".format(ub, d)
+        u = "{}{}/{}.h264".format(ub, d, d.replace("/", ""))
         r = requests.get(u)
         ok = bool(r)
         if ok != (p in ["rw"]):
@@ -153,14 +161,14 @@ def tc1():
 
     # stat filesystem
     for d, p in zip(pdirs, perms):
-        u = "{}/a.h264".format(d)
+        u = "{}/{}.h264".format(d, d.split("test/")[-1].replace("/", ""))
         ok = os.path.exists(u)
         if ok != (p in ["rw", "w"]):
             raise Exception("stat {} with perm {} at {}".format(ok, p, u))
 
     # GET thumbnail, vreify contents
     for d, p in zip(udirs, perms):
-        u = "{}{}/a.h264?th=j".format(ub, d)
+        u = "{}{}/{}.h264?th=j".format(ub, d, d.replace("/", ""))
         r = requests.get(u)
         ok = bool(r and r.content[:3] == b"\xff\xd8\xff")
         if ok != (p in ["rw"]):
@@ -192,9 +200,9 @@ def tc1():
     cpp.stop(True)
 
 
-def run(tc):
+def run(tc, *a):
     try:
-        tc()
+        tc(*a)
     finally:
         try:
             CPP[0].stop(False)
@@ -203,7 +211,8 @@ def run(tc):
 
 
 def main():
-    run(tc1)
+    run(tc1, "")
+    run(tc1, ":c,fk")
 
 
 if __name__ == "__main__":