v0.11.11

.vscode/launch.py

@@ -3,14 +3,12 @@
 # launches 10x faster than mspython debugpy
 # and is stoppable with ^C
 
+import re
 import os
 import sys
 import shlex
-
-sys.path.insert(0, os.getcwd())
-
 import jstyleson
-from copyparty.__main__ import main as copyparty
+import subprocess as sp
 
 with open(".vscode/launch.json", "r", encoding="utf-8") as f:
     tj = f.read()
@@ -25,11 +23,19 @@ except:
     pass
 
 argv = [os.path.expanduser(x) if x.startswith("~") else x for x in argv]
-try:
-    copyparty(["a"] + argv)
-except SystemExit as ex:
-    if ex.code:
-        raise
+
+if re.search(" -j ?[0-9]", " ".join(argv)):
+    argv = [sys.executable, "-m", "copyparty"] + argv
+    sp.check_call(argv)
+else:
+    sys.path.insert(0, os.getcwd())
+    from copyparty.__main__ import main as copyparty
+
+    try:
+        copyparty(["a"] + argv)
+    except SystemExit as ex:
+        if ex.code:
+            raise
 
 print("\n\033[32mokke\033[0m")
 sys.exit(1)

README.md

@@ -68,12 +68,16 @@ you may also want these, especially on servers:
 
 ## notes
 
+general:
+* paper-printing is affected by dark/light-mode! use lightmode for color, darkmode for grayscale
+  * because no browsers currently implement the media-query to do this properly orz
+
+browser-specific:
 * iPhone/iPad: use Firefox to download files
 * Android-Chrome: increase "parallel uploads" for higher speed (android bug)
 * Android-Firefox: takes a while to select files (their fix for ☝️)
 * Desktop-Firefox: ~~may use gigabytes of RAM if your files are massive~~ *seems to be OK now*
-* paper-printing is affected by dark/light-mode! use lightmode for color, darkmode for grayscale
-  * because no browsers currently implement the media-query to do this properly orz
+* Desktop-Firefox: may stop you from deleting folders you've uploaded until you visit `about:memory` and click `Minimize memory usage`
 
 
 ## status
@@ -397,6 +401,8 @@ quick summary of more eccentric web-browsers trying to view a directory index:
   * cross-platform python client available in [./bin/](bin/)
   * [rclone](https://rclone.org/) as client can give ~5x performance, see [./docs/rclone.md](docs/rclone.md)
 
+* sharex (screenshot utility): see [./contrib/sharex.sxcu](contrib/#sharexsxcu)
+
 copyparty returns a truncated sha512sum of your PUT/POST as base64; you can generate the same checksum locally to verify uplaods:
 
     b512(){ printf "$((sha512sum||shasum -a512)|sed -E 's/ .*//;s/(..)/\\x\1/g')"|base64|head -c43;}
@@ -520,20 +526,25 @@ in the `scripts` folder:
 
 roughly sorted by priority
 
-* separate sqlite table per tag
-* audio fingerprinting
 * readme.md as epilogue
+* single sha512 across all up2k chunks? maybe
 * reduce up2k roundtrips
   * start from a chunk index and just go
   * terminate client on bad data
-* `os.copy_file_range` for up2k cloning
-* single sha512 across all up2k chunks? maybe
-* figure out the deal with pixel3a not being connectable as hotspot
-  * pixel3a having unpredictable 3sec latency in general :||||
 
 discarded ideas
 
+* separate sqlite table per tag
+  * performance fixed by skipping some indexes (`+mt.k`)
+* audio fingerprinting
+  * only makes sense if there can be a wasm client and that doesn't exist yet (except for olaf which is agpl hence counts as not existing)
+* `os.copy_file_range` for up2k cloning
+  * almost never hit this path anyways
 * up2k partials ui
+  * feels like there isn't much point
 * cache sha512 chunks on client
+  * too dangerous
 * comment field
+  * nah
 * look into android thumbnail cache file format
+  * absolutely not

bin/copyparty-fuse.py

@@ -54,6 +54,12 @@ MACOS = platform.system() == "Darwin"
 info = log = dbg = None
 
 
+print("{} v{} @ {}".format(
+    platform.python_implementation(),
+    ".".join([str(x) for x in sys.version_info]),
+    sys.executable))
+
+
 try:
     from fuse import FUSE, FuseOSError, Operations
 except:

contrib/README.md

@@ -9,6 +9,16 @@
 * assumes the webserver and copyparty is running on the same server/IP
 * modify `10.13.1.1` as necessary if you wish to support browsers without javascript
 
+### [`sharex.sxcu`](sharex.sxcu)
+* sharex config file to upload screenshots and grab the URL
+* `RequestURL`: full URL to the target folder
+* `pw`: password (remove the `pw` line if anon-write)
+
+however if your copyparty is behind a reverse-proxy, you may want to use [`sharex-html.sxcu`](sharex-html.sxcu) instead:
+* `RequestURL`: full URL to the target folder
+* `URL`: full URL to the root folder (with trailing slash) followed by `$regex:1|1$`
+* `pw`: password (remove `Parameters` if anon-write)
+
 ### [`explorer-nothumbs-nofoldertypes.reg`](explorer-nothumbs-nofoldertypes.reg)
 * disables thumbnails and folder-type detection in windows explorer
 * makes it way faster (especially for slow/networked locations (such as copyparty-fuse))

contrib/sharex-html.sxcu

@@ -0,0 +1,19 @@
+{
+  "Version": "13.5.0",
+  "Name": "copyparty-html",
+  "DestinationType": "ImageUploader",
+  "RequestMethod": "POST",
+  "RequestURL": "http://127.0.0.1:3923/sharex",
+  "Parameters": {
+    "pw": "wark"
+  },
+  "Body": "MultipartFormData",
+  "Arguments": {
+    "act": "bput"
+  },
+  "FileFormName": "f",
+  "RegexList": [
+    "bytes // <a href=\"/([^\"]+)\""
+  ],
+  "URL": "http://127.0.0.1:3923/$regex:1|1$"
+}

contrib/sharex.sxcu

@@ -0,0 +1,17 @@
+{
+  "Version": "13.5.0",
+  "Name": "copyparty",
+  "DestinationType": "ImageUploader",
+  "RequestMethod": "POST",
+  "RequestURL": "http://127.0.0.1:3923/sharex",
+  "Parameters": {
+    "pw": "wark",
+    "j": null
+  },
+  "Body": "MultipartFormData",
+  "Arguments": {
+    "act": "bput"
+  },
+  "FileFormName": "f",
+  "URL": "$json:files[0].url$"
+}

copyparty/__main__.py

@@ -225,6 +225,20 @@ def run_argparse(argv, formatter):
             --ciphers help = available ssl/tls ciphers,
             --ssl-ver help = available ssl/tls versions,
               default is what python considers safe, usually >= TLS1
+            
+            values for --ls:
+              "USR" is a user to browse as; * is anonymous, ** is all users
+              "VOL" is a single volume to scan, default is * (all vols)
+              "FLAG" is flags;
+                "v" in addition to realpaths, print usernames and vpaths
+                "ln" only prints symlinks leaving the volume mountpoint
+                "p" exits 1 if any such symlinks are found
+                "r" resumes startup after the listing
+            examples:
+              --ls '**'          # list all files which are possible to read
+              --ls '**,*,ln'     # check for dangerous symlinks
+              --ls '**,*,ln,p,r' # check, then start normally if safe
+            \033[0m
             """
         ),
     )
@@ -288,6 +302,7 @@ def run_argparse(argv, formatter):
     ap2.add_argument("--ssl-log", metavar="PATH", help="log master secrets")
 
     ap2 = ap.add_argument_group('debug options')
+    ap2.add_argument("--ls", metavar="U[,V[,F]]", help="scan all volumes")
     ap2.add_argument("--log-conn", action="store_true", help="print tcp-server msgs")
     ap2.add_argument("--no-sendfile", action="store_true", help="disable sendfile")
     ap2.add_argument("--no-scandir", action="store_true", help="disable scandir")
@@ -363,6 +378,9 @@ def main(argv=None):
             + "  (if you crash with codec errors then that is why)"
         )
 
+    if WINDOWS and sys.version_info < (3, 6):
+        al.no_scandir = True
+
     # signal.signal(signal.SIGINT, sighandler)
 
     SvcHub(al).run()

copyparty/__version__.py

@@ -1,8 +1,8 @@
 # coding: utf-8
 
-VERSION = (0, 11, 4)
+VERSION = (0, 11, 11)
 CODENAME = "the grid"
-BUILD_DT = (2021, 6, 1)
+BUILD_DT = (2021, 6, 8)
 
 S_VERSION = ".".join(map(str, VERSION))
 S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT)

copyparty/authsrv.py

@@ -7,7 +7,7 @@ import sys
 import stat
 import threading
 
-from .__init__ import PY2, WINDOWS
+from .__init__ import WINDOWS
 from .util import IMPLICATIONS, undot, Pebkac, fsdec, fsenc, statdir, nuprint
 
 
@@ -22,7 +22,7 @@ class VFS(object):
         self.uadm = uadm  # users who are regular admins
         self.flags = flags  # config switches
         self.nodes = {}  # child nodes
-        self.all_vols = {vpath: self}  # flattened recursive
+        self.all_vols = {vpath: self} if realpath else {}  # flattened recursive
 
     def __repr__(self):
         return "VFS({})".format(
@@ -96,6 +96,7 @@ class VFS(object):
         ]
 
     def get(self, vpath, uname, will_read, will_write):
+        # type: (str, str, bool, bool) -> tuple[VFS, str]
         """returns [vfsnode,fs_remainder] if user has the requested permissions"""
         vn, rem = self._find(vpath)
 
@@ -136,6 +137,7 @@ class VFS(object):
             return os.path.realpath(rp)
 
     def ls(self, rem, uname, scandir, incl_wo=False, lstat=False):
+        # type: (str, str, bool, bool, bool) -> tuple[str, str, dict[str, VFS]]
         """return user-readable [fsdir,real,virt] items at vpath"""
         virt_vis = {}  # nodes readable by user
         abspath = self.canonical(rem)
@@ -156,13 +158,21 @@ class VFS(object):
 
         return [abspath, real, virt_vis]
 
-    def walk(self, rel, rem, uname, dots, scandir, lstat=False):
+    def walk(self, rel, rem, seen, uname, dots, scandir, lstat):
         """
         recursively yields from ./rem;
         rel is a unix-style user-defined vpath (not vfs-related)
         """
 
-        fsroot, vfs_ls, vfs_virt = self.ls(rem, uname, scandir, False, lstat)
+        fsroot, vfs_ls, vfs_virt = self.ls(
+            rem, uname, scandir, incl_wo=False, lstat=lstat
+        )
+
+        if seen and not fsroot.startswith(seen[-1]) and fsroot in seen:
+            print("bailing from symlink loop,\n  {}\n  {}".format(seen[-1], fsroot))
+            return
+
+        seen = seen[:] + [fsroot]
         rfiles = [x for x in vfs_ls if not stat.S_ISDIR(x[1].st_mode)]
         rdirs = [x for x in vfs_ls if stat.S_ISDIR(x[1].st_mode)]
 
@@ -177,7 +187,7 @@ class VFS(object):
 
             wrel = (rel + "/" + rdir).lstrip("/")
             wrem = (rem + "/" + rdir).lstrip("/")
-            for x in self.walk(wrel, wrem, uname, scandir, lstat):
+            for x in self.walk(wrel, wrem, seen, uname, dots, scandir, lstat):
                 yield x
 
         for n, vfs in sorted(vfs_virt.items()):
@@ -185,14 +195,16 @@ class VFS(object):
                 continue
 
             wrel = (rel + "/" + n).lstrip("/")
-            for x in vfs.walk(wrel, "", uname, scandir, lstat):
+            for x in vfs.walk(wrel, "", seen, uname, dots, scandir, lstat):
                 yield x
 
     def zipgen(self, vrem, flt, uname, dots, scandir):
         if flt:
             flt = {k: True for k in flt}
 
-        for vpath, apath, files, rd, vd in self.walk("", vrem, uname, dots, scandir):
+        for vpath, apath, files, rd, vd in self.walk(
+            "", vrem, [], uname, dots, scandir, False
+        ):
             if flt:
                 files = [x for x in files if x[0] in flt]
 
@@ -228,21 +240,19 @@ class VFS(object):
             for f in [{"vp": v, "ap": a, "st": n[1]} for v, a, n in files]:
                 yield f
 
-    def user_tree(self, uname, readable=False, writable=False, admin=False):
-        ret = []
-        opt1 = readable and (uname in self.uread or "*" in self.uread)
-        opt2 = writable and (uname in self.uwrite or "*" in self.uwrite)
-        if admin:
-            if opt1 and opt2:
-                ret.append(self.vpath)
-        else:
-            if opt1 or opt2:
-                ret.append(self.vpath)
+    def user_tree(self, uname, readable, writable, admin):
+        is_readable = False
+        if uname in self.uread or "*" in self.uread:
+            readable.append(self.vpath)
+            is_readable = True
+
+        if uname in self.uwrite or "*" in self.uwrite:
+            writable.append(self.vpath)
+            if is_readable:
+                admin.append(self.vpath)
 
         for _, vn in sorted(self.nodes.items()):
-            ret.extend(vn.user_tree(uname, readable, writable, admin))
-
-        return ret
+            vn.user_tree(uname, readable, writable, admin)
 
 
 class AuthSrv(object):
@@ -420,7 +430,7 @@ class AuthSrv(object):
             vfs = VFS(os.path.abspath("."), "", ["*"], ["*"])
         elif "" not in mount:
             # there's volumes but no root; make root inaccessible
-            vfs = VFS(os.path.abspath("."), "")
+            vfs = VFS(None, "")
             vfs.flags["d2d"] = True
 
         maxdepth = 0
@@ -559,3 +569,90 @@ class AuthSrv(object):
 
         # import pprint
         # pprint.pprint({"usr": user, "rd": mread, "wr": mwrite, "mnt": mount})
+
+    def dbg_ls(self):
+        users = self.args.ls
+        vols = "*"
+        flags = []
+
+        try:
+            users, vols = users.split(",", 1)
+        except:
+            pass
+
+        try:
+            vols, flags = vols.split(",", 1)
+            flags = flags.split(",")
+        except:
+            pass
+
+        if users == "**":
+            users = list(self.user.keys()) + ["*"]
+        else:
+            users = [users]
+
+        for u in users:
+            if u not in self.user and u != "*":
+                raise Exception("user not found: " + u)
+
+        if vols == "*":
+            vols = ["/" + x for x in self.vfs.all_vols.keys()]
+        else:
+            vols = [vols]
+
+        for v in vols:
+            if not v.startswith("/"):
+                raise Exception("volumes must start with /")
+
+            if v[1:] not in self.vfs.all_vols:
+                raise Exception("volume not found: " + v)
+
+        self.log({"users": users, "vols": vols, "flags": flags})
+        for k, v in self.vfs.all_vols.items():
+            self.log("/{}: read({}) write({})".format(k, v.uread, v.uwrite))
+
+        flag_v = "v" in flags
+        flag_ln = "ln" in flags
+        flag_p = "p" in flags
+        flag_r = "r" in flags
+
+        n_bads = 0
+        for v in vols:
+            v = v[1:]
+            vtop = "/{}/".format(v) if v else "/"
+            for u in users:
+                self.log("checking /{} as {}".format(v, u))
+                try:
+                    vn, _ = self.vfs.get(v, u, True, False)
+                except:
+                    continue
+
+                atop = vn.realpath
+                g = vn.walk("", "", [], u, True, not self.args.no_scandir, False)
+                for vpath, apath, files, _, _ in g:
+                    fnames = [n[0] for n in files]
+                    vpaths = [vpath + "/" + n for n in fnames] if vpath else fnames
+                    vpaths = [vtop + x for x in vpaths]
+                    apaths = [os.path.join(apath, n) for n in fnames]
+                    files = [[vpath + "/", apath + os.sep]] + list(zip(vpaths, apaths))
+
+                    if flag_ln:
+                        files = [x for x in files if not x[1].startswith(atop + os.sep)]
+                        n_bads += len(files)
+
+                    if flag_v:
+                        msg = [
+                            '# user "{}", vpath "{}"\n{}'.format(u, vp, ap)
+                            for vp, ap in files
+                        ]
+                    else:
+                        msg = [x[1] for x in files]
+
+                    if msg:
+                        nuprint("\n".join(msg))
+
+                if n_bads and flag_p:
+                    raise Exception("found symlink leaving volume, and strict is set")
+
+        if not flag_r:
+            sys.exit(0)

copyparty/broker_mp.py

@@ -44,7 +44,9 @@ class BrokerMp(object):
             proc.clients = {}
             proc.workload = 0
 
-            thr = threading.Thread(target=self.collector, args=(proc,))
+            thr = threading.Thread(
+                target=self.collector, args=(proc,), name="mp-collector"
+            )
             thr.daemon = True
             thr.start()
 
@@ -52,14 +54,19 @@ class BrokerMp(object):
             proc.start()
 
         if not self.args.q:
-            thr = threading.Thread(target=self.debug_load_balancer)
+            thr = threading.Thread(
+                target=self.debug_load_balancer, name="mp-dbg-loadbalancer"
+            )
             thr.daemon = True
             thr.start()
 
     def shutdown(self):
         self.log("broker", "shutting down")
-        for proc in self.procs:
-            thr = threading.Thread(target=proc.q_pend.put([0, "shutdown", []]))
+        for n, proc in enumerate(self.procs):
+            thr = threading.Thread(
+                target=proc.q_pend.put([0, "shutdown", []]),
+                name="mp-shutdown-{}-{}".format(n, len(self.procs)),
+            )
             thr.start()
 
         with self.mutex:

copyparty/broker_mpw.py

@@ -27,7 +27,7 @@ class MpWorker(object):
         self.retpend = {}
         self.retpend_mutex = threading.Lock()
         self.mutex = threading.Lock()
-        self.workload_thr_active = False
+        self.workload_thr_alive = False
 
         # we inherited signal_handler from parent,
         # replace it with something harmless
@@ -35,12 +35,12 @@ class MpWorker(object):
             signal.signal(signal.SIGINT, self.signal_handler)
 
         # instantiate all services here (TODO: inheritance?)
-        self.httpsrv = HttpSrv(self)
+        self.httpsrv = HttpSrv(self, True)
         self.httpsrv.disconnect_func = self.httpdrop
 
         # on winxp and some other platforms,
         # use thr.join() to block all signals
-        thr = threading.Thread(target=self.main)
+        thr = threading.Thread(target=self.main, name="mpw-main")
         thr.daemon = True
         thr.start()
         thr.join()
@@ -79,9 +79,11 @@ class MpWorker(object):
                 self.httpsrv.accept(sck, addr)
 
                 with self.mutex:
-                    if not self.workload_thr_active:
+                    if not self.workload_thr_alive:
                         self.workload_thr_alive = True
-                        thr = threading.Thread(target=self.thr_workload)
+                        thr = threading.Thread(
+                            target=self.thr_workload, name="mpw-workload"
+                        )
                         thr.daemon = True
                         thr.start()
 

copyparty/httpcli.py

@@ -16,6 +16,7 @@ import calendar
 
 from .__init__ import E, PY2, WINDOWS, ANYWIN
 from .util import *  # noqa  # pylint: disable=unused-wildcard-import
+from .authsrv import AuthSrv
 from .szip import StreamZip
 from .star import StreamTar
 
@@ -35,12 +36,13 @@ class HttpCli(object):
     def __init__(self, conn):
         self.t0 = time.time()
         self.conn = conn
-        self.s = conn.s
-        self.sr = conn.sr
+        self.s = conn.s  # type: socket
+        self.sr = conn.sr  # type: Unrecv
         self.ip = conn.addr[0]
-        self.addr = conn.addr
+        self.addr = conn.addr  # type: tuple[str, int]
         self.args = conn.args
-        self.auth = conn.auth
+        self.is_mp = conn.is_mp
+        self.auth = conn.auth  # type: AuthSrv
         self.ico = conn.ico
         self.thumbcli = conn.thumbcli
         self.log_func = conn.log_func
@@ -153,10 +155,8 @@ class HttpCli(object):
 
         pwd = uparam.get("pw")
         self.uname = self.auth.iuser.get(pwd, "*")
-        if self.uname:
-            self.rvol = self.auth.vfs.user_tree(self.uname, readable=True)
-            self.wvol = self.auth.vfs.user_tree(self.uname, writable=True)
-            self.avol = self.auth.vfs.user_tree(self.uname, True, True, True)
+        self.rvol, self.wvol, self.avol = [[], [], []]
+        self.auth.vfs.user_tree(self.uname, self.rvol, self.wvol, self.avol)
 
         ua = self.headers.get("user-agent", "")
         self.is_rclone = ua.startswith("rclone/")
@@ -508,6 +508,7 @@ class HttpCli(object):
         items = items.replace("\r", "").split("\n")
         items = [unquotep(x) for x in items if items]
 
+        self.parser.drop()
         return self.tx_zip(k, v, vn, rem, items, self.args.ed)
 
     def handle_post_json(self):
@@ -851,14 +852,28 @@ class HttpCli(object):
             status = "ERROR"
 
         msg = "{} // {} bytes // {:.3f} MiB/s\n".format(status, sz_total, spd)
+        jmsg = {"status": status, "sz": sz_total, "mbps": round(spd, 3), "files": []}
 
         for sz, sha512, ofn, lfn in files:
-            vpath = self.vpath + "/" + lfn
+            vpath = (self.vpath + "/" if self.vpath else "") + lfn
             msg += 'sha512: {} // {} bytes // <a href="/{}">{}</a>\n'.format(
                 sha512[:56], sz, quotep(vpath), html_escape(ofn, crlf=True)
             )
             # truncated SHA-512 prevents length extension attacks;
             # using SHA-512/224, optionally SHA-512/256 = :64
+            jpart = {
+                "url": "{}://{}/{}".format(
+                    "https" if self.tls else "http",
+                    self.headers.get("host", "copyparty"),
+                    vpath,
+                ),
+                "sha512": sha512[:56],
+                "sz": sz,
+                "fn": lfn,
+                "fn_orig": ofn,
+                "path": vpath,
+            }
+            jmsg["files"].append(jpart)
 
         vspd = self._spd(sz_total, False)
         self.log("{} {}".format(vspd, msg))
@@ -870,7 +885,12 @@ class HttpCli(object):
                 ft = "{}\n{}\n{}\n".format(ft, msg.rstrip(), errmsg)
                 f.write(ft.encode("utf-8"))
 
-        self.redirect(self.vpath, msg=msg, flavor="return to", click=False)
+        if "j" in self.uparam:
+            jtxt = json.dumps(jmsg, indent=2, sort_keys=True)
+            self.reply(jtxt.encode("utf-8", "replace"), mime="application/json")
+        else:
+            self.redirect(self.vpath, msg=msg, flavor="return to", click=False)
+
         self.parser.drop()
         return True
 
@@ -974,6 +994,8 @@ class HttpCli(object):
         cli_lastmod = self.headers.get("if-modified-since")
         if cli_lastmod:
             try:
+                # some browser append "; length=573"
+                cli_lastmod = cli_lastmod.split(";")[0].strip()
                 cli_dt = time.strptime(cli_lastmod, HTTP_TS_FMT)
                 cli_ts = calendar.timegm(cli_dt)
                 return file_lastmod, int(file_ts) > int(cli_ts)
@@ -1143,7 +1165,8 @@ class HttpCli(object):
             if use_sendfile:
                 remains = sendfile_kern(lower, upper, f, self.s)
             else:
-                remains = sendfile_py(lower, upper, f, self.s)
+                actor = self.conn if self.is_mp else None
+                remains = sendfile_py(lower, upper, f, self.s, actor)
 
         if remains > 0:
             logmsg += " \033[31m" + unicode(upper - remains) + "\033[0m"
@@ -1359,15 +1382,31 @@ class HttpCli(object):
         if self.args.no_stack:
             raise Pebkac(403, "disabled by argv")
 
-        ret = []
+        threads = {}
         names = dict([(t.ident, t.name) for t in threading.enumerate()])
         for tid, stack in sys._current_frames().items():
-            ret.append("\n\n# {} ({:x})".format(names.get(tid), tid))
+            name = "{} ({:x})".format(names.get(tid), tid)
+            threads[name] = stack
+
+        rret = []
+        bret = []
+        for name, stack in sorted(threads.items()):
+            ret = ["\n\n# {}".format(name)]
+            pad = None
             for fn, lno, name, line in traceback.extract_stack(stack):
+                fn = os.sep.join(fn.split(os.sep)[-3:])
                 ret.append('File: "{}", line {}, in {}'.format(fn, lno, name))
                 if line:
                     ret.append("  " + str(line.strip()))
+                    if "self.not_empty.wait()" in line:
+                        pad = " " * 4
 
+            if pad:
+                bret += [ret[0]] + [pad + x for x in ret[1:]]
+            else:
+                rret += ret
+
+        ret = rret + bret
         ret = ("<pre>" + "\n".join(ret)).encode("utf-8")
         self.reply(ret)
 
@@ -1401,7 +1440,7 @@ class HttpCli(object):
         try:
             vn, rem = self.auth.vfs.get(top, self.uname, True, False)
             fsroot, vfs_ls, vfs_virt = vn.ls(
-                rem, self.uname, not self.args.no_scandir, True
+                rem, self.uname, not self.args.no_scandir, incl_wo=True
             )
         except:
             vfs_ls = []
@@ -1568,7 +1607,7 @@ class HttpCli(object):
                 return self.tx_zip(k, v, vn, rem, [], self.args.ed)
 
         fsroot, vfs_ls, vfs_virt = vn.ls(
-            rem, self.uname, not self.args.no_scandir, True
+            rem, self.uname, not self.args.no_scandir, incl_wo=True
         )
         stats = {k: v for k, v in vfs_ls}
         vfs_ls = [x[0] for x in vfs_ls]

copyparty/httpconn.py

@@ -35,6 +35,7 @@ class HttpConn(object):
 
         self.args = hsrv.args
         self.auth = hsrv.auth
+        self.is_mp = hsrv.is_mp
         self.cert_path = hsrv.cert_path
 
         enth = HAVE_PIL and not self.args.no_thumb
@@ -174,6 +175,11 @@ class HttpConn(object):
             self.sr = Unrecv(self.s)
 
         while True:
+            if self.is_mp:
+                self.workload += 50
+                if self.workload >= 2 ** 31:
+                    self.workload = 100
+
             cli = HttpCli(self)
             if not cli.run():
                 return

copyparty/httpsrv.py

@@ -25,8 +25,8 @@ except ImportError:
     sys.exit(1)
 
 from .__init__ import E, MACOS
-from .httpconn import HttpConn
 from .authsrv import AuthSrv
+from .httpconn import HttpConn
 
 
 class HttpSrv(object):
@@ -35,8 +35,9 @@ class HttpSrv(object):
     relying on MpSrv for performance (HttpSrv is just plain threads)
     """
 
-    def __init__(self, broker):
+    def __init__(self, broker, is_mp=False):
         self.broker = broker
+        self.is_mp = is_mp
         self.args = broker.args
         self.log = broker.log
 
@@ -66,7 +67,11 @@ class HttpSrv(object):
         if self.args.log_conn:
             self.log("%s %s" % addr, "|%sC-cthr" % ("-" * 5,), c="1;30")
 
-        thr = threading.Thread(target=self.thr_client, args=(sck, addr))
+        thr = threading.Thread(
+            target=self.thr_client,
+            args=(sck, addr),
+            name="httpsrv-{}-{}".format(addr[0].split(".", 2)[-1][-6:], addr[1]),
+        )
         thr.daemon = True
         thr.start()
 
@@ -84,13 +89,16 @@ class HttpSrv(object):
         cli = HttpConn(sck, addr, self)
         with self.mutex:
             self.clients[cli] = 0
-            self.workload += 50
 
-            if not self.workload_thr_alive:
-                self.workload_thr_alive = True
-                thr = threading.Thread(target=self.thr_workload)
-                thr.daemon = True
-                thr.start()
+            if self.is_mp:
+                self.workload += 50
+                if not self.workload_thr_alive:
+                    self.workload_thr_alive = True
+                    thr = threading.Thread(
+                        target=self.thr_workload, name="httpsrv-workload"
+                    )
+                    thr.daemon = True
+                    thr.start()
 
         try:
             if self.args.log_conn:
@@ -99,6 +107,7 @@ class HttpSrv(object):
             cli.run()
 
         finally:
+            sck = cli.s
             if self.args.log_conn:
                 self.log("%s %s" % addr, "|%sC-cdone" % ("-" * 7,), c="1;30")
 

copyparty/ico.py

@@ -1,3 +1,6 @@
+# coding: utf-8
+from __future__ import print_function, unicode_literals
+
 import hashlib
 import colorsys
 

copyparty/star.py

@@ -1,3 +1,6 @@
+# coding: utf-8
+from __future__ import print_function, unicode_literals
+
 import os
 import tarfile
 import threading
@@ -42,7 +45,7 @@ class StreamTar(object):
         fmt = tarfile.GNU_FORMAT
         self.tar = tarfile.open(fileobj=self.qfile, mode="w|", format=fmt)
 
-        w = threading.Thread(target=self._gen)
+        w = threading.Thread(target=self._gen, name="star-gen")
         w.daemon = True
         w.start()
 

copyparty/sutil.py

@@ -1,3 +1,6 @@
+# coding: utf-8
+from __future__ import print_function, unicode_literals
+
 import os
 import time
 import tempfile

copyparty/svchub.py

@@ -39,6 +39,8 @@ class SvcHub(object):
 
         # jank goes here
         auth = AuthSrv(self.args, self.log, False)
+        if args.ls:
+            auth.dbg_ls()
 
         # initiate all services to manage
         self.tcpsrv = TcpSrv(self)
@@ -69,7 +71,7 @@ class SvcHub(object):
         self.broker = Broker(self)
 
     def run(self):
-        thr = threading.Thread(target=self.tcpsrv.run)
+        thr = threading.Thread(target=self.tcpsrv.run, name="svchub-main")
         thr.daemon = True
         thr.start()
 
@@ -93,9 +95,11 @@ class SvcHub(object):
                         break
 
                     if n == 3:
-                        print("waiting for thumbsrv...")
+                        print("waiting for thumbsrv (10sec)...")
 
-            print("nailed it")
+            print("nailed it", end="")
+        finally:
+            print("\033[0m")
 
     def _log_disabled(self, src, msg, c=0):
         pass

copyparty/szip.py

@@ -1,3 +1,6 @@
+# coding: utf-8
+from __future__ import print_function, unicode_literals
+
 import os
 import time
 import zlib

copyparty/th_cli.py

@@ -1,3 +1,6 @@
+# coding: utf-8
+from __future__ import print_function, unicode_literals
+
 import os
 import time
 

copyparty/th_srv.py

@@ -1,3 +1,6 @@
+# coding: utf-8
+from __future__ import print_function, unicode_literals
+
 import os
 import sys
 import time
@@ -114,8 +117,10 @@ class ThumbSrv(object):
         self.stopping = False
         self.nthr = os.cpu_count() if hasattr(os, "cpu_count") else 4
         self.q = Queue(self.nthr * 4)
-        for _ in range(self.nthr):
-            t = threading.Thread(target=self.worker)
+        for n in range(self.nthr):
+            t = threading.Thread(
+                target=self.worker, name="thumb-{}-{}".format(n, self.nthr)
+            )
             t.daemon = True
             t.start()
 
@@ -129,9 +134,9 @@ class ThumbSrv(object):
 
             msg = "cannot create video thumbnails because some of the required programs are not available: "
             msg += ", ".join(missing)
-            self.log(msg, c=1)
+            self.log(msg, c=3)
 
-        t = threading.Thread(target=self.cleaner)
+        t = threading.Thread(target=self.cleaner, name="thumb-cleaner")
         t.daemon = True
         t.start()
 

copyparty/u2idx.py

@@ -126,10 +126,11 @@ class U2idx(object):
                     joins += "inner join mt mt{} on {} = {} ".format(
                         mt_ctr, mt_keycmp, mt_keycmp2
                     )
+                    mt_keycmp = mt_keycmp2
                     if v == "tags":
                         v = "mt{0}.v".format(mt_ctr)
                     else:
-                        v = "mt{0}.k = '{1}' and mt{0}.v".format(mt_ctr, v)
+                        v = "+mt{0}.k = '{1}' and mt{0}.v".format(mt_ctr, v)
 
                 else:
                     raise Pebkac(400, "invalid key [" + v + "]")
@@ -191,11 +192,12 @@ class U2idx(object):
                 self.active_id,
                 done_flag,
             ),
+            name="u2idx-terminator",
         )
         thr.daemon = True
         thr.start()
 
-        if not uq:
+        if not uq or not uv:
             q = "select * from up"
             v = ()
         else:

copyparty/up2k.py

@@ -83,7 +83,7 @@ class Up2k(object):
         if ANYWIN:
             # usually fails to set lastmod too quickly
             self.lastmod_q = Queue()
-            thr = threading.Thread(target=self._lastmodder)
+            thr = threading.Thread(target=self._lastmodder, name="up2k-lastmod")
             thr.daemon = True
             thr.start()
 
@@ -96,7 +96,9 @@ class Up2k(object):
         if self.args.no_fastboot:
             self.deferred_init(all_vols)
         else:
-            t = threading.Thread(target=self.deferred_init, args=(all_vols,))
+            t = threading.Thread(
+                target=self.deferred_init, args=(all_vols,), name="up2k-deferred-init"
+            )
             t.daemon = True
             t.start()
 
@@ -104,20 +106,20 @@ class Up2k(object):
         have_e2d = self.init_indexes(all_vols)
 
         if have_e2d:
-            thr = threading.Thread(target=self._snapshot)
+            thr = threading.Thread(target=self._snapshot, name="up2k-snapshot")
             thr.daemon = True
             thr.start()
 
-            thr = threading.Thread(target=self._hasher)
+            thr = threading.Thread(target=self._hasher, name="up2k-hasher")
             thr.daemon = True
             thr.start()
 
             if self.mtag:
-                thr = threading.Thread(target=self._tagger)
+                thr = threading.Thread(target=self._tagger, name="up2k-tagger")
                 thr.daemon = True
                 thr.start()
 
-                thr = threading.Thread(target=self._run_all_mtp)
+                thr = threading.Thread(target=self._run_all_mtp, name="up2k-mtp-init")
                 thr.daemon = True
                 thr.start()
 
@@ -132,7 +134,11 @@ class Up2k(object):
             return "cannot initiate; scan is already in progress"
 
         args = (all_vols, scan_vols)
-        t = threading.Thread(target=self.init_indexes, args=args)
+        t = threading.Thread(
+            target=self.init_indexes,
+            args=args,
+            name="up2k-rescan-{}".format(scan_vols[0]),
+        )
         t.daemon = True
         t.start()
         return None
@@ -186,12 +192,14 @@ class Up2k(object):
                 self.log("cannot access " + vol.realpath, c=1)
                 continue
 
+            if scan_vols and vol.vpath not in scan_vols:
+                continue
+
             if not self.register_vpath(vol.realpath, vol.flags):
                 # self.log("db not enabled for {}".format(m, vol.realpath))
                 continue
 
-            if vol.vpath in scan_vols or not scan_vols:
-                live_vols.append(vol)
+            live_vols.append(vol)
 
             if vol.vpath not in self.volstate:
                 self.volstate[vol.vpath] = "OFFLINE (pending initialization)"
@@ -271,7 +279,7 @@ class Up2k(object):
         if self.mtag:
             m = "online (running mtp)"
             if scan_vols:
-                thr = threading.Thread(target=self._run_all_mtp)
+                thr = threading.Thread(target=self._run_all_mtp, name="up2k-mtp-scan")
                 thr.daemon = True
         else:
             del self.pp
@@ -288,7 +296,10 @@ class Up2k(object):
     def register_vpath(self, ptop, flags):
         db_path = os.path.join(ptop, ".hist", "up2k.db")
         if ptop in self.registry:
-            return [self.cur[ptop], db_path]
+            try:
+                return [self.cur[ptop], db_path]
+            except:
+                return None
 
         _, flags = self._expr_idx_filter(flags)
 
@@ -370,7 +381,8 @@ class Up2k(object):
         self.pp.msg = "a{} {}".format(self.pp.n, cdir)
         histdir = os.path.join(top, ".hist")
         ret = 0
-        for iname, inf in statdir(self.log, not self.args.no_scandir, False, cdir):
+        g = statdir(self.log, not self.args.no_scandir, False, cdir)
+        for iname, inf in sorted(g):
             abspath = os.path.join(cdir, iname)
             lmod = int(inf.st_mtime)
             if stat.S_ISDIR(inf.st_mode):
@@ -552,9 +564,10 @@ class Up2k(object):
                     last_write = time.time()
                     n_buf = 0
 
-            self._stop_mpool(mpool)
-            with self.mutex:
-                n_add += len(self._flush_mpool(c3))
+            if mpool:
+                self._stop_mpool(mpool)
+                with self.mutex:
+                    n_add += len(self._flush_mpool(c3))
 
             conn.commit()
             c3.close()
@@ -752,7 +765,9 @@ class Up2k(object):
 
         mpool = Queue(nw)
         for _ in range(nw):
-            thr = threading.Thread(target=self._tag_thr, args=(mpool,))
+            thr = threading.Thread(
+                target=self._tag_thr, args=(mpool,), name="up2k-mpool"
+            )
             thr.daemon = True
             thr.start()
 

copyparty/util.py

@@ -193,7 +193,7 @@ class ProgressPrinter(threading.Thread):
     """
 
     def __init__(self):
-        threading.Thread.__init__(self)
+        threading.Thread.__init__(self, name="pp")
         self.daemon = True
         self.msg = None
         self.end = False
@@ -208,6 +208,8 @@ class ProgressPrinter(threading.Thread):
 
             msg = self.msg
             uprint(" {}\033[K\r".format(msg))
+            if PY2:
+                sys.stdout.flush()
 
         print("\033[K", end="")
         sys.stdout.flush()  # necessary on win10 even w/ stderr btw
@@ -262,6 +264,11 @@ def ren_open(fname, *args, **kwargs):
             yield {"orz": [f, fname]}
             return
 
+    if suffix:
+        ext = fname.split(".")[-1]
+        if len(ext) < 7:
+            suffix += "." + ext
+
     orig_name = fname
     bname = fname
     ext = ""
@@ -847,13 +854,14 @@ def yieldfile(fn):
 
 
 def hashcopy(actor, fin, fout):
-    u32_lim = int((2 ** 31) * 0.9)
+    is_mp = actor.is_mp
     hashobj = hashlib.sha512()
     tlen = 0
     for buf in fin:
-        actor.workload += 1
-        if actor.workload > u32_lim:
-            actor.workload = 100  # prevent overflow
+        if is_mp:
+            actor.workload += 1
+            if actor.workload > 2 ** 31:
+                actor.workload = 100
 
         tlen += len(buf)
         hashobj.update(buf)
@@ -865,12 +873,17 @@ def hashcopy(actor, fin, fout):
     return tlen, hashobj.hexdigest(), digest_b64
 
 
-def sendfile_py(lower, upper, f, s):
+def sendfile_py(lower, upper, f, s, actor=None):
     remains = upper - lower
     f.seek(lower)
     while remains > 0:
+        if actor:
+            actor.workload += 1
+            if actor.workload > 2 ** 31:
+                actor.workload = 100
+
         # time.sleep(0.01)
-        buf = f.read(min(4096, remains))
+        buf = f.read(min(1024 * 32, remains))
         if not buf:
             return remains
 

copyparty/web/browser.js

@@ -803,7 +803,10 @@ var thegrid = (function () {
 			r.sz = v;
 			swrite('gridsz', r.sz);
 		}
-		document.documentElement.style.setProperty('--grid-sz', r.sz + 'em');
+		try {
+			document.documentElement.style.setProperty('--grid-sz', r.sz + 'em');
+		}
+		catch (ex) { }
 	}
 	setsz();
 
@@ -820,10 +823,18 @@ var thegrid = (function () {
 		this.setAttribute('class', tr.getAttribute('class'));
 	}
 
+	function bgopen(e) {
+		ev(e);
+		var url = this.getAttribute('href');
+		window.open(url, '_blank');
+	}
+
 	r.loadsel = function () {
-		var ths = QSA('#ggrid>a');
+		var ths = QSA('#ggrid>a'),
+			have_sel = !!QS('#files tr.sel');
+
 		for (var a = 0, aa = ths.length; a < aa; a++) {
-			ths[a].onclick = r.sel ? seltgl : null;
+			ths[a].onclick = r.sel ? seltgl : have_sel ? bgopen : null;
 			ths[a].setAttribute('class', ebi(ths[a].getAttribute('ref')).parentNode.parentNode.getAttribute('class'));
 		}
 		var uns = QS('#ggrid a[ref="unsearch"]');
@@ -965,7 +976,7 @@ document.onkeydown = function (e) {
 	if (k == 'KeyT')
 		return ebi('thumbs').click();
 
-	if (window['thegrid'] && thegrid.en) {
+	if (thegrid.en) {
 		if (k == 'KeyS')
 			return ebi('gridsel').click();
 
@@ -1437,7 +1448,7 @@ var treectl = (function () {
 		if (hpush)
 			get_tree('.', xhr.top);
 
-		enspin('#files');
+		enspin(thegrid.en ? '#gfiles' : '#files');
 	}
 
 	function treegrow(e) {
@@ -1517,6 +1528,7 @@ var treectl = (function () {
 
 		apply_perms(res.perms);
 		despin('#files');
+		despin('#gfiles');
 
 		ebi('pro').innerHTML = res.logues ? res.logues[0] || "" : "";
 		ebi('epi').innerHTML = res.logues ? res.logues[1] || "" : "";
@@ -2051,8 +2063,7 @@ var msel = (function () {
 	}
 	function selui() {
 		clmod(ebi('wtoggle'), 'sel', getsel().length);
-		if (window['thegrid'])
-			thegrid.loadsel();
+		thegrid.loadsel();
 	}
 	function seltgl(e) {
 		ev(e);

copyparty/web/up2k.js

@@ -17,6 +17,7 @@ function goto_up2k() {
 // chrome requires https to use crypto.subtle,
 // usually it's undefined but some chromes throw on invoke
 var up2k = null;
+var sha_js = window.WebAssembly ? 'hw' : 'ac';  // ff53,c57,sa11
 try {
     var cf = crypto.subtle || crypto.webkitSubtle;
     cf.digest('SHA-512', new Uint8Array(1)).then(
@@ -430,13 +431,15 @@ function up2k_init(subtle) {
     // upload ui hidden by default, clicking the header shows it
     function init_deps() {
         if (!subtle && !window.asmCrypto) {
-            showmodal('<h1>loading sha512.js</h1><h2>since ' + shame + '</h2><h4>thanks chrome</h4>');
-            import_js('/.cpr/deps/sha512.js', unmodal);
+            var fn = 'sha512.' + sha_js + '.js';
+            showmodal('<h1>loading ' + fn + '</h1><h2>since ' + shame + '</h2><h4>thanks chrome</h4>');
+            import_js('/.cpr/deps/' + fn, unmodal);
 
             if (is_https)
                 ebi('u2foot').innerHTML = shame + ' so <em>this</em> uploader will do like 500kB/s at best';
             else
-                ebi('u2foot').innerHTML = 'seems like ' + shame + ' so do that if you want more performance';
+                ebi('u2foot').innerHTML = 'seems like ' + shame + ' so do that if you want more performance <span style="color:#' +
+                    (sha_js == 'ac' ? 'c84">(expecting 20' : '8a5">(but dont worry too much, expect 100') + ' MiB/s)</span>';
         }
     }
 
@@ -886,6 +889,10 @@ function up2k_init(subtle) {
         return base64;
     }
 
+    function hex2u8(txt) {
+        return new Uint8Array(txt.match(/.{2}/g).map(function (b) { return parseInt(b, 16); }));
+    }
+
     function get_chunksize(filesize) {
         var chunksize = 1024 * 1024,
             stepsize = 512 * 1024;
@@ -987,10 +994,18 @@ function up2k_init(subtle) {
             if (subtle)
                 subtle.digest('SHA-512', buf).then(hash_done);
             else setTimeout(function () {
-                var hasher = new asmCrypto.Sha512();
-                hasher.process(new Uint8Array(buf));
-                hasher.finish();
-                hash_done(hasher.result);
+                var u8buf = new Uint8Array(buf);
+                if (sha_js == 'hw') {
+                    hashwasm.sha512(u8buf).then(function (v) {
+                        hash_done(hex2u8(v))
+                    });
+                }
+                else {
+                    var hasher = new asmCrypto.Sha512();
+                    hasher.process(u8buf);
+                    hasher.finish();
+                    hash_done(hasher.result);
+                }
             }, 1);
         };
 

copyparty/web/upload.css

@@ -238,6 +238,10 @@
 	color: #fff;
 	font-style: italic;
 }
+#u2foot span {
+	color: #999;
+	font-size: .9em;
+}
 #u2footfoot {
 	margin-bottom: -1em;
 }

docs/notes.sh

@@ -153,6 +153,9 @@ dbg.asyncStore.pendingBreakpoints = {}
 # fix firefox phantom breakpoints
 about:config >> devtools.debugger.prefs-schema-version = -1
 
+# determine server version
+git reset --hard origin/HEAD && git log --format=format:"%H %ai %d" --decorate=full > /dev/shm/revs && cat /dev/shm/revs | while read -r rev extra; do (git reset --hard $rev >/dev/null 2>/dev/null && dsz=$(cat copyparty/web/{util,browser,up2k}.js 2>/dev/null | diff -wNarU0 - <(cat /mnt/Users/ed/Downloads/ref/{util,browser,up2k}.js) | wc -c) && printf '%s %6s %s\n' "$rev" $dsz "$extra") </dev/null; done                
+
 
 ##
 ## http 206

scripts/deps-docker/Dockerfile

@@ -9,6 +9,12 @@ ENV     ver_asmcrypto=5b994303a9d3e27e0915f72a10b6c2c51535a4dc \
         ver_zopfli=1.0.3
 
 
+# TODO
+# sha512.hw.js https://github.com/Daninet/hash-wasm
+# sha512.kc.js https://github.com/chm-diederichs/sha3-wasm
+# awk '/HMAC state/{o=1}  /var HEAP/{o=0}  /function hmac_reset/{o=1}  /return \{/{o=0}  /var __extends =/{o=1}  /var Hash =/{o=0}  /hmac_|pbkdf2_/{next}  o{next}  {gsub(/IllegalStateError/,"Exception")}  {sub(/^ +/,"");sub(/^\/\/ .*/,"");sub(/;$/," ;")}  1' <sha512.ac.js.orig >sha512.ac.js; for fn in sha512.ac.js.orig sha512.ac.js; do wc -c <$fn; wc -c <$fn.gz ; for n in {1..9}; do printf '%8d %d bz\n' $(bzip2 -c$n <$fn | wc -c) $n; done; done
+
+
 # download;
 # the scp url is latin from https://fonts.googleapis.com/css2?family=Source+Code+Pro&display=swap
 RUN     mkdir -p /z/dist/no-pk \

scripts/install-githooks.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+set -ex
+
+[ -e setup.py ] || ..
+[ -e setup.py ] || {
+    echo u wot
+    exit 1
+}
+
+cd .git/hooks
+rm -f pre-commit
+ln -s ../../scripts/run-tests.sh pre-commit

scripts/make-sfx.sh

@@ -32,6 +32,10 @@ gtar=$(command -v gtar || command -v gnutar) || true
 	[ -e /opt/local/bin/bzip2 ] &&
 		bzip2() { /opt/local/bin/bzip2 "$@"; }
 }
+
+gawk=$(command -v gawk || command -v gnuawk || command -v awk)
+awk() { $gawk "$@"; }
+
 pybin=$(command -v python3 || command -v python) || {
 	echo need python
 	exit 1
@@ -194,17 +198,46 @@ tmv "$f"
 
 # up2k goes from 28k to 22k laff
 echo entabbening
-find | grep -E '\.(js|css|html)$' | while IFS= read -r f; do
+find | grep -E '\.css$' | while IFS= read -r f; do
+	awk '{
+		sub(/^[ \t]+/,"");
+		sub(/[ \t]+$/,"");
+		$0=gensub(/^([a-z-]+) *: *(.*[^ ]) *;$/,"\\1:\\2;","1");
+		sub(/ +\{$/,"{");
+		gsub(/, /,",")
+	}
+	!/\}$/ {printf "%s",$0;next}
+	1
+	' <$f | gsed 's/;\}$/}/' >t
+	tmv "$f"
+done
+find | grep -E '\.(js|html)$' | while IFS= read -r f; do
 	unexpand -t 4 --first-only <"$f" >t
 	tmv "$f"
 done
 
+
+gzres() {
+command -v pigz &&
+	pk='pigz -11 -J 34 -I 100' ||
+	pk='gzip'
+
+echo "$pk"
+find | grep -E '\.(js|css)$' | while IFS= read -r f; do
+	echo -n .
+	$pk "$f"
+done
+echo
+}
+gzres
+
+
 echo gen tarlist
 for d in copyparty dep-j2; do find $d -type f; done |
 sed -r 's/(.*)\.(.*)/\2 \1/' | LC_ALL=C sort |
 sed -r 's/([^ ]*) (.*)/\2.\1/' | grep -vE '/list1?$' > list1
 
-(grep -vE 'gz$' list1; grep -E 'gz$' list1) >list
+(grep -vE '\.(gz|br)$' list1; grep -E '\.(gz|br)$' list1) >list || true
 
 echo creating tar
 args=(--owner=1000 --group=1000)

scripts/profile.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+
+import sys
+
+sys.path.insert(0, ".")
+cmd = sys.argv[1]
+
+if cmd == "cpp":
+    from copyparty.__main__ import main
+
+    argv = ["__main__", "-v", "srv::r", "-v", "../../yt:yt:r"]
+    main(argv=argv)
+
+elif cmd == "test":
+    from unittest import main
+
+    argv = ["__main__", "discover", "-s", "tests"]
+    main(module=None, argv=argv)
+
+else:
+    raise Exception()
+
+# import dis; print(dis.dis(main))
+
+
+# macos:
+#   option1) python3.9 -m pip install --user -U vmprof==0.4.9
+#   option2) python3.9 -m pip install --user -U https://github.com/vmprof/vmprof-python/archive/refs/heads/master.zip
+#
+# python -m vmprof -o prof --lines ./scripts/profile.py test
+
+# linux: ~/.local/bin/vmprofshow prof tree | grep -vF '[1m  0.'
+# macos: ~/Library/Python/3.9/bin/vmprofshow prof tree | grep -vF '[1m  0.'
+#   win: %appdata%\..\Roaming\Python\Python39\Scripts\vmprofshow.exe prof tree

scripts/run-tests.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+set -ex
+
+pids=()
+for py in python{2,3}; do
+    $py -m unittest discover -s tests >/dev/null &
+    pids+=($!)
+done
+
+for pid in ${pids[@]}; do
+    wait $pid
+done

scripts/sfx.sh

@@ -47,7 +47,7 @@ grep -E '/(python|pypy)[0-9\.-]*$' >$dir/pys || true
 	printf '\033[1;30mlooking for jinja2 in [%s]\033[0m\n' "$_py" >&2
 	$_py -c 'import jinja2' 2>/dev/null || continue
 	printf '%s\n' "$_py"
-	mv $dir/{,x.}jinja2
+	mv $dir/{,x.}dep-j2
 	break
 done)"
 

tests/test_httpcli.py

@@ -8,13 +8,13 @@ import time
 import shutil
 import pprint
 import tarfile
+import tempfile
 import unittest
-
 from argparse import Namespace
-from copyparty.authsrv import AuthSrv
-from copyparty.httpcli import HttpCli
 
 from tests import util as tu
+from copyparty.authsrv import AuthSrv
+from copyparty.httpcli import HttpCli
 
 
 def hdr(query):
@@ -42,13 +42,15 @@ class Cfg(Namespace):
 
 
 class TestHttpCli(unittest.TestCase):
-    def test(self):
-        td = os.path.join(tu.get_ramdisk(), "vfs")
-        try:
-            shutil.rmtree(td)
-        except OSError:
-            pass
+    def setUp(self):
+        self.td = tu.get_ramdisk()
 
+    def tearDown(self):
+        os.chdir(tempfile.gettempdir())
+        shutil.rmtree(self.td)
+
+    def test(self):
+        td = os.path.join(self.td, "vfs")
         os.mkdir(td)
         os.chdir(td)
 

tests/test_vfs.py

@@ -7,13 +7,12 @@ import json
 import shutil
 import tempfile
 import unittest
-
 from textwrap import dedent
 from argparse import Namespace
-from copyparty.authsrv import AuthSrv
-from copyparty import util
 
 from tests import util as tu
+from copyparty.authsrv import AuthSrv, VFS
+from copyparty import util
 
 
 class Cfg(Namespace):
@@ -25,6 +24,13 @@ class Cfg(Namespace):
 
 
 class TestVFS(unittest.TestCase):
+    def setUp(self):
+        self.td = tu.get_ramdisk()
+
+    def tearDown(self):
+        os.chdir(tempfile.gettempdir())
+        shutil.rmtree(self.td)
+
     def dump(self, vfs):
         print(json.dumps(vfs, indent=4, sort_keys=True, default=lambda o: o.__dict__))
 
@@ -41,6 +47,7 @@ class TestVFS(unittest.TestCase):
         self.assertEqual(util.undot(query), response)
 
     def ls(self, vfs, vpath, uname):
+        # type: (VFS, str, str) -> tuple[str, str, str]
         """helper for resolving and listing a folder"""
         vn, rem = vfs.get(vpath, uname, True, False)
         r1 = vn.ls(rem, uname, False)
@@ -55,12 +62,7 @@ class TestVFS(unittest.TestCase):
         pass
 
     def test(self):
-        td = os.path.join(tu.get_ramdisk(), "vfs")
-        try:
-            shutil.rmtree(td)
-        except OSError:
-            pass
-
+        td = os.path.join(self.td, "vfs")
         os.mkdir(td)
         os.chdir(td)
 
@@ -227,7 +229,7 @@ class TestVFS(unittest.TestCase):
         self.assertEqual(list(v1), list(v2))
 
         # config file parser
-        cfg_path = os.path.join(tu.get_ramdisk(), "test.cfg")
+        cfg_path = os.path.join(self.td, "test.cfg")
         with open(cfg_path, "wb") as f:
             f.write(
                 dedent(
@@ -249,7 +251,7 @@ class TestVFS(unittest.TestCase):
         n = au.vfs
         # root was not defined, so PWD with no access to anyone
         self.assertEqual(n.vpath, "")
-        self.assertEqual(n.realpath, td)
+        self.assertEqual(n.realpath, None)
         self.assertEqual(n.uread, [])
         self.assertEqual(n.uwrite, [])
         self.assertEqual(len(n.nodes), 1)
@@ -260,6 +262,4 @@ class TestVFS(unittest.TestCase):
         self.assertEqual(n.uwrite, ["asd"])
         self.assertEqual(len(n.nodes), 0)
 
-        os.chdir(tempfile.gettempdir())
-        shutil.rmtree(td)
         os.unlink(cfg_path)

tests/util.py

@@ -1,16 +1,36 @@
 import os
+import sys
 import time
+import shutil
 import jinja2
 import tempfile
+import platform
 import subprocess as sp
 
-from copyparty.util import Unrecv
 
+WINDOWS = platform.system() == "Windows"
+ANYWIN = WINDOWS or sys.platform in ["msys"]
+MACOS = platform.system() == "Darwin"
 
 J2_ENV = jinja2.Environment(loader=jinja2.BaseLoader)
 J2_FILES = J2_ENV.from_string("{{ files|join('\n') }}")
 
 
+def nah(*a, **ka):
+    return False
+
+
+if MACOS:
+    import posixpath
+
+    posixpath.islink = nah
+    os.path.islink = nah
+    # 25% faster; until any tests do symlink stuff
+
+
+from copyparty.util import Unrecv
+
+
 def runcmd(*argv):
     p = sp.Popen(argv, stdout=sp.PIPE, stderr=sp.PIPE)
     stdout, stderr = p.communicate()
@@ -28,18 +48,25 @@ def chkcmd(*argv):
 
 
 def get_ramdisk():
+    def subdir(top):
+        ret = os.path.join(top, "cptd-{}".format(os.getpid()))
+        shutil.rmtree(ret, True)
+        os.mkdir(ret)
+        return ret
+
     for vol in ["/dev/shm", "/Volumes/cptd"]:  # nosec (singleton test)
         if os.path.exists(vol):
-            return vol
+            return subdir(vol)
 
     if os.path.exists("/Volumes"):
-        devname, _ = chkcmd("hdiutil", "attach", "-nomount", "ram://32768")
+        # hdiutil eject /Volumes/cptd/
+        devname, _ = chkcmd("hdiutil", "attach", "-nomount", "ram://65536")
         devname = devname.strip()
         print("devname: [{}]".format(devname))
         for _ in range(10):
             try:
                 _, _ = chkcmd("diskutil", "eraseVolume", "HFS+", "cptd", devname)
-                return "/Volumes/cptd"
+                return subdir("/Volumes/cptd")
             except Exception as ex:
                 print(repr(ex))
                 time.sleep(0.25)
@@ -50,7 +77,7 @@ def get_ramdisk():
     try:
         os.mkdir(ret)
     finally:
-        return ret
+        return subdir(ret)
 
 
 class NullBroker(object):
@@ -89,6 +116,7 @@ class VHttpConn(object):
         self.addr = ("127.0.0.1", "42069")
         self.args = args
         self.auth = auth
+        self.is_mp = False
         self.log_func = log
         self.log_src = "a"
         self.lf_url = None