v0.5.4

* show per-connection and per-transfer speeds
* support multiple cookies in parser
* set SameSite=Lax
* restore macos support in sfx.sh
* md-editor: add mojibake/unicode hunter
* md-editor: add table formatter
* md-editor: make bold bolder
* md-editor: more hotkeys
* md-editor: fix saving in fancy
* md-editor: fix eof-scrolling in chrome
* md-editor: fix text erasure with newline
* md-editor: fix backspace behavior in gutter

README.md

@@ -38,7 +38,7 @@ turn your phone or raspi into a portable file server with resumable uploads/down
 * [x] accounts
 * [x] markdown viewer
 * [x] markdown editor
-* [x] FUSE client
+* [x] FUSE client (read-only)
 
 summary: it works! you can use it! (but technically not even close to beta)
 
@@ -49,7 +49,9 @@ summary: it works! you can use it! (but technically not even close to beta)
   * `await fetch('https://127.0.0.1:3923/', {method:"PUT", body: JSON.stringify(foo)});`
   * `var xhr = new XMLHttpRequest(); xhr.open('POST', 'https://127.0.0.1:3923/msgs?raw'); xhr.send('foo');`
 
-* FUSE: mount a copyparty server as a local filesystem (see [./bin/](bin/))
+* FUSE: mount a copyparty server as a local filesystem
+  * cross-platform python client available in [./bin/](bin/)
+  * [rclone](https://rclone.org/) as client can give ~5x performance, see [./docs/rclone.md](docs/rclone.md)
 
 
 # dependencies

bin/README.md

@@ -10,6 +10,8 @@ filecache is default-on for windows and macos;
 
 note that copyparty should run with `-ed` to enable dotfiles (hidden otherwise)
 
+also consider using [../docs/rclone.md](../docs/rclone.md) instead for 5x performance
+
 
 ## to run this on windows:
 * install [winfsp](https://github.com/billziss-gh/winfsp/releases/latest) and [python 3](https://www.python.org/downloads/)

bin/copyparty-fuse.py

@@ -19,6 +19,9 @@ dependencies:
   + on Linux: sudo apk add fuse
   + on Macos: https://osxfuse.github.io/
   + on Windows: https://github.com/billziss-gh/winfsp/releases/latest
+
+get server cert:
+  awk '/-BEGIN CERTIFICATE-/ {a=1} a; /-END CERTIFICATE-/{exit}' <(openssl s_client -connect 127.0.0.1:3923 </dev/null 2>/dev/null) >cert.pem
 """
 
 
@@ -29,21 +32,21 @@ import time
 import stat
 import errno
 import struct
+import codecs
 import builtins
 import platform
+import argparse
 import threading
 import traceback
 import http.client  # py2: httplib
 import urllib.parse
 from datetime import datetime
 from urllib.parse import quote_from_bytes as quote
-
-
-DEBUG = False  # ctrl-f this to configure logging
-
+from urllib.parse import unquote_to_bytes as unquote
 
 WINDOWS = sys.platform == "win32"
 MACOS = platform.system() == "Darwin"
+info = log = dbg = None
 
 
 try:
@@ -104,6 +107,47 @@ def null_log(msg):
     pass
 
 
+def hexler(binary):
+    return binary.replace("\r", "\\r").replace("\n", "\\n")
+    return " ".join(["{}\033[36m{:02x}\033[0m".format(b, ord(b)) for b in binary])
+    return " ".join(map(lambda b: format(ord(b), "02x"), binary))
+
+
+def register_wtf8():
+    def wtf8_enc(text):
+        return str(text).encode("utf-8", "surrogateescape"), len(text)
+
+    def wtf8_dec(binary):
+        return bytes(binary).decode("utf-8", "surrogateescape"), len(binary)
+
+    def wtf8_search(encoding_name):
+        return codecs.CodecInfo(wtf8_enc, wtf8_dec, name="wtf-8")
+
+    codecs.register(wtf8_search)
+
+
+bad_good = {}
+good_bad = {}
+
+
+def enwin(txt):
+    return "".join([bad_good.get(x, x) for x in txt])
+
+    for bad, good in bad_good.items():
+        txt = txt.replace(bad, good)
+
+    return txt
+
+
+def dewin(txt):
+    return "".join([good_bad.get(x, x) for x in txt])
+
+    for bad, good in bad_good.items():
+        txt = txt.replace(good, bad)
+
+    return txt
+
+
 class RecentLog(object):
     def __init__(self):
         self.mtx = threading.Lock()
@@ -138,22 +182,6 @@ class RecentLog(object):
             print("".join(q), end="")
 
 
-if DEBUG:
-    # debug=on,
-    #   windows terminals are slow (cmd.exe, mintty)
-    #   otoh fancy_log beats RecentLog on linux
-    logger = RecentLog().put if WINDOWS else fancy_log
-
-    info = logger
-    log = logger
-    dbg = logger
-else:
-    # debug=off, speed is dontcare
-    info = fancy_log
-    log = null_log
-    dbg = null_log
-
-
 # [windows/cmd/cpy3]  python dev\copyparty\bin\copyparty-fuse.py q: http://192.168.1.159:1234/
 # [windows/cmd/msys2] C:\msys64\mingw64\bin\python3 dev\copyparty\bin\copyparty-fuse.py q: http://192.168.1.159:1234/
 # [windows/mty/msys2] /mingw64/bin/python3 /c/Users/ed/dev/copyparty/bin/copyparty-fuse.py q: http://192.168.1.159:1234/
@@ -183,6 +211,8 @@ def html_dec(txt):
         txt.replace("&lt;", "<")
         .replace("&gt;", ">")
         .replace("&quot;", '"')
+        .replace("&#13;", "\r")
+        .replace("&#10;", "\n")
         .replace("&amp;", "&")
     )
 
@@ -195,10 +225,11 @@ class CacheNode(object):
 
 
 class Gateway(object):
-    def __init__(self, base_url):
-        self.base_url = base_url
+    def __init__(self, ar):
+        self.base_url = ar.base_url
+        self.password = ar.a
 
-        ui = urllib.parse.urlparse(base_url)
+        ui = urllib.parse.urlparse(self.base_url)
         self.web_root = ui.path.strip("/")
         try:
             self.web_host, self.web_port = ui.netloc.split(":")
@@ -208,15 +239,25 @@ class Gateway(object):
             if ui.scheme == "http":
                 self.web_port = 80
             elif ui.scheme == "https":
-                raise Exception("todo")
+                self.web_port = 443
             else:
                 raise Exception("bad url?")
 
+        self.ssl_context = None
+        self.use_tls = ui.scheme.lower() == "https"
+        if self.use_tls:
+            import ssl
+
+            if ar.td:
+                self.ssl_context = ssl._create_unverified_context()
+            elif ar.te:
+                self.ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS)
+                self.ssl_context.load_verify_locations(ar.te)
+
         self.conns = {}
 
     def quotep(self, path):
-        # TODO: mojibake support
-        path = path.encode("utf-8", "ignore")
+        path = path.encode("wtf-8")
         return quote(path, safe="/")
 
     def getconn(self, tid=None):
@@ -226,7 +267,15 @@ class Gateway(object):
         except:
             info("new conn [{}] [{}]".format(self.web_host, self.web_port))
 
-            conn = http.client.HTTPConnection(self.web_host, self.web_port, timeout=260)
+            args = {}
+            if not self.use_tls:
+                C = http.client.HTTPConnection
+            else:
+                C = http.client.HTTPSConnection
+                if self.ssl_context:
+                    args = {"context": self.ssl_context}
+
+            conn = C(self.web_host, self.web_port, timeout=260, **args)
 
             self.conns[tid] = conn
             return conn
@@ -239,41 +288,67 @@ class Gateway(object):
         except:
             pass
 
-    def sendreq(self, *args, **kwargs):
+    def sendreq(self, *args, headers={}, **kwargs):
         tid = get_tid()
+        if self.password:
+            headers["Cookie"] = "=".join(["cppwd", self.password])
+
         try:
             c = self.getconn(tid)
-            c.request(*list(args), **kwargs)
+            c.request(*list(args), headers=headers, **kwargs)
             return c.getresponse()
         except:
-            self.closeconn(tid)
+            dbg("bad conn")
+
+        self.closeconn(tid)
+        try:
             c = self.getconn(tid)
-            c.request(*list(args), **kwargs)
+            c.request(*list(args), headers=headers, **kwargs)
             return c.getresponse()
+        except:
+            info("http connection failed:\n" + traceback.format_exc())
+            if self.use_tls and not self.ssl_context:
+                import ssl
+
+                cert = ssl.get_server_certificate((self.web_host, self.web_port))
+                info("server certificate probably not trusted:\n" + cert)
+
+            raise
 
     def listdir(self, path):
+        if bad_good:
+            path = dewin(path)
+
         web_path = self.quotep("/" + "/".join([self.web_root, path])) + "?dots"
         r = self.sendreq("GET", web_path)
         if r.status != 200:
             self.closeconn()
-            raise Exception(
+            log(
                 "http error {} reading dir {} in {}".format(
                     r.status, web_path, rice_tid()
                 )
             )
+            raise FuseOSError(errno.ENOENT)
+
+        if not r.getheader("Content-Type", "").startswith("text/html"):
+            log("listdir on file: {}".format(path))
+            raise FuseOSError(errno.ENOENT)
 
         try:
             return self.parse_html(r)
         except:
-            traceback.print_exc()
+            info(repr(path) + "\n" + traceback.format_exc())
             raise
 
     def download_file_range(self, path, ofs1, ofs2):
+        if bad_good:
+            path = dewin(path)
+
         web_path = self.quotep("/" + "/".join([self.web_root, path])) + "?raw"
         hdr_range = "bytes={}-{}".format(ofs1, ofs2 - 1)
         info(
             "DL {:4.0f}K\033[36m{:>9}-{:<9}\033[0m{}".format(
-                (ofs2 - ofs1) / 1024.0, ofs1, ofs2 - 1, path
+                (ofs2 - ofs1) / 1024.0, ofs1, ofs2 - 1, hexler(path)
             )
         )
 
@@ -292,7 +367,7 @@ class Gateway(object):
         ret = []
         remainder = b""
         ptn = re.compile(
-            r"^<tr><td>(-|DIR)</td><td><a [^>]+>([^<]+)</a></td><td>([^<]+)</td><td>([^<]+)</td></tr>$"
+            r'^<tr><td>(-|DIR)</td><td><a[^>]* href="([^"]+)"[^>]*>([^<]+)</a></td><td>([^<]+)</td><td>([^<]+)</td></tr>$'
         )
 
         while True:
@@ -314,8 +389,13 @@ class Gateway(object):
                     # print(line)
                     continue
 
-                ftype, fname, fsize, fdate = m.groups()
-                fname = html_dec(fname)
+                ftype, furl, fname, fsize, fdate = m.groups()
+                fname = furl.rstrip("/").split("/")[-1]
+                fname = unquote(fname)
+                fname = fname.decode("wtf-8")
+                if bad_good:
+                    fname = enwin(fname)
+
                 sz = 1
                 ts = 60 * 60 * 24 * 2
                 try:
@@ -358,11 +438,11 @@ class Gateway(object):
 
 
 class CPPF(Operations):
-    def __init__(self, base_url, dircache, filecache):
-        self.gw = Gateway(base_url)
+    def __init__(self, ar):
+        self.gw = Gateway(ar)
         self.junk_fh_ctr = 3
-        self.n_dircache = dircache
-        self.n_filecache = filecache
+        self.n_dircache = ar.cd
+        self.n_filecache = ar.cf
 
         self.dircache = []
         self.dircache_mtx = threading.Lock()
@@ -379,7 +459,11 @@ class CPPF(Operations):
                 cache_path, cache1 = cn.tag
                 cache2 = cache1 + len(cn.data)
                 msg += "\n{:<2} {:>7} {:>10}:{:<9} {}".format(
-                    n, len(cn.data), cache1, cache2, cache_path
+                    n,
+                    len(cn.data),
+                    cache1,
+                    cache2,
+                    cache_path.replace("\r", "\\r").replace("\n", "\\n"),
                 )
         return msg
 
@@ -610,7 +694,7 @@ class CPPF(Operations):
 
     def _readdir(self, path, fh=None):
         path = path.strip("/")
-        log("readdir [{}] [{}]".format(path, fh))
+        log("readdir [{}] [{}]".format(hexler(path), fh))
 
         ret = self.gw.listdir(path)
         if not self.n_dircache:
@@ -637,7 +721,11 @@ class CPPF(Operations):
         path = path.strip("/")
         ofs2 = offset + length
         file_sz = self.getattr(path)["st_size"]
-        log("read {} |{}| {}:{} max {}".format(path, length, offset, ofs2, file_sz))
+        log(
+            "read {} |{}| {}:{} max {}".format(
+                hexler(path), length, offset, ofs2, file_sz
+            )
+        )
         if ofs2 > file_sz:
             ofs2 = file_sz
             log("truncate to |{}| :{}".format(ofs2 - offset, ofs2))
@@ -676,7 +764,9 @@ class CPPF(Operations):
         return ret
 
     def getattr(self, path, fh=None):
-        log("getattr [{}]".format(path))
+        log("getattr [{}]".format(hexler(path)))
+        if WINDOWS:
+            path = enwin(path)  # windows occasionally decodes f0xx to xx
 
         path = path.strip("/")
         try:
@@ -699,11 +789,20 @@ class CPPF(Operations):
             dents = self._readdir(dirpath)
 
         for cache_name, cache_stat, _ in dents:
+            # if "qw" in cache_name and "qw" in fname:
+            #     info(
+            #         "cmp\n  [{}]\n  [{}]\n\n{}\n".format(
+            #             hexler(cache_name),
+            #             hexler(fname),
+            #             "\n".join(traceback.format_stack()[:-1]),
+            #         )
+            #     )
+
             if cache_name == fname:
                 # dbg("=" + repr(cache_stat))
                 return cache_stat
 
-        info("=ENOENT ({})".format(path))
+        info("=ENOENT ({})".format(hexler(path)))
         raise FuseOSError(errno.ENOENT)
 
     access = None
@@ -773,24 +872,24 @@ class CPPF(Operations):
                 raise FuseOSError(errno.ENOENT)
 
         def open(self, path, flags):
-            dbg("open [{}] [{}]".format(path, flags))
+            dbg("open [{}] [{}]".format(hexler(path), flags))
             return self._open(path)
 
         def opendir(self, path):
-            dbg("opendir [{}]".format(path))
+            dbg("opendir [{}]".format(hexler(path)))
             return self._open(path)
 
         def flush(self, path, fh):
-            dbg("flush [{}] [{}]".format(path, fh))
+            dbg("flush [{}] [{}]".format(hexler(path), fh))
 
         def release(self, ino, fi):
-            dbg("release [{}] [{}]".format(ino, fi))
+            dbg("release [{}] [{}]".format(hexler(ino), fi))
 
         def releasedir(self, ino, fi):
-            dbg("releasedir [{}] [{}]".format(ino, fi))
+            dbg("releasedir [{}] [{}]".format(hexler(ino), fi))
 
         def access(self, path, mode):
-            dbg("access [{}] [{}]".format(path, mode))
+            dbg("access [{}] [{}]".format(hexler(path), mode))
             try:
                 x = self.getattr(path)
                 if x["st_mode"] <= 0:
@@ -799,42 +898,84 @@ class CPPF(Operations):
                 raise FuseOSError(errno.ENOENT)
 
 
+class TheArgparseFormatter(
+    argparse.RawTextHelpFormatter, argparse.ArgumentDefaultsHelpFormatter
+):
+    pass
+
+
 def main():
+    global info, log, dbg
+
     # filecache helps for reads that are ~64k or smaller;
     #   linux generally does 128k so the cache is a slowdown,
     #   windows likes to use 4k and 64k so cache is required,
     #   value is numChunks (1~3M each) to keep in the cache
-    nf = 24 if WINDOWS or MACOS else 0
+    nf = 24
 
     # dircache is always a boost,
     #   only want to disable it for tests etc,
     #   value is numSec until an entry goes stale
     nd = 1
 
-    try:
-        local, remote = sys.argv[1:3]
-        filecache = nf if len(sys.argv) <= 3 else int(sys.argv[3])
-        dircache = nd if len(sys.argv) <= 4 else float(sys.argv[4])
-    except:
-        where = "local directory"
-        if WINDOWS:
-            where += " or DRIVE:"
+    where = "local directory"
+    if WINDOWS:
+        where += " or DRIVE:"
 
-        print("need arg 1: " + where)
-        print("need arg 2: root url")
-        print("optional 3: num files in filecache ({})".format(nf))
-        print("optional 4: num seconds / dircache ({})".format(nd))
-        print()
-        print("example:")
-        print("  copyparty-fuse.py ./music http://192.168.1.69:3923/music/")
-        if WINDOWS:
-            print("  copyparty-fuse.py M: http://192.168.1.69:3923/music/")
+    ex_pre = "\n  " + os.path.basename(__file__) + "  "
+    examples = ["http://192.168.1.69:3923/music/  ./music"]
+    if WINDOWS:
+        examples.append("http://192.168.1.69:3923/music/  M:")
 
-        return
+    ap = argparse.ArgumentParser(
+        formatter_class=TheArgparseFormatter,
+        epilog="example:" + ex_pre + ex_pre.join(examples),
+    )
+    ap.add_argument(
+        "-cd", metavar="NUM_SECONDS", type=float, default=nd, help="directory cache"
+    )
+    ap.add_argument(
+        "-cf", metavar="NUM_BLOCKS", type=int, default=nf, help="file cache"
+    )
+    ap.add_argument("-a", metavar="PASSWORD", help="password")
+    ap.add_argument("-d", action="store_true", help="enable debug")
+    ap.add_argument("-te", metavar="PEM_FILE", help="certificate to expect/verify")
+    ap.add_argument("-td", action="store_true", help="disable certificate check")
+    ap.add_argument("base_url", type=str, help="remote copyparty URL to mount")
+    ap.add_argument("local_path", type=str, help=where + " to mount it on")
+    ar = ap.parse_args()
+
+    if ar.d:
+        # windows terminals are slow (cmd.exe, mintty)
+        # otoh fancy_log beats RecentLog on linux
+        logger = RecentLog().put if WINDOWS else fancy_log
+
+        info = logger
+        log = logger
+        dbg = logger
+    else:
+        # debug=off, speed is dontcare
+        info = fancy_log
+        log = null_log
+        dbg = null_log
 
     if WINDOWS:
         os.system("")
 
+        for ch in '<>:"\\|?*':
+            # microsoft maps illegal characters to f0xx
+            # (e000 to f8ff is basic-plane private-use)
+            bad_good[ch] = chr(ord(ch) + 0xF000)
+
+        for n in range(0, 0x100):
+            # map surrogateescape to another private-use area
+            bad_good[chr(n + 0xDC00)] = chr(n + 0xF100)
+
+        for k, v in bad_good.items():
+            good_bad[v] = k
+
+    register_wtf8()
+
     try:
         with open("/etc/fuse.conf", "rb") as f:
             allow_other = b"\nuser_allow_other" in f.read()
@@ -845,7 +986,7 @@ def main():
     if not MACOS:
         args["nonempty"] = True
 
-    FUSE(CPPF(remote, dircache, filecache), local, **args)
+    FUSE(CPPF(ar), ar.local_path, encoding="wtf-8", **args)
 
 
 if __name__ == "__main__":

contrib/README.md

@@ -0,0 +1,19 @@
+### [`copyparty.bat`](copyparty.bat)
+* launches copyparty with no arguments (anon read+write within same folder)
+* intended for windows machines with no python.exe in PATH
+* works on windows, linux and macos
+* assumes `copyparty-sfx.py` was renamed to `copyparty.py` in the same folder as `copyparty.bat`
+
+### [`index.html`](index.html)
+* drop-in redirect from an httpd to copyparty
+* assumes the webserver and copyparty is running on the same server/IP
+* modify `10.13.1.1` as necessary if you wish to support browsers without javascript
+
+# OS integration
+init-scripts to start copyparty as a service
+* [`systemd/copyparty.service`](systemd/copyparty.service)
+* [`openrc/copyparty`](openrc/copyparty)
+
+# Reverse-proxy
+copyparty has basic support for running behind another webserver
+* [`nginx/copyparty.conf`](nginx/copyparty.conf)

contrib/copyparty.bat

@@ -0,0 +1,33 @@
+exec python "$(dirname "$0")"/copyparty.py
+
+@rem on linux, the above will execute and the script will terminate
+@rem on windows, the rest of this script will run
+
+@echo off
+cls
+
+set py=
+for /f %%i in ('where python 2^>nul') do (
+    set "py=%%i"
+    goto c1
+)
+:c1
+
+if [%py%] == [] (
+    for /f %%i in ('where /r "%localappdata%\programs\python" python 2^>nul') do (
+        set "py=%%i"
+        goto c2
+    )
+)
+:c2
+
+if [%py%] == [] set "py=c:\python27\python.exe"
+
+if not exist "%py%" (
+    echo could not find python
+    echo(
+    pause
+    exit /b
+)
+
+start cmd /c %py% "%~dp0\copyparty.py"

contrib/index.html

@@ -0,0 +1,43 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+	<meta charset="utf-8">
+	<title>⇆🎉 redirect</title>
+	<meta http-equiv="X-UA-Compatible" content="IE=edge">
+	<style>
+
+html, body {
+	font-family: sans-serif;
+}
+body {
+	padding: 1em 2em;
+	font-size: 1.5em;
+}
+a {
+	font-size: 1.2em;
+	padding: .1em;
+}
+
+</style>
+</head>
+<body>
+	<span id="desc">you probably want</span> <a id="redir" href="//10.13.1.1:3923/">copyparty</a>
+	<script>
+
+var a = document.getElementById('redir'),
+	proto = window.location.protocol.indexOf('https') === 0 ? 'https' : 'http',
+	loc = window.location.hostname || '127.0.0.1',
+	port = a.getAttribute('href').split(':').pop().split('/')[0],
+	url = proto + '://' + loc + ':' + port + '/';
+
+a.setAttribute('href', url);
+document.getElementById('desc').innerHTML = 'redirecting to';
+
+setTimeout(function() {
+	window.location.href = url;
+}, 500);
+
+</script>
+</body>
+</html>

contrib/nginx/copyparty.conf

@@ -0,0 +1,26 @@
+upstream cpp {
+	server 127.0.0.1:3923;
+	keepalive 120;
+}
+server {
+	listen 443 ssl;
+	listen [::]:443 ssl;
+
+	server_name fs.example.com;
+	
+	location / {
+		proxy_pass http://cpp;
+		proxy_redirect off;
+		# disable buffering (next 4 lines)
+		proxy_http_version 1.1;
+		client_max_body_size 0;
+		proxy_buffering off;
+		proxy_request_buffering off;
+
+		proxy_set_header   Host              $host;
+		proxy_set_header   X-Real-IP         $remote_addr;
+		proxy_set_header   X-Forwarded-For   $proxy_add_x_forwarded_for;
+		proxy_set_header   X-Forwarded-Proto $scheme;
+		proxy_set_header   Connection        "Keep-Alive";
+	}
+}

contrib/openrc/copyparty

@@ -0,0 +1,18 @@
+#!/sbin/openrc-run
+
+# this will start `/usr/local/bin/copyparty-sfx.py`
+# and share '/mnt' with anonymous read+write
+#
+# installation:
+#   cp -pv copyparty /etc/init.d && rc-update add copyparty
+#
+# you may want to:
+#   change '/usr/bin/python' to another interpreter
+#   change '/mnt::a' to another location or permission-set
+
+name="$SVCNAME"
+command_background=true
+pidfile="/var/run/$SVCNAME.pid"
+
+command="/usr/bin/python /usr/local/bin/copyparty-sfx.py"
+command_args="-q -v /mnt::a"

contrib/systemd/copyparty.service

@@ -0,0 +1,19 @@
+# this will start `/usr/local/bin/copyparty-sfx.py`
+# and share '/mnt' with anonymous read+write
+#
+# installation:
+#   cp -pv copyparty.service /etc/systemd/system && systemctl enable --now copyparty
+#
+# you may want to:
+#   change '/usr/bin/python' to another interpreter
+#   change '/mnt::a' to another location or permission-set
+
+[Unit]
+Description=copyparty file server
+
+[Service]
+ExecStart=/usr/bin/python /usr/local/bin/copyparty-sfx.py -q -v /mnt::a
+ExecStartPre=/bin/bash -c 'mkdir -p /run/tmpfiles.d/ && echo "x /tmp/pe-copyparty*" > /run/tmpfiles.d/copyparty.conf'
+
+[Install]
+WantedBy=multi-user.target

copyparty/__main__.py

@@ -128,7 +128,7 @@ def main():
     )
     ap.add_argument("-i", metavar="IP", type=str, default="0.0.0.0", help="ip to bind")
     ap.add_argument("-p", metavar="PORT", type=int, default=3923, help="port to bind")
-    ap.add_argument("-nc", metavar="NUM", type=int, default=16, help="max num clients")
+    ap.add_argument("-nc", metavar="NUM", type=int, default=64, help="max num clients")
     ap.add_argument(
         "-j", metavar="CORES", type=int, default=1, help="max num cpu cores"
     )

copyparty/__version__.py

@@ -1,8 +1,8 @@
 # coding: utf-8
 
-VERSION = (0, 5, 2)
+VERSION = (0, 5, 4)
 CODENAME = "fuse jelly"
-BUILD_DT = (2020, 8, 18)
+BUILD_DT = (2020, 11, 17)
 
 S_VERSION = ".".join(map(str, VERSION))
 S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT)

copyparty/httpcli.py

@@ -16,9 +16,6 @@ from .util import *  # noqa  # pylint: disable=unused-wildcard-import
 
 if not PY2:
     unicode = str
-    from html import escape as html_escape
-else:
-    from cgi import escape as html_escape  # pylint: disable=no-name-in-module
 
 
 class HttpCli(object):
@@ -27,6 +24,7 @@ class HttpCli(object):
     """
 
     def __init__(self, conn):
+        self.t0 = time.time()
         self.conn = conn
         self.s = conn.s
         self.sr = conn.sr
@@ -85,11 +83,15 @@ class HttpCli(object):
         v = self.headers.get("connection", "").lower()
         self.keepalive = not v.startswith("close")
 
+        v = self.headers.get("x-forwarded-for", None)
+        if v is not None and self.conn.addr[0] in ["127.0.0.1", "::1"]:
+            self.log_src = self.conn.set_rproxy(v.split(",")[0])
+
         self.uname = "*"
         if "cookie" in self.headers:
             cookies = self.headers["cookie"].split(";")
             for k, v in [x.split("=", 1) for x in cookies]:
-                if k != "cppwd":
+                if k.strip() != "cppwd":
                     continue
 
                 v = unescape_cookie(v)
@@ -125,6 +127,11 @@ class HttpCli(object):
         self.uparam = uparam
         self.vpath = unquotep(vpath)
 
+        ua = self.headers.get("user-agent", "")
+        if ua.startswith("rclone/"):
+            uparam["raw"] = True
+            uparam["dots"] = True
+
         try:
             if self.mode in ["GET", "HEAD"]:
                 return self.handle_get() and self.keepalive
@@ -141,7 +148,7 @@ class HttpCli(object):
             try:
                 # self.log("pebkac at httpcli.run #2: " + repr(ex))
                 self.keepalive = self._check_nonfatal(ex)
-                self.loud_reply(str(ex), status=ex.code)
+                self.loud_reply("{}: {}".format(str(ex), self.vpath), status=ex.code)
                 return self.keepalive
             except Pebkac:
                 return False
@@ -180,7 +187,8 @@ class HttpCli(object):
         self.send_headers(len(body), status, mime, headers)
 
         try:
-            self.s.sendall(body)
+            if self.mode != "HEAD":
+                self.s.sendall(body)
         except:
             raise Pebkac(400, "client d/c while replying body")
 
@@ -188,7 +196,7 @@ class HttpCli(object):
 
     def loud_reply(self, body, *args, **kwargs):
         self.log(body.rstrip())
-        self.reply(b"<pre>" + body.encode("utf-8"), *list(args), **kwargs)
+        self.reply(b"<pre>" + body.encode("utf-8") + b"\r\n", *list(args), **kwargs)
 
     def handle_get(self):
         logmsg = "{:4} {}".format(self.mode, self.req)
@@ -304,10 +312,19 @@ class HttpCli(object):
         with open(path, "wb", 512 * 1024) as f:
             post_sz, _, sha_b64 = hashcopy(self.conn, reader, f)
 
-        self.log("wrote {}/{} bytes to {}".format(post_sz, remains, path))
+        spd = self._spd(post_sz)
+        self.log("{} wrote {}/{} bytes to {}".format(spd, post_sz, remains, path))
         self.reply("{}\n{}\n".format(post_sz, sha_b64).encode("utf-8"))
         return True
 
+    def _spd(self, nbytes, add=True):
+        if add:
+            self.conn.nbyte += nbytes
+
+        spd1 = get_spd(nbytes, self.t0)
+        spd2 = get_spd(self.conn.nbyte, self.conn.t0)
+        return spd1 + " " + spd2
+
     def handle_post_multipart(self):
         self.parser = MultipartParser(self.log, self.sr, self.headers)
         self.parser.parse()
@@ -447,7 +464,9 @@ class HttpCli(object):
             except:
                 self.log("failed to utime ({}, {})".format(path, times))
 
-        self.loud_reply("thank")
+        spd = self._spd(post_sz)
+        self.log("{} thank".format(spd))
+        self.reply(b"thank")
         return True
 
     def handle_login(self):
@@ -460,7 +479,7 @@ class HttpCli(object):
             msg = "naw dude"
             pwd = "x"  # nosec
 
-        h = {"Set-Cookie": "cppwd={}; Path=/".format(pwd)}
+        h = {"Set-Cookie": "cppwd={}; Path=/; SameSite=Lax".format(pwd)}
         html = self.conn.tpl_msg.render(h1=msg, h2='<a href="/">ack</a>', redir="/")
         self.reply(html.encode("utf-8"), headers=h)
         return True
@@ -493,7 +512,7 @@ class HttpCli(object):
         vpath = "{}/{}".format(self.vpath, sanitized).lstrip("/")
         html = self.conn.tpl_msg.render(
             h2='<a href="/{}">go to /{}</a>'.format(
-                quotep(vpath), html_escape(vpath, quote=False)
+                quotep(vpath), html_escape(vpath)
             ),
             pre="aight",
             click=True,
@@ -527,7 +546,7 @@ class HttpCli(object):
         vpath = "{}/{}".format(self.vpath, sanitized).lstrip("/")
         html = self.conn.tpl_msg.render(
             h2='<a href="/{}?edit">go to /{}?edit</a>'.format(
-                quotep(vpath), html_escape(vpath, quote=False)
+                quotep(vpath), html_escape(vpath)
             ),
             pre="aight",
             click=True,
@@ -572,6 +591,7 @@ class HttpCli(object):
                             raise Pebkac(400, "empty files in post")
 
                         files.append([sz, sha512_hex])
+                        self.conn.nbyte += sz
 
                 except Pebkac:
                     if fn != os.devnull:
@@ -599,7 +619,9 @@ class HttpCli(object):
             # truncated SHA-512 prevents length extension attacks;
             # using SHA-512/224, optionally SHA-512/256 = :64
 
-        self.log(msg)
+        vspd = self._spd(sz_total, False)
+        self.log("{} {}".format(vspd, msg))
+
         if not nullwrite:
             # TODO this is bad
             log_fn = "up.{:.6f}.txt".format(t0)
@@ -621,7 +643,7 @@ class HttpCli(object):
 
         html = self.conn.tpl_msg.render(
             h2='<a href="/{}">return to /{}</a>'.format(
-                quotep(self.vpath), html_escape(self.vpath, quote=False)
+                quotep(self.vpath), html_escape(self.vpath)
             ),
             pre=msg,
         )
@@ -882,6 +904,7 @@ class HttpCli(object):
             self.log(logmsg)
             return True
 
+        ret = True
         with open_func(*open_args) as f:
             remains = upper - lower
             f.seek(lower)
@@ -894,17 +917,17 @@ class HttpCli(object):
                 if remains < len(buf):
                     buf = buf[:remains]
 
-                remains -= len(buf)
-
                 try:
                     self.s.sendall(buf)
+                    remains -= len(buf)
                 except:
                     logmsg += " \033[31m" + str(upper - remains) + "\033[0m"
-                    self.log(logmsg)
-                    return False
+                    ret = False
+                    break
 
-        self.log(logmsg)
-        return True
+        spd = self._spd((upper - lower) - remains)
+        self.log("{},  {}".format(logmsg, spd))
+        return ret
 
     def tx_md(self, fs_path):
         logmsg = "{:4} {} ".format("", self.req)
@@ -938,7 +961,7 @@ class HttpCli(object):
 
         targs = {
             "edit": "edit" in self.uparam,
-            "title": html_escape(self.vpath, quote=False),
+            "title": html_escape(self.vpath),
             "lastmod": int(ts_md * 1000),
             "md": "",
         }
@@ -979,7 +1002,7 @@ class HttpCli(object):
                 else:
                     vpath += "/" + node
 
-                vpnodes.append([quotep(vpath) + "/", html_escape(node, quote=False)])
+                vpnodes.append([quotep(vpath) + "/", html_escape(node)])
 
         vn, rem = self.auth.vfs.get(
             self.vpath, self.uname, self.readable, self.writable
@@ -1054,7 +1077,12 @@ class HttpCli(object):
             dt = datetime.utcfromtimestamp(inf.st_mtime)
             dt = dt.strftime("%Y-%m-%d %H:%M:%S")
 
-            item = [margin, quotep(href), html_escape(fn, quote=False), sz, dt]
+            try:
+                ext = "---" if is_dir else fn.rsplit(".", 1)[1]
+            except:
+                ext = "%"
+
+            item = [margin, quotep(href), html_escape(fn), sz, ext, dt]
             if is_dir:
                 dirs.append(item)
             else:
@@ -1119,7 +1147,7 @@ class HttpCli(object):
             ts=ts,
             prologue=logues[0],
             epilogue=logues[1],
-            title=html_escape(self.vpath, quote=False),
+            title=html_escape(self.vpath),
             srv_info="</span> /// <span>".join(srv_info),
         )
         self.reply(html.encode("utf-8", "replace"))

copyparty/httpconn.py

@@ -4,6 +4,7 @@ from __future__ import print_function, unicode_literals
 import os
 import sys
 import ssl
+import time
 import socket
 
 try:
@@ -41,9 +42,11 @@ class HttpConn(object):
         self.auth = hsrv.auth
         self.cert_path = hsrv.cert_path
 
+        self.t0 = time.time()
+        self.nbyte = 0
         self.workload = 0
         self.log_func = hsrv.log
-        self.log_src = "{} \033[36m{}".format(addr[0], addr[1]).ljust(26)
+        self.set_rproxy()
 
         env = jinja2.Environment()
         env.loader = jinja2.FileSystemLoader(os.path.join(E.mod, "web"))
@@ -53,6 +56,18 @@ class HttpConn(object):
         self.tpl_md = env.get_template("md.html")
         self.tpl_mde = env.get_template("mde.html")
 
+    def set_rproxy(self, ip=None):
+        if ip is None:
+            color = 36
+            ip = self.addr[0]
+            self.rproxy = None
+        else:
+            color = 34
+            self.rproxy = ip
+
+        self.log_src = "{} \033[{}m{}".format(ip, color, self.addr[1]).ljust(26)
+        return self.log_src
+
     def respath(self, res_name):
         return os.path.join(E.mod, "web", res_name)
 

copyparty/util.py

@@ -3,6 +3,7 @@ from __future__ import print_function, unicode_literals
 
 import re
 import sys
+import time
 import base64
 import struct
 import hashlib
@@ -335,18 +336,28 @@ def read_header(sr):
 
 
 def humansize(sz, terse=False):
-    for unit in ['B', 'KiB', 'MiB', 'GiB', 'TiB']:
+    for unit in ["B", "KiB", "MiB", "GiB", "TiB"]:
         if sz < 1024:
             break
-         
-        sz /= 1024.
-   
-    ret = ' '.join([str(sz)[:4].rstrip('.'), unit])
-    
+
+        sz /= 1024.0
+
+    ret = " ".join([str(sz)[:4].rstrip("."), unit])
+
     if not terse:
         return ret
-    
-    return ret.replace('iB', '').replace(' ', '')
+
+    return ret.replace("iB", "").replace(" ", "")
+
+
+def get_spd(nbyte, t0, t=None):
+    if t is None:
+        t = time.time()
+
+    bps = nbyte / ((t - t0) + 0.001)
+    s1 = humansize(nbyte).replace(" ", "\033[33m").replace("iB", "")
+    s2 = humansize(bps).replace(" ", "\033[35m").replace("iB", "")
+    return "{} \033[0m{}/s\033[0m".format(s1, s2)
 
 
 def undot(path):
@@ -398,6 +409,21 @@ def exclude_dotfiles(filepaths):
             yield fpath
 
 
+def html_escape(s, quote=False):
+    """html.escape but also newlines"""
+    s = (
+        s.replace("&", "&amp;")
+        .replace("<", "&lt;")
+        .replace(">", "&gt;")
+        .replace("\r", "&#13;")
+        .replace("\n", "&#10;")
+    )
+    if quote:
+        s = s.replace('"', "&quot;").replace("'", "&#x27;")
+
+    return s
+
+
 def quotep(txt):
     """url quoter which deals with bytes correctly"""
     btxt = w8enc(txt)
@@ -412,8 +438,8 @@ def quotep(txt):
 def unquotep(txt):
     """url unquoter which deals with bytes correctly"""
     btxt = w8enc(txt)
-    unq1 = btxt.replace(b"+", b" ")
-    unq2 = unquote(unq1)
+    # btxt = btxt.replace(b"+", b" ")
+    unq2 = unquote(btxt)
     return w8dec(unq2)
 
 

copyparty/web/browser.html

@@ -34,13 +34,14 @@
                 <th></th>
                 <th>File Name</th>
                 <th sort="int">File Size</th>
+                <th>T</th>
                 <th>Date</th>
             </tr>
         </thead>
         <tbody>
 
 {%- for f in files %}
-<tr><td>{{ f[0] }}</td><td><a href="{{ f[1] }}">{{ f[2] }}</a></td><td>{{ f[3] }}</td><td>{{ f[4] }}</td></tr>
+<tr><td>{{ f[0] }}</td><td><a href="{{ f[1] }}">{{ f[2] }}</a></td><td>{{ f[3] }}</td><td>{{ f[4] }}</td><td>{{ f[5] }}</td></tr>
 {%- endfor %}
 
         </tbody>

copyparty/web/browser.js

@@ -617,8 +617,8 @@ function unblocked() {
 // show ui to manually start playback of a linked song
 function autoplay_blocked(tid) {
 	show_modal(
-		'<div id="blk_play"><a id="blk_go"></a></div>' +
-		'<div id="blk_abrt"><a id="blk_na">Cancel<br />(show file list)</a></div>');
+		'<div id="blk_play"><a href="#" id="blk_go"></a></div>' +
+		'<div id="blk_abrt"><a href="#" id="blk_na">Cancel<br />(show file list)</a></div>');
 
 	var go = o('blk_go');
 	var na = o('blk_na');
@@ -627,7 +627,8 @@ function autoplay_blocked(tid) {
 	fn = decodeURIComponent(fn.replace(/\+/g, ' '));
 
 	go.textContent = 'Play "' + fn + '"';
-	go.onclick = function () {
+	go.onclick = function (e) {
+		if (e) e.preventDefault();
 		unblocked();
 		mp.au.play();
 	};

copyparty/web/md.css

@@ -109,8 +109,12 @@ h2 a, h4 a, h6 a {
 #mp ol>li {
 	margin: .7em 0;
 }
+strong {
+	color: #000;
+}
 p>em,
-li>em {
+li>em,
+td>em {
 	color: #c50;
 	padding: .1em;
 	border-bottom: .1em solid #bbb;
@@ -289,6 +293,32 @@ blink {
 		text-decoration: underline;
 		border: none;
 	}
+	#mh a:hover {
+		color: #000;
+		background: #ddd;
+	}
+	#toolsbox {
+		overflow: hidden;
+		display: inline-block;
+		background: #eee;
+		height: 1.5em;
+		padding: 0 .2em;
+		margin: 0 .2em;
+		position: absolute;
+	}
+	#toolsbox.open {
+		height: auto;
+		overflow: visible;
+		background: #eee;
+		box-shadow: 0 .2em .2em #ccc;
+		padding-bottom: .2em;
+	}
+	#toolsbox a {
+		display: block;
+	}
+	#toolsbox a+a {
+		text-decoration: none;
+	}
 
 
 
@@ -332,8 +362,12 @@ blink {
 	html.dark #m>ol {
 		border-color: #555;
 	}
+	html.dark strong {
+		color: #fff;
+	}
 	html.dark p>em,
-	html.dark li>em {
+	html.dark li>em,
+	html.dark td>em {
 		color: #f94;
 		border-color: #666;
 	}
@@ -371,6 +405,17 @@ blink {
 		color: #ccc;
 		background: none;
 	}
+	html.dark #mh a:hover {
+		background: #333;
+		color: #fff;
+	}
+	html.dark #toolsbox {
+		background: #222;
+	}
+	html.dark #toolsbox.open {
+		box-shadow: 0 .2em .2em #069;
+		border-radius: 0 0 .4em .4em;
+	}
 }
 
 @media screen and (min-width: 66em) {
@@ -541,7 +586,8 @@ blink {
 		color: #240;
 	}
 	html.dark p>em,
-	html.dark li>em {
+	html.dark li>em,
+	html.dark td>em {
 		color: #940;
 	}
 }

copyparty/web/md.html

@@ -17,7 +17,14 @@
 			<a id="save" href="?edit">save</a>
 			<a id="sbs" href="#">sbs</a>
 			<a id="nsbs" href="#">editor</a>
-			<a id="help" href="#">help</a>
+			<div id="toolsbox">
+				<a id="tools" href="#">tools</a>
+				<a id="fmt_table" href="#">prettify table (ctrl-k)</a>
+				<a id="iter_uni" href="#">non-ascii: iterate (ctrl-u)</a>
+				<a id="mark_uni" href="#">non-ascii: markup</a>
+				<a id="cfg_uni" href="#">non-ascii: whitelist</a>
+				<a id="help" href="#">help</a>
+			</div>
 		{%- else %}
 			<a href="?edit">edit (basic)</a>
 			<a href="?edit2">edit (fancy)</a>
@@ -46,6 +53,9 @@ write markdown (most html is 🙆 too)
 
 ## hotkey list
 * `Ctrl-S` to save
+* `Ctrl-E` to toggle mode
+* `Ctrl-K` to prettyprint a table
+* `Ctrl-U` to iterate non-ascii chars
 * `Ctrl-H` / `Ctrl-Shift-H` to create a header
 * `TAB` / `Shift-TAB` to indent/dedent a selection
 

copyparty/web/md2.js

@@ -2,10 +2,16 @@
 var server_md = dom_src.value;
 
 
+// the non-ascii whitelist
+var esc_uni_whitelist = '\\n\\t\\x20-\\x7eÆØÅæøå';
+var js_uni_whitelist = eval('\'' + esc_uni_whitelist + '\'');
+
+
 // dom nodes
 var dom_swrap = document.getElementById('mtw');
 var dom_sbs = document.getElementById('sbs');
 var dom_nsbs = document.getElementById('nsbs');
+var dom_tbox = document.getElementById('toolsbox');
 var dom_ref = (function () {
     var d = document.createElement('div');
     d.setAttribute('id', 'mtr');
@@ -164,7 +170,7 @@ redraw = (function () {
             dst.scrollTop = 0;
             return;
         }
-        if (y + 8 + src.clientHeight > src.scrollHeight) {
+        if (y + 48 + src.clientHeight > src.scrollHeight) {
             dst.scrollTop = dst.scrollHeight - dst.clientHeight;
             return;
         }
@@ -339,7 +345,7 @@ function savechk_cb() {
     server_md = this.txt;
     draw_md();
     toast('font-size:6em;font-family:serif;color:#cf6;width:4em;',
-        'OK✔️<span style="font-size:.2em;color:#999">' + this.ntry + '</span>');
+        'OK✔️<span style="font-size:.2em;color:#999;position:absolute">' + this.ntry + '</span>');
 }
 
 function toast(style, msg) {
@@ -427,6 +433,9 @@ function setsel(s) {
     dom_src.value = [s.pre, s.sel, s.post].join('');
     dom_src.setSelectionRange(s.car, s.cdr, dom_src.selectionDirection);
     dom_src.oninput();
+    // support chrome:
+    dom_src.blur();
+    dom_src.focus();
 }
 
 
@@ -500,7 +509,8 @@ function md_newline() {
     var s = linebounds(true),
         ln = s.md.substring(s.n1, s.n2),
         m1 = /^( *)([0-9]+)(\. +)/.exec(ln),
-        m2 = /^[ \t>+-]*(\* )?/.exec(ln);
+        m2 = /^[ \t>+-]*(\* )?/.exec(ln),
+        drop = dom_src.selectionEnd - dom_src.selectionStart;
 
     var pre = m2[0];
     if (m1 !== null)
@@ -512,7 +522,7 @@ function md_newline() {
 
     s.pre = s.md.substring(0, s.car) + '\n' + pre;
     s.sel = '';
-    s.post = s.md.substring(s.car);
+    s.post = s.md.substring(s.car + drop);
     s.car = s.cdr = s.pre.length;
     setsel(s);
     return false;
@@ -522,11 +532,17 @@ function md_newline() {
 // backspace
 function md_backspace() {
     var s = linebounds(true),
-        ln = s.md.substring(s.n1, s.n2),
-        m = /^[ \t>+-]*(\* )?([0-9]+\. +)?/.exec(ln);
+        o0 = dom_src.selectionStart,
+        left = s.md.slice(s.n1, o0),
+        m = /^[ \t>+-]*(\* )?([0-9]+\. +)?/.exec(left);
 
+    // if car is in whitespace area, do nothing
+    if (/^\s*$/.test(left))
+        return true;
+
+    // same if line is all-whitespace or non-markup
     var v = m[0].replace(/[^ ]/g, " ");
-    if (v === m[0] || v.length !== ln.length)
+    if (v === m[0] || v.length !== left.length)
         return true;
 
     s.pre = s.md.substring(0, s.n1) + v;
@@ -540,8 +556,8 @@ function md_backspace() {
 
 // paragraph jump
 function md_p_jump(down) {
-    var ofs = dom_src.selectionStart;
-    var txt = dom_src.value;
+    var txt = dom_src.value,
+        ofs = dom_src.selectionStart;
 
     if (down) {
         while (txt[ofs] == '\n' && --ofs > 0);
@@ -562,6 +578,218 @@ function md_p_jump(down) {
 }
 
 
+function reLastIndexOf(txt, ptn, end) {
+    var ofs = (typeof end !== 'undefined') ? end : txt.length;
+    end = ofs;
+    while (ofs >= 0) {
+        var sub = txt.slice(ofs, end);
+        if (ptn.test(sub))
+            return ofs;
+
+        ofs--;
+    }
+    return -1;
+}
+
+
+// table formatter
+function fmt_table(e) {
+    if (e) e.preventDefault();
+    //dom_tbox.setAttribute('class', '');
+
+    var txt = dom_src.value,
+        ofs = dom_src.selectionStart,
+        //o0 = txt.lastIndexOf('\n\n', ofs),
+        //o1 = txt.indexOf('\n\n', ofs);
+        o0 = reLastIndexOf(txt, /\n\s*\n/m, ofs),
+        o1 = txt.slice(ofs).search(/\n\s*\n|\n\s*$/m);
+    // note \s contains \n but its fine
+
+    if (o0 < 0)
+        o0 = 0;
+    else {
+        // seek past the hit
+        var m = /\n\s*\n/m.exec(txt.slice(o0));
+        o0 += m[0].length;
+    }
+
+    o1 = o1 < 0 ? txt.length : o1 + ofs;
+
+    var err = 'cannot format table due to ',
+        tab = txt.slice(o0, o1).split(/\s*\n/),
+        re_ind = /^\s*/,
+        ind = tab[1].match(re_ind)[0],
+        r0_ind = tab[0].slice(0, ind.length),
+        lpipe = tab[1].indexOf('|') < tab[1].indexOf('-'),
+        rpipe = tab[1].lastIndexOf('|') > tab[1].lastIndexOf('-'),
+        re_lpipe = lpipe ? /^\s*\|\s*/ : /^\s*/,
+        re_rpipe = rpipe ? /\s*\|\s*$/ : /\s*$/;
+
+    // the second row defines the table,
+    // need to process that first
+    var tmp = tab[0];
+    tab[0] = tab[1];
+    tab[1] = tmp;
+
+    for (var a = 0; a < tab.length; a++) {
+        var row_name = (a == 1) ? 'header' : 'row#' + (a + 1);
+
+        var ind2 = tab[a].match(re_ind)[0];
+        if (ind != ind2 && a != 1)  // the table can be a list entry or something, ignore [0]
+            return alert(err + 'indentation mismatch on row#2 and ' + row_name + ',\n' + tab[a]);
+
+        var t = tab[a].slice(ind.length);
+        t = t.replace(re_lpipe, "");
+        t = t.replace(re_rpipe, "");
+        tab[a] = t.split(/\s*\|\s*/g);
+
+        if (a == 0)
+            ncols = tab[a].length;
+        else if (ncols < tab[a].length)
+            return alert(err + 'num.columns(' + row_name + ') exceeding row#2;  ' + ncols + ' < ' + tab[a].length);
+
+        // if row has less columns than row2, fill them in
+        while (tab[a].length < ncols)
+            tab[a].push('');
+    }
+
+    // aight now swap em back
+    tmp = tab[0];
+    tab[0] = tab[1];
+    tab[1] = tmp;
+
+    var re_align = /^ *(:?)-+(:?) *$/;
+    var align = [];
+    for (var col = 0; col < tab[1].length; col++) {
+        var m = tab[1][col].match(re_align);
+        if (!m)
+            return alert(err + 'invalid column specification, row#2, col ' + (col + 1) + ', [' + tab[1][col] + ']');
+
+        if (m[2]) {
+            if (m[1])
+                align.push('c');
+            else
+                align.push('r');
+        }
+        else
+            align.push('l');
+    }
+
+    var pad = [];
+    var tmax = 0;
+    for (var col = 0; col < ncols; col++) {
+        var max = 0;
+        for (var row = 0; row < tab.length; row++)
+            max = Math.max(max, tab[row][col].length);
+
+        var s = '';
+        for (var n = 0; n < max; n++)
+            s += ' ';
+
+        pad.push(s);
+        tmax = Math.max(max, tmax);
+    }
+
+    var dashes = '';
+    for (var a = 0; a < tmax; a++)
+        dashes += '-';
+
+    var ret = [];
+    for (var row = 0; row < tab.length; row++) {
+        var ln = [];
+        for (var col = 0; col < tab[row].length; col++) {
+            var p = pad[col];
+            var s = tab[row][col];
+
+            if (align[col] == 'l') {
+                s = (s + p).slice(0, p.length);
+            }
+            else if (align[col] == 'r') {
+                s = (p + s).slice(-p.length);
+            }
+            else {
+                var pt = p.length - s.length;
+                var pl = p.slice(0, Math.floor(pt / 2));
+                var pr = p.slice(0, pt - pl.length);
+                s = pl + s + pr;
+            }
+
+            if (row == 1) {
+                if (align[col] == 'l')
+                    s = dashes.slice(0, p.length);
+                else if (align[col] == 'r')
+                    s = dashes.slice(0, p.length - 1) + ':';
+                else
+                    s = ':' + dashes.slice(0, p.length - 2) + ':';
+            }
+            ln.push(s);
+        }
+        ret.push(ind + '| ' + ln.join(' | ') + ' |');
+    }
+
+    // restore any markup in the row0 gutter
+    ret[0] = r0_ind + ret[0].slice(ind.length);
+
+    ret = {
+        "pre": txt.slice(0, o0),
+        "sel": ret.join('\n'),
+        "post": txt.slice(o1),
+        "car": o0,
+        "cdr": o0
+    };
+    setsel(ret);
+}
+
+
+// show unicode
+function mark_uni(e) {
+    if (e) e.preventDefault();
+    dom_tbox.setAttribute('class', '');
+
+    var txt = dom_src.value,
+        ptn = new RegExp('([^' + js_uni_whitelist + ']+)', 'g');
+
+    mod = txt.replace(/\r/g, "").replace(ptn, "\u2588\u2770$1\u2771");
+
+    if (txt == mod) {
+        alert('no results;  no modifications were made');
+        return;
+    }
+    dom_src.value = mod;
+}
+
+
+// iterate unicode
+function iter_uni(e) {
+    if (e) e.preventDefault();
+
+    var txt = dom_src.value,
+        ofs = dom_src.selectionDirection == "forward" ? dom_src.selectionEnd : dom_src.selectionStart,
+        re = new RegExp('([^' + js_uni_whitelist + ']+)'),
+        m = re.exec(txt.slice(ofs));
+
+    if (!m) {
+        alert('no more hits from cursor onwards');
+        return;
+    }
+    ofs += m.index;
+
+    dom_src.setSelectionRange(ofs, ofs + m[0].length, "forward");
+    dom_src.oninput();
+    // support chrome:
+    dom_src.blur();
+    dom_src.focus();
+}
+
+
+// configure whitelist
+function cfg_uni(e) {
+    if (e) e.preventDefault();
+    esc_uni_whitelist = prompt("unicode whitelist", esc_uni_whitelist);
+    js_uni_whitelist = eval('\'' + esc_uni_whitelist + '\'');
+}
+
+
 // hotkeys / toolbar
 (function () {
     function keydown(ev) {
@@ -609,6 +837,19 @@ function md_p_jump(down) {
             if (!ctrl && !ev.shiftKey && kc == 8) {
                 return md_backspace();
             }
+            if (ctrl && (ev.code == "KeyK")) {
+                fmt_table();
+                return false;
+            }
+            if (ctrl && (ev.code == "KeyU")) {
+                iter_uni();
+                return false;
+            }
+            if (ctrl && (ev.code == "KeyE")) {
+                dom_nsbs.click();
+                //fmt_table();
+                return false;
+            }
             var up = ev.code == "ArrowUp" || kc == 38;
             var dn = ev.code == "ArrowDown" || kc == 40;
             if (ctrl && (up || dn)) {
@@ -622,8 +863,17 @@ function md_p_jump(down) {
 })();
 
 
+document.getElementById('tools').onclick = function (e) {
+    if (e) e.preventDefault();
+    var is_open = dom_tbox.getAttribute('class') != 'open';
+    dom_tbox.setAttribute('class', is_open ? 'open' : '');
+};
+
+
 document.getElementById('help').onclick = function (e) {
     if (e) e.preventDefault();
+    dom_tbox.setAttribute('class', '');
+
     var dom = document.getElementById('helpbox');
     var dtxt = dom.getElementsByTagName('textarea');
     if (dtxt.length > 0) {
@@ -638,6 +888,12 @@ document.getElementById('help').onclick = function (e) {
 };
 
 
+document.getElementById('fmt_table').onclick = fmt_table;
+document.getElementById('mark_uni').onclick = mark_uni;
+document.getElementById('iter_uni').onclick = iter_uni;
+document.getElementById('cfg_uni').onclick = cfg_uni;
+
+
 // blame steen
 action_stack = (function () {
     var hist = {

copyparty/web/mde.css

@@ -160,8 +160,12 @@ h2 {
 .mdo ol>li {
 	margin: .7em 0;
 }
+strong {
+	color: #000;
+}
 p>em,
-li>em {
+li>em,
+td>em {
 	color: #c50;
 	padding: .1em;
 	border-bottom: .1em solid #bbb;
@@ -253,8 +257,12 @@ html.dark .mdo>ul,
 html.dark .mdo>ol {
     border-color: #555;
 }
+html.dark strong {
+    color: #fff;
+}
 html.dark p>em,
-html.dark li>em {
+html.dark li>em,
+html.dark td>em {
     color: #f94;
     border-color: #666;
 }

copyparty/web/mde.js

@@ -121,7 +121,7 @@ function save(mde) {
     fd.append("lastmod", (force ? -1 : last_modified));
     fd.append("body", txt);
 
-    var url = (document.location + '').split('?')[0] + '?raw';
+    var url = (document.location + '').split('?')[0];
     var xhr = new XMLHttpRequest();
     xhr.open('POST', url, true);
     xhr.responseType = 'text';

docs/rclone.md

@@ -0,0 +1,62 @@
+# using rclone to mount a remote copyparty server as a local filesystem
+
+speed estimates with server and client on the same win10 machine:
+* `1070 MiB/s` with rclone as both server and client
+* `570 MiB/s` with rclone-client and `copyparty -ed -j16` as server
+* `220 MiB/s` with rclone-client and `copyparty -ed` as server
+* `100 MiB/s` with [../bin/copyparty-fuse.py](../bin/copyparty-fuse.py) as client
+
+when server is on another machine (1gbit LAN),
+* `75 MiB/s` with [../bin/copyparty-fuse.py](../bin/copyparty-fuse.py) as client
+* `92 MiB/s` with rclone-client and `copyparty -ed` as server
+* `103 MiB/s` (connection max) with `copyparty -ed -j16` and all the others
+
+
+# creating the config file
+
+if you want to use password auth, add `headers = Cookie,cppwd=fgsfds` below
+
+
+### on windows clients:
+```
+(
+echo [cpp]
+echo type = http
+echo url = http://127.0.0.1:3923/
+) > %userprofile%\.config\rclone\rclone.conf
+```
+
+also install the windows dependencies: [winfsp](https://github.com/billziss-gh/winfsp/releases/latest)
+
+
+### on unix clients:
+```
+cat > ~/.config/rclone/rclone.conf <<'EOF'
+[cpp]
+type = http
+url = http://127.0.0.1:3923/
+EOF
+```
+
+
+# mounting the copyparty server locally
+```
+rclone.exe mount --vfs-cache-max-age 5s --attr-timeout 5s --dir-cache-time 5s cpp: Z:
+```
+
+
+# use rclone as server too, replacing copyparty
+
+feels out of place but is too good not to mention
+
+```
+rclone.exe serve http --read-only .
+```
+
+* `webdav` gives write-access but `http` is twice as fast
+* `ftp` is buggy, avoid
+
+
+# bugs
+
+* rclone-client throws an exception if you try to read an empty file (should return zero bytes)

scripts/sfx.py

@@ -2,7 +2,7 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
-import re, os, sys, stat, time, shutil, tarfile, hashlib, platform, tempfile
+import re, os, sys, time, shutil, signal, tarfile, hashlib, platform, tempfile
 import subprocess as sp
 
 """
@@ -29,6 +29,7 @@ STAMP = None
 PY2 = sys.version_info[0] == 2
 sys.dont_write_bytecode = True
 me = os.path.abspath(os.path.realpath(__file__))
+cpp = None
 
 
 def eprint(*args, **kwargs):
@@ -191,6 +192,16 @@ def makesfx(tar_src, ver, ts):
 # skip 0
 
 
+def u8(gen):
+    try:
+        for s in gen:
+            yield s.decode("utf-8", "ignore")
+    except:
+        yield s
+        for s in gen:
+            yield s
+
+
 def get_py_win(ret):
     tops = []
     p = str(os.getenv("LocalAppdata"))
@@ -216,11 +227,11 @@ def get_py_win(ret):
     # $WIRESHARK_SLOGAN
     for top in tops:
         try:
-            for name1 in sorted(os.listdir(top), reverse=True):
+            for name1 in u8(sorted(os.listdir(top), reverse=True)):
                 if name1.lower().startswith("python"):
                     path1 = os.path.join(top, name1)
                     try:
-                        for name2 in os.listdir(path1):
+                        for name2 in u8(os.listdir(path1)):
                             if name2.lower() == "python.exe":
                                 path2 = os.path.join(path1, name2)
                                 ret[path2.lower()] = path2
@@ -237,7 +248,7 @@ def get_py_nix(ret):
             next
 
         try:
-            for fn in os.listdir(bindir):
+            for fn in u8(os.listdir(bindir)):
                 if ptn.match(fn):
                     fn = os.path.join(bindir, fn)
                     ret[fn.lower()] = fn
@@ -295,17 +306,19 @@ def hashfile(fn):
 def unpack():
     """unpacks the tar yielded by `data`"""
     name = "pe-copyparty"
+    tag = "v" + str(STAMP)
     withpid = "{}.{}".format(name, os.getpid())
     top = tempfile.gettempdir()
     final = os.path.join(top, name)
     mine = os.path.join(top, withpid)
     tar = os.path.join(mine, "tar")
-    tag_mine = os.path.join(mine, "v" + str(STAMP))
-    tag_final = os.path.join(final, "v" + str(STAMP))
 
-    if os.path.exists(tag_final):
-        msg("found early")
-        return final
+    try:
+        if tag in os.listdir(final):
+            msg("found early")
+            return final
+    except:
+        pass
 
     nwrite = 0
     os.mkdir(mine)
@@ -328,12 +341,15 @@ def unpack():
 
     os.remove(tar)
 
-    with open(tag_mine, "wb") as f:
+    with open(os.path.join(mine, tag), "wb") as f:
         f.write(b"h\n")
 
-    if os.path.exists(tag_final):
-        msg("found late")
-        return final
+    try:
+        if tag in os.listdir(final):
+            msg("found late")
+            return final
+    except:
+        pass
 
     try:
         if os.path.islink(final):
@@ -352,7 +368,7 @@ def unpack():
             msg("reloc fail,", mine)
             return mine
 
-    for fn in os.listdir(top):
+    for fn in u8(os.listdir(top)):
         if fn.startswith(name) and fn not in [name, withpid]:
             try:
                 old = os.path.join(top, fn)
@@ -418,10 +434,15 @@ def get_payload():
 def confirm():
     msg()
     msg("*** hit enter to exit ***")
-    raw_input() if PY2 else input()
+    try:
+        raw_input() if PY2 else input()
+    except:
+        pass
 
 
 def run(tmp, py):
+    global cpp
+
     msg("OK")
     msg("will use:", py)
     msg("bound to:", tmp)
@@ -437,8 +458,11 @@ def run(tmp, py):
         pass
 
     fp_py = os.path.join(tmp, "py")
-    with open(fp_py, "wb") as f:
-        f.write(py.encode("utf-8") + b"\n")
+    try:
+        with open(fp_py, "wb") as f:
+            f.write(py.encode("utf-8") + b"\n")
+    except:
+        pass
 
     # avoid loading ./copyparty.py
     cmd = [
@@ -450,16 +474,21 @@ def run(tmp, py):
     ] + list(sys.argv[1:])
 
     msg("\n", cmd, "\n")
-    p = sp.Popen(str(x) for x in cmd)
+    cpp = sp.Popen(str(x) for x in cmd)
     try:
-        p.wait()
+        cpp.wait()
     except:
-        p.wait()
+        cpp.wait()
 
-    if p.returncode != 0:
+    if cpp.returncode != 0:
         confirm()
 
-    sys.exit(p.returncode)
+    sys.exit(cpp.returncode)
+
+
+def bye(sig, frame):
+    if cpp is not None:
+        cpp.terminate()
 
 
 def main():
@@ -494,6 +523,8 @@ def main():
 
     # skip 0
 
+    signal.signal(signal.SIGTERM, bye)
+
     tmp = unpack()
     fp_py = os.path.join(tmp, "py")
     if os.path.exists(fp_py):

scripts/sfx.sh

@@ -32,8 +32,12 @@ dir="$(
 
 # detect available pythons
 (IFS=:; for d in $PATH; do
-	printf '%s\n' "$d"/python* "$d"/pypy* | tac;
-done) | grep -E '(python|pypy)[0-9\.-]*$' > $dir/pys || true
+	printf '%s\n' "$d"/python* "$d"/pypy*;
+done) |
+(sed -E 's/(.*\/[^/0-9]+)([0-9]?[^/]*)$/\2 \1/' || cat) |
+(sort -nr || cat) |
+(sed -E 's/([^ ]*) (.*)/\2\1/' || cat) |
+grep -E '/(python|pypy)[0-9\.-]*$' >$dir/pys || true
 
 # see if we made a choice before
 [ -z "$pybin" ] && pybin="$(cat $dir/py 2>/dev/null || true)"

scripts/speedtest-fs.py

@@ -0,0 +1,164 @@
+#!/usr/bin/env python
+
+import os
+import sys
+import stat
+import time
+import signal
+import traceback
+import threading
+from queue import Queue
+
+
+"""speedtest-fs: filesystem performance estimate"""
+__author__ = "ed <copyparty@ocv.me>"
+__copyright__ = 2020
+__license__ = "MIT"
+__url__ = "https://github.com/9001/copyparty/"
+
+
+def get_spd(nbyte, nsec):
+    if not nsec:
+        return "0.000 MB   0.000 sec   0.000 MB/s"
+
+    mb = nbyte / (1024 * 1024.0)
+    spd = mb / nsec
+
+    return f"{mb:.3f} MB   {nsec:.3f} sec   {spd:.3f} MB/s"
+
+
+class Inf(object):
+    def __init__(self, t0):
+        self.msgs = []
+        self.errors = []
+        self.reports = []
+        self.mtx_msgs = threading.Lock()
+        self.mtx_reports = threading.Lock()
+
+        self.n_byte = 0
+        self.n_sec = 0
+        self.n_done = 0
+        self.t0 = t0
+
+        thr = threading.Thread(target=self.print_msgs)
+        thr.daemon = True
+        thr.start()
+
+    def msg(self, fn, n_read):
+        with self.mtx_msgs:
+            self.msgs.append(f"{fn} {n_read}")
+
+    def err(self, fn):
+        with self.mtx_reports:
+            self.errors.append(f"{fn}\n{traceback.format_exc()}")
+
+    def print_msgs(self):
+        while True:
+            time.sleep(0.02)
+            with self.mtx_msgs:
+                msgs = self.msgs
+                self.msgs = []
+
+            if not msgs:
+                continue
+
+            msgs = msgs[-64:]
+            msgs = [f"{get_spd(self.n_byte, self.n_sec)}   {x}" for x in msgs]
+            print("\n".join(msgs))
+
+    def report(self, fn, n_byte, n_sec):
+        with self.mtx_reports:
+            self.reports.append([n_byte, n_sec, fn])
+            self.n_byte += n_byte
+            self.n_sec += n_sec
+
+    def done(self):
+        with self.mtx_reports:
+            self.n_done += 1
+
+
+def get_files(dir_path):
+    for fn in os.listdir(dir_path):
+        fn = os.path.join(dir_path, fn)
+        st = os.stat(fn).st_mode
+
+        if stat.S_ISDIR(st):
+            yield from get_files(fn)
+
+        if stat.S_ISREG(st):
+            yield fn
+
+
+def worker(q, inf, read_sz):
+    while True:
+        fn = q.get()
+        if not fn:
+            break
+
+        n_read = 0
+        try:
+            t0 = time.time()
+            with open(fn, "rb") as f:
+                while True:
+                    buf = f.read(read_sz)
+                    if not buf:
+                        break
+
+                    n_read += len(buf)
+                    inf.msg(fn, n_read)
+
+            inf.report(fn, n_read, time.time() - t0)
+        except:
+            inf.err(fn)
+
+    inf.done()
+
+
+def sighandler(signo, frame):
+    os._exit(0)
+
+
+def main():
+    signal.signal(signal.SIGINT, sighandler)
+
+    root = "."
+    if len(sys.argv) > 1:
+        root = sys.argv[1]
+
+    t0 = time.time()
+    q = Queue(256)
+    inf = Inf(t0)
+
+    num_threads = 8
+    read_sz = 32 * 1024
+    for _ in range(num_threads):
+        thr = threading.Thread(target=worker, args=(q, inf, read_sz,))
+        thr.daemon = True
+        thr.start()
+
+    for fn in get_files(root):
+        q.put(fn)
+
+    for _ in range(num_threads):
+        q.put(None)
+
+    while inf.n_done < num_threads:
+        time.sleep(0.1)
+
+    t2 = time.time()
+    print("\n")
+
+    log = inf.reports
+    log.sort()
+    for nbyte, nsec, fn in log[-64:]:
+        print(f"{get_spd(nbyte, nsec)}   {fn}")
+
+    print()
+    print("\n".join(inf.errors))
+
+    print(get_spd(inf.n_byte, t2 - t0))
+
+
+if __name__ == "__main__":
+    main()
+

srv/test.md

@@ -1,5 +1,16 @@
 ### hello world
 
+* qwe
+* asd
+  * zxc
+  * 573
+    * one
+    * two
+    
+  * |||
+    |--|--|
+    |listed|table|
+
 ```
 [72....................................................................]
 [80............................................................................]
@@ -21,6 +32,8 @@
 l[i]=1I;(){}o0O</> var foo = "$(`bar`)"; a's'd
 ```
 
+🔍🌽.📕.🍙🔎
+
 [](#s1)
 [s1](#s1)
 [#s1](#s1)
@@ -121,6 +134,15 @@ a newline toplevel
 | a table | on the right |
 | second row | foo bar |
 
+||
+--|:-:|-:
+a table | big text in this | aaakbfddd
+second row | centred | bbb
+
+||
+--|--|--
+foo
+
 * list entry
 * [x] yes
 * [ ] no
@@ -209,3 +231,7 @@ unrelated neat stuff:
 awk '/./ {printf "%s %d\n", $0, NR; next} 1' <test.md >ln.md
 gawk '{print gensub(/([a-zA-Z\.])/,NR" \\1","1")}' <test.md >ln.md
 ```
+
+a|b|c
+--|--|--
+foo