v0.7.6

.vscode/launch.json

@@ -20,6 +20,13 @@
                 "srv::r:aed:cnodupe"
             ]
         },
+        {
+            "name": "No debug",
+            "preLaunchTask": "no_dbg",
+            "type": "python",
+            //"request": "attach", "port": 42069
+            // fork: nc -l 42069 </dev/null
+        },
         {
             "name": "Run active unit test",
             "type": "python",

.vscode/settings.json

@@ -50,11 +50,9 @@
     "files.associations": {
         "*.makefile": "makefile"
     },
-    "editor.codeActionsOnSaveTimeout": 9001,
-    "editor.formatOnSaveTimeout": 9001,
-    //
-    //  things you may wanna edit:
-    //
-    "python.pythonPath": "/usr/bin/python3",
-    //"python.linting.enabled": true,
+    "python.formatting.blackArgs": [
+        "-t",
+        "py27"
+    ],
+    "python.linting.enabled": true,
 }

.vscode/tasks.json

@@ -5,6 +5,11 @@
             "label": "pre",
             "command": "true;rm -rf inc/* inc/.hist/;mkdir -p inc;",
             "type": "shell"
+        },
+        {
+            "label": "no_dbg",
+            "command": "${config:python.pythonPath} -m copyparty -ed -emp -e2d -e2s -a ed:wark -v srv::r:aed:cnodupe ;exit 1",
+            "type": "shell"
         }
     ]
 }

README.md

@@ -76,8 +76,8 @@ optional, will eventually enable thumbnails:
 # sfx
 
 currently there are two self-contained binaries:
-* `copyparty-sfx.sh` for unix (linux and osx) -- smaller, more robust
-* `copyparty-sfx.py` for windows (unix too) -- crossplatform, beta
+* [copyparty-sfx.py](https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.py) -- pure python, works everywhere
+* [copyparty-sfx.sh](https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.sh) -- smaller, but only for linux and macos
 
 launch either of them (**use sfx.py on systemd**) and it'll unpack and run copyparty, assuming you have python installed of course
 

contrib/README.md

@@ -10,7 +10,12 @@
 * modify `10.13.1.1` as necessary if you wish to support browsers without javascript
 
 ### [`explorer-nothumbs-nofoldertypes.reg`](explorer-nothumbs-nofoldertypes.reg)
-disables thumbnails and folder-type detection in windows explorer, makes it way faster (especially for slow/networked locations (such as copyparty-fuse))
+* disables thumbnails and folder-type detection in windows explorer
+* makes it way faster (especially for slow/networked locations (such as copyparty-fuse))
+
+### [`cfssl.sh`](cfssl.sh)
+* creates CA and server certificates using cfssl
+* give a 3rd argument to install it to your copyparty config
 
 # OS integration
 init-scripts to start copyparty as a service

contrib/cfssl.sh

@@ -0,0 +1,72 @@
+#!/bin/bash
+set -e
+
+# ca-name and server-name
+ca_name="$1"
+srv_name="$2"
+
+[ -z "$srv_name" ] && {
+	echo "need arg 1: ca name"
+	echo "need arg 2: server name"
+	exit 1
+}
+
+
+gen_ca() {
+	(tee /dev/stderr <<EOF
+{"CN": "$ca_name ca",
+"CA": {"expiry":"87600h", "pathlen":0},
+"key": {"algo":"rsa", "size":4096},
+"names": [{"O":"$ca_name ca"}]}
+EOF
+	)|
+	cfssl gencert -initca - |
+	cfssljson -bare ca
+	
+	mv ca-key.pem ca.key
+	rm ca.csr
+}
+
+
+gen_srv() {
+	(tee /dev/stderr <<EOF
+{"key": {"algo":"rsa", "size":4096},
+"names": [{"O":"$ca_name - $srv_name"}]}
+EOF
+	)|
+	cfssl gencert -ca ca.pem -ca-key ca.key \
+		-profile=www -hostname="$srv_name.$ca_name" - |
+	cfssljson -bare "$srv_name"
+
+	mv "$srv_name-key.pem" "$srv_name.key"
+	rm "$srv_name.csr"
+}
+
+
+# create ca if not exist
+[ -e ca.key ] ||
+	gen_ca
+
+# always create server cert
+gen_srv
+
+
+# dump cert info
+show() {
+	openssl x509 -text -noout -in $1 |
+	awk '!o; {o=0} /[0-9a-f:]{16}/{o=1}'
+}
+show ca.pem
+show "$srv_name.pem"
+
+
+# write cert into copyparty config
+[ -z "$3" ] || {
+	mkdir -p ~/.config/copyparty
+	cat "$srv_name".{key,pem} ca.pem >~/.config/copyparty/cert.pem 
+}
+
+
+# rm *.key *.pem
+# cfssl print-defaults config
+# cfssl print-defaults csr

copyparty/__main__.py

@@ -8,7 +8,9 @@ __copyright__ = 2019
 __license__ = "MIT"
 __url__ = "https://github.com/9001/copyparty/"
 
+import re
 import os
+import sys
 import time
 import shutil
 import filecmp
@@ -19,7 +21,13 @@ from textwrap import dedent
 from .__init__ import E, WINDOWS, VT100
 from .__version__ import S_VERSION, S_BUILD_DT, CODENAME
 from .svchub import SvcHub
-from .util import py_desc
+from .util import py_desc, align_tab
+
+HAVE_SSL = True
+try:
+    import ssl
+except:
+    HAVE_SSL = False
 
 
 class RiceFormatter(argparse.HelpFormatter):
@@ -85,6 +93,73 @@ def ensure_cert():
     # printf 'NO\n.\n.\n.\n.\ncopyparty-insecure\n.\n' | faketime '2000-01-01 00:00:00' openssl req -x509 -sha256 -newkey rsa:2048 -keyout insecure.pem -out insecure.pem -days $((($(printf %d 0x7fffffff)-$(date +%s --date=2000-01-01T00:00:00Z))/(60*60*24))) -nodes && ls -al insecure.pem && openssl x509 -in insecure.pem -text -noout
 
 
+def configure_ssl_ver(al):
+    def terse_sslver(txt):
+        txt = txt.lower()
+        for c in ["_", "v", "."]:
+            txt = txt.replace(c, "")
+
+        return txt.replace("tls10", "tls1")
+
+    # oh man i love openssl
+    # check this out
+    # hold my beer
+    ptn = re.compile(r"^OP_NO_(TLS|SSL)v")
+    sslver = terse_sslver(al.ssl_ver).split(",")
+    flags = [k for k in ssl.__dict__ if ptn.match(k)]
+    # SSLv2 SSLv3 TLSv1 TLSv1_1 TLSv1_2 TLSv1_3
+    if "help" in sslver:
+        avail = [terse_sslver(x[6:]) for x in flags]
+        avail = " ".join(sorted(avail) + ["all"])
+        print("\navailable ssl/tls versions:\n  " + avail)
+        sys.exit(0)
+
+    al.ssl_flags_en = 0
+    al.ssl_flags_de = 0
+    for flag in sorted(flags):
+        ver = terse_sslver(flag[6:])
+        num = getattr(ssl, flag)
+        if ver in sslver:
+            al.ssl_flags_en |= num
+        else:
+            al.ssl_flags_de |= num
+
+    if sslver == ["all"]:
+        x = al.ssl_flags_en
+        al.ssl_flags_en = al.ssl_flags_de
+        al.ssl_flags_de = x
+
+    for k in ["ssl_flags_en", "ssl_flags_de"]:
+        num = getattr(al, k)
+        print("{}: {:8x} ({})".format(k, num, num))
+
+    # think i need that beer now
+
+
+def configure_ssl_ciphers(al):
+    ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
+    if al.ssl_ver:
+        ctx.options &= ~al.ssl_flags_en
+        ctx.options |= al.ssl_flags_de
+
+    is_help = al.ciphers == "help"
+
+    if al.ciphers and not is_help:
+        try:
+            ctx.set_ciphers(al.ciphers)
+        except:
+            print("\n\033[1;31mfailed to set ciphers\033[0m\n")
+
+    if not hasattr(ctx, "get_ciphers"):
+        print("cannot read cipher list: openssl or python too old")
+    else:
+        ciphers = [x["description"] for x in ctx.get_ciphers()]
+        print("\n  ".join(["\nenabled ciphers:"] + align_tab(ciphers) + [""]))
+
+    if is_help:
+        sys.exit(0)
+
+
 def main():
     time.strptime("19970815", "%Y%m%d")  # python#7980
     if WINDOWS:
@@ -96,7 +171,8 @@ def main():
     print(f.format(S_VERSION, CODENAME, S_BUILD_DT, desc))
 
     ensure_locale()
-    ensure_cert()
+    if HAVE_SSL:
+        ensure_cert()
 
     ap = argparse.ArgumentParser(
         formatter_class=RiceFormatter,
@@ -133,6 +209,10 @@ def main():
               "save,get" dumps to file and returns the page like a GET
               "print,get" prints the data in the log and returns GET
               (leave out the ",get" to return an error instead)
+
+            --ciphers help = available ssl/tls ciphers,
+            --ssl-ver help = available ssl/tls versions,
+              default is what python considers safe, usually >= TLS1
             """
         ),
     )
@@ -155,6 +235,14 @@ def main():
     ap.add_argument("-nid", action="store_true", help="no info disk-usage")
     ap.add_argument("--no-sendfile", action="store_true", help="disable sendfile")
     ap.add_argument("--urlform", type=str, default="print,get", help="how to handle url-forms")
+
+    ap2 = ap.add_argument_group('SSL/TLS options')
+    ap2.add_argument("--http-only", action="store_true", help="disable ssl/tls")
+    ap2.add_argument("--https-only", action="store_true", help="disable plaintext")
+    ap2.add_argument("--ssl-ver", type=str, help="ssl/tls versions to allow")
+    ap2.add_argument("--ciphers", metavar="LIST", help="set allowed ciphers")
+    ap2.add_argument("--ssl-dbg", action="store_true", help="dump some tls info")
+    ap2.add_argument("--ssl-log", metavar="PATH", help="log master secrets")
     al = ap.parse_args()
     # fmt: on
 
@@ -168,6 +256,15 @@ def main():
     except:
         raise Exception("invalid value for -p")
 
+    if HAVE_SSL:
+        if al.ssl_ver:
+            configure_ssl_ver(al)
+
+        if al.ciphers:
+            configure_ssl_ciphers(al)
+    else:
+        print("\033[33m  ssl module does not exist; cannot enable https\033[0m\n")
+
     SvcHub(al).run()
 
 

copyparty/__version__.py

@@ -1,8 +1,8 @@
 # coding: utf-8
 
-VERSION = (0, 7, 4)
+VERSION = (0, 7, 6)
 CODENAME = "keeping track"
-BUILD_DT = (2021, 2, 4)
+BUILD_DT = (2021, 2, 12)
 
 S_VERSION = ".".join(map(str, VERSION))
 S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT)

copyparty/httpcli.py

@@ -134,6 +134,16 @@ class HttpCli(object):
             uparam["raw"] = True
             uparam["dots"] = True
 
+        if hasattr(self.s, "cipher"):
+            self.ssl_suf = "".join(
+                [
+                    " \033[3{}m{}".format(c, s)
+                    for c, s in zip([6, 3, 6], self.s.cipher())
+                ]
+            )
+        else:
+            self.ssl_suf = ""
+
         try:
             if self.mode in ["GET", "HEAD"]:
                 return self.handle_get() and self.keepalive
@@ -211,7 +221,7 @@ class HttpCli(object):
 
             logmsg += " [\033[36m" + rval + "\033[0m]"
 
-        self.log(logmsg)
+        self.log(logmsg + self.ssl_suf)
 
         # "embedded" resources
         if self.vpath.startswith(".cpr"):
@@ -245,7 +255,7 @@ class HttpCli(object):
         return self.tx_browser()
 
     def handle_options(self):
-        self.log("OPTIONS " + self.req)
+        self.log("OPTIONS " + self.req + self.ssl_suf)
         self.send_headers(
             None,
             204,
@@ -258,7 +268,7 @@ class HttpCli(object):
         return True
 
     def handle_put(self):
-        self.log("PUT " + self.req)
+        self.log("PUT " + self.req + self.ssl_suf)
 
         if self.headers.get("expect", "").lower() == "100-continue":
             try:
@@ -269,7 +279,7 @@ class HttpCli(object):
         return self.handle_stash()
 
     def handle_post(self):
-        self.log("POST " + self.req)
+        self.log("POST " + self.req + self.ssl_suf)
 
         if self.headers.get("expect", "").lower() == "100-continue":
             try:
@@ -927,8 +937,11 @@ class HttpCli(object):
             open_func = open
             # 512 kB is optimal for huge files, use 64k
             open_args = [fsenc(fs_path), "rb", 64 * 1024]
-            if hasattr(os, "sendfile"):
-                use_sendfile = not self.args.no_sendfile
+            use_sendfile = (
+                not self.ssl_suf
+                and not self.args.no_sendfile
+                and hasattr(os, "sendfile")
+            )
 
         #
         # send reply
@@ -1082,6 +1095,10 @@ class HttpCli(object):
         if not self.args.ed or "dots" not in self.uparam:
             vfs_ls = exclude_dotfiles(vfs_ls)
 
+        hidden = []
+        if fsroot.endswith(str(os.sep) + ".hist"):
+            hidden = ["up2k.db", "up2k.snap"]
+
         dirs = []
         files = []
         for fn in vfs_ls:
@@ -1093,6 +1110,8 @@ class HttpCli(object):
 
             if fn in vfs_virt:
                 fspath = vfs_virt[fn].realpath
+            elif fn in hidden:
+                continue
             else:
                 fspath = fsroot + "/" + fn
 

copyparty/httpconn.py

@@ -3,10 +3,15 @@ from __future__ import print_function, unicode_literals
 
 import os
 import sys
-import ssl
 import time
 import socket
 
+HAVE_SSL = True
+try:
+    import ssl
+except:
+    HAVE_SSL = False
+
 try:
     import jinja2
 except ImportError:
@@ -75,9 +80,8 @@ class HttpConn(object):
     def log(self, msg):
         self.log_func(self.log_src, msg)
 
-    def run(self):
+    def _detect_https(self):
         method = None
-        self.sr = None
         if self.cert_path:
             try:
                 method = self.s.recv(4, socket.MSG_PEEK)
@@ -102,16 +106,52 @@ class HttpConn(object):
                 self.s.send(b"HTTP/1.1 400 Bad Request\r\n\r\n" + err.encode("utf-8"))
                 return
 
-        if method not in [None, b"GET ", b"HEAD", b"POST", b"PUT ", b"OPTI"]:
+        return method not in [None, b"GET ", b"HEAD", b"POST", b"PUT ", b"OPTI"]
+
+    def run(self):
+        self.sr = None
+        if self.args.https_only:
+            is_https = True
+        elif self.args.http_only or not HAVE_SSL:
+            is_https = False
+        else:
+            is_https = self._detect_https()
+
+        if is_https:
             if self.sr:
                 self.log("\033[1;31mTODO: cannot do https in jython\033[0m")
                 return
 
             self.log_src = self.log_src.replace("[36m", "[35m")
             try:
-                self.s = ssl.wrap_socket(
-                    self.s, server_side=True, certfile=self.cert_path
-                )
+                ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
+                ctx.load_cert_chain(self.cert_path)
+                if self.args.ssl_ver:
+                    ctx.options &= ~self.args.ssl_flags_en
+                    ctx.options |= self.args.ssl_flags_de
+                    # print(repr(ctx.options))
+
+                if self.args.ssl_log:
+                    try:
+                        ctx.keylog_filename = self.args.ssl_log
+                    except:
+                        self.log("keylog failed; openssl or python too old")
+
+                if self.args.ciphers:
+                    ctx.set_ciphers(self.args.ciphers)
+
+                self.s = ctx.wrap_socket(self.s, server_side=True)
+                if self.args.ssl_dbg and hasattr(self.s, "shared_ciphers"):
+                    overlap = [y[::-1] for y in self.s.shared_ciphers()]
+                    lines = [str(x) for x in (["TLS cipher overlap:"] + overlap)]
+                    self.log("\n".join(lines))
+                    for k, v in [
+                        ["compression", self.s.compression()],
+                        ["ALPN proto", self.s.selected_alpn_protocol()],
+                        ["NPN proto", self.s.selected_npn_protocol()],
+                    ]:
+                        self.log("TLS {}: {}".format(k, v or "nah"))
+
             except Exception as ex:
                 em = str(ex)
 

copyparty/up2k.py

@@ -130,15 +130,19 @@ class Up2k(object):
             if db:
                 # can be symlink so don't `and d.startswith(top)``
                 excl = set([d for d in tops if d != top])
-                self._build_dir([db, 0], top, excl, top)
+                dbw = [db, 0, time.time()]
+                self._build_dir(dbw, top, excl, top)
                 self._drop_lost(db, top)
+                if dbw[1]:
+                    self.log("up2k", "commit {} new files".format(dbw[1]))
+
                 db.commit()
 
     def _build_dir(self, dbw, top, excl, cdir):
         try:
             inodes = [fsdec(x) for x in os.listdir(fsenc(cdir))]
         except Exception as ex:
-            self.log("up2k", "listdir: " + repr(ex))
+            self.log("up2k", "listdir: {} @ [{}]".format(repr(ex), cdir))
             return
 
         histdir = os.path.join(top, ".hist")
@@ -147,7 +151,7 @@ class Up2k(object):
             try:
                 inf = os.stat(fsenc(abspath))
             except Exception as ex:
-                self.log("up2k", "stat: " + repr(ex))
+                self.log("up2k", "stat: {} @ [{}]".format(repr(ex), abspath))
                 continue
 
             if stat.S_ISDIR(inf.st_mode):
@@ -182,15 +186,18 @@ class Up2k(object):
                 try:
                     hashes = self._hashlist_from_file(abspath)
                 except Exception as ex:
-                    self.log("up2k", "hash: " + repr(ex))
+                    self.log("up2k", "hash: {} @ [{}]".format(repr(ex), abspath))
                     continue
 
                 wark = self._wark_from_hashlist(inf.st_size, hashes)
                 self.db_add(dbw[0], wark, rp, inf.st_mtime, inf.st_size)
                 dbw[1] += 1
-                if dbw[1] > 1024:
+                td = time.time() - dbw[2]
+                if dbw[1] > 1024 or td > 60:
+                    self.log("up2k", "commit {} new files".format(dbw[1]))
                     dbw[0].commit()
                     dbw[1] = 0
+                    dbw[2] = time.time()
 
     def _drop_lost(self, db, top):
         rm = []
@@ -201,7 +208,7 @@ class Up2k(object):
                 if not os.path.exists(fsenc(abspath)):
                     rm.append(drp)
             except Exception as ex:
-                self.log("up2k", "stat-rm: " + repr(ex))
+                self.log("up2k", "stat-rm: {} @ [{}]".format(repr(ex), abspath))
 
         if not rm:
             return
@@ -512,8 +519,15 @@ class Up2k(object):
         fsz = os.path.getsize(path)
         csz = self._get_chunksize(fsz)
         ret = []
+        last_print = time.time()
         with open(path, "rb", 512 * 1024) as f:
             while fsz > 0:
+                now = time.time()
+                td = now - last_print
+                if td >= 0.3:
+                    last_print = now
+                    print(" {}    \n\033[A".format(fsz), end="")
+
                 hashobj = hashlib.sha512()
                 rem = min(csz, fsz)
                 fsz -= rem

copyparty/util.py

@@ -108,7 +108,7 @@ def ren_open(fname, *args, **kwargs):
         with open(fname, *args, **kwargs) as f:
             yield {"orz": [f, fname]}
             return
-    
+
     orig_name = fname
     bname = fname
     ext = ""
@@ -632,10 +632,10 @@ def sendfile_kern(lower, upper, f, s):
         except Exception as ex:
             # print("sendfile: " + repr(ex))
             n = 0
-        
+
         if n <= 0:
             return upper - ofs
-        
+
         ofs += n
         # print("sendfile: ok, sent {} now, {} total, {} remains".format(n, ofs - lower, upper - ofs))
 
@@ -718,6 +718,22 @@ def py_desc():
     )
 
 
+def align_tab(lines):
+    rows = []
+    ncols = 0
+    for ln in lines:
+        row = [x for x in ln.split(" ") if x]
+        ncols = max(ncols, len(row))
+        rows.append(row)
+
+    lens = [0] * ncols
+    for row in rows:
+        for n, col in enumerate(row):
+            lens[n] = max(lens[n], len(col))
+
+    return ["".join(x.ljust(y + 2) for x, y in zip(row, lens)) for row in rows]
+
+
 class Pebkac(Exception):
     def __init__(self, code, msg=None):
         super(Pebkac, self).__init__(msg or HTTPCODE[code])

copyparty/web/up2k.js

@@ -219,6 +219,10 @@ function up2k_init(have_crypto) {
             "hash": [],
             "handshake": [],
             "upload": []
+        },
+        "bytes": {
+            "hashed": 0,
+            "uploaded": 0
         }
     };
 
@@ -357,8 +361,11 @@ function up2k_init(have_crypto) {
     }
 
     function hashing_permitted() {
-        var lim = multitask ? 1 : 0;
-        return handshakes_permitted() && lim >=
+        if (multitask) {
+            var ahead = st.bytes.hashed - st.bytes.uploaded;
+            return ahead < 1024 * 1024 * 128;
+        }
+        return handshakes_permitted() && 0 ==
             st.todo.handshake.length +
             st.busy.handshake.length;
     }
@@ -512,6 +519,7 @@ function up2k_init(have_crypto) {
 
         var t = st.todo.hash.shift();
         st.busy.hash.push(t);
+        st.bytes.hashed += t.size;
         t.t1 = new Date().getTime();
 
         var nchunk = 0;
@@ -752,6 +760,7 @@ function up2k_init(have_crypto) {
             xhr.onload = function (xev) {
                 if (xhr.status == 200) {
                     prog(t.n, npart, col_uploaded);
+                    st.bytes.uploaded += cdr - car;
                     st.busy.upload.splice(st.busy.upload.indexOf(upt), 1);
                     t.postlist.splice(t.postlist.indexOf(npart), 1);
                     if (t.postlist.length == 0) {

setup.py

@@ -2,10 +2,8 @@
 # coding: utf-8
 from __future__ import print_function
 
-import io
 import os
 import sys
-from glob import glob
 from shutil import rmtree
 
 setuptools_available = True
@@ -49,7 +47,7 @@ with open(here + "/README.md", "rb") as f:
 about = {}
 if not VERSION:
     with open(os.path.join(here, NAME, "__version__.py"), "rb") as f:
-        exec(f.read().decode("utf-8").split("\n\n", 1)[1], about)
+        exec (f.read().decode("utf-8").split("\n\n", 1)[1], about)
 else:
     about["__version__"] = VERSION
 
@@ -116,6 +114,7 @@ args = {
         "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: Implementation :: CPython",
         "Programming Language :: Python :: Implementation :: PyPy",
         "Environment :: Console",