v1.4.2

README.md

@@ -1036,7 +1036,9 @@ quick outline of the up2k protocol, see [uploading](#uploading) for the web-clie
   * server writes chunks into place based on the hash
 * client does another handshake with the hashlist; server replies with OK or a list of chunks to reupload
 
-up2k has saved a few uploads from becoming corrupted in-transfer already; caught an android phone on wifi redhanded in wireshark with a bitflip, however bup with https would *probably* have noticed as well (thanks to tls also functioning as an integrity check)
+up2k has saved a few uploads from becoming corrupted in-transfer already;
+* caught an android phone on wifi redhanded in wireshark with a bitflip, however bup with https would *probably* have noticed as well (thanks to tls also functioning as an integrity check)
+* also stopped someone from uploading because their ram was bad
 
 regarding the frequent server log message during uploads;  
 `6.0M 106M/s 2.77G 102.9M/s n948 thank 4/0/3/1 10042/7198 00:01:09`

copyparty/__version__.py

@@ -1,8 +1,8 @@
 # coding: utf-8
 
-VERSION = (1, 4, 0)
+VERSION = (1, 4, 2)
 CODENAME = "mostly reliable"
-BUILD_DT = (2022, 9, 23)
+BUILD_DT = (2022, 9, 25)
 
 S_VERSION = ".".join(map(str, VERSION))
 S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT)

copyparty/httpcli.py

@@ -710,7 +710,13 @@ class HttpCli(object):
                 self.log("urlform: {} bytes, {}".format(post_sz, path))
             elif "print" in opt:
                 reader, _ = self.get_body_reader()
-                for buf in reader:
+                buf = b""
+                for rbuf in reader:
+                    buf += rbuf
+                    if not rbuf or len(buf) >= 32768:
+                        break
+
+                if buf:
                     orig = buf.decode("utf-8", "replace")
                     t = "urlform_raw {} @ {}\n  {}\n"
                     self.log(t.format(len(orig), self.vpath, orig))

copyparty/up2k.py

@@ -1958,13 +1958,16 @@ class Up2k(object):
                         "sprs": sprs,  # dontcare; finished anyways
                         "size": dsize,
                         "lmod": dtime,
-                        "life": cj.get("life"),
                         "addr": ip,
                         "at": at,
                         "hash": [],
                         "need": [],
                         "busy": {},
                     }
+                    for k in ["life"]:
+                        if k in cj:
+                            j[k] = cj[k]
+
                     score = (
                         (3 if st.st_dev == dev else 0)
                         + (2 if dp_dir == cj["prel"] else 0)
@@ -2071,11 +2074,14 @@ class Up2k(object):
                     "name",
                     "size",
                     "lmod",
-                    "life",
                     "poke",
                 ]:
                     job[k] = cj[k]
 
+                for k in ["life"]:
+                    if k in cj:
+                        job[k] = cj[k]
+
                 # one chunk may occur multiple times in a file;
                 # filter to unique values for the list of missing chunks
                 # (preserve order to reduce disk thrashing)
@@ -2969,8 +2975,18 @@ class Up2k(object):
             for x in reg.values()
             if x["need"] and now - x["poke"] > self.snap_discard_interval
         ]
-        if rm:
-            t = "dropping {} abandoned uploads in {}".format(len(rm), ptop)
+
+        lost = [
+            x
+            for x in reg.values()
+            if x["need"]
+            and not bos.path.exists(os.path.join(x["ptop"], x["prel"], x["name"]))
+        ]
+
+        if rm or lost:
+            t = "dropping {} abandoned, {} deleted uploads in {}"
+            t = t.format(len(rm), len(lost), ptop)
+            rm.extend(lost)
             vis = [self._vis_job_progress(x) for x in rm]
             self.log("\n".join([t] + vis))
             for job in rm:
@@ -2980,7 +2996,10 @@ class Up2k(object):
                     path = os.path.join(job["ptop"], job["prel"], job["name"])
                     if bos.path.getsize(path) == 0:
                         bos.unlink(path)
+                except:
+                    pass
 
+                try:
                     if len(job["hash"]) == len(job["need"]):
                         # PARTIAL is empty, delete that too
                         path = os.path.join(job["ptop"], job["prel"], job["tnam"])

copyparty/util.py

@@ -1022,7 +1022,7 @@ def ren_open(
                 ]
                 continue
 
-            if ex.errno not in [36, 63] and (not WINDOWS or ex.errno != 22):
+            if ex.errno not in [36, 63, 95] and (not WINDOWS or ex.errno != 22):
                 raise
 
         if not b64:

docs/changelog.md

@@ -1,3 +1,105 @@
+▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀  
+# 2022-0924-1245  `v1.4.1`  fix api compat
+
+* read-only demo server at https://a.ocv.me/pub/demo/
+* latest gzip edition of the sfx: [v1.0.14](https://github.com/9001/copyparty/releases/tag/v1.0.14#:~:text=release-specific%20notes)
+
+# bugfixes
+* [v1.4.0](https://github.com/9001/copyparty/releases/tag/v1.4.0) accidentally required all clients to use the new up2k.js to continue uploading; support the old js too
+
+
+
+▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀  
+# 2022-0923-2053  `v1.4.0`  mostly reliable
+
+* read-only demo server at https://a.ocv.me/pub/demo/
+* latest gzip edition of the sfx: [v1.0.14](https://github.com/9001/copyparty/releases/tag/v1.0.14#:~:text=release-specific%20notes)
+
+## new features
+* huge folders are lazily rendered for a massive speedup, #11
+  * also reduces the number of `?tree` requests; helps a tiny bit on server load
+* [selfdestruct timer](https://github.com/9001/copyparty#self-destruct) on uploaded files -- see link for howto and side-effects
+* ban clients trying to bruteforce passwords
+  * arg `--ban-pw`, default `9,60,1440`, bans for 1440min after 9 wrong passwords in 60min
+  * clients repeatedly trying the same password (due to a bug or whatever) are not counted
+  * does a `/64` range-ban for IPv6 offenders
+  * arg `--ban-404`, disabled by default, bans for excessive 404s / directory-scanning
+    * but that breaks up2k turbo-mode and probably some other eccentric usecases
+* waveform seekbar [(screenshot)](https://user-images.githubusercontent.com/241032/192042695-522b3ec7-6845-494a-abdb-d1c0d0e23801.png)
+* the up2k upload button can do folders recursively now
+  * but only a single folder can be selected at a time, making drag-drop the obvious choice still
+* gridview is now less jank, #12
+* togglebuttons for desktop-notifications and audio-jingle when upload completes
+* stop exposing uploader IPs when avoiding filename collisions
+  * IPs are now HMAC'ed with urandom stored at `~/.config/copyparty/iphash`
+* stop crashing chrome; generate PNGs rather than SVGs for filetype icons
+* terminate connections with SHUT_WR and flush with siocoutq
+  * makes buggy enterprise proxies behave less buggy
+  * do a read-spin on windows for almost the same effect
+* improved upload scheduling
+  * unfortunately removes the `0.0%, NaN:aN, N.aN MB/s` easteregg
+* arg `--magic` enables filetype detection on nameless uploads based on libmagic
+* mtp modifiers to let tagparsers keep their stdout/stderr instead of capturing
+  * `c0` disables all capturing, `c1` captures stdout only, `c2` only stderr, and `c3` (default) captures both
+* arg `--write-uplog` enables the old default of writing upload reports on POSTs
+  * kinda pointless and was causing issues in prisonparty
+* [upload modifiers](https://github.com/9001/copyparty#write) for terse replies and to randomize filenames
+* other optimizations
+  * 30% faster tag collection on directory listings
+  * 8x faster rendering of huge tagsets
+* new mtps [guestbook](https://github.com/9001/copyparty/blob/hovudstraum/bin/mtag/guestbook.py) and [guestbook-read](https://github.com/9001/copyparty/blob/hovudstraum/bin/mtag/guestbook-read.py), for example for comment-fields on uploads
+* arg `--stackmon` now takes dateformat filenames to produce multiple files
+* arg `--mtag-vv` to debug tagparser configs
+* arg `--version` shows copyparty version and exits
+* arg `--license` shows a list of embedded dependencies + their licenses
+* arg `--no-forget` and volflag `:c,noforget` keeps deleted files in the up2k db/index
+  * useful if you're shuffling uploads to s3/gdrive/etc and still want deduplication
+
+## bugfixes
+* upload deduplication using symlinks on windows
+* increase timeouts to run better on servers with extremely overloaded HDDs
+  * arg `--mtag-to` (default 60 sec, was 10) can be reduced for faster tag scanning
+* incorrect filekeys for files symlinked into another volume
+* playback could start mid-song if skipping back and forth between songs
+* use affinity mask to determine how many CPU cores are available
+* restore .bin-suffix for nameless PUT/POSTs (disappeared in v1.0.11)
+* fix glitch in uploader-UI when upload queue is bigger than 1 TiB
+* avoid a firefox race-condition accessing the navigation history
+* sfx tmpdir keepalive when flipflopping between unix users
+* reject anon ftp if anon has no read/write
+* improved autocorrect for poor ffmpeg builds
+* patch popen on older pythons so collecting tags on windows is always possible
+* misc ui/ux fixes
+  * filesearch layout in read-only folders
+  * more comfy fadein/fadeout on play/pause
+  * total-ETA going crazy when an overloaded server drops requests
+  * stop trying to play into the next folder while in search results
+  * improve warnings/errors in the uploader ui
+    * some errors which should have been warnings are now warnings
+    * autohide warnings/errors when they are remedied
+  * delay starting the audiocontext until necessary
+    * reduces cpu-load by 0.2% and fixes chrome claiming the tab is playing audio
+
+# copyparty.exe
+
+now introducing [copyparty.exe](https://github.com/9001/copyparty/releases/download/v1.4.0/copyparty.exe)!   only suitable for the rainiest of days ™
+
+[first thing you'll see](https://user-images.githubusercontent.com/241032/192070274-bfe0bfef-2293-40fc-8852-fcf4f7a90043.png) when you run it is a warning to **«please use the [python-sfx](https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.py) instead»**,
+* `copyparty.exe` was compiled using 32bit python3.7 to support windows7, meaning it won't receive any security patches
+* `copyparty-sfx.py` uses your system libraries instead so it'll stay safe for much longer while also having better performance
+
+so the exe might be super useful in a pinch on a secluded LAN but otherwise *Absolutely Not Recommended*
+
+you can download [ffmpeg](https://ocv.me/stuff/bin/ffmpeg.exe) and [ffprobe](https://ocv.me/stuff/bin/ffprobe.exe) into the same folder if you want multimedia-info, audio-transcoding or thumbnails/spectrograms/waveforms -- those binaries were [built](https://github.com/9001/copyparty/tree/hovudstraum/scripts/pyinstaller#ffmpeg) with just enough features to cover what copyparty wants, but much like copyparty.exe itself (so due to security reasons) it is strongly recommended to instead grab a [recent official build](https://github.com/BtbN/FFmpeg-Builds/releases/download/latest/ffmpeg-master-latest-win64-gpl.zip) every once in a while
+
+## and finally some good news
+
+* the chrome memory leak will be [fixed in v107](https://bugs.chromium.org/p/chromium/issues/detail?id=1354816)
+* and firefox may fix the crash in [v106 or so](https://bugzilla.mozilla.org/show_bug.cgi?id=1790500)
+* and the release title / this season's codename stems from a cpp instance recently being slammed with terabytes of uploads running on a struggling server mostly without breaking a sweat 👍
+
+
+
 ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀  
 # 2022-0818-1724  `v1.3.16`  gc kiting
 

docs/notes.sh

@@ -143,6 +143,31 @@ sqlite3 -readonly up2k.db.key-full 'select w, v from mt where k = "key" order by
 sqlite3 -readonly up2k.db.key-full 'select w, v from mt where k = "key" order by w' > k1; sqlite3 -readonly up2k.db 'select mt.w, mt.v, up.rd, up.fn from mt inner join up on mt.w = substr(up.w,1,16) where mt.k = "key" order by up.rd, up.fn' > k2; ok=0; ng=0; while IFS='|' read w k2 path; do k1="$(grep -E "^$w" k1 | sed -r 's/.*\|//')"; [ "$k1" = "$k2" ] && ok=$((ok+1)) || { ng=$((ng+1)); printf '%3s %3s  %s\n' "$k1" "$k2" "$path"; }; done < <(cat k2); echo "match $ok   diff $ng"
 
 
+##
+## tracking bitflips
+
+l=log.tmux-1662316902  # your logfile (tmux-capture or decompressed -lo)
+
+# grab handshakes to a smaller logfile
+tr -d '\r' <$l | awk '/^.\[36m....-..-...\[0m.?$/{d=substr($0,6,10)} !d{next} /"purl": "/{t=substr($1,6);sub(/[^ ]+ /,"");sub(/ .\[34m[0-9]+ /," ");printf("%s %s %s %s\n",d,t,ip,$0)}' | while read d t ip f; do u=$(date +%s --date="${d}T${t}Z"); printf '%s\n' "$u $ip $f"; done > handshakes
+
+# quick list of affected files
+grep 'your chunk got corrupted somehow' -A1 $l | tr -d '\r' | grep -E '^[a-zA-Z0-9_-]{44}$' | sort | uniq | while IFS= read -r x; do grep -F "$x" handshakes | head -c 200; echo; done | sed -r 's/.*"name": "//' | sort | uniq -cw20
+
+# find all cases of corrupt chunks and print their respective handshakes (if any),
+# timestamps are when the corrupted chunk was received (and also the order they are displayed),
+# first checksum is the expected value from the handshake, second is what got uploaded
+awk <$l '/^.\[36m....-..-...\[0m.?$/{d=substr($0,6,10)} /your chunk got corrupted somehow/{n=2;t=substr($1,6);next} !n{next} {n--;sub(/\r$/,"")} n{a=$0;next} {sub(/.\[0m,.*/,"");printf "%s %s %s %s\n",d,t,a,$0}' |
+while read d t h1 h2; do printf '%s %s\n' $d $t; (
+printf '  %s [%s]\n' $h1 "$(grep -F $h1 <handshakes | head -n 1)"
+printf '  %s [%s]\n' $h2 "$(grep -F $h2 <handshakes | head -n 1)"
+) | sed 's/, "sprs":.*//'; done | less -R
+
+# notes; TODO clean up and put in the readme maybe --
+# quickest way to drop the bad files (if a client generated bad hashes for the initial handshake) is shutting down copyparty and moving aside the unfinished file (both the .PARTIAL and the empty placeholder)
+# BUT the clients will immediately re-handshake the upload with the same bitflipped hashes, so the uploaders have to refresh their browsers before you do that,
+# so maybe just ask them to refresh and do nothing for 6 hours so the timeout kicks in, which deletes the placeholders/name-reservations and you can then manually delete the .PARTIALs at some point later
+
 ##
 ## media
 

scripts/pyinstaller/README.md

@@ -5,3 +5,8 @@ requires a win7 vm which has never been connected to the internet and a host-onl
 first-time setup steps in notes.txt
 
 run build.sh in the vm to fetch src + compile + push a new exe to the linux host for manual publishing
+
+
+## ffmpeg
+
+built with [ffmpeg-windows-build-helpers](https://github.com/rdp/ffmpeg-windows-build-helpers) and [this patch](./ffmpeg.patch) using [these steps](./ffmpeg.txt)