v1.4.6

.gitignore

@@ -22,6 +22,7 @@ copyparty.egg-info/
 *.bak
 
 # derived
+copyparty/res/COPYING.txt
 copyparty/web/deps/
 srv/
 

README.md

@@ -1399,7 +1399,7 @@ first grab the web-dependencies from a previous sfx (assuming you don't need to
 ```sh
 rm -rf copyparty/web/deps
 curl -L https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.py >x.py
-python3 x.py -h
+python3 x.py --version
 rm x.py
 mv /tmp/pe-copyparty/copyparty/web/deps/ copyparty/web/deps/
 ```

copyparty/__version__.py

@@ -1,8 +1,8 @@
 # coding: utf-8
 
-VERSION = (1, 4, 4)
+VERSION = (1, 4, 6)
 CODENAME = "mostly reliable"
-BUILD_DT = (2022, 10, 9)
+BUILD_DT = (2022, 10, 13)
 
 S_VERSION = ".".join(map(str, VERSION))
 S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT)

copyparty/tcpsrv.py

@@ -62,8 +62,8 @@ class TcpSrv(object):
                 for x in nonlocals:
                     eps[x] = "external"
 
-        qr1 = {}
-        qr2 = {}
+        qr1: dict[str, list[int]] = {}
+        qr2: dict[str, list[int]] = {}
         msgs = []
         title_tab: dict[str, dict[str, int]] = {}
         title_vars = [x[1:] for x in self.args.wintitle.split(" ") if x.startswith("$")]

copyparty/th_srv.py

@@ -621,7 +621,7 @@ class ThumbSrv(object):
 
     def _clean(self, cat: str, thumbpath: str) -> int:
         # self.log("cln {}".format(thumbpath))
-        exts = ["jpg", "webp"] if cat == "th" else ["opus", "caf"]
+        exts = ["jpg", "webp", "png"] if cat == "th" else ["opus", "caf"]
         maxage = getattr(self.args, cat + "_maxage")
         now = time.time()
         prev_b64 = None

copyparty/web/browser.js

@@ -166,7 +166,7 @@ var Ls = {
 		"mt_oscv": "show album cover in osd\">art",
 		"mt_mloop": "loop the open folder\">🔁 loop",
 		"mt_mnext": "load the next folder and continue\">📂 next",
-		"mt_cflac": "convert flac to opus\">flac",
+		"mt_cflac": "convert flac / wav to opus\">flac",
 		"mt_caac": "convert aac / m4a to opus\">aac",
 		"mt_coth": "convert all others (not mp3) to opus\">oth",
 		"mt_tint": "background level (0-100) on the seekbar$Nto make buffering less distracting",
@@ -534,14 +534,14 @@ var Ls = {
 
 		"mt_preload": "hent ned litt av neste sang i forkant,$Nslik at pausen i overgangen blir mindre\">forles",
 		"mt_fullpre": "hent ned hele neste sang, ikke bare litt:$N✅ skru på hvis nettet ditt er <b>ustabilt</b>,$N❌ skru av hvis nettet ditt er <b>tregt</b>\">full",
-		"mt_waves": "waveform seekbar:$Nvis volumkart i avspillingsindikatoren\">~s",
+		"mt_waves": "waveform seekbar:$Nvis volumindikator i avspillingsfeltet\">~s",
 		"mt_npclip": "vis knapper for å kopiere info om sangen du hører på\">/np",
 		"mt_octl": "integrering med operativsystemet (fjernkontroll, info-skjerm)\">os-ctl",
 		"mt_oseek": "tillat spoling med fjernkontroll\">spoling",
 		"mt_oscv": "vis album-cover på infoskjermen\">bilde",
 		"mt_mloop": "repeter hele mappen\">🔁 gjenta",
 		"mt_mnext": "hopp til neste mappe og fortsett\">📂 neste",
-		"mt_cflac": "konverter flac-filer til opus\">flac",
+		"mt_cflac": "konverter flac / wav-filer til opus\">flac",
 		"mt_caac": "konverter aac / m4a-filer til to opus\">aac",
 		"mt_coth": "konverter alt annet (men ikke mp3) til opus\">andre",
 		"mt_tint": "nivå av bakgrunnsfarge på søkestripa (0-100),$Ngjør oppdateringer mindre distraherende",
@@ -1197,7 +1197,7 @@ var mpl = (function () {
 		var c = true;
 		if (!have_acode)
 			c = false;
-		else if (/\.flac$/i.exec(url))
+		else if (/\.(wav|flac)$/i.exec(url))
 			c = r.ac_flac;
 		else if (/\.(aac|m4a)$/i.exec(url))
 			c = r.ac_aac;

copyparty/web/splash.js

@@ -12,7 +12,7 @@ var Ls = {
 		"cc1": "klient-konfigurasjon",
 		"h1": "skru av k304",
 		"i1": "skru på k304",
-		"j1": "k304 bryter tilkoplingen for hver HTTP 304. Dette hjelper visse mellomtjenere som kan sette seg fast / plutselig slutter å laste sider, men det reduserer også ytelsen betydelig",
+		"j1": "k304 bryter tilkoplingen for hver HTTP 304. Dette hjelper mot visse mellomtjenere som kan sette seg fast / plutselig slutter å laste sider, men det reduserer også ytelsen betydelig",
 		"k1": "nullstill innstillinger",
 		"l1": "logg inn:",
 		"m1": "velkommen tilbake,",

docs/changelog.md

@@ -1,3 +1,51 @@
+▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀  
+# 2022-1009-0919  `v1.4.5`  qr-code
+
+* read-only demo server at https://a.ocv.me/pub/demo/
+* latest gzip edition of the sfx: [v1.0.14](https://github.com/9001/copyparty/releases/tag/v1.0.14#:~:text=release-specific%20notes)
+
+## new features
+* display a server [qr-code](https://github.com/9001/copyparty#qr-code) [(screenshot)](https://user-images.githubusercontent.com/241032/194728533-6f00849b-c6ac-43c6-9359-83e454d11e00.png) on startup
+  * primarily for running copyparty on a phone and accessing it from another
+  * optionally specify a path or password with `--qrl lootbox/?pw=hunter2`
+  * uses the server's exteral ip (default route) unless `--qri` specifies a domain / ip-prefix
+  * classic cp437 `▄` `▀` for space efficiency; some misbehaving terminals / fonts need `--qrz 2`
+* new permission `G` returns the filekey of uploaded files for users without read-access
+  * when combined with permission `w` and volflag `fk`, uploaded files will not be accessible unless the filekey is provided in the url, and `G` provides the filekey to the uploader unlike `g`
+* filekeys are added to the unpost listing
+
+## bugfixes
+* renaming / moving folders is now **at least 120x faster**
+  * and that's on nvme drives, so probably like 2000x on HDDs
+* uploads to volumes with lifetimes could get instapurged depending on browser and browser settings
+* ux fixes
+  * FINALLY fixed messageboxes appearing offscreen on phones (and some other layout issues)
+  * stop asking about folder-uploads on phones because they dont support it
+  * on android-firefox, default to truncating huge folders with the load-more button due to ff onscroll being buggy
+  * audioplayer looking funky if ffmpeg unavailable
+* waveform-seekbar cache expiration (the thumbcleaner complaining about png files)
+* ie11 panic when opening a folder which contains a file named `up2k`
+  * turns out `<a name=foo>` becomes `window.foo` unless that's already declared somewhere in js -- luckily other browsers "only" do that with IDs
+
+
+
+▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀  
+# 2022-0926-2037  `v1.4.3`  signal in the noise
+
+* read-only demo server at https://a.ocv.me/pub/demo/
+* latest gzip edition of the sfx: [v1.0.14](https://github.com/9001/copyparty/releases/tag/v1.0.14#:~:text=release-specific%20notes)
+
+## new features
+* `--bak-flips` saves a copy of corrupted / bitflipped up2k uploads
+  * comparing against a good copy can help pinpoint the culprit
+  * also see [tracking bitflips](https://github.com/9001/copyparty/blob/hovudstraum/docs/notes.sh#:~:text=tracking%20bitflips)
+
+## bugfixes
+* some edgecases where deleted files didn't get dropped from the db
+  * can reduce performance over time, hitting the filesystem more than necessary
+
+
+
 ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀  
 # 2022-0925-1236  `v1.4.2`  fuhgeddaboudit
 

docs/notes.sh

@@ -67,6 +67,7 @@ mkdir -p "${dirs[@]}"
 for dir in "${dirs[@]}"; do for fn in ふが "$(printf \\xed\\x93)" 'qw,er;ty%20as df?gh+jkl%zxc&vbn <qwe>"rty'"'"'uio&asd&nbsp;fgh'; do echo "$dir" > "$dir/$fn.html"; done; done
 # qw er+ty%20ui%%20op<as>df&gh&amp;jk#zx'cv"bn`m=qw*er^ty?ui@op,as.df-gh_jk
 
+
 ##
 ## upload mojibake
 
@@ -143,6 +144,17 @@ sqlite3 -readonly up2k.db.key-full 'select w, v from mt where k = "key" order by
 sqlite3 -readonly up2k.db.key-full 'select w, v from mt where k = "key" order by w' > k1; sqlite3 -readonly up2k.db 'select mt.w, mt.v, up.rd, up.fn from mt inner join up on mt.w = substr(up.w,1,16) where mt.k = "key" order by up.rd, up.fn' > k2; ok=0; ng=0; while IFS='|' read w k2 path; do k1="$(grep -E "^$w" k1 | sed -r 's/.*\|//')"; [ "$k1" = "$k2" ] && ok=$((ok+1)) || { ng=$((ng+1)); printf '%3s %3s  %s\n' "$k1" "$k2" "$path"; }; done < <(cat k2); echo "match $ok   diff $ng"
 
 
+##
+## scanning for exceptions
+
+cd /dev/shm
+journalctl -aS '720 hour ago' -t python3 -o with-unit --utc | cut -d\  -f2,6- > cpp.log
+tac cpp.log | awk '/RuntimeError: generator ignored GeneratorExit/{n=1} n{n--;if(n==0)print} 1' | grep 'generator ignored GeneratorExit' -C7 | head -n 100
+awk '/Exception ignored in: <generator object StreamZip.gen/{s=1;next} /could not create thumbnail/{s=3;next} s{s--;next} 1' <cpp.log | less -R
+less-search:
+  >: |Exception|Traceback
+
+
 ##
 ## tracking bitflips
 
@@ -168,6 +180,7 @@ printf '  %s [%s]\n' $h2 "$(grep -F $h2 <handshakes | head -n 1)"
 # BUT the clients will immediately re-handshake the upload with the same bitflipped hashes, so the uploaders have to refresh their browsers before you do that,
 # so maybe just ask them to refresh and do nothing for 6 hours so the timeout kicks in, which deletes the placeholders/name-reservations and you can then manually delete the .PARTIALs at some point later
 
+
 ##
 ## media
 

scripts/copyparty-repack.sh

@@ -119,7 +119,7 @@ chmod 755 \
 
 # extract the sfx
 ( cd copyparty-extras/sfx-full/
-./copyparty-sfx.py -h
+./copyparty-sfx.py --version
 )
 
 

scripts/genlic.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+set -e
+
+outfile="$(realpath "$1")"
+
+[ -e genlic.sh ] || cd scripts
+[ -e genlic.sh ]
+
+f=../build/mit.txt
+[ -e $f ] ||
+	curl https://opensource.org/licenses/MIT |
+	awk '/div>/{o=0}o>1;o{o++}/;COPYRIGHT HOLDER/{o=1}' |
+	awk '{gsub(/<[^>]+>/,"")};1' >$f
+
+f=../build/isc.txt
+[ -e $f ] ||
+	curl https://opensource.org/licenses/ISC |
+	awk '/div>/{o=0}o>2;o{o++}/;OWNER/{o=1}' |
+	awk '{gsub(/<[^>]+>/,"")};/./{b=0}!/./{b++}b>1{next}1' >$f
+
+f=../build/3bsd.txt
+[ -e $f ] ||
+	curl https://opensource.org/licenses/BSD-3-Clause |
+	awk '/div>/{o=0}o>1;o{o++}/HOLDER/{o=1}' |
+	awk '{gsub(/<[^>]+>/,"")};1' >$f
+
+f=../build/ofl.txt
+[ -e $f ] ||
+	curl https://opensource.org/licenses/OFL-1.1 |
+	awk '/PREAMBLE/{o=1}/sil\.org/{o=0}!o{next}/./{printf "%s ",$0;next}{print"\n"}' |
+	awk '{gsub(/<[^>]+>/,"");gsub(/^\s+/,"");gsub(/&amp;/,"\\&")}/./{b=0}!/./{b++}b>1{next}1' >$f
+
+(sed -r 's/^L: /License: /;s/^C: /Copyright (c) /' <../docs/lics.txt
+printf '\n\n--- MIT License ---\n\n'; cat ../build/mit.txt
+printf '\n\n--- ISC License ---\n\n'; cat ../build/isc.txt
+printf '\n\n--- BSD 3-Clause License ---\n\n'; cat ../build/3bsd.txt
+printf '\n\n--- SIL Open Font License v1.1 ---\n\n'; cat ../build/ofl.txt
+) |
+while IFS= read -r x; do
+	[ "${x:0:4}" = "--- " ] || {
+		printf '%s\n' "$x"
+		continue
+	}
+	n=${#x}
+	p=$(( (80-n)/2 ))
+	printf "%${p}s\033[07m%s\033[0m\n" "" "$x"
+done > "$outfile"

scripts/make-sfx.sh

@@ -87,6 +87,7 @@ while [ ! -z "$1" ]; do
 		re)     repack=1 ; ;;
 		ox)     use_ox=1 ; ;;
 		gz)     use_gz=1 ; ;;
+		gzz)    shift;use_gzz=$1;use_gz=1; ;;
 		no-fnt) no_fnt=1 ; ;;
 		no-hl)  no_hl=1  ; ;;
 		no-dd)  no_dd=1  ; ;;
@@ -225,46 +226,8 @@ tmpdir="$(
 	# remove type hints before build instead
 	(cd copyparty; "$pybin" ../../scripts/strip_hints/a.py; rm uh)
 
-f=../build/mit.txt
-[ -e $f ] ||
-	curl https://opensource.org/licenses/MIT |
-	awk '/div>/{o=0}o>1;o{o++}/;COPYRIGHT HOLDER/{o=1}' |
-	awk '{gsub(/<[^>]+>/,"")};1' >$f
-
-f=../build/isc.txt
-[ -e $f ] ||
-	curl https://opensource.org/licenses/ISC |
-	awk '/div>/{o=0}o>2;o{o++}/;OWNER/{o=1}' |
-	awk '{gsub(/<[^>]+>/,"")};/./{b=0}!/./{b++}b>1{next}1' >$f
-
-f=../build/3bsd.txt
-[ -e $f ] ||
-	curl https://opensource.org/licenses/BSD-3-Clause |
-	awk '/div>/{o=0}o>1;o{o++}/HOLDER/{o=1}' |
-	awk '{gsub(/<[^>]+>/,"")};1' >$f
-
-f=../build/ofl.txt
-[ -e $f ] ||
-	curl https://opensource.org/licenses/OFL-1.1 |
-	awk '/PREAMBLE/{o=1}/sil\.org/{o=0}!o{next}/./{printf "%s ",$0;next}{print"\n"}' |
-	awk '{gsub(/<[^>]+>/,"");gsub(/^\s+/,"");gsub(/&amp;/,"\\&")}/./{b=0}!/./{b++}b>1{next}1' >$f
-
-(sed -r 's/^L: /License: /;s/^C: /Copyright (c) /' <../docs/lics.txt
-printf '\n\n--- MIT License ---\n\n'; cat ../build/mit.txt
-printf '\n\n--- ISC License ---\n\n'; cat ../build/isc.txt
-printf '\n\n--- BSD 3-Clause License ---\n\n'; cat ../build/3bsd.txt
-printf '\n\n--- SIL Open Font License v1.1 ---\n\n'; cat ../build/ofl.txt
-) |
-while IFS= read -r x; do
-	[ "${x:0:4}" = "--- " ] || {
-		printf '%s\n' "$x"
-		continue
-	}
-	n=${#x}
-	p=$(( (80-n)/2 ))
-	printf "%${p}s\033[07m%s\033[0m\n" "" "$x"
-done > copyparty/res/COPYING.txt
-
+	licfile=$(realpath copyparty/res/COPYING.txt)
+	(cd ../scripts; ./genlic.sh "$licfile")
 }
 
 ver=
@@ -311,6 +274,7 @@ sfx_out=../dist/copyparty-$CSN
 echo cleanup
 find -name '*.pyc' -delete
 find -name __pycache__ -delete
+find -name py.typed -delete
 
 # especially prevent osx from leaking your lan ip (wtf apple)
 find -type f \( -name .DS_Store -or -name ._.DS_Store \) -delete
@@ -483,7 +447,7 @@ nf=$(ls -1 "$zdir"/arc.* | wc -l)
 	pyoxidizer build --release --target-triple $tgt
 	mv $bdir/copyparty.exe dist/
 	cp -pv "$(for d in '/c/Program Files (x86)/Microsoft Visual Studio/'*'/BuildTools/VC/Redist/MSVC'; do
-		find "$d" -name vcruntime140.dll; done | sort | grep -vE '/x64/|/onecore/'ƒ | head -n 1)" dist/
+		find "$d" -name vcruntime140.dll; done | sort | grep -vE '/x64/|/onecore/' | head -n 1)" dist/
 	dist/copyparty.exe --version
 	cp -pv dist/copyparty{,.orig}.exe
 	[ $ultra ] && a="--best --lzma" || a=-1
@@ -510,13 +474,18 @@ done
 echo creating tar
 tar -cf tar "${targs[@]}" --numeric-owner -T list
 
-pc=bzip2
-pe=bz2
-[ $use_gz ] && pc=gzip && pe=gz
+pc="bzip2 -"; pe=bz2
+[ $use_gz ] && pc="gzip -" && pe=gz
+[ $use_gzz ] && pc="pigz -11 -I$use_gzz" && pe=gz
 
 echo compressing tar
-# detect best level; bzip2 -7 is usually better than -9
-for n in {2..9}; do cp tar t.$n; nice $pc  -$n t.$n & done; wait; mv -v $(ls -1S t.*.$pe | tail -n 1) tar.bz2
+for n in {2..9}; do cp tar t.$n; nice $pc$n t.$n & done; wait
+minf=$(for f in t.*.$pe; do
+	s1=$(wc -c <$f)
+	s2=$(tr -d '\r\n\0' <$f | wc -c)
+	echo "$(( s2+(s1-s2)*3 )) $f"
+done | sort -n | awk '{print$2;exit}')
+mv -v $minf tar.bz2
 rm t.* || true
 exts=()
 

scripts/rls.sh

@@ -31,10 +31,9 @@ rm -f ../dist/copyparty-sfx*
 shift
 ./make-sfx.sh "$@"
 f=../dist/copyparty-sfx
-[ -e $f.py ] || 
-    f=../dist/copyparty-sfx-gz
+[ -e $f.py ] && s= || s=-gz
 
-$f.py -h >/dev/null
+$f$s.py --version >/dev/null
 
 [ $parallel -gt 1 ] && {
     printf '\033[%s' s 2r H "0;1;37;44mbruteforcing sfx size -- press enter to terminate" K u "7m $* " K $'27m\n'
@@ -44,9 +43,9 @@ $f.py -h >/dev/null
     for ((a=0; a<$parallel; a++)); do
         while [ -e .sfx-run ]; do
             CSN=sfx$a ./make-sfx.sh re "$@"
-            sz=$(wc -c <$f$a.py | awk '{print$1}')
+            sz=$(wc -c <$f$a$s.py | awk '{print$1}')
             [ $sz -ge $min ] && continue
-            mv $f$a.py $f.py.$sz
+            mv $f$a$s.py $f$s.py.$sz
             min=$sz
         done &
     done
@@ -55,7 +54,7 @@ $f.py -h >/dev/null
 }
 
 while true; do
-    mv $f.py $f.$(wc -c <$f.py | awk '{print$1}').py
+    mv $f$s.py $f$s.$(wc -c <$f$s.py | awk '{print$1}').py
     ./make-sfx.sh re "$@"
 done
 

scripts/sfx.py

@@ -142,6 +142,7 @@ def testchk(cdata):
 
 def encode(data, size, cksum, ver, ts):
     """creates a new sfx; `data` should yield bufs to attach"""
+    nb = 0
     nin = 0
     nout = 0
     skip = False
@@ -151,6 +152,7 @@ def encode(data, size, cksum, ver, ts):
         for ln in src.split("\n"):
             if ln.endswith("# skip 0"):
                 skip = False
+                nb = 9
                 continue
 
             if ln.endswith("# skip 1") or skip:
@@ -160,6 +162,13 @@ def encode(data, size, cksum, ver, ts):
             if ln.strip().startswith("# fmt: "):
                 continue
 
+            if ln:
+                nb = 0
+            else:
+                nb += 1
+                if nb > 2:
+                    continue
+
             unpk += ln + "\n"
 
         for k, v in [
@@ -177,7 +186,7 @@ def encode(data, size, cksum, ver, ts):
             unpk = unpk.replace("\t    ", "\t\t")
 
     with open("sfx.out", "wb") as f:
-        f.write(unpk.encode("utf-8") + b"\n\n# eof\n# ")
+        f.write(unpk.encode("utf-8").rstrip(b"\n") + b"\n\n\n# eof\n# ")
         for buf in data:
             ebuf = buf.replace(b"\n", b"\n#n").replace(b"\r", b"\n#r")
             f.write(ebuf)
@@ -260,6 +269,12 @@ def unpack():
         raise Exception(t.format(CKSUM, SIZE, ck, sz))
 
     with tarfile.open(tar, "r:bz2") as tf:
+        # this is safe against traversal
+        # skip 1
+        # since it will never process user-provided data;
+        # the only possible input is a single tar.bz2
+        # which gets hardcoded into this script at build stage
+        # skip 0
         tf.extractall(mine)
 
     os.remove(tar)

setup.py

@@ -4,6 +4,7 @@ from __future__ import print_function
 
 import os
 import sys
+import subprocess as sp
 from shutil import rmtree
 from setuptools import setup, Command, find_packages
 
@@ -28,6 +29,11 @@ with open(here + "/README.md", "rb") as f:
     txt = f.read().decode("utf-8")
     long_description = txt
 
+try:
+    cmd = "bash scripts/genlic.sh copyparty/res/COPYING.txt"
+    sp.Popen(cmd.split()).wait()
+except:
+    pass
 
 about = {}
 if not VERSION:
@@ -100,6 +106,7 @@ args = {
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
         "Programming Language :: Python :: Implementation :: CPython",
         "Programming Language :: Python :: Implementation :: Jython",
         "Programming Language :: Python :: Implementation :: PyPy",