v1.3.16

.gitignore

@@ -11,7 +11,7 @@ copyparty.egg-info/
 /build/
 /dist/
 /py2/
-/sfx/
+/sfx*
 /unt/
 /log/
 

bin/mtag/mousepad.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+import subprocess as sp
+
+
+"""
+mtp test -- opens a texteditor
+
+usage:
+  -vsrv/v1:v1:r:c,mte=+x1:c,mtp=x1=ad,p,bin/mtag/mousepad.py
+
+explained:
+  c,mte: list of tags to index in this volume
+  c,mtp: add new tag provider
+     x1: dummy tag to provide
+     ad: dontcare if audio or not
+      p: priority 1 (run after initial tag-scan with ffprobe or mutagen)
+"""
+
+
+def main():
+    env = os.environ.copy()
+    env["DISPLAY"] = ":0.0"
+
+    if False:
+        # open the uploaded file
+        fp = sys.argv[-1]
+    else:
+        # display stdin contents (`oth_tags`)
+        fp = "/dev/stdin"
+
+    p = sp.Popen(["/usr/bin/mousepad", fp])
+    p.communicate()
+
+
+main()

contrib/plugins/up2k-hook-ytid.js

@@ -51,6 +51,8 @@ async function a_up2k_namefilter(good_files, nil_files, bad_files, hooks) {
             cname = name,  // will clobber
             sz = fobj.size,
             ids = [],
+            fn_ids = [],
+            md_ids = [],
             id_ok = false,
             m;
 
@@ -71,7 +73,7 @@ async function a_up2k_namefilter(good_files, nil_files, bad_files, hooks) {
 
             cname = cname.replace(m[1], '');
             yt_ids.add(m[1]);
-            ids.push(m[1]);
+            fn_ids.unshift(m[1]);
         }
 
         // look for IDs in video metadata,
@@ -110,10 +112,13 @@ async function a_up2k_namefilter(good_files, nil_files, bad_files, hooks) {
 
                         console.log(`found ${m} @${bofs}, ${name} `);
                         yt_ids.add(m);
-                        if (!has(ids, m)) {
-                            ids.push(m);
+                        if (!has(fn_ids, m) && !has(md_ids, m)) {
+                            md_ids.push(m);
                             md_only.push(`${m} ${name}`);
                         }
+                        else
+                            // id appears several times; make it preferred
+                            md_ids.unshift(m);
 
                         // bail after next iteration
                         chunk = nchunks - 1;
@@ -130,6 +135,13 @@ async function a_up2k_namefilter(good_files, nil_files, bad_files, hooks) {
                 }
             }
         }
+
+        for (var yi of md_ids)
+            ids.push(yi);
+
+        for (var yi of fn_ids)
+            if (!has(ids, yi))
+                ids.push(yi);
     }
 
     if (md_only.length)
@@ -164,6 +176,7 @@ async function a_up2k_namefilter(good_files, nil_files, bad_files, hooks) {
 
     function process_id_list(txt) {
         var wanted_ids = new Set(txt.trim().split('\n')),
+            name_id = {},
             wanted_names = new Set(),  // basenames with a wanted ID
             wanted_files = new Set();  // filedrops
 
@@ -174,8 +187,11 @@ async function a_up2k_namefilter(good_files, nil_files, bad_files, hooks) {
                     wanted_files.add(good_files[a]);
 
                     var m = /(.*)\.(mp4|webm|mkv|flv|opus|ogg|mp3|m4a|aac)$/i.exec(name);
-                    if (m)
+                    if (!m)
+                        continue;
+
                     wanted_names.add(m[1]);
+                    name_id[m[1]] = file_ids[a][b];
 
                     break;
                 }
@@ -189,6 +205,9 @@ async function a_up2k_namefilter(good_files, nil_files, bad_files, hooks) {
                 name = name.replace(/\.[^\.]+$/, '');
                 if (wanted_names.has(name)) {
                     wanted_files.add(good_files[a]);
+
+                    var subdir = `${name_id[name]}-${Date.now()}-${a}`;
+                    good_files[a][1] = subdir + '/' + good_files[a][1].split(/\//g).pop();
                     break;
                 }
             }

copyparty/__main__.py

@@ -335,7 +335,7 @@ def run_argparse(argv: list[str], formatter: Any, retry: bool) -> argparse.Names
     except:
         fk_salt = "hunter2"
 
-    cores = os.cpu_count() if hasattr(os, "cpu_count") else 4
+    cores = (os.cpu_count() if hasattr(os, "cpu_count") else 0) or 4
     hcores = min(cores, 3)  # 4% faster than 4+ on py3.9 @ r5-4500U
 
     sects = [

copyparty/__version__.py

@@ -1,8 +1,8 @@
 # coding: utf-8
 
-VERSION = (1, 3, 13)
+VERSION = (1, 3, 16)
 CODENAME = "god dag"
-BUILD_DT = (2022, 8, 15)
+BUILD_DT = (2022, 8, 18)
 
 S_VERSION = ".".join(map(str, VERSION))
 S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT)

copyparty/mtag.py

@@ -313,7 +313,6 @@ class MTag(object):
                 "tope",
             ],
             "title": ["title", "tit2", "\u00a9nam"],
-            "comment": ["comment"],
             "circle": [
                 "album-artist",
                 "tpe2",

copyparty/up2k.py

@@ -125,7 +125,7 @@ class Up2k(object):
         self.mtp_parsers: dict[str, dict[str, MParser]] = {}
         self.pending_tags: list[tuple[set[str], str, str, dict[str, Any]]] = []
         self.hashq: Queue[tuple[str, str, str, str, float]] = Queue()
-        self.tagq: Queue[tuple[str, str, str, str]] = Queue()
+        self.tagq: Queue[tuple[str, str, str, str, str, float]] = Queue()
         self.tag_event = threading.Condition()
         self.n_hashq = 0
         self.n_tagq = 0
@@ -1288,8 +1288,8 @@ class Up2k(object):
 
             with self.mutex:
                 try:
-                    q = "select rd, fn from up where substr(w,1,16)=? and +w=?"
-                    rd, fn = cur.execute(q, (w[:16], w)).fetchone()
+                    q = "select rd, fn, ip, at from up where substr(w,1,16)=? and +w=?"
+                    rd, fn, ip, at = cur.execute(q, (w[:16], w)).fetchone()
                 except:
                     # file modified/deleted since spooling
                     continue
@@ -1304,9 +1304,14 @@ class Up2k(object):
             abspath = os.path.join(ptop, rd, fn)
             self.pp.msg = "c{} {}".format(nq, abspath)
             if not mpool:
-                n_tags = self._tagscan_file(cur, entags, w, abspath)
+                n_tags = self._tagscan_file(cur, entags, w, abspath, ip, at)
             else:
-                mpool.put(Mpqe({}, entags, w, abspath, {}))
+                if ip:
+                    oth_tags = {"up_ip": ip, "up_at": at}
+                else:
+                    oth_tags = {}
+
+                mpool.put(Mpqe({}, entags, w, abspath, oth_tags))
                 with self.mutex:
                     n_tags = len(self._flush_mpool(cur))
 
@@ -1449,8 +1454,8 @@ class Up2k(object):
                     if w in in_progress:
                         continue
 
-                    q = "select rd, fn from up where substr(w,1,16)=? limit 1"
-                    rd, fn = cur.execute(q, (w,)).fetchone()
+                    q = "select rd, fn, ip, at from up where substr(w,1,16)=? limit 1"
+                    rd, fn, ip, at = cur.execute(q, (w,)).fetchone()
                     rd, fn = s3dec(rd, fn)
                     abspath = os.path.join(ptop, rd, fn)
 
@@ -1472,6 +1477,10 @@ class Up2k(object):
                     else:
                         oth_tags = {}
 
+                    if ip:
+                        oth_tags["up_ip"] = ip
+                        oth_tags["up_at"] = at
+
                     jobs.append(Mpqe(parsers, set(), w, abspath, oth_tags))
                     in_progress[w] = True
 
@@ -1641,6 +1650,8 @@ class Up2k(object):
         entags: set[str],
         wark: str,
         abspath: str,
+        ip: str,
+        at: float
     ) -> int:
         """will mutex"""
         assert self.mtag
@@ -1654,6 +1665,10 @@ class Up2k(object):
             self._log_tag_err("", abspath, ex)
             return 0
 
+        if ip:
+            tags["up_ip"] = ip
+            tags["up_at"] = at
+        
         with self.mutex:
             return self._tag_file(write_cur, entags, wark, abspath, tags)
 
@@ -2295,7 +2310,7 @@ class Up2k(object):
             raise
 
         if "e2t" in self.flags[ptop]:
-            self.tagq.put((ptop, wark, rd, fn))
+            self.tagq.put((ptop, wark, rd, fn, ip, at))
             self.n_tagq += 1
 
         return True
@@ -2941,7 +2956,7 @@ class Up2k(object):
             with self.mutex:
                 self.n_tagq -= 1
 
-            ptop, wark, rd, fn = self.tagq.get()
+            ptop, wark, rd, fn, ip, at = self.tagq.get()
             if "e2t" not in self.flags[ptop]:
                 continue
 
@@ -2952,6 +2967,8 @@ class Up2k(object):
                 ntags1 = len(tags)
                 parsers = self._get_parsers(ptop, tags, abspath)
                 if parsers:
+                    tags["up_ip"] = ip
+                    tags["up_at"] = at
                     tags.update(self.mtag.get_bin(parsers, abspath, tags))
             except Exception as ex:
                 self._log_tag_err("", abspath, ex)

copyparty/web/browser.js

@@ -318,6 +318,10 @@ var Ls = {
 		"u_ehssrch": "server rejected the request to perform search",
 		"u_ehsinit": "server rejected the request to initiate upload",
 		"u_ehsdf": "server ran out of disk space!\n\nwill keep retrying, in case someone\nfrees up enough space to continue",
+		"u_emtleak1": "it looks like your webbrowser may have a memory leak;\nplease",
+		"u_emtleak2": ' <a href="{0}">switch to https (recommended)</a> or ',
+		"u_emtleak3": ' ',
+		"u_emtleak4": "try the following:\n<ul><li>hit <code>F5</code> to refresh the page</li><li>then disable the &nbsp;<code>mt</code>&nbsp; button in the &nbsp;<code>⚙️ settings</code></li><li>and try that upload again</li></ul>Uploads will be a bit slower, but oh well.\nSorry for the trouble !",
 		"u_s404": "not found on server",
 		"u_expl": "explain",
 		"u_tu": '<p class="warn">WARNING: turbo enabled, <span>&nbsp;client may not detect and resume incomplete uploads; see turbo-button tooltip</span></p>',
@@ -656,6 +660,10 @@ var Ls = {
 		"u_ehssrch": "server nektet forespørselen om å utføre søk",
 		"u_ehsinit": "server nektet forespørselen om å begynne en ny opplastning",
 		"u_ehsdf": "serveren er full!\n\nprøver igjen regelmessig,\ni tilfelle noen rydder litt...",
+		"u_emtleak1": "uff, det er mulig at nettleseren din har en minnelekkasje...\nForeslår",
+		"u_emtleak2": ' helst at du <a href="{0}">bytter til https</a>, eller ',
+		"u_emtleak3": ' at du ',
+		"u_emtleak4": "prøver følgende:\n<ul><li>trykk F5 for å laste siden på nytt</li><li>så skru av &nbsp;<code>mt</code>&nbsp; bryteren under &nbsp;<code>⚙️ innstillinger</code></li><li>og forsøk den samme opplastningen igjen</li></ul>Opplastning vil gå litt tregere, men det får så være.\nBeklager bryderiet !",
 		"u_s404": "ikke funnet på serveren",
 		"u_expl": "forklar",
 		"u_tu": '<p class="warn">ADVARSEL: turbo er på, <span>&nbsp;avbrutte opplastninger vil muligens ikke oppdages og gjenopptas; hold musepekeren over turbo-knappen for mer info</span></p>',

copyparty/web/ui.css

@@ -251,6 +251,9 @@ html.y #tth {
 	padding: .3em;
 	text-align: center;
 }
+#modalc a {
+	color: #07b;
+}
 #modalb {
 	position: sticky;
 	text-align: right;

copyparty/web/up2k.js

@@ -808,7 +808,7 @@ function up2k_init(subtle) {
     bcfg_bind(uc, 'turbo', 'u2turbo', turbolvl > 1, draw_turbo, false);
     bcfg_bind(uc, 'datechk', 'u2tdate', turbolvl < 3, null, false);
     bcfg_bind(uc, 'az', 'u2sort', u2sort.indexOf('n') + 1, set_u2sort, false);
-    bcfg_bind(uc, 'hashw', 'hashw', !!window.WebAssembly, set_hashw, false);
+    bcfg_bind(uc, 'hashw', 'hashw', !!window.WebAssembly && (!HTTPS || !CHROME || MOBILE), set_hashw, false);
 
     var st = {
         "files": [],
@@ -847,6 +847,7 @@ function up2k_init(subtle) {
         },
         "car": 0,
         "slow_io": null,
+        "oserr": false,
         "modn": 0,
         "modv": 0,
         "mod0": null
@@ -1365,6 +1366,15 @@ function up2k_init(subtle) {
             etaskip = 0;
     }
 
+    function got_oserr() {
+        if (!hws.length || !uc.hashw || st.oserr)
+            return;
+
+        st.oserr = true;
+        var msg = HTTPS ? L.u_emtleak3 : L.u_emtleak2.format((window.location + '').replace(':', 's:'));
+        modal.alert(L.u_emtleak1 + msg + L.u_emtleak4);
+    }
+
     /////
     ////
     ///   actuator
@@ -1723,6 +1733,7 @@ function up2k_init(subtle) {
                     pvis.seth(t.n, 2, err + ' @ ' + car);
                     console.log('OS-error', reader.error, '@', car);
                     handled = true;
+                    got_oserr();
                 }
 
                 if (handled) {
@@ -1841,6 +1852,8 @@ function up2k_init(subtle) {
                 pvis.seth(t.n, 1, d[1]);
                 pvis.seth(t.n, 2, d[2]);
                 console.log(d[1], d[2]);
+                if (d[1] == 'OS-error')
+                    got_oserr();
 
                 pvis.move(t.n, 'ng');
                 apop(st.busy.hash, t);

copyparty/web/w.hash.js

@@ -8,7 +8,7 @@ function hex2u8(txt) {
 
 var subtle = null;
 try {
-    subtle = crypto.subtle || crypto.webkitSubtle;
+    subtle = crypto.subtle;
     subtle.digest('SHA-512', new Uint8Array(1)).then(
         function (x) { },
         function (x) { load_fb(); }
@@ -23,22 +23,39 @@ function load_fb() {
 }
 
 
+var reader = null,
+    gc1, gc2, gc3,
+    busy = false;
+
+
 onmessage = (d) => {
-    var [nchunk, fobj, car, cdr] = d.data,
-        t0 = Date.now(),
+    if (busy)
+        return postMessage(["panic", 'worker got another task while busy']);
+
+    if (!reader)
         reader = new FileReader();
 
+    var [nchunk, fobj, car, cdr] = d.data,
+        t0 = Date.now();
+
     reader.onload = function (e) {
         try {
+            // chrome gc forgets the filereader output; remind it
+            // (for some chromes, also necessary for subtle)
+            gc1 = e.target.result;
+            gc2 = new Uint8Array(gc1, 0, 1);
+            gc3 = new Uint8Array(gc1, gc1.byteLength - 1);
+
             //console.log('[ w] %d HASH bgin', nchunk);
             postMessage(["read", nchunk, cdr - car, Date.now() - t0]);
-            hash_calc(e.target.result);
+            hash_calc(gc1);
         }
         catch (ex) {
             postMessage(["panic", ex + '']);
         }
     };
     reader.onerror = function () {
+        busy = false;
         var err = reader.error + '';
 
         if (err.indexOf('NotReadableError') !== -1 || // win10-chrome defender
@@ -49,12 +66,19 @@ onmessage = (d) => {
         postMessage(["ferr", err]);
     };
     //console.log('[ w] %d read bgin', nchunk);
+    busy = true;
     reader.readAsArrayBuffer(
         File.prototype.slice.call(fobj, car, cdr));
 
 
     var hash_calc = function (buf) {
         var hash_done = function (hashbuf) {
+            // stop gc from attempting to free early
+            if (!gc1 || !gc2 || !gc3)
+                return console.log('torch went out');
+
+            gc1 = gc2 = gc3 = null;
+            busy = false;
             try {
                 var hslice = new Uint8Array(hashbuf).subarray(0, 33);
                 //console.log('[ w] %d HASH DONE', nchunk);
@@ -65,9 +89,14 @@ onmessage = (d) => {
             }
         };
 
+        // stop gc from attempting to free early
+        if (!gc1 || !gc2 || !gc3)
+            console.log('torch went out');
+
         if (subtle)
             subtle.digest('SHA-512', buf).then(hash_done);
         else {
+            // note: lifting u8buf counterproductive for the chrome gc bug
             var u8buf = new Uint8Array(buf);
             hashwasm.sha512(u8buf).then(function (v) {
                 hash_done(hex2u8(v))

docs/changelog.md

@@ -1,3 +1,38 @@
+▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀  
+# 2022-0817-2302  `v1.3.15`  pls let me stop finding chrome bugs
+
+two browser-bugs in two hours, man i just wanna play horizon
+* read-only demo server at https://a.ocv.me/pub/demo/
+* latest gzip edition of the sfx: [v1.0.14](https://github.com/9001/copyparty/releases/tag/v1.0.14#:~:text=release-specific%20notes)
+
+## bugfixes
+* chrome randomly running out of memory while hashing files and `mt` is enabled
+  * the gc suddenly gives up collecting the filereaders
+  * fixed by reusing a pool of readers instead
+* chrome failing to gc Any Buffers At All while hashing files and `mt` is enabled on plaintext http
+  * this one's funkier, they've repeatedly fixed and broke it like 6 times between chrome 84 and 106
+  * looks like it just forgets about everything that's passed into wasm
+  * no way around it, just show a popup explaining how to disable multithreaded hashing
+
+
+
+▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀  
+# 2022-0815-1825  `v1.3.14`  fix windows db
+
+after two exciting releases, time for something boring
+* read-only demo server at https://a.ocv.me/pub/demo/
+* latest gzip edition of the sfx: [v1.0.14](https://github.com/9001/copyparty/releases/tag/v1.0.14#:~:text=release-specific%20notes)
+
+## new features
+* upload-info (ip and timestamp) is provided to `mtp` tagparser plugins as json
+* tagscanner will index `fmt` (file-format / container type) by default
+  * and `description` can be enabled in `-mte`
+
+## bugfixes
+* [v1.3.12](https://github.com/9001/copyparty/releases/tag/v1.3.12) broke file-indexing on windows if an entire HDD was mounted as a volume
+
+
+
 ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀  
 # 2022-0812-2258  `v1.3.12`  quickboot
 

scripts/rls.sh

@@ -3,7 +3,11 @@ set -e
 
 parallel=2
 
-cd ~/dev/copyparty/scripts
+[ -e make-sfx.sh ] || cd scripts
+[ -e make-sfx.sh ] && [ -e deps-docker ] || {
+    echo cd into the scripts folder first
+    exit 1
+}
 
 v=$1
 
@@ -36,10 +40,14 @@ $f.py -h >/dev/null
     printf '\033[%s' s 2r H "0;1;37;44mbruteforcing sfx size -- press enter to terminate" K u "7m $* " K $'27m\n'
     trap "rm -f .sfx-run; printf '\033[%s' s r u" INT TERM EXIT
     touch .sfx-run
+    min=99999999
     for ((a=0; a<$parallel; a++)); do
         while [ -e .sfx-run ]; do
             CSN=sfx$a ./make-sfx.sh re "$@"
-            mv $f$a.py $f.$(wc -c <$f$a.py | awk '{print$1}').py
+            sz=$(wc -c <$f$a.py | awk '{print$1}')
+            [ $sz -ge $min ] && continue
+            mv $f$a.py $f.py.$sz
+            min=$sz
         done &
     done
     read