v0.11.31

README.md

@@ -23,6 +23,7 @@ turn your phone or raspi into a portable file server with resumable uploads/down
         * [on debian](#on-debian)
     * [notes](#notes)
     * [status](#status)
+    * [testimonials](#testimonials)
 * [bugs](#bugs)
     * [general bugs](#general-bugs)
     * [not my bugs](#not-my-bugs)
@@ -143,6 +144,13 @@ summary: all planned features work! now please enjoy the bloatening
   * ☑ editor (sure why not)
 
 
+## testimonials
+
+small collection of user feedback
+
+`good enough`, `surprisingly correct`, `certified good software`, `just works`, `why`
+
+
 # bugs
 
 * Windows: python 3.7 and older cannot read tags with ffprobe, so use mutagen or upgrade
@@ -618,6 +626,7 @@ roughly sorted by priority
 * reduce up2k roundtrips
   * start from a chunk index and just go
   * terminate client on bad data
+* logging to file
 
 discarded ideas
 
@@ -637,3 +646,6 @@ discarded ideas
   * nah
 * look into android thumbnail cache file format
   * absolutely not
+* indexedDB for hashes, cfg enable/clear/sz, 2gb avail, ~9k for 1g, ~4k for 100m, 500k items before autoeviction
+  * blank hashlist when up-ok to skip handshake
+    * too many confusing side-effects

copyparty/__version__.py

@@ -1,8 +1,8 @@
 # coding: utf-8
 
-VERSION = (0, 11, 29)
+VERSION = (0, 11, 31)
 CODENAME = "the grid"
-BUILD_DT = (2021, 6, 30)
+BUILD_DT = (2021, 7, 4)
 
 S_VERSION = ".".join(map(str, VERSION))
 S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT)

copyparty/httpcli.py

@@ -607,13 +607,14 @@ class HttpCli(object):
                     os.makedirs(fsenc(dst))
             except OSError as ex:
                 self.log("makedirs failed [{}]".format(dst))
-                if ex.errno == 13:
-                    raise Pebkac(500, "the server OS denied write-access")
+                if not os.path.isdir(fsenc(dst)):
+                    if ex.errno == 13:
+                        raise Pebkac(500, "the server OS denied write-access")
 
-                if ex.errno == 17:
-                    raise Pebkac(400, "some file got your folder name")
+                    if ex.errno == 17:
+                        raise Pebkac(400, "some file got your folder name")
 
-                raise Pebkac(500, min_ex())
+                    raise Pebkac(500, min_ex())
             except:
                 raise Pebkac(500, min_ex())
 

copyparty/up2k.py

@@ -1047,8 +1047,9 @@ class Up2k(object):
                         pdir = os.path.join(cj["ptop"], cj["prel"])
                         job["name"] = self._untaken(pdir, cj["name"], now, cj["addr"])
                         dst = os.path.join(job["ptop"], job["prel"], job["name"])
-                        os.unlink(fsenc(dst))  # TODO ed pls
-                        self._symlink(src, dst)
+                        if not self.args.nw:
+                            os.unlink(fsenc(dst))  # TODO ed pls
+                            self._symlink(src, dst)
 
             if not job:
                 job = {

copyparty/web/up2k.js

@@ -225,7 +225,7 @@ function U2pvis(act, btns) {
     this.hashed = function (fobj) {
         var fo = this.tab[fobj.n],
             nb = fo.bt * (++fo.nh / fo.cb.length),
-            p = this.perc(nb, 0, fobj.size, fobj.t1);
+            p = this.perc(nb, 0, fobj.size, fobj.t_hashing);
 
         fo.hp = '{0}%, {1}, {2} MB/s'.format(
             p[0].toFixed(2), p[1], p[2].toFixed(2)
@@ -248,7 +248,7 @@ function U2pvis(act, btns) {
         fo.cb[nchunk] = cbd;
         fo.bd += delta;
 
-        var p = this.perc(fo.bd, fo.bd0, fo.bt, fobj.t3);
+        var p = this.perc(fo.bd, fo.bd0, fo.bt, fobj.t_uploading);
         fo.hp = '{0}%, {1}, {2} MB/s'.format(
             p[0].toFixed(2), p[1], p[2].toFixed(2)
         );
@@ -612,7 +612,7 @@ function up2k_init(subtle) {
         }
         else files = e.target.files;
 
-        if (!files || files.length == 0)
+        if (!files || !files.length)
             return alert('no files selected??');
 
         more_one_file();
@@ -715,8 +715,7 @@ function up2k_init(subtle) {
 
                     pf.push(name);
                     dn.file(function (fobj) {
-                        var idx = pf.indexOf(name);
-                        pf.splice(idx, 1);
+                        apop(pf, name);
                         try {
                             if (fobj.size > 0) {
                                 good.push([fobj, name]);
@@ -739,7 +738,7 @@ function up2k_init(subtle) {
     }
 
     function gotallfiles(good_files, bad_files) {
-        if (bad_files.length > 0) {
+        if (bad_files.length) {
             var ntot = bad_files.length + good_files.length,
                 msg = 'These {0} files (of {1} total) were skipped because they are empty:\n'.format(bad_files.length, ntot);
 
@@ -837,15 +836,25 @@ function up2k_init(subtle) {
     //
 
     function handshakes_permitted() {
-        var lim = multitask ? 1 : 0;
+        if (!st.todo.handshake.length)
+            return true;
 
-        if (lim <
-            st.todo.upload.length +
-            st.busy.upload.length)
+        var cd = st.todo.handshake[0].cooldown;
+        if (cd && cd - Date.now() > 0)
             return false;
 
-        var cd = st.todo.handshake.length ? st.todo.handshake[0].cooldown : 0;
-        if (cd && cd - Date.now() > 0)
+        // keepalive or verify
+        if (st.todo.handshake[0].keepalive ||
+            st.todo.handshake[0].t_uploaded)
+            return true;
+
+        if (parallel_uploads <
+            st.busy.handshake.length)
+            return false;
+
+        if ((multitask ? parallel_uploads : 0) <
+            st.todo.upload.length +
+            st.busy.upload.length)
             return false;
 
         return true;
@@ -880,13 +889,14 @@ function up2k_init(subtle) {
             clearTimeout(tto);
             running = true;
             while (window['vis_exh']) {
-                var is_busy = 0 !=
-                    st.todo.hash.length +
-                    st.todo.handshake.length +
-                    st.todo.upload.length +
-                    st.busy.hash.length +
-                    st.busy.handshake.length +
-                    st.busy.upload.length;
+                var now = Date.now(),
+                    is_busy = 0 !=
+                        st.todo.hash.length +
+                        st.todo.handshake.length +
+                        st.todo.upload.length +
+                        st.busy.hash.length +
+                        st.busy.handshake.length +
+                        st.busy.upload.length;
 
                 if (was_busy != is_busy) {
                     was_busy = is_busy;
@@ -897,7 +907,6 @@ function up2k_init(subtle) {
 
                 if (flag) {
                     if (is_busy) {
-                        var now = Date.now();
                         flag.take(now);
                         if (!flag.ours)
                             return defer();
@@ -909,43 +918,46 @@ function up2k_init(subtle) {
 
                 var mou_ikkai = false;
 
-                if (st.busy.handshake.length > 0 &&
-                    st.busy.handshake[0].busied < Date.now() - 30 * 1000
+                if (st.busy.handshake.length &&
+                    st.busy.handshake[0].t_busied < now - 30 * 1000
                 ) {
                     console.log("retrying stuck handshake");
                     var t = st.busy.handshake.shift();
                     st.todo.handshake.unshift(t);
                 }
 
-                if (st.todo.handshake.length > 0 &&
-                    st.busy.handshake.length == 0 && (
-                        st.todo.handshake[0].t4 || (
-                            handshakes_permitted() &&
-                            st.busy.upload.length < parallel_uploads
-                        )
-                    )
-                ) {
-                    exec_handshake();
-                    mou_ikkai = true;
+                var nprev = -1;
+                for (var a = 0; a < st.todo.upload.length; a++) {
+                    var nf = st.todo.upload[a].nfile;
+                    if (nprev == nf)
+                        continue;
+
+                    nprev = nf;
+                    var t = st.files[nf];
+                    if (now - t.t_busied > 1000 * 30 &&
+                        now - t.t_handshake > 1000 * (21600 - 1800)
+                    ) {
+                        apop(st.todo.handshake, t);
+                        st.todo.handshake.unshift(t);
+                        t.keepalive = true;
+                    }
                 }
 
                 if (handshakes_permitted() &&
-                    st.todo.handshake.length > 0 &&
-                    st.busy.handshake.length == 0 &&
-                    st.busy.upload.length < parallel_uploads) {
+                    st.todo.handshake.length) {
                     exec_handshake();
                     mou_ikkai = true;
                 }
 
-                if (st.todo.upload.length > 0 &&
+                if (st.todo.upload.length &&
                     st.busy.upload.length < parallel_uploads) {
                     exec_upload();
                     mou_ikkai = true;
                 }
 
                 if (hashing_permitted() &&
-                    st.todo.hash.length > 0 &&
-                    st.busy.hash.length == 0) {
+                    st.todo.hash.length &&
+                    !st.busy.hash.length) {
                     exec_hash();
                     mou_ikkai = true;
                 }
@@ -1080,7 +1092,7 @@ function up2k_init(subtle) {
 
                 if (handled) {
                     pvis.move(t.n, 'ng');
-                    st.busy.hash.splice(st.busy.hash.indexOf(t), 1);
+                    apop(st.busy.hash, t);
                     st.bytes.uploaded += t.size;
                     return tasker();
                 }
@@ -1113,15 +1125,15 @@ function up2k_init(subtle) {
                     t.hash.push(hashtab[a]);
                 }
 
-                t.t2 = Date.now();
+                t.t_hashed = Date.now();
                 if (t.n == 0 && window.location.hash == '#dbg') {
-                    var spd = (t.size / ((t.t2 - t.t1) / 1000.)) / (1024 * 1024.);
-                    alert('{0} ms, {1} MB/s\n'.format(t.t2 - t.t1, spd.toFixed(3)) + t.hash.join('\n'));
+                    var spd = (t.size / ((t.t_hashed - t.t_hashing) / 1000.)) / (1024 * 1024.);
+                    alert('{0} ms, {1} MB/s\n'.format(t.t_hashed - t.t_hashing, spd.toFixed(3)) + t.hash.join('\n'));
                 }
 
                 pvis.seth(t.n, 2, 'hashing done');
                 pvis.seth(t.n, 1, '📦 wait');
-                st.busy.hash.splice(st.busy.hash.indexOf(t), 1);
+                apop(st.busy.hash, t);
                 st.todo.handshake.push(t);
                 tasker();
             };
@@ -1144,7 +1156,7 @@ function up2k_init(subtle) {
             }, 1);
         };
 
-        t.t1 = Date.now();
+        t.t_hashing = Date.now();
         segm_next();
     }
 
@@ -1155,30 +1167,41 @@ function up2k_init(subtle) {
 
     function exec_handshake() {
         var t = st.todo.handshake.shift(),
+            keepalive = t.keepalive,
             me = Date.now();
 
         st.busy.handshake.push(t);
-        t.busied = me;
+        t.keepalive = undefined;
+        t.t_busied = me;
+
+        if (keepalive)
+            console.log("sending keepalive handshake", t);
 
         var xhr = new XMLHttpRequest();
         xhr.onerror = function () {
-            if (t.busied != me) {
+            if (t.t_busied != me) {
                 console.log('zombie handshake onerror,', t);
                 return;
             }
             console.log('handshake onerror, retrying', t);
-            st.busy.handshake.splice(st.busy.handshake.indexOf(t), 1);
+            apop(st.busy.handshake, t);
             st.todo.handshake.unshift(t);
+            t.keepalive = keepalive;
             tasker();
         };
         function orz(e) {
-            if (t.busied != me) {
+            if (t.t_busied != me) {
                 console.log('zombie handshake onload,', t);
                 return;
             }
             if (xhr.status == 200) {
-                var response = JSON.parse(xhr.responseText);
+                t.t_handshake = Date.now();
+                if (keepalive) {
+                    apop(st.busy.handshake, t);
+                    return;
+                }
 
+                var response = JSON.parse(xhr.responseText);
                 if (!response.name) {
                     var msg = '',
                         smsg = '';
@@ -1202,7 +1225,7 @@ function up2k_init(subtle) {
                     pvis.seth(t.n, 2, msg);
                     pvis.seth(t.n, 1, smsg);
                     pvis.move(t.n, smsg == '404' ? 'ng' : 'ok');
-                    st.busy.handshake.splice(st.busy.handshake.indexOf(t), 1);
+                    apop(st.busy.handshake, t);
                     st.bytes.uploaded += t.size;
                     t.done = true;
                     tasker();
@@ -1244,7 +1267,7 @@ function up2k_init(subtle) {
                 var done = true,
                     msg = '&#x1f3b7;&#x1f41b;';
 
-                if (t.postlist.length > 0) {
+                if (t.postlist.length) {
                     for (var a = 0; a < t.postlist.length; a++)
                         st.todo.upload.push({
                             'nfile': t.n,
@@ -1255,20 +1278,20 @@ function up2k_init(subtle) {
                     done = false;
                 }
                 pvis.seth(t.n, 1, msg);
-                st.busy.handshake.splice(st.busy.handshake.indexOf(t), 1);
+                apop(st.busy.handshake, t);
 
                 if (done) {
                     t.done = true;
                     st.bytes.uploaded += t.size - t.bytes_uploaded;
-                    var spd1 = (t.size / ((t.t2 - t.t1) / 1000.)) / (1024 * 1024.),
-                        spd2 = (t.size / ((t.t4 - t.t3) / 1000.)) / (1024 * 1024.);
+                    var spd1 = (t.size / ((t.t_hashed - t.t_hashing) / 1000.)) / (1024 * 1024.),
+                        spd2 = (t.size / ((t.t_uploaded - t.t_uploading) / 1000.)) / (1024 * 1024.);
 
                     pvis.seth(t.n, 2, 'hash {0}, up {1} MB/s'.format(
                         spd1.toFixed(2), spd2.toFixed(2)));
 
                     pvis.move(t.n, 'ok');
                 }
-                else t.t4 = undefined;
+                else t.t_uploaded = undefined;
 
                 tasker();
             }
@@ -1287,7 +1310,7 @@ function up2k_init(subtle) {
                     var penalty = rsp.replace(/.*rate-limit /, "").split(' ')[0];
                     console.log("rate-limit: " + penalty);
                     t.cooldown = Date.now() + parseFloat(penalty) * 1000;
-                    st.busy.handshake.splice(st.busy.handshake.indexOf(t), 1);
+                    apop(st.busy.handshake, t);
                     st.todo.handshake.unshift(t);
                     return;
                 }
@@ -1306,7 +1329,7 @@ function up2k_init(subtle) {
                     pvis.seth(t.n, 2, err);
                     pvis.move(t.n, 'ng');
 
-                    st.busy.handshake.splice(st.busy.handshake.indexOf(t), 1);
+                    apop(st.busy.handshake, t);
                     tasker();
                     return;
                 }
@@ -1347,8 +1370,8 @@ function up2k_init(subtle) {
         var npart = upt.npart,
             t = st.files[upt.nfile];
 
-        if (!t.t3)
-            t.t3 = Date.now();
+        if (!t.t_uploading)
+            t.t_uploading = Date.now();
 
         pvis.seth(t.n, 1, "🚀 send");
 
@@ -1367,17 +1390,17 @@ function up2k_init(subtle) {
                 t.bytes_uploaded += cdr - car;
             }
             else if (txt.indexOf('already got that') !== -1) {
-                console.log("ignoring dupe-segment error");
+                console.log("ignoring dupe-segment error", t);
             }
             else {
                 alert("server broke; cu-err {0} on file [{1}]:\n".format(
                     xhr.status, t.name) + (txt || "no further information"));
                 return;
             }
-            st.busy.upload.splice(st.busy.upload.indexOf(upt), 1);
-            t.postlist.splice(t.postlist.indexOf(npart), 1);
-            if (t.postlist.length == 0) {
-                t.t4 = Date.now();
+            apop(st.busy.upload, upt);
+            apop(t.postlist, npart);
+            if (!t.postlist.length) {
+                t.t_uploaded = Date.now();
                 pvis.seth(t.n, 1, 'verifying');
                 st.todo.handshake.unshift(t);
             }

copyparty/web/util.js

@@ -389,6 +389,13 @@ function has(haystack, needle) {
 }
 
 
+function apop(arr, v) {
+    var ofs = arr.indexOf(v);
+    if (ofs !== -1)
+        arr.splice(ofs, 1);
+}
+
+
 function jcp(obj) {
     return JSON.parse(JSON.stringify(obj));
 }

docs/hls.html

@@ -0,0 +1,51 @@
+<!DOCTYPE html><html lang="en"><head>
+	<meta charset="utf-8">
+	<title>hls-test</title>
+	<meta http-equiv="X-UA-Compatible" content="IE=edge">
+</head><body>
+
+<video id="vid" controls></video>
+<script src="hls.light.js"></script>
+<script>
+
+var video = document.getElementById('vid');
+var hls = new Hls({
+	debug: true,
+	autoStartLoad: false
+});
+hls.loadSource('live/v.m3u8');
+hls.attachMedia(video);
+hls.on(Hls.Events.MANIFEST_PARSED, function() {
+	hls.startLoad(0);
+});
+hls.on(Hls.Events.MEDIA_ATTACHED, function() {
+	video.muted = true;
+	video.play();
+});
+
+/*
+general good news:
+- doesn't need fixed-length segments; ok to let x264 pick optimal keyframes and slice on those
+- hls.js polls the m3u8 for new segments, scales the duration accordingly, seeking works great
+- the sfx will grow by 66 KiB since that's how small hls.js can get, wait thats not good
+
+# vod, creates m3u8 at the end, fixed keyframes, v bad
+ffmpeg -hide_banner -threads 0 -flags -global_header -i ..\CowboyBebopMovie-OP1.webm -vf scale=1280:-4,format=yuv420p -ac 2 -c:a libopus -b:a 128k -c:v libx264 -preset slow -crf 24 -maxrate:v 5M -bufsize:v 10M -g 120 -keyint_min 120 -sc_threshold 0 -hls_time 4 -hls_playlist_type vod -hls_segment_filename v%05d.ts v.m3u8
+
+# live, updates m3u8 as it goes, dynamic keyframes, streamable with hls.js
+ffmpeg -hide_banner -threads 0 -flags -global_header -i ..\..\CowboyBebopMovie-OP1.webm -vf scale=1280:-4,format=yuv420p -ac 2 -c:a libopus -b:a 128k -c:v libx264 -preset slow -crf 24 -maxrate:v 5M -bufsize:v 10M -f segment -segment_list v.m3u8 -segment_format mpegts -segment_list_flags live v%05d.ts
+
+# fmp4 (fragmented mp4), doesn't work with hls.js, gets duratoin 149:07:51 (536871s), probably the tkhd/mdhd 0xffffffff (timebase 8000? ok)
+ffmpeg -re -hide_banner -threads 0 -flags +cgop -i ..\..\CowboyBebopMovie-OP1.webm -vf scale=1280:-4,format=yuv420p -ac 2 -c:a libopus -b:a 128k -c:v libx264 -preset slow -crf 24 -maxrate:v 5M -bufsize:v 10M -f segment -segment_list v.m3u8 -segment_format fmp4 -segment_list_flags live v%05d.mp4
+
+# try 2, works, uses tempfiles for m3u8 updates, good, 6% smaller
+ffmpeg -re -hide_banner -threads 0 -flags +cgop -i ..\..\CowboyBebopMovie-OP1.webm -vf scale=1280:-4,format=yuv420p -ac 2 -c:a libopus -b:a 128k -c:v libx264 -preset slow -crf 24 -maxrate:v 5M -bufsize:v 10M -f hls -hls_segment_type fmp4 -hls_list_size 0 -hls_segment_filename v%05d.mp4 v.m3u8
+
+more notes
+- adding -hls_flags single_file makes duration wack during playback (for both fmp4 and ts), ok once finalized and refreshed, gives no size reduction anyways
+- bebop op has good keyframe spacing for testing hls.js, in particular it hops one seg back and immediately resumes if it hits eof with the explicit hls.startLoad(0); otherwise it jumps into the middle of a seg and becomes art
+- can probably -c:v copy most of the time, is there a way to check for cgop? todo
+
+*/
+</script>
+</body></html>