v0.7.7

copyparty/__version__.py

@@ -1,8 +1,8 @@
 # coding: utf-8
 
-VERSION = (0, 7, 6)
+VERSION = (0, 7, 7)
 CODENAME = "keeping track"
-BUILD_DT = (2021, 2, 12)
+BUILD_DT = (2021, 2, 14)
 
 S_VERSION = ".".join(map(str, VERSION))
 S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT)

copyparty/httpcli.py

@@ -34,6 +34,7 @@ class HttpCli(object):
         self.auth = conn.auth
         self.log_func = conn.log_func
         self.log_src = conn.log_src
+        self.tls = hasattr(self.s, "cipher")
 
         self.bufsz = 1024 * 32
         self.absolute_urls = False
@@ -134,16 +135,6 @@ class HttpCli(object):
             uparam["raw"] = True
             uparam["dots"] = True
 
-        if hasattr(self.s, "cipher"):
-            self.ssl_suf = "".join(
-                [
-                    " \033[3{}m{}".format(c, s)
-                    for c, s in zip([6, 3, 6], self.s.cipher())
-                ]
-            )
-        else:
-            self.ssl_suf = ""
-
         try:
             if self.mode in ["GET", "HEAD"]:
                 return self.handle_get() and self.keepalive
@@ -221,7 +212,7 @@ class HttpCli(object):
 
             logmsg += " [\033[36m" + rval + "\033[0m]"
 
-        self.log(logmsg + self.ssl_suf)
+        self.log(logmsg)
 
         # "embedded" resources
         if self.vpath.startswith(".cpr"):
@@ -255,7 +246,7 @@ class HttpCli(object):
         return self.tx_browser()
 
     def handle_options(self):
-        self.log("OPTIONS " + self.req + self.ssl_suf)
+        self.log("OPTIONS " + self.req)
         self.send_headers(
             None,
             204,
@@ -268,7 +259,7 @@ class HttpCli(object):
         return True
 
     def handle_put(self):
-        self.log("PUT " + self.req + self.ssl_suf)
+        self.log("PUT " + self.req)
 
         if self.headers.get("expect", "").lower() == "100-continue":
             try:
@@ -279,7 +270,7 @@ class HttpCli(object):
         return self.handle_stash()
 
     def handle_post(self):
-        self.log("POST " + self.req + self.ssl_suf)
+        self.log("POST " + self.req)
 
         if self.headers.get("expect", "").lower() == "100-continue":
             try:
@@ -496,7 +487,12 @@ class HttpCli(object):
                 self.log("clone {} done".format(cstart[0]))
 
         x = self.conn.hsrv.broker.put(True, "up2k.confirm_chunk", ptop, wark, chash)
-        num_left, path = x.get()
+        x = x.get()
+        try:
+            num_left, path = x
+        except:
+            self.loud_reply(x, status=500)
+            return False
 
         if not WINDOWS and num_left == 0:
             times = (int(time.time()), int(lastmod))
@@ -938,7 +934,7 @@ class HttpCli(object):
             # 512 kB is optimal for huge files, use 64k
             open_args = [fsenc(fs_path), "rb", 64 * 1024]
             use_sendfile = (
-                not self.ssl_suf
+                not self.tls  #
                 and not self.args.no_sendfile
                 and hasattr(os, "sendfile")
             )

copyparty/httpconn.py

@@ -141,6 +141,12 @@ class HttpConn(object):
                     ctx.set_ciphers(self.args.ciphers)
 
                 self.s = ctx.wrap_socket(self.s, server_side=True)
+                msg = [
+                    "\033[1;3{:d}m{}".format(c, s)
+                    for c, s in zip([0, 5, 0], self.s.cipher())
+                ]
+                self.log(" ".join(msg) + "\033[0m")
+
                 if self.args.ssl_dbg and hasattr(self.s, "shared_ciphers"):
                     overlap = [y[::-1] for y in self.s.shared_ciphers()]
                     lines = [str(x) for x in (["TLS cipher overlap:"] + overlap)]

copyparty/httpsrv.py

@@ -78,7 +78,7 @@ class HttpSrv(object):
                 if not MACOS:
                     self.log(
                         "%s %s" % addr,
-                        "shut_rdwr err:\n  {}\n  {}".format(repr(sck), ex),
+                        "\033[1;30mshut({}): {}\033[0m".format(sck.fileno(), ex),
                     )
                 if ex.errno not in [10038, 10054, 107, 57, 9]:
                     # 10038 No longer considered a socket

copyparty/up2k.py

@@ -2,8 +2,9 @@
 from __future__ import print_function, unicode_literals
 
 
-import os
 import re
+import os
+import sys
 import time
 import math
 import json
@@ -421,7 +422,7 @@ class Up2k(object):
                 raise Pebkac(400, "unknown wark")
 
             if chash not in job["need"]:
-                raise Pebkac(200, "already got that but thanks??")
+                raise Pebkac(400, "already got that but thanks??")
 
             nchunk = [n for n, v in enumerate(job["hash"]) if v == chash]
             if not nchunk:
@@ -438,12 +439,19 @@ class Up2k(object):
 
     def confirm_chunk(self, ptop, wark, chash):
         with self.mutex:
-            job = self.registry[ptop][wark]
-            pdir = os.path.join(job["ptop"], job["prel"])
-            src = os.path.join(pdir, job["tnam"])
-            dst = os.path.join(pdir, job["name"])
+            try:
+                job = self.registry[ptop][wark]
+                pdir = os.path.join(job["ptop"], job["prel"])
+                src = os.path.join(pdir, job["tnam"])
+                dst = os.path.join(pdir, job["name"])
+            except Exception as ex:
+                return "confirm_chunk, wark, " + repr(ex)
+
+            try:
+                job["need"].remove(chash)
+            except Exception as ex:
+                return "confirm_chunk, chash, " + repr(ex)
 
-            job["need"].remove(chash)
             ret = len(job["need"])
             if ret > 0:
                 return ret, src
@@ -524,9 +532,10 @@ class Up2k(object):
             while fsz > 0:
                 now = time.time()
                 td = now - last_print
-                if td >= 0.3:
+                if td >= 0.1:
                     last_print = now
-                    print(" {}    \n\033[A".format(fsz), end="")
+                    msg = " {} MB   \r".format(int(fsz / 1024 / 1024))
+                    print(msg, end="", file=sys.stderr)
 
                 hashobj = hashlib.sha512()
                 rem = min(csz, fsz)
@@ -568,6 +577,7 @@ class Up2k(object):
             # self.log("lmod", "got {}".format(len(ready)))
             time.sleep(5)
             for path, times in ready:
+                self.log("lmod", "setting times {} on {}".format(times, path))
                 try:
                     os.utime(fsenc(path), times)
                 except:

copyparty/web/up2k.js

@@ -89,6 +89,104 @@ catch (ex) {
 }
 
 
+function up2k_flagbus() {
+    var flag = {
+        "id": Math.floor(Math.random() * 1024 * 1024 * 1023 * 2),
+        "ch": new BroadcastChannel("up2k_flagbus"),
+        "ours": false,
+        "owner": null,
+        "wants": null,
+        "act": false,
+        "last_tx": ["x", null]
+    };
+    var dbg = function (who, msg) {
+        console.log('flagbus(' + flag.id + '): [' + who + '] ' + msg);
+    };
+    flag.ch.onmessage = function (ev) {
+        var who = ev.data[0],
+            what = ev.data[1];
+
+        if (who == flag.id) {
+            dbg(who, 'hi me (??)');
+            return;
+        }
+        flag.act = new Date().getTime();
+        if (what == "want") {
+            // lowest id wins, don't care if that's us
+            if (who < flag.id) {
+                dbg(who, 'wants (ack)');
+                flag.wants = [who, flag.act];
+            }
+            else {
+                dbg(who, 'wants (ign)');
+            }
+        }
+        else if (what == "have") {
+            dbg(who, 'have');
+            flag.owner = [who, flag.act];
+        }
+        else if (what == "give") {
+            if (flag.owner && flag.owner[0] == who) {
+                flag.owner = null;
+                dbg(who, 'give (ok)');
+            }
+            else {
+                dbg(who, 'give, INVALID, ' + flag.owner);
+            }
+        }
+        else if (what == "hi") {
+            dbg(who, 'hi');
+            flag.ch.postMessage([flag.id, "hey"]);
+        }
+        else {
+            dbg('?', ev.data);
+        }
+    };
+    var tx = function (now, msg) {
+        var td = now - flag.last_tx[1];
+        if (td > 500 || flag.last_tx[0] != msg) {
+            dbg('*', 'tx ' + msg);
+            flag.ch.postMessage([flag.id, msg]);
+            flag.last_tx = [msg, now];
+        }
+    };
+    var do_take = function (now) {
+        //dbg('*', 'do_take');
+        tx(now, "have");
+        flag.owner = [flag.id, now];
+        flag.ours = true;
+    };
+    var do_want = function (now) {
+        //dbg('*', 'do_want');
+        tx(now, "want");
+    };
+    flag.take = function (now) {
+        if (flag.ours) {
+            do_take(now);
+            return;
+        }
+        if (flag.owner && now - flag.owner[1] > 5000) {
+            flag.owner = null;
+        }
+        if (flag.wants && now - flag.wants[1] > 5000) {
+            flag.wants = null;
+        }
+        if (!flag.owner && !flag.wants) {
+            do_take(now);
+            return;
+        }
+        do_want(now);
+    };
+    flag.give = function () {
+        dbg('#', 'put give');
+        flag.ch.postMessage([flag.id, "give"]);
+        flag.owner = null;
+        flag.ours = false;
+    };
+    flag.ch.postMessage([flag.id, 'hi']);
+    return flag;
+}
+
 function up2k_init(have_crypto) {
     //have_crypto = false;
     var need_filereader_cache = undefined;
@@ -202,6 +300,7 @@ function up2k_init(have_crypto) {
     var parallel_uploads = cfg_get('nthread');
     var multitask = bcfg_get('multitask', true);
     var ask_up = bcfg_get('ask_up', true);
+    var flag_en = bcfg_get('flag_en', false);
 
     var col_hashing = '#00bbff';
     var col_hashed = '#004466';
@@ -233,6 +332,9 @@ function up2k_init(have_crypto) {
     if (!bobslice || !window.FileReader || !window.FileList)
         return un2k("this is the basic uploader; up2k needs at least<br />chrome 21 // firefox 13 // edge 12 // opera 12 // safari 5.1");
 
+    var flag = false;
+    apply_flag_cfg();
+
     function nav() {
         ebi('file' + fdom_ctr).click();
     }
@@ -379,8 +481,54 @@ function up2k_init(have_crypto) {
 
             mutex = true;
             while (true) {
+                if (false) {
+                    ebi('srv_info').innerHTML =
+                        new Date().getTime() + ", " +
+                        st.todo.hash.length + ", " +
+                        st.todo.handshake.length + ", " +
+                        st.todo.upload.length + ", " +
+                        st.busy.hash.length + ", " +
+                        st.busy.handshake.length + ", " +
+                        st.busy.upload.length;
+                }
+
+                if (flag) {
+                    var need_flag = 0 !=
+                        st.todo.hash.length +
+                        st.todo.handshake.length +
+                        st.todo.upload.length +
+                        st.busy.hash.length +
+                        st.busy.handshake.length +
+                        st.busy.upload.length;
+
+                    if (need_flag) {
+                        var now = new Date().getTime();
+                        flag.take(now);
+                        if (!flag.ours) {
+                            setTimeout(taskerd, 100);
+                            mutex = false;
+                            return;
+                        }
+                    }
+                    else if (flag.ours) {
+                        flag.give();
+                    }
+                }
+
                 var mou_ikkai = false;
 
+                if (st.todo.handshake.length > 0 &&
+                    st.busy.handshake.length == 0 && (
+                        st.todo.handshake[0].t3 || (
+                            handshakes_permitted() &&
+                            st.busy.upload.length < parallel_uploads
+                        )
+                    )
+                ) {
+                    exec_handshake();
+                    mou_ikkai = true;
+                }
+
                 if (handshakes_permitted() &&
                     st.todo.handshake.length > 0 &&
                     st.busy.handshake.length == 0 &&
@@ -520,6 +668,7 @@ function up2k_init(have_crypto) {
         var t = st.todo.hash.shift();
         st.busy.hash.push(t);
         st.bytes.hashed += t.size;
+        t.bytes_uploaded = 0;
         t.t1 = new Date().getTime();
 
         var nchunk = 0;
@@ -683,11 +832,14 @@ function up2k_init(have_crypto) {
                 st.busy.handshake.splice(st.busy.handshake.indexOf(t), 1);
 
                 if (done) {
+                    st.bytes.uploaded += t.size - t.bytes_uploaded;
                     var spd1 = (t.size / ((t.t2 - t.t1) / 1000.)) / (1024 * 1024.);
                     var spd2 = (t.size / ((t.t3 - t.t2) / 1000.)) / (1024 * 1024.);
                     ebi('f{0}p'.format(t.n)).innerHTML = 'hash {0}, up {1} MB/s'.format(
                         spd1.toFixed(2), spd2.toFixed(2));
                 }
+                else t.t3 = undefined;
+
                 tasker();
             }
             else {
@@ -761,12 +913,13 @@ function up2k_init(have_crypto) {
                 if (xhr.status == 200) {
                     prog(t.n, npart, col_uploaded);
                     st.bytes.uploaded += cdr - car;
+                    t.bytes_uploaded += cdr - car;
                     st.busy.upload.splice(st.busy.upload.indexOf(upt), 1);
                     t.postlist.splice(t.postlist.indexOf(npart), 1);
                     if (t.postlist.length == 0) {
                         t.t3 = new Date().getTime();
                         ebi('f{0}t'.format(t.n)).innerHTML = 'verifying';
-                        st.todo.handshake.push(t);
+                        st.todo.handshake.unshift(t);
                     }
                     tasker();
                 }
@@ -854,6 +1007,28 @@ function up2k_init(have_crypto) {
         bcfg_set('ask_up', ask_up);
     }
 
+    function tgl_flag_en() {
+        flag_en = !flag_en;
+        bcfg_set('flag_en', flag_en);
+        apply_flag_cfg();
+    }
+
+    function apply_flag_cfg() {
+        if (flag_en && !flag) {
+            try {
+                flag = up2k_flagbus();
+            }
+            catch (ex) {
+                console.log("flag error: " + ex.toString());
+                tgl_flag_en();
+            }
+        }
+        else if (!flag_en && flag) {
+            flag.ch.close();
+            flag = false;
+        }
+    }
+
     function nop(ev) {
         ev.preventDefault();
         this.click();
@@ -871,6 +1046,7 @@ function up2k_init(have_crypto) {
     ebi('nthread').addEventListener('input', bumpthread, false);
     ebi('multitask').addEventListener('click', tgl_multitask, false);
     ebi('ask_up').addEventListener('click', tgl_ask_up, false);
+    ebi('flag_en').addEventListener('click', tgl_flag_en, false);
 
     var nodes = ebi('u2conf').getElementsByTagName('a');
     for (var a = nodes.length - 1; a >= 0; a--)

copyparty/web/upload.html

@@ -53,11 +53,15 @@
                     </td>
                     <td rowspan="2" style="padding-left:1.5em">
                         <input type="checkbox" id="multitask" />
-                        <label for="multitask">hash while<br />uploading</label>
+                        <label for="multitask">hash<br />while<br />upping</label>
                     </td>
                     <td rowspan="2">
                         <input type="checkbox" id="ask_up" />
-                        <label for="ask_up">ask for<br />confirmation</label>
+                        <label for="ask_up">ask<br />before<br />start</label>
+                    </td>
+                    <td rowspan="2">
+                        <input type="checkbox" id="flag_en" />
+                        <label for="flag_en">only<br />one tab<br />at once</label>
                     </td>
                 </tr>
             </table>