Compare commits

...

14 Commits

Author SHA1 Message Date
ed
06c6ddffb6 v0.7.7 2021-02-14 02:13:52 +01:00
ed
d29f0c066c logging 2021-02-14 01:32:16 +01:00
ed
c9e4de3346 up2k: fix rejected files not counting as progress 2021-02-13 04:30:46 +01:00
ed
ca0b97f72d oh cool 2021-02-13 03:59:38 +01:00
ed
b38f20b408 up2k: make tabsync optional 2021-02-13 03:45:40 +01:00
ed
05b1dbaf56 up2k: upload semaphore across tabs/windows 2021-02-13 02:57:51 +01:00
ed
b8481e32ba lovely priority inversions 2021-02-12 23:53:13 +01:00
ed
9c03c65e07 v0.7.6 2021-02-12 20:53:29 +01:00
ed
d8ed006b9b up2k: 128 MiB runahead 2021-02-12 20:41:42 +01:00
ed
63c0623a5e vscode: windows support 2021-02-12 19:47:18 +01:00
ed
fd84506db0 don't list up2k db in browser 2021-02-12 19:25:57 +01:00
ed
d8bcb44e44 vscode: no-debug launcher 2021-02-12 19:25:01 +01:00
ed
56a26b0916 up2k: print final commit too 2021-02-12 17:10:08 +01:00
ed
efcf1d6b90 add cfssl.sh 2021-02-12 07:30:20 +00:00
15 changed files with 345 additions and 53 deletions

7
.vscode/launch.json vendored
View File

@@ -20,6 +20,13 @@
"srv::r:aed:cnodupe"
]
},
{
"name": "No debug",
"preLaunchTask": "no_dbg",
"type": "python",
//"request": "attach", "port": 42069
// fork: nc -l 42069 </dev/null
},
{
"name": "Run active unit test",
"type": "python",

View File

@@ -50,15 +50,9 @@
"files.associations": {
"*.makefile": "makefile"
},
"editor.codeActionsOnSaveTimeout": 9001,
"editor.formatOnSaveTimeout": 9001,
//
// things you may wanna edit:
//
"python.pythonPath": "/usr/bin/python3",
"python.formatting.blackArgs": [
"-t",
"py27"
],
//"python.linting.enabled": true,
"python.linting.enabled": true,
}

5
.vscode/tasks.json vendored
View File

@@ -5,6 +5,11 @@
"label": "pre",
"command": "true;rm -rf inc/* inc/.hist/;mkdir -p inc;",
"type": "shell"
},
{
"label": "no_dbg",
"command": "${config:python.pythonPath} -m copyparty -ed -emp -e2d -e2s -a ed:wark -v srv::r:aed:cnodupe ;exit 1",
"type": "shell"
}
]
}

View File

@@ -76,8 +76,8 @@ optional, will eventually enable thumbnails:
# sfx
currently there are two self-contained binaries:
* `copyparty-sfx.sh` for unix (linux and osx) -- smaller, more robust
* `copyparty-sfx.py` for windows (unix too) -- crossplatform, beta
* [copyparty-sfx.py](https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.py) -- pure python, works everywhere
* [copyparty-sfx.sh](https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.sh) -- smaller, but only for linux and macos
launch either of them (**use sfx.py on systemd**) and it'll unpack and run copyparty, assuming you have python installed of course

View File

@@ -10,7 +10,12 @@
* modify `10.13.1.1` as necessary if you wish to support browsers without javascript
### [`explorer-nothumbs-nofoldertypes.reg`](explorer-nothumbs-nofoldertypes.reg)
disables thumbnails and folder-type detection in windows explorer, makes it way faster (especially for slow/networked locations (such as copyparty-fuse))
* disables thumbnails and folder-type detection in windows explorer
* makes it way faster (especially for slow/networked locations (such as copyparty-fuse))
### [`cfssl.sh`](cfssl.sh)
* creates CA and server certificates using cfssl
* give a 3rd argument to install it to your copyparty config
# OS integration
init-scripts to start copyparty as a service

72
contrib/cfssl.sh Executable file
View File

@@ -0,0 +1,72 @@
#!/bin/bash
set -e
# ca-name and server-name
ca_name="$1"
srv_name="$2"
[ -z "$srv_name" ] && {
echo "need arg 1: ca name"
echo "need arg 2: server name"
exit 1
}
gen_ca() {
(tee /dev/stderr <<EOF
{"CN": "$ca_name ca",
"CA": {"expiry":"87600h", "pathlen":0},
"key": {"algo":"rsa", "size":4096},
"names": [{"O":"$ca_name ca"}]}
EOF
)|
cfssl gencert -initca - |
cfssljson -bare ca
mv ca-key.pem ca.key
rm ca.csr
}
gen_srv() {
(tee /dev/stderr <<EOF
{"key": {"algo":"rsa", "size":4096},
"names": [{"O":"$ca_name - $srv_name"}]}
EOF
)|
cfssl gencert -ca ca.pem -ca-key ca.key \
-profile=www -hostname="$srv_name.$ca_name" - |
cfssljson -bare "$srv_name"
mv "$srv_name-key.pem" "$srv_name.key"
rm "$srv_name.csr"
}
# create ca if not exist
[ -e ca.key ] ||
gen_ca
# always create server cert
gen_srv
# dump cert info
show() {
openssl x509 -text -noout -in $1 |
awk '!o; {o=0} /[0-9a-f:]{16}/{o=1}'
}
show ca.pem
show "$srv_name.pem"
# write cert into copyparty config
[ -z "$3" ] || {
mkdir -p ~/.config/copyparty
cat "$srv_name".{key,pem} ca.pem >~/.config/copyparty/cert.pem
}
# rm *.key *.pem
# cfssl print-defaults config
# cfssl print-defaults csr

View File

@@ -144,12 +144,11 @@ def configure_ssl_ciphers(al):
is_help = al.ciphers == "help"
if al.ciphers:
if al.ciphers and not is_help:
try:
ctx.set_ciphers(al.ciphers)
except:
if not is_help:
print("\n\033[1;31mfailed to set ciphers\033[0m\n")
print("\n\033[1;31mfailed to set ciphers\033[0m\n")
if not hasattr(ctx, "get_ciphers"):
print("cannot read cipher list: openssl or python too old")
@@ -211,8 +210,8 @@ def main():
"print,get" prints the data in the log and returns GET
(leave out the ",get" to return an error instead)
see "--ciphers help" for available ssl/tls ciphers,
see "--ssl-ver help" for available ssl/tls versions,
--ciphers help = available ssl/tls ciphers,
--ssl-ver help = available ssl/tls versions,
default is what python considers safe, usually >= TLS1
"""
),

View File

@@ -1,8 +1,8 @@
# coding: utf-8
VERSION = (0, 7, 5)
VERSION = (0, 7, 7)
CODENAME = "keeping track"
BUILD_DT = (2021, 2, 12)
BUILD_DT = (2021, 2, 14)
S_VERSION = ".".join(map(str, VERSION))
S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT)

View File

@@ -34,6 +34,7 @@ class HttpCli(object):
self.auth = conn.auth
self.log_func = conn.log_func
self.log_src = conn.log_src
self.tls = hasattr(self.s, "cipher")
self.bufsz = 1024 * 32
self.absolute_urls = False
@@ -134,16 +135,6 @@ class HttpCli(object):
uparam["raw"] = True
uparam["dots"] = True
if hasattr(self.s, "cipher"):
self.ssl_suf = "".join(
[
" \033[3{}m{}".format(c, s)
for c, s in zip([6, 3, 6], self.s.cipher())
]
)
else:
self.ssl_suf = ""
try:
if self.mode in ["GET", "HEAD"]:
return self.handle_get() and self.keepalive
@@ -221,7 +212,7 @@ class HttpCli(object):
logmsg += " [\033[36m" + rval + "\033[0m]"
self.log(logmsg + self.ssl_suf)
self.log(logmsg)
# "embedded" resources
if self.vpath.startswith(".cpr"):
@@ -255,7 +246,7 @@ class HttpCli(object):
return self.tx_browser()
def handle_options(self):
self.log("OPTIONS " + self.req + self.ssl_suf)
self.log("OPTIONS " + self.req)
self.send_headers(
None,
204,
@@ -268,7 +259,7 @@ class HttpCli(object):
return True
def handle_put(self):
self.log("PUT " + self.req + self.ssl_suf)
self.log("PUT " + self.req)
if self.headers.get("expect", "").lower() == "100-continue":
try:
@@ -279,7 +270,7 @@ class HttpCli(object):
return self.handle_stash()
def handle_post(self):
self.log("POST " + self.req + self.ssl_suf)
self.log("POST " + self.req)
if self.headers.get("expect", "").lower() == "100-continue":
try:
@@ -496,7 +487,12 @@ class HttpCli(object):
self.log("clone {} done".format(cstart[0]))
x = self.conn.hsrv.broker.put(True, "up2k.confirm_chunk", ptop, wark, chash)
num_left, path = x.get()
x = x.get()
try:
num_left, path = x
except:
self.loud_reply(x, status=500)
return False
if not WINDOWS and num_left == 0:
times = (int(time.time()), int(lastmod))
@@ -938,7 +934,7 @@ class HttpCli(object):
# 512 kB is optimal for huge files, use 64k
open_args = [fsenc(fs_path), "rb", 64 * 1024]
use_sendfile = (
not self.ssl_suf
not self.tls #
and not self.args.no_sendfile
and hasattr(os, "sendfile")
)
@@ -1095,6 +1091,10 @@ class HttpCli(object):
if not self.args.ed or "dots" not in self.uparam:
vfs_ls = exclude_dotfiles(vfs_ls)
hidden = []
if fsroot.endswith(str(os.sep) + ".hist"):
hidden = ["up2k.db", "up2k.snap"]
dirs = []
files = []
for fn in vfs_ls:
@@ -1106,6 +1106,8 @@ class HttpCli(object):
if fn in vfs_virt:
fspath = vfs_virt[fn].realpath
elif fn in hidden:
continue
else:
fspath = fsroot + "/" + fn

View File

@@ -141,6 +141,12 @@ class HttpConn(object):
ctx.set_ciphers(self.args.ciphers)
self.s = ctx.wrap_socket(self.s, server_side=True)
msg = [
"\033[1;3{:d}m{}".format(c, s)
for c, s in zip([0, 5, 0], self.s.cipher())
]
self.log(" ".join(msg) + "\033[0m")
if self.args.ssl_dbg and hasattr(self.s, "shared_ciphers"):
overlap = [y[::-1] for y in self.s.shared_ciphers()]
lines = [str(x) for x in (["TLS cipher overlap:"] + overlap)]

View File

@@ -78,7 +78,7 @@ class HttpSrv(object):
if not MACOS:
self.log(
"%s %s" % addr,
"shut_rdwr err:\n {}\n {}".format(repr(sck), ex),
"\033[1;30mshut({}): {}\033[0m".format(sck.fileno(), ex),
)
if ex.errno not in [10038, 10054, 107, 57, 9]:
# 10038 No longer considered a socket

View File

@@ -2,8 +2,9 @@
from __future__ import print_function, unicode_literals
import os
import re
import os
import sys
import time
import math
import json
@@ -130,8 +131,12 @@ class Up2k(object):
if db:
# can be symlink so don't `and d.startswith(top)``
excl = set([d for d in tops if d != top])
self._build_dir([db, 0, time.time()], top, excl, top)
dbw = [db, 0, time.time()]
self._build_dir(dbw, top, excl, top)
self._drop_lost(db, top)
if dbw[1]:
self.log("up2k", "commit {} new files".format(dbw[1]))
db.commit()
def _build_dir(self, dbw, top, excl, cdir):
@@ -190,7 +195,7 @@ class Up2k(object):
dbw[1] += 1
td = time.time() - dbw[2]
if dbw[1] > 1024 or td > 60:
self.log("up2k", "commit {} files".format(dbw[1]))
self.log("up2k", "commit {} new files".format(dbw[1]))
dbw[0].commit()
dbw[1] = 0
dbw[2] = time.time()
@@ -417,7 +422,7 @@ class Up2k(object):
raise Pebkac(400, "unknown wark")
if chash not in job["need"]:
raise Pebkac(200, "already got that but thanks??")
raise Pebkac(400, "already got that but thanks??")
nchunk = [n for n, v in enumerate(job["hash"]) if v == chash]
if not nchunk:
@@ -434,12 +439,19 @@ class Up2k(object):
def confirm_chunk(self, ptop, wark, chash):
with self.mutex:
job = self.registry[ptop][wark]
pdir = os.path.join(job["ptop"], job["prel"])
src = os.path.join(pdir, job["tnam"])
dst = os.path.join(pdir, job["name"])
try:
job = self.registry[ptop][wark]
pdir = os.path.join(job["ptop"], job["prel"])
src = os.path.join(pdir, job["tnam"])
dst = os.path.join(pdir, job["name"])
except Exception as ex:
return "confirm_chunk, wark, " + repr(ex)
try:
job["need"].remove(chash)
except Exception as ex:
return "confirm_chunk, chash, " + repr(ex)
job["need"].remove(chash)
ret = len(job["need"])
if ret > 0:
return ret, src
@@ -520,9 +532,10 @@ class Up2k(object):
while fsz > 0:
now = time.time()
td = now - last_print
if td >= 0.3:
if td >= 0.1:
last_print = now
print(" {} \n\033[A".format(fsz), end="")
msg = " {} MB \r".format(int(fsz / 1024 / 1024))
print(msg, end="", file=sys.stderr)
hashobj = hashlib.sha512()
rem = min(csz, fsz)
@@ -564,6 +577,7 @@ class Up2k(object):
# self.log("lmod", "got {}".format(len(ready)))
time.sleep(5)
for path, times in ready:
self.log("lmod", "setting times {} on {}".format(times, path))
try:
os.utime(fsenc(path), times)
except:

View File

@@ -89,6 +89,104 @@ catch (ex) {
}
function up2k_flagbus() {
var flag = {
"id": Math.floor(Math.random() * 1024 * 1024 * 1023 * 2),
"ch": new BroadcastChannel("up2k_flagbus"),
"ours": false,
"owner": null,
"wants": null,
"act": false,
"last_tx": ["x", null]
};
var dbg = function (who, msg) {
console.log('flagbus(' + flag.id + '): [' + who + '] ' + msg);
};
flag.ch.onmessage = function (ev) {
var who = ev.data[0],
what = ev.data[1];
if (who == flag.id) {
dbg(who, 'hi me (??)');
return;
}
flag.act = new Date().getTime();
if (what == "want") {
// lowest id wins, don't care if that's us
if (who < flag.id) {
dbg(who, 'wants (ack)');
flag.wants = [who, flag.act];
}
else {
dbg(who, 'wants (ign)');
}
}
else if (what == "have") {
dbg(who, 'have');
flag.owner = [who, flag.act];
}
else if (what == "give") {
if (flag.owner && flag.owner[0] == who) {
flag.owner = null;
dbg(who, 'give (ok)');
}
else {
dbg(who, 'give, INVALID, ' + flag.owner);
}
}
else if (what == "hi") {
dbg(who, 'hi');
flag.ch.postMessage([flag.id, "hey"]);
}
else {
dbg('?', ev.data);
}
};
var tx = function (now, msg) {
var td = now - flag.last_tx[1];
if (td > 500 || flag.last_tx[0] != msg) {
dbg('*', 'tx ' + msg);
flag.ch.postMessage([flag.id, msg]);
flag.last_tx = [msg, now];
}
};
var do_take = function (now) {
//dbg('*', 'do_take');
tx(now, "have");
flag.owner = [flag.id, now];
flag.ours = true;
};
var do_want = function (now) {
//dbg('*', 'do_want');
tx(now, "want");
};
flag.take = function (now) {
if (flag.ours) {
do_take(now);
return;
}
if (flag.owner && now - flag.owner[1] > 5000) {
flag.owner = null;
}
if (flag.wants && now - flag.wants[1] > 5000) {
flag.wants = null;
}
if (!flag.owner && !flag.wants) {
do_take(now);
return;
}
do_want(now);
};
flag.give = function () {
dbg('#', 'put give');
flag.ch.postMessage([flag.id, "give"]);
flag.owner = null;
flag.ours = false;
};
flag.ch.postMessage([flag.id, 'hi']);
return flag;
}
function up2k_init(have_crypto) {
//have_crypto = false;
var need_filereader_cache = undefined;
@@ -202,6 +300,7 @@ function up2k_init(have_crypto) {
var parallel_uploads = cfg_get('nthread');
var multitask = bcfg_get('multitask', true);
var ask_up = bcfg_get('ask_up', true);
var flag_en = bcfg_get('flag_en', false);
var col_hashing = '#00bbff';
var col_hashed = '#004466';
@@ -219,6 +318,10 @@ function up2k_init(have_crypto) {
"hash": [],
"handshake": [],
"upload": []
},
"bytes": {
"hashed": 0,
"uploaded": 0
}
};
@@ -229,6 +332,9 @@ function up2k_init(have_crypto) {
if (!bobslice || !window.FileReader || !window.FileList)
return un2k("this is the basic uploader; up2k needs at least<br />chrome 21 // firefox 13 // edge 12 // opera 12 // safari 5.1");
var flag = false;
apply_flag_cfg();
function nav() {
ebi('file' + fdom_ctr).click();
}
@@ -357,8 +463,11 @@ function up2k_init(have_crypto) {
}
function hashing_permitted() {
var lim = multitask ? 1 : 0;
return handshakes_permitted() && lim >=
if (multitask) {
var ahead = st.bytes.hashed - st.bytes.uploaded;
return ahead < 1024 * 1024 * 128;
}
return handshakes_permitted() && 0 ==
st.todo.handshake.length +
st.busy.handshake.length;
}
@@ -372,8 +481,54 @@ function up2k_init(have_crypto) {
mutex = true;
while (true) {
if (false) {
ebi('srv_info').innerHTML =
new Date().getTime() + ", " +
st.todo.hash.length + ", " +
st.todo.handshake.length + ", " +
st.todo.upload.length + ", " +
st.busy.hash.length + ", " +
st.busy.handshake.length + ", " +
st.busy.upload.length;
}
if (flag) {
var need_flag = 0 !=
st.todo.hash.length +
st.todo.handshake.length +
st.todo.upload.length +
st.busy.hash.length +
st.busy.handshake.length +
st.busy.upload.length;
if (need_flag) {
var now = new Date().getTime();
flag.take(now);
if (!flag.ours) {
setTimeout(taskerd, 100);
mutex = false;
return;
}
}
else if (flag.ours) {
flag.give();
}
}
var mou_ikkai = false;
if (st.todo.handshake.length > 0 &&
st.busy.handshake.length == 0 && (
st.todo.handshake[0].t3 || (
handshakes_permitted() &&
st.busy.upload.length < parallel_uploads
)
)
) {
exec_handshake();
mou_ikkai = true;
}
if (handshakes_permitted() &&
st.todo.handshake.length > 0 &&
st.busy.handshake.length == 0 &&
@@ -512,6 +667,8 @@ function up2k_init(have_crypto) {
var t = st.todo.hash.shift();
st.busy.hash.push(t);
st.bytes.hashed += t.size;
t.bytes_uploaded = 0;
t.t1 = new Date().getTime();
var nchunk = 0;
@@ -675,11 +832,14 @@ function up2k_init(have_crypto) {
st.busy.handshake.splice(st.busy.handshake.indexOf(t), 1);
if (done) {
st.bytes.uploaded += t.size - t.bytes_uploaded;
var spd1 = (t.size / ((t.t2 - t.t1) / 1000.)) / (1024 * 1024.);
var spd2 = (t.size / ((t.t3 - t.t2) / 1000.)) / (1024 * 1024.);
ebi('f{0}p'.format(t.n)).innerHTML = 'hash {0}, up {1} MB/s'.format(
spd1.toFixed(2), spd2.toFixed(2));
}
else t.t3 = undefined;
tasker();
}
else {
@@ -752,12 +912,14 @@ function up2k_init(have_crypto) {
xhr.onload = function (xev) {
if (xhr.status == 200) {
prog(t.n, npart, col_uploaded);
st.bytes.uploaded += cdr - car;
t.bytes_uploaded += cdr - car;
st.busy.upload.splice(st.busy.upload.indexOf(upt), 1);
t.postlist.splice(t.postlist.indexOf(npart), 1);
if (t.postlist.length == 0) {
t.t3 = new Date().getTime();
ebi('f{0}t'.format(t.n)).innerHTML = 'verifying';
st.todo.handshake.push(t);
st.todo.handshake.unshift(t);
}
tasker();
}
@@ -845,6 +1007,28 @@ function up2k_init(have_crypto) {
bcfg_set('ask_up', ask_up);
}
function tgl_flag_en() {
flag_en = !flag_en;
bcfg_set('flag_en', flag_en);
apply_flag_cfg();
}
function apply_flag_cfg() {
if (flag_en && !flag) {
try {
flag = up2k_flagbus();
}
catch (ex) {
console.log("flag error: " + ex.toString());
tgl_flag_en();
}
}
else if (!flag_en && flag) {
flag.ch.close();
flag = false;
}
}
function nop(ev) {
ev.preventDefault();
this.click();
@@ -862,6 +1046,7 @@ function up2k_init(have_crypto) {
ebi('nthread').addEventListener('input', bumpthread, false);
ebi('multitask').addEventListener('click', tgl_multitask, false);
ebi('ask_up').addEventListener('click', tgl_ask_up, false);
ebi('flag_en').addEventListener('click', tgl_flag_en, false);
var nodes = ebi('u2conf').getElementsByTagName('a');
for (var a = nodes.length - 1; a >= 0; a--)

View File

@@ -53,11 +53,15 @@
</td>
<td rowspan="2" style="padding-left:1.5em">
<input type="checkbox" id="multitask" />
<label for="multitask">hash while<br />uploading</label>
<label for="multitask">hash<br />while<br />upping</label>
</td>
<td rowspan="2">
<input type="checkbox" id="ask_up" />
<label for="ask_up">ask for<br />confirmation</label>
<label for="ask_up">ask<br />before<br />start</label>
</td>
<td rowspan="2">
<input type="checkbox" id="flag_en" />
<label for="flag_en">only<br />one tab<br />at once</label>
</td>
</tr>
</table>

View File

@@ -2,10 +2,8 @@
# coding: utf-8
from __future__ import print_function
import io
import os
import sys
from glob import glob
from shutil import rmtree
setuptools_available = True
@@ -49,7 +47,7 @@ with open(here + "/README.md", "rb") as f:
about = {}
if not VERSION:
with open(os.path.join(here, NAME, "__version__.py"), "rb") as f:
exec(f.read().decode("utf-8").split("\n\n", 1)[1], about)
exec (f.read().decode("utf-8").split("\n\n", 1)[1], about)
else:
about["__version__"] = VERSION
@@ -116,6 +114,7 @@ args = {
"Programming Language :: Python :: 3.6",
"Programming Language :: Python :: 3.7",
"Programming Language :: Python :: 3.8",
"Programming Language :: Python :: 3.9",
"Programming Language :: Python :: Implementation :: CPython",
"Programming Language :: Python :: Implementation :: PyPy",
"Environment :: Console",