Compare commits
12 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0a3bbc4b4a | ||
|
|
855b93dcf6 | ||
|
|
89b79ba267 | ||
|
|
f5651b7d94 | ||
|
|
1881019ede | ||
|
|
caba4e974c | ||
|
|
bc3c9613bc | ||
|
|
15a3ee252e | ||
|
|
be055961ae | ||
|
|
e3031bdeec | ||
|
|
75917b9f7c | ||
|
|
910732e02c |
46
README.md
46
README.md
@@ -51,8 +51,10 @@ turn your phone or raspi into a portable file server with resumable uploads/down
|
||||
* [sfx](#sfx)
|
||||
* [sfx repack](#sfx-repack)
|
||||
* [install on android](#install-on-android)
|
||||
* [dev env setup](#dev-env-setup)
|
||||
* [how to release](#how-to-release)
|
||||
* [building](#building)
|
||||
* [dev env setup](#dev-env-setup)
|
||||
* [just the sfx](#just-the-sfx)
|
||||
* [complete release](#complete-release)
|
||||
* [todo](#todo)
|
||||
|
||||
|
||||
@@ -141,6 +143,9 @@ summary: all planned features work! now please enjoy the bloatening
|
||||
|
||||
## not my bugs
|
||||
|
||||
* Windows: folders cannot be accessed if the name ends with `.`
|
||||
* python or windows bug
|
||||
|
||||
* Windows: msys2-python 3.8.6 occasionally throws "RuntimeError: release unlocked lock" when leaving a scoped mutex in up2k
|
||||
* this is an msys2 bug, the regular windows edition of python is fine
|
||||
|
||||
@@ -437,7 +442,7 @@ quick summary of more eccentric web-browsers trying to view a directory index:
|
||||
|
||||
copyparty returns a truncated sha512sum of your PUT/POST as base64; you can generate the same checksum locally to verify uplaods:
|
||||
|
||||
b512(){ printf "$((sha512sum||shasum -a512)|sed -E 's/ .*//;s/(..)/\\x\1/g')"|base64|head -c43;}
|
||||
b512(){ printf "$((sha512sum||shasum -a512)|sed -E 's/ .*//;s/(..)/\\x\1/g')"|base64|tr '+/' '-_'|head -c44;}
|
||||
b512 <movie.mkv
|
||||
|
||||
|
||||
@@ -532,18 +537,45 @@ echo $?
|
||||
after the initial setup, you can launch copyparty at any time by running `copyparty` anywhere in Termux
|
||||
|
||||
|
||||
# dev env setup
|
||||
# building
|
||||
|
||||
## dev env setup
|
||||
|
||||
mostly optional; if you need a working env for vscode or similar
|
||||
|
||||
```sh
|
||||
python3 -m venv .venv
|
||||
. .venv/bin/activate
|
||||
pip install jinja2 # mandatory deps
|
||||
pip install Pillow # thumbnail deps
|
||||
pip install jinja2 # mandatory
|
||||
pip install mutagen # audio metadata
|
||||
pip install Pillow pyheif-pillow-opener pillow-avif-plugin # thumbnails
|
||||
pip install black bandit pylint flake8 # vscode tooling
|
||||
```
|
||||
|
||||
|
||||
# how to release
|
||||
## just the sfx
|
||||
|
||||
unless you need to modify something in the web-dependencies, it's faster to grab those from a previous release:
|
||||
|
||||
```sh
|
||||
rm -rf copyparty/web/deps
|
||||
curl -L https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.py >x.py
|
||||
python3 x.py -h
|
||||
rm x.py
|
||||
mv /tmp/pe-copyparty/copyparty/web/deps/ copyparty/web/
|
||||
```
|
||||
|
||||
then build the sfx using any of the following examples:
|
||||
|
||||
```sh
|
||||
./scripts/make-sfx.sh # both python and sh editions
|
||||
./scripts/make-sfx.sh no-sh gz # just python with gzip
|
||||
```
|
||||
|
||||
|
||||
## complete release
|
||||
|
||||
also builds the sfx so disregard the sfx section above
|
||||
|
||||
in the `scripts` folder:
|
||||
|
||||
|
||||
@@ -2,10 +2,13 @@
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import shutil
|
||||
import sqlite3
|
||||
import argparse
|
||||
|
||||
DB_VER = 3
|
||||
DB_VER1 = 3
|
||||
DB_VER2 = 4
|
||||
|
||||
|
||||
def die(msg):
|
||||
@@ -45,18 +48,21 @@ def compare(n1, d1, n2, d2, verbose):
|
||||
nt = next(d1.execute("select count(w) from up"))[0]
|
||||
n = 0
|
||||
miss = 0
|
||||
for w, rd, fn in d1.execute("select w, rd, fn from up"):
|
||||
for w1, rd, fn in d1.execute("select w, rd, fn from up"):
|
||||
n += 1
|
||||
if n % 25_000 == 0:
|
||||
m = f"\033[36mchecked {n:,} of {nt:,} files in {n1} against {n2}\033[0m"
|
||||
print(m)
|
||||
|
||||
q = "select w from up where substr(w,1,16) = ?"
|
||||
hit = d2.execute(q, (w[:16],)).fetchone()
|
||||
if rd.split("/", 1)[0] == ".hist":
|
||||
continue
|
||||
|
||||
q = "select w from up where rd = ? and fn = ?"
|
||||
hit = d2.execute(q, (rd, fn)).fetchone()
|
||||
if not hit:
|
||||
miss += 1
|
||||
if verbose:
|
||||
print(f"file in {n1} missing in {n2}: [{w}] {rd}/{fn}")
|
||||
print(f"file in {n1} missing in {n2}: [{w1}] {rd}/{fn}")
|
||||
|
||||
print(f" {miss} files in {n1} missing in {n2}\n")
|
||||
|
||||
@@ -64,13 +70,28 @@ def compare(n1, d1, n2, d2, verbose):
|
||||
n = 0
|
||||
miss = {}
|
||||
nmiss = 0
|
||||
for w, k, v in d1.execute("select * from mt"):
|
||||
for w1, k, v in d1.execute("select * from mt"):
|
||||
|
||||
n += 1
|
||||
if n % 100_000 == 0:
|
||||
m = f"\033[36mchecked {n:,} of {nt:,} tags in {n1} against {n2}, so far {nmiss} missing tags\033[0m"
|
||||
print(m)
|
||||
|
||||
v2 = d2.execute("select v from mt where w = ? and +k = ?", (w, k)).fetchone()
|
||||
q = "select rd, fn from up where substr(w,1,16) = ?"
|
||||
rd, fn = d1.execute(q, (w1,)).fetchone()
|
||||
if rd.split("/", 1)[0] == ".hist":
|
||||
continue
|
||||
|
||||
q = "select substr(w,1,16) from up where rd = ? and fn = ?"
|
||||
w2 = d2.execute(q, (rd, fn)).fetchone()
|
||||
if w2:
|
||||
w2 = w2[0]
|
||||
|
||||
v2 = None
|
||||
if w2:
|
||||
v2 = d2.execute(
|
||||
"select v from mt where w = ? and +k = ?", (w2, k)
|
||||
).fetchone()
|
||||
if v2:
|
||||
v2 = v2[0]
|
||||
|
||||
@@ -99,9 +120,7 @@ def compare(n1, d1, n2, d2, verbose):
|
||||
miss[k] = 1
|
||||
|
||||
if verbose:
|
||||
q = "select rd, fn from up where substr(w,1,16) = ?"
|
||||
rd, fn = d1.execute(q, (w,)).fetchone()
|
||||
print(f"missing in {n2}: [{w}] [{rd}/{fn}] {k} = {v}")
|
||||
print(f"missing in {n2}: [{w1}] [{rd}/{fn}] {k} = {v}")
|
||||
|
||||
for k, v in sorted(miss.items()):
|
||||
if v:
|
||||
@@ -114,24 +133,35 @@ def copy_mtp(d1, d2, tag, rm):
|
||||
nt = next(d1.execute("select count(w) from mt where k = ?", (tag,)))[0]
|
||||
n = 0
|
||||
ndone = 0
|
||||
for w, k, v in d1.execute("select * from mt where k = ?", (tag,)):
|
||||
for w1, k, v in d1.execute("select * from mt where k = ?", (tag,)):
|
||||
n += 1
|
||||
if n % 25_000 == 0:
|
||||
m = f"\033[36m{n:,} of {nt:,} tags checked, so far {ndone} copied\033[0m"
|
||||
print(m)
|
||||
|
||||
hit = d2.execute("select v from mt where w = ? and +k = ?", (w, k)).fetchone()
|
||||
q = "select rd, fn from up where substr(w,1,16) = ?"
|
||||
rd, fn = d1.execute(q, (w1,)).fetchone()
|
||||
if rd.split("/", 1)[0] == ".hist":
|
||||
continue
|
||||
|
||||
q = "select substr(w,1,16) from up where rd = ? and fn = ?"
|
||||
w2 = d2.execute(q, (rd, fn)).fetchone()
|
||||
if not w2:
|
||||
continue
|
||||
|
||||
w2 = w2[0]
|
||||
hit = d2.execute("select v from mt where w = ? and +k = ?", (w2, k)).fetchone()
|
||||
if hit:
|
||||
hit = hit[0]
|
||||
|
||||
if hit != v:
|
||||
ndone += 1
|
||||
if hit is not None:
|
||||
d2.execute("delete from mt where w = ? and +k = ?", (w, k))
|
||||
d2.execute("delete from mt where w = ? and +k = ?", (w2, k))
|
||||
|
||||
d2.execute("insert into mt values (?,?,?)", (w, k, v))
|
||||
d2.execute("insert into mt values (?,?,?)", (w2, k, v))
|
||||
if rm:
|
||||
d2.execute("delete from mt where w = ? and +k = 't:mtp'", (w,))
|
||||
d2.execute("delete from mt where w = ? and +k = 't:mtp'", (w2,))
|
||||
|
||||
d2.commit()
|
||||
print(f"copied {ndone} {tag} tags over")
|
||||
@@ -168,6 +198,23 @@ def main():
|
||||
db = sqlite3.connect(ar.db)
|
||||
ds = sqlite3.connect(ar.src) if ar.src else None
|
||||
|
||||
# revert journals
|
||||
for d, p in [[db, ar.db], [ds, ar.src]]:
|
||||
if not d:
|
||||
continue
|
||||
|
||||
pj = "{}-journal".format(p)
|
||||
if not os.path.exists(pj):
|
||||
continue
|
||||
|
||||
d.execute("create table foo (bar int)")
|
||||
d.execute("drop table foo")
|
||||
|
||||
if ar.copy:
|
||||
db.close()
|
||||
shutil.copy2(ar.db, "{}.bak.dbtool.{:x}".format(ar.db, int(time.time())))
|
||||
db = sqlite3.connect(ar.db)
|
||||
|
||||
for d, n in [[ds, "src"], [db, "dst"]]:
|
||||
if not d:
|
||||
continue
|
||||
@@ -176,8 +223,8 @@ def main():
|
||||
if ver == "corrupt":
|
||||
die("{} database appears to be corrupt, sorry")
|
||||
|
||||
if ver != DB_VER:
|
||||
m = f"{n} db is version {ver}, this tool only supports version {DB_VER}, please upgrade it with copyparty first"
|
||||
if ver < DB_VER1 or ver > DB_VER2:
|
||||
m = f"{n} db is version {ver}, this tool only supports versions between {DB_VER1} and {DB_VER2}, please upgrade it with copyparty first"
|
||||
die(m)
|
||||
|
||||
if ar.ls:
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
# coding: utf-8
|
||||
|
||||
VERSION = (0, 11, 19)
|
||||
VERSION = (0, 11, 20)
|
||||
CODENAME = "the grid"
|
||||
BUILD_DT = (2021, 6, 19)
|
||||
BUILD_DT = (2021, 6, 20)
|
||||
|
||||
S_VERSION = ".".join(map(str, VERSION))
|
||||
S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT)
|
||||
|
||||
@@ -10,7 +10,7 @@ import hashlib
|
||||
import threading
|
||||
|
||||
from .__init__ import WINDOWS
|
||||
from .util import IMPLICATIONS, undot, Pebkac, fsdec, fsenc, statdir, nuprint
|
||||
from .util import IMPLICATIONS, uncyg, undot, Pebkac, fsdec, fsenc, statdir, nuprint
|
||||
|
||||
|
||||
class VFS(object):
|
||||
@@ -439,8 +439,8 @@ class AuthSrv(object):
|
||||
raise Exception("invalid -v argument: [{}]".format(v_str))
|
||||
|
||||
src, dst, perms = m.groups()
|
||||
if WINDOWS and src.startswith("/"):
|
||||
src = "{}:\\{}".format(src[1], src[3:])
|
||||
if WINDOWS:
|
||||
src = uncyg(src)
|
||||
|
||||
# print("\n".join([src, dst, perms]))
|
||||
src = fsdec(os.path.abspath(fsenc(src)))
|
||||
@@ -469,6 +469,17 @@ class AuthSrv(object):
|
||||
print(m.format(cfg_fn, self.line_ctr))
|
||||
raise
|
||||
|
||||
# case-insensitive; normalize
|
||||
if WINDOWS:
|
||||
cased = {}
|
||||
for k, v in mount.items():
|
||||
try:
|
||||
cased[k] = fsdec(os.path.realpath(fsenc(v)))
|
||||
except:
|
||||
cased[k] = v
|
||||
|
||||
mount = cased
|
||||
|
||||
if not mount:
|
||||
# -h says our defaults are CWD at root and read/write for everyone
|
||||
vfs = VFS(os.path.abspath("."), "", ["*"], ["*"])
|
||||
@@ -524,9 +535,7 @@ class AuthSrv(object):
|
||||
if vflag == "-":
|
||||
pass
|
||||
elif vflag:
|
||||
if WINDOWS and vflag.startswith("/"):
|
||||
vflag = "{}:\\{}".format(vflag[1], vflag[3:])
|
||||
vol.histpath = vflag
|
||||
vol.histpath = uncyg(vflag) if WINDOWS else vflag
|
||||
elif self.args.hist:
|
||||
for nch in range(len(hid)):
|
||||
hpath = os.path.join(self.args.hist, hid[: nch + 1])
|
||||
|
||||
@@ -115,7 +115,7 @@ class HttpCli(object):
|
||||
try:
|
||||
self.ip = vs[n].strip()
|
||||
except:
|
||||
self.ip = vs[-1].strip()
|
||||
self.ip = vs[0].strip()
|
||||
self.log("rproxy={} oob x-fwd {}".format(self.args.rproxy, v), c=3)
|
||||
|
||||
self.log_src = self.conn.set_rproxy(self.ip)
|
||||
@@ -581,8 +581,10 @@ class HttpCli(object):
|
||||
if sub:
|
||||
try:
|
||||
dst = os.path.join(vfs.realpath, rem)
|
||||
if not os.path.isdir(fsenc(dst)):
|
||||
os.makedirs(fsenc(dst))
|
||||
except OSError as ex:
|
||||
self.log("makedirs failed [{}]".format(dst))
|
||||
if ex.errno == 13:
|
||||
raise Pebkac(500, "the server OS denied write-access")
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ import shutil
|
||||
import subprocess as sp
|
||||
|
||||
from .__init__ import PY2, WINDOWS
|
||||
from .util import fsenc, fsdec, REKOBO_LKEY
|
||||
from .util import fsenc, fsdec, uncyg, REKOBO_LKEY
|
||||
|
||||
if not PY2:
|
||||
unicode = str
|
||||
@@ -44,6 +44,9 @@ class MParser(object):
|
||||
while True:
|
||||
try:
|
||||
bp = os.path.expanduser(args)
|
||||
if WINDOWS:
|
||||
bp = uncyg(bp)
|
||||
|
||||
if os.path.exists(bp):
|
||||
self.bin = bp
|
||||
return
|
||||
|
||||
@@ -79,7 +79,14 @@ class TcpSrv(object):
|
||||
if self.args.log_conn:
|
||||
self.log("tcpsrv", "|%sC-acc1" % ("-" * 2,), c="1;30")
|
||||
|
||||
try:
|
||||
# macos throws bad-fd
|
||||
ready, _, _ = select.select(self.srv, [], [])
|
||||
except:
|
||||
ready = []
|
||||
if not self.stopping:
|
||||
raise
|
||||
|
||||
for srv in ready:
|
||||
if self.stopping:
|
||||
break
|
||||
|
||||
@@ -84,14 +84,14 @@ def thumb_path(histpath, rem, mtime, fmt):
|
||||
fn = rem
|
||||
|
||||
if rd:
|
||||
h = hashlib.sha512(fsenc(rd)).digest()[:24]
|
||||
h = hashlib.sha512(fsenc(rd)).digest()
|
||||
b64 = base64.urlsafe_b64encode(h).decode("ascii")[:24]
|
||||
rd = "{}/{}/".format(b64[:2], b64[2:4]).lower() + b64
|
||||
else:
|
||||
rd = "top"
|
||||
|
||||
# could keep original filenames but this is safer re pathlen
|
||||
h = hashlib.sha512(fsenc(fn)).digest()[:24]
|
||||
h = hashlib.sha512(fsenc(fn)).digest()
|
||||
fn = base64.urlsafe_b64encode(h).decode("ascii")[:24]
|
||||
|
||||
return "{}/th/{}/{}.{:x}.{}".format(
|
||||
|
||||
@@ -30,6 +30,7 @@ from .util import (
|
||||
s3dec,
|
||||
statdir,
|
||||
s2hms,
|
||||
min_ex,
|
||||
)
|
||||
from .mtag import MTag, MParser
|
||||
|
||||
@@ -39,6 +40,8 @@ try:
|
||||
except:
|
||||
HAVE_SQLITE3 = False
|
||||
|
||||
DB_VER = 4
|
||||
|
||||
|
||||
class Up2k(object):
|
||||
"""
|
||||
@@ -91,7 +94,7 @@ class Up2k(object):
|
||||
thr.start()
|
||||
|
||||
# static
|
||||
self.r_hash = re.compile("^[0-9a-zA-Z_-]{43}$")
|
||||
self.r_hash = re.compile("^[0-9a-zA-Z_-]{44}$")
|
||||
|
||||
if not HAVE_SQLITE3:
|
||||
self.log("could not initialize sqlite3, will use in-memory registry only")
|
||||
@@ -422,7 +425,10 @@ class Up2k(object):
|
||||
ret += self._build_dir(dbw, top, excl, abspath, nohash)
|
||||
else:
|
||||
# self.log("file: {}".format(abspath))
|
||||
rp = abspath[len(top) :].replace("\\", "/").strip("/")
|
||||
rp = abspath[len(top) + 1 :]
|
||||
if WINDOWS:
|
||||
rp = rp.replace("\\", "/").strip("/")
|
||||
|
||||
rd, fn = rp.rsplit("/", 1) if "/" in rp else ["", rp]
|
||||
sql = "select w, mt, sz from up where rd = ? and fn = ?"
|
||||
try:
|
||||
@@ -887,59 +893,31 @@ class Up2k(object):
|
||||
if not existed and ver is None:
|
||||
return self._create_db(db_path, cur)
|
||||
|
||||
orig_ver = ver
|
||||
if not ver or ver < 3:
|
||||
if ver == DB_VER:
|
||||
try:
|
||||
nfiles = next(cur.execute("select count(w) from up"))[0]
|
||||
self.log("OK: {} |{}|".format(db_path, nfiles))
|
||||
return cur
|
||||
except:
|
||||
self.log("WARN: could not list files; DB corrupt?\n" + min_ex())
|
||||
|
||||
elif ver > DB_VER:
|
||||
m = "database is version {}, this copyparty only supports versions <= {}"
|
||||
raise Exception(m.format(ver, DB_VER))
|
||||
|
||||
bak = "{}.bak.{:x}.v{}".format(db_path, int(time.time()), ver)
|
||||
db = cur.connection
|
||||
cur.close()
|
||||
db.close()
|
||||
msg = "creating new DB (old is bad); backup: {}"
|
||||
if ver:
|
||||
msg = "creating backup before upgrade: {}"
|
||||
msg = "creating new DB (too old to upgrade); backup: {}"
|
||||
|
||||
self.log(msg.format(bak))
|
||||
shutil.copy2(db_path, bak)
|
||||
cur = self._orz(db_path)
|
||||
|
||||
if ver == 1:
|
||||
cur = self._upgrade_v1(cur, db_path)
|
||||
if cur:
|
||||
ver = 2
|
||||
|
||||
if ver == 2:
|
||||
cur = self._create_v3(cur)
|
||||
ver = self._read_ver(cur) if cur else None
|
||||
|
||||
if ver == 3:
|
||||
if orig_ver != ver:
|
||||
cur.connection.commit()
|
||||
cur.execute("vacuum")
|
||||
cur.connection.commit()
|
||||
|
||||
try:
|
||||
nfiles = next(cur.execute("select count(w) from up"))[0]
|
||||
self.log("OK: {} |{}|".format(db_path, nfiles))
|
||||
return cur
|
||||
except Exception as ex:
|
||||
self.log("WARN: could not list files, DB corrupt?\n " + repr(ex))
|
||||
|
||||
if cur:
|
||||
db = cur.connection
|
||||
cur.close()
|
||||
db.close()
|
||||
os.rename(fsenc(db_path), fsenc(bak))
|
||||
|
||||
return self._create_db(db_path, None)
|
||||
|
||||
def _create_db(self, db_path, cur):
|
||||
if not cur:
|
||||
cur = self._orz(db_path)
|
||||
|
||||
self._create_v2(cur)
|
||||
self._create_v3(cur)
|
||||
cur.connection.commit()
|
||||
self.log("created DB at {}".format(db_path))
|
||||
return cur
|
||||
|
||||
def _read_ver(self, cur):
|
||||
for tab in ["ki", "kv"]:
|
||||
try:
|
||||
@@ -951,65 +929,38 @@ class Up2k(object):
|
||||
if rows:
|
||||
return int(rows[0][0])
|
||||
|
||||
def _create_v2(self, cur):
|
||||
for cmd in [
|
||||
r"create table up (w text, mt int, sz int, rd text, fn text)",
|
||||
r"create index up_rd on up(rd)",
|
||||
r"create index up_fn on up(fn)",
|
||||
]:
|
||||
cur.execute(cmd)
|
||||
return cur
|
||||
|
||||
def _create_v3(self, cur):
|
||||
def _create_db(self, db_path, cur):
|
||||
"""
|
||||
collision in 2^(n/2) files where n = bits (6 bits/ch)
|
||||
10*6/2 = 2^30 = 1'073'741'824, 24.1mb idx 1<<(3*10)
|
||||
12*6/2 = 2^36 = 68'719'476'736, 24.8mb idx
|
||||
16*6/2 = 2^48 = 281'474'976'710'656, 26.1mb idx
|
||||
"""
|
||||
for c, ks in [["drop table k", "isv"], ["drop index up_", "w"]]:
|
||||
for k in ks:
|
||||
try:
|
||||
cur.execute(c + k)
|
||||
except:
|
||||
pass
|
||||
if not cur:
|
||||
cur = self._orz(db_path)
|
||||
|
||||
idx = r"create index up_w on up(substr(w,1,16))"
|
||||
if self.no_expr_idx:
|
||||
idx = r"create index up_w on up(w)"
|
||||
|
||||
for cmd in [
|
||||
r"create table up (w text, mt int, sz int, rd text, fn text)",
|
||||
r"create index up_rd on up(rd)",
|
||||
r"create index up_fn on up(fn)",
|
||||
idx,
|
||||
r"create table mt (w text, k text, v int)",
|
||||
r"create index mt_w on mt(w)",
|
||||
r"create index mt_k on mt(k)",
|
||||
r"create index mt_v on mt(v)",
|
||||
r"create table kv (k text, v int)",
|
||||
r"insert into kv values ('sver', 3)",
|
||||
r"insert into kv values ('sver', {})".format(DB_VER),
|
||||
]:
|
||||
cur.execute(cmd)
|
||||
|
||||
cur.connection.commit()
|
||||
self.log("created DB at {}".format(db_path))
|
||||
return cur
|
||||
|
||||
def _upgrade_v1(self, odb, db_path):
|
||||
npath = db_path + ".next"
|
||||
if os.path.exists(npath):
|
||||
os.unlink(npath)
|
||||
|
||||
ndb = self._orz(npath)
|
||||
self._create_v2(ndb)
|
||||
|
||||
c = odb.execute("select * from up")
|
||||
for wark, ts, sz, rp in c:
|
||||
rd, fn = rp.rsplit("/", 1) if "/" in rp else ["", rp]
|
||||
v = (wark, ts, sz, rd, fn)
|
||||
ndb.execute("insert into up values (?,?,?,?,?)", v)
|
||||
|
||||
ndb.connection.commit()
|
||||
ndb.connection.close()
|
||||
odb.connection.close()
|
||||
atomic_move(npath, db_path)
|
||||
return self._orz(db_path)
|
||||
|
||||
def handle_json(self, cj):
|
||||
with self.mutex:
|
||||
if not self.register_vpath(cj["ptop"], cj["vcfg"]):
|
||||
@@ -1316,9 +1267,9 @@ class Up2k(object):
|
||||
hashobj.update(buf)
|
||||
rem -= len(buf)
|
||||
|
||||
digest = hashobj.digest()[:32]
|
||||
digest = hashobj.digest()[:33]
|
||||
digest = base64.urlsafe_b64encode(digest)
|
||||
ret.append(digest.decode("utf-8").rstrip("="))
|
||||
ret.append(digest.decode("utf-8"))
|
||||
|
||||
return ret
|
||||
|
||||
@@ -1518,12 +1469,12 @@ def up2k_wark_from_hashlist(salt, filesize, hashes):
|
||||
ident.extend(hashes)
|
||||
ident = "\n".join(ident)
|
||||
|
||||
wark = hashlib.sha512(ident.encode("utf-8")).digest()
|
||||
wark = hashlib.sha512(ident.encode("utf-8")).digest()[:33]
|
||||
wark = base64.urlsafe_b64encode(wark)
|
||||
return wark.decode("ascii")[:43]
|
||||
return wark.decode("ascii")
|
||||
|
||||
|
||||
def up2k_wark_from_metadata(salt, sz, lastmod, rd, fn):
|
||||
ret = fsenc("{}\n{}\n{}\n{}\n{}".format(salt, lastmod, sz, rd, fn))
|
||||
ret = base64.urlsafe_b64encode(hashlib.sha512(ret).digest())
|
||||
return "#{}".format(ret[:42].decode("ascii"))
|
||||
return "#{}".format(ret.decode("ascii"))[:44]
|
||||
|
||||
@@ -351,7 +351,7 @@ def ren_open(fname, *args, **kwargs):
|
||||
if not b64:
|
||||
b64 = (bname + ext).encode("utf-8", "replace")
|
||||
b64 = hashlib.sha512(b64).digest()[:12]
|
||||
b64 = base64.urlsafe_b64encode(b64).decode("utf-8").rstrip("=")
|
||||
b64 = base64.urlsafe_b64encode(b64).decode("utf-8")
|
||||
|
||||
badlen = len(fname)
|
||||
while len(fname) >= badlen:
|
||||
@@ -648,6 +648,16 @@ def s2hms(s, optional_h=False):
|
||||
return "{}:{:02}:{:02}".format(h, m, s)
|
||||
|
||||
|
||||
def uncyg(path):
|
||||
if len(path) < 2 or not path.startswith("/"):
|
||||
return path
|
||||
|
||||
if len(path) > 2 and path[2] != "/":
|
||||
return path
|
||||
|
||||
return "{}:\\{}".format(path[1], path[3:])
|
||||
|
||||
|
||||
def undot(path):
|
||||
ret = []
|
||||
for node in path.split("/"):
|
||||
@@ -908,8 +918,8 @@ def hashcopy(actor, fin, fout):
|
||||
hashobj.update(buf)
|
||||
fout.write(buf)
|
||||
|
||||
digest32 = hashobj.digest()[:32]
|
||||
digest_b64 = base64.urlsafe_b64encode(digest32).decode("utf-8").rstrip("=")
|
||||
digest = hashobj.digest()[:33]
|
||||
digest_b64 = base64.urlsafe_b64encode(digest).decode("utf-8")
|
||||
|
||||
return tlen, hashobj.hexdigest(), digest_b64
|
||||
|
||||
|
||||
@@ -969,6 +969,9 @@ html.light #treeul a.hl {
|
||||
background: #07a;
|
||||
color: #fff;
|
||||
}
|
||||
html.light #treeul a.hl:hover {
|
||||
background: #059;
|
||||
}
|
||||
html.light #tree li {
|
||||
border-color: #f7f7f7 #fff #ddd #fff;
|
||||
}
|
||||
|
||||
@@ -419,6 +419,9 @@ var widget = (function () {
|
||||
m = ck + 'np: ';
|
||||
|
||||
for (var a = 1, aa = th.length; a < aa; a++) {
|
||||
if (th[a].className.indexOf('min') !== -1)
|
||||
continue;
|
||||
|
||||
var tv = tr[a].textContent,
|
||||
tk = a == 1 ? '' : th[a].getAttribute('name').split('/').slice(-1)[0];
|
||||
|
||||
|
||||
@@ -970,8 +970,8 @@ function up2k_init(subtle) {
|
||||
while (segm_next());
|
||||
|
||||
var hash_done = function (hashbuf) {
|
||||
var hslice = new Uint8Array(hashbuf).subarray(0, 32),
|
||||
b64str = buf2b64(hslice).replace(/=$/, '');
|
||||
var hslice = new Uint8Array(hashbuf).subarray(0, 33),
|
||||
b64str = buf2b64(hslice);
|
||||
|
||||
hashtab[nch] = b64str;
|
||||
t.hash.push(nch);
|
||||
@@ -996,6 +996,7 @@ function up2k_init(subtle) {
|
||||
pvis.seth(t.n, 1, '📦 wait');
|
||||
st.busy.hash.splice(st.busy.hash.indexOf(t), 1);
|
||||
st.todo.handshake.push(t);
|
||||
tasker();
|
||||
};
|
||||
|
||||
if (subtle)
|
||||
@@ -1041,6 +1042,7 @@ function up2k_init(subtle) {
|
||||
console.log('handshake onerror, retrying');
|
||||
st.busy.handshake.splice(st.busy.handshake.indexOf(t), 1);
|
||||
st.todo.handshake.unshift(t);
|
||||
tasker();
|
||||
};
|
||||
xhr.onload = function (e) {
|
||||
if (t.busied != me) {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
FROM alpine:3.13
|
||||
WORKDIR /z
|
||||
ENV ver_asmcrypto=5b994303a9d3e27e0915f72a10b6c2c51535a4dc \
|
||||
ver_hashwasm=4.7.0 \
|
||||
ver_marked=1.1.0 \
|
||||
ver_ogvjs=1.8.0 \
|
||||
ver_mde=2.14.0 \
|
||||
@@ -9,12 +10,6 @@ ENV ver_asmcrypto=5b994303a9d3e27e0915f72a10b6c2c51535a4dc \
|
||||
ver_zopfli=1.0.3
|
||||
|
||||
|
||||
# TODO
|
||||
# sha512.hw.js https://github.com/Daninet/hash-wasm
|
||||
# sha512.kc.js https://github.com/chm-diederichs/sha3-wasm
|
||||
# awk '/HMAC state/{o=1} /var HEAP/{o=0} /function hmac_reset/{o=1} /return \{/{o=0} /var __extends =/{o=1} /var Hash =/{o=0} /hmac_|pbkdf2_/{next} o{next} {gsub(/IllegalStateError/,"Exception")} {sub(/^ +/,"");sub(/^\/\/ .*/,"");sub(/;$/," ;")} 1' <sha512.ac.js.orig >sha512.ac.js; for fn in sha512.ac.js.orig sha512.ac.js; do wc -c <$fn; wc -c <$fn.gz ; for n in {1..9}; do printf '%8d %d bz\n' $(bzip2 -c$n <$fn | wc -c) $n; done; done
|
||||
|
||||
|
||||
# download;
|
||||
# the scp url is latin from https://fonts.googleapis.com/css2?family=Source+Code+Pro&display=swap
|
||||
RUN mkdir -p /z/dist/no-pk \
|
||||
@@ -27,7 +22,11 @@ RUN mkdir -p /z/dist/no-pk \
|
||||
&& wget https://github.com/codemirror/CodeMirror/archive/$ver_codemirror.tar.gz -O codemirror.tgz \
|
||||
&& wget https://github.com/FortAwesome/Font-Awesome/releases/download/$ver_fontawesome/fontawesome-free-$ver_fontawesome-web.zip -O fontawesome.zip \
|
||||
&& wget https://github.com/google/zopfli/archive/zopfli-$ver_zopfli.tar.gz -O zopfli.tgz \
|
||||
&& wget https://github.com/Daninet/hash-wasm/releases/download/v$ver_hashwasm/hash-wasm@$ver_hashwasm.zip -O hash-wasm.zip \
|
||||
&& unzip ogvjs.zip \
|
||||
&& (mkdir hash-wasm \
|
||||
&& cd hash-wasm \
|
||||
&& unzip ../hash-wasm.zip) \
|
||||
&& (tar -xf asmcrypto.tgz \
|
||||
&& cd asmcrypto.js-$ver_asmcrypto \
|
||||
&& npm install ) \
|
||||
@@ -64,7 +63,12 @@ RUN tar -xf zopfli.tgz \
|
||||
RUN cd asmcrypto.js-$ver_asmcrypto \
|
||||
&& echo "export { Sha512 } from './hash/sha512/sha512';" > src/entry-export_all.ts \
|
||||
&& node -r esm build.js \
|
||||
&& mv asmcrypto.all.es5.js /z/dist/sha512.js
|
||||
&& awk '/HMAC state/{o=1} /var HEAP/{o=0} /function hmac_reset/{o=1} /return \{/{o=0} /var __extends =/{o=1} /var Hash =/{o=0} /hmac_|pbkdf2_/{next} o{next} {gsub(/IllegalStateError/,"Exception")} {sub(/^ +/,"");sub(/^\/\/ .*/,"");sub(/;$/," ;")} 1' < asmcrypto.all.es5.js > /z/dist/sha512.ac.js
|
||||
|
||||
|
||||
# build hash-wasm
|
||||
RUN cd hash-wasm \
|
||||
&& mv sha512.umd.min.js /z/dist/sha512.hw.js
|
||||
|
||||
|
||||
# build ogvjs
|
||||
|
||||
Reference in New Issue
Block a user