Compare commits

...

12 Commits

Author SHA1 Message Date
ed
e9ae9782fe v0.11.39 2021-07-13 00:54:23 +02:00
ed
016dba4ca9 v0.11.38 2021-07-13 00:35:34 +02:00
ed
39c7ef305f add a link to clear settings on the js crash page 2021-07-13 00:33:46 +02:00
ed
849c1dc848 video-player: add hotkeys m=mute, f=fullscreen 2021-07-13 00:23:48 +02:00
ed
61414014fe gallery: fix link overlapping image 2021-07-13 00:14:06 +02:00
ed
578a915884 stack/thread monitors in mpw + better thread names 2021-07-12 23:03:52 +02:00
ed
eacafb8a63 add option to log summary of running threads 2021-07-12 22:57:37 +02:00
ed
4446760f74 fix link to ?stack on rootless configs 2021-07-12 22:55:38 +02:00
ed
6da2a083f9 v0.11.37 2021-07-12 00:51:59 +02:00
ed
8837c8f822 print zip/tar errors to log 2021-07-12 00:47:22 +02:00
ed
bac301ed66 get rid of iffy default-args 2021-07-12 00:15:13 +02:00
ed
061db3906d v0.11.36 2021-07-11 06:39:58 +02:00
24 changed files with 180 additions and 101 deletions

View File

@@ -345,7 +345,7 @@ class Gateway(object):
except: except:
pass pass
def sendreq(self, *args, headers={}, **kwargs): def sendreq(self, meth, path, headers, **kwargs):
if self.password: if self.password:
headers["Cookie"] = "=".join(["cppwd", self.password]) headers["Cookie"] = "=".join(["cppwd", self.password])
@@ -354,21 +354,21 @@ class Gateway(object):
if c.rx_path: if c.rx_path:
raise Exception() raise Exception()
c.request(*list(args), headers=headers, **kwargs) c.request(meth, path, headers=headers, **kwargs)
c.rx = c.getresponse() c.rx = c.getresponse()
return c return c
except: except:
tid = threading.current_thread().ident tid = threading.current_thread().ident
dbg( dbg(
"\033[1;37;44mbad conn {:x}\n {}\n {}\033[0m".format( "\033[1;37;44mbad conn {:x}\n {} {}\n {}\033[0m".format(
tid, " ".join(str(x) for x in args), c.rx_path if c else "(null)" tid, meth, path, c.rx_path if c else "(null)"
) )
) )
self.closeconn(c) self.closeconn(c)
c = self.getconn() c = self.getconn()
try: try:
c.request(*list(args), headers=headers, **kwargs) c.request(meth, path, headers=headers, **kwargs)
c.rx = c.getresponse() c.rx = c.getresponse()
return c return c
except: except:
@@ -386,7 +386,7 @@ class Gateway(object):
path = dewin(path) path = dewin(path)
web_path = self.quotep("/" + "/".join([self.web_root, path])) + "?dots" web_path = self.quotep("/" + "/".join([self.web_root, path])) + "?dots"
c = self.sendreq("GET", web_path) c = self.sendreq("GET", web_path, {})
if c.rx.status != 200: if c.rx.status != 200:
self.closeconn(c) self.closeconn(c)
log( log(
@@ -440,7 +440,7 @@ class Gateway(object):
) )
) )
c = self.sendreq("GET", web_path, headers={"Range": hdr_range}) c = self.sendreq("GET", web_path, {"Range": hdr_range})
if c.rx.status != http.client.PARTIAL_CONTENT: if c.rx.status != http.client.PARTIAL_CONTENT:
self.closeconn(c) self.closeconn(c)
raise Exception( raise Exception(

View File

@@ -54,10 +54,13 @@ MACOS = platform.system() == "Darwin"
info = log = dbg = None info = log = dbg = None
print("{} v{} @ {}".format( print(
platform.python_implementation(), "{} v{} @ {}".format(
".".join([str(x) for x in sys.version_info]), platform.python_implementation(),
sys.executable)) ".".join([str(x) for x in sys.version_info]),
sys.executable,
)
)
try: try:
@@ -299,14 +302,14 @@ class Gateway(object):
except: except:
pass pass
def sendreq(self, *args, headers={}, **kwargs): def sendreq(self, meth, path, headers, **kwargs):
tid = get_tid() tid = get_tid()
if self.password: if self.password:
headers["Cookie"] = "=".join(["cppwd", self.password]) headers["Cookie"] = "=".join(["cppwd", self.password])
try: try:
c = self.getconn(tid) c = self.getconn(tid)
c.request(*list(args), headers=headers, **kwargs) c.request(meth, path, headers=headers, **kwargs)
return c.getresponse() return c.getresponse()
except: except:
dbg("bad conn") dbg("bad conn")
@@ -314,7 +317,7 @@ class Gateway(object):
self.closeconn(tid) self.closeconn(tid)
try: try:
c = self.getconn(tid) c = self.getconn(tid)
c.request(*list(args), headers=headers, **kwargs) c.request(meth, path, headers=headers, **kwargs)
return c.getresponse() return c.getresponse()
except: except:
info("http connection failed:\n" + traceback.format_exc()) info("http connection failed:\n" + traceback.format_exc())
@@ -331,7 +334,7 @@ class Gateway(object):
path = dewin(path) path = dewin(path)
web_path = self.quotep("/" + "/".join([self.web_root, path])) + "?dots&ls" web_path = self.quotep("/" + "/".join([self.web_root, path])) + "?dots&ls"
r = self.sendreq("GET", web_path) r = self.sendreq("GET", web_path, {})
if r.status != 200: if r.status != 200:
self.closeconn() self.closeconn()
log( log(
@@ -368,7 +371,7 @@ class Gateway(object):
) )
) )
r = self.sendreq("GET", web_path, headers={"Range": hdr_range}) r = self.sendreq("GET", web_path, {"Range": hdr_range})
if r.status != http.client.PARTIAL_CONTENT: if r.status != http.client.PARTIAL_CONTENT:
self.closeconn() self.closeconn()
raise Exception( raise Exception(

View File

@@ -23,7 +23,7 @@ from textwrap import dedent
from .__init__ import E, WINDOWS, VT100, PY2, unicode from .__init__ import E, WINDOWS, VT100, PY2, unicode
from .__version__ import S_VERSION, S_BUILD_DT, CODENAME from .__version__ import S_VERSION, S_BUILD_DT, CODENAME
from .svchub import SvcHub from .svchub import SvcHub
from .util import py_desc, align_tab, IMPLICATIONS, alltrace from .util import py_desc, align_tab, IMPLICATIONS
HAVE_SSL = True HAVE_SSL = True
try: try:
@@ -191,16 +191,6 @@ def sighandler(sig=None, frame=None):
print("\n".join(msg)) print("\n".join(msg))
def stackmon(fp, ival):
ctr = 0
while True:
ctr += 1
time.sleep(ival)
st = "{}, {}\n{}".format(ctr, time.time(), alltrace())
with open(fp, "wb") as f:
f.write(st.encode("utf-8", "replace"))
def run_argparse(argv, formatter): def run_argparse(argv, formatter):
ap = argparse.ArgumentParser( ap = argparse.ArgumentParser(
formatter_class=formatter, formatter_class=formatter,
@@ -346,6 +336,7 @@ def run_argparse(argv, formatter):
ap2.add_argument("--no-fastboot", action="store_true", help="wait for up2k indexing") ap2.add_argument("--no-fastboot", action="store_true", help="wait for up2k indexing")
ap2.add_argument("--no-htp", action="store_true", help="disable httpserver threadpool, create threads as-needed instead") ap2.add_argument("--no-htp", action="store_true", help="disable httpserver threadpool, create threads as-needed instead")
ap2.add_argument("--stackmon", metavar="P,S", type=u, help="write stacktrace to Path every S second") ap2.add_argument("--stackmon", metavar="P,S", type=u, help="write stacktrace to Path every S second")
ap2.add_argument("--log-thrs", metavar="SEC", type=float, help="list active threads every SEC")
return ap.parse_args(args=argv[1:]) return ap.parse_args(args=argv[1:])
# fmt: on # fmt: on
@@ -385,16 +376,6 @@ def main(argv=None):
except AssertionError: except AssertionError:
al = run_argparse(argv, Dodge11874) al = run_argparse(argv, Dodge11874)
if al.stackmon:
fp, f = al.stackmon.rsplit(",", 1)
f = int(f)
t = threading.Thread(
target=stackmon,
args=(fp, f),
)
t.daemon = True
t.start()
# propagate implications # propagate implications
for k1, k2 in IMPLICATIONS: for k1, k2 in IMPLICATIONS:
if getattr(al, k1): if getattr(al, k1):

View File

@@ -1,8 +1,8 @@
# coding: utf-8 # coding: utf-8
VERSION = (0, 11, 35) VERSION = (0, 11, 39)
CODENAME = "the grid" CODENAME = "the grid"
BUILD_DT = (2021, 7, 11) BUILD_DT = (2021, 7, 13)
S_VERSION = ".".join(map(str, VERSION)) S_VERSION = ".".join(map(str, VERSION))
S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT) S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT)

View File

@@ -16,7 +16,7 @@ from .util import IMPLICATIONS, uncyg, undot, Pebkac, fsdec, fsenc, statdir
class VFS(object): class VFS(object):
"""single level in the virtual fs""" """single level in the virtual fs"""
def __init__(self, log, realpath, vpath, uread=[], uwrite=[], uadm=[], flags={}): def __init__(self, log, realpath, vpath, uread, uwrite, uadm, flags):
self.log = log self.log = log
self.realpath = realpath # absolute path on host filesystem self.realpath = realpath # absolute path on host filesystem
self.vpath = vpath # absolute path in the virtual filesystem self.vpath = vpath # absolute path in the virtual filesystem
@@ -81,7 +81,7 @@ class VFS(object):
# leaf does not exist; create and keep permissions blank # leaf does not exist; create and keep permissions blank
vp = "{}/{}".format(self.vpath, dst).lstrip("/") vp = "{}/{}".format(self.vpath, dst).lstrip("/")
vn = VFS(self.log, src, vp) vn = VFS(self.log, src, vp, [], [], [], {})
vn.dbv = self.dbv or self vn.dbv = self.dbv or self
self.nodes[dst] = vn self.nodes[dst] = vn
return vn return vn
@@ -497,10 +497,10 @@ class AuthSrv(object):
if not mount: if not mount:
# -h says our defaults are CWD at root and read/write for everyone # -h says our defaults are CWD at root and read/write for everyone
vfs = VFS(self.log_func, os.path.abspath("."), "", ["*"], ["*"]) vfs = VFS(self.log_func, os.path.abspath("."), "", ["*"], ["*"], ["*"], {})
elif "" not in mount: elif "" not in mount:
# there's volumes but no root; make root inaccessible # there's volumes but no root; make root inaccessible
vfs = VFS(self.log_func, None, "") vfs = VFS(self.log_func, None, "", [], [], [], {})
vfs.flags["d2d"] = True vfs.flags["d2d"] = True
maxdepth = 0 maxdepth = 0

View File

@@ -27,18 +27,17 @@ class BrokerMp(object):
cores = mp.cpu_count() cores = mp.cpu_count()
self.log("broker", "booting {} subprocesses".format(cores)) self.log("broker", "booting {} subprocesses".format(cores))
for n in range(cores): for n in range(1, cores + 1):
q_pend = mp.Queue(1) q_pend = mp.Queue(1)
q_yield = mp.Queue(64) q_yield = mp.Queue(64)
proc = mp.Process(target=MpWorker, args=(q_pend, q_yield, self.args, n)) proc = mp.Process(target=MpWorker, args=(q_pend, q_yield, self.args, n))
proc.q_pend = q_pend proc.q_pend = q_pend
proc.q_yield = q_yield proc.q_yield = q_yield
proc.nid = n
proc.clients = {} proc.clients = {}
thr = threading.Thread( thr = threading.Thread(
target=self.collector, args=(proc,), name="mp-collector" target=self.collector, args=(proc,), name="mp-sink-{}".format(n)
) )
thr.daemon = True thr.daemon = True
thr.start() thr.start()

View File

@@ -35,7 +35,7 @@ class MpWorker(object):
self.asrv = AuthSrv(args, None, False) self.asrv = AuthSrv(args, None, False)
# instantiate all services here (TODO: inheritance?) # instantiate all services here (TODO: inheritance?)
self.httpsrv = HttpSrv(self, True) self.httpsrv = HttpSrv(self, n)
# on winxp and some other platforms, # on winxp and some other platforms,
# use thr.join() to block all signals # use thr.join() to block all signals

View File

@@ -19,7 +19,7 @@ class BrokerThr(object):
self.mutex = threading.Lock() self.mutex = threading.Lock()
# instantiate all services here (TODO: inheritance?) # instantiate all services here (TODO: inheritance?)
self.httpsrv = HttpSrv(self) self.httpsrv = HttpSrv(self, None)
def shutdown(self): def shutdown(self):
# self.log("broker", "shutting down") # self.log("broker", "shutting down")

View File

@@ -37,7 +37,6 @@ class HttpCli(object):
self.ip = conn.addr[0] self.ip = conn.addr[0]
self.addr = conn.addr # type: tuple[str, int] self.addr = conn.addr # type: tuple[str, int]
self.args = conn.args self.args = conn.args
self.is_mp = conn.is_mp
self.asrv = conn.asrv # type: AuthSrv self.asrv = conn.asrv # type: AuthSrv
self.ico = conn.ico self.ico = conn.ico
self.thumbcli = conn.thumbcli self.thumbcli = conn.thumbcli
@@ -227,7 +226,7 @@ class HttpCli(object):
except Pebkac: except Pebkac:
return False return False
def send_headers(self, length, status=200, mime=None, headers={}): def send_headers(self, length, status=200, mime=None, headers=None):
response = ["{} {} {}".format(self.http_ver, status, HTTPCODE[status])] response = ["{} {} {}".format(self.http_ver, status, HTTPCODE[status])]
if length is not None: if length is not None:
@@ -237,7 +236,8 @@ class HttpCli(object):
response.append("Connection: " + ("Keep-Alive" if self.keepalive else "Close")) response.append("Connection: " + ("Keep-Alive" if self.keepalive else "Close"))
# headers{} overrides anything set previously # headers{} overrides anything set previously
self.out_headers.update(headers) if headers:
self.out_headers.update(headers)
# default to utf8 html if no content-type is set # default to utf8 html if no content-type is set
if not mime: if not mime:
@@ -254,7 +254,7 @@ class HttpCli(object):
except: except:
raise Pebkac(400, "client d/c while replying headers") raise Pebkac(400, "client d/c while replying headers")
def reply(self, body, status=200, mime=None, headers={}): def reply(self, body, status=200, mime=None, headers=None):
# TODO something to reply with user-supplied values safely # TODO something to reply with user-supplied values safely
self.send_headers(len(body), status, mime, headers) self.send_headers(len(body), status, mime, headers)
@@ -270,7 +270,7 @@ class HttpCli(object):
self.log(body.rstrip()) self.log(body.rstrip())
self.reply(b"<pre>" + body.encode("utf-8") + b"\r\n", *list(args), **kwargs) self.reply(b"<pre>" + body.encode("utf-8") + b"\r\n", *list(args), **kwargs)
def urlq(self, add={}, rm=[]): def urlq(self, add, rm):
""" """
generates url query based on uparam (b, pw, all others) generates url query based on uparam (b, pw, all others)
removing anything in rm, adding pairs in add removing anything in rm, adding pairs in add
@@ -342,6 +342,9 @@ class HttpCli(object):
if "tree" in self.uparam: if "tree" in self.uparam:
return self.tx_tree() return self.tx_tree()
if "stack" in self.uparam:
return self.tx_stack()
# conditional redirect to single volumes # conditional redirect to single volumes
if self.vpath == "" and not self.ouparam: if self.vpath == "" and not self.ouparam:
nread = len(self.rvol) nread = len(self.rvol)
@@ -371,9 +374,6 @@ class HttpCli(object):
if "scan" in self.uparam: if "scan" in self.uparam:
return self.scanvol() return self.scanvol()
if "stack" in self.uparam:
return self.tx_stack()
return self.tx_browser() return self.tx_browser()
def handle_options(self): def handle_options(self):
@@ -795,7 +795,7 @@ class HttpCli(object):
vfs, rem = self.asrv.vfs.get(self.vpath, self.uname, False, True) vfs, rem = self.asrv.vfs.get(self.vpath, self.uname, False, True)
self._assert_safe_rem(rem) self._assert_safe_rem(rem)
sanitized = sanitize_fn(new_dir) sanitized = sanitize_fn(new_dir, "", [])
if not nullwrite: if not nullwrite:
fdir = os.path.join(vfs.realpath, rem) fdir = os.path.join(vfs.realpath, rem)
@@ -832,7 +832,7 @@ class HttpCli(object):
if not new_file.endswith(".md"): if not new_file.endswith(".md"):
new_file += ".md" new_file += ".md"
sanitized = sanitize_fn(new_file) sanitized = sanitize_fn(new_file, "", [])
if not nullwrite: if not nullwrite:
fdir = os.path.join(vfs.realpath, rem) fdir = os.path.join(vfs.realpath, rem)
@@ -865,7 +865,7 @@ class HttpCli(object):
if p_file and not nullwrite: if p_file and not nullwrite:
fdir = os.path.join(vfs.realpath, rem) fdir = os.path.join(vfs.realpath, rem)
fname = sanitize_fn( fname = sanitize_fn(
p_file, bad=[".prologue.html", ".epilogue.html"] p_file, "", [".prologue.html", ".epilogue.html"]
) )
if not os.path.isdir(fsenc(fdir)): if not os.path.isdir(fsenc(fdir)):
@@ -1312,7 +1312,7 @@ class HttpCli(object):
fgen = vn.zipgen(rem, items, self.uname, dots, not self.args.no_scandir) fgen = vn.zipgen(rem, items, self.uname, dots, not self.args.no_scandir)
# for f in fgen: print(repr({k: f[k] for k in ["vp", "ap"]})) # for f in fgen: print(repr({k: f[k] for k in ["vp", "ap"]}))
bgen = packer(fgen, utf8="utf" in uarg, pre_crc="crc" in uarg) bgen = packer(self.log, fgen, utf8="utf" in uarg, pre_crc="crc" in uarg)
bsent = 0 bsent = 0
for buf in bgen.gen(): for buf in bgen.gen():
if not buf: if not buf:
@@ -1423,7 +1423,7 @@ class HttpCli(object):
return True return True
def tx_mounts(self): def tx_mounts(self):
suf = self.urlq(rm=["h"]) suf = self.urlq({}, ["h"])
rvol, wvol, avol = [ rvol, wvol, avol = [
[("/" + x).rstrip("/") + "/" for x in y] [("/" + x).rstrip("/") + "/" for x in y]
for y in [self.rvol, self.wvol, self.avol] for y in [self.rvol, self.wvol, self.avol]
@@ -1634,7 +1634,7 @@ class HttpCli(object):
if self.writable: if self.writable:
perms.append("write") perms.append("write")
url_suf = self.urlq() url_suf = self.urlq({}, [])
is_ls = "ls" in self.uparam is_ls = "ls" in self.uparam
tpl = "browser" tpl = "browser"

View File

@@ -34,7 +34,6 @@ class HttpConn(object):
self.args = hsrv.args self.args = hsrv.args
self.asrv = hsrv.asrv self.asrv = hsrv.asrv
self.is_mp = hsrv.is_mp
self.cert_path = hsrv.cert_path self.cert_path = hsrv.cert_path
enth = HAVE_PIL and not self.args.no_thumb enth = HAVE_PIL and not self.args.no_thumb

View File

@@ -27,7 +27,7 @@ except ImportError:
sys.exit(1) sys.exit(1)
from .__init__ import E, PY2, MACOS from .__init__ import E, PY2, MACOS
from .util import spack, min_ex from .util import spack, min_ex, start_stackmon, start_log_thrs
from .httpconn import HttpConn from .httpconn import HttpConn
if PY2: if PY2:
@@ -42,14 +42,14 @@ class HttpSrv(object):
relying on MpSrv for performance (HttpSrv is just plain threads) relying on MpSrv for performance (HttpSrv is just plain threads)
""" """
def __init__(self, broker, is_mp=False): def __init__(self, broker, nid):
self.broker = broker self.broker = broker
self.is_mp = is_mp self.nid = nid
self.args = broker.args self.args = broker.args
self.log = broker.log self.log = broker.log
self.asrv = broker.asrv self.asrv = broker.asrv
self.name = "httpsrv-i{:x}".format(os.getpid()) self.name = "httpsrv" + ("-n{}-i{:x}".format(nid, os.getpid()) if nid else "")
self.mutex = threading.Lock() self.mutex = threading.Lock()
self.stopping = False self.stopping = False
@@ -81,10 +81,18 @@ class HttpSrv(object):
if self.tp_q: if self.tp_q:
self.start_threads(4) self.start_threads(4)
t = threading.Thread(target=self.thr_scaler) name = "httpsrv-scaler" + ("-{}".format(nid) if nid else "")
t = threading.Thread(target=self.thr_scaler, name=name)
t.daemon = True t.daemon = True
t.start() t.start()
if nid:
if self.args.stackmon:
start_stackmon(self.args.stackmon, nid)
if self.args.log_thrs:
start_log_thrs(self.log, self.args.log_thrs, nid)
def start_threads(self, n): def start_threads(self, n):
self.tp_nthr += n self.tp_nthr += n
if self.args.log_htp: if self.args.log_htp:
@@ -93,7 +101,7 @@ class HttpSrv(object):
for _ in range(n): for _ in range(n):
thr = threading.Thread( thr = threading.Thread(
target=self.thr_poolw, target=self.thr_poolw,
name="httpsrv-poolw", name=self.name + "-poolw",
) )
thr.daemon = True thr.daemon = True
thr.start() thr.start()
@@ -115,9 +123,14 @@ class HttpSrv(object):
self.stop_threads(4) self.stop_threads(4)
def listen(self, sck, nlisteners): def listen(self, sck, nlisteners):
ip, port = sck.getsockname()
self.srvs.append(sck) self.srvs.append(sck)
self.nclimax = math.ceil(self.args.nc * 1.0 / nlisteners) self.nclimax = math.ceil(self.args.nc * 1.0 / nlisteners)
t = threading.Thread(target=self.thr_listen, args=(sck,)) t = threading.Thread(
target=self.thr_listen,
args=(sck,),
name="httpsrv-n{}-listen-{}-{}".format(self.nid or "0", ip, port),
)
t.daemon = True t.daemon = True
t.start() t.start()
@@ -158,7 +171,7 @@ class HttpSrv(object):
"""takes an incoming tcp connection and creates a thread to handle it""" """takes an incoming tcp connection and creates a thread to handle it"""
now = time.time() now = time.time()
if self.tp_time and now - self.tp_time > 300: if now - (self.tp_time or now) > 300:
self.tp_q = None self.tp_q = None
if self.tp_q: if self.tp_q:
@@ -181,7 +194,7 @@ class HttpSrv(object):
thr = threading.Thread( thr = threading.Thread(
target=self.thr_client, target=self.thr_client,
args=(sck, addr), args=(sck, addr),
name="httpsrv-{}-{}".format(addr[0].split(".", 2)[-1][-6:], addr[1]), name="httpconn-{}-{}".format(addr[0].split(".", 2)[-1][-6:], addr[1]),
) )
thr.daemon = True thr.daemon = True
thr.start() thr.start()
@@ -198,11 +211,11 @@ class HttpSrv(object):
try: try:
sck, addr = task sck, addr = task
me = threading.current_thread() me = threading.current_thread()
me.name = ( me.name = "httpconn-{}-{}".format(
"httpsrv-{}-{}".format(addr[0].split(".", 2)[-1][-6:], addr[1]), addr[0].split(".", 2)[-1][-6:], addr[1]
) )
self.thr_client(sck, addr) self.thr_client(sck, addr)
me.name = "httpsrv-poolw" me.name = self.name + "-poolw"
except: except:
self.log(self.name, "thr_client: " + min_ex(), 3) self.log(self.name, "thr_client: " + min_ex(), 3)
@@ -228,7 +241,7 @@ class HttpSrv(object):
if self.tp_q.empty(): if self.tp_q.empty():
break break
self.log("httpsrv-i" + str(os.getpid()), "ok bye") self.log(self.name, "ok bye")
def thr_client(self, sck, addr): def thr_client(self, sck, addr):
"""thread managing one tcp client""" """thread managing one tcp client"""

View File

@@ -33,10 +33,11 @@ class QFile(object):
class StreamTar(object): class StreamTar(object):
"""construct in-memory tar file from the given path""" """construct in-memory tar file from the given path"""
def __init__(self, fgen, **kwargs): def __init__(self, log, fgen, **kwargs):
self.ci = 0 self.ci = 0
self.co = 0 self.co = 0
self.qfile = QFile() self.qfile = QFile()
self.log = log
self.fgen = fgen self.fgen = fgen
self.errf = None self.errf = None
@@ -91,7 +92,8 @@ class StreamTar(object):
errors.append([f["vp"], repr(ex)]) errors.append([f["vp"], repr(ex)])
if errors: if errors:
self.errf = errdesc(errors) self.errf, txt = errdesc(errors)
self.log("\n".join(([repr(self.errf)] + txt[1:])))
self.ser(self.errf) self.ser(self.errf)
self.tar.close() self.tar.close()

View File

@@ -25,4 +25,4 @@ def errdesc(errors):
"vp": "archive-errors-{}.txt".format(dt), "vp": "archive-errors-{}.txt".format(dt),
"ap": tf_path, "ap": tf_path,
"st": os.stat(tf_path), "st": os.stat(tf_path),
} }, report

View File

@@ -11,7 +11,7 @@ from datetime import datetime, timedelta
import calendar import calendar
from .__init__ import E, PY2, WINDOWS, MACOS, VT100 from .__init__ import E, PY2, WINDOWS, MACOS, VT100
from .util import mp from .util import mp, start_log_thrs, start_stackmon
from .authsrv import AuthSrv from .authsrv import AuthSrv
from .tcpsrv import TcpSrv from .tcpsrv import TcpSrv
from .up2k import Up2k from .up2k import Up2k
@@ -42,6 +42,12 @@ class SvcHub(object):
if args.lo: if args.lo:
self._setup_logfile(printed) self._setup_logfile(printed)
if args.stackmon:
start_stackmon(args.stackmon, 0)
if args.log_thrs:
start_log_thrs(self.log, args.log_thrs, 0)
# initiate all services to manage # initiate all services to manage
self.asrv = AuthSrv(self.args, self.log, False) self.asrv = AuthSrv(self.args, self.log, False)
if args.ls: if args.ls:

View File

@@ -89,7 +89,7 @@ def gen_hdr(h_pos, fn, sz, lastmod, utf8, crc32, pre_crc):
ret += spack(b"<LL", vsz, vsz) ret += spack(b"<LL", vsz, vsz)
# windows support (the "?" replace below too) # windows support (the "?" replace below too)
fn = sanitize_fn(fn, ok="/") fn = sanitize_fn(fn, "/", [])
bfn = fn.encode("utf-8" if utf8 else "cp437", "replace").replace(b"?", b"_") bfn = fn.encode("utf-8" if utf8 else "cp437", "replace").replace(b"?", b"_")
z64_len = len(z64v) * 8 + 4 if z64v else 0 z64_len = len(z64v) * 8 + 4 if z64v else 0
@@ -183,7 +183,8 @@ def gen_ecdr64_loc(ecdr64_pos):
class StreamZip(object): class StreamZip(object):
def __init__(self, fgen, utf8=False, pre_crc=False): def __init__(self, log, fgen, utf8=False, pre_crc=False):
self.log = log
self.fgen = fgen self.fgen = fgen
self.utf8 = utf8 self.utf8 = utf8
self.pre_crc = pre_crc self.pre_crc = pre_crc
@@ -246,8 +247,8 @@ class StreamZip(object):
errors.append([f["vp"], repr(ex)]) errors.append([f["vp"], repr(ex)])
if errors: if errors:
errf = errdesc(errors) errf, txt = errdesc(errors)
print(repr(errf)) self.log("\n".join(([repr(errf)] + txt[1:])))
for x in self.ser(errf): for x in self.ser(errf):
yield x yield x

View File

@@ -260,7 +260,7 @@ class ThumbSrv(object):
pass # default q = 75 pass # default q = 75
if im.mode not in fmts: if im.mode not in fmts:
print("conv {}".format(im.mode)) # print("conv {}".format(im.mode))
im = im.convert("RGB") im = im.convert("RGB")
im.save(tpath, quality=40, method=6) im.save(tpath, quality=40, method=6)

View File

@@ -195,7 +195,7 @@ class Up2k(object):
return True, ret return True, ret
def init_indexes(self, all_vols, scan_vols=[]): def init_indexes(self, all_vols, scan_vols=None):
self.pp = ProgressPrinter() self.pp = ProgressPrinter()
vols = all_vols.values() vols = all_vols.values()
t0 = time.time() t0 = time.time()
@@ -991,7 +991,7 @@ class Up2k(object):
if cj["ptop"] not in self.registry: if cj["ptop"] not in self.registry:
raise Pebkac(410, "location unavailable") raise Pebkac(410, "location unavailable")
cj["name"] = sanitize_fn(cj["name"], bad=[".prologue.html", ".epilogue.html"]) cj["name"] = sanitize_fn(cj["name"], "", [".prologue.html", ".epilogue.html"])
cj["poke"] = time.time() cj["poke"] = time.time()
wark = self._get_wark(cj) wark = self._get_wark(cj)
now = time.time() now = time.time()

View File

@@ -16,6 +16,7 @@ import mimetypes
import contextlib import contextlib
import subprocess as sp # nosec import subprocess as sp # nosec
from datetime import datetime from datetime import datetime
from collections import Counter
from .__init__ import PY2, WINDOWS, ANYWIN from .__init__ import PY2, WINDOWS, ANYWIN
from .stolen import surrogateescape from .stolen import surrogateescape
@@ -282,6 +283,62 @@ def alltrace():
return "\n".join(rret + bret) return "\n".join(rret + bret)
def start_stackmon(arg_str, nid):
suffix = "-{}".format(nid) if nid else ""
fp, f = arg_str.rsplit(",", 1)
f = int(f)
t = threading.Thread(
target=stackmon,
args=(fp, f, suffix),
name="stackmon" + suffix,
)
t.daemon = True
t.start()
def stackmon(fp, ival, suffix):
ctr = 0
while True:
ctr += 1
time.sleep(ival)
st = "{}, {}\n{}".format(ctr, time.time(), alltrace())
with open(fp + suffix, "wb") as f:
f.write(st.encode("utf-8", "replace"))
def start_log_thrs(logger, ival, nid):
ival = int(ival)
tname = lname = "log-thrs"
if nid:
tname = "logthr-n{}-i{:x}".format(nid, os.getpid())
lname = tname[3:]
t = threading.Thread(
target=log_thrs,
args=(logger, ival, lname),
name=tname,
)
t.daemon = True
t.start()
def log_thrs(log, ival, name):
while True:
time.sleep(ival)
tv = [x.name for x in threading.enumerate()]
tv = [
x.split("-")[0]
if x.startswith("httpconn-") or x.startswith("thumb-")
else "listen"
if "-listen-" in x
else x
for x in tv
if not x.startswith("pydevd.")
]
tv = ["{}\033[36m{}".format(v, k) for k, v in sorted(Counter(tv).items())]
log(name, "\033[0m \033[33m".join(tv), 3)
def min_ex(): def min_ex():
et, ev, tb = sys.exc_info() et, ev, tb = sys.exc_info()
tb = traceback.extract_tb(tb) tb = traceback.extract_tb(tb)
@@ -672,7 +729,7 @@ def undot(path):
return "/".join(ret) return "/".join(ret)
def sanitize_fn(fn, ok="", bad=[]): def sanitize_fn(fn, ok, bad):
if "/" not in ok: if "/" not in ok:
fn = fn.replace("\\", "/").split("/")[-1] fn = fn.replace("\\", "/").split("/")[-1]

View File

@@ -28,11 +28,16 @@ window.baguetteBox = (function () {
isOverlayVisible = false, isOverlayVisible = false,
touch = {}, // start-pos touch = {}, // start-pos
touchFlag = false, // busy touchFlag = false, // busy
re_i = /.+\.(gif|jpe?g|png|webp)/i, re_i = /.+\.(gif|jpe?g|png|webp)(\?|$)/i,
re_v = /.+\.(webm|mp4)/i, re_v = /.+\.(webm|mp4)(\?|$)/i,
data = {}, // all galleries data = {}, // all galleries
imagesElements = [], imagesElements = [],
documentLastFocus = null; documentLastFocus = null,
isFullscreen = false;
var onFSC = function (e) {
isFullscreen = !!document.fullscreenElement;
};
var overlayClickHandler = function (event) { var overlayClickHandler = function (event) {
if (event.target.id.indexOf('baguette-img') !== -1) { if (event.target.id.indexOf('baguette-img') !== -1) {
@@ -226,6 +231,16 @@ window.baguetteBox = (function () {
playpause(); playpause();
else if (k == "KeyU" || k == "KeyO") else if (k == "KeyU" || k == "KeyO")
relseek(k == "KeyU" ? -10 : 10); relseek(k == "KeyU" ? -10 : 10);
else if (k == "KeyM" && vid())
vid().muted = !vid().muted;
else if (k == "KeyF")
try {
if (isFullscreen)
document.exitFullscreen();
else
vid().requestFullscreen();
}
catch (ex) { }
} }
function keyUpHandler(e) { function keyUpHandler(e) {
@@ -335,6 +350,7 @@ window.baguetteBox = (function () {
bind(document, 'keydown', keyDownHandler); bind(document, 'keydown', keyDownHandler);
bind(document, 'keyup', keyUpHandler); bind(document, 'keyup', keyUpHandler);
bind(document, 'fullscreenchange', onFSC);
currentIndex = chosenImageIndex; currentIndex = chosenImageIndex;
touch = { touch = {
count: 0, count: 0,
@@ -387,6 +403,7 @@ window.baguetteBox = (function () {
unbind(document, 'keydown', keyDownHandler); unbind(document, 'keydown', keyDownHandler);
unbind(document, 'keyup', keyUpHandler); unbind(document, 'keyup', keyUpHandler);
unbind(document, 'fullscreenchange', onFSC);
// Fade out and hide the overlay // Fade out and hide the overlay
overlay.className = ''; overlay.className = '';
setTimeout(function () { setTimeout(function () {

View File

@@ -1179,8 +1179,10 @@ html.light #tree::-webkit-scrollbar {
display: inline-block; display: inline-block;
width: auto; width: auto;
height: auto; height: auto;
max-height: 100%;
max-width: 100%; max-width: 100%;
max-height: 100%;
max-height: calc(100% - 1.4em);
margin-bottom: 1.4em;
vertical-align: middle; vertical-align: middle;
box-shadow: 0 0 8px rgba(0, 0, 0, 0.6); box-shadow: 0 0 8px rgba(0, 0, 0, 0.6);
} }
@@ -1189,11 +1191,10 @@ html.light #tree::-webkit-scrollbar {
} }
#baguetteBox-overlay .full-image figcaption { #baguetteBox-overlay .full-image figcaption {
display: block; display: block;
position: absolute; position: fixed;
bottom: 0; bottom: .1em;
width: 100%; width: 100%;
text-align: center; text-align: center;
line-height: 1.8;
white-space: normal; white-space: normal;
color: #ccc; color: #ccc;
} }

View File

@@ -37,6 +37,7 @@ function vis_exh(msg, url, lineNo, columnNo, error) {
html.push('<h2>' + find[a] + '</h2>' + html.push('<h2>' + find[a] + '</h2>' +
esc(String(error[find[a]])).replace(/\n/g, '<br />\n')); esc(String(error[find[a]])).replace(/\n/g, '<br />\n'));
} }
html.push('<p style="border-top:1px solid #999;margin-top:1.5em;font-size:1.4em">if you are stuck here, try to <a href="#" onclick="localStorage.clear();location.reload();">reset copyparty settings</a></p>');
document.body.innerHTML = html.join('\n'); document.body.innerHTML = html.join('\n');
var s = mknod('style'); var s = mknod('style');

View File

@@ -23,10 +23,10 @@ def hdr(query):
class Cfg(Namespace): class Cfg(Namespace):
def __init__(self, a=[], v=[], c=None): def __init__(self, a=None, v=None, c=None):
super(Cfg, self).__init__( super(Cfg, self).__init__(
a=a, a=a or [],
v=v, v=v or [],
c=c, c=c,
rproxy=0, rproxy=0,
ed=False, ed=False,

View File

@@ -16,7 +16,7 @@ from copyparty import util
class Cfg(Namespace): class Cfg(Namespace):
def __init__(self, a=[], v=[], c=None): def __init__(self, a=None, v=None, c=None):
ex = {k: False for k in "nw e2d e2ds e2dsa e2t e2ts e2tsr".split()} ex = {k: False for k in "nw e2d e2ds e2dsa e2t e2ts e2tsr".split()}
ex2 = { ex2 = {
"mtp": [], "mtp": [],
@@ -27,7 +27,7 @@ class Cfg(Namespace):
"rproxy": 0, "rproxy": 0,
} }
ex.update(ex2) ex.update(ex2)
super(Cfg, self).__init__(a=a, v=v, c=c, **ex) super(Cfg, self).__init__(a=a or [], v=v or [], c=c, **ex)
class TestVFS(unittest.TestCase): class TestVFS(unittest.TestCase):

View File

@@ -126,7 +126,6 @@ class VHttpConn(object):
self.hsrv = VHttpSrv() self.hsrv = VHttpSrv()
self.nreq = 0 self.nreq = 0
self.nbyte = 0 self.nbyte = 0
self.workload = 0
self.ico = None self.ico = None
self.thumbcli = None self.thumbcli = None
self.t0 = time.time() self.t0 = time.time()