From 483dd527c61063052aafc67f311f1c9555fbef20 Mon Sep 17 00:00:00 2001 From: ed Date: Tue, 25 May 2021 19:46:35 +0200 Subject: [PATCH] add cache eviction --- README.md | 10 ++--- copyparty/__main__.py | 3 ++ copyparty/httpcli.py | 22 ++++++----- copyparty/svchub.py | 16 ++++++-- copyparty/th_cli.py | 18 ++++++++- copyparty/th_srv.py | 88 ++++++++++++++++++++++++++++++++++++++++++- copyparty/up2k.py | 12 +++--- copyparty/util.py | 26 +++++++++++++ 8 files changed, 166 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index 520592d7..ae53219b 100644 --- a/README.md +++ b/README.md @@ -92,10 +92,10 @@ you may also want these, especially on servers: * browser * ☑ tree-view * ☑ media player - * ✖ thumbnails - * ☑ images - * ☑ videos - * ✖ cache eviction + * ☑ thumbnails + * ☑ images using Pillow + * ☑ videos using FFmpeg + * ☑ cache eviction (max-age; maybe max-size eventually) * ☑ SPA (browse while uploading) * if you use the file-tree on the left only, not folders in the file list * server indexing @@ -106,7 +106,7 @@ you may also want these, especially on servers: * ☑ viewer * ☑ editor (sure why not) -summary: it works! you can use it! (but technically not even close to beta) +summary: it works! # bugs diff --git a/copyparty/__main__.py b/copyparty/__main__.py index 62fbe512..6f8a3763 100644 --- a/copyparty/__main__.py +++ b/copyparty/__main__.py @@ -253,6 +253,9 @@ def run_argparse(argv, formatter): ap2.add_argument("--no-thumb", action="store_true", help="disable all thumbnails") ap2.add_argument("--no-vthumb", action="store_true", help="disable video thumbnails") ap2.add_argument("--thumbsz", metavar="WxH", default="352x352", help="thumbnail res") + ap2.add_argument("--th-poke", metavar="SEC", type=int, default=300, help="activity labeling cooldown") + ap2.add_argument("--th-clean", metavar="SEC", type=int, default=1800, help="cleanup interval") + ap2.add_argument("--th-maxage", metavar="SEC", type=int, default=604800, help="max folder age") ap2 = ap.add_argument_group('database options') ap2.add_argument("-e2d", action="store_true", help="enable up2k database") diff --git a/copyparty/httpcli.py b/copyparty/httpcli.py index 9813678d..32282746 100644 --- a/copyparty/httpcli.py +++ b/copyparty/httpcli.py @@ -286,7 +286,7 @@ class HttpCli(object): # "embedded" resources if self.vpath.startswith(".cpr"): if self.vpath.startswith(".cpr/ico/"): - return self.tx_ico(self.vpath.split("/")[-1]) + return self.tx_ico(self.vpath.split("/")[-1], exact=True) static_path = os.path.join(E.mod, "web/", self.vpath[5:]) return self.tx_file(static_path) @@ -1207,15 +1207,19 @@ class HttpCli(object): self.log("{}, {}".format(logmsg, spd)) return True - def tx_ico(self, ext): - bad = re.compile(r"[](){}[]|^[0-9_-]*$") - n = ext.split(".")[1:][::-1] - ext = "" - for v in n: - if len(v) > 7 or bad.match(v): - break + def tx_ico(self, ext, exact=False): + if not exact: + if ext.endswith("/"): + ext = "a.folder" - ext = "{}.{}".format(v, ext) + bad = re.compile(r"[](){}[]|^[0-9_-]*$") + n = ext.split(".")[1:][::-1] + ext = "" + for v in n: + if len(v) > 7 or bad.match(v): + break + + ext = "{}.{}".format(v, ext) ext = ext.rstrip(".") or "unk" mime, ico = self.ico.get(ext) diff --git a/copyparty/svchub.py b/copyparty/svchub.py index a7d72059..d8769957 100644 --- a/copyparty/svchub.py +++ b/copyparty/svchub.py @@ -11,6 +11,7 @@ import calendar from .__init__ import PY2, WINDOWS, MACOS, VT100 from .util import mp +from .authsrv import AuthSrv from .tcpsrv import TcpSrv from .up2k import Up2k from .th_srv import ThumbSrv, HAVE_PIL @@ -36,14 +37,17 @@ class SvcHub(object): self.log = self._log_disabled if args.q else self._log_enabled + # jank goes here + auth = AuthSrv(self.args, self.log, False) + # initiate all services to manage self.tcpsrv = TcpSrv(self) - self.up2k = Up2k(self) + self.up2k = Up2k(self, auth.vfs.all_vols) self.thumbsrv = None if not args.no_thumb: if HAVE_PIL: - self.thumbsrv = ThumbSrv(self) + self.thumbsrv = ThumbSrv(self, auth.vfs.all_vols) else: msg = "need Pillow to create thumbnails; for example:\n {} -m pip install --user Pillow" self.log("thumb", msg.format(os.path.basename(sys.executable)), c=3) @@ -76,9 +80,13 @@ class SvcHub(object): if self.thumbsrv: self.thumbsrv.shutdown() - print("waiting for thumbsrv...") - while not self.thumbsrv.stopped(): + for n in range(200): # 10s time.sleep(0.05) + if self.thumbsrv.stopped(): + break + + if n == 3: + print("waiting for thumbsrv...") print("nailed it") diff --git a/copyparty/th_cli.py b/copyparty/th_cli.py index 942e269c..e83a8357 100644 --- a/copyparty/th_cli.py +++ b/copyparty/th_cli.py @@ -1,5 +1,7 @@ import os +import time +from .util import Cooldown from .th_srv import thumb_path, THUMBABLE, FMT_FF @@ -8,6 +10,9 @@ class ThumbCli(object): self.broker = broker self.args = broker.args + # cache on both sides for less broker spam + self.cooldown = Cooldown(self.args.th_poke) + def get(self, ptop, rem, mtime): ext = rem.rsplit(".")[-1].lower() if ext not in THUMBABLE: @@ -17,13 +22,22 @@ class ThumbCli(object): return None tpath = thumb_path(ptop, rem, mtime) + ret = None try: st = os.stat(tpath) if st.st_size: - return tpath - return None + ret = tpath + else: + return None except: pass + if ret: + tdir = os.path.dirname(tpath) + if self.cooldown.poke(tdir): + self.broker.put(False, "thumbsrv.poke", tdir) + + return ret + x = self.broker.put(True, "thumbsrv.get", ptop, rem, mtime) return x.get() diff --git a/copyparty/th_srv.py b/copyparty/th_srv.py index 6b3851bd..5eb5f1b4 100644 --- a/copyparty/th_srv.py +++ b/copyparty/th_srv.py @@ -1,12 +1,14 @@ import os import sys +import time +import shutil import base64 import hashlib import threading import subprocess as sp from .__init__ import PY2 -from .util import fsenc, Queue +from .util import fsenc, Queue, Cooldown from .mtag import HAVE_FFMPEG, HAVE_FFPROBE, parse_ffprobe @@ -74,13 +76,16 @@ def thumb_path(ptop, rem, mtime): class ThumbSrv(object): - def __init__(self, hub): + def __init__(self, hub, vols): self.hub = hub + self.vols = [v.realpath for v in vols.values()] + self.args = hub.args self.log_func = hub.log res = hub.args.thumbsz.split("x") self.res = tuple([int(x) for x in res]) + self.poke_cd = Cooldown(self.args.th_poke) self.mutex = threading.Lock() self.busy = {} @@ -108,6 +113,10 @@ class ThumbSrv(object): msg += ", ".join(missing) self.log(msg, c=1) + t = threading.Thread(target=self.cleaner) + t.daemon = True + t.start() + def log(self, msg, c=0): self.log_func("thumb", msg, c) @@ -233,3 +242,78 @@ class ThumbSrv(object): ] p = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE) r = p.communicate() + + def poke(self, tdir): + if not self.poke_cd.poke(tdir): + return + + ts = int(time.time()) + try: + p1 = os.path.dirname(tdir) + p2 = os.path.dirname(p1) + for dp in [tdir, p1, p2]: + os.utime(fsenc(dp), (ts, ts)) + except: + pass + + def cleaner(self): + interval = self.args.th_clean + while True: + time.sleep(interval) + for vol in self.vols: + vol += "/.hist/th" + self.log("cln {}/".format(vol)) + self.clean(vol) + + def clean(self, vol): + # self.log("cln {}".format(vol)) + maxage = self.args.th_maxage + now = time.time() + prev_b64 = None + prev_fp = None + try: + ents = os.listdir(vol) + except: + return + + for f in sorted(ents): + fp = os.path.join(vol, f) + cmp = fp.lower().replace("\\", "/") + + # "top" or b64 prefix/full (a folder) + if len(f) <= 3 or len(f) == 24: + age = now - os.path.getmtime(fp) + if age > maxage: + with self.mutex: + safe = True + for k in self.busy.keys(): + if k.lower().replace("\\", "/").startswith(cmp): + safe = False + break + + if safe: + self.log("rm -rf [{}]".format(fp)) + shutil.rmtree(fp, ignore_errors=True) + else: + self.clean(fp) + continue + + # thumb file + try: + b64, ts, ext = f.split(".") + if len(b64) != 24 or len(ts) != 8 or ext != "jpg": + raise Exception() + + ts = int(ts, 16) + except: + if f != "dir.txt": + self.log("foreign file in thumbs dir: [{}]".format(fp), 1) + + continue + + if b64 == prev_b64: + self.log("rm replaced [{}]".format(fp)) + os.unlink(prev_fp) + + prev_b64 = b64 + prev_fp = fp diff --git a/copyparty/up2k.py b/copyparty/up2k.py index 7744195c..93042185 100644 --- a/copyparty/up2k.py +++ b/copyparty/up2k.py @@ -32,7 +32,6 @@ from .util import ( s2hms, ) from .mtag import MTag -from .authsrv import AuthSrv try: HAVE_SQLITE3 = True @@ -49,10 +48,11 @@ class Up2k(object): * ~/.config flatfiles for active jobs """ - def __init__(self, hub): + def __init__(self, hub, all_vols): self.hub = hub self.args = hub.args self.log_func = hub.log + self.all_vols = all_vols # config self.salt = self.args.salt @@ -92,9 +92,7 @@ class Up2k(object): if not HAVE_SQLITE3: self.log("could not initialize sqlite3, will use in-memory registry only") - # this is kinda jank - auth = AuthSrv(self.args, self.log_func, False) - have_e2d = self.init_indexes(auth) + have_e2d = self.init_indexes() if have_e2d: thr = threading.Thread(target=self._snapshot) @@ -139,9 +137,9 @@ class Up2k(object): return True, ret - def init_indexes(self, auth): + def init_indexes(self): self.pp = ProgressPrinter() - vols = auth.vfs.all_vols.values() + vols = self.all_vols.values() t0 = time.time() have_e2d = False diff --git a/copyparty/util.py b/copyparty/util.py index 416a41da..722c653e 100644 --- a/copyparty/util.py +++ b/copyparty/util.py @@ -124,6 +124,32 @@ class Counter(object): self.v = absval +class Cooldown(object): + def __init__(self, maxage): + self.maxage = maxage + self.mutex = threading.Lock() + self.hist = {} + self.oldest = 0 + + def poke(self, key): + with self.mutex: + now = time.time() + + ret = False + v = self.hist.get(key, 0) + if now - v > self.maxage: + self.hist[key] = now + ret = True + + if self.oldest - now > self.maxage * 2: + self.hist = { + k: v for k, v in self.hist.items() if now - v < self.maxage + } + self.oldest = sorted(self.hist.values())[0] + + return ret + + class Unrecv(object): """ undo any number of socket recv ops