add cache eviction

README.md

@@ -92,10 +92,10 @@ you may also want these, especially on servers:
 * browser
   * ☑ tree-view
   * ☑ media player
-  * ✖ thumbnails
+  * ☑ thumbnails
-    * ☑ images
+    * ☑ images using Pillow
-    * ☑ videos
+    * ☑ videos using FFmpeg
-    * ✖ cache eviction
+    * ☑ cache eviction (max-age; maybe max-size eventually)
   * ☑ SPA (browse while uploading)
     * if you use the file-tree on the left only, not folders in the file list
 * server indexing
@@ -106,7 +106,7 @@ you may also want these, especially on servers:
   * ☑ viewer
   * ☑ editor (sure why not)
 
-summary: it works! you can use it! (but technically not even close to beta)
+summary: it works!
 
 
 # bugs

copyparty/__main__.py

@@ -253,6 +253,9 @@ def run_argparse(argv, formatter):
     ap2.add_argument("--no-thumb", action="store_true", help="disable all thumbnails")
     ap2.add_argument("--no-vthumb", action="store_true", help="disable video thumbnails")
     ap2.add_argument("--thumbsz", metavar="WxH", default="352x352", help="thumbnail res")
+    ap2.add_argument("--th-poke", metavar="SEC", type=int, default=300, help="activity labeling cooldown")
+    ap2.add_argument("--th-clean", metavar="SEC", type=int, default=1800, help="cleanup interval")
+    ap2.add_argument("--th-maxage", metavar="SEC", type=int, default=604800, help="max folder age")
 
     ap2 = ap.add_argument_group('database options')
     ap2.add_argument("-e2d", action="store_true", help="enable up2k database")

copyparty/httpcli.py

@@ -286,7 +286,7 @@ class HttpCli(object):
         # "embedded" resources
         if self.vpath.startswith(".cpr"):
             if self.vpath.startswith(".cpr/ico/"):
-                return self.tx_ico(self.vpath.split("/")[-1])
+                return self.tx_ico(self.vpath.split("/")[-1], exact=True)
 
             static_path = os.path.join(E.mod, "web/", self.vpath[5:])
             return self.tx_file(static_path)
@@ -1207,7 +1207,11 @@ class HttpCli(object):
         self.log("{},  {}".format(logmsg, spd))
         return True
 
-    def tx_ico(self, ext):
+    def tx_ico(self, ext, exact=False):
+        if not exact:
+            if ext.endswith("/"):
+                ext = "a.folder"
+
             bad = re.compile(r"[](){}[]|^[0-9_-]*$")
             n = ext.split(".")[1:][::-1]
             ext = ""

copyparty/svchub.py

@@ -11,6 +11,7 @@ import calendar
 
 from .__init__ import PY2, WINDOWS, MACOS, VT100
 from .util import mp
+from .authsrv import AuthSrv
 from .tcpsrv import TcpSrv
 from .up2k import Up2k
 from .th_srv import ThumbSrv, HAVE_PIL
@@ -36,14 +37,17 @@ class SvcHub(object):
 
         self.log = self._log_disabled if args.q else self._log_enabled
 
+        # jank goes here
+        auth = AuthSrv(self.args, self.log, False)
+
         # initiate all services to manage
         self.tcpsrv = TcpSrv(self)
-        self.up2k = Up2k(self)
+        self.up2k = Up2k(self, auth.vfs.all_vols)
 
         self.thumbsrv = None
         if not args.no_thumb:
             if HAVE_PIL:
-                self.thumbsrv = ThumbSrv(self)
+                self.thumbsrv = ThumbSrv(self, auth.vfs.all_vols)
             else:
                 msg = "need Pillow to create thumbnails; for example:\n  {} -m pip install --user Pillow"
                 self.log("thumb", msg.format(os.path.basename(sys.executable)), c=3)
@@ -76,9 +80,13 @@ class SvcHub(object):
             if self.thumbsrv:
                 self.thumbsrv.shutdown()
 
-                print("waiting for thumbsrv...")
+                for n in range(200):  # 10s
-                while not self.thumbsrv.stopped():
                     time.sleep(0.05)
+                    if self.thumbsrv.stopped():
+                        break
+
+                    if n == 3:
+                        print("waiting for thumbsrv...")
 
             print("nailed it")
 

copyparty/th_cli.py

@@ -1,5 +1,7 @@
 import os
+import time
 
+from .util import Cooldown
 from .th_srv import thumb_path, THUMBABLE, FMT_FF
 
 
@@ -8,6 +10,9 @@ class ThumbCli(object):
         self.broker = broker
         self.args = broker.args
 
+        # cache on both sides for less broker spam
+        self.cooldown = Cooldown(self.args.th_poke)
+
     def get(self, ptop, rem, mtime):
         ext = rem.rsplit(".")[-1].lower()
         if ext not in THUMBABLE:
@@ -17,13 +22,22 @@ class ThumbCli(object):
             return None
 
         tpath = thumb_path(ptop, rem, mtime)
+        ret = None
         try:
             st = os.stat(tpath)
             if st.st_size:
-                return tpath
+                ret = tpath
+            else:
                 return None
         except:
             pass
 
+        if ret:
+            tdir = os.path.dirname(tpath)
+            if self.cooldown.poke(tdir):
+                self.broker.put(False, "thumbsrv.poke", tdir)
+
+            return ret
+
         x = self.broker.put(True, "thumbsrv.get", ptop, rem, mtime)
         return x.get()

copyparty/th_srv.py

@@ -1,12 +1,14 @@
 import os
 import sys
+import time
+import shutil
 import base64
 import hashlib
 import threading
 import subprocess as sp
 
 from .__init__ import PY2
-from .util import fsenc, Queue
+from .util import fsenc, Queue, Cooldown
 from .mtag import HAVE_FFMPEG, HAVE_FFPROBE, parse_ffprobe
 
 
@@ -74,13 +76,16 @@ def thumb_path(ptop, rem, mtime):
 
 
 class ThumbSrv(object):
-    def __init__(self, hub):
+    def __init__(self, hub, vols):
         self.hub = hub
+        self.vols = [v.realpath for v in vols.values()]
+
         self.args = hub.args
         self.log_func = hub.log
 
         res = hub.args.thumbsz.split("x")
         self.res = tuple([int(x) for x in res])
+        self.poke_cd = Cooldown(self.args.th_poke)
 
         self.mutex = threading.Lock()
         self.busy = {}
@@ -108,6 +113,10 @@ class ThumbSrv(object):
             msg += ", ".join(missing)
             self.log(msg, c=1)
 
+        t = threading.Thread(target=self.cleaner)
+        t.daemon = True
+        t.start()
+
     def log(self, msg, c=0):
         self.log_func("thumb", msg, c)
 
@@ -233,3 +242,78 @@ class ThumbSrv(object):
         ]
         p = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE)
         r = p.communicate()
+
+    def poke(self, tdir):
+        if not self.poke_cd.poke(tdir):
+            return
+
+        ts = int(time.time())
+        try:
+            p1 = os.path.dirname(tdir)
+            p2 = os.path.dirname(p1)
+            for dp in [tdir, p1, p2]:
+                os.utime(fsenc(dp), (ts, ts))
+        except:
+            pass
+
+    def cleaner(self):
+        interval = self.args.th_clean
+        while True:
+            time.sleep(interval)
+            for vol in self.vols:
+                vol += "/.hist/th"
+                self.log("cln {}/".format(vol))
+                self.clean(vol)
+
+    def clean(self, vol):
+        # self.log("cln {}".format(vol))
+        maxage = self.args.th_maxage
+        now = time.time()
+        prev_b64 = None
+        prev_fp = None
+        try:
+            ents = os.listdir(vol)
+        except:
+            return
+
+        for f in sorted(ents):
+            fp = os.path.join(vol, f)
+            cmp = fp.lower().replace("\\", "/")
+
+            # "top" or b64 prefix/full (a folder)
+            if len(f) <= 3 or len(f) == 24:
+                age = now - os.path.getmtime(fp)
+                if age > maxage:
+                    with self.mutex:
+                        safe = True
+                        for k in self.busy.keys():
+                            if k.lower().replace("\\", "/").startswith(cmp):
+                                safe = False
+                                break
+
+                        if safe:
+                            self.log("rm -rf [{}]".format(fp))
+                            shutil.rmtree(fp, ignore_errors=True)
+                else:
+                    self.clean(fp)
+                continue
+
+            # thumb file
+            try:
+                b64, ts, ext = f.split(".")
+                if len(b64) != 24 or len(ts) != 8 or ext != "jpg":
+                    raise Exception()
+
+                ts = int(ts, 16)
+            except:
+                if f != "dir.txt":
+                    self.log("foreign file in thumbs dir: [{}]".format(fp), 1)
+
+                continue
+
+            if b64 == prev_b64:
+                self.log("rm replaced [{}]".format(fp))
+                os.unlink(prev_fp)
+
+            prev_b64 = b64
+            prev_fp = fp

copyparty/up2k.py

@@ -32,7 +32,6 @@ from .util import (
     s2hms,
 )
 from .mtag import MTag
-from .authsrv import AuthSrv
 
 try:
     HAVE_SQLITE3 = True
@@ -49,10 +48,11 @@ class Up2k(object):
         * ~/.config flatfiles for active jobs
     """
 
-    def __init__(self, hub):
+    def __init__(self, hub, all_vols):
         self.hub = hub
         self.args = hub.args
         self.log_func = hub.log
+        self.all_vols = all_vols
 
         # config
         self.salt = self.args.salt
@@ -92,9 +92,7 @@ class Up2k(object):
         if not HAVE_SQLITE3:
             self.log("could not initialize sqlite3, will use in-memory registry only")
 
-        # this is kinda jank
+        have_e2d = self.init_indexes()
-        auth = AuthSrv(self.args, self.log_func, False)
-        have_e2d = self.init_indexes(auth)
 
         if have_e2d:
             thr = threading.Thread(target=self._snapshot)
@@ -139,9 +137,9 @@ class Up2k(object):
 
         return True, ret
 
-    def init_indexes(self, auth):
+    def init_indexes(self):
         self.pp = ProgressPrinter()
-        vols = auth.vfs.all_vols.values()
+        vols = self.all_vols.values()
         t0 = time.time()
         have_e2d = False
 

copyparty/util.py

@@ -124,6 +124,32 @@ class Counter(object):
             self.v = absval
 
 
+class Cooldown(object):
+    def __init__(self, maxage):
+        self.maxage = maxage
+        self.mutex = threading.Lock()
+        self.hist = {}
+        self.oldest = 0
+
+    def poke(self, key):
+        with self.mutex:
+            now = time.time()
+
+            ret = False
+            v = self.hist.get(key, 0)
+            if now - v > self.maxage:
+                self.hist[key] = now
+                ret = True
+
+            if self.oldest - now > self.maxage * 2:
+                self.hist = {
+                    k: v for k, v in self.hist.items() if now - v < self.maxage
+                }
+                self.oldest = sorted(self.hist.values())[0]
+
+            return ret
+
+
 class Unrecv(object):
     """
     undo any number of socket recv ops