proactive opus cache expiration

copyparty/__main__.py

@@ -418,6 +418,7 @@ def run_argparse(argv, formatter):
 
     ap2 = ap.add_argument_group('transcoding options')
     ap2.add_argument("--no-acode", action="store_true", help="disable audio transcoding")
+    ap2.add_argument("--ac-maxage", metavar="SEC", type=int, default=86400, help="delete transcode output after SEC seconds")
 
     ap2 = ap.add_argument_group('general db options')
     ap2.add_argument("-e2d", action="store_true", help="enable up2k database")

copyparty/svchub.py

@@ -89,11 +89,18 @@ class SvcHub(object):
                     "thumb", msg.format(" " * 37, os.path.basename(sys.executable)), c=3
                 )
 
+        if not args.no_acode and args.no_thumb:
+            msg = "setting --no-acode because --no-thumb (sorry)"
+            self.log("thumb", msg, c=6)
+            args.no_acode = True
+
         if not args.no_acode and (not HAVE_FFMPEG or not HAVE_FFPROBE):
             msg = "setting --no-acode because either FFmpeg or FFprobe is not available"
             self.log("thumb", msg, c=6)
             args.no_acode = True
 
+        args.th_poke = min(args.th_poke, args.th_maxage, args.ac_maxage)
+
         # decide which worker impl to use
         if self.check_mp_enable():
             from .broker_mp import BrokerMp as Broker

copyparty/th_cli.py

@@ -65,6 +65,11 @@ class ThumbCli(object):
             if self.cooldown.poke(tdir):
                 self.broker.put(False, "thumbsrv.poke", tdir)
 
+            if want_opus:
+                # audio files expire individually
+                if self.cooldown.poke(tpath):
+                    self.broker.put(False, "thumbsrv.poke", tpath)
+
             return ret
 
         x = self.broker.put(True, "thumbsrv.get", ptop, rem, mtime, fmt)

copyparty/th_srv.py

@@ -10,7 +10,7 @@ import threading
 import subprocess as sp
 
 from .__init__ import PY2, unicode
-from .util import fsenc, vsplit, runcmd, Queue, Cooldown, BytesIO, min_ex
+from .util import fsenc, vsplit, statdir, runcmd, Queue, Cooldown, BytesIO, min_ex
 from .bos import bos
 from .mtag import HAVE_FFMPEG, HAVE_FFPROBE, ffprobe
 
@@ -90,10 +90,13 @@ def thumb_path(histpath, rem, mtime, fmt):
     h = hashlib.sha512(fsenc(fn)).digest()
     fn = base64.urlsafe_b64encode(h).decode("ascii")[:24]
 
-    if fmt != "opus":
+    if fmt == "opus":
+        cat = "ac"
+    else:
         fmt = "webp" if fmt == "w" else "jpg"
+        cat = "th"
 
-    return "{}/th/{}/{}.{:x}.{}".format(histpath, rd, fn, int(mtime), fmt)
+    return "{}/{}/{}/{}.{:x}.{}".format(histpath, cat, rd, fn, int(mtime), fmt)
 
 
 class ThumbSrv(object):
@@ -187,6 +190,7 @@ class ThumbSrv(object):
         try:
             st = bos.stat(tpath)
             if st.st_size:
+                self.poke(tpath)
                 return tpath
         except:
             pass
@@ -414,10 +418,9 @@ class ThumbSrv(object):
 
         ts = int(time.time())
         try:
-            p1 = os.path.dirname(tdir)
+            for _ in range(4):
-            p2 = os.path.dirname(p1)
+                bos.utime(tdir, (ts, ts))
-            for dp in [tdir, p1, p2]:
+                tdir = os.path.dirname(tdir)
-                bos.utime(dp, (ts, ts))
         except:
             pass
 
@@ -437,25 +440,36 @@ class ThumbSrv(object):
             self.log("\033[Jcln ok; rm {} dirs".format(ndirs))
 
     def clean(self, histpath):
-        thumbpath = os.path.join(histpath, "th")
+        ret = 0
+        for cat in ["th", "ac"]:
+            ret += self._clean(histpath, cat, None)
+
+        return ret
+
+    def _clean(self, histpath, cat, thumbpath):
+        if not thumbpath:
+            thumbpath = os.path.join(histpath, cat)
+
         # self.log("cln {}".format(thumbpath))
-        maxage = self.args.th_maxage
+        exts = ["jpg", "webp"] if cat == "th" else ["opus"]
+        maxage = getattr(self.args, cat + "_maxage")
         now = time.time()
         prev_b64 = None
         prev_fp = None
         try:
-            ents = bos.listdir(thumbpath)
+            ents = statdir(self.log, not self.args.no_scandir, False, thumbpath)
+            ents = sorted(list(ents))
         except:
             return 0
 
         ndirs = 0
-        for f in sorted(ents):
+        for f, inf in ents:
             fp = os.path.join(thumbpath, f)
             cmp = fp.lower().replace("\\", "/")
 
             # "top" or b64 prefix/full (a folder)
             if len(f) <= 3 or len(f) == 24:
-                age = now - bos.path.getmtime(fp)
+                age = now - inf.st_mtime
                 if age > maxage:
                     with self.mutex:
                         safe = True
@@ -469,16 +483,15 @@ class ThumbSrv(object):
                             self.log("rm -rf [{}]".format(fp))
                             shutil.rmtree(fp, ignore_errors=True)
                 else:
-                    ndirs += self.clean(fp)
+                    self._clean(histpath, cat, fp)
+
                 continue
 
             # thumb file
             try:
                 b64, ts, ext = f.split(".")
-                if len(b64) != 24 or len(ts) != 8 or ext not in ["jpg", "webp"]:
+                if len(b64) != 24 or len(ts) != 8 or ext not in exts:
                     raise Exception()
-
-                ts = int(ts, 16)
             except:
                 if f != "dir.txt":
                     self.log("foreign file in thumbs dir: [{}]".format(fp), 1)
@@ -489,6 +502,10 @@ class ThumbSrv(object):
                 self.log("rm replaced [{}]".format(fp))
                 bos.unlink(prev_fp)
 
+            if cat != "th" and inf.st_mtime + maxage < now:
+                self.log("rm expired [{}]".format(fp))
+                bos.unlink(fp)
+
             prev_b64 = b64
             prev_fp = fp