diff --git a/copyparty/__main__.py b/copyparty/__main__.py index a10b6170..c44ac711 100644 --- a/copyparty/__main__.py +++ b/copyparty/__main__.py @@ -418,6 +418,7 @@ def run_argparse(argv, formatter): ap2 = ap.add_argument_group('transcoding options') ap2.add_argument("--no-acode", action="store_true", help="disable audio transcoding") + ap2.add_argument("--ac-maxage", metavar="SEC", type=int, default=86400, help="delete transcode output after SEC seconds") ap2 = ap.add_argument_group('general db options') ap2.add_argument("-e2d", action="store_true", help="enable up2k database") diff --git a/copyparty/svchub.py b/copyparty/svchub.py index 11da5709..7bc3597e 100644 --- a/copyparty/svchub.py +++ b/copyparty/svchub.py @@ -89,11 +89,18 @@ class SvcHub(object): "thumb", msg.format(" " * 37, os.path.basename(sys.executable)), c=3 ) + if not args.no_acode and args.no_thumb: + msg = "setting --no-acode because --no-thumb (sorry)" + self.log("thumb", msg, c=6) + args.no_acode = True + if not args.no_acode and (not HAVE_FFMPEG or not HAVE_FFPROBE): msg = "setting --no-acode because either FFmpeg or FFprobe is not available" self.log("thumb", msg, c=6) args.no_acode = True + args.th_poke = min(args.th_poke, args.th_maxage, args.ac_maxage) + # decide which worker impl to use if self.check_mp_enable(): from .broker_mp import BrokerMp as Broker diff --git a/copyparty/th_cli.py b/copyparty/th_cli.py index c64f921f..8bb395d5 100644 --- a/copyparty/th_cli.py +++ b/copyparty/th_cli.py @@ -65,6 +65,11 @@ class ThumbCli(object): if self.cooldown.poke(tdir): self.broker.put(False, "thumbsrv.poke", tdir) + if want_opus: + # audio files expire individually + if self.cooldown.poke(tpath): + self.broker.put(False, "thumbsrv.poke", tpath) + return ret x = self.broker.put(True, "thumbsrv.get", ptop, rem, mtime, fmt) diff --git a/copyparty/th_srv.py b/copyparty/th_srv.py index ee2662c3..ce22aae5 100644 --- a/copyparty/th_srv.py +++ b/copyparty/th_srv.py @@ -10,7 +10,7 @@ import threading import subprocess as sp from .__init__ import PY2, unicode -from .util import fsenc, vsplit, runcmd, Queue, Cooldown, BytesIO, min_ex +from .util import fsenc, vsplit, statdir, runcmd, Queue, Cooldown, BytesIO, min_ex from .bos import bos from .mtag import HAVE_FFMPEG, HAVE_FFPROBE, ffprobe @@ -90,10 +90,13 @@ def thumb_path(histpath, rem, mtime, fmt): h = hashlib.sha512(fsenc(fn)).digest() fn = base64.urlsafe_b64encode(h).decode("ascii")[:24] - if fmt != "opus": + if fmt == "opus": + cat = "ac" + else: fmt = "webp" if fmt == "w" else "jpg" + cat = "th" - return "{}/th/{}/{}.{:x}.{}".format(histpath, rd, fn, int(mtime), fmt) + return "{}/{}/{}/{}.{:x}.{}".format(histpath, cat, rd, fn, int(mtime), fmt) class ThumbSrv(object): @@ -187,6 +190,7 @@ class ThumbSrv(object): try: st = bos.stat(tpath) if st.st_size: + self.poke(tpath) return tpath except: pass @@ -414,10 +418,9 @@ class ThumbSrv(object): ts = int(time.time()) try: - p1 = os.path.dirname(tdir) - p2 = os.path.dirname(p1) - for dp in [tdir, p1, p2]: - bos.utime(dp, (ts, ts)) + for _ in range(4): + bos.utime(tdir, (ts, ts)) + tdir = os.path.dirname(tdir) except: pass @@ -437,25 +440,36 @@ class ThumbSrv(object): self.log("\033[Jcln ok; rm {} dirs".format(ndirs)) def clean(self, histpath): - thumbpath = os.path.join(histpath, "th") + ret = 0 + for cat in ["th", "ac"]: + ret += self._clean(histpath, cat, None) + + return ret + + def _clean(self, histpath, cat, thumbpath): + if not thumbpath: + thumbpath = os.path.join(histpath, cat) + # self.log("cln {}".format(thumbpath)) - maxage = self.args.th_maxage + exts = ["jpg", "webp"] if cat == "th" else ["opus"] + maxage = getattr(self.args, cat + "_maxage") now = time.time() prev_b64 = None prev_fp = None try: - ents = bos.listdir(thumbpath) + ents = statdir(self.log, not self.args.no_scandir, False, thumbpath) + ents = sorted(list(ents)) except: return 0 ndirs = 0 - for f in sorted(ents): + for f, inf in ents: fp = os.path.join(thumbpath, f) cmp = fp.lower().replace("\\", "/") # "top" or b64 prefix/full (a folder) if len(f) <= 3 or len(f) == 24: - age = now - bos.path.getmtime(fp) + age = now - inf.st_mtime if age > maxage: with self.mutex: safe = True @@ -469,16 +483,15 @@ class ThumbSrv(object): self.log("rm -rf [{}]".format(fp)) shutil.rmtree(fp, ignore_errors=True) else: - ndirs += self.clean(fp) + self._clean(histpath, cat, fp) + continue # thumb file try: b64, ts, ext = f.split(".") - if len(b64) != 24 or len(ts) != 8 or ext not in ["jpg", "webp"]: + if len(b64) != 24 or len(ts) != 8 or ext not in exts: raise Exception() - - ts = int(ts, 16) except: if f != "dir.txt": self.log("foreign file in thumbs dir: [{}]".format(fp), 1) @@ -489,6 +502,10 @@ class ThumbSrv(object): self.log("rm replaced [{}]".format(fp)) bos.unlink(prev_fp) + if cat != "th" and inf.st_mtime + maxage < now: + self.log("rm expired [{}]".format(fp)) + bos.unlink(fp) + prev_b64 = b64 prev_fp = fp