diff --git a/copyparty/__main__.py b/copyparty/__main__.py index 9c5f9484..6f929c7c 100755 --- a/copyparty/__main__.py +++ b/copyparty/__main__.py @@ -20,7 +20,18 @@ import time import traceback import uuid -from .__init__ import ANYWIN, CORES, EXE, MACOS, PY2, VT100, WINDOWS, E, EnvParams, unicode +from .__init__ import ( + ANYWIN, + CORES, + EXE, + MACOS, + PY2, + VT100, + WINDOWS, + E, + EnvParams, + unicode, +) from .__version__ import CODENAME, S_BUILD_DT, S_VERSION from .authsrv import expand_config_file, split_cfg_ln, upgrade_cfg_fmt from .cfg import flagcats, onedash @@ -1139,6 +1150,7 @@ def add_thumbnail(ap): ap2.add_argument("--th-size", metavar="WxH", default="320x256", help="thumbnail res (volflag=thsize)") ap2.add_argument("--th-mt", metavar="CORES", type=int, default=CORES, help="num cpu cores to use for generating thumbnails") ap2.add_argument("--th-convt", metavar="SEC", type=float, default=60, help="conversion timeout in seconds (volflag=convt)") + ap2.add_argument("--th-ram-max", metavar="GB", type=float, default=6, help="max memory usage (GiB) permitted by thumbnailer; not very accurate") ap2.add_argument("--th-no-crop", action="store_true", help="dynamic height; show full image by default (client can override in UI) (volflag=nocrop)") ap2.add_argument("--th-dec", metavar="LIBS", default="vips,pil,ff", help="image decoders, in order of preference") ap2.add_argument("--th-no-jpg", action="store_true", help="disable jpg output") diff --git a/copyparty/th_srv.py b/copyparty/th_srv.py index 27a16bc9..a5826f1f 100644 --- a/copyparty/th_srv.py +++ b/copyparty/th_srv.py @@ -129,6 +129,8 @@ class ThumbSrv(object): self.mutex = threading.Lock() self.busy: dict[str, list[threading.Condition]] = {} + self.ram: dict[str, float] = {} + self.memcond = threading.Condition(self.mutex) self.stopping = False self.nthr = max(1, self.args.th_mt) @@ -214,7 +216,7 @@ class ThumbSrv(object): with self.mutex: try: self.busy[tpath].append(cond) - self.log("wait {}".format(tpath)) + self.log("joined waiting room for %s" % (tpath,)) except: thdir = os.path.dirname(tpath) bos.makedirs(os.path.join(thdir, "w")) @@ -265,6 +267,23 @@ class ThumbSrv(object): "ffa": self.fmt_ffa, } + def wait4ram(self, need: float, ttpath: str) -> None: + ram = self.args.th_ram_max + if need > ram * 0.99: + t = "file too big; need %.2f GiB RAM, but --th-ram-max is only %.1f" + raise Exception(t % (need, ram)) + + while True: + with self.mutex: + used = sum([v for k, v in self.ram.items() if k != ttpath]) + need + if used < ram: + # self.log("XXX self.ram: %s" % (self.ram,), 5) + self.ram[ttpath] = need + return + with self.memcond: + # self.log("at RAM limit; used %.2f GiB, need %.2f more" % (used-need, need), 1) + self.memcond.wait(3) + def worker(self) -> None: while not self.stopping: task = self.q.get() @@ -330,11 +349,15 @@ class ThumbSrv(object): with self.mutex: subs = self.busy[tpath] del self.busy[tpath] + self.ram.pop(ttpath, None) for x in subs: with x: x.notify_all() + with self.memcond: + self.memcond.notify_all() + with self.mutex: self.nthr -= 1 @@ -366,6 +389,7 @@ class ThumbSrv(object): return im def conv_pil(self, abspath: str, tpath: str, fmt: str, vn: VFS) -> None: + self.wait4ram(0.2, tpath) with Image.open(fsenc(abspath)) as im: try: im = self.fancy_pillow(im, fmt, vn) @@ -395,6 +419,7 @@ class ThumbSrv(object): im.save(tpath, **args) def conv_vips(self, abspath: str, tpath: str, fmt: str, vn: VFS) -> None: + self.wait4ram(0.2, tpath) crops = ["centre", "none"] if fmt.endswith("f"): crops = ["none"] @@ -415,6 +440,7 @@ class ThumbSrv(object): img.write_to_file(tpath, Q=40) def conv_ffmpeg(self, abspath: str, tpath: str, fmt: str, vn: VFS) -> None: + self.wait4ram(0.2, tpath) ret, _ = ffprobe(abspath, int(vn.flags["convt"] / 2)) if not ret: return @@ -517,8 +543,21 @@ class ThumbSrv(object): if "ac" not in ret: raise Exception("not audio") - flt = ( - b"[0:a:0]" + # jt_versi.xm: 405M/839s + dur = ret[".dur"][1] if ".dur" in ret else 300 + need = 0.2 + dur / 3000 + speedup = b"" + if need > self.args.th_ram_max * 0.7: + self.log("waves too big (need %.2f GiB); trying to optimize" % (need,)) + need = 0.2 + dur / 4200 # only helps about this much... + speedup = b"aresample=8000," + if need > self.args.th_ram_max * 0.96: + raise Exception("file too big; cannot waves") + + self.wait4ram(need, tpath) + + flt = b"[0:a:0]" + speedup + flt += ( b"compand=.3|.3:1|1:-90/-60|-60/-40|-40/-30|-20/-20:6:0:-90:0.2" b",volume=2" b",showwavespic=s=2048x64:colors=white" @@ -545,6 +584,15 @@ class ThumbSrv(object): if "ac" not in ret: raise Exception("not audio") + # https://trac.ffmpeg.org/ticket/10797 + # expect 1 GiB every 600 seconds when duration is tricky; + # simple filetypes are generally safer so let's special-case those + safe = ("flac", "wav", "aif", "aiff", "opus") + coeff = 1800 if abspath.split(".")[-1].lower() in safe else 600 + dur = ret[".dur"][1] if ".dur" in ret else 300 + need = 0.2 + dur / coeff + self.wait4ram(need, tpath) + fc = "[0:a:0]aresample=48000{},showspectrumpic=s=640x512,crop=780:544:70:50[o]" if self.args.th_ff_swr: @@ -587,6 +635,7 @@ class ThumbSrv(object): if self.args.no_acode: raise Exception("disabled in server config") + self.wait4ram(0.2, tpath) ret, _ = ffprobe(abspath, int(vn.flags["convt"] / 2)) if "ac" not in ret: raise Exception("not audio")