From b54b7213a7eb10b5764be846e492fae32441726a Mon Sep 17 00:00:00 2001 From: ed Date: Tue, 11 Jul 2023 22:15:37 +0000 Subject: [PATCH] more thumbnailer configs available as volflags: --th-convt = convt --th-no-crop = nocrop --th-size = thsize --- copyparty/__main__.py | 6 ++-- copyparty/authsrv.py | 4 +++ copyparty/cfg.py | 10 ++++-- copyparty/th_srv.py | 81 ++++++++++++++++++++++++------------------- copyparty/up2k.py | 9 +++-- copyparty/util.py | 4 +-- tests/util.py | 5 +-- 7 files changed, 72 insertions(+), 47 deletions(-) diff --git a/copyparty/__main__.py b/copyparty/__main__.py index 944de8b2..032bcbb8 100755 --- a/copyparty/__main__.py +++ b/copyparty/__main__.py @@ -1007,10 +1007,10 @@ def add_thumbnail(ap): ap2.add_argument("--no-thumb", action="store_true", help="disable all thumbnails (volflag=dthumb)") ap2.add_argument("--no-vthumb", action="store_true", help="disable video thumbnails (volflag=dvthumb)") ap2.add_argument("--no-athumb", action="store_true", help="disable audio thumbnails (spectrograms) (volflag=dathumb)") - ap2.add_argument("--th-size", metavar="WxH", default="320x256", help="thumbnail res") + ap2.add_argument("--th-size", metavar="WxH", default="320x256", help="thumbnail res (volflag=thsize)") ap2.add_argument("--th-mt", metavar="CORES", type=int, default=CORES, help="num cpu cores to use for generating thumbnails") - ap2.add_argument("--th-convt", metavar="SEC", type=int, default=60, help="conversion timeout in seconds") - ap2.add_argument("--th-no-crop", action="store_true", help="dynamic height; show full image") + ap2.add_argument("--th-convt", metavar="SEC", type=float, default=60, help="conversion timeout in seconds (volflag=convt)") + ap2.add_argument("--th-no-crop", action="store_true", help="dynamic height; show full image (volflag=nocrop)") ap2.add_argument("--th-dec", metavar="LIBS", default="vips,pil,ff", help="image decoders, in order of preference") ap2.add_argument("--th-no-jpg", action="store_true", help="disable jpg output") ap2.add_argument("--th-no-webp", action="store_true", help="disable webp output") diff --git a/copyparty/authsrv.py b/copyparty/authsrv.py index 419a4755..60b506bd 100644 --- a/copyparty/authsrv.py +++ b/copyparty/authsrv.py @@ -1420,6 +1420,10 @@ class AuthSrv(object): if k in vol.flags: vol.flags[k] = int(vol.flags[k]) + for k in ("convt",): + if k in vol.flags: + vol.flags[k] = float(vol.flags[k]) + for k1, k2 in IMPLICATIONS: if k1 in vol.flags: vol.flags[k2] = True diff --git a/copyparty/cfg.py b/copyparty/cfg.py index cfb8f2fa..c3f9e477 100644 --- a/copyparty/cfg.py +++ b/copyparty/cfg.py @@ -13,6 +13,7 @@ def vf_bmap() -> dict[str, str]: "no_dedup": "copydupes", "no_dupe": "nodupe", "no_forget": "noforget", + "th_no_crop": "nocrop", "dav_auth": "davauth", "dav_rt": "davrt", } @@ -40,8 +41,8 @@ def vf_bmap() -> dict[str, str]: def vf_vmap() -> dict[str, str]: """argv-to-volflag: simple values""" - ret = {} - for k in ("lg_sbf", "md_sbf", "unlist"): + ret = {"th_convt": "convt", "th_size": "thsize"} + for k in ("dbd", "lg_sbf", "md_sbf", "nrand", "unlist"): ret[k] = k return ret @@ -49,7 +50,7 @@ def vf_vmap() -> dict[str, str]: def vf_cmap() -> dict[str, str]: """argv-to-volflag: complex/lists""" ret = {} - for k in ("dbd", "html_head", "mte", "mth", "nrand"): + for k in ("html_head", "mte", "mth"): ret[k] = k return ret @@ -124,6 +125,9 @@ flagcats = { "dvthumb": "disables video thumbnails", "dathumb": "disables audio thumbnails (spectrograms)", "dithumb": "disables image thumbnails", + "thsize": "thumbnail res; WxH", + "nocrop": "disable center-cropping", + "convt": "conversion timeout in seconds", }, "handlers\n(better explained in --help-handlers)": { "on404=PY": "handle 404s by executing PY file", diff --git a/copyparty/th_srv.py b/copyparty/th_srv.py index 1c40e9ff..35070af8 100644 --- a/copyparty/th_srv.py +++ b/copyparty/th_srv.py @@ -13,6 +13,7 @@ import time from queue import Queue from .__init__ import ANYWIN, TYPE_CHECKING +from .authsrv import VFS from .bos import bos from .mtag import HAVE_FFMPEG, HAVE_FFPROBE, ffprobe from .util import ( @@ -110,8 +111,6 @@ class ThumbSrv(object): self.args = hub.args self.log_func = hub.log - res = hub.args.th_size.split("x") - self.res = tuple([int(x) for x in res]) self.poke_cd = Cooldown(self.args.th_poke) self.mutex = threading.Lock() @@ -119,7 +118,7 @@ class ThumbSrv(object): self.stopping = False self.nthr = max(1, self.args.th_mt) - self.q: Queue[Optional[tuple[str, str]]] = Queue(self.nthr * 4) + self.q: Queue[Optional[tuple[str, str, VFS]]] = Queue(self.nthr * 4) for n in range(self.nthr): Daemon(self.worker, "thumb-{}-{}".format(n, self.nthr)) @@ -184,6 +183,10 @@ class ThumbSrv(object): with self.mutex: return not self.nthr + def getres(self, vn: VFS) -> tuple[int, int]: + w, h = vn.flags["thsize"].split("x") + return int(w), int(h) + def get(self, ptop: str, rem: str, mtime: float, fmt: str) -> Optional[str]: histpath = self.asrv.vfs.histtab.get(ptop) if not histpath: @@ -211,7 +214,13 @@ class ThumbSrv(object): do_conv = True if do_conv: - self.q.put((abspath, tpath)) + allvols = list(self.asrv.vfs.all_vols.values()) + vn = next((x for x in allvols if x.realpath == ptop), None) + if not vn: + self.log("ptop [{}] not in {}".format(ptop, allvols), 3) + vn = self.asrv.vfs.all_aps[0][1] + + self.q.put((abspath, tpath, vn)) self.log("conv {} \033[0m{}".format(tpath, abspath), c=6) while not self.stopping: @@ -248,7 +257,7 @@ class ThumbSrv(object): if not task: break - abspath, tpath = task + abspath, tpath, vn = task ext = abspath.split(".")[-1].lower() png_ok = False funs = [] @@ -281,7 +290,7 @@ class ThumbSrv(object): for fun in funs: try: - fun(abspath, ttpath) + fun(abspath, ttpath, vn) break except Exception as ex: msg = "{} could not create thumbnail of {}\n{}" @@ -315,9 +324,10 @@ class ThumbSrv(object): with self.mutex: self.nthr -= 1 - def fancy_pillow(self, im: "Image.Image") -> "Image.Image": + def fancy_pillow(self, im: "Image.Image", vn: VFS) -> "Image.Image": # exif_transpose is expensive (loads full image + unconditional copy) - r = max(*self.res) * 2 + res = self.getres(vn) + r = max(*res) * 2 im.thumbnail((r, r), resample=Image.LANCZOS) try: k = next(k for k, v in ExifTags.TAGS.items() if v == "Orientation") @@ -331,23 +341,23 @@ class ThumbSrv(object): if rot in rots: im = im.transpose(rots[rot]) - if self.args.th_no_crop: - im.thumbnail(self.res, resample=Image.LANCZOS) + if "nocrop" in vn.flags: + im.thumbnail(res, resample=Image.LANCZOS) else: iw, ih = im.size - dw, dh = self.res + dw, dh = res res = (min(iw, dw), min(ih, dh)) im = ImageOps.fit(im, res, method=Image.LANCZOS) return im - def conv_pil(self, abspath: str, tpath: str) -> None: + def conv_pil(self, abspath: str, tpath: str, vn: VFS) -> None: with Image.open(fsenc(abspath)) as im: try: - im = self.fancy_pillow(im) + im = self.fancy_pillow(im, vn) except Exception as ex: self.log("fancy_pillow {}".format(ex), "90") - im.thumbnail(self.res) + im.thumbnail(self.getres(vn)) fmts = ["RGB", "L"] args = {"quality": 40} @@ -370,12 +380,12 @@ class ThumbSrv(object): im.save(tpath, **args) - def conv_vips(self, abspath: str, tpath: str) -> None: + def conv_vips(self, abspath: str, tpath: str, vn: VFS) -> None: crops = ["centre", "none"] - if self.args.th_no_crop: + if "nocrop" in vn.flags: crops = ["none"] - w, h = self.res + w, h = self.getres(vn) kw = {"height": h, "size": "down", "intent": "relative"} for c in crops: @@ -389,8 +399,8 @@ class ThumbSrv(object): img.write_to_file(tpath, Q=40) - def conv_ffmpeg(self, abspath: str, tpath: str) -> None: - ret, _ = ffprobe(abspath, int(self.args.th_convt / 2)) + def conv_ffmpeg(self, abspath: str, tpath: str, vn: VFS) -> None: + ret, _ = ffprobe(abspath, int(vn.flags["convt"] / 2)) if not ret: return @@ -402,12 +412,13 @@ class ThumbSrv(object): seek = [b"-ss", "{:.0f}".format(dur / 3).encode("utf-8")] scale = "scale={0}:{1}:force_original_aspect_ratio=" - if self.args.th_no_crop: + if "nocrop" in vn.flags: scale += "decrease,setsar=1:1" else: scale += "increase,crop={0}:{1},setsar=1:1" - bscale = scale.format(*list(self.res)).encode("utf-8") + res = self.getres(vn) + bscale = scale.format(*list(res)).encode("utf-8") # fmt: off cmd = [ b"ffmpeg", @@ -439,11 +450,11 @@ class ThumbSrv(object): ] cmd += [fsenc(tpath)] - self._run_ff(cmd) + self._run_ff(cmd, vn) - def _run_ff(self, cmd: list[bytes]) -> None: + def _run_ff(self, cmd: list[bytes], vn: VFS) -> None: # self.log((b" ".join(cmd)).decode("utf-8")) - ret, _, serr = runcmd(cmd, timeout=self.args.th_convt) + ret, _, serr = runcmd(cmd, timeout=vn.flags["convt"]) if not ret: return @@ -486,8 +497,8 @@ class ThumbSrv(object): self.log(t + txt, c=c) raise sp.CalledProcessError(ret, (cmd[0], b"...", cmd[-1])) - def conv_waves(self, abspath: str, tpath: str) -> None: - ret, _ = ffprobe(abspath, int(self.args.th_convt / 2)) + def conv_waves(self, abspath: str, tpath: str, vn: VFS) -> None: + ret, _ = ffprobe(abspath, int(vn.flags["convt"] / 2)) if "ac" not in ret: raise Exception("not audio") @@ -512,10 +523,10 @@ class ThumbSrv(object): # fmt: on cmd += [fsenc(tpath)] - self._run_ff(cmd) + self._run_ff(cmd, vn) - def conv_spec(self, abspath: str, tpath: str) -> None: - ret, _ = ffprobe(abspath, int(self.args.th_convt / 2)) + def conv_spec(self, abspath: str, tpath: str, vn: VFS) -> None: + ret, _ = ffprobe(abspath, int(vn.flags["convt"] / 2)) if "ac" not in ret: raise Exception("not audio") @@ -555,13 +566,13 @@ class ThumbSrv(object): ] cmd += [fsenc(tpath)] - self._run_ff(cmd) + self._run_ff(cmd, vn) - def conv_opus(self, abspath: str, tpath: str) -> None: + def conv_opus(self, abspath: str, tpath: str, vn: VFS) -> None: if self.args.no_acode: raise Exception("disabled in server config") - ret, _ = ffprobe(abspath, int(self.args.th_convt / 2)) + ret, _ = ffprobe(abspath, int(vn.flags["convt"] / 2)) if "ac" not in ret: raise Exception("not audio") @@ -597,7 +608,7 @@ class ThumbSrv(object): fsenc(tmp_opus) ] # fmt: on - self._run_ff(cmd) + self._run_ff(cmd, vn) # iOS fails to play some "insufficiently complex" files # (average file shorter than 8 seconds), so of course we @@ -621,7 +632,7 @@ class ThumbSrv(object): fsenc(tpath) ] # fmt: on - self._run_ff(cmd) + self._run_ff(cmd, vn) elif want_caf: # simple remux should be safe @@ -639,7 +650,7 @@ class ThumbSrv(object): fsenc(tpath) ] # fmt: on - self._run_ff(cmd) + self._run_ff(cmd, vn) if tmp_opus != tpath: try: diff --git a/copyparty/up2k.py b/copyparty/up2k.py index 274fb704..b6aaa69b 100644 --- a/copyparty/up2k.py +++ b/copyparty/up2k.py @@ -24,6 +24,7 @@ from queue import Queue from .__init__ import ANYWIN, PY2, TYPE_CHECKING, WINDOWS from .authsrv import LEELOO_DALLAS, VFS, AuthSrv from .bos import bos +from .cfg import vf_bmap, vf_vmap from .fsutil import Fstab from .mtag import MParser, MTag from .util import ( @@ -757,8 +758,9 @@ class Up2k(object): ff = "\033[0;35m{}{:.0}" fv = "\033[0;36m{}:\033[90m{}" fx = set(("html_head",)) - fdl = ("dbd", "lg_sbf", "md_sbf", "mte", "mth", "mtp", "nrand", "rand") - fd = {x: x for x in fdl} + fd = vf_bmap() + fd.update(vf_vmap()) + fd = {v: k for k, v in fd.items()} fl = { k: v for k, v in flags.items() @@ -769,6 +771,9 @@ class Up2k(object): for k, v in fl.items() if k not in fx ] + if not a: + a = ["\033[90mall-default"] + if a: vpath = "?" for k, v in self.asrv.vfs.all_vols.items(): diff --git a/copyparty/util.py b/copyparty/util.py index 263bec25..d3eb06d1 100644 --- a/copyparty/util.py +++ b/copyparty/util.py @@ -2427,7 +2427,7 @@ def killtree(root: int) -> None: def runcmd( - argv: Union[list[bytes], list[str]], timeout: Optional[int] = None, **ka: Any + argv: Union[list[bytes], list[str]], timeout: Optional[float] = None, **ka: Any ) -> tuple[int, str, str]: kill = ka.pop("kill", "t") # [t]ree [m]ain [n]one capture = ka.pop("capture", 3) # 0=none 1=stdout 2=stderr 3=both @@ -2480,7 +2480,7 @@ def chkcmd(argv: Union[list[bytes], list[str]], **ka: Any) -> tuple[str, str]: return sout, serr -def mchkcmd(argv: Union[list[bytes], list[str]], timeout: int = 10) -> None: +def mchkcmd(argv: Union[list[bytes], list[str]], timeout: float = 10) -> None: if PY2: with open(os.devnull, "wb") as f: rv = sp.call(argv, stdout=f, stderr=f) diff --git a/tests/util.py b/tests/util.py index 59b2bb7a..2dc313ce 100644 --- a/tests/util.py +++ b/tests/util.py @@ -98,7 +98,7 @@ class Cfg(Namespace): def __init__(self, a=None, v=None, c=None): ka = {} - ex = "daw dav_auth dav_inf dav_mac dav_rt dotsrch e2d e2ds e2dsa e2t e2ts e2tsr e2v e2vu e2vp ed emp force_js getmod grid hardlink ih ihead magic never_symlink nid nih no_acode no_athumb no_dav no_dedup no_del no_dupe no_logues no_mv no_readme no_robots no_sb_md no_sb_lg no_scandir no_thumb no_vthumb no_zip nrand nw rand smb vc xdev xlink xvol" + ex = "daw dav_auth dav_inf dav_mac dav_rt dotsrch e2d e2ds e2dsa e2t e2ts e2tsr e2v e2vu e2vp ed emp force_js getmod grid hardlink ih ihead magic never_symlink nid nih no_acode no_athumb no_dav no_dedup no_del no_dupe no_logues no_mv no_readme no_robots no_sb_md no_sb_lg no_scandir no_thumb no_vthumb no_zip nrand nw rand smb th_no_crop vc xdev xlink xvol" ka.update(**{k: False for k in ex.split()}) ex = "dotpart no_rescan no_sendfile no_voldump plain_ip" @@ -107,7 +107,7 @@ class Cfg(Namespace): ex = "css_browser hist js_browser no_forget no_hash no_idx" ka.update(**{k: None for k in ex.split()}) - ex = "s_thead s_tbody" + ex = "s_thead s_tbody th_convt" ka.update(**{k: 9 for k in ex.split()}) ex = "df loris re_maxage rproxy rsp_jtr rsp_slp s_wr_slp theme themes turbo" @@ -126,6 +126,7 @@ class Cfg(Namespace): E=E, dbd="wal", s_wr_sz=512 * 1024, + th_size="320x256", unpost=600, u2sort="s", mtp=[],