th: custom thumbnail extractors (#1602)

Signed-off-by: ed <s@ocv.me>
Co-authored-by: ed <s@ocv.me>

bin/thumbs/README.md

@@ -0,0 +1,52 @@
+# thumbnail extractors
+
+extract (or generate) thumbnails for custom file formats or override default thumbnail extraction for standard media files
+
+
+## usage
+
+* load plugins with `--th-extract baz=~/dev/copyparty/bin/thumbs/randomcolor.py`
+  * `baz` is the file extension (may be comma-separated list of extensions)
+  * add multiple different extractor plugins by repeating the `--th-extract` argument
+
+
+## api
+
+### in
+
+each plugin must define a function `main(abspath, **kwargs)`:
+* `abspath` is the path to a file to extract thumbnail for
+* `kwargs` currently receives these keyword arguments:
+  * `vn` is the VFS which contains the requested file
+  * `th_srv` is an instance of [copyparty/th_srv](https://github.com/9001/copyparty/blob/hovudstraum/copyparty/th_srv.py)
+
+### out
+
+the `main()` function must
+* return `None` or raise an `Exception` in case it is unable to extract thumbnail
+* return tuple `fmt, stream, offset, whence, size` otherwise:
+  * `fmt: str` – format (filename extension) of the thumbnail; should be one of the image formats supported by copyparty (see `--th-r-*` options)
+  * `stream: IO[bytes]` – binary file-like object supporting `seek()`, `read()`, `close()` methods; should contain extracted thumbnail image
+  * `offset: int, whence: int` – arguments to `stream.seek()`; copyparty will call `stream.seek(offset, whence)` once before starting reading the stream contents
+  * `size: int | None` – number of bytes to read from the `stream`; if `size` is negative or `None`, copyparty will read until the end of the stream
+
+
+### notes
+
+it is possible to extract or generate custom thumbnails for standard media files too,
+e.g. run copyparty with `--th-extract mp3,m4a,aac,flac,opus=~/dev/copyparty/bin/thumbs/my_custom_mp3_th_extractor.py` and do your magic with audio file thumbnails
+
+in case copyparty doesn't like the `fmt` option, it will `close()` the `stream` without ever calling `seek()` or `read()`
+
+if your scenario involves heavy computation or i/o, it is better to defer the heavy parts until the first `seek()` or `read()` to avoid redundant work in case the result gets rejected based on `fmt` value
+
+if you provide custom `stream` object:
+copyparty performs buffered reading, so expect multiple `read()` calls, respect `size` argument in `read(size)` calls, and maintain correct seek position
+
+
+## examples
+
+
+## some other known plugins seen on the internets
+
+* [fpkg_thumb](https://github.com/kamaeff/copyparty-dumb-fpkgi-handler/blob/master/fpkg_thumb.py) extracts cover images from playstation4 software installation packages ("pkg" and "fpkg" files)

copyparty/__main__.py

@@ -777,6 +777,54 @@ def get_sects():
             ).rstrip()
             + build_flags_desc(),
         ],
+        [
+            "thumb-extractors",
+            "extract thumbnails from custom file formats with python scripts",
+            dedent(
+                """
+            copyparty is able to extract thumbnails from various media formats:
+            images, audio, video, ebooks
+
+            if you want to extract thumbnails from unsupported file formats,
+            you can load a plugin, which extracts image data from custom formats
+
+            load the plugin using --args; for example \033[36m
+             --tx-extract iso,mdf,dmg=~/party-thumb-extractors/th_diskimg.py
+            \033[0m
+            the file must define the function \033[35mmain(abspath, **kwargs)\033[0m:
+             \033[35mabspath\033[0m: path to the file to extract thumbnail from
+             \033[35mkwargs\033[0m currently have:
+             \033[35mvn\033[0m: the VFS which contains the requested file
+             \033[35mrth_srv\033[0m: the copyparty ThumbSrv instance
+
+            if `main` couldn't extract a thumbnail, it must return None;
+            otherwise the return value must be a tuple:
+             \033[35mreturn fmt, stream, offset, whence, size\033[0m
+
+            > \033[32mfmt\033[0m: thumbnail format, should be one of the formats
+                supported by copyparty; e.g. "png", "jpg"
+
+            > \033[32mstream\033[0m: binary file-like object containing the thumbnail,
+                supporting seek(), read(), and close() methods
+
+            > \033[32moffset, whence\033[0m: arguments to seek() method;
+                stream.seek(offset, whence) will be called once before copyparty
+                starts reading from the stream
+
+            > \033[32msize\033[0m: number of bytes to read ftom the stream;
+                if negative, copyparty will read through the end of the stream;
+
+            you can also supply extractors for standard formats to override default
+            thumbnail extraction behaviour; for example \033[36m
+             --tx-extract mp3=~/party-thumb-extractors/th_mp3.py
+            \033[0mwill let you extract or generate custom thumbnails for mp3 files
+
+            \033[1;35mPS!\033[0m the folder that contains the python file should ideally
+              not contain many other python files, and especially nothing
+              with filenames that overlap with modules used by copyparty
+            """
+            ),
+        ],
         [
             "handlers",
             "use plugins to handle certain events",
@@ -1838,8 +1886,19 @@ def add_thumbnail(ap):
     ap2.add_argument("--th-r-ffi", metavar="T,T", type=u, default="apng,avif,avifs,bmp,cbz,dds,dib,epub,fit,fits,fts,gif,hdr,heic,heics,heif,heifs,icns,ico,jp2,jpeg,jpg,jpx,jxl,kra,ora,pbm,pcx,pfm,pgm,png,pnm,ppm,psd,qoi,sgi,tga,tif,tiff,webp,xbm,xpm", help="image formats to decode using ffmpeg")
     ap2.add_argument("--th-r-ffv", metavar="T,T", type=u, default="3gp,asf,av1,avc,avi,flv,h264,h265,hevc,m4v,mjpeg,mjpg,mkv,mov,mp4,mpeg,mpeg2,mpegts,mpg,mpg2,mts,nut,ogm,ogv,rm,ts,vob,webm,wmv", help="video formats to decode using ffmpeg")
     ap2.add_argument("--th-r-ffa", metavar="T,T", type=u, default="aac,ac3,aif,aiff,alac,alaw,amr,apac,ape,au,bcstm,bfstm,brstm,bonk,dfpwm,dts,flac,gsm,ilbc,it,itgz,itxz,itz,m4a,m4b,m4r,mdgz,mdxz,mdz,mka,mo3,mod,mp2,mp3,mpc,mptm,mt2,mulaw,oga,ogg,okt,opus,ra,s3m,s3gz,s3xz,s3z,tak,tta,ulaw,wav,wma,wv,xm,xmgz,xmxz,xmz,xpk", help="audio formats to decode using ffmpeg")
+
+    ap2.add_argument("--th-pil-add", metavar="T", type=u, default="", help="add or remove image formats to decode using pillow listed in \033[33m--th-r-pil\033[0m; example: [\033[32mfb2,pdf,-psd,-bmp\033[0m] (add fb2 and pdf, remove psd and bmp)")
+    ap2.add_argument("--th-vips-add", metavar="T", type=u, default="", help="add or remove formats; same as \033[33m--th-pil-add\033[0m but for pyvips")
+    ap2.add_argument("--th-raw-add", metavar="T", type=u, default="", help="add or remove formats; same as \033[33m--th-pil-add\033[0m but for rawpy/libraw")
+    ap2.add_argument("--th-ffi-add", metavar="T", type=u, default="", help="add or remove formats; same as \033[33m--th-pil-add\033[0m but for ffmpeg images")
+    ap2.add_argument("--th-ffv-add", metavar="T", type=u, default="", help="add or remove formats; same as \033[33m--th-pil-add\033[0m but for ffmpeg videos")
+    ap2.add_argument("--th-ffa-add", metavar="T", type=u, default="", help="add or remove formats; same as \033[33m--th-pil-add\033[0m but for ffmpeg audio")
+
     ap2.add_argument("--th-spec-cnv", metavar="T", type=u, default="it,itgz,itxz,itz,mdgz,mdxz,mdz,mo3,mod,s3m,s3gz,s3xz,s3z,xm,xmgz,xmxz,xmz,xpk", help="audio formats which provoke https://trac.ffmpeg.org/ticket/10797 (huge ram usage for s3xmodit spectrograms)")
     ap2.add_argument("--au-unpk", metavar="E=F.C", type=u, default="mdz=mod.zip, mdgz=mod.gz, mdxz=mod.xz, s3z=s3m.zip, s3gz=s3m.gz, s3xz=s3m.xz, xmz=xm.zip, xmgz=xm.gz, xmxz=xm.xz, itz=it.zip, itgz=it.gz, itxz=it.xz, cbz=jpg.cbz, epub=jpg.epub, kra=png.kra, ora=png.ora", help="audio/image formats to decompress before passing to ffmpeg")
+    ap2.add_argument("--th-extract", metavar="T", type=u, action="append", help="\033[34mREPEATABLE:\033[0m list of file extensions to thumbnail using a custom plugin (a python script); example: [\033[32mmdf,iso,dmg=/handlers/th_diskimg.py\033[0m]")
+    ap2.add_argument("--th-extr-sz", metavar="M", type=int, default=16, help="max num megabytes to allow \033[33m--th-extract\033[0m plugins to extract from each file")
+    ap2.add_argument("--hot-th-extr", action="store_true", help="recompile extractors on each thumbnail extraction -- expensive but convenient when hacking on stuff")
 
 
 def add_transcoding(ap):

copyparty/authsrv.py

@@ -2379,6 +2379,7 @@ class AuthSrv(object):
                 vol.flags["dvthumb"] = True
                 vol.flags["dathumb"] = True
                 vol.flags["dithumb"] = True
+                vol.flags["dethumb"] = True
 
         have_fk = False
         for vol in vfs.all_nodes.values():

copyparty/cfg.py

@@ -316,6 +316,7 @@ flagcats = {
         "dvthumb": "disables video thumbnails",
         "dathumb": "disables audio thumbnails (spectrograms)",
         "dithumb": "disables image thumbnails",
+        "dethumb": "disables custom thumbnails (--th-extract)",
         "pngquant": "compress audio waveforms 33% better",
         "thsize": "thumbnail res; WxH",
         "crop": "center-cropping (y/n/fy/fn)",

copyparty/httpcli.py

@@ -7893,6 +7893,10 @@ class HttpCli(object):
                         ext in self.thumbcli.fmt_pil
                         or ext in self.thumbcli.fmt_vips
                         or ext in self.thumbcli.fmt_ffi
+                        or (
+                            ext in self.thumbcli.thumbable
+                            and ext not in self.thumbcli.thumbable_native
+                        )
                     )
                     is_vid = ext in self.thumbcli.fmt_ffv
                     is_au = ext in self.thumbcli.fmt_ffa

copyparty/svchub.py

@@ -133,6 +133,19 @@ VER_SHARES_DB = 2
 CVE_SEVS = {"low": 1, "medium": 2, "moderate": 2, "high": 3, "critical": 4}
 
 
+def _build_th_fset(f: str, fadd: str) -> set[str]:
+    fset = {x for x in f.replace(" ", "").split(",")}
+    rmset = {""}
+
+    for x in fadd.replace(" ", "").split(","):
+        if x.startswith("-"):
+            rmset.add(x.lstrip("-"))
+        else:
+            fset.add(x)
+
+    return fset - rmset
+
+
 class SvcHub(object):
     """
     Hosts all services which cannot be parallelized due to reliance on monolithic resources.
@@ -441,6 +454,20 @@ class SvcHub(object):
         zlss = [x.strip().lower().split("=", 1) for x in args.au_unpk.split(",")]
         args.au_unpk = {x[0]: x[1] for x in zlss}
 
+        args.th_r_pil = _build_th_fset(args.th_r_pil, args.th_pil_add)
+        args.th_r_vips = _build_th_fset(args.th_r_vips, args.th_vips_add)
+        args.th_r_raw = _build_th_fset(args.th_r_raw, args.th_raw_add)
+        args.th_r_ffi = _build_th_fset(args.th_r_ffi, args.th_ffi_add)
+        args.th_r_ffv = _build_th_fset(args.th_r_ffv, args.th_ffv_add)
+        args.th_r_ffa = _build_th_fset(args.th_r_ffa, args.th_ffa_add)
+
+        th_extract = args.th_extract or []
+        args.th_extract = {}
+        for arg in th_extract:
+            exts, script = arg.split("=", 1)
+            for ext in exts.split(","):
+                args.th_extract[ext.strip().lower()] = script
+
         self.args.th_dec = list(decs.keys())
         self.thumbsrv = None
         want_ff = False
@@ -478,7 +505,7 @@ class SvcHub(object):
                 t = "invalid mp3 transcoding quality [%s] specified; only supports [0] to disable, a CBR value such as [192k], or a CQ/CRF value such as [v2]"
                 raise Exception(t % (args.q_mp3,))
         else:
-            zss = set(args.th_r_ffa.split(",") + args.th_r_ffv.split(","))
+            zss = args.th_r_ffa | args.th_r_ffv
             args.au_unpk = {
                 k: v for k, v in args.au_unpk.items() if v.split(".")[0] not in zss
             }

copyparty/th_cli.py

@@ -34,6 +34,7 @@ class ThumbCli(object):
         self.cooldown = Cooldown(self.args.th_poke) if self.args.th_poke else None
 
         self.thumbable = c["thumbable"]
+        self.thumbable_native = c["thumbable_native"]
         self.fmt_pil = c["pil"]
         self.fmt_vips = c["vips"]
         self.fmt_raw = c["raw"]
@@ -59,6 +60,10 @@ class ThumbCli(object):
         if is_vid and "dvthumb" in dbv.flags:
             return None
 
+        is_custom = ext not in self.thumbable_native
+        if is_custom and "dethumb" in dbv.flags:
+            return None
+
         want_opus = fmt in EXTS_AC
         is_au = ext in self.fmt_ffa
         is_vau = want_opus and ext in self.fmt_ffv
@@ -76,7 +81,7 @@ class ThumbCli(object):
         elif want_opus:
             return None
 
-        is_img = not is_vid and not is_au
+        is_img = not is_vid and not is_au and not is_custom
         if is_img and "dithumb" in dbv.flags:
             return None
 

copyparty/th_srv.py

@@ -38,6 +38,7 @@ from .util import (
     afsenc,
     atomic_move,
     fsenc,
+    loadpy,
     min_ex,
     runcmd,
     statdir,
@@ -323,24 +324,12 @@ class ThumbSrv(object):
         if self.args.th_clean:
             Daemon(self.cleaner, "thumb.cln")
 
-        (
-            self.fmt_pil,
-            self.fmt_vips,
-            self.fmt_raw,
-            self.fmt_ffi,
-            self.fmt_ffv,
-            self.fmt_ffa,
-        ) = [
-            set(y.split(","))
-            for y in [
-                self.args.th_r_pil,
-                self.args.th_r_vips,
-                self.args.th_r_raw,
-                self.args.th_r_ffi,
-                self.args.th_r_ffv,
-                self.args.th_r_ffa,
-            ]
-        ]
+        self.fmt_pil = self.args.th_r_pil
+        self.fmt_vips = self.args.th_r_vips
+        self.fmt_raw = self.args.th_r_raw
+        self.fmt_ffi = self.args.th_r_ffi
+        self.fmt_ffv = self.args.th_r_ffv
+        self.fmt_ffa = self.args.th_r_ffa
 
         if not H_PIL_HEIF:
             for f in "heif heifs heic heics".split(" "):
@@ -359,23 +348,26 @@ class ThumbSrv(object):
                 self.fmt_pil.discard(f)
 
         self.thumbable: set[str] = set()
+        self.thumbable_native: set[str] = set()
         self._build_thumbable()
 
     def _build_thumbable(self) -> None:
-        self.thumbable.clear()
+        self.thumbable_native.clear()
 
         if "pil" in self.args.th_dec:
-            self.thumbable |= self.fmt_pil
+            self.thumbable_native |= self.fmt_pil
 
         if "vips" in self.args.th_dec:
-            self.thumbable |= self.fmt_vips
+            self.thumbable_native |= self.fmt_vips
 
         if "raw" in self.args.th_dec:
-            self.thumbable |= self.fmt_raw
+            self.thumbable_native |= self.fmt_raw
 
         if "ff" in self.args.th_dec:
             for zss in [self.fmt_ffi, self.fmt_ffv, self.fmt_ffa]:
-                self.thumbable |= zss
+                self.thumbable_native |= zss
+
+        self.thumbable = self.thumbable_native | set(self.args.th_extract)
 
     def _log(self, msg: str, c: Union[int, str] = 0) -> None:
         self.log_func("thumb", msg, c)
@@ -467,6 +459,7 @@ class ThumbSrv(object):
     def getcfg(self) -> dict[str, set[str]]:
         return {
             "thumbable": self.thumbable,
+            "thumbable_native": self.thumbable_native,
             "pil": self.fmt_pil,
             "vips": self.fmt_vips,
             "raw": self.fmt_raw,
@@ -549,19 +542,38 @@ class ThumbSrv(object):
             png_ok = False
             funs = []
 
+            tex = tpath.rsplit(".", 1)[-1]
+            want_mp3 = tex == "mp3"
+            want_opus = tex in ("opus", "owa", "caf")
+            want_flac = tex == "flac"
+            want_wav = tex == "wav"
+            want_png = tex == "png"
+            want_au = want_mp3 or want_opus or want_flac or want_wav
+
+            ap_extr = abspath
+            if (
+                ext in self.args.th_extract
+                and not want_au
+                and not want_png
+                and "dethumb" not in vn.flags
+            ):
+                ap_extr = self.run_extractor(self.args.th_extract[ext], abspath, vn)
+                if ap_extr:
+                    ext = ap_extr.rsplit(".", 1)[-1]
+
             if ext in self.args.au_unpk:
                 ap_unpk = au_unpk(self.log, self.args.au_unpk, abspath, vn)
+                ap_unpk = au_unpk(self.log, self.args.au_unpk, ap_extr, vn)
+            elif ext in self.thumbable_native:
+                ap_unpk = ap_extr
             else:
                 ap_unpk = abspath
+                ap_unpk = ""
+
+            if ap_extr and ap_extr != ap_unpk and ap_extr != abspath:
+                wunlink(self.log, ap_extr, vn.flags)
 
             if ap_unpk and not bos.path.exists(tpath):
-                tex = tpath.rsplit(".", 1)[-1]
-                want_mp3 = tex == "mp3"
-                want_opus = tex in ("opus", "owa", "caf")
-                want_flac = tex == "flac"
-                want_wav = tex == "wav"
-                want_png = tex == "png"
-                want_au = want_mp3 or want_opus or want_flac or want_wav
                 for lib in self.args.th_dec:
                     can_au = lib == "ff" and (
                         ext in self.fmt_ffa or ext in self.fmt_ffv
@@ -669,6 +681,66 @@ class ThumbSrv(object):
         with self.mutex:
             self.nthr -= 1
 
+    def run_extractor(self, extr: str, abspath: str, vn: VFS) -> str:
+        try:
+            mod = loadpy(extr, self.args.hot_th_extr)
+        except Exception as ex:
+            self.log("extractor import failed; " + min_ex(), 1)
+            return ""
+
+        fd = 0
+        ret = ""
+        stream = None
+        try:
+            res = mod.main(abspath, vn=vn, th_srv=self)
+            if not res:
+                t = "extractor %r returned no data for file %r"
+                self.log(t % (extr, abspath))
+                return ""
+
+            outext, stream, offset, whence, size = res
+            if (
+                outext not in self.thumbable_native
+                or outext in self.fmt_ffa
+                or outext in self.fmt_ffv
+            ):
+                t = "extractor %r returned unsupported format %r"
+                self.log(t % (extr, outext))
+                return ""
+
+            remains = size if size is not None and size >= 0 else -1
+
+            stream.seek(offset, whence)
+            fd, ret = tempfile.mkstemp("." + outext)
+            bufsz = rsz = self.args.iobuf
+            maxsz = self.args.th_extr_sz * 1048576
+            outsz = 0
+            with os.fdopen(fd, "wb", rsz) as f:
+                fd = 0
+                while True:
+                    if remains < rsz and remains >= 0:
+                        bufsz = remains
+                    buf = stream.read(bufsz)
+                    if not buf:
+                        break
+                    remains -= len(buf)
+                    outsz += len(buf)
+                    if outsz >= maxsz:
+                        raise Exception("too large")
+                    f.write(buf)
+            return ret
+        except Exception as e:
+            if fd:
+                os.close(fd)
+            if ret:
+                wunlink(self.log, ret, vn.flags)
+            t = "failed to extract thumbnail from %r: %s"
+            self.log(t % (abspath, min_ex()))
+            return ""
+        finally:
+            if stream:
+                stream.close()
+
     def fancy_pillow(self, im: "Image.Image", fmt: str, vn: VFS) -> "Image.Image":
         # exif_transpose is expensive (loads full image + unconditional copy)
         res = self.getres(vn, fmt)