drop rawpy, use libraw/dcraw_emu directly;

rawpy is still supported but will not be bundled by default
due to security concerns

dcraw_emu reads more formats than rawpy + gives better quality
(we told rawpy to use embedded thumbs), so also much slower

dcraw_emu must be combined with libvips or pillow (equivalent)

other alternatives considered:

libvips + a full imagemagick does a different subset of formats,
less than dcraw_emu, yet is 3x slower and eats ram

magick wins wrt formats but is even slower (4x of dcraw_emu)

README.md

@@ -302,7 +302,7 @@ also see [comparison to similar software](./docs/versus.md)
     * ☑ realtime streaming of growing files (logfiles and such)
   * ☑ [thumbnails](#thumbnails)
     * ☑ ...of images using Pillow, pyvips, or FFmpeg
-    * ☑ ...of RAW images using rawpy
+    * ☑ ...of RAW images using libraw-dcraw_emu or rawpy
     * ☑ ...of videos using FFmpeg
     * ☑ ...of audio (spectrograms) using FFmpeg
     * ☑ cache eviction (max-age; maybe max-size eventually)
@@ -3206,7 +3206,7 @@ enable [thumbnails](#thumbnails) of...
 * **HEIF pictures:** `pyvips` or `ffmpeg` or `pillow-heif`
 * **AVIF pictures:** `pyvips` or `ffmpeg` or `pillow-avif-plugin` or pillow v11.3+
 * **JPEG XL pictures:** `pyvips` or `ffmpeg`
-* **RAW images:** `rawpy`, plus one of `pyvips` or `Pillow` (for some formats)
+* **RAW photos:** either `libraw dcraw_emu` or `rawpy`, plus either `pyvips` or `Pillow`
 
 enable sending [zeromq messages](#zeromq) from event-hooks: `pyzmq`
 
@@ -3232,6 +3232,7 @@ set any of the following environment variables to disable its associated optiona
 | -------------------- | ------------ |
 | `PRTY_NO_ARGON2`     | disable argon2-cffi password hashing |
 | `PRTY_NO_CFSSL`      | never attempt to generate self-signed certificates using [cfssl](https://github.com/cloudflare/cfssl) |
+| `PRTY_NO_DCRAW`      | disable all [libraw](https://www.libraw.org/homepage)-based thumbnail support for RAW images |
 | `PRTY_NO_FFMPEG`     | **audio transcoding** goes byebye, **thumbnailing** must be handled by Pillow/libvips |
 | `PRTY_NO_FFPROBE`    | **audio transcoding** goes byebye, **thumbnailing** must be handled by Pillow/libvips, **metadata-scanning** must be handled by mutagen |
 | `PRTY_NO_MAGIC`      | do not use [magic](https://pypi.org/project/python-magic/) for filetype detection |
@@ -3246,7 +3247,8 @@ set any of the following environment variables to disable its associated optiona
 | `PRTY_NO_PIL_WEBP`   | disable use of native webp support in Pillow |
 | `PRTY_NO_PSUTIL`     | do not use [psutil](https://pypi.org/project/psutil/) for reaping stuck hooks and plugins on Windows |
 | `PRTY_NO_PYFTPD`     | disable ftp(s) server ([pyftpdlib](https://pypi.org/project/pyftpdlib/)-based) |
-| `PRTY_NO_RAW`        | disable all [rawpy](https://pypi.org/project/rawpy/)-based thumbnail support for RAW images |
+| `PRTY_NO_RAW`        | same as `PRTY_NO_DCRAW` plus `PRTY_NO_RAWPY` |
+| `PRTY_NO_RAWPY`      | disable all [rawpy](https://pypi.org/project/rawpy/)-based thumbnail support for RAW images |
 | `PRTY_NO_VIPS`       | disable all [libvips](https://pypi.org/project/pyvips/)-based thumbnail support; will fallback to Pillow or ffmpeg |
 
 example: `PRTY_NO_PIL=1 python3 copyparty-sfx.py`

copyparty/__main__.py

@@ -1775,8 +1775,8 @@ def add_thumbnail(ap):
     # https://stackoverflow.com/a/47612661
     # ffmpeg -hide_banner -demuxers | awk '/^ D  /{print$2}' | while IFS= read -r x; do ffmpeg -hide_banner -h demuxer=$x; done | grep -E '^Demuxer |extensions:'
     ap2.add_argument("--th-r-pil", metavar="T,T", type=u, default="avif,avifs,blp,bmp,cbz,dcx,dds,dib,emf,eps,epub,fits,flc,fli,fpx,gif,heic,heics,heif,heifs,icns,ico,im,j2p,j2k,jp2,jpeg,jpg,jpx,jxl,pbm,pcx,pgm,png,pnm,ppm,psd,qoi,sgi,spi,tga,tif,tiff,webp,wmf,xbm,xpm", help="image formats to decode using pillow")
-    ap2.add_argument("--th-r-vips", metavar="T,T", type=u, default="3fr,avif,cr2,cr3,crw,dcr,dng,erf,exr,fit,fits,fts,gif,hdr,heic,heics,heif,heifs,jp2,jpeg,jpg,jpx,jxl,k25,mdc,mef,mrw,nef,nii,pfm,pgm,png,ppm,raf,raw,sr2,srf,svg,tif,tiff,webp,x3f", help="image formats to decode using pyvips")
+    ap2.add_argument("--th-r-vips", metavar="T,T", type=u, default="3fr,arw,avif,cr2,cr3,crw,dcr,dng,erf,exr,fit,fits,fts,gif,hdr,heic,heics,heif,heifs,jp2,jpeg,jpg,jpx,jxl,k25,kdc,mdc,mef,mrw,nef,nii,nrw,orf,pfm,pgm,png,ppm,raf,raw,rw2,sr2,srf,srw,svg,tif,tiff,webp,x3f", help="image formats to decode using pyvips")
-    ap2.add_argument("--th-r-raw", metavar="T,T", type=u, default="3fr,arw,cr2,cr3,crw,dcr,dng,erf,k25,kdc,mdc,mef,mos,mrw,nef,nrw,orf,pef,raf,raw,sr2,srf,srw,x3f", help="image formats to decode using rawpy")
+    ap2.add_argument("--th-r-raw", metavar="T,T", type=u, default="3fr,arw,cr2,cr3,crw,dcr,dng,erf,k25,kdc,mdc,mef,mos,mrw,nef,nrw,orf,pef,raf,raw,rw2,sr2,srf,srw,x3f", help="image formats to decode using rawpy (if available) or libraw's dcraw_emu")
     ap2.add_argument("--th-r-ffi", metavar="T,T", type=u, default="apng,avif,avifs,bmp,cbz,dds,dib,epub,fit,fits,fts,gif,hdr,heic,heics,heif,heifs,icns,ico,jp2,jpeg,jpg,jpx,jxl,pbm,pcx,pfm,pgm,png,pnm,ppm,psd,qoi,sgi,tga,tif,tiff,webp,xbm,xpm", help="image formats to decode using ffmpeg")
     ap2.add_argument("--th-r-ffv", metavar="T,T", type=u, default="3gp,asf,av1,avc,avi,flv,h264,h265,hevc,m4v,mjpeg,mjpg,mkv,mov,mp4,mpeg,mpeg2,mpegts,mpg,mpg2,mts,nut,ogm,ogv,rm,ts,vob,webm,wmv", help="video formats to decode using ffmpeg")
     ap2.add_argument("--th-r-ffa", metavar="T,T", type=u, default="aac,ac3,aif,aiff,alac,alaw,amr,apac,ape,au,bcstm,bfstm,brstm,bonk,dfpwm,dts,flac,gsm,ilbc,it,itgz,itxz,itz,m4a,m4b,m4r,mdgz,mdxz,mdz,mka,mo3,mod,mp2,mp3,mpc,mptm,mt2,mulaw,oga,ogg,okt,opus,ra,s3m,s3gz,s3xz,s3z,tak,tta,ulaw,wav,wma,wv,xm,xmgz,xmxz,xmz,xpk", help="audio formats to decode using ffmpeg")

copyparty/svchub.py

@@ -41,10 +41,11 @@ from .th_srv import (
     H_PIL_AVIF,
     H_PIL_HEIF,
     H_PIL_WEBP,
+    HAVE_DCRAW,
     HAVE_FFMPEG,
     HAVE_FFPROBE,
     HAVE_PIL,
-    HAVE_RAW,
+    HAVE_RAWPY,
     HAVE_VIPS,
     ThumbSrv,
 )
@@ -398,7 +399,7 @@ class SvcHub(object):
             decs.pop("vips", None)
         if not HAVE_PIL:
             decs.pop("pil", None)
-        if not HAVE_RAW:
+        if not HAVE_RAWPY and not HAVE_DCRAW:
             decs.pop("raw", None)
         if not HAVE_FFMPEG or not HAVE_FFPROBE:
             decs.pop("ff", None)
@@ -1009,7 +1010,8 @@ class SvcHub(object):
             (HAVE_ZMQ, "pyzmq", "send zeromq messages from event-hooks"),
             (H_PIL_HEIF, "pillow-heif", "read .heif pics with pillow (rarely useful)"),
             (H_PIL_AVIF, "pillow-avif", "read .avif pics with pillow (rarely useful)"),
-            (HAVE_RAW, "rawpy", "read RAW images"),
+            (HAVE_RAWPY, "rawpy", "read RAW images"),
+            (HAVE_DCRAW, "libraw", "read RAW images"),
         ]
         if ANYWIN:
             to_check += [

copyparty/th_srv.py

@@ -18,7 +18,7 @@ from queue import Queue
 from .__init__ import ANYWIN, PY2, TYPE_CHECKING, unicode
 from .authsrv import VFS
 from .bos import bos
-from .mtag import HAVE_FFMPEG, HAVE_FFPROBE, au_unpk, ffprobe
+from .mtag import HAVE_FFMPEG, HAVE_FFPROBE, au_unpk, ffprobe, have_ff
 from .util import BytesIO  # type: ignore
 from .util import (
     FFMPEG_URL,
@@ -201,16 +201,22 @@ except Exception as e:
         logging.warning("libvips found, but failed to load: " + str(e))
 
 
+PRTY_NO_RAW = os.environ.get("PRTY_NO_RAW")
+PRTY_NO_RAWPY = PRTY_NO_RAW or os.environ.get("PRTY_NO_RAWPY")
+PRTY_NO_DCRAW = PRTY_NO_RAW or os.environ.get("PRTY_NO_DCRAW")
 try:
-    if os.environ.get("PRTY_NO_RAW"):
+    if PRTY_NO_RAWPY:
         raise Exception()
 
-    HAVE_RAW = True
+    HAVE_RAWPY = True
     import rawpy
 
     logging.getLogger("rawpy").setLevel(logging.WARNING)
 except:
-    HAVE_RAW = False
+    HAVE_RAWPY = False
+
+
+HAVE_DCRAW = not PRTY_NO_DCRAW and have_ff("dcraw_emu")
 
 
 th_dir_cache = {}
@@ -307,6 +313,8 @@ class ThumbSrv(object):
             if ANYWIN and self.args.no_acode:
                 self.log("download FFmpeg to fix it:\033[0m " + FFMPEG_URL, 3)
 
+        self.conv_raw = self._conv_rawpy if HAVE_RAWPY else self._conv_dcraw
+
         if self.args.th_clean:
             Daemon(self.cleaner, "thumb.cln")
 
@@ -758,8 +766,39 @@ class ThumbSrv(object):
 
         self.conv_image_vips(_loader, tpath, fmt, vn)
 
-    def conv_raw(self, abspath: str, tpath: str, fmt: str, vn: VFS) -> None:
+    def _conv_dcraw(self, abspath: str, tpath: str, fmt: str, vn: VFS) -> None:
-        self.wait4ram(0.2, tpath)
+        self.wait4ram(0.6, tpath)
+        # fmt: off
+        cmd = [
+            b"dcraw_emu",
+            b"-h",  # halfsize
+            b"-o", b"1",  # srgb
+            b"-s", b"0",  # first frame
+            b"-Z", b"-",  # to stdout
+            fsenc(abspath),
+        ]
+        # fmt: on
+        p = sp.Popen(cmd, stdout=sp.PIPE)
+        try:
+            if HAVE_PIL:
+                self.conv_image_pil(Image.open(p.stdout), tpath, fmt, vn)
+            elif HAVE_VIPS:
+                ppm, _ = p.communicate(timeout=vn.flags["convt"])
+
+                def _loader(w: int, kw: dict) -> Any:
+                    return pyvips.Image.thumbnail_buffer(ppm, w, **kw)
+
+                self.conv_image_vips(_loader, tpath, fmt, vn)
+            else:
+                raise Exception(
+                    "either pil or vips is needed to process embedded bitmap thumbnails in raw files"
+                )
+        finally:
+            if p and p.poll() is None:
+                p.kill(9)
+
+    def _conv_rawpy(self, abspath: str, tpath: str, fmt: str, vn: VFS) -> None:
+        self.wait4ram(0.6, tpath)
         with rawpy.imread(abspath) as raw:
             thumb = raw.extract_thumb()
         if thumb.format == rawpy.ThumbFormat.JPEG and tpath.endswith(".jpg"):

docs/th-raw.txt

@@ -0,0 +1,64 @@
+FS=/home/ed/Pictures/rawsamples-ch  # https://rawsamples.ch/index.php/en/ (the 7z)
+
+find $FS -type f | sed -r 's/(.*)\.(.*)/\2                     \1.\2/' | sort | tr '[:upper:]' '[:lower:]' | uniq -cw16 | sort -n | awk '{printf"%s ",$2}'
+FMTS="dcr erf mdc mef ppm sr2 srf mos pdf 3fr tiff nrw kdc tif srw x3f mrw pef dng raw raf arw crw orf nef cr2 rw2 jpg"
+
+for w in $FMTS ; do grep -E "th-r-.*\b$w\b" ~/dev/copyparty/copyparty/__main__.py || echo "$w"; done 
+missing rw2;
+FMTS_CPP=3fr,arw,cr2,cr3,crw,dcr,dng,erf,k25,kdc,mdc,mef,mos,mrw,nef,nrw,orf,pef,raf,raw,sr2,srf,srw,x3f,rw2 
+
+rm -rf $FS/.hist
+time podman run --rm -it -v $FS:/w copyparty/iv -v /w::r --exit=thgen --th-pregen=j 
+find $FS/.hist/th/ -iname '*.jpg' | wc -l
+
+371 0m5.200s 3s
+458 0m5.512s 3s with --th-r-raw=$FMTS_CPP
+443 0m3.967s 2s with --th-r-raw=$FMTS_CPP --th-dec=raw
+
+t0=$(date +%s); for f in $FMTS ; do rm -rf $FS/.hist
+podman run --rm -it -v $FS:/w copyparty/iv -v /w::r --exit=thgen --th-pregen=j --th-dec=raw --th-r-raw=$f -q >/dev/null 2>&1
+printf '%s ' $(find $FS/.hist/th/ -size +0 -iname '*.jpg' | wc -l)
+done;t=$(date +%s);echo $((t-t0))
+
+ 95f 55s --th-dec=ff 
+397f 51s --th-dec=raw (rawpy)
+153f 51s --th-dec=vips (no-magick)
+
+# swithc to persistent for messingaround
+podman run --rm -it -v $FS:/w --entrypoint /bin/ash copyparty/iv 
+apk update
+apk upgrade -lai
+
+t0=$(date +%s); for f in $FMTS ; do rm -rf $FS/.hist
+podman exec -it d171470581ab python3 -m copyparty -v /w::r --exit=thgen --th-pregen=j --th-dec=vips --th-r-vips=$f -q >/dev/null 2>&1
+printf '%s ' $(find $FS/.hist/th/ -size +0 -iname '*.jpg' | wc -l)
+done;t=$(date +%s);echo $((t-t0))
+# equivalent results
+
+apk add imagemagick; t0=$(date +%s); rm -rf /w/.hist/ ; for f in $FMTS ; do rm -f /*.jpg; n=0; find /w -type f -iname "*.$f" | while IFS= read -r x; do magick "$x" -scale 320x /$n.jpg >/dev/null 2>&1 ; [ -s /$n.jpg ] || rm -f /$n.jpg; n=$((n+1)); done; echo -n "$(ls -1 / | grep -F .jpg | wc -l) "; done; t=$(date +%s); echo $((t-t0))
+
+apk add libraw-tools; t0=$(date +%s); rm -rf /w/.hist/ ; for f in $FMTS ; do rm -f /*.jpg; n=0; find /w -type f -iname "*.$f" | while IFS= read -r x; do [ $(dcraw_emu -h -o 1 -s 0 -Z - "$x" 2>/dev/null | wc -c) -gt 1024 ] && touch /$n.jpg; n=$((n+1)); done; echo -n "$(ls -1 / | grep -F .jpg | wc -l) "; done; t=$(date +%s); echo $((t-t0))
+
+dcr erf mdc mef ppm sr2 srf mos pdf 3fr tiff nrw kdc tif srw x3f mrw pef dng raw raf arw crw orf nef cr2 rw2 jpg
+d e m m p s s m p 3 t n k t s x m  p  d  r  r  a  c  o  n  c  r j
+c r d e p r r o d f i r d i r 3 r  e  n  a  a  r  r  r  e  r  w p
+r f c f m 2 f s f r f w c f w f w  f  g  w  f  w  w  f  f  2  2 g
+-----------------------------------------------------------------
+0 0 0 0 1 1 0 0 0 0 3 0 2 6 0 0 0  6 17  0  0  0  0  0 54  0  0 5 = 95,  55s --th-dec=ff
+1 1 0 1 0 1 1 2 0 3 0 4 5 6 7 0 8 17 16  2 30 31 17 45 56 56 87 0 =397,  51s --th-dec=raw  ## rawpy
+1 1 1 1 0 1 1 2 0 3 0 4 5 6 6 0 9 17 18 24 30 31 34 45 56 57 87 0 =440,  87s dcraw_emu
+1 1 1 1 1 1 1 2 0 3 6 4 5 6 6 0 9 17 18 24 30 31 34 45 56 57 87 5 =452, 226s magick-cmd
+0 1 0 1 1 0 0 0 2 3 3 0 3 6 0 0 0  0 17  0  0  0  0  0 56 55  0 5 =153,  51s --th-dec=vips  ## stock
+0 1 0 1 1 1 1 0 2 3 3 4 5 6 6 0 8  0 18 11 30 30 18 39 56 55 87 5 =391, 151s vips + apk add imagemagick imagemagick-raw
+0 1 0 1 1 0 0 0 2 3 3 0 3 6 0 0 8  0 17 11 30  0 18 39 56 55 87 5 =346, 128s vips + apk del imagemagick  (just imagemagick-raw)
+
+xsel -o | tr ' ' '\n' | awk '!$0{next} {t+=$1} END{print t}'
+
+apk add time
+rm -rf /w/.hist; time python3 -m copyparty -v /w::r --exit=thgen --th-pregen=j --th-dec=raw --th-r-raw=$FMTS_CPP ; find /w/.hist/ -iname '*.jpg' -size +0 | wc -l
+
+391f, 0:03.77elapsed  227264maxresident  # rawpy+vips    with-embedded-thumbs
+391f, 0:04.79elapsed  467036maxresident  # rawpy+pillow  with-embedded-thumbs
+434f, 0:29.67elapsed  307724maxresident  # dcraw+vips
+434f, 0:29.34elapsed  327980maxresident  # dcraw+pillow
+374f, 1:49.70elapsed 4574768maxresident  # vips+imagemagick lol lmao

scripts/docker/Dockerfile.dj

@@ -15,22 +15,20 @@ RUN     apk add -U !pyc ${ADD_PKG} \
             tzdata wget mimalloc2 mimalloc2-insecure \
             py3-jinja2 py3-argon2-cffi py3-pyzmq \
             py3-openssl py3-paramiko py3-pillow \
-            py3-pip py3-cffi \
+            py3-pip \
             py3-magic \
             vips-jxl vips-poppler vips-magick \
             py3-numpy fftw libsndfile \
             vamp-sdk vamp-sdk-libs keyfinder-cli \
-            libraw py3-numpy \
+            libraw-tools \
         && apk add -t .bd \
             bash wget gcc g++ make cmake patchelf \
             ffmpeg ffmpeg-dev \
             python3-dev fftw-dev libsndfile-dev \
-            py3-wheel py3-numpy-dev libffi-dev \
+            py3-wheel py3-numpy-dev \
             vamp-sdk-dev \
-            libraw-dev py3-numpy-dev cython \
         && rm -f /usr/lib/python3*/EXTERNALLY-MANAGED \
         && python3 -m pip install pyvips \
-        && python3 -m pip install --no-binary rawpy rawpy \
         && bash install-deps.sh \
         && apk del py3-pip .bd \
         && chmod 777 /root \

scripts/docker/Dockerfile.iv

@@ -12,18 +12,12 @@ RUN     apk add -U !pyc ${ADD_PKG} \
             tzdata wget mimalloc2 mimalloc2-insecure \
             py3-jinja2 py3-argon2-cffi py3-pyzmq \
             py3-openssl py3-paramiko py3-pillow \
-            py3-pip py3-cffi \
+            py3-pip \
             py3-magic \
             vips-jxl vips-poppler vips-magick \
-            libraw py3-numpy \
+            libraw-tools \
-        && apk add -t .bd \
-            bash wget gcc g++ make cmake patchelf \
-            python3-dev py3-wheel libffi-dev \
-            libraw-dev py3-numpy-dev cython \
         && rm -f /usr/lib/python3*/EXTERNALLY-MANAGED \
-        && python3 -m pip install pyvips \
+        && python3 -m pip install pyvips
-        && python3 -m pip install --no-binary rawpy rawpy \
-        && apk del py3-pip .bd
 
 COPY    i innvikler.sh ./
 RUN     ash innvikler.sh iv