diff --git a/README.md b/README.md index b48dfa60..4f887359 100644 --- a/README.md +++ b/README.md @@ -302,7 +302,7 @@ also see [comparison to similar software](./docs/versus.md) * ☑ realtime streaming of growing files (logfiles and such) * ☑ [thumbnails](#thumbnails) * ☑ ...of images using Pillow, pyvips, or FFmpeg - * ☑ ...of RAW images using rawpy + * ☑ ...of RAW images using libraw-dcraw_emu or rawpy * ☑ ...of videos using FFmpeg * ☑ ...of audio (spectrograms) using FFmpeg * ☑ cache eviction (max-age; maybe max-size eventually) @@ -3206,7 +3206,7 @@ enable [thumbnails](#thumbnails) of... * **HEIF pictures:** `pyvips` or `ffmpeg` or `pillow-heif` * **AVIF pictures:** `pyvips` or `ffmpeg` or `pillow-avif-plugin` or pillow v11.3+ * **JPEG XL pictures:** `pyvips` or `ffmpeg` -* **RAW images:** `rawpy`, plus one of `pyvips` or `Pillow` (for some formats) +* **RAW photos:** either `libraw dcraw_emu` or `rawpy`, plus either `pyvips` or `Pillow` enable sending [zeromq messages](#zeromq) from event-hooks: `pyzmq` @@ -3232,6 +3232,7 @@ set any of the following environment variables to disable its associated optiona | -------------------- | ------------ | | `PRTY_NO_ARGON2` | disable argon2-cffi password hashing | | `PRTY_NO_CFSSL` | never attempt to generate self-signed certificates using [cfssl](https://github.com/cloudflare/cfssl) | +| `PRTY_NO_DCRAW` | disable all [libraw](https://www.libraw.org/homepage)-based thumbnail support for RAW images | | `PRTY_NO_FFMPEG` | **audio transcoding** goes byebye, **thumbnailing** must be handled by Pillow/libvips | | `PRTY_NO_FFPROBE` | **audio transcoding** goes byebye, **thumbnailing** must be handled by Pillow/libvips, **metadata-scanning** must be handled by mutagen | | `PRTY_NO_MAGIC` | do not use [magic](https://pypi.org/project/python-magic/) for filetype detection | @@ -3246,7 +3247,8 @@ set any of the following environment variables to disable its associated optiona | `PRTY_NO_PIL_WEBP` | disable use of native webp support in Pillow | | `PRTY_NO_PSUTIL` | do not use [psutil](https://pypi.org/project/psutil/) for reaping stuck hooks and plugins on Windows | | `PRTY_NO_PYFTPD` | disable ftp(s) server ([pyftpdlib](https://pypi.org/project/pyftpdlib/)-based) | -| `PRTY_NO_RAW` | disable all [rawpy](https://pypi.org/project/rawpy/)-based thumbnail support for RAW images | +| `PRTY_NO_RAW` | same as `PRTY_NO_DCRAW` plus `PRTY_NO_RAWPY` | +| `PRTY_NO_RAWPY` | disable all [rawpy](https://pypi.org/project/rawpy/)-based thumbnail support for RAW images | | `PRTY_NO_VIPS` | disable all [libvips](https://pypi.org/project/pyvips/)-based thumbnail support; will fallback to Pillow or ffmpeg | example: `PRTY_NO_PIL=1 python3 copyparty-sfx.py` diff --git a/copyparty/__main__.py b/copyparty/__main__.py index 51ceefab..b2f72c61 100644 --- a/copyparty/__main__.py +++ b/copyparty/__main__.py @@ -1775,8 +1775,8 @@ def add_thumbnail(ap): # https://stackoverflow.com/a/47612661 # ffmpeg -hide_banner -demuxers | awk '/^ D /{print$2}' | while IFS= read -r x; do ffmpeg -hide_banner -h demuxer=$x; done | grep -E '^Demuxer |extensions:' ap2.add_argument("--th-r-pil", metavar="T,T", type=u, default="avif,avifs,blp,bmp,cbz,dcx,dds,dib,emf,eps,epub,fits,flc,fli,fpx,gif,heic,heics,heif,heifs,icns,ico,im,j2p,j2k,jp2,jpeg,jpg,jpx,jxl,pbm,pcx,pgm,png,pnm,ppm,psd,qoi,sgi,spi,tga,tif,tiff,webp,wmf,xbm,xpm", help="image formats to decode using pillow") - ap2.add_argument("--th-r-vips", metavar="T,T", type=u, default="3fr,avif,cr2,cr3,crw,dcr,dng,erf,exr,fit,fits,fts,gif,hdr,heic,heics,heif,heifs,jp2,jpeg,jpg,jpx,jxl,k25,mdc,mef,mrw,nef,nii,pfm,pgm,png,ppm,raf,raw,sr2,srf,svg,tif,tiff,webp,x3f", help="image formats to decode using pyvips") - ap2.add_argument("--th-r-raw", metavar="T,T", type=u, default="3fr,arw,cr2,cr3,crw,dcr,dng,erf,k25,kdc,mdc,mef,mos,mrw,nef,nrw,orf,pef,raf,raw,sr2,srf,srw,x3f", help="image formats to decode using rawpy") + ap2.add_argument("--th-r-vips", metavar="T,T", type=u, default="3fr,arw,avif,cr2,cr3,crw,dcr,dng,erf,exr,fit,fits,fts,gif,hdr,heic,heics,heif,heifs,jp2,jpeg,jpg,jpx,jxl,k25,kdc,mdc,mef,mrw,nef,nii,nrw,orf,pfm,pgm,png,ppm,raf,raw,rw2,sr2,srf,srw,svg,tif,tiff,webp,x3f", help="image formats to decode using pyvips") + ap2.add_argument("--th-r-raw", metavar="T,T", type=u, default="3fr,arw,cr2,cr3,crw,dcr,dng,erf,k25,kdc,mdc,mef,mos,mrw,nef,nrw,orf,pef,raf,raw,rw2,sr2,srf,srw,x3f", help="image formats to decode using rawpy (if available) or libraw's dcraw_emu") ap2.add_argument("--th-r-ffi", metavar="T,T", type=u, default="apng,avif,avifs,bmp,cbz,dds,dib,epub,fit,fits,fts,gif,hdr,heic,heics,heif,heifs,icns,ico,jp2,jpeg,jpg,jpx,jxl,pbm,pcx,pfm,pgm,png,pnm,ppm,psd,qoi,sgi,tga,tif,tiff,webp,xbm,xpm", help="image formats to decode using ffmpeg") ap2.add_argument("--th-r-ffv", metavar="T,T", type=u, default="3gp,asf,av1,avc,avi,flv,h264,h265,hevc,m4v,mjpeg,mjpg,mkv,mov,mp4,mpeg,mpeg2,mpegts,mpg,mpg2,mts,nut,ogm,ogv,rm,ts,vob,webm,wmv", help="video formats to decode using ffmpeg") ap2.add_argument("--th-r-ffa", metavar="T,T", type=u, default="aac,ac3,aif,aiff,alac,alaw,amr,apac,ape,au,bcstm,bfstm,brstm,bonk,dfpwm,dts,flac,gsm,ilbc,it,itgz,itxz,itz,m4a,m4b,m4r,mdgz,mdxz,mdz,mka,mo3,mod,mp2,mp3,mpc,mptm,mt2,mulaw,oga,ogg,okt,opus,ra,s3m,s3gz,s3xz,s3z,tak,tta,ulaw,wav,wma,wv,xm,xmgz,xmxz,xmz,xpk", help="audio formats to decode using ffmpeg") diff --git a/copyparty/svchub.py b/copyparty/svchub.py index 2a4737fc..0f4325d0 100644 --- a/copyparty/svchub.py +++ b/copyparty/svchub.py @@ -41,10 +41,11 @@ from .th_srv import ( H_PIL_AVIF, H_PIL_HEIF, H_PIL_WEBP, + HAVE_DCRAW, HAVE_FFMPEG, HAVE_FFPROBE, HAVE_PIL, - HAVE_RAW, + HAVE_RAWPY, HAVE_VIPS, ThumbSrv, ) @@ -398,7 +399,7 @@ class SvcHub(object): decs.pop("vips", None) if not HAVE_PIL: decs.pop("pil", None) - if not HAVE_RAW: + if not HAVE_RAWPY and not HAVE_DCRAW: decs.pop("raw", None) if not HAVE_FFMPEG or not HAVE_FFPROBE: decs.pop("ff", None) @@ -1009,7 +1010,8 @@ class SvcHub(object): (HAVE_ZMQ, "pyzmq", "send zeromq messages from event-hooks"), (H_PIL_HEIF, "pillow-heif", "read .heif pics with pillow (rarely useful)"), (H_PIL_AVIF, "pillow-avif", "read .avif pics with pillow (rarely useful)"), - (HAVE_RAW, "rawpy", "read RAW images"), + (HAVE_RAWPY, "rawpy", "read RAW images"), + (HAVE_DCRAW, "libraw", "read RAW images"), ] if ANYWIN: to_check += [ diff --git a/copyparty/th_srv.py b/copyparty/th_srv.py index f119c5e5..e63ce14e 100644 --- a/copyparty/th_srv.py +++ b/copyparty/th_srv.py @@ -18,7 +18,7 @@ from queue import Queue from .__init__ import ANYWIN, PY2, TYPE_CHECKING, unicode from .authsrv import VFS from .bos import bos -from .mtag import HAVE_FFMPEG, HAVE_FFPROBE, au_unpk, ffprobe +from .mtag import HAVE_FFMPEG, HAVE_FFPROBE, au_unpk, ffprobe, have_ff from .util import BytesIO # type: ignore from .util import ( FFMPEG_URL, @@ -201,16 +201,22 @@ except Exception as e: logging.warning("libvips found, but failed to load: " + str(e)) +PRTY_NO_RAW = os.environ.get("PRTY_NO_RAW") +PRTY_NO_RAWPY = PRTY_NO_RAW or os.environ.get("PRTY_NO_RAWPY") +PRTY_NO_DCRAW = PRTY_NO_RAW or os.environ.get("PRTY_NO_DCRAW") try: - if os.environ.get("PRTY_NO_RAW"): + if PRTY_NO_RAWPY: raise Exception() - HAVE_RAW = True + HAVE_RAWPY = True import rawpy logging.getLogger("rawpy").setLevel(logging.WARNING) except: - HAVE_RAW = False + HAVE_RAWPY = False + + +HAVE_DCRAW = not PRTY_NO_DCRAW and have_ff("dcraw_emu") th_dir_cache = {} @@ -307,6 +313,8 @@ class ThumbSrv(object): if ANYWIN and self.args.no_acode: self.log("download FFmpeg to fix it:\033[0m " + FFMPEG_URL, 3) + self.conv_raw = self._conv_rawpy if HAVE_RAWPY else self._conv_dcraw + if self.args.th_clean: Daemon(self.cleaner, "thumb.cln") @@ -758,8 +766,39 @@ class ThumbSrv(object): self.conv_image_vips(_loader, tpath, fmt, vn) - def conv_raw(self, abspath: str, tpath: str, fmt: str, vn: VFS) -> None: - self.wait4ram(0.2, tpath) + def _conv_dcraw(self, abspath: str, tpath: str, fmt: str, vn: VFS) -> None: + self.wait4ram(0.6, tpath) + # fmt: off + cmd = [ + b"dcraw_emu", + b"-h", # halfsize + b"-o", b"1", # srgb + b"-s", b"0", # first frame + b"-Z", b"-", # to stdout + fsenc(abspath), + ] + # fmt: on + p = sp.Popen(cmd, stdout=sp.PIPE) + try: + if HAVE_PIL: + self.conv_image_pil(Image.open(p.stdout), tpath, fmt, vn) + elif HAVE_VIPS: + ppm, _ = p.communicate(timeout=vn.flags["convt"]) + + def _loader(w: int, kw: dict) -> Any: + return pyvips.Image.thumbnail_buffer(ppm, w, **kw) + + self.conv_image_vips(_loader, tpath, fmt, vn) + else: + raise Exception( + "either pil or vips is needed to process embedded bitmap thumbnails in raw files" + ) + finally: + if p and p.poll() is None: + p.kill(9) + + def _conv_rawpy(self, abspath: str, tpath: str, fmt: str, vn: VFS) -> None: + self.wait4ram(0.6, tpath) with rawpy.imread(abspath) as raw: thumb = raw.extract_thumb() if thumb.format == rawpy.ThumbFormat.JPEG and tpath.endswith(".jpg"): diff --git a/docs/th-raw.txt b/docs/th-raw.txt new file mode 100644 index 00000000..5a25dee5 --- /dev/null +++ b/docs/th-raw.txt @@ -0,0 +1,64 @@ +FS=/home/ed/Pictures/rawsamples-ch # https://rawsamples.ch/index.php/en/ (the 7z) + +find $FS -type f | sed -r 's/(.*)\.(.*)/\2 \1.\2/' | sort | tr '[:upper:]' '[:lower:]' | uniq -cw16 | sort -n | awk '{printf"%s ",$2}' +FMTS="dcr erf mdc mef ppm sr2 srf mos pdf 3fr tiff nrw kdc tif srw x3f mrw pef dng raw raf arw crw orf nef cr2 rw2 jpg" + +for w in $FMTS ; do grep -E "th-r-.*\b$w\b" ~/dev/copyparty/copyparty/__main__.py || echo "$w"; done +missing rw2; +FMTS_CPP=3fr,arw,cr2,cr3,crw,dcr,dng,erf,k25,kdc,mdc,mef,mos,mrw,nef,nrw,orf,pef,raf,raw,sr2,srf,srw,x3f,rw2 + +rm -rf $FS/.hist +time podman run --rm -it -v $FS:/w copyparty/iv -v /w::r --exit=thgen --th-pregen=j +find $FS/.hist/th/ -iname '*.jpg' | wc -l + +371 0m5.200s 3s +458 0m5.512s 3s with --th-r-raw=$FMTS_CPP +443 0m3.967s 2s with --th-r-raw=$FMTS_CPP --th-dec=raw + +t0=$(date +%s); for f in $FMTS ; do rm -rf $FS/.hist +podman run --rm -it -v $FS:/w copyparty/iv -v /w::r --exit=thgen --th-pregen=j --th-dec=raw --th-r-raw=$f -q >/dev/null 2>&1 +printf '%s ' $(find $FS/.hist/th/ -size +0 -iname '*.jpg' | wc -l) +done;t=$(date +%s);echo $((t-t0)) + + 95f 55s --th-dec=ff +397f 51s --th-dec=raw (rawpy) +153f 51s --th-dec=vips (no-magick) + +# swithc to persistent for messingaround +podman run --rm -it -v $FS:/w --entrypoint /bin/ash copyparty/iv +apk update +apk upgrade -lai + +t0=$(date +%s); for f in $FMTS ; do rm -rf $FS/.hist +podman exec -it d171470581ab python3 -m copyparty -v /w::r --exit=thgen --th-pregen=j --th-dec=vips --th-r-vips=$f -q >/dev/null 2>&1 +printf '%s ' $(find $FS/.hist/th/ -size +0 -iname '*.jpg' | wc -l) +done;t=$(date +%s);echo $((t-t0)) +# equivalent results + +apk add imagemagick; t0=$(date +%s); rm -rf /w/.hist/ ; for f in $FMTS ; do rm -f /*.jpg; n=0; find /w -type f -iname "*.$f" | while IFS= read -r x; do magick "$x" -scale 320x /$n.jpg >/dev/null 2>&1 ; [ -s /$n.jpg ] || rm -f /$n.jpg; n=$((n+1)); done; echo -n "$(ls -1 / | grep -F .jpg | wc -l) "; done; t=$(date +%s); echo $((t-t0)) + +apk add libraw-tools; t0=$(date +%s); rm -rf /w/.hist/ ; for f in $FMTS ; do rm -f /*.jpg; n=0; find /w -type f -iname "*.$f" | while IFS= read -r x; do [ $(dcraw_emu -h -o 1 -s 0 -Z - "$x" 2>/dev/null | wc -c) -gt 1024 ] && touch /$n.jpg; n=$((n+1)); done; echo -n "$(ls -1 / | grep -F .jpg | wc -l) "; done; t=$(date +%s); echo $((t-t0)) + +dcr erf mdc mef ppm sr2 srf mos pdf 3fr tiff nrw kdc tif srw x3f mrw pef dng raw raf arw crw orf nef cr2 rw2 jpg +d e m m p s s m p 3 t n k t s x m p d r r a c o n c r j +c r d e p r r o d f i r d i r 3 r e n a a r r r e r w p +r f c f m 2 f s f r f w c f w f w f g w f w w f f 2 2 g +----------------------------------------------------------------- +0 0 0 0 1 1 0 0 0 0 3 0 2 6 0 0 0 6 17 0 0 0 0 0 54 0 0 5 = 95, 55s --th-dec=ff +1 1 0 1 0 1 1 2 0 3 0 4 5 6 7 0 8 17 16 2 30 31 17 45 56 56 87 0 =397, 51s --th-dec=raw ## rawpy +1 1 1 1 0 1 1 2 0 3 0 4 5 6 6 0 9 17 18 24 30 31 34 45 56 57 87 0 =440, 87s dcraw_emu +1 1 1 1 1 1 1 2 0 3 6 4 5 6 6 0 9 17 18 24 30 31 34 45 56 57 87 5 =452, 226s magick-cmd +0 1 0 1 1 0 0 0 2 3 3 0 3 6 0 0 0 0 17 0 0 0 0 0 56 55 0 5 =153, 51s --th-dec=vips ## stock +0 1 0 1 1 1 1 0 2 3 3 4 5 6 6 0 8 0 18 11 30 30 18 39 56 55 87 5 =391, 151s vips + apk add imagemagick imagemagick-raw +0 1 0 1 1 0 0 0 2 3 3 0 3 6 0 0 8 0 17 11 30 0 18 39 56 55 87 5 =346, 128s vips + apk del imagemagick (just imagemagick-raw) + +xsel -o | tr ' ' '\n' | awk '!$0{next} {t+=$1} END{print t}' + +apk add time +rm -rf /w/.hist; time python3 -m copyparty -v /w::r --exit=thgen --th-pregen=j --th-dec=raw --th-r-raw=$FMTS_CPP ; find /w/.hist/ -iname '*.jpg' -size +0 | wc -l + +391f, 0:03.77elapsed 227264maxresident # rawpy+vips with-embedded-thumbs +391f, 0:04.79elapsed 467036maxresident # rawpy+pillow with-embedded-thumbs +434f, 0:29.67elapsed 307724maxresident # dcraw+vips +434f, 0:29.34elapsed 327980maxresident # dcraw+pillow +374f, 1:49.70elapsed 4574768maxresident # vips+imagemagick lol lmao diff --git a/scripts/docker/Dockerfile.dj b/scripts/docker/Dockerfile.dj index ec82be50..1d1233f0 100644 --- a/scripts/docker/Dockerfile.dj +++ b/scripts/docker/Dockerfile.dj @@ -15,22 +15,20 @@ RUN apk add -U !pyc ${ADD_PKG} \ tzdata wget mimalloc2 mimalloc2-insecure \ py3-jinja2 py3-argon2-cffi py3-pyzmq \ py3-openssl py3-paramiko py3-pillow \ - py3-pip py3-cffi \ + py3-pip \ py3-magic \ vips-jxl vips-poppler vips-magick \ py3-numpy fftw libsndfile \ vamp-sdk vamp-sdk-libs keyfinder-cli \ - libraw py3-numpy \ + libraw-tools \ && apk add -t .bd \ bash wget gcc g++ make cmake patchelf \ ffmpeg ffmpeg-dev \ python3-dev fftw-dev libsndfile-dev \ - py3-wheel py3-numpy-dev libffi-dev \ + py3-wheel py3-numpy-dev \ vamp-sdk-dev \ - libraw-dev py3-numpy-dev cython \ && rm -f /usr/lib/python3*/EXTERNALLY-MANAGED \ && python3 -m pip install pyvips \ - && python3 -m pip install --no-binary rawpy rawpy \ && bash install-deps.sh \ && apk del py3-pip .bd \ && chmod 777 /root \ diff --git a/scripts/docker/Dockerfile.iv b/scripts/docker/Dockerfile.iv index dff98cd0..b85bf846 100644 --- a/scripts/docker/Dockerfile.iv +++ b/scripts/docker/Dockerfile.iv @@ -12,18 +12,12 @@ RUN apk add -U !pyc ${ADD_PKG} \ tzdata wget mimalloc2 mimalloc2-insecure \ py3-jinja2 py3-argon2-cffi py3-pyzmq \ py3-openssl py3-paramiko py3-pillow \ - py3-pip py3-cffi \ + py3-pip \ py3-magic \ vips-jxl vips-poppler vips-magick \ - libraw py3-numpy \ - && apk add -t .bd \ - bash wget gcc g++ make cmake patchelf \ - python3-dev py3-wheel libffi-dev \ - libraw-dev py3-numpy-dev cython \ + libraw-tools \ && rm -f /usr/lib/python3*/EXTERNALLY-MANAGED \ - && python3 -m pip install pyvips \ - && python3 -m pip install --no-binary rawpy rawpy \ - && apk del py3-pip .bd + && python3 -m pip install pyvips COPY i innvikler.sh ./ RUN ash innvikler.sh iv