diff --git a/.gitignore b/.gitignore index 302136b9..4b78a4db 100644 --- a/.gitignore +++ b/.gitignore @@ -35,6 +35,7 @@ scripts/docker/base/test-aac/ scripts/docker/base/whl/ scripts/docker/i/ scripts/deps-docker/uncomment.py +scripts/deps-docker/unhint.py contrib/package/arch/pkg/ contrib/package/arch/src/ diff --git a/README.md b/README.md index b48dfa60..133d962f 100644 --- a/README.md +++ b/README.md @@ -302,7 +302,7 @@ also see [comparison to similar software](./docs/versus.md) * ☑ realtime streaming of growing files (logfiles and such) * ☑ [thumbnails](#thumbnails) * ☑ ...of images using Pillow, pyvips, or FFmpeg - * ☑ ...of RAW images using rawpy + * ☑ ...of RAW images using libraw-dcraw_emu or rawpy * ☑ ...of videos using FFmpeg * ☑ ...of audio (spectrograms) using FFmpeg * ☑ cache eviction (max-age; maybe max-size eventually) @@ -836,6 +836,7 @@ you can also zip a selection of files or folders by clicking them in the browser cool trick: download a folder by appending url-params `?tar&opus` or `?tar&mp3` to transcode all audio files (except aac|m4a|mp3|ogg|opus|wma) to opus/mp3 before they're added to the archive * super useful if you're 5 minutes away from takeoff and realize you don't have any music on your phone but your server only has flac files and downloading those will burn through all your data + there wouldn't be enough time anyways +* and url-param `&name=foo` changes the name of the toplevel folder in the archive to `foo`, and just `&name` removes the folder entirely * and url-param `&nodot` skips dotfiles/dotfolders; they are included by default if your account has permission to see them * and url-params `&j` / `&w` produce jpeg/webm thumbnails/spectrograms instead of the original audio/video/images (`&p` for audio waveforms) * can also be used to pregenerate thumbnails; combine with `--th-maxage=9999999` or `--th-clean=0` @@ -1336,9 +1337,15 @@ using arguments or config files, or a mix of both: sleep better at night by telling copyparty to periodically check whether your version has a [known vulnerability](https://github.com/9001/copyparty/security/advisories) -this feature can be enabled by setting the global-option `--vc-url` to one of the following URLs; all of them provide the same information, so which one you choose is whatever -* `https://api.copyparty.eu/advisories` -* `https://api.github.com/repos/9001/copyparty/security-advisories?per_page=9` +this feature can be enabled by setting the global-option `--vc-url` to one of the following URLs; choose what severity level you want to be notified for: +* `https://api.copyparty.eu/advisories-panic` -- only really bad stuff, the "UPGRADE NOW" kind +* `https://api.copyparty.eu/advisories` -- everything important / noteworthy, "upgrade when you can" +* `https://api.copyparty.eu/advisories-all` -- *everything*, including stuff that's unlikely to affect anyone +* `https://api.github.com/repos/9001/copyparty/security-advisories?per_page=9` -- same as `advisories-all` + +note that `https://api.copyparty.eu/advisories` may (for example) skip some advisories rated `High` but include some `Low`; that's because an easily-reachable `Low` in a default-enabled feature is more severe than a `High` which is a theoretical bug in a contrived use of a fringe feature, but the CVE calculator would still classify that as `High` + +if you want to use the github advisory feed but only care about advisories rated `medium`/`moderate` or higher, then global-option `--vc-sev medium` does that, but see previous paragraph > to see what happens when a bad version is detected, try `--vc-url https://api.copyparty.eu/advisories-test` @@ -1354,6 +1361,7 @@ config file example: vc-url: https://api.copyparty.eu/advisories vc-age: 3 # how many hours to wait between each check vc-exit # emergency-exit if current version is vulnerable + vc-sev: medium # only care about severity 'Medium'/'Moderate' or higher (github-only; don't use this with api.copyparty.eu) ``` @@ -1762,7 +1770,7 @@ avoid traversing into other filesystems using `--xdev` / volflag `:c,xdev`, ski and/or you can `--xvol` / `:c,xvol` to ignore all symlinks leaving the volume's top directory, but still allow bind-mounts pointing elsewhere -* symlinks are permitted with `xvol` if they point into another volume where the user has the same level of access +* symlinks are permitted with `xvol` if they point into another volume where the user also has some sort of access, keeping permissions from outer/initial volume these options will reduce performance; unlikely worst-case estimates are 14% reduction for directory listings, 35% for download-as-tar @@ -3206,7 +3214,7 @@ enable [thumbnails](#thumbnails) of... * **HEIF pictures:** `pyvips` or `ffmpeg` or `pillow-heif` * **AVIF pictures:** `pyvips` or `ffmpeg` or `pillow-avif-plugin` or pillow v11.3+ * **JPEG XL pictures:** `pyvips` or `ffmpeg` -* **RAW images:** `rawpy`, plus one of `pyvips` or `Pillow` (for some formats) +* **RAW photos:** either `libraw dcraw_emu` or `rawpy`, plus either `pyvips` or `Pillow` enable sending [zeromq messages](#zeromq) from event-hooks: `pyzmq` @@ -3218,6 +3226,8 @@ enable [smb](#smb-server) support (**not** recommended): `impacket==0.13.0` to install FFmpeg on Windows, grab [a recent build](https://www.gyan.dev/ffmpeg/builds/ffmpeg-git-full.7z) -- you need `ffmpeg.exe` and `ffprobe.exe` from inside the `bin` folder; copy them into `C:\Windows\System32` or any other folder that's in your `%PATH%` +if your ffmpeg/ffprobe binaries have nonstandard names -- such as `ffmpeg8` (macports) -- set environment variables `PRTY_FFMPEG_BIN` and `PRTY_FFPROBE_BIN` to the corret name (or full path) + ### dependency chickenbits @@ -3232,6 +3242,7 @@ set any of the following environment variables to disable its associated optiona | -------------------- | ------------ | | `PRTY_NO_ARGON2` | disable argon2-cffi password hashing | | `PRTY_NO_CFSSL` | never attempt to generate self-signed certificates using [cfssl](https://github.com/cloudflare/cfssl) | +| `PRTY_NO_DCRAW` | disable all [libraw](https://www.libraw.org/homepage)-based thumbnail support for RAW images | | `PRTY_NO_FFMPEG` | **audio transcoding** goes byebye, **thumbnailing** must be handled by Pillow/libvips | | `PRTY_NO_FFPROBE` | **audio transcoding** goes byebye, **thumbnailing** must be handled by Pillow/libvips, **metadata-scanning** must be handled by mutagen | | `PRTY_NO_MAGIC` | do not use [magic](https://pypi.org/project/python-magic/) for filetype detection | @@ -3246,7 +3257,8 @@ set any of the following environment variables to disable its associated optiona | `PRTY_NO_PIL_WEBP` | disable use of native webp support in Pillow | | `PRTY_NO_PSUTIL` | do not use [psutil](https://pypi.org/project/psutil/) for reaping stuck hooks and plugins on Windows | | `PRTY_NO_PYFTPD` | disable ftp(s) server ([pyftpdlib](https://pypi.org/project/pyftpdlib/)-based) | -| `PRTY_NO_RAW` | disable all [rawpy](https://pypi.org/project/rawpy/)-based thumbnail support for RAW images | +| `PRTY_NO_RAW` | same as `PRTY_NO_DCRAW` plus `PRTY_NO_RAWPY` | +| `PRTY_NO_RAWPY` | disable all [rawpy](https://pypi.org/project/rawpy/)-based thumbnail support for RAW images | | `PRTY_NO_VIPS` | disable all [libvips](https://pypi.org/project/pyvips/)-based thumbnail support; will fallback to Pillow or ffmpeg | example: `PRTY_NO_PIL=1 python3 copyparty-sfx.py` diff --git a/bin/u2c.py b/bin/u2c.py index f0e3f8de..a6a564a1 100755 --- a/bin/u2c.py +++ b/bin/u2c.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 from __future__ import print_function, unicode_literals -S_VERSION = "2.20" -S_BUILD_DT = "2026-04-22" +S_VERSION = "2.21" +S_BUILD_DT = "2026-05-25" """ u2c.py: upload to copyparty @@ -1311,7 +1311,8 @@ class Ctl(object): if self.ar.jw: print("%s %s" % (wark, vp)) else: - zd = datetime.datetime.fromtimestamp(max(0, file.lmod), UTC) + tsdt = datetime.datetime.fromtimestamp + zd = tsdt(max(0, min(2 << 36, file.lmod)), UTC) dt = "%04d-%02d-%02d %02d:%02d:%02d" % ( zd.year, zd.month, diff --git a/contrib/package/arch/PKGBUILD b/contrib/package/arch/PKGBUILD index 3983488a..cabe12fc 100644 --- a/contrib/package/arch/PKGBUILD +++ b/contrib/package/arch/PKGBUILD @@ -3,7 +3,7 @@ # NOTE: You generally shouldn't use this PKGBUILD on Arch, as it is mainly for testing purposes. Install copyparty using pacman instead. pkgname=copyparty -pkgver="1.20.14" +pkgver="1.20.16" pkgrel=1 pkgdesc="File server with accelerated resumable uploads, dedup, WebDAV, SFTP, FTP, TFTP, zeroconf, media indexer, thumbnails++" arch=("any") @@ -24,7 +24,7 @@ optdepends=("ffmpeg: thumbnails for videos, images (slower) and audio, music tag ) source=("https://github.com/9001/${pkgname}/releases/download/v${pkgver}/${pkgname}-${pkgver}.tar.gz") backup=("etc/${pkgname}/copyparty.conf" ) -sha256sums=("8783dc8390be17673d306f424e7a28dd9f9b4fce005e35734d30c1b296707c12") +sha256sums=("625f95d65d95cdd6898510518d013905e6766c7d2ae0ea9ae7d5dec96e89e02d") build() { cd "${srcdir}/${pkgname}-${pkgver}/copyparty/web" diff --git a/contrib/package/makedeb-mpr/PKGBUILD b/contrib/package/makedeb-mpr/PKGBUILD index 770398d1..5c3ff91c 100644 --- a/contrib/package/makedeb-mpr/PKGBUILD +++ b/contrib/package/makedeb-mpr/PKGBUILD @@ -2,7 +2,7 @@ pkgname=copyparty -pkgver=1.20.14 +pkgver=1.20.16 pkgrel=1 pkgdesc="File server with accelerated resumable uploads, dedup, WebDAV, SFTP, FTP, TFTP, zeroconf, media indexer, thumbnails++" arch=("any") @@ -21,7 +21,7 @@ optdepends=("ffmpeg: thumbnails for videos, images (slower) and audio, music tag ) source=("https://github.com/9001/${pkgname}/releases/download/v${pkgver}/${pkgname}-${pkgver}.tar.gz") backup=("/etc/${pkgname}.d/init" ) -sha256sums=("8783dc8390be17673d306f424e7a28dd9f9b4fce005e35734d30c1b296707c12") +sha256sums=("625f95d65d95cdd6898510518d013905e6766c7d2ae0ea9ae7d5dec96e89e02d") build() { cd "${srcdir}/${pkgname}-${pkgver}/copyparty/web" diff --git a/contrib/package/nix/copyparty/pin.json b/contrib/package/nix/copyparty/pin.json index 7ed8f8f1..c393e5d3 100644 --- a/contrib/package/nix/copyparty/pin.json +++ b/contrib/package/nix/copyparty/pin.json @@ -1,5 +1,5 @@ { - "url": "https://github.com/9001/copyparty/releases/download/v1.20.14/copyparty-1.20.14.tar.gz", - "version": "1.20.14", - "hash": "sha256-h4Pcg5C+F2c9MG9CTnoo3Z+bT84AXjVzTTDBspZwfBI=" + "url": "https://github.com/9001/copyparty/releases/download/v1.20.16/copyparty-1.20.16.tar.gz", + "version": "1.20.16", + "hash": "sha256-Yl+V1l2VzdaJhRBRjQE5BeZ2bH0q4Oqa59XeyW6J4C0=" } \ No newline at end of file diff --git a/copyparty/__main__.py b/copyparty/__main__.py index 50394fd9..a9130937 100644 --- a/copyparty/__main__.py +++ b/copyparty/__main__.py @@ -1234,8 +1234,10 @@ def add_general(ap, nc, srvname): ap2.add_argument("--mimes", action="store_true", help="list default mimetype mapping and exit") ap2.add_argument("--rmagic", action="store_true", help="do expensive analysis to improve accuracy of returned mimetypes; will make file-downloads, rss, and webdav slower (volflag=rmagic)") ap2.add_argument("-j", metavar="CORES", type=int, default=1, help="num cpu-cores for uploads/downloads (0=all); keeping the default is almost always best") + ap2.add_argument("--reload-sig", metavar="S", type=u, default=("" if ANYWIN else "USR1"), help="reload server config when unix-signal \033[33mS\033[0m is received; examples: [\033[32mSIGUSR1\033[0m], [\033[32mUSR1\033[0m], [\033[32m10\033[0m]") ap2.add_argument("--vc-url", metavar="URL", type=u, default="", help="URL to check for vulnerable versions (default-disabled)") ap2.add_argument("--vc-age", metavar="HOURS", type=int, default=3, help="how many hours to wait between vulnerability checks") + ap2.add_argument("--vc-sev", metavar="LEVEL", type=u, default="low", help="minimum severity to care about; one of these: \033[32mlow medium high critical\033[0m") ap2.add_argument("--vc-exit", action="store_true", help="panic and exit if current version is vulnerable") ap2.add_argument("--license", action="store_true", help="show licenses and exit") ap2.add_argument("--version", action="store_true", help="show versions and exit") @@ -1653,7 +1655,7 @@ def add_safety(ap): ap2.add_argument("-ss", action="store_true", help="further increase safety: Prevent js-injection, accidental move/delete, broken symlinks, webdav requires login, 404 on 403, ban on excessive 404s.\n └─Alias of\033[32m -s --no-html --no-readme --no-logues --unpost=0 --no-del --no-mv --reflink --dav-auth --vague-403 -nih") ap2.add_argument("-sss", action="store_true", help="further increase safety: Enable logging to disk, scan for dangerous symlinks.\n └─Alias of\033[32m -ss --no-dav --no-logues --no-readme -lo=cpp-%%Y-%%m%%d-%%H%%M%%S.txt.xz --ls=**,*,ln,p,r") ap2.add_argument("--ls", metavar="U[,V[,F]]", type=u, default="", help="do a sanity/safety check of all volumes on startup; arguments \033[33mUSER\033[0m,\033[33mVOL\033[0m,\033[33mFLAGS\033[0m (see \033[33m--help-ls\033[0m); example [\033[32m**,*,ln,p,r\033[0m]") - ap2.add_argument("--xvol", action="store_true", help="never follow symlinks leaving the volume root, unless the link is into another volume where the user has similar access (volflag=xvol)") + ap2.add_argument("--xvol", action="store_true", help="never follow symlinks leaving the volume root, unless the link is into another volume where the user also has access (keeps permissions from the outer/initial volume) (volflag=xvol)") ap2.add_argument("--xdev", action="store_true", help="stay within the filesystem of the volume root; do not descend into other devices (symlink or bind-mount to another HDD, ...) (volflag=xdev)") ap2.add_argument("--vol-nospawn", action="store_true", help="if a volume's folder does not exist on the HDD, then do not create it (continue with warning) (volflag=nospawn)") ap2.add_argument("--vol-or-crash", action="store_true", help="if a volume's folder does not exist on the HDD, then burst into flames (volflag=assert_root)") @@ -1713,6 +1715,7 @@ def add_logging(ap): ap2.add_argument("-lo", metavar="PATH", type=u, default="", help="logfile; use .txt for plaintext or .xz for compressed. Example: \033[32mcpp-%%Y-%%m%%d-%%H%%M%%S.txt.xz\033[0m (NB: some errors may appear on STDOUT only)") ap2.add_argument("--flo", metavar="N", type=int, default=1, help="log format for \033[33m-lo\033[0m; [\033[32m1\033[0m]=classic/colors, [\033[32m2\033[0m]=no-color") ap2.add_argument("--rlo", metavar="TXT", type=u, default=".1", help="logrotate counter format; see \033[33m--help-rlo\033[0m") + ap2.add_argument("--logrot-sig", metavar="S", type=u, default="", help="immediately logrotate when unix-signal \033[33mS\033[0m is received; examples: [\033[32mSIGHUP\033[0m], [\033[32mHUP\033[0m], [\033[32m1\033[0m]") ap2.add_argument("--no-ansi", action="store_true", default=not VT100, help="disable colors; same as environment-variable NO_COLOR") ap2.add_argument("--ansi", action="store_true", help="force colors; overrides environment-variable NO_COLOR") ap2.add_argument("--no-logflush", action="store_true", help="don't flush the logfile after each write; tiny bit faster") @@ -1782,8 +1785,8 @@ def add_thumbnail(ap): # https://stackoverflow.com/a/47612661 # ffmpeg -hide_banner -demuxers | awk '/^ D /{print$2}' | while IFS= read -r x; do ffmpeg -hide_banner -h demuxer=$x; done | grep -E '^Demuxer |extensions:' ap2.add_argument("--th-r-pil", metavar="T,T", type=u, default="avif,avifs,blp,bmp,cbz,dcx,dds,dib,emf,eps,epub,fits,flc,fli,fpx,gif,heic,heics,heif,heifs,icns,ico,im,j2p,j2k,jp2,jpeg,jpg,jpx,jxl,pbm,pcx,pgm,png,pnm,ppm,psd,qoi,sgi,spi,tga,tif,tiff,webp,wmf,xbm,xpm", help="image formats to decode using pillow") - ap2.add_argument("--th-r-vips", metavar="T,T", type=u, default="3fr,avif,cr2,cr3,crw,dcr,dng,erf,exr,fit,fits,fts,gif,hdr,heic,heics,heif,heifs,jp2,jpeg,jpg,jpx,jxl,k25,mdc,mef,mrw,nef,nii,pfm,pgm,png,ppm,raf,raw,sr2,srf,svg,tif,tiff,webp,x3f", help="image formats to decode using pyvips") - ap2.add_argument("--th-r-raw", metavar="T,T", type=u, default="3fr,arw,cr2,cr3,crw,dcr,dng,erf,k25,kdc,mdc,mef,mos,mrw,nef,nrw,orf,pef,raf,raw,sr2,srf,srw,x3f", help="image formats to decode using rawpy") + ap2.add_argument("--th-r-vips", metavar="T,T", type=u, default="3fr,arw,avif,cr2,cr3,crw,dcr,dng,erf,exr,fit,fits,fts,gif,hdr,heic,heics,heif,heifs,jp2,jpeg,jpg,jpx,jxl,k25,kdc,mdc,mef,mrw,nef,nii,nrw,orf,pfm,pgm,png,ppm,raf,raw,rw2,sr2,srf,srw,svg,tif,tiff,webp,x3f", help="image formats to decode using pyvips") + ap2.add_argument("--th-r-raw", metavar="T,T", type=u, default="3fr,arw,cr2,cr3,crw,dcr,dng,erf,k25,kdc,mdc,mef,mos,mrw,nef,nrw,orf,pef,raf,raw,rw2,sr2,srf,srw,x3f", help="image formats to decode using rawpy (if available) or libraw's dcraw_emu") ap2.add_argument("--th-r-ffi", metavar="T,T", type=u, default="apng,avif,avifs,bmp,cbz,dds,dib,epub,fit,fits,fts,gif,hdr,heic,heics,heif,heifs,icns,ico,jp2,jpeg,jpg,jpx,jxl,pbm,pcx,pfm,pgm,png,pnm,ppm,psd,qoi,sgi,tga,tif,tiff,webp,xbm,xpm", help="image formats to decode using ffmpeg") ap2.add_argument("--th-r-ffv", metavar="T,T", type=u, default="3gp,asf,av1,avc,avi,flv,h264,h265,hevc,m4v,mjpeg,mjpg,mkv,mov,mp4,mpeg,mpeg2,mpegts,mpg,mpg2,mts,nut,ogm,ogv,rm,ts,vob,webm,wmv", help="video formats to decode using ffmpeg") ap2.add_argument("--th-r-ffa", metavar="T,T", type=u, default="aac,ac3,aif,aiff,alac,alaw,amr,apac,ape,au,bcstm,bfstm,brstm,bonk,dfpwm,dts,flac,gsm,ilbc,it,itgz,itxz,itz,m4a,m4b,m4r,mdgz,mdxz,mdz,mka,mo3,mod,mp2,mp3,mpc,mptm,mt2,mulaw,oga,ogg,okt,opus,ra,s3m,s3gz,s3xz,s3z,tak,tta,ulaw,wav,wma,wv,xm,xmgz,xmxz,xmz,xpk", help="audio formats to decode using ffmpeg") @@ -1987,10 +1990,15 @@ def add_debug(ap): ap2.add_argument("--no-scandir", action="store_true", help="kernel-bug workaround: disable scandir; do a listdir + stat on each file instead") ap2.add_argument("--no-fastboot", action="store_true", help="wait for initial filesystem indexing before accepting client requests") ap2.add_argument("--no-htp", action="store_true", help="disable httpserver threadpool, create threads as-needed instead") + if ANYWIN or sys.version_info < (3, 7): + ap2.add_argument("--sig-thr", action="store_true", default=True, help=argparse.SUPPRESS) + else: + ap2.add_argument("--sig-thr", action="store_true", help="start separate thread for OS-signals (try this if CTRL-C is busted)") ap2.add_argument("--rm-sck", action="store_true", help="when listening on unix-sockets, do a basic delete+bind instead of the default atomic bind") ap2.add_argument("--srch-dbg", action="store_true", help="explain search processing, and do some extra expensive sanity checks") ap2.add_argument("--rclone-mdns", action="store_true", help="use mdns-domain instead of server-ip on /?hc") ap2.add_argument("--stackmon", metavar="P,S", type=u, default="", help="write stacktrace to \033[33mP\033[0math every \033[33mS\033[0m second, for example --stackmon=\033[32m./st/%%Y-%%m/%%d/%%H%%M.xz,60") + ap2.add_argument("--stack-sig", metavar="S", type=u, default="", help="show stacktrace when unix-signal \033[33mS\033[0m is received; examples: [\033[32mSIGUSR2\033[0m], [\033[32mUSR2\033[0m], [\033[32m12\033[0m]") ap2.add_argument("--log-thrs", metavar="SEC", type=float, default=0.0, help="list active threads every \033[33mSEC\033[0m") ap2.add_argument("--log-fk", metavar="REGEX", type=u, default="", help="log filekey params for files where path matches \033[33mREGEX\033[0m; [\033[32m.\033[0m] (a single dot) = all files") ap2.add_argument("--bak-flips", action="store_true", help="[up2k] if a client uploads a bitflipped/corrupted chunk, store a copy according to \033[33m--bf-nc\033[0m and \033[33m--bf-dir\033[0m") diff --git a/copyparty/__version__.py b/copyparty/__version__.py index 9d2458c4..a33ecb07 100644 --- a/copyparty/__version__.py +++ b/copyparty/__version__.py @@ -1,8 +1,8 @@ # coding: utf-8 -VERSION = (1, 20, 14) +VERSION = (1, 20, 16) CODENAME = "sftp is fine too" -BUILD_DT = (2026, 4, 24) +BUILD_DT = (2026, 5, 26) S_VERSION = ".".join(map(str, VERSION)) S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT) diff --git a/copyparty/authsrv.py b/copyparty/authsrv.py index b261a1f4..4a34824f 100644 --- a/copyparty/authsrv.py +++ b/copyparty/authsrv.py @@ -627,6 +627,9 @@ class VFS(object): t = "%s has no %s in %r => %r => %r" self.log("vfs", t % (uname, msg, vpath, cvpath, ap), 6) + if not err: + return None + t = "you don't have %s-access in %r or below %r" raise Pebkac(err, t % (msg, "/" + cvpath, "/" + vn.vpath)) @@ -858,7 +861,7 @@ class VFS(object): for le in vfs_ls: ap = absreal(os.path.join(fsroot, le[0])) vn2 = self.chk_ap(ap) - if not vn2 or not vn2.get("", uname, True, False): + if not vn2 or not vn2.get("", uname, True, False, err=0): rm1.append(le) _ = [vfs_ls.remove(x) for x in rm1] # type: ignore @@ -900,20 +903,14 @@ class VFS(object): def zipgen( self, - vpath: str, + folder: str, vrem: str, flt: set[str], uname: str, dirs: bool, dots: int, scandir: bool, - wrap: bool = True, ) -> Generator[dict[str, Any], None, None]: - - # if multiselect: add all items to archive root - # if single folder: the folder itself is the top-level item - folder = "" if flt or not wrap else (vpath.split("/")[-1].lstrip(".") or "top") - g = self.walk(folder, vrem, [], uname, [[True, False]], dots, scandir, False) for _, _, vpath, apath, files, rd, vd in g: if flt: @@ -3082,10 +3079,10 @@ class AuthSrv(object): pwds.extend([x.split(":", 1)[1] for x in pwds if ":" in x]) if pwds: if self.ah.on: - zs = r"(\[H\] %s:.*|[?&]%s=)([^&]+)" + zs = r"(\[[HO]\] %s:.*|[?&]%s=)([^&]+)" zs = zs % (self.args.pw_hdr, self.args.pw_urlp) else: - zs = r"(\[H\] %s:.*|=)(" % (self.args.pw_hdr,) + zs = r"(\[[HO]\] %s:.*|=)(" % (self.args.pw_hdr,) zs += "|".join(pwds) + r")([]&; ]|$)" self.re_pwd = re.compile(zs) diff --git a/copyparty/broker_util.py b/copyparty/broker_util.py index 10c778dc..50264e83 100644 --- a/copyparty/broker_util.py +++ b/copyparty/broker_util.py @@ -16,6 +16,7 @@ if True: # pylint: disable=using-constant-test if TYPE_CHECKING: from .httpsrv import HttpSrv + from .svchub import SvcHub class ExceptionalQueue(Queue, object): @@ -50,6 +51,7 @@ class BrokerCli(object): for example resolving httpconn.* in httpcli -- see lines tagged #mypy404 """ + hub: "SvcHub" log: "RootLogger" args: argparse.Namespace asrv: AuthSrv diff --git a/copyparty/httpcli.py b/copyparty/httpcli.py index 4732611d..ca79f484 100644 --- a/copyparty/httpcli.py +++ b/copyparty/httpcli.py @@ -1938,14 +1938,13 @@ class HttpCli(object): # because lstat=true would not recurse into subfolders # and this is a rare case where we actually want that fgen = vn.zipgen( - rem, + "", rem, set(), self.uname, True, 1, not self.args.no_scandir, - wrap=False, ) elif depth == "0": @@ -5313,6 +5312,16 @@ class HttpCli(object): if items: fn = "sel-" + fn + if "name" in self.ouparam: + # user-selected name for toplevel folder, or blank for none + vpath = undot(self.ouparam["name"]) + elif items: + # multiselect; add all items to archive root + vpath = "" + else: + # single folder; the folder itself is the top-level item + vpath = vpath.split("/")[-1].lstrip(".") or "top" + if vn.flags.get("zipmax") and not ( vn.flags.get("zipmaxu") and self.uname != "*" ): @@ -5359,7 +5368,7 @@ class HttpCli(object): if cfmt: self.log("transcoding to [{}]".format(cfmt)) - fgen = gfilter(fgen, self.thumbcli, self.uname, vpath, cfmt) + fgen = gfilter(fgen, self.thumbcli, self.uname, self.vpath, vpath, cfmt) now = time.time() self.dl_id = "%s:%s" % (self.ip, self.addr[1]) @@ -7337,18 +7346,19 @@ class HttpCli(object): dirs = [] files = [] ptn_hr = RE_HR - use_abs_url = ( - not is_opds - and not is_ls - and not is_js - and not self.trailing_slash - and vpath + use_abs_url = is_opds or ( + vpath and not is_ls and not is_js and not self.trailing_slash ) for fn in ls_names: base = "" href = fn if use_abs_url: - base = "/" + vpath + "/" + if is_opds: + base = self.args.SRS + if vpath: + base += vpath + "/" + else: + base = "/" + vpath + "/" href = base + fn if fn in vfs_virt: @@ -7388,7 +7398,7 @@ class HttpCli(object): margin = "-" sz = inf.st_size - zd = datetime.fromtimestamp(max(0, linf.st_mtime), UTC) + zd = datetime.fromtimestamp(max(0, min(2 << 36, linf.st_mtime)), UTC) dt = "%04d-%02d-%02d %02d:%02d:%02d" % ( zd.year, zd.month, @@ -7678,17 +7688,26 @@ class HttpCli(object): ] j2a["opds_osd"] = "%s%s?opds&osd" % (self.args.SRS, quotep(vpath)) - + j2a["opds_id"] = uuid.uuid5(uuid.NAMESPACE_URL, vpath + "/").urn + j2a["opds_title"] = ( + (vpath.rsplit("/", 1)[-1] + "/") if vpath else self.args.bname + ) for item in dirs: href = item["href"] href += ("&" if "?" in href else "?") + "opds" item["href"] = href + item["opds_id"] = uuid.uuid5( + uuid.NAMESPACE_URL, "%s/%s" % (vpath, item["name"]) + ).urn item["iso8601"] = "%sZ" % (item["dt"].replace(" ", "T"),) for item in files: href = item["href"] href += ("&" if "?" in href else "?") + "dl" item["href"] = href + item["opds_id"] = uuid.uuid5( + uuid.NAMESPACE_URL, "%s/%s" % (vpath, item["name"]) + ).urn item["iso8601"] = "%sZ" % (item["dt"].replace(" ", "T"),) if "rmagic" in self.vn.flags: diff --git a/copyparty/httpsrv.py b/copyparty/httpsrv.py index 8e062d76..ecaeaa70 100644 --- a/copyparty/httpsrv.py +++ b/copyparty/httpsrv.py @@ -703,7 +703,7 @@ class HttpSrv(object): if not fmts: continue log("starting for volume /%s" % (vn.vpath,), 6) - g = vn.walk("x", "/", [], LEELOO_DALLAS, [[True]], 2, scandir, False, False) + g = vn.walk("", "/", [], LEELOO_DALLAS, [[True]], 2, scandir, False, False) g = gfilter2(g, self, vn.vpath, fmts.split(",")) for f in g: nfiles += 1 diff --git a/copyparty/mtag.py b/copyparty/mtag.py index 65dcd9b8..d9290783 100644 --- a/copyparty/mtag.py +++ b/copyparty/mtag.py @@ -46,24 +46,34 @@ except: HAVE_MUTAGEN = False -def have_ff(scmd: str) -> bool: - if ANYWIN: +def have_ff(name: str) -> bytes: + uname = name.upper() + if os.environ.get("PRTY_NO_" + uname): + return b"" + + ebin = os.environ.get("PRTY_%s_BIN" % (uname,)) + try: + scmd = (ebin or name).decode("utf-8") + except: + scmd: str = ebin or name + + if ANYWIN and not ebin: scmd += ".exe" if PY2: - print("# checking {}".format(scmd)) - acmd = (scmd + " -version").encode("ascii").split(b" ") + print("# checking %s" % (scmd,)) + bcmd = scmd.encode("utf-8") try: - sp.Popen(acmd, stdout=sp.PIPE, stderr=sp.PIPE).communicate() - return True + sp.Popen([bcmd, b"-version"], stdout=sp.PIPE, stderr=sp.PIPE).communicate() + return bcmd except: - return False + return b"" else: - return bool(shutil.which(scmd)) + return (shutil.which(scmd) or "").encode("utf-8") -HAVE_FFMPEG = not os.environ.get("PRTY_NO_FFMPEG") and have_ff("ffmpeg") -HAVE_FFPROBE = not os.environ.get("PRTY_NO_FFPROBE") and have_ff("ffprobe") +HAVE_FFMPEG = have_ff("ffmpeg") +HAVE_FFPROBE = have_ff("ffprobe") CBZ_PICS = set("png jpg jpeg gif bmp tga tif tiff webp avif jxl".split()) CBZ_01 = re.compile(r"(^|[^0-9v])0+[01]\b") @@ -219,7 +229,7 @@ def ffprobe( ) -> tuple[dict[str, tuple[int, Any]], dict[str, list[Any]], list[Any], dict[str, Any]]: # ffprobe -hide_banner -show_streams -show_format -- cmd = [ - b"ffprobe", + HAVE_FFPROBE, b"-hide_banner", b"-show_streams", b"-show_format", diff --git a/copyparty/sutil.py b/copyparty/sutil.py index 7fc03bab..97c10086 100644 --- a/copyparty/sutil.py +++ b/copyparty/sutil.py @@ -61,6 +61,7 @@ def gfilter( thumbcli: ThumbCli, uname: str, vtop: str, + vname: str, fmt: str, ) -> Generator[dict[str, Any], None, None]: from concurrent.futures import ThreadPoolExecutor @@ -70,7 +71,7 @@ def gfilter( _pools[tp] = 1 try: for f in fgen: - task = tp.submit(enthumb, thumbcli, uname, vtop, f, fmt) + task = tp.submit(enthumb, thumbcli, uname, vtop, vname, f, fmt) pend.append((task, f)) if pend[0][0].done() or len(pend) > CORES * 4: task, f = pend.pop(0) @@ -130,7 +131,7 @@ def gfilter2( try: f = {"vp": vp, "st": fi[1]} task = tp.submit( - enthumb, hsrv.thumbcli, LEELOO_DALLAS, vtop, f, fmt + enthumb, hsrv.thumbcli, LEELOO_DALLAS, vtop, "", f, fmt ) pend.append((task, f)) if pend[0][0].done() or len(pend) > CORES * 4: @@ -152,14 +153,17 @@ def gfilter2( def enthumb( - thumbcli: ThumbCli, uname: str, vtop: str, f: dict[str, Any], fmt: str + thumbcli: ThumbCli, uname: str, vtop: str, vname: str, f: dict[str, Any], fmt: str ) -> dict[str, Any]: rem = f["vp"] ext = rem.rsplit(".", 1)[-1].lower() if (fmt == "mp3" and ext == "mp3") or (fmt == "opus" and ext in TAR_NO_OPUS): raise Exception() - vp = vjoin(vtop, rem.split("/", 1)[1]) + if vname: + vp = vjoin(vtop, rem.split("/", 1)[1]) + else: + vp = vjoin(vtop, rem) vn, rem = thumbcli.asrv.vfs.get(vp, uname, True, False) dbv, vrem = vn.get_dbv(rem) thp = thumbcli.get(dbv, vrem, f["st"].st_mtime, fmt) diff --git a/copyparty/svchub.py b/copyparty/svchub.py index 2a4737fc..b0b1f9ef 100644 --- a/copyparty/svchub.py +++ b/copyparty/svchub.py @@ -41,10 +41,11 @@ from .th_srv import ( H_PIL_AVIF, H_PIL_HEIF, H_PIL_WEBP, + HAVE_DCRAW, HAVE_FFMPEG, HAVE_FFPROBE, HAVE_PIL, - HAVE_RAW, + HAVE_RAWPY, HAVE_VIPS, ThumbSrv, ) @@ -81,6 +82,7 @@ from .util import ( odfusion, pybin, read_utf8, + signame2int, start_log_thrs, start_stackmon, termsize, @@ -106,11 +108,19 @@ if PY2: else: from urllib.request import Request, urlopen +try: + from queue import SimpleQueue +except: + # yuul b. alwright + from queue import Queue as SimpleQueue + VER_IDP_DB = 1 VER_SESSION_DB = 1 VER_SHARES_DB = 2 +CVE_SEVS = {"low": 1, "medium": 2, "moderate": 2, "high": 3, "critical": 4} + class SvcHub(object): """ @@ -139,13 +149,9 @@ class SvcHub(object): self.logf: Optional[typing.TextIO] = None self.logf_base_fn = "" self.is_dut = False # running in unittest; always False - self.stop_req = False self.stopping = False self.stopped = False - self.reload_req = False self.reload_mutex = threading.Lock() - self.stop_cond = threading.Condition() - self.nsigs = 3 self.retcode = 0 self.httpsrv_up = 0 self.qr_tsz = None @@ -155,6 +161,12 @@ class SvcHub(object): self.cmon = 0 self.tstack = 0.0 + self.sig_logrot = -999 + self.sig_reload = -999 + self.sig_stack = -999 + self.nsigs = 7 + self.sig = SimpleQueue() + self.iphash = HMaccas(os.path.join(self.E.cfg, "iphash"), 8) if args.sss or args.s >= 3: @@ -288,6 +300,9 @@ class SvcHub(object): self.log("root", "vc-age too low for copyparty.eu; will use 3 hours") args.vc_age = zi + if args.vc_sev and args.vc_sev not in CVE_SEVS: + self.log("root", "vc-sev %r invalid; will use 'low'" % (args.vc_sev,), 3) + zs = "" if args.th_ram_max < 0.22: zs = "generate thumbnails" @@ -398,7 +413,7 @@ class SvcHub(object): decs.pop("vips", None) if not HAVE_PIL: decs.pop("pil", None) - if not HAVE_RAW: + if not HAVE_RAWPY and not HAVE_DCRAW: decs.pop("raw", None) if not HAVE_FFMPEG or not HAVE_FFPROBE: decs.pop("ff", None) @@ -1009,7 +1024,8 @@ class SvcHub(object): (HAVE_ZMQ, "pyzmq", "send zeromq messages from event-hooks"), (H_PIL_HEIF, "pillow-heif", "read .heif pics with pillow (rarely useful)"), (H_PIL_AVIF, "pillow-avif", "read .avif pics with pillow (rarely useful)"), - (HAVE_RAW, "rawpy", "read RAW images"), + (HAVE_RAWPY, "rawpy", "read RAW images"), + (HAVE_DCRAW, "libraw", "read RAW images"), ] if ANYWIN: to_check += [ @@ -1449,31 +1465,42 @@ class SvcHub(object): sigs = [signal.SIGINT, signal.SIGTERM] if not ANYWIN: - sigs.append(signal.SIGUSR1) + sigs.append(signal.SIGHUP) + + for (opt, mem) in ( + ("logrot_sig", "sig_logrot"), + ("reload_sig", "sig_reload"), + ("stack_sig", "sig_stack"), + ): + zs = getattr(self.args, opt) + if not zs: + continue + zi = signame2int(zs) + setattr(self, mem, zi) + try: + sigs.append(signal.Signals(zi)) + except: + t = "using unknown signal %r as %s" + self.log("root", t % (zi, mem), 3) + sigs.append(zi) for sig in sigs: signal.signal(sig, self.signal_handler) - # macos hangs after shutdown on sigterm with while-sleep, - # windows cannot ^c stop_cond (and win10 does the macos thing but winxp is fine??) - # linux is fine with both, - # never lucky - if ANYWIN: - # msys-python probably fine but >msys-python - Daemon(self.stop_thr, "svchub-sig") + if self.args.sig_thr: + Daemon(self._signal_thr, "svchub-sig") try: - while not self.stop_req: + while not self.stopping: time.sleep(1) except: pass - self.shutdown() # cant join; eats signals on win10 while not self.stopped: time.sleep(0.1) else: - self.stop_thr() + self._signal_thr() def start_zeroconf(self) -> None: self.zc_ngen += 1 @@ -1531,17 +1558,6 @@ class SvcHub(object): self.asrv.load_sessions(True) self.broker.reload_sessions() - def stop_thr(self) -> None: - while not self.stop_req: - with self.stop_cond: - self.stop_cond.wait(9001) - - if self.reload_req: - self.reload_req = False - self.reload(True, True) - - self.shutdown() - def kill9(self, delay: float = 0.0) -> None: if delay > 0.01: time.sleep(delay) @@ -1554,26 +1570,42 @@ class SvcHub(object): os.kill(os.getpid(), signal.SIGKILL) def signal_handler(self, sig: int, frame: Optional[FrameType]) -> None: - if self.stopping: - if self.nsigs <= 0: + if sig in (signal.SIGINT, signal.SIGTERM): + self.nsigs -= 1 + + if self.nsigs == 0: try: threading.Thread(target=self.pr, args=("OMBO BREAKER",)).start() time.sleep(0.1) except: pass + if self.nsigs <= 0: self.kill9() - else: - self.nsigs -= 1 - return - if not ANYWIN and sig == signal.SIGUSR1: - self.reload_req = True + self.sig.put(sig) + + def _signal_thr(self) -> None: + while not self.stopping: + sig = self.sig.get() + self._signal_handler(sig) + + def _signal_handler(self, sig: int) -> None: + if sig == self.sig_logrot: + self.log("root", "signal: logrotate") + dt = datetime.now(self.tz) + self.logf_base_fn = "\t" + self._set_next_day(dt) + + elif sig == self.sig_reload: + self.log("root", "signal: reload") + self.reload(True, True) + + elif sig == self.sig_stack: + self.log("root", "signal: stack%s" % (alltrace(),)) + else: - self.stop_req = True - - with self.stop_cond: - self.stop_cond.notify_all() + self.shutdown() def shutdown(self) -> None: if self.stopping: @@ -1581,10 +1613,8 @@ class SvcHub(object): # start_log_thrs(print, 0.1, 1) + self.nsigs = 3 self.stopping = True - self.stop_req = True - with self.stop_cond: - self.stop_cond.notify_all() ret = 1 try: @@ -1928,6 +1958,7 @@ class SvcHub(object): next_chk = 0 # self.args.vc_age = 2 / 60 fpath = os.path.join(self.E.cfg, "vuln_advisory.json") + minsev = CVE_SEVS.get(self.args.vc_sev, 0) while not self.stopping: now = time.time() if now < next_chk: @@ -1971,10 +2002,13 @@ class SvcHub(object): continue try: + sver = "0.1" advisories = json.loads(jtxt) for adv in advisories: if adv.get("state") == "closed": continue + if CVE_SEVS.get(adv.get("severity"), 9) < minsev: + continue vuln = {} for x in adv["vulnerabilities"]: if x["package"]["name"].lower() == "copyparty": @@ -1992,9 +2026,8 @@ class SvcHub(object): if self.args.vc_exit: self.sigterm() return - else: - t = "%sok; v%s and newer is safe" - self.log("ver-chk", t % (src, sver), 2) + t = "%sok; v%s and newer is safe" + self.log("ver-chk", t % (src, sver), 2) next_chk = time.time() + self.args.vc_age * 3600 - age except Exception as e: t = "failed to process vulnerability advisory; %s" diff --git a/copyparty/th_srv.py b/copyparty/th_srv.py index f119c5e5..f1393a2e 100644 --- a/copyparty/th_srv.py +++ b/copyparty/th_srv.py @@ -18,7 +18,7 @@ from queue import Queue from .__init__ import ANYWIN, PY2, TYPE_CHECKING, unicode from .authsrv import VFS from .bos import bos -from .mtag import HAVE_FFMPEG, HAVE_FFPROBE, au_unpk, ffprobe +from .mtag import HAVE_FFMPEG, HAVE_FFPROBE, au_unpk, ffprobe, have_ff from .util import BytesIO # type: ignore from .util import ( FFMPEG_URL, @@ -201,16 +201,22 @@ except Exception as e: logging.warning("libvips found, but failed to load: " + str(e)) +PRTY_NO_RAW = os.environ.get("PRTY_NO_RAW") +PRTY_NO_RAWPY = PRTY_NO_RAW or os.environ.get("PRTY_NO_RAWPY") +PRTY_NO_DCRAW = PRTY_NO_RAW or os.environ.get("PRTY_NO_DCRAW") try: - if os.environ.get("PRTY_NO_RAW"): + if PRTY_NO_RAWPY: raise Exception() - HAVE_RAW = True + HAVE_RAWPY = True import rawpy logging.getLogger("rawpy").setLevel(logging.WARNING) except: - HAVE_RAW = False + HAVE_RAWPY = False + + +HAVE_DCRAW = not PRTY_NO_DCRAW and have_ff("dcraw_emu") th_dir_cache = {} @@ -224,11 +230,6 @@ def thumb_path(histpath: str, rem: str, mtime: float, fmt: str, ffa: set[str]) - if not rd: rd = "\ntop" - # spectrograms are never cropped; strip fullsize flag - ext = rem.split(".")[-1].lower() - if ext in ffa and fmt[:2] in ("wf", "jf", "xf"): - fmt = fmt.replace("f", "") - dcache = th_dir_cache rd_key = rd + "\n" + fmt rd = dcache.get(rd_key) @@ -307,6 +308,8 @@ class ThumbSrv(object): if ANYWIN and self.args.no_acode: self.log("download FFmpeg to fix it:\033[0m " + FFMPEG_URL, 3) + self.conv_raw = self._conv_rawpy if HAVE_RAWPY else self._conv_dcraw + if self.args.th_clean: Daemon(self.cleaner, "thumb.cln") @@ -758,8 +761,39 @@ class ThumbSrv(object): self.conv_image_vips(_loader, tpath, fmt, vn) - def conv_raw(self, abspath: str, tpath: str, fmt: str, vn: VFS) -> None: - self.wait4ram(0.2, tpath) + def _conv_dcraw(self, abspath: str, tpath: str, fmt: str, vn: VFS) -> None: + self.wait4ram(0.6, tpath) + # fmt: off + cmd = [ + b"dcraw_emu", + b"-h", # halfsize + b"-o", b"1", # srgb + b"-s", b"0", # first frame + b"-Z", b"-", # to stdout + fsenc(abspath), + ] + # fmt: on + p = sp.Popen(cmd, stdout=sp.PIPE) + try: + if HAVE_PIL: + self.conv_image_pil(Image.open(p.stdout), tpath, fmt, vn) + elif HAVE_VIPS: + ppm, _ = p.communicate(timeout=vn.flags["convt"]) + + def _loader(w: int, kw: dict) -> Any: + return pyvips.Image.thumbnail_buffer(ppm, w, **kw) + + self.conv_image_vips(_loader, tpath, fmt, vn) + else: + raise Exception( + "either pil or vips is needed to process embedded bitmap thumbnails in raw files" + ) + finally: + if p and p.poll() is None: + p.kill() + + def _conv_rawpy(self, abspath: str, tpath: str, fmt: str, vn: VFS) -> None: + self.wait4ram(0.6, tpath) with rawpy.imread(abspath) as raw: thumb = raw.extract_thumb() if thumb.format == rawpy.ThumbFormat.JPEG and tpath.endswith(".jpg"): @@ -826,7 +860,7 @@ class ThumbSrv(object): bscale = scale.format(*list(res)).encode("utf-8") # fmt: off cmd = [ - b"ffmpeg", + HAVE_FFMPEG, b"-nostdin", b"-v", b"error", b"-hide_banner" @@ -967,7 +1001,7 @@ class ThumbSrv(object): # fmt: off cmd = [ - b"ffmpeg", + HAVE_FFMPEG, b"-nostdin", b"-v", b"error", b"-hide_banner", @@ -1046,7 +1080,7 @@ class ThumbSrv(object): # fmt: off cmd = [ - b"ffmpeg", + HAVE_FFMPEG, b"-nostdin", b"-v", b"error", b"-hide_banner", @@ -1078,7 +1112,7 @@ class ThumbSrv(object): # fmt: off cmd = [ - b"ffmpeg", + HAVE_FFMPEG, b"-nostdin", b"-v", b"error", b"-hide_banner", @@ -1114,7 +1148,7 @@ class ThumbSrv(object): # fmt: off cmd = [ - b"ffmpeg", + HAVE_FFMPEG, b"-nostdin", b"-v", b"error", b"-hide_banner", @@ -1143,7 +1177,7 @@ class ThumbSrv(object): # fmt: off cmd = [ - b"ffmpeg", + HAVE_FFMPEG, b"-nostdin", b"-v", b"error", b"-hide_banner", @@ -1178,7 +1212,7 @@ class ThumbSrv(object): # fmt: off cmd = [ - b"ffmpeg", + HAVE_FFMPEG, b"-nostdin", b"-v", b"error", b"-hide_banner", @@ -1239,7 +1273,7 @@ class ThumbSrv(object): # fmt: off cmd = [ - b"ffmpeg", + HAVE_FFMPEG, b"-nostdin", b"-v", b"error", b"-hide_banner", @@ -1280,7 +1314,7 @@ class ThumbSrv(object): # fmt: off cmd = [ - b"ffmpeg", + HAVE_FFMPEG, b"-nostdin", b"-v", b"error", b"-hide_banner", @@ -1306,7 +1340,7 @@ class ThumbSrv(object): self.log("conv2 caf-transcode; dur=%d sz=%d q=%s" % (dur, sz, zs), 6) # fmt: off cmd = [ - b"ffmpeg", + HAVE_FFMPEG, b"-nostdin", b"-v", b"error", b"-hide_banner", @@ -1327,7 +1361,7 @@ class ThumbSrv(object): self.log("conv2 caf-remux; dur=%d sz=%d" % (dur, sz), 6) # fmt: off cmd = [ - b"ffmpeg", + HAVE_FFMPEG, b"-nostdin", b"-v", b"error", b"-hide_banner", diff --git a/copyparty/up2k.py b/copyparty/up2k.py index ff1ba97d..957cbc72 100644 --- a/copyparty/up2k.py +++ b/copyparty/up2k.py @@ -3077,7 +3077,7 @@ class Up2k(object): ) zi = cj["lmod"] - bad_mt = zi <= 0 or zi > 0xAAAAAAAA + bad_mt = zi <= 0 or zi > (2 << 36) if bad_mt or vfs.flags.get("up_ts", "") == "fu": # force upload time rather than last-modified cj["lmod"] = int(time.time()) diff --git a/copyparty/util.py b/copyparty/util.py index 2d69d2fe..6396445f 100644 --- a/copyparty/util.py +++ b/copyparty/util.py @@ -1635,6 +1635,16 @@ def expand_osenv_cs(txt) -> str: raise Exception(t) +def signame2int(txt: str) -> int: + try: + return int(txt) + except: + txt = txt.upper() + if not txt.startswith("SIG"): + txt = "SIG" + txt + return int(getattr(signal, txt)) + + def rice_tid() -> str: tid = threading.current_thread().ident c = sunpack(b"B" * 5, spack(b">Q", tid)[-5:]) diff --git a/copyparty/web/opds.xml b/copyparty/web/opds.xml index ea89edf7..d3c52d1a 100644 --- a/copyparty/web/opds.xml +++ b/copyparty/web/opds.xml @@ -1,10 +1,13 @@ + {{ opds_id }} + {{ opds_title | e }} {%- for d in dirs %} + {{ d.opds_id }} {{ d.name | e }} + {{ f.opds_id }} {{ f.name | e }} {{ f.iso8601 }} /dev/null 2>&1 +printf '%s ' $(find $FS/.hist/th/ -size +0 -iname '*.jpg' | wc -l) +done;t=$(date +%s);echo $((t-t0)) + + 95f 55s --th-dec=ff +397f 51s --th-dec=raw (rawpy) +153f 51s --th-dec=vips (no-magick) + +# swithc to persistent for messingaround +podman run --rm -it -v $FS:/w --entrypoint /bin/ash copyparty/iv +apk update +apk upgrade -lai + +t0=$(date +%s); for f in $FMTS ; do rm -rf $FS/.hist +podman exec -it d171470581ab python3 -m copyparty -v /w::r --exit=thgen --th-pregen=j --th-dec=vips --th-r-vips=$f -q >/dev/null 2>&1 +printf '%s ' $(find $FS/.hist/th/ -size +0 -iname '*.jpg' | wc -l) +done;t=$(date +%s);echo $((t-t0)) +# equivalent results + +apk add imagemagick; t0=$(date +%s); rm -rf /w/.hist/ ; for f in $FMTS ; do rm -f /*.jpg; n=0; find /w -type f -iname "*.$f" | while IFS= read -r x; do magick "$x" -scale 320x /$n.jpg >/dev/null 2>&1 ; [ -s /$n.jpg ] || rm -f /$n.jpg; n=$((n+1)); done; echo -n "$(ls -1 / | grep -F .jpg | wc -l) "; done; t=$(date +%s); echo $((t-t0)) + +apk add libraw-tools; t0=$(date +%s); rm -rf /w/.hist/ ; for f in $FMTS ; do rm -f /*.jpg; n=0; find /w -type f -iname "*.$f" | while IFS= read -r x; do [ $(dcraw_emu -h -o 1 -s 0 -Z - "$x" 2>/dev/null | wc -c) -gt 1024 ] && touch /$n.jpg; n=$((n+1)); done; echo -n "$(ls -1 / | grep -F .jpg | wc -l) "; done; t=$(date +%s); echo $((t-t0)) + +dcr erf mdc mef ppm sr2 srf mos pdf 3fr tiff nrw kdc tif srw x3f mrw pef dng raw raf arw crw orf nef cr2 rw2 jpg +d e m m p s s m p 3 t n k t s x m p d r r a c o n c r j +c r d e p r r o d f i r d i r 3 r e n a a r r r e r w p +r f c f m 2 f s f r f w c f w f w f g w f w w f f 2 2 g +----------------------------------------------------------------- +0 0 0 0 1 1 0 0 0 0 3 0 2 6 0 0 0 6 17 0 0 0 0 0 54 0 0 5 = 95, 55s --th-dec=ff +1 1 0 1 0 1 1 2 0 3 0 4 5 6 7 0 8 17 16 2 30 31 17 45 56 56 87 0 =397, 51s --th-dec=raw ## rawpy +1 1 1 1 0 1 1 2 0 3 0 4 5 6 6 0 9 17 18 24 30 31 34 45 56 57 87 0 =440, 87s dcraw_emu +1 1 1 1 1 1 1 2 0 3 6 4 5 6 6 0 9 17 18 24 30 31 34 45 56 57 87 5 =452, 226s magick-cmd +0 1 0 1 1 0 0 0 2 3 3 0 3 6 0 0 0 0 17 0 0 0 0 0 56 55 0 5 =153, 51s --th-dec=vips ## stock +0 1 0 1 1 1 1 0 2 3 3 4 5 6 6 0 8 0 18 11 30 30 18 39 56 55 87 5 =391, 151s vips + apk add imagemagick imagemagick-raw +0 1 0 1 1 0 0 0 2 3 3 0 3 6 0 0 8 0 17 11 30 0 18 39 56 55 87 5 =346, 128s vips + apk del imagemagick (just imagemagick-raw) + +xsel -o | tr ' ' '\n' | awk '!$0{next} {t+=$1} END{print t}' + +apk add time +rm -rf /w/.hist; time python3 -m copyparty -v /w::r --exit=thgen --th-pregen=j --th-dec=raw --th-r-raw=$FMTS_CPP ; find /w/.hist/ -iname '*.jpg' -size +0 | wc -l + +391f, 0:03.77elapsed 227264maxresident # rawpy+vips with-embedded-thumbs +391f, 0:04.79elapsed 467036maxresident # rawpy+pillow with-embedded-thumbs +434f, 0:29.67elapsed 307724maxresident # dcraw+vips +434f, 0:29.34elapsed 327980maxresident # dcraw+pillow +374f, 1:49.70elapsed 4574768maxresident # vips+imagemagick lol lmao diff --git a/scripts/deps-docker/Dockerfile b/scripts/deps-docker/Dockerfile index a79d9426..485461dd 100644 --- a/scripts/deps-docker/Dockerfile +++ b/scripts/deps-docker/Dockerfile @@ -2,7 +2,7 @@ FROM alpine:3.23 WORKDIR /z ENV ver_hashwasm=4.12.0 \ ver_marked=4.3.0 \ - ver_dompf=3.4.1 \ + ver_dompf=3.4.6 \ ver_mde=2.18.0 \ ver_codemirror=5.65.18 \ ver_fontawesome=5.13.0 \ @@ -156,6 +156,7 @@ COPY uncomment.py unhint.py /z RUN mv /z/mfusepy-3.1.1/mfusepy.py /z/dist/ \ && cd /z/dist \ && python3 /z/unhint.py \ + && rm -f uh \ && mv mfusepy.py f1 \ && python3 /z/uncomment.py f1 \ && sed -ri '/self.__critical_exception/d; /^from (typing|collections.abc) import/d' f1 \ diff --git a/scripts/deps-docker/codemirror.patch b/scripts/deps-docker/codemirror.patch index aef47f6f..9e3bfe7f 100644 --- a/scripts/deps-docker/codemirror.patch +++ b/scripts/deps-docker/codemirror.patch @@ -114,12 +114,14 @@ diff -wNarU2 codemirror-5.65.1-orig/src/input/ContentEditableInput.js codemirror + /* let order = getOrder(line, cm.doc.direction), side = "left" if (order) { -@@ -405,4 +406,5 @@ +@@ -405,5 +406,6 @@ side = partPos % 2 ? "right" : "left" } +- let result = nodeAndOffsetInLineMap(info.map, pos.ch, side) + */ - let result = nodeAndOffsetInLineMap(info.map, pos.ch, side) ++ let result = nodeAndOffsetInLineMap(info.map, pos.ch, "left") result.offset = result.collapse == "right" ? result.end : result.start + return result diff -wNarU2 codemirror-5.65.1-orig/src/input/movement.js codemirror-5.65.1/src/input/movement.js --- codemirror-5.65.1-orig/src/input/movement.js 2022-01-20 13:06:23.000000000 +0100 +++ codemirror-5.65.1/src/input/movement.js 2022-02-09 22:50:18.145862052 +0100 diff --git a/scripts/docker/Dockerfile.dj b/scripts/docker/Dockerfile.dj index ec82be50..3796722c 100644 --- a/scripts/docker/Dockerfile.dj +++ b/scripts/docker/Dockerfile.dj @@ -15,22 +15,20 @@ RUN apk add -U !pyc ${ADD_PKG} \ tzdata wget mimalloc2 mimalloc2-insecure \ py3-jinja2 py3-argon2-cffi py3-pyzmq \ py3-openssl py3-paramiko py3-pillow \ - py3-pip py3-cffi \ + py3-pip \ py3-magic \ vips-jxl vips-poppler vips-magick \ py3-numpy fftw libsndfile \ vamp-sdk vamp-sdk-libs keyfinder-cli \ - libraw py3-numpy \ + libraw-tools \ && apk add -t .bd \ bash wget gcc g++ make cmake patchelf \ ffmpeg ffmpeg-dev \ python3-dev fftw-dev libsndfile-dev \ - py3-wheel py3-numpy-dev libffi-dev \ + py3-wheel py3-numpy-dev \ vamp-sdk-dev \ - libraw-dev py3-numpy-dev cython \ && rm -f /usr/lib/python3*/EXTERNALLY-MANAGED \ - && python3 -m pip install pyvips \ - && python3 -m pip install --no-binary rawpy rawpy \ + && python3 -m pip install pyvips --no-build-isolation \ && bash install-deps.sh \ && apk del py3-pip .bd \ && chmod 777 /root \ diff --git a/scripts/docker/Dockerfile.iv b/scripts/docker/Dockerfile.iv index dff98cd0..1b9e7c2a 100644 --- a/scripts/docker/Dockerfile.iv +++ b/scripts/docker/Dockerfile.iv @@ -12,18 +12,12 @@ RUN apk add -U !pyc ${ADD_PKG} \ tzdata wget mimalloc2 mimalloc2-insecure \ py3-jinja2 py3-argon2-cffi py3-pyzmq \ py3-openssl py3-paramiko py3-pillow \ - py3-pip py3-cffi \ + py3-pip \ py3-magic \ vips-jxl vips-poppler vips-magick \ - libraw py3-numpy \ - && apk add -t .bd \ - bash wget gcc g++ make cmake patchelf \ - python3-dev py3-wheel libffi-dev \ - libraw-dev py3-numpy-dev cython \ + libraw-tools \ && rm -f /usr/lib/python3*/EXTERNALLY-MANAGED \ - && python3 -m pip install pyvips \ - && python3 -m pip install --no-binary rawpy rawpy \ - && apk del py3-pip .bd + && python3 -m pip install pyvips --no-build-isolation COPY i innvikler.sh ./ RUN ash innvikler.sh iv diff --git a/scripts/docker/base/arbeidspakke.sh b/scripts/docker/base/arbeidspakke.sh index a191480c..96d54930 100755 --- a/scripts/docker/base/arbeidspakke.sh +++ b/scripts/docker/base/arbeidspakke.sh @@ -48,7 +48,7 @@ sed -ri 's/\bffplay$//; s/\bsdl2-dev\b//' APKBUILD sed -ri 's/--enable-(ladspa|lv2|vaapi|vulkan)/--disable-\1/' APKBUILD sed -ri 's/--enable-lib(aom|ass|drm|fontconfig|freetype|fribidi|harfbuzz|pulse|rist|srt|ssh|v4l2|vidstab|x264|xvid|zimg|vpl)/--disable-lib\1/' APKBUILD sed -ri 's/\b(v4l-utils|libvpx)-dev\b//' APKBUILD # (try to) drop v4l2_m2m, and use builtin vp8/vp9 instead of libvpx for decode -sed -ri 's/(--disable-vulkan)/\1 --disable-devices --disable-hwaccels --disable-encoders --enable-encoder=flac --enable-encoder=libjxl --enable-encoder=libmp3lame --enable-encoder=libopus --enable-encoder=libwebp --enable-encoder=mjpeg --enable-encoder=pcm_s16le --enable-encoder=pcm_s16le_planar --enable-encoder=png --enable-encoder=rawvideo --enable-encoder=vnull --enable-encoder=wrapped_avframe --disable-muxers --enable-muxer=aiff --enable-muxer=apng --enable-muxer=caf --enable-muxer=ffmetadata --enable-muxer=fifo --enable-muxer=flac --enable-muxer=image2 --enable-muxer=image2pipe --enable-muxer=matroska --enable-muxer=matroska_audio --enable-muxer=mjpeg --enable-muxer=mp3 --enable-muxer=null --enable-muxer=opus --enable-muxer=pcm_s16le --enable-muxer=wav --enable-muxer=webm --enable-muxer=webp --enable-muxer=yuv4mpegpipe --disable-filters --enable-filter=anoisesrc --enable-filter=asplit --enable-filter=amerge --enable-filter=amix --enable-filter=aresample --enable-filter=crop --enable-filter=showspectrumpic --enable-filter=showwavespic --enable-filter=convolution --enable-filter=volume --enable-filter=compand --enable-filter=setsar --enable-filter=scale --disable-decoder=av1 --disable-hwaccel=v4l2_m2m --disable-decoder=h263_v4l2m2m --disable-decoder=h264_v4l2m2m --disable-decoder=mpeg1_v4l2m2m --disable-decoder=mpeg2_v4l2m2m --disable-decoder=mpeg4_v4l2m2m --disable-decoder=vc1_v4l2m2m --disable-decoder=vp8_v4l2m2m --disable-decoder=vp9_v4l2m2m --disable-decoder=subrip --disable-decoder=srt --disable-decoder=pgssub --disable-decoder=cc_dec --disable-decoder=dvdsub --disable-decoder=dvbsub --disable-decoder=ssa --disable-decoder=ass --disable-decoder=opus /' APKBUILD +sed -ri 's/(--disable-vulkan)/\1 --disable-devices --disable-hwaccels --disable-encoders --enable-encoder=flac --enable-encoder=libjxl --enable-encoder=libmp3lame --enable-encoder=libopus --enable-encoder=libwebp --enable-encoder=mjpeg --enable-encoder=pcm_f32le --enable-encoder=pcm_s16le --enable-encoder=pcm_s16le_planar --enable-encoder=png --enable-encoder=rawvideo --enable-encoder=vnull --enable-encoder=wrapped_avframe --disable-muxers --enable-muxer=aiff --enable-muxer=apng --enable-muxer=caf --enable-muxer=ffmetadata --enable-muxer=fifo --enable-muxer=flac --enable-muxer=image2 --enable-muxer=image2pipe --enable-muxer=matroska --enable-muxer=matroska_audio --enable-muxer=mjpeg --enable-muxer=mp3 --enable-muxer=null --enable-muxer=opus --enable-muxer=pcm_f32le --enable-muxer=pcm_s16le --enable-muxer=wav --enable-muxer=webm --enable-muxer=webp --enable-muxer=yuv4mpegpipe --disable-filters --enable-filter=anoisesrc --enable-filter=asplit --enable-filter=amerge --enable-filter=amix --enable-filter=aresample --enable-filter=crop --enable-filter=showspectrumpic --enable-filter=showwavespic --enable-filter=convolution --enable-filter=volume --enable-filter=compand --enable-filter=setsar --enable-filter=scale --disable-decoder=av1 --disable-hwaccel=v4l2_m2m --disable-decoder=h263_v4l2m2m --disable-decoder=h264_v4l2m2m --disable-decoder=mpeg1_v4l2m2m --disable-decoder=mpeg2_v4l2m2m --disable-decoder=mpeg4_v4l2m2m --disable-decoder=vc1_v4l2m2m --disable-decoder=vp8_v4l2m2m --disable-decoder=vp9_v4l2m2m --disable-decoder=subrip --disable-decoder=srt --disable-decoder=pgssub --disable-decoder=cc_dec --disable-decoder=dvdsub --disable-decoder=dvbsub --disable-decoder=ssa --disable-decoder=ass --disable-decoder=opus /' APKBUILD # `- s/av1/libdav1d/; s/libvorbis/vorbis/; s/opus/libopus/; libvorbis and mpg123 gets pulled in by openmpt } diff --git a/scripts/docker/make.sh b/scripts/docker/make.sh index 1595f91e..46987417 100755 --- a/scripts/docker/make.sh +++ b/scripts/docker/make.sh @@ -16,7 +16,7 @@ imgs="dj iv min im ac" dhub_order="iv dj min im ac" ghcr_order="ac im min dj iv" ngs=( - iv-{ppc64le,s390x} + iv-{ppc64le,s390x,arm} dj-{ppc64le,s390x,arm} ) diff --git a/scripts/pyinstaller/deps.sha512 b/scripts/pyinstaller/deps.sha512 index 9d6cdd5f..90257c9e 100644 --- a/scripts/pyinstaller/deps.sha512 +++ b/scripts/pyinstaller/deps.sha512 @@ -28,7 +28,7 @@ f4b4e330995ebe96c0bd06e16e5b26062ece9473f06d369775aa68eab261dedcf32dfdd159acaa22 00731cfdd9d5c12efef04a7161c90c1e5ed1dc4677aa88a1d4054aff836f3430df4da5262ed4289c21637358a9e10e5df16f76743cbf5a29bb3a44b146c19cf3 MarkupSafe-3.0.2-cp313-cp313-win_amd64.whl 8a6e2b13a2ec4ef914a5d62aad3db6464d45e525a82e07f6051ed10474eae959069e165dba011aefb8207cdfd55391d73d6f06362c7eb247b08763106709526e mutagen-1.47.0-py3-none-any.whl a726fb46cce24f781fc8b55a3e6dea0a884ebc3b2b400ea74aa02333699f4955a5dc1e2ec5927ac72f35a624401f3f3b442882ba1cc4cadaf9c88558b5b8bdae packaging-25.0-py3-none-any.whl -efc712162da7fb005c8869a7612d2f4983d2d073ec79e16a58e7bf1fcd01c88b1cc26656f0893c68edd2294be7c3990db2f6bd77e7e3f2613539d57994b6a033 pillow-12.1.1-cp313-cp313-win_amd64.whl +5459cfe12d953ed37c481a992e3509536b7997fbd1bb77158d3465d86d3d57af9a16fd4d695374fe6ed30cbb12ac90a2de3000dd92897ddf8bdcfc3e3de831bd pillow-12.2.0-cp313-cp313-win_amd64.whl b9b98714dfca6fa80b0b3f222965724d63be9c54d19435d1fe768e07016913d6db8d6e043fcb185b55a9bd6fe370a80cf961814fc096046a5f4640d99ed575ef pyinstaller-6.15.0-py3-none-win_amd64.whl cad0f7cf39de691813b1d4abc7d33f8bda99a87d9c5886039b814752e8690364150da26fb61b3e28d5698ff57a90e6dcd619ed2b64b04f72b5aadb75e201bdb0 pyinstaller_hooks_contrib-2025.8-py3-none-any.whl 368ea2da3e3bfe765a37c62227e84774853aaabce6954475fa45c873e5547cb5346ca03a0f6a0789af369285bb3464881fed0275a19066913d9d396d5d9b9947 python-3.13.13-amd64.exe diff --git a/scripts/pyinstaller/notes.txt b/scripts/pyinstaller/notes.txt index 7f7b89e6..832b58a7 100644 --- a/scripts/pyinstaller/notes.txt +++ b/scripts/pyinstaller/notes.txt @@ -39,7 +39,7 @@ fns=( MarkupSafe-3.0.2-cp313-cp313-win_amd64.whl mutagen-1.47.0-py3-none-any.whl packaging-25.0-py3-none-any.whl - pillow-12.1.1-cp313-cp313-win_amd64.whl + pillow-12.2.0-cp313-cp313-win_amd64.whl pyinstaller-6.15.0-py3-none-win_amd64.whl pyinstaller_hooks_contrib-2025.8-py3-none-any.whl python-3.13.13-amd64.exe diff --git a/tests/test_idp.py b/tests/test_idp.py index 3378d3b1..b2ec0580 100644 --- a/tests/test_idp.py +++ b/tests/test_idp.py @@ -38,7 +38,10 @@ class TestVFS(unittest.TestCase): unpacked.append(list(sorted(getattr(axs, k)))) pad = len(unpacked) - len(expected) - self.assertEqual(unpacked, expected + [[]] * pad) + want = expected + [[]] * pad + if want[0] and not want[4]: + want[4] = want[0] + self.assertEqual(unpacked, want) def assertAxsAt(self, au, vp, expected): vn = self.nav(au, vp) diff --git a/tests/test_vfs.py b/tests/test_vfs.py index bd7432b7..0278f8f1 100644 --- a/tests/test_vfs.py +++ b/tests/test_vfs.py @@ -188,8 +188,8 @@ class TestVFS(unittest.TestCase): self.assertAxs(n.axs.uread, ["*", "k"]) self.assertAxs(n.axs.uwrite, []) perm_na = (False, False, False, False, False, False, False, False, False) - perm_rw = (True, True, False, False, False, False, False, False, False) - perm_ro = (True, False, False, False, False, False, False, False, False) + perm_rw = (True, True, False, False, True, False, False, False, False) + perm_ro = (True, False, False, False, True, False, False, False, False) self.assertEqual(vfs.can_access("/", "*"), perm_na) self.assertEqual(vfs.can_access("/", "k"), perm_rw) self.assertEqual(vfs.can_access("/a", "*"), perm_ro)