diff --git a/copyparty/__main__.py b/copyparty/__main__.py index 902c322a..fc524ece 100644 --- a/copyparty/__main__.py +++ b/copyparty/__main__.py @@ -1208,7 +1208,8 @@ def add_thumbnail(ap): ap2.add_argument("--th-r-vips", metavar="T,T", type=u, default="avif,exr,fit,fits,fts,gif,hdr,heic,jp2,jpeg,jpg,jpx,jxl,nii,pfm,pgm,png,ppm,svg,tif,tiff,webp", help="image formats to decode using pyvips") ap2.add_argument("--th-r-ffi", metavar="T,T", type=u, default="apng,avif,avifs,bmp,dds,dib,fit,fits,fts,gif,hdr,heic,heics,heif,heifs,icns,ico,jp2,jpeg,jpg,jpx,jxl,pbm,pcx,pfm,pgm,png,pnm,ppm,psd,qoi,sgi,tga,tif,tiff,webp,xbm,xpm", help="image formats to decode using ffmpeg") ap2.add_argument("--th-r-ffv", metavar="T,T", type=u, default="3gp,asf,av1,avc,avi,flv,h264,h265,hevc,m4v,mjpeg,mjpg,mkv,mov,mp4,mpeg,mpeg2,mpegts,mpg,mpg2,mts,nut,ogm,ogv,rm,ts,vob,webm,wmv", help="video formats to decode using ffmpeg") - ap2.add_argument("--th-r-ffa", metavar="T,T", type=u, default="aac,ac3,aif,aiff,alac,alaw,amr,apac,ape,au,bonk,dfpwm,dts,flac,gsm,ilbc,it,m4a,mo3,mod,mp2,mp3,mpc,mptm,mt2,mulaw,ogg,okt,opus,ra,s3m,tak,tta,ulaw,wav,wma,wv,xm,xpk", help="audio formats to decode using ffmpeg") + ap2.add_argument("--th-r-ffa", metavar="T,T", type=u, default="aac,ac3,aif,aiff,alac,alaw,amr,apac,ape,au,bonk,dfpwm,dts,flac,gsm,ilbc,it,itgz,itxz,itz,m4a,mdgz,mdxz,mdz,mo3,mod,mp2,mp3,mpc,mptm,mt2,mulaw,ogg,okt,opus,ra,s3m,s3gz,s3xz,s3z,tak,tta,ulaw,wav,wma,wv,xm,xmgz,xmxz,xmz,xpk", help="audio formats to decode using ffmpeg") + ap2.add_argument("--au-unpk", metavar="E=F.C", type=u, default="mdz=mod.zip, mdgz=mod.gz, mdxz=mod.xz, s3z=s3m.zip, s3gz=s3m.gz, s3xz=s3m.xz, xmz=xm.zip, xmgz=xm.gz, xmxz=xm.xz, itz=it.zip, itgz=it.gz, itxz=it.xz", help="audio formats to decompress before passing to ffmpeg") def add_transcoding(ap): diff --git a/copyparty/mtag.py b/copyparty/mtag.py index 9d2ba6f8..72f5e3da 100644 --- a/copyparty/mtag.py +++ b/copyparty/mtag.py @@ -7,12 +7,15 @@ import os import shutil import subprocess as sp import sys +import tempfile from .__init__ import ANYWIN, EXE, PY2, WINDOWS, E, unicode +from .authsrv import VFS from .bos import bos from .util import ( FFMPEG_URL, REKOBO_LKEY, + VF_CAREFUL, fsenc, min_ex, pybin, @@ -20,12 +23,13 @@ from .util import ( runcmd, sfsenc, uncyg, + wunlink, ) if True: # pylint: disable=using-constant-test - from typing import Any, Union + from typing import Any, Optional, Union - from .util import RootLogger + from .util import NamedLogger, RootLogger def have_ff(scmd: str) -> bool: @@ -107,6 +111,51 @@ class MParser(object): raise Exception() +def au_unpk(log: "NamedLogger", fmt_map: dict[str, str], abspath: str, vn: Optional[VFS] = None) -> str: + ret = "" + try: + ext = abspath.split(".")[-1].lower() + au, pk = fmt_map[ext].split(".") + + fd, ret = tempfile.mkstemp("." + au) + + if pk == "gz": + import gzip + + fi = gzip.GzipFile(abspath, mode="rb") + + elif pk == "xz": + import lzma + + fi = lzma.open(abspath, "rb") + + elif pk == "zip": + import zipfile + + zf = zipfile.ZipFile(abspath, "r") + zil = zf.infolist() + zil = [x for x in zil if x.filename.lower().split(".")[-1] == au] + fi = zf.open(zil[0]) + + with os.fdopen(fd, "wb") as fo: + while True: + buf = fi.read(32768) + if not buf: + break + + fo.write(buf) + + return ret + + except Exception as ex: + if ret: + t = "failed to decompress audio file [%s]: %r" + log(t % (abspath, ex)) + wunlink(log, ret, vn.flags if vn else VF_CAREFUL) + + return abspath + + def ffprobe( abspath: str, timeout: int = 60 ) -> tuple[dict[str, tuple[int, Any]], dict[str, list[Any]]]: @@ -281,7 +330,7 @@ class MTag(object): or_ffprobe = " or FFprobe" if self.backend == "mutagen": - self.get = self.get_mutagen + self._get = self.get_mutagen try: from mutagen import version # noqa: F401 except: @@ -290,7 +339,7 @@ class MTag(object): if self.backend == "ffprobe": self.usable = self.can_ffprobe - self.get = self.get_ffprobe + self._get = self.get_ffprobe self.prefer_mt = True if not HAVE_FFPROBE: @@ -460,6 +509,17 @@ class MTag(object): return r1 + def get(self, abspath: str) -> dict[str, Union[str, float]]: + ext = abspath.split(".")[-1].lower() + if ext not in self.args.au_unpk: + return self._get(abspath) + + ap = au_unpk(self.log, self.args.au_unpk, abspath) + ret = self._get(ap) + if ap != abspath: + wunlink(self.log, ap, VF_CAREFUL) + return ret + def get_mutagen(self, abspath: str) -> dict[str, Union[str, float]]: ret: dict[str, tuple[int, Any]] = {} @@ -553,10 +613,16 @@ class MTag(object): except: raise # might be expected outside cpython + ext = abspath.split(".")[-1].lower() + if ext in self.args.au_unpk: + ap = au_unpk(self.log, self.args.au_unpk, abspath) + else: + ap = abspath + ret: dict[str, Any] = {} for tagname, parser in sorted(parsers.items(), key=lambda x: (x[1].pri, x[0])): try: - cmd = [parser.bin, abspath] + cmd = [parser.bin, ap] if parser.bin.endswith(".py"): cmd = [pybin] + cmd @@ -593,4 +659,7 @@ class MTag(object): t = "mtag error: tagname {}, parser {}, file {} => {}" self.log(t.format(tagname, parser.bin, abspath, min_ex())) + if ap != abspath: + wunlink(self.log, ap, VF_CAREFUL) + return ret diff --git a/copyparty/svchub.py b/copyparty/svchub.py index e69c3d42..50cc14ed 100644 --- a/copyparty/svchub.py +++ b/copyparty/svchub.py @@ -240,6 +240,10 @@ class SvcHub(object): if not HAVE_FFMPEG or not HAVE_FFPROBE: decs.pop("ff", None) + # compressed formats; "s3z=s3m.zip, s3gz=s3m.gz, ..." + zlss = [x.strip().lower().split("=", 1) for x in args.au_unpk.split(",")] + args.au_unpk = {x[0]: x[1] for x in zlss} + self.args.th_dec = list(decs.keys()) self.thumbsrv = None want_ff = False @@ -280,6 +284,8 @@ class SvcHub(object): if not re.match("^(0|[qv][0-9]|[0-9]{2,3}k)$", args.q_mp3.lower()): t = "invalid mp3 transcoding quality [%s] specified; only supports [0] to disable, a CBR value such as [192k], or a CQ/CRF value such as [v2]" raise Exception(t % (args.q_mp3,)) + else: + args.au_unpk = {} args.th_poke = min(args.th_poke, args.th_maxage, args.ac_maxage) diff --git a/copyparty/th_srv.py b/copyparty/th_srv.py index cb673983..2878ed82 100644 --- a/copyparty/th_srv.py +++ b/copyparty/th_srv.py @@ -15,7 +15,7 @@ from queue import Queue from .__init__ import ANYWIN, TYPE_CHECKING from .authsrv import VFS from .bos import bos -from .mtag import HAVE_FFMPEG, HAVE_FFPROBE, ffprobe +from .mtag import HAVE_FFMPEG, HAVE_FFPROBE, au_unpk, ffprobe from .util import BytesIO # type: ignore from .util import ( FFMPEG_URL, @@ -297,6 +297,12 @@ class ThumbSrv(object): ext = abspath.split(".")[-1].lower() png_ok = False funs = [] + + if ext in self.args.au_unpk: + ap_unpk = au_unpk(self.log, self.args.au_unpk, abspath, vn) + else: + ap_unpk = abspath + if not bos.path.exists(tpath): for lib in self.args.th_dec: if lib == "pil" and ext in self.fmt_pil: @@ -328,7 +334,7 @@ class ThumbSrv(object): for fun in funs: try: - fun(abspath, ttpath, fmt, vn) + fun(ap_unpk, ttpath, fmt, vn) break except Exception as ex: msg = "{} could not create thumbnail of {}\n{}" @@ -346,6 +352,9 @@ class ThumbSrv(object): except: pass + if abspath != ap_unpk: + wunlink(self.log, ap_unpk, vn.flags) + try: wrename(self.log, ttpath, tpath, vn.flags) except: diff --git a/copyparty/up2k.py b/copyparty/up2k.py index 6786fcd9..9d8e7e6a 100644 --- a/copyparty/up2k.py +++ b/copyparty/up2k.py @@ -28,6 +28,7 @@ from .fsutil import Fstab from .mtag import MParser, MTag from .util import ( HAVE_SQLITE3, + VF_CAREFUL, SYMTIME, Daemon, MTHash, @@ -90,9 +91,6 @@ CV_EXTS = set(zsg.split(",")) HINT_HISTPATH = "you could try moving the database to another location (preferably an SSD or NVME drive) using either the --hist argument (global option for all volumes), or the hist volflag (just for this volume)" -VF_CAREFUL = {"mv_re_t": 5, "rm_re_t": 5, "mv_re_r": 0.1, "rm_re_r": 0.1} - - class Dbw(object): def __init__(self, c: "sqlite3.Cursor", n: int, t: float) -> None: self.c = c diff --git a/copyparty/util.py b/copyparty/util.py index 5f4be32c..16edceb7 100644 --- a/copyparty/util.py +++ b/copyparty/util.py @@ -358,6 +358,9 @@ APPLESAN_TXT = r"/(__MACOS|Icon\r\r)|/\.(_|DS_Store|AppleDouble|LSOverride|Docum APPLESAN_RE = re.compile(APPLESAN_TXT) +VF_CAREFUL = {"mv_re_t": 5, "rm_re_t": 5, "mv_re_r": 0.1, "rm_re_r": 0.1} + + pybin = sys.executable or "" if EXE: pybin = "" diff --git a/copyparty/web/browser.js b/copyparty/web/browser.js index 3ca186b3..4920a8e8 100644 --- a/copyparty/web/browser.js +++ b/copyparty/web/browser.js @@ -1700,7 +1700,7 @@ catch (ex) { } var re_au_native = (can_ogg || have_acode) ? /\.(aac|flac|m4a|mp3|ogg|opus|wav)$/i : /\.(aac|flac|m4a|mp3|wav)$/i, - re_au_all = /\.(aac|ac3|aif|aiff|alac|alaw|amr|ape|au|dfpwm|dts|flac|gsm|it|m4a|mo3|mod|mp2|mp3|mpc|mptm|mt2|mulaw|ogg|okt|opus|ra|s3m|tak|tta|ulaw|wav|wma|wv|xm|xpk)$/i; + re_au_all = /\.(aac|ac3|aif|aiff|alac|alaw|amr|ape|au|dfpwm|dts|flac|gsm|it|itgz|itxz|itz|m4a|mdgz|mdxz|mdz|mo3|mod|mp2|mp3|mpc|mptm|mt2|mulaw|ogg|okt|opus|ra|s3m|s3gz|s3xz|s3z|tak|tta|ulaw|wav|wma|wv|xm|xmgz|xmxz|xmz|xpk)$/i; // extract songs + add play column diff --git a/docs/notes.sh b/docs/notes.sh index 1e3bc24f..d4e158c0 100644 --- a/docs/notes.sh +++ b/docs/notes.sh @@ -221,6 +221,11 @@ sox -DnV -r8000 -b8 -c1 /dev/shm/a.wav synth 1.1 sin 400 vol 0.02 # play icon calibration pics for w in 150 170 190 210 230 250; do for h in 130 150 170 190 210; do /c/Program\ Files/ImageMagick-7.0.11-Q16-HDRI/magick.exe convert -size ${w}x${h} xc:brown -fill orange -draw "circle $((w/2)),$((h/2)) $((w/2)),$((h/3))" $w-$h.png; done; done +# compress chiptune modules +mkdir gz; for f in *.*; do pigz -c11 -I100 <"$f" >gz/"$f"gz; touch -r "$f" gz/"$f"gz; done +mkdir xz; for f in *.*; do xz -cz9 <"$f" >xz/"$f"xz; touch -r "$f" xz/"$f"xz; done +mkdir z; for f in *.*; do 7z a -tzip -mx=9 -mm=lzma "z/${f}z" "$f" && touch -r "$f" z/"$f"z; done + ## ## vscode