From 1b7634932d9840efe43914e9bfa8f1ac0598c060 Mon Sep 17 00:00:00 2001 From: ed Date: Sat, 19 Aug 2023 19:40:46 +0000 Subject: [PATCH] tar/zip-download: add opus transcoding filter --- README.md | 6 +++- copyparty/__main__.py | 1 + copyparty/httpcli.py | 13 +++++++- copyparty/star.py | 10 ++++++ copyparty/sutil.py | 72 +++++++++++++++++++++++++++++++++++++++++++ copyparty/th_cli.py | 1 + 6 files changed, 101 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 59e5b818..73973c0b 100644 --- a/README.md +++ b/README.md @@ -519,6 +519,10 @@ you can also zip a selection of files or folders by clicking them in the browser ![copyparty-zipsel-fs8](https://user-images.githubusercontent.com/241032/129635374-e5136e01-470a-49b1-a762-848e8a4c9cdc.png) +cool trick: download a folder by appending url-params `?tar&opus` to transcode all audio files (except aac|m4a|mp3|ogg|opus|wma) to opus before they're added to the archive +* super useful if you're 5 minutes away from takeoff and realize you don't have any music on your phone but your server only has flac files and downloading those will burn through all your data + there wouldn't be enough time anyways +* and url-params `&j` / `&w` produce jpeg/webm thumbnails/spectrograms instead of the original audio/video/images + ## uploading @@ -701,7 +705,7 @@ open the `[🎺]` media-player-settings tab to configure it, * `[loop]` keeps looping the folder * `[next]` plays into the next folder * transcode: - * `[flac]` convers `flac` and `wav` files into opus + * `[flac]` converts `flac` and `wav` files into opus * `[aac]` converts `aac` and `m4a` files into opus * `[oth]` converts all other known formats into opus * `aac|ac3|aif|aiff|alac|alaw|amr|ape|au|dfpwm|dts|flac|gsm|it|m4a|mo3|mod|mp2|mp3|mpc|mptm|mt2|mulaw|ogg|okt|opus|ra|s3m|tak|tta|ulaw|wav|wma|wv|xm|xpk` diff --git a/copyparty/__main__.py b/copyparty/__main__.py index 5aac5c08..96255472 100755 --- a/copyparty/__main__.py +++ b/copyparty/__main__.py @@ -1041,6 +1041,7 @@ def add_thumbnail(ap): def add_transcoding(ap): ap2 = ap.add_argument_group('transcoding options') ap2.add_argument("--no-acode", action="store_true", help="disable audio transcoding") + ap2.add_argument("--no-bacode", action="store_true", help="disable batch audio transcoding by folder download (zip/tar)") ap2.add_argument("--ac-maxage", metavar="SEC", type=int, default=86400, help="delete cached transcode output after SEC seconds") diff --git a/copyparty/httpcli.py b/copyparty/httpcli.py index 85411dec..ed637cd3 100644 --- a/copyparty/httpcli.py +++ b/copyparty/httpcli.py @@ -33,7 +33,7 @@ from .__version__ import S_VERSION from .authsrv import VFS # typechk from .bos import bos from .star import StreamTar -from .sutil import StreamArc # typechk +from .sutil import StreamArc, gfilter from .szip import StreamZip from .util import ( HTTPCODE, @@ -2896,6 +2896,16 @@ class HttpCli(object): vpath, rem, set(items), self.uname, dots, False, not self.args.no_scandir ) # for f in fgen: print(repr({k: f[k] for k in ["vp", "ap"]})) + cfmt = "" + if self.thumbcli and not self.args.no_bacode: + for zs in ("opus", "w", "j"): + if zs in self.ouparam or uarg == zs: + cfmt = zs + + if cfmt: + self.log("transcoding to [{}]".format(cfmt)) + fgen = gfilter(fgen, self.thumbcli, self.uname, vpath, cfmt) + bgen = packer(self.log, fgen, utf8="utf" in uarg, pre_crc="crc" in uarg) bsent = 0 for buf in bgen.gen(): @@ -2907,6 +2917,7 @@ class HttpCli(object): bsent += len(buf) except: logmsg += " \033[31m" + unicode(bsent) + "\033[0m" + bgen.stop() break spd = self._spd(bsent) diff --git a/copyparty/star.py b/copyparty/star.py index dc4fc120..113f6d69 100644 --- a/copyparty/star.py +++ b/copyparty/star.py @@ -61,6 +61,7 @@ class StreamTar(StreamArc): Daemon(self._gen, "star-gen") def gen(self) -> Generator[Optional[bytes], None, None]: + buf = b"" try: while True: buf = self.qfile.q.get() @@ -72,6 +73,12 @@ class StreamTar(StreamArc): yield None finally: + while buf: + try: + buf = self.qfile.q.get() + except: + pass + if self.errf: bos.unlink(self.errf["ap"]) @@ -101,6 +108,9 @@ class StreamTar(StreamArc): errors.append((f["vp"], f["err"])) continue + if self.stopped: + break + try: self.ser(f) except: diff --git a/copyparty/sutil.py b/copyparty/sutil.py index b6b21dc0..61c1396b 100644 --- a/copyparty/sutil.py +++ b/copyparty/sutil.py @@ -1,10 +1,14 @@ # coding: utf-8 from __future__ import print_function, unicode_literals +import os import tempfile from datetime import datetime +from .__init__ import CORES from .bos import bos +from .th_cli import ThumbCli +from .util import vjoin if True: # pylint: disable=using-constant-test from typing import Any, Generator, Optional @@ -21,10 +25,78 @@ class StreamArc(object): ): self.log = log self.fgen = fgen + self.stopped = False def gen(self) -> Generator[Optional[bytes], None, None]: raise Exception("override me") + def stop(self) -> None: + self.stopped = True + + +def gfilter( + fgen: Generator[dict[str, Any], None, None], + thumbcli: ThumbCli, + uname: str, + vtop: str, + fmt: str, +) -> Generator[dict[str, Any], None, None]: + from concurrent.futures import ThreadPoolExecutor + + pend = [] + with ThreadPoolExecutor(max_workers=CORES) as tp: + try: + for f in fgen: + task = tp.submit(enthumb, thumbcli, uname, vtop, f, fmt) + pend.append((task, f)) + if pend[0][0].done() or len(pend) > CORES * 4: + task, f = pend.pop(0) + try: + f = task.result(600) + except: + pass + yield f + + for task, f in pend: + try: + f = task.result(600) + except: + pass + yield f + except Exception as ex: + thumbcli.log("gfilter flushing ({})".format(ex)) + for task, f in pend: + try: + task.result(600) + except: + pass + thumbcli.log("gfilter flushed") + + +def enthumb( + thumbcli: ThumbCli, uname: str, vtop: str, f: dict[str, Any], fmt: str +) -> dict[str, Any]: + rem = f["vp"] + ext = rem.rsplit(".", 1)[-1].lower() + if fmt == "opus" and ext in "aac|m4a|mp3|ogg|opus|wma".split("|"): + raise Exception() + + vp = vjoin(vtop, rem.split("/", 1)[1]) + vn, rem = thumbcli.asrv.vfs.get(vp, uname, True, False) + dbv, vrem = vn.get_dbv(rem) + thp = thumbcli.get(dbv, vrem, f["st"].st_mtime, fmt) + if not thp: + raise Exception() + + ext = "jpg" if fmt == "j" else "webp" if fmt == "w" else fmt + sz = bos.path.getsize(thp) + st: os.stat_result = f["st"] + ts = st.st_mtime + f["ap"] = thp + f["vp"] = f["vp"].rsplit(".", 1)[0] + "." + ext + f["st"] = os.stat_result((st.st_mode, -1, -1, 1, 1000, 1000, sz, ts, ts, ts)) + return f + def errdesc(errors: list[tuple[str, str]]) -> tuple[dict[str, Any], list[str]]: report = ["copyparty failed to add the following files to the archive:", ""] diff --git a/copyparty/th_cli.py b/copyparty/th_cli.py index afc15cb8..e0b7b144 100644 --- a/copyparty/th_cli.py +++ b/copyparty/th_cli.py @@ -108,6 +108,7 @@ class ThumbCli(object): if st.st_size: ret = tpath = tp fmt = ret.rsplit(".")[1] + break else: abort = True except: