mirror of
https://github.com/9001/copyparty.git
synced 2025-08-17 09:02:15 -06:00
2b24c50eb7
the default (256 KiB) appears optimal in the most popular scenario (linux host with storage on local physical disk, usually NVMe) was previously a mix of 64 and 512 KiB; now the same value is enforced everywhere download-as-tar is now 20% faster with the default value
122 lines
3.4 KiB
Python
122 lines
3.4 KiB
Python
# coding: utf-8
|
|
from __future__ import print_function, unicode_literals
|
|
|
|
import argparse
|
|
import os
|
|
import tempfile
|
|
from datetime import datetime
|
|
|
|
from .__init__ import CORES
|
|
from .bos import bos
|
|
from .th_cli import ThumbCli
|
|
from .util import UTC, vjoin
|
|
|
|
if True: # pylint: disable=using-constant-test
|
|
from typing import Any, Generator, Optional
|
|
|
|
from .util import NamedLogger
|
|
|
|
|
|
class StreamArc(object):
|
|
def __init__(
|
|
self,
|
|
log: "NamedLogger",
|
|
args: argparse.Namespace,
|
|
fgen: Generator[dict[str, Any], None, None],
|
|
**kwargs: Any
|
|
):
|
|
self.log = log
|
|
self.args = args
|
|
self.fgen = fgen
|
|
self.stopped = False
|
|
|
|
def gen(self) -> Generator[Optional[bytes], None, None]:
|
|
raise Exception("override me")
|
|
|
|
def stop(self) -> None:
|
|
self.stopped = True
|
|
|
|
|
|
def gfilter(
|
|
fgen: Generator[dict[str, Any], None, None],
|
|
thumbcli: ThumbCli,
|
|
uname: str,
|
|
vtop: str,
|
|
fmt: str,
|
|
) -> Generator[dict[str, Any], None, None]:
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
|
|
pend = []
|
|
with ThreadPoolExecutor(max_workers=CORES) as tp:
|
|
try:
|
|
for f in fgen:
|
|
task = tp.submit(enthumb, thumbcli, uname, vtop, f, fmt)
|
|
pend.append((task, f))
|
|
if pend[0][0].done() or len(pend) > CORES * 4:
|
|
task, f = pend.pop(0)
|
|
try:
|
|
f = task.result(600)
|
|
except:
|
|
pass
|
|
yield f
|
|
|
|
for task, f in pend:
|
|
try:
|
|
f = task.result(600)
|
|
except:
|
|
pass
|
|
yield f
|
|
except Exception as ex:
|
|
thumbcli.log("gfilter flushing ({})".format(ex))
|
|
for task, f in pend:
|
|
try:
|
|
task.result(600)
|
|
except:
|
|
pass
|
|
thumbcli.log("gfilter flushed")
|
|
|
|
|
|
def enthumb(
|
|
thumbcli: ThumbCli, uname: str, vtop: str, f: dict[str, Any], fmt: str
|
|
) -> dict[str, Any]:
|
|
rem = f["vp"]
|
|
ext = rem.rsplit(".", 1)[-1].lower()
|
|
if fmt == "opus" and ext in "aac|m4a|mp3|ogg|opus|wma".split("|"):
|
|
raise Exception()
|
|
|
|
vp = vjoin(vtop, rem.split("/", 1)[1])
|
|
vn, rem = thumbcli.asrv.vfs.get(vp, uname, True, False)
|
|
dbv, vrem = vn.get_dbv(rem)
|
|
thp = thumbcli.get(dbv, vrem, f["st"].st_mtime, fmt)
|
|
if not thp:
|
|
raise Exception()
|
|
|
|
ext = "jpg" if fmt == "j" else "webp" if fmt == "w" else fmt
|
|
sz = bos.path.getsize(thp)
|
|
st: os.stat_result = f["st"]
|
|
ts = st.st_mtime
|
|
f["ap"] = thp
|
|
f["vp"] = f["vp"].rsplit(".", 1)[0] + "." + ext
|
|
f["st"] = os.stat_result((st.st_mode, -1, -1, 1, 1000, 1000, sz, ts, ts, ts))
|
|
return f
|
|
|
|
|
|
def errdesc(errors: list[tuple[str, str]]) -> tuple[dict[str, Any], list[str]]:
|
|
report = ["copyparty failed to add the following files to the archive:", ""]
|
|
|
|
for fn, err in errors:
|
|
report.extend([" file: {}".format(fn), "error: {}".format(err), ""])
|
|
|
|
with tempfile.NamedTemporaryFile(prefix="copyparty-", delete=False) as tf:
|
|
tf_path = tf.name
|
|
tf.write("\r\n".join(report).encode("utf-8", "replace"))
|
|
|
|
dt = datetime.now(UTC).strftime("%Y-%m%d-%H%M%S")
|
|
|
|
bos.chmod(tf_path, 0o444)
|
|
return {
|
|
"vp": "archive-errors-{}.txt".format(dt),
|
|
"ap": tf_path,
|
|
"st": bos.stat(tf_path),
|
|
}, report
|