diff --git a/bin/u2c.py b/bin/u2c.py index a6a564a1..0b79c300 100755 --- a/bin/u2c.py +++ b/bin/u2c.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 from __future__ import print_function, unicode_literals -S_VERSION = "2.21" -S_BUILD_DT = "2026-05-25" +S_VERSION = "2.22" +S_BUILD_DT = "2026-06-13" """ u2c.py: upload to copyparty @@ -733,16 +733,22 @@ def up2k_chunksize(filesize): stepsize *= mul -# mostly from copyparty/up2k.py def get_hashlist(file, pcb, mth): # type: (File, Any, Any) -> None + with open(file.abs, "rb", 512 * 1024) as f: + get_hashlist_f(file, pcb, mth, f) + + +# mostly from copyparty/up2k.py +def get_hashlist_f(file, pcb, mth, f): + # type: (File, Any, Any, Any) -> None """generates the up2k hashlist from file contents, inserts it into `file`""" chunk_sz = up2k_chunksize(file.size) file_rem = file.size file_ofs = 0 ret = [] - with open(file.abs, "rb", 512 * 1024) as f: + if True: t0 = time.time() if mth and file.size >= 1024 * 512: @@ -778,6 +784,18 @@ def get_hashlist(file, pcb, mth): file.kchunks[k] = [v1, v2] +def wark_stdin(ar): + file = File(b"", b"", int(ar.files[0]), 0) + get_hashlist_f(file, None, False, sys.stdin if PY2 else sys.stdin.buffer) + if ar.chs: + zsl = ["%s %d %d" % (zsii[0], n, zsii[1]) for n, zsii in enumerate(file.cids)] + print("chs:\n" + "\n".join(zsl)) + zsl = [ar.wsalt, str(file.size)] + [x[0] for x in file.cids] + zb = hashlib.sha512("\n".join(zsl).encode("utf-8")).digest()[:33] + wark = ub64enc(zb).decode("utf-8") + print(wark) + + def printlink(ar, purl, name, fk): if not name: url = purl # srch @@ -1587,6 +1605,7 @@ NOTE: if server has --usernames enabled, then password is "username:password" ap.add_argument("--wsalt", type=unicode, metavar="S", default="hunter2", help="salt to use when creating warks; must match server config") ap.add_argument("--chs", action="store_true", help="verbose (print the hash/offset of each chunk in each file)") ap.add_argument("--jw", action="store_true", help="just identifier+filepath, not mtime/size too") + ap.add_argument("--stdin", action="store_true", help="calculate file-ID of stdin; u2c.py --stdin - $LEN < a.mkv") ap = app.add_argument_group("performance tweaks") ap.add_argument("-j", type=int, metavar="CONNS", default=2, help="parallel connections") @@ -1617,6 +1636,10 @@ NOTE: if server has --usernames enabled, then password is "username:password" except: pass + if ar.stdin: + wark_stdin(ar) + return + # msys2 doesn't uncygpath absolute paths with whitespace if not VT100: zsl = [] diff --git a/copyparty/__main__.py b/copyparty/__main__.py index a9130937..2de35ead 100644 --- a/copyparty/__main__.py +++ b/copyparty/__main__.py @@ -28,6 +28,7 @@ from .__init__ import ( MACOS, PY2, PY36, + UNIX, VT100, WINDOWS, E, @@ -45,6 +46,7 @@ from .util import ( DEF_EXP, DEF_MTE, DEF_MTH, + HAVE_BWRAP, HAVE_IPV6, IMPLICATIONS, JINJA_VER, @@ -1650,6 +1652,29 @@ def add_optouts(ap): def add_safety(ap): + th_bwrap = "" + if HAVE_BWRAP: + zsl = [ + "bwrap", + "--proc /proc", + "--tmpfs /tmp", + "--tmpfs /var", + "--tmpfs /run", + "--dev-bind /dev/null /dev/null", + "--dev-bind /dev/random /dev/random", + "--dev-bind /dev/urandom /dev/urandom", + "--chdir /tmp", + "--clearenv", + "--unshare-all", + "--cap-drop ALL", + "--die-with-parent", + "--new-session", + ] + for d in ("/lib", "/lib64", "/usr/lib", "/usr/lib64"): + if os.path.isdir(d): + zsl.append(" --ro-bind %s %s" % (d, d)) + th_bwrap = " ".join(zsl) + ap2 = ap.add_argument_group("safety options") ap2.add_argument("-s", action="count", default=0, help="increase safety: Disable thumbnails / potentially dangerous software (ffmpeg/pillow/vips), hide partial uploads, avoid crawlers.\n └─Alias of\033[32m --dotpart --no-thumb --no-mtag-ff --no-robots --force-js") ap2.add_argument("-ss", action="store_true", help="further increase safety: Prevent js-injection, accidental move/delete, broken symlinks, webdav requires login, 404 on 403, ban on excessive 404s.\n └─Alias of\033[32m -s --no-html --no-readme --no-logues --unpost=0 --no-del --no-mv --reflink --dav-auth --vague-403 -nih") @@ -1686,6 +1711,8 @@ def add_safety(ap): ap2.add_argument("--loris", metavar="B", type=int, default=60, help="if a client maxes out the server connection limit without sending headers, ban it for \033[33mB\033[0m minutes; disable with [\033[32m0\033[0m]") ap2.add_argument("--acao", metavar="V[,V]", type=u, default="*", help="Access-Control-Allow-Origin; list of origins (domains/IPs without port) to accept requests from; [\033[32mhttps://1.2.3.4\033[0m]. Default [\033[32m*\033[0m] allows requests from all sites but removes cookies and http-auth; only ?pw=hunter2 survives") ap2.add_argument("--acam", metavar="V[,V]", type=u, default="GET,HEAD", help="Access-Control-Allow-Methods; list of methods to accept from offsite ('*' behaves like \033[33m--acao\033[0m's description)") + if not ANYWIN and not UNIX: + ap2.add_argument("--th-bwrap", metavar="CMD", type=u, default=th_bwrap, help="optional bwrap sandbox command for FFmpeg and dcraw (Linux-only)") def add_salt(ap, fk_salt, dk_salt, ah_salt): diff --git a/copyparty/authsrv.py b/copyparty/authsrv.py index 4a34824f..10447c72 100644 --- a/copyparty/authsrv.py +++ b/copyparty/authsrv.py @@ -2328,7 +2328,7 @@ class AuthSrv(object): zs = vol.flags.get("rotf") if zs: use = True - lim.set_rotf(zs, vol.flags.get("rotf_tz") or "UTC") + lim.set_rotf(zs, vol.flags.get("rotf_tz", self.args.rotf_tz) or "UTC") zs = vol.flags.get("maxn") if zs: diff --git a/copyparty/httpcli.py b/copyparty/httpcli.py index 6e68d743..750f5890 100644 --- a/copyparty/httpcli.py +++ b/copyparty/httpcli.py @@ -127,6 +127,7 @@ from .util import ( unescape_cookie, unquotep, vjoin, + vjoins, vol_san, vroots, vsplit, @@ -2057,7 +2058,7 @@ class HttpCli(object): df = {} fgen = itertools.chain([topdir], fgen) - vtop = vjoin(self.args.R, vjoin(vn.vpath, rem)) + vtop = vjoins(self.args.R, vn.vpath, rem) chunksz = 0x7FF8 # preferred by nginx or cf (dunno which) diff --git a/copyparty/mtag.py b/copyparty/mtag.py index d9290783..dd86caa9 100644 --- a/copyparty/mtag.py +++ b/copyparty/mtag.py @@ -74,6 +74,7 @@ def have_ff(name: str) -> bytes: HAVE_FFMPEG = have_ff("ffmpeg") HAVE_FFPROBE = have_ff("ffprobe") +TH_BWRAP = [] CBZ_PICS = set("png jpg jpeg gif bmp tga tif tiff webp avif jxl".split()) CBZ_01 = re.compile(r"(^|[^0-9v])0+[01]\b") @@ -224,17 +225,28 @@ def au_unpk( return abspath +def bwrap(prog: bytes, ap_in: bytes, ap_out: bytes) -> list[bytes]: + if not TH_BWRAP: + return [prog] + ret = TH_BWRAP + [b"--ro-bind", prog, prog, b"--ro-bind", ap_in, ap_in] + if ap_out: + zs = ap_out.rsplit(b"/", 1)[0] + ret += [b"--bind", zs, zs] + ret.append(prog) + return ret + + def ffprobe( abspath: str, timeout: int = 60 ) -> tuple[dict[str, tuple[int, Any]], dict[str, list[Any]], list[Any], dict[str, Any]]: # ffprobe -hide_banner -show_streams -show_format -- - cmd = [ - HAVE_FFPROBE, + bap = fsenc(abspath) + cmd = bwrap(HAVE_FFPROBE, bap, b"") + [ b"-hide_banner", b"-show_streams", b"-show_format", b"--", - fsenc(abspath), + bap, ] rc, so, se = runcmd(cmd, timeout=timeout, nice=True, oom=200) retchk(rc, cmd, se) diff --git a/copyparty/svchub.py b/copyparty/svchub.py index b0b1f9ef..ad837253 100644 --- a/copyparty/svchub.py +++ b/copyparty/svchub.py @@ -27,13 +27,23 @@ if True: # pylint: disable=using-constant-test import typing from typing import Any, Optional, Union -from .__init__ import ANYWIN, EXE, MACOS, PY2, TYPE_CHECKING, E, EnvParams, unicode +from .__init__ import ( + ANYWIN, + EXE, + MACOS, + PY2, + TYPE_CHECKING, + UNIX, + E, + EnvParams, + unicode, +) from .__version__ import S_VERSION, VERSION from .authsrv import BAD_CFG, AuthSrv, derive_args, n_du_who, n_ver_who from .bos import bos from .cert import ensure_cert from .fsutil import ramdisk_chk -from .mtag import HAVE_FFMPEG, HAVE_FFPROBE, HAVE_MUTAGEN +from .mtag import HAVE_FFMPEG, HAVE_FFPROBE, HAVE_MUTAGEN, TH_BWRAP from .pwhash import HAVE_ARGON2 from .sutil import close_pools as sutil_close_pools from .tcpsrv import TcpSrv @@ -42,8 +52,6 @@ from .th_srv import ( H_PIL_HEIF, H_PIL_WEBP, HAVE_DCRAW, - HAVE_FFMPEG, - HAVE_FFPROBE, HAVE_PIL, HAVE_RAWPY, HAVE_VIPS, @@ -51,10 +59,12 @@ from .th_srv import ( ) from .up2k import Up2k from .util import ( + BLOCK_SIGS, DEF_EXP, DEF_MTE, DEF_MTH, FFMPEG_URL, + HAVE_BWRAP, HAVE_PSUTIL, HAVE_SQLITE3, HAVE_ZMQ, @@ -1012,6 +1022,8 @@ class SvcHub(object): fok = [] fng = [] t_ff = "transcode audio, create spectrograms, video thumbnails" + + # fmt: off to_check = [ (HAVE_SQLITE3, "sqlite", "sessions and file/media indexing"), (HAVE_PIL, "pillow", "image thumbnails (plenty fast)"), @@ -1019,6 +1031,7 @@ class SvcHub(object): (H_PIL_WEBP, "pillow-webp", "create thumbnails as webp files"), (HAVE_FFMPEG, "ffmpeg", t_ff + ", good-but-slow image thumbnails"), (HAVE_FFPROBE, "ffprobe", t_ff + ", read audio/media tags"), + (HAVE_BWRAP, "bwrap", "sandbox to make ffmpeg less dangerous", not ANYWIN and not UNIX), (HAVE_MUTAGEN, "mutagen", "read audio tags (ffprobe is better but slower)"), (HAVE_ARGON2, "argon2", "secure password hashing (advanced users only)"), (HAVE_ZMQ, "pyzmq", "send zeromq messages from event-hooks"), @@ -1026,17 +1039,21 @@ class SvcHub(object): (H_PIL_AVIF, "pillow-avif", "read .avif pics with pillow (rarely useful)"), (HAVE_RAWPY, "rawpy", "read RAW images"), (HAVE_DCRAW, "libraw", "read RAW images"), + (HAVE_PSUTIL, "psutil", "improved plugin cleanup (rarely useful)", ANYWIN), ] - if ANYWIN: - to_check += [ - (HAVE_PSUTIL, "psutil", "improved plugin cleanup (rarely useful)") - ] + # fmt: on verbose = self.args.deps if verbose: self.log("dependencies", "") - for have, feat, what in to_check: + for zc in to_check: + try: + have, feat, what = zc + except: + have, feat, what, zb = zc + if not zb: + continue lst = fok if have else fng lst.append((feat, what)) if verbose: @@ -1203,6 +1220,15 @@ class SvcHub(object): vsa = [x.upper() for x in vsa if x] setattr(al, k + "_set", set(vsa)) + zs = "th_bwrap" + for k in zs.split(" "): + zsl = [x for x in str(getattr(al, k)).split(" ") if x] + zbl = [x.encode("ascii", "replace") for x in zsl] + setattr(al, k + "_s", zsl) + setattr(al, k + "_b", zbl) + + TH_BWRAP[:] = al.th_bwrap_b + zs = "dav_ua1 lf_url sus_urls nonsus_urls ua_nodav ua_nodoc ua_nozip" for k in zs.split(" "): vs = getattr(al, k) @@ -1486,6 +1512,8 @@ class SvcHub(object): for sig in sigs: signal.signal(sig, self.signal_handler) + if sig not in BLOCK_SIGS and BLOCK_SIGS: + BLOCK_SIGS.append(sig) if self.args.sig_thr: Daemon(self._signal_thr, "svchub-sig") diff --git a/copyparty/th_srv.py b/copyparty/th_srv.py index f1393a2e..9174c633 100644 --- a/copyparty/th_srv.py +++ b/copyparty/th_srv.py @@ -18,7 +18,7 @@ from queue import Queue from .__init__ import ANYWIN, PY2, TYPE_CHECKING, unicode from .authsrv import VFS from .bos import bos -from .mtag import HAVE_FFMPEG, HAVE_FFPROBE, au_unpk, ffprobe, have_ff +from .mtag import HAVE_FFMPEG, HAVE_FFPROBE, au_unpk, bwrap, ffprobe, have_ff from .util import BytesIO # type: ignore from .util import ( FFMPEG_URL, @@ -763,14 +763,14 @@ class ThumbSrv(object): def _conv_dcraw(self, abspath: str, tpath: str, fmt: str, vn: VFS) -> None: self.wait4ram(0.6, tpath) + bap = fsenc(abspath) # fmt: off - cmd = [ - b"dcraw_emu", + cmd = bwrap(HAVE_DCRAW, bap, b"") + [ b"-h", # halfsize b"-o", b"1", # srgb b"-s", b"0", # first frame b"-Z", b"-", # to stdout - fsenc(abspath), + bap, ] # fmt: on p = sp.Popen(cmd, stdout=sp.PIPE) @@ -858,16 +858,15 @@ class ThumbSrv(object): res = self.getres(vn, fmt) bscale = scale.format(*list(res)).encode("utf-8") + bap_in = fsenc(abspath) + bap_out = fsenc(tpath) # fmt: off - cmd = [ - HAVE_FFMPEG, + cmd = bwrap(HAVE_FFMPEG, bap_in, bap_out) + [ b"-nostdin", b"-v", b"error", b"-hide_banner" - ] - cmd += seek - cmd += [ - b"-i", fsenc(abspath), + ] + seek + [ + b"-i", bap_in, b"-map", imap, b"-vf", bscale, b"-frames:v", b"1", @@ -875,15 +874,15 @@ class ThumbSrv(object): ] # fmt: on - self._ffmpeg_im_o(tpath, vn, cmd) + self._ffmpeg_im_o(bap_out, vn, cmd) - def _ffmpeg_im_o(self, tpath: str, vn: VFS, cmd: list[bytes]) -> None: - if tpath.endswith(".jpg"): + def _ffmpeg_im_o(self, tpath: bytes, vn: VFS, cmd: list[bytes]) -> None: + if tpath.endswith(b".jpg"): cmd += [ b"-q:v", FF_JPG_Q[vn.flags["th_qv"] // 5], # default=?? ] - elif tpath.endswith(".jxl"): + elif tpath.endswith(b".jxl"): cmd += [ b"-q:v", unicode(vn.flags["th_qvx"]).encode("ascii"), # default=?? @@ -898,7 +897,7 @@ class ThumbSrv(object): b"6", # default=4, 0=fast, 6=max ] - cmd += [fsenc(tpath)] + cmd.append(tpath) self._run_ff(cmd, vn, "convt") def _run_ff(self, cmd: list[bytes], vn: VFS, kto: str, oom: int = 400) -> None: @@ -998,20 +997,21 @@ class ThumbSrv(object): b",showwavespic=s=2048x64:colors=white" b",convolution=1 1 1 1 1 1 1 1 1:1 1 1 1 1 1 1 1 1:1 1 1 1 1 1 1 1 1:1 -1 1 -1 5 -1 1 -1 1" # idk what im doing but it looks ok ) + bap_in = fsenc(abspath) + bap_out = fsenc(tpath) # fmt: off - cmd = [ - HAVE_FFMPEG, + cmd = bwrap(HAVE_FFMPEG, bap_in, bap_out) + [ b"-nostdin", b"-v", b"error", b"-hide_banner", - b"-i", fsenc(abspath), + b"-i", bap_in, b"-filter_complex", flt, b"-frames:v", b"1", ] # fmt: on - cmd += [fsenc(tpath)] + cmd.append(bap_out) self._run_ff(cmd, vn, "convt") if "pngquant" in vn.flags: @@ -1078,19 +1078,21 @@ class ThumbSrv(object): except: self.untemp[tpath] = [infile] + bap_in = fsenc(abspath) + bap_out = fsenc(infile) + # fmt: off - cmd = [ - HAVE_FFMPEG, + cmd = bwrap(HAVE_FFMPEG, bap_in, bap_out) + [ b"-nostdin", b"-v", b"error", b"-hide_banner", - b"-i", fsenc(abspath), + b"-i", bap_in, b"-map", b"0:a:0", b"-ac", b"1", b"-ar", b"48000", b"-sample_fmt", b"s16", b"-t", b"900", - b"-y", fsenc(infile), + b"-y", bap_out, ] # fmt: on self._run_ff(cmd, vn, "convt") @@ -1110,20 +1112,22 @@ class ThumbSrv(object): fc = fc.format(fco) + bap_in = fsenc(infile) + bap_out = fsenc(tpath) + # fmt: off - cmd = [ - HAVE_FFMPEG, + cmd = bwrap(HAVE_FFMPEG, bap_in, bap_out) + [ b"-nostdin", b"-v", b"error", b"-hide_banner", - b"-i", fsenc(infile), + b"-i", bap_in, b"-filter_complex", fc.encode("utf-8"), b"-map", b"[o]", b"-frames:v", b"1", ] # fmt: on - self._ffmpeg_im_o(tpath, vn, cmd) + self._ffmpeg_im_o(bap_out, vn, cmd) def conv_mp3(self, abspath: str, tpath: str, fmt: str, vn: VFS) -> None: quality = self.args.q_mp3.lower() @@ -1142,24 +1146,26 @@ class ThumbSrv(object): qk = b"-q:a" qv = quality[1:].encode("ascii") + bap_in = fsenc(abspath) + bap_out = fsenc(tpath) + # extremely conservative choices for output format # (always 2ch 44k1) because if a device is old enough # to not support opus then it's probably also super picky # fmt: off - cmd = [ - HAVE_FFMPEG, + cmd = bwrap(HAVE_FFMPEG, bap_in, bap_out) + [ b"-nostdin", b"-v", b"error", b"-hide_banner", - b"-i", fsenc(abspath), + b"-i", bap_in, ] + self.big_tags(rawtags) + [ b"-map", b"0:a:0", b"-ar", b"44100", b"-ac", b"2", b"-c:a", b"libmp3lame", qk, qv, - fsenc(tpath) + bap_out, ] # fmt: on self._run_ff(cmd, vn, "aconvt", oom=300) @@ -1175,16 +1181,18 @@ class ThumbSrv(object): self.log("conv2 flac", 6) + bap_in = fsenc(abspath) + bap_out = fsenc(tpath) + # fmt: off - cmd = [ - HAVE_FFMPEG, + cmd = bwrap(HAVE_FFMPEG, bap_in, bap_out) + [ b"-nostdin", b"-v", b"error", b"-hide_banner", - b"-i", fsenc(abspath), + b"-i", bap_in, b"-map", b"0:a:0", b"-c:a", b"flac", - fsenc(tpath) + bap_out, ] # fmt: on self._run_ff(cmd, vn, "aconvt", oom=300) @@ -1210,16 +1218,18 @@ class ThumbSrv(object): self.log("conv2 wav", 6) + bap_in = fsenc(abspath) + bap_out = fsenc(tpath) + # fmt: off - cmd = [ - HAVE_FFMPEG, + cmd = bwrap(HAVE_FFMPEG, bap_in, bap_out) + [ b"-nostdin", b"-v", b"error", b"-hide_banner", - b"-i", fsenc(abspath), + b"-i", bap_in, b"-map", b"0:a:0", b"-c:a", codec, - fsenc(tpath) + bap_out, ] # fmt: on self._run_ff(cmd, vn, "aconvt", oom=300) @@ -1271,19 +1281,21 @@ class ThumbSrv(object): except: pass + bap_in = fsenc(abspath) + bap_out = fsenc(tpath) + # fmt: off - cmd = [ - HAVE_FFMPEG, + cmd = bwrap(HAVE_FFMPEG, bap_in, bap_out) + [ b"-nostdin", b"-v", b"error", b"-hide_banner", - b"-i", fsenc(abspath), + b"-i", bap_in, ] + tagset + [ b"-map", b"0:a:0", b"-ac", ac, ] + benc + [ b"-f", container, - fsenc(tpath) + bap_out, ] # fmt: on self._run_ff(cmd, vn, "aconvt", oom=300) @@ -1312,19 +1324,21 @@ class ThumbSrv(object): self.log("conv2 caf-tmp [%s]" % (enc,), 6) benc = enc.encode("ascii").split(b" ") + bap_in = fsenc(abspath) + bap_out = fsenc(tmp_opus) + # fmt: off - cmd = [ - HAVE_FFMPEG, + cmd = bwrap(HAVE_FFMPEG, bap_in, bap_out) + [ b"-nostdin", b"-v", b"error", b"-hide_banner", - b"-i", fsenc(abspath), + b"-i", bap_in, b"-map_metadata", b"-1", b"-map", b"0:a:0", b"-ac", b"2", ] + benc + [ b"-f", b"opus", - fsenc(tmp_opus) + bap_out, ] # fmt: on self._run_ff(cmd, vn, "aconvt", oom=300) @@ -1338,20 +1352,21 @@ class ThumbSrv(object): if dur < 20 or sz < 256 * 1024: zs = bq.decode("ascii") self.log("conv2 caf-transcode; dur=%d sz=%d q=%s" % (dur, sz, zs), 6) + bap_in = fsenc(abspath) + bap_out = fsenc(tpath) # fmt: off - cmd = [ - HAVE_FFMPEG, + cmd = bwrap(HAVE_FFMPEG, bap_in, bap_out) + [ b"-nostdin", b"-v", b"error", b"-hide_banner", - b"-i", fsenc(abspath), + b"-i", bap_in, b"-filter_complex", b"anoisesrc=a=0.001:d=7:c=pink,asplit[l][r]; [l][r]amerge[s]; [0:a:0][s]amix", b"-map_metadata", b"-1", b"-ac", b"2", b"-c:a", b"libopus", b"-b:a", bq, b"-f", b"caf", - fsenc(tpath) + bap_out, ] # fmt: on self._run_ff(cmd, vn, "aconvt", oom=300) @@ -1359,18 +1374,19 @@ class ThumbSrv(object): else: # simple remux should be safe self.log("conv2 caf-remux; dur=%d sz=%d" % (dur, sz), 6) + bap_in = fsenc(tmp_opus) + bap_out = fsenc(tpath) # fmt: off - cmd = [ - HAVE_FFMPEG, + cmd = bwrap(HAVE_FFMPEG, bap_in, bap_out) + [ b"-nostdin", b"-v", b"error", b"-hide_banner", - b"-i", fsenc(tmp_opus), + b"-i", bap_in, b"-map_metadata", b"-1", b"-map", b"0:a:0", b"-c:a", b"copy", b"-f", b"caf", - fsenc(tpath) + bap_out, ] # fmt: on self._run_ff(cmd, vn, "aconvt", oom=300) diff --git a/copyparty/u2idx.py b/copyparty/u2idx.py index ca1014d1..67c9967a 100644 --- a/copyparty/u2idx.py +++ b/copyparty/u2idx.py @@ -22,6 +22,7 @@ from .util import ( quotep, s3dec, vjoin, + vjoins, ) if HAVE_SQLITE3: @@ -437,7 +438,7 @@ class U2idx(object): if rd.startswith("//") or fn.startswith("//"): rd, fn = s3dec(rd, fn) - vp = vjoin(vjoin(vtop, rd), fn) + vp = vjoins(vtop, rd, fn) if vp in seen_rps: continue diff --git a/copyparty/up2k.py b/copyparty/up2k.py index 957cbc72..84fc1ce2 100644 --- a/copyparty/up2k.py +++ b/copyparty/up2k.py @@ -70,6 +70,7 @@ from .util import ( ub64enc, unhumanize, vjoin, + vjoins, vsplit, w8b64dec, w8b64enc, @@ -3037,7 +3038,7 @@ class Up2k(object): raise Pebkac(500, "too many xbu relocs, giving up") ptop = cj["ptop"] - if not self.register_vpath(ptop, cj["vcfg"]): + if not self.register_vpath(ptop, cj.pop("vcfg")): if ptop not in self.registry: raise Pebkac(410, "location unavailable") @@ -3190,7 +3191,7 @@ class Up2k(object): c2 = None for cur, dp_dir, dp_fn in lost: t = "forgetting desynced db entry: %r" - self.log(t % ("/" + vjoin(vjoin(vfs.vpath, dp_dir), dp_fn))) + self.log(t % ("/" + vjoins(vfs.vpath, dp_dir, dp_fn))) self.db_rm(cur, vfs.flags, dp_dir, dp_fn, cj["size"]) if c2 and c2 != cur: c2.connection.commit() @@ -3280,7 +3281,10 @@ class Up2k(object): vfs.lim.nup(cj["addr"]) vfs.lim.bup(cj["addr"], cj["size"]) - if "done" not in job: + if "rvp0" in job and wark in reg: + # xbu reloc; accept wrong path + job["addr"] = cj["addr"] + elif "done" not in job: self.log("unfinished:\n %r\n %r" % (src, dst)) err = "partial upload exists at a different location; please resume uploading here instead:\n" err += "/" + quotep(vsrc) + " " @@ -3370,9 +3374,9 @@ class Up2k(object): x = pathmod(self.vfs, dst, vp, hr["reloc"]) if x: ud1 = (vfs.vpath, job["prel"], job["name"]) + job["rvp0"] = vjoins(*ud1) pdir, _, job["name"], (vfs, rem) = x dst = os.path.join(pdir, job["name"]) - job["vcfg"] = vfs.flags job["ptop"] = vfs.realpath job["vtop"] = vfs.vpath job["prel"] = rem @@ -3380,8 +3384,19 @@ class Up2k(object): ud2 = (vfs.vpath, job["prel"], job["name"]) if ud1 != ud2: # print(json.dumps(job, sort_keys=True, indent=4)) + job["vcfg"] = vfs.flags job["hash"] = cj["hash"] - self.log("xbu reloc1:%d..." % (depth,), 6) + t = "xbu reloc1=%d ptop=%r vtop=%r prel=%r name=%r" + t = t % ( + depth, + job["ptop"], + job["vtop"], + job["prel"], + job["name"], + ) + self.log(t, 6) + zs = djoin(job["ptop"], job["prel"]) + bos.makedirs(zs, vf=vfs.flags) return self._handle_json(job, depth + 1) job["name"] = self._untaken(pdir, job, now) @@ -5263,15 +5278,25 @@ class Up2k(object): x = pathmod(self.vfs, ap_chk, vp_chk, hr["reloc"]) if x: ud1 = (vfs.vpath, job["prel"], job["name"]) + job["rvp0"] = vjoins(*ud1) pdir, _, job["name"], (vfs, rem) = x - job["vcfg"] = vf = vfs.flags + vf = vfs.flags job["ptop"] = vfs.realpath job["vtop"] = vfs.vpath job["prel"] = rem job["name"] = sanitize_fn(job["name"]) ud2 = (vfs.vpath, job["prel"], job["name"]) if ud1 != ud2: - self.log("xbu reloc2:%d..." % (depth,), 6) + job["vcfg"] = vf + t = "xbu reloc2=%d ptop=%r vtop=%r prel=%r name=%r" % ( + depth, + job["ptop"], + job["vtop"], + job["prel"], + job["name"], + ) + self.log(t, 6) + bos.makedirs(djoin(job["ptop"], job["prel"]), vf=vf) return self._handle_json(job, depth + 1) job["name"] = self._untaken(pdir, job, job["t0"]) diff --git a/copyparty/util.py b/copyparty/util.py index 6396445f..62f3b253 100644 --- a/copyparty/util.py +++ b/copyparty/util.py @@ -326,7 +326,14 @@ except: BITNESS = struct.calcsize("P") * 8 -CAN_SIGMASK = not (ANYWIN or PY2 or GRAAL) +try: + if ANYWIN or PY2 or GRAAL or not hasattr(signal, "pthread_sigmask"): + raise Exception() + BLOCK_SIGS = [signal.SIGINT, signal.SIGTERM, signal.SIGHUP, signal.SIGUSR1] + CAN_SIGMASK = True +except: + BLOCK_SIGS = [] + CAN_SIGMASK = False RE_ANSI = re.compile("\033\\[[^mK]*[mK]") @@ -650,6 +657,14 @@ if EXE: pass +try: + if PY2 or ANYWIN: + raise Exception() + HAVE_BWRAP = shutil.which("bwrap") +except: + HAVE_BWRAP = "" + + def py_desc() -> str: interp = platform.python_implementation() py_ver = ".".join([str(x) for x in sys.version_info]) @@ -832,10 +847,8 @@ class Daemon(threading.Thread): self.start() def run(self): - if CAN_SIGMASK: - signal.pthread_sigmask( - signal.SIG_BLOCK, [signal.SIGINT, signal.SIGTERM, signal.SIGUSR1] - ) + if BLOCK_SIGS: + signal.pthread_sigmask(signal.SIG_BLOCK, BLOCK_SIGS) self.fun(*self.a, **self.ka) @@ -1778,9 +1791,7 @@ def log_thrs(log: Callable[[str, str, int], None], ival: float, name: str) -> No def _sigblock(): - signal.pthread_sigmask( - signal.SIG_BLOCK, [signal.SIGINT, signal.SIGTERM, signal.SIGUSR1] - ) + signal.pthread_sigmask(signal.SIG_BLOCK, BLOCK_SIGS) sigblock = _sigblock if CAN_SIGMASK else noop @@ -2676,6 +2687,10 @@ def vjoin(rd: str, fn: str) -> str: return rd or fn +def vjoins(*a: str) -> str: + return "/".join([x for x in a if x]) + + # url-join def ujoin(rd: str, fn: str) -> str: if rd and fn: diff --git a/copyparty/web/browser.js b/copyparty/web/browser.js index 1aa69e9d..9ec4d3e7 100644 --- a/copyparty/web/browser.js +++ b/copyparty/web/browser.js @@ -8999,6 +8999,13 @@ var msel = (function () { return; } + if (this.status == 405) { + tb.value = ''; + sf.textContent = 'already existed'; + treectl.goto(this.vp + uricom_enc(this.dn) + '/', true); + return tree_scrollto(); + } + xhrchk(this, L.fd_xe1, L.fd_xe2); if (this.status !== 201) { diff --git a/scripts/docker/Dockerfile.ac b/scripts/docker/Dockerfile.ac index d549627f..7ad7dca9 100644 --- a/scripts/docker/Dockerfile.ac +++ b/scripts/docker/Dockerfile.ac @@ -9,7 +9,7 @@ ENV XDG_CONFIG_HOME=/cfg ARG ADD_PKG="" RUN apk --no-cache add !pyc ${ADD_PKG} \ - tzdata wget mimalloc2 mimalloc2-insecure \ + tzdata wget mimalloc2 mimalloc2-insecure bubblewrap \ py3-jinja2 py3-argon2-cffi py3-pyzmq \ py3-openssl py3-paramiko py3-pillow diff --git a/scripts/docker/Dockerfile.dj b/scripts/docker/Dockerfile.dj index 3796722c..75d4d6c6 100644 --- a/scripts/docker/Dockerfile.dj +++ b/scripts/docker/Dockerfile.dj @@ -12,7 +12,7 @@ COPY i/bin/mtag/install-deps.sh ./ COPY i/bin/mtag/audio-bpm.py /mtag/ COPY i/bin/mtag/audio-key.py /mtag/ RUN apk add -U !pyc ${ADD_PKG} \ - tzdata wget mimalloc2 mimalloc2-insecure \ + tzdata wget mimalloc2 mimalloc2-insecure bubblewrap \ py3-jinja2 py3-argon2-cffi py3-pyzmq \ py3-openssl py3-paramiko py3-pillow \ py3-pip \ diff --git a/scripts/docker/Dockerfile.iv b/scripts/docker/Dockerfile.iv index 1b9e7c2a..1a8b8aa4 100644 --- a/scripts/docker/Dockerfile.iv +++ b/scripts/docker/Dockerfile.iv @@ -9,7 +9,7 @@ ENV XDG_CONFIG_HOME=/cfg ARG ADD_PKG="" RUN apk add -U !pyc ${ADD_PKG} \ - tzdata wget mimalloc2 mimalloc2-insecure \ + tzdata wget mimalloc2 mimalloc2-insecure bubblewrap \ py3-jinja2 py3-argon2-cffi py3-pyzmq \ py3-openssl py3-paramiko py3-pillow \ py3-pip \ diff --git a/scripts/docker/README.md b/scripts/docker/README.md index 74a12680..37f756e9 100644 --- a/scripts/docker/README.md +++ b/scripts/docker/README.md @@ -54,9 +54,14 @@ with image size after installation and when gzipped [`ac` is recommended](https://hub.docker.com/r/copyparty/ac) since the additional features available in `iv` and `dj` are rarely useful -most editions support `x86`, `x86_64`, `armhf`, `aarch64`, `ppc64le`, `s390x` -* `dj` doesn't run on `ppc64le`, `s390x`, `armhf` -* `iv` doesn't run on `ppc64le`, `s390x` +| this architecture | can run these editions | +| ------------------------------ | ----------------------------- | +| `x86` aka `i386` aka `386` | `min`, `im`, `ac`, `iv`, `dj` | +| `x86_64` aka `x64` aka `amd64` | `min`, `im`, `ac`, `iv`, `dj` | +| `AArch64` aka `arm64/v8` | `min`, `im`, `ac`, `iv`, `dj` | +| `arm32` aka `arm/v7` | `min`, `im`, `ac` | +| `ppc64le` aka `PowerPC` | `min`, `im`, `ac` | +| `s390x` aka `IBM mainframe` | `min`, `im`, `ac` | > NOTE: the following editions are unfinished experiments, and not published anywhere: djd djf djff dju diff --git a/scripts/docker/base/Makefile b/scripts/docker/base/Makefile index 6e8b1dce..a66e5cf4 100644 --- a/scripts/docker/base/Makefile +++ b/scripts/docker/base/Makefile @@ -8,7 +8,7 @@ all: ff: # legally comfy - /usr/bin/time ./build-no265.sh img + /usr/bin/time ./build-no265.sh pull img zlib: diff --git a/scripts/docker/base/arbeidspakke.sh b/scripts/docker/base/arbeidspakke.sh index 96d54930..112f0f71 100755 --- a/scripts/docker/base/arbeidspakke.sh +++ b/scripts/docker/base/arbeidspakke.sh @@ -1,9 +1,13 @@ #!/bin/ash set -e +AVER=3.24 + [ $1 = 1 ] && hub=1 uname -a +apk update +apk upgrade -al apk add alpine-sdk doas wget echo permit nopass root > /etc/doas.d/u.conf cp -pv /root/.abuild/*.pub /etc/apk/keys/ || abuild-keygen -ina @@ -13,7 +17,7 @@ cp -pv /root/.abuild/*.pub /etc/apk/keys/ || abuild-keygen -ina mkdir /ffmpeg cd /ffmpeg -base=https://github.com/alpinelinux/aports/raw/refs/heads/3.23-stable/community/ffmpeg/ +base=https://github.com/alpinelinux/aports/raw/refs/heads/$AVER-stable/community/ffmpeg/ wget ${base}APKBUILD awk t &1 | - tee b/log.$a + awk '{getline a<"/proc/uptime";close("/proc/uptime");sub(/ .*/,"",a);printf"%.2f %s\n",a-p,$0;p=a}' | + tee b/log.$n.$a - touch b/t.$a.2.$(date +%s) + touch b/t.$n.$a.2.$(date +%s) done wt -;wt "" } @@ -89,11 +92,23 @@ echo ok # 50m01.04 s390x # golflympics -# 4:09 x86_64-hub -# 2:57 x86_64 -# 2:54 x86 -# 31:13 aarch64 -# 22:38 armv7 -# 32:17 s390x -# 24:27 ppc64le -# 2:00:35 summa summarum +# 3:48 x86_64-hub +# 2:46 x86_64 +# 2:24 x86 +# 28:50 aarch64 +# 21:34 armv7 +# 31:13 s390x +# 22:50 ppc64le +# 1:53:25 summa summarum + +# for a in version muxers demuxers devices decoders encoders filters pix_fmts layouts sample_fmts ; do ffmpeg -hide_banner -$a; done | nc 192.168.123.1 4321 + +# v=3.24-stable +# echo -n https://dl-cdn.alpinelinux.org/v${v%-*}/releases/x86_64/ >aver +# curl -s $(cat aver)latest-releases.yaml | awk '/alpine-minirootfs-3.*gz$/{print$2;exit}' | grep ... >> aver +# podman import $(cat aver) a324 +# f(){ p=/sys/fs;for w in cgroup user.slice user-1000.slice user@1000.service user.slice ;do p="$p/$w";echo $1>"$p/cgroup.subtree_control";done;} +# f +cpuset +# time nice podman run --cpuset-cpus=1 \ +# grep -E '^[^0].*' -B2 -A1 log.1.amd64 # offbyone, whatever, just eyeball it +# f -cpuset diff --git a/scripts/docker/base/verchk.sh b/scripts/docker/base/verchk.sh index ab05830e..ef25a9f7 100755 --- a/scripts/docker/base/verchk.sh +++ b/scripts/docker/base/verchk.sh @@ -1,16 +1,19 @@ #!/bin/bash set -e -v=3.23 +AVER=3.24 + +v=$AVER-stable +#v=master mkdir -p cver rm -rf cver2 mkdir cver2 cd cver2 curl \ - -Lo1 https://raw.githubusercontent.com/alpinelinux/aports/refs/heads/$v-stable/main/musl/APKBUILD \ - -Lo2 https://raw.githubusercontent.com/alpinelinux/aports/refs/heads/$v-stable/main/python3/APKBUILD \ - -Lo3 https://raw.githubusercontent.com/alpinelinux/aports/refs/heads/$v-stable/community/ffmpeg/APKBUILD \ + -Lo1 https://raw.githubusercontent.com/alpinelinux/aports/refs/heads/$v/main/musl/APKBUILD \ + -Lo2 https://raw.githubusercontent.com/alpinelinux/aports/refs/heads/$v/main/python3/APKBUILD \ + -Lo3 https://raw.githubusercontent.com/alpinelinux/aports/refs/heads/$v/community/ffmpeg/APKBUILD \ ; zlib= ff=