From 57a56073d80fd9e8b93901910eeb8805c7787149 Mon Sep 17 00:00:00 2001 From: ed Date: Sun, 23 Mar 2025 20:15:21 +0000 Subject: [PATCH] use zlib-ng when available download-as-tar-gz becomes 2.4x faster in docker segfaults on windows, so don't use it there does not affect fedora or gentoo, since zlib-ng is already system-default on those also adds a global-option to write list of successful binds to a textfile, for automation / smoketest purposes --- bin/mtag/cksum.py | 6 ++++- copyparty/__main__.py | 2 ++ copyparty/httpcli.py | 2 +- copyparty/mtag.py | 3 +-- copyparty/svchub.py | 2 +- copyparty/szip.py | 3 +-- copyparty/tcpsrv.py | 18 ++++++++++++++ copyparty/up2k.py | 2 +- copyparty/util.py | 28 +++++++++++++++++++--- scripts/docker/Dockerfile.ac | 3 ++- scripts/docker/Dockerfile.dj | 3 ++- scripts/docker/Dockerfile.im | 3 ++- scripts/docker/Dockerfile.iv | 3 ++- scripts/docker/Dockerfile.min | 2 +- scripts/docker/base/Dockerfile.zlibng | 5 ++++ scripts/docker/base/Makefile | 15 ++++++++++++ scripts/docker/innvikler.sh | 34 ++++++++++++++++++++++++++- 17 files changed, 117 insertions(+), 17 deletions(-) create mode 100644 scripts/docker/base/Dockerfile.zlibng create mode 100644 scripts/docker/base/Makefile diff --git a/bin/mtag/cksum.py b/bin/mtag/cksum.py index 94b22f74..41106c66 100755 --- a/bin/mtag/cksum.py +++ b/bin/mtag/cksum.py @@ -2,11 +2,15 @@ import sys import json -import zlib import struct import base64 import hashlib +try: + from zlib_ng import zlib_ng as zlib +except: + import zlib + try: from copyparty.util import fsenc except: diff --git a/copyparty/__main__.py b/copyparty/__main__.py index 790543ef..d664fabd 100644 --- a/copyparty/__main__.py +++ b/copyparty/__main__.py @@ -1029,6 +1029,8 @@ def add_network(ap): ap2.add_argument("--reuseaddr", action="store_true", help="set reuseaddr on listening sockets on windows; allows rapid restart of copyparty at the expense of being able to accidentally start multiple instances") else: ap2.add_argument("--freebind", action="store_true", help="allow listening on IPs which do not yet exist, for example if the network interfaces haven't finished going up. Only makes sense for IPs other than '0.0.0.0', '127.0.0.1', '::', and '::1'. May require running as root (unless net.ipv6.ip_nonlocal_bind)") + ap2.add_argument("--wr-h-eps", metavar="PATH", type=u, default="", help="write list of listening-on ip:port to textfile at \033[33mPATH\033[0m when http-servers have started") + ap2.add_argument("--wr-h-aon", metavar="PATH", type=u, default="", help="write list of accessible-on ip:port to textfile at \033[33mPATH\033[0m when http-servers have started") ap2.add_argument("--s-thead", metavar="SEC", type=int, default=120, help="socket timeout (read request header)") ap2.add_argument("--s-tbody", metavar="SEC", type=float, default=128.0, help="socket timeout (read/write request/response bodies). Use 60 on fast servers (default is extremely safe). Disable with 0 if reverse-proxied for a 2%% speed boost") ap2.add_argument("--s-rd-sz", metavar="B", type=int, default=256*1024, help="socket read size in bytes (indirectly affects filesystem writes; recommendation: keep equal-to or lower-than \033[33m--iobuf\033[0m)") diff --git a/copyparty/httpcli.py b/copyparty/httpcli.py index e9b0250f..3c12595f 100644 --- a/copyparty/httpcli.py +++ b/copyparty/httpcli.py @@ -4,7 +4,6 @@ from __future__ import print_function, unicode_literals import argparse # typechk import copy import errno -import gzip import hashlib import itertools import json @@ -70,6 +69,7 @@ from .util import ( get_df, get_spd, guess_mime, + gzip, gzip_file_orig_sz, gzip_orig_sz, has_resource, diff --git a/copyparty/mtag.py b/copyparty/mtag.py index 583e9328..a972ef33 100644 --- a/copyparty/mtag.py +++ b/copyparty/mtag.py @@ -18,6 +18,7 @@ from .util import ( REKOBO_LKEY, VF_CAREFUL, fsenc, + gzip, min_ex, pybin, retchk, @@ -138,8 +139,6 @@ def au_unpk( fd, ret = tempfile.mkstemp("." + au) if pk == "gz": - import gzip - fi = gzip.GzipFile(abspath, mode="rb") elif pk == "xz": diff --git a/copyparty/svchub.py b/copyparty/svchub.py index 7213a7d2..cd55e735 100644 --- a/copyparty/svchub.py +++ b/copyparty/svchub.py @@ -3,7 +3,6 @@ from __future__ import print_function, unicode_literals import argparse import errno -import gzip import logging import os import re @@ -63,6 +62,7 @@ from .util import ( ansi_re, build_netmap, expat_ver, + gzip, load_ipu, min_ex, mp, diff --git a/copyparty/szip.py b/copyparty/szip.py index 348d8caf..8e0ffee2 100644 --- a/copyparty/szip.py +++ b/copyparty/szip.py @@ -4,12 +4,11 @@ from __future__ import print_function, unicode_literals import calendar import stat import time -import zlib from .authsrv import AuthSrv from .bos import bos from .sutil import StreamArc, errdesc -from .util import min_ex, sanitize_fn, spack, sunpack, yieldfile +from .util import min_ex, sanitize_fn, spack, sunpack, yieldfile, zlib if True: # pylint: disable=using-constant-test from typing import Any, Generator, Optional diff --git a/copyparty/tcpsrv.py b/copyparty/tcpsrv.py index c56ea837..99f178c5 100644 --- a/copyparty/tcpsrv.py +++ b/copyparty/tcpsrv.py @@ -151,9 +151,15 @@ class TcpSrv(object): if just_ll or self.args.ll: ll_ok.add(ip.split("/")[0]) + listening_on = [] + for ip, ports in sorted(ok.items()): + for port in sorted(ports): + listening_on.append("%s %s" % (ip, port)) + qr1: dict[str, list[int]] = {} qr2: dict[str, list[int]] = {} msgs = [] + accessible_on = [] title_tab: dict[str, dict[str, int]] = {} title_vars = [x[1:] for x in self.args.wintitle.split(" ") if x.startswith("$")] t = "available @ {}://{}:{}/ (\033[33m{}\033[0m)" @@ -169,6 +175,10 @@ class TcpSrv(object): ): continue + zs = "%s %s" % (ip, port) + if zs not in accessible_on: + accessible_on.append(zs) + proto = " http" if self.args.http_only: pass @@ -219,6 +229,14 @@ class TcpSrv(object): else: print("\n", end="") + for fn, ls in ( + (self.args.wr_h_eps, listening_on), + (self.args.wr_h_aon, accessible_on), + ): + if fn: + with open(fn, "wb") as f: + f.write(("\n".join(ls)).encode("utf-8")) + if self.args.qr or self.args.qrs: self.qr = self._qr(qr1, qr2) diff --git a/copyparty/up2k.py b/copyparty/up2k.py index e3613926..1f815343 100644 --- a/copyparty/up2k.py +++ b/copyparty/up2k.py @@ -2,7 +2,6 @@ from __future__ import print_function, unicode_literals import errno -import gzip import hashlib import json import math @@ -42,6 +41,7 @@ from .util import ( fsenc, gen_filekey, gen_filekey_dbg, + gzip, hidedir, humansize, min_ex, diff --git a/copyparty/util.py b/copyparty/util.py index 6f773105..cf7be3f3 100644 --- a/copyparty/util.py +++ b/copyparty/util.py @@ -31,6 +31,17 @@ from collections import Counter from ipaddress import IPv4Address, IPv4Network, IPv6Address, IPv6Network from queue import Queue +try: + from zlib_ng import gzip_ng as gzip + from zlib_ng import zlib_ng as zlib + + sys.modules["gzip"] = gzip + # sys.modules["zlib"] = zlib + # `- somehow makes tarfile 3% slower with default malloc, and barely faster with mimalloc +except: + import gzip + import zlib + from .__init__ import ( ANYWIN, EXE, @@ -1453,8 +1464,6 @@ def stackmon(fp: str, ival: float, suffix: str) -> None: buf = st.encode("utf-8", "replace") if fp.endswith(".gz"): - import gzip - # 2459b 2304b 2241b 2202b 2194b 2191b lv3..8 # 0.06s 0.08s 0.11s 0.13s 0.16s 0.19s buf = gzip.compress(buf, compresslevel=6) @@ -4055,9 +4064,22 @@ class WrongPostKey(Pebkac): self.datagen = datagen -_: Any = (mp, BytesIO, quote, unquote, SQLITE_VER, JINJA_VER, PYFTPD_VER, PARTFTPY_VER) +_: Any = ( + gzip, + mp, + zlib, + BytesIO, + quote, + unquote, + SQLITE_VER, + JINJA_VER, + PYFTPD_VER, + PARTFTPY_VER, +) __all__ = [ + "gzip", "mp", + "zlib", "BytesIO", "quote", "unquote", diff --git a/scripts/docker/Dockerfile.ac b/scripts/docker/Dockerfile.ac index e27bbd9f..b5df6f8b 100644 --- a/scripts/docker/Dockerfile.ac +++ b/scripts/docker/Dockerfile.ac @@ -13,7 +13,8 @@ RUN apk --no-cache add !pyc \ ffmpeg COPY i/dist/copyparty-sfx.py innvikler.sh ./ -RUN ash innvikler.sh && rm innvikler.sh +ADD base ./base +RUN ash innvikler.sh ac WORKDIR /w EXPOSE 3923 diff --git a/scripts/docker/Dockerfile.dj b/scripts/docker/Dockerfile.dj index 09e603b0..5b7fc355 100644 --- a/scripts/docker/Dockerfile.dj +++ b/scripts/docker/Dockerfile.dj @@ -31,7 +31,8 @@ RUN apk add -U !pyc \ && ln -s /root/vamp /root/.local / COPY i/dist/copyparty-sfx.py innvikler.sh ./ -RUN ash innvikler.sh && rm innvikler.sh +ADD base ./base +RUN ash innvikler.sh dj WORKDIR /w EXPOSE 3923 diff --git a/scripts/docker/Dockerfile.im b/scripts/docker/Dockerfile.im index 276865bf..a24715ca 100644 --- a/scripts/docker/Dockerfile.im +++ b/scripts/docker/Dockerfile.im @@ -12,7 +12,8 @@ RUN apk --no-cache add !pyc \ py3-jinja2 py3-argon2-cffi py3-pillow py3-mutagen COPY i/dist/copyparty-sfx.py innvikler.sh ./ -RUN ash innvikler.sh && rm innvikler.sh +ADD base ./base +RUN ash innvikler.sh im WORKDIR /w EXPOSE 3923 diff --git a/scripts/docker/Dockerfile.iv b/scripts/docker/Dockerfile.iv index 1accef7c..48264373 100644 --- a/scripts/docker/Dockerfile.iv +++ b/scripts/docker/Dockerfile.iv @@ -21,7 +21,8 @@ RUN apk add -U !pyc \ && apk del py3-pip .bd COPY i/dist/copyparty-sfx.py innvikler.sh ./ -RUN ash innvikler.sh && rm innvikler.sh +ADD base ./base +RUN ash innvikler.sh iv WORKDIR /w EXPOSE 3923 diff --git a/scripts/docker/Dockerfile.min b/scripts/docker/Dockerfile.min index a7f492a3..4712a8ad 100644 --- a/scripts/docker/Dockerfile.min +++ b/scripts/docker/Dockerfile.min @@ -11,7 +11,7 @@ RUN apk --no-cache add !pyc \ py3-jinja2 COPY i/dist/copyparty-sfx.py innvikler.sh ./ -RUN ash innvikler.sh && rm innvikler.sh +RUN ash innvikler.sh min WORKDIR /w EXPOSE 3923 diff --git a/scripts/docker/base/Dockerfile.zlibng b/scripts/docker/base/Dockerfile.zlibng new file mode 100644 index 00000000..da270d0c --- /dev/null +++ b/scripts/docker/base/Dockerfile.zlibng @@ -0,0 +1,5 @@ +FROM alpine:latest +WORKDIR /z + +RUN apk add py3-pip make gcc musl-dev python3-dev +RUN pip wheel https://files.pythonhosted.org/packages/c4/a7/0b7673be5945071e99364a3ac1987b02fc1d416617e97f3e8816d275174e/zlib_ng-0.5.1.tar.gz diff --git a/scripts/docker/base/Makefile b/scripts/docker/base/Makefile new file mode 100644 index 00000000..78eacd28 --- /dev/null +++ b/scripts/docker/base/Makefile @@ -0,0 +1,15 @@ +self := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) + +all: +# build zlib-ng from source so we know how the sausage was made +# (still only doing the archs which are officially supported/tested) + + podman build --arch amd64 -t localhost/cpp-zlibng-amd64:latest -f Dockerfile.zlibng . + podman run --arch amd64 --rm --log-driver=none -i localhost/cpp-zlibng-amd64:latest tar -cC/z . | tar -xv + + podman build --arch arm64 -t localhost/cpp-zlibng-amd64:latest -f Dockerfile.zlibng . + podman run --arch arm64 --rm --log-driver=none -i localhost/cpp-zlibng-amd64:latest tar -cC/z . | tar -xv + +sh: + @printf "\n\033[1;31mopening a shell in the most recently created docker image\033[0m\n" + docker run --rm -it --entrypoint /bin/ash `docker images -aq | head -n 1` diff --git a/scripts/docker/innvikler.sh b/scripts/docker/innvikler.sh index d16561c5..8cceda0f 100644 --- a/scripts/docker/innvikler.sh +++ b/scripts/docker/innvikler.sh @@ -1,6 +1,16 @@ #!/bin/ash set -ex +# use zlib-ng if available +f=/z/base/zlib_ng-0.5.1-cp312-cp312-linux_$(uname -m).whl +[ "$1" != min ] && [ -e $f ] && { + apk add -t .bd !pyc py3-pip + rm -f /usr/lib/python3*/EXTERNALLY-MANAGED + pip install $f + apk del .bd +} +rm -rf /z/base + # cleanup for flavors with python build steps (dj/iv) rm -rf /var/cache/apk/* /root/.cache @@ -40,7 +50,29 @@ find -name __pycache__ | cd /z python3 -m copyparty \ --ign-ebind -p$((1024+RANDOM)),$((1024+RANDOM)),$((1024+RANDOM)) \ - --no-crt -qi127.1 --exit=idx -e2dsa -e2ts + -v .::r --no-crt -qi127.1 --exit=idx -e2dsa -e2ts + +######################################################################## +# test download-as-tar.gz + +t=$(mktemp) +python3 -m copyparty \ + --ign-ebind -p$((1024+RANDOM)),$((1024+RANDOM)),$((1024+RANDOM)) \ + -v .::r --no-crt -qi127.1 --wr-h-eps $t & pid=$! + +for n in $(seq 1 200); do sleep 0.2 + v=$(awk '/^127/{print;n=1;exit}END{exit n-1}' $t) && break +done +[ -z "$v" ] && echo SNAAAAAKE && exit 1 + +wget -O- http://${v/ /:}/?tar=gz:1 | tar -xzO top/innvikler.sh | cmp innvikler.sh + +kill $pid; wait $pid + +######################################################################## # output from -e2d rm -rf .hist + +# goodbye +exec rm innvikler.sh