use zlib-ng when available

download-as-tar-gz becomes 2.4x faster in docker segfaults on windows, so don't use it there does not affect fedora or gentoo, since zlib-ng is already system-default on those also adds a global-option to write list of successful binds to a textfile, for automation / smoketest purposes
2025-08-16 00:22:13 -06:00 · 2025-03-23 20:15:21 +00:00 · 2025-03-23 20:15:21 +00:00 · 57a56073d8
parent 2525d594c5
commit 57a56073d8
17 changed files with 117 additions and 17 deletions
--- a/bin/mtag/cksum.py
+++ b/bin/mtag/cksum.py
@ -2,11 +2,15 @@

 import sys
 import json
-import zlib
 import struct
 import base64
 import hashlib

+try:
+    from zlib_ng import zlib_ng as zlib
+except:
+    import zlib
+
 try:
    from copyparty.util import fsenc
 except:
--- a/copyparty/main.py
+++ b/copyparty/main.py
@ -1029,6 +1029,8 @@ def add_network(ap):
        ap2.add_argument("--reuseaddr", action="store_true", help="set reuseaddr on listening sockets on windows; allows rapid restart of copyparty at the expense of being able to accidentally start multiple instances")
    else:
        ap2.add_argument("--freebind", action="store_true", help="allow listening on IPs which do not yet exist, for example if the network interfaces haven't finished going up. Only makes sense for IPs other than '0.0.0.0', '127.0.0.1', '::', and '::1'. May require running as root (unless net.ipv6.ip_nonlocal_bind)")
+    ap2.add_argument("--wr-h-eps", metavar="PATH", type=u, default="", help="write list of listening-on ip:port to textfile at \033[33mPATH\033[0m when http-servers have started")
+    ap2.add_argument("--wr-h-aon", metavar="PATH", type=u, default="", help="write list of accessible-on ip:port to textfile at \033[33mPATH\033[0m when http-servers have started")
    ap2.add_argument("--s-thead", metavar="SEC", type=int, default=120, help="socket timeout (read request header)")
    ap2.add_argument("--s-tbody", metavar="SEC", type=float, default=128.0, help="socket timeout (read/write request/response bodies). Use 60 on fast servers (default is extremely safe). Disable with 0 if reverse-proxied for a 2%% speed boost")
    ap2.add_argument("--s-rd-sz", metavar="B", type=int, default=256*1024, help="socket read size in bytes (indirectly affects filesystem writes; recommendation: keep equal-to or lower-than \033[33m--iobuf\033[0m)")
--- a/copyparty/httpcli.py
+++ b/copyparty/httpcli.py
@ -4,7 +4,6 @@ from __future__ import print_function, unicode_literals
 import argparse  # typechk
 import copy
 import errno
-import gzip
 import hashlib
 import itertools
 import json
@ -70,6 +69,7 @@ from .util import (
    get_df,
    get_spd,
    guess_mime,
+    gzip,
    gzip_file_orig_sz,
    gzip_orig_sz,
    has_resource,
--- a/copyparty/mtag.py
+++ b/copyparty/mtag.py
@ -18,6 +18,7 @@ from .util import (
    REKOBO_LKEY,
    VF_CAREFUL,
    fsenc,
+    gzip,
    min_ex,
    pybin,
    retchk,
@ -138,8 +139,6 @@ def au_unpk(
        fd, ret = tempfile.mkstemp("." + au)

        if pk == "gz":
-            import gzip
-
            fi = gzip.GzipFile(abspath, mode="rb")

        elif pk == "xz":
--- a/copyparty/svchub.py
+++ b/copyparty/svchub.py
@ -3,7 +3,6 @@ from __future__ import print_function, unicode_literals

 import argparse
 import errno
-import gzip
 import logging
 import os
 import re
@ -63,6 +62,7 @@ from .util import (
    ansi_re,
    build_netmap,
    expat_ver,
+    gzip,
    load_ipu,
    min_ex,
    mp,
--- a/copyparty/szip.py
+++ b/copyparty/szip.py
@ -4,12 +4,11 @@ from __future__ import print_function, unicode_literals
 import calendar
 import stat
 import time
-import zlib

 from .authsrv import AuthSrv
 from .bos import bos
 from .sutil import StreamArc, errdesc
-from .util import min_ex, sanitize_fn, spack, sunpack, yieldfile
+from .util import min_ex, sanitize_fn, spack, sunpack, yieldfile, zlib

 if True:  # pylint: disable=using-constant-test
    from typing import Any, Generator, Optional
--- a/copyparty/tcpsrv.py
+++ b/copyparty/tcpsrv.py
@ -151,9 +151,15 @@ class TcpSrv(object):
            if just_ll or self.args.ll:
                ll_ok.add(ip.split("/")[0])

+        listening_on = []
+        for ip, ports in sorted(ok.items()):
+            for port in sorted(ports):
+                listening_on.append("%s %s" % (ip, port))
+
        qr1: dict[str, list[int]] = {}
        qr2: dict[str, list[int]] = {}
        msgs = []
+        accessible_on = []
        title_tab: dict[str, dict[str, int]] = {}
        title_vars = [x[1:] for x in self.args.wintitle.split(" ") if x.startswith("$")]
        t = "available @ {}://{}:{}/  (\033[33m{}\033[0m)"
@ -169,6 +175,10 @@ class TcpSrv(object):
                ):
                    continue

+                zs = "%s %s" % (ip, port)
+                if zs not in accessible_on:
+                    accessible_on.append(zs)
+
                proto = " http"
                if self.args.http_only:
                    pass
@ -219,6 +229,14 @@ class TcpSrv(object):
        else:
            print("\n", end="")

+        for fn, ls in (
+            (self.args.wr_h_eps, listening_on),
+            (self.args.wr_h_aon, accessible_on),
+        ):
+            if fn:
+                with open(fn, "wb") as f:
+                    f.write(("\n".join(ls)).encode("utf-8"))
+
        if self.args.qr or self.args.qrs:
            self.qr = self._qr(qr1, qr2)

--- a/copyparty/up2k.py
+++ b/copyparty/up2k.py
@ -2,7 +2,6 @@
 from __future__ import print_function, unicode_literals

 import errno
-import gzip
 import hashlib
 import json
 import math
@ -42,6 +41,7 @@ from .util import (
    fsenc,
    gen_filekey,
    gen_filekey_dbg,
+    gzip,
    hidedir,
    humansize,
    min_ex,
--- a/copyparty/util.py
+++ b/copyparty/util.py
@ -31,6 +31,17 @@ from collections import Counter
 from ipaddress import IPv4Address, IPv4Network, IPv6Address, IPv6Network
 from queue import Queue

+try:
+    from zlib_ng import gzip_ng as gzip
+    from zlib_ng import zlib_ng as zlib
+
+    sys.modules["gzip"] = gzip
+    # sys.modules["zlib"] = zlib
+    # `- somehow makes tarfile 3% slower with default malloc, and barely faster with mimalloc
+except:
+    import gzip
+    import zlib
+
 from .__init__ import (
    ANYWIN,
    EXE,
@ -1453,8 +1464,6 @@ def stackmon(fp: str, ival: float, suffix: str) -> None:
        buf = st.encode("utf-8", "replace")

        if fp.endswith(".gz"):
-            import gzip
-
            # 2459b 2304b 2241b 2202b 2194b 2191b lv3..8
            # 0.06s 0.08s 0.11s 0.13s 0.16s 0.19s
            buf = gzip.compress(buf, compresslevel=6)
@ -4055,9 +4064,22 @@ class WrongPostKey(Pebkac):
        self.datagen = datagen


-_: Any = (mp, BytesIO, quote, unquote, SQLITE_VER, JINJA_VER, PYFTPD_VER, PARTFTPY_VER)
+_: Any = (
+    gzip,
+    mp,
+    zlib,
+    BytesIO,
+    quote,
+    unquote,
+    SQLITE_VER,
+    JINJA_VER,
+    PYFTPD_VER,
+    PARTFTPY_VER,
+)
 __all__ = [
+    "gzip",
    "mp",
+    "zlib",
    "BytesIO",
    "quote",
    "unquote",
--- a/scripts/docker/Dockerfile.ac
+++ b/scripts/docker/Dockerfile.ac
@ -13,7 +13,8 @@ RUN     apk --no-cache add !pyc \
            ffmpeg

 COPY    i/dist/copyparty-sfx.py innvikler.sh ./
-RUN     ash innvikler.sh && rm innvikler.sh
+ADD     base ./base
+RUN     ash innvikler.sh ac

 WORKDIR /w
 EXPOSE  3923
--- a/scripts/docker/Dockerfile.dj
+++ b/scripts/docker/Dockerfile.dj
@ -31,7 +31,8 @@ RUN     apk add -U !pyc \
        && ln -s /root/vamp /root/.local /

 COPY    i/dist/copyparty-sfx.py innvikler.sh ./
-RUN     ash innvikler.sh && rm innvikler.sh
+ADD     base ./base
+RUN     ash innvikler.sh dj

 WORKDIR /w
 EXPOSE  3923
--- a/scripts/docker/Dockerfile.im
+++ b/scripts/docker/Dockerfile.im
@ -12,7 +12,8 @@ RUN     apk --no-cache add !pyc \
            py3-jinja2 py3-argon2-cffi py3-pillow py3-mutagen

 COPY    i/dist/copyparty-sfx.py innvikler.sh ./
-RUN     ash innvikler.sh && rm innvikler.sh
+ADD     base ./base
+RUN     ash innvikler.sh im

 WORKDIR /w
 EXPOSE  3923
--- a/scripts/docker/Dockerfile.iv
+++ b/scripts/docker/Dockerfile.iv
@ -21,7 +21,8 @@ RUN     apk add -U !pyc \
        && apk del py3-pip .bd

 COPY    i/dist/copyparty-sfx.py innvikler.sh ./
-RUN     ash innvikler.sh && rm innvikler.sh
+ADD     base ./base
+RUN     ash innvikler.sh iv

 WORKDIR /w
 EXPOSE  3923
--- a/scripts/docker/Dockerfile.min
+++ b/scripts/docker/Dockerfile.min
@ -11,7 +11,7 @@ RUN     apk --no-cache add !pyc \
            py3-jinja2

 COPY    i/dist/copyparty-sfx.py innvikler.sh ./
-RUN     ash innvikler.sh && rm innvikler.sh
+RUN     ash innvikler.sh min

 WORKDIR /w
 EXPOSE  3923
--- a/scripts/docker/base/Dockerfile.zlibng
+++ b/scripts/docker/base/Dockerfile.zlibng
@ -0,0 +1,5 @@
+FROM    alpine:latest
+WORKDIR /z
+
+RUN     apk add py3-pip make gcc musl-dev python3-dev
+RUN     pip wheel https://files.pythonhosted.org/packages/c4/a7/0b7673be5945071e99364a3ac1987b02fc1d416617e97f3e8816d275174e/zlib_ng-0.5.1.tar.gz
--- a/scripts/docker/base/Makefile
+++ b/scripts/docker/base/Makefile
@ -0,0 +1,15 @@
+self := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
+
+all:
+# 	build zlib-ng from source so we know how the sausage was made
+#	(still only doing the archs which are officially supported/tested)
+
+	podman build --arch amd64 -t localhost/cpp-zlibng-amd64:latest -f Dockerfile.zlibng .
+	podman run --arch amd64 --rm --log-driver=none -i localhost/cpp-zlibng-amd64:latest tar -cC/z . | tar -xv
+
+	podman build --arch arm64 -t localhost/cpp-zlibng-amd64:latest -f Dockerfile.zlibng .
+	podman run --arch arm64 --rm --log-driver=none -i localhost/cpp-zlibng-amd64:latest tar -cC/z . | tar -xv
+
+sh:
+	@printf "\n\033[1;31mopening a shell in the most recently created docker image\033[0m\n"
+	docker run --rm -it --entrypoint /bin/ash `docker images -aq | head -n 1`
--- a/scripts/docker/innvikler.sh
+++ b/scripts/docker/innvikler.sh
@ -1,6 +1,16 @@
 #!/bin/ash
 set -ex

+# use zlib-ng if available
+f=/z/base/zlib_ng-0.5.1-cp312-cp312-linux_$(uname -m).whl
+[ "$1" != min ] && [ -e $f ] && {
+  apk add -t .bd !pyc py3-pip
+  rm -f /usr/lib/python3*/EXTERNALLY-MANAGED
+  pip install $f
+  apk del .bd
+}
+rm -rf /z/base
+
 # cleanup for flavors with python build steps (dj/iv)
 rm -rf /var/cache/apk/* /root/.cache

@ -40,7 +50,29 @@ find -name __pycache__ |
 cd /z
 python3 -m copyparty \
  --ign-ebind -p$((1024+RANDOM)),$((1024+RANDOM)),$((1024+RANDOM)) \
-  --no-crt -qi127.1 --exit=idx -e2dsa -e2ts
+  -v .::r --no-crt -qi127.1 --exit=idx -e2dsa -e2ts
+
+########################################################################
+# test download-as-tar.gz
+
+t=$(mktemp)
+python3 -m copyparty \
+  --ign-ebind -p$((1024+RANDOM)),$((1024+RANDOM)),$((1024+RANDOM)) \
+  -v .::r --no-crt -qi127.1 --wr-h-eps $t & pid=$!
+
+for n in $(seq 1 200); do sleep 0.2
+  v=$(awk '/^127/{print;n=1;exit}END{exit n-1}' $t) && break
+done
+[ -z "$v" ] && echo SNAAAAAKE && exit 1
+
+wget -O- http://${v/ /:}/?tar=gz:1 | tar -xzO top/innvikler.sh | cmp innvikler.sh
+
+kill $pid; wait $pid
+
+########################################################################

 # output from -e2d
 rm -rf .hist
+
+# goodbye
+exec rm innvikler.sh