docker: use less RAM at runtime

compile to bytecode so cpython doesn't have to keep it in memory

ram usage reduced by:
* min: 5.4 MiB (32.6 to 27.2)
* ac/im: 5.2 MiB (39.0 to 33.8)
* dj/iv: 10.6 MiB (67.3 to 56.7)

startup time reduced from:
* min: 1.3s to 0.6s
* ac/im: 1.6s to 0.9s
* dj/iv: 2.0s to 1.1s

image size increased by 4 MiB (min), 6 MiB (ac/im/iv), 9 MiB (dj)

ram usage measured on idle with:
while true; do ps aux | grep -E 'R[S]S|no[-]crt'; read -n1; echo; done

startup time measured with:
time podman run --rm -it localhost/copyparty-min-amd64 --exit=idx

copyparty/__main__.py

@@ -497,6 +497,9 @@ def disable_quickedit() -> None:
 
 
 def sfx_tpoke(top: str):
+    if os.environ.get("PRTY_NO_TPOKE"):
+        return
+
     files = [top] + [
         os.path.join(dp, p) for dp, dd, df in os.walk(top) for p in dd + df
     ]

copyparty/fsutil.py

@@ -9,7 +9,7 @@ import time
 from .__init__ import ANYWIN, MACOS
 from .authsrv import AXS, VFS
 from .bos import bos
-from .util import chkcmd, min_ex
+from .util import chkcmd, min_ex, undot
 
 if True:  # pylint: disable=using-constant-test
     from typing import Optional, Union

copyparty/sutil.py

@@ -6,7 +6,7 @@ import tempfile
 from datetime import datetime
 
 from .__init__ import CORES
-from .authsrv import AuthSrv, VFS
+from .authsrv import VFS, AuthSrv
 from .bos import bos
 from .th_cli import ThumbCli
 from .util import UTC, vjoin, vol_san

copyparty/up2k.py

@@ -28,8 +28,8 @@ from .fsutil import Fstab
 from .mtag import MParser, MTag
 from .util import (
     HAVE_SQLITE3,
-    VF_CAREFUL,
     SYMTIME,
+    VF_CAREFUL,
     Daemon,
     MTHash,
     Pebkac,

scripts/docker/Dockerfile.ac

@@ -5,19 +5,16 @@ LABEL   org.opencontainers.image.url="https://github.com/9001/copyparty" \
         org.opencontainers.image.licenses="MIT" \
         org.opencontainers.image.title="copyparty-ac" \
         org.opencontainers.image.description="copyparty with Pillow and FFmpeg (image/audio/video thumbnails, audio transcoding, media tags)"
-ENV     PYTHONPYCACHEPREFIX=/tmp/pyc \
+ENV     XDG_CONFIG_HOME=/cfg
-        XDG_CONFIG_HOME=/cfg
 
 RUN     apk --no-cache add !pyc \
             wget \
-            py3-argon2-cffi py3-pillow \
+            py3-jinja2 py3-argon2-cffi py3-pillow \
-            ffmpeg \
+            ffmpeg
-        && rm -rf /tmp/pyc \
+
-        && mkdir /cfg /w \
+COPY    i/dist/copyparty-sfx.py innvikler.sh ./
-        && chmod 777 /cfg /w \
+RUN     ash innvikler.sh && rm innvikler.sh
-        && echo % /cfg > initcfg
 
-COPY    i/dist/copyparty-sfx.py ./
 WORKDIR /w
 EXPOSE  3923
-ENTRYPOINT ["python3", "/z/copyparty-sfx.py", "--no-crt", "-c", "/z/initcfg"]
+ENTRYPOINT ["python3", "-m", "copyparty", "--no-crt", "-c", "/z/initcfg"]

scripts/docker/Dockerfile.dj

@@ -5,15 +5,14 @@ LABEL   org.opencontainers.image.url="https://github.com/9001/copyparty" \
         org.opencontainers.image.licenses="MIT" \
         org.opencontainers.image.title="copyparty-dj" \
         org.opencontainers.image.description="copyparty with all optional dependencies, including musical key / bpm detection"
-ENV     PYTHONPYCACHEPREFIX=/tmp/pyc \
+ENV     XDG_CONFIG_HOME=/cfg
-        XDG_CONFIG_HOME=/cfg
 
 COPY    i/bin/mtag/install-deps.sh ./
 COPY    i/bin/mtag/audio-bpm.py /mtag/
 COPY    i/bin/mtag/audio-key.py /mtag/
 RUN     apk add -U !pyc \
             wget \
-            py3-argon2-cffi py3-pillow py3-pip py3-cffi \
+            py3-jinja2 py3-argon2-cffi py3-pillow py3-pip py3-cffi \
             ffmpeg \
             vips-jxl vips-heif vips-poppler vips-magick \
             py3-numpy fftw libsndfile \
@@ -27,18 +26,12 @@ RUN     apk add -U !pyc \
         && python3 -m pip install pyvips \
         && bash install-deps.sh \
         && apk del py3-pip .bd \
-        && rm -rf /var/cache/apk/* /tmp/pyc \
         && chmod 777 /root \
-        && ln -s /root/vamp /root/.local / \
+        && ln -s /root/vamp /root/.local /
-        && mkdir /cfg /w \
+
-        && chmod 777 /cfg /w \
+COPY    i/dist/copyparty-sfx.py innvikler.sh ./
-        && echo % /cfg > initcfg
+RUN     ash innvikler.sh && rm innvikler.sh
 
-COPY    i/dist/copyparty-sfx.py ./
 WORKDIR /w
 EXPOSE  3923
-ENTRYPOINT ["python3", "/z/copyparty-sfx.py", "--no-crt", "-c", "/z/initcfg"]
+ENTRYPOINT ["python3", "-m", "copyparty", "--no-crt", "-c", "/z/initcfg"]
-
-# size: 286 MB
-# bpm/key: 529 sec
-# idx-bench: 2352 MB/s

scripts/docker/Dockerfile.im

@@ -5,18 +5,15 @@ LABEL   org.opencontainers.image.url="https://github.com/9001/copyparty" \
         org.opencontainers.image.licenses="MIT" \
         org.opencontainers.image.title="copyparty-im" \
         org.opencontainers.image.description="copyparty with Pillow and Mutagen (image thumbnails, media tags)"
-ENV     PYTHONPYCACHEPREFIX=/tmp/pyc \
+ENV     XDG_CONFIG_HOME=/cfg
-        XDG_CONFIG_HOME=/cfg
 
 RUN     apk --no-cache add !pyc \
             wget \
-            py3-argon2-cffi py3-pillow py3-mutagen \
+            py3-jinja2 py3-argon2-cffi py3-pillow py3-mutagen
-        && rm -rf /tmp/pyc \
+
-        && mkdir /cfg /w \
+COPY    i/dist/copyparty-sfx.py innvikler.sh ./
-        && chmod 777 /cfg /w \
+RUN     ash innvikler.sh && rm innvikler.sh
-        && echo % /cfg > initcfg
 
-COPY    i/dist/copyparty-sfx.py ./
 WORKDIR /w
 EXPOSE  3923
-ENTRYPOINT ["python3", "/z/copyparty-sfx.py", "--no-crt", "-c", "/z/initcfg"]
+ENTRYPOINT ["python3", "-m", "copyparty", "--no-crt", "-c", "/z/initcfg"]

scripts/docker/Dockerfile.iv

@@ -5,12 +5,11 @@ LABEL   org.opencontainers.image.url="https://github.com/9001/copyparty" \
         org.opencontainers.image.licenses="MIT" \
         org.opencontainers.image.title="copyparty-iv" \
         org.opencontainers.image.description="copyparty with Pillow, FFmpeg, libvips (image/audio/video thumbnails, audio transcoding, media tags)"
-ENV     PYTHONPYCACHEPREFIX=/tmp/pyc \
+ENV     XDG_CONFIG_HOME=/cfg
-        XDG_CONFIG_HOME=/cfg
 
 RUN     apk add -U !pyc \
             wget \
-            py3-argon2-cffi py3-pillow py3-pip py3-cffi \
+            py3-jinja2 py3-argon2-cffi py3-pillow py3-pip py3-cffi \
             ffmpeg \
             vips-jxl vips-heif vips-poppler vips-magick \
         && apk add -t .bd \
@@ -18,13 +17,11 @@ RUN     apk add -U !pyc \
             python3-dev py3-wheel \
         && rm -f /usr/lib/python3*/EXTERNALLY-MANAGED \
         && python3 -m pip install pyvips \
-        && apk del py3-pip .bd \
+        && apk del py3-pip .bd
-        && rm -rf /var/cache/apk/* /tmp/pyc \
+
-        && mkdir /cfg /w \
+COPY    i/dist/copyparty-sfx.py innvikler.sh ./
-        && chmod 777 /cfg /w \
+RUN     ash innvikler.sh && rm innvikler.sh
-        && echo % /cfg > initcfg
 
-COPY    i/dist/copyparty-sfx.py ./
 WORKDIR /w
 EXPOSE  3923
-ENTRYPOINT ["python3", "/z/copyparty-sfx.py", "--no-crt", "-c", "/z/initcfg"]
+ENTRYPOINT ["python3", "-m", "copyparty", "--no-crt", "-c", "/z/initcfg"]

scripts/docker/Dockerfile.min

@@ -5,17 +5,14 @@ LABEL   org.opencontainers.image.url="https://github.com/9001/copyparty" \
         org.opencontainers.image.licenses="MIT" \
         org.opencontainers.image.title="copyparty-min" \
         org.opencontainers.image.description="just copyparty, no thumbnails / media tags / audio transcoding"
-ENV     PYTHONPYCACHEPREFIX=/tmp/pyc \
+ENV     XDG_CONFIG_HOME=/cfg
-        XDG_CONFIG_HOME=/cfg
 
 RUN     apk --no-cache add !pyc \
-            python3 \
+            py3-jinja2
-        && rm -rf /tmp/pyc \
+
-        && mkdir /cfg /w \
+COPY    i/dist/copyparty-sfx.py innvikler.sh ./
-        && chmod 777 /cfg /w \
+RUN     ash innvikler.sh && rm innvikler.sh
-        && echo % /cfg > initcfg
 
-COPY    i/dist/copyparty-sfx.py ./
 WORKDIR /w
 EXPOSE  3923
-ENTRYPOINT ["python3", "/z/copyparty-sfx.py", "--no-crt", "--no-thumb", "-c", "/z/initcfg"]
+ENTRYPOINT ["python3", "-m", "copyparty", "--no-crt", "--no-thumb", "-c", "/z/initcfg"]

scripts/docker/innvikler.sh

@@ -0,0 +1,52 @@
+#!/bin/ash
+set -ex
+
+# cleanup for flavors with python build steps (dj/iv)
+rm -rf /var/cache/apk/* /root/.cache
+
+# initial config; common for all flavors
+mkdir /cfg /w
+chmod 777 /cfg /w
+echo % /cfg > initcfg
+
+# unpack sfx and dive in
+python3 copyparty-sfx.py --version
+cd /tmp/pe-copyparty.0
+
+# workaround s390x deadlock on startup
+[ $(uname -m) = s390x ] && cat >copyparty/stolen/ifaddr/_posix.py <<'EOF'
+def get_adapters(include_unconfigured=False):
+    return []
+EOF
+
+# steal the stuff we need
+mv copyparty partftpy ftp/* /usr/lib/python3.*/site-packages/
+
+# golf
+cd /usr/lib/python3.*/
+rm -rf \
+  /tmp/pe-* /z/copyparty-sfx.py \
+  ensurepip pydoc_data turtle.py turtledemo lib2to3
+
+# drop bytecode
+find / -xdev -name __pycache__ -print0 | xargs -0 rm -rf
+
+# build the stuff we want
+python3 -m compileall -qj4 site-packages sqlite3 xml
+
+# drop the stuff we dont
+find -name __pycache__ |
+  grep -E 'ty/web/|/pycpar' |
+  tr '\n' '\0' | xargs -0 rm -rf
+
+# two-for-one:
+# 1) smoketest copyparty even starts
+# 2) build any bytecode we missed
+# this tends to race other builders (alle gode ting er tre)
+cd /z
+python3 -m copyparty \
+  --ign-ebind -p$((1024+RANDOM)),$((1024+RANDOM)),$((1024+RANDOM)) \
+  --no-crt -qi127.1 --exit=idx -e2dsa -e2ts
+
+# output from -e2d
+rm -rf .hist

scripts/docker/make.sh

@@ -6,6 +6,8 @@ set -e
     exit 1
 }
 
+suf=-b1
+suf=
 sarchs="386 amd64 arm/v7 arm64/v8 ppc64le s390x"
 archs="amd64 arm s390x 386 arm64 ppc64le"
 imgs="dj iv min im ac"
@@ -103,11 +105,12 @@ filt=
             # --pull=never does nothing at all btw
             (set -x
             $nice podman build \
+                --squash \
                 --pull=never \
                 --from localhost/alpine-$a \
-                -t copyparty-$i-$a \
+                -t copyparty-$i-$a$suf \
                 -f Dockerfile.$i . ||
-                    (echo $? $i-$a >> err)
+                    (echo $? $i-$a >> err; printf '%096d\n' $(seq 1 42))
             rm -f .blk
             ) 2> >(tee $a.err | sed "s/^/$aa:/" >&2) > >(tee $a.out | sed "s/^/$aa:/") &
         done
@@ -134,9 +137,10 @@ filt=
         variants=
         for a in $archs; do
             [[ " ${ngs[*]} " =~ " $i-$a " ]] && continue
-            variants="$variants containers-storage:localhost/copyparty-$i-$a"
+            variants="$variants containers-storage:localhost/copyparty-$i-$a$suf"
         done
-        podman manifest create copyparty-$i $variants
+        podman manifest rm copyparty-$i$suf || echo "(that's fine btw)"
+        podman manifest create copyparty-$i$suf $variants
     done
 }