diff --git a/README.md b/README.md index 0c2387bf..9b24a08a 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,25 @@ optional, enables thumbnails: * `Pillow` (requires py2.7 or py3.5+) +# sfx + +currently there are two self-contained binaries: +* `copyparty-sfx.sh` for unix (linux and osx) -- smaller, more robust +* `copyparty-sfx.py` for windows (unix too) -- crossplatform, beta + +launch either of them and it'll unpack and run copyparty, assuming you have python installed of course + +pls note that `copyparty-sfx.sh` will fail if you rename `copyparty-sfx.py` to `copyparty.py` and keep it in the same folder because `sys.path` is funky + +if you don't need all the features you can repack the sfx and save a bunch of space, tho currently the only removable feature is the opus/vorbis javascript decoder which is needed by apple devices to play foss audio files + +steps to reduce the sfx size from `720 kB` to `250 kB` roughly: +* run one of the sfx'es once to unpack it +* `./scripts/make-sfx.sh re no-ogv` creates a new pair of sfx + +no internet connection needed, just download an sfx and the repo zip (also if you're on windows use msys2) + + # install on android install [Termux](https://termux.com/) (see [ocv.me/termux](https://ocv.me/termux/)) and then copy-paste this into Termux (long-tap) all at once: @@ -78,6 +97,7 @@ in the `scripts` folder: * run `make -C deps-docker` to build all dependencies * create github release with `make-tgz-release.sh` * upload to pypi with `make-pypi-release.(sh|bat)` +* create sfx with `make-sfx.sh` # todo diff --git a/scripts/make-sfx.sh b/scripts/make-sfx.sh index 9f102e7e..6ca5afb7 100755 --- a/scripts/make-sfx.sh +++ b/scripts/make-sfx.sh @@ -2,69 +2,149 @@ set -e echo -# clean=1 to export clean files from git; -# will use current working tree otherwise -clean=1 -clean= -tar=$( which gtar 2>/dev/null || which tar) -sed=$( which gsed 2>/dev/null || which sed) -find=$(which gfind 2>/dev/null || which find) -sort=$(which gsort 2>/dev/null || which sort) +# optional args: +# +# `clean` uses files from git (everything except web/deps), +# so local changes won't affect the produced sfx +# +# `re` does a repack of an sfx which you already executed once +# (grabs files from the sfx-created tempdir), overrides `clean` +# +# `no-ogv` saves ~500k by removing the opus/vorbis audio codecs +# (only affects apple devices; everything else has native support) -[[ -e copyparty/__main__.py ]] || cd .. -[[ -e copyparty/__main__.py ]] || + +command -v gtar >/dev/null && +command -v gfind >/dev/null && { + tar() { gtar "$@"; } + sed() { gsed "$@"; } + find() { gfind "$@"; } + sort() { gsort "$@"; } +} + +[ -e copyparty/__main__.py ] || cd .. +[ -e copyparty/__main__.py ] || { echo "run me from within the project root folder" echo exit 1 } -$find -name '*.pyc' -delete -$find -name __pycache__ -delete +while [ ! -z "$1" ]; do + [ "$1" = clean ] && clean=1 && shift && continue + [ "$1" = re ] && repack=1 && shift && continue + [ "$1" = no-ogv ] && no_ogv=1 && shift && continue + break +done rm -rf sfx/* mkdir -p sfx build cd sfx -echo collecting jinja2 -f="../build/Jinja2-2.6.tar.gz" -[ -e "$f" ] || - (url=https://files.pythonhosted.org/packages/25/c8/212b1c2fd6df9eaf536384b6c6619c4e70a3afd2dffdd00e5296ffbae940/Jinja2-2.6.tar.gz; - wget -O$f "$url" || curl -L "$url" >$f) +[ $repack ] && { + old="$( + printf '%s\n' "$TMPDIR" /tmp | + awk '/./ {print; exit}' + )/pe-copyparty" -tar -zxf $f -mv Jinja2-*/jinja2 . -rm -rf Jinja2-* - -# msys2 tar is bad, make the best of it -echo collecting source -[ $clean ] && { - (cd .. && git archive master >tar) && tar -xf ../tar copyparty - (cd .. && tar -cf tar copyparty/web/deps) && tar -xf ../tar + echo "repack of files in $old" + cp -pR "$old/"*{jinja2,copyparty} . + mv {x.,}jinja2 2>/dev/null || true +} + +[ $repack ] || { + echo collecting jinja2 + f="../build/Jinja2-2.6.tar.gz" + [ -e "$f" ] || + (url=https://files.pythonhosted.org/packages/25/c8/212b1c2fd6df9eaf536384b6c6619c4e70a3afd2dffdd00e5296ffbae940/Jinja2-2.6.tar.gz; + wget -O$f "$url" || curl -L "$url" >$f) + + tar -zxf $f + mv Jinja2-*/jinja2 . + rm -rf Jinja2-* jinja2/testsuite + + # msys2 tar is bad, make the best of it + echo collecting source + [ $clean ] && { + (cd .. && git archive master >tar) && tar -xf ../tar copyparty + (cd .. && tar -cf tar copyparty/web/deps) && tar -xf ../tar + } + [ $clean ] || { + (cd .. && tar -cf tar copyparty) && tar -xf ../tar + } + rm -f ../tar } -[ $clean ] || { - (cd .. && tar -cf tar copyparty) && tar -xf ../tar -} -rm -f ../tar -echo creating tar ver="$(awk '/^VERSION *= \(/ { gsub(/[^0-9,]/,""); gsub(/,/,"."); print; exit}' < ../copyparty/__version__.py)" -tar -cf tar copyparty jinja2 - -echo compressing tar -bzip2 -9 tar - -echo creating sfx -python ../scripts/sfx.py --sfx-make tar.bz2 $ver +ts=$(date -u +%s) +hts=$(date -u +%Y-%m%d-%H%M%S) # --date=@$ts (thx osx) mkdir -p ../dist -sfx_out=../dist/copyparty-$ver-sfx.py -mv sfx.out $sfx_out -chmod 755 $sfx_out +sfx_out=../dist/copyparty-sfx -printf "done:\n %s\n" "$(realpath $sfx_out)" -cd .. -rm -rf sfx +echo cleanup +find .. -name '*.pyc' -delete +find .. -name __pycache__ -delete + +# especially prevent osx from leaking your lan ip (wtf apple) +find .. -type f \( -name .DS_Store -or -name ._.DS_Store \) -delete +find .. -type f -name ._\* | while IFS= read -r f; do cmp <(printf '\x00\x05\x16') <(head -c 3 -- "$f") && rm -f -- "$f"; done + +echo use smol web deps +rm -f copyparty/web/deps/*.full.* + +# it's fine dw +grep -lE '\.full\.(js|css)' copyparty/web/* | +while IFS= read -r x; do sed -ri 's/\.full\.(js|css)/.\1/g' "$x"; done + +[ $no_ogv ] && + rm -rf copyparty/web/deps/{dynamicaudio,ogv}* copyparty/web/browser.js + +echo creating tar +args=(--owner=1000 --group=1000) +[ "$OSTYPE" = msys ] && + args=() + +tar -cf tar "${args[@]}" --numeric-owner copyparty jinja2 + +echo compressing tar +# detect best level; bzip2 -7 is usually better than -9 +for n in {2..9}; do cp tar t.$n; bzip2 -$n t.$n & done; wait; mv -v $(ls -1S t.*.bz2 | tail -n 1) tar.bz2 +for n in {2..9}; do cp tar t.$n; xz -ze$n t.$n & done; wait; mv -v $(ls -1S t.*.xz | tail -n 1) tar.xz +rm t.* + +echo creating unix sfx +( + sed "s/PACK_TS/$ts/; s/PACK_HTS/$hts/; s/CPP_VER/$ver/" <../scripts/sfx.sh | + grep -E '^sfx_eof$' -B 9001; + cat tar.xz +) >$sfx_out.sh + +echo creating generic sfx +python ../scripts/sfx.py --sfx-make tar.bz2 $ver $ts +mv sfx.out $sfx_out.py +chmod 755 $sfx_out.* + +printf "done:\n" +printf " %s\n" "$(realpath $sfx_out)."{sh,py} +# rm -rf * + +# -rw-r--r-- 1 ed ed 811271 May 5 14:35 tar.bz2 +# -rw-r--r-- 1 ed ed 732016 May 5 14:35 tar.xz + +# -rwxr-xr-x 1 ed ed 830425 May 5 14:35 copyparty-sfx.py* +# -rwxr-xr-x 1 ed ed 734088 May 5 14:35 copyparty-sfx.sh* + +# -rwxr-xr-x 1 ed ed 799690 May 5 14:45 copyparty-sfx.py* +# -rwxr-xr-x 1 ed ed 735004 May 5 14:45 copyparty-sfx.sh* + +# time pigz -11 -J 34 -I 5730 < tar > tar.gz.5730 +# real 8m50.622s +# user 33m9.821s +# -rw-r--r-- 1 ed ed 1136640 May 5 14:50 tar +# -rw-r--r-- 1 ed ed 296334 May 5 14:50 tar.bz2 +# -rw-r--r-- 1 ed ed 324705 May 5 15:01 tar.gz.5730 +# -rw-r--r-- 1 ed ed 257208 May 5 14:50 tar.xz diff --git a/scripts/sfx.py b/scripts/sfx.py index 3266becc..85dfa5b1 100644 --- a/scripts/sfx.py +++ b/scripts/sfx.py @@ -2,16 +2,7 @@ # coding: utf-8 from __future__ import print_function, unicode_literals -import re -import os -import sys -import time -import signal -import shutil -import tarfile -import hashlib -import platform -import tempfile +import re, os, sys, stat, time, shutil, tarfile, hashlib, platform, tempfile import subprocess as sp """ @@ -20,7 +11,7 @@ run me with any version of python, i will unpack and run copyparty (but please don't edit this file with a text editor since that would probably corrupt the binary stuff at the end) -there is zero binaries! just plaintext python scripts all the way down +there's zero binaries! just plaintext python scripts all the way down so you can easily unpack the archive and inspect it for shady stuff the archive data is attached after the b"\n# eof\n" archive marker, @@ -29,12 +20,13 @@ the archive data is attached after the b"\n# eof\n" archive marker, b"\n# " decodes to b"" """ -# metadata set when building the sfx +# set by make-sfx.sh VER = None SIZE = None CKSUM = None STAMP = None +PY2 = sys.version_info[0] == 2 sys.dont_write_bytecode = True me = os.path.abspath(os.path.realpath(__file__)) @@ -46,7 +38,7 @@ def eprint(*args, **kwargs): def msg(*args, **kwargs): if args: - args = ["[SFX] {}".format(args[0])] + list(args[1:]) + args = ["[SFX]", args[0]] + list(args[1:]) eprint(*args, **kwargs) @@ -146,7 +138,7 @@ def testchk(cdata): msg(txt) -def encode(data, size, cksum, ver): +def encode(data, size, cksum, ver, ts): """creates a new sfx; `data` should yield bufs to attach""" nin = 0 nout = 0 @@ -169,7 +161,7 @@ def encode(data, size, cksum, ver): ["VER", '"' + ver + '"'], ["SIZE", size], ["CKSUM", '"' + cksum + '"'], - ["STAMP", int(time.time())], + ["STAMP", ts], ]: v1 = "\n{} = None\n".format(k) v2 = "\n{} = {}\n".format(k, v) @@ -190,10 +182,10 @@ def encode(data, size, cksum, ver): msg("wrote {:x}H bytes ({:x}H after encode)".format(nin, nout)) -def makesfx(tar_src, ver): +def makesfx(tar_src, ver, ts): sz = os.path.getsize(tar_src) cksum = hashfile(tar_src) - encode(yieldfile(tar_src), sz, cksum, ver) + encode(yieldfile(tar_src), sz, cksum, ver, ts) # skip 0 @@ -261,7 +253,9 @@ def read_py(binp): ] p = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE) ver, _ = p.communicate() - return ver.split(b" ")[:3], p.returncode == 0 + ver = ver.decode("utf-8").split(" ")[:3] + ver = [int(x) if x.isdigit() else 0 for x in ver] + return ver, p.returncode == 0 def get_pys(): @@ -277,9 +271,9 @@ def get_pys(): ret = [] for binp in hits.values(): - msg("testing", binp) ver, chk = read_py(binp) ret.append([chk, ver, binp]) + msg("\t".join(str(x) for x in ret[-1])) return ret @@ -300,29 +294,21 @@ def hashfile(fn): def unpack(): """unpacks the tar yielded by `data`""" - tag = "copyparty-{}".format(STAMP) - tmp = tempfile.gettempdir() + name = "pe-copyparty" + withpid = "{}.{}".format(name, os.getpid()) + top = tempfile.gettempdir() + final = os.path.join(top, name) + mine = os.path.join(top, withpid) + tar = os.path.join(mine, "tar") + tag_mine = os.path.join(mine, "v" + str(STAMP)) + tag_final = os.path.join(final, "v" + str(STAMP)) - for fn in os.listdir(tmp): - if fn.startswith("copyparty-") and fn != tag: - try: - old = os.path.join(tmp, fn) - shutil.rmtree(old) - except: - pass + if os.path.exists(tag_final): + msg("found early") + return final - tmp = os.path.join(tmp, tag) - tar = os.path.join(tmp, "tar") - ok = os.path.join(tmp, "ok") - - if os.path.exists(ok): - return tmp - - if os.path.exists(tmp): - shutil.rmtree(tmp) - - os.mkdir(tmp) nwrite = 0 + os.mkdir(mine) with open(tar, "wb") as f: for buf in get_payload(): nwrite += len(buf) @@ -338,14 +324,44 @@ def unpack(): raise Exception(t) with tarfile.open(tar, "r:bz2") as tf: - tf.extractall(tmp) + tf.extractall(mine) os.remove(tar) - with open(ok, "wb") as f: + with open(tag_mine, "wb") as f: + f.write(b"h\n") + + if os.path.exists(tag_final): + msg("found late") + return final + + try: + if os.path.islink(final): + os.remove(final) + else: + shutil.rmtree(final) + except: pass - return tmp + try: + os.symlink(mine, final) + except: + try: + os.rename(mine, final) + except: + msg("reloc fail,", mine) + return mine + + for fn in os.listdir(top): + if fn.startswith(name) and fn not in [name, withpid]: + try: + old = os.path.join(top, fn) + if time.time() - os.path.getmtime(old) > 10: + shutil.rmtree(old) + except: + pass + + return final def get_payload(): @@ -402,50 +418,29 @@ def get_payload(): def confirm(): msg() msg("*** hit enter to exit ***") - try: - raw_input() - except NameError: - input() + raw_input() if PY2 else input() def run(tmp, py): msg("OK") msg("will use:", py) - msg("bound to:", tmp, "\n") + msg("bound to:", tmp) fp_py = os.path.join(tmp, "py") with open(fp_py, "wb") as f: f.write(py.encode("utf-8") + b"\n") - env = os.environ.copy() - try: - libs = "{}:{}".format(tmp, env["PYTHONPATH"]) - except: - libs = tmp + # avoid loading ./copyparty.py + cmd = [ + py, + "-c", + 'import sys, runpy; sys.path.insert(0, r"' + + tmp + + '"); runpy.run_module("copyparty", run_name="__main__")', + ] + list(sys.argv[1:]) - env[str("PYTHONPATH")] = str(libs) - - # skip 1 - if False: - # mingw64 py3.8.2 doesn't emit any prints without -u - env[str("PYTHONUNBUFFERED")] = str("ja") - - # it also doesn't deal with ^C and none of this helps - def orz(sig, frame): - p.terminate() - - signal.signal(signal.SIGINT, orz) - - while True: - try: - time.sleep(9001) - except: - p.terminate() - break - # skip 0 - - cmd = [py, "-m", "copyparty"] + list(sys.argv[1:]) - p = sp.Popen([str(x) for x in cmd], env=env) + msg("\n", cmd, "\n") + p = sp.Popen(str(x) for x in cmd) try: p.wait() except: @@ -458,11 +453,12 @@ def run(tmp, py): def main(): - os.system("") sysver = str(sys.version).replace("\n", "\n" + " " * 18) + pktime = time.strftime("%Y-%m-%d, %H:%M:%S", time.gmtime(STAMP)) + os.system("") msg() msg(" this is: copyparty", VER) - msg(" packed at:", time.strftime("%Y-%m-%d, %H:%M:%S UTC", time.gmtime(STAMP))) + msg(" packed at:", pktime, "UTC,", STAMP) msg("archive is:", me) msg("python bin:", sys.executable) msg("python ver:", platform.python_implementation(), sysver) @@ -477,16 +473,14 @@ def main(): # skip 1 if arg == "--sfx-testgen": - return encode(testptn(), 1, "x", "x") + return encode(testptn(), 1, "x", "x", 1) if arg == "--sfx-testchk": return testchk(get_payload()) if arg == "--sfx-make": - tar, ver = sys.argv[2:] - return makesfx(tar, ver) - - # https://docs.microsoft.com/en-us/windows/win32/shell/knownfolderid?redirectedfrom=MSDN + tar, ver, ts = sys.argv[2:] + return makesfx(tar, ver, ts) # skip 0 @@ -495,13 +489,18 @@ def main(): if os.path.exists(fp_py): with open(fp_py, "rb") as f: py = f.read().decode("utf-8").rstrip() - return run(tmp, py) + + return run(tmp, py) pys = get_pys() pys.sort(reverse=True) j2, ver, py = pys[0] if j2: - shutil.rmtree(os.path.join(tmp, "jinja2")) + try: + os.rename(os.path.join(tmp, "jinja2"), os.path.join(tmp, "x.jinja2")) + except: + pass + return run(tmp, py) msg("\n could not find jinja2; will use py2 + the bundled version\n") diff --git a/scripts/sfx.sh b/scripts/sfx.sh new file mode 100644 index 00000000..c7f7bbde --- /dev/null +++ b/scripts/sfx.sh @@ -0,0 +1,72 @@ +# use current/default shell +set -e + +dir="$( + printf '%s\n' "$TMPDIR" /tmp | + awk '/./ {print; exit}' +)/pe-copyparty" + +[ -e "$dir/vPACK_TS" ] || ( + printf '\033[36munpacking copyparty vCPP_VER (sfx-PACK_HTS)\033[1;30m\n\n' + mkdir -p "$dir.$$" + ofs=$(awk '$0=="sfx_eof" {print NR+1; exit}' < "$0") + + [ -z "$ofs" ] && { + printf '\033[31mabort: could not find SFX boundary\033[0m\n' + exit 1 + } + tail -n +$ofs "$0" | tar -JxC "$dir.$$" + ln -nsf "$dir.$$" "$dir" + printf '\033[0m' + + now=$(date -u +%s) + for d in "$dir".*; do + ts=$(stat -c%Y -- "$d" 2>/dev/null) || + ts=$(stat -f %m%n -- "$d" 2>/dev/null) + + [ $((now-ts)) -gt 300 ] && + rm -rf "$d" + done + echo h > "$dir/vPACK_TS" +) >&2 || exit 1 + +# detect available pythons +(IFS=:; for d in $PATH; do + printf '%s\n' "$d"/python* "$d"/pypy* | tac; +done) | grep -E '(python|pypy)[0-9\.-]*$' > $dir/pys || true + +# see if we made a choice before +[ -z "$pybin" ] && pybin="$(cat $dir/py 2>/dev/null || true)" + +# otherwise find a python with jinja2 +[ -z "$pybin" ] && pybin="$(cat $dir/pys | while IFS= read -r _py; do + printf '\033[1;30mlooking for jinja2 in [%s]\033[0m\n' "$_py" >&2 + $_py -c 'import jinja2' 2>/dev/null || continue + printf '%s\n' "$_py" + mv $dir/{,x.}jinja2 + break +done)" + +# otherwise find python2 (bundled jinja2 is way old) +[ -z "$pybin" ] && { + printf '\033[0;33mcould not find jinja2; will use py2 + the bundled version\033[0m\n' >&2 + pybin="$(cat $dir/pys | while IFS= read -r _py; do + printf '\033[1;30mtesting if py2 [%s]\033[0m\n' "$_py" >&2 + _ver=$($_py -c 'import sys; sys.stdout.write(str(sys.version_info[0]))' 2>/dev/null) || continue + [ $_ver = 2 ] || continue + printf '%s\n' "$_py" + break + done)" +} + +[ -z "$pybin" ] && { + printf '\033[1;31m\ncould not find a python with jinja2 installed; please do one of these:\n\n pip install --user jinja2\n\n install python2\033[0m\n\n' >&2 + exit 1 +} + +printf '\033[1;30musing [%s]. you can reset with this:\n rm -rf %s*\033[0m\n\n' "$pybin" "$dir" +printf '%s\n' "$pybin" > $dir/py + +PYTHONPATH=$dir exec "$pybin" -m copyparty "$@" + +sfx_eof