mirrors/copyparty
1
0
Fork 0
mirror of https://github.com/9001/copyparty.git synced 2025-08-17 09:02:15 -06:00
Code Issues Actions Packages Projects Releases Wiki Activity

crossplatform sfx

Browse source
This commit is contained in:
ed 2020-05-06 00:39:21 +02:00
parent c4bea13be5
commit e0a38ceeee
4 changed files with 299 additions and 128 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
README.md
View file

@ -49,6 +49,25 @@ optional, enables thumbnails:
* `Pillow` (requires py2.7 or py3.5+) * `Pillow` (requires py2.7 or py3.5+)
# sfx
currently there are two self-contained binaries:
* `copyparty-sfx.sh` for unix (linux and osx) -- smaller, more robust
* `copyparty-sfx.py` for windows (unix too) -- crossplatform, beta
launch either of them and it'll unpack and run copyparty, assuming you have python installed of course
pls note that `copyparty-sfx.sh` will fail if you rename `copyparty-sfx.py` to `copyparty.py` and keep it in the same folder because `sys.path` is funky
if you don't need all the features you can repack the sfx and save a bunch of space, tho currently the only removable feature is the opus/vorbis javascript decoder which is needed by apple devices to play foss audio files
steps to reduce the sfx size from `720 kB` to `250 kB` roughly:
* run one of the sfx'es once to unpack it
* `./scripts/make-sfx.sh re no-ogv` creates a new pair of sfx
no internet connection needed, just download an sfx and the repo zip (also if you're on windows use msys2)
# install on android # install on android
install [Termux](https://termux.com/) (see [ocv.me/termux](https://ocv.me/termux/)) and then copy-paste this into Termux (long-tap) all at once: install [Termux](https://termux.com/) (see [ocv.me/termux](https://ocv.me/termux/)) and then copy-paste this into Termux (long-tap) all at once:
@ -78,6 +97,7 @@ in the `scripts` folder:
* run `make -C deps-docker` to build all dependencies * run `make -C deps-docker` to build all dependencies
* create github release with `make-tgz-release.sh` * create github release with `make-tgz-release.sh`
* upload to pypi with `make-pypi-release.(sh|bat)` * upload to pypi with `make-pypi-release.(sh|bat)`
* create sfx with `make-sfx.sh`
# todo # todo

168
scripts/make-sfx.sh
View file

@ -2,69 +2,149 @@
set -e set -e
echo echo
# clean=1 to export clean files from git;
# will use current working tree otherwise
clean=1
clean=
tar=$( which gtar 2>/dev/null || which tar) # optional args:
sed=$( which gsed 2>/dev/null || which sed) #
find=$(which gfind 2>/dev/null || which find) # `clean` uses files from git (everything except web/deps),
sort=$(which gsort 2>/dev/null || which sort) # so local changes won't affect the produced sfx
#
# `re` does a repack of an sfx which you already executed once
# (grabs files from the sfx-created tempdir), overrides `clean`
#
# `no-ogv` saves ~500k by removing the opus/vorbis audio codecs
# (only affects apple devices; everything else has native support)
[[ -e copyparty/__main__.py ]] || cd ..
[[ -e copyparty/__main__.py ]] || command -v gtar >/dev/null &&
command -v gfind >/dev/null && {
tar() { gtar "$@"; }
sed() { gsed "$@"; }
find() { gfind "$@"; }
sort() { gsort "$@"; }
}
[ -e copyparty/__main__.py ] || cd ..
[ -e copyparty/__main__.py ] ||
{ {
echo "run me from within the project root folder" echo "run me from within the project root folder"
echo echo
exit 1 exit 1
} }
$find -name '*.pyc' -delete while [ ! -z "$1" ]; do
$find -name __pycache__ -delete [ "$1" = clean ] && clean=1 && shift && continue
[ "$1" = re ] && repack=1 && shift && continue
[ "$1" = no-ogv ] && no_ogv=1 && shift && continue
break
done
rm -rf sfx/* rm -rf sfx/*
mkdir -p sfx build mkdir -p sfx build
cd sfx cd sfx
echo collecting jinja2 [ $repack ] && {
f="../build/Jinja2-2.6.tar.gz" old="$(
[ -e "$f" ] || printf '%s\n' "$TMPDIR" /tmp |
(url=https://files.pythonhosted.org/packages/25/c8/212b1c2fd6df9eaf536384b6c6619c4e70a3afd2dffdd00e5296ffbae940/Jinja2-2.6.tar.gz; awk '/./ {print; exit}'
wget -O$f "$url" || curl -L "$url" >$f) )/pe-copyparty"
tar -zxf $f echo "repack of files in $old"
mv Jinja2-*/jinja2 . cp -pR "$old/"*{jinja2,copyparty} .
rm -rf Jinja2-* mv {x.,}jinja2 2>/dev/null || true
}
# msys2 tar is bad, make the best of it
echo collecting source [ $repack ] || {
[ $clean ] && { echo collecting jinja2
(cd .. && git archive master >tar) && tar -xf ../tar copyparty f="../build/Jinja2-2.6.tar.gz"
(cd .. && tar -cf tar copyparty/web/deps) && tar -xf ../tar [ -e "$f" ] ||
(url=https://files.pythonhosted.org/packages/25/c8/212b1c2fd6df9eaf536384b6c6619c4e70a3afd2dffdd00e5296ffbae940/Jinja2-2.6.tar.gz;
wget -O$f "$url" || curl -L "$url" >$f)
tar -zxf $f
mv Jinja2-*/jinja2 .
rm -rf Jinja2-* jinja2/testsuite
# msys2 tar is bad, make the best of it
echo collecting source
[ $clean ] && {
(cd .. && git archive master >tar) && tar -xf ../tar copyparty
(cd .. && tar -cf tar copyparty/web/deps) && tar -xf ../tar
}
[ $clean ] || {
(cd .. && tar -cf tar copyparty) && tar -xf ../tar
}
rm -f ../tar
} }
[ $clean ] || {
(cd .. && tar -cf tar copyparty) && tar -xf ../tar
}
rm -f ../tar
echo creating tar
ver="$(awk '/^VERSION *= \(/ { ver="$(awk '/^VERSION *= \(/ {
gsub(/[^0-9,]/,""); gsub(/,/,"."); print; exit}' < ../copyparty/__version__.py)" gsub(/[^0-9,]/,""); gsub(/,/,"."); print; exit}' < ../copyparty/__version__.py)"
tar -cf tar copyparty jinja2 ts=$(date -u +%s)
hts=$(date -u +%Y-%m%d-%H%M%S) # --date=@$ts (thx osx)
echo compressing tar
bzip2 -9 tar
echo creating sfx
python ../scripts/sfx.py --sfx-make tar.bz2 $ver
mkdir -p ../dist mkdir -p ../dist
sfx_out=../dist/copyparty-$ver-sfx.py sfx_out=../dist/copyparty-sfx
mv sfx.out $sfx_out
chmod 755 $sfx_out
printf "done:\n %s\n" "$(realpath $sfx_out)" echo cleanup
cd .. find .. -name '*.pyc' -delete
rm -rf sfx find .. -name __pycache__ -delete
# especially prevent osx from leaking your lan ip (wtf apple)
find .. -type f \( -name .DS_Store -or -name ._.DS_Store \) -delete
find .. -type f -name ._\* | while IFS= read -r f; do cmp <(printf '\x00\x05\x16') <(head -c 3 -- "$f") && rm -f -- "$f"; done
echo use smol web deps
rm -f copyparty/web/deps/*.full.*
# it's fine dw
grep -lE '\.full\.(js|css)' copyparty/web/* |
while IFS= read -r x; do sed -ri 's/\.full\.(js|css)/.\1/g' "$x"; done
[ $no_ogv ] &&
rm -rf copyparty/web/deps/{dynamicaudio,ogv}* copyparty/web/browser.js
echo creating tar
args=(--owner=1000 --group=1000)
[ "$OSTYPE" = msys ] &&
args=()
tar -cf tar "${args[@]}" --numeric-owner copyparty jinja2
echo compressing tar
# detect best level; bzip2 -7 is usually better than -9
for n in {2..9}; do cp tar t.$n; bzip2 -$n t.$n & done; wait; mv -v $(ls -1S t.*.bz2 | tail -n 1) tar.bz2
for n in {2..9}; do cp tar t.$n; xz -ze$n t.$n & done; wait; mv -v $(ls -1S t.*.xz | tail -n 1) tar.xz
rm t.*
echo creating unix sfx
(
sed "s/PACK_TS/$ts/; s/PACK_HTS/$hts/; s/CPP_VER/$ver/" <../scripts/sfx.sh |
grep -E '^sfx_eof$' -B 9001;
cat tar.xz
) >$sfx_out.sh
echo creating generic sfx
python ../scripts/sfx.py --sfx-make tar.bz2 $ver $ts
mv sfx.out $sfx_out.py
chmod 755 $sfx_out.*
printf "done:\n"
printf " %s\n" "$(realpath $sfx_out)."{sh,py}
# rm -rf *
# -rw-r--r-- 1 ed ed 811271 May 5 14:35 tar.bz2
# -rw-r--r-- 1 ed ed 732016 May 5 14:35 tar.xz
# -rwxr-xr-x 1 ed ed 830425 May 5 14:35 copyparty-sfx.py*
# -rwxr-xr-x 1 ed ed 734088 May 5 14:35 copyparty-sfx.sh*
# -rwxr-xr-x 1 ed ed 799690 May 5 14:45 copyparty-sfx.py*
# -rwxr-xr-x 1 ed ed 735004 May 5 14:45 copyparty-sfx.sh*
# time pigz -11 -J 34 -I 5730 < tar > tar.gz.5730
# real 8m50.622s
# user 33m9.821s
# -rw-r--r-- 1 ed ed 1136640 May 5 14:50 tar
# -rw-r--r-- 1 ed ed 296334 May 5 14:50 tar.bz2
# -rw-r--r-- 1 ed ed 324705 May 5 15:01 tar.gz.5730
# -rw-r--r-- 1 ed ed 257208 May 5 14:50 tar.xz

167
scripts/sfx.py
View file

@ -2,16 +2,7 @@
# coding: utf-8 # coding: utf-8
from __future__ import print_function, unicode_literals from __future__ import print_function, unicode_literals
import re import re, os, sys, stat, time, shutil, tarfile, hashlib, platform, tempfile
import os
import sys
import time
import signal
import shutil
import tarfile
import hashlib
import platform
import tempfile
import subprocess as sp import subprocess as sp
""" """
@ -20,7 +11,7 @@ run me with any version of python, i will unpack and run copyparty
(but please don't edit this file with a text editor (but please don't edit this file with a text editor
since that would probably corrupt the binary stuff at the end) since that would probably corrupt the binary stuff at the end)
there is zero binaries! just plaintext python scripts all the way down there's zero binaries! just plaintext python scripts all the way down
so you can easily unpack the archive and inspect it for shady stuff so you can easily unpack the archive and inspect it for shady stuff
the archive data is attached after the b"\n# eof\n" archive marker, the archive data is attached after the b"\n# eof\n" archive marker,
@ -29,12 +20,13 @@ the archive data is attached after the b"\n# eof\n" archive marker,
b"\n# " decodes to b"" b"\n# " decodes to b""
""" """
# metadata set when building the sfx # set by make-sfx.sh
VER = None VER = None
SIZE = None SIZE = None
CKSUM = None CKSUM = None
STAMP = None STAMP = None
PY2 = sys.version_info[0] == 2
sys.dont_write_bytecode = True sys.dont_write_bytecode = True
me = os.path.abspath(os.path.realpath(__file__)) me = os.path.abspath(os.path.realpath(__file__))
@ -46,7 +38,7 @@ def eprint(*args, **kwargs):
def msg(*args, **kwargs): def msg(*args, **kwargs):
if args: if args:
args = ["[SFX] {}".format(args[0])] + list(args[1:]) args = ["[SFX]", args[0]] + list(args[1:])
eprint(*args, **kwargs) eprint(*args, **kwargs)
@ -146,7 +138,7 @@ def testchk(cdata):
msg(txt) msg(txt)
def encode(data, size, cksum, ver): def encode(data, size, cksum, ver, ts):
"""creates a new sfx; `data` should yield bufs to attach""" """creates a new sfx; `data` should yield bufs to attach"""
nin = 0 nin = 0
nout = 0 nout = 0
@ -169,7 +161,7 @@ def encode(data, size, cksum, ver):
["VER", '"' + ver + '"'], ["VER", '"' + ver + '"'],
["SIZE", size], ["SIZE", size],
["CKSUM", '"' + cksum + '"'], ["CKSUM", '"' + cksum + '"'],
["STAMP", int(time.time())], ["STAMP", ts],
]: ]:
v1 = "\n{} = None\n".format(k) v1 = "\n{} = None\n".format(k)
v2 = "\n{} = {}\n".format(k, v) v2 = "\n{} = {}\n".format(k, v)
@ -190,10 +182,10 @@ def encode(data, size, cksum, ver):
msg("wrote {:x}H bytes ({:x}H after encode)".format(nin, nout)) msg("wrote {:x}H bytes ({:x}H after encode)".format(nin, nout))
def makesfx(tar_src, ver): def makesfx(tar_src, ver, ts):
sz = os.path.getsize(tar_src) sz = os.path.getsize(tar_src)
cksum = hashfile(tar_src) cksum = hashfile(tar_src)
encode(yieldfile(tar_src), sz, cksum, ver) encode(yieldfile(tar_src), sz, cksum, ver, ts)
# skip 0 # skip 0
@ -261,7 +253,9 @@ def read_py(binp):
] ]
p = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE) p = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE)
ver, _ = p.communicate() ver, _ = p.communicate()
return ver.split(b" ")[:3], p.returncode == 0 ver = ver.decode("utf-8").split(" ")[:3]
ver = [int(x) if x.isdigit() else 0 for x in ver]
return ver, p.returncode == 0
def get_pys(): def get_pys():
@ -277,9 +271,9 @@ def get_pys():
ret = [] ret = []
for binp in hits.values(): for binp in hits.values():
msg("testing", binp)
ver, chk = read_py(binp) ver, chk = read_py(binp)
ret.append([chk, ver, binp]) ret.append([chk, ver, binp])
msg("\t".join(str(x) for x in ret[-1]))
return ret return ret
@ -300,29 +294,21 @@ def hashfile(fn):
def unpack(): def unpack():
"""unpacks the tar yielded by `data`""" """unpacks the tar yielded by `data`"""
tag = "copyparty-{}".format(STAMP) name = "pe-copyparty"
tmp = tempfile.gettempdir() withpid = "{}.{}".format(name, os.getpid())
top = tempfile.gettempdir()
final = os.path.join(top, name)
mine = os.path.join(top, withpid)
tar = os.path.join(mine, "tar")
tag_mine = os.path.join(mine, "v" + str(STAMP))
tag_final = os.path.join(final, "v" + str(STAMP))
for fn in os.listdir(tmp): if os.path.exists(tag_final):
if fn.startswith("copyparty-") and fn != tag: msg("found early")
try: return final
old = os.path.join(tmp, fn)
shutil.rmtree(old)
except:
pass
tmp = os.path.join(tmp, tag)
tar = os.path.join(tmp, "tar")
ok = os.path.join(tmp, "ok")
if os.path.exists(ok):
return tmp
if os.path.exists(tmp):
shutil.rmtree(tmp)
os.mkdir(tmp)
nwrite = 0 nwrite = 0
os.mkdir(mine)
with open(tar, "wb") as f: with open(tar, "wb") as f:
for buf in get_payload(): for buf in get_payload():
nwrite += len(buf) nwrite += len(buf)
@ -338,14 +324,44 @@ def unpack():
raise Exception(t) raise Exception(t)
with tarfile.open(tar, "r:bz2") as tf: with tarfile.open(tar, "r:bz2") as tf:
tf.extractall(tmp) tf.extractall(mine)
os.remove(tar) os.remove(tar)
with open(ok, "wb") as f: with open(tag_mine, "wb") as f:
f.write(b"h\n")
if os.path.exists(tag_final):
msg("found late")
return final
try:
if os.path.islink(final):
os.remove(final)
else:
shutil.rmtree(final)
except:
pass pass
return tmp try:
os.symlink(mine, final)
except:
try:
os.rename(mine, final)
except:
msg("reloc fail,", mine)
return mine
for fn in os.listdir(top):
if fn.startswith(name) and fn not in [name, withpid]:
try:
old = os.path.join(top, fn)
if time.time() - os.path.getmtime(old) > 10:
shutil.rmtree(old)
except:
pass
return final
def get_payload(): def get_payload():
@ -402,50 +418,29 @@ def get_payload():
def confirm(): def confirm():
msg() msg()
msg("*** hit enter to exit ***") msg("*** hit enter to exit ***")
try: raw_input() if PY2 else input()
raw_input()
except NameError:
input()
def run(tmp, py): def run(tmp, py):
msg("OK") msg("OK")
msg("will use:", py) msg("will use:", py)
msg("bound to:", tmp, "\n") msg("bound to:", tmp)
fp_py = os.path.join(tmp, "py") fp_py = os.path.join(tmp, "py")
with open(fp_py, "wb") as f: with open(fp_py, "wb") as f:
f.write(py.encode("utf-8") + b"\n") f.write(py.encode("utf-8") + b"\n")
env = os.environ.copy() # avoid loading ./copyparty.py
try: cmd = [
libs = "{}:{}".format(tmp, env["PYTHONPATH"]) py,
except: "-c",
libs = tmp 'import sys, runpy; sys.path.insert(0, r"'
+ tmp
+ '"); runpy.run_module("copyparty", run_name="__main__")',
] + list(sys.argv[1:])
env[str("PYTHONPATH")] = str(libs) msg("\n", cmd, "\n")
p = sp.Popen(str(x) for x in cmd)
# skip 1
if False:
# mingw64 py3.8.2 doesn't emit any prints without -u
env[str("PYTHONUNBUFFERED")] = str("ja")
# it also doesn't deal with ^C and none of this helps
def orz(sig, frame):
p.terminate()
signal.signal(signal.SIGINT, orz)
while True:
try:
time.sleep(9001)
except:
p.terminate()
break
# skip 0
cmd = [py, "-m", "copyparty"] + list(sys.argv[1:])
p = sp.Popen([str(x) for x in cmd], env=env)
try: try:
p.wait() p.wait()
except: except:
@ -458,11 +453,12 @@ def run(tmp, py):
def main(): def main():
os.system("")
sysver = str(sys.version).replace("\n", "\n" + " " * 18) sysver = str(sys.version).replace("\n", "\n" + " " * 18)
pktime = time.strftime("%Y-%m-%d, %H:%M:%S", time.gmtime(STAMP))
os.system("")
msg() msg()
msg(" this is: copyparty", VER) msg(" this is: copyparty", VER)
msg(" packed at:", time.strftime("%Y-%m-%d, %H:%M:%S UTC", time.gmtime(STAMP))) msg(" packed at:", pktime, "UTC,", STAMP)
msg("archive is:", me) msg("archive is:", me)
msg("python bin:", sys.executable) msg("python bin:", sys.executable)
msg("python ver:", platform.python_implementation(), sysver) msg("python ver:", platform.python_implementation(), sysver)
@ -477,16 +473,14 @@ def main():
# skip 1 # skip 1
if arg == "--sfx-testgen": if arg == "--sfx-testgen":
return encode(testptn(), 1, "x", "x") return encode(testptn(), 1, "x", "x", 1)
if arg == "--sfx-testchk": if arg == "--sfx-testchk":
return testchk(get_payload()) return testchk(get_payload())
if arg == "--sfx-make": if arg == "--sfx-make":
tar, ver = sys.argv[2:] tar, ver, ts = sys.argv[2:]
return makesfx(tar, ver) return makesfx(tar, ver, ts)
# https://docs.microsoft.com/en-us/windows/win32/shell/knownfolderid?redirectedfrom=MSDN
# skip 0 # skip 0
@ -495,13 +489,18 @@ def main():
if os.path.exists(fp_py): if os.path.exists(fp_py):
with open(fp_py, "rb") as f: with open(fp_py, "rb") as f:
py = f.read().decode("utf-8").rstrip() py = f.read().decode("utf-8").rstrip()
return run(tmp, py)
return run(tmp, py)
pys = get_pys() pys = get_pys()
pys.sort(reverse=True) pys.sort(reverse=True)
j2, ver, py = pys[0] j2, ver, py = pys[0]
if j2: if j2:
shutil.rmtree(os.path.join(tmp, "jinja2")) try:
os.rename(os.path.join(tmp, "jinja2"), os.path.join(tmp, "x.jinja2"))
except:
pass
return run(tmp, py) return run(tmp, py)
msg("\n could not find jinja2; will use py2 + the bundled version\n") msg("\n could not find jinja2; will use py2 + the bundled version\n")

72
scripts/sfx.sh Normal file
View file

@ -0,0 +1,72 @@
# use current/default shell
set -e
dir="$(
printf '%s\n' "$TMPDIR" /tmp |
awk '/./ {print; exit}'
)/pe-copyparty"
[ -e "$dir/vPACK_TS" ] || (
printf '\033[36munpacking copyparty vCPP_VER (sfx-PACK_HTS)\033[1;30m\n\n'
mkdir -p "$dir.$$"
ofs=$(awk '$0=="sfx_eof" {print NR+1; exit}' < "$0")
[ -z "$ofs" ] && {
printf '\033[31mabort: could not find SFX boundary\033[0m\n'
exit 1
}
tail -n +$ofs "$0" | tar -JxC "$dir.$$"
ln -nsf "$dir.$$" "$dir"
printf '\033[0m'
now=$(date -u +%s)
for d in "$dir".*; do
ts=$(stat -c%Y -- "$d" 2>/dev/null) ||
ts=$(stat -f %m%n -- "$d" 2>/dev/null)
[ $((now-ts)) -gt 300 ] &&
rm -rf "$d"
done
echo h > "$dir/vPACK_TS"
) >&2 || exit 1
# detect available pythons
(IFS=:; for d in $PATH; do
printf '%s\n' "$d"/python* "$d"/pypy* | tac;
done) | grep -E '(python|pypy)[0-9\.-]*$' > $dir/pys || true
# see if we made a choice before
[ -z "$pybin" ] && pybin="$(cat $dir/py 2>/dev/null || true)"
# otherwise find a python with jinja2
[ -z "$pybin" ] && pybin="$(cat $dir/pys | while IFS= read -r _py; do
printf '\033[1;30mlooking for jinja2 in [%s]\033[0m\n' "$_py" >&2
$_py -c 'import jinja2' 2>/dev/null || continue
printf '%s\n' "$_py"
mv $dir/{,x.}jinja2
break
done)"
# otherwise find python2 (bundled jinja2 is way old)
[ -z "$pybin" ] && {
printf '\033[0;33mcould not find jinja2; will use py2 + the bundled version\033[0m\n' >&2
pybin="$(cat $dir/pys | while IFS= read -r _py; do
printf '\033[1;30mtesting if py2 [%s]\033[0m\n' "$_py" >&2
_ver=$($_py -c 'import sys; sys.stdout.write(str(sys.version_info[0]))' 2>/dev/null) || continue
[ $_ver = 2 ] || continue
printf '%s\n' "$_py"
break
done)"
}
[ -z "$pybin" ] && {
printf '\033[1;31m\ncould not find a python with jinja2 installed; please do one of these:\n\n pip install --user jinja2\n\n install python2\033[0m\n\n' >&2
exit 1
}
printf '\033[1;30musing [%s]. you can reset with this:\n rm -rf %s*\033[0m\n\n' "$pybin" "$dir"
printf '%s\n' "$pybin" > $dir/py
PYTHONPATH=$dir exec "$pybin" -m copyparty "$@"
sfx_eof