mirrors/copyparty
1
0
Fork 0
mirror of https://github.com/9001/copyparty.git synced 2025-08-17 09:02:15 -06:00
Code Issues Actions Packages Projects Releases Wiki Activity

crossplatform sfx

Browse source
This commit is contained in:
ed 2020-05-06 00:39:21 +02:00
parent c4bea13be5
commit e0a38ceeee
4 changed files with 299 additions and 128 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
README.md
View file

@ -49,6 +49,25 @@ optional, enables thumbnails:
* `Pillow` (requires py2.7 or py3.5+)
# sfx
currently there are two self-contained binaries:
* `copyparty-sfx.sh` for unix (linux and osx) -- smaller, more robust
* `copyparty-sfx.py` for windows (unix too) -- crossplatform, beta
launch either of them and it'll unpack and run copyparty, assuming you have python installed of course
pls note that `copyparty-sfx.sh` will fail if you rename `copyparty-sfx.py` to `copyparty.py` and keep it in the same folder because `sys.path` is funky
if you don't need all the features you can repack the sfx and save a bunch of space, tho currently the only removable feature is the opus/vorbis javascript decoder which is needed by apple devices to play foss audio files
steps to reduce the sfx size from `720 kB` to `250 kB` roughly:
* run one of the sfx'es once to unpack it
* `./scripts/make-sfx.sh re no-ogv` creates a new pair of sfx
no internet connection needed, just download an sfx and the repo zip (also if you're on windows use msys2)
# install on android
install [Termux](https://termux.com/) (see [ocv.me/termux](https://ocv.me/termux/)) and then copy-paste this into Termux (long-tap) all at once:
@ -78,6 +97,7 @@ in the `scripts` folder:
* run `make -C deps-docker` to build all dependencies
* create github release with `make-tgz-release.sh`
* upload to pypi with `make-pypi-release.(sh|bat)`
* create sfx with `make-sfx.sh`
# todo

168
scripts/make-sfx.sh
View file

@ -2,69 +2,149 @@
set -e
echo
# clean=1 to export clean files from git;
# will use current working tree otherwise
clean=1
clean=
tar=$( which gtar 2>/dev/null || which tar)
sed=$( which gsed 2>/dev/null || which sed)
find=$(which gfind 2>/dev/null || which find)
sort=$(which gsort 2>/dev/null || which sort)
# optional args:
#
# `clean` uses files from git (everything except web/deps),
# so local changes won't affect the produced sfx
#
# `re` does a repack of an sfx which you already executed once
# (grabs files from the sfx-created tempdir), overrides `clean`
#
# `no-ogv` saves ~500k by removing the opus/vorbis audio codecs
# (only affects apple devices; everything else has native support)
[[ -e copyparty/__main__.py ]] || cd ..
[[ -e copyparty/__main__.py ]] ||
command -v gtar >/dev/null &&
command -v gfind >/dev/null && {
tar() { gtar "$@"; }
sed() { gsed "$@"; }
find() { gfind "$@"; }
sort() { gsort "$@"; }
}
[ -e copyparty/__main__.py ] || cd ..
[ -e copyparty/__main__.py ] ||
{
echo "run me from within the project root folder"
echo
exit 1
}
$find -name '*.pyc' -delete
$find -name __pycache__ -delete
while [ ! -z "$1" ]; do
[ "$1" = clean ] && clean=1 && shift && continue
[ "$1" = re ] && repack=1 && shift && continue
[ "$1" = no-ogv ] && no_ogv=1 && shift && continue
break
done
rm -rf sfx/*
mkdir -p sfx build
cd sfx
echo collecting jinja2
f="../build/Jinja2-2.6.tar.gz"
[ -e "$f" ] ||
(url=https://files.pythonhosted.org/packages/25/c8/212b1c2fd6df9eaf536384b6c6619c4e70a3afd2dffdd00e5296ffbae940/Jinja2-2.6.tar.gz;
wget -O$f "$url" || curl -L "$url" >$f)
[ $repack ] && {
old="$(
printf '%s\n' "$TMPDIR" /tmp |
awk '/./ {print; exit}'
)/pe-copyparty"
tar -zxf $f
mv Jinja2-*/jinja2 .
rm -rf Jinja2-*
# msys2 tar is bad, make the best of it
echo collecting source
[ $clean ] && {
(cd .. && git archive master >tar) && tar -xf ../tar copyparty
(cd .. && tar -cf tar copyparty/web/deps) && tar -xf ../tar
echo "repack of files in $old"
cp -pR "$old/"*{jinja2,copyparty} .
mv {x.,}jinja2 2>/dev/null || true
}
[ $repack ] || {
echo collecting jinja2
f="../build/Jinja2-2.6.tar.gz"
[ -e "$f" ] ||
(url=https://files.pythonhosted.org/packages/25/c8/212b1c2fd6df9eaf536384b6c6619c4e70a3afd2dffdd00e5296ffbae940/Jinja2-2.6.tar.gz;
wget -O$f "$url" || curl -L "$url" >$f)
tar -zxf $f
mv Jinja2-*/jinja2 .
rm -rf Jinja2-* jinja2/testsuite
# msys2 tar is bad, make the best of it
echo collecting source
[ $clean ] && {
(cd .. && git archive master >tar) && tar -xf ../tar copyparty
(cd .. && tar -cf tar copyparty/web/deps) && tar -xf ../tar
}
[ $clean ] || {
(cd .. && tar -cf tar copyparty) && tar -xf ../tar
}
rm -f ../tar
}
[ $clean ] || {
(cd .. && tar -cf tar copyparty) && tar -xf ../tar
}
rm -f ../tar
echo creating tar
ver="$(awk '/^VERSION *= \(/ {
gsub(/[^0-9,]/,""); gsub(/,/,"."); print; exit}' < ../copyparty/__version__.py)"
tar -cf tar copyparty jinja2
echo compressing tar
bzip2 -9 tar
echo creating sfx
python ../scripts/sfx.py --sfx-make tar.bz2 $ver
ts=$(date -u +%s)
hts=$(date -u +%Y-%m%d-%H%M%S) # --date=@$ts (thx osx)
mkdir -p ../dist
sfx_out=../dist/copyparty-$ver-sfx.py
mv sfx.out $sfx_out
chmod 755 $sfx_out
sfx_out=../dist/copyparty-sfx
printf "done:\n %s\n" "$(realpath $sfx_out)"
cd ..
rm -rf sfx
echo cleanup
find .. -name '*.pyc' -delete
find .. -name __pycache__ -delete
# especially prevent osx from leaking your lan ip (wtf apple)
find .. -type f \( -name .DS_Store -or -name ._.DS_Store \) -delete
find .. -type f -name ._\* | while IFS= read -r f; do cmp <(printf '\x00\x05\x16') <(head -c 3 -- "$f") && rm -f -- "$f"; done
echo use smol web deps
rm -f copyparty/web/deps/*.full.*
# it's fine dw
grep -lE '\.full\.(js|css)' copyparty/web/* |
while IFS= read -r x; do sed -ri 's/\.full\.(js|css)/.\1/g' "$x"; done
[ $no_ogv ] &&
rm -rf copyparty/web/deps/{dynamicaudio,ogv}* copyparty/web/browser.js
echo creating tar
args=(--owner=1000 --group=1000)
[ "$OSTYPE" = msys ] &&
args=()
tar -cf tar "${args[@]}" --numeric-owner copyparty jinja2
echo compressing tar
# detect best level; bzip2 -7 is usually better than -9
for n in {2..9}; do cp tar t.$n; bzip2 -$n t.$n & done; wait; mv -v $(ls -1S t.*.bz2 | tail -n 1) tar.bz2
for n in {2..9}; do cp tar t.$n; xz -ze$n t.$n & done; wait; mv -v $(ls -1S t.*.xz | tail -n 1) tar.xz
rm t.*
echo creating unix sfx
(
sed "s/PACK_TS/$ts/; s/PACK_HTS/$hts/; s/CPP_VER/$ver/" <../scripts/sfx.sh |
grep -E '^sfx_eof$' -B 9001;
cat tar.xz
) >$sfx_out.sh
echo creating generic sfx
python ../scripts/sfx.py --sfx-make tar.bz2 $ver $ts
mv sfx.out $sfx_out.py
chmod 755 $sfx_out.*
printf "done:\n"
printf " %s\n" "$(realpath $sfx_out)."{sh,py}
# rm -rf *
# -rw-r--r-- 1 ed ed 811271 May 5 14:35 tar.bz2
# -rw-r--r-- 1 ed ed 732016 May 5 14:35 tar.xz
# -rwxr-xr-x 1 ed ed 830425 May 5 14:35 copyparty-sfx.py*
# -rwxr-xr-x 1 ed ed 734088 May 5 14:35 copyparty-sfx.sh*
# -rwxr-xr-x 1 ed ed 799690 May 5 14:45 copyparty-sfx.py*
# -rwxr-xr-x 1 ed ed 735004 May 5 14:45 copyparty-sfx.sh*
# time pigz -11 -J 34 -I 5730 < tar > tar.gz.5730
# real 8m50.622s
# user 33m9.821s
# -rw-r--r-- 1 ed ed 1136640 May 5 14:50 tar
# -rw-r--r-- 1 ed ed 296334 May 5 14:50 tar.bz2
# -rw-r--r-- 1 ed ed 324705 May 5 15:01 tar.gz.5730
# -rw-r--r-- 1 ed ed 257208 May 5 14:50 tar.xz

167
scripts/sfx.py
View file

@ -2,16 +2,7 @@
# coding: utf-8
from __future__ import print_function, unicode_literals
import re
import os
import sys
import time
import signal
import shutil
import tarfile
import hashlib
import platform
import tempfile
import re, os, sys, stat, time, shutil, tarfile, hashlib, platform, tempfile
import subprocess as sp
"""
@ -20,7 +11,7 @@ run me with any version of python, i will unpack and run copyparty
(but please don't edit this file with a text editor
since that would probably corrupt the binary stuff at the end)
there is zero binaries! just plaintext python scripts all the way down
there's zero binaries! just plaintext python scripts all the way down
so you can easily unpack the archive and inspect it for shady stuff
the archive data is attached after the b"\n# eof\n" archive marker,
@ -29,12 +20,13 @@ the archive data is attached after the b"\n# eof\n" archive marker,
b"\n# " decodes to b""
"""
# metadata set when building the sfx
# set by make-sfx.sh
VER = None
SIZE = None
CKSUM = None
STAMP = None
PY2 = sys.version_info[0] == 2
sys.dont_write_bytecode = True
me = os.path.abspath(os.path.realpath(__file__))
@ -46,7 +38,7 @@ def eprint(*args, **kwargs):
def msg(*args, **kwargs):
if args:
args = ["[SFX] {}".format(args[0])] + list(args[1:])
args = ["[SFX]", args[0]] + list(args[1:])
eprint(*args, **kwargs)
@ -146,7 +138,7 @@ def testchk(cdata):
msg(txt)
def encode(data, size, cksum, ver):
def encode(data, size, cksum, ver, ts):
"""creates a new sfx; `data` should yield bufs to attach"""
nin = 0
nout = 0
@ -169,7 +161,7 @@ def encode(data, size, cksum, ver):
["VER", '"' + ver + '"'],
["SIZE", size],
["CKSUM", '"' + cksum + '"'],
["STAMP", int(time.time())],
["STAMP", ts],
]:
v1 = "\n{} = None\n".format(k)
v2 = "\n{} = {}\n".format(k, v)
@ -190,10 +182,10 @@ def encode(data, size, cksum, ver):
msg("wrote {:x}H bytes ({:x}H after encode)".format(nin, nout))
def makesfx(tar_src, ver):
def makesfx(tar_src, ver, ts):
sz = os.path.getsize(tar_src)
cksum = hashfile(tar_src)
encode(yieldfile(tar_src), sz, cksum, ver)
encode(yieldfile(tar_src), sz, cksum, ver, ts)
# skip 0
@ -261,7 +253,9 @@ def read_py(binp):
]
p = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE)
ver, _ = p.communicate()
return ver.split(b" ")[:3], p.returncode == 0
ver = ver.decode("utf-8").split(" ")[:3]
ver = [int(x) if x.isdigit() else 0 for x in ver]
return ver, p.returncode == 0
def get_pys():
@ -277,9 +271,9 @@ def get_pys():
ret = []
for binp in hits.values():
msg("testing", binp)
ver, chk = read_py(binp)
ret.append([chk, ver, binp])
msg("\t".join(str(x) for x in ret[-1]))
return ret
@ -300,29 +294,21 @@ def hashfile(fn):
def unpack():
"""unpacks the tar yielded by `data`"""
tag = "copyparty-{}".format(STAMP)
tmp = tempfile.gettempdir()
name = "pe-copyparty"
withpid = "{}.{}".format(name, os.getpid())
top = tempfile.gettempdir()
final = os.path.join(top, name)
mine = os.path.join(top, withpid)
tar = os.path.join(mine, "tar")
tag_mine = os.path.join(mine, "v" + str(STAMP))
tag_final = os.path.join(final, "v" + str(STAMP))
for fn in os.listdir(tmp):
if fn.startswith("copyparty-") and fn != tag:
try:
old = os.path.join(tmp, fn)
shutil.rmtree(old)
except:
pass
if os.path.exists(tag_final):
msg("found early")
return final
tmp = os.path.join(tmp, tag)
tar = os.path.join(tmp, "tar")
ok = os.path.join(tmp, "ok")
if os.path.exists(ok):
return tmp
if os.path.exists(tmp):
shutil.rmtree(tmp)
os.mkdir(tmp)
nwrite = 0
os.mkdir(mine)
with open(tar, "wb") as f:
for buf in get_payload():
nwrite += len(buf)
@ -338,14 +324,44 @@ def unpack():
raise Exception(t)
with tarfile.open(tar, "r:bz2") as tf:
tf.extractall(tmp)
tf.extractall(mine)
os.remove(tar)
with open(ok, "wb") as f:
with open(tag_mine, "wb") as f:
f.write(b"h\n")
if os.path.exists(tag_final):
msg("found late")
return final
try:
if os.path.islink(final):
os.remove(final)
else:
shutil.rmtree(final)
except:
pass
return tmp
try:
os.symlink(mine, final)
except:
try:
os.rename(mine, final)
except:
msg("reloc fail,", mine)
return mine
for fn in os.listdir(top):
if fn.startswith(name) and fn not in [name, withpid]:
try:
old = os.path.join(top, fn)
if time.time() - os.path.getmtime(old) > 10:
shutil.rmtree(old)
except:
pass
return final
def get_payload():
@ -402,50 +418,29 @@ def get_payload():
def confirm():
msg()
msg("*** hit enter to exit ***")
try:
raw_input()
except NameError:
input()
raw_input() if PY2 else input()
def run(tmp, py):
msg("OK")
msg("will use:", py)
msg("bound to:", tmp, "\n")
msg("bound to:", tmp)
fp_py = os.path.join(tmp, "py")
with open(fp_py, "wb") as f:
f.write(py.encode("utf-8") + b"\n")
env = os.environ.copy()
try:
libs = "{}:{}".format(tmp, env["PYTHONPATH"])
except:
libs = tmp
# avoid loading ./copyparty.py
cmd = [
py,
"-c",
'import sys, runpy; sys.path.insert(0, r"'
+ tmp
+ '"); runpy.run_module("copyparty", run_name="__main__")',
] + list(sys.argv[1:])
env[str("PYTHONPATH")] = str(libs)
# skip 1
if False:
# mingw64 py3.8.2 doesn't emit any prints without -u
env[str("PYTHONUNBUFFERED")] = str("ja")
# it also doesn't deal with ^C and none of this helps
def orz(sig, frame):
p.terminate()
signal.signal(signal.SIGINT, orz)
while True:
try:
time.sleep(9001)
except:
p.terminate()
break
# skip 0
cmd = [py, "-m", "copyparty"] + list(sys.argv[1:])
p = sp.Popen([str(x) for x in cmd], env=env)
msg("\n", cmd, "\n")
p = sp.Popen(str(x) for x in cmd)
try:
p.wait()
except:
@ -458,11 +453,12 @@ def run(tmp, py):
def main():
os.system("")
sysver = str(sys.version).replace("\n", "\n" + " " * 18)
pktime = time.strftime("%Y-%m-%d, %H:%M:%S", time.gmtime(STAMP))
os.system("")
msg()
msg(" this is: copyparty", VER)
msg(" packed at:", time.strftime("%Y-%m-%d, %H:%M:%S UTC", time.gmtime(STAMP)))
msg(" packed at:", pktime, "UTC,", STAMP)
msg("archive is:", me)
msg("python bin:", sys.executable)
msg("python ver:", platform.python_implementation(), sysver)
@ -477,16 +473,14 @@ def main():
# skip 1
if arg == "--sfx-testgen":
return encode(testptn(), 1, "x", "x")
return encode(testptn(), 1, "x", "x", 1)
if arg == "--sfx-testchk":
return testchk(get_payload())
if arg == "--sfx-make":
tar, ver = sys.argv[2:]
return makesfx(tar, ver)
# https://docs.microsoft.com/en-us/windows/win32/shell/knownfolderid?redirectedfrom=MSDN
tar, ver, ts = sys.argv[2:]
return makesfx(tar, ver, ts)
# skip 0
@ -495,13 +489,18 @@ def main():
if os.path.exists(fp_py):
with open(fp_py, "rb") as f:
py = f.read().decode("utf-8").rstrip()
return run(tmp, py)
return run(tmp, py)
pys = get_pys()
pys.sort(reverse=True)
j2, ver, py = pys[0]
if j2:
shutil.rmtree(os.path.join(tmp, "jinja2"))
try:
os.rename(os.path.join(tmp, "jinja2"), os.path.join(tmp, "x.jinja2"))
except:
pass
return run(tmp, py)
msg("\n could not find jinja2; will use py2 + the bundled version\n")

72
scripts/sfx.sh Normal file
View file

@ -0,0 +1,72 @@
# use current/default shell
set -e
dir="$(
printf '%s\n' "$TMPDIR" /tmp |
awk '/./ {print; exit}'
)/pe-copyparty"
[ -e "$dir/vPACK_TS" ] || (
printf '\033[36munpacking copyparty vCPP_VER (sfx-PACK_HTS)\033[1;30m\n\n'
mkdir -p "$dir.$$"
ofs=$(awk '$0=="sfx_eof" {print NR+1; exit}' < "$0")
[ -z "$ofs" ] && {
printf '\033[31mabort: could not find SFX boundary\033[0m\n'
exit 1
}
tail -n +$ofs "$0" | tar -JxC "$dir.$$"
ln -nsf "$dir.$$" "$dir"
printf '\033[0m'
now=$(date -u +%s)
for d in "$dir".*; do
ts=$(stat -c%Y -- "$d" 2>/dev/null) ||
ts=$(stat -f %m%n -- "$d" 2>/dev/null)
[ $((now-ts)) -gt 300 ] &&
rm -rf "$d"
done
echo h > "$dir/vPACK_TS"
) >&2 || exit 1
# detect available pythons
(IFS=:; for d in $PATH; do
printf '%s\n' "$d"/python* "$d"/pypy* | tac;
done) | grep -E '(python|pypy)[0-9\.-]*$' > $dir/pys || true
# see if we made a choice before
[ -z "$pybin" ] && pybin="$(cat $dir/py 2>/dev/null || true)"
# otherwise find a python with jinja2
[ -z "$pybin" ] && pybin="$(cat $dir/pys | while IFS= read -r _py; do
printf '\033[1;30mlooking for jinja2 in [%s]\033[0m\n' "$_py" >&2
$_py -c 'import jinja2' 2>/dev/null || continue
printf '%s\n' "$_py"
mv $dir/{,x.}jinja2
break
done)"
# otherwise find python2 (bundled jinja2 is way old)
[ -z "$pybin" ] && {
printf '\033[0;33mcould not find jinja2; will use py2 + the bundled version\033[0m\n' >&2
pybin="$(cat $dir/pys | while IFS= read -r _py; do
printf '\033[1;30mtesting if py2 [%s]\033[0m\n' "$_py" >&2
_ver=$($_py -c 'import sys; sys.stdout.write(str(sys.version_info[0]))' 2>/dev/null) || continue
[ $_ver = 2 ] || continue
printf '%s\n' "$_py"
break
done)"
}
[ -z "$pybin" ] && {
printf '\033[1;31m\ncould not find a python with jinja2 installed; please do one of these:\n\n pip install --user jinja2\n\n install python2\033[0m\n\n' >&2
exit 1
}
printf '\033[1;30musing [%s]. you can reset with this:\n rm -rf %s*\033[0m\n\n' "$pybin" "$dir"
printf '%s\n' "$pybin" > $dir/py
PYTHONPATH=$dir exec "$pybin" -m copyparty "$@"
sfx_eof