mirrors/copyparty
1
0
Fork 0
mirror of https://github.com/9001/copyparty.git synced 2025-08-17 09:02:15 -06:00
Code Issues Actions Packages Projects Releases Wiki Activity

add pyoxidizer (windows-only)

Browse source
This commit is contained in:
ed 2022-09-10 17:33:04 +02:00
parent fbc8ee15da
commit e430b2567a
11 changed files with 276 additions and 80 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

51
copyparty/__init__.py
View file

@ -7,8 +7,6 @@ import sys
import time
try:
from collections.abc import Callable
from typing import TYPE_CHECKING, Any
except:
TYPE_CHECKING = False
@ -39,56 +37,15 @@ except:
CORES = (os.cpu_count() if hasattr(os, "cpu_count") else 0) or 2
def get_unixdir() -> str:
paths: list[tuple[Callable[..., str], str]] = [
(os.environ.get, "XDG_CONFIG_HOME"),
(os.path.expanduser, "~/.config"),
(os.environ.get, "TMPDIR"),
(os.environ.get, "TEMP"),
(os.environ.get, "TMP"),
(unicode, "/tmp"),
]
for chk in [os.listdir, os.mkdir]:
for pf, pa in paths:
try:
p = pf(pa)
# print(chk.__name__, p, pa)
if not p or p.startswith("~"):
continue
p = os.path.normpath(p)
chk(p) # type: ignore
p = os.path.join(p, "copyparty")
if not os.path.isdir(p):
os.mkdir(p)
return p
except:
pass
raise Exception("could not find a writable path for config")
class EnvParams(object):
def __init__(self) -> None:
self.t0 = time.time()
self.mod = os.path.dirname(os.path.realpath(__file__))
if self.mod.endswith("__init__"):
self.mod = os.path.dirname(self.mod)
if sys.platform == "win32":
self.cfg = os.path.normpath(os.environ["APPDATA"] + "/copyparty")
elif sys.platform == "darwin":
self.cfg = os.path.expanduser("~/Library/Preferences/copyparty")
else:
self.cfg = get_unixdir()
self.cfg = self.cfg.replace("\\", "/")
self.mod = None
self.cfg = None
try:
os.makedirs(self.cfg)
self.ox = sys.oxidized
except:
if not os.path.isdir(self.cfg):
raise
self.ox = False
E = EnvParams()

101
copyparty/__main__.py Normal file → Executable file
View file

@ -41,6 +41,7 @@ try:
from types import FrameType
from typing import Any, Optional
from collections.abc import Callable
except:
pass
@ -131,6 +132,79 @@ def warn(msg: str) -> None:
lprint("\033[1mwarning:\033[0;33m {}\033[0m\n".format(msg))
def init_E() -> None:
# __init__ runs 18 times when oxidized; do expensive stuff here
def get_unixdir() -> str:
paths: list[tuple[Callable[..., str], str]] = [
(os.environ.get, "XDG_CONFIG_HOME"),
(os.path.expanduser, "~/.config"),
(os.environ.get, "TMPDIR"),
(os.environ.get, "TEMP"),
(os.environ.get, "TMP"),
(unicode, "/tmp"),
]
for chk in [os.listdir, os.mkdir]:
for pf, pa in paths:
try:
p = pf(pa)
# print(chk.__name__, p, pa)
if not p or p.startswith("~"):
continue
p = os.path.normpath(p)
chk(p) # type: ignore
p = os.path.join(p, "copyparty")
if not os.path.isdir(p):
os.mkdir(p)
return p
except:
pass
raise Exception("could not find a writable path for config")
def _unpack() -> str:
import atexit
import tarfile
import tempfile
from importlib.resources import open_binary
td = tempfile.TemporaryDirectory(prefix="")
atexit.register(td.cleanup)
tdn = td.name
with open_binary("copyparty", "z.tar") as tgz:
with tarfile.open(fileobj=tgz) as tf:
tf.extractall(tdn)
return tdn
try:
E.mod = os.path.dirname(os.path.realpath(__file__))
if E.mod.endswith("__init__"):
E.mod = os.path.dirname(E.mod)
except:
if not E.ox:
raise
E.mod = _unpack()
if sys.platform == "win32":
E.cfg = os.path.normpath(os.environ["APPDATA"] + "/copyparty")
elif sys.platform == "darwin":
E.cfg = os.path.expanduser("~/Library/Preferences/copyparty")
else:
E.cfg = get_unixdir()
E.cfg = E.cfg.replace("\\", "/")
try:
os.makedirs(E.cfg)
except:
if not os.path.isdir(E.cfg):
raise
def ensure_locale() -> None:
for x in [
"en_US.UTF-8",
@ -323,6 +397,21 @@ def disable_quickedit() -> None:
cmode(True, mode | 4)
def showlic() -> None:
p = os.path.join(E.mod, "COPYING.txt")
if not os.path.exists(p):
print("no relevant license info to display")
return
t = " licenses are only relevant to this EXE edition of copyparty, as they are a result of packaging by pyoxidizer"
print("the below" + t + ":\n")
with open(p, "rb") as f:
print(f.read().decode("utf-8", "replace"))
print("\nthe above" + t)
def run_argparse(argv: list[str], formatter: Any, retry: bool) -> argparse.Namespace:
ap = argparse.ArgumentParser(
formatter_class=formatter,
@ -488,6 +577,9 @@ def run_argparse(argv: list[str], formatter: Any, retry: bool) -> argparse.Names
ap2.add_argument("-mcr", metavar="SEC", type=int, default=60, help="md-editor mod-chk rate")
ap2.add_argument("--urlform", metavar="MODE", type=u, default="print,get", help="how to handle url-form POSTs; see --help-urlform")
ap2.add_argument("--wintitle", metavar="TXT", type=u, default="cpp @ $pub", help="window title, for example '$ip-10.1.2.' or '$ip-'")
if E.ox:
ap2.add_argument("--license", action="store_true", help="show licenses and exit")
ap2.add_argument("--version", action="store_true", help="show versions and exit")
ap2 = ap.add_argument_group('upload options')
ap2.add_argument("--dotpart", action="store_true", help="dotfile incomplete uploads, hiding them from clients unless -ed")
@ -680,6 +772,7 @@ def main(argv: Optional[list[str]] = None) -> None:
if WINDOWS:
os.system("rem") # enables colors
init_E()
if argv is None:
argv = sys.argv
@ -695,6 +788,13 @@ def main(argv: Optional[list[str]] = None) -> None:
)
lprint(f)
if "--version" in argv:
sys.exit(0)
if "--license" in argv and E.ox:
showlic()
sys.exit(0)
ensure_locale()
if HAVE_SSL:
ensure_cert()
@ -733,6 +833,7 @@ def main(argv: Optional[list[str]] = None) -> None:
lprint("\n[ {} ]:\n{}\n".format(fmtr, min_ex()))
assert al
al.E = E # __init__ is not shared when oxidized
if WINDOWS and not al.keep_qem:
try:

2
copyparty/ftpd.py
View file

@ -356,7 +356,7 @@ class Ftpd(object):
print(t.format(sys.executable))
sys.exit(1)
h1.certfile = os.path.join(E.cfg, "cert.pem")
h1.certfile = os.path.join(self.args.E.cfg, "cert.pem")
h1.tls_control_required = True
h1.tls_data_required = True

11
copyparty/httpcli.py
View file

@ -24,7 +24,7 @@ try:
except:
pass
from .__init__ import ANYWIN, PY2, TYPE_CHECKING, E, unicode
from .__init__ import ANYWIN, PY2, TYPE_CHECKING, EnvParams, unicode
from .authsrv import VFS # typechk
from .bos import bos
from .star import StreamTar
@ -103,6 +103,7 @@ class HttpCli(object):
self.ip = conn.addr[0]
self.addr: tuple[str, int] = conn.addr
self.args = conn.args # mypy404
self.E: EnvParams = self.args.E
self.asrv = conn.asrv # mypy404
self.ico = conn.ico # mypy404
self.thumbcli = conn.thumbcli # mypy404
@ -553,7 +554,7 @@ class HttpCli(object):
if self.vpath.startswith(".cpr/ico/"):
return self.tx_ico(self.vpath.split("/")[-1], exact=True)
static_path = os.path.join(E.mod, "web/", self.vpath[5:])
static_path = os.path.join(self.E.mod, "web/", self.vpath[5:])
return self.tx_file(static_path)
if "cf_challenge" in self.uparam:
@ -1839,7 +1840,7 @@ class HttpCli(object):
mime, ico = self.ico.get(ext, not exact)
dt = datetime.utcfromtimestamp(E.t0)
dt = datetime.utcfromtimestamp(self.E.t0)
lm = dt.strftime("%a, %d %b %Y %H:%M:%S GMT")
self.reply(ico, mime=mime, headers={"Last-Modified": lm})
return True
@ -1852,7 +1853,7 @@ class HttpCli(object):
return self.tx_404(True)
tpl = "mde" if "edit2" in self.uparam else "md"
html_path = os.path.join(E.mod, "web", "{}.html".format(tpl))
html_path = os.path.join(self.E.mod, "web", "{}.html".format(tpl))
template = self.j2j(tpl)
st = bos.stat(fs_path)
@ -1867,7 +1868,7 @@ class HttpCli(object):
for c, v in [(b"&", 4), (b"<", 3), (b">", 3)]:
sz_md += (len(buf) - len(buf.replace(c, b""))) * v
file_ts = max(ts_md, ts_html, E.t0)
file_ts = max(ts_md, ts_html, self.E.t0)
file_lastmod, do_send = self._chk_lastmod(file_ts)
self.out_headers["Last-Modified"] = file_lastmod
self.out_headers.update(NO_CACHE)

5
copyparty/httpconn.py
View file

@ -15,7 +15,7 @@ except:
HAVE_SSL = False
from . import util as Util
from .__init__ import TYPE_CHECKING, E
from .__init__ import TYPE_CHECKING, EnvParams
from .authsrv import AuthSrv # typechk
from .httpcli import HttpCli
from .ico import Ico
@ -50,6 +50,7 @@ class HttpConn(object):
self.mutex: threading.Lock = hsrv.mutex # mypy404
self.args: argparse.Namespace = hsrv.args # mypy404
self.E: EnvParams = self.args.E
self.asrv: AuthSrv = hsrv.asrv # mypy404
self.cert_path = hsrv.cert_path
self.u2fh: Util.FHC = hsrv.u2fh # mypy404
@ -91,7 +92,7 @@ class HttpConn(object):
return self.log_src
def respath(self, res_name: str) -> str:
return os.path.join(E.mod, "web", res_name)
return os.path.join(self.E.mod, "web", res_name)
def log(self, msg: str, c: Union[int, str] = 0) -> None:
self.log_func(self.log_src, msg, c)

14
copyparty/httpsrv.py
View file

@ -28,7 +28,7 @@ except ImportError:
)
sys.exit(1)
from .__init__ import MACOS, TYPE_CHECKING, E
from .__init__ import MACOS, TYPE_CHECKING, EnvParams
from .bos import bos
from .httpconn import HttpConn
from .util import FHC, min_ex, shut_socket, spack, start_log_thrs, start_stackmon
@ -52,6 +52,7 @@ class HttpSrv(object):
self.broker = broker
self.nid = nid
self.args = broker.args
self.E: EnvParams = self.args.E
self.log = broker.log
self.asrv = broker.asrv
@ -81,14 +82,15 @@ class HttpSrv(object):
self.cb_v = ""
env = jinja2.Environment()
env.loader = jinja2.FileSystemLoader(os.path.join(E.mod, "web"))
env.loader = jinja2.FileSystemLoader(os.path.join(self.E.mod, "web"))
self.j2 = {
x: env.get_template(x + ".html")
for x in ["splash", "browser", "browser2", "msg", "md", "mde", "cf"]
}
self.prism = os.path.exists(os.path.join(E.mod, "web", "deps", "prism.js.gz"))
zs = os.path.join(self.E.mod, "web", "deps", "prism.js.gz")
self.prism = os.path.exists(zs)
cert_path = os.path.join(E.cfg, "cert.pem")
cert_path = os.path.join(self.E.cfg, "cert.pem")
if bos.path.exists(cert_path):
self.cert_path = cert_path
else:
@ -354,9 +356,9 @@ class HttpSrv(object):
if time.time() - self.cb_ts < 1:
return self.cb_v
v = E.t0
v = self.E.t0
try:
with os.scandir(os.path.join(E.mod, "web")) as dh:
with os.scandir(os.path.join(self.E.mod, "web")) as dh:
for fh in dh:
inf = fh.stat()
v = max(v, inf.st_mtime)

8
copyparty/mtag.py
View file

@ -8,7 +8,7 @@ import shutil
import subprocess as sp
import sys
from .__init__ import PY2, WINDOWS, unicode
from .__init__ import E, PY2, WINDOWS, unicode
from .bos import bos
from .util import REKOBO_LKEY, fsenc, min_ex, retchk, runcmd, uncyg
@ -511,11 +511,15 @@ class MTag(object):
if not bos.path.isfile(abspath):
return {}
env = os.environ.copy()
try:
pypath = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
zsl = [str(pypath)] + [str(x) for x in sys.path if x]
pypath = str(os.pathsep.join(zsl))
env = os.environ.copy()
env["PYTHONPATH"] = pypath
except:
if not E.ox:
raise
ret: dict[str, Any] = {}
for tagname, parser in sorted(parsers.items(), key=lambda x: (x[1].pri, x[0])):

5
copyparty/svchub.py
View file

@ -24,7 +24,7 @@ try:
except:
pass
from .__init__ import ANYWIN, MACOS, PY2, VT100, WINDOWS, E, unicode
from .__init__ import ANYWIN, MACOS, PY2, VT100, WINDOWS, EnvParams, unicode
from .authsrv import AuthSrv
from .mtag import HAVE_FFMPEG, HAVE_FFPROBE
from .tcpsrv import TcpSrv
@ -55,6 +55,7 @@ class SvcHub(object):
def __init__(self, args: argparse.Namespace, argv: list[str], printed: str) -> None:
self.args = args
self.argv = argv
self.E: EnvParams = args.E
self.logf: Optional[typing.TextIO] = None
self.logf_base_fn = ""
self.stop_req = False
@ -264,7 +265,7 @@ class SvcHub(object):
msg = "[+] opened logfile [{}]\n".format(fn)
printed += msg
lh.write("t0: {:.3f}\nargv: {}\n\n{}".format(E.t0, " ".join(argv), printed))
lh.write("t0: {:.3f}\nargv: {}\n\n{}".format(self.E.t0, " ".join(argv), printed))
self.logf = lh
self.logf_base_fn = base_fn
print(msg, end="")

52
docs/pyoxidizer.txt Normal file
View file

@ -0,0 +1,52 @@
pyoxidizer doesn't crosscompile yet so need to build in a windows vm,
luckily possible to do mostly airgapped (https-proxy for crates)
none of this is version-specific but doing absolute links just in case
(only exception is py3.8 which is the final win7 ver)
# deps (download on linux host):
https://www.python.org/ftp/python/3.10.7/python-3.10.7-amd64.exe
https://github.com/indygreg/PyOxidizer/releases/download/pyoxidizer%2F0.22.0/pyoxidizer-0.22.0-x86_64-pc-windows-msvc.zip
https://github.com/upx/upx/releases/download/v3.96/upx-3.96-win64.zip
https://static.rust-lang.org/dist/rust-1.61.0-x86_64-pc-windows-msvc.msi
https://github.com/indygreg/python-build-standalone/releases/download/20220528/cpython-3.8.13%2B20220528-i686-pc-windows-msvc-static-noopt-full.tar.zst
# need cl.exe, prefer 2017 -- download on linux host:
https://visualstudio.microsoft.com/downloads/?q=build+tools
https://docs.microsoft.com/en-us/visualstudio/releases/2022/release-history#release-dates-and-build-numbers
https://aka.ms/vs/15/release/vs_buildtools.exe # 2017
https://aka.ms/vs/16/release/vs_buildtools.exe # 2019
https://aka.ms/vs/17/release/vs_buildtools.exe # 2022
https://docs.microsoft.com/en-us/visualstudio/install/workload-component-id-vs-build-tools?view=vs-2017
# use disposable w10 vm to prep offline installer; xfer to linux host with firefox to copyparty
vs_buildtools-2017.exe --add Microsoft.VisualStudio.Workload.MSBuildTools --add Microsoft.VisualStudio.Workload.VCTools --add Microsoft.VisualStudio.Component.Windows10SDK.17763 --layout c:\msbt2017 --lang en-us
# need two proxies on host; s5s or ssh for msys2(socks5), and tinyproxy for rust(http)
UP=- python3 socks5server.py 192.168.123.1 4321
ssh -vND 192.168.123.1:4321 localhost
git clone https://github.com/tinyproxy/tinyproxy.git
./autogen.sh
./configure --prefix=/home/ed/pe/tinyproxy
make -j24 install
printf '%s\n' >cfg "Port 4380" "Listen 192.168.123.1"
./tinyproxy -dccfg
https://github.com/msys2/msys2-installer/releases/download/2022-09-04/msys2-x86_64-20220904.exe
export all_proxy=socks5h://192.168.123.1:4321
# if chat dies after auth (2 messages) it probably failed dns, note the h in socks5h to tunnel dns
pacman -Syuu
pacman -S git patch mingw64/mingw-w64-x86_64-zopfli
cd /c && curl -k https://192.168.123.1:3923/ro/ox/msbt2017/?tar | tar -xv
first install certs from msbt/certificates then admin-cmd `vs_buildtools.exe --noweb`,
default selection (vc++2017-v15.9-v14.16, vc++redist, vc++bt-core) += win10sdk (for io.h)
install rust without documentation, python 3.10, put upx and pyoxidizer into ~/bin,
[cmd.exe] python -m pip install --user -U wheel-0.37.1.tar.gz strip-hints-0.1.10.tar.gz
p=192.168.123.1:4380; export https_proxy=$p; export http_proxy=$p
# and with all of the one-time-setup out of the way,
mkdir /c/d; cd /c/d && curl -k https://192.168.123.1:3923/cpp/gb?pw=wark > gb && git clone gb copyparty
cd /c/d/copyparty/ && curl -k https://192.168.123.1:3923/cpp/patch?pw=wark | patch -p1
cd /c/d/copyparty/scripts && CARGO_HTTP_CHECK_REVOKE=false PATH=/c/Users/$USER/AppData/Local/Programs/Python/Python310:/c/Users/$USER/bin:"$(cygpath "C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\VC\Tools\MSVC\14.16.27023\bin\Hostx86\x86"):$PATH" ./make-sfx.sh ox ultra

47
pyoxidizer.bzl Normal file
View file

@ -0,0 +1,47 @@
# builds win7-i386 exe on win10-ltsc-1809(17763.316)
# see docs/pyoxidizer.txt
def make_exe():
dist = default_python_distribution(flavor="standalone_static", python_version="3.8")
policy = dist.make_python_packaging_policy()
policy.allow_files = True
policy.allow_in_memory_shared_library_loading = True
#policy.bytecode_optimize_level_zero = True
#policy.include_distribution_sources = False # error instantiating embedded Python interpreter: during initializing Python main: init_fs_encoding: failed to get the Python codec of the filesystem encoding
policy.include_distribution_resources = False
policy.include_non_distribution_sources = False
policy.include_test = False
python_config = dist.make_python_interpreter_config()
#python_config.module_search_paths = ["$ORIGIN/lib"]
python_config.run_module = "copyparty"
exe = dist.to_python_executable(
name="copyparty",
config=python_config,
packaging_policy=policy,
)
exe.windows_runtime_dlls_mode = "never"
exe.windows_subsystem = "console"
exe.add_python_resources(exe.read_package_root(
path="sfx",
packages=[
"copyparty",
"jinja2",
"markupsafe",
"pyftpdlib",
]
))
return exe
def make_embedded_resources(exe):
return exe.to_embedded_resources()
def make_install(exe):
files = FileManifest()
files.add_python_resource("copyparty", exe)
return files
register_target("exe", make_exe)
register_target("resources", make_embedded_resources, depends=["exe"], default_build_script=True)
register_target("install", make_install, depends=["exe"], default=True)
resolve_targets()

50
scripts/make-sfx.sh
View file

@ -12,6 +12,8 @@ help() { exec cat <<'EOF'
# `re` does a repack of an sfx which you already executed once
# (grabs files from the sfx-created tempdir), overrides `clean`
#
# `ox` builds a pyoxidizer exe instead of py
#
# `gz` creates a gzip-compressed python sfx instead of bzip2
#
# `lang` limits which languages/translations to include,
@ -56,6 +58,10 @@ gtar=$(command -v gtar || command -v gnutar) || true
gawk=$(command -v gawk || command -v gnuawk || command -v awk)
awk() { $gawk "$@"; }
targs=(--owner=1000 --group=1000)
[ "$OSTYPE" = msys ] &&
targs=()
pybin=$(command -v python3 || command -v python) || {
echo need python
exit 1
@ -79,12 +85,14 @@ while [ ! -z "$1" ]; do
case $1 in
clean) clean=1 ; ;;
re) repack=1 ; ;;
ox) use_ox=1 ; ;;
gz) use_gz=1 ; ;;
no-fnt) no_fnt=1 ; ;;
no-hl) no_hl=1 ; ;;
no-dd) no_dd=1 ; ;;
no-cm) no_cm=1 ; ;;
fast) zopf= ; ;;
ultra) ultra=1 ; ;;
lang) shift;langs="$1"; ;;
*) help ; ;;
esac
@ -303,8 +311,8 @@ rm have
tmv "$f"
done
[ $repack ] || {
# uncomment
[ ! $repack ] && [ ! $use_ox ] && {
# uncomment; oxidized drops 45 KiB but becomes undebuggable
find | grep -E '\.py$' |
grep -vE '__version__' |
tr '\n' '\0' |
@ -348,9 +356,9 @@ find | grep -E '\.(js|html)$' | while IFS= read -r f; do
done
gzres() {
command -v pigz && [ $zopf ] &&
pk="pigz -11 -I $zopf" ||
pk='gzip'
[ $zopf ] && command -v zopfli && pk="zopfli --i$zopf"
[ $zopf ] && command -v pigz && pk="pigz -11 -I $zopf"
[ -z "$pk" ] && pk='gzip'
np=$(nproc)
echo "$pk #$np"
@ -399,6 +407,32 @@ nf=$(ls -1 "$zdir"/arc.* | wc -l)
}
[ $use_ox ] && {
tgt=x86_64-pc-windows-msvc
tgt=i686-pc-windows-msvc # 2M smaller (770k after upx)
bdir=build/$tgt/release/install/copyparty
t="res web"
cp -pv ../$bdir/COPYING.txt copyparty/ &&
t="$t COPYING.txt" ||
echo "copying.txt 404 pls rebuild"
mv ftp/* j2/* copyparty/vend/* .
rm -rf ftp j2 py2 copyparty/vend
(cd copyparty; tar -cvf z.tar $t; rm -rf $t)
cd ..
pyoxidizer build --release --target-triple $tgt
mv $bdir/copyparty.exe dist/
cp -pv "$(cygpath 'C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\VC\Redist\MSVC\14.16.27012\x86\Microsoft.VC141.CRT\vcruntime140.dll')" dist/
dist/copyparty.exe --version
cp -pv dist/copyparty{,.orig}.exe
[ $ultra ] && a="--best --lzma" || a=-1
/bin/time -f %es upx $a dist/copyparty.exe >/dev/null
ls -al dist/copyparty{,.orig}.exe
exit 0
}
echo gen tarlist
for d in copyparty j2 ftp py2; do find $d -type f; done | # strip_hints
sed -r 's/(.*)\.(.*)/\2 \1/' | LC_ALL=C sort |
@ -414,11 +448,7 @@ done
[ $n -eq 50 ] && exit
echo creating tar
args=(--owner=1000 --group=1000)
[ "$OSTYPE" = msys ] &&
args=()
tar -cf tar "${args[@]}" --numeric-owner -T list
tar -cf tar "${targs[@]}" --numeric-owner -T list
pc=bzip2
pe=bz2