diff --git a/copyparty/__init__.py b/copyparty/__init__.py index 28dbc8ba..6ff96050 100644 --- a/copyparty/__init__.py +++ b/copyparty/__init__.py @@ -16,7 +16,9 @@ except: TYPE_CHECKING = False if True: - from typing import Any, Callable + from types import ModuleType + + from typing import Any, Callable, Optional PY2 = sys.version_info < (3,) PY36 = sys.version_info > (3, 6) @@ -51,10 +53,63 @@ try: except: CORES = (os.cpu_count() if hasattr(os, "cpu_count") else 0) or 2 +# all embedded resources to be retrievable over http +zs = """ +web/a/partyfuse.py +web/a/u2c.py +web/a/webdav-cfg.bat +web/baguettebox.js +web/browser.css +web/browser.html +web/browser.js +web/browser2.html +web/cf.html +web/copyparty.gif +web/dd/2.png +web/dd/3.png +web/dd/4.png +web/dd/5.png +web/deps/busy.mp3 +web/deps/easymde.css +web/deps/easymde.js +web/deps/marked.js +web/deps/mini-fa.css +web/deps/mini-fa.woff +web/deps/prism.css +web/deps/prism.js +web/deps/prismd.css +web/deps/scp.woff2 +web/deps/sha512.ac.js +web/deps/sha512.hw.js +web/md.css +web/md.html +web/md.js +web/md2.css +web/md2.js +web/mde.css +web/mde.html +web/mde.js +web/msg.css +web/msg.html +web/shares.css +web/shares.html +web/shares.js +web/splash.css +web/splash.html +web/splash.js +web/svcs.html +web/svcs.js +web/ui.css +web/up2k.js +web/util.js +web/w.hash.js +""" +RES = set(zs.strip().split("\n")) + class EnvParams(object): def __init__(self) -> None: - self.pkg = None + self.pkg: Optional[ModuleType] = None self.t0 = time.time() self.mod = "" self.cfg = "" diff --git a/copyparty/__main__.py b/copyparty/__main__.py index 56d6f443..60b94d51 100644 --- a/copyparty/__main__.py +++ b/copyparty/__main__.py @@ -58,6 +58,7 @@ from .util import ( b64enc, dedent, has_resource, + load_resource, min_ex, pybin, termsize, @@ -217,6 +218,7 @@ def init_E(EE: EnvParams) -> None: raise Exception("could not find a writable path for config") + assert __package__ # !rm E.pkg = sys.modules[__package__] E.mod = os.path.dirname(os.path.realpath(__file__)) if E.mod.endswith("__init__"): @@ -520,14 +522,18 @@ def sfx_tpoke(top: str): def showlic() -> None: - p = os.path.join(E.mod, "res", "COPYING.txt") - if not os.path.exists(p): + try: + with load_resource(E, "res/COPYING.txt") as f: + buf = f.read() + except: + buf = b"" + + if buf: + print(buf.decode("utf-8", "replace")) + else: print("no relevant license info to display") return - with open(p, "rb") as f: - print(f.read().decode("utf-8", "replace")) - def get_sects(): return [ @@ -1567,16 +1573,13 @@ def run_argparse( return ret -def main(argv: Optional[list[str]] = None, rsrc: Optional[str] = None) -> None: +def main(argv: Optional[list[str]] = None) -> None: time.strptime("19970815", "%Y%m%d") # python#7980 if WINDOWS: os.system("rem") # enables colors init_E(E) - if rsrc: # pyz - E.mod = rsrc - if argv is None: argv = sys.argv diff --git a/copyparty/cert.py b/copyparty/cert.py index 7618c876..dcba8d7e 100644 --- a/copyparty/cert.py +++ b/copyparty/cert.py @@ -7,7 +7,7 @@ import shutil import time from .__init__ import ANYWIN -from .util import Netdev, runcmd, wrename, wunlink +from .util import Netdev, load_resource, runcmd, wrename, wunlink HAVE_CFSSL = not os.environ.get("PRTY_NO_CFSSL") @@ -29,13 +29,15 @@ def ensure_cert(log: "RootLogger", args) -> None: i feel awful about this and so should they """ - cert_insec = os.path.join(args.E.mod, "res/insecure.pem") + with load_resource(args.E, "res/insecure.pem") as f: + cert_insec = f.read() cert_appdata = os.path.join(args.E.cfg, "cert.pem") if not os.path.isfile(args.cert): if cert_appdata != args.cert: raise Exception("certificate file does not exist: " + args.cert) - shutil.copy(cert_insec, args.cert) + with open(args.cert, "wb") as f: + f.write(cert_insec) with open(args.cert, "rb") as f: buf = f.read() @@ -50,7 +52,9 @@ def ensure_cert(log: "RootLogger", args) -> None: raise Exception(m + "private key must appear before server certificate") try: - if filecmp.cmp(args.cert, cert_insec): + with open(args.cert, "rb") as f: + active_cert = f.read() + if active_cert == cert_insec: t = "using default TLS certificate; https will be insecure:\033[36m {}" log("cert", t.format(args.cert), 3) except: @@ -151,14 +155,22 @@ def _gen_srv(log: "RootLogger", args, netdevs: dict[str, Netdev]): raise Exception("no useable cert found") expired = time.time() + args.crt_sdays * 60 * 60 * 24 * 0.5 > expiry - cert_insec = os.path.join(args.E.mod, "res/insecure.pem") + if expired: + raise Exception("old server-cert has expired") + for n in names: if n not in inf["sans"]: raise Exception("does not have {}".format(n)) - if expired: - raise Exception("old server-cert has expired") - if not filecmp.cmp(args.cert, cert_insec): + + with load_resource(args.E, "res/insecure.pem") as f: + cert_insec = f.read() + + with open(args.cert, "rb") as f: + active_cert = f.read() + + if active_cert and active_cert != cert_insec: return + except Exception as ex: log("cert", "will create new server-cert; {}".format(ex)) diff --git a/copyparty/httpcli.py b/copyparty/httpcli.py index 7abd37b5..4fa6f914 100644 --- a/copyparty/httpcli.py +++ b/copyparty/httpcli.py @@ -32,7 +32,7 @@ try: except: pass -from .__init__ import ANYWIN, PY2, TYPE_CHECKING, EnvParams, unicode +from .__init__ import ANYWIN, PY2, RES, TYPE_CHECKING, EnvParams, unicode from .__version__ import S_VERSION from .authsrv import VFS # typechk from .bos import bos @@ -67,8 +67,8 @@ from .util import ( get_df, get_spd, guess_mime, - gzip_orig_sz, gzip_file_orig_sz, + gzip_orig_sz, has_resource, hashcopy, hidedir, @@ -1097,13 +1097,17 @@ class HttpCli(object): if self.vpath == ".cpr/metrics": return self.conn.hsrv.metrics.tx(self) - static_path = os.path.join("web", self.vpath[5:]) - if static_path in self.conn.hsrv.statics: - return self.tx_res(static_path) + res_path = "web/" + self.vpath[5:] + if res_path in RES: + ap = os.path.join(self.E.mod, res_path) + if bos.path.exists(ap) or bos.path.exists(ap + ".gz"): + return self.tx_file(ap) + else: + return self.tx_res(res_path) - if not undot(static_path).startswith("web"): + if res_path != undot(res_path): t = "malicious user; attempted path traversal [{}] => [{}]" - self.log(t.format(self.vpath, static_path), 1) + self.log(t.format(self.vpath, res_path), 1) self.cbonk(self.conn.hsrv.gmal, self.req, "trav", "path traversal") self.tx_404() @@ -3415,6 +3419,7 @@ class HttpCli(object): self.args.s_wr_slp, not self.args.no_poll, ) + res.close() if remains > 0: logmsg += " \033[31m" + unicode(file_sz - remains) + "\033[0m" diff --git a/copyparty/httpsrv.py b/copyparty/httpsrv.py index 0e5710f6..bdec07c5 100644 --- a/copyparty/httpsrv.py +++ b/copyparty/httpsrv.py @@ -66,7 +66,6 @@ from .util import ( Magician, Netdev, NetMap, - absreal, build_netmap, has_resource, ipnorm, @@ -76,9 +75,7 @@ from .util import ( spack, start_log_thrs, start_stackmon, - stat_resource, ub64enc, - walk_resources, ) if TYPE_CHECKING: @@ -96,7 +93,7 @@ if not hasattr(socket, "AF_UNIX"): def load_jinja2_resource(E: EnvParams, name: str): - return load_resource(E, os.path.join("web", name), "r").read() + return load_resource(E, "web/" + name, "r").read() class HttpSrv(object): @@ -174,15 +171,12 @@ class HttpSrv(object): "cf", ] self.j2 = {x: env.get_template(x + ".html") for x in jn} - self.prism = has_resource(self.E, os.path.join("web", "deps", "prism.js.gz")) + self.prism = has_resource(self.E, "web/deps/prism.js.gz") self.ipa_nm = build_netmap(self.args.ipa) self.xff_nm = build_netmap(self.args.xff_src) self.xff_lan = build_netmap("lan") - self.statics: set[str] = set() - self._build_statics() - self.ptn_cc = re.compile(r"[\x00-\x1f]") self.ptn_hsafe = re.compile(r"[\x00-\x1f<>\"'&]") @@ -216,14 +210,6 @@ class HttpSrv(object): except: pass - def _build_statics(self) -> None: - for dp, _, df in walk_resources(self.E, "web"): - for fn in df: - ap = os.path.join(dp, fn) - self.statics.add(ap) - if ap.endswith(".gz"): - self.statics.add(ap[:-3]) - def set_netdevs(self, netdevs: dict[str, Netdev]) -> None: ips = set() for ip, _ in self.bound: @@ -543,20 +529,10 @@ class HttpSrv(object): v = self.E.t0 try: - for (base, dirs, files) in walk_resources(self.E, "web"): - inf = stat_resource(self.E, base) - if inf: + with os.scandir(os.path.join(self.E.mod, "web")) as dh: + for fh in dh: + inf = fh.stat() v = max(v, inf.st_mtime) - for d in dirs: - inf = stat_resource(self.E, os.path.join(base, d)) - if inf: - v = max(v, inf.st_mtime) - for f in files: - inf = stat_resource(self.E, os.path.join(base, e)) - if inf: - v = max(v, inf.st_mtime) - # only do top-level - break except: pass diff --git a/copyparty/util.py b/copyparty/util.py index 95e904f5..aa00d693 100644 --- a/copyparty/util.py +++ b/copyparty/util.py @@ -134,7 +134,7 @@ if True: # pylint: disable=using-constant-test from collections.abc import Callable, Iterable import typing - from typing import Any, Generator, Optional, Pattern, Protocol, Union + from typing import Any, Generator, IO, Optional, Pattern, Protocol, Union try: from typing import LiteralString @@ -3420,6 +3420,7 @@ def gzip_orig_sz(fn: str) -> int: with open(fsenc(fn), "rb") as f: return gzip_file_orig_sz(f) + def gzip_file_orig_sz(f) -> int: start = f.tell() f.seek(-4, 2) @@ -3582,144 +3583,95 @@ def _pkg_resource_exists(pkg: str, name: str) -> bool: except NotImplementedError: return False + def stat_resource(E: EnvParams, name: str): path = os.path.join(E.mod, name) if os.path.exists(path): return os.stat(fsenc(path)) return None -def has_resource(E: EnvParams, name: str): + +def _find_impresource_cold(E: EnvParams, name: str): + global _rescache_imp, _find_impresource + + assert impresources # !rm + try: + _rescache_imp = impresources.files(E.pkg) + except ImportError: + return None + + _find_impresource = _find_impresource_warm + return _find_impresource(E, name) + + +def _find_impresource_warm(E: EnvParams, name: str): + if not _rescache_imp: + return None + + try: + return _rescache_res[name] + except: + if len(_rescache_res) > 999: + _rescache_res.clear() + ret = _rescache_imp.joinpath(name) + _rescache_res[name] = ret + return ret + + +_find_impresource = _find_impresource_cold +_rescache_imp = None +_rescache_has = {} +_rescache_res = {} + + +def _has_resource(E: EnvParams, name: str): + try: + return _rescache_has[name] + except: + pass + + if len(_rescache_has) > 999: + _rescache_has.clear() + if impresources: - try: - resources = impresources.files(E.pkg) - except ImportError: - pass - else: - res = resources.joinpath(name) - if res.is_file() or res.is_dir(): - return True + res = _find_impresource(E, name) + if res and res.is_file(): + _rescache_has[name] = True + return True if pkg_resources: if _pkg_resource_exists(E.pkg.__name__, name): + _rescache_has[name] = True return True - return os.path.exists(os.path.join(E.mod, name)) + _rescache_has[name] = False + return False -def load_resource(E: EnvParams, name: str, mode="rb"): +def has_resource(E: EnvParams, name: str): + return _has_resource(E, name) or os.path.exists(os.path.join(E.mod, name)) + + +def load_resource(E: EnvParams, name: str, mode="rb") -> IO[bytes]: + enc = None if "b" in mode else "utf-8" + if impresources: - try: - resources = impresources.files(E.pkg) - except ImportError: - pass - else: - res = resources.joinpath(name) - if res.is_file(): + res = _find_impresource(E, name) + if res and res.is_file(): + if enc: + return res.open(mode, encoding=enc) + else: + # throws if encoding= is mentioned at all return res.open(mode) if pkg_resources: - if _pkg_resource_exists(E.pkg.__name__, name) and not pkg_resources.resource_isdir(E.pkg.__name__, name): + if _pkg_resource_exists(E.pkg.__name__, name): stream = pkg_resources.resource_stream(E.pkg.__name__, name) - if 'b' not in mode: - stream = io.TextIOWrapper(stream) + if enc: + stream = io.TextIOWrapper(stream, encoding=enc) return stream - return open(os.path.join(E.mod, name), mode) - - -def walk_resources(E: EnvParams, name: str): - def walk_idirs(base, r): - queue = [(base, r)] - while queue: - (b, r) = queue.pop(0) - d = [] - f = [] - for e in r.iterdir(): - if e.is_dir(): - d.append(e.name) - queue.append((os.path.join(b, e.name), e)) - elif e.is_file(): - f.append(e.name) - yield (b, d, f) - - def walk_pdirs(base): - queue = [base] - while queue: - b = queue.pop(0) - d = [] - f = [] - for e in pkg_resources.resource_listdir(E.pkg.__name__, b): - if pkg_resources.resource_isdir(E.pkg.__name__, e): - d.append(e) - queue.append(os.path.join(b, e)) - else: - f.append(e) - yield (b, d, f) - - if impresources: - try: - iresources = impresources.files(E.pkg) - except ImportError: - iresources = None - else: - iresources = None - - base_path = os.path.join(E.mod, name) - - def walk_single(base, dirs, files, normalize_base=False, skip_ires=False, skip_pres=False): - if normalize_base: - if base != base_path: - relbase = os.path.relpath(base, base_path) - else: - relbase = name - else: - relbase = base - - ires_dirs = [] - if not skip_ires and iresources: - iresbase = iresources.joinpath(relbase) - if iresbase.is_dir(): - for ientry in iresbase.iterdir(): - if ientry.is_dir() and ientry.name not in dirs: - dirs.append(ientry.name) - ires_dirs.append(ientry.name) - elif ientry.is_file() and ientry.name not in files: - files.append(ientry.name) - - pres_dirs = [] - if not skip_pres and _pkg_resource_exists(E.pkg.__name__, relbase) and pkg_resources.resource_isdir(E.pkg.__name__, relbase): - for pentry in pkg_resources.resource_listdir(E.pkg.__name__, relbase): - ppath = os.path.join(relbase, pentry) - if pkg_resources.resource_isdir(E.pkg.__name__, ppath): - if pentry not in dirs: - dirs.append(pentry) - pres_dirs.append(pentry) - else: - if pentry not in files: - files.append(pentry) - - yield (base, dirs + ires_dirs + pres_dirs, files) - for d in ires_dirs: - for (ibase, idirs, ifiles) in walk_idirs(os.path.join(relbase, d), iresources.joinpath(relbase, d)): - yield from walk_single(ibase, idirs, ifiles, normalize_base=False, skip_ires=True, skip_pres=skip_pres) - for d in pres_dirs: - for (pbase, pdirs, pfiles) in walk_pdirs(os.path.join(relbase, d)): - yield (pbase, pdirs, pfiles) - - normalize_base = False - skip_ires = skip_pres = False - if os.path.isdir(base_path): - walker = os.walk(base_path) - normalize_base = True - elif iresources and iresources.joinpath(name).is_dir(): - walker = walk_idirs(name, iresources.joinpath(name)) - skip_ires = True - elif pkg_resources and _pkg_resource_exists(E.pkg.__name__, name) and pkg_resources.resource_isdir(E.pkg.__name__, name): - walker = walk_pdirs(name) - skip_pres = True - - for (base, dirs, files) in walker: - yield from walk_single(base, dirs, files, normalize_base=normalize_base, skip_ires=skip_ires, skip_pres=skip_pres) + return open(os.path.join(E.mod, name), mode, encoding=enc) class Pebkac(Exception): diff --git a/scripts/make-pyz.sh b/scripts/make-pyz.sh index 6c73eabe..55027cf6 100755 --- a/scripts/make-pyz.sh +++ b/scripts/make-pyz.sh @@ -42,12 +42,6 @@ ver="$(cat ../sfx/ver)" mkdir -p ../dist pyz_out=../dist/copyparty.pyz -echo creating z.tar -( cd copyparty - tar -cf z.tar "${targs[@]}" --numeric-owner web res - rm -rf web res -) - echo creating loader sed -r 's/^(VER = ).*/\1"'"$ver"'"/; s/^(STAMP = ).*/\1'$(date +%s)/ \ <../scripts/ziploader.py \ diff --git a/scripts/make-sfx.sh b/scripts/make-sfx.sh index 728c63a2..474d2f39 100755 --- a/scripts/make-sfx.sh +++ b/scripts/make-sfx.sh @@ -492,8 +492,8 @@ iawk '/^def /{s=0}/^def generate_lorem_ipsum/{s=1}!s' j2/jinja2/utils.py iawk '/^(class|def) /{s=0}/^(class InternationalizationExtension|def _make_new_n?gettext)/{s=1}!s' j2/jinja2/ext.py iawk '/^[^ ]/{s=0}/^def babel_extract/{s=1}!s' j2/jinja2/ext.py ised '/InternationalizationExtension/d' j2/jinja2/ext.py -iawk '/^class/{s=0}/^class (Package|Dict|Function|Prefix|Choice|Module)Loader/{s=1}!s' j2/jinja2/loaders.py -sed -ri '/^from .bccache | (Package|Dict|Function|Prefix|Choice|Module)Loader$/d' j2/jinja2/__init__.py +iawk '/^class/{s=0}/^class (Package|Dict|Prefix|Choice|Module)Loader/{s=1}!s' j2/jinja2/loaders.py +sed -ri '/^from .bccache | (Package|Dict|Prefix|Choice|Module)Loader$/d' j2/jinja2/__init__.py rm -f j2/jinja2/async* j2/jinja2/{bccache,sandbox}.py cat > j2/jinja2/_identifier.py <<'EOF' import re diff --git a/scripts/pyinstaller/build.sh b/scripts/pyinstaller/build.sh index d3c2685d..263246ba 100644 --- a/scripts/pyinstaller/build.sh +++ b/scripts/pyinstaller/build.sh @@ -77,11 +77,14 @@ excl=( email._header_value_parser email.header email.parser + importlib.resources + importlib_resources inspect multiprocessing packaging pdb pickle + pkg_resources PIL.EpsImagePlugin pyftpdlib.prefork urllib.request diff --git a/scripts/ziploader.py b/scripts/ziploader.py index 0f41f29d..4503452d 100644 --- a/scripts/ziploader.py +++ b/scripts/ziploader.py @@ -1,11 +1,6 @@ #!/usr/bin/env python3 -import atexit -import os -import platform import sys -import tarfile -import tempfile import time import traceback @@ -23,20 +18,6 @@ def msg(*a, **ka): print(*a, **ka) -def utime(top): - # avoid cleaners - files = [os.path.join(dp, p) for dp, dd, df in os.walk(top) for p in dd + df] - try: - while True: - t = int(time.time()) - for f in [top] + files: - os.utime(f, (t, t)) - - time.sleep(78123) - except Exception as ex: - print("utime:", ex, f) - - def confirm(rv): msg() msg("retcode", rv if rv else traceback.format_exc()) @@ -51,47 +32,17 @@ def confirm(rv): def run(): - import copyparty from copyparty.__main__ import main as cm - td = tempfile.TemporaryDirectory(prefix="") - atexit.register(td.cleanup) - rsrc = td.name - - try: - from importlib.resources import files - - f = files(copyparty).joinpath("z.tar").open("rb") - except: - from importlib.resources import open_binary - - f = open_binary("copyparty", "z.tar") - - with tarfile.open(fileobj=f) as tf: - try: - tf.extractall(rsrc, filter="tar") - except TypeError: - tf.extractall(rsrc) # nosec (archive is safe) - - f.close() - f = None - - msg(" rsrc dir:", rsrc) - msg() - - sys.argv.append("--sfx-tpoke=" + rsrc) - - cm(rsrc=rsrc) + cm() def main(): - sysver = str(sys.version).replace("\n", "\n" + " " * 18) pktime = time.strftime("%Y-%m-%d, %H:%M:%S", time.gmtime(STAMP)) msg() - msg(" this is: copyparty", VER) - msg(" packed at:", pktime, "UTC,", STAMP) - msg("python bin:", sys.executable) - msg("python ver:", platform.python_implementation(), sysver) + msg("build-time:", pktime, "UTC,", STAMP) + msg("python-bin:", sys.executable) + msg() try: run()