pkgres:

* pyz: yeet the resource tar which is now pointless thanks to pkgres
* cache impresource stuff because pyz lookups are Extremely slow
* prefer tx_file when possible for slightly better performance
* use hardcoded list of expected resources instead of dynamic
   discovery at runtime; much simpler and probably safer
* fix some forgotten resources (copying.txt, insecure.pem)
* fix loading jinja templates on windows

copyparty/__init__.py

@@ -16,7 +16,9 @@ except:
     TYPE_CHECKING = False
 
 if True:
-    from typing import Any, Callable
+    from types import ModuleType
+
+    from typing import Any, Callable, Optional
 
 PY2 = sys.version_info < (3,)
 PY36 = sys.version_info > (3, 6)
@@ -51,10 +53,63 @@ try:
 except:
     CORES = (os.cpu_count() if hasattr(os, "cpu_count") else 0) or 2
 
+# all embedded resources to be retrievable over http
+zs = """
+web/a/partyfuse.py
+web/a/u2c.py
+web/a/webdav-cfg.bat
+web/baguettebox.js
+web/browser.css
+web/browser.html
+web/browser.js
+web/browser2.html
+web/cf.html
+web/copyparty.gif
+web/dd/2.png
+web/dd/3.png
+web/dd/4.png
+web/dd/5.png
+web/deps/busy.mp3
+web/deps/easymde.css
+web/deps/easymde.js
+web/deps/marked.js
+web/deps/mini-fa.css
+web/deps/mini-fa.woff
+web/deps/prism.css
+web/deps/prism.js
+web/deps/prismd.css
+web/deps/scp.woff2
+web/deps/sha512.ac.js
+web/deps/sha512.hw.js
+web/md.css
+web/md.html
+web/md.js
+web/md2.css
+web/md2.js
+web/mde.css
+web/mde.html
+web/mde.js
+web/msg.css
+web/msg.html
+web/shares.css
+web/shares.html
+web/shares.js
+web/splash.css
+web/splash.html
+web/splash.js
+web/svcs.html
+web/svcs.js
+web/ui.css
+web/up2k.js
+web/util.js
+web/w.hash.js
+"""
+RES = set(zs.strip().split("\n"))
+
 
 class EnvParams(object):
     def __init__(self) -> None:
-        self.pkg = None
+        self.pkg: Optional[ModuleType] = None
         self.t0 = time.time()
         self.mod = ""
         self.cfg = ""

copyparty/__main__.py

@@ -58,6 +58,7 @@ from .util import (
     b64enc,
     dedent,
     has_resource,
+    load_resource,
     min_ex,
     pybin,
     termsize,
@@ -217,6 +218,7 @@ def init_E(EE: EnvParams) -> None:
 
         raise Exception("could not find a writable path for config")
 
+    assert __package__  # !rm
     E.pkg = sys.modules[__package__]
     E.mod = os.path.dirname(os.path.realpath(__file__))
     if E.mod.endswith("__init__"):
@@ -520,14 +522,18 @@ def sfx_tpoke(top: str):
 
 
 def showlic() -> None:
-    p = os.path.join(E.mod, "res", "COPYING.txt")
+    try:
-    if not os.path.exists(p):
+        with load_resource(E, "res/COPYING.txt") as f:
+            buf = f.read()
+    except:
+        buf = b""
+
+    if buf:
+        print(buf.decode("utf-8", "replace"))
+    else:
         print("no relevant license info to display")
         return
 
-    with open(p, "rb") as f:
-        print(f.read().decode("utf-8", "replace"))
-
 
 def get_sects():
     return [
@@ -1567,16 +1573,13 @@ def run_argparse(
     return ret
 
 
-def main(argv: Optional[list[str]] = None, rsrc: Optional[str] = None) -> None:
+def main(argv: Optional[list[str]] = None) -> None:
     time.strptime("19970815", "%Y%m%d")  # python#7980
     if WINDOWS:
         os.system("rem")  # enables colors
 
     init_E(E)
 
-    if rsrc:  # pyz
-        E.mod = rsrc
-
     if argv is None:
         argv = sys.argv
 

copyparty/cert.py

@@ -7,7 +7,7 @@ import shutil
 import time
 
 from .__init__ import ANYWIN
-from .util import Netdev, runcmd, wrename, wunlink
+from .util import Netdev, load_resource, runcmd, wrename, wunlink
 
 HAVE_CFSSL = not os.environ.get("PRTY_NO_CFSSL")
 
@@ -29,13 +29,15 @@ def ensure_cert(log: "RootLogger", args) -> None:
 
     i feel awful about this and so should they
     """
-    cert_insec = os.path.join(args.E.mod, "res/insecure.pem")
+    with load_resource(args.E, "res/insecure.pem") as f:
+        cert_insec = f.read()
     cert_appdata = os.path.join(args.E.cfg, "cert.pem")
     if not os.path.isfile(args.cert):
         if cert_appdata != args.cert:
             raise Exception("certificate file does not exist: " + args.cert)
 
-        shutil.copy(cert_insec, args.cert)
+        with open(args.cert, "wb") as f:
+            f.write(cert_insec)
 
     with open(args.cert, "rb") as f:
         buf = f.read()
@@ -50,7 +52,9 @@ def ensure_cert(log: "RootLogger", args) -> None:
             raise Exception(m + "private key must appear before server certificate")
 
     try:
-        if filecmp.cmp(args.cert, cert_insec):
+        with open(args.cert, "rb") as f:
+            active_cert = f.read()
+        if active_cert == cert_insec:
             t = "using default TLS certificate; https will be insecure:\033[36m {}"
             log("cert", t.format(args.cert), 3)
     except:
@@ -151,14 +155,22 @@ def _gen_srv(log: "RootLogger", args, netdevs: dict[str, Netdev]):
             raise Exception("no useable cert found")
 
         expired = time.time() + args.crt_sdays * 60 * 60 * 24 * 0.5 > expiry
-        cert_insec = os.path.join(args.E.mod, "res/insecure.pem")
+        if expired:
+            raise Exception("old server-cert has expired")
+
         for n in names:
             if n not in inf["sans"]:
                 raise Exception("does not have {}".format(n))
-        if expired:
+
-            raise Exception("old server-cert has expired")
+        with load_resource(args.E, "res/insecure.pem") as f:
-        if not filecmp.cmp(args.cert, cert_insec):
+            cert_insec = f.read()
+
+        with open(args.cert, "rb") as f:
+            active_cert = f.read()
+
+        if active_cert and active_cert != cert_insec:
             return
+
     except Exception as ex:
         log("cert", "will create new server-cert; {}".format(ex))
 

copyparty/httpcli.py

@@ -32,7 +32,7 @@ try:
 except:
     pass
 
-from .__init__ import ANYWIN, PY2, TYPE_CHECKING, EnvParams, unicode
+from .__init__ import ANYWIN, PY2, RES, TYPE_CHECKING, EnvParams, unicode
 from .__version__ import S_VERSION
 from .authsrv import VFS  # typechk
 from .bos import bos
@@ -67,8 +67,8 @@ from .util import (
     get_df,
     get_spd,
     guess_mime,
-    gzip_orig_sz,
     gzip_file_orig_sz,
+    gzip_orig_sz,
     has_resource,
     hashcopy,
     hidedir,
@@ -1097,13 +1097,17 @@ class HttpCli(object):
             if self.vpath == ".cpr/metrics":
                 return self.conn.hsrv.metrics.tx(self)
 
-            static_path = os.path.join("web", self.vpath[5:])
+            res_path = "web/" + self.vpath[5:]
-            if static_path in self.conn.hsrv.statics:
+            if res_path in RES:
-                return self.tx_res(static_path)
+                ap = os.path.join(self.E.mod, res_path)
+                if bos.path.exists(ap) or bos.path.exists(ap + ".gz"):
+                    return self.tx_file(ap)
+                else:
+                    return self.tx_res(res_path)
 
-            if not undot(static_path).startswith("web"):
+            if res_path != undot(res_path):
                 t = "malicious user; attempted path traversal [{}] => [{}]"
-                self.log(t.format(self.vpath, static_path), 1)
+                self.log(t.format(self.vpath, res_path), 1)
                 self.cbonk(self.conn.hsrv.gmal, self.req, "trav", "path traversal")
 
             self.tx_404()
@@ -3415,6 +3419,7 @@ class HttpCli(object):
             self.args.s_wr_slp,
             not self.args.no_poll,
         )
+        res.close()
 
         if remains > 0:
             logmsg += " \033[31m" + unicode(file_sz - remains) + "\033[0m"

copyparty/httpsrv.py

@@ -66,7 +66,6 @@ from .util import (
     Magician,
     Netdev,
     NetMap,
-    absreal,
     build_netmap,
     has_resource,
     ipnorm,
@@ -76,9 +75,7 @@ from .util import (
     spack,
     start_log_thrs,
     start_stackmon,
-    stat_resource,
     ub64enc,
-    walk_resources,
 )
 
 if TYPE_CHECKING:
@@ -96,7 +93,7 @@ if not hasattr(socket, "AF_UNIX"):
 
 
 def load_jinja2_resource(E: EnvParams, name: str):
-    return load_resource(E, os.path.join("web", name), "r").read()
+    return load_resource(E, "web/" + name, "r").read()
 
 
 class HttpSrv(object):
@@ -174,15 +171,12 @@ class HttpSrv(object):
             "cf",
         ]
         self.j2 = {x: env.get_template(x + ".html") for x in jn}
-        self.prism = has_resource(self.E, os.path.join("web", "deps", "prism.js.gz"))
+        self.prism = has_resource(self.E, "web/deps/prism.js.gz")
 
         self.ipa_nm = build_netmap(self.args.ipa)
         self.xff_nm = build_netmap(self.args.xff_src)
         self.xff_lan = build_netmap("lan")
 
-        self.statics: set[str] = set()
-        self._build_statics()
-
         self.ptn_cc = re.compile(r"[\x00-\x1f]")
         self.ptn_hsafe = re.compile(r"[\x00-\x1f<>\"'&]")
 
@@ -216,14 +210,6 @@ class HttpSrv(object):
         except:
             pass
 
-    def _build_statics(self) -> None:
-        for dp, _, df in walk_resources(self.E, "web"):
-            for fn in df:
-                ap = os.path.join(dp, fn)
-                self.statics.add(ap)
-                if ap.endswith(".gz"):
-                    self.statics.add(ap[:-3])
-
     def set_netdevs(self, netdevs: dict[str, Netdev]) -> None:
         ips = set()
         for ip, _ in self.bound:
@@ -543,20 +529,10 @@ class HttpSrv(object):
 
             v = self.E.t0
             try:
-                for (base, dirs, files) in walk_resources(self.E, "web"):
+                with os.scandir(os.path.join(self.E.mod, "web")) as dh:
-                    inf = stat_resource(self.E, base)
+                    for fh in dh:
-                    if inf:
+                        inf = fh.stat()
                         v = max(v, inf.st_mtime)
-                    for d in dirs:
-                        inf = stat_resource(self.E, os.path.join(base, d))
-                        if inf:
-                            v = max(v, inf.st_mtime)
-                    for f in files:
-                        inf = stat_resource(self.E, os.path.join(base, e))
-                        if inf:
-                            v = max(v, inf.st_mtime)
-                    # only do top-level
-                    break
             except:
                 pass
 

copyparty/util.py

@@ -134,7 +134,7 @@ if True:  # pylint: disable=using-constant-test
     from collections.abc import Callable, Iterable
 
     import typing
-    from typing import Any, Generator, Optional, Pattern, Protocol, Union
+    from typing import Any, Generator, IO, Optional, Pattern, Protocol, Union
 
     try:
         from typing import LiteralString
@@ -3420,6 +3420,7 @@ def gzip_orig_sz(fn: str) -> int:
     with open(fsenc(fn), "rb") as f:
         return gzip_file_orig_sz(f)
 
+
 def gzip_file_orig_sz(f) -> int:
     start = f.tell()
     f.seek(-4, 2)
@@ -3582,144 +3583,95 @@ def _pkg_resource_exists(pkg: str, name: str) -> bool:
     except NotImplementedError:
         return False
 
+
 def stat_resource(E: EnvParams, name: str):
     path = os.path.join(E.mod, name)
     if os.path.exists(path):
         return os.stat(fsenc(path))
     return None
 
-def has_resource(E: EnvParams, name: str):
+
-    if impresources:
+def _find_impresource_cold(E: EnvParams, name: str):
+    global _rescache_imp, _find_impresource
+
+    assert impresources  # !rm
     try:
-            resources = impresources.files(E.pkg)
+        _rescache_imp = impresources.files(E.pkg)
     except ImportError:
+        return None
+
+    _find_impresource = _find_impresource_warm
+    return _find_impresource(E, name)
+
+
+def _find_impresource_warm(E: EnvParams, name: str):
+    if not _rescache_imp:
+        return None
+
+    try:
+        return _rescache_res[name]
+    except:
+        if len(_rescache_res) > 999:
+            _rescache_res.clear()
+        ret = _rescache_imp.joinpath(name)
+        _rescache_res[name] = ret
+        return ret
+
+
+_find_impresource = _find_impresource_cold
+_rescache_imp = None
+_rescache_has = {}
+_rescache_res = {}
+
+
+def _has_resource(E: EnvParams, name: str):
+    try:
+        return _rescache_has[name]
+    except:
         pass
-        else:
+
-            res = resources.joinpath(name)
+    if len(_rescache_has) > 999:
-            if res.is_file() or res.is_dir():
+        _rescache_has.clear()
+
+    if impresources:
+        res = _find_impresource(E, name)
+        if res and res.is_file():
+            _rescache_has[name] = True
             return True
 
     if pkg_resources:
         if _pkg_resource_exists(E.pkg.__name__, name):
+            _rescache_has[name] = True
             return True
 
-    return os.path.exists(os.path.join(E.mod, name))
+    _rescache_has[name] = False
+    return False
 
 
-def load_resource(E: EnvParams, name: str, mode="rb"):
+def has_resource(E: EnvParams, name: str):
+    return _has_resource(E, name) or os.path.exists(os.path.join(E.mod, name))
+
+
+def load_resource(E: EnvParams, name: str, mode="rb") -> IO[bytes]:
+    enc = None if "b" in mode else "utf-8"
+
     if impresources:
-        try:
+        res = _find_impresource(E, name)
-            resources = impresources.files(E.pkg)
+        if res and res.is_file():
-        except ImportError:
+            if enc:
-            pass
+                return res.open(mode, encoding=enc)
             else:
-            res = resources.joinpath(name)
+                # throws if encoding= is mentioned at all
-            if res.is_file():
                 return res.open(mode)
 
     if pkg_resources:
-        if _pkg_resource_exists(E.pkg.__name__, name) and not pkg_resources.resource_isdir(E.pkg.__name__, name):
+        if _pkg_resource_exists(E.pkg.__name__, name):
             stream = pkg_resources.resource_stream(E.pkg.__name__, name)
-            if 'b' not in mode:
+            if enc:
-                stream = io.TextIOWrapper(stream)
+                stream = io.TextIOWrapper(stream, encoding=enc)
             return stream
 
-    return open(os.path.join(E.mod, name), mode)
+    return open(os.path.join(E.mod, name), mode, encoding=enc)
-
-
-def walk_resources(E: EnvParams, name: str):
-    def walk_idirs(base, r):
-        queue = [(base, r)]
-        while queue:
-            (b, r) = queue.pop(0)
-            d = []
-            f = []
-            for e in r.iterdir():
-                if e.is_dir():
-                    d.append(e.name)
-                    queue.append((os.path.join(b, e.name), e))
-                elif e.is_file():
-                    f.append(e.name)
-            yield (b, d, f)
-
-    def walk_pdirs(base):
-        queue = [base]
-        while queue:
-            b = queue.pop(0)
-            d = []
-            f = []
-            for e in pkg_resources.resource_listdir(E.pkg.__name__, b):
-                if pkg_resources.resource_isdir(E.pkg.__name__, e):
-                    d.append(e)
-                    queue.append(os.path.join(b, e))
-                else:
-                    f.append(e)
-            yield (b, d, f)
-
-    if impresources:
-        try:
-            iresources = impresources.files(E.pkg)
-        except ImportError:
-            iresources = None
-    else:
-        iresources = None
-
-    base_path = os.path.join(E.mod, name)
-
-    def walk_single(base, dirs, files, normalize_base=False, skip_ires=False, skip_pres=False):
-        if normalize_base:
-            if base != base_path:
-                relbase = os.path.relpath(base, base_path)
-            else:
-                relbase = name
-        else:
-            relbase = base
-
-        ires_dirs = []
-        if not skip_ires and iresources:
-            iresbase = iresources.joinpath(relbase)
-            if iresbase.is_dir():
-                for ientry in iresbase.iterdir():
-                    if ientry.is_dir() and ientry.name not in dirs:
-                        dirs.append(ientry.name)
-                        ires_dirs.append(ientry.name)
-                    elif ientry.is_file() and ientry.name not in files:
-                        files.append(ientry.name)
-
-        pres_dirs = []
-        if not skip_pres and _pkg_resource_exists(E.pkg.__name__, relbase) and pkg_resources.resource_isdir(E.pkg.__name__, relbase):
-            for pentry in pkg_resources.resource_listdir(E.pkg.__name__, relbase):
-                ppath = os.path.join(relbase, pentry)
-                if pkg_resources.resource_isdir(E.pkg.__name__, ppath):
-                    if pentry not in dirs:
-                        dirs.append(pentry)
-                        pres_dirs.append(pentry)
-                else:
-                    if pentry not in files:
-                        files.append(pentry)
-
-        yield (base, dirs + ires_dirs + pres_dirs, files)
-        for d in ires_dirs:
-            for (ibase, idirs, ifiles) in walk_idirs(os.path.join(relbase, d), iresources.joinpath(relbase, d)):
-                yield from walk_single(ibase, idirs, ifiles, normalize_base=False, skip_ires=True, skip_pres=skip_pres)
-        for d in pres_dirs:
-            for (pbase, pdirs, pfiles) in walk_pdirs(os.path.join(relbase, d)):
-                yield (pbase, pdirs, pfiles)
-
-    normalize_base = False
-    skip_ires = skip_pres = False
-    if os.path.isdir(base_path):
-        walker = os.walk(base_path)
-        normalize_base = True
-    elif iresources and iresources.joinpath(name).is_dir():
-        walker = walk_idirs(name, iresources.joinpath(name))
-        skip_ires = True
-    elif pkg_resources and _pkg_resource_exists(E.pkg.__name__, name) and pkg_resources.resource_isdir(E.pkg.__name__, name):
-        walker = walk_pdirs(name)
-        skip_pres = True
-
-    for (base, dirs, files) in walker:
-        yield from walk_single(base, dirs, files, normalize_base=normalize_base, skip_ires=skip_ires, skip_pres=skip_pres)
 
 
 class Pebkac(Exception):

scripts/make-pyz.sh

@@ -42,12 +42,6 @@ ver="$(cat ../sfx/ver)"
 mkdir -p ../dist
 pyz_out=../dist/copyparty.pyz
 
-echo creating z.tar
-( cd copyparty
-  tar -cf z.tar "${targs[@]}" --numeric-owner web res
-  rm -rf web res
-)
-
 echo creating loader
 sed -r 's/^(VER = ).*/\1"'"$ver"'"/; s/^(STAMP = ).*/\1'$(date +%s)/ \
     <../scripts/ziploader.py \

scripts/make-sfx.sh

@@ -492,8 +492,8 @@ iawk '/^def /{s=0}/^def generate_lorem_ipsum/{s=1}!s' j2/jinja2/utils.py
 iawk '/^(class|def) /{s=0}/^(class InternationalizationExtension|def _make_new_n?gettext)/{s=1}!s' j2/jinja2/ext.py
 iawk '/^[^ ]/{s=0}/^def babel_extract/{s=1}!s' j2/jinja2/ext.py
 ised '/InternationalizationExtension/d' j2/jinja2/ext.py
-iawk '/^class/{s=0}/^class (Package|Dict|Function|Prefix|Choice|Module)Loader/{s=1}!s' j2/jinja2/loaders.py
+iawk '/^class/{s=0}/^class (Package|Dict|Prefix|Choice|Module)Loader/{s=1}!s' j2/jinja2/loaders.py
-sed -ri '/^from .bccache | (Package|Dict|Function|Prefix|Choice|Module)Loader$/d' j2/jinja2/__init__.py
+sed -ri '/^from .bccache | (Package|Dict|Prefix|Choice|Module)Loader$/d' j2/jinja2/__init__.py
 rm -f j2/jinja2/async* j2/jinja2/{bccache,sandbox}.py
 cat > j2/jinja2/_identifier.py <<'EOF'
 import re

scripts/pyinstaller/build.sh

@@ -77,11 +77,14 @@ excl=(
     email._header_value_parser
     email.header
     email.parser
+    importlib.resources
+    importlib_resources
     inspect
     multiprocessing
     packaging
     pdb
     pickle
+    pkg_resources
     PIL.EpsImagePlugin
     pyftpdlib.prefork
     urllib.request

scripts/ziploader.py

@@ -1,11 +1,6 @@
 #!/usr/bin/env python3
 
-import atexit
-import os
-import platform
 import sys
-import tarfile
-import tempfile
 import time
 import traceback
 
@@ -23,20 +18,6 @@ def msg(*a, **ka):
     print(*a, **ka)
 
 
-def utime(top):
-    # avoid cleaners
-    files = [os.path.join(dp, p) for dp, dd, df in os.walk(top) for p in dd + df]
-    try:
-        while True:
-            t = int(time.time())
-            for f in [top] + files:
-                os.utime(f, (t, t))
-
-            time.sleep(78123)
-    except Exception as ex:
-        print("utime:", ex, f)
-
-
 def confirm(rv):
     msg()
     msg("retcode", rv if rv else traceback.format_exc())
@@ -51,47 +32,17 @@ def confirm(rv):
 
 
 def run():
-    import copyparty
     from copyparty.__main__ import main as cm
 
-    td = tempfile.TemporaryDirectory(prefix="")
+    cm()
-    atexit.register(td.cleanup)
-    rsrc = td.name
-
-    try:
-        from importlib.resources import files
-
-        f = files(copyparty).joinpath("z.tar").open("rb")
-    except:
-        from importlib.resources import open_binary
-
-        f = open_binary("copyparty", "z.tar")
-
-    with tarfile.open(fileobj=f) as tf:
-        try:
-            tf.extractall(rsrc, filter="tar")
-        except TypeError:
-            tf.extractall(rsrc)  # nosec (archive is safe)
-
-    f.close()
-    f = None
-
-    msg("  rsrc dir:", rsrc)
-    msg()
-
-    sys.argv.append("--sfx-tpoke=" + rsrc)
-
-    cm(rsrc=rsrc)
 
 
 def main():
-    sysver = str(sys.version).replace("\n", "\n" + " " * 18)
     pktime = time.strftime("%Y-%m-%d, %H:%M:%S", time.gmtime(STAMP))
     msg()
-    msg("   this is: copyparty", VER)
+    msg("build-time:", pktime, "UTC,", STAMP)
-    msg(" packed at:", pktime, "UTC,", STAMP)
+    msg("python-bin:", sys.executable)
-    msg("python bin:", sys.executable)
+    msg()
-    msg("python ver:", platform.python_implementation(), sysver)
 
     try:
         run()