add types, isort, errorhandling

README.md

@@ -1200,15 +1200,18 @@ journalctl -aS '48 hour ago' -u copyparty | grep -C10 FILENAME | tee bug.log
 
 ## dev env setup
 
-mostly optional; if you need a working env for vscode or similar
+you need python 3.9 or newer due to type hints
+
+the rest is mostly optional; if you need a working env for vscode or similar
 
 ```sh
 python3 -m venv .venv
 . .venv/bin/activate
-pip install jinja2  # mandatory
+pip install jinja2 strip_hints  # MANDATORY
 pip install mutagen  # audio metadata
+pip install pyftpdlib  # ftp server
 pip install Pillow pyheif-pillow-opener pillow-avif-plugin  # thumbnails
-pip install black==21.12b0 bandit pylint flake8  # vscode tooling
+pip install black==21.12b0 click==8.0.2 bandit pylint flake8 isort mypy  # vscode tooling
 ```
 
 

bin/mtag/image-noexif.py

@@ -43,7 +43,6 @@ PS: this requires e2ts to be functional,
 
 import os
 import sys
-import time
 import filecmp
 import subprocess as sp
 

bin/up2k.py

@@ -77,15 +77,15 @@ class File(object):
         self.up_b = 0  # type: int
         self.up_c = 0  # type: int
 
-        # m = "size({}) lmod({}) top({}) rel({}) abs({}) name({})\n"
+        # t = "size({}) lmod({}) top({}) rel({}) abs({}) name({})\n"
-        # eprint(m.format(self.size, self.lmod, self.top, self.rel, self.abs, self.name))
+        # eprint(t.format(self.size, self.lmod, self.top, self.rel, self.abs, self.name))
 
 
 class FileSlice(object):
     """file-like object providing a fixed window into a file"""
 
     def __init__(self, file, cid):
-        # type: (File, str) -> FileSlice
+        # type: (File, str) -> None
 
         self.car, self.len = file.kchunks[cid]
         self.cdr = self.car + self.len
@@ -216,8 +216,8 @@ class CTermsize(object):
             eprint("\033[s\033[r\033[u")
         else:
             self.g = 1 + self.h - margin
-            m = "{0}\033[{1}A".format("\n" * margin, margin)
+            t = "{0}\033[{1}A".format("\n" * margin, margin)
-            eprint("{0}\033[s\033[1;{1}r\033[u".format(m, self.g - 1))
+            eprint("{0}\033[s\033[1;{1}r\033[u".format(t, self.g - 1))
 
 
 ss = CTermsize()
@@ -597,8 +597,8 @@ class Ctl(object):
                         if "/" in name:
                             name = "\033[36m{0}\033[0m/{1}".format(*name.rsplit("/", 1))
 
-                        m = "{0:6.1f}% {1} {2}\033[K"
+                        t = "{0:6.1f}% {1} {2}\033[K"
-                        txt += m.format(p, self.nfiles - f, name)
+                        txt += t.format(p, self.nfiles - f, name)
 
                 txt += "\033[{0}H ".format(ss.g + 2)
             else:
@@ -618,8 +618,8 @@ class Ctl(object):
             nleft = self.nfiles - self.up_f
             tail = "\033[K\033[u" if VT100 else "\r"
 
-            m = "{0} eta @ {1}/s, {2}, {3}# left".format(eta, spd, sleft, nleft)
+            t = "{0} eta @ {1}/s, {2}, {3}# left".format(eta, spd, sleft, nleft)
-            eprint(txt + "\033]0;{0}\033\\\r{0}{1}".format(m, tail))
+            eprint(txt + "\033]0;{0}\033\\\r{0}{1}".format(t, tail))
 
     def cleanup_vt100(self):
         ss.scroll_region(None)
@@ -721,8 +721,8 @@ class Ctl(object):
             if search:
                 if hs:
                     for hit in hs:
-                        m = "found: {0}\n  {1}{2}\n"
+                        t = "found: {0}\n  {1}{2}\n"
-                        print(m.format(upath, burl, hit["rp"]), end="")
+                        print(t.format(upath, burl, hit["rp"]), end="")
                 else:
                     print("NOT found: {0}\n".format(upath), end="")
 

contrib/systemd/copyparty.service

@@ -4,7 +4,7 @@
 # installation:
 #   cp -pv copyparty.service /etc/systemd/system
 #   restorecon -vr /etc/systemd/system/copyparty.service
-#   firewall-cmd --permanent --add-port={80,443,3923}/tcp
+#   firewall-cmd --permanent --add-port={80,443,3923}/tcp  # --zone=libvirt
 #   firewall-cmd --reload
 #   systemctl daemon-reload && systemctl enable --now copyparty
 #

copyparty/__init__.py

@@ -1,21 +1,30 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
-import platform
-import time
-import sys
 import os
+import platform
+import sys
+import time
+
+try:
+    from collections.abc import Callable
+
+    from typing import TYPE_CHECKING, Any
+except:
+    TYPE_CHECKING = False
 
 PY2 = sys.version_info[0] == 2
 if PY2:
     sys.dont_write_bytecode = True
-    unicode = unicode
+    unicode = unicode  # noqa: F821  # pylint: disable=undefined-variable,self-assigning-variable
 else:
     unicode = str
 
-WINDOWS = False
+WINDOWS: Any = (
-if platform.system() == "Windows":
+    [int(x) for x in platform.version().split(".")]
-    WINDOWS = [int(x) for x in platform.version().split(".")]
+    if platform.system() == "Windows"
+    else False
+)
 
 VT100 = not WINDOWS or WINDOWS >= [10, 0, 14393]
 # introduced in anniversary update
@@ -25,8 +34,8 @@ ANYWIN = WINDOWS or sys.platform in ["msys"]
 MACOS = platform.system() == "Darwin"
 
 
-def get_unixdir():
+def get_unixdir() -> str:
-    paths = [
+    paths: list[tuple[Callable[..., str], str]] = [
         (os.environ.get, "XDG_CONFIG_HOME"),
         (os.path.expanduser, "~/.config"),
         (os.environ.get, "TMPDIR"),
@@ -43,7 +52,7 @@ def get_unixdir():
                     continue
 
                 p = os.path.normpath(p)
-                chk(p)
+                chk(p)  # type: ignore
                 p = os.path.join(p, "copyparty")
                 if not os.path.isdir(p):
                     os.mkdir(p)
@@ -56,7 +65,7 @@ def get_unixdir():
 
 
 class EnvParams(object):
-    def __init__(self):
+    def __init__(self) -> None:
         self.t0 = time.time()
         self.mod = os.path.dirname(os.path.realpath(__file__))
         if self.mod.endswith("__init__"):

copyparty/__main__.py

@@ -8,35 +8,42 @@ __copyright__ = 2019
 __license__ = "MIT"
 __url__ = "https://github.com/9001/copyparty/"
 
-import re
+import argparse
-import os
-import sys
-import time
-import shutil
 import filecmp
 import locale
-import argparse
+import os
+import re
+import shutil
+import sys
 import threading
+import time
 import traceback
 from textwrap import dedent
 
-from .__init__ import E, WINDOWS, ANYWIN, VT100, PY2, unicode
+from .__init__ import ANYWIN, PY2, VT100, WINDOWS, E, unicode
-from .__version__ import S_VERSION, S_BUILD_DT, CODENAME
+from .__version__ import CODENAME, S_BUILD_DT, S_VERSION
-from .svchub import SvcHub
-from .util import py_desc, align_tab, IMPLICATIONS, ansi_re, min_ex
 from .authsrv import re_vol
+from .svchub import SvcHub
+from .util import IMPLICATIONS, align_tab, ansi_re, min_ex, py_desc
 
-HAVE_SSL = True
 try:
+    from types import FrameType
+
+    from typing import Any, Optional
+except:
+    pass
+
+try:
+    HAVE_SSL = True
     import ssl
 except:
     HAVE_SSL = False
 
-printed = []
+printed: list[str] = []
 
 
 class RiceFormatter(argparse.HelpFormatter):
-    def _get_help_string(self, action):
+    def _get_help_string(self, action: argparse.Action) -> str:
         """
         same as ArgumentDefaultsHelpFormatter(HelpFormatter)
         except the help += [...] line now has colors
@@ -45,27 +52,27 @@ class RiceFormatter(argparse.HelpFormatter):
         if not VT100:
             fmt = " (default: %(default)s)"
 
-        ret = action.help
+        ret = str(action.help)
-        if "%(default)" not in action.help:
+        if "%(default)" not in ret:
             if action.default is not argparse.SUPPRESS:
                 defaulting_nargs = [argparse.OPTIONAL, argparse.ZERO_OR_MORE]
                 if action.option_strings or action.nargs in defaulting_nargs:
                     ret += fmt
         return ret
 
-    def _fill_text(self, text, width, indent):
+    def _fill_text(self, text: str, width: int, indent: str) -> str:
         """same as RawDescriptionHelpFormatter(HelpFormatter)"""
         return "".join(indent + line + "\n" for line in text.splitlines())
 
 
 class Dodge11874(RiceFormatter):
-    def __init__(self, *args, **kwargs):
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
         kwargs["width"] = 9003
         super(Dodge11874, self).__init__(*args, **kwargs)
 
 
-def lprint(*a, **ka):
+def lprint(*a: Any, **ka: Any) -> None:
-    txt = " ".join(unicode(x) for x in a) + ka.get("end", "\n")
+    txt: str = " ".join(unicode(x) for x in a) + ka.get("end", "\n")
     printed.append(txt)
     if not VT100:
         txt = ansi_re.sub("", txt)
@@ -73,11 +80,11 @@ def lprint(*a, **ka):
     print(txt, **ka)
 
 
-def warn(msg):
+def warn(msg: str) -> None:
     lprint("\033[1mwarning:\033[0;33m {}\033[0m\n".format(msg))
 
 
-def ensure_locale():
+def ensure_locale() -> None:
     for x in [
         "en_US.UTF-8",
         "English_United States.UTF8",
@@ -91,7 +98,7 @@ def ensure_locale():
             continue
 
 
-def ensure_cert():
+def ensure_cert() -> None:
     """
     the default cert (and the entire TLS support) is only here to enable the
     crypto.subtle javascript API, which is necessary due to the webkit guys
@@ -117,8 +124,8 @@ def ensure_cert():
     # printf 'NO\n.\n.\n.\n.\ncopyparty-insecure\n.\n' | faketime '2000-01-01 00:00:00' openssl req -x509 -sha256 -newkey rsa:2048 -keyout insecure.pem -out insecure.pem -days $((($(printf %d 0x7fffffff)-$(date +%s --date=2000-01-01T00:00:00Z))/(60*60*24))) -nodes && ls -al insecure.pem && openssl x509 -in insecure.pem -text -noout
 
 
-def configure_ssl_ver(al):
+def configure_ssl_ver(al: argparse.Namespace) -> None:
-    def terse_sslver(txt):
+    def terse_sslver(txt: str) -> str:
         txt = txt.lower()
         for c in ["_", "v", "."]:
             txt = txt.replace(c, "")
@@ -133,8 +140,8 @@ def configure_ssl_ver(al):
     flags = [k for k in ssl.__dict__ if ptn.match(k)]
     # SSLv2 SSLv3 TLSv1 TLSv1_1 TLSv1_2 TLSv1_3
     if "help" in sslver:
-        avail = [terse_sslver(x[6:]) for x in flags]
+        avail1 = [terse_sslver(x[6:]) for x in flags]
-        avail = " ".join(sorted(avail) + ["all"])
+        avail = " ".join(sorted(avail1) + ["all"])
         lprint("\navailable ssl/tls versions:\n  " + avail)
         sys.exit(0)
 
@@ -160,7 +167,7 @@ def configure_ssl_ver(al):
     # think i need that beer now
 
 
-def configure_ssl_ciphers(al):
+def configure_ssl_ciphers(al: argparse.Namespace) -> None:
     ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
     if al.ssl_ver:
         ctx.options &= ~al.ssl_flags_en
@@ -184,8 +191,8 @@ def configure_ssl_ciphers(al):
         sys.exit(0)
 
 
-def args_from_cfg(cfg_path):
+def args_from_cfg(cfg_path: str) -> list[str]:
-    ret = []
+    ret: list[str] = []
     skip = False
     with open(cfg_path, "rb") as f:
         for ln in [x.decode("utf-8").strip() for x in f]:
@@ -210,29 +217,30 @@ def args_from_cfg(cfg_path):
     return ret
 
 
-def sighandler(sig=None, frame=None):
+def sighandler(sig: Optional[int] = None, frame: Optional[FrameType] = None) -> None:
     msg = [""] * 5
     for th in threading.enumerate():
+        stk = sys._current_frames()[th.ident]  # type: ignore
         msg.append(str(th))
-        msg.extend(traceback.format_stack(sys._current_frames()[th.ident]))
+        msg.extend(traceback.format_stack(stk))
 
     msg.append("\n")
     print("\n".join(msg))
 
 
-def disable_quickedit():
+def disable_quickedit() -> None:
-    import ctypes
     import atexit
+    import ctypes
     from ctypes import wintypes
 
-    def ecb(ok, fun, args):
+    def ecb(ok: bool, fun: Any, args: list[Any]) -> list[Any]:
         if not ok:
-            err = ctypes.get_last_error()
+            err: int = ctypes.get_last_error()  # type: ignore
             if err:
-                raise ctypes.WinError(err)
+                raise ctypes.WinError(err)  # type: ignore
         return args
 
-    k32 = ctypes.WinDLL("kernel32", use_last_error=True)
+    k32 = ctypes.WinDLL("kernel32", use_last_error=True)  # type: ignore
     if PY2:
         wintypes.LPDWORD = ctypes.POINTER(wintypes.DWORD)
 
@@ -242,14 +250,14 @@ def disable_quickedit():
     k32.GetConsoleMode.argtypes = (wintypes.HANDLE, wintypes.LPDWORD)
     k32.SetConsoleMode.argtypes = (wintypes.HANDLE, wintypes.DWORD)
 
-    def cmode(out, mode=None):
+    def cmode(out: bool, mode: Optional[int] = None) -> int:
         h = k32.GetStdHandle(-11 if out else -10)
         if mode:
-            return k32.SetConsoleMode(h, mode)
+            return k32.SetConsoleMode(h, mode)  # type: ignore
 
-        mode = wintypes.DWORD()
+        cmode = wintypes.DWORD()
-        k32.GetConsoleMode(h, ctypes.byref(mode))
+        k32.GetConsoleMode(h, ctypes.byref(cmode))
-        return mode.value
+        return cmode.value
 
     # disable quickedit
     mode = orig_in = cmode(False)
@@ -268,7 +276,7 @@ def disable_quickedit():
             cmode(True, mode | 4)
 
 
-def run_argparse(argv, formatter):
+def run_argparse(argv: list[str], formatter: Any) -> argparse.Namespace:
     ap = argparse.ArgumentParser(
         formatter_class=formatter,
         prog="copyparty",
@@ -596,7 +604,7 @@ def run_argparse(argv, formatter):
     return ret
 
 
-def main(argv=None):
+def main(argv: Optional[list[str]] = None) -> None:
     time.strptime("19970815", "%Y%m%d")  # python#7980
     if WINDOWS:
         os.system("rem")  # enables colors
@@ -618,7 +626,7 @@ def main(argv=None):
             supp = args_from_cfg(v)
             argv.extend(supp)
 
-    deprecated = []
+    deprecated: list[tuple[str, str]] = []
     for dk, nk in deprecated:
         try:
             idx = argv.index(dk)
@@ -650,7 +658,7 @@ def main(argv=None):
     if not VT100:
         al.wintitle = ""
 
-    nstrs = []
+    nstrs: list[str] = []
     anymod = False
     for ostr in al.v or []:
         m = re_vol.match(ostr)

copyparty/bos/bos.py

@@ -2,23 +2,30 @@
 from __future__ import print_function, unicode_literals
 
 import os
-from ..util import fsenc, fsdec, SYMTIME
+
+from ..util import SYMTIME, fsdec, fsenc
 from . import path
 
+try:
+    from typing import Optional
+except:
+    pass
+
+_ = (path,)
 
 # grep -hRiE '(^|[^a-zA-Z_\.-])os\.' . | gsed -r 's/ /\n/g;s/\(/(\n/g' | grep -hRiE '(^|[^a-zA-Z_\.-])os\.' | sort | uniq -c
 # printf 'os\.(%s)' "$(grep ^def bos/__init__.py | gsed -r 's/^def //;s/\(.*//' | tr '\n' '|' | gsed -r 's/.$//')"
 
 
-def chmod(p, mode):
+def chmod(p: str, mode: int) -> None:
     return os.chmod(fsenc(p), mode)
 
 
-def listdir(p="."):
+def listdir(p: str = ".") -> list[str]:
     return [fsdec(x) for x in os.listdir(fsenc(p))]
 
 
-def makedirs(name, mode=0o755, exist_ok=True):
+def makedirs(name: str, mode: int = 0o755, exist_ok: bool = True) -> None:
     bname = fsenc(name)
     try:
         os.makedirs(bname, mode)
@@ -27,31 +34,33 @@ def makedirs(name, mode=0o755, exist_ok=True):
             raise
 
 
-def mkdir(p, mode=0o755):
+def mkdir(p: str, mode: int = 0o755) -> None:
     return os.mkdir(fsenc(p), mode)
 
 
-def rename(src, dst):
+def rename(src: str, dst: str) -> None:
     return os.rename(fsenc(src), fsenc(dst))
 
 
-def replace(src, dst):
+def replace(src: str, dst: str) -> None:
     return os.replace(fsenc(src), fsenc(dst))
 
 
-def rmdir(p):
+def rmdir(p: str) -> None:
     return os.rmdir(fsenc(p))
 
 
-def stat(p):
+def stat(p: str) -> os.stat_result:
     return os.stat(fsenc(p))
 
 
-def unlink(p):
+def unlink(p: str) -> None:
     return os.unlink(fsenc(p))
 
 
-def utime(p, times=None, follow_symlinks=True):
+def utime(
+    p: str, times: Optional[tuple[float, float]] = None, follow_symlinks: bool = True
+) -> None:
     if SYMTIME:
         return os.utime(fsenc(p), times, follow_symlinks=follow_symlinks)
     else:
@@ -60,7 +69,7 @@ def utime(p, times=None, follow_symlinks=True):
 
 if hasattr(os, "lstat"):
 
-    def lstat(p):
+    def lstat(p: str) -> os.stat_result:
         return os.lstat(fsenc(p))
 
 else:

copyparty/bos/path.py

@@ -2,43 +2,44 @@
 from __future__ import print_function, unicode_literals
 
 import os
-from ..util import fsenc, fsdec, SYMTIME
+
+from ..util import SYMTIME, fsdec, fsenc
 
 
-def abspath(p):
+def abspath(p: str) -> str:
     return fsdec(os.path.abspath(fsenc(p)))
 
 
-def exists(p):
+def exists(p: str) -> bool:
     return os.path.exists(fsenc(p))
 
 
-def getmtime(p, follow_symlinks=True):
+def getmtime(p: str, follow_symlinks: bool = True) -> float:
     if not follow_symlinks and SYMTIME:
         return os.lstat(fsenc(p)).st_mtime
     else:
         return os.path.getmtime(fsenc(p))
 
 
-def getsize(p):
+def getsize(p: str) -> int:
     return os.path.getsize(fsenc(p))
 
 
-def isfile(p):
+def isfile(p: str) -> bool:
     return os.path.isfile(fsenc(p))
 
 
-def isdir(p):
+def isdir(p: str) -> bool:
     return os.path.isdir(fsenc(p))
 
 
-def islink(p):
+def islink(p: str) -> bool:
     return os.path.islink(fsenc(p))
 
 
-def lexists(p):
+def lexists(p: str) -> bool:
     return os.path.lexists(fsenc(p))
 
 
-def realpath(p):
+def realpath(p: str) -> str:
     return fsdec(os.path.realpath(fsenc(p)))

copyparty/broker_mp.py

@@ -1,37 +1,56 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
-import time
 import threading
+import time
 
-from .broker_util import try_exec
+import queue
+
+from .__init__ import TYPE_CHECKING
 from .broker_mpw import MpWorker
+from .broker_util import try_exec
 from .util import mp
 
+if TYPE_CHECKING:
+    from .svchub import SvcHub
+
+try:
+    from typing import Any
+except:
+    pass
+
+
+class MProcess(mp.Process):
+    def __init__(
+        self,
+        q_pend: queue.Queue[tuple[int, str, list[Any]]],
+        q_yield: queue.Queue[tuple[int, str, list[Any]]],
+        target: Any,
+        args: Any,
+    ) -> None:
+        super(MProcess, self).__init__(target=target, args=args)
+        self.q_pend = q_pend
+        self.q_yield = q_yield
+
 
 class BrokerMp(object):
     """external api; manages MpWorkers"""
 
-    def __init__(self, hub):
+    def __init__(self, hub: "SvcHub") -> None:
         self.hub = hub
         self.log = hub.log
         self.args = hub.args
 
         self.procs = []
-        self.retpend = {}
-        self.retpend_mutex = threading.Lock()
         self.mutex = threading.Lock()
 
         self.num_workers = self.args.j or mp.cpu_count()
         self.log("broker", "booting {} subprocesses".format(self.num_workers))
         for n in range(1, self.num_workers + 1):
-            q_pend = mp.Queue(1)
+            q_pend: queue.Queue[tuple[int, str, list[Any]]] = mp.Queue(1)
-            q_yield = mp.Queue(64)
+            q_yield: queue.Queue[tuple[int, str, list[Any]]] = mp.Queue(64)
 
-            proc = mp.Process(target=MpWorker, args=(q_pend, q_yield, self.args, n))
+            proc = MProcess(q_pend, q_yield, MpWorker, (q_pend, q_yield, self.args, n))
-            proc.q_pend = q_pend
-            proc.q_yield = q_yield
-            proc.clients = {}
 
             thr = threading.Thread(
                 target=self.collector, args=(proc,), name="mp-sink-{}".format(n)
@@ -42,11 +61,11 @@ class BrokerMp(object):
             self.procs.append(proc)
             proc.start()
 
-    def shutdown(self):
+    def shutdown(self) -> None:
         self.log("broker", "shutting down")
         for n, proc in enumerate(self.procs):
             thr = threading.Thread(
-                target=proc.q_pend.put([0, "shutdown", []]),
+                target=proc.q_pend.put((0, "shutdown", [])),
                 name="mp-shutdown-{}-{}".format(n, len(self.procs)),
             )
             thr.start()
@@ -62,12 +81,12 @@ class BrokerMp(object):
 
             procs.pop()
 
-    def reload(self):
+    def reload(self) -> None:
         self.log("broker", "reloading")
         for _, proc in enumerate(self.procs):
-            proc.q_pend.put([0, "reload", []])
+            proc.q_pend.put((0, "reload", []))
 
-    def collector(self, proc):
+    def collector(self, proc: MProcess) -> None:
         """receive message from hub in other process"""
         while True:
             msg = proc.q_yield.get()
@@ -78,10 +97,7 @@ class BrokerMp(object):
 
             elif dest == "retq":
                 # response from previous ipc call
-                with self.retpend_mutex:
+                raise Exception("invalid broker_mp usage")
-                    retq = self.retpend.pop(retq_id)
-
-                retq.put(args)
 
             else:
                 # new ipc invoking managed service in hub
@@ -93,9 +109,9 @@ class BrokerMp(object):
                 rv = try_exec(retq_id, obj, *args)
 
                 if retq_id:
-                    proc.q_pend.put([retq_id, "retq", rv])
+                    proc.q_pend.put((retq_id, "retq", rv))
 
-    def put(self, want_retval, dest, *args):
+    def say(self, dest: str, *args: Any) -> None:
         """
         send message to non-hub component in other process,
         returns a Queue object which eventually contains the response if want_retval
@@ -103,7 +119,7 @@ class BrokerMp(object):
         """
         if dest == "listen":
             for p in self.procs:
-                p.q_pend.put([0, dest, [args[0], len(self.procs)]])
+                p.q_pend.put((0, dest, [args[0], len(self.procs)]))
 
         elif dest == "cb_httpsrv_up":
             self.hub.cb_httpsrv_up()

copyparty/broker_mpw.py

@@ -1,20 +1,38 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
-import sys
+import argparse
 import signal
+import sys
 import threading
 
-from .broker_util import ExceptionalQueue
+import queue
+
+from .authsrv import AuthSrv
+from .broker_util import BrokerCli, ExceptionalQueue
 from .httpsrv import HttpSrv
 from .util import FAKE_MP
-from .authsrv import AuthSrv
+
+try:
+    from types import FrameType
+
+    from typing import Any, Optional, Union
+except:
+    pass
 
 
-class MpWorker(object):
+class MpWorker(BrokerCli):
     """one single mp instance"""
 
-    def __init__(self, q_pend, q_yield, args, n):
+    def __init__(
+        self,
+        q_pend: queue.Queue[tuple[int, str, list[Any]]],
+        q_yield: queue.Queue[tuple[int, str, list[Any]]],
+        args: argparse.Namespace,
+        n: int,
+    ) -> None:
+        super(MpWorker, self).__init__()
+
         self.q_pend = q_pend
         self.q_yield = q_yield
         self.args = args
@@ -22,7 +40,7 @@ class MpWorker(object):
 
         self.log = self._log_disabled if args.q and not args.lo else self._log_enabled
 
-        self.retpend = {}
+        self.retpend: dict[int, Any] = {}
         self.retpend_mutex = threading.Lock()
         self.mutex = threading.Lock()
 
@@ -45,20 +63,20 @@ class MpWorker(object):
         thr.start()
         thr.join()
 
-    def signal_handler(self, sig, frame):
+    def signal_handler(self, sig: Optional[int], frame: Optional[FrameType]) -> None:
         # print('k')
         pass
 
-    def _log_enabled(self, src, msg, c=0):
+    def _log_enabled(self, src: str, msg: str, c: Union[int, str] = 0) -> None:
-        self.q_yield.put([0, "log", [src, msg, c]])
+        self.q_yield.put((0, "log", [src, msg, c]))
 
-    def _log_disabled(self, src, msg, c=0):
+    def _log_disabled(self, src: str, msg: str, c: Union[int, str] = 0) -> None:
         pass
 
-    def logw(self, msg, c=0):
+    def logw(self, msg: str, c: Union[int, str] = 0) -> None:
         self.log("mp{}".format(self.n), msg, c)
 
-    def main(self):
+    def main(self) -> None:
         while True:
             retq_id, dest, args = self.q_pend.get()
 
@@ -87,15 +105,14 @@ class MpWorker(object):
             else:
                 raise Exception("what is " + str(dest))
 
-    def put(self, want_retval, dest, *args):
+    def ask(self, dest: str, *args: Any) -> ExceptionalQueue:
-        if want_retval:
+        retq = ExceptionalQueue(1)
-            retq = ExceptionalQueue(1)
+        retq_id = id(retq)
-            retq_id = id(retq)
+        with self.retpend_mutex:
-            with self.retpend_mutex:
+            self.retpend[retq_id] = retq
-                self.retpend[retq_id] = retq
-        else:
-            retq = None
-            retq_id = 0
 
-        self.q_yield.put([retq_id, dest, args])
+        self.q_yield.put((retq_id, dest, list(args)))
         return retq
+
+    def say(self, dest: str, *args: Any) -> None:
+        self.q_yield.put((0, dest, list(args)))

copyparty/broker_thr.py

@@ -3,14 +3,25 @@ from __future__ import print_function, unicode_literals
 
 import threading
 
+from .__init__ import TYPE_CHECKING
+from .broker_util import BrokerCli, ExceptionalQueue, try_exec
 from .httpsrv import HttpSrv
-from .broker_util import ExceptionalQueue, try_exec
+
+if TYPE_CHECKING:
+    from .svchub import SvcHub
+
+try:
+    from typing import Any
+except:
+    pass
 
 
-class BrokerThr(object):
+class BrokerThr(BrokerCli):
     """external api; behaves like BrokerMP but using plain threads"""
 
-    def __init__(self, hub):
+    def __init__(self, hub: "SvcHub") -> None:
+        super(BrokerThr, self).__init__()
+
         self.hub = hub
         self.log = hub.log
         self.args = hub.args
@@ -23,29 +34,35 @@ class BrokerThr(object):
         self.httpsrv = HttpSrv(self, None)
         self.reload = self.noop
 
-    def shutdown(self):
+    def shutdown(self) -> None:
         # self.log("broker", "shutting down")
         self.httpsrv.shutdown()
 
-    def noop(self):
+    def noop(self) -> None:
         pass
 
-    def put(self, want_retval, dest, *args):
+    def ask(self, dest: str, *args: Any) -> ExceptionalQueue:
+
+        # new ipc invoking managed service in hub
+        obj = self.hub
+        for node in dest.split("."):
+            obj = getattr(obj, node)
+
+        rv = try_exec(True, obj, *args)
+
+        # pretend we're broker_mp
+        retq = ExceptionalQueue(1)
+        retq.put(rv)
+        return retq
+
+    def say(self, dest: str, *args: Any) -> None:
         if dest == "listen":
             self.httpsrv.listen(args[0], 1)
+            return
 
-        else:
+        # new ipc invoking managed service in hub
-            # new ipc invoking managed service in hub
+        obj = self.hub
-            obj = self.hub
+        for node in dest.split("."):
-            for node in dest.split("."):
+            obj = getattr(obj, node)
-                obj = getattr(obj, node)
 
-            # TODO will deadlock if dest performs another ipc
+        try_exec(False, obj, *args)
-            rv = try_exec(want_retval, obj, *args)
-            if not want_retval:
-                return
-
-            # pretend we're broker_mp
-            retq = ExceptionalQueue(1)
-            retq.put(rv)
-            return retq

copyparty/broker_util.py

@@ -1,14 +1,28 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
-
+import argparse
 import traceback
 
-from .util import Pebkac, Queue
+from queue import Queue
+
+from .__init__ import TYPE_CHECKING
+from .authsrv import AuthSrv
+from .util import Pebkac
+
+try:
+    from typing import Any, Optional, Union
+
+    from .util import RootLogger
+except:
+    pass
+
+if TYPE_CHECKING:
+    from .httpsrv import HttpSrv
 
 
 class ExceptionalQueue(Queue, object):
-    def get(self, block=True, timeout=None):
+    def get(self, block: bool = True, timeout: Optional[float] = None) -> Any:
         rv = super(ExceptionalQueue, self).get(block, timeout)
 
         if isinstance(rv, list):
@@ -21,7 +35,26 @@ class ExceptionalQueue(Queue, object):
         return rv
 
 
-def try_exec(want_retval, func, *args):
+class BrokerCli(object):
+    """
+    helps mypy understand httpsrv.broker but still fails a few levels deeper,
+    for example resolving httpconn.* in httpcli -- see lines tagged #mypy404
+    """
+
+    def __init__(self) -> None:
+        self.log: RootLogger = None
+        self.args: argparse.Namespace = None
+        self.asrv: AuthSrv = None
+        self.httpsrv: "HttpSrv" = None
+
+    def ask(self, dest: str, *args: Any) -> ExceptionalQueue:
+        return ExceptionalQueue(1)
+
+    def say(self, dest: str, *args: Any) -> None:
+        pass
+
+
+def try_exec(want_retval: Union[bool, int], func: Any, *args: list[Any]) -> Any:
     try:
         return func(*args)
 

copyparty/ftpd.py

@@ -1,23 +1,23 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
-import os
-import sys
-import stat
-import time
-import logging
 import argparse
+import logging
+import os
+import stat
+import sys
 import threading
+import time
 
-from pyftpdlib.authorizers import DummyAuthorizer, AuthenticationFailed
+from pyftpdlib.authorizers import AuthenticationFailed, DummyAuthorizer
 from pyftpdlib.filesystems import AbstractedFS, FilesystemError
 from pyftpdlib.handlers import FTPHandler
-from pyftpdlib.servers import FTPServer
 from pyftpdlib.log import config_logging
+from pyftpdlib.servers import FTPServer
 
-from .__init__ import E, PY2
+from .__init__ import PY2, TYPE_CHECKING, E
-from .util import Pebkac, fsenc, exclude_dotfiles
 from .bos import bos
+from .util import Pebkac, exclude_dotfiles, fsenc
 
 try:
     from pyftpdlib.ioloop import IOLoop
@@ -28,58 +28,63 @@ except ImportError:
     from pyftpdlib.ioloop import IOLoop
 
 
-try:
+if TYPE_CHECKING:
-    from typing import TYPE_CHECKING
+    from .svchub import SvcHub
 
-    if TYPE_CHECKING:
+try:
-        from .svchub import SvcHub
+    import typing
-except ImportError:
+    from typing import Any, Optional
+except:
     pass
 
 
 class FtpAuth(DummyAuthorizer):
-    def __init__(self):
+    def __init__(self, hub: "SvcHub") -> None:
         super(FtpAuth, self).__init__()
-        self.hub = None  # type: SvcHub
+        self.hub = hub
 
-    def validate_authentication(self, username, password, handler):
+    def validate_authentication(
+        self, username: str, password: str, handler: Any
+    ) -> None:
         asrv = self.hub.asrv
         if username == "anonymous":
             password = ""
 
         uname = "*"
         if password:
-            uname = asrv.iacct.get(password, None)
+            uname = asrv.iacct.get(password, "")
 
         handler.username = uname
 
         if password and not uname:
             raise AuthenticationFailed("Authentication failed.")
 
-    def get_home_dir(self, username):
+    def get_home_dir(self, username: str) -> str:
         return "/"
 
-    def has_user(self, username):
+    def has_user(self, username: str) -> bool:
         asrv = self.hub.asrv
         return username in asrv.acct
 
-    def has_perm(self, username, perm, path=None):
+    def has_perm(self, username: str, perm: int, path: Optional[str] = None) -> bool:
         return True  # handled at filesystem layer
 
-    def get_perms(self, username):
+    def get_perms(self, username: str) -> str:
         return "elradfmwMT"
 
-    def get_msg_login(self, username):
+    def get_msg_login(self, username: str) -> str:
         return "sup {}".format(username)
 
-    def get_msg_quit(self, username):
+    def get_msg_quit(self, username: str) -> str:
         return "cya"
 
 
 class FtpFs(AbstractedFS):
-    def __init__(self, root, cmd_channel):
+    def __init__(
+        self, root: str, cmd_channel: Any
+    ) -> None:  # pylint: disable=super-init-not-called
         self.h = self.cmd_channel = cmd_channel  # type: FTPHandler
-        self.hub = cmd_channel.hub  # type: SvcHub
+        self.hub: "SvcHub" = cmd_channel.hub
         self.args = cmd_channel.args
 
         self.uname = self.hub.asrv.iacct.get(cmd_channel.password, "*")
@@ -90,7 +95,14 @@ class FtpFs(AbstractedFS):
         self.listdirinfo = self.listdir
         self.chdir(".")
 
-    def v2a(self, vpath, r=False, w=False, m=False, d=False):
+    def v2a(
+        self,
+        vpath: str,
+        r: bool = False,
+        w: bool = False,
+        m: bool = False,
+        d: bool = False,
+    ) -> str:
         try:
             vpath = vpath.replace("\\", "/").lstrip("/")
             vfs, rem = self.hub.asrv.vfs.get(vpath, self.uname, r, w, m, d)
@@ -101,25 +113,32 @@ class FtpFs(AbstractedFS):
         except Pebkac as ex:
             raise FilesystemError(str(ex))
 
-    def rv2a(self, vpath, r=False, w=False, m=False, d=False):
+    def rv2a(
+        self,
+        vpath: str,
+        r: bool = False,
+        w: bool = False,
+        m: bool = False,
+        d: bool = False,
+    ) -> str:
         return self.v2a(os.path.join(self.cwd, vpath), r, w, m, d)
 
-    def ftp2fs(self, ftppath):
+    def ftp2fs(self, ftppath: str) -> str:
         # return self.v2a(ftppath)
         return ftppath  # self.cwd must be vpath
 
-    def fs2ftp(self, fspath):
+    def fs2ftp(self, fspath: str) -> str:
         # raise NotImplementedError()
         return fspath
 
-    def validpath(self, path):
+    def validpath(self, path: str) -> bool:
         if "/.hist/" in path:
             if "/up2k." in path or path.endswith("/dir.txt"):
                 raise FilesystemError("access to this file is forbidden")
 
         return True
 
-    def open(self, filename, mode):
+    def open(self, filename: str, mode: str) -> typing.IO[Any]:
         r = "r" in mode
         w = "w" in mode or "a" in mode or "+" in mode
 
@@ -130,24 +149,24 @@ class FtpFs(AbstractedFS):
         self.validpath(ap)
         return open(fsenc(ap), mode)
 
-    def chdir(self, path):
+    def chdir(self, path: str) -> None:
         self.cwd = join(self.cwd, path)
         x = self.hub.asrv.vfs.can_access(self.cwd.lstrip("/"), self.h.username)
         self.can_read, self.can_write, self.can_move, self.can_delete, self.can_get = x
 
-    def mkdir(self, path):
+    def mkdir(self, path: str) -> None:
         ap = self.rv2a(path, w=True)
         bos.mkdir(ap)
 
-    def listdir(self, path):
+    def listdir(self, path: str) -> list[str]:
         vpath = join(self.cwd, path).lstrip("/")
         try:
             vfs, rem = self.hub.asrv.vfs.get(vpath, self.uname, True, False)
 
-            fsroot, vfs_ls, vfs_virt = vfs.ls(
+            fsroot, vfs_ls1, vfs_virt = vfs.ls(
                 rem, self.uname, not self.args.no_scandir, [[True], [False, True]]
             )
-            vfs_ls = [x[0] for x in vfs_ls]
+            vfs_ls = [x[0] for x in vfs_ls1]
             vfs_ls.extend(vfs_virt.keys())
 
             if not self.args.ed:
@@ -164,11 +183,11 @@ class FtpFs(AbstractedFS):
             r = {x.split("/")[0]: 1 for x in self.hub.asrv.vfs.all_vols.keys()}
             return list(sorted(list(r.keys())))
 
-    def rmdir(self, path):
+    def rmdir(self, path: str) -> None:
         ap = self.rv2a(path, d=True)
         bos.rmdir(ap)
 
-    def remove(self, path):
+    def remove(self, path: str) -> None:
         if self.args.no_del:
             raise FilesystemError("the delete feature is disabled in server config")
 
@@ -178,13 +197,13 @@ class FtpFs(AbstractedFS):
         except Exception as ex:
             raise FilesystemError(str(ex))
 
-    def rename(self, src, dst):
+    def rename(self, src: str, dst: str) -> None:
         if not self.can_move:
             raise FilesystemError("not allowed for user " + self.h.username)
 
         if self.args.no_mv:
-            m = "the rename/move feature is disabled in server config"
+            t = "the rename/move feature is disabled in server config"
-            raise FilesystemError(m)
+            raise FilesystemError(t)
 
         svp = join(self.cwd, src).lstrip("/")
         dvp = join(self.cwd, dst).lstrip("/")
@@ -193,10 +212,10 @@ class FtpFs(AbstractedFS):
         except Exception as ex:
             raise FilesystemError(str(ex))
 
-    def chmod(self, path, mode):
+    def chmod(self, path: str, mode: str) -> None:
         pass
 
-    def stat(self, path):
+    def stat(self, path: str) -> os.stat_result:
         try:
             ap = self.rv2a(path, r=True)
             return bos.stat(ap)
@@ -208,59 +227,59 @@ class FtpFs(AbstractedFS):
 
             return st
 
-    def utime(self, path, timeval):
+    def utime(self, path: str, timeval: float) -> None:
         ap = self.rv2a(path, w=True)
         return bos.utime(ap, (timeval, timeval))
 
-    def lstat(self, path):
+    def lstat(self, path: str) -> os.stat_result:
         ap = self.rv2a(path)
         return bos.lstat(ap)
 
-    def isfile(self, path):
+    def isfile(self, path: str) -> bool:
         st = self.stat(path)
         return stat.S_ISREG(st.st_mode)
 
-    def islink(self, path):
+    def islink(self, path: str) -> bool:
         ap = self.rv2a(path)
         return bos.path.islink(ap)
 
-    def isdir(self, path):
+    def isdir(self, path: str) -> bool:
         try:
             st = self.stat(path)
             return stat.S_ISDIR(st.st_mode)
         except:
             return True
 
-    def getsize(self, path):
+    def getsize(self, path: str) -> int:
         ap = self.rv2a(path)
         return bos.path.getsize(ap)
 
-    def getmtime(self, path):
+    def getmtime(self, path: str) -> float:
         ap = self.rv2a(path)
         return bos.path.getmtime(ap)
 
-    def realpath(self, path):
+    def realpath(self, path: str) -> str:
         return path
 
-    def lexists(self, path):
+    def lexists(self, path: str) -> bool:
         ap = self.rv2a(path)
         return bos.path.lexists(ap)
 
-    def get_user_by_uid(self, uid):
+    def get_user_by_uid(self, uid: int) -> str:
         return "root"
 
-    def get_group_by_uid(self, gid):
+    def get_group_by_uid(self, gid: int) -> str:
         return "root"
 
 
 class FtpHandler(FTPHandler):
     abstracted_fs = FtpFs
-    hub = None  # type: SvcHub
+    hub: "SvcHub" = None
-    args = None  # type: argparse.Namespace
+    args: argparse.Namespace = None
 
-    def __init__(self, conn, server, ioloop=None):
+    def __init__(self, conn: Any, server: Any, ioloop: Any = None) -> None:
-        self.hub = FtpHandler.hub  # type: SvcHub
+        self.hub: "SvcHub" = FtpHandler.hub
-        self.args = FtpHandler.args  # type: argparse.Namespace
+        self.args: argparse.Namespace = FtpHandler.args
 
         if PY2:
             FTPHandler.__init__(self, conn, server, ioloop)
@@ -268,9 +287,10 @@ class FtpHandler(FTPHandler):
             super(FtpHandler, self).__init__(conn, server, ioloop)
 
         # abspath->vpath mapping to resolve log_transfer paths
-        self.vfs_map = {}
+        self.vfs_map: dict[str, str] = {}
 
-    def ftp_STOR(self, file, mode="w"):
+    def ftp_STOR(self, file: str, mode: str = "w") -> Any:
+        # Optional[str]
         vp = join(self.fs.cwd, file).lstrip("/")
         ap = self.fs.v2a(vp)
         self.vfs_map[ap] = vp
@@ -279,7 +299,16 @@ class FtpHandler(FTPHandler):
         # print("ftp_STOR: {} {} OK".format(vp, mode))
         return ret
 
-    def log_transfer(self, cmd, filename, receive, completed, elapsed, bytes):
+    def log_transfer(
+        self,
+        cmd: str,
+        filename: bytes,
+        receive: bool,
+        completed: bool,
+        elapsed: float,
+        bytes: int,
+    ) -> Any:
+        # None
         ap = filename.decode("utf-8", "replace")
         vp = self.vfs_map.pop(ap, None)
         # print("xfer_end: {} => {}".format(ap, vp))
@@ -312,7 +341,7 @@ except:
 
 
 class Ftpd(object):
-    def __init__(self, hub):
+    def __init__(self, hub: "SvcHub") -> None:
         self.hub = hub
         self.args = hub.args
 
@@ -321,24 +350,23 @@ class Ftpd(object):
             hs.append([FtpHandler, self.args.ftp])
         if self.args.ftps:
             try:
-                h = SftpHandler
+                h1 = SftpHandler
             except:
-                m = "\nftps requires pyopenssl;\nplease run the following:\n\n  {} -m pip install --user pyopenssl\n"
+                t = "\nftps requires pyopenssl;\nplease run the following:\n\n  {} -m pip install --user pyopenssl\n"
-                print(m.format(sys.executable))
+                print(t.format(sys.executable))
                 sys.exit(1)
 
-            h.certfile = os.path.join(E.cfg, "cert.pem")
+            h1.certfile = os.path.join(E.cfg, "cert.pem")
-            h.tls_control_required = True
+            h1.tls_control_required = True
-            h.tls_data_required = True
+            h1.tls_data_required = True
 
-            hs.append([h, self.args.ftps])
+            hs.append([h1, self.args.ftps])
 
-        for h in hs:
+        for h_lp in hs:
-            h, lp = h
+            h2, lp = h_lp
-            h.hub = hub
+            h2.hub = hub
-            h.args = hub.args
+            h2.args = hub.args
-            h.authorizer = FtpAuth()
+            h2.authorizer = FtpAuth(hub)
-            h.authorizer.hub = hub
 
             if self.args.ftp_pr:
                 p1, p2 = [int(x) for x in self.args.ftp_pr.split("-")]
@@ -350,10 +378,10 @@ class Ftpd(object):
                     else:
                         p1 += d + 1
 
-                h.passive_ports = list(range(p1, p2 + 1))
+                h2.passive_ports = list(range(p1, p2 + 1))
 
             if self.args.ftp_nat:
-                h.masquerade_address = self.args.ftp_nat
+                h2.masquerade_address = self.args.ftp_nat
 
         if self.args.ftp_dbg:
             config_logging(level=logging.DEBUG)
@@ -363,11 +391,11 @@ class Ftpd(object):
             for h, lp in hs:
                 FTPServer((ip, int(lp)), h, ioloop)
 
-        t = threading.Thread(target=ioloop.loop)
+        thr = threading.Thread(target=ioloop.loop)
-        t.daemon = True
+        thr.daemon = True
-        t.start()
+        thr.start()
 
 
-def join(p1, p2):
+def join(p1: str, p2: str) -> str:
     w = os.path.join(p1, p2.replace("\\", "/"))
     return os.path.normpath(w).replace("\\", "/")

copyparty/httpconn.py

@@ -1,25 +1,36 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
-import re
+import argparse  # typechk
 import os
-import time
+import re
 import socket
+import threading  # typechk
+import time
 
-HAVE_SSL = True
 try:
+    HAVE_SSL = True
     import ssl
 except:
     HAVE_SSL = False
 
-from .__init__ import E
+from . import util as Util
-from .util import Unrecv
+from .__init__ import TYPE_CHECKING, E
+from .authsrv import AuthSrv  # typechk
 from .httpcli import HttpCli
-from .u2idx import U2idx
+from .ico import Ico
+from .mtag import HAVE_FFMPEG
 from .th_cli import ThumbCli
 from .th_srv import HAVE_PIL, HAVE_VIPS
-from .mtag import HAVE_FFMPEG
+from .u2idx import U2idx
-from .ico import Ico
+
+try:
+    from typing import Optional, Pattern, Union
+except:
+    pass
+
+if TYPE_CHECKING:
+    from .httpsrv import HttpSrv
 
 
 class HttpConn(object):
@@ -28,32 +39,37 @@ class HttpConn(object):
     creates an HttpCli for each request (Connection: Keep-Alive)
     """
 
-    def __init__(self, sck, addr, hsrv):
+    def __init__(
+        self, sck: socket.socket, addr: tuple[str, int], hsrv: "HttpSrv"
+    ) -> None:
         self.s = sck
-        self.sr = None  # Type: Unrecv
+        self.sr: Optional[Util._Unrecv] = None
         self.addr = addr
         self.hsrv = hsrv
 
-        self.mutex = hsrv.mutex
+        self.mutex: threading.Lock = hsrv.mutex  # mypy404
-        self.args = hsrv.args
+        self.args: argparse.Namespace = hsrv.args  # mypy404
-        self.asrv = hsrv.asrv
+        self.asrv: AuthSrv = hsrv.asrv  # mypy404
         self.cert_path = hsrv.cert_path
-        self.u2fh = hsrv.u2fh
+        self.u2fh: Util.FHC = hsrv.u2fh  # mypy404
 
         enth = (HAVE_PIL or HAVE_VIPS or HAVE_FFMPEG) and not self.args.no_thumb
-        self.thumbcli = ThumbCli(hsrv) if enth else None
+        self.thumbcli: Optional[ThumbCli] = ThumbCli(hsrv) if enth else None  # mypy404
-        self.ico = Ico(self.args)
+        self.ico: Ico = Ico(self.args)  # mypy404
 
-        self.t0 = time.time()
+        self.t0: float = time.time()  # mypy404
         self.stopping = False
-        self.nreq = 0
+        self.nreq: int = 0  # mypy404
-        self.nbyte = 0
+        self.nbyte: int = 0  # mypy404
-        self.u2idx = None
+        self.u2idx: Optional[U2idx] = None
-        self.log_func = hsrv.log
+        self.log_func: Util.RootLogger = hsrv.log  # mypy404
-        self.lf_url = re.compile(self.args.lf_url) if self.args.lf_url else None
+        self.log_src: str = "httpconn"  # mypy404
+        self.lf_url: Optional[Pattern[str]] = (
+            re.compile(self.args.lf_url) if self.args.lf_url else None
+        )  # mypy404
         self.set_rproxy()
 
-    def shutdown(self):
+    def shutdown(self) -> None:
         self.stopping = True
         try:
             self.s.shutdown(socket.SHUT_RDWR)
@@ -61,7 +77,7 @@ class HttpConn(object):
         except:
             pass
 
-    def set_rproxy(self, ip=None):
+    def set_rproxy(self, ip: Optional[str] = None) -> str:
         if ip is None:
             color = 36
             ip = self.addr[0]
@@ -74,35 +90,35 @@ class HttpConn(object):
         self.log_src = "{} \033[{}m{}".format(ip, color, self.addr[1]).ljust(26)
         return self.log_src
 
-    def respath(self, res_name):
+    def respath(self, res_name: str) -> str:
         return os.path.join(E.mod, "web", res_name)
 
-    def log(self, msg, c=0):
+    def log(self, msg: str, c: Union[int, str] = 0) -> None:
         self.log_func(self.log_src, msg, c)
 
-    def get_u2idx(self):
+    def get_u2idx(self) -> U2idx:
         if not self.u2idx:
             self.u2idx = U2idx(self)
 
         return self.u2idx
 
-    def _detect_https(self):
+    def _detect_https(self) -> bool:
         method = None
         if self.cert_path:
             try:
                 method = self.s.recv(4, socket.MSG_PEEK)
             except socket.timeout:
-                return
+                return False
             except AttributeError:
                 # jython does not support msg_peek; forget about https
                 method = self.s.recv(4)
-                self.sr = Unrecv(self.s, self.log)
+                self.sr = Util.Unrecv(self.s, self.log)
                 self.sr.buf = method
 
                 # jython used to do this, they stopped since it's broken
                 # but reimplementing sendall is out of scope for now
                 if not getattr(self.s, "sendall", None):
-                    self.s.sendall = self.s.send
+                    self.s.sendall = self.s.send  # type: ignore
 
             if len(method) != 4:
                 err = "need at least 4 bytes in the first packet; got {}".format(
@@ -112,17 +128,18 @@ class HttpConn(object):
                     self.log(err)
 
                 self.s.send(b"HTTP/1.1 400 Bad Request\r\n\r\n" + err.encode("utf-8"))
-                return
+                return False
 
         return method not in [None, b"GET ", b"HEAD", b"POST", b"PUT ", b"OPTI"]
 
-    def run(self):
+    def run(self) -> None:
         self.sr = None
         if self.args.https_only:
             is_https = True
         elif self.args.http_only or not HAVE_SSL:
             is_https = False
         else:
+            # raise Exception("asdf")
             is_https = self._detect_https()
 
         if is_https:
@@ -151,14 +168,15 @@ class HttpConn(object):
                 self.s = ctx.wrap_socket(self.s, server_side=True)
                 msg = [
                     "\033[1;3{:d}m{}".format(c, s)
-                    for c, s in zip([0, 5, 0], self.s.cipher())
+                    for c, s in zip([0, 5, 0], self.s.cipher())  # type: ignore
                 ]
                 self.log(" ".join(msg) + "\033[0m")
 
                 if self.args.ssl_dbg and hasattr(self.s, "shared_ciphers"):
-                    overlap = [y[::-1] for y in self.s.shared_ciphers()]
+                    ciphers = self.s.shared_ciphers()
-                    lines = [str(x) for x in (["TLS cipher overlap:"] + overlap)]
+                    assert ciphers
-                    self.log("\n".join(lines))
+                    overlap = [str(y[::-1]) for y in ciphers]
+                    self.log("TLS cipher overlap:" + "\n".join(overlap))
                     for k, v in [
                         ["compression", self.s.compression()],
                         ["ALPN proto", self.s.selected_alpn_protocol()],
@@ -183,7 +201,7 @@ class HttpConn(object):
                 return
 
         if not self.sr:
-            self.sr = Unrecv(self.s, self.log)
+            self.sr = Util.Unrecv(self.s, self.log)
 
         while not self.stopping:
             self.nreq += 1

copyparty/httpsrv.py

@@ -1,13 +1,15 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
-import os
-import sys
-import time
-import math
 import base64
+import math
+import os
 import socket
+import sys
 import threading
+import time
+
+import queue
 
 try:
     import jinja2
@@ -26,15 +28,18 @@ except ImportError:
     )
     sys.exit(1)
 
-from .__init__ import E, PY2, MACOS
+from .__init__ import MACOS, TYPE_CHECKING, E
-from .util import FHC, spack, min_ex, start_stackmon, start_log_thrs
 from .bos import bos
 from .httpconn import HttpConn
+from .util import FHC, min_ex, spack, start_log_thrs, start_stackmon
 
-if PY2:
+if TYPE_CHECKING:
-    import Queue as queue
+    from .broker_util import BrokerCli
-else:
+
-    import queue
+try:
+    from typing import Any, Optional
+except:
+    pass
 
 
 class HttpSrv(object):
@@ -43,7 +48,7 @@ class HttpSrv(object):
     relying on MpSrv for performance (HttpSrv is just plain threads)
     """
 
-    def __init__(self, broker, nid):
+    def __init__(self, broker: "BrokerCli", nid: Optional[int]) -> None:
         self.broker = broker
         self.nid = nid
         self.args = broker.args
@@ -58,17 +63,19 @@ class HttpSrv(object):
 
         self.tp_nthr = 0  # actual
         self.tp_ncli = 0  # fading
-        self.tp_time = None  # latest worker collect
+        self.tp_time = 0.0  # latest worker collect
-        self.tp_q = None if self.args.no_htp else queue.LifoQueue()
+        self.tp_q: Optional[queue.LifoQueue[Any]] = (
-        self.t_periodic = None
+            None if self.args.no_htp else queue.LifoQueue()
+        )
+        self.t_periodic: Optional[threading.Thread] = None
 
         self.u2fh = FHC()
-        self.srvs = []
+        self.srvs: list[socket.socket] = []
         self.ncli = 0  # exact
-        self.clients = {}  # laggy
+        self.clients: set[HttpConn] = set()  # laggy
         self.nclimax = 0
-        self.cb_ts = 0
+        self.cb_ts = 0.0
-        self.cb_v = 0
+        self.cb_v = ""
 
         env = jinja2.Environment()
         env.loader = jinja2.FileSystemLoader(os.path.join(E.mod, "web"))
@@ -82,7 +89,7 @@ class HttpSrv(object):
         if bos.path.exists(cert_path):
             self.cert_path = cert_path
         else:
-            self.cert_path = None
+            self.cert_path = ""
 
         if self.tp_q:
             self.start_threads(4)
@@ -94,19 +101,19 @@ class HttpSrv(object):
             if self.args.log_thrs:
                 start_log_thrs(self.log, self.args.log_thrs, nid)
 
-        self.th_cfg = {}  # type: dict[str, Any]
+        self.th_cfg: dict[str, Any] = {}
         t = threading.Thread(target=self.post_init)
         t.daemon = True
         t.start()
 
-    def post_init(self):
+    def post_init(self) -> None:
         try:
-            x = self.broker.put(True, "thumbsrv.getcfg")
+            x = self.broker.ask("thumbsrv.getcfg")
             self.th_cfg = x.get()
         except:
             pass
 
-    def start_threads(self, n):
+    def start_threads(self, n: int) -> None:
         self.tp_nthr += n
         if self.args.log_htp:
             self.log(self.name, "workers += {} = {}".format(n, self.tp_nthr), 6)
@@ -119,15 +126,16 @@ class HttpSrv(object):
             thr.daemon = True
             thr.start()
 
-    def stop_threads(self, n):
+    def stop_threads(self, n: int) -> None:
         self.tp_nthr -= n
         if self.args.log_htp:
             self.log(self.name, "workers -= {} = {}".format(n, self.tp_nthr), 6)
 
+        assert self.tp_q
         for _ in range(n):
             self.tp_q.put(None)
 
-    def periodic(self):
+    def periodic(self) -> None:
         while True:
             time.sleep(2 if self.tp_ncli or self.ncli else 10)
             with self.mutex:
@@ -141,7 +149,7 @@ class HttpSrv(object):
                     self.t_periodic = None
                     return
 
-    def listen(self, sck, nlisteners):
+    def listen(self, sck: socket.socket, nlisteners: int) -> None:
         ip, port = sck.getsockname()
         self.srvs.append(sck)
         self.nclimax = math.ceil(self.args.nc * 1.0 / nlisteners)
@@ -153,15 +161,15 @@ class HttpSrv(object):
         t.daemon = True
         t.start()
 
-    def thr_listen(self, srv_sck):
+    def thr_listen(self, srv_sck: socket.socket) -> None:
         """listens on a shared tcp server"""
         ip, port = srv_sck.getsockname()
         fno = srv_sck.fileno()
         msg = "subscribed @ {}:{}  f{}".format(ip, port, fno)
         self.log(self.name, msg)
 
-        def fun():
+        def fun() -> None:
-            self.broker.put(False, "cb_httpsrv_up")
+            self.broker.say("cb_httpsrv_up")
 
         threading.Thread(target=fun).start()
 
@@ -185,21 +193,21 @@ class HttpSrv(object):
                 continue
 
             if self.args.log_conn:
-                m = "|{}C-acc2 \033[0;36m{} \033[3{}m{}".format(
+                t = "|{}C-acc2 \033[0;36m{} \033[3{}m{}".format(
                     "-" * 3, ip, port % 8, port
                 )
-                self.log("%s %s" % addr, m, c="1;30")
+                self.log("%s %s" % addr, t, c="1;30")
 
             self.accept(sck, addr)
 
-    def accept(self, sck, addr):
+    def accept(self, sck: socket.socket, addr: tuple[str, int]) -> None:
         """takes an incoming tcp connection and creates a thread to handle it"""
         now = time.time()
 
         if now - (self.tp_time or now) > 300:
-            m = "httpserver threadpool died: tpt {:.2f}, now {:.2f}, nthr {}, ncli {}"
+            t = "httpserver threadpool died: tpt {:.2f}, now {:.2f}, nthr {}, ncli {}"
-            self.log(self.name, m.format(self.tp_time, now, self.tp_nthr, self.ncli), 1)
+            self.log(self.name, t.format(self.tp_time, now, self.tp_nthr, self.ncli), 1)
-            self.tp_time = None
+            self.tp_time = 0
             self.tp_q = None
 
         with self.mutex:
@@ -209,10 +217,10 @@ class HttpSrv(object):
                 if self.nid:
                     name += "-{}".format(self.nid)
 
-                t = threading.Thread(target=self.periodic, name=name)
+                thr = threading.Thread(target=self.periodic, name=name)
-                self.t_periodic = t
+                self.t_periodic = thr
-                t.daemon = True
+                thr.daemon = True
-                t.start()
+                thr.start()
 
             if self.tp_q:
                 self.tp_time = self.tp_time or now
@@ -224,8 +232,8 @@ class HttpSrv(object):
                 return
 
         if not self.args.no_htp:
-            m = "looks like the httpserver threadpool died; please make an issue on github and tell me the story of how you pulled that off, thanks and dog bless\n"
+            t = "looks like the httpserver threadpool died; please make an issue on github and tell me the story of how you pulled that off, thanks and dog bless\n"
-            self.log(self.name, m, 1)
+            self.log(self.name, t, 1)
 
         thr = threading.Thread(
             target=self.thr_client,
@@ -235,14 +243,15 @@ class HttpSrv(object):
         thr.daemon = True
         thr.start()
 
-    def thr_poolw(self):
+    def thr_poolw(self) -> None:
+        assert self.tp_q
         while True:
             task = self.tp_q.get()
             if not task:
                 break
 
             with self.mutex:
-                self.tp_time = None
+                self.tp_time = 0
 
             try:
                 sck, addr = task
@@ -255,7 +264,7 @@ class HttpSrv(object):
             except:
                 self.log(self.name, "thr_client: " + min_ex(), 3)
 
-    def shutdown(self):
+    def shutdown(self) -> None:
         self.stopping = True
         for srv in self.srvs:
             try:
@@ -263,7 +272,7 @@ class HttpSrv(object):
             except:
                 pass
 
-        clients = list(self.clients.keys())
+        clients = list(self.clients)
         for cli in clients:
             try:
                 cli.shutdown()
@@ -279,13 +288,13 @@ class HttpSrv(object):
 
         self.log(self.name, "ok bye")
 
-    def thr_client(self, sck, addr):
+    def thr_client(self, sck: socket.socket, addr: tuple[str, int]) -> None:
         """thread managing one tcp client"""
         sck.settimeout(120)
 
         cli = HttpConn(sck, addr, self)
         with self.mutex:
-            self.clients[cli] = 0
+            self.clients.add(cli)
 
         fno = sck.fileno()
         try:
@@ -328,10 +337,10 @@ class HttpSrv(object):
                     raise
             finally:
                 with self.mutex:
-                    del self.clients[cli]
+                    self.clients.remove(cli)
                     self.ncli -= 1
 
-    def cachebuster(self):
+    def cachebuster(self) -> str:
         if time.time() - self.cb_ts < 1:
             return self.cb_v
 

copyparty/ico.py

@@ -1,28 +1,28 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
-import hashlib
+import argparse  # typechk
 import colorsys
+import hashlib
 
 from .__init__ import PY2
 
 
 class Ico(object):
-    def __init__(self, args):
+    def __init__(self, args: argparse.Namespace) -> None:
         self.args = args
 
-    def get(self, ext, as_thumb):
+    def get(self, ext: str, as_thumb: bool) -> tuple[str, bytes]:
         """placeholder to make thumbnails not break"""
 
-        h = hashlib.md5(ext.encode("utf-8")).digest()[:2]
+        zb = hashlib.md5(ext.encode("utf-8")).digest()[:2]
         if PY2:
-            h = [ord(x) for x in h]
+            zb = [ord(x) for x in zb]
 
-        c1 = colorsys.hsv_to_rgb(h[0] / 256.0, 1, 0.3)
+        c1 = colorsys.hsv_to_rgb(zb[0] / 256.0, 1, 0.3)
-        c2 = colorsys.hsv_to_rgb(h[0] / 256.0, 1, 1)
+        c2 = colorsys.hsv_to_rgb(zb[0] / 256.0, 1, 1)
-        c = list(c1) + list(c2)
+        ci = [int(x * 255) for x in list(c1) + list(c2)]
-        c = [int(x * 255) for x in c]
+        c = "".join(["{:02x}".format(x) for x in ci])
-        c = "".join(["{:02x}".format(x) for x in c])
 
         h = 30
         if not self.args.th_no_crop and as_thumb:
@@ -37,6 +37,6 @@ class Ico(object):
   fill="#{}" font-family="monospace" font-size="14px" style="letter-spacing:.5px">{}</text>
 </g></svg>
 """
-        svg = svg.format(h, c[:6], c[6:], ext).encode("utf-8")
+        svg = svg.format(h, c[:6], c[6:], ext)
 
-        return ["image/svg+xml", svg]
+        return "image/svg+xml", svg.encode("utf-8")

copyparty/mtag.py

@@ -1,18 +1,26 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
-import os
+import argparse
-import sys
 import json
+import os
 import shutil
 import subprocess as sp
+import sys
 
 from .__init__ import PY2, WINDOWS, unicode
-from .util import fsenc, uncyg, runcmd, retchk, REKOBO_LKEY
 from .bos import bos
+from .util import REKOBO_LKEY, fsenc, retchk, runcmd, uncyg
+
+try:
+    from typing import Any, Union
+
+    from .util import RootLogger
+except:
+    pass
 
 
-def have_ff(cmd):
+def have_ff(cmd: str) -> bool:
     if PY2:
         print("# checking {}".format(cmd))
         cmd = (cmd + " -version").encode("ascii").split(b" ")
@@ -30,7 +38,7 @@ HAVE_FFPROBE = have_ff("ffprobe")
 
 
 class MParser(object):
-    def __init__(self, cmdline):
+    def __init__(self, cmdline: str) -> None:
         self.tag, args = cmdline.split("=", 1)
         self.tags = self.tag.split(",")
 
@@ -73,7 +81,9 @@ class MParser(object):
             raise Exception()
 
 
-def ffprobe(abspath, timeout=10):
+def ffprobe(
+    abspath: str, timeout: int = 10
+) -> tuple[dict[str, tuple[int, Any]], dict[str, list[Any]]]:
     cmd = [
         b"ffprobe",
         b"-hide_banner",
@@ -87,15 +97,15 @@ def ffprobe(abspath, timeout=10):
     return parse_ffprobe(so)
 
 
-def parse_ffprobe(txt):
+def parse_ffprobe(txt: str) -> tuple[dict[str, tuple[int, Any]], dict[str, list[Any]]]:
     """ffprobe -show_format -show_streams"""
     streams = []
     fmt = {}
     g = {}
     for ln in [x.rstrip("\r") for x in txt.split("\n")]:
         try:
-            k, v = ln.split("=", 1)
+            sk, sv = ln.split("=", 1)
-            g[k] = v
+            g[sk] = sv
             continue
         except:
             pass
@@ -109,8 +119,8 @@ def parse_ffprobe(txt):
             fmt = g
 
     streams = [fmt] + streams
-    ret = {}  # processed
+    ret: dict[str, Any] = {}  # processed
-    md = {}  # raw tags
+    md: dict[str, list[Any]] = {}  # raw tags
 
     is_audio = fmt.get("format_name") in ["mp3", "ogg", "flac", "wav"]
     if fmt.get("filename", "").split(".")[-1].lower() in ["m4a", "aac"]:
@@ -161,43 +171,43 @@ def parse_ffprobe(txt):
             kvm = [["duration", ".dur"], ["bit_rate", ".q"]]
 
         for sk, rk in kvm:
-            v = strm.get(sk)
+            v1 = strm.get(sk)
-            if v is None:
+            if v1 is None:
                 continue
 
             if rk.startswith("."):
                 try:
-                    v = float(v)
+                    zf = float(v1)
                     v2 = ret.get(rk)
-                    if v2 is None or v > v2:
+                    if v2 is None or zf > v2:
-                        ret[rk] = v
+                        ret[rk] = zf
                 except:
                     # sqlite doesnt care but the code below does
-                    if v not in ["N/A"]:
+                    if v1 not in ["N/A"]:
-                        ret[rk] = v
+                        ret[rk] = v1
             else:
-                ret[rk] = v
+                ret[rk] = v1
 
     if ret.get("vc") == "ansi":  # shellscript
         return {}, {}
 
     for strm in streams:
-        for k, v in strm.items():
+        for sk, sv in strm.items():
-            if not k.startswith("TAG:"):
+            if not sk.startswith("TAG:"):
                 continue
 
-            k = k[4:].strip()
+            sk = sk[4:].strip()
-            v = v.strip()
+            sv = sv.strip()
-            if k and v and k not in md:
+            if sk and sv and sk not in md:
-                md[k] = [v]
+                md[sk] = [sv]
 
-    for k in [".q", ".vq", ".aq"]:
+    for sk in [".q", ".vq", ".aq"]:
-        if k in ret:
+        if sk in ret:
-            ret[k] /= 1000  # bit_rate=320000
+            ret[sk] /= 1000  # bit_rate=320000
 
-    for k in [".q", ".vq", ".aq", ".resw", ".resh"]:
+    for sk in [".q", ".vq", ".aq", ".resw", ".resh"]:
-        if k in ret:
+        if sk in ret:
-            ret[k] = int(ret[k])
+            ret[sk] = int(ret[sk])
 
     if ".fps" in ret:
         fps = ret[".fps"]
@@ -219,13 +229,13 @@ def parse_ffprobe(txt):
     if ".resw" in ret and ".resh" in ret:
         ret["res"] = "{}x{}".format(ret[".resw"], ret[".resh"])
 
-    ret = {k: [0, v] for k, v in ret.items()}
+    zd = {k: (0, v) for k, v in ret.items()}
 
-    return ret, md
+    return zd, md
 
 
 class MTag(object):
-    def __init__(self, log_func, args):
+    def __init__(self, log_func: RootLogger, args: argparse.Namespace) -> None:
         self.log_func = log_func
         self.args = args
         self.usable = True
@@ -242,7 +252,7 @@ class MTag(object):
         if self.backend == "mutagen":
             self.get = self.get_mutagen
             try:
-                import mutagen
+                import mutagen  # noqa: F401  # pylint: disable=unused-import,import-outside-toplevel
             except:
                 self.log("could not load Mutagen, trying FFprobe instead", c=3)
                 self.backend = "ffprobe"
@@ -339,31 +349,33 @@ class MTag(object):
         }
         # self.get = self.compare
 
-    def log(self, msg, c=0):
+    def log(self, msg: str, c: Union[int, str] = 0) -> None:
         self.log_func("mtag", msg, c)
 
-    def normalize_tags(self, ret, md):
+    def normalize_tags(
-        for k, v in dict(md).items():
+        self, parser_output: dict[str, tuple[int, Any]], md: dict[str, list[Any]]
-            if not v:
+    ) -> dict[str, Union[str, float]]:
+        for sk, tv in dict(md).items():
+            if not tv:
                 continue
 
-            k = k.lower().split("::")[0].strip()
+            sk = sk.lower().split("::")[0].strip()
-            mk = self.rmap.get(k)
+            key_mapping = self.rmap.get(sk)
-            if not mk:
+            if not key_mapping:
                 continue
 
-            pref, mk = mk
+            priority, alias = key_mapping
-            if mk not in ret or ret[mk][0] > pref:
+            if alias not in parser_output or parser_output[alias][0] > priority:
-                ret[mk] = [pref, v[0]]
+                parser_output[alias] = (priority, tv[0])
 
-        # take first value
+        # take first value (lowest priority / most preferred)
-        ret = {k: unicode(v[1]).strip() for k, v in ret.items()}
+        ret = {sk: unicode(tv[1]).strip() for sk, tv in parser_output.items()}
 
         # track 3/7 => track 3
-        for k, v in ret.items():
+        for sk, tv in ret.items():
-            if k[0] == ".":
+            if sk[0] == ".":
-                v = v.split("/")[0].strip().lstrip("0")
+                sv = str(tv).split("/")[0].strip().lstrip("0")
-                ret[k] = v or 0
+                ret[sk] = sv or 0
 
         # normalize key notation to rkeobo
         okey = ret.get("key")
@@ -373,7 +385,7 @@ class MTag(object):
 
         return ret
 
-    def compare(self, abspath):
+    def compare(self, abspath: str) -> dict[str, Union[str, float]]:
         if abspath.endswith(".au"):
             return {}
 
@@ -411,7 +423,7 @@ class MTag(object):
 
         return r1
 
-    def get_mutagen(self, abspath):
+    def get_mutagen(self, abspath: str) -> dict[str, Union[str, float]]:
         if not bos.path.isfile(abspath):
             return {}
 
@@ -425,7 +437,7 @@ class MTag(object):
             return self.get_ffprobe(abspath) if self.can_ffprobe else {}
 
         sz = bos.path.getsize(abspath)
-        ret = {".q": [0, int((sz / md.info.length) / 128)]}
+        ret = {".q": (0, int((sz / md.info.length) / 128))}
 
         for attr, k, norm in [
             ["codec", "ac", unicode],
@@ -456,24 +468,24 @@ class MTag(object):
             if k == "ac" and v.startswith("mp4a.40."):
                 v = "aac"
 
-            ret[k] = [0, norm(v)]
+            ret[k] = (0, norm(v))
 
         return self.normalize_tags(ret, md)
 
-    def get_ffprobe(self, abspath):
+    def get_ffprobe(self, abspath: str) -> dict[str, Union[str, float]]:
         if not bos.path.isfile(abspath):
             return {}
 
         ret, md = ffprobe(abspath)
         return self.normalize_tags(ret, md)
 
-    def get_bin(self, parsers, abspath):
+    def get_bin(self, parsers: dict[str, MParser], abspath: str) -> dict[str, Any]:
         if not bos.path.isfile(abspath):
             return {}
 
         pypath = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
-        pypath = [str(pypath)] + [str(x) for x in sys.path if x]
+        zsl = [str(pypath)] + [str(x) for x in sys.path if x]
-        pypath = str(os.pathsep.join(pypath))
+        pypath = str(os.pathsep.join(zsl))
         env = os.environ.copy()
         env["PYTHONPATH"] = pypath
 
@@ -491,9 +503,9 @@ class MTag(object):
                 else:
                     cmd = ["nice"] + cmd
 
-                cmd = [fsenc(x) for x in cmd]
+                bcmd = [fsenc(x) for x in cmd]
-                rc, v, err = runcmd(cmd, **args)
+                rc, v, err = runcmd(bcmd, **args)  # type: ignore
-                retchk(rc, cmd, err, self.log, 5, self.args.mtag_v)
+                retchk(rc, bcmd, err, self.log, 5, self.args.mtag_v)
                 v = v.strip()
                 if not v:
                     continue
@@ -501,10 +513,10 @@ class MTag(object):
                 if "," not in tagname:
                     ret[tagname] = v
                 else:
-                    v = json.loads(v)
+                    zj = json.loads(v)
                     for tag in tagname.split(","):
-                        if tag and tag in v:
+                        if tag and tag in zj:
-                            ret[tag] = v[tag]
+                            ret[tag] = zj[tag]
             except:
                 pass
 

copyparty/star.py

@@ -4,20 +4,29 @@ from __future__ import print_function, unicode_literals
 import tarfile
 import threading
 
-from .sutil import errdesc
+from queue import Queue
-from .util import Queue, fsenc, min_ex
+
 from .bos import bos
+from .sutil import StreamArc, errdesc
+from .util import fsenc, min_ex
+
+try:
+    from typing import Any, Generator, Optional
+
+    from .util import NamedLogger
+except:
+    pass
 
 
-class QFile(object):
+class QFile(object):  # inherit io.StringIO for painful typing
     """file-like object which buffers writes into a queue"""
 
-    def __init__(self):
+    def __init__(self) -> None:
-        self.q = Queue(64)
+        self.q: Queue[Optional[bytes]] = Queue(64)
-        self.bq = []
+        self.bq: list[bytes] = []
         self.nq = 0
 
-    def write(self, buf):
+    def write(self, buf: Optional[bytes]) -> None:
         if buf is None or self.nq >= 240 * 1024:
             self.q.put(b"".join(self.bq))
             self.bq = []
@@ -30,27 +39,32 @@ class QFile(object):
             self.nq += len(buf)
 
 
-class StreamTar(object):
+class StreamTar(StreamArc):
     """construct in-memory tar file from the given path"""
 
-    def __init__(self, log, fgen, **kwargs):
+    def __init__(
+        self,
+        log: NamedLogger,
+        fgen: Generator[dict[str, Any], None, None],
+        **kwargs: Any
+    ):
+        super(StreamTar, self).__init__(log, fgen)
+
         self.ci = 0
         self.co = 0
         self.qfile = QFile()
-        self.log = log
+        self.errf: dict[str, Any] = {}
-        self.fgen = fgen
-        self.errf = None
 
         # python 3.8 changed to PAX_FORMAT as default,
         # waste of space and don't care about the new features
         fmt = tarfile.GNU_FORMAT
-        self.tar = tarfile.open(fileobj=self.qfile, mode="w|", format=fmt)
+        self.tar = tarfile.open(fileobj=self.qfile, mode="w|", format=fmt)  # type: ignore
 
         w = threading.Thread(target=self._gen, name="star-gen")
         w.daemon = True
         w.start()
 
-    def gen(self):
+    def gen(self) -> Generator[Optional[bytes], None, None]:
         while True:
             buf = self.qfile.q.get()
             if not buf:
@@ -63,7 +77,7 @@ class StreamTar(object):
         if self.errf:
             bos.unlink(self.errf["ap"])
 
-    def ser(self, f):
+    def ser(self, f: dict[str, Any]) -> None:
         name = f["vp"]
         src = f["ap"]
         fsi = f["st"]
@@ -76,21 +90,21 @@ class StreamTar(object):
         inf.gid = 0
 
         self.ci += inf.size
-        with open(fsenc(src), "rb", 512 * 1024) as f:
+        with open(fsenc(src), "rb", 512 * 1024) as fo:
-            self.tar.addfile(inf, f)
+            self.tar.addfile(inf, fo)
 
-    def _gen(self):
+    def _gen(self) -> None:
         errors = []
         for f in self.fgen:
             if "err" in f:
-                errors.append([f["vp"], f["err"]])
+                errors.append((f["vp"], f["err"]))
                 continue
 
             try:
                 self.ser(f)
             except:
                 ex = min_ex(5, True).replace("\n", "\n-- ")
-                errors.append([f["vp"], ex])
+                errors.append((f["vp"], ex))
 
         if errors:
             self.errf, txt = errdesc(errors)

copyparty/stolen/surrogateescape.py

@@ -12,23 +12,28 @@ Original source: misc/python/surrogateescape.py in https://bitbucket.org/haypo/m
 
 # This code is released under the Python license and the BSD 2-clause license
 
-import platform
 import codecs
+import platform
 import sys
 
 PY3 = sys.version_info[0] > 2
 WINDOWS = platform.system() == "Windows"
 FS_ERRORS = "surrogateescape"
 
+try:
+    from typing import Any
+except:
+    pass
 
-def u(text):
+
+def u(text: Any) -> str:
     if PY3:
         return text
     else:
         return text.decode("unicode_escape")
 
 
-def b(data):
+def b(data: Any) -> bytes:
     if PY3:
         return data.encode("latin1")
     else:
@@ -43,7 +48,7 @@ else:
     bytes_chr = chr
 
 
-def surrogateescape_handler(exc):
+def surrogateescape_handler(exc: Any) -> tuple[str, int]:
     """
     Pure Python implementation of the PEP 383: the "surrogateescape" error
     handler of Python 3. Undecodable bytes will be replaced by a Unicode
@@ -74,7 +79,7 @@ class NotASurrogateError(Exception):
     pass
 
 
-def replace_surrogate_encode(mystring):
+def replace_surrogate_encode(mystring: str) -> str:
     """
     Returns a (unicode) string, not the more logical bytes, because the codecs
     register_error functionality expects this.
@@ -100,7 +105,7 @@ def replace_surrogate_encode(mystring):
     return str().join(decoded)
 
 
-def replace_surrogate_decode(mybytes):
+def replace_surrogate_decode(mybytes: bytes) -> str:
     """
     Returns a (unicode) string
     """
@@ -121,7 +126,7 @@ def replace_surrogate_decode(mybytes):
     return str().join(decoded)
 
 
-def encodefilename(fn):
+def encodefilename(fn: str) -> bytes:
     if FS_ENCODING == "ascii":
         # ASCII encoder of Python 2 expects that the error handler returns a
         # Unicode string encodable to ASCII, whereas our surrogateescape error
@@ -161,7 +166,7 @@ def encodefilename(fn):
         return fn.encode(FS_ENCODING, FS_ERRORS)
 
 
-def decodefilename(fn):
+def decodefilename(fn: bytes) -> str:
     return fn.decode(FS_ENCODING, FS_ERRORS)
 
 
@@ -181,7 +186,7 @@ if WINDOWS and not PY3:
 FS_ENCODING = codecs.lookup(FS_ENCODING).name
 
 
-def register_surrogateescape():
+def register_surrogateescape() -> None:
     """
     Registers the surrogateescape error handler on Python 2 (only)
     """

copyparty/sutil.py

@@ -6,8 +6,29 @@ from datetime import datetime
 
 from .bos import bos
 
+try:
+    from typing import Any, Generator, Optional
 
-def errdesc(errors):
+    from .util import NamedLogger
+except:
+    pass
+
+
+class StreamArc(object):
+    def __init__(
+        self,
+        log: NamedLogger,
+        fgen: Generator[dict[str, Any], None, None],
+        **kwargs: Any
+    ):
+        self.log = log
+        self.fgen = fgen
+
+    def gen(self) -> Generator[Optional[bytes], None, None]:
+        pass
+
+
+def errdesc(errors: list[tuple[str, str]]) -> tuple[dict[str, Any], list[str]]:
     report = ["copyparty failed to add the following files to the archive:", ""]
 
     for fn, err in errors:

copyparty/svchub.py

@@ -1,41 +1,51 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
+import argparse
+import calendar
 import os
-import sys
-import time
 import shlex
-import string
 import signal
 import socket
+import string
+import sys
 import threading
+import time
 from datetime import datetime, timedelta
-import calendar
 
-from .__init__ import E, PY2, WINDOWS, ANYWIN, MACOS, VT100, unicode
+try:
-from .util import mp, start_log_thrs, start_stackmon, min_ex, ansi_re
+    from types import FrameType
+
+    import typing
+    from typing import Optional, Union
+except:
+    pass
+
+from .__init__ import ANYWIN, MACOS, PY2, VT100, WINDOWS, E, unicode
 from .authsrv import AuthSrv
-from .tcpsrv import TcpSrv
-from .up2k import Up2k
-from .th_srv import ThumbSrv, HAVE_PIL, HAVE_VIPS, HAVE_WEBP
 from .mtag import HAVE_FFMPEG, HAVE_FFPROBE
+from .tcpsrv import TcpSrv
+from .th_srv import HAVE_PIL, HAVE_VIPS, HAVE_WEBP, ThumbSrv
+from .up2k import Up2k
+from .util import ansi_re, min_ex, mp, start_log_thrs, start_stackmon
 
 
 class SvcHub(object):
     """
     Hosts all services which cannot be parallelized due to reliance on monolithic resources.
     Creates a Broker which does most of the heavy stuff; hosted services can use this to perform work:
-        hub.broker.put(want_reply, destination, args_list).
+        hub.broker.<say|ask>(destination, args_list).
 
     Either BrokerThr (plain threads) or BrokerMP (multiprocessing) is used depending on configuration.
     Nothing is returned synchronously; if you want any value returned from the call,
     put() can return a queue (if want_reply=True) which has a blocking get() with the response.
     """
 
-    def __init__(self, args, argv, printed):
+    def __init__(self, args: argparse.Namespace, argv: list[str], printed: str) -> None:
         self.args = args
         self.argv = argv
-        self.logf = None
+        self.logf: Optional[typing.TextIO] = None
+        self.logf_base_fn = ""
         self.stop_req = False
         self.reload_req = False
         self.stopping = False
@@ -59,16 +69,16 @@ class SvcHub(object):
 
         if not args.use_fpool and args.j != 1:
             args.no_fpool = True
-            m = "multithreading enabled with -j {}, so disabling fpool -- this can reduce upload performance on some filesystems"
+            t = "multithreading enabled with -j {}, so disabling fpool -- this can reduce upload performance on some filesystems"
-            self.log("root", m.format(args.j))
+            self.log("root", t.format(args.j))
 
         if not args.no_fpool and args.j != 1:
-            m = "WARNING: --use-fpool combined with multithreading is untested and can probably cause undefined behavior"
+            t = "WARNING: --use-fpool combined with multithreading is untested and can probably cause undefined behavior"
             if ANYWIN:
-                m = 'windows cannot do multithreading without --no-fpool, so enabling that -- note that upload performance will suffer if you have microsoft defender "real-time protection" enabled, so you probably want to use -j 1 instead'
+                t = 'windows cannot do multithreading without --no-fpool, so enabling that -- note that upload performance will suffer if you have microsoft defender "real-time protection" enabled, so you probably want to use -j 1 instead'
                 args.no_fpool = True
 
-            self.log("root", m, c=3)
+            self.log("root", t, c=3)
 
         bri = "zy"[args.theme % 2 :][:1]
         ch = "abcdefghijklmnopqrstuvwx"[int(args.theme / 2)]
@@ -96,8 +106,8 @@ class SvcHub(object):
         self.args.th_dec = list(decs.keys())
         self.thumbsrv = None
         if not args.no_thumb:
-            m = "decoder preference: {}".format(", ".join(self.args.th_dec))
+            t = "decoder preference: {}".format(", ".join(self.args.th_dec))
-            self.log("thumb", m)
+            self.log("thumb", t)
 
             if "pil" in self.args.th_dec and not HAVE_WEBP:
                 msg = "disabling webp thumbnails because either libwebp is not available or your Pillow is too old"
@@ -131,11 +141,11 @@ class SvcHub(object):
         if self.check_mp_enable():
             from .broker_mp import BrokerMp as Broker
         else:
-            from .broker_thr import BrokerThr as Broker
+            from .broker_thr import BrokerThr as Broker  # type: ignore
 
         self.broker = Broker(self)
 
-    def thr_httpsrv_up(self):
+    def thr_httpsrv_up(self) -> None:
         time.sleep(1 if self.args.ign_ebind_all else 5)
         expected = self.broker.num_workers * self.tcpsrv.nsrv
         failed = expected - self.httpsrv_up
@@ -145,20 +155,20 @@ class SvcHub(object):
         if self.args.ign_ebind_all:
             if not self.tcpsrv.srv:
                 for _ in range(self.broker.num_workers):
-                    self.broker.put(False, "cb_httpsrv_up")
+                    self.broker.say("cb_httpsrv_up")
             return
 
         if self.args.ign_ebind and self.tcpsrv.srv:
             return
 
-        m = "{}/{} workers failed to start"
+        t = "{}/{} workers failed to start"
-        m = m.format(failed, expected)
+        t = t.format(failed, expected)
-        self.log("root", m, 1)
+        self.log("root", t, 1)
 
         self.retcode = 1
         os.kill(os.getpid(), signal.SIGTERM)
 
-    def cb_httpsrv_up(self):
+    def cb_httpsrv_up(self) -> None:
         self.httpsrv_up += 1
         if self.httpsrv_up != self.broker.num_workers:
             return
@@ -171,9 +181,9 @@ class SvcHub(object):
         thr.daemon = True
         thr.start()
 
-    def _logname(self):
+    def _logname(self) -> str:
         dt = datetime.utcnow()
-        fn = self.args.lo
+        fn = str(self.args.lo)
         for fs in "YmdHMS":
             fs = "%" + fs
             if fs in fn:
@@ -181,7 +191,7 @@ class SvcHub(object):
 
         return fn
 
-    def _setup_logfile(self, printed):
+    def _setup_logfile(self, printed: str) -> None:
         base_fn = fn = sel_fn = self._logname()
         if fn != self.args.lo:
             ctr = 0
@@ -203,8 +213,6 @@ class SvcHub(object):
 
             lh = codecs.open(fn, "w", encoding="utf-8", errors="replace")
 
-        lh.base_fn = base_fn
-
         argv = [sys.executable] + self.argv
         if hasattr(shlex, "quote"):
             argv = [shlex.quote(x) for x in argv]
@@ -215,9 +223,10 @@ class SvcHub(object):
         printed += msg
         lh.write("t0: {:.3f}\nargv: {}\n\n{}".format(E.t0, " ".join(argv), printed))
         self.logf = lh
+        self.logf_base_fn = base_fn
         print(msg, end="")
 
-    def run(self):
+    def run(self) -> None:
         self.tcpsrv.run()
 
         thr = threading.Thread(target=self.thr_httpsrv_up)
@@ -252,7 +261,7 @@ class SvcHub(object):
         else:
             self.stop_thr()
 
-    def reload(self):
+    def reload(self) -> str:
         if self.reloading:
             return "cannot reload; already in progress"
 
@@ -262,7 +271,7 @@ class SvcHub(object):
         t.start()
         return "reload initiated"
 
-    def _reload(self):
+    def _reload(self) -> None:
         self.log("root", "reload scheduled")
         with self.up2k.mutex:
             self.asrv.reload()
@@ -271,7 +280,7 @@ class SvcHub(object):
 
         self.reloading = False
 
-    def stop_thr(self):
+    def stop_thr(self) -> None:
         while not self.stop_req:
             with self.stop_cond:
                 self.stop_cond.wait(9001)
@@ -282,7 +291,7 @@ class SvcHub(object):
 
         self.shutdown()
 
-    def signal_handler(self, sig, frame):
+    def signal_handler(self, sig: int, frame: Optional[FrameType]) -> None:
         if self.stopping:
             return
 
@@ -294,7 +303,7 @@ class SvcHub(object):
         with self.stop_cond:
             self.stop_cond.notify_all()
 
-    def shutdown(self):
+    def shutdown(self) -> None:
         if self.stopping:
             return
 
@@ -337,7 +346,7 @@ class SvcHub(object):
 
             sys.exit(ret)
 
-    def _log_disabled(self, src, msg, c=0):
+    def _log_disabled(self, src: str, msg: str, c: Union[int, str] = 0) -> None:
         if not self.logf:
             return
 
@@ -349,8 +358,8 @@ class SvcHub(object):
             if now >= self.next_day:
                 self._set_next_day()
 
-    def _set_next_day(self):
+    def _set_next_day(self) -> None:
-        if self.next_day and self.logf and self.logf.base_fn != self._logname():
+        if self.next_day and self.logf and self.logf_base_fn != self._logname():
             self.logf.close()
             self._setup_logfile("")
 
@@ -364,7 +373,7 @@ class SvcHub(object):
         dt = dt.replace(hour=0, minute=0, second=0)
         self.next_day = calendar.timegm(dt.utctimetuple())
 
-    def _log_enabled(self, src, msg, c=0):
+    def _log_enabled(self, src: str, msg: str, c: Union[int, str] = 0) -> None:
         """handles logging from all components"""
         with self.log_mutex:
             now = time.time()
@@ -401,7 +410,7 @@ class SvcHub(object):
             if self.logf:
                 self.logf.write(msg)
 
-    def check_mp_support(self):
+    def check_mp_support(self) -> str:
         vmin = sys.version_info[1]
         if WINDOWS:
             msg = "need python 3.3 or newer for multiprocessing;"
@@ -415,16 +424,16 @@ class SvcHub(object):
                 return msg
 
         try:
-            x = mp.Queue(1)
+            x: mp.Queue[tuple[str, str]] = mp.Queue(1)
-            x.put(["foo", "bar"])
+            x.put(("foo", "bar"))
             if x.get()[0] != "foo":
                 raise Exception()
         except:
             return "multiprocessing is not supported on your platform;"
 
-        return None
+        return ""
 
-    def check_mp_enable(self):
+    def check_mp_enable(self) -> bool:
         if self.args.j == 1:
             return False
 
@@ -447,18 +456,18 @@ class SvcHub(object):
             self.log("svchub", "cannot efficiently use multiple CPU cores")
             return False
 
-    def sd_notify(self):
+    def sd_notify(self) -> None:
         try:
-            addr = os.getenv("NOTIFY_SOCKET")
+            zb = os.getenv("NOTIFY_SOCKET")
-            if not addr:
+            if not zb:
                 return
 
-            addr = unicode(addr)
+            addr = unicode(zb)
             if addr.startswith("@"):
                 addr = "\0" + addr[1:]
 
-            m = "".join(x for x in addr if x in string.printable)
+            t = "".join(x for x in addr if x in string.printable)
-            self.log("sd_notify", m)
+            self.log("sd_notify", t)
 
             sck = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
             sck.connect(addr)

copyparty/szip.py

@@ -1,16 +1,23 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
+import calendar
 import time
 import zlib
-import calendar
 
-from .sutil import errdesc
-from .util import yieldfile, sanitize_fn, spack, sunpack, min_ex
 from .bos import bos
+from .sutil import StreamArc, errdesc
+from .util import min_ex, sanitize_fn, spack, sunpack, yieldfile
+
+try:
+    from typing import Any, Generator, Optional
+
+    from .util import NamedLogger
+except:
+    pass
 
 
-def dostime2unix(buf):
+def dostime2unix(buf: bytes) -> int:
     t, d = sunpack(b"<HH", buf)
 
     ts = (t & 0x1F) * 2
@@ -29,7 +36,7 @@ def dostime2unix(buf):
     return int(calendar.timegm(dt))
 
 
-def unixtime2dos(ts):
+def unixtime2dos(ts: int) -> bytes:
     tt = time.gmtime(ts + 1)
     dy, dm, dd, th, tm, ts = list(tt)[:6]
 
@@ -41,14 +48,22 @@ def unixtime2dos(ts):
         return b"\x00\x00\x21\x00"
 
 
-def gen_fdesc(sz, crc32, z64):
+def gen_fdesc(sz: int, crc32: int, z64: bool) -> bytes:
     ret = b"\x50\x4b\x07\x08"
     fmt = b"<LQQ" if z64 else b"<LLL"
     ret += spack(fmt, crc32, sz, sz)
     return ret
 
 
-def gen_hdr(h_pos, fn, sz, lastmod, utf8, crc32, pre_crc):
+def gen_hdr(
+    h_pos: Optional[int],
+    fn: str,
+    sz: int,
+    lastmod: int,
+    utf8: bool,
+    icrc32: int,
+    pre_crc: bool,
+) -> bytes:
     """
     does regular file headers
     and the central directory meme if h_pos is set
@@ -67,8 +82,8 @@ def gen_hdr(h_pos, fn, sz, lastmod, utf8, crc32, pre_crc):
     # confusingly this doesn't bump if h_pos
     req_ver = b"\x2d\x00" if z64 else b"\x0a\x00"
 
-    if crc32:
+    if icrc32:
-        crc32 = spack(b"<L", crc32)
+        crc32 = spack(b"<L", icrc32)
     else:
         crc32 = b"\x00" * 4
 
@@ -129,7 +144,9 @@ def gen_hdr(h_pos, fn, sz, lastmod, utf8, crc32, pre_crc):
     return ret
 
 
-def gen_ecdr(items, cdir_pos, cdir_end):
+def gen_ecdr(
+    items: list[tuple[str, int, int, int, int]], cdir_pos: int, cdir_end: int
+) -> tuple[bytes, bool]:
     """
     summary of all file headers,
     usually the zipfile footer unless something clamps
@@ -154,10 +171,12 @@ def gen_ecdr(items, cdir_pos, cdir_end):
     # 2b comment length
     ret += b"\x00\x00"
 
-    return [ret, need_64]
+    return ret, need_64
 
 
-def gen_ecdr64(items, cdir_pos, cdir_end):
+def gen_ecdr64(
+    items: list[tuple[str, int, int, int, int]], cdir_pos: int, cdir_end: int
+) -> bytes:
     """
     z64 end of central directory
     added when numfiles or a headerptr clamps
@@ -181,7 +200,7 @@ def gen_ecdr64(items, cdir_pos, cdir_end):
     return ret
 
 
-def gen_ecdr64_loc(ecdr64_pos):
+def gen_ecdr64_loc(ecdr64_pos: int) -> bytes:
     """
     z64 end of central directory locator
     points to ecdr64
@@ -196,21 +215,27 @@ def gen_ecdr64_loc(ecdr64_pos):
     return ret
 
 
-class StreamZip(object):
+class StreamZip(StreamArc):
-    def __init__(self, log, fgen, utf8=False, pre_crc=False):
+    def __init__(
-        self.log = log
+        self,
-        self.fgen = fgen
+        log: NamedLogger,
+        fgen: Generator[dict[str, Any], None, None],
+        utf8: bool = False,
+        pre_crc: bool = False,
+    ) -> None:
+        super(StreamZip, self).__init__(log, fgen)
+
         self.utf8 = utf8
         self.pre_crc = pre_crc
 
         self.pos = 0
-        self.items = []
+        self.items: list[tuple[str, int, int, int, int]] = []
 
-    def _ct(self, buf):
+    def _ct(self, buf: bytes) -> bytes:
         self.pos += len(buf)
         return buf
 
-    def ser(self, f):
+    def ser(self, f: dict[str, Any]) -> Generator[bytes, None, None]:
         name = f["vp"]
         src = f["ap"]
         st = f["st"]
@@ -218,9 +243,8 @@ class StreamZip(object):
         sz = st.st_size
         ts = st.st_mtime
 
-        crc = None
+        crc = 0
         if self.pre_crc:
-            crc = 0
             for buf in yieldfile(src):
                 crc = zlib.crc32(buf, crc)
 
@@ -230,7 +254,6 @@ class StreamZip(object):
         buf = gen_hdr(None, name, sz, ts, self.utf8, crc, self.pre_crc)
         yield self._ct(buf)
 
-        crc = crc or 0
         for buf in yieldfile(src):
             if not self.pre_crc:
                 crc = zlib.crc32(buf, crc)
@@ -239,7 +262,7 @@ class StreamZip(object):
 
         crc &= 0xFFFFFFFF
 
-        self.items.append([name, sz, ts, crc, h_pos])
+        self.items.append((name, sz, ts, crc, h_pos))
 
         z64 = sz >= 4 * 1024 * 1024 * 1024
 
@@ -247,11 +270,11 @@ class StreamZip(object):
             buf = gen_fdesc(sz, crc, z64)
             yield self._ct(buf)
 
-    def gen(self):
+    def gen(self) -> Generator[bytes, None, None]:
         errors = []
         for f in self.fgen:
             if "err" in f:
-                errors.append([f["vp"], f["err"]])
+                errors.append((f["vp"], f["err"]))
                 continue
 
             try:
@@ -259,7 +282,7 @@ class StreamZip(object):
                     yield x
             except:
                 ex = min_ex(5, True).replace("\n", "\n-- ")
-                errors.append([f["vp"], ex])
+                errors.append((f["vp"], ex))
 
         if errors:
             errf, txt = errdesc(errors)

copyparty/tcpsrv.py

@@ -2,12 +2,15 @@
 from __future__ import print_function, unicode_literals
 
 import re
-import sys
 import socket
+import sys
 
-from .__init__ import MACOS, ANYWIN, unicode
+from .__init__ import ANYWIN, MACOS, TYPE_CHECKING, unicode
 from .util import chkcmd
 
+if TYPE_CHECKING:
+    from .svchub import SvcHub
+
 
 class TcpSrv(object):
     """
@@ -15,16 +18,16 @@ class TcpSrv(object):
     which then uses the least busy HttpSrv to handle it
     """
 
-    def __init__(self, hub):
+    def __init__(self, hub: "SvcHub"):
         self.hub = hub
         self.args = hub.args
         self.log = hub.log
 
         self.stopping = False
 
-        self.srv = []
+        self.srv: list[socket.socket] = []
         self.nsrv = 0
-        ok = {}
+        ok: dict[str, list[int]] = {}
         for ip in self.args.i:
             ok[ip] = []
             for port in self.args.p:
@@ -34,8 +37,8 @@ class TcpSrv(object):
                     ok[ip].append(port)
                 except Exception as ex:
                     if self.args.ign_ebind or self.args.ign_ebind_all:
-                        m = "could not listen on {}:{}: {}"
+                        t = "could not listen on {}:{}: {}"
-                        self.log("tcpsrv", m.format(ip, port, ex), c=3)
+                        self.log("tcpsrv", t.format(ip, port, ex), c=3)
                     else:
                         raise
 
@@ -55,9 +58,9 @@ class TcpSrv(object):
                     eps[x] = "external"
 
         msgs = []
-        title_tab = {}
+        title_tab: dict[str, dict[str, int]] = {}
         title_vars = [x[1:] for x in self.args.wintitle.split(" ") if x.startswith("$")]
-        m = "available @ {}://{}:{}/  (\033[33m{}\033[0m)"
+        t = "available @ {}://{}:{}/  (\033[33m{}\033[0m)"
         for ip, desc in sorted(eps.items(), key=lambda x: x[1]):
             for port in sorted(self.args.p):
                 if port not in ok.get(ip, ok.get("0.0.0.0", [])):
@@ -69,7 +72,7 @@ class TcpSrv(object):
                 elif self.args.https_only or port == 443:
                     proto = "https"
 
-                msgs.append(m.format(proto, ip, port, desc))
+                msgs.append(t.format(proto, ip, port, desc))
 
                 if not self.args.wintitle:
                     continue
@@ -98,13 +101,13 @@ class TcpSrv(object):
 
         if msgs:
             msgs[-1] += "\n"
-            for m in msgs:
+            for t in msgs:
-                self.log("tcpsrv", m)
+                self.log("tcpsrv", t)
 
         if self.args.wintitle:
             self._set_wintitle(title_tab)
 
-    def _listen(self, ip, port):
+    def _listen(self, ip: str, port: int) -> None:
         srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
         srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
         srv.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
@@ -120,7 +123,7 @@ class TcpSrv(object):
                 raise
             raise Exception(e)
 
-    def run(self):
+    def run(self) -> None:
         for srv in self.srv:
             srv.listen(self.args.nc)
             ip, port = srv.getsockname()
@@ -130,9 +133,9 @@ class TcpSrv(object):
             if self.args.q:
                 print(msg)
 
-            self.hub.broker.put(False, "listen", srv)
+            self.hub.broker.say("listen", srv)
 
-    def shutdown(self):
+    def shutdown(self) -> None:
         self.stopping = True
         try:
             for srv in self.srv:
@@ -142,14 +145,14 @@ class TcpSrv(object):
 
         self.log("tcpsrv", "ok bye")
 
-    def ips_linux_ifconfig(self):
+    def ips_linux_ifconfig(self) -> dict[str, str]:
         # for termux
         try:
             txt, _ = chkcmd(["ifconfig"])
         except:
             return {}
 
-        eps = {}
+        eps: dict[str, str] = {}
         dev = None
         ip = None
         up = None
@@ -171,7 +174,7 @@ class TcpSrv(object):
 
         return eps
 
-    def ips_linux(self):
+    def ips_linux(self) -> dict[str, str]:
         try:
             txt, _ = chkcmd(["ip", "addr"])
         except:
@@ -180,21 +183,21 @@ class TcpSrv(object):
         r = re.compile(r"^\s+inet ([^ ]+)/.* (.*)")
         ri = re.compile(r"^\s*[0-9]+\s*:.*")
         up = False
-        eps = {}
+        eps: dict[str, str] = {}
         for ln in txt.split("\n"):
             if ri.match(ln):
                 up = "UP" in re.split("[>,< ]", ln)
 
             try:
-                ip, dev = r.match(ln.rstrip()).groups()
+                ip, dev = r.match(ln.rstrip()).groups()  # type: ignore
                 eps[ip] = dev + ("" if up else ", \033[31mLINK-DOWN")
             except:
                 pass
 
         return eps
 
-    def ips_macos(self):
+    def ips_macos(self) -> dict[str, str]:
-        eps = {}
+        eps: dict[str, str] = {}
         try:
             txt, _ = chkcmd(["ifconfig"])
         except:
@@ -202,7 +205,7 @@ class TcpSrv(object):
 
         rdev = re.compile(r"^([^ ]+):")
         rip = re.compile(r"^\tinet ([0-9\.]+) ")
-        dev = None
+        dev = "UNKNOWN"
         for ln in txt.split("\n"):
             m = rdev.match(ln)
             if m:
@@ -211,17 +214,17 @@ class TcpSrv(object):
             m = rip.match(ln)
             if m:
                 eps[m.group(1)] = dev
-                dev = None
+                dev = "UNKNOWN"
 
         return eps
 
-    def ips_windows_ipconfig(self):
+    def ips_windows_ipconfig(self) -> tuple[dict[str, str], set[str]]:
-        eps = {}
+        eps: dict[str, str] = {}
-        offs = {}
+        offs: set[str] = set()
         try:
             txt, _ = chkcmd(["ipconfig"])
         except:
-            return eps
+            return eps, offs
 
         rdev = re.compile(r"(^[^ ].*):$")
         rip = re.compile(r"^ +IPv?4? [^:]+: *([0-9\.]{7,15})$")
@@ -231,12 +234,12 @@ class TcpSrv(object):
             m = rdev.match(ln)
             if m:
                 if dev and dev not in eps.values():
-                    offs[dev] = 1
+                    offs.add(dev)
 
                 dev = m.group(1).split(" adapter ", 1)[-1]
 
             if dev and roff.match(ln):
-                offs[dev] = 1
+                offs.add(dev)
                 dev = None
 
             m = rip.match(ln)
@@ -245,12 +248,12 @@ class TcpSrv(object):
                 dev = None
 
         if dev and dev not in eps.values():
-            offs[dev] = 1
+            offs.add(dev)
 
         return eps, offs
 
-    def ips_windows_netsh(self):
+    def ips_windows_netsh(self) -> dict[str, str]:
-        eps = {}
+        eps: dict[str, str] = {}
         try:
             txt, _ = chkcmd("netsh interface ip show address".split())
         except:
@@ -270,7 +273,7 @@ class TcpSrv(object):
 
         return eps
 
-    def detect_interfaces(self, listen_ips):
+    def detect_interfaces(self, listen_ips: list[str]) -> dict[str, str]:
         if MACOS:
             eps = self.ips_macos()
         elif ANYWIN:
@@ -317,7 +320,7 @@ class TcpSrv(object):
 
         return eps
 
-    def _set_wintitle(self, vs):
+    def _set_wintitle(self, vs: dict[str, dict[str, int]]) -> None:
         vs["all"] = vs.get("all", {"Local-Only": 1})
         vs["pub"] = vs.get("pub", vs["all"])
 

copyparty/th_cli.py

@@ -3,13 +3,23 @@ from __future__ import print_function, unicode_literals
 
 import os
 
-from .util import Cooldown
+from .__init__ import TYPE_CHECKING
-from .th_srv import thumb_path, HAVE_WEBP
+from .authsrv import VFS
 from .bos import bos
+from .th_srv import HAVE_WEBP, thumb_path
+from .util import Cooldown
+
+try:
+    from typing import Optional, Union
+except:
+    pass
+
+if TYPE_CHECKING:
+    from .httpsrv import HttpSrv
 
 
 class ThumbCli(object):
-    def __init__(self, hsrv):
+    def __init__(self, hsrv: "HttpSrv") -> None:
         self.broker = hsrv.broker
         self.log_func = hsrv.log
         self.args = hsrv.args
@@ -34,10 +44,10 @@ class ThumbCli(object):
         d = next((x for x in self.args.th_dec if x in ("vips", "pil")), None)
         self.can_webp = HAVE_WEBP or d == "vips"
 
-    def log(self, msg, c=0):
+    def log(self, msg: str, c: Union[int, str] = 0) -> None:
         self.log_func("thumbcli", msg, c)
 
-    def get(self, dbv, rem, mtime, fmt):
+    def get(self, dbv: VFS, rem: str, mtime: float, fmt: str) -> Optional[str]:
         ptop = dbv.realpath
         ext = rem.rsplit(".")[-1].lower()
         if ext not in self.thumbable or "dthumb" in dbv.flags:
@@ -106,17 +116,17 @@ class ThumbCli(object):
         if ret:
             tdir = os.path.dirname(tpath)
             if self.cooldown.poke(tdir):
-                self.broker.put(False, "thumbsrv.poke", tdir)
+                self.broker.say("thumbsrv.poke", tdir)
 
             if want_opus:
                 # audio files expire individually
                 if self.cooldown.poke(tpath):
-                    self.broker.put(False, "thumbsrv.poke", tpath)
+                    self.broker.say("thumbsrv.poke", tpath)
 
             return ret
 
         if abort:
             return None
 
-        x = self.broker.put(True, "thumbsrv.get", ptop, rem, mtime, fmt)
+        x = self.broker.ask("thumbsrv.get", ptop, rem, mtime, fmt)
-        return x.get()
+        return x.get()  # type: ignore

copyparty/th_srv.py

@@ -1,18 +1,28 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
-import os
-import time
-import shutil
 import base64
 import hashlib
-import threading
+import os
+import shutil
 import subprocess as sp
+import threading
+import time
 
-from .util import fsenc, vsplit, statdir, runcmd, Queue, Cooldown, BytesIO, min_ex
+from queue import Queue
+
+from .__init__ import TYPE_CHECKING
 from .bos import bos
 from .mtag import HAVE_FFMPEG, HAVE_FFPROBE, ffprobe
+from .util import BytesIO, Cooldown, fsenc, min_ex, runcmd, statdir, vsplit
 
+try:
+    from typing import Optional, Union
+except:
+    pass
+
+if TYPE_CHECKING:
+    from .svchub import SvcHub
 
 HAVE_PIL = False
 HAVE_HEIF = False
@@ -20,7 +30,7 @@ HAVE_AVIF = False
 HAVE_WEBP = False
 
 try:
-    from PIL import Image, ImageOps, ExifTags
+    from PIL import ExifTags, Image, ImageOps
 
     HAVE_PIL = True
     try:
@@ -47,14 +57,13 @@ except:
     pass
 
 try:
-    import pyvips
-
     HAVE_VIPS = True
+    import pyvips
 except:
     HAVE_VIPS = False
 
 
-def thumb_path(histpath, rem, mtime, fmt):
+def thumb_path(histpath: str, rem: str, mtime: float, fmt: str) -> str:
     # base16 = 16 = 256
     # b64-lc = 38 = 1444
     # base64 = 64 = 4096
@@ -80,7 +89,7 @@ def thumb_path(histpath, rem, mtime, fmt):
 
 
 class ThumbSrv(object):
-    def __init__(self, hub):
+    def __init__(self, hub: "SvcHub") -> None:
         self.hub = hub
         self.asrv = hub.asrv
         self.args = hub.args
@@ -91,17 +100,17 @@ class ThumbSrv(object):
         self.poke_cd = Cooldown(self.args.th_poke)
 
         self.mutex = threading.Lock()
-        self.busy = {}
+        self.busy: dict[str, list[threading.Condition]] = {}
         self.stopping = False
         self.nthr = max(1, self.args.th_mt)
 
-        self.q = Queue(self.nthr * 4)
+        self.q: Queue[Optional[tuple[str, str]]] = Queue(self.nthr * 4)
         for n in range(self.nthr):
-            t = threading.Thread(
+            thr = threading.Thread(
                 target=self.worker, name="thumb-{}-{}".format(n, self.nthr)
             )
-            t.daemon = True
+            thr.daemon = True
-            t.start()
+            thr.start()
 
         want_ff = not self.args.no_vthumb or not self.args.no_athumb
         if want_ff and (not HAVE_FFMPEG or not HAVE_FFPROBE):
@@ -122,7 +131,7 @@ class ThumbSrv(object):
             t.start()
 
         self.fmt_pil, self.fmt_vips, self.fmt_ffi, self.fmt_ffv, self.fmt_ffa = [
-            {x: True for x in y.split(",")}
+            set(y.split(","))
             for y in [
                 self.args.th_r_pil,
                 self.args.th_r_vips,
@@ -134,37 +143,37 @@ class ThumbSrv(object):
 
         if not HAVE_HEIF:
             for f in "heif heifs heic heics".split(" "):
-                self.fmt_pil.pop(f, None)
+                self.fmt_pil.discard(f)
 
         if not HAVE_AVIF:
             for f in "avif avifs".split(" "):
-                self.fmt_pil.pop(f, None)
+                self.fmt_pil.discard(f)
 
-        self.thumbable = {}
+        self.thumbable: set[str] = set()
 
         if "pil" in self.args.th_dec:
-            self.thumbable.update(self.fmt_pil)
+            self.thumbable |= self.fmt_pil
 
         if "vips" in self.args.th_dec:
-            self.thumbable.update(self.fmt_vips)
+            self.thumbable |= self.fmt_vips
 
         if "ff" in self.args.th_dec:
-            for t in [self.fmt_ffi, self.fmt_ffv, self.fmt_ffa]:
+            for zss in [self.fmt_ffi, self.fmt_ffv, self.fmt_ffa]:
-                self.thumbable.update(t)
+                self.thumbable |= zss
 
-    def log(self, msg, c=0):
+    def log(self, msg: str, c: Union[int, str] = 0) -> None:
         self.log_func("thumb", msg, c)
 
-    def shutdown(self):
+    def shutdown(self) -> None:
         self.stopping = True
         for _ in range(self.nthr):
             self.q.put(None)
 
-    def stopped(self):
+    def stopped(self) -> bool:
         with self.mutex:
             return not self.nthr
 
-    def get(self, ptop, rem, mtime, fmt):
+    def get(self, ptop: str, rem: str, mtime: float, fmt: str) -> Optional[str]:
         histpath = self.asrv.vfs.histtab.get(ptop)
         if not histpath:
             self.log("no histpath for [{}]".format(ptop))
@@ -191,7 +200,7 @@ class ThumbSrv(object):
                 do_conv = True
 
         if do_conv:
-            self.q.put([abspath, tpath])
+            self.q.put((abspath, tpath))
             self.log("conv {} \033[0m{}".format(tpath, abspath), c=6)
 
         while not self.stopping:
@@ -212,7 +221,7 @@ class ThumbSrv(object):
 
         return None
 
-    def getcfg(self):
+    def getcfg(self) -> dict[str, set[str]]:
         return {
             "thumbable": self.thumbable,
             "pil": self.fmt_pil,
@@ -222,7 +231,7 @@ class ThumbSrv(object):
             "ffa": self.fmt_ffa,
         }
 
-    def worker(self):
+    def worker(self) -> None:
         while not self.stopping:
             task = self.q.get()
             if not task:
@@ -253,7 +262,7 @@ class ThumbSrv(object):
                 except:
                     msg = "{} could not create thumbnail of {}\n{}"
                     msg = msg.format(fun.__name__, abspath, min_ex())
-                    c = 1 if "<Signals.SIG" in msg else "1;30"
+                    c: Union[str, int] = 1 if "<Signals.SIG" in msg else "1;30"
                     self.log(msg, c)
                     with open(tpath, "wb") as _:
                         pass
@@ -269,7 +278,7 @@ class ThumbSrv(object):
         with self.mutex:
             self.nthr -= 1
 
-    def fancy_pillow(self, im):
+    def fancy_pillow(self, im: "Image.Image") -> "Image.Image":
         # exif_transpose is expensive (loads full image + unconditional copy)
         r = max(*self.res) * 2
         im.thumbnail((r, r), resample=Image.LANCZOS)
@@ -295,7 +304,7 @@ class ThumbSrv(object):
 
         return im
 
-    def conv_pil(self, abspath, tpath):
+    def conv_pil(self, abspath: str, tpath: str) -> None:
         with Image.open(fsenc(abspath)) as im:
             try:
                 im = self.fancy_pillow(im)
@@ -324,7 +333,7 @@ class ThumbSrv(object):
 
             im.save(tpath, **args)
 
-    def conv_vips(self, abspath, tpath):
+    def conv_vips(self, abspath: str, tpath: str) -> None:
         crops = ["centre", "none"]
         if self.args.th_no_crop:
             crops = ["none"]
@@ -342,18 +351,17 @@ class ThumbSrv(object):
 
         img.write_to_file(tpath, Q=40)
 
-    def conv_ffmpeg(self, abspath, tpath):
+    def conv_ffmpeg(self, abspath: str, tpath: str) -> None:
         ret, _ = ffprobe(abspath)
         if not ret:
             return
 
         ext = abspath.rsplit(".")[-1].lower()
         if ext in ["h264", "h265"] or ext in self.fmt_ffi:
-            seek = []
+            seek: list[bytes] = []
         else:
             dur = ret[".dur"][1] if ".dur" in ret else 4
-            seek = "{:.0f}".format(dur / 3)
+            seek = [b"-ss", "{:.0f}".format(dur / 3).encode("utf-8")]
-            seek = [b"-ss", seek.encode("utf-8")]
 
         scale = "scale={0}:{1}:force_original_aspect_ratio="
         if self.args.th_no_crop:
@@ -361,7 +369,7 @@ class ThumbSrv(object):
         else:
             scale += "increase,crop={0}:{1},setsar=1:1"
 
-        scale = scale.format(*list(self.res)).encode("utf-8")
+        bscale = scale.format(*list(self.res)).encode("utf-8")
         # fmt: off
         cmd = [
             b"ffmpeg",
@@ -373,7 +381,7 @@ class ThumbSrv(object):
         cmd += [
             b"-i", fsenc(abspath),
             b"-map", b"0:v:0",
-            b"-vf", scale,
+            b"-vf", bscale,
             b"-frames:v", b"1",
             b"-metadata:s:v:0", b"rotate=0",
         ]
@@ -395,14 +403,14 @@ class ThumbSrv(object):
         cmd += [fsenc(tpath)]
         self._run_ff(cmd)
 
-    def _run_ff(self, cmd):
+    def _run_ff(self, cmd: list[bytes]) -> None:
         # self.log((b" ".join(cmd)).decode("utf-8"))
         ret, _, serr = runcmd(cmd, timeout=self.args.th_convt)
         if not ret:
             return
 
-        c = "1;30"
+        c: Union[str, int] = "1;30"
-        m = "FFmpeg failed (probably a corrupt video file):\n"
+        t = "FFmpeg failed (probably a corrupt video file):\n"
         if cmd[-1].lower().endswith(b".webp") and (
             "Error selecting an encoder" in serr
             or "Automatic encoder selection failed" in serr
@@ -410,14 +418,14 @@ class ThumbSrv(object):
             or "Please choose an encoder manually" in serr
         ):
             self.args.th_ff_jpg = True
-            m = "FFmpeg failed because it was compiled without libwebp; enabling --th-ff-jpg to force jpeg output:\n"
+            t = "FFmpeg failed because it was compiled without libwebp; enabling --th-ff-jpg to force jpeg output:\n"
             c = 1
 
         if (
             "Requested resampling engine is unavailable" in serr
             or "output pad on Parsed_aresample_" in serr
         ):
-            m = "FFmpeg failed because it was compiled without libsox; you must set --th-ff-swr to force swr resampling:\n"
+            t = "FFmpeg failed because it was compiled without libsox; you must set --th-ff-swr to force swr resampling:\n"
             c = 1
 
         lines = serr.strip("\n").split("\n")
@@ -428,10 +436,10 @@ class ThumbSrv(object):
         if len(txt) > 5000:
             txt = txt[:2500] + "...\nff: [...]\nff: ..." + txt[-2500:]
 
-        self.log(m + txt, c=c)
+        self.log(t + txt, c=c)
         raise sp.CalledProcessError(ret, (cmd[0], b"...", cmd[-1]))
 
-    def conv_spec(self, abspath, tpath):
+    def conv_spec(self, abspath: str, tpath: str) -> None:
         ret, _ = ffprobe(abspath)
         if "ac" not in ret:
             raise Exception("not audio")
@@ -473,7 +481,7 @@ class ThumbSrv(object):
         cmd += [fsenc(tpath)]
         self._run_ff(cmd)
 
-    def conv_opus(self, abspath, tpath):
+    def conv_opus(self, abspath: str, tpath: str) -> None:
         if self.args.no_acode:
             raise Exception("disabled in server config")
 
@@ -521,7 +529,7 @@ class ThumbSrv(object):
             # fmt: on
             self._run_ff(cmd)
 
-    def poke(self, tdir):
+    def poke(self, tdir: str) -> None:
         if not self.poke_cd.poke(tdir):
             return
 
@@ -533,7 +541,7 @@ class ThumbSrv(object):
         except:
             pass
 
-    def cleaner(self):
+    def cleaner(self) -> None:
         interval = self.args.th_clean
         while True:
             time.sleep(interval)
@@ -548,14 +556,14 @@ class ThumbSrv(object):
 
             self.log("\033[Jcln ok; rm {} dirs".format(ndirs))
 
-    def clean(self, histpath):
+    def clean(self, histpath: str) -> int:
         ret = 0
         for cat in ["th", "ac"]:
-            ret += self._clean(histpath, cat, None)
+            ret += self._clean(histpath, cat, "")
 
         return ret
 
-    def _clean(self, histpath, cat, thumbpath):
+    def _clean(self, histpath: str, cat: str, thumbpath: str) -> int:
         if not thumbpath:
             thumbpath = os.path.join(histpath, cat)
 
@@ -564,10 +572,10 @@ class ThumbSrv(object):
         maxage = getattr(self.args, cat + "_maxage")
         now = time.time()
         prev_b64 = None
-        prev_fp = None
+        prev_fp = ""
         try:
-            ents = statdir(self.log, not self.args.no_scandir, False, thumbpath)
+            t1 = statdir(self.log_func, not self.args.no_scandir, False, thumbpath)
-            ents = sorted(list(ents))
+            ents = sorted(list(t1))
         except:
             return 0
 

copyparty/u2idx.py

@@ -1,34 +1,37 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
-import re
-import os
-import time
 import calendar
+import os
+import re
 import threading
+import time
 from operator import itemgetter
 
-from .__init__ import ANYWIN, unicode
+from .__init__ import ANYWIN, TYPE_CHECKING, unicode
-from .util import absreal, s3dec, Pebkac, min_ex, gen_filekey, quotep
 from .bos import bos
 from .up2k import up2k_wark_from_hashlist
+from .util import HAVE_SQLITE3, Pebkac, absreal, gen_filekey, min_ex, quotep, s3dec
 
-
+if HAVE_SQLITE3:
-try:
-    HAVE_SQLITE3 = True
     import sqlite3
-except:
-    HAVE_SQLITE3 = False
-
 
 try:
     from pathlib import Path
 except:
     pass
 
+try:
+    from typing import Any, Optional, Union
+except:
+    pass
+
+if TYPE_CHECKING:
+    from .httpconn import HttpConn
+
 
 class U2idx(object):
-    def __init__(self, conn):
+    def __init__(self, conn: "HttpConn") -> None:
         self.log_func = conn.log_func
         self.asrv = conn.asrv
         self.args = conn.args
@@ -38,19 +41,21 @@ class U2idx(object):
             self.log("your python does not have sqlite3; searching will be disabled")
             return
 
-        self.active_id = None
+        self.active_id = ""
-        self.active_cur = None
+        self.active_cur: Optional["sqlite3.Cursor"] = None
-        self.cur = {}
+        self.cur: dict[str, "sqlite3.Cursor"] = {}
-        self.mem_cur = sqlite3.connect(":memory:")
+        self.mem_cur = sqlite3.connect(":memory:").cursor()
         self.mem_cur.execute(r"create table a (b text)")
 
-        self.p_end = None
+        self.p_end = 0.0
-        self.p_dur = 0
+        self.p_dur = 0.0
 
-    def log(self, msg, c=0):
+    def log(self, msg: str, c: Union[int, str] = 0) -> None:
         self.log_func("u2idx", msg, c)
 
-    def fsearch(self, vols, body):
+    def fsearch(
+        self, vols: list[tuple[str, str, dict[str, Any]]], body: dict[str, Any]
+    ) -> list[dict[str, Any]]:
         """search by up2k hashlist"""
         if not HAVE_SQLITE3:
             return []
@@ -60,14 +65,14 @@ class U2idx(object):
         wark = up2k_wark_from_hashlist(self.args.salt, fsize, fhash)
 
         uq = "substr(w,1,16) = ? and w = ?"
-        uv = [wark[:16], wark]
+        uv: list[Union[str, int]] = [wark[:16], wark]
 
         try:
             return self.run_query(vols, uq, uv, True, False, 99999)[0]
         except:
             raise Pebkac(500, min_ex())
 
-    def get_cur(self, ptop):
+    def get_cur(self, ptop: str) -> Optional["sqlite3.Cursor"]:
         if not HAVE_SQLITE3:
             return None
 
@@ -103,13 +108,16 @@ class U2idx(object):
         self.cur[ptop] = cur
         return cur
 
-    def search(self, vols, uq, lim):
+    def search(
+        self, vols: list[tuple[str, str, dict[str, Any]]], uq: str, lim: int
+    ) -> tuple[list[dict[str, Any]], list[str]]:
         """search by query params"""
         if not HAVE_SQLITE3:
-            return []
+            return [], []
 
         q = ""
-        va = []
+        v: Union[str, int] = ""
+        va: list[Union[str, int]] = []
         have_up = False  # query has up.* operands
         have_mt = False
         is_key = True
@@ -202,7 +210,7 @@ class U2idx(object):
                     "%Y",
                 ]:
                     try:
-                        v = calendar.timegm(time.strptime(v, fmt))
+                        v = calendar.timegm(time.strptime(str(v), fmt))
                         break
                     except:
                         pass
@@ -230,11 +238,12 @@ class U2idx(object):
 
             # lowercase tag searches
             m = ptn_lc.search(q)
-            if not m or not ptn_lcv.search(unicode(v)):
+            zs = unicode(v)
+            if not m or not ptn_lcv.search(zs):
                 continue
 
             va.pop()
-            va.append(v.lower())
+            va.append(zs.lower())
             q = q[: m.start()]
 
             field, oper = m.groups()
@@ -248,8 +257,16 @@ class U2idx(object):
         except Exception as ex:
             raise Pebkac(500, repr(ex))
 
-    def run_query(self, vols, uq, uv, have_up, have_mt, lim):
+    def run_query(
-        done_flag = []
+        self,
+        vols: list[tuple[str, str, dict[str, Any]]],
+        uq: str,
+        uv: list[Union[str, int]],
+        have_up: bool,
+        have_mt: bool,
+        lim: int,
+    ) -> tuple[list[dict[str, Any]], list[str]]:
+        done_flag: list[bool] = []
         self.active_id = "{:.6f}_{}".format(
             time.time(), threading.current_thread().ident
         )
@@ -266,13 +283,11 @@ class U2idx(object):
 
         if not uq or not uv:
             uq = "select * from up"
-            uv = ()
+            uv = []
         elif have_mt:
             uq = "select up.*, substr(up.w,1,16) mtw from up where " + uq
-            uv = tuple(uv)
         else:
             uq = "select up.* from up where " + uq
-            uv = tuple(uv)
 
         self.log("qs: {!r} {!r}".format(uq, uv))
 
@@ -292,11 +307,10 @@ class U2idx(object):
                     v = vtop + "/"
 
                 vuv.append(v)
-            vuv = tuple(vuv)
 
             sret = []
             fk = flags.get("fk")
-            c = cur.execute(uq, vuv)
+            c = cur.execute(uq, tuple(vuv))
             for hit in c:
                 w, ts, sz, rd, fn, ip, at = hit[:7]
                 lim -= 1
@@ -340,7 +354,7 @@ class U2idx(object):
             # print("[{}] {}".format(ptop, sret))
 
         done_flag.append(True)
-        self.active_id = None
+        self.active_id = ""
 
         # undupe hits from multiple metadata keys
         if len(ret) > 1:
@@ -354,11 +368,12 @@ class U2idx(object):
 
         return ret, list(taglist.keys())
 
-    def terminator(self, identifier, done_flag):
+    def terminator(self, identifier: str, done_flag: list[bool]) -> None:
         for _ in range(self.timeout):
             time.sleep(1)
             if done_flag:
                 return
 
         if identifier == self.active_id:
+            assert self.active_cur
             self.active_cur.connection.interrupt()

scripts/make-pypi-release.sh

@@ -90,6 +90,15 @@ function have() {
 have setuptools
 have wheel
 have twine
+
+# remove type hints to support python < 3.9
+rm -rf build/pypi
+mkdir -p build/pypi
+cp -pR setup.py README.md LICENSE copyparty tests bin scripts/strip_hints build/pypi/
+cd build/pypi
+tar --strip-components=2 -xf ../strip-hints-0.1.10.tar.gz strip-hints-0.1.10/src/strip_hints
+python3 -c 'from strip_hints.a import uh; uh("copyparty")'
+
 ./setup.py clean2
 ./setup.py sdist bdist_wheel --universal
 

scripts/make-sfx.sh

@@ -76,7 +76,7 @@ while [ ! -z "$1" ]; do
 		no-hl)  no_hl=1  ; ;;
 		no-dd)  no_dd=1  ; ;;
 		no-cm)  no_cm=1  ; ;;
-		fast)   zopf=100 ; ;;
+		fast)   zopf=    ; ;;
 		lang)   shift;langs="$1"; ;;
 		*)      help     ; ;;
 	esac
@@ -106,7 +106,7 @@ tmpdir="$(
 [ $repack ] && {
 	old="$tmpdir/pe-copyparty"
 	echo "repack of files in $old"
-	cp -pR "$old/"*{dep-j2,dep-ftp,copyparty} .
+	cp -pR "$old/"*{j2,ftp,copyparty} .
 }
 
 [ $repack ] || {
@@ -130,8 +130,8 @@ tmpdir="$(
 	mv MarkupSafe-*/src/markupsafe .
 	rm -rf MarkupSafe-* markupsafe/_speedups.c
 
-	mkdir dep-j2/
+	mkdir j2/
-	mv {markupsafe,jinja2} dep-j2/
+	mv {markupsafe,jinja2} j2/
 
 	echo collecting pyftpdlib
 	f="../build/pyftpdlib-1.5.6.tar.gz"
@@ -143,8 +143,8 @@ tmpdir="$(
 	mv pyftpdlib-release-*/pyftpdlib .
 	rm -rf pyftpdlib-release-* pyftpdlib/test
 
-	mkdir dep-ftp/
+	mkdir ftp/
-	mv pyftpdlib dep-ftp/
+	mv pyftpdlib ftp/
 
 	echo collecting asyncore, asynchat
 	for n in asyncore.py asynchat.py; do
@@ -154,6 +154,24 @@ tmpdir="$(
 			wget -O$f "$url" || curl -L "$url" >$f)
 	done
 
+    # enable this to dynamically remove type hints at startup,
+    # in case a future python version can use them for performance
+	true || (
+		echo collecting strip-hints
+		f=../build/strip-hints-0.1.10.tar.gz
+		[ -e $f ] ||
+			(url=https://files.pythonhosted.org/packages/9c/d4/312ddce71ee10f7e0ab762afc027e07a918f1c0e1be5b0069db5b0e7542d/strip-hints-0.1.10.tar.gz;
+			wget -O$f "$url" || curl -L "$url" >$f)
+
+		tar -zxf $f
+		mv strip-hints-0.1.10/src/strip_hints .
+		rm -rf strip-hints-* strip_hints/import_hooks*
+		sed -ri 's/[a-z].* as import_hooks$/"""a"""/' strip_hints/*.py
+
+		cp -pR ../scripts/strip_hints/ .
+	)
+	cp -pR ../scripts/py2/ .
+
 	# msys2 tar is bad, make the best of it
 	echo collecting source
 	[ $clean ] && {
@@ -170,6 +188,9 @@ tmpdir="$(
 	for n in asyncore.py asynchat.py; do
 		awk 'NR<4||NR>27;NR==4{print"# license: https://opensource.org/licenses/ISC\n"}' ../build/$n >copyparty/vend/$n
 	done
+
+	# remove type hints before build instead
+	(cd copyparty; python3 ../../scripts/strip_hints/a.py; rm uh)
 }
 
 ver=
@@ -274,17 +295,23 @@ rm have
 		tmv "$f"
 	done
 
-[ $repack ] ||
+[ $repack ] || {
-find | grep -E '\.py$' |
+	# uncomment
-  grep -vE '__version__' |
+	find | grep -E '\.py$' |
-  tr '\n' '\0' |
+		grep -vE '__version__' |
-  xargs -0 "$pybin" ../scripts/uncomment.py
+		tr '\n' '\0' |
+		xargs -0 "$pybin" ../scripts/uncomment.py
 
-f=dep-j2/jinja2/constants.py
+	# py2-compat
+	#find | grep -E '\.py$' | while IFS= read -r x; do
+	#	sed -ri '/: TypeAlias = /d' "$x"; done
+}
+
+f=j2/jinja2/constants.py
 awk '/^LOREM_IPSUM_WORDS/{o=1;print "LOREM_IPSUM_WORDS = u\"a\"";next} !o; /"""/{o=0}' <$f >t
 tmv "$f"
 
-grep -rLE '^#[^a-z]*coding: utf-8' dep-j2 |
+grep -rLE '^#[^a-z]*coding: utf-8' j2 |
 while IFS= read -r f; do
 	(echo "# coding: utf-8"; cat "$f") >t
 	tmv "$f"
@@ -313,7 +340,7 @@ find | grep -E '\.(js|html)$' | while IFS= read -r f; do
 done
 
 gzres() {
-	command -v pigz &&
+	command -v pigz && [ $zopf ] &&
 		pk="pigz -11 -I $zopf" ||
 		pk='gzip'
 
@@ -354,7 +381,8 @@ nf=$(ls -1 "$zdir"/arc.* | wc -l)
 }
 [ $use_zdir ] && {
 	arcs=("$zdir"/arc.*)
-	arc="${arcs[$RANDOM % ${#arcs[@]} ] }"
+	n=$(( $RANDOM % ${#arcs[@]} ))
+	arc="${arcs[n]}"
 	echo "using $arc"
 	tar -xf "$arc"
 	for f in copyparty/web/*.gz; do
@@ -364,7 +392,7 @@ nf=$(ls -1 "$zdir"/arc.* | wc -l)
 
 
 echo gen tarlist
-for d in copyparty dep-j2 dep-ftp; do find $d -type f; done |
+for d in copyparty j2 ftp py2; do find $d -type f; done |  # strip_hints
 sed -r 's/(.*)\.(.*)/\2 \1/' | LC_ALL=C sort |
 sed -r 's/([^ ]*) (.*)/\2.\1/' | grep -vE '/list1?$' > list1
 

scripts/run-tests.sh

@@ -1,13 +1,23 @@
 #!/bin/bash
 set -ex
 
+rm -rf unt
+mkdir -p unt/srv
+cp -pR copyparty tests unt/
+cd unt
+python3 ../scripts/strip_hints/a.py
+
 pids=()
 for py in python{2,3}; do
+    PYTHONPATH=
+    [ $py = python2 ] && PYTHONPATH=../scripts/py2
+    export PYTHONPATH
+
     nice $py -m unittest discover -s tests >/dev/null &
     pids+=($!)
 done
 
-python3 scripts/test/smoketest.py &
+python3 ../scripts/test/smoketest.py &
 pids+=($!)
 
 for pid in ${pids[@]}; do

scripts/sfx.py

@@ -379,9 +379,20 @@ def run(tmp, j2, ftp):
     t.daemon = True
     t.start()
 
-    ld = (("", ""), (j2, "dep-j2"), (ftp, "dep-ftp"))
+    ld = (("", ""), (j2, "j2"), (ftp, "ftp"), (not PY2, "py2"))
     ld = [os.path.join(tmp, b) for a, b in ld if not a]
 
+    # skip 1
+    # enable this to dynamically remove type hints at startup,
+    # in case a future python version can use them for performance
+    if sys.version_info < (3, 10) and False:
+        sys.path.insert(0, ld[0])
+
+        from strip_hints.a import uh
+
+        uh(tmp + "/copyparty")
+    # skip 0
+
     if any([re.match(r"^-.*j[0-9]", x) for x in sys.argv]):
         run_s(ld)
     else:

scripts/sfx.sh

@@ -47,7 +47,7 @@ grep -E '/(python|pypy)[0-9\.-]*$' >$dir/pys || true
 	printf '\033[1;30mlooking for jinja2 in [%s]\033[0m\n' "$_py" >&2
 	$_py -c 'import jinja2' 2>/dev/null || continue
 	printf '%s\n' "$_py"
-	mv $dir/{,x.}dep-j2
+	mv $dir/{,x.}j2
 	break
 done)"
 

scripts/strip_hints/a.py

@@ -0,0 +1,57 @@
+# coding: utf-8
+from __future__ import print_function, unicode_literals
+
+import re
+import os
+import sys
+from strip_hints import strip_file_to_string
+
+
+# list unique types used in hints:
+# rm -rf unt && cp -pR copyparty unt && (cd unt && python3 ../scripts/strip_hints/a.py)
+# diff -wNarU1 copyparty unt | grep -E '^\-' | sed -r 's/[^][, ]+://g; s/[^][, ]+[[(]//g; s/[],()<>{} -]/\n/g' | grep -E .. | sort | uniq -c | sort -n
+
+
+def pr(m):
+    sys.stderr.write(m)
+    sys.stderr.flush()
+
+
+def uh(top):
+    if os.path.exists(top + "/uh"):
+        return
+
+    libs = "typing|types|collections\.abc"
+    ptn = re.compile(r"^(\s*)(from (?:{0}) import |import (?:{0})\b).*".format(libs))
+
+    # pr("building support for your python ver")
+    pr("unhinting")
+    for (dp, _, fns) in os.walk(top):
+        for fn in fns:
+            if not fn.endswith(".py"):
+                continue
+
+            pr(".")
+            fp = os.path.join(dp, fn)
+            cs = strip_file_to_string(fp, no_ast=True, to_empty=True)
+
+            # remove expensive imports too
+            lns = []
+            for ln in cs.split("\n"):
+                m = ptn.match(ln)
+                if m:
+                    ln = m.group(1) + "raise Exception()"
+
+                lns.append(ln)
+
+            cs = "\n".join(lns)
+            with open(fp, "wb") as f:
+                f.write(cs.encode("utf-8"))
+
+    pr("k\n\n")
+    with open(top + "/uh", "wb") as f:
+        f.write(b"a")
+
+
+if __name__ == "__main__":
+    uh(".")

scripts/test/race.py

@@ -58,13 +58,13 @@ class CState(threading.Thread):
                         remotes.append("?")
                         remotes_ok = False
 
-            m = []
+            ta = []
             for conn, remote in zip(self.cs, remotes):
                 stage = len(conn.st)
-                m.append(f"\033[3{colors[stage]}m{remote}")
+                ta.append(f"\033[3{colors[stage]}m{remote}")
 
-            m = " ".join(m)
+            t = " ".join(ta)
-            print(f"{m}\033[0m\n\033[A", end="")
+            print(f"{t}\033[0m\n\033[A", end="")
 
 
 def allget(cs, urls):

scripts/test/smoketest.py

@@ -72,6 +72,8 @@ def tc1(vflags):
     for _ in range(10):
         try:
             os.mkdir(td)
+            if os.path.exists(td):
+                break
         except:
             time.sleep(0.1)  # win10
 

tests/test_vfs.py

@@ -85,7 +85,7 @@ class TestVFS(unittest.TestCase):
         pass
 
     def assertAxs(self, dct, lst):
-        t1 = list(sorted(dct.keys()))
+        t1 = list(sorted(dct))
         t2 = list(sorted(lst))
         self.assertEqual(t1, t2)
 
@@ -208,10 +208,10 @@ class TestVFS(unittest.TestCase):
         self.assertEqual(n.realpath, os.path.join(td, "a"))
         self.assertAxs(n.axs.uread, ["*"])
         self.assertAxs(n.axs.uwrite, [])
-        self.assertEqual(vfs.can_access("/", "*"), [False, False, False, False, False])
+        self.assertEqual(vfs.can_access("/", "*"), (False, False, False, False, False))
-        self.assertEqual(vfs.can_access("/", "k"), [True, True, False, False, False])
+        self.assertEqual(vfs.can_access("/", "k"), (True, True, False, False, False))
-        self.assertEqual(vfs.can_access("/a", "*"), [True, False, False, False, False])
+        self.assertEqual(vfs.can_access("/a", "*"), (True, False, False, False, False))
-        self.assertEqual(vfs.can_access("/a", "k"), [True, False, False, False, False])
+        self.assertEqual(vfs.can_access("/a", "k"), (True, False, False, False, False))
 
         # breadth-first construction
         vfs = AuthSrv(
@@ -279,7 +279,7 @@ class TestVFS(unittest.TestCase):
         n = au.vfs
         # root was not defined, so PWD with no access to anyone
         self.assertEqual(n.vpath, "")
-        self.assertEqual(n.realpath, None)
+        self.assertEqual(n.realpath, "")
         self.assertAxs(n.axs.uread, [])
         self.assertAxs(n.axs.uwrite, [])
         self.assertEqual(len(n.nodes), 1)

tests/util.py

@@ -90,7 +90,10 @@ def get_ramdisk():
 
 
 class NullBroker(object):
-    def put(*args):
+    def say(*args):
+        pass
+
+    def ask(*args):
         pass