mirror of
https://github.com/9001/copyparty.git
synced 2025-08-16 08:32:13 -06:00
add types, isort, errorhandling
This commit is contained in:
ed
parent
0b6f102436
commit
438384425a
|
|
@ -1200,15 +1200,18 @@ journalctl -aS '48 hour ago' -u copyparty | grep -C10 FILENAME | tee bug.log
|
|||
|
||||
## dev env setup
|
||||
|
||||
mostly optional; if you need a working env for vscode or similar
|
||||
you need python 3.9 or newer due to type hints
|
||||
|
||||
the rest is mostly optional; if you need a working env for vscode or similar
|
||||
|
||||
```sh
|
||||
python3 -m venv .venv
|
||||
. .venv/bin/activate
|
||||
pip install jinja2 # mandatory
|
||||
pip install jinja2 strip_hints # MANDATORY
|
||||
pip install mutagen # audio metadata
|
||||
pip install pyftpdlib # ftp server
|
||||
pip install Pillow pyheif-pillow-opener pillow-avif-plugin # thumbnails
|
||||
pip install black==21.12b0 bandit pylint flake8 # vscode tooling
|
||||
pip install black==21.12b0 click==8.0.2 bandit pylint flake8 isort mypy # vscode tooling
|
||||
```
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -43,7 +43,6 @@ PS: this requires e2ts to be functional,
|
|||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import filecmp
|
||||
import subprocess as sp
|
||||
|
||||
|
|
|
|||
22
bin/up2k.py
22
bin/up2k.py
|
|
@ -77,15 +77,15 @@ class File(object):
|
|||
self.up_b = 0 # type: int
|
||||
self.up_c = 0 # type: int
|
||||
|
||||
# m = "size({}) lmod({}) top({}) rel({}) abs({}) name({})\n"
|
||||
# eprint(m.format(self.size, self.lmod, self.top, self.rel, self.abs, self.name))
|
||||
# t = "size({}) lmod({}) top({}) rel({}) abs({}) name({})\n"
|
||||
# eprint(t.format(self.size, self.lmod, self.top, self.rel, self.abs, self.name))
|
||||
|
||||
|
||||
class FileSlice(object):
|
||||
"""file-like object providing a fixed window into a file"""
|
||||
|
||||
def __init__(self, file, cid):
|
||||
# type: (File, str) -> FileSlice
|
||||
# type: (File, str) -> None
|
||||
|
||||
self.car, self.len = file.kchunks[cid]
|
||||
self.cdr = self.car + self.len
|
||||
|
|
@ -216,8 +216,8 @@ class CTermsize(object):
|
|||
eprint("\033[s\033[r\033[u")
|
||||
else:
|
||||
self.g = 1 + self.h - margin
|
||||
m = "{0}\033[{1}A".format("\n" * margin, margin)
|
||||
eprint("{0}\033[s\033[1;{1}r\033[u".format(m, self.g - 1))
|
||||
t = "{0}\033[{1}A".format("\n" * margin, margin)
|
||||
eprint("{0}\033[s\033[1;{1}r\033[u".format(t, self.g - 1))
|
||||
|
||||
|
||||
ss = CTermsize()
|
||||
|
|
@ -597,8 +597,8 @@ class Ctl(object):
|
|||
if "/" in name:
|
||||
name = "\033[36m{0}\033[0m/{1}".format(*name.rsplit("/", 1))
|
||||
|
||||
m = "{0:6.1f}% {1} {2}\033[K"
|
||||
txt += m.format(p, self.nfiles - f, name)
|
||||
t = "{0:6.1f}% {1} {2}\033[K"
|
||||
txt += t.format(p, self.nfiles - f, name)
|
||||
|
||||
txt += "\033[{0}H ".format(ss.g + 2)
|
||||
else:
|
||||
|
|
@ -618,8 +618,8 @@ class Ctl(object):
|
|||
nleft = self.nfiles - self.up_f
|
||||
tail = "\033[K\033[u" if VT100 else "\r"
|
||||
|
||||
m = "{0} eta @ {1}/s, {2}, {3}# left".format(eta, spd, sleft, nleft)
|
||||
eprint(txt + "\033]0;{0}\033\\\r{0}{1}".format(m, tail))
|
||||
t = "{0} eta @ {1}/s, {2}, {3}# left".format(eta, spd, sleft, nleft)
|
||||
eprint(txt + "\033]0;{0}\033\\\r{0}{1}".format(t, tail))
|
||||
|
||||
def cleanup_vt100(self):
|
||||
ss.scroll_region(None)
|
||||
|
|
@ -721,8 +721,8 @@ class Ctl(object):
|
|||
if search:
|
||||
if hs:
|
||||
for hit in hs:
|
||||
m = "found: {0}\n {1}{2}\n"
|
||||
print(m.format(upath, burl, hit["rp"]), end="")
|
||||
t = "found: {0}\n {1}{2}\n"
|
||||
print(t.format(upath, burl, hit["rp"]), end="")
|
||||
else:
|
||||
print("NOT found: {0}\n".format(upath), end="")
|
||||
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@
|
|||
# installation:
|
||||
# cp -pv copyparty.service /etc/systemd/system
|
||||
# restorecon -vr /etc/systemd/system/copyparty.service
|
||||
# firewall-cmd --permanent --add-port={80,443,3923}/tcp
|
||||
# firewall-cmd --permanent --add-port={80,443,3923}/tcp # --zone=libvirt
|
||||
# firewall-cmd --reload
|
||||
# systemctl daemon-reload && systemctl enable --now copyparty
|
||||
#
|
||||
|
|
|
|||
|
|
@ -1,21 +1,30 @@
|
|||
# coding: utf-8
|
||||
from __future__ import print_function, unicode_literals
|
||||
|
||||
import platform
|
||||
import time
|
||||
import sys
|
||||
import os
|
||||
import platform
|
||||
import sys
|
||||
import time
|
||||
|
||||
try:
|
||||
from collections.abc import Callable
|
||||
|
||||
from typing import TYPE_CHECKING, Any
|
||||
except:
|
||||
TYPE_CHECKING = False
|
||||
|
||||
PY2 = sys.version_info[0] == 2
|
||||
if PY2:
|
||||
sys.dont_write_bytecode = True
|
||||
unicode = unicode
|
||||
unicode = unicode # noqa: F821 # pylint: disable=undefined-variable,self-assigning-variable
|
||||
else:
|
||||
unicode = str
|
||||
|
||||
WINDOWS = False
|
||||
if platform.system() == "Windows":
|
||||
WINDOWS = [int(x) for x in platform.version().split(".")]
|
||||
WINDOWS: Any = (
|
||||
[int(x) for x in platform.version().split(".")]
|
||||
if platform.system() == "Windows"
|
||||
else False
|
||||
)
|
||||
|
||||
VT100 = not WINDOWS or WINDOWS >= [10, 0, 14393]
|
||||
# introduced in anniversary update
|
||||
|
|
@ -25,8 +34,8 @@ ANYWIN = WINDOWS or sys.platform in ["msys"]
|
|||
MACOS = platform.system() == "Darwin"
|
||||
|
||||
|
||||
def get_unixdir():
|
||||
paths = [
|
||||
def get_unixdir() -> str:
|
||||
paths: list[tuple[Callable[..., str], str]] = [
|
||||
(os.environ.get, "XDG_CONFIG_HOME"),
|
||||
(os.path.expanduser, "~/.config"),
|
||||
(os.environ.get, "TMPDIR"),
|
||||
|
|
@ -43,7 +52,7 @@ def get_unixdir():
|
|||
continue
|
||||
|
||||
p = os.path.normpath(p)
|
||||
chk(p)
|
||||
chk(p) # type: ignore
|
||||
p = os.path.join(p, "copyparty")
|
||||
if not os.path.isdir(p):
|
||||
os.mkdir(p)
|
||||
|
|
@ -56,7 +65,7 @@ def get_unixdir():
|
|||
|
||||
|
||||
class EnvParams(object):
|
||||
def __init__(self):
|
||||
def __init__(self) -> None:
|
||||
self.t0 = time.time()
|
||||
self.mod = os.path.dirname(os.path.realpath(__file__))
|
||||
if self.mod.endswith("__init__"):
|
||||
|
|
|
|||
|
|
@ -8,35 +8,42 @@ __copyright__ = 2019
|
|||
__license__ = "MIT"
|
||||
__url__ = "https://github.com/9001/copyparty/"
|
||||
|
||||
import re
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import shutil
|
||||
import argparse
|
||||
import filecmp
|
||||
import locale
|
||||
import argparse
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import traceback
|
||||
from textwrap import dedent
|
||||
|
||||
from .__init__ import E, WINDOWS, ANYWIN, VT100, PY2, unicode
|
||||
from .__version__ import S_VERSION, S_BUILD_DT, CODENAME
|
||||
from .svchub import SvcHub
|
||||
from .util import py_desc, align_tab, IMPLICATIONS, ansi_re, min_ex
|
||||
from .__init__ import ANYWIN, PY2, VT100, WINDOWS, E, unicode
|
||||
from .__version__ import CODENAME, S_BUILD_DT, S_VERSION
|
||||
from .authsrv import re_vol
|
||||
from .svchub import SvcHub
|
||||
from .util import IMPLICATIONS, align_tab, ansi_re, min_ex, py_desc
|
||||
|
||||
HAVE_SSL = True
|
||||
try:
|
||||
from types import FrameType
|
||||
|
||||
from typing import Any, Optional
|
||||
except:
|
||||
pass
|
||||
|
||||
try:
|
||||
HAVE_SSL = True
|
||||
import ssl
|
||||
except:
|
||||
HAVE_SSL = False
|
||||
|
||||
printed = []
|
||||
printed: list[str] = []
|
||||
|
||||
|
||||
class RiceFormatter(argparse.HelpFormatter):
|
||||
def _get_help_string(self, action):
|
||||
def _get_help_string(self, action: argparse.Action) -> str:
|
||||
"""
|
||||
same as ArgumentDefaultsHelpFormatter(HelpFormatter)
|
||||
except the help += [...] line now has colors
|
||||
|
|
@ -45,27 +52,27 @@ class RiceFormatter(argparse.HelpFormatter):
|
|||
if not VT100:
|
||||
fmt = " (default: %(default)s)"
|
||||
|
||||
ret = action.help
|
||||
if "%(default)" not in action.help:
|
||||
ret = str(action.help)
|
||||
if "%(default)" not in ret:
|
||||
if action.default is not argparse.SUPPRESS:
|
||||
defaulting_nargs = [argparse.OPTIONAL, argparse.ZERO_OR_MORE]
|
||||
if action.option_strings or action.nargs in defaulting_nargs:
|
||||
ret += fmt
|
||||
return ret
|
||||
|
||||
def _fill_text(self, text, width, indent):
|
||||
def _fill_text(self, text: str, width: int, indent: str) -> str:
|
||||
"""same as RawDescriptionHelpFormatter(HelpFormatter)"""
|
||||
return "".join(indent + line + "\n" for line in text.splitlines())
|
||||
|
||||
|
||||
class Dodge11874(RiceFormatter):
|
||||
def __init__(self, *args, **kwargs):
|
||||
def __init__(self, *args: Any, **kwargs: Any) -> None:
|
||||
kwargs["width"] = 9003
|
||||
super(Dodge11874, self).__init__(*args, **kwargs)
|
||||
|
||||
|
||||
def lprint(*a, **ka):
|
||||
txt = " ".join(unicode(x) for x in a) + ka.get("end", "\n")
|
||||
def lprint(*a: Any, **ka: Any) -> None:
|
||||
txt: str = " ".join(unicode(x) for x in a) + ka.get("end", "\n")
|
||||
printed.append(txt)
|
||||
if not VT100:
|
||||
txt = ansi_re.sub("", txt)
|
||||
|
|
@ -73,11 +80,11 @@ def lprint(*a, **ka):
|
|||
print(txt, **ka)
|
||||
|
||||
|
||||
def warn(msg):
|
||||
def warn(msg: str) -> None:
|
||||
lprint("\033[1mwarning:\033[0;33m {}\033[0m\n".format(msg))
|
||||
|
||||
|
||||
def ensure_locale():
|
||||
def ensure_locale() -> None:
|
||||
for x in [
|
||||
"en_US.UTF-8",
|
||||
"English_United States.UTF8",
|
||||
|
|
@ -91,7 +98,7 @@ def ensure_locale():
|
|||
continue
|
||||
|
||||
|
||||
def ensure_cert():
|
||||
def ensure_cert() -> None:
|
||||
"""
|
||||
the default cert (and the entire TLS support) is only here to enable the
|
||||
crypto.subtle javascript API, which is necessary due to the webkit guys
|
||||
|
|
@ -117,8 +124,8 @@ def ensure_cert():
|
|||
# printf 'NO\n.\n.\n.\n.\ncopyparty-insecure\n.\n' | faketime '2000-01-01 00:00:00' openssl req -x509 -sha256 -newkey rsa:2048 -keyout insecure.pem -out insecure.pem -days $((($(printf %d 0x7fffffff)-$(date +%s --date=2000-01-01T00:00:00Z))/(60*60*24))) -nodes && ls -al insecure.pem && openssl x509 -in insecure.pem -text -noout
|
||||
|
||||
|
||||
def configure_ssl_ver(al):
|
||||
def terse_sslver(txt):
|
||||
def configure_ssl_ver(al: argparse.Namespace) -> None:
|
||||
def terse_sslver(txt: str) -> str:
|
||||
txt = txt.lower()
|
||||
for c in ["_", "v", "."]:
|
||||
txt = txt.replace(c, "")
|
||||
|
|
@ -133,8 +140,8 @@ def configure_ssl_ver(al):
|
|||
flags = [k for k in ssl.__dict__ if ptn.match(k)]
|
||||
# SSLv2 SSLv3 TLSv1 TLSv1_1 TLSv1_2 TLSv1_3
|
||||
if "help" in sslver:
|
||||
avail = [terse_sslver(x[6:]) for x in flags]
|
||||
avail = " ".join(sorted(avail) + ["all"])
|
||||
avail1 = [terse_sslver(x[6:]) for x in flags]
|
||||
avail = " ".join(sorted(avail1) + ["all"])
|
||||
lprint("\navailable ssl/tls versions:\n " + avail)
|
||||
sys.exit(0)
|
||||
|
||||
|
|
@ -160,7 +167,7 @@ def configure_ssl_ver(al):
|
|||
# think i need that beer now
|
||||
|
||||
|
||||
def configure_ssl_ciphers(al):
|
||||
def configure_ssl_ciphers(al: argparse.Namespace) -> None:
|
||||
ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
|
||||
if al.ssl_ver:
|
||||
ctx.options &= ~al.ssl_flags_en
|
||||
|
|
@ -184,8 +191,8 @@ def configure_ssl_ciphers(al):
|
|||
sys.exit(0)
|
||||
|
||||
|
||||
def args_from_cfg(cfg_path):
|
||||
ret = []
|
||||
def args_from_cfg(cfg_path: str) -> list[str]:
|
||||
ret: list[str] = []
|
||||
skip = False
|
||||
with open(cfg_path, "rb") as f:
|
||||
for ln in [x.decode("utf-8").strip() for x in f]:
|
||||
|
|
@ -210,29 +217,30 @@ def args_from_cfg(cfg_path):
|
|||
return ret
|
||||
|
||||
|
||||
def sighandler(sig=None, frame=None):
|
||||
def sighandler(sig: Optional[int] = None, frame: Optional[FrameType] = None) -> None:
|
||||
msg = [""] * 5
|
||||
for th in threading.enumerate():
|
||||
stk = sys._current_frames()[th.ident] # type: ignore
|
||||
msg.append(str(th))
|
||||
msg.extend(traceback.format_stack(sys._current_frames()[th.ident]))
|
||||
msg.extend(traceback.format_stack(stk))
|
||||
|
||||
msg.append("\n")
|
||||
print("\n".join(msg))
|
||||
|
||||
|
||||
def disable_quickedit():
|
||||
import ctypes
|
||||
def disable_quickedit() -> None:
|
||||
import atexit
|
||||
import ctypes
|
||||
from ctypes import wintypes
|
||||
|
||||
def ecb(ok, fun, args):
|
||||
def ecb(ok: bool, fun: Any, args: list[Any]) -> list[Any]:
|
||||
if not ok:
|
||||
err = ctypes.get_last_error()
|
||||
err: int = ctypes.get_last_error() # type: ignore
|
||||
if err:
|
||||
raise ctypes.WinError(err)
|
||||
raise ctypes.WinError(err) # type: ignore
|
||||
return args
|
||||
|
||||
k32 = ctypes.WinDLL("kernel32", use_last_error=True)
|
||||
k32 = ctypes.WinDLL("kernel32", use_last_error=True) # type: ignore
|
||||
if PY2:
|
||||
wintypes.LPDWORD = ctypes.POINTER(wintypes.DWORD)
|
||||
|
||||
|
|
@ -242,14 +250,14 @@ def disable_quickedit():
|
|||
k32.GetConsoleMode.argtypes = (wintypes.HANDLE, wintypes.LPDWORD)
|
||||
k32.SetConsoleMode.argtypes = (wintypes.HANDLE, wintypes.DWORD)
|
||||
|
||||
def cmode(out, mode=None):
|
||||
def cmode(out: bool, mode: Optional[int] = None) -> int:
|
||||
h = k32.GetStdHandle(-11 if out else -10)
|
||||
if mode:
|
||||
return k32.SetConsoleMode(h, mode)
|
||||
return k32.SetConsoleMode(h, mode) # type: ignore
|
||||
|
||||
mode = wintypes.DWORD()
|
||||
k32.GetConsoleMode(h, ctypes.byref(mode))
|
||||
return mode.value
|
||||
cmode = wintypes.DWORD()
|
||||
k32.GetConsoleMode(h, ctypes.byref(cmode))
|
||||
return cmode.value
|
||||
|
||||
# disable quickedit
|
||||
mode = orig_in = cmode(False)
|
||||
|
|
@ -268,7 +276,7 @@ def disable_quickedit():
|
|||
cmode(True, mode | 4)
|
||||
|
||||
|
||||
def run_argparse(argv, formatter):
|
||||
def run_argparse(argv: list[str], formatter: Any) -> argparse.Namespace:
|
||||
ap = argparse.ArgumentParser(
|
||||
formatter_class=formatter,
|
||||
prog="copyparty",
|
||||
|
|
@ -596,7 +604,7 @@ def run_argparse(argv, formatter):
|
|||
return ret
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
def main(argv: Optional[list[str]] = None) -> None:
|
||||
time.strptime("19970815", "%Y%m%d") # python#7980
|
||||
if WINDOWS:
|
||||
os.system("rem") # enables colors
|
||||
|
|
@ -618,7 +626,7 @@ def main(argv=None):
|
|||
supp = args_from_cfg(v)
|
||||
argv.extend(supp)
|
||||
|
||||
deprecated = []
|
||||
deprecated: list[tuple[str, str]] = []
|
||||
for dk, nk in deprecated:
|
||||
try:
|
||||
idx = argv.index(dk)
|
||||
|
|
@ -650,7 +658,7 @@ def main(argv=None):
|
|||
if not VT100:
|
||||
al.wintitle = ""
|
||||
|
||||
nstrs = []
|
||||
nstrs: list[str] = []
|
||||
anymod = False
|
||||
for ostr in al.v or []:
|
||||
m = re_vol.match(ostr)
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -2,23 +2,30 @@
|
|||
from __future__ import print_function, unicode_literals
|
||||
|
||||
import os
|
||||
from ..util import fsenc, fsdec, SYMTIME
|
||||
|
||||
from ..util import SYMTIME, fsdec, fsenc
|
||||
from . import path
|
||||
|
||||
try:
|
||||
from typing import Optional
|
||||
except:
|
||||
pass
|
||||
|
||||
_ = (path,)
|
||||
|
||||
# grep -hRiE '(^|[^a-zA-Z_\.-])os\.' . | gsed -r 's/ /\n/g;s/\(/(\n/g' | grep -hRiE '(^|[^a-zA-Z_\.-])os\.' | sort | uniq -c
|
||||
# printf 'os\.(%s)' "$(grep ^def bos/__init__.py | gsed -r 's/^def //;s/\(.*//' | tr '\n' '|' | gsed -r 's/.$//')"
|
||||
|
||||
|
||||
def chmod(p, mode):
|
||||
def chmod(p: str, mode: int) -> None:
|
||||
return os.chmod(fsenc(p), mode)
|
||||
|
||||
|
||||
def listdir(p="."):
|
||||
def listdir(p: str = ".") -> list[str]:
|
||||
return [fsdec(x) for x in os.listdir(fsenc(p))]
|
||||
|
||||
|
||||
def makedirs(name, mode=0o755, exist_ok=True):
|
||||
def makedirs(name: str, mode: int = 0o755, exist_ok: bool = True) -> None:
|
||||
bname = fsenc(name)
|
||||
try:
|
||||
os.makedirs(bname, mode)
|
||||
|
|
@ -27,31 +34,33 @@ def makedirs(name, mode=0o755, exist_ok=True):
|
|||
raise
|
||||
|
||||
|
||||
def mkdir(p, mode=0o755):
|
||||
def mkdir(p: str, mode: int = 0o755) -> None:
|
||||
return os.mkdir(fsenc(p), mode)
|
||||
|
||||
|
||||
def rename(src, dst):
|
||||
def rename(src: str, dst: str) -> None:
|
||||
return os.rename(fsenc(src), fsenc(dst))
|
||||
|
||||
|
||||
def replace(src, dst):
|
||||
def replace(src: str, dst: str) -> None:
|
||||
return os.replace(fsenc(src), fsenc(dst))
|
||||
|
||||
|
||||
def rmdir(p):
|
||||
def rmdir(p: str) -> None:
|
||||
return os.rmdir(fsenc(p))
|
||||
|
||||
|
||||
def stat(p):
|
||||
def stat(p: str) -> os.stat_result:
|
||||
return os.stat(fsenc(p))
|
||||
|
||||
|
||||
def unlink(p):
|
||||
def unlink(p: str) -> None:
|
||||
return os.unlink(fsenc(p))
|
||||
|
||||
|
||||
def utime(p, times=None, follow_symlinks=True):
|
||||
def utime(
|
||||
p: str, times: Optional[tuple[float, float]] = None, follow_symlinks: bool = True
|
||||
) -> None:
|
||||
if SYMTIME:
|
||||
return os.utime(fsenc(p), times, follow_symlinks=follow_symlinks)
|
||||
else:
|
||||
|
|
@ -60,7 +69,7 @@ def utime(p, times=None, follow_symlinks=True):
|
|||
|
||||
if hasattr(os, "lstat"):
|
||||
|
||||
def lstat(p):
|
||||
def lstat(p: str) -> os.stat_result:
|
||||
return os.lstat(fsenc(p))
|
||||
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -2,43 +2,44 @@
|
|||
from __future__ import print_function, unicode_literals
|
||||
|
||||
import os
|
||||
from ..util import fsenc, fsdec, SYMTIME
|
||||
|
||||
from ..util import SYMTIME, fsdec, fsenc
|
||||
|
||||
|
||||
def abspath(p):
|
||||
def abspath(p: str) -> str:
|
||||
return fsdec(os.path.abspath(fsenc(p)))
|
||||
|
||||
|
||||
def exists(p):
|
||||
def exists(p: str) -> bool:
|
||||
return os.path.exists(fsenc(p))
|
||||
|
||||
|
||||
def getmtime(p, follow_symlinks=True):
|
||||
def getmtime(p: str, follow_symlinks: bool = True) -> float:
|
||||
if not follow_symlinks and SYMTIME:
|
||||
return os.lstat(fsenc(p)).st_mtime
|
||||
else:
|
||||
return os.path.getmtime(fsenc(p))
|
||||
|
||||
|
||||
def getsize(p):
|
||||
def getsize(p: str) -> int:
|
||||
return os.path.getsize(fsenc(p))
|
||||
|
||||
|
||||
def isfile(p):
|
||||
def isfile(p: str) -> bool:
|
||||
return os.path.isfile(fsenc(p))
|
||||
|
||||
|
||||
def isdir(p):
|
||||
def isdir(p: str) -> bool:
|
||||
return os.path.isdir(fsenc(p))
|
||||
|
||||
|
||||
def islink(p):
|
||||
def islink(p: str) -> bool:
|
||||
return os.path.islink(fsenc(p))
|
||||
|
||||
|
||||
def lexists(p):
|
||||
def lexists(p: str) -> bool:
|
||||
return os.path.lexists(fsenc(p))
|
||||
|
||||
|
||||
def realpath(p):
|
||||
def realpath(p: str) -> str:
|
||||
return fsdec(os.path.realpath(fsenc(p)))
|
||||
|
|
|
|||
|
|
@ -1,37 +1,56 @@
|
|||
# coding: utf-8
|
||||
from __future__ import print_function, unicode_literals
|
||||
|
||||
import time
|
||||
import threading
|
||||
import time
|
||||
|
||||
from .broker_util import try_exec
|
||||
import queue
|
||||
|
||||
from .__init__ import TYPE_CHECKING
|
||||
from .broker_mpw import MpWorker
|
||||
from .broker_util import try_exec
|
||||
from .util import mp
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .svchub import SvcHub
|
||||
|
||||
try:
|
||||
from typing import Any
|
||||
except:
|
||||
pass
|
||||
|
||||
|
||||
class MProcess(mp.Process):
|
||||
def __init__(
|
||||
self,
|
||||
q_pend: queue.Queue[tuple[int, str, list[Any]]],
|
||||
q_yield: queue.Queue[tuple[int, str, list[Any]]],
|
||||
target: Any,
|
||||
args: Any,
|
||||
) -> None:
|
||||
super(MProcess, self).__init__(target=target, args=args)
|
||||
self.q_pend = q_pend
|
||||
self.q_yield = q_yield
|
||||
|
||||
|
||||
class BrokerMp(object):
|
||||
"""external api; manages MpWorkers"""
|
||||
|
||||
def __init__(self, hub):
|
||||
def __init__(self, hub: "SvcHub") -> None:
|
||||
self.hub = hub
|
||||
self.log = hub.log
|
||||
self.args = hub.args
|
||||
|
||||
self.procs = []
|
||||
self.retpend = {}
|
||||
self.retpend_mutex = threading.Lock()
|
||||
self.mutex = threading.Lock()
|
||||
|
||||
self.num_workers = self.args.j or mp.cpu_count()
|
||||
self.log("broker", "booting {} subprocesses".format(self.num_workers))
|
||||
for n in range(1, self.num_workers + 1):
|
||||
q_pend = mp.Queue(1)
|
||||
q_yield = mp.Queue(64)
|
||||
q_pend: queue.Queue[tuple[int, str, list[Any]]] = mp.Queue(1)
|
||||
q_yield: queue.Queue[tuple[int, str, list[Any]]] = mp.Queue(64)
|
||||
|
||||
proc = mp.Process(target=MpWorker, args=(q_pend, q_yield, self.args, n))
|
||||
proc.q_pend = q_pend
|
||||
proc.q_yield = q_yield
|
||||
proc.clients = {}
|
||||
proc = MProcess(q_pend, q_yield, MpWorker, (q_pend, q_yield, self.args, n))
|
||||
|
||||
thr = threading.Thread(
|
||||
target=self.collector, args=(proc,), name="mp-sink-{}".format(n)
|
||||
|
|
@ -42,11 +61,11 @@ class BrokerMp(object):
|
|||
self.procs.append(proc)
|
||||
proc.start()
|
||||
|
||||
def shutdown(self):
|
||||
def shutdown(self) -> None:
|
||||
self.log("broker", "shutting down")
|
||||
for n, proc in enumerate(self.procs):
|
||||
thr = threading.Thread(
|
||||
target=proc.q_pend.put([0, "shutdown", []]),
|
||||
target=proc.q_pend.put((0, "shutdown", [])),
|
||||
name="mp-shutdown-{}-{}".format(n, len(self.procs)),
|
||||
)
|
||||
thr.start()
|
||||
|
|
@ -62,12 +81,12 @@ class BrokerMp(object):
|
|||
|
||||
procs.pop()
|
||||
|
||||
def reload(self):
|
||||
def reload(self) -> None:
|
||||
self.log("broker", "reloading")
|
||||
for _, proc in enumerate(self.procs):
|
||||
proc.q_pend.put([0, "reload", []])
|
||||
proc.q_pend.put((0, "reload", []))
|
||||
|
||||
def collector(self, proc):
|
||||
def collector(self, proc: MProcess) -> None:
|
||||
"""receive message from hub in other process"""
|
||||
while True:
|
||||
msg = proc.q_yield.get()
|
||||
|
|
@ -78,10 +97,7 @@ class BrokerMp(object):
|
|||
|
||||
elif dest == "retq":
|
||||
# response from previous ipc call
|
||||
with self.retpend_mutex:
|
||||
retq = self.retpend.pop(retq_id)
|
||||
|
||||
retq.put(args)
|
||||
raise Exception("invalid broker_mp usage")
|
||||
|
||||
else:
|
||||
# new ipc invoking managed service in hub
|
||||
|
|
@ -93,9 +109,9 @@ class BrokerMp(object):
|
|||
rv = try_exec(retq_id, obj, *args)
|
||||
|
||||
if retq_id:
|
||||
proc.q_pend.put([retq_id, "retq", rv])
|
||||
proc.q_pend.put((retq_id, "retq", rv))
|
||||
|
||||
def put(self, want_retval, dest, *args):
|
||||
def say(self, dest: str, *args: Any) -> None:
|
||||
"""
|
||||
send message to non-hub component in other process,
|
||||
returns a Queue object which eventually contains the response if want_retval
|
||||
|
|
@ -103,7 +119,7 @@ class BrokerMp(object):
|
|||
"""
|
||||
if dest == "listen":
|
||||
for p in self.procs:
|
||||
p.q_pend.put([0, dest, [args[0], len(self.procs)]])
|
||||
p.q_pend.put((0, dest, [args[0], len(self.procs)]))
|
||||
|
||||
elif dest == "cb_httpsrv_up":
|
||||
self.hub.cb_httpsrv_up()
|
||||
|
|
|
|||
|
|
@ -1,20 +1,38 @@
|
|||
# coding: utf-8
|
||||
from __future__ import print_function, unicode_literals
|
||||
|
||||
import sys
|
||||
import argparse
|
||||
import signal
|
||||
import sys
|
||||
import threading
|
||||
|
||||
from .broker_util import ExceptionalQueue
|
||||
import queue
|
||||
|
||||
from .authsrv import AuthSrv
|
||||
from .broker_util import BrokerCli, ExceptionalQueue
|
||||
from .httpsrv import HttpSrv
|
||||
from .util import FAKE_MP
|
||||
from .authsrv import AuthSrv
|
||||
|
||||
try:
|
||||
from types import FrameType
|
||||
|
||||
from typing import Any, Optional, Union
|
||||
except:
|
||||
pass
|
||||
|
||||
|
||||
class MpWorker(object):
|
||||
class MpWorker(BrokerCli):
|
||||
"""one single mp instance"""
|
||||
|
||||
def __init__(self, q_pend, q_yield, args, n):
|
||||
def __init__(
|
||||
self,
|
||||
q_pend: queue.Queue[tuple[int, str, list[Any]]],
|
||||
q_yield: queue.Queue[tuple[int, str, list[Any]]],
|
||||
args: argparse.Namespace,
|
||||
n: int,
|
||||
) -> None:
|
||||
super(MpWorker, self).__init__()
|
||||
|
||||
self.q_pend = q_pend
|
||||
self.q_yield = q_yield
|
||||
self.args = args
|
||||
|
|
@ -22,7 +40,7 @@ class MpWorker(object):
|
|||
|
||||
self.log = self._log_disabled if args.q and not args.lo else self._log_enabled
|
||||
|
||||
self.retpend = {}
|
||||
self.retpend: dict[int, Any] = {}
|
||||
self.retpend_mutex = threading.Lock()
|
||||
self.mutex = threading.Lock()
|
||||
|
||||
|
|
@ -45,20 +63,20 @@ class MpWorker(object):
|
|||
thr.start()
|
||||
thr.join()
|
||||
|
||||
def signal_handler(self, sig, frame):
|
||||
def signal_handler(self, sig: Optional[int], frame: Optional[FrameType]) -> None:
|
||||
# print('k')
|
||||
pass
|
||||
|
||||
def _log_enabled(self, src, msg, c=0):
|
||||
self.q_yield.put([0, "log", [src, msg, c]])
|
||||
def _log_enabled(self, src: str, msg: str, c: Union[int, str] = 0) -> None:
|
||||
self.q_yield.put((0, "log", [src, msg, c]))
|
||||
|
||||
def _log_disabled(self, src, msg, c=0):
|
||||
def _log_disabled(self, src: str, msg: str, c: Union[int, str] = 0) -> None:
|
||||
pass
|
||||
|
||||
def logw(self, msg, c=0):
|
||||
def logw(self, msg: str, c: Union[int, str] = 0) -> None:
|
||||
self.log("mp{}".format(self.n), msg, c)
|
||||
|
||||
def main(self):
|
||||
def main(self) -> None:
|
||||
while True:
|
||||
retq_id, dest, args = self.q_pend.get()
|
||||
|
||||
|
|
@ -87,15 +105,14 @@ class MpWorker(object):
|
|||
else:
|
||||
raise Exception("what is " + str(dest))
|
||||
|
||||
def put(self, want_retval, dest, *args):
|
||||
if want_retval:
|
||||
retq = ExceptionalQueue(1)
|
||||
retq_id = id(retq)
|
||||
with self.retpend_mutex:
|
||||
self.retpend[retq_id] = retq
|
||||
else:
|
||||
retq = None
|
||||
retq_id = 0
|
||||
def ask(self, dest: str, *args: Any) -> ExceptionalQueue:
|
||||
retq = ExceptionalQueue(1)
|
||||
retq_id = id(retq)
|
||||
with self.retpend_mutex:
|
||||
self.retpend[retq_id] = retq
|
||||
|
||||
self.q_yield.put([retq_id, dest, args])
|
||||
self.q_yield.put((retq_id, dest, list(args)))
|
||||
return retq
|
||||
|
||||
def say(self, dest: str, *args: Any) -> None:
|
||||
self.q_yield.put((0, dest, list(args)))
|
||||
|
|
|
|||
|
|
@ -3,14 +3,25 @@ from __future__ import print_function, unicode_literals
|
|||
|
||||
import threading
|
||||
|
||||
from .__init__ import TYPE_CHECKING
|
||||
from .broker_util import BrokerCli, ExceptionalQueue, try_exec
|
||||
from .httpsrv import HttpSrv
|
||||
from .broker_util import ExceptionalQueue, try_exec
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .svchub import SvcHub
|
||||
|
||||
try:
|
||||
from typing import Any
|
||||
except:
|
||||
pass
|
||||
|
||||
|
||||
class BrokerThr(object):
|
||||
class BrokerThr(BrokerCli):
|
||||
"""external api; behaves like BrokerMP but using plain threads"""
|
||||
|
||||
def __init__(self, hub):
|
||||
def __init__(self, hub: "SvcHub") -> None:
|
||||
super(BrokerThr, self).__init__()
|
||||
|
||||
self.hub = hub
|
||||
self.log = hub.log
|
||||
self.args = hub.args
|
||||
|
|
@ -23,29 +34,35 @@ class BrokerThr(object):
|
|||
self.httpsrv = HttpSrv(self, None)
|
||||
self.reload = self.noop
|
||||
|
||||
def shutdown(self):
|
||||
def shutdown(self) -> None:
|
||||
# self.log("broker", "shutting down")
|
||||
self.httpsrv.shutdown()
|
||||
|
||||
def noop(self):
|
||||
def noop(self) -> None:
|
||||
pass
|
||||
|
||||
def put(self, want_retval, dest, *args):
|
||||
def ask(self, dest: str, *args: Any) -> ExceptionalQueue:
|
||||
|
||||
# new ipc invoking managed service in hub
|
||||
obj = self.hub
|
||||
for node in dest.split("."):
|
||||
obj = getattr(obj, node)
|
||||
|
||||
rv = try_exec(True, obj, *args)
|
||||
|
||||
# pretend we're broker_mp
|
||||
retq = ExceptionalQueue(1)
|
||||
retq.put(rv)
|
||||
return retq
|
||||
|
||||
def say(self, dest: str, *args: Any) -> None:
|
||||
if dest == "listen":
|
||||
self.httpsrv.listen(args[0], 1)
|
||||
return
|
||||
|
||||
else:
|
||||
# new ipc invoking managed service in hub
|
||||
obj = self.hub
|
||||
for node in dest.split("."):
|
||||
obj = getattr(obj, node)
|
||||
# new ipc invoking managed service in hub
|
||||
obj = self.hub
|
||||
for node in dest.split("."):
|
||||
obj = getattr(obj, node)
|
||||
|
||||
# TODO will deadlock if dest performs another ipc
|
||||
rv = try_exec(want_retval, obj, *args)
|
||||
if not want_retval:
|
||||
return
|
||||
|
||||
# pretend we're broker_mp
|
||||
retq = ExceptionalQueue(1)
|
||||
retq.put(rv)
|
||||
return retq
|
||||
try_exec(False, obj, *args)
|
||||
|
|
|
|||
|
|
@ -1,14 +1,28 @@
|
|||
# coding: utf-8
|
||||
from __future__ import print_function, unicode_literals
|
||||
|
||||
|
||||
import argparse
|
||||
import traceback
|
||||
|
||||
from .util import Pebkac, Queue
|
||||
from queue import Queue
|
||||
|
||||
from .__init__ import TYPE_CHECKING
|
||||
from .authsrv import AuthSrv
|
||||
from .util import Pebkac
|
||||
|
||||
try:
|
||||
from typing import Any, Optional, Union
|
||||
|
||||
from .util import RootLogger
|
||||
except:
|
||||
pass
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .httpsrv import HttpSrv
|
||||
|
||||
|
||||
class ExceptionalQueue(Queue, object):
|
||||
def get(self, block=True, timeout=None):
|
||||
def get(self, block: bool = True, timeout: Optional[float] = None) -> Any:
|
||||
rv = super(ExceptionalQueue, self).get(block, timeout)
|
||||
|
||||
if isinstance(rv, list):
|
||||
|
|
@ -21,7 +35,26 @@ class ExceptionalQueue(Queue, object):
|
|||
return rv
|
||||
|
||||
|
||||
def try_exec(want_retval, func, *args):
|
||||
class BrokerCli(object):
|
||||
"""
|
||||
helps mypy understand httpsrv.broker but still fails a few levels deeper,
|
||||
for example resolving httpconn.* in httpcli -- see lines tagged #mypy404
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.log: RootLogger = None
|
||||
self.args: argparse.Namespace = None
|
||||
self.asrv: AuthSrv = None
|
||||
self.httpsrv: "HttpSrv" = None
|
||||
|
||||
def ask(self, dest: str, *args: Any) -> ExceptionalQueue:
|
||||
return ExceptionalQueue(1)
|
||||
|
||||
def say(self, dest: str, *args: Any) -> None:
|
||||
pass
|
||||
|
||||
|
||||
def try_exec(want_retval: Union[bool, int], func: Any, *args: list[Any]) -> Any:
|
||||
try:
|
||||
return func(*args)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,23 +1,23 @@
|
|||
# coding: utf-8
|
||||
from __future__ import print_function, unicode_literals
|
||||
|
||||
import os
|
||||
import sys
|
||||
import stat
|
||||
import time
|
||||
import logging
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
import stat
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
|
||||
from pyftpdlib.authorizers import DummyAuthorizer, AuthenticationFailed
|
||||
from pyftpdlib.authorizers import AuthenticationFailed, DummyAuthorizer
|
||||
from pyftpdlib.filesystems import AbstractedFS, FilesystemError
|
||||
from pyftpdlib.handlers import FTPHandler
|
||||
from pyftpdlib.servers import FTPServer
|
||||
from pyftpdlib.log import config_logging
|
||||
from pyftpdlib.servers import FTPServer
|
||||
|
||||
from .__init__ import E, PY2
|
||||
from .util import Pebkac, fsenc, exclude_dotfiles
|
||||
from .__init__ import PY2, TYPE_CHECKING, E
|
||||
from .bos import bos
|
||||
from .util import Pebkac, exclude_dotfiles, fsenc
|
||||
|
||||
try:
|
||||
from pyftpdlib.ioloop import IOLoop
|
||||
|
|
@ -28,58 +28,63 @@ except ImportError:
|
|||
from pyftpdlib.ioloop import IOLoop
|
||||
|
||||
|
||||
try:
|
||||
from typing import TYPE_CHECKING
|
||||
if TYPE_CHECKING:
|
||||
from .svchub import SvcHub
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .svchub import SvcHub
|
||||
except ImportError:
|
||||
try:
|
||||
import typing
|
||||
from typing import Any, Optional
|
||||
except:
|
||||
pass
|
||||
|
||||
|
||||
class FtpAuth(DummyAuthorizer):
|
||||
def __init__(self):
|
||||
def __init__(self, hub: "SvcHub") -> None:
|
||||
super(FtpAuth, self).__init__()
|
||||
self.hub = None # type: SvcHub
|
||||
self.hub = hub
|
||||
|
||||
def validate_authentication(self, username, password, handler):
|
||||
def validate_authentication(
|
||||
self, username: str, password: str, handler: Any
|
||||
) -> None:
|
||||
asrv = self.hub.asrv
|
||||
if username == "anonymous":
|
||||
password = ""
|
||||
|
||||
uname = "*"
|
||||
if password:
|
||||
uname = asrv.iacct.get(password, None)
|
||||
uname = asrv.iacct.get(password, "")
|
||||
|
||||
handler.username = uname
|
||||
|
||||
if password and not uname:
|
||||
raise AuthenticationFailed("Authentication failed.")
|
||||
|
||||
def get_home_dir(self, username):
|
||||
def get_home_dir(self, username: str) -> str:
|
||||
return "/"
|
||||
|
||||
def has_user(self, username):
|
||||
def has_user(self, username: str) -> bool:
|
||||
asrv = self.hub.asrv
|
||||
return username in asrv.acct
|
||||
|
||||
def has_perm(self, username, perm, path=None):
|
||||
def has_perm(self, username: str, perm: int, path: Optional[str] = None) -> bool:
|
||||
return True # handled at filesystem layer
|
||||
|
||||
def get_perms(self, username):
|
||||
def get_perms(self, username: str) -> str:
|
||||
return "elradfmwMT"
|
||||
|
||||
def get_msg_login(self, username):
|
||||
def get_msg_login(self, username: str) -> str:
|
||||
return "sup {}".format(username)
|
||||
|
||||
def get_msg_quit(self, username):
|
||||
def get_msg_quit(self, username: str) -> str:
|
||||
return "cya"
|
||||
|
||||
|
||||
class FtpFs(AbstractedFS):
|
||||
def __init__(self, root, cmd_channel):
|
||||
def __init__(
|
||||
self, root: str, cmd_channel: Any
|
||||
) -> None: # pylint: disable=super-init-not-called
|
||||
self.h = self.cmd_channel = cmd_channel # type: FTPHandler
|
||||
self.hub = cmd_channel.hub # type: SvcHub
|
||||
self.hub: "SvcHub" = cmd_channel.hub
|
||||
self.args = cmd_channel.args
|
||||
|
||||
self.uname = self.hub.asrv.iacct.get(cmd_channel.password, "*")
|
||||
|
|
@ -90,7 +95,14 @@ class FtpFs(AbstractedFS):
|
|||
self.listdirinfo = self.listdir
|
||||
self.chdir(".")
|
||||
|
||||
def v2a(self, vpath, r=False, w=False, m=False, d=False):
|
||||
def v2a(
|
||||
self,
|
||||
vpath: str,
|
||||
r: bool = False,
|
||||
w: bool = False,
|
||||
m: bool = False,
|
||||
d: bool = False,
|
||||
) -> str:
|
||||
try:
|
||||
vpath = vpath.replace("\\", "/").lstrip("/")
|
||||
vfs, rem = self.hub.asrv.vfs.get(vpath, self.uname, r, w, m, d)
|
||||
|
|
@ -101,25 +113,32 @@ class FtpFs(AbstractedFS):
|
|||
except Pebkac as ex:
|
||||
raise FilesystemError(str(ex))
|
||||
|
||||
def rv2a(self, vpath, r=False, w=False, m=False, d=False):
|
||||
def rv2a(
|
||||
self,
|
||||
vpath: str,
|
||||
r: bool = False,
|
||||
w: bool = False,
|
||||
m: bool = False,
|
||||
d: bool = False,
|
||||
) -> str:
|
||||
return self.v2a(os.path.join(self.cwd, vpath), r, w, m, d)
|
||||
|
||||
def ftp2fs(self, ftppath):
|
||||
def ftp2fs(self, ftppath: str) -> str:
|
||||
# return self.v2a(ftppath)
|
||||
return ftppath # self.cwd must be vpath
|
||||
|
||||
def fs2ftp(self, fspath):
|
||||
def fs2ftp(self, fspath: str) -> str:
|
||||
# raise NotImplementedError()
|
||||
return fspath
|
||||
|
||||
def validpath(self, path):
|
||||
def validpath(self, path: str) -> bool:
|
||||
if "/.hist/" in path:
|
||||
if "/up2k." in path or path.endswith("/dir.txt"):
|
||||
raise FilesystemError("access to this file is forbidden")
|
||||
|
||||
return True
|
||||
|
||||
def open(self, filename, mode):
|
||||
def open(self, filename: str, mode: str) -> typing.IO[Any]:
|
||||
r = "r" in mode
|
||||
w = "w" in mode or "a" in mode or "+" in mode
|
||||
|
||||
|
|
@ -130,24 +149,24 @@ class FtpFs(AbstractedFS):
|
|||
self.validpath(ap)
|
||||
return open(fsenc(ap), mode)
|
||||
|
||||
def chdir(self, path):
|
||||
def chdir(self, path: str) -> None:
|
||||
self.cwd = join(self.cwd, path)
|
||||
x = self.hub.asrv.vfs.can_access(self.cwd.lstrip("/"), self.h.username)
|
||||
self.can_read, self.can_write, self.can_move, self.can_delete, self.can_get = x
|
||||
|
||||
def mkdir(self, path):
|
||||
def mkdir(self, path: str) -> None:
|
||||
ap = self.rv2a(path, w=True)
|
||||
bos.mkdir(ap)
|
||||
|
||||
def listdir(self, path):
|
||||
def listdir(self, path: str) -> list[str]:
|
||||
vpath = join(self.cwd, path).lstrip("/")
|
||||
try:
|
||||
vfs, rem = self.hub.asrv.vfs.get(vpath, self.uname, True, False)
|
||||
|
||||
fsroot, vfs_ls, vfs_virt = vfs.ls(
|
||||
fsroot, vfs_ls1, vfs_virt = vfs.ls(
|
||||
rem, self.uname, not self.args.no_scandir, [[True], [False, True]]
|
||||
)
|
||||
vfs_ls = [x[0] for x in vfs_ls]
|
||||
vfs_ls = [x[0] for x in vfs_ls1]
|
||||
vfs_ls.extend(vfs_virt.keys())
|
||||
|
||||
if not self.args.ed:
|
||||
|
|
@ -164,11 +183,11 @@ class FtpFs(AbstractedFS):
|
|||
r = {x.split("/")[0]: 1 for x in self.hub.asrv.vfs.all_vols.keys()}
|
||||
return list(sorted(list(r.keys())))
|
||||
|
||||
def rmdir(self, path):
|
||||
def rmdir(self, path: str) -> None:
|
||||
ap = self.rv2a(path, d=True)
|
||||
bos.rmdir(ap)
|
||||
|
||||
def remove(self, path):
|
||||
def remove(self, path: str) -> None:
|
||||
if self.args.no_del:
|
||||
raise FilesystemError("the delete feature is disabled in server config")
|
||||
|
||||
|
|
@ -178,13 +197,13 @@ class FtpFs(AbstractedFS):
|
|||
except Exception as ex:
|
||||
raise FilesystemError(str(ex))
|
||||
|
||||
def rename(self, src, dst):
|
||||
def rename(self, src: str, dst: str) -> None:
|
||||
if not self.can_move:
|
||||
raise FilesystemError("not allowed for user " + self.h.username)
|
||||
|
||||
if self.args.no_mv:
|
||||
m = "the rename/move feature is disabled in server config"
|
||||
raise FilesystemError(m)
|
||||
t = "the rename/move feature is disabled in server config"
|
||||
raise FilesystemError(t)
|
||||
|
||||
svp = join(self.cwd, src).lstrip("/")
|
||||
dvp = join(self.cwd, dst).lstrip("/")
|
||||
|
|
@ -193,10 +212,10 @@ class FtpFs(AbstractedFS):
|
|||
except Exception as ex:
|
||||
raise FilesystemError(str(ex))
|
||||
|
||||
def chmod(self, path, mode):
|
||||
def chmod(self, path: str, mode: str) -> None:
|
||||
pass
|
||||
|
||||
def stat(self, path):
|
||||
def stat(self, path: str) -> os.stat_result:
|
||||
try:
|
||||
ap = self.rv2a(path, r=True)
|
||||
return bos.stat(ap)
|
||||
|
|
@ -208,59 +227,59 @@ class FtpFs(AbstractedFS):
|
|||
|
||||
return st
|
||||
|
||||
def utime(self, path, timeval):
|
||||
def utime(self, path: str, timeval: float) -> None:
|
||||
ap = self.rv2a(path, w=True)
|
||||
return bos.utime(ap, (timeval, timeval))
|
||||
|
||||
def lstat(self, path):
|
||||
def lstat(self, path: str) -> os.stat_result:
|
||||
ap = self.rv2a(path)
|
||||
return bos.lstat(ap)
|
||||
|
||||
def isfile(self, path):
|
||||
def isfile(self, path: str) -> bool:
|
||||
st = self.stat(path)
|
||||
return stat.S_ISREG(st.st_mode)
|
||||
|
||||
def islink(self, path):
|
||||
def islink(self, path: str) -> bool:
|
||||
ap = self.rv2a(path)
|
||||
return bos.path.islink(ap)
|
||||
|
||||
def isdir(self, path):
|
||||
def isdir(self, path: str) -> bool:
|
||||
try:
|
||||
st = self.stat(path)
|
||||
return stat.S_ISDIR(st.st_mode)
|
||||
except:
|
||||
return True
|
||||
|
||||
def getsize(self, path):
|
||||
def getsize(self, path: str) -> int:
|
||||
ap = self.rv2a(path)
|
||||
return bos.path.getsize(ap)
|
||||
|
||||
def getmtime(self, path):
|
||||
def getmtime(self, path: str) -> float:
|
||||
ap = self.rv2a(path)
|
||||
return bos.path.getmtime(ap)
|
||||
|
||||
def realpath(self, path):
|
||||
def realpath(self, path: str) -> str:
|
||||
return path
|
||||
|
||||
def lexists(self, path):
|
||||
def lexists(self, path: str) -> bool:
|
||||
ap = self.rv2a(path)
|
||||
return bos.path.lexists(ap)
|
||||
|
||||
def get_user_by_uid(self, uid):
|
||||
def get_user_by_uid(self, uid: int) -> str:
|
||||
return "root"
|
||||
|
||||
def get_group_by_uid(self, gid):
|
||||
def get_group_by_uid(self, gid: int) -> str:
|
||||
return "root"
|
||||
|
||||
|
||||
class FtpHandler(FTPHandler):
|
||||
abstracted_fs = FtpFs
|
||||
hub = None # type: SvcHub
|
||||
args = None # type: argparse.Namespace
|
||||
hub: "SvcHub" = None
|
||||
args: argparse.Namespace = None
|
||||
|
||||
def __init__(self, conn, server, ioloop=None):
|
||||
self.hub = FtpHandler.hub # type: SvcHub
|
||||
self.args = FtpHandler.args # type: argparse.Namespace
|
||||
def __init__(self, conn: Any, server: Any, ioloop: Any = None) -> None:
|
||||
self.hub: "SvcHub" = FtpHandler.hub
|
||||
self.args: argparse.Namespace = FtpHandler.args
|
||||
|
||||
if PY2:
|
||||
FTPHandler.__init__(self, conn, server, ioloop)
|
||||
|
|
@ -268,9 +287,10 @@ class FtpHandler(FTPHandler):
|
|||
super(FtpHandler, self).__init__(conn, server, ioloop)
|
||||
|
||||
# abspath->vpath mapping to resolve log_transfer paths
|
||||
self.vfs_map = {}
|
||||
self.vfs_map: dict[str, str] = {}
|
||||
|
||||
def ftp_STOR(self, file, mode="w"):
|
||||
def ftp_STOR(self, file: str, mode: str = "w") -> Any:
|
||||
# Optional[str]
|
||||
vp = join(self.fs.cwd, file).lstrip("/")
|
||||
ap = self.fs.v2a(vp)
|
||||
self.vfs_map[ap] = vp
|
||||
|
|
@ -279,7 +299,16 @@ class FtpHandler(FTPHandler):
|
|||
# print("ftp_STOR: {} {} OK".format(vp, mode))
|
||||
return ret
|
||||
|
||||
def log_transfer(self, cmd, filename, receive, completed, elapsed, bytes):
|
||||
def log_transfer(
|
||||
self,
|
||||
cmd: str,
|
||||
filename: bytes,
|
||||
receive: bool,
|
||||
completed: bool,
|
||||
elapsed: float,
|
||||
bytes: int,
|
||||
) -> Any:
|
||||
# None
|
||||
ap = filename.decode("utf-8", "replace")
|
||||
vp = self.vfs_map.pop(ap, None)
|
||||
# print("xfer_end: {} => {}".format(ap, vp))
|
||||
|
|
@ -312,7 +341,7 @@ except:
|
|||
|
||||
|
||||
class Ftpd(object):
|
||||
def __init__(self, hub):
|
||||
def __init__(self, hub: "SvcHub") -> None:
|
||||
self.hub = hub
|
||||
self.args = hub.args
|
||||
|
||||
|
|
@ -321,24 +350,23 @@ class Ftpd(object):
|
|||
hs.append([FtpHandler, self.args.ftp])
|
||||
if self.args.ftps:
|
||||
try:
|
||||
h = SftpHandler
|
||||
h1 = SftpHandler
|
||||
except:
|
||||
m = "\nftps requires pyopenssl;\nplease run the following:\n\n {} -m pip install --user pyopenssl\n"
|
||||
print(m.format(sys.executable))
|
||||
t = "\nftps requires pyopenssl;\nplease run the following:\n\n {} -m pip install --user pyopenssl\n"
|
||||
print(t.format(sys.executable))
|
||||
sys.exit(1)
|
||||
|
||||
h.certfile = os.path.join(E.cfg, "cert.pem")
|
||||
h.tls_control_required = True
|
||||
h.tls_data_required = True
|
||||
h1.certfile = os.path.join(E.cfg, "cert.pem")
|
||||
h1.tls_control_required = True
|
||||
h1.tls_data_required = True
|
||||
|
||||
hs.append([h, self.args.ftps])
|
||||
hs.append([h1, self.args.ftps])
|
||||
|
||||
for h in hs:
|
||||
h, lp = h
|
||||
h.hub = hub
|
||||
h.args = hub.args
|
||||
h.authorizer = FtpAuth()
|
||||
h.authorizer.hub = hub
|
||||
for h_lp in hs:
|
||||
h2, lp = h_lp
|
||||
h2.hub = hub
|
||||
h2.args = hub.args
|
||||
h2.authorizer = FtpAuth(hub)
|
||||
|
||||
if self.args.ftp_pr:
|
||||
p1, p2 = [int(x) for x in self.args.ftp_pr.split("-")]
|
||||
|
|
@ -350,10 +378,10 @@ class Ftpd(object):
|
|||
else:
|
||||
p1 += d + 1
|
||||
|
||||
h.passive_ports = list(range(p1, p2 + 1))
|
||||
h2.passive_ports = list(range(p1, p2 + 1))
|
||||
|
||||
if self.args.ftp_nat:
|
||||
h.masquerade_address = self.args.ftp_nat
|
||||
h2.masquerade_address = self.args.ftp_nat
|
||||
|
||||
if self.args.ftp_dbg:
|
||||
config_logging(level=logging.DEBUG)
|
||||
|
|
@ -363,11 +391,11 @@ class Ftpd(object):
|
|||
for h, lp in hs:
|
||||
FTPServer((ip, int(lp)), h, ioloop)
|
||||
|
||||
t = threading.Thread(target=ioloop.loop)
|
||||
t.daemon = True
|
||||
t.start()
|
||||
thr = threading.Thread(target=ioloop.loop)
|
||||
thr.daemon = True
|
||||
thr.start()
|
||||
|
||||
|
||||
def join(p1, p2):
|
||||
def join(p1: str, p2: str) -> str:
|
||||
w = os.path.join(p1, p2.replace("\\", "/"))
|
||||
return os.path.normpath(w).replace("\\", "/")
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -1,25 +1,36 @@
|
|||
# coding: utf-8
|
||||
from __future__ import print_function, unicode_literals
|
||||
|
||||
import re
|
||||
import argparse # typechk
|
||||
import os
|
||||
import time
|
||||
import re
|
||||
import socket
|
||||
import threading # typechk
|
||||
import time
|
||||
|
||||
HAVE_SSL = True
|
||||
try:
|
||||
HAVE_SSL = True
|
||||
import ssl
|
||||
except:
|
||||
HAVE_SSL = False
|
||||
|
||||
from .__init__ import E
|
||||
from .util import Unrecv
|
||||
from . import util as Util
|
||||
from .__init__ import TYPE_CHECKING, E
|
||||
from .authsrv import AuthSrv # typechk
|
||||
from .httpcli import HttpCli
|
||||
from .u2idx import U2idx
|
||||
from .ico import Ico
|
||||
from .mtag import HAVE_FFMPEG
|
||||
from .th_cli import ThumbCli
|
||||
from .th_srv import HAVE_PIL, HAVE_VIPS
|
||||
from .mtag import HAVE_FFMPEG
|
||||
from .ico import Ico
|
||||
from .u2idx import U2idx
|
||||
|
||||
try:
|
||||
from typing import Optional, Pattern, Union
|
||||
except:
|
||||
pass
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .httpsrv import HttpSrv
|
||||
|
||||
|
||||
class HttpConn(object):
|
||||
|
|
@ -28,32 +39,37 @@ class HttpConn(object):
|
|||
creates an HttpCli for each request (Connection: Keep-Alive)
|
||||
"""
|
||||
|
||||
def __init__(self, sck, addr, hsrv):
|
||||
def __init__(
|
||||
self, sck: socket.socket, addr: tuple[str, int], hsrv: "HttpSrv"
|
||||
) -> None:
|
||||
self.s = sck
|
||||
self.sr = None # Type: Unrecv
|
||||
self.sr: Optional[Util._Unrecv] = None
|
||||
self.addr = addr
|
||||
self.hsrv = hsrv
|
||||
|
||||
self.mutex = hsrv.mutex
|
||||
self.args = hsrv.args
|
||||
self.asrv = hsrv.asrv
|
||||
self.mutex: threading.Lock = hsrv.mutex # mypy404
|
||||
self.args: argparse.Namespace = hsrv.args # mypy404
|
||||
self.asrv: AuthSrv = hsrv.asrv # mypy404
|
||||
self.cert_path = hsrv.cert_path
|
||||
self.u2fh = hsrv.u2fh
|
||||
self.u2fh: Util.FHC = hsrv.u2fh # mypy404
|
||||
|
||||
enth = (HAVE_PIL or HAVE_VIPS or HAVE_FFMPEG) and not self.args.no_thumb
|
||||
self.thumbcli = ThumbCli(hsrv) if enth else None
|
||||
self.ico = Ico(self.args)
|
||||
self.thumbcli: Optional[ThumbCli] = ThumbCli(hsrv) if enth else None # mypy404
|
||||
self.ico: Ico = Ico(self.args) # mypy404
|
||||
|
||||
self.t0 = time.time()
|
||||
self.t0: float = time.time() # mypy404
|
||||
self.stopping = False
|
||||
self.nreq = 0
|
||||
self.nbyte = 0
|
||||
self.u2idx = None
|
||||
self.log_func = hsrv.log
|
||||
self.lf_url = re.compile(self.args.lf_url) if self.args.lf_url else None
|
||||
self.nreq: int = 0 # mypy404
|
||||
self.nbyte: int = 0 # mypy404
|
||||
self.u2idx: Optional[U2idx] = None
|
||||
self.log_func: Util.RootLogger = hsrv.log # mypy404
|
||||
self.log_src: str = "httpconn" # mypy404
|
||||
self.lf_url: Optional[Pattern[str]] = (
|
||||
re.compile(self.args.lf_url) if self.args.lf_url else None
|
||||
) # mypy404
|
||||
self.set_rproxy()
|
||||
|
||||
def shutdown(self):
|
||||
def shutdown(self) -> None:
|
||||
self.stopping = True
|
||||
try:
|
||||
self.s.shutdown(socket.SHUT_RDWR)
|
||||
|
|
@ -61,7 +77,7 @@ class HttpConn(object):
|
|||
except:
|
||||
pass
|
||||
|
||||
def set_rproxy(self, ip=None):
|
||||
def set_rproxy(self, ip: Optional[str] = None) -> str:
|
||||
if ip is None:
|
||||
color = 36
|
||||
ip = self.addr[0]
|
||||
|
|
@ -74,35 +90,35 @@ class HttpConn(object):
|
|||
self.log_src = "{} \033[{}m{}".format(ip, color, self.addr[1]).ljust(26)
|
||||
return self.log_src
|
||||
|
||||
def respath(self, res_name):
|
||||
def respath(self, res_name: str) -> str:
|
||||
return os.path.join(E.mod, "web", res_name)
|
||||
|
||||
def log(self, msg, c=0):
|
||||
def log(self, msg: str, c: Union[int, str] = 0) -> None:
|
||||
self.log_func(self.log_src, msg, c)
|
||||
|
||||
def get_u2idx(self):
|
||||
def get_u2idx(self) -> U2idx:
|
||||
if not self.u2idx:
|
||||
self.u2idx = U2idx(self)
|
||||
|
||||
return self.u2idx
|
||||
|
||||
def _detect_https(self):
|
||||
def _detect_https(self) -> bool:
|
||||
method = None
|
||||
if self.cert_path:
|
||||
try:
|
||||
method = self.s.recv(4, socket.MSG_PEEK)
|
||||
except socket.timeout:
|
||||
return
|
||||
return False
|
||||
except AttributeError:
|
||||
# jython does not support msg_peek; forget about https
|
||||
method = self.s.recv(4)
|
||||
self.sr = Unrecv(self.s, self.log)
|
||||
self.sr = Util.Unrecv(self.s, self.log)
|
||||
self.sr.buf = method
|
||||
|
||||
# jython used to do this, they stopped since it's broken
|
||||
# but reimplementing sendall is out of scope for now
|
||||
if not getattr(self.s, "sendall", None):
|
||||
self.s.sendall = self.s.send
|
||||
self.s.sendall = self.s.send # type: ignore
|
||||
|
||||
if len(method) != 4:
|
||||
err = "need at least 4 bytes in the first packet; got {}".format(
|
||||
|
|
@ -112,17 +128,18 @@ class HttpConn(object):
|
|||
self.log(err)
|
||||
|
||||
self.s.send(b"HTTP/1.1 400 Bad Request\r\n\r\n" + err.encode("utf-8"))
|
||||
return
|
||||
return False
|
||||
|
||||
return method not in [None, b"GET ", b"HEAD", b"POST", b"PUT ", b"OPTI"]
|
||||
|
||||
def run(self):
|
||||
def run(self) -> None:
|
||||
self.sr = None
|
||||
if self.args.https_only:
|
||||
is_https = True
|
||||
elif self.args.http_only or not HAVE_SSL:
|
||||
is_https = False
|
||||
else:
|
||||
# raise Exception("asdf")
|
||||
is_https = self._detect_https()
|
||||
|
||||
if is_https:
|
||||
|
|
@ -151,14 +168,15 @@ class HttpConn(object):
|
|||
self.s = ctx.wrap_socket(self.s, server_side=True)
|
||||
msg = [
|
||||
"\033[1;3{:d}m{}".format(c, s)
|
||||
for c, s in zip([0, 5, 0], self.s.cipher())
|
||||
for c, s in zip([0, 5, 0], self.s.cipher()) # type: ignore
|
||||
]
|
||||
self.log(" ".join(msg) + "\033[0m")
|
||||
|
||||
if self.args.ssl_dbg and hasattr(self.s, "shared_ciphers"):
|
||||
overlap = [y[::-1] for y in self.s.shared_ciphers()]
|
||||
lines = [str(x) for x in (["TLS cipher overlap:"] + overlap)]
|
||||
self.log("\n".join(lines))
|
||||
ciphers = self.s.shared_ciphers()
|
||||
assert ciphers
|
||||
overlap = [str(y[::-1]) for y in ciphers]
|
||||
self.log("TLS cipher overlap:" + "\n".join(overlap))
|
||||
for k, v in [
|
||||
["compression", self.s.compression()],
|
||||
["ALPN proto", self.s.selected_alpn_protocol()],
|
||||
|
|
@ -183,7 +201,7 @@ class HttpConn(object):
|
|||
return
|
||||
|
||||
if not self.sr:
|
||||
self.sr = Unrecv(self.s, self.log)
|
||||
self.sr = Util.Unrecv(self.s, self.log)
|
||||
|
||||
while not self.stopping:
|
||||
self.nreq += 1
|
||||
|
|
|
|||
|
|
@ -1,13 +1,15 @@
|
|||
# coding: utf-8
|
||||
from __future__ import print_function, unicode_literals
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import math
|
||||
import base64
|
||||
import math
|
||||
import os
|
||||
import socket
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
|
||||
import queue
|
||||
|
||||
try:
|
||||
import jinja2
|
||||
|
|
@ -26,15 +28,18 @@ except ImportError:
|
|||
)
|
||||
sys.exit(1)
|
||||
|
||||
from .__init__ import E, PY2, MACOS
|
||||
from .util import FHC, spack, min_ex, start_stackmon, start_log_thrs
|
||||
from .__init__ import MACOS, TYPE_CHECKING, E
|
||||
from .bos import bos
|
||||
from .httpconn import HttpConn
|
||||
from .util import FHC, min_ex, spack, start_log_thrs, start_stackmon
|
||||
|
||||
if PY2:
|
||||
import Queue as queue
|
||||
else:
|
||||
import queue
|
||||
if TYPE_CHECKING:
|
||||
from .broker_util import BrokerCli
|
||||
|
||||
try:
|
||||
from typing import Any, Optional
|
||||
except:
|
||||
pass
|
||||
|
||||
|
||||
class HttpSrv(object):
|
||||
|
|
@ -43,7 +48,7 @@ class HttpSrv(object):
|
|||
relying on MpSrv for performance (HttpSrv is just plain threads)
|
||||
"""
|
||||
|
||||
def __init__(self, broker, nid):
|
||||
def __init__(self, broker: "BrokerCli", nid: Optional[int]) -> None:
|
||||
self.broker = broker
|
||||
self.nid = nid
|
||||
self.args = broker.args
|
||||
|
|
@ -58,17 +63,19 @@ class HttpSrv(object):
|
|||
|
||||
self.tp_nthr = 0 # actual
|
||||
self.tp_ncli = 0 # fading
|
||||
self.tp_time = None # latest worker collect
|
||||
self.tp_q = None if self.args.no_htp else queue.LifoQueue()
|
||||
self.t_periodic = None
|
||||
self.tp_time = 0.0 # latest worker collect
|
||||
self.tp_q: Optional[queue.LifoQueue[Any]] = (
|
||||
None if self.args.no_htp else queue.LifoQueue()
|
||||
)
|
||||
self.t_periodic: Optional[threading.Thread] = None
|
||||
|
||||
self.u2fh = FHC()
|
||||
self.srvs = []
|
||||
self.srvs: list[socket.socket] = []
|
||||
self.ncli = 0 # exact
|
||||
self.clients = {} # laggy
|
||||
self.clients: set[HttpConn] = set() # laggy
|
||||
self.nclimax = 0
|
||||
self.cb_ts = 0
|
||||
self.cb_v = 0
|
||||
self.cb_ts = 0.0
|
||||
self.cb_v = ""
|
||||
|
||||
env = jinja2.Environment()
|
||||
env.loader = jinja2.FileSystemLoader(os.path.join(E.mod, "web"))
|
||||
|
|
@ -82,7 +89,7 @@ class HttpSrv(object):
|
|||
if bos.path.exists(cert_path):
|
||||
self.cert_path = cert_path
|
||||
else:
|
||||
self.cert_path = None
|
||||
self.cert_path = ""
|
||||
|
||||
if self.tp_q:
|
||||
self.start_threads(4)
|
||||
|
|
@ -94,19 +101,19 @@ class HttpSrv(object):
|
|||
if self.args.log_thrs:
|
||||
start_log_thrs(self.log, self.args.log_thrs, nid)
|
||||
|
||||
self.th_cfg = {} # type: dict[str, Any]
|
||||
self.th_cfg: dict[str, Any] = {}
|
||||
t = threading.Thread(target=self.post_init)
|
||||
t.daemon = True
|
||||
t.start()
|
||||
|
||||
def post_init(self):
|
||||
def post_init(self) -> None:
|
||||
try:
|
||||
x = self.broker.put(True, "thumbsrv.getcfg")
|
||||
x = self.broker.ask("thumbsrv.getcfg")
|
||||
self.th_cfg = x.get()
|
||||
except:
|
||||
pass
|
||||
|
||||
def start_threads(self, n):
|
||||
def start_threads(self, n: int) -> None:
|
||||
self.tp_nthr += n
|
||||
if self.args.log_htp:
|
||||
self.log(self.name, "workers += {} = {}".format(n, self.tp_nthr), 6)
|
||||
|
|
@ -119,15 +126,16 @@ class HttpSrv(object):
|
|||
thr.daemon = True
|
||||
thr.start()
|
||||
|
||||
def stop_threads(self, n):
|
||||
def stop_threads(self, n: int) -> None:
|
||||
self.tp_nthr -= n
|
||||
if self.args.log_htp:
|
||||
self.log(self.name, "workers -= {} = {}".format(n, self.tp_nthr), 6)
|
||||
|
||||
assert self.tp_q
|
||||
for _ in range(n):
|
||||
self.tp_q.put(None)
|
||||
|
||||
def periodic(self):
|
||||
def periodic(self) -> None:
|
||||
while True:
|
||||
time.sleep(2 if self.tp_ncli or self.ncli else 10)
|
||||
with self.mutex:
|
||||
|
|
@ -141,7 +149,7 @@ class HttpSrv(object):
|
|||
self.t_periodic = None
|
||||
return
|
||||
|
||||
def listen(self, sck, nlisteners):
|
||||
def listen(self, sck: socket.socket, nlisteners: int) -> None:
|
||||
ip, port = sck.getsockname()
|
||||
self.srvs.append(sck)
|
||||
self.nclimax = math.ceil(self.args.nc * 1.0 / nlisteners)
|
||||
|
|
@ -153,15 +161,15 @@ class HttpSrv(object):
|
|||
t.daemon = True
|
||||
t.start()
|
||||
|
||||
def thr_listen(self, srv_sck):
|
||||
def thr_listen(self, srv_sck: socket.socket) -> None:
|
||||
"""listens on a shared tcp server"""
|
||||
ip, port = srv_sck.getsockname()
|
||||
fno = srv_sck.fileno()
|
||||
msg = "subscribed @ {}:{} f{}".format(ip, port, fno)
|
||||
self.log(self.name, msg)
|
||||
|
||||
def fun():
|
||||
self.broker.put(False, "cb_httpsrv_up")
|
||||
def fun() -> None:
|
||||
self.broker.say("cb_httpsrv_up")
|
||||
|
||||
threading.Thread(target=fun).start()
|
||||
|
||||
|
|
@ -185,21 +193,21 @@ class HttpSrv(object):
|
|||
continue
|
||||
|
||||
if self.args.log_conn:
|
||||
m = "|{}C-acc2 \033[0;36m{} \033[3{}m{}".format(
|
||||
t = "|{}C-acc2 \033[0;36m{} \033[3{}m{}".format(
|
||||
"-" * 3, ip, port % 8, port
|
||||
)
|
||||
self.log("%s %s" % addr, m, c="1;30")
|
||||
self.log("%s %s" % addr, t, c="1;30")
|
||||
|
||||
self.accept(sck, addr)
|
||||
|
||||
def accept(self, sck, addr):
|
||||
def accept(self, sck: socket.socket, addr: tuple[str, int]) -> None:
|
||||
"""takes an incoming tcp connection and creates a thread to handle it"""
|
||||
now = time.time()
|
||||
|
||||
if now - (self.tp_time or now) > 300:
|
||||
m = "httpserver threadpool died: tpt {:.2f}, now {:.2f}, nthr {}, ncli {}"
|
||||
self.log(self.name, m.format(self.tp_time, now, self.tp_nthr, self.ncli), 1)
|
||||
self.tp_time = None
|
||||
t = "httpserver threadpool died: tpt {:.2f}, now {:.2f}, nthr {}, ncli {}"
|
||||
self.log(self.name, t.format(self.tp_time, now, self.tp_nthr, self.ncli), 1)
|
||||
self.tp_time = 0
|
||||
self.tp_q = None
|
||||
|
||||
with self.mutex:
|
||||
|
|
@ -209,10 +217,10 @@ class HttpSrv(object):
|
|||
if self.nid:
|
||||
name += "-{}".format(self.nid)
|
||||
|
||||
t = threading.Thread(target=self.periodic, name=name)
|
||||
self.t_periodic = t
|
||||
t.daemon = True
|
||||
t.start()
|
||||
thr = threading.Thread(target=self.periodic, name=name)
|
||||
self.t_periodic = thr
|
||||
thr.daemon = True
|
||||
thr.start()
|
||||
|
||||
if self.tp_q:
|
||||
self.tp_time = self.tp_time or now
|
||||
|
|
@ -224,8 +232,8 @@ class HttpSrv(object):
|
|||
return
|
||||
|
||||
if not self.args.no_htp:
|
||||
m = "looks like the httpserver threadpool died; please make an issue on github and tell me the story of how you pulled that off, thanks and dog bless\n"
|
||||
self.log(self.name, m, 1)
|
||||
t = "looks like the httpserver threadpool died; please make an issue on github and tell me the story of how you pulled that off, thanks and dog bless\n"
|
||||
self.log(self.name, t, 1)
|
||||
|
||||
thr = threading.Thread(
|
||||
target=self.thr_client,
|
||||
|
|
@ -235,14 +243,15 @@ class HttpSrv(object):
|
|||
thr.daemon = True
|
||||
thr.start()
|
||||
|
||||
def thr_poolw(self):
|
||||
def thr_poolw(self) -> None:
|
||||
assert self.tp_q
|
||||
while True:
|
||||
task = self.tp_q.get()
|
||||
if not task:
|
||||
break
|
||||
|
||||
with self.mutex:
|
||||
self.tp_time = None
|
||||
self.tp_time = 0
|
||||
|
||||
try:
|
||||
sck, addr = task
|
||||
|
|
@ -255,7 +264,7 @@ class HttpSrv(object):
|
|||
except:
|
||||
self.log(self.name, "thr_client: " + min_ex(), 3)
|
||||
|
||||
def shutdown(self):
|
||||
def shutdown(self) -> None:
|
||||
self.stopping = True
|
||||
for srv in self.srvs:
|
||||
try:
|
||||
|
|
@ -263,7 +272,7 @@ class HttpSrv(object):
|
|||
except:
|
||||
pass
|
||||
|
||||
clients = list(self.clients.keys())
|
||||
clients = list(self.clients)
|
||||
for cli in clients:
|
||||
try:
|
||||
cli.shutdown()
|
||||
|
|
@ -279,13 +288,13 @@ class HttpSrv(object):
|
|||
|
||||
self.log(self.name, "ok bye")
|
||||
|
||||
def thr_client(self, sck, addr):
|
||||
def thr_client(self, sck: socket.socket, addr: tuple[str, int]) -> None:
|
||||
"""thread managing one tcp client"""
|
||||
sck.settimeout(120)
|
||||
|
||||
cli = HttpConn(sck, addr, self)
|
||||
with self.mutex:
|
||||
self.clients[cli] = 0
|
||||
self.clients.add(cli)
|
||||
|
||||
fno = sck.fileno()
|
||||
try:
|
||||
|
|
@ -328,10 +337,10 @@ class HttpSrv(object):
|
|||
raise
|
||||
finally:
|
||||
with self.mutex:
|
||||
del self.clients[cli]
|
||||
self.clients.remove(cli)
|
||||
self.ncli -= 1
|
||||
|
||||
def cachebuster(self):
|
||||
def cachebuster(self) -> str:
|
||||
if time.time() - self.cb_ts < 1:
|
||||
return self.cb_v
|
||||
|
||||
|
|
|
|||
|
|
@ -1,28 +1,28 @@
|
|||
# coding: utf-8
|
||||
from __future__ import print_function, unicode_literals
|
||||
|
||||
import hashlib
|
||||
import argparse # typechk
|
||||
import colorsys
|
||||
import hashlib
|
||||
|
||||
from .__init__ import PY2
|
||||
|
||||
|
||||
class Ico(object):
|
||||
def __init__(self, args):
|
||||
def __init__(self, args: argparse.Namespace) -> None:
|
||||
self.args = args
|
||||
|
||||
def get(self, ext, as_thumb):
|
||||
def get(self, ext: str, as_thumb: bool) -> tuple[str, bytes]:
|
||||
"""placeholder to make thumbnails not break"""
|
||||
|
||||
h = hashlib.md5(ext.encode("utf-8")).digest()[:2]
|
||||
zb = hashlib.md5(ext.encode("utf-8")).digest()[:2]
|
||||
if PY2:
|
||||
h = [ord(x) for x in h]
|
||||
zb = [ord(x) for x in zb]
|
||||
|
||||
c1 = colorsys.hsv_to_rgb(h[0] / 256.0, 1, 0.3)
|
||||
c2 = colorsys.hsv_to_rgb(h[0] / 256.0, 1, 1)
|
||||
c = list(c1) + list(c2)
|
||||
c = [int(x * 255) for x in c]
|
||||
c = "".join(["{:02x}".format(x) for x in c])
|
||||
c1 = colorsys.hsv_to_rgb(zb[0] / 256.0, 1, 0.3)
|
||||
c2 = colorsys.hsv_to_rgb(zb[0] / 256.0, 1, 1)
|
||||
ci = [int(x * 255) for x in list(c1) + list(c2)]
|
||||
c = "".join(["{:02x}".format(x) for x in ci])
|
||||
|
||||
h = 30
|
||||
if not self.args.th_no_crop and as_thumb:
|
||||
|
|
@ -37,6 +37,6 @@ class Ico(object):
|
|||
fill="#{}" font-family="monospace" font-size="14px" style="letter-spacing:.5px">{}</text>
|
||||
</g></svg>
|
||||
"""
|
||||
svg = svg.format(h, c[:6], c[6:], ext).encode("utf-8")
|
||||
svg = svg.format(h, c[:6], c[6:], ext)
|
||||
|
||||
return ["image/svg+xml", svg]
|
||||
return "image/svg+xml", svg.encode("utf-8")
|
||||
|
|
|
|||
|
|
@ -1,18 +1,26 @@
|
|||
# coding: utf-8
|
||||
from __future__ import print_function, unicode_literals
|
||||
|
||||
import os
|
||||
import sys
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess as sp
|
||||
import sys
|
||||
|
||||
from .__init__ import PY2, WINDOWS, unicode
|
||||
from .util import fsenc, uncyg, runcmd, retchk, REKOBO_LKEY
|
||||
from .bos import bos
|
||||
from .util import REKOBO_LKEY, fsenc, retchk, runcmd, uncyg
|
||||
|
||||
try:
|
||||
from typing import Any, Union
|
||||
|
||||
from .util import RootLogger
|
||||
except:
|
||||
pass
|
||||
|
||||
|
||||
def have_ff(cmd):
|
||||
def have_ff(cmd: str) -> bool:
|
||||
if PY2:
|
||||
print("# checking {}".format(cmd))
|
||||
cmd = (cmd + " -version").encode("ascii").split(b" ")
|
||||
|
|
@ -30,7 +38,7 @@ HAVE_FFPROBE = have_ff("ffprobe")
|
|||
|
||||
|
||||
class MParser(object):
|
||||
def __init__(self, cmdline):
|
||||
def __init__(self, cmdline: str) -> None:
|
||||
self.tag, args = cmdline.split("=", 1)
|
||||
self.tags = self.tag.split(",")
|
||||
|
||||
|
|
@ -73,7 +81,9 @@ class MParser(object):
|
|||
raise Exception()
|
||||
|
||||
|
||||
def ffprobe(abspath, timeout=10):
|
||||
def ffprobe(
|
||||
abspath: str, timeout: int = 10
|
||||
) -> tuple[dict[str, tuple[int, Any]], dict[str, list[Any]]]:
|
||||
cmd = [
|
||||
b"ffprobe",
|
||||
b"-hide_banner",
|
||||
|
|
@ -87,15 +97,15 @@ def ffprobe(abspath, timeout=10):
|
|||
return parse_ffprobe(so)
|
||||
|
||||
|
||||
def parse_ffprobe(txt):
|
||||
def parse_ffprobe(txt: str) -> tuple[dict[str, tuple[int, Any]], dict[str, list[Any]]]:
|
||||
"""ffprobe -show_format -show_streams"""
|
||||
streams = []
|
||||
fmt = {}
|
||||
g = {}
|
||||
for ln in [x.rstrip("\r") for x in txt.split("\n")]:
|
||||
try:
|
||||
k, v = ln.split("=", 1)
|
||||
g[k] = v
|
||||
sk, sv = ln.split("=", 1)
|
||||
g[sk] = sv
|
||||
continue
|
||||
except:
|
||||
pass
|
||||
|
|
@ -109,8 +119,8 @@ def parse_ffprobe(txt):
|
|||
fmt = g
|
||||
|
||||
streams = [fmt] + streams
|
||||
ret = {} # processed
|
||||
md = {} # raw tags
|
||||
ret: dict[str, Any] = {} # processed
|
||||
md: dict[str, list[Any]] = {} # raw tags
|
||||
|
||||
is_audio = fmt.get("format_name") in ["mp3", "ogg", "flac", "wav"]
|
||||
if fmt.get("filename", "").split(".")[-1].lower() in ["m4a", "aac"]:
|
||||
|
|
@ -161,43 +171,43 @@ def parse_ffprobe(txt):
|
|||
kvm = [["duration", ".dur"], ["bit_rate", ".q"]]
|
||||
|
||||
for sk, rk in kvm:
|
||||
v = strm.get(sk)
|
||||
if v is None:
|
||||
v1 = strm.get(sk)
|
||||
if v1 is None:
|
||||
continue
|
||||
|
||||
if rk.startswith("."):
|
||||
try:
|
||||
v = float(v)
|
||||
zf = float(v1)
|
||||
v2 = ret.get(rk)
|
||||
if v2 is None or v > v2:
|
||||
ret[rk] = v
|
||||
if v2 is None or zf > v2:
|
||||
ret[rk] = zf
|
||||
except:
|
||||
# sqlite doesnt care but the code below does
|
||||
if v not in ["N/A"]:
|
||||
ret[rk] = v
|
||||
if v1 not in ["N/A"]:
|
||||
ret[rk] = v1
|
||||
else:
|
||||
ret[rk] = v
|
||||
ret[rk] = v1
|
||||
|
||||
if ret.get("vc") == "ansi": # shellscript
|
||||
return {}, {}
|
||||
|
||||
for strm in streams:
|
||||
for k, v in strm.items():
|
||||
if not k.startswith("TAG:"):
|
||||
for sk, sv in strm.items():
|
||||
if not sk.startswith("TAG:"):
|
||||
continue
|
||||
|
||||
k = k[4:].strip()
|
||||
v = v.strip()
|
||||
if k and v and k not in md:
|
||||
md[k] = [v]
|
||||
sk = sk[4:].strip()
|
||||
sv = sv.strip()
|
||||
if sk and sv and sk not in md:
|
||||
md[sk] = [sv]
|
||||
|
||||
for k in [".q", ".vq", ".aq"]:
|
||||
if k in ret:
|
||||
ret[k] /= 1000 # bit_rate=320000
|
||||
for sk in [".q", ".vq", ".aq"]:
|
||||
if sk in ret:
|
||||
ret[sk] /= 1000 # bit_rate=320000
|
||||
|
||||
for k in [".q", ".vq", ".aq", ".resw", ".resh"]:
|
||||
if k in ret:
|
||||
ret[k] = int(ret[k])
|
||||
for sk in [".q", ".vq", ".aq", ".resw", ".resh"]:
|
||||
if sk in ret:
|
||||
ret[sk] = int(ret[sk])
|
||||
|
||||
if ".fps" in ret:
|
||||
fps = ret[".fps"]
|
||||
|
|
@ -219,13 +229,13 @@ def parse_ffprobe(txt):
|
|||
if ".resw" in ret and ".resh" in ret:
|
||||
ret["res"] = "{}x{}".format(ret[".resw"], ret[".resh"])
|
||||
|
||||
ret = {k: [0, v] for k, v in ret.items()}
|
||||
zd = {k: (0, v) for k, v in ret.items()}
|
||||
|
||||
return ret, md
|
||||
return zd, md
|
||||
|
||||
|
||||
class MTag(object):
|
||||
def __init__(self, log_func, args):
|
||||
def __init__(self, log_func: RootLogger, args: argparse.Namespace) -> None:
|
||||
self.log_func = log_func
|
||||
self.args = args
|
||||
self.usable = True
|
||||
|
|
@ -242,7 +252,7 @@ class MTag(object):
|
|||
if self.backend == "mutagen":
|
||||
self.get = self.get_mutagen
|
||||
try:
|
||||
import mutagen
|
||||
import mutagen # noqa: F401 # pylint: disable=unused-import,import-outside-toplevel
|
||||
except:
|
||||
self.log("could not load Mutagen, trying FFprobe instead", c=3)
|
||||
self.backend = "ffprobe"
|
||||
|
|
@ -339,31 +349,33 @@ class MTag(object):
|
|||
}
|
||||
# self.get = self.compare
|
||||
|
||||
def log(self, msg, c=0):
|
||||
def log(self, msg: str, c: Union[int, str] = 0) -> None:
|
||||
self.log_func("mtag", msg, c)
|
||||
|
||||
def normalize_tags(self, ret, md):
|
||||
for k, v in dict(md).items():
|
||||
if not v:
|
||||
def normalize_tags(
|
||||
self, parser_output: dict[str, tuple[int, Any]], md: dict[str, list[Any]]
|
||||
) -> dict[str, Union[str, float]]:
|
||||
for sk, tv in dict(md).items():
|
||||
if not tv:
|
||||
continue
|
||||
|
||||
k = k.lower().split("::")[0].strip()
|
||||
mk = self.rmap.get(k)
|
||||
if not mk:
|
||||
sk = sk.lower().split("::")[0].strip()
|
||||
key_mapping = self.rmap.get(sk)
|
||||
if not key_mapping:
|
||||
continue
|
||||
|
||||
pref, mk = mk
|
||||
if mk not in ret or ret[mk][0] > pref:
|
||||
ret[mk] = [pref, v[0]]
|
||||
priority, alias = key_mapping
|
||||
if alias not in parser_output or parser_output[alias][0] > priority:
|
||||
parser_output[alias] = (priority, tv[0])
|
||||
|
||||
# take first value
|
||||
ret = {k: unicode(v[1]).strip() for k, v in ret.items()}
|
||||
# take first value (lowest priority / most preferred)
|
||||
ret = {sk: unicode(tv[1]).strip() for sk, tv in parser_output.items()}
|
||||
|
||||
# track 3/7 => track 3
|
||||
for k, v in ret.items():
|
||||
if k[0] == ".":
|
||||
v = v.split("/")[0].strip().lstrip("0")
|
||||
ret[k] = v or 0
|
||||
for sk, tv in ret.items():
|
||||
if sk[0] == ".":
|
||||
sv = str(tv).split("/")[0].strip().lstrip("0")
|
||||
ret[sk] = sv or 0
|
||||
|
||||
# normalize key notation to rkeobo
|
||||
okey = ret.get("key")
|
||||
|
|
@ -373,7 +385,7 @@ class MTag(object):
|
|||
|
||||
return ret
|
||||
|
||||
def compare(self, abspath):
|
||||
def compare(self, abspath: str) -> dict[str, Union[str, float]]:
|
||||
if abspath.endswith(".au"):
|
||||
return {}
|
||||
|
||||
|
|
@ -411,7 +423,7 @@ class MTag(object):
|
|||
|
||||
return r1
|
||||
|
||||
def get_mutagen(self, abspath):
|
||||
def get_mutagen(self, abspath: str) -> dict[str, Union[str, float]]:
|
||||
if not bos.path.isfile(abspath):
|
||||
return {}
|
||||
|
||||
|
|
@ -425,7 +437,7 @@ class MTag(object):
|
|||
return self.get_ffprobe(abspath) if self.can_ffprobe else {}
|
||||
|
||||
sz = bos.path.getsize(abspath)
|
||||
ret = {".q": [0, int((sz / md.info.length) / 128)]}
|
||||
ret = {".q": (0, int((sz / md.info.length) / 128))}
|
||||
|
||||
for attr, k, norm in [
|
||||
["codec", "ac", unicode],
|
||||
|
|
@ -456,24 +468,24 @@ class MTag(object):
|
|||
if k == "ac" and v.startswith("mp4a.40."):
|
||||
v = "aac"
|
||||
|
||||
ret[k] = [0, norm(v)]
|
||||
ret[k] = (0, norm(v))
|
||||
|
||||
return self.normalize_tags(ret, md)
|
||||
|
||||
def get_ffprobe(self, abspath):
|
||||
def get_ffprobe(self, abspath: str) -> dict[str, Union[str, float]]:
|
||||
if not bos.path.isfile(abspath):
|
||||
return {}
|
||||
|
||||
ret, md = ffprobe(abspath)
|
||||
return self.normalize_tags(ret, md)
|
||||
|
||||
def get_bin(self, parsers, abspath):
|
||||
def get_bin(self, parsers: dict[str, MParser], abspath: str) -> dict[str, Any]:
|
||||
if not bos.path.isfile(abspath):
|
||||
return {}
|
||||
|
||||
pypath = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
|
||||
pypath = [str(pypath)] + [str(x) for x in sys.path if x]
|
||||
pypath = str(os.pathsep.join(pypath))
|
||||
zsl = [str(pypath)] + [str(x) for x in sys.path if x]
|
||||
pypath = str(os.pathsep.join(zsl))
|
||||
env = os.environ.copy()
|
||||
env["PYTHONPATH"] = pypath
|
||||
|
||||
|
|
@ -491,9 +503,9 @@ class MTag(object):
|
|||
else:
|
||||
cmd = ["nice"] + cmd
|
||||
|
||||
cmd = [fsenc(x) for x in cmd]
|
||||
rc, v, err = runcmd(cmd, **args)
|
||||
retchk(rc, cmd, err, self.log, 5, self.args.mtag_v)
|
||||
bcmd = [fsenc(x) for x in cmd]
|
||||
rc, v, err = runcmd(bcmd, **args) # type: ignore
|
||||
retchk(rc, bcmd, err, self.log, 5, self.args.mtag_v)
|
||||
v = v.strip()
|
||||
if not v:
|
||||
continue
|
||||
|
|
@ -501,10 +513,10 @@ class MTag(object):
|
|||
if "," not in tagname:
|
||||
ret[tagname] = v
|
||||
else:
|
||||
v = json.loads(v)
|
||||
zj = json.loads(v)
|
||||
for tag in tagname.split(","):
|
||||
if tag and tag in v:
|
||||
ret[tag] = v[tag]
|
||||
if tag and tag in zj:
|
||||
ret[tag] = zj[tag]
|
||||
except:
|
||||
pass
|
||||
|
||||
|
|
|
|||
|
|
@ -4,20 +4,29 @@ from __future__ import print_function, unicode_literals
|
|||
import tarfile
|
||||
import threading
|
||||
|
||||
from .sutil import errdesc
|
||||
from .util import Queue, fsenc, min_ex
|
||||
from queue import Queue
|
||||
|
||||
from .bos import bos
|
||||
from .sutil import StreamArc, errdesc
|
||||
from .util import fsenc, min_ex
|
||||
|
||||
try:
|
||||
from typing import Any, Generator, Optional
|
||||
|
||||
from .util import NamedLogger
|
||||
except:
|
||||
pass
|
||||
|
||||
|
||||
class QFile(object):
|
||||
class QFile(object): # inherit io.StringIO for painful typing
|
||||
"""file-like object which buffers writes into a queue"""
|
||||
|
||||
def __init__(self):
|
||||
self.q = Queue(64)
|
||||
self.bq = []
|
||||
def __init__(self) -> None:
|
||||
self.q: Queue[Optional[bytes]] = Queue(64)
|
||||
self.bq: list[bytes] = []
|
||||
self.nq = 0
|
||||
|
||||
def write(self, buf):
|
||||
def write(self, buf: Optional[bytes]) -> None:
|
||||
if buf is None or self.nq >= 240 * 1024:
|
||||
self.q.put(b"".join(self.bq))
|
||||
self.bq = []
|
||||
|
|
@ -30,27 +39,32 @@ class QFile(object):
|
|||
self.nq += len(buf)
|
||||
|
||||
|
||||
class StreamTar(object):
|
||||
class StreamTar(StreamArc):
|
||||
"""construct in-memory tar file from the given path"""
|
||||
|
||||
def __init__(self, log, fgen, **kwargs):
|
||||
def __init__(
|
||||
self,
|
||||
log: NamedLogger,
|
||||
fgen: Generator[dict[str, Any], None, None],
|
||||
**kwargs: Any
|
||||
):
|
||||
super(StreamTar, self).__init__(log, fgen)
|
||||
|
||||
self.ci = 0
|
||||
self.co = 0
|
||||
self.qfile = QFile()
|
||||
self.log = log
|
||||
self.fgen = fgen
|
||||
self.errf = None
|
||||
self.errf: dict[str, Any] = {}
|
||||
|
||||
# python 3.8 changed to PAX_FORMAT as default,
|
||||
# waste of space and don't care about the new features
|
||||
fmt = tarfile.GNU_FORMAT
|
||||
self.tar = tarfile.open(fileobj=self.qfile, mode="w|", format=fmt)
|
||||
self.tar = tarfile.open(fileobj=self.qfile, mode="w|", format=fmt) # type: ignore
|
||||
|
||||
w = threading.Thread(target=self._gen, name="star-gen")
|
||||
w.daemon = True
|
||||
w.start()
|
||||
|
||||
def gen(self):
|
||||
def gen(self) -> Generator[Optional[bytes], None, None]:
|
||||
while True:
|
||||
buf = self.qfile.q.get()
|
||||
if not buf:
|
||||
|
|
@ -63,7 +77,7 @@ class StreamTar(object):
|
|||
if self.errf:
|
||||
bos.unlink(self.errf["ap"])
|
||||
|
||||
def ser(self, f):
|
||||
def ser(self, f: dict[str, Any]) -> None:
|
||||
name = f["vp"]
|
||||
src = f["ap"]
|
||||
fsi = f["st"]
|
||||
|
|
@ -76,21 +90,21 @@ class StreamTar(object):
|
|||
inf.gid = 0
|
||||
|
||||
self.ci += inf.size
|
||||
with open(fsenc(src), "rb", 512 * 1024) as f:
|
||||
self.tar.addfile(inf, f)
|
||||
with open(fsenc(src), "rb", 512 * 1024) as fo:
|
||||
self.tar.addfile(inf, fo)
|
||||
|
||||
def _gen(self):
|
||||
def _gen(self) -> None:
|
||||
errors = []
|
||||
for f in self.fgen:
|
||||
if "err" in f:
|
||||
errors.append([f["vp"], f["err"]])
|
||||
errors.append((f["vp"], f["err"]))
|
||||
continue
|
||||
|
||||
try:
|
||||
self.ser(f)
|
||||
except:
|
||||
ex = min_ex(5, True).replace("\n", "\n-- ")
|
||||
errors.append([f["vp"], ex])
|
||||
errors.append((f["vp"], ex))
|
||||
|
||||
if errors:
|
||||
self.errf, txt = errdesc(errors)
|
||||
|
|
|
|||
|
|
@ -12,23 +12,28 @@ Original source: misc/python/surrogateescape.py in https://bitbucket.org/haypo/m
|
|||
|
||||
# This code is released under the Python license and the BSD 2-clause license
|
||||
|
||||
import platform
|
||||
import codecs
|
||||
import platform
|
||||
import sys
|
||||
|
||||
PY3 = sys.version_info[0] > 2
|
||||
WINDOWS = platform.system() == "Windows"
|
||||
FS_ERRORS = "surrogateescape"
|
||||
|
||||
try:
|
||||
from typing import Any
|
||||
except:
|
||||
pass
|
||||
|
||||
def u(text):
|
||||
|
||||
def u(text: Any) -> str:
|
||||
if PY3:
|
||||
return text
|
||||
else:
|
||||
return text.decode("unicode_escape")
|
||||
|
||||
|
||||
def b(data):
|
||||
def b(data: Any) -> bytes:
|
||||
if PY3:
|
||||
return data.encode("latin1")
|
||||
else:
|
||||
|
|
@ -43,7 +48,7 @@ else:
|
|||
bytes_chr = chr
|
||||
|
||||
|
||||
def surrogateescape_handler(exc):
|
||||
def surrogateescape_handler(exc: Any) -> tuple[str, int]:
|
||||
"""
|
||||
Pure Python implementation of the PEP 383: the "surrogateescape" error
|
||||
handler of Python 3. Undecodable bytes will be replaced by a Unicode
|
||||
|
|
@ -74,7 +79,7 @@ class NotASurrogateError(Exception):
|
|||
pass
|
||||
|
||||
|
||||
def replace_surrogate_encode(mystring):
|
||||
def replace_surrogate_encode(mystring: str) -> str:
|
||||
"""
|
||||
Returns a (unicode) string, not the more logical bytes, because the codecs
|
||||
register_error functionality expects this.
|
||||
|
|
@ -100,7 +105,7 @@ def replace_surrogate_encode(mystring):
|
|||
return str().join(decoded)
|
||||
|
||||
|
||||
def replace_surrogate_decode(mybytes):
|
||||
def replace_surrogate_decode(mybytes: bytes) -> str:
|
||||
"""
|
||||
Returns a (unicode) string
|
||||
"""
|
||||
|
|
@ -121,7 +126,7 @@ def replace_surrogate_decode(mybytes):
|
|||
return str().join(decoded)
|
||||
|
||||
|
||||
def encodefilename(fn):
|
||||
def encodefilename(fn: str) -> bytes:
|
||||
if FS_ENCODING == "ascii":
|
||||
# ASCII encoder of Python 2 expects that the error handler returns a
|
||||
# Unicode string encodable to ASCII, whereas our surrogateescape error
|
||||
|
|
@ -161,7 +166,7 @@ def encodefilename(fn):
|
|||
return fn.encode(FS_ENCODING, FS_ERRORS)
|
||||
|
||||
|
||||
def decodefilename(fn):
|
||||
def decodefilename(fn: bytes) -> str:
|
||||
return fn.decode(FS_ENCODING, FS_ERRORS)
|
||||
|
||||
|
||||
|
|
@ -181,7 +186,7 @@ if WINDOWS and not PY3:
|
|||
FS_ENCODING = codecs.lookup(FS_ENCODING).name
|
||||
|
||||
|
||||
def register_surrogateescape():
|
||||
def register_surrogateescape() -> None:
|
||||
"""
|
||||
Registers the surrogateescape error handler on Python 2 (only)
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -6,8 +6,29 @@ from datetime import datetime
|
|||
|
||||
from .bos import bos
|
||||
|
||||
try:
|
||||
from typing import Any, Generator, Optional
|
||||
|
||||
def errdesc(errors):
|
||||
from .util import NamedLogger
|
||||
except:
|
||||
pass
|
||||
|
||||
|
||||
class StreamArc(object):
|
||||
def __init__(
|
||||
self,
|
||||
log: NamedLogger,
|
||||
fgen: Generator[dict[str, Any], None, None],
|
||||
**kwargs: Any
|
||||
):
|
||||
self.log = log
|
||||
self.fgen = fgen
|
||||
|
||||
def gen(self) -> Generator[Optional[bytes], None, None]:
|
||||
pass
|
||||
|
||||
|
||||
def errdesc(errors: list[tuple[str, str]]) -> tuple[dict[str, Any], list[str]]:
|
||||
report = ["copyparty failed to add the following files to the archive:", ""]
|
||||
|
||||
for fn, err in errors:
|
||||
|
|
|
|||
|
|
@ -1,41 +1,51 @@
|
|||
# coding: utf-8
|
||||
from __future__ import print_function, unicode_literals
|
||||
|
||||
import argparse
|
||||
import calendar
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import shlex
|
||||
import string
|
||||
import signal
|
||||
import socket
|
||||
import string
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from datetime import datetime, timedelta
|
||||
import calendar
|
||||
|
||||
from .__init__ import E, PY2, WINDOWS, ANYWIN, MACOS, VT100, unicode
|
||||
from .util import mp, start_log_thrs, start_stackmon, min_ex, ansi_re
|
||||
try:
|
||||
from types import FrameType
|
||||
|
||||
import typing
|
||||
from typing import Optional, Union
|
||||
except:
|
||||
pass
|
||||
|
||||
from .__init__ import ANYWIN, MACOS, PY2, VT100, WINDOWS, E, unicode
|
||||
from .authsrv import AuthSrv
|
||||
from .tcpsrv import TcpSrv
|
||||
from .up2k import Up2k
|
||||
from .th_srv import ThumbSrv, HAVE_PIL, HAVE_VIPS, HAVE_WEBP
|
||||
from .mtag import HAVE_FFMPEG, HAVE_FFPROBE
|
||||
from .tcpsrv import TcpSrv
|
||||
from .th_srv import HAVE_PIL, HAVE_VIPS, HAVE_WEBP, ThumbSrv
|
||||
from .up2k import Up2k
|
||||
from .util import ansi_re, min_ex, mp, start_log_thrs, start_stackmon
|
||||
|
||||
|
||||
class SvcHub(object):
|
||||
"""
|
||||
Hosts all services which cannot be parallelized due to reliance on monolithic resources.
|
||||
Creates a Broker which does most of the heavy stuff; hosted services can use this to perform work:
|
||||
hub.broker.put(want_reply, destination, args_list).
|
||||
hub.broker.<say|ask>(destination, args_list).
|
||||
|
||||
Either BrokerThr (plain threads) or BrokerMP (multiprocessing) is used depending on configuration.
|
||||
Nothing is returned synchronously; if you want any value returned from the call,
|
||||
put() can return a queue (if want_reply=True) which has a blocking get() with the response.
|
||||
"""
|
||||
|
||||
def __init__(self, args, argv, printed):
|
||||
def __init__(self, args: argparse.Namespace, argv: list[str], printed: str) -> None:
|
||||
self.args = args
|
||||
self.argv = argv
|
||||
self.logf = None
|
||||
self.logf: Optional[typing.TextIO] = None
|
||||
self.logf_base_fn = ""
|
||||
self.stop_req = False
|
||||
self.reload_req = False
|
||||
self.stopping = False
|
||||
|
|
@ -59,16 +69,16 @@ class SvcHub(object):
|
|||
|
||||
if not args.use_fpool and args.j != 1:
|
||||
args.no_fpool = True
|
||||
m = "multithreading enabled with -j {}, so disabling fpool -- this can reduce upload performance on some filesystems"
|
||||
self.log("root", m.format(args.j))
|
||||
t = "multithreading enabled with -j {}, so disabling fpool -- this can reduce upload performance on some filesystems"
|
||||
self.log("root", t.format(args.j))
|
||||
|
||||
if not args.no_fpool and args.j != 1:
|
||||
m = "WARNING: --use-fpool combined with multithreading is untested and can probably cause undefined behavior"
|
||||
t = "WARNING: --use-fpool combined with multithreading is untested and can probably cause undefined behavior"
|
||||
if ANYWIN:
|
||||
m = 'windows cannot do multithreading without --no-fpool, so enabling that -- note that upload performance will suffer if you have microsoft defender "real-time protection" enabled, so you probably want to use -j 1 instead'
|
||||
t = 'windows cannot do multithreading without --no-fpool, so enabling that -- note that upload performance will suffer if you have microsoft defender "real-time protection" enabled, so you probably want to use -j 1 instead'
|
||||
args.no_fpool = True
|
||||
|
||||
self.log("root", m, c=3)
|
||||
self.log("root", t, c=3)
|
||||
|
||||
bri = "zy"[args.theme % 2 :][:1]
|
||||
ch = "abcdefghijklmnopqrstuvwx"[int(args.theme / 2)]
|
||||
|
|
@ -96,8 +106,8 @@ class SvcHub(object):
|
|||
self.args.th_dec = list(decs.keys())
|
||||
self.thumbsrv = None
|
||||
if not args.no_thumb:
|
||||
m = "decoder preference: {}".format(", ".join(self.args.th_dec))
|
||||
self.log("thumb", m)
|
||||
t = "decoder preference: {}".format(", ".join(self.args.th_dec))
|
||||
self.log("thumb", t)
|
||||
|
||||
if "pil" in self.args.th_dec and not HAVE_WEBP:
|
||||
msg = "disabling webp thumbnails because either libwebp is not available or your Pillow is too old"
|
||||
|
|
@ -131,11 +141,11 @@ class SvcHub(object):
|
|||
if self.check_mp_enable():
|
||||
from .broker_mp import BrokerMp as Broker
|
||||
else:
|
||||
from .broker_thr import BrokerThr as Broker
|
||||
from .broker_thr import BrokerThr as Broker # type: ignore
|
||||
|
||||
self.broker = Broker(self)
|
||||
|
||||
def thr_httpsrv_up(self):
|
||||
def thr_httpsrv_up(self) -> None:
|
||||
time.sleep(1 if self.args.ign_ebind_all else 5)
|
||||
expected = self.broker.num_workers * self.tcpsrv.nsrv
|
||||
failed = expected - self.httpsrv_up
|
||||
|
|
@ -145,20 +155,20 @@ class SvcHub(object):
|
|||
if self.args.ign_ebind_all:
|
||||
if not self.tcpsrv.srv:
|
||||
for _ in range(self.broker.num_workers):
|
||||
self.broker.put(False, "cb_httpsrv_up")
|
||||
self.broker.say("cb_httpsrv_up")
|
||||
return
|
||||
|
||||
if self.args.ign_ebind and self.tcpsrv.srv:
|
||||
return
|
||||
|
||||
m = "{}/{} workers failed to start"
|
||||
m = m.format(failed, expected)
|
||||
self.log("root", m, 1)
|
||||
t = "{}/{} workers failed to start"
|
||||
t = t.format(failed, expected)
|
||||
self.log("root", t, 1)
|
||||
|
||||
self.retcode = 1
|
||||
os.kill(os.getpid(), signal.SIGTERM)
|
||||
|
||||
def cb_httpsrv_up(self):
|
||||
def cb_httpsrv_up(self) -> None:
|
||||
self.httpsrv_up += 1
|
||||
if self.httpsrv_up != self.broker.num_workers:
|
||||
return
|
||||
|
|
@ -171,9 +181,9 @@ class SvcHub(object):
|
|||
thr.daemon = True
|
||||
thr.start()
|
||||
|
||||
def _logname(self):
|
||||
def _logname(self) -> str:
|
||||
dt = datetime.utcnow()
|
||||
fn = self.args.lo
|
||||
fn = str(self.args.lo)
|
||||
for fs in "YmdHMS":
|
||||
fs = "%" + fs
|
||||
if fs in fn:
|
||||
|
|
@ -181,7 +191,7 @@ class SvcHub(object):
|
|||
|
||||
return fn
|
||||
|
||||
def _setup_logfile(self, printed):
|
||||
def _setup_logfile(self, printed: str) -> None:
|
||||
base_fn = fn = sel_fn = self._logname()
|
||||
if fn != self.args.lo:
|
||||
ctr = 0
|
||||
|
|
@ -203,8 +213,6 @@ class SvcHub(object):
|
|||
|
||||
lh = codecs.open(fn, "w", encoding="utf-8", errors="replace")
|
||||
|
||||
lh.base_fn = base_fn
|
||||
|
||||
argv = [sys.executable] + self.argv
|
||||
if hasattr(shlex, "quote"):
|
||||
argv = [shlex.quote(x) for x in argv]
|
||||
|
|
@ -215,9 +223,10 @@ class SvcHub(object):
|
|||
printed += msg
|
||||
lh.write("t0: {:.3f}\nargv: {}\n\n{}".format(E.t0, " ".join(argv), printed))
|
||||
self.logf = lh
|
||||
self.logf_base_fn = base_fn
|
||||
print(msg, end="")
|
||||
|
||||
def run(self):
|
||||
def run(self) -> None:
|
||||
self.tcpsrv.run()
|
||||
|
||||
thr = threading.Thread(target=self.thr_httpsrv_up)
|
||||
|
|
@ -252,7 +261,7 @@ class SvcHub(object):
|
|||
else:
|
||||
self.stop_thr()
|
||||
|
||||
def reload(self):
|
||||
def reload(self) -> str:
|
||||
if self.reloading:
|
||||
return "cannot reload; already in progress"
|
||||
|
||||
|
|
@ -262,7 +271,7 @@ class SvcHub(object):
|
|||
t.start()
|
||||
return "reload initiated"
|
||||
|
||||
def _reload(self):
|
||||
def _reload(self) -> None:
|
||||
self.log("root", "reload scheduled")
|
||||
with self.up2k.mutex:
|
||||
self.asrv.reload()
|
||||
|
|
@ -271,7 +280,7 @@ class SvcHub(object):
|
|||
|
||||
self.reloading = False
|
||||
|
||||
def stop_thr(self):
|
||||
def stop_thr(self) -> None:
|
||||
while not self.stop_req:
|
||||
with self.stop_cond:
|
||||
self.stop_cond.wait(9001)
|
||||
|
|
@ -282,7 +291,7 @@ class SvcHub(object):
|
|||
|
||||
self.shutdown()
|
||||
|
||||
def signal_handler(self, sig, frame):
|
||||
def signal_handler(self, sig: int, frame: Optional[FrameType]) -> None:
|
||||
if self.stopping:
|
||||
return
|
||||
|
||||
|
|
@ -294,7 +303,7 @@ class SvcHub(object):
|
|||
with self.stop_cond:
|
||||
self.stop_cond.notify_all()
|
||||
|
||||
def shutdown(self):
|
||||
def shutdown(self) -> None:
|
||||
if self.stopping:
|
||||
return
|
||||
|
||||
|
|
@ -337,7 +346,7 @@ class SvcHub(object):
|
|||
|
||||
sys.exit(ret)
|
||||
|
||||
def _log_disabled(self, src, msg, c=0):
|
||||
def _log_disabled(self, src: str, msg: str, c: Union[int, str] = 0) -> None:
|
||||
if not self.logf:
|
||||
return
|
||||
|
||||
|
|
@ -349,8 +358,8 @@ class SvcHub(object):
|
|||
if now >= self.next_day:
|
||||
self._set_next_day()
|
||||
|
||||
def _set_next_day(self):
|
||||
if self.next_day and self.logf and self.logf.base_fn != self._logname():
|
||||
def _set_next_day(self) -> None:
|
||||
if self.next_day and self.logf and self.logf_base_fn != self._logname():
|
||||
self.logf.close()
|
||||
self._setup_logfile("")
|
||||
|
||||
|
|
@ -364,7 +373,7 @@ class SvcHub(object):
|
|||
dt = dt.replace(hour=0, minute=0, second=0)
|
||||
self.next_day = calendar.timegm(dt.utctimetuple())
|
||||
|
||||
def _log_enabled(self, src, msg, c=0):
|
||||
def _log_enabled(self, src: str, msg: str, c: Union[int, str] = 0) -> None:
|
||||
"""handles logging from all components"""
|
||||
with self.log_mutex:
|
||||
now = time.time()
|
||||
|
|
@ -401,7 +410,7 @@ class SvcHub(object):
|
|||
if self.logf:
|
||||
self.logf.write(msg)
|
||||
|
||||
def check_mp_support(self):
|
||||
def check_mp_support(self) -> str:
|
||||
vmin = sys.version_info[1]
|
||||
if WINDOWS:
|
||||
msg = "need python 3.3 or newer for multiprocessing;"
|
||||
|
|
@ -415,16 +424,16 @@ class SvcHub(object):
|
|||
return msg
|
||||
|
||||
try:
|
||||
x = mp.Queue(1)
|
||||
x.put(["foo", "bar"])
|
||||
x: mp.Queue[tuple[str, str]] = mp.Queue(1)
|
||||
x.put(("foo", "bar"))
|
||||
if x.get()[0] != "foo":
|
||||
raise Exception()
|
||||
except:
|
||||
return "multiprocessing is not supported on your platform;"
|
||||
|
||||
return None
|
||||
return ""
|
||||
|
||||
def check_mp_enable(self):
|
||||
def check_mp_enable(self) -> bool:
|
||||
if self.args.j == 1:
|
||||
return False
|
||||
|
||||
|
|
@ -447,18 +456,18 @@ class SvcHub(object):
|
|||
self.log("svchub", "cannot efficiently use multiple CPU cores")
|
||||
return False
|
||||
|
||||
def sd_notify(self):
|
||||
def sd_notify(self) -> None:
|
||||
try:
|
||||
addr = os.getenv("NOTIFY_SOCKET")
|
||||
if not addr:
|
||||
zb = os.getenv("NOTIFY_SOCKET")
|
||||
if not zb:
|
||||
return
|
||||
|
||||
addr = unicode(addr)
|
||||
addr = unicode(zb)
|
||||
if addr.startswith("@"):
|
||||
addr = "\0" + addr[1:]
|
||||
|
||||
m = "".join(x for x in addr if x in string.printable)
|
||||
self.log("sd_notify", m)
|
||||
t = "".join(x for x in addr if x in string.printable)
|
||||
self.log("sd_notify", t)
|
||||
|
||||
sck = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
|
||||
sck.connect(addr)
|
||||
|
|
|
|||
|
|
@ -1,16 +1,23 @@
|
|||
# coding: utf-8
|
||||
from __future__ import print_function, unicode_literals
|
||||
|
||||
import calendar
|
||||
import time
|
||||
import zlib
|
||||
import calendar
|
||||
|
||||
from .sutil import errdesc
|
||||
from .util import yieldfile, sanitize_fn, spack, sunpack, min_ex
|
||||
from .bos import bos
|
||||
from .sutil import StreamArc, errdesc
|
||||
from .util import min_ex, sanitize_fn, spack, sunpack, yieldfile
|
||||
|
||||
try:
|
||||
from typing import Any, Generator, Optional
|
||||
|
||||
from .util import NamedLogger
|
||||
except:
|
||||
pass
|
||||
|
||||
|
||||
def dostime2unix(buf):
|
||||
def dostime2unix(buf: bytes) -> int:
|
||||
t, d = sunpack(b"<HH", buf)
|
||||
|
||||
ts = (t & 0x1F) * 2
|
||||
|
|
@ -29,7 +36,7 @@ def dostime2unix(buf):
|
|||
return int(calendar.timegm(dt))
|
||||
|
||||
|
||||
def unixtime2dos(ts):
|
||||
def unixtime2dos(ts: int) -> bytes:
|
||||
tt = time.gmtime(ts + 1)
|
||||
dy, dm, dd, th, tm, ts = list(tt)[:6]
|
||||
|
||||
|
|
@ -41,14 +48,22 @@ def unixtime2dos(ts):
|
|||
return b"\x00\x00\x21\x00"
|
||||
|
||||
|
||||
def gen_fdesc(sz, crc32, z64):
|
||||
def gen_fdesc(sz: int, crc32: int, z64: bool) -> bytes:
|
||||
ret = b"\x50\x4b\x07\x08"
|
||||
fmt = b"<LQQ" if z64 else b"<LLL"
|
||||
ret += spack(fmt, crc32, sz, sz)
|
||||
return ret
|
||||
|
||||
|
||||
def gen_hdr(h_pos, fn, sz, lastmod, utf8, crc32, pre_crc):
|
||||
def gen_hdr(
|
||||
h_pos: Optional[int],
|
||||
fn: str,
|
||||
sz: int,
|
||||
lastmod: int,
|
||||
utf8: bool,
|
||||
icrc32: int,
|
||||
pre_crc: bool,
|
||||
) -> bytes:
|
||||
"""
|
||||
does regular file headers
|
||||
and the central directory meme if h_pos is set
|
||||
|
|
@ -67,8 +82,8 @@ def gen_hdr(h_pos, fn, sz, lastmod, utf8, crc32, pre_crc):
|
|||
# confusingly this doesn't bump if h_pos
|
||||
req_ver = b"\x2d\x00" if z64 else b"\x0a\x00"
|
||||
|
||||
if crc32:
|
||||
crc32 = spack(b"<L", crc32)
|
||||
if icrc32:
|
||||
crc32 = spack(b"<L", icrc32)
|
||||
else:
|
||||
crc32 = b"\x00" * 4
|
||||
|
||||
|
|
@ -129,7 +144,9 @@ def gen_hdr(h_pos, fn, sz, lastmod, utf8, crc32, pre_crc):
|
|||
return ret
|
||||
|
||||
|
||||
def gen_ecdr(items, cdir_pos, cdir_end):
|
||||
def gen_ecdr(
|
||||
items: list[tuple[str, int, int, int, int]], cdir_pos: int, cdir_end: int
|
||||
) -> tuple[bytes, bool]:
|
||||
"""
|
||||
summary of all file headers,
|
||||
usually the zipfile footer unless something clamps
|
||||
|
|
@ -154,10 +171,12 @@ def gen_ecdr(items, cdir_pos, cdir_end):
|
|||
# 2b comment length
|
||||
ret += b"\x00\x00"
|
||||
|
||||
return [ret, need_64]
|
||||
return ret, need_64
|
||||
|
||||
|
||||
def gen_ecdr64(items, cdir_pos, cdir_end):
|
||||
def gen_ecdr64(
|
||||
items: list[tuple[str, int, int, int, int]], cdir_pos: int, cdir_end: int
|
||||
) -> bytes:
|
||||
"""
|
||||
z64 end of central directory
|
||||
added when numfiles or a headerptr clamps
|
||||
|
|
@ -181,7 +200,7 @@ def gen_ecdr64(items, cdir_pos, cdir_end):
|
|||
return ret
|
||||
|
||||
|
||||
def gen_ecdr64_loc(ecdr64_pos):
|
||||
def gen_ecdr64_loc(ecdr64_pos: int) -> bytes:
|
||||
"""
|
||||
z64 end of central directory locator
|
||||
points to ecdr64
|
||||
|
|
@ -196,21 +215,27 @@ def gen_ecdr64_loc(ecdr64_pos):
|
|||
return ret
|
||||
|
||||
|
||||
class StreamZip(object):
|
||||
def __init__(self, log, fgen, utf8=False, pre_crc=False):
|
||||
self.log = log
|
||||
self.fgen = fgen
|
||||
class StreamZip(StreamArc):
|
||||
def __init__(
|
||||
self,
|
||||
log: NamedLogger,
|
||||
fgen: Generator[dict[str, Any], None, None],
|
||||
utf8: bool = False,
|
||||
pre_crc: bool = False,
|
||||
) -> None:
|
||||
super(StreamZip, self).__init__(log, fgen)
|
||||
|
||||
self.utf8 = utf8
|
||||
self.pre_crc = pre_crc
|
||||
|
||||
self.pos = 0
|
||||
self.items = []
|
||||
self.items: list[tuple[str, int, int, int, int]] = []
|
||||
|
||||
def _ct(self, buf):
|
||||
def _ct(self, buf: bytes) -> bytes:
|
||||
self.pos += len(buf)
|
||||
return buf
|
||||
|
||||
def ser(self, f):
|
||||
def ser(self, f: dict[str, Any]) -> Generator[bytes, None, None]:
|
||||
name = f["vp"]
|
||||
src = f["ap"]
|
||||
st = f["st"]
|
||||
|
|
@ -218,9 +243,8 @@ class StreamZip(object):
|
|||
sz = st.st_size
|
||||
ts = st.st_mtime
|
||||
|
||||
crc = None
|
||||
crc = 0
|
||||
if self.pre_crc:
|
||||
crc = 0
|
||||
for buf in yieldfile(src):
|
||||
crc = zlib.crc32(buf, crc)
|
||||
|
||||
|
|
@ -230,7 +254,6 @@ class StreamZip(object):
|
|||
buf = gen_hdr(None, name, sz, ts, self.utf8, crc, self.pre_crc)
|
||||
yield self._ct(buf)
|
||||
|
||||
crc = crc or 0
|
||||
for buf in yieldfile(src):
|
||||
if not self.pre_crc:
|
||||
crc = zlib.crc32(buf, crc)
|
||||
|
|
@ -239,7 +262,7 @@ class StreamZip(object):
|
|||
|
||||
crc &= 0xFFFFFFFF
|
||||
|
||||
self.items.append([name, sz, ts, crc, h_pos])
|
||||
self.items.append((name, sz, ts, crc, h_pos))
|
||||
|
||||
z64 = sz >= 4 * 1024 * 1024 * 1024
|
||||
|
||||
|
|
@ -247,11 +270,11 @@ class StreamZip(object):
|
|||
buf = gen_fdesc(sz, crc, z64)
|
||||
yield self._ct(buf)
|
||||
|
||||
def gen(self):
|
||||
def gen(self) -> Generator[bytes, None, None]:
|
||||
errors = []
|
||||
for f in self.fgen:
|
||||
if "err" in f:
|
||||
errors.append([f["vp"], f["err"]])
|
||||
errors.append((f["vp"], f["err"]))
|
||||
continue
|
||||
|
||||
try:
|
||||
|
|
@ -259,7 +282,7 @@ class StreamZip(object):
|
|||
yield x
|
||||
except:
|
||||
ex = min_ex(5, True).replace("\n", "\n-- ")
|
||||
errors.append([f["vp"], ex])
|
||||
errors.append((f["vp"], ex))
|
||||
|
||||
if errors:
|
||||
errf, txt = errdesc(errors)
|
||||
|
|
|
|||
|
|
@ -2,12 +2,15 @@
|
|||
from __future__ import print_function, unicode_literals
|
||||
|
||||
import re
|
||||
import sys
|
||||
import socket
|
||||
import sys
|
||||
|
||||
from .__init__ import MACOS, ANYWIN, unicode
|
||||
from .__init__ import ANYWIN, MACOS, TYPE_CHECKING, unicode
|
||||
from .util import chkcmd
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .svchub import SvcHub
|
||||
|
||||
|
||||
class TcpSrv(object):
|
||||
"""
|
||||
|
|
@ -15,16 +18,16 @@ class TcpSrv(object):
|
|||
which then uses the least busy HttpSrv to handle it
|
||||
"""
|
||||
|
||||
def __init__(self, hub):
|
||||
def __init__(self, hub: "SvcHub"):
|
||||
self.hub = hub
|
||||
self.args = hub.args
|
||||
self.log = hub.log
|
||||
|
||||
self.stopping = False
|
||||
|
||||
self.srv = []
|
||||
self.srv: list[socket.socket] = []
|
||||
self.nsrv = 0
|
||||
ok = {}
|
||||
ok: dict[str, list[int]] = {}
|
||||
for ip in self.args.i:
|
||||
ok[ip] = []
|
||||
for port in self.args.p:
|
||||
|
|
@ -34,8 +37,8 @@ class TcpSrv(object):
|
|||
ok[ip].append(port)
|
||||
except Exception as ex:
|
||||
if self.args.ign_ebind or self.args.ign_ebind_all:
|
||||
m = "could not listen on {}:{}: {}"
|
||||
self.log("tcpsrv", m.format(ip, port, ex), c=3)
|
||||
t = "could not listen on {}:{}: {}"
|
||||
self.log("tcpsrv", t.format(ip, port, ex), c=3)
|
||||
else:
|
||||
raise
|
||||
|
||||
|
|
@ -55,9 +58,9 @@ class TcpSrv(object):
|
|||
eps[x] = "external"
|
||||
|
||||
msgs = []
|
||||
title_tab = {}
|
||||
title_tab: dict[str, dict[str, int]] = {}
|
||||
title_vars = [x[1:] for x in self.args.wintitle.split(" ") if x.startswith("$")]
|
||||
m = "available @ {}://{}:{}/ (\033[33m{}\033[0m)"
|
||||
t = "available @ {}://{}:{}/ (\033[33m{}\033[0m)"
|
||||
for ip, desc in sorted(eps.items(), key=lambda x: x[1]):
|
||||
for port in sorted(self.args.p):
|
||||
if port not in ok.get(ip, ok.get("0.0.0.0", [])):
|
||||
|
|
@ -69,7 +72,7 @@ class TcpSrv(object):
|
|||
elif self.args.https_only or port == 443:
|
||||
proto = "https"
|
||||
|
||||
msgs.append(m.format(proto, ip, port, desc))
|
||||
msgs.append(t.format(proto, ip, port, desc))
|
||||
|
||||
if not self.args.wintitle:
|
||||
continue
|
||||
|
|
@ -98,13 +101,13 @@ class TcpSrv(object):
|
|||
|
||||
if msgs:
|
||||
msgs[-1] += "\n"
|
||||
for m in msgs:
|
||||
self.log("tcpsrv", m)
|
||||
for t in msgs:
|
||||
self.log("tcpsrv", t)
|
||||
|
||||
if self.args.wintitle:
|
||||
self._set_wintitle(title_tab)
|
||||
|
||||
def _listen(self, ip, port):
|
||||
def _listen(self, ip: str, port: int) -> None:
|
||||
srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||
srv.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
|
||||
|
|
@ -120,7 +123,7 @@ class TcpSrv(object):
|
|||
raise
|
||||
raise Exception(e)
|
||||
|
||||
def run(self):
|
||||
def run(self) -> None:
|
||||
for srv in self.srv:
|
||||
srv.listen(self.args.nc)
|
||||
ip, port = srv.getsockname()
|
||||
|
|
@ -130,9 +133,9 @@ class TcpSrv(object):
|
|||
if self.args.q:
|
||||
print(msg)
|
||||
|
||||
self.hub.broker.put(False, "listen", srv)
|
||||
self.hub.broker.say("listen", srv)
|
||||
|
||||
def shutdown(self):
|
||||
def shutdown(self) -> None:
|
||||
self.stopping = True
|
||||
try:
|
||||
for srv in self.srv:
|
||||
|
|
@ -142,14 +145,14 @@ class TcpSrv(object):
|
|||
|
||||
self.log("tcpsrv", "ok bye")
|
||||
|
||||
def ips_linux_ifconfig(self):
|
||||
def ips_linux_ifconfig(self) -> dict[str, str]:
|
||||
# for termux
|
||||
try:
|
||||
txt, _ = chkcmd(["ifconfig"])
|
||||
except:
|
||||
return {}
|
||||
|
||||
eps = {}
|
||||
eps: dict[str, str] = {}
|
||||
dev = None
|
||||
ip = None
|
||||
up = None
|
||||
|
|
@ -171,7 +174,7 @@ class TcpSrv(object):
|
|||
|
||||
return eps
|
||||
|
||||
def ips_linux(self):
|
||||
def ips_linux(self) -> dict[str, str]:
|
||||
try:
|
||||
txt, _ = chkcmd(["ip", "addr"])
|
||||
except:
|
||||
|
|
@ -180,21 +183,21 @@ class TcpSrv(object):
|
|||
r = re.compile(r"^\s+inet ([^ ]+)/.* (.*)")
|
||||
ri = re.compile(r"^\s*[0-9]+\s*:.*")
|
||||
up = False
|
||||
eps = {}
|
||||
eps: dict[str, str] = {}
|
||||
for ln in txt.split("\n"):
|
||||
if ri.match(ln):
|
||||
up = "UP" in re.split("[>,< ]", ln)
|
||||
|
||||
try:
|
||||
ip, dev = r.match(ln.rstrip()).groups()
|
||||
ip, dev = r.match(ln.rstrip()).groups() # type: ignore
|
||||
eps[ip] = dev + ("" if up else ", \033[31mLINK-DOWN")
|
||||
except:
|
||||
pass
|
||||
|
||||
return eps
|
||||
|
||||
def ips_macos(self):
|
||||
eps = {}
|
||||
def ips_macos(self) -> dict[str, str]:
|
||||
eps: dict[str, str] = {}
|
||||
try:
|
||||
txt, _ = chkcmd(["ifconfig"])
|
||||
except:
|
||||
|
|
@ -202,7 +205,7 @@ class TcpSrv(object):
|
|||
|
||||
rdev = re.compile(r"^([^ ]+):")
|
||||
rip = re.compile(r"^\tinet ([0-9\.]+) ")
|
||||
dev = None
|
||||
dev = "UNKNOWN"
|
||||
for ln in txt.split("\n"):
|
||||
m = rdev.match(ln)
|
||||
if m:
|
||||
|
|
@ -211,17 +214,17 @@ class TcpSrv(object):
|
|||
m = rip.match(ln)
|
||||
if m:
|
||||
eps[m.group(1)] = dev
|
||||
dev = None
|
||||
dev = "UNKNOWN"
|
||||
|
||||
return eps
|
||||
|
||||
def ips_windows_ipconfig(self):
|
||||
eps = {}
|
||||
offs = {}
|
||||
def ips_windows_ipconfig(self) -> tuple[dict[str, str], set[str]]:
|
||||
eps: dict[str, str] = {}
|
||||
offs: set[str] = set()
|
||||
try:
|
||||
txt, _ = chkcmd(["ipconfig"])
|
||||
except:
|
||||
return eps
|
||||
return eps, offs
|
||||
|
||||
rdev = re.compile(r"(^[^ ].*):$")
|
||||
rip = re.compile(r"^ +IPv?4? [^:]+: *([0-9\.]{7,15})$")
|
||||
|
|
@ -231,12 +234,12 @@ class TcpSrv(object):
|
|||
m = rdev.match(ln)
|
||||
if m:
|
||||
if dev and dev not in eps.values():
|
||||
offs[dev] = 1
|
||||
offs.add(dev)
|
||||
|
||||
dev = m.group(1).split(" adapter ", 1)[-1]
|
||||
|
||||
if dev and roff.match(ln):
|
||||
offs[dev] = 1
|
||||
offs.add(dev)
|
||||
dev = None
|
||||
|
||||
m = rip.match(ln)
|
||||
|
|
@ -245,12 +248,12 @@ class TcpSrv(object):
|
|||
dev = None
|
||||
|
||||
if dev and dev not in eps.values():
|
||||
offs[dev] = 1
|
||||
offs.add(dev)
|
||||
|
||||
return eps, offs
|
||||
|
||||
def ips_windows_netsh(self):
|
||||
eps = {}
|
||||
def ips_windows_netsh(self) -> dict[str, str]:
|
||||
eps: dict[str, str] = {}
|
||||
try:
|
||||
txt, _ = chkcmd("netsh interface ip show address".split())
|
||||
except:
|
||||
|
|
@ -270,7 +273,7 @@ class TcpSrv(object):
|
|||
|
||||
return eps
|
||||
|
||||
def detect_interfaces(self, listen_ips):
|
||||
def detect_interfaces(self, listen_ips: list[str]) -> dict[str, str]:
|
||||
if MACOS:
|
||||
eps = self.ips_macos()
|
||||
elif ANYWIN:
|
||||
|
|
@ -317,7 +320,7 @@ class TcpSrv(object):
|
|||
|
||||
return eps
|
||||
|
||||
def _set_wintitle(self, vs):
|
||||
def _set_wintitle(self, vs: dict[str, dict[str, int]]) -> None:
|
||||
vs["all"] = vs.get("all", {"Local-Only": 1})
|
||||
vs["pub"] = vs.get("pub", vs["all"])
|
||||
|
||||
|
|
|
|||
|
|
@ -3,13 +3,23 @@ from __future__ import print_function, unicode_literals
|
|||
|
||||
import os
|
||||
|
||||
from .util import Cooldown
|
||||
from .th_srv import thumb_path, HAVE_WEBP
|
||||
from .__init__ import TYPE_CHECKING
|
||||
from .authsrv import VFS
|
||||
from .bos import bos
|
||||
from .th_srv import HAVE_WEBP, thumb_path
|
||||
from .util import Cooldown
|
||||
|
||||
try:
|
||||
from typing import Optional, Union
|
||||
except:
|
||||
pass
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .httpsrv import HttpSrv
|
||||
|
||||
|
||||
class ThumbCli(object):
|
||||
def __init__(self, hsrv):
|
||||
def __init__(self, hsrv: "HttpSrv") -> None:
|
||||
self.broker = hsrv.broker
|
||||
self.log_func = hsrv.log
|
||||
self.args = hsrv.args
|
||||
|
|
@ -34,10 +44,10 @@ class ThumbCli(object):
|
|||
d = next((x for x in self.args.th_dec if x in ("vips", "pil")), None)
|
||||
self.can_webp = HAVE_WEBP or d == "vips"
|
||||
|
||||
def log(self, msg, c=0):
|
||||
def log(self, msg: str, c: Union[int, str] = 0) -> None:
|
||||
self.log_func("thumbcli", msg, c)
|
||||
|
||||
def get(self, dbv, rem, mtime, fmt):
|
||||
def get(self, dbv: VFS, rem: str, mtime: float, fmt: str) -> Optional[str]:
|
||||
ptop = dbv.realpath
|
||||
ext = rem.rsplit(".")[-1].lower()
|
||||
if ext not in self.thumbable or "dthumb" in dbv.flags:
|
||||
|
|
@ -106,17 +116,17 @@ class ThumbCli(object):
|
|||
if ret:
|
||||
tdir = os.path.dirname(tpath)
|
||||
if self.cooldown.poke(tdir):
|
||||
self.broker.put(False, "thumbsrv.poke", tdir)
|
||||
self.broker.say("thumbsrv.poke", tdir)
|
||||
|
||||
if want_opus:
|
||||
# audio files expire individually
|
||||
if self.cooldown.poke(tpath):
|
||||
self.broker.put(False, "thumbsrv.poke", tpath)
|
||||
self.broker.say("thumbsrv.poke", tpath)
|
||||
|
||||
return ret
|
||||
|
||||
if abort:
|
||||
return None
|
||||
|
||||
x = self.broker.put(True, "thumbsrv.get", ptop, rem, mtime, fmt)
|
||||
return x.get()
|
||||
x = self.broker.ask("thumbsrv.get", ptop, rem, mtime, fmt)
|
||||
return x.get() # type: ignore
|
||||
|
|
|
|||
|
|
@ -1,18 +1,28 @@
|
|||
# coding: utf-8
|
||||
from __future__ import print_function, unicode_literals
|
||||
|
||||
import os
|
||||
import time
|
||||
import shutil
|
||||
import base64
|
||||
import hashlib
|
||||
import threading
|
||||
import os
|
||||
import shutil
|
||||
import subprocess as sp
|
||||
import threading
|
||||
import time
|
||||
|
||||
from .util import fsenc, vsplit, statdir, runcmd, Queue, Cooldown, BytesIO, min_ex
|
||||
from queue import Queue
|
||||
|
||||
from .__init__ import TYPE_CHECKING
|
||||
from .bos import bos
|
||||
from .mtag import HAVE_FFMPEG, HAVE_FFPROBE, ffprobe
|
||||
from .util import BytesIO, Cooldown, fsenc, min_ex, runcmd, statdir, vsplit
|
||||
|
||||
try:
|
||||
from typing import Optional, Union
|
||||
except:
|
||||
pass
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .svchub import SvcHub
|
||||
|
||||
HAVE_PIL = False
|
||||
HAVE_HEIF = False
|
||||
|
|
@ -20,7 +30,7 @@ HAVE_AVIF = False
|
|||
HAVE_WEBP = False
|
||||
|
||||
try:
|
||||
from PIL import Image, ImageOps, ExifTags
|
||||
from PIL import ExifTags, Image, ImageOps
|
||||
|
||||
HAVE_PIL = True
|
||||
try:
|
||||
|
|
@ -47,14 +57,13 @@ except:
|
|||
pass
|
||||
|
||||
try:
|
||||
import pyvips
|
||||
|
||||
HAVE_VIPS = True
|
||||
import pyvips
|
||||
except:
|
||||
HAVE_VIPS = False
|
||||
|
||||
|
||||
def thumb_path(histpath, rem, mtime, fmt):
|
||||
def thumb_path(histpath: str, rem: str, mtime: float, fmt: str) -> str:
|
||||
# base16 = 16 = 256
|
||||
# b64-lc = 38 = 1444
|
||||
# base64 = 64 = 4096
|
||||
|
|
@ -80,7 +89,7 @@ def thumb_path(histpath, rem, mtime, fmt):
|
|||
|
||||
|
||||
class ThumbSrv(object):
|
||||
def __init__(self, hub):
|
||||
def __init__(self, hub: "SvcHub") -> None:
|
||||
self.hub = hub
|
||||
self.asrv = hub.asrv
|
||||
self.args = hub.args
|
||||
|
|
@ -91,17 +100,17 @@ class ThumbSrv(object):
|
|||
self.poke_cd = Cooldown(self.args.th_poke)
|
||||
|
||||
self.mutex = threading.Lock()
|
||||
self.busy = {}
|
||||
self.busy: dict[str, list[threading.Condition]] = {}
|
||||
self.stopping = False
|
||||
self.nthr = max(1, self.args.th_mt)
|
||||
|
||||
self.q = Queue(self.nthr * 4)
|
||||
self.q: Queue[Optional[tuple[str, str]]] = Queue(self.nthr * 4)
|
||||
for n in range(self.nthr):
|
||||
t = threading.Thread(
|
||||
thr = threading.Thread(
|
||||
target=self.worker, name="thumb-{}-{}".format(n, self.nthr)
|
||||
)
|
||||
t.daemon = True
|
||||
t.start()
|
||||
thr.daemon = True
|
||||
thr.start()
|
||||
|
||||
want_ff = not self.args.no_vthumb or not self.args.no_athumb
|
||||
if want_ff and (not HAVE_FFMPEG or not HAVE_FFPROBE):
|
||||
|
|
@ -122,7 +131,7 @@ class ThumbSrv(object):
|
|||
t.start()
|
||||
|
||||
self.fmt_pil, self.fmt_vips, self.fmt_ffi, self.fmt_ffv, self.fmt_ffa = [
|
||||
{x: True for x in y.split(",")}
|
||||
set(y.split(","))
|
||||
for y in [
|
||||
self.args.th_r_pil,
|
||||
self.args.th_r_vips,
|
||||
|
|
@ -134,37 +143,37 @@ class ThumbSrv(object):
|
|||
|
||||
if not HAVE_HEIF:
|
||||
for f in "heif heifs heic heics".split(" "):
|
||||
self.fmt_pil.pop(f, None)
|
||||
self.fmt_pil.discard(f)
|
||||
|
||||
if not HAVE_AVIF:
|
||||
for f in "avif avifs".split(" "):
|
||||
self.fmt_pil.pop(f, None)
|
||||
self.fmt_pil.discard(f)
|
||||
|
||||
self.thumbable = {}
|
||||
self.thumbable: set[str] = set()
|
||||
|
||||
if "pil" in self.args.th_dec:
|
||||
self.thumbable.update(self.fmt_pil)
|
||||
self.thumbable |= self.fmt_pil
|
||||
|
||||
if "vips" in self.args.th_dec:
|
||||
self.thumbable.update(self.fmt_vips)
|
||||
self.thumbable |= self.fmt_vips
|
||||
|
||||
if "ff" in self.args.th_dec:
|
||||
for t in [self.fmt_ffi, self.fmt_ffv, self.fmt_ffa]:
|
||||
self.thumbable.update(t)
|
||||
for zss in [self.fmt_ffi, self.fmt_ffv, self.fmt_ffa]:
|
||||
self.thumbable |= zss
|
||||
|
||||
def log(self, msg, c=0):
|
||||
def log(self, msg: str, c: Union[int, str] = 0) -> None:
|
||||
self.log_func("thumb", msg, c)
|
||||
|
||||
def shutdown(self):
|
||||
def shutdown(self) -> None:
|
||||
self.stopping = True
|
||||
for _ in range(self.nthr):
|
||||
self.q.put(None)
|
||||
|
||||
def stopped(self):
|
||||
def stopped(self) -> bool:
|
||||
with self.mutex:
|
||||
return not self.nthr
|
||||
|
||||
def get(self, ptop, rem, mtime, fmt):
|
||||
def get(self, ptop: str, rem: str, mtime: float, fmt: str) -> Optional[str]:
|
||||
histpath = self.asrv.vfs.histtab.get(ptop)
|
||||
if not histpath:
|
||||
self.log("no histpath for [{}]".format(ptop))
|
||||
|
|
@ -191,7 +200,7 @@ class ThumbSrv(object):
|
|||
do_conv = True
|
||||
|
||||
if do_conv:
|
||||
self.q.put([abspath, tpath])
|
||||
self.q.put((abspath, tpath))
|
||||
self.log("conv {} \033[0m{}".format(tpath, abspath), c=6)
|
||||
|
||||
while not self.stopping:
|
||||
|
|
@ -212,7 +221,7 @@ class ThumbSrv(object):
|
|||
|
||||
return None
|
||||
|
||||
def getcfg(self):
|
||||
def getcfg(self) -> dict[str, set[str]]:
|
||||
return {
|
||||
"thumbable": self.thumbable,
|
||||
"pil": self.fmt_pil,
|
||||
|
|
@ -222,7 +231,7 @@ class ThumbSrv(object):
|
|||
"ffa": self.fmt_ffa,
|
||||
}
|
||||
|
||||
def worker(self):
|
||||
def worker(self) -> None:
|
||||
while not self.stopping:
|
||||
task = self.q.get()
|
||||
if not task:
|
||||
|
|
@ -253,7 +262,7 @@ class ThumbSrv(object):
|
|||
except:
|
||||
msg = "{} could not create thumbnail of {}\n{}"
|
||||
msg = msg.format(fun.__name__, abspath, min_ex())
|
||||
c = 1 if "<Signals.SIG" in msg else "1;30"
|
||||
c: Union[str, int] = 1 if "<Signals.SIG" in msg else "1;30"
|
||||
self.log(msg, c)
|
||||
with open(tpath, "wb") as _:
|
||||
pass
|
||||
|
|
@ -269,7 +278,7 @@ class ThumbSrv(object):
|
|||
with self.mutex:
|
||||
self.nthr -= 1
|
||||
|
||||
def fancy_pillow(self, im):
|
||||
def fancy_pillow(self, im: "Image.Image") -> "Image.Image":
|
||||
# exif_transpose is expensive (loads full image + unconditional copy)
|
||||
r = max(*self.res) * 2
|
||||
im.thumbnail((r, r), resample=Image.LANCZOS)
|
||||
|
|
@ -295,7 +304,7 @@ class ThumbSrv(object):
|
|||
|
||||
return im
|
||||
|
||||
def conv_pil(self, abspath, tpath):
|
||||
def conv_pil(self, abspath: str, tpath: str) -> None:
|
||||
with Image.open(fsenc(abspath)) as im:
|
||||
try:
|
||||
im = self.fancy_pillow(im)
|
||||
|
|
@ -324,7 +333,7 @@ class ThumbSrv(object):
|
|||
|
||||
im.save(tpath, **args)
|
||||
|
||||
def conv_vips(self, abspath, tpath):
|
||||
def conv_vips(self, abspath: str, tpath: str) -> None:
|
||||
crops = ["centre", "none"]
|
||||
if self.args.th_no_crop:
|
||||
crops = ["none"]
|
||||
|
|
@ -342,18 +351,17 @@ class ThumbSrv(object):
|
|||
|
||||
img.write_to_file(tpath, Q=40)
|
||||
|
||||
def conv_ffmpeg(self, abspath, tpath):
|
||||
def conv_ffmpeg(self, abspath: str, tpath: str) -> None:
|
||||
ret, _ = ffprobe(abspath)
|
||||
if not ret:
|
||||
return
|
||||
|
||||
ext = abspath.rsplit(".")[-1].lower()
|
||||
if ext in ["h264", "h265"] or ext in self.fmt_ffi:
|
||||
seek = []
|
||||
seek: list[bytes] = []
|
||||
else:
|
||||
dur = ret[".dur"][1] if ".dur" in ret else 4
|
||||
seek = "{:.0f}".format(dur / 3)
|
||||
seek = [b"-ss", seek.encode("utf-8")]
|
||||
seek = [b"-ss", "{:.0f}".format(dur / 3).encode("utf-8")]
|
||||
|
||||
scale = "scale={0}:{1}:force_original_aspect_ratio="
|
||||
if self.args.th_no_crop:
|
||||
|
|
@ -361,7 +369,7 @@ class ThumbSrv(object):
|
|||
else:
|
||||
scale += "increase,crop={0}:{1},setsar=1:1"
|
||||
|
||||
scale = scale.format(*list(self.res)).encode("utf-8")
|
||||
bscale = scale.format(*list(self.res)).encode("utf-8")
|
||||
# fmt: off
|
||||
cmd = [
|
||||
b"ffmpeg",
|
||||
|
|
@ -373,7 +381,7 @@ class ThumbSrv(object):
|
|||
cmd += [
|
||||
b"-i", fsenc(abspath),
|
||||
b"-map", b"0:v:0",
|
||||
b"-vf", scale,
|
||||
b"-vf", bscale,
|
||||
b"-frames:v", b"1",
|
||||
b"-metadata:s:v:0", b"rotate=0",
|
||||
]
|
||||
|
|
@ -395,14 +403,14 @@ class ThumbSrv(object):
|
|||
cmd += [fsenc(tpath)]
|
||||
self._run_ff(cmd)
|
||||
|
||||
def _run_ff(self, cmd):
|
||||
def _run_ff(self, cmd: list[bytes]) -> None:
|
||||
# self.log((b" ".join(cmd)).decode("utf-8"))
|
||||
ret, _, serr = runcmd(cmd, timeout=self.args.th_convt)
|
||||
if not ret:
|
||||
return
|
||||
|
||||
c = "1;30"
|
||||
m = "FFmpeg failed (probably a corrupt video file):\n"
|
||||
c: Union[str, int] = "1;30"
|
||||
t = "FFmpeg failed (probably a corrupt video file):\n"
|
||||
if cmd[-1].lower().endswith(b".webp") and (
|
||||
"Error selecting an encoder" in serr
|
||||
or "Automatic encoder selection failed" in serr
|
||||
|
|
@ -410,14 +418,14 @@ class ThumbSrv(object):
|
|||
or "Please choose an encoder manually" in serr
|
||||
):
|
||||
self.args.th_ff_jpg = True
|
||||
m = "FFmpeg failed because it was compiled without libwebp; enabling --th-ff-jpg to force jpeg output:\n"
|
||||
t = "FFmpeg failed because it was compiled without libwebp; enabling --th-ff-jpg to force jpeg output:\n"
|
||||
c = 1
|
||||
|
||||
if (
|
||||
"Requested resampling engine is unavailable" in serr
|
||||
or "output pad on Parsed_aresample_" in serr
|
||||
):
|
||||
m = "FFmpeg failed because it was compiled without libsox; you must set --th-ff-swr to force swr resampling:\n"
|
||||
t = "FFmpeg failed because it was compiled without libsox; you must set --th-ff-swr to force swr resampling:\n"
|
||||
c = 1
|
||||
|
||||
lines = serr.strip("\n").split("\n")
|
||||
|
|
@ -428,10 +436,10 @@ class ThumbSrv(object):
|
|||
if len(txt) > 5000:
|
||||
txt = txt[:2500] + "...\nff: [...]\nff: ..." + txt[-2500:]
|
||||
|
||||
self.log(m + txt, c=c)
|
||||
self.log(t + txt, c=c)
|
||||
raise sp.CalledProcessError(ret, (cmd[0], b"...", cmd[-1]))
|
||||
|
||||
def conv_spec(self, abspath, tpath):
|
||||
def conv_spec(self, abspath: str, tpath: str) -> None:
|
||||
ret, _ = ffprobe(abspath)
|
||||
if "ac" not in ret:
|
||||
raise Exception("not audio")
|
||||
|
|
@ -473,7 +481,7 @@ class ThumbSrv(object):
|
|||
cmd += [fsenc(tpath)]
|
||||
self._run_ff(cmd)
|
||||
|
||||
def conv_opus(self, abspath, tpath):
|
||||
def conv_opus(self, abspath: str, tpath: str) -> None:
|
||||
if self.args.no_acode:
|
||||
raise Exception("disabled in server config")
|
||||
|
||||
|
|
@ -521,7 +529,7 @@ class ThumbSrv(object):
|
|||
# fmt: on
|
||||
self._run_ff(cmd)
|
||||
|
||||
def poke(self, tdir):
|
||||
def poke(self, tdir: str) -> None:
|
||||
if not self.poke_cd.poke(tdir):
|
||||
return
|
||||
|
||||
|
|
@ -533,7 +541,7 @@ class ThumbSrv(object):
|
|||
except:
|
||||
pass
|
||||
|
||||
def cleaner(self):
|
||||
def cleaner(self) -> None:
|
||||
interval = self.args.th_clean
|
||||
while True:
|
||||
time.sleep(interval)
|
||||
|
|
@ -548,14 +556,14 @@ class ThumbSrv(object):
|
|||
|
||||
self.log("\033[Jcln ok; rm {} dirs".format(ndirs))
|
||||
|
||||
def clean(self, histpath):
|
||||
def clean(self, histpath: str) -> int:
|
||||
ret = 0
|
||||
for cat in ["th", "ac"]:
|
||||
ret += self._clean(histpath, cat, None)
|
||||
ret += self._clean(histpath, cat, "")
|
||||
|
||||
return ret
|
||||
|
||||
def _clean(self, histpath, cat, thumbpath):
|
||||
def _clean(self, histpath: str, cat: str, thumbpath: str) -> int:
|
||||
if not thumbpath:
|
||||
thumbpath = os.path.join(histpath, cat)
|
||||
|
||||
|
|
@ -564,10 +572,10 @@ class ThumbSrv(object):
|
|||
maxage = getattr(self.args, cat + "_maxage")
|
||||
now = time.time()
|
||||
prev_b64 = None
|
||||
prev_fp = None
|
||||
prev_fp = ""
|
||||
try:
|
||||
ents = statdir(self.log, not self.args.no_scandir, False, thumbpath)
|
||||
ents = sorted(list(ents))
|
||||
t1 = statdir(self.log_func, not self.args.no_scandir, False, thumbpath)
|
||||
ents = sorted(list(t1))
|
||||
except:
|
||||
return 0
|
||||
|
||||
|
|
|
|||
|
|
@ -1,34 +1,37 @@
|
|||
# coding: utf-8
|
||||
from __future__ import print_function, unicode_literals
|
||||
|
||||
import re
|
||||
import os
|
||||
import time
|
||||
import calendar
|
||||
import os
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
from operator import itemgetter
|
||||
|
||||
from .__init__ import ANYWIN, unicode
|
||||
from .util import absreal, s3dec, Pebkac, min_ex, gen_filekey, quotep
|
||||
from .__init__ import ANYWIN, TYPE_CHECKING, unicode
|
||||
from .bos import bos
|
||||
from .up2k import up2k_wark_from_hashlist
|
||||
from .util import HAVE_SQLITE3, Pebkac, absreal, gen_filekey, min_ex, quotep, s3dec
|
||||
|
||||
|
||||
try:
|
||||
HAVE_SQLITE3 = True
|
||||
if HAVE_SQLITE3:
|
||||
import sqlite3
|
||||
except:
|
||||
HAVE_SQLITE3 = False
|
||||
|
||||
|
||||
try:
|
||||
from pathlib import Path
|
||||
except:
|
||||
pass
|
||||
|
||||
try:
|
||||
from typing import Any, Optional, Union
|
||||
except:
|
||||
pass
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .httpconn import HttpConn
|
||||
|
||||
|
||||
class U2idx(object):
|
||||
def __init__(self, conn):
|
||||
def __init__(self, conn: "HttpConn") -> None:
|
||||
self.log_func = conn.log_func
|
||||
self.asrv = conn.asrv
|
||||
self.args = conn.args
|
||||
|
|
@ -38,19 +41,21 @@ class U2idx(object):
|
|||
self.log("your python does not have sqlite3; searching will be disabled")
|
||||
return
|
||||
|
||||
self.active_id = None
|
||||
self.active_cur = None
|
||||
self.cur = {}
|
||||
self.mem_cur = sqlite3.connect(":memory:")
|
||||
self.active_id = ""
|
||||
self.active_cur: Optional["sqlite3.Cursor"] = None
|
||||
self.cur: dict[str, "sqlite3.Cursor"] = {}
|
||||
self.mem_cur = sqlite3.connect(":memory:").cursor()
|
||||
self.mem_cur.execute(r"create table a (b text)")
|
||||
|
||||
self.p_end = None
|
||||
self.p_dur = 0
|
||||
self.p_end = 0.0
|
||||
self.p_dur = 0.0
|
||||
|
||||
def log(self, msg, c=0):
|
||||
def log(self, msg: str, c: Union[int, str] = 0) -> None:
|
||||
self.log_func("u2idx", msg, c)
|
||||
|
||||
def fsearch(self, vols, body):
|
||||
def fsearch(
|
||||
self, vols: list[tuple[str, str, dict[str, Any]]], body: dict[str, Any]
|
||||
) -> list[dict[str, Any]]:
|
||||
"""search by up2k hashlist"""
|
||||
if not HAVE_SQLITE3:
|
||||
return []
|
||||
|
|
@ -60,14 +65,14 @@ class U2idx(object):
|
|||
wark = up2k_wark_from_hashlist(self.args.salt, fsize, fhash)
|
||||
|
||||
uq = "substr(w,1,16) = ? and w = ?"
|
||||
uv = [wark[:16], wark]
|
||||
uv: list[Union[str, int]] = [wark[:16], wark]
|
||||
|
||||
try:
|
||||
return self.run_query(vols, uq, uv, True, False, 99999)[0]
|
||||
except:
|
||||
raise Pebkac(500, min_ex())
|
||||
|
||||
def get_cur(self, ptop):
|
||||
def get_cur(self, ptop: str) -> Optional["sqlite3.Cursor"]:
|
||||
if not HAVE_SQLITE3:
|
||||
return None
|
||||
|
||||
|
|
@ -103,13 +108,16 @@ class U2idx(object):
|
|||
self.cur[ptop] = cur
|
||||
return cur
|
||||
|
||||
def search(self, vols, uq, lim):
|
||||
def search(
|
||||
self, vols: list[tuple[str, str, dict[str, Any]]], uq: str, lim: int
|
||||
) -> tuple[list[dict[str, Any]], list[str]]:
|
||||
"""search by query params"""
|
||||
if not HAVE_SQLITE3:
|
||||
return []
|
||||
return [], []
|
||||
|
||||
q = ""
|
||||
va = []
|
||||
v: Union[str, int] = ""
|
||||
va: list[Union[str, int]] = []
|
||||
have_up = False # query has up.* operands
|
||||
have_mt = False
|
||||
is_key = True
|
||||
|
|
@ -202,7 +210,7 @@ class U2idx(object):
|
|||
"%Y",
|
||||
]:
|
||||
try:
|
||||
v = calendar.timegm(time.strptime(v, fmt))
|
||||
v = calendar.timegm(time.strptime(str(v), fmt))
|
||||
break
|
||||
except:
|
||||
pass
|
||||
|
|
@ -230,11 +238,12 @@ class U2idx(object):
|
|||
|
||||
# lowercase tag searches
|
||||
m = ptn_lc.search(q)
|
||||
if not m or not ptn_lcv.search(unicode(v)):
|
||||
zs = unicode(v)
|
||||
if not m or not ptn_lcv.search(zs):
|
||||
continue
|
||||
|
||||
va.pop()
|
||||
va.append(v.lower())
|
||||
va.append(zs.lower())
|
||||
q = q[: m.start()]
|
||||
|
||||
field, oper = m.groups()
|
||||
|
|
@ -248,8 +257,16 @@ class U2idx(object):
|
|||
except Exception as ex:
|
||||
raise Pebkac(500, repr(ex))
|
||||
|
||||
def run_query(self, vols, uq, uv, have_up, have_mt, lim):
|
||||
done_flag = []
|
||||
def run_query(
|
||||
self,
|
||||
vols: list[tuple[str, str, dict[str, Any]]],
|
||||
uq: str,
|
||||
uv: list[Union[str, int]],
|
||||
have_up: bool,
|
||||
have_mt: bool,
|
||||
lim: int,
|
||||
) -> tuple[list[dict[str, Any]], list[str]]:
|
||||
done_flag: list[bool] = []
|
||||
self.active_id = "{:.6f}_{}".format(
|
||||
time.time(), threading.current_thread().ident
|
||||
)
|
||||
|
|
@ -266,13 +283,11 @@ class U2idx(object):
|
|||
|
||||
if not uq or not uv:
|
||||
uq = "select * from up"
|
||||
uv = ()
|
||||
uv = []
|
||||
elif have_mt:
|
||||
uq = "select up.*, substr(up.w,1,16) mtw from up where " + uq
|
||||
uv = tuple(uv)
|
||||
else:
|
||||
uq = "select up.* from up where " + uq
|
||||
uv = tuple(uv)
|
||||
|
||||
self.log("qs: {!r} {!r}".format(uq, uv))
|
||||
|
||||
|
|
@ -292,11 +307,10 @@ class U2idx(object):
|
|||
v = vtop + "/"
|
||||
|
||||
vuv.append(v)
|
||||
vuv = tuple(vuv)
|
||||
|
||||
sret = []
|
||||
fk = flags.get("fk")
|
||||
c = cur.execute(uq, vuv)
|
||||
c = cur.execute(uq, tuple(vuv))
|
||||
for hit in c:
|
||||
w, ts, sz, rd, fn, ip, at = hit[:7]
|
||||
lim -= 1
|
||||
|
|
@ -340,7 +354,7 @@ class U2idx(object):
|
|||
# print("[{}] {}".format(ptop, sret))
|
||||
|
||||
done_flag.append(True)
|
||||
self.active_id = None
|
||||
self.active_id = ""
|
||||
|
||||
# undupe hits from multiple metadata keys
|
||||
if len(ret) > 1:
|
||||
|
|
@ -354,11 +368,12 @@ class U2idx(object):
|
|||
|
||||
return ret, list(taglist.keys())
|
||||
|
||||
def terminator(self, identifier, done_flag):
|
||||
def terminator(self, identifier: str, done_flag: list[bool]) -> None:
|
||||
for _ in range(self.timeout):
|
||||
time.sleep(1)
|
||||
if done_flag:
|
||||
return
|
||||
|
||||
if identifier == self.active_id:
|
||||
assert self.active_cur
|
||||
self.active_cur.connection.interrupt()
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
|
|
@ -90,6 +90,15 @@ function have() {
|
|||
have setuptools
|
||||
have wheel
|
||||
have twine
|
||||
|
||||
# remove type hints to support python < 3.9
|
||||
rm -rf build/pypi
|
||||
mkdir -p build/pypi
|
||||
cp -pR setup.py README.md LICENSE copyparty tests bin scripts/strip_hints build/pypi/
|
||||
cd build/pypi
|
||||
tar --strip-components=2 -xf ../strip-hints-0.1.10.tar.gz strip-hints-0.1.10/src/strip_hints
|
||||
python3 -c 'from strip_hints.a import uh; uh("copyparty")'
|
||||
|
||||
./setup.py clean2
|
||||
./setup.py sdist bdist_wheel --universal
|
||||
|
||||
|
|
|
|||
|
|
@ -76,7 +76,7 @@ while [ ! -z "$1" ]; do
|
|||
no-hl) no_hl=1 ; ;;
|
||||
no-dd) no_dd=1 ; ;;
|
||||
no-cm) no_cm=1 ; ;;
|
||||
fast) zopf=100 ; ;;
|
||||
fast) zopf= ; ;;
|
||||
lang) shift;langs="$1"; ;;
|
||||
*) help ; ;;
|
||||
esac
|
||||
|
|
@ -106,7 +106,7 @@ tmpdir="$(
|
|||
[ $repack ] && {
|
||||
old="$tmpdir/pe-copyparty"
|
||||
echo "repack of files in $old"
|
||||
cp -pR "$old/"*{dep-j2,dep-ftp,copyparty} .
|
||||
cp -pR "$old/"*{j2,ftp,copyparty} .
|
||||
}
|
||||
|
||||
[ $repack ] || {
|
||||
|
|
@ -130,8 +130,8 @@ tmpdir="$(
|
|||
mv MarkupSafe-*/src/markupsafe .
|
||||
rm -rf MarkupSafe-* markupsafe/_speedups.c
|
||||
|
||||
mkdir dep-j2/
|
||||
mv {markupsafe,jinja2} dep-j2/
|
||||
mkdir j2/
|
||||
mv {markupsafe,jinja2} j2/
|
||||
|
||||
echo collecting pyftpdlib
|
||||
f="../build/pyftpdlib-1.5.6.tar.gz"
|
||||
|
|
@ -143,8 +143,8 @@ tmpdir="$(
|
|||
mv pyftpdlib-release-*/pyftpdlib .
|
||||
rm -rf pyftpdlib-release-* pyftpdlib/test
|
||||
|
||||
mkdir dep-ftp/
|
||||
mv pyftpdlib dep-ftp/
|
||||
mkdir ftp/
|
||||
mv pyftpdlib ftp/
|
||||
|
||||
echo collecting asyncore, asynchat
|
||||
for n in asyncore.py asynchat.py; do
|
||||
|
|
@ -154,6 +154,24 @@ tmpdir="$(
|
|||
wget -O$f "$url" || curl -L "$url" >$f)
|
||||
done
|
||||
|
||||
# enable this to dynamically remove type hints at startup,
|
||||
# in case a future python version can use them for performance
|
||||
true || (
|
||||
echo collecting strip-hints
|
||||
f=../build/strip-hints-0.1.10.tar.gz
|
||||
[ -e $f ] ||
|
||||
(url=https://files.pythonhosted.org/packages/9c/d4/312ddce71ee10f7e0ab762afc027e07a918f1c0e1be5b0069db5b0e7542d/strip-hints-0.1.10.tar.gz;
|
||||
wget -O$f "$url" || curl -L "$url" >$f)
|
||||
|
||||
tar -zxf $f
|
||||
mv strip-hints-0.1.10/src/strip_hints .
|
||||
rm -rf strip-hints-* strip_hints/import_hooks*
|
||||
sed -ri 's/[a-z].* as import_hooks$/"""a"""/' strip_hints/*.py
|
||||
|
||||
cp -pR ../scripts/strip_hints/ .
|
||||
)
|
||||
cp -pR ../scripts/py2/ .
|
||||
|
||||
# msys2 tar is bad, make the best of it
|
||||
echo collecting source
|
||||
[ $clean ] && {
|
||||
|
|
@ -170,6 +188,9 @@ tmpdir="$(
|
|||
for n in asyncore.py asynchat.py; do
|
||||
awk 'NR<4||NR>27;NR==4{print"# license: https://opensource.org/licenses/ISC\n"}' ../build/$n >copyparty/vend/$n
|
||||
done
|
||||
|
||||
# remove type hints before build instead
|
||||
(cd copyparty; python3 ../../scripts/strip_hints/a.py; rm uh)
|
||||
}
|
||||
|
||||
ver=
|
||||
|
|
@ -274,17 +295,23 @@ rm have
|
|||
tmv "$f"
|
||||
done
|
||||
|
||||
[ $repack ] ||
|
||||
find | grep -E '\.py$' |
|
||||
grep -vE '__version__' |
|
||||
tr '\n' '\0' |
|
||||
xargs -0 "$pybin" ../scripts/uncomment.py
|
||||
[ $repack ] || {
|
||||
# uncomment
|
||||
find | grep -E '\.py$' |
|
||||
grep -vE '__version__' |
|
||||
tr '\n' '\0' |
|
||||
xargs -0 "$pybin" ../scripts/uncomment.py
|
||||
|
||||
f=dep-j2/jinja2/constants.py
|
||||
# py2-compat
|
||||
#find | grep -E '\.py$' | while IFS= read -r x; do
|
||||
# sed -ri '/: TypeAlias = /d' "$x"; done
|
||||
}
|
||||
|
||||
f=j2/jinja2/constants.py
|
||||
awk '/^LOREM_IPSUM_WORDS/{o=1;print "LOREM_IPSUM_WORDS = u\"a\"";next} !o; /"""/{o=0}' <$f >t
|
||||
tmv "$f"
|
||||
|
||||
grep -rLE '^#[^a-z]*coding: utf-8' dep-j2 |
|
||||
grep -rLE '^#[^a-z]*coding: utf-8' j2 |
|
||||
while IFS= read -r f; do
|
||||
(echo "# coding: utf-8"; cat "$f") >t
|
||||
tmv "$f"
|
||||
|
|
@ -313,7 +340,7 @@ find | grep -E '\.(js|html)$' | while IFS= read -r f; do
|
|||
done
|
||||
|
||||
gzres() {
|
||||
command -v pigz &&
|
||||
command -v pigz && [ $zopf ] &&
|
||||
pk="pigz -11 -I $zopf" ||
|
||||
pk='gzip'
|
||||
|
||||
|
|
@ -354,7 +381,8 @@ nf=$(ls -1 "$zdir"/arc.* | wc -l)
|
|||
}
|
||||
[ $use_zdir ] && {
|
||||
arcs=("$zdir"/arc.*)
|
||||
arc="${arcs[$RANDOM % ${#arcs[@]} ] }"
|
||||
n=$(( $RANDOM % ${#arcs[@]} ))
|
||||
arc="${arcs[n]}"
|
||||
echo "using $arc"
|
||||
tar -xf "$arc"
|
||||
for f in copyparty/web/*.gz; do
|
||||
|
|
@ -364,7 +392,7 @@ nf=$(ls -1 "$zdir"/arc.* | wc -l)
|
|||
|
||||
|
||||
echo gen tarlist
|
||||
for d in copyparty dep-j2 dep-ftp; do find $d -type f; done |
|
||||
for d in copyparty j2 ftp py2; do find $d -type f; done | # strip_hints
|
||||
sed -r 's/(.*)\.(.*)/\2 \1/' | LC_ALL=C sort |
|
||||
sed -r 's/([^ ]*) (.*)/\2.\1/' | grep -vE '/list1?$' > list1
|
||||
|
||||
|
|
|
|||
|
|
@ -1,13 +1,23 @@
|
|||
#!/bin/bash
|
||||
set -ex
|
||||
|
||||
rm -rf unt
|
||||
mkdir -p unt/srv
|
||||
cp -pR copyparty tests unt/
|
||||
cd unt
|
||||
python3 ../scripts/strip_hints/a.py
|
||||
|
||||
pids=()
|
||||
for py in python{2,3}; do
|
||||
PYTHONPATH=
|
||||
[ $py = python2 ] && PYTHONPATH=../scripts/py2
|
||||
export PYTHONPATH
|
||||
|
||||
nice $py -m unittest discover -s tests >/dev/null &
|
||||
pids+=($!)
|
||||
done
|
||||
|
||||
python3 scripts/test/smoketest.py &
|
||||
python3 ../scripts/test/smoketest.py &
|
||||
pids+=($!)
|
||||
|
||||
for pid in ${pids[@]}; do
|
||||
|
|
|
|||
|
|
@ -379,9 +379,20 @@ def run(tmp, j2, ftp):
|
|||
t.daemon = True
|
||||
t.start()
|
||||
|
||||
ld = (("", ""), (j2, "dep-j2"), (ftp, "dep-ftp"))
|
||||
ld = (("", ""), (j2, "j2"), (ftp, "ftp"), (not PY2, "py2"))
|
||||
ld = [os.path.join(tmp, b) for a, b in ld if not a]
|
||||
|
||||
# skip 1
|
||||
# enable this to dynamically remove type hints at startup,
|
||||
# in case a future python version can use them for performance
|
||||
if sys.version_info < (3, 10) and False:
|
||||
sys.path.insert(0, ld[0])
|
||||
|
||||
from strip_hints.a import uh
|
||||
|
||||
uh(tmp + "/copyparty")
|
||||
# skip 0
|
||||
|
||||
if any([re.match(r"^-.*j[0-9]", x) for x in sys.argv]):
|
||||
run_s(ld)
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -47,7 +47,7 @@ grep -E '/(python|pypy)[0-9\.-]*$' >$dir/pys || true
|
|||
printf '\033[1;30mlooking for jinja2 in [%s]\033[0m\n' "$_py" >&2
|
||||
$_py -c 'import jinja2' 2>/dev/null || continue
|
||||
printf '%s\n' "$_py"
|
||||
mv $dir/{,x.}dep-j2
|
||||
mv $dir/{,x.}j2
|
||||
break
|
||||
done)"
|
||||
|
||||
|
|
|
|||
57
scripts/strip_hints/a.py
Normal file
57
scripts/strip_hints/a.py
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
# coding: utf-8
|
||||
from __future__ import print_function, unicode_literals
|
||||
|
||||
import re
|
||||
import os
|
||||
import sys
|
||||
from strip_hints import strip_file_to_string
|
||||
|
||||
|
||||
# list unique types used in hints:
|
||||
# rm -rf unt && cp -pR copyparty unt && (cd unt && python3 ../scripts/strip_hints/a.py)
|
||||
# diff -wNarU1 copyparty unt | grep -E '^\-' | sed -r 's/[^][, ]+://g; s/[^][, ]+[[(]//g; s/[],()<>{} -]/\n/g' | grep -E .. | sort | uniq -c | sort -n
|
||||
|
||||
|
||||
def pr(m):
|
||||
sys.stderr.write(m)
|
||||
sys.stderr.flush()
|
||||
|
||||
|
||||
def uh(top):
|
||||
if os.path.exists(top + "/uh"):
|
||||
return
|
||||
|
||||
libs = "typing|types|collections\.abc"
|
||||
ptn = re.compile(r"^(\s*)(from (?:{0}) import |import (?:{0})\b).*".format(libs))
|
||||
|
||||
# pr("building support for your python ver")
|
||||
pr("unhinting")
|
||||
for (dp, _, fns) in os.walk(top):
|
||||
for fn in fns:
|
||||
if not fn.endswith(".py"):
|
||||
continue
|
||||
|
||||
pr(".")
|
||||
fp = os.path.join(dp, fn)
|
||||
cs = strip_file_to_string(fp, no_ast=True, to_empty=True)
|
||||
|
||||
# remove expensive imports too
|
||||
lns = []
|
||||
for ln in cs.split("\n"):
|
||||
m = ptn.match(ln)
|
||||
if m:
|
||||
ln = m.group(1) + "raise Exception()"
|
||||
|
||||
lns.append(ln)
|
||||
|
||||
cs = "\n".join(lns)
|
||||
with open(fp, "wb") as f:
|
||||
f.write(cs.encode("utf-8"))
|
||||
|
||||
pr("k\n\n")
|
||||
with open(top + "/uh", "wb") as f:
|
||||
f.write(b"a")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
uh(".")
|
||||
|
|
@ -58,13 +58,13 @@ class CState(threading.Thread):
|
|||
remotes.append("?")
|
||||
remotes_ok = False
|
||||
|
||||
m = []
|
||||
ta = []
|
||||
for conn, remote in zip(self.cs, remotes):
|
||||
stage = len(conn.st)
|
||||
m.append(f"\033[3{colors[stage]}m{remote}")
|
||||
ta.append(f"\033[3{colors[stage]}m{remote}")
|
||||
|
||||
m = " ".join(m)
|
||||
print(f"{m}\033[0m\n\033[A", end="")
|
||||
t = " ".join(ta)
|
||||
print(f"{t}\033[0m\n\033[A", end="")
|
||||
|
||||
|
||||
def allget(cs, urls):
|
||||
|
|
|
|||
|
|
@ -72,6 +72,8 @@ def tc1(vflags):
|
|||
for _ in range(10):
|
||||
try:
|
||||
os.mkdir(td)
|
||||
if os.path.exists(td):
|
||||
break
|
||||
except:
|
||||
time.sleep(0.1) # win10
|
||||
|
||||
|
|
|
|||
|
|
@ -85,7 +85,7 @@ class TestVFS(unittest.TestCase):
|
|||
pass
|
||||
|
||||
def assertAxs(self, dct, lst):
|
||||
t1 = list(sorted(dct.keys()))
|
||||
t1 = list(sorted(dct))
|
||||
t2 = list(sorted(lst))
|
||||
self.assertEqual(t1, t2)
|
||||
|
||||
|
|
@ -208,10 +208,10 @@ class TestVFS(unittest.TestCase):
|
|||
self.assertEqual(n.realpath, os.path.join(td, "a"))
|
||||
self.assertAxs(n.axs.uread, ["*"])
|
||||
self.assertAxs(n.axs.uwrite, [])
|
||||
self.assertEqual(vfs.can_access("/", "*"), [False, False, False, False, False])
|
||||
self.assertEqual(vfs.can_access("/", "k"), [True, True, False, False, False])
|
||||
self.assertEqual(vfs.can_access("/a", "*"), [True, False, False, False, False])
|
||||
self.assertEqual(vfs.can_access("/a", "k"), [True, False, False, False, False])
|
||||
self.assertEqual(vfs.can_access("/", "*"), (False, False, False, False, False))
|
||||
self.assertEqual(vfs.can_access("/", "k"), (True, True, False, False, False))
|
||||
self.assertEqual(vfs.can_access("/a", "*"), (True, False, False, False, False))
|
||||
self.assertEqual(vfs.can_access("/a", "k"), (True, False, False, False, False))
|
||||
|
||||
# breadth-first construction
|
||||
vfs = AuthSrv(
|
||||
|
|
@ -279,7 +279,7 @@ class TestVFS(unittest.TestCase):
|
|||
n = au.vfs
|
||||
# root was not defined, so PWD with no access to anyone
|
||||
self.assertEqual(n.vpath, "")
|
||||
self.assertEqual(n.realpath, None)
|
||||
self.assertEqual(n.realpath, "")
|
||||
self.assertAxs(n.axs.uread, [])
|
||||
self.assertAxs(n.axs.uwrite, [])
|
||||
self.assertEqual(len(n.nodes), 1)
|
||||
|
|
|
|||
|
|
@ -90,7 +90,10 @@ def get_ramdisk():
|
|||
|
||||
|
||||
class NullBroker(object):
|
||||
def put(*args):
|
||||
def say(*args):
|
||||
pass
|
||||
|
||||
def ask(*args):
|
||||
pass
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue