diff --git a/README.md b/README.md index 75f74a55..9b34ddb1 100644 --- a/README.md +++ b/README.md @@ -695,6 +695,7 @@ if you set `--no-hash [...]` globally, you can enable hashing for specific volum set upload rules using volume flags, some examples: * `:c,sz=1k-3m` sets allowed filesize between 1 KiB and 3 MiB inclusive (suffixes: `b`, `k`, `m`, `g`) +* `:c,df=4g` block uploads if there would be less than 4 GiB free disk space afterwards * `:c,nosub` disallow uploading into subdirectories; goes well with `rotn` and `rotf`: * `:c,rotn=1000,2` moves uploads into subfolders, up to 1000 files in each folder before making a new one, two levels deep (must be at least 1) * `:c,rotf=%Y/%m/%d/%H` enforces files to be uploaded into a structure of subfolders according to that date format diff --git a/copyparty/__main__.py b/copyparty/__main__.py index 72abfebe..e5807ddb 100644 --- a/copyparty/__main__.py +++ b/copyparty/__main__.py @@ -382,6 +382,7 @@ def run_argparse(argv: list[str], formatter: Any, retry: bool) -> argparse.Names \033[36mmaxn=250,600\033[35m max 250 uploads over 15min \033[36mmaxb=1g,300\033[35m max 1 GiB over 5min (suffixes: b, k, m, g) \033[36msz=1k-3m\033[35m allow filesizes between 1 KiB and 3MiB + \033[36mdf=1g\033[35m ensure 1 GiB free disk space \033[0mupload rotation: (moves all uploads into the specified folder structure) @@ -482,6 +483,7 @@ def run_argparse(argv: list[str], formatter: Any, retry: bool) -> argparse.Names ap2.add_argument("--hardlink", action="store_true", help="prefer hardlinks instead of symlinks when possible (within same filesystem)") ap2.add_argument("--never-symlink", action="store_true", help="do not fallback to symlinks when a hardlink cannot be made") ap2.add_argument("--no-dedup", action="store_true", help="disable symlink/hardlink creation; copy file contents instead") + ap2.add_argument("--df", metavar="GiB", type=float, default=0, help="ensure GiB free disk space by rejecting upload requests") ap2.add_argument("--sparse", metavar="MiB", type=int, default=4, help="windows-only: minimum size of incoming uploads through up2k before they are made into sparse files") ap2.add_argument("--turbo", metavar="LVL", type=int, default=0, help="configure turbo-mode in up2k client; 0 = off and warn if enabled, 1 = off, 2 = on, 3 = on and disable datecheck") ap2.add_argument("--u2sort", metavar="TXT", type=u, default="s", help="upload order; s=smallest-first, n=alphabetical, fs=force-s, fn=force-n -- alphabetical is a bit slower on fiber/LAN but makes it easier to eyeball if everything went fine") diff --git a/copyparty/authsrv.py b/copyparty/authsrv.py index 8d0f3b97..185767d8 100644 --- a/copyparty/authsrv.py +++ b/copyparty/authsrv.py @@ -20,6 +20,8 @@ from .util import ( Pebkac, absreal, fsenc, + get_df, + humansize, relchk, statdir, uncyg, @@ -72,15 +74,23 @@ class AXS(object): class Lim(object): - def __init__(self) -> None: + def __init__(self, log_func: Optional["RootLogger"]) -> None: + self.log_func = log_func + + self.reg: Optional[dict[str, dict[str, Any]]] = None # up2k registry + self.nups: dict[str, list[float]] = {} # num tracker self.bups: dict[str, list[tuple[float, int]]] = {} # byte tracker list self.bupc: dict[str, int] = {} # byte tracker cache self.nosub = False # disallow subdirectories - self.smin = -1 # filesize min - self.smax = -1 # filesize max + self.dfl = 0 # free disk space limit + self.dft = 0 # last-measured time + self.dfv: Optional[int] = 0 # currently free + + self.smin = 0 # filesize min + self.smax = 0 # filesize max self.bwin = 0 # bytes window self.bmax = 0 # bytes max @@ -92,18 +102,34 @@ class Lim(object): self.rotf = "" # rot datefmt self.rot_re = re.compile("") # rotf check + def log(self, msg: str, c: Union[int, str] = 0) -> None: + if self.log_func: + self.log_func("up-lim", msg, c) + def set_rotf(self, fmt: str) -> None: self.rotf = fmt r = re.escape(fmt).replace("%Y", "[0-9]{4}").replace("%j", "[0-9]{3}") r = re.sub("%[mdHMSWU]", "[0-9]{2}", r) self.rot_re = re.compile("(^|/)" + r + "$") - def all(self, ip: str, rem: str, sz: float, abspath: str) -> tuple[str, str]: + def all( + self, + ip: str, + rem: str, + sz: int, + abspath: str, + reg: Optional[dict[str, dict[str, Any]]] = None, + ) -> tuple[str, str]: + if reg is not None and self.reg is None: + self.reg = reg + self.dft = 0 + self.chk_nup(ip) self.chk_bup(ip) self.chk_rem(rem) if sz != -1: self.chk_sz(sz) + self.chk_df(abspath, sz) # side effects; keep last-ish ap2, vp2 = self.rot(abspath) if abspath == ap2: @@ -111,13 +137,33 @@ class Lim(object): return ap2, ("{}/{}".format(rem, vp2) if rem else vp2) - def chk_sz(self, sz: float) -> None: - if self.smin != -1 and sz < self.smin: + def chk_sz(self, sz: int) -> None: + if sz < self.smin: raise Pebkac(400, "file too small") - if self.smax != -1 and sz > self.smax: + if self.smax and sz > self.smax: raise Pebkac(400, "file too big") + def chk_df(self, abspath: str, sz: int, already_written: bool = False) -> None: + if not self.dfl: + return + + if self.dft < time.time(): + self.dft = int(time.time()) + 300 + self.dfv = get_df(abspath)[0] + for j in list(self.reg.values()) if self.reg else []: + self.dfv -= int(j["size"] / len(j["hash"]) * len(j["need"])) + + if already_written: + sz = 0 + + if self.dfv - sz < self.dfl: + self.dft = min(self.dft, int(time.time()) + 10) + t = "server HDD is full; {} free, need {}" + raise Pebkac(500, t.format(humansize(self.dfv - self.dfl), humansize(sz))) + + self.dfv -= int(sz) + def chk_rem(self, rem: str) -> None: if self.nosub and rem: raise Pebkac(500, "no subdirectories allowed") @@ -226,7 +272,7 @@ class VFS(object): def __init__( self, - log: Optional[RootLogger], + log: Optional["RootLogger"], realpath: str, vpath: str, axs: AXS, @@ -569,7 +615,7 @@ class AuthSrv(object): def __init__( self, args: argparse.Namespace, - log_func: Optional[RootLogger], + log_func: Optional["RootLogger"], warn_anonwrite: bool = True, ) -> None: self.args = args @@ -917,13 +963,20 @@ class AuthSrv(object): vfs.histtab = {zv.realpath: zv.histpath for zv in vfs.all_vols.values()} for vol in vfs.all_vols.values(): - lim = Lim() + lim = Lim(self.log_func) use = False if vol.flags.get("nosub"): use = True lim.nosub = True + zs = vol.flags.get("df") or ( + "{}g".format(self.args.df) if self.args.df else "" + ) + if zs: + use = True + lim.dfl = unhumanize(zs) + zs = vol.flags.get("sz") if zs: use = True @@ -1126,7 +1179,7 @@ class AuthSrv(object): u = u if u else "\033[36m--none--\033[0m" t += "\n| {}: {}".format(txt, u) - if "e2v" in zv.flags and zv.axs.uwrite: + if "e2v" in zv.flags: e2vs.append(zv.vpath or "/") t += "\n" diff --git a/copyparty/broker_util.py b/copyparty/broker_util.py index b0d44575..7645632b 100644 --- a/copyparty/broker_util.py +++ b/copyparty/broker_util.py @@ -42,7 +42,7 @@ class BrokerCli(object): """ def __init__(self) -> None: - self.log: RootLogger = None + self.log: "RootLogger" = None self.args: argparse.Namespace = None self.asrv: AuthSrv = None self.httpsrv: "HttpSrv" = None diff --git a/copyparty/fsutil.py b/copyparty/fsutil.py index 26a7911a..af8aa884 100644 --- a/copyparty/fsutil.py +++ b/copyparty/fsutil.py @@ -1,7 +1,11 @@ # coding: utf-8 from __future__ import print_function, unicode_literals -import ctypes +try: + import ctypes +except: + pass + import os import re import time @@ -19,7 +23,7 @@ except: class Fstab(object): - def __init__(self, log: RootLogger): + def __init__(self, log: "RootLogger"): self.log_func = log self.trusted = False @@ -136,7 +140,7 @@ class Fstab(object): def get_w32(self, path: str) -> str: # list mountpoints: fsutil fsinfo drives - + assert ctypes from ctypes.wintypes import BOOL, DWORD, LPCWSTR, LPDWORD, LPWSTR, MAX_PATH def echk(rc: int, fun: Any, args: Any) -> None: diff --git a/copyparty/httpcli.py b/copyparty/httpcli.py index 8dc27462..388ae939 100644 --- a/copyparty/httpcli.py +++ b/copyparty/httpcli.py @@ -24,12 +24,7 @@ try: except: pass -try: - import ctypes -except: - pass - -from .__init__ import ANYWIN, PY2, TYPE_CHECKING, WINDOWS, E, unicode +from .__init__ import ANYWIN, PY2, TYPE_CHECKING, E, unicode from .authsrv import VFS # typechk from .bos import bos from .star import StreamTar @@ -48,6 +43,7 @@ from .util import ( fsenc, gen_filekey, gencookie, + get_df, get_spd, guess_mime, gzip_orig_sz, @@ -1294,7 +1290,12 @@ class HttpCli(object): lim.chk_nup(self.ip) try: - max_sz = lim.smax if lim else 0 + max_sz = 0 + if lim: + v1 = lim.smax + v2 = lim.dfv - lim.dfl + max_sz = min(v1, v2) if v1 and v2 else v1 or v2 + with ren_open(tnam, "wb", 512 * 1024, **open_args) as zfw: f, tnam = zfw["orz"] tabspath = os.path.join(fdir, tnam) @@ -1309,6 +1310,7 @@ class HttpCli(object): lim.nup(self.ip) lim.bup(self.ip, sz) try: + lim.chk_df(tabspath, sz, True) lim.chk_sz(sz) lim.chk_bup(self.ip) lim.chk_nup(self.ip) @@ -2322,26 +2324,14 @@ class HttpCli(object): except: self.log("#wow #whoa") - try: - # some fuses misbehave - if not self.args.nid: - if WINDOWS: - try: - bfree = ctypes.c_ulonglong(0) - ctypes.windll.kernel32.GetDiskFreeSpaceExW( # type: ignore - ctypes.c_wchar_p(abspath), None, None, ctypes.pointer(bfree) - ) - srv_info.append(humansize(bfree.value) + " free") - except: - pass - else: - sv = os.statvfs(fsenc(abspath)) - free = humansize(sv.f_frsize * sv.f_bfree, True) - total = humansize(sv.f_frsize * sv.f_blocks, True) - - srv_info.append("{} free of {}".format(free, total)) - except: - pass + if not self.args.nid: + free, total = get_df(abspath) + if total is not None: + h1 = humansize(free or 0) + h2 = humansize(total) + srv_info.append("{} free of {}".format(h1, h2)) + elif free is not None: + srv_info.append(humansize(free, True) + " free") srv_infot = " // ".join(srv_info) diff --git a/copyparty/httpconn.py b/copyparty/httpconn.py index 06bbb539..7c6c168b 100644 --- a/copyparty/httpconn.py +++ b/copyparty/httpconn.py @@ -62,7 +62,7 @@ class HttpConn(object): self.nreq: int = 0 # mypy404 self.nbyte: int = 0 # mypy404 self.u2idx: Optional[U2idx] = None - self.log_func: Util.RootLogger = hsrv.log # mypy404 + self.log_func: "Util.RootLogger" = hsrv.log # mypy404 self.log_src: str = "httpconn" # mypy404 self.lf_url: Optional[Pattern[str]] = ( re.compile(self.args.lf_url) if self.args.lf_url else None diff --git a/copyparty/mtag.py b/copyparty/mtag.py index 0f09b1db..b790823f 100644 --- a/copyparty/mtag.py +++ b/copyparty/mtag.py @@ -248,7 +248,7 @@ def parse_ffprobe(txt: str) -> tuple[dict[str, tuple[int, Any]], dict[str, list[ class MTag(object): - def __init__(self, log_func: RootLogger, args: argparse.Namespace) -> None: + def __init__(self, log_func: "RootLogger", args: argparse.Namespace) -> None: self.log_func = log_func self.args = args self.usable = True diff --git a/copyparty/star.py b/copyparty/star.py index 21c2703c..43b04287 100644 --- a/copyparty/star.py +++ b/copyparty/star.py @@ -44,7 +44,7 @@ class StreamTar(StreamArc): def __init__( self, - log: NamedLogger, + log: "NamedLogger", fgen: Generator[dict[str, Any], None, None], **kwargs: Any ): diff --git a/copyparty/sutil.py b/copyparty/sutil.py index 506e389f..98f2a88d 100644 --- a/copyparty/sutil.py +++ b/copyparty/sutil.py @@ -17,7 +17,7 @@ except: class StreamArc(object): def __init__( self, - log: NamedLogger, + log: "NamedLogger", fgen: Generator[dict[str, Any], None, None], **kwargs: Any ): diff --git a/copyparty/szip.py b/copyparty/szip.py index 178f8a61..f6b25175 100644 --- a/copyparty/szip.py +++ b/copyparty/szip.py @@ -218,7 +218,7 @@ def gen_ecdr64_loc(ecdr64_pos: int) -> bytes: class StreamZip(StreamArc): def __init__( self, - log: NamedLogger, + log: "NamedLogger", fgen: Generator[dict[str, Any], None, None], utf8: bool = False, pre_crc: bool = False, diff --git a/copyparty/up2k.py b/copyparty/up2k.py index ccddcd19..6be3c7cb 100644 --- a/copyparty/up2k.py +++ b/copyparty/up2k.py @@ -196,7 +196,7 @@ class Up2k(object): def _block(self, why: str) -> None: self.blocked = why - self.log("uploads are temporarily blocked due to " + why, 3) + self.log("uploads temporarily blocked due to " + why, 3) def _unblock(self) -> None: self.blocked = None @@ -1657,7 +1657,7 @@ class Up2k(object): if vfs.lim: ap1 = os.path.join(cj["ptop"], cj["prel"]) ap2, cj["prel"] = vfs.lim.all( - cj["addr"], cj["prel"], cj["size"], ap1 + cj["addr"], cj["prel"], cj["size"], ap1, reg ) bos.makedirs(ap2) vfs.lim.nup(cj["addr"]) diff --git a/copyparty/util.py b/copyparty/util.py index 3e81cf30..55842fe7 100644 --- a/copyparty/util.py +++ b/copyparty/util.py @@ -24,6 +24,11 @@ from datetime import datetime from .__init__ import ANYWIN, PY2, TYPE_CHECKING, VT100, WINDOWS from .stolen import surrogateescape +try: + import ctypes +except: + pass + try: HAVE_SQLITE3 = True import sqlite3 # pylint: disable=unused-import # typechk @@ -243,7 +248,7 @@ class _Unrecv(object): undo any number of socket recv ops """ - def __init__(self, s: socket.socket, log: Optional[NamedLogger]) -> None: + def __init__(self, s: socket.socket, log: Optional["NamedLogger"]) -> None: self.s = s self.log = log self.buf: bytes = b"" @@ -287,7 +292,7 @@ class _LUnrecv(object): with expensive debug logging """ - def __init__(self, s: socket.socket, log: Optional[NamedLogger]) -> None: + def __init__(self, s: socket.socket, log: Optional["NamedLogger"]) -> None: self.s = s self.log = log self.buf = b"" @@ -662,7 +667,9 @@ def ren_open( class MultipartParser(object): - def __init__(self, log_func: NamedLogger, sr: Unrecv, http_headers: dict[str, str]): + def __init__( + self, log_func: "NamedLogger", sr: Unrecv, http_headers: dict[str, str] + ): self.sr = sr self.log = log_func self.headers = http_headers @@ -1207,6 +1214,24 @@ def atomic_move(usrc: str, udst: str) -> None: os.rename(src, dst) +def get_df(abspath: str) -> tuple[Optional[int], Optional[int]]: + try: + # some fuses misbehave + if ANYWIN: + bfree = ctypes.c_ulonglong(0) + ctypes.windll.kernel32.GetDiskFreeSpaceExW( # type: ignore + ctypes.c_wchar_p(abspath), None, None, ctypes.pointer(bfree) + ) + return (bfree.value, None) + else: + sv = os.statvfs(fsenc(abspath)) + free = sv.f_frsize * sv.f_bfree + total = sv.f_frsize * sv.f_blocks + return (free, total) + except: + return (None, None) + + def read_socket(sr: Unrecv, total_size: int) -> Generator[bytes, None, None]: remains = total_size while remains > 0: @@ -1233,7 +1258,7 @@ def read_socket_unbounded(sr: Unrecv) -> Generator[bytes, None, None]: def read_socket_chunked( - sr: Unrecv, log: Optional[NamedLogger] = None + sr: Unrecv, log: Optional["NamedLogger"] = None ) -> Generator[bytes, None, None]: err = "upload aborted: expected chunk length, got [{}] |{}| instead" while True: @@ -1311,7 +1336,7 @@ def hashcopy( def sendfile_py( - log: NamedLogger, + log: "NamedLogger", lower: int, upper: int, f: typing.BinaryIO, @@ -1339,7 +1364,7 @@ def sendfile_py( def sendfile_kern( - log: NamedLogger, + log: "NamedLogger", lower: int, upper: int, f: typing.BinaryIO, @@ -1380,7 +1405,7 @@ def sendfile_kern( def statdir( - logger: Optional[RootLogger], scandir: bool, lstat: bool, top: str + logger: Optional["RootLogger"], scandir: bool, lstat: bool, top: str ) -> Generator[tuple[str, os.stat_result], None, None]: if lstat and ANYWIN: lstat = False @@ -1423,7 +1448,7 @@ def statdir( def rmdirs( - logger: RootLogger, scandir: bool, lstat: bool, top: str, depth: int + logger: "RootLogger", scandir: bool, lstat: bool, top: str, depth: int ) -> tuple[list[str], list[str]]: """rmdir all descendants, then self""" if not os.path.isdir(fsenc(top)): @@ -1644,7 +1669,7 @@ def retchk( rc: int, cmd: Union[list[bytes], list[str]], serr: str, - logger: Optional[NamedLogger] = None, + logger: Optional["NamedLogger"] = None, color: Union[int, str] = 0, verbose: bool = False, ) -> None: diff --git a/copyparty/web/browser.js b/copyparty/web/browser.js index d79e0b98..5330ebb6 100644 --- a/copyparty/web/browser.js +++ b/copyparty/web/browser.js @@ -311,6 +311,7 @@ var Ls = { "u_ehsfin": "server rejected the request to finalize upload", "u_ehssrch": "server rejected the request to perform search", "u_ehsinit": "server rejected the request to initiate upload", + "u_ehsdf": "server ran out of disk space!\n\nwill keep retrying, in case someone\nfrees up enough space to continue", "u_s404": "not found on server", "u_expl": "explain", "u_tu": '

WARNING: turbo enabled,  client may not detect and resume incomplete uploads; see turbo-button tooltip

', @@ -642,6 +643,7 @@ var Ls = { "u_ehsfin": "server nektet forespørselen om å ferdigstille filen", "u_ehssrch": "server nektet forespørselen om å utføre søk", "u_ehsinit": "server nektet forespørselen om å begynne en ny opplastning", + "u_ehsdf": "serveren er full!\n\nprøver igjen regelmessig,\ni tilfelle noen rydder litt...", "u_s404": "ikke funnet på serveren", "u_expl": "forklar", "u_tu": '

ADVARSEL: turbo er på,  avbrutte opplastninger vil muligens ikke oppdages og gjenopptas; hold musepekeren over turbo-knappen for mer info

', diff --git a/copyparty/web/up2k.js b/copyparty/web/up2k.js index c6547d55..e72b6c7a 100644 --- a/copyparty/web/up2k.js +++ b/copyparty/web/up2k.js @@ -2011,6 +2011,9 @@ function up2k_init(subtle) { t.want_recheck = true; } } + if (rsp.indexOf('server HDD is full') + 1) + return toast.err(0, L.u_ehsdf + "\n\n" + rsp.replace(/.*; /, '')); + if (err != "") { pvis.seth(t.n, 1, "ERROR"); pvis.seth(t.n, 2, err); diff --git a/docs/notes.md b/docs/notes.md new file mode 100644 index 00000000..9f4a9733 --- /dev/null +++ b/docs/notes.md @@ -0,0 +1,10 @@ +# up2k.js + +## potato detection + +* tsk 0.25/8.4/31.5 bzw 1.27/22.9/18 = 77% (38.4s, 49.7s) + * 4c locale #1313, ff-102,deb-11 @ ryzen4500u wifi -> win10 + * profiling shows 2sec heavy gc every 2sec + +* tsk 0.41/4.1/10 bzw 1.41/9.9/7 = 73% (13.3s, 18.2s) + * 4c locale #1313, ch-103,deb-11 @ ryzen4500u wifi -> win10