ensure free disk space

README.md

@@ -695,6 +695,7 @@ if you set `--no-hash [...]` globally, you can enable hashing for specific volum
 set upload rules using volume flags,  some examples:
 
 * `:c,sz=1k-3m` sets allowed filesize between 1 KiB and 3 MiB inclusive (suffixes: `b`, `k`, `m`, `g`)
+* `:c,df=4g` block uploads if there would be less than 4 GiB free disk space afterwards
 * `:c,nosub` disallow uploading into subdirectories; goes well with `rotn` and `rotf`:
 * `:c,rotn=1000,2` moves uploads into subfolders, up to 1000 files in each folder before making a new one, two levels deep (must be at least 1)
 * `:c,rotf=%Y/%m/%d/%H` enforces files to be uploaded into a structure of subfolders according to that date format

copyparty/__main__.py

@@ -382,6 +382,7 @@ def run_argparse(argv: list[str], formatter: Any, retry: bool) -> argparse.Names
               \033[36mmaxn=250,600\033[35m max 250 uploads over 15min
               \033[36mmaxb=1g,300\033[35m max 1 GiB over 5min (suffixes: b, k, m, g)
               \033[36msz=1k-3m\033[35m allow filesizes between 1 KiB and 3MiB
+              \033[36mdf=1g\033[35m ensure 1 GiB free disk space
 
             \033[0mupload rotation:
             (moves all uploads into the specified folder structure)
@@ -482,6 +483,7 @@ def run_argparse(argv: list[str], formatter: Any, retry: bool) -> argparse.Names
     ap2.add_argument("--hardlink", action="store_true", help="prefer hardlinks instead of symlinks when possible (within same filesystem)")
     ap2.add_argument("--never-symlink", action="store_true", help="do not fallback to symlinks when a hardlink cannot be made")
     ap2.add_argument("--no-dedup", action="store_true", help="disable symlink/hardlink creation; copy file contents instead")
+    ap2.add_argument("--df", metavar="GiB", type=float, default=0, help="ensure GiB free disk space by rejecting upload requests")
     ap2.add_argument("--sparse", metavar="MiB", type=int, default=4, help="windows-only: minimum size of incoming uploads through up2k before they are made into sparse files")
     ap2.add_argument("--turbo", metavar="LVL", type=int, default=0, help="configure turbo-mode in up2k client; 0 = off and warn if enabled, 1 = off, 2 = on, 3 = on and disable datecheck")
     ap2.add_argument("--u2sort", metavar="TXT", type=u, default="s", help="upload order; s=smallest-first, n=alphabetical, fs=force-s, fn=force-n -- alphabetical is a bit slower on fiber/LAN but makes it easier to eyeball if everything went fine")

copyparty/authsrv.py

@@ -20,6 +20,8 @@ from .util import (
     Pebkac,
     absreal,
     fsenc,
+    get_df,
+    humansize,
     relchk,
     statdir,
     uncyg,
@@ -72,15 +74,23 @@ class AXS(object):
 
 
 class Lim(object):
-    def __init__(self) -> None:
+    def __init__(self, log_func: Optional["RootLogger"]) -> None:
+        self.log_func = log_func
+
+        self.reg: Optional[dict[str, dict[str, Any]]] = None  # up2k registry
+
         self.nups: dict[str, list[float]] = {}  # num tracker
         self.bups: dict[str, list[tuple[float, int]]] = {}  # byte tracker list
         self.bupc: dict[str, int] = {}  # byte tracker cache
 
         self.nosub = False  # disallow subdirectories
 
-        self.smin = -1  # filesize min
+        self.dfl = 0  # free disk space limit
-        self.smax = -1  # filesize max
+        self.dft = 0  # last-measured time
+        self.dfv: Optional[int] = 0  # currently free
+
+        self.smin = 0  # filesize min
+        self.smax = 0  # filesize max
 
         self.bwin = 0  # bytes window
         self.bmax = 0  # bytes max
@@ -92,18 +102,34 @@ class Lim(object):
         self.rotf = ""  # rot datefmt
         self.rot_re = re.compile("")  # rotf check
 
+    def log(self, msg: str, c: Union[int, str] = 0) -> None:
+        if self.log_func:
+            self.log_func("up-lim", msg, c)
+
     def set_rotf(self, fmt: str) -> None:
         self.rotf = fmt
         r = re.escape(fmt).replace("%Y", "[0-9]{4}").replace("%j", "[0-9]{3}")
         r = re.sub("%[mdHMSWU]", "[0-9]{2}", r)
         self.rot_re = re.compile("(^|/)" + r + "$")
 
-    def all(self, ip: str, rem: str, sz: float, abspath: str) -> tuple[str, str]:
+    def all(
+        self,
+        ip: str,
+        rem: str,
+        sz: int,
+        abspath: str,
+        reg: Optional[dict[str, dict[str, Any]]] = None,
+    ) -> tuple[str, str]:
+        if reg is not None and self.reg is None:
+            self.reg = reg
+            self.dft = 0
+
         self.chk_nup(ip)
         self.chk_bup(ip)
         self.chk_rem(rem)
         if sz != -1:
             self.chk_sz(sz)
+            self.chk_df(abspath, sz)  # side effects; keep last-ish
 
         ap2, vp2 = self.rot(abspath)
         if abspath == ap2:
@@ -111,13 +137,33 @@ class Lim(object):
 
         return ap2, ("{}/{}".format(rem, vp2) if rem else vp2)
 
-    def chk_sz(self, sz: float) -> None:
+    def chk_sz(self, sz: int) -> None:
-        if self.smin != -1 and sz < self.smin:
+        if sz < self.smin:
             raise Pebkac(400, "file too small")
 
-        if self.smax != -1 and sz > self.smax:
+        if self.smax and sz > self.smax:
             raise Pebkac(400, "file too big")
 
+    def chk_df(self, abspath: str, sz: int, already_written: bool = False) -> None:
+        if not self.dfl:
+            return
+
+        if self.dft < time.time():
+            self.dft = int(time.time()) + 300
+            self.dfv = get_df(abspath)[0]
+            for j in list(self.reg.values()) if self.reg else []:
+                self.dfv -= int(j["size"] / len(j["hash"]) * len(j["need"]))
+
+            if already_written:
+                sz = 0
+
+        if self.dfv - sz < self.dfl:
+            self.dft = min(self.dft, int(time.time()) + 10)
+            t = "server HDD is full; {} free, need {}"
+            raise Pebkac(500, t.format(humansize(self.dfv - self.dfl), humansize(sz)))
+
+        self.dfv -= int(sz)
+
     def chk_rem(self, rem: str) -> None:
         if self.nosub and rem:
             raise Pebkac(500, "no subdirectories allowed")
@@ -226,7 +272,7 @@ class VFS(object):
 
     def __init__(
         self,
-        log: Optional[RootLogger],
+        log: Optional["RootLogger"],
         realpath: str,
         vpath: str,
         axs: AXS,
@@ -569,7 +615,7 @@ class AuthSrv(object):
     def __init__(
         self,
         args: argparse.Namespace,
-        log_func: Optional[RootLogger],
+        log_func: Optional["RootLogger"],
         warn_anonwrite: bool = True,
     ) -> None:
         self.args = args
@@ -917,13 +963,20 @@ class AuthSrv(object):
         vfs.histtab = {zv.realpath: zv.histpath for zv in vfs.all_vols.values()}
 
         for vol in vfs.all_vols.values():
-            lim = Lim()
+            lim = Lim(self.log_func)
             use = False
 
             if vol.flags.get("nosub"):
                 use = True
                 lim.nosub = True
 
+            zs = vol.flags.get("df") or (
+                "{}g".format(self.args.df) if self.args.df else ""
+            )
+            if zs:
+                use = True
+                lim.dfl = unhumanize(zs)
+
             zs = vol.flags.get("sz")
             if zs:
                 use = True
@@ -1126,7 +1179,7 @@ class AuthSrv(object):
                 u = u if u else "\033[36m--none--\033[0m"
                 t += "\n|  {}:  {}".format(txt, u)
 
-            if "e2v" in zv.flags and zv.axs.uwrite:
+            if "e2v" in zv.flags:
                 e2vs.append(zv.vpath or "/")
 
             t += "\n"

copyparty/broker_util.py

@@ -42,7 +42,7 @@ class BrokerCli(object):
     """
 
     def __init__(self) -> None:
-        self.log: RootLogger = None
+        self.log: "RootLogger" = None
         self.args: argparse.Namespace = None
         self.asrv: AuthSrv = None
         self.httpsrv: "HttpSrv" = None

copyparty/fsutil.py

@@ -1,7 +1,11 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
-import ctypes
+try:
+    import ctypes
+except:
+    pass
+
 import os
 import re
 import time
@@ -19,7 +23,7 @@ except:
 
 
 class Fstab(object):
-    def __init__(self, log: RootLogger):
+    def __init__(self, log: "RootLogger"):
         self.log_func = log
 
         self.trusted = False
@@ -136,7 +140,7 @@ class Fstab(object):
 
     def get_w32(self, path: str) -> str:
         # list mountpoints: fsutil fsinfo drives
-
+        assert ctypes
         from ctypes.wintypes import BOOL, DWORD, LPCWSTR, LPDWORD, LPWSTR, MAX_PATH
 
         def echk(rc: int, fun: Any, args: Any) -> None:

copyparty/httpcli.py

@@ -24,12 +24,7 @@ try:
 except:
     pass
 
-try:
+from .__init__ import ANYWIN, PY2, TYPE_CHECKING, E, unicode
-    import ctypes
-except:
-    pass
-
-from .__init__ import ANYWIN, PY2, TYPE_CHECKING, WINDOWS, E, unicode
 from .authsrv import VFS  # typechk
 from .bos import bos
 from .star import StreamTar
@@ -48,6 +43,7 @@ from .util import (
     fsenc,
     gen_filekey,
     gencookie,
+    get_df,
     get_spd,
     guess_mime,
     gzip_orig_sz,
@@ -1294,7 +1290,12 @@ class HttpCli(object):
                     lim.chk_nup(self.ip)
 
                 try:
-                    max_sz = lim.smax if lim else 0
+                    max_sz = 0
+                    if lim:
+                        v1 = lim.smax
+                        v2 = lim.dfv - lim.dfl
+                        max_sz = min(v1, v2) if v1 and v2 else v1 or v2
+
                     with ren_open(tnam, "wb", 512 * 1024, **open_args) as zfw:
                         f, tnam = zfw["orz"]
                         tabspath = os.path.join(fdir, tnam)
@@ -1309,6 +1310,7 @@ class HttpCli(object):
                         lim.nup(self.ip)
                         lim.bup(self.ip, sz)
                         try:
+                            lim.chk_df(tabspath, sz, True)
                             lim.chk_sz(sz)
                             lim.chk_bup(self.ip)
                             lim.chk_nup(self.ip)
@@ -2322,26 +2324,14 @@ class HttpCli(object):
         except:
             self.log("#wow #whoa")
 
-        try:
+        if not self.args.nid:
-            # some fuses misbehave
+            free, total = get_df(abspath)
-            if not self.args.nid:
+            if total is not None:
-                if WINDOWS:
+                h1 = humansize(free or 0)
-                    try:
+                h2 = humansize(total)
-                        bfree = ctypes.c_ulonglong(0)
+                srv_info.append("{} free of {}".format(h1, h2))
-                        ctypes.windll.kernel32.GetDiskFreeSpaceExW(  # type: ignore
+            elif free is not None:
-                            ctypes.c_wchar_p(abspath), None, None, ctypes.pointer(bfree)
+                srv_info.append(humansize(free, True) + " free")
-                        )
-                        srv_info.append(humansize(bfree.value) + " free")
-                    except:
-                        pass
-                else:
-                    sv = os.statvfs(fsenc(abspath))
-                    free = humansize(sv.f_frsize * sv.f_bfree, True)
-                    total = humansize(sv.f_frsize * sv.f_blocks, True)
-
-                    srv_info.append("{} free of {}".format(free, total))
-        except:
-            pass
 
         srv_infot = "</span> // <span>".join(srv_info)
 

copyparty/httpconn.py

@@ -62,7 +62,7 @@ class HttpConn(object):
         self.nreq: int = 0  # mypy404
         self.nbyte: int = 0  # mypy404
         self.u2idx: Optional[U2idx] = None
-        self.log_func: Util.RootLogger = hsrv.log  # mypy404
+        self.log_func: "Util.RootLogger" = hsrv.log  # mypy404
         self.log_src: str = "httpconn"  # mypy404
         self.lf_url: Optional[Pattern[str]] = (
             re.compile(self.args.lf_url) if self.args.lf_url else None

copyparty/mtag.py

@@ -248,7 +248,7 @@ def parse_ffprobe(txt: str) -> tuple[dict[str, tuple[int, Any]], dict[str, list[
 
 
 class MTag(object):
-    def __init__(self, log_func: RootLogger, args: argparse.Namespace) -> None:
+    def __init__(self, log_func: "RootLogger", args: argparse.Namespace) -> None:
         self.log_func = log_func
         self.args = args
         self.usable = True

copyparty/star.py

@@ -44,7 +44,7 @@ class StreamTar(StreamArc):
 
     def __init__(
         self,
-        log: NamedLogger,
+        log: "NamedLogger",
         fgen: Generator[dict[str, Any], None, None],
         **kwargs: Any
     ):

copyparty/sutil.py

@@ -17,7 +17,7 @@ except:
 class StreamArc(object):
     def __init__(
         self,
-        log: NamedLogger,
+        log: "NamedLogger",
         fgen: Generator[dict[str, Any], None, None],
         **kwargs: Any
     ):

copyparty/szip.py

@@ -218,7 +218,7 @@ def gen_ecdr64_loc(ecdr64_pos: int) -> bytes:
 class StreamZip(StreamArc):
     def __init__(
         self,
-        log: NamedLogger,
+        log: "NamedLogger",
         fgen: Generator[dict[str, Any], None, None],
         utf8: bool = False,
         pre_crc: bool = False,

copyparty/up2k.py

@@ -196,7 +196,7 @@ class Up2k(object):
 
     def _block(self, why: str) -> None:
         self.blocked = why
-        self.log("uploads are temporarily blocked due to " + why, 3)
+        self.log("uploads temporarily blocked due to " + why, 3)
 
     def _unblock(self) -> None:
         self.blocked = None
@@ -1657,7 +1657,7 @@ class Up2k(object):
                 if vfs.lim:
                     ap1 = os.path.join(cj["ptop"], cj["prel"])
                     ap2, cj["prel"] = vfs.lim.all(
-                        cj["addr"], cj["prel"], cj["size"], ap1
+                        cj["addr"], cj["prel"], cj["size"], ap1, reg
                     )
                     bos.makedirs(ap2)
                     vfs.lim.nup(cj["addr"])

copyparty/util.py

@@ -24,6 +24,11 @@ from datetime import datetime
 from .__init__ import ANYWIN, PY2, TYPE_CHECKING, VT100, WINDOWS
 from .stolen import surrogateescape
 
+try:
+    import ctypes
+except:
+    pass
+
 try:
     HAVE_SQLITE3 = True
     import sqlite3  # pylint: disable=unused-import  # typechk
@@ -243,7 +248,7 @@ class _Unrecv(object):
     undo any number of socket recv ops
     """
 
-    def __init__(self, s: socket.socket, log: Optional[NamedLogger]) -> None:
+    def __init__(self, s: socket.socket, log: Optional["NamedLogger"]) -> None:
         self.s = s
         self.log = log
         self.buf: bytes = b""
@@ -287,7 +292,7 @@ class _LUnrecv(object):
     with expensive debug logging
     """
 
-    def __init__(self, s: socket.socket, log: Optional[NamedLogger]) -> None:
+    def __init__(self, s: socket.socket, log: Optional["NamedLogger"]) -> None:
         self.s = s
         self.log = log
         self.buf = b""
@@ -662,7 +667,9 @@ def ren_open(
 
 
 class MultipartParser(object):
-    def __init__(self, log_func: NamedLogger, sr: Unrecv, http_headers: dict[str, str]):
+    def __init__(
+        self, log_func: "NamedLogger", sr: Unrecv, http_headers: dict[str, str]
+    ):
         self.sr = sr
         self.log = log_func
         self.headers = http_headers
@@ -1207,6 +1214,24 @@ def atomic_move(usrc: str, udst: str) -> None:
         os.rename(src, dst)
 
 
+def get_df(abspath: str) -> tuple[Optional[int], Optional[int]]:
+    try:
+        # some fuses misbehave
+        if ANYWIN:
+            bfree = ctypes.c_ulonglong(0)
+            ctypes.windll.kernel32.GetDiskFreeSpaceExW(  # type: ignore
+                ctypes.c_wchar_p(abspath), None, None, ctypes.pointer(bfree)
+            )
+            return (bfree.value, None)
+        else:
+            sv = os.statvfs(fsenc(abspath))
+            free = sv.f_frsize * sv.f_bfree
+            total = sv.f_frsize * sv.f_blocks
+            return (free, total)
+    except:
+        return (None, None)
+
+
 def read_socket(sr: Unrecv, total_size: int) -> Generator[bytes, None, None]:
     remains = total_size
     while remains > 0:
@@ -1233,7 +1258,7 @@ def read_socket_unbounded(sr: Unrecv) -> Generator[bytes, None, None]:
 
 
 def read_socket_chunked(
-    sr: Unrecv, log: Optional[NamedLogger] = None
+    sr: Unrecv, log: Optional["NamedLogger"] = None
 ) -> Generator[bytes, None, None]:
     err = "upload aborted: expected chunk length, got [{}] |{}| instead"
     while True:
@@ -1311,7 +1336,7 @@ def hashcopy(
 
 
 def sendfile_py(
-    log: NamedLogger,
+    log: "NamedLogger",
     lower: int,
     upper: int,
     f: typing.BinaryIO,
@@ -1339,7 +1364,7 @@ def sendfile_py(
 
 
 def sendfile_kern(
-    log: NamedLogger,
+    log: "NamedLogger",
     lower: int,
     upper: int,
     f: typing.BinaryIO,
@@ -1380,7 +1405,7 @@ def sendfile_kern(
 
 
 def statdir(
-    logger: Optional[RootLogger], scandir: bool, lstat: bool, top: str
+    logger: Optional["RootLogger"], scandir: bool, lstat: bool, top: str
 ) -> Generator[tuple[str, os.stat_result], None, None]:
     if lstat and ANYWIN:
         lstat = False
@@ -1423,7 +1448,7 @@ def statdir(
 
 
 def rmdirs(
-    logger: RootLogger, scandir: bool, lstat: bool, top: str, depth: int
+    logger: "RootLogger", scandir: bool, lstat: bool, top: str, depth: int
 ) -> tuple[list[str], list[str]]:
     """rmdir all descendants, then self"""
     if not os.path.isdir(fsenc(top)):
@@ -1644,7 +1669,7 @@ def retchk(
     rc: int,
     cmd: Union[list[bytes], list[str]],
     serr: str,
-    logger: Optional[NamedLogger] = None,
+    logger: Optional["NamedLogger"] = None,
     color: Union[int, str] = 0,
     verbose: bool = False,
 ) -> None:

copyparty/web/browser.js

@@ -311,6 +311,7 @@ var Ls = {
 		"u_ehsfin": "server rejected the request to finalize upload",
 		"u_ehssrch": "server rejected the request to perform search",
 		"u_ehsinit": "server rejected the request to initiate upload",
+		"u_ehsdf": "server ran out of disk space!\n\nwill keep retrying, in case someone\nfrees up enough space to continue",
 		"u_s404": "not found on server",
 		"u_expl": "explain",
 		"u_tu": '<p class="warn">WARNING: turbo enabled, <span>&nbsp;client may not detect and resume incomplete uploads; see turbo-button tooltip</span></p>',
@@ -642,6 +643,7 @@ var Ls = {
 		"u_ehsfin": "server nektet forespørselen om å ferdigstille filen",
 		"u_ehssrch": "server nektet forespørselen om å utføre søk",
 		"u_ehsinit": "server nektet forespørselen om å begynne en ny opplastning",
+		"u_ehsdf": "serveren er full!\n\nprøver igjen regelmessig,\ni tilfelle noen rydder litt...",
 		"u_s404": "ikke funnet på serveren",
 		"u_expl": "forklar",
 		"u_tu": '<p class="warn">ADVARSEL: turbo er på, <span>&nbsp;avbrutte opplastninger vil muligens ikke oppdages og gjenopptas; hold musepekeren over turbo-knappen for mer info</span></p>',

copyparty/web/up2k.js

@@ -2011,6 +2011,9 @@ function up2k_init(subtle) {
                         t.want_recheck = true;
                     }
                 }
+                if (rsp.indexOf('server HDD is full') + 1)
+                    return toast.err(0, L.u_ehsdf + "\n\n" + rsp.replace(/.*; /, ''));
+
                 if (err != "") {
                     pvis.seth(t.n, 1, "ERROR");
                     pvis.seth(t.n, 2, err);

docs/notes.md

@@ -0,0 +1,10 @@
+# up2k.js
+
+## potato detection
+
+* tsk 0.25/8.4/31.5 bzw 1.27/22.9/18 = 77% (38.4s, 49.7s)
+  * 4c locale #1313, ff-102,deb-11 @ ryzen4500u wifi -> win10
+  * profiling shows 2sec heavy gc every 2sec
+
+* tsk 0.41/4.1/10 bzw 1.41/9.9/7 = 73% (13.3s, 18.2s)
+  * 4c locale #1313, ch-103,deb-11 @ ryzen4500u wifi -> win10