ensure free disk space

This commit is contained in:
ed 2022-07-17 22:33:08 +02:00
parent cebda5028a
commit 4a76663fb2
16 changed files with 148 additions and 58 deletions

View file

@ -695,6 +695,7 @@ if you set `--no-hash [...]` globally, you can enable hashing for specific volum
set upload rules using volume flags, some examples:
* `:c,sz=1k-3m` sets allowed filesize between 1 KiB and 3 MiB inclusive (suffixes: `b`, `k`, `m`, `g`)
* `:c,df=4g` block uploads if there would be less than 4 GiB free disk space afterwards
* `:c,nosub` disallow uploading into subdirectories; goes well with `rotn` and `rotf`:
* `:c,rotn=1000,2` moves uploads into subfolders, up to 1000 files in each folder before making a new one, two levels deep (must be at least 1)
* `:c,rotf=%Y/%m/%d/%H` enforces files to be uploaded into a structure of subfolders according to that date format

View file

@ -382,6 +382,7 @@ def run_argparse(argv: list[str], formatter: Any, retry: bool) -> argparse.Names
\033[36mmaxn=250,600\033[35m max 250 uploads over 15min
\033[36mmaxb=1g,300\033[35m max 1 GiB over 5min (suffixes: b, k, m, g)
\033[36msz=1k-3m\033[35m allow filesizes between 1 KiB and 3MiB
\033[36mdf=1g\033[35m ensure 1 GiB free disk space
\033[0mupload rotation:
(moves all uploads into the specified folder structure)
@ -482,6 +483,7 @@ def run_argparse(argv: list[str], formatter: Any, retry: bool) -> argparse.Names
ap2.add_argument("--hardlink", action="store_true", help="prefer hardlinks instead of symlinks when possible (within same filesystem)")
ap2.add_argument("--never-symlink", action="store_true", help="do not fallback to symlinks when a hardlink cannot be made")
ap2.add_argument("--no-dedup", action="store_true", help="disable symlink/hardlink creation; copy file contents instead")
ap2.add_argument("--df", metavar="GiB", type=float, default=0, help="ensure GiB free disk space by rejecting upload requests")
ap2.add_argument("--sparse", metavar="MiB", type=int, default=4, help="windows-only: minimum size of incoming uploads through up2k before they are made into sparse files")
ap2.add_argument("--turbo", metavar="LVL", type=int, default=0, help="configure turbo-mode in up2k client; 0 = off and warn if enabled, 1 = off, 2 = on, 3 = on and disable datecheck")
ap2.add_argument("--u2sort", metavar="TXT", type=u, default="s", help="upload order; s=smallest-first, n=alphabetical, fs=force-s, fn=force-n -- alphabetical is a bit slower on fiber/LAN but makes it easier to eyeball if everything went fine")

View file

@ -20,6 +20,8 @@ from .util import (
Pebkac,
absreal,
fsenc,
get_df,
humansize,
relchk,
statdir,
uncyg,
@ -72,15 +74,23 @@ class AXS(object):
class Lim(object):
def __init__(self) -> None:
def __init__(self, log_func: Optional["RootLogger"]) -> None:
self.log_func = log_func
self.reg: Optional[dict[str, dict[str, Any]]] = None # up2k registry
self.nups: dict[str, list[float]] = {} # num tracker
self.bups: dict[str, list[tuple[float, int]]] = {} # byte tracker list
self.bupc: dict[str, int] = {} # byte tracker cache
self.nosub = False # disallow subdirectories
self.smin = -1 # filesize min
self.smax = -1 # filesize max
self.dfl = 0 # free disk space limit
self.dft = 0 # last-measured time
self.dfv: Optional[int] = 0 # currently free
self.smin = 0 # filesize min
self.smax = 0 # filesize max
self.bwin = 0 # bytes window
self.bmax = 0 # bytes max
@ -92,18 +102,34 @@ class Lim(object):
self.rotf = "" # rot datefmt
self.rot_re = re.compile("") # rotf check
def log(self, msg: str, c: Union[int, str] = 0) -> None:
if self.log_func:
self.log_func("up-lim", msg, c)
def set_rotf(self, fmt: str) -> None:
self.rotf = fmt
r = re.escape(fmt).replace("%Y", "[0-9]{4}").replace("%j", "[0-9]{3}")
r = re.sub("%[mdHMSWU]", "[0-9]{2}", r)
self.rot_re = re.compile("(^|/)" + r + "$")
def all(self, ip: str, rem: str, sz: float, abspath: str) -> tuple[str, str]:
def all(
self,
ip: str,
rem: str,
sz: int,
abspath: str,
reg: Optional[dict[str, dict[str, Any]]] = None,
) -> tuple[str, str]:
if reg is not None and self.reg is None:
self.reg = reg
self.dft = 0
self.chk_nup(ip)
self.chk_bup(ip)
self.chk_rem(rem)
if sz != -1:
self.chk_sz(sz)
self.chk_df(abspath, sz) # side effects; keep last-ish
ap2, vp2 = self.rot(abspath)
if abspath == ap2:
@ -111,13 +137,33 @@ class Lim(object):
return ap2, ("{}/{}".format(rem, vp2) if rem else vp2)
def chk_sz(self, sz: float) -> None:
if self.smin != -1 and sz < self.smin:
def chk_sz(self, sz: int) -> None:
if sz < self.smin:
raise Pebkac(400, "file too small")
if self.smax != -1 and sz > self.smax:
if self.smax and sz > self.smax:
raise Pebkac(400, "file too big")
def chk_df(self, abspath: str, sz: int, already_written: bool = False) -> None:
if not self.dfl:
return
if self.dft < time.time():
self.dft = int(time.time()) + 300
self.dfv = get_df(abspath)[0]
for j in list(self.reg.values()) if self.reg else []:
self.dfv -= int(j["size"] / len(j["hash"]) * len(j["need"]))
if already_written:
sz = 0
if self.dfv - sz < self.dfl:
self.dft = min(self.dft, int(time.time()) + 10)
t = "server HDD is full; {} free, need {}"
raise Pebkac(500, t.format(humansize(self.dfv - self.dfl), humansize(sz)))
self.dfv -= int(sz)
def chk_rem(self, rem: str) -> None:
if self.nosub and rem:
raise Pebkac(500, "no subdirectories allowed")
@ -226,7 +272,7 @@ class VFS(object):
def __init__(
self,
log: Optional[RootLogger],
log: Optional["RootLogger"],
realpath: str,
vpath: str,
axs: AXS,
@ -569,7 +615,7 @@ class AuthSrv(object):
def __init__(
self,
args: argparse.Namespace,
log_func: Optional[RootLogger],
log_func: Optional["RootLogger"],
warn_anonwrite: bool = True,
) -> None:
self.args = args
@ -917,13 +963,20 @@ class AuthSrv(object):
vfs.histtab = {zv.realpath: zv.histpath for zv in vfs.all_vols.values()}
for vol in vfs.all_vols.values():
lim = Lim()
lim = Lim(self.log_func)
use = False
if vol.flags.get("nosub"):
use = True
lim.nosub = True
zs = vol.flags.get("df") or (
"{}g".format(self.args.df) if self.args.df else ""
)
if zs:
use = True
lim.dfl = unhumanize(zs)
zs = vol.flags.get("sz")
if zs:
use = True
@ -1126,7 +1179,7 @@ class AuthSrv(object):
u = u if u else "\033[36m--none--\033[0m"
t += "\n| {}: {}".format(txt, u)
if "e2v" in zv.flags and zv.axs.uwrite:
if "e2v" in zv.flags:
e2vs.append(zv.vpath or "/")
t += "\n"

View file

@ -42,7 +42,7 @@ class BrokerCli(object):
"""
def __init__(self) -> None:
self.log: RootLogger = None
self.log: "RootLogger" = None
self.args: argparse.Namespace = None
self.asrv: AuthSrv = None
self.httpsrv: "HttpSrv" = None

View file

@ -1,7 +1,11 @@
# coding: utf-8
from __future__ import print_function, unicode_literals
import ctypes
try:
import ctypes
except:
pass
import os
import re
import time
@ -19,7 +23,7 @@ except:
class Fstab(object):
def __init__(self, log: RootLogger):
def __init__(self, log: "RootLogger"):
self.log_func = log
self.trusted = False
@ -136,7 +140,7 @@ class Fstab(object):
def get_w32(self, path: str) -> str:
# list mountpoints: fsutil fsinfo drives
assert ctypes
from ctypes.wintypes import BOOL, DWORD, LPCWSTR, LPDWORD, LPWSTR, MAX_PATH
def echk(rc: int, fun: Any, args: Any) -> None:

View file

@ -24,12 +24,7 @@ try:
except:
pass
try:
import ctypes
except:
pass
from .__init__ import ANYWIN, PY2, TYPE_CHECKING, WINDOWS, E, unicode
from .__init__ import ANYWIN, PY2, TYPE_CHECKING, E, unicode
from .authsrv import VFS # typechk
from .bos import bos
from .star import StreamTar
@ -48,6 +43,7 @@ from .util import (
fsenc,
gen_filekey,
gencookie,
get_df,
get_spd,
guess_mime,
gzip_orig_sz,
@ -1294,7 +1290,12 @@ class HttpCli(object):
lim.chk_nup(self.ip)
try:
max_sz = lim.smax if lim else 0
max_sz = 0
if lim:
v1 = lim.smax
v2 = lim.dfv - lim.dfl
max_sz = min(v1, v2) if v1 and v2 else v1 or v2
with ren_open(tnam, "wb", 512 * 1024, **open_args) as zfw:
f, tnam = zfw["orz"]
tabspath = os.path.join(fdir, tnam)
@ -1309,6 +1310,7 @@ class HttpCli(object):
lim.nup(self.ip)
lim.bup(self.ip, sz)
try:
lim.chk_df(tabspath, sz, True)
lim.chk_sz(sz)
lim.chk_bup(self.ip)
lim.chk_nup(self.ip)
@ -2322,26 +2324,14 @@ class HttpCli(object):
except:
self.log("#wow #whoa")
try:
# some fuses misbehave
if not self.args.nid:
if WINDOWS:
try:
bfree = ctypes.c_ulonglong(0)
ctypes.windll.kernel32.GetDiskFreeSpaceExW( # type: ignore
ctypes.c_wchar_p(abspath), None, None, ctypes.pointer(bfree)
)
srv_info.append(humansize(bfree.value) + " free")
except:
pass
else:
sv = os.statvfs(fsenc(abspath))
free = humansize(sv.f_frsize * sv.f_bfree, True)
total = humansize(sv.f_frsize * sv.f_blocks, True)
srv_info.append("{} free of {}".format(free, total))
except:
pass
if not self.args.nid:
free, total = get_df(abspath)
if total is not None:
h1 = humansize(free or 0)
h2 = humansize(total)
srv_info.append("{} free of {}".format(h1, h2))
elif free is not None:
srv_info.append(humansize(free, True) + " free")
srv_infot = "</span> // <span>".join(srv_info)

View file

@ -62,7 +62,7 @@ class HttpConn(object):
self.nreq: int = 0 # mypy404
self.nbyte: int = 0 # mypy404
self.u2idx: Optional[U2idx] = None
self.log_func: Util.RootLogger = hsrv.log # mypy404
self.log_func: "Util.RootLogger" = hsrv.log # mypy404
self.log_src: str = "httpconn" # mypy404
self.lf_url: Optional[Pattern[str]] = (
re.compile(self.args.lf_url) if self.args.lf_url else None

View file

@ -248,7 +248,7 @@ def parse_ffprobe(txt: str) -> tuple[dict[str, tuple[int, Any]], dict[str, list[
class MTag(object):
def __init__(self, log_func: RootLogger, args: argparse.Namespace) -> None:
def __init__(self, log_func: "RootLogger", args: argparse.Namespace) -> None:
self.log_func = log_func
self.args = args
self.usable = True

View file

@ -44,7 +44,7 @@ class StreamTar(StreamArc):
def __init__(
self,
log: NamedLogger,
log: "NamedLogger",
fgen: Generator[dict[str, Any], None, None],
**kwargs: Any
):

View file

@ -17,7 +17,7 @@ except:
class StreamArc(object):
def __init__(
self,
log: NamedLogger,
log: "NamedLogger",
fgen: Generator[dict[str, Any], None, None],
**kwargs: Any
):

View file

@ -218,7 +218,7 @@ def gen_ecdr64_loc(ecdr64_pos: int) -> bytes:
class StreamZip(StreamArc):
def __init__(
self,
log: NamedLogger,
log: "NamedLogger",
fgen: Generator[dict[str, Any], None, None],
utf8: bool = False,
pre_crc: bool = False,

View file

@ -196,7 +196,7 @@ class Up2k(object):
def _block(self, why: str) -> None:
self.blocked = why
self.log("uploads are temporarily blocked due to " + why, 3)
self.log("uploads temporarily blocked due to " + why, 3)
def _unblock(self) -> None:
self.blocked = None
@ -1657,7 +1657,7 @@ class Up2k(object):
if vfs.lim:
ap1 = os.path.join(cj["ptop"], cj["prel"])
ap2, cj["prel"] = vfs.lim.all(
cj["addr"], cj["prel"], cj["size"], ap1
cj["addr"], cj["prel"], cj["size"], ap1, reg
)
bos.makedirs(ap2)
vfs.lim.nup(cj["addr"])

View file

@ -24,6 +24,11 @@ from datetime import datetime
from .__init__ import ANYWIN, PY2, TYPE_CHECKING, VT100, WINDOWS
from .stolen import surrogateescape
try:
import ctypes
except:
pass
try:
HAVE_SQLITE3 = True
import sqlite3 # pylint: disable=unused-import # typechk
@ -243,7 +248,7 @@ class _Unrecv(object):
undo any number of socket recv ops
"""
def __init__(self, s: socket.socket, log: Optional[NamedLogger]) -> None:
def __init__(self, s: socket.socket, log: Optional["NamedLogger"]) -> None:
self.s = s
self.log = log
self.buf: bytes = b""
@ -287,7 +292,7 @@ class _LUnrecv(object):
with expensive debug logging
"""
def __init__(self, s: socket.socket, log: Optional[NamedLogger]) -> None:
def __init__(self, s: socket.socket, log: Optional["NamedLogger"]) -> None:
self.s = s
self.log = log
self.buf = b""
@ -662,7 +667,9 @@ def ren_open(
class MultipartParser(object):
def __init__(self, log_func: NamedLogger, sr: Unrecv, http_headers: dict[str, str]):
def __init__(
self, log_func: "NamedLogger", sr: Unrecv, http_headers: dict[str, str]
):
self.sr = sr
self.log = log_func
self.headers = http_headers
@ -1207,6 +1214,24 @@ def atomic_move(usrc: str, udst: str) -> None:
os.rename(src, dst)
def get_df(abspath: str) -> tuple[Optional[int], Optional[int]]:
try:
# some fuses misbehave
if ANYWIN:
bfree = ctypes.c_ulonglong(0)
ctypes.windll.kernel32.GetDiskFreeSpaceExW( # type: ignore
ctypes.c_wchar_p(abspath), None, None, ctypes.pointer(bfree)
)
return (bfree.value, None)
else:
sv = os.statvfs(fsenc(abspath))
free = sv.f_frsize * sv.f_bfree
total = sv.f_frsize * sv.f_blocks
return (free, total)
except:
return (None, None)
def read_socket(sr: Unrecv, total_size: int) -> Generator[bytes, None, None]:
remains = total_size
while remains > 0:
@ -1233,7 +1258,7 @@ def read_socket_unbounded(sr: Unrecv) -> Generator[bytes, None, None]:
def read_socket_chunked(
sr: Unrecv, log: Optional[NamedLogger] = None
sr: Unrecv, log: Optional["NamedLogger"] = None
) -> Generator[bytes, None, None]:
err = "upload aborted: expected chunk length, got [{}] |{}| instead"
while True:
@ -1311,7 +1336,7 @@ def hashcopy(
def sendfile_py(
log: NamedLogger,
log: "NamedLogger",
lower: int,
upper: int,
f: typing.BinaryIO,
@ -1339,7 +1364,7 @@ def sendfile_py(
def sendfile_kern(
log: NamedLogger,
log: "NamedLogger",
lower: int,
upper: int,
f: typing.BinaryIO,
@ -1380,7 +1405,7 @@ def sendfile_kern(
def statdir(
logger: Optional[RootLogger], scandir: bool, lstat: bool, top: str
logger: Optional["RootLogger"], scandir: bool, lstat: bool, top: str
) -> Generator[tuple[str, os.stat_result], None, None]:
if lstat and ANYWIN:
lstat = False
@ -1423,7 +1448,7 @@ def statdir(
def rmdirs(
logger: RootLogger, scandir: bool, lstat: bool, top: str, depth: int
logger: "RootLogger", scandir: bool, lstat: bool, top: str, depth: int
) -> tuple[list[str], list[str]]:
"""rmdir all descendants, then self"""
if not os.path.isdir(fsenc(top)):
@ -1644,7 +1669,7 @@ def retchk(
rc: int,
cmd: Union[list[bytes], list[str]],
serr: str,
logger: Optional[NamedLogger] = None,
logger: Optional["NamedLogger"] = None,
color: Union[int, str] = 0,
verbose: bool = False,
) -> None:

View file

@ -311,6 +311,7 @@ var Ls = {
"u_ehsfin": "server rejected the request to finalize upload",
"u_ehssrch": "server rejected the request to perform search",
"u_ehsinit": "server rejected the request to initiate upload",
"u_ehsdf": "server ran out of disk space!\n\nwill keep retrying, in case someone\nfrees up enough space to continue",
"u_s404": "not found on server",
"u_expl": "explain",
"u_tu": '<p class="warn">WARNING: turbo enabled, <span>&nbsp;client may not detect and resume incomplete uploads; see turbo-button tooltip</span></p>',
@ -642,6 +643,7 @@ var Ls = {
"u_ehsfin": "server nektet forespørselen om å ferdigstille filen",
"u_ehssrch": "server nektet forespørselen om å utføre søk",
"u_ehsinit": "server nektet forespørselen om å begynne en ny opplastning",
"u_ehsdf": "serveren er full!\n\nprøver igjen regelmessig,\ni tilfelle noen rydder litt...",
"u_s404": "ikke funnet på serveren",
"u_expl": "forklar",
"u_tu": '<p class="warn">ADVARSEL: turbo er på, <span>&nbsp;avbrutte opplastninger vil muligens ikke oppdages og gjenopptas; hold musepekeren over turbo-knappen for mer info</span></p>',

View file

@ -2011,6 +2011,9 @@ function up2k_init(subtle) {
t.want_recheck = true;
}
}
if (rsp.indexOf('server HDD is full') + 1)
return toast.err(0, L.u_ehsdf + "\n\n" + rsp.replace(/.*; /, ''));
if (err != "") {
pvis.seth(t.n, 1, "ERROR");
pvis.seth(t.n, 2, err);

10
docs/notes.md Normal file
View file

@ -0,0 +1,10 @@
# up2k.js
## potato detection
* tsk 0.25/8.4/31.5 bzw 1.27/22.9/18 = 77% (38.4s, 49.7s)
* 4c locale #1313, ff-102,deb-11 @ ryzen4500u wifi -> win10
* profiling shows 2sec heavy gc every 2sec
* tsk 0.41/4.1/10 bzw 1.41/9.9/7 = 73% (13.3s, 18.2s)
* 4c locale #1313, ch-103,deb-11 @ ryzen4500u wifi -> win10