diff --git a/README.md b/README.md index 4d522451..83803df9 100644 --- a/README.md +++ b/README.md @@ -980,6 +980,8 @@ set upload rules using volflags, some examples: * `:c,sz=1k-3m` sets allowed filesize between 1 KiB and 3 MiB inclusive (suffixes: `b`, `k`, `m`, `g`) * `:c,df=4g` block uploads if there would be less than 4 GiB free disk space afterwards +* `:c,vmaxb=1g` block uploads if total volume size would exceed 1 GiB afterwards +* `:c,vmaxn=4k` block uploads if volume would contain more than 4096 files afterwards * `:c,nosub` disallow uploading into subdirectories; goes well with `rotn` and `rotf`: * `:c,rotn=1000,2` moves uploads into subfolders, up to 1000 files in each folder before making a new one, two levels deep (must be at least 1) * `:c,rotf=%Y/%m/%d/%H` enforces files to be uploaded into a structure of subfolders according to that date format diff --git a/copyparty/__main__.py b/copyparty/__main__.py index 90ba6991..7cff4682 100755 --- a/copyparty/__main__.py +++ b/copyparty/__main__.py @@ -980,7 +980,7 @@ def add_ui(ap, retry): ap2.add_argument("--lang", metavar="LANG", type=u, default="eng", help="language") ap2.add_argument("--theme", metavar="NUM", type=int, default=0, help="default theme to use") ap2.add_argument("--themes", metavar="NUM", type=int, default=8, help="number of themes installed") - ap2.add_argument("--unlist", metavar="REGEX", type=u, default="", help="don't show files matching REGEX in file list. Purely cosmetic! Does not affect API calls, just the browser. Example: [\033[32m\.(js|css)$\033[0m] (volflag=unlist)") + ap2.add_argument("--unlist", metavar="REGEX", type=u, default="", help="don't show files matching REGEX in file list. Purely cosmetic! Does not affect API calls, just the browser. Example: [\033[32m\\.(js|css)$\033[0m] (volflag=unlist)") ap2.add_argument("--favico", metavar="TXT", type=u, default="c 000 none" if retry else "🎉 000 none", help="\033[33mfavicon-text\033[0m [ \033[33mforeground\033[0m [ \033[33mbackground\033[0m ] ], set blank to disable") ap2.add_argument("--mpmc", metavar="URL", type=u, default="", help="change the mediaplayer-toggle mouse cursor; URL to a folder with {2..5}.png inside (or disable with [\033[32m.\033[0m])") ap2.add_argument("--js-browser", metavar="L", type=u, help="URL to additional JS to include") diff --git a/copyparty/authsrv.py b/copyparty/authsrv.py index fac29b9a..e8efd45d 100644 --- a/copyparty/authsrv.py +++ b/copyparty/authsrv.py @@ -40,7 +40,10 @@ if True: # pylint: disable=using-constant-test from .util import NamedLogger, RootLogger if TYPE_CHECKING: - pass + from .broker_mp import BrokerMp + from .broker_thr import BrokerThr + from .broker_util import BrokerCli + # Vflags: TypeAlias = dict[str, str | bool | float | list[str]] # Vflags: TypeAlias = dict[str, Any] # Mflags: TypeAlias = dict[str, Vflags] @@ -90,6 +93,8 @@ class Lim(object): self.dfl = 0 # free disk space limit self.dft = 0 # last-measured time self.dfv = 0 # currently free + self.vbmax = 0 # volume bytes max + self.vnmax = 0 # volume max num files self.smin = 0 # filesize min self.smax = 0 # filesize max @@ -119,8 +124,11 @@ class Lim(object): ip: str, rem: str, sz: int, + ptop: str, abspath: str, + broker: Optional[Union["BrokerCli", "BrokerMp", "BrokerThr"]] = None, reg: Optional[dict[str, dict[str, Any]]] = None, + volgetter: str = "up2k.get_volsize", ) -> tuple[str, str]: if reg is not None and self.reg is None: self.reg = reg @@ -131,6 +139,7 @@ class Lim(object): self.chk_rem(rem) if sz != -1: self.chk_sz(sz) + self.chk_vsz(broker, ptop, sz, volgetter) self.chk_df(abspath, sz) # side effects; keep last-ish ap2, vp2 = self.rot(abspath) @@ -146,6 +155,25 @@ class Lim(object): if self.smax and sz > self.smax: raise Pebkac(400, "file too big") + def chk_vsz( + self, + broker: Optional[Union["BrokerCli", "BrokerMp", "BrokerThr"]], + ptop: str, + sz: int, + volgetter: str = "up2k.get_volsize", + ) -> None: + if not broker or not self.vbmax + self.vnmax: + return + + x = broker.ask(volgetter, ptop) + nbytes, nfiles = x.get() + + if self.vbmax and self.vbmax < nbytes + sz: + raise Pebkac(400, "volume has exceeded max size") + + if self.vnmax and self.vnmax < nfiles + 1: + raise Pebkac(400, "volume has exceeded max num.files") + def chk_df(self, abspath: str, sz: int, already_written: bool = False) -> None: if not self.dfl: return @@ -266,7 +294,7 @@ class Lim(object): self.bupc[ip] = mark if mark >= self.bmax: - raise Pebkac(429, "ingress saturated") + raise Pebkac(429, "upload size limit exceeded") class VFS(object): @@ -1290,6 +1318,16 @@ class AuthSrv(object): use = True lim.bmax, lim.bwin = [unhumanize(x) for x in zs.split(",")] + zs = vol.flags.get("vmaxb") + if zs: + use = True + lim.vbmax = unhumanize(zs) + + zs = vol.flags.get("vmaxn") + if zs: + use = True + lim.vnmax = unhumanize(zs) + if use: vol.lim = lim diff --git a/copyparty/broker_mp.py b/copyparty/broker_mp.py index 6978cfb9..bbadc3fe 100644 --- a/copyparty/broker_mp.py +++ b/copyparty/broker_mp.py @@ -9,7 +9,7 @@ import queue from .__init__ import CORES, TYPE_CHECKING from .broker_mpw import MpWorker -from .broker_util import try_exec +from .broker_util import ExceptionalQueue, try_exec from .util import Daemon, mp if TYPE_CHECKING: @@ -107,6 +107,19 @@ class BrokerMp(object): if retq_id: proc.q_pend.put((retq_id, "retq", rv)) + def ask(self, dest: str, *args: Any) -> ExceptionalQueue: + + # new non-ipc invoking managed service in hub + obj = self.hub + for node in dest.split("."): + obj = getattr(obj, node) + + rv = try_exec(True, obj, *args) + + retq = ExceptionalQueue(1) + retq.put(rv) + return retq + def say(self, dest: str, *args: Any) -> None: """ send message to non-hub component in other process, diff --git a/copyparty/cert.py b/copyparty/cert.py index a6c28132..59c64563 100644 --- a/copyparty/cert.py +++ b/copyparty/cert.py @@ -10,6 +10,9 @@ from .util import Netdev, runcmd HAVE_CFSSL = True +if True: # pylint: disable=using-constant-test + from .util import RootLogger + def ensure_cert(log: "RootLogger", args) -> None: """ diff --git a/copyparty/cfg.py b/copyparty/cfg.py index 494cdb88..35008d51 100644 --- a/copyparty/cfg.py +++ b/copyparty/cfg.py @@ -78,7 +78,9 @@ flagcats = { }, "upload rules": { "maxn=250,600": "max 250 uploads over 15min", - "maxb=1g,300": "max 1 GiB over 5min (suffixes: b, k, m, g)", + "maxb=1g,300": "max 1 GiB over 5min (suffixes: b, k, m, g, t)", + "vmaxb=1g": "total volume size max 1 GiB (suffixes: b, k, m, g, t)", + "vmaxn=4k": "max 4096 files in volume (suffixes: b, k, m, g, t)", "rand": "force randomized filenames, 9 chars long by default", "nrand=N": "randomized filenames are N chars long", "sz=1k-3m": "allow filesizes between 1 KiB and 3MiB", diff --git a/copyparty/httpcli.py b/copyparty/httpcli.py index a72d79e4..bba3f580 100644 --- a/copyparty/httpcli.py +++ b/copyparty/httpcli.py @@ -1358,7 +1358,9 @@ class HttpCli(object): lim = vfs.get_dbv(rem)[0].lim fdir = vfs.canonical(rem) if lim: - fdir, rem = lim.all(self.ip, rem, remains, fdir) + fdir, rem = lim.all( + self.ip, rem, remains, vfs.realpath, fdir, self.conn.hsrv.broker + ) fn = None if rem and not self.trailing_slash and not bos.path.isdir(fdir): @@ -1491,6 +1493,7 @@ class HttpCli(object): lim.bup(self.ip, post_sz) try: lim.chk_sz(post_sz) + lim.chk_vsz(self.conn.hsrv.broker, vfs.realpath, post_sz) except: bos.unlink(path) raise @@ -2101,7 +2104,9 @@ class HttpCli(object): lim = vfs.get_dbv(rem)[0].lim fdir_base = vfs.canonical(rem) if lim: - fdir_base, rem = lim.all(self.ip, rem, -1, fdir_base) + fdir_base, rem = lim.all( + self.ip, rem, -1, vfs.realpath, fdir_base, self.conn.hsrv.broker + ) upload_vpath = "{}/{}".format(vfs.vpath, rem).strip("/") if not nullwrite: bos.makedirs(fdir_base) @@ -2194,6 +2199,7 @@ class HttpCli(object): try: lim.chk_df(tabspath, sz, True) lim.chk_sz(sz) + lim.chk_vsz(self.conn.hsrv.broker, vfs.realpath, sz) lim.chk_bup(self.ip) lim.chk_nup(self.ip) except: @@ -2369,7 +2375,7 @@ class HttpCli(object): fp = vfs.canonical(rp) lim = vfs.get_dbv(rem)[0].lim if lim: - fp, rp = lim.all(self.ip, rp, clen, fp) + fp, rp = lim.all(self.ip, rp, clen, vfs.realpath, fp, self.conn.hsrv.broker) bos.makedirs(fp) fp = os.path.join(fp, fn) @@ -2451,6 +2457,7 @@ class HttpCli(object): lim.bup(self.ip, sz) try: lim.chk_sz(sz) + lim.chk_vsz(self.conn.hsrv.broker, vfs.realpath, sz) except: bos.unlink(fp) raise diff --git a/copyparty/up2k.py b/copyparty/up2k.py index 97df5851..4f1371a9 100644 --- a/copyparty/up2k.py +++ b/copyparty/up2k.py @@ -41,6 +41,7 @@ from .util import ( gen_filekey, gen_filekey_dbg, hidedir, + humansize, min_ex, quotep, rand_name, @@ -56,6 +57,7 @@ from .util import ( sfsenc, spack, statdir, + unhumanize, vjoin, vsplit, w8b64dec, @@ -125,6 +127,8 @@ class Up2k(object): self.registry: dict[str, dict[str, dict[str, Any]]] = {} self.flags: dict[str, dict[str, Any]] = {} self.droppable: dict[str, list[str]] = {} + self.volnfiles: dict["sqlite3.Cursor", int] = {} + self.volsize: dict["sqlite3.Cursor", int] = {} self.volstate: dict[str, str] = {} self.vol_act: dict[str, float] = {} self.busy_aps: set[str] = set() @@ -261,6 +265,20 @@ class Up2k(object): } return json.dumps(ret, indent=4) + def get_volsize(self, ptop: str) -> tuple[int, int]: + with self.mutex: + return self._get_volsize(ptop) + + def _get_volsize(self, ptop: str) -> tuple[int, int]: + cur = self.cur[ptop] + nbytes = self.volsize[cur] + nfiles = self.volnfiles[cur] + for j in list(self.registry.get(ptop, {}).values()): + nbytes += j["size"] + nfiles += 1 + + return (nbytes, nfiles) + def rescan( self, all_vols: dict[str, VFS], scan_vols: list[str], wait: bool, fscan: bool ) -> str: @@ -810,6 +828,8 @@ class Up2k(object): try: cur = self._open_db(db_path) self.cur[ptop] = cur + self.volsize[cur] = 0 + self.volnfiles[cur] = 0 # speeds measured uploading 520 small files on a WD20SPZX (SMR 2.5" 5400rpm 4kb) dbd = flags["dbd"] @@ -917,6 +937,24 @@ class Up2k(object): db.c.connection.commit() + if vol.flags.get("vmaxb") or vol.flags.get("vmaxn"): + zs = "select count(sz), sum(sz) from up" + vn, vb = db.c.execute(zs).fetchone() + vb = vb or 0 + vb += vn * 2048 + self.volsize[db.c] = vb + self.volnfiles[db.c] = vn + vmaxb = unhumanize(vol.flags.get("vmaxb") or "0") + vmaxn = unhumanize(vol.flags.get("vmaxn") or "0") + t = "{} / {} ( {} / {} files) in {}".format( + humansize(vb, True), + humansize(vmaxb, True), + humansize(vn, True).rstrip("B"), + humansize(vmaxn, True).rstrip("B"), + vol.realpath, + ) + self.log(t) + return True, bool(n_add or n_rm or do_vac) def _build_dir( @@ -1092,7 +1130,7 @@ class Up2k(object): top, rp, dts, lmod, dsz, sz ) self.log(t) - self.db_rm(db.c, rd, fn) + self.db_rm(db.c, rd, fn, 0) ret += 1 db.n += 1 in_db = [] @@ -1175,7 +1213,7 @@ class Up2k(object): rm_files = [x for x in hits if x not in seen_files] n_rm = len(rm_files) for fn in rm_files: - self.db_rm(db.c, rd, fn) + self.db_rm(db.c, rd, fn, 0) if n_rm: self.log("forgot {} deleted files".format(n_rm)) @@ -2284,7 +2322,9 @@ class Up2k(object): if lost: c2 = None for cur, dp_dir, dp_fn in lost: - self.db_rm(cur, dp_dir, dp_fn) + t = "forgetting deleted file: /{}" + self.log(t.format(vjoin(vjoin(vfs.vpath, dp_dir), dp_fn))) + self.db_rm(cur, dp_dir, dp_fn, cj["size"]) if c2 and c2 != cur: c2.connection.commit() @@ -2418,7 +2458,14 @@ class Up2k(object): if vfs.lim: ap2, cj["prel"] = vfs.lim.all( - cj["addr"], cj["prel"], cj["size"], ap1, reg + cj["addr"], + cj["prel"], + cj["size"], + cj["ptop"], + ap1, + self.hub.broker, + reg, + "up2k._get_volsize", ) bos.makedirs(ap2) vfs.lim.nup(cj["addr"]) @@ -2736,7 +2783,7 @@ class Up2k(object): self._symlink(dst, d2, self.flags[ptop], lmod=lmod) if cur: - self.db_rm(cur, rd, fn) + self.db_rm(cur, rd, fn, job["size"]) self.db_add(cur, vflags, rd, fn, lmod, *z2[3:]) if cur: @@ -2779,7 +2826,7 @@ class Up2k(object): self.db_act = self.vol_act[ptop] = time.time() try: - self.db_rm(cur, rd, fn) + self.db_rm(cur, rd, fn, sz) self.db_add( cur, vflags, @@ -2809,13 +2856,17 @@ class Up2k(object): return True - def db_rm(self, db: "sqlite3.Cursor", rd: str, fn: str) -> None: + def db_rm(self, db: "sqlite3.Cursor", rd: str, fn: str, sz: int) -> None: sql = "delete from up where rd = ? and fn = ?" try: - db.execute(sql, (rd, fn)) + r = db.execute(sql, (rd, fn)) except: assert self.mem_cur - db.execute(sql, s3enc(self.mem_cur, rd, fn)) + r = db.execute(sql, s3enc(self.mem_cur, rd, fn)) + + if r.rowcount: + self.volsize[db] -= sz + self.volnfiles[db] -= 1 def db_add( self, @@ -2844,6 +2895,9 @@ class Up2k(object): v = (wark, int(ts), sz, rd, fn, ip or "", int(at or 0)) db.execute(sql, v) + self.volsize[db] += sz + self.volnfiles[db] += 1 + xau = False if skip_xau else vflags.get("xau") dst = djoin(ptop, rd, fn) if xau and not runhook( @@ -2991,12 +3045,12 @@ class Up2k(object): break abspath = djoin(adir, fn) + st = bos.stat(abspath) volpath = "{}/{}".format(vrem, fn).strip("/") vpath = "{}/{}".format(dbv.vpath, volpath).strip("/") self.log("rm {}\n {}".format(vpath, abspath)) _ = dbv.get(volpath, uname, *permsets[0]) if xbd: - st = bos.stat(abspath) if not runhook( self.log, xbd, @@ -3020,14 +3074,26 @@ class Up2k(object): try: ptop = dbv.realpath cur, wark, _, _, _, _ = self._find_from_vpath(ptop, volpath) - self._forget_file(ptop, volpath, cur, wark, True) + self._forget_file(ptop, volpath, cur, wark, True, st.st_size) finally: if cur: cur.connection.commit() bos.unlink(abspath) if xad: - runhook(self.log, xad, abspath, vpath, "", uname, 0, 0, ip, 0, "") + runhook( + self.log, + xad, + abspath, + vpath, + "", + uname, + st.st_mtime, + st.st_size, + ip, + 0, + "", + ) if is_dir: ok, ng = rmdirs(self.log_func, scandir, True, atop, 1) @@ -3203,7 +3269,7 @@ class Up2k(object): if c2 and c2 != c1: self._copy_tags(c1, c2, w) - self._forget_file(svn.realpath, srem, c1, w, c1 != c2) + self._forget_file(svn.realpath, srem, c1, w, c1 != c2, fsize) self._relink(w, svn.realpath, srem, dabs) curs.add(c1) @@ -3279,6 +3345,7 @@ class Up2k(object): cur: Optional["sqlite3.Cursor"], wark: Optional[str], drop_tags: bool, + sz: int, ) -> None: """forgets file in db, fixes symlinks, does not delete""" srd, sfn = vsplit(vrem) @@ -3293,7 +3360,7 @@ class Up2k(object): q = "delete from mt where w=?" cur.execute(q, (wark[:16],)) - self.db_rm(cur, srd, sfn) + self.db_rm(cur, srd, sfn, sz) reg = self.registry.get(ptop) if reg: diff --git a/copyparty/util.py b/copyparty/util.py index bef44165..bc184278 100644 --- a/copyparty/util.py +++ b/copyparty/util.py @@ -1626,7 +1626,12 @@ def unhumanize(sz: str) -> int: pass mc = sz[-1:].lower() - mi = {"k": 1024, "m": 1024 * 1024, "g": 1024 * 1024 * 1024}.get(mc, 1) + mi = { + "k": 1024, + "m": 1024 * 1024, + "g": 1024 * 1024 * 1024, + "t": 1024 * 1024 * 1024 * 1024, + }.get(mc, 1) return int(float(sz[:-1]) * mi)