volflags to limit volume size / num files; closes #40

README.md

@@ -980,6 +980,8 @@ set upload rules using volflags,  some examples:
 
 * `:c,sz=1k-3m` sets allowed filesize between 1 KiB and 3 MiB inclusive (suffixes: `b`, `k`, `m`, `g`)
 * `:c,df=4g` block uploads if there would be less than 4 GiB free disk space afterwards
+* `:c,vmaxb=1g` block uploads if total volume size would exceed 1 GiB afterwards
+* `:c,vmaxn=4k` block uploads if volume would contain more than 4096 files afterwards
 * `:c,nosub` disallow uploading into subdirectories; goes well with `rotn` and `rotf`:
 * `:c,rotn=1000,2` moves uploads into subfolders, up to 1000 files in each folder before making a new one, two levels deep (must be at least 1)
 * `:c,rotf=%Y/%m/%d/%H` enforces files to be uploaded into a structure of subfolders according to that date format

copyparty/__main__.py

@@ -980,7 +980,7 @@ def add_ui(ap, retry):
     ap2.add_argument("--lang", metavar="LANG", type=u, default="eng", help="language")
     ap2.add_argument("--theme", metavar="NUM", type=int, default=0, help="default theme to use")
     ap2.add_argument("--themes", metavar="NUM", type=int, default=8, help="number of themes installed")
-    ap2.add_argument("--unlist", metavar="REGEX", type=u, default="", help="don't show files matching REGEX in file list. Purely cosmetic! Does not affect API calls, just the browser. Example: [\033[32m\.(js|css)$\033[0m] (volflag=unlist)")
+    ap2.add_argument("--unlist", metavar="REGEX", type=u, default="", help="don't show files matching REGEX in file list. Purely cosmetic! Does not affect API calls, just the browser. Example: [\033[32m\\.(js|css)$\033[0m] (volflag=unlist)")
     ap2.add_argument("--favico", metavar="TXT", type=u, default="c 000 none" if retry else "🎉 000 none", help="\033[33mfavicon-text\033[0m [ \033[33mforeground\033[0m [ \033[33mbackground\033[0m ] ], set blank to disable")
     ap2.add_argument("--mpmc", metavar="URL", type=u, default="", help="change the mediaplayer-toggle mouse cursor; URL to a folder with {2..5}.png inside (or disable with [\033[32m.\033[0m])")
     ap2.add_argument("--js-browser", metavar="L", type=u, help="URL to additional JS to include")

copyparty/authsrv.py

@@ -40,7 +40,10 @@ if True:  # pylint: disable=using-constant-test
     from .util import NamedLogger, RootLogger
 
 if TYPE_CHECKING:
-    pass
+    from .broker_mp import BrokerMp
+    from .broker_thr import BrokerThr
+    from .broker_util import BrokerCli
+
     # Vflags: TypeAlias = dict[str, str | bool | float | list[str]]
     # Vflags: TypeAlias = dict[str, Any]
     # Mflags: TypeAlias = dict[str, Vflags]
@@ -90,6 +93,8 @@ class Lim(object):
         self.dfl = 0  # free disk space limit
         self.dft = 0  # last-measured time
         self.dfv = 0  # currently free
+        self.vbmax = 0  # volume bytes max
+        self.vnmax = 0  # volume max num files
 
         self.smin = 0  # filesize min
         self.smax = 0  # filesize max
@@ -119,8 +124,11 @@ class Lim(object):
         ip: str,
         rem: str,
         sz: int,
+        ptop: str,
         abspath: str,
+        broker: Optional[Union["BrokerCli", "BrokerMp", "BrokerThr"]] = None,
         reg: Optional[dict[str, dict[str, Any]]] = None,
+        volgetter: str = "up2k.get_volsize",
     ) -> tuple[str, str]:
         if reg is not None and self.reg is None:
             self.reg = reg
@@ -131,6 +139,7 @@ class Lim(object):
         self.chk_rem(rem)
         if sz != -1:
             self.chk_sz(sz)
+            self.chk_vsz(broker, ptop, sz, volgetter)
             self.chk_df(abspath, sz)  # side effects; keep last-ish
 
         ap2, vp2 = self.rot(abspath)
@@ -146,6 +155,25 @@ class Lim(object):
         if self.smax and sz > self.smax:
             raise Pebkac(400, "file too big")
 
+    def chk_vsz(
+        self,
+        broker: Optional[Union["BrokerCli", "BrokerMp", "BrokerThr"]],
+        ptop: str,
+        sz: int,
+        volgetter: str = "up2k.get_volsize",
+    ) -> None:
+        if not broker or not self.vbmax + self.vnmax:
+            return
+
+        x = broker.ask(volgetter, ptop)
+        nbytes, nfiles = x.get()
+
+        if self.vbmax and self.vbmax < nbytes + sz:
+            raise Pebkac(400, "volume has exceeded max size")
+
+        if self.vnmax and self.vnmax < nfiles + 1:
+            raise Pebkac(400, "volume has exceeded max num.files")
+
     def chk_df(self, abspath: str, sz: int, already_written: bool = False) -> None:
         if not self.dfl:
             return
@@ -266,7 +294,7 @@ class Lim(object):
 
         self.bupc[ip] = mark
         if mark >= self.bmax:
-            raise Pebkac(429, "ingress saturated")
+            raise Pebkac(429, "upload size limit exceeded")
 
 
 class VFS(object):
@@ -1290,6 +1318,16 @@ class AuthSrv(object):
                 use = True
                 lim.bmax, lim.bwin = [unhumanize(x) for x in zs.split(",")]
 
+            zs = vol.flags.get("vmaxb")
+            if zs:
+                use = True
+                lim.vbmax = unhumanize(zs)
+
+            zs = vol.flags.get("vmaxn")
+            if zs:
+                use = True
+                lim.vnmax = unhumanize(zs)
+
             if use:
                 vol.lim = lim
 

copyparty/broker_mp.py

@@ -9,7 +9,7 @@ import queue
 
 from .__init__ import CORES, TYPE_CHECKING
 from .broker_mpw import MpWorker
-from .broker_util import try_exec
+from .broker_util import ExceptionalQueue, try_exec
 from .util import Daemon, mp
 
 if TYPE_CHECKING:
@@ -107,6 +107,19 @@ class BrokerMp(object):
                 if retq_id:
                     proc.q_pend.put((retq_id, "retq", rv))
 
+    def ask(self, dest: str, *args: Any) -> ExceptionalQueue:
+
+        # new non-ipc invoking managed service in hub
+        obj = self.hub
+        for node in dest.split("."):
+            obj = getattr(obj, node)
+
+        rv = try_exec(True, obj, *args)
+
+        retq = ExceptionalQueue(1)
+        retq.put(rv)
+        return retq
+
     def say(self, dest: str, *args: Any) -> None:
         """
         send message to non-hub component in other process,

copyparty/cert.py

@@ -10,6 +10,9 @@ from .util import Netdev, runcmd
 
 HAVE_CFSSL = True
 
+if True:  # pylint: disable=using-constant-test
+    from .util import RootLogger
+
 
 def ensure_cert(log: "RootLogger", args) -> None:
     """

copyparty/cfg.py

@@ -78,7 +78,9 @@ flagcats = {
     },
     "upload rules": {
         "maxn=250,600": "max 250 uploads over 15min",
-        "maxb=1g,300": "max 1 GiB over 5min (suffixes: b, k, m, g)",
+        "maxb=1g,300": "max 1 GiB over 5min (suffixes: b, k, m, g, t)",
+        "vmaxb=1g": "total volume size max 1 GiB (suffixes: b, k, m, g, t)",
+        "vmaxn=4k": "max 4096 files in volume (suffixes: b, k, m, g, t)",
         "rand": "force randomized filenames, 9 chars long by default",
         "nrand=N": "randomized filenames are N chars long",
         "sz=1k-3m": "allow filesizes between 1 KiB and 3MiB",

copyparty/httpcli.py

@@ -1358,7 +1358,9 @@ class HttpCli(object):
         lim = vfs.get_dbv(rem)[0].lim
         fdir = vfs.canonical(rem)
         if lim:
-            fdir, rem = lim.all(self.ip, rem, remains, fdir)
+            fdir, rem = lim.all(
+                self.ip, rem, remains, vfs.realpath, fdir, self.conn.hsrv.broker
+            )
 
         fn = None
         if rem and not self.trailing_slash and not bos.path.isdir(fdir):
@@ -1491,6 +1493,7 @@ class HttpCli(object):
             lim.bup(self.ip, post_sz)
             try:
                 lim.chk_sz(post_sz)
+                lim.chk_vsz(self.conn.hsrv.broker, vfs.realpath, post_sz)
             except:
                 bos.unlink(path)
                 raise
@@ -2101,7 +2104,9 @@ class HttpCli(object):
         lim = vfs.get_dbv(rem)[0].lim
         fdir_base = vfs.canonical(rem)
         if lim:
-            fdir_base, rem = lim.all(self.ip, rem, -1, fdir_base)
+            fdir_base, rem = lim.all(
+                self.ip, rem, -1, vfs.realpath, fdir_base, self.conn.hsrv.broker
+            )
             upload_vpath = "{}/{}".format(vfs.vpath, rem).strip("/")
             if not nullwrite:
                 bos.makedirs(fdir_base)
@@ -2194,6 +2199,7 @@ class HttpCli(object):
                         try:
                             lim.chk_df(tabspath, sz, True)
                             lim.chk_sz(sz)
+                            lim.chk_vsz(self.conn.hsrv.broker, vfs.realpath, sz)
                             lim.chk_bup(self.ip)
                             lim.chk_nup(self.ip)
                         except:
@@ -2369,7 +2375,7 @@ class HttpCli(object):
         fp = vfs.canonical(rp)
         lim = vfs.get_dbv(rem)[0].lim
         if lim:
-            fp, rp = lim.all(self.ip, rp, clen, fp)
+            fp, rp = lim.all(self.ip, rp, clen, vfs.realpath, fp, self.conn.hsrv.broker)
             bos.makedirs(fp)
 
         fp = os.path.join(fp, fn)
@@ -2451,6 +2457,7 @@ class HttpCli(object):
             lim.bup(self.ip, sz)
             try:
                 lim.chk_sz(sz)
+                lim.chk_vsz(self.conn.hsrv.broker, vfs.realpath, sz)
             except:
                 bos.unlink(fp)
                 raise

copyparty/up2k.py

@@ -41,6 +41,7 @@ from .util import (
     gen_filekey,
     gen_filekey_dbg,
     hidedir,
+    humansize,
     min_ex,
     quotep,
     rand_name,
@@ -56,6 +57,7 @@ from .util import (
     sfsenc,
     spack,
     statdir,
+    unhumanize,
     vjoin,
     vsplit,
     w8b64dec,
@@ -125,6 +127,8 @@ class Up2k(object):
         self.registry: dict[str, dict[str, dict[str, Any]]] = {}
         self.flags: dict[str, dict[str, Any]] = {}
         self.droppable: dict[str, list[str]] = {}
+        self.volnfiles: dict["sqlite3.Cursor", int] = {}
+        self.volsize: dict["sqlite3.Cursor", int] = {}
         self.volstate: dict[str, str] = {}
         self.vol_act: dict[str, float] = {}
         self.busy_aps: set[str] = set()
@@ -261,6 +265,20 @@ class Up2k(object):
         }
         return json.dumps(ret, indent=4)
 
+    def get_volsize(self, ptop: str) -> tuple[int, int]:
+        with self.mutex:
+            return self._get_volsize(ptop)
+
+    def _get_volsize(self, ptop: str) -> tuple[int, int]:
+        cur = self.cur[ptop]
+        nbytes = self.volsize[cur]
+        nfiles = self.volnfiles[cur]
+        for j in list(self.registry.get(ptop, {}).values()):
+            nbytes += j["size"]
+            nfiles += 1
+
+        return (nbytes, nfiles)
+
     def rescan(
         self, all_vols: dict[str, VFS], scan_vols: list[str], wait: bool, fscan: bool
     ) -> str:
@@ -810,6 +828,8 @@ class Up2k(object):
         try:
             cur = self._open_db(db_path)
             self.cur[ptop] = cur
+            self.volsize[cur] = 0
+            self.volnfiles[cur] = 0
 
             # speeds measured uploading 520 small files on a WD20SPZX (SMR 2.5" 5400rpm 4kb)
             dbd = flags["dbd"]
@@ -917,6 +937,24 @@ class Up2k(object):
 
             db.c.connection.commit()
 
+            if vol.flags.get("vmaxb") or vol.flags.get("vmaxn"):
+                zs = "select count(sz), sum(sz) from up"
+                vn, vb = db.c.execute(zs).fetchone()
+                vb = vb or 0
+                vb += vn * 2048
+                self.volsize[db.c] = vb
+                self.volnfiles[db.c] = vn
+                vmaxb = unhumanize(vol.flags.get("vmaxb") or "0")
+                vmaxn = unhumanize(vol.flags.get("vmaxn") or "0")
+                t = "{} / {}  ( {} / {} files) in {}".format(
+                    humansize(vb, True),
+                    humansize(vmaxb, True),
+                    humansize(vn, True).rstrip("B"),
+                    humansize(vmaxn, True).rstrip("B"),
+                    vol.realpath,
+                )
+                self.log(t)
+
             return True, bool(n_add or n_rm or do_vac)
 
     def _build_dir(
@@ -1092,7 +1130,7 @@ class Up2k(object):
                         top, rp, dts, lmod, dsz, sz
                     )
                     self.log(t)
-                    self.db_rm(db.c, rd, fn)
+                    self.db_rm(db.c, rd, fn, 0)
                     ret += 1
                     db.n += 1
                     in_db = []
@@ -1175,7 +1213,7 @@ class Up2k(object):
         rm_files = [x for x in hits if x not in seen_files]
         n_rm = len(rm_files)
         for fn in rm_files:
-            self.db_rm(db.c, rd, fn)
+            self.db_rm(db.c, rd, fn, 0)
 
         if n_rm:
             self.log("forgot {} deleted files".format(n_rm))
@@ -2284,7 +2322,9 @@ class Up2k(object):
             if lost:
                 c2 = None
                 for cur, dp_dir, dp_fn in lost:
-                    self.db_rm(cur, dp_dir, dp_fn)
+                    t = "forgetting deleted file: /{}"
+                    self.log(t.format(vjoin(vjoin(vfs.vpath, dp_dir), dp_fn)))
+                    self.db_rm(cur, dp_dir, dp_fn, cj["size"])
                     if c2 and c2 != cur:
                         c2.connection.commit()
 
@@ -2418,7 +2458,14 @@ class Up2k(object):
 
                 if vfs.lim:
                     ap2, cj["prel"] = vfs.lim.all(
-                        cj["addr"], cj["prel"], cj["size"], ap1, reg
+                        cj["addr"],
+                        cj["prel"],
+                        cj["size"],
+                        cj["ptop"],
+                        ap1,
+                        self.hub.broker,
+                        reg,
+                        "up2k._get_volsize",
                     )
                     bos.makedirs(ap2)
                     vfs.lim.nup(cj["addr"])
@@ -2736,7 +2783,7 @@ class Up2k(object):
 
             self._symlink(dst, d2, self.flags[ptop], lmod=lmod)
             if cur:
-                self.db_rm(cur, rd, fn)
+                self.db_rm(cur, rd, fn, job["size"])
                 self.db_add(cur, vflags, rd, fn, lmod, *z2[3:])
 
         if cur:
@@ -2779,7 +2826,7 @@ class Up2k(object):
 
         self.db_act = self.vol_act[ptop] = time.time()
         try:
-            self.db_rm(cur, rd, fn)
+            self.db_rm(cur, rd, fn, sz)
             self.db_add(
                 cur,
                 vflags,
@@ -2809,13 +2856,17 @@ class Up2k(object):
 
         return True
 
-    def db_rm(self, db: "sqlite3.Cursor", rd: str, fn: str) -> None:
+    def db_rm(self, db: "sqlite3.Cursor", rd: str, fn: str, sz: int) -> None:
         sql = "delete from up where rd = ? and fn = ?"
         try:
-            db.execute(sql, (rd, fn))
+            r = db.execute(sql, (rd, fn))
         except:
             assert self.mem_cur
-            db.execute(sql, s3enc(self.mem_cur, rd, fn))
+            r = db.execute(sql, s3enc(self.mem_cur, rd, fn))
+
+        if r.rowcount:
+            self.volsize[db] -= sz
+            self.volnfiles[db] -= 1
 
     def db_add(
         self,
@@ -2844,6 +2895,9 @@ class Up2k(object):
             v = (wark, int(ts), sz, rd, fn, ip or "", int(at or 0))
             db.execute(sql, v)
 
+        self.volsize[db] += sz
+        self.volnfiles[db] += 1
+
         xau = False if skip_xau else vflags.get("xau")
         dst = djoin(ptop, rd, fn)
         if xau and not runhook(
@@ -2991,12 +3045,12 @@ class Up2k(object):
                         break
 
                 abspath = djoin(adir, fn)
+                st = bos.stat(abspath)
                 volpath = "{}/{}".format(vrem, fn).strip("/")
                 vpath = "{}/{}".format(dbv.vpath, volpath).strip("/")
                 self.log("rm {}\n  {}".format(vpath, abspath))
                 _ = dbv.get(volpath, uname, *permsets[0])
                 if xbd:
-                    st = bos.stat(abspath)
                     if not runhook(
                         self.log,
                         xbd,
@@ -3020,14 +3074,26 @@ class Up2k(object):
                     try:
                         ptop = dbv.realpath
                         cur, wark, _, _, _, _ = self._find_from_vpath(ptop, volpath)
-                        self._forget_file(ptop, volpath, cur, wark, True)
+                        self._forget_file(ptop, volpath, cur, wark, True, st.st_size)
                     finally:
                         if cur:
                             cur.connection.commit()
 
                 bos.unlink(abspath)
                 if xad:
-                    runhook(self.log, xad, abspath, vpath, "", uname, 0, 0, ip, 0, "")
+                    runhook(
+                        self.log,
+                        xad,
+                        abspath,
+                        vpath,
+                        "",
+                        uname,
+                        st.st_mtime,
+                        st.st_size,
+                        ip,
+                        0,
+                        "",
+                    )
 
         if is_dir:
             ok, ng = rmdirs(self.log_func, scandir, True, atop, 1)
@@ -3203,7 +3269,7 @@ class Up2k(object):
             if c2 and c2 != c1:
                 self._copy_tags(c1, c2, w)
 
-            self._forget_file(svn.realpath, srem, c1, w, c1 != c2)
+            self._forget_file(svn.realpath, srem, c1, w, c1 != c2, fsize)
             self._relink(w, svn.realpath, srem, dabs)
             curs.add(c1)
 
@@ -3279,6 +3345,7 @@ class Up2k(object):
         cur: Optional["sqlite3.Cursor"],
         wark: Optional[str],
         drop_tags: bool,
+        sz: int,
     ) -> None:
         """forgets file in db, fixes symlinks, does not delete"""
         srd, sfn = vsplit(vrem)
@@ -3293,7 +3360,7 @@ class Up2k(object):
                 q = "delete from mt where w=?"
                 cur.execute(q, (wark[:16],))
 
-            self.db_rm(cur, srd, sfn)
+            self.db_rm(cur, srd, sfn, sz)
 
         reg = self.registry.get(ptop)
         if reg:

copyparty/util.py

@@ -1626,7 +1626,12 @@ def unhumanize(sz: str) -> int:
         pass
 
     mc = sz[-1:].lower()
-    mi = {"k": 1024, "m": 1024 * 1024, "g": 1024 * 1024 * 1024}.get(mc, 1)
+    mi = {
+        "k": 1024,
+        "m": 1024 * 1024,
+        "g": 1024 * 1024 * 1024,
+        "t": 1024 * 1024 * 1024 * 1024,
+    }.get(mc, 1)
     return int(float(sz[:-1]) * mi)