add upload lifetimes

README.md

@@ -487,6 +487,7 @@ you can set upload rules using volume flags, some examples:
   * if someone uploads to `/foo/bar` the path would be rewritten to `/foo/bar/2021/08/06/23` for example
   * but the actual value is not verified, just the structure, so the uploader can choose any values which conform to the format string
     * just to avoid additional complexity in up2k which is enough of a mess already
+* `:c,lifetime=300` delete uploaded files when they become 5 minutes old
 
 you can also set transaction limits which apply per-IP and per-volume, but these assume `-j 1` (default) otherwise the limits will be off, for example `-j 4` would allow anywhere between 1x and 4x the limits you set depending on which processing node the client gets routed to
 

bin/mtag/res/yt-ipr.conf

@@ -28,6 +28,8 @@ c sz=16k-1m
 c maxn=10,300
 # move uploads into subfolders: YEAR-MONTH / DAY-HOUR / <upload>
 c rotf=%Y-%m/%d-%H
+# delete uploads when they are 24 hours old
+c lifetime=86400
 # add the parser and tell copyparty what tags it can expect from it
 c mtp=yt-id,yt-title,yt-author,yt-channel,yt-views,yt-private,yt-manifest,yt-expires=bin/mtag/yt-ipr.py
 # decide which tags we want to index and in what order

copyparty/__main__.py

@@ -210,9 +210,9 @@ def run_argparse(argv, formatter):
             dedent(
                 """
             -a takes username:password,
-            -v takes src:dst:perm1:perm2:permN:cflag1:cflag2:cflagN:...
+            -v takes src:dst:perm1:perm2:permN:volflag1:volflag2:volflagN:...
                where "perm" is "accesslevels,username1,username2,..."
-               and "cflag" is config flags to set on this volume
+               and "volflag" is config flags to set on this volume
             
             list of accesslevels:
               "r" (read):   list folder contents, download files
@@ -220,7 +220,7 @@ def run_argparse(argv, formatter):
               "m" (move):   move files and folders; need "w" at destination
               "d" (delete): permanently delete files and folders
 
-            too many cflags to list here, see the other sections
+            too many volflags to list here, see the other sections
 
             example:\033[35m
               -a ed:hunter2 -v .::r:rw,ed -v ../inc:dump:w:rw,ed:c,nodupe  \033[36m
@@ -241,11 +241,11 @@ def run_argparse(argv, formatter):
             ),
         ],
         [
-            "cflags",
+            "flags",
-            "list of cflags",
+            "list of volflags",
             dedent(
                 """
-            cflags are appended to volume definitions, for example,
+            volflags are appended to volume definitions, for example,
             to create a write-only volume with the \033[33mnodupe\033[0m and \033[32mnosub\033[0m flags:
               \033[35m-v /mnt/inc:/inc:w\033[33m:c,nodupe\033[32m:c,nosub
 
@@ -264,9 +264,10 @@ def run_argparse(argv, formatter):
             (moves all uploads into the specified folder structure)
               \033[36mrotn=100,3\033[35m 3 levels of subfolders with 100 entries in each
               \033[36mrotf=%Y-%m/%d-%H\033[35m date-formatted organizing
+              \033[36mlifetime=3600\033[35m uploads are deleted after 1 hour
             
             \033[0mdatabase, general:
-              \033[36me2d\033[35m sets -e2d (all -e2* args can be set using ce2* cflags)
+              \033[36me2d\033[35m sets -e2d (all -e2* args can be set using ce2* volflags)
               \033[36md2t\033[35m disables metadata collection, overrides -e2t*
               \033[36md2d\033[35m disables all database stuff, overrides -e2*
               \033[36mdhash\033[35m disables file hashing on initial scans, also ehash
@@ -354,6 +355,7 @@ def run_argparse(argv, formatter):
     ap2.add_argument("-nih", action="store_true", help="no info hostname")
     ap2.add_argument("-nid", action="store_true", help="no info disk-usage")
     ap2.add_argument("--no-zip", action="store_true", help="disable download as zip/tar")
+    ap2.add_argument("--no-lifetime", action="store_true", help="disable automatic deletion of uploads after a certain time (lifetime volflag)")
 
     ap2 = ap.add_argument_group('safety options')
     ap2.add_argument("--ls", metavar="U[,V[,F]]", type=u, help="scan all volumes; arguments USER,VOL,FLAGS; example [**,*,ln,p,r]")
@@ -392,7 +394,7 @@ def run_argparse(argv, formatter):
     ap2.add_argument("--hist", metavar="PATH", type=u, help="where to store volume data (db, thumbs)")
     ap2.add_argument("--no-hash", action="store_true", help="disable hashing during e2ds folder scans")
     ap2.add_argument("--re-int", metavar="SEC", type=int, default=30, help="disk rescan check interval")
-    ap2.add_argument("--re-maxage", metavar="SEC", type=int, default=0, help="disk rescan volume interval, 0=off, can be set per-volume with the 'scan' cflag")
+    ap2.add_argument("--re-maxage", metavar="SEC", type=int, default=0, help="disk rescan volume interval, 0=off, can be set per-volume with the 'scan' volflag")
     ap2.add_argument("--srch-time", metavar="SEC", type=int, default=30, help="search deadline")
     
     ap2 = ap.add_argument_group('metadata db options')

copyparty/authsrv.py

@@ -25,6 +25,9 @@ from .util import (
 from .bos import bos
 
 
+LEELOO_DALLAS = "leeloo_dallas"
+
+
 class AXS(object):
     def __init__(self, uread=None, uwrite=None, umove=None, udel=None):
         self.uread = {} if uread is None else {k: 1 for k in uread}
@@ -327,7 +330,7 @@ class VFS(object):
             [will_move, c.umove, "move"],
             [will_del, c.udel, "delete"],
         ]:
-            if req and (uname not in d and "*" not in d):
+            if req and (uname not in d and "*" not in d) and uname != LEELOO_DALLAS:
                 m = "you don't have {}-access for this location"
                 raise Pebkac(403, m.format(msg))
 
@@ -554,6 +557,9 @@ class AuthSrv(object):
 
     def _read_vol_str(self, lvl, uname, axs, flags):
         # type: (str, str, AXS, any) -> None
+        if lvl.strip("crwmd"):
+            raise Exception("invalid volume flag: {},{}".format(lvl, uname))
+
         if lvl == "c":
             cval = True
             if "=" in uname:
@@ -709,6 +715,9 @@ class AuthSrv(object):
             )
             raise Exception("invalid config")
 
+        if LEELOO_DALLAS in all_users:
+            raise Exception("sorry, reserved username: " + LEELOO_DALLAS)
+
         promote = []
         demote = []
         for vol in vfs.all_vols.values():

copyparty/up2k.py

@@ -36,7 +36,7 @@ from .util import (
     min_ex,
 )
 from .bos import bos
-from .authsrv import AuthSrv
+from .authsrv import AuthSrv, LEELOO_DALLAS
 from .mtag import MTag, MParser
 
 try:
@@ -192,21 +192,55 @@ class Up2k(object):
                     if now - volage[vp] >= maxage:
                         self.need_rescan[vp] = 1
 
-                if not self.need_rescan:
-                    continue
-
                 vols = list(sorted(self.need_rescan.keys()))
                 self.need_rescan = {}
 
-            err = self.rescan(self.asrv.vfs.all_vols, vols)
+            if vols:
-            if err:
+                err = self.rescan(self.asrv.vfs.all_vols, vols)
-                for v in vols:
+                if err:
-                    self.need_rescan[v] = True
+                    for v in vols:
+                        self.need_rescan[v] = True
 
+                    continue
+
+                for v in vols:
+                    volage[v] = now
+
+            if self.args.no_lifetime:
                 continue
 
-            for v in vols:
+            for vp, vol in sorted(self.asrv.vfs.all_vols.items()):
-                volage[v] = now
+                lifetime = vol.flags.get("lifetime")
+                if not lifetime:
+                    continue
+
+                cur = self.cur.get(vol.realpath)
+                if not cur:
+                    continue
+
+                nrm = 0
+                deadline = time.time() - int(lifetime)
+                q = "select rd, fn from up where at > 0 and at < ? limit 100"
+                while True:
+                    with self.mutex:
+                        hits = cur.execute(q, (deadline,)).fetchall()
+
+                    if not hits:
+                        break
+
+                    for rd, fn in hits:
+                        if rd.startswith("//") or fn.startswith("//"):
+                            rd, fn = s3dec(rd, fn)
+
+                        fvp = "{}/{}".format(rd, fn).strip("/")
+                        if vp:
+                            fvp = "{}/{}".format(vp, fvp)
+
+                        self._handle_rm(LEELOO_DALLAS, None, fvp)
+                        nrm += 1
+
+                if nrm:
+                    self.log("{} files graduated in {}".format(nrm, vp))
 
     def _vis_job_progress(self, job):
         perc = 100 - (len(job["need"]) * 100.0 / len(job["hash"]))