From c1d77e10411f139f762fd3f3a245d47c9a8bb872 Mon Sep 17 00:00:00 2001 From: ed Date: Mon, 9 Aug 2021 22:17:41 +0200 Subject: [PATCH] add upload lifetimes --- README.md | 1 + bin/mtag/res/yt-ipr.conf | 2 ++ copyparty/__main__.py | 18 ++++++++------ copyparty/authsrv.py | 11 +++++++- copyparty/up2k.py | 54 ++++++++++++++++++++++++++++++++-------- 5 files changed, 67 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index d74f8f4f..f3625a29 100644 --- a/README.md +++ b/README.md @@ -487,6 +487,7 @@ you can set upload rules using volume flags, some examples: * if someone uploads to `/foo/bar` the path would be rewritten to `/foo/bar/2021/08/06/23` for example * but the actual value is not verified, just the structure, so the uploader can choose any values which conform to the format string * just to avoid additional complexity in up2k which is enough of a mess already +* `:c,lifetime=300` delete uploaded files when they become 5 minutes old you can also set transaction limits which apply per-IP and per-volume, but these assume `-j 1` (default) otherwise the limits will be off, for example `-j 4` would allow anywhere between 1x and 4x the limits you set depending on which processing node the client gets routed to diff --git a/bin/mtag/res/yt-ipr.conf b/bin/mtag/res/yt-ipr.conf index f95f0aee..1aa22423 100644 --- a/bin/mtag/res/yt-ipr.conf +++ b/bin/mtag/res/yt-ipr.conf @@ -28,6 +28,8 @@ c sz=16k-1m c maxn=10,300 # move uploads into subfolders: YEAR-MONTH / DAY-HOUR / c rotf=%Y-%m/%d-%H +# delete uploads when they are 24 hours old +c lifetime=86400 # add the parser and tell copyparty what tags it can expect from it c mtp=yt-id,yt-title,yt-author,yt-channel,yt-views,yt-private,yt-manifest,yt-expires=bin/mtag/yt-ipr.py # decide which tags we want to index and in what order diff --git a/copyparty/__main__.py b/copyparty/__main__.py index f1064433..625ff138 100644 --- a/copyparty/__main__.py +++ b/copyparty/__main__.py @@ -210,9 +210,9 @@ def run_argparse(argv, formatter): dedent( """ -a takes username:password, - -v takes src:dst:perm1:perm2:permN:cflag1:cflag2:cflagN:... + -v takes src:dst:perm1:perm2:permN:volflag1:volflag2:volflagN:... where "perm" is "accesslevels,username1,username2,..." - and "cflag" is config flags to set on this volume + and "volflag" is config flags to set on this volume list of accesslevels: "r" (read): list folder contents, download files @@ -220,7 +220,7 @@ def run_argparse(argv, formatter): "m" (move): move files and folders; need "w" at destination "d" (delete): permanently delete files and folders - too many cflags to list here, see the other sections + too many volflags to list here, see the other sections example:\033[35m -a ed:hunter2 -v .::r:rw,ed -v ../inc:dump:w:rw,ed:c,nodupe \033[36m @@ -241,11 +241,11 @@ def run_argparse(argv, formatter): ), ], [ - "cflags", - "list of cflags", + "flags", + "list of volflags", dedent( """ - cflags are appended to volume definitions, for example, + volflags are appended to volume definitions, for example, to create a write-only volume with the \033[33mnodupe\033[0m and \033[32mnosub\033[0m flags: \033[35m-v /mnt/inc:/inc:w\033[33m:c,nodupe\033[32m:c,nosub @@ -264,9 +264,10 @@ def run_argparse(argv, formatter): (moves all uploads into the specified folder structure) \033[36mrotn=100,3\033[35m 3 levels of subfolders with 100 entries in each \033[36mrotf=%Y-%m/%d-%H\033[35m date-formatted organizing + \033[36mlifetime=3600\033[35m uploads are deleted after 1 hour \033[0mdatabase, general: - \033[36me2d\033[35m sets -e2d (all -e2* args can be set using ce2* cflags) + \033[36me2d\033[35m sets -e2d (all -e2* args can be set using ce2* volflags) \033[36md2t\033[35m disables metadata collection, overrides -e2t* \033[36md2d\033[35m disables all database stuff, overrides -e2* \033[36mdhash\033[35m disables file hashing on initial scans, also ehash @@ -354,6 +355,7 @@ def run_argparse(argv, formatter): ap2.add_argument("-nih", action="store_true", help="no info hostname") ap2.add_argument("-nid", action="store_true", help="no info disk-usage") ap2.add_argument("--no-zip", action="store_true", help="disable download as zip/tar") + ap2.add_argument("--no-lifetime", action="store_true", help="disable automatic deletion of uploads after a certain time (lifetime volflag)") ap2 = ap.add_argument_group('safety options') ap2.add_argument("--ls", metavar="U[,V[,F]]", type=u, help="scan all volumes; arguments USER,VOL,FLAGS; example [**,*,ln,p,r]") @@ -392,7 +394,7 @@ def run_argparse(argv, formatter): ap2.add_argument("--hist", metavar="PATH", type=u, help="where to store volume data (db, thumbs)") ap2.add_argument("--no-hash", action="store_true", help="disable hashing during e2ds folder scans") ap2.add_argument("--re-int", metavar="SEC", type=int, default=30, help="disk rescan check interval") - ap2.add_argument("--re-maxage", metavar="SEC", type=int, default=0, help="disk rescan volume interval, 0=off, can be set per-volume with the 'scan' cflag") + ap2.add_argument("--re-maxage", metavar="SEC", type=int, default=0, help="disk rescan volume interval, 0=off, can be set per-volume with the 'scan' volflag") ap2.add_argument("--srch-time", metavar="SEC", type=int, default=30, help="search deadline") ap2 = ap.add_argument_group('metadata db options') diff --git a/copyparty/authsrv.py b/copyparty/authsrv.py index de5b3290..672d6c58 100644 --- a/copyparty/authsrv.py +++ b/copyparty/authsrv.py @@ -25,6 +25,9 @@ from .util import ( from .bos import bos +LEELOO_DALLAS = "leeloo_dallas" + + class AXS(object): def __init__(self, uread=None, uwrite=None, umove=None, udel=None): self.uread = {} if uread is None else {k: 1 for k in uread} @@ -327,7 +330,7 @@ class VFS(object): [will_move, c.umove, "move"], [will_del, c.udel, "delete"], ]: - if req and (uname not in d and "*" not in d): + if req and (uname not in d and "*" not in d) and uname != LEELOO_DALLAS: m = "you don't have {}-access for this location" raise Pebkac(403, m.format(msg)) @@ -554,6 +557,9 @@ class AuthSrv(object): def _read_vol_str(self, lvl, uname, axs, flags): # type: (str, str, AXS, any) -> None + if lvl.strip("crwmd"): + raise Exception("invalid volume flag: {},{}".format(lvl, uname)) + if lvl == "c": cval = True if "=" in uname: @@ -709,6 +715,9 @@ class AuthSrv(object): ) raise Exception("invalid config") + if LEELOO_DALLAS in all_users: + raise Exception("sorry, reserved username: " + LEELOO_DALLAS) + promote = [] demote = [] for vol in vfs.all_vols.values(): diff --git a/copyparty/up2k.py b/copyparty/up2k.py index 734d3129..44a60f87 100644 --- a/copyparty/up2k.py +++ b/copyparty/up2k.py @@ -36,7 +36,7 @@ from .util import ( min_ex, ) from .bos import bos -from .authsrv import AuthSrv +from .authsrv import AuthSrv, LEELOO_DALLAS from .mtag import MTag, MParser try: @@ -192,21 +192,55 @@ class Up2k(object): if now - volage[vp] >= maxage: self.need_rescan[vp] = 1 - if not self.need_rescan: - continue - vols = list(sorted(self.need_rescan.keys())) self.need_rescan = {} - err = self.rescan(self.asrv.vfs.all_vols, vols) - if err: - for v in vols: - self.need_rescan[v] = True + if vols: + err = self.rescan(self.asrv.vfs.all_vols, vols) + if err: + for v in vols: + self.need_rescan[v] = True + continue + + for v in vols: + volage[v] = now + + if self.args.no_lifetime: continue - for v in vols: - volage[v] = now + for vp, vol in sorted(self.asrv.vfs.all_vols.items()): + lifetime = vol.flags.get("lifetime") + if not lifetime: + continue + + cur = self.cur.get(vol.realpath) + if not cur: + continue + + nrm = 0 + deadline = time.time() - int(lifetime) + q = "select rd, fn from up where at > 0 and at < ? limit 100" + while True: + with self.mutex: + hits = cur.execute(q, (deadline,)).fetchall() + + if not hits: + break + + for rd, fn in hits: + if rd.startswith("//") or fn.startswith("//"): + rd, fn = s3dec(rd, fn) + + fvp = "{}/{}".format(rd, fn).strip("/") + if vp: + fvp = "{}/{}".format(vp, fvp) + + self._handle_rm(LEELOO_DALLAS, None, fvp) + nrm += 1 + + if nrm: + self.log("{} files graduated in {}".format(nrm, vp)) def _vis_job_progress(self, job): perc = 100 - (len(job["need"]) * 100.0 / len(job["hash"]))