From 9b7b9262aaf7197b0954754d9872ad2822d4b09e Mon Sep 17 00:00:00 2001 From: ed Date: Wed, 25 Jan 2023 21:46:15 +0000 Subject: [PATCH] promote dedup control to volflags --- copyparty/__main__.py | 9 ++++++--- copyparty/authsrv.py | 6 ++++++ copyparty/svchub.py | 3 --- copyparty/up2k.py | 27 +++++++++++++++++---------- 4 files changed, 29 insertions(+), 16 deletions(-) diff --git a/copyparty/__main__.py b/copyparty/__main__.py index c0f74d05..a020b2f8 100755 --- a/copyparty/__main__.py +++ b/copyparty/__main__.py @@ -498,6 +498,9 @@ def get_sects(): \033[0muploads, general: \033[36mnodupe\033[35m rejects existing files (instead of symlinking them) + \033[36mhardlink\033[35m does dedup with hardlinks instead of symlinks + \033[36mneversymlink\033[35m disables symlink fallback; full copy instead + \033[36mcopydupes\033[35m disables dedup, always saves full copies of dupes \033[36mnosub\033[35m forces all uploads into the top folder of the vfs \033[36mmagic$\033[35m enables filetype detection for nameless uploads \033[36mgz\033[35m allows server-side gzip of uploads with ?gz (also c,xz) @@ -695,9 +698,9 @@ def add_upload(ap): ap2.add_argument("--reg-cap", metavar="N", type=int, default=38400, help="max number of uploads to keep in memory when running without -e2d; roughly 1 MiB RAM per 600") ap2.add_argument("--no-fpool", action="store_true", help="disable file-handle pooling -- instead, repeatedly close and reopen files during upload (very slow on windows)") ap2.add_argument("--use-fpool", action="store_true", help="force file-handle pooling, even when it might be dangerous (multiprocessing, filesystems lacking sparse-files support, ...)") - ap2.add_argument("--hardlink", action="store_true", help="prefer hardlinks instead of symlinks when possible (within same filesystem)") - ap2.add_argument("--never-symlink", action="store_true", help="do not fallback to symlinks when a hardlink cannot be made") - ap2.add_argument("--no-dedup", action="store_true", help="disable symlink/hardlink creation; copy file contents instead") + ap2.add_argument("--hardlink", action="store_true", help="prefer hardlinks instead of symlinks when possible (within same filesystem) (volflag=hardlink)") + ap2.add_argument("--never-symlink", action="store_true", help="do not fallback to symlinks when a hardlink cannot be made (volflag=neversymlink)") + ap2.add_argument("--no-dedup", action="store_true", help="disable symlink/hardlink creation; copy file contents instead (volflag=copydupes") ap2.add_argument("--no-dupe", action="store_true", help="reject duplicate files during upload; only matches within the same volume (volflag=nodupe)") ap2.add_argument("--no-snap", action="store_true", help="disable snapshots -- forget unfinished uploads on shutdown; don't create .hist/up2k.snap files -- abandoned/interrupted uploads must be cleaned up manually") ap2.add_argument("--magic", action="store_true", help="enable filetype detection on nameless uploads (volflag=magic)") diff --git a/copyparty/authsrv.py b/copyparty/authsrv.py index f943b5c1..75751420 100644 --- a/copyparty/authsrv.py +++ b/copyparty/authsrv.py @@ -1121,6 +1121,9 @@ class AuthSrv(object): for ga, vf in ( ("no_forget", "noforget"), ("no_dupe", "nodupe"), + ("hardlink", "hardlink"), + ("never_symlink", "neversymlink"), + ("no_dedup", "copydupes"), ("magic", "magic"), ("xlink", "xlink"), ): @@ -1217,6 +1220,9 @@ class AuthSrv(object): self.log(t.format(vol.vpath), 1) del vol.flags["lifetime"] + if vol.flags.get("neversymlink") and not vol.flags.get("hardlink"): + vol.flags["copydupes"] = True + # verify tags mentioned by -mt[mp] are used by -mte local_mtp = {} local_only_mtp = {} diff --git a/copyparty/svchub.py b/copyparty/svchub.py index 3029d446..263206ee 100644 --- a/copyparty/svchub.py +++ b/copyparty/svchub.py @@ -151,9 +151,6 @@ class SvcHub(object): ch = "abcdefghijklmnopqrstuvwx"[int(args.theme / 2)] args.theme = "{0}{1} {0} {1}".format(ch, bri) - if not args.hardlink and args.never_symlink: - args.no_dedup = True - if args.log_fk: args.log_fk = re.compile(args.log_fk) diff --git a/copyparty/up2k.py b/copyparty/up2k.py index b83da3bf..8ea3bd2d 100644 --- a/copyparty/up2k.py +++ b/copyparty/up2k.py @@ -2159,7 +2159,8 @@ class Up2k(object): if not self.args.nw: bos.unlink(dst) # TODO ed pls try: - self._symlink(src, dst, lmod=cj["lmod"]) + dst_flags = self.flags[job["ptop"]] + self._symlink(src, dst, dst_flags, lmod=cj["lmod"]) except: if not n4g: raise @@ -2257,7 +2258,12 @@ class Up2k(object): return zfw["orz"][1] def _symlink( - self, src: str, dst: str, verbose: bool = True, lmod: float = 0 + self, + src: str, + dst: str, + flags: dict[str, Any], + verbose: bool = True, + lmod: float = 0, ) -> None: if verbose: self.log("linking dupe:\n {0}\n {1}".format(src, dst)) @@ -2267,7 +2273,7 @@ class Up2k(object): linked = False try: - if self.args.no_dedup: + if "copydupes" in flags: raise Exception("disabled in config") lsrc = src @@ -2297,12 +2303,12 @@ class Up2k(object): ldst = ldst.replace("/", "\\") try: - if self.args.hardlink: + if "hardlink" in flags: os.link(fsenc(src), fsenc(dst)) linked = True except Exception as ex: self.log("cannot hardlink: " + repr(ex)) - if self.args.never_symlink: + if "neversymlink" in flags: raise Exception("symlink-fallback disabled in cfg") if not linked: @@ -2485,7 +2491,7 @@ class Up2k(object): if os.path.exists(d2): continue - self._symlink(dst, d2, lmod=lmod) + self._symlink(dst, d2, self.flags[ptop], lmod=lmod) if cur: self.db_rm(cur, rd, fn) self.db_add(cur, wark, rd, fn, *z2[-4:]) @@ -2790,7 +2796,7 @@ class Up2k(object): self.log(t.format(sabs, dabs, dlabs)) mt = bos.path.getmtime(sabs, False) bos.unlink(sabs) - self._symlink(dlabs, dabs, False, lmod=mt) + self._symlink(dlabs, dabs, dvn.flags, False, lmod=mt) # folders are too scary, schedule rescan of both vols self.need_rescan.add(svn.vpath) @@ -2972,14 +2978,14 @@ class Up2k(object): bos.unlink(slabs) bos.rename(sabs, slabs) bos.utime(slabs, (int(time.time()), int(mt)), False) - self._symlink(slabs, sabs, False) + self._symlink(slabs, sabs, self.flags.get(ptop) or {}, False) full[slabs] = (ptop, rem) sabs = slabs if not dabs: dabs = list(sorted(full.keys()))[0] - for alink in links: + for alink, parts in links.items(): lmod = None try: if alink != sabs and absreal(alink) != sabs: @@ -2991,7 +2997,8 @@ class Up2k(object): except: pass - self._symlink(dabs, alink, False, lmod=lmod or 0) + flags = self.flags.get(parts[0]) or {} + self._symlink(dabs, alink, flags, False, lmod=lmod or 0) return len(full) + len(links)