promote dedup control to volflags

copyparty/__main__.py

@@ -498,6 +498,9 @@ def get_sects():
 
             \033[0muploads, general:
               \033[36mnodupe\033[35m rejects existing files (instead of symlinking them)
+              \033[36mhardlink\033[35m does dedup with hardlinks instead of symlinks
+              \033[36mneversymlink\033[35m disables symlink fallback; full copy instead
+              \033[36mcopydupes\033[35m disables dedup, always saves full copies of dupes
               \033[36mnosub\033[35m forces all uploads into the top folder of the vfs
               \033[36mmagic$\033[35m enables filetype detection for nameless uploads
               \033[36mgz\033[35m allows server-side gzip of uploads with ?gz (also c,xz)
@@ -695,9 +698,9 @@ def add_upload(ap):
     ap2.add_argument("--reg-cap", metavar="N", type=int, default=38400, help="max number of uploads to keep in memory when running without -e2d; roughly 1 MiB RAM per 600")
     ap2.add_argument("--no-fpool", action="store_true", help="disable file-handle pooling -- instead, repeatedly close and reopen files during upload (very slow on windows)")
     ap2.add_argument("--use-fpool", action="store_true", help="force file-handle pooling, even when it might be dangerous (multiprocessing, filesystems lacking sparse-files support, ...)")
-    ap2.add_argument("--hardlink", action="store_true", help="prefer hardlinks instead of symlinks when possible (within same filesystem)")
+    ap2.add_argument("--hardlink", action="store_true", help="prefer hardlinks instead of symlinks when possible (within same filesystem) (volflag=hardlink)")
-    ap2.add_argument("--never-symlink", action="store_true", help="do not fallback to symlinks when a hardlink cannot be made")
+    ap2.add_argument("--never-symlink", action="store_true", help="do not fallback to symlinks when a hardlink cannot be made (volflag=neversymlink)")
-    ap2.add_argument("--no-dedup", action="store_true", help="disable symlink/hardlink creation; copy file contents instead")
+    ap2.add_argument("--no-dedup", action="store_true", help="disable symlink/hardlink creation; copy file contents instead (volflag=copydupes")
     ap2.add_argument("--no-dupe", action="store_true", help="reject duplicate files during upload; only matches within the same volume (volflag=nodupe)")
     ap2.add_argument("--no-snap", action="store_true", help="disable snapshots -- forget unfinished uploads on shutdown; don't create .hist/up2k.snap files -- abandoned/interrupted uploads must be cleaned up manually")
     ap2.add_argument("--magic", action="store_true", help="enable filetype detection on nameless uploads (volflag=magic)")

copyparty/authsrv.py

@@ -1121,6 +1121,9 @@ class AuthSrv(object):
             for ga, vf in (
                 ("no_forget", "noforget"),
                 ("no_dupe", "nodupe"),
+                ("hardlink", "hardlink"),
+                ("never_symlink", "neversymlink"),
+                ("no_dedup", "copydupes"),
                 ("magic", "magic"),
                 ("xlink", "xlink"),
             ):
@@ -1217,6 +1220,9 @@ class AuthSrv(object):
                 self.log(t.format(vol.vpath), 1)
                 del vol.flags["lifetime"]
 
+            if vol.flags.get("neversymlink") and not vol.flags.get("hardlink"):
+                vol.flags["copydupes"] = True
+
             # verify tags mentioned by -mt[mp] are used by -mte
             local_mtp = {}
             local_only_mtp = {}

copyparty/svchub.py

@@ -151,9 +151,6 @@ class SvcHub(object):
         ch = "abcdefghijklmnopqrstuvwx"[int(args.theme / 2)]
         args.theme = "{0}{1} {0} {1}".format(ch, bri)
 
-        if not args.hardlink and args.never_symlink:
-            args.no_dedup = True
-
         if args.log_fk:
             args.log_fk = re.compile(args.log_fk)
 

copyparty/up2k.py

@@ -2159,7 +2159,8 @@ class Up2k(object):
                         if not self.args.nw:
                             bos.unlink(dst)  # TODO ed pls
                             try:
-                                self._symlink(src, dst, lmod=cj["lmod"])
+                                dst_flags = self.flags[job["ptop"]]
+                                self._symlink(src, dst, dst_flags, lmod=cj["lmod"])
                             except:
                                 if not n4g:
                                     raise
@@ -2257,7 +2258,12 @@ class Up2k(object):
             return zfw["orz"][1]
 
     def _symlink(
-        self, src: str, dst: str, verbose: bool = True, lmod: float = 0
+        self,
+        src: str,
+        dst: str,
+        flags: dict[str, Any],
+        verbose: bool = True,
+        lmod: float = 0,
     ) -> None:
         if verbose:
             self.log("linking dupe:\n  {0}\n  {1}".format(src, dst))
@@ -2267,7 +2273,7 @@ class Up2k(object):
 
         linked = False
         try:
-            if self.args.no_dedup:
+            if "copydupes" in flags:
                 raise Exception("disabled in config")
 
             lsrc = src
@@ -2297,12 +2303,12 @@ class Up2k(object):
                 ldst = ldst.replace("/", "\\")
 
             try:
-                if self.args.hardlink:
+                if "hardlink" in flags:
                     os.link(fsenc(src), fsenc(dst))
                     linked = True
             except Exception as ex:
                 self.log("cannot hardlink: " + repr(ex))
-                if self.args.never_symlink:
+                if "neversymlink" in flags:
                     raise Exception("symlink-fallback disabled in cfg")
 
             if not linked:
@@ -2485,7 +2491,7 @@ class Up2k(object):
             if os.path.exists(d2):
                 continue
 
-            self._symlink(dst, d2, lmod=lmod)
+            self._symlink(dst, d2, self.flags[ptop], lmod=lmod)
             if cur:
                 self.db_rm(cur, rd, fn)
                 self.db_add(cur, wark, rd, fn, *z2[-4:])
@@ -2790,7 +2796,7 @@ class Up2k(object):
             self.log(t.format(sabs, dabs, dlabs))
             mt = bos.path.getmtime(sabs, False)
             bos.unlink(sabs)
-            self._symlink(dlabs, dabs, False, lmod=mt)
+            self._symlink(dlabs, dabs, dvn.flags, False, lmod=mt)
 
             # folders are too scary, schedule rescan of both vols
             self.need_rescan.add(svn.vpath)
@@ -2972,14 +2978,14 @@ class Up2k(object):
             bos.unlink(slabs)
             bos.rename(sabs, slabs)
             bos.utime(slabs, (int(time.time()), int(mt)), False)
-            self._symlink(slabs, sabs, False)
+            self._symlink(slabs, sabs, self.flags.get(ptop) or {}, False)
             full[slabs] = (ptop, rem)
             sabs = slabs
 
         if not dabs:
             dabs = list(sorted(full.keys()))[0]
 
-        for alink in links:
+        for alink, parts in links.items():
             lmod = None
             try:
                 if alink != sabs and absreal(alink) != sabs:
@@ -2991,7 +2997,8 @@ class Up2k(object):
             except:
                 pass
 
-            self._symlink(dabs, alink, False, lmod=lmod or 0)
+            flags = self.flags.get(parts[0]) or {}
+            self._symlink(dabs, alink, flags, False, lmod=lmod or 0)
 
         return len(full) + len(links)