From 11d1267f8c284de82e38ecf87f684733cbe5fb5d Mon Sep 17 00:00:00 2001
From: ed <s@ocv.me>
Date: Wed, 7 Sep 2022 01:07:21 +0200
Subject: [PATCH] option to keep files in index when deleted

---
 copyparty/__main__.py |  2 ++
 copyparty/authsrv.py  |  4 ++++
 copyparty/up2k.py     | 21 ++++++++++++++++-----
 3 files changed, 22 insertions(+), 5 deletions(-)

diff --git a/copyparty/__main__.py b/copyparty/__main__.py
index fb4463b6..15f09337 100644
--- a/copyparty/__main__.py
+++ b/copyparty/__main__.py
@@ -413,6 +413,7 @@ def run_argparse(argv: list[str], formatter: Any, retry: bool) -> argparse.Names
               \033[36mscan=60\033[35m scan for new files every 60sec, same as --re-maxage
               \033[36mnohash=\\.iso$\033[35m skips hashing file contents if path matches *.iso
               \033[36mnoidx=\\.iso$\033[35m fully ignores the contents at paths matching *.iso
+              \033[36mnoforget$\033[35m don't forget files when deleted from disk
               \033[36mxdev\033[35m do not descend into other filesystems
               \033[36mxvol\033[35m skip symlinks leaving the volume root
 
@@ -611,6 +612,7 @@ def run_argparse(argv: list[str], formatter: Any, retry: bool) -> argparse.Names
     ap2.add_argument("--no-hash", metavar="PTN", type=u, help="regex: disable hashing of matching paths during e2ds folder scans")
     ap2.add_argument("--no-idx", metavar="PTN", type=u, help="regex: disable indexing of matching paths during e2ds folder scans")
     ap2.add_argument("--no-dhash", action="store_true", help="disable rescan acceleration; do full database integrity check -- makes the db ~5%% smaller and bootup/rescans 3~10x slower")
+    ap2.add_argument("--no-forget", action="store_true", help="never forget indexed files, even when deleted from disk -- makes it impossible to ever upload the same file twice")
     ap2.add_argument("--xdev", action="store_true", help="do not descend into other filesystems (symlink or bind-mount to another HDD, ...)")
     ap2.add_argument("--xvol", action="store_true", help="skip symlinks leaving the volume root")
     ap2.add_argument("--hash-mt", metavar="CORES", type=int, default=hcores, help="num cpu cores to use for file hashing; set 0 or 1 for single-core hashing")
diff --git a/copyparty/authsrv.py b/copyparty/authsrv.py
index 8b7508ee..d10cc075 100644
--- a/copyparty/authsrv.py
+++ b/copyparty/authsrv.py
@@ -1071,6 +1071,10 @@ class AuthSrv(object):
                 if getattr(self.args, k):
                     vol.flags[k] = True
 
+            for ga, vf in [["no_forget", "noforget"]]:
+                if getattr(self.args, ga):
+                    vol.flags[vf] = True
+
             for k1, k2 in IMPLICATIONS:
                 if k1 in vol.flags:
                     vol.flags[k2] = True
diff --git a/copyparty/up2k.py b/copyparty/up2k.py
index 2527ca47..b50699d2 100644
--- a/copyparty/up2k.py
+++ b/copyparty/up2k.py
@@ -683,6 +683,7 @@ class Up2k(object):
         top = vol.realpath
         rei = vol.flags.get("noidx")
         reh = vol.flags.get("nohash")
+        n4g = bool(vol.flags.get("noforget"))
 
         dev = 0
         if vol.flags.get("xdev"):
@@ -720,11 +721,13 @@ class Up2k(object):
                     rtop,
                     rei,
                     reh,
+                    n4g,
                     [],
                     dev,
                     bool(vol.flags.get("xvol")),
                 )
-                n_rm = self._drop_lost(db.c, top, excl)
+                if not n4g:
+                    n_rm = self._drop_lost(db.c, top, excl)
             except Exception as ex:
                 t = "failed to index volume [{}]:\n{}"
                 self.log(t.format(top, min_ex()), c=1)
@@ -754,6 +757,7 @@ class Up2k(object):
         rcdir: str,
         rei: Optional[Pattern[str]],
         reh: Optional[Pattern[str]],
+        n4g: bool,
         seen: list[str],
         dev: int,
         xvol: bool,
@@ -809,7 +813,7 @@ class Up2k(object):
                 # self.log(" dir: {}".format(abspath))
                 try:
                     ret += self._build_dir(
-                        db, top, excl, abspath, rap, rei, reh, seen, dev, xvol
+                        db, top, excl, abspath, rap, rei, reh, n4g, seen, dev, xvol
                     )
                 except:
                     t = "failed to index subdir [{}]:\n{}"
@@ -953,6 +957,9 @@ class Up2k(object):
                 db.c.execute(q, (erd, erd + "/"))
                 ret += n
 
+        if n4g:
+            return ret
+
         # drop missing files
         q = "select fn from up where rd = ?"
         try:
@@ -1910,6 +1917,8 @@ class Up2k(object):
         with self.mutex:
             cur = self.cur.get(cj["ptop"])
             reg = self.registry[cj["ptop"]]
+            vfs = self.asrv.vfs.all_vols[cj["vtop"]]
+            n4g = vfs.flags.get("noforget")
             if cur:
                 if self.no_expr_idx:
                     q = r"select * from up where w = ?"
@@ -1931,7 +1940,10 @@ class Up2k(object):
                             # broken symlink
                             raise Exception()
                     except:
-                        continue
+                        if n4g:
+                            st = os.stat_result((0, -1, -1, 0, 0, 0, 0, 0, 0, 0))
+                        else:
+                            continue
 
                     j = {
                         "name": dp_fn,
@@ -1972,7 +1984,7 @@ class Up2k(object):
                                 break
                         except:
                             # missing; restart
-                            if not self.args.nw:
+                            if not self.args.nw and not n4g:
                                 job = None
                             break
                 else:
@@ -2023,7 +2035,6 @@ class Up2k(object):
                             cur.connection.commit()
 
             if not job:
-                vfs = self.asrv.vfs.all_vols[cj["vtop"]]
                 if vfs.lim:
                     ap1 = djoin(cj["ptop"], cj["prel"])
                     ap2, cj["prel"] = vfs.lim.all(