use parent db/thumbs in jump-volumes

README.md

@@ -37,6 +37,7 @@ turn your phone or raspi into a portable file server with resumable uploads/down
     * [other tricks](#other-tricks)
 * [searching](#searching)
     * [search configuration](#search-configuration)
+    * [database location](#database-location)
     * [metadata from audio files](#metadata-from-audio-files)
     * [file parser plugins](#file-parser-plugins)
     * [complete examples](#complete-examples)
@@ -126,7 +127,6 @@ summary: all planned features work! now please enjoy the bloatening
 
 * all volumes must exist / be available on startup; up2k (mtp especially) gets funky otherwise
 * cannot mount something at `/d1/d2/d3` unless `d2` exists inside `d1`
-* hiding the contents at url `/d1/d2/d3` using `-v :d1/d2/d3:cd2d` has the side-effect of creating databases (for files/tags) inside folders d1 and d2, and those databases take precedence over the main db at the top of the vfs - this means all files in d2 and below will be reindexed unless you already had a vfs entry at or below d2
 * probably more, pls let me know
 
 ## not my bugs

copyparty/authsrv.py

@@ -25,6 +25,7 @@ class VFS(object):
         self.flags = flags  # config switches
         self.nodes = {}  # child nodes
         self.histtab = None  # all realpath->histpath
+        self.dbv = None  # closest full/non-jump parent
 
         if realpath:
             self.histpath = os.path.join(realpath, ".hist")  # db / thumbcache
@@ -64,8 +65,9 @@ class VFS(object):
                 self.uread,
                 self.uwrite,
                 self.uadm,
-                self.flags,
+                self._copy_flags(name),
             )
+            vn.dbv = self.dbv or self
             self.nodes[name] = vn
             return vn.add(src, dst)
 
@@ -76,9 +78,27 @@ class VFS(object):
         # leaf does not exist; create and keep permissions blank
         vp = "{}/{}".format(self.vpath, dst).lstrip("/")
         vn = VFS(src, vp)
+        vn.dbv = self.dbv or self
         self.nodes[dst] = vn
         return vn
 
+    def _copy_flags(self, name):
+        flags = {k: v for k, v in self.flags.items()}
+        hist = flags.get("hist")
+        if hist and hist != "-":
+            flags["hist"] = "{}/{}".format(hist.rstrip("/"), name)
+
+        return flags
+
+    def bubble_flags(self):
+        if self.dbv:
+            for k, v in self.dbv.flags:
+                if k not in ["hist"]:
+                    self.flags[k] = v
+
+        for v in self.nodes.values():
+            v.bubble_flags()
+
     def _find(self, vpath):
         """return [vfs,remainder]"""
         vpath = undot(vpath)
@@ -117,6 +137,15 @@ class VFS(object):
 
         return vn, rem
 
+    def get_dbv(self, vrem):
+        dbv = self.dbv
+        if not dbv:
+            return self, vrem
+
+        vrem = [self.vpath[len(dbv.vpath) + 1 :], vrem]
+        vrem = "/".join([x for x in vrem if x])
+        return dbv, vrem
+
     def canonical(self, rem):
         """returns the canonical path (fully-resolved absolute fs path)"""
         rp = self.realpath
@@ -461,6 +490,7 @@ class AuthSrv(object):
             v.uwrite = mwrite[dst]
             v.uadm = madm[dst]
             v.flags = mflags[dst]
+            v.dbv = None
 
         vfs.all_vols = {}
         vfs.get_all_vols(vfs.all_vols)
@@ -480,6 +510,8 @@ class AuthSrv(object):
             )
             raise Exception("invalid config")
 
+        promote = []
+        demote = []
         for vol in vfs.all_vols.values():
             hid = hashlib.sha512(fsenc(vol.realpath)).digest()
             hid = base64.b32encode(hid).decode("ascii").lower()
@@ -517,6 +549,27 @@ class AuthSrv(object):
                     break
 
             vol.histpath = os.path.realpath(vol.histpath)
+            if vol.dbv:
+                if os.path.exists(os.path.join(vol.histpath, "up2k.db")):
+                    promote.append(vol)
+                    vol.dbv = None
+                else:
+                    demote.append(vol)
+
+        # discard jump-vols
+        for v in demote:
+            vfs.all_vols.pop(v.vpath)
+
+        if promote:
+            msg = [
+                "\n  the following jump-volumes were generated to assist the vfs.\n  As they contain a database (probably from v0.11.11 or older),\n  they are promoted to full volumes:"
+            ]
+            for vol in promote:
+                msg.append(
+                    "  /{}  ({})  ({})".format(vol.vpath, vol.realpath, vol.histpath)
+                )
+
+            self.log("\n\n".join(msg) + "\n", c=3)
 
         vfs.histtab = {v.realpath: v.histpath for v in vfs.all_vols.values()}
 
@@ -610,6 +663,8 @@ class AuthSrv(object):
         if errors:
             sys.exit(1)
 
+        vfs.bubble_flags()
+
         try:
             v, _ = vfs.get("/", "*", False, True)
             if self.warn_anonwrite and os.getcwd() == v.realpath:

copyparty/httpcli.py

@@ -443,8 +443,10 @@ class HttpCli(object):
         with open(fsenc(path), "wb", 512 * 1024) as f:
             post_sz, _, sha_b64 = hashcopy(self.conn, reader, f)
 
+        vfs, vrem = vfs.get_dbv(rem)
+
         self.conn.hsrv.broker.put(
-            False, "up2k.hash_file", vfs.realpath, vfs.flags, rem, fn
+            False, "up2k.hash_file", vfs.realpath, vfs.flags, vrem, fn
         )
 
         return post_sz, sha_b64, remains, path
@@ -551,12 +553,13 @@ class HttpCli(object):
             body["name"] = name
 
         vfs, rem = self.conn.auth.vfs.get(self.vpath, self.uname, False, True)
+        dbv, vrem = vfs.get_dbv(rem)
 
-        body["vtop"] = vfs.vpath
+        body["vtop"] = dbv.vpath
-        body["ptop"] = vfs.realpath
+        body["ptop"] = dbv.realpath
-        body["prel"] = rem
+        body["prel"] = vrem
         body["addr"] = self.ip
-        body["vcfg"] = vfs.flags
+        body["vcfg"] = dbv.flags
 
         if sub:
             try:
@@ -578,8 +581,14 @@ class HttpCli(object):
 
     def handle_search(self, body):
         vols = []
+        seen = {}
         for vtop in self.rvol:
             vfs, _ = self.conn.auth.vfs.get(vtop, self.uname, True, False)
+            vfs = vfs.dbv or vfs
+            if vfs in seen:
+                continue
+
+            seen[vfs] = True
             vols.append([vfs.vpath, vfs.realpath, vfs.flags])
 
         idx = self.conn.get_u2idx()
@@ -636,7 +645,7 @@ class HttpCli(object):
             raise Pebkac(400, "need hash and wark headers for binary POST")
 
         vfs, _ = self.conn.auth.vfs.get(self.vpath, self.uname, False, True)
-        ptop = vfs.realpath
+        ptop = (vfs.dbv or vfs).realpath
 
         x = self.conn.hsrv.broker.put(True, "up2k.handle_chunk", ptop, wark, chash)
         response = x.get()
@@ -817,8 +826,14 @@ class HttpCli(object):
                             raise Pebkac(400, "empty files in post")
 
                         files.append([sz, sha512_hex, p_file, fname])
+                        dbv, vrem = vfs.get_dbv(vrem)
                         self.conn.hsrv.broker.put(
-                            False, "up2k.hash_file", vfs.realpath, vfs.flags, rem, fname
+                            False,
+                            "up2k.hash_file",
+                            dbv.realpath,
+                            dbv.flags,
+                            vrem,
+                            fname,
                         )
                         self.conn.nbyte += sz
 
@@ -1483,6 +1498,7 @@ class HttpCli(object):
             self.vpath, self.uname, self.readable, self.writable
         )
         abspath = vn.canonical(rem)
+        dbv, vrem = vn.get_dbv(rem)
 
         try:
             st = os.stat(fsenc(abspath))
@@ -1497,7 +1513,9 @@ class HttpCli(object):
             if th_fmt is not None:
                 thp = None
                 if self.thumbcli:
-                    thp = self.thumbcli.get(vn.realpath, rem, int(st.st_mtime), th_fmt)
+                    thp = self.thumbcli.get(
+                        dbv.realpath, vrem, int(st.st_mtime), th_fmt
+                    )
 
                 if thp:
                     return self.tx_file(thp)
@@ -1640,7 +1658,7 @@ class HttpCli(object):
         icur = None
         if "e2t" in vn.flags:
             idx = self.conn.get_u2idx()
-            icur = idx.get_cur(vn.realpath)
+            icur = idx.get_cur(dbv.realpath)
 
         dirs = []
         files = []
@@ -1708,6 +1726,9 @@ class HttpCli(object):
             rd = f["rd"]
             del f["rd"]
             if icur:
+                if vn != dbv:
+                    _, rd = vn.get_dbv(rd)
+
                 q = "select w from up where rd = ? and fn = ?"
                 try:
                     r = icur.execute(q, (rd, fn)).fetchone()

tests/test_vfs.py

@@ -113,13 +113,13 @@ class TestVFS(unittest.TestCase):
         n = vfs.nodes["a"]
         self.assertEqual(len(vfs.nodes), 1)
         self.assertEqual(n.vpath, "a")
-        self.assertEqual(n.realpath, td + "/a")
+        self.assertEqual(n.realpath, os.path.join(td, "a"))
         self.assertEqual(n.uread, ["*", "k"])
         self.assertEqual(n.uwrite, ["k"])
         n = n.nodes["ac"]
         self.assertEqual(len(vfs.nodes), 1)
         self.assertEqual(n.vpath, "a/ac")
-        self.assertEqual(n.realpath, td + "/a/ac")
+        self.assertEqual(n.realpath, os.path.join(td, "a", "ac"))
         self.assertEqual(n.uread, ["*", "k"])
         self.assertEqual(n.uwrite, ["k"])
         n = n.nodes["acb"]