diff --git a/README.md b/README.md index f1f445c1..df2fc7c0 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,7 @@ turn your phone or raspi into a portable file server with resumable uploads/down * [other tricks](#other-tricks) * [searching](#searching) * [search configuration](#search-configuration) + * [database location](#database-location) * [metadata from audio files](#metadata-from-audio-files) * [file parser plugins](#file-parser-plugins) * [complete examples](#complete-examples) @@ -126,7 +127,6 @@ summary: all planned features work! now please enjoy the bloatening * all volumes must exist / be available on startup; up2k (mtp especially) gets funky otherwise * cannot mount something at `/d1/d2/d3` unless `d2` exists inside `d1` -* hiding the contents at url `/d1/d2/d3` using `-v :d1/d2/d3:cd2d` has the side-effect of creating databases (for files/tags) inside folders d1 and d2, and those databases take precedence over the main db at the top of the vfs - this means all files in d2 and below will be reindexed unless you already had a vfs entry at or below d2 * probably more, pls let me know ## not my bugs diff --git a/copyparty/authsrv.py b/copyparty/authsrv.py index 1281d720..5c83b68b 100644 --- a/copyparty/authsrv.py +++ b/copyparty/authsrv.py @@ -25,6 +25,7 @@ class VFS(object): self.flags = flags # config switches self.nodes = {} # child nodes self.histtab = None # all realpath->histpath + self.dbv = None # closest full/non-jump parent if realpath: self.histpath = os.path.join(realpath, ".hist") # db / thumbcache @@ -64,8 +65,9 @@ class VFS(object): self.uread, self.uwrite, self.uadm, - self.flags, + self._copy_flags(name), ) + vn.dbv = self.dbv or self self.nodes[name] = vn return vn.add(src, dst) @@ -76,9 +78,27 @@ class VFS(object): # leaf does not exist; create and keep permissions blank vp = "{}/{}".format(self.vpath, dst).lstrip("/") vn = VFS(src, vp) + vn.dbv = self.dbv or self self.nodes[dst] = vn return vn + def _copy_flags(self, name): + flags = {k: v for k, v in self.flags.items()} + hist = flags.get("hist") + if hist and hist != "-": + flags["hist"] = "{}/{}".format(hist.rstrip("/"), name) + + return flags + + def bubble_flags(self): + if self.dbv: + for k, v in self.dbv.flags: + if k not in ["hist"]: + self.flags[k] = v + + for v in self.nodes.values(): + v.bubble_flags() + def _find(self, vpath): """return [vfs,remainder]""" vpath = undot(vpath) @@ -117,6 +137,15 @@ class VFS(object): return vn, rem + def get_dbv(self, vrem): + dbv = self.dbv + if not dbv: + return self, vrem + + vrem = [self.vpath[len(dbv.vpath) + 1 :], vrem] + vrem = "/".join([x for x in vrem if x]) + return dbv, vrem + def canonical(self, rem): """returns the canonical path (fully-resolved absolute fs path)""" rp = self.realpath @@ -461,6 +490,7 @@ class AuthSrv(object): v.uwrite = mwrite[dst] v.uadm = madm[dst] v.flags = mflags[dst] + v.dbv = None vfs.all_vols = {} vfs.get_all_vols(vfs.all_vols) @@ -480,6 +510,8 @@ class AuthSrv(object): ) raise Exception("invalid config") + promote = [] + demote = [] for vol in vfs.all_vols.values(): hid = hashlib.sha512(fsenc(vol.realpath)).digest() hid = base64.b32encode(hid).decode("ascii").lower() @@ -517,6 +549,27 @@ class AuthSrv(object): break vol.histpath = os.path.realpath(vol.histpath) + if vol.dbv: + if os.path.exists(os.path.join(vol.histpath, "up2k.db")): + promote.append(vol) + vol.dbv = None + else: + demote.append(vol) + + # discard jump-vols + for v in demote: + vfs.all_vols.pop(v.vpath) + + if promote: + msg = [ + "\n the following jump-volumes were generated to assist the vfs.\n As they contain a database (probably from v0.11.11 or older),\n they are promoted to full volumes:" + ] + for vol in promote: + msg.append( + " /{} ({}) ({})".format(vol.vpath, vol.realpath, vol.histpath) + ) + + self.log("\n\n".join(msg) + "\n", c=3) vfs.histtab = {v.realpath: v.histpath for v in vfs.all_vols.values()} @@ -610,6 +663,8 @@ class AuthSrv(object): if errors: sys.exit(1) + vfs.bubble_flags() + try: v, _ = vfs.get("/", "*", False, True) if self.warn_anonwrite and os.getcwd() == v.realpath: diff --git a/copyparty/httpcli.py b/copyparty/httpcli.py index a9187eef..346e1152 100644 --- a/copyparty/httpcli.py +++ b/copyparty/httpcli.py @@ -443,8 +443,10 @@ class HttpCli(object): with open(fsenc(path), "wb", 512 * 1024) as f: post_sz, _, sha_b64 = hashcopy(self.conn, reader, f) + vfs, vrem = vfs.get_dbv(rem) + self.conn.hsrv.broker.put( - False, "up2k.hash_file", vfs.realpath, vfs.flags, rem, fn + False, "up2k.hash_file", vfs.realpath, vfs.flags, vrem, fn ) return post_sz, sha_b64, remains, path @@ -551,12 +553,13 @@ class HttpCli(object): body["name"] = name vfs, rem = self.conn.auth.vfs.get(self.vpath, self.uname, False, True) + dbv, vrem = vfs.get_dbv(rem) - body["vtop"] = vfs.vpath - body["ptop"] = vfs.realpath - body["prel"] = rem + body["vtop"] = dbv.vpath + body["ptop"] = dbv.realpath + body["prel"] = vrem body["addr"] = self.ip - body["vcfg"] = vfs.flags + body["vcfg"] = dbv.flags if sub: try: @@ -578,8 +581,14 @@ class HttpCli(object): def handle_search(self, body): vols = [] + seen = {} for vtop in self.rvol: vfs, _ = self.conn.auth.vfs.get(vtop, self.uname, True, False) + vfs = vfs.dbv or vfs + if vfs in seen: + continue + + seen[vfs] = True vols.append([vfs.vpath, vfs.realpath, vfs.flags]) idx = self.conn.get_u2idx() @@ -636,7 +645,7 @@ class HttpCli(object): raise Pebkac(400, "need hash and wark headers for binary POST") vfs, _ = self.conn.auth.vfs.get(self.vpath, self.uname, False, True) - ptop = vfs.realpath + ptop = (vfs.dbv or vfs).realpath x = self.conn.hsrv.broker.put(True, "up2k.handle_chunk", ptop, wark, chash) response = x.get() @@ -817,8 +826,14 @@ class HttpCli(object): raise Pebkac(400, "empty files in post") files.append([sz, sha512_hex, p_file, fname]) + dbv, vrem = vfs.get_dbv(vrem) self.conn.hsrv.broker.put( - False, "up2k.hash_file", vfs.realpath, vfs.flags, rem, fname + False, + "up2k.hash_file", + dbv.realpath, + dbv.flags, + vrem, + fname, ) self.conn.nbyte += sz @@ -1483,6 +1498,7 @@ class HttpCli(object): self.vpath, self.uname, self.readable, self.writable ) abspath = vn.canonical(rem) + dbv, vrem = vn.get_dbv(rem) try: st = os.stat(fsenc(abspath)) @@ -1497,7 +1513,9 @@ class HttpCli(object): if th_fmt is not None: thp = None if self.thumbcli: - thp = self.thumbcli.get(vn.realpath, rem, int(st.st_mtime), th_fmt) + thp = self.thumbcli.get( + dbv.realpath, vrem, int(st.st_mtime), th_fmt + ) if thp: return self.tx_file(thp) @@ -1640,7 +1658,7 @@ class HttpCli(object): icur = None if "e2t" in vn.flags: idx = self.conn.get_u2idx() - icur = idx.get_cur(vn.realpath) + icur = idx.get_cur(dbv.realpath) dirs = [] files = [] @@ -1708,6 +1726,9 @@ class HttpCli(object): rd = f["rd"] del f["rd"] if icur: + if vn != dbv: + _, rd = vn.get_dbv(rd) + q = "select w from up where rd = ? and fn = ?" try: r = icur.execute(q, (rd, fn)).fetchone() diff --git a/tests/test_vfs.py b/tests/test_vfs.py index 625c6c6f..81977c2b 100644 --- a/tests/test_vfs.py +++ b/tests/test_vfs.py @@ -113,13 +113,13 @@ class TestVFS(unittest.TestCase): n = vfs.nodes["a"] self.assertEqual(len(vfs.nodes), 1) self.assertEqual(n.vpath, "a") - self.assertEqual(n.realpath, td + "/a") + self.assertEqual(n.realpath, os.path.join(td, "a")) self.assertEqual(n.uread, ["*", "k"]) self.assertEqual(n.uwrite, ["k"]) n = n.nodes["ac"] self.assertEqual(len(vfs.nodes), 1) self.assertEqual(n.vpath, "a/ac") - self.assertEqual(n.realpath, td + "/a/ac") + self.assertEqual(n.realpath, os.path.join(td, "a", "ac")) self.assertEqual(n.uread, ["*", "k"]) self.assertEqual(n.uwrite, ["k"]) n = n.nodes["acb"]