mirror of
https://github.com/9001/copyparty.git
synced 2025-08-17 09:02:15 -06:00
use parent db/thumbs in jump-volumes
This commit is contained in:
ed
parent
1078d933b4
commit
5e03b3ca38
|
|
@ -37,6 +37,7 @@ turn your phone or raspi into a portable file server with resumable uploads/down
|
|||
* [other tricks](#other-tricks)
|
||||
* [searching](#searching)
|
||||
* [search configuration](#search-configuration)
|
||||
* [database location](#database-location)
|
||||
* [metadata from audio files](#metadata-from-audio-files)
|
||||
* [file parser plugins](#file-parser-plugins)
|
||||
* [complete examples](#complete-examples)
|
||||
|
|
@ -126,7 +127,6 @@ summary: all planned features work! now please enjoy the bloatening
|
|||
|
||||
* all volumes must exist / be available on startup; up2k (mtp especially) gets funky otherwise
|
||||
* cannot mount something at `/d1/d2/d3` unless `d2` exists inside `d1`
|
||||
* hiding the contents at url `/d1/d2/d3` using `-v :d1/d2/d3:cd2d` has the side-effect of creating databases (for files/tags) inside folders d1 and d2, and those databases take precedence over the main db at the top of the vfs - this means all files in d2 and below will be reindexed unless you already had a vfs entry at or below d2
|
||||
* probably more, pls let me know
|
||||
|
||||
## not my bugs
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ class VFS(object):
|
|||
self.flags = flags # config switches
|
||||
self.nodes = {} # child nodes
|
||||
self.histtab = None # all realpath->histpath
|
||||
self.dbv = None # closest full/non-jump parent
|
||||
|
||||
if realpath:
|
||||
self.histpath = os.path.join(realpath, ".hist") # db / thumbcache
|
||||
|
|
@ -64,8 +65,9 @@ class VFS(object):
|
|||
self.uread,
|
||||
self.uwrite,
|
||||
self.uadm,
|
||||
self.flags,
|
||||
self._copy_flags(name),
|
||||
)
|
||||
vn.dbv = self.dbv or self
|
||||
self.nodes[name] = vn
|
||||
return vn.add(src, dst)
|
||||
|
||||
|
|
@ -76,9 +78,27 @@ class VFS(object):
|
|||
# leaf does not exist; create and keep permissions blank
|
||||
vp = "{}/{}".format(self.vpath, dst).lstrip("/")
|
||||
vn = VFS(src, vp)
|
||||
vn.dbv = self.dbv or self
|
||||
self.nodes[dst] = vn
|
||||
return vn
|
||||
|
||||
def _copy_flags(self, name):
|
||||
flags = {k: v for k, v in self.flags.items()}
|
||||
hist = flags.get("hist")
|
||||
if hist and hist != "-":
|
||||
flags["hist"] = "{}/{}".format(hist.rstrip("/"), name)
|
||||
|
||||
return flags
|
||||
|
||||
def bubble_flags(self):
|
||||
if self.dbv:
|
||||
for k, v in self.dbv.flags:
|
||||
if k not in ["hist"]:
|
||||
self.flags[k] = v
|
||||
|
||||
for v in self.nodes.values():
|
||||
v.bubble_flags()
|
||||
|
||||
def _find(self, vpath):
|
||||
"""return [vfs,remainder]"""
|
||||
vpath = undot(vpath)
|
||||
|
|
@ -117,6 +137,15 @@ class VFS(object):
|
|||
|
||||
return vn, rem
|
||||
|
||||
def get_dbv(self, vrem):
|
||||
dbv = self.dbv
|
||||
if not dbv:
|
||||
return self, vrem
|
||||
|
||||
vrem = [self.vpath[len(dbv.vpath) + 1 :], vrem]
|
||||
vrem = "/".join([x for x in vrem if x])
|
||||
return dbv, vrem
|
||||
|
||||
def canonical(self, rem):
|
||||
"""returns the canonical path (fully-resolved absolute fs path)"""
|
||||
rp = self.realpath
|
||||
|
|
@ -461,6 +490,7 @@ class AuthSrv(object):
|
|||
v.uwrite = mwrite[dst]
|
||||
v.uadm = madm[dst]
|
||||
v.flags = mflags[dst]
|
||||
v.dbv = None
|
||||
|
||||
vfs.all_vols = {}
|
||||
vfs.get_all_vols(vfs.all_vols)
|
||||
|
|
@ -480,6 +510,8 @@ class AuthSrv(object):
|
|||
)
|
||||
raise Exception("invalid config")
|
||||
|
||||
promote = []
|
||||
demote = []
|
||||
for vol in vfs.all_vols.values():
|
||||
hid = hashlib.sha512(fsenc(vol.realpath)).digest()
|
||||
hid = base64.b32encode(hid).decode("ascii").lower()
|
||||
|
|
@ -517,6 +549,27 @@ class AuthSrv(object):
|
|||
break
|
||||
|
||||
vol.histpath = os.path.realpath(vol.histpath)
|
||||
if vol.dbv:
|
||||
if os.path.exists(os.path.join(vol.histpath, "up2k.db")):
|
||||
promote.append(vol)
|
||||
vol.dbv = None
|
||||
else:
|
||||
demote.append(vol)
|
||||
|
||||
# discard jump-vols
|
||||
for v in demote:
|
||||
vfs.all_vols.pop(v.vpath)
|
||||
|
||||
if promote:
|
||||
msg = [
|
||||
"\n the following jump-volumes were generated to assist the vfs.\n As they contain a database (probably from v0.11.11 or older),\n they are promoted to full volumes:"
|
||||
]
|
||||
for vol in promote:
|
||||
msg.append(
|
||||
" /{} ({}) ({})".format(vol.vpath, vol.realpath, vol.histpath)
|
||||
)
|
||||
|
||||
self.log("\n\n".join(msg) + "\n", c=3)
|
||||
|
||||
vfs.histtab = {v.realpath: v.histpath for v in vfs.all_vols.values()}
|
||||
|
||||
|
|
@ -610,6 +663,8 @@ class AuthSrv(object):
|
|||
if errors:
|
||||
sys.exit(1)
|
||||
|
||||
vfs.bubble_flags()
|
||||
|
||||
try:
|
||||
v, _ = vfs.get("/", "*", False, True)
|
||||
if self.warn_anonwrite and os.getcwd() == v.realpath:
|
||||
|
|
|
|||
|
|
@ -443,8 +443,10 @@ class HttpCli(object):
|
|||
with open(fsenc(path), "wb", 512 * 1024) as f:
|
||||
post_sz, _, sha_b64 = hashcopy(self.conn, reader, f)
|
||||
|
||||
vfs, vrem = vfs.get_dbv(rem)
|
||||
|
||||
self.conn.hsrv.broker.put(
|
||||
False, "up2k.hash_file", vfs.realpath, vfs.flags, rem, fn
|
||||
False, "up2k.hash_file", vfs.realpath, vfs.flags, vrem, fn
|
||||
)
|
||||
|
||||
return post_sz, sha_b64, remains, path
|
||||
|
|
@ -551,12 +553,13 @@ class HttpCli(object):
|
|||
body["name"] = name
|
||||
|
||||
vfs, rem = self.conn.auth.vfs.get(self.vpath, self.uname, False, True)
|
||||
dbv, vrem = vfs.get_dbv(rem)
|
||||
|
||||
body["vtop"] = vfs.vpath
|
||||
body["ptop"] = vfs.realpath
|
||||
body["prel"] = rem
|
||||
body["vtop"] = dbv.vpath
|
||||
body["ptop"] = dbv.realpath
|
||||
body["prel"] = vrem
|
||||
body["addr"] = self.ip
|
||||
body["vcfg"] = vfs.flags
|
||||
body["vcfg"] = dbv.flags
|
||||
|
||||
if sub:
|
||||
try:
|
||||
|
|
@ -578,8 +581,14 @@ class HttpCli(object):
|
|||
|
||||
def handle_search(self, body):
|
||||
vols = []
|
||||
seen = {}
|
||||
for vtop in self.rvol:
|
||||
vfs, _ = self.conn.auth.vfs.get(vtop, self.uname, True, False)
|
||||
vfs = vfs.dbv or vfs
|
||||
if vfs in seen:
|
||||
continue
|
||||
|
||||
seen[vfs] = True
|
||||
vols.append([vfs.vpath, vfs.realpath, vfs.flags])
|
||||
|
||||
idx = self.conn.get_u2idx()
|
||||
|
|
@ -636,7 +645,7 @@ class HttpCli(object):
|
|||
raise Pebkac(400, "need hash and wark headers for binary POST")
|
||||
|
||||
vfs, _ = self.conn.auth.vfs.get(self.vpath, self.uname, False, True)
|
||||
ptop = vfs.realpath
|
||||
ptop = (vfs.dbv or vfs).realpath
|
||||
|
||||
x = self.conn.hsrv.broker.put(True, "up2k.handle_chunk", ptop, wark, chash)
|
||||
response = x.get()
|
||||
|
|
@ -817,8 +826,14 @@ class HttpCli(object):
|
|||
raise Pebkac(400, "empty files in post")
|
||||
|
||||
files.append([sz, sha512_hex, p_file, fname])
|
||||
dbv, vrem = vfs.get_dbv(vrem)
|
||||
self.conn.hsrv.broker.put(
|
||||
False, "up2k.hash_file", vfs.realpath, vfs.flags, rem, fname
|
||||
False,
|
||||
"up2k.hash_file",
|
||||
dbv.realpath,
|
||||
dbv.flags,
|
||||
vrem,
|
||||
fname,
|
||||
)
|
||||
self.conn.nbyte += sz
|
||||
|
||||
|
|
@ -1483,6 +1498,7 @@ class HttpCli(object):
|
|||
self.vpath, self.uname, self.readable, self.writable
|
||||
)
|
||||
abspath = vn.canonical(rem)
|
||||
dbv, vrem = vn.get_dbv(rem)
|
||||
|
||||
try:
|
||||
st = os.stat(fsenc(abspath))
|
||||
|
|
@ -1497,7 +1513,9 @@ class HttpCli(object):
|
|||
if th_fmt is not None:
|
||||
thp = None
|
||||
if self.thumbcli:
|
||||
thp = self.thumbcli.get(vn.realpath, rem, int(st.st_mtime), th_fmt)
|
||||
thp = self.thumbcli.get(
|
||||
dbv.realpath, vrem, int(st.st_mtime), th_fmt
|
||||
)
|
||||
|
||||
if thp:
|
||||
return self.tx_file(thp)
|
||||
|
|
@ -1640,7 +1658,7 @@ class HttpCli(object):
|
|||
icur = None
|
||||
if "e2t" in vn.flags:
|
||||
idx = self.conn.get_u2idx()
|
||||
icur = idx.get_cur(vn.realpath)
|
||||
icur = idx.get_cur(dbv.realpath)
|
||||
|
||||
dirs = []
|
||||
files = []
|
||||
|
|
@ -1708,6 +1726,9 @@ class HttpCli(object):
|
|||
rd = f["rd"]
|
||||
del f["rd"]
|
||||
if icur:
|
||||
if vn != dbv:
|
||||
_, rd = vn.get_dbv(rd)
|
||||
|
||||
q = "select w from up where rd = ? and fn = ?"
|
||||
try:
|
||||
r = icur.execute(q, (rd, fn)).fetchone()
|
||||
|
|
|
|||
|
|
@ -113,13 +113,13 @@ class TestVFS(unittest.TestCase):
|
|||
n = vfs.nodes["a"]
|
||||
self.assertEqual(len(vfs.nodes), 1)
|
||||
self.assertEqual(n.vpath, "a")
|
||||
self.assertEqual(n.realpath, td + "/a")
|
||||
self.assertEqual(n.realpath, os.path.join(td, "a"))
|
||||
self.assertEqual(n.uread, ["*", "k"])
|
||||
self.assertEqual(n.uwrite, ["k"])
|
||||
n = n.nodes["ac"]
|
||||
self.assertEqual(len(vfs.nodes), 1)
|
||||
self.assertEqual(n.vpath, "a/ac")
|
||||
self.assertEqual(n.realpath, td + "/a/ac")
|
||||
self.assertEqual(n.realpath, os.path.join(td, "a", "ac"))
|
||||
self.assertEqual(n.uread, ["*", "k"])
|
||||
self.assertEqual(n.uwrite, ["k"])
|
||||
n = n.nodes["acb"]
|
||||
|
|
|
|||
Loading…
Reference in a new issue