mirror of
https://github.com/9001/copyparty.git
synced 2025-08-18 09:22:31 -06:00
live db/tags rescan
This commit is contained in:
parent
eb5aaddba4
commit
0be7c5e2d8
|
@ -294,6 +294,8 @@ the same arguments can be set as volume flags, in addition to `d2d` and `d2t` fo
|
||||||
|
|
||||||
`e2tsr` is probably always overkill, since `e2ds`/`e2dsa` would pick up any file modifications and cause `e2ts` to reindex those
|
`e2tsr` is probably always overkill, since `e2ds`/`e2dsa` would pick up any file modifications and cause `e2ts` to reindex those
|
||||||
|
|
||||||
|
the rescan button in the admin panel has no effect unless the volume has `-e2ds` or higher
|
||||||
|
|
||||||
|
|
||||||
## metadata from audio files
|
## metadata from audio files
|
||||||
|
|
||||||
|
|
|
@ -249,6 +249,10 @@ def run_argparse(argv, formatter):
|
||||||
ap.add_argument("--urlform", metavar="MODE", type=str, default="print,get", help="how to handle url-forms")
|
ap.add_argument("--urlform", metavar="MODE", type=str, default="print,get", help="how to handle url-forms")
|
||||||
ap.add_argument("--salt", type=str, default="hunter2", help="up2k file-hash salt")
|
ap.add_argument("--salt", type=str, default="hunter2", help="up2k file-hash salt")
|
||||||
|
|
||||||
|
ap2 = ap.add_argument_group('admin panel options')
|
||||||
|
ap2.add_argument("--no-rescan", action="store_true", help="disable ?scan (volume reindexing)")
|
||||||
|
ap2.add_argument("--no-stack", action="store_true", help="disable ?stack (list all stacks)")
|
||||||
|
|
||||||
ap2 = ap.add_argument_group('thumbnail options')
|
ap2 = ap.add_argument_group('thumbnail options')
|
||||||
ap2.add_argument("--no-thumb", action="store_true", help="disable all thumbnails")
|
ap2.add_argument("--no-thumb", action="store_true", help="disable all thumbnails")
|
||||||
ap2.add_argument("--no-vthumb", action="store_true", help="disable video thumbnails")
|
ap2.add_argument("--no-vthumb", action="store_true", help="disable video thumbnails")
|
||||||
|
@ -287,6 +291,7 @@ def run_argparse(argv, formatter):
|
||||||
ap2.add_argument("--log-conn", action="store_true", help="print tcp-server msgs")
|
ap2.add_argument("--log-conn", action="store_true", help="print tcp-server msgs")
|
||||||
ap2.add_argument("--no-sendfile", action="store_true", help="disable sendfile")
|
ap2.add_argument("--no-sendfile", action="store_true", help="disable sendfile")
|
||||||
ap2.add_argument("--no-scandir", action="store_true", help="disable scandir")
|
ap2.add_argument("--no-scandir", action="store_true", help="disable scandir")
|
||||||
|
ap2.add_argument("--no-fastboot", action="store_true", help="wait for up2k indexing")
|
||||||
ap2.add_argument("--ihead", metavar="HEADER", action='append', help="dump incoming header")
|
ap2.add_argument("--ihead", metavar="HEADER", action='append', help="dump incoming header")
|
||||||
ap2.add_argument("--lf-url", metavar="RE", type=str, default=r"^/\.cpr/|\?th=[wj]$", help="dont log URLs matching")
|
ap2.add_argument("--lf-url", metavar="RE", type=str, default=r"^/\.cpr/|\?th=[wj]$", help="dont log URLs matching")
|
||||||
|
|
||||||
|
|
|
@ -14,11 +14,12 @@ from .util import IMPLICATIONS, undot, Pebkac, fsdec, fsenc, statdir, nuprint
|
||||||
class VFS(object):
|
class VFS(object):
|
||||||
"""single level in the virtual fs"""
|
"""single level in the virtual fs"""
|
||||||
|
|
||||||
def __init__(self, realpath, vpath, uread=[], uwrite=[], flags={}):
|
def __init__(self, realpath, vpath, uread=[], uwrite=[], uadm=[], flags={}):
|
||||||
self.realpath = realpath # absolute path on host filesystem
|
self.realpath = realpath # absolute path on host filesystem
|
||||||
self.vpath = vpath # absolute path in the virtual filesystem
|
self.vpath = vpath # absolute path in the virtual filesystem
|
||||||
self.uread = uread # users who can read this
|
self.uread = uread # users who can read this
|
||||||
self.uwrite = uwrite # users who can write this
|
self.uwrite = uwrite # users who can write this
|
||||||
|
self.uadm = uadm # users who are regular admins
|
||||||
self.flags = flags # config switches
|
self.flags = flags # config switches
|
||||||
self.nodes = {} # child nodes
|
self.nodes = {} # child nodes
|
||||||
self.all_vols = {vpath: self} # flattened recursive
|
self.all_vols = {vpath: self} # flattened recursive
|
||||||
|
@ -27,7 +28,7 @@ class VFS(object):
|
||||||
return "VFS({})".format(
|
return "VFS({})".format(
|
||||||
", ".join(
|
", ".join(
|
||||||
"{}={!r}".format(k, self.__dict__[k])
|
"{}={!r}".format(k, self.__dict__[k])
|
||||||
for k in "realpath vpath uread uwrite flags".split()
|
for k in "realpath vpath uread uwrite uadm flags".split()
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -52,6 +53,7 @@ class VFS(object):
|
||||||
"{}/{}".format(self.vpath, name).lstrip("/"),
|
"{}/{}".format(self.vpath, name).lstrip("/"),
|
||||||
self.uread,
|
self.uread,
|
||||||
self.uwrite,
|
self.uwrite,
|
||||||
|
self.uadm,
|
||||||
self.flags,
|
self.flags,
|
||||||
)
|
)
|
||||||
self._trk(vn)
|
self._trk(vn)
|
||||||
|
@ -226,15 +228,19 @@ class VFS(object):
|
||||||
for f in [{"vp": v, "ap": a, "st": n[1]} for v, a, n in files]:
|
for f in [{"vp": v, "ap": a, "st": n[1]} for v, a, n in files]:
|
||||||
yield f
|
yield f
|
||||||
|
|
||||||
def user_tree(self, uname, readable=False, writable=False):
|
def user_tree(self, uname, readable=False, writable=False, admin=False):
|
||||||
ret = []
|
ret = []
|
||||||
opt1 = readable and (uname in self.uread or "*" in self.uread)
|
opt1 = readable and (uname in self.uread or "*" in self.uread)
|
||||||
opt2 = writable and (uname in self.uwrite or "*" in self.uwrite)
|
opt2 = writable and (uname in self.uwrite or "*" in self.uwrite)
|
||||||
|
if admin:
|
||||||
|
if opt1 and opt2:
|
||||||
|
ret.append(self.vpath)
|
||||||
|
else:
|
||||||
if opt1 or opt2:
|
if opt1 or opt2:
|
||||||
ret.append(self.vpath)
|
ret.append(self.vpath)
|
||||||
|
|
||||||
for _, vn in sorted(self.nodes.items()):
|
for _, vn in sorted(self.nodes.items()):
|
||||||
ret.extend(vn.user_tree(uname, readable, writable))
|
ret.extend(vn.user_tree(uname, readable, writable, admin))
|
||||||
|
|
||||||
return ret
|
return ret
|
||||||
|
|
||||||
|
@ -269,7 +275,7 @@ class AuthSrv(object):
|
||||||
|
|
||||||
yield prev, True
|
yield prev, True
|
||||||
|
|
||||||
def _parse_config_file(self, fd, user, mread, mwrite, mflags, mount):
|
def _parse_config_file(self, fd, user, mread, mwrite, madm, mflags, mount):
|
||||||
vol_src = None
|
vol_src = None
|
||||||
vol_dst = None
|
vol_dst = None
|
||||||
self.line_ctr = 0
|
self.line_ctr = 0
|
||||||
|
@ -301,6 +307,7 @@ class AuthSrv(object):
|
||||||
mount[vol_dst] = vol_src
|
mount[vol_dst] = vol_src
|
||||||
mread[vol_dst] = []
|
mread[vol_dst] = []
|
||||||
mwrite[vol_dst] = []
|
mwrite[vol_dst] = []
|
||||||
|
madm[vol_dst] = []
|
||||||
mflags[vol_dst] = {}
|
mflags[vol_dst] = {}
|
||||||
continue
|
continue
|
||||||
|
|
||||||
|
@ -311,10 +318,15 @@ class AuthSrv(object):
|
||||||
uname = "*"
|
uname = "*"
|
||||||
|
|
||||||
self._read_vol_str(
|
self._read_vol_str(
|
||||||
lvl, uname, mread[vol_dst], mwrite[vol_dst], mflags[vol_dst]
|
lvl,
|
||||||
|
uname,
|
||||||
|
mread[vol_dst],
|
||||||
|
mwrite[vol_dst],
|
||||||
|
madm[vol_dst],
|
||||||
|
mflags[vol_dst],
|
||||||
)
|
)
|
||||||
|
|
||||||
def _read_vol_str(self, lvl, uname, mr, mw, mf):
|
def _read_vol_str(self, lvl, uname, mr, mw, ma, mf):
|
||||||
if lvl == "c":
|
if lvl == "c":
|
||||||
cval = True
|
cval = True
|
||||||
if "=" in uname:
|
if "=" in uname:
|
||||||
|
@ -332,6 +344,9 @@ class AuthSrv(object):
|
||||||
if lvl in "wa":
|
if lvl in "wa":
|
||||||
mw.append(uname)
|
mw.append(uname)
|
||||||
|
|
||||||
|
if lvl == "a":
|
||||||
|
ma.append(uname)
|
||||||
|
|
||||||
def _read_volflag(self, flags, name, value, is_list):
|
def _read_volflag(self, flags, name, value, is_list):
|
||||||
if name not in ["mtp"]:
|
if name not in ["mtp"]:
|
||||||
flags[name] = value
|
flags[name] = value
|
||||||
|
@ -355,6 +370,7 @@ class AuthSrv(object):
|
||||||
user = {} # username:password
|
user = {} # username:password
|
||||||
mread = {} # mountpoint:[username]
|
mread = {} # mountpoint:[username]
|
||||||
mwrite = {} # mountpoint:[username]
|
mwrite = {} # mountpoint:[username]
|
||||||
|
madm = {} # mountpoint:[username]
|
||||||
mflags = {} # mountpoint:[flag]
|
mflags = {} # mountpoint:[flag]
|
||||||
mount = {} # dst:src (mountpoint:realpath)
|
mount = {} # dst:src (mountpoint:realpath)
|
||||||
|
|
||||||
|
@ -378,17 +394,22 @@ class AuthSrv(object):
|
||||||
mount[dst] = src
|
mount[dst] = src
|
||||||
mread[dst] = []
|
mread[dst] = []
|
||||||
mwrite[dst] = []
|
mwrite[dst] = []
|
||||||
|
madm[dst] = []
|
||||||
mflags[dst] = {}
|
mflags[dst] = {}
|
||||||
|
|
||||||
perms = perms.split(":")
|
perms = perms.split(":")
|
||||||
for (lvl, uname) in [[x[0], x[1:]] for x in perms]:
|
for (lvl, uname) in [[x[0], x[1:]] for x in perms]:
|
||||||
self._read_vol_str(lvl, uname, mread[dst], mwrite[dst], mflags[dst])
|
self._read_vol_str(
|
||||||
|
lvl, uname, mread[dst], mwrite[dst], madm[dst], mflags[dst]
|
||||||
|
)
|
||||||
|
|
||||||
if self.args.c:
|
if self.args.c:
|
||||||
for cfg_fn in self.args.c:
|
for cfg_fn in self.args.c:
|
||||||
with open(cfg_fn, "rb") as f:
|
with open(cfg_fn, "rb") as f:
|
||||||
try:
|
try:
|
||||||
self._parse_config_file(f, user, mread, mwrite, mflags, mount)
|
self._parse_config_file(
|
||||||
|
f, user, mread, mwrite, madm, mflags, mount
|
||||||
|
)
|
||||||
except:
|
except:
|
||||||
m = "\n\033[1;31m\nerror in config file {} on line {}:\n\033[0m"
|
m = "\n\033[1;31m\nerror in config file {} on line {}:\n\033[0m"
|
||||||
print(m.format(cfg_fn, self.line_ctr))
|
print(m.format(cfg_fn, self.line_ctr))
|
||||||
|
@ -410,12 +431,15 @@ class AuthSrv(object):
|
||||||
|
|
||||||
if dst == "":
|
if dst == "":
|
||||||
# rootfs was mapped; fully replaces the default CWD vfs
|
# rootfs was mapped; fully replaces the default CWD vfs
|
||||||
vfs = VFS(mount[dst], dst, mread[dst], mwrite[dst], mflags[dst])
|
vfs = VFS(
|
||||||
|
mount[dst], dst, mread[dst], mwrite[dst], madm[dst], mflags[dst]
|
||||||
|
)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
v = vfs.add(mount[dst], dst)
|
v = vfs.add(mount[dst], dst)
|
||||||
v.uread = mread[dst]
|
v.uread = mread[dst]
|
||||||
v.uwrite = mwrite[dst]
|
v.uwrite = mwrite[dst]
|
||||||
|
v.uadm = madm[dst]
|
||||||
v.flags = mflags[dst]
|
v.flags = mflags[dst]
|
||||||
|
|
||||||
missing_users = {}
|
missing_users = {}
|
||||||
|
|
|
@ -10,6 +10,7 @@ import json
|
||||||
import string
|
import string
|
||||||
import socket
|
import socket
|
||||||
import ctypes
|
import ctypes
|
||||||
|
import traceback
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
import calendar
|
import calendar
|
||||||
|
|
||||||
|
@ -155,6 +156,7 @@ class HttpCli(object):
|
||||||
if self.uname:
|
if self.uname:
|
||||||
self.rvol = self.auth.vfs.user_tree(self.uname, readable=True)
|
self.rvol = self.auth.vfs.user_tree(self.uname, readable=True)
|
||||||
self.wvol = self.auth.vfs.user_tree(self.uname, writable=True)
|
self.wvol = self.auth.vfs.user_tree(self.uname, writable=True)
|
||||||
|
self.avol = self.auth.vfs.user_tree(self.uname, True, True, True)
|
||||||
|
|
||||||
ua = self.headers.get("user-agent", "")
|
ua = self.headers.get("user-agent", "")
|
||||||
self.is_rclone = ua.startswith("rclone/")
|
self.is_rclone = ua.startswith("rclone/")
|
||||||
|
@ -326,6 +328,12 @@ class HttpCli(object):
|
||||||
self.vpath = None
|
self.vpath = None
|
||||||
return self.tx_mounts()
|
return self.tx_mounts()
|
||||||
|
|
||||||
|
if "scan" in self.uparam:
|
||||||
|
return self.scanvol()
|
||||||
|
|
||||||
|
if "stack" in self.uparam:
|
||||||
|
return self.tx_stack()
|
||||||
|
|
||||||
return self.tx_browser()
|
return self.tx_browser()
|
||||||
|
|
||||||
def handle_options(self):
|
def handle_options(self):
|
||||||
|
@ -1304,10 +1312,61 @@ class HttpCli(object):
|
||||||
suf = self.urlq(rm=["h"])
|
suf = self.urlq(rm=["h"])
|
||||||
rvol = [x + "/" if x else x for x in self.rvol]
|
rvol = [x + "/" if x else x for x in self.rvol]
|
||||||
wvol = [x + "/" if x else x for x in self.wvol]
|
wvol = [x + "/" if x else x for x in self.wvol]
|
||||||
html = self.j2("splash", this=self, rvol=rvol, wvol=wvol, url_suf=suf)
|
|
||||||
|
vstate = {}
|
||||||
|
if self.avol and not self.args.no_rescan:
|
||||||
|
x = self.conn.hsrv.broker.put(True, "up2k.get_volstate")
|
||||||
|
vstate = json.loads(x.get())
|
||||||
|
|
||||||
|
html = self.j2(
|
||||||
|
"splash",
|
||||||
|
this=self,
|
||||||
|
rvol=rvol,
|
||||||
|
wvol=wvol,
|
||||||
|
avol=self.avol,
|
||||||
|
vstate=vstate,
|
||||||
|
url_suf=suf,
|
||||||
|
)
|
||||||
self.reply(html.encode("utf-8"), headers=NO_STORE)
|
self.reply(html.encode("utf-8"), headers=NO_STORE)
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
def scanvol(self):
|
||||||
|
if not self.readable or not self.writable:
|
||||||
|
raise Pebkac(403, "not admin")
|
||||||
|
|
||||||
|
if self.args.no_rescan:
|
||||||
|
raise Pebkac(403, "disabled by argv")
|
||||||
|
|
||||||
|
vn, _ = self.auth.vfs.get(self.vpath, self.uname, True, True)
|
||||||
|
|
||||||
|
args = [self.auth.vfs.all_vols, [vn.vpath]]
|
||||||
|
x = self.conn.hsrv.broker.put(True, "up2k.rescan", *args)
|
||||||
|
x = x.get()
|
||||||
|
if not x:
|
||||||
|
self.redirect("", "?h")
|
||||||
|
return ""
|
||||||
|
|
||||||
|
raise Pebkac(500, x)
|
||||||
|
|
||||||
|
def tx_stack(self):
|
||||||
|
if not self.readable or not self.writable:
|
||||||
|
raise Pebkac(403, "not admin")
|
||||||
|
|
||||||
|
if self.args.no_stack:
|
||||||
|
raise Pebkac(403, "disabled by argv")
|
||||||
|
|
||||||
|
ret = []
|
||||||
|
names = dict([(t.ident, t.name) for t in threading.enumerate()])
|
||||||
|
for tid, stack in sys._current_frames().items():
|
||||||
|
ret.append("\n\n# {} ({:x})".format(names.get(tid), tid))
|
||||||
|
for fn, lno, name, line in traceback.extract_stack(stack):
|
||||||
|
ret.append('File: "{}", line {}, in {}'.format(fn, lno, name))
|
||||||
|
if line:
|
||||||
|
ret.append(" " + str(line.strip()))
|
||||||
|
|
||||||
|
ret = ("<pre>" + "\n".join(ret)).encode("utf-8")
|
||||||
|
self.reply(ret)
|
||||||
|
|
||||||
def tx_tree(self):
|
def tx_tree(self):
|
||||||
top = self.uparam["tree"] or ""
|
top = self.uparam["tree"] or ""
|
||||||
dst = self.vpath
|
dst = self.vpath
|
||||||
|
|
|
@ -52,7 +52,6 @@ class Up2k(object):
|
||||||
self.hub = hub
|
self.hub = hub
|
||||||
self.args = hub.args
|
self.args = hub.args
|
||||||
self.log_func = hub.log
|
self.log_func = hub.log
|
||||||
self.all_vols = all_vols
|
|
||||||
|
|
||||||
# config
|
# config
|
||||||
self.salt = self.args.salt
|
self.salt = self.args.salt
|
||||||
|
@ -61,12 +60,14 @@ class Up2k(object):
|
||||||
self.mutex = threading.Lock()
|
self.mutex = threading.Lock()
|
||||||
self.hashq = Queue()
|
self.hashq = Queue()
|
||||||
self.tagq = Queue()
|
self.tagq = Queue()
|
||||||
|
self.volstate = {}
|
||||||
self.registry = {}
|
self.registry = {}
|
||||||
self.entags = {}
|
self.entags = {}
|
||||||
self.flags = {}
|
self.flags = {}
|
||||||
self.cur = {}
|
self.cur = {}
|
||||||
self.mtag = None
|
self.mtag = None
|
||||||
self.pending_tags = None
|
self.pending_tags = None
|
||||||
|
self.mtp_parsers = {}
|
||||||
|
|
||||||
self.mem_cur = None
|
self.mem_cur = None
|
||||||
self.sqlite_ver = None
|
self.sqlite_ver = None
|
||||||
|
@ -92,7 +93,15 @@ class Up2k(object):
|
||||||
if not HAVE_SQLITE3:
|
if not HAVE_SQLITE3:
|
||||||
self.log("could not initialize sqlite3, will use in-memory registry only")
|
self.log("could not initialize sqlite3, will use in-memory registry only")
|
||||||
|
|
||||||
have_e2d = self.init_indexes()
|
if self.args.no_fastboot:
|
||||||
|
self.deferred_init(all_vols)
|
||||||
|
else:
|
||||||
|
t = threading.Thread(target=self.deferred_init, args=(all_vols,))
|
||||||
|
t.daemon = True
|
||||||
|
t.start()
|
||||||
|
|
||||||
|
def deferred_init(self, all_vols):
|
||||||
|
have_e2d = self.init_indexes(all_vols)
|
||||||
|
|
||||||
if have_e2d:
|
if have_e2d:
|
||||||
thr = threading.Thread(target=self._snapshot)
|
thr = threading.Thread(target=self._snapshot)
|
||||||
|
@ -115,6 +124,19 @@ class Up2k(object):
|
||||||
def log(self, msg, c=0):
|
def log(self, msg, c=0):
|
||||||
self.log_func("up2k", msg + "\033[K", c)
|
self.log_func("up2k", msg + "\033[K", c)
|
||||||
|
|
||||||
|
def get_volstate(self):
|
||||||
|
return json.dumps(self.volstate, indent=4)
|
||||||
|
|
||||||
|
def rescan(self, all_vols, scan_vols):
|
||||||
|
if hasattr(self, "pp"):
|
||||||
|
return "cannot initiate; scan is already in progress"
|
||||||
|
|
||||||
|
args = (all_vols, scan_vols)
|
||||||
|
t = threading.Thread(target=self.init_indexes, args=args)
|
||||||
|
t.daemon = True
|
||||||
|
t.start()
|
||||||
|
return None
|
||||||
|
|
||||||
def _vis_job_progress(self, job):
|
def _vis_job_progress(self, job):
|
||||||
perc = 100 - (len(job["need"]) * 100.0 / len(job["hash"]))
|
perc = 100 - (len(job["need"]) * 100.0 / len(job["hash"]))
|
||||||
path = os.path.join(job["ptop"], job["prel"], job["name"])
|
path = os.path.join(job["ptop"], job["prel"], job["name"])
|
||||||
|
@ -137,9 +159,9 @@ class Up2k(object):
|
||||||
|
|
||||||
return True, ret
|
return True, ret
|
||||||
|
|
||||||
def init_indexes(self):
|
def init_indexes(self, all_vols, scan_vols=[]):
|
||||||
self.pp = ProgressPrinter()
|
self.pp = ProgressPrinter()
|
||||||
vols = self.all_vols.values()
|
vols = all_vols.values()
|
||||||
t0 = time.time()
|
t0 = time.time()
|
||||||
have_e2d = False
|
have_e2d = False
|
||||||
|
|
||||||
|
@ -159,24 +181,32 @@ class Up2k(object):
|
||||||
for vol in vols:
|
for vol in vols:
|
||||||
try:
|
try:
|
||||||
os.listdir(vol.realpath)
|
os.listdir(vol.realpath)
|
||||||
|
if not self.register_vpath(vol.realpath, vol.flags):
|
||||||
|
raise Exception()
|
||||||
|
|
||||||
|
if vol.vpath in scan_vols or not scan_vols:
|
||||||
live_vols.append(vol)
|
live_vols.append(vol)
|
||||||
|
|
||||||
|
if vol.vpath not in self.volstate:
|
||||||
|
self.volstate[vol.vpath] = "OFFLINE (not initialized)"
|
||||||
except:
|
except:
|
||||||
self.log("cannot access " + vol.realpath, c=1)
|
# self.log("db not enabled for {}".format(m, vol.realpath))
|
||||||
|
pass
|
||||||
|
|
||||||
vols = live_vols
|
vols = live_vols
|
||||||
|
need_vac = {}
|
||||||
|
|
||||||
need_mtag = False
|
need_mtag = False
|
||||||
for vol in vols:
|
for vol in vols:
|
||||||
if "e2t" in vol.flags:
|
if "e2t" in vol.flags:
|
||||||
need_mtag = True
|
need_mtag = True
|
||||||
|
|
||||||
if need_mtag:
|
if need_mtag and not self.mtag:
|
||||||
self.mtag = MTag(self.log_func, self.args)
|
self.mtag = MTag(self.log_func, self.args)
|
||||||
if not self.mtag.usable:
|
if not self.mtag.usable:
|
||||||
self.mtag = None
|
self.mtag = None
|
||||||
|
|
||||||
# e2ds(a) volumes first,
|
# e2ds(a) volumes first
|
||||||
# also covers tags where e2ts is set
|
|
||||||
for vol in vols:
|
for vol in vols:
|
||||||
en = {}
|
en = {}
|
||||||
if "mte" in vol.flags:
|
if "mte" in vol.flags:
|
||||||
|
@ -188,26 +218,45 @@ class Up2k(object):
|
||||||
have_e2d = True
|
have_e2d = True
|
||||||
|
|
||||||
if "e2ds" in vol.flags:
|
if "e2ds" in vol.flags:
|
||||||
r = self._build_file_index(vol, vols)
|
self.volstate[vol.vpath] = "busy (hashing files)"
|
||||||
if not r:
|
_, vac = self._build_file_index(vol, list(all_vols.values()))
|
||||||
needed_mutagen = True
|
if vac:
|
||||||
|
need_vac[vol] = True
|
||||||
|
|
||||||
|
if "e2ts" not in vol.flags:
|
||||||
|
m = "online, idle"
|
||||||
|
else:
|
||||||
|
m = "online (tags pending)"
|
||||||
|
|
||||||
|
self.volstate[vol.vpath] = m
|
||||||
|
|
||||||
# open the rest + do any e2ts(a)
|
# open the rest + do any e2ts(a)
|
||||||
needed_mutagen = False
|
needed_mutagen = False
|
||||||
for vol in vols:
|
for vol in vols:
|
||||||
r = self.register_vpath(vol.realpath, vol.flags)
|
if "e2ts" not in vol.flags:
|
||||||
if not r or "e2ts" not in vol.flags:
|
|
||||||
continue
|
continue
|
||||||
|
|
||||||
cur, db_path, sz0 = r
|
m = "online (reading tags)"
|
||||||
n_add, n_rm, success = self._build_tags_index(vol.realpath)
|
self.volstate[vol.vpath] = m
|
||||||
|
self.log("{} [{}]".format(m, vol.realpath))
|
||||||
|
|
||||||
|
nadd, nrm, success = self._build_tags_index(vol)
|
||||||
if not success:
|
if not success:
|
||||||
needed_mutagen = True
|
needed_mutagen = True
|
||||||
|
|
||||||
if n_add or n_rm:
|
if nadd or nrm:
|
||||||
self.vac(cur, db_path, n_add, n_rm, sz0)
|
need_vac[vol] = True
|
||||||
|
|
||||||
|
self.volstate[vol.vpath] = "online (mtp soon)"
|
||||||
|
|
||||||
|
for vol in need_vac:
|
||||||
|
cur, _ = self.register_vpath(vol.realpath, vol.flags)
|
||||||
|
with self.mutex:
|
||||||
|
cur.connection.commit()
|
||||||
|
cur.execute("vacuum")
|
||||||
|
|
||||||
self.pp.end = True
|
self.pp.end = True
|
||||||
|
|
||||||
msg = "{} volumes in {:.2f} sec"
|
msg = "{} volumes in {:.2f} sec"
|
||||||
self.log(msg.format(len(vols), time.time() - t0))
|
self.log(msg.format(len(vols), time.time() - t0))
|
||||||
|
|
||||||
|
@ -215,12 +264,28 @@ class Up2k(object):
|
||||||
msg = "could not read tags because no backends are available (mutagen or ffprobe)"
|
msg = "could not read tags because no backends are available (mutagen or ffprobe)"
|
||||||
self.log(msg, c=1)
|
self.log(msg, c=1)
|
||||||
|
|
||||||
|
thr = None
|
||||||
|
if self.mtag:
|
||||||
|
m = "online (running mtp)"
|
||||||
|
if scan_vols:
|
||||||
|
thr = threading.Thread(target=self._run_all_mtp)
|
||||||
|
thr.daemon = True
|
||||||
|
else:
|
||||||
|
del self.pp
|
||||||
|
m = "online, idle"
|
||||||
|
|
||||||
|
for vol in vols:
|
||||||
|
self.volstate[vol.vpath] = m
|
||||||
|
|
||||||
|
if thr:
|
||||||
|
thr.start()
|
||||||
|
|
||||||
return have_e2d
|
return have_e2d
|
||||||
|
|
||||||
def register_vpath(self, ptop, flags):
|
def register_vpath(self, ptop, flags):
|
||||||
with self.mutex:
|
db_path = os.path.join(ptop, ".hist", "up2k.db")
|
||||||
if ptop in self.registry:
|
if ptop in self.registry:
|
||||||
return None
|
return [self.cur[ptop], db_path]
|
||||||
|
|
||||||
_, flags = self._expr_idx_filter(flags)
|
_, flags = self._expr_idx_filter(flags)
|
||||||
|
|
||||||
|
@ -263,18 +328,10 @@ class Up2k(object):
|
||||||
except:
|
except:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
db_path = os.path.join(ptop, ".hist", "up2k.db")
|
|
||||||
if ptop in self.cur:
|
|
||||||
return None
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
sz0 = 0
|
|
||||||
if os.path.exists(db_path):
|
|
||||||
sz0 = os.path.getsize(db_path) // 1024
|
|
||||||
|
|
||||||
cur = self._open_db(db_path)
|
cur = self._open_db(db_path)
|
||||||
self.cur[ptop] = cur
|
self.cur[ptop] = cur
|
||||||
return [cur, db_path, sz0]
|
return [cur, db_path]
|
||||||
except:
|
except:
|
||||||
msg = "cannot use database at [{}]:\n{}"
|
msg = "cannot use database at [{}]:\n{}"
|
||||||
self.log(msg.format(ptop, traceback.format_exc()))
|
self.log(msg.format(ptop, traceback.format_exc()))
|
||||||
|
@ -284,12 +341,10 @@ class Up2k(object):
|
||||||
def _build_file_index(self, vol, all_vols):
|
def _build_file_index(self, vol, all_vols):
|
||||||
do_vac = False
|
do_vac = False
|
||||||
top = vol.realpath
|
top = vol.realpath
|
||||||
reg = self.register_vpath(top, vol.flags)
|
with self.mutex:
|
||||||
if not reg:
|
cur, _ = self.register_vpath(top, vol.flags)
|
||||||
return
|
|
||||||
|
|
||||||
_, db_path, sz0 = reg
|
dbw = [cur, 0, time.time()]
|
||||||
dbw = [reg[0], 0, time.time()]
|
|
||||||
self.pp.n = next(dbw[0].execute("select count(w) from up"))[0]
|
self.pp.n = next(dbw[0].execute("select count(w) from up"))[0]
|
||||||
|
|
||||||
excl = [
|
excl = [
|
||||||
|
@ -297,28 +352,16 @@ class Up2k(object):
|
||||||
for d in all_vols
|
for d in all_vols
|
||||||
if d != vol and (d.vpath.startswith(vol.vpath + "/") or not vol.vpath)
|
if d != vol and (d.vpath.startswith(vol.vpath + "/") or not vol.vpath)
|
||||||
]
|
]
|
||||||
|
if WINDOWS:
|
||||||
|
excl = [x.replace("/", "\\") for x in excl]
|
||||||
|
|
||||||
n_add = self._build_dir(dbw, top, set(excl), top)
|
n_add = self._build_dir(dbw, top, set(excl), top)
|
||||||
n_rm = self._drop_lost(dbw[0], top)
|
n_rm = self._drop_lost(dbw[0], top)
|
||||||
if dbw[1]:
|
if dbw[1]:
|
||||||
self.log("commit {} new files".format(dbw[1]))
|
self.log("commit {} new files".format(dbw[1]))
|
||||||
dbw[0].connection.commit()
|
dbw[0].connection.commit()
|
||||||
|
|
||||||
n_add, n_rm, success = self._build_tags_index(vol.realpath)
|
return True, n_add or n_rm or do_vac
|
||||||
|
|
||||||
dbw[0].connection.commit()
|
|
||||||
if n_add or n_rm or do_vac:
|
|
||||||
self.vac(dbw[0], db_path, n_add, n_rm, sz0)
|
|
||||||
|
|
||||||
return success
|
|
||||||
|
|
||||||
def vac(self, cur, db_path, n_add, n_rm, sz0):
|
|
||||||
sz1 = os.path.getsize(db_path) // 1024
|
|
||||||
cur.execute("vacuum")
|
|
||||||
sz2 = os.path.getsize(db_path) // 1024
|
|
||||||
msg = "{} new, {} del, {} kB vacced, {} kB gain, {} kB now".format(
|
|
||||||
n_add, n_rm, sz1 - sz2, sz2 - sz0, sz2
|
|
||||||
)
|
|
||||||
self.log(msg)
|
|
||||||
|
|
||||||
def _build_dir(self, dbw, top, excl, cdir):
|
def _build_dir(self, dbw, top, excl, cdir):
|
||||||
self.pp.msg = "a{} {}".format(self.pp.n, cdir)
|
self.pp.msg = "a{} {}".format(self.pp.n, cdir)
|
||||||
|
@ -413,25 +456,29 @@ class Up2k(object):
|
||||||
|
|
||||||
return len(rm)
|
return len(rm)
|
||||||
|
|
||||||
def _build_tags_index(self, ptop):
|
def _build_tags_index(self, vol):
|
||||||
|
ptop = vol.realpath
|
||||||
|
with self.mutex:
|
||||||
|
_, db_path = self.register_vpath(ptop, vol.flags)
|
||||||
entags = self.entags[ptop]
|
entags = self.entags[ptop]
|
||||||
flags = self.flags[ptop]
|
flags = self.flags[ptop]
|
||||||
cur = self.cur[ptop]
|
cur = self.cur[ptop]
|
||||||
|
|
||||||
n_add = 0
|
n_add = 0
|
||||||
n_rm = 0
|
n_rm = 0
|
||||||
n_buf = 0
|
n_buf = 0
|
||||||
last_write = time.time()
|
last_write = time.time()
|
||||||
|
|
||||||
if "e2tsr" in flags:
|
if "e2tsr" in flags:
|
||||||
|
with self.mutex:
|
||||||
n_rm = cur.execute("select count(w) from mt").fetchone()[0]
|
n_rm = cur.execute("select count(w) from mt").fetchone()[0]
|
||||||
if n_rm:
|
if n_rm:
|
||||||
self.log("discarding {} media tags for a full rescan".format(n_rm))
|
self.log("discarding {} media tags for a full rescan".format(n_rm))
|
||||||
cur.execute("delete from mt")
|
cur.execute("delete from mt")
|
||||||
else:
|
|
||||||
self.log("volume has e2tsr but there are no media tags to discard")
|
|
||||||
|
|
||||||
# integrity: drop tags for tracks that were deleted
|
# integrity: drop tags for tracks that were deleted
|
||||||
if "e2t" in flags:
|
if "e2t" in flags:
|
||||||
|
with self.mutex:
|
||||||
drops = []
|
drops = []
|
||||||
c2 = cur.connection.cursor()
|
c2 = cur.connection.cursor()
|
||||||
up_q = "select w from up where substr(w,1,16) = ?"
|
up_q = "select w from up where substr(w,1,16) = ?"
|
||||||
|
@ -452,6 +499,10 @@ class Up2k(object):
|
||||||
return n_add, n_rm, True
|
return n_add, n_rm, True
|
||||||
|
|
||||||
# add tags for new files
|
# add tags for new files
|
||||||
|
gcur = cur
|
||||||
|
with self.mutex:
|
||||||
|
gcur.connection.commit()
|
||||||
|
|
||||||
if "e2ts" in flags:
|
if "e2ts" in flags:
|
||||||
if not self.mtag:
|
if not self.mtag:
|
||||||
return n_add, n_rm, False
|
return n_add, n_rm, False
|
||||||
|
@ -460,8 +511,10 @@ class Up2k(object):
|
||||||
if self.mtag.prefer_mt and not self.args.no_mtag_mt:
|
if self.mtag.prefer_mt and not self.args.no_mtag_mt:
|
||||||
mpool = self._start_mpool()
|
mpool = self._start_mpool()
|
||||||
|
|
||||||
c2 = cur.connection.cursor()
|
conn = sqlite3.connect(db_path, timeout=15)
|
||||||
c3 = cur.connection.cursor()
|
cur = conn.cursor()
|
||||||
|
c2 = conn.cursor()
|
||||||
|
c3 = conn.cursor()
|
||||||
n_left = cur.execute("select count(w) from up").fetchone()[0]
|
n_left = cur.execute("select count(w) from up").fetchone()[0]
|
||||||
for w, rd, fn in cur.execute("select w, rd, fn from up"):
|
for w, rd, fn in cur.execute("select w, rd, fn from up"):
|
||||||
n_left -= 1
|
n_left -= 1
|
||||||
|
@ -483,6 +536,7 @@ class Up2k(object):
|
||||||
n_tags = self._tag_file(c3, *args)
|
n_tags = self._tag_file(c3, *args)
|
||||||
else:
|
else:
|
||||||
mpool.put(["mtag"] + args)
|
mpool.put(["mtag"] + args)
|
||||||
|
with self.mutex:
|
||||||
n_tags = len(self._flush_mpool(c3))
|
n_tags = len(self._flush_mpool(c3))
|
||||||
|
|
||||||
n_add += n_tags
|
n_add += n_tags
|
||||||
|
@ -495,15 +549,22 @@ class Up2k(object):
|
||||||
last_write = time.time()
|
last_write = time.time()
|
||||||
n_buf = 0
|
n_buf = 0
|
||||||
|
|
||||||
self._stop_mpool(mpool, c3)
|
self._stop_mpool(mpool)
|
||||||
|
with self.mutex:
|
||||||
|
n_add += len(self._flush_mpool(c3))
|
||||||
|
|
||||||
|
conn.commit()
|
||||||
c3.close()
|
c3.close()
|
||||||
c2.close()
|
c2.close()
|
||||||
|
cur.close()
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
with self.mutex:
|
||||||
|
gcur.connection.commit()
|
||||||
|
|
||||||
return n_add, n_rm, True
|
return n_add, n_rm, True
|
||||||
|
|
||||||
def _flush_mpool(self, wcur):
|
def _flush_mpool(self, wcur):
|
||||||
with self.mutex:
|
|
||||||
ret = []
|
ret = []
|
||||||
for x in self.pending_tags:
|
for x in self.pending_tags:
|
||||||
self._tag_file(wcur, *x)
|
self._tag_file(wcur, *x)
|
||||||
|
@ -514,7 +575,6 @@ class Up2k(object):
|
||||||
|
|
||||||
def _run_all_mtp(self):
|
def _run_all_mtp(self):
|
||||||
t0 = time.time()
|
t0 = time.time()
|
||||||
self.mtp_parsers = {}
|
|
||||||
for ptop, flags in self.flags.items():
|
for ptop, flags in self.flags.items():
|
||||||
if "mtp" in flags:
|
if "mtp" in flags:
|
||||||
self._run_one_mtp(ptop)
|
self._run_one_mtp(ptop)
|
||||||
|
@ -523,10 +583,11 @@ class Up2k(object):
|
||||||
msg = "mtp finished in {:.2f} sec ({})"
|
msg = "mtp finished in {:.2f} sec ({})"
|
||||||
self.log(msg.format(td, s2hms(td, True)))
|
self.log(msg.format(td, s2hms(td, True)))
|
||||||
|
|
||||||
def _run_one_mtp(self, ptop):
|
del self.pp
|
||||||
db_path = os.path.join(ptop, ".hist", "up2k.db")
|
for k in list(self.volstate.keys()):
|
||||||
sz0 = os.path.getsize(db_path) // 1024
|
self.volstate[k] = "online, idle"
|
||||||
|
|
||||||
|
def _run_one_mtp(self, ptop):
|
||||||
entags = self.entags[ptop]
|
entags = self.entags[ptop]
|
||||||
|
|
||||||
parsers = {}
|
parsers = {}
|
||||||
|
@ -585,9 +646,8 @@ class Up2k(object):
|
||||||
jobs.append([parsers, None, w, abspath])
|
jobs.append([parsers, None, w, abspath])
|
||||||
in_progress[w] = True
|
in_progress[w] = True
|
||||||
|
|
||||||
done = self._flush_mpool(wcur)
|
|
||||||
|
|
||||||
with self.mutex:
|
with self.mutex:
|
||||||
|
done = self._flush_mpool(wcur)
|
||||||
for w in done:
|
for w in done:
|
||||||
to_delete[w] = True
|
to_delete[w] = True
|
||||||
in_progress.pop(w)
|
in_progress.pop(w)
|
||||||
|
@ -628,15 +688,16 @@ class Up2k(object):
|
||||||
with self.mutex:
|
with self.mutex:
|
||||||
cur.connection.commit()
|
cur.connection.commit()
|
||||||
|
|
||||||
done = self._stop_mpool(mpool, wcur)
|
self._stop_mpool(mpool)
|
||||||
with self.mutex:
|
with self.mutex:
|
||||||
|
done = self._flush_mpool(wcur)
|
||||||
for w in done:
|
for w in done:
|
||||||
q = "delete from mt where w = ? and k = 't:mtp'"
|
q = "delete from mt where w = ? and k = 't:mtp'"
|
||||||
cur.execute(q, (w,))
|
cur.execute(q, (w,))
|
||||||
|
|
||||||
cur.connection.commit()
|
cur.connection.commit()
|
||||||
if n_done:
|
if n_done:
|
||||||
self.vac(cur, db_path, n_done, 0, sz0)
|
cur.execute("vacuum")
|
||||||
|
|
||||||
wcur.close()
|
wcur.close()
|
||||||
cur.close()
|
cur.close()
|
||||||
|
@ -693,7 +754,7 @@ class Up2k(object):
|
||||||
|
|
||||||
return mpool
|
return mpool
|
||||||
|
|
||||||
def _stop_mpool(self, mpool, wcur):
|
def _stop_mpool(self, mpool):
|
||||||
if not mpool:
|
if not mpool:
|
||||||
return
|
return
|
||||||
|
|
||||||
|
@ -701,8 +762,6 @@ class Up2k(object):
|
||||||
mpool.put(None)
|
mpool.put(None)
|
||||||
|
|
||||||
mpool.join()
|
mpool.join()
|
||||||
done = self._flush_mpool(wcur)
|
|
||||||
return done
|
|
||||||
|
|
||||||
def _tag_thr(self, q):
|
def _tag_thr(self, q):
|
||||||
while True:
|
while True:
|
||||||
|
|
|
@ -26,6 +26,13 @@ a {
|
||||||
border-radius: .2em;
|
border-radius: .2em;
|
||||||
padding: .2em .8em;
|
padding: .2em .8em;
|
||||||
}
|
}
|
||||||
|
td, th {
|
||||||
|
padding: .3em .6em;
|
||||||
|
text-align: left;
|
||||||
|
}
|
||||||
|
.btns {
|
||||||
|
margin: 1em 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
html.dark,
|
html.dark,
|
||||||
|
|
|
@ -13,6 +13,23 @@
|
||||||
<div id="wrap">
|
<div id="wrap">
|
||||||
<p>hello {{ this.uname }}</p>
|
<p>hello {{ this.uname }}</p>
|
||||||
|
|
||||||
|
{%- if avol %}
|
||||||
|
<h1>admin panel:</h1>
|
||||||
|
<table>
|
||||||
|
<thead><tr><th>vol</th><th>action</th><th>status</th></tr></thead>
|
||||||
|
<tbody>
|
||||||
|
{% for mp in avol %}
|
||||||
|
{%- if mp in vstate and vstate[mp] %}
|
||||||
|
<tr><td>/{{ mp }}</td><td><a href="/{{ mp }}?scan">rescan</a></td><td>{{ vstate[mp] }}</td></tr>
|
||||||
|
{%- endif %}
|
||||||
|
{% endfor %}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
<div class="btns">
|
||||||
|
<a href="/{{ avol[0] }}?stack">dump stack</a>
|
||||||
|
</div>
|
||||||
|
{%- endif %}
|
||||||
|
|
||||||
{%- if rvol %}
|
{%- if rvol %}
|
||||||
<h1>you can browse these:</h1>
|
<h1>you can browse these:</h1>
|
||||||
<ul>
|
<ul>
|
||||||
|
|
Loading…
Reference in a new issue