From 8441206e26c5b78dccdb643735418e5ed57411dd Mon Sep 17 00:00:00 2001 From: ed Date: Mon, 1 Mar 2021 02:50:10 +0100 Subject: [PATCH] read media-tags from files (for display/searching) --- README.md | 35 +++- copyparty/__main__.py | 33 +++- copyparty/authsrv.py | 41 ++++- copyparty/httpcli.py | 62 ++++++- copyparty/httpconn.py | 6 +- copyparty/mtag.py | 305 ++++++++++++++++++++++++++++++++ copyparty/svchub.py | 8 - copyparty/u2idx.py | 103 +++++++---- copyparty/up2k.py | 351 ++++++++++++++++++++++++++++--------- copyparty/web/browser.css | 13 +- copyparty/web/browser.html | 26 ++- copyparty/web/browser.js | 99 +++++++++-- copyparty/web/up2k.js | 4 +- copyparty/web/util.js | 16 +- 14 files changed, 922 insertions(+), 180 deletions(-) create mode 100644 copyparty/mtag.py diff --git a/README.md b/README.md index a984e84e..f66209dd 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,40 @@ path/name queries are space-separated, AND'ed together, and words are negated wi * path: `shibayan -bossa` finds all files where one of the folders contain `shibayan` but filters out any results where `bossa` exists somewhere in the path * name: `demetori styx` gives you [good stuff](https://www.youtube.com/watch?v=zGh0g14ZJ8I&list=PL3A147BD151EE5218&index=9) -other metadata (like song tags etc) are not yet indexed for searching + +## search configuration + +searching relies on two databases, the up2k filetree (`-e2d`) and the metadata tags (`-e2t`). Configuration can be done through arguments, volume flags, or a mix of both. + +through arguments: +* `-e2d` enables file indexing on upload +* `-e2ds` scans writable folders on startup +* `-e2dsa` scans all mounted volumes (including readonly ones) +* `-e2t` enables metadata indexing on upload +* `-e2ts` scans for tags in all files that don't have tags yet +* `-e2tsr` deletes all existing tags, so a full reindex + +in addition to `d2d` and `d2t`, the same arguments can be set as volume flags: +* `-v ~/music::ce2dsa:ce2tsr` does a full reindex of everything on startup +* `-v ~/music::cd2d` disables **all** indexing, even if any `-e2*` are on +* `-v ~/music::cd2t` disables all `-e2t*` (tags), does not affect `-e2d*` + +`e2tsr` is probably always overkill, since `e2ds`/`e2dsa` would pick up any file modifications and cause `e2ts` to reindex those + +`-mte` decides which tags to index and display in the browser (and also the display order), this can be changed per-volume: +* `-v ~/music::cmte=title,artist` indexes and displays *title* followed by *artist* + +if you add/remove a tag from `mte` you will need to run with `-e2tsr` once to rebuild the database, otherwise only new files will be affected + +`-mtm` can be used to add or redefine a metadata mapping, say you have media files with `foo` and `bar` tags and you want them to display as `qux` in the browser (preferring `foo` if both are present), then do `-mtm qux=foo,bar` and now you can `-mte artist,title,qux` + +see the beautiful mess of a dictionary in [mtag.py](https://github.com/9001/copyparty/blob/master/copyparty/mtag.py) for the default mappings (should cover mp3,opus,flac,m4a,wav,aif,) + +`--no-mutagen` disables mutagen and uses ffprobe instead, which... +* is about 20x slower than mutagen +* catches a few tags that mutagen doesn't +* avoids pulling any GPL code into copyparty +* more importantly runs ffprobe on incoming files which is bad if your ffmpeg has a cve # client examples diff --git a/copyparty/__main__.py b/copyparty/__main__.py index 63a402b7..30628444 100644 --- a/copyparty/__main__.py +++ b/copyparty/__main__.py @@ -198,7 +198,7 @@ def main(): and "cflag" is config flags to set on this volume list of cflags: - cnodupe rejects existing files (instead of symlinking them) + "cnodupe" rejects existing files (instead of symlinking them) example:\033[35m -a ed:hunter2 -v .::r:aed -v ../inc:dump:w:aed:cnodupe \033[36m @@ -239,9 +239,6 @@ def main(): ap.add_argument("-q", action="store_true", help="quiet") ap.add_argument("-ed", action="store_true", help="enable ?dots") ap.add_argument("-emp", action="store_true", help="enable markdown plugins") - ap.add_argument("-e2d", action="store_true", help="enable up2k database") - ap.add_argument("-e2ds", action="store_true", help="enable up2k db-scanner, sets -e2d") - ap.add_argument("-e2dsa", action="store_true", help="scan all folders (for search), sets -e2ds") ap.add_argument("-mcr", metavar="SEC", type=int, default=60, help="md-editor mod-chk rate") ap.add_argument("-nw", action="store_true", help="disable writes (benchmark)") ap.add_argument("-nih", action="store_true", help="no info hostname") @@ -250,6 +247,18 @@ def main(): ap.add_argument("--urlform", type=str, default="print,get", help="how to handle url-forms") ap.add_argument("--salt", type=str, default="hunter2", help="up2k file-hash salt") + ap2 = ap.add_argument_group('database options') + ap2.add_argument("-e2d", action="store_true", help="enable up2k database") + ap2.add_argument("-e2ds", action="store_true", help="enable up2k db-scanner, sets -e2d") + ap2.add_argument("-e2dsa", action="store_true", help="scan all folders (for search), sets -e2ds") + ap2.add_argument("-e2t", action="store_true", help="enable metadata indexing") + ap2.add_argument("-e2ts", action="store_true", help="enable metadata scanner, sets -e2t") + ap2.add_argument("-e2tsr", action="store_true", help="rescan all metadata, sets -e2ts") + ap2.add_argument("--no-mutagen", action="store_true", help="use ffprobe for tags instead") + ap2.add_argument("-mtm", metavar="M=t,t,t", action="append", type=str, help="add/replace metadata mapping") + ap2.add_argument("-mte", metavar="M,M,M", type=str, help="tags to index/display (comma-sep.)", + default="circle,album,.tn,artist,title,.dur,.q") + ap2 = ap.add_argument_group('SSL/TLS options') ap2.add_argument("--http-only", action="store_true", help="disable ssl/tls") ap2.add_argument("--https-only", action="store_true", help="disable plaintext") @@ -257,14 +266,20 @@ def main(): ap2.add_argument("--ciphers", metavar="LIST", help="set allowed ciphers") ap2.add_argument("--ssl-dbg", action="store_true", help="dump some tls info") ap2.add_argument("--ssl-log", metavar="PATH", help="log master secrets") + al = ap.parse_args() # fmt: on - if al.e2dsa: - al.e2ds = True - - if al.e2ds: - al.e2d = True + # propagate implications + for k1, k2 in [ + ["e2dsa", "e2ds"], + ["e2ds", "e2d"], + ["e2tsr", "e2ts"], + ["e2ts", "e2t"], + ["e2t", "e2d"], + ]: + if getattr(al, k1): + setattr(al, k2, True) al.i = al.i.split(",") try: diff --git a/copyparty/authsrv.py b/copyparty/authsrv.py index 6d888fbc..c0833756 100644 --- a/copyparty/authsrv.py +++ b/copyparty/authsrv.py @@ -206,8 +206,11 @@ class AuthSrv(object): if lvl in "wa": mwrite[vol_dst].append(uname) if lvl == "c": - # config option, currently switches only - mflags[vol_dst][uname] = True + cval = True + if "=" in uname: + uname, cval = uname.split("=", 1) + + mflags[vol_dst][uname] = cval def reload(self): """ @@ -248,12 +251,19 @@ class AuthSrv(object): perms = perms.split(":") for (lvl, uname) in [[x[0], x[1:]] for x in perms]: if lvl == "c": - # config option, currently switches only - mflags[dst][uname] = True + cval = True + if "=" in uname: + uname, cval = uname.split("=", 1) + + mflags[dst][uname] = cval + continue + if uname == "": uname = "*" + if lvl in "ra": mread[dst].append(uname) + if lvl in "wa": mwrite[dst].append(uname) @@ -268,6 +278,7 @@ class AuthSrv(object): elif "" not in mount: # there's volumes but no root; make root inaccessible vfs = VFS(os.path.abspath("."), "") + vfs.flags["d2d"] = True maxdepth = 0 for dst in sorted(mount.keys(), key=lambda x: (x.count("/"), len(x))): @@ -300,15 +311,27 @@ class AuthSrv(object): ) raise Exception("invalid config") + for vol in vfs.all_vols.values(): + if (self.args.e2ds and vol.uwrite) or self.args.e2dsa: + vol.flags["e2ds"] = True + + if self.args.e2d: + vol.flags["e2d"] = True + + for k in ["e2t", "e2ts", "e2tsr"]: + if getattr(self.args, k): + vol.flags[k] = True + + # default tag-list if unset + if "mte" not in vol.flags: + vol.flags["mte"] = self.args.mte + try: v, _ = vfs.get("/", "*", False, True) if self.warn_anonwrite and os.getcwd() == v.realpath: self.warn_anonwrite = False - self.log( - "\033[31manyone can read/write the current directory: {}\033[0m".format( - v.realpath - ) - ) + msg = "\033[31manyone can read/write the current directory: {}\033[0m" + self.log(msg.format(v.realpath)) except Pebkac: self.warn_anonwrite = True diff --git a/copyparty/httpcli.py b/copyparty/httpcli.py index dc09f252..ba5587d7 100644 --- a/copyparty/httpcli.py +++ b/copyparty/httpcli.py @@ -222,6 +222,9 @@ class HttpCli(object): static_path = os.path.join(E.mod, "web/", self.vpath[5:]) return self.tx_file(static_path) + if "tree" in self.uparam: + return self.tx_tree() + # conditional redirect to single volumes if self.vpath == "" and not self.uparam: nread = len(self.rvol) @@ -246,9 +249,6 @@ class HttpCli(object): self.vpath = None return self.tx_mounts() - if "tree" in self.uparam: - return self.tx_tree() - return self.tx_browser() def handle_options(self): @@ -428,7 +428,6 @@ class HttpCli(object): body["ptop"] = vfs.realpath body["prel"] = rem body["addr"] = self.ip - body["flag"] = vfs.flags x = self.conn.hsrv.broker.put(True, "up2k.handle_json", body) response = x.get() @@ -445,20 +444,31 @@ class HttpCli(object): vols.append([vfs.vpath, vfs.realpath, vfs.flags]) idx = self.conn.get_u2idx() + t0 = time.time() if "srch" in body: # search by up2k hashlist vbody = copy.deepcopy(body) vbody["hash"] = len(vbody["hash"]) self.log("qj: " + repr(vbody)) hits = idx.fsearch(vols, body) - self.log("q#: " + repr(hits)) + self.log("q#: {} ({:.2f}s)".format(repr(hits), time.time() - t0)) + taglist = [] else: # search by query params self.log("qj: " + repr(body)) - hits = idx.search(vols, body) - self.log("q#: " + str(len(hits))) + hits, taglist = idx.search(vols, body) + self.log("q#: {} ({:.2f}s)".format(len(hits), time.time() - t0)) - r = json.dumps(hits).encode("utf-8") + order = [] + cfg = self.args.mte.split(",") + for t in cfg: + if t in taglist: + order.append(t) + for t in taglist: + if t not in order: + order.append(t) + + r = json.dumps({"hits": hits, "tag_order": order}).encode("utf-8") self.reply(r, mime="application/json") return True @@ -1186,6 +1196,11 @@ class HttpCli(object): is_ls = "ls" in self.uparam + icur = None + if "e2t" in vn.flags: + idx = self.conn.get_u2idx() + icur = idx.get_cur(vn.realpath) + dirs = [] files = [] for fn in vfs_ls: @@ -1241,6 +1256,31 @@ class HttpCli(object): dirs.append(item) else: files.append(item) + item["rd"] = rem + + taglist = {} + for f in files: + fn = f["name"] + rd = f["rd"] + del f["rd"] + if icur: + q = "select w from up where rd = ? and fn = ?" + r = icur.execute(q, (rd, fn)).fetchone() + if not r: + continue + + w = r[0][:16] + tags = {} + for k, v in icur.execute("select k, v from mt where w = ?", (w,)): + taglist[k] = True + tags[k] = v + + f["tags"] = tags + + if icur: + taglist = [k for k in self.args.mte.split(",") if k in taglist] + for f in dirs: + f["tags"] = {} srv_info = [] @@ -1293,6 +1333,7 @@ class HttpCli(object): "srvinf": srv_info, "perms": perms, "logues": logues, + "taglist": taglist, } ret = json.dumps(ret) self.reply(ret.encode("utf-8", "replace"), mime="application/json") @@ -1309,7 +1350,10 @@ class HttpCli(object): files=dirs, ts=ts, perms=json.dumps(perms), - have_up2k_idx=self.args.e2d, + taglist=taglist, + tag_order=json.dumps(self.args.mte.split(",")), + have_up2k_idx=("e2d" in vn.flags), + have_tags_idx=("e2t" in vn.flags), logues=logues, title=html_escape(self.vpath), srv_info=srv_info, diff --git a/copyparty/httpconn.py b/copyparty/httpconn.py index 8c6ac30f..cb808e0e 100644 --- a/copyparty/httpconn.py +++ b/copyparty/httpconn.py @@ -20,10 +20,12 @@ except ImportError: you do not have jinja2 installed,\033[33m choose one of these:\033[0m * apt install python-jinja2 - * python3 -m pip install --user jinja2 + * {} -m pip install --user jinja2 * (try another python version, if you have one) * (try copyparty.sfx instead) -""" +""".format( + os.path.basename(sys.executable) + ) ) sys.exit(1) diff --git a/copyparty/mtag.py b/copyparty/mtag.py new file mode 100644 index 00000000..60bff287 --- /dev/null +++ b/copyparty/mtag.py @@ -0,0 +1,305 @@ +# coding: utf-8 +from __future__ import print_function, unicode_literals +from math import fabs + +import re +import os +import sys +import shutil +import subprocess as sp + +from .__init__ import PY2, WINDOWS +from .util import fsenc, fsdec + + +class MTag(object): + def __init__(self, log_func, args): + self.log_func = log_func + self.usable = True + mappings = args.mtm + backend = "ffprobe" if args.no_mutagen else "mutagen" + + if backend == "mutagen": + self.get = self.get_mutagen + try: + import mutagen + except: + self.log("could not load mutagen, trying ffprobe instead") + backend = "ffprobe" + + if backend == "ffprobe": + self.get = self.get_ffprobe + # about 20x slower + if PY2: + cmd = ["ffprobe", "-version"] + try: + sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE) + except: + self.usable = False + else: + if not shutil.which("ffprobe"): + self.usable = False + + if not self.usable: + msg = "\033[31mneed mutagen or ffprobe to read media tags so please run this:\n {} -m pip install --user mutagen \033[0m" + self.log(msg.format(os.path.basename(sys.executable))) + return + + # https://picard-docs.musicbrainz.org/downloads/MusicBrainz_Picard_Tag_Map.html + tagmap = { + "album": ["album", "talb", "\u00a9alb", "original-album", "toal"], + "artist": [ + "artist", + "tpe1", + "\u00a9art", + "composer", + "performer", + "arranger", + "\u00a9wrt", + "tcom", + "tpe3", + "original-artist", + "tope", + ], + "title": ["title", "tit2", "\u00a9nam"], + "circle": [ + "album-artist", + "tpe2", + "aart", + "conductor", + "organization", + "band", + ], + ".tn": ["tracknumber", "trck", "trkn", "track"], + "genre": ["genre", "tcon", "\u00a9gen"], + "date": [ + "original-release-date", + "release-date", + "date", + "tdrc", + "\u00a9day", + "original-date", + "original-year", + "tyer", + "tdor", + "tory", + "year", + "creation-time", + ], + ".bpm": ["bpm", "tbpm", "tmpo", "tbp"], + "key": ["initial-key", "tkey", "key"], + "comment": ["comment", "comm", "\u00a9cmt", "comments", "description"], + } + + if mappings: + for k, v in [x.split("=") for x in mappings]: + tagmap[k] = v.split(",") + + self.tagmap = {} + for k, vs in tagmap.items(): + vs2 = [] + for v in vs: + if "-" not in v: + vs2.append(v) + continue + + vs2.append(v.replace("-", " ")) + vs2.append(v.replace("-", "_")) + vs2.append(v.replace("-", "")) + + self.tagmap[k] = vs2 + + self.rmap = { + v: [n, k] for k, vs in self.tagmap.items() for n, v in enumerate(vs) + } + # self.get = self.compare + + def log(self, msg): + self.log_func("mtag", msg) + + def normalize_tags(self, ret, md): + for k, v in dict(md).items(): + if not v: + continue + + k = k.lower().split("::")[0].strip() + mk = self.rmap.get(k) + if not mk: + continue + + pref, mk = mk + if mk not in ret or ret[mk][0] > pref: + ret[mk] = [pref, v[0]] + + # take first value + ret = {k: str(v[1]).strip() for k, v in ret.items()} + + # track 3/7 => track 3 + for k, v in ret.items(): + if k[0] == ".": + v = v.split("/")[0].strip().lstrip("0") + ret[k] = v or 0 + + return ret + + def compare(self, abspath): + if abspath.endswith(".au"): + return {} + + print("\n" + abspath) + r1 = self.get_mutagen(abspath) + r2 = self.get_ffprobe(abspath) + + keys = {} + for d in [r1, r2]: + for k in d.keys(): + keys[k] = True + + diffs = [] + l1 = [] + l2 = [] + for k in sorted(keys.keys()): + if k in [".q", ".dur"]: + continue # lenient + + v1 = r1.get(k) + v2 = r2.get(k) + if v1 == v2: + print(" ", k, v1) + elif v1 != "0000": # ffprobe date=0 + diffs.append(k) + print(" 1", k, v1) + print(" 2", k, v2) + if v1: + l1.append(k) + if v2: + l2.append(k) + + if diffs: + raise Exception() + + return r1 + + def get_mutagen(self, abspath): + import mutagen + + try: + md = mutagen.File(abspath, easy=True) + x = md.info.length + except Exception as ex: + return {} + + ret = {} + try: + dur = int(md.info.length) + try: + q = int(md.info.bitrate / 1024) + except: + q = int((os.path.getsize(abspath) / dur) / 128) + + ret[".dur"] = [0, dur] + ret[".q"] = [0, q] + except: + pass + + return self.normalize_tags(ret, md) + + def get_ffprobe(self, abspath): + cmd = ["ffprobe", "-hide_banner", "--", fsenc(abspath)] + p = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE) + r = p.communicate() + txt = r[1].decode("utf-8", "replace") + txt = [x.rstrip("\r") for x in txt.split("\n")] + + """ + note: + tags which contain newline will be truncated on first \n, + ffmpeg emits \n and spacepads the : to align visually + note: + the Stream ln always mentions Audio: if audio + the Stream ln usually has kb/s, is more accurate + the Duration ln always has kb/s + the Metadata: after Chapter may contain BPM info, + title : Tempo: 126.0 + + Input #0, wav, + Metadata: + date : + Duration: + Chapter # + Metadata: + title : + + Input #0, mp3, + Metadata: + album : + Duration: + Stream #0:0: Audio: + Stream #0:1: Video: + Metadata: + comment : + """ + + ptn_md_beg = re.compile("^( +)Metadata:$") + ptn_md_kv = re.compile("^( +)([^:]+) *: (.*)") + ptn_dur = re.compile("^ *Duration: ([^ ]+)(, |$)") + ptn_br1 = re.compile("^ *Duration: .*, bitrate: ([0-9]+) kb/s(, |$)") + ptn_br2 = re.compile("^ *Stream.*: Audio:.* ([0-9]+) kb/s(, |$)") + ptn_audio = re.compile("^ *Stream .*: Audio: ") + ptn_au_parent = re.compile("^ *(Input #|Stream .*: Audio: )") + + ret = {} + md = {} + in_md = False + is_audio = False + au_parent = False + for ln in txt: + m = ptn_md_kv.match(ln) + if m and in_md and len(m.group(1)) == in_md: + _, k, v = [x.strip() for x in m.groups()] + if k != "" and v != "": + md[k] = [v] + continue + else: + in_md = False + + m = ptn_md_beg.match(ln) + if m and au_parent: + in_md = len(m.group(1)) + 2 + continue + + au_parent = bool(ptn_au_parent.search(ln)) + + if ptn_audio.search(ln): + is_audio = True + + m = ptn_dur.search(ln) + if m: + sec = 0 + tstr = m.group(1) + if tstr.lower() != "n/a": + try: + tf = tstr.split(",")[0].split(".")[0].split(":") + for f in tf: + sec *= 60 + sec += int(f) + except: + self.log( + "\033[33minvalid timestr from ffmpeg: [{}]".format(tstr) + ) + + ret[".dur"] = sec + m = ptn_br1.search(ln) + if m: + ret[".q"] = m.group(1) + + m = ptn_br2.search(ln) + if m: + ret[".q"] = m.group(1) + + if not is_audio: + return {} + + ret = {k: [0, v] for k, v in ret.items()} + + return self.normalize_tags(ret, md) diff --git a/copyparty/svchub.py b/copyparty/svchub.py index 6d0299b0..c31f2ee5 100644 --- a/copyparty/svchub.py +++ b/copyparty/svchub.py @@ -39,14 +39,6 @@ class SvcHub(object): self.tcpsrv = TcpSrv(self) self.up2k = Up2k(self) - if self.args.e2ds: - auth = AuthSrv(self.args, self.log, False) - vols = auth.vfs.all_vols.values() - if not self.args.e2dsa: - vols = [x for x in vols if x.uwrite] - - self.up2k.build_indexes(vols) - # decide which worker impl to use if self.check_mp_enable(): from .broker_mp import BrokerMp as Broker diff --git a/copyparty/u2idx.py b/copyparty/u2idx.py index 76e046cc..62cd8950 100644 --- a/copyparty/u2idx.py +++ b/copyparty/u2idx.py @@ -37,7 +37,19 @@ class U2idx(object): fsize = body["size"] fhash = body["hash"] wark = up2k_wark_from_hashlist(self.args.salt, fsize, fhash) - return self.run_query(vols, "select * from up where w = ?", [wark]) + return self.run_query(vols, "w = ?", [wark], "", []) + + def get_cur(self, ptop): + cur = self.cur.get(ptop) + if cur: + return cur + + cur = _open(ptop) + if not cur: + return None + + self.cur[ptop] = cur + return cur def search(self, vols, body): """search by query params""" @@ -45,53 +57,74 @@ class U2idx(object): return [] qobj = {} - _conv_sz(qobj, body, "sz_min", "sz >= ?") - _conv_sz(qobj, body, "sz_max", "sz <= ?") - _conv_dt(qobj, body, "dt_min", "mt >= ?") - _conv_dt(qobj, body, "dt_max", "mt <= ?") - for seg, dk in [["path", "rd"], ["name", "fn"]]: + _conv_sz(qobj, body, "sz_min", "up.sz >= ?") + _conv_sz(qobj, body, "sz_max", "up.sz <= ?") + _conv_dt(qobj, body, "dt_min", "up.mt >= ?") + _conv_dt(qobj, body, "dt_max", "up.mt <= ?") + for seg, dk in [["path", "up.rd"], ["name", "up.fn"]]: if seg in body: _conv_txt(qobj, body, seg, dk) - qstr = "select * from up" - qv = [] - if qobj: - qk = [] - for k, v in sorted(qobj.items()): - qk.append(k.split("\n")[0]) - qv.append(v) + uq, uv = _sqlize(qobj) - qstr = " and ".join(qk) - qstr = "select * from up where " + qstr + tq = "" + tv = [] + qobj = {} + if "tags" in body: + _conv_txt(qobj, body, "tags", "mt.v") + tq, tv = _sqlize(qobj) - return self.run_query(vols, qstr, qv) + return self.run_query(vols, uq, uv, tq, tv) - def run_query(self, vols, qstr, qv): - qv = tuple(qv) - self.log("qs: {} {}".format(qstr, repr(qv))) + def run_query(self, vols, uq, uv, tq, tv): + self.log("qs: {} {} , {} {}".format(uq, repr(uv), tq, repr(tv))) ret = [] - lim = 100 + lim = 1000 + taglist = {} for (vtop, ptop, flags) in vols: - cur = self.cur.get(ptop) + cur = self.get_cur(ptop) if not cur: - cur = _open(ptop) - if not cur: - continue + continue - self.cur[ptop] = cur - # self.log("idx /{} @ {} {}".format(vtop, ptop, flags)) + if not tq: + if not uq: + q = "select * from up" + v = () + else: + q = "select * from up where " + uq + v = tuple(uv) + else: + # naive assumption: tags first + q = "select up.* from up inner join mt on substr(up.w,1,16) = mt.w where {}" + q = q.format(" and ".join([tq, uq]) if uq else tq) + v = tuple(tv + uv) - c = cur.execute(qstr, qv) - for _, ts, sz, rd, fn in c: + sret = [] + c = cur.execute(q, v) + for hit in c: + w, ts, sz, rd, fn = hit lim -= 1 if lim <= 0: break rp = os.path.join(vtop, rd, fn).replace("\\", "/") - ret.append({"ts": int(ts), "sz": sz, "rp": rp}) + sret.append({"ts": int(ts), "sz": sz, "rp": rp, "w": w[:16]}) - return ret + for hit in sret: + w = hit["w"] + del hit["w"] + tags = {} + q = "select k, v from mt where w = ?" + for k, v in cur.execute(q, (w,)): + taglist[k] = True + tags[k] = v + + hit["tags"] = tags + + ret.extend(sret) + + return ret, taglist.keys() def _open(ptop): @@ -146,3 +179,13 @@ def _conv_txt(q, body, k, sql): qk = "{} {} like {}?{}".format(sql, inv, head, tail) q[qk + "\n" + v] = u8safe(v) + + +def _sqlize(qobj): + keys = [] + values = [] + for k, v in sorted(qobj.items()): + keys.append(k.split("\n")[0]) + values.append(v) + + return " and ".join(keys), values diff --git a/copyparty/up2k.py b/copyparty/up2k.py index 90eea6e9..715cdb38 100644 --- a/copyparty/up2k.py +++ b/copyparty/up2k.py @@ -3,6 +3,7 @@ from __future__ import print_function, unicode_literals import re import os +import sys import time import math import json @@ -12,6 +13,7 @@ import shutil import base64 import hashlib import threading +import traceback from copy import deepcopy from .__init__ import WINDOWS @@ -27,6 +29,8 @@ from .util import ( w8b64enc, w8b64dec, ) +from .mtag import MTag +from .authsrv import AuthSrv try: HAVE_SQLITE3 = True @@ -55,12 +59,14 @@ class Up2k(object): # state self.mutex = threading.Lock() self.registry = {} + self.entags = {} + self.flags = {} self.cur = {} self.mem_cur = None if HAVE_SQLITE3: # mojibake detector - self.mem_cur = sqlite3.connect(":memory:", check_same_thread=False).cursor() + self.mem_cur = self._orz(":memory:") self.mem_cur.execute(r"create table a (b text)") if WINDOWS: @@ -70,10 +76,9 @@ class Up2k(object): thr.daemon = True thr.start() - if self.persist: - thr = threading.Thread(target=self._snapshot) - thr.daemon = True - thr.start() + self.mtag = MTag(self.log_func, self.args) + if not self.mtag.usable: + self.mtag = None # static self.r_hash = re.compile("^[0-9a-zA-Z_-]{43}$") @@ -81,6 +86,15 @@ class Up2k(object): if self.persist and not HAVE_SQLITE3: self.log("could not initialize sqlite3, will use in-memory registry only") + # this is kinda jank + auth = AuthSrv(self.args, self.log, False) + self.init_indexes(auth) + + if self.persist: + thr = threading.Thread(target=self._snapshot) + thr.daemon = True + thr.start() + def log(self, msg): self.log_func("up2k", msg + "\033[K") @@ -119,7 +133,49 @@ class Up2k(object): return ret - def register_vpath(self, ptop): + def init_indexes(self, auth): + self.pp = ProgressPrinter() + vols = auth.vfs.all_vols.values() + t0 = time.time() + + # e2ds(a) volumes first, + # also covers tags where e2ts is set + for vol in vols: + en = {} + if "mte" in vol.flags: + en = {k: True for k in vol.flags["mte"].split(",")} + + self.entags[vol.realpath] = en + + if "e2ds" in vol.flags: + r = self._build_file_index(vol, vols) + if not r: + needed_mutagen = True + + # open the rest + do any e2ts(a) + needed_mutagen = False + for vol in vols: + r = self.register_vpath(vol.realpath, vol.flags) + if not r or "e2ts" not in vol.flags: + continue + + cur, db_path, sz0 = r + n_add, n_rm, success = self._build_tags_index(vol.realpath) + if not success: + needed_mutagen = True + + if n_add or n_rm: + self.vac(cur, db_path, n_add, n_rm, sz0) + + self.pp.end = True + msg = "{} volumes in {:.2f} sec" + self.log(msg.format(len(vols), time.time() - t0)) + + if needed_mutagen: + msg = "\033[31mcould not read tags because no backends are available (mutagen or ffprobe)\033[0m" + self.log(msg) + + def register_vpath(self, ptop, flags): with self.mutex: if ptop in self.registry: return None @@ -138,8 +194,9 @@ class Up2k(object): m = [m] + self._vis_reg_progress(reg) self.log("\n".join(m)) + self.flags[ptop] = flags self.registry[ptop] = reg - if not self.persist or not HAVE_SQLITE3: + if not self.persist or not HAVE_SQLITE3 or "d2d" in flags: return None try: @@ -152,47 +209,54 @@ class Up2k(object): return None try: + sz0 = 0 + if os.path.exists(db_path): + sz0 = os.path.getsize(db_path) // 1024 + cur = self._open_db(db_path) self.cur[ptop] = cur - return cur - except Exception as ex: - self.log("cannot use database at [{}]: {}".format(ptop, repr(ex))) + return [cur, db_path, sz0] + except: + msg = "cannot use database at [{}]:\n{}" + self.log(msg.format(ptop, traceback.format_exc())) return None - def build_indexes(self, writeables): - tops = [d.realpath for d in writeables] - self.pp = ProgressPrinter() - t0 = time.time() - for top in tops: - dbw = [self.register_vpath(top), 0, time.time()] - if not dbw[0]: - continue + def _build_file_index(self, vol, all_vols): + do_vac = False + top = vol.realpath + reg = self.register_vpath(top, vol.flags) + if not reg: + return - self.pp.n = next(dbw[0].execute("select count(w) from up"))[0] - db_path = os.path.join(top, ".hist", "up2k.db") - sz0 = os.path.getsize(db_path) // 1024 - - # can be symlink so don't `and d.startswith(top)`` - excl = set([d for d in tops if d != top]) - n_add = self._build_dir(dbw, top, excl, top) - n_rm = self._drop_lost(dbw[0], top) - if dbw[1]: - self.log("commit {} new files".format(dbw[1])) + _, db_path, sz0 = reg + dbw = [reg[0], 0, time.time()] + self.pp.n = next(dbw[0].execute("select count(w) from up"))[0] + # can be symlink so don't `and d.startswith(top)`` + excl = set([d.realpath for d in all_vols if d != vol]) + n_add = self._build_dir(dbw, top, excl, top) + n_rm = self._drop_lost(dbw[0], top) + if dbw[1]: + self.log("commit {} new files".format(dbw[1])) dbw[0].connection.commit() - if n_add or n_rm: - db_path = os.path.join(top, ".hist", "up2k.db") - sz1 = os.path.getsize(db_path) // 1024 - dbw[0].execute("vacuum") - sz2 = os.path.getsize(db_path) // 1024 - msg = "{} new, {} del, {} kB vacced, {} kB gain, {} kB now".format( - n_add, n_rm, sz1 - sz2, sz2 - sz0, sz2 - ) - self.log(msg) - self.pp.end = True - self.log("{} volumes in {:.2f} sec".format(len(tops), time.time() - t0)) + n_add, n_rm, success = self._build_tags_index(vol.realpath) + + dbw[0].connection.commit() + if n_add or n_rm or do_vac: + self.vac(dbw[0], db_path, n_add, n_rm, sz0) + + return success + + def vac(self, cur, db_path, n_add, n_rm, sz0): + sz1 = os.path.getsize(db_path) // 1024 + cur.execute("vacuum") + sz2 = os.path.getsize(db_path) // 1024 + msg = "{} new, {} del, {} kB vacced, {} kB gain, {} kB now".format( + n_add, n_rm, sz1 - sz2, sz2 - sz0, sz2 + ) + self.log(msg) def _build_dir(self, dbw, top, excl, cdir): try: @@ -298,39 +362,144 @@ class Up2k(object): return len(rm) + def _build_tags_index(self, ptop): + entags = self.entags[ptop] + flags = self.flags[ptop] + cur = self.cur[ptop] + n_add = 0 + n_rm = 0 + n_buf = 0 + last_write = time.time() + + if "e2tsr" in flags: + n_rm = cur.execute("select count(w) from mt").fetchone()[0] + if n_rm: + self.log("discarding {} media tags for a full rescan".format(n_rm)) + cur.execute("delete from mt") + else: + self.log("volume has e2tsr but there are no media tags to discard") + + # integrity: drop tags for tracks that were deleted + if "e2t" in flags: + drops = [] + c2 = cur.connection.cursor() + up_q = "select w from up where substr(w,1,16) = ?" + for (w,) in cur.execute("select w from mt"): + if not c2.execute(up_q, (w,)).fetchone(): + drops.append(w) + c2.close() + + if drops: + msg = "discarding media tags for {} deleted files" + self.log(msg.format(len(drops))) + n_rm += len(drops) + for w in drops: + cur.execute("delete from up where rowid = ?", (w,)) + + # bail if a volume flag disables indexing + if "d2t" in flags or "d2d" in flags: + return n_add, n_rm, True + + # add tags for new files + if "e2ts" in flags: + if not self.mtag: + return n_add, n_rm, False + + c2 = cur.connection.cursor() + n_left = cur.execute("select count(w) from up").fetchone()[0] + for w, rd, fn in cur.execute("select w, rd, fn from up"): + n_left -= 1 + q = "select w from mt where w = ?" + if c2.execute(q, (w[:16],)).fetchone(): + continue + + abspath = os.path.join(ptop, rd, fn) + self.pp.msg = "c{} {}".format(n_left, abspath) + tags = self.mtag.get(abspath) + tags = {k: v for k, v in tags.items() if k in entags} + if not tags: + # indicate scanned without tags + tags = {"x": 0} + + for k, v in tags.items(): + q = "insert into mt values (?,?,?)" + c2.execute(q, (w[:16], k, v)) + n_add += 1 + n_buf += 1 + + td = time.time() - last_write + if n_buf >= 4096 or td >= 60: + self.log("commit {} new tags".format(n_buf)) + cur.connection.commit() + last_write = time.time() + n_buf = 0 + + c2.close() + + return n_add, n_rm, True + + def _orz(self, db_path): + return sqlite3.connect(db_path, check_same_thread=False).cursor() + def _open_db(self, db_path): existed = os.path.exists(db_path) - cur = sqlite3.connect(db_path, check_same_thread=False).cursor() + cur = self._orz(db_path) try: ver = self._read_ver(cur) - - if ver == 1: - cur = self._upgrade_v1(cur, db_path) - ver = self._read_ver(cur) - - if ver == 2: - try: - nfiles = next(cur.execute("select count(w) from up"))[0] - self.log("found DB at {} |{}|".format(db_path, nfiles)) - return cur - except Exception as ex: - self.log("WARN: could not list files, DB corrupt?\n " + repr(ex)) - - if ver is not None: - self.log("REPLACING unsupported DB (v.{}) at {}".format(ver, db_path)) - elif not existed: - raise Exception("whatever") - - conn = cur.connection - cur.close() - conn.close() - os.unlink(db_path) - cur = sqlite3.connect(db_path, check_same_thread=False).cursor() except: - pass + ver = None + if not existed: + return self._create_db(db_path, cur) + + orig_ver = ver + if not ver or ver < 3: + bak = "{}.bak.{:x}.v{}".format(db_path, int(time.time()), ver) + db = cur.connection + cur.close() + db.close() + msg = "creating new DB (old is bad); backup: {}" + if ver: + msg = "creating backup before upgrade: {}" + + self.log(msg.format(bak)) + shutil.copy2(db_path, bak) + cur = self._orz(db_path) + + if ver == 1: + cur = self._upgrade_v1(cur, db_path) + if cur: + ver = 2 + + if ver == 2: + cur = self._create_v3(cur) + ver = self._read_ver(cur) if cur else None + + if ver == 3: + if orig_ver != ver: + cur.connection.commit() + cur.execute("vacuum") + cur.connection.commit() + + try: + nfiles = next(cur.execute("select count(w) from up"))[0] + self.log("OK: {} |{}|".format(db_path, nfiles)) + return cur + except Exception as ex: + self.log("WARN: could not list files, DB corrupt?\n " + repr(ex)) + + if cur: + db = cur.connection + cur.close() + db.close() + + return self._create_db(db_path, None) + + def _create_db(self, db_path, cur): + if not cur: + cur = self._orz(db_path) - # sqlite is variable-width only, no point in using char/nchar/varchar self._create_v2(cur) + self._create_v3(cur) cur.connection.commit() self.log("created DB at {}".format(db_path)) return cur @@ -348,24 +517,45 @@ class Up2k(object): def _create_v2(self, cur): for cmd in [ - r"create table ks (k text, v text)", - r"create table ki (k text, v int)", r"create table up (w text, mt int, sz int, rd text, fn text)", - r"insert into ki values ('sver', 2)", - r"create index up_w on up(w)", r"create index up_rd on up(rd)", r"create index up_fn on up(fn)", ]: cur.execute(cmd) + return cur + + def _create_v3(self, cur): + """ + collision in 2^(n/2) files where n = bits (6 bits/ch) + 10*6/2 = 2^30 = 1'073'741'824, 24.1mb idx + 12*6/2 = 2^36 = 68'719'476'736, 24.8mb idx + 16*6/2 = 2^48 = 281'474'976'710'656, 26.1mb idx + """ + for c, ks in [["drop table k", "isv"], ["drop index up_", "w"]]: + for k in ks: + try: + cur.execute(c + k) + except: + pass + + for cmd in [ + r"create index up_w on up(substr(w,1,16))", + r"create table mt (w text, k text, v int)", + r"create index mt_w on mt(w)", + r"create index mt_k on mt(k)", + r"create index mt_v on mt(v)", + r"create table kv (k text, v int)", + r"insert into kv values ('sver', 3)", + ]: + cur.execute(cmd) + return cur def _upgrade_v1(self, odb, db_path): - self.log("\033[33mupgrading v1 to v2:\033[0m {}".format(db_path)) - npath = db_path + ".next" if os.path.exists(npath): os.unlink(npath) - ndb = sqlite3.connect(npath, check_same_thread=False).cursor() + ndb = self._orz(npath) self._create_v2(ndb) c = odb.execute("select * from up") @@ -377,14 +567,10 @@ class Up2k(object): ndb.connection.commit() ndb.connection.close() odb.connection.close() - bpath = db_path + ".bak.v1" - self.log("success; backup at: " + bpath) - atomic_move(db_path, bpath) atomic_move(npath, db_path) - return sqlite3.connect(db_path, check_same_thread=False).cursor() + return self._orz(db_path) def handle_json(self, cj): - self.register_vpath(cj["ptop"]) cj["name"] = sanitize_fn(cj["name"]) cj["poke"] = time.time() wark = self._get_wark(cj) @@ -394,7 +580,10 @@ class Up2k(object): cur = self.cur.get(cj["ptop"], None) reg = self.registry[cj["ptop"]] if cur: - cur = cur.execute(r"select * from up where w = ?", (wark,)) + cur = cur.execute( + r"select * from up where substr(w,1,16) = ? and w = ?", + (wark[:16], wark,), + ) for _, dtime, dsize, dp_dir, dp_fn in cur: if dp_dir.startswith("//") or dp_fn.startswith("//"): dp_dir, dp_fn = self.w8dec(dp_dir, dp_fn) @@ -407,7 +596,6 @@ class Up2k(object): "prel": dp_dir, "vtop": cj["vtop"], "ptop": cj["ptop"], - "flag": cj["flag"], "size": dsize, "lmod": dtime, "hash": [], @@ -444,7 +632,7 @@ class Up2k(object): err = "partial upload exists at a different location; please resume uploading here instead:\n" err += "/" + vsrc + " " raise Pebkac(400, err) - elif "nodupe" in job["flag"]: + elif "nodupe" in self.flags[job["ptop"]]: self.log("dupe-reject:\n {0}\n {1}".format(src, dst)) err = "upload rejected, file already exists:\n/" + vsrc + " " raise Pebkac(400, err) @@ -474,7 +662,6 @@ class Up2k(object): "vtop", "ptop", "prel", - "flag", "name", "size", "lmod", @@ -603,7 +790,7 @@ class Up2k(object): def db_add(self, db, wark, rd, fn, ts, sz): sql = "insert into up values (?,?,?,?,?)" - v = (wark, ts, sz, rd, fn) + v = (wark, int(ts), sz, rd, fn) try: db.execute(sql, v) except: diff --git a/copyparty/web/browser.css b/copyparty/web/browser.css index 7d8d929a..5453a6e8 100644 --- a/copyparty/web/browser.css +++ b/copyparty/web/browser.css @@ -46,7 +46,7 @@ body { display: none; } #files { - border-collapse: collapse; + border-spacing: 0; margin-top: 2em; z-index: 1; position: relative; @@ -94,6 +94,13 @@ a, margin: 0; padding: 0 .5em; } +#files td { + border-bottom: 1px solid #111; + max-width: 30em; +} +#files tr+tr td { + border-top: 1px solid #383838; +} #files tbody td:nth-child(3) { font-family: monospace; font-size: 1.3em; @@ -111,6 +118,7 @@ a, #files tbody tr:last-child td { padding-bottom: 1.3em; border-bottom: .5em solid #444; + white-space: nowrap; } #files thead th[style] { width: auto !important; @@ -400,14 +408,13 @@ input[type="checkbox"]:checked+label { color: #fff; } #files td div a { - display: table-cell; + display: inline-block; white-space: nowrap; } #files td div a:last-child { width: 100%; } #files td div { - display: table; border-collapse: collapse; width: 100%; } diff --git a/copyparty/web/browser.html b/copyparty/web/browser.html index 5bf98cf5..d9bbc576 100644 --- a/copyparty/web/browser.html +++ b/copyparty/web/browser.html @@ -26,7 +26,11 @@ {%- include 'upload.html' %} @@ -55,7 +59,14 @@ File Name - File Size + Size + {%- for k in taglist %} + {%- if k.startswith('.') %} + {{ k[1:] }} + {%- else %} + {{ k[0]|upper }}{{ k[1:] }} + {%- endif %} + {%- endfor %} T Date @@ -63,7 +74,13 @@ {%- for f in files %} -{{ f.lead }}{{ f.name|e }}{{ f.sz }}{{ f.ext }}{{ f.dt }} + {{ f.lead }}{{ f.name|e }}{{ f.sz }} + {%- if f.tags is defined %} + {%- for k in taglist %} + {{ f.tags[k] }} + {%- endfor %} + {%- endif %} + {{ f.ext }}{{ f.dt }} {%- endfor %} @@ -86,7 +103,10 @@ - + + diff --git a/copyparty/web/browser.js b/copyparty/web/browser.js index 37d1741b..da7420a1 100644 --- a/copyparty/web/browser.js +++ b/copyparty/web/browser.js @@ -472,7 +472,7 @@ function play(tid, call_depth) { o.setAttribute('id', 'thx_js'); if (window.history && history.replaceState) { var nurl = (document.location + '').split('#')[0] + '#' + oid; - history.replaceState(ebi('files').tBodies[0].innerHTML, nurl, nurl); + history.replaceState(ebi('files').innerHTML, nurl, nurl); } else { document.location.hash = oid; @@ -591,6 +591,12 @@ function autoplay_blocked() { ["name", "name", "name contains   (negate with -nope)", "46"] ] ]; + + if (document.querySelector('#srch_form.tags')) + sconf.push(["tags", + ["tags", "tags", "tags contains", "46"] + ]); + var html = []; var orig_html = null; for (var a = 0; a < sconf.length; a++) { @@ -653,6 +659,9 @@ function autoplay_blocked() { return; } + var res = JSON.parse(this.responseText), + tagord = res.tag_order; + var ofiles = ebi('files'); if (ofiles.getAttribute('ts') > this.ts) return; @@ -660,10 +669,11 @@ function autoplay_blocked() { ebi('path').style.display = 'none'; ebi('tree').style.display = 'none'; - var html = ['-close search results']; - var res = JSON.parse(this.responseText); - for (var a = 0; a < res.length; a++) { - var r = res[a], + var html = mk_files_header(tagord); + html.push(''); + html.push('-close search results'); + for (var a = 0; a < res.hits.length; a++) { + var r = res.hits[a], ts = parseInt(r.ts), sz = esc(r.sz + ''), rp = esc(r.rp + ''), @@ -674,14 +684,29 @@ function autoplay_blocked() { ext = '%'; links = links.join(''); - html.push('-
' + links + '
' + sz + - '' + ext + '' + unix2iso(ts) + ''); + var nodes = ['-
' + links + '
', sz]; + for (var b = 0; b < tagord.length; b++) { + var k = tagord[b], + v = r.tags[k] || ""; + + if (k == "dur") { + var sv = s2ms(v); + nodes[nodes.length - 1] += '' + sv; + continue; + } + + nodes.push(v); + } + + nodes = nodes.concat([ext, unix2iso(ts)]); + html.push(nodes.join('')); + html.push(''); } if (!orig_html) - orig_html = ebi('files').tBodies[0].innerHTML; + orig_html = ebi('files').innerHTML; - ofiles.tBodies[0].innerHTML = html.join('\n'); + ofiles.innerHTML = html.join('\n'); ofiles.setAttribute("ts", this.ts); reload_browser(); @@ -692,7 +717,7 @@ function autoplay_blocked() { ev(e); ebi('path').style.display = 'inline-block'; ebi('tree').style.display = 'block'; - ebi('files').tBodies[0].innerHTML = orig_html; + ebi('files').innerHTML = orig_html; orig_html = null; reload_browser(); } @@ -851,17 +876,34 @@ function autoplay_blocked() { ebi('srv_info').innerHTML = '' + res.srvinf + ''; var nodes = res.dirs.concat(res.files); var top = this.top; - var html = []; + var html = mk_files_header(res.taglist); + html.push(''); for (var a = 0; a < nodes.length; a++) { var r = nodes[a], - ln = '' + r.lead + '' + esc(decodeURIComponent(r.href)) + ''; + ln = ['' + r.lead + '' + esc(decodeURIComponent(r.href)) + '', r.sz]; - ln = [ln, r.sz, r.ext, unix2iso(r.ts)].join(''); + for (var b = 0; b < res.taglist.length; b++) { + var k = res.taglist[b], + v = (r.tags || {})[k] || ""; + + if (k[0] == '.') + k = k.slice(1); + + if (k == "dur") { + var sv = s2ms(v); + ln[ln.length - 1] += '' + sv; + continue; + } + ln.push(v); + } + ln = ln.concat([r.ext, unix2iso(r.ts)]).join(''); html.push(ln + ''); } + html.push(''); html = html.join('\n'); - ebi('files').tBodies[0].innerHTML = html; + ebi('files').innerHTML = html; + history.pushState(html, this.top, this.top); apply_perms(res.perms); despin('#files'); @@ -924,7 +966,7 @@ function autoplay_blocked() { window.onpopstate = function (e) { console.log(e.url + ' ,, ' + ((e.state + '').slice(0, 64))); if (e.state) { - ebi('files').tBodies[0].innerHTML = e.state; + ebi('files').innerHTML = e.state; reload_tree(); reload_browser(); } @@ -932,7 +974,7 @@ function autoplay_blocked() { if (window.history && history.pushState) { var u = get_vpath(); - history.replaceState(ebi('files').tBodies[0].innerHTML, u, u); + history.replaceState(ebi('files').innerHTML, u, u); } })(); @@ -989,6 +1031,28 @@ function apply_perms(perms) { } +function mk_files_header(taglist) { + var html = ['', '', 'File Name', 'Size']; + for (var a = 0; a < taglist.length; a++) { + var tag = taglist[a]; + var c1 = tag.slice(0, 1).toUpperCase(); + tag = c1 + tag.slice(1); + if (c1 == '.') + tag = '' + tag.slice(1); + else + tag = '' + tag; + + html.push(tag + ''); + } + html = html.concat([ + 'T', + 'Date', + '', + ]); + return html; +} + + function reload_browser(not_mp) { makeSortable(ebi('files')); @@ -1012,6 +1076,7 @@ function reload_browser(not_mp) { hsz = sz.replace(/\B(?=(\d{3})+(?!\d))/g, " "); oo[a].textContent = hsz; + oo[a].setAttribute("sortv", sz); } if (!not_mp) { diff --git a/copyparty/web/up2k.js b/copyparty/web/up2k.js index fa52c6b0..f6b750ea 100644 --- a/copyparty/web/up2k.js +++ b/copyparty/web/up2k.js @@ -772,13 +772,13 @@ function up2k_init(have_crypto) { if (!response.name) { var msg = ''; var smsg = ''; - if (!response || !response.length) { + if (!response || !response.hits || !response.hits.length) { msg = 'not found on server'; smsg = '404'; } else { smsg = 'found'; - var hit = response[0], + var hit = response.hits[0], msg = linksplit(hit.rp).join(''), tr = unix2iso(hit.ts), tu = unix2iso(t.lmod), diff --git a/copyparty/web/util.js b/copyparty/web/util.js index c3e49c7a..1d1575b9 100644 --- a/copyparty/web/util.js +++ b/copyparty/web/util.js @@ -76,7 +76,7 @@ function import_js(url, cb) { function sortTable(table, col) { - var tb = table.tBodies[0], // use `` to ignore `` and `` rows + var tb = table.tBodies[0], th = table.tHead.rows[0].cells, tr = Array.prototype.slice.call(tb.rows, 0), i, reverse = th[col].className == 'sort1' ? -1 : 1; @@ -90,11 +90,11 @@ function sortTable(table, col) { if (!b.cells[col]) return 1; - var v1 = a.cells[col].textContent.trim(); - var v2 = b.cells[col].textContent.trim(); + var v1 = a.cells[col].getAttribute('sortv') || a.cells[col].textContent.trim(); + var v2 = b.cells[col].getAttribute('sortv') || b.cells[col].textContent.trim(); if (stype == 'int') { - v1 = parseInt(v1.replace(/,/g, '')); - v2 = parseInt(v2.replace(/,/g, '')); + v1 = parseInt(v1.replace(/,/g, '')) || 0; + v2 = parseInt(v2.replace(/,/g, '')) || 0; return reverse * (v1 - v2); } return reverse * (v1.localeCompare(v2)); @@ -225,6 +225,12 @@ function unix2iso(ts) { } +function s2ms(s) { + var m = Math.floor(s / 60); + return m + ":" + ("0" + (s - m * 60)).slice(-2); +} + + function has(haystack, needle) { for (var a = 0; a < haystack.length; a++) if (haystack[a] == needle)