diff --git a/.vscode/launch.json b/.vscode/launch.json index d0fa0ad1..fdaaf284 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -12,8 +12,7 @@ //"-nw", "-ed", "-emp", - "-e2d", - "-e2s", + "-e2dsa", "-a", "ed:wark", "-v", diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 592d8710..f4a637f1 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -8,7 +8,7 @@ }, { "label": "no_dbg", - "command": "${config:python.pythonPath} -m copyparty -ed -emp -e2d -e2s -a ed:wark -v srv::r:aed:cnodupe ;exit 1", + "command": "${config:python.pythonPath} -m copyparty -ed -emp -e2dsa -a ed:wark -v srv::r:aed:cnodupe ;exit 1", "type": "shell" } ] diff --git a/copyparty/__main__.py b/copyparty/__main__.py index ab8e17c7..63a402b7 100644 --- a/copyparty/__main__.py +++ b/copyparty/__main__.py @@ -174,6 +174,18 @@ def main(): if HAVE_SSL: ensure_cert() + deprecated = [["-e2s", "-e2ds"]] + for dk, nk in deprecated: + try: + idx = sys.argv.index(dk) + except: + continue + + msg = "\033[1;31mWARNING:\033[0;1m\n {} \033[0;33mwas replaced with\033[0;1m {} \033[0;33mand will be removed\n\033[0m" + print(msg.format(dk, nk)) + sys.argv[idx] = nk + time.sleep(2) + ap = argparse.ArgumentParser( formatter_class=RiceFormatter, prog="copyparty", @@ -228,13 +240,15 @@ def main(): ap.add_argument("-ed", action="store_true", help="enable ?dots") ap.add_argument("-emp", action="store_true", help="enable markdown plugins") ap.add_argument("-e2d", action="store_true", help="enable up2k database") - ap.add_argument("-e2s", action="store_true", help="enable up2k db-scanner") + ap.add_argument("-e2ds", action="store_true", help="enable up2k db-scanner, sets -e2d") + ap.add_argument("-e2dsa", action="store_true", help="scan all folders (for search), sets -e2ds") ap.add_argument("-mcr", metavar="SEC", type=int, default=60, help="md-editor mod-chk rate") ap.add_argument("-nw", action="store_true", help="disable writes (benchmark)") ap.add_argument("-nih", action="store_true", help="no info hostname") ap.add_argument("-nid", action="store_true", help="no info disk-usage") ap.add_argument("--no-sendfile", action="store_true", help="disable sendfile") ap.add_argument("--urlform", type=str, default="print,get", help="how to handle url-forms") + ap.add_argument("--salt", type=str, default="hunter2", help="up2k file-hash salt") ap2 = ap.add_argument_group('SSL/TLS options') ap2.add_argument("--http-only", action="store_true", help="disable ssl/tls") @@ -246,6 +260,12 @@ def main(): al = ap.parse_args() # fmt: on + if al.e2dsa: + al.e2ds = True + + if al.e2ds: + al.e2d = True + al.i = al.i.split(",") try: if "-" in al.p: diff --git a/copyparty/authsrv.py b/copyparty/authsrv.py index bd7b72dd..6d888fbc 100644 --- a/copyparty/authsrv.py +++ b/copyparty/authsrv.py @@ -19,6 +19,11 @@ class VFS(object): self.uwrite = uwrite # users who can write this self.flags = flags # config switches self.nodes = {} # child nodes + self.all_vols = {vpath: self} # flattened recursive + + def _trk(self, vol): + self.all_vols[vol.vpath] = vol + return vol def add(self, src, dst): """get existing, or add new path to the vfs""" @@ -30,7 +35,7 @@ class VFS(object): name, dst = dst.split("/", 1) if name in self.nodes: # exists; do not manipulate permissions - return self.nodes[name].add(src, dst) + return self._trk(self.nodes[name].add(src, dst)) vn = VFS( "{}/{}".format(self.realpath, name), @@ -40,7 +45,7 @@ class VFS(object): self.flags, ) self.nodes[name] = vn - return vn.add(src, dst) + return self._trk(vn.add(src, dst)) if dst in self.nodes: # leaf exists; return as-is @@ -50,7 +55,7 @@ class VFS(object): vp = "{}/{}".format(self.vpath, dst).lstrip("/") vn = VFS(src, vp) self.nodes[dst] = vn - return vn + return self._trk(vn) def _find(self, vpath): """return [vfs,remainder]""" @@ -257,7 +262,6 @@ class AuthSrv(object): with open(cfg_fn, "rb") as f: self._parse_config_file(f, user, mread, mwrite, mflags, mount) - self.all_writable = [] if not mount: # -h says our defaults are CWD at root and read/write for everyone vfs = VFS(os.path.abspath("."), "", ["*"], ["*"]) @@ -280,11 +284,6 @@ class AuthSrv(object): v.uread = mread[dst] v.uwrite = mwrite[dst] v.flags = mflags[dst] - if v.uwrite: - self.all_writable.append(v) - - if vfs.uwrite and vfs not in self.all_writable: - self.all_writable.append(vfs) missing_users = {} for d in [mread, mwrite]: diff --git a/copyparty/httpcli.py b/copyparty/httpcli.py index 36544bd1..e7bb0877 100644 --- a/copyparty/httpcli.py +++ b/copyparty/httpcli.py @@ -5,6 +5,7 @@ import os import stat import gzip import time +import copy import json import socket import ctypes @@ -125,15 +126,15 @@ class HttpCli(object): k, v = k.split("=", 1) uparam[k.lower()] = v.strip() else: - uparam[k.lower()] = True + uparam[k.lower()] = False self.uparam = uparam self.vpath = unquotep(vpath) ua = self.headers.get("user-agent", "") if ua.startswith("rclone/"): - uparam["raw"] = True - uparam["dots"] = True + uparam["raw"] = False + uparam["dots"] = False try: if self.mode in ["GET", "HEAD"]: @@ -237,12 +238,15 @@ class HttpCli(object): ) if not self.readable and not self.writable: self.log("inaccessible: [{}]".format(self.vpath)) - self.uparam = {"h": True} + self.uparam = {"h": False} if "h" in self.uparam: self.vpath = None return self.tx_mounts() + if "tree" in self.uparam: + return self.tx_tree() + return self.tx_browser() def handle_options(self): @@ -401,6 +405,9 @@ class HttpCli(object): except: raise Pebkac(422, "you POSTed invalid json") + if "srch" in self.uparam or "srch" in body: + return self.handle_search(body) + # prefer this over undot; no reason to allow traversion if "/" in body["name"]: raise Pebkac(400, "folders verboten") @@ -426,6 +433,30 @@ class HttpCli(object): self.reply(response.encode("utf-8"), mime="application/json") return True + def handle_search(self, body): + vols = [] + for vtop in self.rvol: + vfs, _ = self.conn.auth.vfs.get(vtop, self.uname, True, False) + vols.append([vfs.vpath, vfs.realpath, vfs.flags]) + + idx = self.conn.get_u2idx() + if "srch" in body: + # search by up2k hashlist + vbody = copy.deepcopy(body) + vbody["hash"] = len(vbody["hash"]) + self.log("qj: " + repr(vbody)) + hits = idx.fsearch(vols, body) + self.log("qh: " + repr(hits)) + else: + # search by query params + self.log("qj: " + repr(body)) + hits = idx.search(vols, body) + self.log("qh: " + str(len(hits))) + + r = json.dumps(hits).encode("utf-8") + self.reply(r, mime="application/json") + return True + def handle_post_binary(self): try: remains = int(self.headers["content-length"]) @@ -1037,6 +1068,60 @@ class HttpCli(object): self.reply(html.encode("utf-8")) return True + def tx_tree(self): + top = self.uparam["tree"] or "" + dst = self.vpath + if top in [".", ".."]: + top = undot(self.vpath + "/" + top) + + if top == dst: + dst = "" + elif top: + if not dst.startswith(top + "/"): + raise Pebkac(400, "arg funk") + + dst = dst[len(top) + 1 :] + + ret = self.gen_tree(top, dst) + ret = json.dumps(ret) + self.reply(ret.encode("utf-8")) + return True + + def gen_tree(self, top, target): + ret = {} + excl = None + if target: + excl, target = (target.split("/", 1) + [""])[:2] + ret["k" + excl] = self.gen_tree("/".join([top, excl]).strip("/"), target) + + try: + vn, rem = self.auth.vfs.get(top, self.uname, self.readable, self.writable) + fsroot, vfs_ls, vfs_virt = vn.ls(rem, self.uname) + except: + vfs_ls = [] + vfs_virt = {} + for v in self.rvol: + d1, d2 = v.rsplit("/", 1) if "/" in v else ["", v] + if d1 == top: + vfs_virt[d2] = 0 + + dirs = [] + + if not self.args.ed or "dots" not in self.uparam: + vfs_ls = exclude_dotfiles(vfs_ls) + + for fn in [x for x in vfs_ls if x != excl]: + abspath = os.path.join(fsroot, fn) + if os.path.isdir(abspath): + dirs.append(fn) + + for x in vfs_virt.keys(): + if x != excl: + dirs.append(x) + + ret["a"] = dirs + return ret + def tx_browser(self): vpath = "" vpnodes = [["", "/"]] @@ -1062,8 +1147,7 @@ class HttpCli(object): if abspath.endswith(".md") and "raw" not in self.uparam: return self.tx_md(abspath) - bad = "{0}.hist{0}up2k.".format(os.sep) - if abspath.endswith(bad + "db") or abspath.endswith(bad + "snap"): + if rem.startswith(".hist/up2k."): raise Pebkac(403) return self.tx_file(abspath) @@ -1092,8 +1176,8 @@ class HttpCli(object): vfs_ls = exclude_dotfiles(vfs_ls) hidden = [] - if fsroot.endswith(str(os.sep) + ".hist"): - hidden = ["up2k.db", "up2k.snap"] + if rem == ".hist": + hidden = ["up2k."] dirs = [] files = [] @@ -1106,7 +1190,7 @@ class HttpCli(object): if fn in vfs_virt: fspath = vfs_virt[fn].realpath - elif fn in hidden: + elif hidden and any(fn.startswith(x) for x in hidden): continue else: fspath = fsroot + "/" + fn @@ -1193,12 +1277,19 @@ class HttpCli(object): # ts = "?{}".format(time.time()) dirs.extend(files) + + if "ls" in self.uparam: + ret = json.dumps(dirs) + self.reply(ret.encode("utf-8", "replace")) + return True + html = self.conn.tpl_browser.render( vdir=quotep(self.vpath), vpnodes=vpnodes, files=dirs, can_upload=self.writable, can_read=self.readable, + have_up2k_idx=self.args.e2d, ts=ts, prologue=logues[0], epilogue=logues[1], diff --git a/copyparty/httpconn.py b/copyparty/httpconn.py index 1cb321cb..8c6ac30f 100644 --- a/copyparty/httpconn.py +++ b/copyparty/httpconn.py @@ -30,6 +30,7 @@ except ImportError: from .__init__ import E from .util import Unrecv from .httpcli import HttpCli +from .u2idx import U2idx class HttpConn(object): @@ -50,6 +51,7 @@ class HttpConn(object): self.t0 = time.time() self.nbyte = 0 self.workload = 0 + self.u2idx = None self.log_func = hsrv.log self.set_rproxy() @@ -80,6 +82,12 @@ class HttpConn(object): def log(self, msg): self.log_func(self.log_src, msg) + def get_u2idx(self): + if not self.u2idx: + self.u2idx = U2idx(self.args, self.log_func) + + return self.u2idx + def _detect_https(self): method = None if self.cert_path: diff --git a/copyparty/svchub.py b/copyparty/svchub.py index 0a8b73cf..cc17e30a 100644 --- a/copyparty/svchub.py +++ b/copyparty/svchub.py @@ -39,9 +39,13 @@ class SvcHub(object): self.tcpsrv = TcpSrv(self) self.up2k = Up2k(self) - if self.args.e2d and self.args.e2s: + if self.args.e2ds: auth = AuthSrv(self.args, self.log, False) - self.up2k.build_indexes(auth.all_writable) + vols = auth.vfs.all_vols.values() + if not self.args.e2dsa: + vols = [x for x in vols if x.uwrite] + + self.up2k.build_indexes(vols) # decide which worker impl to use if self.check_mp_enable(): @@ -79,7 +83,7 @@ class SvcHub(object): now = time.time() if now >= self.next_day: dt = datetime.utcfromtimestamp(now) - print("\033[36m{}\033[0m".format(dt.strftime("%Y-%m-%d"))) + print("\033[36m{}\033[0m\n".format(dt.strftime("%Y-%m-%d")), end="") # unix timestamp of next 00:00:00 (leap-seconds safe) day_now = dt.day @@ -89,7 +93,7 @@ class SvcHub(object): dt = dt.replace(hour=0, minute=0, second=0) self.next_day = calendar.timegm(dt.utctimetuple()) - fmt = "\033[36m{} \033[33m{:21} \033[0m{}" + fmt = "\033[36m{} \033[33m{:21} \033[0m{}\n" if not VT100: fmt = "{} {:21} {}" if "\033" in msg: @@ -100,12 +104,12 @@ class SvcHub(object): ts = datetime.utcfromtimestamp(now).strftime("%H:%M:%S.%f")[:-3] msg = fmt.format(ts, src, msg) try: - print(msg) + print(msg, end="") except UnicodeEncodeError: try: - print(msg.encode("utf-8", "replace").decode()) + print(msg.encode("utf-8", "replace").decode(), end="") except: - print(msg.encode("ascii", "replace").decode()) + print(msg.encode("ascii", "replace").decode(), end="") def check_mp_support(self): vmin = sys.version_info[1] diff --git a/copyparty/u2idx.py b/copyparty/u2idx.py new file mode 100644 index 00000000..59f2a1dd --- /dev/null +++ b/copyparty/u2idx.py @@ -0,0 +1,146 @@ +# coding: utf-8 +from __future__ import print_function, unicode_literals + +import os +from datetime import datetime + +from .util import u8safe +from .up2k import up2k_wark_from_hashlist + + +try: + HAVE_SQLITE3 = True + import sqlite3 +except: + HAVE_SQLITE3 = False + + +class U2idx(object): + def __init__(self, args, log_func): + self.args = args + self.log_func = log_func + + if not HAVE_SQLITE3: + self.log("could not load sqlite3; searchign wqill be disabled") + return + + self.dbs = {} + + def log(self, msg): + self.log_func("u2idx", msg) + + def fsearch(self, vols, body): + """search by up2k hashlist""" + if not HAVE_SQLITE3: + return [] + + fsize = body["size"] + fhash = body["hash"] + wark = up2k_wark_from_hashlist(self.args.salt, fsize, fhash) + return self.run_query(vols, "select * from up where w = ?", [wark]) + + def search(self, vols, body): + """search by query params""" + if not HAVE_SQLITE3: + return [] + + qobj = {} + _conv_sz(qobj, body, "sz_min", "sz >= ?") + _conv_sz(qobj, body, "sz_max", "sz <= ?") + _conv_dt(qobj, body, "dt_min", "mt >= ?") + _conv_dt(qobj, body, "dt_max", "mt <= ?") + for seg, dk in [["path", "rd"], ["name", "fn"]]: + for inv in ["no", "yes"]: + jk = "{}_{}".format(seg, inv) + if jk in body: + _conv_txt(qobj, body, jk, dk) + + qstr = "select * from up" + qv = [] + if qobj: + qk = [] + for k, v in sorted(qobj.items()): + qk.append(k) + qv.append(v) + + qstr = " and ".join(qk) + qstr = "select * from up where " + qstr + + return self.run_query(vols, qstr, qv) + + def run_query(self, vols, qstr, qv): + qv = tuple(qv) + self.log("qs: " + qstr) + self.log("qv: " + repr(qv)) + + ret = [] + lim = 100 + for (vtop, ptop, flags) in vols: + db = self.dbs.get(ptop) + if not db: + db = _open(ptop) + if not db: + continue + + self.dbs[ptop] = db + self.log("idx /{} @ {} {}".format(vtop, ptop, flags)) + + c = db.execute(qstr, qv) + for _, ts, sz, rd, fn in c: + lim -= 1 + if lim <= 0: + break + + rp = os.path.join(vtop, rd, fn).replace("\\", "/") + ret.append({"ts": int(ts), "sz": sz, "rp": rp}) + + return ret + + +def _open(ptop): + db_path = os.path.join(ptop, ".hist", "up2k.db") + if os.path.exists(db_path): + return sqlite3.connect(db_path) + + +def _conv_sz(q, body, k, sql): + if k in body: + q[sql] = int(float(body[k]) * 1024 * 1024) + + +def _conv_dt(q, body, k, sql): + if k not in body: + return + + v = body[k].upper().rstrip("Z").replace(",", " ").replace("T", " ") + while " " in v: + v = v.replace(" ", " ") + + for fmt in ["%Y-%m-%d %H:%M:%S", "%Y-%m-%d %H:%M", "%Y-%m-%d %H", "%Y-%m-%d"]: + try: + ts = datetime.strptime(v, fmt).timestamp() + break + except: + ts = None + + if ts: + q[sql] = ts + + +def _conv_txt(q, body, k, sql): + v = body[k] + print("[" + v + "]") + + head = "'%'||" + if v.startswith("^"): + head = "" + v = v[1:] + + tail = "||'%'" + if v.endswith("$"): + tail = "" + v = v[:-1] + + inv = "not" if k.endswith("_no") else "" + qk = "{} {} like {}?{}".format(sql, inv, head, tail) + q[qk] = u8safe(v) diff --git a/copyparty/up2k.py b/copyparty/up2k.py index 27053f70..ed9ddc5a 100644 --- a/copyparty/up2k.py +++ b/copyparty/up2k.py @@ -1,7 +1,6 @@ # coding: utf-8 from __future__ import print_function, unicode_literals - import re import os import sys @@ -17,15 +16,23 @@ import threading from copy import deepcopy from .__init__ import WINDOWS -from .util import Pebkac, Queue, fsdec, fsenc, sanitize_fn, ren_open, atomic_move +from .util import ( + Pebkac, + Queue, + ProgressPrinter, + fsdec, + fsenc, + sanitize_fn, + ren_open, + atomic_move, + u8safe, +) -HAVE_SQLITE3 = False try: - import sqlite3 - HAVE_SQLITE3 = True + import sqlite3 except: - pass + HAVE_SQLITE3 = False class Up2k(object): @@ -39,11 +46,11 @@ class Up2k(object): def __init__(self, broker): self.broker = broker self.args = broker.args - self.log = broker.log + self.log_func = broker.log self.persist = self.args.e2d # config - self.salt = "hunter2" # TODO: config + self.salt = broker.args.salt # state self.mutex = threading.Lock() @@ -66,8 +73,16 @@ class Up2k(object): self.r_hash = re.compile("^[0-9a-zA-Z_-]{43}$") if self.persist and not HAVE_SQLITE3: - m = "could not initialize sqlite3, will use in-memory registry only" - self.log("up2k", m) + self.log("could not initialize sqlite3, will use in-memory registry only") + + def log(self, msg): + self.log_func("up2k", msg + "\033[K") + + def _u8(self, rd, fn): + s_rd = u8safe(rd) + s_fn = u8safe(fn) + self.log("u8safe retry:\n [{}] [{}]\n [{}] [{}]".format(rd, fn, s_rd, s_fn)) + return (s_rd, s_fn) def _vis_job_progress(self, job): perc = 100 - (len(job["need"]) * 100.0 / len(job["hash"])) @@ -98,7 +113,7 @@ class Up2k(object): m = "loaded snap {} |{}|".format(path, len(reg.keys())) m = [m] + self._vis_reg_progress(reg) - self.log("up2k", "\n".join(m)) + self.log("\n".join(m)) self.registry[ptop] = reg if not self.persist or not HAVE_SQLITE3: @@ -119,57 +134,86 @@ class Up2k(object): self.db[ptop] = db return db except Exception as ex: - m = "failed to open [{}]: {}".format(ptop, repr(ex)) - self.log("up2k", m) + self.log("cannot use database at [{}]: {}".format(ptop, repr(ex))) return None def build_indexes(self, writeables): tops = [d.realpath for d in writeables] + self.pp = ProgressPrinter() + t0 = time.time() for top in tops: db = self.register_vpath(top) - if db: - # can be symlink so don't `and d.startswith(top)`` - excl = set([d for d in tops if d != top]) - dbw = [db, 0, time.time()] - self._build_dir(dbw, top, excl, top) - self._drop_lost(db, top) - if dbw[1]: - self.log("up2k", "commit {} new files".format(dbw[1])) + if not db: + continue - db.commit() + self.pp.n = next(db.execute("select count(w) from up"))[0] + db_path = os.path.join(top, ".hist", "up2k.db") + sz0 = os.path.getsize(db_path) // 1024 + + # can be symlink so don't `and d.startswith(top)`` + excl = set([d for d in tops if d != top]) + dbw = [db, 0, time.time()] + + n_add = self._build_dir(dbw, top, excl, top) + n_rm = self._drop_lost(db, top) + if dbw[1]: + self.log("commit {} new files".format(dbw[1])) + + db.commit() + if n_add or n_rm: + db_path = os.path.join(top, ".hist", "up2k.db") + sz1 = os.path.getsize(db_path) // 1024 + db.execute("vacuum") + sz2 = os.path.getsize(db_path) // 1024 + msg = "{} new, {} del, {} kB vacced, {} kB gain, {} kB now".format( + n_add, n_rm, sz1 - sz2, sz2 - sz0, sz2 + ) + self.log(msg) + + self.pp.end = True + self.log("{} volumes in {:.2f} sec".format(len(tops), time.time() - t0)) def _build_dir(self, dbw, top, excl, cdir): try: inodes = [fsdec(x) for x in os.listdir(fsenc(cdir))] except Exception as ex: - self.log("up2k", "listdir: {} @ [{}]".format(repr(ex), cdir)) - return + self.log("listdir: {} @ [{}]".format(repr(ex), cdir)) + return 0 + self.pp.msg = "a{} {}".format(self.pp.n, cdir) histdir = os.path.join(top, ".hist") + ret = 0 for inode in inodes: abspath = os.path.join(cdir, inode) try: inf = os.stat(fsenc(abspath)) except Exception as ex: - self.log("up2k", "stat: {} @ [{}]".format(repr(ex), abspath)) + self.log("stat: {} @ [{}]".format(repr(ex), abspath)) continue if stat.S_ISDIR(inf.st_mode): if abspath in excl or abspath == histdir: continue - # self.log("up2k", " dir: {}".format(abspath)) - self._build_dir(dbw, top, excl, abspath) + # self.log(" dir: {}".format(abspath)) + ret += self._build_dir(dbw, top, excl, abspath) else: - # self.log("up2k", "file: {}".format(abspath)) + # self.log("file: {}".format(abspath)) rp = abspath[len(top) :].replace("\\", "/").strip("/") - c = dbw[0].execute("select * from up where rp = ?", (rp,)) + rd, fn = rp.rsplit("/", 1) if "/" in rp else ["", rp] + sql = "select * from up where rd = ? and fn = ?" + try: + c = dbw[0].execute(sql, (rd, fn)) + except: + c = dbw[0].execute(sql, self._u8(rd, fn)) + in_db = list(c.fetchall()) if in_db: - _, dts, dsz, _ = in_db[0] + self.pp.n -= 1 + _, dts, dsz, _, _ = in_db[0] if len(in_db) > 1: m = "WARN: multiple entries: [{}] => [{}] ({})" - self.log("up2k", m.format(top, rp, len(in_db))) + self.log(m.format(top, rp, len(in_db))) dts = -1 if dts == inf.st_mtime and dsz == inf.st_size: @@ -178,68 +222,80 @@ class Up2k(object): m = "reindex [{}] => [{}] ({}/{}) ({}/{})".format( top, rp, dts, inf.st_mtime, dsz, inf.st_size ) - self.log("up2k", m) - self.db_rm(dbw[0], rp) + self.log(m) + self.db_rm(dbw[0], rd, fn) + ret += 1 dbw[1] += 1 in_db = None - self.log("up2k", "file: {}".format(abspath)) + self.pp.msg = "a{} {}".format(self.pp.n, abspath) + if inf.st_size > 1024 * 1024: + self.log("file: {}".format(abspath)) + try: hashes = self._hashlist_from_file(abspath) except Exception as ex: - self.log("up2k", "hash: {} @ [{}]".format(repr(ex), abspath)) + self.log("hash: {} @ [{}]".format(repr(ex), abspath)) continue - wark = self._wark_from_hashlist(inf.st_size, hashes) - self.db_add(dbw[0], wark, rp, inf.st_mtime, inf.st_size) + wark = up2k_wark_from_hashlist(self.salt, inf.st_size, hashes) + self.db_add(dbw[0], wark, rd, fn, inf.st_mtime, inf.st_size) dbw[1] += 1 + ret += 1 td = time.time() - dbw[2] - if dbw[1] > 1024 or td > 60: - self.log("up2k", "commit {} new files".format(dbw[1])) + if dbw[1] >= 4096 or td >= 60: + self.log("commit {} new files".format(dbw[1])) dbw[0].commit() dbw[1] = 0 dbw[2] = time.time() + return ret def _drop_lost(self, db, top): rm = [] + nchecked = 0 + nfiles = next(db.execute("select count(w) from up"))[0] c = db.execute("select * from up") - for dwark, dts, dsz, drp in c: - abspath = os.path.join(top, drp) + for dwark, dts, dsz, drd, dfn in c: + nchecked += 1 + abspath = os.path.join(top, drd, dfn) + # almost zero overhead dw + self.pp.msg = "b{} {}".format(nfiles - nchecked, abspath) try: if not os.path.exists(fsenc(abspath)): - rm.append(drp) + rm.append([drd, dfn]) except Exception as ex: - self.log("up2k", "stat-rm: {} @ [{}]".format(repr(ex), abspath)) + self.log("stat-rm: {} @ [{}]".format(repr(ex), abspath)) - if not rm: - return + if rm: + self.log("forgetting {} deleted files".format(len(rm))) + for rd, fn in rm: + self.db_rm(db, rd, fn) - self.log("up2k", "forgetting {} deleted files".format(len(rm))) - for rp in rm: - self.db_rm(db, rp) + return len(rm) def _open_db(self, db_path): + existed = os.path.exists(db_path) conn = sqlite3.connect(db_path, check_same_thread=False) try: - c = conn.execute(r"select * from kv where k = 'sver'") - rows = c.fetchall() - if rows: - ver = rows[0][1] - else: - self.log("up2k", "WARN: no sver in kv, DB corrupt?") - ver = "unknown" + ver = self._read_ver(conn) - if ver == "1": + if ver == 1: + conn = self._upgrade_v1(conn, db_path) + ver = self._read_ver(conn) + + if ver == 2: try: nfiles = next(conn.execute("select count(w) from up"))[0] - self.log("up2k", "found DB at {} |{}|".format(db_path, nfiles)) + self.log("found DB at {} |{}|".format(db_path, nfiles)) return conn except Exception as ex: - m = "WARN: could not list files, DB corrupt?\n " + repr(ex) - self.log("up2k", m) + self.log("WARN: could not list files, DB corrupt?\n " + repr(ex)) + + if ver is not None: + self.log("REPLACING unsupported DB (v.{}) at {}".format(ver, db_path)) + elif not existed: + raise Exception("whatever") - m = "REPLACING unsupported DB (v.{}) at {}".format(ver, db_path) - self.log("up2k", m) conn.close() os.unlink(db_path) conn = sqlite3.connect(db_path, check_same_thread=False) @@ -247,17 +303,58 @@ class Up2k(object): pass # sqlite is variable-width only, no point in using char/nchar/varchar + self._create_v2(conn) + conn.commit() + self.log("created DB at {}".format(db_path)) + return conn + + def _read_ver(self, conn): + for tab in ["ki", "kv"]: + try: + c = conn.execute(r"select v from {} where k = 'sver'".format(tab)) + except: + continue + + rows = c.fetchall() + if rows: + return int(rows[0][0]) + + def _create_v2(self, conn): for cmd in [ - r"create table kv (k text, v text)", - r"create table up (w text, mt int, sz int, rp text)", - r"insert into kv values ('sver', '1')", + r"create table ks (k text, v text)", + r"create table ki (k text, v int)", + r"create table up (w text, mt int, sz int, rd text, fn text)", + r"insert into ki values ('sver', 2)", r"create index up_w on up(w)", + r"create index up_rd on up(rd)", + r"create index up_fn on up(fn)", ]: conn.execute(cmd) - conn.commit() - self.log("up2k", "created DB at {}".format(db_path)) - return conn + def _upgrade_v1(self, odb, db_path): + self.log("\033[33mupgrading v1 to v2:\033[0m {}".format(db_path)) + + npath = db_path + ".next" + if os.path.exists(npath): + os.unlink(npath) + + ndb = sqlite3.connect(npath, check_same_thread=False) + self._create_v2(ndb) + + c = odb.execute("select * from up") + for wark, ts, sz, rp in c: + rd, fn = rp.rsplit("/", 1) if "/" in rp else ["", rp] + v = (wark, ts, sz, rd, fn) + ndb.execute("insert into up values (?,?,?,?,?)", v) + + ndb.commit() + ndb.close() + odb.close() + bpath = db_path + ".bak.v1" + self.log("success; backup at: " + bpath) + atomic_move(db_path, bpath) + atomic_move(npath, db_path) + return sqlite3.connect(db_path, check_same_thread=False) def handle_json(self, cj): self.register_vpath(cj["ptop"]) @@ -271,19 +368,13 @@ class Up2k(object): reg = self.registry[cj["ptop"]] if db: cur = db.execute(r"select * from up where w = ?", (wark,)) - for _, dtime, dsize, dp_rel in cur: - dp_abs = os.path.join(cj["ptop"], dp_rel).replace("\\", "/") + for _, dtime, dsize, dp_dir, dp_fn in cur: + dp_abs = os.path.join(cj["ptop"], dp_dir, dp_fn).replace("\\", "/") # relying on path.exists to return false on broken symlinks if os.path.exists(fsenc(dp_abs)): - try: - prel, name = dp_rel.rsplit("/", 1) - except: - prel = "" - name = dp_rel - job = { - "name": name, - "prel": prel, + "name": dp_fn, + "prel": dp_dir, "vtop": cj["vtop"], "ptop": cj["ptop"], "flag": cj["flag"], @@ -319,12 +410,12 @@ class Up2k(object): vsrc = os.path.join(job["vtop"], job["prel"], job["name"]) vsrc = vsrc.replace("\\", "/") # just for prints anyways if job["need"]: - self.log("up2k", "unfinished:\n {0}\n {1}".format(src, dst)) + self.log("unfinished:\n {0}\n {1}".format(src, dst)) err = "partial upload exists at a different location; please resume uploading here instead:\n" err += vsrc + " " raise Pebkac(400, err) elif "nodupe" in job["flag"]: - self.log("up2k", "dupe-reject:\n {0}\n {1}".format(src, dst)) + self.log("dupe-reject:\n {0}\n {1}".format(src, dst)) err = "upload rejected, file already exists:\n " + vsrc + " " raise Pebkac(400, err) else: @@ -389,7 +480,7 @@ class Up2k(object): def _symlink(self, src, dst): # TODO store this in linktab so we never delete src if there are links to it - self.log("up2k", "linking dupe:\n {0}\n {1}".format(src, dst)) + self.log("linking dupe:\n {0}\n {1}".format(src, dst)) try: lsrc = src ldst = dst @@ -412,7 +503,7 @@ class Up2k(object): lsrc = "../" * (len(lsrc) - 1) + "/".join(lsrc) os.symlink(fsenc(lsrc), fsenc(ldst)) except (AttributeError, OSError) as ex: - self.log("up2k", "cannot symlink; creating copy: " + repr(ex)) + self.log("cannot symlink; creating copy: " + repr(ex)) shutil.copy2(fsenc(src), fsenc(dst)) def handle_chunk(self, ptop, wark, chash): @@ -430,7 +521,7 @@ class Up2k(object): job["poke"] = time.time() - chunksize = self._get_chunksize(job["size"]) + chunksize = up2k_chunksize(job["size"]) ofs = [chunksize * x for x in nchunk] path = os.path.join(job["ptop"], job["prel"], job["tnam"]) @@ -463,33 +554,31 @@ class Up2k(object): db = self.db.get(job["ptop"], None) if db: - rp = os.path.join(job["prel"], job["name"]).replace("\\", "/") - self.db_rm(db, rp) - self.db_add(db, job["wark"], rp, job["lmod"], job["size"]) + j = job + self.db_rm(db, j["prel"], j["name"]) + self.db_add(db, j["wark"], j["prel"], j["name"], j["lmod"], j["size"]) db.commit() del self.registry[ptop][wark] # in-memory registry is reserved for unfinished uploads return ret, dst - def _get_chunksize(self, filesize): - chunksize = 1024 * 1024 - stepsize = 512 * 1024 - while True: - for mul in [1, 2]: - nchunks = math.ceil(filesize * 1.0 / chunksize) - if nchunks <= 256 or chunksize >= 32 * 1024 * 1024: - return chunksize + def db_rm(self, db, rd, fn): + sql = "delete from up where rd = ? and fn = ?" + try: + db.execute(sql, (rd, fn)) + except: + db.execute(sql, self._u8(rd, fn)) - chunksize += stepsize - stepsize *= mul - - def db_rm(self, db, rp): - db.execute("delete from up where rp = ?", (rp,)) - - def db_add(self, db, wark, rp, ts, sz): - v = (wark, ts, sz, rp) - db.execute("insert into up values (?,?,?,?)", v) + def db_add(self, db, wark, rd, fn, ts, sz): + sql = "insert into up values (?,?,?,?,?)" + v = (wark, ts, sz, rd, fn) + try: + db.execute(sql, v) + except: + rd, fn = self._u8(rd, fn) + v = (wark, ts, sz, rd, fn) + db.execute(sql, v) def _get_wark(self, cj): if len(cj["name"]) > 1024 or len(cj["hash"]) > 512 * 1024: # 16TiB @@ -507,36 +596,17 @@ class Up2k(object): except: cj["lmod"] = int(time.time()) - wark = self._wark_from_hashlist(cj["size"], cj["hash"]) + wark = up2k_wark_from_hashlist(self.salt, cj["size"], cj["hash"]) return wark - def _wark_from_hashlist(self, filesize, hashes): - """ server-reproducible file identifier, independent of name or location """ - ident = [self.salt, str(filesize)] - ident.extend(hashes) - ident = "\n".join(ident) - - hasher = hashlib.sha512() - hasher.update(ident.encode("utf-8")) - digest = hasher.digest()[:32] - - wark = base64.urlsafe_b64encode(digest) - return wark.decode("utf-8").rstrip("=") - def _hashlist_from_file(self, path): fsz = os.path.getsize(path) - csz = self._get_chunksize(fsz) + csz = up2k_chunksize(fsz) ret = [] last_print = time.time() with open(path, "rb", 512 * 1024) as f: while fsz > 0: - now = time.time() - td = now - last_print - if td >= 0.1: - last_print = now - msg = " {} MB \r".format(int(fsz / 1024 / 1024)) - print(msg, end="", file=sys.stderr) - + self.pp.msg = msg = "{} MB".format(int(fsz / 1024 / 1024)) hashobj = hashlib.sha512() rem = min(csz, fsz) fsz -= rem @@ -599,7 +669,7 @@ class Up2k(object): if rm: m = "dropping {} abandoned uploads in {}".format(len(rm), k) vis = [self._vis_job_progress(x) for x in rm] - self.log("up2k", "\n".join([m] + vis)) + self.log("\n".join([m] + vis)) for job in rm: del reg[job["wark"]] try: @@ -635,5 +705,32 @@ class Up2k(object): atomic_move(path2, path) - self.log("up2k", "snap: {} |{}|".format(path, len(reg.keys()))) + self.log("snap: {} |{}|".format(path, len(reg.keys()))) prev[k] = etag + + +def up2k_chunksize(filesize): + chunksize = 1024 * 1024 + stepsize = 512 * 1024 + while True: + for mul in [1, 2]: + nchunks = math.ceil(filesize * 1.0 / chunksize) + if nchunks <= 256 or chunksize >= 32 * 1024 * 1024: + return chunksize + + chunksize += stepsize + stepsize *= mul + + +def up2k_wark_from_hashlist(salt, filesize, hashes): + """ server-reproducible file identifier, independent of name or location """ + ident = [salt, str(filesize)] + ident.extend(hashes) + ident = "\n".join(ident) + + hasher = hashlib.sha512() + hasher.update(ident.encode("utf-8")) + digest = hasher.digest()[:32] + + wark = base64.urlsafe_b64encode(digest) + return wark.decode("utf-8").rstrip("=") diff --git a/copyparty/util.py b/copyparty/util.py index b5b37090..038990ec 100644 --- a/copyparty/util.py +++ b/copyparty/util.py @@ -99,6 +99,32 @@ class Unrecv(object): self.buf = buf + self.buf +class ProgressPrinter(threading.Thread): + """ + periodically print progress info without linefeeds + """ + + def __init__(self): + threading.Thread.__init__(self) + self.daemon = True + self.msg = None + self.end = False + self.start() + + def run(self): + msg = None + while not self.end: + time.sleep(0.05) + if msg == self.msg or self.end: + continue + + msg = self.msg + print(" {}\033[K\r".format(msg), end="") + + print("\033[K", end="") + sys.stdout.flush() # necessary on win10 even w/ stderr btw + + @contextlib.contextmanager def ren_open(fname, *args, **kwargs): fdir = kwargs.pop("fdir", None) @@ -146,7 +172,7 @@ def ren_open(fname, *args, **kwargs): except OSError as ex_: ex = ex_ - if ex.errno != 36: + if ex.errno not in [36, 63] and (not WINDOWS or ex.errno != 22): raise if not b64: @@ -480,6 +506,13 @@ def sanitize_fn(fn): return fn.strip() +def u8safe(txt): + try: + return txt.encode("utf-8", "xmlcharrefreplace").decode("utf-8", "replace") + except: + return txt.encode("utf-8", "replace").decode("utf-8", "replace") + + def exclude_dotfiles(filepaths): for fpath in filepaths: if not fpath.split("/")[-1].startswith("."): diff --git a/copyparty/web/browser.css b/copyparty/web/browser.css index bdc8ca57..baedfa50 100644 --- a/copyparty/web/browser.css +++ b/copyparty/web/browser.css @@ -39,15 +39,27 @@ body { margin: 1.3em 0 0 0; font-size: 1.4em; } +#path #entree { + margin-left: -.7em; +} +#treetab { + display: none; +} #files { border-collapse: collapse; margin-top: 2em; + z-index: 1; + position: relative; } #files tbody a { display: block; padding: .3em 0; } -a { +#files[ts] tbody div a { + color: #f5a; +} +a, +#files[ts] tbody div a:last-child { color: #fc5; padding: .2em; text-decoration: none; @@ -156,7 +168,7 @@ a.play.act { height: 100%; background: #333; font-size: 2.5em; - z-index:99; + z-index: 99; } #blk_play, #blk_abrt { @@ -190,6 +202,7 @@ a.play.act { bottom: -6em; height: 6em; width: 100%; + z-index: 3; transition: bottom 0.15s; } #widget.open { @@ -214,6 +227,9 @@ a.play.act { 75% {cursor: url(/.cpr/dd/5.png), pointer} 85% {cursor: url(/.cpr/dd/1.png), pointer} } +@keyframes spin { + 100% {transform: rotate(360deg)} +} #wtoggle { position: absolute; top: -1.2em; @@ -273,3 +289,207 @@ a.play.act { width: calc(100% - 10.5em); background: rgba(0,0,0,0.2); } + + + +.opview { + display: none; +} +.opview.act { + display: block; +} +#ops a { + color: #fc5; + font-size: 1.5em; + padding: .25em .3em; + margin: 0; + outline: none; +} +#ops a.act { + background: #281838; + border-radius: 0 0 .2em .2em; + border-bottom: .3em solid #d90; + box-shadow: 0 -.15em .2em #000 inset; + padding-bottom: .3em; +} +#ops i { + font-size: 1.5em; +} +#ops i:before { + content: 'x'; + color: #282828; + text-shadow: 0 0 .08em #01a7e1; + position: relative; +} +#ops i:after { + content: 'x'; + color: #282828; + text-shadow: 0 0 .08em #ff3f1a; + margin-left: -.35em; + font-size: 1.05em; +} +#ops, +.opbox { + border: 1px solid #3a3a3a; + box-shadow: 0 0 1em #222 inset; +} +#ops { + display: none; + background: #333; + margin: 1.7em 1.5em 0 1.5em; + padding: .3em .6em; + border-radius: .3em; + border-width: .15em 0; +} +.opbox { + background: #2d2d2d; + margin: 1.5em 0 0 0; + padding: .5em; + border-radius: 0 1em 1em 0; + border-width: .15em .3em .3em 0; + max-width: 40em; +} +.opbox input { + margin: .5em; +} +.opview input[type=text] { + color: #fff; + background: #383838; + border: none; + box-shadow: 0 0 .3em #222; + border-bottom: 1px solid #fc5; + border-radius: .2em; + padding: .2em .3em; +} +input[type="checkbox"]+label { + color: #f5a; +} +input[type="checkbox"]:checked+label { + color: #fc5; +} + + + +#op_search table { + border: 1px solid #3a3a3a; + box-shadow: 0 0 1em #222 inset; + background: #2d2d2d; + border-radius: .4em; + margin: 1.4em; + margin-bottom: 0; + padding: 0 .5em .5em 0; +} +#srch_form td { + padding: .6em .6em; +} +#op_search input { + margin: 0; +} +#srch_q { + white-space: pre; +} +#files td div span { + color: #fff; + padding: 0 .4em; + font-weight: bold; + font-style: italic; +} +#files td div a:hover { + background: #444; + color: #fff; +} +#files td div a { + display: table-cell; + white-space: nowrap; +} +#files td div a:last-child { + width: 100%; +} +#files td div { + display: table; + border-collapse: collapse; + width: 100%; +} +#files td div a:last-child { + width: 100%; +} +#tree, +#treefiles { + vertical-align: top; +} +#tree { + padding-top: 2em; +} +#detree { + padding: .3em .5em; + font-size: 1.5em; +} +#treefiles #files tbody { + border-radius: 0 .7em 0 .7em; +} +#treefiles #files thead th:nth-child(1) { + border-radius: .7em 0 0 0; +} +#tree li { + list-style: none; +} +#tree ul, +#tree li { + padding: 0; + margin: 0; +} +#tree ul { + border-left: .2em solid #444; +} +#tree li { + margin-left: 1em; +} +#tree a.hl { + color: #400; + background: #fc4; + border-radius: .3em; + text-shadow: none; +} +#tree li { + white-space: nowrap; +} +#tree a { + display: inline-block; +} +#tree a+a { + width: calc(100% - 2em); + background: #333; +} +#tree a+a:hover { + background: #222; + color: #fff; +} +#treeul { + position: relative; + overflow: hidden; + left: -1.7em; +} +#treeul:hover { + z-index: 2; + overflow: visible; +} +#treeul:hover a+a { + width: auto; + min-width: calc(100% - 2em); +} +#treeul a:first-child { + font-family: monospace, monospace; +} +#treefiles { + opacity: 1; + transition: opacity 0.2s ease-in-out; +} +#tree:hover+#treefiles { + opacity: .8; +} +.dumb_loader_thing { + display: inline-block; + margin: 1em; + font-size: 3em; + animation: spin 1s linear infinite; +} diff --git a/copyparty/web/browser.html b/copyparty/web/browser.html index 9d326ecc..ff995edb 100644 --- a/copyparty/web/browser.html +++ b/copyparty/web/browser.html @@ -13,11 +13,33 @@ +
+ --- + {%- if can_read %} + 🔎 + {%- endif %} + {%- if can_upload %} + 🚀 + 🎈 + 📂 + 📝 + 📟 + {%- endif %} +
+ + {%- if can_read %} + + {%- endif %} + {%- if can_upload %} {%- include 'upload.html' %} {%- endif %}

+ 🌲 {%- for n in vpnodes %} {{ n[1] }} {%- endfor %} @@ -28,6 +50,18 @@
{{ prologue }}
{%- endif %} + + + + + +
+ 🍞... +
    +
    🌲
    +
+
+ diff --git a/copyparty/web/browser.js b/copyparty/web/browser.js index e6373704..03e45953 100644 --- a/copyparty/web/browser.js +++ b/copyparty/web/browser.js @@ -8,6 +8,8 @@ function dbg(msg) { function ev(e) { e = e || window.event; + if (!e) + return; if (e.preventDefault) e.preventDefault() @@ -23,7 +25,7 @@ makeSortable(ebi('files')); // extract songs + add play column -var mp = (function () { +function init_mp() { var tracks = []; var ret = { 'au': null, @@ -37,7 +39,8 @@ var mp = (function () { var trs = ebi('files').getElementsByTagName('tbody')[0].getElementsByTagName('tr'); for (var a = 0, aa = trs.length; a < aa; a++) { var tds = trs[a].getElementsByTagName('td'); - var link = tds[1].getElementsByTagName('a')[0]; + var link = tds[1].getElementsByTagName('a'); + link = link[link.length - 1]; var url = link.getAttribute('href'); var m = re_audio.exec(url); @@ -71,7 +74,8 @@ var mp = (function () { }; return ret; -})(); +} +var mp = init_mp(); // toggle player widget @@ -466,7 +470,13 @@ function play(tid, call_depth) { var o = ebi(oid); o.setAttribute('id', 'thx_js'); - location.hash = oid; + if (window.history && history.replaceState) { + var nurl = (document.location + '').split('#')[0] + '#' + oid; + history.replaceState(ebi('files').tBodies[0].innerHTML, nurl, nurl); + } + else { + document.location.hash = oid; + } o.setAttribute('id', oid); pbar.drawbuf(); @@ -561,3 +571,384 @@ function autoplay_blocked() { //widget.open(); + + +// search +(function () { + var sconf = [ + ["size", + ["szl", "sz_min", "minimum MiB", ""], + ["szu", "sz_max", "maximum MiB", ""] + ], + ["date", + ["dtl", "dt_min", "min. iso8601", ""], + ["dtu", "dt_max", "max. iso8601", ""] + ], + ["path", + ["pn", "path_no", "path NOT contains", "30"], + ["py", "path_yes", "path contains", "30"] + ], + ["name", + ["nn", "name_no", "name NOT contains", "30"], + ["ny", "name_yes", "name contains", "30"] + ] + ]; + var html = []; + for (var a = 0; a < sconf.length; a++) { + html.push(''); + for (var b = 1; b < 3; b++) { + var hn = "srch_" + sconf[a][b][0]; + html.push( + ''); + } + html.push(''); + } + ebi('srch_form').innerHTML = html.join('\n'); + + var o = document.querySelectorAll('#op_search input[type="text"]'); + for (var a = 0; a < o.length; a++) { + o[a].oninput = ev_search_input; + } + + var search_timeout; + + function ev_search_input() { + var v = this.value; + var chk = ebi(this.getAttribute('id').slice(0, -1) + 'c'); + chk.checked = ((v + '').length > 0); + clearTimeout(search_timeout); + search_timeout = setTimeout(do_search, 100); + } + + function do_search() { + clearTimeout(search_timeout); + var params = {}; + var o = document.querySelectorAll('#op_search input[type="text"]'); + for (var a = 0; a < o.length; a++) { + var chk = ebi(o[a].getAttribute('id').slice(0, -1) + 'c'); + if (!chk.checked) + continue; + + params[o[a].getAttribute('name')] = o[a].value; + } + // ebi('srch_q').textContent = JSON.stringify(params, null, 4); + var xhr = new XMLHttpRequest(); + xhr.open('POST', '/?srch', true); + xhr.onreadystatechange = xhr_search_results; + xhr.ts = new Date().getTime(); + xhr.send(JSON.stringify(params)); + } + + function xhr_search_results() { + if (this.readyState != XMLHttpRequest.DONE) + return; + + if (this.status !== 200) { + alert('ah fug\n' + this.status + ": " + this.responseText); + return; + } + + var ofiles = ebi('files'); + if (ofiles.getAttribute('ts') > this.ts) + return; + + ebi('path').style.display = 'none'; + ebi('tree').style.display = 'none'; + + var html = []; + var res = JSON.parse(this.responseText); + for (var a = 0; a < res.length; a++) { + var r = res[a], + ts = parseInt(r.ts), + sz = esc(r.sz + ''), + rp = esc(r.rp + ''), + ext = rp.lastIndexOf('.') > 0 ? rp.split('.').slice(-1)[0] : '%', + links = linksplit(rp); + + ts = new Date(ts * 1000).toISOString().replace("T", " ").slice(0, -5); + + if (ext.length > 8) + ext = '%'; + + links = links.join(''); + html.push(''); + } + + ofiles.tBodies[0].innerHTML = html.join('\n'); + ofiles.setAttribute("ts", this.ts); + reload_browser(); + } +})(); + + +// tree +(function () { + var treedata = null; + + function entree(e) { + ev(e); + ebi('path').style.display = 'none'; + + var treetab = ebi('treetab'); + var treefiles = ebi('treefiles'); + + treetab.style.display = 'table'; + + var pro = ebi('pro'); + if (pro) + treefiles.appendChild(pro); + + treefiles.appendChild(ebi('files')); + + var epi = ebi('epi'); + if (epi) + treefiles.appendChild(epi); + + localStorage.setItem('entreed', 'tree'); + get_tree("", get_vpath()); + } + + function get_tree(top, dst) { + var xhr = new XMLHttpRequest(); + xhr.top = top; + xhr.dst = dst; + xhr.open('GET', dst + '?tree=' + top, true); + xhr.onreadystatechange = recvtree; + xhr.send(); + } + + function recvtree() { + if (this.readyState != XMLHttpRequest.DONE) + return; + + if (this.status !== 200) { + alert('ah fug\n' + this.status + ": " + this.responseText); + return; + } + + var top = this.top == '.' ? this.dst : this.top, + name = top.split('/').slice(-2)[0], + rtop = top.replace(/^\/+/, ""); + + try { + var res = JSON.parse(this.responseText); + } + catch (ex) { + return; + } + var html = parsetree(res, rtop); + if (!this.top) { + html = '
  • -[root]\n
    • '; + } + else { + html = '-' + esc(name) + "" + html; + + var links = document.querySelectorAll('#tree a+a'); + for (var a = 0, aa = links.length; a < aa; a++) { + if (links[a].getAttribute('href') == top) { + var o = links[a].parentNode; + if (!o.getElementsByTagName('li').length) + o.innerHTML = html; + //else + // links[a].previousSibling.textContent = '-'; + } + } + } + document.querySelector('#treeul>li>a+a').textContent = '[root]'; + reload_tree(); + + var q = '#tree'; + var nq = 0; + while (true) { + nq++; + q += '>ul>li'; + if (!document.querySelector(q)) + break; + } + ebi('treeul').style.width = (24 + nq) + 'em'; + } + + function reload_tree() { + var cdir = get_vpath(); + var links = document.querySelectorAll('#tree a+a'); + for (var a = 0, aa = links.length; a < aa; a++) { + var href = links[a].getAttribute('href'); + links[a].setAttribute('class', href == cdir ? 'hl' : ''); + links[a].onclick = treego; + } + links = document.querySelectorAll('#tree li>a:first-child'); + for (var a = 0, aa = links.length; a < aa; a++) { + links[a].setAttribute('dst', links[a].nextSibling.getAttribute('href')); + links[a].onclick = treegrow; + } + } + + function treego(e) { + ev(e); + if (this.getAttribute('class') == 'hl') { + treegrow.call(this.previousSibling, e); + return; + } + var xhr = new XMLHttpRequest(); + xhr.top = this.getAttribute('href'); + xhr.open('GET', xhr.top + '?ls', true); + xhr.onreadystatechange = recvls; + xhr.send(); + get_tree('.', xhr.top); + } + + function treegrow(e) { + ev(e); + if (this.textContent == '-') { + while (this.nextSibling.nextSibling) { + var rm = this.nextSibling.nextSibling; + rm.parentNode.removeChild(rm); + this.textContent = '+'; + } + return; + } + var dst = this.getAttribute('dst'); + get_tree('.', dst); + } + + function recvls() { + if (this.readyState != XMLHttpRequest.DONE) + return; + + if (this.status !== 200) { + alert('ah fug\n' + this.status + ": " + this.responseText); + return; + } + + try { + var res = JSON.parse(this.responseText); + } + catch (ex) { + window.location = this.top; + return; + } + var top = this.top; + var html = []; + for (var a = 0; a < res.length; a++) { + var ln = ''; + + for (var b = 3; b < res[a].length; b++) { + ln += ''; + } + html.push(ln + '') + } + html = html.join('\n'); + ebi('files').tBodies[0].innerHTML = html; + history.pushState(html, this.top, this.top); + + var o = ebi('pro'); + if (o) o.parentNode.removeChild(o); + + o = ebi('epi'); + if (o) o.parentNode.removeChild(o); + + reload_tree(); + reload_browser(); + } + + function parsetree(res, top) { + var ret = ''; + for (var a = 0; a < res.a.length; a++) { + res['k' + res.a[a]] = 0; + } + delete res['a']; + var keys = Object.keys(res); + keys.sort(); + for (var a = 0; a < keys.length; a++) { + var kk = keys[a], + k = kk.slice(1), + url = '/' + (top ? top + k : k) + '/', + ek = esc(k), + sym = res[kk] ? '-' : '+', + link = '' + sym + '' + ek + ''; + + if (res[kk]) { + var subtree = parsetree(res[kk], url.slice(1)); + ret += '
  • ' + link + '\n
    • \n'; + } + else { + ret += '
  • ' + link + '
    • \n'; + } + } + return ret; + } + + function detree(e) { + ev(e); + var treetab = ebi('treetab'); + + var pro = ebi('pro'); + if (pro) + treetab.parentNode.insertBefore(pro, treetab); + + treetab.parentNode.insertBefore(ebi('files'), treetab.nextSibling); + + var epi = ebi('epi'); + if (epi) + treetab.parentNode.insertBefore(epi, ebi('files').nextSibling); + + ebi('path').style.display = 'inline-block'; + treetab.style.display = 'none'; + + localStorage.setItem('entreed', 'na'); + } + + ebi('entree').onclick = entree; + ebi('detree').onclick = detree; + if (window.localStorage && localStorage.getItem('entreed') == 'tree') + entree(); + + window.onpopstate = function (e) { + console.log(e.url + ' ,, ' + ((e.state + '').slice(0, 64))); + if (e.state) { + ebi('files').tBodies[0].innerHTML = e.state; + reload_tree(); + reload_browser(); + } + }; + + if (window.history && history.pushState) { + var u = get_vpath(); + history.replaceState(ebi('files').tBodies[0].innerHTML, u, u); + } +})(); + + +function reload_browser() { + makeSortable(ebi('files')); + + var parts = get_vpath().split('/'); + var rm = document.querySelectorAll('#path>a+a+a'); + for (a = rm.length - 1; a >= 0; a--) + rm[a].parentNode.removeChild(rm[a]); + + var link = '/'; + for (var a = 1; a < parts.length - 1; a++) { + link += parts[a] + '/'; + var o = document.createElement('a'); + o.setAttribute('href', link); + o.innerHTML = parts[a]; + ebi('path').appendChild(o); + } + + if (mp && mp.au) { + mp.au.pause(); + mp.au = null; + } + widget.close(); + mp = init_mp(); +} diff --git a/copyparty/web/md2.css b/copyparty/web/md2.css index 655cf040..9e55f709 100644 --- a/copyparty/web/md2.css +++ b/copyparty/web/md2.css @@ -124,5 +124,3 @@ html.dark #toast { transition: opacity 0.2s ease-in-out; opacity: 1; } - -# mt {opacity: .5;top:1px} diff --git a/copyparty/web/up2k.js b/copyparty/web/up2k.js index 058b1bc2..37da44cd 100644 --- a/copyparty/web/up2k.js +++ b/copyparty/web/up2k.js @@ -3,51 +3,6 @@ window.onerror = vis_exh; -(function () { - var ops = document.querySelectorAll('#ops>a'); - for (var a = 0; a < ops.length; a++) { - ops[a].onclick = opclick; - } -})(); - - -function opclick(ev) { - if (ev) //ie - ev.preventDefault(); - - var dest = this.getAttribute('data-dest'); - goto(dest); - - // writing a blank value makes ie8 segfault w - if (window.localStorage) - localStorage.setItem('opmode', dest || '.'); - - var input = document.querySelector('.opview.act input:not([type="hidden"])') - if (input) - input.focus(); -} - - -function goto(dest) { - var obj = document.querySelectorAll('.opview.act'); - for (var a = obj.length - 1; a >= 0; a--) - obj[a].classList.remove('act'); - - obj = document.querySelectorAll('#ops>a'); - for (var a = obj.length - 1; a >= 0; a--) - obj[a].classList.remove('act'); - - if (dest) { - ebi('op_' + dest).classList.add('act'); - document.querySelector('#ops>a[data-dest=' + dest + ']').classList.add('act'); - - var fn = window['goto_' + dest]; - if (fn) - fn(); - } -} - - function goto_up2k() { if (up2k === false) return goto('bup'); @@ -59,17 +14,6 @@ function goto_up2k() { } -(function () { - goto(); - if (window.localStorage) { - var op = localStorage.getItem('opmode'); - if (op !== null && op !== '.') - goto(op); - } - ebi('ops').style.display = 'block'; -})(); - - // chrome requires https to use crypto.subtle, // usually it's undefined but some chromes throw on invoke var up2k = null; @@ -255,7 +199,7 @@ function up2k_init(have_crypto) { // handle user intent to use the basic uploader instead ebi('u2nope').onclick = function (e) { e.preventDefault(); - setmsg(''); + setmsg(); goto('bup'); }; @@ -279,13 +223,17 @@ function up2k_init(have_crypto) { } function bcfg_get(name, defval) { + var o = ebi(name); + if (!o) + return defval; + var val = localStorage.getItem(name); if (val === null) val = defval; else val = (val == '1'); - ebi(name).checked = val; + o.checked = val; return val; } @@ -293,7 +241,10 @@ function up2k_init(have_crypto) { localStorage.setItem( name, val ? '1' : '0'); - ebi(name).checked = val; + var o = ebi(name); + if (o) + o.checked = val; + return val; } @@ -301,6 +252,7 @@ function up2k_init(have_crypto) { var multitask = bcfg_get('multitask', true); var ask_up = bcfg_get('ask_up', true); var flag_en = bcfg_get('flag_en', false); + var fsearch = bcfg_get('fsearch', false); var col_hashing = '#00bbff'; var col_hashed = '#004466'; @@ -334,6 +286,7 @@ function up2k_init(have_crypto) { var flag = false; apply_flag_cfg(); + apply_fsearch_cfg(); function nav() { ebi('file' + fdom_ctr).click(); @@ -404,7 +357,7 @@ function up2k_init(have_crypto) { for (var a = 0; a < good_files.length; a++) msg.push(good_files[a].name); - if (ask_up && !confirm(msg.join('\n'))) + if (ask_up && !fsearch && !confirm(msg.join('\n'))) return; for (var a = 0; a < good_files.length; a++) { @@ -795,6 +748,34 @@ function up2k_init(have_crypto) { if (xhr.status == 200) { var response = JSON.parse(xhr.responseText); + if (!response.name) { + var msg = ''; + var smsg = ''; + if (!response || !response.length) { + msg = 'not found on server'; + smsg = '404'; + } + else { + smsg = 'found'; + var hit = response[0], + links = linksplit(hit.rp), + msg = links.join(''), + tr = new Date(hit.ts * 1000).toISOString().replace("T", " ").slice(0, -5), + tu = new Date(t.lmod * 1000).toISOString().replace("T", " ").slice(0, -5), + diff = parseInt(t.lmod) - parseInt(hit.ts), + cdiff = (Math.abs(diff) <= 2) ? '3c0' : 'f0b', + sdiff = 'diff ' + diff; + + msg += '
    ' + tr + ' (srv), ' + tu + ' (You), ' + sdiff + '    '; + } + ebi('f{0}p'.format(t.n)).innerHTML = msg; + ebi('f{0}t'.format(t.n)).innerHTML = smsg; + st.busy.handshake.splice(st.busy.handshake.indexOf(t), 1); + st.bytes.uploaded += t.size; + tasker(); + return; + } + if (response.name !== t.name) { // file exists; server renamed us t.name = response.name; @@ -867,14 +848,19 @@ function up2k_init(have_crypto) { "no further information")); } }; - xhr.open('POST', post_url + 'handshake.php', true); - xhr.responseType = 'text'; - xhr.send(JSON.stringify({ + + var req = { "name": t.name, "size": t.size, "lmod": t.lmod, "hash": t.hash - })); + }; + if (fsearch) + req.srch = 1; + + xhr.open('POST', post_url + 'handshake.php', true); + xhr.responseType = 'text'; + xhr.send(JSON.stringify(req)); } ///// @@ -966,6 +952,46 @@ function up2k_init(have_crypto) { /// config ui // + function onresize(ev) { + var bar = ebi('ops'), + wpx = innerWidth, + fpx = parseInt(getComputedStyle(bar)['font-size']), + wem = wpx * 1.0 / fpx, + wide = wem > 54, + parent = ebi(wide ? 'u2btn_cw' : 'u2btn_ct'), + btn = ebi('u2btn'); + + //console.log([wpx, fpx, wem]); + if (btn.parentNode !== parent) { + parent.appendChild(btn); + ebi('u2conf').setAttribute('class', wide ? 'has_btn' : ''); + } + } + window.onresize = onresize; + onresize(); + + function desc_show(ev) { + var msg = this.getAttribute('alt'); + msg = msg.replace(/\$N/g, "
    "); + var cdesc = ebi('u2cdesc'); + cdesc.innerHTML = msg; + cdesc.setAttribute('class', 'show'); + } + function desc_hide(ev) { + ebi('u2cdesc').setAttribute('class', ''); + } + var o = document.querySelectorAll('#u2conf *[alt]'); + for (var a = o.length - 1; a >= 0; a--) { + o[a].parentNode.getElementsByTagName('input')[0].setAttribute('alt', o[a].getAttribute('alt')); + } + var o = document.querySelectorAll('#u2conf *[alt]'); + for (var a = 0; a < o.length; a++) { + o[a].onfocus = desc_show; + o[a].onblur = desc_hide; + o[a].onmouseenter = desc_show; + o[a].onmouseleave = desc_hide; + } + function bumpthread(dir) { try { dir.stopPropagation(); @@ -1007,6 +1033,21 @@ function up2k_init(have_crypto) { bcfg_set('ask_up', ask_up); } + function tgl_fsearch() { + fsearch = !fsearch; + bcfg_set('fsearch', fsearch); + apply_fsearch_cfg(); + } + + function apply_fsearch_cfg() { + var ks = ['multitask', 'ask_up']; + for (var a = 0; a < ks.length; a++) { + var lbl = document.querySelector('label[for="' + ks[a] + '"]'); + lbl.setAttribute('class', fsearch ? 'gray' : ''); + } + ebi('u2tab').setAttribute('class', fsearch ? 'srch' : ''); + } + function tgl_flag_en() { flag_en = !flag_en; bcfg_set('flag_en', flag_en); @@ -1047,6 +1088,9 @@ function up2k_init(have_crypto) { ebi('multitask').addEventListener('click', tgl_multitask, false); ebi('ask_up').addEventListener('click', tgl_ask_up, false); ebi('flag_en').addEventListener('click', tgl_flag_en, false); + var o = ebi('fsearch'); + if (o) + o.addEventListener('click', tgl_fsearch, false); var nodes = ebi('u2conf').getElementsByTagName('a'); for (var a = nodes.length - 1; a >= 0; a--) diff --git a/copyparty/web/upload.css b/copyparty/web/upload.css index 2355d0ce..426eff31 100644 --- a/copyparty/web/upload.css +++ b/copyparty/web/upload.css @@ -1,92 +1,4 @@ -.opview { - display: none; -} -.opview.act { - display: block; -} -#ops a { - color: #fc5; - font-size: 1.5em; - padding: 0 .3em; - margin: 0; - outline: none; -} -#ops a.act { - text-decoration: underline; -} -/* -#ops a+a:after, -#ops a:first-child:after { - content: 'x'; - color: #282828; - text-shadow: 0 0 .08em #01a7e1; - margin-left: .3em; - position: relative; -} -#ops a+a:before { - content: 'x'; - color: #282828; - text-shadow: 0 0 .08em #ff3f1a; - margin-right: .3em; - margin-left: -.3em; -} -#ops a:last-child:after { - content: ''; -} -#ops a.act:before, -#ops a.act:after { - text-decoration: none !important; -} -*/ -#ops i { - font-size: 1.5em; -} -#ops i:before { - content: 'x'; - color: #282828; - text-shadow: 0 0 .08em #01a7e1; - position: relative; -} -#ops i:after { - content: 'x'; - color: #282828; - text-shadow: 0 0 .08em #ff3f1a; - margin-left: -.35em; - font-size: 1.05em; -} -#ops, -.opbox { - border: 1px solid #3a3a3a; - box-shadow: 0 0 1em #222 inset; -} -#ops { - display: none; - background: #333; - margin: 1.7em 1.5em 0 1.5em; - padding: .3em .6em; - border-radius: .3em; - border-width: .15em 0; -} -.opbox { - background: #2d2d2d; - margin: 1.5em 0 0 0; - padding: .5em; - border-radius: 0 1em 1em 0; - border-width: .15em .3em .3em 0; - max-width: 40em; -} -.opbox input { - margin: .5em; -} -.opbox input[type=text] { - color: #fff; - background: #383838; - border: none; - box-shadow: 0 0 .3em #222; - border-bottom: 1px solid #fc5; - border-radius: .2em; - padding: .2em .3em; -} + #op_up2k { padding: 0 1em 1em 1em; } @@ -117,17 +29,22 @@ background: linear-gradient(to bottom, #367 0%, #489 50%, #38788a 51%, #367 100%); filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#489', endColorstr='#38788a', GradientType=0); text-decoration: none; - line-height: 1.5em; + line-height: 1.3em; border: 1px solid #222; border-radius: .4em; text-align: center; - font-size: 2em; - margin: 1em auto; - padding: 1em 0; - width: 12em; + font-size: 1.5em; + margin: .5em auto; + padding: .6em 0; + width: 16em; cursor: pointer; box-shadow: .4em .4em 0 #111; } +#u2conf #u2btn { + margin: -1em 0; + padding: .7em 0; + width: 100%; +} #u2notbtn { display: none; text-align: center; @@ -142,6 +59,9 @@ width: calc(100% - 2em); max-width: 100em; } +#u2tab.srch { + max-width: none; +} #u2tab td { border: 1px solid #ccc; border-width: 0 0px 1px 0; @@ -153,12 +73,19 @@ #u2tab td:nth-child(3) { width: 40%; } +#u2tab.srch td:nth-child(3) { + font-family: sans-serif; + width: auto; +} #u2tab tr+tr:hover td { background: #222; } #u2conf { margin: 1em auto; - width: 26em; + width: 30em; +} +#u2conf.has_btn { + width: 46em; } #u2conf * { text-align: center; @@ -194,16 +121,70 @@ #u2conf input+a { background: #d80; } +#u2conf label { + font-size: 1.6em; + width: 2em; + height: 1em; + padding: .4em 0; + display: block; + user-select: none; + border-radius: .25em; +} +#u2conf input[type="checkbox"] { + position: relative; + opacity: .02; + top: 2em; +} #u2conf input[type="checkbox"]+label { - color: #f5a; + position: relative; + background: #603; + border-bottom: .2em solid #a16; + box-shadow: 0 .1em .3em #a00 inset; } #u2conf input[type="checkbox"]:checked+label { - color: #fc5; + background: #6a1; + border-bottom: .2em solid #efa; + box-shadow: 0 .1em .5em #0c0; +} +#u2conf input[type="checkbox"]+label:hover { + box-shadow: 0 .1em .3em #fb0; + border-color: #fb0; +} +#u2conf input[type="checkbox"]+label.gray { + background: #777; + border-color: #ccc; + box-shadow: none; + opacity: .25; +} +#u2cdesc { + position: absolute; + width: 34em; + left: calc(50% - 15em); + background: #222; + border: 0 solid #555; + text-align: center; + overflow: hidden; + margin: 0 -2em; + height: 0; + padding: 0 1em; + opacity: .1; + transition: all 0.14s ease-in-out; + border-radius: .4em; + box-shadow: 0 .2em .5em #222; +} +#u2cdesc.show { + padding: 1em; + height: auto; + border-width: .2em 0; + opacity: 1; } #u2foot { color: #fff; font-style: italic; } +#u2footfoot { + margin-bottom: -1em; +} .prog { font-family: monospace; } @@ -225,3 +206,9 @@ bottom: 0; background: #0a0; } +.prog>a>span { + font-weight: bold; + font-style: italic; + color: #fff; + padding-left: .2em; +} \ No newline at end of file diff --git a/copyparty/web/upload.html b/copyparty/web/upload.html index aa2d468f..ab167a82 100644 --- a/copyparty/web/upload.html +++ b/copyparty/web/upload.html @@ -1,10 +1,3 @@ -
    @@ -34,7 +27,7 @@
    - +
    @@ -44,6 +37,25 @@

    ' + sconf[a][0] + '    		\n' + + '\n' + + '
    -
    ' + links + '
    		' + sz + + '' + ext + '' + ts + '
    ' + res[a][0] + '' + res[a][2] + '' + res[a][b] + '
    + + + + {%- if have_up2k_idx %} + + {%- endif %} + - - -
    parallel uploads + + + + + + + + + + + +
    @@ -51,26 +63,18 @@ + - - - - - - - - -
    +
    +
    -
    - drop files here
    - (or click me) +
    +
    + drop files here
    + (or click me) +
    @@ -82,5 +86,5 @@

    -

    ( if you don't need lastmod timestamps, resumable uploads or progress bars just use the basic uploader)

    +

    ( if you don't need lastmod timestamps, resumable uploads or progress bars just use the basic uploader)

    diff --git a/copyparty/web/util.js b/copyparty/web/util.js index 50cf70df..41b31348 100644 --- a/copyparty/web/util.js +++ b/copyparty/web/util.js @@ -106,4 +106,108 @@ function makeSortable(table) { sortTable(table, i); }; }(i)); -} \ No newline at end of file +} + + + +(function () { + var ops = document.querySelectorAll('#ops>a'); + for (var a = 0; a < ops.length; a++) { + ops[a].onclick = opclick; + } +})(); + + +function opclick(ev) { + if (ev) //ie + ev.preventDefault(); + + var dest = this.getAttribute('data-dest'); + goto(dest); + + // writing a blank value makes ie8 segfault w + if (window.localStorage) + localStorage.setItem('opmode', dest || '.'); + + var input = document.querySelector('.opview.act input:not([type="hidden"])') + if (input) + input.focus(); +} + + +function goto(dest) { + var obj = document.querySelectorAll('.opview.act'); + for (var a = obj.length - 1; a >= 0; a--) + obj[a].classList.remove('act'); + + obj = document.querySelectorAll('#ops>a'); + for (var a = obj.length - 1; a >= 0; a--) + obj[a].classList.remove('act'); + + var others = ['path', 'files', 'widget']; + for (var a = 0; a < others.length; a++) + ebi(others[a]).classList.remove('hidden'); + + if (dest) { + var ui = ebi('op_' + dest); + ui.classList.add('act'); + document.querySelector('#ops>a[data-dest=' + dest + ']').classList.add('act'); + + var fn = window['goto_' + dest]; + if (fn) + fn(); + } +} + + +(function () { + goto(); + if (window.localStorage) { + var op = localStorage.getItem('opmode'); + if (op !== null && op !== '.') + goto(op); + } + ebi('ops').style.display = 'block'; +})(); + + +function linksplit(rp) { + var ret = []; + var apath = '/'; + while (rp) { + var link = rp; + var ofs = rp.indexOf('/'); + if (ofs === -1) { + rp = null; + } + else { + link = rp.slice(0, ofs + 1); + rp = rp.slice(ofs + 1); + } + var vlink = link; + if (link.indexOf('/') !== -1) + vlink = link.slice(0, -1) + '/'; + + ret.push('' + vlink + ''); + apath += link; + } + return ret; +} + + +function get_evpath() { + var ret = document.location.pathname; + + if (ret.indexOf('/') !== 0) + ret = '/' + ret; + + if (ret.lastIndexOf('/') !== ret.length - 1) + ret += '/'; + + return ret; +} + + +function get_vpath() { + return decodeURIComponent(get_evpath()); +}