Merge branch 'master' of gh:9001/copyparty

copyparty/__main__.py

@@ -278,6 +278,7 @@ def main():
     ap2.add_argument("-mte", metavar="M,M,M", type=str, help="tags to index/display (comma-sep.)",
         default="circle,album,.tn,artist,title,.bpm,key,.dur,.q")
     ap2.add_argument("-mtp", metavar="M=[f,]bin", action="append", type=str, help="read tag M using bin")
+    ap2.add_argument("--srch-time", metavar="SEC", type=int, default=30, help="search deadline")
 
     ap2 = ap.add_argument_group('SSL/TLS options')
     ap2.add_argument("--http-only", action="store_true", help="disable ssl/tls")

copyparty/httpcli.py

@@ -45,7 +45,7 @@ class HttpCli(object):
         self.log_func(self.log_src, msg, c)
 
     def _check_nonfatal(self, ex):
-        return ex.code < 400 or ex.code == 404
+        return ex.code < 400 or ex.code in [404, 429]
 
     def _assert_safe_rem(self, rem):
         # sanity check to prevent any disasters
@@ -450,19 +450,30 @@ class HttpCli(object):
 
         idx = self.conn.get_u2idx()
         t0 = time.time()
+        if idx.p_end:
+            penalty = 0.7
+            t_idle = t0 - idx.p_end
+            if idx.p_dur > 0.7 and t_idle < penalty:
+                m = "rate-limit ({:.1f} sec), cost {:.2f}, idle {:.2f}"
+                raise Pebkac(429, m.format(penalty, idx.p_dur, t_idle))
+
         if "srch" in body:
             # search by up2k hashlist
             vbody = copy.deepcopy(body)
             vbody["hash"] = len(vbody["hash"])
             self.log("qj: " + repr(vbody))
             hits = idx.fsearch(vols, body)
-            self.log("q#: {} ({:.2f}s)".format(repr(hits), time.time() - t0))
+            msg = repr(hits)
             taglist = []
         else:
             # search by query params
             self.log("qj: " + repr(body))
             hits, taglist = idx.search(vols, body)
-            self.log("q#: {} ({:.2f}s)".format(len(hits), time.time() - t0))
+            msg = len(hits)
+
+        idx.p_end = time.time()
+        idx.p_dur = idx.p_end - t0
+        self.log("q#: {} ({:.2f}s)".format(msg, idx.p_dur))
 
         order = []
         cfg = self.args.mte.split(",")

copyparty/u2idx.py

@@ -3,6 +3,8 @@ from __future__ import print_function, unicode_literals
 
 import re
 import os
+import time
+import threading
 from datetime import datetime
 
 from .util import u8safe, s3dec, html_escape, Pebkac
@@ -20,6 +22,7 @@ class U2idx(object):
     def __init__(self, args, log_func):
         self.args = args
         self.log_func = log_func
+        self.timeout = args.srch_time
 
         if not HAVE_SQLITE3:
             self.log("could not load sqlite3; searchign wqill be disabled")
@@ -29,6 +32,9 @@ class U2idx(object):
         self.mem_cur = sqlite3.connect(":memory:")
         self.mem_cur.execute(r"create table a (b text)")
 
+        self.p_end = None
+        self.p_dur = 0
+
     def log(self, msg, c=0):
         self.log_func("u2idx", msg, c)
 
@@ -44,7 +50,10 @@ class U2idx(object):
         uq = "substr(w,1,16) = ? and w = ?"
         uv = [wark[:16], wark]
 
-        return self.run_query(vols, uq, uv, "", [])[0]
+        try:
+            return self.run_query(vols, uq, uv, "", [])[0]
+        except Exception as ex:
+            raise Pebkac(500, repr(ex))
 
     def get_cur(self, ptop):
         cur = self.cur.get(ptop)
@@ -81,11 +90,20 @@ class U2idx(object):
         if "adv" in body:
             _conv_adv(qobj, body, "adv")
 
-        return self.run_query(vols, uq, uv, qobj)
+        try:
+            return self.run_query(vols, uq, uv, qobj)
+        except Exception as ex:
+            raise Pebkac(500, repr(ex))
 
     def run_query(self, vols, uq, uv, targs):
         self.log("qs: {} {} ,  {}".format(uq, repr(uv), repr(targs)))
 
+        done_flag = []
+        self.active_id = "{:.6f}_{}".format(time.time(), threading.current_thread().ident)
+        thr = threading.Thread(target=self.terminator, args=(self.active_id, done_flag, ))
+        thr.daemon = True
+        thr.start()
+
         if not targs:
             if not uq:
                 q = "select * from up"
@@ -124,6 +142,8 @@ class U2idx(object):
             if not cur:
                 continue
 
+            self.active_cur = cur
+
             sret = []
             c = cur.execute(q, v)
             for hit in c:
@@ -151,8 +171,20 @@ class U2idx(object):
 
             ret.extend(sret)
 
+        done_flag.append(True)
+        self.active_id = None
+
         return ret, list(taglist.keys())
 
+    def terminator(self, identifier, done_flag):
+        for _ in range(self.timeout):
+            time.sleep(1)
+            if done_flag:
+                return
+
+        if identifier == self.active_id:
+            self.active_cur.connection.interrupt()
+
 
 def _open(ptop):
     db_path = os.path.join(ptop, ".hist", "up2k.db")

copyparty/up2k.py

@@ -621,7 +621,8 @@ class Up2k(object):
             wcur.close()
             cur.close()
 
-        self.log("mtp finished")
+        if n_done:
+            self.log("mtp finished")
 
     def _start_mpool(self):
         if WINDOWS and False:

copyparty/util.py

@@ -57,6 +57,7 @@ HTTPCODE = {
     413: "Payload Too Large",
     416: "Requested Range Not Satisfiable",
     422: "Unprocessable Entity",
+    429: "Too Many Requests",
     500: "Internal Server Error",
     501: "Not Implemented",
 }

copyparty/web/browser.css

@@ -431,7 +431,8 @@ input[type="checkbox"]:checked+label {
 #srch_q {
 	white-space: pre;
 	color: #f80;
-	margin: .2em 0 0 1.6em
+	height: 1em;
+	margin: .2em 0 -1em 1.6em;
 }
 #files td div span {
 	color: #fff;
@@ -641,4 +642,4 @@ input[type="checkbox"]:checked+label {
 	border-radius: .3em;
 	font-family: monospace, monospace;
 	line-height: 2em;
-}
+}

copyparty/web/browser.js

@@ -653,7 +653,14 @@ document.onkeydown = function (e) {
 		o[a].oninput = ev_search_input;
 	}
 
+	function srch_msg(err, txt) {
+		var o = ebi('srch_q');
+		o.textContent = txt;
+		o.style.color = err ? '#f09' : '#c90';
+	}
+
 	var search_timeout;
+	var search_in_progress = 0;
 
 	function ev_search_input() {
 		var v = this.value;
@@ -663,10 +670,14 @@ document.onkeydown = function (e) {
 			chk.checked = ((v + '').length > 0);
 		}
 		clearTimeout(search_timeout);
-		search_timeout = setTimeout(do_search, 100);
+		var now = new Date().getTime();
+		if (now - search_in_progress > 30 * 1000)
+			search_timeout = setTimeout(do_search, 100);
 	}
 
 	function do_search() {
+		search_in_progress = new Date().getTime();
+		srch_msg(false, "searching...");
 		clearTimeout(search_timeout);
 		var params = {};
 		var o = document.querySelectorAll('#op_search input[type="text"]');
@@ -690,10 +701,16 @@ document.onkeydown = function (e) {
 			return;
 
 		if (this.status !== 200) {
-			ebi('srch_q').textContent = "http " + this.status + ": " + this.responseText;
+			var msg = this.responseText;
+			if (msg.indexOf('<pre>') === 0)
+				msg = msg.slice(5);
+
+			srch_msg(true, "http " + this.status + ": " + msg);
+			search_in_progress = 0;
 			return;
 		}
-		ebi('srch_q').textContent = '';
+		search_in_progress = 0;
+		srch_msg(false, '');
 
 		var res = JSON.parse(this.responseText),
 			tagord = res.tag_order;