diff --git a/copyparty/httpcli.py b/copyparty/httpcli.py index b02ab2e8..61223095 100644 --- a/copyparty/httpcli.py +++ b/copyparty/httpcli.py @@ -1373,11 +1373,13 @@ class HttpCli(object): for y in [self.rvol, self.wvol, self.avol] ] - vstate = {} if self.avol and not self.args.no_rescan: - x = self.conn.hsrv.broker.put(True, "up2k.get_volstate") - vstate = json.loads(x.get()) - vstate = {("/" + k).rstrip("/") + "/": v for k, v in vstate.items()} + x = self.conn.hsrv.broker.put(True, "up2k.get_state") + vs = json.loads(x.get()) + vstate = {("/" + k).rstrip("/") + "/": v for k, v in vs["volstate"].items()} + else: + vstate = {} + vs = {"scanning": None, "hashq": None, "tagq": None} html = self.j2( "splash", @@ -1386,6 +1388,9 @@ class HttpCli(object): wvol=wvol, avol=avol, vstate=vstate, + scanning=vs["scanning"], + hashq=vs["hashq"], + tagq=vs["tagq"], url_suf=suf, ) self.reply(html.encode("utf-8"), headers=NO_STORE) diff --git a/copyparty/mtag.py b/copyparty/mtag.py index 6a5d272b..4ae7a3dd 100644 --- a/copyparty/mtag.py +++ b/copyparty/mtag.py @@ -1,7 +1,6 @@ # coding: utf-8 from __future__ import print_function, unicode_literals -import re import os import sys import json diff --git a/copyparty/up2k.py b/copyparty/up2k.py index 2ba907a0..420950bc 100644 --- a/copyparty/up2k.py +++ b/copyparty/up2k.py @@ -61,6 +61,8 @@ class Up2k(object): self.mutex = threading.Lock() self.hashq = Queue() self.tagq = Queue() + self.n_hashq = 0 + self.n_tagq = 0 self.volstate = {} self.registry = {} self.entags = {} @@ -129,8 +131,14 @@ class Up2k(object): def log(self, msg, c=0): self.log_func("up2k", msg + "\033[K", c) - def get_volstate(self): - return json.dumps(self.volstate, indent=4) + def get_state(self): + ret = { + "volstate": self.volstate, + "scanning": hasattr(self, "pp"), + "hashq": self.n_hashq, + "tagq": self.n_tagq, + } + return json.dumps(ret, indent=4) def rescan(self, all_vols, scan_vols): if hasattr(self, "pp"): @@ -1233,6 +1241,7 @@ class Up2k(object): if "e2t" in self.flags[ptop]: self.tagq.put([ptop, wark, rd, fn]) + self.n_tagq += 1 return True @@ -1410,7 +1419,13 @@ class Up2k(object): prev[ptop] = etag def _tagger(self): + with self.mutex: + self.n_tagq += 1 + while True: + with self.mutex: + self.n_tagq -= 1 + ptop, wark, rd, fn = self.tagq.get() if "e2t" not in self.flags[ptop]: continue @@ -1441,8 +1456,16 @@ class Up2k(object): self.log("tagged {} ({}+{})".format(abspath, ntags1, len(tags) - ntags1)) def _hasher(self): + with self.mutex: + self.n_hashq += 1 + while True: + with self.mutex: + self.n_hashq -= 1 + # self.log("hashq {}".format(self.n_hashq)) + ptop, rd, fn = self.hashq.get() + # self.log("hashq {} pop {}/{}/{}".format(self.n_hashq, ptop, rd, fn)) if "e2d" not in self.flags[ptop]: continue @@ -1458,6 +1481,8 @@ class Up2k(object): with self.mutex: self.register_vpath(ptop, flags) self.hashq.put([ptop, rd, fn]) + self.n_hashq += 1 + # self.log("hashq {} push {}/{}/{}".format(self.n_hashq, ptop, rd, fn)) def up2k_chunksize(filesize): diff --git a/copyparty/web/splash.css b/copyparty/web/splash.css index 47a4a7f7..af8eafbd 100644 --- a/copyparty/web/splash.css +++ b/copyparty/web/splash.css @@ -26,10 +26,20 @@ a { border-radius: .2em; padding: .2em .8em; } -td, th { +table { + border-collapse: collapse; +} +.vols td, +.vols th { padding: .3em .6em; text-align: left; } +.num td { + padding: .1em .7em .1em 0; +} +.num td:first-child { + text-align: right; +} .btns { margin: 1em 0; } diff --git a/copyparty/web/splash.html b/copyparty/web/splash.html index e3bd332a..f1f065de 100644 --- a/copyparty/web/splash.html +++ b/copyparty/web/splash.html @@ -15,16 +15,24 @@ {%- if avol %}

admin panel:

- - - - {% for mp in avol %} - {%- if mp in vstate and vstate[mp] %} - - {%- endif %} - {% endfor %} - -
volactionstatus
{{ mp }}rescan{{ vstate[mp] }}
+
+ + + + +
scanning{{ scanning }}
hash-q{{ hashq }}
tag-q{{ tagq }}
+ 		+ + + + {% for mp in avol %} + {%- if mp in vstate and vstate[mp] %} + + {%- endif %} + {% endfor %} + +
volactionstatus
{{ mp }}rescan{{ vstate[mp] }}
+
dump stack
diff --git a/scripts/run-tests.sh b/scripts/run-tests.sh index eb93d44b..48aaf075 100755 --- a/scripts/run-tests.sh +++ b/scripts/run-tests.sh @@ -3,10 +3,13 @@ set -ex pids=() for py in python{2,3}; do - $py -m unittest discover -s tests >/dev/null & + nice $py -m unittest discover -s tests >/dev/null & pids+=($!) done +python3 scripts/test/smoketest.py & +pids+=($!) + for pid in ${pids[@]}; do wait $pid done diff --git a/scripts/test/smoketest.py b/scripts/test/smoketest.py index 3ff06082..d6dff9df 100644 --- a/scripts/test/smoketest.py +++ b/scripts/test/smoketest.py @@ -1,18 +1,25 @@ import os import sys import time -import signal +import shlex import shutil +import signal import tempfile import requests import threading import subprocess as sp +CPP = [] + + class Cpp(object): def __init__(self, args): + args = [sys.executable, "-m", "copyparty"] + args + print(" ".join([shlex.quote(x) for x in args])) + self.ls_pre = set(list(os.listdir())) - self.p = sp.Popen([sys.executable, "-m", "copyparty"] + args) + self.p = sp.Popen(args) # , stdout=sp.PIPE, stderr=sp.PIPE) self.t = threading.Thread(target=self._run) @@ -23,10 +30,11 @@ class Cpp(object): self.so, self.se = self.p.communicate() def stop(self, wait): - # self.p.kill() - os.kill(self.p.pid, signal.SIGINT) if wait: - self.t.join() + os.kill(self.p.pid, signal.SIGINT) + self.t.join(timeout=2) + else: + self.p.kill() # macos py3.8 def clean(self): t = os.listdir() @@ -34,6 +42,22 @@ class Cpp(object): if f not in self.ls_pre and f.startswith("up."): os.unlink(f) + def await_idle(self, ub, timeout): + req = ["scanningFalse", "hash-q0", "tag-q0"] + u = ub + "?h" + for _ in range(timeout * 10): + try: + time.sleep(0.1) + r = requests.get(u, timeout=0.1) + for x in req: + if x not in r.text: + print("ST: miss " + x) + raise Exception() + print("ST: idle") + return + except: + pass + def tc1(): ub = "http://127.0.0.1:4321/" @@ -61,10 +85,10 @@ def tc1(): ovid = f.read() args = [ - "-p", - "4321", + "-p4321", "-e2dsa", "-e2tsr", + "--no-mutagen", "--th-ff-jpg", "--hist", os.path.join(td, "dbm"), @@ -89,30 +113,24 @@ def tc1(): hp = None if pd.endswith("st/a"): - hp = os.path.join(td, "db1") + hp = hpaths[ud] = os.path.join(td, "db1") elif pd[:-1].endswith("a/j/"): - hp = os.path.join(td, "dbm") + hpaths[ud] = os.path.join(td, "dbm") + hp = None else: hp = "-" + hpaths[ud] = os.path.join(pd, ".hist") - hpaths[ud] = os.path.join(pd, ".hist") if hp == "-" else hp - args += ["-v", "{}:{}:{}:chist={}".format(pd, ud, p, hp)] + arg = "{}:{}:{}".format(pd, ud, p, hp) + if hp: + arg += ":chist=" + hp + + args += ["-v", arg] - # print(repr(args)) # return cpp = Cpp(args) - - up = False - for n in range(30): - try: - time.sleep(0.1) - requests.get(ub + "?h", timeout=0.1) - up = True - break - except: - pass - - assert up + CPP.append(cpp) + cpp.await_idle(ub, 3) for d in udirs: vid = ovid + "\n{}".format(d).encode("utf-8") @@ -147,6 +165,7 @@ def tc1(): raise Exception("thumb {} with perm {} at {}".format(ok, p, u)) # check tags + cpp.await_idle(ub, 5) for d, p in zip(udirs, perms): u = "{}{}?ls".format(ub, d) r = requests.get(u) @@ -163,7 +182,7 @@ def tc1(): raise Exception("ls {} with perm {} at {}".format(ok, p, u)) if (tag and p != "a") or (not tag and p == "a"): - raise Exception("tag {} with perm {} at {}".format(ok, p, u)) + raise Exception("tag {} with perm {} at {}".format(tag, p, u)) if tag is not None and tag != "48x32": raise Exception("tag [{}] at {}".format(tag, u)) @@ -171,8 +190,18 @@ def tc1(): cpp.stop(True) +def run(tc): + try: + tc() + finally: + try: + CPP[0].stop(False) + except: + pass + + def main(): - tc1() + run(tc1) if __name__ == "__main__":