more status on admin panel

copyparty/httpcli.py

@@ -1373,11 +1373,13 @@ class HttpCli(object):
             for y in [self.rvol, self.wvol, self.avol]
         ]
 
-        vstate = {}
         if self.avol and not self.args.no_rescan:
-            x = self.conn.hsrv.broker.put(True, "up2k.get_volstate")
-            vstate = json.loads(x.get())
-            vstate = {("/" + k).rstrip("/") + "/": v for k, v in vstate.items()}
+            x = self.conn.hsrv.broker.put(True, "up2k.get_state")
+            vs = json.loads(x.get())
+            vstate = {("/" + k).rstrip("/") + "/": v for k, v in vs["volstate"].items()}
+        else:
+            vstate = {}
+            vs = {"scanning": None, "hashq": None, "tagq": None}
 
         html = self.j2(
             "splash",
@@ -1386,6 +1388,9 @@ class HttpCli(object):
             wvol=wvol,
             avol=avol,
             vstate=vstate,
+            scanning=vs["scanning"],
+            hashq=vs["hashq"],
+            tagq=vs["tagq"],
             url_suf=suf,
         )
         self.reply(html.encode("utf-8"), headers=NO_STORE)

copyparty/mtag.py

@@ -1,7 +1,6 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
-import re
 import os
 import sys
 import json

copyparty/up2k.py

@@ -61,6 +61,8 @@ class Up2k(object):
         self.mutex = threading.Lock()
         self.hashq = Queue()
         self.tagq = Queue()
+        self.n_hashq = 0
+        self.n_tagq = 0
         self.volstate = {}
         self.registry = {}
         self.entags = {}
@@ -129,8 +131,14 @@ class Up2k(object):
     def log(self, msg, c=0):
         self.log_func("up2k", msg + "\033[K", c)
 
-    def get_volstate(self):
-        return json.dumps(self.volstate, indent=4)
+    def get_state(self):
+        ret = {
+            "volstate": self.volstate,
+            "scanning": hasattr(self, "pp"),
+            "hashq": self.n_hashq,
+            "tagq": self.n_tagq,
+        }
+        return json.dumps(ret, indent=4)
 
     def rescan(self, all_vols, scan_vols):
         if hasattr(self, "pp"):
@@ -1233,6 +1241,7 @@ class Up2k(object):
 
         if "e2t" in self.flags[ptop]:
             self.tagq.put([ptop, wark, rd, fn])
+            self.n_tagq += 1
 
         return True
 
@@ -1410,7 +1419,13 @@ class Up2k(object):
         prev[ptop] = etag
 
     def _tagger(self):
+        with self.mutex:
+            self.n_tagq += 1
+
         while True:
+            with self.mutex:
+                self.n_tagq -= 1
+
             ptop, wark, rd, fn = self.tagq.get()
             if "e2t" not in self.flags[ptop]:
                 continue
@@ -1441,8 +1456,16 @@ class Up2k(object):
             self.log("tagged {} ({}+{})".format(abspath, ntags1, len(tags) - ntags1))
 
     def _hasher(self):
+        with self.mutex:
+            self.n_hashq += 1
+
         while True:
+            with self.mutex:
+                self.n_hashq -= 1
+            # self.log("hashq {}".format(self.n_hashq))
+
             ptop, rd, fn = self.hashq.get()
+            # self.log("hashq {} pop {}/{}/{}".format(self.n_hashq, ptop, rd, fn))
             if "e2d" not in self.flags[ptop]:
                 continue
 
@@ -1458,6 +1481,8 @@ class Up2k(object):
         with self.mutex:
             self.register_vpath(ptop, flags)
             self.hashq.put([ptop, rd, fn])
+            self.n_hashq += 1
+        # self.log("hashq {} push {}/{}/{}".format(self.n_hashq, ptop, rd, fn))
 
 
 def up2k_chunksize(filesize):

copyparty/web/splash.css

@@ -26,10 +26,20 @@ a {
 	border-radius: .2em;
 	padding: .2em .8em;
 }
-td, th {
+table {
+	border-collapse: collapse;
+}
+.vols td,
+.vols th {
 	padding: .3em .6em;
 	text-align: left;
 }
+.num td {
+	padding: .1em .7em .1em 0;
+}
+.num td:first-child {
+	text-align: right;
+}
 .btns {
 	margin: 1em 0;
 }

copyparty/web/splash.html

@@ -15,16 +15,24 @@
 
         {%- if avol %}
         <h1>admin panel:</h1>
-        <table>
-            <thead><tr><th>vol</th><th>action</th><th>status</th></tr></thead>
-            <tbody>
-                {% for mp in avol %}
-                {%- if mp in vstate and vstate[mp] %}
-                <tr><td><a href="{{ mp }}{{ url_suf }}">{{ mp }}</a></td><td><a href="{{ mp }}?scan">rescan</a></td><td>{{ vstate[mp] }}</td></tr>
-                {%- endif %}
-                {% endfor %}
-            </tbody>
-        </table>
+        <table><tr><td> <!-- hehehe -->
+            <table class="num">
+                <tr><td>scanning</td><td>{{ scanning }}</td></tr>
+                <tr><td>hash-q</td><td>{{ hashq }}</td></tr>
+                <tr><td>tag-q</td><td>{{ tagq }}</td></tr>
+            </table>            
+        </td><td>
+            <table class="vols">
+                <thead><tr><th>vol</th><th>action</th><th>status</th></tr></thead>
+                <tbody>
+                    {% for mp in avol %}
+                    {%- if mp in vstate and vstate[mp] %}
+                    <tr><td><a href="{{ mp }}{{ url_suf }}">{{ mp }}</a></td><td><a href="{{ mp }}?scan">rescan</a></td><td>{{ vstate[mp] }}</td></tr>
+                    {%- endif %}
+                    {% endfor %}
+                </tbody>
+            </table>
+        </td></tr></table>
         <div class="btns">
             <a href="{{ avol[0] }}?stack">dump stack</a>
         </div>

scripts/run-tests.sh

@@ -3,10 +3,13 @@ set -ex
 
 pids=()
 for py in python{2,3}; do
-    $py -m unittest discover -s tests >/dev/null &
+    nice $py -m unittest discover -s tests >/dev/null &
     pids+=($!)
 done
 
+python3 scripts/test/smoketest.py &
+pids+=($!)
+
 for pid in ${pids[@]}; do
     wait $pid
 done

scripts/test/smoketest.py

@@ -1,18 +1,25 @@
 import os
 import sys
 import time
-import signal
+import shlex
 import shutil
+import signal
 import tempfile
 import requests
 import threading
 import subprocess as sp
 
 
+CPP = []
+
+
 class Cpp(object):
     def __init__(self, args):
+        args = [sys.executable, "-m", "copyparty"] + args
+        print(" ".join([shlex.quote(x) for x in args]))
+
         self.ls_pre = set(list(os.listdir()))
-        self.p = sp.Popen([sys.executable, "-m", "copyparty"] + args)
+        self.p = sp.Popen(args)
         # , stdout=sp.PIPE, stderr=sp.PIPE)
 
         self.t = threading.Thread(target=self._run)
@@ -23,10 +30,11 @@ class Cpp(object):
         self.so, self.se = self.p.communicate()
 
     def stop(self, wait):
-        # self.p.kill()
-        os.kill(self.p.pid, signal.SIGINT)
         if wait:
-            self.t.join()
+            os.kill(self.p.pid, signal.SIGINT)
+            self.t.join(timeout=2)
+        else:
+            self.p.kill()  # macos py3.8
 
     def clean(self):
         t = os.listdir()
@@ -34,6 +42,22 @@ class Cpp(object):
             if f not in self.ls_pre and f.startswith("up."):
                 os.unlink(f)
 
+    def await_idle(self, ub, timeout):
+        req = ["scanning</td><td>False", "hash-q</td><td>0", "tag-q</td><td>0"]
+        u = ub + "?h"
+        for _ in range(timeout * 10):
+            try:
+                time.sleep(0.1)
+                r = requests.get(u, timeout=0.1)
+                for x in req:
+                    if x not in r.text:
+                        print("ST: miss " + x)
+                        raise Exception()
+                print("ST: idle")
+                return
+            except:
+                pass
+
 
 def tc1():
     ub = "http://127.0.0.1:4321/"
@@ -61,10 +85,10 @@ def tc1():
         ovid = f.read()
 
     args = [
-        "-p",
-        "4321",
+        "-p4321",
         "-e2dsa",
         "-e2tsr",
+        "--no-mutagen",
         "--th-ff-jpg",
         "--hist",
         os.path.join(td, "dbm"),
@@ -89,30 +113,24 @@ def tc1():
 
         hp = None
         if pd.endswith("st/a"):
-            hp = os.path.join(td, "db1")
+            hp = hpaths[ud] = os.path.join(td, "db1")
         elif pd[:-1].endswith("a/j/"):
-            hp = os.path.join(td, "dbm")
+            hpaths[ud] = os.path.join(td, "dbm")
+            hp = None
         else:
             hp = "-"
+            hpaths[ud] = os.path.join(pd, ".hist")
 
-        hpaths[ud] = os.path.join(pd, ".hist") if hp == "-" else hp
-        args += ["-v", "{}:{}:{}:chist={}".format(pd, ud, p, hp)]
+        arg = "{}:{}:{}".format(pd, ud, p, hp)
+        if hp:
+            arg += ":chist=" + hp
+
+        args += ["-v", arg]
 
-    # print(repr(args))
     # return
     cpp = Cpp(args)
-
-    up = False
-    for n in range(30):
-        try:
-            time.sleep(0.1)
-            requests.get(ub + "?h", timeout=0.1)
-            up = True
-            break
-        except:
-            pass
-
-    assert up
+    CPP.append(cpp)
+    cpp.await_idle(ub, 3)
 
     for d in udirs:
         vid = ovid + "\n{}".format(d).encode("utf-8")
@@ -147,6 +165,7 @@ def tc1():
             raise Exception("thumb {} with perm {} at {}".format(ok, p, u))
 
     # check tags
+    cpp.await_idle(ub, 5)
     for d, p in zip(udirs, perms):
         u = "{}{}?ls".format(ub, d)
         r = requests.get(u)
@@ -163,7 +182,7 @@ def tc1():
             raise Exception("ls {} with perm {} at {}".format(ok, p, u))
 
         if (tag and p != "a") or (not tag and p == "a"):
-            raise Exception("tag {} with perm {} at {}".format(ok, p, u))
+            raise Exception("tag {} with perm {} at {}".format(tag, p, u))
 
         if tag is not None and tag != "48x32":
             raise Exception("tag [{}] at {}".format(tag, u))
@@ -171,8 +190,18 @@ def tc1():
     cpp.stop(True)
 
 
+def run(tc):
+    try:
+        tc()
+    finally:
+        try:
+            CPP[0].stop(False)
+        except:
+            pass
+
+
 def main():
-    tc1()
+    run(tc1)
 
 
 if __name__ == "__main__":