From 60ac68d000ba0ed2874565bacbffbf2e77cb9ebf Mon Sep 17 00:00:00 2001
From: ed <s@ocv.me>
Date: Fri, 11 Jun 2021 23:01:13 +0200
Subject: [PATCH] single authsrv instance per process

---
 .vscode/launch.json       |  9 ++---
 copyparty/authsrv.py      |  6 ++-
 copyparty/broker_mpw.py   |  2 +-
 copyparty/broker_thr.py   |  4 +-
 copyparty/httpcli.py      | 39 +++++++++---------
 copyparty/httpconn.py     |  4 +-
 copyparty/httpsrv.py      |  2 +-
 copyparty/svchub.py       | 13 +++---
 copyparty/th_cli.py       |  6 +--
 copyparty/th_srv.py       | 33 +++++++--------
 copyparty/u2idx.py        | 14 +++----
 copyparty/up2k.py         | 39 +++++++++---------
 scripts/test/smoketest.py | 84 ++++++++++++++++++++++++++++++++-------
 tests/test_httpcli.py     |  6 +--
 tests/util.py             |  4 +-
 15 files changed, 156 insertions(+), 109 deletions(-)

diff --git a/.vscode/launch.json b/.vscode/launch.json
index 0d6b7584..b1e94a04 100644
--- a/.vscode/launch.json
+++ b/.vscode/launch.json
@@ -16,12 +16,9 @@
                 "-e2ts",
                 "-mtp",
                 ".bpm=f,bin/mtag/audio-bpm.py",
-                "-a",
-                "ed:wark",
-                "-v",
-                "srv::r:aed:cnodupe",
-                "-v",
-                "dist:dist:r"
+                "-aed:wark",
+                "-vsrv::r:aed:cnodupe",
+                "-vdist:dist:r"
             ]
         },
         {
diff --git a/copyparty/authsrv.py b/copyparty/authsrv.py
index 2cbd18d7..4755b9dc 100644
--- a/copyparty/authsrv.py
+++ b/copyparty/authsrv.py
@@ -43,7 +43,9 @@ class VFS(object):
         )
 
     def get_all_vols(self, outdict):
-        outdict[self.vpath] = self
+        if self.realpath:
+            outdict[self.vpath] = self
+
         for v in self.nodes.values():
             v.get_all_vols(outdict)
 
@@ -60,7 +62,7 @@ class VFS(object):
                 return self.nodes[name].add(src, dst)
 
             vn = VFS(
-                os.path.join(self.realpath, name) if self.realpath else name,
+                os.path.join(self.realpath, name) if self.realpath else None,
                 "{}/{}".format(self.vpath, name).lstrip("/"),
                 self.uread,
                 self.uwrite,
diff --git a/copyparty/broker_mpw.py b/copyparty/broker_mpw.py
index 8e27a5a9..d82202a2 100644
--- a/copyparty/broker_mpw.py
+++ b/copyparty/broker_mpw.py
@@ -36,7 +36,7 @@ class MpWorker(object):
             signal.signal(signal.SIGINT, self.signal_handler)
 
         # starting to look like a good idea
-        self.authsrv = AuthSrv(args, None, False)
+        self.asrv = AuthSrv(args, None, False)
 
         # instantiate all services here (TODO: inheritance?)
         self.httpsrv = HttpSrv(self, True)
diff --git a/copyparty/broker_thr.py b/copyparty/broker_thr.py
index b98e336f..64decbc2 100644
--- a/copyparty/broker_thr.py
+++ b/copyparty/broker_thr.py
@@ -15,12 +15,10 @@ class BrokerThr(object):
         self.hub = hub
         self.log = hub.log
         self.args = hub.args
+        self.asrv = hub.asrv
 
         self.mutex = threading.Lock()
 
-        # starting to look like a good idea
-        self.authsrv = AuthSrv(self.args, None, False)
-
         # instantiate all services here (TODO: inheritance?)
         self.httpsrv = HttpSrv(self)
         self.httpsrv.disconnect_func = self.httpdrop
diff --git a/copyparty/httpcli.py b/copyparty/httpcli.py
index a7a18eae..f50b7e5b 100644
--- a/copyparty/httpcli.py
+++ b/copyparty/httpcli.py
@@ -42,7 +42,7 @@ class HttpCli(object):
         self.addr = conn.addr  # type: tuple[str, int]
         self.args = conn.args
         self.is_mp = conn.is_mp
-        self.auth = conn.auth  # type: AuthSrv
+        self.asrv = conn.asrv  # type: AuthSrv
         self.ico = conn.ico
         self.thumbcli = conn.thumbcli
         self.log_func = conn.log_func
@@ -154,9 +154,9 @@ class HttpCli(object):
         self.vpath = unquotep(vpath)
 
         pwd = uparam.get("pw")
-        self.uname = self.auth.iuser.get(pwd, "*")
+        self.uname = self.asrv.iuser.get(pwd, "*")
         self.rvol, self.wvol, self.avol = [[], [], []]
-        self.auth.vfs.user_tree(self.uname, self.rvol, self.wvol, self.avol)
+        self.asrv.vfs.user_tree(self.uname, self.rvol, self.wvol, self.avol)
 
         ua = self.headers.get("user-agent", "")
         self.is_rclone = ua.startswith("rclone/")
@@ -321,9 +321,7 @@ class HttpCli(object):
                     self.redirect(vpath, flavor="redirecting to", use302=True)
                     return True
 
-        self.readable, self.writable = self.conn.auth.vfs.can_access(
-            self.vpath, self.uname
-        )
+        self.readable, self.writable = self.asrv.vfs.can_access(self.vpath, self.uname)
         if not self.readable and not self.writable:
             if self.vpath:
                 self.log("inaccessible: [{}]".format(self.vpath))
@@ -440,7 +438,7 @@ class HttpCli(object):
 
     def dump_to_file(self):
         reader, remains = self.get_body_reader()
-        vfs, rem = self.conn.auth.vfs.get(self.vpath, self.uname, False, True)
+        vfs, rem = self.asrv.vfs.get(self.vpath, self.uname, False, True)
         fdir = os.path.join(vfs.realpath, rem)
 
         addr = self.ip.replace(":", ".")
@@ -509,7 +507,7 @@ class HttpCli(object):
         if v is None:
             raise Pebkac(422, "need zip or tar keyword")
 
-        vn, rem = self.auth.vfs.get(self.vpath, self.uname, True, False)
+        vn, rem = self.asrv.vfs.get(self.vpath, self.uname, True, False)
         items = self.parser.require("files", 1024 * 1024)
         if not items:
             raise Pebkac(422, "need files list")
@@ -559,7 +557,7 @@ class HttpCli(object):
             self.vpath = "/".join([self.vpath, sub]).strip("/")
             body["name"] = name
 
-        vfs, rem = self.conn.auth.vfs.get(self.vpath, self.uname, False, True)
+        vfs, rem = self.asrv.vfs.get(self.vpath, self.uname, False, True)
         dbv, vrem = vfs.get_dbv(rem)
 
         body["vtop"] = dbv.vpath
@@ -590,7 +588,7 @@ class HttpCli(object):
         vols = []
         seen = {}
         for vtop in self.rvol:
-            vfs, _ = self.conn.auth.vfs.get(vtop, self.uname, True, False)
+            vfs, _ = self.asrv.vfs.get(vtop, self.uname, True, False)
             vfs = vfs.dbv or vfs
             if vfs in seen:
                 continue
@@ -651,7 +649,7 @@ class HttpCli(object):
         except KeyError:
             raise Pebkac(400, "need hash and wark headers for binary POST")
 
-        vfs, _ = self.conn.auth.vfs.get(self.vpath, self.uname, False, True)
+        vfs, _ = self.asrv.vfs.get(self.vpath, self.uname, False, True)
         ptop = (vfs.dbv or vfs).realpath
 
         x = self.conn.hsrv.broker.put(True, "up2k.handle_chunk", ptop, wark, chash)
@@ -724,7 +722,7 @@ class HttpCli(object):
         pwd = self.parser.require("cppwd", 64)
         self.parser.drop()
 
-        if pwd in self.auth.iuser:
+        if pwd in self.asrv.iuser:
             msg = "login ok"
             dt = datetime.utcfromtimestamp(time.time() + 60 * 60 * 24 * 365)
             exp = dt.strftime("%a, %d %b %Y %H:%M:%S GMT")
@@ -743,7 +741,7 @@ class HttpCli(object):
         self.parser.drop()
 
         nullwrite = self.args.nw
-        vfs, rem = self.conn.auth.vfs.get(self.vpath, self.uname, False, True)
+        vfs, rem = self.asrv.vfs.get(self.vpath, self.uname, False, True)
         self._assert_safe_rem(rem)
 
         sanitized = sanitize_fn(new_dir)
@@ -772,7 +770,7 @@ class HttpCli(object):
         self.parser.drop()
 
         nullwrite = self.args.nw
-        vfs, rem = self.conn.auth.vfs.get(self.vpath, self.uname, False, True)
+        vfs, rem = self.asrv.vfs.get(self.vpath, self.uname, False, True)
         self._assert_safe_rem(rem)
 
         if not new_file.endswith(".md"):
@@ -796,7 +794,7 @@ class HttpCli(object):
 
     def handle_plain_upload(self):
         nullwrite = self.args.nw
-        vfs, rem = self.conn.auth.vfs.get(self.vpath, self.uname, False, True)
+        vfs, rem = self.asrv.vfs.get(self.vpath, self.uname, False, True)
         self._assert_safe_rem(rem)
 
         files = []
@@ -937,7 +935,7 @@ class HttpCli(object):
             raise Pebkac(400, "could not read lastmod from request")
 
         nullwrite = self.args.nw
-        vfs, rem = self.conn.auth.vfs.get(self.vpath, self.uname, False, True)
+        vfs, rem = self.asrv.vfs.get(self.vpath, self.uname, False, True)
         self._assert_safe_rem(rem)
 
         # TODO:
@@ -1400,9 +1398,10 @@ class HttpCli(object):
         if self.args.no_rescan:
             raise Pebkac(403, "disabled by argv")
 
-        vn, _ = self.auth.vfs.get(self.vpath, self.uname, True, True)
+        vn, _ = self.asrv.vfs.get(self.vpath, self.uname, True, True)
+
+        args = [self.asrv.vfs.all_vols, [vn.vpath]]
 
-        args = [self.auth.vfs.all_vols, [vn.vpath]]
         x = self.conn.hsrv.broker.put(True, "up2k.rescan", *args)
         x = x.get()
         if not x:
@@ -1474,7 +1473,7 @@ class HttpCli(object):
             ret["k" + quotep(excl)] = sub
 
         try:
-            vn, rem = self.auth.vfs.get(top, self.uname, True, False)
+            vn, rem = self.asrv.vfs.get(top, self.uname, True, False)
             fsroot, vfs_ls, vfs_virt = vn.ls(
                 rem, self.uname, not self.args.no_scandir, incl_wo=True
             )
@@ -1515,7 +1514,7 @@ class HttpCli(object):
 
                 vpnodes.append([quotep(vpath) + "/", html_escape(node, crlf=True)])
 
-        vn, rem = self.auth.vfs.get(
+        vn, rem = self.asrv.vfs.get(
             self.vpath, self.uname, self.readable, self.writable
         )
         abspath = vn.canonical(rem)
diff --git a/copyparty/httpconn.py b/copyparty/httpconn.py
index 59fe9d40..db7e7f86 100644
--- a/copyparty/httpconn.py
+++ b/copyparty/httpconn.py
@@ -34,7 +34,7 @@ class HttpConn(object):
         self.hsrv = hsrv
 
         self.args = hsrv.args
-        self.auth = hsrv.auth
+        self.asrv = hsrv.asrv
         self.is_mp = hsrv.is_mp
         self.cert_path = hsrv.cert_path
 
@@ -71,7 +71,7 @@ class HttpConn(object):
 
     def get_u2idx(self):
         if not self.u2idx:
-            self.u2idx = U2idx(self.args, self.log_func, self.auth.vfs)
+            self.u2idx = U2idx(self)
 
         return self.u2idx
 
diff --git a/copyparty/httpsrv.py b/copyparty/httpsrv.py
index a757e3ae..be81c422 100644
--- a/copyparty/httpsrv.py
+++ b/copyparty/httpsrv.py
@@ -40,7 +40,7 @@ class HttpSrv(object):
         self.is_mp = is_mp
         self.args = broker.args
         self.log = broker.log
-        self.auth = broker.authsrv
+        self.asrv = broker.asrv
 
         self.disconnect_func = None
         self.mutex = threading.Lock()
diff --git a/copyparty/svchub.py b/copyparty/svchub.py
index 654ec2c6..2dd711bd 100644
--- a/copyparty/svchub.py
+++ b/copyparty/svchub.py
@@ -37,14 +37,13 @@ class SvcHub(object):
 
         self.log = self._log_disabled if args.q else self._log_enabled
 
-        # jank goes here
-        auth = AuthSrv(self.args, self.log, False)
-        if args.ls:
-            auth.dbg_ls()
-
         # initiate all services to manage
+        self.asrv = AuthSrv(self.args, self.log, False)
+        if args.ls:
+            self.asrv.dbg_ls()
+
         self.tcpsrv = TcpSrv(self)
-        self.up2k = Up2k(self, auth.vfs)
+        self.up2k = Up2k(self)
 
         self.thumbsrv = None
         if not args.no_thumb:
@@ -54,7 +53,7 @@ class SvcHub(object):
                     msg = "setting --th-no-webp because either libwebp is not available or your Pillow is too old"
                     self.log("thumb", msg, c=3)
 
-                self.thumbsrv = ThumbSrv(self, auth.vfs)
+                self.thumbsrv = ThumbSrv(self)
             else:
                 msg = "need Pillow to create thumbnails; for example:\n{}{} -m pip install --user Pillow\n"
                 self.log(
diff --git a/copyparty/th_cli.py b/copyparty/th_cli.py
index 49d7f29d..a3b96c5b 100644
--- a/copyparty/th_cli.py
+++ b/copyparty/th_cli.py
@@ -11,7 +11,7 @@ class ThumbCli(object):
     def __init__(self, broker):
         self.broker = broker
         self.args = broker.args
-        self.hist = broker.authsrv.vfs.histtab
+        self.asrv = broker.asrv
 
         # cache on both sides for less broker spam
         self.cooldown = Cooldown(self.args.th_poke)
@@ -32,8 +32,8 @@ class ThumbCli(object):
             if self.args.th_no_webp or (is_vid and self.args.th_ff_jpg):
                 fmt = "j"
 
-        hist = self.hist[ptop]
-        tpath = thumb_path(hist, rem, mtime, fmt)
+        histpath = self.asrv.vfs.histtab[ptop]
+        tpath = thumb_path(histpath, rem, mtime, fmt)
         ret = None
         try:
             st = os.stat(tpath)
diff --git a/copyparty/th_srv.py b/copyparty/th_srv.py
index 1a7f5e49..cc10d4c6 100644
--- a/copyparty/th_srv.py
+++ b/copyparty/th_srv.py
@@ -2,7 +2,6 @@
 from __future__ import print_function, unicode_literals
 
 import os
-import sys
 import time
 import shutil
 import base64
@@ -74,7 +73,7 @@ if HAVE_FFMPEG and HAVE_FFPROBE:
     THUMBABLE.update(FMT_FF)
 
 
-def thumb_path(hist, rem, mtime, fmt):
+def thumb_path(histpath, rem, mtime, fmt):
     # base16 = 16 = 256
     # b64-lc = 38 = 1444
     # base64 = 64 = 4096
@@ -96,16 +95,14 @@ def thumb_path(hist, rem, mtime, fmt):
     fn = base64.urlsafe_b64encode(h).decode("ascii")[:24]
 
     return "{}/th/{}/{}.{:x}.{}".format(
-        hist, rd, fn, int(mtime), "webp" if fmt == "w" else "jpg"
+        histpath, rd, fn, int(mtime), "webp" if fmt == "w" else "jpg"
     )
 
 
 class ThumbSrv(object):
-    def __init__(self, hub, vfs):
+    def __init__(self, hub):
         self.hub = hub
-        self.vols = [v.realpath for v in vfs.all_vols.values()]
-        self.hist = vfs.histtab
-
+        self.asrv = hub.asrv
         self.args = hub.args
         self.log_func = hub.log
 
@@ -154,8 +151,8 @@ class ThumbSrv(object):
             return not self.nthr
 
     def get(self, ptop, rem, mtime, fmt):
-        hist = self.hist[ptop]
-        tpath = thumb_path(hist, rem, mtime, fmt)
+        histpath = self.asrv.vfs.histtab[ptop]
+        tpath = thumb_path(histpath, rem, mtime, fmt)
         abspath = os.path.join(ptop, rem)
         cond = threading.Condition()
         with self.mutex:
@@ -331,29 +328,29 @@ class ThumbSrv(object):
         interval = self.args.th_clean
         while True:
             time.sleep(interval)
-            for vol, hist in self.hist.items():
-                if hist.startswith(vol):
-                    self.log("\033[Jcln {}/\033[A".format(hist))
+            for vol, histpath in self.asrv.vfs.histtab.items():
+                if histpath.startswith(vol):
+                    self.log("\033[Jcln {}/\033[A".format(histpath))
                 else:
-                    self.log("\033[Jcln {} ({})/\033[A".format(hist, vol))
+                    self.log("\033[Jcln {} ({})/\033[A".format(histpath, vol))
 
-                self.clean(hist)
+                self.clean(histpath)
 
             self.log("\033[Jcln ok")
 
-    def clean(self, hist):
-        # self.log("cln {}".format(hist))
+    def clean(self, histpath):
+        # self.log("cln {}".format(histpath))
         maxage = self.args.th_maxage
         now = time.time()
         prev_b64 = None
         prev_fp = None
         try:
-            ents = os.listdir(hist)
+            ents = os.listdir(histpath)
         except:
             return
 
         for f in sorted(ents):
-            fp = os.path.join(hist, f)
+            fp = os.path.join(histpath, f)
             cmp = fp.lower().replace("\\", "/")
 
             # "top" or b64 prefix/full (a folder)
diff --git a/copyparty/u2idx.py b/copyparty/u2idx.py
index c5091b5e..49f4a860 100644
--- a/copyparty/u2idx.py
+++ b/copyparty/u2idx.py
@@ -19,12 +19,11 @@ except:
 
 
 class U2idx(object):
-    def __init__(self, args, log_func, vfs):
-        self.args = args
-        self.log_func = log_func
-        self.vfs = vfs
-
-        self.timeout = args.srch_time
+    def __init__(self, conn):
+        self.log_func = conn.log_func
+        self.asrv = conn.asrv
+        self.args = conn.args
+        self.timeout = self.args.srch_time
 
         if not HAVE_SQLITE3:
             self.log("could not load sqlite3; searchign wqill be disabled")
@@ -62,7 +61,8 @@ class U2idx(object):
         if cur:
             return cur
 
-        db_path = os.path.join(self.vfs.histtab[ptop], "up2k.db")
+        histpath = self.asrv.vfs.histtab[ptop]
+        db_path = os.path.join(histpath, "up2k.db")
         if not os.path.exists(db_path):
             return None
 
diff --git a/copyparty/up2k.py b/copyparty/up2k.py
index 5418091f..2ba907a0 100644
--- a/copyparty/up2k.py
+++ b/copyparty/up2k.py
@@ -48,10 +48,9 @@ class Up2k(object):
         * ~/.config flatfiles for active jobs
     """
 
-    def __init__(self, hub, vfs):
+    def __init__(self, hub):
         self.hub = hub
-        self.vfs = vfs
-        # TODO stop passing around vfs, do auth or broker instead
+        self.asrv = hub.asrv
         self.args = hub.args
         self.log_func = hub.log
 
@@ -96,17 +95,17 @@ class Up2k(object):
             self.log("could not initialize sqlite3, will use in-memory registry only")
 
         if self.args.no_fastboot:
-            self.deferred_init(vfs.all_vols)
+            self.deferred_init()
         else:
             t = threading.Thread(
                 target=self.deferred_init,
-                args=(vfs.all_vols,),
                 name="up2k-deferred-init",
             )
             t.daemon = True
             t.start()
 
-    def deferred_init(self, all_vols):
+    def deferred_init(self):
+        all_vols = self.asrv.vfs.all_vols
         have_e2d = self.init_indexes(all_vols)
 
         if have_e2d:
@@ -300,7 +299,8 @@ class Up2k(object):
         return have_e2d
 
     def register_vpath(self, ptop, flags):
-        db_path = os.path.join(self.vfs.histtab[ptop], "up2k.db")
+        histpath = self.asrv.vfs.histtab[ptop]
+        db_path = os.path.join(histpath, "up2k.db")
         if ptop in self.registry:
             try:
                 return [self.cur[ptop], db_path]
@@ -320,7 +320,7 @@ class Up2k(object):
             self.log(" ".join(sorted(a)) + "\033[0m")
 
         reg = {}
-        path = os.path.join(self.vfs.histtab[ptop], "up2k.snap")
+        path = os.path.join(histpath, "up2k.snap")
         if "e2d" in flags and os.path.exists(path):
             with gzip.GzipFile(path, "rb") as f:
                 j = f.read().decode("utf-8")
@@ -344,7 +344,7 @@ class Up2k(object):
             return None
 
         try:
-            os.makedirs(self.vfs.histtab[ptop])
+            os.makedirs(histpath)
         except:
             pass
 
@@ -386,7 +386,7 @@ class Up2k(object):
 
     def _build_dir(self, dbw, top, excl, cdir, nohash):
         self.pp.msg = "a{} {}".format(self.pp.n, cdir)
-        histdir = self.vfs.histtab[top]
+        histpath = self.asrv.vfs.histtab[top]
         ret = 0
         g = statdir(self.log, not self.args.no_scandir, False, cdir)
         for iname, inf in sorted(g):
@@ -394,7 +394,7 @@ class Up2k(object):
             lmod = int(inf.st_mtime)
             sz = inf.st_size
             if stat.S_ISDIR(inf.st_mode):
-                if abspath in excl or abspath == histdir:
+                if abspath in excl or abspath == histpath:
                     continue
                 # self.log(" dir: {}".format(abspath))
                 ret += self._build_dir(dbw, top, excl, abspath, nohash)
@@ -1358,11 +1358,12 @@ class Up2k(object):
                 for k, reg in self.registry.items():
                     self._snap_reg(prev, k, reg, discard_interval)
 
-    def _snap_reg(self, prev, k, reg, discard_interval):
+    def _snap_reg(self, prev, ptop, reg, discard_interval):
         now = time.time()
+        histpath = self.asrv.vfs.histtab[ptop]
         rm = [x for x in reg.values() if now - x["poke"] > discard_interval]
         if rm:
-            m = "dropping {} abandoned uploads in {}".format(len(rm), k)
+            m = "dropping {} abandoned uploads in {}".format(len(rm), ptop)
             vis = [self._vis_job_progress(x) for x in rm]
             self.log("\n".join([m] + vis))
             for job in rm:
@@ -1380,21 +1381,21 @@ class Up2k(object):
                 except:
                     pass
 
-        path = os.path.join(self.vfs.histtab[k], "up2k.snap")
+        path = os.path.join(histpath, "up2k.snap")
         if not reg:
-            if k not in prev or prev[k] is not None:
-                prev[k] = None
+            if ptop not in prev or prev[ptop] is not None:
+                prev[ptop] = None
                 if os.path.exists(fsenc(path)):
                     os.unlink(fsenc(path))
             return
 
         newest = max(x["poke"] for _, x in reg.items()) if reg else 0
         etag = [len(reg), newest]
-        if etag == prev.get(k, None):
+        if etag == prev.get(ptop, None):
             return
 
         try:
-            os.makedirs(self.vfs.histtab[k])
+            os.makedirs(histpath)
         except:
             pass
 
@@ -1406,7 +1407,7 @@ class Up2k(object):
         atomic_move(path2, path)
 
         self.log("snap: {} |{}|".format(path, len(reg.keys())))
-        prev[k] = etag
+        prev[ptop] = etag
 
     def _tagger(self):
         while True:
diff --git a/scripts/test/smoketest.py b/scripts/test/smoketest.py
index 4abea8df..3ff06082 100644
--- a/scripts/test/smoketest.py
+++ b/scripts/test/smoketest.py
@@ -11,6 +11,7 @@ import subprocess as sp
 
 class Cpp(object):
     def __init__(self, args):
+        self.ls_pre = set(list(os.listdir()))
         self.p = sp.Popen([sys.executable, "-m", "copyparty"] + args)
         # , stdout=sp.PIPE, stderr=sp.PIPE)
 
@@ -27,27 +28,29 @@ class Cpp(object):
         if wait:
             self.t.join()
 
-
-def main():
-    t1 = set(list(os.listdir()))
-    try:
-        main2()
-    finally:
-        t2 = os.listdir()
-        for f in t2:
-            if f not in t1 and f.startswith("up."):
+    def clean(self):
+        t = os.listdir()
+        for f in t:
+            if f not in self.ls_pre and f.startswith("up."):
                 os.unlink(f)
 
 
-def main2():
+def tc1():
     ub = "http://127.0.0.1:4321/"
     td = os.path.join("srv", "smoketest")
     try:
         shutil.rmtree(td)
     except:
-        pass
+        if os.path.exists(td):
+            raise
 
-    os.mkdir(td)
+    for _ in range(10):
+        try:
+            os.mkdir(td)
+        except:
+            time.sleep(0.1)  # win10
+
+    assert os.path.exists(td)
 
     vidp = os.path.join(tempfile.gettempdir(), "smoketest.h264")
     if not os.path.exists(vidp):
@@ -57,8 +60,17 @@ def main2():
     with open(vidp, "rb") as f:
         ovid = f.read()
 
-    args = ["-p", "4321", "-e2dsa", "-e2tsr", "--th-ff-jpg"]
+    args = [
+        "-p",
+        "4321",
+        "-e2dsa",
+        "-e2tsr",
+        "--th-ff-jpg",
+        "--hist",
+        os.path.join(td, "dbm"),
+    ]
     pdirs = []
+    hpaths = {}
 
     for d1 in ["r", "w", "a"]:
         pdirs.append("{}/{}".format(td, d1))
@@ -72,8 +84,19 @@ def main2():
     udirs = [x.split("/", 2)[2] for x in pdirs]
     perms = [x.rstrip("j/")[-1] for x in pdirs]
     for pd, ud, p in zip(pdirs, udirs, perms):
-        # args += ["-v", "{}:{}:{}".format(d.split("/", 1)[1], d, d[-1])]
-        args += ["-v", "{}:{}:{}".format(pd, ud, p)]
+        if ud[-1] == "j":
+            continue
+
+        hp = None
+        if pd.endswith("st/a"):
+            hp = os.path.join(td, "db1")
+        elif pd[:-1].endswith("a/j/"):
+            hp = os.path.join(td, "dbm")
+        else:
+            hp = "-"
+
+        hpaths[ud] = os.path.join(pd, ".hist") if hp == "-" else hp
+        args += ["-v", "{}:{}:{}:chist={}".format(pd, ud, p, hp)]
 
     # print(repr(args))
     # return
@@ -98,6 +121,9 @@ def main2():
         except:
             pass
 
+    cpp.clean()
+
+    # GET permission
     for d, p in zip(udirs, perms):
         u = "{}{}/a.h264".format(ub, d)
         r = requests.get(u)
@@ -105,12 +131,14 @@ def main2():
         if ok != (p in ["a"]):
             raise Exception("get {} with perm {} at {}".format(ok, p, u))
 
+    # stat filesystem
     for d, p in zip(pdirs, perms):
         u = "{}/a.h264".format(d)
         ok = os.path.exists(u)
         if ok != (p in ["a", "w"]):
             raise Exception("stat {} with perm {} at {}".format(ok, p, u))
 
+    # GET thumbnail, vreify contents
     for d, p in zip(udirs, perms):
         u = "{}{}/a.h264?th=j".format(ub, d)
         r = requests.get(u)
@@ -118,8 +146,34 @@ def main2():
         if ok != (p in ["a"]):
             raise Exception("thumb {} with perm {} at {}".format(ok, p, u))
 
+    # check tags
+    for d, p in zip(udirs, perms):
+        u = "{}{}?ls".format(ub, d)
+        r = requests.get(u)
+        j = r.json() if r else False
+        tag = None
+        if j:
+            for f in j["files"]:
+                tag = tag or f["tags"].get("res")
+
+        r_ok = bool(j)
+        w_ok = bool(r_ok and j.get("files"))
+
+        if not r_ok or w_ok != (p in ["a"]):
+            raise Exception("ls {} with perm {} at {}".format(ok, p, u))
+
+        if (tag and p != "a") or (not tag and p == "a"):
+            raise Exception("tag {} with perm {} at {}".format(ok, p, u))
+
+        if tag is not None and tag != "48x32":
+            raise Exception("tag [{}] at {}".format(tag, u))
+
     cpp.stop(True)
 
 
+def main():
+    tc1()
+
+
 if __name__ == "__main__":
     main()
diff --git a/tests/test_httpcli.py b/tests/test_httpcli.py
index f3bfdcda..007f00e3 100644
--- a/tests/test_httpcli.py
+++ b/tests/test_httpcli.py
@@ -101,7 +101,7 @@ class TestHttpCli(unittest.TestCase):
             pprint.pprint(vcfg)
 
             self.args = Cfg(v=vcfg, a=["o:o", "x:x"])
-            self.auth = AuthSrv(self.args, self.log)
+            self.asrv = AuthSrv(self.args, self.log)
             vfiles = [x for x in allfiles if x.startswith(top)]
             for fp in vfiles:
                 rok, wok = self.can_rw(fp)
@@ -190,12 +190,12 @@ class TestHttpCli(unittest.TestCase):
     def put(self, url):
         buf = "PUT /{0} HTTP/1.1\r\nCookie: cppwd=o\r\nConnection: close\r\nContent-Length: {1}\r\n\r\nok {0}\n"
         buf = buf.format(url, len(url) + 4).encode("utf-8")
-        conn = tu.VHttpConn(self.args, self.auth, self.log, buf)
+        conn = tu.VHttpConn(self.args, self.asrv, self.log, buf)
         HttpCli(conn).run()
         return conn.s._reply.decode("utf-8").split("\r\n\r\n", 1)
 
     def curl(self, url, binary=False):
-        conn = tu.VHttpConn(self.args, self.auth, self.log, hdr(url))
+        conn = tu.VHttpConn(self.args, self.asrv, self.log, hdr(url))
         HttpCli(conn).run()
         if binary:
             h, b = conn.s._reply.split(b"\r\n\r\n", 1)
diff --git a/tests/util.py b/tests/util.py
index 1c6dd706..e04c31ce 100644
--- a/tests/util.py
+++ b/tests/util.py
@@ -110,12 +110,12 @@ class VHttpSrv(object):
 
 
 class VHttpConn(object):
-    def __init__(self, args, auth, log, buf):
+    def __init__(self, args, asrv, log, buf):
         self.s = VSock(buf)
         self.sr = Unrecv(self.s)
         self.addr = ("127.0.0.1", "42069")
         self.args = args
-        self.auth = auth
+        self.asrv = asrv
         self.is_mp = False
         self.log_func = log
         self.log_src = "a"