fix race

.vscode/launch.py

@@ -6,10 +6,14 @@
 import re
 import os
 import sys
+
+print(sys.executable)
+
 import shlex
 import jstyleson
 import subprocess as sp
 
+
 with open(".vscode/launch.json", "r", encoding="utf-8") as f:
     tj = f.read()
 

copyparty/authsrv.py

@@ -43,9 +43,9 @@ class VFS(object):
         )
 
     def get_all_vols(self, outdict):
+        outdict[self.vpath] = self
         for v in self.nodes.values():
             v.get_all_vols(outdict)
-            outdict[v.vpath] = v
 
     def add(self, src, dst):
         """get existing, or add new path to the vfs"""

copyparty/up2k.py

@@ -188,25 +188,27 @@ class Up2k(object):
                 self.log(msg, c=3)
 
         live_vols = []
-        for vol in vols:
+        with self.mutex:
-            try:
+            # only need to protect register_vpath but all in one go feels right
-                os.listdir(vol.realpath)
+            for vol in vols:
-            except:
+                try:
-                self.volstate[vol.vpath] = "OFFLINE (cannot access folder)"
+                    os.listdir(vol.realpath)
-                self.log("cannot access " + vol.realpath, c=1)
+                except:
-                continue
+                    self.volstate[vol.vpath] = "OFFLINE (cannot access folder)"
+                    self.log("cannot access " + vol.realpath, c=1)
+                    continue
 
-            if scan_vols and vol.vpath not in scan_vols:
+                if scan_vols and vol.vpath not in scan_vols:
-                continue
+                    continue
 
-            if not self.register_vpath(vol.realpath, vol.flags):
+                if not self.register_vpath(vol.realpath, vol.flags):
-                # self.log("db not enabled for {}".format(m, vol.realpath))
+                    # self.log("db not enable for {}".format(m, vol.realpath))
-                continue
+                    continue
 
-            live_vols.append(vol)
+                live_vols.append(vol)
 
-            if vol.vpath not in self.volstate:
+                if vol.vpath not in self.volstate:
-                self.volstate[vol.vpath] = "OFFLINE (pending initialization)"
+                    self.volstate[vol.vpath] = "OFFLINE (pending initialization)"
 
         vols = live_vols
         need_vac = {}
@@ -987,9 +989,10 @@ class Up2k(object):
         return self._orz(db_path)
 
     def handle_json(self, cj):
-        if not self.register_vpath(cj["ptop"], cj["vcfg"]):
+        with self.mutex:
-            if cj["ptop"] not in self.registry:
+            if not self.register_vpath(cj["ptop"], cj["vcfg"]):
-                raise Pebkac(410, "location unavailable")
+                if cj["ptop"] not in self.registry:
+                    raise Pebkac(410, "location unavailable")
 
         cj["name"] = sanitize_fn(cj["name"], bad=[".prologue.html", ".epilogue.html"])
         cj["poke"] = time.time()
@@ -1411,6 +1414,7 @@ class Up2k(object):
             if "e2t" not in self.flags[ptop]:
                 continue
 
+            # self.log("\n  " + repr([ptop, rd, fn]))
             abspath = os.path.join(ptop, rd, fn)
             tags = self.mtag.get(abspath)
             ntags1 = len(tags)
@@ -1450,8 +1454,9 @@ class Up2k(object):
                 self.idx_wark(ptop, wark, rd, fn, inf.st_mtime, inf.st_size)
 
     def hash_file(self, ptop, flags, rd, fn):
-        self.register_vpath(ptop, flags)
+        with self.mutex:
-        self.hashq.put([ptop, rd, fn])
+            self.register_vpath(ptop, flags)
+            self.hashq.put([ptop, rd, fn])
 
 
 def up2k_chunksize(filesize):

scripts/test/smoketest.py

@@ -29,6 +29,17 @@ class Cpp(object):
 
 
 def main():
+    t1 = set(list(os.listdir()))
+    try:
+        main2()
+    finally:
+        t2 = os.listdir()
+        for f in t2:
+            if f not in t1 and f.startswith("up."):
+                os.unlink(f)
+
+
+def main2():
     ub = "http://127.0.0.1:4321/"
     td = os.path.join("srv", "smoketest")
     try:
@@ -54,15 +65,18 @@ def main():
         pdirs.append("{}/{}/j".format(td, d1))
         for d2 in ["r", "w", "a"]:
             d = os.path.join(td, d1, "j", d2)
-            pdirs.append(d.replace("\\", "/"))
+            pdirs.append(d)
             os.makedirs(d)
 
+    pdirs = [x.replace("\\", "/") for x in pdirs]
     udirs = [x.split("/", 2)[2] for x in pdirs]
     perms = [x.rstrip("j/")[-1] for x in pdirs]
     for pd, ud, p in zip(pdirs, udirs, perms):
         # args += ["-v", "{}:{}:{}".format(d.split("/", 1)[1], d, d[-1])]
         args += ["-v", "{}:{}:{}".format(pd, ud, p)]
 
+    # print(repr(args))
+    # return
     cpp = Cpp(args)
 
     up = False
@@ -79,7 +93,10 @@ def main():
 
     for d in udirs:
         vid = ovid + "\n{}".format(d).encode("utf-8")
-        requests.post(ub + d, data={"act": "bput"}, files={"f": ("a.h264", vid)})
+        try:
+            requests.post(ub + d, data={"act": "bput"}, files={"f": ("a.h264", vid)})
+        except:
+            pass
 
     for d, p in zip(udirs, perms):
         u = "{}{}/a.h264".format(ub, d)