fuse: cache options

bin/copyparty-fuse.py

@@ -194,7 +194,7 @@ class Gateway(object):
     def download_file_range(self, path, ofs1, ofs2):
         web_path = self.quotep("/" + "/".join([self.web_root, path])) + "?raw"
         hdr_range = "bytes={}-{}".format(ofs1, ofs2 - 1)
-        log("downloading {}".format(hdr_range))
+        log("downloading {:4.0f}K, {}".format((ofs2 - ofs1) / 1024.0, hdr_range))
 
         r = self.sendreq("GET", web_path, headers={"Range": hdr_range})
         if r.status != http.client.PARTIAL_CONTENT:
@@ -277,9 +277,11 @@ class Gateway(object):
 
 
 class CPPF(Operations):
-    def __init__(self, base_url):
+    def __init__(self, base_url, dircache, filecache):
         self.gw = Gateway(base_url)
         self.junk_fh_ctr = 3
+        self.n_dircache = dircache
+        self.n_filecache = filecache
 
         self.dircache = []
         self.dircache_mtx = threading.Lock()
@@ -289,12 +291,22 @@ class CPPF(Operations):
 
         info("up")
 
+    def _describe(self):
+        msg = ""
+        for n, cn in enumerate(self.filecache):
+            cache_path, cache1 = cn.tag
+            cache2 = cache1 + len(cn.data)
+            msg += "\n#{} = {}  {}  {}\n{}\n".format(
+                n, cache1, len(cn.data), cache2, cache_path
+            )
+        return msg
+
     def clean_dircache(self):
         """not threadsafe"""
         now = time.time()
         cutoff = 0
         for cn in self.dircache:
-            if now - cn.ts > 1:
+            if now - cn.ts > self.n_dircache:
                 cutoff += 1
             else:
                 break
@@ -411,7 +423,18 @@ class CPPF(Operations):
 
                     continue
 
-                raise Exception("what")
+                msg = "cache fallthrough\n{} {} {}\n{} {} {}\n{} {} --\n".format(
+                    get1,
+                    get2,
+                    get2 - get1,
+                    cache1,
+                    cache2,
+                    cache2 - cache1,
+                    get1 - cache1,
+                    get2 - cache2,
+                )
+                msg += self._describe()
+                raise Exception(msg)
 
         if car and cdr:
             dbg("<cache> have both")
@@ -420,7 +443,9 @@ class CPPF(Operations):
             if len(ret) == get2 - get1:
                 return ret
 
-            raise Exception("{} + {} != {} - {}".format(len(car), len(cdr), get2, get1))
+            msg = "{} + {} != {} - {}".format(len(car), len(cdr), get2, get1)
+            msg += self._describe()
+            raise Exception(msg)
 
         elif cdr:
             h_end = get1 + (get2 - get1) - len(cdr)
@@ -482,18 +507,20 @@ class CPPF(Operations):
 
         cn = CacheNode([path, h_ofs], buf)
         with self.filecache_mtx:
-            if len(self.filecache) > 6:
+            if len(self.filecache) >= self.n_filecache:
                 self.filecache = self.filecache[1:] + [cn]
             else:
                 self.filecache.append(cn)
 
         return ret
 
-    def readdir(self, path, fh=None):
+    def _readdir(self, path, fh=None):
         path = path.strip("/")
         log("readdir [{}] [{}]".format(path, fh))
 
         ret = self.gw.listdir(path)
+        if not self.n_dircache:
+            return ret
 
         with self.dircache_mtx:
             cn = CacheNode(path, ret)
@@ -502,6 +529,9 @@ class CPPF(Operations):
 
         return ret
 
+    def readdir(self, path, fh=None):
+        return [".", ".."] + self._readdir(path, fh)
+
     def read(self, path, length, offset, fh=None):
         path = path.strip("/")
 
@@ -516,9 +546,10 @@ class CPPF(Operations):
         if file_sz == 0 or offset >= ofs2:
             return b""
 
-        # toggle cache here i suppose
+        if not self.n_filecache:
-        # return self.get_cached_file(path, offset, ofs2, file_sz)
+            return self.gw.download_file_range(path, offset, ofs2)
-        return self.gw.download_file_range(path, offset, ofs2)
+
+        return self.get_cached_file(path, offset, ofs2, file_sz)
 
     def getattr(self, path, fh=None):
         log("getattr [{}]".format(path))
@@ -532,7 +563,7 @@ class CPPF(Operations):
 
         if not path:
             ret = self.gw.stat_dir(time.time())
-            dbg("=" + repr(ret))
+            # dbg("=" + repr(ret))
             return ret
 
         cn = self.get_cached_dir(dirpath)
@@ -541,11 +572,11 @@ class CPPF(Operations):
             dents = cn.data
         else:
             log("cache miss")
-            dents = self.readdir(dirpath)
+            dents = self._readdir(dirpath)
 
         for cache_name, cache_stat, _ in dents:
             if cache_name == fname:
-                dbg("=" + repr(cache_stat))
+                # dbg("=" + repr(cache_stat))
                 return cache_stat
 
         log("=404 ({})".format(path))
@@ -646,7 +677,9 @@ class CPPF(Operations):
 
 def main():
     try:
-        local, remote = sys.argv[1:]
+        local, remote = sys.argv[1:3]
+        filecache = 7 if len(sys.argv) <= 3 else int(sys.argv[3])
+        dircache = 1 if len(sys.argv) <= 4 else float(sys.argv[4])
     except:
         where = "local directory"
         if WINDOWS:
@@ -654,6 +687,8 @@ def main():
 
         print("need arg 1: " + where)
         print("need arg 2: root url")
+        print("optional 3: num files in filecache (7)")
+        print("optional 4: num seconds / dircache (1)")
         print()
         print("example:")
         print("  copyparty-fuse.py ./music http://192.168.1.69:3923/music/")
@@ -666,7 +701,7 @@ def main():
         os.system("")
 
     FUSE(
-        CPPF(remote),
+        CPPF(remote, dircache, filecache),
         local,
         foreground=True,
         nothreads=True,

scripts/fusefuzz.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+
+import os
+
+"""
+mkdir -p /dev/shm/fusefuzz/{r,v}
+PYTHONPATH=.. python3 -m copyparty -v /dev/shm/fusefuzz/r::r -i 127.0.0.1
+../bin/copyparty-fuse.py /dev/shm/fusefuzz/v http://127.0.0.1:3923/ 2 0
+(d="$PWD"; cd /dev/shm/fusefuzz && "$d"/fusefuzz.py)
+"""
+
+
+def main():
+    for n in range(5):
+        with open(f"r/{n}", "wb") as f:
+            f.write(b"h" * n)
+
+    for fsz in range(1024 * 1024 * 2 - 3, 1024 * 1024 * 2 + 3):
+        with open("r/f", "wb", fsz) as f:
+            f.write(b"\xab" * fsz)
+
+        for rsz in range(62, 66):
+            ofslist = [0, 1, 2]
+            for n in range(3):
+                ofslist.append(fsz - n)
+                ofslist.append(fsz - (rsz * 1 + n))
+                ofslist.append(fsz - (rsz * 2 + n))
+
+            for ofs0 in ofslist:
+                for shift in range(-3, 3):
+                    print(f"fsz {fsz} rsz {rsz} ofs {ofs0} shift {shift}")
+                    ofs = ofs0
+                    if ofs < 0 or ofs >= fsz:
+                        continue
+
+                    for n in range(1, 3):
+                        with open(f"v/{n}", "rb") as f:
+                            f.read()
+
+                    prev_ofs = -99
+                    with open("r/f", "rb", rsz) as rf:
+                        with open("v/f", "rb", rsz) as vf:
+                            while True:
+                                ofs += shift
+                                if ofs < 0 or ofs > fsz or ofs == prev_ofs:
+                                    break
+
+                                prev_ofs = ofs
+
+                                if ofs != rf.tell():
+                                    rf.seek(ofs)
+                                    vf.seek(ofs)
+
+                                rb = rf.read(rsz)
+                                vb = vf.read(rsz)
+
+                                print(
+                                    f"fsz {fsz} rsz {rsz} ofs {ofs0} shift {shift} ofs {ofs} = {len(rb)}"
+                                )
+
+                                if rb != vb:
+                                    raise Exception(f"{len(rb)} != {len(vb)}")
+
+                                if not rb:
+                                    break
+
+                                ofs += len(rb)
+
+
+if __name__ == "__main__":
+    main()