detect sparse support from st_blocks

copyparty/__main__.py

@@ -481,7 +481,6 @@ def run_argparse(argv: list[str], formatter: Any) -> argparse.Namespace:
     ap2.add_argument("--hardlink", action="store_true", help="prefer hardlinks instead of symlinks when possible (within same filesystem)")
     ap2.add_argument("--never-symlink", action="store_true", help="do not fallback to symlinks when a hardlink cannot be made")
     ap2.add_argument("--no-dedup", action="store_true", help="disable symlink/hardlink creation; copy file contents instead")
-    ap2.add_argument("--thickfs", metavar="REGEX", type=u, default="fat|vfat|ex.?fat|hpfs|fuse|sdcardfs", help="filesystems which dont support sparse files")
     ap2.add_argument("--sparse", metavar="MiB", type=int, default=4, help="windows-only: minimum size of incoming uploads through up2k before they are made into sparse files")
     ap2.add_argument("--turbo", metavar="LVL", type=int, default=0, help="configure turbo-mode in up2k client; 0 = off and warn if enabled, 1 = off, 2 = on, 3 = on and disable datecheck")
 

copyparty/fsutil.py

@@ -29,7 +29,7 @@ class Fstab(object):
         self.log_func("fstab", msg + "\033[K", c)
 
     def get(self, path: str):
-        if time.time() - self.age > 600 or len(self.cache) > 9000:
+        if len(self.cache) > 9000:
             self.age = time.time()
             self.tab = None
             self.cache = {}
@@ -89,6 +89,16 @@ class Fstab(object):
 
         self.tab = tab
 
+    def relabel(self, path: str, nval: str):
+        ptn = re.compile(r"^[^\\/]*")
+        vn, _ = self.tab._find(path)
+        visit = [vn]
+        while visit:
+            vn = visit.pop()
+            vn.realpath = ptn.sub(nval, vn.realpath)
+            visit.extend(list(vn.nodes.values()))
+        self.cache = {}
+
     def get_unix(self, path: str):
         if not self.tab:
             self.build_tab()

copyparty/up2k.py

@@ -132,7 +132,6 @@ class Up2k(object):
             thr.start()
 
         self.fstab = Fstab(self.log_func)
-        self.no_sparse = re.compile(self.args.thickfs)
 
         if self.args.no_fastboot:
             self.deferred_init()
@@ -1327,7 +1326,7 @@ class Up2k(object):
 
         # check if filesystem supports sparse files;
         # refuse out-of-order / multithreaded uploading if sprs False
-        sprs = not self.no_sparse.match(self.fstab.get(pdir))
+        sprs = self.fstab.get(pdir) != "ng"
 
         with self.mutex:
             cur = self.cur.get(cj["ptop"])
@@ -2214,31 +2213,49 @@ class Up2k(object):
 
         dip = job["addr"].replace(":", ".")
         suffix = "-{:.6f}-{}".format(job["t0"], dip)
-        t0 = time.time()
         with ren_open(tnam, "wb", fdir=pdir, suffix=suffix) as zfw:
             f, job["tnam"] = zfw["orz"]
+            abspath = os.path.join(pdir, job["tnam"])
+            sprs = job["sprs"]
+            sz = job["size"]
+            relabel = False
             if (
                 ANYWIN
-                and job["sprs"]
+                and sprs
                 and self.args.sparse
-                and self.args.sparse * 1024 * 1024 <= job["size"]
+                and self.args.sparse * 1024 * 1024 <= sz
             ):
-                fp = os.path.join(pdir, job["tnam"])
                 try:
-                    sp.check_call(["fsutil", "sparse", "setflag", fp])
+                    sp.check_call(["fsutil", "sparse", "setflag", abspath])
                 except:
-                    self.log("could not sparse [{}]".format(fp), 3)
+                    self.log("could not sparse [{}]".format(abspath), 3)
+                    relabel = True
+                    sprs = False
 
-            if job["hash"] and job["sprs"]:
-                f.seek(job["size"] - 1)
+            if not ANYWIN and sprs and sz > 1024 * 1024:
+                fs = self.fstab.get(pdir)
+                if fs != "ok":
+                    relabel = True
+                    f.seek(1024 * 1024 - 1)
+                    f.write(b"e")
+                    f.flush()
+                    try:
+                        nblk = bos.stat(abspath).st_blocks
+                        sprs = nblk < 2048
+                    except:
+                        sprs = False
+
+            if relabel:
+                t = "sparse files {} on {} filesystem at {}"
+                nv = "ok" if sprs else "ng"
+                self.log(t.format(nv, self.fstab.get(pdir), pdir))
+                self.fstab.relabel(pdir, nv)
+                job["sprs"] = sprs
+
+            if job["hash"] and sprs:
+                f.seek(sz - 1)
                 f.write(b"e")
 
-        td = time.time() - t0
-        if td > 3 and not ANYWIN:
-            t = "WARNING: filesystem [{}] at [{}] probably does support sparse files; adjust the list in --thickfs and maybe create a github issue (please mention the filesystem + any related info about your setup if you do)"
-            fs = self.fstab.get(pdir)
-            self.log(t.format(fs, pdir), 1)
-
         if not job["hash"]:
             self._finish_upload(job["ptop"], job["wark"])