configurable list of sparse-supporting filesystems +

close nonsparse files after each write to force flush

copyparty/__main__.py

@@ -474,14 +474,15 @@ def run_argparse(argv: list[str], formatter: Any) -> argparse.Namespace:
 
     ap2 = ap.add_argument_group('upload options')
     ap2.add_argument("--dotpart", action="store_true", help="dotfile incomplete uploads, hiding them from clients unless -ed")
-    ap2.add_argument("--sparse", metavar="MiB", type=int, default=4, help="windows-only: minimum size of incoming uploads through up2k before they are made into sparse files")
     ap2.add_argument("--unpost", metavar="SEC", type=int, default=3600*12, help="grace period where uploads can be deleted by the uploader, even without delete permissions; 0=disabled")
+    ap2.add_argument("--reg-cap", metavar="N", type=int, default=38400, help="max number of uploads to keep in memory when running without -e2d; roughly 1 MiB RAM per 600")
     ap2.add_argument("--no-fpool", action="store_true", help="disable file-handle pooling -- instead, repeatedly close and reopen files during upload")
     ap2.add_argument("--use-fpool", action="store_true", help="force file-handle pooling, even if copyparty thinks you're better off without -- probably useful on nfs and cow filesystems (zfs, btrfs)")
     ap2.add_argument("--hardlink", action="store_true", help="prefer hardlinks instead of symlinks when possible (within same filesystem)")
     ap2.add_argument("--never-symlink", action="store_true", help="do not fallback to symlinks when a hardlink cannot be made")
     ap2.add_argument("--no-dedup", action="store_true", help="disable symlink/hardlink creation; copy file contents instead")
-    ap2.add_argument("--reg-cap", metavar="N", type=int, default=38400, help="max number of uploads to keep in memory when running without -e2d; roughly 1 MiB RAM per 600")
+    ap2.add_argument("--thickfs", metavar="REGEX", type=u, default="fat|vfat|ex.?fat|hpfs|fuse", help="filesystems which dont support sparse files")
+    ap2.add_argument("--sparse", metavar="MiB", type=int, default=4, help="windows-only: minimum size of incoming uploads through up2k before they are made into sparse files")
     ap2.add_argument("--turbo", metavar="LVL", type=int, default=0, help="configure turbo-mode in up2k client; 0 = off and warn if enabled, 1 = off, 2 = on, 3 = on and disable datecheck")
 
     ap2 = ap.add_argument_group('network options')

copyparty/fsutil.py

@@ -21,25 +21,6 @@ class Fstab(object):
     def __init__(self, log: RootLogger):
         self.log_func = log
 
-        self.no_sparse = set(
-            [
-                "fuse",  # termux-sdcard
-                "vfat",  # linux-efi
-                "fat32",
-                "fat16",
-                "fat12",
-                "fat-32",
-                "fat-16",
-                "fat-12",
-                "fat 32",
-                "fat 16",
-                "fat 12",
-                "exfat",
-                "ex-fat",
-                "ex fat",
-                "hpfs",  # macos
-            ]
-        )
         self.tab: Optional[VFS] = None
         self.cache: dict[str, str] = {}
         self.age = 0.0

copyparty/httpcli.py

@@ -1048,7 +1048,7 @@ class HttpCli(object):
             reader = read_socket(self.sr, remains)
 
             f = None
-            fpool = not self.args.no_fpool and (not ANYWIN or sprs)
+            fpool = not self.args.no_fpool and sprs
             if fpool:
                 with self.mutex:
                     try:

copyparty/up2k.py

@@ -132,6 +132,7 @@ class Up2k(object):
             thr.start()
 
         self.fstab = Fstab(self.log_func)
+        self.no_sparse = re.compile(self.args.thickfs)
 
         if self.args.no_fastboot:
             self.deferred_init()
@@ -1326,7 +1327,7 @@ class Up2k(object):
 
         # check if filesystem supports sparse files;
         # refuse out-of-order / multithreaded uploading if sprs False
-        sprs = self.fstab.get(pdir) not in self.fstab.no_sparse
+        sprs = not self.no_sparse.match(self.fstab.get(pdir))
 
         with self.mutex:
             cur = self.cur.get(cj["ptop"])
@@ -1359,7 +1360,7 @@ class Up2k(object):
                         "prel": dp_dir,
                         "vtop": cj["vtop"],
                         "ptop": cj["ptop"],
-                        "sprs": sprs,
+                        "sprs": sprs,  # dontcare; finished anyways
                         "size": dsize,
                         "lmod": dtime,
                         "addr": ip,
@@ -1494,7 +1495,7 @@ class Up2k(object):
                 "purl": purl,
                 "size": job["size"],
                 "lmod": job["lmod"],
-                "sprs": sprs,
+                "sprs": job.get("sprs", sprs),
                 "hash": job["need"],
                 "wark": wark,
             }
@@ -2213,6 +2214,7 @@ class Up2k(object):
 
         dip = job["addr"].replace(":", ".")
         suffix = "-{:.6f}-{}".format(job["t0"], dip)
+        t0 = time.time()
         with ren_open(tnam, "wb", fdir=pdir, suffix=suffix) as zfw:
             f, job["tnam"] = zfw["orz"]
             if (
@@ -2231,6 +2233,12 @@ class Up2k(object):
                 f.seek(job["size"] - 1)
                 f.write(b"e")
 
+        td = time.time() - t0
+        if td > 3 and not ANYWIN:
+            t = "WARNING: filesystem [{}] at [{}] probably does support sparse files; adjust the list in --thickfs and maybe create a github issue (please mention the filesystem + any related info about your setup if you do)"
+            fs = self.fstab.get(pdir)
+            self.log(t.format(fs, pdir), 1)
+
         if not job["hash"]:
             self._finish_upload(job["ptop"], job["wark"])