diff --git a/copyparty/__main__.py b/copyparty/__main__.py index 7a2206c1..310db30e 100644 --- a/copyparty/__main__.py +++ b/copyparty/__main__.py @@ -481,7 +481,6 @@ def run_argparse(argv: list[str], formatter: Any) -> argparse.Namespace: ap2.add_argument("--hardlink", action="store_true", help="prefer hardlinks instead of symlinks when possible (within same filesystem)") ap2.add_argument("--never-symlink", action="store_true", help="do not fallback to symlinks when a hardlink cannot be made") ap2.add_argument("--no-dedup", action="store_true", help="disable symlink/hardlink creation; copy file contents instead") - ap2.add_argument("--thickfs", metavar="REGEX", type=u, default="fat|vfat|ex.?fat|hpfs|fuse|sdcardfs", help="filesystems which dont support sparse files") ap2.add_argument("--sparse", metavar="MiB", type=int, default=4, help="windows-only: minimum size of incoming uploads through up2k before they are made into sparse files") ap2.add_argument("--turbo", metavar="LVL", type=int, default=0, help="configure turbo-mode in up2k client; 0 = off and warn if enabled, 1 = off, 2 = on, 3 = on and disable datecheck") diff --git a/copyparty/fsutil.py b/copyparty/fsutil.py index 02036c8b..cc68733c 100644 --- a/copyparty/fsutil.py +++ b/copyparty/fsutil.py @@ -29,7 +29,7 @@ class Fstab(object): self.log_func("fstab", msg + "\033[K", c) def get(self, path: str): - if time.time() - self.age > 600 or len(self.cache) > 9000: + if len(self.cache) > 9000: self.age = time.time() self.tab = None self.cache = {} @@ -89,6 +89,16 @@ class Fstab(object): self.tab = tab + def relabel(self, path: str, nval: str): + ptn = re.compile(r"^[^\\/]*") + vn, _ = self.tab._find(path) + visit = [vn] + while visit: + vn = visit.pop() + vn.realpath = ptn.sub(nval, vn.realpath) + visit.extend(list(vn.nodes.values())) + self.cache = {} + def get_unix(self, path: str): if not self.tab: self.build_tab() diff --git a/copyparty/up2k.py b/copyparty/up2k.py index 98abddad..bde9e84c 100644 --- a/copyparty/up2k.py +++ b/copyparty/up2k.py @@ -132,7 +132,6 @@ class Up2k(object): thr.start() self.fstab = Fstab(self.log_func) - self.no_sparse = re.compile(self.args.thickfs) if self.args.no_fastboot: self.deferred_init() @@ -1327,7 +1326,7 @@ class Up2k(object): # check if filesystem supports sparse files; # refuse out-of-order / multithreaded uploading if sprs False - sprs = not self.no_sparse.match(self.fstab.get(pdir)) + sprs = self.fstab.get(pdir) != "ng" with self.mutex: cur = self.cur.get(cj["ptop"]) @@ -2214,31 +2213,49 @@ class Up2k(object): dip = job["addr"].replace(":", ".") suffix = "-{:.6f}-{}".format(job["t0"], dip) - t0 = time.time() with ren_open(tnam, "wb", fdir=pdir, suffix=suffix) as zfw: f, job["tnam"] = zfw["orz"] + abspath = os.path.join(pdir, job["tnam"]) + sprs = job["sprs"] + sz = job["size"] + relabel = False if ( ANYWIN - and job["sprs"] + and sprs and self.args.sparse - and self.args.sparse * 1024 * 1024 <= job["size"] + and self.args.sparse * 1024 * 1024 <= sz ): - fp = os.path.join(pdir, job["tnam"]) try: - sp.check_call(["fsutil", "sparse", "setflag", fp]) + sp.check_call(["fsutil", "sparse", "setflag", abspath]) except: - self.log("could not sparse [{}]".format(fp), 3) + self.log("could not sparse [{}]".format(abspath), 3) + relabel = True + sprs = False - if job["hash"] and job["sprs"]: - f.seek(job["size"] - 1) + if not ANYWIN and sprs and sz > 1024 * 1024: + fs = self.fstab.get(pdir) + if fs != "ok": + relabel = True + f.seek(1024 * 1024 - 1) + f.write(b"e") + f.flush() + try: + nblk = bos.stat(abspath).st_blocks + sprs = nblk < 2048 + except: + sprs = False + + if relabel: + t = "sparse files {} on {} filesystem at {}" + nv = "ok" if sprs else "ng" + self.log(t.format(nv, self.fstab.get(pdir), pdir)) + self.fstab.relabel(pdir, nv) + job["sprs"] = sprs + + if job["hash"] and sprs: + f.seek(sz - 1) f.write(b"e") - td = time.time() - t0 - if td > 3 and not ANYWIN: - t = "WARNING: filesystem [{}] at [{}] probably does support sparse files; adjust the list in --thickfs and maybe create a github issue (please mention the filesystem + any related info about your setup if you do)" - fs = self.fstab.get(pdir) - self.log(t.format(fs, pdir), 1) - if not job["hash"]: self._finish_upload(job["ptop"], job["wark"])