From 505a8fc6f696b4721637e03ace6453ccfcd74711 Mon Sep 17 00:00:00 2001 From: ed Date: Wed, 21 Apr 2021 18:32:21 +0200 Subject: [PATCH] up2k: sparse alloc on windows --- copyparty/__main__.py | 1 + copyparty/up2k.py | 22 ++++++++++++++++++++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/copyparty/__main__.py b/copyparty/__main__.py index 30c80250..ee5acc67 100644 --- a/copyparty/__main__.py +++ b/copyparty/__main__.py @@ -247,6 +247,7 @@ def run_argparse(argv, formatter): ap.add_argument("--no-zip", action="store_true", help="disable download as zip/tar") ap.add_argument("--no-sendfile", action="store_true", help="disable sendfile (for debugging)") ap.add_argument("--no-scandir", action="store_true", help="disable scandir (for debugging)") + ap.add_argument("--sparse", metavar="MiB", type=int, default=4, help="up2k min.size threshold (mswin-only)") ap.add_argument("--urlform", metavar="MODE", type=str, default="print,get", help="how to handle url-forms") ap.add_argument("--salt", type=str, default="hunter2", help="up2k file-hash salt") diff --git a/copyparty/up2k.py b/copyparty/up2k.py index 674a79cc..236beb26 100644 --- a/copyparty/up2k.py +++ b/copyparty/up2k.py @@ -1111,7 +1111,8 @@ class Up2k(object): atomic_move(src, dst) if ANYWIN: - self.lastmod_q.put([dst, (int(time.time()), int(job["lmod"]))]) + a = [dst, job["size"], (int(time.time()), int(job["lmod"]))] + self.lastmod_q.put(a) # legit api sware 2 me mum if self.idx_wark( @@ -1212,6 +1213,17 @@ class Up2k(object): suffix = ".{:.6f}-{}".format(job["t0"], job["addr"]) with ren_open(tnam, "wb", fdir=pdir, suffix=suffix) as f: f, job["tnam"] = f["orz"] + if ( + ANYWIN + and self.args.sparse + and self.args.sparse * 1024 * 1024 <= job["size"] + ): + fp = os.path.join(pdir, job["tnam"]) + try: + sp.check_call(["fsutil", "sparse", "setflag", fp]) + except: + self.log("could not sparse [{}]".format(fp), 3) + f.seek(job["size"] - 1) f.write(b"e") @@ -1223,13 +1235,19 @@ class Up2k(object): # self.log("lmod: got {}".format(len(ready))) time.sleep(5) - for path, times in ready: + for path, sz, times in ready: self.log("lmod: setting times {} on {}".format(times, path)) try: os.utime(fsenc(path), times) except: self.log("lmod: failed to utime ({}, {})".format(path, times)) + if self.args.sparse and self.args.sparse * 1024 * 1024 <= sz: + try: + sp.check_call(["fsutil", "sparse", "setflag", path, "0"]) + except: + self.log("could not unsparse [{}]".format(path), 3) + def _snapshot(self): persist_interval = 30 # persist unfinished uploads index every 30 sec discard_interval = 21600 # drop unfinished uploads after 6 hours inactivity