From 8785d2f9feed82633f3ce79eaa37a5d7150c91f2 Mon Sep 17 00:00:00 2001 From: ed Date: Fri, 8 Mar 2024 18:20:29 +0000 Subject: [PATCH] add volflag `sparse` to force use of sparse files; this improves performance on s3-backed volumes noktuas reported on discord that the upload performance was unexpectedly poor when writing to an s3 bucket through a JuiceFS fuse-mount, only getting 1.5 MiB/s with copyparty, meanwhile a regular filecopy averaged 30 MiB/s plus the issue was that s3 does not support sparse files, so copyparty would fall back to sequential uploading, and also disable fpool, causing JuiceFS to repeatedly commit the same 5 MiB range to the storage provider as each chunk arrived from the client by forcing use of sparse files, s3 adapters such as JuiceFS and geesefs will "only" write the entire file to s3 *twice*, initially it writes the full filesize of zerobytes (depending on adapter, hopefully using gzip compression to reduce the bandwidth necessary) and then the actual file data in an adapter-specific chunksize with this volflag, copyparty appears to reach the full expected speed --- copyparty/cfg.py | 1 + copyparty/up2k.py | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/copyparty/cfg.py b/copyparty/cfg.py index 10e921e8..abfb6c13 100644 --- a/copyparty/cfg.py +++ b/copyparty/cfg.py @@ -116,6 +116,7 @@ flagcats = { "hardlink": "does dedup with hardlinks instead of symlinks", "neversymlink": "disables symlink fallback; full copy instead", "copydupes": "disables dedup, always saves full copies of dupes", + "sparse": "force use of sparse files, mainly for s3-backed storage", "daw": "enable full WebDAV write support (dangerous);\nPUT-operations will now \033[1;31mOVERWRITE\033[0;35m existing files", "nosub": "forces all uploads into the top folder of the vfs", "magic": "enables filetype detection for nameless uploads", diff --git a/copyparty/up2k.py b/copyparty/up2k.py index 9abcfd7a..372d0c6a 100644 --- a/copyparty/up2k.py +++ b/copyparty/up2k.py @@ -3956,7 +3956,13 @@ class Up2k(object): if not ANYWIN and sprs and sz > 1024 * 1024: fs = self.fstab.get(pdir) - if fs != "ok": + if fs == "ok": + pass + elif "sparse" in self.flags[job["ptop"]]: + t = "volflag 'sparse' is forcing use of sparse files for uploads to [%s]" + self.log(t % (job["ptop"],)) + relabel = True + else: relabel = True f.seek(1024 * 1024 - 1) f.write(b"e")