From 917380ddbb5c1456d8797ece01832793f9a0a6f8 Mon Sep 17 00:00:00 2001 From: ed Date: Sun, 19 Jan 2025 16:28:40 +0000 Subject: [PATCH] add nosparse volflag + update s3 readme: may improve upload performance in some particular uncommon scenarios, for example if hdd-writes are uncached, and/or the hdd is drastically slower than the network throughput one particular usecase where nosparse *might* improve performance is when the upload destination is cloud-storage provided by FUSE (for example an s3 bucket) but this is educated guesswork --- README.md | 6 +++++- copyparty/cfg.py | 1 + copyparty/up2k.py | 14 ++++++++++---- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 6d9b5192..be859ba3 100644 --- a/README.md +++ b/README.md @@ -1545,12 +1545,16 @@ connecting to an aws s3 bucket and similar there is no built-in support for this, but you can use FUSE-software such as [rclone](https://rclone.org/) / [geesefs](https://github.com/yandex-cloud/geesefs) / [JuiceFS](https://juicefs.com/en/) to first mount your cloud storage as a local disk, and then let copyparty use (a folder in) that disk as a volume -you may experience poor upload performance this way, but that can sometimes be fixed by specifying the volflag `sparse` to force the use of sparse files; this has improved the upload speeds from `1.5 MiB/s` to over `80 MiB/s` in one case, but note that you are also more likely to discover funny bugs in your FUSE software this way, so buckle up +you will probably get decent speeds with the default config, however most likely restricted to using one TCP connection per file, so the upload-client won't be able to send multiple chunks in parallel + +> before [v1.13.5](https://github.com/9001/copyparty/releases/tag/v1.13.5) it was recommended to use the volflag `sparse` to force-allow multiple chunks in parallel; this would improve the upload-speed from `1.5 MiB/s` to over `80 MiB/s` at the risk of provoking latent bugs in S3 or JuiceFS. But v1.13.5 added chunk-stitching, so this is now probably much less important. On the contrary, `nosparse` *may* now increase performance in some cases. Please try all three options (default, `sparse`, `nosparse`) as the optimal choice depends on your network conditions and software stack (both the FUSE-driver and cloud-server) someone has also tested geesefs in combination with [gocryptfs](https://nuetzlich.net/gocryptfs/) with surprisingly good results, getting 60 MiB/s upload speeds on a gbit line, but JuiceFS won with 80 MiB/s using its built-in encryption you may improve performance by specifying larger values for `--iobuf` / `--s-rd-sz` / `--s-wr-sz` +> if you've experimented with this and made interesting observations, please share your findings so we can add a section with specific recommendations :-) + ## hiding from google diff --git a/copyparty/cfg.py b/copyparty/cfg.py index 5e5f4b16..3ad325a1 100644 --- a/copyparty/cfg.py +++ b/copyparty/cfg.py @@ -144,6 +144,7 @@ flagcats = { "noclone": "take dupe data from clients, even if available on HDD", "nodupe": "rejects existing files (instead of linking/cloning them)", "sparse": "force use of sparse files, mainly for s3-backed storage", + "nosparse": "deny use of sparse files, mainly for slow storage", "daw": "enable full WebDAV write support (dangerous);\nPUT-operations will now \033[1;31mOVERWRITE\033[0;35m existing files", "nosub": "forces all uploads into the top folder of the vfs", "magic": "enables filetype detection for nameless uploads", diff --git a/copyparty/up2k.py b/copyparty/up2k.py index df9115b1..70603401 100644 --- a/copyparty/up2k.py +++ b/copyparty/up2k.py @@ -4905,7 +4905,8 @@ class Up2k(object): except: pass - xbu = self.flags[job["ptop"]].get("xbu") + vf = self.flags[job["ptop"]] + xbu = vf.get("xbu") ap_chk = djoin(pdir, job["name"]) vp_chk = djoin(job["vtop"], job["prel"], job["name"]) if xbu: @@ -4935,7 +4936,7 @@ class Up2k(object): if x: zvfs = vfs pdir, _, job["name"], (vfs, rem) = x - job["vcfg"] = vfs.flags + job["vcfg"] = vf = vfs.flags job["ptop"] = vfs.realpath job["vtop"] = vfs.vpath job["prel"] = rem @@ -4985,8 +4986,13 @@ class Up2k(object): fs = self.fstab.get(pdir) if fs == "ok": pass - elif "sparse" in self.flags[job["ptop"]]: - t = "volflag 'sparse' is forcing use of sparse files for uploads to [%s]" + elif "nosparse" in vf: + t = "volflag 'nosparse' is preventing creation of sparse files for uploads to [%s]" + self.log(t % (job["ptop"],)) + relabel = True + sprs = False + elif "sparse" in vf: + t = "volflag 'sparse' is forcing creation of sparse files for uploads to [%s]" self.log(t % (job["ptop"],)) relabel = True else: