mirror of
https://github.com/9001/copyparty.git
synced 2025-08-17 09:02:15 -06:00
add volflag sparse
to force use of sparse files;
this improves performance on s3-backed volumes noktuas reported on discord that the upload performance was unexpectedly poor when writing to an s3 bucket through a JuiceFS fuse-mount, only getting 1.5 MiB/s with copyparty, meanwhile a regular filecopy averaged 30 MiB/s plus the issue was that s3 does not support sparse files, so copyparty would fall back to sequential uploading, and also disable fpool, causing JuiceFS to repeatedly commit the same 5 MiB range to the storage provider as each chunk arrived from the client by forcing use of sparse files, s3 adapters such as JuiceFS and geesefs will "only" write the entire file to s3 *twice*, initially it writes the full filesize of zerobytes (depending on adapter, hopefully using gzip compression to reduce the bandwidth necessary) and then the actual file data in an adapter-specific chunksize with this volflag, copyparty appears to reach the full expected speed
This commit is contained in:
parent
d744f3ff8f
commit
8785d2f9fe
|
@ -116,6 +116,7 @@ flagcats = {
|
|||
"hardlink": "does dedup with hardlinks instead of symlinks",
|
||||
"neversymlink": "disables symlink fallback; full copy instead",
|
||||
"copydupes": "disables dedup, always saves full copies of dupes",
|
||||
"sparse": "force use of sparse files, mainly for s3-backed storage",
|
||||
"daw": "enable full WebDAV write support (dangerous);\nPUT-operations will now \033[1;31mOVERWRITE\033[0;35m existing files",
|
||||
"nosub": "forces all uploads into the top folder of the vfs",
|
||||
"magic": "enables filetype detection for nameless uploads",
|
||||
|
|
|
@ -3956,7 +3956,13 @@ class Up2k(object):
|
|||
|
||||
if not ANYWIN and sprs and sz > 1024 * 1024:
|
||||
fs = self.fstab.get(pdir)
|
||||
if fs != "ok":
|
||||
if fs == "ok":
|
||||
pass
|
||||
elif "sparse" in self.flags[job["ptop"]]:
|
||||
t = "volflag 'sparse' is forcing use of sparse files for uploads to [%s]"
|
||||
self.log(t % (job["ptop"],))
|
||||
relabel = True
|
||||
else:
|
||||
relabel = True
|
||||
f.seek(1024 * 1024 - 1)
|
||||
f.write(b"e")
|
||||
|
|
Loading…
Reference in a new issue