From 2a37e81bd848fbd9c70fda1b3695544c50e77d4a Mon Sep 17 00:00:00 2001 From: ed Date: Sat, 1 Apr 2023 10:21:21 +0000 Subject: [PATCH] add rclone optimization, closes #21 --- README.md | 11 ++++++----- copyparty/web/svcs.html | 5 ++--- docs/rclone.md | 6 ++++-- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index f83d4eca..c05bf1ed 100644 --- a/README.md +++ b/README.md @@ -1206,7 +1206,9 @@ sync folders to/from copyparty the commandline uploader [up2k.py](https://github.com/9001/copyparty/tree/hovudstraum/bin#up2kpy) with `--dr` is the best way to sync a folder to copyparty; verifies checksums and does files in parallel, and deletes unexpected files on the server after upload has finished which makes file-renames really cheap (it'll rename serverside and skip uploading) -alternatively there is [rclone](./docs/rclone.md) which allows for bidirectional sync and is *way* more flexible (stream files straight from sftp/s3/gcs to copyparty for instance), although syncing to copyparty is about 5x slower than up2k.py if you have many small files in particular +alternatively there is [rclone](./docs/rclone.md) which allows for bidirectional sync and is *way* more flexible (stream files straight from sftp/s3/gcs to copyparty for instance), although there is no integrity check and it won't work with files over 100 MiB if copyparty is behind cloudflare + +* starting from rclone v1.63 (currently [in beta](https://beta.rclone.org/?filter=latest)), rclone will also be faster than up2k.py ## mount as drive @@ -1215,11 +1217,10 @@ a remote copyparty server as a local filesystem; go to the control-panel and cl alternatively, some alternatives roughly sorted by speed (unreproducible benchmark), best first: -* [rclone-http](./docs/rclone.md) (25s), read-only +* [rclone-webdav](./docs/rclone.md) (25s), read/WRITE ((v1.63-beta)[https://beta.rclone.org/?filter=latest]) +* [rclone-http](./docs/rclone.md) (26s), read-only +* [partyfuse.py](./bin/#partyfusepy) (35s), read-only * [rclone-ftp](./docs/rclone.md) (47s), read/WRITE -* [rclone-webdav](./docs/rclone.md) (51s), read/WRITE - * copyparty-1.5.0's webdav server is faster than rclone-1.60.0 (69s) -* [partyfuse.py](./bin/#partyfusepy) (71s), read-only * davfs2 (103s), read/WRITE, *very fast* on small files * [win10-webdav](#webdav-server) (138s), read/WRITE * [win10-smb2](#smb-server) (387s), read/WRITE diff --git a/copyparty/web/svcs.html b/copyparty/web/svcs.html index a3ab01d1..cad6095c 100644 --- a/copyparty/web/svcs.html +++ b/copyparty/web/svcs.html @@ -43,10 +43,9 @@

WebDAV

-

note: rclone-FTP is a bit faster, so {% if args.ftp or args.ftps %}try that first{% else %}consider enabling FTP in server settings{% endif %}

if you can, install winfsp+rclone and then paste this in cmd:

-                rclone config create {{ aname }}-dav webdav url=http{{ s }}://{{ rip }}{{ hport }} vendor=owncloud{% if accs %} user=k pass={{ pw }}{% endif %}
+                rclone config create {{ aname }}-dav webdav url=http{{ s }}://{{ rip }}{{ hport }} vendor=owncloud pacer_min_sleep=0.01ms{% if accs %} user=k pass={{ pw }}{% endif %}
                 rclone mount --vfs-cache-mode writes --dir-cache-time 5s {{ aname }}-dav:{{ rvp }} W:
{% if s %} @@ -71,7 +70,7 @@

or you can use rclone instead, which is much slower but doesn't require root (plus it keeps lastmodified on upload):

-                rclone config create {{ aname }}-dav webdav url=http{{ s }}://{{ rip }}{{ hport }} vendor=owncloud{% if accs %} user=k pass={{ pw }}{% endif %}
+                rclone config create {{ aname }}-dav webdav url=http{{ s }}://{{ rip }}{{ hport }} vendor=owncloud pacer_min_sleep=0.01ms{% if accs %} user=k pass={{ pw }}{% endif %}
                 rclone mount --vfs-cache-mode writes --dir-cache-time 5s {{ aname }}-dav:{{ rvp }} mp
{% if s %} diff --git a/docs/rclone.md b/docs/rclone.md index 3c20074a..c0792816 100644 --- a/docs/rclone.md +++ b/docs/rclone.md @@ -29,11 +29,13 @@ echo type = webdav echo vendor = owncloud echo url = http://127.0.0.1:3923/ echo headers = Cookie,cppwd=hunter2 +echo pacer_min_sleep = 0.01ms echo( echo [cpp-ro] echo type = http echo url = http://127.0.0.1:3923/ echo headers = Cookie,cppwd=hunter2 +echo pacer_min_sleep = 0.01ms ) > %userprofile%\.config\rclone\rclone.conf ``` @@ -48,11 +50,13 @@ type = webdav vendor = owncloud url = http://127.0.0.1:3923/ headers = Cookie,cppwd=hunter2 +pacer_min_sleep = 0.01ms [cpp-ro] type = http url = http://127.0.0.1:3923/ headers = Cookie,cppwd=hunter2 +pacer_min_sleep = 0.01ms EOF ``` @@ -74,8 +78,6 @@ note that the up2k client [up2k.py](https://github.com/9001/copyparty/tree/hovud rclone sync /usr/share/icons/ cpp-rw:fds/ ``` -TODO: rclone bug? `--transfers=4` doesn't seem to do anything (it does one request at a time), doesn't matter if the webdav server is copyparty or rclone - # use rclone as server too, replacing copyparty