use sendfile when possible

README.md

@@ -137,11 +137,11 @@ in the `scripts` folder:
 
 roughly sorted by priority
 
-* `os.sendfile` (and `os.copy_file_range` for up2k cloning)
 * reduce up2k roundtrips
   * start from a chunk index and just go
   * terminate client on bad data
 * drop onto folders
+* `os.copy_file_range` for up2k cloning
 * support pillow-simd
 * cache sha512 chunks on client
 * ~~symlink existing files on upload~~

copyparty/__main__.py

@@ -139,6 +139,7 @@ def main():
     ap.add_argument("-nw", action="store_true", help="disable writes (benchmark)")
     ap.add_argument("-nih", action="store_true", help="no info hostname")
     ap.add_argument("-nid", action="store_true", help="no info disk-usage")
+    ap.add_argument("--no-sendfile", action="store_true", help="disable sendfile")
     al = ap.parse_args()
 
     SvcHub(al).run()

copyparty/httpcli.py

@@ -884,6 +884,7 @@ class HttpCli(object):
 
             logtail += " [\033[36m{}-{}\033[0m]".format(lower, upper)
 
+        use_sendfile = False
         if decompress:
             open_func = gzip.open
             open_args = [fsenc(fs_path), "rb"]
@@ -893,6 +894,8 @@ class HttpCli(object):
             open_func = open
             # 512 kB is optimal for huge files, use 64k
             open_args = [fsenc(fs_path), "rb", 64 * 1024]
+            if hasattr(os, 'sendfile'):
+                use_sendfile = not self.args.no_sendfile
 
         #
         # send reply
@@ -915,24 +918,13 @@ class HttpCli(object):
 
         ret = True
         with open_func(*open_args) as f:
-            remains = upper - lower
+            if use_sendfile:
-            f.seek(lower)
+                remains = sendfile_kern(lower, upper, f, self.s)
-            while remains > 0:
+            else:
-                # time.sleep(0.01)
+                remains = sendfile_py(lower, upper, f, self.s)
-                buf = f.read(4096)
+        
-                if not buf:
+        if remains > 0:
-                    break
+            logmsg += " \033[31m" + str(upper - remains) + "\033[0m"
-
-                if remains < len(buf):
-                    buf = buf[:remains]
-
-                try:
-                    self.s.sendall(buf)
-                    remains -= len(buf)
-                except:
-                    logmsg += " \033[31m" + str(upper - remains) + "\033[0m"
-                    ret = False
-                    break
 
         spd = self._spd((upper - lower) - remains)
         self.log("{},  {}".format(logmsg, spd))

copyparty/util.py

@@ -6,6 +6,7 @@ import os
 import sys
 import time
 import base64
+import select
 import struct
 import hashlib
 import platform
@@ -591,6 +592,46 @@ def hashcopy(actor, fin, fout):
     return tlen, hashobj.hexdigest(), digest_b64
 
 
+def sendfile_py(lower, upper, f, s):
+    remains = upper - lower
+    f.seek(lower)
+    while remains > 0:
+        # time.sleep(0.01)
+        buf = f.read(min(4096, remains))
+        if not buf:
+            return remains
+
+        try:
+            s.sendall(buf)
+            remains -= len(buf)
+        except:
+            return remains
+
+    return 0
+
+
+def sendfile_kern(lower, upper, f, s):
+    out_fd = s.fileno()
+    in_fd = f.fileno()
+    ofs = lower
+    while ofs < upper:
+        try:
+            req = min(2 ** 30, upper - ofs)
+            select.select([], [out_fd], [], 10)
+            n = os.sendfile(out_fd, in_fd, ofs, req)
+        except Exception as ex:
+            # print("sendfile: " + repr(ex))
+            n = 0
+        
+        if n <= 0:
+            return upper - ofs
+        
+        ofs += n
+        # print("sendfile: ok, sent {} now, {} total, {} remains".format(n, ofs - lower, upper - ofs))
+
+    return 0
+
+
 def unescape_cookie(orig):
     # mw=idk; doot=qwe%2Crty%3Basd+fgh%2Bjkl%25zxc%26vbn  # qwe,rty;asd fgh+jkl%zxc&vbn
     ret = ""

scripts/fusefuzz.py

@@ -4,10 +4,10 @@ import os
 import time
 
 """
-mkdir -p /dev/shm/fusefuzz/{r,v}
+td=/dev/shm/; [ -e $td ] || td=$HOME; mkdir -p $td/fusefuzz/{r,v}
-PYTHONPATH=.. python3 -m copyparty -v /dev/shm/fusefuzz/r::r -i 127.0.0.1
+PYTHONPATH=.. python3 -m copyparty -v $td/fusefuzz/r::r -i 127.0.0.1
-../bin/copyparty-fuse.py /dev/shm/fusefuzz/v http://127.0.0.1:3923/ 2 0
+../bin/copyparty-fuse.py http://127.0.0.1:3923/ $td/fusefuzz/v -cf 2 -cd 0.5
-(d="$PWD"; cd /dev/shm/fusefuzz && "$d"/fusefuzz.py)
+(d="$PWD"; cd $td/fusefuzz && "$d"/fusefuzz.py)
 """