From d58988a033cb8997fb799a1a75949488ccee2e50 Mon Sep 17 00:00:00 2001
From: ed <s@ocv.me>
Date: Thu, 7 Jan 2021 00:50:42 +0100
Subject: [PATCH] use sendfile when possible

---
 README.md             |  2 +-
 copyparty/__main__.py |  1 +
 copyparty/httpcli.py  | 28 ++++++++++------------------
 copyparty/util.py     | 41 +++++++++++++++++++++++++++++++++++++++++
 scripts/fusefuzz.py   |  8 ++++----
 5 files changed, 57 insertions(+), 23 deletions(-)

diff --git a/README.md b/README.md
index c23ca019..1dd74525 100644
--- a/README.md
+++ b/README.md
@@ -137,11 +137,11 @@ in the `scripts` folder:
 
 roughly sorted by priority
 
-* `os.sendfile` (and `os.copy_file_range` for up2k cloning)
 * reduce up2k roundtrips
   * start from a chunk index and just go
   * terminate client on bad data
 * drop onto folders
+* `os.copy_file_range` for up2k cloning
 * support pillow-simd
 * cache sha512 chunks on client
 * ~~symlink existing files on upload~~
diff --git a/copyparty/__main__.py b/copyparty/__main__.py
index d4d5b786..66652e4e 100644
--- a/copyparty/__main__.py
+++ b/copyparty/__main__.py
@@ -139,6 +139,7 @@ def main():
     ap.add_argument("-nw", action="store_true", help="disable writes (benchmark)")
     ap.add_argument("-nih", action="store_true", help="no info hostname")
     ap.add_argument("-nid", action="store_true", help="no info disk-usage")
+    ap.add_argument("--no-sendfile", action="store_true", help="disable sendfile")
     al = ap.parse_args()
 
     SvcHub(al).run()
diff --git a/copyparty/httpcli.py b/copyparty/httpcli.py
index 4cc2f622..263fe6cf 100644
--- a/copyparty/httpcli.py
+++ b/copyparty/httpcli.py
@@ -884,6 +884,7 @@ class HttpCli(object):
 
             logtail += " [\033[36m{}-{}\033[0m]".format(lower, upper)
 
+        use_sendfile = False
         if decompress:
             open_func = gzip.open
             open_args = [fsenc(fs_path), "rb"]
@@ -893,6 +894,8 @@ class HttpCli(object):
             open_func = open
             # 512 kB is optimal for huge files, use 64k
             open_args = [fsenc(fs_path), "rb", 64 * 1024]
+            if hasattr(os, 'sendfile'):
+                use_sendfile = not self.args.no_sendfile
 
         #
         # send reply
@@ -915,24 +918,13 @@ class HttpCli(object):
 
         ret = True
         with open_func(*open_args) as f:
-            remains = upper - lower
-            f.seek(lower)
-            while remains > 0:
-                # time.sleep(0.01)
-                buf = f.read(4096)
-                if not buf:
-                    break
-
-                if remains < len(buf):
-                    buf = buf[:remains]
-
-                try:
-                    self.s.sendall(buf)
-                    remains -= len(buf)
-                except:
-                    logmsg += " \033[31m" + str(upper - remains) + "\033[0m"
-                    ret = False
-                    break
+            if use_sendfile:
+                remains = sendfile_kern(lower, upper, f, self.s)
+            else:
+                remains = sendfile_py(lower, upper, f, self.s)
+        
+        if remains > 0:
+            logmsg += " \033[31m" + str(upper - remains) + "\033[0m"
 
         spd = self._spd((upper - lower) - remains)
         self.log("{},  {}".format(logmsg, spd))
diff --git a/copyparty/util.py b/copyparty/util.py
index b0d84077..cf24e654 100644
--- a/copyparty/util.py
+++ b/copyparty/util.py
@@ -6,6 +6,7 @@ import os
 import sys
 import time
 import base64
+import select
 import struct
 import hashlib
 import platform
@@ -591,6 +592,46 @@ def hashcopy(actor, fin, fout):
     return tlen, hashobj.hexdigest(), digest_b64
 
 
+def sendfile_py(lower, upper, f, s):
+    remains = upper - lower
+    f.seek(lower)
+    while remains > 0:
+        # time.sleep(0.01)
+        buf = f.read(min(4096, remains))
+        if not buf:
+            return remains
+
+        try:
+            s.sendall(buf)
+            remains -= len(buf)
+        except:
+            return remains
+
+    return 0
+
+
+def sendfile_kern(lower, upper, f, s):
+    out_fd = s.fileno()
+    in_fd = f.fileno()
+    ofs = lower
+    while ofs < upper:
+        try:
+            req = min(2 ** 30, upper - ofs)
+            select.select([], [out_fd], [], 10)
+            n = os.sendfile(out_fd, in_fd, ofs, req)
+        except Exception as ex:
+            # print("sendfile: " + repr(ex))
+            n = 0
+        
+        if n <= 0:
+            return upper - ofs
+        
+        ofs += n
+        # print("sendfile: ok, sent {} now, {} total, {} remains".format(n, ofs - lower, upper - ofs))
+
+    return 0
+
+
 def unescape_cookie(orig):
     # mw=idk; doot=qwe%2Crty%3Basd+fgh%2Bjkl%25zxc%26vbn  # qwe,rty;asd fgh+jkl%zxc&vbn
     ret = ""
diff --git a/scripts/fusefuzz.py b/scripts/fusefuzz.py
index c2e2c52b..36884a8e 100755
--- a/scripts/fusefuzz.py
+++ b/scripts/fusefuzz.py
@@ -4,10 +4,10 @@ import os
 import time
 
 """
-mkdir -p /dev/shm/fusefuzz/{r,v}
-PYTHONPATH=.. python3 -m copyparty -v /dev/shm/fusefuzz/r::r -i 127.0.0.1
-../bin/copyparty-fuse.py /dev/shm/fusefuzz/v http://127.0.0.1:3923/ 2 0
-(d="$PWD"; cd /dev/shm/fusefuzz && "$d"/fusefuzz.py)
+td=/dev/shm/; [ -e $td ] || td=$HOME; mkdir -p $td/fusefuzz/{r,v}
+PYTHONPATH=.. python3 -m copyparty -v $td/fusefuzz/r::r -i 127.0.0.1
+../bin/copyparty-fuse.py http://127.0.0.1:3923/ $td/fusefuzz/v -cf 2 -cd 0.5
+(d="$PWD"; cd $td/fusefuzz && "$d"/fusefuzz.py)
 """