make tar 6x faster (1.8 GiB/s)

copyparty/__main__.py

@@ -177,11 +177,14 @@ def sighandler(signal=None, frame=None):
     print("\n".join(msg))
 
 
-def main():
+def main(argv=None):
     time.strptime("19970815", "%Y%m%d")  # python#7980
     if WINDOWS:
         os.system("rem")  # enables colors
 
+    if argv is None:
+        argv = sys.argv
+
     desc = py_desc().replace("[", "\033[1;30m[")
 
     f = '\033[36mcopyparty v{} "\033[35m{}\033[36m" ({})\n{}\033[0m\n'
@@ -194,13 +197,13 @@ def main():
     deprecated = [["-e2s", "-e2ds"]]
     for dk, nk in deprecated:
         try:
-            idx = sys.argv.index(dk)
+            idx = argv.index(dk)
         except:
             continue
 
         msg = "\033[1;31mWARNING:\033[0;1m\n  {} \033[0;33mwas replaced with\033[0;1m {} \033[0;33mand will be removed\n\033[0m"
         print(msg.format(dk, nk))
-        sys.argv[idx] = nk
+        argv[idx] = nk
         time.sleep(2)
 
     ap = argparse.ArgumentParser(
@@ -290,7 +293,7 @@ def main():
     ap2.add_argument("--ssl-dbg", action="store_true", help="dump some tls info")
     ap2.add_argument("--ssl-log", metavar="PATH", help="log master secrets")
     
-    al = ap.parse_args()
+    al = ap.parse_args(args=argv[1:])
     # fmt: on
 
     # propagate implications

copyparty/star.py

@@ -11,9 +11,20 @@ class QFile(object):
 
     def __init__(self):
         self.q = Queue(64)
+        self.bq = []
+        self.nq = 0
 
     def write(self, buf):
-        self.q.put(buf)
+        if buf is None or self.nq >= 240 * 1024:
+            self.q.put(b"".join(self.bq))
+            self.bq = []
+            self.nq = 0
+
+        if buf is None:
+            self.q.put(None)
+        else:
+            self.bq.append(buf)
+            self.nq += len(buf)
 
 
 class StreamTar(object):
@@ -38,7 +49,7 @@ class StreamTar(object):
     def gen(self):
         while True:
             buf = self.qfile.q.get()
-            if buf is None:
+            if not buf:
                 break
 
             self.co += len(buf)
@@ -81,4 +92,4 @@ class StreamTar(object):
             self.ser(self.errf)
 
         self.tar.close()
-        self.qfile.q.put(None)
+        self.qfile.write(None)