finish adding zip-crc (semi-streaming)

copyparty/httpcli.py

@@ -1104,7 +1104,7 @@ class HttpCli(object):
 
         fgen = vn.zipgen(rem, items, self.uname, dots, not self.args.no_scandir)
         # for f in fgen: print(repr({k: f[k] for k in ["vp", "ap"]}))
-        bgen = packer(fgen, utf8="utf" in uarg)
+        bgen = packer(fgen, utf8="utf" in uarg, pre_crc="crc" in uarg)
         bsent = 0
         for buf in bgen.gen():
             if not buf:

copyparty/szip.py

@@ -1,10 +1,9 @@
-import os
 import time
 import zlib
 import struct
 from datetime import datetime
 
-from .util import fsenc
+from .util import yieldfile
 
 
 def dostime2unix(buf):
@@ -195,19 +194,23 @@ class StreamZip(object):
             sz = st.st_size
             ts = st.st_mtime + 1
 
+            crc = 0
+            if self.pre_crc:
+                crc = 0
+                for buf in yieldfile(src):
+                    crc = zlib.crc32(buf, crc)
+
+                crc &= 0xFFFFFFFF
+
             h_pos = self.pos
             buf = gen_hdr(None, name, sz, ts, self.utf8, None, self.pre_crc)
             yield self._ct(buf)
 
-            crc = 0
+            for buf in yieldfile(src):
-            with open(fsenc(src), "rb", 512 * 1024) as f:
+                if not self.pre_crc:
-                while True:
-                    buf = f.read(64 * 1024)
-                    if not buf:
-                        break
-
                     crc = zlib.crc32(buf, crc)
-                    yield self._ct(buf)
+
+                yield self._ct(buf)
 
             crc &= 0xFFFFFFFF
 

copyparty/util.py

@@ -780,6 +780,16 @@ def read_socket_chunked(sr, log=None):
         sr.recv(2)  # \r\n after each chunk too
 
 
+def yieldfile(fn):
+    with open(fsenc(fn), "rb", 512 * 1024) as f:
+        while True:
+            buf = f.read(64 * 1024)
+            if not buf:
+                break
+
+            yield buf
+
+
 def hashcopy(actor, fin, fout):
     u32_lim = int((2 ** 31) * 0.9)
     hashobj = hashlib.sha512()