From 13e5c96cabb7063856297197c0a22e219c4317cf Mon Sep 17 00:00:00 2001
From: ed <s@ocv.me>
Date: Sat, 27 Mar 2021 01:27:12 +0100
Subject: [PATCH] finish adding zip-crc (semi-streaming)

---
 copyparty/httpcli.py |  2 +-
 copyparty/szip.py    | 23 +++++++++++++----------
 copyparty/util.py    | 10 ++++++++++
 3 files changed, 24 insertions(+), 11 deletions(-)

diff --git a/copyparty/httpcli.py b/copyparty/httpcli.py
index eccf9fd0..72cefd04 100644
--- a/copyparty/httpcli.py
+++ b/copyparty/httpcli.py
@@ -1104,7 +1104,7 @@ class HttpCli(object):
 
         fgen = vn.zipgen(rem, items, self.uname, dots, not self.args.no_scandir)
         # for f in fgen: print(repr({k: f[k] for k in ["vp", "ap"]}))
-        bgen = packer(fgen, utf8="utf" in uarg)
+        bgen = packer(fgen, utf8="utf" in uarg, pre_crc="crc" in uarg)
         bsent = 0
         for buf in bgen.gen():
             if not buf:
diff --git a/copyparty/szip.py b/copyparty/szip.py
index f5cfa0dd..42e0bcdd 100644
--- a/copyparty/szip.py
+++ b/copyparty/szip.py
@@ -1,10 +1,9 @@
-import os
 import time
 import zlib
 import struct
 from datetime import datetime
 
-from .util import fsenc
+from .util import yieldfile
 
 
 def dostime2unix(buf):
@@ -195,19 +194,23 @@ class StreamZip(object):
             sz = st.st_size
             ts = st.st_mtime + 1
 
+            crc = 0
+            if self.pre_crc:
+                crc = 0
+                for buf in yieldfile(src):
+                    crc = zlib.crc32(buf, crc)
+
+                crc &= 0xFFFFFFFF
+
             h_pos = self.pos
             buf = gen_hdr(None, name, sz, ts, self.utf8, None, self.pre_crc)
             yield self._ct(buf)
 
-            crc = 0
-            with open(fsenc(src), "rb", 512 * 1024) as f:
-                while True:
-                    buf = f.read(64 * 1024)
-                    if not buf:
-                        break
-
+            for buf in yieldfile(src):
+                if not self.pre_crc:
                     crc = zlib.crc32(buf, crc)
-                    yield self._ct(buf)
+
+                yield self._ct(buf)
 
             crc &= 0xFFFFFFFF
 
diff --git a/copyparty/util.py b/copyparty/util.py
index 85db3199..3fe7c51d 100644
--- a/copyparty/util.py
+++ b/copyparty/util.py
@@ -780,6 +780,16 @@ def read_socket_chunked(sr, log=None):
         sr.recv(2)  # \r\n after each chunk too
 
 
+def yieldfile(fn):
+    with open(fsenc(fn), "rb", 512 * 1024) as f:
+        while True:
+            buf = f.read(64 * 1024)
+            if not buf:
+                break
+
+            yield buf
+
+
 def hashcopy(actor, fin, fout):
     u32_lim = int((2 ** 31) * 0.9)
     hashobj = hashlib.sha512()