up2k: fix hs after bitflips / net-glitch

chunk stitching could cause handshakes to initiate
a new upload rather than resume an ongoing one

copyparty/httpcli.py

@@ -2421,13 +2421,13 @@ class HttpCli(object):
         finally:
             if locked:
                 # now block until all chunks released+confirmed
-                x = broker.ask("up2k.confirm_chunks", ptop, wark, locked)
+                x = broker.ask("up2k.confirm_chunks", ptop, wark, written, locked)
                 num_left, t = x.get()
                 if num_left < 0:
                     self.loud_reply(t, status=500)
                     return False
                 t = "got %d more chunks, %d left"
-                self.log(t % (len(locked), num_left), 6)
+                self.log(t % (len(written), num_left), 6)
 
         if num_left < 0:
             raise Pebkac(500, "unconfirmed; see serverlog")

copyparty/up2k.py

@@ -3451,19 +3451,19 @@ class Up2k(object):
             self.mutex.release()
             return -1, ""
         try:
-            return self._confirm_chunks(ptop, wark, chashes)
+            return self._confirm_chunks(ptop, wark, chashes, chashes)
         finally:
             self.reg_mutex.release()
             self.mutex.release()
 
     def confirm_chunks(
-        self, ptop: str, wark: str, chashes: list[str]
+        self, ptop: str, wark: str, written: list[str], locked: list[str]
     ) -> tuple[int, str]:
         with self.mutex, self.reg_mutex:
-            return self._confirm_chunks(ptop, wark, chashes)
+            return self._confirm_chunks(ptop, wark, written, locked)
 
     def _confirm_chunks(
-        self, ptop: str, wark: str, chashes: list[str]
+        self, ptop: str, wark: str, written: list[str], locked: list[str]
     ) -> tuple[int, str]:
         if True:
             self.db_act = self.vol_act[ptop] = time.time()
@@ -3475,11 +3475,11 @@ class Up2k(object):
             except Exception as ex:
                 return -2, "confirm_chunk, wark(%r)" % (ex,)  # type: ignore
 
-            for chash in chashes:
+            for chash in locked:
                 job["busy"].pop(chash, None)
 
             try:
-                for chash in chashes:
+                for chash in written:
                     job["need"].remove(chash)
             except Exception as ex:
                 return -2, "confirm_chunk, chash(%s) %r" % (chash, ex)  # type: ignore

docs/notes.sh

@@ -257,6 +257,8 @@ for d in /usr /var; do find $d -type f -size +30M 2>/dev/null; done | while IFS=
 
 # up2k worst-case testfiles: create 64 GiB (256 x 256 MiB) of sparse files; each file takes 1 MiB disk space; each 1 MiB chunk is globally unique
 for f in {0..255}; do echo $f; truncate -s 256M $f; b1=$(printf '%02x' $f); for o in {0..255}; do b2=$(printf '%02x' $o); printf "\x$b1\x$b2" | dd of=$f bs=2 seek=$((o*1024*1024)) conv=notrunc 2>/dev/null; done; done
+# create 6.06G file with 16 bytes of unique data at start+end of each 32M chunk
+sz=6509559808; truncate -s $sz f; csz=33554432; sz=$((sz/16)); step=$((csz/16)); ofs=0; while [ $ofs -lt $sz ]; do dd if=/dev/urandom of=f bs=16 count=2 seek=$ofs conv=notrunc iflag=fullblock; [ $ofs = 0 ] && ofs=$((ofs+step-1)) || ofs=$((ofs+step)); done
 
 # py2 on osx
 brew install python@2