add chunksize cheat-sheet

README.md

@@ -92,6 +92,7 @@ turn almost any device into a file server with resumable uploads/downloads using
     * [listen on port 80 and 443](#listen-on-port-80-and-443) - become a *real* webserver
     * [reverse-proxy](#reverse-proxy) - running copyparty next to other websites
         * [real-ip](#real-ip) - teaching copyparty how to see client IPs
+        * [reverse-proxy performance](#reverse-proxy-performance)
     * [prometheus](#prometheus) - metrics/stats can be enabled
     * [other extremely specific features](#other-extremely-specific-features) - you'll never find a use for these
         * [custom mimetypes](#custom-mimetypes) - change the association of a file extension

docs/chunksizes.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+
+# there's far better ways to do this but its 4am and i dont wanna think
+
+# just pypy it my dude
+
+import math
+
+def humansize(sz, terse=False):
+    for unit in ["B", "KiB", "MiB", "GiB", "TiB"]:
+        if sz < 1024:
+            break
+
+        sz /= 1024.0
+
+    ret = " ".join([str(sz)[:4].rstrip("."), unit])
+
+    if not terse:
+        return ret
+
+    return ret.replace("iB", "").replace(" ", "")
+
+
+def up2k_chunksize(filesize):
+    chunksize = 1024 * 1024
+    stepsize = 512 * 1024
+    while True:
+        for mul in [1, 2]:
+            nchunks = math.ceil(filesize * 1.0 / chunksize)
+            if nchunks <= 256 or (chunksize >= 32 * 1024 * 1024 and nchunks <= 4096):
+                return chunksize
+
+            chunksize += stepsize
+            stepsize *= mul
+
+
+def main():
+    prev = 1048576
+    n = n0 = 524288
+    while True:
+        csz = up2k_chunksize(n)
+        if csz > prev:
+            print(f"| {n-n0:>18_} | {humansize(n-n0):>8} | {prev:>13_} | {humansize(prev):>8} |".replace("_", " "))
+            prev = csz
+        n += n0
+
+
+main()

docs/devnotes.md

@@ -6,6 +6,7 @@
     * [up2k](#up2k) - quick outline of the up2k protocol
         * [why not tus](#why-not-tus) - I didn't know about [tus](https://tus.io/)
         * [why chunk-hashes](#why-chunk-hashes) - a single sha512 would be better, right?
+        * [list of chunk-sizes](#list-of-chunk-sizes) - specific chunksizes are enforced
 * [hashed passwords](#hashed-passwords) - regarding the curious decisions
 * [http api](#http-api)
     * [read](#read)
@@ -95,6 +96,44 @@ hashwasm would solve the streaming issue but reduces hashing speed for sha512 (x
 
 * blake2 might be a better choice since xxh is non-cryptographic, but that gets ~15 MiB/s on slower androids
 
+### list of chunk-sizes
+
+specific chunksizes are enforced  depending on total filesize
+
+each pair of filesize/chunksize is the largest filesize which will use its listed chunksize; a 512 MiB file will use chunksize 2 MiB, but if the file is one byte larger than 512 MiB then it becomes 3 MiB
+
+for the purpose of performance (or dodging arbitrary proxy limitations), it is possible to upload combined and/or partial chunks using stitching and/or subchunks respectively
+
+| filesize           | filesize | chunksize     | chunksz |
+| -----------------: | -------: | ------------: | ------: |
+|        268 435 456 |  256 MiB |     1 048 576 | 1.0 MiB |
+|        402 653 184 |  384 MiB |     1 572 864 | 1.5 MiB |
+|        536 870 912 |  512 MiB |     2 097 152 | 2.0 MiB |
+|        805 306 368 |  768 MiB |     3 145 728 | 3.0 MiB |
+|      1 073 741 824 |  1.0 GiB |     4 194 304 | 4.0 MiB |
+|      1 610 612 736 |  1.5 GiB |     6 291 456 | 6.0 MiB |
+|      2 147 483 648 |  2.0 GiB |     8 388 608 | 8.0 MiB |
+|      3 221 225 472 |  3.0 GiB |    12 582 912 |  12 MiB |
+|      4 294 967 296 |  4.0 GiB |    16 777 216 |  16 MiB |
+|      6 442 450 944 |  6.0 GiB |    25 165 824 |  24 MiB |
+|    137 438 953 472 |  128 GiB |    33 554 432 |  32 MiB |
+|    206 158 430 208 |  192 GiB |    50 331 648 |  48 MiB |
+|    274 877 906 944 |  256 GiB |    67 108 864 |  64 MiB |
+|    412 316 860 416 |  384 GiB |   100 663 296 |  96 MiB |
+|    549 755 813 888 |  512 GiB |   134 217 728 | 128 MiB |
+|    824 633 720 832 |  768 GiB |   201 326 592 | 192 MiB |
+|  1 099 511 627 776 |  1.0 TiB |   268 435 456 | 256 MiB |
+|  1 649 267 441 664 |  1.5 TiB |   402 653 184 | 384 MiB |
+|  2 199 023 255 552 |  2.0 TiB |   536 870 912 | 512 MiB |
+|  3 298 534 883 328 |  3.0 TiB |   805 306 368 | 768 MiB |
+|  4 398 046 511 104 |  4.0 TiB | 1 073 741 824 | 1.0 GiB |
+|  6 597 069 766 656 |  6.0 TiB | 1 610 612 736 | 1.5 GiB |
+|  8 796 093 022 208 |  8.0 TiB | 2 147 483 648 | 2.0 GiB |
+| 13 194 139 533 312 | 12.0 TiB | 3 221 225 472 | 3.0 GiB |
+| 17 592 186 044 416 | 16.0 TiB | 4 294 967 296 | 4.0 GiB |
+| 26 388 279 066 624 | 24.0 TiB | 6 442 450 944 | 6.0 GiB |
+| 35 184 372 088 832 | 32.0 TiB | 8 589 934 592 | 8.0 GiB |
+
 
 # hashed passwords
 

docs/notes.sh

@@ -259,6 +259,12 @@ for d in /usr /var; do find $d -type f -size +30M 2>/dev/null; done | while IFS=
 for f in {0..255}; do echo $f; truncate -s 256M $f; b1=$(printf '%02x' $f); for o in {0..255}; do b2=$(printf '%02x' $o); printf "\x$b1\x$b2" | dd of=$f bs=2 seek=$((o*1024*1024)) conv=notrunc 2>/dev/null; done; done
 # create 6.06G file with 16 bytes of unique data at start+end of each 32M chunk
 sz=6509559808; truncate -s $sz f; csz=33554432; sz=$((sz/16)); step=$((csz/16)); ofs=0; while [ $ofs -lt $sz ]; do dd if=/dev/urandom of=f bs=16 count=2 seek=$ofs conv=notrunc iflag=fullblock; [ $ofs = 0 ] && ofs=$((ofs+step-1)) || ofs=$((ofs+step)); done
+# same but for chunksizes 16M (3.1G), 24M (4.1G), 48M (128.1G)
+sz=3321225472;   csz=16777216;
+sz=4394967296;   csz=25165824;
+sz=6509559808;   csz=33554432;
+sz=138438953472; csz=50331648;
+f=csz-$csz; truncate -s $sz $f; sz=$((sz/16)); step=$((csz/16)); ofs=0; while [ $ofs -lt $sz ]; do dd if=/dev/urandom of=$f bs=16 count=2 seek=$ofs conv=notrunc iflag=fullblock; [ $ofs = 0 ] && ofs=$((ofs+step-1)) || ofs=$((ofs+step)); done
 
 # py2 on osx
 brew install python@2

scripts/toc.sh

@@ -20,7 +20,7 @@ cat $f | awk '
     o{next}
     /^#/{s=1;rs=0;pr()}
     /^#* *(nix package)/{rs=1}
-    /^#* *(themes|install on android|dev env setup|just the sfx|complete release|optional gpl stuff|nixos module)|```/{s=rs}
+    /^#* *(themes|install on android|dev env setup|just the sfx|complete release|optional gpl stuff|nixos module|reverse-proxy perf)|```/{s=rs}
     /^#/{
         lv=length($1);
         sub(/[^ ]+ /,"");