diff --git a/README.md b/README.md index 881cf2ef..855874ae 100644 --- a/README.md +++ b/README.md @@ -92,6 +92,7 @@ turn almost any device into a file server with resumable uploads/downloads using * [listen on port 80 and 443](#listen-on-port-80-and-443) - become a *real* webserver * [reverse-proxy](#reverse-proxy) - running copyparty next to other websites * [real-ip](#real-ip) - teaching copyparty how to see client IPs + * [reverse-proxy performance](#reverse-proxy-performance) * [prometheus](#prometheus) - metrics/stats can be enabled * [other extremely specific features](#other-extremely-specific-features) - you'll never find a use for these * [custom mimetypes](#custom-mimetypes) - change the association of a file extension diff --git a/docs/chunksizes.py b/docs/chunksizes.py new file mode 100755 index 00000000..7b24d664 --- /dev/null +++ b/docs/chunksizes.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 + +# there's far better ways to do this but its 4am and i dont wanna think + +# just pypy it my dude + +import math + +def humansize(sz, terse=False): + for unit in ["B", "KiB", "MiB", "GiB", "TiB"]: + if sz < 1024: + break + + sz /= 1024.0 + + ret = " ".join([str(sz)[:4].rstrip("."), unit]) + + if not terse: + return ret + + return ret.replace("iB", "").replace(" ", "") + + +def up2k_chunksize(filesize): + chunksize = 1024 * 1024 + stepsize = 512 * 1024 + while True: + for mul in [1, 2]: + nchunks = math.ceil(filesize * 1.0 / chunksize) + if nchunks <= 256 or (chunksize >= 32 * 1024 * 1024 and nchunks <= 4096): + return chunksize + + chunksize += stepsize + stepsize *= mul + + +def main(): + prev = 1048576 + n = n0 = 524288 + while True: + csz = up2k_chunksize(n) + if csz > prev: + print(f"| {n-n0:>18_} | {humansize(n-n0):>8} | {prev:>13_} | {humansize(prev):>8} |".replace("_", " ")) + prev = csz + n += n0 + + +main() diff --git a/docs/devnotes.md b/docs/devnotes.md index ebb3777f..7fee3f5e 100644 --- a/docs/devnotes.md +++ b/docs/devnotes.md @@ -6,6 +6,7 @@ * [up2k](#up2k) - quick outline of the up2k protocol * [why not tus](#why-not-tus) - I didn't know about [tus](https://tus.io/) * [why chunk-hashes](#why-chunk-hashes) - a single sha512 would be better, right? + * [list of chunk-sizes](#list-of-chunk-sizes) - specific chunksizes are enforced * [hashed passwords](#hashed-passwords) - regarding the curious decisions * [http api](#http-api) * [read](#read) @@ -95,6 +96,44 @@ hashwasm would solve the streaming issue but reduces hashing speed for sha512 (x * blake2 might be a better choice since xxh is non-cryptographic, but that gets ~15 MiB/s on slower androids +### list of chunk-sizes + +specific chunksizes are enforced depending on total filesize + +each pair of filesize/chunksize is the largest filesize which will use its listed chunksize; a 512 MiB file will use chunksize 2 MiB, but if the file is one byte larger than 512 MiB then it becomes 3 MiB + +for the purpose of performance (or dodging arbitrary proxy limitations), it is possible to upload combined and/or partial chunks using stitching and/or subchunks respectively + +| filesize | filesize | chunksize | chunksz | +| -----------------: | -------: | ------------: | ------: | +| 268 435 456 | 256 MiB | 1 048 576 | 1.0 MiB | +| 402 653 184 | 384 MiB | 1 572 864 | 1.5 MiB | +| 536 870 912 | 512 MiB | 2 097 152 | 2.0 MiB | +| 805 306 368 | 768 MiB | 3 145 728 | 3.0 MiB | +| 1 073 741 824 | 1.0 GiB | 4 194 304 | 4.0 MiB | +| 1 610 612 736 | 1.5 GiB | 6 291 456 | 6.0 MiB | +| 2 147 483 648 | 2.0 GiB | 8 388 608 | 8.0 MiB | +| 3 221 225 472 | 3.0 GiB | 12 582 912 | 12 MiB | +| 4 294 967 296 | 4.0 GiB | 16 777 216 | 16 MiB | +| 6 442 450 944 | 6.0 GiB | 25 165 824 | 24 MiB | +| 137 438 953 472 | 128 GiB | 33 554 432 | 32 MiB | +| 206 158 430 208 | 192 GiB | 50 331 648 | 48 MiB | +| 274 877 906 944 | 256 GiB | 67 108 864 | 64 MiB | +| 412 316 860 416 | 384 GiB | 100 663 296 | 96 MiB | +| 549 755 813 888 | 512 GiB | 134 217 728 | 128 MiB | +| 824 633 720 832 | 768 GiB | 201 326 592 | 192 MiB | +| 1 099 511 627 776 | 1.0 TiB | 268 435 456 | 256 MiB | +| 1 649 267 441 664 | 1.5 TiB | 402 653 184 | 384 MiB | +| 2 199 023 255 552 | 2.0 TiB | 536 870 912 | 512 MiB | +| 3 298 534 883 328 | 3.0 TiB | 805 306 368 | 768 MiB | +| 4 398 046 511 104 | 4.0 TiB | 1 073 741 824 | 1.0 GiB | +| 6 597 069 766 656 | 6.0 TiB | 1 610 612 736 | 1.5 GiB | +| 8 796 093 022 208 | 8.0 TiB | 2 147 483 648 | 2.0 GiB | +| 13 194 139 533 312 | 12.0 TiB | 3 221 225 472 | 3.0 GiB | +| 17 592 186 044 416 | 16.0 TiB | 4 294 967 296 | 4.0 GiB | +| 26 388 279 066 624 | 24.0 TiB | 6 442 450 944 | 6.0 GiB | +| 35 184 372 088 832 | 32.0 TiB | 8 589 934 592 | 8.0 GiB | + # hashed passwords diff --git a/docs/notes.sh b/docs/notes.sh index e50b1ce9..3c8798cc 100644 --- a/docs/notes.sh +++ b/docs/notes.sh @@ -259,6 +259,12 @@ for d in /usr /var; do find $d -type f -size +30M 2>/dev/null; done | while IFS= for f in {0..255}; do echo $f; truncate -s 256M $f; b1=$(printf '%02x' $f); for o in {0..255}; do b2=$(printf '%02x' $o); printf "\x$b1\x$b2" | dd of=$f bs=2 seek=$((o*1024*1024)) conv=notrunc 2>/dev/null; done; done # create 6.06G file with 16 bytes of unique data at start+end of each 32M chunk sz=6509559808; truncate -s $sz f; csz=33554432; sz=$((sz/16)); step=$((csz/16)); ofs=0; while [ $ofs -lt $sz ]; do dd if=/dev/urandom of=f bs=16 count=2 seek=$ofs conv=notrunc iflag=fullblock; [ $ofs = 0 ] && ofs=$((ofs+step-1)) || ofs=$((ofs+step)); done +# same but for chunksizes 16M (3.1G), 24M (4.1G), 48M (128.1G) +sz=3321225472; csz=16777216; +sz=4394967296; csz=25165824; +sz=6509559808; csz=33554432; +sz=138438953472; csz=50331648; +f=csz-$csz; truncate -s $sz $f; sz=$((sz/16)); step=$((csz/16)); ofs=0; while [ $ofs -lt $sz ]; do dd if=/dev/urandom of=$f bs=16 count=2 seek=$ofs conv=notrunc iflag=fullblock; [ $ofs = 0 ] && ofs=$((ofs+step-1)) || ofs=$((ofs+step)); done # py2 on osx brew install python@2 diff --git a/scripts/toc.sh b/scripts/toc.sh index d0bbf814..8c3e22e9 100755 --- a/scripts/toc.sh +++ b/scripts/toc.sh @@ -20,7 +20,7 @@ cat $f | awk ' o{next} /^#/{s=1;rs=0;pr()} /^#* *(nix package)/{rs=1} - /^#* *(themes|install on android|dev env setup|just the sfx|complete release|optional gpl stuff|nixos module)|```/{s=rs} + /^#* *(themes|install on android|dev env setup|just the sfx|complete release|optional gpl stuff|nixos module|reverse-proxy perf)|```/{s=rs} /^#/{ lv=length($1); sub(/[^ ]+ /,"");