mirror of
https://github.com/9001/copyparty.git
synced 2025-08-16 08:32:13 -06:00
add chunksize cheat-sheet
This commit is contained in:
ed
parent
fcf32a935b
commit
f5753ba720
|
|
@ -92,6 +92,7 @@ turn almost any device into a file server with resumable uploads/downloads using
|
|||
* [listen on port 80 and 443](#listen-on-port-80-and-443) - become a *real* webserver
|
||||
* [reverse-proxy](#reverse-proxy) - running copyparty next to other websites
|
||||
* [real-ip](#real-ip) - teaching copyparty how to see client IPs
|
||||
* [reverse-proxy performance](#reverse-proxy-performance)
|
||||
* [prometheus](#prometheus) - metrics/stats can be enabled
|
||||
* [other extremely specific features](#other-extremely-specific-features) - you'll never find a use for these
|
||||
* [custom mimetypes](#custom-mimetypes) - change the association of a file extension
|
||||
|
|
|
|||
48
docs/chunksizes.py
Executable file
48
docs/chunksizes.py
Executable file
|
|
@ -0,0 +1,48 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
# there's far better ways to do this but its 4am and i dont wanna think
|
||||
|
||||
# just pypy it my dude
|
||||
|
||||
import math
|
||||
|
||||
def humansize(sz, terse=False):
|
||||
for unit in ["B", "KiB", "MiB", "GiB", "TiB"]:
|
||||
if sz < 1024:
|
||||
break
|
||||
|
||||
sz /= 1024.0
|
||||
|
||||
ret = " ".join([str(sz)[:4].rstrip("."), unit])
|
||||
|
||||
if not terse:
|
||||
return ret
|
||||
|
||||
return ret.replace("iB", "").replace(" ", "")
|
||||
|
||||
|
||||
def up2k_chunksize(filesize):
|
||||
chunksize = 1024 * 1024
|
||||
stepsize = 512 * 1024
|
||||
while True:
|
||||
for mul in [1, 2]:
|
||||
nchunks = math.ceil(filesize * 1.0 / chunksize)
|
||||
if nchunks <= 256 or (chunksize >= 32 * 1024 * 1024 and nchunks <= 4096):
|
||||
return chunksize
|
||||
|
||||
chunksize += stepsize
|
||||
stepsize *= mul
|
||||
|
||||
|
||||
def main():
|
||||
prev = 1048576
|
||||
n = n0 = 524288
|
||||
while True:
|
||||
csz = up2k_chunksize(n)
|
||||
if csz > prev:
|
||||
print(f"| {n-n0:>18_} | {humansize(n-n0):>8} | {prev:>13_} | {humansize(prev):>8} |".replace("_", " "))
|
||||
prev = csz
|
||||
n += n0
|
||||
|
||||
|
||||
main()
|
||||
|
|
@ -6,6 +6,7 @@
|
|||
* [up2k](#up2k) - quick outline of the up2k protocol
|
||||
* [why not tus](#why-not-tus) - I didn't know about [tus](https://tus.io/)
|
||||
* [why chunk-hashes](#why-chunk-hashes) - a single sha512 would be better, right?
|
||||
* [list of chunk-sizes](#list-of-chunk-sizes) - specific chunksizes are enforced
|
||||
* [hashed passwords](#hashed-passwords) - regarding the curious decisions
|
||||
* [http api](#http-api)
|
||||
* [read](#read)
|
||||
|
|
@ -95,6 +96,44 @@ hashwasm would solve the streaming issue but reduces hashing speed for sha512 (x
|
|||
|
||||
* blake2 might be a better choice since xxh is non-cryptographic, but that gets ~15 MiB/s on slower androids
|
||||
|
||||
### list of chunk-sizes
|
||||
|
||||
specific chunksizes are enforced depending on total filesize
|
||||
|
||||
each pair of filesize/chunksize is the largest filesize which will use its listed chunksize; a 512 MiB file will use chunksize 2 MiB, but if the file is one byte larger than 512 MiB then it becomes 3 MiB
|
||||
|
||||
for the purpose of performance (or dodging arbitrary proxy limitations), it is possible to upload combined and/or partial chunks using stitching and/or subchunks respectively
|
||||
|
||||
| filesize | filesize | chunksize | chunksz |
|
||||
| -----------------: | -------: | ------------: | ------: |
|
||||
| 268 435 456 | 256 MiB | 1 048 576 | 1.0 MiB |
|
||||
| 402 653 184 | 384 MiB | 1 572 864 | 1.5 MiB |
|
||||
| 536 870 912 | 512 MiB | 2 097 152 | 2.0 MiB |
|
||||
| 805 306 368 | 768 MiB | 3 145 728 | 3.0 MiB |
|
||||
| 1 073 741 824 | 1.0 GiB | 4 194 304 | 4.0 MiB |
|
||||
| 1 610 612 736 | 1.5 GiB | 6 291 456 | 6.0 MiB |
|
||||
| 2 147 483 648 | 2.0 GiB | 8 388 608 | 8.0 MiB |
|
||||
| 3 221 225 472 | 3.0 GiB | 12 582 912 | 12 MiB |
|
||||
| 4 294 967 296 | 4.0 GiB | 16 777 216 | 16 MiB |
|
||||
| 6 442 450 944 | 6.0 GiB | 25 165 824 | 24 MiB |
|
||||
| 137 438 953 472 | 128 GiB | 33 554 432 | 32 MiB |
|
||||
| 206 158 430 208 | 192 GiB | 50 331 648 | 48 MiB |
|
||||
| 274 877 906 944 | 256 GiB | 67 108 864 | 64 MiB |
|
||||
| 412 316 860 416 | 384 GiB | 100 663 296 | 96 MiB |
|
||||
| 549 755 813 888 | 512 GiB | 134 217 728 | 128 MiB |
|
||||
| 824 633 720 832 | 768 GiB | 201 326 592 | 192 MiB |
|
||||
| 1 099 511 627 776 | 1.0 TiB | 268 435 456 | 256 MiB |
|
||||
| 1 649 267 441 664 | 1.5 TiB | 402 653 184 | 384 MiB |
|
||||
| 2 199 023 255 552 | 2.0 TiB | 536 870 912 | 512 MiB |
|
||||
| 3 298 534 883 328 | 3.0 TiB | 805 306 368 | 768 MiB |
|
||||
| 4 398 046 511 104 | 4.0 TiB | 1 073 741 824 | 1.0 GiB |
|
||||
| 6 597 069 766 656 | 6.0 TiB | 1 610 612 736 | 1.5 GiB |
|
||||
| 8 796 093 022 208 | 8.0 TiB | 2 147 483 648 | 2.0 GiB |
|
||||
| 13 194 139 533 312 | 12.0 TiB | 3 221 225 472 | 3.0 GiB |
|
||||
| 17 592 186 044 416 | 16.0 TiB | 4 294 967 296 | 4.0 GiB |
|
||||
| 26 388 279 066 624 | 24.0 TiB | 6 442 450 944 | 6.0 GiB |
|
||||
| 35 184 372 088 832 | 32.0 TiB | 8 589 934 592 | 8.0 GiB |
|
||||
|
||||
|
||||
# hashed passwords
|
||||
|
||||
|
|
|
|||
|
|
@ -259,6 +259,12 @@ for d in /usr /var; do find $d -type f -size +30M 2>/dev/null; done | while IFS=
|
|||
for f in {0..255}; do echo $f; truncate -s 256M $f; b1=$(printf '%02x' $f); for o in {0..255}; do b2=$(printf '%02x' $o); printf "\x$b1\x$b2" | dd of=$f bs=2 seek=$((o*1024*1024)) conv=notrunc 2>/dev/null; done; done
|
||||
# create 6.06G file with 16 bytes of unique data at start+end of each 32M chunk
|
||||
sz=6509559808; truncate -s $sz f; csz=33554432; sz=$((sz/16)); step=$((csz/16)); ofs=0; while [ $ofs -lt $sz ]; do dd if=/dev/urandom of=f bs=16 count=2 seek=$ofs conv=notrunc iflag=fullblock; [ $ofs = 0 ] && ofs=$((ofs+step-1)) || ofs=$((ofs+step)); done
|
||||
# same but for chunksizes 16M (3.1G), 24M (4.1G), 48M (128.1G)
|
||||
sz=3321225472; csz=16777216;
|
||||
sz=4394967296; csz=25165824;
|
||||
sz=6509559808; csz=33554432;
|
||||
sz=138438953472; csz=50331648;
|
||||
f=csz-$csz; truncate -s $sz $f; sz=$((sz/16)); step=$((csz/16)); ofs=0; while [ $ofs -lt $sz ]; do dd if=/dev/urandom of=$f bs=16 count=2 seek=$ofs conv=notrunc iflag=fullblock; [ $ofs = 0 ] && ofs=$((ofs+step-1)) || ofs=$((ofs+step)); done
|
||||
|
||||
# py2 on osx
|
||||
brew install python@2
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ cat $f | awk '
|
|||
o{next}
|
||||
/^#/{s=1;rs=0;pr()}
|
||||
/^#* *(nix package)/{rs=1}
|
||||
/^#* *(themes|install on android|dev env setup|just the sfx|complete release|optional gpl stuff|nixos module)|```/{s=rs}
|
||||
/^#* *(themes|install on android|dev env setup|just the sfx|complete release|optional gpl stuff|nixos module|reverse-proxy perf)|```/{s=rs}
|
||||
/^#/{
|
||||
lv=length($1);
|
||||
sub(/[^ ]+ /,"");
|
||||
|
|
|
|||
Loading…
Reference in a new issue