From e76a50cb9d248fa206f11558b6d7116764d9a62f Mon Sep 17 00:00:00 2001 From: ed Date: Fri, 22 Sep 2023 20:40:52 +0000 Subject: [PATCH] add indexer benchmark + bump default num cores from 4 to 5 and make the mtag deps build better on fedora --- bin/mtag/install-deps.sh | 5 +-- copyparty/__main__.py | 5 ++- scripts/bench/filehash.sh | 67 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 74 insertions(+), 3 deletions(-) create mode 100755 scripts/bench/filehash.sh diff --git a/bin/mtag/install-deps.sh b/bin/mtag/install-deps.sh index 8c76f117..d07c8a05 100755 --- a/bin/mtag/install-deps.sh +++ b/bin/mtag/install-deps.sh @@ -250,8 +250,9 @@ install_vamp() { rm -- *.tar.gz cd beatroot-vamp-v1.0 [ -e ~/pe/vamp-sdk ] && - sed -ri 's`^(CFLAGS :=.*)`\1 -I'$HOME'/pe/vamp-sdk/include`' Makefile.linux - make -f Makefile.linux -j4 LDFLAGS=-L$HOME/pe/vamp-sdk/lib + sed -ri 's`^(CFLAGS :=.*)`\1 -I'$HOME'/pe/vamp-sdk/include`' Makefile.linux || + sed -ri 's`^(CFLAGS :=.*)`\1 -I/usr/include/vamp-sdk`' Makefile.linux + make -f Makefile.linux -j4 LDFLAGS="-L$HOME/pe/vamp-sdk/lib -L/usr/lib64" # /home/ed/vamp /home/ed/.vamp /usr/local/lib/vamp mkdir ~/vamp cp -pv beatroot-vamp.* ~/vamp/ diff --git a/copyparty/__main__.py b/copyparty/__main__.py index 0fcfb2e6..ec75dfc6 100755 --- a/copyparty/__main__.py +++ b/copyparty/__main__.py @@ -1200,7 +1200,10 @@ def run_argparse( fk_salt = get_fk_salt(cert_path) ah_salt = get_ah_salt() - hcores = min(CORES, 4) # optimal on py3.11 @ r5-4500U + # alpine peaks at 5 threads for some reason, + # all others scale past that (but try to avoid SMT), + # 5 should be plenty anyways (3 GiB/s on most machines) + hcores = min(CORES, 5 if CORES > 8 else 4) tty = os.environ.get("TERM", "").lower() == "linux" diff --git a/scripts/bench/filehash.sh b/scripts/bench/filehash.sh new file mode 100755 index 00000000..876107d0 --- /dev/null +++ b/scripts/bench/filehash.sh @@ -0,0 +1,67 @@ +#!/bin/bash +set -euo pipefail + +# check how fast copyparty is able to hash files during indexing +# assuming an infinitely fast HDD to read from (alternatively, +# checks whether you will be bottlenecked by CPU or HDD) +# +# uses copyparty's default config of using, well, it's complicated: +# * if you have more than 8 cores, then 5 threads, +# * if you have between 4 and 8, then 4 threads, +# * anything less and it takes your number of cores +# +# can be adjusted with --hash-mt (but alpine caps out at 5) + +[ $# -ge 1 ] || { + echo 'need arg 1: path to copyparty-sfx.py' + echo ' (remaining args will be passed on to copyparty,' + echo ' for example to tweak the hasher settings)' + exit 1 +} +sfx="$1" +shift +sfx="$(realpath "$sfx" || readlink -e "$sfx" || echo "$sfx")" +awk=$(which gawk || which awk) + +# try to use /dev/shm to avoid hitting filesystems at all, +# otherwise fallback to mktemp which probably uses /tmp +td=/dev/shm/cppbenchtmp +mkdir $td || td=$(mktemp -d) +trap "rm -rf $td" INT TERM EXIT +cd $td + +echo creating 256 MiB testfile in $td +head -c $((1024*1024*256)) /dev/urandom > 1 + +echo creating 127 symlinks to it +for n in $(seq 2 128); do ln -s 1 $n; done + +echo warming up cache +cat 1 >/dev/null + +echo ok lets go +python3 "$sfx" -p39204 -e2dsa --dbd=yolo --exit=idx -lo=t "$@" + +echo and the results are... +$awk '/1 volumes in / {printf "%s MiB/s\n", 256*128/$(NF-1)}'