From f327bdb6b45ef32a18cf7911a5e7fb04808e8aa1 Mon Sep 17 00:00:00 2001 From: ed Date: Mon, 15 Mar 2021 23:12:13 +0100 Subject: [PATCH] =?UTF-8?q?never=20trust=20tags=20=E3=81=8B=E3=81=97?= =?UTF-8?q?=E3=82=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/music-analysis.sh | 71 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 62 insertions(+), 9 deletions(-) diff --git a/docs/music-analysis.sh b/docs/music-analysis.sh index c10d1d62..f4e5af9e 100644 --- a/docs/music-analysis.sh +++ b/docs/music-analysis.sh @@ -9,27 +9,80 @@ python3 -m pip install --user librosa git clone https://github.com/librosa/librosa.git +# correct bpm for tracks with bad tags +br=' +/Trip Trip Trip\(Hardcore Edit\).mp3/ {v=176} +/World!!.BIG_SOS/ {v=175} +/\/08\..*\(BIG_SOS Bootleg\)\.mp3/ {v=175} +/もってけ！セーラ服.Asterisk DnB/ {v=175} +/Rondo\(Asterisk DnB Re.mp3/ {v=175} +/Ray Nautica 175 Edit/ {v=175;x="thunk"} +/TOKIMEKI Language.Jauz/ {v=174} +/YUPPUN Hardcore Remix\).mp3/ {v=174;x="keeps drifting"} +/(èâAâï.î╧ûδ|バーチャリアル.狐耶)J-Core Remix\).mp3/ {v=172;x="hard"} +/lucky train..Freezer/ {v=170} +/Alf zero Bootleg ReMix/ {v=170} +/Prisoner of Love.Kacky/ {v=170} +/火炎 .Qota/ {v=170} +/\(hu-zin Bootleg\)\.mp3/ {v=170} +/15. STRAIGHT BET\(Milynn Bootleg\)\.mp3/ {v=170} +/\/13.*\(Milynn Bootleg\)\.mp3/ {v=167;x="way hard"} +/COLOR PLANET .10SAI . nijikon Remix\)\.mp3/ {v=165} +/11\. (朝はご飯派|Æ⌐é═é▓ö╤öh)\.mp3/ {v=162} +/09\. Where.s the core/ {v=160} +/PLANET\(Koushif Jersey Club Bootleg\)remaster.mp3/ {v=160;x="starts ez turns bs"} +/kened Soul - Madeon x Angel Beats!.mp3/ {v=160} +/Dear Moments\(Mother Harlot Bootleg\)\.mp3/ {v=150} +/POWER.Ringos UKG/ {v=140} +/ブルー・フィールド\(Ringos UKG Remix\).mp3/ {v=135} +/プラチナジェット.Ringo Remix..mp3/ {v=131.2} +/Mirrorball Love \(TKM Bootleg Mix\).mp3/ {v=130} +/Photon Melodies \(TKM Bootleg Mix\).mp3/ {v=128} +/Trap of Love \(TKM Bootleg Mix\).mp3/ {v=128} +/One Step \(TKM Bootleg Mix\)\.mp3/ {v=126} +/04 (トリカムイ岩|âgâèâJâÇâCèΓ).mp3/ {v=125} +/Get your Wish \(NAWN REMIX\)\.mp3/ {v=95} +/Flicker .Nitro Fun/ {v=92} +/\/14\..*suicat Remix/ {v=85.5;x="tricky"} +/Yanagi Nagi - Harumodoki \(EO Remix\)\.mp3/ {v=150} +/Azure - Nicology\.mp3/ {v=128;x="off by 5 how"} +' + + # afun host, collects/grades the results -runfun() { cores=8; touch run; rm -f /dev/shm/mres.*; t00=$(date +%s); tbc() { bc | sed -r 's/(\.[0-9]{2}).*/\1/'; }; for ((core=0; core<$cores; core++)); do sqlite3 /mnt/Users/ed/Music/.hist/up2k.db 'select dur.w, dur.v, bpm.v from mt bpm join mt dur on bpm.w = dur.w where bpm.k = ".bpm" and dur.k = ".dur" order by dur.w' | uniq -w16 | while IFS=\| read w dur bpm; do sqlite3 /mnt/Users/ed/Music/.hist/up2k.db "select rd, fn from up where substr(w,1,16) = '$w'" | sed -r "s/^/$bpm /"; done | grep mir/cr | tr \| / | while read bpm fn; do [ -e run ] || break; n=$((n+1)); ncore=$((n%cores)); [ $ncore -eq $core ] || continue; t0=$(date +%s.%N); (afun || exit 1; t=$(date +%s.%N); td=$(echo "scale=3; $t - $t0" | tbc); bd=$(echo "scale=3; $bpm / $py" | tbc); printf '%4s sec, %4s orig, %6s py, %4s div, %s\n' $td $bpm $py $bd "$fn") | tee -a /dev/shm/mres.$ncore; rv=${PIPESTATUS[0]}; [ $rv -eq 0 ] || { echo "FAULT($rv): $fn"; }; done & done; wait 2>/dev/null; cat /dev/shm/mres.* | awk '$8!="div,"{next} $5!~/^[0-9\.]+/{next} 1;{meta=$3;det=$5;div=meta/det} div<0.7{det/=2} div>1.3{det*=2} {idet=sprintf("%.0f",det)} {idiff=idet-meta} meta>idet{idiff=meta-idet} idiff==0{n0++;next} idiff==1{n1++;next} idiff>10{nx++;next} {n10++} END {printf "ok: %d 1off: %d 10off: %d fail: %d\n",n0,n1,n10,nx}'; te=$(date +%s); echo $((te-t00)) sec spent; } +runfun() { cores=8; touch run; rm -f /dev/shm/mres.*; t00=$(date +%s); tbc() { bc | sed -r 's/(\.[0-9]{2}).*/\1/'; }; for ((core=0; core<$cores; core++)); do sqlite3 /mnt/Users/ed/Music/.hist/up2k.db 'select dur.w, dur.v, bpm.v from mt bpm join mt dur on bpm.w = dur.w where bpm.k = ".bpm" and dur.k = ".dur" order by dur.w' | uniq -w16 | while IFS=\| read w dur bpm; do sqlite3 /mnt/Users/ed/Music/.hist/up2k.db "select rd, fn from up where substr(w,1,16) = '$w'" | sed -r "s/^/$bpm /"; done | grep mir/cr | tr \| / | awk '{v=$1;sub(/[^ ]+ /,"")} '"$br"' {printf "%s %s\n",v,$0}' | while read bpm fn; do [ -e run ] || break; n=$((n+1)); ncore=$((n%cores)); [ $ncore -eq $core ] || continue; t0=$(date +%s.%N); (afun || exit 1; t=$(date +%s.%N); td=$(echo "scale=3; $t - $t0" | tbc); bd=$(echo "scale=3; $bpm / $py" | tbc); printf '%4s sec, %4s orig, %6s py, %4s div, %s\n' $td $bpm $py $bd "$fn") | tee -a /dev/shm/mres.$ncore; rv=${PIPESTATUS[0]}; [ $rv -eq 0 ] || { echo "FAULT($rv): $fn"; }; done & done; wait 2>/dev/null; cat /dev/shm/mres.* | awk 'function prt(c) {printf "\033[3%sm%s\033[0m\n",c,$0} $8!="div,"{next} $5!~/^[0-9\.]+/{next} {meta=$3;det=$5;div=meta/det} div<0.7{det/=2} div>1.3{det*=2} {idet=sprintf("%.0f",det)} {idiff=idet-meta} meta>idet{idiff=meta-idet} idiff==0{n0++;prt(6);next} idiff==1{n1++;prt(3);next} idiff>10{nx++;prt(1);next} {n10++;prt(5)} END {printf "ok: %d 1off: %2s (%3s) 10off: %2s (%3s) fail: %2s\n",n0,n1,n0+n1,n10,n0+n1+n10,nx}'; te=$(date +%s); echo $((te-t00)) sec spent; } -# ok: 4 1off: 59 10off: 65 fail: 53 # 105 sec, librosa @ 8c archvm on 3700x w10 -afun() { ffmpeg -hide_banner -v fatal -nostdin -ss $((dur/3)) -y -i /mnt/Users/ed/Music/"$fn" -t 60 /dev/shm/$core.wav || return 1; py="$(/home/ed/src/librosa/examples/beat_tracker.py /dev/shm/$core.wav x 2>&1 | awk 'BEGIN {v=1} /^Estimated tempo: /{v=$3} END {print v}')"; } +# ok: 8 1off: 62 ( 70) 10off: 86 (156) fail: 25 # 105 sec, librosa @ 8c archvm on 3700x w10 +# ok: 4 1off: 59 ( 63) 10off: 65 (128) fail: 53 # using original tags (bad) +afun() { ffmpeg -hide_banner -v fatal -nostdin -ss $((dur/3)) -y -i /mnt/Users/ed/Music/"$fn" -t 60 /dev/shm/$core.wav || return 1; py="$(/home/ed/src/librosa/examples/beat_tracker.py /dev/shm/$core.wav x 2>&1 | awk 'BEGIN {v=1} /^Estimated tempo: /{v=$3} END {print v}')"; } runfun -# ok: 109 1off: 4 10off: 9 fail: 59 # 51 sec, vamp-example-fixedtempo -afun() { ffmpeg -hide_banner -v fatal -nostdin -ss $((dur/3)) -y -i /mnt/Users/ed/Music/"$fn" -ac 1 -ar 22050 -f f32le /dev/shm/$core.pcm || return 1; py="$(python3 -c 'import vamp; import numpy as np; f = open("/dev/shm/'$core'.pcm", "rb"); d = np.fromfile(f, dtype=np.float32); c = vamp.collect(d, 22050, "vamp-example-plugins:fixedtempo", parameters={"maxdflen":40}); print(c["list"][0]["label"].split(" ")[0])')"; } +# ok: 119 1off: 5 (124) 10off: 8 (132) fail: 49 # 51 sec, vamp-example-fixedtempo +# ok: 109 1off: 4 (113) 10off: 9 (122) fail: 59 # bad-tags +afun() { ffmpeg -hide_banner -v fatal -nostdin -ss $((dur/3)) -y -i /mnt/Users/ed/Music/"$fn" -ac 1 -ar 22050 -f f32le /dev/shm/$core.pcm || return 1; py="$(python3 -c 'import vamp; import numpy as np; f = open("/dev/shm/'$core'.pcm", "rb"); d = np.fromfile(f, dtype=np.float32); c = vamp.collect(d, 22050, "vamp-example-plugins:fixedtempo", parameters={"maxdflen":40}); print(c["list"][0]["label"].split(" ")[0])')"; }; runfun -# ok: 80 1off: 48 10off: 11 fail: 42 # 61 sec, vamp-qm-tempotracker -afun() { ffmpeg -hide_banner -v fatal -nostdin -ss $((dur/3)) -y -i /mnt/Users/ed/Music/"$fn" -ac 1 -ar 22050 -f f32le /dev/shm/$core.pcm || return 1; py="$(python3 -c 'import vamp; import numpy as np; f = open("/dev/shm/'$core'.pcm", "rb"); d = np.fromfile(f, dtype=np.float32); c = vamp.collect(d, 22050, "qm-vamp-plugins:qm-tempotracker", parameters={"inputtempo":150}); v = [float(x["label"].split(" ")[0]) for x in c["list"] if x["label"]]; v = list(sorted(v))[len(v)//4:-len(v)//4]; print(round(sum(v) / len(v), 1))')"; } +# ok: 102 1off: 61 (163) 10off: 12 (175) fail: 6 # 61 sec, vamp-qm-tempotracker +# ok: 80 1off: 48 (128) 10off: 11 (139) fail: 42 # bad-tags +afun() { ffmpeg -hide_banner -v fatal -nostdin -ss $((dur/3)) -y -i /mnt/Users/ed/Music/"$fn" -ac 1 -ar 22050 -f f32le /dev/shm/$core.pcm || return 1; py="$(python3 -c 'import vamp; import numpy as np; f = open("/dev/shm/'$core'.pcm", "rb"); d = np.fromfile(f, dtype=np.float32); c = vamp.collect(d, 22050, "qm-vamp-plugins:qm-tempotracker", parameters={"inputtempo":150}); v = [float(x["label"].split(" ")[0]) for x in c["list"] if x["label"]]; v = list(sorted(v))[len(v)//4:-len(v)//4]; print(round(sum(v) / len(v), 1))')"; }; runfun -# ok: 101 1off: 22 10off: 16 fail: 39 # 51 sec, vamp-beatroot +# ok: 133 1off: 32 (165) 10off: 12 (177) fail: 3 # 51 sec, vamp-beatroot +# ok: 101 1off: 22 (123) 10off: 16 (139) fail: 39 # bad-tags # note: some tracks fully fail to analyze (unlike the others which always provide a guess) -afun() { ffmpeg -hide_banner -v fatal -nostdin -ss $((dur/3)) -y -i /mnt/Users/ed/Music/"$fn" -ac 1 -ar 22050 -f f32le /dev/shm/$core.pcm || return 1; py="$(python3 -c 'import vamp; import numpy as np; f = open("/dev/shm/'$core'.pcm", "rb"); d = np.fromfile(f, dtype=np.float32); c = vamp.collect(d, 22050, "beatroot-vamp:beatroot"); cl=c["list"]; print(round(60*((len(cl)-1)/(float(cl[-1]["timestamp"]-cl[1]["timestamp"]))), 2))')"; } +afun() { ffmpeg -hide_banner -v fatal -nostdin -ss $((dur/3)) -y -i /mnt/Users/ed/Music/"$fn" -ac 1 -ar 22050 -f f32le /dev/shm/$core.pcm || return 1; py="$(python3 -c 'import vamp; import numpy as np; f = open("/dev/shm/'$core'.pcm", "rb"); d = np.fromfile(f, dtype=np.float32); c = vamp.collect(d, 22050, "beatroot-vamp:beatroot"); cl=c["list"]; print(round(60*((len(cl)-1)/(float(cl[-1]["timestamp"]-cl[1]["timestamp"]))), 2))')"; }; runfun +# ok: 124 1off: 9 (133) 10off: 40 (173) fail: 8 # 231 sec, essentia/full +# ok: 109 1off: 8 (117) 10off: 22 (139) fail: 42 # bad-tags +afun() { ffmpeg -hide_banner -v fatal -nostdin -ss $((dur/3)) -y -i /mnt/Users/ed/Music/"$fn" -ac 1 -ar 44100 /dev/shm/$core.wav || return 1; py="$(python3 -c 'import essentia; import essentia.standard as es; fe, fef = es.MusicExtractor(lowlevelStats=["mean", "stdev"], rhythmStats=["mean", "stdev"], tonalStats=["mean", "stdev"])("/dev/shm/'$core'.wav"); print("{:.2f}".format(fe["rhythm.bpm"]))')"; }; runfun + + +# ok: 113 1off: 18 (131) 10off: 46 (177) fail: 4 # 134 sec, essentia/re2013 +# ok: 101 1off: 15 (116) 10off: 26 (142) fail: 39 # bad-tags +afun() { ffmpeg -hide_banner -v fatal -nostdin -ss $((dur/3)) -y -i /mnt/Users/ed/Music/"$fn" -ac 1 -ar 44100 /dev/shm/$core.wav || return 1; py="$(python3 -c 'from essentia.standard import *; a=MonoLoader(filename="/dev/shm/'$core'.wav")(); bpm,beats,confidence,_,intervals=RhythmExtractor2013(method="multifeature")(a); print("{:.2f}".format(bpm))')"; }; runfun + ######################################################################## ##