better task recovery on restart

This commit is contained in:
ed 2021-03-21 05:57:24 +01:00
parent f991f74983
commit f5cb9f92b9
3 changed files with 49 additions and 9 deletions

8
bin/mtag/sleep.py Normal file
View file

@ -0,0 +1,8 @@
#!/usr/bin/env python
import time
import random
v = random.random() * 6
time.sleep(v)
print(f"{v:.2f}")

View file

@ -469,7 +469,7 @@ class Up2k(object):
n_tags = self._tag_file(c3, *args) n_tags = self._tag_file(c3, *args)
else: else:
mpool.put(["mtag"] + args) mpool.put(["mtag"] + args)
n_tags = self._flush_mpool(c3) n_tags = len(self._flush_mpool(c3))
n_add += n_tags n_add += n_tags
n_buf += n_tags n_buf += n_tags
@ -490,10 +490,10 @@ class Up2k(object):
def _flush_mpool(self, wcur): def _flush_mpool(self, wcur):
with self.mutex: with self.mutex:
ret = 0 ret = []
for x in self.pending_tags: for x in self.pending_tags:
self._tag_file(wcur, *x) self._tag_file(wcur, *x)
ret += 1 ret.append(x[1])
self.pending_tags = [] self.pending_tags = []
return ret return ret
@ -552,6 +552,8 @@ class Up2k(object):
t_prev = time.time() t_prev = time.time()
n_prev = n_left n_prev = n_left
n_done = 0 n_done = 0
to_delete = {}
in_progress = {}
while True: while True:
with self.mutex: with self.mutex:
q = "select w from mt where k = 't:mtp' limit ?" q = "select w from mt where k = 't:mtp' limit ?"
@ -559,9 +561,6 @@ class Up2k(object):
warks = [x[0] for x in warks] warks = [x[0] for x in warks]
jobs = [] jobs = []
for w in warks: for w in warks:
q = "delete from mt where w = ? and k = 't:mtp'"
cur.execute(q, (w,))
q = "select rd, fn from up where substr(w,1,16)=? limit 1" q = "select rd, fn from up where substr(w,1,16)=? limit 1"
rd, fn = cur.execute(q, (w,)).fetchone() rd, fn = cur.execute(q, (w,)).fetchone()
rd, fn = s3dec(rd, fn) rd, fn = s3dec(rd, fn)
@ -573,15 +572,32 @@ class Up2k(object):
if ".dur" not in have: if ".dur" not in have:
# skip non-audio # skip non-audio
to_delete[w] = True
n_left -= 1 n_left -= 1
continue continue
if w in in_progress:
continue
task_parsers = { task_parsers = {
k: v for k, v in parsers.items() if k in force or k not in have k: v for k, v in parsers.items() if k in force or k not in have
} }
jobs.append([task_parsers, None, w, abspath]) jobs.append([task_parsers, None, w, abspath])
in_progress[w] = True
n_done += self._flush_mpool(wcur) done = self._flush_mpool(wcur)
with self.mutex:
for w in done:
to_delete[w] = True
in_progress.pop(w)
n_done += 1
for w in to_delete.keys():
q = "delete from mt where w = ? and k = 't:mtp'"
cur.execute(q, (w,))
to_delete = {}
if not warks: if not warks:
break break
@ -612,8 +628,12 @@ class Up2k(object):
with self.mutex: with self.mutex:
cur.connection.commit() cur.connection.commit()
self._stop_mpool(mpool, wcur) done = self._stop_mpool(mpool, wcur)
with self.mutex: with self.mutex:
for w in done:
q = "delete from mt where w = ? and k = 't:mtp'"
cur.execute(q, (w,))
cur.connection.commit() cur.connection.commit()
if n_done: if n_done:
self.vac(cur, db_path, n_done, 0, sz0) self.vac(cur, db_path, n_done, 0, sz0)
@ -654,13 +674,15 @@ class Up2k(object):
mpool.put(None) mpool.put(None)
mpool.join() mpool.join()
self._flush_mpool(wcur) done = self._flush_mpool(wcur)
if WINDOWS and False: if WINDOWS and False:
nah = open(os.devnull, "wb") nah = open(os.devnull, "wb")
wmic = f"processid={os.getpid()}" wmic = f"processid={os.getpid()}"
wmic = ["wmic", "process", "where", wmic, "call", "setpriority"] wmic = ["wmic", "process", "where", wmic, "call", "setpriority"]
sp.call(wmic + ["below normal"], stdout=nah, stderr=nah) sp.call(wmic + ["below normal"], stdout=nah, stderr=nah)
return done
def _tag_thr(self, q): def _tag_thr(self, q):
while True: while True:
task = q.get() task = q.get()

View file

@ -78,6 +78,16 @@ time sqlite3 up2k.db 'select mt1.w from mt mt1 inner join mt mt2 on mt1.w = mt2.
cat warks | while IFS= read -r x; do sqlite3 up2k.db "delete from mt where w = '$x'"; done cat warks | while IFS= read -r x; do sqlite3 up2k.db "delete from mt where w = '$x'"; done
##
## media
# split track into test files
e=6; s=10; d=~/dev/copyparty/srv/aus; n=1; p=0; e=$((e*60)); rm -rf $d; mkdir $d; while true; do ffmpeg -hide_banner -ss $p -i 'nervous_testpilot - office.mp3' -c copy -t $s $d/$(printf %04d $n).mp3; n=$((n+1)); p=$((p+s)); [ $p -gt $e ] && break; done
-v srv/aus:aus:r:ce2dsa:ce2ts:cmtp=fgsfds=bin/mtag/sleep.py
sqlite3 .hist/up2k.db 'select * from mt where k="fgsfds" or k="t:mtp"' | tee /dev/stderr | wc -l
## ##
## vscode ## vscode