mirror of
https://github.com/9001/copyparty.git
synced 2025-08-17 09:02:15 -06:00
better task recovery on restart
This commit is contained in:
parent
f991f74983
commit
f5cb9f92b9
8
bin/mtag/sleep.py
Normal file
8
bin/mtag/sleep.py
Normal file
|
@ -0,0 +1,8 @@
|
||||||
|
#!/usr/bin/env python
|
||||||
|
|
||||||
|
import time
|
||||||
|
import random
|
||||||
|
|
||||||
|
v = random.random() * 6
|
||||||
|
time.sleep(v)
|
||||||
|
print(f"{v:.2f}")
|
|
@ -469,7 +469,7 @@ class Up2k(object):
|
||||||
n_tags = self._tag_file(c3, *args)
|
n_tags = self._tag_file(c3, *args)
|
||||||
else:
|
else:
|
||||||
mpool.put(["mtag"] + args)
|
mpool.put(["mtag"] + args)
|
||||||
n_tags = self._flush_mpool(c3)
|
n_tags = len(self._flush_mpool(c3))
|
||||||
|
|
||||||
n_add += n_tags
|
n_add += n_tags
|
||||||
n_buf += n_tags
|
n_buf += n_tags
|
||||||
|
@ -490,10 +490,10 @@ class Up2k(object):
|
||||||
|
|
||||||
def _flush_mpool(self, wcur):
|
def _flush_mpool(self, wcur):
|
||||||
with self.mutex:
|
with self.mutex:
|
||||||
ret = 0
|
ret = []
|
||||||
for x in self.pending_tags:
|
for x in self.pending_tags:
|
||||||
self._tag_file(wcur, *x)
|
self._tag_file(wcur, *x)
|
||||||
ret += 1
|
ret.append(x[1])
|
||||||
|
|
||||||
self.pending_tags = []
|
self.pending_tags = []
|
||||||
return ret
|
return ret
|
||||||
|
@ -552,6 +552,8 @@ class Up2k(object):
|
||||||
t_prev = time.time()
|
t_prev = time.time()
|
||||||
n_prev = n_left
|
n_prev = n_left
|
||||||
n_done = 0
|
n_done = 0
|
||||||
|
to_delete = {}
|
||||||
|
in_progress = {}
|
||||||
while True:
|
while True:
|
||||||
with self.mutex:
|
with self.mutex:
|
||||||
q = "select w from mt where k = 't:mtp' limit ?"
|
q = "select w from mt where k = 't:mtp' limit ?"
|
||||||
|
@ -559,9 +561,6 @@ class Up2k(object):
|
||||||
warks = [x[0] for x in warks]
|
warks = [x[0] for x in warks]
|
||||||
jobs = []
|
jobs = []
|
||||||
for w in warks:
|
for w in warks:
|
||||||
q = "delete from mt where w = ? and k = 't:mtp'"
|
|
||||||
cur.execute(q, (w,))
|
|
||||||
|
|
||||||
q = "select rd, fn from up where substr(w,1,16)=? limit 1"
|
q = "select rd, fn from up where substr(w,1,16)=? limit 1"
|
||||||
rd, fn = cur.execute(q, (w,)).fetchone()
|
rd, fn = cur.execute(q, (w,)).fetchone()
|
||||||
rd, fn = s3dec(rd, fn)
|
rd, fn = s3dec(rd, fn)
|
||||||
|
@ -573,15 +572,32 @@ class Up2k(object):
|
||||||
|
|
||||||
if ".dur" not in have:
|
if ".dur" not in have:
|
||||||
# skip non-audio
|
# skip non-audio
|
||||||
|
to_delete[w] = True
|
||||||
n_left -= 1
|
n_left -= 1
|
||||||
continue
|
continue
|
||||||
|
|
||||||
|
if w in in_progress:
|
||||||
|
continue
|
||||||
|
|
||||||
task_parsers = {
|
task_parsers = {
|
||||||
k: v for k, v in parsers.items() if k in force or k not in have
|
k: v for k, v in parsers.items() if k in force or k not in have
|
||||||
}
|
}
|
||||||
jobs.append([task_parsers, None, w, abspath])
|
jobs.append([task_parsers, None, w, abspath])
|
||||||
|
in_progress[w] = True
|
||||||
|
|
||||||
n_done += self._flush_mpool(wcur)
|
done = self._flush_mpool(wcur)
|
||||||
|
|
||||||
|
with self.mutex:
|
||||||
|
for w in done:
|
||||||
|
to_delete[w] = True
|
||||||
|
in_progress.pop(w)
|
||||||
|
n_done += 1
|
||||||
|
|
||||||
|
for w in to_delete.keys():
|
||||||
|
q = "delete from mt where w = ? and k = 't:mtp'"
|
||||||
|
cur.execute(q, (w,))
|
||||||
|
|
||||||
|
to_delete = {}
|
||||||
|
|
||||||
if not warks:
|
if not warks:
|
||||||
break
|
break
|
||||||
|
@ -612,8 +628,12 @@ class Up2k(object):
|
||||||
with self.mutex:
|
with self.mutex:
|
||||||
cur.connection.commit()
|
cur.connection.commit()
|
||||||
|
|
||||||
self._stop_mpool(mpool, wcur)
|
done = self._stop_mpool(mpool, wcur)
|
||||||
with self.mutex:
|
with self.mutex:
|
||||||
|
for w in done:
|
||||||
|
q = "delete from mt where w = ? and k = 't:mtp'"
|
||||||
|
cur.execute(q, (w,))
|
||||||
|
|
||||||
cur.connection.commit()
|
cur.connection.commit()
|
||||||
if n_done:
|
if n_done:
|
||||||
self.vac(cur, db_path, n_done, 0, sz0)
|
self.vac(cur, db_path, n_done, 0, sz0)
|
||||||
|
@ -654,13 +674,15 @@ class Up2k(object):
|
||||||
mpool.put(None)
|
mpool.put(None)
|
||||||
|
|
||||||
mpool.join()
|
mpool.join()
|
||||||
self._flush_mpool(wcur)
|
done = self._flush_mpool(wcur)
|
||||||
if WINDOWS and False:
|
if WINDOWS and False:
|
||||||
nah = open(os.devnull, "wb")
|
nah = open(os.devnull, "wb")
|
||||||
wmic = f"processid={os.getpid()}"
|
wmic = f"processid={os.getpid()}"
|
||||||
wmic = ["wmic", "process", "where", wmic, "call", "setpriority"]
|
wmic = ["wmic", "process", "where", wmic, "call", "setpriority"]
|
||||||
sp.call(wmic + ["below normal"], stdout=nah, stderr=nah)
|
sp.call(wmic + ["below normal"], stdout=nah, stderr=nah)
|
||||||
|
|
||||||
|
return done
|
||||||
|
|
||||||
def _tag_thr(self, q):
|
def _tag_thr(self, q):
|
||||||
while True:
|
while True:
|
||||||
task = q.get()
|
task = q.get()
|
||||||
|
|
|
@ -78,6 +78,16 @@ time sqlite3 up2k.db 'select mt1.w from mt mt1 inner join mt mt2 on mt1.w = mt2.
|
||||||
cat warks | while IFS= read -r x; do sqlite3 up2k.db "delete from mt where w = '$x'"; done
|
cat warks | while IFS= read -r x; do sqlite3 up2k.db "delete from mt where w = '$x'"; done
|
||||||
|
|
||||||
|
|
||||||
|
##
|
||||||
|
## media
|
||||||
|
|
||||||
|
# split track into test files
|
||||||
|
e=6; s=10; d=~/dev/copyparty/srv/aus; n=1; p=0; e=$((e*60)); rm -rf $d; mkdir $d; while true; do ffmpeg -hide_banner -ss $p -i 'nervous_testpilot - office.mp3' -c copy -t $s $d/$(printf %04d $n).mp3; n=$((n+1)); p=$((p+s)); [ $p -gt $e ] && break; done
|
||||||
|
|
||||||
|
-v srv/aus:aus:r:ce2dsa:ce2ts:cmtp=fgsfds=bin/mtag/sleep.py
|
||||||
|
sqlite3 .hist/up2k.db 'select * from mt where k="fgsfds" or k="t:mtp"' | tee /dev/stderr | wc -l
|
||||||
|
|
||||||
|
|
||||||
##
|
##
|
||||||
## vscode
|
## vscode
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue