mirrors/copyparty
1
0
Fork 0
mirror of https://github.com/9001/copyparty.git synced 2025-08-18 01:22:13 -06:00
Code Issues Actions Packages Projects Releases Wiki Activity

do up2k snapshot on shutdown

Browse source
This commit is contained in:
ed 2021-07-17 14:48:35 +02:00
parent 5c7debd900
commit 3ca41be2b4
2 changed files with 22 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
copyparty/svchub.py
View file

@ -181,6 +181,7 @@ class SvcHub(object):
self.tcpsrv.shutdown() self.tcpsrv.shutdown()
self.broker.shutdown() self.broker.shutdown()
self.up2k.shutdown()
if self.thumbsrv: if self.thumbsrv:
self.thumbsrv.shutdown() self.thumbsrv.shutdown()

34
copyparty/up2k.py
View file

@ -1363,19 +1363,22 @@ class Up2k(object):
self.log("could not unsparse [{}]".format(path), 3) self.log("could not unsparse [{}]".format(path), 3)
def _snapshot(self): def _snapshot(self):
persist_interval = 30 # persist unfinished uploads index every 30 sec self.snap_persist_interval = 300 # persist unfinished index every 5 min
discard_interval = 21600 # drop unfinished uploads after 6 hours inactivity self.snap_discard_interval = 21600 # drop unfinished after 6 hours inactivity
prev = {} self.snap_prev = {}
while True: while True:
time.sleep(persist_interval) time.sleep(self.snap_persist_interval)
with self.mutex: self.do_snapshot()
for k, reg in self.registry.items():
self._snap_reg(prev, k, reg, discard_interval)
def _snap_reg(self, prev, ptop, reg, discard_interval): def do_snapshot(self):
with self.mutex:
for k, reg in self.registry.items():
self._snap_reg(k, reg)
def _snap_reg(self, ptop, reg):
now = time.time() now = time.time()
histpath = self.asrv.vfs.histtab[ptop] histpath = self.asrv.vfs.histtab[ptop]
rm = [x for x in reg.values() if now - x["poke"] > discard_interval] rm = [x for x in reg.values() if now - x["poke"] > self.snap_discard_interval]
if rm: if rm:
m = "dropping {} abandoned uploads in {}".format(len(rm), ptop) m = "dropping {} abandoned uploads in {}".format(len(rm), ptop)
vis = [self._vis_job_progress(x) for x in rm] vis = [self._vis_job_progress(x) for x in rm]
@ -1397,15 +1400,15 @@ class Up2k(object):
path = os.path.join(histpath, "up2k.snap") path = os.path.join(histpath, "up2k.snap")
if not reg: if not reg:
if ptop not in prev or prev[ptop] is not None: if ptop not in self.snap_prev or self.snap_prev[ptop] is not None:
prev[ptop] = None self.snap_prev[ptop] = None
if os.path.exists(fsenc(path)): if os.path.exists(fsenc(path)):
os.unlink(fsenc(path)) os.unlink(fsenc(path))
return return
newest = max(x["poke"] for _, x in reg.items()) if reg else 0 newest = max(x["poke"] for _, x in reg.items()) if reg else 0
etag = [len(reg), newest] etag = [len(reg), newest]
if etag == prev.get(ptop): if etag == self.snap_prev.get(ptop):
return return
try: try:
@ -1421,7 +1424,7 @@ class Up2k(object):
atomic_move(path2, path) atomic_move(path2, path)
self.log("snap: {} |{}|".format(path, len(reg.keys()))) self.log("snap: {} |{}|".format(path, len(reg.keys())))
prev[ptop] = etag self.snap_prev[ptop] = etag
def _tagger(self): def _tagger(self):
with self.mutex: with self.mutex:
@ -1489,6 +1492,11 @@ class Up2k(object):
self.n_hashq += 1 self.n_hashq += 1
# self.log("hashq {} push {}/{}/{}".format(self.n_hashq, ptop, rd, fn)) # self.log("hashq {} push {}/{}/{}".format(self.n_hashq, ptop, rd, fn))
def shutdown(self):
if hasattr(self, "snap_prev"):
self.log("writing snapshot")
self.do_snapshot()
def up2k_chunksize(filesize): def up2k_chunksize(filesize):
chunksize = 1024 * 1024 chunksize = 1024 * 1024