mirror of
https://github.com/9001/copyparty.git
synced 2025-08-17 09:02:15 -06:00
optimize non-e2d ram usage down to 10% or so
drop chunk-hashes in the up2k snap, plus other insignificant attribs to reduce both the snapfile size and the ram usage by about 90% reduces startup/shutdown time by a lot since there's less to serdes (does not affect -e2d which was already optimal) other changes: * improve incoming-eta accuracy when the initial handshake was made a long time before the upload actually started * move the list of incoming files in the controlpanel to the top
This commit is contained in:
ed
parent
047176b297
commit
88a1c5ca5d
|
|
@ -328,7 +328,7 @@ class Up2k(object):
|
|||
zt = (
|
||||
ineed / ihash,
|
||||
job["size"],
|
||||
int(job["t0"]),
|
||||
int(job["t0c"]),
|
||||
int(job["poke"]),
|
||||
djoin(vtop, job["prel"], job["name"]),
|
||||
)
|
||||
|
|
@ -364,10 +364,9 @@ class Up2k(object):
|
|||
continue
|
||||
addr = (ip or "\n") if cfg in (1, 2) else ""
|
||||
user = (uname or "\n") if cfg in (1, 3) else ""
|
||||
drp = self.droppable.get(ptop, {})
|
||||
for wark, job in tab2.items():
|
||||
for job in tab2.values():
|
||||
if (
|
||||
wark in drp
|
||||
"done" in job
|
||||
or (user and user != job["user"])
|
||||
or (addr and addr != job["addr"])
|
||||
):
|
||||
|
|
@ -408,9 +407,8 @@ class Up2k(object):
|
|||
for ptop, tab2 in self.registry.items():
|
||||
nbytes = 0
|
||||
nfiles = 0
|
||||
drp = self.droppable.get(ptop, {})
|
||||
for wark, job in tab2.items():
|
||||
if wark in drp:
|
||||
for job in tab2.values():
|
||||
if "done" in job:
|
||||
continue
|
||||
|
||||
nfiles += 1
|
||||
|
|
@ -1057,6 +1055,7 @@ class Up2k(object):
|
|||
|
||||
reg = {}
|
||||
drp = None
|
||||
emptylist = []
|
||||
snap = os.path.join(histpath, "up2k.snap")
|
||||
if bos.path.exists(snap):
|
||||
with gzip.GzipFile(snap, "rb") as f:
|
||||
|
|
@ -1073,6 +1072,9 @@ class Up2k(object):
|
|||
fp = djoin(job["ptop"], job["prel"], job["name"])
|
||||
if bos.path.exists(fp):
|
||||
reg[k] = job
|
||||
if "done" in job:
|
||||
job["need"] = job["hash"] = emptylist
|
||||
continue
|
||||
job["poke"] = time.time()
|
||||
job["busy"] = {}
|
||||
else:
|
||||
|
|
@ -3021,7 +3023,7 @@ class Up2k(object):
|
|||
|
||||
job = deepcopy(job)
|
||||
job["wark"] = wark
|
||||
job["at"] = cj.get("at") or time.time()
|
||||
job["at"] = cj.get("at") or now
|
||||
zs = "vtop ptop prel name lmod host user addr poke"
|
||||
for k in zs.split():
|
||||
job[k] = cj.get(k) or ""
|
||||
|
|
@ -3346,6 +3348,9 @@ class Up2k(object):
|
|||
self.log("unknown wark [{}], known: {}".format(wark, known))
|
||||
raise Pebkac(400, "unknown wark" + SSEELOG)
|
||||
|
||||
if "t0c" not in job:
|
||||
job["t0c"] = time.time()
|
||||
|
||||
if len(chashes) > 1 and len(chashes[1]) < 44:
|
||||
# first hash is full-length; expand remaining ones
|
||||
uniq = []
|
||||
|
|
@ -3527,7 +3532,11 @@ class Up2k(object):
|
|||
if self.idx_wark(vflags, *z2):
|
||||
del self.registry[ptop][wark]
|
||||
else:
|
||||
self.registry[ptop][wark]["done"] = 1
|
||||
for k in "host tnam busy sprs poke t0c".split():
|
||||
del job[k]
|
||||
job["t0"] = int(job["t0"])
|
||||
job["hash"] = []
|
||||
job["done"] = 1
|
||||
self.regdrop(ptop, wark)
|
||||
|
||||
if wake_sr:
|
||||
|
|
@ -4720,7 +4729,11 @@ class Up2k(object):
|
|||
bos.unlink(path)
|
||||
return
|
||||
|
||||
newest = float(max(x["poke"] for _, x in reg.items()) if reg else 0)
|
||||
newest = float(
|
||||
max(x["t0"] if "done" in x else x["poke"] for _, x in reg.items())
|
||||
if reg
|
||||
else 0
|
||||
)
|
||||
etag = (len(reg), newest)
|
||||
if etag == self.snap_prev.get(ptop):
|
||||
return
|
||||
|
|
@ -4731,12 +4744,15 @@ class Up2k(object):
|
|||
path2 = "{}.{}".format(path, os.getpid())
|
||||
body = {"droppable": self.droppable[ptop], "registry": reg}
|
||||
j = json.dumps(body, sort_keys=True, separators=(",\n", ": ")).encode("utf-8")
|
||||
# j = re.sub(r'"(need|hash)": \[\],\n', "", j) # bytes=slow, utf8=hungry
|
||||
j = j.replace(b'"need": [],\n', b"") # surprisingly optimal
|
||||
j = j.replace(b'"hash": [],\n', b"")
|
||||
with gzip.GzipFile(path2, "wb") as f:
|
||||
f.write(j)
|
||||
|
||||
atomic_move(self.log, path2, path, VF_CAREFUL)
|
||||
|
||||
self.log("snap: {} |{}|".format(path, len(reg.keys())))
|
||||
self.log("snap: %s |%d| %.2fs" % (path, len(reg), time.time() - now))
|
||||
self.snap_prev[ptop] = etag
|
||||
|
||||
def _tagger(self) -> None:
|
||||
|
|
|
|||
|
|
@ -32,6 +32,18 @@
|
|||
</div>
|
||||
{%- endif %}
|
||||
|
||||
{%- if ups %}
|
||||
<h1 id="aa">incoming files:</h1>
|
||||
<table class="vols">
|
||||
<thead><tr><th>%</th><th>speed</th><th>eta</th><th>idle</th><th>dir</th><th>file</th></tr></thead>
|
||||
<tbody>
|
||||
{% for u in ups %}
|
||||
<tr><td>{{ u[0] }}</td><td>{{ u[1] }}</td><td>{{ u[2] }}</td><td>{{ u[3] }}</td><td><a href="{{ u[4] }}">{{ u[5]|e }}</a></td><td>{{ u[6]|e }}</td></tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
{%- endif %}
|
||||
|
||||
{%- if avol %}
|
||||
<h1>admin panel:</h1>
|
||||
<table><tr><td> <!-- hehehe -->
|
||||
|
|
@ -60,18 +72,6 @@
|
|||
</div>
|
||||
{%- endif %}
|
||||
|
||||
{%- if ups %}
|
||||
<h1 id="aa">incoming files:</h1>
|
||||
<table class="vols">
|
||||
<thead><tr><th>%</th><th>speed</th><th>eta</th><th>idle</th><th>dir</th><th>file</th></tr></thead>
|
||||
<tbody>
|
||||
{% for u in ups %}
|
||||
<tr><td>{{ u[0] }}</td><td>{{ u[1] }}</td><td>{{ u[2] }}</td><td>{{ u[3] }}</td><td><a href="{{ u[4] }}">{{ u[5]|e }}</a></td><td>{{ u[6]|e }}</td></tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
{%- endif %}
|
||||
|
||||
{%- if rvol %}
|
||||
<h1 id="f">you can browse:</h1>
|
||||
<ul>
|
||||
|
|
|
|||
Loading…
Reference in a new issue