mirror of
https://github.com/9001/copyparty.git
synced 2025-10-31 04:32:20 -06:00
100 lines
2.9 KiB
Python
Executable file
100 lines
2.9 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
|
|
"""
|
|
unforget.py: rebuild db from logfiles
|
|
2022-09-07, v0.1, ed <irc.rizon.net>, MIT-Licensed
|
|
https://github.com/9001/copyparty/blob/hovudstraum/bin/unforget.py
|
|
|
|
only makes sense if running copyparty with --no-forget
|
|
(e.g. immediately shifting uploads to other storage)
|
|
|
|
usage:
|
|
xz -d < log | ./unforget.py .hist/up2k.db
|
|
|
|
"""
|
|
|
|
import re
|
|
import sys
|
|
import json
|
|
import base64
|
|
import sqlite3
|
|
import argparse
|
|
|
|
|
|
FS_ENCODING = sys.getfilesystemencoding()
|
|
|
|
|
|
class APF(argparse.ArgumentDefaultsHelpFormatter, argparse.RawDescriptionHelpFormatter):
|
|
pass
|
|
|
|
|
|
mem_cur = sqlite3.connect(":memory:").cursor()
|
|
mem_cur.execute(r"create table a (b text)")
|
|
|
|
|
|
def s3enc(rd: str, fn: str) -> tuple[str, str]:
|
|
ret: list[str] = []
|
|
for v in [rd, fn]:
|
|
try:
|
|
mem_cur.execute("select * from a where b = ?", (v,))
|
|
ret.append(v)
|
|
except:
|
|
wtf8 = v.encode(FS_ENCODING, "surrogateescape")
|
|
ret.append("//" + base64.urlsafe_b64encode(wtf8).decode("ascii"))
|
|
|
|
return ret[0], ret[1]
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("db")
|
|
ar = ap.parse_args()
|
|
|
|
db = sqlite3.connect(ar.db).cursor()
|
|
ptn_times = re.compile(r"no more chunks, setting times \(([0-9]+)")
|
|
at = 0
|
|
ctr = 0
|
|
|
|
for ln in [x.decode("utf-8", "replace").rstrip() for x in sys.stdin.buffer]:
|
|
if "no more chunks, setting times (" in ln:
|
|
m = ptn_times.search(ln)
|
|
if m:
|
|
at = int(m.group(1))
|
|
|
|
if '"hash": []' in ln:
|
|
try:
|
|
ofs = ln.find("{")
|
|
j = json.loads(ln[ofs:])
|
|
except:
|
|
continue
|
|
|
|
w = j["wark"]
|
|
if db.execute("select w from up where w = ?", (w,)).fetchone():
|
|
continue
|
|
|
|
# PYTHONPATH=/home/ed/dev/copyparty/ python3 -m copyparty -e2dsa -v foo:foo:rwmd,ed -aed:wark --no-forget
|
|
# 05:34:43.845 127.0.0.1 42496 no more chunks, setting times (1662528883, 1658001882)
|
|
# 05:34:43.863 127.0.0.1 42496 {"name": "f\"2", "purl": "/foo/bar/baz/", "size": 1674, "lmod": 1658001882, "sprs": true, "hash": [], "wark": "LKIWpp2jEAh9dH3fu-DobuURFGEKlODXDGTpZ1otMhUg"}
|
|
# | w | mt | sz | rd | fn | ip | at |
|
|
# | LKIWpp2jEAh9dH3fu-DobuURFGEKlODXDGTpZ1otMhUg | 1658001882 | 1674 | bar/baz | f"2 | 127.0.0.1 | 1662528883 |
|
|
|
|
rd, fn = s3enc(j["purl"].strip("/"), j["name"])
|
|
ip = ln.split(" ")[1].split("m")[-1]
|
|
|
|
q = "insert into up values (?,?,?,?,?,?,?)"
|
|
v = (w, int(j["lmod"]), int(j["size"]), rd, fn, ip, at)
|
|
db.execute(q, v)
|
|
ctr += 1
|
|
if ctr % 1024 == 1023:
|
|
print(f"{ctr} commit...")
|
|
db.connection.commit()
|
|
|
|
if ctr:
|
|
db.connection.commit()
|
|
|
|
print(f"unforgot {ctr} files")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|