From 1e7de5d14f00f8821b1a32f3009f4328b1a5e04a Mon Sep 17 00:00:00 2001 From: ed Date: Thu, 23 Apr 2026 22:35:15 +0000 Subject: [PATCH] new hooks: reloc-by-wark; closes #1395 --- bin/hooks/README.md | 4 ++ bin/hooks/reloc-by-wark-xau.py | 92 ++++++++++++++++++++++++++++++++++ bin/hooks/reloc-by-wark-xbu.py | 69 +++++++++++++++++++++++++ copyparty/up2k.py | 31 ++++++++++-- copyparty/util.py | 9 ++-- 5 files changed, 199 insertions(+), 6 deletions(-) create mode 100644 bin/hooks/reloc-by-wark-xau.py create mode 100644 bin/hooks/reloc-by-wark-xbu.py diff --git a/bin/hooks/README.md b/bin/hooks/README.md index c0ce7bf1..04574248 100644 --- a/bin/hooks/README.md +++ b/bin/hooks/README.md @@ -38,6 +38,10 @@ these are `--xiu` hooks; unlike `xbu` and `xau` (which get executed on every sin * this hook uses the `I` flag which makes it 140x faster, but if the plugin has a bug it may crash copyparty +# more upload stuff +* combine [reloc-by-wark-xbu.py](reloc-by-wark-xbu.py) and [reloc-by-wark-xau.py](reloc-by-wark-xau.py) to rename uploads to the checksum of the file contents + + # on message * [wget.py](wget.py) lets you download files by POSTing URLs to copyparty * [wget-i.py](wget-i.py) is an import-safe modification of this hook (starts 140x faster, but higher chance of bugs) diff --git a/bin/hooks/reloc-by-wark-xau.py b/bin/hooks/reloc-by-wark-xau.py new file mode 100644 index 00000000..3779a350 --- /dev/null +++ b/bin/hooks/reloc-by-wark-xau.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 + +import os +import sys + + +_ = r""" +rename incoming uploads according to the "wark" (the file identifier) +which is basically but not exactly a sha512 hash of the file contents + +NOTE: this does NOT work with up2k uploads (dragdrop into browser); + combine this hook with reloc-by-wark-xbu.py to fix that + +example usage as global config: + -e2d --xau I,c,bin/hooks/reloc-by-wark-xau.py + +parameters explained, + e2d = enable up2k database (mandatory for xau hooks) + xau = execute before upload + I = import this hook for performance; do not fork / subprocess + c = "check"; reject upload if this hook crashes due to a bug + +example usage as a volflag (per-volume config): + -v srv/inc:inc:r:rw,ed:c,e2d,xau=I,c,bin/hooks/reloc-by-wark-xau.py + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + + (share filesystem-path srv/inc as volume /inc, + readable by everyone, read-write for user 'ed', + running this plugin on all uploads with the params explained above) + +example usage as a volflag in a copyparty config file: + [/inc] + srv/inc + accs: + r: * + rw: ed + flags: + e2d, xau: I,c,bin/hooks/reloc-by-wark-xau.py +""" + + +def main(inf): + if inf.get("wark"): + # this is an up2k upload; nothing to be done, just bail + return {} + + abspath = inf["ap"] # filesystem path to the uploaded file + + # we don't have the wark yet so need to calculate it; + # generating a regular sha512 would of course be much easier, + # but then filenames would be different depending on how the + # file was uploaded (laaame) so let's do it the hard way + + # use the standard up2k-salt which nobody ever changes: + salt = "hunter2" + + # to generate the wark we'll need some functions from copyparty; + # follow the trail to the copyparty module and grab them from there: + + import inspect + + libpath = inspect.getfile(inf["log"]) + libpath = os.path.dirname(os.path.dirname(libpath)) + sys.path.insert(0, libpath) + + from copyparty.up2k import up2k_hashlist_from_file, up2k_wark_from_hashlist + + chunklist, st = up2k_hashlist_from_file(abspath) + wark = up2k_wark_from_hashlist(salt, st.st_size, chunklist) + + # okay nice + # the rest of the code below is just copied from reloc-by-wark-xbu.py + # ------------------------------------------------------------------------- + + # grab the original filename from the vpath... + vdir, fn = os.path.split(inf["vp"]) + + # ...to retain the original file extension, if any + try: + fn, ext = fn.rsplit(".", 1) + except: + ext = "" + + # use the first 16 characters; 12 bytes of entropy, + # roughly one collision for every 26 million files + fn = wark[:16] + + if ext: + ext = ext.lower() + fn += "." + ext + + return {"reloc": {"fn": fn}} diff --git a/bin/hooks/reloc-by-wark-xbu.py b/bin/hooks/reloc-by-wark-xbu.py new file mode 100644 index 00000000..8576ad59 --- /dev/null +++ b/bin/hooks/reloc-by-wark-xbu.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 + +import os + + +_ = r""" +rename incoming uploads according to the "wark" (the file identifier) +which is basically but not exactly a sha512 hash of the file contents + +NOTE: this only works for up2k uploads (dragdrop into browser); + combine this with reloc-by-wark-xau.py to cover the other protocols + +example usage as global config: + -e2d --xbu I,c,bin/hooks/reloc-by-wark-xbu.py + +parameters explained, + e2d = enable up2k database (mandatory for xbu hooks) + xbu = execute before upload + I = import this hook for performance; do not fork / subprocess + c = "check"; reject upload if this hook crashes due to a bug + +example usage as a volflag (per-volume config): + -v srv/inc:inc:r:rw,ed:c,e2d,xbu=I,c,bin/hooks/reloc-by-wark-xbu.py + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + + (share filesystem-path srv/inc as volume /inc, + readable by everyone, read-write for user 'ed', + running this plugin on all uploads with the params explained above) + +example usage as a volflag in a copyparty config file: + [/inc] + srv/inc + accs: + r: * + rw: ed + flags: + e2d, xbu: I,c,bin/hooks/reloc-by-wark-xbu.py +""" + + +def main(inf): + wark = inf.get("wark") + if not wark: + # not an up2k upload, so we don't have the hash; + + # option 1: let upload proceed with original filename + return {} + + # option 2: reject the upload + return {"rejectmsg": "only up2k uploads are allowed in this volume"} + + # grab the original filename from the vpath... + vdir, fn = os.path.split(inf["vp"]) + + # ...to retain the original file extension, if any + try: + fn, ext = fn.rsplit(".", 1) + except: + ext = "" + + # use the first 16 characters; 12 bytes of entropy, + # roughly one collision for every 26 million files + fn = wark[:16] + + if ext: + ext = ext.lower() + fn += "." + ext + + return {"reloc": {"fn": fn}} diff --git a/copyparty/up2k.py b/copyparty/up2k.py index c3d7901d..35c3ae29 100644 --- a/copyparty/up2k.py +++ b/copyparty/up2k.py @@ -3357,7 +3357,7 @@ class Up2k(object): job["size"], job["addr"], job["at"], - None, + [dwark], ) t = hr.get("rejectmsg") or "" if t or hr.get("rc") != 0: @@ -4069,7 +4069,7 @@ class Up2k(object): sz, ip, at or time.time(), - None, + [dwark], ) t = hr.get("rejectmsg") or "" if t or hr.get("rc") != 0: @@ -5251,7 +5251,7 @@ class Up2k(object): job["size"], job["addr"], job["t0"], - None, + [job["dwrk"]], ) t = hr.get("rejectmsg") or "" if t or hr.get("rc") != 0: @@ -5708,6 +5708,31 @@ def up2k_chunksize(filesize: int) -> int: stepsize *= mul +def up2k_hashlist_from_file(path: str) -> tuple[list[str], os.stat_result]: + """not used by copyparty itself, only by some hooks""" + st = bos.stat(path) + fsz = st.st_size + csz = up2k_chunksize(fsz) + ret = [] + with open(fsenc(path), "rb", 256*1024) as f: + while fsz > 0: + hashobj = hashlib.sha512() + rem = min(csz, fsz) + fsz -= rem + while rem > 0: + buf = f.read(min(rem, 64 * 1024)) + if not buf: + raise Exception("EOF at " + str(f.tell())) + + hashobj.update(buf) + rem -= len(buf) + + digest = hashobj.digest()[:33] + ret.append(ub64enc(digest).decode("ascii")) + + return ret, st + + def up2k_wark_from_hashlist(salt: str, filesize: int, hashes: list[str]) -> str: """server-reproducible file identifier, independent of name or location""" values = [salt, str(filesize)] + hashes diff --git a/copyparty/util.py b/copyparty/util.py index 00260140..6138aeae 100644 --- a/copyparty/util.py +++ b/copyparty/util.py @@ -4133,8 +4133,11 @@ def _runhook( "src": src, } if txt: - ja["txt"] = txt[0] - ja["body"] = txt[1] + if src in ("xm", "xban"): + ja["txt"] = txt[0] + ja["body"] = txt[1] + else: + ja["wark"] = txt[0] # acshually the dwark but less confusing if imp: ja["log"] = log mod = loadpy(acmd[0], False) @@ -4247,7 +4250,7 @@ def runhook( else: ret[k] = v except Exception as ex: - (log or print)("hook: %r, %s" % (ex, ex)) + (log or print)("hook failed; %s:\n%s" % (ex, min_ex())) if ",c," in "," + cmd: return {"rc": 1} break