new hooks: reloc-by-wark; closes #1395

bin/hooks/README.md

@@ -38,6 +38,10 @@ these are `--xiu` hooks; unlike `xbu` and `xau` (which get executed on every sin
   * this hook uses the `I` flag which makes it 140x faster, but if the plugin has a bug it may crash copyparty
 
 
+# more upload stuff
+* combine [reloc-by-wark-xbu.py](reloc-by-wark-xbu.py) and [reloc-by-wark-xau.py](reloc-by-wark-xau.py) to rename uploads to the checksum of the file contents
+
+
 # on message
 * [wget.py](wget.py) lets you download files by POSTing URLs to copyparty
   * [wget-i.py](wget-i.py) is an import-safe modification of this hook (starts 140x faster, but higher chance of bugs)

bin/hooks/reloc-by-wark-xau.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+
+
+_ = r"""
+rename incoming uploads according to the "wark" (the file identifier)
+which is basically but not exactly a sha512 hash of the file contents
+
+NOTE: this does NOT work with up2k uploads (dragdrop into browser);
+  combine this hook with reloc-by-wark-xbu.py to fix that
+
+example usage as global config:
+    -e2d --xau I,c,bin/hooks/reloc-by-wark-xau.py
+
+parameters explained,
+    e2d = enable up2k database (mandatory for xau hooks)
+    xau = execute before upload
+    I   = import this hook for performance; do not fork / subprocess
+    c   = "check"; reject upload if this hook crashes due to a bug
+
+example usage as a volflag (per-volume config):
+    -v srv/inc:inc:r:rw,ed:c,e2d,xau=I,c,bin/hooks/reloc-by-wark-xau.py
+                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+    (share filesystem-path srv/inc as volume /inc,
+     readable by everyone, read-write for user 'ed',
+     running this plugin on all uploads with the params explained above)
+
+example usage as a volflag in a copyparty config file:
+    [/inc]
+      srv/inc
+      accs:
+        r: *
+        rw: ed
+      flags:
+        e2d, xau: I,c,bin/hooks/reloc-by-wark-xau.py
+"""
+
+
+def main(inf):
+    if inf.get("wark"):
+        # this is an up2k upload; nothing to be done, just bail
+        return {}
+
+    abspath = inf["ap"]  # filesystem path to the uploaded file
+
+    # we don't have the wark yet so need to calculate it;
+    # generating a regular sha512 would of course be much easier,
+    # but then filenames would be different depending on how the
+    # file was uploaded (laaame) so let's do it the hard way
+
+    # use the standard up2k-salt which nobody ever changes:
+    salt = "hunter2"
+
+    # to generate the wark we'll need some functions from copyparty;
+    # follow the trail to the copyparty module and grab them from there:
+
+    import inspect
+
+    libpath = inspect.getfile(inf["log"])
+    libpath = os.path.dirname(os.path.dirname(libpath))
+    sys.path.insert(0, libpath)
+
+    from copyparty.up2k import up2k_hashlist_from_file, up2k_wark_from_hashlist
+
+    chunklist, st = up2k_hashlist_from_file(abspath)
+    wark = up2k_wark_from_hashlist(salt, st.st_size, chunklist)
+
+    # okay nice
+    # the rest of the code below is just copied from reloc-by-wark-xbu.py
+    # -------------------------------------------------------------------------
+
+    # grab the original filename from the vpath...
+    vdir, fn = os.path.split(inf["vp"])
+
+    # ...to retain the original file extension, if any
+    try:
+        fn, ext = fn.rsplit(".", 1)
+    except:
+        ext = ""
+
+    # use the first 16 characters; 12 bytes of entropy,
+    # roughly one collision for every 26 million files
+    fn = wark[:16]
+
+    if ext:
+        ext = ext.lower()
+        fn += "." + ext
+
+    return {"reloc": {"fn": fn}}

bin/hooks/reloc-by-wark-xbu.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+
+import os
+
+
+_ = r"""
+rename incoming uploads according to the "wark" (the file identifier)
+which is basically but not exactly a sha512 hash of the file contents
+
+NOTE: this only works for up2k uploads (dragdrop into browser);
+  combine this with reloc-by-wark-xau.py to cover the other protocols
+
+example usage as global config:
+    -e2d --xbu I,c,bin/hooks/reloc-by-wark-xbu.py
+
+parameters explained,
+    e2d = enable up2k database (mandatory for xbu hooks)
+    xbu = execute before upload
+    I   = import this hook for performance; do not fork / subprocess
+    c   = "check"; reject upload if this hook crashes due to a bug
+
+example usage as a volflag (per-volume config):
+    -v srv/inc:inc:r:rw,ed:c,e2d,xbu=I,c,bin/hooks/reloc-by-wark-xbu.py
+                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+    (share filesystem-path srv/inc as volume /inc,
+     readable by everyone, read-write for user 'ed',
+     running this plugin on all uploads with the params explained above)
+
+example usage as a volflag in a copyparty config file:
+    [/inc]
+      srv/inc
+      accs:
+        r: *
+        rw: ed
+      flags:
+        e2d, xbu: I,c,bin/hooks/reloc-by-wark-xbu.py
+"""
+
+
+def main(inf):
+    wark = inf.get("wark")
+    if not wark:
+        # not an up2k upload, so we don't have the hash;
+
+        # option 1: let upload proceed with original filename
+        return {}
+
+        # option 2: reject the upload
+        return {"rejectmsg": "only up2k uploads are allowed in this volume"}
+
+    # grab the original filename from the vpath...
+    vdir, fn = os.path.split(inf["vp"])
+
+    # ...to retain the original file extension, if any
+    try:
+        fn, ext = fn.rsplit(".", 1)
+    except:
+        ext = ""
+
+    # use the first 16 characters; 12 bytes of entropy,
+    # roughly one collision for every 26 million files
+    fn = wark[:16]
+
+    if ext:
+        ext = ext.lower()
+        fn += "." + ext
+
+    return {"reloc": {"fn": fn}}

copyparty/up2k.py

@@ -3357,7 +3357,7 @@ class Up2k(object):
                                 job["size"],
                                 job["addr"],
                                 job["at"],
-                                None,
+                                [dwark],
                             )
                             t = hr.get("rejectmsg") or ""
                             if t or hr.get("rc") != 0:
@@ -4069,7 +4069,7 @@ class Up2k(object):
                 sz,
                 ip,
                 at or time.time(),
-                None,
+                [dwark],
             )
             t = hr.get("rejectmsg") or ""
             if t or hr.get("rc") != 0:
@@ -5251,7 +5251,7 @@ class Up2k(object):
                 job["size"],
                 job["addr"],
                 job["t0"],
-                None,
+                [job["dwrk"]],
             )
             t = hr.get("rejectmsg") or ""
             if t or hr.get("rc") != 0:
@@ -5708,6 +5708,31 @@ def up2k_chunksize(filesize: int) -> int:
             stepsize *= mul
 
 
+def up2k_hashlist_from_file(path: str) -> tuple[list[str], os.stat_result]:
+    """not used by copyparty itself, only by some hooks"""
+    st = bos.stat(path)
+    fsz = st.st_size
+    csz = up2k_chunksize(fsz)
+    ret = []
+    with open(fsenc(path), "rb", 256*1024) as f:
+        while fsz > 0:
+            hashobj = hashlib.sha512()
+            rem = min(csz, fsz)
+            fsz -= rem
+            while rem > 0:
+                buf = f.read(min(rem, 64 * 1024))
+                if not buf:
+                    raise Exception("EOF at " + str(f.tell()))
+
+                hashobj.update(buf)
+                rem -= len(buf)
+
+            digest = hashobj.digest()[:33]
+            ret.append(ub64enc(digest).decode("ascii"))
+
+    return ret, st
+
+
 def up2k_wark_from_hashlist(salt: str, filesize: int, hashes: list[str]) -> str:
     """server-reproducible file identifier, independent of name or location"""
     values = [salt, str(filesize)] + hashes

copyparty/util.py

@@ -4133,8 +4133,11 @@ def _runhook(
             "src": src,
         }
         if txt:
+            if src in ("xm", "xban"):
                 ja["txt"] = txt[0]
                 ja["body"] = txt[1]
+            else:
+                ja["wark"] = txt[0]  # acshually the dwark but less confusing
         if imp:
             ja["log"] = log
             mod = loadpy(acmd[0], False)
@@ -4247,7 +4250,7 @@ def runhook(
                 else:
                     ret[k] = v
         except Exception as ex:
-            (log or print)("hook: %r, %s" % (ex, ex))
+            (log or print)("hook failed; %s:\n%s" % (ex, min_ex()))
             if ",c," in "," + cmd:
                 return {"rc": 1}
             break