hmac uploader-ip when avoiding filename collisions

copyparty/__main__.py

@@ -583,6 +583,7 @@ def run_argparse(argv: list[str], formatter: Any, retry: bool) -> argparse.Names
 
     ap2 = ap.add_argument_group('upload options')
     ap2.add_argument("--dotpart", action="store_true", help="dotfile incomplete uploads, hiding them from clients unless -ed")
+    ap2.add_argument("--plain-ip", action="store_true", help="when avoiding filename collisions by appending the uploader's ip to the filename: append the plaintext ip instead of salting and hashing the ip")
     ap2.add_argument("--unpost", metavar="SEC", type=int, default=3600*12, help="grace period where uploads can be deleted by the uploader, even without delete permissions; 0=disabled")
     ap2.add_argument("--reg-cap", metavar="N", type=int, default=38400, help="max number of uploads to keep in memory when running without -e2d; roughly 1 MiB RAM per 600")
     ap2.add_argument("--no-fpool", action="store_true", help="disable file-handle pooling -- instead, repeatedly close and reopen files during upload")

copyparty/broker_mpw.py

@@ -6,12 +6,13 @@ import signal
 import sys
 import threading
 
+import os
 import queue
 
 from .authsrv import AuthSrv
 from .broker_util import BrokerCli, ExceptionalQueue
 from .httpsrv import HttpSrv
-from .util import FAKE_MP
+from .util import FAKE_MP, HMaccas
 
 try:
     from types import FrameType
@@ -54,6 +55,7 @@ class MpWorker(BrokerCli):
         self.asrv = AuthSrv(args, None, False)
 
         # instantiate all services here (TODO: inheritance?)
+        self.iphash = HMaccas(os.path.join(self.args.E.cfg, "iphash"), 8)
         self.httpsrv = HttpSrv(self, n)
 
         # on winxp and some other platforms,

copyparty/broker_thr.py

@@ -1,11 +1,13 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
+import os
 import threading
 
 from .__init__ import TYPE_CHECKING
 from .broker_util import BrokerCli, ExceptionalQueue, try_exec
 from .httpsrv import HttpSrv
+from .util import HMaccas
 
 if TYPE_CHECKING:
     from .svchub import SvcHub
@@ -31,6 +33,7 @@ class BrokerThr(BrokerCli):
         self.num_workers = 1
 
         # instantiate all services here (TODO: inheritance?)
+        self.iphash = HMaccas(os.path.join(self.args.E.cfg, "iphash"), 8)
         self.httpsrv = HttpSrv(self, None)
         self.reload = self.noop
 

copyparty/broker_util.py

@@ -8,7 +8,7 @@ from queue import Queue
 
 from .__init__ import TYPE_CHECKING
 from .authsrv import AuthSrv
-from .util import Pebkac
+from .util import Pebkac, HMaccas
 
 try:
     from typing import Any, Optional, Union
@@ -46,6 +46,7 @@ class BrokerCli(object):
         self.args: argparse.Namespace = None
         self.asrv: AuthSrv = None
         self.httpsrv: "HttpSrv" = None
+        self.iphash: HMaccas = None
 
     def ask(self, dest: str, *args: Any) -> ExceptionalQueue:
         return ExceptionalQueue(1)

copyparty/httpcli.py

@@ -123,7 +123,6 @@ class HttpCli(object):
         self.ua = " "
         self.is_rclone = False
         self.is_ancient = False
-        self.dip = " "
         self.ouparam: dict[str, str] = {}
         self.uparam: dict[str, str] = {}
         self.cookies: dict[str, str] = {}
@@ -264,8 +263,6 @@ class HttpCli(object):
 
                 self.log_src = self.conn.set_rproxy(self.ip)
 
-        self.dip = self.ip.replace(":", ".")
-
         if self.args.ihead:
             keys = self.args.ihead
             if "*" in keys:
@@ -403,6 +400,12 @@ class HttpCli(object):
             except Pebkac:
                 return False
 
+    def dip(self) -> str:
+        if self.args.plain_ip:
+            return self.ip.replace(":", ".")
+        else:
+            return self.conn.iphash.s(self.ip)
+
     def permit_caching(self) -> None:
         cache = self.uparam.get("cache")
         if cache is None:
@@ -778,7 +781,7 @@ class HttpCli(object):
             else:
                 self.log("fallthrough? thats a bug", 1)
 
-        suffix = "-{:.6f}-{}".format(time.time(), self.dip)
+        suffix = "-{:.6f}-{}".format(time.time(), self.dip())
         if not fn:
             suffix += ".bin"
             fn = "put" + suffix
@@ -1262,6 +1265,7 @@ class HttpCli(object):
         files: list[tuple[int, str, str, str, str, str]] = []
         # sz, sha_hex, sha_b64, p_file, fname, abspath
         errmsg = ""
+        dip = self.dip()
         t0 = time.time()
         try:
             assert self.parser.gen
@@ -1278,7 +1282,7 @@ class HttpCli(object):
                     if not bos.path.isdir(fdir):
                         raise Pebkac(404, "that folder does not exist")
 
-                    suffix = "-{:.6f}-{}".format(time.time(), self.dip)
+                    suffix = "-{:.6f}-{}".format(time.time(), dip)
                     open_args = {"fdir": fdir, "suffix": suffix}
 
                     # reserve destination filename

copyparty/httpconn.py

@@ -23,7 +23,7 @@ from .mtag import HAVE_FFMPEG
 from .th_cli import ThumbCli
 from .th_srv import HAVE_PIL, HAVE_VIPS
 from .u2idx import U2idx
-from .util import shut_socket
+from .util import HMaccas, shut_socket
 
 try:
     from typing import Optional, Pattern, Union
@@ -54,6 +54,7 @@ class HttpConn(object):
         self.asrv: AuthSrv = hsrv.asrv  # mypy404
         self.cert_path = hsrv.cert_path
         self.u2fh: Util.FHC = hsrv.u2fh  # mypy404
+        self.iphash: HMaccas = hsrv.broker.iphash
 
         enth = (HAVE_PIL or HAVE_VIPS or HAVE_FFMPEG) and not self.args.no_thumb
         self.thumbcli: Optional[ThumbCli] = ThumbCli(hsrv) if enth else None  # mypy404

copyparty/svchub.py

@@ -32,6 +32,7 @@ from .th_srv import HAVE_PIL, HAVE_VIPS, HAVE_WEBP, ThumbSrv
 from .up2k import Up2k
 from .util import (
     VERSIONS,
+    HMaccas,
     alltrace,
     ansi_re,
     min_ex,
@@ -72,6 +73,8 @@ class SvcHub(object):
         self.next_day = 0
         self.tstack = 0.0
 
+        self.iphash = HMaccas(os.path.join(self.E.cfg, "iphash"), 8)
+
         if args.sss or args.s >= 3:
             args.ss = True
             args.lo = args.lo or "cpp-%Y-%m%d-%H%M%S.txt.xz"

copyparty/up2k.py

@@ -2101,9 +2101,12 @@ class Up2k(object):
         if self.args.nw:
             return fname
 
-        # TODO broker which avoid this race and
+        if self.args.plain_ip:
-        # provides a new filename if taken (same as bup)
+            dip = ip.replace(":", ".")
-        suffix = "-{:.6f}-{}".format(ts, ip.replace(":", "."))
+        else:
+            dip = self.hub.iphash.s(ip)
+
+        suffix = "-{:.6f}-{}".format(ts, dip)
         with ren_open(fname, "wb", fdir=fdir, suffix=suffix) as zfw:
             return zfw["orz"][1]
 
@@ -2842,7 +2845,11 @@ class Up2k(object):
                 del self.registry[job["ptop"]][job["wark"]]
             return
 
+        if self.args.plain_ip:
             dip = job["addr"].replace(":", ".")
+        else:
+            dip = self.hub.iphash.s(job["addr"])
+
         suffix = "-{:.6f}-{}".format(job["t0"], dip)
         with ren_open(tnam, "wb", fdir=pdir, suffix=suffix) as zfw:
             f, job["tnam"] = zfw["orz"]

copyparty/util.py

@@ -4,6 +4,7 @@ from __future__ import print_function, unicode_literals
 import base64
 import contextlib
 import hashlib
+import hmac
 import math
 import mimetypes
 import os
@@ -597,6 +598,33 @@ class MTHash(object):
         return nch, udig, ofs0, chunk_sz
 
 
+class HMaccas(object):
+    def __init__(self, keypath: str, retlen: int) -> None:
+        self.retlen = retlen
+        self.cache: dict[bytes, str] = {}
+        try:
+            with open(keypath, "rb") as f:
+                self.key = f.read()
+                if len(self.key) != 64:
+                    raise Exception()
+        except:
+            self.key = os.urandom(64)
+            with open(keypath, "wb") as f:
+                f.write(self.key)
+
+    def b(self, msg: bytes) -> str:
+        try:
+            return self.cache[msg]
+        except:
+            zb = hmac.new(self.key, msg, hashlib.sha512).digest()
+            zs = base64.urlsafe_b64encode(zb)[: self.retlen].decode("utf-8")
+            self.cache[msg] = zs
+            return zs
+
+    def s(self, msg: str) -> str:
+        return self.b(msg.encode("utf-8", "replace"))
+
+
 def uprint(msg: str) -> None:
     try:
         print(msg, end="")