diff --git a/copyparty/__main__.py b/copyparty/__main__.py index cb5396e1..04c63b48 100755 --- a/copyparty/__main__.py +++ b/copyparty/__main__.py @@ -583,6 +583,7 @@ def run_argparse(argv: list[str], formatter: Any, retry: bool) -> argparse.Names ap2 = ap.add_argument_group('upload options') ap2.add_argument("--dotpart", action="store_true", help="dotfile incomplete uploads, hiding them from clients unless -ed") + ap2.add_argument("--plain-ip", action="store_true", help="when avoiding filename collisions by appending the uploader's ip to the filename: append the plaintext ip instead of salting and hashing the ip") ap2.add_argument("--unpost", metavar="SEC", type=int, default=3600*12, help="grace period where uploads can be deleted by the uploader, even without delete permissions; 0=disabled") ap2.add_argument("--reg-cap", metavar="N", type=int, default=38400, help="max number of uploads to keep in memory when running without -e2d; roughly 1 MiB RAM per 600") ap2.add_argument("--no-fpool", action="store_true", help="disable file-handle pooling -- instead, repeatedly close and reopen files during upload") diff --git a/copyparty/broker_mpw.py b/copyparty/broker_mpw.py index 4dbec5b1..51eb51df 100644 --- a/copyparty/broker_mpw.py +++ b/copyparty/broker_mpw.py @@ -6,12 +6,13 @@ import signal import sys import threading +import os import queue from .authsrv import AuthSrv from .broker_util import BrokerCli, ExceptionalQueue from .httpsrv import HttpSrv -from .util import FAKE_MP +from .util import FAKE_MP, HMaccas try: from types import FrameType @@ -54,6 +55,7 @@ class MpWorker(BrokerCli): self.asrv = AuthSrv(args, None, False) # instantiate all services here (TODO: inheritance?) + self.iphash = HMaccas(os.path.join(self.args.E.cfg, "iphash"), 8) self.httpsrv = HttpSrv(self, n) # on winxp and some other platforms, diff --git a/copyparty/broker_thr.py b/copyparty/broker_thr.py index 51c25d41..b5ce5b06 100644 --- a/copyparty/broker_thr.py +++ b/copyparty/broker_thr.py @@ -1,11 +1,13 @@ # coding: utf-8 from __future__ import print_function, unicode_literals +import os import threading from .__init__ import TYPE_CHECKING from .broker_util import BrokerCli, ExceptionalQueue, try_exec from .httpsrv import HttpSrv +from .util import HMaccas if TYPE_CHECKING: from .svchub import SvcHub @@ -31,6 +33,7 @@ class BrokerThr(BrokerCli): self.num_workers = 1 # instantiate all services here (TODO: inheritance?) + self.iphash = HMaccas(os.path.join(self.args.E.cfg, "iphash"), 8) self.httpsrv = HttpSrv(self, None) self.reload = self.noop diff --git a/copyparty/broker_util.py b/copyparty/broker_util.py index 7645632b..d74a622a 100644 --- a/copyparty/broker_util.py +++ b/copyparty/broker_util.py @@ -8,7 +8,7 @@ from queue import Queue from .__init__ import TYPE_CHECKING from .authsrv import AuthSrv -from .util import Pebkac +from .util import Pebkac, HMaccas try: from typing import Any, Optional, Union @@ -46,6 +46,7 @@ class BrokerCli(object): self.args: argparse.Namespace = None self.asrv: AuthSrv = None self.httpsrv: "HttpSrv" = None + self.iphash: HMaccas = None def ask(self, dest: str, *args: Any) -> ExceptionalQueue: return ExceptionalQueue(1) diff --git a/copyparty/httpcli.py b/copyparty/httpcli.py index d9014576..0f25d2a7 100644 --- a/copyparty/httpcli.py +++ b/copyparty/httpcli.py @@ -123,7 +123,6 @@ class HttpCli(object): self.ua = " " self.is_rclone = False self.is_ancient = False - self.dip = " " self.ouparam: dict[str, str] = {} self.uparam: dict[str, str] = {} self.cookies: dict[str, str] = {} @@ -264,8 +263,6 @@ class HttpCli(object): self.log_src = self.conn.set_rproxy(self.ip) - self.dip = self.ip.replace(":", ".") - if self.args.ihead: keys = self.args.ihead if "*" in keys: @@ -403,6 +400,12 @@ class HttpCli(object): except Pebkac: return False + def dip(self) -> str: + if self.args.plain_ip: + return self.ip.replace(":", ".") + else: + return self.conn.iphash.s(self.ip) + def permit_caching(self) -> None: cache = self.uparam.get("cache") if cache is None: @@ -778,7 +781,7 @@ class HttpCli(object): else: self.log("fallthrough? thats a bug", 1) - suffix = "-{:.6f}-{}".format(time.time(), self.dip) + suffix = "-{:.6f}-{}".format(time.time(), self.dip()) if not fn: suffix += ".bin" fn = "put" + suffix @@ -1262,6 +1265,7 @@ class HttpCli(object): files: list[tuple[int, str, str, str, str, str]] = [] # sz, sha_hex, sha_b64, p_file, fname, abspath errmsg = "" + dip = self.dip() t0 = time.time() try: assert self.parser.gen @@ -1278,7 +1282,7 @@ class HttpCli(object): if not bos.path.isdir(fdir): raise Pebkac(404, "that folder does not exist") - suffix = "-{:.6f}-{}".format(time.time(), self.dip) + suffix = "-{:.6f}-{}".format(time.time(), dip) open_args = {"fdir": fdir, "suffix": suffix} # reserve destination filename diff --git a/copyparty/httpconn.py b/copyparty/httpconn.py index 9938ac0a..298cbc56 100644 --- a/copyparty/httpconn.py +++ b/copyparty/httpconn.py @@ -23,7 +23,7 @@ from .mtag import HAVE_FFMPEG from .th_cli import ThumbCli from .th_srv import HAVE_PIL, HAVE_VIPS from .u2idx import U2idx -from .util import shut_socket +from .util import HMaccas, shut_socket try: from typing import Optional, Pattern, Union @@ -54,6 +54,7 @@ class HttpConn(object): self.asrv: AuthSrv = hsrv.asrv # mypy404 self.cert_path = hsrv.cert_path self.u2fh: Util.FHC = hsrv.u2fh # mypy404 + self.iphash: HMaccas = hsrv.broker.iphash enth = (HAVE_PIL or HAVE_VIPS or HAVE_FFMPEG) and not self.args.no_thumb self.thumbcli: Optional[ThumbCli] = ThumbCli(hsrv) if enth else None # mypy404 diff --git a/copyparty/svchub.py b/copyparty/svchub.py index a8545640..b844a463 100644 --- a/copyparty/svchub.py +++ b/copyparty/svchub.py @@ -32,6 +32,7 @@ from .th_srv import HAVE_PIL, HAVE_VIPS, HAVE_WEBP, ThumbSrv from .up2k import Up2k from .util import ( VERSIONS, + HMaccas, alltrace, ansi_re, min_ex, @@ -72,6 +73,8 @@ class SvcHub(object): self.next_day = 0 self.tstack = 0.0 + self.iphash = HMaccas(os.path.join(self.E.cfg, "iphash"), 8) + if args.sss or args.s >= 3: args.ss = True args.lo = args.lo or "cpp-%Y-%m%d-%H%M%S.txt.xz" diff --git a/copyparty/up2k.py b/copyparty/up2k.py index 4d7a0141..047911da 100644 --- a/copyparty/up2k.py +++ b/copyparty/up2k.py @@ -2101,9 +2101,12 @@ class Up2k(object): if self.args.nw: return fname - # TODO broker which avoid this race and - # provides a new filename if taken (same as bup) - suffix = "-{:.6f}-{}".format(ts, ip.replace(":", ".")) + if self.args.plain_ip: + dip = ip.replace(":", ".") + else: + dip = self.hub.iphash.s(ip) + + suffix = "-{:.6f}-{}".format(ts, dip) with ren_open(fname, "wb", fdir=fdir, suffix=suffix) as zfw: return zfw["orz"][1] @@ -2842,7 +2845,11 @@ class Up2k(object): del self.registry[job["ptop"]][job["wark"]] return - dip = job["addr"].replace(":", ".") + if self.args.plain_ip: + dip = job["addr"].replace(":", ".") + else: + dip = self.hub.iphash.s(job["addr"]) + suffix = "-{:.6f}-{}".format(job["t0"], dip) with ren_open(tnam, "wb", fdir=pdir, suffix=suffix) as zfw: f, job["tnam"] = zfw["orz"] diff --git a/copyparty/util.py b/copyparty/util.py index b806e040..3368305b 100644 --- a/copyparty/util.py +++ b/copyparty/util.py @@ -4,6 +4,7 @@ from __future__ import print_function, unicode_literals import base64 import contextlib import hashlib +import hmac import math import mimetypes import os @@ -597,6 +598,33 @@ class MTHash(object): return nch, udig, ofs0, chunk_sz +class HMaccas(object): + def __init__(self, keypath: str, retlen: int) -> None: + self.retlen = retlen + self.cache: dict[bytes, str] = {} + try: + with open(keypath, "rb") as f: + self.key = f.read() + if len(self.key) != 64: + raise Exception() + except: + self.key = os.urandom(64) + with open(keypath, "wb") as f: + f.write(self.key) + + def b(self, msg: bytes) -> str: + try: + return self.cache[msg] + except: + zb = hmac.new(self.key, msg, hashlib.sha512).digest() + zs = base64.urlsafe_b64encode(zb)[: self.retlen].decode("utf-8") + self.cache[msg] = zs + return zs + + def s(self, msg: str) -> str: + return self.b(msg.encode("utf-8", "replace")) + + def uprint(msg: str) -> None: try: print(msg, end="")