hmac uploader-ip when avoiding filename collisions

This commit is contained in:
ed 2022-09-11 08:27:45 +02:00
parent 0484f97c9c
commit 0006f933a2
9 changed files with 62 additions and 12 deletions

View file

@ -583,6 +583,7 @@ def run_argparse(argv: list[str], formatter: Any, retry: bool) -> argparse.Names
ap2 = ap.add_argument_group('upload options')
ap2.add_argument("--dotpart", action="store_true", help="dotfile incomplete uploads, hiding them from clients unless -ed")
ap2.add_argument("--plain-ip", action="store_true", help="when avoiding filename collisions by appending the uploader's ip to the filename: append the plaintext ip instead of salting and hashing the ip")
ap2.add_argument("--unpost", metavar="SEC", type=int, default=3600*12, help="grace period where uploads can be deleted by the uploader, even without delete permissions; 0=disabled")
ap2.add_argument("--reg-cap", metavar="N", type=int, default=38400, help="max number of uploads to keep in memory when running without -e2d; roughly 1 MiB RAM per 600")
ap2.add_argument("--no-fpool", action="store_true", help="disable file-handle pooling -- instead, repeatedly close and reopen files during upload")

View file

@ -6,12 +6,13 @@ import signal
import sys
import threading
import os
import queue
from .authsrv import AuthSrv
from .broker_util import BrokerCli, ExceptionalQueue
from .httpsrv import HttpSrv
from .util import FAKE_MP
from .util import FAKE_MP, HMaccas
try:
from types import FrameType
@ -54,6 +55,7 @@ class MpWorker(BrokerCli):
self.asrv = AuthSrv(args, None, False)
# instantiate all services here (TODO: inheritance?)
self.iphash = HMaccas(os.path.join(self.args.E.cfg, "iphash"), 8)
self.httpsrv = HttpSrv(self, n)
# on winxp and some other platforms,

View file

@ -1,11 +1,13 @@
# coding: utf-8
from __future__ import print_function, unicode_literals
import os
import threading
from .__init__ import TYPE_CHECKING
from .broker_util import BrokerCli, ExceptionalQueue, try_exec
from .httpsrv import HttpSrv
from .util import HMaccas
if TYPE_CHECKING:
from .svchub import SvcHub
@ -31,6 +33,7 @@ class BrokerThr(BrokerCli):
self.num_workers = 1
# instantiate all services here (TODO: inheritance?)
self.iphash = HMaccas(os.path.join(self.args.E.cfg, "iphash"), 8)
self.httpsrv = HttpSrv(self, None)
self.reload = self.noop

View file

@ -8,7 +8,7 @@ from queue import Queue
from .__init__ import TYPE_CHECKING
from .authsrv import AuthSrv
from .util import Pebkac
from .util import Pebkac, HMaccas
try:
from typing import Any, Optional, Union
@ -46,6 +46,7 @@ class BrokerCli(object):
self.args: argparse.Namespace = None
self.asrv: AuthSrv = None
self.httpsrv: "HttpSrv" = None
self.iphash: HMaccas = None
def ask(self, dest: str, *args: Any) -> ExceptionalQueue:
return ExceptionalQueue(1)

View file

@ -123,7 +123,6 @@ class HttpCli(object):
self.ua = " "
self.is_rclone = False
self.is_ancient = False
self.dip = " "
self.ouparam: dict[str, str] = {}
self.uparam: dict[str, str] = {}
self.cookies: dict[str, str] = {}
@ -264,8 +263,6 @@ class HttpCli(object):
self.log_src = self.conn.set_rproxy(self.ip)
self.dip = self.ip.replace(":", ".")
if self.args.ihead:
keys = self.args.ihead
if "*" in keys:
@ -403,6 +400,12 @@ class HttpCli(object):
except Pebkac:
return False
def dip(self) -> str:
if self.args.plain_ip:
return self.ip.replace(":", ".")
else:
return self.conn.iphash.s(self.ip)
def permit_caching(self) -> None:
cache = self.uparam.get("cache")
if cache is None:
@ -778,7 +781,7 @@ class HttpCli(object):
else:
self.log("fallthrough? thats a bug", 1)
suffix = "-{:.6f}-{}".format(time.time(), self.dip)
suffix = "-{:.6f}-{}".format(time.time(), self.dip())
if not fn:
suffix += ".bin"
fn = "put" + suffix
@ -1262,6 +1265,7 @@ class HttpCli(object):
files: list[tuple[int, str, str, str, str, str]] = []
# sz, sha_hex, sha_b64, p_file, fname, abspath
errmsg = ""
dip = self.dip()
t0 = time.time()
try:
assert self.parser.gen
@ -1278,7 +1282,7 @@ class HttpCli(object):
if not bos.path.isdir(fdir):
raise Pebkac(404, "that folder does not exist")
suffix = "-{:.6f}-{}".format(time.time(), self.dip)
suffix = "-{:.6f}-{}".format(time.time(), dip)
open_args = {"fdir": fdir, "suffix": suffix}
# reserve destination filename

View file

@ -23,7 +23,7 @@ from .mtag import HAVE_FFMPEG
from .th_cli import ThumbCli
from .th_srv import HAVE_PIL, HAVE_VIPS
from .u2idx import U2idx
from .util import shut_socket
from .util import HMaccas, shut_socket
try:
from typing import Optional, Pattern, Union
@ -54,6 +54,7 @@ class HttpConn(object):
self.asrv: AuthSrv = hsrv.asrv # mypy404
self.cert_path = hsrv.cert_path
self.u2fh: Util.FHC = hsrv.u2fh # mypy404
self.iphash: HMaccas = hsrv.broker.iphash
enth = (HAVE_PIL or HAVE_VIPS or HAVE_FFMPEG) and not self.args.no_thumb
self.thumbcli: Optional[ThumbCli] = ThumbCli(hsrv) if enth else None # mypy404

View file

@ -32,6 +32,7 @@ from .th_srv import HAVE_PIL, HAVE_VIPS, HAVE_WEBP, ThumbSrv
from .up2k import Up2k
from .util import (
VERSIONS,
HMaccas,
alltrace,
ansi_re,
min_ex,
@ -72,6 +73,8 @@ class SvcHub(object):
self.next_day = 0
self.tstack = 0.0
self.iphash = HMaccas(os.path.join(self.E.cfg, "iphash"), 8)
if args.sss or args.s >= 3:
args.ss = True
args.lo = args.lo or "cpp-%Y-%m%d-%H%M%S.txt.xz"

View file

@ -2101,9 +2101,12 @@ class Up2k(object):
if self.args.nw:
return fname
# TODO broker which avoid this race and
# provides a new filename if taken (same as bup)
suffix = "-{:.6f}-{}".format(ts, ip.replace(":", "."))
if self.args.plain_ip:
dip = ip.replace(":", ".")
else:
dip = self.hub.iphash.s(ip)
suffix = "-{:.6f}-{}".format(ts, dip)
with ren_open(fname, "wb", fdir=fdir, suffix=suffix) as zfw:
return zfw["orz"][1]
@ -2842,7 +2845,11 @@ class Up2k(object):
del self.registry[job["ptop"]][job["wark"]]
return
dip = job["addr"].replace(":", ".")
if self.args.plain_ip:
dip = job["addr"].replace(":", ".")
else:
dip = self.hub.iphash.s(job["addr"])
suffix = "-{:.6f}-{}".format(job["t0"], dip)
with ren_open(tnam, "wb", fdir=pdir, suffix=suffix) as zfw:
f, job["tnam"] = zfw["orz"]

View file

@ -4,6 +4,7 @@ from __future__ import print_function, unicode_literals
import base64
import contextlib
import hashlib
import hmac
import math
import mimetypes
import os
@ -597,6 +598,33 @@ class MTHash(object):
return nch, udig, ofs0, chunk_sz
class HMaccas(object):
def __init__(self, keypath: str, retlen: int) -> None:
self.retlen = retlen
self.cache: dict[bytes, str] = {}
try:
with open(keypath, "rb") as f:
self.key = f.read()
if len(self.key) != 64:
raise Exception()
except:
self.key = os.urandom(64)
with open(keypath, "wb") as f:
f.write(self.key)
def b(self, msg: bytes) -> str:
try:
return self.cache[msg]
except:
zb = hmac.new(self.key, msg, hashlib.sha512).digest()
zs = base64.urlsafe_b64encode(zb)[: self.retlen].decode("utf-8")
self.cache[msg] = zs
return zs
def s(self, msg: str) -> str:
return self.b(msg.encode("utf-8", "replace"))
def uprint(msg: str) -> None:
try:
print(msg, end="")