diff --git a/copyparty/__main__.py b/copyparty/__main__.py index 2439f698..f6a1216a 100755 --- a/copyparty/__main__.py +++ b/copyparty/__main__.py @@ -750,6 +750,9 @@ def run_argparse(argv: list[str], formatter: Any, retry: bool) -> argparse.Names ap2.add_argument("--stackmon", metavar="P,S", type=u, help="write stacktrace to Path every S second, for example --stackmon=./st/%%Y-%%m/%%d/%%H%%M.xz,60") ap2.add_argument("--log-thrs", metavar="SEC", type=float, help="list active threads every SEC") ap2.add_argument("--log-fk", metavar="REGEX", type=u, default="", help="log filekey params for files where path matches REGEX; '.' (a single dot) = all files") + ap2.add_argument("--bak-flips", action="store_true", help="[up2k] if a client uploads a bitflipped/corrupted chunk, store a copy according to --bf-nc and --bf-dir") + ap2.add_argument("--bf-nc", metavar="NUM", type=int, default=200, help="bak-flips: stop if there's more than NUM files at --kf-dir already; default: 6.3 GiB max (200*32M)") + ap2.add_argument("--bf-dir", metavar="PATH", type=u, default="bf", help="bak-flips: store corrupted chunks at PATH; default: folder named 'bf' wherever copyparty was started") # fmt: on ap2 = ap.add_argument_group("help sections") diff --git a/copyparty/httpcli.py b/copyparty/httpcli.py index cf8c8ce5..3ee2c8c1 100644 --- a/copyparty/httpcli.py +++ b/copyparty/httpcli.py @@ -82,6 +82,7 @@ from .util import ( try: from typing import Any, Generator, Match, Optional, Pattern, Type, Union + import typing except: pass @@ -920,6 +921,38 @@ class HttpCli(object): self.reply(t.encode("utf-8")) return True + def bakflip(self, f: typing.BinaryIO, ofs: int, sz: int, sha: str) -> None: + if not self.args.bak_flips or self.args.nw: + return + + sdir = self.args.bf_dir + fp = os.path.join(sdir, sha) + if bos.path.exists(fp): + return self.log("no bakflip; have it", 6) + + if not bos.path.isdir(sdir): + bos.makedirs(sdir) + + if len(bos.listdir(sdir)) >= self.args.bf_nc: + return self.log("no bakflip; too many", 3) + + nrem = sz + f.seek(ofs) + with open(fp, "wb") as fo: + while nrem: + buf = f.read(min(nrem, 512 * 1024)) + if not buf: + break + + nrem -= len(buf) + fo.write(buf) + + if nrem: + self.log("bakflip truncated; {} remains".format(nrem), 1) + atomic_move(fp, fp + ".trunc") + else: + self.log("bakflip ok", 2) + def rand_name(self, fdir: str, fn: str, rnd: int) -> str: ok = False try: @@ -1177,6 +1210,11 @@ class HttpCli(object): post_sz, _, sha_b64 = hashcopy(reader, f, self.args.s_wr_slp) if sha_b64 != chash: + try: + self.bakflip(f, cstart[0], post_sz, sha_b64) + except: + self.log("bakflip failed: " + min_ex()) + t = "your chunk got corrupted somehow (received {} bytes); expected vs received hash:\n{}\n{}" raise Pebkac(400, t.format(post_sz, chash, sha_b64)) diff --git a/copyparty/svchub.py b/copyparty/svchub.py index ecaf7975..fafd7ccd 100644 --- a/copyparty/svchub.py +++ b/copyparty/svchub.py @@ -24,7 +24,7 @@ try: except: pass -from .__init__ import ANYWIN, MACOS, PY2, VT100, WINDOWS, EnvParams, unicode +from .__init__ import ANYWIN, MACOS, VT100, EnvParams, unicode from .authsrv import AuthSrv from .mtag import HAVE_FFMPEG, HAVE_FFPROBE from .tcpsrv import TcpSrv diff --git a/copyparty/util.py b/copyparty/util.py index d343974a..6b78c262 100644 --- a/copyparty/util.py +++ b/copyparty/util.py @@ -1786,7 +1786,7 @@ def yieldfile(fn: str) -> Generator[bytes, None, None]: def hashcopy( - fin: Union[typing.BinaryIO, Generator[bytes, None, None]], + fin: Generator[bytes, None, None], fout: Union[typing.BinaryIO, typing.IO[Any]], slp: int = 0, max_sz: int = 0,