From c5a000d2ae628504d89427dbc65254a1f9b4856c Mon Sep 17 00:00:00 2001 From: ed Date: Mon, 2 Dec 2024 13:51:39 +0000 Subject: [PATCH] url-option for upload checksum type url-param / header `ck` specifies hashing algo; md5 sha1 sha256 sha512 b2 blake2 b2s blake2s value 'no' or blank disables checksumming, for when copyparty is running on ancient gear and you don't really care about file integrity --- copyparty/httpcli.py | 84 +++++++++++++++++++++++++++++++++++----- copyparty/util.py | 21 +++++++++- copyparty/web/browser.js | 12 ++++++ docs/devnotes.md | 10 +++++ 4 files changed, 116 insertions(+), 11 deletions(-) diff --git a/copyparty/httpcli.py b/copyparty/httpcli.py index bb759a04..219efda0 100644 --- a/copyparty/httpcli.py +++ b/copyparty/httpcli.py @@ -14,6 +14,7 @@ import re import socket import stat import string +import sys import threading # typechk import time import uuid @@ -76,6 +77,7 @@ from .util import ( html_escape, humansize, ipnorm, + justcopy, load_resource, loadpy, log_reloc, @@ -124,6 +126,8 @@ if not hasattr(socket, "AF_UNIX"): _ = (argparse, threading) +USED4SEC = {"usedforsecurity": False} if sys.version_info > (3, 9) else {} + NO_CACHE = {"Cache-Control": "no-cache"} ALL_COOKIES = "k304 no304 js idxh dots cppwd cppws".split() @@ -137,6 +141,10 @@ READMES = [[0, ["preadme.md", "PREADME.md"]], [1, ["readme.md", "README.md"]]] RSS_SORT = {"m": "mt", "u": "at", "n": "fn", "s": "sz"} +A_FILE = os.stat_result( + (0o644, -1, -1, 1, 1000, 1000, 8, 0x39230101, 0x39230101, 0x39230101) +) + class HttpCli(object): """ @@ -2060,10 +2068,31 @@ class HttpCli(object): # small toctou, but better than clobbering a hardlink wunlink(self.log, path, vfs.flags) + hasher = None + copier = hashcopy + if "ck" in self.ouparam or "ck" in self.headers: + zs = self.ouparam.get("ck") or self.headers.get("ck") or "" + if not zs or zs == "no": + copier = justcopy + elif zs == "md5": + hasher = hashlib.md5(**USED4SEC) + elif zs == "sha1": + hasher = hashlib.sha1(**USED4SEC) + elif zs == "sha256": + hasher = hashlib.sha256(**USED4SEC) + elif zs in ("blake2", "b2"): + hasher = hashlib.blake2b(**USED4SEC) + elif zs in ("blake2s", "b2s"): + hasher = hashlib.blake2s(**USED4SEC) + elif zs == "sha512": + pass + else: + raise Pebkac(500, "unknown hash alg") + f, fn = ren_open(fn, *open_a, **params) try: path = os.path.join(fdir, fn) - post_sz, sha_hex, sha_b64 = hashcopy(reader, f, None, 0, self.args.s_wr_slp) + post_sz, sha_hex, sha_b64 = copier(reader, f, hasher, 0, self.args.s_wr_slp) finally: f.close() @@ -2300,8 +2329,8 @@ class HttpCli(object): # kinda silly but has the least side effects return self.handle_new_md() - if act == "bput": - return self.handle_plain_upload(file0) + if act in ("bput", "uput"): + return self.handle_plain_upload(file0, act == "uput") if act == "tput": return self.handle_text_upload() @@ -2918,13 +2947,41 @@ class HttpCli(object): ) def handle_plain_upload( - self, file0: list[tuple[str, Optional[str], Generator[bytes, None, None]]] + self, + file0: list[tuple[str, Optional[str], Generator[bytes, None, None]]], + nohash: bool, ) -> bool: assert self.parser nullwrite = self.args.nw vfs, rem = self.asrv.vfs.get(self.vpath, self.uname, False, True) self._assert_safe_rem(rem) + halg = "sha512" + hasher = None + copier = hashcopy + if nohash: + halg = "" + copier = justcopy + elif "ck" in self.ouparam or "ck" in self.headers: + halg = self.ouparam.get("ck") or self.headers.get("ck") or "" + if not halg or halg == "no": + copier = justcopy + halg = "" + elif halg == "md5": + hasher = hashlib.md5(**USED4SEC) + elif halg == "sha1": + hasher = hashlib.sha1(**USED4SEC) + elif halg == "sha256": + hasher = hashlib.sha256(**USED4SEC) + elif halg in ("blake2", "b2"): + hasher = hashlib.blake2b(**USED4SEC) + elif halg in ("blake2s", "b2s"): + hasher = hashlib.blake2s(**USED4SEC) + elif halg == "sha512": + pass + else: + raise Pebkac(500, "unknown hash alg") + upload_vpath = self.vpath lim = vfs.get_dbv(rem)[0].lim fdir_base = vfs.canonical(rem) @@ -3054,8 +3111,8 @@ class HttpCli(object): try: tabspath = os.path.join(fdir, tnam) self.log("writing to {}".format(tabspath)) - sz, sha_hex, sha_b64 = hashcopy( - p_data, f, None, max_sz, self.args.s_wr_slp + sz, sha_hex, sha_b64 = copier( + p_data, f, hasher, max_sz, self.args.s_wr_slp ) if sz == 0: raise Pebkac(400, "empty files in post") @@ -3187,10 +3244,15 @@ class HttpCli(object): jmsg["error"] = errmsg errmsg = "ERROR: " + errmsg + if halg: + file_fmt = '{0}: {1} // {2} // {3} bytes // {5} {6}\n' + else: + file_fmt = '{3} bytes // {5} {6}\n' + for sz, sha_hex, sha_b64, ofn, lfn, ap in files: vsuf = "" if (self.can_read or self.can_upget) and "fk" in vfs.flags: - st = bos.stat(ap) + st = A_FILE if nullwrite else bos.stat(ap) alg = 2 if "fka" in vfs.flags else 1 vsuf = "?k=" + self.gen_fk( alg, @@ -3205,7 +3267,8 @@ class HttpCli(object): vpath = "{}/{}".format(upload_vpath, lfn).strip("/") rel_url = quotep(self.args.RS + vpath) + vsuf - msg += 'sha512: {} // {} // {} bytes // {} {}\n'.format( + msg += file_fmt.format( + halg, sha_hex[:56], sha_b64, sz, @@ -3221,13 +3284,14 @@ class HttpCli(object): self.host, rel_url, ), - "sha512": sha_hex[:56], - "sha_b64": sha_b64, "sz": sz, "fn": lfn, "fn_orig": ofn, "path": rel_url, } + if halg: + jpart[halg] = sha_hex[:56] + jpart["sha_b64"] = sha_b64 jmsg["files"].append(jpart) vspd = self._spd(sz_total, False) diff --git a/copyparty/util.py b/copyparty/util.py index 1ce33e69..6c3cc8ee 100644 --- a/copyparty/util.py +++ b/copyparty/util.py @@ -2796,6 +2796,26 @@ def yieldfile(fn: str, bufsz: int) -> Generator[bytes, None, None]: yield buf +def justcopy( + fin: Generator[bytes, None, None], + fout: Union[typing.BinaryIO, typing.IO[Any]], + hashobj: Optional["hashlib._Hash"], + max_sz: int, + slp: float, +) -> tuple[int, str, str]: + tlen = 0 + for buf in fin: + tlen += len(buf) + if max_sz and tlen > max_sz: + continue + + fout.write(buf) + if slp: + time.sleep(slp) + + return tlen, "checksum-disabled", "checksum-disabled" + + def hashcopy( fin: Generator[bytes, None, None], fout: Union[typing.BinaryIO, typing.IO[Any]], @@ -3506,7 +3526,6 @@ def runhook( txt: str, ) -> dict[str, Any]: assert broker or up2k # !rm - asrv = (broker or up2k).asrv args = (broker or up2k).args vp = vp.replace("\\", "/") ret = {"rc": 0} diff --git a/copyparty/web/browser.js b/copyparty/web/browser.js index 718e56c4..3710db9d 100644 --- a/copyparty/web/browser.js +++ b/copyparty/web/browser.js @@ -521,6 +521,7 @@ var Ls = { "u_pott": "

files:   {0} finished,   {1} failed,   {2} busy,   {3} queued

", "u_ever": "this is the basic uploader; up2k needs at least
chrome 21 // firefox 13 // edge 12 // opera 12 // safari 5.1", "u_su2k": 'this is the basic uploader; up2k is better', + "u_uput": 'optimize for speed (skip checksum)', "u_ewrite": 'you do not have write-access to this folder', "u_eread": 'you do not have read-access to this folder', "u_enoi": 'file-search is not enabled in server config', @@ -1105,6 +1106,7 @@ var Ls = { "u_pott": "

filer:   {0} ferdig,   {1} feilet,   {2} behandles,   {3} i kø

", "u_ever": "dette er den primitive opplasteren; up2k krever minst:
chrome 21 // firefox 13 // edge 12 // opera 12 // safari 5.1", "u_su2k": 'dette er den primitive opplasteren; up2k er bedre', + "u_uput": 'litt raskere (uten sha512)', "u_ewrite": 'du har ikke skrivetilgang i denne mappen', "u_eread": 'du har ikke lesetilgang i denne mappen', "u_enoi": 'filsøk er deaktivert i serverkonfigurasjonen', @@ -1689,6 +1691,7 @@ var Ls = { "u_pott": "

个文件：   {0} 已完成，   {1} 失败，   {2} 正在处理，   {3} 排队中

", "u_ever": "这是基本的上传工具； up2k 需要至少
chrome 21 // firefox 13 // edge 12 // opera 12 // safari 5.1", "u_su2k": '这是基本的上传工具；up2k 更好', + "u_uput": '提高速度（跳过校验和）', "u_ewrite": '你对这个文件夹没有写入权限', "u_eread": '你对这个文件夹没有读取权限', "u_enoi": '文件搜索在服务器配置中未启用', @@ -1935,6 +1938,10 @@ ebi('op_up2k').innerHTML = ( ebi('wrap').insertBefore(mknod('div', 'lazy'), ebi('epi')); +var x = ebi('bbsw'); +x.parentNode.insertBefore(mknod('div', null, + ''), x); + (function () { var o = mknod('div'); @@ -4210,6 +4217,11 @@ function eval_hash() { } bcfg_bind(props, 'mcmp', 'au_compact', false, setacmp); setacmp(); + + // toggle bup checksums + ebi('uput').onchange = function() { + QS('#op_bup input[name="act"]').value = this.checked ? 'uput' : 'bput'; + }; })(); diff --git a/docs/devnotes.md b/docs/devnotes.md index f9cf9b72..2719cc4e 100644 --- a/docs/devnotes.md +++ b/docs/devnotes.md @@ -170,10 +170,14 @@ authenticate using header `Cookie: cppwd=foo` or url param `&pw=foo` | method | params | body | result | |--|--|--|--| | PUT | | (binary data) | upload into file at URL | +| PUT | `?ck` | (binary data) | upload without checksum gen (faster) | +| PUT | `?ck=md5` | (binary data) | return md5 instead of sha512 | | PUT | `?gz` | (binary data) | compress with gzip and write into file at URL | | PUT | `?xz` | (binary data) | compress with xz and write into file at URL | | mPOST | | `f=FILE` | upload `FILE` into the folder at URL | | mPOST | `?j` | `f=FILE` | ...and reply with json | +| mPOST | `?ck` | `f=FILE` | ...and disable checksum gen (faster) | +| mPOST | `?ck=md5` | `f=FILE` | ...and return md5 instead of sha512 | | mPOST | `?replace` | `f=FILE` | ...and overwrite existing files | | mPOST | `?media` | `f=FILE` | ...and return medialink (not hotlink) | | mPOST | | `act=mkdir`, `name=foo` | create directory `foo` at URL | @@ -192,6 +196,12 @@ upload modifiers: | `Accept: url` | `want=url` | return just the file URL | | `Rand: 4` | `rand=4` | generate random filename with 4 characters | | `Life: 30` | `life=30` | delete file after 30 seconds | +| `CK: no` | `ck` | disable serverside checksum (maybe faster) | +| `CK: md5` | `ck=md5` | return md5 checksum instead of sha512 | +| `CK: sha1` | `ck=sha1` | return sha1 checksum | +| `CK: sha256` | `ck=sha256` | return sha256 checksum | +| `CK: b2` | `ck=b2` | return blake2b checksum | +| `CK: b2s` | `ck=b2s` | return blake2s checksum | * `life` only has an effect if the volume has a lifetime, and the volume lifetime must be greater than the file's