url-option for upload checksum type

url-param / header `ck` specifies hashing algo;
md5 sha1 sha256 sha512 b2 blake2 b2s blake2s

value 'no' or blank disables checksumming,
for when copyparty is running on ancient gear
and you don't really care about file integrity

copyparty/httpcli.py

@@ -14,6 +14,7 @@ import re
 import socket
 import stat
 import string
+import sys
 import threading  # typechk
 import time
 import uuid
@@ -76,6 +77,7 @@ from .util import (
     html_escape,
     humansize,
     ipnorm,
+    justcopy,
     load_resource,
     loadpy,
     log_reloc,
@@ -124,6 +126,8 @@ if not hasattr(socket, "AF_UNIX"):
 
 _ = (argparse, threading)
 
+USED4SEC = {"usedforsecurity": False} if sys.version_info > (3, 9) else {}
+
 NO_CACHE = {"Cache-Control": "no-cache"}
 
 ALL_COOKIES = "k304 no304 js idxh dots cppwd cppws".split()
@@ -137,6 +141,10 @@ READMES = [[0, ["preadme.md", "PREADME.md"]], [1, ["readme.md", "README.md"]]]
 
 RSS_SORT = {"m": "mt", "u": "at", "n": "fn", "s": "sz"}
 
+A_FILE = os.stat_result(
+    (0o644, -1, -1, 1, 1000, 1000, 8, 0x39230101, 0x39230101, 0x39230101)
+)
+
 
 class HttpCli(object):
     """
@@ -2060,10 +2068,31 @@ class HttpCli(object):
                 # small toctou, but better than clobbering a hardlink
                 wunlink(self.log, path, vfs.flags)
 
+        hasher = None
+        copier = hashcopy
+        if "ck" in self.ouparam or "ck" in self.headers:
+            zs = self.ouparam.get("ck") or self.headers.get("ck") or ""
+            if not zs or zs == "no":
+                copier = justcopy
+            elif zs == "md5":
+                hasher = hashlib.md5(**USED4SEC)
+            elif zs == "sha1":
+                hasher = hashlib.sha1(**USED4SEC)
+            elif zs == "sha256":
+                hasher = hashlib.sha256(**USED4SEC)
+            elif zs in ("blake2", "b2"):
+                hasher = hashlib.blake2b(**USED4SEC)
+            elif zs in ("blake2s", "b2s"):
+                hasher = hashlib.blake2s(**USED4SEC)
+            elif zs == "sha512":
+                pass
+            else:
+                raise Pebkac(500, "unknown hash alg")
+
         f, fn = ren_open(fn, *open_a, **params)
         try:
             path = os.path.join(fdir, fn)
-            post_sz, sha_hex, sha_b64 = hashcopy(reader, f, None, 0, self.args.s_wr_slp)
+            post_sz, sha_hex, sha_b64 = copier(reader, f, hasher, 0, self.args.s_wr_slp)
         finally:
             f.close()
 
@@ -2300,8 +2329,8 @@ class HttpCli(object):
             # kinda silly but has the least side effects
             return self.handle_new_md()
 
-        if act == "bput":
-            return self.handle_plain_upload(file0)
+        if act in ("bput", "uput"):
+            return self.handle_plain_upload(file0, act == "uput")
 
         if act == "tput":
             return self.handle_text_upload()
@@ -2918,13 +2947,41 @@ class HttpCli(object):
         )
 
     def handle_plain_upload(
-        self, file0: list[tuple[str, Optional[str], Generator[bytes, None, None]]]
+        self,
+        file0: list[tuple[str, Optional[str], Generator[bytes, None, None]]],
+        nohash: bool,
     ) -> bool:
         assert self.parser
         nullwrite = self.args.nw
         vfs, rem = self.asrv.vfs.get(self.vpath, self.uname, False, True)
         self._assert_safe_rem(rem)
 
+        halg = "sha512"
+        hasher = None
+        copier = hashcopy
+        if nohash:
+            halg = ""
+            copier = justcopy
+        elif "ck" in self.ouparam or "ck" in self.headers:
+            halg = self.ouparam.get("ck") or self.headers.get("ck") or ""
+            if not halg or halg == "no":
+                copier = justcopy
+                halg = ""
+            elif halg == "md5":
+                hasher = hashlib.md5(**USED4SEC)
+            elif halg == "sha1":
+                hasher = hashlib.sha1(**USED4SEC)
+            elif halg == "sha256":
+                hasher = hashlib.sha256(**USED4SEC)
+            elif halg in ("blake2", "b2"):
+                hasher = hashlib.blake2b(**USED4SEC)
+            elif halg in ("blake2s", "b2s"):
+                hasher = hashlib.blake2s(**USED4SEC)
+            elif halg == "sha512":
+                pass
+            else:
+                raise Pebkac(500, "unknown hash alg")
+
         upload_vpath = self.vpath
         lim = vfs.get_dbv(rem)[0].lim
         fdir_base = vfs.canonical(rem)
@@ -3054,8 +3111,8 @@ class HttpCli(object):
                     try:
                         tabspath = os.path.join(fdir, tnam)
                         self.log("writing to {}".format(tabspath))
-                        sz, sha_hex, sha_b64 = hashcopy(
-                            p_data, f, None, max_sz, self.args.s_wr_slp
+                        sz, sha_hex, sha_b64 = copier(
+                            p_data, f, hasher, max_sz, self.args.s_wr_slp
                         )
                         if sz == 0:
                             raise Pebkac(400, "empty files in post")
@@ -3187,10 +3244,15 @@ class HttpCli(object):
             jmsg["error"] = errmsg
             errmsg = "ERROR: " + errmsg
 
+        if halg:
+            file_fmt = '{0}: {1} // {2} // {3} bytes // <a href="/{4}">{5}</a> {6}\n'
+        else:
+            file_fmt = '{3} bytes // <a href="/{4}">{5}</a> {6}\n'
+
         for sz, sha_hex, sha_b64, ofn, lfn, ap in files:
             vsuf = ""
             if (self.can_read or self.can_upget) and "fk" in vfs.flags:
-                st = bos.stat(ap)
+                st = A_FILE if nullwrite else bos.stat(ap)
                 alg = 2 if "fka" in vfs.flags else 1
                 vsuf = "?k=" + self.gen_fk(
                     alg,
@@ -3205,7 +3267,8 @@ class HttpCli(object):
 
             vpath = "{}/{}".format(upload_vpath, lfn).strip("/")
             rel_url = quotep(self.args.RS + vpath) + vsuf
-            msg += 'sha512: {} // {} // {} bytes // <a href="/{}">{}</a> {}\n'.format(
+            msg += file_fmt.format(
+                halg,
                 sha_hex[:56],
                 sha_b64,
                 sz,
@@ -3221,13 +3284,14 @@ class HttpCli(object):
                     self.host,
                     rel_url,
                 ),
-                "sha512": sha_hex[:56],
-                "sha_b64": sha_b64,
                 "sz": sz,
                 "fn": lfn,
                 "fn_orig": ofn,
                 "path": rel_url,
             }
+            if halg:
+                jpart[halg] = sha_hex[:56]
+                jpart["sha_b64"] = sha_b64
             jmsg["files"].append(jpart)
 
         vspd = self._spd(sz_total, False)

copyparty/util.py

@@ -2796,6 +2796,26 @@ def yieldfile(fn: str, bufsz: int) -> Generator[bytes, None, None]:
             yield buf
 
 
+def justcopy(
+    fin: Generator[bytes, None, None],
+    fout: Union[typing.BinaryIO, typing.IO[Any]],
+    hashobj: Optional["hashlib._Hash"],
+    max_sz: int,
+    slp: float,
+) -> tuple[int, str, str]:
+    tlen = 0
+    for buf in fin:
+        tlen += len(buf)
+        if max_sz and tlen > max_sz:
+            continue
+
+        fout.write(buf)
+        if slp:
+            time.sleep(slp)
+
+    return tlen, "checksum-disabled", "checksum-disabled"
+
+
 def hashcopy(
     fin: Generator[bytes, None, None],
     fout: Union[typing.BinaryIO, typing.IO[Any]],
@@ -3506,7 +3526,6 @@ def runhook(
     txt: str,
 ) -> dict[str, Any]:
     assert broker or up2k  # !rm
-    asrv = (broker or up2k).asrv
     args = (broker or up2k).args
     vp = vp.replace("\\", "/")
     ret = {"rc": 0}

copyparty/web/browser.js

@@ -521,6 +521,7 @@ var Ls = {
 		"u_pott": "<p>files: &nbsp; <b>{0}</b> finished, &nbsp; <b>{1}</b> failed, &nbsp; <b>{2}</b> busy, &nbsp; <b>{3}</b> queued</p>",
 		"u_ever": "this is the basic uploader; up2k needs at least<br>chrome 21 // firefox 13 // edge 12 // opera 12 // safari 5.1",
 		"u_su2k": 'this is the basic uploader; <a href="#" id="u2yea">up2k</a> is better',
+		"u_uput": 'optimize for speed (skip checksum)',
 		"u_ewrite": 'you do not have write-access to this folder',
 		"u_eread": 'you do not have read-access to this folder',
 		"u_enoi": 'file-search is not enabled in server config',
@@ -1105,6 +1106,7 @@ var Ls = {
 		"u_pott": "<p>filer: &nbsp; <b>{0}</b> ferdig, &nbsp; <b>{1}</b> feilet, &nbsp; <b>{2}</b> behandles, &nbsp; <b>{3}</b> i kø</p>",
 		"u_ever": "dette er den primitive opplasteren; up2k krever minst:<br>chrome 21 // firefox 13 // edge 12 // opera 12 // safari 5.1",
 		"u_su2k": 'dette er den primitive opplasteren; <a href="#" id="u2yea">up2k</a> er bedre',
+		"u_uput": 'litt raskere (uten sha512)',
 		"u_ewrite": 'du har ikke skrivetilgang i denne mappen',
 		"u_eread": 'du har ikke lesetilgang i denne mappen',
 		"u_enoi": 'filsøk er deaktivert i serverkonfigurasjonen',
@@ -1689,6 +1691,7 @@ var Ls = {
 		"u_pott": "<p>个文件： &nbsp; <b>{0}</b> 已完成， &nbsp; <b>{1}</b> 失败， &nbsp; <b>{2}</b> 正在处理， &nbsp; <b>{3}</b> 排队中</p>",
 		"u_ever": "这是基本的上传工具； up2k 需要至少<br>chrome 21 // firefox 13 // edge 12 // opera 12 // safari 5.1",
 		"u_su2k": '这是基本的上传工具；<a href="#" id="u2yea">up2k</a> 更好',
+		"u_uput": '提高速度（跳过校验和）',
 		"u_ewrite": '你对这个文件夹没有写入权限',
 		"u_eread": '你对这个文件夹没有读取权限',
 		"u_enoi": '文件搜索在服务器配置中未启用',
@@ -1935,6 +1938,10 @@ ebi('op_up2k').innerHTML = (
 
 ebi('wrap').insertBefore(mknod('div', 'lazy'), ebi('epi'));
 
+var x = ebi('bbsw');
+x.parentNode.insertBefore(mknod('div', null,
+	'<input type="checkbox" id="uput" name="uput"><label for="uput">' + L.u_uput + '</label>'), x);
+
 
 (function () {
 	var o = mknod('div');
@@ -4210,6 +4217,11 @@ function eval_hash() {
 	}
 	bcfg_bind(props, 'mcmp', 'au_compact', false, setacmp);
 	setacmp();
+
+	// toggle bup checksums
+	ebi('uput').onchange = function() {
+		QS('#op_bup input[name="act"]').value = this.checked ? 'uput' : 'bput';
+	};
 })();
 
 

docs/devnotes.md

@@ -170,10 +170,14 @@ authenticate using header `Cookie: cppwd=foo` or url param `&pw=foo`
 | method | params | body | result |
 |--|--|--|--|
 | PUT | | (binary data) | upload into file at URL |
+| PUT | `?ck` | (binary data) | upload without checksum gen (faster) |
+| PUT | `?ck=md5` | (binary data) | return md5 instead of sha512 |
 | PUT | `?gz` | (binary data) | compress with gzip and write into file at URL |
 | PUT | `?xz` | (binary data) | compress with xz and write into file at URL |
 | mPOST | | `f=FILE` | upload `FILE` into the folder at URL |
 | mPOST | `?j` | `f=FILE` | ...and reply with json |
+| mPOST | `?ck` | `f=FILE` | ...and disable checksum gen (faster) |
+| mPOST | `?ck=md5` | `f=FILE` | ...and return md5 instead of sha512 |
 | mPOST | `?replace` | `f=FILE` | ...and overwrite existing files |
 | mPOST | `?media` | `f=FILE` | ...and return medialink (not hotlink) |
 | mPOST | | `act=mkdir`, `name=foo` | create directory `foo` at URL |
@@ -192,6 +196,12 @@ upload modifiers:
 | `Accept: url` | `want=url` | return just the file URL |
 | `Rand: 4` | `rand=4` | generate random filename with 4 characters |
 | `Life: 30` | `life=30` | delete file after 30 seconds |
+| `CK: no` | `ck` | disable serverside checksum (maybe faster) |
+| `CK: md5` | `ck=md5` | return md5 checksum instead of sha512 |
+| `CK: sha1` | `ck=sha1` | return sha1 checksum |
+| `CK: sha256` | `ck=sha256` | return sha256 checksum |
+| `CK: b2` | `ck=b2` | return blake2b checksum |
+| `CK: b2s` | `ck=b2s` | return blake2s checksum |
 
 * `life` only has an effect if the volume has a lifetime, and the volume lifetime must be greater than the file's