From be055961ae72aa09a2ea4cde351efa61bdbed9bb Mon Sep 17 00:00:00 2001 From: ed Date: Sun, 20 Jun 2021 15:32:36 +0200 Subject: [PATCH] adjust up2k hashlen to match base64 window --- README.md | 2 +- copyparty/th_srv.py | 4 +- copyparty/up2k.py | 124 ++++++++++++------------------------------ copyparty/util.py | 6 +- copyparty/web/up2k.js | 4 +- 5 files changed, 44 insertions(+), 96 deletions(-) diff --git a/README.md b/README.md index 6da0721b..84346f03 100644 --- a/README.md +++ b/README.md @@ -439,7 +439,7 @@ quick summary of more eccentric web-browsers trying to view a directory index: copyparty returns a truncated sha512sum of your PUT/POST as base64; you can generate the same checksum locally to verify uplaods: - b512(){ printf "$((sha512sum||shasum -a512)|sed -E 's/ .*//;s/(..)/\\x\1/g')"|base64|head -c43;} + b512(){ printf "$((sha512sum||shasum -a512)|sed -E 's/ .*//;s/(..)/\\x\1/g')"|base64|head -c44;} b512 DB_VER: + m = "database is version {}, this copyparty only supports versions <= {}" + raise Exception(m.format(ver, DB_VER)) + + bak = "{}.bak.{:x}.v{}".format(db_path, int(time.time()), ver) + db = cur.connection + cur.close() + db.close() + msg = "creating new DB (old is bad); backup: {}" + if ver: + msg = "creating new DB (too old to upgrade); backup: {}" + + self.log(msg.format(bak)) + os.rename(fsenc(db_path), fsenc(bak)) return self._create_db(db_path, None) - def _create_db(self, db_path, cur): - if not cur: - cur = self._orz(db_path) - - self._create_v2(cur) - self._create_v3(cur) - cur.connection.commit() - self.log("created DB at {}".format(db_path)) - return cur - def _read_ver(self, cur): for tab in ["ki", "kv"]: try: @@ -951,65 +926,38 @@ class Up2k(object): if rows: return int(rows[0][0]) - def _create_v2(self, cur): - for cmd in [ - r"create table up (w text, mt int, sz int, rd text, fn text)", - r"create index up_rd on up(rd)", - r"create index up_fn on up(fn)", - ]: - cur.execute(cmd) - return cur - - def _create_v3(self, cur): + def _create_db(self, db_path, cur): """ collision in 2^(n/2) files where n = bits (6 bits/ch) 10*6/2 = 2^30 = 1'073'741'824, 24.1mb idx 1<<(3*10) 12*6/2 = 2^36 = 68'719'476'736, 24.8mb idx 16*6/2 = 2^48 = 281'474'976'710'656, 26.1mb idx """ - for c, ks in [["drop table k", "isv"], ["drop index up_", "w"]]: - for k in ks: - try: - cur.execute(c + k) - except: - pass + if not cur: + cur = self._orz(db_path) idx = r"create index up_w on up(substr(w,1,16))" if self.no_expr_idx: idx = r"create index up_w on up(w)" for cmd in [ + r"create table up (w text, mt int, sz int, rd text, fn text)", + r"create index up_rd on up(rd)", + r"create index up_fn on up(fn)", idx, r"create table mt (w text, k text, v int)", r"create index mt_w on mt(w)", r"create index mt_k on mt(k)", r"create index mt_v on mt(v)", r"create table kv (k text, v int)", - r"insert into kv values ('sver', 3)", + r"insert into kv values ('sver', {})".format(DB_VER), ]: cur.execute(cmd) + + cur.connection.commit() + self.log("created DB at {}".format(db_path)) return cur - def _upgrade_v1(self, odb, db_path): - npath = db_path + ".next" - if os.path.exists(npath): - os.unlink(npath) - - ndb = self._orz(npath) - self._create_v2(ndb) - - c = odb.execute("select * from up") - for wark, ts, sz, rp in c: - rd, fn = rp.rsplit("/", 1) if "/" in rp else ["", rp] - v = (wark, ts, sz, rd, fn) - ndb.execute("insert into up values (?,?,?,?,?)", v) - - ndb.connection.commit() - ndb.connection.close() - odb.connection.close() - atomic_move(npath, db_path) - return self._orz(db_path) - def handle_json(self, cj): with self.mutex: if not self.register_vpath(cj["ptop"], cj["vcfg"]): @@ -1316,9 +1264,9 @@ class Up2k(object): hashobj.update(buf) rem -= len(buf) - digest = hashobj.digest()[:32] + digest = hashobj.digest()[:33] digest = base64.urlsafe_b64encode(digest) - ret.append(digest.decode("utf-8").rstrip("=")) + ret.append(digest.decode("utf-8")) return ret @@ -1518,12 +1466,12 @@ def up2k_wark_from_hashlist(salt, filesize, hashes): ident.extend(hashes) ident = "\n".join(ident) - wark = hashlib.sha512(ident.encode("utf-8")).digest() + wark = hashlib.sha512(ident.encode("utf-8")).digest()[:33] wark = base64.urlsafe_b64encode(wark) - return wark.decode("ascii")[:43] + return wark.decode("ascii") def up2k_wark_from_metadata(salt, sz, lastmod, rd, fn): ret = fsenc("{}\n{}\n{}\n{}\n{}".format(salt, lastmod, sz, rd, fn)) ret = base64.urlsafe_b64encode(hashlib.sha512(ret).digest()) - return "#{}".format(ret[:42].decode("ascii")) + return "#{}".format(ret.decode("ascii"))[:44] diff --git a/copyparty/util.py b/copyparty/util.py index 4a2308ec..d7efb9dc 100644 --- a/copyparty/util.py +++ b/copyparty/util.py @@ -351,7 +351,7 @@ def ren_open(fname, *args, **kwargs): if not b64: b64 = (bname + ext).encode("utf-8", "replace") b64 = hashlib.sha512(b64).digest()[:12] - b64 = base64.urlsafe_b64encode(b64).decode("utf-8").rstrip("=") + b64 = base64.urlsafe_b64encode(b64).decode("utf-8") badlen = len(fname) while len(fname) >= badlen: @@ -908,8 +908,8 @@ def hashcopy(actor, fin, fout): hashobj.update(buf) fout.write(buf) - digest32 = hashobj.digest()[:32] - digest_b64 = base64.urlsafe_b64encode(digest32).decode("utf-8").rstrip("=") + digest = hashobj.digest()[:33] + digest_b64 = base64.urlsafe_b64encode(digest).decode("utf-8") return tlen, hashobj.hexdigest(), digest_b64 diff --git a/copyparty/web/up2k.js b/copyparty/web/up2k.js index 707b16d9..32098f2e 100644 --- a/copyparty/web/up2k.js +++ b/copyparty/web/up2k.js @@ -970,8 +970,8 @@ function up2k_init(subtle) { while (segm_next()); var hash_done = function (hashbuf) { - var hslice = new Uint8Array(hashbuf).subarray(0, 32), - b64str = buf2b64(hslice).replace(/=$/, ''); + var hslice = new Uint8Array(hashbuf).subarray(0, 33), + b64str = buf2b64(hslice); hashtab[nch] = b64str; t.hash.push(nch);