From 5ce9060e5c203fb33b11984f57f703ae25175be2 Mon Sep 17 00:00:00 2001 From: ed Date: Wed, 10 Aug 2022 01:09:54 +0200 Subject: [PATCH] up2k.js: do hashing in web-workers --- copyparty/web/browser.js | 7 ++ copyparty/web/up2k.js | 156 ++++++++++++++++++++++++++++++++------- copyparty/web/w.hash.js | 76 +++++++++++++++++++ 3 files changed, 213 insertions(+), 26 deletions(-) create mode 100644 copyparty/web/w.hash.js diff --git a/copyparty/web/browser.js b/copyparty/web/browser.js index 050e2919..95f7558d 100644 --- a/copyparty/web/browser.js +++ b/copyparty/web/browser.js @@ -117,6 +117,8 @@ var Ls = { "cut_az": "upload files in alphabetical order, rather than smallest-file-first$N$Nalphabetical order can make it easier to eyeball if something went wrong on the server, but it makes uploading slightly slower on fiber / LAN", + "cut_mt": "use multithreading to accelerate file hashing$N$Nthis uses web-workers and requires$Nmore RAM (up to 512 MiB extra)$N$N35% faster https, 450% faster http", + "cft_text": "favicon text (blank and refresh to disable)", "cft_fg": "foreground color", "cft_bg": "background color", @@ -289,6 +291,7 @@ var Ls = { "u_https2": "switch to https", "u_https3": "for much better performance", "u_ancient": 'your browser is impressively ancient -- maybe you should use bup instead', + "u_nowork": "need firefox 53+ or chrome 57+ or iOS 11+", "u_enpot": 'switch to potato UI (may improve upload speed)', "u_depot": 'switch to fancy UI (may reduce upload speed)', "u_gotpot": 'switching to the potato UI for improved upload speed,\n\nfeel free to disagree and switch back!', @@ -451,6 +454,8 @@ var Ls = { "cut_az": "last opp filer i alfabetisk rekkefølge, istedenfor minste-fil-først$N$Nalfabetisk kan gjøre det lettere å anslå om alt gikk bra, men er bittelitt tregere på fiber / LAN", + "cut_mt": "raskere befaring ved å bruke hele CPU'en$N$Ndenne funksjonen anvender web-workers$Nog krever mer RAM (opptil 512 MiB ekstra)$N$N35% raskere https, 450% raskere http", + "cft_text": "ikontekst (blank ut og last siden på nytt for å deaktivere)", "cft_fg": "farge", "cft_bg": "bakgrunnsfarge", @@ -623,6 +628,7 @@ var Ls = { "u_https2": "bytte til https", "u_https3": "for mye høyere hastighet", "u_ancient": 'nettleseren din er prehistorisk -- mulig du burde bruke bup istedenfor', + "u_nowork": "krever firefox 53+, chrome 57+, eller iOS 11+", "u_enpot": 'bytt til enkelt UI (gir sannsynlig raskere opplastning)', "u_depot": 'bytt til snæsent UI (gir sannsynlig tregere opplastning)', "u_gotpot": 'byttet til et enklere UI for å laste opp raskere,\n\ndu kan gjerne bytte tilbake altså!', @@ -844,6 +850,7 @@ ebi('op_cfg').innerHTML = ( '
\n' + '

' + L.cl_uopts + '

\n' + '
\n' + + ' mt\n' + ' turbo\n' + ' date-chk\n' + ' 💤\n' + diff --git a/copyparty/web/up2k.js b/copyparty/web/up2k.js index 8e9908a2..a00018df 100644 --- a/copyparty/web/up2k.js +++ b/copyparty/web/up2k.js @@ -16,6 +16,7 @@ function goto_up2k() { // usually it's undefined but some chromes throw on invoke var up2k = null, up2k_hooks = [], + hws = [], sha_js = window.WebAssembly ? 'hw' : 'ac', // ff53,c57,sa11 m = 'will use ' + sha_js + ' instead of native sha512 due to'; @@ -718,6 +719,13 @@ function up2k_init(subtle) { "gotallfiles": [gotallfiles] // hooks }; + if (window.WebAssembly) { + for (var a = 0; a < Math.min(navigator.hardwareConcurrency || 4, 16); a++) + hws.push(new Worker('/.cpr/w.hash.js')); + + console.log(hws.length + " hashers ready"); + } + function showmodal(msg) { ebi('u2notbtn').innerHTML = msg; ebi('u2btn').style.display = 'none'; @@ -790,7 +798,6 @@ function up2k_init(subtle) { var parallel_uploads = icfg_get('nthread'), uc = {}, fdom_ctr = 0, - min_filebuf = 0, biggest_file = 0; bcfg_bind(uc, 'multitask', 'multitask', true, null, false); @@ -801,6 +808,7 @@ function up2k_init(subtle) { bcfg_bind(uc, 'turbo', 'u2turbo', turbolvl > 1, draw_turbo, false); bcfg_bind(uc, 'datechk', 'u2tdate', turbolvl < 3, null, false); bcfg_bind(uc, 'az', 'u2sort', u2sort.indexOf('n') + 1, set_u2sort, false); + bcfg_bind(uc, 'hashw', 'hashw', !!window.WebAssembly, set_hashw, false); var st = { "files": [], @@ -1288,8 +1296,12 @@ function up2k_init(subtle) { if (!nhash) { var h = L.u_etadone.format(humansize(st.bytes.hashed), pvis.ctr.ok + pvis.ctr.ng); - if (st.eta.h !== h) + if (st.eta.h !== h) { st.eta.h = ebi('u2etah').innerHTML = h; + console.log('{0} hash, {1} up'.format( + f2f(st.time.hashing, 2), + f2f(st.time.uploading, 2))); + } } if (!nsend && !nhash) { @@ -1665,6 +1677,7 @@ function up2k_init(subtle) { var t = st.todo.hash.shift(); st.busy.hash.push(t); st.nfile.hash = t.n; + t.t_hashing = Date.now(); var bpend = 0, nchunk = 0, @@ -1675,30 +1688,23 @@ function up2k_init(subtle) { pvis.setab(t.n, nchunks); pvis.move(t.n, 'bz'); + if (hws.length && uc.hashw) + return wexec_hash(t, chunksize, nchunks); + var segm_next = function () { - if (nchunk >= nchunks || (bpend > chunksize && bpend >= min_filebuf)) + if (nchunk >= nchunks || bpend) return false; var reader = new FileReader(), nch = nchunk++, car = nch * chunksize, - cdr = car + chunksize, - t0 = Date.now(); + cdr = Math.min(chunksize + car, t.size); - if (cdr >= t.size) - cdr = t.size; - - bpend += cdr - car; st.bytes.hashed += cdr - car; function orz(e) { - if (!min_filebuf && nch == 1) { - min_filebuf = 1; - var td = Date.now() - t0; - if (td > 50) { - min_filebuf = 32 * 1024 * 1024; - } - } + bpend--; + segm_next(); hash_calc(nch, e.target.result); } reader.onload = function (e) { @@ -1726,6 +1732,7 @@ function up2k_init(subtle) { toast.err(0, 'y o u b r o k e i t\nfile: ' + esc(t.name + '') + '\nerror: ' + err); }; + bpend++; reader.readAsArrayBuffer( bobslice.call(t.fobj, car, cdr)); @@ -1733,8 +1740,6 @@ function up2k_init(subtle) { }; var hash_calc = function (nch, buf) { - while (segm_next()); - var orz = function (hashbuf) { var hslice = new Uint8Array(hashbuf).subarray(0, 33), b64str = buf2b64(hslice); @@ -1742,15 +1747,12 @@ function up2k_init(subtle) { hashtab[nch] = b64str; t.hash.push(nch); pvis.hashed(t); - - bpend -= buf.byteLength; - if (t.hash.length < nchunks) { + if (t.hash.length < nchunks) return segm_next(); - } + t.hash = []; - for (var a = 0; a < nchunks; a++) { + for (var a = 0; a < nchunks; a++) t.hash.push(hashtab[a]); - } t.t_hashed = Date.now(); @@ -1782,11 +1784,106 @@ function up2k_init(subtle) { } }, 1); }; - - t.t_hashing = Date.now(); segm_next(); } + function wexec_hash(t, chunksize, nchunks) { + var nchunk = 0, + reading = 0, + max_readers = 1, //uc.multitask ? 2 : 1, + free = [], + busy = {}, + nbusy = 0, + hashtab = {}, + mem = (is_touch ? 128 : 256) * 1024 * 1024; + + for (var a = 0; a < hws.length; a++) { + var w = hws[a]; + free.push(w); + w.onmessage = onmsg; + mem -= chunksize; + if (mem <= 0) + break; + } + + function go_next() { + if (reading >= max_readers || !free.length || nchunk >= nchunks) + return; + + var w = free.pop(), + car = nchunk * chunksize, + cdr = Math.min(chunksize + car, t.size); + + //console.log('[P ] %d read bgin (%d reading, %d busy)', nchunk, reading + 1, nbusy + 1); + w.postMessage([nchunk, t.fobj, car, cdr]); + busy[nchunk] = w; + nbusy++; + reading++; + nchunk++; + } + + function onmsg(d) { + d = d.data; + var k = d[0]; + + if (k == "panic") + return vis_exh(d[1], 'up2k.js', '', '', d[1]); + + if (k == "fail") { + pvis.seth(t.n, 1, d[1]); + pvis.seth(t.n, 2, d[2]); + console.log(d[1], d[2]); + + pvis.move(t.n, 'ng'); + apop(st.busy.hash, t); + st.bytes.finished += t.size; + return; + } + + if (k == "ferr") + return toast.err(0, 'y o u b r o k e i t\nfile: ' + esc(t.name + '') + '\nerror: ' + d[1]); + + if (k == "read") { + reading--; + //console.log('[P ] %d read DONE (%d reading, %d busy)', d[1], reading, nbusy); + return go_next(); + } + + if (k == "done") { + var nchunk = d[1], + hslice = d[2], + sz = d[3]; + + free.push(busy[nchunk]); + delete busy[nchunk]; + nbusy--; + + //console.log('[P ] %d HASH DONE (%d reading, %d busy)', nchunk, reading, nbusy); + + hashtab[nchunk] = buf2b64(hslice); + st.bytes.hashed += sz; + t.hash.push(nchunk); + pvis.hashed(t); + + if (t.hash.length < nchunks) + return nbusy < 2 && go_next(); + + t.hash = []; + for (var a = 0; a < nchunks; a++) + t.hash.push(hashtab[a]); + + t.t_hashed = Date.now(); + + pvis.seth(t.n, 2, L.u_hashdone); + pvis.seth(t.n, 1, '📦 wait'); + apop(st.busy.hash, t); + st.todo.handshake.push(t); + tasker(); + } + } + go_next(); + } + ///// //// /// head @@ -2366,6 +2463,13 @@ function up2k_init(subtle) { localStorage.removeItem('u2sort'); } + function set_hashw() { + if (!window.WebAssembly) { + bcfg_set('hashw', false); + toast.err(10, L.u_nowork); + } + } + ebi('nthread_add').onclick = function (e) { ev(e); bumpthread(1); diff --git a/copyparty/web/w.hash.js b/copyparty/web/w.hash.js new file mode 100644 index 00000000..2cc7acb9 --- /dev/null +++ b/copyparty/web/w.hash.js @@ -0,0 +1,76 @@ +"use strict"; + + +function hex2u8(txt) { + return new Uint8Array(txt.match(/.{2}/g).map(function (b) { return parseInt(b, 16); })); +} + + +var subtle = null; +try { + subtle = crypto.subtle || crypto.webkitSubtle; + subtle.digest('SHA-512', new Uint8Array(1)).then( + function (x) { }, + function (x) { load_fb(); } + ); +} +catch (ex) { + load_fb(); +} +function load_fb() { + subtle = null; + importScripts('/.cpr/deps/sha512.hw.js'); +} + + +onmessage = (d) => { + var [nchunk, fobj, car, cdr] = d.data, + reader = new FileReader(); + + reader.onload = function (e) { + try { + //console.log('[ w] %d HASH bgin', nchunk); + postMessage(["read", nchunk]); + hash_calc(e.target.result); + } + catch (ex) { + postMessage(["panic", ex + '']); + } + }; + reader.onerror = function () { + var err = reader.error + ''; + + if (err.indexOf('NotReadableError') !== -1 || // win10-chrome defender + err.indexOf('NotFoundError') !== -1 // macos-firefox permissions + ) + return postMessage(["fail", 'OS-error', err + ' @ ' + car]); + + postMessage(["ferr", err]); + }; + //console.log('[ w] %d read bgin', nchunk); + reader.readAsArrayBuffer( + File.prototype.slice.call(fobj, car, cdr)); + + + var hash_calc = function (buf) { + var hash_done = function (hashbuf) { + try { + var hslice = new Uint8Array(hashbuf).subarray(0, 33); + //console.log('[ w] %d HASH DONE', nchunk); + postMessage(["done", nchunk, hslice, cdr - car]); + } + catch (ex) { + postMessage(["panic", ex + '']); + } + }; + + if (subtle) + subtle.digest('SHA-512', buf).then(hash_done); + else { + var u8buf = new Uint8Array(buf); + hashwasm.sha512(u8buf).then(function (v) { + hash_done(hex2u8(v)) + }); + } + }; +}