up2k.js: do hashing in web-workers

copyparty/web/browser.js

@@ -117,6 +117,8 @@ var Ls = {
 
 		"cut_az": "upload files in alphabetical order, rather than smallest-file-first$N$Nalphabetical order can make it easier to eyeball if something went wrong on the server, but it makes uploading slightly slower on fiber / LAN",
 
+		"cut_mt": "use multithreading to accelerate file hashing$N$Nthis uses web-workers and requires$Nmore RAM (up to 512 MiB extra)$N$N35% faster https, 450% faster http",
+
 		"cft_text": "favicon text (blank and refresh to disable)",
 		"cft_fg": "foreground color",
 		"cft_bg": "background color",
@@ -289,6 +291,7 @@ var Ls = {
 		"u_https2": "switch to https",
 		"u_https3": "for much better performance",
 		"u_ancient": 'your browser is impressively ancient -- maybe you should <a href="#" onclick="goto(\'bup\')">use bup instead</a>',
+		"u_nowork": "need firefox 53+ or chrome 57+ or iOS 11+",
 		"u_enpot": 'switch to <a href="#">potato UI</a> (may improve upload speed)',
 		"u_depot": 'switch to <a href="#">fancy UI</a> (may reduce upload speed)',
 		"u_gotpot": 'switching to the potato UI for improved upload speed,\n\nfeel free to disagree and switch back!',
@@ -451,6 +454,8 @@ var Ls = {
 
 		"cut_az": "last opp filer i alfabetisk rekkefølge, istedenfor minste-fil-først$N$Nalfabetisk kan gjøre det lettere å anslå om alt gikk bra, men er bittelitt tregere på fiber / LAN",
 
+		"cut_mt": "raskere befaring ved å bruke hele CPU'en$N$Ndenne funksjonen anvender web-workers$Nog krever mer RAM (opptil 512 MiB ekstra)$N$N35% raskere https, 450% raskere http",
+
 		"cft_text": "ikontekst (blank ut og last siden på nytt for å deaktivere)",
 		"cft_fg": "farge",
 		"cft_bg": "bakgrunnsfarge",
@@ -623,6 +628,7 @@ var Ls = {
 		"u_https2": "bytte til https",
 		"u_https3": "for mye høyere hastighet",
 		"u_ancient": 'nettleseren din er prehistorisk -- mulig du burde <a href="#" onclick="goto(\'bup\')">bruke bup istedenfor</a>',
+		"u_nowork": "krever firefox 53+, chrome 57+, eller iOS 11+",
 		"u_enpot": 'bytt til <a href="#">enkelt UI</a> (gir sannsynlig raskere opplastning)',
 		"u_depot": 'bytt til <a href="#">snæsent UI</a> (gir sannsynlig tregere opplastning)',
 		"u_gotpot": 'byttet til et enklere UI for å laste opp raskere,\n\ndu kan gjerne bytte tilbake altså!',
@@ -844,6 +850,7 @@ ebi('op_cfg').innerHTML = (
 	'<div>\n' +
 	'	<h3>' + L.cl_uopts + '</h3>\n' +
 	'	<div>\n' +
+	'		<a id="hashw" class="tgl btn" href="#" tt="' + L.cut_mt + '">mt</a>\n' +
 	'		<a id="u2turbo" class="tgl btn ttb" href="#" tt="' + L.cut_turbo + '">turbo</a>\n' +
 	'		<a id="u2tdate" class="tgl btn ttb" href="#" tt="' + L.cut_datechk + '">date-chk</a>\n' +
 	'		<a id="flag_en" class="tgl btn" href="#" tt="' + L.cut_flag + '">💤</a>\n' +

copyparty/web/up2k.js

@@ -16,6 +16,7 @@ function goto_up2k() {
 // usually it's undefined but some chromes throw on invoke
 var up2k = null,
     up2k_hooks = [],
+    hws = [],
     sha_js = window.WebAssembly ? 'hw' : 'ac',  // ff53,c57,sa11
     m = 'will use ' + sha_js + ' instead of native sha512 due to';
 
@@ -718,6 +719,13 @@ function up2k_init(subtle) {
         "gotallfiles": [gotallfiles]  // hooks
     };
 
+    if (window.WebAssembly) {
+        for (var a = 0; a < Math.min(navigator.hardwareConcurrency || 4, 16); a++)
+            hws.push(new Worker('/.cpr/w.hash.js'));
+
+        console.log(hws.length + " hashers ready");
+    }
+
     function showmodal(msg) {
         ebi('u2notbtn').innerHTML = msg;
         ebi('u2btn').style.display = 'none';
@@ -790,7 +798,6 @@ function up2k_init(subtle) {
     var parallel_uploads = icfg_get('nthread'),
         uc = {},
         fdom_ctr = 0,
-        min_filebuf = 0,
         biggest_file = 0;
 
     bcfg_bind(uc, 'multitask', 'multitask', true, null, false);
@@ -801,6 +808,7 @@ function up2k_init(subtle) {
     bcfg_bind(uc, 'turbo', 'u2turbo', turbolvl > 1, draw_turbo, false);
     bcfg_bind(uc, 'datechk', 'u2tdate', turbolvl < 3, null, false);
     bcfg_bind(uc, 'az', 'u2sort', u2sort.indexOf('n') + 1, set_u2sort, false);
+    bcfg_bind(uc, 'hashw', 'hashw', !!window.WebAssembly, set_hashw, false);
 
     var st = {
         "files": [],
@@ -1288,8 +1296,12 @@ function up2k_init(subtle) {
 
         if (!nhash) {
             var h = L.u_etadone.format(humansize(st.bytes.hashed), pvis.ctr.ok + pvis.ctr.ng);
-            if (st.eta.h !== h)
+            if (st.eta.h !== h) {
                 st.eta.h = ebi('u2etah').innerHTML = h;
+                console.log('{0} hash, {1} up'.format(
+                    f2f(st.time.hashing, 2),
+                    f2f(st.time.uploading, 2)));
+            }
         }
 
         if (!nsend && !nhash) {
@@ -1665,6 +1677,7 @@ function up2k_init(subtle) {
         var t = st.todo.hash.shift();
         st.busy.hash.push(t);
         st.nfile.hash = t.n;
+        t.t_hashing = Date.now();
 
         var bpend = 0,
             nchunk = 0,
@@ -1675,30 +1688,23 @@ function up2k_init(subtle) {
         pvis.setab(t.n, nchunks);
         pvis.move(t.n, 'bz');
 
+        if (hws.length && uc.hashw)
+            return wexec_hash(t, chunksize, nchunks);
+
         var segm_next = function () {
-            if (nchunk >= nchunks || (bpend > chunksize && bpend >= min_filebuf))
+            if (nchunk >= nchunks || bpend)
                 return false;
 
             var reader = new FileReader(),
                 nch = nchunk++,
                 car = nch * chunksize,
-                cdr = car + chunksize,
+                cdr = Math.min(chunksize + car, t.size);
-                t0 = Date.now();
 
-            if (cdr >= t.size)
-                cdr = t.size;
-
-            bpend += cdr - car;
             st.bytes.hashed += cdr - car;
 
             function orz(e) {
-                if (!min_filebuf && nch == 1) {
+                bpend--;
-                    min_filebuf = 1;
+                segm_next();
-                    var td = Date.now() - t0;
-                    if (td > 50) {
-                        min_filebuf = 32 * 1024 * 1024;
-                    }
-                }
                 hash_calc(nch, e.target.result);
             }
             reader.onload = function (e) {
@@ -1726,6 +1732,7 @@ function up2k_init(subtle) {
 
                 toast.err(0, 'y o u   b r o k e    i t\nfile: ' + esc(t.name + '') + '\nerror: ' + err);
             };
+            bpend++;
             reader.readAsArrayBuffer(
                 bobslice.call(t.fobj, car, cdr));
 
@@ -1733,8 +1740,6 @@ function up2k_init(subtle) {
         };
 
         var hash_calc = function (nch, buf) {
-            while (segm_next());
-
             var orz = function (hashbuf) {
                 var hslice = new Uint8Array(hashbuf).subarray(0, 33),
                     b64str = buf2b64(hslice);
@@ -1742,15 +1747,12 @@ function up2k_init(subtle) {
                 hashtab[nch] = b64str;
                 t.hash.push(nch);
                 pvis.hashed(t);
-
+                if (t.hash.length < nchunks)
-                bpend -= buf.byteLength;
-                if (t.hash.length < nchunks) {
                     return segm_next();
-                }
+
                 t.hash = [];
-                for (var a = 0; a < nchunks; a++) {
+                for (var a = 0; a < nchunks; a++)
                     t.hash.push(hashtab[a]);
-                }
 
                 t.t_hashed = Date.now();
 
@@ -1782,11 +1784,106 @@ function up2k_init(subtle) {
                 }
             }, 1);
         };
-
-        t.t_hashing = Date.now();
         segm_next();
     }
 
+    function wexec_hash(t, chunksize, nchunks) {
+        var nchunk = 0,
+            reading = 0,
+            max_readers = 1, //uc.multitask ? 2 : 1,
+            free = [],
+            busy = {},
+            nbusy = 0,
+            hashtab = {},
+            mem = (is_touch ? 128 : 256) * 1024 * 1024;
+
+        for (var a = 0; a < hws.length; a++) {
+            var w = hws[a];
+            free.push(w);
+            w.onmessage = onmsg;
+            mem -= chunksize;
+            if (mem <= 0)
+                break;
+        }
+
+        function go_next() {
+            if (reading >= max_readers || !free.length || nchunk >= nchunks)
+                return;
+
+            var w = free.pop(),
+                car = nchunk * chunksize,
+                cdr = Math.min(chunksize + car, t.size);
+
+            //console.log('[P ] %d read bgin (%d reading, %d busy)', nchunk, reading + 1, nbusy + 1);
+            w.postMessage([nchunk, t.fobj, car, cdr]);
+            busy[nchunk] = w;
+            nbusy++;
+            reading++;
+            nchunk++;
+        }
+
+        function onmsg(d) {
+            d = d.data;
+            var k = d[0];
+
+            if (k == "panic")
+                return vis_exh(d[1], 'up2k.js', '', '', d[1]);
+
+            if (k == "fail") {
+                pvis.seth(t.n, 1, d[1]);
+                pvis.seth(t.n, 2, d[2]);
+                console.log(d[1], d[2]);
+
+                pvis.move(t.n, 'ng');
+                apop(st.busy.hash, t);
+                st.bytes.finished += t.size;
+                return;
+            }
+
+            if (k == "ferr")
+                return toast.err(0, 'y o u   b r o k e    i t\nfile: ' + esc(t.name + '') + '\nerror: ' + d[1]);
+
+            if (k == "read") {
+                reading--;
+                //console.log('[P ] %d read DONE (%d reading, %d busy)', d[1], reading, nbusy);
+                return go_next();
+            }
+
+            if (k == "done") {
+                var nchunk = d[1],
+                    hslice = d[2],
+                    sz = d[3];
+
+                free.push(busy[nchunk]);
+                delete busy[nchunk];
+                nbusy--;
+
+                //console.log('[P ] %d HASH DONE (%d reading, %d busy)', nchunk, reading, nbusy);
+
+                hashtab[nchunk] = buf2b64(hslice);
+                st.bytes.hashed += sz;
+                t.hash.push(nchunk);
+                pvis.hashed(t);
+
+                if (t.hash.length < nchunks)
+                    return nbusy < 2 && go_next();
+
+                t.hash = [];
+                for (var a = 0; a < nchunks; a++)
+                    t.hash.push(hashtab[a]);
+
+                t.t_hashed = Date.now();
+
+                pvis.seth(t.n, 2, L.u_hashdone);
+                pvis.seth(t.n, 1, '📦 wait');
+                apop(st.busy.hash, t);
+                st.todo.handshake.push(t);
+                tasker();
+            }
+        }
+        go_next();
+    }
+
     /////
     ////
     ///   head
@@ -2366,6 +2463,13 @@ function up2k_init(subtle) {
         localStorage.removeItem('u2sort');
     }
 
+    function set_hashw() {
+        if (!window.WebAssembly) {
+            bcfg_set('hashw', false);
+            toast.err(10, L.u_nowork);
+        }
+    }
+
     ebi('nthread_add').onclick = function (e) {
         ev(e);
         bumpthread(1);

copyparty/web/w.hash.js

@@ -0,0 +1,76 @@
+"use strict";
+
+
+function hex2u8(txt) {
+    return new Uint8Array(txt.match(/.{2}/g).map(function (b) { return parseInt(b, 16); }));
+}
+
+
+var subtle = null;
+try {
+    subtle = crypto.subtle || crypto.webkitSubtle;
+    subtle.digest('SHA-512', new Uint8Array(1)).then(
+        function (x) { },
+        function (x) { load_fb(); }
+    );
+}
+catch (ex) {
+    load_fb();
+}
+function load_fb() {
+    subtle = null;
+    importScripts('/.cpr/deps/sha512.hw.js');
+}
+
+
+onmessage = (d) => {
+    var [nchunk, fobj, car, cdr] = d.data,
+        reader = new FileReader();
+
+    reader.onload = function (e) {
+        try {
+            //console.log('[ w] %d HASH bgin', nchunk);
+            postMessage(["read", nchunk]);
+            hash_calc(e.target.result);
+        }
+        catch (ex) {
+            postMessage(["panic", ex + '']);
+        }
+    };
+    reader.onerror = function () {
+        var err = reader.error + '';
+
+        if (err.indexOf('NotReadableError') !== -1 || // win10-chrome defender
+            err.indexOf('NotFoundError') !== -1  // macos-firefox permissions
+        )
+            return postMessage(["fail", 'OS-error', err + ' @ ' + car]);
+
+        postMessage(["ferr", err]);
+    };
+    //console.log('[ w] %d read bgin', nchunk);
+    reader.readAsArrayBuffer(
+        File.prototype.slice.call(fobj, car, cdr));
+
+
+    var hash_calc = function (buf) {
+        var hash_done = function (hashbuf) {
+            try {
+                var hslice = new Uint8Array(hashbuf).subarray(0, 33);
+                //console.log('[ w] %d HASH DONE', nchunk);
+                postMessage(["done", nchunk, hslice, cdr - car]);
+            }
+            catch (ex) {
+                postMessage(["panic", ex + '']);
+            }
+        };
+
+        if (subtle)
+            subtle.digest('SHA-512', buf).then(hash_done);
+        else {
+            var u8buf = new Uint8Array(buf);
+            hashwasm.sha512(u8buf).then(function (v) {
+                hash_done(hex2u8(v))
+            });
+        }
+    };
+}