mirror of
https://github.com/9001/copyparty.git
synced 2025-08-17 17:12:13 -06:00
up2k.js: do hashing in web-workers
This commit is contained in:
ed
parent
f727d5cb5a
commit
5ce9060e5c
|
|
@ -117,6 +117,8 @@ var Ls = {
|
|||
|
||||
"cut_az": "upload files in alphabetical order, rather than smallest-file-first$N$Nalphabetical order can make it easier to eyeball if something went wrong on the server, but it makes uploading slightly slower on fiber / LAN",
|
||||
|
||||
"cut_mt": "use multithreading to accelerate file hashing$N$Nthis uses web-workers and requires$Nmore RAM (up to 512 MiB extra)$N$N35% faster https, 450% faster http",
|
||||
|
||||
"cft_text": "favicon text (blank and refresh to disable)",
|
||||
"cft_fg": "foreground color",
|
||||
"cft_bg": "background color",
|
||||
|
|
@ -289,6 +291,7 @@ var Ls = {
|
|||
"u_https2": "switch to https",
|
||||
"u_https3": "for much better performance",
|
||||
"u_ancient": 'your browser is impressively ancient -- maybe you should <a href="#" onclick="goto(\'bup\')">use bup instead</a>',
|
||||
"u_nowork": "need firefox 53+ or chrome 57+ or iOS 11+",
|
||||
"u_enpot": 'switch to <a href="#">potato UI</a> (may improve upload speed)',
|
||||
"u_depot": 'switch to <a href="#">fancy UI</a> (may reduce upload speed)',
|
||||
"u_gotpot": 'switching to the potato UI for improved upload speed,\n\nfeel free to disagree and switch back!',
|
||||
|
|
@ -451,6 +454,8 @@ var Ls = {
|
|||
|
||||
"cut_az": "last opp filer i alfabetisk rekkefølge, istedenfor minste-fil-først$N$Nalfabetisk kan gjøre det lettere å anslå om alt gikk bra, men er bittelitt tregere på fiber / LAN",
|
||||
|
||||
"cut_mt": "raskere befaring ved å bruke hele CPU'en$N$Ndenne funksjonen anvender web-workers$Nog krever mer RAM (opptil 512 MiB ekstra)$N$N35% raskere https, 450% raskere http",
|
||||
|
||||
"cft_text": "ikontekst (blank ut og last siden på nytt for å deaktivere)",
|
||||
"cft_fg": "farge",
|
||||
"cft_bg": "bakgrunnsfarge",
|
||||
|
|
@ -623,6 +628,7 @@ var Ls = {
|
|||
"u_https2": "bytte til https",
|
||||
"u_https3": "for mye høyere hastighet",
|
||||
"u_ancient": 'nettleseren din er prehistorisk -- mulig du burde <a href="#" onclick="goto(\'bup\')">bruke bup istedenfor</a>',
|
||||
"u_nowork": "krever firefox 53+, chrome 57+, eller iOS 11+",
|
||||
"u_enpot": 'bytt til <a href="#">enkelt UI</a> (gir sannsynlig raskere opplastning)',
|
||||
"u_depot": 'bytt til <a href="#">snæsent UI</a> (gir sannsynlig tregere opplastning)',
|
||||
"u_gotpot": 'byttet til et enklere UI for å laste opp raskere,\n\ndu kan gjerne bytte tilbake altså!',
|
||||
|
|
@ -844,6 +850,7 @@ ebi('op_cfg').innerHTML = (
|
|||
'<div>\n' +
|
||||
' <h3>' + L.cl_uopts + '</h3>\n' +
|
||||
' <div>\n' +
|
||||
' <a id="hashw" class="tgl btn" href="#" tt="' + L.cut_mt + '">mt</a>\n' +
|
||||
' <a id="u2turbo" class="tgl btn ttb" href="#" tt="' + L.cut_turbo + '">turbo</a>\n' +
|
||||
' <a id="u2tdate" class="tgl btn ttb" href="#" tt="' + L.cut_datechk + '">date-chk</a>\n' +
|
||||
' <a id="flag_en" class="tgl btn" href="#" tt="' + L.cut_flag + '">💤</a>\n' +
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ function goto_up2k() {
|
|||
// usually it's undefined but some chromes throw on invoke
|
||||
var up2k = null,
|
||||
up2k_hooks = [],
|
||||
hws = [],
|
||||
sha_js = window.WebAssembly ? 'hw' : 'ac', // ff53,c57,sa11
|
||||
m = 'will use ' + sha_js + ' instead of native sha512 due to';
|
||||
|
||||
|
|
@ -718,6 +719,13 @@ function up2k_init(subtle) {
|
|||
"gotallfiles": [gotallfiles] // hooks
|
||||
};
|
||||
|
||||
if (window.WebAssembly) {
|
||||
for (var a = 0; a < Math.min(navigator.hardwareConcurrency || 4, 16); a++)
|
||||
hws.push(new Worker('/.cpr/w.hash.js'));
|
||||
|
||||
console.log(hws.length + " hashers ready");
|
||||
}
|
||||
|
||||
function showmodal(msg) {
|
||||
ebi('u2notbtn').innerHTML = msg;
|
||||
ebi('u2btn').style.display = 'none';
|
||||
|
|
@ -790,7 +798,6 @@ function up2k_init(subtle) {
|
|||
var parallel_uploads = icfg_get('nthread'),
|
||||
uc = {},
|
||||
fdom_ctr = 0,
|
||||
min_filebuf = 0,
|
||||
biggest_file = 0;
|
||||
|
||||
bcfg_bind(uc, 'multitask', 'multitask', true, null, false);
|
||||
|
|
@ -801,6 +808,7 @@ function up2k_init(subtle) {
|
|||
bcfg_bind(uc, 'turbo', 'u2turbo', turbolvl > 1, draw_turbo, false);
|
||||
bcfg_bind(uc, 'datechk', 'u2tdate', turbolvl < 3, null, false);
|
||||
bcfg_bind(uc, 'az', 'u2sort', u2sort.indexOf('n') + 1, set_u2sort, false);
|
||||
bcfg_bind(uc, 'hashw', 'hashw', !!window.WebAssembly, set_hashw, false);
|
||||
|
||||
var st = {
|
||||
"files": [],
|
||||
|
|
@ -1288,8 +1296,12 @@ function up2k_init(subtle) {
|
|||
|
||||
if (!nhash) {
|
||||
var h = L.u_etadone.format(humansize(st.bytes.hashed), pvis.ctr.ok + pvis.ctr.ng);
|
||||
if (st.eta.h !== h)
|
||||
if (st.eta.h !== h) {
|
||||
st.eta.h = ebi('u2etah').innerHTML = h;
|
||||
console.log('{0} hash, {1} up'.format(
|
||||
f2f(st.time.hashing, 2),
|
||||
f2f(st.time.uploading, 2)));
|
||||
}
|
||||
}
|
||||
|
||||
if (!nsend && !nhash) {
|
||||
|
|
@ -1665,6 +1677,7 @@ function up2k_init(subtle) {
|
|||
var t = st.todo.hash.shift();
|
||||
st.busy.hash.push(t);
|
||||
st.nfile.hash = t.n;
|
||||
t.t_hashing = Date.now();
|
||||
|
||||
var bpend = 0,
|
||||
nchunk = 0,
|
||||
|
|
@ -1675,30 +1688,23 @@ function up2k_init(subtle) {
|
|||
pvis.setab(t.n, nchunks);
|
||||
pvis.move(t.n, 'bz');
|
||||
|
||||
if (hws.length && uc.hashw)
|
||||
return wexec_hash(t, chunksize, nchunks);
|
||||
|
||||
var segm_next = function () {
|
||||
if (nchunk >= nchunks || (bpend > chunksize && bpend >= min_filebuf))
|
||||
if (nchunk >= nchunks || bpend)
|
||||
return false;
|
||||
|
||||
var reader = new FileReader(),
|
||||
nch = nchunk++,
|
||||
car = nch * chunksize,
|
||||
cdr = car + chunksize,
|
||||
t0 = Date.now();
|
||||
cdr = Math.min(chunksize + car, t.size);
|
||||
|
||||
if (cdr >= t.size)
|
||||
cdr = t.size;
|
||||
|
||||
bpend += cdr - car;
|
||||
st.bytes.hashed += cdr - car;
|
||||
|
||||
function orz(e) {
|
||||
if (!min_filebuf && nch == 1) {
|
||||
min_filebuf = 1;
|
||||
var td = Date.now() - t0;
|
||||
if (td > 50) {
|
||||
min_filebuf = 32 * 1024 * 1024;
|
||||
}
|
||||
}
|
||||
bpend--;
|
||||
segm_next();
|
||||
hash_calc(nch, e.target.result);
|
||||
}
|
||||
reader.onload = function (e) {
|
||||
|
|
@ -1726,6 +1732,7 @@ function up2k_init(subtle) {
|
|||
|
||||
toast.err(0, 'y o u b r o k e i t\nfile: ' + esc(t.name + '') + '\nerror: ' + err);
|
||||
};
|
||||
bpend++;
|
||||
reader.readAsArrayBuffer(
|
||||
bobslice.call(t.fobj, car, cdr));
|
||||
|
||||
|
|
@ -1733,8 +1740,6 @@ function up2k_init(subtle) {
|
|||
};
|
||||
|
||||
var hash_calc = function (nch, buf) {
|
||||
while (segm_next());
|
||||
|
||||
var orz = function (hashbuf) {
|
||||
var hslice = new Uint8Array(hashbuf).subarray(0, 33),
|
||||
b64str = buf2b64(hslice);
|
||||
|
|
@ -1742,15 +1747,12 @@ function up2k_init(subtle) {
|
|||
hashtab[nch] = b64str;
|
||||
t.hash.push(nch);
|
||||
pvis.hashed(t);
|
||||
|
||||
bpend -= buf.byteLength;
|
||||
if (t.hash.length < nchunks) {
|
||||
if (t.hash.length < nchunks)
|
||||
return segm_next();
|
||||
}
|
||||
|
||||
t.hash = [];
|
||||
for (var a = 0; a < nchunks; a++) {
|
||||
for (var a = 0; a < nchunks; a++)
|
||||
t.hash.push(hashtab[a]);
|
||||
}
|
||||
|
||||
t.t_hashed = Date.now();
|
||||
|
||||
|
|
@ -1782,11 +1784,106 @@ function up2k_init(subtle) {
|
|||
}
|
||||
}, 1);
|
||||
};
|
||||
|
||||
t.t_hashing = Date.now();
|
||||
segm_next();
|
||||
}
|
||||
|
||||
function wexec_hash(t, chunksize, nchunks) {
|
||||
var nchunk = 0,
|
||||
reading = 0,
|
||||
max_readers = 1, //uc.multitask ? 2 : 1,
|
||||
free = [],
|
||||
busy = {},
|
||||
nbusy = 0,
|
||||
hashtab = {},
|
||||
mem = (is_touch ? 128 : 256) * 1024 * 1024;
|
||||
|
||||
for (var a = 0; a < hws.length; a++) {
|
||||
var w = hws[a];
|
||||
free.push(w);
|
||||
w.onmessage = onmsg;
|
||||
mem -= chunksize;
|
||||
if (mem <= 0)
|
||||
break;
|
||||
}
|
||||
|
||||
function go_next() {
|
||||
if (reading >= max_readers || !free.length || nchunk >= nchunks)
|
||||
return;
|
||||
|
||||
var w = free.pop(),
|
||||
car = nchunk * chunksize,
|
||||
cdr = Math.min(chunksize + car, t.size);
|
||||
|
||||
//console.log('[P ] %d read bgin (%d reading, %d busy)', nchunk, reading + 1, nbusy + 1);
|
||||
w.postMessage([nchunk, t.fobj, car, cdr]);
|
||||
busy[nchunk] = w;
|
||||
nbusy++;
|
||||
reading++;
|
||||
nchunk++;
|
||||
}
|
||||
|
||||
function onmsg(d) {
|
||||
d = d.data;
|
||||
var k = d[0];
|
||||
|
||||
if (k == "panic")
|
||||
return vis_exh(d[1], 'up2k.js', '', '', d[1]);
|
||||
|
||||
if (k == "fail") {
|
||||
pvis.seth(t.n, 1, d[1]);
|
||||
pvis.seth(t.n, 2, d[2]);
|
||||
console.log(d[1], d[2]);
|
||||
|
||||
pvis.move(t.n, 'ng');
|
||||
apop(st.busy.hash, t);
|
||||
st.bytes.finished += t.size;
|
||||
return;
|
||||
}
|
||||
|
||||
if (k == "ferr")
|
||||
return toast.err(0, 'y o u b r o k e i t\nfile: ' + esc(t.name + '') + '\nerror: ' + d[1]);
|
||||
|
||||
if (k == "read") {
|
||||
reading--;
|
||||
//console.log('[P ] %d read DONE (%d reading, %d busy)', d[1], reading, nbusy);
|
||||
return go_next();
|
||||
}
|
||||
|
||||
if (k == "done") {
|
||||
var nchunk = d[1],
|
||||
hslice = d[2],
|
||||
sz = d[3];
|
||||
|
||||
free.push(busy[nchunk]);
|
||||
delete busy[nchunk];
|
||||
nbusy--;
|
||||
|
||||
//console.log('[P ] %d HASH DONE (%d reading, %d busy)', nchunk, reading, nbusy);
|
||||
|
||||
hashtab[nchunk] = buf2b64(hslice);
|
||||
st.bytes.hashed += sz;
|
||||
t.hash.push(nchunk);
|
||||
pvis.hashed(t);
|
||||
|
||||
if (t.hash.length < nchunks)
|
||||
return nbusy < 2 && go_next();
|
||||
|
||||
t.hash = [];
|
||||
for (var a = 0; a < nchunks; a++)
|
||||
t.hash.push(hashtab[a]);
|
||||
|
||||
t.t_hashed = Date.now();
|
||||
|
||||
pvis.seth(t.n, 2, L.u_hashdone);
|
||||
pvis.seth(t.n, 1, '📦 wait');
|
||||
apop(st.busy.hash, t);
|
||||
st.todo.handshake.push(t);
|
||||
tasker();
|
||||
}
|
||||
}
|
||||
go_next();
|
||||
}
|
||||
|
||||
/////
|
||||
////
|
||||
/// head
|
||||
|
|
@ -2366,6 +2463,13 @@ function up2k_init(subtle) {
|
|||
localStorage.removeItem('u2sort');
|
||||
}
|
||||
|
||||
function set_hashw() {
|
||||
if (!window.WebAssembly) {
|
||||
bcfg_set('hashw', false);
|
||||
toast.err(10, L.u_nowork);
|
||||
}
|
||||
}
|
||||
|
||||
ebi('nthread_add').onclick = function (e) {
|
||||
ev(e);
|
||||
bumpthread(1);
|
||||
|
|
|
|||
76
copyparty/web/w.hash.js
Normal file
76
copyparty/web/w.hash.js
Normal file
|
|
@ -0,0 +1,76 @@
|
|||
"use strict";
|
||||
|
||||
|
||||
function hex2u8(txt) {
|
||||
return new Uint8Array(txt.match(/.{2}/g).map(function (b) { return parseInt(b, 16); }));
|
||||
}
|
||||
|
||||
|
||||
var subtle = null;
|
||||
try {
|
||||
subtle = crypto.subtle || crypto.webkitSubtle;
|
||||
subtle.digest('SHA-512', new Uint8Array(1)).then(
|
||||
function (x) { },
|
||||
function (x) { load_fb(); }
|
||||
);
|
||||
}
|
||||
catch (ex) {
|
||||
load_fb();
|
||||
}
|
||||
function load_fb() {
|
||||
subtle = null;
|
||||
importScripts('/.cpr/deps/sha512.hw.js');
|
||||
}
|
||||
|
||||
|
||||
onmessage = (d) => {
|
||||
var [nchunk, fobj, car, cdr] = d.data,
|
||||
reader = new FileReader();
|
||||
|
||||
reader.onload = function (e) {
|
||||
try {
|
||||
//console.log('[ w] %d HASH bgin', nchunk);
|
||||
postMessage(["read", nchunk]);
|
||||
hash_calc(e.target.result);
|
||||
}
|
||||
catch (ex) {
|
||||
postMessage(["panic", ex + '']);
|
||||
}
|
||||
};
|
||||
reader.onerror = function () {
|
||||
var err = reader.error + '';
|
||||
|
||||
if (err.indexOf('NotReadableError') !== -1 || // win10-chrome defender
|
||||
err.indexOf('NotFoundError') !== -1 // macos-firefox permissions
|
||||
)
|
||||
return postMessage(["fail", 'OS-error', err + ' @ ' + car]);
|
||||
|
||||
postMessage(["ferr", err]);
|
||||
};
|
||||
//console.log('[ w] %d read bgin', nchunk);
|
||||
reader.readAsArrayBuffer(
|
||||
File.prototype.slice.call(fobj, car, cdr));
|
||||
|
||||
|
||||
var hash_calc = function (buf) {
|
||||
var hash_done = function (hashbuf) {
|
||||
try {
|
||||
var hslice = new Uint8Array(hashbuf).subarray(0, 33);
|
||||
//console.log('[ w] %d HASH DONE', nchunk);
|
||||
postMessage(["done", nchunk, hslice, cdr - car]);
|
||||
}
|
||||
catch (ex) {
|
||||
postMessage(["panic", ex + '']);
|
||||
}
|
||||
};
|
||||
|
||||
if (subtle)
|
||||
subtle.digest('SHA-512', buf).then(hash_done);
|
||||
else {
|
||||
var u8buf = new Uint8Array(buf);
|
||||
hashwasm.sha512(u8buf).then(function (v) {
|
||||
hash_done(hex2u8(v))
|
||||
});
|
||||
}
|
||||
};
|
||||
}
|
||||
Loading…
Reference in a new issue