From ec50788987bf8bcfb091d8bf6b8d808f267ed526 Mon Sep 17 00:00:00 2001 From: ed Date: Fri, 10 Jan 2025 05:29:55 +0000 Subject: [PATCH] up2k.js: 10x faster hashing on android-chrome when hashing files on android-chrome, read a contiguous range of several chunks at a time, ensuring each read is at least 48 MiB and then slice that cache into the correct chunksizes for hashing especially on GrapheneOS Vanadium (where webworkers are forbidden), improves worst-case speed (filesize <= 256 MiB) from 13 to 139 MiB/s 48M was chosen wrt RAM usage (48*4 MiB); a target read-size of 16M would have given 76 MiB/s, 32M = 117 MiB/s, and 64M = 154 MiB/s additionally, on all platforms (not just android and/or chrome), allow async hashing of <= 3 chunks in parallel on main-thread when chunksize <= 48 MiB, and <= 2 at <= 96 MiB; this gives decent speeds approaching that of webworkers (around 50%) this is a new take on c06d928bb57640cec8b4a7beeed1fe0d5bf74f12 which was removed in 184af0c6031694b5f19c9fcdc412f6fea98b9843 when a chrome-beta temporarily fixed the poor file-read performance (afaict the fix was reverted and never made it to chrome stable) as for why any of this is necessary, the security features in android have the unfortunate side-effect of making file-reads from web-browsers extremely expensive; this is especially noticeable in android-chrome, where file-hashing is painfully slow, around 9 MiB/s worst-case this is due to a fixed-time overhead for each read operation; reading 1 MiB takes 60 msec, while reading 16 MiB takes 112 msec --- copyparty/web/up2k.js | 79 ++++++++++++++++++++++++++++++++++++------- copyparty/web/util.js | 2 +- 2 files changed, 68 insertions(+), 13 deletions(-) diff --git a/copyparty/web/up2k.js b/copyparty/web/up2k.js index f6ef7aff..107a06ec 100644 --- a/copyparty/web/up2k.js +++ b/copyparty/web/up2k.js @@ -881,7 +881,7 @@ function up2k_init(subtle) { bcfg_bind(uc, 'turbo', 'u2turbo', turbolvl > 1, draw_turbo); bcfg_bind(uc, 'datechk', 'u2tdate', turbolvl < 3, null); bcfg_bind(uc, 'az', 'u2sort', u2sort.indexOf('n') + 1, set_u2sort); - bcfg_bind(uc, 'hashw', 'hashw', !!WebAssembly && (!subtle || !CHROME || MOBILE || VCHROME >= 107), set_hashw); + bcfg_bind(uc, 'hashw', 'hashw', !!WebAssembly && !(CHROME && MOBILE) && (!subtle || !CHROME), set_hashw); bcfg_bind(uc, 'upnag', 'upnag', false, set_upnag); bcfg_bind(uc, 'upsfx', 'upsfx', false, set_upsfx); @@ -1972,32 +1972,84 @@ function up2k_init(subtle) { nchunk = 0, chunksize = get_chunksize(t.size), nchunks = Math.ceil(t.size / chunksize), + csz_mib = chunksize / 1048576, + tread = t.t_hashing, + cache_buf = null, + cache_car = 0, + cache_cdr = 0, + hashers = 0, hashtab = {}; + // resolving subtle.digest w/o worker takes 1sec on blur if the actx hack breaks + var use_workers = hws.length && !hws_ng && uc.hashw && (nchunks > 1 || document.visibilityState == 'hidden'), + hash_par = (!subtle && !use_workers) ? 0 : csz_mib < 48 ? 2 : csz_mib < 96 ? 1 : 0; + pvis.setab(t.n, nchunks); pvis.move(t.n, 'bz'); - if (hws.length && !hws_ng && uc.hashw && (nchunks > 1 || document.visibilityState == 'hidden')) - // resolving subtle.digest w/o worker takes 1sec on blur if the actx hack breaks + if (use_workers) return wexec_hash(t, chunksize, nchunks); var segm_next = function () { if (nchunk >= nchunks || bpend) return false; - var reader = new FileReader(), - nch = nchunk++, + var nch = nchunk++, car = nch * chunksize, cdr = Math.min(chunksize + car, t.size); st.bytes.hashed += cdr - car; st.etac.h++; - var orz = function (e) { - bpend--; - segm_next(); - hash_calc(nch, e.target.result); + if (MOBILE && CHROME && st.slow_io === null && nch == 1 && cdr - car >= 1024 * 512) { + var spd = Math.floor((cdr - car) / (Date.now() + 1 - tread)); + st.slow_io = spd < 40 * 1024; + console.log('spd {0}, slow: {1}'.format(spd, st.slow_io)); } + + if (cdr <= cache_cdr && car >= cache_car) { + try { + var ofs = car - cache_car, + ofs2 = ofs + (cdr - car), + buf = cache_buf.subarray(ofs, ofs2); + + hash_calc(nch, buf); + } + catch (ex) { + vis_exh(ex + '', 'up2k.js', '', '', ex); + } + return; + } + + var reader = new FileReader(), + fr_cdr = cdr; + + if (st.slow_io) { + var step = cdr - car, + tgt = 48 * 1048576; + + while (step && fr_cdr - car < tgt) + fr_cdr += step; + if (fr_cdr - car > tgt && fr_cdr > cdr) + fr_cdr -= step; + if (fr_cdr > t.size) + fr_cdr = t.size; + } + + var orz = function (e) { + bpend = 0; + var buf = e.target.result; + if (fr_cdr > cdr) { + cache_buf = new Uint8Array(buf); + cache_car = car; + cache_cdr = fr_cdr; + buf = cache_buf.subarray(0, cdr - car); + } + if (hashers < hash_par) + segm_next(); + + hash_calc(nch, buf); + }; reader.onload = function (e) { try { orz(e); } catch (ex) { vis_exh(ex + '', 'up2k.js', '', '', ex); } }; @@ -2024,17 +2076,20 @@ function up2k_init(subtle) { toast.err(0, 'y o u b r o k e i t\nfile: ' + esc(t.name + '') + '\nerror: ' + err); }; - bpend++; - reader.readAsArrayBuffer(t.fobj.slice(car, cdr)); + bpend = 1; + tread = Date.now(); + reader.readAsArrayBuffer(t.fobj.slice(car, fr_cdr)); return true; }; var hash_calc = function (nch, buf) { + hashers++; var orz = function (hashbuf) { var hslice = new Uint8Array(hashbuf).subarray(0, 33), b64str = buf2b64(hslice); + hashers--; hashtab[nch] = b64str; t.hash.push(nch); pvis.hashed(t); @@ -3044,7 +3099,7 @@ function up2k_init(subtle) { new_state = false; fixed = true; } - if (new_state === undefined) + if (new_state === undefined && preferred === undefined) new_state = can_write ? false : have_up2k_idx ? true : undefined; } diff --git a/copyparty/web/util.js b/copyparty/web/util.js index 94b2f2e5..d53bc496 100644 --- a/copyparty/web/util.js +++ b/copyparty/web/util.js @@ -29,7 +29,7 @@ var wah = '', HTTPS = ('' + location).indexOf('https:') === 0, TOUCH = 'ontouchstart' in window, MOBILE = TOUCH, - CHROME = !!window.chrome, + CHROME = !!window.chrome, // safari=false VCHROME = CHROME ? 1 : 0, IE = /Trident\//.test(navigator.userAgent), FIREFOX = ('netscape' in window) && / rv:/.test(navigator.userAgent),