From 184af0c6031694b5f19c9fcdc412f6fea98b9843 Mon Sep 17 00:00:00 2001 From: ed Date: Fri, 5 Jul 2019 19:01:04 +0000 Subject: [PATCH] workaround android-chrome bug --- README.md | 8 +++ copyparty/httpcli.py | 11 +-- copyparty/svchub.py | 2 +- copyparty/up2k.py | 2 +- copyparty/web/splash.html | 7 -- copyparty/web/up2k.js | 114 ++++++++++++++++++++++++++------ scripts/deps-docker/forge.patch | 14 ---- 7 files changed, 112 insertions(+), 46 deletions(-) delete mode 100644 scripts/deps-docker/forge.patch diff --git a/README.md b/README.md index a597b755..97e3be8e 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,14 @@ turn your phone or raspi into a portable file server with resumable uploads/down * code standard: `black` +## notes + +* iPhone/iPad: use Firefox to download files +* Android-Chrome: set max "parallel uploads" for 200% upload speed (android bug) +* Android-Firefox: takes a while to select files (in order to avoid the above android-chrome issue) +* Desktop-Firefox: may use gigabytes of RAM if your connection is great and your files are massive + + ## status * [x] sanic multipart parser diff --git a/copyparty/httpcli.py b/copyparty/httpcli.py index 4cff3bf8..e47193ec 100644 --- a/copyparty/httpcli.py +++ b/copyparty/httpcli.py @@ -302,8 +302,8 @@ class HttpCli(object): if sha_b64 != chash: raise Pebkac( 400, - "your chunk got corrupted somehow:\n{} expected,\n{} received ({} bytes)".format( - chash, sha_b64, post_sz + "your chunk got corrupted somehow (received {} bytes); expected vs received hash:\n{}\n{}".format( + post_sz, chash, sha_b64 ), ) @@ -447,7 +447,8 @@ class HttpCli(object): do_send = True status = 200 extra_headers = [] - logmsg = "{:4} {} {}".format("", self.req, "200 OK") + logmsg = "{:4} {} ".format("", self.req) + logtail = "" # # if request is for foo.js, check if we have foo.js.gz @@ -516,7 +517,7 @@ class HttpCli(object): "Content-Range: bytes {}-{}/{}".format(lower, upper - 1, file_sz) ) - logmsg += " [\033[36m" + str(lower) + "-" + str(upper) + "\033[0m]" + logtail += " [\033[36m{}-{}\033[0m]".format(lower, upper) # # Accept-Encoding and UA decides if we can send gzip as-is @@ -546,6 +547,8 @@ class HttpCli(object): # # send reply + logmsg += str(status) + logtail + mime = mimetypes.guess_type(req_path)[0] or "application/octet-stream" headers = [ diff --git a/copyparty/svchub.py b/copyparty/svchub.py index 430f4a4a..b901d909 100644 --- a/copyparty/svchub.py +++ b/copyparty/svchub.py @@ -77,7 +77,7 @@ class SvcHub(object): dt = dt.replace(hour=0, minute=0, second=0) self.next_day = calendar.timegm(dt.utctimetuple()) - ts = datetime.utcfromtimestamp(now).strftime("%H:%M:%S") + ts = datetime.utcfromtimestamp(now).strftime("%H:%M:%S.%f")[:-3] print("\033[36m{} \033[33m{:21} \033[0m{}".format(ts, src, msg)) def check_mp_support(self): diff --git a/copyparty/up2k.py b/copyparty/up2k.py index 47c51c95..bbcc2a80 100644 --- a/copyparty/up2k.py +++ b/copyparty/up2k.py @@ -108,7 +108,7 @@ class Up2k(object): while True: for mul in [1, 2]: nchunks = math.ceil(filesize * 1.0 / chunksize) - if nchunks <= 256: + if nchunks <= 256 or chunksize >= 32 * 1024 * 1024: return chunksize chunksize += stepsize diff --git a/copyparty/web/splash.html b/copyparty/web/splash.html index 1c3d60f1..4295ebfd 100644 --- a/copyparty/web/splash.html +++ b/copyparty/web/splash.html @@ -35,13 +35,6 @@ - -

[debug] fallback upload

-
- -
-
-
diff --git a/copyparty/web/up2k.js b/copyparty/web/up2k.js index 31ebcc8a..1dcddcc8 100644 --- a/copyparty/web/up2k.js +++ b/copyparty/web/up2k.js @@ -76,6 +76,7 @@ function o(id) { function up2k_init(have_crypto) { //have_crypto = false; + var need_filereader_cache = undefined; // show modal message function showmodal(msg) { @@ -448,7 +449,7 @@ function up2k_init(have_crypto) { while (true) { for (var mul = 1; mul <= 2; mul++) { var nchunks = Math.ceil(filesize / chunksize); - if (nchunks <= 256) + if (nchunks <= 256 || chunksize >= 32 * 1024 * 1024) return chunksize; chunksize += stepsize; @@ -457,7 +458,50 @@ function up2k_init(have_crypto) { } } + function test_filereader_speed(segm_err) { + var f = st.todo.hash[0].fobj, + sz = Math.min(2, f.size), + reader = new FileReader(), + t0, ctr = 0; + + var segm_next = function () { + var t = new Date().getTime(), + td = t - t0; + + if (++ctr > 2) { + need_filereader_cache = td > 50; + st.busy.hash.pop(); + return; + } + t0 = t; + reader.onload = segm_next; + reader.onerror = segm_err; + reader.readAsArrayBuffer( + bobslice.call(f, 0, sz)); + }; + + segm_next(); + } + + function ensure_rendered(func) { + var hidden = false; + var keys = ['hidden', 'msHidden', 'webkitHidden']; + for (var a = 0; a < keys.length; a++) + if (typeof document[keys[a]] !== "undefined") + hidden = document[keys[a]]; + + if (hidden) + return func(); + + window.requestAnimationFrame(func); + } + function exec_hash() { + if (need_filereader_cache === undefined) { + st.busy.hash.push(1); + return test_filereader_speed(segm_err); + } + var t = st.todo.hash.shift(); st.busy.hash.push(t); @@ -465,6 +509,19 @@ function up2k_init(have_crypto) { var chunksize = get_chunksize(t.size); var nchunks = Math.ceil(t.size / chunksize); + // android-chrome has 180ms latency on FileReader calls, + // detect this and do 32MB at a time + var cache_buf = undefined, + cache_ofs = 0, + subchunks = 2; + + while (subchunks * chunksize <= 32 * 1024 * 1024) + subchunks++; + + subchunks--; + if (!need_filereader_cache) + subchunks = 1; + var pb_html = ''; var pb_perc = 99.9 / nchunks; for (var a = 0; a < nchunks; a++) @@ -473,13 +530,17 @@ function up2k_init(have_crypto) { o('f{0}p'.format(t.n)).innerHTML = pb_html; + var reader = new FileReader(); + var segm_next = function () { - var reader = new FileReader(); + if (cache_buf) { + return hash_calc(); + } reader.onload = segm_load; reader.onerror = segm_err; var car = nchunk * chunksize; - var cdr = car + chunksize; + var cdr = car + chunksize * subchunks; if (cdr >= t.size) cdr = t.size; @@ -490,24 +551,39 @@ function up2k_init(have_crypto) { }; var segm_load = function (ev) { - var filebuf = ev.target.result; - var hashbuf; - if (have_crypto) - crypto.subtle.digest('SHA-512', filebuf).then(hash_done); + cache_buf = ev.target.result; + cache_ofs = 0; + hash_calc(); + }; + + var hash_calc = function () { + var buf = cache_buf; + if (chunksize >= buf.byteLength) + cache_buf = undefined; else { - var ofs = 0; - var eof = filebuf.byteLength; - var hasher = new asmCrypto.Sha512(); - //hasher.process(new Uint8Array(filebuf)); - while (ofs < eof) { - // saves memory, doesn't affect perf - var ofs2 = Math.min(eof, ofs + 1024 * 1024); - hasher.process(new Uint8Array(filebuf.slice(ofs, ofs2))); - ofs = ofs2; - } - hasher.finish(); - hash_done(hasher.result); + var ofs = cache_ofs; + var ofs2 = ofs + Math.min(chunksize, cache_buf.byteLength - cache_ofs); + cache_ofs = ofs2; + buf = new Uint8Array(cache_buf).subarray(ofs, ofs2); + if (ofs2 >= cache_buf.byteLength) + cache_buf = undefined; } + + var func = function () { + if (have_crypto) + crypto.subtle.digest('SHA-512', buf).then(hash_done); + else { + var hasher = new asmCrypto.Sha512(); + hasher.process(new Uint8Array(buf)); + hasher.finish(); + hash_done(hasher.result); + } + }; + + if (cache_buf) + ensure_rendered(func); + else + func(); }; var hash_done = function (hashbuf) { diff --git a/scripts/deps-docker/forge.patch b/scripts/deps-docker/forge.patch deleted file mode 100644 index 29ecbf4f..00000000 --- a/scripts/deps-docker/forge.patch +++ /dev/null @@ -1,14 +0,0 @@ -diff -NarU3 forge-0.8.5/webpack.config.js forge-0.8.5-mod/webpack.config.js ---- forge-0.8.5/webpack.config.js 2019-06-19 03:05:22.000000000 +0200 -+++ forge-0.8.5-mod/webpack.config.js 2019-06-26 00:11:42.108019160 +0200 -@@ -31,6 +31,10 @@ - library: null, - libraryTarget: null - } -+ ,{ -+ entry: ['./lib/sha512.js', './lib/forge.js'], -+ filenameBase: 'forge.sha512' -+ } - // Custom builds can be created by specifying the high level files you need - // webpack will pull in dependencies as needed. - //