workaround android-chrome bug

README.md

@@ -13,6 +13,14 @@ turn your phone or raspi into a portable file server with resumable uploads/down
 * code standard: `black`
 
 
+## notes
+
+* iPhone/iPad: use Firefox to download files
+* Android-Chrome: set max "parallel uploads" for 200% upload speed (android bug)
+* Android-Firefox: takes a while to select files (in order to avoid the above android-chrome issue)
+* Desktop-Firefox: may use gigabytes of RAM if your connection is great and your files are massive
+
+
 ## status
 
 * [x] sanic multipart parser

copyparty/httpcli.py

@@ -302,8 +302,8 @@ class HttpCli(object):
             if sha_b64 != chash:
                 raise Pebkac(
                     400,
-                    "your chunk got corrupted somehow:\n{} expected,\n{} received ({} bytes)".format(
+                    "your chunk got corrupted somehow (received {} bytes); expected vs received hash:\n{}\n{}".format(
-                        chash, sha_b64, post_sz
+                        post_sz, chash, sha_b64
                     ),
                 )
 
@@ -447,7 +447,8 @@ class HttpCli(object):
         do_send = True
         status = 200
         extra_headers = []
-        logmsg = "{:4} {} {}".format("", self.req, "200 OK")
+        logmsg = "{:4} {} ".format("", self.req)
+        logtail = ""
 
         #
         # if request is for foo.js, check if we have foo.js.gz
@@ -516,7 +517,7 @@ class HttpCli(object):
                 "Content-Range: bytes {}-{}/{}".format(lower, upper - 1, file_sz)
             )
 
-            logmsg += " [\033[36m" + str(lower) + "-" + str(upper) + "\033[0m]"
+            logtail += " [\033[36m{}-{}\033[0m]".format(lower, upper)
 
         #
         # Accept-Encoding and UA decides if we can send gzip as-is
@@ -546,6 +547,8 @@ class HttpCli(object):
         #
         # send reply
 
+        logmsg += str(status) + logtail
+
         mime = mimetypes.guess_type(req_path)[0] or "application/octet-stream"
 
         headers = [

copyparty/svchub.py

@@ -77,7 +77,7 @@ class SvcHub(object):
                 dt = dt.replace(hour=0, minute=0, second=0)
                 self.next_day = calendar.timegm(dt.utctimetuple())
 
-            ts = datetime.utcfromtimestamp(now).strftime("%H:%M:%S")
+            ts = datetime.utcfromtimestamp(now).strftime("%H:%M:%S.%f")[:-3]
             print("\033[36m{} \033[33m{:21} \033[0m{}".format(ts, src, msg))
 
     def check_mp_support(self):

copyparty/up2k.py

@@ -108,7 +108,7 @@ class Up2k(object):
         while True:
             for mul in [1, 2]:
                 nchunks = math.ceil(filesize * 1.0 / chunksize)
-                if nchunks <= 256:
+                if nchunks <= 256 or chunksize >= 32 * 1024 * 1024:
                     return chunksize
 
                 chunksize += stepsize

copyparty/web/splash.html

@@ -35,13 +35,6 @@
                 <input type="submit" value="Login" />
             </form>
         </ul>
-
-        <h1>[debug] fallback upload</h1>
-        <form method="post" enctype="multipart/form-data">
-            <input type="hidden" name="act" value="bput" />
-            <h5><input type="file" name="f" multiple></h5>
-            <h5><input type="submit" value="start upload"></h5>
-        </form>
     </div>
     <!-- script src="/.cpr/splash.js"></script -->
 </body>

copyparty/web/up2k.js

@@ -76,6 +76,7 @@ function o(id) {
 
 function up2k_init(have_crypto) {
     //have_crypto = false;
+    var need_filereader_cache = undefined;
 
     // show modal message
     function showmodal(msg) {
@@ -448,7 +449,7 @@ function up2k_init(have_crypto) {
         while (true) {
             for (var mul = 1; mul <= 2; mul++) {
                 var nchunks = Math.ceil(filesize / chunksize);
-                if (nchunks <= 256)
+                if (nchunks <= 256 || chunksize >= 32 * 1024 * 1024)
                     return chunksize;
 
                 chunksize += stepsize;
@@ -457,7 +458,50 @@ function up2k_init(have_crypto) {
         }
     }
 
+    function test_filereader_speed(segm_err) {
+        var f = st.todo.hash[0].fobj,
+            sz = Math.min(2, f.size),
+            reader = new FileReader(),
+            t0, ctr = 0;
+
+        var segm_next = function () {
+            var t = new Date().getTime(),
+                td = t - t0;
+
+            if (++ctr > 2) {
+                need_filereader_cache = td > 50;
+                st.busy.hash.pop();
+                return;
+            }
+            t0 = t;
+            reader.onload = segm_next;
+            reader.onerror = segm_err;
+            reader.readAsArrayBuffer(
+                bobslice.call(f, 0, sz));
+        };
+
+        segm_next();
+    }
+
+    function ensure_rendered(func) {
+        var hidden = false;
+        var keys = ['hidden', 'msHidden', 'webkitHidden'];
+        for (var a = 0; a < keys.length; a++)
+            if (typeof document[keys[a]] !== "undefined")
+                hidden = document[keys[a]];
+
+        if (hidden)
+            return func();
+
+        window.requestAnimationFrame(func);
+    }
+
     function exec_hash() {
+        if (need_filereader_cache === undefined) {
+            st.busy.hash.push(1);
+            return test_filereader_speed(segm_err);
+        }
+
         var t = st.todo.hash.shift();
         st.busy.hash.push(t);
 
@@ -465,6 +509,19 @@ function up2k_init(have_crypto) {
         var chunksize = get_chunksize(t.size);
         var nchunks = Math.ceil(t.size / chunksize);
 
+        // android-chrome has 180ms latency on FileReader calls,
+        // detect this and do 32MB at a time
+        var cache_buf = undefined,
+            cache_ofs = 0,
+            subchunks = 2;
+
+        while (subchunks * chunksize <= 32 * 1024 * 1024)
+            subchunks++;
+
+        subchunks--;
+        if (!need_filereader_cache)
+            subchunks = 1;
+
         var pb_html = '';
         var pb_perc = 99.9 / nchunks;
         for (var a = 0; a < nchunks; a++)
@@ -473,13 +530,17 @@ function up2k_init(have_crypto) {
 
         o('f{0}p'.format(t.n)).innerHTML = pb_html;
 
+        var reader = new FileReader();
+
         var segm_next = function () {
-            var reader = new FileReader();
+            if (cache_buf) {
+                return hash_calc();
+            }
             reader.onload = segm_load;
             reader.onerror = segm_err;
 
             var car = nchunk * chunksize;
-            var cdr = car + chunksize;
+            var cdr = car + chunksize * subchunks;
             if (cdr >= t.size)
                 cdr = t.size;
 
@@ -490,24 +551,39 @@ function up2k_init(have_crypto) {
         };
 
         var segm_load = function (ev) {
-            var filebuf = ev.target.result;
+            cache_buf = ev.target.result;
-            var hashbuf;
+            cache_ofs = 0;
-            if (have_crypto)
+            hash_calc();
-                crypto.subtle.digest('SHA-512', filebuf).then(hash_done);
+        };
+
+        var hash_calc = function () {
+            var buf = cache_buf;
+            if (chunksize >= buf.byteLength)
+                cache_buf = undefined;
             else {
-                var ofs = 0;
+                var ofs = cache_ofs;
-                var eof = filebuf.byteLength;
+                var ofs2 = ofs + Math.min(chunksize, cache_buf.byteLength - cache_ofs);
-                var hasher = new asmCrypto.Sha512();
+                cache_ofs = ofs2;
-                //hasher.process(new Uint8Array(filebuf));
+                buf = new Uint8Array(cache_buf).subarray(ofs, ofs2);
-                while (ofs < eof) {
+                if (ofs2 >= cache_buf.byteLength)
-                    // saves memory, doesn't affect perf
+                    cache_buf = undefined;
-                    var ofs2 = Math.min(eof, ofs + 1024 * 1024);
-                    hasher.process(new Uint8Array(filebuf.slice(ofs, ofs2)));
-                    ofs = ofs2;
-                }
-                hasher.finish();
-                hash_done(hasher.result);
             }
+
+            var func = function () {
+                if (have_crypto)
+                    crypto.subtle.digest('SHA-512', buf).then(hash_done);
+                else {
+                    var hasher = new asmCrypto.Sha512();
+                    hasher.process(new Uint8Array(buf));
+                    hasher.finish();
+                    hash_done(hasher.result);
+                }
+            };
+
+            if (cache_buf)
+                ensure_rendered(func);
+            else
+                func();
         };
 
         var hash_done = function (hashbuf) {

scripts/deps-docker/forge.patch

@@ -1,14 +0,0 @@
-diff -NarU3 forge-0.8.5/webpack.config.js forge-0.8.5-mod/webpack.config.js
---- forge-0.8.5/webpack.config.js	2019-06-19 03:05:22.000000000 +0200
-+++ forge-0.8.5-mod/webpack.config.js	2019-06-26 00:11:42.108019160 +0200
-@@ -31,6 +31,10 @@
-     library: null,
-     libraryTarget: null
-   }
-+  ,{
-+    entry: ['./lib/sha512.js', './lib/forge.js'],
-+    filenameBase: 'forge.sha512'
-+  }
-   // Custom builds can be created by specifying the high level files you need
-   // webpack will pull in dependencies as needed.
-   //