diff --git a/README.md b/README.md
index a597b755..97e3be8e 100644
--- a/README.md
+++ b/README.md
@@ -13,6 +13,14 @@ turn your phone or raspi into a portable file server with resumable uploads/down
* code standard: `black`
+## notes
+
+* iPhone/iPad: use Firefox to download files
+* Android-Chrome: set max "parallel uploads" for 200% upload speed (android bug)
+* Android-Firefox: takes a while to select files (in order to avoid the above android-chrome issue)
+* Desktop-Firefox: may use gigabytes of RAM if your connection is great and your files are massive
+
+
## status
* [x] sanic multipart parser
diff --git a/copyparty/httpcli.py b/copyparty/httpcli.py
index 4cff3bf8..e47193ec 100644
--- a/copyparty/httpcli.py
+++ b/copyparty/httpcli.py
@@ -302,8 +302,8 @@ class HttpCli(object):
if sha_b64 != chash:
raise Pebkac(
400,
- "your chunk got corrupted somehow:\n{} expected,\n{} received ({} bytes)".format(
- chash, sha_b64, post_sz
+ "your chunk got corrupted somehow (received {} bytes); expected vs received hash:\n{}\n{}".format(
+ post_sz, chash, sha_b64
),
)
@@ -447,7 +447,8 @@ class HttpCli(object):
do_send = True
status = 200
extra_headers = []
- logmsg = "{:4} {} {}".format("", self.req, "200 OK")
+ logmsg = "{:4} {} ".format("", self.req)
+ logtail = ""
#
# if request is for foo.js, check if we have foo.js.gz
@@ -516,7 +517,7 @@ class HttpCli(object):
"Content-Range: bytes {}-{}/{}".format(lower, upper - 1, file_sz)
)
- logmsg += " [\033[36m" + str(lower) + "-" + str(upper) + "\033[0m]"
+ logtail += " [\033[36m{}-{}\033[0m]".format(lower, upper)
#
# Accept-Encoding and UA decides if we can send gzip as-is
@@ -546,6 +547,8 @@ class HttpCli(object):
#
# send reply
+ logmsg += str(status) + logtail
+
mime = mimetypes.guess_type(req_path)[0] or "application/octet-stream"
headers = [
diff --git a/copyparty/svchub.py b/copyparty/svchub.py
index 430f4a4a..b901d909 100644
--- a/copyparty/svchub.py
+++ b/copyparty/svchub.py
@@ -77,7 +77,7 @@ class SvcHub(object):
dt = dt.replace(hour=0, minute=0, second=0)
self.next_day = calendar.timegm(dt.utctimetuple())
- ts = datetime.utcfromtimestamp(now).strftime("%H:%M:%S")
+ ts = datetime.utcfromtimestamp(now).strftime("%H:%M:%S.%f")[:-3]
print("\033[36m{} \033[33m{:21} \033[0m{}".format(ts, src, msg))
def check_mp_support(self):
diff --git a/copyparty/up2k.py b/copyparty/up2k.py
index 47c51c95..bbcc2a80 100644
--- a/copyparty/up2k.py
+++ b/copyparty/up2k.py
@@ -108,7 +108,7 @@ class Up2k(object):
while True:
for mul in [1, 2]:
nchunks = math.ceil(filesize * 1.0 / chunksize)
- if nchunks <= 256:
+ if nchunks <= 256 or chunksize >= 32 * 1024 * 1024:
return chunksize
chunksize += stepsize
diff --git a/copyparty/web/splash.html b/copyparty/web/splash.html
index 1c3d60f1..4295ebfd 100644
--- a/copyparty/web/splash.html
+++ b/copyparty/web/splash.html
@@ -35,13 +35,6 @@
-
- [debug] fallback upload
-
diff --git a/copyparty/web/up2k.js b/copyparty/web/up2k.js
index 31ebcc8a..1dcddcc8 100644
--- a/copyparty/web/up2k.js
+++ b/copyparty/web/up2k.js
@@ -76,6 +76,7 @@ function o(id) {
function up2k_init(have_crypto) {
//have_crypto = false;
+ var need_filereader_cache = undefined;
// show modal message
function showmodal(msg) {
@@ -448,7 +449,7 @@ function up2k_init(have_crypto) {
while (true) {
for (var mul = 1; mul <= 2; mul++) {
var nchunks = Math.ceil(filesize / chunksize);
- if (nchunks <= 256)
+ if (nchunks <= 256 || chunksize >= 32 * 1024 * 1024)
return chunksize;
chunksize += stepsize;
@@ -457,7 +458,50 @@ function up2k_init(have_crypto) {
}
}
+ function test_filereader_speed(segm_err) {
+ var f = st.todo.hash[0].fobj,
+ sz = Math.min(2, f.size),
+ reader = new FileReader(),
+ t0, ctr = 0;
+
+ var segm_next = function () {
+ var t = new Date().getTime(),
+ td = t - t0;
+
+ if (++ctr > 2) {
+ need_filereader_cache = td > 50;
+ st.busy.hash.pop();
+ return;
+ }
+ t0 = t;
+ reader.onload = segm_next;
+ reader.onerror = segm_err;
+ reader.readAsArrayBuffer(
+ bobslice.call(f, 0, sz));
+ };
+
+ segm_next();
+ }
+
+ function ensure_rendered(func) {
+ var hidden = false;
+ var keys = ['hidden', 'msHidden', 'webkitHidden'];
+ for (var a = 0; a < keys.length; a++)
+ if (typeof document[keys[a]] !== "undefined")
+ hidden = document[keys[a]];
+
+ if (hidden)
+ return func();
+
+ window.requestAnimationFrame(func);
+ }
+
function exec_hash() {
+ if (need_filereader_cache === undefined) {
+ st.busy.hash.push(1);
+ return test_filereader_speed(segm_err);
+ }
+
var t = st.todo.hash.shift();
st.busy.hash.push(t);
@@ -465,6 +509,19 @@ function up2k_init(have_crypto) {
var chunksize = get_chunksize(t.size);
var nchunks = Math.ceil(t.size / chunksize);
+ // android-chrome has 180ms latency on FileReader calls,
+ // detect this and do 32MB at a time
+ var cache_buf = undefined,
+ cache_ofs = 0,
+ subchunks = 2;
+
+ while (subchunks * chunksize <= 32 * 1024 * 1024)
+ subchunks++;
+
+ subchunks--;
+ if (!need_filereader_cache)
+ subchunks = 1;
+
var pb_html = '';
var pb_perc = 99.9 / nchunks;
for (var a = 0; a < nchunks; a++)
@@ -473,13 +530,17 @@ function up2k_init(have_crypto) {
o('f{0}p'.format(t.n)).innerHTML = pb_html;
+ var reader = new FileReader();
+
var segm_next = function () {
- var reader = new FileReader();
+ if (cache_buf) {
+ return hash_calc();
+ }
reader.onload = segm_load;
reader.onerror = segm_err;
var car = nchunk * chunksize;
- var cdr = car + chunksize;
+ var cdr = car + chunksize * subchunks;
if (cdr >= t.size)
cdr = t.size;
@@ -490,24 +551,39 @@ function up2k_init(have_crypto) {
};
var segm_load = function (ev) {
- var filebuf = ev.target.result;
- var hashbuf;
- if (have_crypto)
- crypto.subtle.digest('SHA-512', filebuf).then(hash_done);
+ cache_buf = ev.target.result;
+ cache_ofs = 0;
+ hash_calc();
+ };
+
+ var hash_calc = function () {
+ var buf = cache_buf;
+ if (chunksize >= buf.byteLength)
+ cache_buf = undefined;
else {
- var ofs = 0;
- var eof = filebuf.byteLength;
- var hasher = new asmCrypto.Sha512();
- //hasher.process(new Uint8Array(filebuf));
- while (ofs < eof) {
- // saves memory, doesn't affect perf
- var ofs2 = Math.min(eof, ofs + 1024 * 1024);
- hasher.process(new Uint8Array(filebuf.slice(ofs, ofs2)));
- ofs = ofs2;
- }
- hasher.finish();
- hash_done(hasher.result);
+ var ofs = cache_ofs;
+ var ofs2 = ofs + Math.min(chunksize, cache_buf.byteLength - cache_ofs);
+ cache_ofs = ofs2;
+ buf = new Uint8Array(cache_buf).subarray(ofs, ofs2);
+ if (ofs2 >= cache_buf.byteLength)
+ cache_buf = undefined;
}
+
+ var func = function () {
+ if (have_crypto)
+ crypto.subtle.digest('SHA-512', buf).then(hash_done);
+ else {
+ var hasher = new asmCrypto.Sha512();
+ hasher.process(new Uint8Array(buf));
+ hasher.finish();
+ hash_done(hasher.result);
+ }
+ };
+
+ if (cache_buf)
+ ensure_rendered(func);
+ else
+ func();
};
var hash_done = function (hashbuf) {
diff --git a/scripts/deps-docker/forge.patch b/scripts/deps-docker/forge.patch
deleted file mode 100644
index 29ecbf4f..00000000
--- a/scripts/deps-docker/forge.patch
+++ /dev/null
@@ -1,14 +0,0 @@
-diff -NarU3 forge-0.8.5/webpack.config.js forge-0.8.5-mod/webpack.config.js
---- forge-0.8.5/webpack.config.js 2019-06-19 03:05:22.000000000 +0200
-+++ forge-0.8.5-mod/webpack.config.js 2019-06-26 00:11:42.108019160 +0200
-@@ -31,6 +31,10 @@
- library: null,
- libraryTarget: null
- }
-+ ,{
-+ entry: ['./lib/sha512.js', './lib/forge.js'],
-+ filenameBase: 'forge.sha512'
-+ }
- // Custom builds can be created by specifying the high level files you need
- // webpack will pull in dependencies as needed.
- //