From 1c69ccc6cde82441d9884e13ba562967792abeb0 Mon Sep 17 00:00:00 2001 From: ed Date: Sat, 13 Aug 2022 00:58:49 +0200 Subject: [PATCH] v1.3.12 --- README.md | 11 ++++++++ bin/up2k.py | 3 +- copyparty/__main__.py | 2 +- copyparty/__version__.py | 4 +-- copyparty/up2k.py | 3 +- docs/changelog.md | 61 ++++++++++++++++++++++++++++++++++++++++ 6 files changed, 79 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 3f352c9c..b02d3101 100644 --- a/README.md +++ b/README.md @@ -249,12 +249,18 @@ some improvement ideas * Windows: if the `up2k.db` (filesystem index) is on a samba-share or network disk, you'll get unpredictable behavior if the share is disconnected for a bit * use `--hist` or the `hist` volflag (`-v [...]:c,hist=/tmp/foo`) to place the db on a local disk instead * all volumes must exist / be available on startup; up2k (mtp especially) gets funky otherwise +* [the database can get stuck](https://github.com/9001/copyparty/issues/10) + * has only happened once but that is once too many + * luckily not dangerous for file integrity and doesn't really stop uploads or anything like that + * but would really appreciate some logs if anyone ever runs into it again * probably more, pls let me know ## not my bugs * [Chrome issue 1317069](https://bugs.chromium.org/p/chromium/issues/detail?id=1317069) -- if you try to upload a folder which contains symlinks by dragging it into the browser, the symlinked files will not get uploaded +* [Chrome issue 1352210](https://bugs.chromium.org/p/chromium/issues/detail?id=1352210) -- plaintext http may be faster at filehashing than https (but also extremely CPU-intensive) + * iPhones: the volume control doesn't work because [apple doesn't want it to](https://developer.apple.com/library/archive/documentation/AudioVideo/Conceptual/Using_HTML5_Audio_Video/Device-SpecificConsiderations/Device-SpecificConsiderations.html#//apple_ref/doc/uid/TP40009523-CH5-SW11) * *future workaround:* enable the equalizer, make it all-zero, and set a negative boost to reduce the volume * "future" because `AudioContext` is broken in the current iOS version (15.1), maybe one day... @@ -1008,6 +1014,10 @@ this is due to `crypto.subtle` [not yet](https://github.com/w3c/webcrypto/issues as a result, the hashes are much less useful than they could have been (search the server by sha512, provide the sha512 in the response http headers, ...) +however it allows for hashing multiple chunks in parallel, greatly increasing upload speed from fast storage (NVMe, raid-0 and such) + +* both the [browser uploader](#uploading) and the [commandline one](https://github.com/9001/copyparty/blob/hovudstraum/bin/up2k.py) does this now, allowing for fast uploading even from plaintext http + hashwasm would solve the streaming issue but reduces hashing speed for sha512 (xxh128 does 6 GiB/s), and it would make old browsers and [iphones](https://bugs.webkit.org/show_bug.cgi?id=228552) unsupported * blake2 might be a better choice since xxh is non-cryptographic, but that gets ~15 MiB/s on slower androids @@ -1041,6 +1051,7 @@ when uploading files, * if you're cpu-bottlenecked, or the browser is maxing a cpu core: * up to 30% faster uploads if you hide the upload status list by switching away from the `[🚀]` up2k ui-tab (or closing it) + * optionally you can switch to the lightweight potato ui by clicking the `[🥔]` * switching to another browser-tab also works, the favicon will update every 10 seconds in that case * unlikely to be a problem, but can happen when uploding many small files, or your internet is too fast, or PC too slow diff --git a/bin/up2k.py b/bin/up2k.py index d2e58faa..d5142ae3 100755 --- a/bin/up2k.py +++ b/bin/up2k.py @@ -354,7 +354,8 @@ def walkdir(err, top, seen): """recursive statdir""" atop = os.path.abspath(os.path.realpath(top)) if atop in seen: - return err.append((top, "recursive-symlink")) + err.append((top, "recursive-symlink")) + return seen = seen[:] + [atop] for ap, inf in sorted(statdir(err, top)): diff --git a/copyparty/__main__.py b/copyparty/__main__.py index 50ebefda..6ebdcbda 100644 --- a/copyparty/__main__.py +++ b/copyparty/__main__.py @@ -611,7 +611,7 @@ def run_argparse(argv: list[str], formatter: Any, retry: bool) -> argparse.Names ap2.add_argument("--hist", metavar="PATH", type=u, help="where to store volume data (db, thumbs)") ap2.add_argument("--no-hash", metavar="PTN", type=u, help="regex: disable hashing of matching paths during e2ds folder scans") ap2.add_argument("--no-idx", metavar="PTN", type=u, help="regex: disable indexing of matching paths during e2ds folder scans") - ap2.add_argument("--no-dhash", action="store_true", help="disable rescan acceleration; do full database integrity check (makes the db ~5%% smaller)") + ap2.add_argument("--no-dhash", action="store_true", help="disable rescan acceleration; do full database integrity check -- makes the db ~5%% smaller and bootup/rescans 3~10x slower") ap2.add_argument("--xdev", action="store_true", help="do not descend into other filesystems (symlink or bind-mount to another HDD, ...)") ap2.add_argument("--xvol", action="store_true", help="skip symlinks leaving the volume root") ap2.add_argument("--hash-mt", metavar="CORES", type=int, default=hcores, help="num cpu cores to use for file hashing; set 0 or 1 for single-core hashing") diff --git a/copyparty/__version__.py b/copyparty/__version__.py index 11c119fa..8f925501 100644 --- a/copyparty/__version__.py +++ b/copyparty/__version__.py @@ -1,8 +1,8 @@ # coding: utf-8 -VERSION = (1, 3, 11) +VERSION = (1, 3, 12) CODENAME = "god dag" -BUILD_DT = (2022, 8, 10) +BUILD_DT = (2022, 8, 13) S_VERSION = ".".join(map(str, VERSION)) S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT) diff --git a/copyparty/up2k.py b/copyparty/up2k.py index b9043240..34315beb 100644 --- a/copyparty/up2k.py +++ b/copyparty/up2k.py @@ -1810,7 +1810,7 @@ class Up2k(object): ]: cur.execute(cmd) - cur.connection.commit() + self._add_dhash_tab(cur) self.log("created DB at {}".format(db_path)) return cur @@ -1830,6 +1830,7 @@ class Up2k(object): for cmd in [ r"create table dh (d text, h text)", r"create index dh_d on dh(d)", + r"insert into kv values ('tagscan',1)", ]: cur.execute(cmd) diff --git a/docs/changelog.md b/docs/changelog.md index 8eb88707..b526dc4c 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,3 +1,64 @@ +▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ +# 2022-0810-2135 `v1.3.11` webworkers + +* read-only demo server at https://a.ocv.me/pub/demo/ +* latest gzip edition of the sfx: [v1.0.14](https://github.com/9001/copyparty/releases/tag/v1.0.14#:~:text=release-specific%20notes) + +## new features +* multithreaded file hashing! **300%** average speed increase + * when uploading files through the browser client, based on web-workers + * `4.5x` faster on http from a laptop -- `146` -> `670` MiB/s + * ` 30%` faster on https from a laptop -- `552` -> `716` MiB/s + * `4.2x` faster on http from android -- `13.5` -> `57.1` MiB/s + * `5.3x` faster on https from android -- `13.8` -> `73.3` MiB/s + * can be disabled using the `mt` togglebtn in the settings pane, for example if your phone runs out of memory (it eats ~250 MiB extra RAM) + * `2.3x` faster [u2cli](https://github.com/9001/copyparty/tree/hovudstraum/bin#up2kpy) (cmd-line client) -- `398` -> `930` MiB/s + * `2.4x` faster filesystem indexing on the server + * thx to @kipukun for the webworker suggestion! + +## bugfixes +* ux: reset scroll when navigating into a new folder +* u2cli: better errormsg if the server's tls certificate got rejected +* js: more futureproof cloudflare-challenge detection (they got a new one recently) + +## other changes +* print warning if the python interpreter was built with an unsafe sqlite +* u2cli: add helpful messages on how to make it run on python 2.6 + +**trivia:** due to a [chrome bug](https://bugs.chromium.org/p/chromium/issues/detail?id=1352210), http can sometimes be faster than https now ¯\\\_(ツ)\_/¯ + + + +▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ +# 2022-0803-2340 `v1.3.10` folders first + +* read-only demo server at https://a.ocv.me/pub/demo/ +* latest gzip edition of the sfx: [v1.0.14](https://github.com/9001/copyparty/releases/tag/v1.0.14#:~:text=release-specific%20notes) + +## new features +* faster + * tag scanner + * on windows: uploading to fat32 or smb +* toggle-button to sort folders before files (default-on) + * almost the same as before, but now also when sorting by size / date +* repeatedly hit `ctrl-c` to force-quit if everything dies +* new file-indexing guards + * `--xdev` / volflag `:c,xdev` stops if it hits another filesystem (bindmount/symlink) + * `--xvol` / volflag `:c,xvol` does not follow symlinks pointing outside the volume + * only affects file indexing -- does NOT prevent access! + +## bugfixes +* forget uploads that failed to initialize (allows retry in another folder) +* wrong filekeys in upload response if volume path contained a symlink +* faster shutdown on `ctrl-c` while hashing huge files +* ux: fix navpane covering files on horizontal scroll + +## other changes +* include version info in the base64 crash-message +* ux: make upload errors more visible on mobile + + + ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ # 2022-0727-1407 `v1.3.8` more async