From 7704b9c8a21cd0da35d6d0680cdd7e178871fec7 Mon Sep 17 00:00:00 2001 From: ed Date: Sat, 10 Dec 2022 10:01:33 +0000 Subject: [PATCH] sqlite durability profiles --- copyparty/__main__.py | 39 +++++++++++++++++++++++++++------------ copyparty/authsrv.py | 11 +++++------ copyparty/u2idx.py | 2 +- copyparty/up2k.py | 38 ++++++++++++++++++++++++++++++-------- copyparty/util.py | 3 --- 5 files changed, 63 insertions(+), 30 deletions(-) diff --git a/copyparty/__main__.py b/copyparty/__main__.py index a4857ac4..24f1ed8b 100755 --- a/copyparty/__main__.py +++ b/copyparty/__main__.py @@ -540,10 +540,7 @@ def run_argparse( \033[36mnohash=\\.iso$\033[35m skips hashing file contents if path matches *.iso \033[36mnoidx=\\.iso$\033[35m fully ignores the contents at paths matching *.iso \033[36mnoforget$\033[35m don't forget files when deleted from disk - \033[36mnowal\033[35m guarantee zero dataloss on powerloss by disabling wal - \033[36mwal\033[35m enable wal (default; overrides --no-wal) - \033[36mnosync\033[35m switch to unsafe mode for extremely fast uploads - \033[36msync\033[35m normal corruption-protected db config + \033[36mdbd=[acid|swal|wal|yolo]\033[35m database speed-durability tradeoff \033[36mxlink$\033[35m cross-volume dupe detection / linking \033[36mxdev\033[35m do not descend into other filesystems \033[36mxvol\033[35m skip symlinks leaving the volume root @@ -604,6 +601,25 @@ def run_argparse( """ ), ], + [ + "dbd", + "database durability profiles", + dedent( + """ + mainly affects uploads of many small files on slow HDDs; speeds measured uploading 520 files on a WD20SPZX (SMR 2.5" 5400rpm 4kb) + + \033[32macid\033[0m = extremely safe but slow; the old default. Should never lose any data no matter what + + \033[32mswal\033[0m = 2.4x faster uploads yet 99.9%% as safe -- theoretical chance of losing metadata for the ~200 most recently uploaded files if there's a power-loss or your OS crashes + + \033[32mwal\033[0m = another 21x faster on HDDs yet 90%% as safe; same pitfall as \033[33mswal\033[0m except more likely + + \033[32myolo\033[0m = another 1.5x faster, and removes the occasional sudden upload-pause while the disk syncs, but now you're at risk of losing the entire database in a powerloss / OS-crash + + profiles can be set globally (--dbd=yolo), or per-volume with volflags: -v ~/Music:music:r:c,dbd=acid + """ + ), + ], ] # fmt: off @@ -673,15 +689,15 @@ def run_argparse( ap2 = ap.add_argument_group("Zeroconf options") ap2.add_argument("-z", action="store_true", help="enable all zeroconf backends (mdns, ssdp)") - ap2.add_argument("--z-on", metavar="NICS/NETS", type=u, default="", help="enable zeroconf ONLY on the comma-separated list of subnets and/or interface names/indexes\n └─example: \033[32meth0, wlo1, virhost0, 192.168.123.0/24, fd00:fda::/96\033[0m") - ap2.add_argument("--z-off", metavar="NICS/NETS", type=u, default="", help="disable zeroconf on the comma-separated list of subnets and/or interface names/indexes") + ap2.add_argument("--z-on", metavar="NETS", type=u, default="", help="enable zeroconf ONLY on the comma-separated list of subnets and/or interface names/indexes\n └─example: \033[32meth0, wlo1, virhost0, 192.168.123.0/24, fd00:fda::/96\033[0m") + ap2.add_argument("--z-off", metavar="NETS", type=u, default="", help="disable zeroconf on the comma-separated list of subnets and/or interface names/indexes") ap2.add_argument("-zv", action="store_true", help="verbose all zeroconf backends") ap2.add_argument("--mc-hop", metavar="SEC", type=int, default=0, help="rejoin multicast groups every SEC seconds (workaround for some switches/routers which cause mDNS to suddenly stop working after some time); try [\033[32m300\033[0m] or [\033[32m180\033[0m]") ap2 = ap.add_argument_group("Zeroconf-mDNS options:") ap2.add_argument("--zm", action="store_true", help="announce the enabled protocols over mDNS (multicast DNS-SD) -- compatible with KDE, gnome, macOS, ...") - ap2.add_argument("--zm-on", metavar="NICS/NETS", type=u, default="", help="enable zeroconf ONLY on the comma-separated list of subnets and/or interface names/indexes") - ap2.add_argument("--zm-off", metavar="NICS/NETS", type=u, default="", help="disable zeroconf on the comma-separated list of subnets and/or interface names/indexes") + ap2.add_argument("--zm-on", metavar="NETS", type=u, default="", help="enable zeroconf ONLY on the comma-separated list of subnets and/or interface names/indexes") + ap2.add_argument("--zm-off", metavar="NETS", type=u, default="", help="disable zeroconf on the comma-separated list of subnets and/or interface names/indexes") ap2.add_argument("--zm4", action="store_true", help="IPv4 only -- try this if some clients can't connect") ap2.add_argument("--zm6", action="store_true", help="IPv6 only") ap2.add_argument("--zmv", action="store_true", help="verbose mdns") @@ -697,8 +713,8 @@ def run_argparse( ap2 = ap.add_argument_group("Zeroconf-SSDP options:") ap2.add_argument("--zs", action="store_true", help="announce the enabled protocols over SSDP -- compatible with Windows") - ap2.add_argument("--zs-on", metavar="NICS/NETS", type=u, default="", help="enable zeroconf ONLY on the comma-separated list of subnets and/or interface names/indexes") - ap2.add_argument("--zs-off", metavar="NICS/NETS", type=u, default="", help="disable zeroconf on the comma-separated list of subnets and/or interface names/indexes") + ap2.add_argument("--zs-on", metavar="NETS", type=u, default="", help="enable zeroconf ONLY on the comma-separated list of subnets and/or interface names/indexes") + ap2.add_argument("--zs-off", metavar="NETS", type=u, default="", help="disable zeroconf on the comma-separated list of subnets and/or interface names/indexes") ap2.add_argument("--zsv", action="store_true", help="verbose SSDP") ap2.add_argument("--zsl", metavar="PATH", type=u, default="/?hc", help="location to include in the url (or a complete external URL), for example [\033[32mpriv/?pw=hunter2\033[0m] (goes directly to /priv/ with password hunter2) or [\033[32m?hc=priv&pw=hunter2\033[0m] (shows mounting options for /priv/ with password)") ap2.add_argument("--zsid", metavar="UUID", type=u, default=uuid.uuid4().urn[4:], help="USN (device identifier) to announce") @@ -819,8 +835,7 @@ def run_argparse( ap2.add_argument("--no-idx", metavar="PTN", type=u, help="regex: disable indexing of matching paths during e2ds folder scans (volflag=noidx)") ap2.add_argument("--no-dhash", action="store_true", help="disable rescan acceleration; do full database integrity check -- makes the db ~5%% smaller and bootup/rescans 3~10x slower") ap2.add_argument("--no-forget", action="store_true", help="never forget indexed files, even when deleted from disk -- makes it impossible to ever upload the same file twice (volflag=noforget)") - ap2.add_argument("--no-wal", action="store_true", help="1%% faster searches, more reliable upload performance, and slightly more resistant to dataloss, but makes uploads up to 2x slower (volflag=nowal)") - ap2.add_argument("--no-sync", action="store_true", help="make uploads extremely fast, but removes corruption protection -- if the OS crashes, you might lose the db (volflag=nosync)") + ap2.add_argument("--dbd", metavar="PROFILE", default="wal", help="database durability profile; sets the tradeoff between robustness and speed, see --help-dbd (volflag=dbd)") ap2.add_argument("--xlink", action="store_true", help="on upload: check all volumes for dupes, not just the target volume (volflag=xlink)") ap2.add_argument("--xdev", action="store_true", help="do not descend into other filesystems (symlink or bind-mount to another HDD, ...) (volflag=xdev)") ap2.add_argument("--xvol", action="store_true", help="skip symlinks leaving the volume root (volflag=xvol)") diff --git a/copyparty/authsrv.py b/copyparty/authsrv.py index 03ed41f5..1b168fb3 100644 --- a/copyparty/authsrv.py +++ b/copyparty/authsrv.py @@ -15,7 +15,6 @@ from datetime import datetime from .__init__ import ANYWIN, TYPE_CHECKING, WINDOWS from .bos import bos from .util import ( - DROPLICATIONS, IMPLICATIONS, META_NOBOTS, SQLITE_VER, @@ -1123,8 +1122,6 @@ class AuthSrv(object): for ga, vf in ( ("no_forget", "noforget"), ("no_dupe", "nodupe"), - ("no_wal", "nowal"), - ("no_sync", "nosync"), ("magic", "magic"), ("xlink", "xlink"), ): @@ -1139,9 +1136,11 @@ class AuthSrv(object): if k1 in vol.flags: vol.flags[k2] = False - for k1, k2 in DROPLICATIONS: - if k1 in vol.flags: - vol.flags.pop(k2) + dbds = "acid|swal|wal|yolo" + vol.flags["dbd"] = dbd = vol.flags.get("dbd") or self.args.dbd + if dbd not in dbds.split("|"): + t = "invalid dbd [{}]; must be one of [{}]" + raise Exception(t.format(dbd, dbds)) # default tag cfgs if unset if "mte" not in vol.flags: diff --git a/copyparty/u2idx.py b/copyparty/u2idx.py index 2c357bba..5233ca22 100644 --- a/copyparty/u2idx.py +++ b/copyparty/u2idx.py @@ -97,7 +97,7 @@ class U2idx(object): return None cur = None - if ANYWIN and self.args.no_wal: + if ANYWIN and bos.path.exists(db_path + "-wal"): uri = "" try: uri = "{}?mode=ro&nolock=1".format(Path(db_path).as_uri()) diff --git a/copyparty/up2k.py b/copyparty/up2k.py index c7312e12..fb189273 100644 --- a/copyparty/up2k.py +++ b/copyparty/up2k.py @@ -558,7 +558,7 @@ class Up2k(object): return False for vol in all_vols.values(): - if "nowal" in vol.flags: + if vol.flags["dbd"] == "acid": continue reg = self.register_vpath(vol.realpath, vol.flags) @@ -696,15 +696,37 @@ class Up2k(object): cur = self._open_db(db_path) self.cur[ptop] = cur + # speeds measured uploading 520 small files on a WD20SPZX (SMR 2.5" 5400rpm 4kb) + dbd = flags["dbd"] + if dbd == "acid": + # 217.5s; python-defaults + zs = "delete" + sync = "full" + elif dbd == "swal": + # 88.0s; still 99.9% safe (can lose a bit of on OS crash) + zs = "wal" + sync = "full" + elif dbd == "yolo": + # 2.7s; may lose entire db on OS crash + zs = "wal" + sync = "off" + else: + # 4.1s; corruption-safe but more likely to lose wal + zs = "wal" + sync = "normal" + try: - zs = "delete" if "nowal" in flags else "wal" - cur.execute("pragma journal_mode=" + zs) - except: - pass - - if "nosync" in flags: - cur.execute("pragma synchronous=0") + amode = cur.execute("pragma journal_mode=" + zs).fetchone()[0] + if amode.lower() != zs.lower(): + t = "sqlite failed to set journal_mode {}; got {}" + raise Exception(t.format(zs, amode)) + except Exception as ex: + if sync != "off": + sync = "full" + t = "reverting to sync={} because {}" + self.log(t.format(sync, ex)) + cur.execute("pragma synchronous=" + sync) cur.connection.commit() return cur, db_path except: diff --git a/copyparty/util.py b/copyparty/util.py index ee8c79eb..dcc52bdc 100644 --- a/copyparty/util.py +++ b/copyparty/util.py @@ -202,9 +202,6 @@ if ANYWIN: UNPLICATIONS = [["no_dav", "daw"]] -DROPLICATIONS = [["wal", "nowal"], ["sync", "nosync"]] - - MIMES = { "opus": "audio/ogg; codecs=opus", }