switch to wal for ~2x faster uploads

copyparty/__main__.py

@@ -540,6 +540,8 @@ def run_argparse(
               \033[36mnohash=\\.iso$\033[35m skips hashing file contents if path matches *.iso
               \033[36mnoidx=\\.iso$\033[35m fully ignores the contents at paths matching *.iso
               \033[36mnoforget$\033[35m don't forget files when deleted from disk
+              \033[36mnowal\033[35m guarantee zero dataloss on powerloss by disabling wal
+              \033[36mwal\033[35m enable wal (default; overrides --no-wal)
               \033[36mxlink$\033[35m cross-volume dupe detection / linking
               \033[36mxdev\033[35m do not descend into other filesystems
               \033[36mxvol\033[35m skip symlinks leaving the volume root
@@ -813,6 +815,7 @@ def run_argparse(
     ap2.add_argument("--no-idx", metavar="PTN", type=u, help="regex: disable indexing of matching paths during e2ds folder scans (volflag=noidx)")
     ap2.add_argument("--no-dhash", action="store_true", help="disable rescan acceleration; do full database integrity check -- makes the db ~5%% smaller and bootup/rescans 3~10x slower")
     ap2.add_argument("--no-forget", action="store_true", help="never forget indexed files, even when deleted from disk -- makes it impossible to ever upload the same file twice (volflag=noforget)")
+    ap2.add_argument("--no-wal", action="store_true", help="1%% faster searches, more reliable upload performance, and slightly more resistant to dataloss, but makes uploads up to 2x slower (volflag=nowal)")
     ap2.add_argument("--xlink", action="store_true", help="on upload: check all volumes for dupes, not just the target volume (volflag=xlink)")
     ap2.add_argument("--xdev", action="store_true", help="do not descend into other filesystems (symlink or bind-mount to another HDD, ...) (volflag=xdev)")
     ap2.add_argument("--xvol", action="store_true", help="skip symlinks leaving the volume root (volflag=xvol)")

copyparty/authsrv.py

@@ -15,6 +15,7 @@ from datetime import datetime
 from .__init__ import ANYWIN, TYPE_CHECKING, WINDOWS
 from .bos import bos
 from .util import (
+    DROPLICATIONS,
     IMPLICATIONS,
     META_NOBOTS,
     SQLITE_VER,
@@ -1122,6 +1123,7 @@ class AuthSrv(object):
             for ga, vf in (
                 ("no_forget", "noforget"),
                 ("no_dupe", "nodupe"),
+                ("no_wal", "nowal"),
                 ("magic", "magic"),
                 ("xlink", "xlink"),
             ):
@@ -1136,6 +1138,10 @@ class AuthSrv(object):
                 if k1 in vol.flags:
                     vol.flags[k2] = False
 
+            for k1, k2 in DROPLICATIONS:
+                if k1 in vol.flags:
+                    vol.flags.pop(k2)
+
             # default tag cfgs if unset
             if "mte" not in vol.flags:
                 vol.flags["mte"] = self.args.mte

copyparty/httpcli.py

@@ -1302,7 +1302,7 @@ class HttpCli(object):
             if self.args.dotpart:
                 tnam = "." + tnam
 
-            if ("daw" in vfs.flags and self.can_delete) or (
+            if (vfs.flags.get("daw") and self.can_delete) or (
                 not bos.path.exists(os.path.join(fdir, tnam))
                 and bos.path.exists(path)
                 and not bos.path.getsize(path)

copyparty/u2idx.py

@@ -97,14 +97,17 @@ class U2idx(object):
             return None
 
         cur = None
-        if ANYWIN:
+        if ANYWIN and self.args.no_wal:
             uri = ""
             try:
                 uri = "{}?mode=ro&nolock=1".format(Path(db_path).as_uri())
                 cur = sqlite3.connect(uri, 2, uri=True).cursor()
+                cur.execute('pragma table_info("up")').fetchone()
                 self.log("ro: {}".format(db_path))
             except:
                 self.log("could not open read-only: {}\n{}".format(uri, min_ex()))
+                # may not fail until the pragma so unset it
+                cur = None
 
         if not cur:
             # on windows, this steals the write-lock from up2k.deferred_init --

copyparty/up2k.py

@@ -659,6 +659,13 @@ class Up2k(object):
         try:
             cur = self._open_db(db_path)
             self.cur[ptop] = cur
+
+            try:
+                zs = "delete" if "nowal" in flags else "wal"
+                cur.execute("pragma journal_mode=" + zs)
+            except:
+                pass
+
             return cur, db_path
         except:
             msg = "cannot use database at [{}]:\n{}"

copyparty/util.py

@@ -202,6 +202,9 @@ if ANYWIN:
 UNPLICATIONS = [["no_dav", "daw"]]
 
 
+DROPLICATIONS = [["wal", "nowal"]]
+
+
 MIMES = {
     "opus": "audio/ogg; codecs=opus",
 }