add --srch-nfkc (closes #1555);

option to unicode-normalize filepaths during search; useful for:
* macos servers with cjk filenames
* cjk stuff originating from a macos device

README.md

@@ -2576,6 +2576,7 @@ buggy feature? rip it out  by setting any of the following environment variables
 | `PRTY_NO_IPV6`       | disable some ipv6 support (should not be necessary since windows 2000) |
 | `PRTY_NO_LZMA`       | disable streaming xz compression of incoming uploads |
 | `PRTY_NO_MP`         | disable all use of the python `multiprocessing` module (actual multithreading, cpu-count for parsers/thumbnailers) |
+| `PRTY_NO_UNIDATA`    | disable loading the unicodedata c-extension; saves 400k ram, breaks `--srch-nfkc` |
 | `PRTY_NO_SQLITE`     | disable all database-related functionality (file indexing, metadata indexing, most file deduplication logic) |
 | `PRTY_NO_TLS`        | disable native HTTPS support; if you still want to accept HTTPS connections then TLS must now be terminated by a reverse-proxy |
 | `PRTY_NO_TPOKE`      | disable systemd-tmpfilesd avoider |

copyparty/__main__.py

@@ -1882,7 +1882,8 @@ def add_db_general(ap, hcores):
     ap2.add_argument("--hash-mt", metavar="CORES", type=int, default=hcores, help="num cpu cores to use for file hashing; set 0 or 1 for single-core hashing")
     ap2.add_argument("--re-maxage", metavar="SEC", type=int, default=0, help="rescan filesystem for changes every \033[33mSEC\033[0m seconds; 0=off (volflag=scan)")
     ap2.add_argument("--db-act", metavar="SEC", type=float, default=10.0, help="defer any scheduled volume reindexing until \033[33mSEC\033[0m seconds after last db write (uploads, renames, ...)")
-    ap2.add_argument("--srch-icase", action="store_true", help="case-insensitive search for all unicode characters (the default is icase for just ascii). NOTE: will make searches much slower (around 4x), and NOTE: only applies to filenames/paths, not tags")
+    ap2.add_argument("--srch-icase", action="store_true", help="case-insensitive search for all unicode characters (the default is icase for just ascii). NOTE: will make searches much slower (around 3x), and NOTE: only applies to filenames/paths, not tags")
+    ap2.add_argument("--srch-nfkc", action="store_true", help="case-insensitive and unicode-normalization-insensitive search (NFC/NFD) for filenames/paths; slightly slower than \033[33m--srch-icase\033[0m (about 15%% slower / 87%% as fast)")
     ap2.add_argument("--srch-time", metavar="SEC", type=int, default=45, help="search deadline -- terminate searches running for more than \033[33mSEC\033[0m seconds")
     ap2.add_argument("--srch-hits", metavar="N", type=int, default=7999, help="max search results to allow clients to fetch; 125 results will be shown initially")
     ap2.add_argument("--srch-excl", metavar="PTN", type=u, default="", help="regex: exclude files from search results if the file-URL matches \033[33mPTN\033[0m (case-sensitive). Example: [\033[32mpassword|logs/[0-9]\033[0m] any URL containing 'password' or 'logs/DIGIT' (volflag=srch_excl)")

copyparty/authsrv.py

@@ -3037,6 +3037,10 @@ class AuthSrv(object):
                 t = "hint: enable upload deduplication with --dedup (but see readme for consequences)"
                 self.log(t, 6)
 
+            if MACOS and not self.args.srch_nfkc:
+                t = "hint: enable --srch-nfkc to improve unicode filename/path search on MacOS"
+                self.log(t, 6)
+
             zv, _ = vfs.get("/", "*", False, False)
             zs = zv.realpath.lower()
             if zs in ("/", "c:\\") or zs.startswith(r"c:\windows"):

copyparty/u2idx.py

@@ -28,6 +28,14 @@ from .util import (
 if HAVE_SQLITE3:
     import sqlite3
 
+try:
+    if os.environ.get("PRTY_NO_UNIDATA"):
+        raise Exception()
+
+    import unicodedata
+except:
+    pass
+
 try:
     from pathlib import Path
 except:
@@ -43,6 +51,14 @@ if PY2:
     range = xrange  # type: ignore
 
 
+def norm_icase(txt: str) -> str:
+    return txt.casefold() if txt else txt
+
+
+def norm_nfkc(txt: str) -> str:
+    return unicodedata.normalize("NFKC", txt).casefold() if txt else txt
+
+
 class U2idx(object):
     def __init__(self, hsrv: "HttpSrv") -> None:
         self.log_func = hsrv.log
@@ -55,6 +71,10 @@ class U2idx(object):
             return
 
         if self.args.srch_icase:
+            if self.args.srch_nfkc:
+                self.normfun = norm_nfkc
+            else:
+                self.normfun = norm_icase
             self._open_db = self._open_db_icase
         else:
             self._open_db = self._open_db_std
@@ -82,7 +102,7 @@ class U2idx(object):
 
     def _open_db_icase(self, *args, **kwargs):
         db = self._open_db_std(*args, **kwargs)
-        db.create_function("casefold", 1, lambda x: x.casefold() if x else x)
+        db.create_function("casefold", 1, self.normfun)
         return db
 
     def shutdown(self) -> None:
@@ -323,7 +343,7 @@ class U2idx(object):
 
             if icase and "casefold(" in q:
                 try:
-                    v = unicode(v).casefold()
+                    v = self.normfun(unicode(v))
                 except:
                     v = unicode(v).lower()
 

copyparty/util.py

@@ -429,6 +429,7 @@ IMPLICATIONS = [
     ["e2vu", "e2v"],
     ["e2vp", "e2v"],
     ["e2v", "e2d"],
+    ["srch_nfkc", "srch_icase"],
     ["hardlink_only", "hardlink"],
     ["hardlink", "dedup"],
     ["tftpvv", "tftpv"],