diff --git a/README.md b/README.md index d368c031..12483cf0 100644 --- a/README.md +++ b/README.md @@ -2576,6 +2576,7 @@ buggy feature? rip it out by setting any of the following environment variables | `PRTY_NO_IPV6` | disable some ipv6 support (should not be necessary since windows 2000) | | `PRTY_NO_LZMA` | disable streaming xz compression of incoming uploads | | `PRTY_NO_MP` | disable all use of the python `multiprocessing` module (actual multithreading, cpu-count for parsers/thumbnailers) | +| `PRTY_NO_UNIDATA` | disable loading the unicodedata c-extension; saves 400k ram, breaks `--srch-nfkc` | | `PRTY_NO_SQLITE` | disable all database-related functionality (file indexing, metadata indexing, most file deduplication logic) | | `PRTY_NO_TLS` | disable native HTTPS support; if you still want to accept HTTPS connections then TLS must now be terminated by a reverse-proxy | | `PRTY_NO_TPOKE` | disable systemd-tmpfilesd avoider | diff --git a/copyparty/__main__.py b/copyparty/__main__.py index 6e2921bb..c28a9fd2 100644 --- a/copyparty/__main__.py +++ b/copyparty/__main__.py @@ -1882,7 +1882,8 @@ def add_db_general(ap, hcores): ap2.add_argument("--hash-mt", metavar="CORES", type=int, default=hcores, help="num cpu cores to use for file hashing; set 0 or 1 for single-core hashing") ap2.add_argument("--re-maxage", metavar="SEC", type=int, default=0, help="rescan filesystem for changes every \033[33mSEC\033[0m seconds; 0=off (volflag=scan)") ap2.add_argument("--db-act", metavar="SEC", type=float, default=10.0, help="defer any scheduled volume reindexing until \033[33mSEC\033[0m seconds after last db write (uploads, renames, ...)") - ap2.add_argument("--srch-icase", action="store_true", help="case-insensitive search for all unicode characters (the default is icase for just ascii). NOTE: will make searches much slower (around 4x), and NOTE: only applies to filenames/paths, not tags") + ap2.add_argument("--srch-icase", action="store_true", help="case-insensitive search for all unicode characters (the default is icase for just ascii). NOTE: will make searches much slower (around 3x), and NOTE: only applies to filenames/paths, not tags") + ap2.add_argument("--srch-nfkc", action="store_true", help="case-insensitive and unicode-normalization-insensitive search (NFC/NFD) for filenames/paths; slightly slower than \033[33m--srch-icase\033[0m (about 15%% slower / 87%% as fast)") ap2.add_argument("--srch-time", metavar="SEC", type=int, default=45, help="search deadline -- terminate searches running for more than \033[33mSEC\033[0m seconds") ap2.add_argument("--srch-hits", metavar="N", type=int, default=7999, help="max search results to allow clients to fetch; 125 results will be shown initially") ap2.add_argument("--srch-excl", metavar="PTN", type=u, default="", help="regex: exclude files from search results if the file-URL matches \033[33mPTN\033[0m (case-sensitive). Example: [\033[32mpassword|logs/[0-9]\033[0m] any URL containing 'password' or 'logs/DIGIT' (volflag=srch_excl)") diff --git a/copyparty/authsrv.py b/copyparty/authsrv.py index 9fa9b7d8..33180133 100644 --- a/copyparty/authsrv.py +++ b/copyparty/authsrv.py @@ -3037,6 +3037,10 @@ class AuthSrv(object): t = "hint: enable upload deduplication with --dedup (but see readme for consequences)" self.log(t, 6) + if MACOS and not self.args.srch_nfkc: + t = "hint: enable --srch-nfkc to improve unicode filename/path search on MacOS" + self.log(t, 6) + zv, _ = vfs.get("/", "*", False, False) zs = zv.realpath.lower() if zs in ("/", "c:\\") or zs.startswith(r"c:\windows"): diff --git a/copyparty/u2idx.py b/copyparty/u2idx.py index 67c9967a..63beffc1 100644 --- a/copyparty/u2idx.py +++ b/copyparty/u2idx.py @@ -28,6 +28,14 @@ from .util import ( if HAVE_SQLITE3: import sqlite3 +try: + if os.environ.get("PRTY_NO_UNIDATA"): + raise Exception() + + import unicodedata +except: + pass + try: from pathlib import Path except: @@ -43,6 +51,14 @@ if PY2: range = xrange # type: ignore +def norm_icase(txt: str) -> str: + return txt.casefold() if txt else txt + + +def norm_nfkc(txt: str) -> str: + return unicodedata.normalize("NFKC", txt).casefold() if txt else txt + + class U2idx(object): def __init__(self, hsrv: "HttpSrv") -> None: self.log_func = hsrv.log @@ -55,6 +71,10 @@ class U2idx(object): return if self.args.srch_icase: + if self.args.srch_nfkc: + self.normfun = norm_nfkc + else: + self.normfun = norm_icase self._open_db = self._open_db_icase else: self._open_db = self._open_db_std @@ -82,7 +102,7 @@ class U2idx(object): def _open_db_icase(self, *args, **kwargs): db = self._open_db_std(*args, **kwargs) - db.create_function("casefold", 1, lambda x: x.casefold() if x else x) + db.create_function("casefold", 1, self.normfun) return db def shutdown(self) -> None: @@ -323,7 +343,7 @@ class U2idx(object): if icase and "casefold(" in q: try: - v = unicode(v).casefold() + v = self.normfun(unicode(v)) except: v = unicode(v).lower() diff --git a/copyparty/util.py b/copyparty/util.py index e3d8cd43..ac78ad9f 100644 --- a/copyparty/util.py +++ b/copyparty/util.py @@ -429,6 +429,7 @@ IMPLICATIONS = [ ["e2vu", "e2v"], ["e2vp", "e2v"], ["e2v", "e2d"], + ["srch_nfkc", "srch_icase"], ["hardlink_only", "hardlink"], ["hardlink", "dedup"], ["tftpvv", "tftpv"],