From d162502c385622c7cac9fcab48f807ca8067a71f Mon Sep 17 00:00:00 2001 From: ed Date: Mon, 7 Jul 2025 01:05:57 +0200 Subject: [PATCH] add idp-volume persistence (optional); it keeps track of all seen users/groups by default, but nothing takes effect unless --idp-store=3 or 2 --- copyparty/__main__.py | 3 ++ copyparty/authsrv.py | 66 +++++++++++++++++++++-- copyparty/svchub.py | 121 +++++++++++++++++++++++++++++++++--------- tests/util.py | 2 +- 4 files changed, 162 insertions(+), 30 deletions(-) diff --git a/copyparty/__main__.py b/copyparty/__main__.py index 7cfee783..a92770d7 100644 --- a/copyparty/__main__.py +++ b/copyparty/__main__.py @@ -1093,12 +1093,15 @@ def add_cert(ap, cert_path): def add_auth(ap): + idp_db = os.path.join(E.cfg, "idp.db") ses_db = os.path.join(E.cfg, "sessions.db") ap2 = ap.add_argument_group('IdP / identity provider / user authentication options') ap2.add_argument("--idp-h-usr", metavar="HN", type=u, default="", help="bypass the copyparty authentication checks if the request-header \033[33mHN\033[0m contains a username to associate the request with (for use with authentik/oauth/...)\n\033[1;31mWARNING:\033[0m if you enable this, make sure clients are unable to specify this header themselves; must be washed away and replaced by a reverse-proxy") ap2.add_argument("--idp-h-grp", metavar="HN", type=u, default="", help="assume the request-header \033[33mHN\033[0m contains the groupname of the requesting user; can be referenced in config files for group-based access control") ap2.add_argument("--idp-h-key", metavar="HN", type=u, default="", help="optional but recommended safeguard; your reverse-proxy will insert a secret header named \033[33mHN\033[0m into all requests, and the other IdP headers will be ignored if this header is not present") ap2.add_argument("--idp-gsep", metavar="RE", type=u, default="|:;+,", help="if there are multiple groups in \033[33m--idp-h-grp\033[0m, they are separated by one of the characters in \033[33mRE\033[0m") + ap2.add_argument("--idp-db", metavar="PATH", type=u, default=idp_db, help="where to store the known IdP users/groups (if you run multiple copyparty instances, make sure they use different DBs)") + ap2.add_argument("--idp-store", metavar="N", type=int, default=1, help="how to use \033[33m--idp-db\033[0m; [\033[32m0\033[0m] = entirely disable, [\033[32m1\033[0m] = write-only (effectively disabled), [\033[32m2\033[0m] = remember users, [\033[32m3\033[0m] = remember users and groups.\nNOTE: Will remember and restore the IdP-volumes of all users for all eternity if set to 2 or 3, even when user is deleted from your IdP") ap2.add_argument("--no-bauth", action="store_true", help="disable basic-authentication support; do not accept passwords from the 'Authenticate' header at all. NOTE: This breaks support for the android app") ap2.add_argument("--bauth-last", action="store_true", help="keeps basic-authentication enabled, but only as a last-resort; if a cookie is also provided then the cookie wins") ap2.add_argument("--ses-db", metavar="PATH", type=u, default=ses_db, help="where to store the sessions database (if you run multiple copyparty instances, make sure they use different DBs)") diff --git a/copyparty/authsrv.py b/copyparty/authsrv.py index 5e42f07a..91e84051 100644 --- a/copyparty/authsrv.py +++ b/copyparty/authsrv.py @@ -21,6 +21,7 @@ from .util import ( DEF_MTE, DEF_MTH, EXTS, + HAVE_SQLITE3, IMPLICATIONS, MIMES, SQLITE_VER, @@ -32,6 +33,7 @@ from .util import ( afsenc, get_df, humansize, + min_ex, odfusion, read_utf8, relchk, @@ -44,6 +46,9 @@ from .util import ( vsplit, ) +if HAVE_SQLITE3: + import sqlite3 + if True: # pylint: disable=using-constant-test from collections.abc import Iterable @@ -935,6 +940,10 @@ class AuthSrv(object): return False self.idp_accs[uname] = gnames + try: + self._update_idp_db(uname, gname) + except: + self.log("failed to update the --idp-db:\n%s" % (min_ex(),), 3) t = "reinitializing due to new user from IdP: [%r:%r]" self.log(t % (uname, gnames), 3) @@ -947,6 +956,22 @@ class AuthSrv(object): broker.ask("reload", False, True).get() return True + def _update_idp_db(self, uname, gname): + if not self.args.idp_store: + return + + assert sqlite3 # type: ignore # !rm + + db = sqlite3.connect(self.args.idp_db) + cur = db.cursor() + + cur.execute("delete from us where un = ?", (uname,)) + cur.execute("insert into us values (?,?)", (uname, gname)) + + db.commit() + cur.close() + db.close() + def _map_volume_idp( self, src: str, @@ -1095,6 +1120,7 @@ class AuthSrv(object): * any non-zero value from IdP group header * otherwise take --grps / [groups] """ + self.load_idp_db(bool(self.idp_accs)) ret = {un: gns[:] for un, gns in self.idp_accs.items()} ret.update({zs: [""] for zs in acct if zs not in ret}) for gn, uns in grps.items(): @@ -1655,7 +1681,7 @@ class AuthSrv(object): shr = enshare[1:-1] shrs = enshare[1:] if enshare: - import sqlite3 + assert sqlite3 # type: ignore # !rm zsd = {"d2d": True, "tcolor": self.args.tcolor} shv = VFS(self.log_func, "", shr, shr, AXS(), zsd) @@ -2621,6 +2647,40 @@ class AuthSrv(object): zs = str(vol.flags.get("tcolor") or self.args.tcolor) vol.flags["tcolor"] = zs.lstrip("#") + def load_idp_db(self, quiet=False) -> None: + # mutex me + level = self.args.idp_store + if level < 2 or not self.args.idp_h_usr: + return + + assert sqlite3 # type: ignore # !rm + + db = sqlite3.connect(self.args.idp_db) + cur = db.cursor() + + gsep = self.args.idp_gsep + n = [] + for uname, gname in cur.execute("select un, gs from us"): + if level < 3: + if uname in self.idp_accs: + continue + gname = "" + gnames = [x.strip() for x in gsep.split(gname)] + gnames.sort() + + # self.idp_usr_gh[uname] = gname + self.idp_accs[uname] = gnames + n.append(uname) + + cur.close() + db.close() + + if n and not quiet: + t = ", ".join(n[:9]) + if len(n) > 9: + t += "..." + self.log("found %d IdP users in db (%s)" % (len(n), t)) + def load_sessions(self, quiet=False) -> None: # mutex me if self.args.no_ses: @@ -2628,7 +2688,7 @@ class AuthSrv(object): self.sesa = {} return - import sqlite3 + assert sqlite3 # type: ignore # !rm ases = {} blen = (self.args.ses_len // 4) * 4 # 3 bytes in 4 chars @@ -2675,7 +2735,7 @@ class AuthSrv(object): if self.args.no_ses: return - import sqlite3 + assert sqlite3 # type: ignore # !rm db = sqlite3.connect(self.args.ses_db) cur = db.cursor() diff --git a/copyparty/svchub.py b/copyparty/svchub.py index e4f1110a..e7e3b86c 100644 --- a/copyparty/svchub.py +++ b/copyparty/svchub.py @@ -88,6 +88,7 @@ if PY2: range = xrange # type: ignore +VER_IDP_DB = 1 VER_SESSION_DB = 1 VER_SHARES_DB = 2 @@ -258,11 +259,15 @@ class SvcHub(object): self.log("root", "effective %s is %s" % (zs, getattr(args, zs))) if args.ah_cli or args.ah_gen: + args.idp_store = 0 args.no_ses = True args.shr = "" + if args.idp_store and args.idp_h_usr: + self.setup_db("idp") + if not self.args.no_ses: - self.setup_session_db() + self.setup_db("ses") args.shr1 = "" if args.shr: @@ -421,26 +426,58 @@ class SvcHub(object): except: pass - def setup_session_db(self) -> None: + def _db_onfail_ses(self) -> None: + self.args.no_ses = True + + def _db_onfail_idp(self) -> None: + self.args.idp_store = 0 + + def setup_db(self, which: str) -> None: + """ + the "non-mission-critical" databases; if something looks broken then just nuke it + """ + if which == "ses": + native_ver = VER_SESSION_DB + db_path = self.args.ses_db + desc = "sessions-db" + pathopt = "ses-db" + sanchk_q = "select count(*) from us" + createfun = self._create_session_db + failfun = self._db_onfail_ses + elif which == "idp": + native_ver = VER_IDP_DB + db_path = self.args.idp_db + desc = "idp-db" + pathopt = "idp-db" + sanchk_q = "select count(*) from us" + createfun = self._create_idp_db + failfun = self._db_onfail_idp + else: + raise Exception("unknown cachetype") + + if not db_path.endswith(".db"): + zs = "config option --%s (the %s) was configured to [%s] which is invalid; must be a filepath ending with .db" + self.log("root", zs % (pathopt, desc, db_path), 1) + raise Exception(BAD_CFG) + if not HAVE_SQLITE3: - self.args.no_ses = True - t = "WARNING: sqlite3 not available; disabling sessions, will use plaintext passwords in cookies" - self.log("root", t, 3) + failfun() + if which == "ses": + zs = "disabling sessions, will use plaintext passwords in cookies" + elif which == "idp": + zs = "disabling idp-db, will be unable to remember IdP-volumes after a restart" + self.log("root", "WARNING: sqlite3 not available; %s" % (zs,), 3) return assert sqlite3 # type: ignore # !rm - # policy: - # the sessions-db is whatever, if something looks broken then just nuke it - - db_path = self.args.ses_db db_lock = db_path + ".lock" try: create = not os.path.getsize(db_path) except: create = True zs = "creating new" if create else "opening" - self.log("root", "%s sessions-db %s" % (zs, db_path)) + self.log("root", "%s %s %s" % (zs, desc, db_path)) for tries in range(2): sver = 0 @@ -450,17 +487,19 @@ class SvcHub(object): try: zs = "select v from kv where k='sver'" sver = cur.execute(zs).fetchall()[0][0] - if sver > VER_SESSION_DB: - zs = "this version of copyparty only understands session-db v%d and older; the db is v%d" - raise Exception(zs % (VER_SESSION_DB, sver)) + if sver > native_ver: + zs = "this version of copyparty only understands %s v%d and older; the db is v%d" + raise Exception(zs % (desc, native_ver, sver)) - cur.execute("select count(*) from us").fetchone() + cur.execute(sanchk_q).fetchone() except: if sver: raise - sver = 1 - self._create_session_db(cur) - err = self._verify_session_db(cur, sver, db_path) + sver = createfun(cur) + + err = self._verify_db( + cur, which, pathopt, db_path, desc, sver, native_ver + ) if err: tries = 99 self.args.no_ses = True @@ -468,10 +507,10 @@ class SvcHub(object): break except Exception as ex: - if tries or sver > VER_SESSION_DB: + if tries or sver > native_ver: raise - t = "sessions-db is unusable; deleting and recreating: %r" - self.log("root", t % (ex,), 3) + t = "%s is unusable; deleting and recreating: %r" + self.log("root", t % (desc, ex), 3) try: cur.close() # type: ignore except: @@ -486,7 +525,7 @@ class SvcHub(object): pass os.unlink(db_path) - def _create_session_db(self, cur: "sqlite3.Cursor") -> None: + def _create_session_db(self, cur: "sqlite3.Cursor") -> int: sch = [ r"create table kv (k text, v int)", r"create table us (un text, si text, t0 int)", @@ -499,8 +538,31 @@ class SvcHub(object): for cmd in sch: cur.execute(cmd) self.log("root", "created new sessions-db") + return 1 - def _verify_session_db(self, cur: "sqlite3.Cursor", sver: int, db_path: str) -> str: + def _create_idp_db(self, cur: "sqlite3.Cursor") -> int: + sch = [ + r"create table kv (k text, v int)", + r"create table us (un text, gs text)", + # username, groups + r"create index us_un on us(un)", + r"insert into kv values ('sver', 1)", + ] + for cmd in sch: + cur.execute(cmd) + self.log("root", "created new idp-db") + return 1 + + def _verify_db( + self, + cur: "sqlite3.Cursor", + which: str, + pathopt: str, + db_path: str, + desc: str, + sver: int, + native_ver: int, + ) -> str: # ensure writable (maybe owned by other user) db = cur.connection @@ -512,9 +574,16 @@ class SvcHub(object): except: owner = 0 + if which == "ses": + cons = "Will now disable sessions and instead use plaintext passwords in cookies." + elif which == "idp": + cons = "Each IdP-volume will not become available until its associated user sends their first request." + else: + raise Exception() + if not lock_file(db_path + ".lock"): - t = "the sessions-db [%s] is already in use by another copyparty instance (pid:%d). This is not supported; please provide another database with --ses-db or give this copyparty-instance its entirely separate config-folder by setting another path in the XDG_CONFIG_HOME env-var. You can also disable this safeguard by setting env-var PRTY_NO_DB_LOCK=1. Will now disable sessions and instead use plaintext passwords in cookies." - return t % (db_path, owner) + t = "the %s [%s] is already in use by another copyparty instance (pid:%d). This is not supported; please provide another database with --%s or give this copyparty-instance its entirely separate config-folder by setting another path in the XDG_CONFIG_HOME env-var. You can also disable this safeguard by setting env-var PRTY_NO_DB_LOCK=1. %s" + return t % (desc, db_path, owner, pathopt, cons) vars = (("pid", os.getpid()), ("ts", int(time.time() * 1000))) if owner: @@ -526,9 +595,9 @@ class SvcHub(object): for k, v in vars: cur.execute("insert into kv values(?, ?)", (k, v)) - if sver < VER_SESSION_DB: + if sver < native_ver: cur.execute("delete from kv where k='sver'") - cur.execute("insert into kv values('sver',?)", (VER_SESSION_DB,)) + cur.execute("insert into kv values('sver',?)", (native_ver,)) db.commit() cur.close() diff --git a/tests/util.py b/tests/util.py index cdc6b6c4..427bdf3d 100644 --- a/tests/util.py +++ b/tests/util.py @@ -158,7 +158,7 @@ class Cfg(Namespace): ex = "au_vol dl_list mtab_age reg_cap s_thead s_tbody tail_tmax tail_who th_convt ups_who zip_who" ka.update(**{k: 9 for k in ex.split()}) - ex = "db_act forget_ip k304 loris no304 nosubtle re_maxage rproxy rsp_jtr rsp_slp s_wr_slp snap_wri theme themes turbo u2ow zipmaxn zipmaxs" + ex = "db_act forget_ip idp_store k304 loris no304 nosubtle re_maxage rproxy rsp_jtr rsp_slp s_wr_slp snap_wri theme themes turbo u2ow zipmaxn zipmaxs" ka.update(**{k: 0 for k in ex.split()}) ex = "ah_alg bname chpw_db doctitle df exit favico idp_h_usr ipa html_head lg_sba lg_sbf log_fk md_sba md_sbf name og_desc og_site og_th og_title og_title_a og_title_v og_title_i shr tcolor textfiles unlist vname xff_src zipmaxt R RS SR"