add idp-volume persistence (optional);

it keeps track of all seen users/groups by default,
but nothing takes effect unless --idp-store=3 or 2

copyparty/__main__.py

@@ -1093,12 +1093,15 @@ def add_cert(ap, cert_path):
 
 
 def add_auth(ap):
+    idp_db = os.path.join(E.cfg, "idp.db")
     ses_db = os.path.join(E.cfg, "sessions.db")
     ap2 = ap.add_argument_group('IdP / identity provider / user authentication options')
     ap2.add_argument("--idp-h-usr", metavar="HN", type=u, default="", help="bypass the copyparty authentication checks if the request-header \033[33mHN\033[0m contains a username to associate the request with (for use with authentik/oauth/...)\n\033[1;31mWARNING:\033[0m if you enable this, make sure clients are unable to specify this header themselves; must be washed away and replaced by a reverse-proxy")
     ap2.add_argument("--idp-h-grp", metavar="HN", type=u, default="", help="assume the request-header \033[33mHN\033[0m contains the groupname of the requesting user; can be referenced in config files for group-based access control")
     ap2.add_argument("--idp-h-key", metavar="HN", type=u, default="", help="optional but recommended safeguard; your reverse-proxy will insert a secret header named \033[33mHN\033[0m into all requests, and the other IdP headers will be ignored if this header is not present")
     ap2.add_argument("--idp-gsep", metavar="RE", type=u, default="|:;+,", help="if there are multiple groups in \033[33m--idp-h-grp\033[0m, they are separated by one of the characters in \033[33mRE\033[0m")
+    ap2.add_argument("--idp-db", metavar="PATH", type=u, default=idp_db, help="where to store the known IdP users/groups (if you run multiple copyparty instances, make sure they use different DBs)")
+    ap2.add_argument("--idp-store", metavar="N", type=int, default=1, help="how to use \033[33m--idp-db\033[0m; [\033[32m0\033[0m] = entirely disable, [\033[32m1\033[0m] = write-only (effectively disabled), [\033[32m2\033[0m] = remember users, [\033[32m3\033[0m] = remember users and groups.\nNOTE: Will remember and restore the IdP-volumes of all users for all eternity if set to 2 or 3, even when user is deleted from your IdP")
     ap2.add_argument("--no-bauth", action="store_true", help="disable basic-authentication support; do not accept passwords from the 'Authenticate' header at all. NOTE: This breaks support for the android app")
     ap2.add_argument("--bauth-last", action="store_true", help="keeps basic-authentication enabled, but only as a last-resort; if a cookie is also provided then the cookie wins")
     ap2.add_argument("--ses-db", metavar="PATH", type=u, default=ses_db, help="where to store the sessions database (if you run multiple copyparty instances, make sure they use different DBs)")

copyparty/authsrv.py

@@ -21,6 +21,7 @@ from .util import (
     DEF_MTE,
     DEF_MTH,
     EXTS,
+    HAVE_SQLITE3,
     IMPLICATIONS,
     MIMES,
     SQLITE_VER,
@@ -32,6 +33,7 @@ from .util import (
     afsenc,
     get_df,
     humansize,
+    min_ex,
     odfusion,
     read_utf8,
     relchk,
@@ -44,6 +46,9 @@ from .util import (
     vsplit,
 )
 
+if HAVE_SQLITE3:
+    import sqlite3
+
 if True:  # pylint: disable=using-constant-test
     from collections.abc import Iterable
 
@@ -935,6 +940,10 @@ class AuthSrv(object):
                 return False
 
             self.idp_accs[uname] = gnames
+            try:
+                self._update_idp_db(uname, gname)
+            except:
+                self.log("failed to update the --idp-db:\n%s" % (min_ex(),), 3)
 
             t = "reinitializing due to new user from IdP: [%r:%r]"
             self.log(t % (uname, gnames), 3)
@@ -947,6 +956,22 @@ class AuthSrv(object):
         broker.ask("reload", False, True).get()
         return True
 
+    def _update_idp_db(self, uname, gname):
+        if not self.args.idp_store:
+            return
+
+        assert sqlite3  # type: ignore  # !rm
+
+        db = sqlite3.connect(self.args.idp_db)
+        cur = db.cursor()
+
+        cur.execute("delete from us where un = ?", (uname,))
+        cur.execute("insert into us values (?,?)", (uname, gname))
+
+        db.commit()
+        cur.close()
+        db.close()
+
     def _map_volume_idp(
         self,
         src: str,
@@ -1095,6 +1120,7 @@ class AuthSrv(object):
          * any non-zero value from IdP group header
          * otherwise take --grps / [groups]
         """
+        self.load_idp_db(bool(self.idp_accs))
         ret = {un: gns[:] for un, gns in self.idp_accs.items()}
         ret.update({zs: [""] for zs in acct if zs not in ret})
         for gn, uns in grps.items():
@@ -1655,7 +1681,7 @@ class AuthSrv(object):
         shr = enshare[1:-1]
         shrs = enshare[1:]
         if enshare:
-            import sqlite3
+            assert sqlite3  # type: ignore  # !rm
 
             zsd = {"d2d": True, "tcolor": self.args.tcolor}
             shv = VFS(self.log_func, "", shr, shr, AXS(), zsd)
@@ -2621,6 +2647,40 @@ class AuthSrv(object):
             zs = str(vol.flags.get("tcolor") or self.args.tcolor)
             vol.flags["tcolor"] = zs.lstrip("#")
 
+    def load_idp_db(self, quiet=False) -> None:
+        # mutex me
+        level = self.args.idp_store
+        if level < 2 or not self.args.idp_h_usr:
+            return
+
+        assert sqlite3  # type: ignore  # !rm
+
+        db = sqlite3.connect(self.args.idp_db)
+        cur = db.cursor()
+
+        gsep = self.args.idp_gsep
+        n = []
+        for uname, gname in cur.execute("select un, gs from us"):
+            if level < 3:
+                if uname in self.idp_accs:
+                    continue
+                gname = ""
+            gnames = [x.strip() for x in gsep.split(gname)]
+            gnames.sort()
+
+            # self.idp_usr_gh[uname] = gname
+            self.idp_accs[uname] = gnames
+            n.append(uname)
+
+        cur.close()
+        db.close()
+
+        if n and not quiet:
+            t = ", ".join(n[:9])
+            if len(n) > 9:
+                t += "..."
+            self.log("found %d IdP users in db (%s)" % (len(n), t))
+
     def load_sessions(self, quiet=False) -> None:
         # mutex me
         if self.args.no_ses:
@@ -2628,7 +2688,7 @@ class AuthSrv(object):
             self.sesa = {}
             return
 
-        import sqlite3
+        assert sqlite3  # type: ignore  # !rm
 
         ases = {}
         blen = (self.args.ses_len // 4) * 4  # 3 bytes in 4 chars
@@ -2675,7 +2735,7 @@ class AuthSrv(object):
         if self.args.no_ses:
             return
 
-        import sqlite3
+        assert sqlite3  # type: ignore  # !rm
 
         db = sqlite3.connect(self.args.ses_db)
         cur = db.cursor()

copyparty/svchub.py

@@ -88,6 +88,7 @@ if PY2:
     range = xrange  # type: ignore
 
 
+VER_IDP_DB = 1
 VER_SESSION_DB = 1
 VER_SHARES_DB = 2
 
@@ -258,11 +259,15 @@ class SvcHub(object):
                 self.log("root", "effective %s is %s" % (zs, getattr(args, zs)))
 
         if args.ah_cli or args.ah_gen:
+            args.idp_store = 0
             args.no_ses = True
             args.shr = ""
 
+        if args.idp_store and args.idp_h_usr:
+            self.setup_db("idp")
+
         if not self.args.no_ses:
-            self.setup_session_db()
+            self.setup_db("ses")
 
         args.shr1 = ""
         if args.shr:
@@ -421,26 +426,58 @@ class SvcHub(object):
             except:
                 pass
 
-    def setup_session_db(self) -> None:
+    def _db_onfail_ses(self) -> None:
+        self.args.no_ses = True
+
+    def _db_onfail_idp(self) -> None:
+        self.args.idp_store = 0
+
+    def setup_db(self, which: str) -> None:
+        """
+        the "non-mission-critical" databases; if something looks broken then just nuke it
+        """
+        if which == "ses":
+            native_ver = VER_SESSION_DB
+            db_path = self.args.ses_db
+            desc = "sessions-db"
+            pathopt = "ses-db"
+            sanchk_q = "select count(*) from us"
+            createfun = self._create_session_db
+            failfun = self._db_onfail_ses
+        elif which == "idp":
+            native_ver = VER_IDP_DB
+            db_path = self.args.idp_db
+            desc = "idp-db"
+            pathopt = "idp-db"
+            sanchk_q = "select count(*) from us"
+            createfun = self._create_idp_db
+            failfun = self._db_onfail_idp
+        else:
+            raise Exception("unknown cachetype")
+
+        if not db_path.endswith(".db"):
+            zs = "config option --%s (the %s) was configured to [%s] which is invalid; must be a filepath ending with .db"
+            self.log("root", zs % (pathopt, desc, db_path), 1)
+            raise Exception(BAD_CFG)
+
         if not HAVE_SQLITE3:
-            self.args.no_ses = True
+            failfun()
-            t = "WARNING: sqlite3 not available; disabling sessions, will use plaintext passwords in cookies"
+            if which == "ses":
-            self.log("root", t, 3)
+                zs = "disabling sessions, will use plaintext passwords in cookies"
+            elif which == "idp":
+                zs = "disabling idp-db, will be unable to remember IdP-volumes after a restart"
+            self.log("root", "WARNING: sqlite3 not available; %s" % (zs,), 3)
             return
 
         assert sqlite3  # type: ignore  # !rm
 
-        # policy:
-        # the sessions-db is whatever, if something looks broken then just nuke it
-
-        db_path = self.args.ses_db
         db_lock = db_path + ".lock"
         try:
             create = not os.path.getsize(db_path)
         except:
             create = True
         zs = "creating new" if create else "opening"
-        self.log("root", "%s sessions-db %s" % (zs, db_path))
+        self.log("root", "%s %s %s" % (zs, desc, db_path))
 
         for tries in range(2):
             sver = 0
@@ -450,17 +487,19 @@ class SvcHub(object):
                 try:
                     zs = "select v from kv where k='sver'"
                     sver = cur.execute(zs).fetchall()[0][0]
-                    if sver > VER_SESSION_DB:
+                    if sver > native_ver:
-                        zs = "this version of copyparty only understands session-db v%d and older; the db is v%d"
+                        zs = "this version of copyparty only understands %s v%d and older; the db is v%d"
-                        raise Exception(zs % (VER_SESSION_DB, sver))
+                        raise Exception(zs % (desc, native_ver, sver))
 
-                    cur.execute("select count(*) from us").fetchone()
+                    cur.execute(sanchk_q).fetchone()
                 except:
                     if sver:
                         raise
-                    sver = 1
+                    sver = createfun(cur)
-                    self._create_session_db(cur)
+
-                err = self._verify_session_db(cur, sver, db_path)
+                err = self._verify_db(
+                    cur, which, pathopt, db_path, desc, sver, native_ver
+                )
                 if err:
                     tries = 99
                     self.args.no_ses = True
@@ -468,10 +507,10 @@ class SvcHub(object):
                 break
 
             except Exception as ex:
-                if tries or sver > VER_SESSION_DB:
+                if tries or sver > native_ver:
                     raise
-                t = "sessions-db is unusable; deleting and recreating: %r"
+                t = "%s is unusable; deleting and recreating: %r"
-                self.log("root", t % (ex,), 3)
+                self.log("root", t % (desc, ex), 3)
                 try:
                     cur.close()  # type: ignore
                 except:
@@ -486,7 +525,7 @@ class SvcHub(object):
                     pass
                 os.unlink(db_path)
 
-    def _create_session_db(self, cur: "sqlite3.Cursor") -> None:
+    def _create_session_db(self, cur: "sqlite3.Cursor") -> int:
         sch = [
             r"create table kv (k text, v int)",
             r"create table us (un text, si text, t0 int)",
@@ -499,8 +538,31 @@ class SvcHub(object):
         for cmd in sch:
             cur.execute(cmd)
         self.log("root", "created new sessions-db")
+        return 1
 
-    def _verify_session_db(self, cur: "sqlite3.Cursor", sver: int, db_path: str) -> str:
+    def _create_idp_db(self, cur: "sqlite3.Cursor") -> int:
+        sch = [
+            r"create table kv (k text, v int)",
+            r"create table us (un text, gs text)",
+            # username, groups
+            r"create index us_un on us(un)",
+            r"insert into kv values ('sver', 1)",
+        ]
+        for cmd in sch:
+            cur.execute(cmd)
+        self.log("root", "created new idp-db")
+        return 1
+
+    def _verify_db(
+        self,
+        cur: "sqlite3.Cursor",
+        which: str,
+        pathopt: str,
+        db_path: str,
+        desc: str,
+        sver: int,
+        native_ver: int,
+    ) -> str:
         # ensure writable (maybe owned by other user)
         db = cur.connection
 
@@ -512,9 +574,16 @@ class SvcHub(object):
         except:
             owner = 0
 
+        if which == "ses":
+            cons = "Will now disable sessions and instead use plaintext passwords in cookies."
+        elif which == "idp":
+            cons = "Each IdP-volume will not become available until its associated user sends their first request."
+        else:
+            raise Exception()
+
         if not lock_file(db_path + ".lock"):
-            t = "the sessions-db [%s] is already in use by another copyparty instance (pid:%d). This is not supported; please provide another database with --ses-db or give this copyparty-instance its entirely separate config-folder by setting another path in the XDG_CONFIG_HOME env-var. You can also disable this safeguard by setting env-var PRTY_NO_DB_LOCK=1. Will now disable sessions and instead use plaintext passwords in cookies."
+            t = "the %s [%s] is already in use by another copyparty instance (pid:%d). This is not supported; please provide another database with --%s or give this copyparty-instance its entirely separate config-folder by setting another path in the XDG_CONFIG_HOME env-var. You can also disable this safeguard by setting env-var PRTY_NO_DB_LOCK=1. %s"
-            return t % (db_path, owner)
+            return t % (desc, db_path, owner, pathopt, cons)
 
         vars = (("pid", os.getpid()), ("ts", int(time.time() * 1000)))
         if owner:
@@ -526,9 +595,9 @@ class SvcHub(object):
             for k, v in vars:
                 cur.execute("insert into kv values(?, ?)", (k, v))
 
-        if sver < VER_SESSION_DB:
+        if sver < native_ver:
             cur.execute("delete from kv where k='sver'")
-            cur.execute("insert into kv values('sver',?)", (VER_SESSION_DB,))
+            cur.execute("insert into kv values('sver',?)", (native_ver,))
 
         db.commit()
         cur.close()

tests/util.py

@@ -158,7 +158,7 @@ class Cfg(Namespace):
         ex = "au_vol dl_list mtab_age reg_cap s_thead s_tbody tail_tmax tail_who th_convt ups_who zip_who"
         ka.update(**{k: 9 for k in ex.split()})
 
-        ex = "db_act forget_ip k304 loris no304 nosubtle re_maxage rproxy rsp_jtr rsp_slp s_wr_slp snap_wri theme themes turbo u2ow zipmaxn zipmaxs"
+        ex = "db_act forget_ip idp_store k304 loris no304 nosubtle re_maxage rproxy rsp_jtr rsp_slp s_wr_slp snap_wri theme themes turbo u2ow zipmaxn zipmaxs"
         ka.update(**{k: 0 for k in ex.split()})
 
         ex = "ah_alg bname chpw_db doctitle df exit favico idp_h_usr ipa html_head lg_sba lg_sbf log_fk md_sba md_sbf name og_desc og_site og_th og_title og_title_a og_title_v og_title_i shr tcolor textfiles unlist vname xff_src zipmaxt R RS SR"