add config-file preprocessor (%include)

README.md

@@ -667,7 +667,7 @@ for the above example to work, add the commandline argument `-e2ts` to also scan
 # server config
 
 using arguments or config files, or a mix of both:
-* config files (`-c some.conf`) can set additional commandline arguments; see [./docs/example.conf](docs/example.conf)
+* config files (`-c some.conf`) can set additional commandline arguments; see [./docs/example.conf](docs/example.conf) and [./docs/example2.conf](docs/example2.conf)
 * `kill -s USR1` (same as `systemctl reload copyparty`) to reload accounts and volumes from config files without restarting
   * or click the `[reload cfg]` button in the control-panel when logged in as admin 
 

copyparty/__main__.py

@@ -22,7 +22,7 @@ from textwrap import dedent
 
 from .__init__ import ANYWIN, CORES, PY2, VT100, WINDOWS, E, EnvParams, unicode
 from .__version__ import CODENAME, S_BUILD_DT, S_VERSION
-from .authsrv import re_vol
+from .authsrv import re_vol, expand_config_file
 from .svchub import SvcHub
 from .util import (
     IMPLICATIONS,
@@ -317,27 +317,29 @@ def configure_ssl_ciphers(al: argparse.Namespace) -> None:
 
 
 def args_from_cfg(cfg_path: str) -> list[str]:
+    lines: list[str] = []
+    expand_config_file(lines, cfg_path, "")
+
     ret: list[str] = []
     skip = False
-    with open(cfg_path, "rb") as f:
+    for ln in lines:
-        for ln in [x.decode("utf-8").strip() for x in f]:
+        if not ln:
-            if not ln:
+            skip = False
-                skip = False
+            continue
-                continue
 
-            if ln.startswith("#"):
+        if ln.startswith("#"):
-                continue
+            continue
 
-            if not ln.startswith("-"):
+        if not ln.startswith("-"):
-                continue
+            continue
 
-            if skip:
+        if skip:
-                continue
+            continue
 
-            try:
+        try:
-                ret.extend(ln.split(" ", 1))
+            ret.extend(ln.split(" ", 1))
-            except:
+        except:
-                ret.append(ln)
+            ret.append(ln)
 
     return ret
 
@@ -837,7 +839,13 @@ def main(argv: Optional[list[str]] = None) -> None:
         ensure_cert()
 
     for k, v in zip(argv[1:], argv[2:]):
-        if k == "-c":
+        if k == "-c" and os.path.isfile(v):
+            supp = args_from_cfg(v)
+            argv.extend(supp)
+
+    for k in argv[1:]:
+        v = k[2:]
+        if k.startswith("-c") and v and os.path.isfile(v):
             supp = args_from_cfg(v)
             argv.extend(supp)
 

copyparty/authsrv.py

@@ -705,7 +705,8 @@ class AuthSrv(object):
 
     def _parse_config_file(
         self,
-        fd: typing.BinaryIO,
+        fp: str,
+        cfg_lines: list[str],
         acct: dict[str, str],
         daxs: dict[str, AXS],
         mflags: dict[str, dict[str, Any]],
@@ -715,7 +716,8 @@ class AuthSrv(object):
         vol_src = None
         vol_dst = None
         self.line_ctr = 0
-        for ln in [x.decode("utf-8").strip() for x in fd]:
+        expand_config_file(cfg_lines, fp, "")
+        for ln in cfg_lines:
             self.line_ctr += 1
             if not ln and vol_src is not None:
                 vol_src = None
@@ -744,6 +746,9 @@ class AuthSrv(object):
                 if not vol_dst.startswith("/"):
                     raise Exception('invalid mountpoint "{}"'.format(vol_dst))
 
+                if vol_src.startswith("~"):
+                    vol_src = os.path.expanduser(vol_src)
+
                 # cfg files override arguments and previous files
                 vol_src = absreal(vol_src)
                 vol_dst = vol_dst.strip("/")
@@ -760,7 +765,7 @@ class AuthSrv(object):
                 t = "WARNING (config-file): permission flag 'a' is deprecated; please use 'rw' instead"
                 self.log(t, 1)
 
-            assert vol_dst
+            assert vol_dst is not None
             self._read_vol_str(lvl, uname, daxs[vol_dst], mflags[vol_dst])
 
     def _read_vol_str(
@@ -869,13 +874,16 @@ class AuthSrv(object):
 
         if self.args.c:
             for cfg_fn in self.args.c:
-                with open(cfg_fn, "rb") as f:
+                lns: list[str] = []
-                    try:
+                try:
-                        self._parse_config_file(f, acct, daxs, mflags, mount)
+                    self._parse_config_file(cfg_fn, lns, acct, daxs, mflags, mount)
-                    except:
+                except:
-                        t = "\n\033[1;31m\nerror in config file {} on line {}:\n\033[0m"
+                    lns = lns[: self.line_ctr]
-                        self.log(t.format(cfg_fn, self.line_ctr), 1)
+                    slns = ["{:4}: {}".format(n, s) for n, s in enumerate(lns, 1)]
-                        raise
+                    t = "\033[1;31m\nerror @ line {}, included from {}\033[0m"
+                    t = t.format(self.line_ctr, cfg_fn)
+                    self.log("\n{0}\n{1}{0}".format(t, "\n".join(slns)))
+                    raise
 
         # case-insensitive; normalize
         if WINDOWS:
@@ -1419,3 +1427,33 @@ class AuthSrv(object):
 
         if not flag_r:
             sys.exit(0)
+
+
+def expand_config_file(ret: list[str], fp: str, ipath: str) -> None:
+    """expand all % file includes"""
+    fp = absreal(fp)
+    ipath += " -> " + fp
+    ret.append("#\033[36m opening cfg file{}\033[0m".format(ipath))
+    if len(ipath.split(" -> ")) > 64:
+        raise Exception("hit max depth of 64 includes")
+
+    if os.path.isdir(fp):
+        for fn in sorted(os.listdir(fp)):
+            fp2 = os.path.join(fp, fn)
+            if not os.path.isfile(fp2):
+                continue  # dont recurse
+
+            expand_config_file(ret, fp2, ipath)
+        return
+
+    with open(fp, "rb") as f:
+        for ln in [x.decode("utf-8").strip() for x in f]:
+            if ln.startswith("% "):
+                fp2 = ln[1:].strip()
+                fp2 = os.path.join(os.path.dirname(fp), fp2)
+                expand_config_file(ret, fp2, ipath)
+                continue
+
+            ret.append(ln)
+
+    ret.append("#\033[36m closed{}\033[0m".format(ipath))

docs/copyparty.d/foo/another.conf

@@ -0,0 +1,5 @@
+# this file gets included twice from ../some.conf,
+# setting user permissions for a volume
+rw usr1
+r usr2
+% sibling.conf

docs/copyparty.d/foo/sibling.conf

@@ -0,0 +1,3 @@
+# and this config file gets included from ./another.conf,
+# adding a final permission for each of the two volumes in ../some.conf
+m usr1 usr2

docs/copyparty.d/some.conf

@@ -0,0 +1,26 @@
+# lets make two volumes with the same accounts/permissions for both;
+# first declare the accounts just once:
+u usr1:passw0rd
+u usr2:letmein
+
+# and listen on 127.0.0.1 only, port 2434
+-i 127.0.0.1
+-p 2434
+
+# share /usr/share/games from the server filesystem
+/usr/share/games
+/vidya
+# include config file with volume permissions
+% foo/another.conf
+
+# and share your ~/Music folder too
+~/Music
+/bangers
+% foo/another.conf
+
+# which should result in each of the volumes getting the following permissions:
+# usr1 read/write/move
+# usr2 read/move
+#
+# because another.conf sets the read/write permissions before it
+# includes sibling.conf which adds the move permission

docs/example2.conf

@@ -0,0 +1,13 @@
+# you can include additional config like this
+# (the space after the % is important)
+#
+# since copyparty.d is a folder, it'll include all *.conf
+# files inside (not recursively) in alphabetical order
+# (not necessarily same as numerical/natural order)
+#
+# paths are relative from the location of each included file
+# unless the path is absolute, for example % /etc/copyparty.d
+#
+# max include depth is 64
+
+% copyparty.d