diff --git a/README.md b/README.md index 6efe89ae..3cbf5289 100644 --- a/README.md +++ b/README.md @@ -789,6 +789,8 @@ other notes, * files named `README.md` / `readme.md` will be rendered after directory listings unless `--no-readme` (but `.epilogue.html` takes precedence) +* `README.md` and `*logue.html` can contain placeholder values which are replaced server-side before embedding into directory listings; see `--help-exp` + ## searching diff --git a/copyparty/__main__.py b/copyparty/__main__.py index 1e259432..3f2ef2b0 100755 --- a/copyparty/__main__.py +++ b/copyparty/__main__.py @@ -27,6 +27,7 @@ from .authsrv import expand_config_file, re_vol, split_cfg_ln, upgrade_cfg_fmt from .cfg import flagcats, onedash from .svchub import SvcHub from .util import ( + DEF_EXP, DEF_MTE, DEF_MTH, IMPLICATIONS, @@ -646,6 +647,47 @@ def get_sects(): """ ), ], + [ + "exp", + "text expansion", + dedent( + """ + specify --exp or the "exp" volflag to enable placeholder expansions + in README.md / .prologue.html / .epilogue.html + + --exp-md (volflag exp_md) holds the list of placeholders which can be + expanded in READMEs, and --exp-lg (volflag exp_lg) likewise for logues; + any placeholder not given in those lists will be ignored and shown as-is + + the default list will expand the following placeholders: + \033[36m{{self.ip}} \033[35mclient ip + \033[36m{{self.ua}} \033[35mclient user-agent + \033[36m{{self.uname}} \033[35mclient username + \033[36m{{self.host}} \033[35mthe "Host" header, or the server's external IP otherwise + \033[36m{{cfg.name}} \033[35mthe --name global-config + \033[36m{{cfg.logout}} \033[35mthe --logout global-config + \033[36m{{vf.scan}} \033[35mthe "scan" volflag + \033[36m{{vf.thsize}} \033[35mthumbnail size + \033[36m{{srv.itime}} \033[35mserver time in seconds + \033[36m{{srv.htime}} \033[35mserver time as YY-mm-dd, HH:MM:SS (UTC) + \033[36m{{hdr.cf_ipcountry}} \033[35mthe "CF-IPCountry" client header (probably blank) + \033[0m + so the following types of placeholders can be added to the lists: + * any client header can be accessed through {{hdr.*}} + * any variable in httpcli.py can be accessed through {{self.*}} + * any global server setting can be accessed through {{cfg.*}} + * any volflag can be accessed through {{vf.*}} + + remove vf.scan from default list using --exp-md /vf.scan + add "accept" header to def. list using --exp-md +hdr.accept + + for performance reasons, expansion only happens while embedding + documents into directory listings, and when accessing a ?doc=... + link, but never otherwise, so if you click a -txt- link you'll + have to refresh the page to apply expansion + """ + ), + ], [ "ls", "volume inspection", @@ -776,8 +818,6 @@ def add_general(ap, nc, srvname): ap2.add_argument("-a", metavar="ACCT", type=u, action="append", help="add account, \033[33mUSER\033[0m:\033[33mPASS\033[0m; example [\033[32med:wark\033[0m]") ap2.add_argument("-v", metavar="VOL", type=u, action="append", help="add volume, \033[33mSRC\033[0m:\033[33mDST\033[0m:\033[33mFLAG\033[0m; examples [\033[32m.::r\033[0m], [\033[32m/mnt/nas/music:/music:r:aed\033[0m]") ap2.add_argument("-ed", action="store_true", help="enable the ?dots url parameter / client option which allows clients to see dotfiles / hidden files") - ap2.add_argument("-emp", action="store_true", help="enable markdown plugins -- neat but dangerous, big XSS risk") - ap2.add_argument("-mcr", metavar="SEC", type=int, default=60, help="md-editor mod-chk rate") ap2.add_argument("--urlform", metavar="MODE", type=u, default="print,get", help="how to handle url-form POSTs; see --help-urlform") ap2.add_argument("--wintitle", metavar="TXT", type=u, default="cpp @ $pub", help="window title, for example [\033[32m$ip-10.1.2.\033[0m] or [\033[32m$ip-]") ap2.add_argument("--name", metavar="TXT", type=u, default=srvname, help="server name (displayed topleft in browser and in mDNS)") @@ -1144,6 +1184,15 @@ def add_db_metadata(ap): ap2.add_argument("-mtp", metavar="M=[f,]BIN", type=u, action="append", help="read tag M using program BIN to parse the file") +def add_txt(ap): + ap2 = ap.add_argument_group('textfile options') + ap2.add_argument("-mcr", metavar="SEC", type=int, default=60, help="textfile editor checks for serverside changes every SEC seconds") + ap2.add_argument("-emp", action="store_true", help="enable markdown plugins -- neat but dangerous, big XSS risk") + ap2.add_argument("--exp", action="store_true", help="enable textfile expansion -- replace {{self.ip}} and such; see --help-exp (volflag=exp)") + ap2.add_argument("--exp-md", metavar="V,V,V", type=u, default=DEF_EXP, help="comma/space-separated list of placeholders to expand in markdown files; add/remove stuff on the default list with +hdr_foo or /vf.scan (volflag=exp_md)") + ap2.add_argument("--exp-lg", metavar="V,V,V", type=u, default=DEF_EXP, help="comma/space-separated list of placeholders to expand in prologue/epilogue files (volflag=exp_lg)") + + def add_ui(ap, retry): ap2 = ap.add_argument_group('ui options') ap2.add_argument("--grid", action="store_true", help="show grid/thumbnails by default (volflag=grid)") @@ -1237,6 +1286,7 @@ def run_argparse( add_handlers(ap) add_hooks(ap) add_stats(ap) + add_txt(ap) add_ui(ap, retry) add_admin(ap) add_logging(ap) diff --git a/copyparty/authsrv.py b/copyparty/authsrv.py index 40d89123..ffe2a7a3 100644 --- a/copyparty/authsrv.py +++ b/copyparty/authsrv.py @@ -1479,15 +1479,11 @@ class AuthSrv(object): raise Exception(t.format(dbd, dbds)) # default tag cfgs if unset - if "mte" not in vol.flags: - vol.flags["mte"] = self.args.mte.copy() - else: - vol.flags["mte"] = odfusion(self.args.mte, vol.flags["mte"]) - - if "mth" not in vol.flags: - vol.flags["mth"] = self.args.mth.copy() - else: - vol.flags["mth"] = odfusion(self.args.mth, vol.flags["mth"]) + for k in ("mte", "mth", "exp_md", "exp_lg"): + if k not in vol.flags: + vol.flags[k] = getattr(self.args, k).copy() + else: + vol.flags[k] = odfusion(getattr(self.args, k), vol.flags[k]) # append additive args from argv to volflags hooks = "xbu xau xiu xbr xar xbd xad xm xban".split() diff --git a/copyparty/cfg.py b/copyparty/cfg.py index 81dd6a2c..53865236 100644 --- a/copyparty/cfg.py +++ b/copyparty/cfg.py @@ -17,7 +17,6 @@ def vf_bmap() -> dict[str, str]: "no_thumb": "dthumb", "no_vthumb": "dvthumb", "no_athumb": "dathumb", - "re_maxage": "scan", "th_no_crop": "nocrop", "dav_auth": "davauth", "dav_rt": "davrt", @@ -33,6 +32,7 @@ def vf_bmap() -> dict[str, str]: "e2v", "e2vu", "e2vp", + "exp", "grid", "hardlink", "magic", @@ -52,10 +52,19 @@ def vf_vmap() -> dict[str, str]: ret = { "no_hash": "nohash", "no_idx": "noidx", + "re_maxage": "scan", "th_convt": "convt", "th_size": "thsize", } - for k in ("dbd", "lg_sbf", "md_sbf", "nrand", "sort", "unlist", "u2ts"): + for k in ( + "dbd", + "lg_sbf", + "md_sbf", + "nrand", + "sort", + "unlist", + "u2ts", + ): ret[k] = k return ret @@ -64,6 +73,8 @@ def vf_cmap() -> dict[str, str]: """argv-to-volflag: complex/lists""" ret = {} for k in ( + "exp_lg", + "exp_md", "html_head", "mte", "mth", diff --git a/copyparty/httpcli.py b/copyparty/httpcli.py index 26652f1a..8879826d 100644 --- a/copyparty/httpcli.py +++ b/copyparty/httpcli.py @@ -2726,6 +2726,29 @@ class HttpCli(object): return file_lastmod, True + def _expand(self, txt: str, phs: list[str]) -> str: + for ph in phs: + if ph.startswith("hdr."): + sv = str(self.headers.get(ph[4:], "")) + elif ph.startswith("self."): + sv = str(getattr(self, ph[5:], "")) + elif ph.startswith("cfg."): + sv = str(getattr(self.args, ph[4:], "")) + elif ph.startswith("vf."): + sv = str(self.vn.flags.get(ph[3:]) or "") + elif ph == "srv.itime": + sv = str(int(time.time())) + elif ph == "srv.htime": + sv = datetime.now(UTC).strftime("%Y-%m-%d, %H:%M:%S") + else: + self.log("unknown placeholder in server config: [%s]" % (ph), 3) + continue + + sv = self.conn.hsrv.ptn_hsafe.sub("_", sv) + txt = txt.replace("{{%s}}" % (ph,), sv) + + return txt + def tx_file(self, req_path: str) -> bool: status = 200 logmsg = "{:4} {} ".format("", self.req) @@ -3052,7 +3075,7 @@ class HttpCli(object): self.reply(ico, mime=mime, headers={"Last-Modified": lm}) return True - def tx_md(self, fs_path: str) -> bool: + def tx_md(self, vn: VFS, fs_path: str) -> bool: logmsg = " %s @%s " % (self.req, self.uname) if not self.can_write: @@ -3069,9 +3092,16 @@ class HttpCli(object): st = bos.stat(html_path) ts_html = st.st_mtime + max_sz = 1024 * self.args.txt_max sz_md = 0 lead = b"" + fullfile = b"" for buf in yieldfile(fs_path): + if sz_md < max_sz: + fullfile += buf + else: + fullfile = b"" + if not sz_md and b"\n" in buf[:2]: lead = buf[: buf.find(b"\n") + 1] sz_md += len(lead) @@ -3080,6 +3110,21 @@ class HttpCli(object): for c, v in [(b"&", 4), (b"<", 3), (b">", 3)]: sz_md += (len(buf) - len(buf.replace(c, b""))) * v + if ( + fullfile + and "exp" in vn.flags + and "edit" not in self.uparam + and "edit2" not in self.uparam + and vn.flags.get("exp_md") + ): + fulltxt = fullfile.decode("utf-8", "replace") + fulltxt = self._expand(fulltxt, vn.flags.get("exp_md") or []) + fullfile = fulltxt.encode("utf-8", "replace") + + if fullfile: + fullfile = html_bescape(fullfile) + sz_md = len(lead) + len(fullfile) + file_ts = int(max(ts_md, ts_html, self.E.t0)) file_lastmod, do_send = self._chk_lastmod(file_ts) self.out_headers["Last-Modified"] = file_lastmod @@ -3121,8 +3166,11 @@ class HttpCli(object): try: self.s.sendall(html[0] + lead) - for buf in yieldfile(fs_path): - self.s.sendall(html_bescape(buf)) + if fullfile: + self.s.sendall(fullfile) + else: + for buf in yieldfile(fs_path): + self.s.sendall(html_bescape(buf)) self.s.sendall(html[1]) @@ -3753,7 +3801,7 @@ class HttpCli(object): or "edit2" in self.uparam ) ): - return self.tx_md(abspath) + return self.tx_md(vn, abspath) return self.tx_file(abspath) @@ -3815,6 +3863,10 @@ class HttpCli(object): if bos.path.exists(fn): with open(fsenc(fn), "rb") as f: logues[n] = f.read().decode("utf-8") + if "exp" in vn.flags: + logues[n] = self._expand( + logues[n], vn.flags.get("exp_lg") or [] + ) readme = "" if not self.args.no_readme and not logues[1]: @@ -3824,6 +3876,8 @@ class HttpCli(object): with open(fsenc(fn), "rb") as f: readme = f.read().decode("utf-8") break + if readme and "exp" in vn.flags: + readme = self._expand(readme, vn.flags.get("exp_md") or []) vf = vn.flags unlist = vf.get("unlist", "") @@ -4134,6 +4188,12 @@ class HttpCli(object): if sz < 1024 * self.args.txt_max: with open(fsenc(docpath), "rb") as f: doctxt = f.read().decode("utf-8", "replace") + + if doc.lower().endswith(".md") and "exp" in vn.flags: + doctxt = self._expand(doctxt, vn.flags.get("exp_md") or []) + else: + self.log("doc 2big: [{}]".format(doc), c=6) + doctxt = "( size of textfile exceeds serverside limit )" else: self.log("doc 404: [{}]".format(doc), c=6) doctxt = "( textfile not found )" diff --git a/copyparty/httpsrv.py b/copyparty/httpsrv.py index 286e9461..2f253dad 100644 --- a/copyparty/httpsrv.py +++ b/copyparty/httpsrv.py @@ -149,6 +149,7 @@ class HttpSrv(object): self._build_statics() self.ptn_cc = re.compile(r"[\x00-\x1f]") + self.ptn_hsafe = re.compile(r"[\x00-\x1f<>\"'&]") self.mallow = "GET HEAD POST PUT DELETE OPTIONS".split() if not self.args.no_dav: diff --git a/copyparty/svchub.py b/copyparty/svchub.py index 6ef6d37e..2fce9c67 100644 --- a/copyparty/svchub.py +++ b/copyparty/svchub.py @@ -39,6 +39,7 @@ from .util import ( FFMPEG_URL, VERSIONS, Daemon, + DEF_EXP, DEF_MTE, DEF_MTH, Garda, @@ -442,6 +443,10 @@ class SvcHub(object): mth = ODict.fromkeys(DEF_MTH.split(","), True) al.mth = odfusion(mth, al.mth) + exp = ODict.fromkeys(DEF_EXP.split(" "), True) + al.exp_md = odfusion(exp, al.exp_md.replace(" ", ",")) + al.exp_lg = odfusion(exp, al.exp_lg.replace(" ", ",")) + for k in ["no_hash", "no_idx"]: ptn = getattr(self.args, k) if ptn: diff --git a/copyparty/up2k.py b/copyparty/up2k.py index 23206049..d873417d 100644 --- a/copyparty/up2k.py +++ b/copyparty/up2k.py @@ -814,7 +814,7 @@ class Up2k(object): if str(fl[k1]) == str(getattr(self.args, k2)): del fl[k1] else: - fl[k1] = ",".join(x for x in fl) + fl[k1] = ",".join(x for x in fl[k1]) a = [ (ft if v is True else ff if v is False else fv).format(k, str(v)) for k, v in fl.items() diff --git a/copyparty/util.py b/copyparty/util.py index 6f4ad76d..26133374 100644 --- a/copyparty/util.py +++ b/copyparty/util.py @@ -289,6 +289,8 @@ EXTS["vnd.mozilla.apng"] = "png" MAGIC_MAP = {"jpeg": "jpg"} +DEF_EXP = "self.ip self.ua self.uname self.host cfg.name cfg.logout vf.scan vf.thsize hdr.cf_ipcountry srv.itime srv.htime" + DEF_MTE = "circle,album,.tn,artist,title,.bpm,key,.dur,.q,.vq,.aq,vc,ac,fmt,res,.fps,ahash,vhash" DEF_MTH = ".vq,.aq,vc,ac,fmt,res,.fps" @@ -1809,15 +1811,18 @@ def exclude_dotfiles(filepaths: list[str]) -> list[str]: def odfusion(base: ODict[str, bool], oth: str) -> ODict[str, bool]: # merge an "ordered set" (just a dict really) with another list of keys + words0 = [x for x in oth.split(",") if x] + words1 = [x for x in oth[1:].split(",") if x] + ret = base.copy() if oth.startswith("+"): - for k in oth[1:].split(","): + for k in words1: ret[k] = True elif oth[:1] in ("-", "/"): - for k in oth[1:].split(","): + for k in words1: ret.pop(k, None) else: - ret = ODict.fromkeys(oth.split(","), True) + ret = ODict.fromkeys(words0, True) return ret diff --git a/srv/expand/README.md b/srv/expand/README.md new file mode 100644 index 00000000..423c0737 --- /dev/null +++ b/srv/expand/README.md @@ -0,0 +1,26 @@ +## text expansion + +enable expansion of placeholder variables in `README.md` and prologue/epilogue files with `--exp` and customize the list of allowed placeholders to expand using `--exp-md` and `--exp-lg` + +| explanation | placeholder | +| -------------------- | -------------------- | +| your ip address | {{self.ip}} | +| your user-agent | {{self.ua}} | +| your username | {{self.uname}} | +| the `Host` you see | {{self.host}} | +| server unix time | {{srv.itime}} | +| server datetime | {{srv.htime}} | +| server name | {{cfg.name}} | +| logout after | {{cfg.logout}} hours | +| vol reindex interval | {{vf.scan}} | +| thumbnail size | {{vf.thsize}} | +| your country | {{hdr.cf_ipcountry}} | + +placeholders starting with... +* `self.` are grabbed from copyparty's internal state; anything in `httpcli.py` is fair game +* `cfg.` are the global server settings +* `vf.` are the volflags of the current volume +* `hdr.` are grabbed from the client headers; any header is supported, just add it (in lowercase) to the allowlist +* `srv.` are processed inside the `_expand` function in httpcli + +for example (bad example), `hdr_cf_ipcountry` maps to the header `CF-IPCountry` (which is generated by cloudflare before the request is passed on to your server / copyparty) diff --git a/tests/util.py b/tests/util.py index 82f7d26b..3d3683db 100644 --- a/tests/util.py +++ b/tests/util.py @@ -109,7 +109,7 @@ class Cfg(Namespace): def __init__(self, a=None, v=None, c=None): ka = {} - ex = "daw dav_auth dav_inf dav_mac dav_rt dotsrch e2d e2ds e2dsa e2t e2ts e2tsr e2v e2vu e2vp ed emp force_js getmod grid hardlink ih ihead magic never_symlink nid nih no_acode no_athumb no_dav no_dedup no_del no_dupe no_logues no_mv no_readme no_robots no_sb_md no_sb_lg no_scandir no_tarcmp no_thumb no_vthumb no_zip nrand nw rand smb th_no_crop vague_403 vc ver xdev xlink xvol" + ex = "daw dav_auth dav_inf dav_mac dav_rt dotsrch e2d e2ds e2dsa e2t e2ts e2tsr e2v e2vu e2vp ed emp exp force_js getmod grid hardlink ih ihead magic never_symlink nid nih no_acode no_athumb no_dav no_dedup no_del no_dupe no_logues no_mv no_readme no_robots no_sb_md no_sb_lg no_scandir no_tarcmp no_thumb no_vthumb no_zip nrand nw rand smb th_no_crop vague_403 vc ver xdev xlink xvol" ka.update(**{k: False for k in ex.split()}) ex = "dotpart no_rescan no_sendfile no_voldump plain_ip" @@ -130,6 +130,9 @@ class Cfg(Namespace): ex = "on403 on404 xad xar xau xban xbd xbr xbu xiu xm" ka.update(**{k: [] for k in ex.split()}) + ex = "exp_lg exp_md" + ka.update(**{k: {} for k in ex.split()}) + super(Cfg, self).__init__( a=a or [], v=v or [],