mirrors/copyparty
1
0
Fork 0
mirror of https://github.com/9001/copyparty.git synced 2025-08-16 08:32:13 -06:00
Code Issues Actions Packages Projects Releases Wiki Activity

placeholder expansion in readme and logues; closes #56

Browse source 
also fixes the "scan" volflag which broke in v1.9.14
This commit is contained in:
ed 2023-10-24 16:37:32 +00:00
parent 063fa3efde
commit 1f75314463
11 changed files with 181 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
README.md
View file

@ -789,6 +789,8 @@ other notes,
* files named `README.md` / `readme.md` will be rendered after directory listings unless `--no-readme` (but `.epilogue.html` takes precedence)
* `README.md` and `*logue.html` can contain placeholder values which are replaced server-side before embedding into directory listings; see `--help-exp`
## searching

54
copyparty/__main__.py
View file

@ -27,6 +27,7 @@ from .authsrv import expand_config_file, re_vol, split_cfg_ln, upgrade_cfg_fmt
from .cfg import flagcats, onedash
from .svchub import SvcHub
from .util import (
DEF_EXP,
DEF_MTE,
DEF_MTH,
IMPLICATIONS,
@ -646,6 +647,47 @@ def get_sects():
"""
),
],
[
"exp",
"text expansion",
dedent(
"""
specify --exp or the "exp" volflag to enable placeholder expansions
in README.md / .prologue.html / .epilogue.html
--exp-md (volflag exp_md) holds the list of placeholders which can be
expanded in READMEs, and --exp-lg (volflag exp_lg) likewise for logues;
any placeholder not given in those lists will be ignored and shown as-is
the default list will expand the following placeholders:
\033[36m{{self.ip}} \033[35mclient ip
\033[36m{{self.ua}} \033[35mclient user-agent
\033[36m{{self.uname}} \033[35mclient username
\033[36m{{self.host}} \033[35mthe "Host" header, or the server's external IP otherwise
\033[36m{{cfg.name}} \033[35mthe --name global-config
\033[36m{{cfg.logout}} \033[35mthe --logout global-config
\033[36m{{vf.scan}} \033[35mthe "scan" volflag
\033[36m{{vf.thsize}} \033[35mthumbnail size
\033[36m{{srv.itime}} \033[35mserver time in seconds
\033[36m{{srv.htime}} \033[35mserver time as YY-mm-dd, HH:MM:SS (UTC)
\033[36m{{hdr.cf_ipcountry}} \033[35mthe "CF-IPCountry" client header (probably blank)
\033[0m
so the following types of placeholders can be added to the lists:
* any client header can be accessed through {{hdr.*}}
* any variable in httpcli.py can be accessed through {{self.*}}
* any global server setting can be accessed through {{cfg.*}}
* any volflag can be accessed through {{vf.*}}
remove vf.scan from default list using --exp-md /vf.scan
add "accept" header to def. list using --exp-md +hdr.accept
for performance reasons, expansion only happens while embedding
documents into directory listings, and when accessing a ?doc=...
link, but never otherwise, so if you click a -txt- link you'll
have to refresh the page to apply expansion
"""
),
],
[
"ls",
"volume inspection",
@ -776,8 +818,6 @@ def add_general(ap, nc, srvname):
ap2.add_argument("-a", metavar="ACCT", type=u, action="append", help="add account, \033[33mUSER\033[0m:\033[33mPASS\033[0m; example [\033[32med:wark\033[0m]")
ap2.add_argument("-v", metavar="VOL", type=u, action="append", help="add volume, \033[33mSRC\033[0m:\033[33mDST\033[0m:\033[33mFLAG\033[0m; examples [\033[32m.::r\033[0m], [\033[32m/mnt/nas/music:/music:r:aed\033[0m]")
ap2.add_argument("-ed", action="store_true", help="enable the ?dots url parameter / client option which allows clients to see dotfiles / hidden files")
ap2.add_argument("-emp", action="store_true", help="enable markdown plugins -- neat but dangerous, big XSS risk")
ap2.add_argument("-mcr", metavar="SEC", type=int, default=60, help="md-editor mod-chk rate")
ap2.add_argument("--urlform", metavar="MODE", type=u, default="print,get", help="how to handle url-form POSTs; see --help-urlform")
ap2.add_argument("--wintitle", metavar="TXT", type=u, default="cpp @ $pub", help="window title, for example [\033[32m$ip-10.1.2.\033[0m] or [\033[32m$ip-]")
ap2.add_argument("--name", metavar="TXT", type=u, default=srvname, help="server name (displayed topleft in browser and in mDNS)")
@ -1144,6 +1184,15 @@ def add_db_metadata(ap):
ap2.add_argument("-mtp", metavar="M=[f,]BIN", type=u, action="append", help="read tag M using program BIN to parse the file")
def add_txt(ap):
ap2 = ap.add_argument_group('textfile options')
ap2.add_argument("-mcr", metavar="SEC", type=int, default=60, help="textfile editor checks for serverside changes every SEC seconds")
ap2.add_argument("-emp", action="store_true", help="enable markdown plugins -- neat but dangerous, big XSS risk")
ap2.add_argument("--exp", action="store_true", help="enable textfile expansion -- replace {{self.ip}} and such; see --help-exp (volflag=exp)")
ap2.add_argument("--exp-md", metavar="V,V,V", type=u, default=DEF_EXP, help="comma/space-separated list of placeholders to expand in markdown files; add/remove stuff on the default list with +hdr_foo or /vf.scan (volflag=exp_md)")
ap2.add_argument("--exp-lg", metavar="V,V,V", type=u, default=DEF_EXP, help="comma/space-separated list of placeholders to expand in prologue/epilogue files (volflag=exp_lg)")
def add_ui(ap, retry):
ap2 = ap.add_argument_group('ui options')
ap2.add_argument("--grid", action="store_true", help="show grid/thumbnails by default (volflag=grid)")
@ -1237,6 +1286,7 @@ def run_argparse(
add_handlers(ap)
add_hooks(ap)
add_stats(ap)
add_txt(ap)
add_ui(ap, retry)
add_admin(ap)
add_logging(ap)

14
copyparty/authsrv.py
View file

@ -1479,15 +1479,11 @@ class AuthSrv(object):
raise Exception(t.format(dbd, dbds))
# default tag cfgs if unset
if "mte" not in vol.flags:
vol.flags["mte"] = self.args.mte.copy()
else:
vol.flags["mte"] = odfusion(self.args.mte, vol.flags["mte"])
if "mth" not in vol.flags:
vol.flags["mth"] = self.args.mth.copy()
else:
vol.flags["mth"] = odfusion(self.args.mth, vol.flags["mth"])
for k in ("mte", "mth", "exp_md", "exp_lg"):
if k not in vol.flags:
vol.flags[k] = getattr(self.args, k).copy()
else:
vol.flags[k] = odfusion(getattr(self.args, k), vol.flags[k])
# append additive args from argv to volflags
hooks = "xbu xau xiu xbr xar xbd xad xm xban".split()

15
copyparty/cfg.py
View file

@ -17,7 +17,6 @@ def vf_bmap() -> dict[str, str]:
"no_thumb": "dthumb",
"no_vthumb": "dvthumb",
"no_athumb": "dathumb",
"re_maxage": "scan",
"th_no_crop": "nocrop",
"dav_auth": "davauth",
"dav_rt": "davrt",
@ -33,6 +32,7 @@ def vf_bmap() -> dict[str, str]:
"e2v",
"e2vu",
"e2vp",
"exp",
"grid",
"hardlink",
"magic",
@ -52,10 +52,19 @@ def vf_vmap() -> dict[str, str]:
ret = {
"no_hash": "nohash",
"no_idx": "noidx",
"re_maxage": "scan",
"th_convt": "convt",
"th_size": "thsize",
}
for k in ("dbd", "lg_sbf", "md_sbf", "nrand", "sort", "unlist", "u2ts"):
for k in (
"dbd",
"lg_sbf",
"md_sbf",
"nrand",
"sort",
"unlist",
"u2ts",
):
ret[k] = k
return ret
@ -64,6 +73,8 @@ def vf_cmap() -> dict[str, str]:
"""argv-to-volflag: complex/lists"""
ret = {}
for k in (
"exp_lg",
"exp_md",
"html_head",
"mte",
"mth",

68
copyparty/httpcli.py
View file

@ -2726,6 +2726,29 @@ class HttpCli(object):
return file_lastmod, True
def _expand(self, txt: str, phs: list[str]) -> str:
for ph in phs:
if ph.startswith("hdr."):
sv = str(self.headers.get(ph[4:], ""))
elif ph.startswith("self."):
sv = str(getattr(self, ph[5:], ""))
elif ph.startswith("cfg."):
sv = str(getattr(self.args, ph[4:], ""))
elif ph.startswith("vf."):
sv = str(self.vn.flags.get(ph[3:]) or "")
elif ph == "srv.itime":
sv = str(int(time.time()))
elif ph == "srv.htime":
sv = datetime.now(UTC).strftime("%Y-%m-%d, %H:%M:%S")
else:
self.log("unknown placeholder in server config: [%s]" % (ph), 3)
continue
sv = self.conn.hsrv.ptn_hsafe.sub("_", sv)
txt = txt.replace("{{%s}}" % (ph,), sv)
return txt
def tx_file(self, req_path: str) -> bool:
status = 200
logmsg = "{:4} {} ".format("", self.req)
@ -3052,7 +3075,7 @@ class HttpCli(object):
self.reply(ico, mime=mime, headers={"Last-Modified": lm})
return True
def tx_md(self, fs_path: str) -> bool:
def tx_md(self, vn: VFS, fs_path: str) -> bool:
logmsg = " %s @%s " % (self.req, self.uname)
if not self.can_write:
@ -3069,9 +3092,16 @@ class HttpCli(object):
st = bos.stat(html_path)
ts_html = st.st_mtime
max_sz = 1024 * self.args.txt_max
sz_md = 0
lead = b""
fullfile = b""
for buf in yieldfile(fs_path):
if sz_md < max_sz:
fullfile += buf
else:
fullfile = b""
if not sz_md and b"\n" in buf[:2]:
lead = buf[: buf.find(b"\n") + 1]
sz_md += len(lead)
@ -3080,6 +3110,21 @@ class HttpCli(object):
for c, v in [(b"&", 4), (b"<", 3), (b">", 3)]:
sz_md += (len(buf) - len(buf.replace(c, b""))) * v
if (
fullfile
and "exp" in vn.flags
and "edit" not in self.uparam
and "edit2" not in self.uparam
and vn.flags.get("exp_md")
):
fulltxt = fullfile.decode("utf-8", "replace")
fulltxt = self._expand(fulltxt, vn.flags.get("exp_md") or [])
fullfile = fulltxt.encode("utf-8", "replace")
if fullfile:
fullfile = html_bescape(fullfile)
sz_md = len(lead) + len(fullfile)
file_ts = int(max(ts_md, ts_html, self.E.t0))
file_lastmod, do_send = self._chk_lastmod(file_ts)
self.out_headers["Last-Modified"] = file_lastmod
@ -3121,8 +3166,11 @@ class HttpCli(object):
try:
self.s.sendall(html[0] + lead)
for buf in yieldfile(fs_path):
self.s.sendall(html_bescape(buf))
if fullfile:
self.s.sendall(fullfile)
else:
for buf in yieldfile(fs_path):
self.s.sendall(html_bescape(buf))
self.s.sendall(html[1])
@ -3753,7 +3801,7 @@ class HttpCli(object):
or "edit2" in self.uparam
)
):
return self.tx_md(abspath)
return self.tx_md(vn, abspath)
return self.tx_file(abspath)
@ -3815,6 +3863,10 @@ class HttpCli(object):
if bos.path.exists(fn):
with open(fsenc(fn), "rb") as f:
logues[n] = f.read().decode("utf-8")
if "exp" in vn.flags:
logues[n] = self._expand(
logues[n], vn.flags.get("exp_lg") or []
)
readme = ""
if not self.args.no_readme and not logues[1]:
@ -3824,6 +3876,8 @@ class HttpCli(object):
with open(fsenc(fn), "rb") as f:
readme = f.read().decode("utf-8")
break
if readme and "exp" in vn.flags:
readme = self._expand(readme, vn.flags.get("exp_md") or [])
vf = vn.flags
unlist = vf.get("unlist", "")
@ -4134,6 +4188,12 @@ class HttpCli(object):
if sz < 1024 * self.args.txt_max:
with open(fsenc(docpath), "rb") as f:
doctxt = f.read().decode("utf-8", "replace")
if doc.lower().endswith(".md") and "exp" in vn.flags:
doctxt = self._expand(doctxt, vn.flags.get("exp_md") or [])
else:
self.log("doc 2big: [{}]".format(doc), c=6)
doctxt = "( size of textfile exceeds serverside limit )"
else:
self.log("doc 404: [{}]".format(doc), c=6)
doctxt = "( textfile not found )"

1
copyparty/httpsrv.py
View file

@ -149,6 +149,7 @@ class HttpSrv(object):
self._build_statics()
self.ptn_cc = re.compile(r"[\x00-\x1f]")
self.ptn_hsafe = re.compile(r"[\x00-\x1f<>\"'&]")
self.mallow = "GET HEAD POST PUT DELETE OPTIONS".split()
if not self.args.no_dav:

5
copyparty/svchub.py
View file

@ -39,6 +39,7 @@ from .util import (
FFMPEG_URL,
VERSIONS,
Daemon,
DEF_EXP,
DEF_MTE,
DEF_MTH,
Garda,
@ -442,6 +443,10 @@ class SvcHub(object):
mth = ODict.fromkeys(DEF_MTH.split(","), True)
al.mth = odfusion(mth, al.mth)
exp = ODict.fromkeys(DEF_EXP.split(" "), True)
al.exp_md = odfusion(exp, al.exp_md.replace(" ", ","))
al.exp_lg = odfusion(exp, al.exp_lg.replace(" ", ","))
for k in ["no_hash", "no_idx"]:
ptn = getattr(self.args, k)
if ptn:

2
copyparty/up2k.py
View file

@ -814,7 +814,7 @@ class Up2k(object):
if str(fl[k1]) == str(getattr(self.args, k2)):
del fl[k1]
else:
fl[k1] = ",".join(x for x in fl)
fl[k1] = ",".join(x for x in fl[k1])
a = [
(ft if v is True else ff if v is False else fv).format(k, str(v))
for k, v in fl.items()

11
copyparty/util.py
View file

@ -289,6 +289,8 @@ EXTS["vnd.mozilla.apng"] = "png"
MAGIC_MAP = {"jpeg": "jpg"}
DEF_EXP = "self.ip self.ua self.uname self.host cfg.name cfg.logout vf.scan vf.thsize hdr.cf_ipcountry srv.itime srv.htime"
DEF_MTE = "circle,album,.tn,artist,title,.bpm,key,.dur,.q,.vq,.aq,vc,ac,fmt,res,.fps,ahash,vhash"
DEF_MTH = ".vq,.aq,vc,ac,fmt,res,.fps"
@ -1809,15 +1811,18 @@ def exclude_dotfiles(filepaths: list[str]) -> list[str]:
def odfusion(base: ODict[str, bool], oth: str) -> ODict[str, bool]:
# merge an "ordered set" (just a dict really) with another list of keys
words0 = [x for x in oth.split(",") if x]
words1 = [x for x in oth[1:].split(",") if x]
ret = base.copy()
if oth.startswith("+"):
for k in oth[1:].split(","):
for k in words1:
ret[k] = True
elif oth[:1] in ("-", "/"):
for k in oth[1:].split(","):
for k in words1:
ret.pop(k, None)
else:
ret = ODict.fromkeys(oth.split(","), True)
ret = ODict.fromkeys(words0, True)
return ret

26
srv/expand/README.md Normal file
View file

@ -0,0 +1,26 @@
## text expansion
enable expansion of placeholder variables in `README.md` and prologue/epilogue files with `--exp` and customize the list of allowed placeholders to expand using `--exp-md` and `--exp-lg`
| explanation | placeholder |
| -------------------- | -------------------- |
| your ip address | {{self.ip}} |
| your user-agent | {{self.ua}} |
| your username | {{self.uname}} |
| the `Host` you see | {{self.host}} |
| server unix time | {{srv.itime}} |
| server datetime | {{srv.htime}} |
| server name | {{cfg.name}} |
| logout after | {{cfg.logout}} hours |
| vol reindex interval | {{vf.scan}} |
| thumbnail size | {{vf.thsize}} |
| your country | {{hdr.cf_ipcountry}} |
placeholders starting with...
* `self.` are grabbed from copyparty's internal state; anything in `httpcli.py` is fair game
* `cfg.` are the global server settings
* `vf.` are the volflags of the current volume
* `hdr.` are grabbed from the client headers; any header is supported, just add it (in lowercase) to the allowlist
* `srv.` are processed inside the `_expand` function in httpcli
for example (bad example), `hdr_cf_ipcountry` maps to the header `CF-IPCountry` (which is generated by cloudflare before the request is passed on to your server / copyparty)

5
tests/util.py
View file

@ -109,7 +109,7 @@ class Cfg(Namespace):
def __init__(self, a=None, v=None, c=None):
ka = {}
ex = "daw dav_auth dav_inf dav_mac dav_rt dotsrch e2d e2ds e2dsa e2t e2ts e2tsr e2v e2vu e2vp ed emp force_js getmod grid hardlink ih ihead magic never_symlink nid nih no_acode no_athumb no_dav no_dedup no_del no_dupe no_logues no_mv no_readme no_robots no_sb_md no_sb_lg no_scandir no_tarcmp no_thumb no_vthumb no_zip nrand nw rand smb th_no_crop vague_403 vc ver xdev xlink xvol"
ex = "daw dav_auth dav_inf dav_mac dav_rt dotsrch e2d e2ds e2dsa e2t e2ts e2tsr e2v e2vu e2vp ed emp exp force_js getmod grid hardlink ih ihead magic never_symlink nid nih no_acode no_athumb no_dav no_dedup no_del no_dupe no_logues no_mv no_readme no_robots no_sb_md no_sb_lg no_scandir no_tarcmp no_thumb no_vthumb no_zip nrand nw rand smb th_no_crop vague_403 vc ver xdev xlink xvol"
ka.update(**{k: False for k in ex.split()})
ex = "dotpart no_rescan no_sendfile no_voldump plain_ip"
@ -130,6 +130,9 @@ class Cfg(Namespace):
ex = "on403 on404 xad xar xau xban xbd xbr xbu xiu xm"
ka.update(**{k: [] for k in ex.split()})
ex = "exp_lg exp_md"
ka.update(**{k: {} for k in ex.split()})
super(Cfg, self).__init__(
a=a or [],
v=v or [],