From 0c50ea175785fbeb8e7fdc8bf6ec87b0c12ca9da Mon Sep 17 00:00:00 2001 From: ed Date: Sat, 16 Dec 2023 15:38:48 +0000 Subject: [PATCH] list dotfiles only for specific volumes or users (#66): * permission `.` grants dotfile visibility if user has `r` too * `-ed` will grant dotfiles to all `r` accounts (same as before) * volflag `dots` likewise also drops compatibility for pre-0.12.0 `-v` syntax (`-v .::red` will no longer translate to `-v .::r,ed`) --- README.md | 15 +++++- copyparty/__main__.py | 49 ++++--------------- copyparty/authsrv.py | 102 ++++++++++++++++++++++---------------- copyparty/cfg.py | 7 ++- copyparty/ftpd.py | 8 +-- copyparty/httpcli.py | 47 ++++++++---------- copyparty/u2idx.py | 30 ++++++------ copyparty/up2k.py | 7 +-- copyparty/util.py | 11 +++++ tests/test_dots.py | 111 ++++++++++++++++++++++++++++++++++++++++++ tests/test_vfs.py | 11 ++--- tests/util.py | 75 ++++++++++++++++++---------- 12 files changed, 306 insertions(+), 167 deletions(-) create mode 100644 tests/test_dots.py diff --git a/README.md b/README.md index 81f01f9c..2cc4fbec 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,7 @@ turn almost any device into a file server with resumable uploads/downloads using * [FAQ](#FAQ) - "frequently" asked questions * [accounts and volumes](#accounts-and-volumes) - per-folder, per-user permissions * [shadowing](#shadowing) - hiding specific subfolders + * [dotfiles](#dotfiles) - unix-style hidden files/folders * [the browser](#the-browser) - accessing a copyparty server using a web-browser * [tabs](#tabs) - the main tabs in the ui * [hotkeys](#hotkeys) - the browser has the following hotkeys @@ -368,6 +369,7 @@ permissions: * `w` (write): upload files, move files *into* this folder * `m` (move): move files/folders *from* this folder * `d` (delete): delete files/folders +* `.` (dots): user can ask to show dotfiles in directory listings * `g` (get): only download files, cannot see folder contents or zip/tar * `G` (upget): same as `g` except uploaders get to see their own [filekeys](#filekeys) (see `fk` in examples below) * `h` (html): same as `g` except folders return their index.html, and filekeys are not necessary for index.html @@ -399,6 +401,17 @@ hiding specific subfolders by mounting another volume on top of them for example `-v /mnt::r -v /var/empty:web/certs:r` mounts the server folder `/mnt` as the webroot, but another volume is mounted at `/web/certs` -- so visitors can only see the contents of `/mnt` and `/mnt/web` (at URLs `/` and `/web`), but not `/mnt/web/certs` because URL `/web/certs` is mapped to `/var/empty` +## dotfiles + +unix-style hidden files/folders by starting the name with a dot + +anyone can access these if they know the name, but they normally don't appear in directory listings + +a client can request to see dotfiles in directory listings if global option `-ed` is specified, or the volume has volflag `dots`, or the user has permission `.` + +dotfiles do not appear in search results unless one of the above is true, **and** the global option / volflag `dotsrch` is set + + # the browser accessing a copyparty server using a web-browser @@ -539,7 +552,7 @@ select which type of archive you want in the `[⚙️] config` tab: * gzip default level is `3` (0=fast, 9=best), change with `?tar=gz:9` * xz default level is `1` (0=fast, 9=best), change with `?tar=xz:9` * bz2 default level is `2` (1=fast, 9=best), change with `?tar=bz2:9` -* hidden files (dotfiles) are excluded unless `-ed` +* hidden files ([dotfiles](#dotfiles)) are excluded unless account is allowed to list them * `up2k.db` and `dir.txt` is always excluded * bsdtar supports streaming unzipping: `curl foo?zip=utf8 | bsdtar -xv` * good, because copyparty's zip is faster than tar on small files diff --git a/copyparty/__main__.py b/copyparty/__main__.py index 0525671e..32f841e8 100755 --- a/copyparty/__main__.py +++ b/copyparty/__main__.py @@ -19,11 +19,10 @@ import threading import time import traceback import uuid -from textwrap import dedent from .__init__ import ANYWIN, CORES, EXE, PY2, VT100, WINDOWS, E, EnvParams, unicode from .__version__ import CODENAME, S_BUILD_DT, S_VERSION -from .authsrv import expand_config_file, re_vol, split_cfg_ln, upgrade_cfg_fmt +from .authsrv import expand_config_file, split_cfg_ln, upgrade_cfg_fmt from .cfg import flagcats, onedash from .svchub import SvcHub from .util import ( @@ -37,6 +36,7 @@ from .util import ( UNPLICATIONS, align_tab, ansi_re, + dedent, min_ex, py_desc, pybin, @@ -498,6 +498,7 @@ def get_sects(): "g" (get): download files, but cannot see folder contents "G" (upget): "get", but can see filekeys of their own uploads "h" (html): "get", but folders return their index.html + "." (dots): user can ask to show dotfiles in listings "a" (admin): can see uploader IPs, config-reload too many volflags to list here, see --help-flags @@ -705,6 +706,7 @@ def get_sects(): \033[36mln\033[0m only prints symlinks leaving the volume mountpoint \033[36mp\033[0m exits 1 if any such symlinks are found \033[36mr\033[0m resumes startup after the listing + examples: --ls '**' # list all files which are possible to read --ls '**,*,ln' # check for dangerous symlinks @@ -738,9 +740,12 @@ def get_sects(): """ when \033[36m--ah-alg\033[0m is not the default [\033[32mnone\033[0m], all account passwords must be hashed - passwords can be hashed on the commandline with \033[36m--ah-gen\033[0m, but copyparty will also hash and print any passwords that are non-hashed (password which do not start with '+') and then terminate afterwards + passwords can be hashed on the commandline with \033[36m--ah-gen\033[0m, but + copyparty will also hash and print any passwords that are non-hashed + (password which do not start with '+') and then terminate afterwards - \033[36m--ah-alg\033[0m specifies the hashing algorithm and a list of optional comma-separated arguments: + \033[36m--ah-alg\033[0m specifies the hashing algorithm and a + list of optional comma-separated arguments: \033[36m--ah-alg argon2\033[0m # which is the same as: \033[36m--ah-alg argon2,3,256,4,19\033[0m @@ -821,7 +826,7 @@ def add_general(ap, nc, srvname): ap2.add_argument("-j", metavar="CORES", type=int, default=1, help="max num cpu cores, 0=all") ap2.add_argument("-a", metavar="ACCT", type=u, action="append", help="add account, \033[33mUSER\033[0m:\033[33mPASS\033[0m; example [\033[32med:wark\033[0m]") ap2.add_argument("-v", metavar="VOL", type=u, action="append", help="add volume, \033[33mSRC\033[0m:\033[33mDST\033[0m:\033[33mFLAG\033[0m; examples [\033[32m.::r\033[0m], [\033[32m/mnt/nas/music:/music:r:aed\033[0m]") - ap2.add_argument("-ed", action="store_true", help="enable the ?dots url parameter / client option which allows clients to see dotfiles / hidden files") + ap2.add_argument("-ed", action="store_true", help="enable the ?dots url parameter / client option which allows clients to see dotfiles / hidden files (volflag=dots)") ap2.add_argument("--urlform", metavar="MODE", type=u, default="print,get", help="how to handle url-form POSTs; see \033[33m--help-urlform\033[0m") ap2.add_argument("--wintitle", metavar="TXT", type=u, default="cpp @ $pub", help="server terminal title, for example [\033[32m$ip-10.1.2.\033[0m] or [\033[32m$ip-]") ap2.add_argument("--name", metavar="TXT", type=u, default=srvname, help="server name (displayed topleft in browser and in mDNS)") @@ -1457,40 +1462,6 @@ def main(argv: Optional[list[str]] = None) -> None: if al.ansi: al.wintitle = "" - nstrs: list[str] = [] - anymod = False - for ostr in al.v or []: - m = re_vol.match(ostr) - if not m: - # not our problem - nstrs.append(ostr) - continue - - src, dst, perms = m.groups() - na = [src, dst] - mod = False - for opt in perms.split(":"): - if re.match("c[^,]", opt): - mod = True - na.append("c," + opt[1:]) - elif re.sub("^[rwmdgGha]*", "", opt) and "," not in opt: - mod = True - perm = opt[0] - na.append(perm + "," + opt[1:]) - else: - na.append(opt) - - nstr = ":".join(na) - nstrs.append(nstr if mod else ostr) - if mod: - msg = "\033[1;31mWARNING:\033[0;1m\n -v {} \033[0;33mwas replaced with\033[0;1m\n -v {} \n\033[0m" - lprint(msg.format(ostr, nstr)) - anymod = True - - if anymod: - al.v = nstrs - time.sleep(2) - # propagate implications for k1, k2 in IMPLICATIONS: if getattr(al, k1): diff --git a/copyparty/authsrv.py b/copyparty/authsrv.py index 55f66bc4..fb90e842 100644 --- a/copyparty/authsrv.py +++ b/copyparty/authsrv.py @@ -72,6 +72,7 @@ class AXS(object): upget: Optional[Union[list[str], set[str]]] = None, uhtml: Optional[Union[list[str], set[str]]] = None, uadmin: Optional[Union[list[str], set[str]]] = None, + udot: Optional[Union[list[str], set[str]]] = None, ) -> None: self.uread: set[str] = set(uread or []) self.uwrite: set[str] = set(uwrite or []) @@ -81,9 +82,10 @@ class AXS(object): self.upget: set[str] = set(upget or []) self.uhtml: set[str] = set(uhtml or []) self.uadmin: set[str] = set(uadmin or []) + self.udot: set[str] = set(udot or []) def __repr__(self) -> str: - ks = "uread uwrite umove udel uget upget uhtml uadmin".split() + ks = "uread uwrite umove udel uget upget uhtml uadmin udot".split() return "AXS(%s)" % (", ".join("%s=%r" % (k, self.__dict__[k]) for k in ks),) @@ -336,6 +338,8 @@ class VFS(object): self.apget: dict[str, list[str]] = {} self.ahtml: dict[str, list[str]] = {} self.aadmin: dict[str, list[str]] = {} + self.adot: dict[str, list[str]] = {} + self.all_vols: dict[str, VFS] = {} if realpath: rp = realpath + ("" if realpath.endswith(os.sep) else os.sep) @@ -445,8 +449,8 @@ class VFS(object): def can_access( self, vpath: str, uname: str - ) -> tuple[bool, bool, bool, bool, bool, bool, bool]: - """can Read,Write,Move,Delete,Get,Upget,Admin""" + ) -> tuple[bool, bool, bool, bool, bool, bool, bool, bool]: + """can Read,Write,Move,Delete,Get,Upget,Admin,Dot""" if vpath: vn, _ = self._find(undot(vpath)) else: @@ -454,13 +458,14 @@ class VFS(object): c = vn.axs return ( - uname in c.uread or "*" in c.uread, - uname in c.uwrite or "*" in c.uwrite, - uname in c.umove or "*" in c.umove, - uname in c.udel or "*" in c.udel, - uname in c.uget or "*" in c.uget, - uname in c.upget or "*" in c.upget, - uname in c.uadmin or "*" in c.uadmin, + uname in c.uread, + uname in c.uwrite, + uname in c.umove, + uname in c.udel, + uname in c.uget, + uname in c.upget, + uname in c.uadmin, + uname in c.udot, ) # skip uhtml because it's rarely needed @@ -492,7 +497,7 @@ class VFS(object): (will_del, c.udel, "delete"), (will_get, c.uget, "get"), ]: - if req and (uname not in d and "*" not in d) and uname != LEELOO_DALLAS: + if req and uname not in d and uname != LEELOO_DALLAS: if vpath != cvpath and vpath != "." and self.log: ap = vn.canonical(rem) t = "{} has no {} in [{}] => [{}] => [{}]" @@ -553,7 +558,7 @@ class VFS(object): for pset in permsets: ok = True for req, lst in zip(pset, axs): - if req and uname not in lst and "*" not in lst: + if req and uname not in lst: ok = False if ok: break @@ -577,7 +582,7 @@ class VFS(object): seen: list[str], uname: str, permsets: list[list[bool]], - dots: bool, + wantdots: bool, scandir: bool, lstat: bool, subvols: bool = True, @@ -621,6 +626,10 @@ class VFS(object): rm1.append(le) _ = [vfs_ls.remove(x) for x in rm1] # type: ignore + dots_ok = wantdots and uname in dbv.axs.udot + if not dots_ok: + vfs_ls = [x for x in vfs_ls if "/." not in "/" + x[0]] + seen = seen[:] + [fsroot] rfiles = [x for x in vfs_ls if not stat.S_ISDIR(x[1].st_mode)] rdirs = [x for x in vfs_ls if stat.S_ISDIR(x[1].st_mode)] @@ -633,13 +642,13 @@ class VFS(object): yield dbv, vrem, rel, fsroot, rfiles, rdirs, vfs_virt for rdir, _ in rdirs: - if not dots and rdir.startswith("."): + if not dots_ok and rdir.startswith("."): continue wrel = (rel + "/" + rdir).lstrip("/") wrem = (rem + "/" + rdir).lstrip("/") for x in self.walk( - wrel, wrem, seen, uname, permsets, dots, scandir, lstat, subvols + wrel, wrem, seen, uname, permsets, wantdots, scandir, lstat, subvols ): yield x @@ -647,11 +656,13 @@ class VFS(object): return for n, vfs in sorted(vfs_virt.items()): - if not dots and n.startswith("."): + if not dots_ok and n.startswith("."): continue wrel = (rel + "/" + n).lstrip("/") - for x in vfs.walk(wrel, "", seen, uname, permsets, dots, scandir, lstat): + for x in vfs.walk( + wrel, "", seen, uname, permsets, wantdots, scandir, lstat + ): yield x def zipgen( @@ -660,7 +671,6 @@ class VFS(object): vrem: str, flt: set[str], uname: str, - dots: bool, dirs: bool, scandir: bool, wrap: bool = True, @@ -670,7 +680,7 @@ class VFS(object): # if single folder: the folder itself is the top-level item folder = "" if flt or not wrap else (vpath.split("/")[-1].lstrip(".") or "top") - g = self.walk(folder, vrem, [], uname, [[True, False]], dots, scandir, False) + g = self.walk(folder, vrem, [], uname, [[True, False]], True, scandir, False) for _, _, vpath, apath, files, rd, vd in g: if flt: files = [x for x in files if x[0] in flt] @@ -689,18 +699,6 @@ class VFS(object): apaths = [os.path.join(apath, n) for n in fnames] ret = list(zip(vpaths, apaths, files)) - if not dots: - # dotfile filtering based on vpath (intended visibility) - ret = [x for x in ret if "/." not in "/" + x[0]] - - zel = [ze for ze in rd if ze[0].startswith(".")] - for ze in zel: - rd.remove(ze) - - zsl = [zs for zs in vd.keys() if zs.startswith(".")] - for zs in zsl: - del vd[zs] - for f in [{"vp": v, "ap": a, "st": n[1]} for v, a, n in ret]: yield f @@ -958,16 +956,17 @@ class AuthSrv(object): try: self._l(ln, 5, "volume access config:") sk, sv = ln.split(":") - if re.sub("[rwmdgGha]", "", sk) or not sk: + if re.sub("[rwmdgGha.]", "", sk) or not sk: err = "invalid accs permissions list; " raise Exception(err) if " " in re.sub(", *", "", sv).strip(): err = "list of users is not comma-separated; " raise Exception(err) + assert vp is not None self._read_vol_str(sk, sv.replace(" ", ""), daxs[vp], mflags[vp]) continue except: - err += "accs entries must be 'rwmdgGha: user1, user2, ...'" + err += "accs entries must be 'rwmdgGha.: user1, user2, ...'" raise Exception(err + SBADCFG) if cat == catf: @@ -986,9 +985,11 @@ class AuthSrv(object): fstr += "," + sk else: fstr += ",{}={}".format(sk, sv) + assert vp is not None self._read_vol_str("c", fstr[1:], daxs[vp], mflags[vp]) fstr = "" if fstr: + assert vp is not None self._read_vol_str("c", fstr[1:], daxs[vp], mflags[vp]) continue except: @@ -1003,8 +1004,9 @@ class AuthSrv(object): def _read_vol_str( self, lvl: str, uname: str, axs: AXS, flags: dict[str, Any] ) -> None: - if lvl.strip("crwmdgGha"): - raise Exception("invalid volflag: {},{}".format(lvl, uname)) + if lvl.strip("crwmdgGha."): + t = "%s,%s" % (lvl, uname) if uname else lvl + raise Exception("invalid config value (volume or volflag): %s" % (t,)) if lvl == "c": cval: Union[bool, str] = True @@ -1032,6 +1034,7 @@ class AuthSrv(object): ("w", axs.uwrite), ("m", axs.umove), ("d", axs.udel), + (".", axs.udot), ("a", axs.uadmin), ("h", axs.uhtml), ("h", axs.uget), @@ -1110,7 +1113,7 @@ class AuthSrv(object): if self.args.v: # list of src:dst:permset:permset:... - # permset is [,username][,username] or ,[=args] + # permset is [,username][,username] or ,[=args] for v_str in self.args.v: m = re_vol.match(v_str) if not m: @@ -1200,14 +1203,21 @@ class AuthSrv(object): vol.all_vps.sort(key=lambda x: len(x[0]), reverse=True) vol.root = vfs - for perm in "read write move del get pget html admin".split(): + for perm in "read write move del get pget html admin dot".split(): axs_key = "u" + perm unames = ["*"] + list(acct.keys()) + for vp, vol in vfs.all_vols.items(): + zx = getattr(vol.axs, axs_key) + if "*" in zx: + for usr in unames: + zx.add(usr) + + # aread,... = dict[uname, list[volnames] or []] umap: dict[str, list[str]] = {x: [] for x in unames} for usr in unames: for vp, vol in vfs.all_vols.items(): zx = getattr(vol.axs, axs_key) - if usr in zx or "*" in zx: + if usr in zx: umap[usr].append(vp) umap[usr].sort() setattr(vfs, "a" + perm, umap) @@ -1224,6 +1234,7 @@ class AuthSrv(object): axs.upget, axs.uhtml, axs.uadmin, + axs.udot, ]: for usr in d: all_users[usr] = 1 @@ -1632,6 +1643,11 @@ class AuthSrv(object): vol.flags.pop(k[1:], None) vol.flags.pop(k) + for vol in vfs.all_vols.values(): + if vol.flags.get("dots"): + for name in vol.axs.uread: + vol.axs.udot.add(name) + if errors: sys.exit(1) @@ -1650,12 +1666,14 @@ class AuthSrv(object): [" write", "uwrite"], [" move", "umove"], ["delete", "udel"], + [" dots", "udot"], [" get", "uget"], - [" upget", "upget"], + [" upGet", "upget"], [" html", "uhtml"], ["uadmin", "uadmin"], ]: u = list(sorted(getattr(zv.axs, attr))) + u = ["*"] if "*" in u else u u = ", ".join("\033[35meverybody\033[0m" if x == "*" else x for x in u) u = u if u else "\033[36m--none--\033[0m" t += "\n| {}: {}".format(txt, u) @@ -1812,7 +1830,7 @@ class AuthSrv(object): raise Exception("volume not found: " + zs) self.log(str({"users": users, "vols": vols, "flags": flags})) - t = "/{}: read({}) write({}) move({}) del({}) get({}) upget({}) uadmin({})" + t = "/{}: read({}) write({}) move({}) del({}) dots({}) get({}) upGet({}) uadmin({})" for k, zv in self.vfs.all_vols.items(): vc = zv.axs vs = [ @@ -1821,6 +1839,7 @@ class AuthSrv(object): vc.uwrite, vc.umove, vc.udel, + vc.udot, vc.uget, vc.upget, vc.uhtml, @@ -1963,6 +1982,7 @@ class AuthSrv(object): "w": "uwrite", "m": "umove", "d": "udel", + ".": "udot", "g": "uget", "G": "upget", "h": "uhtml", @@ -2169,7 +2189,7 @@ def upgrade_cfg_fmt( else: sn = sn.replace(",", ", ") ret.append(" " + sn) - elif sn[:1] in "rwmdgGha": + elif sn[:1] in "rwmdgGha.": if cat != catx: cat = catx ret.append(cat) diff --git a/copyparty/cfg.py b/copyparty/cfg.py index 53865236..f7e0261a 100644 --- a/copyparty/cfg.py +++ b/copyparty/cfg.py @@ -9,6 +9,9 @@ onedash = set(zs.split()) def vf_bmap() -> dict[str, str]: """argv-to-volflag: simple bools""" ret = { + "dav_auth": "davauth", + "dav_rt": "davrt", + "ed": "dots", "never_symlink": "neversymlink", "no_dedup": "copydupes", "no_dupe": "nodupe", @@ -18,8 +21,6 @@ def vf_bmap() -> dict[str, str]: "no_vthumb": "dvthumb", "no_athumb": "dathumb", "th_no_crop": "nocrop", - "dav_auth": "davauth", - "dav_rt": "davrt", } for k in ( "dotsrch", @@ -98,6 +99,7 @@ permdescs = { "w": 'write; upload files; need "r" to see the uploads', "m": 'move; move files and folders; need "w" at destination', "d": "delete; permanently delete files and folders", + ".": "dots; user can ask to show dotfiles in listings", "g": "get; download files, but cannot see folder contents", "G": 'upget; same as "g" but can see filekeys of their own uploads', "h": 'html; same as "g" but folders return their index.html', @@ -202,6 +204,7 @@ flagcats = { "nohtml": "return html and markdown as text/html", }, "others": { + "dots": "allow all users with read-access to\nenable the option to show dotfiles in listings", "fk=8": 'generates per-file accesskeys,\nwhich are then required at the "g" permission;\nkeys are invalidated if filesize or inode changes', "fka=8": 'generates slightly weaker per-file accesskeys,\nwhich are then required at the "g" permission;\nnot affected by filesize or inode numbers', "davauth": "ask webdav clients to login for all folders", diff --git a/copyparty/ftpd.py b/copyparty/ftpd.py index 13bca7bf..2bc99c38 100644 --- a/copyparty/ftpd.py +++ b/copyparty/ftpd.py @@ -73,6 +73,7 @@ class FtpAuth(DummyAuthorizer): asrv = self.hub.asrv uname = "*" if username != "anonymous": + uname = "" for zs in (password, username): zs = asrv.iacct.get(asrv.ah.hash(zs), "") if zs: @@ -132,7 +133,7 @@ class FtpFs(AbstractedFS): self.can_read = self.can_write = self.can_move = False self.can_delete = self.can_get = self.can_upget = False - self.can_admin = False + self.can_admin = self.can_dot = False self.listdirinfo = self.listdir self.chdir(".") @@ -167,7 +168,7 @@ class FtpFs(AbstractedFS): if not avfs: raise FSE(t.format(vpath), 1) - cr, cw, cm, cd, _, _, _ = avfs.can_access("", self.h.uname) + cr, cw, cm, cd, _, _, _, _ = avfs.can_access("", self.h.uname) if r and not cr or w and not cw or m and not cm or d and not cd: raise FSE(t.format(vpath), 1) @@ -243,6 +244,7 @@ class FtpFs(AbstractedFS): self.can_get, self.can_upget, self.can_admin, + self.can_dot, ) = avfs.can_access("", self.h.uname) def mkdir(self, path: str) -> None: @@ -265,7 +267,7 @@ class FtpFs(AbstractedFS): vfs_ls = [x[0] for x in vfs_ls1] vfs_ls.extend(vfs_virt.keys()) - if not self.args.ed: + if not self.can_dot: vfs_ls = exclude_dotfiles(vfs_ls) vfs_ls.sort() diff --git a/copyparty/httpcli.py b/copyparty/httpcli.py index 900c9ec6..f59dfb32 100644 --- a/copyparty/httpcli.py +++ b/copyparty/httpcli.py @@ -154,10 +154,6 @@ class HttpCli(object): self.pw = " " self.rvol = [" "] self.wvol = [" "] - self.mvol = [" "] - self.dvol = [" "] - self.gvol = [" "] - self.upvol = [" "] self.avol = [" "] self.do_log = True self.can_read = False @@ -167,6 +163,7 @@ class HttpCli(object): self.can_get = False self.can_upget = False self.can_admin = False + self.can_dot = False self.out_headerlist: list[tuple[str, str]] = [] self.out_headers: dict[str, str] = {} self.html_head = " " @@ -467,10 +464,6 @@ class HttpCli(object): self.rvol = self.asrv.vfs.aread[self.uname] self.wvol = self.asrv.vfs.awrite[self.uname] - self.mvol = self.asrv.vfs.amove[self.uname] - self.dvol = self.asrv.vfs.adel[self.uname] - self.gvol = self.asrv.vfs.aget[self.uname] - self.upvol = self.asrv.vfs.apget[self.uname] self.avol = self.asrv.vfs.aadmin[self.uname] if self.pw and ( @@ -503,6 +496,7 @@ class HttpCli(object): self.can_get, self.can_upget, self.can_admin, + self.can_dot, ) = ( avn.can_access("", self.uname) if avn else [False] * 7 ) @@ -1131,7 +1125,6 @@ class HttpCli(object): rem, set(), self.uname, - self.args.ed, True, not self.args.no_scandir, wrap=False, @@ -1145,7 +1138,7 @@ class HttpCli(object): [[True, False]], lstat="davrt" not in vn.flags, ) - if not self.args.ed: + if not self.can_dot: names = set(exclude_dotfiles([x[0] for x in vfs_ls])) vfs_ls = [x for x in vfs_ls if x[0] in names] @@ -1910,7 +1903,7 @@ class HttpCli(object): items = [unquotep(x) for x in items if items] self.parser.drop() - return self.tx_zip(k, v, "", vn, rem, items, self.args.ed) + return self.tx_zip(k, v, "", vn, rem, items) def handle_post_json(self) -> bool: try: @@ -1996,10 +1989,10 @@ class HttpCli(object): def handle_search(self, body: dict[str, Any]) -> bool: idx = self.conn.get_u2idx() if not idx or not hasattr(idx, "p_end"): - raise Pebkac(500, "sqlite3 is not available on the server; cannot search") + raise Pebkac(500, "server busy, or sqlite3 not available; cannot search") - vols = [] - seen = {} + vols: list[VFS] = [] + seen: dict[VFS, bool] = {} for vtop in self.rvol: vfs, _ = self.asrv.vfs.get(vtop, self.uname, True, False) vfs = vfs.dbv or vfs @@ -2007,7 +2000,7 @@ class HttpCli(object): continue seen[vfs] = True - vols.append((vfs.vpath, vfs.realpath, vfs.flags)) + vols.append(vfs) t0 = time.time() if idx.p_end: @@ -2022,7 +2015,7 @@ class HttpCli(object): vbody = copy.deepcopy(body) vbody["hash"] = len(vbody["hash"]) self.log("qj: " + repr(vbody)) - hits = idx.fsearch(vols, body) + hits = idx.fsearch(self.uname, vols, body) msg: Any = repr(hits) taglist: list[str] = [] trunc = False @@ -2031,7 +2024,7 @@ class HttpCli(object): q = body["q"] n = body.get("n", self.args.srch_hits) self.log("qj: {} |{}|".format(q, n)) - hits, taglist, trunc = idx.search(vols, q, n) + hits, taglist, trunc = idx.search(self.uname, vols, q, n) msg = len(hits) idx.p_end = time.time() @@ -3002,7 +2995,6 @@ class HttpCli(object): vn: VFS, rem: str, items: list[str], - dots: bool, ) -> bool: if self.args.no_zip: raise Pebkac(400, "not enabled") @@ -3059,7 +3051,7 @@ class HttpCli(object): self.send_headers(None, mime=mime, headers={"Content-Disposition": cdis}) fgen = vn.zipgen( - vpath, rem, set(items), self.uname, dots, False, not self.args.no_scandir + vpath, rem, set(items), self.uname, False, not self.args.no_scandir ) # for f in fgen: print(repr({k: f[k] for k in ["vp", "ap"]})) cfmt = "" @@ -3473,6 +3465,7 @@ class HttpCli(object): ret["k" + quotep(excl)] = sub vfs = self.asrv.vfs + dots = False try: vn, rem = vfs.get(top, self.uname, True, False) fsroot, vfs_ls, vfs_virt = vn.ls( @@ -3481,6 +3474,7 @@ class HttpCli(object): not self.args.no_scandir, [[True, False], [False, True]], ) + dots = self.uname in vn.axs.udot except: vfs_ls = [] vfs_virt = {} @@ -3493,7 +3487,7 @@ class HttpCli(object): dirnames = [x[0] for x in vfs_ls if stat.S_ISDIR(x[1].st_mode)] - if not self.args.ed or "dots" not in self.uparam: + if not dots or "dots" not in self.uparam: dirnames = exclude_dotfiles(dirnames) for fn in [x for x in dirnames if x != excl]: @@ -3529,7 +3523,8 @@ class HttpCli(object): fk_vols = { vol: (vol.flags["fk"], 2 if "fka" in vol.flags else 1) for vp, vol in self.asrv.vfs.all_vols.items() - if "fk" in vol.flags and (vp in self.rvol or vp in self.upvol) + if "fk" in vol.flags + and (self.uname in vol.axs.uread or self.uname in vol.axs.upget) } for vol in self.asrv.vfs.all_vols.values(): cur = idx.get_cur(vol.realpath) @@ -3800,7 +3795,7 @@ class HttpCli(object): elif self.can_get and self.avn: axs = self.avn.axs - if self.uname not in axs.uhtml and "*" not in axs.uhtml: + if self.uname not in axs.uhtml: pass elif is_dir: for fn in ("index.htm", "index.html"): @@ -4021,7 +4016,7 @@ class HttpCli(object): for k in ["zip", "tar"]: v = self.uparam.get(k) if v is not None: - return self.tx_zip(k, v, self.vpath, vn, rem, [], self.args.ed) + return self.tx_zip(k, v, self.vpath, vn, rem, []) fsroot, vfs_ls, vfs_virt = vn.ls( rem, @@ -4052,13 +4047,13 @@ class HttpCli(object): pass # show dotfiles if permitted and requested - if not self.args.ed or ( + if not self.can_dot or ( "dots" not in self.uparam and (is_ls or "dots" not in self.cookies) ): ls_names = exclude_dotfiles(ls_names) - add_fk = vn.flags.get("fk") - fk_alg = 2 if "fka" in vn.flags else 1 + add_fk = vf.get("fk") + fk_alg = 2 if "fka" in vf else 1 dirs = [] files = [] diff --git a/copyparty/u2idx.py b/copyparty/u2idx.py index e839fd67..90736094 100644 --- a/copyparty/u2idx.py +++ b/copyparty/u2idx.py @@ -9,7 +9,7 @@ import time from operator import itemgetter from .__init__ import ANYWIN, TYPE_CHECKING, unicode -from .authsrv import LEELOO_DALLAS +from .authsrv import LEELOO_DALLAS, VFS from .bos import bos from .up2k import up2k_wark_from_hashlist from .util import ( @@ -63,7 +63,7 @@ class U2idx(object): self.log_func("u2idx", msg, c) def fsearch( - self, vols: list[tuple[str, str, dict[str, Any]]], body: dict[str, Any] + self, uname: str, vols: list[VFS], body: dict[str, Any] ) -> list[dict[str, Any]]: """search by up2k hashlist""" if not HAVE_SQLITE3: @@ -77,7 +77,7 @@ class U2idx(object): uv: list[Union[str, int]] = [wark[:16], wark] try: - return self.run_query(vols, uq, uv, True, False, 99999)[0] + return self.run_query(uname, vols, uq, uv, False, 99999)[0] except: raise Pebkac(500, min_ex()) @@ -122,7 +122,7 @@ class U2idx(object): return cur def search( - self, vols: list[tuple[str, str, dict[str, Any]]], uq: str, lim: int + self, uname: str, vols: list[VFS], uq: str, lim: int ) -> tuple[list[dict[str, Any]], list[str], bool]: """search by query params""" if not HAVE_SQLITE3: @@ -131,7 +131,6 @@ class U2idx(object): q = "" v: Union[str, int] = "" va: list[Union[str, int]] = [] - have_up = False # query has up.* operands have_mt = False is_key = True is_size = False @@ -176,26 +175,21 @@ class U2idx(object): if v == "size": v = "up.sz" is_size = True - have_up = True elif v == "date": v = "up.mt" is_date = True - have_up = True elif v == "up_at": v = "up.at" is_date = True - have_up = True elif v == "path": v = "trim(?||up.rd,'/')" va.append("\nrd") - have_up = True elif v == "name": v = "up.fn" - have_up = True elif v == "tags" or ptn_mt.match(v): have_mt = True @@ -271,22 +265,22 @@ class U2idx(object): q += " lower({}) {} ? ) ".format(field, oper) try: - return self.run_query(vols, q, va, have_up, have_mt, lim) + return self.run_query(uname, vols, q, va, have_mt, lim) except Exception as ex: raise Pebkac(500, repr(ex)) def run_query( self, - vols: list[tuple[str, str, dict[str, Any]]], + uname: str, + vols: list[VFS], uq: str, uv: list[Union[str, int]], - have_up: bool, have_mt: bool, lim: int, ) -> tuple[list[dict[str, Any]], list[str], bool]: if self.args.srch_dbg: t = "searching across all %s volumes in which the user has 'r' (full read access):\n %s" - zs = "\n ".join(["/%s = %s" % (x[0], x[1]) for x in vols]) + zs = "\n ".join(["/%s = %s" % (x.vpath, x.realpath) for x in vols]) self.log(t % (len(vols), zs), 5) done_flag: list[bool] = [] @@ -315,10 +309,14 @@ class U2idx(object): clamped = False taglist = {} - for (vtop, ptop, flags) in vols: + for vol in vols: if lim < 0: break + vtop = vol.vpath + ptop = vol.realpath + flags = vol.flags + cur = self.get_cur(ptop) if not cur: continue @@ -343,7 +341,7 @@ class U2idx(object): sret = [] fk = flags.get("fk") - dots = flags.get("dotsrch") + dots = flags.get("dotsrch") and uname in vol.axs.udot fk_alg = 2 if "fka" in flags else 1 c = cur.execute(uq, tuple(vuv)) for hit in c: diff --git a/copyparty/up2k.py b/copyparty/up2k.py index 32e8685e..0e95619d 100644 --- a/copyparty/up2k.py +++ b/copyparty/up2k.py @@ -2664,12 +2664,7 @@ class Up2k(object): not ret["hash"] and "fk" in vfs.flags and not self.args.nw - and ( - cj["user"] in vfs.axs.uread - or cj["user"] in vfs.axs.upget - or "*" in vfs.axs.uread - or "*" in vfs.axs.upget - ) + and (cj["user"] in vfs.axs.uread or cj["user"] in vfs.axs.upget) ): alg = 2 if "fka" in vfs.flags else 1 ap = absreal(djoin(job["ptop"], job["prel"], job["name"])) diff --git a/copyparty/util.py b/copyparty/util.py index 97c8820a..6bfe0b34 100644 --- a/copyparty/util.py +++ b/copyparty/util.py @@ -1074,6 +1074,17 @@ def nuprint(msg: str) -> None: uprint("{}\n".format(msg)) +def dedent(txt: str) -> str: + pad = 64 + lns = txt.replace("\r", "").split("\n") + for ln in lns: + zs = ln.lstrip() + pad2 = len(ln) - len(zs) + if zs and pad > pad2: + pad = pad2 + return "\n".join([ln[pad:] for ln in lns]) + + def rice_tid() -> str: tid = threading.current_thread().ident c = sunpack(b"B" * 5, spack(b">Q", tid)[-5:]) diff --git a/tests/test_dots.py b/tests/test_dots.py new file mode 100644 index 00000000..5822dfdd --- /dev/null +++ b/tests/test_dots.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +# coding: utf-8 +from __future__ import print_function, unicode_literals + +import io +import os +import shutil +import tarfile +import tempfile +import unittest + +from copyparty.authsrv import AuthSrv +from copyparty.httpcli import HttpCli +from copyparty.up2k import Up2k +from copyparty.u2idx import U2idx +from tests import util as tu +from tests.util import Cfg + + +def hdr(query, uname): + h = "GET /%s HTTP/1.1\r\nPW: %s\r\nConnection: close\r\n\r\n" + return (h % (query, uname)).encode("utf-8") + + +class TestHttpCli(unittest.TestCase): + def setUp(self): + self.td = tu.get_ramdisk() + + def tearDown(self): + os.chdir(tempfile.gettempdir()) + shutil.rmtree(self.td) + + def test(self): + td = os.path.join(self.td, "vfs") + os.mkdir(td) + os.chdir(td) + + # topDir volA volA/*dirA .volB .volB/*dirB + spaths = " t .t a a/da a/.da .b .b/db .b/.db" + for n, dirpath in enumerate(spaths.split(" ")): + if dirpath: + os.makedirs(dirpath) + + for pfx in "f", ".f": + filepath = pfx + str(n) + if dirpath: + filepath = os.path.join(dirpath, filepath) + + with open(filepath, "wb") as f: + f.write(filepath.encode("utf-8")) + + vcfg = [ + ".::r,u1:r.,u2", + "a:a:r,u1:r,u2", + ".b:.b:r.,u1:r,u2" + ] + self.args = Cfg(v=vcfg, a=["u1:u1", "u2:u2"], e2dsa=True) + self.asrv = AuthSrv(self.args, self.log) + + self.assertEqual(self.tardir("", "u1"), "f0 t/f1 a/f3 a/da/f4") + self.assertEqual(self.tardir(".t", "u1"), "f2") + self.assertEqual(self.tardir(".b", "u1"), ".f6 f6 .db/.f8 .db/f8 db/.f7 db/f7") + + zs = ".f0 f0 .t/.f2 .t/f2 t/.f1 t/f1 .b/f6 .b/db/f7 a/f3 a/da/f4" + self.assertEqual(self.tardir("", "u2"), zs) + + self.assertEqual(self.curl("?tar", "x")[1][:17], "\nJ2EOT") + + # search + up2k = Up2k(self) + u2idx = U2idx(self) + allvols = list(self.asrv.vfs.all_vols.values()) + + x = u2idx.search("u1", allvols, "", 999) + x = " ".join(sorted([x["rp"] for x in x[0]])) + # u1 can see dotfiles in volB so they should be included + xe = ".b/.db/.f8 .b/.db/f8 .b/.f6 .b/db/.f7 .b/db/f7 .b/f6 a/da/f4 a/f3 f0 t/f1" + self.assertEqual(x, xe) + + x = u2idx.search("u2", allvols, "", 999) + x = " ".join(sorted([x["rp"] for x in x[0]])) + self.assertEqual(x, ".f0 .t/.f2 .t/f2 a/da/f4 a/f3 f0 t/.f1 t/f1") + + self.args = Cfg(v=vcfg, a=["u1:u1", "u2:u2"], dotsrch=False) + self.asrv = AuthSrv(self.args, self.log) + u2idx = U2idx(self) + + x = u2idx.search("u1", self.asrv.vfs.all_vols.values(), "", 999) + x = " ".join(sorted([x["rp"] for x in x[0]])) + # u1 can see dotfiles in volB so they should be included + xe = "a/da/f4 a/f3 f0 t/f1" + self.assertEqual(x, xe) + + def tardir(self, url, uname): + h, b = self.curl("/" + url + "?tar", uname, True) + tar = tarfile.open(fileobj=io.BytesIO(b), mode="r|").getnames() + top = ("top" if not url else url.lstrip(".").split("/")[0]) + "/" + assert len(tar) == len([x for x in tar if x.startswith(top)]) + return " ".join([x[len(top):] for x in tar]) + + def curl(self, url, uname, binary=False): + conn = tu.VHttpConn(self.args, self.asrv, self.log, hdr(url, uname)) + HttpCli(conn).run() + if binary: + h, b = conn.s._reply.split(b"\r\n\r\n", 1) + return [h.decode("utf-8"), b] + + return conn.s._reply.decode("utf-8").split("\r\n\r\n", 1) + + def log(self, src, msg, c=0): + print(msg) diff --git a/tests/test_vfs.py b/tests/test_vfs.py index b79cc178..0d89260a 100644 --- a/tests/test_vfs.py +++ b/tests/test_vfs.py @@ -7,7 +7,6 @@ import os import shutil import tempfile import unittest -from textwrap import dedent from copyparty import util from copyparty.authsrv import VFS, AuthSrv @@ -175,11 +174,11 @@ class TestVFS(unittest.TestCase): self.assertEqual(len(vfs.nodes), 1) self.assertEqual(n.vpath, "a") self.assertEqual(n.realpath, os.path.join(td, "a")) - self.assertAxs(n.axs.uread, ["*"]) + self.assertAxs(n.axs.uread, ["*", "k"]) self.assertAxs(n.axs.uwrite, []) - perm_na = (False, False, False, False, False, False, False) - perm_rw = (True, True, False, False, False, False, False) - perm_ro = (True, False, False, False, False, False, False) + perm_na = (False, False, False, False, False, False, False, False) + perm_rw = (True, True, False, False, False, False, False, False) + perm_ro = (True, False, False, False, False, False, False, False) self.assertEqual(vfs.can_access("/", "*"), perm_na) self.assertEqual(vfs.can_access("/", "k"), perm_rw) self.assertEqual(vfs.can_access("/a", "*"), perm_ro) @@ -232,7 +231,7 @@ class TestVFS(unittest.TestCase): cfg_path = os.path.join(self.td, "test.cfg") with open(cfg_path, "wb") as f: f.write( - dedent( + util.dedent( """ u a:123 u asd:fgh:jkl diff --git a/tests/util.py b/tests/util.py index 4caa56de..4593b848 100644 --- a/tests/util.py +++ b/tests/util.py @@ -44,6 +44,7 @@ if MACOS: from copyparty.__init__ import E from copyparty.__main__ import init_E from copyparty.util import FHC, Garda, Unrecv +from copyparty.u2idx import U2idx init_E(E) @@ -106,51 +107,59 @@ def get_ramdisk(): class Cfg(Namespace): - def __init__(self, a=None, v=None, c=None): + def __init__(self, a=None, v=None, c=None, **ka0): ka = {} - ex = "daw dav_auth dav_inf dav_mac dav_rt dotsrch e2d e2ds e2dsa e2t e2ts e2tsr e2v e2vu e2vp ed emp exp force_js getmod grid hardlink ih ihead magic never_symlink nid nih no_acode no_athumb no_dav no_dedup no_del no_dupe no_logues no_mv no_readme no_robots no_sb_md no_sb_lg no_scandir no_tarcmp no_thumb no_vthumb no_zip nrand nw rand smb th_no_crop vague_403 vc ver xdev xlink xvol" + ex = "daw dav_auth dav_inf dav_mac dav_rt e2d e2ds e2dsa e2t e2ts e2tsr e2v e2vu e2vp ed emp exp force_js getmod grid hardlink ih ihead magic never_symlink nid nih no_acode no_athumb no_dav no_dedup no_del no_dupe no_lifetime no_logues no_mv no_readme no_robots no_sb_md no_sb_lg no_scandir no_tarcmp no_thumb no_vthumb no_zip nrand nw rand smb srch_dbg stats th_no_crop vague_403 vc ver xdev xlink xvol" ka.update(**{k: False for k in ex.split()}) - ex = "dotpart no_rescan no_sendfile no_voldump plain_ip" + ex = "dotpart dotsrch no_dhash no_fastboot no_rescan no_sendfile no_voldump re_dhash plain_ip" ka.update(**{k: True for k in ex.split()}) ex = "ah_cli ah_gen css_browser hist ipa_re js_browser no_forget no_hash no_idx nonsus_urls" ka.update(**{k: None for k in ex.split()}) - ex = "s_thead s_tbody th_convt" + ex = "hash_mt srch_time" + ka.update(**{k: 1 for k in ex.split()}) + + ex = "reg_cap s_thead s_tbody th_convt" ka.update(**{k: 9 for k in ex.split()}) - ex = "df loris re_maxage rproxy rsp_jtr rsp_slp s_wr_slp theme themes turbo" + ex = "db_act df loris re_maxage rproxy rsp_jtr rsp_slp s_wr_slp snap_wri theme themes turbo" ka.update(**{k: 0 for k in ex.split()}) - ex = "ah_alg bname doctitle favico hdr_au_usr html_head lg_sbf log_fk md_sbf name textfiles unlist vname R RS SR" + ex = "ah_alg bname doctitle exit favico hdr_au_usr html_head lg_sbf log_fk md_sbf name textfiles unlist vname R RS SR" ka.update(**{k: "" for k in ex.split()}) ex = "on403 on404 xad xar xau xban xbd xbr xbu xiu xm" ka.update(**{k: [] for k in ex.split()}) - ex = "exp_lg exp_md" + ex = "exp_lg exp_md th_coversd" ka.update(**{k: {} for k in ex.split()}) + ka.update(ka0) + super(Cfg, self).__init__( a=a or [], v=v or [], c=c, E=E, dbd="wal", - s_wr_sz=512 * 1024, - th_size="320x256", fk_salt="a" * 16, - unpost=600, - u2sort="s", - u2ts="c", - sort="href", - mtp=[], + lang="eng", + log_badpwd=1, + logout=573, mte={"a": True}, mth={}, - lang="eng", - logout=573, + mtp=[], + s_wr_sz=512 * 1024, + sort="href", + srch_hits=99999, + th_size="320x256", + u2sort="s", + u2ts="c", + unpost=600, + warksalt="hunter2", **ka ) @@ -186,11 +195,16 @@ class VSock(object): class VHttpSrv(object): - def __init__(self): + def __init__(self, args, asrv, log): + self.args = args + self.asrv = asrv + self.log = log + self.broker = NullBroker() self.prism = None self.bans = {} self.nreq = 0 + self.nsus = 0 aliases = ["splash", "browser", "browser2", "msg", "md", "mde"] self.j2 = {x: J2_FILES for x in aliases} @@ -200,31 +214,38 @@ class VHttpSrv(object): self.g403 = Garda("") self.gurl = Garda("") + self.u2idx = None self.ptn_cc = re.compile(r"[\x00-\x1f]") def cachebuster(self): return "a" + def get_u2idx(self): + self.u2idx = self.u2idx or U2idx(self) + return self.u2idx + class VHttpConn(object): def __init__(self, args, asrv, log, buf): + self.t0 = time.time() self.s = VSock(buf) self.sr = Unrecv(self.s, None) # type: ignore + self.aclose = {} self.addr = ("127.0.0.1", "42069") self.args = args self.asrv = asrv - self.nid = None + self.bans = {} + self.freshen_pwd = 0.0 + self.hsrv = VHttpSrv(args, asrv, log) + self.ico = None + self.lf_url = None self.log_func = log self.log_src = "a" - self.lf_url = None - self.hsrv = VHttpSrv() - self.bans = {} - self.aclose = {} - self.u2fh = FHC() self.mutex = threading.Lock() - self.nreq = -1 self.nbyte = 0 - self.ico = None + self.nid = None + self.nreq = -1 self.thumbcli = None - self.freshen_pwd = 0.0 - self.t0 = time.time() + self.u2fh = FHC() + + self.get_u2idx = self.hsrv.get_u2idx \ No newline at end of file