mirrors/copyparty
1
0
Fork 0
mirror of https://github.com/9001/copyparty.git synced 2025-08-17 09:02:15 -06:00
Code Issues Actions Packages Projects Releases Wiki Activity

add hook side-effects; closes #86

Browse source 
hooks can now interrupt or redirect actions, and initiate
related actions, by printing json on stdout with commands

mainly to mitigate limitations such as sharex/sharex#3992

xbr/xau can redirect uploads to other destinations with `reloc`
and most hooks can initiate indexing or deletion of additional
files by giving a list of vpaths in json-keys `idx` or `del`

there are limitations;
* xbu/xau effects don't apply to ftp, tftp, smb
* xau will intentionally fail if a reloc destination exists
* xau effects do not apply to up2k

also provides more details for hooks:
* xbu/xau: basic-uploader vpath with filename
* xbr/xar: add client ip
This commit is contained in:
ed 2024-08-11 14:52:32 +00:00
parent 20669c73d3
commit 6c94a63f1c
15 changed files with 799 additions and 181 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
README.md
View file

@ -1313,6 +1313,8 @@ you can set hooks before and/or after an event happens, and currently you can ho
there's a bunch of flags and stuff, see `--help-hooks` there's a bunch of flags and stuff, see `--help-hooks`
if you want to write your own hooks, see [devnotes](./docs/devnotes.md#event-hooks)
### upload events ### upload events

4
bin/hooks/into-the-cache-it-goes.py
View file

@ -41,8 +41,8 @@ parameters explained,
t10 = abort download and continue if it takes longer than 10sec t10 = abort download and continue if it takes longer than 10sec
example usage as a volflag (per-volume config): example usage as a volflag (per-volume config):
-v srv/inc:inc:r:rw,ed:xau=j,t10,bin/hooks/into-the-cache-it-goes.py -v srv/inc:inc:r:rw,ed:c,xau=j,t10,bin/hooks/into-the-cache-it-goes.py
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(share filesystem-path srv/inc as volume /inc, (share filesystem-path srv/inc as volume /inc,
readable by everyone, read-write for user 'ed', readable by everyone, read-write for user 'ed',

94
bin/hooks/reloc-by-ext.py Normal file
View file

@ -0,0 +1,94 @@
#!/usr/bin/env python3
import json
import os
import sys
_ = r"""
relocate/redirect incoming uploads according to file extension
example usage as global config:
--xbu j,c1,bin/hooks/reloc-by-ext.py
parameters explained,
xbu = execute before upload
j = this hook needs upload information as json (not just the filename)
c1 = this hook returns json on stdout, so tell copyparty to read that
example usage as a volflag (per-volume config):
-v srv/inc:inc:r:rw,ed:c,xbu=j,c1,bin/hooks/reloc-by-ext.py
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(share filesystem-path srv/inc as volume /inc,
readable by everyone, read-write for user 'ed',
running this plugin on all uploads with the params explained above)
example usage as a volflag in a copyparty config file:
[/inc]
srv/inc
accs:
r: *
rw: ed
flags:
xbu: j,c1,bin/hooks/reloc-by-ext.py
note: this only works with the basic uploader (sharex and such),
does not work with up2k / dragdrop into browser
note: this could also work as an xau hook (after-upload), but
because it doesn't need to read the file contents its better
as xbu (before-upload) since that's safer / less buggy
"""
PICS = "avif bmp gif heic heif jpeg jpg jxl png psd qoi tga tif tiff webp"
VIDS = "3gp asf avi flv mkv mov mp4 mpeg mpeg2 mpegts mpg mpg2 nut ogm ogv rm ts vob webm wmv"
MUSIC = "aac aif aiff alac amr ape dfpwm flac m4a mp3 ogg opus ra tak tta wav wma wv"
def main():
inf = json.loads(sys.argv[1])
vdir, fn = os.path.split(inf["vp"])
try:
fn, ext = fn.rsplit(".", 1)
except:
# no file extension; abort
return
ext = ext.lower()
##
## some example actions to take; pick one by
## selecting it inside the print at the end:
##
# create a subfolder named after the filetype and move it into there
into_subfolder = {"vp": ext}
# move it into a toplevel folder named after the filetype
into_toplevel = {"vp": "/" + ext}
# move it into a filetype-named folder next to the target folder
into_sibling = {"vp": "../" + ext}
# move images into "/just/pics", vids into "/just/vids",
# music into "/just/tunes", and anything else as-is
if ext in PICS.split():
by_category = {"vp": "/just/pics"}
elif ext in VIDS.split():
by_category = {"vp": "/just/vids"}
elif ext in MUSIC.split():
by_category = {"vp": "/just/tunes"}
else:
by_category = {}
# now choose the effect to apply; can be any of these:
# into_subfolder into_toplevel into_sibling by_category
effect = into_subfolder
print(json.dumps({"reloc": effect}))
if __name__ == "__main__":
main()

6
copyparty/__main__.py
View file

@ -704,6 +704,11 @@ def get_sects():
\033[36mxban\033[0m can be used to overrule / cancel a user ban event; \033[36mxban\033[0m can be used to overrule / cancel a user ban event;
if the program returns 0 (true/OK) then the ban will NOT happen if the program returns 0 (true/OK) then the ban will NOT happen
effects can be used to redirect uploads into other
locations, and to delete or index other files based
on new uploads, but with certain limitations. See
bin/hooks/reloc* and docs/devnotes.md#hook-effects
except for \033[36mxm\033[0m, only one hook / one action can run at a time, except for \033[36mxm\033[0m, only one hook / one action can run at a time,
so it's recommended to use the \033[36mf\033[0m flag unless you really need so it's recommended to use the \033[36mf\033[0m flag unless you really need
to wait for the hook to finish before continuing (without \033[36mf\033[0m to wait for the hook to finish before continuing (without \033[36mf\033[0m
@ -1132,6 +1137,7 @@ def add_hooks(ap):
ap2.add_argument("--xad", metavar="CMD", type=u, action="append", help="execute \033[33mCMD\033[0m after a file delete") ap2.add_argument("--xad", metavar="CMD", type=u, action="append", help="execute \033[33mCMD\033[0m after a file delete")
ap2.add_argument("--xm", metavar="CMD", type=u, action="append", help="execute \033[33mCMD\033[0m on message") ap2.add_argument("--xm", metavar="CMD", type=u, action="append", help="execute \033[33mCMD\033[0m on message")
ap2.add_argument("--xban", metavar="CMD", type=u, action="append", help="execute \033[33mCMD\033[0m if someone gets banned (pw/404/403/url)") ap2.add_argument("--xban", metavar="CMD", type=u, action="append", help="execute \033[33mCMD\033[0m if someone gets banned (pw/404/403/url)")
ap2.add_argument("--hook-v", action="store_true", help="verbose hooks")
def add_stats(ap): def add_stats(ap):

6
copyparty/authsrv.py
View file

@ -521,8 +521,8 @@ class VFS(object):
t = "{} has no {} in [{}] => [{}] => [{}]" t = "{} has no {} in [{}] => [{}] => [{}]"
self.log("vfs", t.format(uname, msg, vpath, cvpath, ap), 6) self.log("vfs", t.format(uname, msg, vpath, cvpath, ap), 6)
t = 'you don\'t have %s-access in "/%s"' t = 'you don\'t have %s-access in "/%s" or below "/%s"'
raise Pebkac(err, t % (msg, cvpath)) raise Pebkac(err, t % (msg, cvpath, vn.vpath))
return vn, rem return vn, rem
@ -1898,7 +1898,7 @@ class AuthSrv(object):
self.log(t.format(vol.vpath), 1) self.log(t.format(vol.vpath), 1)
del vol.flags["lifetime"] del vol.flags["lifetime"]
needs_e2d = [x for x in hooks if x != "xm"] needs_e2d = [x for x in hooks if x in ("xau", "xiu")]
drop = [x for x in needs_e2d if vol.flags.get(x)] drop = [x for x in needs_e2d if vol.flags.get(x)]
if drop: if drop:
t = 'removing [{}] from volume "/{}" because e2d is disabled' t = 'removing [{}] from volume "/{}" because e2d is disabled'

7
copyparty/ftpd.py
View file

@ -353,7 +353,7 @@ class FtpFs(AbstractedFS):
svp = join(self.cwd, src).lstrip("/") svp = join(self.cwd, src).lstrip("/")
dvp = join(self.cwd, dst).lstrip("/") dvp = join(self.cwd, dst).lstrip("/")
try: try:
self.hub.up2k.handle_mv(self.uname, svp, dvp) self.hub.up2k.handle_mv(self.uname, self.h.cli_ip, svp, dvp)
except Exception as ex: except Exception as ex:
raise FSE(str(ex)) raise FSE(str(ex))
@ -471,6 +471,9 @@ class FtpHandler(FTPHandler):
xbu = vfs.flags.get("xbu") xbu = vfs.flags.get("xbu")
if xbu and not runhook( if xbu and not runhook(
None, None,
None,
self.hub.up2k,
"xbu.ftpd",
xbu, xbu,
ap, ap,
vp, vp,
@ -480,7 +483,7 @@ class FtpHandler(FTPHandler):
0, 0,
0, 0,
self.cli_ip, self.cli_ip,
0, time.time(),
"", "",
): ):
raise FSE("Upload blocked by xbu server config") raise FSE("Upload blocked by xbu server config")

260
copyparty/httpcli.py
View file

@ -73,7 +73,9 @@ from .util import (
humansize, humansize,
ipnorm, ipnorm,
loadpy, loadpy,
log_reloc,
min_ex, min_ex,
pathmod,
quotep, quotep,
rand_name, rand_name,
read_header, read_header,
@ -695,6 +697,9 @@ class HttpCli(object):
xban = self.vn.flags.get("xban") xban = self.vn.flags.get("xban")
if not xban or not runhook( if not xban or not runhook(
self.log, self.log,
self.conn.hsrv.broker,
None,
"xban",
xban, xban,
self.vn.canonical(self.rem), self.vn.canonical(self.rem),
self.vpath, self.vpath,
@ -1172,7 +1177,8 @@ class HttpCli(object):
if self.args.no_dav: if self.args.no_dav:
raise Pebkac(405, "WebDAV is disabled in server config") raise Pebkac(405, "WebDAV is disabled in server config")
vn, rem = self.asrv.vfs.get(self.vpath, self.uname, False, False, err=401) vn = self.vn
rem = self.rem
tap = vn.canonical(rem) tap = vn.canonical(rem)
if "davauth" in vn.flags and self.uname == "*": if "davauth" in vn.flags and self.uname == "*":
@ -1556,8 +1562,8 @@ class HttpCli(object):
self.log("PUT %s @%s" % (self.req, self.uname)) self.log("PUT %s @%s" % (self.req, self.uname))
if not self.can_write: if not self.can_write:
t = "user {} does not have write-access here" t = "user %s does not have write-access under /%s"
raise Pebkac(403, t.format(self.uname)) raise Pebkac(403, t % (self.uname, self.vn.vpath))
if not self.args.no_dav and self._applesan(): if not self.args.no_dav and self._applesan():
return self.headers.get("content-length") == "0" return self.headers.get("content-length") == "0"
@ -1632,6 +1638,9 @@ class HttpCli(object):
if xm: if xm:
runhook( runhook(
self.log, self.log,
self.conn.hsrv.broker,
None,
"xm",
xm, xm,
self.vn.canonical(self.rem), self.vn.canonical(self.rem),
self.vpath, self.vpath,
@ -1780,11 +1789,15 @@ class HttpCli(object):
if xbu: if xbu:
at = time.time() - lifetime at = time.time() - lifetime
if not runhook( vp = vjoin(self.vpath, fn) if nameless else self.vpath
hr = runhook(
self.log, self.log,
self.conn.hsrv.broker,
None,
"xbu.http.dump",
xbu, xbu,
path, path,
self.vpath, vp,
self.host, self.host,
self.uname, self.uname,
self.asrv.vfs.get_perms(self.vpath, self.uname), self.asrv.vfs.get_perms(self.vpath, self.uname),
@ -1793,10 +1806,25 @@ class HttpCli(object):
self.ip, self.ip,
at, at,
"", "",
): )
if not hr:
t = "upload blocked by xbu server config" t = "upload blocked by xbu server config"
self.log(t, 1) self.log(t, 1)
raise Pebkac(403, t) raise Pebkac(403, t)
if hr.get("reloc"):
x = pathmod(self.asrv.vfs, path, vp, hr["reloc"])
if x:
if self.args.hook_v:
log_reloc(self.log, hr["reloc"], x, path, vp, fn, vfs, rem)
fdir, self.vpath, fn, (vfs, rem) = x
if self.args.nw:
fn = os.devnull
else:
bos.makedirs(fdir)
path = os.path.join(fdir, fn)
if not nameless:
self.vpath = vjoin(self.vpath, fn)
params["fdir"] = fdir
if is_put and not (self.args.no_dav or self.args.nw) and bos.path.exists(path): if is_put and not (self.args.no_dav or self.args.nw) and bos.path.exists(path):
# allow overwrite if... # allow overwrite if...
@ -1871,24 +1899,45 @@ class HttpCli(object):
fn = fn2 fn = fn2
path = path2 path = path2
if xau and not runhook( if xau:
self.log, vp = vjoin(self.vpath, fn) if nameless else self.vpath
xau, hr = runhook(
path, self.log,
self.vpath, self.conn.hsrv.broker,
self.host, None,
self.uname, "xau.http.dump",
self.asrv.vfs.get_perms(self.vpath, self.uname), xau,
mt, path,
post_sz, vp,
self.ip, self.host,
at, self.uname,
"", self.asrv.vfs.get_perms(self.vpath, self.uname),
): mt,
t = "upload blocked by xau server config" post_sz,
self.log(t, 1) self.ip,
wunlink(self.log, path, vfs.flags) at,
raise Pebkac(403, t) "",
)
if not hr:
t = "upload blocked by xau server config"
self.log(t, 1)
wunlink(self.log, path, vfs.flags)
raise Pebkac(403, t)
if hr.get("reloc"):
x = pathmod(self.asrv.vfs, path, vp, hr["reloc"])
if x:
if self.args.hook_v:
log_reloc(self.log, hr["reloc"], x, path, vp, fn, vfs, rem)
fdir, self.vpath, fn, (vfs, rem) = x
bos.makedirs(fdir)
path2 = os.path.join(fdir, fn)
atomic_move(self.log, path, path2, vfs.flags)
path = path2
if not nameless:
self.vpath = vjoin(self.vpath, fn)
sz = bos.path.getsize(path)
else:
sz = post_sz
vfs, rem = vfs.get_dbv(rem) vfs, rem = vfs.get_dbv(rem)
self.conn.hsrv.broker.say( self.conn.hsrv.broker.say(
@ -1911,7 +1960,7 @@ class HttpCli(object):
alg, alg,
self.args.fk_salt, self.args.fk_salt,
path, path,
post_sz, sz,
0 if ANYWIN else bos.stat(path).st_ino, 0 if ANYWIN else bos.stat(path).st_ino,
)[: vfs.flags["fk"]] )[: vfs.flags["fk"]]
@ -2536,18 +2585,15 @@ class HttpCli(object):
fname = sanitize_fn( fname = sanitize_fn(
p_file or "", "", [".prologue.html", ".epilogue.html"] p_file or "", "", [".prologue.html", ".epilogue.html"]
) )
abspath = os.path.join(fdir, fname)
suffix = "-%.6f-%s" % (time.time(), dip)
if p_file and not nullwrite: if p_file and not nullwrite:
if rnd: if rnd:
fname = rand_name(fdir, fname, rnd) fname = rand_name(fdir, fname, rnd)
if not bos.path.isdir(fdir):
raise Pebkac(404, "that folder does not exist")
suffix = "-{:.6f}-{}".format(time.time(), dip)
open_args = {"fdir": fdir, "suffix": suffix} open_args = {"fdir": fdir, "suffix": suffix}
if "replace" in self.uparam: if "replace" in self.uparam:
abspath = os.path.join(fdir, fname)
if not self.can_delete: if not self.can_delete:
self.log("user not allowed to overwrite with ?replace") self.log("user not allowed to overwrite with ?replace")
elif bos.path.exists(abspath): elif bos.path.exists(abspath):
@ -2557,6 +2603,58 @@ class HttpCli(object):
except: except:
t = "toctou while deleting for ?replace: %s" t = "toctou while deleting for ?replace: %s"
self.log(t % (abspath,)) self.log(t % (abspath,))
else:
open_args = {}
tnam = fname = os.devnull
fdir = abspath = ""
if xbu:
at = time.time() - lifetime
hr = runhook(
self.log,
self.conn.hsrv.broker,
None,
"xbu.http.bup",
xbu,
abspath,
vjoin(upload_vpath, fname),
self.host,
self.uname,
self.asrv.vfs.get_perms(upload_vpath, self.uname),
at,
0,
self.ip,
at,
"",
)
if not hr:
t = "upload blocked by xbu server config"
self.log(t, 1)
raise Pebkac(403, t)
if hr.get("reloc"):
zs = vjoin(upload_vpath, fname)
x = pathmod(self.asrv.vfs, abspath, zs, hr["reloc"])
if x:
if self.args.hook_v:
log_reloc(
self.log,
hr["reloc"],
x,
abspath,
zs,
fname,
vfs,
rem,
)
fdir, upload_vpath, fname, (vfs, rem) = x
abspath = os.path.join(fdir, fname)
if nullwrite:
fdir = abspath = ""
else:
open_args["fdir"] = fdir
if p_file and not nullwrite:
bos.makedirs(fdir)
# reserve destination filename # reserve destination filename
with ren_open(fname, "wb", fdir=fdir, suffix=suffix) as zfw: with ren_open(fname, "wb", fdir=fdir, suffix=suffix) as zfw:
@ -2572,26 +2670,6 @@ class HttpCli(object):
tnam = fname = os.devnull tnam = fname = os.devnull
fdir = abspath = "" fdir = abspath = ""
if xbu:
at = time.time() - lifetime
if not runhook(
self.log,
xbu,
abspath,
self.vpath,
self.host,
self.uname,
self.asrv.vfs.get_perms(self.vpath, self.uname),
at,
0,
self.ip,
at,
"",
):
t = "upload blocked by xbu server config"
self.log(t, 1)
raise Pebkac(403, t)
if lim: if lim:
lim.chk_bup(self.ip) lim.chk_bup(self.ip)
lim.chk_nup(self.ip) lim.chk_nup(self.ip)
@ -2634,29 +2712,58 @@ class HttpCli(object):
tabspath = "" tabspath = ""
at = time.time() - lifetime
if xau:
hr = runhook(
self.log,
self.conn.hsrv.broker,
None,
"xau.http.bup",
xau,
abspath,
vjoin(upload_vpath, fname),
self.host,
self.uname,
self.asrv.vfs.get_perms(upload_vpath, self.uname),
at,
sz,
self.ip,
at,
"",
)
if not hr:
t = "upload blocked by xau server config"
self.log(t, 1)
wunlink(self.log, abspath, vfs.flags)
raise Pebkac(403, t)
if hr.get("reloc"):
zs = vjoin(upload_vpath, fname)
x = pathmod(self.asrv.vfs, abspath, zs, hr["reloc"])
if x:
if self.args.hook_v:
log_reloc(
self.log,
hr["reloc"],
x,
abspath,
zs,
fname,
vfs,
rem,
)
fdir, upload_vpath, fname, (vfs, rem) = x
ap2 = os.path.join(fdir, fname)
if nullwrite:
fdir = ap2 = ""
else:
bos.makedirs(fdir)
atomic_move(self.log, abspath, ap2, vfs.flags)
abspath = ap2
sz = bos.path.getsize(abspath)
files.append( files.append(
(sz, sha_hex, sha_b64, p_file or "(discarded)", fname, abspath) (sz, sha_hex, sha_b64, p_file or "(discarded)", fname, abspath)
) )
at = time.time() - lifetime
if xau and not runhook(
self.log,
xau,
abspath,
self.vpath,
self.host,
self.uname,
self.asrv.vfs.get_perms(self.vpath, self.uname),
at,
sz,
self.ip,
at,
"",
):
t = "upload blocked by xau server config"
self.log(t, 1)
wunlink(self.log, abspath, vfs.flags)
raise Pebkac(403, t)
dbv, vrem = vfs.get_dbv(rem) dbv, vrem = vfs.get_dbv(rem)
self.conn.hsrv.broker.say( self.conn.hsrv.broker.say(
"up2k.hash_file", "up2k.hash_file",
@ -2712,13 +2819,14 @@ class HttpCli(object):
for sz, sha_hex, sha_b64, ofn, lfn, ap in files: for sz, sha_hex, sha_b64, ofn, lfn, ap in files:
vsuf = "" vsuf = ""
if (self.can_read or self.can_upget) and "fk" in vfs.flags: if (self.can_read or self.can_upget) and "fk" in vfs.flags:
st = bos.stat(ap)
alg = 2 if "fka" in vfs.flags else 1 alg = 2 if "fka" in vfs.flags else 1
vsuf = "?k=" + self.gen_fk( vsuf = "?k=" + self.gen_fk(
alg, alg,
self.args.fk_salt, self.args.fk_salt,
ap, ap,
sz, st.st_size,
0 if ANYWIN or not ap else bos.stat(ap).st_ino, 0 if ANYWIN or not ap else st.st_ino,
)[: vfs.flags["fk"]] )[: vfs.flags["fk"]]
if "media" in self.uparam or "medialinks" in vfs.flags: if "media" in self.uparam or "medialinks" in vfs.flags:
@ -2885,6 +2993,9 @@ class HttpCli(object):
if xbu: if xbu:
if not runhook( if not runhook(
self.log, self.log,
self.conn.hsrv.broker,
None,
"xbu.http.txt",
xbu, xbu,
fp, fp,
self.vpath, self.vpath,
@ -2924,6 +3035,9 @@ class HttpCli(object):
xau = vfs.flags.get("xau") xau = vfs.flags.get("xau")
if xau and not runhook( if xau and not runhook(
self.log, self.log,
self.conn.hsrv.broker,
None,
"xau.http.txt",
xau, xau,
fp, fp,
self.vpath, self.vpath,
@ -4156,7 +4270,7 @@ class HttpCli(object):
if self.args.no_mv: if self.args.no_mv:
raise Pebkac(403, "the rename/move feature is disabled in server config") raise Pebkac(403, "the rename/move feature is disabled in server config")
x = self.conn.hsrv.broker.ask("up2k.handle_mv", self.uname, vsrc, vdst) x = self.conn.hsrv.broker.ask("up2k.handle_mv", self.uname, self.ip, vsrc, vdst)
self.loud_reply(x.get(), status=201) self.loud_reply(x.get(), status=201)
return True return True

22
copyparty/smbd.py
View file

@ -187,6 +187,8 @@ class SMB(object):
debug('%s("%s", %s) %s @%s\033[K\033[0m', caller, vpath, str(a), perms, uname) debug('%s("%s", %s) %s @%s\033[K\033[0m', caller, vpath, str(a), perms, uname)
vfs, rem = self.asrv.vfs.get(vpath, uname, *perms) vfs, rem = self.asrv.vfs.get(vpath, uname, *perms)
if not vfs.realpath:
raise Exception("unmapped vfs")
return vfs, vfs.canonical(rem) return vfs, vfs.canonical(rem)
def _listdir(self, vpath: str, *a: Any, **ka: Any) -> list[str]: def _listdir(self, vpath: str, *a: Any, **ka: Any) -> list[str]:
@ -195,6 +197,8 @@ class SMB(object):
uname = self._uname() uname = self._uname()
# debug('listdir("%s", %s) @%s\033[K\033[0m', vpath, str(a), uname) # debug('listdir("%s", %s) @%s\033[K\033[0m', vpath, str(a), uname)
vfs, rem = self.asrv.vfs.get(vpath, uname, False, False) vfs, rem = self.asrv.vfs.get(vpath, uname, False, False)
if not vfs.realpath:
raise Exception("unmapped vfs")
_, vfs_ls, vfs_virt = vfs.ls( _, vfs_ls, vfs_virt = vfs.ls(
rem, uname, not self.args.no_scandir, [[False, False]] rem, uname, not self.args.no_scandir, [[False, False]]
) )
@ -240,7 +244,21 @@ class SMB(object):
xbu = vfs.flags.get("xbu") xbu = vfs.flags.get("xbu")
if xbu and not runhook( if xbu and not runhook(
self.nlog, xbu, ap, vpath, "", "", "", 0, 0, "1.7.6.2", 0, "" self.nlog,
None,
self.hub.up2k,
"xbu.smb",
xbu,
ap,
vpath,
"",
"",
"",
0,
0,
"1.7.6.2",
time.time(),
"",
): ):
yeet("blocked by xbu server config: " + vpath) yeet("blocked by xbu server config: " + vpath)
@ -297,7 +315,7 @@ class SMB(object):
t = "blocked rename (no-move-acc %s): /%s @%s" t = "blocked rename (no-move-acc %s): /%s @%s"
yeet(t % (vfs1.axs.umove, vp1, uname)) yeet(t % (vfs1.axs.umove, vp1, uname))
self.hub.up2k.handle_mv(uname, vp1, vp2) self.hub.up2k.handle_mv(uname, "1.7.6.2", vp1, vp2)
try: try:
bos.makedirs(ap2) bos.makedirs(ap2)
except: except:

20
copyparty/tftpd.py
View file

@ -244,6 +244,8 @@ class Tftpd(object):
debug('%s("%s", %s) %s\033[K\033[0m', caller, vpath, str(a), perms) debug('%s("%s", %s) %s\033[K\033[0m', caller, vpath, str(a), perms)
vfs, rem = self.asrv.vfs.get(vpath, "*", *perms) vfs, rem = self.asrv.vfs.get(vpath, "*", *perms)
if not vfs.realpath:
raise Exception("unmapped vfs")
return vfs, vfs.canonical(rem) return vfs, vfs.canonical(rem)
def _ls(self, vpath: str, raddress: str, rport: int, force=False) -> Any: def _ls(self, vpath: str, raddress: str, rport: int, force=False) -> Any:
@ -331,7 +333,21 @@ class Tftpd(object):
xbu = vfs.flags.get("xbu") xbu = vfs.flags.get("xbu")
if xbu and not runhook( if xbu and not runhook(
self.nlog, xbu, ap, vpath, "", "", "", 0, 0, "8.3.8.7", 0, "" self.nlog,
None,
self.hub.up2k,
"xbu.tftpd",
xbu,
ap,
vpath,
"",
"",
"",
0,
0,
"8.3.8.7",
time.time(),
"",
): ):
yeet("blocked by xbu server config: " + vpath) yeet("blocked by xbu server config: " + vpath)
@ -339,7 +355,7 @@ class Tftpd(object):
return self._ls(vpath, "", 0, True) return self._ls(vpath, "", 0, True)
if not a: if not a:
a = [self.args.iobuf] a = (self.args.iobuf,)
return open(ap, mode, *a, **ka) return open(ap, mode, *a, **ka)

268
copyparty/up2k.py
View file

@ -46,6 +46,7 @@ from .util import (
hidedir, hidedir,
humansize, humansize,
min_ex, min_ex,
pathmod,
quotep, quotep,
rand_name, rand_name,
ren_open, ren_open,
@ -165,6 +166,7 @@ class Up2k(object):
self.xiu_ptn = re.compile(r"(?:^|,)i([0-9]+)") self.xiu_ptn = re.compile(r"(?:^|,)i([0-9]+)")
self.xiu_busy = False # currently running hook self.xiu_busy = False # currently running hook
self.xiu_asleep = True # needs rescan_cond poke to schedule self self.xiu_asleep = True # needs rescan_cond poke to schedule self
self.fx_backlog: list[tuple[str, dict[str, str], str]] = []
self.cur: dict[str, "sqlite3.Cursor"] = {} self.cur: dict[str, "sqlite3.Cursor"] = {}
self.mem_cur = None self.mem_cur = None
@ -2544,7 +2546,7 @@ class Up2k(object):
if self.mutex.acquire(timeout=10): if self.mutex.acquire(timeout=10):
got_lock = True got_lock = True
with self.reg_mutex: with self.reg_mutex:
return self._handle_json(cj) ret = self._handle_json(cj)
else: else:
t = "cannot receive uploads right now;\nserver busy with {}.\nPlease wait; the client will retry..." t = "cannot receive uploads right now;\nserver busy with {}.\nPlease wait; the client will retry..."
raise Pebkac(503, t.format(self.blocked or "[unknown]")) raise Pebkac(503, t.format(self.blocked or "[unknown]"))
@ -2552,11 +2554,16 @@ class Up2k(object):
if not PY2: if not PY2:
raise raise
with self.mutex, self.reg_mutex: with self.mutex, self.reg_mutex:
return self._handle_json(cj) ret = self._handle_json(cj)
finally: finally:
if got_lock: if got_lock:
self.mutex.release() self.mutex.release()
if self.fx_backlog:
self.do_fx_backlog()
return ret
def _handle_json(self, cj: dict[str, Any]) -> dict[str, Any]: def _handle_json(self, cj: dict[str, Any]) -> dict[str, Any]:
ptop = cj["ptop"] ptop = cj["ptop"]
if not self.register_vpath(ptop, cj["vcfg"]): if not self.register_vpath(ptop, cj["vcfg"]):
@ -2758,28 +2765,43 @@ class Up2k(object):
job["name"] = rand_name( job["name"] = rand_name(
pdir, cj["name"], vfs.flags["nrand"] pdir, cj["name"], vfs.flags["nrand"]
) )
else:
job["name"] = self._untaken(pdir, cj, now)
dst = djoin(job["ptop"], job["prel"], job["name"]) dst = djoin(job["ptop"], job["prel"], job["name"])
xbu = vfs.flags.get("xbu") xbu = vfs.flags.get("xbu")
if xbu and not runhook( if xbu:
self.log, vp = djoin(job["vtop"], job["prel"], job["name"])
xbu, # type: ignore hr = runhook(
dst, self.log,
job["vtop"], None,
job["host"], self,
job["user"], "xbu.up2k.dupe",
self.asrv.vfs.get_perms(job["vtop"], job["user"]), xbu, # type: ignore
job["lmod"], dst,
job["size"], vp,
job["addr"], job["host"],
job["at"], job["user"],
"", self.asrv.vfs.get_perms(job["vtop"], job["user"]),
): job["lmod"],
t = "upload blocked by xbu server config: {}".format(dst) job["size"],
self.log(t, 1) job["addr"],
raise Pebkac(403, t) job["at"],
"",
)
if not hr:
t = "upload blocked by xbu server config: %s" % (dst,)
self.log(t, 1)
raise Pebkac(403, t)
if hr.get("reloc"):
x = pathmod(self.asrv.vfs, dst, vp, hr["reloc"])
if x:
pdir, _, job["name"], (vfs, rem) = x
dst = os.path.join(pdir, job["name"])
job["ptop"] = vfs.realpath
job["vtop"] = vfs.vpath
job["prel"] = rem
bos.makedirs(pdir)
job["name"] = self._untaken(pdir, cj, now)
if not self.args.nw: if not self.args.nw:
dvf: dict[str, Any] = vfs.flags dvf: dict[str, Any] = vfs.flags
@ -3142,6 +3164,9 @@ class Up2k(object):
with self.mutex, self.reg_mutex: with self.mutex, self.reg_mutex:
self._finish_upload(ptop, wark) self._finish_upload(ptop, wark)
if self.fx_backlog:
self.do_fx_backlog()
def _finish_upload(self, ptop: str, wark: str) -> None: def _finish_upload(self, ptop: str, wark: str) -> None:
"""mutex(main,reg) me""" """mutex(main,reg) me"""
try: try:
@ -3335,25 +3360,30 @@ class Up2k(object):
xau = False if skip_xau else vflags.get("xau") xau = False if skip_xau else vflags.get("xau")
dst = djoin(ptop, rd, fn) dst = djoin(ptop, rd, fn)
if xau and not runhook( if xau:
self.log, hr = runhook(
xau, self.log,
dst, None,
djoin(vtop, rd, fn), self,
host, "xau.up2k",
usr, xau,
self.asrv.vfs.get_perms(djoin(vtop, rd, fn), usr), dst,
int(ts), djoin(vtop, rd, fn),
sz, host,
ip, usr,
at or time.time(), self.asrv.vfs.get_perms(djoin(vtop, rd, fn), usr),
"", ts,
): sz,
t = "upload blocked by xau server config" ip,
self.log(t, 1) at or time.time(),
wunlink(self.log, dst, vflags) "",
self.registry[ptop].pop(wark, None) )
raise Pebkac(403, t) if not hr:
t = "upload blocked by xau server config"
self.log(t, 1)
wunlink(self.log, dst, vflags)
self.registry[ptop].pop(wark, None)
raise Pebkac(403, t)
xiu = vflags.get("xiu") xiu = vflags.get("xiu")
if xiu: if xiu:
@ -3537,6 +3567,9 @@ class Up2k(object):
if xbd: if xbd:
if not runhook( if not runhook(
self.log, self.log,
None,
self,
"xbd",
xbd, xbd,
abspath, abspath,
vpath, vpath,
@ -3546,7 +3579,7 @@ class Up2k(object):
stl.st_mtime, stl.st_mtime,
st.st_size, st.st_size,
ip, ip,
0, time.time(),
"", "",
): ):
t = "delete blocked by xbd server config: {}" t = "delete blocked by xbd server config: {}"
@ -3571,6 +3604,9 @@ class Up2k(object):
if xad: if xad:
runhook( runhook(
self.log, self.log,
None,
self,
"xad",
xad, xad,
abspath, abspath,
vpath, vpath,
@ -3580,7 +3616,7 @@ class Up2k(object):
stl.st_mtime, stl.st_mtime,
st.st_size, st.st_size,
ip, ip,
0, time.time(),
"", "",
) )
@ -3596,7 +3632,7 @@ class Up2k(object):
return n_files, ok + ok2, ng + ng2 return n_files, ok + ok2, ng + ng2
def handle_mv(self, uname: str, svp: str, dvp: str) -> str: def handle_mv(self, uname: str, ip: str, svp: str, dvp: str) -> str:
if svp == dvp or dvp.startswith(svp + "/"): if svp == dvp or dvp.startswith(svp + "/"):
raise Pebkac(400, "mv: cannot move parent into subfolder") raise Pebkac(400, "mv: cannot move parent into subfolder")
@ -3613,7 +3649,7 @@ class Up2k(object):
if stat.S_ISREG(st.st_mode) or stat.S_ISLNK(st.st_mode): if stat.S_ISREG(st.st_mode) or stat.S_ISLNK(st.st_mode):
with self.mutex: with self.mutex:
try: try:
ret = self._mv_file(uname, svp, dvp, curs) ret = self._mv_file(uname, ip, svp, dvp, curs)
finally: finally:
for v in curs: for v in curs:
v.connection.commit() v.connection.commit()
@ -3646,7 +3682,7 @@ class Up2k(object):
raise Pebkac(500, "mv: bug at {}, top {}".format(svpf, svp)) raise Pebkac(500, "mv: bug at {}, top {}".format(svpf, svp))
dvpf = dvp + svpf[len(svp) :] dvpf = dvp + svpf[len(svp) :]
self._mv_file(uname, svpf, dvpf, curs) self._mv_file(uname, ip, svpf, dvpf, curs)
finally: finally:
for v in curs: for v in curs:
v.connection.commit() v.connection.commit()
@ -3671,7 +3707,7 @@ class Up2k(object):
return "k" return "k"
def _mv_file( def _mv_file(
self, uname: str, svp: str, dvp: str, curs: set["sqlite3.Cursor"] self, uname: str, ip: str, svp: str, dvp: str, curs: set["sqlite3.Cursor"]
) -> str: ) -> str:
"""mutex(main) me; will mutex(reg)""" """mutex(main) me; will mutex(reg)"""
svn, srem = self.asrv.vfs.get(svp, uname, True, False, True) svn, srem = self.asrv.vfs.get(svp, uname, True, False, True)
@ -3705,21 +3741,27 @@ class Up2k(object):
except: except:
pass # broken symlink; keep as-is pass # broken symlink; keep as-is
ftime = stl.st_mtime
fsize = st.st_size
xbr = svn.flags.get("xbr") xbr = svn.flags.get("xbr")
xar = dvn.flags.get("xar") xar = dvn.flags.get("xar")
if xbr: if xbr:
if not runhook( if not runhook(
self.log, self.log,
None,
self,
"xbr",
xbr, xbr,
sabs, sabs,
svp, svp,
"", "",
uname, uname,
self.asrv.vfs.get_perms(svp, uname), self.asrv.vfs.get_perms(svp, uname),
stl.st_mtime, ftime,
st.st_size, fsize,
"", ip,
0, time.time(),
"", "",
): ):
t = "move blocked by xbr server config: {}".format(svp) t = "move blocked by xbr server config: {}".format(svp)
@ -3747,16 +3789,19 @@ class Up2k(object):
if xar: if xar:
runhook( runhook(
self.log, self.log,
None,
self,
"xar.ln",
xar, xar,
dabs, dabs,
dvp, dvp,
"", "",
uname, uname,
self.asrv.vfs.get_perms(dvp, uname), self.asrv.vfs.get_perms(dvp, uname),
0, ftime,
0, fsize,
"", ip,
0, time.time(),
"", "",
) )
@ -3765,13 +3810,6 @@ class Up2k(object):
c1, w, ftime_, fsize_, ip, at = self._find_from_vpath(svn.realpath, srem) c1, w, ftime_, fsize_, ip, at = self._find_from_vpath(svn.realpath, srem)
c2 = self.cur.get(dvn.realpath) c2 = self.cur.get(dvn.realpath)
if ftime_ is None:
ftime = stl.st_mtime
fsize = st.st_size
else:
ftime = ftime_
fsize = fsize_ or 0
has_dupes = False has_dupes = False
if w: if w:
assert c1 assert c1
@ -3779,7 +3817,9 @@ class Up2k(object):
self._copy_tags(c1, c2, w) self._copy_tags(c1, c2, w)
with self.reg_mutex: with self.reg_mutex:
has_dupes = self._forget_file(svn.realpath, srem, c1, w, is_xvol, fsize) has_dupes = self._forget_file(
svn.realpath, srem, c1, w, is_xvol, fsize_ or fsize
)
if not is_xvol: if not is_xvol:
has_dupes = self._relink(w, svn.realpath, srem, dabs) has_dupes = self._relink(w, svn.realpath, srem, dabs)
@ -3849,7 +3889,7 @@ class Up2k(object):
if is_link: if is_link:
try: try:
times = (int(time.time()), int(stl.st_mtime)) times = (int(time.time()), int(ftime))
bos.utime(dabs, times, False) bos.utime(dabs, times, False)
except: except:
pass pass
@ -3859,16 +3899,19 @@ class Up2k(object):
if xar: if xar:
runhook( runhook(
self.log, self.log,
None,
self,
"xar.mv",
xar, xar,
dabs, dabs,
dvp, dvp,
"", "",
uname, uname,
self.asrv.vfs.get_perms(dvp, uname), self.asrv.vfs.get_perms(dvp, uname),
0, ftime,
0, fsize,
"", ip,
0, time.time(),
"", "",
) )
@ -4152,23 +4195,35 @@ class Up2k(object):
xbu = self.flags[job["ptop"]].get("xbu") xbu = self.flags[job["ptop"]].get("xbu")
ap_chk = djoin(pdir, job["name"]) ap_chk = djoin(pdir, job["name"])
vp_chk = djoin(job["vtop"], job["prel"], job["name"]) vp_chk = djoin(job["vtop"], job["prel"], job["name"])
if xbu and not runhook( if xbu:
self.log, hr = runhook(
xbu, self.log,
ap_chk, None,
vp_chk, self,
job["host"], "xbu.up2k",
job["user"], xbu,
self.asrv.vfs.get_perms(vp_chk, job["user"]), ap_chk,
int(job["lmod"]), vp_chk,
job["size"], job["host"],
job["addr"], job["user"],
int(job["t0"]), self.asrv.vfs.get_perms(vp_chk, job["user"]),
"", job["lmod"],
): job["size"],
t = "upload blocked by xbu server config: {}".format(vp_chk) job["addr"],
self.log(t, 1) job["t0"],
raise Pebkac(403, t) "",
)
if not hr:
t = "upload blocked by xbu server config: {}".format(vp_chk)
self.log(t, 1)
raise Pebkac(403, t)
if hr.get("reloc"):
x = pathmod(self.asrv.vfs, ap_chk, vp_chk, hr["reloc"])
if x:
pdir, _, job["name"], (vfs, rem) = x
job["ptop"] = vfs.realpath
job["vtop"] = vfs.vpath
job["prel"] = rem
tnam = job["name"] + ".PARTIAL" tnam = job["name"] + ".PARTIAL"
if self.args.dotpart: if self.args.dotpart:
@ -4442,6 +4497,9 @@ class Up2k(object):
with self.rescan_cond: with self.rescan_cond:
self.rescan_cond.notify_all() self.rescan_cond.notify_all()
if self.fx_backlog:
self.do_fx_backlog()
return True return True
def hash_file( def hash_file(
@ -4473,6 +4531,48 @@ class Up2k(object):
self.hashq.put(zt) self.hashq.put(zt)
self.n_hashq += 1 self.n_hashq += 1
def do_fx_backlog(self):
with self.mutex, self.reg_mutex:
todo = self.fx_backlog
self.fx_backlog = []
for act, hr, req_vp in todo:
self.hook_fx(act, hr, req_vp)
def hook_fx(self, act: str, hr: dict[str, str], req_vp: str) -> None:
bad = [k for k in hr if k != "vp"]
if bad:
t = "got unsupported key in %s from hook: %s"
raise Exception(t % (act, bad))
for fvp in hr.get("vp") or []:
# expect vpath including filename; either absolute
# or relative to the client's vpath (request url)
if fvp.startswith("/"):
fvp, fn = vsplit(fvp[1:])
fvp = "/" + fvp
else:
fvp, fn = vsplit(fvp)
x = pathmod(self.asrv.vfs, "", req_vp, {"vp": fvp, "fn": fn})
if not x:
t = "hook_fx(%s): failed to resolve %s based on %s"
self.log(t % (act, fvp, req_vp))
continue
ap, rd, fn, (vn, rem) = x
vp = vjoin(rd, fn)
if not vp:
raise Exception("hook_fx: blank vp from pathmod")
if act == "idx":
rd = rd[len(vn.vpath) :].strip("/")
self.hash_file(
vn.realpath, vn.vpath, vn.flags, rd, fn, "", time.time(), "", True
)
if act == "del":
self._handle_rm(LEELOO_DALLAS, "", vp, [], False, False)
def shutdown(self) -> None: def shutdown(self) -> None:
self.stop = True self.stop = True

131
copyparty/util.py
View file

@ -146,6 +146,8 @@ if TYPE_CHECKING:
import magic import magic
from .authsrv import VFS from .authsrv import VFS
from .broker_util import BrokerCli
from .up2k import Up2k
FAKE_MP = False FAKE_MP = False
@ -2117,6 +2119,72 @@ def ujoin(rd: str, fn: str) -> str:
return rd or fn return rd or fn
def log_reloc(
log: "NamedLogger",
re: dict[str, str],
pm: tuple[str, str, str, tuple["VFS", str]],
ap: str,
vp: str,
fn: str,
vn: "VFS",
rem: str,
) -> None:
nap, nvp, nfn, (nvn, nrem) = pm
t = "reloc %s:\nold ap [%s]\nnew ap [%s\033[36m/%s\033[0m]\nold vp [%s]\nnew vp [%s\033[36m/%s\033[0m]\nold fn [%s]\nnew fn [%s]\nold vfs [%s]\nnew vfs [%s]\nold rem [%s]\nnew rem [%s]"
log(t % (re, ap, nap, nfn, vp, nvp, nfn, fn, nfn, vn.vpath, nvn.vpath, rem, nrem))
def pathmod(
vfs: "VFS", ap: str, vp: str, mod: dict[str, str]
) -> Optional[tuple[str, str, str, tuple["VFS", str]]]:
# vfs: authsrv.vfs
# ap: original abspath to a file
# vp: original urlpath to a file
# mod: modification (ap/vp/fn)
nvp = "\n" # new vpath
ap = os.path.dirname(ap)
vp, fn = vsplit(vp)
if mod.get("fn"):
fn = mod["fn"]
nvp = vp
for ref, k in ((ap, "ap"), (vp, "vp")):
if k not in mod:
continue
ms = mod[k].replace(os.sep, "/")
if ms.startswith("/"):
np = ms
elif k == "vp":
np = undot(vjoin(ref, ms))
else:
np = os.path.abspath(os.path.join(ref, ms))
if k == "vp":
nvp = np.lstrip("/")
continue
# try to map abspath to vpath
np = np.replace("/", os.sep)
for vn_ap, vn in vfs.all_aps:
if not np.startswith(vn_ap):
continue
zs = np[len(vn_ap) :].replace(os.sep, "/")
nvp = vjoin(vn.vpath, zs)
break
if nvp == "\n":
return None
vn, rem = vfs.get(nvp, "*", False, False)
if not vn.realpath:
raise Exception("unmapped vfs")
ap = vn.canonical(rem)
return ap, nvp, fn, (vn, rem)
def _w8dec2(txt: bytes) -> str: def _w8dec2(txt: bytes) -> str:
"""decodes filesystem-bytes to wtf8""" """decodes filesystem-bytes to wtf8"""
return surrogateescape.decodefilename(txt) return surrogateescape.decodefilename(txt)
@ -3130,6 +3198,7 @@ def runihook(
def _runhook( def _runhook(
log: Optional["NamedLogger"], log: Optional["NamedLogger"],
src: str,
cmd: str, cmd: str,
ap: str, ap: str,
vp: str, vp: str,
@ -3141,14 +3210,16 @@ def _runhook(
ip: str, ip: str,
at: float, at: float,
txt: str, txt: str,
) -> bool: ) -> dict[str, Any]:
ret = {"rc": 0}
areq, chk, fork, jtxt, wait, sp_ka, acmd = _parsehook(log, cmd) areq, chk, fork, jtxt, wait, sp_ka, acmd = _parsehook(log, cmd)
if areq: if areq:
for ch in areq: for ch in areq:
if ch not in perms: if ch not in perms:
t = "user %s not allowed to run hook %s; need perms %s, have %s" t = "user %s not allowed to run hook %s; need perms %s, have %s"
log(t % (uname, cmd, areq, perms)) if log:
return True # fallthrough to next hook log(t % (uname, cmd, areq, perms))
return ret # fallthrough to next hook
if jtxt: if jtxt:
ja = { ja = {
"ap": ap, "ap": ap,
@ -3160,6 +3231,7 @@ def _runhook(
"host": host, "host": host,
"user": uname, "user": uname,
"perms": perms, "perms": perms,
"src": src,
"txt": txt, "txt": txt,
} }
arg = json.dumps(ja) arg = json.dumps(ja)
@ -3178,18 +3250,34 @@ def _runhook(
else: else:
rc, v, err = runcmd(bcmd, **sp_ka) # type: ignore rc, v, err = runcmd(bcmd, **sp_ka) # type: ignore
if chk and rc: if chk and rc:
ret["rc"] = rc
retchk(rc, bcmd, err, log, 5) retchk(rc, bcmd, err, log, 5)
return False else:
try:
ret = json.loads(v)
except:
ret = {}
try:
if "stdout" not in ret:
ret["stdout"] = v
if "rc" not in ret:
ret["rc"] = rc
except:
ret = {"rc": rc, "stdout": v}
wait -= time.time() - t0 wait -= time.time() - t0
if wait > 0: if wait > 0:
time.sleep(wait) time.sleep(wait)
return True return ret
def runhook( def runhook(
log: Optional["NamedLogger"], log: Optional["NamedLogger"],
broker: Optional["BrokerCli"],
up2k: Optional["Up2k"],
src: str,
cmds: list[str], cmds: list[str],
ap: str, ap: str,
vp: str, vp: str,
@ -3201,19 +3289,42 @@ def runhook(
ip: str, ip: str,
at: float, at: float,
txt: str, txt: str,
) -> bool: ) -> dict[str, Any]:
assert broker or up2k
asrv = (broker or up2k).asrv
args = (broker or up2k).args
vp = vp.replace("\\", "/") vp = vp.replace("\\", "/")
ret = {"rc": 0}
for cmd in cmds: for cmd in cmds:
try: try:
if not _runhook(log, cmd, ap, vp, host, uname, perms, mt, sz, ip, at, txt): hr = _runhook(
return False log, src, cmd, ap, vp, host, uname, perms, mt, sz, ip, at, txt
)
if log and args.hook_v:
log("hook(%s) [%s] => \033[32m%s" % (src, cmd, hr), 6)
if not hr:
return {}
for k, v in hr.items():
if k in ("idx", "del") and v:
if broker:
broker.say("up2k.hook_fx", k, v, vp)
else:
up2k.fx_backlog.append((k, v, vp))
elif k == "reloc" and v:
# idk, just take the last one ig
ret["reloc"] = v
elif k in ret:
if k == "rc" and v:
ret[k] = v
else:
ret[k] = v
except Exception as ex: except Exception as ex:
(log or print)("hook: {}".format(ex)) (log or print)("hook: {}".format(ex))
if ",c," in "," + cmd: if ",c," in "," + cmd:
return False return {}
break break
return True return ret
def loadpy(ap: str, hot: bool) -> Any: def loadpy(ap: str, hot: bool) -> Any:

28
docs/devnotes.md
View file

@ -12,6 +12,8 @@
* [write](#write) * [write](#write)
* [admin](#admin) * [admin](#admin)
* [general](#general) * [general](#general)
* [event hooks](#event-hooks) - on writing your own [hooks](../README.md#event-hooks)
* [hook effects](#hook-effects) - hooks can cause intentional side-effects
* [assumptions](#assumptions) * [assumptions](#assumptions)
* [mdns](#mdns) * [mdns](#mdns)
* [sfx repack](#sfx-repack) - reduce the size of an sfx by removing features * [sfx repack](#sfx-repack) - reduce the size of an sfx by removing features
@ -204,6 +206,32 @@ upload modifiers:
| GET | `?pw=x` | logout | | GET | `?pw=x` | logout |
# event hooks
on writing your own [hooks](../README.md#event-hooks)
## hook effects
hooks can cause intentional side-effects, such as redirecting an upload into another location, or creating+indexing additional files, or deleting existing files, by returning json on stdout
* `reloc` can redirect uploads before/after uploading has finished, based on filename, extension, file contents, uploader ip/name etc.
* `idx` informs copyparty about a new file to index as a consequence of this upload
* `del` tells copyparty to delete an unrelated file by vpath
for these to take effect, the hook must be defined with the `c1` flag; see example [reloc-by-ext](https://github.com/9001/copyparty/blob/hovudstraum/bin/hooks/reloc-by-ext.py)
a subset of effect types are available for a subset of hook types,
* most hook types (xbu/xau/xbr/xar/xbd/xad/xm) support `idx` and `del` for all http protocols (up2k / basic-uploader / webdav), but not ftp/tftp/smb
* most hook types will abort/reject the action if the hook returns nonzero, assuming flag `c` is given, see examples [reject-extension](https://github.com/9001/copyparty/blob/hovudstraum/bin/hooks/reject-extension.py) and [reject-mimetype](https://github.com/9001/copyparty/blob/hovudstraum/bin/hooks/reject-mimetype.py)
* `xbu` supports `reloc` for all http protocols (up2k / basic-uploader / webdav), but not ftp/tftp/smb
* `xau` supports `reloc` for basic-uploader / webdav only, not up2k or ftp/tftp/smb
* so clients like sharex are supported, but not dragdrop into browser
to trigger indexing of files `/foo/1.txt` and `/foo/bar/2.txt`, a hook can `print(json.dumps({"idx":{"vp":["/foo/1.txt","/foo/bar/2.txt"]}}))` (and replace "idx" with "del" to delete instead)
* note: paths starting with `/` are absolute URLs, but you can also do `../3.txt` relative to the destination folder of each uploaded file
# assumptions # assumptions
## mdns ## mdns

4
docs/versus.md
View file

@ -175,6 +175,7 @@ symbol legend,
| ┗ randomize filename | █ | | | | | | | █ | █ | | | | | | ┗ randomize filename | █ | | | | | | | █ | █ | | | | |
| ┗ mimetype reject-list | ╱ | | | | | | | | • | ╱ | | ╱ | • | | ┗ mimetype reject-list | ╱ | | | | | | | | • | ╱ | | ╱ | • |
| ┗ extension reject-list | ╱ | | | | | | | █ | • | ╱ | | ╱ | • | | ┗ extension reject-list | ╱ | | | | | | | █ | • | ╱ | | ╱ | • |
| ┗ upload routing | █ | | | | | | | | | | | | |
| checksums provided | | | | █ | █ | | | | █ | ╱ | | | | | checksums provided | | | | █ | █ | | | | █ | ╱ | | | |
| cloud storage backend | ╱ | ╱ | ╱ | █ | █ | █ | ╱ | | | ╱ | █ | █ | ╱ | | cloud storage backend | ╱ | ╱ | ╱ | █ | █ | █ | ╱ | | | ╱ | █ | █ | ╱ |
@ -188,6 +189,9 @@ symbol legend,
* `race the beam` = files can be downloaded while they're still uploading; downloaders are slowed down such that the uploader is always ahead * `race the beam` = files can be downloaded while they're still uploading; downloaders are slowed down such that the uploader is always ahead
* `upload routing` = depending on filetype / contents / uploader etc., the file can be redirected to another location or otherwise transformed; mitigates limitations such as [sharex#3992](https://github.com/ShareX/ShareX/issues/3992)
* copyparty example: [reloc-by-ext](https://github.com/9001/copyparty/tree/hovudstraum/bin/hooks#before-upload)
* `checksums provided` = when downloading a file from the server, the file's checksum is provided for verification client-side * `checksums provided` = when downloading a file from the server, the file's checksum is provided for verification client-side
* `cloud storage backend` = able to serve files from (and write to) s3 or similar cloud services; `╱` means the software can do this with some help from `rclone mount` as a bridge * `cloud storage backend` = able to serve files from (and write to) s3 or similar cloud services; `╱` means the software can do this with some help from `rclone mount` as a bridge

111
tests/test_hooks.py Normal file
View file

@ -0,0 +1,111 @@
#!/usr/bin/env python3
# coding: utf-8
from __future__ import print_function, unicode_literals
import os
import shutil
import tempfile
import unittest
from copyparty.authsrv import AuthSrv
from copyparty.httpcli import HttpCli
from tests import util as tu
from tests.util import Cfg
def hdr(query):
h = "GET /{} HTTP/1.1\r\nPW: o\r\nConnection: close\r\n\r\n"
return h.format(query).encode("utf-8")
class TestHooks(unittest.TestCase):
def setUp(self):
self.td = tu.get_ramdisk()
def tearDown(self):
os.chdir(tempfile.gettempdir())
shutil.rmtree(self.td)
def reset(self):
td = os.path.join(self.td, "vfs")
if os.path.exists(td):
shutil.rmtree(td)
os.mkdir(td)
os.chdir(td)
return td
def test(self):
vcfg = ["a/b/c/d:c/d:A", "a:a:r"]
scenarios = (
('{"vp":"x/y"}', "c/d/a.png", "c/d/x/y/a.png"),
('{"vp":"x/y"}', "c/d/e/a.png", "c/d/e/x/y/a.png"),
('{"vp":"../x/y"}', "c/d/e/a.png", "c/d/x/y/a.png"),
('{"ap":"x/y"}', "c/d/a.png", "c/d/x/y/a.png"),
('{"ap":"x/y"}', "c/d/e/a.png", "c/d/e/x/y/a.png"),
('{"ap":"../x/y"}', "c/d/e/a.png", "c/d/x/y/a.png"),
('{"ap":"../x/y"}', "c/d/a.png", "a/b/c/x/y/a.png"),
('{"fn":"b.png"}', "c/d/a.png", "c/d/b.png"),
('{"vp":"x","fn":"b.png"}', "c/d/a.png", "c/d/x/b.png"),
)
for x in scenarios:
print("\n\n\n", x)
hooktxt, url_up, url_dl = x
for hooktype in ("xbu", "xau"):
for upfun in (self.put, self.bup):
self.reset()
self.makehook("""print('{"reloc":%s}')""" % (hooktxt,))
ka = {hooktype: ["j,c1,h.py"]}
self.args = Cfg(v=vcfg, a=["o:o"], e2d=True, **ka)
self.asrv = AuthSrv(self.args, self.log)
h, b = upfun(url_up)
self.assertIn("201 Created", h)
h, b = self.curl(url_dl)
self.assertEqual(b, "ok %s\n" % (url_up))
def makehook(self, hs):
with open("h.py", "wb") as f:
f.write(hs.encode("utf-8"))
def put(self, url):
buf = "PUT /{0} HTTP/1.1\r\nPW: o\r\nConnection: close\r\nContent-Length: {1}\r\n\r\nok {0}\n"
buf = buf.format(url, len(url) + 4).encode("utf-8")
print("PUT -->", buf)
conn = tu.VHttpConn(self.args, self.asrv, self.log, buf)
HttpCli(conn).run()
ret = conn.s._reply.decode("utf-8").split("\r\n\r\n", 1)
print("PUT <--", ret)
return ret
def bup(self, url):
hdr = "POST /%s HTTP/1.1\r\nPW: o\r\nConnection: close\r\nContent-Type: multipart/form-data; boundary=XD\r\nContent-Length: %d\r\n\r\n"
bdy = '--XD\r\nContent-Disposition: form-data; name="act"\r\n\r\nbput\r\n--XD\r\nContent-Disposition: form-data; name="f"; filename="%s"\r\n\r\n'
ftr = "\r\n--XD--\r\n"
try:
url, fn = url.rsplit("/", 1)
except:
fn = url
url = ""
buf = (bdy % (fn,) + "ok %s/%s\n" % (url, fn) + ftr).encode("utf-8")
buf = (hdr % (url, len(buf))).encode("utf-8") + buf
print("PoST -->", buf)
conn = tu.VHttpConn(self.args, self.asrv, self.log, buf)
HttpCli(conn).run()
ret = conn.s._reply.decode("utf-8").split("\r\n\r\n", 1)
print("POST <--", ret)
return ret
def curl(self, url, binary=False):
conn = tu.VHttpConn(self.args, self.asrv, self.log, hdr(url))
HttpCli(conn).run()
if binary:
h, b = conn.s._reply.split(b"\r\n\r\n", 1)
return [h.decode("utf-8"), b]
return conn.s._reply.decode("utf-8").split("\r\n\r\n", 1)
def log(self, src, msg, c=0):
print(msg)

17
tests/util.py
View file

@ -68,6 +68,13 @@ def chkcmd(argv):
def get_ramdisk(): def get_ramdisk():
def subdir(top): def subdir(top):
for d in os.listdir(top):
if not d.startswith("cptd-"):
continue
p = os.path.join(top, d)
st = os.stat(p)
if time.time() - st.st_mtime > 300:
shutil.rmtree(p)
ret = os.path.join(top, "cptd-{}".format(os.getpid())) ret = os.path.join(top, "cptd-{}".format(os.getpid()))
shutil.rmtree(ret, True) shutil.rmtree(ret, True)
os.mkdir(ret) os.mkdir(ret)
@ -111,10 +118,10 @@ class Cfg(Namespace):
def __init__(self, a=None, v=None, c=None, **ka0): def __init__(self, a=None, v=None, c=None, **ka0):
ka = {} ka = {}
ex = "daw dav_auth dav_inf dav_mac dav_rt e2d e2ds e2dsa e2t e2ts e2tsr e2v e2vu e2vp early_ban ed emp exp force_js getmod grid gsel hardlink ih ihead magic never_symlink nid nih no_acode no_athumb no_dav no_dedup no_del no_dupe no_lifetime no_logues no_mv no_pipe no_poll no_readme no_robots no_sb_md no_sb_lg no_scandir no_tarcmp no_thumb no_vthumb no_zip nrand nw og og_no_head og_s_title q rand smb srch_dbg stats uqe vague_403 vc ver xdev xlink xvol" ex = "daw dav_auth dav_inf dav_mac dav_rt e2d e2ds e2dsa e2t e2ts e2tsr e2v e2vu e2vp early_ban ed emp exp force_js getmod grid gsel hardlink ih ihead magic never_symlink nid nih no_acode no_athumb no_dav no_dedup no_del no_dupe no_lifetime no_logues no_mv no_pipe no_poll no_readme no_robots no_sb_md no_sb_lg no_scandir no_tarcmp no_thumb no_vthumb no_zip nrand nw og og_no_head og_s_title q rand smb srch_dbg stats uqe vague_403 vc ver write_uplog xdev xlink xvol"
ka.update(**{k: False for k in ex.split()}) ka.update(**{k: False for k in ex.split()})
ex = "dotpart dotsrch no_dhash no_fastboot no_rescan no_sendfile no_snap no_voldump re_dhash plain_ip" ex = "dotpart dotsrch hook_v no_dhash no_fastboot no_rescan no_sendfile no_snap no_voldump re_dhash plain_ip"
ka.update(**{k: True for k in ex.split()}) ka.update(**{k: True for k in ex.split()})
ex = "ah_cli ah_gen css_browser hist js_browser mime mimes no_forget no_hash no_idx nonsus_urls og_tpl og_ua" ex = "ah_cli ah_gen css_browser hist js_browser mime mimes no_forget no_hash no_idx nonsus_urls og_tpl og_ua"
@ -178,6 +185,10 @@ class Cfg(Namespace):
class NullBroker(object): class NullBroker(object):
def __init__(self, args, asrv):
self.args = args
self.asrv = asrv
def say(self, *args): def say(self, *args):
pass pass
@ -213,7 +224,7 @@ class VHttpSrv(object):
self.asrv = asrv self.asrv = asrv
self.log = log self.log = log
self.broker = NullBroker() self.broker = NullBroker(args, asrv)
self.prism = None self.prism = None
self.bans = {} self.bans = {}
self.nreq = 0 self.nreq = 0