mirror of
https://github.com/9001/copyparty.git
synced 2025-08-16 08:32:13 -06:00
add hook side-effects; closes #86
hooks can now interrupt or redirect actions, and initiate related actions, by printing json on stdout with commands mainly to mitigate limitations such as sharex/sharex#3992 xbr/xau can redirect uploads to other destinations with `reloc` and most hooks can initiate indexing or deletion of additional files by giving a list of vpaths in json-keys `idx` or `del` there are limitations; * xbu/xau effects don't apply to ftp, tftp, smb * xau will intentionally fail if a reloc destination exists * xau effects do not apply to up2k also provides more details for hooks: * xbu/xau: basic-uploader vpath with filename * xbr/xar: add client ip
This commit is contained in:
ed
parent
20669c73d3
commit
6c94a63f1c
|
|
@ -1313,6 +1313,8 @@ you can set hooks before and/or after an event happens, and currently you can ho
|
|||
|
||||
there's a bunch of flags and stuff, see `--help-hooks`
|
||||
|
||||
if you want to write your own hooks, see [devnotes](./docs/devnotes.md#event-hooks)
|
||||
|
||||
|
||||
### upload events
|
||||
|
||||
|
|
|
|||
|
|
@ -41,8 +41,8 @@ parameters explained,
|
|||
t10 = abort download and continue if it takes longer than 10sec
|
||||
|
||||
example usage as a volflag (per-volume config):
|
||||
-v srv/inc:inc:r:rw,ed:xau=j,t10,bin/hooks/into-the-cache-it-goes.py
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
-v srv/inc:inc:r:rw,ed:c,xau=j,t10,bin/hooks/into-the-cache-it-goes.py
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
(share filesystem-path srv/inc as volume /inc,
|
||||
readable by everyone, read-write for user 'ed',
|
||||
|
|
|
|||
94
bin/hooks/reloc-by-ext.py
Normal file
94
bin/hooks/reloc-by-ext.py
Normal file
|
|
@ -0,0 +1,94 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
|
||||
_ = r"""
|
||||
relocate/redirect incoming uploads according to file extension
|
||||
|
||||
example usage as global config:
|
||||
--xbu j,c1,bin/hooks/reloc-by-ext.py
|
||||
|
||||
parameters explained,
|
||||
xbu = execute before upload
|
||||
j = this hook needs upload information as json (not just the filename)
|
||||
c1 = this hook returns json on stdout, so tell copyparty to read that
|
||||
|
||||
example usage as a volflag (per-volume config):
|
||||
-v srv/inc:inc:r:rw,ed:c,xbu=j,c1,bin/hooks/reloc-by-ext.py
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
(share filesystem-path srv/inc as volume /inc,
|
||||
readable by everyone, read-write for user 'ed',
|
||||
running this plugin on all uploads with the params explained above)
|
||||
|
||||
example usage as a volflag in a copyparty config file:
|
||||
[/inc]
|
||||
srv/inc
|
||||
accs:
|
||||
r: *
|
||||
rw: ed
|
||||
flags:
|
||||
xbu: j,c1,bin/hooks/reloc-by-ext.py
|
||||
|
||||
note: this only works with the basic uploader (sharex and such),
|
||||
does not work with up2k / dragdrop into browser
|
||||
|
||||
note: this could also work as an xau hook (after-upload), but
|
||||
because it doesn't need to read the file contents its better
|
||||
as xbu (before-upload) since that's safer / less buggy
|
||||
"""
|
||||
|
||||
|
||||
PICS = "avif bmp gif heic heif jpeg jpg jxl png psd qoi tga tif tiff webp"
|
||||
VIDS = "3gp asf avi flv mkv mov mp4 mpeg mpeg2 mpegts mpg mpg2 nut ogm ogv rm ts vob webm wmv"
|
||||
MUSIC = "aac aif aiff alac amr ape dfpwm flac m4a mp3 ogg opus ra tak tta wav wma wv"
|
||||
|
||||
|
||||
def main():
|
||||
inf = json.loads(sys.argv[1])
|
||||
vdir, fn = os.path.split(inf["vp"])
|
||||
|
||||
try:
|
||||
fn, ext = fn.rsplit(".", 1)
|
||||
except:
|
||||
# no file extension; abort
|
||||
return
|
||||
|
||||
ext = ext.lower()
|
||||
|
||||
##
|
||||
## some example actions to take; pick one by
|
||||
## selecting it inside the print at the end:
|
||||
##
|
||||
|
||||
# create a subfolder named after the filetype and move it into there
|
||||
into_subfolder = {"vp": ext}
|
||||
|
||||
# move it into a toplevel folder named after the filetype
|
||||
into_toplevel = {"vp": "/" + ext}
|
||||
|
||||
# move it into a filetype-named folder next to the target folder
|
||||
into_sibling = {"vp": "../" + ext}
|
||||
|
||||
# move images into "/just/pics", vids into "/just/vids",
|
||||
# music into "/just/tunes", and anything else as-is
|
||||
if ext in PICS.split():
|
||||
by_category = {"vp": "/just/pics"}
|
||||
elif ext in VIDS.split():
|
||||
by_category = {"vp": "/just/vids"}
|
||||
elif ext in MUSIC.split():
|
||||
by_category = {"vp": "/just/tunes"}
|
||||
else:
|
||||
by_category = {}
|
||||
|
||||
# now choose the effect to apply; can be any of these:
|
||||
# into_subfolder into_toplevel into_sibling by_category
|
||||
effect = into_subfolder
|
||||
print(json.dumps({"reloc": effect}))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -704,6 +704,11 @@ def get_sects():
|
|||
\033[36mxban\033[0m can be used to overrule / cancel a user ban event;
|
||||
if the program returns 0 (true/OK) then the ban will NOT happen
|
||||
|
||||
effects can be used to redirect uploads into other
|
||||
locations, and to delete or index other files based
|
||||
on new uploads, but with certain limitations. See
|
||||
bin/hooks/reloc* and docs/devnotes.md#hook-effects
|
||||
|
||||
except for \033[36mxm\033[0m, only one hook / one action can run at a time,
|
||||
so it's recommended to use the \033[36mf\033[0m flag unless you really need
|
||||
to wait for the hook to finish before continuing (without \033[36mf\033[0m
|
||||
|
|
@ -1132,6 +1137,7 @@ def add_hooks(ap):
|
|||
ap2.add_argument("--xad", metavar="CMD", type=u, action="append", help="execute \033[33mCMD\033[0m after a file delete")
|
||||
ap2.add_argument("--xm", metavar="CMD", type=u, action="append", help="execute \033[33mCMD\033[0m on message")
|
||||
ap2.add_argument("--xban", metavar="CMD", type=u, action="append", help="execute \033[33mCMD\033[0m if someone gets banned (pw/404/403/url)")
|
||||
ap2.add_argument("--hook-v", action="store_true", help="verbose hooks")
|
||||
|
||||
|
||||
def add_stats(ap):
|
||||
|
|
|
|||
|
|
@ -521,8 +521,8 @@ class VFS(object):
|
|||
t = "{} has no {} in [{}] => [{}] => [{}]"
|
||||
self.log("vfs", t.format(uname, msg, vpath, cvpath, ap), 6)
|
||||
|
||||
t = 'you don\'t have %s-access in "/%s"'
|
||||
raise Pebkac(err, t % (msg, cvpath))
|
||||
t = 'you don\'t have %s-access in "/%s" or below "/%s"'
|
||||
raise Pebkac(err, t % (msg, cvpath, vn.vpath))
|
||||
|
||||
return vn, rem
|
||||
|
||||
|
|
@ -1898,7 +1898,7 @@ class AuthSrv(object):
|
|||
self.log(t.format(vol.vpath), 1)
|
||||
del vol.flags["lifetime"]
|
||||
|
||||
needs_e2d = [x for x in hooks if x != "xm"]
|
||||
needs_e2d = [x for x in hooks if x in ("xau", "xiu")]
|
||||
drop = [x for x in needs_e2d if vol.flags.get(x)]
|
||||
if drop:
|
||||
t = 'removing [{}] from volume "/{}" because e2d is disabled'
|
||||
|
|
|
|||
|
|
@ -353,7 +353,7 @@ class FtpFs(AbstractedFS):
|
|||
svp = join(self.cwd, src).lstrip("/")
|
||||
dvp = join(self.cwd, dst).lstrip("/")
|
||||
try:
|
||||
self.hub.up2k.handle_mv(self.uname, svp, dvp)
|
||||
self.hub.up2k.handle_mv(self.uname, self.h.cli_ip, svp, dvp)
|
||||
except Exception as ex:
|
||||
raise FSE(str(ex))
|
||||
|
||||
|
|
@ -471,6 +471,9 @@ class FtpHandler(FTPHandler):
|
|||
xbu = vfs.flags.get("xbu")
|
||||
if xbu and not runhook(
|
||||
None,
|
||||
None,
|
||||
self.hub.up2k,
|
||||
"xbu.ftpd",
|
||||
xbu,
|
||||
ap,
|
||||
vp,
|
||||
|
|
@ -480,7 +483,7 @@ class FtpHandler(FTPHandler):
|
|||
0,
|
||||
0,
|
||||
self.cli_ip,
|
||||
0,
|
||||
time.time(),
|
||||
"",
|
||||
):
|
||||
raise FSE("Upload blocked by xbu server config")
|
||||
|
|
|
|||
|
|
@ -73,7 +73,9 @@ from .util import (
|
|||
humansize,
|
||||
ipnorm,
|
||||
loadpy,
|
||||
log_reloc,
|
||||
min_ex,
|
||||
pathmod,
|
||||
quotep,
|
||||
rand_name,
|
||||
read_header,
|
||||
|
|
@ -695,6 +697,9 @@ class HttpCli(object):
|
|||
xban = self.vn.flags.get("xban")
|
||||
if not xban or not runhook(
|
||||
self.log,
|
||||
self.conn.hsrv.broker,
|
||||
None,
|
||||
"xban",
|
||||
xban,
|
||||
self.vn.canonical(self.rem),
|
||||
self.vpath,
|
||||
|
|
@ -1172,7 +1177,8 @@ class HttpCli(object):
|
|||
if self.args.no_dav:
|
||||
raise Pebkac(405, "WebDAV is disabled in server config")
|
||||
|
||||
vn, rem = self.asrv.vfs.get(self.vpath, self.uname, False, False, err=401)
|
||||
vn = self.vn
|
||||
rem = self.rem
|
||||
tap = vn.canonical(rem)
|
||||
|
||||
if "davauth" in vn.flags and self.uname == "*":
|
||||
|
|
@ -1556,8 +1562,8 @@ class HttpCli(object):
|
|||
self.log("PUT %s @%s" % (self.req, self.uname))
|
||||
|
||||
if not self.can_write:
|
||||
t = "user {} does not have write-access here"
|
||||
raise Pebkac(403, t.format(self.uname))
|
||||
t = "user %s does not have write-access under /%s"
|
||||
raise Pebkac(403, t % (self.uname, self.vn.vpath))
|
||||
|
||||
if not self.args.no_dav and self._applesan():
|
||||
return self.headers.get("content-length") == "0"
|
||||
|
|
@ -1632,6 +1638,9 @@ class HttpCli(object):
|
|||
if xm:
|
||||
runhook(
|
||||
self.log,
|
||||
self.conn.hsrv.broker,
|
||||
None,
|
||||
"xm",
|
||||
xm,
|
||||
self.vn.canonical(self.rem),
|
||||
self.vpath,
|
||||
|
|
@ -1780,11 +1789,15 @@ class HttpCli(object):
|
|||
|
||||
if xbu:
|
||||
at = time.time() - lifetime
|
||||
if not runhook(
|
||||
vp = vjoin(self.vpath, fn) if nameless else self.vpath
|
||||
hr = runhook(
|
||||
self.log,
|
||||
self.conn.hsrv.broker,
|
||||
None,
|
||||
"xbu.http.dump",
|
||||
xbu,
|
||||
path,
|
||||
self.vpath,
|
||||
vp,
|
||||
self.host,
|
||||
self.uname,
|
||||
self.asrv.vfs.get_perms(self.vpath, self.uname),
|
||||
|
|
@ -1793,10 +1806,25 @@ class HttpCli(object):
|
|||
self.ip,
|
||||
at,
|
||||
"",
|
||||
):
|
||||
)
|
||||
if not hr:
|
||||
t = "upload blocked by xbu server config"
|
||||
self.log(t, 1)
|
||||
raise Pebkac(403, t)
|
||||
if hr.get("reloc"):
|
||||
x = pathmod(self.asrv.vfs, path, vp, hr["reloc"])
|
||||
if x:
|
||||
if self.args.hook_v:
|
||||
log_reloc(self.log, hr["reloc"], x, path, vp, fn, vfs, rem)
|
||||
fdir, self.vpath, fn, (vfs, rem) = x
|
||||
if self.args.nw:
|
||||
fn = os.devnull
|
||||
else:
|
||||
bos.makedirs(fdir)
|
||||
path = os.path.join(fdir, fn)
|
||||
if not nameless:
|
||||
self.vpath = vjoin(self.vpath, fn)
|
||||
params["fdir"] = fdir
|
||||
|
||||
if is_put and not (self.args.no_dav or self.args.nw) and bos.path.exists(path):
|
||||
# allow overwrite if...
|
||||
|
|
@ -1871,24 +1899,45 @@ class HttpCli(object):
|
|||
fn = fn2
|
||||
path = path2
|
||||
|
||||
if xau and not runhook(
|
||||
self.log,
|
||||
xau,
|
||||
path,
|
||||
self.vpath,
|
||||
self.host,
|
||||
self.uname,
|
||||
self.asrv.vfs.get_perms(self.vpath, self.uname),
|
||||
mt,
|
||||
post_sz,
|
||||
self.ip,
|
||||
at,
|
||||
"",
|
||||
):
|
||||
t = "upload blocked by xau server config"
|
||||
self.log(t, 1)
|
||||
wunlink(self.log, path, vfs.flags)
|
||||
raise Pebkac(403, t)
|
||||
if xau:
|
||||
vp = vjoin(self.vpath, fn) if nameless else self.vpath
|
||||
hr = runhook(
|
||||
self.log,
|
||||
self.conn.hsrv.broker,
|
||||
None,
|
||||
"xau.http.dump",
|
||||
xau,
|
||||
path,
|
||||
vp,
|
||||
self.host,
|
||||
self.uname,
|
||||
self.asrv.vfs.get_perms(self.vpath, self.uname),
|
||||
mt,
|
||||
post_sz,
|
||||
self.ip,
|
||||
at,
|
||||
"",
|
||||
)
|
||||
if not hr:
|
||||
t = "upload blocked by xau server config"
|
||||
self.log(t, 1)
|
||||
wunlink(self.log, path, vfs.flags)
|
||||
raise Pebkac(403, t)
|
||||
if hr.get("reloc"):
|
||||
x = pathmod(self.asrv.vfs, path, vp, hr["reloc"])
|
||||
if x:
|
||||
if self.args.hook_v:
|
||||
log_reloc(self.log, hr["reloc"], x, path, vp, fn, vfs, rem)
|
||||
fdir, self.vpath, fn, (vfs, rem) = x
|
||||
bos.makedirs(fdir)
|
||||
path2 = os.path.join(fdir, fn)
|
||||
atomic_move(self.log, path, path2, vfs.flags)
|
||||
path = path2
|
||||
if not nameless:
|
||||
self.vpath = vjoin(self.vpath, fn)
|
||||
sz = bos.path.getsize(path)
|
||||
else:
|
||||
sz = post_sz
|
||||
|
||||
vfs, rem = vfs.get_dbv(rem)
|
||||
self.conn.hsrv.broker.say(
|
||||
|
|
@ -1911,7 +1960,7 @@ class HttpCli(object):
|
|||
alg,
|
||||
self.args.fk_salt,
|
||||
path,
|
||||
post_sz,
|
||||
sz,
|
||||
0 if ANYWIN else bos.stat(path).st_ino,
|
||||
)[: vfs.flags["fk"]]
|
||||
|
||||
|
|
@ -2536,18 +2585,15 @@ class HttpCli(object):
|
|||
fname = sanitize_fn(
|
||||
p_file or "", "", [".prologue.html", ".epilogue.html"]
|
||||
)
|
||||
abspath = os.path.join(fdir, fname)
|
||||
suffix = "-%.6f-%s" % (time.time(), dip)
|
||||
if p_file and not nullwrite:
|
||||
if rnd:
|
||||
fname = rand_name(fdir, fname, rnd)
|
||||
|
||||
if not bos.path.isdir(fdir):
|
||||
raise Pebkac(404, "that folder does not exist")
|
||||
|
||||
suffix = "-{:.6f}-{}".format(time.time(), dip)
|
||||
open_args = {"fdir": fdir, "suffix": suffix}
|
||||
|
||||
if "replace" in self.uparam:
|
||||
abspath = os.path.join(fdir, fname)
|
||||
if not self.can_delete:
|
||||
self.log("user not allowed to overwrite with ?replace")
|
||||
elif bos.path.exists(abspath):
|
||||
|
|
@ -2557,6 +2603,58 @@ class HttpCli(object):
|
|||
except:
|
||||
t = "toctou while deleting for ?replace: %s"
|
||||
self.log(t % (abspath,))
|
||||
else:
|
||||
open_args = {}
|
||||
tnam = fname = os.devnull
|
||||
fdir = abspath = ""
|
||||
|
||||
if xbu:
|
||||
at = time.time() - lifetime
|
||||
hr = runhook(
|
||||
self.log,
|
||||
self.conn.hsrv.broker,
|
||||
None,
|
||||
"xbu.http.bup",
|
||||
xbu,
|
||||
abspath,
|
||||
vjoin(upload_vpath, fname),
|
||||
self.host,
|
||||
self.uname,
|
||||
self.asrv.vfs.get_perms(upload_vpath, self.uname),
|
||||
at,
|
||||
0,
|
||||
self.ip,
|
||||
at,
|
||||
"",
|
||||
)
|
||||
if not hr:
|
||||
t = "upload blocked by xbu server config"
|
||||
self.log(t, 1)
|
||||
raise Pebkac(403, t)
|
||||
if hr.get("reloc"):
|
||||
zs = vjoin(upload_vpath, fname)
|
||||
x = pathmod(self.asrv.vfs, abspath, zs, hr["reloc"])
|
||||
if x:
|
||||
if self.args.hook_v:
|
||||
log_reloc(
|
||||
self.log,
|
||||
hr["reloc"],
|
||||
x,
|
||||
abspath,
|
||||
zs,
|
||||
fname,
|
||||
vfs,
|
||||
rem,
|
||||
)
|
||||
fdir, upload_vpath, fname, (vfs, rem) = x
|
||||
abspath = os.path.join(fdir, fname)
|
||||
if nullwrite:
|
||||
fdir = abspath = ""
|
||||
else:
|
||||
open_args["fdir"] = fdir
|
||||
|
||||
if p_file and not nullwrite:
|
||||
bos.makedirs(fdir)
|
||||
|
||||
# reserve destination filename
|
||||
with ren_open(fname, "wb", fdir=fdir, suffix=suffix) as zfw:
|
||||
|
|
@ -2572,26 +2670,6 @@ class HttpCli(object):
|
|||
tnam = fname = os.devnull
|
||||
fdir = abspath = ""
|
||||
|
||||
if xbu:
|
||||
at = time.time() - lifetime
|
||||
if not runhook(
|
||||
self.log,
|
||||
xbu,
|
||||
abspath,
|
||||
self.vpath,
|
||||
self.host,
|
||||
self.uname,
|
||||
self.asrv.vfs.get_perms(self.vpath, self.uname),
|
||||
at,
|
||||
0,
|
||||
self.ip,
|
||||
at,
|
||||
"",
|
||||
):
|
||||
t = "upload blocked by xbu server config"
|
||||
self.log(t, 1)
|
||||
raise Pebkac(403, t)
|
||||
|
||||
if lim:
|
||||
lim.chk_bup(self.ip)
|
||||
lim.chk_nup(self.ip)
|
||||
|
|
@ -2634,29 +2712,58 @@ class HttpCli(object):
|
|||
|
||||
tabspath = ""
|
||||
|
||||
at = time.time() - lifetime
|
||||
if xau:
|
||||
hr = runhook(
|
||||
self.log,
|
||||
self.conn.hsrv.broker,
|
||||
None,
|
||||
"xau.http.bup",
|
||||
xau,
|
||||
abspath,
|
||||
vjoin(upload_vpath, fname),
|
||||
self.host,
|
||||
self.uname,
|
||||
self.asrv.vfs.get_perms(upload_vpath, self.uname),
|
||||
at,
|
||||
sz,
|
||||
self.ip,
|
||||
at,
|
||||
"",
|
||||
)
|
||||
if not hr:
|
||||
t = "upload blocked by xau server config"
|
||||
self.log(t, 1)
|
||||
wunlink(self.log, abspath, vfs.flags)
|
||||
raise Pebkac(403, t)
|
||||
if hr.get("reloc"):
|
||||
zs = vjoin(upload_vpath, fname)
|
||||
x = pathmod(self.asrv.vfs, abspath, zs, hr["reloc"])
|
||||
if x:
|
||||
if self.args.hook_v:
|
||||
log_reloc(
|
||||
self.log,
|
||||
hr["reloc"],
|
||||
x,
|
||||
abspath,
|
||||
zs,
|
||||
fname,
|
||||
vfs,
|
||||
rem,
|
||||
)
|
||||
fdir, upload_vpath, fname, (vfs, rem) = x
|
||||
ap2 = os.path.join(fdir, fname)
|
||||
if nullwrite:
|
||||
fdir = ap2 = ""
|
||||
else:
|
||||
bos.makedirs(fdir)
|
||||
atomic_move(self.log, abspath, ap2, vfs.flags)
|
||||
abspath = ap2
|
||||
sz = bos.path.getsize(abspath)
|
||||
|
||||
files.append(
|
||||
(sz, sha_hex, sha_b64, p_file or "(discarded)", fname, abspath)
|
||||
)
|
||||
at = time.time() - lifetime
|
||||
if xau and not runhook(
|
||||
self.log,
|
||||
xau,
|
||||
abspath,
|
||||
self.vpath,
|
||||
self.host,
|
||||
self.uname,
|
||||
self.asrv.vfs.get_perms(self.vpath, self.uname),
|
||||
at,
|
||||
sz,
|
||||
self.ip,
|
||||
at,
|
||||
"",
|
||||
):
|
||||
t = "upload blocked by xau server config"
|
||||
self.log(t, 1)
|
||||
wunlink(self.log, abspath, vfs.flags)
|
||||
raise Pebkac(403, t)
|
||||
|
||||
dbv, vrem = vfs.get_dbv(rem)
|
||||
self.conn.hsrv.broker.say(
|
||||
"up2k.hash_file",
|
||||
|
|
@ -2712,13 +2819,14 @@ class HttpCli(object):
|
|||
for sz, sha_hex, sha_b64, ofn, lfn, ap in files:
|
||||
vsuf = ""
|
||||
if (self.can_read or self.can_upget) and "fk" in vfs.flags:
|
||||
st = bos.stat(ap)
|
||||
alg = 2 if "fka" in vfs.flags else 1
|
||||
vsuf = "?k=" + self.gen_fk(
|
||||
alg,
|
||||
self.args.fk_salt,
|
||||
ap,
|
||||
sz,
|
||||
0 if ANYWIN or not ap else bos.stat(ap).st_ino,
|
||||
st.st_size,
|
||||
0 if ANYWIN or not ap else st.st_ino,
|
||||
)[: vfs.flags["fk"]]
|
||||
|
||||
if "media" in self.uparam or "medialinks" in vfs.flags:
|
||||
|
|
@ -2885,6 +2993,9 @@ class HttpCli(object):
|
|||
if xbu:
|
||||
if not runhook(
|
||||
self.log,
|
||||
self.conn.hsrv.broker,
|
||||
None,
|
||||
"xbu.http.txt",
|
||||
xbu,
|
||||
fp,
|
||||
self.vpath,
|
||||
|
|
@ -2924,6 +3035,9 @@ class HttpCli(object):
|
|||
xau = vfs.flags.get("xau")
|
||||
if xau and not runhook(
|
||||
self.log,
|
||||
self.conn.hsrv.broker,
|
||||
None,
|
||||
"xau.http.txt",
|
||||
xau,
|
||||
fp,
|
||||
self.vpath,
|
||||
|
|
@ -4156,7 +4270,7 @@ class HttpCli(object):
|
|||
if self.args.no_mv:
|
||||
raise Pebkac(403, "the rename/move feature is disabled in server config")
|
||||
|
||||
x = self.conn.hsrv.broker.ask("up2k.handle_mv", self.uname, vsrc, vdst)
|
||||
x = self.conn.hsrv.broker.ask("up2k.handle_mv", self.uname, self.ip, vsrc, vdst)
|
||||
self.loud_reply(x.get(), status=201)
|
||||
return True
|
||||
|
||||
|
|
|
|||
|
|
@ -187,6 +187,8 @@ class SMB(object):
|
|||
|
||||
debug('%s("%s", %s) %s @%s\033[K\033[0m', caller, vpath, str(a), perms, uname)
|
||||
vfs, rem = self.asrv.vfs.get(vpath, uname, *perms)
|
||||
if not vfs.realpath:
|
||||
raise Exception("unmapped vfs")
|
||||
return vfs, vfs.canonical(rem)
|
||||
|
||||
def _listdir(self, vpath: str, *a: Any, **ka: Any) -> list[str]:
|
||||
|
|
@ -195,6 +197,8 @@ class SMB(object):
|
|||
uname = self._uname()
|
||||
# debug('listdir("%s", %s) @%s\033[K\033[0m', vpath, str(a), uname)
|
||||
vfs, rem = self.asrv.vfs.get(vpath, uname, False, False)
|
||||
if not vfs.realpath:
|
||||
raise Exception("unmapped vfs")
|
||||
_, vfs_ls, vfs_virt = vfs.ls(
|
||||
rem, uname, not self.args.no_scandir, [[False, False]]
|
||||
)
|
||||
|
|
@ -240,7 +244,21 @@ class SMB(object):
|
|||
|
||||
xbu = vfs.flags.get("xbu")
|
||||
if xbu and not runhook(
|
||||
self.nlog, xbu, ap, vpath, "", "", "", 0, 0, "1.7.6.2", 0, ""
|
||||
self.nlog,
|
||||
None,
|
||||
self.hub.up2k,
|
||||
"xbu.smb",
|
||||
xbu,
|
||||
ap,
|
||||
vpath,
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
0,
|
||||
0,
|
||||
"1.7.6.2",
|
||||
time.time(),
|
||||
"",
|
||||
):
|
||||
yeet("blocked by xbu server config: " + vpath)
|
||||
|
||||
|
|
@ -297,7 +315,7 @@ class SMB(object):
|
|||
t = "blocked rename (no-move-acc %s): /%s @%s"
|
||||
yeet(t % (vfs1.axs.umove, vp1, uname))
|
||||
|
||||
self.hub.up2k.handle_mv(uname, vp1, vp2)
|
||||
self.hub.up2k.handle_mv(uname, "1.7.6.2", vp1, vp2)
|
||||
try:
|
||||
bos.makedirs(ap2)
|
||||
except:
|
||||
|
|
|
|||
|
|
@ -244,6 +244,8 @@ class Tftpd(object):
|
|||
|
||||
debug('%s("%s", %s) %s\033[K\033[0m', caller, vpath, str(a), perms)
|
||||
vfs, rem = self.asrv.vfs.get(vpath, "*", *perms)
|
||||
if not vfs.realpath:
|
||||
raise Exception("unmapped vfs")
|
||||
return vfs, vfs.canonical(rem)
|
||||
|
||||
def _ls(self, vpath: str, raddress: str, rport: int, force=False) -> Any:
|
||||
|
|
@ -331,7 +333,21 @@ class Tftpd(object):
|
|||
|
||||
xbu = vfs.flags.get("xbu")
|
||||
if xbu and not runhook(
|
||||
self.nlog, xbu, ap, vpath, "", "", "", 0, 0, "8.3.8.7", 0, ""
|
||||
self.nlog,
|
||||
None,
|
||||
self.hub.up2k,
|
||||
"xbu.tftpd",
|
||||
xbu,
|
||||
ap,
|
||||
vpath,
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
0,
|
||||
0,
|
||||
"8.3.8.7",
|
||||
time.time(),
|
||||
"",
|
||||
):
|
||||
yeet("blocked by xbu server config: " + vpath)
|
||||
|
||||
|
|
@ -339,7 +355,7 @@ class Tftpd(object):
|
|||
return self._ls(vpath, "", 0, True)
|
||||
|
||||
if not a:
|
||||
a = [self.args.iobuf]
|
||||
a = (self.args.iobuf,)
|
||||
|
||||
return open(ap, mode, *a, **ka)
|
||||
|
||||
|
|
|
|||
|
|
@ -46,6 +46,7 @@ from .util import (
|
|||
hidedir,
|
||||
humansize,
|
||||
min_ex,
|
||||
pathmod,
|
||||
quotep,
|
||||
rand_name,
|
||||
ren_open,
|
||||
|
|
@ -165,6 +166,7 @@ class Up2k(object):
|
|||
self.xiu_ptn = re.compile(r"(?:^|,)i([0-9]+)")
|
||||
self.xiu_busy = False # currently running hook
|
||||
self.xiu_asleep = True # needs rescan_cond poke to schedule self
|
||||
self.fx_backlog: list[tuple[str, dict[str, str], str]] = []
|
||||
|
||||
self.cur: dict[str, "sqlite3.Cursor"] = {}
|
||||
self.mem_cur = None
|
||||
|
|
@ -2544,7 +2546,7 @@ class Up2k(object):
|
|||
if self.mutex.acquire(timeout=10):
|
||||
got_lock = True
|
||||
with self.reg_mutex:
|
||||
return self._handle_json(cj)
|
||||
ret = self._handle_json(cj)
|
||||
else:
|
||||
t = "cannot receive uploads right now;\nserver busy with {}.\nPlease wait; the client will retry..."
|
||||
raise Pebkac(503, t.format(self.blocked or "[unknown]"))
|
||||
|
|
@ -2552,11 +2554,16 @@ class Up2k(object):
|
|||
if not PY2:
|
||||
raise
|
||||
with self.mutex, self.reg_mutex:
|
||||
return self._handle_json(cj)
|
||||
ret = self._handle_json(cj)
|
||||
finally:
|
||||
if got_lock:
|
||||
self.mutex.release()
|
||||
|
||||
if self.fx_backlog:
|
||||
self.do_fx_backlog()
|
||||
|
||||
return ret
|
||||
|
||||
def _handle_json(self, cj: dict[str, Any]) -> dict[str, Any]:
|
||||
ptop = cj["ptop"]
|
||||
if not self.register_vpath(ptop, cj["vcfg"]):
|
||||
|
|
@ -2758,28 +2765,43 @@ class Up2k(object):
|
|||
job["name"] = rand_name(
|
||||
pdir, cj["name"], vfs.flags["nrand"]
|
||||
)
|
||||
else:
|
||||
job["name"] = self._untaken(pdir, cj, now)
|
||||
|
||||
dst = djoin(job["ptop"], job["prel"], job["name"])
|
||||
xbu = vfs.flags.get("xbu")
|
||||
if xbu and not runhook(
|
||||
self.log,
|
||||
xbu, # type: ignore
|
||||
dst,
|
||||
job["vtop"],
|
||||
job["host"],
|
||||
job["user"],
|
||||
self.asrv.vfs.get_perms(job["vtop"], job["user"]),
|
||||
job["lmod"],
|
||||
job["size"],
|
||||
job["addr"],
|
||||
job["at"],
|
||||
"",
|
||||
):
|
||||
t = "upload blocked by xbu server config: {}".format(dst)
|
||||
self.log(t, 1)
|
||||
raise Pebkac(403, t)
|
||||
if xbu:
|
||||
vp = djoin(job["vtop"], job["prel"], job["name"])
|
||||
hr = runhook(
|
||||
self.log,
|
||||
None,
|
||||
self,
|
||||
"xbu.up2k.dupe",
|
||||
xbu, # type: ignore
|
||||
dst,
|
||||
vp,
|
||||
job["host"],
|
||||
job["user"],
|
||||
self.asrv.vfs.get_perms(job["vtop"], job["user"]),
|
||||
job["lmod"],
|
||||
job["size"],
|
||||
job["addr"],
|
||||
job["at"],
|
||||
"",
|
||||
)
|
||||
if not hr:
|
||||
t = "upload blocked by xbu server config: %s" % (dst,)
|
||||
self.log(t, 1)
|
||||
raise Pebkac(403, t)
|
||||
if hr.get("reloc"):
|
||||
x = pathmod(self.asrv.vfs, dst, vp, hr["reloc"])
|
||||
if x:
|
||||
pdir, _, job["name"], (vfs, rem) = x
|
||||
dst = os.path.join(pdir, job["name"])
|
||||
job["ptop"] = vfs.realpath
|
||||
job["vtop"] = vfs.vpath
|
||||
job["prel"] = rem
|
||||
bos.makedirs(pdir)
|
||||
|
||||
job["name"] = self._untaken(pdir, cj, now)
|
||||
|
||||
if not self.args.nw:
|
||||
dvf: dict[str, Any] = vfs.flags
|
||||
|
|
@ -3142,6 +3164,9 @@ class Up2k(object):
|
|||
with self.mutex, self.reg_mutex:
|
||||
self._finish_upload(ptop, wark)
|
||||
|
||||
if self.fx_backlog:
|
||||
self.do_fx_backlog()
|
||||
|
||||
def _finish_upload(self, ptop: str, wark: str) -> None:
|
||||
"""mutex(main,reg) me"""
|
||||
try:
|
||||
|
|
@ -3335,25 +3360,30 @@ class Up2k(object):
|
|||
|
||||
xau = False if skip_xau else vflags.get("xau")
|
||||
dst = djoin(ptop, rd, fn)
|
||||
if xau and not runhook(
|
||||
self.log,
|
||||
xau,
|
||||
dst,
|
||||
djoin(vtop, rd, fn),
|
||||
host,
|
||||
usr,
|
||||
self.asrv.vfs.get_perms(djoin(vtop, rd, fn), usr),
|
||||
int(ts),
|
||||
sz,
|
||||
ip,
|
||||
at or time.time(),
|
||||
"",
|
||||
):
|
||||
t = "upload blocked by xau server config"
|
||||
self.log(t, 1)
|
||||
wunlink(self.log, dst, vflags)
|
||||
self.registry[ptop].pop(wark, None)
|
||||
raise Pebkac(403, t)
|
||||
if xau:
|
||||
hr = runhook(
|
||||
self.log,
|
||||
None,
|
||||
self,
|
||||
"xau.up2k",
|
||||
xau,
|
||||
dst,
|
||||
djoin(vtop, rd, fn),
|
||||
host,
|
||||
usr,
|
||||
self.asrv.vfs.get_perms(djoin(vtop, rd, fn), usr),
|
||||
ts,
|
||||
sz,
|
||||
ip,
|
||||
at or time.time(),
|
||||
"",
|
||||
)
|
||||
if not hr:
|
||||
t = "upload blocked by xau server config"
|
||||
self.log(t, 1)
|
||||
wunlink(self.log, dst, vflags)
|
||||
self.registry[ptop].pop(wark, None)
|
||||
raise Pebkac(403, t)
|
||||
|
||||
xiu = vflags.get("xiu")
|
||||
if xiu:
|
||||
|
|
@ -3537,6 +3567,9 @@ class Up2k(object):
|
|||
if xbd:
|
||||
if not runhook(
|
||||
self.log,
|
||||
None,
|
||||
self,
|
||||
"xbd",
|
||||
xbd,
|
||||
abspath,
|
||||
vpath,
|
||||
|
|
@ -3546,7 +3579,7 @@ class Up2k(object):
|
|||
stl.st_mtime,
|
||||
st.st_size,
|
||||
ip,
|
||||
0,
|
||||
time.time(),
|
||||
"",
|
||||
):
|
||||
t = "delete blocked by xbd server config: {}"
|
||||
|
|
@ -3571,6 +3604,9 @@ class Up2k(object):
|
|||
if xad:
|
||||
runhook(
|
||||
self.log,
|
||||
None,
|
||||
self,
|
||||
"xad",
|
||||
xad,
|
||||
abspath,
|
||||
vpath,
|
||||
|
|
@ -3580,7 +3616,7 @@ class Up2k(object):
|
|||
stl.st_mtime,
|
||||
st.st_size,
|
||||
ip,
|
||||
0,
|
||||
time.time(),
|
||||
"",
|
||||
)
|
||||
|
||||
|
|
@ -3596,7 +3632,7 @@ class Up2k(object):
|
|||
|
||||
return n_files, ok + ok2, ng + ng2
|
||||
|
||||
def handle_mv(self, uname: str, svp: str, dvp: str) -> str:
|
||||
def handle_mv(self, uname: str, ip: str, svp: str, dvp: str) -> str:
|
||||
if svp == dvp or dvp.startswith(svp + "/"):
|
||||
raise Pebkac(400, "mv: cannot move parent into subfolder")
|
||||
|
||||
|
|
@ -3613,7 +3649,7 @@ class Up2k(object):
|
|||
if stat.S_ISREG(st.st_mode) or stat.S_ISLNK(st.st_mode):
|
||||
with self.mutex:
|
||||
try:
|
||||
ret = self._mv_file(uname, svp, dvp, curs)
|
||||
ret = self._mv_file(uname, ip, svp, dvp, curs)
|
||||
finally:
|
||||
for v in curs:
|
||||
v.connection.commit()
|
||||
|
|
@ -3646,7 +3682,7 @@ class Up2k(object):
|
|||
raise Pebkac(500, "mv: bug at {}, top {}".format(svpf, svp))
|
||||
|
||||
dvpf = dvp + svpf[len(svp) :]
|
||||
self._mv_file(uname, svpf, dvpf, curs)
|
||||
self._mv_file(uname, ip, svpf, dvpf, curs)
|
||||
finally:
|
||||
for v in curs:
|
||||
v.connection.commit()
|
||||
|
|
@ -3671,7 +3707,7 @@ class Up2k(object):
|
|||
return "k"
|
||||
|
||||
def _mv_file(
|
||||
self, uname: str, svp: str, dvp: str, curs: set["sqlite3.Cursor"]
|
||||
self, uname: str, ip: str, svp: str, dvp: str, curs: set["sqlite3.Cursor"]
|
||||
) -> str:
|
||||
"""mutex(main) me; will mutex(reg)"""
|
||||
svn, srem = self.asrv.vfs.get(svp, uname, True, False, True)
|
||||
|
|
@ -3705,21 +3741,27 @@ class Up2k(object):
|
|||
except:
|
||||
pass # broken symlink; keep as-is
|
||||
|
||||
ftime = stl.st_mtime
|
||||
fsize = st.st_size
|
||||
|
||||
xbr = svn.flags.get("xbr")
|
||||
xar = dvn.flags.get("xar")
|
||||
if xbr:
|
||||
if not runhook(
|
||||
self.log,
|
||||
None,
|
||||
self,
|
||||
"xbr",
|
||||
xbr,
|
||||
sabs,
|
||||
svp,
|
||||
"",
|
||||
uname,
|
||||
self.asrv.vfs.get_perms(svp, uname),
|
||||
stl.st_mtime,
|
||||
st.st_size,
|
||||
"",
|
||||
0,
|
||||
ftime,
|
||||
fsize,
|
||||
ip,
|
||||
time.time(),
|
||||
"",
|
||||
):
|
||||
t = "move blocked by xbr server config: {}".format(svp)
|
||||
|
|
@ -3747,16 +3789,19 @@ class Up2k(object):
|
|||
if xar:
|
||||
runhook(
|
||||
self.log,
|
||||
None,
|
||||
self,
|
||||
"xar.ln",
|
||||
xar,
|
||||
dabs,
|
||||
dvp,
|
||||
"",
|
||||
uname,
|
||||
self.asrv.vfs.get_perms(dvp, uname),
|
||||
0,
|
||||
0,
|
||||
"",
|
||||
0,
|
||||
ftime,
|
||||
fsize,
|
||||
ip,
|
||||
time.time(),
|
||||
"",
|
||||
)
|
||||
|
||||
|
|
@ -3765,13 +3810,6 @@ class Up2k(object):
|
|||
c1, w, ftime_, fsize_, ip, at = self._find_from_vpath(svn.realpath, srem)
|
||||
c2 = self.cur.get(dvn.realpath)
|
||||
|
||||
if ftime_ is None:
|
||||
ftime = stl.st_mtime
|
||||
fsize = st.st_size
|
||||
else:
|
||||
ftime = ftime_
|
||||
fsize = fsize_ or 0
|
||||
|
||||
has_dupes = False
|
||||
if w:
|
||||
assert c1
|
||||
|
|
@ -3779,7 +3817,9 @@ class Up2k(object):
|
|||
self._copy_tags(c1, c2, w)
|
||||
|
||||
with self.reg_mutex:
|
||||
has_dupes = self._forget_file(svn.realpath, srem, c1, w, is_xvol, fsize)
|
||||
has_dupes = self._forget_file(
|
||||
svn.realpath, srem, c1, w, is_xvol, fsize_ or fsize
|
||||
)
|
||||
|
||||
if not is_xvol:
|
||||
has_dupes = self._relink(w, svn.realpath, srem, dabs)
|
||||
|
|
@ -3849,7 +3889,7 @@ class Up2k(object):
|
|||
|
||||
if is_link:
|
||||
try:
|
||||
times = (int(time.time()), int(stl.st_mtime))
|
||||
times = (int(time.time()), int(ftime))
|
||||
bos.utime(dabs, times, False)
|
||||
except:
|
||||
pass
|
||||
|
|
@ -3859,16 +3899,19 @@ class Up2k(object):
|
|||
if xar:
|
||||
runhook(
|
||||
self.log,
|
||||
None,
|
||||
self,
|
||||
"xar.mv",
|
||||
xar,
|
||||
dabs,
|
||||
dvp,
|
||||
"",
|
||||
uname,
|
||||
self.asrv.vfs.get_perms(dvp, uname),
|
||||
0,
|
||||
0,
|
||||
"",
|
||||
0,
|
||||
ftime,
|
||||
fsize,
|
||||
ip,
|
||||
time.time(),
|
||||
"",
|
||||
)
|
||||
|
||||
|
|
@ -4152,23 +4195,35 @@ class Up2k(object):
|
|||
xbu = self.flags[job["ptop"]].get("xbu")
|
||||
ap_chk = djoin(pdir, job["name"])
|
||||
vp_chk = djoin(job["vtop"], job["prel"], job["name"])
|
||||
if xbu and not runhook(
|
||||
self.log,
|
||||
xbu,
|
||||
ap_chk,
|
||||
vp_chk,
|
||||
job["host"],
|
||||
job["user"],
|
||||
self.asrv.vfs.get_perms(vp_chk, job["user"]),
|
||||
int(job["lmod"]),
|
||||
job["size"],
|
||||
job["addr"],
|
||||
int(job["t0"]),
|
||||
"",
|
||||
):
|
||||
t = "upload blocked by xbu server config: {}".format(vp_chk)
|
||||
self.log(t, 1)
|
||||
raise Pebkac(403, t)
|
||||
if xbu:
|
||||
hr = runhook(
|
||||
self.log,
|
||||
None,
|
||||
self,
|
||||
"xbu.up2k",
|
||||
xbu,
|
||||
ap_chk,
|
||||
vp_chk,
|
||||
job["host"],
|
||||
job["user"],
|
||||
self.asrv.vfs.get_perms(vp_chk, job["user"]),
|
||||
job["lmod"],
|
||||
job["size"],
|
||||
job["addr"],
|
||||
job["t0"],
|
||||
"",
|
||||
)
|
||||
if not hr:
|
||||
t = "upload blocked by xbu server config: {}".format(vp_chk)
|
||||
self.log(t, 1)
|
||||
raise Pebkac(403, t)
|
||||
if hr.get("reloc"):
|
||||
x = pathmod(self.asrv.vfs, ap_chk, vp_chk, hr["reloc"])
|
||||
if x:
|
||||
pdir, _, job["name"], (vfs, rem) = x
|
||||
job["ptop"] = vfs.realpath
|
||||
job["vtop"] = vfs.vpath
|
||||
job["prel"] = rem
|
||||
|
||||
tnam = job["name"] + ".PARTIAL"
|
||||
if self.args.dotpart:
|
||||
|
|
@ -4442,6 +4497,9 @@ class Up2k(object):
|
|||
with self.rescan_cond:
|
||||
self.rescan_cond.notify_all()
|
||||
|
||||
if self.fx_backlog:
|
||||
self.do_fx_backlog()
|
||||
|
||||
return True
|
||||
|
||||
def hash_file(
|
||||
|
|
@ -4473,6 +4531,48 @@ class Up2k(object):
|
|||
self.hashq.put(zt)
|
||||
self.n_hashq += 1
|
||||
|
||||
def do_fx_backlog(self):
|
||||
with self.mutex, self.reg_mutex:
|
||||
todo = self.fx_backlog
|
||||
self.fx_backlog = []
|
||||
for act, hr, req_vp in todo:
|
||||
self.hook_fx(act, hr, req_vp)
|
||||
|
||||
def hook_fx(self, act: str, hr: dict[str, str], req_vp: str) -> None:
|
||||
bad = [k for k in hr if k != "vp"]
|
||||
if bad:
|
||||
t = "got unsupported key in %s from hook: %s"
|
||||
raise Exception(t % (act, bad))
|
||||
|
||||
for fvp in hr.get("vp") or []:
|
||||
# expect vpath including filename; either absolute
|
||||
# or relative to the client's vpath (request url)
|
||||
if fvp.startswith("/"):
|
||||
fvp, fn = vsplit(fvp[1:])
|
||||
fvp = "/" + fvp
|
||||
else:
|
||||
fvp, fn = vsplit(fvp)
|
||||
|
||||
x = pathmod(self.asrv.vfs, "", req_vp, {"vp": fvp, "fn": fn})
|
||||
if not x:
|
||||
t = "hook_fx(%s): failed to resolve %s based on %s"
|
||||
self.log(t % (act, fvp, req_vp))
|
||||
continue
|
||||
|
||||
ap, rd, fn, (vn, rem) = x
|
||||
vp = vjoin(rd, fn)
|
||||
if not vp:
|
||||
raise Exception("hook_fx: blank vp from pathmod")
|
||||
|
||||
if act == "idx":
|
||||
rd = rd[len(vn.vpath) :].strip("/")
|
||||
self.hash_file(
|
||||
vn.realpath, vn.vpath, vn.flags, rd, fn, "", time.time(), "", True
|
||||
)
|
||||
|
||||
if act == "del":
|
||||
self._handle_rm(LEELOO_DALLAS, "", vp, [], False, False)
|
||||
|
||||
def shutdown(self) -> None:
|
||||
self.stop = True
|
||||
|
||||
|
|
|
|||
|
|
@ -146,6 +146,8 @@ if TYPE_CHECKING:
|
|||
import magic
|
||||
|
||||
from .authsrv import VFS
|
||||
from .broker_util import BrokerCli
|
||||
from .up2k import Up2k
|
||||
|
||||
FAKE_MP = False
|
||||
|
||||
|
|
@ -2117,6 +2119,72 @@ def ujoin(rd: str, fn: str) -> str:
|
|||
return rd or fn
|
||||
|
||||
|
||||
def log_reloc(
|
||||
log: "NamedLogger",
|
||||
re: dict[str, str],
|
||||
pm: tuple[str, str, str, tuple["VFS", str]],
|
||||
ap: str,
|
||||
vp: str,
|
||||
fn: str,
|
||||
vn: "VFS",
|
||||
rem: str,
|
||||
) -> None:
|
||||
nap, nvp, nfn, (nvn, nrem) = pm
|
||||
t = "reloc %s:\nold ap [%s]\nnew ap [%s\033[36m/%s\033[0m]\nold vp [%s]\nnew vp [%s\033[36m/%s\033[0m]\nold fn [%s]\nnew fn [%s]\nold vfs [%s]\nnew vfs [%s]\nold rem [%s]\nnew rem [%s]"
|
||||
log(t % (re, ap, nap, nfn, vp, nvp, nfn, fn, nfn, vn.vpath, nvn.vpath, rem, nrem))
|
||||
|
||||
|
||||
def pathmod(
|
||||
vfs: "VFS", ap: str, vp: str, mod: dict[str, str]
|
||||
) -> Optional[tuple[str, str, str, tuple["VFS", str]]]:
|
||||
# vfs: authsrv.vfs
|
||||
# ap: original abspath to a file
|
||||
# vp: original urlpath to a file
|
||||
# mod: modification (ap/vp/fn)
|
||||
|
||||
nvp = "\n" # new vpath
|
||||
ap = os.path.dirname(ap)
|
||||
vp, fn = vsplit(vp)
|
||||
if mod.get("fn"):
|
||||
fn = mod["fn"]
|
||||
nvp = vp
|
||||
|
||||
for ref, k in ((ap, "ap"), (vp, "vp")):
|
||||
if k not in mod:
|
||||
continue
|
||||
|
||||
ms = mod[k].replace(os.sep, "/")
|
||||
if ms.startswith("/"):
|
||||
np = ms
|
||||
elif k == "vp":
|
||||
np = undot(vjoin(ref, ms))
|
||||
else:
|
||||
np = os.path.abspath(os.path.join(ref, ms))
|
||||
|
||||
if k == "vp":
|
||||
nvp = np.lstrip("/")
|
||||
continue
|
||||
|
||||
# try to map abspath to vpath
|
||||
np = np.replace("/", os.sep)
|
||||
for vn_ap, vn in vfs.all_aps:
|
||||
if not np.startswith(vn_ap):
|
||||
continue
|
||||
zs = np[len(vn_ap) :].replace(os.sep, "/")
|
||||
nvp = vjoin(vn.vpath, zs)
|
||||
break
|
||||
|
||||
if nvp == "\n":
|
||||
return None
|
||||
|
||||
vn, rem = vfs.get(nvp, "*", False, False)
|
||||
if not vn.realpath:
|
||||
raise Exception("unmapped vfs")
|
||||
|
||||
ap = vn.canonical(rem)
|
||||
return ap, nvp, fn, (vn, rem)
|
||||
|
||||
|
||||
def _w8dec2(txt: bytes) -> str:
|
||||
"""decodes filesystem-bytes to wtf8"""
|
||||
return surrogateescape.decodefilename(txt)
|
||||
|
|
@ -3130,6 +3198,7 @@ def runihook(
|
|||
|
||||
def _runhook(
|
||||
log: Optional["NamedLogger"],
|
||||
src: str,
|
||||
cmd: str,
|
||||
ap: str,
|
||||
vp: str,
|
||||
|
|
@ -3141,14 +3210,16 @@ def _runhook(
|
|||
ip: str,
|
||||
at: float,
|
||||
txt: str,
|
||||
) -> bool:
|
||||
) -> dict[str, Any]:
|
||||
ret = {"rc": 0}
|
||||
areq, chk, fork, jtxt, wait, sp_ka, acmd = _parsehook(log, cmd)
|
||||
if areq:
|
||||
for ch in areq:
|
||||
if ch not in perms:
|
||||
t = "user %s not allowed to run hook %s; need perms %s, have %s"
|
||||
log(t % (uname, cmd, areq, perms))
|
||||
return True # fallthrough to next hook
|
||||
if log:
|
||||
log(t % (uname, cmd, areq, perms))
|
||||
return ret # fallthrough to next hook
|
||||
if jtxt:
|
||||
ja = {
|
||||
"ap": ap,
|
||||
|
|
@ -3160,6 +3231,7 @@ def _runhook(
|
|||
"host": host,
|
||||
"user": uname,
|
||||
"perms": perms,
|
||||
"src": src,
|
||||
"txt": txt,
|
||||
}
|
||||
arg = json.dumps(ja)
|
||||
|
|
@ -3178,18 +3250,34 @@ def _runhook(
|
|||
else:
|
||||
rc, v, err = runcmd(bcmd, **sp_ka) # type: ignore
|
||||
if chk and rc:
|
||||
ret["rc"] = rc
|
||||
retchk(rc, bcmd, err, log, 5)
|
||||
return False
|
||||
else:
|
||||
try:
|
||||
ret = json.loads(v)
|
||||
except:
|
||||
ret = {}
|
||||
|
||||
try:
|
||||
if "stdout" not in ret:
|
||||
ret["stdout"] = v
|
||||
if "rc" not in ret:
|
||||
ret["rc"] = rc
|
||||
except:
|
||||
ret = {"rc": rc, "stdout": v}
|
||||
|
||||
wait -= time.time() - t0
|
||||
if wait > 0:
|
||||
time.sleep(wait)
|
||||
|
||||
return True
|
||||
return ret
|
||||
|
||||
|
||||
def runhook(
|
||||
log: Optional["NamedLogger"],
|
||||
broker: Optional["BrokerCli"],
|
||||
up2k: Optional["Up2k"],
|
||||
src: str,
|
||||
cmds: list[str],
|
||||
ap: str,
|
||||
vp: str,
|
||||
|
|
@ -3201,19 +3289,42 @@ def runhook(
|
|||
ip: str,
|
||||
at: float,
|
||||
txt: str,
|
||||
) -> bool:
|
||||
) -> dict[str, Any]:
|
||||
assert broker or up2k
|
||||
asrv = (broker or up2k).asrv
|
||||
args = (broker or up2k).args
|
||||
vp = vp.replace("\\", "/")
|
||||
ret = {"rc": 0}
|
||||
for cmd in cmds:
|
||||
try:
|
||||
if not _runhook(log, cmd, ap, vp, host, uname, perms, mt, sz, ip, at, txt):
|
||||
return False
|
||||
hr = _runhook(
|
||||
log, src, cmd, ap, vp, host, uname, perms, mt, sz, ip, at, txt
|
||||
)
|
||||
if log and args.hook_v:
|
||||
log("hook(%s) [%s] => \033[32m%s" % (src, cmd, hr), 6)
|
||||
if not hr:
|
||||
return {}
|
||||
for k, v in hr.items():
|
||||
if k in ("idx", "del") and v:
|
||||
if broker:
|
||||
broker.say("up2k.hook_fx", k, v, vp)
|
||||
else:
|
||||
up2k.fx_backlog.append((k, v, vp))
|
||||
elif k == "reloc" and v:
|
||||
# idk, just take the last one ig
|
||||
ret["reloc"] = v
|
||||
elif k in ret:
|
||||
if k == "rc" and v:
|
||||
ret[k] = v
|
||||
else:
|
||||
ret[k] = v
|
||||
except Exception as ex:
|
||||
(log or print)("hook: {}".format(ex))
|
||||
if ",c," in "," + cmd:
|
||||
return False
|
||||
return {}
|
||||
break
|
||||
|
||||
return True
|
||||
return ret
|
||||
|
||||
|
||||
def loadpy(ap: str, hot: bool) -> Any:
|
||||
|
|
|
|||
|
|
@ -12,6 +12,8 @@
|
|||
* [write](#write)
|
||||
* [admin](#admin)
|
||||
* [general](#general)
|
||||
* [event hooks](#event-hooks) - on writing your own [hooks](../README.md#event-hooks)
|
||||
* [hook effects](#hook-effects) - hooks can cause intentional side-effects
|
||||
* [assumptions](#assumptions)
|
||||
* [mdns](#mdns)
|
||||
* [sfx repack](#sfx-repack) - reduce the size of an sfx by removing features
|
||||
|
|
@ -204,6 +206,32 @@ upload modifiers:
|
|||
| GET | `?pw=x` | logout |
|
||||
|
||||
|
||||
# event hooks
|
||||
|
||||
on writing your own [hooks](../README.md#event-hooks)
|
||||
|
||||
## hook effects
|
||||
|
||||
hooks can cause intentional side-effects, such as redirecting an upload into another location, or creating+indexing additional files, or deleting existing files, by returning json on stdout
|
||||
|
||||
* `reloc` can redirect uploads before/after uploading has finished, based on filename, extension, file contents, uploader ip/name etc.
|
||||
* `idx` informs copyparty about a new file to index as a consequence of this upload
|
||||
* `del` tells copyparty to delete an unrelated file by vpath
|
||||
|
||||
for these to take effect, the hook must be defined with the `c1` flag; see example [reloc-by-ext](https://github.com/9001/copyparty/blob/hovudstraum/bin/hooks/reloc-by-ext.py)
|
||||
|
||||
a subset of effect types are available for a subset of hook types,
|
||||
|
||||
* most hook types (xbu/xau/xbr/xar/xbd/xad/xm) support `idx` and `del` for all http protocols (up2k / basic-uploader / webdav), but not ftp/tftp/smb
|
||||
* most hook types will abort/reject the action if the hook returns nonzero, assuming flag `c` is given, see examples [reject-extension](https://github.com/9001/copyparty/blob/hovudstraum/bin/hooks/reject-extension.py) and [reject-mimetype](https://github.com/9001/copyparty/blob/hovudstraum/bin/hooks/reject-mimetype.py)
|
||||
* `xbu` supports `reloc` for all http protocols (up2k / basic-uploader / webdav), but not ftp/tftp/smb
|
||||
* `xau` supports `reloc` for basic-uploader / webdav only, not up2k or ftp/tftp/smb
|
||||
* so clients like sharex are supported, but not dragdrop into browser
|
||||
|
||||
to trigger indexing of files `/foo/1.txt` and `/foo/bar/2.txt`, a hook can `print(json.dumps({"idx":{"vp":["/foo/1.txt","/foo/bar/2.txt"]}}))` (and replace "idx" with "del" to delete instead)
|
||||
* note: paths starting with `/` are absolute URLs, but you can also do `../3.txt` relative to the destination folder of each uploaded file
|
||||
|
||||
|
||||
# assumptions
|
||||
|
||||
## mdns
|
||||
|
|
|
|||
|
|
@ -175,6 +175,7 @@ symbol legend,
|
|||
| ┗ randomize filename | █ | | | | | | | █ | █ | | | | |
|
||||
| ┗ mimetype reject-list | ╱ | | | | | | | | • | ╱ | | ╱ | • |
|
||||
| ┗ extension reject-list | ╱ | | | | | | | █ | • | ╱ | | ╱ | • |
|
||||
| ┗ upload routing | █ | | | | | | | | | | | | |
|
||||
| checksums provided | | | | █ | █ | | | | █ | ╱ | | | |
|
||||
| cloud storage backend | ╱ | ╱ | ╱ | █ | █ | █ | ╱ | | | ╱ | █ | █ | ╱ |
|
||||
|
||||
|
|
@ -188,6 +189,9 @@ symbol legend,
|
|||
|
||||
* `race the beam` = files can be downloaded while they're still uploading; downloaders are slowed down such that the uploader is always ahead
|
||||
|
||||
* `upload routing` = depending on filetype / contents / uploader etc., the file can be redirected to another location or otherwise transformed; mitigates limitations such as [sharex#3992](https://github.com/ShareX/ShareX/issues/3992)
|
||||
* copyparty example: [reloc-by-ext](https://github.com/9001/copyparty/tree/hovudstraum/bin/hooks#before-upload)
|
||||
|
||||
* `checksums provided` = when downloading a file from the server, the file's checksum is provided for verification client-side
|
||||
|
||||
* `cloud storage backend` = able to serve files from (and write to) s3 or similar cloud services; `╱` means the software can do this with some help from `rclone mount` as a bridge
|
||||
|
|
|
|||
111
tests/test_hooks.py
Normal file
111
tests/test_hooks.py
Normal file
|
|
@ -0,0 +1,111 @@
|
|||
#!/usr/bin/env python3
|
||||
# coding: utf-8
|
||||
from __future__ import print_function, unicode_literals
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
from copyparty.authsrv import AuthSrv
|
||||
from copyparty.httpcli import HttpCli
|
||||
from tests import util as tu
|
||||
from tests.util import Cfg
|
||||
|
||||
|
||||
def hdr(query):
|
||||
h = "GET /{} HTTP/1.1\r\nPW: o\r\nConnection: close\r\n\r\n"
|
||||
return h.format(query).encode("utf-8")
|
||||
|
||||
|
||||
class TestHooks(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.td = tu.get_ramdisk()
|
||||
|
||||
def tearDown(self):
|
||||
os.chdir(tempfile.gettempdir())
|
||||
shutil.rmtree(self.td)
|
||||
|
||||
def reset(self):
|
||||
td = os.path.join(self.td, "vfs")
|
||||
if os.path.exists(td):
|
||||
shutil.rmtree(td)
|
||||
os.mkdir(td)
|
||||
os.chdir(td)
|
||||
return td
|
||||
|
||||
def test(self):
|
||||
vcfg = ["a/b/c/d:c/d:A", "a:a:r"]
|
||||
|
||||
scenarios = (
|
||||
('{"vp":"x/y"}', "c/d/a.png", "c/d/x/y/a.png"),
|
||||
('{"vp":"x/y"}', "c/d/e/a.png", "c/d/e/x/y/a.png"),
|
||||
('{"vp":"../x/y"}', "c/d/e/a.png", "c/d/x/y/a.png"),
|
||||
('{"ap":"x/y"}', "c/d/a.png", "c/d/x/y/a.png"),
|
||||
('{"ap":"x/y"}', "c/d/e/a.png", "c/d/e/x/y/a.png"),
|
||||
('{"ap":"../x/y"}', "c/d/e/a.png", "c/d/x/y/a.png"),
|
||||
('{"ap":"../x/y"}', "c/d/a.png", "a/b/c/x/y/a.png"),
|
||||
('{"fn":"b.png"}', "c/d/a.png", "c/d/b.png"),
|
||||
('{"vp":"x","fn":"b.png"}', "c/d/a.png", "c/d/x/b.png"),
|
||||
)
|
||||
|
||||
for x in scenarios:
|
||||
print("\n\n\n", x)
|
||||
hooktxt, url_up, url_dl = x
|
||||
for hooktype in ("xbu", "xau"):
|
||||
for upfun in (self.put, self.bup):
|
||||
self.reset()
|
||||
self.makehook("""print('{"reloc":%s}')""" % (hooktxt,))
|
||||
ka = {hooktype: ["j,c1,h.py"]}
|
||||
self.args = Cfg(v=vcfg, a=["o:o"], e2d=True, **ka)
|
||||
self.asrv = AuthSrv(self.args, self.log)
|
||||
|
||||
h, b = upfun(url_up)
|
||||
self.assertIn("201 Created", h)
|
||||
h, b = self.curl(url_dl)
|
||||
self.assertEqual(b, "ok %s\n" % (url_up))
|
||||
|
||||
def makehook(self, hs):
|
||||
with open("h.py", "wb") as f:
|
||||
f.write(hs.encode("utf-8"))
|
||||
|
||||
def put(self, url):
|
||||
buf = "PUT /{0} HTTP/1.1\r\nPW: o\r\nConnection: close\r\nContent-Length: {1}\r\n\r\nok {0}\n"
|
||||
buf = buf.format(url, len(url) + 4).encode("utf-8")
|
||||
print("PUT -->", buf)
|
||||
conn = tu.VHttpConn(self.args, self.asrv, self.log, buf)
|
||||
HttpCli(conn).run()
|
||||
ret = conn.s._reply.decode("utf-8").split("\r\n\r\n", 1)
|
||||
print("PUT <--", ret)
|
||||
return ret
|
||||
|
||||
def bup(self, url):
|
||||
hdr = "POST /%s HTTP/1.1\r\nPW: o\r\nConnection: close\r\nContent-Type: multipart/form-data; boundary=XD\r\nContent-Length: %d\r\n\r\n"
|
||||
bdy = '--XD\r\nContent-Disposition: form-data; name="act"\r\n\r\nbput\r\n--XD\r\nContent-Disposition: form-data; name="f"; filename="%s"\r\n\r\n'
|
||||
ftr = "\r\n--XD--\r\n"
|
||||
try:
|
||||
url, fn = url.rsplit("/", 1)
|
||||
except:
|
||||
fn = url
|
||||
url = ""
|
||||
|
||||
buf = (bdy % (fn,) + "ok %s/%s\n" % (url, fn) + ftr).encode("utf-8")
|
||||
buf = (hdr % (url, len(buf))).encode("utf-8") + buf
|
||||
print("PoST -->", buf)
|
||||
conn = tu.VHttpConn(self.args, self.asrv, self.log, buf)
|
||||
HttpCli(conn).run()
|
||||
ret = conn.s._reply.decode("utf-8").split("\r\n\r\n", 1)
|
||||
print("POST <--", ret)
|
||||
return ret
|
||||
|
||||
def curl(self, url, binary=False):
|
||||
conn = tu.VHttpConn(self.args, self.asrv, self.log, hdr(url))
|
||||
HttpCli(conn).run()
|
||||
if binary:
|
||||
h, b = conn.s._reply.split(b"\r\n\r\n", 1)
|
||||
return [h.decode("utf-8"), b]
|
||||
|
||||
return conn.s._reply.decode("utf-8").split("\r\n\r\n", 1)
|
||||
|
||||
def log(self, src, msg, c=0):
|
||||
print(msg)
|
||||
|
|
@ -68,6 +68,13 @@ def chkcmd(argv):
|
|||
|
||||
def get_ramdisk():
|
||||
def subdir(top):
|
||||
for d in os.listdir(top):
|
||||
if not d.startswith("cptd-"):
|
||||
continue
|
||||
p = os.path.join(top, d)
|
||||
st = os.stat(p)
|
||||
if time.time() - st.st_mtime > 300:
|
||||
shutil.rmtree(p)
|
||||
ret = os.path.join(top, "cptd-{}".format(os.getpid()))
|
||||
shutil.rmtree(ret, True)
|
||||
os.mkdir(ret)
|
||||
|
|
@ -111,10 +118,10 @@ class Cfg(Namespace):
|
|||
def __init__(self, a=None, v=None, c=None, **ka0):
|
||||
ka = {}
|
||||
|
||||
ex = "daw dav_auth dav_inf dav_mac dav_rt e2d e2ds e2dsa e2t e2ts e2tsr e2v e2vu e2vp early_ban ed emp exp force_js getmod grid gsel hardlink ih ihead magic never_symlink nid nih no_acode no_athumb no_dav no_dedup no_del no_dupe no_lifetime no_logues no_mv no_pipe no_poll no_readme no_robots no_sb_md no_sb_lg no_scandir no_tarcmp no_thumb no_vthumb no_zip nrand nw og og_no_head og_s_title q rand smb srch_dbg stats uqe vague_403 vc ver xdev xlink xvol"
|
||||
ex = "daw dav_auth dav_inf dav_mac dav_rt e2d e2ds e2dsa e2t e2ts e2tsr e2v e2vu e2vp early_ban ed emp exp force_js getmod grid gsel hardlink ih ihead magic never_symlink nid nih no_acode no_athumb no_dav no_dedup no_del no_dupe no_lifetime no_logues no_mv no_pipe no_poll no_readme no_robots no_sb_md no_sb_lg no_scandir no_tarcmp no_thumb no_vthumb no_zip nrand nw og og_no_head og_s_title q rand smb srch_dbg stats uqe vague_403 vc ver write_uplog xdev xlink xvol"
|
||||
ka.update(**{k: False for k in ex.split()})
|
||||
|
||||
ex = "dotpart dotsrch no_dhash no_fastboot no_rescan no_sendfile no_snap no_voldump re_dhash plain_ip"
|
||||
ex = "dotpart dotsrch hook_v no_dhash no_fastboot no_rescan no_sendfile no_snap no_voldump re_dhash plain_ip"
|
||||
ka.update(**{k: True for k in ex.split()})
|
||||
|
||||
ex = "ah_cli ah_gen css_browser hist js_browser mime mimes no_forget no_hash no_idx nonsus_urls og_tpl og_ua"
|
||||
|
|
@ -178,6 +185,10 @@ class Cfg(Namespace):
|
|||
|
||||
|
||||
class NullBroker(object):
|
||||
def __init__(self, args, asrv):
|
||||
self.args = args
|
||||
self.asrv = asrv
|
||||
|
||||
def say(self, *args):
|
||||
pass
|
||||
|
||||
|
|
@ -213,7 +224,7 @@ class VHttpSrv(object):
|
|||
self.asrv = asrv
|
||||
self.log = log
|
||||
|
||||
self.broker = NullBroker()
|
||||
self.broker = NullBroker(args, asrv)
|
||||
self.prism = None
|
||||
self.bans = {}
|
||||
self.nreq = 0
|
||||
|
|
|
|||
Loading…
Reference in a new issue