single authsrv instance per process

This commit is contained in:
ed 2021-06-11 23:01:13 +02:00
parent fbe656957d
commit 60ac68d000
15 changed files with 156 additions and 109 deletions

9
.vscode/launch.json vendored
View file

@ -16,12 +16,9 @@
"-e2ts", "-e2ts",
"-mtp", "-mtp",
".bpm=f,bin/mtag/audio-bpm.py", ".bpm=f,bin/mtag/audio-bpm.py",
"-a", "-aed:wark",
"ed:wark", "-vsrv::r:aed:cnodupe",
"-v", "-vdist:dist:r"
"srv::r:aed:cnodupe",
"-v",
"dist:dist:r"
] ]
}, },
{ {

View file

@ -43,7 +43,9 @@ class VFS(object):
) )
def get_all_vols(self, outdict): def get_all_vols(self, outdict):
outdict[self.vpath] = self if self.realpath:
outdict[self.vpath] = self
for v in self.nodes.values(): for v in self.nodes.values():
v.get_all_vols(outdict) v.get_all_vols(outdict)
@ -60,7 +62,7 @@ class VFS(object):
return self.nodes[name].add(src, dst) return self.nodes[name].add(src, dst)
vn = VFS( vn = VFS(
os.path.join(self.realpath, name) if self.realpath else name, os.path.join(self.realpath, name) if self.realpath else None,
"{}/{}".format(self.vpath, name).lstrip("/"), "{}/{}".format(self.vpath, name).lstrip("/"),
self.uread, self.uread,
self.uwrite, self.uwrite,

View file

@ -36,7 +36,7 @@ class MpWorker(object):
signal.signal(signal.SIGINT, self.signal_handler) signal.signal(signal.SIGINT, self.signal_handler)
# starting to look like a good idea # starting to look like a good idea
self.authsrv = AuthSrv(args, None, False) self.asrv = AuthSrv(args, None, False)
# instantiate all services here (TODO: inheritance?) # instantiate all services here (TODO: inheritance?)
self.httpsrv = HttpSrv(self, True) self.httpsrv = HttpSrv(self, True)

View file

@ -15,12 +15,10 @@ class BrokerThr(object):
self.hub = hub self.hub = hub
self.log = hub.log self.log = hub.log
self.args = hub.args self.args = hub.args
self.asrv = hub.asrv
self.mutex = threading.Lock() self.mutex = threading.Lock()
# starting to look like a good idea
self.authsrv = AuthSrv(self.args, None, False)
# instantiate all services here (TODO: inheritance?) # instantiate all services here (TODO: inheritance?)
self.httpsrv = HttpSrv(self) self.httpsrv = HttpSrv(self)
self.httpsrv.disconnect_func = self.httpdrop self.httpsrv.disconnect_func = self.httpdrop

View file

@ -42,7 +42,7 @@ class HttpCli(object):
self.addr = conn.addr # type: tuple[str, int] self.addr = conn.addr # type: tuple[str, int]
self.args = conn.args self.args = conn.args
self.is_mp = conn.is_mp self.is_mp = conn.is_mp
self.auth = conn.auth # type: AuthSrv self.asrv = conn.asrv # type: AuthSrv
self.ico = conn.ico self.ico = conn.ico
self.thumbcli = conn.thumbcli self.thumbcli = conn.thumbcli
self.log_func = conn.log_func self.log_func = conn.log_func
@ -154,9 +154,9 @@ class HttpCli(object):
self.vpath = unquotep(vpath) self.vpath = unquotep(vpath)
pwd = uparam.get("pw") pwd = uparam.get("pw")
self.uname = self.auth.iuser.get(pwd, "*") self.uname = self.asrv.iuser.get(pwd, "*")
self.rvol, self.wvol, self.avol = [[], [], []] self.rvol, self.wvol, self.avol = [[], [], []]
self.auth.vfs.user_tree(self.uname, self.rvol, self.wvol, self.avol) self.asrv.vfs.user_tree(self.uname, self.rvol, self.wvol, self.avol)
ua = self.headers.get("user-agent", "") ua = self.headers.get("user-agent", "")
self.is_rclone = ua.startswith("rclone/") self.is_rclone = ua.startswith("rclone/")
@ -321,9 +321,7 @@ class HttpCli(object):
self.redirect(vpath, flavor="redirecting to", use302=True) self.redirect(vpath, flavor="redirecting to", use302=True)
return True return True
self.readable, self.writable = self.conn.auth.vfs.can_access( self.readable, self.writable = self.asrv.vfs.can_access(self.vpath, self.uname)
self.vpath, self.uname
)
if not self.readable and not self.writable: if not self.readable and not self.writable:
if self.vpath: if self.vpath:
self.log("inaccessible: [{}]".format(self.vpath)) self.log("inaccessible: [{}]".format(self.vpath))
@ -440,7 +438,7 @@ class HttpCli(object):
def dump_to_file(self): def dump_to_file(self):
reader, remains = self.get_body_reader() reader, remains = self.get_body_reader()
vfs, rem = self.conn.auth.vfs.get(self.vpath, self.uname, False, True) vfs, rem = self.asrv.vfs.get(self.vpath, self.uname, False, True)
fdir = os.path.join(vfs.realpath, rem) fdir = os.path.join(vfs.realpath, rem)
addr = self.ip.replace(":", ".") addr = self.ip.replace(":", ".")
@ -509,7 +507,7 @@ class HttpCli(object):
if v is None: if v is None:
raise Pebkac(422, "need zip or tar keyword") raise Pebkac(422, "need zip or tar keyword")
vn, rem = self.auth.vfs.get(self.vpath, self.uname, True, False) vn, rem = self.asrv.vfs.get(self.vpath, self.uname, True, False)
items = self.parser.require("files", 1024 * 1024) items = self.parser.require("files", 1024 * 1024)
if not items: if not items:
raise Pebkac(422, "need files list") raise Pebkac(422, "need files list")
@ -559,7 +557,7 @@ class HttpCli(object):
self.vpath = "/".join([self.vpath, sub]).strip("/") self.vpath = "/".join([self.vpath, sub]).strip("/")
body["name"] = name body["name"] = name
vfs, rem = self.conn.auth.vfs.get(self.vpath, self.uname, False, True) vfs, rem = self.asrv.vfs.get(self.vpath, self.uname, False, True)
dbv, vrem = vfs.get_dbv(rem) dbv, vrem = vfs.get_dbv(rem)
body["vtop"] = dbv.vpath body["vtop"] = dbv.vpath
@ -590,7 +588,7 @@ class HttpCli(object):
vols = [] vols = []
seen = {} seen = {}
for vtop in self.rvol: for vtop in self.rvol:
vfs, _ = self.conn.auth.vfs.get(vtop, self.uname, True, False) vfs, _ = self.asrv.vfs.get(vtop, self.uname, True, False)
vfs = vfs.dbv or vfs vfs = vfs.dbv or vfs
if vfs in seen: if vfs in seen:
continue continue
@ -651,7 +649,7 @@ class HttpCli(object):
except KeyError: except KeyError:
raise Pebkac(400, "need hash and wark headers for binary POST") raise Pebkac(400, "need hash and wark headers for binary POST")
vfs, _ = self.conn.auth.vfs.get(self.vpath, self.uname, False, True) vfs, _ = self.asrv.vfs.get(self.vpath, self.uname, False, True)
ptop = (vfs.dbv or vfs).realpath ptop = (vfs.dbv or vfs).realpath
x = self.conn.hsrv.broker.put(True, "up2k.handle_chunk", ptop, wark, chash) x = self.conn.hsrv.broker.put(True, "up2k.handle_chunk", ptop, wark, chash)
@ -724,7 +722,7 @@ class HttpCli(object):
pwd = self.parser.require("cppwd", 64) pwd = self.parser.require("cppwd", 64)
self.parser.drop() self.parser.drop()
if pwd in self.auth.iuser: if pwd in self.asrv.iuser:
msg = "login ok" msg = "login ok"
dt = datetime.utcfromtimestamp(time.time() + 60 * 60 * 24 * 365) dt = datetime.utcfromtimestamp(time.time() + 60 * 60 * 24 * 365)
exp = dt.strftime("%a, %d %b %Y %H:%M:%S GMT") exp = dt.strftime("%a, %d %b %Y %H:%M:%S GMT")
@ -743,7 +741,7 @@ class HttpCli(object):
self.parser.drop() self.parser.drop()
nullwrite = self.args.nw nullwrite = self.args.nw
vfs, rem = self.conn.auth.vfs.get(self.vpath, self.uname, False, True) vfs, rem = self.asrv.vfs.get(self.vpath, self.uname, False, True)
self._assert_safe_rem(rem) self._assert_safe_rem(rem)
sanitized = sanitize_fn(new_dir) sanitized = sanitize_fn(new_dir)
@ -772,7 +770,7 @@ class HttpCli(object):
self.parser.drop() self.parser.drop()
nullwrite = self.args.nw nullwrite = self.args.nw
vfs, rem = self.conn.auth.vfs.get(self.vpath, self.uname, False, True) vfs, rem = self.asrv.vfs.get(self.vpath, self.uname, False, True)
self._assert_safe_rem(rem) self._assert_safe_rem(rem)
if not new_file.endswith(".md"): if not new_file.endswith(".md"):
@ -796,7 +794,7 @@ class HttpCli(object):
def handle_plain_upload(self): def handle_plain_upload(self):
nullwrite = self.args.nw nullwrite = self.args.nw
vfs, rem = self.conn.auth.vfs.get(self.vpath, self.uname, False, True) vfs, rem = self.asrv.vfs.get(self.vpath, self.uname, False, True)
self._assert_safe_rem(rem) self._assert_safe_rem(rem)
files = [] files = []
@ -937,7 +935,7 @@ class HttpCli(object):
raise Pebkac(400, "could not read lastmod from request") raise Pebkac(400, "could not read lastmod from request")
nullwrite = self.args.nw nullwrite = self.args.nw
vfs, rem = self.conn.auth.vfs.get(self.vpath, self.uname, False, True) vfs, rem = self.asrv.vfs.get(self.vpath, self.uname, False, True)
self._assert_safe_rem(rem) self._assert_safe_rem(rem)
# TODO: # TODO:
@ -1400,9 +1398,10 @@ class HttpCli(object):
if self.args.no_rescan: if self.args.no_rescan:
raise Pebkac(403, "disabled by argv") raise Pebkac(403, "disabled by argv")
vn, _ = self.auth.vfs.get(self.vpath, self.uname, True, True) vn, _ = self.asrv.vfs.get(self.vpath, self.uname, True, True)
args = [self.asrv.vfs.all_vols, [vn.vpath]]
args = [self.auth.vfs.all_vols, [vn.vpath]]
x = self.conn.hsrv.broker.put(True, "up2k.rescan", *args) x = self.conn.hsrv.broker.put(True, "up2k.rescan", *args)
x = x.get() x = x.get()
if not x: if not x:
@ -1474,7 +1473,7 @@ class HttpCli(object):
ret["k" + quotep(excl)] = sub ret["k" + quotep(excl)] = sub
try: try:
vn, rem = self.auth.vfs.get(top, self.uname, True, False) vn, rem = self.asrv.vfs.get(top, self.uname, True, False)
fsroot, vfs_ls, vfs_virt = vn.ls( fsroot, vfs_ls, vfs_virt = vn.ls(
rem, self.uname, not self.args.no_scandir, incl_wo=True rem, self.uname, not self.args.no_scandir, incl_wo=True
) )
@ -1515,7 +1514,7 @@ class HttpCli(object):
vpnodes.append([quotep(vpath) + "/", html_escape(node, crlf=True)]) vpnodes.append([quotep(vpath) + "/", html_escape(node, crlf=True)])
vn, rem = self.auth.vfs.get( vn, rem = self.asrv.vfs.get(
self.vpath, self.uname, self.readable, self.writable self.vpath, self.uname, self.readable, self.writable
) )
abspath = vn.canonical(rem) abspath = vn.canonical(rem)

View file

@ -34,7 +34,7 @@ class HttpConn(object):
self.hsrv = hsrv self.hsrv = hsrv
self.args = hsrv.args self.args = hsrv.args
self.auth = hsrv.auth self.asrv = hsrv.asrv
self.is_mp = hsrv.is_mp self.is_mp = hsrv.is_mp
self.cert_path = hsrv.cert_path self.cert_path = hsrv.cert_path
@ -71,7 +71,7 @@ class HttpConn(object):
def get_u2idx(self): def get_u2idx(self):
if not self.u2idx: if not self.u2idx:
self.u2idx = U2idx(self.args, self.log_func, self.auth.vfs) self.u2idx = U2idx(self)
return self.u2idx return self.u2idx

View file

@ -40,7 +40,7 @@ class HttpSrv(object):
self.is_mp = is_mp self.is_mp = is_mp
self.args = broker.args self.args = broker.args
self.log = broker.log self.log = broker.log
self.auth = broker.authsrv self.asrv = broker.asrv
self.disconnect_func = None self.disconnect_func = None
self.mutex = threading.Lock() self.mutex = threading.Lock()

View file

@ -37,14 +37,13 @@ class SvcHub(object):
self.log = self._log_disabled if args.q else self._log_enabled self.log = self._log_disabled if args.q else self._log_enabled
# jank goes here
auth = AuthSrv(self.args, self.log, False)
if args.ls:
auth.dbg_ls()
# initiate all services to manage # initiate all services to manage
self.asrv = AuthSrv(self.args, self.log, False)
if args.ls:
self.asrv.dbg_ls()
self.tcpsrv = TcpSrv(self) self.tcpsrv = TcpSrv(self)
self.up2k = Up2k(self, auth.vfs) self.up2k = Up2k(self)
self.thumbsrv = None self.thumbsrv = None
if not args.no_thumb: if not args.no_thumb:
@ -54,7 +53,7 @@ class SvcHub(object):
msg = "setting --th-no-webp because either libwebp is not available or your Pillow is too old" msg = "setting --th-no-webp because either libwebp is not available or your Pillow is too old"
self.log("thumb", msg, c=3) self.log("thumb", msg, c=3)
self.thumbsrv = ThumbSrv(self, auth.vfs) self.thumbsrv = ThumbSrv(self)
else: else:
msg = "need Pillow to create thumbnails; for example:\n{}{} -m pip install --user Pillow\n" msg = "need Pillow to create thumbnails; for example:\n{}{} -m pip install --user Pillow\n"
self.log( self.log(

View file

@ -11,7 +11,7 @@ class ThumbCli(object):
def __init__(self, broker): def __init__(self, broker):
self.broker = broker self.broker = broker
self.args = broker.args self.args = broker.args
self.hist = broker.authsrv.vfs.histtab self.asrv = broker.asrv
# cache on both sides for less broker spam # cache on both sides for less broker spam
self.cooldown = Cooldown(self.args.th_poke) self.cooldown = Cooldown(self.args.th_poke)
@ -32,8 +32,8 @@ class ThumbCli(object):
if self.args.th_no_webp or (is_vid and self.args.th_ff_jpg): if self.args.th_no_webp or (is_vid and self.args.th_ff_jpg):
fmt = "j" fmt = "j"
hist = self.hist[ptop] histpath = self.asrv.vfs.histtab[ptop]
tpath = thumb_path(hist, rem, mtime, fmt) tpath = thumb_path(histpath, rem, mtime, fmt)
ret = None ret = None
try: try:
st = os.stat(tpath) st = os.stat(tpath)

View file

@ -2,7 +2,6 @@
from __future__ import print_function, unicode_literals from __future__ import print_function, unicode_literals
import os import os
import sys
import time import time
import shutil import shutil
import base64 import base64
@ -74,7 +73,7 @@ if HAVE_FFMPEG and HAVE_FFPROBE:
THUMBABLE.update(FMT_FF) THUMBABLE.update(FMT_FF)
def thumb_path(hist, rem, mtime, fmt): def thumb_path(histpath, rem, mtime, fmt):
# base16 = 16 = 256 # base16 = 16 = 256
# b64-lc = 38 = 1444 # b64-lc = 38 = 1444
# base64 = 64 = 4096 # base64 = 64 = 4096
@ -96,16 +95,14 @@ def thumb_path(hist, rem, mtime, fmt):
fn = base64.urlsafe_b64encode(h).decode("ascii")[:24] fn = base64.urlsafe_b64encode(h).decode("ascii")[:24]
return "{}/th/{}/{}.{:x}.{}".format( return "{}/th/{}/{}.{:x}.{}".format(
hist, rd, fn, int(mtime), "webp" if fmt == "w" else "jpg" histpath, rd, fn, int(mtime), "webp" if fmt == "w" else "jpg"
) )
class ThumbSrv(object): class ThumbSrv(object):
def __init__(self, hub, vfs): def __init__(self, hub):
self.hub = hub self.hub = hub
self.vols = [v.realpath for v in vfs.all_vols.values()] self.asrv = hub.asrv
self.hist = vfs.histtab
self.args = hub.args self.args = hub.args
self.log_func = hub.log self.log_func = hub.log
@ -154,8 +151,8 @@ class ThumbSrv(object):
return not self.nthr return not self.nthr
def get(self, ptop, rem, mtime, fmt): def get(self, ptop, rem, mtime, fmt):
hist = self.hist[ptop] histpath = self.asrv.vfs.histtab[ptop]
tpath = thumb_path(hist, rem, mtime, fmt) tpath = thumb_path(histpath, rem, mtime, fmt)
abspath = os.path.join(ptop, rem) abspath = os.path.join(ptop, rem)
cond = threading.Condition() cond = threading.Condition()
with self.mutex: with self.mutex:
@ -331,29 +328,29 @@ class ThumbSrv(object):
interval = self.args.th_clean interval = self.args.th_clean
while True: while True:
time.sleep(interval) time.sleep(interval)
for vol, hist in self.hist.items(): for vol, histpath in self.asrv.vfs.histtab.items():
if hist.startswith(vol): if histpath.startswith(vol):
self.log("\033[Jcln {}/\033[A".format(hist)) self.log("\033[Jcln {}/\033[A".format(histpath))
else: else:
self.log("\033[Jcln {} ({})/\033[A".format(hist, vol)) self.log("\033[Jcln {} ({})/\033[A".format(histpath, vol))
self.clean(hist) self.clean(histpath)
self.log("\033[Jcln ok") self.log("\033[Jcln ok")
def clean(self, hist): def clean(self, histpath):
# self.log("cln {}".format(hist)) # self.log("cln {}".format(histpath))
maxage = self.args.th_maxage maxage = self.args.th_maxage
now = time.time() now = time.time()
prev_b64 = None prev_b64 = None
prev_fp = None prev_fp = None
try: try:
ents = os.listdir(hist) ents = os.listdir(histpath)
except: except:
return return
for f in sorted(ents): for f in sorted(ents):
fp = os.path.join(hist, f) fp = os.path.join(histpath, f)
cmp = fp.lower().replace("\\", "/") cmp = fp.lower().replace("\\", "/")
# "top" or b64 prefix/full (a folder) # "top" or b64 prefix/full (a folder)

View file

@ -19,12 +19,11 @@ except:
class U2idx(object): class U2idx(object):
def __init__(self, args, log_func, vfs): def __init__(self, conn):
self.args = args self.log_func = conn.log_func
self.log_func = log_func self.asrv = conn.asrv
self.vfs = vfs self.args = conn.args
self.timeout = self.args.srch_time
self.timeout = args.srch_time
if not HAVE_SQLITE3: if not HAVE_SQLITE3:
self.log("could not load sqlite3; searchign wqill be disabled") self.log("could not load sqlite3; searchign wqill be disabled")
@ -62,7 +61,8 @@ class U2idx(object):
if cur: if cur:
return cur return cur
db_path = os.path.join(self.vfs.histtab[ptop], "up2k.db") histpath = self.asrv.vfs.histtab[ptop]
db_path = os.path.join(histpath, "up2k.db")
if not os.path.exists(db_path): if not os.path.exists(db_path):
return None return None

View file

@ -48,10 +48,9 @@ class Up2k(object):
* ~/.config flatfiles for active jobs * ~/.config flatfiles for active jobs
""" """
def __init__(self, hub, vfs): def __init__(self, hub):
self.hub = hub self.hub = hub
self.vfs = vfs self.asrv = hub.asrv
# TODO stop passing around vfs, do auth or broker instead
self.args = hub.args self.args = hub.args
self.log_func = hub.log self.log_func = hub.log
@ -96,17 +95,17 @@ class Up2k(object):
self.log("could not initialize sqlite3, will use in-memory registry only") self.log("could not initialize sqlite3, will use in-memory registry only")
if self.args.no_fastboot: if self.args.no_fastboot:
self.deferred_init(vfs.all_vols) self.deferred_init()
else: else:
t = threading.Thread( t = threading.Thread(
target=self.deferred_init, target=self.deferred_init,
args=(vfs.all_vols,),
name="up2k-deferred-init", name="up2k-deferred-init",
) )
t.daemon = True t.daemon = True
t.start() t.start()
def deferred_init(self, all_vols): def deferred_init(self):
all_vols = self.asrv.vfs.all_vols
have_e2d = self.init_indexes(all_vols) have_e2d = self.init_indexes(all_vols)
if have_e2d: if have_e2d:
@ -300,7 +299,8 @@ class Up2k(object):
return have_e2d return have_e2d
def register_vpath(self, ptop, flags): def register_vpath(self, ptop, flags):
db_path = os.path.join(self.vfs.histtab[ptop], "up2k.db") histpath = self.asrv.vfs.histtab[ptop]
db_path = os.path.join(histpath, "up2k.db")
if ptop in self.registry: if ptop in self.registry:
try: try:
return [self.cur[ptop], db_path] return [self.cur[ptop], db_path]
@ -320,7 +320,7 @@ class Up2k(object):
self.log(" ".join(sorted(a)) + "\033[0m") self.log(" ".join(sorted(a)) + "\033[0m")
reg = {} reg = {}
path = os.path.join(self.vfs.histtab[ptop], "up2k.snap") path = os.path.join(histpath, "up2k.snap")
if "e2d" in flags and os.path.exists(path): if "e2d" in flags and os.path.exists(path):
with gzip.GzipFile(path, "rb") as f: with gzip.GzipFile(path, "rb") as f:
j = f.read().decode("utf-8") j = f.read().decode("utf-8")
@ -344,7 +344,7 @@ class Up2k(object):
return None return None
try: try:
os.makedirs(self.vfs.histtab[ptop]) os.makedirs(histpath)
except: except:
pass pass
@ -386,7 +386,7 @@ class Up2k(object):
def _build_dir(self, dbw, top, excl, cdir, nohash): def _build_dir(self, dbw, top, excl, cdir, nohash):
self.pp.msg = "a{} {}".format(self.pp.n, cdir) self.pp.msg = "a{} {}".format(self.pp.n, cdir)
histdir = self.vfs.histtab[top] histpath = self.asrv.vfs.histtab[top]
ret = 0 ret = 0
g = statdir(self.log, not self.args.no_scandir, False, cdir) g = statdir(self.log, not self.args.no_scandir, False, cdir)
for iname, inf in sorted(g): for iname, inf in sorted(g):
@ -394,7 +394,7 @@ class Up2k(object):
lmod = int(inf.st_mtime) lmod = int(inf.st_mtime)
sz = inf.st_size sz = inf.st_size
if stat.S_ISDIR(inf.st_mode): if stat.S_ISDIR(inf.st_mode):
if abspath in excl or abspath == histdir: if abspath in excl or abspath == histpath:
continue continue
# self.log(" dir: {}".format(abspath)) # self.log(" dir: {}".format(abspath))
ret += self._build_dir(dbw, top, excl, abspath, nohash) ret += self._build_dir(dbw, top, excl, abspath, nohash)
@ -1358,11 +1358,12 @@ class Up2k(object):
for k, reg in self.registry.items(): for k, reg in self.registry.items():
self._snap_reg(prev, k, reg, discard_interval) self._snap_reg(prev, k, reg, discard_interval)
def _snap_reg(self, prev, k, reg, discard_interval): def _snap_reg(self, prev, ptop, reg, discard_interval):
now = time.time() now = time.time()
histpath = self.asrv.vfs.histtab[ptop]
rm = [x for x in reg.values() if now - x["poke"] > discard_interval] rm = [x for x in reg.values() if now - x["poke"] > discard_interval]
if rm: if rm:
m = "dropping {} abandoned uploads in {}".format(len(rm), k) m = "dropping {} abandoned uploads in {}".format(len(rm), ptop)
vis = [self._vis_job_progress(x) for x in rm] vis = [self._vis_job_progress(x) for x in rm]
self.log("\n".join([m] + vis)) self.log("\n".join([m] + vis))
for job in rm: for job in rm:
@ -1380,21 +1381,21 @@ class Up2k(object):
except: except:
pass pass
path = os.path.join(self.vfs.histtab[k], "up2k.snap") path = os.path.join(histpath, "up2k.snap")
if not reg: if not reg:
if k not in prev or prev[k] is not None: if ptop not in prev or prev[ptop] is not None:
prev[k] = None prev[ptop] = None
if os.path.exists(fsenc(path)): if os.path.exists(fsenc(path)):
os.unlink(fsenc(path)) os.unlink(fsenc(path))
return return
newest = max(x["poke"] for _, x in reg.items()) if reg else 0 newest = max(x["poke"] for _, x in reg.items()) if reg else 0
etag = [len(reg), newest] etag = [len(reg), newest]
if etag == prev.get(k, None): if etag == prev.get(ptop, None):
return return
try: try:
os.makedirs(self.vfs.histtab[k]) os.makedirs(histpath)
except: except:
pass pass
@ -1406,7 +1407,7 @@ class Up2k(object):
atomic_move(path2, path) atomic_move(path2, path)
self.log("snap: {} |{}|".format(path, len(reg.keys()))) self.log("snap: {} |{}|".format(path, len(reg.keys())))
prev[k] = etag prev[ptop] = etag
def _tagger(self): def _tagger(self):
while True: while True:

View file

@ -11,6 +11,7 @@ import subprocess as sp
class Cpp(object): class Cpp(object):
def __init__(self, args): def __init__(self, args):
self.ls_pre = set(list(os.listdir()))
self.p = sp.Popen([sys.executable, "-m", "copyparty"] + args) self.p = sp.Popen([sys.executable, "-m", "copyparty"] + args)
# , stdout=sp.PIPE, stderr=sp.PIPE) # , stdout=sp.PIPE, stderr=sp.PIPE)
@ -27,27 +28,29 @@ class Cpp(object):
if wait: if wait:
self.t.join() self.t.join()
def clean(self):
def main(): t = os.listdir()
t1 = set(list(os.listdir())) for f in t:
try: if f not in self.ls_pre and f.startswith("up."):
main2()
finally:
t2 = os.listdir()
for f in t2:
if f not in t1 and f.startswith("up."):
os.unlink(f) os.unlink(f)
def main2(): def tc1():
ub = "http://127.0.0.1:4321/" ub = "http://127.0.0.1:4321/"
td = os.path.join("srv", "smoketest") td = os.path.join("srv", "smoketest")
try: try:
shutil.rmtree(td) shutil.rmtree(td)
except: except:
pass if os.path.exists(td):
raise
os.mkdir(td) for _ in range(10):
try:
os.mkdir(td)
except:
time.sleep(0.1) # win10
assert os.path.exists(td)
vidp = os.path.join(tempfile.gettempdir(), "smoketest.h264") vidp = os.path.join(tempfile.gettempdir(), "smoketest.h264")
if not os.path.exists(vidp): if not os.path.exists(vidp):
@ -57,8 +60,17 @@ def main2():
with open(vidp, "rb") as f: with open(vidp, "rb") as f:
ovid = f.read() ovid = f.read()
args = ["-p", "4321", "-e2dsa", "-e2tsr", "--th-ff-jpg"] args = [
"-p",
"4321",
"-e2dsa",
"-e2tsr",
"--th-ff-jpg",
"--hist",
os.path.join(td, "dbm"),
]
pdirs = [] pdirs = []
hpaths = {}
for d1 in ["r", "w", "a"]: for d1 in ["r", "w", "a"]:
pdirs.append("{}/{}".format(td, d1)) pdirs.append("{}/{}".format(td, d1))
@ -72,8 +84,19 @@ def main2():
udirs = [x.split("/", 2)[2] for x in pdirs] udirs = [x.split("/", 2)[2] for x in pdirs]
perms = [x.rstrip("j/")[-1] for x in pdirs] perms = [x.rstrip("j/")[-1] for x in pdirs]
for pd, ud, p in zip(pdirs, udirs, perms): for pd, ud, p in zip(pdirs, udirs, perms):
# args += ["-v", "{}:{}:{}".format(d.split("/", 1)[1], d, d[-1])] if ud[-1] == "j":
args += ["-v", "{}:{}:{}".format(pd, ud, p)] continue
hp = None
if pd.endswith("st/a"):
hp = os.path.join(td, "db1")
elif pd[:-1].endswith("a/j/"):
hp = os.path.join(td, "dbm")
else:
hp = "-"
hpaths[ud] = os.path.join(pd, ".hist") if hp == "-" else hp
args += ["-v", "{}:{}:{}:chist={}".format(pd, ud, p, hp)]
# print(repr(args)) # print(repr(args))
# return # return
@ -98,6 +121,9 @@ def main2():
except: except:
pass pass
cpp.clean()
# GET permission
for d, p in zip(udirs, perms): for d, p in zip(udirs, perms):
u = "{}{}/a.h264".format(ub, d) u = "{}{}/a.h264".format(ub, d)
r = requests.get(u) r = requests.get(u)
@ -105,12 +131,14 @@ def main2():
if ok != (p in ["a"]): if ok != (p in ["a"]):
raise Exception("get {} with perm {} at {}".format(ok, p, u)) raise Exception("get {} with perm {} at {}".format(ok, p, u))
# stat filesystem
for d, p in zip(pdirs, perms): for d, p in zip(pdirs, perms):
u = "{}/a.h264".format(d) u = "{}/a.h264".format(d)
ok = os.path.exists(u) ok = os.path.exists(u)
if ok != (p in ["a", "w"]): if ok != (p in ["a", "w"]):
raise Exception("stat {} with perm {} at {}".format(ok, p, u)) raise Exception("stat {} with perm {} at {}".format(ok, p, u))
# GET thumbnail, vreify contents
for d, p in zip(udirs, perms): for d, p in zip(udirs, perms):
u = "{}{}/a.h264?th=j".format(ub, d) u = "{}{}/a.h264?th=j".format(ub, d)
r = requests.get(u) r = requests.get(u)
@ -118,8 +146,34 @@ def main2():
if ok != (p in ["a"]): if ok != (p in ["a"]):
raise Exception("thumb {} with perm {} at {}".format(ok, p, u)) raise Exception("thumb {} with perm {} at {}".format(ok, p, u))
# check tags
for d, p in zip(udirs, perms):
u = "{}{}?ls".format(ub, d)
r = requests.get(u)
j = r.json() if r else False
tag = None
if j:
for f in j["files"]:
tag = tag or f["tags"].get("res")
r_ok = bool(j)
w_ok = bool(r_ok and j.get("files"))
if not r_ok or w_ok != (p in ["a"]):
raise Exception("ls {} with perm {} at {}".format(ok, p, u))
if (tag and p != "a") or (not tag and p == "a"):
raise Exception("tag {} with perm {} at {}".format(ok, p, u))
if tag is not None and tag != "48x32":
raise Exception("tag [{}] at {}".format(tag, u))
cpp.stop(True) cpp.stop(True)
def main():
tc1()
if __name__ == "__main__": if __name__ == "__main__":
main() main()

View file

@ -101,7 +101,7 @@ class TestHttpCli(unittest.TestCase):
pprint.pprint(vcfg) pprint.pprint(vcfg)
self.args = Cfg(v=vcfg, a=["o:o", "x:x"]) self.args = Cfg(v=vcfg, a=["o:o", "x:x"])
self.auth = AuthSrv(self.args, self.log) self.asrv = AuthSrv(self.args, self.log)
vfiles = [x for x in allfiles if x.startswith(top)] vfiles = [x for x in allfiles if x.startswith(top)]
for fp in vfiles: for fp in vfiles:
rok, wok = self.can_rw(fp) rok, wok = self.can_rw(fp)
@ -190,12 +190,12 @@ class TestHttpCli(unittest.TestCase):
def put(self, url): def put(self, url):
buf = "PUT /{0} HTTP/1.1\r\nCookie: cppwd=o\r\nConnection: close\r\nContent-Length: {1}\r\n\r\nok {0}\n" buf = "PUT /{0} HTTP/1.1\r\nCookie: cppwd=o\r\nConnection: close\r\nContent-Length: {1}\r\n\r\nok {0}\n"
buf = buf.format(url, len(url) + 4).encode("utf-8") buf = buf.format(url, len(url) + 4).encode("utf-8")
conn = tu.VHttpConn(self.args, self.auth, self.log, buf) conn = tu.VHttpConn(self.args, self.asrv, self.log, buf)
HttpCli(conn).run() HttpCli(conn).run()
return conn.s._reply.decode("utf-8").split("\r\n\r\n", 1) return conn.s._reply.decode("utf-8").split("\r\n\r\n", 1)
def curl(self, url, binary=False): def curl(self, url, binary=False):
conn = tu.VHttpConn(self.args, self.auth, self.log, hdr(url)) conn = tu.VHttpConn(self.args, self.asrv, self.log, hdr(url))
HttpCli(conn).run() HttpCli(conn).run()
if binary: if binary:
h, b = conn.s._reply.split(b"\r\n\r\n", 1) h, b = conn.s._reply.split(b"\r\n\r\n", 1)

View file

@ -110,12 +110,12 @@ class VHttpSrv(object):
class VHttpConn(object): class VHttpConn(object):
def __init__(self, args, auth, log, buf): def __init__(self, args, asrv, log, buf):
self.s = VSock(buf) self.s = VSock(buf)
self.sr = Unrecv(self.s) self.sr = Unrecv(self.s)
self.addr = ("127.0.0.1", "42069") self.addr = ("127.0.0.1", "42069")
self.args = args self.args = args
self.auth = auth self.asrv = asrv
self.is_mp = False self.is_mp = False
self.log_func = log self.log_func = log
self.log_src = "a" self.log_src = "a"