mirror of
https://github.com/9001/copyparty.git
synced 2025-08-17 09:02:15 -06:00
single authsrv instance per process
This commit is contained in:
parent
fbe656957d
commit
60ac68d000
9
.vscode/launch.json
vendored
9
.vscode/launch.json
vendored
|
@ -16,12 +16,9 @@
|
||||||
"-e2ts",
|
"-e2ts",
|
||||||
"-mtp",
|
"-mtp",
|
||||||
".bpm=f,bin/mtag/audio-bpm.py",
|
".bpm=f,bin/mtag/audio-bpm.py",
|
||||||
"-a",
|
"-aed:wark",
|
||||||
"ed:wark",
|
"-vsrv::r:aed:cnodupe",
|
||||||
"-v",
|
"-vdist:dist:r"
|
||||||
"srv::r:aed:cnodupe",
|
|
||||||
"-v",
|
|
||||||
"dist:dist:r"
|
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
|
|
@ -43,7 +43,9 @@ class VFS(object):
|
||||||
)
|
)
|
||||||
|
|
||||||
def get_all_vols(self, outdict):
|
def get_all_vols(self, outdict):
|
||||||
outdict[self.vpath] = self
|
if self.realpath:
|
||||||
|
outdict[self.vpath] = self
|
||||||
|
|
||||||
for v in self.nodes.values():
|
for v in self.nodes.values():
|
||||||
v.get_all_vols(outdict)
|
v.get_all_vols(outdict)
|
||||||
|
|
||||||
|
@ -60,7 +62,7 @@ class VFS(object):
|
||||||
return self.nodes[name].add(src, dst)
|
return self.nodes[name].add(src, dst)
|
||||||
|
|
||||||
vn = VFS(
|
vn = VFS(
|
||||||
os.path.join(self.realpath, name) if self.realpath else name,
|
os.path.join(self.realpath, name) if self.realpath else None,
|
||||||
"{}/{}".format(self.vpath, name).lstrip("/"),
|
"{}/{}".format(self.vpath, name).lstrip("/"),
|
||||||
self.uread,
|
self.uread,
|
||||||
self.uwrite,
|
self.uwrite,
|
||||||
|
|
|
@ -36,7 +36,7 @@ class MpWorker(object):
|
||||||
signal.signal(signal.SIGINT, self.signal_handler)
|
signal.signal(signal.SIGINT, self.signal_handler)
|
||||||
|
|
||||||
# starting to look like a good idea
|
# starting to look like a good idea
|
||||||
self.authsrv = AuthSrv(args, None, False)
|
self.asrv = AuthSrv(args, None, False)
|
||||||
|
|
||||||
# instantiate all services here (TODO: inheritance?)
|
# instantiate all services here (TODO: inheritance?)
|
||||||
self.httpsrv = HttpSrv(self, True)
|
self.httpsrv = HttpSrv(self, True)
|
||||||
|
|
|
@ -15,12 +15,10 @@ class BrokerThr(object):
|
||||||
self.hub = hub
|
self.hub = hub
|
||||||
self.log = hub.log
|
self.log = hub.log
|
||||||
self.args = hub.args
|
self.args = hub.args
|
||||||
|
self.asrv = hub.asrv
|
||||||
|
|
||||||
self.mutex = threading.Lock()
|
self.mutex = threading.Lock()
|
||||||
|
|
||||||
# starting to look like a good idea
|
|
||||||
self.authsrv = AuthSrv(self.args, None, False)
|
|
||||||
|
|
||||||
# instantiate all services here (TODO: inheritance?)
|
# instantiate all services here (TODO: inheritance?)
|
||||||
self.httpsrv = HttpSrv(self)
|
self.httpsrv = HttpSrv(self)
|
||||||
self.httpsrv.disconnect_func = self.httpdrop
|
self.httpsrv.disconnect_func = self.httpdrop
|
||||||
|
|
|
@ -42,7 +42,7 @@ class HttpCli(object):
|
||||||
self.addr = conn.addr # type: tuple[str, int]
|
self.addr = conn.addr # type: tuple[str, int]
|
||||||
self.args = conn.args
|
self.args = conn.args
|
||||||
self.is_mp = conn.is_mp
|
self.is_mp = conn.is_mp
|
||||||
self.auth = conn.auth # type: AuthSrv
|
self.asrv = conn.asrv # type: AuthSrv
|
||||||
self.ico = conn.ico
|
self.ico = conn.ico
|
||||||
self.thumbcli = conn.thumbcli
|
self.thumbcli = conn.thumbcli
|
||||||
self.log_func = conn.log_func
|
self.log_func = conn.log_func
|
||||||
|
@ -154,9 +154,9 @@ class HttpCli(object):
|
||||||
self.vpath = unquotep(vpath)
|
self.vpath = unquotep(vpath)
|
||||||
|
|
||||||
pwd = uparam.get("pw")
|
pwd = uparam.get("pw")
|
||||||
self.uname = self.auth.iuser.get(pwd, "*")
|
self.uname = self.asrv.iuser.get(pwd, "*")
|
||||||
self.rvol, self.wvol, self.avol = [[], [], []]
|
self.rvol, self.wvol, self.avol = [[], [], []]
|
||||||
self.auth.vfs.user_tree(self.uname, self.rvol, self.wvol, self.avol)
|
self.asrv.vfs.user_tree(self.uname, self.rvol, self.wvol, self.avol)
|
||||||
|
|
||||||
ua = self.headers.get("user-agent", "")
|
ua = self.headers.get("user-agent", "")
|
||||||
self.is_rclone = ua.startswith("rclone/")
|
self.is_rclone = ua.startswith("rclone/")
|
||||||
|
@ -321,9 +321,7 @@ class HttpCli(object):
|
||||||
self.redirect(vpath, flavor="redirecting to", use302=True)
|
self.redirect(vpath, flavor="redirecting to", use302=True)
|
||||||
return True
|
return True
|
||||||
|
|
||||||
self.readable, self.writable = self.conn.auth.vfs.can_access(
|
self.readable, self.writable = self.asrv.vfs.can_access(self.vpath, self.uname)
|
||||||
self.vpath, self.uname
|
|
||||||
)
|
|
||||||
if not self.readable and not self.writable:
|
if not self.readable and not self.writable:
|
||||||
if self.vpath:
|
if self.vpath:
|
||||||
self.log("inaccessible: [{}]".format(self.vpath))
|
self.log("inaccessible: [{}]".format(self.vpath))
|
||||||
|
@ -440,7 +438,7 @@ class HttpCli(object):
|
||||||
|
|
||||||
def dump_to_file(self):
|
def dump_to_file(self):
|
||||||
reader, remains = self.get_body_reader()
|
reader, remains = self.get_body_reader()
|
||||||
vfs, rem = self.conn.auth.vfs.get(self.vpath, self.uname, False, True)
|
vfs, rem = self.asrv.vfs.get(self.vpath, self.uname, False, True)
|
||||||
fdir = os.path.join(vfs.realpath, rem)
|
fdir = os.path.join(vfs.realpath, rem)
|
||||||
|
|
||||||
addr = self.ip.replace(":", ".")
|
addr = self.ip.replace(":", ".")
|
||||||
|
@ -509,7 +507,7 @@ class HttpCli(object):
|
||||||
if v is None:
|
if v is None:
|
||||||
raise Pebkac(422, "need zip or tar keyword")
|
raise Pebkac(422, "need zip or tar keyword")
|
||||||
|
|
||||||
vn, rem = self.auth.vfs.get(self.vpath, self.uname, True, False)
|
vn, rem = self.asrv.vfs.get(self.vpath, self.uname, True, False)
|
||||||
items = self.parser.require("files", 1024 * 1024)
|
items = self.parser.require("files", 1024 * 1024)
|
||||||
if not items:
|
if not items:
|
||||||
raise Pebkac(422, "need files list")
|
raise Pebkac(422, "need files list")
|
||||||
|
@ -559,7 +557,7 @@ class HttpCli(object):
|
||||||
self.vpath = "/".join([self.vpath, sub]).strip("/")
|
self.vpath = "/".join([self.vpath, sub]).strip("/")
|
||||||
body["name"] = name
|
body["name"] = name
|
||||||
|
|
||||||
vfs, rem = self.conn.auth.vfs.get(self.vpath, self.uname, False, True)
|
vfs, rem = self.asrv.vfs.get(self.vpath, self.uname, False, True)
|
||||||
dbv, vrem = vfs.get_dbv(rem)
|
dbv, vrem = vfs.get_dbv(rem)
|
||||||
|
|
||||||
body["vtop"] = dbv.vpath
|
body["vtop"] = dbv.vpath
|
||||||
|
@ -590,7 +588,7 @@ class HttpCli(object):
|
||||||
vols = []
|
vols = []
|
||||||
seen = {}
|
seen = {}
|
||||||
for vtop in self.rvol:
|
for vtop in self.rvol:
|
||||||
vfs, _ = self.conn.auth.vfs.get(vtop, self.uname, True, False)
|
vfs, _ = self.asrv.vfs.get(vtop, self.uname, True, False)
|
||||||
vfs = vfs.dbv or vfs
|
vfs = vfs.dbv or vfs
|
||||||
if vfs in seen:
|
if vfs in seen:
|
||||||
continue
|
continue
|
||||||
|
@ -651,7 +649,7 @@ class HttpCli(object):
|
||||||
except KeyError:
|
except KeyError:
|
||||||
raise Pebkac(400, "need hash and wark headers for binary POST")
|
raise Pebkac(400, "need hash and wark headers for binary POST")
|
||||||
|
|
||||||
vfs, _ = self.conn.auth.vfs.get(self.vpath, self.uname, False, True)
|
vfs, _ = self.asrv.vfs.get(self.vpath, self.uname, False, True)
|
||||||
ptop = (vfs.dbv or vfs).realpath
|
ptop = (vfs.dbv or vfs).realpath
|
||||||
|
|
||||||
x = self.conn.hsrv.broker.put(True, "up2k.handle_chunk", ptop, wark, chash)
|
x = self.conn.hsrv.broker.put(True, "up2k.handle_chunk", ptop, wark, chash)
|
||||||
|
@ -724,7 +722,7 @@ class HttpCli(object):
|
||||||
pwd = self.parser.require("cppwd", 64)
|
pwd = self.parser.require("cppwd", 64)
|
||||||
self.parser.drop()
|
self.parser.drop()
|
||||||
|
|
||||||
if pwd in self.auth.iuser:
|
if pwd in self.asrv.iuser:
|
||||||
msg = "login ok"
|
msg = "login ok"
|
||||||
dt = datetime.utcfromtimestamp(time.time() + 60 * 60 * 24 * 365)
|
dt = datetime.utcfromtimestamp(time.time() + 60 * 60 * 24 * 365)
|
||||||
exp = dt.strftime("%a, %d %b %Y %H:%M:%S GMT")
|
exp = dt.strftime("%a, %d %b %Y %H:%M:%S GMT")
|
||||||
|
@ -743,7 +741,7 @@ class HttpCli(object):
|
||||||
self.parser.drop()
|
self.parser.drop()
|
||||||
|
|
||||||
nullwrite = self.args.nw
|
nullwrite = self.args.nw
|
||||||
vfs, rem = self.conn.auth.vfs.get(self.vpath, self.uname, False, True)
|
vfs, rem = self.asrv.vfs.get(self.vpath, self.uname, False, True)
|
||||||
self._assert_safe_rem(rem)
|
self._assert_safe_rem(rem)
|
||||||
|
|
||||||
sanitized = sanitize_fn(new_dir)
|
sanitized = sanitize_fn(new_dir)
|
||||||
|
@ -772,7 +770,7 @@ class HttpCli(object):
|
||||||
self.parser.drop()
|
self.parser.drop()
|
||||||
|
|
||||||
nullwrite = self.args.nw
|
nullwrite = self.args.nw
|
||||||
vfs, rem = self.conn.auth.vfs.get(self.vpath, self.uname, False, True)
|
vfs, rem = self.asrv.vfs.get(self.vpath, self.uname, False, True)
|
||||||
self._assert_safe_rem(rem)
|
self._assert_safe_rem(rem)
|
||||||
|
|
||||||
if not new_file.endswith(".md"):
|
if not new_file.endswith(".md"):
|
||||||
|
@ -796,7 +794,7 @@ class HttpCli(object):
|
||||||
|
|
||||||
def handle_plain_upload(self):
|
def handle_plain_upload(self):
|
||||||
nullwrite = self.args.nw
|
nullwrite = self.args.nw
|
||||||
vfs, rem = self.conn.auth.vfs.get(self.vpath, self.uname, False, True)
|
vfs, rem = self.asrv.vfs.get(self.vpath, self.uname, False, True)
|
||||||
self._assert_safe_rem(rem)
|
self._assert_safe_rem(rem)
|
||||||
|
|
||||||
files = []
|
files = []
|
||||||
|
@ -937,7 +935,7 @@ class HttpCli(object):
|
||||||
raise Pebkac(400, "could not read lastmod from request")
|
raise Pebkac(400, "could not read lastmod from request")
|
||||||
|
|
||||||
nullwrite = self.args.nw
|
nullwrite = self.args.nw
|
||||||
vfs, rem = self.conn.auth.vfs.get(self.vpath, self.uname, False, True)
|
vfs, rem = self.asrv.vfs.get(self.vpath, self.uname, False, True)
|
||||||
self._assert_safe_rem(rem)
|
self._assert_safe_rem(rem)
|
||||||
|
|
||||||
# TODO:
|
# TODO:
|
||||||
|
@ -1400,9 +1398,10 @@ class HttpCli(object):
|
||||||
if self.args.no_rescan:
|
if self.args.no_rescan:
|
||||||
raise Pebkac(403, "disabled by argv")
|
raise Pebkac(403, "disabled by argv")
|
||||||
|
|
||||||
vn, _ = self.auth.vfs.get(self.vpath, self.uname, True, True)
|
vn, _ = self.asrv.vfs.get(self.vpath, self.uname, True, True)
|
||||||
|
|
||||||
|
args = [self.asrv.vfs.all_vols, [vn.vpath]]
|
||||||
|
|
||||||
args = [self.auth.vfs.all_vols, [vn.vpath]]
|
|
||||||
x = self.conn.hsrv.broker.put(True, "up2k.rescan", *args)
|
x = self.conn.hsrv.broker.put(True, "up2k.rescan", *args)
|
||||||
x = x.get()
|
x = x.get()
|
||||||
if not x:
|
if not x:
|
||||||
|
@ -1474,7 +1473,7 @@ class HttpCli(object):
|
||||||
ret["k" + quotep(excl)] = sub
|
ret["k" + quotep(excl)] = sub
|
||||||
|
|
||||||
try:
|
try:
|
||||||
vn, rem = self.auth.vfs.get(top, self.uname, True, False)
|
vn, rem = self.asrv.vfs.get(top, self.uname, True, False)
|
||||||
fsroot, vfs_ls, vfs_virt = vn.ls(
|
fsroot, vfs_ls, vfs_virt = vn.ls(
|
||||||
rem, self.uname, not self.args.no_scandir, incl_wo=True
|
rem, self.uname, not self.args.no_scandir, incl_wo=True
|
||||||
)
|
)
|
||||||
|
@ -1515,7 +1514,7 @@ class HttpCli(object):
|
||||||
|
|
||||||
vpnodes.append([quotep(vpath) + "/", html_escape(node, crlf=True)])
|
vpnodes.append([quotep(vpath) + "/", html_escape(node, crlf=True)])
|
||||||
|
|
||||||
vn, rem = self.auth.vfs.get(
|
vn, rem = self.asrv.vfs.get(
|
||||||
self.vpath, self.uname, self.readable, self.writable
|
self.vpath, self.uname, self.readable, self.writable
|
||||||
)
|
)
|
||||||
abspath = vn.canonical(rem)
|
abspath = vn.canonical(rem)
|
||||||
|
|
|
@ -34,7 +34,7 @@ class HttpConn(object):
|
||||||
self.hsrv = hsrv
|
self.hsrv = hsrv
|
||||||
|
|
||||||
self.args = hsrv.args
|
self.args = hsrv.args
|
||||||
self.auth = hsrv.auth
|
self.asrv = hsrv.asrv
|
||||||
self.is_mp = hsrv.is_mp
|
self.is_mp = hsrv.is_mp
|
||||||
self.cert_path = hsrv.cert_path
|
self.cert_path = hsrv.cert_path
|
||||||
|
|
||||||
|
@ -71,7 +71,7 @@ class HttpConn(object):
|
||||||
|
|
||||||
def get_u2idx(self):
|
def get_u2idx(self):
|
||||||
if not self.u2idx:
|
if not self.u2idx:
|
||||||
self.u2idx = U2idx(self.args, self.log_func, self.auth.vfs)
|
self.u2idx = U2idx(self)
|
||||||
|
|
||||||
return self.u2idx
|
return self.u2idx
|
||||||
|
|
||||||
|
|
|
@ -40,7 +40,7 @@ class HttpSrv(object):
|
||||||
self.is_mp = is_mp
|
self.is_mp = is_mp
|
||||||
self.args = broker.args
|
self.args = broker.args
|
||||||
self.log = broker.log
|
self.log = broker.log
|
||||||
self.auth = broker.authsrv
|
self.asrv = broker.asrv
|
||||||
|
|
||||||
self.disconnect_func = None
|
self.disconnect_func = None
|
||||||
self.mutex = threading.Lock()
|
self.mutex = threading.Lock()
|
||||||
|
|
|
@ -37,14 +37,13 @@ class SvcHub(object):
|
||||||
|
|
||||||
self.log = self._log_disabled if args.q else self._log_enabled
|
self.log = self._log_disabled if args.q else self._log_enabled
|
||||||
|
|
||||||
# jank goes here
|
|
||||||
auth = AuthSrv(self.args, self.log, False)
|
|
||||||
if args.ls:
|
|
||||||
auth.dbg_ls()
|
|
||||||
|
|
||||||
# initiate all services to manage
|
# initiate all services to manage
|
||||||
|
self.asrv = AuthSrv(self.args, self.log, False)
|
||||||
|
if args.ls:
|
||||||
|
self.asrv.dbg_ls()
|
||||||
|
|
||||||
self.tcpsrv = TcpSrv(self)
|
self.tcpsrv = TcpSrv(self)
|
||||||
self.up2k = Up2k(self, auth.vfs)
|
self.up2k = Up2k(self)
|
||||||
|
|
||||||
self.thumbsrv = None
|
self.thumbsrv = None
|
||||||
if not args.no_thumb:
|
if not args.no_thumb:
|
||||||
|
@ -54,7 +53,7 @@ class SvcHub(object):
|
||||||
msg = "setting --th-no-webp because either libwebp is not available or your Pillow is too old"
|
msg = "setting --th-no-webp because either libwebp is not available or your Pillow is too old"
|
||||||
self.log("thumb", msg, c=3)
|
self.log("thumb", msg, c=3)
|
||||||
|
|
||||||
self.thumbsrv = ThumbSrv(self, auth.vfs)
|
self.thumbsrv = ThumbSrv(self)
|
||||||
else:
|
else:
|
||||||
msg = "need Pillow to create thumbnails; for example:\n{}{} -m pip install --user Pillow\n"
|
msg = "need Pillow to create thumbnails; for example:\n{}{} -m pip install --user Pillow\n"
|
||||||
self.log(
|
self.log(
|
||||||
|
|
|
@ -11,7 +11,7 @@ class ThumbCli(object):
|
||||||
def __init__(self, broker):
|
def __init__(self, broker):
|
||||||
self.broker = broker
|
self.broker = broker
|
||||||
self.args = broker.args
|
self.args = broker.args
|
||||||
self.hist = broker.authsrv.vfs.histtab
|
self.asrv = broker.asrv
|
||||||
|
|
||||||
# cache on both sides for less broker spam
|
# cache on both sides for less broker spam
|
||||||
self.cooldown = Cooldown(self.args.th_poke)
|
self.cooldown = Cooldown(self.args.th_poke)
|
||||||
|
@ -32,8 +32,8 @@ class ThumbCli(object):
|
||||||
if self.args.th_no_webp or (is_vid and self.args.th_ff_jpg):
|
if self.args.th_no_webp or (is_vid and self.args.th_ff_jpg):
|
||||||
fmt = "j"
|
fmt = "j"
|
||||||
|
|
||||||
hist = self.hist[ptop]
|
histpath = self.asrv.vfs.histtab[ptop]
|
||||||
tpath = thumb_path(hist, rem, mtime, fmt)
|
tpath = thumb_path(histpath, rem, mtime, fmt)
|
||||||
ret = None
|
ret = None
|
||||||
try:
|
try:
|
||||||
st = os.stat(tpath)
|
st = os.stat(tpath)
|
||||||
|
|
|
@ -2,7 +2,6 @@
|
||||||
from __future__ import print_function, unicode_literals
|
from __future__ import print_function, unicode_literals
|
||||||
|
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
import time
|
import time
|
||||||
import shutil
|
import shutil
|
||||||
import base64
|
import base64
|
||||||
|
@ -74,7 +73,7 @@ if HAVE_FFMPEG and HAVE_FFPROBE:
|
||||||
THUMBABLE.update(FMT_FF)
|
THUMBABLE.update(FMT_FF)
|
||||||
|
|
||||||
|
|
||||||
def thumb_path(hist, rem, mtime, fmt):
|
def thumb_path(histpath, rem, mtime, fmt):
|
||||||
# base16 = 16 = 256
|
# base16 = 16 = 256
|
||||||
# b64-lc = 38 = 1444
|
# b64-lc = 38 = 1444
|
||||||
# base64 = 64 = 4096
|
# base64 = 64 = 4096
|
||||||
|
@ -96,16 +95,14 @@ def thumb_path(hist, rem, mtime, fmt):
|
||||||
fn = base64.urlsafe_b64encode(h).decode("ascii")[:24]
|
fn = base64.urlsafe_b64encode(h).decode("ascii")[:24]
|
||||||
|
|
||||||
return "{}/th/{}/{}.{:x}.{}".format(
|
return "{}/th/{}/{}.{:x}.{}".format(
|
||||||
hist, rd, fn, int(mtime), "webp" if fmt == "w" else "jpg"
|
histpath, rd, fn, int(mtime), "webp" if fmt == "w" else "jpg"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
class ThumbSrv(object):
|
class ThumbSrv(object):
|
||||||
def __init__(self, hub, vfs):
|
def __init__(self, hub):
|
||||||
self.hub = hub
|
self.hub = hub
|
||||||
self.vols = [v.realpath for v in vfs.all_vols.values()]
|
self.asrv = hub.asrv
|
||||||
self.hist = vfs.histtab
|
|
||||||
|
|
||||||
self.args = hub.args
|
self.args = hub.args
|
||||||
self.log_func = hub.log
|
self.log_func = hub.log
|
||||||
|
|
||||||
|
@ -154,8 +151,8 @@ class ThumbSrv(object):
|
||||||
return not self.nthr
|
return not self.nthr
|
||||||
|
|
||||||
def get(self, ptop, rem, mtime, fmt):
|
def get(self, ptop, rem, mtime, fmt):
|
||||||
hist = self.hist[ptop]
|
histpath = self.asrv.vfs.histtab[ptop]
|
||||||
tpath = thumb_path(hist, rem, mtime, fmt)
|
tpath = thumb_path(histpath, rem, mtime, fmt)
|
||||||
abspath = os.path.join(ptop, rem)
|
abspath = os.path.join(ptop, rem)
|
||||||
cond = threading.Condition()
|
cond = threading.Condition()
|
||||||
with self.mutex:
|
with self.mutex:
|
||||||
|
@ -331,29 +328,29 @@ class ThumbSrv(object):
|
||||||
interval = self.args.th_clean
|
interval = self.args.th_clean
|
||||||
while True:
|
while True:
|
||||||
time.sleep(interval)
|
time.sleep(interval)
|
||||||
for vol, hist in self.hist.items():
|
for vol, histpath in self.asrv.vfs.histtab.items():
|
||||||
if hist.startswith(vol):
|
if histpath.startswith(vol):
|
||||||
self.log("\033[Jcln {}/\033[A".format(hist))
|
self.log("\033[Jcln {}/\033[A".format(histpath))
|
||||||
else:
|
else:
|
||||||
self.log("\033[Jcln {} ({})/\033[A".format(hist, vol))
|
self.log("\033[Jcln {} ({})/\033[A".format(histpath, vol))
|
||||||
|
|
||||||
self.clean(hist)
|
self.clean(histpath)
|
||||||
|
|
||||||
self.log("\033[Jcln ok")
|
self.log("\033[Jcln ok")
|
||||||
|
|
||||||
def clean(self, hist):
|
def clean(self, histpath):
|
||||||
# self.log("cln {}".format(hist))
|
# self.log("cln {}".format(histpath))
|
||||||
maxage = self.args.th_maxage
|
maxage = self.args.th_maxage
|
||||||
now = time.time()
|
now = time.time()
|
||||||
prev_b64 = None
|
prev_b64 = None
|
||||||
prev_fp = None
|
prev_fp = None
|
||||||
try:
|
try:
|
||||||
ents = os.listdir(hist)
|
ents = os.listdir(histpath)
|
||||||
except:
|
except:
|
||||||
return
|
return
|
||||||
|
|
||||||
for f in sorted(ents):
|
for f in sorted(ents):
|
||||||
fp = os.path.join(hist, f)
|
fp = os.path.join(histpath, f)
|
||||||
cmp = fp.lower().replace("\\", "/")
|
cmp = fp.lower().replace("\\", "/")
|
||||||
|
|
||||||
# "top" or b64 prefix/full (a folder)
|
# "top" or b64 prefix/full (a folder)
|
||||||
|
|
|
@ -19,12 +19,11 @@ except:
|
||||||
|
|
||||||
|
|
||||||
class U2idx(object):
|
class U2idx(object):
|
||||||
def __init__(self, args, log_func, vfs):
|
def __init__(self, conn):
|
||||||
self.args = args
|
self.log_func = conn.log_func
|
||||||
self.log_func = log_func
|
self.asrv = conn.asrv
|
||||||
self.vfs = vfs
|
self.args = conn.args
|
||||||
|
self.timeout = self.args.srch_time
|
||||||
self.timeout = args.srch_time
|
|
||||||
|
|
||||||
if not HAVE_SQLITE3:
|
if not HAVE_SQLITE3:
|
||||||
self.log("could not load sqlite3; searchign wqill be disabled")
|
self.log("could not load sqlite3; searchign wqill be disabled")
|
||||||
|
@ -62,7 +61,8 @@ class U2idx(object):
|
||||||
if cur:
|
if cur:
|
||||||
return cur
|
return cur
|
||||||
|
|
||||||
db_path = os.path.join(self.vfs.histtab[ptop], "up2k.db")
|
histpath = self.asrv.vfs.histtab[ptop]
|
||||||
|
db_path = os.path.join(histpath, "up2k.db")
|
||||||
if not os.path.exists(db_path):
|
if not os.path.exists(db_path):
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
|
@ -48,10 +48,9 @@ class Up2k(object):
|
||||||
* ~/.config flatfiles for active jobs
|
* ~/.config flatfiles for active jobs
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, hub, vfs):
|
def __init__(self, hub):
|
||||||
self.hub = hub
|
self.hub = hub
|
||||||
self.vfs = vfs
|
self.asrv = hub.asrv
|
||||||
# TODO stop passing around vfs, do auth or broker instead
|
|
||||||
self.args = hub.args
|
self.args = hub.args
|
||||||
self.log_func = hub.log
|
self.log_func = hub.log
|
||||||
|
|
||||||
|
@ -96,17 +95,17 @@ class Up2k(object):
|
||||||
self.log("could not initialize sqlite3, will use in-memory registry only")
|
self.log("could not initialize sqlite3, will use in-memory registry only")
|
||||||
|
|
||||||
if self.args.no_fastboot:
|
if self.args.no_fastboot:
|
||||||
self.deferred_init(vfs.all_vols)
|
self.deferred_init()
|
||||||
else:
|
else:
|
||||||
t = threading.Thread(
|
t = threading.Thread(
|
||||||
target=self.deferred_init,
|
target=self.deferred_init,
|
||||||
args=(vfs.all_vols,),
|
|
||||||
name="up2k-deferred-init",
|
name="up2k-deferred-init",
|
||||||
)
|
)
|
||||||
t.daemon = True
|
t.daemon = True
|
||||||
t.start()
|
t.start()
|
||||||
|
|
||||||
def deferred_init(self, all_vols):
|
def deferred_init(self):
|
||||||
|
all_vols = self.asrv.vfs.all_vols
|
||||||
have_e2d = self.init_indexes(all_vols)
|
have_e2d = self.init_indexes(all_vols)
|
||||||
|
|
||||||
if have_e2d:
|
if have_e2d:
|
||||||
|
@ -300,7 +299,8 @@ class Up2k(object):
|
||||||
return have_e2d
|
return have_e2d
|
||||||
|
|
||||||
def register_vpath(self, ptop, flags):
|
def register_vpath(self, ptop, flags):
|
||||||
db_path = os.path.join(self.vfs.histtab[ptop], "up2k.db")
|
histpath = self.asrv.vfs.histtab[ptop]
|
||||||
|
db_path = os.path.join(histpath, "up2k.db")
|
||||||
if ptop in self.registry:
|
if ptop in self.registry:
|
||||||
try:
|
try:
|
||||||
return [self.cur[ptop], db_path]
|
return [self.cur[ptop], db_path]
|
||||||
|
@ -320,7 +320,7 @@ class Up2k(object):
|
||||||
self.log(" ".join(sorted(a)) + "\033[0m")
|
self.log(" ".join(sorted(a)) + "\033[0m")
|
||||||
|
|
||||||
reg = {}
|
reg = {}
|
||||||
path = os.path.join(self.vfs.histtab[ptop], "up2k.snap")
|
path = os.path.join(histpath, "up2k.snap")
|
||||||
if "e2d" in flags and os.path.exists(path):
|
if "e2d" in flags and os.path.exists(path):
|
||||||
with gzip.GzipFile(path, "rb") as f:
|
with gzip.GzipFile(path, "rb") as f:
|
||||||
j = f.read().decode("utf-8")
|
j = f.read().decode("utf-8")
|
||||||
|
@ -344,7 +344,7 @@ class Up2k(object):
|
||||||
return None
|
return None
|
||||||
|
|
||||||
try:
|
try:
|
||||||
os.makedirs(self.vfs.histtab[ptop])
|
os.makedirs(histpath)
|
||||||
except:
|
except:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
@ -386,7 +386,7 @@ class Up2k(object):
|
||||||
|
|
||||||
def _build_dir(self, dbw, top, excl, cdir, nohash):
|
def _build_dir(self, dbw, top, excl, cdir, nohash):
|
||||||
self.pp.msg = "a{} {}".format(self.pp.n, cdir)
|
self.pp.msg = "a{} {}".format(self.pp.n, cdir)
|
||||||
histdir = self.vfs.histtab[top]
|
histpath = self.asrv.vfs.histtab[top]
|
||||||
ret = 0
|
ret = 0
|
||||||
g = statdir(self.log, not self.args.no_scandir, False, cdir)
|
g = statdir(self.log, not self.args.no_scandir, False, cdir)
|
||||||
for iname, inf in sorted(g):
|
for iname, inf in sorted(g):
|
||||||
|
@ -394,7 +394,7 @@ class Up2k(object):
|
||||||
lmod = int(inf.st_mtime)
|
lmod = int(inf.st_mtime)
|
||||||
sz = inf.st_size
|
sz = inf.st_size
|
||||||
if stat.S_ISDIR(inf.st_mode):
|
if stat.S_ISDIR(inf.st_mode):
|
||||||
if abspath in excl or abspath == histdir:
|
if abspath in excl or abspath == histpath:
|
||||||
continue
|
continue
|
||||||
# self.log(" dir: {}".format(abspath))
|
# self.log(" dir: {}".format(abspath))
|
||||||
ret += self._build_dir(dbw, top, excl, abspath, nohash)
|
ret += self._build_dir(dbw, top, excl, abspath, nohash)
|
||||||
|
@ -1358,11 +1358,12 @@ class Up2k(object):
|
||||||
for k, reg in self.registry.items():
|
for k, reg in self.registry.items():
|
||||||
self._snap_reg(prev, k, reg, discard_interval)
|
self._snap_reg(prev, k, reg, discard_interval)
|
||||||
|
|
||||||
def _snap_reg(self, prev, k, reg, discard_interval):
|
def _snap_reg(self, prev, ptop, reg, discard_interval):
|
||||||
now = time.time()
|
now = time.time()
|
||||||
|
histpath = self.asrv.vfs.histtab[ptop]
|
||||||
rm = [x for x in reg.values() if now - x["poke"] > discard_interval]
|
rm = [x for x in reg.values() if now - x["poke"] > discard_interval]
|
||||||
if rm:
|
if rm:
|
||||||
m = "dropping {} abandoned uploads in {}".format(len(rm), k)
|
m = "dropping {} abandoned uploads in {}".format(len(rm), ptop)
|
||||||
vis = [self._vis_job_progress(x) for x in rm]
|
vis = [self._vis_job_progress(x) for x in rm]
|
||||||
self.log("\n".join([m] + vis))
|
self.log("\n".join([m] + vis))
|
||||||
for job in rm:
|
for job in rm:
|
||||||
|
@ -1380,21 +1381,21 @@ class Up2k(object):
|
||||||
except:
|
except:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
path = os.path.join(self.vfs.histtab[k], "up2k.snap")
|
path = os.path.join(histpath, "up2k.snap")
|
||||||
if not reg:
|
if not reg:
|
||||||
if k not in prev or prev[k] is not None:
|
if ptop not in prev or prev[ptop] is not None:
|
||||||
prev[k] = None
|
prev[ptop] = None
|
||||||
if os.path.exists(fsenc(path)):
|
if os.path.exists(fsenc(path)):
|
||||||
os.unlink(fsenc(path))
|
os.unlink(fsenc(path))
|
||||||
return
|
return
|
||||||
|
|
||||||
newest = max(x["poke"] for _, x in reg.items()) if reg else 0
|
newest = max(x["poke"] for _, x in reg.items()) if reg else 0
|
||||||
etag = [len(reg), newest]
|
etag = [len(reg), newest]
|
||||||
if etag == prev.get(k, None):
|
if etag == prev.get(ptop, None):
|
||||||
return
|
return
|
||||||
|
|
||||||
try:
|
try:
|
||||||
os.makedirs(self.vfs.histtab[k])
|
os.makedirs(histpath)
|
||||||
except:
|
except:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
@ -1406,7 +1407,7 @@ class Up2k(object):
|
||||||
atomic_move(path2, path)
|
atomic_move(path2, path)
|
||||||
|
|
||||||
self.log("snap: {} |{}|".format(path, len(reg.keys())))
|
self.log("snap: {} |{}|".format(path, len(reg.keys())))
|
||||||
prev[k] = etag
|
prev[ptop] = etag
|
||||||
|
|
||||||
def _tagger(self):
|
def _tagger(self):
|
||||||
while True:
|
while True:
|
||||||
|
|
|
@ -11,6 +11,7 @@ import subprocess as sp
|
||||||
|
|
||||||
class Cpp(object):
|
class Cpp(object):
|
||||||
def __init__(self, args):
|
def __init__(self, args):
|
||||||
|
self.ls_pre = set(list(os.listdir()))
|
||||||
self.p = sp.Popen([sys.executable, "-m", "copyparty"] + args)
|
self.p = sp.Popen([sys.executable, "-m", "copyparty"] + args)
|
||||||
# , stdout=sp.PIPE, stderr=sp.PIPE)
|
# , stdout=sp.PIPE, stderr=sp.PIPE)
|
||||||
|
|
||||||
|
@ -27,27 +28,29 @@ class Cpp(object):
|
||||||
if wait:
|
if wait:
|
||||||
self.t.join()
|
self.t.join()
|
||||||
|
|
||||||
|
def clean(self):
|
||||||
def main():
|
t = os.listdir()
|
||||||
t1 = set(list(os.listdir()))
|
for f in t:
|
||||||
try:
|
if f not in self.ls_pre and f.startswith("up."):
|
||||||
main2()
|
|
||||||
finally:
|
|
||||||
t2 = os.listdir()
|
|
||||||
for f in t2:
|
|
||||||
if f not in t1 and f.startswith("up."):
|
|
||||||
os.unlink(f)
|
os.unlink(f)
|
||||||
|
|
||||||
|
|
||||||
def main2():
|
def tc1():
|
||||||
ub = "http://127.0.0.1:4321/"
|
ub = "http://127.0.0.1:4321/"
|
||||||
td = os.path.join("srv", "smoketest")
|
td = os.path.join("srv", "smoketest")
|
||||||
try:
|
try:
|
||||||
shutil.rmtree(td)
|
shutil.rmtree(td)
|
||||||
except:
|
except:
|
||||||
pass
|
if os.path.exists(td):
|
||||||
|
raise
|
||||||
|
|
||||||
os.mkdir(td)
|
for _ in range(10):
|
||||||
|
try:
|
||||||
|
os.mkdir(td)
|
||||||
|
except:
|
||||||
|
time.sleep(0.1) # win10
|
||||||
|
|
||||||
|
assert os.path.exists(td)
|
||||||
|
|
||||||
vidp = os.path.join(tempfile.gettempdir(), "smoketest.h264")
|
vidp = os.path.join(tempfile.gettempdir(), "smoketest.h264")
|
||||||
if not os.path.exists(vidp):
|
if not os.path.exists(vidp):
|
||||||
|
@ -57,8 +60,17 @@ def main2():
|
||||||
with open(vidp, "rb") as f:
|
with open(vidp, "rb") as f:
|
||||||
ovid = f.read()
|
ovid = f.read()
|
||||||
|
|
||||||
args = ["-p", "4321", "-e2dsa", "-e2tsr", "--th-ff-jpg"]
|
args = [
|
||||||
|
"-p",
|
||||||
|
"4321",
|
||||||
|
"-e2dsa",
|
||||||
|
"-e2tsr",
|
||||||
|
"--th-ff-jpg",
|
||||||
|
"--hist",
|
||||||
|
os.path.join(td, "dbm"),
|
||||||
|
]
|
||||||
pdirs = []
|
pdirs = []
|
||||||
|
hpaths = {}
|
||||||
|
|
||||||
for d1 in ["r", "w", "a"]:
|
for d1 in ["r", "w", "a"]:
|
||||||
pdirs.append("{}/{}".format(td, d1))
|
pdirs.append("{}/{}".format(td, d1))
|
||||||
|
@ -72,8 +84,19 @@ def main2():
|
||||||
udirs = [x.split("/", 2)[2] for x in pdirs]
|
udirs = [x.split("/", 2)[2] for x in pdirs]
|
||||||
perms = [x.rstrip("j/")[-1] for x in pdirs]
|
perms = [x.rstrip("j/")[-1] for x in pdirs]
|
||||||
for pd, ud, p in zip(pdirs, udirs, perms):
|
for pd, ud, p in zip(pdirs, udirs, perms):
|
||||||
# args += ["-v", "{}:{}:{}".format(d.split("/", 1)[1], d, d[-1])]
|
if ud[-1] == "j":
|
||||||
args += ["-v", "{}:{}:{}".format(pd, ud, p)]
|
continue
|
||||||
|
|
||||||
|
hp = None
|
||||||
|
if pd.endswith("st/a"):
|
||||||
|
hp = os.path.join(td, "db1")
|
||||||
|
elif pd[:-1].endswith("a/j/"):
|
||||||
|
hp = os.path.join(td, "dbm")
|
||||||
|
else:
|
||||||
|
hp = "-"
|
||||||
|
|
||||||
|
hpaths[ud] = os.path.join(pd, ".hist") if hp == "-" else hp
|
||||||
|
args += ["-v", "{}:{}:{}:chist={}".format(pd, ud, p, hp)]
|
||||||
|
|
||||||
# print(repr(args))
|
# print(repr(args))
|
||||||
# return
|
# return
|
||||||
|
@ -98,6 +121,9 @@ def main2():
|
||||||
except:
|
except:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
cpp.clean()
|
||||||
|
|
||||||
|
# GET permission
|
||||||
for d, p in zip(udirs, perms):
|
for d, p in zip(udirs, perms):
|
||||||
u = "{}{}/a.h264".format(ub, d)
|
u = "{}{}/a.h264".format(ub, d)
|
||||||
r = requests.get(u)
|
r = requests.get(u)
|
||||||
|
@ -105,12 +131,14 @@ def main2():
|
||||||
if ok != (p in ["a"]):
|
if ok != (p in ["a"]):
|
||||||
raise Exception("get {} with perm {} at {}".format(ok, p, u))
|
raise Exception("get {} with perm {} at {}".format(ok, p, u))
|
||||||
|
|
||||||
|
# stat filesystem
|
||||||
for d, p in zip(pdirs, perms):
|
for d, p in zip(pdirs, perms):
|
||||||
u = "{}/a.h264".format(d)
|
u = "{}/a.h264".format(d)
|
||||||
ok = os.path.exists(u)
|
ok = os.path.exists(u)
|
||||||
if ok != (p in ["a", "w"]):
|
if ok != (p in ["a", "w"]):
|
||||||
raise Exception("stat {} with perm {} at {}".format(ok, p, u))
|
raise Exception("stat {} with perm {} at {}".format(ok, p, u))
|
||||||
|
|
||||||
|
# GET thumbnail, vreify contents
|
||||||
for d, p in zip(udirs, perms):
|
for d, p in zip(udirs, perms):
|
||||||
u = "{}{}/a.h264?th=j".format(ub, d)
|
u = "{}{}/a.h264?th=j".format(ub, d)
|
||||||
r = requests.get(u)
|
r = requests.get(u)
|
||||||
|
@ -118,8 +146,34 @@ def main2():
|
||||||
if ok != (p in ["a"]):
|
if ok != (p in ["a"]):
|
||||||
raise Exception("thumb {} with perm {} at {}".format(ok, p, u))
|
raise Exception("thumb {} with perm {} at {}".format(ok, p, u))
|
||||||
|
|
||||||
|
# check tags
|
||||||
|
for d, p in zip(udirs, perms):
|
||||||
|
u = "{}{}?ls".format(ub, d)
|
||||||
|
r = requests.get(u)
|
||||||
|
j = r.json() if r else False
|
||||||
|
tag = None
|
||||||
|
if j:
|
||||||
|
for f in j["files"]:
|
||||||
|
tag = tag or f["tags"].get("res")
|
||||||
|
|
||||||
|
r_ok = bool(j)
|
||||||
|
w_ok = bool(r_ok and j.get("files"))
|
||||||
|
|
||||||
|
if not r_ok or w_ok != (p in ["a"]):
|
||||||
|
raise Exception("ls {} with perm {} at {}".format(ok, p, u))
|
||||||
|
|
||||||
|
if (tag and p != "a") or (not tag and p == "a"):
|
||||||
|
raise Exception("tag {} with perm {} at {}".format(ok, p, u))
|
||||||
|
|
||||||
|
if tag is not None and tag != "48x32":
|
||||||
|
raise Exception("tag [{}] at {}".format(tag, u))
|
||||||
|
|
||||||
cpp.stop(True)
|
cpp.stop(True)
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
tc1()
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
main()
|
main()
|
||||||
|
|
|
@ -101,7 +101,7 @@ class TestHttpCli(unittest.TestCase):
|
||||||
pprint.pprint(vcfg)
|
pprint.pprint(vcfg)
|
||||||
|
|
||||||
self.args = Cfg(v=vcfg, a=["o:o", "x:x"])
|
self.args = Cfg(v=vcfg, a=["o:o", "x:x"])
|
||||||
self.auth = AuthSrv(self.args, self.log)
|
self.asrv = AuthSrv(self.args, self.log)
|
||||||
vfiles = [x for x in allfiles if x.startswith(top)]
|
vfiles = [x for x in allfiles if x.startswith(top)]
|
||||||
for fp in vfiles:
|
for fp in vfiles:
|
||||||
rok, wok = self.can_rw(fp)
|
rok, wok = self.can_rw(fp)
|
||||||
|
@ -190,12 +190,12 @@ class TestHttpCli(unittest.TestCase):
|
||||||
def put(self, url):
|
def put(self, url):
|
||||||
buf = "PUT /{0} HTTP/1.1\r\nCookie: cppwd=o\r\nConnection: close\r\nContent-Length: {1}\r\n\r\nok {0}\n"
|
buf = "PUT /{0} HTTP/1.1\r\nCookie: cppwd=o\r\nConnection: close\r\nContent-Length: {1}\r\n\r\nok {0}\n"
|
||||||
buf = buf.format(url, len(url) + 4).encode("utf-8")
|
buf = buf.format(url, len(url) + 4).encode("utf-8")
|
||||||
conn = tu.VHttpConn(self.args, self.auth, self.log, buf)
|
conn = tu.VHttpConn(self.args, self.asrv, self.log, buf)
|
||||||
HttpCli(conn).run()
|
HttpCli(conn).run()
|
||||||
return conn.s._reply.decode("utf-8").split("\r\n\r\n", 1)
|
return conn.s._reply.decode("utf-8").split("\r\n\r\n", 1)
|
||||||
|
|
||||||
def curl(self, url, binary=False):
|
def curl(self, url, binary=False):
|
||||||
conn = tu.VHttpConn(self.args, self.auth, self.log, hdr(url))
|
conn = tu.VHttpConn(self.args, self.asrv, self.log, hdr(url))
|
||||||
HttpCli(conn).run()
|
HttpCli(conn).run()
|
||||||
if binary:
|
if binary:
|
||||||
h, b = conn.s._reply.split(b"\r\n\r\n", 1)
|
h, b = conn.s._reply.split(b"\r\n\r\n", 1)
|
||||||
|
|
|
@ -110,12 +110,12 @@ class VHttpSrv(object):
|
||||||
|
|
||||||
|
|
||||||
class VHttpConn(object):
|
class VHttpConn(object):
|
||||||
def __init__(self, args, auth, log, buf):
|
def __init__(self, args, asrv, log, buf):
|
||||||
self.s = VSock(buf)
|
self.s = VSock(buf)
|
||||||
self.sr = Unrecv(self.s)
|
self.sr = Unrecv(self.s)
|
||||||
self.addr = ("127.0.0.1", "42069")
|
self.addr = ("127.0.0.1", "42069")
|
||||||
self.args = args
|
self.args = args
|
||||||
self.auth = auth
|
self.asrv = asrv
|
||||||
self.is_mp = False
|
self.is_mp = False
|
||||||
self.log_func = log
|
self.log_func = log
|
||||||
self.log_src = "a"
|
self.log_src = "a"
|
||||||
|
|
Loading…
Reference in a new issue