add unpost

This commit is contained in:
ed 2021-07-29 23:53:08 +02:00
parent 0c625a4e62
commit c164fc58a2
12 changed files with 323 additions and 44 deletions

View file

@ -264,9 +264,12 @@ def run_argparse(argv, formatter):
ap2.add_argument("-ed", action="store_true", help="enable ?dots")
ap2.add_argument("-emp", action="store_true", help="enable markdown plugins")
ap2.add_argument("-mcr", metavar="SEC", type=int, default=60, help="md-editor mod-chk rate")
ap2.add_argument("--urlform", metavar="MODE", type=u, default="print,get", help="how to handle url-forms; examples: [stash], [save,get]")
ap2 = ap.add_argument_group('upload options')
ap2.add_argument("--dotpart", action="store_true", help="dotfile incomplete uploads")
ap2.add_argument("--sparse", metavar="MiB", type=int, default=4, help="up2k min.size threshold (mswin-only)")
ap2.add_argument("--urlform", metavar="MODE", type=u, default="print,get", help="how to handle url-forms; examples: [stash], [save,get]")
ap2.add_argument("--unpost", metavar="SEC", type=int, default=3600*12, help="grace period where uploads can be deleted by the uploader, even without delete permissions; 0=disabled")
ap2 = ap.add_argument_group('network options')
ap2.add_argument("-i", metavar="IP", type=u, default="0.0.0.0", help="ip to bind (comma-sep.)")
@ -319,25 +322,27 @@ def run_argparse(argv, formatter):
ap2.add_argument("--th-maxage", metavar="SEC", type=int, default=604800, help="max folder age")
ap2.add_argument("--th-covers", metavar="N,N", type=u, default="folder.png,folder.jpg,cover.png,cover.jpg", help="folder thumbnails to stat for")
ap2 = ap.add_argument_group('database options')
ap2 = ap.add_argument_group('general db options')
ap2.add_argument("-e2d", action="store_true", help="enable up2k database")
ap2.add_argument("-e2ds", action="store_true", help="enable up2k db-scanner, sets -e2d")
ap2.add_argument("-e2dsa", action="store_true", help="scan all folders (for search), sets -e2ds")
ap2.add_argument("--hist", metavar="PATH", type=u, help="where to store volume state")
ap2.add_argument("--no-hash", action="store_true", help="disable hashing during e2ds folder scans")
ap2.add_argument("--re-int", metavar="SEC", type=int, default=30, help="disk rescan check interval")
ap2.add_argument("--re-maxage", metavar="SEC", type=int, default=0, help="disk rescan volume interval (0=off)")
ap2.add_argument("--srch-time", metavar="SEC", type=int, default=30, help="search deadline")
ap2 = ap.add_argument_group('metadata db options')
ap2.add_argument("-e2t", action="store_true", help="enable metadata indexing")
ap2.add_argument("-e2ts", action="store_true", help="enable metadata scanner, sets -e2t")
ap2.add_argument("-e2tsr", action="store_true", help="rescan all metadata, sets -e2ts")
ap2.add_argument("--hist", metavar="PATH", type=u, help="where to store volume state")
ap2.add_argument("--no-hash", action="store_true", help="disable hashing during e2ds folder scans")
ap2.add_argument("--no-mutagen", action="store_true", help="use FFprobe for tags instead")
ap2.add_argument("--no-mtag-mt", action="store_true", help="disable tag-read parallelism")
ap2.add_argument("--no-mtag-ff", action="store_true", help="never use FFprobe as tag reader")
ap2.add_argument("--re-int", metavar="SEC", type=int, default=30, help="disk rescan check interval")
ap2.add_argument("--re-maxage", metavar="SEC", type=int, default=0, help="disk rescan volume interval (0=off)")
ap2.add_argument("-mtm", metavar="M=t,t,t", type=u, action="append", help="add/replace metadata mapping")
ap2.add_argument("-mte", metavar="M,M,M", type=u, help="tags to index/display (comma-sep.)",
default="circle,album,.tn,artist,title,.bpm,key,.dur,.q,.vq,.aq,ac,vc,res,.fps")
ap2.add_argument("-mtp", metavar="M=[f,]bin", type=u, action="append", help="read tag M using bin")
ap2.add_argument("--srch-time", metavar="SEC", type=int, default=30, help="search deadline")
ap2 = ap.add_argument_group('appearance options')
ap2.add_argument("--css-browser", metavar="L", type=u, help="URL to additional CSS to include")

View file

@ -795,7 +795,7 @@ class AuthSrv(object):
atop = vn.realpath
g = vn.walk(
"", "", [], u, True, [[True]], not self.args.no_scandir, False
vn.vpath, "", [], u, [[True]], True, not self.args.no_scandir, False
)
for _, _, vpath, apath, files, _, _ in g:
fnames = [n[0] for n in files]

View file

@ -61,7 +61,10 @@ class HttpCli(object):
a, b = m.groups()
return "=\033[7m {} \033[27m{}".format(self.asrv.iacct[a], b)
def _check_nonfatal(self, ex):
def _check_nonfatal(self, ex, post):
if post:
return ex.code < 300
return ex.code < 400 or ex.code in [404, 429]
def _assert_safe_rem(self, rem):
@ -103,7 +106,7 @@ class HttpCli(object):
self.req = "[junk]"
self.http_ver = "HTTP/1.1"
# self.log("pebkac at httpcli.run #1: " + repr(ex))
self.keepalive = self._check_nonfatal(ex)
self.keepalive = False
self.loud_reply(unicode(ex), status=ex.code)
return self.keepalive
@ -216,7 +219,8 @@ class HttpCli(object):
except Pebkac as ex:
try:
# self.log("pebkac at httpcli.run #2: " + repr(ex))
if not self._check_nonfatal(ex):
post = self.mode in ["POST", "PUT"] or "content-length" in self.headers
if not self._check_nonfatal(ex, post):
self.keepalive = False
self.log("{}\033[0m, {}".format(str(ex), self.vpath), 3)
@ -345,7 +349,7 @@ class HttpCli(object):
if "tree" in self.uparam:
return self.tx_tree()
if "stack" in self.uparam:
if not self.vpath and "stack" in self.uparam:
return self.tx_stack()
# conditional redirect to single volumes
@ -377,13 +381,16 @@ class HttpCli(object):
if "move" in self.uparam:
return self.handle_mv()
if "h" in self.uparam:
self.vpath = None
return self.tx_mounts()
if "scan" in self.uparam:
return self.scanvol()
if not self.vpath:
if "h" in self.uparam:
return self.tx_mounts()
if "ups" in self.uparam:
return self.tx_ups()
return self.tx_browser()
def handle_options(self):
@ -599,6 +606,9 @@ class HttpCli(object):
if "srch" in self.uparam or "srch" in body:
return self.handle_search(body)
if "delete" in self.uparam:
return self.handle_rm(body)
# up2k-php compat
for k in "chunkpit.php", "handshake.php":
if self.vpath.endswith(k):
@ -1551,14 +1561,52 @@ class HttpCli(object):
ret["a"] = dirs
return ret
def handle_rm(self):
if not self.can_delete:
def tx_ups(self):
if not self.args.unpost:
raise Pebkac(400, "the unpost feature was disabled by server config")
filt = self.uparam.get("filter")
lm = "ups [{}]".format(filt)
self.log(lm)
ret = []
t0 = time.time()
idx = self.conn.get_u2idx()
lim = time.time() - self.args.unpost
for vol in self.asrv.vfs.all_vols.values():
cur = idx.get_cur(vol.realpath)
if not cur:
continue
q = "select sz, rd, fn, at from up where ip=? and at>?"
for sz, rd, fn, at in cur.execute(q, (self.ip, lim)):
vp = "/" + "/".join([rd, fn]).strip("/")
if filt and filt not in vp:
continue
ret.append({"vp": vp, "sz": sz, "at": at})
if len(ret) > 3000:
ret.sort(key=lambda x: x["at"], reverse=True)
ret = ret[:2000]
ret.sort(key=lambda x: x["at"], reverse=True)
ret = ret[:2000]
jtxt = json.dumps(ret, indent=2, sort_keys=True).encode("utf-8", "replace")
self.log("{} #{} {:.2f}sec".format(lm, len(ret), time.time() - t0))
self.reply(jtxt, mime="application/json")
def handle_rm(self, req=None):
if not req and not self.can_delete:
raise Pebkac(403, "not allowed for user " + self.uname)
if self.args.no_del:
raise Pebkac(403, "disabled by argv")
x = self.conn.hsrv.broker.put(True, "up2k.handle_rm", self.uname, self.vpath)
if not req:
req = [self.vpath]
x = self.conn.hsrv.broker.put(True, "up2k.handle_rm", self.uname, self.ip, req)
self.loud_reply(x.get())
def handle_mv(self):
@ -1711,6 +1759,7 @@ class HttpCli(object):
"have_mv": (not self.args.no_mv),
"have_del": (not self.args.no_del),
"have_zip": (not self.args.no_zip),
"have_unpost": (self.args.unpost > 0),
"have_b_u": (self.can_write and self.uparam.get("b") == "u"),
"url_suf": url_suf,
"logues": logues,

View file

@ -1330,43 +1330,89 @@ class Up2k(object):
v = (wark, int(ts), sz, rd, fn, ip or "", int(at or 0))
db.execute(sql, v)
def handle_rm(self, uname, vpath):
permsets = [[True, False, False, True]]
vn, rem = self.asrv.vfs.get(vpath, uname, *permsets[0])
def handle_rm(self, uname, ip, vpaths):
n_files = 0
ok = {}
ng = {}
for vp in vpaths:
a, b, c = self._handle_rm(uname, ip, vp)
n_files += a
for k in b:
ok[k] = 1
for k in c:
ng[k] = 1
ng = {k: 1 for k in ng if k not in ok}
ok = len(ok)
ng = len(ng)
return "deleted {} files (and {}/{} folders)".format(n_files, ok, ok + ng)
def _handle_rm(self, uname, ip, vpath):
try:
permsets = [[True, False, False, True]]
vn, rem = self.asrv.vfs.get(vpath, uname, *permsets[0])
unpost = False
except:
# unpost with missing permissions? try read+write and verify with db
if not self.args.unpost:
raise Pebkac(400, "the unpost feature was disabled by server config")
unpost = True
permsets = [[True, True]]
vn, rem = self.asrv.vfs.get(vpath, uname, *permsets[0])
_, _, _, _, dip, dat = self._find_from_vpath(vn.realpath, rem)
m = "you cannot delete this: "
if not dip:
m += "file not found"
elif dip != ip:
m += "not uploaded by (You)"
elif dat < time.time() - self.args.unpost:
m += "uploaded too long ago"
else:
m = None
if m:
raise Pebkac(400, m)
ptop = vn.realpath
atop = vn.canonical(rem)
atop = vn.canonical(rem, False)
adir, fn = os.path.split(atop)
st = bos.lstat(atop)
scandir = not self.args.no_scandir
if stat.S_ISLNK(st.st_mode) or stat.S_ISREG(st.st_mode):
dbv, vrem = self.asrv.vfs.get(vpath, uname, *permsets[0])
dbv, vrem = dbv.get_dbv(vrem)
g = [[dbv, vrem, os.path.dirname(vpath), adir, [[fn, 0]], [], []]]
voldir = vsplit(vrem)[0]
vpath_dir = vsplit(vpath)[0]
g = [[dbv, voldir, vpath_dir, adir, [[fn, 0]], [], []]]
else:
g = vn.walk("", rem, [], uname, permsets, True, scandir, True)
if unpost:
raise Pebkac(400, "cannot unpost folders")
n_files = 0
for dbv, vrem, _, adir, files, rd, vd in g:
for fn in [x[0] for x in files]:
n_files += 1
abspath = os.path.join(adir, fn)
vpath = "{}/{}".format(vrem, fn).strip("/")
volpath = "{}/{}".format(vrem, fn).strip("/")
vpath = "{}/{}".format(dbv.vpath, volpath).strip("/")
self.log("rm {}\n {}".format(vpath, abspath))
_ = dbv.get(vrem, uname, *permsets[0])
_ = dbv.get(volpath, uname, *permsets[0])
with self.mutex:
try:
ptop = dbv.realpath
cur, wark, _, _, _, _ = self._find_from_vpath(ptop, vrem)
self._forget_file(ptop, vpath, cur, wark)
cur, wark, _, _, _, _ = self._find_from_vpath(ptop, volpath)
self._forget_file(ptop, volpath, cur, wark)
finally:
cur.connection.commit()
bos.unlink(abspath)
rm = rmdirs(self.log_func, scandir, True, atop)
ok = len(rm[0])
ng = len(rm[1])
return "deleted {} files (and {}/{} folders)".format(n_files, ok, ok + ng)
return n_files, rm[0], rm[1]
def handle_mv(self, uname, svp, dvp):
svn, srem = self.asrv.vfs.get(svp, uname, True, False, True)

View file

@ -1063,6 +1063,9 @@ def statdir(logger, scandir, lstat, top):
def rmdirs(logger, scandir, lstat, top):
if not os.path.exists(fsenc(top)) or not os.path.isdir(fsenc(top)):
top = os.path.dirname(top)
dirs = statdir(logger, scandir, lstat, top)
dirs = [x[0] for x in dirs if stat.S_ISDIR(x[1].st_mode)]
dirs = [os.path.join(top, x) for x in dirs]

View file

@ -78,6 +78,9 @@ pre, code, tt {
border-radius: .5em 0 0 .5em;
transition: left .3s, width .3s, padding .3s, opacity .3s;
}
#toast pre {
margin: 0;
}
#toast.vis {
right: 1.3em;
transform: unset;
@ -952,7 +955,8 @@ input.eq_gain {
color: #300;
background: #fea;
}
.opwide {
.opwide,
#op_unpost {
max-width: none;
margin-right: 1.5em;
}
@ -1054,6 +1058,16 @@ html.light #ggrid a:hover {
color: #015;
box-shadow: 0 .1em .5em #aaa;
}
#op_unpost {
padding: 1em;
}
#op_unpost td {
padding: .2em .4em;
}
#op_unpost a {
margin: 0;
padding: 0;
}
#pvol,
#barbuf,
#barpos,

View file

@ -59,6 +59,8 @@
</form>
</div>
<div id="op_unpost" class="opview opbox"></div>
<div id="op_up2k" class="opview"></div>
<div id="op_cfg" class="opview opbox opwide"></div>
@ -128,6 +130,7 @@
have_tags_idx = {{ have_tags_idx|tojson }},
have_mv = {{ have_mv|tojson }},
have_del = {{ have_del|tojson }},
have_unpost = {{ have_unpost|tojson }},
have_zip = {{ have_zip|tojson }};
</script>
<script src="/.cpr/util.js?_={{ ts }}"></script>

View file

@ -12,6 +12,7 @@ ebi('ops').innerHTML = (
'<a href="#" data-dest="" tt="close submenu">---</a>\n' +
(have_up2k_idx ? (
'<a href="#" data-perm="read" data-dest="search" tt="search for files by attributes, path/name, music tags, or any combination of those.$N$N&lt;code&gt;foo bar&lt;/code&gt; = must contain both foo and bar,$N&lt;code&gt;foo -bar&lt;/code&gt; = must contain foo but not bar,$N&lt;code&gt;^yana .opus$&lt;/code&gt; = must start with yana and have the opus extension">🔎</a>\n' +
(have_del && have_unpost ? '<a href="#" data-dest="unpost" tt="unpost: delete your recent uploads">🧯</a>\n' : '') +
'<a href="#" data-dest="up2k" tt="up2k: upload files (if you have write-access) or toggle into the search-mode and drag files onto the search button to see if they exist somewhere on the server">🚀</a>\n'
) : (
'<a href="#" data-perm="write" data-dest="up2k" tt="up2k: upload files with resume support (close your browser and drop the same files in later)">🚀</a>\n'
@ -213,17 +214,6 @@ function goto(dest) {
}
(function () {
goto();
var op = sread('opmode');
if (op !== null && op !== '.')
try {
goto(op);
}
catch (ex) { }
})();
var have_webp = null;
(function () {
var img = new Image();
@ -3435,6 +3425,160 @@ function ev_row_tgl(e) {
}
var unpost = (function () {
ebi('op_unpost').innerHTML = (
"you can delete your recent uploads below &ndash; click the fire-extinguisher icon to refresh" +
'<p>optional filter:&nbsp; URL must contain <input type="text" id="unpost_filt" size="20" /><a id="unpost_nofilt" href="#">clear filter</a></p>' +
'<div id="unpost"></div>'
);
var r = {},
ct = ebi('unpost'),
filt = ebi('unpost_filt');
r.files = [];
r.me = null;
r.load = function () {
var me = Date.now(),
html = [];
function unpost_load_cb() {
if (this.readyState != XMLHttpRequest.DONE)
return;
if (this.status !== 200) {
var msg = this.responseText;
toast.err(9, 'unpost-load failed:\n' + msg);
ebi('op_unpost').innerHTML = html.join('\n');
return;
}
var res = JSON.parse(this.responseText);
if (res.length) {
if (res.length == 2000)
html.push("<p>showing first 2000 files (use the filter)");
else
html.push("<p>" + res.length + " uploads can be deleted");
html.push(" &ndash; sorted by upload time &ndash; most recent first:</p>");
html.push("<table><thead><tr><td></td><td>time</td><td>size</td><td>file</td></tr></thead><tbody>");
}
else
html.push("<p>sike! no uploads " + (filt.value ? 'matching that filter' : '') + " are sufficiently recent</p>");
var mods = [1000, 100, 10];
for (var a = 0; a < res.length; a++) {
for (var b = 0; b < mods.length; b++)
if (a % mods[b] == 0 && res.length > a + mods[b] / 10)
html.push(
'<tr><td></td><td colspan="3" style="padding:.5em">' +
'<a me="' + me + '" class="n' + a + '" n2="' + (a + mods[b]) +
'" href="#">delete the next ' + Math.min(mods[b], res.length - a) + ' files below</a></td></tr>');
html.push(
'<tr><td><a me="' + me + '" class="n' + a + '" href="#">delete</a></td>' +
'<td>' + unix2iso(res[a].at) + '</td>' +
'<td>' + res[a].sz + '</td>' +
'<td>' + linksplit(res[a].vp).join(' ') + '</td></tr>');
}
html.push("</tbody></table>");
ct.innerHTML = html.join('\n');
r.files = res;
r.me = me;
}
var q = '/?ups';
if (filt.value)
q += '&filter=' + uricom_enc(filt.value, true);
var xhr = new XMLHttpRequest();
xhr.open('GET', q, true);
xhr.onreadystatechange = unpost_load_cb;
xhr.send();
ct.innerHTML = "<p><em>loading your recent uploads...</em></p>";
};
function unpost_delete_cb() {
if (this.readyState != XMLHttpRequest.DONE)
return;
if (this.status !== 200) {
var msg = this.responseText;
toast.err(9, 'unpost-delete failed:\n' + msg);
return;
}
for (var a = this.n; a < this.n2; a++) {
var o = QSA('#op_unpost a.n' + a);
for (var b = 0; b < o.length; b++) {
var o2 = o[b].closest('tr');
o2.parentNode.removeChild(o2);
}
}
toast.ok(5, this.responseText);
if (!QS('#op_unpost a[me]'))
ebi(goto_unpost());
}
ct.onclick = function (e) {
var tgt = e.target.closest('a[me]');
if (!tgt)
return;
if (!tgt.getAttribute('href'))
return;
var ame = tgt.getAttribute('me');
if (ame != r.me)
return toast.err(0, 'something broke, please try a refresh');
var n = parseInt(tgt.className.slice(1)),
n2 = parseInt(tgt.getAttribute('n2') || n + 1),
req = [];
for (var a = n; a < n2; a++)
if (QS('#op_unpost a.n' + a))
req.push(r.files[a].vp);
var links = QSA('#op_unpost a.n' + n);
for (var a = 0, aa = links.length; a < aa; a++) {
links[a].removeAttribute('href');
links[a].innerHTML = '[busy]';
}
toast.inf(0, "deleting " + req.length + " files...");
var xhr = new XMLHttpRequest();
xhr.n = n;
xhr.n2 = n2;
xhr.open('POST', '/?delete', true);
xhr.onreadystatechange = unpost_delete_cb;
xhr.send(JSON.stringify(req));
};
var tfilt = null;
filt.oninput = function () {
clearTimeout(tfilt);
tfilt = setTimeout(r.load, 250);
};
ebi('unpost_nofilt').onclick = function () {
filt.value = '';
r.load();
};
return r;
})();
function goto_unpost(e) {
unpost.load();
}
function reload_mp() {
if (mp && mp.au) {
mp.au.pause();

View file

@ -41,6 +41,9 @@ html, body {
text-shadow: 1px 1px 0 #000;
color: #fff;
}
#toast pre {
margin: 0;
}
#toastc {
display: inline-block;
position: absolute;

View file

@ -75,7 +75,7 @@ function set_jumpto() {
}
function jumpto(ev) {
var tgt = ev.target || ev.srcElement;
var tgt = ev.target;
var ln = null;
while (tgt && !ln) {
ln = tgt.getAttribute('data-ln');

View file

@ -1773,3 +1773,14 @@ if (QS('#op_up2k.act'))
goto_up2k();
apply_perms(perms);
(function () {
goto();
var op = sread('opmode');
if (op !== null && op !== '.')
try {
goto(op);
}
catch (ex) { }
})();

View file

@ -31,6 +31,7 @@ class Cfg(Namespace):
rproxy=0,
ed=False,
nw=False,
unpost=600,
no_mv=False,
no_del=False,
no_zip=False,