initial thumbnail and icon stuff

This commit is contained in:
ed 2021-05-25 03:37:01 +02:00
parent cbc449036f
commit 4dff726310
16 changed files with 533 additions and 45 deletions

View file

@ -93,6 +93,9 @@ you may also want these, especially on servers:
* ☑ tree-view
* ☑ media player
* ✖ thumbnails
* ☑ images
* ✖ videos
* ✖ cache eviction
* ☑ SPA (browse while uploading)
* if you use the file-tree on the left only, not folders in the file list
* server indexing
@ -405,9 +408,18 @@ quick outline of the up2k protocol, see [uploading](#uploading) for the web-clie
* either `mutagen` (fast, pure-python, skips a few tags, makes copyparty GPL? idk)
* or `FFprobe` (20x slower, more accurate, possibly dangerous depending on your distro and users)
**optional,** will eventually enable thumbnails:
**optional,** enables thumbnails:
* `Pillow` (requires py2.7 or py3.5+)
**optional,** enables reading HEIF pictures:
* `pyheif-pillow-opener` (requires Linux or a C compiler)
## install recommended dependencies
```
python -m pip install --user -U jinja2 mutagen Pillow
```
## optional gpl stuff
@ -481,6 +493,10 @@ in the `scripts` folder:
roughly sorted by priority
* mtag mediainfo (multitag)
* thumbnail expiration
* touch cachedir on access with cooldown
* drop dir if older than X and near maxsize
* drop outdated thumbs
* separate sqlite table per tag
* audio fingerprinting
* readme.md as epilogue
@ -488,7 +504,6 @@ roughly sorted by priority
* start from a chunk index and just go
* terminate client on bad data
* `os.copy_file_range` for up2k cloning
* support pillow-simd
* single sha512 across all up2k chunks? maybe
* figure out the deal with pixel3a not being connectable as hotspot
* pixel3a having unpredictable 3sec latency in general :||||

View file

@ -2,6 +2,7 @@
from __future__ import print_function, unicode_literals
import platform
import time
import sys
import os
@ -23,6 +24,7 @@ MACOS = platform.system() == "Darwin"
class EnvParams(object):
def __init__(self):
self.t0 = time.time()
self.mod = os.path.dirname(os.path.realpath(__file__))
if self.mod.endswith("__init__"):
self.mod = os.path.dirname(self.mod)

View file

@ -245,6 +245,7 @@ def run_argparse(argv, formatter):
ap.add_argument("-nid", action="store_true", help="no info disk-usage")
ap.add_argument("--dotpart", action="store_true", help="dotfile incomplete uploads")
ap.add_argument("--no-zip", action="store_true", help="disable download as zip/tar")
ap.add_argument("--no-thumb", action="store_true", help="disable thumbnails")
ap.add_argument("--sparse", metavar="MiB", type=int, default=4, help="up2k min.size threshold (mswin-only)")
ap.add_argument("--urlform", metavar="MODE", type=str, default="print,get", help="how to handle url-forms")
ap.add_argument("--salt", type=str, default="hunter2", help="up2k file-hash salt")

View file

@ -36,6 +36,8 @@ class HttpCli(object):
self.addr = conn.addr
self.args = conn.args
self.auth = conn.auth
self.ico = conn.ico
self.thumbcli = conn.thumbcli
self.log_func = conn.log_func
self.log_src = conn.log_src
self.tls = hasattr(self.s, "cipher")
@ -283,6 +285,9 @@ class HttpCli(object):
# "embedded" resources
if self.vpath.startswith(".cpr"):
if self.vpath.startswith(".cpr/ico/"):
return self.tx_ico(self.vpath.split("/")[-1])
static_path = os.path.join(E.mod, "web/", self.vpath[5:])
return self.tx_file(static_path)
@ -1112,7 +1117,7 @@ class HttpCli(object):
self.send_headers(
length=upper - lower,
status=status,
mime=guess_mime(req_path)[0] or "application/octet-stream",
mime=guess_mime(req_path)[0],
)
logmsg += unicode(status) + logtail
@ -1202,6 +1207,23 @@ class HttpCli(object):
self.log("{}, {}".format(logmsg, spd))
return True
def tx_ico(self, ext):
n = ext.split(".")[::-1]
ext = ""
for v in n:
if len(v) > 7:
break
ext = "{}.{}".format(v, ext)
ext = ext.rstrip(".") or "unk"
mime, ico = self.ico.get(ext)
dt = datetime.utcfromtimestamp(E.t0)
lm = dt.strftime("%a, %d %b %Y %H:%M:%S GMT")
self.reply(ico, mime=mime, headers={"Last-Modified": lm})
return True
def tx_md(self, fs_path):
logmsg = "{:4} {} ".format("", self.req)
@ -1346,10 +1368,30 @@ class HttpCli(object):
)
abspath = vn.canonical(rem)
if not os.path.exists(fsenc(abspath)):
# print(abspath)
try:
st = os.stat(fsenc(abspath))
except:
raise Pebkac(404)
if self.readable and not stat.S_ISDIR(st.st_mode):
if abspath.endswith(".md") and "raw" not in self.uparam:
return self.tx_md(abspath)
if rem.startswith(".hist/up2k."):
raise Pebkac(403)
if "th" in self.uparam:
thp = None
if self.thumbcli:
thp = self.thumbcli.get(vn.realpath, rem, int(st.st_mtime))
if thp:
return self.tx_file(thp)
return self.tx_ico(rem)
return self.tx_file(abspath)
srv_info = []
try:
@ -1431,22 +1473,13 @@ class HttpCli(object):
self.reply(ret.encode("utf-8", "replace"), mime="application/json")
return True
if not os.path.isdir(fsenc(abspath)):
if not stat.S_ISDIR(st.st_mode):
raise Pebkac(404)
html = self.j2(tpl, **j2a)
self.reply(html.encode("utf-8", "replace"))
return True
if not os.path.isdir(fsenc(abspath)):
if abspath.endswith(".md") and "raw" not in self.uparam:
return self.tx_md(abspath)
if rem.startswith(".hist/up2k."):
raise Pebkac(403)
return self.tx_file(abspath)
for k in ["zip", "tar"]:
v = self.uparam.get(k)
if v is not None:

View file

@ -17,6 +17,9 @@ from .__init__ import E
from .util import Unrecv
from .httpcli import HttpCli
from .u2idx import U2idx
from .th_cli import ThumbCli
from .th_srv import HAVE_PIL
from .ico import Ico
class HttpConn(object):
@ -34,6 +37,10 @@ class HttpConn(object):
self.auth = hsrv.auth
self.cert_path = hsrv.cert_path
enth = HAVE_PIL and not self.args.no_thumb
self.thumbcli = ThumbCli(hsrv.broker) if enth else None
self.ico = Ico()
self.t0 = time.time()
self.nbyte = 0
self.workload = 0

35
copyparty/ico.py Normal file
View file

@ -0,0 +1,35 @@
import hashlib
import colorsys
class Ico(object):
def __init__(self):
pass
def get(self, ext):
"""placeholder to make thumbnails not break"""
if False:
h = hashlib.md5(ext.encode("utf-8")).digest()[:6]
lo = [int(x / 3) for x in h]
hi = [int(x / 3 + 170) for x in h]
c = lo[:3] + hi[3:6]
else:
h = hashlib.md5(ext.encode("utf-8")).digest()[:2]
c1 = colorsys.hsv_to_rgb(h[0] / 256.0, 1, 0.3)
c2 = colorsys.hsv_to_rgb(h[0] / 256.0, 1, 1)
c = list(c1) + list(c2)
c = [int(x * 255) for x in c]
c = "".join(["{:02x}".format(x) for x in c])
svg = """\
<?xml version="1.0" encoding="UTF-8"?>
<svg version="1.1" viewBox="0 0 100 30" xmlns="http://www.w3.org/2000/svg"><g>
<rect width="100%" height="100%" fill="#{}" />
<text x="50%" y="50%" dominant-baseline="middle" text-anchor="middle" fill="#{}" font-family="sans-serif" font-size="16px" xml:space="preserve">{}</text>
</g></svg>
"""
svg = svg.format(c[:6], c[6:], ext).encode("utf-8")
return ["image/svg+xml", svg]

View file

@ -9,9 +9,10 @@ from datetime import datetime, timedelta
import calendar
from .__init__ import PY2, WINDOWS, MACOS, VT100
from .util import mp
from .tcpsrv import TcpSrv
from .up2k import Up2k
from .util import mp
from .th_srv import ThumbSrv, HAVE_PIL
class SvcHub(object):
@ -38,6 +39,9 @@ class SvcHub(object):
self.tcpsrv = TcpSrv(self)
self.up2k = Up2k(self)
enth = HAVE_PIL and not args.no_thumb
self.thumbsrv = ThumbSrv(self) if enth else None
# decide which worker impl to use
if self.check_mp_enable():
from .broker_mp import BrokerMp as Broker
@ -63,6 +67,13 @@ class SvcHub(object):
self.tcpsrv.shutdown()
self.broker.shutdown()
if self.thumbsrv:
self.thumbsrv.shutdown()
print("waiting for thumbsrv...")
while not self.thumbsrv.stopped():
time.sleep(0.05)
print("nailed it")
def _log_disabled(self, src, msg, c=0):

21
copyparty/th_cli.py Normal file
View file

@ -0,0 +1,21 @@
import os
from .th_srv import thumb_path, THUMBABLE
class ThumbCli(object):
def __init__(self, broker):
self.broker = broker
self.args = broker.args
def get(self, ptop, rem, mtime):
ext = rem.rsplit(".")[-1].lower()
if ext not in THUMBABLE:
return None
tpath = thumb_path(ptop, rem, mtime)
if os.path.exists(tpath):
return tpath
x = self.broker.put(True, "thumbsrv.get", ptop, rem, mtime)
return x.get()

152
copyparty/th_srv.py Normal file
View file

@ -0,0 +1,152 @@
import os
import base64
import hashlib
import threading
try:
HAVE_PIL = True
from PIL import Image
try:
HAVE_HEIF = True
from pyheif_pillow_opener import register_heif_opener
register_heif_opener()
except:
HAVE_HEIF = False
except:
HAVE_PIL = False
from .util import fsenc, Queue
# https://pillow.readthedocs.io/en/stable/handbook/image-file-formats.html
FMT_PIL = "bmp dib gif icns ico jpg jpeg jp2 jpx pcx png pbm pgm ppm pnm sgi tga tif tiff webp xbm dds xpm"
FMT_PIL = {x: True for x in FMT_PIL.split(" ") if x}
THUMBABLE = FMT_PIL
def thumb_path(ptop, rem, mtime):
# base16 = 16 = 256
# b64-lc = 38 = 1444
# base64 = 64 = 4096
try:
rd, fn = rem.rsplit("/", 1)
except:
rd = ""
fn = rem
if rd:
h = hashlib.sha512(fsenc(rd)).digest()[:24]
b64 = base64.urlsafe_b64encode(h).decode("ascii")[:24]
rd = "{}/{}/".format(b64[:2], b64[2:4]).lower() + b64
else:
rd = "top"
# could keep original filenames but this is safer re pathlen
h = hashlib.sha512(fsenc(fn)).digest()[:24]
fn = base64.urlsafe_b64encode(h).decode("ascii")[:24]
return "{}/.hist/th/{}/{}.{:x}.jpg".format(ptop, rd, fn, int(mtime))
class ThumbSrv(object):
def __init__(self, hub):
self.hub = hub
self.log_func = hub.log
self.mutex = threading.Lock()
self.busy = {}
self.stopping = False
self.nthr = os.cpu_count() if hasattr(os, "cpu_count") else 4
self.q = Queue(self.nthr * 4)
for _ in range(self.nthr):
t = threading.Thread(target=self.worker)
t.daemon = True
t.start()
def log(self, msg, c=0):
self.log_func("thumb", msg, c)
def shutdown(self):
self.stopping = True
for _ in range(self.nthr):
self.q.put(None)
def stopped(self):
with self.mutex:
return not self.nthr
def get(self, ptop, rem, mtime):
tpath = thumb_path(ptop, rem, mtime)
abspath = os.path.join(ptop, rem)
cond = threading.Condition()
with self.mutex:
try:
self.busy[tpath].append(cond)
self.log("conv {}".format(tpath))
except:
thdir = os.path.dirname(tpath)
try:
os.makedirs(thdir)
except:
pass
inf_path = os.path.join(thdir, "dir.txt")
if not os.path.exists(inf_path):
with open(inf_path, "wb") as f:
f.write(fsenc(os.path.dirname(abspath)))
self.busy[tpath] = [cond]
self.q.put([abspath, tpath])
self.log("CONV {}".format(tpath))
while not self.stopping:
with self.mutex:
if tpath not in self.busy:
break
with cond:
cond.wait()
if not os.path.exists(tpath):
return None
return tpath
def worker(self):
while not self.stopping:
task = self.q.get()
if not task:
break
abspath, tpath = task
ext = abspath.split(".")[-1].lower()
fun = None
if not os.path.exists(tpath):
if ext in FMT_PIL:
fun = self.conv_pil
if fun:
fun(abspath, tpath)
with self.mutex:
subs = self.busy[tpath]
del self.busy[tpath]
for x in subs:
with x:
x.notify_all()
with self.mutex:
self.nthr -= 1
def conv_pil(self, abspath, tpath):
try:
with Image.open(abspath) as im:
if im.mode in ("RGBA", "P"):
im = im.convert("RGB")
im.thumbnail((256, 256))
im.save(tpath)
except:
pass

View file

@ -49,13 +49,13 @@ class Up2k(object):
* ~/.config flatfiles for active jobs
"""
def __init__(self, broker):
self.broker = broker
self.args = broker.args
self.log_func = broker.log
def __init__(self, hub):
self.hub = hub
self.args = hub.args
self.log_func = hub.log
# config
self.salt = broker.args.salt
self.salt = self.args.salt
# state
self.mutex = threading.Lock()

View file

@ -914,11 +914,11 @@ def unescape_cookie(orig):
return ret
def guess_mime(url):
def guess_mime(url, fallback="application/octet-stream"):
if url.endswith(".md"):
return ["text/plain; charset=UTF-8"]
return mimetypes.guess_type(url)
return mimetypes.guess_type(url) or fallback
def runcmd(*argv):

View file

@ -1,3 +1,6 @@
:root {
--grid-sz: 10em;
}
* {
line-height: 1.2em;
}
@ -550,8 +553,7 @@ input[type="checkbox"]:checked+label {
left: -1.7em;
width: calc(100% + 1.3em);
}
.tglbtn,
#tree>a+a {
.btn {
padding: .2em .4em;
font-size: 1.2em;
background: #2a2a2a;
@ -561,12 +563,10 @@ input[type="checkbox"]:checked+label {
position: relative;
top: -.2em;
}
.tglbtn:hover,
#tree>a+a:hover {
.btn:hover {
background: #805;
}
.tglbtn.on,
#tree>a+a.on {
.tgl.btn.on {
background: #fc4;
color: #400;
text-shadow: none;
@ -711,6 +711,40 @@ input[type="checkbox"]:checked+label {
font-family: monospace, monospace;
line-height: 2em;
}
#griden.on+#thumbs {
opacity: .3;
}
#ghead {
background: #3c3c3c;
border: 1px solid #444;
border-radius: .3em;
padding: .5em;
margin: 0 1.5em 0 .4em;
}
#ghead .btn {
position: relative;
top: 0;
}
#ggrid {
padding-top: .5em;
}
#ggrid a {
display: inline-block;
width: var(--grid-sz);
vertical-align: top;
overflow-wrap: break-word;
background: #383838;
border: 1px solid #444;
border-radius: .3em;
padding: .3em .6em;
margin: .5em;
}
#ggrid a img {
max-width: var(--grid-sz);
max-height: var(--grid-sz);
margin: 0 auto .5em auto;
display: block;
}
#pvol,
#barbuf,
#barpos,
@ -725,6 +759,21 @@ input[type="checkbox"]:checked+label {
html.light {
color: #333;
background: #eee;
@ -746,18 +795,15 @@ html.light #ops a.act {
html.light #op_cfg h3 {
border-color: #ccc;
}
html.light .tglbtn,
html.light #tree > a + a {
html.light .btn {
color: #666;
background: #ddd;
box-shadow: none;
}
html.light .tglbtn:hover,
html.light #tree > a + a:hover {
html.light .btn:hover {
background: #caf;
}
html.light .tglbtn.on,
html.light #tree > a + a.on {
html.light .tgl.btn.on {
background: #4a0;
color: #fff;
}

View file

@ -43,6 +43,8 @@
<div>
<a id="tooltips" class="tgl btn" href="#">tooltips</a>
<a id="lightmode" class="tgl btn" href="#">lightmode</a>
<a id="griden" class="tgl btn" href="#">the grid</a>
<a id="thumbs" class="tgl btn" href="#">thumbs</a>
</div>
{%- if have_zip %}
<h3>folder download</h3>
@ -61,8 +63,8 @@
<div id="tree">
<a href="#" id="detree">🍞...</a>
<a href="#" step="2" id="twobytwo">+</a>
<a href="#" step="-2" id="twig">&ndash;</a>
<a href="#" class="btn" step="2" id="twobytwo">+</a>
<a href="#" class="btn" step="-2" id="twig">&ndash;</a>
<a href="#" class="tgl btn" id="dyntree">a</a>
<ul id="treeul"></ul>
<div id="thx_ff">&nbsp;</div>

View file

@ -696,6 +696,157 @@ function autoplay_blocked(seek) {
})();
var thegrid = (function () {
var lfiles = ebi('files');
var gfiles = document.createElement('div');
gfiles.setAttribute('id', 'gfiles');
gfiles.style.display = 'none';
gfiles.innerHTML = (
'<div id="ghead">' +
'<a href="#" class="tgl btn" id="gridsel">multiselect</a> &nbsp; zoom ' +
'<a href="#" class="btn" z="-1.2">&ndash;</a> ' +
'<a href="#" class="btn" z="1.2">+</a> &nbsp; sort by: ' +
'<a href="#" s="href">name</a>, ' +
'<a href="#" s="sz">size</a>, ' +
'<a href="#" s="ts">date</a>, ' +
'<a href="#" s="ext">type</a>' +
'</div>' +
'<div id="ggrid"></div>'
);
lfiles.parentNode.insertBefore(gfiles, lfiles);
var r = {
'thumbs': bcfg_get('thumbs', true),
'en': bcfg_get('griden', false),
'sel': bcfg_get('gridsel', false),
'sz': fcfg_get('gridsz', 10),
'isdirty': true
};
ebi('thumbs').onclick = function (e) {
ev(e);
r.thumbs = !r.thumbs;
bcfg_set('thumbs', r.thumbs);
if (r.en) {
loadgrid();
}
};
ebi('griden').onclick = function (e) {
ev(e);
r.en = !r.en;
bcfg_set('griden', r.en);
if (r.en) {
loadgrid();
}
else {
lfiles.style.display = '';
gfiles.style.display = 'none';
}
};
var click = function (e) {
ev(e);
var s = this.getAttribute('s'),
z = this.getAttribute('z');
if (z)
return setsz(z > 0 ? r.sz * z : r.sz / (-z));
var t = lfiles.tHead.rows[0].cells;
for (var a = 0; a < t.length; a++)
if (t[a].getAttribute('name') == s) {
t[a].click();
break;
}
r.setdirty();
};
var links = QSA('#ghead>a');
for (var a = 0; a < links.length; a++)
links[a].onclick = click;
ebi('gridsel').onclick = function (e) {
ev(e);
r.sel = !r.sel;
bcfg_set('gridsel', r.sel);
};
r.setvis = function (vis) {
(r.en ? gfiles : lfiles).style.display = vis ? '' : 'none';
}
r.setdirty = function () {
r.dirty = true;
if (r.en) {
loadgrid();
}
}
function setsz(v) {
if (v !== undefined) {
r.sz = v;
swrite('gridsz', r.sz);
}
document.documentElement.style.setProperty('--grid-sz', r.sz + 'em');
}
setsz();
function loadgrid() {
if (!r.dirty)
return;
var html = [];
var tr = lfiles.tBodies[0].rows;
for (var a = 0; a < tr.length; a++) {
var ao = tr[a].cells[1].firstChild,
href = esc(ao.getAttribute('href')),
isdir = href.split('?')[0].slice(-1)[0] == '/',
ihref = href;
if (isdir) {
ihref = '/.cpr/ico/folder'
}
else if (r.thumbs) {
ihref += ihref.indexOf('?') === -1 ? '?th' : '&th';
}
else {
var ar = href.split('?')[0].split('.');
if (ar.length > 1)
ar = ar.slice(1);
ihref = '';
ar.reverse();
for (var b = 0; b < ar.length; b++) {
if (ar[b].length > 7)
break;
ihref = ar[b] + '.' + ihref;
}
if (!ihref) {
ihref = 'unk.';
}
ihref = '/.cpr/ico/' + ihref.slice(0, -1);
}
html.push('<a href="' + href +
'"><img src="' + ihref + '">' + // /.cpr/dd/1.png
ao.innerHTML + '</a>');
}
lfiles.style.display = 'none';
gfiles.style.display = 'block';
ebi('ggrid').innerHTML = html.join('\n');
}
if (r.en) {
loadgrid();
}
return r;
})();
function tree_neigh(n) {
var links = QSA('#treeul li>a+a');
if (!links.length) {
@ -763,6 +914,12 @@ document.onkeydown = function (e) {
if (k == 'KeyP')
return tree_up();
if (k == 'KeyG')
return ebi('griden').click();
if (k == 'KeyT')
return ebi('thumbs').click();
};
@ -1391,7 +1548,7 @@ function apply_perms(perms) {
up2k.set_fsearch();
ebi('widget').style.display = have_read ? '' : 'none';
ebi('files').style.display = have_read ? '' : 'none';
thegrid.setvis(have_read);
if (!have_read)
goto('up2k');
}
@ -1892,6 +2049,8 @@ function reload_browser(not_mp) {
if (window['up2k'])
up2k.set_fsearch();
thegrid.setdirty();
}
reload_browser(true);
mukey.render();

View file

@ -456,11 +456,15 @@ function jwrite(key, val) {
}
function icfg_get(name, defval) {
return parseInt(fcfg_get(name, defval));
}
function fcfg_get(name, defval) {
var o = ebi(name);
var val = parseInt(sread(name));
var val = parseFloat(sread(name));
if (isNaN(val))
return parseInt(o ? o.value : defval);
return parseFloat(o ? o.value : defval);
if (o)
o.value = val;

View file

@ -1,7 +1,7 @@
# recipe for building an exe with nuitka (extreme jank edition)
#
# NOTE: on win7 the exe immediately hits a c0000005 in kernelbase.dll
# but win10 builds work on win10
# NOTE: win7 and win10 builds both work on win10 but
# on win7 they immediately c0000005 in kernelbase.dll
#
# first install python-3.6.8-amd64.exe
# [x] add to path