diff --git a/README.md b/README.md index e821ae17..0fcb0d85 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,9 @@ you may also want these, especially on servers: * ☑ tree-view * ☑ media player * ✖ thumbnails + * ☑ images + * ✖ videos + * ✖ cache eviction * ☑ SPA (browse while uploading) * if you use the file-tree on the left only, not folders in the file list * server indexing @@ -405,9 +408,18 @@ quick outline of the up2k protocol, see [uploading](#uploading) for the web-clie * either `mutagen` (fast, pure-python, skips a few tags, makes copyparty GPL? idk) * or `FFprobe` (20x slower, more accurate, possibly dangerous depending on your distro and users) -**optional,** will eventually enable thumbnails: +**optional,** enables thumbnails: * `Pillow` (requires py2.7 or py3.5+) +**optional,** enables reading HEIF pictures: +* `pyheif-pillow-opener` (requires Linux or a C compiler) + + +## install recommended dependencies +``` +python -m pip install --user -U jinja2 mutagen Pillow +``` + ## optional gpl stuff @@ -481,6 +493,10 @@ in the `scripts` folder: roughly sorted by priority * mtag mediainfo (multitag) +* thumbnail expiration + * touch cachedir on access with cooldown + * drop dir if older than X and near maxsize + * drop outdated thumbs * separate sqlite table per tag * audio fingerprinting * readme.md as epilogue @@ -488,7 +504,6 @@ roughly sorted by priority * start from a chunk index and just go * terminate client on bad data * `os.copy_file_range` for up2k cloning -* support pillow-simd * single sha512 across all up2k chunks? maybe * figure out the deal with pixel3a not being connectable as hotspot * pixel3a having unpredictable 3sec latency in general :|||| diff --git a/copyparty/__init__.py b/copyparty/__init__.py index d82b655f..3e0ec43d 100644 --- a/copyparty/__init__.py +++ b/copyparty/__init__.py @@ -2,6 +2,7 @@ from __future__ import print_function, unicode_literals import platform +import time import sys import os @@ -23,6 +24,7 @@ MACOS = platform.system() == "Darwin" class EnvParams(object): def __init__(self): + self.t0 = time.time() self.mod = os.path.dirname(os.path.realpath(__file__)) if self.mod.endswith("__init__"): self.mod = os.path.dirname(self.mod) diff --git a/copyparty/__main__.py b/copyparty/__main__.py index 3c70e1f2..95f1ee78 100644 --- a/copyparty/__main__.py +++ b/copyparty/__main__.py @@ -245,6 +245,7 @@ def run_argparse(argv, formatter): ap.add_argument("-nid", action="store_true", help="no info disk-usage") ap.add_argument("--dotpart", action="store_true", help="dotfile incomplete uploads") ap.add_argument("--no-zip", action="store_true", help="disable download as zip/tar") + ap.add_argument("--no-thumb", action="store_true", help="disable thumbnails") ap.add_argument("--sparse", metavar="MiB", type=int, default=4, help="up2k min.size threshold (mswin-only)") ap.add_argument("--urlform", metavar="MODE", type=str, default="print,get", help="how to handle url-forms") ap.add_argument("--salt", type=str, default="hunter2", help="up2k file-hash salt") diff --git a/copyparty/httpcli.py b/copyparty/httpcli.py index 60e66cb1..e1da2ee0 100644 --- a/copyparty/httpcli.py +++ b/copyparty/httpcli.py @@ -36,6 +36,8 @@ class HttpCli(object): self.addr = conn.addr self.args = conn.args self.auth = conn.auth + self.ico = conn.ico + self.thumbcli = conn.thumbcli self.log_func = conn.log_func self.log_src = conn.log_src self.tls = hasattr(self.s, "cipher") @@ -283,6 +285,9 @@ class HttpCli(object): # "embedded" resources if self.vpath.startswith(".cpr"): + if self.vpath.startswith(".cpr/ico/"): + return self.tx_ico(self.vpath.split("/")[-1]) + static_path = os.path.join(E.mod, "web/", self.vpath[5:]) return self.tx_file(static_path) @@ -1112,7 +1117,7 @@ class HttpCli(object): self.send_headers( length=upper - lower, status=status, - mime=guess_mime(req_path)[0] or "application/octet-stream", + mime=guess_mime(req_path)[0], ) logmsg += unicode(status) + logtail @@ -1202,6 +1207,23 @@ class HttpCli(object): self.log("{}, {}".format(logmsg, spd)) return True + def tx_ico(self, ext): + n = ext.split(".")[::-1] + ext = "" + for v in n: + if len(v) > 7: + break + + ext = "{}.{}".format(v, ext) + + ext = ext.rstrip(".") or "unk" + mime, ico = self.ico.get(ext) + + dt = datetime.utcfromtimestamp(E.t0) + lm = dt.strftime("%a, %d %b %Y %H:%M:%S GMT") + self.reply(ico, mime=mime, headers={"Last-Modified": lm}) + return True + def tx_md(self, fs_path): logmsg = "{:4} {} ".format("", self.req) @@ -1346,10 +1368,30 @@ class HttpCli(object): ) abspath = vn.canonical(rem) - if not os.path.exists(fsenc(abspath)): - # print(abspath) + try: + st = os.stat(fsenc(abspath)) + except: raise Pebkac(404) + if self.readable and not stat.S_ISDIR(st.st_mode): + if abspath.endswith(".md") and "raw" not in self.uparam: + return self.tx_md(abspath) + + if rem.startswith(".hist/up2k."): + raise Pebkac(403) + + if "th" in self.uparam: + thp = None + if self.thumbcli: + thp = self.thumbcli.get(vn.realpath, rem, int(st.st_mtime)) + + if thp: + return self.tx_file(thp) + + return self.tx_ico(rem) + + return self.tx_file(abspath) + srv_info = [] try: @@ -1431,22 +1473,13 @@ class HttpCli(object): self.reply(ret.encode("utf-8", "replace"), mime="application/json") return True - if not os.path.isdir(fsenc(abspath)): + if not stat.S_ISDIR(st.st_mode): raise Pebkac(404) html = self.j2(tpl, **j2a) self.reply(html.encode("utf-8", "replace")) return True - if not os.path.isdir(fsenc(abspath)): - if abspath.endswith(".md") and "raw" not in self.uparam: - return self.tx_md(abspath) - - if rem.startswith(".hist/up2k."): - raise Pebkac(403) - - return self.tx_file(abspath) - for k in ["zip", "tar"]: v = self.uparam.get(k) if v is not None: diff --git a/copyparty/httpconn.py b/copyparty/httpconn.py index 2f600bc9..cbcaeb2c 100644 --- a/copyparty/httpconn.py +++ b/copyparty/httpconn.py @@ -17,6 +17,9 @@ from .__init__ import E from .util import Unrecv from .httpcli import HttpCli from .u2idx import U2idx +from .th_cli import ThumbCli +from .th_srv import HAVE_PIL +from .ico import Ico class HttpConn(object): @@ -34,6 +37,10 @@ class HttpConn(object): self.auth = hsrv.auth self.cert_path = hsrv.cert_path + enth = HAVE_PIL and not self.args.no_thumb + self.thumbcli = ThumbCli(hsrv.broker) if enth else None + self.ico = Ico() + self.t0 = time.time() self.nbyte = 0 self.workload = 0 diff --git a/copyparty/ico.py b/copyparty/ico.py new file mode 100644 index 00000000..4f0fae68 --- /dev/null +++ b/copyparty/ico.py @@ -0,0 +1,35 @@ +import hashlib +import colorsys + + +class Ico(object): + def __init__(self): + pass + + def get(self, ext): + """placeholder to make thumbnails not break""" + + if False: + h = hashlib.md5(ext.encode("utf-8")).digest()[:6] + lo = [int(x / 3) for x in h] + hi = [int(x / 3 + 170) for x in h] + c = lo[:3] + hi[3:6] + else: + h = hashlib.md5(ext.encode("utf-8")).digest()[:2] + c1 = colorsys.hsv_to_rgb(h[0] / 256.0, 1, 0.3) + c2 = colorsys.hsv_to_rgb(h[0] / 256.0, 1, 1) + c = list(c1) + list(c2) + c = [int(x * 255) for x in c] + + c = "".join(["{:02x}".format(x) for x in c]) + + svg = """\ + + + +{} + +""" + svg = svg.format(c[:6], c[6:], ext).encode("utf-8") + + return ["image/svg+xml", svg] diff --git a/copyparty/svchub.py b/copyparty/svchub.py index 80f7d627..727dfef0 100644 --- a/copyparty/svchub.py +++ b/copyparty/svchub.py @@ -9,9 +9,10 @@ from datetime import datetime, timedelta import calendar from .__init__ import PY2, WINDOWS, MACOS, VT100 +from .util import mp from .tcpsrv import TcpSrv from .up2k import Up2k -from .util import mp +from .th_srv import ThumbSrv, HAVE_PIL class SvcHub(object): @@ -38,6 +39,9 @@ class SvcHub(object): self.tcpsrv = TcpSrv(self) self.up2k = Up2k(self) + enth = HAVE_PIL and not args.no_thumb + self.thumbsrv = ThumbSrv(self) if enth else None + # decide which worker impl to use if self.check_mp_enable(): from .broker_mp import BrokerMp as Broker @@ -63,6 +67,13 @@ class SvcHub(object): self.tcpsrv.shutdown() self.broker.shutdown() + if self.thumbsrv: + self.thumbsrv.shutdown() + + print("waiting for thumbsrv...") + while not self.thumbsrv.stopped(): + time.sleep(0.05) + print("nailed it") def _log_disabled(self, src, msg, c=0): diff --git a/copyparty/th_cli.py b/copyparty/th_cli.py new file mode 100644 index 00000000..1dde3ca4 --- /dev/null +++ b/copyparty/th_cli.py @@ -0,0 +1,21 @@ +import os + +from .th_srv import thumb_path, THUMBABLE + + +class ThumbCli(object): + def __init__(self, broker): + self.broker = broker + self.args = broker.args + + def get(self, ptop, rem, mtime): + ext = rem.rsplit(".")[-1].lower() + if ext not in THUMBABLE: + return None + + tpath = thumb_path(ptop, rem, mtime) + if os.path.exists(tpath): + return tpath + + x = self.broker.put(True, "thumbsrv.get", ptop, rem, mtime) + return x.get() diff --git a/copyparty/th_srv.py b/copyparty/th_srv.py new file mode 100644 index 00000000..33c58c74 --- /dev/null +++ b/copyparty/th_srv.py @@ -0,0 +1,152 @@ +import os +import base64 +import hashlib +import threading + +try: + HAVE_PIL = True + from PIL import Image + + try: + HAVE_HEIF = True + from pyheif_pillow_opener import register_heif_opener + + register_heif_opener() + except: + HAVE_HEIF = False +except: + HAVE_PIL = False + +from .util import fsenc, Queue + +# https://pillow.readthedocs.io/en/stable/handbook/image-file-formats.html +FMT_PIL = "bmp dib gif icns ico jpg jpeg jp2 jpx pcx png pbm pgm ppm pnm sgi tga tif tiff webp xbm dds xpm" +FMT_PIL = {x: True for x in FMT_PIL.split(" ") if x} +THUMBABLE = FMT_PIL + + +def thumb_path(ptop, rem, mtime): + # base16 = 16 = 256 + # b64-lc = 38 = 1444 + # base64 = 64 = 4096 + try: + rd, fn = rem.rsplit("/", 1) + except: + rd = "" + fn = rem + + if rd: + h = hashlib.sha512(fsenc(rd)).digest()[:24] + b64 = base64.urlsafe_b64encode(h).decode("ascii")[:24] + rd = "{}/{}/".format(b64[:2], b64[2:4]).lower() + b64 + else: + rd = "top" + + # could keep original filenames but this is safer re pathlen + h = hashlib.sha512(fsenc(fn)).digest()[:24] + fn = base64.urlsafe_b64encode(h).decode("ascii")[:24] + + return "{}/.hist/th/{}/{}.{:x}.jpg".format(ptop, rd, fn, int(mtime)) + + +class ThumbSrv(object): + def __init__(self, hub): + self.hub = hub + self.log_func = hub.log + + self.mutex = threading.Lock() + self.busy = {} + self.stopping = False + self.nthr = os.cpu_count() if hasattr(os, "cpu_count") else 4 + self.q = Queue(self.nthr * 4) + for _ in range(self.nthr): + t = threading.Thread(target=self.worker) + t.daemon = True + t.start() + + def log(self, msg, c=0): + self.log_func("thumb", msg, c) + + def shutdown(self): + self.stopping = True + for _ in range(self.nthr): + self.q.put(None) + + def stopped(self): + with self.mutex: + return not self.nthr + + def get(self, ptop, rem, mtime): + tpath = thumb_path(ptop, rem, mtime) + abspath = os.path.join(ptop, rem) + cond = threading.Condition() + with self.mutex: + try: + self.busy[tpath].append(cond) + self.log("conv {}".format(tpath)) + except: + thdir = os.path.dirname(tpath) + try: + os.makedirs(thdir) + except: + pass + + inf_path = os.path.join(thdir, "dir.txt") + if not os.path.exists(inf_path): + with open(inf_path, "wb") as f: + f.write(fsenc(os.path.dirname(abspath))) + + self.busy[tpath] = [cond] + self.q.put([abspath, tpath]) + self.log("CONV {}".format(tpath)) + + while not self.stopping: + with self.mutex: + if tpath not in self.busy: + break + + with cond: + cond.wait() + + if not os.path.exists(tpath): + return None + + return tpath + + def worker(self): + while not self.stopping: + task = self.q.get() + if not task: + break + + abspath, tpath = task + ext = abspath.split(".")[-1].lower() + fun = None + if not os.path.exists(tpath): + if ext in FMT_PIL: + fun = self.conv_pil + + if fun: + fun(abspath, tpath) + + with self.mutex: + subs = self.busy[tpath] + del self.busy[tpath] + + for x in subs: + with x: + x.notify_all() + + with self.mutex: + self.nthr -= 1 + + def conv_pil(self, abspath, tpath): + try: + with Image.open(abspath) as im: + if im.mode in ("RGBA", "P"): + im = im.convert("RGB") + + im.thumbnail((256, 256)) + im.save(tpath) + except: + pass \ No newline at end of file diff --git a/copyparty/up2k.py b/copyparty/up2k.py index 171d5cd9..7744195c 100644 --- a/copyparty/up2k.py +++ b/copyparty/up2k.py @@ -49,13 +49,13 @@ class Up2k(object): * ~/.config flatfiles for active jobs """ - def __init__(self, broker): - self.broker = broker - self.args = broker.args - self.log_func = broker.log + def __init__(self, hub): + self.hub = hub + self.args = hub.args + self.log_func = hub.log # config - self.salt = broker.args.salt + self.salt = self.args.salt # state self.mutex = threading.Lock() diff --git a/copyparty/util.py b/copyparty/util.py index f741c47d..416a41da 100644 --- a/copyparty/util.py +++ b/copyparty/util.py @@ -914,11 +914,11 @@ def unescape_cookie(orig): return ret -def guess_mime(url): +def guess_mime(url, fallback="application/octet-stream"): if url.endswith(".md"): return ["text/plain; charset=UTF-8"] - return mimetypes.guess_type(url) + return mimetypes.guess_type(url) or fallback def runcmd(*argv): diff --git a/copyparty/web/browser.css b/copyparty/web/browser.css index 3658de5b..0b4a7b25 100644 --- a/copyparty/web/browser.css +++ b/copyparty/web/browser.css @@ -1,3 +1,6 @@ +:root { + --grid-sz: 10em; +} * { line-height: 1.2em; } @@ -550,8 +553,7 @@ input[type="checkbox"]:checked+label { left: -1.7em; width: calc(100% + 1.3em); } -.tglbtn, -#tree>a+a { +.btn { padding: .2em .4em; font-size: 1.2em; background: #2a2a2a; @@ -561,12 +563,10 @@ input[type="checkbox"]:checked+label { position: relative; top: -.2em; } -.tglbtn:hover, -#tree>a+a:hover { +.btn:hover { background: #805; } -.tglbtn.on, -#tree>a+a.on { +.tgl.btn.on { background: #fc4; color: #400; text-shadow: none; @@ -711,6 +711,40 @@ input[type="checkbox"]:checked+label { font-family: monospace, monospace; line-height: 2em; } +#griden.on+#thumbs { + opacity: .3; +} +#ghead { + background: #3c3c3c; + border: 1px solid #444; + border-radius: .3em; + padding: .5em; + margin: 0 1.5em 0 .4em; +} +#ghead .btn { + position: relative; + top: 0; +} +#ggrid { + padding-top: .5em; +} +#ggrid a { + display: inline-block; + width: var(--grid-sz); + vertical-align: top; + overflow-wrap: break-word; + background: #383838; + border: 1px solid #444; + border-radius: .3em; + padding: .3em .6em; + margin: .5em; +} +#ggrid a img { + max-width: var(--grid-sz); + max-height: var(--grid-sz); + margin: 0 auto .5em auto; + display: block; +} #pvol, #barbuf, #barpos, @@ -725,6 +759,21 @@ input[type="checkbox"]:checked+label { + + + + + + + + + + + + + + + html.light { color: #333; background: #eee; @@ -746,18 +795,15 @@ html.light #ops a.act { html.light #op_cfg h3 { border-color: #ccc; } -html.light .tglbtn, -html.light #tree > a + a { +html.light .btn { color: #666; background: #ddd; box-shadow: none; } -html.light .tglbtn:hover, -html.light #tree > a + a:hover { +html.light .btn:hover { background: #caf; } -html.light .tglbtn.on, -html.light #tree > a + a.on { +html.light .tgl.btn.on { background: #4a0; color: #fff; } diff --git a/copyparty/web/browser.html b/copyparty/web/browser.html index d4065542..decd96a0 100644 --- a/copyparty/web/browser.html +++ b/copyparty/web/browser.html @@ -41,8 +41,10 @@

switches

- tooltips - lightmode + tooltips + lightmode + the grid + thumbs
{%- if have_zip %}

folder download

@@ -61,9 +63,9 @@
🍞... - + - – - a + + + – + a
 
diff --git a/copyparty/web/browser.js b/copyparty/web/browser.js index b7292ae7..e3f8c589 100644 --- a/copyparty/web/browser.js +++ b/copyparty/web/browser.js @@ -696,6 +696,157 @@ function autoplay_blocked(seek) { })(); +var thegrid = (function () { + var lfiles = ebi('files'); + var gfiles = document.createElement('div'); + gfiles.setAttribute('id', 'gfiles'); + gfiles.style.display = 'none'; + gfiles.innerHTML = ( + '
' + + 'multiselect   zoom ' + + '– ' + + '+   sort by: ' + + 'name, ' + + 'size, ' + + 'date, ' + + 'type' + + '
' + + '
' + ); + lfiles.parentNode.insertBefore(gfiles, lfiles); + + var r = { + 'thumbs': bcfg_get('thumbs', true), + 'en': bcfg_get('griden', false), + 'sel': bcfg_get('gridsel', false), + 'sz': fcfg_get('gridsz', 10), + 'isdirty': true + }; + + ebi('thumbs').onclick = function (e) { + ev(e); + r.thumbs = !r.thumbs; + bcfg_set('thumbs', r.thumbs); + if (r.en) { + loadgrid(); + } + }; + + ebi('griden').onclick = function (e) { + ev(e); + r.en = !r.en; + bcfg_set('griden', r.en); + if (r.en) { + loadgrid(); + } + else { + lfiles.style.display = ''; + gfiles.style.display = 'none'; + } + }; + + var click = function (e) { + ev(e); + var s = this.getAttribute('s'), + z = this.getAttribute('z'); + + if (z) + return setsz(z > 0 ? r.sz * z : r.sz / (-z)); + + var t = lfiles.tHead.rows[0].cells; + for (var a = 0; a < t.length; a++) + if (t[a].getAttribute('name') == s) { + t[a].click(); + break; + } + + r.setdirty(); + }; + + var links = QSA('#ghead>a'); + for (var a = 0; a < links.length; a++) + links[a].onclick = click; + + ebi('gridsel').onclick = function (e) { + ev(e); + r.sel = !r.sel; + bcfg_set('gridsel', r.sel); + }; + + r.setvis = function (vis) { + (r.en ? gfiles : lfiles).style.display = vis ? '' : 'none'; + } + + r.setdirty = function () { + r.dirty = true; + if (r.en) { + loadgrid(); + } + } + + function setsz(v) { + if (v !== undefined) { + r.sz = v; + swrite('gridsz', r.sz); + } + document.documentElement.style.setProperty('--grid-sz', r.sz + 'em'); + } + setsz(); + + function loadgrid() { + if (!r.dirty) + return; + + var html = []; + var tr = lfiles.tBodies[0].rows; + for (var a = 0; a < tr.length; a++) { + var ao = tr[a].cells[1].firstChild, + href = esc(ao.getAttribute('href')), + isdir = href.split('?')[0].slice(-1)[0] == '/', + ihref = href; + + if (isdir) { + ihref = '/.cpr/ico/folder' + } + else if (r.thumbs) { + ihref += ihref.indexOf('?') === -1 ? '?th' : '&th'; + } + else { + var ar = href.split('?')[0].split('.'); + if (ar.length > 1) + ar = ar.slice(1); + + ihref = ''; + ar.reverse(); + for (var b = 0; b < ar.length; b++) { + if (ar[b].length > 7) + break; + + ihref = ar[b] + '.' + ihref; + } + if (!ihref) { + ihref = 'unk.'; + } + ihref = '/.cpr/ico/' + ihref.slice(0, -1); + } + + html.push('' + // /.cpr/dd/1.png + ao.innerHTML + ''); + } + lfiles.style.display = 'none'; + gfiles.style.display = 'block'; + ebi('ggrid').innerHTML = html.join('\n'); + } + + if (r.en) { + loadgrid(); + } + + return r; +})(); + + function tree_neigh(n) { var links = QSA('#treeul li>a+a'); if (!links.length) { @@ -763,6 +914,12 @@ document.onkeydown = function (e) { if (k == 'KeyP') return tree_up(); + + if (k == 'KeyG') + return ebi('griden').click(); + + if (k == 'KeyT') + return ebi('thumbs').click(); }; @@ -1391,7 +1548,7 @@ function apply_perms(perms) { up2k.set_fsearch(); ebi('widget').style.display = have_read ? '' : 'none'; - ebi('files').style.display = have_read ? '' : 'none'; + thegrid.setvis(have_read); if (!have_read) goto('up2k'); } @@ -1892,6 +2049,8 @@ function reload_browser(not_mp) { if (window['up2k']) up2k.set_fsearch(); + + thegrid.setdirty(); } reload_browser(true); mukey.render(); diff --git a/copyparty/web/util.js b/copyparty/web/util.js index fd2f8e7c..ce27579b 100644 --- a/copyparty/web/util.js +++ b/copyparty/web/util.js @@ -456,11 +456,15 @@ function jwrite(key, val) { } function icfg_get(name, defval) { + return parseInt(fcfg_get(name, defval)); +} + +function fcfg_get(name, defval) { var o = ebi(name); - var val = parseInt(sread(name)); + var val = parseFloat(sread(name)); if (isNaN(val)) - return parseInt(o ? o.value : defval); + return parseFloat(o ? o.value : defval); if (o) o.value = val; diff --git a/docs/nuitka.txt b/docs/nuitka.txt index 12478341..75c7143c 100644 --- a/docs/nuitka.txt +++ b/docs/nuitka.txt @@ -1,7 +1,7 @@ # recipe for building an exe with nuitka (extreme jank edition) # -# NOTE: on win7 the exe immediately hits a c0000005 in kernelbase.dll -# but win10 builds work on win10 +# NOTE: win7 and win10 builds both work on win10 but +# on win7 they immediately c0000005 in kernelbase.dll # # first install python-3.6.8-amd64.exe # [x] add to path