diff --git a/.gitignore b/.gitignore index f5da4311..c8cd8d26 100644 --- a/.gitignore +++ b/.gitignore @@ -8,8 +8,7 @@ copyparty.egg-info/ buildenv/ build/ dist/ -*.rst -.env/ +.venv/ # sublime *.sublime-workspace diff --git a/.vscode/settings.json b/.vscode/settings.json index 2a96d0e2..a3e2455d 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -55,6 +55,6 @@ // // things you may wanna edit: // - "python.pythonPath": ".env/bin/python", + "python.pythonPath": ".venv/bin/python", //"python.linting.enabled": true, } \ No newline at end of file diff --git a/README.md b/README.md index 39eeaf5d..a7d18013 100644 --- a/README.md +++ b/README.md @@ -61,8 +61,8 @@ after the initial setup (and restarting bash), you can launch copyparty at any t # dev env setup ```sh -python3 -m venv .env -. .env/bin/activate +python3 -m venv .venv +. .venv/bin/activate pip install jinja2 # mandatory deps pip install Pillow # thumbnail deps pip install black bandit pylint flake8 # vscode tooling diff --git a/copyparty/__main__.py b/copyparty/__main__.py index f9727ff8..31c0258f 100644 --- a/copyparty/__main__.py +++ b/copyparty/__main__.py @@ -28,8 +28,6 @@ class RiceFormatter(argparse.HelpFormatter): except the help += [...] line now has colors """ fmt = "\033[36m (default: \033[35m%(default)s\033[36m)\033[0m" - if WINDOWS: - fmt = " (default: %(default)s)" help = action.help if "%(default)" not in action.help: @@ -85,6 +83,9 @@ def ensure_cert(): def main(): + if WINDOWS: + os.system("") # enables colors + f = "\033[36mcopyparty v{} ({})\n python v{}\033[0m\n" print(f.format(S_VERSION, S_BUILD_DT, py_desc())) diff --git a/copyparty/broker_mp.py b/copyparty/broker_mp.py index 8b028049..0e02091a 100644 --- a/copyparty/broker_mp.py +++ b/copyparty/broker_mp.py @@ -141,8 +141,6 @@ class BrokerMp(object): def debug_load_balancer(self): fmt = "\033[1m{}\033[0;36m{:4}\033[0m " - if WINDOWS: - fmt = "({}{:4})" last = "" while self.procs: diff --git a/copyparty/httpcli.py b/copyparty/httpcli.py index 6f22f48e..b7bee43b 100644 --- a/copyparty/httpcli.py +++ b/copyparty/httpcli.py @@ -10,7 +10,7 @@ from datetime import datetime import calendar import mimetypes -from .__init__ import E, PY2 +from .__init__ import E, PY2, WINDOWS from .util import * # noqa # pylint: disable=unused-wildcard-import if not PY2: @@ -195,7 +195,7 @@ class HttpCli(object): return self.tx_mounts() return self.tx_browser() - + def handle_post(self): self.log("POST " + self.req) @@ -224,12 +224,15 @@ class HttpCli(object): act = self.parser.require("act", 64) - if act == "bput": - return self.handle_plain_upload() - if act == "login": return self.handle_login() + if act == "mkdir": + return self.handle_mkdir() + + if act == "bput": + return self.handle_plain_upload() + raise Pebkac(422, 'invalid action "{}"'.format(act)) def handle_post_json(self): @@ -292,7 +295,7 @@ class HttpCli(object): x = self.conn.hsrv.broker.put(True, "up2k.handle_chunk", wark, chash) response = x.get() - chunksize, cstart, path = response + chunksize, cstart, path, lastmod = response if self.args.nw: path = os.devnull @@ -336,7 +339,15 @@ class HttpCli(object): self.log("clone {} done".format(cstart[0])) x = self.conn.hsrv.broker.put(True, "up2k.confirm_chunk", wark, chash) - response = x.get() + num_left = x.get() + + if not WINDOWS and num_left == 0: + times = (int(time.time()), int(lastmod)) + self.log("no more chunks, setting times {}".format(times)) + try: + os.utime(path, times) + except: + self.log("failed to utime ({}, {})".format(path, times)) self.loud_reply("thank") return True @@ -356,6 +367,36 @@ class HttpCli(object): self.reply(html.encode("utf-8"), headers=h) return True + def handle_mkdir(self): + new_dir = self.parser.require("name", 512) + self.parser.drop() + + nullwrite = self.args.nw + vfs, rem = self.conn.auth.vfs.get(self.vpath, self.uname, False, True) + + # rem is escaped at this point, + # this is just a sanity check to prevent any disasters + if rem.startswith("/") or rem.startswith("../") or "/../" in rem: + raise Exception("that was close") + + if not nullwrite: + fdir = os.path.join(vfs.realpath, rem) + fn = os.path.join(fdir, sanitize_fn(new_dir)) + + if not os.path.isdir(fsenc(fdir)): + raise Pebkac(404, "that folder does not exist") + + os.mkdir(fsenc(fn)) + + html = self.conn.tpl_msg.render( + h2='return to /{}'.format( + quotep(self.vpath), html_escape(self.vpath, quote=False) + ), + pre="aight", + ) + self.reply(html.encode("utf-8", "replace")) + return True + def handle_plain_upload(self): nullwrite = self.args.nw vfs, rem = self.conn.auth.vfs.get(self.vpath, self.uname, False, True) @@ -620,7 +661,9 @@ class HttpCli(object): vpnodes.append([quotep(vpath) + "/", html_escape(node, quote=False)]) - vn, rem = self.auth.vfs.get(self.vpath, self.uname, self.readable, self.writable) + vn, rem = self.auth.vfs.get( + self.vpath, self.uname, self.readable, self.writable + ) abspath = vn.canonical(rem) if not os.path.exists(fsenc(abspath)): @@ -684,7 +727,7 @@ class HttpCli(object): ts=ts, prologue=logues[0], epilogue=logues[1], + title=quotep(self.vpath), ) self.reply(html.encode("utf-8", "replace")) return True - diff --git a/copyparty/httpsrv.py b/copyparty/httpsrv.py index 11ad9406..c1b09f9f 100644 --- a/copyparty/httpsrv.py +++ b/copyparty/httpsrv.py @@ -75,10 +75,12 @@ class HttpSrv(object): sck.shutdown(socket.SHUT_RDWR) sck.close() except (OSError, socket.error) as ex: - if ex.errno not in [107, 57, 9]: - # 107 Transport endpoint not connected - # 57 Socket is not connected - # 9 Bad file descriptor + # self.log(str(addr), "shut_rdwr err: " + repr(sck)) + if ex.errno not in [10038, 107, 57, 9]: + # 10038 No longer considered a socket + # 107 Transport endpoint not connected + # 57 Socket is not connected + # 9 Bad file descriptor raise finally: with self.mutex: diff --git a/copyparty/svchub.py b/copyparty/svchub.py index ee328639..b41db72b 100644 --- a/copyparty/svchub.py +++ b/copyparty/svchub.py @@ -85,16 +85,7 @@ class SvcHub(object): self.next_day = calendar.timegm(dt.utctimetuple()) ts = datetime.utcfromtimestamp(now).strftime("%H:%M:%S.%f")[:-3] - - if not WINDOWS: - fmt = "\033[36m{} \033[33m{:21} \033[0m{}" - else: - fmt = "{} {:21} {}" - if "\033" in msg: - msg = self.ansi_re.sub("", msg) - if "\033" in src: - src = self.ansi_re.sub("", src) - + fmt = "\033[36m{} \033[33m{:21} \033[0m{}" msg = fmt.format(ts, src, msg) try: print(msg) diff --git a/copyparty/up2k.py b/copyparty/up2k.py index bbcc2a80..5707aaa8 100644 --- a/copyparty/up2k.py +++ b/copyparty/up2k.py @@ -9,8 +9,10 @@ import math import base64 import hashlib import threading +from queue import Queue from copy import deepcopy +from .__init__ import WINDOWS from .util import Pebkac @@ -35,6 +37,13 @@ class Up2k(object): self.registry = {} self.mutex = threading.Lock() + if WINDOWS: + # usually fails to set lastmod too quickly + self.lastmod_q = Queue() + thr = threading.Thread(target=self._lastmodder) + thr.daemon = True + thr.start() + # static self.r_hash = re.compile("^[0-9a-zA-Z_-]{43}$") @@ -56,6 +65,7 @@ class Up2k(object): # client-provided, sanitized by _get_wark: "name": cj["name"], "size": cj["size"], + "lmod": cj["lmod"], "hash": deepcopy(cj["hash"]), } @@ -74,6 +84,7 @@ class Up2k(object): return { "name": job["name"], "size": job["size"], + "lmod": job["lmod"], "hash": job["need"], "wark": wark, } @@ -96,11 +107,19 @@ class Up2k(object): path = os.path.join(job["vdir"], job["name"]) - return [chunksize, ofs, path] + return [chunksize, ofs, path, job["lmod"]] def confirm_chunk(self, wark, chash): with self.mutex: - self.registry[wark]["need"].remove(chash) + job = self.registry[wark] + job["need"].remove(chash) + ret = len(job["need"]) + + if WINDOWS and ret == 0: + path = os.path.join(job["vdir"], job["name"]) + self.lastmod_q.put([path, (int(time.time()), int(job["lmod"]))]) + + return ret def _get_chunksize(self, filesize): chunksize = 1024 * 1024 @@ -115,7 +134,7 @@ class Up2k(object): stepsize *= mul def _get_wark(self, cj): - if len(cj["name"]) > 1024 or len(cj["hash"]) > 256: + if len(cj["name"]) > 1024 or len(cj["hash"]) > 512 * 1024: # 16TiB raise Pebkac(400, "name or numchunks not according to spec") for k in cj["hash"]: @@ -124,6 +143,12 @@ class Up2k(object): 400, "at least one hash is not according to spec: {}".format(k) ) + # try to use client-provided timestamp, don't care if it fails somehow + try: + cj["lmod"] = int(cj["lmod"]) + except: + cj["lmod"] = int(time.time()) + # server-reproducible file identifier, independent of name or location ident = [self.salt, str(cj["size"])] ident.extend(cj["hash"]) @@ -143,3 +168,16 @@ class Up2k(object): f.seek(job["size"] - 1) f.write(b"e") + def _lastmodder(self): + while True: + ready = [] + while not self.lastmod_q.empty(): + ready.append(self.lastmod_q.get()) + + # self.log("lmod", "got {}".format(len(ready))) + time.sleep(5) + for path, times in ready: + try: + os.utime(path, times) + except: + self.log("lmod", "failed to utime ({}, {})".format(path, times)) diff --git a/copyparty/web/browser.html b/copyparty/web/browser.html index 9592ac50..eaf4f6b4 100644 --- a/copyparty/web/browser.html +++ b/copyparty/web/browser.html @@ -3,7 +3,7 @@ - copyparty + ⇆🎉 {{ title }} diff --git a/copyparty/web/up2k.js b/copyparty/web/up2k.js index 8add148a..ebe6a12c 100644 --- a/copyparty/web/up2k.js +++ b/copyparty/web/up2k.js @@ -58,22 +58,61 @@ function o(id) { (function () { - // chrome requires https to use crypto.subtle, - // usually it's undefined but some chromes throw on invoke - try { - crypto.subtle.digest( - 'SHA-512', new Uint8Array(1) - ).then( - function (x) { up2k_init(true) }, - function (x) { up2k_init(false) } - ); - } - catch (ex) { - up2k_init(false); + var ops = document.querySelectorAll('#ops>a'); + for (var a = 0; a < ops.length; a++) { + ops[a].onclick = opclick; } })(); +function opclick(ev) { + ev.preventDefault(); + goto(this.getAttribute('data-dest')); +} + + +function goto(dest) { + var obj = document.querySelectorAll('.opview.act'); + for (var a = obj.length - 1; a >= 0; a--) + obj[a].setAttribute('class', 'opview'); + + var obj = document.querySelectorAll('#ops>a'); + for (var a = obj.length - 1; a >= 0; a--) + obj[a].setAttribute('class', ''); + + document.querySelector('#ops>a[data-dest=' + dest + ']').setAttribute('class', 'act'); + document.getElementById('op_' + dest).setAttribute('class', 'opview act'); + + var fn = window['goto_' + dest]; + if (fn) + fn(); +} + + +function goto_up2k() { + if (!up2k) + return setTimeout(goto_up2k, 100); + + up2k.init_deps(); +} + + +// chrome requires https to use crypto.subtle, +// usually it's undefined but some chromes throw on invoke +var up2k = null; +try { + crypto.subtle.digest( + 'SHA-512', new Uint8Array(1) + ).then( + function (x) { up2k = up2k_init(true) }, + function (x) { up2k = up2k_init(false) } + ); +} +catch (ex) { + up2k = up2k_init(false); +} + + function up2k_init(have_crypto) { //have_crypto = false; var need_filereader_cache = undefined; @@ -94,7 +133,7 @@ function up2k_init(have_crypto) { o('u2notbtn').innerHTML = ''; } - var post_url = o('bup').getElementsByTagName('form')[0].getAttribute('action'); + var post_url = o('op_bup').getElementsByTagName('form')[0].getAttribute('action'); if (post_url && post_url.charAt(post_url.length - 1) !== '/') post_url += '/'; @@ -105,13 +144,7 @@ function up2k_init(have_crypto) { shame = 'your browser is impressively ancient'; // upload ui hidden by default, clicking the header shows it - function toggle_upload_visible(ev) { - if (ev) - ev.preventDefault(); - - o('u2tgl').style.display = 'none'; - o('u2body').style.display = 'block'; - + function init_deps() { if (!have_crypto && !window.asmCrypto) { showmodal('

loading sha512.js

since ' + shame + '

thanks chrome

'); import_js('/.cpr/deps/sha512.js', unmodal); @@ -122,11 +155,10 @@ function up2k_init(have_crypto) { o('u2foot').innerHTML = 'seems like ' + shame + ' so do that if you want more performance'; } }; - o('u2tgl').onclick = toggle_upload_visible; - + // show uploader if the user only has write-access if (!o('files')) - toggle_upload_visible(); + goto('up2k'); // shows or clears an error message in the basic uploader ui function setmsg(msg) { @@ -142,8 +174,7 @@ function up2k_init(have_crypto) { // switches to the basic uploader with msg as error message function un2k(msg) { - o('up2k').style.display = 'none'; - o('bup').style.display = 'block'; + goto('bup'); setmsg(msg); } @@ -218,10 +249,6 @@ function up2k_init(have_crypto) { if (!bobslice || !window.FileReader || !window.FileList) return un2k("this is the basic uploader; up2k needs at least
chrome 21 // firefox 13 // edge 12 // opera 12 // safari 5.1"); - // probably safe now - o('up2k').style.display = 'block'; - o('bup').style.display = 'none'; - function nav() { o('file' + fdom_ctr).click(); } @@ -272,12 +299,15 @@ function up2k_init(have_crypto) { bad_files.push([a, fobj.name]); continue; } + var now = new Date().getTime(); + var lmod = fobj.lastModified || now; var entry = { "n": parseInt(st.files.length.toString()), - "t0": new Date().getTime(), // TODO remove probably + "t0": now, // TODO remove probably "fobj": fobj, "name": fobj.name, "size": fobj.size, + "lmod": lmod / 1000, "hash": [] }; @@ -689,6 +719,7 @@ function up2k_init(have_crypto) { xhr.send(JSON.stringify({ "name": t.name, "size": t.size, + "lmod": t.lmod, "hash": t.hash })); } @@ -838,4 +869,6 @@ function up2k_init(have_crypto) { nodes[a].addEventListener('touchend', nop, false); bumpthread({ "target": 1 }) + + return { "init_deps": init_deps } } diff --git a/copyparty/web/upload.css b/copyparty/web/upload.css index 3ead19e2..202782ac 100644 --- a/copyparty/web/upload.css +++ b/copyparty/web/upload.css @@ -1,19 +1,97 @@ -#bup { - padding: .5em .5em .5em .3em; - margin: 1em 0 2em 0; - background: #2d2d2d; - border-radius: 0 1em 1em 0; +.opview { + display: none; +} +.opview.act { + display: block; +} +#ops a { + color: #fc5; + font-size: 1.5em; + padding: 0 .3em; + margin: 0; + outline: none; +} +#ops a.act { + text-decoration: underline; +} +/* +#ops a+a:after, +#ops a:first-child:after { + content: 'x'; + color: #282828; + text-shadow: 0 0 .08em #01a7e1; + margin-left: .3em; + position: relative; +} +#ops a+a:before { + content: 'x'; + color: #282828; + text-shadow: 0 0 .08em #ff3f1a; + margin-right: .3em; + margin-left: -.3em; +} +#ops a:last-child:after { + content: ''; +} +#ops a.act:before, +#ops a.act:after { + text-decoration: none !important; +} +*/ +#ops i { + font-size: 1.5em; +} +#ops i:before { + content: 'x'; + color: #282828; + text-shadow: 0 0 .08em #01a7e1; + position: relative; +} +#ops i:after { + content: 'x'; + color: #282828; + text-shadow: 0 0 .08em #ff3f1a; + margin-left: -.35em; + font-size: 1.05em; +} +#ops, +#op_bup, +#op_mkdir { border: 1px solid #3a3a3a; - border-width: 0 .3em .3em 0; box-shadow: 0 0 1em #222 inset; +} +#ops { + display: inline-block; + background: #333; + margin: 1em 1.5em; + padding: .3em .6em; + border-radius: .3em; + border-width: .15em 0; +} +#op_bup, +#op_mkdir { + background: #2d2d2d; + margin: 1em 0 2em 0; + padding: .5em; + border-radius: 0 1em 1em 0; + border-width: .15em .3em .3em 0; max-width: 40em; } -#bup input { +#op_mkdir input, +#op_bup input { margin: .5em; } -#up2k { - display: none; - padding: 0 1em; +#op_mkdir input[type=text] { + color: #fff; + background: #383838; + border: none; + box-shadow: 0 0 .3em #222; + border-bottom: 1px solid #fc5; + border-radius: .2em; + padding: .2em .3em; +} +#op_up2k { + padding: 0 1em 1em 1em; } #u2form { position: absolute; @@ -29,16 +107,6 @@ color: #f87; padding: .5em; } -#u2tgl { - color: #fc5; - font-size: 1.5em; - margin: .5em 0 1em 0; - display: block; -} -#u2body { - display: none; - padding-bottom: 1em; -} #u2form { width: 2px; height: 2px; diff --git a/copyparty/web/upload.html b/copyparty/web/upload.html index 27111494..5abaca42 100644 --- a/copyparty/web/upload.html +++ b/copyparty/web/upload.html @@ -1,4 +1,9 @@ -
+
up2kbupmkdir
+ +
@@ -7,10 +12,16 @@
-
- you can upload here -
-
+
+
+ + + +
+
+ +
+
@@ -45,6 +56,5 @@

-

( if you don't need resumable uploads and progress bars just use the basic uploader)

-
+

( if you don't need resumable uploads and progress bars just use the basic uploader)