add mkdir + keep mtime + bump max-size

.gitignore

@@ -8,8 +8,7 @@ copyparty.egg-info/
 buildenv/
 build/
 dist/
-*.rst
+.venv/
-.env/
 
 # sublime
 *.sublime-workspace

.vscode/settings.json

@@ -55,6 +55,6 @@
     //
     //  things you may wanna edit:
     //
-    "python.pythonPath": ".env/bin/python",
+    "python.pythonPath": ".venv/bin/python",
     //"python.linting.enabled": true,
 }

README.md

@@ -61,8 +61,8 @@ after the initial setup (and restarting bash), you can launch copyparty at any t
 # dev env setup
 
 ```sh
-python3 -m venv .env
+python3 -m venv .venv
-. .env/bin/activate
+. .venv/bin/activate
 pip install jinja2  # mandatory deps
 pip install Pillow  # thumbnail deps
 pip install black bandit pylint flake8  # vscode tooling

copyparty/__main__.py

@@ -28,8 +28,6 @@ class RiceFormatter(argparse.HelpFormatter):
         except the help += [...] line now has colors
         """
         fmt = "\033[36m (default: \033[35m%(default)s\033[36m)\033[0m"
-        if WINDOWS:
-            fmt = " (default: %(default)s)"
 
         help = action.help
         if "%(default)" not in action.help:
@@ -85,6 +83,9 @@ def ensure_cert():
 
 
 def main():
+    if WINDOWS:
+        os.system("")  # enables colors
+
     f = "\033[36mcopyparty v{} ({})\n   python v{}\033[0m\n"
     print(f.format(S_VERSION, S_BUILD_DT, py_desc()))
 

copyparty/broker_mp.py

@@ -141,8 +141,6 @@ class BrokerMp(object):
 
     def debug_load_balancer(self):
         fmt = "\033[1m{}\033[0;36m{:4}\033[0m "
-        if WINDOWS:
-            fmt = "({}{:4})"
 
         last = ""
         while self.procs:

copyparty/httpcli.py

@@ -10,7 +10,7 @@ from datetime import datetime
 import calendar
 import mimetypes
 
-from .__init__ import E, PY2
+from .__init__ import E, PY2, WINDOWS
 from .util import *  # noqa  # pylint: disable=unused-wildcard-import
 
 if not PY2:
@@ -224,12 +224,15 @@ class HttpCli(object):
 
         act = self.parser.require("act", 64)
 
-        if act == "bput":
-            return self.handle_plain_upload()
-
         if act == "login":
             return self.handle_login()
 
+        if act == "mkdir":
+            return self.handle_mkdir()
+
+        if act == "bput":
+            return self.handle_plain_upload()
+
         raise Pebkac(422, 'invalid action "{}"'.format(act))
 
     def handle_post_json(self):
@@ -292,7 +295,7 @@ class HttpCli(object):
 
         x = self.conn.hsrv.broker.put(True, "up2k.handle_chunk", wark, chash)
         response = x.get()
-        chunksize, cstart, path = response
+        chunksize, cstart, path, lastmod = response
 
         if self.args.nw:
             path = os.devnull
@@ -336,7 +339,15 @@ class HttpCli(object):
                 self.log("clone {} done".format(cstart[0]))
 
         x = self.conn.hsrv.broker.put(True, "up2k.confirm_chunk", wark, chash)
-        response = x.get()
+        num_left = x.get()
+
+        if not WINDOWS and num_left == 0:
+            times = (int(time.time()), int(lastmod))
+            self.log("no more chunks, setting times {}".format(times))
+            try:
+                os.utime(path, times)
+            except:
+                self.log("failed to utime ({}, {})".format(path, times))
 
         self.loud_reply("thank")
         return True
@@ -356,6 +367,36 @@ class HttpCli(object):
         self.reply(html.encode("utf-8"), headers=h)
         return True
 
+    def handle_mkdir(self):
+        new_dir = self.parser.require("name", 512)
+        self.parser.drop()
+
+        nullwrite = self.args.nw
+        vfs, rem = self.conn.auth.vfs.get(self.vpath, self.uname, False, True)
+
+        # rem is escaped at this point,
+        # this is just a sanity check to prevent any disasters
+        if rem.startswith("/") or rem.startswith("../") or "/../" in rem:
+            raise Exception("that was close")
+
+        if not nullwrite:
+            fdir = os.path.join(vfs.realpath, rem)
+            fn = os.path.join(fdir, sanitize_fn(new_dir))
+
+            if not os.path.isdir(fsenc(fdir)):
+                raise Pebkac(404, "that folder does not exist")
+
+            os.mkdir(fsenc(fn))
+
+        html = self.conn.tpl_msg.render(
+            h2='<a href="/{}">return to /{}</a>'.format(
+                quotep(self.vpath), html_escape(self.vpath, quote=False)
+            ),
+            pre="aight",
+        )
+        self.reply(html.encode("utf-8", "replace"))
+        return True
+
     def handle_plain_upload(self):
         nullwrite = self.args.nw
         vfs, rem = self.conn.auth.vfs.get(self.vpath, self.uname, False, True)
@@ -620,7 +661,9 @@ class HttpCli(object):
 
                 vpnodes.append([quotep(vpath) + "/", html_escape(node, quote=False)])
 
-        vn, rem = self.auth.vfs.get(self.vpath, self.uname, self.readable, self.writable)
+        vn, rem = self.auth.vfs.get(
+            self.vpath, self.uname, self.readable, self.writable
+        )
         abspath = vn.canonical(rem)
 
         if not os.path.exists(fsenc(abspath)):
@@ -684,7 +727,7 @@ class HttpCli(object):
             ts=ts,
             prologue=logues[0],
             epilogue=logues[1],
+            title=quotep(self.vpath),
         )
         self.reply(html.encode("utf-8", "replace"))
         return True
-

copyparty/httpsrv.py

@@ -75,10 +75,12 @@ class HttpSrv(object):
                 sck.shutdown(socket.SHUT_RDWR)
                 sck.close()
             except (OSError, socket.error) as ex:
-                if ex.errno not in [107, 57, 9]:
+                # self.log(str(addr), "shut_rdwr err: " + repr(sck))
-                    # 107 Transport endpoint not connected
+                if ex.errno not in [10038, 107, 57, 9]:
-                    #  57 Socket is not connected
+                    # 10038 No longer considered a socket
-                    #   9 Bad file descriptor
+                    #   107 Transport endpoint not connected
+                    #    57 Socket is not connected
+                    #     9 Bad file descriptor
                     raise
             finally:
                 with self.mutex:

copyparty/svchub.py

@@ -85,16 +85,7 @@ class SvcHub(object):
                 self.next_day = calendar.timegm(dt.utctimetuple())
 
             ts = datetime.utcfromtimestamp(now).strftime("%H:%M:%S.%f")[:-3]
-
+            fmt = "\033[36m{} \033[33m{:21} \033[0m{}"
-            if not WINDOWS:
-                fmt = "\033[36m{} \033[33m{:21} \033[0m{}"
-            else:
-                fmt = "{} {:21} {}"
-                if "\033" in msg:
-                    msg = self.ansi_re.sub("", msg)
-                if "\033" in src:
-                    src = self.ansi_re.sub("", src)
-
             msg = fmt.format(ts, src, msg)
             try:
                 print(msg)

copyparty/up2k.py

@@ -9,8 +9,10 @@ import math
 import base64
 import hashlib
 import threading
+from queue import Queue
 from copy import deepcopy
 
+from .__init__ import WINDOWS
 from .util import Pebkac
 
 
@@ -35,6 +37,13 @@ class Up2k(object):
         self.registry = {}
         self.mutex = threading.Lock()
 
+        if WINDOWS:
+            # usually fails to set lastmod too quickly
+            self.lastmod_q = Queue()
+            thr = threading.Thread(target=self._lastmodder)
+            thr.daemon = True
+            thr.start()
+
         # static
         self.r_hash = re.compile("^[0-9a-zA-Z_-]{43}$")
 
@@ -56,6 +65,7 @@ class Up2k(object):
                     # client-provided, sanitized by _get_wark:
                     "name": cj["name"],
                     "size": cj["size"],
+                    "lmod": cj["lmod"],
                     "hash": deepcopy(cj["hash"]),
                 }
 
@@ -74,6 +84,7 @@ class Up2k(object):
             return {
                 "name": job["name"],
                 "size": job["size"],
+                "lmod": job["lmod"],
                 "hash": job["need"],
                 "wark": wark,
             }
@@ -96,11 +107,19 @@ class Up2k(object):
 
         path = os.path.join(job["vdir"], job["name"])
 
-        return [chunksize, ofs, path]
+        return [chunksize, ofs, path, job["lmod"]]
 
     def confirm_chunk(self, wark, chash):
         with self.mutex:
-            self.registry[wark]["need"].remove(chash)
+            job = self.registry[wark]
+            job["need"].remove(chash)
+            ret = len(job["need"])
+
+            if WINDOWS and ret == 0:
+                path = os.path.join(job["vdir"], job["name"])
+                self.lastmod_q.put([path, (int(time.time()), int(job["lmod"]))])
+
+            return ret
 
     def _get_chunksize(self, filesize):
         chunksize = 1024 * 1024
@@ -115,7 +134,7 @@ class Up2k(object):
                 stepsize *= mul
 
     def _get_wark(self, cj):
-        if len(cj["name"]) > 1024 or len(cj["hash"]) > 256:
+        if len(cj["name"]) > 1024 or len(cj["hash"]) > 512 * 1024:  # 16TiB
             raise Pebkac(400, "name or numchunks not according to spec")
 
         for k in cj["hash"]:
@@ -124,6 +143,12 @@ class Up2k(object):
                     400, "at least one hash is not according to spec: {}".format(k)
                 )
 
+        # try to use client-provided timestamp, don't care if it fails somehow
+        try:
+            cj["lmod"] = int(cj["lmod"])
+        except:
+            cj["lmod"] = int(time.time())
+
         # server-reproducible file identifier, independent of name or location
         ident = [self.salt, str(cj["size"])]
         ident.extend(cj["hash"])
@@ -143,3 +168,16 @@ class Up2k(object):
             f.seek(job["size"] - 1)
             f.write(b"e")
 
+    def _lastmodder(self):
+        while True:
+            ready = []
+            while not self.lastmod_q.empty():
+                ready.append(self.lastmod_q.get())
+
+            # self.log("lmod", "got {}".format(len(ready)))
+            time.sleep(5)
+            for path, times in ready:
+                try:
+                    os.utime(path, times)
+                except:
+                    self.log("lmod", "failed to utime ({}, {})".format(path, times))

copyparty/web/browser.html

@@ -3,7 +3,7 @@
 
 <head>
     <meta charset="utf-8">
-    <title>copyparty</title>
+    <title>⇆🎉 {{ title }}</title>
     <meta http-equiv="X-UA-Compatible" content="IE=edge">
     <meta name="viewport" content="width=device-width, initial-scale=0.8">
     <link rel="stylesheet" type="text/css" media="screen" href="/.cpr/browser.css{{ ts }}">

copyparty/web/up2k.js

@@ -58,22 +58,61 @@ function o(id) {
 
 
 (function () {
-    // chrome requires https to use crypto.subtle,
+    var ops = document.querySelectorAll('#ops>a');
-    // usually it's undefined but some chromes throw on invoke
+    for (var a = 0; a < ops.length; a++) {
-    try {
+        ops[a].onclick = opclick;
-        crypto.subtle.digest(
-            'SHA-512', new Uint8Array(1)
-        ).then(
-            function (x) { up2k_init(true) },
-            function (x) { up2k_init(false) }
-        );
-    }
-    catch (ex) {
-        up2k_init(false);
     }
 })();
 
 
+function opclick(ev) {
+    ev.preventDefault();
+    goto(this.getAttribute('data-dest'));
+}
+
+
+function goto(dest) {
+    var obj = document.querySelectorAll('.opview.act');
+    for (var a = obj.length - 1; a >= 0; a--)
+        obj[a].setAttribute('class', 'opview');
+
+    var obj = document.querySelectorAll('#ops>a');
+    for (var a = obj.length - 1; a >= 0; a--)
+        obj[a].setAttribute('class', '');
+
+    document.querySelector('#ops>a[data-dest=' + dest + ']').setAttribute('class', 'act');
+    document.getElementById('op_' + dest).setAttribute('class', 'opview act');
+
+    var fn = window['goto_' + dest];
+    if (fn)
+        fn();
+}
+
+
+function goto_up2k() {
+    if (!up2k)
+        return setTimeout(goto_up2k, 100);
+
+    up2k.init_deps();
+}
+
+
+// chrome requires https to use crypto.subtle,
+// usually it's undefined but some chromes throw on invoke
+var up2k = null;
+try {
+    crypto.subtle.digest(
+        'SHA-512', new Uint8Array(1)
+    ).then(
+        function (x) { up2k = up2k_init(true) },
+        function (x) { up2k = up2k_init(false) }
+    );
+}
+catch (ex) {
+    up2k = up2k_init(false);
+}
+
+
 function up2k_init(have_crypto) {
     //have_crypto = false;
     var need_filereader_cache = undefined;
@@ -94,7 +133,7 @@ function up2k_init(have_crypto) {
         o('u2notbtn').innerHTML = '';
     }
 
-    var post_url = o('bup').getElementsByTagName('form')[0].getAttribute('action');
+    var post_url = o('op_bup').getElementsByTagName('form')[0].getAttribute('action');
     if (post_url && post_url.charAt(post_url.length - 1) !== '/')
         post_url += '/';
 
@@ -105,13 +144,7 @@ function up2k_init(have_crypto) {
         shame = 'your browser is impressively ancient';
 
     // upload ui hidden by default, clicking the header shows it
-    function toggle_upload_visible(ev) {
+    function init_deps() {
-        if (ev)
-            ev.preventDefault();
-        
-        o('u2tgl').style.display = 'none';
-        o('u2body').style.display = 'block';
-
         if (!have_crypto && !window.asmCrypto) {
             showmodal('<h1>loading sha512.js</h1><h2>since ' + shame + '</h2><h4>thanks chrome</h4>');
             import_js('/.cpr/deps/sha512.js', unmodal);
@@ -122,11 +155,10 @@ function up2k_init(have_crypto) {
                 o('u2foot').innerHTML = 'seems like ' + shame + ' so do that if you want more performance';
         }
     };
-    o('u2tgl').onclick = toggle_upload_visible;
 
     // show uploader if the user only has write-access
     if (!o('files'))
-        toggle_upload_visible();
+        goto('up2k');
 
     // shows or clears an error message in the basic uploader ui
     function setmsg(msg) {
@@ -142,8 +174,7 @@ function up2k_init(have_crypto) {
 
     // switches to the basic uploader with msg as error message
     function un2k(msg) {
-        o('up2k').style.display = 'none';
+        goto('bup');
-        o('bup').style.display = 'block';
         setmsg(msg);
     }
 
@@ -218,10 +249,6 @@ function up2k_init(have_crypto) {
     if (!bobslice || !window.FileReader || !window.FileList)
         return un2k("this is the basic uploader; up2k needs at least<br />chrome 21 // firefox 13 // edge 12 // opera 12 // safari 5.1");
 
-    // probably safe now
-    o('up2k').style.display = 'block';
-    o('bup').style.display = 'none';
-
     function nav() {
         o('file' + fdom_ctr).click();
     }
@@ -272,12 +299,15 @@ function up2k_init(have_crypto) {
                 bad_files.push([a, fobj.name]);
                 continue;
             }
+            var now = new Date().getTime();
+            var lmod = fobj.lastModified || now;
             var entry = {
                 "n": parseInt(st.files.length.toString()),
-                "t0": new Date().getTime(),  // TODO remove probably
+                "t0": now,  // TODO remove probably
                 "fobj": fobj,
                 "name": fobj.name,
                 "size": fobj.size,
+                "lmod": lmod / 1000,
                 "hash": []
             };
 
@@ -689,6 +719,7 @@ function up2k_init(have_crypto) {
         xhr.send(JSON.stringify({
             "name": t.name,
             "size": t.size,
+            "lmod": t.lmod,
             "hash": t.hash
         }));
     }
@@ -838,4 +869,6 @@ function up2k_init(have_crypto) {
         nodes[a].addEventListener('touchend', nop, false);
 
     bumpthread({ "target": 1 })
+
+    return { "init_deps": init_deps }
 }

copyparty/web/upload.css

@@ -1,19 +1,97 @@
-#bup {
+.opview {
-	padding: .5em .5em .5em .3em;
+	display: none;
-	margin: 1em 0 2em 0;
+}
-	background: #2d2d2d;
+.opview.act {
-	border-radius: 0 1em 1em 0;
+	display: block;
+}
+#ops a {
+	color: #fc5;
+	font-size: 1.5em;
+	padding: 0 .3em;
+	margin: 0;
+	outline: none;
+}
+#ops a.act {
+	text-decoration: underline;
+}
+/*
+#ops a+a:after,
+#ops a:first-child:after {
+	content: 'x';
+	color: #282828;
+	text-shadow: 0 0 .08em #01a7e1;
+	margin-left: .3em;
+	position: relative;
+}
+#ops a+a:before {
+	content: 'x';
+	color: #282828;
+	text-shadow: 0 0 .08em #ff3f1a;
+	margin-right: .3em;
+	margin-left: -.3em;
+}
+#ops a:last-child:after {
+	content: '';
+}
+#ops a.act:before,
+#ops a.act:after {
+	text-decoration: none !important;
+}
+*/
+#ops i {
+	font-size: 1.5em;
+}
+#ops i:before {
+	content: 'x';
+	color: #282828;
+	text-shadow: 0 0 .08em #01a7e1;
+	position: relative;
+}
+#ops i:after {
+	content: 'x';
+	color: #282828;
+	text-shadow: 0 0 .08em #ff3f1a;
+	margin-left: -.35em;
+	font-size: 1.05em;
+}
+#ops,
+#op_bup,
+#op_mkdir {
 	border: 1px solid #3a3a3a;
-	border-width: 0 .3em .3em 0;
 	box-shadow: 0 0 1em #222 inset;
+}
+#ops {
+	display: inline-block;
+	background: #333;
+	margin: 1em 1.5em;
+	padding: .3em .6em;
+	border-radius: .3em;
+	border-width: .15em 0;
+}
+#op_bup,
+#op_mkdir {
+	background: #2d2d2d;
+	margin: 1em 0 2em 0;
+	padding: .5em;
+	border-radius: 0 1em 1em 0;
+	border-width: .15em .3em .3em 0;
 	max-width: 40em;
 }
-#bup input {
+#op_mkdir input,
+#op_bup input {
 	margin: .5em;
 }
-#up2k {
+#op_mkdir input[type=text] {
-	display: none;
+	color: #fff;
-	padding: 0 1em;
+	background: #383838;
+	border: none;
+	box-shadow: 0 0 .3em #222;
+	border-bottom: 1px solid #fc5;
+	border-radius: .2em;
+	padding: .2em .3em;
+}
+#op_up2k {
+	padding: 0 1em 1em 1em;
 }
 #u2form {
 	position: absolute;
@@ -29,16 +107,6 @@
 	color: #f87;
 	padding: .5em;
 }
-#u2tgl {
-	color: #fc5;
-	font-size: 1.5em;
-	margin: .5em 0 1em 0;
-	display: block;
-}
-#u2body {
-	display: none;
-	padding-bottom: 1em;
-}
 #u2form {
 	width: 2px;
 	height: 2px;

copyparty/web/upload.html

@@ -1,4 +1,9 @@
-    <div id="bup">
+    <div id="ops"><a
+        href="#" data-dest="up2k">up2k</a><i></i><a
+        href="#" data-dest="bup">bup</a><i></i><a
+        href="#" data-dest="mkdir">mkdir</a></div>
+
+    <div id="op_bup" class="opview">
         <div id="u2err"></div>
         <form method="post" enctype="multipart/form-data" accept-charset="utf-8" action="/{{ vdir }}">
             <input type="hidden" name="act" value="bput" />
@@ -7,10 +12,16 @@
         </form>
     </div>
 
-    <div id="up2k">
+    <div id="op_mkdir" class="opview">
-        <a href="#" id="u2tgl">you can upload here</a>
+        <form method="post" enctype="multipart/form-data" accept-charset="utf-8" action="/{{ vdir }}">
-        <form id="u2form" method="POST" enctype="multipart/form-data" onsubmit="return false;"></form>
+            <input type="hidden" name="act" value="mkdir" />
-        <div id="u2body">
+            <input type="text" name="name" size="30">
+            <input type="submit" value="mkdir">
+        </form>
+    </div>
+
+    <div id="op_up2k" class="opview">
+        <form id="u2form" method="post" enctype="multipart/form-data" onsubmit="return false;"></form>
 
             <table id="u2conf">
                 <tr>
@@ -45,6 +56,5 @@
             </table>
 
             <p id="u2foot"></p>
-            <p>( if you don't need resumable uploads and progress bars just use the <a href="#" id="u2nope" onclick="javascript:un2k();">basic uploader</a>)</p>
+            <p>( if you don't need resumable uploads and progress bars just use the <a href="#" id="u2nope" onclick="javascript:goto('bup');">basic uploader</a>)</p>
-        </div>
     </div>