download as tar + utf8 zip + optimize walk

copyparty/authsrv.py

@@ -175,16 +175,31 @@ class VFS(object):
                     pass
 
         for rel, (vn, rem) in vtops:
-            for vpath, apath, files, _, _ in vn.walk(rel, rem, uname, dots, scandir):
+            for vpath, apath, files, rd, vd in vn.walk(rel, rem, uname, dots, scandir):
                 # print(repr([vpath, apath, [x[0] for x in files]]))
-                files = [x for x in files if dots or not x[0].startswith(".")]
                 fnames = [n[0] for n in files]
                 vpaths = [vpath + "/" + n for n in fnames] if vpath else fnames
                 apaths = [os.path.join(apath, n) for n in fnames]
-                for f in [
-                    {"vp": vp, "ap": ap, "st": n[1]}
-                    for vp, ap, n in zip(vpaths, apaths, files)
-                ]:
+                files = list(zip(vpaths, apaths, files))
+
+                if not dots:
+                    # dotfile filtering based on vpath (intended visibility)
+                    files = [x for x in files if "/." not in "/" + x[0]]
+
+                    rm = [x for x in rd if x[0].startswith(".")]
+                    for x in rm:
+                        rd.remove(x)
+
+                    rm = [k for k in vd.keys() if k.startswith(".")]
+                    for x in rm:
+                        del vd[x]
+
+                # up2k filetring based on actual abspath
+                files = [
+                    x for x in files if "{0}.hist{0}up2k.".format(os.sep) not in x[1]
+                ]
+
+                for f in [{"vp": v, "ap": a, "st": n[1]} for v, a, n in files]:
                     yield f
 
     def user_tree(self, uname, readable=False, writable=False):

copyparty/httpcli.py

@@ -1047,18 +1047,22 @@ class HttpCli(object):
         self.log("{},  {}".format(logmsg, spd))
         return ret
 
-    def tx_zip(self, vn, rem, items, dots):
+    def tx_zip(self, fmt, uarg, vn, rem, items, dots):
         if self.args.no_zip:
             raise Pebkac(400, "not enabled")
 
         logmsg = "{:4} {} ".format("", self.req)
         self.keepalive = False
 
-        fmt = "zip"
+        if not uarg:
+            uarg = ""
+
         if fmt == "tar":
             mime = "application/x-tar"
+            packer = StreamTar
         else:
             mime = "application/zip"
+            packer = StreamZip
 
         fn = items[0] if items and items[0] else self.vpath
         if fn:
@@ -1071,12 +1075,13 @@ class HttpCli(object):
         )
 
         bascii = unicode(string.ascii_letters + string.digits).encode("utf-8")
+        chcon = ord if PY2 else int
         ufn = b"".join(
             [
                 chr(x).encode("utf-8")
                 if x in bascii
-                else "%{:02x}".format(x).encode("ascii")
-                for x in fn.encode("utf-8")
+                else "%{:02x}".format(chcon(x)).encode("ascii")
+                for x in fn.encode("utf-8", "xmlcharrefreplace")
             ]
         ).decode("ascii")
 
@@ -1086,7 +1091,7 @@ class HttpCli(object):
 
         fgen = vn.zipgen(rem, items, self.uname, dots, not self.args.no_scandir)
         # for f in fgen: print(repr({k: f[k] for k in ["vp", "ap"]}))
-        bgen = StreamZip(fgen, False, False)
+        bgen = packer(fgen, utf8="utf" in uarg)
         bsent = 0
         for buf in bgen.gen():
             if not buf:
@@ -1249,8 +1254,10 @@ class HttpCli(object):
 
             return self.tx_file(abspath)
 
-        if "zip" in self.uparam:
-            return self.tx_zip(vn, rem, [], False)
+        for k in ["zip", "tar"]:
+            v = self.uparam.get(k)
+            if v is not None:
+                return self.tx_zip(k, v, vn, rem, [], self.args.ed)
 
         fsroot, vfs_ls, vfs_virt = vn.ls(rem, self.uname, not self.args.no_scandir)
         stats = {k: v for k, v in vfs_ls}
@@ -1316,7 +1323,7 @@ class HttpCli(object):
                 if self.args.no_zip:
                     margin = "DIR"
                 else:
-                    margin = '<a href="{}?zip">zip</a>'.format(html_escape(href))
+                    margin = '<a href="{}?zip">zip</a>'.format(quotep(href))
             elif fn in hist:
                 margin = '<a href="{}.hist/{}">#{}</a>'.format(
                     base, html_escape(hist[fn][2], quote=True), hist[fn][0]

copyparty/star.py

@@ -17,11 +17,11 @@ class QFile(object):
 class StreamTar(object):
     """construct in-memory tar file from the given path"""
 
-    def __init__(self, top, fgen):
+    def __init__(self, fgen, **kwargs):
         self.ci = 0
         self.co = 0
         self.qfile = QFile()
-        self.srcfiles = fgen
+        self.fgen = fgen
 
         # python 3.8 changed to PAX_FORMAT as default,
         # waste of space and don't care about the new features

copyparty/szip.py

@@ -174,7 +174,7 @@ def gen_ecdr64_loc(ecdr64_pos):
 
 
 class StreamZip(object):
-    def __init__(self, fgen, utf8, pre_crc):
+    def __init__(self, fgen, utf8=False, pre_crc=False):
         self.fgen = fgen
         self.utf8 = utf8
         self.pre_crc = pre_crc

copyparty/web/up2k.js

@@ -278,7 +278,7 @@ function up2k_init(have_crypto) {
         }
         else files = e.target.files;
 
-        if (files.length == 0)
+        if (!files || files.length == 0)
             return alert('no files selected??');
 
         more_one_file();