download folders as zip

README.md

@@ -72,7 +72,7 @@ you may also want these, especially on servers:
   * ☑ symlink/discard existing files (content-matching)
 * download
   * ☑ single files in browser
-  * ✖ folders as zip files
+  * ☑ folders as zip files *(not in release yet)*
   * ☑ FUSE client (read-only)
 * browser
   * ☑ tree-view

copyparty/__main__.py

@@ -261,6 +261,7 @@ def main():
     ap.add_argument("-nw", action="store_true", help="disable writes (benchmark)")
     ap.add_argument("-nih", action="store_true", help="no info hostname")
     ap.add_argument("-nid", action="store_true", help="no info disk-usage")
+    ap.add_argument("--no-zip", action="store_true", help="disable download as zip/tar")
     ap.add_argument("--no-sendfile", action="store_true", help="disable sendfile (for debugging)")
     ap.add_argument("--no-scandir", action="store_true", help="disable scandir (for debugging)")
     ap.add_argument("--urlform", metavar="MODE", type=str, default="print,get", help="how to handle url-forms")

copyparty/authsrv.py

@@ -4,6 +4,7 @@ from __future__ import print_function, unicode_literals
 import re
 import os
 import sys
+import stat
 import threading
 
 from .__init__ import PY2, WINDOWS
@@ -127,6 +128,64 @@ class VFS(object):
 
         return [abspath, real, virt_vis]
 
+    def walk(self, rel, rem, uname, dots, scandir, lstat=False):
+        """
+        recursively yields from ./rem;
+        rel is a unix-style user-defined vpath (not vfs-related)
+        """
+
+        fsroot, vfs_ls, vfs_virt = self.ls(rem, uname, scandir, lstat)
+        rfiles = [x for x in vfs_ls if not stat.S_ISDIR(x[1].st_mode)]
+        rdirs = [x for x in vfs_ls if stat.S_ISDIR(x[1].st_mode)]
+
+        rfiles.sort()
+        rdirs.sort()
+
+        yield rel, fsroot, rfiles, rdirs, vfs_virt
+
+        for rdir, _ in rdirs:
+            if not dots and rdir.startswith("."):
+                continue
+
+            wrel = (rel + "/" + rdir).lstrip("/")
+            wrem = (rem + "/" + rdir).lstrip("/")
+            for x in self.walk(wrel, wrem, uname, scandir, lstat):
+                yield x
+
+        for n, vfs in sorted(vfs_virt.items()):
+            if not dots and n.startswith("."):
+                continue
+
+            wrel = (rel + "/" + n).lstrip("/")
+            for x in vfs.walk(wrel, "", uname, scandir, lstat):
+                yield x
+
+    def zipgen(self, rems, uname, dots, scandir):
+        vtops = [["", [self, ""]]]
+        if rems:
+            # list of subfolders to zip was provided,
+            # add all the ones uname is allowed to access
+            vtops = []
+            for rem in rems:
+                try:
+                    vn = self.get(rem, uname, True, False)
+                    vtops.append([rem, vn])
+                except:
+                    pass
+
+        for rel, (vn, rem) in vtops:
+            for vpath, apath, files, _, _ in vn.walk(rel, rem, uname, dots, scandir):
+                # print(repr([vpath, apath, [x[0] for x in files]]))
+                files = [x for x in files if dots or not x[0].startswith(".")]
+                fnames = [n[0] for n in files]
+                vpaths = [vpath + "/" + n for n in fnames] if vpath else fnames
+                apaths = [os.path.join(apath, n) for n in fnames]
+                for f in [
+                    {"vp": vp, "ap": ap, "st": n[1]}
+                    for vp, ap, n in zip(vpaths, apaths, files)
+                ]:
+                    yield f
+
     def user_tree(self, uname, readable=False, writable=False):
         ret = []
         opt1 = readable and (uname in self.uread or "*" in self.uread)

copyparty/httpcli.py

@@ -7,6 +7,7 @@ import gzip
 import time
 import copy
 import json
+import string
 import socket
 import ctypes
 from datetime import datetime
@@ -14,6 +15,8 @@ import calendar
 
 from .__init__ import E, PY2, WINDOWS
 from .util import *  # noqa  # pylint: disable=unused-wildcard-import
+from .szip import StreamZip
+from .star import StreamTar
 
 if not PY2:
     unicode = str
@@ -1044,6 +1047,63 @@ class HttpCli(object):
         self.log("{},  {}".format(logmsg, spd))
         return ret
 
+    def tx_zip(self, vn, rems, dots):
+        if self.args.no_zip:
+            raise Pebkac(400, "not enabled")
+
+        logmsg = "{:4} {} ".format("", self.req)
+        self.keepalive = False
+
+        fmt = "zip"
+        if fmt == "tar":
+            mime = "application/x-tar"
+        else:
+            mime = "application/zip"
+
+        if rems and rems[0]:
+            fn = rems[0]
+        else:
+            fn = self.vpath.rstrip("/").split("/")[-1]
+
+        if not fn:
+            fn = self.headers.get("host", "hey")
+
+        afn = "".join(
+            [x if x in (string.ascii_letters + string.digits) else "_" for x in fn]
+        )
+
+        ufn = "".join(
+            [
+                x
+                if x in (string.ascii_letters + string.digits)
+                else "%{:02x}".format(ord(x))
+                for x in fn
+            ]
+        )
+
+        cdis = 'attachment; filename="{}.{}", filename*=UTF-8''{}.{}"
+        cdis = cdis.format(afn, fmt, ufn, fmt)
+        self.send_headers(None, mime=mime, headers={"Content-Disposition": cdis})
+
+        fgen = vn.zipgen(rems, self.uname, dots, not self.args.no_scandir)
+        # for f in fgen: print(repr({k: f[k] for k in ["vp", "ap"]}))
+        bgen = StreamZip(fgen, False, False)
+        bsent = 0
+        for buf in bgen.gen():
+            if not buf:
+                break
+
+            try:
+                self.s.sendall(buf)
+                bsent += len(buf)
+            except:
+                logmsg += " \033[31m" + unicode(bsent) + "\033[0m"
+                break
+
+        spd = self._spd(bsent)
+        self.log("{},  {}".format(logmsg, spd))
+        return True
+
     def tx_md(self, fs_path):
         logmsg = "{:4} {} ".format("", self.req)
 
@@ -1190,6 +1250,9 @@ class HttpCli(object):
 
             return self.tx_file(abspath)
 
+        if "zip" in self.uparam:
+            return self.tx_zip(vn, None, False)
+
         fsroot, vfs_ls, vfs_virt = vn.ls(rem, self.uname, not self.args.no_scandir)
         stats = {k: v for k, v in vfs_ls}
         vfs_ls = [x[0] for x in vfs_ls]
@@ -1250,8 +1313,11 @@ class HttpCli(object):
 
             is_dir = stat.S_ISDIR(inf.st_mode)
             if is_dir:
-                margin = "DIR"
                 href += "/"
+                if self.args.no_zip:
+                    margin = "DIR"
+                else:
+                    margin = '<a href="{}?zip">zip</a>'.format(html_escape(href))
             elif fn in hist:
                 margin = '<a href="{}.hist/{}">#{}</a>'.format(
                     base, html_escape(hist[fn][2], quote=True), hist[fn][0]

copyparty/star.py

@@ -1,8 +1,7 @@
-import os
 import tarfile
 import threading
 
-from .util import Queue
+from .util import Queue, fsenc
 
 
 class QFile(object):
@@ -46,11 +45,11 @@ class StreamTar(object):
 
     def _gen(self):
         for f in self.fgen:
-            src = f["a"]
-            name = f["n"]
-            inf = tarfile.TarInfo(name=name)
+            name = f["vp"]
+            src = f["ap"]
+            fsi = f["st"]
 
-            fsi = os.stat(src)
+            inf = tarfile.TarInfo(name=name)
             inf.mode = fsi.st_mode
             inf.size = fsi.st_size
             inf.mtime = fsi.st_mtime
@@ -58,7 +57,7 @@ class StreamTar(object):
             inf.gid = 0
 
             self.ci += inf.size
-            with open(src, "rb") as f:
+            with open(fsenc(src), "rb", 512 * 1024) as f:
                 self.tar.addfile(inf, f)
 
         self.tar.close()

copyparty/szip.py

@@ -174,8 +174,7 @@ def gen_ecdr64_loc(ecdr64_pos):
 
 
 class StreamZip(object):
-    def __init__(self, top, fgen, utf8, pre_crc):
-        self.top = top
+    def __init__(self, fgen, utf8, pre_crc):
         self.fgen = fgen
         self.utf8 = utf8
         self.pre_crc = pre_crc
@@ -189,10 +188,10 @@ class StreamZip(object):
 
     def gen(self):
         for f in self.fgen:
-            src = f["a"]
-            name = f["n"]
+            name = f["vp"]
+            src = f["ap"]
+            st = f["st"]
 
-            st = os.stat(fsenc(src))
             sz = st.st_size
             ts = st.st_mtime + 1
 
@@ -201,9 +200,9 @@ class StreamZip(object):
             yield self._ct(buf)
 
             crc = 0
-            with open(src, "rb") as f:
+            with open(fsenc(src), "rb", 512 * 1024) as f:
                 while True:
-                    buf = f.read(32768)
+                    buf = f.read(64 * 1024)
                     if not buf:
                         break