remove speedbumps

copyparty/__main__.py

@@ -243,7 +243,8 @@ def main():
     ap.add_argument("-nw", action="store_true", help="disable writes (benchmark)")
     ap.add_argument("-nih", action="store_true", help="no info hostname")
     ap.add_argument("-nid", action="store_true", help="no info disk-usage")
-    ap.add_argument("--no-sendfile", action="store_true", help="disable sendfile")
+    ap.add_argument("--no-sendfile", action="store_true", help="disable sendfile (for debugging)")
+    ap.add_argument("--no-scandir", action="store_true", help="disable scandir (for debugging)")
     ap.add_argument("--urlform", type=str, default="print,get", help="how to handle url-forms")
     ap.add_argument("--salt", type=str, default="hunter2", help="up2k file-hash salt")
 

copyparty/authsrv.py

@@ -6,7 +6,7 @@ import re
 import threading
 
 from .__init__ import PY2, WINDOWS
-from .util import undot, Pebkac, fsdec, fsenc
+from .util import undot, Pebkac, fsdec, fsenc, statdir
 
 
 class VFS(object):
@@ -102,12 +102,11 @@ class VFS(object):
 
         return fsdec(os.path.realpath(fsenc(rp)))
 
-    def ls(self, rem, uname):
+    def ls(self, rem, uname, scandir, lstat=False):
         """return user-readable [fsdir,real,virt] items at vpath"""
         virt_vis = {}  # nodes readable by user
         abspath = self.canonical(rem)
-        items = os.listdir(fsenc(abspath))
-        real = [fsdec(x) for x in items]
+        real = list(statdir(print, scandir, lstat, abspath))
         real.sort()
         if not rem:
             for name, vn2 in sorted(self.nodes.items()):
@@ -115,7 +114,7 @@ class VFS(object):
                     virt_vis[name] = vn2
 
             # no vfs nodes in the list of real inodes
-            real = [x for x in real if x not in self.nodes]
+            real = [x for x in real if x[0] not in self.nodes]
 
         return [abspath, real, virt_vis]
 
@@ -315,7 +314,7 @@ class AuthSrv(object):
             if (self.args.e2ds and vol.uwrite) or self.args.e2dsa:
                 vol.flags["e2ds"] = True
 
-            if self.args.e2d:
+            if self.args.e2d or "e2ds" in vol.flags:
                 vol.flags["e2d"] = True
 
             for k in ["e2t", "e2ts", "e2tsr"]:

copyparty/httpcli.py

@@ -1119,7 +1119,7 @@ class HttpCli(object):
 
         try:
             vn, rem = self.auth.vfs.get(top, self.uname, True, False)
-            fsroot, vfs_ls, vfs_virt = vn.ls(rem, self.uname)
+            fsroot, vfs_ls, vfs_virt = vn.ls(rem, self.uname, not self.args.no_sendfile)
         except:
             vfs_ls = []
             vfs_virt = {}
@@ -1130,13 +1130,13 @@ class HttpCli(object):
 
         dirs = []
 
+        vfs_ls = [x[0] for x in vfs_virt if stat.S_ISDIR(x[1])]
+
         if not self.args.ed or "dots" not in self.uparam:
             vfs_ls = exclude_dotfiles(vfs_ls)
 
         for fn in [x for x in vfs_ls if x != excl]:
-            abspath = os.path.join(fsroot, fn)
-            if os.path.isdir(abspath):
-                dirs.append(fn)
+            dirs.append(os.path.join(fsroot, fn))
 
         for x in vfs_virt.keys():
             if x != excl:
@@ -1175,7 +1175,9 @@ class HttpCli(object):
 
             return self.tx_file(abspath)
 
-        fsroot, vfs_ls, vfs_virt = vn.ls(rem, self.uname)
+        fsroot, vfs_ls, vfs_virt = vn.ls(rem, self.uname, not self.args.no_sendfile)
+        stats = {k: v for k, v in vfs_ls}
+        vfs_ls = [x[0] for x in vfs_ls]
         vfs_ls.extend(vfs_virt.keys())
 
         # check for old versions of files,
@@ -1226,7 +1228,7 @@ class HttpCli(object):
                 fspath = fsroot + "/" + fn
 
             try:
-                inf = os.stat(fsenc(fspath))
+                inf = stats[fn]
             except:
                 self.log("broken symlink: {}".format(repr(fspath)))
                 continue

copyparty/up2k.py

@@ -3,7 +3,6 @@ from __future__ import print_function, unicode_literals
 
 import re
 import os
-import sys
 import time
 import math
 import json
@@ -28,6 +27,7 @@ from .util import (
     atomic_move,
     w8b64enc,
     w8b64dec,
+    statdir,
 )
 from .mtag import MTag
 from .authsrv import AuthSrv
@@ -284,23 +284,11 @@ class Up2k(object):
         self.log(msg)
 
     def _build_dir(self, dbw, top, excl, cdir):
-        try:
-            inodes = [fsdec(x) for x in os.listdir(fsenc(cdir))]
-        except Exception as ex:
-            self.log("listdir: {} @ [{}]".format(repr(ex), cdir))
-            return 0
-
         self.pp.msg = "a{} {}".format(self.pp.n, cdir)
         histdir = os.path.join(top, ".hist")
         ret = 0
-        for inode in inodes:
-            abspath = os.path.join(cdir, inode)
-            try:
-                inf = os.stat(fsenc(abspath))
-            except Exception as ex:
-                self.log("stat: {} @ [{}]".format(repr(ex), abspath))
-                continue
-
+        for iname, inf in statdir(self.log, not self.args.no_scandir, False, cdir):
+            abspath = os.path.join(cdir, iname)
             lmod = int(inf.st_mtime)
             if stat.S_ISDIR(inf.st_mode):
                 if abspath in excl or abspath == histdir:

copyparty/util.py

@@ -521,9 +521,7 @@ def u8safe(txt):
 
 
 def exclude_dotfiles(filepaths):
-    for fpath in filepaths:
-        if not fpath.split("/")[-1].startswith("."):
-            yield fpath
+    return [x for x in filepaths if not x.split("/")[-1].startswith(".")]
 
 
 def html_escape(s, quote=False):
@@ -726,6 +724,30 @@ def sendfile_kern(lower, upper, f, s):
     return 0
 
 
+def statdir(logger, scandir, lstat, top):
+    try:
+        btop = fsenc(top)
+        if scandir and hasattr(os, "scandir"):
+            src = "scandir"
+            with os.scandir(btop) as dh:
+                for fh in dh:
+                    try:
+                        yield [fsdec(fh.name), fh.stat(follow_symlinks=not lstat)]
+                    except Exception as ex:
+                        logger("scan-stat: {} @ {}".format(repr(ex), fsdec(fh.path)))
+        else:
+            src = "listdir"
+            fun = os.lstat if lstat else os.stat
+            for name in os.listdir(btop):
+                abspath = os.path.join(btop, name)
+                try:
+                    yield [fsdec(name), fun(abspath)]
+                except Exception as ex:
+                    logger("list-stat: {} @ {}".format(repr(ex), fsdec(abspath)))
+    except Exception as ex:
+        logger("{}: {} @ {}".format(src, repr(ex), top))
+
+
 def unescape_cookie(orig):
     # mw=idk; doot=qwe%2Crty%3Basd+fgh%2Bjkl%25zxc%26vbn  # qwe,rty;asd fgh+jkl%zxc&vbn
     ret = ""