From da01413b7baac477f20d0cc9dab582d7658d7ac7 Mon Sep 17 00:00:00 2001 From: ed Date: Thu, 4 Mar 2021 01:21:04 +0100 Subject: [PATCH] remove speedbumps --- copyparty/__main__.py | 3 ++- copyparty/authsrv.py | 11 +++++------ copyparty/httpcli.py | 14 ++++++++------ copyparty/up2k.py | 18 +++--------------- copyparty/util.py | 28 +++++++++++++++++++++++++--- 5 files changed, 43 insertions(+), 31 deletions(-) diff --git a/copyparty/__main__.py b/copyparty/__main__.py index 09798a17..3dc2b136 100644 --- a/copyparty/__main__.py +++ b/copyparty/__main__.py @@ -243,7 +243,8 @@ def main(): ap.add_argument("-nw", action="store_true", help="disable writes (benchmark)") ap.add_argument("-nih", action="store_true", help="no info hostname") ap.add_argument("-nid", action="store_true", help="no info disk-usage") - ap.add_argument("--no-sendfile", action="store_true", help="disable sendfile") + ap.add_argument("--no-sendfile", action="store_true", help="disable sendfile (for debugging)") + ap.add_argument("--no-scandir", action="store_true", help="disable scandir (for debugging)") ap.add_argument("--urlform", type=str, default="print,get", help="how to handle url-forms") ap.add_argument("--salt", type=str, default="hunter2", help="up2k file-hash salt") diff --git a/copyparty/authsrv.py b/copyparty/authsrv.py index c0833756..3cb44a68 100644 --- a/copyparty/authsrv.py +++ b/copyparty/authsrv.py @@ -6,7 +6,7 @@ import re import threading from .__init__ import PY2, WINDOWS -from .util import undot, Pebkac, fsdec, fsenc +from .util import undot, Pebkac, fsdec, fsenc, statdir class VFS(object): @@ -102,12 +102,11 @@ class VFS(object): return fsdec(os.path.realpath(fsenc(rp))) - def ls(self, rem, uname): + def ls(self, rem, uname, scandir, lstat=False): """return user-readable [fsdir,real,virt] items at vpath""" virt_vis = {} # nodes readable by user abspath = self.canonical(rem) - items = os.listdir(fsenc(abspath)) - real = [fsdec(x) for x in items] + real = list(statdir(print, scandir, lstat, abspath)) real.sort() if not rem: for name, vn2 in sorted(self.nodes.items()): @@ -115,7 +114,7 @@ class VFS(object): virt_vis[name] = vn2 # no vfs nodes in the list of real inodes - real = [x for x in real if x not in self.nodes] + real = [x for x in real if x[0] not in self.nodes] return [abspath, real, virt_vis] @@ -315,7 +314,7 @@ class AuthSrv(object): if (self.args.e2ds and vol.uwrite) or self.args.e2dsa: vol.flags["e2ds"] = True - if self.args.e2d: + if self.args.e2d or "e2ds" in vol.flags: vol.flags["e2d"] = True for k in ["e2t", "e2ts", "e2tsr"]: diff --git a/copyparty/httpcli.py b/copyparty/httpcli.py index 13ea0f68..2bf47e32 100644 --- a/copyparty/httpcli.py +++ b/copyparty/httpcli.py @@ -1119,7 +1119,7 @@ class HttpCli(object): try: vn, rem = self.auth.vfs.get(top, self.uname, True, False) - fsroot, vfs_ls, vfs_virt = vn.ls(rem, self.uname) + fsroot, vfs_ls, vfs_virt = vn.ls(rem, self.uname, not self.args.no_sendfile) except: vfs_ls = [] vfs_virt = {} @@ -1130,13 +1130,13 @@ class HttpCli(object): dirs = [] + vfs_ls = [x[0] for x in vfs_virt if stat.S_ISDIR(x[1])] + if not self.args.ed or "dots" not in self.uparam: vfs_ls = exclude_dotfiles(vfs_ls) for fn in [x for x in vfs_ls if x != excl]: - abspath = os.path.join(fsroot, fn) - if os.path.isdir(abspath): - dirs.append(fn) + dirs.append(os.path.join(fsroot, fn)) for x in vfs_virt.keys(): if x != excl: @@ -1175,7 +1175,9 @@ class HttpCli(object): return self.tx_file(abspath) - fsroot, vfs_ls, vfs_virt = vn.ls(rem, self.uname) + fsroot, vfs_ls, vfs_virt = vn.ls(rem, self.uname, not self.args.no_sendfile) + stats = {k: v for k, v in vfs_ls} + vfs_ls = [x[0] for x in vfs_ls] vfs_ls.extend(vfs_virt.keys()) # check for old versions of files, @@ -1226,7 +1228,7 @@ class HttpCli(object): fspath = fsroot + "/" + fn try: - inf = os.stat(fsenc(fspath)) + inf = stats[fn] except: self.log("broken symlink: {}".format(repr(fspath))) continue diff --git a/copyparty/up2k.py b/copyparty/up2k.py index 7aad942b..e9a768cd 100644 --- a/copyparty/up2k.py +++ b/copyparty/up2k.py @@ -3,7 +3,6 @@ from __future__ import print_function, unicode_literals import re import os -import sys import time import math import json @@ -28,6 +27,7 @@ from .util import ( atomic_move, w8b64enc, w8b64dec, + statdir, ) from .mtag import MTag from .authsrv import AuthSrv @@ -284,23 +284,11 @@ class Up2k(object): self.log(msg) def _build_dir(self, dbw, top, excl, cdir): - try: - inodes = [fsdec(x) for x in os.listdir(fsenc(cdir))] - except Exception as ex: - self.log("listdir: {} @ [{}]".format(repr(ex), cdir)) - return 0 - self.pp.msg = "a{} {}".format(self.pp.n, cdir) histdir = os.path.join(top, ".hist") ret = 0 - for inode in inodes: - abspath = os.path.join(cdir, inode) - try: - inf = os.stat(fsenc(abspath)) - except Exception as ex: - self.log("stat: {} @ [{}]".format(repr(ex), abspath)) - continue - + for iname, inf in statdir(self.log, not self.args.no_scandir, False, cdir): + abspath = os.path.join(cdir, iname) lmod = int(inf.st_mtime) if stat.S_ISDIR(inf.st_mode): if abspath in excl or abspath == histdir: diff --git a/copyparty/util.py b/copyparty/util.py index 733bfb80..37d71718 100644 --- a/copyparty/util.py +++ b/copyparty/util.py @@ -521,9 +521,7 @@ def u8safe(txt): def exclude_dotfiles(filepaths): - for fpath in filepaths: - if not fpath.split("/")[-1].startswith("."): - yield fpath + return [x for x in filepaths if not x.split("/")[-1].startswith(".")] def html_escape(s, quote=False): @@ -726,6 +724,30 @@ def sendfile_kern(lower, upper, f, s): return 0 +def statdir(logger, scandir, lstat, top): + try: + btop = fsenc(top) + if scandir and hasattr(os, "scandir"): + src = "scandir" + with os.scandir(btop) as dh: + for fh in dh: + try: + yield [fsdec(fh.name), fh.stat(follow_symlinks=not lstat)] + except Exception as ex: + logger("scan-stat: {} @ {}".format(repr(ex), fsdec(fh.path))) + else: + src = "listdir" + fun = os.lstat if lstat else os.stat + for name in os.listdir(btop): + abspath = os.path.join(btop, name) + try: + yield [fsdec(name), fun(abspath)] + except Exception as ex: + logger("list-stat: {} @ {}".format(repr(ex), fsdec(abspath))) + except Exception as ex: + logger("{}: {} @ {}".format(src, repr(ex), top)) + + def unescape_cookie(orig): # mw=idk; doot=qwe%2Crty%3Basd+fgh%2Bjkl%25zxc%26vbn # qwe,rty;asd fgh+jkl%zxc&vbn ret = ""