From b5f2fe2f0aaee36ecad19bc17d50da2f4d6a157b Mon Sep 17 00:00:00 2001 From: ed Date: Sun, 13 Feb 2022 03:10:53 +0100 Subject: [PATCH] add ftpd --- README.md | 12 +++ copyparty/__main__.py | 6 ++ copyparty/bos/bos.py | 13 ++- copyparty/bos/path.py | 4 + copyparty/ftpd.py | 231 ++++++++++++++++++++++++++++++++++++++++++ copyparty/svchub.py | 5 + scripts/make-sfx.sh | 17 +++- scripts/sfx.ls | 1 + scripts/sfx.py | 2 +- setup.py | 6 +- 10 files changed, 289 insertions(+), 8 deletions(-) create mode 100644 copyparty/ftpd.py diff --git a/README.md b/README.md index d92ff196..36292a0a 100644 --- a/README.md +++ b/README.md @@ -154,6 +154,7 @@ feature summary * ☑ multiprocessing (actual multithreading) * ☑ volumes (mountpoints) * ☑ [accounts](#accounts-and-volumes) + * ☑ [ftp-server](#ftp-server) * upload * ☑ basic: plain multipart, ie6 support * ☑ [up2k](#uploading): js, resumable, multithreaded @@ -237,6 +238,8 @@ some improvement ideas ## general bugs +* Windows: if the up2k db is on a samba-share or network disk, you'll get unpredictable behavior if the share is disconnected for a bit + * use `--hist` or the `hist` volflag (`-v [...]:c,hist=/tmp/foo`) to place the db on a local disk instead * all volumes must exist / be available on startup; up2k (mtp especially) gets funky otherwise * probably more, pls let me know @@ -621,6 +624,15 @@ using arguments or config files, or a mix of both: * or click the `[reload cfg]` button in the control-panel when logged in as admin +## ftp-server + +an FTP server can be started using `--ftp 2121` (or any other port) + +* based on [pyftpdlib](https://github.com/giampaolo/pyftpdlib) +* needs a dedicated port (cannot share with the HTTP/HTTPS API) +* runs in active mode by default, you probably want `--ftp-r` + + ## file indexing file indexing relies on two database tables, the up2k filetree (`-e2d`) and the metadata tags (`-e2t`), stored in `.hist/up2k.db`. Configuration can be done through arguments, volume flags, or a mix of both. diff --git a/copyparty/__main__.py b/copyparty/__main__.py index 551defb2..2a20f47b 100644 --- a/copyparty/__main__.py +++ b/copyparty/__main__.py @@ -445,6 +445,12 @@ def run_argparse(argv, formatter): ap2.add_argument("--ssl-dbg", action="store_true", help="dump some tls info") ap2.add_argument("--ssl-log", metavar="PATH", type=u, help="log master secrets") + ap2 = ap.add_argument_group('FTP options') + ap2.add_argument("--ftp", metavar="PORT", type=int, help="enable FTP server on PORT") + ap2.add_argument("--ftp-debug", action="store_true", help="enable debug logging") + ap2.add_argument("--ftp-nat", metavar="ADDR", type=u, help="the NAT address to use for passive connections") + ap2.add_argument("--ftp-r", metavar="P-P", type=u, help="the range of TCP ports to use for passive connections, for example 12000-13000") + ap2 = ap.add_argument_group('opt-outs') ap2.add_argument("-nw", action="store_true", help="disable writes (benchmark)") ap2.add_argument("--keep-qem", action="store_true", help="do not disable quick-edit-mode on windows") diff --git a/copyparty/bos/bos.py b/copyparty/bos/bos.py index 37672f85..d5e003cf 100644 --- a/copyparty/bos/bos.py +++ b/copyparty/bos/bos.py @@ -18,10 +18,6 @@ def listdir(p="."): return [fsdec(x) for x in os.listdir(fsenc(p))] -def lstat(p): - return os.lstat(fsenc(p)) - - def makedirs(name, mode=0o755, exist_ok=True): bname = fsenc(name) try: @@ -60,3 +56,12 @@ def utime(p, times=None, follow_symlinks=True): return os.utime(fsenc(p), times, follow_symlinks=follow_symlinks) else: return os.utime(fsenc(p), times) + + +if hasattr(os, "lstat"): + + def lstat(p): + return os.lstat(fsenc(p)) + +else: + lstat = stat diff --git a/copyparty/bos/path.py b/copyparty/bos/path.py index 0ea543f6..066453b0 100644 --- a/copyparty/bos/path.py +++ b/copyparty/bos/path.py @@ -36,5 +36,9 @@ def islink(p): return os.path.islink(fsenc(p)) +def lexists(p): + return os.path.lexists(fsenc(p)) + + def realpath(p): return fsdec(os.path.realpath(fsenc(p))) diff --git a/copyparty/ftpd.py b/copyparty/ftpd.py new file mode 100644 index 00000000..5e40e823 --- /dev/null +++ b/copyparty/ftpd.py @@ -0,0 +1,231 @@ +# coding: utf-8 +from __future__ import print_function, unicode_literals + +import os +import stat +import logging +import threading +from typing import TYPE_CHECKING +from pyftpdlib.authorizers import DummyAuthorizer, AuthenticationFailed +from pyftpdlib.filesystems import AbstractedFS, FilesystemError +from pyftpdlib.handlers import FTPHandler +from pyftpdlib.servers import FTPServer +from pyftpdlib.ioloop import IOLoop +from pyftpdlib.log import config_logging + +from .util import Pebkac, fsenc, exclude_dotfiles +from .bos import bos +from .authsrv import AuthSrv + +if TYPE_CHECKING: + from .svchub import SvcHub + from .authsrv import AuthSrv + + +class FtpAuth(DummyAuthorizer): + def __init__(self): + super(FtpAuth, self).__init__() + self.hub = None # type: SvcHub + + def validate_authentication(self, username, password, handler): + asrv = self.hub.asrv + if username == "anonymous": + password = "" + + uname = "*" + if password: + uname = asrv.iacct.get(password, None) + + handler.username = uname + + if password and not uname: + raise AuthenticationFailed("Authentication failed.") + + def get_home_dir(self, username): + return "/" + + def has_user(self, username): + asrv = self.hub.asrv + return username in asrv.acct + + def has_perm(self, username, perm, path=None): + return True # handled at filesystem layer + + def get_perms(self, username): + return "elradfmwMT" + + def get_msg_login(self, username): + return "sup" + + def get_msg_quit(self, username): + return "cya" + + +class FtpFs(AbstractedFS): + def __init__(self, root, cmd_channel): + self.h = self.cmd_channel = cmd_channel # type: FTPHandler + self.asrv = cmd_channel.asrv # type: AuthSrv + self.args = cmd_channel.args + + self.uname = self.asrv.iacct.get(cmd_channel.password, "*") + + self.cwd = "/" # pyftpdlib convention of leading slash + self.root = "/var/lib/empty" + + self.listdirinfo = self.listdir + + def v2a(self, vpath, r=False, w=False, m=False, d=False): + try: + vpath = vpath.lstrip("/") + vfs, rem = self.asrv.vfs.get(vpath, self.uname, r, w, m, d) + return os.path.join(vfs.realpath, rem) + except Pebkac as ex: + raise FilesystemError(str(ex)) + + def rv2a(self, vpath, r=False, w=False, m=False, d=False): + return self.v2a(os.path.join(self.cwd, vpath), r, w, m, d) + + def ftp2fs(self, ftppath): + # return self.v2a(ftppath) + return ftppath # self.cwd must be vpath + + def fs2ftp(self, fspath): + # raise NotImplementedError() + return fspath + + def validpath(self, path): + # other funcs handle permission checking implicitly + return True + + def open(self, filename, mode): + ap = self.rv2a(filename, "r" in mode, "w" in mode) + if "w" in mode and bos.path.exists(ap): + raise FilesystemError("cannot open existing file for writing") + + return open(fsenc(ap), mode) + + def chdir(self, path): + self.cwd = os.path.join(self.cwd, path) + + def mkdir(self, path): + ap = self.rv2a(path, w=True) + bos.mkdir(ap) + + def listdir(self, path): + try: + vpath = os.path.join(self.cwd, path).lstrip("/") + vfs, rem = self.asrv.vfs.get(vpath, self.uname, True, False) + + fsroot, vfs_ls, vfs_virt = vfs.ls( + rem, self.uname, not self.args.no_scandir, [[True], [False, True]] + ) + vfs_ls = [x[0] for x in vfs_ls] + vfs_ls.extend(vfs_virt.keys()) + + if not self.args.ed: + vfs_ls = exclude_dotfiles(vfs_ls) + + vfs_ls.sort() + return vfs_ls + except Exception as ex: + # display write-only folders as empty + return [] + + def rmdir(self, path): + ap = self.rv2a(path, d=True) + bos.rmdir(ap) + + def remove(self, path): + ap = self.rv2a(path, d=True) + bos.unlink(ap) + + def rename(self, src, dst): + raise NotImplementedError() + + def chmod(self, path, mode): + pass + + def stat(self, path): + try: + ap = self.rv2a(path, r=True) + return bos.stat(ap) + except: + ap = self.rv2a(path) + st = bos.stat(ap) + if not stat.S_ISDIR(st.st_mode): + raise + + return st + + def utime(self, path, timeval): + ap = self.rv2a(path, w=True) + return bos.utime(ap, (timeval, timeval)) + + def lstat(self, path): + ap = self.rv2a(path) + return bos.lstat(ap) + + def isfile(self, path): + st = self.stat(path) + return stat.S_ISREG(st.st_mode) + + def islink(self, path): + ap = self.rv2a(path) + return bos.path.islink(ap) + + def isdir(self, path): + st = self.stat(path) + return stat.S_ISDIR(st.st_mode) + + def getsize(self, path): + ap = self.rv2a(path) + return bos.path.getsize(ap) + + def getmtime(self, path): + ap = self.rv2a(path) + return bos.path.getmtime(ap) + + def realpath(self, path): + ap = self.rv2a(path) + return bos.path.isdir(ap) + + def lexists(self, path): + ap = self.rv2a(path) + return bos.path.lexists(ap) + + def get_user_by_uid(self, uid): + return "root" + + def get_group_by_uid(self, gid): + return "root" + + +class Ftpd(object): + def __init__(self, hub): + self.hub = hub + self.args = hub.args + + h = FTPHandler + h.asrv = hub.asrv + h.args = hub.args + h.abstracted_fs = FtpFs + h.authorizer = FtpAuth() + h.authorizer.hub = hub + + if self.args.ftp_r: + p1, p2 = [int(x) for x in self.args.ftp_r.split("-")] + h.passive_ports = list(range(p1, p2 + 1)) + + if self.args.ftp_nat: + h.masquerade_address = self.args.ftp_nat + + if self.args.ftp_debug: + config_logging(level=logging.DEBUG) + + ioloop = IOLoop() + for ip in self.args.i: + FTPServer((ip, int(self.args.ftp)), h, ioloop) + + t = threading.Thread(target=ioloop.loop) + t.daemon = True + t.start() diff --git a/copyparty/svchub.py b/copyparty/svchub.py index 764adeac..cf9956bc 100644 --- a/copyparty/svchub.py +++ b/copyparty/svchub.py @@ -105,6 +105,11 @@ class SvcHub(object): args.th_poke = min(args.th_poke, args.th_maxage, args.ac_maxage) + if args.ftp: + from .ftpd import Ftpd + + self.ftpd = Ftpd(self) + # decide which worker impl to use if self.check_mp_enable(): from .broker_mp import BrokerMp as Broker diff --git a/scripts/make-sfx.sh b/scripts/make-sfx.sh index c6f761f4..26fe6ddb 100755 --- a/scripts/make-sfx.sh +++ b/scripts/make-sfx.sh @@ -107,7 +107,7 @@ tmpdir="$( [ $repack ] && { old="$tmpdir/pe-copyparty" echo "repack of files in $old" - cp -pR "$old/"*{dep-j2,copyparty} . + cp -pR "$old/"*{dep-j2,dep-ftp,copyparty} . } [ $repack ] || { @@ -134,6 +134,19 @@ tmpdir="$( mkdir dep-j2/ mv {markupsafe,jinja2} dep-j2/ + echo collecting pyftpdlib + f="../build/pyftpdlib-1.5.6.tar.gz" + [ -e "$f" ] || + (url=https://github.com/giampaolo/pyftpdlib/archive/refs/tags/release-1.5.6.tar.gz; + wget -O$f "$url" || curl -L "$url" >$f) + + tar -zxf $f + mv pyftpdlib-release-*/pyftpdlib . + rm -rf pyftpdlib-release-* pyftpdlib/test + + mkdir dep-ftp/ + mv pyftpdlib dep-ftp/ + # msys2 tar is bad, make the best of it echo collecting source [ $clean ] && { @@ -331,7 +344,7 @@ nf=$(ls -1 "$zdir"/arc.* | wc -l) echo gen tarlist -for d in copyparty dep-j2; do find $d -type f; done | +for d in copyparty dep-j2 dep-ftp; do find $d -type f; done | sed -r 's/(.*)\.(.*)/\2 \1/' | LC_ALL=C sort | sed -r 's/([^ ]*) (.*)/\2.\1/' | grep -vE '/list1?$' > list1 diff --git a/scripts/sfx.ls b/scripts/sfx.ls index ced73b2a..da5420ad 100644 --- a/scripts/sfx.ls +++ b/scripts/sfx.ls @@ -11,6 +11,7 @@ copyparty/broker_mp.py, copyparty/broker_mpw.py, copyparty/broker_thr.py, copyparty/broker_util.py, +copyparty/ftpd.py, copyparty/httpcli.py, copyparty/httpconn.py, copyparty/httpsrv.py, diff --git a/scripts/sfx.py b/scripts/sfx.py index da509a26..191014ff 100644 --- a/scripts/sfx.py +++ b/scripts/sfx.py @@ -387,7 +387,7 @@ def run(tmp, j2): t.daemon = True t.start() - ld = [tmp, os.path.join(tmp, "dep-j2")] + ld = [os.path.join(tmp, x) for x in ["", "dep-ftp", "dep-j2"]] if j2: del ld[-1] diff --git a/setup.py b/setup.py index bb18ab5c..bbc60abc 100755 --- a/setup.py +++ b/setup.py @@ -112,7 +112,11 @@ args = { "data_files": data_files, "packages": find_packages(), "install_requires": ["jinja2"], - "extras_require": {"thumbnails": ["Pillow"], "audiotags": ["mutagen"]}, + "extras_require": { + "thumbnails": ["Pillow"], + "audiotags": ["mutagen"], + "ftpd": ["pyftpdlib"], + }, "entry_points": {"console_scripts": ["copyparty = copyparty.__main__:main"]}, "scripts": ["bin/copyparty-fuse.py", "bin/up2k.py"], "cmdclass": {"clean2": clean2},