diff --git a/README.md b/README.md index c1ab7bd3..2a276594 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,7 @@ try the **[read-only demo server](https://a.ocv.me/pub/demo/)** 👀 running fro * [qr-code](#qr-code) - print a qr-code [(screenshot)](https://user-images.githubusercontent.com/241032/194728533-6f00849b-c6ac-43c6-9359-83e454d11e00.png) for quick access * [ftp server](#ftp-server) - an FTP server can be started using `--ftp 3921` * [webdav server](#webdav-server) - with read-write support + * [connecting to webdav from windows](#connecting-to-webdav-from-windows) - using the GUI * [smb server](#smb-server) - unsafe, slow, not recommended for wan * [file indexing](#file-indexing) - enables dedup and music search ++ * [exclude-patterns](#exclude-patterns) - to save some time diff --git a/bin/up2k.py b/bin/up2k.py index 94e4c676..7f699ce2 100755 --- a/bin/up2k.py +++ b/bin/up2k.py @@ -69,6 +69,14 @@ VT100 = platform.system() != "Windows" req_ses = requests.Session() +class Daemon(threading.Thread): + def __init__(self, target, name=None, a=None): + # type: (Any, Any, Any) -> None + threading.Thread.__init__(self, target=target, args=a or (), name=name) + self.daemon = True + self.start() + + class File(object): """an up2k upload task; represents a single file""" @@ -543,14 +551,6 @@ def upload(req_ses, file, cid, pw): f.f.close() -class Daemon(threading.Thread): - def __init__(self, target, name=None, a=None): - # type: (Any, Any, Any) -> None - threading.Thread.__init__(self, target=target, args=a or (), name=name) - self.daemon = True - self.start() - - class Ctl(object): """ this will be the coordinator which runs everything in parallel diff --git a/copyparty/__main__.py b/copyparty/__main__.py index 207ce0c4..8c3e80d5 100755 --- a/copyparty/__main__.py +++ b/copyparty/__main__.py @@ -673,7 +673,9 @@ def run_argparse( ap2.add_argument("--logout", metavar="H", type=float, default="8086", help="logout clients after H hours of inactivity; [\033[32m0.0028\033[0m]=10sec, [\033[32m0.1\033[0m]=6min, [\033[32m24\033[0m]=day, [\033[32m168\033[0m]=week, [\033[32m720\033[0m]=month, [\033[32m8760\033[0m]=year)") ap2.add_argument("--ban-pw", metavar="N,W,B", type=u, default="9,60,1440", help="more than \033[33mN\033[0m wrong passwords in \033[33mW\033[0m minutes = ban for \033[33mB\033[0m minutes; disable with [\033[32mno\033[0m]") ap2.add_argument("--ban-404", metavar="N,W,B", type=u, default="no", help="hitting more than \033[33mN\033[0m 404's in \033[33mW\033[0m minutes = ban for \033[33mB\033[0m minutes (disabled by default since turbo-up2k counts as 404s)") - ap2.add_argument("--cd-dos", metavar="MIN", type=int, default=10, help="if a client maxes out the server connection limit, downgrade it from connection:keep-alive to connection:close for MIN minutes (and also kill its active connections) -- disable with 0") + ap2.add_argument("--aclose", metavar="MIN", type=int, default=10, help="if a client maxes out the server connection limit, downgrade it from connection:keep-alive to connection:close for MIN minutes (and also kill its active connections) -- disable with 0") + ap2.add_argument("--loris1", metavar="W,B", type=u, default="60,60", help="if a client takes more than W seconds to finish sending headers, ban it for B minutes; disable with [\033[32mno\033[0m]") + ap2.add_argument("--loris2", metavar="B", type=int, default=60, help="if a client maxes out the server connection limit without sending headers, ban it for B minutes; disable with [\033[32m0\033[0m]") ap2 = ap.add_argument_group('shutdown options') ap2.add_argument("--ign-ebind", action="store_true", help="continue running even if it's impossible to listen on some of the requested endpoints") diff --git a/copyparty/httpcli.py b/copyparty/httpcli.py index 0e07a572..c0c386d4 100644 --- a/copyparty/httpcli.py +++ b/copyparty/httpcli.py @@ -44,6 +44,7 @@ from .util import ( META_NOBOTS, MultipartParser, Pebkac, + Slowloris, UnrecvEOF, alltrace, atomic_move, @@ -61,6 +62,7 @@ from .util import ( html_escape, http_ts, humansize, + ipnorm, min_ex, quotep, read_header, @@ -124,6 +126,7 @@ class HttpCli(object): # placeholders; assigned by run() self.keepalive = False self.is_https = False + self.in_hdr_recv = True self.headers: dict[str, str] = {} self.mode = " " self.req = " " @@ -211,9 +214,14 @@ class HttpCli(object): self.is_https = False self.headers = {} self.hint = "" + + if self.is_banned(): + return False + try: self.s.settimeout(2) - headerlines = read_header(self.sr) + headerlines = read_header(self.sr, self.args.loris1w) + self.in_hdr_recv = False if not headerlines: return False @@ -244,6 +252,13 @@ class HttpCli(object): self.loud_reply(unicode(ex), status=ex.code, headers=h, volsan=True) return self.keepalive + except Slowloris: + ip = ipnorm(self.ip) + self.conn.bans[ip] = int(time.time() + self.args.loris1b * 60) + t = "slowloris (infinite-headers): {} banned for {} min" + self.log(t.format(ip, self.args.loris1b), 1) + return False + self.ua = self.headers.get("user-agent", "") self.is_rclone = self.ua.startswith("rclone/") self.is_ancient = self.ua.startswith("Mozilla/4.") @@ -273,23 +288,12 @@ class HttpCli(object): self.log_src = self.conn.set_rproxy(self.ip) - if self.conn.bans or self.conn.aclose: - ip = self.ip - if ":" in ip and not PY2: - ip = IPv6Address(ip).exploded[:-20] - - bans = self.conn.bans - if ip in bans: - rt = bans[ip] - time.time() - if rt < 0: - self.log("client unbanned", 3) - del bans[ip] - else: - self.log("banned for {:.0f} sec".format(rt), 6) - self.reply(b"thank you for playing", 403) - return False + if self.is_banned(): + return False + if self.conn.aclose: nka = self.conn.aclose + ip = ipnorm(self.ip) if ip in nka: rt = nka[ip] - time.time() if rt < 0: @@ -467,6 +471,26 @@ class HttpCli(object): else: return self.conn.iphash.s(self.ip) + def is_banned(self) -> bool: + if not self.conn.bans: + return False + + bans = self.conn.bans + ip = ipnorm(self.ip) + if ip not in bans: + return False + + rt = bans[ip] - time.time() + if rt < 0: + self.log("client unbanned", 3) + del bans[ip] + return False + + self.log("banned for {:.0f} sec".format(rt), 6) + zb = b"HTTP/1.0 403 Forbidden\r\n\r\nthank you for playing" + self.s.sendall(zb) + return True + def permit_caching(self) -> None: cache = self.uparam.get("cache") if cache is None: diff --git a/copyparty/httpsrv.py b/copyparty/httpsrv.py index 68248bad..14bf26b5 100644 --- a/copyparty/httpsrv.py +++ b/copyparty/httpsrv.py @@ -33,7 +33,7 @@ except ImportError: ) sys.exit(1) -from .__init__ import MACOS, TYPE_CHECKING, EnvParams, PY2 +from .__init__ import MACOS, TYPE_CHECKING, EnvParams from .bos import bos from .httpconn import HttpConn from .util import ( @@ -42,6 +42,7 @@ from .util import ( Daemon, Garda, Magician, + ipnorm, min_ex, shut_socket, spack, @@ -77,7 +78,6 @@ class HttpSrv(object): self.magician = Magician() self.gpwd = Garda(self.args.ban_pw) self.g404 = Garda(self.args.ban_404) - self.gdos = Garda("1,{0},{0}".format(self.args.cd_dos)) self.bans: dict[str, int] = {} self.aclose: dict[str, int] = {} @@ -206,42 +206,56 @@ class HttpSrv(object): spins += 1 time.sleep(0.1) - if spins != 30 or not self.args.cd_dos: + if spins != 50 or not self.args.aclose: continue ipfreq: dict[str, int] = {} with self.mutex: for c in self.clients: + ip = ipnorm(c.ip) try: - ipfreq[c.ip] += 1 + ipfreq[ip] += 1 except: - ipfreq[c.ip] = 1 + ipfreq[ip] = 1 ip, n = sorted(ipfreq.items(), key=lambda x: x[1], reverse=True)[0] if n < self.nclimax / 2: continue - rt, nip = self.gdos.bonk(ip, "") - self.aclose[nip] = rt + self.aclose[ip] = int(time.time() + self.args.aclose * 60) nclose = 0 + nloris = 0 + nconn = 0 with self.mutex: for c in self.clients: - cip = c.ip - if ":" in cip and not PY2: - cip = IPv6Address(cip).exploded[:-20] - - if nip != cip: + cip = ipnorm(c.ip) + if ip != cip: continue + nconn += 1 try: - if c.nreq >= 1 or c.cli.keepalive: + if ( + c.nreq >= 1 + or not c.cli + or c.cli.in_hdr_recv + or c.cli.keepalive + ): Daemon(c.shutdown) nclose += 1 + if c.nreq <= 0 and (not c.cli or c.cli.in_hdr_recv): + nloris += 1 except: pass - t = "{} downgraded to connection:close for {} min; dropped {} connections" - self.log(self.name, t.format(nip, self.args.cd_dos, nclose), 1) + t = "{} downgraded to connection:close for {} min; dropped {}/{} connections" + self.log(self.name, t.format(ip, self.args.aclose, nclose, nconn), 1) + + if nloris < nconn / 2: + continue + + t = "slowloris (idle-conn): {} banned for {} min" + self.log(self.name, t.format(ip, self.args.loris2, nclose), 1) + self.bans[ip] = int(time.time() + self.args.loris2 * 60) if self.args.log_conn: self.log(self.name, "|%sC-acc1" % ("-" * 2,), c="90") @@ -356,7 +370,7 @@ class HttpSrv(object): with self.mutex: self.clients.add(cli) - print("{}\n".format(len(self.clients)), end="") + # print("{}\n".format(len(self.clients)), end="") fno = sck.fileno() try: if self.args.log_conn: diff --git a/copyparty/svchub.py b/copyparty/svchub.py index 44af8e9d..40f54def 100644 --- a/copyparty/svchub.py +++ b/copyparty/svchub.py @@ -145,6 +145,9 @@ class SvcHub(object): self.log("root", "max clients: {}".format(self.args.nc)) + if not self._process_config(): + raise Exception("bad config") + self.tcpsrv = TcpSrv(self) self.up2k = Up2k(self) @@ -250,6 +253,16 @@ class SvcHub(object): Daemon(self.sd_notify, "sd-notify") + def _process_config(self) -> bool: + if self.args.loris1 == "no": + self.args.loris1 = "0,0" + + i1, i2 = self.args.loris1.split(",") + self.args.loris1w = int(i1) + self.args.loris1b = int(i2) + + return True + def _setlimits(self) -> None: try: import resource diff --git a/copyparty/util.py b/copyparty/util.py index fac9748c..f1c85521 100644 --- a/copyparty/util.py +++ b/copyparty/util.py @@ -26,7 +26,7 @@ from datetime import datetime from queue import Queue -from .__init__ import ANYWIN, MACOS, PY2, TYPE_CHECKING, VT100, WINDOWS +from .__init__ import ANYWIN, MACOS, PY2, TYPE_CHECKING, VT100, WINDOWS, unicode from .__version__ import S_BUILD_DT, S_VERSION from .stolen import surrogateescape @@ -1134,7 +1134,7 @@ class MultipartParser(object): rfc1341/rfc1521/rfc2047/rfc2231/rfc2388/rfc6266/the-real-world (only the fallback non-js uploader relies on these filenames) """ - for ln in read_header(self.sr): + for ln in read_header(self.sr, 0): self.log(ln) m = self.re_ctype.match(ln) @@ -1334,9 +1334,13 @@ def get_boundary(headers: dict[str, str]) -> str: return m.group(2) -def read_header(sr: Unrecv) -> list[str]: +def read_header(sr: Unrecv, loris: int) -> list[str]: + t0 = time.time() ret = b"" while True: + if loris and time.time() - t0 > loris: + raise Slowloris() + try: ret += sr.recv(1024) except: @@ -1560,6 +1564,17 @@ def exclude_dotfiles(filepaths: list[str]) -> list[str]: return [x for x in filepaths if not x.split("/")[-1].startswith(".")] +def _ipnorm3(ip: str) -> str: + if ":" in ip: + # assume /64 clients; drop 4 groups + return IPv6Address(ip).exploded[:-20] + + return ip + + +ipnorm = _ipnorm3 if not PY2 else unicode + + def http_ts(ts: int) -> str: file_dt = datetime.utcfromtimestamp(ts) return file_dt.strftime(HTTP_TS_FMT) @@ -2416,3 +2431,7 @@ class Pebkac(Exception): def __repr__(self) -> str: return "Pebkac({}, {})".format(self.code, repr(self.args)) + + +class Slowloris(Exception): + pass diff --git a/tests/util.py b/tests/util.py index d270bda1..4de33b77 100644 --- a/tests/util.py +++ b/tests/util.py @@ -107,7 +107,7 @@ class Cfg(Namespace): ex = "css_browser hist js_browser no_hash no_idx no_forget" ka.update(**{k: None for k in ex.split()}) - ex = "re_maxage rproxy rsp_slp s_wr_slp theme themes turbo df" + ex = "re_maxage rproxy rsp_slp s_wr_slp theme themes turbo df loris1w loris1b loris2" ka.update(**{k: 0 for k in ex.split()}) ex = "doctitle favico html_head mth textfiles log_fk"