From 3312c6f5bdfd2b5d343a08cd5e665d66a7e3a584 Mon Sep 17 00:00:00 2001 From: ed Date: Mon, 31 Oct 2022 22:42:47 +0000 Subject: [PATCH] autoclose connection-flooding clients --- README.md | 5 +++- copyparty/__main__.py | 1 + copyparty/httpcli.py | 39 ++++++++++++++++++---------- copyparty/httpconn.py | 11 +++++--- copyparty/httpsrv.py | 59 ++++++++++++++++++++++++++++++++++++++----- copyparty/util.py | 11 +++++++- tests/util.py | 11 +++++--- 7 files changed, 109 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 56b6a726..c1ab7bd3 100644 --- a/README.md +++ b/README.md @@ -724,11 +724,14 @@ connecting from commandline (win7 or later; `wark`=password): * optionally allows/enables login over plaintext http * optionally disables wpad for ~100x performance +better yet, you could skip the windows-builtin webdav support entirely and instead [connect using rclone](./docs/rclone.md) which is 3x faster and way less buggy! + known client bugs: * win7+ doesn't actually send the password to the server when reauthenticating after a reboot unless you first try to login with an incorrect password and then switch to the correct password * or just type your password into the username field instead to get around it entirely * connecting to a folder which allows anonymous read will make writing impossible, as windows has decided it doesn't need to login -* win7+ opens a new tcp connection for every file and sometimes forgets to close them, eventually needing a reboot + * workaround: connect twice; first to a folder which requires auth, then to the folder you actually want, and leave both of those mounted +* win7+ may open a new tcp connection for every file and sometimes forgets to close them, eventually needing a reboot * maybe NIC-related (??), happens with win10-ltsc on e1000e but not virtio * windows cannot access folders which contain filenames with invalid unicode or forbidden characters (`<>:"/\|?*`), or names ending with `.` * winxp cannot show unicode characters outside of *some range* diff --git a/copyparty/__main__.py b/copyparty/__main__.py index 6473e2e6..207ce0c4 100755 --- a/copyparty/__main__.py +++ b/copyparty/__main__.py @@ -673,6 +673,7 @@ def run_argparse( ap2.add_argument("--logout", metavar="H", type=float, default="8086", help="logout clients after H hours of inactivity; [\033[32m0.0028\033[0m]=10sec, [\033[32m0.1\033[0m]=6min, [\033[32m24\033[0m]=day, [\033[32m168\033[0m]=week, [\033[32m720\033[0m]=month, [\033[32m8760\033[0m]=year)") ap2.add_argument("--ban-pw", metavar="N,W,B", type=u, default="9,60,1440", help="more than \033[33mN\033[0m wrong passwords in \033[33mW\033[0m minutes = ban for \033[33mB\033[0m minutes; disable with [\033[32mno\033[0m]") ap2.add_argument("--ban-404", metavar="N,W,B", type=u, default="no", help="hitting more than \033[33mN\033[0m 404's in \033[33mW\033[0m minutes = ban for \033[33mB\033[0m minutes (disabled by default since turbo-up2k counts as 404s)") + ap2.add_argument("--cd-dos", metavar="MIN", type=int, default=10, help="if a client maxes out the server connection limit, downgrade it from connection:keep-alive to connection:close for MIN minutes (and also kill its active connections) -- disable with 0") ap2 = ap.add_argument_group('shutdown options') ap2.add_argument("--ign-ebind", action="store_true", help="continue running even if it's impossible to listen on some of the requested endpoints") diff --git a/copyparty/httpcli.py b/copyparty/httpcli.py index 7f231411..0e07a572 100644 --- a/copyparty/httpcli.py +++ b/copyparty/httpcli.py @@ -118,7 +118,6 @@ class HttpCli(object): self.u2fh = conn.u2fh # mypy404 self.log_func = conn.log_func # mypy404 self.log_src = conn.log_src # mypy404 - self.bans = conn.hsrv.bans self.gen_fk = self._gen_fk if self.args.log_fk else gen_filekey self.tls: bool = hasattr(self.s, "cipher") @@ -213,6 +212,7 @@ class HttpCli(object): self.headers = {} self.hint = "" try: + self.s.settimeout(2) headerlines = read_header(self.sr) if not headerlines: return False @@ -244,18 +244,13 @@ class HttpCli(object): self.loud_reply(unicode(ex), status=ex.code, headers=h, volsan=True) return self.keepalive - if self.args.rsp_slp: - time.sleep(self.args.rsp_slp) - self.ua = self.headers.get("user-agent", "") self.is_rclone = self.ua.startswith("rclone/") self.is_ancient = self.ua.startswith("Mozilla/4.") zs = self.headers.get("connection", "").lower() - self.keepalive = ( - not zs.startswith("close") - and (self.http_ver != "HTTP/1.0" or zs == "keep-alive") - and "Microsoft-WebDAV" not in self.ua + self.keepalive = not zs.startswith("close") and ( + self.http_ver != "HTTP/1.0" or zs == "keep-alive" ) self.is_https = ( self.headers.get("x-forwarded-proto", "").lower() == "https" or self.tls @@ -278,21 +273,31 @@ class HttpCli(object): self.log_src = self.conn.set_rproxy(self.ip) - if self.bans: + if self.conn.bans or self.conn.aclose: ip = self.ip if ":" in ip and not PY2: ip = IPv6Address(ip).exploded[:-20] - if ip in self.bans: - ban = self.bans[ip] - time.time() - if ban < 0: + bans = self.conn.bans + if ip in bans: + rt = bans[ip] - time.time() + if rt < 0: self.log("client unbanned", 3) - del self.bans[ip] + del bans[ip] else: - self.log("banned for {:.0f} sec".format(ban), 6) + self.log("banned for {:.0f} sec".format(rt), 6) self.reply(b"thank you for playing", 403) return False + nka = self.conn.aclose + if ip in nka: + rt = nka[ip] - time.time() + if rt < 0: + self.log("client uncapped", 3) + del nka[ip] + else: + self.keepalive = False + if self.args.ihead: keys = self.args.ihead if "*" in keys: @@ -324,6 +329,9 @@ class HttpCli(object): self.ouparam = {k: zs for k, zs in uparam.items()} + if self.args.rsp_slp: + time.sleep(self.args.rsp_slp) + zso = self.headers.get("cookie") if zso: zsll = [x.split("=", 1) for x in zso.split(";") if "=" in x] @@ -507,6 +515,7 @@ class HttpCli(object): try: # best practice to separate headers and body into different packets + self.s.settimeout(None) self.s.sendall("\r\n".join(response).encode("utf-8") + b"\r\n\r\n") except: raise Pebkac(400, "client d/c while replying headers") @@ -1068,6 +1077,7 @@ class HttpCli(object): if self.headers.get("expect", "").lower() == "100-continue": try: + self.s.settimeout(None) self.s.sendall(b"HTTP/1.1 100 Continue\r\n\r\n") except: raise Pebkac(400, "client d/c before 100 continue") @@ -1079,6 +1089,7 @@ class HttpCli(object): if self.headers.get("expect", "").lower() == "100-continue": try: + self.s.settimeout(None) self.s.sendall(b"HTTP/1.1 100 Continue\r\n\r\n") except: raise Pebkac(400, "client d/c before 100 continue") diff --git a/copyparty/httpconn.py b/copyparty/httpconn.py index b1d37b0f..d7a29cb0 100644 --- a/copyparty/httpconn.py +++ b/copyparty/httpconn.py @@ -46,6 +46,7 @@ class HttpConn(object): ) -> None: self.s = sck self.sr: Optional[Util._Unrecv] = None + self.cli: Optional[HttpCli] = None self.addr = addr self.hsrv = hsrv @@ -56,6 +57,8 @@ class HttpConn(object): self.cert_path = hsrv.cert_path self.u2fh: Util.FHC = hsrv.u2fh # mypy404 self.iphash: HMaccas = hsrv.broker.iphash + self.bans: dict[str, int] = hsrv.bans + self.aclose: dict[str, int] = hsrv.aclose enth = (HAVE_PIL or HAVE_VIPS or HAVE_FFMPEG) and not self.args.no_thumb self.thumbcli: Optional[ThumbCli] = ThumbCli(hsrv) if enth else None # mypy404 @@ -63,7 +66,7 @@ class HttpConn(object): self.t0: float = time.time() # mypy404 self.stopping = False - self.nreq: int = 0 # mypy404 + self.nreq: int = -1 # mypy404 self.nbyte: int = 0 # mypy404 self.u2idx: Optional[U2idx] = None self.log_func: "Util.RootLogger" = hsrv.log # mypy404 @@ -138,6 +141,8 @@ class HttpConn(object): return not method or not bool(PTN_HTTP.match(method)) def run(self) -> None: + self.s.settimeout(10) + self.sr = None if self.args.https_only: is_https = True @@ -206,6 +211,6 @@ class HttpConn(object): while not self.stopping: self.nreq += 1 - cli = HttpCli(self) - if not cli.run(): + self.cli = HttpCli(self) + if not self.cli.run(): return diff --git a/copyparty/httpsrv.py b/copyparty/httpsrv.py index 17647fd4..68248bad 100644 --- a/copyparty/httpsrv.py +++ b/copyparty/httpsrv.py @@ -11,6 +11,11 @@ import time import queue +try: + from ipaddress import IPv6Address +except: + pass + try: import jinja2 except ImportError: @@ -28,7 +33,7 @@ except ImportError: ) sys.exit(1) -from .__init__ import MACOS, TYPE_CHECKING, EnvParams +from .__init__ import MACOS, TYPE_CHECKING, EnvParams, PY2 from .bos import bos from .httpconn import HttpConn from .util import ( @@ -70,9 +75,11 @@ class HttpSrv(object): nsuf = "-n{}-i{:x}".format(nid, os.getpid()) if nid else "" self.magician = Magician() - self.bans: dict[str, int] = {} self.gpwd = Garda(self.args.ban_pw) self.g404 = Garda(self.args.ban_404) + self.gdos = Garda("1,{0},{0}".format(self.args.cd_dos)) + self.bans: dict[str, int] = {} + self.aclose: dict[str, int] = {} self.name = "hsrv" + nsuf self.mutex = threading.Lock() @@ -192,10 +199,49 @@ class HttpSrv(object): if self.args.log_conn: self.log(self.name, "|%sC-ncli" % ("-" * 1,), c="90") - if self.ncli >= self.nclimax: - self.log(self.name, "at connection limit; waiting", 3) - while self.ncli >= self.nclimax: - time.sleep(0.1) + spins = 0 + while self.ncli >= self.nclimax: + if not spins: + self.log(self.name, "at connection limit; waiting", 3) + + spins += 1 + time.sleep(0.1) + if spins != 30 or not self.args.cd_dos: + continue + + ipfreq: dict[str, int] = {} + with self.mutex: + for c in self.clients: + try: + ipfreq[c.ip] += 1 + except: + ipfreq[c.ip] = 1 + + ip, n = sorted(ipfreq.items(), key=lambda x: x[1], reverse=True)[0] + if n < self.nclimax / 2: + continue + + rt, nip = self.gdos.bonk(ip, "") + self.aclose[nip] = rt + nclose = 0 + with self.mutex: + for c in self.clients: + cip = c.ip + if ":" in cip and not PY2: + cip = IPv6Address(cip).exploded[:-20] + + if nip != cip: + continue + + try: + if c.nreq >= 1 or c.cli.keepalive: + Daemon(c.shutdown) + nclose += 1 + except: + pass + + t = "{} downgraded to connection:close for {} min; dropped {} connections" + self.log(self.name, t.format(nip, self.args.cd_dos, nclose), 1) if self.args.log_conn: self.log(self.name, "|%sC-acc1" % ("-" * 2,), c="90") @@ -310,6 +356,7 @@ class HttpSrv(object): with self.mutex: self.clients.add(cli) + print("{}\n".format(len(self.clients)), end="") fno = sck.fileno() try: if self.args.log_conn: diff --git a/copyparty/util.py b/copyparty/util.py index 0b840648..fac9748c 100644 --- a/copyparty/util.py +++ b/copyparty/util.py @@ -401,7 +401,16 @@ class _Unrecv(object): self.buf = self.buf[nbytes:] return ret - ret = self.s.recv(nbytes) + while True: + try: + ret = self.s.recv(nbytes) + break + except socket.timeout: + continue + except: + ret = b"" + break + if not ret: raise UnrecvEOF("client stopped sending data") diff --git a/tests/util.py b/tests/util.py index bc7f0f6d..d270bda1 100644 --- a/tests/util.py +++ b/tests/util.py @@ -98,10 +98,10 @@ class Cfg(Namespace): def __init__(self, a=None, v=None, c=None): ka = {} - ex = "e2d e2ds e2dsa e2t e2ts e2tsr e2v e2vu e2vp dav daw dav_inf dav_mac xdev xvol ed emp force_js ihead magic no_acode no_athumb no_del no_logues no_mv no_readme no_robots no_scandir no_thumb no_vthumb no_zip nid nih nw" + ex = "e2d e2ds e2dsa e2t e2ts e2tsr e2v e2vu e2vp no_dav daw dav_inf dav_mac xdev xvol ed emp force_js ihead magic no_acode no_athumb no_del no_logues no_mv no_readme no_robots no_scandir no_thumb no_vthumb no_zip nid nih nw" ka.update(**{k: False for k in ex.split()}) - ex = "no_rescan no_sendfile no_voldump plain_ip" + ex = "no_rescan no_sendfile no_voldump plain_ip dotpart" ka.update(**{k: True for k in ex.split()}) ex = "css_browser hist js_browser no_hash no_idx no_forget" @@ -155,6 +155,9 @@ class VSock(object): def getsockname(self): return ("a", 1) + def settimeout(self, a): + pass + class VHttpSrv(object): def __init__(self): @@ -181,9 +184,11 @@ class VHttpConn(object): self.log_src = "a" self.lf_url = None self.hsrv = VHttpSrv() + self.bans = {} + self.aclose = {} self.u2fh = FHC() self.mutex = threading.Lock() - self.nreq = 0 + self.nreq = -1 self.nbyte = 0 self.ico = None self.thumbcli = None