autoclose connection-flooding clients

README.md

@@ -724,11 +724,14 @@ connecting from commandline (win7 or later; `wark`=password):
 * optionally allows/enables login over plaintext http
 * optionally disables wpad for ~100x performance
 
+better yet, you could skip the windows-builtin webdav support entirely and instead [connect using rclone](./docs/rclone.md) which is 3x faster and way less buggy!
+
 known client bugs:
 * win7+ doesn't actually send the password to the server when reauthenticating after a reboot unless you first try to login with an incorrect password and then switch to the correct password
   * or just type your password into the username field instead to get around it entirely
 * connecting to a folder which allows anonymous read will make writing impossible, as windows has decided it doesn't need to login
-* win7+ opens a new tcp connection for every file and sometimes forgets to close them, eventually needing a reboot
+  * workaround: connect twice; first to a folder which requires auth, then to the folder you actually want, and leave both of those mounted
+* win7+ may open a new tcp connection for every file and sometimes forgets to close them, eventually needing a reboot
   * maybe NIC-related (??), happens with win10-ltsc on e1000e but not virtio
 * windows cannot access folders which contain filenames with invalid unicode or forbidden characters (`<>:"/\|?*`), or names ending with `.`
 * winxp cannot show unicode characters outside of *some range*

copyparty/__main__.py

@@ -673,6 +673,7 @@ def run_argparse(
     ap2.add_argument("--logout", metavar="H", type=float, default="8086", help="logout clients after H hours of inactivity; [\033[32m0.0028\033[0m]=10sec, [\033[32m0.1\033[0m]=6min, [\033[32m24\033[0m]=day, [\033[32m168\033[0m]=week, [\033[32m720\033[0m]=month, [\033[32m8760\033[0m]=year)")
     ap2.add_argument("--ban-pw", metavar="N,W,B", type=u, default="9,60,1440", help="more than \033[33mN\033[0m wrong passwords in \033[33mW\033[0m minutes = ban for \033[33mB\033[0m minutes; disable with [\033[32mno\033[0m]")
     ap2.add_argument("--ban-404", metavar="N,W,B", type=u, default="no", help="hitting more than \033[33mN\033[0m 404's in \033[33mW\033[0m minutes = ban for \033[33mB\033[0m minutes (disabled by default since turbo-up2k counts as 404s)")
+    ap2.add_argument("--cd-dos", metavar="MIN", type=int, default=10, help="if a client maxes out the server connection limit, downgrade it from connection:keep-alive to connection:close for MIN minutes (and also kill its active connections) -- disable with 0")
 
     ap2 = ap.add_argument_group('shutdown options')
     ap2.add_argument("--ign-ebind", action="store_true", help="continue running even if it's impossible to listen on some of the requested endpoints")

copyparty/httpcli.py

@@ -118,7 +118,6 @@ class HttpCli(object):
         self.u2fh = conn.u2fh  # mypy404
         self.log_func = conn.log_func  # mypy404
         self.log_src = conn.log_src  # mypy404
-        self.bans = conn.hsrv.bans
         self.gen_fk = self._gen_fk if self.args.log_fk else gen_filekey
         self.tls: bool = hasattr(self.s, "cipher")
 
@@ -213,6 +212,7 @@ class HttpCli(object):
         self.headers = {}
         self.hint = ""
         try:
+            self.s.settimeout(2)
             headerlines = read_header(self.sr)
             if not headerlines:
                 return False
@@ -244,18 +244,13 @@ class HttpCli(object):
             self.loud_reply(unicode(ex), status=ex.code, headers=h, volsan=True)
             return self.keepalive
 
-        if self.args.rsp_slp:
-            time.sleep(self.args.rsp_slp)
-
         self.ua = self.headers.get("user-agent", "")
         self.is_rclone = self.ua.startswith("rclone/")
         self.is_ancient = self.ua.startswith("Mozilla/4.")
 
         zs = self.headers.get("connection", "").lower()
-        self.keepalive = (
+        self.keepalive = not zs.startswith("close") and (
-            not zs.startswith("close")
+            self.http_ver != "HTTP/1.0" or zs == "keep-alive"
-            and (self.http_ver != "HTTP/1.0" or zs == "keep-alive")
-            and "Microsoft-WebDAV" not in self.ua
         )
         self.is_https = (
             self.headers.get("x-forwarded-proto", "").lower() == "https" or self.tls
@@ -278,21 +273,31 @@ class HttpCli(object):
 
                 self.log_src = self.conn.set_rproxy(self.ip)
 
-        if self.bans:
+        if self.conn.bans or self.conn.aclose:
             ip = self.ip
             if ":" in ip and not PY2:
                 ip = IPv6Address(ip).exploded[:-20]
 
-            if ip in self.bans:
+            bans = self.conn.bans
-                ban = self.bans[ip] - time.time()
+            if ip in bans:
-                if ban < 0:
+                rt = bans[ip] - time.time()
+                if rt < 0:
                     self.log("client unbanned", 3)
-                    del self.bans[ip]
+                    del bans[ip]
                 else:
-                    self.log("banned for {:.0f} sec".format(ban), 6)
+                    self.log("banned for {:.0f} sec".format(rt), 6)
                     self.reply(b"thank you for playing", 403)
                     return False
 
+            nka = self.conn.aclose
+            if ip in nka:
+                rt = nka[ip] - time.time()
+                if rt < 0:
+                    self.log("client uncapped", 3)
+                    del nka[ip]
+                else:
+                    self.keepalive = False
+
         if self.args.ihead:
             keys = self.args.ihead
             if "*" in keys:
@@ -324,6 +329,9 @@ class HttpCli(object):
 
         self.ouparam = {k: zs for k, zs in uparam.items()}
 
+        if self.args.rsp_slp:
+            time.sleep(self.args.rsp_slp)
+
         zso = self.headers.get("cookie")
         if zso:
             zsll = [x.split("=", 1) for x in zso.split(";") if "=" in x]
@@ -507,6 +515,7 @@ class HttpCli(object):
 
         try:
             # best practice to separate headers and body into different packets
+            self.s.settimeout(None)
             self.s.sendall("\r\n".join(response).encode("utf-8") + b"\r\n\r\n")
         except:
             raise Pebkac(400, "client d/c while replying headers")
@@ -1068,6 +1077,7 @@ class HttpCli(object):
 
         if self.headers.get("expect", "").lower() == "100-continue":
             try:
+                self.s.settimeout(None)
                 self.s.sendall(b"HTTP/1.1 100 Continue\r\n\r\n")
             except:
                 raise Pebkac(400, "client d/c before 100 continue")
@@ -1079,6 +1089,7 @@ class HttpCli(object):
 
         if self.headers.get("expect", "").lower() == "100-continue":
             try:
+                self.s.settimeout(None)
                 self.s.sendall(b"HTTP/1.1 100 Continue\r\n\r\n")
             except:
                 raise Pebkac(400, "client d/c before 100 continue")

copyparty/httpconn.py

@@ -46,6 +46,7 @@ class HttpConn(object):
     ) -> None:
         self.s = sck
         self.sr: Optional[Util._Unrecv] = None
+        self.cli: Optional[HttpCli] = None
         self.addr = addr
         self.hsrv = hsrv
 
@@ -56,6 +57,8 @@ class HttpConn(object):
         self.cert_path = hsrv.cert_path
         self.u2fh: Util.FHC = hsrv.u2fh  # mypy404
         self.iphash: HMaccas = hsrv.broker.iphash
+        self.bans: dict[str, int] = hsrv.bans
+        self.aclose: dict[str, int] = hsrv.aclose
 
         enth = (HAVE_PIL or HAVE_VIPS or HAVE_FFMPEG) and not self.args.no_thumb
         self.thumbcli: Optional[ThumbCli] = ThumbCli(hsrv) if enth else None  # mypy404
@@ -63,7 +66,7 @@ class HttpConn(object):
 
         self.t0: float = time.time()  # mypy404
         self.stopping = False
-        self.nreq: int = 0  # mypy404
+        self.nreq: int = -1  # mypy404
         self.nbyte: int = 0  # mypy404
         self.u2idx: Optional[U2idx] = None
         self.log_func: "Util.RootLogger" = hsrv.log  # mypy404
@@ -138,6 +141,8 @@ class HttpConn(object):
         return not method or not bool(PTN_HTTP.match(method))
 
     def run(self) -> None:
+        self.s.settimeout(10)
+
         self.sr = None
         if self.args.https_only:
             is_https = True
@@ -206,6 +211,6 @@ class HttpConn(object):
 
         while not self.stopping:
             self.nreq += 1
-            cli = HttpCli(self)
+            self.cli = HttpCli(self)
-            if not cli.run():
+            if not self.cli.run():
                 return

copyparty/httpsrv.py

@@ -11,6 +11,11 @@ import time
 
 import queue
 
+try:
+    from ipaddress import IPv6Address
+except:
+    pass
+
 try:
     import jinja2
 except ImportError:
@@ -28,7 +33,7 @@ except ImportError:
     )
     sys.exit(1)
 
-from .__init__ import MACOS, TYPE_CHECKING, EnvParams
+from .__init__ import MACOS, TYPE_CHECKING, EnvParams, PY2
 from .bos import bos
 from .httpconn import HttpConn
 from .util import (
@@ -70,9 +75,11 @@ class HttpSrv(object):
 
         nsuf = "-n{}-i{:x}".format(nid, os.getpid()) if nid else ""
         self.magician = Magician()
-        self.bans: dict[str, int] = {}
         self.gpwd = Garda(self.args.ban_pw)
         self.g404 = Garda(self.args.ban_404)
+        self.gdos = Garda("1,{0},{0}".format(self.args.cd_dos))
+        self.bans: dict[str, int] = {}
+        self.aclose: dict[str, int] = {}
 
         self.name = "hsrv" + nsuf
         self.mutex = threading.Lock()
@@ -192,10 +199,49 @@ class HttpSrv(object):
             if self.args.log_conn:
                 self.log(self.name, "|%sC-ncli" % ("-" * 1,), c="90")
 
-            if self.ncli >= self.nclimax:
+            spins = 0
-                self.log(self.name, "at connection limit; waiting", 3)
+            while self.ncli >= self.nclimax:
-                while self.ncli >= self.nclimax:
+                if not spins:
-                    time.sleep(0.1)
+                    self.log(self.name, "at connection limit; waiting", 3)
+
+                spins += 1
+                time.sleep(0.1)
+                if spins != 30 or not self.args.cd_dos:
+                    continue
+
+                ipfreq: dict[str, int] = {}
+                with self.mutex:
+                    for c in self.clients:
+                        try:
+                            ipfreq[c.ip] += 1
+                        except:
+                            ipfreq[c.ip] = 1
+
+                ip, n = sorted(ipfreq.items(), key=lambda x: x[1], reverse=True)[0]
+                if n < self.nclimax / 2:
+                    continue
+
+                rt, nip = self.gdos.bonk(ip, "")
+                self.aclose[nip] = rt
+                nclose = 0
+                with self.mutex:
+                    for c in self.clients:
+                        cip = c.ip
+                        if ":" in cip and not PY2:
+                            cip = IPv6Address(cip).exploded[:-20]
+
+                        if nip != cip:
+                            continue
+
+                        try:
+                            if c.nreq >= 1 or c.cli.keepalive:
+                                Daemon(c.shutdown)
+                                nclose += 1
+                        except:
+                            pass
+
+                t = "{} downgraded to connection:close for {} min; dropped {} connections"
+                self.log(self.name, t.format(nip, self.args.cd_dos, nclose), 1)
 
             if self.args.log_conn:
                 self.log(self.name, "|%sC-acc1" % ("-" * 2,), c="90")
@@ -310,6 +356,7 @@ class HttpSrv(object):
         with self.mutex:
             self.clients.add(cli)
 
+        print("{}\n".format(len(self.clients)), end="")
         fno = sck.fileno()
         try:
             if self.args.log_conn:

copyparty/util.py

@@ -401,7 +401,16 @@ class _Unrecv(object):
             self.buf = self.buf[nbytes:]
             return ret
 
-        ret = self.s.recv(nbytes)
+        while True:
+            try:
+                ret = self.s.recv(nbytes)
+                break
+            except socket.timeout:
+                continue
+            except:
+                ret = b""
+                break
+
         if not ret:
             raise UnrecvEOF("client stopped sending data")
 

tests/util.py

@@ -98,10 +98,10 @@ class Cfg(Namespace):
     def __init__(self, a=None, v=None, c=None):
         ka = {}
 
-        ex = "e2d e2ds e2dsa e2t e2ts e2tsr e2v e2vu e2vp dav daw dav_inf dav_mac xdev xvol ed emp force_js ihead magic no_acode no_athumb no_del no_logues no_mv no_readme no_robots no_scandir no_thumb no_vthumb no_zip nid nih nw"
+        ex = "e2d e2ds e2dsa e2t e2ts e2tsr e2v e2vu e2vp no_dav daw dav_inf dav_mac xdev xvol ed emp force_js ihead magic no_acode no_athumb no_del no_logues no_mv no_readme no_robots no_scandir no_thumb no_vthumb no_zip nid nih nw"
         ka.update(**{k: False for k in ex.split()})
 
-        ex = "no_rescan no_sendfile no_voldump plain_ip"
+        ex = "no_rescan no_sendfile no_voldump plain_ip dotpart"
         ka.update(**{k: True for k in ex.split()})
 
         ex = "css_browser hist js_browser no_hash no_idx no_forget"
@@ -155,6 +155,9 @@ class VSock(object):
     def getsockname(self):
         return ("a", 1)
 
+    def settimeout(self, a):
+        pass
+
 
 class VHttpSrv(object):
     def __init__(self):
@@ -181,9 +184,11 @@ class VHttpConn(object):
         self.log_src = "a"
         self.lf_url = None
         self.hsrv = VHttpSrv()
+        self.bans = {}
+        self.aclose = {}
         self.u2fh = FHC()
         self.mutex = threading.Lock()
-        self.nreq = 0
+        self.nreq = -1
         self.nbyte = 0
         self.ico = None
         self.thumbcli = None