From 2ab8924e2d7dcad7ebe4421c053404508ca5b55f Mon Sep 17 00:00:00 2001 From: ed Date: Fri, 22 Nov 2024 22:21:43 +0000 Subject: [PATCH] tests/debug: plug some resource leaks --- copyparty/u2idx.py | 9 +++++++++ copyparty/up2k.py | 5 +++++ tests/test_cp.py | 3 ++- tests/test_dedup.py | 2 ++ tests/test_dots.py | 21 ++++++++++++++++++++- tests/test_hooks.py | 19 ++++++++++++++++--- tests/test_httpcli.py | 2 ++ tests/test_metrics.py | 2 ++ tests/test_mv.py | 2 ++ tests/util.py | 14 +++++++++++++- 10 files changed, 73 insertions(+), 6 deletions(-) diff --git a/copyparty/u2idx.py b/copyparty/u2idx.py index af6d96a3..8201c246 100644 --- a/copyparty/u2idx.py +++ b/copyparty/u2idx.py @@ -70,6 +70,9 @@ class U2idx(object): self.log_func("u2idx", msg, c) def shutdown(self) -> None: + if not HAVE_SQLITE3: + return + for cur in self.cur.values(): db = cur.connection try: @@ -80,6 +83,12 @@ class U2idx(object): cur.close() db.close() + for cur in (self.mem_cur, self.sh_cur): + if cur: + db = cur.connection + cur.close() + db.close() + def fsearch( self, uname: str, vols: list[VFS], body: dict[str, Any] ) -> list[dict[str, Any]]: diff --git a/copyparty/up2k.py b/copyparty/up2k.py index e9056846..d314862f 100644 --- a/copyparty/up2k.py +++ b/copyparty/up2k.py @@ -5297,6 +5297,11 @@ class Up2k(object): cur.close() db.close() + if self.mem_cur: + db = self.mem_cur.connection + self.mem_cur.close() + db.close() + self.registry = {} diff --git a/tests/test_cp.py b/tests/test_cp.py index 7e0f2010..06c65db5 100644 --- a/tests/test_cp.py +++ b/tests/test_cp.py @@ -19,6 +19,8 @@ class TestDedup(unittest.TestCase): self.td = tu.get_ramdisk() def tearDown(self): + if self.conn: + self.conn.shutdown() os.chdir(tempfile.gettempdir()) shutil.rmtree(self.td) @@ -62,7 +64,6 @@ class TestDedup(unittest.TestCase): self.conn = None self.fstab = None - self.ctr = 0 # 2304 for dedup, act_exp in product(tc_dedup, tcs): action, expect = act_exp.split(" ", 1) t = "dedup:%s action:%s" % (dedup, action) diff --git a/tests/test_dedup.py b/tests/test_dedup.py index 11c75c44..f8fb83c2 100644 --- a/tests/test_dedup.py +++ b/tests/test_dedup.py @@ -34,6 +34,8 @@ class TestDedup(unittest.TestCase): ] def tearDown(self): + if self.conn: + self.conn.shutdown() os.chdir(tempfile.gettempdir()) shutil.rmtree(self.td) diff --git a/tests/test_dots.py b/tests/test_dots.py index 9f5944b8..eeef0630 100644 --- a/tests/test_dots.py +++ b/tests/test_dots.py @@ -17,6 +17,11 @@ from copyparty.up2k import Up2k from tests import util as tu from tests.util import Cfg +try: + from typing import Optional +except: + pass + def hdr(query, uname): h = "GET /%s HTTP/1.1\r\nPW: %s\r\nConnection: close\r\n\r\n" @@ -29,12 +34,21 @@ class TestDots(unittest.TestCase): self.is_dut = True def setUp(self): + self.conn: Optional[tu.VHttpConn] = None self.td = tu.get_ramdisk() def tearDown(self): + if self.conn: + self.conn.shutdown() os.chdir(tempfile.gettempdir()) shutil.rmtree(self.td) + def cinit(self): + if self.conn: + self.conn.shutdown() + self.conn = None + self.conn = tu.VHttpConn(self.args, self.asrv, self.log, b"") + def test_dots(self): td = os.path.join(self.td, "vfs") os.mkdir(td) @@ -57,6 +71,7 @@ class TestDots(unittest.TestCase): vcfg = [".::r,u1:r.,u2", "a:a:r,u1:r,u2", ".b:.b:r.,u1:r,u2"] self.args = Cfg(v=vcfg, a=["u1:u1", "u2:u2"], e2dsa=True) self.asrv = AuthSrv(self.args, self.log) + self.cinit() self.assertEqual(self.tardir("", "u1"), "f0 t/f1 a/f3 a/da/f4") self.assertEqual(self.tardir(".t", "u1"), "f2") @@ -88,6 +103,7 @@ class TestDots(unittest.TestCase): self.args = Cfg(v=vcfg, a=["u1:u1", "u2:u2"], dotsrch=False, e2d=True) self.asrv = AuthSrv(self.args, self.log) u2idx = U2idx(self) + self.cinit() x = u2idx.search("u1", self.asrv.vfs.all_vols.values(), "", 999) x = " ".join(sorted([x["rp"] for x in x[0]])) @@ -113,6 +129,8 @@ class TestDots(unittest.TestCase): ] self.args = Cfg(v=vcfg, a=["u1:u1", "u2:u2"]) self.asrv = AuthSrv(self.args, self.log) + self.cinit() + zj = json.loads(self.curl("?ls", "u1")[1]) url = "?k=" + zj["dk"] # should descend into folders, but not other volumes: @@ -148,6 +166,7 @@ class TestDots(unittest.TestCase): self.args = Cfg(v=vcfg, a=["u1:u1", "u2:u2"]) self.asrv = AuthSrv(self.args, self.log) + self.cinit() dk = {} for d in "dk dks dk,fk dks,fk".split(): @@ -353,7 +372,7 @@ class TestDots(unittest.TestCase): def curl(self, url, uname, binary=False, req=b""): req = req or hdr(url, uname) - conn = tu.VHttpConn(self.args, self.asrv, self.log, req) + conn = self.conn.setbuf(req) HttpCli(conn).run() if binary: h, b = conn.s._reply.split(b"\r\n\r\n", 1) diff --git a/tests/test_hooks.py b/tests/test_hooks.py index 93de60c2..232f4dbf 100644 --- a/tests/test_hooks.py +++ b/tests/test_hooks.py @@ -12,6 +12,11 @@ from copyparty.httpcli import HttpCli from tests import util as tu from tests.util import Cfg +try: + from typing import Optional +except: + pass + def hdr(query): h = "GET /{} HTTP/1.1\r\nPW: o\r\nConnection: close\r\n\r\n" @@ -20,6 +25,7 @@ def hdr(query): class TestHooks(unittest.TestCase): def setUp(self): + self.conn: Optional[tu.VHttpConn] = None self.td = tu.get_ramdisk() def tearDown(self): @@ -34,6 +40,12 @@ class TestHooks(unittest.TestCase): os.chdir(td) return td + def cinit(self): + if self.conn: + self.conn.shutdown() + self.conn = None + self.conn = tu.VHttpConn(self.args, self.asrv, self.log, b"") + def test(self): vcfg = ["a/b/c/d:c/d:A", "a:a:r"] @@ -59,6 +71,7 @@ class TestHooks(unittest.TestCase): ka = {hooktype: ["j,c1,h.py"]} self.args = Cfg(v=vcfg, a=["o:o"], e2d=True, **ka) self.asrv = AuthSrv(self.args, self.log) + self.cinit() h, b = upfun(url_up) self.assertIn("201 Created", h) @@ -73,7 +86,7 @@ class TestHooks(unittest.TestCase): buf = "PUT /{0} HTTP/1.1\r\nPW: o\r\nConnection: close\r\nContent-Length: {1}\r\n\r\nok {0}\n" buf = buf.format(url, len(url) + 4).encode("utf-8") print("PUT -->", buf) - conn = tu.VHttpConn(self.args, self.asrv, self.log, buf) + conn = self.conn.setbuf(buf) HttpCli(conn).run() ret = conn.s._reply.decode("utf-8").split("\r\n\r\n", 1) print("PUT <--", ret) @@ -92,14 +105,14 @@ class TestHooks(unittest.TestCase): buf = (bdy % (fn,) + "ok %s/%s\n" % (url, fn) + ftr).encode("utf-8") buf = (hdr % (url, len(buf))).encode("utf-8") + buf print("PoST -->", buf) - conn = tu.VHttpConn(self.args, self.asrv, self.log, buf) + conn = self.conn.setbuf(buf) HttpCli(conn).run() ret = conn.s._reply.decode("utf-8").split("\r\n\r\n", 1) print("POST <--", ret) return ret def curl(self, url, binary=False): - conn = tu.VHttpConn(self.args, self.asrv, self.log, hdr(url)) + conn = self.conn.setbuf(hdr(url)) HttpCli(conn).run() if binary: h, b = conn.s._reply.split(b"\r\n\r\n", 1) diff --git a/tests/test_httpcli.py b/tests/test_httpcli.py index 0df9efa7..3ffeb7f1 100644 --- a/tests/test_httpcli.py +++ b/tests/test_httpcli.py @@ -178,6 +178,8 @@ class TestHttpCli(unittest.TestCase): ap = os.path.join(vn.realpath, rem) os.unlink(ap) + self.conn.shutdown() + def can_rw(self, fp): # lowest non-neutral folder declares permissions expect = fp.split("/")[:-1] diff --git a/tests/test_metrics.py b/tests/test_metrics.py index 16bdae06..3cd9c305 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -27,6 +27,8 @@ class TestMetrics(unittest.TestCase): os.chdir(self.td) def tearDown(self): + if self.conn: + self.conn.shutdown() os.chdir(tempfile.gettempdir()) shutil.rmtree(self.td) diff --git a/tests/test_mv.py b/tests/test_mv.py index 707618f9..3ee6b62a 100644 --- a/tests/test_mv.py +++ b/tests/test_mv.py @@ -25,6 +25,8 @@ class TestDedup(unittest.TestCase): self.td = tu.get_ramdisk() def tearDown(self): + if not PY2 and self.conn: + self.conn.shutdown() os.chdir(tempfile.gettempdir()) shutil.rmtree(self.td) diff --git a/tests/util.py b/tests/util.py index a133ae55..00cb2159 100644 --- a/tests/util.py +++ b/tests/util.py @@ -122,7 +122,7 @@ class Cfg(Namespace): def __init__(self, a=None, v=None, c=None, **ka0): ka = {} - ex = "chpw daw dav_auth dav_inf dav_mac dav_rt e2d e2ds e2dsa e2t e2ts e2tsr e2v e2vu e2vp early_ban ed emp exp force_js getmod grid gsel hardlink ih ihead magic hardlink_only nid nih no_acode no_athumb no_clone no_cp no_dav no_db_ip no_del no_dirsz no_dupe no_lifetime no_logues no_mv no_pipe no_poll no_readme no_robots no_sb_md no_sb_lg no_scandir no_tarcmp no_thumb no_vthumb no_zip nrand nw og og_no_head og_s_title ohead q rand re_dirsz rss smb srch_dbg stats uqe vague_403 vc ver write_uplog xdev xlink xvol zs" + ex = "chpw daw dav_auth dav_inf dav_mac dav_rt e2d e2ds e2dsa e2t e2ts e2tsr e2v e2vu e2vp early_ban ed emp exp force_js getmod grid gsel hardlink ih ihead magic hardlink_only nid nih no_acode no_athumb no_clone no_cp no_dav no_db_ip no_del no_dirsz no_dupe no_lifetime no_logues no_mv no_pipe no_poll no_readme no_robots no_sb_md no_sb_lg no_scandir no_tarcmp no_thumb no_vthumb no_zip nrand nsort nw og og_no_head og_s_title ohead q rand re_dirsz rss smb srch_dbg stats uqe vague_403 vc ver write_uplog xdev xlink xvol zs" ka.update(**{k: False for k in ex.split()}) ex = "dedup dotpart dotsrch hook_v no_dhash no_fastboot no_fpool no_htp no_rescan no_sendfile no_ses no_snap no_up_list no_voldump re_dhash plain_ip" @@ -278,6 +278,10 @@ class VHttpSrv(object): self.u2idx = self.u2idx or U2idx(self) return self.u2idx + def shutdown(self): + if self.u2idx: + self.u2idx.shutdown() + class VHttpSrvUp2k(VHttpSrv): def __init__(self, args, asrv, log): @@ -285,6 +289,11 @@ class VHttpSrvUp2k(VHttpSrv): self.hub = VHub(args, asrv, log) self.broker = VBrokerThr(self.hub) + def shutdown(self): + self.hub.up2k.shutdown() + if self.u2idx: + self.u2idx.shutdown() + class VHttpConn(object): def __init__(self, args, asrv, log, buf, use_up2k=False): @@ -322,6 +331,9 @@ class VHttpConn(object): self.sr = Unrecv(self.s, None) # type: ignore return self + def shutdown(self): + self.hsrv.shutdown() + if WINDOWS: os.system("rem")