From 2bf9055cae3b04baf235fc3f7bbb2d3b3751e4df Mon Sep 17 00:00:00 2001 From: ed Date: Sun, 10 Nov 2024 15:43:19 +0000 Subject: [PATCH] detect free RAM on startup for sane defaults * if free ram on startup is less than 2 GiB, use smaller chunks for parallel file hashing * if --th-max-ram is lower than 0.25 (256 MiB), print a warning that thumbnails will not work * make thumbnail cleaner immediately do a sweep on startup, forgetting any failed conversions so they can be retried in case the memory limit was increased since last run --- copyparty/__main__.py | 6 +++++- copyparty/svchub.py | 9 +++++++++ copyparty/th_srv.py | 9 +++++++-- copyparty/util.py | 24 +++++++++++++++++++++++- 4 files changed, 44 insertions(+), 4 deletions(-) diff --git a/copyparty/__main__.py b/copyparty/__main__.py index f4d95e85..a5917762 100644 --- a/copyparty/__main__.py +++ b/copyparty/__main__.py @@ -50,6 +50,8 @@ from .util import ( PARTFTPY_VER, PY_DESC, PYFTPD_VER, + RAM_AVAIL, + RAM_TOTAL, SQLITE_VER, UNPLICATIONS, Daemon, @@ -1325,6 +1327,8 @@ def add_admin(ap): def add_thumbnail(ap): + th_ram = (RAM_AVAIL or RAM_TOTAL or 9) * 0.6 + th_ram = int(max(min(th_ram, 6), 1) * 10) / 10 ap2 = ap.add_argument_group('thumbnail options') ap2.add_argument("--no-thumb", action="store_true", help="disable all thumbnails (volflag=dthumb)") ap2.add_argument("--no-vthumb", action="store_true", help="disable video thumbnails (volflag=dvthumb)") @@ -1332,7 +1336,7 @@ def add_thumbnail(ap): ap2.add_argument("--th-size", metavar="WxH", default="320x256", help="thumbnail res (volflag=thsize)") ap2.add_argument("--th-mt", metavar="CORES", type=int, default=CORES, help="num cpu cores to use for generating thumbnails") ap2.add_argument("--th-convt", metavar="SEC", type=float, default=60.0, help="conversion timeout in seconds (volflag=convt)") - ap2.add_argument("--th-ram-max", metavar="GB", type=float, default=6.0, help="max memory usage (GiB) permitted by thumbnailer; not very accurate") + ap2.add_argument("--th-ram-max", metavar="GB", type=float, default=th_ram, help="max memory usage (GiB) permitted by thumbnailer; not very accurate") ap2.add_argument("--th-crop", metavar="TXT", type=u, default="y", help="crop thumbnails to 4:3 or keep dynamic height; client can override in UI unless force. [\033[32my\033[0m]=crop, [\033[32mn\033[0m]=nocrop, [\033[32mfy\033[0m]=force-y, [\033[32mfn\033[0m]=force-n (volflag=crop)") ap2.add_argument("--th-x3", metavar="TXT", type=u, default="n", help="show thumbs at 3x resolution; client can override in UI unless force. [\033[32my\033[0m]=yes, [\033[32mn\033[0m]=no, [\033[32mfy\033[0m]=force-yes, [\033[32mfn\033[0m]=force-no (volflag=th3x)") ap2.add_argument("--th-dec", metavar="LIBS", default="vips,pil,ff", help="image decoders, in order of preference") diff --git a/copyparty/svchub.py b/copyparty/svchub.py index 20cb7bb3..b0068386 100644 --- a/copyparty/svchub.py +++ b/copyparty/svchub.py @@ -211,6 +211,15 @@ class SvcHub(object): t = "WARNING: --s-rd-sz (%d) is larger than --iobuf (%d); this may lead to reduced performance" self.log("root", t % (args.s_rd_sz, args.iobuf), 3) + zs = "" + if args.th_ram_max < 0.22: + zs = "generate thumbnails" + elif args.th_ram_max < 1: + zs = "generate audio waveforms or spectrograms" + if zs: + t = "WARNING: --th-ram-max is very small (%.2f GiB); will not be able to %s" + self.log("root", t % (args.th_ram_max, zs), 3) + if args.chpw and args.idp_h_usr: t = "ERROR: user-changeable passwords is incompatible with IdP/identity-providers; you must disable either --chpw or --idp-h-usr" self.log("root", t, 1) diff --git a/copyparty/th_srv.py b/copyparty/th_srv.py index dd45596c..dcfcde17 100644 --- a/copyparty/th_srv.py +++ b/copyparty/th_srv.py @@ -20,7 +20,6 @@ from .util import ( FFMPEG_URL, Cooldown, Daemon, - Pebkac, afsenc, fsenc, min_ex, @@ -164,6 +163,7 @@ class ThumbSrv(object): self.ram: dict[str, float] = {} self.memcond = threading.Condition(self.mutex) self.stopping = False + self.rm_nullthumbs = True # forget failed conversions on startup self.nthr = max(1, self.args.th_mt) self.q: Queue[Optional[tuple[str, str, str, VFS]]] = Queue(self.nthr * 4) @@ -862,7 +862,6 @@ class ThumbSrv(object): def cleaner(self) -> None: interval = self.args.th_clean while True: - time.sleep(interval) ndirs = 0 for vol, histpath in self.asrv.vfs.histtab.items(): if histpath.startswith(vol): @@ -876,6 +875,8 @@ class ThumbSrv(object): self.log("\033[Jcln err in %s: %r" % (histpath, ex), 3) self.log("\033[Jcln ok; rm {} dirs".format(ndirs)) + self.rm_nullthumbs = False + time.sleep(interval) def clean(self, histpath: str) -> int: ret = 0 @@ -939,6 +940,10 @@ class ThumbSrv(object): continue + if self.rm_nullthumbs and not inf.st_size: + bos.unlink(fp) + continue + if b64 == prev_b64: self.log("rm replaced [{}]".format(fp)) bos.unlink(prev_fp) diff --git a/copyparty/util.py b/copyparty/util.py index 260250fc..0d590d07 100644 --- a/copyparty/util.py +++ b/copyparty/util.py @@ -436,6 +436,27 @@ UNHUMANIZE_UNITS = { VF_CAREFUL = {"mv_re_t": 5, "rm_re_t": 5, "mv_re_r": 0.1, "rm_re_r": 0.1} +def read_ram() -> tuple[float, float]: + a = b = 0 + try: + with open("/proc/meminfo", "rb", 0x10000) as f: + zsl = f.read(0x10000).decode("ascii", "replace").split("\n") + + p = re.compile("^MemTotal:.* kB") + zs = next((x for x in zsl if p.match(x))) + a = int((int(zs.split()[1]) / 0x100000) * 100) / 100 + + p = re.compile("^MemAvailable:.* kB") + zs = next((x for x in zsl if p.match(x))) + b = int((int(zs.split()[1]) / 0x100000) * 100) / 100 + except: + pass + return a, b + + +RAM_TOTAL, RAM_AVAIL = read_ram() + + pybin = sys.executable or "" if EXE: pybin = "" @@ -1030,6 +1051,7 @@ class MTHash(object): self.sz = 0 self.csz = 0 self.stop = False + self.readsz = 1024 * 1024 * (2 if (RAM_AVAIL or 2) < 1 else 12) self.omutex = threading.Lock() self.imutex = threading.Lock() self.work_q: Queue[int] = Queue() @@ -1105,7 +1127,7 @@ class MTHash(object): while chunk_rem > 0: with self.imutex: f.seek(ofs) - buf = f.read(min(chunk_rem, 1024 * 1024 * 12)) + buf = f.read(min(chunk_rem, self.readsz)) if not buf: raise Exception("EOF at " + str(ofs))