From 5d19f23372cbbf5a6973c252c228bc8ba5bc330d Mon Sep 17 00:00:00 2001 From: ed Date: Mon, 29 Aug 2022 19:24:48 +0200 Subject: [PATCH] accurate num.cores detection --- bin/up2k.py | 2 +- copyparty/__init__.py | 5 +++++ copyparty/__main__.py | 9 ++++----- copyparty/broker_mp.py | 4 ++-- copyparty/svchub.py | 5 ++++- 5 files changed, 16 insertions(+), 9 deletions(-) diff --git a/bin/up2k.py b/bin/up2k.py index d5142ae3..0d16f966 100755 --- a/bin/up2k.py +++ b/bin/up2k.py @@ -929,7 +929,7 @@ def main(): if not VT100: os.system("rem") # enables colors - cores = os.cpu_count() if hasattr(os, "cpu_count") else 4 + cores = (os.cpu_count() if hasattr(os, "cpu_count") else 0) or 2 hcores = min(cores, 3) # 4% faster than 4+ on py3.9 @ r5-4500U # fmt: off diff --git a/copyparty/__init__.py b/copyparty/__init__.py index d2fdbdd9..a882a305 100644 --- a/copyparty/__init__.py +++ b/copyparty/__init__.py @@ -33,6 +33,11 @@ ANYWIN = WINDOWS or sys.platform in ["msys", "cygwin"] MACOS = platform.system() == "Darwin" +try: + CORES = len(os.sched_getaffinity(0)) +except: + CORES = (os.cpu_count() if hasattr(os, "cpu_count") else 0) or 2 + def get_unixdir() -> str: paths: list[tuple[Callable[..., str], str]] = [ diff --git a/copyparty/__main__.py b/copyparty/__main__.py index 30662e1a..111a6c32 100644 --- a/copyparty/__main__.py +++ b/copyparty/__main__.py @@ -20,7 +20,7 @@ import time import traceback from textwrap import dedent -from .__init__ import ANYWIN, PY2, VT100, WINDOWS, E, unicode +from .__init__ import ANYWIN, PY2, VT100, WINDOWS, E, unicode, CORES from .__version__ import CODENAME, S_BUILD_DT, S_VERSION from .authsrv import re_vol from .svchub import SvcHub @@ -335,8 +335,7 @@ def run_argparse(argv: list[str], formatter: Any, retry: bool) -> argparse.Names except: fk_salt = "hunter2" - cores = (os.cpu_count() if hasattr(os, "cpu_count") else 0) or 4 - hcores = min(cores, 3) # 4% faster than 4+ on py3.9 @ r5-4500U + hcores = min(CORES, 3) # 4% faster than 4+ on py3.9 @ r5-4500U sects = [ [ @@ -576,7 +575,7 @@ def run_argparse(argv: list[str], formatter: Any, retry: bool) -> argparse.Names ap2.add_argument("--no-athumb", action="store_true", help="disable audio thumbnails (spectrograms)") ap2.add_argument("--no-vthumb", action="store_true", help="disable video thumbnails") ap2.add_argument("--th-size", metavar="WxH", default="320x256", help="thumbnail res") - ap2.add_argument("--th-mt", metavar="CORES", type=int, default=cores, help="num cpu cores to use for generating thumbnails") + ap2.add_argument("--th-mt", metavar="CORES", type=int, default=CORES, help="num cpu cores to use for generating thumbnails") ap2.add_argument("--th-convt", metavar="SEC", type=int, default=60, help="conversion timeout in seconds") ap2.add_argument("--th-no-crop", action="store_true", help="dynamic height; show full image") ap2.add_argument("--th-dec", metavar="LIBS", default="vips,pil,ff", help="image decoders, in order of preference") @@ -626,7 +625,7 @@ def run_argparse(argv: list[str], formatter: Any, retry: bool) -> argparse.Names ap2.add_argument("-e2tsr", action="store_true", help="delete all metadata from DB and do a full rescan; sets -e2ts") ap2.add_argument("--no-mutagen", action="store_true", help="use FFprobe for tags instead; will catch more tags") ap2.add_argument("--no-mtag-ff", action="store_true", help="never use FFprobe as tag reader; is probably safer") - ap2.add_argument("--mtag-mt", metavar="CORES", type=int, default=cores, help="num cpu cores to use for tag scanning") + ap2.add_argument("--mtag-mt", metavar="CORES", type=int, default=CORES, help="num cpu cores to use for tag scanning") ap2.add_argument("--mtag-v", action="store_true", help="verbose tag scanning; print errors from mtp subprocesses and such") ap2.add_argument("-mtm", metavar="M=t,t,t", type=u, action="append", help="add/replace metadata mapping") ap2.add_argument("-mte", metavar="M,M,M", type=u, help="tags to index/display (comma-sep.)", diff --git a/copyparty/broker_mp.py b/copyparty/broker_mp.py index c7bfed70..df4f8c26 100644 --- a/copyparty/broker_mp.py +++ b/copyparty/broker_mp.py @@ -6,7 +6,7 @@ import time import queue -from .__init__ import TYPE_CHECKING +from .__init__ import TYPE_CHECKING, CORES from .broker_mpw import MpWorker from .broker_util import try_exec from .util import mp @@ -44,7 +44,7 @@ class BrokerMp(object): self.procs = [] self.mutex = threading.Lock() - self.num_workers = self.args.j or mp.cpu_count() + self.num_workers = self.args.j or CORES self.log("broker", "booting {} subprocesses".format(self.num_workers)) for n in range(1, self.num_workers + 1): q_pend: queue.Queue[tuple[int, str, list[Any]]] = mp.Queue(1) diff --git a/copyparty/svchub.py b/copyparty/svchub.py index 1e1f87e2..786a43dd 100644 --- a/copyparty/svchub.py +++ b/copyparty/svchub.py @@ -501,7 +501,10 @@ class SvcHub(object): if self.args.j == 1: return False - if mp.cpu_count() <= 1: + try: + if mp.cpu_count() <= 1: + raise Exception() + except: self.log("svchub", "only one CPU detected; multiprocessing disabled") return False