From 995cd10df881ef592ab51ab004fe373ef2cef776 Mon Sep 17 00:00:00 2001 From: ed Date: Sun, 4 Sep 2022 21:21:54 +0200 Subject: [PATCH] bump timeouts for zfs / bursty filesystems --- README.md | 4 +++- copyparty/__main__.py | 3 ++- copyparty/mtag.py | 6 +++--- copyparty/th_srv.py | 6 +++--- copyparty/up2k.py | 4 ++-- copyparty/util.py | 2 +- 6 files changed, 14 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 301957dc..43c0e1a3 100644 --- a/README.md +++ b/README.md @@ -810,12 +810,14 @@ see the beautiful mess of a dictionary in [mtag.py](https://github.com/9001/copy * avoids pulling any GPL code into copyparty * more importantly runs FFprobe on incoming files which is bad if your FFmpeg has a cve +`--mtag-to` sets the tag-scan timeout; very high default (60 sec) to cater for zfs and other randomly-freezing filesystems. Lower values like 10 are usually safe, allowing for faster processing of tricky files + ## file parser plugins provide custom parsers to index additional tags, also see [./bin/mtag/README.md](./bin/mtag/README.md) -copyparty can invoke external programs to collect additional metadata for files using `mtp` (either as argument or volflag), there is a default timeout of 30sec, and only files which contain audio get analyzed by default (see ay/an/ad below) +copyparty can invoke external programs to collect additional metadata for files using `mtp` (either as argument or volflag), there is a default timeout of 60sec, and only files which contain audio get analyzed by default (see ay/an/ad below) * `-mtp .bpm=~/bin/audio-bpm.py` will execute `~/bin/audio-bpm.py` with the audio file as argument 1 to provide the `.bpm` tag, if that does not exist in the audio metadata * `-mtp key=f,t5,~/bin/audio-key.py` uses `~/bin/audio-key.py` to get the `key` tag, replacing any existing metadata tag (`f,`), aborting if it takes longer than 5sec (`t5,`) diff --git a/copyparty/__main__.py b/copyparty/__main__.py index f4f3d3cc..fb4463b6 100644 --- a/copyparty/__main__.py +++ b/copyparty/__main__.py @@ -616,7 +616,7 @@ def run_argparse(argv: list[str], formatter: Any, retry: bool) -> argparse.Names ap2.add_argument("--hash-mt", metavar="CORES", type=int, default=hcores, help="num cpu cores to use for file hashing; set 0 or 1 for single-core hashing") ap2.add_argument("--re-maxage", metavar="SEC", type=int, default=0, help="disk rescan volume interval, 0=off, can be set per-volume with the 'scan' volflag") ap2.add_argument("--db-act", metavar="SEC", type=float, default=10, help="defer any scheduled volume reindexing until SEC seconds after last db write (uploads, renames, ...)") - ap2.add_argument("--srch-time", metavar="SEC", type=int, default=30, help="search deadline -- terminate searches running for more than SEC seconds") + ap2.add_argument("--srch-time", metavar="SEC", type=int, default=45, help="search deadline -- terminate searches running for more than SEC seconds") ap2.add_argument("--srch-hits", metavar="N", type=int, default=7999, help="max search results to allow clients to fetch; 125 results will be shown initially") ap2 = ap.add_argument_group('metadata db options') @@ -625,6 +625,7 @@ def run_argparse(argv: list[str], formatter: Any, retry: bool) -> argparse.Names ap2.add_argument("-e2tsr", action="store_true", help="delete all metadata from DB and do a full rescan; sets -e2ts") ap2.add_argument("--no-mutagen", action="store_true", help="use FFprobe for tags instead; will catch more tags") ap2.add_argument("--no-mtag-ff", action="store_true", help="never use FFprobe as tag reader; is probably safer") + ap2.add_argument("--mtag-to", metavar="SEC", type=int, default=60, help="timeout for ffprobe tag-scan") ap2.add_argument("--mtag-mt", metavar="CORES", type=int, default=CORES, help="num cpu cores to use for tag scanning") ap2.add_argument("--mtag-v", action="store_true", help="verbose tag scanning; print errors from mtp subprocesses and such") ap2.add_argument("--mtag-vv", action="store_true", help="debug mtp settings") diff --git a/copyparty/mtag.py b/copyparty/mtag.py index ac958440..7133ac80 100644 --- a/copyparty/mtag.py +++ b/copyparty/mtag.py @@ -42,7 +42,7 @@ class MParser(object): self.tag, args = cmdline.split("=", 1) self.tags = self.tag.split(",") - self.timeout = 30 + self.timeout = 60 self.force = False self.kill = "t" # tree; all children recursively self.capture = 3 # outputs to consume @@ -97,7 +97,7 @@ class MParser(object): def ffprobe( - abspath: str, timeout: int = 10 + abspath: str, timeout: int = 60 ) -> tuple[dict[str, tuple[int, Any]], dict[str, list[Any]]]: cmd = [ b"ffprobe", @@ -502,7 +502,7 @@ class MTag(object): if not bos.path.isfile(abspath): return {} - ret, md = ffprobe(abspath) + ret, md = ffprobe(abspath, self.args.mtag_to) return self.normalize_tags(ret, md) def get_bin( diff --git a/copyparty/th_srv.py b/copyparty/th_srv.py index 8ea48d05..2585a517 100644 --- a/copyparty/th_srv.py +++ b/copyparty/th_srv.py @@ -352,7 +352,7 @@ class ThumbSrv(object): img.write_to_file(tpath, Q=40) def conv_ffmpeg(self, abspath: str, tpath: str) -> None: - ret, _ = ffprobe(abspath) + ret, _ = ffprobe(abspath, int(self.args.th_convt / 2)) if not ret: return @@ -440,7 +440,7 @@ class ThumbSrv(object): raise sp.CalledProcessError(ret, (cmd[0], b"...", cmd[-1])) def conv_spec(self, abspath: str, tpath: str) -> None: - ret, _ = ffprobe(abspath) + ret, _ = ffprobe(abspath, int(self.args.th_convt / 2)) if "ac" not in ret: raise Exception("not audio") @@ -485,7 +485,7 @@ class ThumbSrv(object): if self.args.no_acode: raise Exception("disabled in server config") - ret, _ = ffprobe(abspath) + ret, _ = ffprobe(abspath, int(self.args.th_convt / 2)) if "ac" not in ret: raise Exception("not audio") diff --git a/copyparty/up2k.py b/copyparty/up2k.py index 7320021f..2527ca47 100644 --- a/copyparty/up2k.py +++ b/copyparty/up2k.py @@ -135,7 +135,7 @@ class Up2k(object): self.mem_cur = None self.sqlite_ver = None self.no_expr_idx = False - self.timeout = int(max(self.args.srch_time, 5) * 1.2) + 1 + self.timeout = int(max(self.args.srch_time, 50) * 1.2) + 1 self.spools: set[tempfile.SpooledTemporaryFile[bytes]] = set() if HAVE_SQLITE3: # mojibake detector @@ -1320,7 +1320,7 @@ class Up2k(object): nq -= 1 td = time.time() - last_write - if n_buf >= 4096 or td >= max(1, self.timeout - 1): + if n_buf >= 4096 or td >= self.timeout / 2: self.log("commit {} new tags".format(n_buf)) with self.mutex: cur.connection.commit() diff --git a/copyparty/util.py b/copyparty/util.py index 660ba03f..cc8f01ee 100644 --- a/copyparty/util.py +++ b/copyparty/util.py @@ -1405,7 +1405,7 @@ def db_ex_chk(log: "NamedLogger", ex: Exception, db_path: str) -> bool: def lsof(log: "NamedLogger", abspath: str) -> None: try: - rc, so, se = runcmd([b"lsof", b"-R", fsenc(abspath)], timeout=5) + rc, so, se = runcmd([b"lsof", b"-R", fsenc(abspath)], timeout=45) zs = (so.strip() + "\n" + se.strip()).strip() log("lsof {} = {}\n{}".format(abspath, rc, zs), 3) except: