bump timeouts for zfs / bursty filesystems

README.md

@@ -810,12 +810,14 @@ see the beautiful mess of a dictionary in [mtag.py](https://github.com/9001/copy
 * avoids pulling any GPL code into copyparty
 * more importantly runs FFprobe on incoming files which is bad if your FFmpeg has a cve
 
+`--mtag-to` sets the tag-scan timeout; very high default (60 sec) to cater for zfs and other randomly-freezing filesystems. Lower values like 10 are usually safe, allowing for faster processing of tricky files
+
 
 ## file parser plugins
 
 provide custom parsers to index additional tags, also see [./bin/mtag/README.md](./bin/mtag/README.md)
 
-copyparty can invoke external programs to collect additional metadata for files using `mtp` (either as argument or volflag), there is a default timeout of 30sec, and only files which contain audio get analyzed by default (see ay/an/ad below)
+copyparty can invoke external programs to collect additional metadata for files using `mtp` (either as argument or volflag), there is a default timeout of 60sec, and only files which contain audio get analyzed by default (see ay/an/ad below)
 
 * `-mtp .bpm=~/bin/audio-bpm.py` will execute `~/bin/audio-bpm.py` with the audio file as argument 1 to provide the `.bpm` tag, if that does not exist in the audio metadata
 * `-mtp key=f,t5,~/bin/audio-key.py` uses `~/bin/audio-key.py` to get the `key` tag, replacing any existing metadata tag (`f,`), aborting if it takes longer than 5sec (`t5,`)

copyparty/__main__.py

@@ -616,7 +616,7 @@ def run_argparse(argv: list[str], formatter: Any, retry: bool) -> argparse.Names
     ap2.add_argument("--hash-mt", metavar="CORES", type=int, default=hcores, help="num cpu cores to use for file hashing; set 0 or 1 for single-core hashing")
     ap2.add_argument("--re-maxage", metavar="SEC", type=int, default=0, help="disk rescan volume interval, 0=off, can be set per-volume with the 'scan' volflag")
     ap2.add_argument("--db-act", metavar="SEC", type=float, default=10, help="defer any scheduled volume reindexing until SEC seconds after last db write (uploads, renames, ...)")
-    ap2.add_argument("--srch-time", metavar="SEC", type=int, default=30, help="search deadline -- terminate searches running for more than SEC seconds")
+    ap2.add_argument("--srch-time", metavar="SEC", type=int, default=45, help="search deadline -- terminate searches running for more than SEC seconds")
     ap2.add_argument("--srch-hits", metavar="N", type=int, default=7999, help="max search results to allow clients to fetch; 125 results will be shown initially")
 
     ap2 = ap.add_argument_group('metadata db options')
@@ -625,6 +625,7 @@ def run_argparse(argv: list[str], formatter: Any, retry: bool) -> argparse.Names
     ap2.add_argument("-e2tsr", action="store_true", help="delete all metadata from DB and do a full rescan; sets -e2ts")
     ap2.add_argument("--no-mutagen", action="store_true", help="use FFprobe for tags instead; will catch more tags")
     ap2.add_argument("--no-mtag-ff", action="store_true", help="never use FFprobe as tag reader; is probably safer")
+    ap2.add_argument("--mtag-to", metavar="SEC", type=int, default=60, help="timeout for ffprobe tag-scan")
     ap2.add_argument("--mtag-mt", metavar="CORES", type=int, default=CORES, help="num cpu cores to use for tag scanning")
     ap2.add_argument("--mtag-v", action="store_true", help="verbose tag scanning; print errors from mtp subprocesses and such")
     ap2.add_argument("--mtag-vv", action="store_true", help="debug mtp settings")

copyparty/mtag.py

@@ -42,7 +42,7 @@ class MParser(object):
         self.tag, args = cmdline.split("=", 1)
         self.tags = self.tag.split(",")
 
-        self.timeout = 30
+        self.timeout = 60
         self.force = False
         self.kill = "t"  # tree; all children recursively
         self.capture = 3  # outputs to consume
@@ -97,7 +97,7 @@ class MParser(object):
 
 
 def ffprobe(
-    abspath: str, timeout: int = 10
+    abspath: str, timeout: int = 60
 ) -> tuple[dict[str, tuple[int, Any]], dict[str, list[Any]]]:
     cmd = [
         b"ffprobe",
@@ -502,7 +502,7 @@ class MTag(object):
         if not bos.path.isfile(abspath):
             return {}
 
-        ret, md = ffprobe(abspath)
+        ret, md = ffprobe(abspath, self.args.mtag_to)
         return self.normalize_tags(ret, md)
 
     def get_bin(

copyparty/th_srv.py

@@ -352,7 +352,7 @@ class ThumbSrv(object):
         img.write_to_file(tpath, Q=40)
 
     def conv_ffmpeg(self, abspath: str, tpath: str) -> None:
-        ret, _ = ffprobe(abspath)
+        ret, _ = ffprobe(abspath, int(self.args.th_convt / 2))
         if not ret:
             return
 
@@ -440,7 +440,7 @@ class ThumbSrv(object):
         raise sp.CalledProcessError(ret, (cmd[0], b"...", cmd[-1]))
 
     def conv_spec(self, abspath: str, tpath: str) -> None:
-        ret, _ = ffprobe(abspath)
+        ret, _ = ffprobe(abspath, int(self.args.th_convt / 2))
         if "ac" not in ret:
             raise Exception("not audio")
 
@@ -485,7 +485,7 @@ class ThumbSrv(object):
         if self.args.no_acode:
             raise Exception("disabled in server config")
 
-        ret, _ = ffprobe(abspath)
+        ret, _ = ffprobe(abspath, int(self.args.th_convt / 2))
         if "ac" not in ret:
             raise Exception("not audio")
 

copyparty/up2k.py

@@ -135,7 +135,7 @@ class Up2k(object):
         self.mem_cur = None
         self.sqlite_ver = None
         self.no_expr_idx = False
-        self.timeout = int(max(self.args.srch_time, 5) * 1.2) + 1
+        self.timeout = int(max(self.args.srch_time, 50) * 1.2) + 1
         self.spools: set[tempfile.SpooledTemporaryFile[bytes]] = set()
         if HAVE_SQLITE3:
             # mojibake detector
@@ -1320,7 +1320,7 @@ class Up2k(object):
             nq -= 1
 
             td = time.time() - last_write
-            if n_buf >= 4096 or td >= max(1, self.timeout - 1):
+            if n_buf >= 4096 or td >= self.timeout / 2:
                 self.log("commit {} new tags".format(n_buf))
                 with self.mutex:
                     cur.connection.commit()

copyparty/util.py

@@ -1405,7 +1405,7 @@ def db_ex_chk(log: "NamedLogger", ex: Exception, db_path: str) -> bool:
 
 def lsof(log: "NamedLogger", abspath: str) -> None:
     try:
-        rc, so, se = runcmd([b"lsof", b"-R", fsenc(abspath)], timeout=5)
+        rc, so, se = runcmd([b"lsof", b"-R", fsenc(abspath)], timeout=45)
         zs = (so.strip() + "\n" + se.strip()).strip()
         log("lsof {} = {}\n{}".format(abspath, rc, zs), 3)
     except: