From a86f09fa46e3560d7b37dd0412b45d34c9d749f9 Mon Sep 17 00:00:00 2001 From: ed Date: Sat, 29 May 2021 04:18:57 +0200 Subject: [PATCH] mtp: file extension filtering --- README.md | 2 +- copyparty/mtag.py | 47 ++++++++++++++++++++++++-- copyparty/up2k.py | 84 ++++++++++++++--------------------------------- 3 files changed, 70 insertions(+), 63 deletions(-) diff --git a/README.md b/README.md index 50626721..b170c7da 100644 --- a/README.md +++ b/README.md @@ -327,7 +327,7 @@ copyparty can invoke external programs to collect additional metadata for files *but wait, there's more!* `-mtp` can be used for non-audio files as well using the `a` flag: `ay` only do audio files, `an` only do non-audio files, or `ad` do all files (d as in dontcare) * `-mtp ext=an,~/bin/file-ext.py` runs `~/bin/file-ext.py` to get the `ext` tag only if file is not audio (`an`) -* `-mtp arch,built,ver,orig=an,~/bin/exe.py` runs `~/bin/exe.py` to get properties about windows-binaries only if file is not audio (`an`) +* `-mtp arch,built,ver,orig=an,eexe,edll,~/bin/exe.py` runs `~/bin/exe.py` to get properties about windows-binaries only if file is not audio (`an`) and file extension is exe or dll ## complete examples diff --git a/copyparty/mtag.py b/copyparty/mtag.py index 2de74435..dde4629c 100644 --- a/copyparty/mtag.py +++ b/copyparty/mtag.py @@ -31,6 +31,47 @@ HAVE_FFMPEG = have_ff("ffmpeg") HAVE_FFPROBE = have_ff("ffprobe") +class MParser(object): + def __init__(self, cmdline): + self.tag, args = cmdline.split("=", 1) + self.tags = self.tag.split(",") + + self.timeout = 30 + self.force = False + self.audio = "y" + self.ext = [] + + while True: + try: + bp = os.path.expanduser(args) + if os.path.exists(bp): + self.bin = bp + return + except: + pass + + arg, args = args.split(",", 1) + arg = arg.lower() + + if arg.startswith("a"): + self.audio = arg[1:] # [r]equire [n]ot [d]ontcare + continue + + if arg == "f": + self.force = True + continue + + if arg.startswith("t"): + self.timeout = int(arg[1:]) + continue + + if arg.startswith("e"): + self.ext.append(arg[1:]) + continue + + raise Exception() + + def ffprobe(abspath): cmd = [ b"ffprobe", @@ -383,10 +424,10 @@ class MTag(object): env["PYTHONPATH"] = pypath ret = {} - for tagname, (binpath, timeout) in parsers.items(): + for tagname, mp in parsers.items(): try: - cmd = [sys.executable, binpath, abspath] - args = {"env": env, "timeout": timeout} + cmd = [sys.executable, mp.bin, abspath] + args = {"env": env, "timeout": mp.timeout} if WINDOWS: args["creationflags"] = 0x4000 diff --git a/copyparty/up2k.py b/copyparty/up2k.py index 085f5eae..728071cf 100644 --- a/copyparty/up2k.py +++ b/copyparty/up2k.py @@ -31,7 +31,7 @@ from .util import ( statdir, s2hms, ) -from .mtag import MTag +from .mtag import MTag, MParser try: HAVE_SQLITE3 = True @@ -509,8 +509,6 @@ class Up2k(object): def _run_all_mtp(self): t0 = time.time() - self.mtp_audio = {} - self.mtp_force = {} self.mtp_parsers = {} for ptop, flags in self.flags.items(): if "mtp" in flags: @@ -526,58 +524,18 @@ class Up2k(object): entags = self.entags[ptop] - audio = {} # [r]equire [n]ot [d]ontcare - force = {} # bool - timeout = {} # int parsers = {} for parser in self.flags[ptop]["mtp"]: - orig = parser - tag, parser = parser.split("=", 1) + try: + parser = MParser(parser) + except: + self.log("invalid argument: " + parser, 1) + return - skip = False - for t in tag.split(","): - if t and t not in entags: - skip = True + for tag in entags: + if tag in parser.tags: + parsers[parser.tag] = parser - if skip: - continue - - audio[tag] = "y" - - while True: - try: - bp = os.path.expanduser(parser) - if os.path.exists(bp): - parsers[tag] = [bp, timeout.get(tag, 30)] - break - except: - pass - - try: - arg, parser = parser.split(",", 1) - arg = arg.lower() - - if arg.startswith("a"): - audio[tag] = arg[1:] - continue - - if arg == "f": - force[tag] = True - continue - - if arg.startswith("t"): - timeout[tag] = int(arg[1:]) - continue - - raise Exception() - - except: - self.log("invalid argument: " + orig, 1) - return - - # todo audio/force => parser attributes - self.mtp_audio[ptop] = audio - self.mtp_force[ptop] = force self.mtp_parsers[ptop] = parsers q = "select count(w) from mt where k = 't:mtp'" @@ -610,7 +568,7 @@ class Up2k(object): have = cur.execute(q, (w,)).fetchall() have = [x[0] for x in have] - parsers = self._get_parsers(ptop, have) + parsers = self._get_parsers(ptop, have, abspath) if not parsers: to_delete[w] = True n_left -= 1 @@ -678,29 +636,37 @@ class Up2k(object): wcur.close() cur.close() - def _get_parsers(self, ptop, have): + def _get_parsers(self, ptop, have, abspath): try: all_parsers = self.mtp_parsers[ptop] except: return {} - audio = self.mtp_audio[ptop] - force = self.mtp_force[ptop] entags = self.entags[ptop] parsers = {} for k, v in all_parsers.items(): if "ac" in entags or ".aq" in entags: if "ac" in have or ".aq" in have: # is audio, require non-audio? - if audio[k] == "n": + if v.audio == "n": continue # is not audio, require audio? - elif audio[k] == "y": + elif v.audio == "y": continue + match = False + if v.ext: + for ext in v.ext: + if abspath.lower().endswith("." + ext): + match = True + break + + if not match: + continue + parsers[k] = v - parsers = {k: v for k, v in parsers.items() if k in force or k not in have} + parsers = {k: v for k, v in parsers.items() if v.force or k not in have} return parsers def _start_mpool(self): @@ -1348,7 +1314,7 @@ class Up2k(object): abspath = os.path.join(ptop, rd, fn) tags = self.mtag.get(abspath) ntags1 = len(tags) - parsers = self._get_parsers(ptop, tags) + parsers = self._get_parsers(ptop, tags, abspath) if parsers: tags.update(self.mtag.get_bin(parsers, abspath))