From 0fcfe79994e520a44865e1ff35c928051bdd4f68 Mon Sep 17 00:00:00 2001 From: ed Date: Sun, 16 May 2021 07:04:18 +0200 Subject: [PATCH] general-purpose file parsing --- README.md | 4 ++++ bin/mtag/file-ext.py | 9 ++++++++ copyparty/httpcli.py | 2 +- copyparty/up2k.py | 53 +++++++++++++++++++++++++++++++------------- 4 files changed, 52 insertions(+), 16 deletions(-) create mode 100644 bin/mtag/file-ext.py diff --git a/README.md b/README.md index 9f99b121..3936b2c4 100644 --- a/README.md +++ b/README.md @@ -299,6 +299,10 @@ copyparty can invoke external programs to collect additional metadata for files * `-mtp key=f,t5,~/bin/audio-key.py` uses `~/bin/audio-key.py` to get the `key` tag, replacing any existing metadata tag (`f,`), aborting if it takes longer than 5sec (`t5,`) * `-v ~/music::r:cmtp=.bpm=~/bin/audio-bpm.py:cmtp=key=f,t5,~/bin/audio-key.py` both as a per-volume config wow this is getting ugly +*but wait, there's more!* `-mtp` can be used for non-audio files as well using the `a` flag: `ay` only do audio files, `an` audio files are skipped, or `ad` always do it (d as in dontcare) + +* `-mtp ext=an,~/bin/file-ext.py` runs `~/bin/file-ext.py` to get the `ext` tag only if file is not audio (`an`) + ## complete examples diff --git a/bin/mtag/file-ext.py b/bin/mtag/file-ext.py new file mode 100644 index 00000000..76bb0597 --- /dev/null +++ b/bin/mtag/file-ext.py @@ -0,0 +1,9 @@ +#!/usr/bin/env python + +import sys + +""" +example that just prints the file extension +""" + +print(sys.argv[1].split(".")[-1]) diff --git a/copyparty/httpcli.py b/copyparty/httpcli.py index 6310141e..1ddaacb5 100644 --- a/copyparty/httpcli.py +++ b/copyparty/httpcli.py @@ -674,7 +674,7 @@ class HttpCli(object): self.log("failed to utime ({}, {})".format(path, times)) spd = self._spd(post_sz) - self.log("{} thank".format(spd)) + self.log("binpost {} thank".format(spd)) self.reply(b"thank") return True diff --git a/copyparty/up2k.py b/copyparty/up2k.py index b49b42d3..9ec26b0c 100644 --- a/copyparty/up2k.py +++ b/copyparty/up2k.py @@ -511,6 +511,7 @@ class Up2k(object): def _run_all_mtp(self): t0 = time.time() + self.mtp_audio = {} self.mtp_force = {} self.mtp_parsers = {} for ptop, flags in self.flags.items(): @@ -527,8 +528,9 @@ class Up2k(object): entags = self.entags[ptop] - force = {} - timeout = {} + audio = {} # [r]equire [n]ot [d]ontcare + force = {} # bool + timeout = {} # int parsers = {} for parser in self.flags[ptop]["mtp"]: orig = parser @@ -536,6 +538,8 @@ class Up2k(object): if tag not in entags: continue + audio[tag] = "y" + while True: try: bp = os.path.expanduser(parser) @@ -549,6 +553,10 @@ class Up2k(object): arg, parser = parser.split(",", 1) arg = arg.lower() + if arg.startswith("a"): + audio[tag] = arg[1:] + continue + if arg == "f": force[tag] = True continue @@ -563,6 +571,8 @@ class Up2k(object): self.log("invalid argument: " + orig, 1) return + # todo audio/force => parser attributes + self.mtp_audio[ptop] = audio self.mtp_force[ptop] = force self.mtp_parsers[ptop] = parsers @@ -596,8 +606,8 @@ class Up2k(object): have = cur.execute(q, (w,)).fetchall() have = [x[0] for x in have] - if ".dur" not in have and ".dur" in entags: - # skip non-audio + parsers = self._get_parsers(ptop, have) + if not parsers: to_delete[w] = True n_left -= 1 continue @@ -605,10 +615,7 @@ class Up2k(object): if w in in_progress: continue - task_parsers = { - k: v for k, v in parsers.items() if k in force or k not in have - } - jobs.append([task_parsers, None, w, abspath]) + jobs.append([parsers, None, w, abspath]) in_progress[w] = True done = self._flush_mpool(wcur) @@ -667,6 +674,26 @@ class Up2k(object): wcur.close() cur.close() + def _get_parsers(self, ptop, have): + audio = self.mtp_audio[ptop] + force = self.mtp_force[ptop] + entags = self.entags[ptop] + parsers = {} + for k, v in self.mtp_parsers[ptop].items(): + if ".dur" in entags: + if ".dur" in have: + # is audio, require non-audio? + if audio[k] == "n": + continue + # is not audio, require audio? + elif audio[k] == "y": + continue + + parsers[k] = v + + parsers = {k: v for k, v in parsers.items() if k in force or k not in have} + return parsers + def _start_mpool(self): # mp.pool.ThreadPool and concurrent.futures.ThreadPoolExecutor # both do crazy runahead so lets reinvent another wheel @@ -1308,13 +1335,9 @@ class Up2k(object): abspath = os.path.join(ptop, rd, fn) tags = self.mtag.get(abspath) ntags1 = len(tags) - if self.mtp_parsers.get(ptop, {}): - parser = { - k: v - for k, v in self.mtp_parsers[ptop].items() - if k in self.mtp_force[ptop] or k not in tags - } - tags.update(self.mtag.get_bin(parser, abspath)) + parsers = self._get_parsers(ptop, tags) + if parsers: + tags.update(self.mtag.get_bin(parsers, abspath)) with self.mutex: cur = self.cur[ptop]