diff --git a/copyparty/__main__.py b/copyparty/__main__.py index 7606ccc8..6e871d4a 100755 --- a/copyparty/__main__.py +++ b/copyparty/__main__.py @@ -436,7 +436,7 @@ def disable_quickedit() -> None: if PY2: wintypes.LPDWORD = ctypes.POINTER(wintypes.DWORD) - k32.GetStdHandle.errcheck = ecb # type: ignore + k32.GetStdHandle.errcheck = ecb # type: ignore k32.GetConsoleMode.errcheck = ecb # type: ignore k32.SetConsoleMode.errcheck = ecb # type: ignore k32.GetConsoleMode.argtypes = (wintypes.HANDLE, wintypes.LPDWORD) diff --git a/copyparty/httpconn.py b/copyparty/httpconn.py index 10a5ce68..60c8e729 100644 --- a/copyparty/httpconn.py +++ b/copyparty/httpconn.py @@ -112,32 +112,30 @@ class HttpConn(object): return self.u2idx def _detect_https(self) -> bool: - method = None - if True: - try: - method = self.s.recv(4, socket.MSG_PEEK) - except socket.timeout: - return False - except AttributeError: - # jython does not support msg_peek; forget about https - method = self.s.recv(4) - self.sr = Util.Unrecv(self.s, self.log) - self.sr.buf = method + try: + method = self.s.recv(4, socket.MSG_PEEK) + except socket.timeout: + return False + except AttributeError: + # jython does not support msg_peek; forget about https + method = self.s.recv(4) + self.sr = Util.Unrecv(self.s, self.log) + self.sr.buf = method - # jython used to do this, they stopped since it's broken - # but reimplementing sendall is out of scope for now - if not getattr(self.s, "sendall", None): - self.s.sendall = self.s.send # type: ignore + # jython used to do this, they stopped since it's broken + # but reimplementing sendall is out of scope for now + if not getattr(self.s, "sendall", None): + self.s.sendall = self.s.send # type: ignore - if len(method) != 4: - err = "need at least 4 bytes in the first packet; got {}".format( - len(method) - ) - if method: - self.log(err) + if len(method) != 4: + err = "need at least 4 bytes in the first packet; got {}".format( + len(method) + ) + if method: + self.log(err) - self.s.send(b"HTTP/1.1 400 Bad Request\r\n\r\n" + err.encode("utf-8")) - return False + self.s.send(b"HTTP/1.1 400 Bad Request\r\n\r\n" + err.encode("utf-8")) + return False return not method or not bool(PTN_HTTP.match(method)) diff --git a/copyparty/stolen/ifaddr/_shared.py b/copyparty/stolen/ifaddr/_shared.py index c6958284..519c4598 100644 --- a/copyparty/stolen/ifaddr/_shared.py +++ b/copyparty/stolen/ifaddr/_shared.py @@ -61,7 +61,7 @@ class Adapter(object): ) -if True: +if True: # pylint: disable=using-constant-test # Type of an IPv4 address (a string in "xxx.xxx.xxx.xxx" format) _IPv4Address = str diff --git a/copyparty/svchub.py b/copyparty/svchub.py index e45bfd25..a6082ef8 100644 --- a/copyparty/svchub.py +++ b/copyparty/svchub.py @@ -457,7 +457,7 @@ class SvcHub(object): def _ipa2re(self, txt) -> Optional[re.Pattern]: if txt in ("any", "0", ""): return None - + zs = txt.replace(" ", "").replace(".", "\\.").replace(",", "|") return re.compile("^(?:" + zs + ")") diff --git a/copyparty/up2k.py b/copyparty/up2k.py index 565b3d8f..20494b5c 100644 --- a/copyparty/up2k.py +++ b/copyparty/up2k.py @@ -419,50 +419,49 @@ class Up2k(object): def _check_lifetimes(self) -> float: now = time.time() timeout = now + 9001 - if now: # diff-golf - for vp, vol in sorted(self.asrv.vfs.all_vols.items()): - lifetime = vol.flags.get("lifetime") - if not lifetime: - continue + for vp, vol in sorted(self.asrv.vfs.all_vols.items()): + lifetime = vol.flags.get("lifetime") + if not lifetime: + continue - cur = self.cur.get(vol.realpath) - if not cur: - continue + cur = self.cur.get(vol.realpath) + if not cur: + continue - nrm = 0 - deadline = time.time() - lifetime - timeout = min(timeout, now + lifetime) - q = "select rd, fn from up where at > 0 and at < ? limit 100" - while True: - with self.mutex: - hits = cur.execute(q, (deadline,)).fetchall() - - if not hits: - break - - for rd, fn in hits: - if rd.startswith("//") or fn.startswith("//"): - rd, fn = s3dec(rd, fn) - - fvp = ("%s/%s" % (rd, fn)).strip("/") - if vp: - fvp = "%s/%s" % (vp, fvp) - - self._handle_rm(LEELOO_DALLAS, "", fvp, [], True) - nrm += 1 - - if nrm: - self.log("{} files graduated in {}".format(nrm, vp)) - - if timeout < 10: - continue - - q = "select at from up where at > 0 order by at limit 1" + nrm = 0 + deadline = time.time() - lifetime + timeout = min(timeout, now + lifetime) + q = "select rd, fn from up where at > 0 and at < ? limit 100" + while True: with self.mutex: - hits = cur.execute(q).fetchone() + hits = cur.execute(q, (deadline,)).fetchall() - if hits: - timeout = min(timeout, now + lifetime - (now - hits[0])) + if not hits: + break + + for rd, fn in hits: + if rd.startswith("//") or fn.startswith("//"): + rd, fn = s3dec(rd, fn) + + fvp = ("%s/%s" % (rd, fn)).strip("/") + if vp: + fvp = "%s/%s" % (vp, fvp) + + self._handle_rm(LEELOO_DALLAS, "", fvp, [], True) + nrm += 1 + + if nrm: + self.log("{} files graduated in {}".format(nrm, vp)) + + if timeout < 10: + continue + + q = "select at from up where at > 0 order by at limit 1" + with self.mutex: + hits = cur.execute(q).fetchone() + + if hits: + timeout = min(timeout, now + lifetime - (now - hits[0])) return timeout @@ -1217,72 +1216,70 @@ class Up2k(object): abspath = os.path.join(cdir, fn) nohash = reh.search(abspath) if reh else False - if fn: # diff-golf + sql = "select w, mt, sz, at from up where rd = ? and fn = ?" + try: + c = db.c.execute(sql, (rd, fn)) + except: + c = db.c.execute(sql, s3enc(self.mem_cur, rd, fn)) - sql = "select w, mt, sz, at from up where rd = ? and fn = ?" - try: - c = db.c.execute(sql, (rd, fn)) - except: - c = db.c.execute(sql, s3enc(self.mem_cur, rd, fn)) + in_db = list(c.fetchall()) + if in_db: + self.pp.n -= 1 + dw, dts, dsz, at = in_db[0] + if len(in_db) > 1: + t = "WARN: multiple entries: [{}] => [{}] |{}|\n{}" + rep_db = "\n".join([repr(x) for x in in_db]) + self.log(t.format(top, rp, len(in_db), rep_db)) + dts = -1 - in_db = list(c.fetchall()) - if in_db: - self.pp.n -= 1 - dw, dts, dsz, at = in_db[0] - if len(in_db) > 1: - t = "WARN: multiple entries: [{}] => [{}] |{}|\n{}" - rep_db = "\n".join([repr(x) for x in in_db]) - self.log(t.format(top, rp, len(in_db), rep_db)) - dts = -1 + if fat32 and abs(dts - lmod) == 1: + dts = lmod - if fat32 and abs(dts - lmod) == 1: - dts = lmod + if dts == lmod and dsz == sz and (nohash or dw[0] != "#" or not sz): + continue - if dts == lmod and dsz == sz and (nohash or dw[0] != "#" or not sz): - continue - - t = "reindex [{}] => [{}] ({}/{}) ({}/{})".format( - top, rp, dts, lmod, dsz, sz - ) - self.log(t) - self.db_rm(db.c, rd, fn, 0) - ret += 1 - db.n += 1 - in_db = [] - else: - at = 0 - - self.pp.msg = "a{} {}".format(self.pp.n, abspath) - - if nohash or not sz: - wark = up2k_wark_from_metadata(self.salt, sz, lmod, rd, fn) - else: - if sz > 1024 * 1024: - self.log("file: {}".format(abspath)) - - try: - hashes = self._hashlist_from_file( - abspath, "a{}, ".format(self.pp.n) - ) - except Exception as ex: - self.log("hash: {} @ [{}]".format(repr(ex), abspath)) - continue - - if not hashes: - return -1 - - wark = up2k_wark_from_hashlist(self.salt, sz, hashes) - - # skip upload hooks by not providing vflags - self.db_add(db.c, {}, rd, fn, lmod, sz, "", "", wark, "", "", "", at) - db.n += 1 + t = "reindex [{}] => [{}] ({}/{}) ({}/{})".format( + top, rp, dts, lmod, dsz, sz + ) + self.log(t) + self.db_rm(db.c, rd, fn, 0) ret += 1 - td = time.time() - db.t - if db.n >= 4096 or td >= 60: - self.log("commit {} new files".format(db.n)) - db.c.connection.commit() - db.n = 0 - db.t = time.time() + db.n += 1 + in_db = [] + else: + at = 0 + + self.pp.msg = "a{} {}".format(self.pp.n, abspath) + + if nohash or not sz: + wark = up2k_wark_from_metadata(self.salt, sz, lmod, rd, fn) + else: + if sz > 1024 * 1024: + self.log("file: {}".format(abspath)) + + try: + hashes = self._hashlist_from_file( + abspath, "a{}, ".format(self.pp.n) + ) + except Exception as ex: + self.log("hash: {} @ [{}]".format(repr(ex), abspath)) + continue + + if not hashes: + return -1 + + wark = up2k_wark_from_hashlist(self.salt, sz, hashes) + + # skip upload hooks by not providing vflags + self.db_add(db.c, {}, rd, fn, lmod, sz, "", "", wark, "", "", "", at) + db.n += 1 + ret += 1 + td = time.time() - db.t + if db.n >= 4096 or td >= 60: + self.log("commit {} new files".format(db.n)) + db.c.connection.commit() + db.n = 0 + db.t = time.time() if not self.args.no_dhash: db.c.execute("delete from dh where d = ?", (drd,)) # type: ignore diff --git a/copyparty/util.py b/copyparty/util.py index 5e39c31f..97c8820a 100644 --- a/copyparty/util.py +++ b/copyparty/util.py @@ -1823,7 +1823,9 @@ def exclude_dotfiles(filepaths: list[str]) -> list[str]: return [x for x in filepaths if not x.split("/")[-1].startswith(".")] -def odfusion(base: Union[ODict[str, bool], ODict["LiteralString", bool]], oth: str) -> ODict[str, bool]: +def odfusion( + base: Union[ODict[str, bool], ODict["LiteralString", bool]], oth: str +) -> ODict[str, bool]: # merge an "ordered set" (just a dict really) with another list of keys words0 = [x for x in oth.split(",") if x] words1 = [x for x in oth[1:].split(",") if x] diff --git a/scripts/docker/README.md b/scripts/docker/README.md index 9f46e83e..d90cd975 100644 --- a/scripts/docker/README.md +++ b/scripts/docker/README.md @@ -19,7 +19,7 @@ docker run --rm -it -u 1000 -p 3923:3923 -v /mnt/nas:/w -v $PWD/cfgdir:/cfg copy this example is also available as a podman-compatible [docker-compose yaml](https://github.com/9001/copyparty/blob/hovudstraum/docs/examples/docker/basic-docker-compose); example usage: `docker-compose up` (you may need to `systemctl enable --now podman.socket` or similar) -i'm unfamiliar with docker-compose and alternatives so let me know if this section could be better 🙏 +i'm not very familiar with containers, so let me know if this section could be better 🙏 ## configuration