diff --git a/copyparty/authsrv.py b/copyparty/authsrv.py index daab6091..063fb68f 100644 --- a/copyparty/authsrv.py +++ b/copyparty/authsrv.py @@ -10,7 +10,7 @@ import hashlib import threading from .__init__ import WINDOWS -from .util import IMPLICATIONS, uncyg, undot, Pebkac, fsdec, fsenc, statdir +from .util import IMPLICATIONS, uncyg, undot, absreal, Pebkac, fsdec, fsenc, statdir class AXS(object): @@ -185,27 +185,7 @@ class VFS(object): if rem: rp += "/" + rem - try: - return fsdec(os.path.realpath(fsenc(rp))) - except: - if not WINDOWS: - raise - - # cpython bug introduced in 3.8, still exists in 3.9.1; - # some win7sp1 and win10:20H2 boxes cannot realpath a - # networked drive letter such as b"n:" or b"n:\\" - # - # requirements to trigger: - # * bytestring (not unicode str) - # * just the drive letter (subfolders are ok) - # * networked drive (regular disks and vmhgfs are ok) - # * on an enterprise network (idk, cannot repro with samba) - # - # hits the following exceptions in succession: - # * access denied at L601: "path = _getfinalpathname(path)" - # * "cant concat str to bytes" at L621: "return path + tail" - # - return os.path.realpath(rp) + return absreal(rp) def ls(self, rem, uname, scandir, permsets, lstat=False): # type: (str, str, bool, list[list[bool]], bool) -> tuple[str, str, dict[str, VFS]] @@ -500,7 +480,7 @@ class AuthSrv(object): cased = {} for k, v in mount.items(): try: - cased[k] = fsdec(os.path.realpath(fsenc(v))) + cased[k] = absreal(v) except: cased[k] = v @@ -597,7 +577,7 @@ class AuthSrv(object): vol.histpath = hpath break - vol.histpath = os.path.realpath(vol.histpath) + vol.histpath = absreal(vol.histpath) if vol.dbv: if os.path.exists(os.path.join(vol.histpath, "up2k.db")): promote.append(vol) diff --git a/copyparty/httpcli.py b/copyparty/httpcli.py index c672ddd4..41c98e8b 100644 --- a/copyparty/httpcli.py +++ b/copyparty/httpcli.py @@ -1558,19 +1558,15 @@ class HttpCli(object): if self.args.no_mv: raise Pebkac(403, "disabled by argv") + # full path of new loc (incl filename) dst = self.uparam.get("to") if dst is None: raise Pebkac(400, "need dst vpath") - svn, srem = self.asrv.vfs.get(self.vpath, self.uname, True, False, True) - dvn, drem = self.asrv.vfs.get(dst, self.uname, False, True) - src = svn.canonical(srem) - dst = dvn.canonical(drem) - - if not srem: - raise Pebkac(400, "cannot move a mountpoint") - - self.loud_reply("mv [{}] to [{}]".format(src, dst)) + x = self.conn.hsrv.broker.put( + True, "up2k.handle_mv", self.uname, self.vpath, dst + ) + self.loud_reply(x.get()) def tx_browser(self): vpath = "" diff --git a/copyparty/th_srv.py b/copyparty/th_srv.py index 11f3b2a1..4593a109 100644 --- a/copyparty/th_srv.py +++ b/copyparty/th_srv.py @@ -10,7 +10,7 @@ import threading import subprocess as sp from .__init__ import PY2, unicode -from .util import fsenc, runcmd, Queue, Cooldown, BytesIO, min_ex +from .util import fsenc, vsplit, runcmd, Queue, Cooldown, BytesIO, min_ex from .mtag import HAVE_FFMPEG, HAVE_FFPROBE, ffprobe @@ -73,12 +73,7 @@ def thumb_path(histpath, rem, mtime, fmt): # base16 = 16 = 256 # b64-lc = 38 = 1444 # base64 = 64 = 4096 - try: - rd, fn = rem.rsplit("/", 1) - except: - rd = "" - fn = rem - + rd, fn = vsplit(rem) if rd: h = hashlib.sha512(fsenc(rd)).digest() b64 = base64.urlsafe_b64encode(h).decode("ascii")[:24] diff --git a/copyparty/up2k.py b/copyparty/up2k.py index b293162c..b4964f63 100644 --- a/copyparty/up2k.py +++ b/copyparty/up2k.py @@ -23,9 +23,11 @@ from .util import ( ProgressPrinter, fsdec, fsenc, + absreal, sanitize_fn, ren_open, atomic_move, + vsplit, s3enc, s3dec, statdir, @@ -418,7 +420,7 @@ class Up2k(object): if not ANYWIN: try: # a bit expensive but worth - rcdir = os.path.realpath(cdir) + rcdir = absreal(cdir) except: pass @@ -1277,6 +1279,7 @@ class Up2k(object): dirs = {} permsets = [[True, False, False, True]] vn, rem = self.asrv.vfs.get(vpath, uname, *permsets[0]) + ptop = vn.realpath atop = vn.canonical(rem) adir, fn = os.path.split(atop) @@ -1300,7 +1303,11 @@ class Up2k(object): # dbv, vrem = dbv.get_dbv(vrem) _ = dbv.get(vrem, uname, *permsets[0]) with self.mutex: - self._drop_file(dbv.realpath, vpath) + ptop = dbv.realpath + cur, wark = self._find_from_vpath(ptop, vrem) + self._forget_file(ptop, vpath, cur, wark) + + os.unlink(abspath) n_dirs = 0 for d in dirs.keys(): @@ -1312,24 +1319,162 @@ class Up2k(object): return "deleted {} files (and {}/{} folders)".format(n_files, n_dirs, len(dirs)) - def _drop_file(self, ptop, vrem): + def _handle_mv(self, uname, svp, dvp): + svn, srem = self.asrv.vfs.get(svp, uname, True, False, True) + dvn, drem = self.asrv.vfs.get(dvp, uname, False, True) + sabs = svn.canonical(srem) + dabs = dvn.canonical(drem) + drd, dfn = vsplit(drem) + + if not srem: + raise Pebkac(400, "mv: cannot move a mountpoint") + + if os.path.exists(dabs): + raise Pebkac(400, "mv: target file exists") + + c1, w = self._find_from_vpath(svn.realpath, srem) + c2 = self.cur.get(dvn.realpath) + if c1 and c2: + q = "select rd, fn from up where substr(w,1,16)=? and w=?" + hit = c2.execute(q, (w[:16], w)).fetchone() + if hit: + # found in dest vol, just need a symlink + rd, fn = hit + if rd.startswith("//") or fn.startswith("//"): + rd, fn = s3dec(rd, fn) + + slabs = "{}/{}".join(rd, fn).strip("/") + slabs = absreal(os.path.join(dvn.realpath, slabs)) + if os.path.exists(fsenc(slabs)): + self.log("mv: quick relink, nice") + self._symlink(fsenc(slabs), fsenc(dabs)) + st = os.stat(fsenc(sabs)) + self.db_add(c2, w, drd, dfn, st.st_mtime, st.st_size) + os.unlink(fsenc(sabs)) + else: + self.log("mv: file in db missing? whatever, fixed") + os.rename(fsenc(sabs), fsenc(slabs)) + + self._forget_file(svn.realpath, srem, c1, w) + return "k" + + # not found in dst vol; copy info + self.log("mv: plain move") + self._copy_tags(c1, c2, w) + self._forget_file(svn.realpath, srem, c1, w) + st = os.stat(fsenc(sabs)) + self.db_add(c2, w, drd, dfn, st.st_mtime, st.st_size) + os.rename(fsenc(sabs), fsenc(dabs)) + return "k" + + def _copy_tags(self, csrc, cdst, wark): + """copy all tags for wark from src-db to dst-db""" + w = wark[:16] + + if cdst.execute("select * from mt where w=? limit 1", (w,)).fetchone(): + return # existing tags in dest db + + for _, k, v in csrc.execute("select * from mt where w=?", (w,)): + cdst.execute("insert into mt values(?,?,?)", (w, k, v)) + + def _find_from_vpath(self, ptop, vrem): cur = self.cur.get(ptop) - if cur: - q = "delete from up where rd=? and fn=?" - rd, fn = os.path.split(vrem) - self.log("{}, [{}], [{}]".format(q, rd, fn)) - # self.db_rm(cur, rd, fn) + if not cur: + return None, None + + rd, fn = vsplit(vrem) + q = "select w from up where rd=? and fn=? limit 1" + try: + c = cur.execute(q, (rd, fn)) + except: + c = cur.execute(q, s3enc(self.mem_cur, rd, fn)) + + wark = c.fetchone() + if wark: + return cur, wark[0] + return cur, None + + def _forget_file(self, ptop, vrem, cur, wark): + """forgets file in db, fixes symlinks, does not delete""" + fn = vrem.split("/")[-1] + wark = None + dupes = [] + + self.log("forgetting {}".format(vrem)) + if wark: + # found in db; find dupes + wark = wark[0] + self.log("found {} in db".format(wark)) + + q = "select rd, fn from up where substr(w,1,16)=? and w=?" + for rd, fn in cur.execute(q, (wark[:16], wark)): + if rd.startswith("//") or fn.startswith("//"): + rd, fn = s3dec(rd, fn) + + dvrem = "/".join([rd, fn]).strip("/") + if vrem != dvrem: + dupes.append(dvrem) + self.log("found {} dupe: {}".format(q, dvrem)) + + if dupes: + # fix symlinks + self._relink(ptop, dupes, vrem, None) + else: + # drop tags + q = "delete from mt where w=?" + cur.execute(q, (wark[:16],)) reg = self.registry.get(ptop) if reg: - wark = [ - x - for x, y in reg.items() - if fn in [y["name"], y.get("tnam")] and y["prel"] == vrem - ] - if wark: - self.log("forgetting wark {}".format(wark[0])) - del reg[wark[0]] + if not wark: + wark = [ + x + for x, y in reg.items() + if fn in [y["name"], y.get("tnam")] and y["prel"] == vrem + ] + + if wark and wark in reg: + m = "forgetting partial upload {} ({})" + p = self._vis_job_progress(wark) + self.log(m.format(wark, p)) + del reg[wark] + + def _relink(self, ptop, dupes, vp1, vp2): + """ + update symlinks from file at vp1 to vp2 (rename), + or to first remaining full if no vp2 (delete) + """ + + if not dupes: + return + + def gabs(v): + return fsdec(os.path.abspath(fsenc(os.path.join(ptop, v)))) + + full = {} + links = {} + for vp in dupes: + ap = gabs(vp) + d = links if os.path.islink(ap) else full + d[vp] = ap + + if not vp2 and not full: + # deleting final remaining full copy; swap it with a symlink + dvp = links.keys()[0] + dabs = links.pop(dvp) + sabs = gabs(vp1) + self.log("linkswap [{}] and [{}]".format(sabs, dabs)) + os.unlink(dabs) + os.rename(sabs, dabs) + os.link(sabs, dabs) + full[vp1] = sabs + + dvp = vp2 if vp2 else full.keys()[0] + dabs = gabs(dvp) + for alink in links.values(): + self.log("relinking [{}] to [{}]".format(alink, dabs)) + os.unlink(alink) + os.link(alink, dabs) def _get_wark(self, cj): if len(cj["name"]) > 1024 or len(cj["hash"]) > 512 * 1024: # 16TiB diff --git a/copyparty/util.py b/copyparty/util.py index f9883afa..0969a77e 100644 --- a/copyparty/util.py +++ b/copyparty/util.py @@ -758,6 +758,19 @@ def sanitize_fn(fn, ok, bad): return fn.strip() +def absreal(fpath): + try: + return fsdec(os.path.abspath(os.path.realpath(fsenc(fpath)))) + except: + if not WINDOWS: + raise + + # cpython bug introduced in 3.8, still exists in 3.9.1, + # some win7sp1 and win10:20H2 boxes cannot realpath a + # networked drive letter such as b"n:" or b"n:\\" + return os.path.abspath(os.path.realpath(fpath)) + + def u8safe(txt): try: return txt.encode("utf-8", "xmlcharrefreplace").decode("utf-8", "replace") @@ -815,6 +828,13 @@ def unquotep(txt): return w8dec(unq2) +def vsplit(vpath): + if "/" not in vpath: + return "", vpath + + return vpath.rsplit("/", 1) + + def w8dec(txt): """decodes filesystem-bytes to wtf8""" if PY2: