mv/rm (serverside), 100% untested

copyparty/authsrv.py

@@ -10,7 +10,7 @@ import hashlib
 import threading
 
 from .__init__ import WINDOWS
-from .util import IMPLICATIONS, uncyg, undot, Pebkac, fsdec, fsenc, statdir
+from .util import IMPLICATIONS, uncyg, undot, absreal, Pebkac, fsdec, fsenc, statdir
 
 
 class AXS(object):
@@ -185,27 +185,7 @@ class VFS(object):
         if rem:
             rp += "/" + rem
 
-        try:
+        return absreal(rp)
-            return fsdec(os.path.realpath(fsenc(rp)))
-        except:
-            if not WINDOWS:
-                raise
-
-            # cpython bug introduced in 3.8, still exists in 3.9.1;
-            # some win7sp1 and win10:20H2 boxes cannot realpath a
-            # networked drive letter such as b"n:" or b"n:\\"
-            #
-            # requirements to trigger:
-            #  * bytestring (not unicode str)
-            #  * just the drive letter (subfolders are ok)
-            #  * networked drive (regular disks and vmhgfs are ok)
-            #  * on an enterprise network (idk, cannot repro with samba)
-            #
-            # hits the following exceptions in succession:
-            #  * access denied at L601: "path = _getfinalpathname(path)"
-            #  * "cant concat str to bytes" at L621: "return path + tail"
-            #
-            return os.path.realpath(rp)
 
     def ls(self, rem, uname, scandir, permsets, lstat=False):
         # type: (str, str, bool, list[list[bool]], bool) -> tuple[str, str, dict[str, VFS]]
@@ -500,7 +480,7 @@ class AuthSrv(object):
             cased = {}
             for k, v in mount.items():
                 try:
-                    cased[k] = fsdec(os.path.realpath(fsenc(v)))
+                    cased[k] = absreal(v)
                 except:
                     cased[k] = v
 
@@ -597,7 +577,7 @@ class AuthSrv(object):
                     vol.histpath = hpath
                     break
 
-            vol.histpath = os.path.realpath(vol.histpath)
+            vol.histpath = absreal(vol.histpath)
             if vol.dbv:
                 if os.path.exists(os.path.join(vol.histpath, "up2k.db")):
                     promote.append(vol)

copyparty/httpcli.py

@@ -1558,19 +1558,15 @@ class HttpCli(object):
         if self.args.no_mv:
             raise Pebkac(403, "disabled by argv")
 
+        # full path of new loc (incl filename)
         dst = self.uparam.get("to")
         if dst is None:
             raise Pebkac(400, "need dst vpath")
 
-        svn, srem = self.asrv.vfs.get(self.vpath, self.uname, True, False, True)
+        x = self.conn.hsrv.broker.put(
-        dvn, drem = self.asrv.vfs.get(dst, self.uname, False, True)
+            True, "up2k.handle_mv", self.uname, self.vpath, dst
-        src = svn.canonical(srem)
+        )
-        dst = dvn.canonical(drem)
+        self.loud_reply(x.get())
-
-        if not srem:
-            raise Pebkac(400, "cannot move a mountpoint")
-
-        self.loud_reply("mv [{}] to [{}]".format(src, dst))
 
     def tx_browser(self):
         vpath = ""

copyparty/th_srv.py

@@ -10,7 +10,7 @@ import threading
 import subprocess as sp
 
 from .__init__ import PY2, unicode
-from .util import fsenc, runcmd, Queue, Cooldown, BytesIO, min_ex
+from .util import fsenc, vsplit, runcmd, Queue, Cooldown, BytesIO, min_ex
 from .mtag import HAVE_FFMPEG, HAVE_FFPROBE, ffprobe
 
 
@@ -73,12 +73,7 @@ def thumb_path(histpath, rem, mtime, fmt):
     # base16 = 16 = 256
     # b64-lc = 38 = 1444
     # base64 = 64 = 4096
-    try:
+    rd, fn = vsplit(rem)
-        rd, fn = rem.rsplit("/", 1)
-    except:
-        rd = ""
-        fn = rem
-
     if rd:
         h = hashlib.sha512(fsenc(rd)).digest()
         b64 = base64.urlsafe_b64encode(h).decode("ascii")[:24]

copyparty/up2k.py

@@ -23,9 +23,11 @@ from .util import (
     ProgressPrinter,
     fsdec,
     fsenc,
+    absreal,
     sanitize_fn,
     ren_open,
     atomic_move,
+    vsplit,
     s3enc,
     s3dec,
     statdir,
@@ -418,7 +420,7 @@ class Up2k(object):
         if not ANYWIN:
             try:
                 # a bit expensive but worth
-                rcdir = os.path.realpath(cdir)
+                rcdir = absreal(cdir)
             except:
                 pass
 
@@ -1277,6 +1279,7 @@ class Up2k(object):
         dirs = {}
         permsets = [[True, False, False, True]]
         vn, rem = self.asrv.vfs.get(vpath, uname, *permsets[0])
+        ptop = vn.realpath
         atop = vn.canonical(rem)
         adir, fn = os.path.split(atop)
 
@@ -1300,7 +1303,11 @@ class Up2k(object):
                 # dbv, vrem = dbv.get_dbv(vrem)
                 _ = dbv.get(vrem, uname, *permsets[0])
                 with self.mutex:
-                    self._drop_file(dbv.realpath, vpath)
+                    ptop = dbv.realpath
+                    cur, wark = self._find_from_vpath(ptop, vrem)
+                    self._forget_file(ptop, vpath, cur, wark)
+
+                os.unlink(abspath)
 
         n_dirs = 0
         for d in dirs.keys():
@@ -1312,24 +1319,162 @@ class Up2k(object):
 
         return "deleted {} files (and {}/{} folders)".format(n_files, n_dirs, len(dirs))
 
-    def _drop_file(self, ptop, vrem):
+    def _handle_mv(self, uname, svp, dvp):
+        svn, srem = self.asrv.vfs.get(svp, uname, True, False, True)
+        dvn, drem = self.asrv.vfs.get(dvp, uname, False, True)
+        sabs = svn.canonical(srem)
+        dabs = dvn.canonical(drem)
+        drd, dfn = vsplit(drem)
+
+        if not srem:
+            raise Pebkac(400, "mv: cannot move a mountpoint")
+
+        if os.path.exists(dabs):
+            raise Pebkac(400, "mv: target file exists")
+
+        c1, w = self._find_from_vpath(svn.realpath, srem)
+        c2 = self.cur.get(dvn.realpath)
+        if c1 and c2:
+            q = "select rd, fn from up where substr(w,1,16)=? and w=?"
+            hit = c2.execute(q, (w[:16], w)).fetchone()
+            if hit:
+                # found in dest vol, just need a symlink
+                rd, fn = hit
+                if rd.startswith("//") or fn.startswith("//"):
+                    rd, fn = s3dec(rd, fn)
+
+                slabs = "{}/{}".join(rd, fn).strip("/")
+                slabs = absreal(os.path.join(dvn.realpath, slabs))
+                if os.path.exists(fsenc(slabs)):
+                    self.log("mv: quick relink, nice")
+                    self._symlink(fsenc(slabs), fsenc(dabs))
+                    st = os.stat(fsenc(sabs))
+                    self.db_add(c2, w, drd, dfn, st.st_mtime, st.st_size)
+                    os.unlink(fsenc(sabs))
+                else:
+                    self.log("mv: file in db missing? whatever, fixed")
+                    os.rename(fsenc(sabs), fsenc(slabs))
+
+                self._forget_file(svn.realpath, srem, c1, w)
+                return "k"
+
+            # not found in dst vol; copy info
+            self.log("mv: plain move")
+            self._copy_tags(c1, c2, w)
+            self._forget_file(svn.realpath, srem, c1, w)
+            st = os.stat(fsenc(sabs))
+            self.db_add(c2, w, drd, dfn, st.st_mtime, st.st_size)
+            os.rename(fsenc(sabs), fsenc(dabs))
+            return "k"
+
+    def _copy_tags(self, csrc, cdst, wark):
+        """copy all tags for wark from src-db to dst-db"""
+        w = wark[:16]
+
+        if cdst.execute("select * from mt where w=? limit 1", (w,)).fetchone():
+            return  # existing tags in dest db
+
+        for _, k, v in csrc.execute("select * from mt where w=?", (w,)):
+            cdst.execute("insert into mt values(?,?,?)", (w, k, v))
+
+    def _find_from_vpath(self, ptop, vrem):
         cur = self.cur.get(ptop)
-        if cur:
+        if not cur:
-            q = "delete from up where rd=? and fn=?"
+            return None, None
-            rd, fn = os.path.split(vrem)
+
-            self.log("{}, [{}], [{}]".format(q, rd, fn))
+        rd, fn = vsplit(vrem)
-            # self.db_rm(cur, rd, fn)
+        q = "select w from up where rd=? and fn=? limit 1"
+        try:
+            c = cur.execute(q, (rd, fn))
+        except:
+            c = cur.execute(q, s3enc(self.mem_cur, rd, fn))
+
+        wark = c.fetchone()
+        if wark:
+            return cur, wark[0]
+        return cur, None
+
+    def _forget_file(self, ptop, vrem, cur, wark):
+        """forgets file in db, fixes symlinks, does not delete"""
+        fn = vrem.split("/")[-1]
+        wark = None
+        dupes = []
+
+        self.log("forgetting {}".format(vrem))
+        if wark:
+            # found in db; find dupes
+            wark = wark[0]
+            self.log("found {} in db".format(wark))
+
+            q = "select rd, fn from up where substr(w,1,16)=? and w=?"
+            for rd, fn in cur.execute(q, (wark[:16], wark)):
+                if rd.startswith("//") or fn.startswith("//"):
+                    rd, fn = s3dec(rd, fn)
+
+                dvrem = "/".join([rd, fn]).strip("/")
+                if vrem != dvrem:
+                    dupes.append(dvrem)
+                    self.log("found {} dupe: {}".format(q, dvrem))
+
+        if dupes:
+            # fix symlinks
+            self._relink(ptop, dupes, vrem, None)
+        else:
+            # drop tags
+            q = "delete from mt where w=?"
+            cur.execute(q, (wark[:16],))
 
         reg = self.registry.get(ptop)
         if reg:
+            if not wark:
                 wark = [
                     x
                     for x, y in reg.items()
                     if fn in [y["name"], y.get("tnam")] and y["prel"] == vrem
                 ]
-            if wark:
+
-                self.log("forgetting wark {}".format(wark[0]))
+            if wark and wark in reg:
-                del reg[wark[0]]
+                m = "forgetting partial upload {} ({})"
+                p = self._vis_job_progress(wark)
+                self.log(m.format(wark, p))
+                del reg[wark]
+
+    def _relink(self, ptop, dupes, vp1, vp2):
+        """
+        update symlinks from file at vp1 to vp2 (rename),
+        or to first remaining full if no vp2 (delete)
+        """
+
+        if not dupes:
+            return
+
+        def gabs(v):
+            return fsdec(os.path.abspath(fsenc(os.path.join(ptop, v))))
+
+        full = {}
+        links = {}
+        for vp in dupes:
+            ap = gabs(vp)
+            d = links if os.path.islink(ap) else full
+            d[vp] = ap
+
+        if not vp2 and not full:
+            # deleting final remaining full copy; swap it with a symlink
+            dvp = links.keys()[0]
+            dabs = links.pop(dvp)
+            sabs = gabs(vp1)
+            self.log("linkswap [{}] and [{}]".format(sabs, dabs))
+            os.unlink(dabs)
+            os.rename(sabs, dabs)
+            os.link(sabs, dabs)
+            full[vp1] = sabs
+
+        dvp = vp2 if vp2 else full.keys()[0]
+        dabs = gabs(dvp)
+        for alink in links.values():
+            self.log("relinking [{}] to [{}]".format(alink, dabs))
+            os.unlink(alink)
+            os.link(alink, dabs)
 
     def _get_wark(self, cj):
         if len(cj["name"]) > 1024 or len(cj["hash"]) > 512 * 1024:  # 16TiB

copyparty/util.py

@@ -758,6 +758,19 @@ def sanitize_fn(fn, ok, bad):
     return fn.strip()
 
 
+def absreal(fpath):
+    try:
+        return fsdec(os.path.abspath(os.path.realpath(fsenc(fpath))))
+    except:
+        if not WINDOWS:
+            raise
+
+        # cpython bug introduced in 3.8, still exists in 3.9.1,
+        # some win7sp1 and win10:20H2 boxes cannot realpath a
+        # networked drive letter such as b"n:" or b"n:\\"
+        return os.path.abspath(os.path.realpath(fpath))
+
+
 def u8safe(txt):
     try:
         return txt.encode("utf-8", "xmlcharrefreplace").decode("utf-8", "replace")
@@ -815,6 +828,13 @@ def unquotep(txt):
     return w8dec(unq2)
 
 
+def vsplit(vpath):
+    if "/" not in vpath:
+        return "", vpath
+
+    return vpath.rsplit("/", 1)
+
+
 def w8dec(txt):
     """decodes filesystem-bytes to wtf8"""
     if PY2: