cross-filesystem-move fixes

* nonlocal markdown backups
* relocation-hooks

tested on macos, to be verified on Linux/windows

copyparty/cert.py

@@ -1,13 +1,11 @@
 import calendar
 import errno
-import filecmp
 import json
 import os
-import shutil
 import time
 
 from .__init__ import ANYWIN
-from .util import Netdev, load_resource, runcmd, wrename, wunlink
+from .util import Netdev, atomic_move, load_resource, runcmd, wunlink
 
 HAVE_CFSSL = not os.environ.get("PRTY_NO_CFSSL")
 
@@ -122,7 +120,7 @@ def _gen_ca(log: "RootLogger", args):
         wunlink(nlog, bname + ".key", VF)
     except:
         pass
-    wrename(nlog, bname + "-key.pem", bname + ".key", VF)
+    atomic_move(nlog, bname + "-key.pem", bname + ".key", VF)
     wunlink(nlog, bname + ".csr", VF)
 
     log("cert", "new ca OK", 2)
@@ -215,7 +213,7 @@ def _gen_srv(log: "RootLogger", args, netdevs: dict[str, Netdev]):
         wunlink(nlog, bname + ".key", VF)
     except:
         pass
-    wrename(nlog, bname + "-key.pem", bname + ".key", VF)
+    atomic_move(nlog, bname + "-key.pem", bname + ".key", VF)
     wunlink(nlog, bname + ".csr", VF)
 
     with open(os.path.join(args.crt_dir, "ca.pem"), "rb") as f:

copyparty/httpcli.py

@@ -113,7 +113,6 @@ from .util import (
     vol_san,
     vroots,
     vsplit,
-    wrename,
     wunlink,
     yieldfile,
 )
@@ -3569,7 +3568,7 @@ class HttpCli(object):
                 except:
                     pass
             if dp:
-                wrename(self.log, fp, os.path.join(dp, mfile2), vfs.flags)
+                atomic_move(self.log, fp, os.path.join(dp, mfile2), vfs.flags)
 
         assert self.parser.gen  # !rm
         p_field, _, p_data = next(self.parser.gen)

copyparty/th_srv.py

@@ -23,6 +23,7 @@ from .util import (
     VF_CAREFUL,
     Cooldown,
     Daemon,
+    atomic_move,
     afsenc,
     fsenc,
     min_ex,
@@ -30,7 +31,6 @@ from .util import (
     statdir,
     ub64enc,
     vsplit,
-    wrename,
     wunlink,
 )
 
@@ -412,7 +412,7 @@ class ThumbSrv(object):
                 wunlink(self.log, ap_unpk, vn.flags)
 
             try:
-                wrename(self.log, ttpath, tpath, vn.flags)
+                atomic_move(self.log, ttpath, tpath, vn.flags)
             except Exception as ex:
                 if not os.path.exists(tpath):
                     t = "failed to move  [%s]  to  [%s]:  %r"
@@ -677,7 +677,7 @@ class ThumbSrv(object):
                 except:
                     pass
             else:
-                wrename(self.log, wtpath, tpath, vn.flags)
+                atomic_move(self.log, wtpath, tpath, vn.flags)
 
     def conv_spec(self, abspath: str, tpath: str, fmt: str, vn: VFS) -> None:
         ret, _ = ffprobe(abspath, int(vn.flags["convt"] / 2))

copyparty/util.py

@@ -2583,6 +2583,11 @@ def _fs_mvrm(
             now = time.time()
             if ex.errno == errno.ENOENT:
                 return False
+            if not attempt and ex.errno == errno.EXDEV:
+                t = "using copy+delete (%s)\n  %s\n  %s"
+                log(t % (ex.strerror, src, dst))
+                osfun = shutil.move
+                continue
             if now - t0 > maxtime or attempt == 90209:
                 raise
             if not attempt:
@@ -2607,15 +2612,18 @@ def atomic_move(log: "NamedLogger", src: str, dst: str, flags: dict[str, Any]) -
     elif flags.get("mv_re_t"):
         _fs_mvrm(log, src, dst, True, flags)
     else:
-        os.replace(bsrc, bdst)
-
-
-def wrename(log: "NamedLogger", src: str, dst: str, flags: dict[str, Any]) -> bool:
-    if not flags.get("mv_re_t"):
-        os.rename(fsenc(src), fsenc(dst))
-        return True
-
-    return _fs_mvrm(log, src, dst, False, flags)
+        try:
+            os.replace(bsrc, bdst)
+        except OSError as ex:
+            if ex.errno != errno.EXDEV:
+                raise
+            t = "using copy+delete (%s);\n  %s\n  %s"
+            log(t % (ex.strerror, src, dst))
+            try:
+                os.unlink(bdst)
+            except:
+                pass
+            shutil.move(bsrc, bdst)
 
 
 def wunlink(log: "NamedLogger", abspath: str, flags: dict[str, Any]) -> bool: