retry deleting busy files on windows:

some clients (clonezilla-webdav) rapidly create and delete files;
this fails if copyparty is still hashing the file (usually the case)

and the same thing can probably happen due to antivirus etc

add global-option --rm-retry (volflag rm_retry) specifying
for how long (and how quickly) to keep retrying the deletion

default: retry for 5sec on windows, 0sec (disabled) on everything else
because this is only a problem on windows

copyparty/__main__.py

@@ -859,6 +859,12 @@ def add_qr(ap, tty):
     ap2.add_argument("--qrz", metavar="N", type=int, default=0, help="[\033[32m1\033[0m]=1x, [\033[32m2\033[0m]=2x, [\033[32m0\033[0m]=auto (try [\033[32m2\033[0m] on broken fonts)")
 
 
+def add_fs(ap):
+    ap2 = ap.add_argument_group("filesystem options")
+    rm_re_def = "5/0.1" if ANYWIN else "0/0"
+    ap2.add_argument("--rm-retry", metavar="T/R", type=u, default=rm_re_def, help="if a file cannot be deleted because it is busy, continue trying for \033[33mT\033[0m seconds, retry every \033[33mR\033[0m seconds; disable with 0/0 (volflag=rm_retry)")
+
+
 def add_upload(ap):
     ap2 = ap.add_argument_group('upload options')
     ap2.add_argument("--dotpart", action="store_true", help="dotfile incomplete uploads, hiding them from clients unless \033[33m-ed\033[0m")
@@ -1308,6 +1314,7 @@ def run_argparse(
     add_zeroconf(ap)
     add_zc_mdns(ap)
     add_zc_ssdp(ap)
+    add_fs(ap)
     add_upload(ap)
     add_db_general(ap, hcores)
     add_db_metadata(ap)

copyparty/authsrv.py

@@ -1494,6 +1494,14 @@ class AuthSrv(object):
                 if k in vol.flags:
                     vol.flags[k] = float(vol.flags[k])
 
+            try:
+                zs1, zs2 = vol.flags["rm_retry"].split("/")
+                vol.flags["rm_re_t"] = float(zs1)
+                vol.flags["rm_re_r"] = float(zs2)
+            except:
+                t = 'volume "/%s" has invalid rm_retry [%s]'
+                raise Exception(t % (vol.vpath, vol.flags.get("rm_retry")))
+
             for k1, k2 in IMPLICATIONS:
                 if k1 in vol.flags:
                     vol.flags[k2] = True
@@ -1505,8 +1513,8 @@ class AuthSrv(object):
             dbds = "acid|swal|wal|yolo"
             vol.flags["dbd"] = dbd = vol.flags.get("dbd") or self.args.dbd
             if dbd not in dbds.split("|"):
-                t = "invalid dbd [{}]; must be one of [{}]"
+                t = 'volume "/%s" has invalid dbd [%s]; must be one of [%s]'
-                raise Exception(t.format(dbd, dbds))
+                raise Exception(t % (vol.vpath, dbd, dbds))
 
             # default tag cfgs if unset
             for k in ("mte", "mth", "exp_md", "exp_lg"):

copyparty/cfg.py

@@ -62,6 +62,7 @@ def vf_vmap() -> dict[str, str]:
         "lg_sbf",
         "md_sbf",
         "nrand",
+        "rm_retry",
         "sort",
         "unlist",
         "u2ts",
@@ -208,6 +209,7 @@ flagcats = {
         "dots": "allow all users with read-access to\nenable the option to show dotfiles in listings",
         "fk=8": 'generates per-file accesskeys,\nwhich are then required at the "g" permission;\nkeys are invalidated if filesize or inode changes',
         "fka=8": 'generates slightly weaker per-file accesskeys,\nwhich are then required at the "g" permission;\nnot affected by filesize or inode numbers',
+        "rm_retry": "ms-windows: timeout for deleting busy files",
         "davauth": "ask webdav clients to login for all folders",
         "davrt": "show lastmod time of symlink destination, not the link itself\n(note: this option is always enabled for recursive listings)",
     },

copyparty/httpcli.py

@@ -88,6 +88,7 @@ from .util import (
     vjoin,
     vol_san,
     vsplit,
+    wunlink,
     yieldfile,
 )
 
@@ -1691,7 +1692,7 @@ class HttpCli(object):
                 and bos.path.getmtime(path) >= time.time() - self.args.blank_wt
             ):
                 # small toctou, but better than clobbering a hardlink
-                bos.unlink(path)
+                wunlink(self.log, path, vfs.flags)
 
         with ren_open(fn, *open_a, **params) as zfw:
             f, fn = zfw["orz"]
@@ -1705,7 +1706,7 @@ class HttpCli(object):
                 lim.chk_sz(post_sz)
                 lim.chk_vsz(self.conn.hsrv.broker, vfs.realpath, post_sz)
             except:
-                bos.unlink(path)
+                wunlink(self.log, path, vfs.flags)
                 raise
 
         if self.args.nw:
@@ -1758,7 +1759,7 @@ class HttpCli(object):
         ):
             t = "upload blocked by xau server config"
             self.log(t, 1)
-            os.unlink(path)
+            wunlink(self.log, path, vfs.flags)
             raise Pebkac(403, t)
 
         vfs, rem = vfs.get_dbv(rem)
@@ -2439,8 +2440,8 @@ class HttpCli(object):
                             lim.chk_nup(self.ip)
                         except:
                             if not nullwrite:
-                                bos.unlink(tabspath)
+                                wunlink(self.log, tabspath, vfs.flags)
-                                bos.unlink(abspath)
+                                wunlink(self.log, abspath, vfs.flags)
                             fname = os.devnull
                             raise
 
@@ -2468,7 +2469,7 @@ class HttpCli(object):
                     ):
                         t = "upload blocked by xau server config"
                         self.log(t, 1)
-                        os.unlink(abspath)
+                        wunlink(self.log, abspath, vfs.flags)
                         raise Pebkac(403, t)
 
                     dbv, vrem = vfs.get_dbv(rem)
@@ -2712,7 +2713,7 @@ class HttpCli(object):
                 raise Pebkac(403, t)
 
         if bos.path.exists(fp):
-            bos.unlink(fp)
+            wunlink(self.log, fp, vfs.flags)
 
         with open(fsenc(fp), "wb", 512 * 1024) as f:
             sz, sha512, _ = hashcopy(p_data, f, self.args.s_wr_slp)
@@ -2724,7 +2725,7 @@ class HttpCli(object):
                 lim.chk_sz(sz)
                 lim.chk_vsz(self.conn.hsrv.broker, vfs.realpath, sz)
             except:
-                bos.unlink(fp)
+                wunlink(self.log, fp, vfs.flags)
                 raise
 
         new_lastmod = bos.stat(fp).st_mtime
@@ -2747,7 +2748,7 @@ class HttpCli(object):
         ):
             t = "save blocked by xau server config"
             self.log(t, 1)
-            os.unlink(fp)
+            wunlink(self.log, fp, vfs.flags)
             raise Pebkac(403, t)
 
         vfs, rem = vfs.get_dbv(rem)

copyparty/svchub.py

@@ -460,6 +460,13 @@ class SvcHub(object):
             if ptn:
                 setattr(self.args, k, re.compile(ptn))
 
+        try:
+            zf1, zf2 = self.args.rm_retry.split("/")
+            self.args.rm_re_t = float(zf1)
+            self.args.rm_re_r = float(zf2)
+        except:
+            raise Exception("invalid --rm-retry [%s]" % (self.args.rm_retry,))
+
         return True
 
     def _ipa2re(self, txt) -> Optional[re.Pattern]:

copyparty/th_srv.py

@@ -28,6 +28,7 @@ from .util import (
     runcmd,
     statdir,
     vsplit,
+    wunlink,
 )
 
 if True:  # pylint: disable=using-constant-test
@@ -317,7 +318,7 @@ class ThumbSrv(object):
             tdir, tfn = os.path.split(tpath)
             ttpath = os.path.join(tdir, "w", tfn)
             try:
-                bos.unlink(ttpath)
+                wunlink(self.log, ttpath, vn.flags)
             except:
                 pass
 
@@ -337,7 +338,7 @@ class ThumbSrv(object):
                     else:
                         # ffmpeg may spawn empty files on windows
                         try:
-                            os.unlink(ttpath)
+                            wunlink(self.log, ttpath, vn.flags)
                         except:
                             pass
 
@@ -651,7 +652,7 @@ class ThumbSrv(object):
         if want_caf:
             tmp_opus = tpath + ".opus"
             try:
-                bos.unlink(tmp_opus)
+                wunlink(self.log, tmp_opus, vn.flags)
             except:
                 pass
 
@@ -718,7 +719,7 @@ class ThumbSrv(object):
 
         if tmp_opus != tpath:
             try:
-                bos.unlink(tmp_opus)
+                wunlink(self.log, tmp_opus, vn.flags)
             except:
                 pass
 
@@ -745,7 +746,10 @@ class ThumbSrv(object):
                 else:
                     self.log("\033[Jcln {} ({})/\033[A".format(histpath, vol))
 
-                ndirs += self.clean(histpath)
+                try:
+                    ndirs += self.clean(histpath)
+                except Exception as ex:
+                    self.log("\033[Jcln err in %s: %r" % (histpath, ex), 3)
 
             self.log("\033[Jcln ok; rm {} dirs".format(ndirs))
 

copyparty/up2k.py

@@ -64,6 +64,7 @@ from .util import (
     vsplit,
     w8b64dec,
     w8b64enc,
+    wunlink,
 )
 
 try:
@@ -808,7 +809,7 @@ class Up2k(object):
         ft = "\033[0;32m{}{:.0}"
         ff = "\033[0;35m{}{:.0}"
         fv = "\033[0;36m{}:\033[90m{}"
-        fx = set(("html_head",))
+        fx = set(("html_head", "rm_re_t", "rm_re_r"))
         fd = vf_bmap()
         fd.update(vf_cmap())
         fd.update(vf_vmap())
@@ -2585,12 +2586,13 @@ class Up2k(object):
                             raise Pebkac(403, t)
 
                         if not self.args.nw:
+                            dvf: dict[str, Any] = vfs.flags
                             try:
                                 dvf = self.flags[job["ptop"]]
                                 self._symlink(src, dst, dvf, lmod=cj["lmod"], rm=True)
                             except:
                                 if bos.path.exists(dst):
-                                    bos.unlink(dst)
+                                    wunlink(self.log, dst, dvf)
                                 if not n4g:
                                     raise
 
@@ -2699,7 +2701,7 @@ class Up2k(object):
         fp = djoin(fdir, fname)
         if job.get("replace") and bos.path.exists(fp):
             self.log("replacing existing file at {}".format(fp))
-            bos.unlink(fp)
+            wunlink(self.log, fp, self.flags.get(job["ptop"]) or {})
 
         if self.args.plain_ip:
             dip = ip.replace(":", ".")
@@ -2757,7 +2759,7 @@ class Up2k(object):
                 ldst = ldst.replace("/", "\\")
 
             if rm and bos.path.exists(dst):
-                bos.unlink(dst)
+                wunlink(self.log, dst, flags)
 
             try:
                 if "hardlink" in flags:
@@ -2773,7 +2775,7 @@ class Up2k(object):
                     Path(ldst).symlink_to(lsrc)
                     if not bos.path.exists(dst):
                         try:
-                            bos.unlink(dst)
+                            wunlink(self.log, dst, flags)
                         except:
                             pass
                         t = "the created symlink [%s] did not resolve to [%s]"
@@ -3076,7 +3078,7 @@ class Up2k(object):
         ):
             t = "upload blocked by xau server config"
             self.log(t, 1)
-            bos.unlink(dst)
+            wunlink(self.log, dst, vflags)
             self.registry[ptop].pop(wark, None)
             raise Pebkac(403, t)
 
@@ -3247,7 +3249,7 @@ class Up2k(object):
                         if cur:
                             cur.connection.commit()
 
-                bos.unlink(abspath)
+                wunlink(self.log, abspath, dbv.flags)
                 if xad:
                     runhook(
                         self.log,
@@ -3402,7 +3404,7 @@ class Up2k(object):
             t = "moving symlink from [{}] to [{}], target [{}]"
             self.log(t.format(sabs, dabs, dlabs))
             mt = bos.path.getmtime(sabs, False)
-            bos.unlink(sabs)
+            wunlink(self.log, sabs, svn.flags)
             self._symlink(dlabs, dabs, dvn.flags, False, lmod=mt)
 
             # folders are too scary, schedule rescan of both vols
@@ -3469,7 +3471,7 @@ class Up2k(object):
                 dlink = os.path.join(os.path.dirname(sabs), dlink)
                 dlink = bos.path.abspath(dlink)
                 self._symlink(dlink, dabs, dvn.flags, lmod=ftime)
-                bos.unlink(sabs)
+                wunlink(self.log, sabs, svn.flags)
             else:
                 atomic_move(sabs, dabs)
 
@@ -3484,7 +3486,7 @@ class Up2k(object):
                 shutil.copy2(b1, b2)
             except:
                 try:
-                    os.unlink(b2)
+                    wunlink(self.log, dabs, dvn.flags)
                 except:
                     pass
 
@@ -3496,7 +3498,7 @@ class Up2k(object):
                     zb = os.readlink(b1)
                     os.symlink(zb, b2)
                 except:
-                    os.unlink(b2)
+                    wunlink(self.log, dabs, dvn.flags)
                     raise
 
             if is_link:
@@ -3506,7 +3508,7 @@ class Up2k(object):
                 except:
                     pass
 
-            os.unlink(b1)
+            wunlink(self.log, sabs, svn.flags)
 
         if xar:
             runhook(self.log, xar, dabs, dvp, "", uname, 0, 0, "", 0, "")
@@ -3646,10 +3648,11 @@ class Up2k(object):
             ptop, rem = links.pop(slabs)
             self.log("linkswap [{}] and [{}]".format(sabs, slabs))
             mt = bos.path.getmtime(slabs, False)
-            bos.unlink(slabs)
+            flags = self.flags.get(ptop) or {}
+            wunlink(self.log, slabs, flags)
             bos.rename(sabs, slabs)
             bos.utime(slabs, (int(time.time()), int(mt)), False)
-            self._symlink(slabs, sabs, self.flags.get(ptop) or {}, False)
+            self._symlink(slabs, sabs, flags, False)
             full[slabs] = (ptop, rem)
             sabs = slabs
 
@@ -3695,13 +3698,13 @@ class Up2k(object):
                 self.log(t % (ex, ex), 3)
 
             self.log("relinking [%s] to [%s]" % (alink, dabs))
+            flags = self.flags.get(parts[0]) or {}
             try:
                 lmod = bos.path.getmtime(alink, False)
-                bos.unlink(alink)
+                wunlink(self.log, alink, flags)
             except:
                 pass
 
-            flags = self.flags.get(parts[0]) or {}
             self._symlink(dabs, alink, flags, False, lmod=lmod or 0)
 
         return len(full) + len(links)

copyparty/util.py

@@ -2078,6 +2078,41 @@ def atomic_move(usrc: str, udst: str) -> None:
         os.rename(src, dst)
 
 
+def wunlink(log: "NamedLogger", abspath: str, flags: dict[str, Any]) -> bool:
+    maxtime = flags.get("rm_re_t", 0.0)
+    bpath = fsenc(abspath)
+    if not maxtime:
+        os.unlink(bpath)
+        return True
+
+    chill = flags.get("rm_re_r", 0.0)
+    if chill < 0.001:
+        chill = 0.1
+
+    t0 = now = time.time()
+    for attempt in range(90210):
+        try:
+            os.unlink(bpath)
+            if attempt:
+                now = time.time()
+                t = "deleted in %.2f sec, attempt %d"
+                log(t % (now - t0, attempt + 1))
+            return True
+        except OSError as ex:
+            now = time.time()
+            if ex.errno == errno.ENOENT:
+                return False
+            if now - t0 > maxtime or attempt == 90209:
+                raise
+            if not attempt:
+                t = "delete failed (err.%d); retrying for %d sec: %s"
+                log(t % (ex.errno, maxtime + 0.99, abspath))
+
+        time.sleep(chill)
+
+    return False  # makes pylance happy
+
+
 def get_df(abspath: str) -> tuple[Optional[int], Optional[int]]:
     try:
         # some fuses misbehave