mirrors/copyparty
1
0
Fork 0
mirror of https://github.com/9001/copyparty.git synced 2025-08-17 09:02:15 -06:00
Code Issues Actions Packages Projects Releases Wiki Activity

retry deleting busy files on windows:

Browse source 
some clients (clonezilla-webdav) rapidly create and delete files;
this fails if copyparty is still hashing the file (usually the case)

and the same thing can probably happen due to antivirus etc

add global-option --rm-retry (volflag rm_retry) specifying
for how long (and how quickly) to keep retrying the deletion

default: retry for 5sec on windows, 0sec (disabled) on everything else
because this is only a problem on windows
This commit is contained in:
ed 2024-01-17 20:27:53 +00:00
parent d999d3a921
commit 3313503ea5
8 changed files with 99 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
copyparty/__main__.py
View file

@ -859,6 +859,12 @@ def add_qr(ap, tty):
ap2.add_argument("--qrz", metavar="N", type=int, default=0, help="[\033[32m1\033[0m]=1x, [\033[32m2\033[0m]=2x, [\033[32m0\033[0m]=auto (try [\033[32m2\033[0m] on broken fonts)")
def add_fs(ap):
ap2 = ap.add_argument_group("filesystem options")
rm_re_def = "5/0.1" if ANYWIN else "0/0"
ap2.add_argument("--rm-retry", metavar="T/R", type=u, default=rm_re_def, help="if a file cannot be deleted because it is busy, continue trying for \033[33mT\033[0m seconds, retry every \033[33mR\033[0m seconds; disable with 0/0 (volflag=rm_retry)")
def add_upload(ap):
ap2 = ap.add_argument_group('upload options')
ap2.add_argument("--dotpart", action="store_true", help="dotfile incomplete uploads, hiding them from clients unless \033[33m-ed\033[0m")
@ -1308,6 +1314,7 @@ def run_argparse(
add_zeroconf(ap)
add_zc_mdns(ap)
add_zc_ssdp(ap)
add_fs(ap)
add_upload(ap)
add_db_general(ap, hcores)
add_db_metadata(ap)

12
copyparty/authsrv.py
View file

@ -1494,6 +1494,14 @@ class AuthSrv(object):
if k in vol.flags:
vol.flags[k] = float(vol.flags[k])
try:
zs1, zs2 = vol.flags["rm_retry"].split("/")
vol.flags["rm_re_t"] = float(zs1)
vol.flags["rm_re_r"] = float(zs2)
except:
t = 'volume "/%s" has invalid rm_retry [%s]'
raise Exception(t % (vol.vpath, vol.flags.get("rm_retry")))
for k1, k2 in IMPLICATIONS:
if k1 in vol.flags:
vol.flags[k2] = True
@ -1505,8 +1513,8 @@ class AuthSrv(object):
dbds = "acid|swal|wal|yolo"
vol.flags["dbd"] = dbd = vol.flags.get("dbd") or self.args.dbd
if dbd not in dbds.split("|"):
t = "invalid dbd [{}]; must be one of [{}]"
raise Exception(t.format(dbd, dbds))
t = 'volume "/%s" has invalid dbd [%s]; must be one of [%s]'
raise Exception(t % (vol.vpath, dbd, dbds))
# default tag cfgs if unset
for k in ("mte", "mth", "exp_md", "exp_lg"):

2
copyparty/cfg.py
View file

@ -62,6 +62,7 @@ def vf_vmap() -> dict[str, str]:
"lg_sbf",
"md_sbf",
"nrand",
"rm_retry",
"sort",
"unlist",
"u2ts",
@ -208,6 +209,7 @@ flagcats = {
"dots": "allow all users with read-access to\nenable the option to show dotfiles in listings",
"fk=8": 'generates per-file accesskeys,\nwhich are then required at the "g" permission;\nkeys are invalidated if filesize or inode changes',
"fka=8": 'generates slightly weaker per-file accesskeys,\nwhich are then required at the "g" permission;\nnot affected by filesize or inode numbers',
"rm_retry": "ms-windows: timeout for deleting busy files",
"davauth": "ask webdav clients to login for all folders",
"davrt": "show lastmod time of symlink destination, not the link itself\n(note: this option is always enabled for recursive listings)",
},

19
copyparty/httpcli.py
View file

@ -88,6 +88,7 @@ from .util import (
vjoin,
vol_san,
vsplit,
wunlink,
yieldfile,
)
@ -1691,7 +1692,7 @@ class HttpCli(object):
and bos.path.getmtime(path) >= time.time() - self.args.blank_wt
):
# small toctou, but better than clobbering a hardlink
bos.unlink(path)
wunlink(self.log, path, vfs.flags)
with ren_open(fn, *open_a, **params) as zfw:
f, fn = zfw["orz"]
@ -1705,7 +1706,7 @@ class HttpCli(object):
lim.chk_sz(post_sz)
lim.chk_vsz(self.conn.hsrv.broker, vfs.realpath, post_sz)
except:
bos.unlink(path)
wunlink(self.log, path, vfs.flags)
raise
if self.args.nw:
@ -1758,7 +1759,7 @@ class HttpCli(object):
):
t = "upload blocked by xau server config"
self.log(t, 1)
os.unlink(path)
wunlink(self.log, path, vfs.flags)
raise Pebkac(403, t)
vfs, rem = vfs.get_dbv(rem)
@ -2439,8 +2440,8 @@ class HttpCli(object):
lim.chk_nup(self.ip)
except:
if not nullwrite:
bos.unlink(tabspath)
bos.unlink(abspath)
wunlink(self.log, tabspath, vfs.flags)
wunlink(self.log, abspath, vfs.flags)
fname = os.devnull
raise
@ -2468,7 +2469,7 @@ class HttpCli(object):
):
t = "upload blocked by xau server config"
self.log(t, 1)
os.unlink(abspath)
wunlink(self.log, abspath, vfs.flags)
raise Pebkac(403, t)
dbv, vrem = vfs.get_dbv(rem)
@ -2712,7 +2713,7 @@ class HttpCli(object):
raise Pebkac(403, t)
if bos.path.exists(fp):
bos.unlink(fp)
wunlink(self.log, fp, vfs.flags)
with open(fsenc(fp), "wb", 512 * 1024) as f:
sz, sha512, _ = hashcopy(p_data, f, self.args.s_wr_slp)
@ -2724,7 +2725,7 @@ class HttpCli(object):
lim.chk_sz(sz)
lim.chk_vsz(self.conn.hsrv.broker, vfs.realpath, sz)
except:
bos.unlink(fp)
wunlink(self.log, fp, vfs.flags)
raise
new_lastmod = bos.stat(fp).st_mtime
@ -2747,7 +2748,7 @@ class HttpCli(object):
):
t = "save blocked by xau server config"
self.log(t, 1)
os.unlink(fp)
wunlink(self.log, fp, vfs.flags)
raise Pebkac(403, t)
vfs, rem = vfs.get_dbv(rem)

7
copyparty/svchub.py
View file

@ -460,6 +460,13 @@ class SvcHub(object):
if ptn:
setattr(self.args, k, re.compile(ptn))
try:
zf1, zf2 = self.args.rm_retry.split("/")
self.args.rm_re_t = float(zf1)
self.args.rm_re_r = float(zf2)
except:
raise Exception("invalid --rm-retry [%s]" % (self.args.rm_retry,))
return True
def _ipa2re(self, txt) -> Optional[re.Pattern]:

12
copyparty/th_srv.py
View file

@ -28,6 +28,7 @@ from .util import (
runcmd,
statdir,
vsplit,
wunlink,
)
if True: # pylint: disable=using-constant-test
@ -317,7 +318,7 @@ class ThumbSrv(object):
tdir, tfn = os.path.split(tpath)
ttpath = os.path.join(tdir, "w", tfn)
try:
bos.unlink(ttpath)
wunlink(self.log, ttpath, vn.flags)
except:
pass
@ -337,7 +338,7 @@ class ThumbSrv(object):
else:
# ffmpeg may spawn empty files on windows
try:
os.unlink(ttpath)
wunlink(self.log, ttpath, vn.flags)
except:
pass
@ -651,7 +652,7 @@ class ThumbSrv(object):
if want_caf:
tmp_opus = tpath + ".opus"
try:
bos.unlink(tmp_opus)
wunlink(self.log, tmp_opus, vn.flags)
except:
pass
@ -718,7 +719,7 @@ class ThumbSrv(object):
if tmp_opus != tpath:
try:
bos.unlink(tmp_opus)
wunlink(self.log, tmp_opus, vn.flags)
except:
pass
@ -745,7 +746,10 @@ class ThumbSrv(object):
else:
self.log("\033[Jcln {} ({})/\033[A".format(histpath, vol))
try:
ndirs += self.clean(histpath)
except Exception as ex:
self.log("\033[Jcln err in %s: %r" % (histpath, ex), 3)
self.log("\033[Jcln ok; rm {} dirs".format(ndirs))

35
copyparty/up2k.py
View file

@ -64,6 +64,7 @@ from .util import (
vsplit,
w8b64dec,
w8b64enc,
wunlink,
)
try:
@ -808,7 +809,7 @@ class Up2k(object):
ft = "\033[0;32m{}{:.0}"
ff = "\033[0;35m{}{:.0}"
fv = "\033[0;36m{}:\033[90m{}"
fx = set(("html_head",))
fx = set(("html_head", "rm_re_t", "rm_re_r"))
fd = vf_bmap()
fd.update(vf_cmap())
fd.update(vf_vmap())
@ -2585,12 +2586,13 @@ class Up2k(object):
raise Pebkac(403, t)
if not self.args.nw:
dvf: dict[str, Any] = vfs.flags
try:
dvf = self.flags[job["ptop"]]
self._symlink(src, dst, dvf, lmod=cj["lmod"], rm=True)
except:
if bos.path.exists(dst):
bos.unlink(dst)
wunlink(self.log, dst, dvf)
if not n4g:
raise
@ -2699,7 +2701,7 @@ class Up2k(object):
fp = djoin(fdir, fname)
if job.get("replace") and bos.path.exists(fp):
self.log("replacing existing file at {}".format(fp))
bos.unlink(fp)
wunlink(self.log, fp, self.flags.get(job["ptop"]) or {})
if self.args.plain_ip:
dip = ip.replace(":", ".")
@ -2757,7 +2759,7 @@ class Up2k(object):
ldst = ldst.replace("/", "\\")
if rm and bos.path.exists(dst):
bos.unlink(dst)
wunlink(self.log, dst, flags)
try:
if "hardlink" in flags:
@ -2773,7 +2775,7 @@ class Up2k(object):
Path(ldst).symlink_to(lsrc)
if not bos.path.exists(dst):
try:
bos.unlink(dst)
wunlink(self.log, dst, flags)
except:
pass
t = "the created symlink [%s] did not resolve to [%s]"
@ -3076,7 +3078,7 @@ class Up2k(object):
):
t = "upload blocked by xau server config"
self.log(t, 1)
bos.unlink(dst)
wunlink(self.log, dst, vflags)
self.registry[ptop].pop(wark, None)
raise Pebkac(403, t)
@ -3247,7 +3249,7 @@ class Up2k(object):
if cur:
cur.connection.commit()
bos.unlink(abspath)
wunlink(self.log, abspath, dbv.flags)
if xad:
runhook(
self.log,
@ -3402,7 +3404,7 @@ class Up2k(object):
t = "moving symlink from [{}] to [{}], target [{}]"
self.log(t.format(sabs, dabs, dlabs))
mt = bos.path.getmtime(sabs, False)
bos.unlink(sabs)
wunlink(self.log, sabs, svn.flags)
self._symlink(dlabs, dabs, dvn.flags, False, lmod=mt)
# folders are too scary, schedule rescan of both vols
@ -3469,7 +3471,7 @@ class Up2k(object):
dlink = os.path.join(os.path.dirname(sabs), dlink)
dlink = bos.path.abspath(dlink)
self._symlink(dlink, dabs, dvn.flags, lmod=ftime)
bos.unlink(sabs)
wunlink(self.log, sabs, svn.flags)
else:
atomic_move(sabs, dabs)
@ -3484,7 +3486,7 @@ class Up2k(object):
shutil.copy2(b1, b2)
except:
try:
os.unlink(b2)
wunlink(self.log, dabs, dvn.flags)
except:
pass
@ -3496,7 +3498,7 @@ class Up2k(object):
zb = os.readlink(b1)
os.symlink(zb, b2)
except:
os.unlink(b2)
wunlink(self.log, dabs, dvn.flags)
raise
if is_link:
@ -3506,7 +3508,7 @@ class Up2k(object):
except:
pass
os.unlink(b1)
wunlink(self.log, sabs, svn.flags)
if xar:
runhook(self.log, xar, dabs, dvp, "", uname, 0, 0, "", 0, "")
@ -3646,10 +3648,11 @@ class Up2k(object):
ptop, rem = links.pop(slabs)
self.log("linkswap [{}] and [{}]".format(sabs, slabs))
mt = bos.path.getmtime(slabs, False)
bos.unlink(slabs)
flags = self.flags.get(ptop) or {}
wunlink(self.log, slabs, flags)
bos.rename(sabs, slabs)
bos.utime(slabs, (int(time.time()), int(mt)), False)
self._symlink(slabs, sabs, self.flags.get(ptop) or {}, False)
self._symlink(slabs, sabs, flags, False)
full[slabs] = (ptop, rem)
sabs = slabs
@ -3695,13 +3698,13 @@ class Up2k(object):
self.log(t % (ex, ex), 3)
self.log("relinking [%s] to [%s]" % (alink, dabs))
flags = self.flags.get(parts[0]) or {}
try:
lmod = bos.path.getmtime(alink, False)
bos.unlink(alink)
wunlink(self.log, alink, flags)
except:
pass
flags = self.flags.get(parts[0]) or {}
self._symlink(dabs, alink, flags, False, lmod=lmod or 0)
return len(full) + len(links)

35
copyparty/util.py
View file

@ -2078,6 +2078,41 @@ def atomic_move(usrc: str, udst: str) -> None:
os.rename(src, dst)
def wunlink(log: "NamedLogger", abspath: str, flags: dict[str, Any]) -> bool:
maxtime = flags.get("rm_re_t", 0.0)
bpath = fsenc(abspath)
if not maxtime:
os.unlink(bpath)
return True
chill = flags.get("rm_re_r", 0.0)
if chill < 0.001:
chill = 0.1
t0 = now = time.time()
for attempt in range(90210):
try:
os.unlink(bpath)
if attempt:
now = time.time()
t = "deleted in %.2f sec, attempt %d"
log(t % (now - t0, attempt + 1))
return True
except OSError as ex:
now = time.time()
if ex.errno == errno.ENOENT:
return False
if now - t0 > maxtime or attempt == 90209:
raise
if not attempt:
t = "delete failed (err.%d); retrying for %d sec: %s"
log(t % (ex.errno, maxtime + 0.99, abspath))
time.sleep(chill)
return False # makes pylance happy
def get_df(abspath: str) -> tuple[Optional[int], Optional[int]]:
try:
# some fuses misbehave