revert 68c6794d (v1.6.2) and fix it better:

moving deduplicated files between volumes could drop some links
This commit is contained in:
ed 2023-10-07 22:25:44 +00:00
parent 99e9cba1f7
commit 753e3cfbaf
3 changed files with 69 additions and 25 deletions

View file

@ -803,7 +803,7 @@ def add_upload(ap):
ap2.add_argument("--use-fpool", action="store_true", help="force file-handle pooling, even when it might be dangerous (multiprocessing, filesystems lacking sparse-files support, ...)")
ap2.add_argument("--hardlink", action="store_true", help="prefer hardlinks instead of symlinks when possible (within same filesystem) (volflag=hardlink)")
ap2.add_argument("--never-symlink", action="store_true", help="do not fallback to symlinks when a hardlink cannot be made (volflag=neversymlink)")
ap2.add_argument("--no-dedup", action="store_true", help="disable symlink/hardlink creation; copy file contents instead (volflag=copydupes")
ap2.add_argument("--no-dedup", action="store_true", help="disable symlink/hardlink creation; copy file contents instead (volflag=copydupes)")
ap2.add_argument("--no-dupe", action="store_true", help="reject duplicate files during upload; only matches within the same volume (volflag=nodupe)")
ap2.add_argument("--no-snap", action="store_true", help="disable snapshots -- forget unfinished uploads on shutdown; don't create .hist/up2k.snap files -- abandoned/interrupted uploads must be cleaned up manually")
ap2.add_argument("--rand", action="store_true", help="force randomized filenames, --nrand chars long (volflag=rand)")

View file

@ -162,7 +162,8 @@ flagcats = {
"nohtml": "return html and markdown as text/html",
},
"others": {
"fk=8": 'generates per-file accesskeys,\nwhich will then be required at the "g" permission',
"fk=8": 'generates per-file accesskeys,\nwhich are then required at the "g" permission;\nkeys are invalidated if filesize or inode changes',
"fka=8": 'generates slightly weaker per-file accesskeys,\nwhich are then required at the "g" permission;\nnot affected by filesize or inode numbers',
"davauth": "ask webdav clients to login for all folders",
"davrt": "show lastmod time of symlink destination, not the link itself\n(note: this option is always enabled for recursive listings)",
},

View file

@ -2680,7 +2680,7 @@ class Up2k(object):
fs2 = bos.stat(os.path.dirname(dst)).st_dev
if fs1 == 0 or fs2 == 0:
# py2 on winxp or other unsupported combination
raise OSError(38, "filesystem does not have st_dev")
raise OSError(errno.ENOSYS, "filesystem does not have st_dev")
elif fs1 == fs2:
# same fs; make symlink as relative as possible
spl = r"[\\/]" if WINDOWS else "/"
@ -3300,10 +3300,15 @@ class Up2k(object):
if bos.path.exists(dabs):
raise Pebkac(400, "mv2: target file exists")
stl = bos.lstat(sabs)
try:
st = bos.stat(sabs)
except:
st = stl
xbr = svn.flags.get("xbr")
xar = dvn.flags.get("xar")
if xbr:
st = bos.stat(sabs)
if not runhook(
self.log, xbr, sabs, svp, "", uname, st.st_mtime, st.st_size, "", 0, ""
):
@ -3311,9 +3316,16 @@ class Up2k(object):
self.log(t, 1)
raise Pebkac(405, t)
is_xvol = svn.realpath != dvn.realpath
if stat.S_ISLNK(stl.st_mode):
is_dirlink = stat.S_ISDIR(st.st_mode)
is_link = True
else:
is_link = is_dirlink = False
bos.makedirs(os.path.dirname(dabs))
if bos.path.islink(sabs):
if is_dirlink:
dlabs = absreal(sabs)
t = "moving symlink from [{}] to [{}], target [{}]"
self.log(t.format(sabs, dabs, dlabs))
@ -3336,36 +3348,22 @@ class Up2k(object):
c2 = self.cur.get(dvn.realpath)
if ftime_ is None:
st = bos.stat(sabs)
ftime = st.st_mtime
fsize = st.st_size
else:
ftime = ftime_
fsize = fsize_ or 0
try:
atomic_move(sabs, dabs)
except OSError as ex:
if ex.errno != errno.EXDEV:
raise
self.log("cross-device move:\n {}\n {}".format(sabs, dabs))
b1, b2 = fsenc(sabs), fsenc(dabs)
try:
shutil.copy2(b1, b2)
except:
os.unlink(b2)
raise
os.unlink(b1)
has_dupes = False
if w:
assert c1
if c2 and c2 != c1:
self._copy_tags(c1, c2, w)
self._forget_file(svn.realpath, srem, c1, w, c1 != c2, fsize)
self._relink(w, svn.realpath, srem, dabs)
has_dupes = self._forget_file(svn.realpath, srem, c1, w, is_xvol, fsize)
if not is_xvol:
has_dupes = self._relink(w, svn.realpath, srem, dabs)
curs.add(c1)
if c2:
@ -3388,6 +3386,47 @@ class Up2k(object):
else:
self.log("not found in src db: [{}]".format(svp))
try:
if is_xvol and has_dupes:
raise OSError(errno.EXDEV, "src is symlink")
atomic_move(sabs, dabs)
except OSError as ex:
if ex.errno != errno.EXDEV:
raise
self.log("using copy+delete (%s):\n %s\n %s" % (ex.strerror, sabs, dabs))
b1, b2 = fsenc(sabs), fsenc(dabs)
is_link = os.path.islink(b1) # due to _relink
try:
shutil.copy2(b1, b2)
except:
try:
os.unlink(b2)
except:
pass
if not is_link:
raise
# broken symlink? keep it as-is
try:
zb = os.readlink(b1)
os.symlink(zb, b2)
except:
os.unlink(b2)
raise
if is_link:
try:
times = (int(time.time()), int(stl.st_mtime))
bos.utime(dabs, times, False)
except:
pass
os.unlink(b1)
if xar:
runhook(self.log, xar, dabs, dvp, "", uname, 0, 0, "", 0, "")
@ -3441,14 +3480,16 @@ class Up2k(object):
wark: Optional[str],
drop_tags: bool,
sz: int,
) -> None:
) -> bool:
"""forgets file in db, fixes symlinks, does not delete"""
srd, sfn = vsplit(vrem)
has_dupes = False
self.log("forgetting {}".format(vrem))
if wark and cur:
self.log("found {} in db".format(wark))
if drop_tags:
if self._relink(wark, ptop, vrem, ""):
has_dupes = True
drop_tags = False
if drop_tags:
@ -3476,6 +3517,8 @@ class Up2k(object):
assert wark
del reg[wark]
return has_dupes
def _relink(self, wark: str, sptop: str, srem: str, dabs: str) -> int:
"""
update symlinks from file at svn/srem to dabs (rename),