From 753e3cfbafb4abb2bb027287e6a0ac18f6725f6a Mon Sep 17 00:00:00 2001 From: ed Date: Sat, 7 Oct 2023 22:25:44 +0000 Subject: [PATCH] revert 68c6794d (v1.6.2) and fix it better: moving deduplicated files between volumes could drop some links --- copyparty/__main__.py | 2 +- copyparty/cfg.py | 3 +- copyparty/up2k.py | 89 ++++++++++++++++++++++++++++++++----------- 3 files changed, 69 insertions(+), 25 deletions(-) diff --git a/copyparty/__main__.py b/copyparty/__main__.py index 691c0009..905ce5e6 100755 --- a/copyparty/__main__.py +++ b/copyparty/__main__.py @@ -803,7 +803,7 @@ def add_upload(ap): ap2.add_argument("--use-fpool", action="store_true", help="force file-handle pooling, even when it might be dangerous (multiprocessing, filesystems lacking sparse-files support, ...)") ap2.add_argument("--hardlink", action="store_true", help="prefer hardlinks instead of symlinks when possible (within same filesystem) (volflag=hardlink)") ap2.add_argument("--never-symlink", action="store_true", help="do not fallback to symlinks when a hardlink cannot be made (volflag=neversymlink)") - ap2.add_argument("--no-dedup", action="store_true", help="disable symlink/hardlink creation; copy file contents instead (volflag=copydupes") + ap2.add_argument("--no-dedup", action="store_true", help="disable symlink/hardlink creation; copy file contents instead (volflag=copydupes)") ap2.add_argument("--no-dupe", action="store_true", help="reject duplicate files during upload; only matches within the same volume (volflag=nodupe)") ap2.add_argument("--no-snap", action="store_true", help="disable snapshots -- forget unfinished uploads on shutdown; don't create .hist/up2k.snap files -- abandoned/interrupted uploads must be cleaned up manually") ap2.add_argument("--rand", action="store_true", help="force randomized filenames, --nrand chars long (volflag=rand)") diff --git a/copyparty/cfg.py b/copyparty/cfg.py index db52bbd1..7fde3c6c 100644 --- a/copyparty/cfg.py +++ b/copyparty/cfg.py @@ -162,7 +162,8 @@ flagcats = { "nohtml": "return html and markdown as text/html", }, "others": { - "fk=8": 'generates per-file accesskeys,\nwhich will then be required at the "g" permission', + "fk=8": 'generates per-file accesskeys,\nwhich are then required at the "g" permission;\nkeys are invalidated if filesize or inode changes', + "fka=8": 'generates slightly weaker per-file accesskeys,\nwhich are then required at the "g" permission;\nnot affected by filesize or inode numbers', "davauth": "ask webdav clients to login for all folders", "davrt": "show lastmod time of symlink destination, not the link itself\n(note: this option is always enabled for recursive listings)", }, diff --git a/copyparty/up2k.py b/copyparty/up2k.py index 42577ca5..fc424554 100644 --- a/copyparty/up2k.py +++ b/copyparty/up2k.py @@ -2680,7 +2680,7 @@ class Up2k(object): fs2 = bos.stat(os.path.dirname(dst)).st_dev if fs1 == 0 or fs2 == 0: # py2 on winxp or other unsupported combination - raise OSError(38, "filesystem does not have st_dev") + raise OSError(errno.ENOSYS, "filesystem does not have st_dev") elif fs1 == fs2: # same fs; make symlink as relative as possible spl = r"[\\/]" if WINDOWS else "/" @@ -3300,10 +3300,15 @@ class Up2k(object): if bos.path.exists(dabs): raise Pebkac(400, "mv2: target file exists") + stl = bos.lstat(sabs) + try: + st = bos.stat(sabs) + except: + st = stl + xbr = svn.flags.get("xbr") xar = dvn.flags.get("xar") if xbr: - st = bos.stat(sabs) if not runhook( self.log, xbr, sabs, svp, "", uname, st.st_mtime, st.st_size, "", 0, "" ): @@ -3311,9 +3316,16 @@ class Up2k(object): self.log(t, 1) raise Pebkac(405, t) + is_xvol = svn.realpath != dvn.realpath + if stat.S_ISLNK(stl.st_mode): + is_dirlink = stat.S_ISDIR(st.st_mode) + is_link = True + else: + is_link = is_dirlink = False + bos.makedirs(os.path.dirname(dabs)) - if bos.path.islink(sabs): + if is_dirlink: dlabs = absreal(sabs) t = "moving symlink from [{}] to [{}], target [{}]" self.log(t.format(sabs, dabs, dlabs)) @@ -3336,36 +3348,22 @@ class Up2k(object): c2 = self.cur.get(dvn.realpath) if ftime_ is None: - st = bos.stat(sabs) ftime = st.st_mtime fsize = st.st_size else: ftime = ftime_ fsize = fsize_ or 0 - try: - atomic_move(sabs, dabs) - except OSError as ex: - if ex.errno != errno.EXDEV: - raise - - self.log("cross-device move:\n {}\n {}".format(sabs, dabs)) - b1, b2 = fsenc(sabs), fsenc(dabs) - try: - shutil.copy2(b1, b2) - except: - os.unlink(b2) - raise - - os.unlink(b1) - + has_dupes = False if w: assert c1 if c2 and c2 != c1: self._copy_tags(c1, c2, w) - self._forget_file(svn.realpath, srem, c1, w, c1 != c2, fsize) - self._relink(w, svn.realpath, srem, dabs) + has_dupes = self._forget_file(svn.realpath, srem, c1, w, is_xvol, fsize) + if not is_xvol: + has_dupes = self._relink(w, svn.realpath, srem, dabs) + curs.add(c1) if c2: @@ -3388,6 +3386,47 @@ class Up2k(object): else: self.log("not found in src db: [{}]".format(svp)) + try: + if is_xvol and has_dupes: + raise OSError(errno.EXDEV, "src is symlink") + + atomic_move(sabs, dabs) + + except OSError as ex: + if ex.errno != errno.EXDEV: + raise + + self.log("using copy+delete (%s):\n %s\n %s" % (ex.strerror, sabs, dabs)) + b1, b2 = fsenc(sabs), fsenc(dabs) + is_link = os.path.islink(b1) # due to _relink + try: + shutil.copy2(b1, b2) + except: + try: + os.unlink(b2) + except: + pass + + if not is_link: + raise + + # broken symlink? keep it as-is + try: + zb = os.readlink(b1) + os.symlink(zb, b2) + except: + os.unlink(b2) + raise + + if is_link: + try: + times = (int(time.time()), int(stl.st_mtime)) + bos.utime(dabs, times, False) + except: + pass + + os.unlink(b1) + if xar: runhook(self.log, xar, dabs, dvp, "", uname, 0, 0, "", 0, "") @@ -3441,14 +3480,16 @@ class Up2k(object): wark: Optional[str], drop_tags: bool, sz: int, - ) -> None: + ) -> bool: """forgets file in db, fixes symlinks, does not delete""" srd, sfn = vsplit(vrem) + has_dupes = False self.log("forgetting {}".format(vrem)) if wark and cur: self.log("found {} in db".format(wark)) if drop_tags: if self._relink(wark, ptop, vrem, ""): + has_dupes = True drop_tags = False if drop_tags: @@ -3476,6 +3517,8 @@ class Up2k(object): assert wark del reg[wark] + return has_dupes + def _relink(self, wark: str, sptop: str, srem: str, dabs: str) -> int: """ update symlinks from file at svn/srem to dabs (rename),