mirrors/copyparty
1
0
Fork 0
mirror of https://github.com/9001/copyparty.git synced 2025-08-17 00:52:16 -06:00
Code Issues Actions Packages Projects Releases Wiki Activity

support long filepaths on win7 + misc windows fixes

Browse source
This commit is contained in:
ed 2023-02-10 18:37:37 +00:00
parent d7f1951e44
commit a4b56c74c7
13 changed files with 179 additions and 84 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
README.md
View file

@ -1350,7 +1350,7 @@ enable [thumbnails](#thumbnails) of...
* **AVIF pictures:** `pyvips` or `ffmpeg` or `pillow-avif-plugin` * **AVIF pictures:** `pyvips` or `ffmpeg` or `pillow-avif-plugin`
* **JPEG XL pictures:** `pyvips` or `ffmpeg` * **JPEG XL pictures:** `pyvips` or `ffmpeg`
enable [smb](#smb-server) support: enable [smb](#smb-server) support (**not** recommended):
* `impacket==0.10.0` * `impacket==0.10.0`
`pyvips` gives higher quality thumbnails than `Pillow` and is 320% faster, using 270% more ram: `sudo apt install libvips42 && python3 -m pip install --user -U pyvips` `pyvips` gives higher quality thumbnails than `Pillow` and is 320% faster, using 270% more ram: `sudo apt install libvips42 && python3 -m pip install --user -U pyvips`

2
bin/mtag/image-noexif.py
View file

@ -61,7 +61,7 @@ def main():
os.chdir(cwd) os.chdir(cwd)
f1 = fsenc(fn) f1 = fsenc(fn)
f2 = os.path.join(b"noexif", f1) f2 = fsenc(os.path.join(b"noexif", fn))
cmd = [ cmd = [
b"exiftool", b"exiftool",
b"-exif:all=", b"-exif:all=",

5
copyparty/__main__.py
View file

@ -36,8 +36,10 @@ from .util import (
UNPLICATIONS, UNPLICATIONS,
align_tab, align_tab,
ansi_re, ansi_re,
is_exe,
min_ex, min_ex,
py_desc, py_desc,
pybin,
termsize, termsize,
wrap, wrap,
) )
@ -1065,6 +1067,9 @@ def main(argv: Optional[list[str]] = None) -> None:
showlic() showlic()
sys.exit(0) sys.exit(0)
if is_exe:
print("pybin: {}\n".format(pybin), end="")
ensure_locale() ensure_locale()
if HAVE_SSL: if HAVE_SSL:
ensure_cert() ensure_cert()

8
copyparty/authsrv.py
View file

@ -14,7 +14,7 @@ from datetime import datetime
from .__init__ import ANYWIN, TYPE_CHECKING, WINDOWS from .__init__ import ANYWIN, TYPE_CHECKING, WINDOWS
from .bos import bos from .bos import bos
from .cfg import vf_bmap, vf_vmap, vf_cmap, onedash, flagdescs, permdescs from .cfg import vf_bmap, vf_vmap, vf_cmap, flagdescs, permdescs
from .util import ( from .util import (
IMPLICATIONS, IMPLICATIONS,
META_NOBOTS, META_NOBOTS,
@ -22,7 +22,7 @@ from .util import (
UNPLICATIONS, UNPLICATIONS,
Pebkac, Pebkac,
absreal, absreal,
fsenc, afsenc,
get_df, get_df,
humansize, humansize,
relchk, relchk,
@ -1083,7 +1083,7 @@ class AuthSrv(object):
promote = [] promote = []
demote = [] demote = []
for vol in vfs.all_vols.values(): for vol in vfs.all_vols.values():
zb = hashlib.sha512(fsenc(vol.realpath)).digest() zb = hashlib.sha512(afsenc(vol.realpath)).digest()
hid = base64.b32encode(zb).decode("ascii").lower() hid = base64.b32encode(zb).decode("ascii").lower()
vflag = vol.flags.get("hist") vflag = vol.flags.get("hist")
if vflag == "-": if vflag == "-":
@ -1102,7 +1102,7 @@ class AuthSrv(object):
except: except:
owner = None owner = None
me = fsenc(vol.realpath).rstrip() me = afsenc(vol.realpath).rstrip()
if owner not in [None, me]: if owner not in [None, me]:
continue continue

18
copyparty/ftpd.py
View file

@ -21,6 +21,7 @@ from .util import (
exclude_dotfiles, exclude_dotfiles,
fsenc, fsenc,
ipnorm, ipnorm,
pybin,
relchk, relchk,
sanitize_fn, sanitize_fn,
vjoin, vjoin,
@ -135,7 +136,8 @@ class FtpFs(AbstractedFS):
try: try:
vpath = vpath.replace("\\", "/").lstrip("/") vpath = vpath.replace("\\", "/").lstrip("/")
rd, fn = os.path.split(vpath) rd, fn = os.path.split(vpath)
if ANYWIN and not relchk(rd): if ANYWIN and relchk(rd):
logging.warning("malicious vpath: %s", vpath)
raise FilesystemError("unsupported characters in filepath") raise FilesystemError("unsupported characters in filepath")
fn = sanitize_fn(fn or "", "", [".prologue.html", ".epilogue.html"]) fn = sanitize_fn(fn or "", "", [".prologue.html", ".epilogue.html"])
@ -417,7 +419,7 @@ class Ftpd(object):
h1 = SftpHandler h1 = SftpHandler
except: except:
t = "\nftps requires pyopenssl;\nplease run the following:\n\n {} -m pip install --user pyopenssl\n" t = "\nftps requires pyopenssl;\nplease run the following:\n\n {} -m pip install --user pyopenssl\n"
print(t.format(sys.executable)) print(t.format(pybin))
sys.exit(1) sys.exit(1)
h1.certfile = os.path.join(self.args.E.cfg, "cert.pem") h1.certfile = os.path.join(self.args.E.cfg, "cert.pem")
@ -450,10 +452,18 @@ class Ftpd(object):
lgr = logging.getLogger("pyftpdlib") lgr = logging.getLogger("pyftpdlib")
lgr.setLevel(logging.DEBUG if self.args.ftpv else logging.INFO) lgr.setLevel(logging.DEBUG if self.args.ftpv else logging.INFO)
ips = self.args.i
if "::" in ips:
ips.append("0.0.0.0")
ioloop = IOLoop() ioloop = IOLoop()
for ip in self.args.i: for ip in ips:
for h, lp in hs: for h, lp in hs:
FTPServer((ip, int(lp)), h, ioloop) try:
FTPServer((ip, int(lp)), h, ioloop)
except:
if ip != "0.0.0.0" or "::" not in ips:
raise
Daemon(ioloop.loop, "ftp") Daemon(ioloop.loop, "ftp")

30
copyparty/mtag.py
View file

@ -10,7 +10,17 @@ import sys
from .__init__ import PY2, WINDOWS, E, unicode from .__init__ import PY2, WINDOWS, E, unicode
from .bos import bos from .bos import bos
from .util import REKOBO_LKEY, fsenc, min_ex, retchk, runcmd, uncyg from .util import (
REKOBO_LKEY,
sfsenc,
fsenc,
is_exe,
min_ex,
pybin,
retchk,
runcmd,
uncyg,
)
if True: # pylint: disable=using-constant-test if True: # pylint: disable=using-constant-test
from typing import Any, Union from typing import Any, Union
@ -285,9 +295,14 @@ class MTag(object):
self.log(msg, c=3) self.log(msg, c=3)
if not self.usable: if not self.usable:
if is_exe:
t = "need ffmpeg to read media tags; copyparty.exe cannot use mutagen"
self.log(t)
return
msg = "need Mutagen{} to read media tags so please run this:\n{}{} -m pip install --user mutagen\n" msg = "need Mutagen{} to read media tags so please run this:\n{}{} -m pip install --user mutagen\n"
pybin = os.path.basename(sys.executable) pyname = os.path.basename(pybin)
self.log(msg.format(or_ffprobe, " " * 37, pybin), c=1) self.log(msg.format(or_ffprobe, " " * 37, pyname), c=1)
return return
# https://picard-docs.musicbrainz.org/downloads/MusicBrainz_Picard_Tag_Map.html # https://picard-docs.musicbrainz.org/downloads/MusicBrainz_Picard_Tag_Map.html
@ -519,12 +534,15 @@ class MTag(object):
env = os.environ.copy() env = os.environ.copy()
try: try:
if is_exe:
raise Exception()
pypath = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) pypath = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
zsl = [str(pypath)] + [str(x) for x in sys.path if x] zsl = [str(pypath)] + [str(x) for x in sys.path if x]
pypath = str(os.pathsep.join(zsl)) pypath = str(os.pathsep.join(zsl))
env["PYTHONPATH"] = pypath env["PYTHONPATH"] = pypath
except: except:
if not E.ox: if not E.ox and not is_exe:
raise raise
ret: dict[str, Any] = {} ret: dict[str, Any] = {}
@ -532,7 +550,7 @@ class MTag(object):
try: try:
cmd = [parser.bin, abspath] cmd = [parser.bin, abspath]
if parser.bin.endswith(".py"): if parser.bin.endswith(".py"):
cmd = [sys.executable] + cmd cmd = [pybin] + cmd
args = { args = {
"env": env, "env": env,
@ -551,7 +569,7 @@ class MTag(object):
else: else:
cmd = ["nice"] + cmd cmd = ["nice"] + cmd
bcmd = [fsenc(x) for x in cmd] bcmd = [sfsenc(x) for x in cmd[:-1]] + [fsenc(cmd[-1])]
rc, v, err = runcmd(bcmd, **args) # type: ignore rc, v, err = runcmd(bcmd, **args) # type: ignore
retchk(rc, bcmd, err, self.log, 5, self.args.mtag_v) retchk(rc, bcmd, err, self.log, 5, self.args.mtag_v)
v = v.strip() v = v.strip()

8
copyparty/smbd.py
View file

@ -12,7 +12,7 @@ from types import SimpleNamespace
from .__init__ import ANYWIN, TYPE_CHECKING from .__init__ import ANYWIN, TYPE_CHECKING
from .authsrv import LEELOO_DALLAS, VFS from .authsrv import LEELOO_DALLAS, VFS
from .bos import bos from .bos import bos
from .util import Daemon, min_ex from .util import Daemon, is_exe, min_ex, pybin
if True: # pylint: disable=using-constant-test if True: # pylint: disable=using-constant-test
from typing import Any from typing import Any
@ -42,8 +42,12 @@ class SMB(object):
from impacket import smbserver from impacket import smbserver
from impacket.ntlm import compute_lmhash, compute_nthash from impacket.ntlm import compute_lmhash, compute_nthash
except ImportError: except ImportError:
if is_exe:
print("copyparty.exe cannot do SMB")
sys.exit(1)
m = "\033[36m\n{}\033[31m\n\nERROR: need 'impacket'; please run this command:\033[33m\n {} -m pip install --user impacket\n\033[0m" m = "\033[36m\n{}\033[31m\n\nERROR: need 'impacket'; please run this command:\033[33m\n {} -m pip install --user impacket\n\033[0m"
print(m.format(min_ex(), sys.executable)) print(m.format(min_ex(), pybin))
sys.exit(1) sys.exit(1)
# patch vfs into smbserver.os # patch vfs into smbserver.os

5
copyparty/svchub.py
View file

@ -44,6 +44,7 @@ from .util import (
ansi_re, ansi_re,
min_ex, min_ex,
mp, mp,
pybin,
start_log_thrs, start_log_thrs,
start_stackmon, start_stackmon,
) )
@ -206,7 +207,7 @@ class SvcHub(object):
self.thumbsrv = ThumbSrv(self) self.thumbsrv = ThumbSrv(self)
else: else:
msg = "need either Pillow, pyvips, or FFmpeg to create thumbnails; for example:\n{0}{1} -m pip install --user Pillow\n{0}{1} -m pip install --user pyvips\n{0}apt install ffmpeg" msg = "need either Pillow, pyvips, or FFmpeg to create thumbnails; for example:\n{0}{1} -m pip install --user Pillow\n{0}{1} -m pip install --user pyvips\n{0}apt install ffmpeg"
msg = msg.format(" " * 37, os.path.basename(sys.executable)) msg = msg.format(" " * 37, os.path.basename(pybin))
self.log("thumb", msg, c=3) self.log("thumb", msg, c=3)
if not args.no_acode and args.no_thumb: if not args.no_acode and args.no_thumb:
@ -413,7 +414,7 @@ class SvcHub(object):
lh = codecs.open(fn, "w", encoding="utf-8", errors="replace") lh = codecs.open(fn, "w", encoding="utf-8", errors="replace")
argv = [sys.executable] + self.argv argv = [pybin] + self.argv
if hasattr(shlex, "quote"): if hasattr(shlex, "quote"):
argv = [shlex.quote(x) for x in argv] argv = [shlex.quote(x) for x in argv]
else: else:

7
copyparty/th_srv.py
View file

@ -20,6 +20,7 @@ from .util import (
Cooldown, Cooldown,
Daemon, Daemon,
Pebkac, Pebkac,
afsenc,
fsenc, fsenc,
min_ex, min_ex,
runcmd, runcmd,
@ -82,14 +83,14 @@ def thumb_path(histpath: str, rem: str, mtime: float, fmt: str) -> str:
# base64 = 64 = 4096 # base64 = 64 = 4096
rd, fn = vsplit(rem) rd, fn = vsplit(rem)
if rd: if rd:
h = hashlib.sha512(fsenc(rd)).digest() h = hashlib.sha512(afsenc(rd)).digest()
b64 = base64.urlsafe_b64encode(h).decode("ascii")[:24] b64 = base64.urlsafe_b64encode(h).decode("ascii")[:24]
rd = "{}/{}/".format(b64[:2], b64[2:4]).lower() + b64 rd = "{}/{}/".format(b64[:2], b64[2:4]).lower() + b64
else: else:
rd = "top" rd = "top"
# could keep original filenames but this is safer re pathlen # could keep original filenames but this is safer re pathlen
h = hashlib.sha512(fsenc(fn)).digest() h = hashlib.sha512(afsenc(fn)).digest()
fn = base64.urlsafe_b64encode(h).decode("ascii")[:24] fn = base64.urlsafe_b64encode(h).decode("ascii")[:24]
if fmt in ("opus", "caf"): if fmt in ("opus", "caf"):
@ -201,7 +202,7 @@ class ThumbSrv(object):
inf_path = os.path.join(thdir, "dir.txt") inf_path = os.path.join(thdir, "dir.txt")
if not bos.path.exists(inf_path): if not bos.path.exists(inf_path):
with open(inf_path, "wb") as f: with open(inf_path, "wb") as f:
f.write(fsenc(os.path.dirname(abspath))) f.write(afsenc(os.path.dirname(abspath)))
self.busy[tpath] = [cond] self.busy[tpath] = [cond]
do_conv = True do_conv = True

71
copyparty/up2k.py
View file

@ -37,6 +37,7 @@ from .util import (
atomic_move, atomic_move,
db_ex_chk, db_ex_chk,
djoin, djoin,
sfsenc,
fsenc, fsenc,
gen_filekey, gen_filekey,
gen_filekey_dbg, gen_filekey_dbg,
@ -395,7 +396,7 @@ class Up2k(object):
def _vis_job_progress(self, job: dict[str, Any]) -> str: def _vis_job_progress(self, job: dict[str, Any]) -> str:
perc = 100 - (len(job["need"]) * 100.0 / len(job["hash"])) perc = 100 - (len(job["need"]) * 100.0 / len(job["hash"]))
path = os.path.join(job["ptop"], job["prel"], job["name"]) path = djoin(job["ptop"], job["prel"], job["name"])
return "{:5.1f}% {}".format(perc, path) return "{:5.1f}% {}".format(perc, path)
def _vis_reg_progress(self, reg: dict[str, dict[str, Any]]) -> list[str]: def _vis_reg_progress(self, reg: dict[str, dict[str, Any]]) -> list[str]:
@ -691,7 +692,7 @@ class Up2k(object):
pass pass
for k, job in reg2.items(): for k, job in reg2.items():
path = os.path.join(job["ptop"], job["prel"], job["name"]) path = djoin(job["ptop"], job["prel"], job["name"])
if bos.path.exists(path): if bos.path.exists(path):
reg[k] = job reg[k] = job
job["poke"] = time.time() job["poke"] = time.time()
@ -1086,7 +1087,7 @@ class Up2k(object):
else: else:
rd = drd rd = drd
abspath = os.path.join(top, rd) abspath = djoin(top, rd)
self.pp.msg = "b{} {}".format(ndirs - nchecked, abspath) self.pp.msg = "b{} {}".format(ndirs - nchecked, abspath)
try: try:
if os.path.isdir(abspath): if os.path.isdir(abspath):
@ -1127,7 +1128,7 @@ class Up2k(object):
if crd != rd: if crd != rd:
crd = rd crd = rd
try: try:
cdc = set(os.listdir(os.path.join(top, rd))) cdc = set(os.listdir(djoin(top, rd)))
except: except:
cdc.clear() cdc.clear()
@ -1203,7 +1204,7 @@ class Up2k(object):
rd = drd rd = drd
fn = dfn fn = dfn
abspath = os.path.join(ptop, rd, fn) abspath = djoin(ptop, rd, fn)
if rei and rei.search(abspath): if rei and rei.search(abspath):
continue continue
@ -1412,7 +1413,7 @@ class Up2k(object):
q = "insert into mt values (?,'t:mtp','a')" q = "insert into mt values (?,'t:mtp','a')"
cur.execute(q, (w[:16],)) cur.execute(q, (w[:16],))
abspath = os.path.join(ptop, rd, fn) abspath = djoin(ptop, rd, fn)
self.pp.msg = "c{} {}".format(nq, abspath) self.pp.msg = "c{} {}".format(nq, abspath)
if not mpool: if not mpool:
n_tags = self._tagscan_file(cur, entags, w, abspath, ip, at) n_tags = self._tagscan_file(cur, entags, w, abspath, ip, at)
@ -1576,7 +1577,7 @@ class Up2k(object):
q = "select rd, fn, ip, at from up where substr(w,1,16)=? limit 1" q = "select rd, fn, ip, at from up where substr(w,1,16)=? limit 1"
rd, fn, ip, at = cur.execute(q, (w,)).fetchone() rd, fn, ip, at = cur.execute(q, (w,)).fetchone()
rd, fn = s3dec(rd, fn) rd, fn = s3dec(rd, fn)
abspath = os.path.join(ptop, rd, fn) abspath = djoin(ptop, rd, fn)
q = "select k from mt where w = ?" q = "select k from mt where w = ?"
zq = cur.execute(q, (w,)).fetchall() zq = cur.execute(q, (w,)).fetchall()
@ -2053,7 +2054,7 @@ class Up2k(object):
if dp_dir.startswith("//") or dp_fn.startswith("//"): if dp_dir.startswith("//") or dp_fn.startswith("//"):
dp_dir, dp_fn = s3dec(dp_dir, dp_fn) dp_dir, dp_fn = s3dec(dp_dir, dp_fn)
dp_abs = "/".join([ptop, dp_dir, dp_fn]) dp_abs = djoin(ptop, dp_dir, dp_fn)
try: try:
st = bos.stat(dp_abs) st = bos.stat(dp_abs)
if stat.S_ISLNK(st.st_mode): if stat.S_ISLNK(st.st_mode):
@ -2128,7 +2129,7 @@ class Up2k(object):
# ensure the files haven't been deleted manually # ensure the files haven't been deleted manually
names = [job[x] for x in ["name", "tnam"] if x in job] names = [job[x] for x in ["name", "tnam"] if x in job]
for fn in names: for fn in names:
path = os.path.join(job["ptop"], job["prel"], fn) path = djoin(job["ptop"], job["prel"], fn)
try: try:
if bos.path.getsize(path) > 0: if bos.path.getsize(path) > 0:
# upload completed or both present # upload completed or both present
@ -2140,9 +2141,9 @@ class Up2k(object):
break break
else: else:
# file contents match, but not the path # file contents match, but not the path
src = os.path.join(job["ptop"], job["prel"], job["name"]) src = djoin(job["ptop"], job["prel"], job["name"])
dst = os.path.join(cj["ptop"], cj["prel"], cj["name"]) dst = djoin(cj["ptop"], cj["prel"], cj["name"])
vsrc = os.path.join(job["vtop"], job["prel"], job["name"]) vsrc = djoin(job["vtop"], job["prel"], job["name"])
vsrc = vsrc.replace("\\", "/") # just for prints anyways vsrc = vsrc.replace("\\", "/") # just for prints anyways
if job["need"]: if job["need"]:
self.log("unfinished:\n {0}\n {1}".format(src, dst)) self.log("unfinished:\n {0}\n {1}".format(src, dst))
@ -2180,7 +2181,7 @@ class Up2k(object):
else: else:
job["name"] = self._untaken(pdir, cj, now) job["name"] = self._untaken(pdir, cj, now)
dst = os.path.join(job["ptop"], job["prel"], job["name"]) dst = djoin(job["ptop"], job["prel"], job["name"])
if not self.args.nw: if not self.args.nw:
try: try:
dvf = self.flags[job["ptop"]] dvf = self.flags[job["ptop"]]
@ -2270,7 +2271,7 @@ class Up2k(object):
and "fk" in vfs.flags and "fk" in vfs.flags
and (cj["user"] in vfs.axs.uread or cj["user"] in vfs.axs.upget) and (cj["user"] in vfs.axs.uread or cj["user"] in vfs.axs.upget)
): ):
ap = absreal(os.path.join(job["ptop"], job["prel"], job["name"])) ap = absreal(djoin(job["ptop"], job["prel"], job["name"]))
ino = 0 if ANYWIN else bos.stat(ap).st_ino ino = 0 if ANYWIN else bos.stat(ap).st_ino
fk = self.gen_fk(self.args.fk_salt, ap, job["size"], ino) fk = self.gen_fk(self.args.fk_salt, ap, job["size"], ino)
ret["fk"] = fk[: vfs.flags["fk"]] ret["fk"] = fk[: vfs.flags["fk"]]
@ -2284,7 +2285,7 @@ class Up2k(object):
if self.args.nw: if self.args.nw:
return fname return fname
fp = os.path.join(fdir, fname) fp = djoin(fdir, fname)
if job.get("replace") and bos.path.exists(fp): if job.get("replace") and bos.path.exists(fp):
self.log("replacing existing file at {}".format(fp)) self.log("replacing existing file at {}".format(fp))
bos.unlink(fp) bos.unlink(fp)
@ -2397,7 +2398,7 @@ class Up2k(object):
t = "that chunk is already being written to:\n {}\n {} {}/{}\n {}" t = "that chunk is already being written to:\n {}\n {} {}/{}\n {}"
raise Pebkac(400, t.format(wark, chash, idx, nh, job["name"])) raise Pebkac(400, t.format(wark, chash, idx, nh, job["name"]))
path = os.path.join(job["ptop"], job["prel"], job["tnam"]) path = djoin(job["ptop"], job["prel"], job["tnam"])
chunksize = up2k_chunksize(job["size"]) chunksize = up2k_chunksize(job["size"])
ofs = [chunksize * x for x in nchunk] ofs = [chunksize * x for x in nchunk]
@ -2428,9 +2429,9 @@ class Up2k(object):
self.db_act = time.time() self.db_act = time.time()
try: try:
job = self.registry[ptop][wark] job = self.registry[ptop][wark]
pdir = os.path.join(job["ptop"], job["prel"]) pdir = djoin(job["ptop"], job["prel"])
src = os.path.join(pdir, job["tnam"]) src = djoin(pdir, job["tnam"])
dst = os.path.join(pdir, job["name"]) dst = djoin(pdir, job["name"])
except Exception as ex: except Exception as ex:
return "confirm_chunk, wark, " + repr(ex) # type: ignore return "confirm_chunk, wark, " + repr(ex) # type: ignore
@ -2463,9 +2464,9 @@ class Up2k(object):
self.db_act = time.time() self.db_act = time.time()
try: try:
job = self.registry[ptop][wark] job = self.registry[ptop][wark]
pdir = os.path.join(job["ptop"], job["prel"]) pdir = djoin(job["ptop"], job["prel"])
src = os.path.join(pdir, job["tnam"]) src = djoin(pdir, job["tnam"])
dst = os.path.join(pdir, job["name"]) dst = djoin(pdir, job["name"])
except Exception as ex: except Exception as ex:
raise Pebkac(500, "finish_upload, wark, " + repr(ex)) raise Pebkac(500, "finish_upload, wark, " + repr(ex))
@ -2532,7 +2533,7 @@ class Up2k(object):
cur = self.cur.get(ptop) cur = self.cur.get(ptop)
for rd, fn, lmod in dupes: for rd, fn, lmod in dupes:
d2 = os.path.join(ptop, rd, fn) d2 = djoin(ptop, rd, fn)
if os.path.exists(d2): if os.path.exists(d2):
continue continue
@ -2710,7 +2711,7 @@ class Up2k(object):
break break
n_files += 1 n_files += 1
abspath = os.path.join(adir, fn) abspath = djoin(adir, fn)
volpath = "{}/{}".format(vrem, fn).strip("/") volpath = "{}/{}".format(vrem, fn).strip("/")
vpath = "{}/{}".format(dbv.vpath, volpath).strip("/") vpath = "{}/{}".format(dbv.vpath, volpath).strip("/")
self.log("rm {}\n {}".format(vpath, abspath)) self.log("rm {}\n {}".format(vpath, abspath))
@ -2989,14 +2990,14 @@ class Up2k(object):
or to first remaining full if no dabs (delete) or to first remaining full if no dabs (delete)
""" """
dupes = [] dupes = []
sabs = os.path.join(sptop, srem) sabs = djoin(sptop, srem)
q = "select rd, fn from up where substr(w,1,16)=? and w=?" q = "select rd, fn from up where substr(w,1,16)=? and w=?"
for ptop, cur in self.cur.items(): for ptop, cur in self.cur.items():
for rd, fn in cur.execute(q, (wark[:16], wark)): for rd, fn in cur.execute(q, (wark[:16], wark)):
if rd.startswith("//") or fn.startswith("//"): if rd.startswith("//") or fn.startswith("//"):
rd, fn = s3dec(rd, fn) rd, fn = s3dec(rd, fn)
dvrem = "/".join([rd, fn]).strip("/") dvrem = vjoin(rd, fn).strip("/")
if ptop != sptop or srem != dvrem: if ptop != sptop or srem != dvrem:
dupes.append([ptop, dvrem]) dupes.append([ptop, dvrem])
self.log("found {} dupe: [{}] {}".format(wark, ptop, dvrem)) self.log("found {} dupe: [{}] {}".format(wark, ptop, dvrem))
@ -3007,7 +3008,7 @@ class Up2k(object):
full: dict[str, tuple[str, str]] = {} full: dict[str, tuple[str, str]] = {}
links: dict[str, tuple[str, str]] = {} links: dict[str, tuple[str, str]] = {}
for ptop, vp in dupes: for ptop, vp in dupes:
ap = os.path.join(ptop, vp) ap = djoin(ptop, vp)
try: try:
d = links if bos.path.islink(ap) else full d = links if bos.path.islink(ap) else full
d[ap] = (ptop, vp) d[ap] = (ptop, vp)
@ -3111,7 +3112,7 @@ class Up2k(object):
def _new_upload(self, job: dict[str, Any]) -> None: def _new_upload(self, job: dict[str, Any]) -> None:
pdir = djoin(job["ptop"], job["prel"]) pdir = djoin(job["ptop"], job["prel"])
if not job["size"] and bos.path.isfile(os.path.join(pdir, job["name"])): if not job["size"] and bos.path.isfile(djoin(pdir, job["name"])):
return return
self.registry[job["ptop"]][job["wark"]] = job self.registry[job["ptop"]][job["wark"]] = job
@ -3156,7 +3157,7 @@ class Up2k(object):
suffix = "-{:.6f}-{}".format(job["t0"], dip) suffix = "-{:.6f}-{}".format(job["t0"], dip)
with ren_open(tnam, "wb", fdir=pdir, suffix=suffix) as zfw: with ren_open(tnam, "wb", fdir=pdir, suffix=suffix) as zfw:
f, job["tnam"] = zfw["orz"] f, job["tnam"] = zfw["orz"]
abspath = os.path.join(pdir, job["tnam"]) abspath = djoin(pdir, job["tnam"])
sprs = job["sprs"] sprs = job["sprs"]
sz = job["size"] sz = job["size"]
relabel = False relabel = False
@ -3255,7 +3256,7 @@ class Up2k(object):
x x
for x in reg.values() for x in reg.values()
if x["need"] if x["need"]
and not bos.path.exists(os.path.join(x["ptop"], x["prel"], x["name"])) and not bos.path.exists(djoin(x["ptop"], x["prel"], x["name"]))
] ]
if rm or lost: if rm or lost:
@ -3268,7 +3269,7 @@ class Up2k(object):
del reg[job["wark"]] del reg[job["wark"]]
try: try:
# remove the filename reservation # remove the filename reservation
path = os.path.join(job["ptop"], job["prel"], job["name"]) path = djoin(job["ptop"], job["prel"], job["name"])
if bos.path.getsize(path) == 0: if bos.path.getsize(path) == 0:
bos.unlink(path) bos.unlink(path)
except: except:
@ -3277,7 +3278,7 @@ class Up2k(object):
try: try:
if len(job["hash"]) == len(job["need"]): if len(job["hash"]) == len(job["need"]):
# PARTIAL is empty, delete that too # PARTIAL is empty, delete that too
path = os.path.join(job["ptop"], job["prel"], job["tnam"]) path = djoin(job["ptop"], job["prel"], job["tnam"])
bos.unlink(path) bos.unlink(path)
except: except:
pass pass
@ -3326,7 +3327,7 @@ class Up2k(object):
continue continue
# self.log("\n " + repr([ptop, rd, fn])) # self.log("\n " + repr([ptop, rd, fn]))
abspath = os.path.join(ptop, rd, fn) abspath = djoin(ptop, rd, fn)
try: try:
tags = self.mtag.get(abspath) tags = self.mtag.get(abspath)
ntags1 = len(tags) ntags1 = len(tags)
@ -3376,7 +3377,7 @@ class Up2k(object):
if "e2d" not in self.flags[ptop]: if "e2d" not in self.flags[ptop]:
continue continue
abspath = os.path.join(ptop, rd, fn) abspath = djoin(ptop, rd, fn)
self.log("hashing " + abspath) self.log("hashing " + abspath)
inf = bos.stat(abspath) inf = bos.stat(abspath)
if not inf.st_size: if not inf.st_size:
@ -3455,6 +3456,6 @@ def up2k_wark_from_hashlist(salt: str, filesize: int, hashes: list[str]) -> str:
def up2k_wark_from_metadata(salt: str, sz: int, lastmod: int, rd: str, fn: str) -> str: def up2k_wark_from_metadata(salt: str, sz: int, lastmod: int, rd: str, fn: str) -> str:
ret = fsenc("{}\n{}\n{}\n{}\n{}".format(salt, lastmod, sz, rd, fn)) ret = sfsenc("{}\n{}\n{}\n{}\n{}".format(salt, lastmod, sz, rd, fn))
ret = base64.urlsafe_b64encode(hashlib.sha512(ret).digest()) ret = base64.urlsafe_b64encode(hashlib.sha512(ret).digest())
return "#{}".format(ret.decode("ascii"))[:44] return "#{}".format(ret.decode("ascii"))[:44]

67
copyparty/util.py
View file

@ -14,6 +14,7 @@ import os
import platform import platform
import re import re
import select import select
import shutil
import signal import signal
import socket import socket
import stat import stat
@ -290,6 +291,20 @@ REKOBO_KEY = {
REKOBO_LKEY = {k.lower(): v for k, v in REKOBO_KEY.items()} REKOBO_LKEY = {k.lower(): v for k, v in REKOBO_KEY.items()}
pybin = sys.executable or ""
is_exe = bool(getattr(sys, "frozen", False))
if is_exe:
pybin = ""
for p in "python3 python".split():
try:
p = shutil.which(p)
if p:
pybin = p
break
except:
pass
def py_desc() -> str: def py_desc() -> str:
interp = platform.python_implementation() interp = platform.python_implementation()
py_ver = ".".join([str(x) for x in sys.version_info]) py_ver = ".".join([str(x) for x in sys.version_info])
@ -1704,7 +1719,7 @@ def relchk(rp: str) -> str:
def absreal(fpath: str) -> str: def absreal(fpath: str) -> str:
try: try:
return fsdec(os.path.abspath(os.path.realpath(fsenc(fpath)))) return fsdec(os.path.abspath(os.path.realpath(afsenc(fpath))))
except: except:
if not WINDOWS: if not WINDOWS:
raise raise
@ -1824,6 +1839,24 @@ def _w8enc3(txt: str) -> bytes:
return txt.encode(FS_ENCODING, "surrogateescape") return txt.encode(FS_ENCODING, "surrogateescape")
def _msdec(txt: bytes) -> str:
ret = txt.decode(FS_ENCODING, "surrogateescape")
return ret[4:] if ret.startswith("\\\\?\\") else ret
def _msaenc(txt: str) -> bytes:
return txt.replace("/", "\\").encode(FS_ENCODING, "surrogateescape")
def _msenc(txt: str) -> bytes:
txt = txt.replace("/", "\\")
if ":" not in txt and not txt.startswith("\\\\"):
txt = absreal(txt)
ret = txt.encode(FS_ENCODING, "surrogateescape")
return ret if ret.startswith(b"\\\\?\\") else b"\\\\?\\" + ret
w8dec = _w8dec3 if not PY2 else _w8dec2 w8dec = _w8dec3 if not PY2 else _w8dec2
w8enc = _w8enc3 if not PY2 else _w8enc2 w8enc = _w8enc3 if not PY2 else _w8enc2
@ -1838,8 +1871,13 @@ def w8b64enc(txt: str) -> str:
return base64.urlsafe_b64encode(w8enc(txt)).decode("ascii") return base64.urlsafe_b64encode(w8enc(txt)).decode("ascii")
if not PY2 or not WINDOWS: if not PY2 and WINDOWS:
fsenc = w8enc sfsenc = w8enc
afsenc = _msaenc
fsenc = _msenc
fsdec = _msdec
elif not PY2 or not WINDOWS:
fsenc = afsenc = sfsenc = w8enc
fsdec = w8dec fsdec = w8dec
else: else:
# moonrunes become \x3f with bytestrings, # moonrunes become \x3f with bytestrings,
@ -1850,7 +1888,7 @@ else:
def _not_actually_mbcs_dec(txt: bytes) -> str: def _not_actually_mbcs_dec(txt: bytes) -> str:
return txt return txt
fsenc = _not_actually_mbcs_enc fsenc = afsenc = sfsenc = _not_actually_mbcs_enc
fsdec = _not_actually_mbcs_dec fsdec = _not_actually_mbcs_dec
@ -2514,12 +2552,17 @@ def _runhook(
log(t.format(arg, ocmd)) log(t.format(arg, ocmd))
env = os.environ.copy() env = os.environ.copy()
# try: try:
pypath = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) if is_exe:
zsl = [str(pypath)] + [str(x) for x in sys.path if x] raise Exception()
pypath = str(os.pathsep.join(zsl))
env["PYTHONPATH"] = pypath pypath = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
# except: if not E.ox: raise zsl = [str(pypath)] + [str(x) for x in sys.path if x]
pypath = str(os.pathsep.join(zsl))
env["PYTHONPATH"] = pypath
except:
if not is_exe:
raise
ka = { ka = {
"env": env, "env": env,
@ -2545,9 +2588,9 @@ def _runhook(
acmd = [cmd, arg] acmd = [cmd, arg]
if cmd.endswith(".py"): if cmd.endswith(".py"):
acmd = [sys.executable] + acmd acmd = [pybin] + acmd
bcmd = [fsenc(x) for x in acmd] bcmd = [fsenc(x) if x == ap else sfsenc(x) for x in acmd]
t0 = time.time() t0 = time.time()
if fork: if fork:

38
scripts/pyinstaller/build.sh
View file

@ -44,21 +44,33 @@ read a b c d _ < <(
) )
sed -r 's/1,2,3,0/'$a,$b,$c,$d'/;s/1\.2\.3/'$a.$b.$c/ <loader.rc >loader.rc2 sed -r 's/1,2,3,0/'$a,$b,$c,$d'/;s/1\.2\.3/'$a.$b.$c/ <loader.rc >loader.rc2
excl=(
copyparty.broker_mp
copyparty.broker_mpw
ctypes.macholib
curses
inspect
multiprocessing
pdb
pickle
pyftpdlib.prefork
urllib.request
urllib.response
urllib.robotparser
zipfile
)
false || excl+=(
PIL
PIL.ExifTags
PIL.Image
PIL.ImageDraw
PIL.ImageOps
)
excl=( "${excl[@]/#/--exclude-module }" )
$APPDATA/python/python37/scripts/pyinstaller \ $APPDATA/python/python37/scripts/pyinstaller \
-y --clean -p mods --upx-dir=. \ -y --clean -p mods --upx-dir=. \
--exclude-module copyparty.broker_mp \ ${excl[*]} \
--exclude-module copyparty.broker_mpw \
--exclude-module curses \
--exclude-module ctypes.macholib \
--exclude-module inspect \
--exclude-module multiprocessing \
--exclude-module pdb \
--exclude-module pickle \
--exclude-module pyftpdlib.prefork \
--exclude-module urllib.request \
--exclude-module urllib.response \
--exclude-module urllib.robotparser \
--exclude-module zipfile \
--version-file loader.rc2 -i loader.ico -n copyparty -c -F loader.py \ --version-file loader.rc2 -i loader.ico -n copyparty -c -F loader.py \
--add-data 'mods/copyparty/res;copyparty/res' \ --add-data 'mods/copyparty/res;copyparty/res' \
--add-data 'mods/copyparty/web;copyparty/web' --add-data 'mods/copyparty/web;copyparty/web'

2
tests/util.py
View file

@ -98,7 +98,7 @@ class Cfg(Namespace):
def __init__(self, a=None, v=None, c=None): def __init__(self, a=None, v=None, c=None):
ka = {} ka = {}
ex = "daw dav_inf dav_mac dotsrch e2d e2ds e2dsa e2t e2ts e2tsr e2v e2vu e2vp ed emp force_js getmod hardlink ihead magic never_symlink nid nih no_acode no_athumb no_dav no_dedup no_del no_dupe no_logues no_mv no_readme no_robots no_sb_md no_sb_lg no_scandir no_thumb no_vthumb no_zip nw xdev xlink xvol" ex = "daw dav_inf dav_mac dotsrch e2d e2ds e2dsa e2t e2ts e2tsr e2v e2vu e2vp ed emp force_js getmod hardlink ihead magic never_symlink nid nih no_acode no_athumb no_dav no_dedup no_del no_dupe no_logues no_mv no_readme no_robots no_sb_md no_sb_lg no_scandir no_thumb no_vthumb no_zip nrand nw rand vc xdev xlink xvol"
ka.update(**{k: False for k in ex.split()}) ka.update(**{k: False for k in ex.split()})
ex = "dotpart no_rescan no_sendfile no_voldump plain_ip" ex = "dotpart no_rescan no_sendfile no_voldump plain_ip"