readme: add help links

in addition to write-perms, also drop move-perms from ramdisks
since that is another potential source for confusion

additionally, write-access was correctly prevented, but
the ui would still indicate write permission, so fix that too

.github/ISSUE_TEMPLATE/bug_report.md

@@ -7,34 +7,48 @@ assignees: '9001'
 
 ---
 
-NOTE:
-all of the below are optional, consider them as inspiration, delete and rewrite at will, thx md
+<!-- NOTE:
+**please use english, or include an english translation.** aside from that,  
+all of the below are optional, consider them as inspiration, delete and rewrite at will, thx md -->
 
-
-**Describe the bug**
+### Describe the bug
 a description of what the bug is
 
-**To Reproduce**
+### To Reproduce
 List of steps to reproduce the issue, or, if it's hard to reproduce, then at least a detailed explanation of what you did to run into it
 
-**Expected behavior**
+### Expected behavior
 a description of what you expected to happen
 
-**Screenshots**
+### Screenshots
 if applicable, add screenshots to help explain your problem, such as the kickass crashpage :^)
 
-**Server details**
-if the issue is possibly on the server-side, then mention some of the following:
-* server OS / version: 
-* python version: 
-* copyparty arguments: 
-* filesystem (`lsblk -f` on linux): 
+### Server details (if you are using docker/podman)
+remove the ones that are not relevant:
+* **server OS / version:** 
+* **how you're running copyparty:** (docker/podman/something-else)
+* **docker image:** (variant, version, and arch if you know)
+* **copyparty arguments and/or config-file:** 
 
-**Client details**
+### Server details (if you're NOT using docker/podman)
+remove the ones that are not relevant:
+* **server OS / version:** 
+* **what copyparty did you grab:** (sfx/exe/pip/arch/...)
+* **how you're running it:** (in a terminal, as a systemd-service, ...)
+* run copyparty with `--version` and grab the last 3 lines (they start with `copyparty`, `CPython`, `sqlite`) and paste them below this line:
+* **copyparty arguments and/or config-file:** 
+
+### Client details
 if the issue is possibly on the client-side, then mention some of the following:
 * the device type and model: 
 * OS version: 
 * browser version: 
 
-**Additional context**
+### The rest of the stack
+if you are connecting directly to copyparty then that's cool, otherwise please mention everything else between copyparty and the browser (reverseproxy, tunnels, etc.)
+
+### Server log
+if the issue might be server-related, include everything that appears in the copyparty log during startup, and also anything else you think might be relevant
+
+### Additional context
 any other context about the problem here

.github/ISSUE_TEMPLATE/feature_request.md

@@ -7,7 +7,9 @@ assignees: '9001'
 
 ---
 
-all of the below are optional, consider them as inspiration, delete and rewrite at will
+<!-- NOTE:
+**please use english, or include an english translation.** aside from that,  
+all of the below are optional, consider them as inspiration, delete and rewrite at will -->
 
 **is your feature request related to a problem? Please describe.**
 a description of what the problem is, for example, `I'm always frustrated when [...]` or `Why is it not possible to [...]`

.gitignore

@@ -43,3 +43,7 @@ scripts/docker/*.err
 
 # nix build output link
 result
+result-*
+
+# IDEA config
+.idea/

.vscode/launch.json

@@ -3,7 +3,7 @@
     "configurations": [
         {
             "name": "Run copyparty",
-            "type": "python",
+            "type": "debugpy",
             "request": "launch",
             "module": "copyparty",
             "console": "integratedTerminal",
@@ -11,30 +11,46 @@
             "justMyCode": false,
             "env": {
                 "PYDEVD_DISABLE_FILE_VALIDATION": "1",
-                "PYTHONWARNINGS": "always", //error
+                "PYTHONWARNINGS": "always"  //error
             },
             "args": [
-                //"-nw",
-                "-ed",
-                "-emp",
+                //"-nw",  // no-write; for testing uploads without writing to disk
+                //"-q",  // quiet; speedboost when console output is not needed
+
+                // # increase debugger performance:
+                //"no-htp",
+                //"hash-mt=0",
+                //"mtag-mt=1",
+                //"th-mt=1",
+
+                // # listen for FTP and TFTP
+                "--ftp=3921",
+                "--ftp-pr=12000-12099",
+                "--tftp=3969",
+
+                // # listen on all IPv6, all IPv4, and unix-socket
+                "-i::,unix:777:a.sock",
+
+                // # misc
+                "--dedup",
                 "-e2dsa",
                 "-e2ts",
-                "-mtp=.bpm=f,bin/mtag/audio-bpm.py",
+                "--rss",
+                "--shr=/shr",
+                "--stats",
+                "-z",
+
+                // # users + volumes
                 "-aed:wark",
-                "-vsrv::r:rw,ed:c,dupe",
-                "-vdist:dist:r"
+                "-vdist:dist:r",
+                "-vsrv::r:rw,ed",
+                "-vsrv/junk:junk:r:A,ed",
+                "--ver"
             ]
         },
-        {
-            "name": "No debug",
-            "preLaunchTask": "no_dbg",
-            "type": "python",
-            //"request": "attach", "port": 42069
-            // fork: nc -l 42069 </dev/null
-        },
         {
             "name": "Run active unit test",
-            "type": "python",
+            "type": "debugpy",
             "request": "launch",
             "module": "unittest",
             "console": "integratedTerminal",
@@ -51,6 +67,6 @@
             "program": "${file}",
             "console": "integratedTerminal",
             "justMyCode": false
-        },
+        }
     ]
 }

CONTRIBUTING.md

@@ -1,8 +1,21 @@
-* do something cool
+* **found a bug?** [create an issue!](https://github.com/9001/copyparty/issues) or let me know in the [discord](https://discord.gg/25J8CdTT6G) :>
+* **fixed a bug?** create a PR or post a patch! big thx in advance :>
+* **have a cool idea?** let's discuss it! anywhere's fine, you choose.
 
-really tho, send a PR or an issue or whatever, all appreciated, anything goes, just behave aight 👍👍
+but please:
+
+
+
+# do not use AI / LLM when writing code
+
+copyparty is 100% organic, free-range, human-written software!
+
+> ⚠ you are now entering a no-copilot zone
+
+the *only* place where LLM/AI *may* be accepted is for [localization](https://github.com/9001/copyparty/tree/hovudstraum/docs/rice#translations) if you are fluent and have confirmed that the translation is accurate.
+
+sorry for the harsh tone, but this is important to me 🙏
 
-but to be more specific,
 
 
 # contribution ideas
@@ -26,7 +39,9 @@ if you wanna have a go at coding it up yourself then maybe mention the idea on d
 
 aside from documentation and ideas, some other things that would be cool to have some help with is:
 
-* **translations** -- the copyparty web-UI has translations for english and norwegian at the top of [browser.js](https://github.com/9001/copyparty/blob/hovudstraum/copyparty/web/browser.js); if you'd like to add a translation for another language then that'd be welcome! and if that language has a grammar that doesn't fit into the way the strings are assembled, then we'll fix that as we go :>
+* **translations** -- the copyparty web-UI has translations in [copyparty/web/tl](https://github.com/9001/copyparty/tree/hovudstraum/copyparty/web/tl); if you'd like to [add a translation](https://github.com/9001/copyparty/tree/hovudstraum/docs/rice#translations) for another language then that'd be welcome! and if that language has a grammar that doesn't fit into the way the strings are assembled, then we'll fix that as we go :>
+
+  * but please note that support for [RTL (Right-to-Left) languages](https://en.wikipedia.org/wiki/Right-to-left_script) is currently not planned, since the javascript is a bit too jank for that
 
 * **UI ideas** -- at some point I was thinking of rewriting the UI in react/preact/something-not-vanilla-javascript, but I'll admit the comfiness of not having any build stage combined with raw performance has kinda convinced me otherwise :p but I'd be very open to ideas on how the UI could be improved, or be more intuitive.
 

SECURITY.md

@@ -1,9 +1,7 @@
 # Security Policy
 
-if you hit something extra juicy pls let me know on either of the following
+if you hit something extra juicy pls let me know on one of the following:
 * email -- `copyparty@ocv.ze` except `ze` should be `me`
-* [mastodon dm](https://layer8.space/@tripflag) -- `@tripflag@layer8.space`
 * [github private vulnerability report](https://github.com/9001/copyparty/security/advisories/new), wow that form is complicated
-* [twitter dm](https://twitter.com/tripflag) (if im somehow not banned yet)
 
 no bug bounties sorry! all i can offer is greetz in the release notes

bin/README.md

@@ -1,7 +1,7 @@
 # [`u2c.py`](u2c.py)
 * command-line up2k client [(webm)](https://ocv.me/stuff/u2cli.webm)
 * file uploads, file-search, autoresume of aborted/broken uploads
-* sync local folder to server
+* [sync local folder to server](https://github.com/9001/copyparty/#folder-sync)
 * generally faster than browsers
 * if something breaks just restart it
 
@@ -78,3 +78,6 @@ cd /mnt/nas/music/.hist
 # [`prisonparty.sh`](prisonparty.sh)
 * run copyparty in a chroot, preventing any accidental file access
 * creates bindmounts for /bin, /lib, and so on, see `sysdirs=`
+
+# [`bubbleparty.sh`](bubbleparty.sh)
+* run copyparty in an isolated process, preventing any accidental file access and more

bin/bubbleparty.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+# usage: ./bubbleparty.sh ./copyparty-sfx.py ....
+bwrap \
+  --unshare-all \
+  --ro-bind /usr /usr \
+  --ro-bind /bin /bin \
+  --ro-bind /lib /lib \
+  --ro-bind /etc/resolv.conf /etc/resolv.conf \
+  --dev-bind /dev /dev \
+  --dir /tmp \
+  --dir /var \
+  --bind "$(pwd)" "$(pwd)" \
+  --share-net \
+  --die-with-parent \
+  --file 11 /etc/passwd \
+  --file 12 /etc/group \
+  "$@" \
+  11< <(getent passwd $(id -u) 65534) \
+  12< <(getent group $(id -g) 65534) 

bin/dbtool.py

@@ -8,7 +8,7 @@ import sqlite3
 import argparse
 
 DB_VER1 = 3
-DB_VER2 = 5
+DB_VER2 = 6
 
 BY_PATH = None
 NC = None
@@ -39,7 +39,7 @@ def ls(db):
     print(f"{nfiles} files")
     print(f"{ntags} tags\n")
 
-    print("number of occurences for each tag,")
+    print("number of occurrences for each tag,")
     print(" 'x' = file has no tags")
     print(" 't:mtp' = the mtp flag (file not mtp processed yet)")
     print()

bin/handlers/README.md

@@ -20,6 +20,8 @@ each plugin must define a `main()` which takes 3 arguments;
 
 ## on404
 
+* [redirect.py](redirect.py) sends an HTTP 301 or 302, redirecting the client to another page/file
+* [randpic.py](randpic.py) redirects `/foo/bar/randpic.jpg` to a random pic in `/foo/bar/`
 * [sorry.py](answer.py) replies with a custom message instead of the usual 404
 * [nooo.py](nooo.py) replies with an endless noooooooooooooo
 * [never404.py](never404.py) 100% guarantee that 404 will never be a thing again as it automatically creates dummy files whenever necessary

bin/handlers/randpic.py

@@ -0,0 +1,35 @@
+import os
+import random
+from urllib.parse import quote
+
+
+# assuming /foo/bar/ is a valid URL but /foo/bar/randpic.png does not exist,
+# hijack the 404 with a redirect to a random pic in that folder
+#
+# thx to lia & kipu for the idea
+
+
+def main(cli, vn, rem):
+    req_fn = rem.split("/")[-1]
+    if not cli.can_read or not req_fn.startswith("randpic"):
+        return
+
+    req_abspath = vn.canonical(rem)
+    req_ap_dir = os.path.dirname(req_abspath)
+    files_in_dir = os.listdir(req_ap_dir)
+
+    if "." in req_fn:
+        file_ext = "." + req_fn.split(".")[-1]
+        files_in_dir = [x for x in files_in_dir if x.lower().endswith(file_ext)]
+
+    if not files_in_dir:
+        return
+
+    selected_file = random.choice(files_in_dir)
+
+    req_url = "/".join([vn.vpath, rem]).strip("/")
+    req_dir = req_url.rsplit("/", 1)[0]
+    new_url = "/".join([req_dir, quote(selected_file)]).strip("/")
+
+    cli.reply(b"redirecting...", 302, headers={"Location": "/" + new_url})
+    return "true"

bin/handlers/redirect.py

@@ -0,0 +1,52 @@
+# if someone hits a 404, redirect them to another location
+
+
+def send_http_302_temporary_redirect(cli, new_path):
+    """
+    replies with an HTTP 302, which is a temporary redirect;
+    "new_path" can be any of the following:
+      - "http://a.com/" would redirect to another website,
+      - "/foo/bar" would redirect to /foo/bar on the same server;
+          note the leading '/' in the location which is important
+    """
+    cli.reply(b"redirecting...", 302, headers={"Location": new_path})
+
+
+def send_http_301_permanent_redirect(cli, new_path):
+    """
+    replies with an HTTP 301, which is a permanent redirect;
+    otherwise identical to send_http_302_temporary_redirect
+    """
+    cli.reply(b"redirecting...", 301, headers={"Location": new_path})
+
+
+def send_errorpage_with_redirect_link(cli, new_path):
+    """
+    replies with a website explaining that the page has moved;
+    "new_path" must be an absolute location on the same server
+    but without a leading '/', so for example "foo/bar"
+    would redirect to "/foo/bar"
+    """
+    cli.redirect(new_path, click=False, msg="this page has moved")
+
+
+def main(cli, vn, rem):
+    """
+    this is the function that gets called by copyparty;
+    note that vn.vpath and cli.vpath does not have a leading '/'
+    so we're adding the slash in the debug messages below
+    """
+    print(f"this client just hit a 404: {cli.ip}")
+    print(f"they were accessing this volume: /{vn.vpath}")
+    print(f"and the original request-path (straight from the URL) was /{cli.vpath}")
+    print(f"...which resolves to the following filesystem path: {vn.canonical(rem)}")
+
+    new_path = "/foo/bar/"
+    print(f"will now redirect the client to {new_path}")
+
+    # uncomment one of these:
+    send_http_302_temporary_redirect(cli, new_path)
+    # send_http_301_permanent_redirect(cli, new_path)
+    # send_errorpage_with_redirect_link(cli, new_path)
+
+    return "true"

bin/hooks/README.md

@@ -4,6 +4,11 @@ these programs either take zero arguments, or a filepath (the affected file), or
 
 run copyparty with `--help-hooks` for usage details / hook type explanations (xm/xbu/xau/xiu/xbc/xac/xbr/xar/xbd/xad/xban)
 
+in particular, if a hook is loaded into copyparty with the hook-flag `c` ("check") then its exit-code controls the action that launched the hook:
+* exit-code `0` = allow the action, and/or continue running the next hook
+* exit-code `100` = allow the action, and stop running any remaining consecutive hooks
+* anything else = reject/prevent the original action, and don't run the remaining hooks
+
 > **note:** in addition to event hooks (the stuff described here), copyparty has another api to run your programs/scripts while providing way more information such as audio tags / video codecs / etc and optionally daisychaining data between scripts in a processing pipeline; if that's what you want then see [mtp plugins](../mtag/) instead
 
 
@@ -14,6 +19,8 @@ run copyparty with `--help-hooks` for usage details / hook type explanations (xm
 * [discord-announce.py](discord-announce.py) announces new uploads on discord using webhooks ([example](https://user-images.githubusercontent.com/241032/215304439-1c1cb3c8-ec6f-4c17-9f27-81f969b1811a.png))
 * [reject-mimetype.py](reject-mimetype.py) rejects uploads unless the mimetype is acceptable
 * [into-the-cache-it-goes.py](into-the-cache-it-goes.py) avoids bugs in caching proxies by immediately downloading each file that is uploaded
+* [podcast-normalizer.py](podcast-normalizer.py) creates a second file with dynamic-range-compression whenever an audio file is uploaded
+  * good example of the `idx` [hook effect](https://github.com/9001/copyparty/blob/hovudstraum/docs/devnotes.md#hook-effects) to tell copyparty about additional files to scan/index
 
 
 # upload batches
@@ -25,10 +32,20 @@ these are `--xiu` hooks; unlike `xbu` and `xau` (which get executed on every sin
 # before upload
 * [reject-extension.py](reject-extension.py) rejects uploads if they match a list of file extensions
 * [reloc-by-ext.py](reloc-by-ext.py) redirects an upload to another destination based on the file extension
+  * good example of the `reloc` [hook effect](https://github.com/9001/copyparty/blob/hovudstraum/docs/devnotes.md#hook-effects)
+* [reject-and-explain.py](reject-and-explain.py) shows a custom error-message when it rejects an upload
+* [reject-ramdisk.py](reject-ramdisk.py) rejects the upload if the destination is a ramdisk
+  * this hook uses the `I` flag which makes it 140x faster, but if the plugin has a bug it may crash copyparty
 
 
 # on message
 * [wget.py](wget.py) lets you download files by POSTing URLs to copyparty
+  * [wget-i.py](wget-i.py) is an import-safe modification of this hook (starts 140x faster, but higher chance of bugs)
 * [qbittorrent-magnet.py](qbittorrent-magnet.py) starts downloading a torrent if you post a magnet url
 * [usb-eject.py](usb-eject.py) adds web-UI buttons to safe-remove usb flashdrives shared through copyparty
 * [msg-log.py](msg-log.py) is a guestbook; logs messages to a doc in the same folder
+
+
+# general concept demos
+* [import-me.py](import-me.py) shows how the `I` flag makes the hook 140x faster (but you need to be Very Careful when writing the plugin)
+  * [wget-i.py](wget-i.py) is an import-safe modification of [wget.py](wget.py)

bin/hooks/import-me.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env python3
+
+from typing import Any
+
+_ = r"""
+the fastest hook in the west
+(runs directly inside copyparty, not as a subprocess)
+
+example usage as global config:
+    --xbu I,bin/hooks/import-me.py
+
+example usage as a volflag (per-volume config):
+    -v srv/inc:inc:r:rw,ed:c,xbu=I,bin/hooks/import-me.py
+                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+    (share filesystem-path srv/inc as volume /inc,
+     readable by everyone, read-write for user 'ed',
+     running this plugin on all uploads with the params listed below)
+
+example usage as a volflag in a copyparty config file:
+    [/inc]
+      srv/inc
+      accs:
+        r: *
+        rw: ed
+      flags:
+        xbu: I,bin/hooks/import-me.py
+
+parameters explained,
+    I = import; do not fork / subprocess
+
+IMPORTANT NOTE:
+    because this hook is running inside copyparty, you need to
+    be EXCEPTIONALLY CAREFUL to avoid side-effects, for example
+    DO NOT os.chdir() or anything like that, and also make sure
+    that the name of this file is unique (cannot be the same as
+    an existing python module/library)
+"""
+
+
+def main(ka: dict[str, Any]) -> dict[str, Any]:
+    # "ka" is a dictionary with info from copyparty...
+
+    # but because we are running inside copyparty, we don't need such courtesies;
+    import inspect
+
+    cf = inspect.currentframe().f_back.f_back.f_back
+    t = "hello from hook; I am able to peek into copyparty's memory like so:\n  function name: %s\n  variables:\n    %s\n"
+    t2 = "\n    ".join([("%r: %r" % (k, v))[:99] for k, v in cf.f_locals.items()][:9])
+    logger = ka["log"]
+    logger(t % (cf.f_code, t2))
+
+    # must return a dictionary with:
+    #  "rc": the retcode; 0 is ok
+    return {"rc": 0}

bin/hooks/notify.py

@@ -9,7 +9,7 @@ from plyer import notification
 _ = r"""
 show os notification on upload; works on windows, linux, macos, android
 
-depdencies:
+dependencies:
     windows: python3 -m pip install --user -U plyer
     linux:   python3 -m pip install --user -U plyer
     macos:   python3 -m pip install --user -U plyer pyobjus

bin/hooks/podcast-normalizer.py

@@ -0,0 +1,121 @@
+#!/usr/bin/env python3
+
+import json
+import os
+import sys
+import subprocess as sp
+
+
+_ = r"""
+sends all uploaded audio files through an aggressive
+dynamic-range-compressor to even out the volume levels
+
+dependencies:
+    ffmpeg
+
+being an xau hook, this gets eXecuted After Upload completion
+    but before copyparty has started hashing/indexing the file, so
+    we'll create a second normalized copy in a subfolder and tell
+    copyparty to hash/index that additional file as well
+
+example usage as global config:
+    -e2d -e2t --xau j,c1,bin/hooks/podcast-normalizer.py
+
+parameters explained,
+    e2d/e2t = enable database and metadata indexing
+    xau = execute after upload
+    j   = this hook needs upload information as json (not just the filename)
+    c1  = this hook returns json on stdout, so tell copyparty to read that
+
+example usage as a volflag (per-volume config):
+    -v srv/inc/pods:inc/pods:r:rw,ed:c,xau=j,c1,bin/hooks/podcast-normalizer.py
+                                     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+    (share fs-path srv/inc/pods at URL /inc/pods,
+     readable by all, read-write for user ed,
+     running this xau (exec-after-upload) plugin for all uploaded files)
+
+example usage as a volflag in a copyparty config file:
+    [/inc/pods]
+      srv/inc/pods
+      accs:
+        r: *
+        rw: ed
+      flags:
+        e2d  # enables file indexing
+        e2t  # metadata tags too
+        xau: j,c1,bin/hooks/podcast-normalizer.py
+
+"""
+
+########################################################################
+### CONFIG
+
+# filetypes to process; ignores everything else
+EXTS = "mp3 flac ogg oga opus m4a aac wav wma"
+
+# the name of the subdir to put the normalized files in
+SUBDIR = "normalized"
+
+########################################################################
+
+
+# try to enable support for crazy filenames
+try:
+    from copyparty.util import fsenc
+except:
+
+    def fsenc(p):
+        return p.encode("utf-8")
+
+
+def main():
+    # read info from copyparty
+    inf = json.loads(sys.argv[1])
+    vpath = inf["vp"]
+    abspath = inf["ap"]
+
+    # check if the file-extension is on the to-be-processed list
+    ext = abspath.lower().split(".")[-1]
+    if ext not in EXTS.split():
+        return
+
+    # jump into the folder where the file was uploaded
+    # and create the subfolder to place the normalized copy inside
+    dirpath, filename = os.path.split(abspath)
+    os.chdir(fsenc(dirpath))
+    os.makedirs(SUBDIR, exist_ok=True)
+
+    # the input and output filenames to give ffmpeg
+    fname_in = fsenc(f"./{filename}")
+    fname_out = fsenc(f"{SUBDIR}/{filename}.opus")
+
+    # fmt: off
+    # create and run the ffmpeg command
+    cmd = [
+        b"ffmpeg",
+        b"-nostdin",
+        b"-hide_banner",
+        b"-i", fname_in,
+        b"-af", b"dynaudnorm=f=100:g=9",  # the normalizer config
+        b"-c:a", b"libopus",
+        b"-b:a", b"128k",
+        fname_out,
+    ]
+    # fmt: on
+    sp.check_output(cmd)
+
+    # and finally, tell copyparty about the new file
+    # so it appears in the database and rss-feed:
+    vpath = f"{SUBDIR}/{filename}.opus"
+    print(json.dumps({"idx": {"vp": [vpath]}}))
+
+    # (it's fine to give it a relative path like that; it gets
+    #  resolved relative to the folder the file was uploaded into)
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except Exception as ex:
+        print("podcast-normalizer failed; %r" % (ex,))

bin/hooks/reject-and-explain.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+
+import json
+import os
+import re
+import sys
+
+
+_ = r"""
+reject file upload (with a nice explanation why)
+
+example usage as global config:
+    --xbu j,c1,bin/hooks/reject-and-explain.py
+
+example usage as a volflag (per-volume config):
+    -v srv/inc:inc:r:rw,ed:c,xbu=j,c1,bin/hooks/reject-and-explain.py
+                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+    (share filesystem-path srv/inc as volume /inc,
+     readable by everyone, read-write for user 'ed',
+     running this plugin on all uploads with the params listed below)
+
+example usage as a volflag in a copyparty config file:
+    [/inc]
+      srv/inc
+      accs:
+        r: *
+        rw: ed
+      flags:
+        xbu: j,c1,bin/hooks/reject-and-explain.py
+
+parameters explained,
+    xbu = execute-before-upload  (can also be xau, execute-after-upload)
+    j   = this hook needs upload information as json (not just the filename)
+    c1  = this hook returns json on stdout, so tell copyparty to read that
+"""
+
+
+def main():
+    inf = json.loads(sys.argv[1])
+    vdir, fn = os.path.split(inf["vp"])
+    print("inf[vp] = %r" % (inf["vp"],), file=sys.stderr)
+
+    # the following is what decides if we'll accept the upload or reject it:
+    # we check if the upload-folder url matches the following regex-pattern:
+    ok = re.search(r"(^|/)day[0-9]+$", vdir, re.IGNORECASE)
+
+    if ok:
+        # allow the upload
+        print("{}")
+        return
+
+    # the upload was rejected; display the following errortext:
+    errmsg = "Files can only be uploaded into a folder named 'DayN' where N is a number, for example 'Day573'. This file was REJECTED: "
+    errmsg += inf["vp"]  # if you want to mention the file's url
+    print(json.dumps({"rejectmsg": errmsg}))
+
+
+if __name__ == "__main__":
+    main()

bin/hooks/reject-ramdisk.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+
+import os
+import threading
+from argparse import Namespace
+
+from jinja2.nodes import Name
+from copyparty.fsutil import Fstab
+from typing import Any, Optional
+
+
+_ = r"""
+reject an upload if the target folder is on a ramdisk; useful when you
+have a volume where some folders inside are ramdisks but others aren't
+
+example usage as global config:
+    --xbu I,bin/hooks/reject-ramdisk.py
+
+example usage as a volflag (per-volume config):
+    -v srv/inc:inc:r:rw,ed:c,xbu=I,bin/hooks/reject-ramdisk.py
+                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+    (share filesystem-path srv/inc as volume /inc,
+     readable by everyone, read-write for user 'ed',
+     running this plugin on all uploads with the params listed below)
+
+example usage as a volflag in a copyparty config file:
+    [/inc]
+      srv/inc
+      accs:
+        r: *
+        rw: ed
+      flags:
+        xbu: I,bin/hooks/reject-ramdisk.py
+
+parameters explained,
+    I = import; do not fork / subprocess
+
+IMPORTANT NOTE:
+    because this hook is imported inside copyparty, you need to
+    be EXCEPTIONALLY CAREFUL to avoid side-effects, for example
+    DO NOT os.chdir() or anything like that, and also make sure
+    that the name of this file is unique (cannot be the same as
+    an existing python module/library)
+"""
+
+
+mutex = threading.Lock()
+fstab: Optional[Fstab] = None
+
+
+def main(ka: dict[str, Any]) -> dict[str, Any]:
+    global fstab
+    with mutex:
+        log = ka["log"]  # this is a copyparty NamedLogger function
+        if not fstab:
+            log("<HOOK:RAMDISK> creating fstab", 6)
+            args = Namespace()
+            args.mtab_age = 1  # cache the filesystem info for 1 sec
+            fstab = Fstab(log, args, False)
+
+        ap = ka["ap"]  # abspath the upload is going to
+        fs, mp = fstab.get(ap)  # figure out what the filesystem is
+        ramdisk = fs in ("tmpfs", "overlay")  # looks like a ramdisk?
+
+        # log("<HOOK:RAMDISK> fs=%r" % (fs,))
+
+        if ramdisk:
+            t = "Upload REJECTED because destination is a ramdisk"
+            return {"rc": 1, "rejectmsg": t}
+
+        return {"rc": 0}

bin/hooks/reloc-by-ext.py

@@ -71,6 +71,9 @@ def main():
     ## selecting it inside the print at the end:
     ##
 
+    # move all uploads to one specific folder
+    into_junk = {"vp": "/junk"}
+
     # create a subfolder named after the filetype and move it into there
     into_subfolder = {"vp": ext}
 
@@ -92,8 +95,8 @@ def main():
         by_category = {}  # no action
 
     # now choose the default effect to apply; can be any of these:
-    # into_subfolder  into_toplevel  into_sibling  by_category
-    effect = {"vp": "/junk"}
+    # into_junk  into_subfolder  into_toplevel  into_sibling  by_category
+    effect = into_sibling
 
     ##
     ## but we can keep going, adding more speicifc rules

bin/hooks/usb-eject.js

@@ -1,15 +1,17 @@
 // see usb-eject.py for usage
 
 function usbclick() {
-    QS('#treeul a[href="/usb/"]').click();
+    var o = QS('#treeul a[dst="/usb/"]') || QS('#treepar a[dst="/usb/"]');
+    if (o)
+        o.click();
 }
 
 function eject_cb() {
-    var t = this.responseText;
+    var t = ('' + this.responseText).trim();
     if (t.indexOf('can be safely unplugged') < 0 && t.indexOf('Device can be removed') < 0)
         return toast.err(30, 'usb eject failed:\n\n' + t);
 
-    toast.ok(5, esc(t.replace(/ - /g, '\n\n')));
+    toast.ok(5, esc(t.replace(/ - /g, '\n\n')).trim());
     usbclick(); setTimeout(usbclick, 10);
 };
 
@@ -21,10 +23,15 @@ function add_eject_2(a) {
     var v = aw[2],
         k = 'umount_' + v;
 
-    qsr('#' + k);
-    a.appendChild(mknod('span', k, '⏏'), a);
-
+    for (var b = 0; b < 9; b++) {
         var o = ebi(k);
+        if (!o)
+            break;
+        o.parentNode.removeChild(o);
+    }
+
+    a.appendChild(mknod('span', k, '⏏'), a);
+    o = ebi(k);
     o.style.cssText = 'position:absolute; right:1em; margin-top:-.2em; font-size:1.3em';
     o.onclick = function (e) {
         ev(e);
@@ -38,8 +45,9 @@ function add_eject_2(a) {
 };
 
 function add_eject() {
-    for (var a of QSA('#treeul a[href^="/usb/"]'))
-        add_eject_2(a);
+    var o = QSA('#treeul a[href^="/usb/"]') || QSA('#treepar a[href^="/usb/"]');
+    for (var a = o.length - 1; a > 0; a--)
+        add_eject_2(o[a]);
 };
 
 (function() {

bin/hooks/usb-eject.py

@@ -4,6 +4,7 @@ import os
 import stat
 import subprocess as sp
 import sys
+from urllib.parse import unquote_to_bytes as unquote
 
 
 """
@@ -14,13 +15,13 @@ remove those flashdrives, then boy howdy are you in the right place :D
 put usb-eject.js in the webroot (or somewhere else http-accessible)
 then run copyparty with these args:
 
-   -v /run/media/ed:/usb:A:c,hist=/tmp/junk
+   -v /run/media/egon:/usb:A:c,hist=/tmp/junk
    --xm=c1,bin/hooks/usb-eject.py
    --js-browser=/usb-eject.js
 
 which does the following respectively,
 
-  * share all of /run/media/ed as /usb with admin for everyone
+  * share all of /run/media/egon as /usb with admin for everyone
      and put the histpath somewhere it won't cause trouble
   * run the usb-eject hook with stdout redirect to the web-ui
   * add the complementary usb-eject.js to the browser
@@ -28,18 +29,33 @@ which does the following respectively,
 """
 
 
+MOUNT_BASE = b"/run/media/egon/"
+
+
 def main():
     try:
-        label = sys.argv[1].split(":usb-eject:")[1].split(":")[0]
-        mp = "/run/media/ed/" + label
+        msg = sys.argv[1]
+        if msg.startswith("upload-queue-empty;"):
+            return
+        label = msg.split(":usb-eject:")[1].split(":")[0]
+        mp = MOUNT_BASE + unquote(label)
         # print("ejecting [%s]... " % (mp,), end="")
-        mp = os.path.abspath(os.path.realpath(mp.encode("utf-8")))
+        mp = os.path.abspath(os.path.realpath(mp))
         st = os.lstat(mp)
-        if not stat.S_ISDIR(st.st_mode):
+        if not stat.S_ISDIR(st.st_mode) or not mp.startswith(MOUNT_BASE):
             raise Exception("not a regular directory")
 
-        cmd = [b"gio", b"mount", b"-e", mp]
-        print(sp.check_output(cmd).decode("utf-8", "replace").strip())
+        # if you're running copyparty as root (thx for the faith)
+        # you'll need something like this to make dbus talkative
+        cmd = b"sudo -u egon DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/1000/bus gio mount -e"
+
+        # but if copyparty and the ui-session is running
+        # as the same user (good) then this is plenty
+        cmd = b"gio mount -e"
+
+        cmd = cmd.split(b" ") + [mp]
+        ret = sp.check_output(cmd).decode("utf-8", "replace")
+        print(ret.strip() or (label + " can be safely unplugged"))
 
     except Exception as ex:
         print("unmount failed: %r" % (ex,))

bin/hooks/wget-i.py

@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+
+import os
+import threading
+import subprocess as sp
+
+
+_ = r"""
+use copyparty as a file downloader by POSTing URLs as
+application/x-www-form-urlencoded (for example using the
+📟 message-to-server-log in the web-ui)
+
+this hook is a modified copy of wget.py, modified to
+make it import-safe so it can be run with the 'I' flag,
+which speeds up the startup time of the hook by 140x
+
+example usage as global config:
+    --xm aw,I,bin/hooks/wget-i.py
+
+parameters explained,
+    xm = execute on message-to-server-log
+    aw = only users with write-access can use this
+    I = import; do not fork / subprocess
+
+example usage as a volflag (per-volume config):
+    -v srv/inc:inc:r:rw,ed:c,xm=aw,I,bin/hooks/wget.py
+                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+    (share filesystem-path srv/inc as volume /inc,
+     readable by everyone, read-write for user 'ed',
+     running this plugin on all messages with the params explained above)
+
+example usage as a volflag in a copyparty config file:
+    [/inc]
+      srv/inc
+      accs:
+        r: *
+        rw: ed
+      flags:
+        xm: aw,I,bin/hooks/wget.py
+
+the volflag examples only kicks in if you send the message
+while you're in the /inc folder (or any folder below there)
+
+IMPORTANT NOTE:
+    because this hook uses the 'I' flag to run inside copyparty,
+    many other flags will not work (f,j,c3,t3600 as seen in the
+    original wget.py), and furthermore + more importantly we
+    need to be EXCEPTIONALLY CAREFUL to avoid side-effects, so
+    the os.chdir has been replaced with cwd=dirpath for example
+"""
+
+
+def do_stuff(inf):
+    """
+    worker function which is executed in another thread to
+    avoid blocking copyparty while the download is running,
+    since we cannot use the 'f,t3600' hook-flags with 'I'
+    """
+
+    # first things first; grab the logger-function which copyparty is letting us borrow
+    log = inf["log"]
+
+    url = inf["txt"]
+    if url.startswith("upload-queue-empty;"):
+        return
+
+    if "://" not in url:
+        url = "https://" + url
+
+    proto = url.split("://")[0].lower()
+    if proto not in ("http", "https", "ftp", "ftps"):
+        raise Exception("bad proto {}".format(proto))
+
+    dirpath = inf["ap"]
+
+    name = url.split("?")[0].split("/")[-1]
+    msg = "-- DOWNLOADING " + name
+    log(msg)
+    tfn = os.path.join(dirpath, msg)
+    open(tfn, "wb").close()
+
+    cmd = ["wget", "--trust-server-names", "-nv", "--", url]
+
+    try:
+        # two things to note here:
+        # - cannot use the `c3` hook-flag with `I` so mute output with stdout=sp.DEVNULL instead;
+        # - MUST NOT use os.chdir with 'I' so use cwd=dirpath instead 
+        sp.check_call(cmd, cwd=dirpath, stdout=sp.DEVNULL)
+    except:
+        t = "-- FAILED TO DOWNLOAD " + name
+        log(t, 3)  # 3=yellow=warning
+        open(os.path.join(dirpath, t), "wb").close()
+        raise  # have copyparty scream about the details in the log
+
+    os.unlink(tfn)
+
+
+def main(inf):
+    threading.Thread(target=do_stuff, args=(inf,), daemon=True).start()

bin/hooks/wget.py

@@ -47,6 +47,9 @@ while you're in the /inc folder (or any folder below there)
 def main():
     inf = json.loads(sys.argv[1])
     url = inf["txt"]
+    if url.startswith("upload-queue-empty;"):
+        return
+
     if "://" not in url:
         url = "https://" + url
 
@@ -66,7 +69,7 @@ def main():
     try:
         sp.check_call(cmd)
     except:
-        t = "-- FAILED TO DONWLOAD " + name
+        t = "-- FAILED TO DOWNLOAD " + name
         print(f"{t}\n", end="")
         open(t, "wb").close()
 

bin/mtag/README.md

@@ -68,3 +68,8 @@ instead of affecting all volumes, you can set the options for just one volume li
 * `:c,mtp=key=f,audio-key.py`
 * `:c,mtp=.bpm=f,audio-bpm.py`
 * `:c,mtp=ahash,vhash=f,media-hash.py`
+
+
+# tips & tricks
+
+* to delete tags for all files below `blog*` and rescan that, `sqlite3 .hist/up2k.db "delete from mt where w in (select substr(w,1,16) from up where rd like 'blog%')";`

bin/mtag/cksum.py

@@ -2,11 +2,15 @@
 
 import sys
 import json
-import zlib
 import struct
 import base64
 import hashlib
 
+try:
+    from zlib_ng import zlib_ng as zlib
+except:
+    import zlib
+
 try:
     from copyparty.util import fsenc
 except:

bin/mtag/geotag.py

@@ -0,0 +1,53 @@
+import json
+import re
+import sys
+
+from copyparty.util import fsenc, runcmd
+
+
+"""
+uses exiftool to geotag images based on embedded gps coordinates in exif data
+
+adds four new metadata keys:
+    .gps_lat = latitute
+    .gps_lon = longitude
+    .masl = meters above sea level
+    city = "city, subregion, region"
+
+usage: -mtp .masl,.gps_lat,.gps_lon,city=ad,t10,bin/mtag/geotag.py
+
+example: https://a.ocv.me/pub/blog/j7/8/?grid=0
+"""
+
+
+def main():
+    cmd = b"exiftool -api geolocation -n".split(b" ")
+    rc, so, se = runcmd(cmd + [fsenc(sys.argv[1])])
+    ptn = re.compile("([^:]*[^ :]) *: (.*)")
+    city = ["", "", ""]
+    ret = {}
+    for ln in so.split("\n"):
+        m = ptn.match(ln)
+        if not m:
+            continue
+        k, v = m.groups()
+        if k == "Geolocation City":
+            city[2] = v
+        elif k == "Geolocation Subregion":
+            city[1] = v
+        elif k == "Geolocation Region":
+            city[0] = v
+        elif k == "GPS Latitude":
+            ret[".gps_lat"] = "%.04f" % (float(v),)
+        elif k == "GPS Longitude":
+            ret[".gps_lon"] = "%.04f" % (float(v),)
+        elif k == "GPS Altitude":
+            ret[".masl"] = str(int(float(v)))
+    v = ", ".join(city).strip(", ")
+    if v:
+        ret["city"] = v
+    print(json.dumps(ret))
+
+
+if __name__ == "__main__":
+    main()

bin/mtag/guestbook-read.py

@@ -7,7 +7,7 @@ example copyparty config to use this:
   --urlform save,get -vsrv/hello:hello:w:c,e2ts,mtp=guestbook=t10,ad,p,bin/mtag/guestbook-read.py:mte=+guestbook
 
 explained:
-  for realpath srv/hello (served at /hello), write-only for eveyrone,
+  for realpath srv/hello (served at /hello), write-only for everyone,
   enable file analysis on upload (e2ts),
   use mtp plugin "bin/mtag/guestbook-read.py" to provide metadata tag "guestbook",
   do this on all uploads regardless of extension,

bin/mtag/guestbook.py

@@ -11,7 +11,7 @@ example copyparty config to use this:
   --urlform save,get -vsrv/hello:hello:w:c,e2ts,mtp=xgb=ebin,t10,ad,p,bin/mtag/guestbook.py:mte=+xgb
 
 explained:
-  for realpath srv/hello (served at /hello),write-only for eveyrone,
+  for realpath srv/hello (served at /hello),write-only for everyone,
   enable file analysis on upload (e2ts),
   use mtp plugin "bin/mtag/guestbook.py" to provide metadata tag "xgb",
   do this on all uploads with the file extension "bin",

bin/mtag/install-deps.sh

@@ -22,6 +22,8 @@ set -e
 #   modifies the keyfinder python lib to load the .so in ~/pe
 
 
+export FORCE_COLOR=1
+
 linux=1
 
 win=
@@ -187,11 +189,14 @@ install_keyfinder() {
 		exit 1
 	}
 
+	x=${-//[^x]/}; set -x; cat /etc/alpine-release
 	# rm -rf /Users/ed/Library/Python/3.9/lib/python/site-packages/*keyfinder*
 	CFLAGS="-I$h/pe/keyfinder/include -I/opt/local/include -I/usr/include/ffmpeg" \
+	CXXFLAGS="-I$h/pe/keyfinder/include -I/opt/local/include -I/usr/include/ffmpeg" \
 	LDFLAGS="-L$h/pe/keyfinder/lib -L$h/pe/keyfinder/lib64 -L/opt/local/lib" \
-	PKG_CONFIG_PATH=/c/msys64/mingw64/lib/pkgconfig \
+	PKG_CONFIG_PATH="/c/msys64/mingw64/lib/pkgconfig:$h/pe/keyfinder/lib/pkgconfig" \
 	$pybin -m pip install --user keyfinder
+	[ "$x" ] || set +x
 
 	pypath="$($pybin -c 'import keyfinder; print(keyfinder.__file__)')"
 	for pyso in "${pypath%/*}"/*.so; do

bin/mtag/wget.py

@@ -84,7 +84,7 @@ def main():
         #   on success, delete the .bin file which contains the URL
         os.unlink(fp)
     except:
-        open("-- FAILED TO DONWLOAD " + name, "wb").close()
+        open("-- FAILED TO DOWNLOAD " + name, "wb").close()
 
     os.unlink(tfn)
     print(url)

bin/partyfuse.py

@@ -6,8 +6,8 @@ __copyright__ = 2019
 __license__ = "MIT"
 __url__ = "https://github.com/9001/copyparty/"
 
-S_VERSION = "2.0"
-S_BUILD_DT = "2024-10-01"
+S_VERSION = "2.1"
+S_BUILD_DT = "2025-09-06"
 
 """
 mount a copyparty server (local or remote) as a filesystem
@@ -99,7 +99,7 @@ except:
     elif MACOS:
         libfuse = "install https://osxfuse.github.io/"
     else:
-        libfuse = "apt install libfuse3-3\n    modprobe fuse"
+        libfuse = "apt install libfuse2\n    modprobe fuse"
 
     m = """\033[33m
   could not import fuse; these may help:
@@ -359,7 +359,7 @@ class Gateway(object):
     def sendreq(self, meth, path, headers, **kwargs):
         tid = get_tid()
         if self.password:
-            headers["Cookie"] = "=".join(["cppwd", self.password])
+            headers["PW"] = self.password
 
         try:
             c = self.getconn(tid)
@@ -902,9 +902,7 @@ class CPPF(Operations):
         return ret
 
     def _readdir(self, path, fh=None):
-        path = path.strip("/")
-        dbg("readdir %r [%s]", path, fh)
-
+        dbg("dircache miss")
         ret = self.gw.listdir(path)
         if not self.n_dircache:
             return ret
@@ -914,11 +912,17 @@ class CPPF(Operations):
             self.dircache.append(cn)
             self.clean_dircache()
 
-        # import pprint; pprint.pprint(ret)
         return ret
 
     def readdir(self, path, fh=None):
-        return [".", ".."] + list(self._readdir(path, fh))
+        dbg("readdir %r [%s]", path, fh)
+        path = path.strip("/")
+        cn = self.get_cached_dir(path)
+        if cn:
+            ret = cn.data
+        else:
+            ret = self._readdir(path, fh)
+        return [".", ".."] + list(ret)
 
     def read(self, path, length, offset, fh=None):
         req_max = 1024 * 1024 * 8
@@ -993,7 +997,6 @@ class CPPF(Operations):
         if cn:
             dents = cn.data
         else:
-            dbg("cache miss")
             dents = self._readdir(dirpath)
 
         try:
@@ -1141,10 +1144,15 @@ def main():
     if WINDOWS:
         examples.append("http://192.168.1.69:3923/music/  M:")
 
+    epi = "example:" + ex_pre + ex_pre.join(examples)
+    epi += """\n
+NOTE: if server has --usernames enabled, then password is "username:password"
+"""
+
     ap = argparse.ArgumentParser(
         formatter_class=TheArgparseFormatter,
         description="mount a copyparty server as a local filesystem -- " + ver,
-        epilog="example:" + ex_pre + ex_pre.join(examples),
+        epilog=epi,
     )
     # fmt: off
     ap.add_argument("base_url", type=str, help="remote copyparty URL to mount")

bin/prisonparty.sh

@@ -141,7 +141,7 @@ chmod 777 "$jail/tmp"
 
 
 # create a dev
-(cd $jail; mkdir -p dev; cd dev
+(cd "$jail"; mkdir -p dev; cd dev
 [ -e null ]    || mknod -m 666 null    c 1 3
 [ -e zero ]    || mknod -m 666 zero    c 1 5
 [ -e random ]  || mknod -m 444 random  c 1 8

bin/u2c.py

@@ -1,8 +1,8 @@
 #!/usr/bin/env python3
 from __future__ import print_function, unicode_literals
 
-S_VERSION = "2.8"
-S_BUILD_DT = "2025-01-21"
+S_VERSION = "2.15"
+S_BUILD_DT = "2025-10-25"
 
 """
 u2c.py: upload to copyparty
@@ -10,7 +10,7 @@ u2c.py: upload to copyparty
 https://github.com/9001/copyparty/blob/hovudstraum/bin/u2c.py
 
 - dependencies: no
-- supports python 2.6, 2.7, and 3.3 through 3.12
+- supports python 2.6, 2.7, and 3.3 through 3.14
 - if something breaks just try again and it'll autoresume
 """
 
@@ -52,6 +52,7 @@ if PY2:
 
     sys.dont_write_bytecode = True
     bytes = str
+    files_decoder = lambda s: unicode(s, "utf8")
 else:
     from urllib.parse import quote_from_bytes as quote
     from urllib.parse import unquote_to_bytes as unquote
@@ -61,6 +62,7 @@ else:
     from queue import Queue
 
     unicode = str
+    files_decoder = unicode
 
 
 WTF8 = "replace" if PY2 else "surrogateescape"
@@ -230,10 +232,15 @@ class HCli(object):
 
 MJ = "application/json"
 MO = "application/octet-stream"
+MM = "application/x-www-form-urlencoded"
 CLEN = "Content-Length"
 
 web = None  # type: HCli
 
+links = []  # type: list[str]
+linkmtx = threading.Lock()
+linkfile = None
+
 
 class File(object):
     """an up2k upload task; represents a single file"""
@@ -584,9 +591,10 @@ def undns(url):
 
 def _scd(err, top):
     """non-recursive listing of directory contents, along with stat() info"""
+    top_ = os.path.join(top, b"")
     with os.scandir(top) as dh:
         for fh in dh:
-            abspath = os.path.join(top, fh.name)
+            abspath = top_ + fh.name
             try:
                 yield [abspath, fh.stat()]
             except Exception as ex:
@@ -595,8 +603,9 @@ def _scd(err, top):
 
 def _lsd(err, top):
     """non-recursive listing of directory contents, along with stat() info"""
+    top_ = os.path.join(top, b"")
     for name in os.listdir(top):
-        abspath = os.path.join(top, name)
+        abspath = top_ + name
         try:
             yield [abspath, os.stat(abspath)]
         except Exception as ex:
@@ -671,7 +680,7 @@ def walkdirs(err, tops, excl):
                 yield stop, ap[len(stop) :].lstrip(sep), inf
         else:
             d, n = top.rsplit(sep, 1)
-            yield d, n, os.stat(top)
+            yield d or b"/", n, os.stat(top)
 
 
 # mostly from copyparty/util.py
@@ -761,6 +770,29 @@ def get_hashlist(file, pcb, mth):
             file.kchunks[k] = [v1, v2]
 
 
+def printlink(ar, purl, name, fk):
+    if not name:
+        url = purl  # srch
+    else:
+        name = quotep(name.encode("utf-8", WTF8)).decode("utf-8")
+        if fk:
+            url = "%s%s?k=%s" % (purl, name, fk)
+        else:
+            url = "%s%s" % (purl, name)
+
+    url = "%s/%s" % (ar.burl, url.lstrip("/"))
+
+    with linkmtx:
+        if ar.u:
+            links.append(url)
+        if ar.ud:
+            print(url)
+        if linkfile:
+            zs = "%s\n" % (url,)
+            zb = zs.encode("utf-8", "replace")
+            linkfile.write(zb)
+
+
 def handshake(ar, file, search):
     # type: (argparse.Namespace, File, bool) -> tuple[list[str], bool]
     """
@@ -780,7 +812,9 @@ def handshake(ar, file, search):
     else:
         if ar.touch:
             req["umod"] = True
-        if ar.ow:
+        if ar.owo:
+            req["replace"] = "mt"
+        elif ar.ow:
             req["replace"] = True
 
     file.recheck = False
@@ -832,12 +866,17 @@ def handshake(ar, file, search):
         raise Exception(txt)
 
     if search:
+        if ar.uon and r["hits"]:
+            printlink(ar, r["hits"][0]["rp"], "", "")
         return r["hits"], False
 
     file.url = quotep(r["purl"].encode("utf-8", WTF8)).decode("utf-8")
     file.name = r["name"]
     file.wark = r["wark"]
 
+    if ar.uon and not r["hash"]:
+        printlink(ar, file.url, r["name"], r.get("fk"))
+
     return r["hash"], r["sprs"]
 
 
@@ -941,6 +980,7 @@ class Ctl(object):
         self.nfiles, self.nbytes = self.stats
         self.filegen = walkdirs([], ar.files, ar.x)
         self.recheck = []  # type: list[File]
+        self.last_file = None
 
         if ar.safe:
             self._safe()
@@ -977,6 +1017,11 @@ class Ctl(object):
 
             self._fancy()
 
+        file = self.last_file
+        if self.up_br and file:
+            zs = quotep(file.name.encode("utf-8", WTF8))
+            web.req("POST", file.url, {}, b"msg=upload-queue-empty;" + zs, MM)
+
         self.ok = not self.errs
 
     def _safe(self):
@@ -1187,9 +1232,7 @@ class Ctl(object):
                             while req:
                                 print("DELETING ~%s#%s" % (srd, len(req)))
                                 body = json.dumps(req).encode("utf-8")
-                                sc, txt = web.req(
-                                    "POST", self.ar.url + "?delete", {}, body, MJ
-                                )
+                                sc, txt = web.req("POST", "/?delete", {}, body, MJ)
                                 if sc == 413 and "json 2big" in txt:
                                     print(" (delete request too big; slicing...)")
                                     req = req[: len(req) // 2]
@@ -1255,7 +1298,7 @@ class Ctl(object):
                 if self.ar.jw:
                     print("%s  %s" % (wark, vp))
                 else:
-                    zd = datetime.datetime.fromtimestamp(file.lmod, UTC)
+                    zd = datetime.datetime.fromtimestamp(max(0, file.lmod), UTC)
                     dt = "%04d-%02d-%02d %02d:%02d:%02d" % (
                         zd.year,
                         zd.month,
@@ -1417,6 +1460,7 @@ class Ctl(object):
 
             file = fsl.file
             cids = fsl.cids
+            self.last_file = file
 
             with self.mutex:
                 if not self.uploader_busy:
@@ -1472,7 +1516,7 @@ class APF(argparse.ArgumentDefaultsHelpFormatter, argparse.RawDescriptionHelpFor
 
 
 def main():
-    global web
+    global web, linkfile
 
     time.strptime("19970815", "%Y%m%d")  # python#7980
     "".encode("idna")  # python#29288
@@ -1491,14 +1535,14 @@ def main():
 
     # fmt: off
     ap = app = argparse.ArgumentParser(formatter_class=APF, description="copyparty up2k uploader / filesearch tool  " + ver, epilog="""
-NOTE:
-source file/folder selection uses rsync syntax, meaning that:
+NOTE: source file/folder selection uses rsync syntax, meaning that:
   "foo" uploads the entire folder to URL/foo/
   "foo/" uploads the CONTENTS of the folder into URL/
+NOTE: if server has --usernames enabled, then password is "username:password"
 """)
 
     ap.add_argument("url", type=unicode, help="server url, including destination folder")
-    ap.add_argument("files", type=unicode, nargs="+", help="files and/or folders to process")
+    ap.add_argument("files", type=files_decoder, nargs="+", help="files and/or folders to process")
     ap.add_argument("-v", action="store_true", help="verbose")
     ap.add_argument("-a", metavar="PASSWD", help="password or $filepath")
     ap.add_argument("-s", action="store_true", help="file-search (disables upload)")
@@ -1506,9 +1550,15 @@ source file/folder selection uses rsync syntax, meaning that:
     ap.add_argument("--ok", action="store_true", help="continue even if some local files are inaccessible")
     ap.add_argument("--touch", action="store_true", help="if last-modified timestamps differ, push local to server (need write+delete perms)")
     ap.add_argument("--ow", action="store_true", help="overwrite existing files instead of autorenaming")
+    ap.add_argument("--owo", action="store_true", help="overwrite existing files if server-file is older")
     ap.add_argument("--spd", action="store_true", help="print speeds for each file")
     ap.add_argument("--version", action="store_true", help="show version and exit")
 
+    ap = app.add_argument_group("print links")
+    ap.add_argument("-u", action="store_true", help="print list of download-links after all uploads finished")
+    ap.add_argument("-ud", action="store_true", help="print download-link after each upload finishes")
+    ap.add_argument("-uf", type=unicode, metavar="PATH", help="print list of download-links to file")
+
     ap = app.add_argument_group("compatibility")
     ap.add_argument("--cls", action="store_true", help="clear screen before start")
     ap.add_argument("--rh", type=int, metavar="TRIES", default=0, help="resolve server hostname before upload (good for buggy networks, but TLS certs will break)")
@@ -1594,6 +1644,10 @@ source file/folder selection uses rsync syntax, meaning that:
     ar.x = "|".join(ar.x or [])
 
     setattr(ar, "wlist", ar.url == "-")
+    setattr(ar, "uon", ar.u or ar.ud or ar.uf)
+
+    if ar.uf:
+        linkfile = open(ar.uf, "wb")
 
     for k in "dl dr drd wlist".split():
         errs = []
@@ -1656,6 +1710,12 @@ source file/folder selection uses rsync syntax, meaning that:
         ar.z = True
         ctl = Ctl(ar, ctl.stats)
 
+    if links:
+        print()
+        print("\n".join(links))
+    if linkfile:
+        linkfile.close()
+
     if ctl.errs:
         print("WARNING: %d errors" % (ctl.errs))
 

bin/zmq-recv.py

@@ -61,6 +61,8 @@ def rep_server():
         print("copyparty says %r" % (sck.recv_string(),))
         reply = b"thx"
         # reply = b"return 1"  # non-zero to block an upload
+        # reply = b'{"rc":1}'  # or as json, that's fine too
+        # reply = b'{"rejectmsg":"naw dude"}'  # or custom message
         sck.send(reply)
 
 

contrib/README.md

@@ -7,6 +7,12 @@
 * works on windows, linux and macos
 * assumes `copyparty-sfx.py` was renamed to `copyparty.py` in the same folder as `copyparty.bat`
 
+### [`setup-ashell.sh`](setup-ashell.sh)
+* run copyparty on an iPhone/iPad using [a-Shell](https://holzschu.github.io/a-Shell_iOS/)
+* not very useful due to limitations in iOS:
+  * not able to share all of your phone's storage
+  * cannot run in the background
+
 ### [`index.html`](index.html)
 * drop-in redirect from an httpd to copyparty
 * assumes the webserver and copyparty is running on the same server/IP
@@ -50,6 +56,9 @@
 * give a 3rd argument to install it to your copyparty config
 * systemd service at [`systemd/cfssl.service`](systemd/cfssl.service)
 
+### [`zfs-tune.py`](zfs-tune.py)
+* optimizes databases for optimal performance when stored on a zfs filesystem; also see [openzfs docs](https://openzfs.github.io/openzfs-docs/Performance%20and%20Tuning/Workload%20Tuning.html#database-workloads) and specifically the SQLite subsection
+
 # OS integration
 init-scripts to start copyparty as a service
 * [`systemd/copyparty.service`](systemd/copyparty.service) runs the sfx normally

contrib/nginx/copyparty.conf

@@ -2,19 +2,38 @@
 # not accept more consecutive clients than what copyparty is able to;
 # nginx default is 512  (worker_processes 1, worker_connections 512)
 #
+# ======================================================================
+#
+# to reverse-proxy a specific path/subpath/location below a domain
+# (rather than a complete subdomain), for example "/qw/er", you must
+# run copyparty with --rp-loc /qw/as and also change the following:
+# 	location / {
+# 		proxy_pass http://cpp_tcp;
+# to this:
+# 	location /qw/er/ {
+# 		proxy_pass http://cpp_tcp/qw/er/;
+#
+# ======================================================================
+#
 # rarely, in some extreme usecases, it can be good to add -j0
 # (40'000 requests per second, or 20gbps upload/download in parallel)
 # but this is usually counterproductive and slightly buggy
 #
+# ======================================================================
+#
 # on fedora/rhel, remember to setsebool -P httpd_can_network_connect 1
 #
-# if you are behind cloudflare (or another protection service),
+# ======================================================================
+#
+# if you are behind cloudflare (or another CDN/WAF/protection service),
 # remember to reject all connections which are not coming from your
 # protection service -- for cloudflare in particular, you can
 # generate the list of permitted IP ranges like so:
 #   (curl -s https://www.cloudflare.com/ips-v{4,6} | sed 's/^/allow /; s/$/;/'; echo; echo "deny all;") > /etc/nginx/cloudflare-only.conf
 #
-# and then enable it below by uncomenting the cloudflare-only.conf line
+# and then enable it below by uncommenting the cloudflare-only.conf line
+#
+# ======================================================================
 
 
 upstream cpp_tcp {
@@ -66,13 +85,13 @@ server {
 		proxy_buffer_size 16k;
 		proxy_busy_buffers_size 24k;
 
+		proxy_set_header   Connection        "Keep-Alive";
 		proxy_set_header   Host              $host;
 		proxy_set_header   X-Real-IP         $remote_addr;
-		proxy_set_header   X-Forwarded-For   $proxy_add_x_forwarded_for;
-		# NOTE: with cloudflare you want this instead:
-		#proxy_set_header   X-Forwarded-For   $http_cf_connecting_ip;
 		proxy_set_header   X-Forwarded-Proto $scheme;
-		proxy_set_header   Connection        "Keep-Alive";
+		proxy_set_header   X-Forwarded-For   $proxy_add_x_forwarded_for;
+		# NOTE: with cloudflare you want this X-Forwarded-For instead:
+		#proxy_set_header   X-Forwarded-For   $http_cf_connecting_ip;
 	}
 }
 

contrib/nixos/modules/copyparty.nix

@@ -1,9 +1,13 @@
-{ config, pkgs, lib, ... }:
-
+{
+  config,
+  pkgs,
+  lib,
+  ...
+}:
 with lib;
-
 let
-  mkKeyValue = key: value:
+  mkKeyValue =
+    key: value:
     if value == true then
       # sets with a true boolean value are coerced to just the key name
       key
@@ -15,9 +19,10 @@ let
 
   mkAttrsString = value: (generators.toKeyValue { inherit mkKeyValue; } value);
 
-  mkValueString = value:
+  mkValueString =
+    value:
     if isList value then
-      (concatStringsSep ", " (map mkValueString value))
+      (concatStringsSep "," (map mkValueString value))
     else if isAttrs value then
       "\n" + (mkAttrsString value)
     else
@@ -43,19 +48,24 @@ let
 
   accountsWithPlaceholders = mapAttrs (name: attrs: passwordPlaceholder name);
 
+  volumesWithoutVariables = filterAttrs (k: v: !(hasInfix "\${" v.path)) cfg.volumes;
+
   configStr = ''
     ${mkSection "global" cfg.settings}
+    ${cfg.globalExtraConfig}
     ${mkSection "accounts" (accountsWithPlaceholders cfg.accounts)}
+    ${mkSection "groups" cfg.groups}
     ${concatStringsSep "\n" (mapAttrsToList mkVolume cfg.volumes)}
   '';
 
-  name = "copyparty";
   cfg = config.services.copyparty;
-  configFile = pkgs.writeText "${name}.conf" configStr;
-  runtimeConfigPath = "/run/${name}/${name}.conf";
-  home = "/var/lib/${name}";
-  defaultShareDir = "${home}/data";
-in {
+  configFile = pkgs.writeText "copyparty.conf" configStr;
+  runtimeConfigPath = "/run/copyparty/copyparty.conf";
+  externalCacheDir = "/var/cache/copyparty";
+  externalStateDir = "/var/lib/copyparty";
+  defaultShareDir = "${externalStateDir}/data";
+in
+{
   options.services.copyparty = {
     enable = mkEnableOption "web-based file manager";
 
@@ -68,6 +78,35 @@ in {
       '';
     };
 
+    mkHashWrapper = mkOption {
+      type = types.bool;
+      default = true;
+      description = ''
+        Make a shell script wrapper called 'copyparty-hash' with all options set here,
+        that launches the hashing cli.
+      '';
+    };
+
+    user = mkOption {
+      type = types.str;
+      default = "copyparty";
+      description = ''
+        The user that copyparty will run under.
+
+        If changed from default, you are responsible for making sure the user exists.
+      '';
+    };
+
+    group = mkOption {
+      type = types.str;
+      default = "copyparty";
+      description = ''
+        The group that copyparty will run under.
+
+        If changed from default, you are responsible for making sure the user exists.
+      '';
+    };
+
     openFilesLimit = mkOption {
       default = 4096;
       type = types.either types.int types.str;
@@ -79,22 +118,34 @@ in {
       description = ''
         Global settings to apply.
         Directly maps to values in the [global] section of the copyparty config.
+        Cannot set "c" or "hist", those are set by this module.
         See `${getExe cfg.package} --help` for more details.
       '';
       default = {
         i = "127.0.0.1";
         no-reload = true;
+        hist = externalCacheDir;
       };
       example = literalExpression ''
         {
           i = "0.0.0.0";
           no-reload = true;
+          hist = ${externalCacheDir};
         }
       '';
     };
 
+    globalExtraConfig = mkOption {
+      type = types.str;
+      default = "";
+      description = "Appended to the end of the [global] section verbatim. This is useful for flags which are used in a repeating manner (e.g. ipu: 255.255.255.1=user) which can't be repeated in the settings = {} attribute set.";
+    };
+
     accounts = mkOption {
-      type = types.attrsOf (types.submodule ({ ... }: {
+      type = types.attrsOf (
+        types.submodule (
+          { ... }:
+          {
             options = {
               passwordFile = mkOption {
                 type = types.str;
@@ -105,7 +156,9 @@ in {
                 example = "/run/keys/copyparty/ed";
               };
             };
-      }));
+          }
+        )
+      );
       description = ''
         A set of copyparty accounts to create.
       '';
@@ -117,11 +170,27 @@ in {
       '';
     };
 
+    groups = mkOption {
+      type = types.attrsOf (types.listOf types.str);
+      description = ''
+        A set of copyparty groups to create and the users that should be part of each group.
+      '';
+      default = { };
+      example = literalExpression ''
+        {
+          group_name = [ "user1" "user2" ];
+        };
+      '';
+    };
+
     volumes = mkOption {
-      type = types.attrsOf (types.submodule ({ ... }: {
+      type = types.attrsOf (
+        types.submodule (
+          { ... }:
+          {
             options = {
               path = mkOption {
-            type = types.str;
+                type = types.path;
                 description = ''
                   Path of a directory to share.
                 '';
@@ -180,12 +249,16 @@ in {
                 default = { };
               };
             };
-      }));
+          }
+        )
+      );
       description = "A set of copyparty volumes to create";
       default = {
         "/" = {
           path = defaultShareDir;
-          access = { r = "*"; };
+          access = {
+            r = "*";
+          };
         };
       };
       example = literalExpression ''
@@ -204,52 +277,63 @@ in {
     };
   };
 
-  config = mkIf cfg.enable {
+  config = mkIf cfg.enable (
+    let
+      command = "${getExe cfg.package} -c ${runtimeConfigPath}";
+    in
+    {
       systemd.services.copyparty = {
         description = "http file sharing hub";
         wantedBy = [ "multi-user.target" ];
 
         environment = {
           PYTHONUNBUFFERED = "true";
-        XDG_CONFIG_HOME = "${home}/.config";
+          XDG_CONFIG_HOME = externalStateDir;
         };
 
-      preStart = let
-        replaceSecretCommand = name: attrs:
-          "${getExe pkgs.replace-secret} '${
-            passwordPlaceholder name
-          }' '${attrs.passwordFile}' ${runtimeConfigPath}";
-      in ''
+        preStart =
+          let
+            replaceSecretCommand =
+              name: attrs:
+              "${getExe pkgs.replace-secret} '${passwordPlaceholder name}' '${attrs.passwordFile}' ${runtimeConfigPath}";
+          in
+          ''
             set -euo pipefail
             install -m 600 ${configFile} ${runtimeConfigPath}
-        ${concatStringsSep "\n"
-        (mapAttrsToList replaceSecretCommand cfg.accounts)}
+            ${concatStringsSep "\n" (mapAttrsToList replaceSecretCommand cfg.accounts)}
           '';
 
         serviceConfig = {
           Type = "simple";
-        ExecStart = "${getExe cfg.package} -c ${runtimeConfigPath}";
-
+          ExecStart = command;
           # Hardening options
-        User = "copyparty";
-        Group = "copyparty";
-        RuntimeDirectory = name;
+          User = cfg.user;
+          Group = cfg.group;
+          RuntimeDirectory = [ "copyparty" ];
           RuntimeDirectoryMode = "0700";
-        StateDirectory = [ name "${name}/data" "${name}/.config" ];
+          StateDirectory = [ "copyparty" ];
           StateDirectoryMode = "0700";
-        WorkingDirectory = home;
-        TemporaryFileSystem = "/:ro";
+          CacheDirectory = lib.mkIf (cfg.settings ? hist) [ "copyparty" ];
+          CacheDirectoryMode = lib.mkIf (cfg.settings ? hist) "0700";
+          WorkingDirectory = externalStateDir;
           BindReadOnlyPaths = [
             "/nix/store"
             "-/etc/resolv.conf"
             "-/etc/nsswitch.conf"
+            "-/etc/group"
             "-/etc/hosts"
             "-/etc/localtime"
           ] ++ (mapAttrsToList (k: v: "-${v.passwordFile}") cfg.accounts);
-        BindPaths = [ home ] ++ (mapAttrsToList (k: v: v.path) cfg.volumes);
-        # Would re-mount paths ignored by temporary root
-        #ProtectSystem = "strict";
-        ProtectHome = true;
+          BindPaths =
+            (if cfg.settings ? hist then [ cfg.settings.hist ] else [ ])
+            ++ [ externalStateDir ]
+            ++ (mapAttrsToList (k: v: v.path) volumesWithoutVariables);
+          # ProtectSystem = "strict";
+          # Note that unlike what 'ro' implies,
+          # this actually makes it impossible to read anything in the root FS,
+          # except for things explicitly mounted via `RuntimeDirectory`, `StateDirectory`, `CacheDirectory`, and `BindReadOnlyPaths`.
+          # This is because TemporaryFileSystem creates a *new* *empty* filesystem for the process, so only bindmounts are visible.
+          TemporaryFileSystem = "/:ro";
           PrivateTmp = true;
           PrivateDevices = true;
           ProtectKernelTunables = true;
@@ -269,15 +353,60 @@ in {
           NoNewPrivileges = true;
           LockPersonality = true;
           RestrictRealtime = true;
+          MemoryDenyWriteExecute = true;
         };
       };
 
-    users.groups.copyparty = { };
-    users.users.copyparty = {
+      # ensure volumes exist:
+      systemd.tmpfiles.settings."copyparty" = (
+        lib.attrsets.mapAttrs' (
+          name: value:
+          lib.attrsets.nameValuePair (value.path) {
+            d = {
+              #: in front of things means it wont change it if the directory already exists.
+              group = ":${cfg.group}";
+              user = ":${cfg.user}";
+              mode = ":${
+                # Use volume permissions if set
+                if (value.flags ? chmod_d) then
+                  value.flags.chmod_d
+                # Else, use global permission if set
+                else if (cfg.settings ? chmod-d) then
+                  cfg.settings.chmod-d
+                # Else, use the default permission
+                else
+                  "755"
+              }";
+            };
+          }
+        ) volumesWithoutVariables
+      );
+
+      users.groups = lib.mkIf (cfg.group == "copyparty") {
+        copyparty = { };
+      };
+      users.users = lib.mkIf (cfg.user == "copyparty") {
+        copyparty = {
           description = "Service user for copyparty";
-      group = "copyparty";
-      home = home;
+          group = cfg.group;
+          home = externalStateDir;
           isSystemUser = true;
         };
       };
+      environment.systemPackages = lib.mkIf cfg.mkHashWrapper [
+        (pkgs.writeShellScriptBin "copyparty-hash" ''
+          set -a  # automatically export variables
+          # set same environment variables as the systemd service
+          ${lib.pipe config.systemd.services.copyparty.environment [
+            (lib.filterAttrs (n: v: v != null && n != "PATH"))
+            (lib.mapAttrs (_: v: "${v}"))
+            (lib.toShellVars)
+          ]}
+          PATH=${config.systemd.services.copyparty.environment.PATH}:$PATH
+
+          exec ${command} --ah-cli
+        '')
+      ];
+    }
+  );
 }

contrib/package/arch/PKGBUILD

@@ -1,57 +1,48 @@
 # Maintainer: icxes <dev.null@need.moe>
+# Contributor: Morgan Adamiec <morganamilo@archlinux.org>
+# NOTE: You generally shouldn't use this PKGBUILD on Arch, as it is mainly for testing purposes. Install copyparty using pacman instead.
+
 pkgname=copyparty
-pkgver="1.16.9"
+pkgver="1.19.21"
 pkgrel=1
 pkgdesc="File server with accelerated resumable uploads, dedup, WebDAV, FTP, TFTP, zeroconf, media indexer, thumbnails++"
 arch=("any")
 url="https://github.com/9001/${pkgname}"
 license=('MIT')
-depends=("python" "lsof" "python-jinja")
+depends=("bash" "python" "lsof" "python-jinja")
 makedepends=("python-wheel" "python-setuptools" "python-build" "python-installer" "make" "pigz")
 optdepends=("ffmpeg: thumbnails for videos, images (slower) and audio, music tags"
-            "cfssl: generate TLS certificates on startup (pointless when reverse-proxied)"
+            "cfssl: generate TLS certificates on startup"
             "python-mutagen: music tags (alternative)"
             "python-pillow: thumbnails for images"
             "python-pyvips: thumbnails for images (higher quality, faster, uses more ram)"
-            "libkeyfinder-git: detection of musical keys" 
-            "qm-vamp-plugins: BPM detection" 
+            "libkeyfinder: detection of musical keys"
             "python-pyopenssl: ftps functionality"
             "python-pyzmq: send zeromq messages from event-hooks"
             "python-argon2-cffi: hashed passwords in config"
-            "python-impacket-git: smb support (bad idea)"
 )
 source=("https://github.com/9001/${pkgname}/releases/download/v${pkgver}/${pkgname}-${pkgver}.tar.gz")
-backup=("etc/${pkgname}.d/init" )
-sha256sums=("3e8f3c24c699aa41e0d51db6d781e453979c77abc34c919063b5bddd64d27bb0")
+backup=("etc/${pkgname}/copyparty.conf" )
+sha256sums=("44723a823f218e52aaec6075695973a75b8663c9202c80fd73f48e52c61acd42")
 
 build() {
+    cd "${srcdir}/${pkgname}-${pkgver}/copyparty/web"
+    make
+
     cd "${srcdir}/${pkgname}-${pkgver}"
-    
-    pushd copyparty/web
-    make -j$(nproc)
-    rm Makefile
-    popd
-    
-    python3 -m build -wn
+    python -m build --wheel --no-isolation
 }
 
 package() {
     cd "${srcdir}/${pkgname}-${pkgver}"
-    python3 -m installer -d "$pkgdir" dist/*.whl
+    python -m installer --destdir="$pkgdir" dist/*.whl
 
-    install -dm755 "${pkgdir}/etc/${pkgname}.d"
+    install -dm755 "${pkgdir}/etc/${pkgname}"
     install -Dm755 "bin/prisonparty.sh" "${pkgdir}/usr/bin/prisonparty"
-    install -Dm644 "contrib/package/arch/${pkgname}.conf" "${pkgdir}/etc/${pkgname}.d/init"
-    install -Dm644 "contrib/package/arch/${pkgname}.service" "${pkgdir}/usr/lib/systemd/system/${pkgname}.service"
-    install -Dm644 "contrib/package/arch/prisonparty.service" "${pkgdir}/usr/lib/systemd/system/prisonparty.service"
-    install -Dm644 "contrib/package/arch/index.md" "${pkgdir}/var/lib/${pkgname}-jail/README.md"
+    install -Dm644 "contrib/systemd/${pkgname}.conf" "${pkgdir}/etc/${pkgname}/copyparty.conf"
+    install -Dm644 "contrib/systemd/${pkgname}@.service" "${pkgdir}/usr/lib/systemd/system/${pkgname}@.service"
+    install -Dm644 "contrib/systemd/${pkgname}-user.service" "${pkgdir}/usr/lib/systemd/user/${pkgname}.service"
+    install -Dm644 "contrib/systemd/prisonparty@.service" "${pkgdir}/usr/lib/systemd/system/prisonparty@.service"
+    install -Dm644 "contrib/systemd/index.md" "${pkgdir}/var/lib/${pkgname}-jail/README.md"
     install -Dm644 "LICENSE" "${pkgdir}/usr/share/licenses/${pkgname}/LICENSE"
-
-    find /etc/${pkgname}.d -iname '*.conf' 2>/dev/null | grep -qE . && return
-    echo "┏━━━━━━━━━━━━━━━──-"
-    echo "┃ Configure ${pkgname} by adding .conf files into /etc/${pkgname}.d/"
-    echo "┃ and maybe copy+edit one of the following to /etc/systemd/system/:"
-    echo "┣━♦ /usr/lib/systemd/system/${pkgname}.service   (standard)"
-    echo "┣━♦ /usr/lib/systemd/system/prisonparty.service (chroot)"
-    echo "┗━━━━━━━━━━━━━━━──-"
 }

contrib/package/makedeb-mpr/PKGBUILD

@@ -0,0 +1,44 @@
+# Contributor: Beethoven <beethovenisadog@protonmail.com>
+
+
+pkgname=copyparty
+pkgver=1.19.21
+pkgrel=1
+pkgdesc="File server with accelerated resumable uploads, dedup, WebDAV, FTP, TFTP, zeroconf, media indexer, thumbnails++"
+arch=("any")
+url="https://github.com/9001/${pkgname}"
+license=('MIT')
+depends=("bash" "python3" "lsof" "python3-jinja2")
+makedepends=("python3-wheel" "python3-setuptools" "python3-build" "python3-installer" "make" "pigz")
+optdepends=("ffmpeg: thumbnails for videos, images (slower) and audio, music tags"
+            "golang-cfssl: generate TLS certificates on startup"
+            "python3-mutagen: music tags (alternative)"
+            "python3-pil: thumbnails for images"
+            "python3-openssl: ftps functionality"
+            "python3-zmq: send zeromq messages from event-hooks"
+            "python3-argon2: hashed passwords in config"
+)
+source=("https://github.com/9001/${pkgname}/releases/download/v${pkgver}/${pkgname}-${pkgver}.tar.gz")
+backup=("/etc/${pkgname}.d/init" )
+sha256sums=("44723a823f218e52aaec6075695973a75b8663c9202c80fd73f48e52c61acd42")
+
+build() {
+    cd "${srcdir}/${pkgname}-${pkgver}/copyparty/web"
+    make
+
+    cd "${srcdir}/${pkgname}-${pkgver}"
+    python -m build --wheel --no-isolation
+}
+
+package() {
+    cd "${srcdir}/${pkgname}-${pkgver}"
+    python -m installer --destdir="$pkgdir" dist/*.whl
+
+    install -dm755 "${pkgdir}/etc/${pkgname}.d"
+    install -Dm755 "bin/prisonparty.sh" "${pkgdir}/usr/bin/prisonparty"
+    install -Dm644 "contrib/package/makedeb-mpr/${pkgname}.conf" "${pkgdir}/etc/${pkgname}.d/init"
+    install -Dm644 "contrib/package/makedeb-mpr/${pkgname}.service" "${pkgdir}/usr/lib/systemd/system/${pkgname}.service"
+    install -Dm644 "contrib/package/makedeb-mpr/prisonparty.service" "${pkgdir}/usr/lib/systemd/system/prisonparty.service"
+    install -Dm644 "contrib/package/makedeb-mpr/index.md" "${pkgdir}/var/lib/${pkgname}-jail/README.md"
+    install -Dm644 "LICENSE" "${pkgdir}/usr/share/licenses/${pkgname}/LICENSE"
+}

contrib/package/makedeb-mpr/copyparty.service

@@ -26,7 +26,7 @@ Environment=XDG_CONFIG_HOME=/home/cpp/.config
 ExecStartPre=+/bin/bash -c 'mkdir -p /run/tmpfiles.d/ && echo "x /tmp/pe-copyparty*" > /run/tmpfiles.d/copyparty.conf'
 
 # run copyparty
-ExecStart=/usr/bin/python3 /usr/bin/copyparty -c /etc/copyparty.d/init
+ExecStart=/usr/bin/python3 /usr/local/bin/copyparty -c /etc/copyparty.d/init
 
 [Install]
 WantedBy=multi-user.target

contrib/package/nix/copyparty/default.nix

@@ -1,45 +1,139 @@
-{ lib, stdenv, makeWrapper, fetchurl, utillinux, python, jinja2, impacket, pyftpdlib, pyopenssl, argon2-cffi, pillow, pyvips, pyzmq, ffmpeg, mutagen,
+{
+  lib,
+  buildPythonApplication,
+  fetchurl,
+  util-linux,
+  python,
+  setuptools,
+  jinja2,
+  impacket,
+  pyopenssl,
+  cfssl,
+  argon2-cffi,
+  pillow,
+  pyvips,
+  pyzmq,
+  ffmpeg,
+  mutagen,
+  pyftpdlib,
+  magic,
+  partftpy,
+  fusepy, # for partyfuse
 
-# use argon2id-hashed passwords in config files (sha2 is always available)
-withHashedPasswords ? true,
+  # use argon2id-hashed passwords in config files (sha2 is always available)
+  withHashedPasswords ? true,
 
-# generate TLS certificates on startup (pointless when reverse-proxied)
-withCertgen ? false,
+  # generate TLS certificates on startup (pointless when reverse-proxied)
+  withCertgen ? false,
 
-# create thumbnails with Pillow; faster than FFmpeg / MediaProcessing
-withThumbnails ? true,
+  # create thumbnails with Pillow; faster than FFmpeg / MediaProcessing
+  withThumbnails ? true,
 
-# create thumbnails with PyVIPS; even faster, uses more memory
-# -- can be combined with Pillow to support more filetypes
-withFastThumbnails ? false,
+  # create thumbnails with PyVIPS; even faster, uses more memory
+  # -- can be combined with Pillow to support more filetypes
+  withFastThumbnails ? false,
 
-# enable FFmpeg; thumbnails for most filetypes (also video and audio), extract audio metadata, transcode audio to opus
-# -- possibly dangerous if you allow anonymous uploads, since FFmpeg has a huge attack surface
-# -- can be combined with Thumbnails and/or FastThumbnails, since FFmpeg is slower than both
-withMediaProcessing ? true,
+  # enable FFmpeg; thumbnails for most filetypes (also video and audio), extract audio metadata, transcode audio to opus
+  # -- possibly dangerous if you allow anonymous uploads, since FFmpeg has a huge attack surface
+  # -- can be combined with Thumbnails and/or FastThumbnails, since FFmpeg is slower than both
+  withMediaProcessing ? true,
 
-# if MediaProcessing is not enabled, you probably want this instead (less accurate, but much safer and faster)
-withBasicAudioMetadata ? false,
+  # if MediaProcessing is not enabled, you probably want this instead (less accurate, but much safer and faster)
+  withBasicAudioMetadata ? false,
 
-# send ZeroMQ messages from event-hooks
-withZeroMQ ? true,
+  # send ZeroMQ messages from event-hooks
+  withZeroMQ ? true,
 
-# enable FTPS support in the FTP server
-withFTPS ? false,
+  # enable FTP server
+  withFTP ? true,
 
-# samba/cifs server; dangerous and buggy, enable if you really need it
-withSMB ? false,
+  # enable FTPS support in the FTP server
+  withFTPS ? false,
 
+  # enable TFTP server
+  withTFTP ? false,
+
+  # samba/cifs server; dangerous and buggy, enable if you really need it
+  withSMB ? false,
+
+  # enables filetype detection for nameless uploads
+  withMagic ? false,
+
+  # extra packages to add to the PATH
+  extraPackages ? [ ],
+
+  # function that accepts a python packageset and returns a list of packages to
+  # be added to the python venv. useful for scripts and such that require
+  # additional dependencies
+  extraPythonPackages ? (_p: [ ]),
+
+  # to build stable + unstable with the same file
+  stable ? true,
+
+  # for commit date, only used when stable = false
+  copypartyFlake ? null,
+
+  nix-gitignore,
 }:
 
 let
   pinData = lib.importJSON ./pin.json;
-  pyEnv = python.withPackages (ps:
-    with ps; [
+  runtimeDeps = ([ util-linux ] ++ extraPackages ++ lib.optional withMediaProcessing ffmpeg);
+  inherit (copypartyFlake) lastModifiedDate;
+  # ex: "1970" "01" "01"
+  dateStringsZeroPrefixed = {
+    year = builtins.substring 0 4 lastModifiedDate;
+    month = builtins.substring 4 2 lastModifiedDate;
+    day = builtins.substring 6 2 lastModifiedDate;
+  };
+  # ex: "1970" "1" "1"
+  dateStringsShort = builtins.mapAttrs (_: val: toString (lib.toIntBase10 val)) dateStringsZeroPrefixed;
+  unstableVersion =
+    if copypartyFlake == null then
+      "${pinData.version}-unstable"
+    else
+      with dateStringsZeroPrefixed; "${pinData.version}-unstable-${year}-${month}-${day}"
+  ;
+  version = if stable then pinData.version else unstableVersion;
+  stableSrc = fetchurl {
+    inherit (pinData) url hash;
+  };
+  root = ../../../..;
+  unstableSrc = nix-gitignore.gitignoreSource [] root;
+  src = if stable then stableSrc else unstableSrc;
+  rev = copypartyFlake.shortRev or copypartyFlake.dirtyShortRev or "unknown";
+  unstableCodename = "unstable" + (lib.optionalString (copypartyFlake != null) "-${rev}");
+in
+buildPythonApplication {
+  pname = "copyparty";
+  inherit version src;
+  postPatch = lib.optionalString (!stable) ''
+    old_src="$(mktemp -d)"
+    tar -C "$old_src" -xf ${stableSrc}
+    declare -a folders
+    folders=("$old_src"/*)
+    count_folders="''${#folders[@]}"
+    if [[ $count_folders != 1 ]]; then
+      declare -p folders
+      echo "Expected 1 folder, found $count_folders" >&2
+      exit 1
+    fi
+    old_src_folder="''${folders[0]}"
+    cp -r "$old_src_folder"/copyparty/web/deps copyparty/web/deps
+    sed -i 's/^CODENAME =.*$/CODENAME = "${unstableCodename}"/' copyparty/__version__.py
+    ${lib.optionalString (copypartyFlake != null) (with dateStringsShort; ''
+      sed -i 's/^BUILD_DT =.*$/BUILD_DT = (${year}, ${month}, ${day})/' copyparty/__version__.py
+    '')}
+  '';
+  dependencies =
+    [
       jinja2
+      fusepy
     ]
     ++ lib.optional withSMB impacket
+    ++ lib.optional withFTP pyftpdlib
     ++ lib.optional withFTPS pyopenssl
+    ++ lib.optional withTFTP partftpy
     ++ lib.optional withCertgen cfssl
     ++ lib.optional withThumbnails pillow
     ++ lib.optional withFastThumbnails pyvips
@@ -47,21 +141,24 @@ let
     ++ lib.optional withBasicAudioMetadata mutagen
     ++ lib.optional withHashedPasswords argon2-cffi
     ++ lib.optional withZeroMQ pyzmq
-    );
-in stdenv.mkDerivation {
-  pname = "copyparty";
-  version = pinData.version;
-  src = fetchurl {
-    url = pinData.url;
-    hash = pinData.hash;
-  };
-  buildInputs = [ makeWrapper ];
-  dontUnpack = true;
-  dontBuild = true;
-  installPhase = ''
-    install -Dm755 $src $out/share/copyparty-sfx.py
-    makeWrapper ${pyEnv.interpreter} $out/bin/copyparty \
-      --set PATH '${lib.makeBinPath ([ utillinux ] ++ lib.optional withMediaProcessing ffmpeg)}:$PATH' \
-      --add-flags "$out/share/copyparty-sfx.py"
+    ++ lib.optional withMagic magic
+    ++ (extraPythonPackages python.pkgs);
+  makeWrapperArgs = [ "--prefix PATH : ${lib.makeBinPath runtimeDeps}" ];
+
+  pyproject = true;
+  build-system = [
+    setuptools
+  ];
+  meta = {
+    description = "Turn almost any device into a file server";
+    longDescription = ''
+      Portable file server with accelerated resumable uploads, dedup, WebDAV,
+      FTP, TFTP, zeroconf, media indexer, thumbnails++ all in one file, no deps
     '';
+    homepage = "https://github.com/9001/copyparty";
+    changelog = "https://github.com/9001/copyparty/releases/tag/v${pinData.version}";
+    license = lib.licenses.mit;
+    mainProgram = "copyparty";
+    sourceProvenance = [ lib.sourceTypes.fromSource ];
+  };
 }

contrib/package/nix/copyparty/pin.json

@@ -1,5 +1,5 @@
 {
-    "url": "https://github.com/9001/copyparty/releases/download/v1.16.9/copyparty-sfx.py",
-    "version": "1.16.9",
-    "hash": "sha256-456L3IHzf8ups3L9pTBZJMQjML8AlsQI66HZohDyEIA="
+    "url": "https://github.com/9001/copyparty/releases/download/v1.19.21/copyparty-1.19.21.tar.gz",
+    "version": "1.19.21",
+    "hash": "sha256-RHI6gj8hjlKq7GB1aVlzp1uGY8kgLID9c/SOUsYazUI="
 }

contrib/package/nix/copyparty/update.py

@@ -11,14 +11,14 @@ import base64
 import json
 import hashlib
 import sys
-import re
+import tarfile
 from pathlib import Path
 
 OUTPUT_FILE = Path("pin.json")
-TARGET_ASSET = "copyparty-sfx.py"
+TARGET_ASSET = lambda version: f"copyparty-{version}.tar.gz"
 HASH_TYPE = "sha256"
 LATEST_RELEASE_URL = "https://api.github.com/repos/9001/copyparty/releases/latest"
-DOWNLOAD_URL = lambda version: f"https://github.com/9001/copyparty/releases/download/v{version}/{TARGET_ASSET}"
+DOWNLOAD_URL = lambda version: f"https://github.com/9001/copyparty/releases/download/v{version}/{TARGET_ASSET(version)}"
 
 
 def get_formatted_hash(binary):
@@ -29,11 +29,13 @@ def get_formatted_hash(binary):
     return f"{HASH_TYPE}-{encoded_hash}"
 
 
-def version_from_sfx(binary):
-    result = re.search(b'^VER = "(.*)"$', binary, re.MULTILINE)
-    if result:
-        return result.groups(1)[0].decode("ascii")
+def version_from_tar_gz(path):
+    with tarfile.open(path) as tarball:
+        release_name = tarball.getmembers()[0].name
+        prefix = "copyparty-"
 
+        if release_name.startswith(prefix):
+            return release_name.replace(prefix, "")
     raise ValueError("version not found in provided file")
 
 
@@ -42,7 +44,7 @@ def remote_release_pin():
 
     response = requests.get(LATEST_RELEASE_URL).json()
     version = response["tag_name"].lstrip("v")
-    asset_info = [a for a in response["assets"] if a["name"] == TARGET_ASSET][0]
+    asset_info = [a for a in response["assets"] if a["name"] == TARGET_ASSET(version)][0]
     download_url = asset_info["browser_download_url"]
     asset = requests.get(download_url)
     formatted_hash = get_formatted_hash(asset.content)
@@ -52,10 +54,9 @@ def remote_release_pin():
 
 
 def local_release_pin(path):
-    asset = path.read_bytes()
-    version = version_from_sfx(asset)
+    version = version_from_tar_gz(path)
     download_url = DOWNLOAD_URL(version)
-    formatted_hash = get_formatted_hash(asset)
+    formatted_hash = get_formatted_hash(path.read_bytes())
 
     result = {"url": download_url, "version": version, "hash": formatted_hash}
     return result

contrib/package/nix/overlay.nix

@@ -0,0 +1,31 @@
+final: prev:
+let
+  fullAttrs = {
+    withHashedPasswords = true;
+    withCertgen = true;
+    withThumbnails = true;
+    withFastThumbnails = true;
+    withMediaProcessing = true;
+    withBasicAudioMetadata = true;
+    withZeroMQ = true;
+    withFTP = true;
+    withFTPS = true;
+    withTFTP = true;
+    withSMB = true;
+    withMagic = true;
+  };
+
+  call = attrs: final.python3.pkgs.callPackage ./copyparty ({ ffmpeg = final.ffmpeg-full; } // attrs);
+in
+{
+  copyparty = call { stable = true; };
+  copyparty-unstable = call { stable = false; };
+  copyparty-full = call (fullAttrs // { stable = true; });
+  copyparty-unstable-full = call (fullAttrs // { stable = false; });
+
+  python3 = prev.python3.override {
+    packageOverrides = pyFinal: pyPrev: {
+      partftpy = pyFinal.callPackage ./partftpy { };
+    };
+  };
+}

contrib/package/nix/partftpy/default.nix

@@ -0,0 +1,30 @@
+{
+  lib,
+  buildPythonPackage,
+  fetchurl,
+  setuptools,
+}:
+let
+  pinData = lib.importJSON ./pin.json;
+in
+
+buildPythonPackage rec {
+  pname = "partftpy";
+  inherit (pinData) version;
+  pyproject = true;
+
+  src = fetchurl {
+    inherit (pinData) url hash;
+  };
+
+  build-system = [ setuptools ];
+
+  pythonImportsCheck = [ "partftpy.TftpServer" ];
+
+  meta = {
+    description = "Pure Python TFTP library  (copyparty edition)";
+    homepage = "https://github.com/9001/partftpy";
+    changelog = "https://github.com/9001/partftpy/releases/tag/${version}";
+    license = lib.licenses.mit;
+  };
+}

contrib/package/nix/partftpy/pin.json

@@ -0,0 +1,5 @@
+{
+    "url": "https://github.com/9001/partftpy/releases/download/v0.4.0/partftpy-0.4.0.tar.gz",
+    "version": "0.4.0",
+    "hash": "sha256-5Q2zyuJ892PGZmb+YXg0ZPW/DK8RDL1uE0j5HPd4We0="
+}

contrib/package/nix/partftpy/update.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+
+# Update the Nix package pin
+#
+# Usage: ./update.sh
+
+import base64
+import json
+import hashlib
+import sys
+from pathlib import Path
+
+OUTPUT_FILE = Path("pin.json")
+TARGET_ASSET = lambda version: f"partftpy-{version}.tar.gz"
+HASH_TYPE = "sha256"
+LATEST_RELEASE_URL = "https://api.github.com/repos/9001/partftpy/releases/latest"
+
+
+def get_formatted_hash(binary):
+    hasher = hashlib.new("sha256")
+    hasher.update(binary)
+    asset_hash = hasher.digest()
+    encoded_hash = base64.b64encode(asset_hash).decode("ascii")
+    return f"{HASH_TYPE}-{encoded_hash}"
+
+
+def remote_release_pin():
+    import requests
+
+    response = requests.get(LATEST_RELEASE_URL).json()
+    version = response["tag_name"].lstrip("v")
+    asset_info = [a for a in response["assets"] if a["name"] == TARGET_ASSET(version)][0]
+    download_url = asset_info["browser_download_url"]
+    asset = requests.get(download_url)
+    formatted_hash = get_formatted_hash(asset.content)
+
+    result = {"url": download_url, "version": version, "hash": formatted_hash}
+    return result
+
+
+def main():
+    result = remote_release_pin()
+
+    print(result)
+    json_result = json.dumps(result, indent=4)
+    OUTPUT_FILE.write_text(json_result)
+
+
+if __name__ == "__main__":
+    main()

contrib/package/rpm/copyparty.spec

@@ -0,0 +1,62 @@
+Name:           copyparty
+Version:        $pkgver
+Release:        $pkgrel
+License:        MIT
+Group:          Utilities
+URL:            https://github.com/9001/copyparty
+Source0:        copyparty-$pkgver.tar.gz
+Summary:        File server with accelerated resumable uploads, dedup, WebDAV, FTP, TFTP, zeroconf, media indexer, thumbnails++
+BuildArch:      noarch
+BuildRequires:  python3, python3-devel, pyproject-rpm-macros, python-setuptools, python-wheel, make
+Requires:       python3, (python3-jinja2 or python-jinja2), lsof
+Recommends:     ffmpeg, (golang-github-cloudflare-cfssl or cfssl), python-mutagen, python-pillow, python-pyvips
+Recommends:     qm-vamp-plugins, python-argon2-cffi, (python-pyopenssl or pyopenssl), python-impacket
+
+%description
+Portable file server with accelerated resumable uploads, dedup, WebDAV, FTP, TFTP, zeroconf, media indexer, thumbnails++ all in one file, no deps
+
+See release at https://github.com/9001/copyparty/releases
+
+%global debug_package %{nil}
+
+%generate_buildrequires
+%pyproject_buildrequires
+
+%prep
+%setup -q
+
+%build
+cd "copyparty/web"
+make
+cd -
+%pyproject_wheel
+
+%install
+mkdir -p %{buildroot}%{_bindir}
+mkdir -p %{buildroot}%{_libdir}/systemd/{system,user}
+mkdir -p %{buildroot}/etc/%{name}
+mkdir -p %{buildroot}/var/lib/%{name}-jail
+mkdir -p %{buildroot}%{_datadir}/licenses/%{name}
+
+%pyproject_install
+%pyproject_save_files copyparty
+
+install -m 0755 bin/prisonparty.sh                       %{buildroot}%{_bindir}/prisonpary.sh
+install -m 0644 contrib/systemd/%{name}.conf             %{buildroot}/etc/%{name}/%{name}.conf
+install -m 0644 contrib/systemd/%{name}@.service         %{buildroot}%{_libdir}/systemd/system/%{name}@.service
+install -m 0644 contrib/systemd/%{name}-user.service     %{buildroot}%{_libdir}/systemd/user/%{name}.service
+install -m 0644 contrib/systemd/prisonparty@.service     %{buildroot}%{_libdir}/systemd/system/prisonparty@.service
+install -m 0644 contrib/systemd/index.md                 %{buildroot}/var/lib/%{name}-jail/README.md
+install -m 0644 LICENSE                                  %{buildroot}%{_datadir}/licenses/%{name}/LICENSE
+
+%files -n copyparty -f %{pyproject_files}
+%license LICENSE
+%{_bindir}/copyparty
+%{_bindir}/partyfuse
+%{_bindir}/u2c
+%{_bindir}/prisonpary.sh
+/etc/%{name}/%{name}.conf
+%{_libdir}/systemd/system/%{name}@.service
+%{_libdir}/systemd/user/%{name}.service
+%{_libdir}/systemd/system/prisonparty@.service
+/var/lib/%{name}-jail/README.md

contrib/plugins/README.md

@@ -15,6 +15,7 @@ save one of these as `.epilogue.html` inside a folder to customize it:
 point `--js-browser` to one of these by URL:
 
 * [`minimal-up2k.js`](minimal-up2k.js) is similar to the above `minimal-up2k.html` except it applies globally to all write-only folders
+* [`quickmove.js`](quickmove.js) adds a hotkey to move selected files into a subfolder
 * [`up2k-hooks.js`](up2k-hooks.js) lets you specify a ruleset for files to skip uploading
   * [`up2k-hook-ytid.js`](up2k-hook-ytid.js) is a more specific example checking youtube-IDs against some API
 
@@ -38,3 +39,9 @@ point `--css-browser` to one of these by URL:
 
 * turns copyparty into chromecast just more flexible (and probably way more buggy)
 * usage: put the js somewhere in the webroot and `--js-browser /memes/meadup.js`
+
+
+
+# junk
+
+* [**rave.js**](./rave.js): april-fools joke, [demo (epilepsy warning)](https://cd.ocv.me/b/d2/d21/#af-9b927c42,sorthref), not maintained, very buggy

contrib/plugins/graft-thumbs.js

@@ -0,0 +1,117 @@
+// USAGE:
+//   place this file somewhere in the webroot and then
+//   python3 -m copyparty --js-browser /.res/graft-thumbs.js
+//
+// DESCRIPTION:
+//   this is a gridview plugin which, for each file in a folder,
+//   looks for another file with the same filename (but with a
+//   different file extension)
+//
+//   if one of those files is an image and the other is not,
+//   then this plugin assumes the image is a "sidecar thumbnail"
+//   for the other file, and it will graft the image thumbnail
+//   onto the non-image file (for example an mp3)
+//
+//   optional feature 1, default-enabled:
+//   the image-file is then hidden from the directory listing
+//
+//   optional feature 2, default-enabled:
+//   when clicking the audio file, the image will also open
+
+
+(function() {
+
+	// `graft_thumbs` assumes the gridview has just been rendered;
+	// it looks for sidecars, and transplants those thumbnails onto
+	// the other file with the same basename (filename sans extension)
+
+	var graft_thumbs = function () {
+		if (!thegrid.en)
+			return;  // not in grid mode
+
+		var files = msel.getall(),
+			pairs = {};
+
+		console.log(files);
+
+		for (var a = 0; a < files.length; a++) {
+			var file = files[a],
+				is_pic = /\.(jpe?g|png|gif|webp)$/i.exec(file.vp),
+				is_audio = re_au_all.exec(file.vp),
+				basename = file.vp.replace(/\.[^\.]+$/, ""),
+				entry = pairs[basename];
+
+			if (!entry)
+				// first time seeing this basename; create a new entry in pairs
+				entry = pairs[basename] = {};
+
+			if (is_pic)
+				entry.thumb = file;
+			else if (is_audio)
+				entry.audio = file;
+		}
+
+		var basenames = Object.keys(pairs);
+		for (var a = 0; a < basenames.length; a++)
+			(function(a) {
+				var pair = pairs[basenames[a]];
+
+				if (!pair.thumb || !pair.audio)
+					return;  // not a matching pair of files
+
+				var img_thumb = QS('#ggrid a[ref="' + pair.thumb.id + '"] img[onload]'),
+					img_audio = QS('#ggrid a[ref="' + pair.audio.id + '"] img[onload]');
+
+				if (!img_thumb || !img_audio)
+					return;  // something's wrong... let's bail
+
+				// alright, graft the thumb...
+				img_audio.src = img_thumb.src;
+
+				// ...and hide the sidecar
+				img_thumb.closest('a').style.display = 'none';
+
+				// ...and add another onclick-handler to the audio,
+				// so it also opens the pic while playing the song
+				img_audio.addEventListener('click', function() {
+					img_thumb.click();
+					return false;  // let it bubble to the next listener
+				});
+
+			})(a);
+	};
+
+	// ...and then the trick! near the end of loadgrid,
+	// thegrid.bagit is called to initialize the baguettebox
+	// (image/video gallery); this is the perfect function to
+	// "hook" (hijack) so we can run our code :^)
+
+	// need to grab a backup of the original function first,
+	var orig_func = thegrid.bagit;
+
+	// and then replace it with our own:
+	thegrid.bagit = function (isrc) {
+
+		if (isrc !== '#ggrid')
+			// we only want to modify the grid, so
+			// let the original function handle this one
+			return orig_func(isrc);
+
+		graft_thumbs();
+
+		// when changing directories, the grid is
+		// rendered before msel returns the correct
+		// filenames, so schedule another run:
+		setTimeout(graft_thumbs, 1);
+
+		// and finally, call the original thegrid.bagit function
+		return orig_func(isrc);
+	};
+
+	if (ls0) {
+		// the server included an initial listing json (ls0),
+		// so the grid has already been rendered without our hook
+		graft_thumbs();
+	}
+
+})();

contrib/plugins/minimal-up2k.js

@@ -12,6 +12,23 @@ almost the same as minimal-up2k.html except this one...:
 
  -- looks slightly better
 
+
+========================
+== USAGE INSTRUCTIONS ==
+
+1. create a volume which anyone can read from (if you haven't already)
+2. copy this file into that volume, so anyone can download it
+3. enable the plugin by telling the webbrowser to load this file;
+    assuming the URL to the public volume is /res/, and
+    assuming you're using config-files, then add this to your config:
+
+    [global]
+      js-browser: /res/minimal-up2k.js
+
+alternatively, if you're not using config-files, then
+add the following commandline argument instead:
+  --js-browser=/res/minimal-up2k.js
+
 */
 
 var u2min = `

contrib/plugins/quickmove.js

@@ -0,0 +1,140 @@
+"use strict";
+
+
+// USAGE:
+//   place this file somewhere in the webroot, 
+//   for example in a folder named ".res" to hide it, and then
+//   python3 copyparty-sfx.py -v .::A --js-browser /.res/quickmove.js
+//
+// DESCRIPTION:
+//   the command above launches copyparty with one single volume;
+//   ".::A" = current folder as webroot, and everyone has Admin
+//
+//   the plugin adds hotkey "W" which moves all selected files
+//   into a subfolder named "foobar" inside the current folder
+
+
+(function() {
+
+    var action_to_perform = ask_for_confirmation_and_then_move;
+    // this decides what the new hotkey should do;
+    //  ask_for_confirmation_and_then_move  =  show a yes/no box,
+    //  move_selected_files  =  just move the files immediately
+
+    var move_destination = "foobar";
+    // this is the target folder to move files to;
+    // by default it is a subfolder of the current folder,
+    // but it can also be an absolute path like "/foo/bar"
+
+    // ===
+    // ===   END OF CONFIG
+    // ===
+
+    var main_hotkey_handler,  // copyparty's original hotkey handler
+        plugin_enabler,  // timer to engage this plugin when safe
+        files_to_move;  // list of files to move
+
+    function ask_for_confirmation_and_then_move() {
+        var num_files = msel.getsel().length,
+            msg = "move the selected " + num_files + " files?";
+
+        if (!num_files)
+            return toast.warn(2, 'no files were selected to be moved');
+
+        modal.confirm(msg, move_selected_files, null);
+    }
+
+    function move_selected_files() {
+        var selection = msel.getsel();
+
+        if (!selection.length)
+            return toast.warn(2, 'no files were selected to be moved');
+
+        if (thegrid.bbox) {
+            // close image/video viewer
+            thegrid.bbox = null;
+            baguetteBox.destroy();
+        }
+
+        files_to_move = [];
+        for (var a = 0; a < selection.length; a++)
+            files_to_move.push(selection[a].vp);
+
+        move_next_file();
+    }
+
+    function move_next_file() {
+        var num_files = files_to_move.length,
+            filepath = files_to_move.pop(),
+            filename = vsplit(filepath)[1];
+
+        toast.inf(10, "moving " + num_files + " files...\n\n" + filename);
+
+        var dst = move_destination;
+
+        if (!dst.endsWith('/'))
+            // must have a trailing slash, so add it
+            dst += '/';
+
+        if (!dst.startsWith('/'))
+            // destination is a relative path, so prefix current folder path
+            dst = get_evpath() + dst;
+
+        // and finally append the filename
+        dst += '/' + filename;
+
+        // prepare the move-request to be sent
+        var xhr = new XHR();
+        xhr.onload = xhr.onerror = function() {
+            if (this.status !== 201)
+                return toast.err(30, 'move failed: ' + esc(this.responseText));
+
+            if (files_to_move.length)
+                return move_next_file();  // still more files to go
+
+            toast.ok(1, 'move OK');
+            treectl.goto(); // reload the folder contents
+        };
+        xhr.open('POST', filepath + '?move=' + dst);
+        xhr.send();
+    }
+
+    function our_hotkey_handler(e) {
+        // bail if either ALT, CTRL, or SHIFT is pressed
+        if (anymod(e))
+            return main_hotkey_handler(e);  // let copyparty handle this keystroke
+
+        var keycode = (e.key || e.code) + '',
+    		ae = document.activeElement,
+		    aet = ae && ae != document.body ? ae.nodeName.toLowerCase() : '';
+
+        // check the current aet (active element type),
+        // only continue if one of the following currently has input focus:
+        //   nothing | link | button | table-row | table-cell | div | text
+        if (aet && !/^(a|button|tr|td|div|pre)$/.test(aet))
+            return main_hotkey_handler(e);  // let copyparty handle this keystroke
+
+        if (keycode == 'w' || keycode == 'KeyW') {
+            // okay, this one's for us... do the thing
+            action_to_perform();
+            return ev(e);
+        }
+
+        return main_hotkey_handler(e);  // let copyparty handle this keystroke
+    }
+
+    function enable_plugin() {
+        if (!window.hotkeys_attached)
+            return console.log('quickmove is waiting for the page to finish loading');
+
+        clearInterval(plugin_enabler);
+        main_hotkey_handler = document.onkeydown;
+        document.onkeydown = our_hotkey_handler;
+        console.log('quickmove is now enabled');
+    }
+
+    // copyparty doesn't enable its hotkeys until the page
+    // has finished loading, so we'll wait for that too
+    plugin_enabler = setInterval(enable_plugin, 100);
+
+})();

contrib/podman-systemd/README.md

@@ -0,0 +1,173 @@
+# copyparty with Podman and Systemd
+
+Use this configuration if you want to run copyparty in a Podman container, with the reliability of running the container under a systemd service.
+
+Documentation for `.container` files can be found in the [Container unit](https://docs.podman.io/en/latest/markdown/podman-systemd.unit.5.html#container-units-container) docs. Systemd does not understand `.container` files natively, so Podman converts these to `.service` files with a [systemd-generator](https://www.freedesktop.org/software/systemd/man/latest/systemd.generator.html). This process is transparent, but sometimes needs to be debugged in case your `.container` file is malformed. There are instructions to debug the systemd generator in the Troubleshooting section below.
+
+To run copyparty in this way, you must already have podman installed. To install Podman, see: https://podman.io/docs/installation
+
+There is a sample configuration file in the same directory as this file (`copyparty.conf`).
+
+## Run the container as root
+
+Running the container as the root user is easy to set up, but less secure. There are instructions in the next section to run the container as a rootless user if you'd rather run the container like that.
+
+First, change this line in the `copyparty.container` file to reflect the directory you want to share. By default, it shares `/mnt/` but you'll probably want to change that.
+
+```
+# Change /mnt to something you want to share
+Volume=/mnt:/w:z
+```
+
+Note that you can select the owner and group of this volume by changing the `uid:` and `gid:` of the volume in `copyparty.conf`, but for simplicity let's assume you want it to be owned by `root:root`.
+
+To install and start copyparty with Podman and systemd as the root user, run the following:
+
+```shell
+sudo mkdir -pv /etc/systemd/container/ /etc/copyparty/
+sudo cp -v copyparty.container /etc/systemd/containers/
+sudo cp -v copyparty.conf /etc/copyparty/
+sudo systemctl daemon-reload
+sudo systemctl start copyparty
+```
+
+Note: You can't "enable" this kind of Podman service. The `[Install]` section of the `.container` file effectively handles enabling the service so that it starts when the server reboots.
+
+You can see the status of the service with:
+
+```shell
+sudo systemctl status -a copyparty
+```
+
+You can see (and follow) the logs with either of these commands:
+
+```shell
+sudo podman logs -f copyparty
+
+# -a is required or else you'll get output like: copyparty[549025]: [649B blob data]
+sudo journalctl -a -f -u copyparty
+```
+
+## Run the container as a non-root user
+
+This configuration is more secure, but is more involved and requires ensuring files have proper permissions. You will need a root user account to do some of this setup.
+
+First, you need a user to run the container as. In this example we'll create a "podman" user with UID=1001 and GID=1001.
+
+```shell
+sudo groupadd -g 1001 podman
+sudo useradd -u 1001 -m podman
+sudo usermod -aG podman podman
+sudo loginctl enable-linger podman
+# Set a strong password for this user
+sudo -u podman passwd
+```
+
+The `enable-linger` command allows the podman user to run systemd user services that persist even when the user is not logged in. You could use a user that already exists in the system to run this service as, just make sure to run `loginctl enable-linger USERNAME` for that user.
+
+Next, change these lines in the `copyparty.container` file to reflect the config directory and the directory you want to share. By default, the config shares `/home/podman/copyparty/sharing/` but you'll probably want to change this:
+
+```
+# Change to reflect your non-root user's home directory
+Volume=/home/podman/copyparty/config:/cfg:z
+
+# Change to the directory you want to share
+Volume=/home/podman/copyparty/sharing:/w:z
+```
+
+Make sure the podman user has read/write access to both of these directories.
+
+Next, **log in to the server as the podman user**.
+
+To install and start copyparty as the non-root podman user, run the following:
+
+```shell
+mkdir -pv /home/podman/.config/containers/systemd/ /home/podman/copyparty/config
+cp -v copyparty.container /home/podman/.config/containers/systemd/copyparty.container
+cp -v copyparty.conf /home/podman/copyparty/config
+systemctl --user daemon-reload
+systemctl --user start copyparty
+```
+
+**Important note: Never use `sudo` with `systemctl --user`!**
+
+You can check the status of the user service with:
+
+```shell
+systemctl --user status -a copyparty
+```
+
+You can see (and follow) the logs with:
+
+```shell
+podman logs -f copyparty
+
+journalctl --user -a -f -u copyparty
+```
+
+## Troubleshooting
+
+If the container fails to start, and you've modified the `.container` service, it's likely that your `.container` file failed to be translated into a `.service` file. You can debug the podman service generator with this command:
+
+```shell
+sudo /usr/lib/systemd/system-generators/podman-system-generator --dryrun
+```
+
+## Allowing Traffic from Outside your Server
+
+To allow traffic on port 3923 of your server, you should run:
+
+```shell
+sudo firewall-cmd --permanent --add-port=3923/tcp
+sudo firewall-cmd --reload
+```
+
+Otherwise, you won't be able to access the copyparty server from anywhere other than the server itself.
+
+## Updating copyparty
+
+To update the version of copyparty used in the container, you can:
+
+```shell
+# If root:
+sudo podman pull docker.io/copyparty/ac:latest
+sudo systemctl restart copyparty
+
+# If non-root:
+podman pull docker.io/copyparty/ac:latest
+systemctl --user restart copyparty
+```
+
+Or, you can change the pinned version of the image in the `[Container]` section of the `.container` file and run:
+
+```shell
+# If root:
+sudo systemctl daemon-reload
+sudo systemctl restart copyparty
+
+# If non-root:
+systemctl --user daemon-reload
+systemctl --user restart copyparty
+```
+
+Podman will pull the image you've specified when restarting. If you have it set to `:latest`, Podman does not know to re-pull the container.
+
+### Enabling auto-update
+
+Alternatively, you can enable auto-updates by un-commenting this line:
+
+```
+# AutoUpdate=registry
+```
+
+You will also need to enable the [podman auto-updater service](https://docs.podman.io/en/latest/markdown/podman-auto-update.1.html) with:
+
+```shell
+# If root:
+sudo systemctl enable podman-auto-update.timer podman-auto-update.service
+
+# If non-root:
+systemctl --user enable podman-auto-update.timer podman-auto-update.service
+```
+
+This works best if you always want the latest version of copyparty. The auto-updater runs once every 24 hours.

contrib/podman-systemd/copyparty.conf

@@ -0,0 +1,36 @@
+[global]
+  e2dsa  # enable file indexing and filesystem scanning
+  e2ts   # and enable multimedia indexing
+  ansi   # and colors in log messages
+
+  # uncomment the line starting with q, lo: to log to a file instead of stdout/journalctl;
+  # $LOGS_DIRECTORY is usually /var/log/copyparty (comes from systemd)
+  # and copyparty replaces %Y-%m%d with Year-MonthDay, so the
+  # full path will be something like /var/log/copyparty/2023-1130.txt
+  # (note: enable compression by adding .xz at the end)
+  # q, lo: $LOGS_DIRECTORY/%Y-%m%d.log
+
+  # p: 80,443,3923   # listen on 80/443 as well (requires CAP_NET_BIND_SERVICE)
+  # i: 127.0.0.1     # only allow connections from localhost (reverse-proxies)
+  # ftp: 3921        # enable ftp server on port 3921
+  # p: 3939          # listen on another port
+  # df: 16           # stop accepting uploads if less than 16 GB free disk space
+  # ver              # show copyparty version in the controlpanel
+  # grid             # show thumbnails/grid-view by default
+  # theme: 2         # monokai
+  # name: datasaver  # change the server-name that's displayed in the browser
+  # stats, nos-dup   # enable the prometheus endpoint, but disable the dupes counter (too slow)
+  # no-robots, force-js  # make it harder for search engines to read your server
+
+
+[accounts]
+  ed: wark  # username: password
+
+
+[/]            # create a volume at "/" (the webroot), which will
+  /w           # share the contents of the "/w" folder
+  accs:
+    rw: *      # everyone gets read-write access, but
+    rwmda: ed  # the user "ed" gets read-write-move-delete-admin
+    # uid: 1000  # If you're running as root, you can change the owner of this volume here
+    # gid: 1000  # If you're running as root, you can change the group of this volume here

contrib/podman-systemd/copyparty.container

@@ -0,0 +1,55 @@
+[Container]
+# It's recommended to replace :latest with a specific version
+# for example: docker.io/copyparty/ac:1.19.15
+Image=docker.io/copyparty/ac:latest
+ContainerName=copyparty
+
+# Uncomment to enable auto-updates
+# AutoUpdate=registry
+
+# Environment variables
+# enable mimalloc by replacing "NOPE" with "2" for a nice speed-boost (will use twice as much ram)
+Environment=LD_PRELOAD=/usr/lib/libmimalloc-secure.so.NOPE
+# ensures log-messages are not delayed (but can reduce speed a tiny bit)
+Environment=PYTHONUNBUFFERED=1
+
+# Ports
+PublishPort=3923:3923
+
+
+# Volumes (PLEASE LOOK!)
+
+# Rootful setup:
+#   Leave as-is
+# Non-root setup:
+#   Change /etc/copyparty to /home/<USER>/copyparty/config
+Volume=/etc/copyparty:/cfg:z
+
+# Rootful setup:
+#   Change /mnt to the directory you want to share
+# Non-root setup:
+#   Change /mnt to something owned by your user, e.g., /home/<USER>/copyparty/sharing:/w:z
+Volume=/mnt:/w:z
+
+
+# Give the container time to stop in case the thumbnailer is still running.
+# It's allowed to continue finishing up for 10s after the shutdown signal, give it a 5s buffer
+StopTimeout=15
+
+# hide it from logs with "/._" so it matches the default --lf-url filter
+HealthCmd="wget --spider -q 127.0.0.1:3923/?reset=/._"
+HealthInterval=1m
+HealthTimeout=2s
+HealthRetries=5
+HealthStartPeriod=15s
+
+[Unit]
+After=default.target
+
+[Install]
+# Start by default on boot
+WantedBy=default.target
+
+[Service]
+# Give the container time to start in case it needs to pull the image
+TimeoutStartSec=600

contrib/setup-ashell.sh

@@ -0,0 +1,71 @@
+#!/bin/bash
+#
+# this script will install copyparty onto an iOS device (iPhone/iPad)
+#
+# step 1: install a-Shell:
+#   https://apps.apple.com/us/app/a-shell/id1473805438
+#
+# step 2: copypaste the following command into a-Shell:
+#   curl -L https://github.com/9001/copyparty/raw/refs/heads/hovudstraum/contrib/setup-ashell.sh
+#
+# step 3: launch copyparty with this command: cpp
+#
+# if you ever want to upgrade copyparty, just repeat step 2
+
+
+
+cd "$HOME/Documents"
+curl -Locopyparty https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.py
+
+
+
+# create the config file? (cannot use heredoc because body too large)
+[ -e cpc ] || {
+echo '[global]' >cpc
+echo '  p: 80, 443, 3923  # enable http and https on these ports' >>cpc
+echo '  e2dsa      # enable file indexing and filesystem scanning' >>cpc
+echo '  e2ts       # and enable multimedia indexing' >>cpc
+echo '  ver        # show copyparty version in the controlpanel' >>cpc
+echo '  qrz: 2     # enable qr-code and make it big' >>cpc
+echo '  qrp: 1     # reduce qr-code padding' >>cpc
+echo '  qr-fg: -1  # optimize for basic/simple terminals' >>cpc
+echo '  qr-wait: 0.3  # less chance of getting scrolled away' >>cpc
+echo '' >>cpc
+echo '  # enable these by uncommenting them:' >>cpc
+echo '  # ftp: 21    # enable ftp server on port 21' >>cpc
+echo '  # tftp: 69   # enable tftp server on port 69' >>cpc
+echo '' >>cpc
+echo '[/]' >>cpc
+echo '  ~/Documents' >>cpc
+echo '  accs:' >>cpc
+echo '    A: *' >>cpc
+}
+
+
+
+# create the launcher?
+[ -e cpp ] || {
+echo '#!/bin/sh' >cpp
+echo '' >>cpp
+echo '# change the font so the qr-code draws correctly:' >>cpp
+echo 'config -n "Menlo"  # name' >>cpp
+echo 'config -s 8  # size' >>cpp
+echo '' >>cpp
+echo '# launch copyparty' >>cpp
+echo 'exec copyparty -c cpc "$@"' >>cpp
+}
+
+
+
+chmod 755 copyparty cpp
+echo
+echo =================================
+echo
+echo 'okay, all done!'
+echo
+echo 'you can edit your config'
+echo 'with this command: vim cpc'
+echo
+echo 'you can run copyparty'
+echo 'with this command: cpp'
+echo

contrib/systemd/copyparty-user.service

@@ -0,0 +1,26 @@
+# this will start `/usr/bin/copyparty`
+# and read config from `$HOME/.config/copyparty.conf`
+#
+# unless you add -q to disable logging, you may want to remove the
+#   following line to allow buffering (slightly better performance):
+#   Environment=PYTHONUNBUFFERED=x
+
+[Unit]
+Description=copyparty file server
+
+[Service]
+Type=notify
+SyslogIdentifier=copyparty
+WorkingDirectory=/var/lib/copyparty-jail
+Environment=PYTHONUNBUFFERED=x
+Environment=PRTY_CONFIG=%h/.config/copyparty/copyparty.conf
+ExecReload=/bin/kill -s USR1 $MAINPID
+
+# ensure there is a config
+ExecStartPre=/bin/bash -c 'if [[ ! -f %h/.config/copyparty/copyparty.conf ]]; then mkdir -p %h/.config/copyparty; cp /etc/copyparty/copyparty.conf %h/.config/copyparty/copyparty.conf; fi'
+
+# run copyparty
+ExecStart=/usr/bin/python3 /usr/bin/copyparty
+
+[Install]
+WantedBy=default.target

contrib/systemd/copyparty.conf

@@ -1,42 +1,13 @@
-# not actually YAML but lets pretend:
-# -*- mode: yaml -*-
-# vim: ft=yaml:
-
-
-# put this file in /etc/
-
-
 [global]
-  e2dsa  # enable file indexing and filesystem scanning
-  e2ts   # and enable multimedia indexing
-  ansi   # and colors in log messages
-
-  # disable logging to stdout/journalctl and log to a file instead;
-  # $LOGS_DIRECTORY is usually /var/log/copyparty (comes from systemd)
-  # and copyparty replaces %Y-%m%d with Year-MonthDay, so the
-  # full path will be something like /var/log/copyparty/2023-1130.txt
-  # (note: enable compression by adding .xz at the end)
-  q, lo: $LOGS_DIRECTORY/%Y-%m%d.log
-
-  # p: 80,443,3923   # listen on 80/443 as well (requires CAP_NET_BIND_SERVICE)
-  # i: 127.0.0.1     # only allow connections from localhost (reverse-proxies)
-  # ftp: 3921        # enable ftp server on port 3921
-  # p: 3939          # listen on another port
-  # df: 16           # stop accepting uploads if less than 16 GB free disk space
-  # ver              # show copyparty version in the controlpanel
-  # grid             # show thumbnails/grid-view by default
-  # theme: 2         # monokai
-  # name: datasaver  # change the server-name that's displayed in the browser
-  # stats, nos-dup   # enable the prometheus endpoint, but disable the dupes counter (too slow)
-  # no-robots, force-js  # make it harder for search engines to read your server
-
+  i: 127.0.0.1
 
 [accounts]
-  ed: wark  # username: password
+  user: password
 
-
-[/]            # create a volume at "/" (the webroot), which will
-  /mnt         # share the contents of the "/mnt" folder
+[/]
+  /var/lib/copyparty-jail
   accs:
-    rw: *      # everyone gets read-write access, but
-    rwmda: ed  # the user "ed" gets read-write-move-delete-admin
+    r: *
+    rwdma: user
+  flags:
+    grid

contrib/systemd/copyparty.example.conf

@@ -0,0 +1,42 @@
+# not actually YAML but lets pretend:
+# -*- mode: yaml -*-
+# vim: ft=yaml:
+
+
+# put this file in /etc/
+
+
+[global]
+  e2dsa  # enable file indexing and filesystem scanning
+  e2ts   # and enable multimedia indexing
+  ansi   # and colors in log messages
+
+  # disable logging to stdout/journalctl and log to a file instead;
+  # $LOGS_DIRECTORY is usually /var/log/copyparty (comes from systemd)
+  # and copyparty replaces %Y-%m%d with Year-MonthDay, so the
+  # full path will be something like /var/log/copyparty/2023-1130.txt
+  # (note: enable compression by adding .xz at the end)
+  q, lo: $LOGS_DIRECTORY/%Y-%m%d.log
+
+  # p: 80,443,3923   # listen on 80/443 as well (requires CAP_NET_BIND_SERVICE)
+  # i: 127.0.0.1     # only allow connections from localhost (reverse-proxies)
+  # ftp: 3921        # enable ftp server on port 3921
+  # p: 3939          # listen on another port
+  # df: 16           # stop accepting uploads if less than 16 GB free disk space
+  # ver              # show copyparty version in the controlpanel
+  # grid             # show thumbnails/grid-view by default
+  # theme: 2         # monokai
+  # name: datasaver  # change the server-name that's displayed in the browser
+  # stats, nos-dup   # enable the prometheus endpoint, but disable the dupes counter (too slow)
+  # no-robots, force-js  # make it harder for search engines to read your server
+
+
+[accounts]
+  ed: wark  # username: password
+
+
+[/]            # create a volume at "/" (the webroot), which will
+  /mnt         # share the contents of the "/mnt" folder
+  accs:
+    rw: *      # everyone gets read-write access, but
+    rwmda: ed  # the user "ed" gets read-write-move-delete-admin

contrib/systemd/copyparty@.service

@@ -0,0 +1,30 @@
+# this will start `/usr/bin/copyparty`
+# and read config from `/etc/copyparty/copyparty.conf`
+#
+# the %i refers to whatever you put after the copyparty@
+#   so with copyparty@foo.service, %i == foo
+#
+# unless you add -q to disable logging, you may want to remove the
+#   following line to allow buffering (slightly better performance):
+#   Environment=PYTHONUNBUFFERED=x
+
+[Unit]
+Description=copyparty file server
+
+[Service]
+Type=notify
+SyslogIdentifier=copyparty
+WorkingDirectory=/var/lib/copyparty-jail
+Environment=PYTHONUNBUFFERED=x
+Environment=PRTY_CONFIG=/etc/copyparty/copyparty.conf
+ExecReload=/bin/kill -s USR1 $MAINPID
+
+# user to run as + where the TLS certificate is (if any)
+User=%i
+Environment=XDG_CONFIG_HOME=/home/%i/.config
+
+# run copyparty
+ExecStart=/usr/bin/python3 /usr/bin/copyparty
+
+[Install]
+WantedBy=multi-user.target

contrib/systemd/index.md

@@ -0,0 +1,10 @@
+this is `/var/lib/copyparty-jail`, the fallback webroot when copyparty has not yet been configured
+
+please edit `/etc/copyparty/copyparty.conf` (if running as a system service)
+or `$HOME/.config/copyparty/copyparty.conf` if running as a user service
+
+a basic configuration example is available at https://github.com/9001/copyparty/blob/hovudstraum/contrib/systemd/copyparty.example.conf
+a configuration example that explains most flags is available at https://github.com/9001/copyparty/blob/hovudstraum/docs/chungus.conf
+
+the full list of configuration options can be seen at https://ocv.me/copyparty/helptext.html 
+or by running `copyparty --help`

contrib/systemd/prisonparty@.service

@@ -0,0 +1,38 @@
+# this will start `/usr/bin/copyparty`
+# in a chroot, preventing accidental access elsewhere,
+# and read copyparty config from `/etc/copyparty/copyparty.conf`
+#
+# expose additional filesystem locations to copyparty
+#   by listing them between the last `%i` and `--`
+#
+# `%i %i` = user/group to run copyparty as; can be IDs (1000 1000)
+#   the %i refers to whatever you put after the prisonparty@
+#   so with prisonparty@foo.service, %i == foo
+#
+# unless you add -q to disable logging, you may want to remove the
+#   following line to allow buffering (slightly better performance):
+#   Environment=PYTHONUNBUFFERED=x
+
+[Unit]
+Description=copyparty file server
+
+[Service]
+Type=notify
+SyslogIdentifier=prisonparty
+WorkingDirectory=/var/lib/copyparty-jail
+Environment=PYTHONUNBUFFERED=x
+Environment=PRTY_CONFIG=/etc/copyparty/copyparty.conf
+ExecReload=/bin/kill -s USR1 $MAINPID
+
+# user to run as + where the TLS certificate is (if any)
+User=%i
+Environment=XDG_CONFIG_HOME=/home/%i/.config
+
+# run copyparty
+ExecStart=/bin/bash /usr/bin/prisonparty /var/lib/copyparty-jail %i %i \
+  /etc/copyparty \
+  -- \
+  /usr/bin/python3 /usr/bin/copyparty
+
+[Install]
+WantedBy=multi-user.target

contrib/traefik/copyparty.yaml

@@ -1,5 +1,18 @@
-# ./traefik --experimental.fastproxy=true --entrypoints.web.address=:8080 --providers.file.filename=copyparty.yaml
+# ./traefik --configFile=copyparty.yaml
 
+entryPoints:
+  web:
+    address: :8080
+    transport:
+      # don't disconnect during big uploads
+      respondingTimeouts:
+        readTimeout: "0s"
+log:
+  level: DEBUG
+providers:
+  file:
+    # WARNING: must be same filename as current file
+    filename: "copyparty.yaml"
 http:
   services:
     service-cpp:

contrib/zfs-tune.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python3
+
+import os
+import sqlite3
+import sys
+import traceback
+
+
+"""
+when the up2k-database is stored on a zfs volume, this may give
+slightly higher performance (actual gains not measured yet)
+
+NOTE: must be applied in combination with the related advice in the openzfs documentation;
+https://openzfs.github.io/openzfs-docs/Performance%20and%20Tuning/Workload%20Tuning.html#database-workloads
+and see specifically the SQLite subsection
+
+it is assumed that all databases are stored in a single location,
+for example with `--hist /var/store/hists`
+
+three alternatives for running this script:
+
+1. copy it into /var/store/hists and run "python3 zfs-tune.py s"
+    (s = modify all databases below folder containing script)
+
+2. cd into /var/store/hists and run "python3 ~/zfs-tune.py w"
+    (w = modify all databases below current working directory)
+
+3. python3 ~/zfs-tune.py /var/store/hists
+
+if you use docker, run copyparty with `--hist /cfg/hists`, copy this script into /cfg, and run this:
+podman run --rm -it --entrypoint /usr/bin/python3 ghcr.io/9001/copyparty-ac /cfg/zfs-tune.py s
+
+"""
+
+
+PAGESIZE = 65536
+
+
+# borrowed from copyparty; short efficient stacktrace for errors
+def min_ex(max_lines: int = 8, reverse: bool = False) -> str:
+    et, ev, tb = sys.exc_info()
+    stb = traceback.extract_tb(tb) if tb else traceback.extract_stack()[:-1]
+    fmt = "%s:%d <%s>: %s"
+    ex = [fmt % (fp.split(os.sep)[-1], ln, fun, txt) for fp, ln, fun, txt in stb]
+    if et or ev or tb:
+        ex.append("[%s] %s" % (et.__name__ if et else "(anonymous)", ev))
+    return "\n".join(ex[-max_lines:][:: -1 if reverse else 1])
+
+
+def set_pagesize(db_path):
+    try:
+        # check current page_size
+        with sqlite3.connect(db_path) as db:
+            v = db.execute("pragma page_size").fetchone()[0]
+            if v == PAGESIZE:
+                print(" `-- OK")
+                return
+
+        # https://www.sqlite.org/pragma.html#pragma_page_size
+        #  `- disable wal; set pagesize; vacuum
+        #      (copyparty will reenable wal if necessary)
+
+        with sqlite3.connect(db_path) as db:
+            db.execute("pragma journal_mode=delete")
+            db.commit()
+
+        with sqlite3.connect(db_path) as db:
+            db.execute(f"pragma page_size = {PAGESIZE}")
+            db.execute("vacuum")
+
+        print(" `-- new pagesize OK")
+
+    except Exception:
+        err = min_ex().replace("\n", "\n -- ")
+        print(f"FAILED: {db_path}\n -- {err}")
+
+
+def main():
+    top = os.path.dirname(os.path.abspath(__file__))
+    cwd = os.path.abspath(os.getcwd())
+    try:
+        x = sys.argv[1]
+    except:
+        print(f"""
+this script takes one mandatory argument:
+specify 's' to start recursing from folder containing this script file ({top})
+specify 'w' to start recursing from the current working directory ({cwd})
+specify a path to start recursing from there
+""")
+        sys.exit(1)
+
+    if x.lower() == "w":
+        top = cwd
+    elif x.lower() != "s":
+        top = x
+
+    for dirpath, dirs, files in os.walk(top):
+        for fname in files:
+            if not fname.endswith(".db"):
+                continue
+            db_path = os.path.join(dirpath, fname)
+            print(db_path)
+            set_pagesize(db_path)
+
+
+if __name__ == "__main__":
+    main()

copyparty/__init__.py

@@ -55,7 +55,7 @@ except:
 zs = """
 web/a/partyfuse.py
 web/a/u2c.py
-web/a/webdav-cfg.bat
+web/a/webdav-cfg.txt
 web/baguettebox.js
 web/browser.css
 web/browser.html
@@ -63,10 +63,6 @@ web/browser.js
 web/browser2.html
 web/cf.html
 web/copyparty.gif
-web/dd/2.png
-web/dd/3.png
-web/dd/4.png
-web/dd/5.png
 web/deps/busy.mp3
 web/deps/easymde.css
 web/deps/easymde.js
@@ -80,6 +76,7 @@ web/deps/prismd.css
 web/deps/scp.woff2
 web/deps/sha512.ac.js
 web/deps/sha512.hw.js
+web/idp.html
 web/iiam.gif
 web/md.css
 web/md.html
@@ -91,6 +88,7 @@ web/mde.html
 web/mde.js
 web/msg.css
 web/msg.html
+web/opds.xml
 web/rups.css
 web/rups.html
 web/rups.js
@@ -102,19 +100,45 @@ web/splash.html
 web/splash.js
 web/svcs.html
 web/svcs.js
+web/tl/chi.js
+web/tl/cze.js
+web/tl/deu.js
+web/tl/epo.js
+web/tl/fin.js
+web/tl/fra.js
+web/tl/grc.js
+web/tl/ita.js
+web/tl/kor.js
+web/tl/nld.js
+web/tl/nno.js
+web/tl/nor.js
+web/tl/pol.js
+web/tl/por.js
+web/tl/rus.js
+web/tl/spa.js
+web/tl/swe.js
+web/tl/tur.js
+web/tl/ukr.js
 web/ui.css
 web/up2k.js
 web/util.js
 web/w.hash.js
 """
 RES = set(zs.strip().split("\n"))
+RESM = {
+    "web/a/partyfuse.txt": "web/a/partyfuse.py",
+    "web/a/u2c.txt": "web/a/u2c.py",
+    "web/a/webdav-cfg.bat": "web/a/webdav-cfg.txt",
+}
 
 
 class EnvParams(object):
     def __init__(self) -> None:
         self.t0 = time.time()
         self.mod = ""
+        self.mod_ = ""
         self.cfg = ""
+        self.scfg = True
 
 
 E = EnvParams()

copyparty/__version__.py

@@ -1,8 +1,8 @@
 # coding: utf-8
 
-VERSION = (1, 16, 10)
-CODENAME = "COPYparty"
-BUILD_DT = (2025, 1, 25)
+VERSION = (1, 19, 21)
+CODENAME = "usernames"
+BUILD_DT = (2025, 12, 2)
 
 S_VERSION = ".".join(map(str, VERSION))
 S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT)

copyparty/bos/bos.py

@@ -2,15 +2,22 @@
 from __future__ import print_function, unicode_literals
 
 import os
+import time
 
 from ..util import SYMTIME, fsdec, fsenc
 from . import path as path
 
 if True:  # pylint: disable=using-constant-test
-    from typing import Any, Optional
+    from typing import Any, Optional, Union
 
-_ = (path,)
-__all__ = ["path"]
+    from ..util import NamedLogger
+
+MKD_755 = {"chmod_d": 0o755}
+MKD_700 = {"chmod_d": 0o700}
+UTIME_CLAMPS = ((max, -2147483647), (max, 1), (min, 4294967294), (min, 2147483646))
+
+_ = (path, MKD_755, MKD_700, UTIME_CLAMPS)
+__all__ = ["path", "MKD_755", "MKD_700", "UTIME_CLAMPS"]
 
 # grep -hRiE '(^|[^a-zA-Z_\.-])os\.' . | gsed -r 's/ /\n/g;s/\(/(\n/g' | grep -hRiE '(^|[^a-zA-Z_\.-])os\.' | sort | uniq -c
 # printf 'os\.(%s)' "$(grep ^def bos/__init__.py | gsed -r 's/^def //;s/\(.*//' | tr '\n' '|' | gsed -r 's/.$//')"
@@ -20,19 +27,39 @@ def chmod(p: str, mode: int) -> None:
     return os.chmod(fsenc(p), mode)
 
 
+def chown(p: str, uid: int, gid: int) -> None:
+    return os.chown(fsenc(p), uid, gid)
+
+
 def listdir(p: str = ".") -> list[str]:
     return [fsdec(x) for x in os.listdir(fsenc(p))]
 
 
-def makedirs(name: str, mode: int = 0o755, exist_ok: bool = True) -> bool:
+def makedirs(name: str, vf: dict[str, Any] = MKD_755, exist_ok: bool = True) -> bool:
+    # os.makedirs does 777 for all but leaf; this does mode on all
+    todo = []
     bname = fsenc(name)
-    try:
-        os.makedirs(bname, mode)
-        return True
-    except:
-        if not exist_ok or not os.path.isdir(bname):
-            raise
+    while bname:
+        if os.path.isdir(bname) or bname in todo:
+            break
+        todo.append(bname)
+        bname = os.path.dirname(bname)
+    if not todo:
+        if not exist_ok:
+            os.mkdir(bname)  # to throw
         return False
+    mode = vf["chmod_d"]
+    chown = "chown" in vf
+    for zb in todo[::-1]:
+        try:
+            os.mkdir(zb, mode)
+            if chown:
+                os.chown(zb, vf["uid"], vf["gid"])
+        except:
+            if os.path.isdir(zb):
+                continue
+            raise
+    return True
 
 
 def mkdir(p: str, mode: int = 0o755) -> None:
@@ -76,6 +103,44 @@ def utime(
         return os.utime(fsenc(p), times)
 
 
+def utime_c(
+    log: Union["NamedLogger", Any],
+    p: str,
+    ts: int,
+    follow_symlinks: bool = True,
+    throw: bool = False,
+) -> Optional[int]:
+    clamp = 0
+    ov = ts
+    bp = fsenc(p)
+    now = int(time.time())
+    while True:
+        try:
+            if SYMTIME:
+                os.utime(bp, (now, ts), follow_symlinks=follow_symlinks)
+            else:
+                os.utime(bp, (now, ts))
+            if clamp:
+                t = "filesystem rejected utime(%r); clamped %s to %s"
+                log(t % (p, ov, ts))
+            return ts
+        except Exception as ex:
+            pv = ts
+            while clamp < len(UTIME_CLAMPS):
+                fun, cv = UTIME_CLAMPS[clamp]
+                ts = fun(ts, cv)
+                clamp += 1
+                if ts != pv:
+                    break
+            if clamp >= len(UTIME_CLAMPS):
+                if throw:
+                    raise
+                else:
+                    t = "could not utime(%r) to %s; %s, %r"
+                    log(t % (p, ov, ex, ex))
+                    return None
+
+
 if hasattr(os, "lstat"):
 
     def lstat(p: str) -> os.stat_result:

copyparty/broker_mpw.py

@@ -12,6 +12,7 @@ import queue
 from .__init__ import ANYWIN
 from .authsrv import AuthSrv
 from .broker_util import BrokerCli, ExceptionalQueue, NotExQueue
+from .fsutil import ramdisk_chk
 from .httpsrv import HttpSrv
 from .util import FAKE_MP, Daemon, HMaccas
 
@@ -56,6 +57,7 @@ class MpWorker(BrokerCli):
 
         # starting to look like a good idea
         self.asrv = AuthSrv(args, None, False)
+        ramdisk_chk(self.asrv)
 
         # instantiate all services here (TODO: inheritance?)
         self.iphash = HMaccas(os.path.join(self.args.E.cfg, "iphash"), 8)
@@ -99,6 +101,7 @@ class MpWorker(BrokerCli):
             if dest == "reload":
                 self.logw("mpw.asrv reloading")
                 self.asrv.reload()
+                ramdisk_chk(self.asrv)
                 self.logw("mpw.asrv reloaded")
                 continue
 

copyparty/broker_util.py

@@ -2,7 +2,6 @@
 from __future__ import print_function, unicode_literals
 
 import argparse
-import traceback
 
 from queue import Queue
 

copyparty/cert.py

@@ -1,13 +1,12 @@
 import calendar
 import errno
-import filecmp
 import json
 import os
 import shutil
 import time
 
 from .__init__ import ANYWIN
-from .util import Netdev, load_resource, runcmd, wrename, wunlink
+from .util import Netdev, atomic_move, load_resource, runcmd, wunlink
 
 HAVE_CFSSL = not os.environ.get("PRTY_NO_CFSSL")
 
@@ -21,6 +20,19 @@ else:
     VF = {"mv_re_t": 0, "rm_re_t": 0}
 
 
+def _sp_err(exe, what, rc, so, se, sin):
+    try:
+        zs = shutil.which(exe)
+    except:
+        zs = "<?>"
+    try:
+        zi = os.path.getsize(zs)
+    except:
+        zi = 0
+    t = "failed to %s; error %s using %s (%s):\n  STDOUT: %s\n  STDERR: %s\n  STDIN: %s\n"
+    raise Exception(t % (what, rc, zs, zi, so, se, sin.decode("utf-8")))
+
+
 def ensure_cert(log: "RootLogger", args) -> None:
     """
     the default cert (and the entire TLS support) is only here to enable the
@@ -109,20 +121,20 @@ def _gen_ca(log: "RootLogger", args):
     cmd = "cfssl gencert -initca -"
     rc, so, se = runcmd(cmd.split(), 30, sin=sin)
     if rc:
-        raise Exception("failed to create ca-cert: {}, {}".format(rc, se), 3)
+        _sp_err("cfssl", "create ca-cert", rc, so, se, sin)
 
     cmd = "cfssljson -bare ca"
     sin = so.encode("utf-8")
     rc, so, se = runcmd(cmd.split(), 10, sin=sin, cwd=args.crt_dir)
     if rc:
-        raise Exception("failed to translate ca-cert: {}, {}".format(rc, se), 3)
+        _sp_err("cfssljson", "translate ca-cert", rc, so, se, sin)
 
     bname = os.path.join(args.crt_dir, "ca")
     try:
         wunlink(nlog, bname + ".key", VF)
     except:
         pass
-    wrename(nlog, bname + "-key.pem", bname + ".key", VF)
+    atomic_move(nlog, bname + "-key.pem", bname + ".key", VF)
     wunlink(nlog, bname + ".csr", VF)
 
     log("cert", "new ca OK", 2)
@@ -132,6 +144,7 @@ def _gen_srv(log: "RootLogger", args, netdevs: dict[str, Netdev]):
     nlog: "NamedLogger" = lambda msg, c=0: log("cert-gen-srv", msg, c)
 
     names = args.crt_ns.split(",") if args.crt_ns else []
+    names = [x.strip() for x in names]
     if not args.crt_exact:
         for n in names[:]:
             names.append("*.{}".format(n))
@@ -202,20 +215,20 @@ def _gen_srv(log: "RootLogger", args, netdevs: dict[str, Netdev]):
     acmd = cmd.split() + ["-hostname=" + ",".join(names), "-"]
     rc, so, se = runcmd(acmd, 30, sin=sin, cwd=args.crt_dir)
     if rc:
-        raise Exception("failed to create cert: {}, {}".format(rc, se))
+        _sp_err("cfssl", "create cert", rc, so, se, sin)
 
     cmd = "cfssljson -bare srv"
     sin = so.encode("utf-8")
     rc, so, se = runcmd(cmd.split(), 10, sin=sin, cwd=args.crt_dir)
     if rc:
-        raise Exception("failed to translate cert: {}, {}".format(rc, se))
+        _sp_err("cfssljson", "translate cert", rc, so, se, sin)
 
     bname = os.path.join(args.crt_dir, "srv")
     try:
         wunlink(nlog, bname + ".key", VF)
     except:
         pass
-    wrename(nlog, bname + "-key.pem", bname + ".key", VF)
+    atomic_move(nlog, bname + "-key.pem", bname + ".key", VF)
     wunlink(nlog, bname + ".csr", VF)
 
     with open(os.path.join(args.crt_dir, "ca.pem"), "rb") as f:

copyparty/cfg.py

@@ -5,6 +5,9 @@ from __future__ import print_function, unicode_literals
 zs = "a c e2d e2ds e2dsa e2t e2ts e2tsr e2v e2vp e2vu ed emp i j lo mcr mte mth mtm mtp nb nc nid nih nth nw p q s ss sss v z zv"
 onedash = set(zs.split())
 
+# verify that all volflags are documented here:
+# grep volflag= __main__.py | sed -r 's/.*volflag=//;s/\).*//' | sort | uniq | while IFS= read -r x; do grep -E "\"$x(=[^ \"]+)?\": \"" cfg.py || printf '%s\n' "$x"; done
+
 
 def vf_bmap() -> dict[str, str]:
     """argv-to-volflag: simple bools"""
@@ -16,15 +19,18 @@ def vf_bmap() -> dict[str, str]:
         "no_clone": "noclone",
         "no_dirsz": "nodirsz",
         "no_dupe": "nodupe",
+        "no_dupe_m": "nodupem",
         "no_forget": "noforget",
         "no_pipe": "nopipe",
         "no_robots": "norobots",
+        "no_tail": "notail",
         "no_thumb": "dthumb",
         "no_vthumb": "dvthumb",
         "no_athumb": "dathumb",
     }
     for k in (
         "dedup",
+        "dlni",
         "dotsrch",
         "e2d",
         "e2ds",
@@ -40,17 +46,35 @@ def vf_bmap() -> dict[str, str]:
         "gsel",
         "hardlink",
         "magic",
+        "md_no_br",
+        "no_db_ip",
         "no_sb_md",
         "no_sb_lg",
         "nsort",
         "og",
         "og_no_head",
         "og_s_title",
+        "opds",
         "rand",
+        "reflink",
+        "rm_partial",
+        "rmagic",
         "rss",
+        "ui_noacci",
+        "ui_nocpla",
+        "ui_nolbar",
+        "ui_nombar",
+        "ui_nonav",
+        "ui_notree",
+        "ui_norepl",
+        "ui_nosrvi",
+        "ui_noctxb",
+        "wo_up_readme",
+        "wram",
         "xdev",
         "xlink",
         "xvol",
+        "zipmaxu",
     ):
         ret[k] = k
     return ret
@@ -59,6 +83,7 @@ def vf_bmap() -> dict[str, str]:
 def vf_vmap() -> dict[str, str]:
     """argv-to-volflag: simple values"""
     ret = {
+        "ac_convt": "aconvt",
         "no_hash": "nohash",
         "no_idx": "noidx",
         "re_maxage": "scan",
@@ -69,14 +94,24 @@ def vf_vmap() -> dict[str, str]:
         "th_x3": "th3x",
     }
     for k in (
+        "bup_ck",
+        "casechk",
+        "chmod_d",
+        "chmod_f",
         "dbd",
+        "du_who",
+        "ufavico",
+        "forget_ip",
         "hsortn",
         "html_head",
+        "html_head_s",
         "lg_sbf",
         "md_sbf",
         "lg_sba",
         "md_sba",
+        "md_hist",
         "nrand",
+        "u2ow",
         "og_desc",
         "og_site",
         "og_th",
@@ -86,14 +121,31 @@ def vf_vmap() -> dict[str, str]:
         "og_title_i",
         "og_tpl",
         "og_ua",
+        "opds_exts",
+        "put_ck",
+        "put_name",
         "mv_retry",
         "rm_retry",
+        "shr_who",
         "sort",
+        "tail_fd",
+        "tail_rate",
+        "tail_tmax",
+        "tail_who",
         "tcolor",
+        "th_spec_p",
+        "txt_eol",
         "unlist",
         "u2abort",
         "u2ts",
+        "uid",
+        "gid",
+        "unp_who",
         "ups_who",
+        "zip_who",
+        "zipmaxn",
+        "zipmaxs",
+        "zipmaxt",
     ):
         ret[k] = k
     return ret
@@ -105,6 +157,7 @@ def vf_cmap() -> dict[str, str]:
     for k in (
         "exp_lg",
         "exp_md",
+        "ext_th",
         "mte",
         "mth",
         "mtp",
@@ -143,15 +196,27 @@ flagcats = {
         "dedup": "enable symlink-based file deduplication",
         "hardlink": "enable hardlink-based file deduplication,\nwith fallback on symlinks when that is impossible",
         "hardlinkonly": "dedup with hardlink only, never symlink;\nmake a full copy if hardlink is impossible",
+        "reflink": "enable reflink-based file deduplication,\nwith fallback on full copy when that is impossible",
         "safededup": "verify on-disk data before using it for dedup",
         "noclone": "take dupe data from clients, even if available on HDD",
         "nodupe": "rejects existing files (instead of linking/cloning them)",
+        "nodupem": "rejects existing files during moves as well",
+        "chmod_d=755": "unix-permission for new dirs/folders",
+        "chmod_f=644": "unix-permission for new files",
+        "uid=573": "change owner of new files/folders to unix-user 573",
+        "gid=999": "change owner of new files/folders to unix-group 999",
+        "wram": "allow uploading into ramdisks",
         "sparse": "force use of sparse files, mainly for s3-backed storage",
         "nosparse": "deny use of sparse files, mainly for slow storage",
+        "rm_partial": "delete unfinished uploads from HDD when they timeout",
         "daw": "enable full WebDAV write support (dangerous);\nPUT-operations will now \033[1;31mOVERWRITE\033[0;35m existing files",
         "nosub": "forces all uploads into the top folder of the vfs",
         "magic": "enables filetype detection for nameless uploads",
-        "gz": "allows server-side gzip of uploads with ?gz (also c,xz)",
+        "put_name": "fallback filename for nameless uploads",
+        "put_ck": "default checksum-hasher for PUT/WebDAV uploads",
+        "bup_ck": "default checksum-hasher for bup/basic uploads",
+        "gz": "allows server-side gzip compression of uploads with ?gz",
+        "xz": "allows server-side lzma compression of uploads with ?xz",
         "pk": "forces server-side compression, optional arg: xz,9",
     },
     "upload rules": {
@@ -160,8 +225,10 @@ flagcats = {
         "vmaxb=1g": "total volume size max 1 GiB (suffixes: b, k, m, g, t)",
         "vmaxn=4k": "max 4096 files in volume (suffixes: b, k, m, g, t)",
         "medialinks": "return medialinks for non-up2k uploads (not hotlinks)",
+        "wo_up_readme": "write-only users can upload logues without getting renamed",
         "rand": "force randomized filenames, 9 chars long by default",
         "nrand=N": "randomized filenames are N chars long",
+        "u2ow=N": "overwrite existing files? 0=no 1=if-older 2=always",
         "u2ts=fc": "[f]orce [c]lient-last-modified or [u]pload-time",
         "u2abort=1": "allow aborting unfinished uploads? 0=no 1=strict 2=ip-chk 3=acct-chk",
         "sz=1k-3m": "allow filesizes between 1 KiB and 3MiB",
@@ -170,6 +237,7 @@ flagcats = {
     "upload rotation\n(moves all uploads into the specified folder structure)": {
         "rotn=100,3": "3 levels of subfolders with 100 entries in each",
         "rotf=%Y-%m/%d-%H": "date-formatted organizing",
+        "rotf_tz=Europe/Oslo": "timezone (default=UTC)",
         "lifetime=3600": "uploads are deleted after 1 hour",
     },
     "database, general": {
@@ -178,19 +246,27 @@ flagcats = {
         "e2dsa": "scans all folders for new files on startup; also sets -e2d",
         "e2t": "enable multimedia indexing; makes it possible to search for tags",
         "e2ts": "scan existing files for tags on startup; also sets -e2t",
-        "e2tsa": "delete all metadata from DB (full rescan); also sets -e2ts",
+        "e2tsr": "delete all metadata from DB (full rescan); also sets -e2ts",
         "d2ts": "disables metadata collection for existing files",
+        "e2v": "verify integrity on startup by hashing files and comparing to db",
+        "e2vu": "when e2v fails, update the db (assume on-disk files are good)",
+        "e2vp": "when e2v fails, panic and quit copyparty",
         "d2ds": "disables onboot indexing, overrides -e2ds*",
         "d2t": "disables metadata collection, overrides -e2t*",
         "d2v": "disables file verification, overrides -e2v*",
         "d2d": "disables all database stuff, overrides -e2*",
         "hist=/tmp/cdb": "puts thumbnails and indexes at that location",
+        "dbpath=/tmp/cdb": "puts indexes at that location",
+        "landmark=foo": "disable db if file foo doesn't exist",
         "scan=60": "scan for new files every 60sec, same as --re-maxage",
         "nohash=\\.iso$": "skips hashing file contents if path matches *.iso",
         "noidx=\\.iso$": "fully ignores the contents at paths matching *.iso",
         "noforget": "don't forget files when deleted from disk",
+        "forget_ip=43200": "forget uploader-IP after 30 days (GDPR)",
+        "no_db_ip": "never store uploader-IP in the db; disables unpost",
         "fat32": "avoid excessive reindexing on android sdcardfs",
         "dbd=[acid|swal|wal|yolo]": "database speed-durability tradeoff",
+        "casechk=auto": "actively prevent case-insensitive filesystem? y/n",
         "xlink": "cross-volume dupe detection / linking (dangerous)",
         "xdev": "do not descend into other filesystems",
         "xvol": "do not follow symlinks leaving the volume root",
@@ -199,6 +275,8 @@ flagcats = {
         "srch_excl": "exclude search results with URL matching this regex",
     },
     'database, audio tags\n"mte", "mth", "mtp", "mtm" all work the same as -mte, -mth, ...': {
+        "mte=artist,title": "media-tags to index/display",
+        "mth=fmt,res,ac": "media-tags to hide by default",
         "mtp=.bpm=f,audio-bpm.py": 'uses the "audio-bpm.py" program to\ngenerate ".bpm" tags from uploads (f = overwrite tags)',
         "mtp=ahash,vhash=media-hash.py": "collects two tags at once",
     },
@@ -211,7 +289,10 @@ flagcats = {
         "thsize": "thumbnail res; WxH",
         "crop": "center-cropping (y/n/fy/fn)",
         "th3x": "3x resolution (y/n/fy/fn)",
-        "convt": "conversion timeout in seconds",
+        "convt": "convert-to-image timeout in seconds",
+        "aconvt": "convert-to-audio timeout in seconds",
+        "th_spec_p=1": "make spectrograms? 0=never 1=fallback 2=always",
+        "ext_th=s=/b.png": "use /b.png as thumbnail for file-extension s",
     },
     "handlers\n(better explained in --help-handlers)": {
         "on404=PY": "handle 404s by executing PY file",
@@ -234,10 +315,20 @@ flagcats = {
         "grid": "show grid/thumbnails by default",
         "gsel": "select files in grid by ctrl-click",
         "sort": "default sort order",
+        "nsort": "natural-sort of leading digits in filenames",
+        "hsortn": "number of sort-rules to add to media URLs",
+        "ufavico=URL": "per-volume favicon (.ico/png/gif/svg)",
         "unlist": "dont list files matching REGEX",
+        "dlni": "force-download (no-inline) files on click",
         "html_head=TXT": "includes TXT in the <head>, or @PATH for file at PATH",
+        "html_head_s=TXT": "additional static text in the html <head>",
+        "tcolor=#fc0": "theme color (a hint for webbrowsers, discord, etc.)",
+        "nodirsz": "don't show total folder size",
+        "du_who=all": "show disk-usage info to everyone",
         "robots": "allows indexing by search engines (default)",
         "norobots": "kindly asks search engines to leave",
+        "unlistcr": "don't list read-access in controlpanel",
+        "unlistcw": "don't list write-access in controlpanel",
         "no_sb_md": "disable js sandbox for markdown files",
         "no_sb_lg": "disable js sandbox for prologue/epilogue",
         "sb_md": "enable js sandbox for markdown files (default)",
@@ -247,11 +338,67 @@ flagcats = {
         "md_sba": "value of iframe allow-prop for markdown-sandbox",
         "lg_sba": "value of iframe allow-prop for *logue-sandbox",
         "nohtml": "return html and markdown as text/html",
+        "ui_noacci": "hide account-info in the UI",
+        "ui_nocpla": "hide cpanel-link in the UI",
+        "ui_nolbar": "hide link-bar in the UI",
+        "ui_nombar": "hide top-menu in the UI",
+        "ui_nonav": "hide navpane+breadcrumbs in the UI",
+        "ui_notree": "hide navpane in the UI",
+        "ui_norepl": "hide repl-button in the UI",
+        "ui_nosrvi": "hide server-info in the UI",
+        "ui_noctxb": "hide context-buttons in the UI",
+    },
+    "opengraph (discord embeds)": {
+        "og": "enable OG (disables hotlinking)",
+        "og_site": "sitename; defaults to --name, disable with '-'",
+        "og_desc": "description text for all files; disable with '-'",
+        "og_th=jf": "thumbnail format; j / jf / jf3 / w / w3 / ...",
+        "og_title_a": "audio title format; default: {{ artist }} - {{ title }}",
+        "og_title_v": "video title format; default: {{ title }}",
+        "og_title_i": "image title format; default: {{ title }}",
+        "og_title=foo": "fallback title if there's nothing in the db",
+        "og_s_title": "force default title; do not read from tags",
+        "og_tpl": "custom html; see --og-tpl in --help",
+        "og_no_head": "you want to add tags manually with og_tpl",
+        "og_ua": "if defined: only send OG html if useragent matches this regex",
+    },
+    "opds": {
+        "opds": "enable OPDS",
+        "opds_exts": "file formats to list in OPDS feeds; leave empty to show everything",
+    },
+    "textfiles": {
+        "md_no_br": "newline only on double-newline or two tailing spaces",
+        "md_hist": "where to put markdown backups; s=subfolder, v=volHist, n=nope",
+        "exp": "enable textfile expansion; see --help-exp",
+        "exp_md": "placeholders to expand in markdown files; see --help",
+        "exp_lg": "placeholders to expand in prologue/epilogue; see --help",
+        "txt_eol=lf": "enable EOL conversion when writing docs (LF or CRLF)",
+    },
+    "tailing": {
+        "notail": "disable ?tail (download a growing file continuously)",
+        "tail_fd=1": "check if file was replaced (new fd) every 1 sec",
+        "tail_rate=0.2": "check for new data every 0.2 sec",
+        "tail_tmax=30": "kill connection after 30 sec",
+        "tail_who=2": "restrict ?tail access (1=admins,2=authed,3=everyone)",
     },
     "others": {
         "dots": "allow all users with read-access to\nenable the option to show dotfiles in listings",
         "fk=8": 'generates per-file accesskeys,\nwhich are then required at the "g" permission;\nkeys are invalidated if filesize or inode changes',
         "fka=8": 'generates slightly weaker per-file accesskeys,\nwhich are then required at the "g" permission;\nnot affected by filesize or inode numbers',
+        "dk=8": 'generates per-directory accesskeys,\nwhich are then required at the "g" permission;\nkeys are invalidated if filesize or inode changes',
+        "dks": "per-directory accesskeys allow browsing into subdirs",
+        "dky": 'allow seeing files (not folders) inside a specific folder\nwith "g" perm, and does not require a valid dirkey to do so',
+        "rss": "allow '?rss' URL suffix (experimental)",
+        "rmagic": "expensive analysis for mimetype accuracy",
+        "shr_who=auth": "who can create shares? no/auth/a",
+        "unp_who=2": "unpost only if same... 1=ip+name, 2=ip, 3=name",
+        "ups_who=2": "restrict viewing the list of recent uploads",
+        "zip_who=2": "restrict access to download-as-zip/tar",
+        "zipmaxn=9k": "reject download-as-zip if more than 9000 files",
+        "zipmaxs=2g": "reject download-as-zip if size over 2 GiB",
+        "zipmaxt=no": "reply with 'no' if download-as-zip exceeds max",
+        "zipmaxu": "zip-size-limit does not apply to authenticated users",
+        "nopipe": "disable race-the-beam (download unfinished uploads)",
         "mv_retry": "ms-windows: timeout for renaming busy files",
         "rm_retry": "ms-windows: timeout for deleting busy files",
         "davauth": "ask webdav clients to login for all folders",
@@ -261,3 +408,10 @@ flagcats = {
 
 
 flagdescs = {k.split("=")[0]: v for tab in flagcats.values() for k, v in tab.items()}
+
+
+if True:  # so it gets removed in release-builds
+    for fun in [vf_bmap, vf_cmap, vf_vmap]:
+        for k in fun().values():
+            if k not in flagdescs:
+                raise Exception("undocumented volflag: " + k)

copyparty/dxml.py

@@ -65,6 +65,9 @@ DXMLParser = _DXMLParser
 
 
 def parse_xml(txt: str) -> ET.Element:
+    """
+    Parse XML into an xml.etree.ElementTree.Element while defusing some unsafe parts.
+    """
     parser = DXMLParser()
     parser.feed(txt)
     return parser.close()  # type: ignore

copyparty/fsutil.py

@@ -7,7 +7,7 @@ import re
 import time
 
 from .__init__ import ANYWIN, MACOS
-from .authsrv import AXS, VFS
+from .authsrv import AXS, VFS, AuthSrv
 from .bos import bos
 from .util import chkcmd, min_ex, undot
 
@@ -18,22 +18,25 @@ if True:  # pylint: disable=using-constant-test
 
 
 class Fstab(object):
-    def __init__(self, log: "RootLogger", args: argparse.Namespace):
+    def __init__(self, log: "RootLogger", args: argparse.Namespace, verbose: bool):
         self.log_func = log
+        self.verbose = verbose
 
         self.warned = False
         self.trusted = False
         self.tab: Optional[VFS] = None
         self.oldtab: Optional[VFS] = None
         self.srctab = "a"
-        self.cache: dict[str, str] = {}
+        self.cache: dict[str, tuple[str, str]] = {}
         self.age = 0.0
         self.maxage = args.mtab_age
 
     def log(self, msg: str, c: Union[int, str] = 0) -> None:
+        if not c or self.verbose:
+            return
         self.log_func("fstab", msg, c)
 
-    def get(self, path: str) -> str:
+    def get(self, path: str) -> tuple[str, str]:
         now = time.time()
         if now - self.age > self.maxage or len(self.cache) > 9000:
             self.age = now
@@ -41,6 +44,7 @@ class Fstab(object):
             self.tab = None
             self.cache = {}
 
+        mp = ""
         fs = "ext4"
         msg = "failed to determine filesystem at %r; assuming %s\n%s"
 
@@ -50,7 +54,7 @@ class Fstab(object):
                 path = self._winpath(path)
             except:
                 self.log(msg % (path, fs, min_ex()), 3)
-                return fs
+                return fs, ""
 
         path = undot(path)
         try:
@@ -59,14 +63,14 @@ class Fstab(object):
             pass
 
         try:
-            fs = self.get_w32(path) if ANYWIN else self.get_unix(path)
+            fs, mp = self.get_w32(path) if ANYWIN else self.get_unix(path)
         except:
             self.log(msg % (path, fs, min_ex()), 3)
 
         fs = fs.lower()
-        self.cache[path] = fs
-        self.log("found %s at %r" % (fs, path))
-        return fs
+        self.cache[path] = (fs, mp)
+        self.log("found %s at %r, %r" % (fs, mp, path))
+        return fs, mp
 
     def _winpath(self, path: str) -> str:
         # try to combine volume-label + st_dev (vsn)
@@ -78,42 +82,58 @@ class Fstab(object):
             return vid
 
     def build_fallback(self) -> None:
-        self.tab = VFS(self.log_func, "idk", "/", AXS(), {})
+        self.tab = VFS(self.log_func, "idk", "/", "/", AXS(), {})
         self.trusted = False
 
-    def build_tab(self) -> None:
-        self.log("inspecting mtab for changes")
-
+    def _from_sp_mount(self) -> dict[str, str]:
         sptn = r"^.*? on (.*) type ([^ ]+) \(.*"
         if MACOS:
             sptn = r"^.*? on (.*) \(([^ ]+), .*"
 
         ptn = re.compile(sptn)
         so, _ = chkcmd(["mount"])
-        tab1: list[tuple[str, str]] = []
-        atab = []
+        dtab: dict[str, str] = {}
         for ln in so.split("\n"):
             m = ptn.match(ln)
             if not m:
                 continue
 
             zs1, zs2 = m.groups()
-            tab1.append((str(zs1), str(zs2)))
-            atab.append(ln)
+            dtab[str(zs1)] = str(zs2)
+
+        return dtab
+
+    def _from_proc(self) -> dict[str, str]:
+        ret: dict[str, str] = {}
+        with open("/proc/self/mounts", "rb", 262144) as f:
+            src = f.read(262144).decode("utf-8", "replace").split("\n")
+        for zsl in [x.split(" ") for x in src]:
+            if len(zsl) < 3:
+                continue
+            zs = zsl[1]
+            zs = zs.replace("\\011", "\t").replace("\\040", " ").replace("\\134", "\\")
+            ret[zs] = zsl[2]
+        return ret
+
+    def build_tab(self) -> None:
+        self.log("inspecting mtab for changes")
+        dtab = self._from_sp_mount() if MACOS else self._from_proc()
 
         # keep empirically-correct values if mounttab unchanged
-        srctab = "\n".join(sorted(atab))
+        srctab = str(sorted(dtab.items()))
         if srctab == self.srctab:
             self.tab = self.oldtab
             return
 
         self.log("mtab has changed; reevaluating support for sparse files")
 
+        tab1 = list(dtab.items())
         tab1.sort(key=lambda x: (len(x[0]), x[0]))
         path1, fs1 = tab1[0]
-        tab = VFS(self.log_func, fs1, path1, AXS(), {})
+        tab = VFS(self.log_func, fs1, path1, path1, AXS(), {})
         for path, fs in tab1[1:]:
-            tab.add(fs, path.lstrip("/"))
+            zs = path.lstrip("/")
+            tab.add(fs, zs, zs)
 
         self.tab = tab
         self.srctab = srctab
@@ -130,9 +150,10 @@ class Fstab(object):
         if not self.trusted:
             # no mtab access; have to build as we go
             if "/" in rem:
-                self.tab.add("idk", os.path.join(vn.vpath, rem.split("/")[0]))
+                zs = os.path.join(vn.vpath, rem.split("/")[0])
+                self.tab.add("idk", zs, zs)
             if rem:
-                self.tab.add(nval, path)
+                self.tab.add(nval, path, path)
             else:
                 vn.realpath = nval
 
@@ -144,7 +165,7 @@ class Fstab(object):
             vn.realpath = ptn.sub(nval, vn.realpath)
             visit.extend(list(vn.nodes.values()))
 
-    def get_unix(self, path: str) -> str:
+    def get_unix(self, path: str) -> tuple[str, str]:
         if not self.tab:
             try:
                 self.build_tab()
@@ -153,20 +174,50 @@ class Fstab(object):
                 # prisonparty or other restrictive environment
                 if not self.warned:
                     self.warned = True
-                    self.log("failed to build tab:\n{}".format(min_ex()), 3)
+                    t = "failed to associate fs-mounts with the VFS (this is fine):\n%s"
+                    self.log(t % (min_ex(),), 6)
                 self.build_fallback()
 
         assert self.tab  # !rm
         ret = self.tab._find(path)[0]
         if self.trusted or path == ret.vpath:
-            return ret.realpath.split("/")[0]
+            return ret.realpath.split("/")[0], ret.vpath
         else:
-            return "idk"
+            return "idk", ""
 
-    def get_w32(self, path: str) -> str:
+    def get_w32(self, path: str) -> tuple[str, str]:
         if not self.tab:
             self.build_fallback()
 
         assert self.tab  # !rm
         ret = self.tab._find(path)[0]
-        return ret.realpath
+        return ret.realpath, ""
+
+
+def ramdisk_chk(asrv: AuthSrv) -> None:
+    # should have been in authsrv but that's a circular import
+    mods = []
+    ramfs = ("tmpfs", "overlay")
+    log = asrv.log_func or print
+    fstab = Fstab(log, asrv.args, False)
+    for vn in asrv.vfs.all_nodes.values():
+        if not vn.axs.uwrite or "wram" in vn.flags:
+            continue
+        ap = vn.realpath
+        if not ap or os.path.isfile(ap):
+            continue
+        fs, mp = fstab.get(ap)
+        mp = "/" + mp.strip("/")
+        if fs == "tmpfs" or (mp == "/" and fs in ramfs):
+            mods.append((vn.vpath, ap, fs, mp))
+            vn.axs.uwrite.clear()
+            vn.axs.umove.clear()
+            for un, ztsp in list(vn.uaxs.items()):
+                zsl = list(ztsp)
+                zsl[1] = False
+                zsl[2] = False
+                vn.uaxs[un] = zsl
+    if mods:
+        t = "WARNING: write-access was removed from the following volumes because they are not mapped to an actual HDD for storage! All uploaded data would live in RAM only, and all uploaded files would be LOST on next reboot. To allow uploading and ignore this hazard, enable the 'wram' option (global/volflag). List of affected volumes:"
+        t2 = ["\n  volume=[/%s], abspath=%r, type=%s, root=%r" % x for x in mods]
+        log("vfs", t + "".join(t2) + "\n", 1)

copyparty/ftpd.py

@@ -19,6 +19,7 @@ from .__init__ import PY2, TYPE_CHECKING
 from .authsrv import VFS
 from .bos import bos
 from .util import (
+    FN_EMB,
     VF_CAREFUL,
     Daemon,
     ODict,
@@ -30,6 +31,7 @@ from .util import (
     relchk,
     runhook,
     sanitize_fn,
+    set_fperms,
     vjoin,
     wunlink,
 )
@@ -66,13 +68,13 @@ class FtpAuth(DummyAuthorizer):
         if ip.startswith("::ffff:"):
             ip = ip[7:]
 
-        ip = ipnorm(ip)
+        ipn = ipnorm(ip)
         bans = self.hub.bans
-        if ip in bans:
-            rt = bans[ip] - time.time()
+        if ipn in bans:
+            rt = bans[ipn] - time.time()
             if rt < 0:
                 logging.info("client unbanned")
-                del bans[ip]
+                del bans[ipn]
             else:
                 raise AuthenticationFailed("banned")
 
@@ -81,7 +83,12 @@ class FtpAuth(DummyAuthorizer):
         uname = "*"
         if username != "anonymous":
             uname = ""
-            for zs in (password, username):
+            if args.usernames:
+                alts = ["%s:%s" % (username, password)]
+            else:
+                alts = password, username
+
+            for zs in alts:
                 zs = asrv.iacct.get(asrv.ah.hash(zs), "")
                 if zs:
                     uname = zs
@@ -89,6 +96,10 @@ class FtpAuth(DummyAuthorizer):
 
         if args.ipu and uname == "*":
             uname = args.ipu_iu[args.ipu_nm.map(ip)]
+        if args.ipr and uname in args.ipr_u:
+            if not args.ipr_u[uname].map(ip):
+                logging.warning("username [%s] rejected by --ipr", uname)
+                uname = "*"
 
         if not uname or not (asrv.vfs.aread.get(uname) or asrv.vfs.awrite.get(uname)):
             g = self.hub.gpwd
@@ -141,10 +152,6 @@ class FtpFs(AbstractedFS):
         self.cwd = "/"  # pyftpdlib convention of leading slash
         self.root = "/var/lib/empty"
 
-        self.can_read = self.can_write = self.can_move = False
-        self.can_delete = self.can_get = self.can_upget = False
-        self.can_admin = self.can_dot = False
-
         self.listdirinfo = self.listdir
         self.chdir(".")
 
@@ -170,6 +177,16 @@ class FtpFs(AbstractedFS):
             fn = sanitize_fn(fn or "", "")
             vpath = vjoin(rd, fn)
             vfs, rem = self.hub.asrv.vfs.get(vpath, self.uname, r, w, m, d)
+            if (
+                w
+                and fn.lower() in FN_EMB
+                and self.h.uname not in vfs.axs.uread
+                and "wo_up_readme" not in vfs.flags
+            ):
+                fn = "_wo_" + fn
+                vpath = vjoin(rd, fn)
+                vfs, rem = self.hub.asrv.vfs.get(vpath, self.uname, r, w, m, d)
+
             if not vfs.realpath:
                 t = "No filesystem mounted at [{}]"
                 raise FSE(t.format(vpath))
@@ -181,10 +198,13 @@ class FtpFs(AbstractedFS):
                 if not avfs:
                     raise FSE(t.format(vpath), 1)
 
-                cr, cw, cm, cd, _, _, _, _ = avfs.can_access("", self.h.uname)
+                cr, cw, cm, cd, _, _, _, _, _ = avfs.uaxs[self.h.uname]
                 if r and not cr or w and not cw or m and not cm or d and not cd:
                     raise FSE(t.format(vpath), 1)
 
+            if "bcasechk" in vfs.flags and not vfs.casechk(rem, True):
+                raise FSE("No such file or directory", 1)
+
             return os.path.join(vfs.realpath, rem), vfs, rem
         except Pebkac as ex:
             raise FSE(str(ex))
@@ -197,7 +217,7 @@ class FtpFs(AbstractedFS):
         m: bool = False,
         d: bool = False,
     ) -> tuple[str, VFS, str]:
-        return self.v2a(os.path.join(self.cwd, vpath), r, w, m, d)
+        return self.v2a(join(self.cwd, vpath), r, w, m, d)
 
     def ftp2fs(self, ftppath: str) -> str:
         # return self.v2a(ftppath)
@@ -218,7 +238,7 @@ class FtpFs(AbstractedFS):
         r = "r" in mode
         w = "w" in mode or "a" in mode or "+" in mode
 
-        ap = self.rv2a(filename, r, w)[0]
+        ap, vfs, _ = self.rv2a(filename, r, w)
         self.validpath(ap)
         if w:
             try:
@@ -230,8 +250,9 @@ class FtpFs(AbstractedFS):
                 td = 0
 
         if w and need_unlink:
+            assert td  # type: ignore  # !rm
             if td >= -1 and td <= self.args.ftp_wt:
-                # within permitted timeframe; unlink and accept
+                # within permitted timeframe; allow overwrite or resume
                 do_it = True
             elif self.args.no_del or self.args.ftp_no_ow:
                 # file too old, or overwrite not allowed; reject
@@ -248,13 +269,23 @@ class FtpFs(AbstractedFS):
             if not do_it:
                 raise FSE("File already exists")
 
+            # Don't unlink file for append mode
+            elif "a" not in mode:
                 wunlink(self.log, ap, VF_CAREFUL)
 
-        return open(fsenc(ap), mode, self.args.iobuf)
+        ret = open(fsenc(ap), mode, self.args.iobuf)
+        if w and "fperms" in vfs.flags:
+            set_fperms(ret, vfs.flags)
+
+        return ret
 
     def chdir(self, path: str) -> None:
         nwd = join(self.cwd, path)
         vfs, rem = self.hub.asrv.vfs.get(nwd, self.uname, False, False)
+        if not vfs.realpath:
+            self.cwd = nwd
+            return
+
         ap = vfs.canonical(rem)
         try:
             st = bos.stat(ap)
@@ -269,20 +300,10 @@ class FtpFs(AbstractedFS):
             raise FSE("Permission denied", 1)
 
         self.cwd = nwd
-        (
-            self.can_read,
-            self.can_write,
-            self.can_move,
-            self.can_delete,
-            self.can_get,
-            self.can_upget,
-            self.can_admin,
-            self.can_dot,
-        ) = avfs.can_access("", self.h.uname)
 
     def mkdir(self, path: str) -> None:
-        ap = self.rv2a(path, w=True)[0]
-        bos.makedirs(ap)  # filezilla expects this
+        ap, vfs, _ = self.rv2a(path, w=True)
+        bos.makedirs(ap, vf=vfs.flags)  # filezilla expects this
 
     def listdir(self, path: str) -> list[str]:
         vpath = join(self.cwd, path)
@@ -301,7 +322,7 @@ class FtpFs(AbstractedFS):
             vfs_ls = [x[0] for x in vfs_ls1]
             vfs_ls.extend(vfs_virt.keys())
 
-            if not self.can_dot:
+            if self.uname not in vfs.axs.udot:
                 vfs_ls = exclude_dotfiles(vfs_ls)
 
             vfs_ls.sort()
@@ -349,16 +370,13 @@ class FtpFs(AbstractedFS):
             raise FSE(str(ex))
 
     def rename(self, src: str, dst: str) -> None:
-        if not self.can_move:
-            raise FSE("Not allowed for user " + self.h.uname)
-
         if self.args.no_mv:
             raise FSE("The rename/move feature is disabled in server config")
 
         svp = join(self.cwd, src).lstrip("/")
         dvp = join(self.cwd, dst).lstrip("/")
         try:
-            self.hub.up2k.handle_mv(self.uname, self.h.cli_ip, svp, dvp)
+            self.hub.up2k.handle_mv("", self.uname, self.h.cli_ip, svp, dvp)
         except Exception as ex:
             raise FSE(str(ex))
 
@@ -382,7 +400,7 @@ class FtpFs(AbstractedFS):
 
     def utime(self, path: str, timeval: float) -> None:
         ap = self.rv2a(path, w=True)[0]
-        return bos.utime(ap, (timeval, timeval))
+        bos.utime_c(logging.warning, ap, int(timeval), False)
 
     def lstat(self, path: str) -> os.stat_result:
         ap = self.rv2a(path)[0]
@@ -471,10 +489,15 @@ class FtpHandler(FTPHandler):
     def ftp_STOR(self, file: str, mode: str = "w") -> Any:
         # Optional[str]
         vp = join(self.fs.cwd, file).lstrip("/")
+        try:
             ap, vfs, rem = self.fs.v2a(vp, w=True)
+        except Exception as ex:
+            self.respond("550 %s" % (ex,), logging.info)
+            return
         self.vfs_map[ap] = vp
         xbu = vfs.flags.get("xbu")
-        if xbu and not runhook(
+        if xbu:
+            hr = runhook(
                 None,
                 None,
                 self.hub.up2k,
@@ -489,9 +512,14 @@ class FtpHandler(FTPHandler):
                 0,
                 self.cli_ip,
                 time.time(),
-            "",
-        ):
-            raise FSE("Upload blocked by xbu server config")
+                None,
+            )
+            t = hr.get("rejectmsg") or ""
+            if t or hr.get("rc") != 0:
+                if not t:
+                    t = "Upload blocked by xbu server config: %r" % (vp,)
+                self.respond("550 %s" % (t,), logging.info)
+                return
 
         # print("ftp_STOR: {} {} => {}".format(vp, mode, ap))
         ret = FTPHandler.ftp_STOR(self, file, mode)
@@ -591,7 +619,7 @@ class Ftpd(object):
         if "::" in ips:
             ips.append("0.0.0.0")
 
-        ips = [x for x in ips if "unix:" not in x]
+        ips = [x for x in ips if not x.startswith(("unix:", "fd:"))]
 
         if self.args.ftp4:
             ips = [x for x in ips if ":" not in x]

copyparty/httpconn.py

@@ -224,3 +224,6 @@ class HttpConn(object):
             if self.u2idx:
                 self.hsrv.put_u2idx(str(self.addr), self.u2idx)
                 self.u2idx = None
+
+            if self.rproxy:
+                self.set_rproxy()

copyparty/httpsrv.py

@@ -70,6 +70,7 @@ from .util import (
     build_netmap,
     has_resource,
     ipnorm,
+    load_ipr,
     load_ipu,
     load_resource,
     min_ex,
@@ -123,6 +124,7 @@ class HttpSrv(object):
         self.nm = NetMap([], [])
         self.ssdp: Optional["SSDPr"] = None
         self.gpwd = Garda(self.args.ban_pw)
+        self.gpwc = Garda(self.args.ban_pwc)
         self.g404 = Garda(self.args.ban_404)
         self.g403 = Garda(self.args.ban_403)
         self.g422 = Garda(self.args.ban_422, False)
@@ -175,6 +177,7 @@ class HttpSrv(object):
             "browser",
             "browser2",
             "cf",
+            "idp",
             "md",
             "mde",
             "msg",
@@ -184,6 +187,7 @@ class HttpSrv(object):
             "svcs",
         ]
         self.j2 = {x: env.get_template(x + ".html") for x in jn}
+        self.j2["opds"] = env.get_template("opds.xml")
         self.prism = has_resource(self.E, "web/deps/prism.js.gz")
 
         if self.args.ipu:
@@ -191,6 +195,11 @@ class HttpSrv(object):
         else:
             self.ipu_iu = self.ipu_nm = None
 
+        if self.args.ipr:
+            self.ipr = load_ipr(self.log, self.args.ipr)
+        else:
+            self.ipr = None
+
         self.ipa_nm = build_netmap(self.args.ipa)
         self.xff_nm = build_netmap(self.args.xff_src)
         self.xff_lan = build_netmap("lan")
@@ -313,6 +322,8 @@ class HttpSrv(object):
 
         Daemon(self.broker.say, "sig-hsrv-up1", ("cb_httpsrv_up",))
 
+        saddr = ("", 0)  # fwd-decl for `except TypeError as ex:`
+
         while not self.stopping:
             if self.args.log_conn:
                 self.log(self.name, "|%sC-ncli" % ("-" * 1,), c="90")
@@ -320,7 +331,8 @@ class HttpSrv(object):
             spins = 0
             while self.ncli >= self.nclimax:
                 if not spins:
-                    self.log(self.name, "at connection limit; waiting", 3)
+                    t = "at connection limit (global-option 'nc'); waiting"
+                    self.log(self.name, t, 3)
 
                 spins += 1
                 time.sleep(0.1)
@@ -371,8 +383,8 @@ class HttpSrv(object):
                 if nloris < nconn / 2:
                     continue
 
-                t = "slowloris (idle-conn): {} banned for {} min"
-                self.log(self.name, t.format(ip, self.args.loris, nclose), 1)
+                t = "slow%s (idle-conn): %s banned for %d min"  # slowloris
+                self.log(self.name, t % ("loris", ip, self.args.loris), 1)
                 self.bans[ip] = int(time.time() + self.args.loris * 60)
 
             if self.args.log_conn:
@@ -394,6 +406,19 @@ class HttpSrv(object):
                 self.log(self.name, "accept({}): {}".format(fno, ex), c=6)
                 time.sleep(0.02)
                 continue
+            except TypeError as ex:
+                # on macOS, accept() may return a None saddr if blocked by LittleSnitch;
+                # unicode(saddr[0]) ==> TypeError: 'NoneType' object is not subscriptable
+                if tcp and not saddr:
+                    t = "accept(%s): failed to accept connection from client due to firewall or network issue"
+                    self.log(self.name, t % (fno,), c=3)
+                    try:
+                        sck.close()  # type: ignore
+                    except:
+                        pass
+                    time.sleep(0.02)
+                    continue
+                raise
 
             if self.args.log_conn:
                 t = "|{}C-acc2 \033[0;36m{} \033[3{}m{}".format(
@@ -547,7 +572,7 @@ class HttpSrv(object):
 
             v = self.E.t0
             try:
-                with os.scandir(os.path.join(self.E.mod, "web")) as dh:
+                with os.scandir(self.E.mod_ + "web") as dh:
                     for fh in dh:
                         inf = fh.stat()
                         v = max(v, inf.st_mtime)

copyparty/ico.py

@@ -94,10 +94,21 @@ class Ico(object):
 <?xml version="1.0" encoding="UTF-8"?>
 <svg version="1.1" viewBox="0 0 100 {}" xmlns="http://www.w3.org/2000/svg"><g>
 <rect width="100%" height="100%" fill="#{}" />
-<text x="50%" y="50%" dominant-baseline="middle" text-anchor="middle" xml:space="preserve"
+<text x="50%" y="{}" dominant-baseline="middle" text-anchor="middle" xml:space="preserve"
   fill="#{}" font-family="monospace" font-size="14px" style="letter-spacing:.5px">{}</text>
 </g></svg>
 """
-        svg = svg.format(h, c[:6], c[6:], html_escape(ext, True))
+
+        txt = html_escape(ext, True)
+        if "\n" in txt:
+            lines = txt.split("\n")
+            n = len(lines)
+            y = "20%" if n == 2 else "10%" if n == 3 else "0"
+            zs = '<tspan x="50%%" dy="1.2em">%s</tspan>'
+            txt = "".join([zs % (x,) for x in lines])
+        else:
+            y = "50%"
+
+        svg = svg.format(h, c[:6], y, c[6:], txt)
 
         return "image/svg+xml", svg.encode("utf-8")

copyparty/mdns.py

@@ -2,6 +2,7 @@
 from __future__ import print_function, unicode_literals
 
 import errno
+import os
 import random
 import select
 import socket
@@ -12,9 +13,16 @@ from ipaddress import IPv4Network, IPv6Network
 from .__init__ import TYPE_CHECKING
 from .__init__ import unicode as U
 from .multicast import MC_Sck, MCast
-from .stolen.dnslib import AAAA
-from .stolen.dnslib import CLASS as DC
-from .stolen.dnslib import (
+from .util import IP6_LL, CachedSet, Daemon, Netdev, list_ips, min_ex
+
+try:
+    if os.getenv("PRTY_SYS_ALL") or os.getenv("PRTY_SYS_DNSLIB"):
+        raise ImportError()
+    from .stolen.dnslib import (
+        AAAA,
+    )
+    from .stolen.dnslib import CLASS as DC
+    from .stolen.dnslib import (
         NSEC,
         PTR,
         QTYPE,
@@ -26,8 +34,31 @@ from .stolen.dnslib import (
         DNSQuestion,
         DNSRecord,
         set_avahi_379,
-)
-from .util import CachedSet, Daemon, Netdev, list_ips, min_ex
+    )
+
+    DNS_VND = True
+except ImportError:
+    DNS_VND = False
+    from dnslib import (
+        AAAA,
+    )
+    from dnslib import CLASS as DC
+    from dnslib import (
+        NSEC,
+        PTR,
+        QTYPE,
+        RR,
+        SRV,
+        TXT,
+        A,
+        Bimap,
+        DNSHeader,
+        DNSQuestion,
+        DNSRecord,
+    )
+
+    DC.forward[0x8001] = "F_IN"
+    DC.reverse["F_IN"] = 0x8001
 
 if TYPE_CHECKING:
     from .svchub import SvcHub
@@ -35,6 +66,11 @@ if TYPE_CHECKING:
 if True:  # pylint: disable=using-constant-test
     from typing import Any, Optional, Union
 
+if os.getenv("PRTY_MODSPEC"):
+    from inspect import getsourcefile
+
+    print("PRTY_MODSPEC: dnslib:", getsourcefile(A))
+
 
 MDNS4 = "224.0.0.251"
 MDNS6 = "ff02::fb"
@@ -73,10 +109,11 @@ class MDNS(MCast):
         self.ngen = ngen
         self.ttl = 300
 
-        if not self.args.zm_nwa_1:
+        if not self.args.zm_nwa_1 and DNS_VND:
             set_avahi_379()
 
-        zs = self.args.name + ".local."
+        zs = self.args.zm_fqdn or (self.args.name + ".local")
+        zs = zs.replace("--name", self.args.name).rstrip(".") + "."
         zs = zs.encode("ascii", "replace").decode("ascii", "replace")
         self.hn = "-".join(x for x in zs.split("?") if x) or (
             "vault-{}".format(random.randint(1, 255))
@@ -99,9 +136,14 @@ class MDNS(MCast):
         self.log_func(self.logsrc, msg, c)
 
     def build_svcs(self) -> tuple[dict[str, dict[str, Any]], set[str]]:
+        ar = self.args
         zms = self.args.zms
-        http = {"port": 80 if 80 in self.args.p else self.args.p[0]}
-        https = {"port": 443 if 443 in self.args.p else self.args.p[0]}
+
+        zi = ar.zm_http
+        http = {"port": zi if zi != -1 else 80 if 80 in ar.p else ar.p[0]}
+        zi = ar.zm_https
+        https = {"port": zi if zi != -1 else 443 if 443 in ar.p else ar.p[0]}
+
         webdav = http.copy()
         webdavs = https.copy()
         webdav["u"] = webdavs["u"] = "u"  # KDE requires username
@@ -126,16 +168,16 @@ class MDNS(MCast):
 
         svcs: dict[str, dict[str, Any]] = {}
 
-        if "d" in zms:
+        if "d" in zms and http["port"]:
             svcs["_webdav._tcp.local."] = webdav
 
-        if "D" in zms:
+        if "D" in zms and https["port"]:
             svcs["_webdavs._tcp.local."] = webdavs
 
-        if "h" in zms:
+        if "h" in zms and http["port"]:
             svcs["_http._tcp.local."] = http
 
-        if "H" in zms:
+        if "H" in zms and https["port"]:
             svcs["_https._tcp.local."] = https
 
         if "f" in zms.lower():
@@ -374,7 +416,7 @@ class MDNS(MCast):
         cip = addr[0]
         v6 = ":" in cip
         if (cip.startswith("169.254") and not self.ll_ok) or (
-            v6 and not cip.startswith("fe80")
+            v6 and not cip.startswith(IP6_LL)
         ):
             return
 

copyparty/metrics.py

@@ -17,10 +17,10 @@ class Metrics(object):
         self.hsrv = hsrv
 
     def tx(self, cli: "HttpCli") -> bool:
-        if not cli.avol:
-            raise Pebkac(403, "not allowed for user " + cli.uname)
-
         args = cli.args
+        if not cli.avol and cli.uname.lower() not in args.stats_u_set:
+            raise Pebkac(403, "'stats' not allowed for user " + cli.uname)
+
         if not args.stats:
             raise Pebkac(403, "the stats feature is not enabled in server config")
 

copyparty/mtag.py

@@ -18,6 +18,7 @@ from .util import (
     REKOBO_LKEY,
     VF_CAREFUL,
     fsenc,
+    gzip,
     min_ex,
     pybin,
     retchk,
@@ -28,7 +29,7 @@ from .util import (
 )
 
 if True:  # pylint: disable=using-constant-test
-    from typing import Any, Optional, Union
+    from typing import IO, Any, Optional, Union
 
     from .util import NamedLogger, RootLogger
 
@@ -66,6 +67,8 @@ HAVE_FFPROBE = not os.environ.get("PRTY_NO_FFPROBE") and have_ff("ffprobe")
 CBZ_PICS = set("png jpg jpeg gif bmp tga tif tiff webp avif".split())
 CBZ_01 = re.compile(r"(^|[^0-9v])0+[01]\b")
 
+FMT_AU = set("mp3 ogg flac wav".split())
+
 
 class MParser(object):
     def __init__(self, cmdline: str) -> None:
@@ -138,8 +141,6 @@ def au_unpk(
         fd, ret = tempfile.mkstemp("." + au)
 
         if pk == "gz":
-            import gzip
-
             fi = gzip.GzipFile(abspath, mode="rb")
 
         elif pk == "xz":
@@ -167,12 +168,17 @@ def au_unpk(
             znil = [x for x in znil if "cover" in x[0]] or znil
             znil = [x for x in znil if CBZ_01.search(x[0])] or znil
             t = "cbz: %d files, %d hits" % (nf, len(znil))
-            if znil:
-                t += ", using " + znil[0][1].filename
-            log(t)
             if not znil:
                 raise Exception("no images inside cbz")
-            fi = zf.open(znil[0][1])
+            using = sorted(znil)[0][1].filename
+            if znil:
+                t += ", using " + using
+            log(t)
+            fi = zf.open(using)
+
+        elif pk == "epub":
+            fi = get_cover_from_epub(log, abspath)
+            assert fi  # !rm
 
         else:
             raise Exception("unknown compression %s" % (pk,))
@@ -194,16 +200,17 @@ def au_unpk(
 
     except Exception as ex:
         if ret:
-            t = "failed to decompress audio file %r: %r"
+            t = "failed to decompress file %r: %r"
             log(t % (abspath, ex))
             wunlink(log, ret, vn.flags if vn else VF_CAREFUL)
+            return ""
 
         return abspath
 
 
 def ffprobe(
     abspath: str, timeout: int = 60
-) -> tuple[dict[str, tuple[int, Any]], dict[str, list[Any]]]:
+) -> tuple[dict[str, tuple[int, Any]], dict[str, list[Any]], list[Any], dict[str, Any]]:
     cmd = [
         b"ffprobe",
         b"-hide_banner",
@@ -217,8 +224,17 @@ def ffprobe(
     return parse_ffprobe(so)
 
 
-def parse_ffprobe(txt: str) -> tuple[dict[str, tuple[int, Any]], dict[str, list[Any]]]:
-    """ffprobe -show_format -show_streams"""
+def parse_ffprobe(
+    txt: str,
+) -> tuple[dict[str, tuple[int, Any]], dict[str, list[Any]], list[Any], dict[str, Any]]:
+    """
+    txt: output from ffprobe -show_format -show_streams
+    returns:
+     * normalized tags
+     * original/raw tags
+     * list of streams
+     * format props
+    """
     streams = []
     fmt = {}
     g = {}
@@ -242,7 +258,7 @@ def parse_ffprobe(txt: str) -> tuple[dict[str, tuple[int, Any]], dict[str, list[
     ret: dict[str, Any] = {}  # processed
     md: dict[str, list[Any]] = {}  # raw tags
 
-    is_audio = fmt.get("format_name") in ["mp3", "ogg", "flac", "wav"]
+    is_audio = fmt.get("format_name") in FMT_AU
     if fmt.get("filename", "").split(".")[-1].lower() in ["m4a", "aac"]:
         is_audio = True
 
@@ -270,6 +286,8 @@ def parse_ffprobe(txt: str) -> tuple[dict[str, tuple[int, Any]], dict[str, list[
                 ["channel_layout", "chs"],
                 ["sample_rate", ".hz"],
                 ["bit_rate", ".aq"],
+                ["bits_per_sample", ".bps"],
+                ["bits_per_raw_sample", ".bprs"],
                 ["duration", ".dur"],
             ]
 
@@ -309,7 +327,7 @@ def parse_ffprobe(txt: str) -> tuple[dict[str, tuple[int, Any]], dict[str, list[
                 ret[rk] = v1
 
     if ret.get("vc") == "ansi":  # shellscript
-        return {}, {}
+        return {}, {}, [], {}
 
     for strm in streams:
         for sk, sv in strm.items():
@@ -358,7 +376,83 @@ def parse_ffprobe(txt: str) -> tuple[dict[str, tuple[int, Any]], dict[str, list[
     zero = int("0")
     zd = {k: (zero, v) for k, v in ret.items()}
 
-    return zd, md
+    return zd, md, streams, fmt
+
+
+def get_cover_from_epub(log: "NamedLogger", abspath: str) -> Optional[IO[bytes]]:
+    import zipfile
+
+    from .dxml import parse_xml
+
+    try:
+        from urlparse import urljoin  # Python2
+    except ImportError:
+        from urllib.parse import urljoin  # Python3
+
+    with zipfile.ZipFile(abspath, "r") as z:
+        # First open the container file to find the package document (.opf file)
+        try:
+            container_root = parse_xml(z.read("META-INF/container.xml").decode())
+        except KeyError:
+            log("epub: no container file found in %s" % (abspath,))
+            return None
+
+        # https://www.w3.org/TR/epub-33/#sec-container.xml-rootfile-elem
+        container_ns = {"": "urn:oasis:names:tc:opendocument:xmlns:container"}
+        # One file could contain multiple package documents, default to the first one
+        rootfile_path = container_root.find("./rootfiles/rootfile", container_ns).get(
+            "full-path"
+        )
+
+        # Then open the first package document to find the path of the cover image
+        try:
+            package_root = parse_xml(z.read(rootfile_path).decode())
+        except KeyError:
+            log("epub: no package document found in %s" % (abspath,))
+            return None
+
+        # https://www.w3.org/TR/epub-33/#sec-package-doc
+        package_ns = {"": "http://www.idpf.org/2007/opf"}
+        # https://www.w3.org/TR/epub-33/#sec-cover-image
+        coverimage_path_node = package_root.find(
+            "./manifest/item[@properties='cover-image']", package_ns
+        )
+        if coverimage_path_node is not None:
+            coverimage_path = coverimage_path_node.get("href")
+        else:
+            # This might be an EPUB2 file, try the legacy way of specifying covers
+            coverimage_path = _get_cover_from_epub2(log, package_root, package_ns)
+
+        if not coverimage_path:
+            raise Exception("no cover inside epub")
+
+        # This url is either absolute (in the .epub) or relative to the package document
+        adjusted_cover_path = urljoin(rootfile_path, coverimage_path)
+
+        try:
+            return z.open(adjusted_cover_path)
+        except KeyError:
+            t = "epub: cover specified in package document, but doesn't exist: %s"
+            log(t % (adjusted_cover_path,))
+
+
+def _get_cover_from_epub2(
+    log: "NamedLogger", package_root, package_ns
+) -> Optional[str]:
+    # <meta name="cover" content="id-to-cover-image"> in <metadata>, then
+    # <item> in <manifest>
+    xn = package_root.find("./metadata/meta[@name='cover']", package_ns)
+    cover_id = xn.get("content") if xn is not None else None
+
+    if not cover_id:
+        return None
+
+    for node in package_root.iterfind("./manifest/item", package_ns):
+        if node.get("id") == cover_id:
+            cover_path = node.get("href")
+            return cover_path
+
+    return None
 
 
 class MTag(object):
@@ -424,7 +518,6 @@ class MTag(object):
                 "album-artist",
                 "tpe2",
                 "aart",
-                "conductor",
                 "organization",
                 "band",
             ],
@@ -558,6 +651,9 @@ class MTag(object):
             return self._get(abspath)
 
         ap = au_unpk(self.log, self.args.au_unpk, abspath)
+        if not ap:
+            return {}
+
         ret = self._get(ap)
         if ap != abspath:
             wunlink(self.log, ap, VF_CAREFUL)
@@ -629,7 +725,7 @@ class MTag(object):
         if not bos.path.isfile(abspath):
             return {}
 
-        ret, md = ffprobe(abspath, self.args.mtag_to)
+        ret, md, _, _ = ffprobe(abspath, self.args.mtag_to)
 
         if self.args.mtag_vv:
             for zd in (ret, dict(md)):
@@ -663,6 +759,9 @@ class MTag(object):
             ap = abspath
 
         ret: dict[str, Any] = {}
+        if not ap:
+            return ret
+
         for tagname, parser in sorted(parsers.items(), key=lambda x: (x[1].pri, x[0])):
             try:
                 cmd = [parser.bin, ap]

copyparty/multicast.py

@@ -15,7 +15,7 @@ from ipaddress import (
 )
 
 from .__init__ import MACOS, TYPE_CHECKING
-from .util import Daemon, Netdev, find_prefix, min_ex, spack
+from .util import IP6_LL, IP64_LL, Daemon, Netdev, find_prefix, min_ex, spack
 
 if TYPE_CHECKING:
     from .svchub import SvcHub
@@ -96,7 +96,10 @@ class MCast(object):
     def create_servers(self) -> list[str]:
         bound: list[str] = []
         netdevs = self.hub.tcpsrv.netdevs
-        ips = [x[0] for x in self.hub.tcpsrv.bound]
+        blist = self.hub.tcpsrv.bound
+        if self.args.http_no_tcp:
+            blist = self.hub.tcpsrv.seen_eps
+        ips = [x[0] for x in blist]
 
         if "::" in ips:
             ips = [x for x in ips if x != "::"] + list(
@@ -145,7 +148,7 @@ class MCast(object):
         all_selected = ips[:]
 
         # discard non-linklocal ipv6
-        ips = [x for x in ips if ":" not in x or x.startswith("fe80")]
+        ips = [x for x in ips if ":" not in x or x.startswith(IP6_LL)]
 
         if not ips:
             raise NoIPs()
@@ -163,6 +166,7 @@ class MCast(object):
             sck.settimeout(None)
             sck.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
             try:
+                # safe for this purpose; https://lwn.net/Articles/853637/
                 sck.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
             except:
                 pass
@@ -182,11 +186,7 @@ class MCast(object):
                 srv.ips[oth_ip.split("/")[0]] = ipaddress.ip_network(oth_ip, False)
 
             # gvfs breaks if a linklocal ip appears in a dns reply
-            ll = {
-                k: v
-                for k, v in srv.ips.items()
-                if k.startswith("169.254") or k.startswith("fe80")
-            }
+            ll = {k: v for k, v in srv.ips.items() if k.startswith(IP64_LL)}
             rt = {k: v for k, v in srv.ips.items() if k not in ll}
 
             if self.args.ll or not rt:

copyparty/pwhash.py

@@ -15,7 +15,7 @@ try:
         raise Exception()
 
     HAVE_ARGON2 = True
-    from argon2 import __version__ as argon2ver
+    from argon2 import exceptions as argon2ex
 except:
     HAVE_ARGON2 = False
 
@@ -25,6 +25,7 @@ class PWHash(object):
         self.args = args
 
         zsl = args.ah_alg.split(",")
+        zsl = [x.strip() for x in zsl]
         alg = zsl[0]
         if alg == "none":
             alg = ""
@@ -147,6 +148,10 @@ class PWHash(object):
     def cli(self) -> None:
         import getpass
 
+        if self.args.usernames:
+            t = "since you have enabled --usernames, please provide username:password"
+            print(t)
+
         while True:
             try:
                 p1 = getpass.getpass("password> ")

copyparty/qrkode.py

@@ -0,0 +1,112 @@
+# coding: utf-8
+from __future__ import print_function, unicode_literals
+
+import os
+
+try:
+    if os.getenv("PRTY_SYS_ALL") or os.getenv("PRTY_SYS_QRCG"):
+        raise ImportError()
+    from .stolen.qrcodegen import QrCode
+
+    qrgen = QrCode.encode_binary
+    VENDORED = True
+except ImportError:
+    VENDORED = False
+    from qrcodegen import QrCode
+
+if os.getenv("PRTY_MODSPEC"):
+    from inspect import getsourcefile
+
+    print("PRTY_MODSPEC: qrcode:", getsourcefile(QrCode))
+
+if True:  # pylint: disable=using-constant-test
+    import typing
+    from typing import Any, Optional, Sequence, Union
+
+
+if not VENDORED:
+
+    def _qrgen(data: Union[bytes, Sequence[int]]) -> "QrCode":
+        ret = None
+        V = QrCode.Ecc
+        for e in [V.HIGH, V.QUARTILE, V.MEDIUM, V.LOW]:
+            qr = QrCode.encode_binary(data, e)
+            qr.size = qr._size
+            qr.modules = qr._modules
+            if not ret or ret.size > qr.size:
+                ret = qr
+        return ret
+
+    qrgen = _qrgen
+
+
+def qr2txt(qr: QrCode, zoom: int = 1, pad: int = 4) -> str:
+    tab = qr.modules
+    sz = qr.size
+    if sz % 2 and zoom == 1:
+        tab.append([False] * sz)
+
+    tab = [[False] * sz] * pad + tab + [[False] * sz] * pad
+    tab = [[False] * pad + x + [False] * pad for x in tab]
+
+    rows: list[str] = []
+    if zoom == 1:
+        for y in range(0, len(tab), 2):
+            row = ""
+            for x in range(len(tab[y])):
+                v = 2 if tab[y][x] else 0
+                v += 1 if tab[y + 1][x] else 0
+                row += " ▄▀█"[v]
+            rows.append(row)
+    else:
+        for tr in tab:
+            row = ""
+            for zb in tr:
+                row += " █"[int(zb)] * 2
+            rows.append(row)
+
+    return "\n".join(rows)
+
+
+def qr2png(
+    qr: QrCode,
+    zoom: int,
+    pad: int,
+    bg: Optional[tuple[int, int, int]],
+    fg: Optional[tuple[int, int, int]],
+    ap: str,
+) -> None:
+    from PIL import Image
+
+    tab = qr.modules
+    sz = qr.size
+    psz = sz + pad * 2
+    if bg:
+        img = Image.new("RGB", (psz, psz), bg)
+    else:
+        img = Image.new("RGBA", (psz, psz), (0, 0, 0, 0))
+        fg = (fg[0], fg[1], fg[2], 255)
+    for y in range(sz):
+        for x in range(sz):
+            if tab[y][x]:
+                img.putpixel((x + pad, y + pad), fg)
+    if zoom != 1:
+        img = img.resize((sz * zoom, sz * zoom), Image.Resampling.NEAREST)
+    img.save(ap)
+
+
+def qr2svg(qr: QrCode, border: int) -> str:
+    parts: list[str] = []
+    for y in range(qr.size):
+        sy = border + y
+        for x in range(qr.size):
+            if qr.modules[y][x]:
+                parts.append("M%d,%dh1v1h-1z" % (border + x, sy))
+    t = """\
+<?xml version="1.0" encoding="UTF-8"?>
+<svg xmlns="http://www.w3.org/2000/svg" version="1.1" viewBox="0 0 {0} {0}" stroke="none">
+<rect width="100%" height="100%" fill="#F7F7F7"/>
+<path d="{1}" fill="#111111"/>
+</svg>
+"""
+    return t.format(qr.size + border * 2, " ".join(parts))

copyparty/smbd.py

@@ -246,7 +246,8 @@ class SMB(object):
 
             ap = absreal(ap)
             xbu = vfs.flags.get("xbu")
-            if xbu and not runhook(
+            if xbu:
+                hr = runhook(
                     self.nlog,
                     None,
                     self.hub.up2k,
@@ -261,9 +262,13 @@ class SMB(object):
                     0,
                     "1.7.6.2",
                     time.time(),
-                "",
-            ):
-                yeet("blocked by xbu server config: %r" % (vpath,))
+                    None,
+                )
+                t = hr.get("rejectmsg") or ""
+                if t or hr.get("rc") != 0:
+                    if not t:
+                        t = "blocked by xbu server config: %r" % (vpath,)
+                    yeet(t)
 
         ret = bos.open(ap, flags, *a, mode=chmod, **ka)
         if wr:
@@ -318,9 +323,9 @@ class SMB(object):
             t = "blocked rename (no-move-acc %s): /%s @%s"
             yeet(t % (vfs1.axs.umove, vp1, uname))
 
-        self.hub.up2k.handle_mv(uname, "1.7.6.2", vp1, vp2)
+        self.hub.up2k.handle_mv("", uname, "1.7.6.2", vp1, vp2)
         try:
-            bos.makedirs(ap2)
+            bos.makedirs(ap2, vf=vfs2.flags)
         except:
             pass
 
@@ -334,7 +339,7 @@ class SMB(object):
             t = "blocked mkdir (no-write-acc %s): /%s @%s"
             yeet(t % (vfs.axs.uwrite, vpath, uname))
 
-        return bos.mkdir(ap)
+        return bos.mkdir(ap, vfs.flags["chmod_d"])
 
     def _stat(self, vpath: str, *a: Any, **ka: Any) -> os.stat_result:
         try:
@@ -373,7 +378,7 @@ class SMB(object):
             t = "blocked utime (no-write-acc %s): /%s @%s"
             yeet(t % (vfs.axs.uwrite, vpath, uname))
 
-        return bos.utime(ap, times)
+        bos.utime_c(info, ap, int(times[1]), False)
 
     def _p_exists(self, vpath: str) -> bool:
         # ap = "?"

copyparty/stolen/qrcodegen.py

@@ -4,7 +4,7 @@
 # https://github.com/nayuki/QR-Code-generator/blob/daa3114/python/qrcodegen.py
 # the original ^ is extremely well commented so refer to that for explanations
 
-# hacks: binary-only, auto-ecc, render, py2-compat
+# hacks: binary-only, auto-ecc, py2-compat
 
 from __future__ import print_function, unicode_literals
 
@@ -173,33 +173,6 @@ class QrCode(object):
         self._apply_mask(msk)  # Apply the final choice of mask
         self._draw_format_bits(msk)  # Overwrite old format bits
 
-    def render(self, zoom=1, pad=4) -> str:
-        tab = self.modules
-        sz = self.size
-        if sz % 2 and zoom == 1:
-            tab.append([False] * sz)
-
-        tab = [[False] * sz] * pad + tab + [[False] * sz] * pad
-        tab = [[False] * pad + x + [False] * pad for x in tab]
-
-        rows: list[str] = []
-        if zoom == 1:
-            for y in range(0, len(tab), 2):
-                row = ""
-                for x in range(len(tab[y])):
-                    v = 2 if tab[y][x] else 0
-                    v += 1 if tab[y + 1][x] else 0
-                    row += " ▄▀█"[v]
-                rows.append(row)
-        else:
-            for tr in tab:
-                row = ""
-                for zb in tr:
-                    row += " █"[int(zb)] * 2
-                rows.append(row)
-
-        return "\n".join(rows)
-
     def _draw_function_patterns(self) -> None:
         # Draw horizontal and vertical timing patterns
         for i in range(self.size):
@@ -594,20 +567,3 @@ def _get_bit(x: int, i: int) -> bool:
 
 class DataTooLongError(ValueError):
     pass
-
-
-def qr2svg(qr: QrCode, border: int) -> str:
-    parts: list[str] = []
-    for y in range(qr.size):
-        sy = border + y
-        for x in range(qr.size):
-            if qr.modules[y][x]:
-                parts.append("M%d,%dh1v1h-1z" % (border + x, sy))
-    t = """\
-<?xml version="1.0" encoding="UTF-8"?>
-<svg xmlns="http://www.w3.org/2000/svg" version="1.1" viewBox="0 0 {0} {0}" stroke="none">
-<rect width="100%" height="100%" fill="#F7F7F7"/>
-<path d="{1}" fill="#111111"/>
-</svg>
-"""
-    return t.format(qr.size + border * 2, " ".join(parts))

copyparty/sutil.py

@@ -17,6 +17,9 @@ if True:  # pylint: disable=using-constant-test
     from .util import NamedLogger
 
 
+TAR_NO_OPUS = set("aac|m4a|mp3|oga|ogg|opus|wma".split("|"))
+
+
 class StreamArc(object):
     def __init__(
         self,
@@ -82,9 +85,7 @@ def enthumb(
 ) -> dict[str, Any]:
     rem = f["vp"]
     ext = rem.rsplit(".", 1)[-1].lower()
-    if (fmt == "mp3" and ext == "mp3") or (
-        fmt == "opus" and ext in "aac|m4a|mp3|ogg|opus|wma".split("|")
-    ):
+    if (fmt == "mp3" and ext == "mp3") or (fmt == "opus" and ext in TAR_NO_OPUS):
         raise Exception()
 
     vp = vjoin(vtop, rem.split("/", 1)[1])

copyparty/svchub.py

@@ -2,8 +2,8 @@
 from __future__ import print_function, unicode_literals
 
 import argparse
+import atexit
 import errno
-import gzip
 import logging
 import os
 import re
@@ -27,8 +27,10 @@ if True:  # pylint: disable=using-constant-test
     from typing import Any, Optional, Union
 
 from .__init__ import ANYWIN, EXE, MACOS, PY2, TYPE_CHECKING, E, EnvParams, unicode
-from .authsrv import BAD_CFG, AuthSrv
+from .authsrv import BAD_CFG, AuthSrv, derive_args, n_du_who, n_ver_who
+from .bos import bos
 from .cert import ensure_cert
+from .fsutil import ramdisk_chk
 from .mtag import HAVE_FFMPEG, HAVE_FFPROBE, HAVE_MUTAGEN
 from .pwhash import HAVE_ARGON2
 from .tcpsrv import TcpSrv
@@ -38,6 +40,7 @@ from .th_srv import (
     HAVE_FFPROBE,
     HAVE_HEIF,
     HAVE_PIL,
+    HAVE_RAW,
     HAVE_VIPS,
     HAVE_WEBP,
     ThumbSrv,
@@ -51,6 +54,7 @@ from .util import (
     HAVE_PSUTIL,
     HAVE_SQLITE3,
     HAVE_ZMQ,
+    RE_ANSI,
     URL_BUG,
     UTC,
     VERSIONS,
@@ -60,19 +64,26 @@ from .util import (
     HMaccas,
     ODict,
     alltrace,
-    ansi_re,
     build_netmap,
     expat_ver,
+    gzip,
+    html_escape,
+    load_ipr,
     load_ipu,
+    lock_file,
     min_ex,
     mp,
     odfusion,
     pybin,
     start_log_thrs,
     start_stackmon,
+    termsize,
     ub64enc,
 )
 
+if HAVE_SQLITE3:
+    import sqlite3
+
 if TYPE_CHECKING:
     try:
         from .mdns import MDNS
@@ -84,6 +95,11 @@ if PY2:
     range = xrange  # type: ignore
 
 
+VER_IDP_DB = 1
+VER_SESSION_DB = 1
+VER_SHARES_DB = 2
+
+
 class SvcHub(object):
     """
     Hosts all services which cannot be parallelized due to reliance on monolithic resources.
@@ -120,6 +136,7 @@ class SvcHub(object):
         self.nsigs = 3
         self.retcode = 0
         self.httpsrv_up = 0
+        self.qr_tsz = None
 
         self.log_mutex = threading.Lock()
         self.cday = 0
@@ -141,7 +158,8 @@ class SvcHub(object):
             args.unpost = 0
             args.no_del = True
             args.no_mv = True
-            args.hardlink = True
+            args.reflink = True
+            args.dav_auth = True
             args.vague_403 = True
             args.nih = True
 
@@ -158,6 +176,7 @@ class SvcHub(object):
         # for non-http clients (ftp, tftp)
         self.bans: dict[str, int] = {}
         self.gpwd = Garda(self.args.ban_pw)
+        self.gpwc = Garda(self.args.ban_pwc)
         self.g404 = Garda(self.args.ban_404)
         self.g403 = Garda(self.args.ban_403)
         self.g422 = Garda(self.args.ban_422, False)
@@ -186,8 +205,14 @@ class SvcHub(object):
 
         if not args.use_fpool and args.j != 1:
             args.no_fpool = True
-            t = "multithreading enabled with -j {}, so disabling fpool -- this can reduce upload performance on some filesystems"
-            self.log("root", t.format(args.j))
+            t = "multithreading enabled with -j {}, so disabling fpool -- this can reduce upload performance on some filesystems, and make some antivirus-softwares "
+            c = 0
+            if ANYWIN:
+                t += "(especially Microsoft Defender) stress your CPU and HDD severely during big uploads"
+                c = 3
+            else:
+                t += "consume more resources (CPU/HDD) than normal"
+            self.log("root", t.format(args.j), c)
 
         if not args.no_fpool and args.j != 1:
             t = "WARNING: ignoring --use-fpool because multithreading (-j{}) is enabled"
@@ -223,8 +248,8 @@ class SvcHub(object):
             t = "WARNING: --th-ram-max is very small (%.2f GiB); will not be able to %s"
             self.log("root", t % (args.th_ram_max, zs), 3)
 
-        if args.chpw and args.idp_h_usr:
-            t = "ERROR: user-changeable passwords is incompatible with IdP/identity-providers; you must disable either --chpw or --idp-h-usr"
+        if args.chpw and args.have_idp_hdrs and "pw" not in args.auth_ord.split(","):
+            t = "ERROR: user-changeable passwords is not compatible with your current configuration. Choose one of these options to fix it:\n option1: disable --chpw\n option2: remove all use of IdP features; --idp-*\n option3: change --auth-ord to something like pw,idp,ipu"
             self.log("root", t, 1)
             raise Exception(t)
 
@@ -239,8 +264,24 @@ class SvcHub(object):
             setattr(args, "ipu_iu", iu)
             setattr(args, "ipu_nm", nm)
 
+        if args.ipr:
+            ipr = load_ipr(self.log, args.ipr, True)
+            setattr(args, "ipr_u", ipr)
+
+        for zs in "ah_salt fk_salt dk_salt".split():
+            if getattr(args, "show_%s" % (zs,)):
+                self.log("root", "effective %s is %s" % (zs, getattr(args, zs)))
+
+        if args.ah_cli or args.ah_gen:
+            args.idp_store = 0
+            args.no_ses = True
+            args.shr = ""
+
+        if args.idp_store and args.have_idp_hdrs:
+            self.setup_db("idp")
+
         if not self.args.no_ses:
-            self.setup_session_db()
+            self.setup_db("ses")
 
         args.shr1 = ""
         if args.shr:
@@ -250,6 +291,17 @@ class SvcHub(object):
         ch = "abcdefghijklmnopqrstuvwx"[int(args.theme / 2)]
         args.theme = "{0}{1} {0} {1}".format(ch, bri)
 
+        if args.no_stack:
+            args.stack_who = "no"
+
+        if args.nid:
+            args.du_who = "no"
+        args.du_iwho = n_du_who(args.du_who)
+
+        if args.ver and args.ver_who == "no":
+            args.ver_who = "all"
+        args.ver_iwho = n_ver_who(args.ver_who)
+
         if args.nih:
             args.vname = ""
             args.doctitle = args.doctitle.replace(" @ --name", "")
@@ -263,6 +315,7 @@ class SvcHub(object):
 
         # initiate all services to manage
         self.asrv = AuthSrv(self.args, self.log, dargs=self.dargs)
+        ramdisk_chk(self.asrv)
 
         if args.cgen:
             self.asrv.cgen()
@@ -287,11 +340,13 @@ class SvcHub(object):
 
         self._feature_test()
 
-        decs = {k: 1 for k in self.args.th_dec.split(",")}
+        decs = {k.strip(): 1 for k in self.args.th_dec.split(",")}
         if not HAVE_VIPS:
             decs.pop("vips", None)
         if not HAVE_PIL:
             decs.pop("pil", None)
+        if not HAVE_RAW:
+            decs.pop("raw", None)
         if not HAVE_FFMPEG or not HAVE_FFPROBE:
             decs.pop("ff", None)
 
@@ -340,7 +395,10 @@ class SvcHub(object):
                 t = "invalid mp3 transcoding quality [%s] specified; only supports [0] to disable, a CBR value such as [192k], or a CQ/CRF value such as [v2]"
                 raise Exception(t % (args.q_mp3,))
         else:
-            args.au_unpk = {}
+            zss = set(args.th_r_ffa.split(",") + args.th_r_ffv.split(","))
+            args.au_unpk = {
+                k: v for k, v in args.au_unpk.items() if v.split(".")[0] not in zss
+            }
 
         args.th_poke = min(args.th_poke, args.th_maxage, args.ac_maxage)
 
@@ -393,39 +451,100 @@ class SvcHub(object):
 
         # create netmaps early to avoid firewall gaps,
         # but the mutex blocks multiprocessing startup
-        for zs in "ipu_iu ftp_ipa_nm tftp_ipa_nm".split():
+        for zs in "ipu_nm ftp_ipa_nm tftp_ipa_nm".split():
             try:
                 getattr(args, zs).mutex = threading.Lock()
             except:
                 pass
+        if args.ipr:
+            for nm in args.ipr_u.values():
+                nm.mutex = threading.Lock()
 
-    def setup_session_db(self) -> None:
-        if not HAVE_SQLITE3:
+    def _db_onfail_ses(self) -> None:
         self.args.no_ses = True
-            t = "WARNING: sqlite3 not available; disabling sessions, will use plaintext passwords in cookies"
-            self.log("root", t, 3)
+
+    def _db_onfail_idp(self) -> None:
+        self.args.idp_store = 0
+
+    def setup_db(self, which: str) -> None:
+        """
+        the "non-mission-critical" databases; if something looks broken then just nuke it
+        """
+        if which == "ses":
+            native_ver = VER_SESSION_DB
+            db_path = self.args.ses_db
+            desc = "sessions-db"
+            pathopt = "ses-db"
+            sanchk_q = "select count(*) from us"
+            createfun = self._create_session_db
+            failfun = self._db_onfail_ses
+        elif which == "idp":
+            native_ver = VER_IDP_DB
+            db_path = self.args.idp_db
+            desc = "idp-db"
+            pathopt = "idp-db"
+            sanchk_q = "select count(*) from us"
+            createfun = self._create_idp_db
+            failfun = self._db_onfail_idp
+        else:
+            raise Exception("unknown cachetype")
+
+        if not db_path.endswith(".db"):
+            zs = "config option --%s (the %s) was configured to [%s] which is invalid; must be a filepath ending with .db"
+            self.log("root", zs % (pathopt, desc, db_path), 1)
+            raise Exception(BAD_CFG)
+
+        if not HAVE_SQLITE3:
+            failfun()
+            if which == "ses":
+                zs = "disabling sessions, will use plaintext passwords in cookies"
+            elif which == "idp":
+                zs = "disabling idp-db, will be unable to remember IdP-volumes after a restart"
+            self.log("root", "WARNING: sqlite3 not available; %s" % (zs,), 3)
             return
 
-        import sqlite3
+        assert sqlite3  # type: ignore  # !rm
 
+        db_lock = db_path + ".lock"
+        try:
+            create = not os.path.getsize(db_path)
+        except:
             create = True
-        db_path = self.args.ses_db
-        self.log("root", "opening sessions-db %s" % (db_path,))
-        for n in range(2):
+        zs = "creating new" if create else "opening"
+        self.log("root", "%s %s %s" % (zs, desc, db_path))
+
+        for tries in range(2):
+            sver = 0
             try:
                 db = sqlite3.connect(db_path)
                 cur = db.cursor()
                 try:
-                    cur.execute("select count(*) from us").fetchone()
-                    create = False
-                    break
+                    zs = "select v from kv where k='sver'"
+                    sver = cur.execute(zs).fetchall()[0][0]
+                    if sver > native_ver:
+                        zs = "this version of copyparty only understands %s v%d and older; the db is v%d"
+                        raise Exception(zs % (desc, native_ver, sver))
+
+                    cur.execute(sanchk_q).fetchone()
                 except:
-                    pass
-            except Exception as ex:
-                if n:
+                    if sver:
                         raise
-                t = "sessions-db corrupt; deleting and recreating: %r"
-                self.log("root", t % (ex,), 3)
+                    sver = createfun(cur)
+
+                err = self._verify_db(
+                    cur, which, pathopt, db_path, desc, sver, native_ver
+                )
+                if err:
+                    tries = 99
+                    self.args.no_ses = True
+                    self.log("root", err, 3)
+                break
+
+            except Exception as ex:
+                if tries or sver > native_ver:
+                    raise
+                t = "%s is unusable; deleting and recreating: %r"
+                self.log("root", t % (desc, ex), 3)
                 try:
                     cur.close()  # type: ignore
                 except:
@@ -434,8 +553,13 @@ class SvcHub(object):
                     db.close()  # type: ignore
                 except:
                     pass
+                try:
+                    os.unlink(db_lock)
+                except:
+                    pass
                 os.unlink(db_path)
 
+    def _create_session_db(self, cur: "sqlite3.Cursor") -> int:
         sch = [
             r"create table kv (k text, v int)",
             r"create table us (un text, si text, t0 int)",
@@ -445,17 +569,74 @@ class SvcHub(object):
             r"create index us_t0 on us(t0)",
             r"insert into kv values ('sver', 1)",
         ]
-
-        assert db  # type: ignore  # !rm
-        assert cur  # type: ignore  # !rm
-        if create:
         for cmd in sch:
             cur.execute(cmd)
         self.log("root", "created new sessions-db")
-            db.commit()
+        return 1
 
+    def _create_idp_db(self, cur: "sqlite3.Cursor") -> int:
+        sch = [
+            r"create table kv (k text, v int)",
+            r"create table us (un text, gs text)",
+            # username, groups
+            r"create index us_un on us(un)",
+            r"insert into kv values ('sver', 1)",
+        ]
+        for cmd in sch:
+            cur.execute(cmd)
+        self.log("root", "created new idp-db")
+        return 1
+
+    def _verify_db(
+        self,
+        cur: "sqlite3.Cursor",
+        which: str,
+        pathopt: str,
+        db_path: str,
+        desc: str,
+        sver: int,
+        native_ver: int,
+    ) -> str:
+        # ensure writable (maybe owned by other user)
+        db = cur.connection
+
+        try:
+            zil = cur.execute("select v from kv where k='pid'").fetchall()
+            if len(zil) > 1:
+                raise Exception()
+            owner = zil[0][0]
+        except:
+            owner = 0
+
+        if which == "ses":
+            cons = "Will now disable sessions and instead use plaintext passwords in cookies."
+        elif which == "idp":
+            cons = "Each IdP-volume will not become available until its associated user sends their first request."
+        else:
+            raise Exception()
+
+        if not lock_file(db_path + ".lock"):
+            t = "the %s [%s] is already in use by another copyparty instance (pid:%d). This is not supported; please provide another database with --%s or give this copyparty-instance its entirely separate config-folder by setting another path in the XDG_CONFIG_HOME env-var. You can also disable this safeguard by setting env-var PRTY_NO_DB_LOCK=1. %s"
+            return t % (desc, db_path, owner, pathopt, cons)
+
+        vars = (("pid", os.getpid()), ("ts", int(time.time() * 1000)))
+        if owner:
+            # wear-estimate: 2 cells; offsets 0x10, 0x50, 0x19720
+            for k, v in vars:
+                cur.execute("update kv set v=? where k=?", (v, k))
+        else:
+            # wear-estimate: 3~4 cells; offsets 0x10, 0x50, 0x19180, 0x19710, 0x36000, 0x360b0, 0x36b90
+            for k, v in vars:
+                cur.execute("insert into kv values(?, ?)", (k, v))
+
+        if sver < native_ver:
+            cur.execute("delete from kv where k='sver'")
+            cur.execute("insert into kv values('sver',?)", (native_ver,))
+
+        db.commit()
         cur.close()
         db.close()
+        return ""
 
     def setup_share_db(self) -> None:
         al = self.args
@@ -464,7 +645,7 @@ class SvcHub(object):
             al.shr = ""
             return
 
-        import sqlite3
+        assert sqlite3  # type: ignore  # !rm
 
         al.shr = al.shr.strip("/")
         if "/" in al.shr or not al.shr:
@@ -475,34 +656,48 @@ class SvcHub(object):
         al.shr = "/%s/" % (al.shr,)
         al.shr1 = al.shr[1:]
 
-        create = True
-        modified = False
+        # policy:
+        # the shares-db is important, so panic if something is wrong
+
         db_path = self.args.shr_db
-        self.log("root", "opening shares-db %s" % (db_path,))
-        for n in range(2):
+        db_lock = db_path + ".lock"
+        try:
+            create = not os.path.getsize(db_path)
+        except:
+            create = True
+        zs = "creating new" if create else "opening"
+        self.log("root", "%s shares-db %s" % (zs, db_path))
+
+        sver = 0
         try:
             db = sqlite3.connect(db_path)
             cur = db.cursor()
-                try:
+            if not create:
+                zs = "select v from kv where k='sver'"
+                sver = cur.execute(zs).fetchall()[0][0]
+                if sver > VER_SHARES_DB:
+                    zs = "this version of copyparty only understands shares-db v%d and older; the db is v%d"
+                    raise Exception(zs % (VER_SHARES_DB, sver))
+
                 cur.execute("select count(*) from sh").fetchone()
-                    create = False
-                    break
-                except:
-                    pass
         except Exception as ex:
-                if n:
+            t = "could not open shares-db; will now panic...\nthe following database must be repaired or deleted before you can launch copyparty:\n%s\n\nERROR: %s\n\nadditional details:\n%s\n"
+            self.log("root", t % (db_path, ex, min_ex()), 1)
             raise
-                t = "shares-db corrupt; deleting and recreating: %r"
-                self.log("root", t % (ex,), 3)
+
         try:
-                    cur.close()  # type: ignore
+            zil = cur.execute("select v from kv where k='pid'").fetchall()
+            if len(zil) > 1:
+                raise Exception()
+            owner = zil[0][0]
         except:
-                    pass
-                try:
-                    db.close()  # type: ignore
-                except:
-                    pass
-                os.unlink(db_path)
+            owner = 0
+
+        if not lock_file(db_lock):
+            t = "the shares-db [%s] is already in use by another copyparty instance (pid:%d). This is not supported; please provide another database with --shr-db or give this copyparty-instance its entirely separate config-folder by setting another path in the XDG_CONFIG_HOME env-var. You can also disable this safeguard by setting env-var PRTY_NO_DB_LOCK=1. Will now panic."
+            t = t % (db_path, owner)
+            self.log("root", t, 1)
+            raise Exception(t)
 
         sch1 = [
             r"create table kv (k text, v int)",
@@ -514,34 +709,37 @@ class SvcHub(object):
             r"create index sf_k on sf(k)",
             r"create index sh_k on sh(k)",
             r"create index sh_t1 on sh(t1)",
+            r"insert into kv values ('sver', 2)",
         ]
 
         assert db  # type: ignore  # !rm
         assert cur  # type: ignore  # !rm
-        if create:
-            dver = 2
-            modified = True
+        if not sver:
+            sver = VER_SHARES_DB
             for cmd in sch1 + sch2:
                 cur.execute(cmd)
             self.log("root", "created new shares-db")
-        else:
-            (dver,) = cur.execute("select v from kv where k = 'sver'").fetchall()[0]
 
-        if dver == 1:
-            modified = True
+        if sver == 1:
             for cmd in sch2:
                 cur.execute(cmd)
             cur.execute("update sh set st = 0")
             self.log("root", "shares-db schema upgrade ok")
 
-        if modified:
-            for cmd in [
-                r"delete from kv where k = 'sver'",
-                r"insert into kv values ('sver', %d)" % (2,),
-            ]:
-                cur.execute(cmd)
-            db.commit()
+        if sver < VER_SHARES_DB:
+            cur.execute("delete from kv where k='sver'")
+            cur.execute("insert into kv values('sver',?)", (VER_SHARES_DB,))
 
+        vars = (("pid", os.getpid()), ("ts", int(time.time() * 1000)))
+        if owner:
+            # wear-estimate: same as sessions-db
+            for k, v in vars:
+                cur.execute("update kv set v=? where k=?", (v, k))
+        else:
+            for k, v in vars:
+                cur.execute("insert into kv values(?, ?)", (k, v))
+
+        db.commit()
         cur.close()
         db.close()
 
@@ -606,6 +804,84 @@ class SvcHub(object):
     def sigterm(self) -> None:
         self.signal_handler(signal.SIGTERM, None)
 
+    def sticky_qr(self) -> None:
+        self._sticky_qr()
+
+    def _unsticky_qr(self, flush=True) -> None:
+        print("\033[s\033[J\033[r\033[u", file=sys.stderr, end="")
+        if flush:
+            sys.stderr.flush()
+
+    def _sticky_qr(self, force: bool = False) -> None:
+        sz = termsize()
+        if self.qr_tsz == sz:
+            if not force:
+                return
+        else:
+            force = False
+
+        if self.qr_tsz:
+            self._unsticky_qr(False)
+        else:
+            atexit.register(self._unsticky_qr)
+
+        tw, th = self.qr_tsz = sz
+        zs1, qr = self.tcpsrv.qr.split("\n", 1)
+        url, colr = zs1.split(" ", 1)
+        nl = len(qr.split("\n"))  # numlines
+        lp = 3 if nl * 2 + 4 < tw else 0  # leftpad
+        lp0 = lp
+        if self.args.qr_pin == 2:
+            url = ""
+        else:
+            while lp and (nl + lp) * 2 + len(url) + 1 > tw:
+                lp -= 1
+            if (nl + lp) * 2 + len(url) + 1 > tw:
+                qr = url + "\n" + qr
+                url = ""
+                nl += 1
+                lp = lp0
+        sh = 1 + th - nl
+        if lp:
+            zs = " " * lp
+            qr = zs + qr.replace("\n", "\n" + zs)
+        if url:
+            url = "%s\033[%d;%dH%s\033[0m" % (colr, sh + 1, (nl + lp) * 2, url)
+        qr = colr + qr
+
+        t = "%s\033[%dA" % ("\n" * nl, nl)
+        t = "%s\033[s\033[1;%dr\033[%dH%s%s\033[u" % (t, sh - 1, sh, qr, url)
+        if not force:
+            self.log("qr", "sticky-qrcode %sx%s,%s" % (tw, th, sh), 6)
+        self.pr(t, file=sys.stderr, end="")
+
+    def _qr_thr(self):
+        qr = self.tcpsrv.qr
+        w8 = self.args.qr_wait
+        if w8:
+            time.sleep(w8)
+            self.log("qr-code", qr)
+        if self.args.qr_stdout:
+            self.pr(self.tcpsrv.qr)
+        if self.args.qr_stderr:
+            self.pr(self.tcpsrv.qr, file=sys.stderr)
+        w8 = self.args.qr_every
+        msg = "%s\033[%dA" % (qr, len(qr.split("\n")))
+        while w8:
+            time.sleep(w8)
+            if self.stopping:
+                break
+            if self.args.qr_pin:
+                self._sticky_qr(True)
+            else:
+                self.log("qr-code", msg)
+        w8 = self.args.qr_winch
+        while w8:
+            time.sleep(w8)
+            if self.stopping:
+                break
+            self._sticky_qr()
+
     def cb_httpsrv_up(self) -> None:
         self.httpsrv_up += 1
         if self.httpsrv_up != self.broker.num_workers:
@@ -618,7 +894,17 @@ class SvcHub(object):
                 break
 
         if self.tcpsrv.qr:
+            if self.args.qr_pin:
+                self.sticky_qr()
+            if self.args.qr_wait or self.args.qr_every or self.args.qr_winch:
+                Daemon(self._qr_thr, "qr")
+            else:
+                if not self.args.qr_pin:
                     self.log("qr-code", self.tcpsrv.qr)
+                if self.args.qr_stdout:
+                    self.pr(self.tcpsrv.qr)
+                if self.args.qr_stderr:
+                    self.pr(self.tcpsrv.qr, file=sys.stderr)
         else:
             self.log("root", "workers OK\n")
 
@@ -645,6 +931,7 @@ class SvcHub(object):
             (HAVE_ZMQ, "pyzmq", "send zeromq messages from event-hooks"),
             (HAVE_HEIF, "pillow-heif", "read .heif images with pillow (rarely useful)"),
             (HAVE_AVIF, "pillow-avif", "read .avif images with pillow (rarely useful)"),
+            (HAVE_RAW, "rawpy", "read RAW images"),
         ]
         if ANYWIN:
             to_check += [
@@ -679,19 +966,11 @@ class SvcHub(object):
                 t += ", "
             t += "\033[0mNG: \033[35m" + sng
 
-        t += "\033[0m, see --deps"
-        self.log("dependencies", t, 6)
+        t += "\033[0m, see --deps (this is fine btw)"
+        self.log("optional-dependencies", t, 6)
 
     def _check_env(self) -> None:
-        try:
-            files = os.listdir(E.cfg)
-        except:
-            files = []
-
-        hits = [x for x in files if x.lower().endswith(".conf")]
-        if hits:
-            t = "WARNING: found config files in [%s]: %s\n  config files are not expected here, and will NOT be loaded (unless your setup is intentionally hella funky)"
-            self.log("root", t % (E.cfg, ", ".join(hits)), 3)
+        al = self.args
 
         if self.args.no_bauth:
             t = "WARNING: --no-bauth disables support for the Android app; you may want to use --bauth-last instead"
@@ -699,6 +978,21 @@ class SvcHub(object):
             if self.args.bauth_last:
                 self.log("root", "WARNING: ignoring --bauth-last due to --no-bauth", 3)
 
+        have_tcp = False
+        for zs in al.i:
+            if not zs.startswith(("unix:", "fd:")):
+                have_tcp = True
+        if not have_tcp:
+            zb = False
+            zs = "z zm zm4 zm6 zmv zmvv zs zsv zv"
+            for zs in zs.split():
+                if getattr(al, zs, False):
+                    setattr(al, zs, False)
+                    zb = True
+            if zb:
+                t = "not listening on any ip-addresses (only unix-sockets and/or FDs); cannot enable zeroconf/mdns/ssdp as requested"
+                self.log("root", t, 3)
+
         if not self.args.no_dav:
             from .dxml import DXML_OK
 
@@ -708,6 +1002,24 @@ class SvcHub(object):
                     t = "WARNING:\nDisabling WebDAV support because dxml selftest failed. Please report this bug;\n%s\n...and include the following information in the bug-report:\n%s | expat %s\n"
                     self.log("root", t % (URL_BUG, VERSIONS, expat_ver()), 1)
 
+        if not E.scfg and not al.unsafe_state and not os.getenv("PRTY_UNSAFE_STATE"):
+            t = "because runtime config is currently being stored in an untrusted emergency-fallback location. Please fix your environment so either XDG_CONFIG_HOME or ~/.config can be used instead, or disable this safeguard with --unsafe-state or env-var PRTY_UNSAFE_STATE=1."
+            if not al.no_ses:
+                al.no_ses = True
+                t2 = "A consequence of this misconfiguration is that passwords will now be sent in the HTTP-header of every request!"
+                self.log("root", "WARNING:\nWill disable sessions %s %s" % (t, t2), 1)
+            if al.idp_store == 1:
+                al.idp_store = 0
+                self.log("root", "WARNING:\nDisabling --idp-store %s" % (t,), 3)
+            if al.idp_store:
+                t2 = "ERROR: Cannot enable --idp-store %s" % (t,)
+                self.log("root", t2, 1)
+                raise Exception(t2)
+            if al.shr:
+                t2 = "ERROR: Cannot enable shares %s" % (t,)
+                self.log("root", t2, 1)
+                raise Exception(t2)
+
     def _process_config(self) -> bool:
         al = self.args
 
@@ -763,13 +1075,20 @@ class SvcHub(object):
             vl = [os.path.expandvars(os.path.expanduser(x)) for x in vl]
             setattr(al, k, vl)
 
-        for k in "lo hist ssl_log".split(" "):
+        for k in "lo hist dbpath ssl_log".split(" "):
             vs = getattr(al, k)
             if vs:
                 vs = os.path.expandvars(os.path.expanduser(vs))
                 setattr(al, k, vs)
 
-        for k in "sus_urls nonsus_urls".split(" "):
+        for k in "idp_adm stats_u".split(" "):
+            vs = getattr(al, k)
+            vsa = [x.strip() for x in vs.split(",")]
+            vsa = [x.lower() for x in vsa if x]
+            setattr(al, k + "_set", set(vsa))
+
+        zs = "dav_ua1 sus_urls nonsus_urls ua_nodoc ua_nozip"
+        for k in zs.split(" "):
             vs = getattr(al, k)
             if not vs or vs == "no":
                 setattr(al, k, None)
@@ -789,10 +1108,23 @@ class SvcHub(object):
             al.sus_urls = None
 
         al.xff_hdr = al.xff_hdr.lower()
-        al.idp_h_usr = al.idp_h_usr.lower()
+        al.idp_h_usr = [x.lower() for x in al.idp_h_usr or []]
         al.idp_h_grp = al.idp_h_grp.lower()
         al.idp_h_key = al.idp_h_key.lower()
 
+        al.idp_hm_usr_p = {}
+        for zs0 in al.idp_hm_usr or []:
+            try:
+                sep = zs0[:1]
+                hn, zs1, zs2 = zs0[1:].split(sep)
+                hn = hn.lower()
+                if hn in al.idp_hm_usr_p:
+                    al.idp_hm_usr_p[hn][zs1] = zs2
+                else:
+                    al.idp_hm_usr_p[hn] = {zs1: zs2}
+            except:
+                raise Exception("invalid --idp-hm-usr [%s]" % (zs0,))
+
         al.ftp_ipa_nm = build_netmap(al.ftp_ipa or al.ipa, True)
         al.tftp_ipa_nm = build_netmap(al.tftp_ipa or al.ipa, True)
 
@@ -837,12 +1169,21 @@ class SvcHub(object):
         except:
             raise Exception("invalid --mv-retry [%s]" % (self.args.mv_retry,))
 
+        al.js_utc = "false" if al.localtime else "true"
+
         al.tcolor = al.tcolor.lstrip("#")
         if len(al.tcolor) == 3:  # fc5 => ffcc55
             al.tcolor = "".join([x * 2 for x in al.tcolor])
 
+        if self.args.name_url:
+            zs = html_escape(self.args.name_url, True, True)
+            zs = '<a href="%s">%s</a>' % (zs, self.args.name)
+        else:
+            zs = self.args.name
+        self.args.name_html = zs
+
         zs = al.u2sz
-        zsl = zs.split(",")
+        zsl = [x.strip() for x in zs.split(",")]
         if len(zsl) not in (1, 3):
             t = "invalid --u2sz; must be either one number, or a comma-separated list of three numbers (min,default,max)"
             raise Exception(t)
@@ -859,6 +1200,7 @@ class SvcHub(object):
             zi2 = zi
         al.u2sz = ",".join(zsl)
 
+        derive_args(al)
         return True
 
     def _ipa2re(self, txt) -> Optional[re.Pattern]:
@@ -931,7 +1273,7 @@ class SvcHub(object):
 
         fn = sel_fn
         try:
-            os.makedirs(os.path.dirname(fn))
+            bos.makedirs(os.path.dirname(fn))
         except:
             pass
 
@@ -948,6 +1290,9 @@ class SvcHub(object):
 
             lh = codecs.open(fn, "w", encoding="utf-8", errors="replace")
 
+        if getattr(self.args, "free_umask", False):
+            os.fchmod(lh.fileno(), 0o644)
+
         argv = [pybin] + self.argv
         if hasattr(shlex, "quote"):
             argv = [shlex.quote(x) for x in argv]
@@ -1031,6 +1376,7 @@ class SvcHub(object):
         with self.reload_mutex:
             self.log("root", "reloading config")
             self.asrv.reload(9 if up2k else 4)
+            ramdisk_chk(self.asrv)
             if up2k:
                 self.up2k.reload(rescan_all_vols)
                 t += "; volumes are now reinitializing"
@@ -1215,11 +1561,18 @@ class SvcHub(object):
 
             fmt = "\033[36m%s \033[33m%-21s \033[0m%s\n"
             if self.no_ansi:
-                fmt = "%s %-21s %s\n"
+                if c == 1:
+                    fmt = "%s %-21s CRIT: %s\n"
+                elif c == 3:
+                    fmt = "%s %-21s WARN: %s\n"
+                elif c == 6:
+                    fmt = "%s %-21s  BTW: %s\n"
+                else:
+                    fmt = "%s %-21s  LOG: %s\n"
                 if "\033" in msg:
-                    msg = ansi_re.sub("", msg)
+                    msg = RE_ANSI.sub("", msg)
                 if "\033" in src:
-                    src = ansi_re.sub("", src)
+                    src = RE_ANSI.sub("", src)
             elif c:
                 if isinstance(c, int):
                     msg = "\033[3%sm%s\033[0m" % (c, msg)
@@ -1260,7 +1613,7 @@ class SvcHub(object):
                 raise
 
     def check_mp_support(self) -> str:
-        if MACOS:
+        if MACOS and not os.environ.get("PRTY_FORCE_MP"):
             return "multiprocessing is wonky on mac osx;"
         elif sys.version_info < (3, 3):
             return "need python 3.3 or newer for multiprocessing;"
@@ -1280,7 +1633,7 @@ class SvcHub(object):
             return False
 
         try:
-            if mp.cpu_count() <= 1:
+            if mp.cpu_count() <= 1 and not os.environ.get("PRTY_FORCE_MP"):
                 raise Exception()
         except:
             self.log("svchub", "only one CPU detected; multiprocessing disabled")

copyparty/szip.py

@@ -4,12 +4,11 @@ from __future__ import print_function, unicode_literals
 import calendar
 import stat
 import time
-import zlib
 
 from .authsrv import AuthSrv
 from .bos import bos
 from .sutil import StreamArc, errdesc
-from .util import min_ex, sanitize_fn, spack, sunpack, yieldfile
+from .util import min_ex, sanitize_fn, spack, sunpack, yieldfile, zlib
 
 if True:  # pylint: disable=using-constant-test
     from typing import Any, Generator, Optional
@@ -55,6 +54,7 @@ def gen_fdesc(sz: int, crc32: int, z64: bool) -> bytes:
 
 def gen_hdr(
     h_pos: Optional[int],
+    z64: bool,
     fn: str,
     sz: int,
     lastmod: int,
@@ -71,7 +71,6 @@ def gen_hdr(
     # appnote 4.5 / zip 3.0 (2008) / unzip 6.0 (2009) says to add z64
     # extinfo for values which exceed H, but that becomes an off-by-one
     # (can't tell if it was clamped or exactly maxval), make it obvious
-    z64 = sz >= 0xFFFFFFFF
     z64v = [sz, sz] if z64 else []
     if h_pos and h_pos >= 0xFFFFFFFF:
         # central, also consider ptr to original header
@@ -245,6 +244,7 @@ class StreamZip(StreamArc):
 
         sz = st.st_size
         ts = st.st_mtime
+        h_pos = self.pos
 
         crc = 0
         if self.pre_crc:
@@ -253,8 +253,12 @@ class StreamZip(StreamArc):
 
             crc &= 0xFFFFFFFF
 
-        h_pos = self.pos
-        buf = gen_hdr(None, name, sz, ts, self.utf8, crc, self.pre_crc)
+        # some unzip-programs expect a 64bit data-descriptor
+        # even if the only 32bit-exceeding value is the offset,
+        # so force that by placeholdering the filesize too
+        z64 = h_pos >= 0xFFFFFFFF or sz >= 0xFFFFFFFF
+
+        buf = gen_hdr(None, z64, name, sz, ts, self.utf8, crc, self.pre_crc)
         yield self._ct(buf)
 
         for buf in yieldfile(src, self.args.iobuf):
@@ -267,8 +271,6 @@ class StreamZip(StreamArc):
 
         self.items.append((name, sz, ts, crc, h_pos))
 
-        z64 = sz >= 4 * 1024 * 1024 * 1024
-
         if z64 or not self.pre_crc:
             buf = gen_fdesc(sz, crc, z64)
             yield self._ct(buf)
@@ -307,7 +309,8 @@ class StreamZip(StreamArc):
 
             cdir_pos = self.pos
             for name, sz, ts, crc, h_pos in self.items:
-                buf = gen_hdr(h_pos, name, sz, ts, self.utf8, crc, self.pre_crc)
+                z64 = h_pos >= 0xFFFFFFFF or sz >= 0xFFFFFFFF
+                buf = gen_hdr(h_pos, z64, name, sz, ts, self.utf8, crc, self.pre_crc)
                 mbuf += self._ct(buf)
                 if len(mbuf) >= 16384:
                     yield mbuf

copyparty/tcpsrv.py

@@ -9,24 +9,26 @@ import time
 
 from .__init__ import ANYWIN, PY2, TYPE_CHECKING, unicode
 from .cert import gencert
-from .stolen.qrcodegen import QrCode
+from .qrkode import QrCode, qr2png, qr2svg, qr2txt, qrgen
 from .util import (
     E_ACCESS,
     E_ADDR_IN_USE,
     E_ADDR_NOT_AVAIL,
     E_UNREACH,
     HAVE_IPV6,
+    IP6_LL,
     IP6ALL,
     VF_CAREFUL,
     Netdev,
     atomic_move,
+    get_adapters,
     min_ex,
     sunpack,
     termsize,
 )
 
-if True:
-    from typing import Generator, Union
+if True:  # pylint: disable=using-constant-test
+    from typing import Generator, Optional, Union
 
 if TYPE_CHECKING:
     from .svchub import SvcHub
@@ -58,6 +60,7 @@ class TcpSrv(object):
         self.stopping = False
         self.srv: list[socket.socket] = []
         self.bound: list[tuple[str, int]] = []
+        self.seen_eps: list[tuple[str, int]] = []  # also skipped by uds-only
         self.netdevs: dict[str, Netdev] = {}
         self.netlist = ""
         self.nsrv = 0
@@ -140,25 +143,31 @@ class TcpSrv(object):
         # keep IPv6 LL-only nics
         ll_ok: set[str] = set()
         for ip, nd in self.netdevs.items():
-            if not ip.startswith("fe80"):
+            if not ip.startswith(IP6_LL):
                 continue
 
             just_ll = True
             for ip2, nd2 in self.netdevs.items():
-                if nd == nd2 and ":" in ip2 and not ip2.startswith("fe80"):
+                if nd == nd2 and ":" in ip2 and not ip2.startswith(IP6_LL):
                     just_ll = False
 
             if just_ll or self.args.ll:
                 ll_ok.add(ip.split("/")[0])
 
+        listening_on = []
+        for ip, ports in sorted(ok.items()):
+            for port in sorted(ports):
+                listening_on.append("%s %s" % (ip, port))
+
         qr1: dict[str, list[int]] = {}
         qr2: dict[str, list[int]] = {}
         msgs = []
+        accessible_on = []
         title_tab: dict[str, dict[str, int]] = {}
         title_vars = [x[1:] for x in self.args.wintitle.split(" ") if x.startswith("$")]
         t = "available @ {}://{}:{}/  (\033[33m{}\033[0m)"
         for ip, desc in sorted(eps.items(), key=lambda x: x[1]):
-            if ip.startswith("fe80") and ip not in ll_ok:
+            if ip.startswith(IP6_LL) and ip not in ll_ok:
                 continue
 
             for port in sorted(self.args.p):
@@ -169,6 +178,10 @@ class TcpSrv(object):
                 ):
                     continue
 
+                zs = "%s %s" % (ip, port)
+                if zs not in accessible_on:
+                    accessible_on.append(zs)
+
                 proto = " http"
                 if self.args.http_only:
                     pass
@@ -219,6 +232,14 @@ class TcpSrv(object):
         else:
             print("\n", end="")
 
+        for fn, ls in (
+            (self.args.wr_h_eps, listening_on),
+            (self.args.wr_h_aon, accessible_on),
+        ):
+            if fn:
+                with open(fn, "wb") as f:
+                    f.write(("\n".join(ls)).encode("utf-8"))
+
         if self.args.qr or self.args.qrs:
             self.qr = self._qr(qr1, qr2)
 
@@ -227,8 +248,10 @@ class TcpSrv(object):
 
     def _listen(self, ip: str, port: int) -> None:
         uds_perm = uds_gid = -1
-        if "unix:" in ip:
+        bound: Optional[socket.socket] = None
         tcp = False
+
+        if "unix:" in ip:
             ipv = socket.AF_UNIX
             uds = ip.split(":")
             ip = uds[-1]
@@ -241,7 +264,12 @@ class TcpSrv(object):
                     import grp
 
                     uds_gid = grp.getgrnam(uds[2]).gr_gid
+        elif "fd:" in ip:
+            fd = ip[3:]
+            bound = socket.socket(fileno=int(fd))
 
+            tcp = bound.proto == socket.IPPROTO_TCP
+            ipv = bound.family
         elif ":" in ip:
             tcp = True
             ipv = socket.AF_INET6
@@ -249,7 +277,7 @@ class TcpSrv(object):
             tcp = True
             ipv = socket.AF_INET
 
-        srv = socket.socket(ipv, socket.SOCK_STREAM)
+        srv = bound or socket.socket(ipv, socket.SOCK_STREAM)
 
         if not ANYWIN or self.args.reuseaddr:
             srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
@@ -264,17 +292,28 @@ class TcpSrv(object):
         except:
             pass  # will create another ipv4 socket instead
 
-        if not ANYWIN and self.args.freebind:
+        if getattr(self.args, "freebind", False):
             srv.setsockopt(socket.SOL_IP, socket.IP_FREEBIND, 1)
 
+        if bound:
+            self.srv.append(srv)
+            return
+
         try:
             if tcp:
+                if self.args.http_no_tcp:
+                    self.seen_eps.append((ip, port))
+                    return
                 srv.bind((ip, port))
             else:
                 if ANYWIN or self.args.rm_sck:
                     if os.path.exists(ip):
                         os.unlink(ip)
                     srv.bind(ip)
+                    if uds_gid != -1:
+                        os.chown(ip, -1, uds_gid)
+                    if uds_perm != -1:
+                        os.chmod(ip, uds_perm)
                 else:
                     tf = "%s.%d" % (ip, os.getpid())
                     if os.path.exists(tf):
@@ -375,6 +414,7 @@ class TcpSrv(object):
 
         self.srv = srvs
         self.bound = bound
+        self.seen_eps = list(set(self.seen_eps + bound))
         self.nsrv = len(srvs)
         self._distribute_netdevs()
 
@@ -417,9 +457,7 @@ class TcpSrv(object):
             self._distribute_netdevs()
 
     def detect_interfaces(self, listen_ips: list[str]) -> dict[str, Netdev]:
-        from .stolen.ifaddr import get_adapters
-
-        listen_ips = [x for x in listen_ips if "unix:" not in x]
+        listen_ips = [x for x in listen_ips if not x.startswith(("unix:", "fd:"))]
 
         nics = get_adapters(True)
         eps: dict[str, Netdev] = {}
@@ -548,7 +586,7 @@ class TcpSrv(object):
         ip = None
         ips = list(t1) + list(t2)
         qri = self.args.qri
-        if self.args.zm and not qri:
+        if self.args.zm and not qri and ips:
             name = self.args.name + ".local"
             t1[name] = next(v for v in (t1 or t2).values())
             ips = [name] + ips
@@ -565,8 +603,7 @@ class TcpSrv(object):
         if not ip:
             return ""
 
-        if ":" in ip:
-            ip = "[{}]".format(ip)
+        hip = "[%s]" % (ip,) if ":" in ip else ip
 
         if self.args.http_only:
             https = ""
@@ -578,7 +615,7 @@ class TcpSrv(object):
         ports = t1.get(ip, t2.get(ip, []))
         dport = 443 if https else 80
         port = "" if dport in ports or not ports else ":{}".format(ports[0])
-        txt = "http{}://{}{}/{}".format(https, ip, port, self.args.qrl)
+        txt = "http{}://{}{}/{}".format(https, hip, port, self.args.qrl)
 
         btxt = txt.encode("utf-8")
         if PY2:
@@ -586,9 +623,17 @@ class TcpSrv(object):
 
         fg = self.args.qr_fg
         bg = self.args.qr_bg
+        nocolor = fg == -1
+        if nocolor:
+            fg = 0
+
         pad = self.args.qrp
         zoom = self.args.qrz
-        qrc = QrCode.encode_binary(btxt)
+        qrc = qrgen(btxt)
+
+        for zs in self.args.qr_file or []:
+            self._qr2file(qrc, zs)
+
         if zoom == 0:
             try:
                 tw, th = termsize()
@@ -597,13 +642,15 @@ class TcpSrv(object):
             except:
                 zoom = 1
 
-        qr = qrc.render(zoom, pad)
+        qr = qr2txt(qrc, zoom, pad)
         if self.args.no_ansi:
             return "{}\n{}".format(txt, qr)
 
         halfc = "\033[40;48;5;{0}m{1}\033[47;48;5;{2}m"
         if not fg:
             halfc = "\033[0;40m{1}\033[0;47m"
+        if nocolor:
+            halfc = "\033[0;7m{1}\033[0m"
 
         def ansify(m: re.Match) -> str:
             return halfc.format(fg, " " * len(m.group(1)), bg)
@@ -613,6 +660,8 @@ class TcpSrv(object):
 
         qr = qr.replace("\n", "\033[K\n") + "\033[K"  # win10do
         cc = " \033[0;38;5;{0};47;48;5;{1}m" if fg else " \033[0;30;47m"
+        if nocolor:
+            cc = " \033[0m"
         t = cc + "\n{2}\033[999G\033[0m\033[J"
         t = t.format(fg, bg, qr)
         if ANYWIN:
@@ -620,3 +669,29 @@ class TcpSrv(object):
             t = t.replace("\n", "`\n`")
 
         return txt + t
+
+    def _qr2file(self, qrc: QrCode, txt: str):
+        if ".txt:" in txt or ".svg:" in txt:
+            ap, zs1, zs2 = txt.rsplit(":", 2)
+            bg = fg = ""
+        else:
+            ap, zs1, zs2, bg, fg = txt.rsplit(":", 4)
+        zoom = int(zs1)
+        pad = int(zs2)
+
+        if ap.endswith(".txt"):
+            if zoom not in (1, 2):
+                raise Exception("invalid zoom for qr.txt; must be 1 or 2")
+            with open(ap, "wb") as f:
+                f.write(qr2txt(qrc, zoom, pad).encode("utf-8"))
+        elif ap.endswith(".svg"):
+            with open(ap, "wb") as f:
+                f.write(qr2svg(qrc, pad).encode("utf-8"))
+        else:
+            qr2png(qrc, zoom, pad, self._h2i(bg), self._h2i(fg), ap)
+
+    def _h2i(self, hs):
+        try:
+            return tuple(int(hs[i : i + 2], 16) for i in (0, 2, 4))
+        except:
+            return None

copyparty/tftpd.py

@@ -36,7 +36,20 @@ from partftpy.TftpShared import TftpException
 from .__init__ import EXE, PY2, TYPE_CHECKING
 from .authsrv import VFS
 from .bos import bos
-from .util import UTC, BytesIO, Daemon, ODict, exclude_dotfiles, min_ex, runhook, undot
+from .util import (
+    FN_EMB,
+    UTC,
+    BytesIO,
+    Daemon,
+    ODict,
+    exclude_dotfiles,
+    min_ex,
+    runhook,
+    set_fperms,
+    undot,
+    vjoin,
+    vsplit,
+)
 
 if True:  # pylint: disable=using-constant-test
     from typing import Any, Union
@@ -166,7 +179,7 @@ class Tftpd(object):
         if "::" in ips:
             ips.append("0.0.0.0")
 
-        ips = [x for x in ips if "unix:" not in x]
+        ips = [x for x in ips if not x.startswith(("unix:", "fd:"))]
 
         if self.args.tftp4:
             ips = [x for x in ips if ":" not in x]
@@ -244,16 +257,25 @@ class Tftpd(object):
         for srv in srvs:
             srv.stop()
 
-    def _v2a(self, caller: str, vpath: str, perms: list, *a: Any) -> tuple[VFS, str]:
+    def _v2a(
+        self, caller: str, vpath: str, perms: list, *a: Any
+    ) -> tuple[VFS, str, str]:
         vpath = vpath.replace("\\", "/").lstrip("/")
         if not perms:
             perms = [True, True]
 
         debug('%s("%s", %s) %s\033[K\033[0m', caller, vpath, str(a), perms)
         vfs, rem = self.asrv.vfs.get(vpath, "*", *perms)
+        if perms[1] and "*" not in vfs.axs.uread and "wo_up_readme" not in vfs.flags:
+            zs, fn = vsplit(vpath)
+            if fn.lower() in FN_EMB:
+                vpath = vjoin(zs, "_wo_" + fn)
+                vfs, rem = self.asrv.vfs.get(vpath, "*", *perms)
+
         if not vfs.realpath:
             raise Exception("unmapped vfs")
-        return vfs, vfs.canonical(rem)
+
+        return vfs, vpath, vfs.canonical(rem)
 
     def _ls(self, vpath: str, raddress: str, rport: int, force=False) -> Any:
         # generate file listing if vpath is dir.txt and return as file object
@@ -263,6 +285,7 @@ class Tftpd(object):
             if not ptn or not ptn.match(fn.lower()):
                 return None
 
+        tsdt = datetime.fromtimestamp
         vn, rem = self.asrv.vfs.get(vpath, "*", True, False)
         fsroot, vfs_ls, vfs_virt = vn.ls(
             rem,
@@ -275,7 +298,7 @@ class Tftpd(object):
         dirs1 = [(v.st_mtime, v.st_size, k + "/") for k, v in vfs_ls if k in dnames]
         fils1 = [(v.st_mtime, v.st_size, k) for k, v in vfs_ls if k not in dnames]
         real1 = dirs1 + fils1
-        realt = [(datetime.fromtimestamp(mt, UTC), sz, fn) for mt, sz, fn in real1]
+        realt = [(tsdt(max(0, mt), UTC), sz, fn) for mt, sz, fn in real1]
         reals = [
             (
                 "%04d-%02d-%02d %02d:%02d:%02d"
@@ -331,7 +354,7 @@ class Tftpd(object):
         else:
             raise Exception("bad mode %s" % (mode,))
 
-        vfs, ap = self._v2a("open", vpath, [rd, wr])
+        vfs, vpath, ap = self._v2a("open", vpath, [rd, wr])
         if wr:
             if "*" not in vfs.axs.uwrite:
                 yeet("blocked write; folder not world-writable: /%s" % (vpath,))
@@ -340,7 +363,8 @@ class Tftpd(object):
                 yeet("blocked write; folder not world-deletable: /%s" % (vpath,))
 
             xbu = vfs.flags.get("xbu")
-            if xbu and not runhook(
+            if xbu:
+                hr = runhook(
                     self.nlog,
                     None,
                     self.hub.up2k,
@@ -355,9 +379,13 @@ class Tftpd(object):
                     0,
                     "8.3.8.7",
                     time.time(),
-                "",
-            ):
-                yeet("blocked by xbu server config: %r" % (vpath,))
+                    None,
+                )
+                t = hr.get("rejectmsg") or ""
+                if t or hr.get("rc") != 0:
+                    if not t:
+                        t = "upload blocked by xbu server config: %r" % (vpath,)
+                    yeet(t)
 
         if not self.args.tftp_nols and bos.path.isdir(ap):
             return self._ls(vpath, "", 0, True)
@@ -365,18 +393,24 @@ class Tftpd(object):
         if not a:
             a = (self.args.iobuf,)
 
-        return open(ap, mode, *a, **ka)
+        ret = open(ap, mode, *a, **ka)
+        if wr and "fperms" in vfs.flags:
+            set_fperms(ret, vfs.flags)
+
+        return ret
 
     def _mkdir(self, vpath: str, *a) -> None:
-        vfs, ap = self._v2a("mkdir", vpath, [])
+        vfs, _, ap = self._v2a("mkdir", vpath, [False, True])
         if "*" not in vfs.axs.uwrite:
             yeet("blocked mkdir; folder not world-writable: /%s" % (vpath,))
 
-        return bos.mkdir(ap)
+        bos.mkdir(ap, vfs.flags["chmod_d"])
+        if "chown" in vfs.flags:
+            bos.chown(ap, vfs.flags["uid"], vfs.flags["gid"])
 
     def _unlink(self, vpath: str) -> None:
         # return bos.unlink(self._v2a("stat", vpath, *a)[1])
-        vfs, ap = self._v2a("delete", vpath, [True, False, False, True])
+        vfs, _, ap = self._v2a("delete", vpath, [True, False, False, True])
 
         try:
             inf = bos.stat(ap)
@@ -400,7 +434,7 @@ class Tftpd(object):
 
     def _p_exists(self, vpath: str) -> bool:
         try:
-            ap = self._v2a("p.exists", vpath, [False, False])[1]
+            ap = self._v2a("p.exists", vpath, [False, False])[2]
             bos.stat(ap)
             return True
         except:
@@ -408,7 +442,7 @@ class Tftpd(object):
 
     def _p_isdir(self, vpath: str) -> bool:
         try:
-            st = bos.stat(self._v2a("p.isdir", vpath, [False, False])[1])
+            st = bos.stat(self._v2a("p.isdir", vpath, [False, False])[2])
             ret = stat.S_ISDIR(st.st_mode)
             return ret
         except:

copyparty/th_cli.py

@@ -1,13 +1,15 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
+import errno
 import os
+import stat
 
 from .__init__ import TYPE_CHECKING
 from .authsrv import VFS
 from .bos import bos
 from .th_srv import EXTS_AC, HAVE_WEBP, thumb_path
-from .util import Cooldown
+from .util import Cooldown, Pebkac
 
 if True:  # pylint: disable=using-constant-test
     from typing import Optional, Union
@@ -16,6 +18,9 @@ if TYPE_CHECKING:
     from .httpsrv import HttpSrv
 
 
+IOERROR = "reading the file was denied by the server os; either due to filesystem permissions, selinux, apparmor, or similar:\n%r"
+
+
 class ThumbCli(object):
     def __init__(self, hsrv: "HttpSrv") -> None:
         self.broker = hsrv.broker
@@ -31,11 +36,15 @@ class ThumbCli(object):
             if not c:
                 raise Exception()
         except:
-            c = {k: set() for k in ["thumbable", "pil", "vips", "ffi", "ffv", "ffa"]}
+            c = {
+                k: set()
+                for k in ["thumbable", "pil", "vips", "raw", "ffi", "ffv", "ffa"]
+            }
 
         self.thumbable = c["thumbable"]
         self.fmt_pil = c["pil"]
         self.fmt_vips = c["vips"]
+        self.fmt_raw = c["raw"]
         self.fmt_ffi = c["ffi"]
         self.fmt_ffv = c["ffv"]
         self.fmt_ffa = c["ffa"]
@@ -83,7 +92,7 @@ class ThumbCli(object):
         if rem.startswith(".hist/th/") and rem.split(".")[-1] in ["webp", "jpg", "png"]:
             return os.path.join(ptop, rem)
 
-        if fmt[:1] in "jw":
+        if fmt[:1] in "jw" and fmt != "wav":
             sfmt = fmt[:1]
 
             if sfmt == "j" and self.args.th_no_jpg:
@@ -124,7 +133,7 @@ class ThumbCli(object):
 
         tpath = thumb_path(histpath, rem, mtime, fmt, self.fmt_ffa)
         tpaths = [tpath]
-        if fmt == "w":
+        if fmt[:1] == "w" and fmt != "wav":
             # also check for jpg (maybe webp is unavailable)
             tpaths.append(tpath.rsplit(".", 1)[0] + ".jpg")
 
@@ -157,8 +166,22 @@ class ThumbCli(object):
         if abort:
             return None
 
-        if not bos.path.getsize(os.path.join(ptop, rem)):
+        ap = os.path.join(ptop, rem)
+        try:
+            st = bos.stat(ap)
+            if not st.st_size or not stat.S_ISREG(st.st_mode):
                 return None
 
+            with open(ap, "rb", 4) as f:
+                if not f.read(4):
+                    raise Exception()
+        except OSError as ex:
+            if ex.errno == errno.ENOENT:
+                raise Pebkac(404)
+            else:
+                raise Pebkac(500, IOERROR % (ex,))
+        except Exception as ex:
+            raise Pebkac(500, IOERROR % (ex,))
+
         x = self.broker.ask("thumbsrv.get", ptop, rem, mtime, fmt)
         return x.get()  # type: ignore

copyparty/th_srv.py

@@ -2,10 +2,13 @@
 from __future__ import print_function, unicode_literals
 
 import hashlib
+import io
 import logging
 import os
+import re
 import shutil
 import subprocess as sp
+import tempfile
 import threading
 import time
 
@@ -18,16 +21,17 @@ from .mtag import HAVE_FFMPEG, HAVE_FFPROBE, au_unpk, ffprobe
 from .util import BytesIO  # type: ignore
 from .util import (
     FFMPEG_URL,
+    VF_CAREFUL,
     Cooldown,
     Daemon,
     afsenc,
+    atomic_move,
     fsenc,
     min_ex,
     runcmd,
     statdir,
     ub64enc,
     vsplit,
-    wrename,
     wunlink,
 )
 
@@ -47,7 +51,11 @@ HAVE_AVIF = False
 HAVE_WEBP = False
 
 EXTS_TH = set(["jpg", "webp", "png"])
-EXTS_AC = set(["opus", "owa", "caf", "mp3"])
+EXTS_AC = set(["opus", "owa", "caf", "mp3", "flac", "wav"])
+EXTS_SPEC_SAFE = set("aif aiff flac mp3 opus wav".split())
+
+PTN_TS = re.compile("^-?[0-9a-f]{8,10}$")
+
 
 try:
     if os.environ.get("PRTY_NO_PIL"):
@@ -78,6 +86,9 @@ try:
         if os.environ.get("PRTY_NO_PIL_HEIF"):
             raise Exception()
 
+        try:
+            from pillow_heif import register_heif_opener
+        except ImportError:
             from pyheif_pillow_opener import register_heif_opener
 
         register_heif_opener()
@@ -89,6 +100,10 @@ try:
         if os.environ.get("PRTY_NO_PIL_AVIF"):
             raise Exception()
 
+        if ".avif" in Image.registered_extensions():
+            HAVE_AVIF = True
+            raise Exception()
+
         import pillow_avif  # noqa: F401  # pylint: disable=unused-import
 
         HAVE_AVIF = True
@@ -101,14 +116,28 @@ except:
 
 try:
     if os.environ.get("PRTY_NO_VIPS"):
-        raise Exception()
+        raise ImportError()
 
     HAVE_VIPS = True
     import pyvips
 
     logging.getLogger("pyvips").setLevel(logging.WARNING)
-except:
+except Exception as e:
     HAVE_VIPS = False
+    if not isinstance(e, ImportError):
+        logging.warning("libvips found, but failed to load: " + str(e))
+
+
+try:
+    if os.environ.get("PRTY_NO_RAW"):
+        raise Exception()
+
+    HAVE_RAW = True
+    import rawpy
+
+    logging.getLogger("rawpy").setLevel(logging.WARNING)
+except:
+    HAVE_RAW = False
 
 
 th_dir_cache = {}
@@ -163,12 +192,15 @@ class ThumbSrv(object):
 
         self.mutex = threading.Lock()
         self.busy: dict[str, list[threading.Condition]] = {}
+        self.untemp: dict[str, list[str]] = {}
         self.ram: dict[str, float] = {}
         self.memcond = threading.Condition(self.mutex)
         self.stopping = False
         self.rm_nullthumbs = True  # forget failed conversions on startup
         self.nthr = max(1, self.args.th_mt)
 
+        self.exts_spec_unsafe = set(self.args.th_spec_cnv.split(","))
+
         self.q: Queue[Optional[tuple[str, str, str, VFS]]] = Queue(self.nthr * 4)
         for n in range(self.nthr):
             Daemon(self.worker, "thumb-{}-{}".format(n, self.nthr))
@@ -191,11 +223,19 @@ class ThumbSrv(object):
         if self.args.th_clean:
             Daemon(self.cleaner, "thumb.cln")
 
-        self.fmt_pil, self.fmt_vips, self.fmt_ffi, self.fmt_ffv, self.fmt_ffa = [
+        (
+            self.fmt_pil,
+            self.fmt_vips,
+            self.fmt_raw,
+            self.fmt_ffi,
+            self.fmt_ffv,
+            self.fmt_ffa,
+        ) = [
             set(y.split(","))
             for y in [
                 self.args.th_r_pil,
                 self.args.th_r_vips,
+                self.args.th_r_raw,
                 self.args.th_r_ffi,
                 self.args.th_r_ffv,
                 self.args.th_r_ffa,
@@ -218,6 +258,9 @@ class ThumbSrv(object):
         if "vips" in self.args.th_dec:
             self.thumbable |= self.fmt_vips
 
+        if "raw" in self.args.th_dec:
+            self.thumbable |= self.fmt_raw
+
         if "ff" in self.args.th_dec:
             for zss in [self.fmt_ffi, self.fmt_ffv, self.fmt_ffa]:
                 self.thumbable |= zss
@@ -227,6 +270,9 @@ class ThumbSrv(object):
 
     def shutdown(self) -> None:
         self.stopping = True
+        Daemon(self._fire_sentinels, "thumbstopper")
+
+    def _fire_sentinels(self):
         for _ in range(self.nthr):
             self.q.put(None)
 
@@ -255,7 +301,8 @@ class ThumbSrv(object):
                 self.log("joined waiting room for %r" % (tpath,))
             except:
                 thdir = os.path.dirname(tpath)
-                bos.makedirs(os.path.join(thdir, "w"))
+                chmod = bos.MKD_700 if self.args.free_umask else bos.MKD_755
+                bos.makedirs(os.path.join(thdir, "w"), vf=chmod)
 
                 inf_path = os.path.join(thdir, "dir.txt")
                 if not bos.path.exists(inf_path):
@@ -270,7 +317,7 @@ class ThumbSrv(object):
             vn = next((x for x in allvols if x.realpath == ptop), None)
             if not vn:
                 self.log("ptop %r not in %s" % (ptop, allvols), 3)
-                vn = self.asrv.vfs.all_aps[0][1]
+                vn = self.asrv.vfs.all_aps[0][1][0]
 
             self.q.put((abspath, tpath, fmt, vn))
             self.log("conv %r :%s \033[0m%r" % (tpath, fmt, abspath), 6)
@@ -298,6 +345,7 @@ class ThumbSrv(object):
             "thumbable": self.thumbable,
             "pil": self.fmt_pil,
             "vips": self.fmt_vips,
+            "raw": self.fmt_raw,
             "ffi": self.fmt_ffi,
             "ffv": self.fmt_ffv,
             "ffa": self.fmt_ffa,
@@ -336,12 +384,14 @@ class ThumbSrv(object):
             else:
                 ap_unpk = abspath
 
-            if not bos.path.exists(tpath):
+            if ap_unpk and not bos.path.exists(tpath):
                 tex = tpath.rsplit(".", 1)[-1]
                 want_mp3 = tex == "mp3"
                 want_opus = tex in ("opus", "owa", "caf")
+                want_flac = tex == "flac"
+                want_wav = tex == "wav"
                 want_png = tex == "png"
-                want_au = want_mp3 or want_opus
+                want_au = want_mp3 or want_opus or want_flac or want_wav
                 for lib in self.args.th_dec:
                     can_au = lib == "ff" and (
                         ext in self.fmt_ffa or ext in self.fmt_ffv
@@ -351,11 +401,17 @@ class ThumbSrv(object):
                         funs.append(self.conv_pil)
                     elif lib == "vips" and ext in self.fmt_vips:
                         funs.append(self.conv_vips)
+                    elif lib == "raw" and ext in self.fmt_raw:
+                        funs.append(self.conv_raw)
                     elif can_au and (want_png or want_au):
                         if want_opus:
                             funs.append(self.conv_opus)
                         elif want_mp3:
                             funs.append(self.conv_mp3)
+                        elif want_flac:
+                            funs.append(self.conv_flac)
+                        elif want_wav:
+                            funs.append(self.conv_wav)
                         elif want_png:
                             funs.append(self.conv_waves)
                             png_ok = True
@@ -371,12 +427,14 @@ class ThumbSrv(object):
             except:
                 pass
 
+            conv_ok = False
             for fun in funs:
                 try:
                     if not png_ok and tpath.endswith(".png"):
                         raise Exception("png only allowed for waveforms")
 
                     fun(ap_unpk, ttpath, fmt, vn)
+                    conv_ok = True
                     break
                 except Exception as ex:
                     msg = "%s could not create thumbnail of %r\n%s"
@@ -385,8 +443,12 @@ class ThumbSrv(object):
                     self.log(msg, c)
                     if getattr(ex, "returncode", 0) != 321:
                         if fun == funs[-1]:
+                            try:
                                 with open(ttpath, "wb") as _:
                                     pass
+                            except Exception as ex:
+                                t = "failed to create the file [%s]: %r"
+                                self.log(t % (ttpath, ex), 3)
                     else:
                         # ffmpeg may spawn empty files on windows
                         try:
@@ -394,18 +456,33 @@ class ThumbSrv(object):
                         except:
                             pass
 
-            if abspath != ap_unpk:
+            if abspath != ap_unpk and ap_unpk:
                 wunlink(self.log, ap_unpk, vn.flags)
 
             try:
-                wrename(self.log, ttpath, tpath, vn.flags)
+                atomic_move(self.log, ttpath, tpath, vn.flags)
+            except Exception as ex:
+                if conv_ok and not os.path.exists(tpath):
+                    t = "failed to move  [%s]  to  [%s]:  %r"
+                    self.log(t % (ttpath, tpath, ex), 3)
+                elif not conv_ok:
+                    try:
+                        open(tpath, "ab").close()
                     except:
                         pass
 
+            untemp = []
             with self.mutex:
                 subs = self.busy[tpath]
                 del self.busy[tpath]
                 self.ram.pop(ttpath, None)
+                untemp = self.untemp.pop(ttpath, None) or []
+
+            for ap in untemp:
+                try:
+                    wunlink(self.log, ap, VF_CAREFUL)
+                except:
+                    pass
 
             for x in subs:
                 with x:
@@ -444,9 +521,7 @@ class ThumbSrv(object):
 
         return im
 
-    def conv_pil(self, abspath: str, tpath: str, fmt: str, vn: VFS) -> None:
-        self.wait4ram(0.2, tpath)
-        with Image.open(fsenc(abspath)) as im:
+    def conv_image_pil(self, im: "Image.Image", tpath: str, fmt: str, vn: VFS) -> None:
         try:
             im = self.fancy_pillow(im, fmt, vn)
         except Exception as ex:
@@ -474,6 +549,11 @@ class ThumbSrv(object):
 
         im.save(tpath, **args)
 
+    def conv_pil(self, abspath: str, tpath: str, fmt: str, vn: VFS) -> None:
+        self.wait4ram(0.2, tpath)
+        with Image.open(fsenc(abspath)) as im:
+            self.conv_image_pil(im, tpath, fmt, vn)
+
     def conv_vips(self, abspath: str, tpath: str, fmt: str, vn: VFS) -> None:
         self.wait4ram(0.2, tpath)
         crops = ["centre", "none"]
@@ -495,9 +575,53 @@ class ThumbSrv(object):
         assert img  # type: ignore  # !rm
         img.write_to_file(tpath, Q=40)
 
+    def conv_raw(self, abspath: str, tpath: str, fmt: str, vn: VFS) -> None:
+        self.wait4ram(0.2, tpath)
+        with rawpy.imread(abspath) as raw:
+            thumb = raw.extract_thumb()
+        if thumb.format == rawpy.ThumbFormat.JPEG and tpath.endswith(".jpg"):
+            # if we have a jpg thumbnail and no webp output is available,
+            # just write the jpg directly (it'll be the wrong size, but it's fast)
+            with open(tpath, "wb") as f:
+                f.write(thumb.data)
+        if HAVE_VIPS:
+            crops = ["centre", "none"]
+            if "f" in fmt:
+                crops = ["none"]
+            w, h = self.getres(vn, fmt)
+            kw = {"height": h, "size": "down", "intent": "relative"}
+
+            for c in crops:
+                try:
+                    kw["crop"] = c
+                    if thumb.format == rawpy.ThumbFormat.BITMAP:
+                        img = pyvips.Image.new_from_array(
+                            thumb.data, interpretation="rgb"
+                        )
+                        img = img.thumbnail_image(w, **kw)
+                    else:
+                        img = pyvips.Image.thumbnail_buffer(thumb.data, w, **kw)
+                    break
+                except:
+                    if c == crops[-1]:
+                        raise
+
+            assert img  # type: ignore  # !rm
+            img.write_to_file(tpath, Q=40)
+        elif HAVE_PIL:
+            if thumb.format == rawpy.ThumbFormat.BITMAP:
+                im = Image.fromarray(thumb.data, "RGB")
+            else:
+                im = Image.open(io.BytesIO(thumb.data))
+            self.conv_image_pil(im, tpath, fmt, vn)
+        else:
+            raise Exception(
+                "either pil or vips is needed to process embedded bitmap thumbnails in raw files"
+            )
+
     def conv_ffmpeg(self, abspath: str, tpath: str, fmt: str, vn: VFS) -> None:
         self.wait4ram(0.2, tpath)
-        ret, _ = ffprobe(abspath, int(vn.flags["convt"] / 2))
+        ret, _, _, _ = ffprobe(abspath, int(vn.flags["convt"] / 2))
         if not ret:
             return
 
@@ -508,6 +632,17 @@ class ThumbSrv(object):
             dur = ret[".dur"][1] if ".dur" in ret else 4
             seek = [b"-ss", "{:.0f}".format(dur / 3).encode("utf-8")]
 
+        self._ffmpeg_im(abspath, tpath, fmt, vn, seek, b"0:v:0")
+
+    def _ffmpeg_im(
+        self,
+        abspath: str,
+        tpath: str,
+        fmt: str,
+        vn: VFS,
+        seek: list[bytes],
+        imap: bytes,
+    ) -> None:
         scale = "scale={0}:{1}:force_original_aspect_ratio="
         if "f" in fmt:
             scale += "decrease,setsar=1:1"
@@ -526,7 +661,7 @@ class ThumbSrv(object):
         cmd += seek
         cmd += [
             b"-i", fsenc(abspath),
-            b"-map", b"0:v:0",
+            b"-map", imap,
             b"-vf", bscale,
             b"-frames:v", b"1",
             b"-metadata:s:v:0", b"rotate=0",
@@ -547,16 +682,16 @@ class ThumbSrv(object):
             ]
 
         cmd += [fsenc(tpath)]
-        self._run_ff(cmd, vn)
+        self._run_ff(cmd, vn, "convt")
 
-    def _run_ff(self, cmd: list[bytes], vn: VFS, oom: int = 400) -> None:
+    def _run_ff(self, cmd: list[bytes], vn: VFS, kto: str, oom: int = 400) -> None:
         # self.log((b" ".join(cmd)).decode("utf-8"))
-        ret, _, serr = runcmd(cmd, timeout=vn.flags["convt"], nice=True, oom=oom)
+        ret, _, serr = runcmd(cmd, timeout=vn.flags[kto], nice=True, oom=oom)
         if not ret:
             return
 
         c: Union[str, int] = "90"
-        t = "FFmpeg failed (probably a corrupt video file):\n"
+        t = "FFmpeg failed (probably a corrupt file):\n"
         if (
             (not self.args.th_ff_jpg or time.time() - int(self.args.th_ff_jpg) < 60)
             and cmd[-1].lower().endswith(b".webp")
@@ -595,7 +730,7 @@ class ThumbSrv(object):
         raise sp.CalledProcessError(ret, (cmd[0], b"...", cmd[-1]))
 
     def conv_waves(self, abspath: str, tpath: str, fmt: str, vn: VFS) -> None:
-        ret, _ = ffprobe(abspath, int(vn.flags["convt"] / 2))
+        ret, _, _, _ = ffprobe(abspath, int(vn.flags["convt"] / 2))
         if "ac" not in ret:
             raise Exception("not audio")
 
@@ -633,7 +768,7 @@ class ThumbSrv(object):
         # fmt: on
 
         cmd += [fsenc(tpath)]
-        self._run_ff(cmd, vn)
+        self._run_ff(cmd, vn, "convt")
 
         if "pngquant" in vn.flags:
             wtpath = tpath + ".png"
@@ -652,22 +787,70 @@ class ThumbSrv(object):
                 except:
                     pass
             else:
-                wrename(self.log, wtpath, tpath, vn.flags)
+                atomic_move(self.log, wtpath, tpath, vn.flags)
+
+    def conv_emb_cv(
+        self, abspath: str, tpath: str, fmt: str, vn: VFS, strm: dict[str, Any]
+    ) -> None:
+        self.wait4ram(0.2, tpath)
+        self._ffmpeg_im(
+            abspath, tpath, fmt, vn, [], b"0:" + strm["index"].encode("ascii")
+        )
 
     def conv_spec(self, abspath: str, tpath: str, fmt: str, vn: VFS) -> None:
-        ret, _ = ffprobe(abspath, int(vn.flags["convt"] / 2))
+        ret, raw, strms, ctnr = ffprobe(abspath, int(vn.flags["convt"] / 2))
         if "ac" not in ret:
             raise Exception("not audio")
 
+        want_spec = vn.flags.get("th_spec_p", 1)
+        if want_spec < 2:
+            for strm in strms:
+                if (
+                    strm.get("codec_type") == "video"
+                    and strm.get("DISPOSITION:attached_pic") == "1"
+                ):
+                    return self.conv_emb_cv(abspath, tpath, fmt, vn, strm)
+
+        if not want_spec:
+            raise Exception("spectrograms forbidden by volflag")
+
+        fext = abspath.split(".")[-1].lower()
+
         # https://trac.ffmpeg.org/ticket/10797
         # expect 1 GiB every 600 seconds when duration is tricky;
         # simple filetypes are generally safer so let's special-case those
-        safe = ("flac", "wav", "aif", "aiff", "opus")
-        coeff = 1800 if abspath.split(".")[-1].lower() in safe else 600
-        dur = ret[".dur"][1] if ".dur" in ret else 300
+        coeff = 1800 if fext in EXTS_SPEC_SAFE else 600
+        dur = ret[".dur"][1] if ".dur" in ret else 900
         need = 0.2 + dur / coeff
         self.wait4ram(need, tpath)
 
+        infile = abspath
+        if dur >= 900 or fext in self.exts_spec_unsafe:
+            with tempfile.NamedTemporaryFile(suffix=".spec.flac", delete=False) as f:
+                f.write(b"h")
+                infile = f.name
+                try:
+                    self.untemp[tpath].append(infile)
+                except:
+                    self.untemp[tpath] = [infile]
+
+            # fmt: off
+            cmd = [
+                b"ffmpeg",
+                b"-nostdin",
+                b"-v", b"error",
+                b"-hide_banner",
+                b"-i", fsenc(abspath),
+                b"-map", b"0:a:0",
+                b"-ac", b"1",
+                b"-ar", b"48000",
+                b"-sample_fmt", b"s16",
+                b"-t", b"900",
+                b"-y", fsenc(infile),
+            ]
+            # fmt: on
+            self._run_ff(cmd, vn, "convt")
+
         fc = "[0:a:0]aresample=48000{},showspectrumpic=s="
         if "3" in fmt:
             fc += "1280x1024,crop=1420:1056:70:48[o]"
@@ -687,7 +870,7 @@ class ThumbSrv(object):
             b"-nostdin",
             b"-v", b"error",
             b"-hide_banner",
-            b"-i", fsenc(abspath),
+            b"-i", fsenc(infile),
             b"-filter_complex", fc.encode("utf-8"),
             b"-map", b"[o]",
             b"-frames:v", b"1",
@@ -708,7 +891,7 @@ class ThumbSrv(object):
             ]
 
         cmd += [fsenc(tpath)]
-        self._run_ff(cmd, vn)
+        self._run_ff(cmd, vn, "convt")
 
     def conv_mp3(self, abspath: str, tpath: str, fmt: str, vn: VFS) -> None:
         quality = self.args.q_mp3.lower()
@@ -716,7 +899,7 @@ class ThumbSrv(object):
             raise Exception("disabled in server config")
 
         self.wait4ram(0.2, tpath)
-        tags, rawtags = ffprobe(abspath, int(vn.flags["convt"] / 2))
+        tags, rawtags, _, _ = ffprobe(abspath, int(vn.flags["convt"] / 2))
         if "ac" not in tags:
             raise Exception("not audio")
 
@@ -747,14 +930,74 @@ class ThumbSrv(object):
             fsenc(tpath)
         ]
         # fmt: on
-        self._run_ff(cmd, vn, oom=300)
+        self._run_ff(cmd, vn, "aconvt", oom=300)
+
+    def conv_flac(self, abspath: str, tpath: str, fmt: str, vn: VFS) -> None:
+        if self.args.no_acode or not self.args.allow_flac:
+            raise Exception("flac not permitted in server config")
+
+        self.wait4ram(0.2, tpath)
+        tags, _, _, _ = ffprobe(abspath, int(vn.flags["convt"] / 2))
+        if "ac" not in tags:
+            raise Exception("not audio")
+
+        self.log("conv2 flac", 6)
+
+        # fmt: off
+        cmd = [
+            b"ffmpeg",
+            b"-nostdin",
+            b"-v", b"error",
+            b"-hide_banner",
+            b"-i", fsenc(abspath),
+            b"-map", b"0:a:0",
+            b"-c:a", b"flac",
+            fsenc(tpath)
+        ]
+        # fmt: on
+        self._run_ff(cmd, vn, "aconvt", oom=300)
+
+    def conv_wav(self, abspath: str, tpath: str, fmt: str, vn: VFS) -> None:
+        if self.args.no_acode or not self.args.allow_wav:
+            raise Exception("wav not permitted in server config")
+
+        self.wait4ram(0.2, tpath)
+        tags, _, _, _ = ffprobe(abspath, int(vn.flags["convt"] / 2))
+        if "ac" not in tags:
+            raise Exception("not audio")
+
+        bits = tags[".bps"][1]
+        if bits == 0.0:
+            bits = tags[".bprs"][1]
+
+        codec = b"pcm_s32le"
+        if bits <= 16.0:
+            codec = b"pcm_s16le"
+        elif bits <= 24.0:
+            codec = b"pcm_s24le"
+
+        self.log("conv2 wav", 6)
+
+        # fmt: off
+        cmd = [
+            b"ffmpeg",
+            b"-nostdin",
+            b"-v", b"error",
+            b"-hide_banner",
+            b"-i", fsenc(abspath),
+            b"-map", b"0:a:0",
+            b"-c:a", codec,
+            fsenc(tpath)
+        ]
+        # fmt: on
+        self._run_ff(cmd, vn, "aconvt", oom=300)
 
     def conv_opus(self, abspath: str, tpath: str, fmt: str, vn: VFS) -> None:
         if self.args.no_acode or not self.args.q_opus:
             raise Exception("disabled in server config")
 
         self.wait4ram(0.2, tpath)
-        tags, rawtags = ffprobe(abspath, int(vn.flags["convt"] / 2))
+        tags, rawtags, _, _ = ffprobe(abspath, int(vn.flags["convt"] / 2))
         if "ac" not in tags:
             raise Exception("not audio")
 
@@ -803,7 +1046,7 @@ class ThumbSrv(object):
             fsenc(tpath)
         ]
         # fmt: on
-        self._run_ff(cmd, vn, oom=300)
+        self._run_ff(cmd, vn, "aconvt", oom=300)
 
     def _conv_caf(
         self,
@@ -843,7 +1086,7 @@ class ThumbSrv(object):
             fsenc(tmp_opus)
         ]
         # fmt: on
-        self._run_ff(cmd, vn, oom=300)
+        self._run_ff(cmd, vn, "aconvt", oom=300)
 
         # iOS fails to play some "insufficiently complex" files
         # (average file shorter than 8 seconds), so of course we
@@ -870,7 +1113,7 @@ class ThumbSrv(object):
                 fsenc(tpath)
             ]
             # fmt: on
-            self._run_ff(cmd, vn, oom=300)
+            self._run_ff(cmd, vn, "aconvt", oom=300)
 
         else:
             # simple remux should be safe
@@ -889,7 +1132,7 @@ class ThumbSrv(object):
                 fsenc(tpath)
             ]
             # fmt: on
-            self._run_ff(cmd, vn, oom=300)
+            self._run_ff(cmd, vn, "aconvt", oom=300)
 
         try:
             wunlink(self.log, tmp_opus, vn.flags)
@@ -991,6 +1234,8 @@ class ThumbSrv(object):
             # thumb file
             try:
                 b64, ts, ext = f.split(".")
+                if len(ts) > 8 and PTN_TS.match(ts):
+                    ts = "yeahokay"
                 if len(b64) != 24 or len(ts) != 8 or ext not in exts:
                     raise Exception()
             except:

copyparty/u2idx.py

@@ -53,6 +53,11 @@ class U2idx(object):
             self.log("your python does not have sqlite3; searching will be disabled")
             return
 
+        if self.args.srch_icase:
+            self._open_db = self._open_db_icase
+        else:
+            self._open_db = self._open_db_std
+
         assert sqlite3  # type: ignore  # !rm
 
         self.active_id = ""
@@ -69,6 +74,16 @@ class U2idx(object):
     def log(self, msg: str, c: Union[int, str] = 0) -> None:
         self.log_func("u2idx", msg, c)
 
+    def _open_db_std(self, *args, **kwargs):
+        assert sqlite3  # type: ignore  # !rm
+        kwargs["check_same_thread"] = False
+        return sqlite3.connect(*args, **kwargs)
+
+    def _open_db_icase(self, *args, **kwargs):
+        db = self._open_db_std(*args, **kwargs)
+        db.create_function("casefold", 1, lambda x: x.casefold() if x else x)
+        return db
+
     def shutdown(self) -> None:
         if not HAVE_SQLITE3:
             return
@@ -134,9 +149,9 @@ class U2idx(object):
         assert sqlite3  # type: ignore  # !rm
 
         ptop = vn.realpath
-        histpath = self.asrv.vfs.histtab.get(ptop)
+        histpath = self.asrv.vfs.dbpaths.get(ptop)
         if not histpath:
-            self.log("no histpath for %r" % (ptop,))
+            self.log("no dbpath for %r" % (ptop,))
             return None
 
         db_path = os.path.join(histpath, "up2k.db")
@@ -148,8 +163,7 @@ class U2idx(object):
             uri = ""
             try:
                 uri = "{}?mode=ro&nolock=1".format(Path(db_path).as_uri())
-                db = sqlite3.connect(uri, timeout=2, uri=True, check_same_thread=False)
-                cur = db.cursor()
+                cur = self._open_db(uri, timeout=2, uri=True).cursor()
                 cur.execute('pragma table_info("up")').fetchone()
                 self.log("ro: %r" % (db_path,))
             except:
@@ -160,7 +174,7 @@ class U2idx(object):
         if not cur:
             # on windows, this steals the write-lock from up2k.deferred_init --
             # seen on win 10.0.17763.2686, py 3.10.4, sqlite 3.37.2
-            cur = sqlite3.connect(db_path, timeout=2, check_same_thread=False).cursor()
+            cur = self._open_db(db_path, timeout=2).cursor()
             self.log("opened %r" % (db_path,))
 
         self.cur[ptop] = cur
@@ -173,6 +187,8 @@ class U2idx(object):
         if not HAVE_SQLITE3:
             return [], [], False
 
+        icase = self.args.srch_icase
+
         q = ""
         v: Union[str, int] = ""
         va: list[Union[str, int]] = []
@@ -180,6 +196,7 @@ class U2idx(object):
         is_key = True
         is_size = False
         is_date = False
+        is_wark = False
         field_end = ""  # closing parenthesis or whatever
         kw_key = ["(", ")", "and ", "or ", "not "]
         kw_val = ["==", "=", "!=", ">", ">=", "<", "<=", "like "]
@@ -198,6 +215,8 @@ class U2idx(object):
                     is_key = kw in kw_key
                     uq = uq[len(kw) :]
                     ok = True
+                    if is_wark:
+                        kw = "= "
                     q += kw
                     break
 
@@ -232,9 +251,17 @@ class U2idx(object):
                 elif v == "path":
                     v = "trim(?||up.rd,'/')"
                     va.append("\nrd")
+                    if icase:
+                        v = "casefold(%s)" % (v,)
 
                 elif v == "name":
                     v = "up.fn"
+                    if icase:
+                        v = "casefold(%s)" % (v,)
+
+                elif v == "w":
+                    v = "substr(up.w,1,16)"
+                    is_wark = True
 
                 elif v == "tags" or ptn_mt.match(v):
                     have_mt = True
@@ -247,7 +274,7 @@ class U2idx(object):
                     v = "exists(select 1 from mt where mt.w = mtw and " + vq
 
                 else:
-                    raise Pebkac(400, "invalid key [{}]".format(v))
+                    raise Pebkac(400, "invalid key %r" % (v,))
 
                 q += v + " "
                 continue
@@ -276,6 +303,14 @@ class U2idx(object):
                 is_size = False
                 v = int(float(v) * 1024 * 1024)
 
+            elif is_wark:
+                is_wark = False
+                v = v.strip("*")
+                if len(v) > 16:
+                    v = v[:16]
+                if len(v) < 16:
+                    raise Pebkac(400, "w/filehash must be 16+ chars")
+
             else:
                 if v.startswith("*"):
                     head = "'%'||"
@@ -285,6 +320,12 @@ class U2idx(object):
                     tail = "||'%'"
                     v = v[:-1]
 
+            if icase and "casefold(" in q:
+                try:
+                    v = unicode(v).casefold()
+                except:
+                    v = unicode(v).lower()
+
             q += " {}?{} ".format(head, tail)
             va.append(v)
             is_key = True
@@ -319,7 +360,7 @@ class U2idx(object):
         uname: str,
         vols: list[VFS],
         uq: str,
-        uv: list[Union[str, int]],
+        uv: Union[list[str], list[Union[str, int]]],
         have_mt: bool,
         sort: bool,
         lim: int,
@@ -391,7 +432,7 @@ class U2idx(object):
             fk_alg = 2 if "fka" in flags else 1
             c = cur.execute(uq, tuple(vuv))
             for hit in c:
-                w, ts, sz, rd, fn, ip, at = hit[:7]
+                w, ts, sz, rd, fn = hit[:5]
 
                 if rd.startswith("//") or fn.startswith("//"):
                     rd, fn = s3dec(rd, fn)