mirror of
https://github.com/Tyrrrz/DiscordChatExporter.git
synced 2026-06-09 15:52:37 -06:00
fix(host): stop forcing compose -T so operator logs stream live
podman-compose and docker compose allocate a pseudo-TTY by default; always passing -T block-buffered export progress. Omit -T for operator runs and set DCE_COMPOSE_TTY=0 only for cron log append. Adds compose TTY smokes and cron job env assertion.
This commit is contained in:
parent
d8742c5c7b
commit
14796e9c09
43
docs/plans/2026-06-04-050-feat-compose-tty-live-logs-plan.md
Normal file
43
docs/plans/2026-06-04-050-feat-compose-tty-live-logs-plan.md
Normal file
|
|
@ -0,0 +1,43 @@
|
||||||
|
---
|
||||||
|
title: "feat: Allocate compose TTY for live operator scrape logs"
|
||||||
|
type: feat
|
||||||
|
status: complete
|
||||||
|
date: 2026-06-04
|
||||||
|
origin: /lfg — plan 048 host tee still shows frozen validation logs; yes_general temp grows but podman run -T block-buffers CLI progress
|
||||||
|
---
|
||||||
|
|
||||||
|
# feat: Allocate compose TTY for live operator scrape logs
|
||||||
|
|
||||||
|
## Problem
|
||||||
|
|
||||||
|
Plan 048 streams host-side compose output via `tee`, but `compose_run_args` always passes `-T` (no pseudo-TTY). Containerized DiscordChatExporter progress lines stay block-buffered when piped, so operator validation logs remain silent for hours during large exports.
|
||||||
|
|
||||||
|
## Requirements
|
||||||
|
|
||||||
|
| ID | Requirement |
|
||||||
|
|----|-------------|
|
||||||
|
| R1 | `compose_run_args` omits `-T` when `DCE_COMPOSE_TTY` is unset or non-zero; passes `-T` only when `DCE_COMPOSE_TTY=0` |
|
||||||
|
| R2 | `setup-cron.sh` cron job sets `DCE_COMPOSE_TTY=0` (non-interactive log append) |
|
||||||
|
| R3 | Host smoke asserts default omits `-T` and `DCE_COMPOSE_TTY=0` passes `-T` |
|
||||||
|
| R4 | Cron smoke asserts installed job includes `DCE_COMPOSE_TTY=0` |
|
||||||
|
| R5 | `run-all-smokes.sh` passes |
|
||||||
|
|
||||||
|
## Implementation
|
||||||
|
|
||||||
|
- `scripts/run-discord-scrape-host.sh` — `compose_tty_flag()` + use in all compose branches; document env in usage
|
||||||
|
- `scripts/setup-cron.sh` — prefix cron job with `DCE_COMPOSE_TTY=0`
|
||||||
|
- `scripts/tests/run-discord-scrape-host-smoke.sh` — compose arg capture for `-t`/`-T`
|
||||||
|
- `scripts/tests/setup-cron-smoke.sh` — grep crontab for `DCE_COMPOSE_TTY=0`
|
||||||
|
|
||||||
|
## Verification
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./scripts/tests/run-discord-scrape-host-smoke.sh
|
||||||
|
./scripts/tests/setup-cron-smoke.sh
|
||||||
|
DCE_MIN_FREE_MB=0 ./scripts/run-all-smokes.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
## Out of scope
|
||||||
|
|
||||||
|
- yes_general catch-up completion
|
||||||
|
- Container memory limits
|
||||||
|
|
@ -37,6 +37,8 @@ Environment:
|
||||||
DISCORD_TOKEN Direct token value (highest precedence after refresh).
|
DISCORD_TOKEN Direct token value (highest precedence after refresh).
|
||||||
DISCORD_TOKEN_FILE Optional path to a file containing the Discord token.
|
DISCORD_TOKEN_FILE Optional path to a file containing the Discord token.
|
||||||
DCE_REAUTH_COMMAND Optional absolute path to an executable reauth script under the repo root.
|
DCE_REAUTH_COMMAND Optional absolute path to an executable reauth script under the repo root.
|
||||||
|
DCE_COMPOSE_TTY When zero, compose run passes -T (no pseudo-TTY). Default omits -T
|
||||||
|
so compose backends allocate a TTY for line-buffered progress logs.
|
||||||
|
|
||||||
Notes:
|
Notes:
|
||||||
When $ENV_FILE is missing, exported DISCORD_TOKEN or DISCORD_TOKEN_FILE is used instead.
|
When $ENV_FILE is missing, exported DISCORD_TOKEN or DISCORD_TOKEN_FILE is used instead.
|
||||||
|
|
@ -255,11 +257,19 @@ resolve_compose_bin() {
|
||||||
COMPOSE_BIN=""
|
COMPOSE_BIN=""
|
||||||
}
|
}
|
||||||
|
|
||||||
|
compose_tty_flag() {
|
||||||
|
if [[ "${DCE_COMPOSE_TTY:-1}" == "0" ]]; then
|
||||||
|
printf '%s' '-T'
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
compose_run_args() {
|
compose_run_args() {
|
||||||
local -n _out=$1
|
local -n _out=$1
|
||||||
local subcommand=$2
|
local subcommand=$2
|
||||||
|
local tty_flag
|
||||||
shift 2
|
shift 2
|
||||||
|
|
||||||
|
tty_flag=$(compose_tty_flag)
|
||||||
resolve_compose_bin
|
resolve_compose_bin
|
||||||
|
|
||||||
_out=()
|
_out=()
|
||||||
|
|
@ -269,7 +279,9 @@ compose_run_args() {
|
||||||
--env-file "$COMPOSE_ENV_FILE"
|
--env-file "$COMPOSE_ENV_FILE"
|
||||||
-f "$COMPOSE_FILE"
|
-f "$COMPOSE_FILE"
|
||||||
run
|
run
|
||||||
-T
|
)
|
||||||
|
[[ -n "$tty_flag" ]] && _out+=("$tty_flag")
|
||||||
|
_out+=(
|
||||||
--rm
|
--rm
|
||||||
discord-scraper
|
discord-scraper
|
||||||
"$subcommand"
|
"$subcommand"
|
||||||
|
|
@ -280,7 +292,9 @@ compose_run_args() {
|
||||||
--env-file "$COMPOSE_ENV_FILE"
|
--env-file "$COMPOSE_ENV_FILE"
|
||||||
-f "$COMPOSE_FILE"
|
-f "$COMPOSE_FILE"
|
||||||
run
|
run
|
||||||
-T
|
)
|
||||||
|
[[ -n "$tty_flag" ]] && _out+=("$tty_flag")
|
||||||
|
_out+=(
|
||||||
--rm
|
--rm
|
||||||
discord-scraper
|
discord-scraper
|
||||||
"$subcommand"
|
"$subcommand"
|
||||||
|
|
@ -292,7 +306,9 @@ compose_run_args() {
|
||||||
--env-file "$COMPOSE_ENV_FILE"
|
--env-file "$COMPOSE_ENV_FILE"
|
||||||
-f "$COMPOSE_FILE"
|
-f "$COMPOSE_FILE"
|
||||||
run
|
run
|
||||||
-T
|
)
|
||||||
|
[[ -n "$tty_flag" ]] && _out+=("$tty_flag")
|
||||||
|
_out+=(
|
||||||
--rm
|
--rm
|
||||||
discord-scraper
|
discord-scraper
|
||||||
"$subcommand"
|
"$subcommand"
|
||||||
|
|
|
||||||
|
|
@ -348,7 +348,7 @@ main() {
|
||||||
lock_prefix=""
|
lock_prefix=""
|
||||||
fi
|
fi
|
||||||
|
|
||||||
job_line="$cron_line cd $(printf '%q' "$REPO_ROOT") && ${lock_prefix}${scrape_command}>> $(printf '%q' "$LOG_FILE") 2>&1"
|
job_line="$cron_line cd $(printf '%q' "$REPO_ROOT") && DCE_COMPOSE_TTY=0 ${lock_prefix}${scrape_command}>> $(printf '%q' "$LOG_FILE") 2>&1"
|
||||||
|
|
||||||
local cron_block
|
local cron_block
|
||||||
cron_block=$(printf '%s\n%s\n%s\n' "$begin_marker" "$job_line" "$end_marker")
|
cron_block=$(printf '%s\n%s\n%s\n' "$begin_marker" "$job_line" "$end_marker")
|
||||||
|
|
|
||||||
|
|
@ -103,6 +103,24 @@ run_host() {
|
||||||
"$REPO_ROOT/scripts/run-discord-scrape-host.sh" scrape --target demo
|
"$REPO_ROOT/scripts/run-discord-scrape-host.sh" scrape --target demo
|
||||||
}
|
}
|
||||||
|
|
||||||
|
run_host_compose_capture() {
|
||||||
|
local env_path=${1:-$ENV_FILE}
|
||||||
|
local compose_bin=$2
|
||||||
|
local args_log=$3
|
||||||
|
shift 3
|
||||||
|
local -a extra_env=( "$@" )
|
||||||
|
|
||||||
|
env -u DISCORD_TOKEN \
|
||||||
|
DCE_SKIP_SCRAPE_LOCK=1 \
|
||||||
|
DCE_COMPOSE_BIN="$compose_bin" \
|
||||||
|
DCE_REPO_ROOT="$REPO_ROOT" \
|
||||||
|
DCE_ENV_FILE="$env_path" \
|
||||||
|
DCE_COMPOSE_FILE="$COMPOSE_FILE" \
|
||||||
|
FAKE_COMPOSE_ARGS_LOG="$args_log" \
|
||||||
|
"${extra_env[@]}" \
|
||||||
|
"$REPO_ROOT/scripts/run-discord-scrape-host.sh" scrape --target demo
|
||||||
|
}
|
||||||
|
|
||||||
run_host_with_shell_token() {
|
run_host_with_shell_token() {
|
||||||
local mode=$1
|
local mode=$1
|
||||||
local missing_env_path=$2
|
local missing_env_path=$2
|
||||||
|
|
@ -173,4 +191,33 @@ grep -q streaming-line2 "$STREAM_OUTPUT" || {
|
||||||
exit 1
|
exit 1
|
||||||
}
|
}
|
||||||
|
|
||||||
|
COMPOSE_TTY_LOG="$TMP_DIR/compose-tty-default.log"
|
||||||
|
FAKE_COMPOSE="$TMP_DIR/fake-compose"
|
||||||
|
cat >"$FAKE_COMPOSE" <<'EOF'
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
printf '%s\n' "$*" >>"${FAKE_COMPOSE_ARGS_LOG:?}"
|
||||||
|
printf 'run succeeded\n'
|
||||||
|
EOF
|
||||||
|
chmod +x "$FAKE_COMPOSE"
|
||||||
|
|
||||||
|
run_host_compose_capture "$ENV_FILE" "$FAKE_COMPOSE" "$COMPOSE_TTY_LOG" >/dev/null
|
||||||
|
grep -q ' run --rm ' "$COMPOSE_TTY_LOG" || {
|
||||||
|
echo "expected default compose run to omit -T for live TTY allocation" >&2
|
||||||
|
cat "$COMPOSE_TTY_LOG" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
grep -qE '(^|[[:space:]])-T([[:space:]]|$)' "$COMPOSE_TTY_LOG" && {
|
||||||
|
echo "expected default compose run not to pass -T" >&2
|
||||||
|
cat "$COMPOSE_TTY_LOG" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
COMPOSE_NOTTY_LOG="$TMP_DIR/compose-tty-off.log"
|
||||||
|
run_host_compose_capture "$ENV_FILE" "$FAKE_COMPOSE" "$COMPOSE_NOTTY_LOG" DCE_COMPOSE_TTY=0 >/dev/null
|
||||||
|
grep -qE '(^|[[:space:]])-T([[:space:]]|$)' "$COMPOSE_NOTTY_LOG" || {
|
||||||
|
echo "expected DCE_COMPOSE_TTY=0 compose run to use -T" >&2
|
||||||
|
cat "$COMPOSE_NOTTY_LOG" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
echo "run-discord-scrape-host smoke test passed"
|
echo "run-discord-scrape-host smoke test passed"
|
||||||
|
|
|
||||||
|
|
@ -86,6 +86,7 @@ grep -q '^MAILTO=test@example.com$' "$CRONTAB_FILE" || { echo "expected unrelate
|
||||||
[[ "$(grep -c '^# BEGIN discord-scrape$' "$CRONTAB_FILE")" == "1" ]] || { echo "expected exactly one managed cron block after install" >&2; exit 1; }
|
[[ "$(grep -c '^# BEGIN discord-scrape$' "$CRONTAB_FILE")" == "1" ]] || { echo "expected exactly one managed cron block after install" >&2; exit 1; }
|
||||||
grep -q 'compose --env-file' "$DOCKER_LOG" || { echo "expected docker preflight to run during install" >&2; exit 1; }
|
grep -q 'compose --env-file' "$DOCKER_LOG" || { echo "expected docker preflight to run during install" >&2; exit 1; }
|
||||||
grep -q 'scripts/run-discord-scrape-host.sh' "$CRONTAB_FILE" || { echo "expected cron job to run host wrapper" >&2; exit 1; }
|
grep -q 'scripts/run-discord-scrape-host.sh' "$CRONTAB_FILE" || { echo "expected cron job to run host wrapper" >&2; exit 1; }
|
||||||
|
grep -q 'DCE_COMPOSE_TTY=0' "$CRONTAB_FILE" || { echo "expected cron job to disable compose TTY for log append" >&2; exit 1; }
|
||||||
|
|
||||||
run_setup
|
run_setup
|
||||||
[[ "$(grep -c '^# BEGIN discord-scrape$' "$CRONTAB_FILE")" == "1" ]] || { echo "expected exactly one managed cron block after reinstall" >&2; exit 1; }
|
[[ "$(grep -c '^# BEGIN discord-scrape$' "$CRONTAB_FILE")" == "1" ]] || { echo "expected exactly one managed cron block after reinstall" >&2; exit 1; }
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue