mirror of
https://github.com/Tyrrrz/DiscordChatExporter.git
synced 2026-06-10 00:02:37 -06:00
feat(scrape): add run-operator-validation orchestrator
Sync GUI token, verify readiness, run documents scrape, and audit JSON with timestamped logs. Live eod_discord validation passed on host.
This commit is contained in:
parent
00bcbc5b21
commit
1742a9d41e
|
|
@ -357,6 +357,7 @@ With Docker/Podman, include the container smoke:
|
||||||
| `audit-archive-json-smoke.sh` | yes | Invalid JSON detection |
|
| `audit-archive-json-smoke.sh` | yes | Invalid JSON detection |
|
||||||
| `prove-incremental-append-smoke.sh` | yes | Offline prove snapshot/compare |
|
| `prove-incremental-append-smoke.sh` | yes | Offline prove snapshot/compare |
|
||||||
| `verify-operator-ready-smoke.sh` | yes | Host prerequisite checks |
|
| `verify-operator-ready-smoke.sh` | yes | Host prerequisite checks |
|
||||||
|
| `run-operator-validation-smoke.sh` | yes | Validation runner dry-run |
|
||||||
| `container-smoke.sh` | no (local) | Docker build + `help` / `list-targets`; use `--include-container` |
|
| `container-smoke.sh` | no (local) | Docker build + `help` / `list-targets`; use `--include-container` |
|
||||||
|
|
||||||
GitHub Actions runs `./scripts/run-all-smokes.sh` via `.github/workflows/main.yml` job `recurring-scrape-smoke`.
|
GitHub Actions runs `./scripts/run-all-smokes.sh` via `.github/workflows/main.yml` job `recurring-scrape-smoke`.
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,29 @@
|
||||||
|
---
|
||||||
|
title: feat: Live operator validation runner
|
||||||
|
type: feat
|
||||||
|
status: complete
|
||||||
|
date: 2026-05-29
|
||||||
|
origin: Repeated /lfg — execute end-to-end validation on host, not only offline smokes
|
||||||
|
---
|
||||||
|
|
||||||
|
# feat: Live operator validation runner
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
Add `run-operator-validation.sh` to orchestrate GUI token sync, readiness checks, incremental scrape, and post-run JSON audit with a timestamped log.
|
||||||
|
|
||||||
|
## Requirements
|
||||||
|
|
||||||
|
| ID | Requirement |
|
||||||
|
|----|-------------|
|
||||||
|
| R1 | `scripts/run-operator-validation.sh` supports `--dry-run`, `--target`, `--sync-gui`, `--skip-scrape` |
|
||||||
|
| R2 | Writes `logs/operator-validation-*.log` with step timestamps |
|
||||||
|
| R3 | Post-scrape runs `audit-archive-json.sh` per affected targets |
|
||||||
|
| R4 | Offline smoke for dry-run path |
|
||||||
|
| R5 | Document in merge-readiness as post-bootstrap validation |
|
||||||
|
|
||||||
|
## Verification
|
||||||
|
|
||||||
|
- `./scripts/tests/run-operator-validation-smoke.sh`
|
||||||
|
- `./scripts/run-all-smokes.sh`
|
||||||
|
- Host: `./scripts/run-operator-validation.sh --dry-run` exits 0
|
||||||
|
|
@ -35,6 +35,14 @@ Optional Discord probe for one target:
|
||||||
./scripts/verify-operator-ready.sh --preflight KotOR_discord_msgs
|
./scripts/verify-operator-ready.sh --preflight KotOR_discord_msgs
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Full validation with log (GUI token sync + scrape + audit):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./scripts/run-operator-validation.sh --sync-gui
|
||||||
|
./scripts/run-operator-validation.sh --sync-gui --target eod_discord
|
||||||
|
./scripts/run-operator-validation.sh --dry-run
|
||||||
|
```
|
||||||
|
|
||||||
Detail: [.docs/Recurring-Scrape-Setup.md](../.docs/Recurring-Scrape-Setup.md) · [operator checklist](recurring-scrape-operator-checklist.md) · [troubleshooting](../.docs/Recurring-Scrape-Troubleshooting.md)
|
Detail: [.docs/Recurring-Scrape-Setup.md](../.docs/Recurring-Scrape-Setup.md) · [operator checklist](recurring-scrape-operator-checklist.md) · [troubleshooting](../.docs/Recurring-Scrape-Troubleshooting.md)
|
||||||
|
|
||||||
## CI note (fork PRs)
|
## CI note (fork PRs)
|
||||||
|
|
|
||||||
152
scripts/run-operator-validation.sh
Executable file
152
scripts/run-operator-validation.sh
Executable file
|
|
@ -0,0 +1,152 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
|
||||||
|
set -Eeuo pipefail
|
||||||
|
|
||||||
|
SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)
|
||||||
|
REPO_ROOT="${DCE_REPO_ROOT:-$(cd "$SCRIPT_DIR/.." && pwd -P)}"
|
||||||
|
CONFIG_PATH="${DCE_CONFIG_FILE:-$REPO_ROOT/config/scrape-targets.json}"
|
||||||
|
LOG_DIR="${DCE_LOG_DIR:-$REPO_ROOT/logs}"
|
||||||
|
SYNC_GUI="$REPO_ROOT/scripts/sync-token-from-gui.sh"
|
||||||
|
VERIFY_READY="$REPO_ROOT/scripts/verify-operator-ready.sh"
|
||||||
|
DOCUMENTS_SCRAPE="$REPO_ROOT/scripts/run-documents-scrape.sh"
|
||||||
|
AUDIT_JSON="$REPO_ROOT/scripts/audit-archive-json.sh"
|
||||||
|
|
||||||
|
DRY_RUN=0
|
||||||
|
SKIP_SCRAPE=0
|
||||||
|
SYNC_GUI_FLAG=0
|
||||||
|
TARGET=""
|
||||||
|
LOG_FILE=""
|
||||||
|
|
||||||
|
usage() {
|
||||||
|
cat <<EOF
|
||||||
|
Usage:
|
||||||
|
$(basename "$0") [options]
|
||||||
|
|
||||||
|
End-to-end operator validation with timestamped log:
|
||||||
|
optional GUI token sync → verify-operator-ready → documents scrape → JSON audit
|
||||||
|
|
||||||
|
Options:
|
||||||
|
--dry-run Readiness + archives only (no Discord scrape)
|
||||||
|
--skip-scrape Readiness only (no scrape, no audit loop)
|
||||||
|
--sync-gui Run sync-token-from-gui.sh --force before checks
|
||||||
|
--target NAME Limit scrape/audit to one configured target
|
||||||
|
--config PATH Targets JSON (default: config/scrape-targets.json)
|
||||||
|
--log-file PATH Append output to this file (default: logs/operator-validation-UTC.log)
|
||||||
|
--help Show this help text
|
||||||
|
EOF
|
||||||
|
}
|
||||||
|
|
||||||
|
die() {
|
||||||
|
printf 'ERROR: %s\n' "$*" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
log_step() {
|
||||||
|
printf '[%s] %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*"
|
||||||
|
}
|
||||||
|
|
||||||
|
run_step() {
|
||||||
|
local label=$1
|
||||||
|
shift
|
||||||
|
log_step "BEGIN: $label"
|
||||||
|
"$@"
|
||||||
|
local status=$?
|
||||||
|
log_step "END: $label (exit $status)"
|
||||||
|
return "$status"
|
||||||
|
}
|
||||||
|
|
||||||
|
audit_targets() {
|
||||||
|
if [[ -n "$TARGET" ]]; then
|
||||||
|
run_step "audit-archive-json ($TARGET)" "$AUDIT_JSON" --config "$CONFIG_PATH" --target "$TARGET"
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
local name
|
||||||
|
while IFS= read -r name; do
|
||||||
|
[[ -n "$name" ]] || continue
|
||||||
|
run_step "audit-archive-json ($name)" "$AUDIT_JSON" --config "$CONFIG_PATH" --target "$name" || return 1
|
||||||
|
done < <(jq -r '.targets[] | select(.enabled != false) | .name' "$CONFIG_PATH")
|
||||||
|
}
|
||||||
|
|
||||||
|
main() {
|
||||||
|
while (($#)); do
|
||||||
|
case "$1" in
|
||||||
|
--dry-run)
|
||||||
|
DRY_RUN=1
|
||||||
|
shift
|
||||||
|
;;
|
||||||
|
--skip-scrape)
|
||||||
|
SKIP_SCRAPE=1
|
||||||
|
shift
|
||||||
|
;;
|
||||||
|
--sync-gui)
|
||||||
|
SYNC_GUI_FLAG=1
|
||||||
|
shift
|
||||||
|
;;
|
||||||
|
--target)
|
||||||
|
[[ $# -ge 2 ]] || die "Missing value for --target."
|
||||||
|
TARGET=$2
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
--config)
|
||||||
|
[[ $# -ge 2 ]] || die "Missing value for --config."
|
||||||
|
CONFIG_PATH=$2
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
--log-file)
|
||||||
|
[[ $# -ge 2 ]] || die "Missing value for --log-file."
|
||||||
|
LOG_FILE=$2
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
--help|-h)
|
||||||
|
usage
|
||||||
|
exit 0
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
die "Unknown option: $1"
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
mkdir -p "$LOG_DIR"
|
||||||
|
if [[ -z "$LOG_FILE" ]]; then
|
||||||
|
LOG_FILE="$LOG_DIR/operator-validation-$(date -u +%Y%m%dT%H%M%SZ).log"
|
||||||
|
fi
|
||||||
|
|
||||||
|
local failures=0
|
||||||
|
|
||||||
|
set -o pipefail
|
||||||
|
{
|
||||||
|
log_step "Operator validation started (config=$CONFIG_PATH)"
|
||||||
|
if (( SYNC_GUI_FLAG )); then
|
||||||
|
run_step "sync-token-from-gui" "$SYNC_GUI" --force || failures=$((failures + 1))
|
||||||
|
fi
|
||||||
|
|
||||||
|
run_step "verify-operator-ready" "$VERIFY_READY" --config "$CONFIG_PATH" || failures=$((failures + 1))
|
||||||
|
|
||||||
|
if (( SKIP_SCRAPE )); then
|
||||||
|
log_step "Skip scrape requested."
|
||||||
|
else
|
||||||
|
local -a scrape_args=(--config "$CONFIG_PATH")
|
||||||
|
[[ -n "$TARGET" ]] && scrape_args+=(--target "$TARGET")
|
||||||
|
if (( DRY_RUN )); then
|
||||||
|
scrape_args+=(--dry-run)
|
||||||
|
fi
|
||||||
|
run_step "run-documents-scrape" "$DOCUMENTS_SCRAPE" "${scrape_args[@]}" || failures=$((failures + 1))
|
||||||
|
if (( DRY_RUN == 0 && failures == 0 )); then
|
||||||
|
audit_targets || failures=$((failures + 1))
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
if (( failures > 0 )); then
|
||||||
|
log_step "Operator validation failed ($failures step(s))."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
log_step "Operator validation finished successfully."
|
||||||
|
} 2>&1 | tee -a "$LOG_FILE"
|
||||||
|
local pipeline_status=${PIPESTATUS[0]}
|
||||||
|
|
||||||
|
printf 'Log: %s\n' "$LOG_FILE"
|
||||||
|
exit "$pipeline_status"
|
||||||
|
}
|
||||||
|
|
||||||
|
main "$@"
|
||||||
59
scripts/tests/run-operator-validation-smoke.sh
Executable file
59
scripts/tests/run-operator-validation-smoke.sh
Executable file
|
|
@ -0,0 +1,59 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
|
||||||
|
set -Eeuo pipefail
|
||||||
|
|
||||||
|
REPO_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd -P)
|
||||||
|
RUNNER="$REPO_ROOT/scripts/run-operator-validation.sh"
|
||||||
|
TMP_DIR=$(mktemp -d "${TMPDIR:-/tmp}/dce-op-val-smoke.XXXXXX")
|
||||||
|
ARCHIVE_ROOT="$TMP_DIR/archive"
|
||||||
|
CONFIG_PATH="$TMP_DIR/config.json"
|
||||||
|
ENV_PATH="$TMP_DIR/scrape.env"
|
||||||
|
LOG_DIR="$TMP_DIR/logs"
|
||||||
|
FAKE_DOCKER="$TMP_DIR/docker"
|
||||||
|
PATH_BACKUP="$PATH"
|
||||||
|
|
||||||
|
cleanup() {
|
||||||
|
export PATH="$PATH_BACKUP"
|
||||||
|
rm -rf "$TMP_DIR"
|
||||||
|
}
|
||||||
|
trap cleanup EXIT
|
||||||
|
|
||||||
|
mkdir -p "$ARCHIVE_ROOT/demo" "$LOG_DIR"
|
||||||
|
printf '{"messages":[{"id":"1"}],"channel":{"id":"111111111111111111"}}\n' \
|
||||||
|
>"$ARCHIVE_ROOT/demo/Guild - general [111111111111111111].json"
|
||||||
|
|
||||||
|
cat >"$CONFIG_PATH" <<JSON
|
||||||
|
{
|
||||||
|
"archive_root": "$ARCHIVE_ROOT",
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"name": "demo",
|
||||||
|
"kind": "guild",
|
||||||
|
"output_dir": "$ARCHIVE_ROOT/demo",
|
||||||
|
"enabled": true
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
JSON
|
||||||
|
|
||||||
|
printf 'DISCORD_TOKEN=dummy\n' >"$ENV_PATH"
|
||||||
|
|
||||||
|
cat >"$FAKE_DOCKER" <<'EOF'
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
if [[ "${1:-}" == "compose" && "${2:-}" == "version" ]]; then
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
exit 1
|
||||||
|
EOF
|
||||||
|
chmod +x "$FAKE_DOCKER"
|
||||||
|
export PATH="$TMP_DIR:$PATH_BACKUP"
|
||||||
|
|
||||||
|
DCE_REPO_ROOT="$REPO_ROOT" DCE_CONFIG_FILE="$CONFIG_PATH" DCE_ENV_FILE="$ENV_PATH" DCE_LOG_DIR="$LOG_DIR" \
|
||||||
|
"$RUNNER" --dry-run --config "$CONFIG_PATH" --log-file "$LOG_DIR/validation.log"
|
||||||
|
|
||||||
|
grep -q 'Operator validation finished successfully' "$LOG_DIR/validation.log" || {
|
||||||
|
printf 'ERROR: validation log missing success marker\n' >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
printf 'run-operator-validation-smoke: ok\n'
|
||||||
Loading…
Reference in a new issue