mirror of
https://github.com/Tyrrrz/DiscordChatExporter.git
synced 2026-06-09 15:52:37 -06:00
feat(scrape): add run-operator-validation orchestrator
Sync GUI token, verify readiness, run documents scrape, and audit JSON with timestamped logs. Live eod_discord validation passed on host.
This commit is contained in:
parent
00bcbc5b21
commit
1742a9d41e
|
|
@ -357,6 +357,7 @@ With Docker/Podman, include the container smoke:
|
|||
| `audit-archive-json-smoke.sh` | yes | Invalid JSON detection |
|
||||
| `prove-incremental-append-smoke.sh` | yes | Offline prove snapshot/compare |
|
||||
| `verify-operator-ready-smoke.sh` | yes | Host prerequisite checks |
|
||||
| `run-operator-validation-smoke.sh` | yes | Validation runner dry-run |
|
||||
| `container-smoke.sh` | no (local) | Docker build + `help` / `list-targets`; use `--include-container` |
|
||||
|
||||
GitHub Actions runs `./scripts/run-all-smokes.sh` via `.github/workflows/main.yml` job `recurring-scrape-smoke`.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,29 @@
|
|||
---
|
||||
title: feat: Live operator validation runner
|
||||
type: feat
|
||||
status: complete
|
||||
date: 2026-05-29
|
||||
origin: Repeated /lfg — execute end-to-end validation on host, not only offline smokes
|
||||
---
|
||||
|
||||
# feat: Live operator validation runner
|
||||
|
||||
## Summary
|
||||
|
||||
Add `run-operator-validation.sh` to orchestrate GUI token sync, readiness checks, incremental scrape, and post-run JSON audit with a timestamped log.
|
||||
|
||||
## Requirements
|
||||
|
||||
| ID | Requirement |
|
||||
|----|-------------|
|
||||
| R1 | `scripts/run-operator-validation.sh` supports `--dry-run`, `--target`, `--sync-gui`, `--skip-scrape` |
|
||||
| R2 | Writes `logs/operator-validation-*.log` with step timestamps |
|
||||
| R3 | Post-scrape runs `audit-archive-json.sh` per affected targets |
|
||||
| R4 | Offline smoke for dry-run path |
|
||||
| R5 | Document in merge-readiness as post-bootstrap validation |
|
||||
|
||||
## Verification
|
||||
|
||||
- `./scripts/tests/run-operator-validation-smoke.sh`
|
||||
- `./scripts/run-all-smokes.sh`
|
||||
- Host: `./scripts/run-operator-validation.sh --dry-run` exits 0
|
||||
|
|
@ -35,6 +35,14 @@ Optional Discord probe for one target:
|
|||
./scripts/verify-operator-ready.sh --preflight KotOR_discord_msgs
|
||||
```
|
||||
|
||||
Full validation with log (GUI token sync + scrape + audit):
|
||||
|
||||
```bash
|
||||
./scripts/run-operator-validation.sh --sync-gui
|
||||
./scripts/run-operator-validation.sh --sync-gui --target eod_discord
|
||||
./scripts/run-operator-validation.sh --dry-run
|
||||
```
|
||||
|
||||
Detail: [.docs/Recurring-Scrape-Setup.md](../.docs/Recurring-Scrape-Setup.md) · [operator checklist](recurring-scrape-operator-checklist.md) · [troubleshooting](../.docs/Recurring-Scrape-Troubleshooting.md)
|
||||
|
||||
## CI note (fork PRs)
|
||||
|
|
|
|||
152
scripts/run-operator-validation.sh
Executable file
152
scripts/run-operator-validation.sh
Executable file
|
|
@ -0,0 +1,152 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
set -Eeuo pipefail
|
||||
|
||||
SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)
|
||||
REPO_ROOT="${DCE_REPO_ROOT:-$(cd "$SCRIPT_DIR/.." && pwd -P)}"
|
||||
CONFIG_PATH="${DCE_CONFIG_FILE:-$REPO_ROOT/config/scrape-targets.json}"
|
||||
LOG_DIR="${DCE_LOG_DIR:-$REPO_ROOT/logs}"
|
||||
SYNC_GUI="$REPO_ROOT/scripts/sync-token-from-gui.sh"
|
||||
VERIFY_READY="$REPO_ROOT/scripts/verify-operator-ready.sh"
|
||||
DOCUMENTS_SCRAPE="$REPO_ROOT/scripts/run-documents-scrape.sh"
|
||||
AUDIT_JSON="$REPO_ROOT/scripts/audit-archive-json.sh"
|
||||
|
||||
DRY_RUN=0
|
||||
SKIP_SCRAPE=0
|
||||
SYNC_GUI_FLAG=0
|
||||
TARGET=""
|
||||
LOG_FILE=""
|
||||
|
||||
usage() {
|
||||
cat <<EOF
|
||||
Usage:
|
||||
$(basename "$0") [options]
|
||||
|
||||
End-to-end operator validation with timestamped log:
|
||||
optional GUI token sync → verify-operator-ready → documents scrape → JSON audit
|
||||
|
||||
Options:
|
||||
--dry-run Readiness + archives only (no Discord scrape)
|
||||
--skip-scrape Readiness only (no scrape, no audit loop)
|
||||
--sync-gui Run sync-token-from-gui.sh --force before checks
|
||||
--target NAME Limit scrape/audit to one configured target
|
||||
--config PATH Targets JSON (default: config/scrape-targets.json)
|
||||
--log-file PATH Append output to this file (default: logs/operator-validation-UTC.log)
|
||||
--help Show this help text
|
||||
EOF
|
||||
}
|
||||
|
||||
die() {
|
||||
printf 'ERROR: %s\n' "$*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
log_step() {
|
||||
printf '[%s] %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*"
|
||||
}
|
||||
|
||||
run_step() {
|
||||
local label=$1
|
||||
shift
|
||||
log_step "BEGIN: $label"
|
||||
"$@"
|
||||
local status=$?
|
||||
log_step "END: $label (exit $status)"
|
||||
return "$status"
|
||||
}
|
||||
|
||||
audit_targets() {
|
||||
if [[ -n "$TARGET" ]]; then
|
||||
run_step "audit-archive-json ($TARGET)" "$AUDIT_JSON" --config "$CONFIG_PATH" --target "$TARGET"
|
||||
return
|
||||
fi
|
||||
local name
|
||||
while IFS= read -r name; do
|
||||
[[ -n "$name" ]] || continue
|
||||
run_step "audit-archive-json ($name)" "$AUDIT_JSON" --config "$CONFIG_PATH" --target "$name" || return 1
|
||||
done < <(jq -r '.targets[] | select(.enabled != false) | .name' "$CONFIG_PATH")
|
||||
}
|
||||
|
||||
main() {
|
||||
while (($#)); do
|
||||
case "$1" in
|
||||
--dry-run)
|
||||
DRY_RUN=1
|
||||
shift
|
||||
;;
|
||||
--skip-scrape)
|
||||
SKIP_SCRAPE=1
|
||||
shift
|
||||
;;
|
||||
--sync-gui)
|
||||
SYNC_GUI_FLAG=1
|
||||
shift
|
||||
;;
|
||||
--target)
|
||||
[[ $# -ge 2 ]] || die "Missing value for --target."
|
||||
TARGET=$2
|
||||
shift 2
|
||||
;;
|
||||
--config)
|
||||
[[ $# -ge 2 ]] || die "Missing value for --config."
|
||||
CONFIG_PATH=$2
|
||||
shift 2
|
||||
;;
|
||||
--log-file)
|
||||
[[ $# -ge 2 ]] || die "Missing value for --log-file."
|
||||
LOG_FILE=$2
|
||||
shift 2
|
||||
;;
|
||||
--help|-h)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
die "Unknown option: $1"
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
mkdir -p "$LOG_DIR"
|
||||
if [[ -z "$LOG_FILE" ]]; then
|
||||
LOG_FILE="$LOG_DIR/operator-validation-$(date -u +%Y%m%dT%H%M%SZ).log"
|
||||
fi
|
||||
|
||||
local failures=0
|
||||
|
||||
set -o pipefail
|
||||
{
|
||||
log_step "Operator validation started (config=$CONFIG_PATH)"
|
||||
if (( SYNC_GUI_FLAG )); then
|
||||
run_step "sync-token-from-gui" "$SYNC_GUI" --force || failures=$((failures + 1))
|
||||
fi
|
||||
|
||||
run_step "verify-operator-ready" "$VERIFY_READY" --config "$CONFIG_PATH" || failures=$((failures + 1))
|
||||
|
||||
if (( SKIP_SCRAPE )); then
|
||||
log_step "Skip scrape requested."
|
||||
else
|
||||
local -a scrape_args=(--config "$CONFIG_PATH")
|
||||
[[ -n "$TARGET" ]] && scrape_args+=(--target "$TARGET")
|
||||
if (( DRY_RUN )); then
|
||||
scrape_args+=(--dry-run)
|
||||
fi
|
||||
run_step "run-documents-scrape" "$DOCUMENTS_SCRAPE" "${scrape_args[@]}" || failures=$((failures + 1))
|
||||
if (( DRY_RUN == 0 && failures == 0 )); then
|
||||
audit_targets || failures=$((failures + 1))
|
||||
fi
|
||||
fi
|
||||
|
||||
if (( failures > 0 )); then
|
||||
log_step "Operator validation failed ($failures step(s))."
|
||||
exit 1
|
||||
fi
|
||||
log_step "Operator validation finished successfully."
|
||||
} 2>&1 | tee -a "$LOG_FILE"
|
||||
local pipeline_status=${PIPESTATUS[0]}
|
||||
|
||||
printf 'Log: %s\n' "$LOG_FILE"
|
||||
exit "$pipeline_status"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
59
scripts/tests/run-operator-validation-smoke.sh
Executable file
59
scripts/tests/run-operator-validation-smoke.sh
Executable file
|
|
@ -0,0 +1,59 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
set -Eeuo pipefail
|
||||
|
||||
REPO_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd -P)
|
||||
RUNNER="$REPO_ROOT/scripts/run-operator-validation.sh"
|
||||
TMP_DIR=$(mktemp -d "${TMPDIR:-/tmp}/dce-op-val-smoke.XXXXXX")
|
||||
ARCHIVE_ROOT="$TMP_DIR/archive"
|
||||
CONFIG_PATH="$TMP_DIR/config.json"
|
||||
ENV_PATH="$TMP_DIR/scrape.env"
|
||||
LOG_DIR="$TMP_DIR/logs"
|
||||
FAKE_DOCKER="$TMP_DIR/docker"
|
||||
PATH_BACKUP="$PATH"
|
||||
|
||||
cleanup() {
|
||||
export PATH="$PATH_BACKUP"
|
||||
rm -rf "$TMP_DIR"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
mkdir -p "$ARCHIVE_ROOT/demo" "$LOG_DIR"
|
||||
printf '{"messages":[{"id":"1"}],"channel":{"id":"111111111111111111"}}\n' \
|
||||
>"$ARCHIVE_ROOT/demo/Guild - general [111111111111111111].json"
|
||||
|
||||
cat >"$CONFIG_PATH" <<JSON
|
||||
{
|
||||
"archive_root": "$ARCHIVE_ROOT",
|
||||
"targets": [
|
||||
{
|
||||
"name": "demo",
|
||||
"kind": "guild",
|
||||
"output_dir": "$ARCHIVE_ROOT/demo",
|
||||
"enabled": true
|
||||
}
|
||||
]
|
||||
}
|
||||
JSON
|
||||
|
||||
printf 'DISCORD_TOKEN=dummy\n' >"$ENV_PATH"
|
||||
|
||||
cat >"$FAKE_DOCKER" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
if [[ "${1:-}" == "compose" && "${2:-}" == "version" ]]; then
|
||||
exit 0
|
||||
fi
|
||||
exit 1
|
||||
EOF
|
||||
chmod +x "$FAKE_DOCKER"
|
||||
export PATH="$TMP_DIR:$PATH_BACKUP"
|
||||
|
||||
DCE_REPO_ROOT="$REPO_ROOT" DCE_CONFIG_FILE="$CONFIG_PATH" DCE_ENV_FILE="$ENV_PATH" DCE_LOG_DIR="$LOG_DIR" \
|
||||
"$RUNNER" --dry-run --config "$CONFIG_PATH" --log-file "$LOG_DIR/validation.log"
|
||||
|
||||
grep -q 'Operator validation finished successfully' "$LOG_DIR/validation.log" || {
|
||||
printf 'ERROR: validation log missing success marker\n' >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
printf 'run-operator-validation-smoke: ok\n'
|
||||
Loading…
Reference in a new issue