From e1667c6cf02db992a135fe107aa33913d36777b8 Mon Sep 17 00:00:00 2001 From: Copilot Date: Wed, 3 Jun 2026 12:37:32 -0500 Subject: [PATCH] feat(scrape): shared KotOR catch-up hint lib (plan 086) Extract kotor-catchup-hint.sh for operator-handoff, verify-operator-ready, and bootstrap; extend smokes and scrape.env.example. --- ...-04-086-feat-kotor-hint-shared-lib-plan.md | 47 +++++++++++++++++++ docs/recurring-scrape-merge-readiness.md | 2 + scrape.env.example | 2 +- scripts/bootstrap-recurring-scrape.sh | 4 ++ scripts/lib/kotor-catchup-hint.sh | 19 ++++++++ scripts/operator-handoff.sh | 17 ++----- .../tests/bootstrap-recurring-scrape-smoke.sh | 34 ++++++++++++++ scripts/tests/verify-operator-ready-smoke.sh | 29 ++++++++++++ scripts/verify-operator-ready.sh | 3 ++ 9 files changed, 142 insertions(+), 15 deletions(-) create mode 100644 docs/plans/2026-06-04-086-feat-kotor-hint-shared-lib-plan.md create mode 100644 scripts/lib/kotor-catchup-hint.sh diff --git a/docs/plans/2026-06-04-086-feat-kotor-hint-shared-lib-plan.md b/docs/plans/2026-06-04-086-feat-kotor-hint-shared-lib-plan.md new file mode 100644 index 00000000..2835e29f --- /dev/null +++ b/docs/plans/2026-06-04-086-feat-kotor-hint-shared-lib-plan.md @@ -0,0 +1,47 @@ +--- +title: "feat: shared KotOR catch-up hint lib" +type: feat +status: complete +date: 2026-06-04 +origin: /lfg — plan 085 added handoff hint; verify-operator-ready and bootstrap still omit KotOR wrapper +--- + +# feat: shared KotOR catch-up hint lib + +## Summary + +Extract `scripts/lib/kotor-catchup-hint.sh` from operator-handoff and call it from `verify-operator-ready.sh` and `bootstrap-recurring-scrape.sh` when `KotOR_discord_msgs` is enabled. + +## Requirements + +| ID | Requirement | +|----|-------------| +| R1 | Shared lib prints wrapper + summary inspect lines | +| R2 | `operator-handoff.sh` sources lib (no duplicated jq logic) | +| R3 | `verify-operator-ready.sh` prints hint after "Operator ready. Next:" | +| R4 | `bootstrap-recurring-scrape.sh` prints hint after dry-run and live bootstrap complete | +| R5 | `scrape.env.example` cites `run-kotor-yes-general-catchup.sh` for yes_general | +| R6 | Smokes assert hint from verify + bootstrap KotOR fixtures | +| R7 | `DCE_MIN_FREE_MB=0 ./scripts/run-all-smokes.sh` → 24/24 | + +## Implementation Units + +### U1. Lib + script wiring + +**Files:** `scripts/lib/kotor-catchup-hint.sh`, `scripts/operator-handoff.sh`, `scripts/verify-operator-ready.sh`, `scripts/bootstrap-recurring-scrape.sh`, `scrape.env.example` + +### U2. Smokes + docs + +**Files:** `scripts/tests/verify-operator-ready-smoke.sh`, `scripts/tests/bootstrap-recurring-scrape-smoke.sh`, `docs/recurring-scrape-merge-readiness.md` + +## Verification + +```bash +DCE_MIN_FREE_MB=0 ./scripts/run-all-smokes.sh +``` + +## Scope Boundaries + +### Deferred + +- Live KotOR catch-up on operator host diff --git a/docs/recurring-scrape-merge-readiness.md b/docs/recurring-scrape-merge-readiness.md index a6285c0b..f86f2f40 100644 --- a/docs/recurring-scrape-merge-readiness.md +++ b/docs/recurring-scrape-merge-readiness.md @@ -208,6 +208,8 @@ DCE_MIN_FREE_MB=0 ./scripts/run-operator-validation.sh \ **Plan 085 (2026-06-04):** `operator-handoff.sh` prints KotOR catch-up hint when `KotOR_discord_msgs` is enabled. +**Plan 086 (2026-06-04):** Shared `lib/kotor-catchup-hint.sh`; verify-operator-ready + bootstrap print same hint. + **Disk:** ~65 GiB free on `/home` (2026-05-30); large channel merges still need headroom. ## CI note (fork PRs) diff --git a/scrape.env.example b/scrape.env.example index 5fc89f4f..108c6546 100644 --- a/scrape.env.example +++ b/scrape.env.example @@ -20,8 +20,8 @@ DCE_GID=1000 DCE_USERNS_MODE= # Optional: raise scrape container memory for multi-year channel catch-up (yes_general, etc.). +# KotOR yes_general one-command path: ./scripts/run-kotor-yes-general-catchup.sh # Examples: 8g, 8192m. Default 0 = no compose memory cap. -# Optional: raise container memory for large multi-year channel catch-up (compose mem_limit). # Per-target: set container_memory on a target in config/scrape-targets.json (single --target runs). # Global override (wins over config): uncomment below. # DCE_CONTAINER_MEMORY=8g diff --git a/scripts/bootstrap-recurring-scrape.sh b/scripts/bootstrap-recurring-scrape.sh index 55a42b3e..4f5e79c6 100755 --- a/scripts/bootstrap-recurring-scrape.sh +++ b/scripts/bootstrap-recurring-scrape.sh @@ -10,6 +10,8 @@ COMPOSE_FILE="${DCE_COMPOSE_FILE:-$REPO_ROOT/docker-compose.yml}" HOST_RUNNER="$REPO_ROOT/scripts/run-discord-scrape-host.sh" VERIFY_SCRIPT="$REPO_ROOT/scripts/verify-documents-archives.sh" SETUP_AUTH="$REPO_ROOT/scripts/setup-scrape-auth.sh" +# shellcheck source=lib/kotor-catchup-hint.sh +source "$SCRIPT_DIR/lib/kotor-catchup-hint.sh" DRY_RUN=0 SKIP_BUILD=0 @@ -118,6 +120,7 @@ main() { if (( DRY_RUN == 1 )); then printf 'Dry run complete: archive paths verified under configured output_dir values.\n' printf 'Next: cp scrape.env.example scrape.env, set DISCORD_TOKEN, then rerun without --dry-run.\n' + print_kotor_catchup_hint "$CONFIG_PATH" exit 0 fi @@ -161,6 +164,7 @@ main() { printf ' Scrape now: %s\n' "$REPO_ROOT/scripts/run-documents-scrape.sh" printf ' Install cron: %s --dry-run\n' "$REPO_ROOT/scripts/setup-cron.sh" + print_kotor_catchup_hint "$CONFIG_PATH" } main "$@" diff --git a/scripts/lib/kotor-catchup-hint.sh b/scripts/lib/kotor-catchup-hint.sh new file mode 100644 index 00000000..1189b351 --- /dev/null +++ b/scripts/lib/kotor-catchup-hint.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +# Shared post-check hint when KotOR_discord_msgs is enabled in scrape config. + +kotor_catchup_enabled() { + local config_path=$1 + command -v jq >/dev/null 2>&1 || return 1 + [[ -f "$config_path" ]] || return 1 + jq -e '.targets[] | select(.name == "KotOR_discord_msgs" and (.enabled // true) == true)' \ + "$config_path" >/dev/null 2>&1 +} + +print_kotor_catchup_hint() { + local config_path=$1 + kotor_catchup_enabled "$config_path" || return 0 + printf '\nKotOR yes_general catch-up (channel 221726893064454144):\n' + printf ' ./scripts/run-kotor-yes-general-catchup.sh\n' + printf ' ./scripts/print-scrape-summary.sh logs/kotor-yes-general.summary.json\n' +} diff --git a/scripts/operator-handoff.sh b/scripts/operator-handoff.sh index ef78fcd5..53f3694b 100755 --- a/scripts/operator-handoff.sh +++ b/scripts/operator-handoff.sh @@ -6,6 +6,8 @@ SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P) REPO_ROOT="${DCE_REPO_ROOT:-$(cd "$SCRIPT_DIR/.." && pwd -P)}" # shellcheck source=lib/scrape-run-plan.sh source "$SCRIPT_DIR/lib/scrape-run-plan.sh" +# shellcheck source=lib/kotor-catchup-hint.sh +source "$SCRIPT_DIR/lib/kotor-catchup-hint.sh" CONFIG_PATH="${DCE_CONFIG_FILE:-$REPO_ROOT/config/scrape-targets.json}" VERIFY_READY="$REPO_ROOT/scripts/verify-operator-ready.sh" DOCUMENTS_SCRAPE="$REPO_ROOT/scripts/run-documents-scrape.sh" @@ -60,19 +62,6 @@ require_command() { command -v "$1" >/dev/null 2>&1 || die "Required command '$1' is missing." } -kotor_catchup_enabled() { - require_command jq - jq -e '.targets[] | select(.name == "KotOR_discord_msgs" and (.enabled // true) == true)' \ - "$CONFIG_PATH" >/dev/null 2>&1 -} - -print_kotor_catchup_hint() { - kotor_catchup_enabled || return 0 - printf '\nKotOR yes_general catch-up (channel 221726893064454144):\n' - printf ' ./scripts/run-kotor-yes-general-catchup.sh\n' - printf ' ./scripts/print-scrape-summary.sh logs/kotor-yes-general.summary.json\n' -} - main() { while (($#)); do case "$1" in @@ -154,7 +143,7 @@ main() { if (( ! SALVAGE_ONLY )); then printf ' ./scripts/setup-cron.sh --dry-run\n' fi - print_kotor_catchup_hint + print_kotor_catchup_hint "$CONFIG_PATH" } main "$@" diff --git a/scripts/tests/bootstrap-recurring-scrape-smoke.sh b/scripts/tests/bootstrap-recurring-scrape-smoke.sh index c0daa032..c7b54ae3 100755 --- a/scripts/tests/bootstrap-recurring-scrape-smoke.sh +++ b/scripts/tests/bootstrap-recurring-scrape-smoke.sh @@ -52,4 +52,38 @@ if [[ "$bootstrap_status" -ne 0 ]] || ! grep -q 'Dry run complete' <<<"$bootstra exit 1 fi +KOTOR_ARCHIVE="$TMP_DIR/archive/kotor" +mkdir -p "$KOTOR_ARCHIVE" +printf '{"messages":[{"id":"1","timestamp":"2020-01-01T00:00:00+00:00"}],"channel":{"id":"221726893064454144"}}\n' \ + >"$KOTOR_ARCHIVE/Guild - yes_general [221726893064454144].json" +KOTOR_CONFIG="$TMP_DIR/kotor-config.json" +cat >"$KOTOR_CONFIG" <&1) +kotor_status=$? +set -e + +if [[ "$kotor_status" -ne 0 ]] || ! grep -q 'Dry run complete' <<<"$kotor_output"; then + printf 'bootstrap KotOR dry-run failed (status=%s)\n' "$kotor_status" >&2 + printf '%s\n' "$kotor_output" >&2 + exit 1 +fi +grep -q 'run-kotor-yes-general-catchup.sh' <<<"$kotor_output" || { + printf 'bootstrap dry-run missing KotOR catch-up hint\n' >&2 + exit 1 +} + printf 'bootstrap-recurring-scrape-smoke: ok\n' diff --git a/scripts/tests/verify-operator-ready-smoke.sh b/scripts/tests/verify-operator-ready-smoke.sh index ad1fcc6b..e8423f4d 100755 --- a/scripts/tests/verify-operator-ready-smoke.sh +++ b/scripts/tests/verify-operator-ready-smoke.sh @@ -92,4 +92,33 @@ if DCE_MIN_FREE_MB=0 DCE_REPO_ROOT="$REPO_ROOT" DCE_CONFIG_FILE="$CONFIG_PATH" D exit 1 fi +KOTOR_ARCHIVE="$TMP_DIR/archive/kotor" +mkdir -p "$KOTOR_ARCHIVE" +printf '{"messages":[{"id":"1"}],"channel":{"id":"221726893064454144"}}\n' \ + >"$KOTOR_ARCHIVE/Guild - yes_general [221726893064454144].json" +KOTOR_CONFIG="$TMP_DIR/kotor-config.json" +cat >"$KOTOR_CONFIG" <&1 +) +grep -q 'run-kotor-yes-general-catchup.sh' <<<"$kotor_output" || { + printf 'ERROR: verify-operator-ready missing KotOR catch-up hint\n' >&2 + printf '%s\n' "$kotor_output" >&2 + exit 1 +} + printf 'verify-operator-ready-smoke: ok\n' diff --git a/scripts/verify-operator-ready.sh b/scripts/verify-operator-ready.sh index 11fd0afa..29808b89 100755 --- a/scripts/verify-operator-ready.sh +++ b/scripts/verify-operator-ready.sh @@ -6,6 +6,8 @@ SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P) REPO_ROOT="${DCE_REPO_ROOT:-$(cd "$SCRIPT_DIR/.." && pwd -P)}" # shellcheck source=lib/scrape-run-plan.sh source "$SCRIPT_DIR/lib/scrape-run-plan.sh" +# shellcheck source=lib/kotor-catchup-hint.sh +source "$SCRIPT_DIR/lib/kotor-catchup-hint.sh" CONFIG_PATH="${DCE_CONFIG_FILE:-$REPO_ROOT/config/scrape-targets.json}" ENV_FILE="${DCE_ENV_FILE:-$REPO_ROOT/scrape.env}" HOST_RUNNER="$REPO_ROOT/scripts/run-discord-scrape-host.sh" @@ -197,6 +199,7 @@ main() { printf '\nOperator ready. Next:\n' printf ' ./scripts/run-documents-scrape.sh\n' printf ' ./scripts/setup-cron.sh --dry-run\n' + print_kotor_catchup_hint "$CONFIG_PATH" } main "$@"