mirror of
https://github.com/Tyrrrz/DiscordChatExporter.git
synced 2026-06-09 15:52:37 -06:00
feat(scrape): shared KotOR catch-up hint lib (plan 086)
Extract kotor-catchup-hint.sh for operator-handoff, verify-operator-ready, and bootstrap; extend smokes and scrape.env.example.
This commit is contained in:
parent
79129433c7
commit
e1667c6cf0
47
docs/plans/2026-06-04-086-feat-kotor-hint-shared-lib-plan.md
Normal file
47
docs/plans/2026-06-04-086-feat-kotor-hint-shared-lib-plan.md
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
---
|
||||
title: "feat: shared KotOR catch-up hint lib"
|
||||
type: feat
|
||||
status: complete
|
||||
date: 2026-06-04
|
||||
origin: /lfg — plan 085 added handoff hint; verify-operator-ready and bootstrap still omit KotOR wrapper
|
||||
---
|
||||
|
||||
# feat: shared KotOR catch-up hint lib
|
||||
|
||||
## Summary
|
||||
|
||||
Extract `scripts/lib/kotor-catchup-hint.sh` from operator-handoff and call it from `verify-operator-ready.sh` and `bootstrap-recurring-scrape.sh` when `KotOR_discord_msgs` is enabled.
|
||||
|
||||
## Requirements
|
||||
|
||||
| ID | Requirement |
|
||||
|----|-------------|
|
||||
| R1 | Shared lib prints wrapper + summary inspect lines |
|
||||
| R2 | `operator-handoff.sh` sources lib (no duplicated jq logic) |
|
||||
| R3 | `verify-operator-ready.sh` prints hint after "Operator ready. Next:" |
|
||||
| R4 | `bootstrap-recurring-scrape.sh` prints hint after dry-run and live bootstrap complete |
|
||||
| R5 | `scrape.env.example` cites `run-kotor-yes-general-catchup.sh` for yes_general |
|
||||
| R6 | Smokes assert hint from verify + bootstrap KotOR fixtures |
|
||||
| R7 | `DCE_MIN_FREE_MB=0 ./scripts/run-all-smokes.sh` → 24/24 |
|
||||
|
||||
## Implementation Units
|
||||
|
||||
### U1. Lib + script wiring
|
||||
|
||||
**Files:** `scripts/lib/kotor-catchup-hint.sh`, `scripts/operator-handoff.sh`, `scripts/verify-operator-ready.sh`, `scripts/bootstrap-recurring-scrape.sh`, `scrape.env.example`
|
||||
|
||||
### U2. Smokes + docs
|
||||
|
||||
**Files:** `scripts/tests/verify-operator-ready-smoke.sh`, `scripts/tests/bootstrap-recurring-scrape-smoke.sh`, `docs/recurring-scrape-merge-readiness.md`
|
||||
|
||||
## Verification
|
||||
|
||||
```bash
|
||||
DCE_MIN_FREE_MB=0 ./scripts/run-all-smokes.sh
|
||||
```
|
||||
|
||||
## Scope Boundaries
|
||||
|
||||
### Deferred
|
||||
|
||||
- Live KotOR catch-up on operator host
|
||||
|
|
@ -208,6 +208,8 @@ DCE_MIN_FREE_MB=0 ./scripts/run-operator-validation.sh \
|
|||
|
||||
**Plan 085 (2026-06-04):** `operator-handoff.sh` prints KotOR catch-up hint when `KotOR_discord_msgs` is enabled.
|
||||
|
||||
**Plan 086 (2026-06-04):** Shared `lib/kotor-catchup-hint.sh`; verify-operator-ready + bootstrap print same hint.
|
||||
|
||||
**Disk:** ~65 GiB free on `/home` (2026-05-30); large channel merges still need headroom.
|
||||
|
||||
## CI note (fork PRs)
|
||||
|
|
|
|||
|
|
@ -20,8 +20,8 @@ DCE_GID=1000
|
|||
DCE_USERNS_MODE=
|
||||
|
||||
# Optional: raise scrape container memory for multi-year channel catch-up (yes_general, etc.).
|
||||
# KotOR yes_general one-command path: ./scripts/run-kotor-yes-general-catchup.sh
|
||||
# Examples: 8g, 8192m. Default 0 = no compose memory cap.
|
||||
# Optional: raise container memory for large multi-year channel catch-up (compose mem_limit).
|
||||
# Per-target: set container_memory on a target in config/scrape-targets.json (single --target runs).
|
||||
# Global override (wins over config): uncomment below.
|
||||
# DCE_CONTAINER_MEMORY=8g
|
||||
|
|
|
|||
|
|
@ -10,6 +10,8 @@ COMPOSE_FILE="${DCE_COMPOSE_FILE:-$REPO_ROOT/docker-compose.yml}"
|
|||
HOST_RUNNER="$REPO_ROOT/scripts/run-discord-scrape-host.sh"
|
||||
VERIFY_SCRIPT="$REPO_ROOT/scripts/verify-documents-archives.sh"
|
||||
SETUP_AUTH="$REPO_ROOT/scripts/setup-scrape-auth.sh"
|
||||
# shellcheck source=lib/kotor-catchup-hint.sh
|
||||
source "$SCRIPT_DIR/lib/kotor-catchup-hint.sh"
|
||||
|
||||
DRY_RUN=0
|
||||
SKIP_BUILD=0
|
||||
|
|
@ -118,6 +120,7 @@ main() {
|
|||
if (( DRY_RUN == 1 )); then
|
||||
printf 'Dry run complete: archive paths verified under configured output_dir values.\n'
|
||||
printf 'Next: cp scrape.env.example scrape.env, set DISCORD_TOKEN, then rerun without --dry-run.\n'
|
||||
print_kotor_catchup_hint "$CONFIG_PATH"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
|
|
@ -161,6 +164,7 @@ main() {
|
|||
|
||||
printf ' Scrape now: %s\n' "$REPO_ROOT/scripts/run-documents-scrape.sh"
|
||||
printf ' Install cron: %s --dry-run\n' "$REPO_ROOT/scripts/setup-cron.sh"
|
||||
print_kotor_catchup_hint "$CONFIG_PATH"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
|
|
|
|||
19
scripts/lib/kotor-catchup-hint.sh
Normal file
19
scripts/lib/kotor-catchup-hint.sh
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
# Shared post-check hint when KotOR_discord_msgs is enabled in scrape config.
|
||||
|
||||
kotor_catchup_enabled() {
|
||||
local config_path=$1
|
||||
command -v jq >/dev/null 2>&1 || return 1
|
||||
[[ -f "$config_path" ]] || return 1
|
||||
jq -e '.targets[] | select(.name == "KotOR_discord_msgs" and (.enabled // true) == true)' \
|
||||
"$config_path" >/dev/null 2>&1
|
||||
}
|
||||
|
||||
print_kotor_catchup_hint() {
|
||||
local config_path=$1
|
||||
kotor_catchup_enabled "$config_path" || return 0
|
||||
printf '\nKotOR yes_general catch-up (channel 221726893064454144):\n'
|
||||
printf ' ./scripts/run-kotor-yes-general-catchup.sh\n'
|
||||
printf ' ./scripts/print-scrape-summary.sh logs/kotor-yes-general.summary.json\n'
|
||||
}
|
||||
|
|
@ -6,6 +6,8 @@ SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)
|
|||
REPO_ROOT="${DCE_REPO_ROOT:-$(cd "$SCRIPT_DIR/.." && pwd -P)}"
|
||||
# shellcheck source=lib/scrape-run-plan.sh
|
||||
source "$SCRIPT_DIR/lib/scrape-run-plan.sh"
|
||||
# shellcheck source=lib/kotor-catchup-hint.sh
|
||||
source "$SCRIPT_DIR/lib/kotor-catchup-hint.sh"
|
||||
CONFIG_PATH="${DCE_CONFIG_FILE:-$REPO_ROOT/config/scrape-targets.json}"
|
||||
VERIFY_READY="$REPO_ROOT/scripts/verify-operator-ready.sh"
|
||||
DOCUMENTS_SCRAPE="$REPO_ROOT/scripts/run-documents-scrape.sh"
|
||||
|
|
@ -60,19 +62,6 @@ require_command() {
|
|||
command -v "$1" >/dev/null 2>&1 || die "Required command '$1' is missing."
|
||||
}
|
||||
|
||||
kotor_catchup_enabled() {
|
||||
require_command jq
|
||||
jq -e '.targets[] | select(.name == "KotOR_discord_msgs" and (.enabled // true) == true)' \
|
||||
"$CONFIG_PATH" >/dev/null 2>&1
|
||||
}
|
||||
|
||||
print_kotor_catchup_hint() {
|
||||
kotor_catchup_enabled || return 0
|
||||
printf '\nKotOR yes_general catch-up (channel 221726893064454144):\n'
|
||||
printf ' ./scripts/run-kotor-yes-general-catchup.sh\n'
|
||||
printf ' ./scripts/print-scrape-summary.sh logs/kotor-yes-general.summary.json\n'
|
||||
}
|
||||
|
||||
main() {
|
||||
while (($#)); do
|
||||
case "$1" in
|
||||
|
|
@ -154,7 +143,7 @@ main() {
|
|||
if (( ! SALVAGE_ONLY )); then
|
||||
printf ' ./scripts/setup-cron.sh --dry-run\n'
|
||||
fi
|
||||
print_kotor_catchup_hint
|
||||
print_kotor_catchup_hint "$CONFIG_PATH"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
|
|
|
|||
|
|
@ -52,4 +52,38 @@ if [[ "$bootstrap_status" -ne 0 ]] || ! grep -q 'Dry run complete' <<<"$bootstra
|
|||
exit 1
|
||||
fi
|
||||
|
||||
KOTOR_ARCHIVE="$TMP_DIR/archive/kotor"
|
||||
mkdir -p "$KOTOR_ARCHIVE"
|
||||
printf '{"messages":[{"id":"1","timestamp":"2020-01-01T00:00:00+00:00"}],"channel":{"id":"221726893064454144"}}\n' \
|
||||
>"$KOTOR_ARCHIVE/Guild - yes_general [221726893064454144].json"
|
||||
KOTOR_CONFIG="$TMP_DIR/kotor-config.json"
|
||||
cat >"$KOTOR_CONFIG" <<JSON
|
||||
{
|
||||
"archive_root": "$TMP_DIR",
|
||||
"targets": [
|
||||
{
|
||||
"name": "KotOR_discord_msgs",
|
||||
"kind": "guild",
|
||||
"output_dir": "$KOTOR_ARCHIVE",
|
||||
"enabled": true
|
||||
}
|
||||
]
|
||||
}
|
||||
JSON
|
||||
|
||||
set +e
|
||||
kotor_output=$("$BOOTSTRAP" --dry-run --config "$KOTOR_CONFIG" 2>&1)
|
||||
kotor_status=$?
|
||||
set -e
|
||||
|
||||
if [[ "$kotor_status" -ne 0 ]] || ! grep -q 'Dry run complete' <<<"$kotor_output"; then
|
||||
printf 'bootstrap KotOR dry-run failed (status=%s)\n' "$kotor_status" >&2
|
||||
printf '%s\n' "$kotor_output" >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -q 'run-kotor-yes-general-catchup.sh' <<<"$kotor_output" || {
|
||||
printf 'bootstrap dry-run missing KotOR catch-up hint\n' >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
printf 'bootstrap-recurring-scrape-smoke: ok\n'
|
||||
|
|
|
|||
|
|
@ -92,4 +92,33 @@ if DCE_MIN_FREE_MB=0 DCE_REPO_ROOT="$REPO_ROOT" DCE_CONFIG_FILE="$CONFIG_PATH" D
|
|||
exit 1
|
||||
fi
|
||||
|
||||
KOTOR_ARCHIVE="$TMP_DIR/archive/kotor"
|
||||
mkdir -p "$KOTOR_ARCHIVE"
|
||||
printf '{"messages":[{"id":"1"}],"channel":{"id":"221726893064454144"}}\n' \
|
||||
>"$KOTOR_ARCHIVE/Guild - yes_general [221726893064454144].json"
|
||||
KOTOR_CONFIG="$TMP_DIR/kotor-config.json"
|
||||
cat >"$KOTOR_CONFIG" <<JSON
|
||||
{
|
||||
"archive_root": "$ARCHIVE_ROOT",
|
||||
"targets": [
|
||||
{
|
||||
"name": "KotOR_discord_msgs",
|
||||
"kind": "guild",
|
||||
"output_dir": "$KOTOR_ARCHIVE",
|
||||
"enabled": true
|
||||
}
|
||||
]
|
||||
}
|
||||
JSON
|
||||
|
||||
kotor_output=$(
|
||||
DCE_MIN_FREE_MB=0 DCE_REPO_ROOT="$REPO_ROOT" DCE_CONFIG_FILE="$KOTOR_CONFIG" DCE_ENV_FILE="$ENV_PATH" \
|
||||
"$VERIFY" --config "$KOTOR_CONFIG" 2>&1
|
||||
)
|
||||
grep -q 'run-kotor-yes-general-catchup.sh' <<<"$kotor_output" || {
|
||||
printf 'ERROR: verify-operator-ready missing KotOR catch-up hint\n' >&2
|
||||
printf '%s\n' "$kotor_output" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
printf 'verify-operator-ready-smoke: ok\n'
|
||||
|
|
|
|||
|
|
@ -6,6 +6,8 @@ SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)
|
|||
REPO_ROOT="${DCE_REPO_ROOT:-$(cd "$SCRIPT_DIR/.." && pwd -P)}"
|
||||
# shellcheck source=lib/scrape-run-plan.sh
|
||||
source "$SCRIPT_DIR/lib/scrape-run-plan.sh"
|
||||
# shellcheck source=lib/kotor-catchup-hint.sh
|
||||
source "$SCRIPT_DIR/lib/kotor-catchup-hint.sh"
|
||||
CONFIG_PATH="${DCE_CONFIG_FILE:-$REPO_ROOT/config/scrape-targets.json}"
|
||||
ENV_FILE="${DCE_ENV_FILE:-$REPO_ROOT/scrape.env}"
|
||||
HOST_RUNNER="$REPO_ROOT/scripts/run-discord-scrape-host.sh"
|
||||
|
|
@ -197,6 +199,7 @@ main() {
|
|||
printf '\nOperator ready. Next:\n'
|
||||
printf ' ./scripts/run-documents-scrape.sh\n'
|
||||
printf ' ./scripts/setup-cron.sh --dry-run\n'
|
||||
print_kotor_catchup_hint "$CONFIG_PATH"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
|
|
|
|||
Loading…
Reference in a new issue