mirror of
https://github.com/Tyrrrz/DiscordChatExporter.git
synced 2026-06-09 15:52:37 -06:00
feat(verify): show per-target container_memory in operator checks
Archive verify table adds MEM column; verify-operator-ready lists config target memory when global DCE_CONTAINER_MEMORY is unset.
This commit is contained in:
parent
8ca55f299b
commit
aa85fe50fa
|
|
@ -0,0 +1,42 @@
|
|||
---
|
||||
title: "feat: Show per-target container_memory in verify output"
|
||||
type: feat
|
||||
status: complete
|
||||
date: 2026-06-04
|
||||
origin: /lfg — plan 067 added container_memory to config; verify-operator-ready still only shows global DCE_CONTAINER_MEMORY
|
||||
---
|
||||
|
||||
# feat: Show per-target container_memory in verify output
|
||||
|
||||
## Summary
|
||||
|
||||
Surface optional `container_memory` in `verify-documents-archives.sh` archive table and print config-driven target memory hints in `verify-operator-ready.sh` when global `DCE_CONTAINER_MEMORY` is unset.
|
||||
|
||||
## Requirements
|
||||
|
||||
| ID | Requirement |
|
||||
|----|-------------|
|
||||
| R1 | Archive verify table includes MEM column (`container_memory` or `-`) |
|
||||
| R2 | `verify-operator-ready` lists targets with config memory when global cap unset |
|
||||
| R3 | Global `DCE_CONTAINER_MEMORY` line unchanged when set in scrape.env |
|
||||
| R4 | Smokes assert MEM column and target memory hint |
|
||||
| R5 | `DCE_MIN_FREE_MB=0 ./scripts/run-all-smokes.sh` → 21/21 |
|
||||
|
||||
## Implementation Units
|
||||
|
||||
### U1. Verify scripts
|
||||
|
||||
**Files:** `scripts/verify-documents-archives.sh`, `scripts/verify-operator-ready.sh`, smokes
|
||||
|
||||
## Verification
|
||||
|
||||
```bash
|
||||
DCE_MIN_FREE_MB=0 ./scripts/run-all-smokes.sh
|
||||
```
|
||||
|
||||
## Scope Boundaries
|
||||
|
||||
### Deferred
|
||||
|
||||
- Live KotOR catch-up on host
|
||||
- Structured JSON run logs
|
||||
|
|
@ -162,6 +162,8 @@ DCE_MIN_FREE_MB=0 ./scripts/run-operator-validation.sh \
|
|||
|
||||
**Plan 067 (2026-06-04):** Optional per-target `container_memory` in `scrape-targets.json` (single `--target` runs); `KotOR_discord_msgs` defaults to `8g`.
|
||||
|
||||
**Plan 068 (2026-06-04):** `verify-documents-archives` MEM column and `verify-operator-ready` target memory hints when global cap unset.
|
||||
|
||||
**Disk:** ~65 GiB free on `/home` (2026-05-30); large channel merges still need headroom.
|
||||
|
||||
## CI note (fork PRs)
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ cat >"$CONFIG_PATH" <<JSON
|
|||
"name": "demo",
|
||||
"kind": "guild",
|
||||
"output_dir": "$ARCHIVE_ROOT/demo",
|
||||
"container_memory": "4g",
|
||||
"enabled": true
|
||||
}
|
||||
]
|
||||
|
|
@ -46,6 +47,16 @@ grep -q 'Operator ready' <<<"$mem_output" || {
|
|||
printf '%s\n' "$mem_output" >&2
|
||||
exit 1
|
||||
}
|
||||
grep -q 'target memory: demo → 4g' <<<"$mem_output" || {
|
||||
printf 'ERROR: expected per-target memory hint in verify output\n' >&2
|
||||
printf '%s\n' "$mem_output" >&2
|
||||
exit 1
|
||||
}
|
||||
grep -qE 'demo[[:space:]].*4g' <<<"$mem_output" || {
|
||||
printf 'ERROR: expected MEM column for demo target\n' >&2
|
||||
printf '%s\n' "$mem_output" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
printf 'DISCORD_TOKEN=dummy\nDCE_CONTAINER_MEMORY=8g\n' >"$ENV_PATH"
|
||||
mem_output=$(
|
||||
|
|
@ -57,6 +68,10 @@ grep -q 'container memory: 8g' <<<"$mem_output" || {
|
|||
printf '%s\n' "$mem_output" >&2
|
||||
exit 1
|
||||
}
|
||||
if grep -q 'target memory: demo → 4g' <<<"$mem_output"; then
|
||||
printf 'ERROR: per-target memory hint should be hidden when global cap is set\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
cat >"$FAKE_DOCKER" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
|
|
|
|||
|
|
@ -77,17 +77,18 @@ main() {
|
|||
[[ -n "$archive_root" ]] || die "Config is missing archive_root."
|
||||
|
||||
printf 'Archive root: %s\n\n' "$archive_root"
|
||||
printf '%-28s %-40s %8s %8s %8s %s\n' "TARGET" "OUTPUT_DIR" "JSON" "SEEDED" "MAP" "STATUS"
|
||||
printf '%-28s %-40s %8s %8s %8s %s\n' "------" "----------" "----" "------" "-----" "------"
|
||||
printf '%-28s %-40s %8s %8s %8s %6s %s\n' "TARGET" "OUTPUT_DIR" "JSON" "SEEDED" "MAP" "MEM" "STATUS"
|
||||
printf '%-28s %-40s %8s %8s %8s %6s %s\n' "------" "----------" "----" "------" "-----" "---" "------"
|
||||
|
||||
local target_json name output_dir enabled json_count seeded_count map_count map_file status
|
||||
local target_json name output_dir enabled json_count seeded_count map_count map_file mem status
|
||||
while IFS= read -r target_json; do
|
||||
name=$(jq -r '.name' <<<"$target_json")
|
||||
output_dir=$(jq -r '.output_dir' <<<"$target_json")
|
||||
enabled=$(jq -r 'if has("enabled") then .enabled else true end' <<<"$target_json")
|
||||
|
||||
if [[ "$enabled" == "false" ]]; then
|
||||
printf '%-28s %-40s %8s %8s %8s %s\n' "$name" "$output_dir" "-" "-" "-" "disabled"
|
||||
mem=$(jq -r '.container_memory // "-"' <<<"$target_json")
|
||||
printf '%-28s %-40s %8s %8s %8s %6s %s\n' "$name" "$output_dir" "-" "-" "-" "$mem" "disabled"
|
||||
continue
|
||||
fi
|
||||
|
||||
|
|
@ -115,7 +116,10 @@ main() {
|
|||
fi
|
||||
fi
|
||||
|
||||
printf '%-28s %-40s %8s %8s %8s %s\n' "$name" "$output_dir" "$json_count" "$seeded_count" "$map_count" "$status"
|
||||
mem=$(jq -r '.container_memory // "-"' <<<"$target_json")
|
||||
[[ -n "$mem" && "$mem" != "null" ]] || mem="-"
|
||||
|
||||
printf '%-28s %-40s %8s %8s %8s %6s %s\n' "$name" "$output_dir" "$json_count" "$seeded_count" "$map_count" "$mem" "$status"
|
||||
done < <(jq -c '.targets[]' "$CONFIG_PATH")
|
||||
|
||||
printf '\n'
|
||||
|
|
|
|||
|
|
@ -4,6 +4,8 @@ set -Eeuo pipefail
|
|||
|
||||
SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)
|
||||
REPO_ROOT="${DCE_REPO_ROOT:-$(cd "$SCRIPT_DIR/.." && pwd -P)}"
|
||||
# shellcheck source=lib/scrape-run-plan.sh
|
||||
source "$SCRIPT_DIR/lib/scrape-run-plan.sh"
|
||||
CONFIG_PATH="${DCE_CONFIG_FILE:-$REPO_ROOT/config/scrape-targets.json}"
|
||||
ENV_FILE="${DCE_ENV_FILE:-$REPO_ROOT/scrape.env}"
|
||||
HOST_RUNNER="$REPO_ROOT/scripts/run-discord-scrape-host.sh"
|
||||
|
|
@ -116,6 +118,29 @@ print_container_memory() {
|
|||
printf 'container memory: %s (compose mem_limit)\n' "$mem"
|
||||
}
|
||||
|
||||
print_config_target_memory() {
|
||||
local global_mem="" name mem
|
||||
|
||||
if [[ -f "$ENV_FILE" ]]; then
|
||||
global_mem=$(grep -E '^[[:space:]]*DCE_CONTAINER_MEMORY=' "$ENV_FILE" 2>/dev/null | tail -1 | cut -d= -f2- | tr -d '\r' || true)
|
||||
fi
|
||||
if [[ -z "$global_mem" && -n "${DCE_CONTAINER_MEMORY:-}" ]]; then
|
||||
global_mem="$DCE_CONTAINER_MEMORY"
|
||||
fi
|
||||
global_mem=${global_mem#"${global_mem%%[![:space:]]*}"}
|
||||
global_mem=${global_mem%"${global_mem##*[![:space:]]}"}
|
||||
if [[ -n "$global_mem" && "$global_mem" != "0" ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
while IFS= read -r name; do
|
||||
[[ -n "$name" ]] || continue
|
||||
mem=$(target_container_memory "$CONFIG_PATH" "$name")
|
||||
[[ -n "$mem" && "$mem" != "null" ]] || continue
|
||||
printf 'target memory: %s → %s (single --target scrape)\n' "$name" "$mem"
|
||||
done < <(enabled_target_names "$CONFIG_PATH")
|
||||
}
|
||||
|
||||
main() {
|
||||
while (($#)); do
|
||||
case "$1" in
|
||||
|
|
@ -159,6 +184,7 @@ main() {
|
|||
resolve_compose
|
||||
check_auth
|
||||
print_container_memory
|
||||
print_config_target_memory
|
||||
printf 'config: %s\n\n' "$CONFIG_PATH"
|
||||
|
||||
DCE_PRIMARY_CONFIG="$CONFIG_PATH" "$VERIFY_ARCHIVES" --config "$CONFIG_PATH"
|
||||
|
|
|
|||
Loading…
Reference in a new issue