mirror of
https://github.com/Tyrrrz/DiscordChatExporter.git
synced 2026-06-09 15:52:37 -06:00
fix(scrape): skip SIGTERM/SIGINT export aborts like OOM
Stopping validation with kill/Ctrl+C returned exit 143/130 and failed the whole target instead of SKIPPED + preserve partial. Added smoke for exit 143; gitignore .dce-scrape.lock.
This commit is contained in:
parent
b9bb4bbe64
commit
ee62078f5b
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -14,6 +14,7 @@ TestResults/
|
||||||
# Local automation secrets and logs
|
# Local automation secrets and logs
|
||||||
scrape.env
|
scrape.env
|
||||||
.discord-token
|
.discord-token
|
||||||
|
.dce-scrape.lock
|
||||||
agentdecompile_projects/
|
agentdecompile_projects/
|
||||||
logs/
|
logs/
|
||||||
.compound-engineering/*.local.yaml
|
.compound-engineering/*.local.yaml
|
||||||
|
|
|
||||||
37
docs/plans/2026-06-04-047-fix-skip-sigterm-export-plan.md
Normal file
37
docs/plans/2026-06-04-047-fix-skip-sigterm-export-plan.md
Normal file
|
|
@ -0,0 +1,37 @@
|
||||||
|
---
|
||||||
|
title: "fix: Treat SIGTERM/SIGINT export aborts as skippable"
|
||||||
|
type: fix
|
||||||
|
status: complete
|
||||||
|
date: 2026-06-04
|
||||||
|
origin: /lfg — yes_general died with fatal ERROR instead of SKIPPED when validation was stopped
|
||||||
|
---
|
||||||
|
|
||||||
|
# fix: Treat SIGTERM/SIGINT export aborts as skippable
|
||||||
|
|
||||||
|
## Problem
|
||||||
|
|
||||||
|
Killing overlapping validation (`kill` / Ctrl+C) sends SIGTERM/SIGINT to the export CLI. Bash exit codes **143** (128+15) and **130** (128+2) were not in the abort-skip list, so `scrape_target` called `die` and failed the whole target — losing partial temp progress instead of SKIPPED + preserve.
|
||||||
|
|
||||||
|
Observed: `ERROR: Channel 221726893064454144 failed` at 7% progress in `logs/kotor-validation-20260604.log`.
|
||||||
|
|
||||||
|
## Requirements
|
||||||
|
|
||||||
|
| ID | Requirement |
|
||||||
|
|----|-------------|
|
||||||
|
| R1 | `export_channel_incremental` returns skip (2) on exit 130 and 143, same as 134/137/139 |
|
||||||
|
| R2 | Partial temp preserved on those skips when `temp_export` non-empty |
|
||||||
|
| R3 | Offline smoke covers exit 143 skip path |
|
||||||
|
| R4 | `.gitignore` includes `.dce-scrape.lock` |
|
||||||
|
| R5 | `run-all-smokes.sh` passes; docs updated |
|
||||||
|
|
||||||
|
## Verification
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./scripts/tests/run-discord-scrape-smoke.sh
|
||||||
|
DCE_MIN_FREE_MB=0 ./scripts/run-all-smokes.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
## Out of scope
|
||||||
|
|
||||||
|
- Completing yes_general catch-up inside LFG
|
||||||
|
- Container memory tuning
|
||||||
|
|
@ -115,6 +115,8 @@ DCE_MIN_FREE_MB=0 ./scripts/run-operator-validation.sh --sync-gui --per-target -
|
||||||
|
|
||||||
\* Audit failed before plan 045 because truncated partial exports under `.dce-temp/` were scanned as archives. After fix, audit passes while partial temps exist.
|
\* Audit failed before plan 045 because truncated partial exports under `.dce-temp/` were scanned as archives. After fix, audit passes while partial temps exist.
|
||||||
|
|
||||||
|
**Plan 047 (2026-06-04):** Treat SIGTERM (143) and SIGINT (130) export exits as skippable aborts — stopping validation no longer fails the whole target with `ERROR: Channel failed`. `.dce-scrape.lock` gitignored.
|
||||||
|
|
||||||
**Plan 046 (2026-06-04):** `run-discord-scrape-host.sh scrape` holds non-blocking `flock` on `.dce-scrape.lock` so overlapping manual/cron validation cannot spawn twin yes_general exports. Stop duplicate runs before restarting KotOR validation.
|
**Plan 046 (2026-06-04):** `run-discord-scrape-host.sh scrape` holds non-blocking `flock` on `.dce-scrape.lock` so overlapping manual/cron validation cannot spawn twin yes_general exports. Stop duplicate runs before restarting KotOR validation.
|
||||||
|
|
||||||
**Plan 045 (2026-06-04):** `audit-archive-json.sh` and `verify-documents-archives.sh` skip `*/.dce-temp/*` (in-progress partial exports). Salvage run 2026-06-03: 7 merged, 17 unchanged, 3 skipped (+5404 messages); yes_general OOM-skipped with partial temps preserved for next salvage.
|
**Plan 045 (2026-06-04):** `audit-archive-json.sh` and `verify-documents-archives.sh` skip `*/.dce-temp/*` (in-progress partial exports). Salvage run 2026-06-03: 7 merged, 17 unchanged, 3 skipped (+5404 messages); yes_general OOM-skipped with partial temps preserved for next salvage.
|
||||||
|
|
|
||||||
|
|
@ -731,8 +731,8 @@ export_channel_incremental() {
|
||||||
return 0
|
return 0
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# SIGABRT (134), SIGKILL/OOM (137), SIGSEGV (139) — bash reports these when the CLI crashes.
|
# SIGINT (130), SIGTERM (143), SIGABRT (134), SIGKILL/OOM (137), SIGSEGV (139)
|
||||||
if (( export_status == 134 || export_status == 137 || export_status == 139 )); then
|
if (( export_status == 130 || export_status == 143 || export_status == 134 || export_status == 137 || export_status == 139 )); then
|
||||||
log "Skipping channel $channel_id (export process aborted, exit $export_status)."
|
log "Skipping channel $channel_id (export process aborted, exit $export_status)."
|
||||||
[[ -s "$export_log" ]] && cat "$export_log" >&2
|
[[ -s "$export_log" ]] && cat "$export_log" >&2
|
||||||
rm -f "$export_log"
|
rm -f "$export_log"
|
||||||
|
|
|
||||||
|
|
@ -135,6 +135,14 @@ cat >"$CONFIG_PATH" <<JSON
|
||||||
"guild_ids": [],
|
"guild_ids": [],
|
||||||
"guild_name_patterns": []
|
"guild_name_patterns": []
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"name": "skip-sigterm",
|
||||||
|
"kind": "guild",
|
||||||
|
"output_dir": "$ARCHIVE_ROOT/skip-sigterm",
|
||||||
|
"channel_ids": ["111", "143"],
|
||||||
|
"guild_ids": [],
|
||||||
|
"guild_name_patterns": []
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"name": "salvage-stale",
|
"name": "salvage-stale",
|
||||||
"kind": "guild",
|
"kind": "guild",
|
||||||
|
|
@ -199,6 +207,12 @@ case "$subcommand" in
|
||||||
exit 134
|
exit 134
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
if [[ "$channel" == "143" ]]; then
|
||||||
|
cp "$fixture_dir/salvage-truncated.json" "$output"
|
||||||
|
echo "Terminated" >&2
|
||||||
|
exit 143
|
||||||
|
fi
|
||||||
|
|
||||||
case "$mode" in
|
case "$mode" in
|
||||||
initial) cp "$fixture_dir/append-existing.json" "$output" ;;
|
initial) cp "$fixture_dir/append-existing.json" "$output" ;;
|
||||||
append) cp "$fixture_dir/append-incremental.json" "$output" ;;
|
append) cp "$fixture_dir/append-incremental.json" "$output" ;;
|
||||||
|
|
@ -392,6 +406,16 @@ partial_temp_dirs=( "$ARCHIVE_ROOT/skip-abort/.dce-temp"/export.134.* )
|
||||||
[[ -d "${partial_temp_dirs[0]}" ]] || { echo "expected partial temp dir preserved for channel 134" >&2; exit 1; }
|
[[ -d "${partial_temp_dirs[0]}" ]] || { echo "expected partial temp dir preserved for channel 134" >&2; exit 1; }
|
||||||
[[ -s "${partial_temp_dirs[0]}/export.json" ]] || { echo "expected partial export.json preserved for channel 134" >&2; exit 1; }
|
[[ -s "${partial_temp_dirs[0]}/export.json" ]] || { echo "expected partial export.json preserved for channel 134" >&2; exit 1; }
|
||||||
|
|
||||||
|
mkdir -p "$ARCHIVE_ROOT/skip-sigterm"
|
||||||
|
cp "$FIXTURE_DIR/append-existing.json" "$ARCHIVE_ROOT/skip-sigterm/$DEFAULT_FILE_NAME"
|
||||||
|
SKIP_SIGTERM_LOG="$TMP_DIR/skip-sigterm.log"
|
||||||
|
run_wrapper skip-sigterm append 2>"$SKIP_SIGTERM_LOG"
|
||||||
|
SKIP_SIGTERM_DEST="$ARCHIVE_ROOT/skip-sigterm/$DEFAULT_FILE_NAME"
|
||||||
|
[[ "$(jq -r '.messages | length' "$SKIP_SIGTERM_DEST")" == "3" ]] || { echo "expected skip-sigterm to append accessible channel" >&2; exit 1; }
|
||||||
|
grep -q 'SKIPPED.*143' "$SKIP_SIGTERM_LOG" || { echo "expected SKIPPED line for sigterm channel 143" >&2; exit 1; }
|
||||||
|
grep -q 'exit 143' "$SKIP_SIGTERM_LOG" || { echo "expected sigterm exit logged for channel 143" >&2; exit 1; }
|
||||||
|
grep -q 'Preserving partial export temp' "$SKIP_SIGTERM_LOG" || { echo "expected partial temp preserved on sigterm channel 143" >&2; exit 1; }
|
||||||
|
|
||||||
# Salvage stale temp export smoke
|
# Salvage stale temp export smoke
|
||||||
mkdir -p "$ARCHIVE_ROOT/salvage-stale"
|
mkdir -p "$ARCHIVE_ROOT/salvage-stale"
|
||||||
cp "$FIXTURE_DIR/append-existing.json" "$ARCHIVE_ROOT/salvage-stale/$DEFAULT_FILE_NAME"
|
cp "$FIXTURE_DIR/append-existing.json" "$ARCHIVE_ROOT/salvage-stale/$DEFAULT_FILE_NAME"
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue