From 8dd6491288bbf8b2ec97de0994c4918ce6da5abf Mon Sep 17 00:00:00 2001 From: ScreenTinker Date: Tue, 30 Jun 2026 22:03:00 -0500 Subject: [PATCH] fix(#146) P1.3: per-feature env kill switches + fallout doc section Every new subsystem is disable-able via env (flip + restart, no redeploy/bisect): - FLAP_LIMITER_ENABLED=false -> flap limiter always allows. - OTA_DOWNLOAD_GUARD_ENABLED=false -> download guard always admits. - MAINTENANCE_BAND_GATE_ENABLED=false -> interval maintenance ignores band. - CONNECT_RATE_QUARANTINE_TRIPS=0 -> quarantine off (already; confirmed). Startup prune is never band-gated regardless. Kill switches table added to the fallout doc. Tests assert each OFF behaviour. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/146-hardening-fallout.md | 17 ++++++++ server/config.js | 5 +++ server/db/database.js | 2 +- server/lib/flap-limiter.js | 1 + server/services/heartbeat.js | 2 +- server/test/kill-switches.test.js | 67 +++++++++++++++++++++++++++++++ 6 files changed, 92 insertions(+), 2 deletions(-) create mode 100644 server/test/kill-switches.test.js diff --git a/docs/146-hardening-fallout.md b/docs/146-hardening-fallout.md index 338ff2a..4ce9add 100644 --- a/docs/146-hardening-fallout.md +++ b/docs/146-hardening-fallout.md @@ -73,6 +73,23 @@ watch, and the measured worst-case blocking cost per hot path. | `event_loop_lag` telemetry | synchronous INSERT per sample | buffered, **batch-inserted every 10s** | | `device:register` (4s flapper) | full register + `buildPlaylistPayload` every ~4s | refused at the gate (identity resolve + one indexed SELECT), cheap | +## Kill switches (env — disable a subsystem with a flip + restart, no redeploy) +Every new subsystem is disable-able so a misfire on alpha is neutralized without a code +change or bisect. All read at process start. + +| Subsystem | Env | Disable value | Effect when off | +|---|---|---|---| +| Flap limiter | `FLAP_LIMITER_ENABLED` | `false` | `check()` always allows — no connect-frequency limiting | +| Auto-quarantine | `CONNECT_RATE_QUARANTINE_TRIPS` | `0` | flappers still cool down, but are never quarantined | +| Download guard | `OTA_DOWNLOAD_GUARD_ENABLED` | `false` | `/download/apk` never sheds (no concurrency/rate/band caps) | +| Maintenance band-gate | `MAINTENANCE_BAND_GATE_ENABLED` | `false` | interval maintenance runs regardless of loop-lag band | +| Flap window / cap (tune, not off) | `CONNECT_RATE_MAX`, `CONNECT_RATE_WINDOW_MS` | raise `MAX` | loosen if healthy devices are refused | +| Download caps (tune) | `OTA_DOWNLOAD_MAX_CONCURRENT`, `OTA_DOWNLOAD_MAX_PER_WINDOW` | raise | loosen the elevated-band caps | +| Prune batch (tune) | `STATUS_LOG_PRUNE_BATCH` | lower | smaller batches = smaller max block | + +Note the startup prune is intentionally NEVER band-gated (it must clear a boot-time +backlog); `MAINTENANCE_BAND_GATE_ENABLED` only affects the interval run. + ## Interlock note Item A ends the prune-induced restart loop; Item B's in-memory flap state now persists long enough to bite (it used to be wiped every ~40s by the restart). The two are a pair: diff --git a/server/config.js b/server/config.js index 51d4f6f..b459449 100644 --- a/server/config.js +++ b/server/config.js @@ -139,6 +139,7 @@ module.exports = { // (device_id -> fingerprint -> device_token -> ONE global anon bucket), NEVER IP // (SNAT collapses the fleet into one key). In-memory (persists now that Item A ends // the restart loop); bounded by an idle sweep + the single anon bucket. + flapLimiterEnabled: process.env.FLAP_LIMITER_ENABLED !== 'false', // #146 P1.3 kill switch connectRateWindowMs: parseInt(process.env.CONNECT_RATE_WINDOW_MS) || 300000, // 5 min connectRateMax: parseInt(process.env.CONNECT_RATE_MAX) || 20, // per identity per window connectRateAnonMax: parseInt(process.env.CONNECT_RATE_ANON_MAX) || 60, // the shared global anon bucket, higher (collective) @@ -178,6 +179,10 @@ module.exports = { // batches, so no sweep can block the loop regardless of table size. Keep well under // the ~50ms invariant per batch. statusLogPruneBatch: parseInt(process.env.STATUS_LOG_PRUNE_BATCH) || 2000, + // #146 P1.3 kill switch: when false, interval maintenance runs regardless of loop-lag + // band (disables the band-gate that skips maintenance while loaded). Startup prune is + // never band-gated regardless. + maintenanceBandGateEnabled: process.env.MAINTENANCE_BAND_GATE_ENABLED !== 'false', // #146 hardening (Item C) — /download/apk GLOBAL guards (NOT per-IP; SNAT collapses // the fleet to one IP). Concurrency + rate caps + critical-band shed protect the loop // and IO from a download flood; the aggregate counter makes a flood VISIBLE (the old diff --git a/server/db/database.js b/server/db/database.js index fc1e578..d0e2ccb 100644 --- a/server/db/database.js +++ b/server/db/database.js @@ -774,7 +774,7 @@ const { applyTenantDeleteCascade } = require('../lib/tenant-cascade-migration'); let _statusPruneRunning = false; async function pruneStatusLog(opts = {}) { if (_statusPruneRunning) return 0; // re-entrancy: work runs once - if (opts.bandGate && currentBand() !== 'normal') return 0; + if (opts.bandGate && config.maintenanceBandGateEnabled && currentBand() !== 'normal') return 0; _statusPruneRunning = true; try { const batch = config.statusLogPruneBatch; diff --git a/server/lib/flap-limiter.js b/server/lib/flap-limiter.js index 07b4b31..fc5857f 100644 --- a/server/lib/flap-limiter.js +++ b/server/lib/flap-limiter.js @@ -34,6 +34,7 @@ function maxFor(key) { return key === ANON_KEY ? config.connectRateAnonMax : con // reason: 'quarantined' (in-memory time-limited), 'flap-cooldown' (post-trip), 'flap-rate' // (the trip edge). `quarantined:true` marks the START of a quarantine (log once). function check(key, now = Date.now()) { + if (!config.flapLimiterEnabled) return { allow: true }; // #146 P1.3 kill switch let s = state.get(key); if (!s) { s = { hits: [], blockedUntil: 0, lastSeen: now, trips: 0, tripWinStart: now, quarantinedUntil: 0 }; state.set(key, s); } s.lastSeen = now; diff --git a/server/services/heartbeat.js b/server/services/heartbeat.js index 6d1dc0f..b1c9824 100644 --- a/server/services/heartbeat.js +++ b/server/services/heartbeat.js @@ -81,7 +81,7 @@ async function prunePlayLogs() { let _maintRunning = false; async function runMaintenance() { if (_maintRunning) return; - if (currentBand() !== 'normal') return; + if (config.maintenanceBandGateEnabled && currentBand() !== 'normal') return; // #146 P1.3 kill switch _maintRunning = true; try { await pruneProvisioningDevices(); diff --git a/server/test/kill-switches.test.js b/server/test/kill-switches.test.js new file mode 100644 index 0000000..c4c61db --- /dev/null +++ b/server/test/kill-switches.test.js @@ -0,0 +1,67 @@ +'use strict'; + +// #146 P1.3 — every new subsystem must be disable-able via env WITHOUT a code change, so +// a misfire on alpha is an env flip + restart (no redeploy). The libs read config.* at +// CALL time, so these tests flip the resolved config value at runtime (equivalent to the +// env being set) and assert the OFF behaviour. + +const os = require('node:os'); +const path = require('node:path'); +const crypto = require('node:crypto'); +process.env.DATA_DIR = path.join(os.tmpdir(), 'st-kill-' + crypto.randomBytes(4).toString('hex')); + +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const config = require('../config'); +const flap = require('../lib/flap-limiter'); +const guard = require('../lib/ota-download-guard'); +const chunked = require('../lib/chunked-prune'); +const { db, pruneStatusLog } = require('../db/database'); + +test('FLAP_LIMITER_ENABLED=false -> flap limiter always allows', () => { + flap.reset(); + config.flapLimiterEnabled = false; + try { for (let i = 0; i < 1000; i++) assert.equal(flap.check('d:x', i).allow, true); } + finally { config.flapLimiterEnabled = true; } +}); + +test('OTA_DOWNLOAD_GUARD_ENABLED=false -> download guard always admits (even critical)', () => { + config.otaDownloadGuardEnabled = false; + try { + const s = guard.newState(); + for (let i = 0; i < 50; i++) assert.equal(guard.admit(s, 'critical').allow, true, 'disabled -> allow even under critical'); + assert.equal(s.shed, 0); + } finally { config.otaDownloadGuardEnabled = true; } +}); + +test('CONNECT_RATE_QUARANTINE_TRIPS=0 -> never quarantines (only cools down)', () => { + flap.reset(); + const orig = { trips: config.connectRateQuarantineTrips, max: config.connectRateMax, cd: config.connectRateCooldownMs }; + config.connectRateQuarantineTrips = 0; config.connectRateMax = 1; config.connectRateCooldownMs = 1; + try { + let quarantined = false, now = 0; + for (let t = 0; t < 20; t++) { // many trips + flap.check('d:q0', now); const r = flap.check('d:q0', now + 1); // 2 hits > max 1 -> trip + if (r.quarantined) quarantined = true; + now += 100; // past the 1ms cooldown + } + assert.equal(quarantined, false, 'trips=0 disables quarantine'); + } finally { Object.assign(config, { connectRateQuarantineTrips: orig.trips, connectRateMax: orig.max, connectRateCooldownMs: orig.cd }); } +}); + +test('MAINTENANCE_BAND_GATE_ENABLED=false -> interval prune runs even under load', async () => { + db.exec('DELETE FROM device_status_log'); + const ins = db.prepare("INSERT INTO device_status_log (device_id, status, timestamp) VALUES ('d', 'online', ?)"); + const oldTs = Math.floor(Date.now() / 1000) - 10 * 86400; // older than retention + for (let i = 0; i < 10; i++) ins.run(oldTs); + + chunked.__setBandForTest(() => 'critical'); + // sanity: with the band-gate ON, a band-gated run is a no-op under critical + assert.equal(await pruneStatusLog({ bandGate: true }), 0, 'gate ON -> skipped while critical'); + + config.maintenanceBandGateEnabled = false; + try { + const deleted = await pruneStatusLog({ bandGate: true }); + assert.ok(deleted > 0, 'gate OFF -> maintenance runs even under critical'); + } finally { config.maintenanceBandGateEnabled = true; chunked.__setBandForTest(() => 'normal'); } +});