fix(#146) P1.3: per-feature env kill switches + fallout doc section

Every new subsystem is disable-able via env (flip + restart, no redeploy/bisect):
- FLAP_LIMITER_ENABLED=false -> flap limiter always allows.
- OTA_DOWNLOAD_GUARD_ENABLED=false -> download guard always admits.
- MAINTENANCE_BAND_GATE_ENABLED=false -> interval maintenance ignores band.
- CONNECT_RATE_QUARANTINE_TRIPS=0 -> quarantine off (already; confirmed).
Startup prune is never band-gated regardless. Kill switches table added to the fallout
doc. Tests assert each OFF behaviour.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

docs/146-hardening-fallout.md

@@ -73,6 +73,23 @@ watch, and the measured worst-case blocking cost per hot path.
 | `event_loop_lag` telemetry | synchronous INSERT per sample | buffered, **batch-inserted every 10s** |
 | `device:register` (4s flapper) | full register + `buildPlaylistPayload` every ~4s | refused at the gate (identity resolve + one indexed SELECT), cheap |
 
+## Kill switches (env — disable a subsystem with a flip + restart, no redeploy)
+Every new subsystem is disable-able so a misfire on alpha is neutralized without a code
+change or bisect. All read at process start.
+
+| Subsystem | Env | Disable value | Effect when off |
+|---|---|---|---|
+| Flap limiter | `FLAP_LIMITER_ENABLED` | `false` | `check()` always allows — no connect-frequency limiting |
+| Auto-quarantine | `CONNECT_RATE_QUARANTINE_TRIPS` | `0` | flappers still cool down, but are never quarantined |
+| Download guard | `OTA_DOWNLOAD_GUARD_ENABLED` | `false` | `/download/apk` never sheds (no concurrency/rate/band caps) |
+| Maintenance band-gate | `MAINTENANCE_BAND_GATE_ENABLED` | `false` | interval maintenance runs regardless of loop-lag band |
+| Flap window / cap (tune, not off) | `CONNECT_RATE_MAX`, `CONNECT_RATE_WINDOW_MS` | raise `MAX` | loosen if healthy devices are refused |
+| Download caps (tune) | `OTA_DOWNLOAD_MAX_CONCURRENT`, `OTA_DOWNLOAD_MAX_PER_WINDOW` | raise | loosen the elevated-band caps |
+| Prune batch (tune) | `STATUS_LOG_PRUNE_BATCH` | lower | smaller batches = smaller max block |
+
+Note the startup prune is intentionally NEVER band-gated (it must clear a boot-time
+backlog); `MAINTENANCE_BAND_GATE_ENABLED` only affects the interval run.
+
 ## Interlock note
 Item A ends the prune-induced restart loop; Item B's in-memory flap state now persists
 long enough to bite (it used to be wiped every ~40s by the restart). The two are a pair:

server/config.js

@@ -139,6 +139,7 @@ module.exports = {
   // (device_id -> fingerprint -> device_token -> ONE global anon bucket), NEVER IP
   // (SNAT collapses the fleet into one key). In-memory (persists now that Item A ends
   // the restart loop); bounded by an idle sweep + the single anon bucket.
+  flapLimiterEnabled: process.env.FLAP_LIMITER_ENABLED !== 'false',              // #146 P1.3 kill switch
   connectRateWindowMs: parseInt(process.env.CONNECT_RATE_WINDOW_MS) || 300000,   // 5 min
   connectRateMax: parseInt(process.env.CONNECT_RATE_MAX) || 20,                  // per identity per window
   connectRateAnonMax: parseInt(process.env.CONNECT_RATE_ANON_MAX) || 60,         // the shared global anon bucket, higher (collective)
@@ -178,6 +179,10 @@ module.exports = {
   // batches, so no sweep can block the loop regardless of table size. Keep well under
   // the ~50ms invariant per batch.
   statusLogPruneBatch: parseInt(process.env.STATUS_LOG_PRUNE_BATCH) || 2000,
+  // #146 P1.3 kill switch: when false, interval maintenance runs regardless of loop-lag
+  // band (disables the band-gate that skips maintenance while loaded). Startup prune is
+  // never band-gated regardless.
+  maintenanceBandGateEnabled: process.env.MAINTENANCE_BAND_GATE_ENABLED !== 'false',
   // #146 hardening (Item C) — /download/apk GLOBAL guards (NOT per-IP; SNAT collapses
   // the fleet to one IP). Concurrency + rate caps + critical-band shed protect the loop
   // and IO from a download flood; the aggregate counter makes a flood VISIBLE (the old

server/db/database.js

@@ -774,7 +774,7 @@ const { applyTenantDeleteCascade } = require('../lib/tenant-cascade-migration');
 let _statusPruneRunning = false;
 async function pruneStatusLog(opts = {}) {
   if (_statusPruneRunning) return 0;                  // re-entrancy: work runs once
-  if (opts.bandGate && currentBand() !== 'normal') return 0;
+  if (opts.bandGate && config.maintenanceBandGateEnabled && currentBand() !== 'normal') return 0;
   _statusPruneRunning = true;
   try {
     const batch = config.statusLogPruneBatch;

server/lib/flap-limiter.js

@@ -34,6 +34,7 @@ function maxFor(key) { return key === ANON_KEY ? config.connectRateAnonMax : con
 // reason: 'quarantined' (in-memory time-limited), 'flap-cooldown' (post-trip), 'flap-rate'
 // (the trip edge). `quarantined:true` marks the START of a quarantine (log once).
 function check(key, now = Date.now()) {
+  if (!config.flapLimiterEnabled) return { allow: true };   // #146 P1.3 kill switch
   let s = state.get(key);
   if (!s) { s = { hits: [], blockedUntil: 0, lastSeen: now, trips: 0, tripWinStart: now, quarantinedUntil: 0 }; state.set(key, s); }
   s.lastSeen = now;

server/services/heartbeat.js

@@ -81,7 +81,7 @@ async function prunePlayLogs() {
 let _maintRunning = false;
 async function runMaintenance() {
   if (_maintRunning) return;
-  if (currentBand() !== 'normal') return;
+  if (config.maintenanceBandGateEnabled && currentBand() !== 'normal') return;   // #146 P1.3 kill switch
   _maintRunning = true;
   try {
     await pruneProvisioningDevices();

server/test/kill-switches.test.js

@@ -0,0 +1,67 @@
+'use strict';
+
+// #146 P1.3 — every new subsystem must be disable-able via env WITHOUT a code change, so
+// a misfire on alpha is an env flip + restart (no redeploy). The libs read config.* at
+// CALL time, so these tests flip the resolved config value at runtime (equivalent to the
+// env being set) and assert the OFF behaviour.
+
+const os = require('node:os');
+const path = require('node:path');
+const crypto = require('node:crypto');
+process.env.DATA_DIR = path.join(os.tmpdir(), 'st-kill-' + crypto.randomBytes(4).toString('hex'));
+
+const { test } = require('node:test');
+const assert = require('node:assert/strict');
+const config = require('../config');
+const flap = require('../lib/flap-limiter');
+const guard = require('../lib/ota-download-guard');
+const chunked = require('../lib/chunked-prune');
+const { db, pruneStatusLog } = require('../db/database');
+
+test('FLAP_LIMITER_ENABLED=false -> flap limiter always allows', () => {
+  flap.reset();
+  config.flapLimiterEnabled = false;
+  try { for (let i = 0; i < 1000; i++) assert.equal(flap.check('d:x', i).allow, true); }
+  finally { config.flapLimiterEnabled = true; }
+});
+
+test('OTA_DOWNLOAD_GUARD_ENABLED=false -> download guard always admits (even critical)', () => {
+  config.otaDownloadGuardEnabled = false;
+  try {
+    const s = guard.newState();
+    for (let i = 0; i < 50; i++) assert.equal(guard.admit(s, 'critical').allow, true, 'disabled -> allow even under critical');
+    assert.equal(s.shed, 0);
+  } finally { config.otaDownloadGuardEnabled = true; }
+});
+
+test('CONNECT_RATE_QUARANTINE_TRIPS=0 -> never quarantines (only cools down)', () => {
+  flap.reset();
+  const orig = { trips: config.connectRateQuarantineTrips, max: config.connectRateMax, cd: config.connectRateCooldownMs };
+  config.connectRateQuarantineTrips = 0; config.connectRateMax = 1; config.connectRateCooldownMs = 1;
+  try {
+    let quarantined = false, now = 0;
+    for (let t = 0; t < 20; t++) {              // many trips
+      flap.check('d:q0', now); const r = flap.check('d:q0', now + 1);   // 2 hits > max 1 -> trip
+      if (r.quarantined) quarantined = true;
+      now += 100;                               // past the 1ms cooldown
+    }
+    assert.equal(quarantined, false, 'trips=0 disables quarantine');
+  } finally { Object.assign(config, { connectRateQuarantineTrips: orig.trips, connectRateMax: orig.max, connectRateCooldownMs: orig.cd }); }
+});
+
+test('MAINTENANCE_BAND_GATE_ENABLED=false -> interval prune runs even under load', async () => {
+  db.exec('DELETE FROM device_status_log');
+  const ins = db.prepare("INSERT INTO device_status_log (device_id, status, timestamp) VALUES ('d', 'online', ?)");
+  const oldTs = Math.floor(Date.now() / 1000) - 10 * 86400;   // older than retention
+  for (let i = 0; i < 10; i++) ins.run(oldTs);
+
+  chunked.__setBandForTest(() => 'critical');
+  // sanity: with the band-gate ON, a band-gated run is a no-op under critical
+  assert.equal(await pruneStatusLog({ bandGate: true }), 0, 'gate ON -> skipped while critical');
+
+  config.maintenanceBandGateEnabled = false;
+  try {
+    const deleted = await pruneStatusLog({ bandGate: true });
+    assert.ok(deleted > 0, 'gate OFF -> maintenance runs even under critical');
+  } finally { config.maintenanceBandGateEnabled = true; chunked.__setBandForTest(() => 'normal'); }
+});