diff --git a/server/config.js b/server/config.js index 5311195..51d4f6f 100644 --- a/server/config.js +++ b/server/config.js @@ -182,6 +182,7 @@ module.exports = { // the fleet to one IP). Concurrency + rate caps + critical-band shed protect the loop // and IO from a download flood; the aggregate counter makes a flood VISIBLE (the old // per-IP-per-10min log throttle hid it under SNAT). + otaDownloadGuardEnabled: process.env.OTA_DOWNLOAD_GUARD_ENABLED !== 'false', // #146 P1.3 kill switch otaDownloadMaxConcurrent: parseInt(process.env.OTA_DOWNLOAD_MAX_CONCURRENT) || 10, otaDownloadMaxPerWindow: parseInt(process.env.OTA_DOWNLOAD_MAX_PER_WINDOW) || 120, otaDownloadWindowMs: parseInt(process.env.OTA_DOWNLOAD_WINDOW_MS) || 60000, diff --git a/server/lib/ota-download-guard.js b/server/lib/ota-download-guard.js index ca72d82..e6bb749 100644 --- a/server/lib/ota-download-guard.js +++ b/server/lib/ota-download-guard.js @@ -12,17 +12,29 @@ function newState() { return { inFlight: 0, windowStart: 0, windowCount: 0, serv // admit(state, band, now) -> { allow, status?, retryAfter?, summary? } // summary (when a window just rolled) = { served, shed } to log, else null. // NEVER takes an IP — admission is global by construction. +// +// #146 P1.2: caps are BAND-AWARE — a LOAD-TIME backstop, not a healthy-state limiter. +// - normal : serve FREELY (no concurrency/rate cap) so a coordinated rollout of the +// whole fleet isn't staggered while the server is perfectly healthy. +// - elevated : the configured concurrency/rate caps engage (early backpressure). +// - critical : shed (503) — the real protection. +// Kill switch: OTA_DOWNLOAD_GUARD_ENABLED=false disables everything (always allow). function admit(state, band, now = Date.now()) { let summary = null; if (now - state.windowStart >= config.otaDownloadWindowMs) { if (state.served || state.shed) summary = { served: state.served, shed: state.shed, inFlight: state.inFlight }; state.windowStart = now; state.windowCount = 0; state.served = 0; state.shed = 0; } - const overGlobal = state.inFlight >= config.otaDownloadMaxConcurrent || state.windowCount >= config.otaDownloadMaxPerWindow; - if (band === 'critical' || overGlobal) { - state.shed++; - return { allow: false, status: 503, retryAfter: band === 'critical' ? 30 : 10, summary }; + + if (config.otaDownloadGuardEnabled) { + if (band === 'critical') { state.shed++; return { allow: false, status: 503, retryAfter: 30, summary }; } + if (band === 'elevated') { + const overGlobal = state.inFlight >= config.otaDownloadMaxConcurrent || state.windowCount >= config.otaDownloadMaxPerWindow; + if (overGlobal) { state.shed++; return { allow: false, status: 503, retryAfter: 10, summary }; } + } + // band === 'normal': no cap — serve freely. } + state.inFlight++; state.windowCount++; state.served++; return { allow: true, summary }; } diff --git a/server/test/ota-hardening.test.js b/server/test/ota-hardening.test.js index bf092c3..07a6985 100644 --- a/server/test/ota-hardening.test.js +++ b/server/test/ota-hardening.test.js @@ -34,23 +34,29 @@ test('apk-cache: get() never touches the filesystem (resolution cached at boot/r assert.equal(calls, 0, 'get() does no fs; 1000 reads = 0 statSync'); }); -test('download guard: global concurrency cap -> 503 (not per-IP)', () => { +test('band-aware: NORMAL band serves freely (no cap) — healthy rollout not staggered', () => { const s = guard.newState(); - assert.equal(guard.admit(s, 'normal').allow, true); - assert.equal(guard.admit(s, 'normal').allow, true); - assert.equal(guard.admit(s, 'normal').allow, true); // 3 in-flight = cap - const over = guard.admit(s, 'normal'); + for (let i = 0; i < 50; i++) assert.equal(guard.admit(s, 'normal').allow, true, 'normal band never caps'); + assert.equal(s.shed, 0, 'nothing shed while healthy'); +}); + +test('download guard: ELEVATED band applies the global concurrency cap -> 503 (not per-IP)', () => { + const s = guard.newState(); + assert.equal(guard.admit(s, 'elevated').allow, true); + assert.equal(guard.admit(s, 'elevated').allow, true); + assert.equal(guard.admit(s, 'elevated').allow, true); // 3 in-flight = cap + const over = guard.admit(s, 'elevated'); assert.equal(over.allow, false); assert.equal(over.status, 503); assert.ok(over.retryAfter > 0); - guard.release(s); // free one slot - assert.equal(guard.admit(s, 'normal').allow, true, 'a freed slot admits again'); + guard.release(s); // free one slot + assert.equal(guard.admit(s, 'elevated').allow, true, 'a freed slot admits again'); }); -test('download guard: global per-window rate cap -> 503', () => { +test('download guard: ELEVATED band applies the global per-window rate cap -> 503', () => { const s = guard.newState(); - for (let i = 0; i < 5; i++) { assert.equal(guard.admit(s, 'normal').allow, true); guard.release(s); } // 5 served this window - const over = guard.admit(s, 'normal'); + for (let i = 0; i < 5; i++) { assert.equal(guard.admit(s, 'elevated').allow, true); guard.release(s); } // 5 served this window + const over = guard.admit(s, 'elevated'); assert.equal(over.allow, false, '6th in the window is shed'); assert.equal(over.status, 503); }); diff --git a/server/test/storm-harness.test.js b/server/test/storm-harness.test.js index c2f30ad..76c4eed 100644 --- a/server/test/storm-harness.test.js +++ b/server/test/storm-harness.test.js @@ -53,7 +53,7 @@ test('storm: bloated-table sweep + flapper + OTA flood — loop stays responsive for (let round = 0; round < 300; round++) { for (let i = 0; i < 40; i++) { if (!flap.check('d:storm-flapper').allow) flapRefused++; // one hard flapper (identity-keyed, never IP) - const a = otaGuard.admit(guardState, 'normal'); // global download admission + const a = otaGuard.admit(guardState, 'elevated'); // under load -> band-aware caps engage if (a.allow) { otaServed++; otaGuard.release(guardState); } else otaShed++; } await new Promise((r) => setImmediate(r));