diff --git a/server/db/database.js b/server/db/database.js index d0e2ccb..94a00f0 100644 --- a/server/db/database.js +++ b/server/db/database.js @@ -772,10 +772,13 @@ const { applyTenantDeleteCascade } = require('../lib/tenant-cascade-migration'); // table self-heals on next deploy without a restart. // Rides idx_device_status_log_device_ts(device_id, timestamp). let _statusPruneRunning = false; +let _lastPrune = { deleted: 0, ms: 0, at: 0 }; // #146 P3.8: soak observability +function getMaintenanceStats() { return { ..._lastPrune, running: _statusPruneRunning }; } async function pruneStatusLog(opts = {}) { if (_statusPruneRunning) return 0; // re-entrancy: work runs once if (opts.bandGate && config.maintenanceBandGateEnabled && currentBand() !== 'normal') return 0; _statusPruneRunning = true; + const _t0 = Date.now(); try { const batch = config.statusLogPruneBatch; const cap = config.statusLogMaxRowsPerDevice; @@ -796,6 +799,7 @@ async function pruneStatusLog(opts = {}) { await yieldTick(); // breathe between devices } if (total > 0) console.log(`[status-log] pruned ${total} row(s) (per-device, newest ${cap}/device + ${config.statusLogRetentionDays}d retention, batches of ${batch})`); + _lastPrune = { deleted: total, ms: Date.now() - _t0, at: Math.floor(Date.now() / 1000) }; return total; } catch (_) { return 0; } finally { _statusPruneRunning = false; } } @@ -873,4 +877,4 @@ try { const { verifyAndRepairSchema } = require('../lib/schema-check'); verifyAndRepairSchema(db); -module.exports = { db, pruneTelemetry, pruneScreenshots, pruneStatusLog }; +module.exports = { db, pruneTelemetry, pruneScreenshots, pruneStatusLog, getMaintenanceStats }; diff --git a/server/lib/flap-limiter.js b/server/lib/flap-limiter.js index fc5857f..96e63c4 100644 --- a/server/lib/flap-limiter.js +++ b/server/lib/flap-limiter.js @@ -88,4 +88,10 @@ function startSweep() { function reset() { state.clear(); } // tests function _size() { return state.size; } -module.exports = { check, sweep, startSweep, reset, _size }; +// #146 P3.8: soak observability — bucket count + currently-quarantined count. +function stats(now = Date.now()) { + let quarantined = 0; + for (const [, s] of state) if (now < s.quarantinedUntil) quarantined++; + return { buckets: state.size, quarantined }; +} +module.exports = { check, sweep, startSweep, reset, _size, stats }; diff --git a/server/lib/ota-download-guard.js b/server/lib/ota-download-guard.js index e6bb749..dd5a911 100644 --- a/server/lib/ota-download-guard.js +++ b/server/lib/ota-download-guard.js @@ -42,4 +42,10 @@ function admit(state, band, now = Date.now()) { // release() — call when a served response finishes/closes (once). function release(state) { state.inFlight = Math.max(0, state.inFlight - 1); } -module.exports = { newState, admit, release }; +// #146 P3.8: the production singleton state + a stats view for /api/status. Tests use +// newState() for isolation. +const _prod = newState(); +function prodState() { return _prod; } +function stats() { return { inFlight: _prod.inFlight, servedThisWindow: _prod.served, shedThisWindow: _prod.shed, windowCount: _prod.windowCount }; } + +module.exports = { newState, admit, release, prodState, stats }; diff --git a/server/routes/status.js b/server/routes/status.js index e2c62ea..b207ba8 100644 --- a/server/routes/status.js +++ b/server/routes/status.js @@ -8,6 +8,11 @@ const config = require('../config'); const VERSION = require('../version'); const { PLATFORM_ROLES } = require('../middleware/auth'); const loopLag = require('../services/loop-lag'); +// #146 P3.8: soak observability — internal limiter/maintenance states. +const flapLimiter = require('../lib/flap-limiter'); +const otaDownloadGuard = require('../lib/ota-download-guard'); +const logCoalescer = require('../lib/log-coalescer'); +const { getMaintenanceStats } = require('../db/database'); // Public status page router.get('/', (req, res) => { @@ -28,6 +33,14 @@ router.get('/', (req, res) => { // #142: current event-loop lag snapshot, so site lag is diagnosable from the // health endpoint independent of any throttling. Cheap (in-memory read). loop_lag: loopLag.getLag(), + // #146 P3.8: soak observability — see the limiters biting without grepping logs. + // Aggregate counts only (no device ids / secrets); cheap in-memory reads. + debug: { + flap: flapLimiter.stats(), // { buckets, quarantined } + ota_download: otaDownloadGuard.stats(), // { inFlight, servedThisWindow, shedThisWindow, windowCount } + maintenance: getMaintenanceStats(), // { deleted, ms, at, running } of the last status-log prune + log_coalescer_buffer: logCoalescer._size(), + }, }); }); diff --git a/server/server.js b/server/server.js index 6a2e794..7c34561 100644 --- a/server/server.js +++ b/server/server.js @@ -737,7 +737,7 @@ function logOtaCheck(deviceId, client, latest, available, reason) { // #146 Item C: GLOBAL download admission (lib/ota-download-guard) — concurrency + rate // caps + critical-band shed, NEVER per-IP (SNAT). Single bounded rolling state. const otaDownloadGuard = require('./lib/ota-download-guard'); -const otaDownloadState = otaDownloadGuard.newState(); +const otaDownloadState = otaDownloadGuard.prodState(); // #146 P3.8: shared singleton so /api/status can read stats app.get('/download/apk', (req, res) => { const apk = apkCache.get(); diff --git a/server/test/loop-lag-integration.test.js b/server/test/loop-lag-integration.test.js index 8dbb35a..1ff5fba 100644 --- a/server/test/loop-lag-integration.test.js +++ b/server/test/loop-lag-integration.test.js @@ -49,6 +49,13 @@ test('/api/status exposes a current loop_lag snapshot', async () => { assert.ok(['normal', 'elevated', 'critical'].includes(body.loop_lag.band), 'band is a valid level'); assert.equal(typeof body.loop_lag.p99_ms, 'number', 'p99_ms is numeric'); assert.equal(typeof body.loop_lag.mean_ms, 'number', 'mean_ms is numeric'); + // #146 P3.8: soak observability block + assert.ok(body.debug, 'debug block present'); + assert.equal(typeof body.debug.flap.buckets, 'number', 'flap bucket count exposed'); + assert.equal(typeof body.debug.flap.quarantined, 'number', 'flap quarantine count exposed'); + assert.equal(typeof body.debug.ota_download.inFlight, 'number', 'download in-flight exposed'); + assert.ok('maintenance' in body.debug && typeof body.debug.maintenance.ms === 'number', 'last-prune stats exposed'); + assert.equal(typeof body.debug.log_coalescer_buffer, 'number', 'coalescer buffer size exposed'); }); test('lag samples are persisted AND bounded by retention prune (not unbounded)', async () => {