diff --git a/docs/146-hardening-fallout.md b/docs/146-hardening-fallout.md index 126aed9..a056f99 100644 --- a/docs/146-hardening-fallout.md +++ b/docs/146-hardening-fallout.md @@ -90,6 +90,13 @@ story on its own — no client trust: global cap engaging (only under elevated/critical). - `maintenance.sweepsTotal` — confirms the prune is FIRING on its interval, not stalled (with `deleted`/`ms` for cost). All aggregate-only, cheap in-memory reads. +- **`devices_connected`** — the ALWAYS-ON live-fleet gauge (top-level, next to `loop_lag`, + never gated): devices with a live WS socket THIS INSTANT (from the heartbeat connection + map), NOT `devices.status='online'` (which lags by the offline-timeout). The `debug` + block is now **admin-toggleable** (Admin tab → "Status endpoint" → "Expose /api/status + debug metrics"; persisted in `app_settings`, default follows `STATUS_DEBUG_ENABLED`); + the toggle takes effect on the next status poll with **no restart**, and when off the + `debug` key is omitted entirely while `loop_lag` + `devices_connected` remain. ## Before / after — worst-case synchronous blocking (measured) | Hot path | Before | After (measured) | diff --git a/frontend/js/api.js b/frontend/js/api.js index 6ff0649..990a616 100644 --- a/frontend/js/api.js +++ b/frontend/js/api.js @@ -209,6 +209,9 @@ export const api = { // Instance-level default branding (#15, platform admin). adminGetBranding: () => request('/admin/branding'), adminSetBranding: (data) => request('/admin/branding', { method: 'PUT', body: JSON.stringify(data) }), + // #146: toggle the /api/status debug block exposure (platform-admin only). + adminGetStatusDebug: () => request('/admin/status-debug'), + adminSetStatusDebug: (enabled) => request('/admin/status-debug', { method: 'PUT', body: JSON.stringify({ enabled }) }), // Per-user workspace membership management (platform Users page modal). adminGetUserWorkspaces: (id) => request(`/admin/users/${id}/workspaces`), diff --git a/frontend/js/views/admin.js b/frontend/js/views/admin.js index 19061b4..bd35f95 100644 --- a/frontend/js/views/admin.js +++ b/frontend/js/views/admin.js @@ -92,6 +92,11 @@ export async function render(container) {

${t('admin.system')}

${t('common.loading')}

+ +
+

Status endpoint

+

${t('common.loading')}

+
`; // Add User (#10): platform admin provisions a user into ANY workspace. The @@ -122,6 +127,7 @@ export async function render(container) { loadBranding(); loadPlans(); loadSystem(); + loadStatusDebug(); } @@ -341,6 +347,29 @@ async function loadUsers() { } catch (err) { el.innerHTML = `

${esc(err.message)}

`; } } +// #146: toggle /api/status debug-metrics exposure. Mirrors loadBranding's +// load-then-save pattern; takes effect on the next status poll (no restart). +async function loadStatusDebug() { + const el = document.getElementById('statusDebugForm'); + if (!el) return; + let enabled = false; + try { enabled = (await api.adminGetStatusDebug()).enabled; } + catch (e) { el.innerHTML = `

${esc(e.message || 'Failed to load')}

`; return; } + el.innerHTML = ` + +

Adds internal limiter/prune/OTA counters to the public status endpoint. Off by default.

+ `; + document.getElementById('statusDebugChk').onchange = async (e) => { + const chk = e.target; + chk.disabled = true; + try { await api.adminSetStatusDebug(chk.checked); showToast('Status debug ' + (chk.checked ? 'enabled' : 'disabled'), 'success'); } + catch (err) { showToast(err.message, 'error'); chk.checked = !chk.checked; } + finally { chk.disabled = false; } + }; +} + async function loadPlans() { const el = document.getElementById('plansTable'); try { diff --git a/server/config.js b/server/config.js index 3c799d9..158a60c 100644 --- a/server/config.js +++ b/server/config.js @@ -195,6 +195,9 @@ module.exports = { // #146 observability: rolling window for the /api/status.debug throughput counters, so // "lastWindow" is comparable across subsystems. debugStatsWindowMs: parseInt(process.env.DEBUG_STATS_WINDOW_MS) || 60000, + // #146: env DEFAULT for the /api/status debug block; a persisted app_settings value + // (admin toggle) overrides this once set. Default on (matches prior behavior). + statusDebugEnabled: process.env.STATUS_DEBUG_ENABLED !== 'false', // #146 Item E — coalescing log flush + batched event_loop_lag telemetry. logCoalesceFlushMs: parseInt(process.env.LOG_COALESCE_FLUSH_MS) || 30000, lagFlushMs: parseInt(process.env.LAG_FLUSH_MS) || 10000, diff --git a/server/db/database.js b/server/db/database.js index 3041017..d93f1f4 100644 --- a/server/db/database.js +++ b/server/db/database.js @@ -238,6 +238,9 @@ const migrations = [ // #146: index the provisioning-cleanup predicate so the chunked prune's batch // subquery is an index range, not a full devices scan under a provisioning flood. "CREATE INDEX IF NOT EXISTS idx_devices_provisioning ON devices(status, created_at)", + // #146: minimal global key/value settings for admin-toggleable runtime flags (none + // existed — ai_settings is per-workspace, white_labels is branding). + "CREATE TABLE IF NOT EXISTS app_settings (key TEXT PRIMARY KEY, value TEXT NOT NULL, updated_at INTEGER NOT NULL DEFAULT (strftime('%s','now')))", // #143: operator device kill switch. blocked=1 refuses the device at the first // register gate on its next reconnect (no restart). Hand-settable by direct SQLite: // UPDATE devices SET blocked = 1 WHERE id = ''; (0 to unblock) diff --git a/server/lib/app-settings.js b/server/lib/app-settings.js new file mode 100644 index 0000000..2ca1467 --- /dev/null +++ b/server/lib/app-settings.js @@ -0,0 +1,41 @@ +'use strict'; +// #146 — minimal global key/value settings for admin-toggleable RUNTIME flags. No +// generic settings table existed (ai_settings is per-workspace, white_labels is +// branding), so this adds one (app_settings). Values are CACHED in memory and refreshed +// on write, so a hot path — e.g. /api/status, polled under load — reads a cached boolean, +// never a per-poll DB read. + +const { db } = require('../db/database'); + +const cache = new Map(); // key -> string value +let loaded = false; + +function loadAll() { + cache.clear(); + try { for (const r of db.prepare('SELECT key, value FROM app_settings').all()) cache.set(r.key, r.value); } catch (_) { /* table may not exist yet */ } + loaded = true; +} + +function get(key, dflt) { + if (!loaded) loadAll(); + return cache.has(key) ? cache.get(key) : dflt; +} + +// Persist + refresh the cache so the change takes effect immediately (no restart). +function set(key, value) { + const v = String(value); + db.prepare("INSERT INTO app_settings (key, value, updated_at) VALUES (?, ?, strftime('%s','now')) ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated_at = excluded.updated_at").run(key, v); + cache.set(key, v); + loaded = true; +} + +// Boolean read with an env-default fallback: the PERSISTED value overrides once set, +// else the caller's env default applies. +function getBool(key, envDefault) { + const v = get(key, undefined); + if (v === undefined) return !!envDefault; + return v === 'true' || v === '1'; +} +function setBool(key, value) { set(key, value ? 'true' : 'false'); } + +module.exports = { get, set, getBool, setBool, __reload: loadAll }; diff --git a/server/routes/admin.js b/server/routes/admin.js index f33873b..a8a832e 100644 --- a/server/routes/admin.js +++ b/server/routes/admin.js @@ -359,4 +359,21 @@ router.put('/branding', requirePlatformAdmin, (req, res) => { res.json(platformDefaultRow(db)); }); +// ===================== /api/status debug exposure (#146) ===================== +// Platform-admin only. Toggles whether /api/status includes the internal `debug` block +// (limiter/prune/OTA counters). Persisted in app_settings + cached, so it takes effect +// on the NEXT status poll with no restart. Default follows STATUS_DEBUG_ENABLED env. +const appSettings = require('../lib/app-settings'); +const config = require('../config'); + +router.get('/status-debug', requirePlatformAdmin, (req, res) => { + res.json({ enabled: appSettings.getBool('status_debug_enabled', config.statusDebugEnabled) }); +}); +router.put('/status-debug', requirePlatformAdmin, (req, res) => { + const enabled = !!req.body.enabled; + appSettings.setBool('status_debug_enabled', enabled); // persists + refreshes the cache + logActivity(req.user.id, 'admin_set_status_debug', `enabled: ${enabled}`, null, getClientIp(req), null); + res.json({ enabled }); +}); + module.exports = router; diff --git a/server/routes/status.js b/server/routes/status.js index 0a8c9f2..eca3274 100644 --- a/server/routes/status.js +++ b/server/routes/status.js @@ -14,19 +14,15 @@ const otaBreaker = require('../lib/ota-breaker'); const otaDownloadGuard = require('../lib/ota-download-guard'); const logCoalescer = require('../lib/log-coalescer'); const { getMaintenanceStats } = require('../db/database'); +const heartbeat = require('../services/heartbeat'); +const appSettings = require('../lib/app-settings'); // Public status page router.get('/', (req, res) => { - const totalDevices = db.prepare('SELECT COUNT(*) as count FROM devices').get().count; - const onlineDevices = db.prepare("SELECT COUNT(*) as count FROM devices WHERE status = 'online'").get().count; - const totalContent = db.prepare('SELECT COUNT(*) as count FROM content').get().count; - const totalUsers = db.prepare('SELECT COUNT(*) as count FROM users').get().count; const uptime = process.uptime(); - - // Public status - minimal info only (no user counts, no server internals) const version = VERSION; - res.json({ + const body = { status: 'ok', version, uptime_human: formatUptime(uptime), @@ -34,18 +30,26 @@ router.get('/', (req, res) => { // #142: current event-loop lag snapshot, so site lag is diagnosable from the // health endpoint independent of any throttling. Cheap (in-memory read). loop_lag: loopLag.getLag(), - // #146 P3.8: soak observability — see the limiters biting without grepping logs. - // Aggregate counts only (no device ids / secrets); cheap in-memory reads. - debug: { - // gauges + THROUGHPUT (total + last completed window) so the server tells the - // flapper/flood story on its own — aggregate only, no ids/secrets. + // #146: ALWAYS-ON live-fleet gauge — devices with a live WS socket THIS INSTANT + // (from the heartbeat connection map), NOT devices.status='online' (which lags by + // the offline-timeout). The single most-glanced operational number; never gated. + devices_connected: heartbeat.getConnectedCount(), + }; + + // #146: the debug block is admin-toggleable (app_settings.status_debug_enabled), + // defaulting to the STATUS_DEBUG_ENABLED env behavior. Cheap cached boolean. When off, + // the `debug` key is omitted entirely. Aggregate counts only (no ids/secrets). + if (appSettings.getBool('status_debug_enabled', config.statusDebugEnabled)) { + body.debug = { flap: flapLimiter.stats(), // buckets, quarantined, refused{Total,LastWindow}, quarantineStarts{Total,LastWindow} ota_breaker: otaBreaker.stats(), // rateBackoff{Total,LastWindow} ota_download: otaDownloadGuard.stats(), // inFlight, served/shed ThisWindow + Total maintenance: getMaintenanceStats(), // deleted, ms, at, running, sweepsTotal log_coalescer_buffer: logCoalescer._size(), - }, - }); + }; + } + + res.json(body); }); function formatUptime(seconds) { diff --git a/server/services/heartbeat.js b/server/services/heartbeat.js index b1c9824..c0930bc 100644 --- a/server/services/heartbeat.js +++ b/server/services/heartbeat.js @@ -114,6 +114,13 @@ function getAllConnections() { return deviceConnections; } +// #146: LIVE connected-device count — the set with a live socket THIS INSTANT. Cheap +// in-memory read. Distinct from devices.status='online' (persisted, lags by the +// offline-timeout). Surfaced as /api/status.devices_connected. +function getConnectedCount() { + return deviceConnections.size; +} + // #142: sweep unclaimed provisioning devices older than 24h (imported devices keep a // user_id and are preserved). #146: now async + CHUNKED (rides idx_devices_provisioning) // so a provisioning-junk flood can't delete-cascade a huge batch in one synchronous @@ -137,5 +144,6 @@ module.exports = { removeConnection, getConnection, getAllConnections, + getConnectedCount, pruneProvisioningDevices }; diff --git a/server/test/loop-lag-integration.test.js b/server/test/loop-lag-integration.test.js index f1a363d..b8f8e87 100644 --- a/server/test/loop-lag-integration.test.js +++ b/server/test/loop-lag-integration.test.js @@ -46,6 +46,7 @@ test('/api/status exposes a current loop_lag snapshot', async () => { const r = await fetch(BASE + '/api/status'); const body = await r.json(); assert.ok(body.loop_lag, 'loop_lag present on /api/status'); + assert.equal(typeof body.devices_connected, 'number', 'devices_connected always-on live-fleet gauge'); assert.ok(['normal', 'elevated', 'critical'].includes(body.loop_lag.band), 'band is a valid level'); assert.equal(typeof body.loop_lag.p99_ms, 'number', 'p99_ms is numeric'); assert.equal(typeof body.loop_lag.mean_ms, 'number', 'mean_ms is numeric'); diff --git a/server/test/observability-units.test.js b/server/test/observability-units.test.js new file mode 100644 index 0000000..a530025 --- /dev/null +++ b/server/test/observability-units.test.js @@ -0,0 +1,33 @@ +'use strict'; + +// #146 — unit coverage for the two new primitives behind the /api/status changes. + +const os = require('node:os'); +const path = require('node:path'); +const crypto = require('node:crypto'); +process.env.DATA_DIR = path.join(os.tmpdir(), 'st-obsunit-' + crypto.randomBytes(4).toString('hex')); + +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const heartbeat = require('../services/heartbeat'); +const appSettings = require('../lib/app-settings'); + +test('heartbeat.getConnectedCount reflects the live connection map (not DB status)', () => { + const start = heartbeat.getConnectedCount(); + heartbeat.registerConnection('dev-a', 'sock-a'); + heartbeat.registerConnection('dev-b', 'sock-b'); + assert.equal(heartbeat.getConnectedCount(), start + 2, 'count rises with registered sockets'); + heartbeat.removeConnection('dev-a'); + assert.equal(heartbeat.getConnectedCount(), start + 1, 'count falls when a socket leaves'); + heartbeat.removeConnection('dev-b'); + assert.equal(heartbeat.getConnectedCount(), start); +}); + +test('app-settings: env default until set, then persisted value overrides (cached)', () => { + assert.equal(appSettings.getBool('status_debug_enabled', true), true, 'falls back to env default when unset'); + assert.equal(appSettings.getBool('status_debug_enabled', false), false, 'default honored when unset'); + appSettings.setBool('status_debug_enabled', false); + assert.equal(appSettings.getBool('status_debug_enabled', true), false, 'persisted false overrides the (true) default'); + appSettings.setBool('status_debug_enabled', true); + assert.equal(appSettings.getBool('status_debug_enabled', false), true, 'persisted true overrides the (false) default'); +}); diff --git a/server/test/status-debug-toggle.test.js b/server/test/status-debug-toggle.test.js new file mode 100644 index 0000000..e541ccf --- /dev/null +++ b/server/test/status-debug-toggle.test.js @@ -0,0 +1,87 @@ +'use strict'; + +// #146 — /api/status: always-on live-fleet gauge (devices_connected, from the WS +// connection map) + admin-toggleable debug block. Booted server + JWT + DB access. + +const { test, before, after } = require('node:test'); +const assert = require('node:assert/strict'); +const { spawn } = require('node:child_process'); +const path = require('node:path'); +const os = require('node:os'); +const fs = require('node:fs'); +const crypto = require('node:crypto'); +const Database = require('better-sqlite3'); +const ioClient = require('socket.io-client'); + +const PORT = 3998; +const BASE = `http://127.0.0.1:${PORT}`; +const DATA_DIR = path.join(os.tmpdir(), 'st-statusdbg-' + crypto.randomBytes(4).toString('hex')); +let proc, db; + +before(async () => { + const logFd = fs.openSync(path.join(os.tmpdir(), 'st-statusdbg.log'), 'w'); + proc = spawn('node', ['server.js'], { + cwd: path.join(__dirname, '..'), + env: { ...process.env, DATA_DIR, SELF_HOSTED: 'true', PORT: String(PORT), NODE_ENV: 'test' }, // no STATUS_DEBUG_ENABLED -> default ON + stdio: ['ignore', logFd, logFd], + }); + let up = false; + for (let i = 0; i < 80; i++) { try { const r = await fetch(BASE + '/api/status'); if (r.ok) { up = true; break; } } catch { /* */ } await new Promise(r => setTimeout(r, 250)); } + if (!up) throw new Error('server did not boot'); + db = new Database(path.join(DATA_DIR, 'db', 'remote_display.db')); +}); +after(() => { try { db && db.close(); } catch { /* */ } try { proc.kill('SIGKILL'); } catch { /* */ } }); + +const status = async () => (await fetch(BASE + '/api/status')).json(); +const reg = (o) => ({ method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(o) }); +const put = (tok, o) => ({ method: 'PUT', headers: tok ? { Authorization: 'Bearer ' + tok, 'Content-Type': 'application/json' } : { 'Content-Type': 'application/json' }, body: JSON.stringify(o) }); + +test('devices_connected is always present, numeric, and reflects the LIVE socket map', async () => { + const b = await status(); + assert.equal(typeof b.devices_connected, 'number', 'devices_connected always present + numeric'); + const before = b.devices_connected; + + // open a real device socket -> the connection map (and the count) must move + const s = ioClient(`${BASE}/device`, { transports: ['websocket'], reconnection: false, forceNew: true }); + await new Promise((resolve) => { + s.on('connect', () => s.emit('device:register', { pairing_code: String(crypto.randomInt(100000, 1000000)) })); + s.on('device:registered', resolve); + setTimeout(resolve, 3000); + }); + await new Promise(r => setTimeout(r, 150)); + const during = (await status()).devices_connected; + assert.ok(during >= before + 1, `devices_connected rose with a live socket (${before} -> ${during})`); + try { s.close(); } catch { /* */ } +}); + +test('debug block: present by default (env), and gated by the admin flag', async () => { + // default (no env override) -> ON + let b = await status(); + assert.ok(b.debug, 'debug present by default'); + assert.equal(typeof b.debug.flap.buckets, 'number'); + + // register an admin + a normal user; promote the admin in the DB (role read from DB). + const adminEmail = 'ad' + crypto.randomBytes(4).toString('hex') + '@x.local'; + const userEmail = 'u' + crypto.randomBytes(4).toString('hex') + '@x.local'; + const adminTok = (await (await fetch(BASE + '/api/auth/register', reg({ email: adminEmail, password: 'Passw0rd123' }))).json()).token; + const userTok = (await (await fetch(BASE + '/api/auth/register', reg({ email: userEmail, password: 'Passw0rd123' }))).json()).token; + db.prepare("UPDATE users SET role = 'platform_admin' WHERE email = ?").run(adminEmail); + + // non-admin cannot flip it + assert.equal((await fetch(BASE + '/api/admin/status-debug', put(userTok, { enabled: false }))).status, 403, 'non-admin denied'); + // unauthenticated cannot flip it + assert.equal((await fetch(BASE + '/api/admin/status-debug', put(null, { enabled: false }))).status, 401, 'anon denied'); + + // admin flips OFF -> debug omitted; loop_lag + devices_connected remain + const off = await fetch(BASE + '/api/admin/status-debug', put(adminTok, { enabled: false })); + assert.equal(off.status, 200); + b = await status(); + assert.equal('debug' in b, false, 'debug key omitted entirely when off'); + assert.ok(b.loop_lag, 'loop_lag still present when debug off'); + assert.equal(typeof b.devices_connected, 'number', 'devices_connected still present when debug off'); + + // admin flips ON -> debug back, no restart + assert.equal((await fetch(BASE + '/api/admin/status-debug', put(adminTok, { enabled: true }))).status, 200); + b = await status(); + assert.ok(b.debug, 'debug back on after re-enable, no restart'); +});