feat(#146): always-on devices_connected + admin-toggleable /api/status debug block

1. devices_connected (always on, never gated): a top-level /api/status field next to
   loop_lag = LIVE WS socket count from the heartbeat connection map (getConnectedCount),
   NOT devices.status='online' (which lags by the offline-timeout). The single
   most-glanced operational number, so it can't disappear when debug is off. Also dropped
   4 dead per-poll COUNT(*) queries the route computed but never returned.

2. debug block behind an admin flag: new minimal app_settings KV table (none existed;
   ai_settings is per-workspace, white_labels is branding) + lib/app-settings.js (cached,
   refresh-on-write so status polls read a cached boolean, not a DB row).
   routes/status.js includes `debug` ONLY when status_debug_enabled is on (persisted value
   overrides the STATUS_DEBUG_ENABLED env default); when off the key is omitted entirely.

3. Admin toggle: GET/PUT /api/admin/status-debug (requirePlatformAdmin, mirrors the
   branding endpoints) + a checkbox in the Admin tab "Status endpoint" section
   (mirrors the branding checkbox). Takes effect on the next poll, no restart.

Tests: devices_connected always present+numeric and rises with a live socket (booted +
socket.io-client); debug present by default, admin flips OFF -> key omitted (loop_lag +
devices_connected remain) -> ON again, no restart; non-admin 403, anon 401; unit coverage
for getConnectedCount + app-settings default/override. Suite 289/289.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

docs/146-hardening-fallout.md

@@ -90,6 +90,13 @@ story on its own — no client trust:
   global cap engaging (only under elevated/critical).
 - `maintenance.sweepsTotal` — confirms the prune is FIRING on its interval, not stalled
   (with `deleted`/`ms` for cost). All aggregate-only, cheap in-memory reads.
+- **`devices_connected`** — the ALWAYS-ON live-fleet gauge (top-level, next to `loop_lag`,
+  never gated): devices with a live WS socket THIS INSTANT (from the heartbeat connection
+  map), NOT `devices.status='online'` (which lags by the offline-timeout). The `debug`
+  block is now **admin-toggleable** (Admin tab → "Status endpoint" → "Expose /api/status
+  debug metrics"; persisted in `app_settings`, default follows `STATUS_DEBUG_ENABLED`);
+  the toggle takes effect on the next status poll with **no restart**, and when off the
+  `debug` key is omitted entirely while `loop_lag` + `devices_connected` remain.
 
 ## Before / after — worst-case synchronous blocking (measured)
 | Hot path | Before | After (measured) |

frontend/js/api.js

@@ -209,6 +209,9 @@ export const api = {
   // Instance-level default branding (#15, platform admin).
   adminGetBranding: () => request('/admin/branding'),
   adminSetBranding: (data) => request('/admin/branding', { method: 'PUT', body: JSON.stringify(data) }),
+  // #146: toggle the /api/status debug block exposure (platform-admin only).
+  adminGetStatusDebug: () => request('/admin/status-debug'),
+  adminSetStatusDebug: (enabled) => request('/admin/status-debug', { method: 'PUT', body: JSON.stringify({ enabled }) }),
 
   // Per-user workspace membership management (platform Users page modal).
   adminGetUserWorkspaces: (id) => request(`/admin/users/${id}/workspaces`),

frontend/js/views/admin.js

@@ -92,6 +92,11 @@ export async function render(container) {
       <h3>${t('admin.system')}</h3>
       <div id="systemInfo"><p style="color:var(--text-muted)">${t('common.loading')}</p></div>
     </div>
+
+    <div class="settings-section">
+      <h3>Status endpoint</h3>
+      <div id="statusDebugForm"><p style="color:var(--text-muted)">${t('common.loading')}</p></div>
+    </div>
   `;
 
   // Add User (#10): platform admin provisions a user into ANY workspace. The
@@ -122,6 +127,7 @@ export async function render(container) {
   loadBranding();
   loadPlans();
   loadSystem();
+  loadStatusDebug();
 
 }
 
@@ -341,6 +347,29 @@ async function loadUsers() {
   } catch (err) { el.innerHTML = `<p style="color:var(--danger)">${esc(err.message)}</p>`; }
 }
 
+// #146: toggle /api/status debug-metrics exposure. Mirrors loadBranding's
+// load-then-save pattern; takes effect on the next status poll (no restart).
+async function loadStatusDebug() {
+  const el = document.getElementById('statusDebugForm');
+  if (!el) return;
+  let enabled = false;
+  try { enabled = (await api.adminGetStatusDebug()).enabled; }
+  catch (e) { el.innerHTML = `<p style="color:var(--danger)">${esc(e.message || 'Failed to load')}</p>`; return; }
+  el.innerHTML = `
+    <label style="display:flex;align-items:center;gap:8px;font-size:13px;cursor:pointer">
+      <input type="checkbox" id="statusDebugChk" ${enabled ? 'checked' : ''}> Expose /api/status debug metrics
+    </label>
+    <p style="color:var(--text-muted);font-size:12px;margin:4px 0 0 24px">Adds internal limiter/prune/OTA counters to the public status endpoint. Off by default.</p>
+  `;
+  document.getElementById('statusDebugChk').onchange = async (e) => {
+    const chk = e.target;
+    chk.disabled = true;
+    try { await api.adminSetStatusDebug(chk.checked); showToast('Status debug ' + (chk.checked ? 'enabled' : 'disabled'), 'success'); }
+    catch (err) { showToast(err.message, 'error'); chk.checked = !chk.checked; }
+    finally { chk.disabled = false; }
+  };
+}
+
 async function loadPlans() {
   const el = document.getElementById('plansTable');
   try {

server/config.js

@@ -195,6 +195,9 @@ module.exports = {
   // #146 observability: rolling window for the /api/status.debug throughput counters, so
   // "lastWindow" is comparable across subsystems.
   debugStatsWindowMs: parseInt(process.env.DEBUG_STATS_WINDOW_MS) || 60000,
+  // #146: env DEFAULT for the /api/status debug block; a persisted app_settings value
+  // (admin toggle) overrides this once set. Default on (matches prior behavior).
+  statusDebugEnabled: process.env.STATUS_DEBUG_ENABLED !== 'false',
   // #146 Item E — coalescing log flush + batched event_loop_lag telemetry.
   logCoalesceFlushMs: parseInt(process.env.LOG_COALESCE_FLUSH_MS) || 30000,
   lagFlushMs: parseInt(process.env.LAG_FLUSH_MS) || 10000,

server/db/database.js

@@ -238,6 +238,9 @@ const migrations = [
   // #146: index the provisioning-cleanup predicate so the chunked prune's batch
   // subquery is an index range, not a full devices scan under a provisioning flood.
   "CREATE INDEX IF NOT EXISTS idx_devices_provisioning ON devices(status, created_at)",
+  // #146: minimal global key/value settings for admin-toggleable runtime flags (none
+  // existed — ai_settings is per-workspace, white_labels is branding).
+  "CREATE TABLE IF NOT EXISTS app_settings (key TEXT PRIMARY KEY, value TEXT NOT NULL, updated_at INTEGER NOT NULL DEFAULT (strftime('%s','now')))",
   // #143: operator device kill switch. blocked=1 refuses the device at the first
   // register gate on its next reconnect (no restart). Hand-settable by direct SQLite:
   //   UPDATE devices SET blocked = 1 WHERE id = '<device_id>';  (0 to unblock)

server/lib/app-settings.js

@@ -0,0 +1,41 @@
+'use strict';
+// #146 — minimal global key/value settings for admin-toggleable RUNTIME flags. No
+// generic settings table existed (ai_settings is per-workspace, white_labels is
+// branding), so this adds one (app_settings). Values are CACHED in memory and refreshed
+// on write, so a hot path — e.g. /api/status, polled under load — reads a cached boolean,
+// never a per-poll DB read.
+
+const { db } = require('../db/database');
+
+const cache = new Map();   // key -> string value
+let loaded = false;
+
+function loadAll() {
+  cache.clear();
+  try { for (const r of db.prepare('SELECT key, value FROM app_settings').all()) cache.set(r.key, r.value); } catch (_) { /* table may not exist yet */ }
+  loaded = true;
+}
+
+function get(key, dflt) {
+  if (!loaded) loadAll();
+  return cache.has(key) ? cache.get(key) : dflt;
+}
+
+// Persist + refresh the cache so the change takes effect immediately (no restart).
+function set(key, value) {
+  const v = String(value);
+  db.prepare("INSERT INTO app_settings (key, value, updated_at) VALUES (?, ?, strftime('%s','now')) ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated_at = excluded.updated_at").run(key, v);
+  cache.set(key, v);
+  loaded = true;
+}
+
+// Boolean read with an env-default fallback: the PERSISTED value overrides once set,
+// else the caller's env default applies.
+function getBool(key, envDefault) {
+  const v = get(key, undefined);
+  if (v === undefined) return !!envDefault;
+  return v === 'true' || v === '1';
+}
+function setBool(key, value) { set(key, value ? 'true' : 'false'); }
+
+module.exports = { get, set, getBool, setBool, __reload: loadAll };

server/routes/admin.js

@@ -359,4 +359,21 @@ router.put('/branding', requirePlatformAdmin, (req, res) => {
   res.json(platformDefaultRow(db));
 });
 
+// ===================== /api/status debug exposure (#146) =====================
+// Platform-admin only. Toggles whether /api/status includes the internal `debug` block
+// (limiter/prune/OTA counters). Persisted in app_settings + cached, so it takes effect
+// on the NEXT status poll with no restart. Default follows STATUS_DEBUG_ENABLED env.
+const appSettings = require('../lib/app-settings');
+const config = require('../config');
+
+router.get('/status-debug', requirePlatformAdmin, (req, res) => {
+  res.json({ enabled: appSettings.getBool('status_debug_enabled', config.statusDebugEnabled) });
+});
+router.put('/status-debug', requirePlatformAdmin, (req, res) => {
+  const enabled = !!req.body.enabled;
+  appSettings.setBool('status_debug_enabled', enabled);   // persists + refreshes the cache
+  logActivity(req.user.id, 'admin_set_status_debug', `enabled: ${enabled}`, null, getClientIp(req), null);
+  res.json({ enabled });
+});
+
 module.exports = router;

server/routes/status.js

@@ -14,19 +14,15 @@ const otaBreaker = require('../lib/ota-breaker');
 const otaDownloadGuard = require('../lib/ota-download-guard');
 const logCoalescer = require('../lib/log-coalescer');
 const { getMaintenanceStats } = require('../db/database');
+const heartbeat = require('../services/heartbeat');
+const appSettings = require('../lib/app-settings');
 
 // Public status page
 router.get('/', (req, res) => {
-  const totalDevices = db.prepare('SELECT COUNT(*) as count FROM devices').get().count;
-  const onlineDevices = db.prepare("SELECT COUNT(*) as count FROM devices WHERE status = 'online'").get().count;
-  const totalContent = db.prepare('SELECT COUNT(*) as count FROM content').get().count;
-  const totalUsers = db.prepare('SELECT COUNT(*) as count FROM users').get().count;
   const uptime = process.uptime();
-
-  // Public status - minimal info only (no user counts, no server internals)
   const version = VERSION;
 
-  res.json({
+  const body = {
     status: 'ok',
     version,
     uptime_human: formatUptime(uptime),
@@ -34,18 +30,26 @@ router.get('/', (req, res) => {
     // #142: current event-loop lag snapshot, so site lag is diagnosable from the
     // health endpoint independent of any throttling. Cheap (in-memory read).
     loop_lag: loopLag.getLag(),
-    // #146 P3.8: soak observability — see the limiters biting without grepping logs.
-    // Aggregate counts only (no device ids / secrets); cheap in-memory reads.
-    debug: {
-      // gauges + THROUGHPUT (total + last completed window) so the server tells the
-      // flapper/flood story on its own — aggregate only, no ids/secrets.
+    // #146: ALWAYS-ON live-fleet gauge — devices with a live WS socket THIS INSTANT
+    // (from the heartbeat connection map), NOT devices.status='online' (which lags by
+    // the offline-timeout). The single most-glanced operational number; never gated.
+    devices_connected: heartbeat.getConnectedCount(),
+  };
+
+  // #146: the debug block is admin-toggleable (app_settings.status_debug_enabled),
+  // defaulting to the STATUS_DEBUG_ENABLED env behavior. Cheap cached boolean. When off,
+  // the `debug` key is omitted entirely. Aggregate counts only (no ids/secrets).
+  if (appSettings.getBool('status_debug_enabled', config.statusDebugEnabled)) {
+    body.debug = {
       flap: flapLimiter.stats(),                  // buckets, quarantined, refused{Total,LastWindow}, quarantineStarts{Total,LastWindow}
       ota_breaker: otaBreaker.stats(),            // rateBackoff{Total,LastWindow}
       ota_download: otaDownloadGuard.stats(),     // inFlight, served/shed ThisWindow + Total
       maintenance: getMaintenanceStats(),         // deleted, ms, at, running, sweepsTotal
       log_coalescer_buffer: logCoalescer._size(),
-    },
-  });
+    };
+  }
+
+  res.json(body);
 });
 
 function formatUptime(seconds) {

server/services/heartbeat.js

@@ -114,6 +114,13 @@ function getAllConnections() {
   return deviceConnections;
 }
 
+// #146: LIVE connected-device count — the set with a live socket THIS INSTANT. Cheap
+// in-memory read. Distinct from devices.status='online' (persisted, lags by the
+// offline-timeout). Surfaced as /api/status.devices_connected.
+function getConnectedCount() {
+  return deviceConnections.size;
+}
+
 // #142: sweep unclaimed provisioning devices older than 24h (imported devices keep a
 // user_id and are preserved). #146: now async + CHUNKED (rides idx_devices_provisioning)
 // so a provisioning-junk flood can't delete-cascade a huge batch in one synchronous
@@ -137,5 +144,6 @@ module.exports = {
   removeConnection,
   getConnection,
   getAllConnections,
+  getConnectedCount,
   pruneProvisioningDevices
 };

server/test/loop-lag-integration.test.js

@@ -46,6 +46,7 @@ test('/api/status exposes a current loop_lag snapshot', async () => {
   const r = await fetch(BASE + '/api/status');
   const body = await r.json();
   assert.ok(body.loop_lag, 'loop_lag present on /api/status');
+  assert.equal(typeof body.devices_connected, 'number', 'devices_connected always-on live-fleet gauge');
   assert.ok(['normal', 'elevated', 'critical'].includes(body.loop_lag.band), 'band is a valid level');
   assert.equal(typeof body.loop_lag.p99_ms, 'number', 'p99_ms is numeric');
   assert.equal(typeof body.loop_lag.mean_ms, 'number', 'mean_ms is numeric');

server/test/observability-units.test.js

@@ -0,0 +1,33 @@
+'use strict';
+
+// #146 — unit coverage for the two new primitives behind the /api/status changes.
+
+const os = require('node:os');
+const path = require('node:path');
+const crypto = require('node:crypto');
+process.env.DATA_DIR = path.join(os.tmpdir(), 'st-obsunit-' + crypto.randomBytes(4).toString('hex'));
+
+const { test } = require('node:test');
+const assert = require('node:assert/strict');
+const heartbeat = require('../services/heartbeat');
+const appSettings = require('../lib/app-settings');
+
+test('heartbeat.getConnectedCount reflects the live connection map (not DB status)', () => {
+  const start = heartbeat.getConnectedCount();
+  heartbeat.registerConnection('dev-a', 'sock-a');
+  heartbeat.registerConnection('dev-b', 'sock-b');
+  assert.equal(heartbeat.getConnectedCount(), start + 2, 'count rises with registered sockets');
+  heartbeat.removeConnection('dev-a');
+  assert.equal(heartbeat.getConnectedCount(), start + 1, 'count falls when a socket leaves');
+  heartbeat.removeConnection('dev-b');
+  assert.equal(heartbeat.getConnectedCount(), start);
+});
+
+test('app-settings: env default until set, then persisted value overrides (cached)', () => {
+  assert.equal(appSettings.getBool('status_debug_enabled', true), true, 'falls back to env default when unset');
+  assert.equal(appSettings.getBool('status_debug_enabled', false), false, 'default honored when unset');
+  appSettings.setBool('status_debug_enabled', false);
+  assert.equal(appSettings.getBool('status_debug_enabled', true), false, 'persisted false overrides the (true) default');
+  appSettings.setBool('status_debug_enabled', true);
+  assert.equal(appSettings.getBool('status_debug_enabled', false), true, 'persisted true overrides the (false) default');
+});

server/test/status-debug-toggle.test.js

@@ -0,0 +1,87 @@
+'use strict';
+
+// #146 — /api/status: always-on live-fleet gauge (devices_connected, from the WS
+// connection map) + admin-toggleable debug block. Booted server + JWT + DB access.
+
+const { test, before, after } = require('node:test');
+const assert = require('node:assert/strict');
+const { spawn } = require('node:child_process');
+const path = require('node:path');
+const os = require('node:os');
+const fs = require('node:fs');
+const crypto = require('node:crypto');
+const Database = require('better-sqlite3');
+const ioClient = require('socket.io-client');
+
+const PORT = 3998;
+const BASE = `http://127.0.0.1:${PORT}`;
+const DATA_DIR = path.join(os.tmpdir(), 'st-statusdbg-' + crypto.randomBytes(4).toString('hex'));
+let proc, db;
+
+before(async () => {
+  const logFd = fs.openSync(path.join(os.tmpdir(), 'st-statusdbg.log'), 'w');
+  proc = spawn('node', ['server.js'], {
+    cwd: path.join(__dirname, '..'),
+    env: { ...process.env, DATA_DIR, SELF_HOSTED: 'true', PORT: String(PORT), NODE_ENV: 'test' },  // no STATUS_DEBUG_ENABLED -> default ON
+    stdio: ['ignore', logFd, logFd],
+  });
+  let up = false;
+  for (let i = 0; i < 80; i++) { try { const r = await fetch(BASE + '/api/status'); if (r.ok) { up = true; break; } } catch { /* */ } await new Promise(r => setTimeout(r, 250)); }
+  if (!up) throw new Error('server did not boot');
+  db = new Database(path.join(DATA_DIR, 'db', 'remote_display.db'));
+});
+after(() => { try { db && db.close(); } catch { /* */ } try { proc.kill('SIGKILL'); } catch { /* */ } });
+
+const status = async () => (await fetch(BASE + '/api/status')).json();
+const reg = (o) => ({ method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(o) });
+const put = (tok, o) => ({ method: 'PUT', headers: tok ? { Authorization: 'Bearer ' + tok, 'Content-Type': 'application/json' } : { 'Content-Type': 'application/json' }, body: JSON.stringify(o) });
+
+test('devices_connected is always present, numeric, and reflects the LIVE socket map', async () => {
+  const b = await status();
+  assert.equal(typeof b.devices_connected, 'number', 'devices_connected always present + numeric');
+  const before = b.devices_connected;
+
+  // open a real device socket -> the connection map (and the count) must move
+  const s = ioClient(`${BASE}/device`, { transports: ['websocket'], reconnection: false, forceNew: true });
+  await new Promise((resolve) => {
+    s.on('connect', () => s.emit('device:register', { pairing_code: String(crypto.randomInt(100000, 1000000)) }));
+    s.on('device:registered', resolve);
+    setTimeout(resolve, 3000);
+  });
+  await new Promise(r => setTimeout(r, 150));
+  const during = (await status()).devices_connected;
+  assert.ok(during >= before + 1, `devices_connected rose with a live socket (${before} -> ${during})`);
+  try { s.close(); } catch { /* */ }
+});
+
+test('debug block: present by default (env), and gated by the admin flag', async () => {
+  // default (no env override) -> ON
+  let b = await status();
+  assert.ok(b.debug, 'debug present by default');
+  assert.equal(typeof b.debug.flap.buckets, 'number');
+
+  // register an admin + a normal user; promote the admin in the DB (role read from DB).
+  const adminEmail = 'ad' + crypto.randomBytes(4).toString('hex') + '@x.local';
+  const userEmail = 'u' + crypto.randomBytes(4).toString('hex') + '@x.local';
+  const adminTok = (await (await fetch(BASE + '/api/auth/register', reg({ email: adminEmail, password: 'Passw0rd123' }))).json()).token;
+  const userTok = (await (await fetch(BASE + '/api/auth/register', reg({ email: userEmail, password: 'Passw0rd123' }))).json()).token;
+  db.prepare("UPDATE users SET role = 'platform_admin' WHERE email = ?").run(adminEmail);
+
+  // non-admin cannot flip it
+  assert.equal((await fetch(BASE + '/api/admin/status-debug', put(userTok, { enabled: false }))).status, 403, 'non-admin denied');
+  // unauthenticated cannot flip it
+  assert.equal((await fetch(BASE + '/api/admin/status-debug', put(null, { enabled: false }))).status, 401, 'anon denied');
+
+  // admin flips OFF -> debug omitted; loop_lag + devices_connected remain
+  const off = await fetch(BASE + '/api/admin/status-debug', put(adminTok, { enabled: false }));
+  assert.equal(off.status, 200);
+  b = await status();
+  assert.equal('debug' in b, false, 'debug key omitted entirely when off');
+  assert.ok(b.loop_lag, 'loop_lag still present when debug off');
+  assert.equal(typeof b.devices_connected, 'number', 'devices_connected still present when debug off');
+
+  // admin flips ON -> debug back, no restart
+  assert.equal((await fetch(BASE + '/api/admin/status-debug', put(adminTok, { enabled: true }))).status, 200);
+  b = await status();
+  assert.ok(b.debug, 'debug back on after re-enable, no restart');
+});