diff --git a/server/config.js b/server/config.js index 2157010..0ec940b 100644 --- a/server/config.js +++ b/server/config.js @@ -176,6 +176,14 @@ module.exports = { // batches, so no sweep can block the loop regardless of table size. Keep well under // the ~50ms invariant per batch. statusLogPruneBatch: parseInt(process.env.STATUS_LOG_PRUNE_BATCH) || 2000, + // #146 hardening (Item C) — /download/apk GLOBAL guards (NOT per-IP; SNAT collapses + // the fleet to one IP). Concurrency + rate caps + critical-band shed protect the loop + // and IO from a download flood; the aggregate counter makes a flood VISIBLE (the old + // per-IP-per-10min log throttle hid it under SNAT). + otaDownloadMaxConcurrent: parseInt(process.env.OTA_DOWNLOAD_MAX_CONCURRENT) || 10, + otaDownloadMaxPerWindow: parseInt(process.env.OTA_DOWNLOAD_MAX_PER_WINDOW) || 120, + otaDownloadWindowMs: parseInt(process.env.OTA_DOWNLOAD_WINDOW_MS) || 60000, + otaApkRefreshMs: parseInt(process.env.OTA_APK_REFRESH_MS) || 60000, // #146 device_status_log write batching (lib/status-log-writer.js). Status // transitions are buffered and coalesced to the NET state per device per flush, // so a flapping device writes ~1 row/flush instead of a row per transition — diff --git a/server/lib/apk-cache.js b/server/lib/apk-cache.js new file mode 100644 index 0000000..29f409d --- /dev/null +++ b/server/lib/apk-cache.js @@ -0,0 +1,39 @@ +'use strict'; +// #146 hardening (Item C) — cache the OTA APK resolution so no /api/update/check or +// /download/apk does a per-request synchronous filesystem call. The path/size/mtime are +// resolved once at boot and refreshed on an interval (like the frontend-hash refresh), +// so a poll/download flood can't turn into an existsSync/statSync flood on the loop. + +const fs = require('fs'); +const path = require('path'); +const config = require('../config'); + +// A copy under DATA_DIR wins (container operators mount /data/ScreenTinker.apk), +// else the legacy in-repo root path — same order as the old resolveApkPath(). +function candidates() { + return [path.join(config.dataDir, 'ScreenTinker.apk'), path.join(__dirname, '..', '..', 'ScreenTinker.apk')]; +} + +let cache = { path: null, exists: false, size: 0, mtime: 0 }; + +function refresh() { + for (const p of candidates()) { + try { const st = fs.statSync(p); cache = { path: p, exists: true, size: st.size, mtime: st.mtimeMs }; return cache; } catch (_) { /* next */ } + } + cache = { path: null, exists: false, size: 0, mtime: 0 }; + return cache; +} + +function get() { return cache; } + +let timer = null; +function start() { + refresh(); // resolve once at boot + if (!timer) { + timer = setInterval(refresh, config.otaApkRefreshMs); + if (timer.unref) timer.unref(); + } + return cache; +} + +module.exports = { start, refresh, get }; diff --git a/server/lib/ota-download-guard.js b/server/lib/ota-download-guard.js new file mode 100644 index 0000000..ca72d82 --- /dev/null +++ b/server/lib/ota-download-guard.js @@ -0,0 +1,33 @@ +'use strict'; +// #146 Item C — GLOBAL admission control for /download/apk. NOT per-IP: the fleet SNATs +// to one IP, so per-IP would collapse the fleet into one bucket. Concurrency + rate caps +// + critical-band shed protect the loop and IO from a download flood; a per-window +// aggregate makes the flood visible. Pure + testable; mutates the passed rolling state. + +const config = require('../config'); + +// newState() — the single bounded rolling counter the endpoint keeps. +function newState() { return { inFlight: 0, windowStart: 0, windowCount: 0, served: 0, shed: 0 }; } + +// admit(state, band, now) -> { allow, status?, retryAfter?, summary? } +// summary (when a window just rolled) = { served, shed } to log, else null. +// NEVER takes an IP — admission is global by construction. +function admit(state, band, now = Date.now()) { + let summary = null; + if (now - state.windowStart >= config.otaDownloadWindowMs) { + if (state.served || state.shed) summary = { served: state.served, shed: state.shed, inFlight: state.inFlight }; + state.windowStart = now; state.windowCount = 0; state.served = 0; state.shed = 0; + } + const overGlobal = state.inFlight >= config.otaDownloadMaxConcurrent || state.windowCount >= config.otaDownloadMaxPerWindow; + if (band === 'critical' || overGlobal) { + state.shed++; + return { allow: false, status: 503, retryAfter: band === 'critical' ? 30 : 10, summary }; + } + state.inFlight++; state.windowCount++; state.served++; + return { allow: true, summary }; +} + +// release() — call when a served response finishes/closes (once). +function release(state) { state.inFlight = Math.max(0, state.inFlight - 1); } + +module.exports = { newState, admit, release }; diff --git a/server/server.js b/server/server.js index af72b9e..2d44e91 100644 --- a/server/server.js +++ b/server/server.js @@ -583,34 +583,44 @@ const otaBreaker = require('./lib/ota-breaker'); otaBreaker.startSweep(); // #144: periodically evict idle breaker buckets so keyed state stays bounded require('./lib/reconnect-throttle').startSweep(); // #146: same, for the reconnect throttle's per-device buckets require('./lib/flap-limiter').startSweep(); // #146 Item B: evict idle flap-limiter buckets +const apkCache = require('./lib/apk-cache'); +apkCache.start(); // #146 Item C: resolve APK path/size/mtime once + refresh on interval (no per-request fs) +const { getBand } = require('./services/loop-lag'); // #146 Item C: critical-band download shed app.get('/api/update/check', (req, res) => { const currentVersion = req.query.version; const deviceId = req.query.device_id || null; // #144: optional; beta4+ clients send it for per-device keying const latestVersion = VERSION; - // #144: circuit-breaker + phantom-version guard (replaces the old string-inequality - // offer). Keys per device_id when present, else per reported version. Rate-trips a - // looping client in seconds; never offers a downgrade or a superseded/garbage version. + // #144: circuit-breaker + phantom-version guard. Keys per device_id when present, else + // per reported version (NOT IP — SNAT). Rate-trips a looping client in seconds. const verdict = otaBreaker.decide(currentVersion, latestVersion, deviceId); - const apkPath = resolveApkPath(); // existsSync x2 (cheap) - const apkExists = apkPath !== null; - const updateAvailable = !!verdict.update_available && apkExists; // never offer if APK missing - const apkSize = updateAvailable ? fs.statSync(apkPath).size : 0; // statSync only when actually offering (don't stat on every looped poll) - const apkModified = updateAvailable ? fs.statSync(apkPath).mtimeMs : 0; - if (verdict.log) console.log(verdict.log); // once-per-event (trip / unrecognized) - // #96: keep the per-check line observable; now also shows the breaker reason + device_id. - console.log(`[ota] update check from ${getClientIp(req)}: device=${deviceId || 'none'} client=${currentVersion || 'unknown'} latest=${latestVersion} update_available=${updateAvailable} reason=${verdict.reason} apk=${apkExists ? 'present' : 'MISSING'}`); + // #146 Item C: EARLY-RETURN before any filesystem work when we won't serve + // (rate-backoff, up-to-date, phantom, client-newer, …). A looping client that gets + // rate-backoff does ZERO fs calls — the flood can't turn into a statSync flood. + if (!verdict.update_available) { + if (verdict.log) console.log(verdict.log); + logOtaCheck(deviceId, currentVersion, latestVersion, false, verdict.reason); + return res.json({ + latest_version: latestVersion, current_version: currentVersion || 'unknown', + update_available: false, reason: verdict.reason, download_url: '/download/apk', + apk_size: 0, apk_modified: 0, + ...(verdict.retry_after_seconds ? { retry_after_seconds: verdict.retry_after_seconds } : {}), + }); + } + // Offering — read the CACHED apk metadata (no per-request statSync; refreshed on an + // interval by apkCache). Never offer if the APK isn't actually present. + const apk = apkCache.get(); + const updateAvailable = apk.exists; + if (verdict.log) console.log(verdict.log); + logOtaCheck(deviceId, currentVersion, latestVersion, updateAvailable, updateAvailable ? verdict.reason : 'apk-missing'); res.json({ - latest_version: latestVersion, - current_version: currentVersion || 'unknown', - update_available: updateAvailable, - reason: verdict.reason, // #144: breaker decision, for observability (additive; old clients ignore) + latest_version: latestVersion, current_version: currentVersion || 'unknown', + update_available: updateAvailable, reason: updateAvailable ? verdict.reason : 'apk-missing', download_url: '/download/apk', - apk_size: apkSize, - apk_modified: apkModified, - ...(verdict.retry_after_seconds ? { retry_after_seconds: verdict.retry_after_seconds } : {}), + apk_size: updateAvailable ? apk.size : 0, + apk_modified: updateAvailable ? apk.mtime : 0, }); }); @@ -715,40 +725,40 @@ app.post('/api/provision/pair', requireAuth, resolveTenancy, checkDeviceLimit, ( res.json(updated); }); -// Resolve the OTA APK. A copy under the data dir (DATA_DIR) wins, so a container -// operator can mount one at /data/ScreenTinker.apk; otherwise the legacy in-repo -// root path (unchanged when DATA_DIR is unset). Returns null if neither exists. -function resolveApkPath() { - for (const p of [path.join(config.dataDir, 'ScreenTinker.apk'), path.join(__dirname, '..', 'ScreenTinker.apk')]) { - if (fs.existsSync(p)) return p; - } - return null; +// #146 Item C: OTA update-check log. One line per check today; Item E coalesces the +// high-frequency lines. Never keys on IP for any decision (SNAT). Kept as a helper so +// both the offer and no-offer paths log consistently. +function logOtaCheck(deviceId, client, latest, available, reason) { + console.log(`[ota] update check: device=${deviceId || 'none'} client=${client || 'unknown'} latest=${latest} update_available=${available} reason=${reason}`); } -// #139: a device that can't silently install re-downloads the APK every check cycle. Don't -// word a download as "in progress" (it may be a stuck loop, not progress), and rate-limit the -// line to once per IP per window so a looping device can't flood the log. -const otaDownloadLoggedAt = new Map(); // ip -> last-logged ms -const OTA_DOWNLOAD_LOG_WINDOW_MS = 10 * 60 * 1000; +// #146 Item C: GLOBAL download admission (lib/ota-download-guard) — concurrency + rate +// caps + critical-band shed, NEVER per-IP (SNAT). Single bounded rolling state. +const otaDownloadGuard = require('./lib/ota-download-guard'); +const otaDownloadState = otaDownloadGuard.newState(); -// Serve APK download app.get('/download/apk', (req, res) => { - const apkPath = resolveApkPath(); - if (apkPath) { - const ip = getClientIp(req); - const now = Date.now(); - if (now - (otaDownloadLoggedAt.get(ip) || 0) > OTA_DOWNLOAD_LOG_WINDOW_MS) { - otaDownloadLoggedAt.set(ip, now); - console.log(`[ota] APK served to ${ip} (${fs.statSync(apkPath).size} bytes)`); - } - res.setHeader('Content-Type', 'application/vnd.android.package-archive'); - res.setHeader('Content-Disposition', 'attachment; filename="ScreenTinker.apk"'); - res.setHeader('Cache-Control', 'no-cache'); - res.sendFile(apkPath); - } else { - console.warn(`[ota] APK download requested by ${getClientIp(req)} but no APK is available (404)`); - res.status(404).send(`APK Not Found

APK Not Available

The Android APK has not been compiled yet. To build it from source:

cd android
./gradlew assembleDebug
cp app/build/outputs/apk/debug/app-debug.apk ../ScreenTinker.apk

See the README for full build instructions.

In Docker, mount a built APK at /data/ScreenTinker.apk (the data dir).

Alternatively, use the web player in any browser.

`); + const apk = apkCache.get(); + if (!apk.exists) { + return res.status(404).send(`APK Not Found

APK Not Available

The Android APK has not been compiled yet. In Docker, mount a built APK at /data/ScreenTinker.apk, or use the web player.

`); } + + const verdict = otaDownloadGuard.admit(otaDownloadState, getBand()); + if (verdict.summary) { + console.log(`[ota] downloads last ${Math.round(config.otaDownloadWindowMs / 1000)}s: ${verdict.summary.served} served, ${verdict.summary.shed} shed (in-flight ${verdict.summary.inFlight})`); + } + if (!verdict.allow) { + res.setHeader('Retry-After', String(verdict.retryAfter)); + return res.status(verdict.status).json({ error: 'download capacity reached, retry shortly', retry_after: verdict.retryAfter }); + } + + let released = false; + const release = () => { if (released) return; released = true; otaDownloadGuard.release(otaDownloadState); }; + res.on('finish', release); res.on('close', release); + res.setHeader('Content-Type', 'application/vnd.android.package-archive'); + res.setHeader('Content-Disposition', 'attachment; filename="ScreenTinker.apk"'); + res.setHeader('Cache-Control', 'no-cache'); + res.sendFile(apk.path, (err) => { if (err) release(); }); }); // SPA fallback for app routes. Unmatched /api/ paths return 404 so misrouted diff --git a/server/test/ota-hardening.test.js b/server/test/ota-hardening.test.js new file mode 100644 index 0000000..bf092c3 --- /dev/null +++ b/server/test/ota-hardening.test.js @@ -0,0 +1,70 @@ +'use strict'; + +// #146 hardening (Item C) — OTA under SNAT. Cached APK resolution (no per-request fs) +// + GLOBAL download admission (concurrency + rate + critical-band shed, never per-IP). + +const os = require('node:os'); +const path = require('node:path'); +const fs = require('node:fs'); +const crypto = require('node:crypto'); +process.env.DATA_DIR = path.join(os.tmpdir(), 'st-otahard-' + crypto.randomBytes(4).toString('hex')); +process.env.OTA_DOWNLOAD_MAX_CONCURRENT = '3'; +process.env.OTA_DOWNLOAD_MAX_PER_WINDOW = '5'; +process.env.OTA_DOWNLOAD_WINDOW_MS = '100000'; + +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const guard = require('../lib/ota-download-guard'); +const apkCache = require('../lib/apk-cache'); + +test('apk-cache: get() never touches the filesystem (resolution cached at boot/refresh)', () => { + // seed a fake APK under DATA_DIR and refresh once + const apk = path.join(process.env.DATA_DIR, 'ScreenTinker.apk'); + fs.mkdirSync(process.env.DATA_DIR, { recursive: true }); + fs.writeFileSync(apk, 'FAKEAPKBYTES'); + const c = apkCache.refresh(); + assert.equal(c.exists, true); + assert.equal(c.size, 12); + + // count fs.statSync calls across many get()s -> must be ZERO (a poll/download flood + // can't become a statSync flood). + const realStat = fs.statSync; let calls = 0; + fs.statSync = (...a) => { calls++; return realStat(...a); }; + try { for (let i = 0; i < 1000; i++) apkCache.get(); } finally { fs.statSync = realStat; } + assert.equal(calls, 0, 'get() does no fs; 1000 reads = 0 statSync'); +}); + +test('download guard: global concurrency cap -> 503 (not per-IP)', () => { + const s = guard.newState(); + assert.equal(guard.admit(s, 'normal').allow, true); + assert.equal(guard.admit(s, 'normal').allow, true); + assert.equal(guard.admit(s, 'normal').allow, true); // 3 in-flight = cap + const over = guard.admit(s, 'normal'); + assert.equal(over.allow, false); + assert.equal(over.status, 503); + assert.ok(over.retryAfter > 0); + guard.release(s); // free one slot + assert.equal(guard.admit(s, 'normal').allow, true, 'a freed slot admits again'); +}); + +test('download guard: global per-window rate cap -> 503', () => { + const s = guard.newState(); + for (let i = 0; i < 5; i++) { assert.equal(guard.admit(s, 'normal').allow, true); guard.release(s); } // 5 served this window + const over = guard.admit(s, 'normal'); + assert.equal(over.allow, false, '6th in the window is shed'); + assert.equal(over.status, 503); +}); + +test('download guard: critical band sheds regardless of caps', () => { + const s = guard.newState(); + const v = guard.admit(s, 'critical'); + assert.equal(v.allow, false); + assert.equal(v.retryAfter, 30, 'critical band asks for a longer backoff'); +}); + +test('download guard admission takes NO ip argument (global by construction)', () => { + // admit(state, band, now) — there is no IP parameter, so it cannot key on IP. + assert.equal(guard.admit.length <= 3, true); + const s = guard.newState(); + assert.equal(guard.admit(s, 'normal').allow, true); // works with zero IP context +});