mirror of
https://github.com/screentinker/screentinker.git
synced 2026-08-14 06:16:20 -06:00
fix(#146) C: OTA hardening under SNAT — no per-request fs, global download caps
The fleet SNATs to one IP, so nothing on the OTA path may key on IP. - /api/update/check: EARLY-RETURN before any filesystem call when the breaker won't offer (rate-backoff / up-to-date / phantom / client-newer). A looping client that gets rate-backoff now does ZERO fs — the flood can't become a statSync flood. - lib/apk-cache.js: resolve APK path/size/mtime once at boot + refresh on an interval; the check/download endpoints read cached metadata (get() does no fs, proven by test). - lib/ota-download-guard.js + /download/apk: GLOBAL concurrency + rate caps + critical- band shed (503 Retry-After), NEVER per-IP. Replaces the per-IP-per-10min log throttle (which hid the flood under SNAT) with a per-window served/shed aggregate so a download flood is VISIBLE. Bounded single rolling-state object; in-flight released on finish/close. - Breaker unchanged; no IP limiting or device_id requirement added (legacy field clients send no device_id on OTA checks — must keep working). Tests: apk-cache get() = 0 statSync over 1000 reads; download guard sheds past global concurrency + per-window rate + critical band; admit() has no IP parameter. Suite 259/259. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
ScreenTinker
parent
9e3222a503
commit
f037dd476a
|
|
@ -176,6 +176,14 @@ module.exports = {
|
|||
// batches, so no sweep can block the loop regardless of table size. Keep well under
|
||||
// the ~50ms invariant per batch.
|
||||
statusLogPruneBatch: parseInt(process.env.STATUS_LOG_PRUNE_BATCH) || 2000,
|
||||
// #146 hardening (Item C) — /download/apk GLOBAL guards (NOT per-IP; SNAT collapses
|
||||
// the fleet to one IP). Concurrency + rate caps + critical-band shed protect the loop
|
||||
// and IO from a download flood; the aggregate counter makes a flood VISIBLE (the old
|
||||
// per-IP-per-10min log throttle hid it under SNAT).
|
||||
otaDownloadMaxConcurrent: parseInt(process.env.OTA_DOWNLOAD_MAX_CONCURRENT) || 10,
|
||||
otaDownloadMaxPerWindow: parseInt(process.env.OTA_DOWNLOAD_MAX_PER_WINDOW) || 120,
|
||||
otaDownloadWindowMs: parseInt(process.env.OTA_DOWNLOAD_WINDOW_MS) || 60000,
|
||||
otaApkRefreshMs: parseInt(process.env.OTA_APK_REFRESH_MS) || 60000,
|
||||
// #146 device_status_log write batching (lib/status-log-writer.js). Status
|
||||
// transitions are buffered and coalesced to the NET state per device per flush,
|
||||
// so a flapping device writes ~1 row/flush instead of a row per transition —
|
||||
|
|
|
|||
39
server/lib/apk-cache.js
Normal file
39
server/lib/apk-cache.js
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
'use strict';
|
||||
// #146 hardening (Item C) — cache the OTA APK resolution so no /api/update/check or
|
||||
// /download/apk does a per-request synchronous filesystem call. The path/size/mtime are
|
||||
// resolved once at boot and refreshed on an interval (like the frontend-hash refresh),
|
||||
// so a poll/download flood can't turn into an existsSync/statSync flood on the loop.
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const config = require('../config');
|
||||
|
||||
// A copy under DATA_DIR wins (container operators mount /data/ScreenTinker.apk),
|
||||
// else the legacy in-repo root path — same order as the old resolveApkPath().
|
||||
function candidates() {
|
||||
return [path.join(config.dataDir, 'ScreenTinker.apk'), path.join(__dirname, '..', '..', 'ScreenTinker.apk')];
|
||||
}
|
||||
|
||||
let cache = { path: null, exists: false, size: 0, mtime: 0 };
|
||||
|
||||
function refresh() {
|
||||
for (const p of candidates()) {
|
||||
try { const st = fs.statSync(p); cache = { path: p, exists: true, size: st.size, mtime: st.mtimeMs }; return cache; } catch (_) { /* next */ }
|
||||
}
|
||||
cache = { path: null, exists: false, size: 0, mtime: 0 };
|
||||
return cache;
|
||||
}
|
||||
|
||||
function get() { return cache; }
|
||||
|
||||
let timer = null;
|
||||
function start() {
|
||||
refresh(); // resolve once at boot
|
||||
if (!timer) {
|
||||
timer = setInterval(refresh, config.otaApkRefreshMs);
|
||||
if (timer.unref) timer.unref();
|
||||
}
|
||||
return cache;
|
||||
}
|
||||
|
||||
module.exports = { start, refresh, get };
|
||||
33
server/lib/ota-download-guard.js
Normal file
33
server/lib/ota-download-guard.js
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
'use strict';
|
||||
// #146 Item C — GLOBAL admission control for /download/apk. NOT per-IP: the fleet SNATs
|
||||
// to one IP, so per-IP would collapse the fleet into one bucket. Concurrency + rate caps
|
||||
// + critical-band shed protect the loop and IO from a download flood; a per-window
|
||||
// aggregate makes the flood visible. Pure + testable; mutates the passed rolling state.
|
||||
|
||||
const config = require('../config');
|
||||
|
||||
// newState() — the single bounded rolling counter the endpoint keeps.
|
||||
function newState() { return { inFlight: 0, windowStart: 0, windowCount: 0, served: 0, shed: 0 }; }
|
||||
|
||||
// admit(state, band, now) -> { allow, status?, retryAfter?, summary? }
|
||||
// summary (when a window just rolled) = { served, shed } to log, else null.
|
||||
// NEVER takes an IP — admission is global by construction.
|
||||
function admit(state, band, now = Date.now()) {
|
||||
let summary = null;
|
||||
if (now - state.windowStart >= config.otaDownloadWindowMs) {
|
||||
if (state.served || state.shed) summary = { served: state.served, shed: state.shed, inFlight: state.inFlight };
|
||||
state.windowStart = now; state.windowCount = 0; state.served = 0; state.shed = 0;
|
||||
}
|
||||
const overGlobal = state.inFlight >= config.otaDownloadMaxConcurrent || state.windowCount >= config.otaDownloadMaxPerWindow;
|
||||
if (band === 'critical' || overGlobal) {
|
||||
state.shed++;
|
||||
return { allow: false, status: 503, retryAfter: band === 'critical' ? 30 : 10, summary };
|
||||
}
|
||||
state.inFlight++; state.windowCount++; state.served++;
|
||||
return { allow: true, summary };
|
||||
}
|
||||
|
||||
// release() — call when a served response finishes/closes (once).
|
||||
function release(state) { state.inFlight = Math.max(0, state.inFlight - 1); }
|
||||
|
||||
module.exports = { newState, admit, release };
|
||||
104
server/server.js
104
server/server.js
|
|
@ -583,34 +583,44 @@ const otaBreaker = require('./lib/ota-breaker');
|
|||
otaBreaker.startSweep(); // #144: periodically evict idle breaker buckets so keyed state stays bounded
|
||||
require('./lib/reconnect-throttle').startSweep(); // #146: same, for the reconnect throttle's per-device buckets
|
||||
require('./lib/flap-limiter').startSweep(); // #146 Item B: evict idle flap-limiter buckets
|
||||
const apkCache = require('./lib/apk-cache');
|
||||
apkCache.start(); // #146 Item C: resolve APK path/size/mtime once + refresh on interval (no per-request fs)
|
||||
const { getBand } = require('./services/loop-lag'); // #146 Item C: critical-band download shed
|
||||
app.get('/api/update/check', (req, res) => {
|
||||
const currentVersion = req.query.version;
|
||||
const deviceId = req.query.device_id || null; // #144: optional; beta4+ clients send it for per-device keying
|
||||
const latestVersion = VERSION;
|
||||
|
||||
// #144: circuit-breaker + phantom-version guard (replaces the old string-inequality
|
||||
// offer). Keys per device_id when present, else per reported version. Rate-trips a
|
||||
// looping client in seconds; never offers a downgrade or a superseded/garbage version.
|
||||
// #144: circuit-breaker + phantom-version guard. Keys per device_id when present, else
|
||||
// per reported version (NOT IP — SNAT). Rate-trips a looping client in seconds.
|
||||
const verdict = otaBreaker.decide(currentVersion, latestVersion, deviceId);
|
||||
const apkPath = resolveApkPath(); // existsSync x2 (cheap)
|
||||
const apkExists = apkPath !== null;
|
||||
const updateAvailable = !!verdict.update_available && apkExists; // never offer if APK missing
|
||||
const apkSize = updateAvailable ? fs.statSync(apkPath).size : 0; // statSync only when actually offering (don't stat on every looped poll)
|
||||
const apkModified = updateAvailable ? fs.statSync(apkPath).mtimeMs : 0;
|
||||
|
||||
if (verdict.log) console.log(verdict.log); // once-per-event (trip / unrecognized)
|
||||
// #96: keep the per-check line observable; now also shows the breaker reason + device_id.
|
||||
console.log(`[ota] update check from ${getClientIp(req)}: device=${deviceId || 'none'} client=${currentVersion || 'unknown'} latest=${latestVersion} update_available=${updateAvailable} reason=${verdict.reason} apk=${apkExists ? 'present' : 'MISSING'}`);
|
||||
// #146 Item C: EARLY-RETURN before any filesystem work when we won't serve
|
||||
// (rate-backoff, up-to-date, phantom, client-newer, …). A looping client that gets
|
||||
// rate-backoff does ZERO fs calls — the flood can't turn into a statSync flood.
|
||||
if (!verdict.update_available) {
|
||||
if (verdict.log) console.log(verdict.log);
|
||||
logOtaCheck(deviceId, currentVersion, latestVersion, false, verdict.reason);
|
||||
return res.json({
|
||||
latest_version: latestVersion, current_version: currentVersion || 'unknown',
|
||||
update_available: false, reason: verdict.reason, download_url: '/download/apk',
|
||||
apk_size: 0, apk_modified: 0,
|
||||
...(verdict.retry_after_seconds ? { retry_after_seconds: verdict.retry_after_seconds } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
// Offering — read the CACHED apk metadata (no per-request statSync; refreshed on an
|
||||
// interval by apkCache). Never offer if the APK isn't actually present.
|
||||
const apk = apkCache.get();
|
||||
const updateAvailable = apk.exists;
|
||||
if (verdict.log) console.log(verdict.log);
|
||||
logOtaCheck(deviceId, currentVersion, latestVersion, updateAvailable, updateAvailable ? verdict.reason : 'apk-missing');
|
||||
res.json({
|
||||
latest_version: latestVersion,
|
||||
current_version: currentVersion || 'unknown',
|
||||
update_available: updateAvailable,
|
||||
reason: verdict.reason, // #144: breaker decision, for observability (additive; old clients ignore)
|
||||
latest_version: latestVersion, current_version: currentVersion || 'unknown',
|
||||
update_available: updateAvailable, reason: updateAvailable ? verdict.reason : 'apk-missing',
|
||||
download_url: '/download/apk',
|
||||
apk_size: apkSize,
|
||||
apk_modified: apkModified,
|
||||
...(verdict.retry_after_seconds ? { retry_after_seconds: verdict.retry_after_seconds } : {}),
|
||||
apk_size: updateAvailable ? apk.size : 0,
|
||||
apk_modified: updateAvailable ? apk.mtime : 0,
|
||||
});
|
||||
});
|
||||
|
||||
|
|
@ -715,40 +725,40 @@ app.post('/api/provision/pair', requireAuth, resolveTenancy, checkDeviceLimit, (
|
|||
res.json(updated);
|
||||
});
|
||||
|
||||
// Resolve the OTA APK. A copy under the data dir (DATA_DIR) wins, so a container
|
||||
// operator can mount one at /data/ScreenTinker.apk; otherwise the legacy in-repo
|
||||
// root path (unchanged when DATA_DIR is unset). Returns null if neither exists.
|
||||
function resolveApkPath() {
|
||||
for (const p of [path.join(config.dataDir, 'ScreenTinker.apk'), path.join(__dirname, '..', 'ScreenTinker.apk')]) {
|
||||
if (fs.existsSync(p)) return p;
|
||||
}
|
||||
return null;
|
||||
// #146 Item C: OTA update-check log. One line per check today; Item E coalesces the
|
||||
// high-frequency lines. Never keys on IP for any decision (SNAT). Kept as a helper so
|
||||
// both the offer and no-offer paths log consistently.
|
||||
function logOtaCheck(deviceId, client, latest, available, reason) {
|
||||
console.log(`[ota] update check: device=${deviceId || 'none'} client=${client || 'unknown'} latest=${latest} update_available=${available} reason=${reason}`);
|
||||
}
|
||||
|
||||
// #139: a device that can't silently install re-downloads the APK every check cycle. Don't
|
||||
// word a download as "in progress" (it may be a stuck loop, not progress), and rate-limit the
|
||||
// line to once per IP per window so a looping device can't flood the log.
|
||||
const otaDownloadLoggedAt = new Map(); // ip -> last-logged ms
|
||||
const OTA_DOWNLOAD_LOG_WINDOW_MS = 10 * 60 * 1000;
|
||||
// #146 Item C: GLOBAL download admission (lib/ota-download-guard) — concurrency + rate
|
||||
// caps + critical-band shed, NEVER per-IP (SNAT). Single bounded rolling state.
|
||||
const otaDownloadGuard = require('./lib/ota-download-guard');
|
||||
const otaDownloadState = otaDownloadGuard.newState();
|
||||
|
||||
// Serve APK download
|
||||
app.get('/download/apk', (req, res) => {
|
||||
const apkPath = resolveApkPath();
|
||||
if (apkPath) {
|
||||
const ip = getClientIp(req);
|
||||
const now = Date.now();
|
||||
if (now - (otaDownloadLoggedAt.get(ip) || 0) > OTA_DOWNLOAD_LOG_WINDOW_MS) {
|
||||
otaDownloadLoggedAt.set(ip, now);
|
||||
console.log(`[ota] APK served to ${ip} (${fs.statSync(apkPath).size} bytes)`);
|
||||
}
|
||||
res.setHeader('Content-Type', 'application/vnd.android.package-archive');
|
||||
res.setHeader('Content-Disposition', 'attachment; filename="ScreenTinker.apk"');
|
||||
res.setHeader('Cache-Control', 'no-cache');
|
||||
res.sendFile(apkPath);
|
||||
} else {
|
||||
console.warn(`[ota] APK download requested by ${getClientIp(req)} but no APK is available (404)`);
|
||||
res.status(404).send(`<!DOCTYPE html><html><head><title>APK Not Found</title><style>body{font-family:-apple-system,system-ui,sans-serif;display:flex;justify-content:center;align-items:center;min-height:100vh;margin:0;background:#0f172a;color:#e2e8f0}div{text-align:center;max-width:500px;padding:24px}h1{color:#f87171;font-size:24px}code{background:#1e293b;padding:2px 8px;border-radius:4px;font-size:14px}p{line-height:1.6;color:#94a3b8}</style></head><body><div><h1>APK Not Available</h1><p>The Android APK has not been compiled yet. To build it from source:</p><p><code>cd android</code><br><code>./gradlew assembleDebug</code><br><code>cp app/build/outputs/apk/debug/app-debug.apk ../ScreenTinker.apk</code></p><p>See the <a href="/" style="color:#3b82f6">README</a> for full build instructions.</p><p>In Docker, mount a built APK at <code>/data/ScreenTinker.apk</code> (the data dir).</p><p>Alternatively, use the <a href="/player" style="color:#3b82f6">web player</a> in any browser.</p></div></body></html>`);
|
||||
const apk = apkCache.get();
|
||||
if (!apk.exists) {
|
||||
return res.status(404).send(`<!DOCTYPE html><html><head><title>APK Not Found</title><style>body{font-family:-apple-system,system-ui,sans-serif;display:flex;justify-content:center;align-items:center;min-height:100vh;margin:0;background:#0f172a;color:#e2e8f0}div{text-align:center;max-width:500px;padding:24px}h1{color:#f87171;font-size:24px}code{background:#1e293b;padding:2px 8px;border-radius:4px;font-size:14px}p{line-height:1.6;color:#94a3b8}</style></head><body><div><h1>APK Not Available</h1><p>The Android APK has not been compiled yet. In Docker, mount a built APK at <code>/data/ScreenTinker.apk</code>, or use the <a href="/player" style="color:#3b82f6">web player</a>.</p></div></body></html>`);
|
||||
}
|
||||
|
||||
const verdict = otaDownloadGuard.admit(otaDownloadState, getBand());
|
||||
if (verdict.summary) {
|
||||
console.log(`[ota] downloads last ${Math.round(config.otaDownloadWindowMs / 1000)}s: ${verdict.summary.served} served, ${verdict.summary.shed} shed (in-flight ${verdict.summary.inFlight})`);
|
||||
}
|
||||
if (!verdict.allow) {
|
||||
res.setHeader('Retry-After', String(verdict.retryAfter));
|
||||
return res.status(verdict.status).json({ error: 'download capacity reached, retry shortly', retry_after: verdict.retryAfter });
|
||||
}
|
||||
|
||||
let released = false;
|
||||
const release = () => { if (released) return; released = true; otaDownloadGuard.release(otaDownloadState); };
|
||||
res.on('finish', release); res.on('close', release);
|
||||
res.setHeader('Content-Type', 'application/vnd.android.package-archive');
|
||||
res.setHeader('Content-Disposition', 'attachment; filename="ScreenTinker.apk"');
|
||||
res.setHeader('Cache-Control', 'no-cache');
|
||||
res.sendFile(apk.path, (err) => { if (err) release(); });
|
||||
});
|
||||
|
||||
// SPA fallback for app routes. Unmatched /api/ paths return 404 so misrouted
|
||||
|
|
|
|||
70
server/test/ota-hardening.test.js
Normal file
70
server/test/ota-hardening.test.js
Normal file
|
|
@ -0,0 +1,70 @@
|
|||
'use strict';
|
||||
|
||||
// #146 hardening (Item C) — OTA under SNAT. Cached APK resolution (no per-request fs)
|
||||
// + GLOBAL download admission (concurrency + rate + critical-band shed, never per-IP).
|
||||
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const fs = require('node:fs');
|
||||
const crypto = require('node:crypto');
|
||||
process.env.DATA_DIR = path.join(os.tmpdir(), 'st-otahard-' + crypto.randomBytes(4).toString('hex'));
|
||||
process.env.OTA_DOWNLOAD_MAX_CONCURRENT = '3';
|
||||
process.env.OTA_DOWNLOAD_MAX_PER_WINDOW = '5';
|
||||
process.env.OTA_DOWNLOAD_WINDOW_MS = '100000';
|
||||
|
||||
const { test } = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const guard = require('../lib/ota-download-guard');
|
||||
const apkCache = require('../lib/apk-cache');
|
||||
|
||||
test('apk-cache: get() never touches the filesystem (resolution cached at boot/refresh)', () => {
|
||||
// seed a fake APK under DATA_DIR and refresh once
|
||||
const apk = path.join(process.env.DATA_DIR, 'ScreenTinker.apk');
|
||||
fs.mkdirSync(process.env.DATA_DIR, { recursive: true });
|
||||
fs.writeFileSync(apk, 'FAKEAPKBYTES');
|
||||
const c = apkCache.refresh();
|
||||
assert.equal(c.exists, true);
|
||||
assert.equal(c.size, 12);
|
||||
|
||||
// count fs.statSync calls across many get()s -> must be ZERO (a poll/download flood
|
||||
// can't become a statSync flood).
|
||||
const realStat = fs.statSync; let calls = 0;
|
||||
fs.statSync = (...a) => { calls++; return realStat(...a); };
|
||||
try { for (let i = 0; i < 1000; i++) apkCache.get(); } finally { fs.statSync = realStat; }
|
||||
assert.equal(calls, 0, 'get() does no fs; 1000 reads = 0 statSync');
|
||||
});
|
||||
|
||||
test('download guard: global concurrency cap -> 503 (not per-IP)', () => {
|
||||
const s = guard.newState();
|
||||
assert.equal(guard.admit(s, 'normal').allow, true);
|
||||
assert.equal(guard.admit(s, 'normal').allow, true);
|
||||
assert.equal(guard.admit(s, 'normal').allow, true); // 3 in-flight = cap
|
||||
const over = guard.admit(s, 'normal');
|
||||
assert.equal(over.allow, false);
|
||||
assert.equal(over.status, 503);
|
||||
assert.ok(over.retryAfter > 0);
|
||||
guard.release(s); // free one slot
|
||||
assert.equal(guard.admit(s, 'normal').allow, true, 'a freed slot admits again');
|
||||
});
|
||||
|
||||
test('download guard: global per-window rate cap -> 503', () => {
|
||||
const s = guard.newState();
|
||||
for (let i = 0; i < 5; i++) { assert.equal(guard.admit(s, 'normal').allow, true); guard.release(s); } // 5 served this window
|
||||
const over = guard.admit(s, 'normal');
|
||||
assert.equal(over.allow, false, '6th in the window is shed');
|
||||
assert.equal(over.status, 503);
|
||||
});
|
||||
|
||||
test('download guard: critical band sheds regardless of caps', () => {
|
||||
const s = guard.newState();
|
||||
const v = guard.admit(s, 'critical');
|
||||
assert.equal(v.allow, false);
|
||||
assert.equal(v.retryAfter, 30, 'critical band asks for a longer backoff');
|
||||
});
|
||||
|
||||
test('download guard admission takes NO ip argument (global by construction)', () => {
|
||||
// admit(state, band, now) — there is no IP parameter, so it cannot key on IP.
|
||||
assert.equal(guard.admit.length <= 3, true);
|
||||
const s = guard.newState();
|
||||
assert.equal(guard.admit(s, 'normal').allow, true); // works with zero IP context
|
||||
});
|
||||
Loading…
Reference in a new issue