// WAL checkpointer WORKER (worker_threads). Runs OFF the main event-loop thread so the // synchronous, fsync-heavy checkpoint never blocks the loop (the ~60s p99 spike). // // CRITICAL: this worker opens its OWN better-sqlite3 Database() handle against the same // file. better-sqlite3 handles are NOT thread-safe, so the main thread's handle is never // shared into the worker — only the dbPath STRING is passed via workerData. SQLite WAL is // designed for multiple connections to the same file, so a second connection checkpointing // while the main connection writes is safe. const { workerData, parentPort } = require('worker_threads'); const fs = require('fs'); const Database = require('better-sqlite3'); const { dbPath, intervalMs, highWaterBytes, starvationRuns, starvationFloorBytes, escalateCooldownMs } = workerData; // Fault injection for TESTS ONLY (env-gated; inert in prod). Exits immediately on start so // the controller's respawn / autocheckpoint-fallback path can be exercised deterministically. if (process.env.WAL_CKPT_FAIL_START) process.exit(1); // Fresh, worker-owned connection (NOT the main handle). const db = new Database(dbPath); db.pragma('busy_timeout = 5000'); // wait (on THIS worker thread) through the main writer's brief locks db.pragma('wal_autocheckpoint = 0'); // this connection must never auto-checkpoint either const walFile = dbPath + '-wal'; function walBytes() { try { return fs.statSync(walFile).size; } catch { return 0; } } let lastBytes = 0; let growthRuns = 0; // consecutive PASSIVE runs where the WAL failed to shrink let lastTruncateAt = 0; // #240: when we last blocked for a TRUNCATE (0 = never) let coolingReported = false; let timer = null; function tick() { try { // PASSIVE never blocks writers, but skips frames pinned by active readers/writers — // so on its own it can perpetually under-checkpoint. That's what the guard below bounds. db.pragma('wal_checkpoint(PASSIVE)', { simple: false }); const bytes = walBytes(); // --- STARVATION BOUND (this is where "WAL cannot grow forever" is enforced) --- // Escalating forces a TRUNCATE, which BLOCKS until it has checkpointed everything and // truncated the file to 0. #240: "fine here on the worker" was only ever half true — // the fsync is off the loop, but SQLite's locks are held across CONNECTIONS, so the // main thread's next statement waits it out in the busy handler. Hence the gates below. if (bytes > lastBytes) growthRuns++; else growthRuns = 0; const overHighWater = bytes > highWaterBytes; // #240: TRUNCATE blocks ACROSS connections — the main thread's next statement waits in // SQLite's busy handler for the whole checkpoint — so the growth signal alone must not // be able to spend it. Two gates, because either on its own leaves the hole open: // FLOOR: a WAL in the lower half of its budget has little to reclaim; blocking for it // is pure cost. (Ungated, every morning fleet power-on wave bought a loop stall.) // COOLDOWN: a WAL that already sits ABOVE the floor would otherwise escalate on every // burst forever. However long the pressure lasts, we stall the loop at most once // per window and let PASSIVE do the rest. // overHighWater bypasses both — a runaway WAL is the one case worth blocking for. const sinceLast = Date.now() - lastTruncateAt; const starved = growthRuns >= starvationRuns && bytes >= starvationFloorBytes; const cooling = starved && lastTruncateAt > 0 && sinceLast < escalateCooldownMs; if (cooling && !overHighWater) { // Report the transition only — a starved-and-cooling state persists for the whole // window and this check runs every interval; one line, not a log flood. if (!coolingReported) { coolingReported = true; post(`starvation escalation held off (WAL ${(bytes / 1e6).toFixed(1)}MB, last TRUNCATE ${Math.round(sinceLast / 1000)}s ago) — PASSIVE continues`); } lastBytes = bytes; return; } if (overHighWater || starved) { lastTruncateAt = Date.now(); coolingReported = false; const r = db.pragma('wal_checkpoint(TRUNCATE)', { simple: false }); const after = walBytes(); // #240: TRUNCATE does NOT throw when it can't get the locks — it returns busy=1 having // sat on SQLite's busy timeout for its full duration. Measured at ~4.9s with a single // reader mid-transaction, reclaiming nothing, while every main-thread statement waited // behind it. Say so plainly: a silent 5-second loss is the worst thing this can do. const busy = Array.isArray(r) && r[0] && r[0].busy === 1; post(`escalated TRUNCATE (${overHighWater ? 'high-water' : 'starvation'}): WAL ${(bytes / 1e6).toFixed(1)}MB -> ${(after / 1e6).toFixed(1)}MB${busy ? ' — BUSY: reclaimed nothing, blocked writers for the busy timeout' : ''}`); growthRuns = 0; lastBytes = after; } else { lastBytes = bytes; } } catch (e) { post('checkpoint error: ' + (e && e.message)); } } function post(log) { try { parentPort && parentPort.postMessage({ log }); } catch (_) {} } timer = setInterval(tick, intervalMs); // Clean shutdown: stop the timer, close our connection, exit THIS worker thread. parentPort && parentPort.on('message', (m) => { if (m && m.stop) { if (timer) { clearInterval(timer); timer = null; } try { db.close(); } catch (_) {} process.exit(0); } });