screentinker/server/test/wal-checkpoint-starvation-floor.test.js

// #240 — the WAL checkpointer's STARVATION escalation must be gated on WAL size.
//
// Why this test exists: TRUNCATE is the blocking checkpoint form and it blocks across
// connections, so a main-thread statement issued during one waits in SQLite's busy handler
// for its whole duration. The old rule escalated on growth ALONE — three consecutive
// PASSIVE runs where the WAL got bigger — which any sustained write burst satisfies. A
// customer's fleet powering on in the morning bought itself a multi-second event-loop
// stall against a WAL of a couple of MB, where a blocking checkpoint had nothing to
// reclaim in the first place.
//
// The decision is deliberately tested as the pure predicate the worker evaluates rather
// than by driving a real worker thread: the property that matters is WHEN we are willing
// to block, and that must not silently regress behind a timing-dependent test.
const { test } = require('node:test');
const assert = require('node:assert/strict');
const fs = require('fs');
const os = require('os');
const path = require('path');
const config = require('../config');

const MB = 1024 * 1024;

// The worker's escalation rule, kept in one place so the assertions below read as policy.
function shouldEscalate({ bytes, growthRuns, sinceLastTruncateMs, everTruncated,
                          highWaterBytes, starvationRuns, starvationFloorBytes, escalateCooldownMs }) {
  const overHighWater = bytes > highWaterBytes;
  const starved = growthRuns >= starvationRuns && bytes >= starvationFloorBytes;
  const cooling = starved && everTruncated && sinceLastTruncateMs < escalateCooldownMs;
  if (cooling && !overHighWater) return { overHighWater, starved, cooling, escalate: false };
  return { overHighWater, starved, cooling, escalate: overHighWater || starved };
}

const RULE = {
  highWaterBytes: config.walCheckpointHighWaterMB * MB,
  starvationRuns: config.walCheckpointStarvationRuns,
  starvationFloorBytes: config.walCheckpointStarvationFloorMB * MB,
  escalateCooldownMs: config.walCheckpointEscalateCooldownMs,
  sinceLastTruncateMs: Infinity,
  everTruncated: false,
};

test('#240: a small WAL growing across runs no longer triggers the blocking TRUNCATE', () => {
  // The morning-wave shape: sustained writes, WAL grew every run, still only 2MB.
  const r = shouldEscalate({ ...RULE, bytes: 2 * MB, growthRuns: 5 });
  assert.equal(r.escalate, false, 'growth alone must not escalate while the WAL is small');
});

test('#240: growth still escalates once the WAL is actually large', () => {
  const r = shouldEscalate({ ...RULE, bytes: config.walCheckpointStarvationFloorMB * MB, growthRuns: config.walCheckpointStarvationRuns });
  assert.equal(r.starved, true, 'at the floor, sustained growth is real starvation');
  assert.equal(r.escalate, true);
});

test('#240: the high-water backstop is untouched — the WAL still cannot grow unbounded', () => {
  // No growth signal at all (a single huge run), well over the high-water mark.
  const r = shouldEscalate({ ...RULE, bytes: (config.walCheckpointHighWaterMB + 1) * MB, growthRuns: 0 });
  assert.equal(r.overHighWater, true);
  assert.equal(r.escalate, true, 'high-water must escalate regardless of the growth counter');
});

test('#240: the floor sits below the high-water mark, so the two rules cannot invert', () => {
  assert.ok(
    config.walCheckpointStarvationFloorMB < config.walCheckpointHighWaterMB,
    'a floor at or above high-water would make the starvation rule dead code'
  );
});

// The floor on its own does NOT close this. Bold's WAL sat at 6.2MB against a 16MB
// high-water — already above any sane floor — so a morning wave would still have escalated
// on every burst. The cooldown is what bounds how often our own maintenance may stall the
// loop, regardless of how long the write pressure lasts.
test('#240: a WAL already above the floor escalates ONCE, then holds off', () => {
  const big = { ...RULE, bytes: 12 * MB, growthRuns: 5 };

  const first = shouldEscalate({ ...big });
  assert.equal(first.escalate, true, 'the first sustained-growth burst still escalates');

  const during = shouldEscalate({ ...big, everTruncated: true, sinceLastTruncateMs: 30_000 });
  assert.equal(during.cooling, true);
  assert.equal(during.escalate, false, 'a second burst inside the cooldown must not stall the loop again');

  const after = shouldEscalate({ ...big, everTruncated: true, sinceLastTruncateMs: config.walCheckpointEscalateCooldownMs + 1 });
  assert.equal(after.escalate, true, 'once the window passes, escalation is available again');
});

test('#240: the cooldown never delays the runaway-WAL backstop', () => {
  const runaway = {
    ...RULE, bytes: (config.walCheckpointHighWaterMB + 1) * MB, growthRuns: 5,
    everTruncated: true, sinceLastTruncateMs: 1000,   // deep inside the cooldown
  };
  const r = shouldEscalate(runaway);
  assert.equal(r.escalate, true, 'over high-water must escalate even mid-cooldown — that rule is the safety net');
});

test('#240: the worker actually applies the floor it is handed', () => {
  const src = fs.readFileSync(path.join(__dirname, '..', 'db', 'wal-checkpointer-worker.js'), 'utf8');
  assert.match(src, /growthRuns >= starvationRuns && bytes >= starvationFloorBytes/,
    'worker starvation check must include the size floor');
  assert.match(src, /starvationFloorBytes/, 'worker must destructure starvationFloorBytes from workerData');
  assert.match(src, /sinceLast < escalateCooldownMs/, 'worker must apply the escalation cooldown');
  const ctl = fs.readFileSync(path.join(__dirname, '..', 'db', 'wal-checkpointer.js'), 'utf8');
  assert.match(ctl, /starvationFloorBytes:\s*config\.walCheckpointStarvationFloorMB/,
    'controller must pass the floor through workerData — an undefined floor would make every comparison false');
  assert.match(ctl, /escalateCooldownMs:\s*config\.walCheckpointEscalateCooldownMs/,
    'controller must pass the cooldown through workerData');
});

// Measured, not assumed: with a single reader mid-transaction, TRUNCATE returns busy=1
// after sitting on its 5s busy timeout and reclaims nothing (probe: WAL 8.8MB -> 8.8MB,
// worst main-thread write 4,936ms). That outcome must not be logged as a success.
// The startup line is where an operator learns the policy. It went stale the moment the gates were
// added — it still promised "3 growing runs" with no mention of the floor or the cooldown, so it
// described a checkpointer that no longer existed. A log line that states a rule has to state the
// whole rule, and nothing but a test keeps the two in step.
test('#240: the startup line states the WHOLE escalation policy', () => {
  const ctl = fs.readFileSync(path.join(__dirname, '..', 'db', 'wal-checkpointer.js'), 'utf8');
  // Anchor forward from the message text, not back to `return worker;` — the idempotence guard at
  // the top of startWalCheckpointer() returns first, so slicing to it yields nothing at all.
  const start = ctl.indexOf('off-thread checkpointer started');
  assert.ok(start > 0, 'the startup line is gone entirely');
  const line = ctl.slice(start, start + 800);
  for (const knob of [
    'walCheckpointIntervalMs',
    'walCheckpointHighWaterMB',
    'walCheckpointStarvationRuns',
    'walCheckpointStarvationFloorMB',
    'walCheckpointEscalateCooldownMs',
  ]) {
    assert.ok(line.includes(knob), `the startup line must report ${knob} — an operator reads it as the policy`);
  }
});

test('#240: a TRUNCATE that reclaimed nothing says so', () => {
  const src = fs.readFileSync(path.join(__dirname, '..', 'db', 'wal-checkpointer-worker.js'), 'utf8');
  assert.match(src, /busy === 1/, 'worker must inspect the checkpoint result');
  assert.match(src, /reclaimed nothing/, 'a busy TRUNCATE must be reported as the loss it is');
});

test('#240: the unrecoverable-worker fallback no longer blocks the loop for a small WAL', () => {
  const ctl = fs.readFileSync(path.join(__dirname, '..', 'db', 'wal-checkpointer.js'), 'utf8');
  assert.match(ctl, /wal_checkpoint\(\$\{over \? 'TRUNCATE' : 'PASSIVE'\}\)/,
    'fallback reclaim must pick TRUNCATE only when the WAL is over the high-water mark');
});

test('#240: /api/status exposes the sticky fallback state', () => {
  const { getCheckpointerState } = require('../db/wal-checkpointer');
  const s = getCheckpointerState();
  // Not started in this process — the point is the shape, and that reading it is safe
  // before startWalCheckpointer() has ever run (status is served during boot too).
  assert.deepEqual(Object.keys(s).sort(), ['fallbackEngaged', 'respawns', 'walBytes', 'worker']);
  assert.equal(s.fallbackEngaged, false);
  assert.equal(typeof s.walBytes, 'number');
});