fix circuit-breaker off-by-one, ENOENT, and reset-on-throw + tests

- getDelay indexed by attempt (1-based) into a 0-indexed array, so the
  leading 0 was unreachable and every "after a crash" delay was shifted
  up one slot. Use attempt - 1 so the documented schedule (0s → 0s →
  10s → 30s → 2min → 5min → 15min cap) actually holds.
- enforceStartupBackoff runs before initDb (which creates DATA_DIR), so
  on a fresh checkout fs.writeFileSync hit ENOENT. write() now
  mkdirSync's DATA_DIR first.
- shutdown() didn't run resetCircuitBreaker if teardownChannelAdapters
  threw, so a graceful exit with a teardown error would be counted as a
  crash on the next start. Wrap teardown in try/finally.
- Adds src/circuit-breaker.test.ts: state transitions, full schedule
  (parameterized), reset-window expiry, malformed file, and the
  fresh-install path.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
gavrielc
2026-04-28 22:51:11 +03:00
parent 2bf296b04a
commit 336e01d2a1
3 changed files with 213 additions and 5 deletions

View File

@@ -6,7 +6,9 @@ import { log } from './log.js';
const CB_PATH = path.join(DATA_DIR, 'circuit-breaker.json');
const RESET_WINDOW_MS = 60 * 60 * 1000; // 1 hour
const BACKOFF_SCHEDULE_S = [0, 0, 10, 30, 120, 300, 900]; // index = attempt number, 6+ capped at 15min
// Index = number of consecutive crashes (0 = clean start, attempt 1).
// 6+ crashes capped at 15min.
const BACKOFF_SCHEDULE_S = [0, 0, 10, 30, 120, 300, 900];
interface CircuitBreakerState {
attempt: number;
@@ -23,11 +25,14 @@ function read(): CircuitBreakerState | null {
}
function write(state: CircuitBreakerState): void {
// The breaker runs before initDb (which is what creates DATA_DIR), so on a
// fresh checkout the dir may not exist yet.
fs.mkdirSync(DATA_DIR, { recursive: true });
fs.writeFileSync(CB_PATH, JSON.stringify(state, null, 2) + '\n');
}
function getDelay(attempt: number): number {
const idx = Math.min(attempt, BACKOFF_SCHEDULE_S.length - 1);
const idx = Math.min(attempt - 1, BACKOFF_SCHEDULE_S.length - 1);
return BACKOFF_SCHEDULE_S[idx];
}