feat: correlate server errors and monitor application health

src/ops/monitor.test.ts

@@ -0,0 +1,39 @@
+import { expect, test } from "bun:test";
+import { fixture } from "../habits/test-fixture";
+import { createApi } from "../api";
+import { healthObserver, probeHealth } from "./monitor";
+
+test("unexpected database errors are correlated without logging secrets", async () => {
+  const f = fixture(); const events: Record<string, unknown>[] = [];
+  const app = createApi(f.db, { origin: f.origin, clientId: "", clientSecret: "", cookieSecret: "test-secret-at-least-32-characters" }, undefined, undefined, undefined, undefined, { log: event => events.push(event) });
+  f.close();
+  const response = await app.request(`${f.origin}/api/health?token=private-secret`, { headers: { Cookie: "private-cookie", "X-Request-ID": "untrusted" } });
+  expect(response.status).toBe(500);
+  const body = await response.json();
+  expect(body.requestId).toBe(response.headers.get("x-request-id"));
+  expect(body.requestId).not.toBe("untrusted");
+  expect(events[0]?.requestId).toBe(body.requestId);
+  expect(events[0]?.type).toBeDefined();
+  expect(JSON.stringify(events)).not.toContain("private");
+});
+
+test("failed backups degrade health, and probes validate status and body", async () => {
+  const f = fixture();
+  try {
+    const app = createApi(f.db, { origin: f.origin, clientId: "", clientSecret: "", cookieSecret: "test-secret-at-least-32-characters" }, undefined, undefined, undefined, undefined, { healthy: () => false });
+    const response = await app.request(`${f.origin}/api/health`);
+    expect(response.status).toBe(503); expect(await response.json()).toEqual({ status: "degraded" });
+    const server = Bun.serve({ hostname: "127.0.0.1", port: 0, fetch: () => Response.json({ status: "ok" }) });
+    try { expect((await probeHealth(String(server.url))).healthy).toBe(true); } finally { await server.stop(true); }
+    expect((await probeHealth(String(server.url))).healthy).toBe(false);
+  } finally { f.close(); }
+});
+
+test("monitor reports sustained failure and recovery once per transition", () => {
+  const events: Record<string, unknown>[] = [];
+  const observe = healthObserver(event => events.push(event));
+  observe({ healthy: true, reason: "ok" });
+  for (let i = 0; i < 5; i++) observe({ healthy: false, reason: "HTTP 503" });
+  observe({ healthy: true, reason: "ok" }); observe({ healthy: true, reason: "ok" });
+  expect(events.map(event => event.event)).toEqual(["monitor_started", "health_failed", "health_recovered"]);
+});