feat: correlate server errors and monitor application health

src/api.ts

@@ -1,3 +1,4 @@
+import { errorDetails } from "./ops/monitor";
 import { createAccountRoutes } from "./account/routes";
 import { createHabitRoutes } from "./habits/routes";
 import { ApiError } from "./habits/service";
@@ -9,11 +10,18 @@ import type { FetchDiscord } from "./auth/discord";
 import { createSharingRoutes, type DiscordFetch } from "./sharing/routes";
 import type { DiscordSharingConfig } from "./sharing/config";
 
-export function createApi(db: AppDatabase, config: AuthConfig, request?: FetchDiscord, now?: () => number, discordRequest?: DiscordFetch, sharingConfig: DiscordSharingConfig = { token: "", channelId: "" }) {
+export function createApi(db: AppDatabase, config: AuthConfig, request?: FetchDiscord, now?: () => number, discordRequest?: DiscordFetch, sharingConfig: DiscordSharingConfig = { token: "", channelId: "" }, operations: { healthy?: () => boolean; log?: (event: Record<string, unknown>) => void } = {}) {
   const app = new Hono<AuthEnv>();
+  app.use("*", async (c, next) => {
+    const requestId = crypto.randomUUID();
+    c.set("requestId", requestId); c.header("X-Request-ID", requestId);
+    await next();
+  });
   const auth = createAuth(db, config, request, now);
   app.get("/api/health", c => {
+    c.header("Cache-Control", "no-store");
     db.get(sql`SELECT 1`);
+    if (operations.healthy && !operations.healthy()) return c.json({ status: "degraded" }, 503);
     return c.json({ status: "ok" });
   });
   app.route("/api/auth", auth.routes);
@@ -25,7 +33,11 @@ export function createApi(db: AppDatabase, config: AuthConfig, request?: FetchDi
   app.onError((_error, c) => {
     c.header("Cache-Control", "no-store");
     if (_error instanceof ApiError) return c.json({ error: _error.message }, _error.status);
-    return c.json({ error: "Internal server error" }, 500);
+    const requestId = c.get("requestId");
+    const event = { event: "request_failed", timestamp: new Date().toISOString(), requestId, method: c.req.method,
+      route: c.req.routePath, ...errorDetails(_error) };
+    (operations.log ?? (event => console.error(JSON.stringify(event))))(event);
+    return c.json({ error: "Internal server error", requestId }, 500);
   });
   return app;
 }

src/auth/index.ts

@@ -12,7 +12,7 @@ import { fetchDiscordProfile, type FetchDiscord } from "./discord";
 const { users, sessions } = schema;
 const SESSION_SECONDS = 30 * 24 * 60 * 60;
 const STATE_SECONDS = 10 * 60;
-export type AuthEnv = { Variables: { user: PublicUser } };
+export type AuthEnv = { Variables: { user: PublicUser; requestId: string } };
 export type AppDatabase = BunSQLiteDatabase<typeof schema>;
 export const hashToken = (token: string) => createHash("sha256").update(token).digest("hex");
 const randomToken = () => randomBytes(32).toString("base64url");

src/index.ts

@@ -9,7 +9,7 @@ import index from "./index.html";
 const backups = process.env.NODE_ENV === "production" ? startBackups(sqlite, backupConfig(databasePath)) : undefined;
 if (import.meta.hot) import.meta.hot.dispose(() => backups?.stop());
 
-const app = createApi(db, readAuthConfig(), undefined, undefined, undefined, readDiscordSharingConfig());
+const app = createApi(db, readAuthConfig(), undefined, undefined, undefined, readDiscordSharingConfig(), { healthy: () => !backups?.state.enabled || (!backups.state.failed && backups.state.lastSuccess > 0) });
 
 const server = Bun.serve({
   hostname: "127.0.0.1",

src/lib/dashboard.ts

@@ -20,7 +20,7 @@ export async function habitRequest<T>(
       throw new Error("Your session has expired. Sign in again to continue.");
     const body = await response.json().catch(() => null);
     throw new Error(
-      body?.error || "Could not reach your workspace. Please try again.",
+      (body?.error || "Could not reach your workspace. Please try again.") + (response.status >= 500 && body?.requestId ? ` Reference: ${body.requestId}` : ""),
     );
   }
   return response.status === 204

src/ops/monitor.test.ts

@@ -0,0 +1,39 @@
+import { expect, test } from "bun:test";
+import { fixture } from "../habits/test-fixture";
+import { createApi } from "../api";
+import { healthObserver, probeHealth } from "./monitor";
+
+test("unexpected database errors are correlated without logging secrets", async () => {
+  const f = fixture(); const events: Record<string, unknown>[] = [];
+  const app = createApi(f.db, { origin: f.origin, clientId: "", clientSecret: "", cookieSecret: "test-secret-at-least-32-characters" }, undefined, undefined, undefined, undefined, { log: event => events.push(event) });
+  f.close();
+  const response = await app.request(`${f.origin}/api/health?token=private-secret`, { headers: { Cookie: "private-cookie", "X-Request-ID": "untrusted" } });
+  expect(response.status).toBe(500);
+  const body = await response.json();
+  expect(body.requestId).toBe(response.headers.get("x-request-id"));
+  expect(body.requestId).not.toBe("untrusted");
+  expect(events[0]?.requestId).toBe(body.requestId);
+  expect(events[0]?.type).toBeDefined();
+  expect(JSON.stringify(events)).not.toContain("private");
+});
+
+test("failed backups degrade health, and probes validate status and body", async () => {
+  const f = fixture();
+  try {
+    const app = createApi(f.db, { origin: f.origin, clientId: "", clientSecret: "", cookieSecret: "test-secret-at-least-32-characters" }, undefined, undefined, undefined, undefined, { healthy: () => false });
+    const response = await app.request(`${f.origin}/api/health`);
+    expect(response.status).toBe(503); expect(await response.json()).toEqual({ status: "degraded" });
+    const server = Bun.serve({ hostname: "127.0.0.1", port: 0, fetch: () => Response.json({ status: "ok" }) });
+    try { expect((await probeHealth(String(server.url))).healthy).toBe(true); } finally { await server.stop(true); }
+    expect((await probeHealth(String(server.url))).healthy).toBe(false);
+  } finally { f.close(); }
+});
+
+test("monitor reports sustained failure and recovery once per transition", () => {
+  const events: Record<string, unknown>[] = [];
+  const observe = healthObserver(event => events.push(event));
+  observe({ healthy: true, reason: "ok" });
+  for (let i = 0; i < 5; i++) observe({ healthy: false, reason: "HTTP 503" });
+  observe({ healthy: true, reason: "ok" }); observe({ healthy: true, reason: "ok" });
+  expect(events.map(event => event.event)).toEqual(["monitor_started", "health_failed", "health_recovered"]);
+});

src/ops/monitor.ts

@@ -0,0 +1,34 @@
+export type HealthResult = { healthy: boolean; reason: string };
+export async function probeHealth(url: string, request: typeof fetch = fetch): Promise<HealthResult> {
+  try {
+    const response = await request(url, { signal: AbortSignal.timeout(10000), redirect: "error", cache: "no-store" });
+    if (!response.ok) return { healthy: false, reason: `HTTP ${response.status}` };
+    const body = await response.json() as { status?: string };
+    return body.status === "ok" ? { healthy: true, reason: "ok" } : { healthy: false, reason: "Invalid health response" };
+  } catch { return { healthy: false, reason: "Health check unavailable" }; }
+}
+export function healthObserver(emit: (event: Record<string, unknown>) => void, threshold = 3) {
+  let failures = 0;
+  let incident = false;
+  let initialized = false;
+  return (result: HealthResult) => {
+    failures = result.healthy ? 0 : failures + 1;
+    const failed = failures >= threshold;
+    const event = !initialized && result.healthy ? "monitor_started" : failed && !incident ? "health_failed" : incident && result.healthy ? "health_recovered" : null;
+    if (event) emit({ event, timestamp: new Date().toISOString(), reason: result.reason, failures });
+    if (failed) incident = true;
+    if (result.healthy) incident = false;
+    initialized = true;
+  };
+}
+
+/** Keep secrets, SQL values, request bodies, OAuth queries and cookies out of logs. */
+export function errorDetails(error: Error) {
+  return {
+    type: ["Error", "TypeError", "RangeError", "SyntaxError", "SQLiteError"].includes(error.name) ? error.name : "Error",
+    frames: (error.stack ?? "").split("\n").slice(1).flatMap(line => {
+      const location = line.match(/([A-Za-z0-9_.-]+\.(?:js|ts|tsx):\d+:\d+)/);
+      return location ? [location[1]] : [];
+    }).slice(0, 8),
+  };
+}