feat: correlate server errors and monitor application health

.env.example

@@ -23,3 +23,5 @@ BACKUP_INTERVAL_HOURS=24
 BACKUP_RETAIN=7
 # Existing mounted off-machine backup directory (absolute path recommended).
 BACKUP_REPLICA_DIR=
+# Optional public HTTPS health URL for the independent monitor.
+MONITOR_URL=

docs/OPERATIONS.md

@@ -43,3 +43,31 @@ before reopening access. Discord images already posted are separate from app dat
 Verification: `bun test src/ops/backups.test.ts` exercises WAL progress, restored
 content, revoked sessions, replicas, retention, missing storage, and refusal to
 overwrite an existing destination.
+
+## Monitoring
+
+Every API response includes a server-generated `X-Request-ID`. Unexpected errors
+return that reference to the browser and write a JSON `request_failed` event to
+stderr with the request ID, method, route template, error type, and stack locations.
+Error messages, SQL values, request bodies, cookies, and query strings are omitted
+so credentials and habit content do not enter logs. Production source maps in
+`dist/` help map stack locations back to the source. Backup outcomes also emit
+`backup_completed` and `backup_failed` events.
+
+`GET /api/health` checks SQLite and returns 503 if enabled automatic backups are
+failing or have never succeeded. It returns no account data, paths, or credentials.
+Run the health monitor independently of the application process:
+
+```sh
+bun run monitor --once
+bun run monitor
+```
+
+`MONITOR_URL` defaults to `http://127.0.0.1:$PORT/api/health`. For an outside-in
+check, run the monitor on another machine and set it to the public HTTPS endpoint.
+The one-shot command exits nonzero on failure. Continuous mode checks every minute,
+times out after ten seconds, reports `health_failed` after three consecutive
+failures, and emits one `health_recovered` event on recovery. It stays quiet while
+state is unchanged. Run it under a process supervisor and route these JSON events
+to your deployment platform's alerting/log collection. No external alert provider
+or off-machine monitor is provisioned automatically by this repository.

package.json

@@ -15,7 +15,8 @@
     "test:coverage": "bun test --coverage",
     "test:smoke": "bun run build && bun scripts/api-smoke.ts",
     "db:backup": "NODE_ENV=production bun --env-file=.env --env-file=.env.production scripts/backup.ts",
-    "db:restore": "bun scripts/restore.ts"
+    "db:restore": "bun scripts/restore.ts",
+    "monitor": "bun --env-file=.env --env-file=.env.production scripts/monitor.ts"
   },
   "dependencies": {
     "@nivo/calendar": "^0.99.0",

scripts/monitor.ts

@@ -0,0 +1,21 @@
+import { probeHealth, healthObserver } from "../src/ops/monitor";
+const target = new URL(process.env.MONITOR_URL || `http://127.0.0.1:${process.env.PORT ?? 3000}/api/health`);
+if (!["http:", "https:"].includes(target.protocol) || target.username || target.password) throw new Error("MONITOR_URL must be an HTTP(S) URL without credentials");
+const emit = (event: Record<string, unknown>) => console.log(JSON.stringify(event));
+if (process.argv.includes("--once")) {
+  const result = await probeHealth(target.href);
+  emit({ event: "health_probe", ...result });
+  process.exitCode = result.healthy ? 0 : 1;
+} else {
+  const observe = healthObserver(emit);
+  let running = false;
+  const check = async () => {
+    if (running) return;
+    running = true;
+    try { observe(await probeHealth(target.href)); } finally { running = false; }
+  };
+  await check();
+  const timer = setInterval(() => void check(), 60000);
+  const stop = () => { clearInterval(timer); process.exit(0); };
+  process.on("SIGTERM", stop); process.on("SIGINT", stop);
+}

src/api.ts

@@ -1,3 +1,4 @@
+import { errorDetails } from "./ops/monitor";
 import { createAccountRoutes } from "./account/routes";
 import { createHabitRoutes } from "./habits/routes";
 import { ApiError } from "./habits/service";
@@ -9,11 +10,18 @@ import type { FetchDiscord } from "./auth/discord";
 import { createSharingRoutes, type DiscordFetch } from "./sharing/routes";
 import type { DiscordSharingConfig } from "./sharing/config";
 
-export function createApi(db: AppDatabase, config: AuthConfig, request?: FetchDiscord, now?: () => number, discordRequest?: DiscordFetch, sharingConfig: DiscordSharingConfig = { token: "", channelId: "" }) {
+export function createApi(db: AppDatabase, config: AuthConfig, request?: FetchDiscord, now?: () => number, discordRequest?: DiscordFetch, sharingConfig: DiscordSharingConfig = { token: "", channelId: "" }, operations: { healthy?: () => boolean; log?: (event: Record<string, unknown>) => void } = {}) {
   const app = new Hono<AuthEnv>();
+  app.use("*", async (c, next) => {
+    const requestId = crypto.randomUUID();
+    c.set("requestId", requestId); c.header("X-Request-ID", requestId);
+    await next();
+  });
   const auth = createAuth(db, config, request, now);
   app.get("/api/health", c => {
+    c.header("Cache-Control", "no-store");
     db.get(sql`SELECT 1`);
+    if (operations.healthy && !operations.healthy()) return c.json({ status: "degraded" }, 503);
     return c.json({ status: "ok" });
   });
   app.route("/api/auth", auth.routes);
@@ -25,7 +33,11 @@ export function createApi(db: AppDatabase, config: AuthConfig, request?: FetchDi
   app.onError((_error, c) => {
     c.header("Cache-Control", "no-store");
     if (_error instanceof ApiError) return c.json({ error: _error.message }, _error.status);
-    return c.json({ error: "Internal server error" }, 500);
+    const requestId = c.get("requestId");
+    const event = { event: "request_failed", timestamp: new Date().toISOString(), requestId, method: c.req.method,
+      route: c.req.routePath, ...errorDetails(_error) };
+    (operations.log ?? (event => console.error(JSON.stringify(event))))(event);
+    return c.json({ error: "Internal server error", requestId }, 500);
   });
   return app;
 }

src/auth/index.ts

@@ -12,7 +12,7 @@ import { fetchDiscordProfile, type FetchDiscord } from "./discord";
 const { users, sessions } = schema;
 const SESSION_SECONDS = 30 * 24 * 60 * 60;
 const STATE_SECONDS = 10 * 60;
-export type AuthEnv = { Variables: { user: PublicUser } };
+export type AuthEnv = { Variables: { user: PublicUser; requestId: string } };
 export type AppDatabase = BunSQLiteDatabase<typeof schema>;
 export const hashToken = (token: string) => createHash("sha256").update(token).digest("hex");
 const randomToken = () => randomBytes(32).toString("base64url");

src/index.ts

@@ -9,7 +9,7 @@ import index from "./index.html";
 const backups = process.env.NODE_ENV === "production" ? startBackups(sqlite, backupConfig(databasePath)) : undefined;
 if (import.meta.hot) import.meta.hot.dispose(() => backups?.stop());
 
-const app = createApi(db, readAuthConfig(), undefined, undefined, undefined, readDiscordSharingConfig());
+const app = createApi(db, readAuthConfig(), undefined, undefined, undefined, readDiscordSharingConfig(), { healthy: () => !backups?.state.enabled || (!backups.state.failed && backups.state.lastSuccess > 0) });
 
 const server = Bun.serve({
   hostname: "127.0.0.1",

src/lib/dashboard.ts

@@ -20,7 +20,7 @@ export async function habitRequest<T>(
       throw new Error("Your session has expired. Sign in again to continue.");
     const body = await response.json().catch(() => null);
     throw new Error(
-      body?.error || "Could not reach your workspace. Please try again.",
+      (body?.error || "Could not reach your workspace. Please try again.") + (response.status >= 500 && body?.requestId ? ` Reference: ${body.requestId}` : ""),
     );
   }
   return response.status === 204

src/ops/monitor.test.ts

@@ -0,0 +1,39 @@
+import { expect, test } from "bun:test";
+import { fixture } from "../habits/test-fixture";
+import { createApi } from "../api";
+import { healthObserver, probeHealth } from "./monitor";
+
+test("unexpected database errors are correlated without logging secrets", async () => {
+  const f = fixture(); const events: Record<string, unknown>[] = [];
+  const app = createApi(f.db, { origin: f.origin, clientId: "", clientSecret: "", cookieSecret: "test-secret-at-least-32-characters" }, undefined, undefined, undefined, undefined, { log: event => events.push(event) });
+  f.close();
+  const response = await app.request(`${f.origin}/api/health?token=private-secret`, { headers: { Cookie: "private-cookie", "X-Request-ID": "untrusted" } });
+  expect(response.status).toBe(500);
+  const body = await response.json();
+  expect(body.requestId).toBe(response.headers.get("x-request-id"));
+  expect(body.requestId).not.toBe("untrusted");
+  expect(events[0]?.requestId).toBe(body.requestId);
+  expect(events[0]?.type).toBeDefined();
+  expect(JSON.stringify(events)).not.toContain("private");
+});
+
+test("failed backups degrade health, and probes validate status and body", async () => {
+  const f = fixture();
+  try {
+    const app = createApi(f.db, { origin: f.origin, clientId: "", clientSecret: "", cookieSecret: "test-secret-at-least-32-characters" }, undefined, undefined, undefined, undefined, { healthy: () => false });
+    const response = await app.request(`${f.origin}/api/health`);
+    expect(response.status).toBe(503); expect(await response.json()).toEqual({ status: "degraded" });
+    const server = Bun.serve({ hostname: "127.0.0.1", port: 0, fetch: () => Response.json({ status: "ok" }) });
+    try { expect((await probeHealth(String(server.url))).healthy).toBe(true); } finally { await server.stop(true); }
+    expect((await probeHealth(String(server.url))).healthy).toBe(false);
+  } finally { f.close(); }
+});
+
+test("monitor reports sustained failure and recovery once per transition", () => {
+  const events: Record<string, unknown>[] = [];
+  const observe = healthObserver(event => events.push(event));
+  observe({ healthy: true, reason: "ok" });
+  for (let i = 0; i < 5; i++) observe({ healthy: false, reason: "HTTP 503" });
+  observe({ healthy: true, reason: "ok" }); observe({ healthy: true, reason: "ok" });
+  expect(events.map(event => event.event)).toEqual(["monitor_started", "health_failed", "health_recovered"]);
+});

src/ops/monitor.ts

@@ -0,0 +1,34 @@
+export type HealthResult = { healthy: boolean; reason: string };
+export async function probeHealth(url: string, request: typeof fetch = fetch): Promise<HealthResult> {
+  try {
+    const response = await request(url, { signal: AbortSignal.timeout(10000), redirect: "error", cache: "no-store" });
+    if (!response.ok) return { healthy: false, reason: `HTTP ${response.status}` };
+    const body = await response.json() as { status?: string };
+    return body.status === "ok" ? { healthy: true, reason: "ok" } : { healthy: false, reason: "Invalid health response" };
+  } catch { return { healthy: false, reason: "Health check unavailable" }; }
+}
+export function healthObserver(emit: (event: Record<string, unknown>) => void, threshold = 3) {
+  let failures = 0;
+  let incident = false;
+  let initialized = false;
+  return (result: HealthResult) => {
+    failures = result.healthy ? 0 : failures + 1;
+    const failed = failures >= threshold;
+    const event = !initialized && result.healthy ? "monitor_started" : failed && !incident ? "health_failed" : incident && result.healthy ? "health_recovered" : null;
+    if (event) emit({ event, timestamp: new Date().toISOString(), reason: result.reason, failures });
+    if (failed) incident = true;
+    if (result.healthy) incident = false;
+    initialized = true;
+  };
+}
+
+/** Keep secrets, SQL values, request bodies, OAuth queries and cookies out of logs. */
+export function errorDetails(error: Error) {
+  return {
+    type: ["Error", "TypeError", "RangeError", "SyntaxError", "SQLiteError"].includes(error.name) ? error.name : "Error",
+    frames: (error.stack ?? "").split("\n").slice(1).flatMap(line => {
+      const location = line.match(/([A-Za-z0-9_.-]+\.(?:js|ts|tsx):\d+:\d+)/);
+      return location ? [location[1]] : [];
+    }).slice(0, 8),
+  };
+}