diff --git a/.env.example b/.env.example index 09f4da7..766fc7d 100644 --- a/.env.example +++ b/.env.example @@ -23,3 +23,5 @@ BACKUP_INTERVAL_HOURS=24 BACKUP_RETAIN=7 # Existing mounted off-machine backup directory (absolute path recommended). BACKUP_REPLICA_DIR= +# Optional public HTTPS health URL for the independent monitor. +MONITOR_URL= diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index 280177c..85e940b 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -43,3 +43,31 @@ before reopening access. Discord images already posted are separate from app dat Verification: `bun test src/ops/backups.test.ts` exercises WAL progress, restored content, revoked sessions, replicas, retention, missing storage, and refusal to overwrite an existing destination. + +## Monitoring + +Every API response includes a server-generated `X-Request-ID`. Unexpected errors +return that reference to the browser and write a JSON `request_failed` event to +stderr with the request ID, method, route template, error type, and stack locations. +Error messages, SQL values, request bodies, cookies, and query strings are omitted +so credentials and habit content do not enter logs. Production source maps in +`dist/` help map stack locations back to the source. Backup outcomes also emit +`backup_completed` and `backup_failed` events. + +`GET /api/health` checks SQLite and returns 503 if enabled automatic backups are +failing or have never succeeded. It returns no account data, paths, or credentials. +Run the health monitor independently of the application process: + +```sh +bun run monitor --once +bun run monitor +``` + +`MONITOR_URL` defaults to `http://127.0.0.1:$PORT/api/health`. For an outside-in +check, run the monitor on another machine and set it to the public HTTPS endpoint. +The one-shot command exits nonzero on failure. Continuous mode checks every minute, +times out after ten seconds, reports `health_failed` after three consecutive +failures, and emits one `health_recovered` event on recovery. It stays quiet while +state is unchanged. Run it under a process supervisor and route these JSON events +to your deployment platform's alerting/log collection. No external alert provider +or off-machine monitor is provisioned automatically by this repository. diff --git a/package.json b/package.json index dabcb9c..351601f 100644 --- a/package.json +++ b/package.json @@ -15,7 +15,8 @@ "test:coverage": "bun test --coverage", "test:smoke": "bun run build && bun scripts/api-smoke.ts", "db:backup": "NODE_ENV=production bun --env-file=.env --env-file=.env.production scripts/backup.ts", - "db:restore": "bun scripts/restore.ts" + "db:restore": "bun scripts/restore.ts", + "monitor": "bun --env-file=.env --env-file=.env.production scripts/monitor.ts" }, "dependencies": { "@nivo/calendar": "^0.99.0", diff --git a/scripts/monitor.ts b/scripts/monitor.ts new file mode 100644 index 0000000..025f8ea --- /dev/null +++ b/scripts/monitor.ts @@ -0,0 +1,21 @@ +import { probeHealth, healthObserver } from "../src/ops/monitor"; +const target = new URL(process.env.MONITOR_URL || `http://127.0.0.1:${process.env.PORT ?? 3000}/api/health`); +if (!["http:", "https:"].includes(target.protocol) || target.username || target.password) throw new Error("MONITOR_URL must be an HTTP(S) URL without credentials"); +const emit = (event: Record) => console.log(JSON.stringify(event)); +if (process.argv.includes("--once")) { + const result = await probeHealth(target.href); + emit({ event: "health_probe", ...result }); + process.exitCode = result.healthy ? 0 : 1; +} else { + const observe = healthObserver(emit); + let running = false; + const check = async () => { + if (running) return; + running = true; + try { observe(await probeHealth(target.href)); } finally { running = false; } + }; + await check(); + const timer = setInterval(() => void check(), 60000); + const stop = () => { clearInterval(timer); process.exit(0); }; + process.on("SIGTERM", stop); process.on("SIGINT", stop); +} diff --git a/src/api.ts b/src/api.ts index 6c51dfc..3e293da 100644 --- a/src/api.ts +++ b/src/api.ts @@ -1,3 +1,4 @@ +import { errorDetails } from "./ops/monitor"; import { createAccountRoutes } from "./account/routes"; import { createHabitRoutes } from "./habits/routes"; import { ApiError } from "./habits/service"; @@ -9,11 +10,18 @@ import type { FetchDiscord } from "./auth/discord"; import { createSharingRoutes, type DiscordFetch } from "./sharing/routes"; import type { DiscordSharingConfig } from "./sharing/config"; -export function createApi(db: AppDatabase, config: AuthConfig, request?: FetchDiscord, now?: () => number, discordRequest?: DiscordFetch, sharingConfig: DiscordSharingConfig = { token: "", channelId: "" }) { +export function createApi(db: AppDatabase, config: AuthConfig, request?: FetchDiscord, now?: () => number, discordRequest?: DiscordFetch, sharingConfig: DiscordSharingConfig = { token: "", channelId: "" }, operations: { healthy?: () => boolean; log?: (event: Record) => void } = {}) { const app = new Hono(); + app.use("*", async (c, next) => { + const requestId = crypto.randomUUID(); + c.set("requestId", requestId); c.header("X-Request-ID", requestId); + await next(); + }); const auth = createAuth(db, config, request, now); app.get("/api/health", c => { + c.header("Cache-Control", "no-store"); db.get(sql`SELECT 1`); + if (operations.healthy && !operations.healthy()) return c.json({ status: "degraded" }, 503); return c.json({ status: "ok" }); }); app.route("/api/auth", auth.routes); @@ -25,7 +33,11 @@ export function createApi(db: AppDatabase, config: AuthConfig, request?: FetchDi app.onError((_error, c) => { c.header("Cache-Control", "no-store"); if (_error instanceof ApiError) return c.json({ error: _error.message }, _error.status); - return c.json({ error: "Internal server error" }, 500); + const requestId = c.get("requestId"); + const event = { event: "request_failed", timestamp: new Date().toISOString(), requestId, method: c.req.method, + route: c.req.routePath, ...errorDetails(_error) }; + (operations.log ?? (event => console.error(JSON.stringify(event))))(event); + return c.json({ error: "Internal server error", requestId }, 500); }); return app; } diff --git a/src/auth/index.ts b/src/auth/index.ts index 3272b2c..d0ec127 100644 --- a/src/auth/index.ts +++ b/src/auth/index.ts @@ -12,7 +12,7 @@ import { fetchDiscordProfile, type FetchDiscord } from "./discord"; const { users, sessions } = schema; const SESSION_SECONDS = 30 * 24 * 60 * 60; const STATE_SECONDS = 10 * 60; -export type AuthEnv = { Variables: { user: PublicUser } }; +export type AuthEnv = { Variables: { user: PublicUser; requestId: string } }; export type AppDatabase = BunSQLiteDatabase; export const hashToken = (token: string) => createHash("sha256").update(token).digest("hex"); const randomToken = () => randomBytes(32).toString("base64url"); diff --git a/src/index.ts b/src/index.ts index 4ff24ec..a5378a3 100644 --- a/src/index.ts +++ b/src/index.ts @@ -9,7 +9,7 @@ import index from "./index.html"; const backups = process.env.NODE_ENV === "production" ? startBackups(sqlite, backupConfig(databasePath)) : undefined; if (import.meta.hot) import.meta.hot.dispose(() => backups?.stop()); -const app = createApi(db, readAuthConfig(), undefined, undefined, undefined, readDiscordSharingConfig()); +const app = createApi(db, readAuthConfig(), undefined, undefined, undefined, readDiscordSharingConfig(), { healthy: () => !backups?.state.enabled || (!backups.state.failed && backups.state.lastSuccess > 0) }); const server = Bun.serve({ hostname: "127.0.0.1", diff --git a/src/lib/dashboard.ts b/src/lib/dashboard.ts index ed3d50d..40c8002 100644 --- a/src/lib/dashboard.ts +++ b/src/lib/dashboard.ts @@ -20,7 +20,7 @@ export async function habitRequest( throw new Error("Your session has expired. Sign in again to continue."); const body = await response.json().catch(() => null); throw new Error( - body?.error || "Could not reach your workspace. Please try again.", + (body?.error || "Could not reach your workspace. Please try again.") + (response.status >= 500 && body?.requestId ? ` Reference: ${body.requestId}` : ""), ); } return response.status === 204 diff --git a/src/ops/monitor.test.ts b/src/ops/monitor.test.ts new file mode 100644 index 0000000..337d76a --- /dev/null +++ b/src/ops/monitor.test.ts @@ -0,0 +1,39 @@ +import { expect, test } from "bun:test"; +import { fixture } from "../habits/test-fixture"; +import { createApi } from "../api"; +import { healthObserver, probeHealth } from "./monitor"; + +test("unexpected database errors are correlated without logging secrets", async () => { + const f = fixture(); const events: Record[] = []; + const app = createApi(f.db, { origin: f.origin, clientId: "", clientSecret: "", cookieSecret: "test-secret-at-least-32-characters" }, undefined, undefined, undefined, undefined, { log: event => events.push(event) }); + f.close(); + const response = await app.request(`${f.origin}/api/health?token=private-secret`, { headers: { Cookie: "private-cookie", "X-Request-ID": "untrusted" } }); + expect(response.status).toBe(500); + const body = await response.json(); + expect(body.requestId).toBe(response.headers.get("x-request-id")); + expect(body.requestId).not.toBe("untrusted"); + expect(events[0]?.requestId).toBe(body.requestId); + expect(events[0]?.type).toBeDefined(); + expect(JSON.stringify(events)).not.toContain("private"); +}); + +test("failed backups degrade health, and probes validate status and body", async () => { + const f = fixture(); + try { + const app = createApi(f.db, { origin: f.origin, clientId: "", clientSecret: "", cookieSecret: "test-secret-at-least-32-characters" }, undefined, undefined, undefined, undefined, { healthy: () => false }); + const response = await app.request(`${f.origin}/api/health`); + expect(response.status).toBe(503); expect(await response.json()).toEqual({ status: "degraded" }); + const server = Bun.serve({ hostname: "127.0.0.1", port: 0, fetch: () => Response.json({ status: "ok" }) }); + try { expect((await probeHealth(String(server.url))).healthy).toBe(true); } finally { await server.stop(true); } + expect((await probeHealth(String(server.url))).healthy).toBe(false); + } finally { f.close(); } +}); + +test("monitor reports sustained failure and recovery once per transition", () => { + const events: Record[] = []; + const observe = healthObserver(event => events.push(event)); + observe({ healthy: true, reason: "ok" }); + for (let i = 0; i < 5; i++) observe({ healthy: false, reason: "HTTP 503" }); + observe({ healthy: true, reason: "ok" }); observe({ healthy: true, reason: "ok" }); + expect(events.map(event => event.event)).toEqual(["monitor_started", "health_failed", "health_recovered"]); +}); diff --git a/src/ops/monitor.ts b/src/ops/monitor.ts new file mode 100644 index 0000000..63689f1 --- /dev/null +++ b/src/ops/monitor.ts @@ -0,0 +1,34 @@ +export type HealthResult = { healthy: boolean; reason: string }; +export async function probeHealth(url: string, request: typeof fetch = fetch): Promise { + try { + const response = await request(url, { signal: AbortSignal.timeout(10000), redirect: "error", cache: "no-store" }); + if (!response.ok) return { healthy: false, reason: `HTTP ${response.status}` }; + const body = await response.json() as { status?: string }; + return body.status === "ok" ? { healthy: true, reason: "ok" } : { healthy: false, reason: "Invalid health response" }; + } catch { return { healthy: false, reason: "Health check unavailable" }; } +} +export function healthObserver(emit: (event: Record) => void, threshold = 3) { + let failures = 0; + let incident = false; + let initialized = false; + return (result: HealthResult) => { + failures = result.healthy ? 0 : failures + 1; + const failed = failures >= threshold; + const event = !initialized && result.healthy ? "monitor_started" : failed && !incident ? "health_failed" : incident && result.healthy ? "health_recovered" : null; + if (event) emit({ event, timestamp: new Date().toISOString(), reason: result.reason, failures }); + if (failed) incident = true; + if (result.healthy) incident = false; + initialized = true; + }; +} + +/** Keep secrets, SQL values, request bodies, OAuth queries and cookies out of logs. */ +export function errorDetails(error: Error) { + return { + type: ["Error", "TypeError", "RangeError", "SyntaxError", "SQLiteError"].includes(error.name) ? error.name : "Error", + frames: (error.stack ?? "").split("\n").slice(1).flatMap(line => { + const location = line.match(/([A-Za-z0-9_.-]+\.(?:js|ts|tsx):\d+:\d+)/); + return location ? [location[1]] : []; + }).slice(0, 8), + }; +}