From 661bc6c346290ff85784c9673ccf29b9ddff382e Mon Sep 17 00:00:00 2001 From: syntaxbullet Date: Fri, 4 Sep 2026 18:28:28 +0200 Subject: [PATCH] feat: automate verified SQLite backups and safe recovery --- .env.example | 7 +++ docs/OPERATIONS.md | 45 +++++++++++++++++++ package.json | 4 +- scripts/backup.ts | 5 +++ scripts/migrate.ts | 6 +++ scripts/restore.ts | 5 +++ scripts/start.ts | 4 ++ src/index.ts | 7 ++- src/ops/backups.test.ts | 50 +++++++++++++++++++++ src/ops/backups.ts | 96 +++++++++++++++++++++++++++++++++++++++++ 10 files changed, 227 insertions(+), 2 deletions(-) create mode 100644 docs/OPERATIONS.md create mode 100644 scripts/backup.ts create mode 100644 scripts/restore.ts create mode 100644 src/ops/backups.test.ts create mode 100644 src/ops/backups.ts diff --git a/.env.example b/.env.example index 375e515..09f4da7 100644 --- a/.env.example +++ b/.env.example @@ -16,3 +16,10 @@ DISCORD_BOT_TOKEN= DISCORD_SHARING_CHANNEL_ID= # Optional comma-separated Discord user IDs permitted to post. Empty permits all accounts. DISCORD_SHARING_ALLOWED_USER_IDS= +# Production snapshots, plus a snapshot before migrations. Paths resolve beside DATABASE_PATH. +BACKUP_ENABLED=true +BACKUP_DIR=backups +BACKUP_INTERVAL_HOURS=24 +BACKUP_RETAIN=7 +# Existing mounted off-machine backup directory (absolute path recommended). +BACKUP_REPLICA_DIR= diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md new file mode 100644 index 0000000..280177c --- /dev/null +++ b/docs/OPERATIONS.md @@ -0,0 +1,45 @@ +# Deployment operations + +## Backups and recovery + +Production creates a verified SQLite snapshot before migrations and automatically +when the latest snapshot is older than `BACKUP_INTERVAL_HOURS` (24 by default). +The server checks once a minute. Snapshot files are private (0600), created with +SQLite `VACUUM INTO`, and checked for integrity and foreign-key errors before being +published. Seven snapshots are retained by default; `BACKUP_RETAIN` accepts 2–365. +Pre-migration snapshots are mandatory; a failure prevents migration. Set +`BACKUP_ENABLED=false` only to disable the periodic runner if an external backup +system already handles it. Manual `bun run db:backup` always takes a snapshot. + +`BACKUP_DIR` defaults to `backups` beside the database. Relative backup paths are +resolved relative to the database directory, consistently in CLI and production. +Use a durable volume for both the database and backups. Configure +`BACKUP_REPLICA_DIR` to an existing mounted directory on a separate machine/storage +service to keep an off-machine copy. The app does not provision or authenticate +that storage. If the directory is unavailable, backup reports failure and does +not prune old snapshots. Local-only backups do not protect against machine loss. +Snapshots run synchronously; for larger databases use the backup CLI from an +external scheduler and disable the in-process periodic runner. + +To rehearse a restore without touching the live database: + +```sh +bun run db:backup +bun run db:restore /absolute/path/to/backup.sqlite /absolute/path/to/recovered.sqlite +``` + +Restore requires a new destination, verifies integrity, and revokes all sessions +in the restored copy. It never overwrites a database or its WAL/SHM sidecars. +Test the recovered copy with the matching release and an isolated local port. +For an actual recovery, stop the service, set `DATABASE_PATH` to the recovered +file, then start the service and sign in again. Keep the original file for rollback. +Do not copy only the live `.sqlite` file: recent commits may be in its WAL. + +Backups include account data and session hashes. Protect backup storage and expire +copies according to your published retention period. Restoring an older backup +can restore deleted accounts; reapply deletions performed since the snapshot +before reopening access. Discord images already posted are separate from app data. + +Verification: `bun test src/ops/backups.test.ts` exercises WAL progress, restored +content, revoked sessions, replicas, retention, missing storage, and refusal to +overwrite an existing destination. diff --git a/package.json b/package.json index 02363a4..dabcb9c 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,9 @@ "test": "bun test", "db:migrate:production": "NODE_ENV=production bun --env-file=.env --env-file=.env.production scripts/migrate.ts", "test:coverage": "bun test --coverage", - "test:smoke": "bun run build && bun scripts/api-smoke.ts" + "test:smoke": "bun run build && bun scripts/api-smoke.ts", + "db:backup": "NODE_ENV=production bun --env-file=.env --env-file=.env.production scripts/backup.ts", + "db:restore": "bun scripts/restore.ts" }, "dependencies": { "@nivo/calendar": "^0.99.0", diff --git a/scripts/backup.ts b/scripts/backup.ts new file mode 100644 index 0000000..a0396be --- /dev/null +++ b/scripts/backup.ts @@ -0,0 +1,5 @@ +import { Database } from "bun:sqlite"; +import { databasePath } from "../src/db/config"; +import { backupConfig, createBackup } from "../src/ops/backups"; +const db = new Database(databasePath, { readonly: true, strict: true }); +try { console.log(createBackup(db, backupConfig(databasePath))); } finally { db.close(); } diff --git a/scripts/migrate.ts b/scripts/migrate.ts index 4f8daf3..6205fce 100644 --- a/scripts/migrate.ts +++ b/scripts/migrate.ts @@ -1,7 +1,13 @@ +import { backupConfig, createBackup } from "../src/ops/backups"; +import { databasePath } from "../src/db/config"; import { migrate } from "drizzle-orm/bun-sqlite/migrator"; import { db, sqlite } from "../src/db"; try { + if (process.env.NODE_ENV === "production" && sqlite.query("SELECT name FROM sqlite_master WHERE name = 'habits'").get()) { + // Fail before migrations if the safety snapshot cannot be created. + createBackup(sqlite, backupConfig(databasePath)); + } migrate(db, { migrationsFolder: "./drizzle" }); console.log("Database migrations applied."); } finally { diff --git a/scripts/restore.ts b/scripts/restore.ts new file mode 100644 index 0000000..c9847c6 --- /dev/null +++ b/scripts/restore.ts @@ -0,0 +1,5 @@ +import { resolve } from "node:path"; +import { restoreBackup } from "../src/ops/backups"; +const [source, destination] = process.argv.slice(2); +if (!source || !destination || process.argv.length !== 4) throw new Error("Usage: bun scripts/restore.ts BACKUP NEW_DATABASE_PATH"); +console.log(`Restored and verified: ${restoreBackup(resolve(source), resolve(destination))}. All sessions revoked.`); diff --git a/scripts/start.ts b/scripts/start.ts index 2a8e252..87af03d 100644 --- a/scripts/start.ts +++ b/scripts/start.ts @@ -1,9 +1,13 @@ +import { backupConfig } from "../src/ops/backups"; import { databasePath } from "../src/db/config"; import { fileURLToPath } from "node:url"; // Bun's built HTML assets resolve from dist; keep the database path absolute. process.env.DATABASE_PATH = databasePath; process.env.NODE_ENV = "production"; +const backups = backupConfig(databasePath); +process.env.BACKUP_DIR = backups.directory; +if (backups.replica) process.env.BACKUP_REPLICA_DIR = backups.replica; process.chdir(fileURLToPath(new URL("../dist", import.meta.url))); const entrypoint = new URL("../dist/index.js", import.meta.url).href; await import(entrypoint); diff --git a/src/index.ts b/src/index.ts index 85e8a89..4ff24ec 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,9 +1,14 @@ -import { db } from "./db"; +import { backupConfig, startBackups } from "./ops/backups"; +import { databasePath } from "./db/config"; +import { db, sqlite } from "./db"; import { createApi } from "./api"; import { readAuthConfig } from "./auth/config"; import { readDiscordSharingConfig } from "./sharing/config"; import index from "./index.html"; +const backups = process.env.NODE_ENV === "production" ? startBackups(sqlite, backupConfig(databasePath)) : undefined; +if (import.meta.hot) import.meta.hot.dispose(() => backups?.stop()); + const app = createApi(db, readAuthConfig(), undefined, undefined, undefined, readDiscordSharingConfig()); const server = Bun.serve({ diff --git a/src/ops/backups.test.ts b/src/ops/backups.test.ts new file mode 100644 index 0000000..9389f83 --- /dev/null +++ b/src/ops/backups.test.ts @@ -0,0 +1,50 @@ +import { expect, test } from "bun:test"; +import { Database } from "bun:sqlite"; +import { mkdtempSync, mkdirSync, readdirSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { fixture } from "../habits/test-fixture"; +import { backupConfig, createBackup, latestBackupTime, restoreBackup, startBackups } from "./backups"; + +test("WAL snapshots restore committed progress, revoke sessions, replicate, and retain only owned snapshots", async () => { + const directory = mkdtempSync(join(tmpdir(), "minabot-backup-")); + const f = fixture(join(directory, "live.sqlite")); + try { + f.sqlite.exec("PRAGMA journal_mode=WAL"); + const config = backupConfig(join(directory, "live.sqlite"), { BACKUP_RETAIN: "2", BACKUP_REPLICA_DIR: "replica" }); + mkdirSync(config.replica!); + const habit = await f.json("/habits", "POST", { name: "Water", method: "count", target: 8, unit: "cups" }, 201); + await f.json(`/habits/${habit.id}/days/2026-09-04/progress`, "PUT", { count: 6 }); + const first = createBackup(f.sqlite, config); + expect(statSync(first).mode & 0o777).toBe(0o600); + expect(latestBackupTime(config)).toBeGreaterThan(0); + const destination = join(directory, "restored.sqlite"); + restoreBackup(first, destination); + const restored = new Database(destination, { readonly: true }); + try { + expect(restored.query("SELECT count FROM habit_days").get()).toEqual({ count: 6 }); + expect(restored.query("SELECT * FROM sessions").all()).toEqual([]); + expect(restored.query("PRAGMA integrity_check").get()).toEqual({ integrity_check: "ok" }); + } finally { restored.close(); } + expect(() => restoreBackup(first, destination)).toThrow("new path"); + writeFileSync(join(config.directory, "keep.txt"), "unrelated"); + createBackup(f.sqlite, config); createBackup(f.sqlite, config); + expect(readdirSync(config.directory).filter(name => name.endsWith(".sqlite"))).toHaveLength(2); + expect(readdirSync(config.replica!)).toHaveLength(2); + expect(readdirSync(config.directory)).toContain("keep.txt"); + const runner = startBackups(f.sqlite, config); + expect(runner.state.failed).toBe(false); expect(runner.state.lastSuccess).toBeGreaterThan(0); runner.stop(); + } finally { f.close(); rmSync(directory, { recursive: true, force: true }); } +}); + +test("invalid backups and missing replica mounts fail without overwriting data", () => { + const directory = mkdtempSync(join(tmpdir(), "minabot-backup-failure-")); + const f = fixture(); + try { + const bad = join(directory, "bad.sqlite"); writeFileSync(bad, "not a database"); + expect(() => restoreBackup(bad, join(directory, "target.sqlite"))).toThrow(); + const config = backupConfig(join(directory, "live.sqlite"), { BACKUP_REPLICA_DIR: "missing" }); + expect(() => createBackup(f.sqlite, config)).toThrow(); + expect(latestBackupTime(config)).toBe(0); + } finally { f.close(); rmSync(directory, { recursive: true, force: true }); } +}); diff --git a/src/ops/backups.ts b/src/ops/backups.ts new file mode 100644 index 0000000..0dabd90 --- /dev/null +++ b/src/ops/backups.ts @@ -0,0 +1,96 @@ +import { Database } from "bun:sqlite"; +import { chmodSync, closeSync, constants, copyFileSync, existsSync, fsyncSync, linkSync, mkdirSync, openSync, readdirSync, renameSync, rmSync, statSync, unlinkSync } from "node:fs"; +import { dirname, join, resolve } from "node:path"; + +export type BackupConfig = { directory: string; replica?: string; retain: number; intervalMs: number; enabled: boolean }; +const filenamePattern = /^minabot-\d{4}-\d{2}-\d{2}T[\d-]+Z-[a-f0-9-]+\.sqlite$/; +export function backupConfig(database: string, env = process.env): BackupConfig { + const retain = Number(env.BACKUP_RETAIN ?? 7); + const hours = Number(env.BACKUP_INTERVAL_HOURS ?? 24); + if (!Number.isInteger(retain) || retain < 2 || retain > 365 || !Number.isFinite(hours) || hours < 1 || hours > 168) + throw new Error("BACKUP_RETAIN must be 2–365 and BACKUP_INTERVAL_HOURS must be 1–168"); + const base = dirname(database); + const directory = resolve(base, env.BACKUP_DIR ?? "backups"); + const replica = env.BACKUP_REPLICA_DIR ? resolve(base, env.BACKUP_REPLICA_DIR) : undefined; + if (replica === directory) throw new Error("The backup replica must use a separate directory"); + return { directory, replica, retain, intervalMs: hours * 3600000, enabled: env.BACKUP_ENABLED !== "false" }; +} +function durable(path: string) { + chmodSync(path, 0o600); + const fd = openSync(path, "r"); try { fsyncSync(fd); } finally { closeSync(fd); } +} +export function verifyBackup(path: string) { + const db = new Database(path, { readonly: true, strict: true }); + try { + const integrity = db.query("PRAGMA integrity_check").all() as { integrity_check: string }[]; + if (integrity.length !== 1 || integrity[0]?.integrity_check !== "ok" || db.query("PRAGMA foreign_key_check").all().length) + throw new Error("Backup integrity check failed"); + db.query("SELECT id FROM users LIMIT 1").all(); + db.query("SELECT id FROM habits LIMIT 1").all(); + db.query("SELECT hash FROM __drizzle_migrations LIMIT 1").all(); + } finally { db.close(); } +} +function snapshots(directory: string) { + if (!existsSync(directory)) return []; + return readdirSync(directory).filter(name => filenamePattern.test(name)).sort().reverse(); +} +export function latestBackupTime(config: BackupConfig) { + const latest = snapshots(config.directory)[0]; + if (!latest || (config.replica && !existsSync(join(config.replica, latest)))) return 0; + return statSync(join(config.directory, latest)).mtimeMs; +} +export function createBackup(database: Database, config: BackupConfig) { + const name = `minabot-${new Date().toISOString().replace(/[:.]/g, "-")}-${crypto.randomUUID()}.sqlite`; + mkdirSync(config.directory, { recursive: true, mode: 0o700 }); + const path = join(config.directory, name); + const staging = `${path}.partial`; + mkdirSync(staging, { mode: 0o700 }); + const temporary = join(staging, "snapshot.sqlite"); + try { + // VACUUM INTO includes committed WAL transactions in one consistent snapshot. + // SQLite creates the output inside a private staging directory. + database.query("VACUUM INTO ?").run(temporary); + verifyBackup(temporary); durable(temporary); renameSync(temporary, path); + if (config.replica) { + // Require the configured mount to exist; never silently create an absent mount. + if (!statSync(config.replica).isDirectory()) throw new Error("Backup replica directory unavailable"); + const replica = join(config.replica, name); + copyFileSync(path, `${replica}.partial`, constants.COPYFILE_EXCL); + durable(`${replica}.partial`); verifyBackup(`${replica}.partial`); renameSync(`${replica}.partial`, replica); + } + for (const directory of [config.directory, ...(config.replica ? [config.replica] : [])]) + for (const old of snapshots(directory).slice(config.retain)) unlinkSync(join(directory, old)); + return path; + } finally { rmSync(staging, { recursive: true, force: true }); } +} +export function restoreBackup(source: string, destination: string) { + if (existsSync(destination) || existsSync(`${destination}-wal`) || existsSync(`${destination}-shm`)) + throw new Error("Restore destination must be a new path with no SQLite sidecars"); + verifyBackup(source); + mkdirSync(dirname(destination), { recursive: true, mode: 0o700 }); + const temporary = `${destination}.${crypto.randomUUID()}.partial`; + try { + copyFileSync(source, temporary, constants.COPYFILE_EXCL); chmodSync(temporary, 0o600); + const restored = new Database(temporary); + try { restored.exec("PRAGMA journal_mode=DELETE; DELETE FROM sessions;"); } finally { restored.close(); } + verifyBackup(temporary); durable(temporary); + // Link is exclusive: a concurrently-created destination is never overwritten. + linkSync(temporary, destination); + } finally { if (existsSync(temporary)) unlinkSync(temporary); } + return destination; +} +export function startBackups(database: Database, config: BackupConfig) { + const state = { enabled: config.enabled, lastSuccess: 0, failed: false }; + if (!config.enabled) return { state, stop() {} }; + const tick = () => { + try { + const latest = latestBackupTime(config); + if (latest && Date.now() - latest < config.intervalMs) { state.lastSuccess = latest; state.failed = false; return; } + createBackup(database, config); state.lastSuccess = Date.now(); state.failed = false; + console.log(JSON.stringify({ event: "backup_completed", timestamp: new Date().toISOString() })); + } catch { state.failed = true; console.error(JSON.stringify({ event: "backup_failed", timestamp: new Date().toISOString() })); } + }; + tick(); + const timer = setInterval(tick, 60000); timer.unref(); + return { state, stop: () => clearInterval(timer) }; +}