feat: automate verified SQLite backups and safe recovery

.env.example

@@ -16,3 +16,10 @@ DISCORD_BOT_TOKEN=
 DISCORD_SHARING_CHANNEL_ID=
 # Optional comma-separated Discord user IDs permitted to post. Empty permits all accounts.
 DISCORD_SHARING_ALLOWED_USER_IDS=
+# Production snapshots, plus a snapshot before migrations. Paths resolve beside DATABASE_PATH.
+BACKUP_ENABLED=true
+BACKUP_DIR=backups
+BACKUP_INTERVAL_HOURS=24
+BACKUP_RETAIN=7
+# Existing mounted off-machine backup directory (absolute path recommended).
+BACKUP_REPLICA_DIR=

docs/OPERATIONS.md

@@ -0,0 +1,45 @@
+# Deployment operations
+
+## Backups and recovery
+
+Production creates a verified SQLite snapshot before migrations and automatically
+when the latest snapshot is older than `BACKUP_INTERVAL_HOURS` (24 by default).
+The server checks once a minute. Snapshot files are private (0600), created with
+SQLite `VACUUM INTO`, and checked for integrity and foreign-key errors before being
+published. Seven snapshots are retained by default; `BACKUP_RETAIN` accepts 2–365.
+Pre-migration snapshots are mandatory; a failure prevents migration. Set
+`BACKUP_ENABLED=false` only to disable the periodic runner if an external backup
+system already handles it. Manual `bun run db:backup` always takes a snapshot.
+
+`BACKUP_DIR` defaults to `backups` beside the database. Relative backup paths are
+resolved relative to the database directory, consistently in CLI and production.
+Use a durable volume for both the database and backups. Configure
+`BACKUP_REPLICA_DIR` to an existing mounted directory on a separate machine/storage
+service to keep an off-machine copy. The app does not provision or authenticate
+that storage. If the directory is unavailable, backup reports failure and does
+not prune old snapshots. Local-only backups do not protect against machine loss.
+Snapshots run synchronously; for larger databases use the backup CLI from an
+external scheduler and disable the in-process periodic runner.
+
+To rehearse a restore without touching the live database:
+
+```sh
+bun run db:backup
+bun run db:restore /absolute/path/to/backup.sqlite /absolute/path/to/recovered.sqlite
+```
+
+Restore requires a new destination, verifies integrity, and revokes all sessions
+in the restored copy. It never overwrites a database or its WAL/SHM sidecars.
+Test the recovered copy with the matching release and an isolated local port.
+For an actual recovery, stop the service, set `DATABASE_PATH` to the recovered
+file, then start the service and sign in again. Keep the original file for rollback.
+Do not copy only the live `.sqlite` file: recent commits may be in its WAL.
+
+Backups include account data and session hashes. Protect backup storage and expire
+copies according to your published retention period. Restoring an older backup
+can restore deleted accounts; reapply deletions performed since the snapshot
+before reopening access. Discord images already posted are separate from app data.
+
+Verification: `bun test src/ops/backups.test.ts` exercises WAL progress, restored
+content, revoked sessions, replicas, retention, missing storage, and refusal to
+overwrite an existing destination.

package.json

@@ -13,7 +13,9 @@
     "test": "bun test",
     "db:migrate:production": "NODE_ENV=production bun --env-file=.env --env-file=.env.production scripts/migrate.ts",
     "test:coverage": "bun test --coverage",
-    "test:smoke": "bun run build && bun scripts/api-smoke.ts"
+    "test:smoke": "bun run build && bun scripts/api-smoke.ts",
+    "db:backup": "NODE_ENV=production bun --env-file=.env --env-file=.env.production scripts/backup.ts",
+    "db:restore": "bun scripts/restore.ts"
   },
   "dependencies": {
     "@nivo/calendar": "^0.99.0",

scripts/backup.ts

@@ -0,0 +1,5 @@
+import { Database } from "bun:sqlite";
+import { databasePath } from "../src/db/config";
+import { backupConfig, createBackup } from "../src/ops/backups";
+const db = new Database(databasePath, { readonly: true, strict: true });
+try { console.log(createBackup(db, backupConfig(databasePath))); } finally { db.close(); }

scripts/migrate.ts

@@ -1,7 +1,13 @@
+import { backupConfig, createBackup } from "../src/ops/backups";
+import { databasePath } from "../src/db/config";
 import { migrate } from "drizzle-orm/bun-sqlite/migrator";
 import { db, sqlite } from "../src/db";
 
 try {
+  if (process.env.NODE_ENV === "production" && sqlite.query("SELECT name FROM sqlite_master WHERE name = 'habits'").get()) {
+    // Fail before migrations if the safety snapshot cannot be created.
+    createBackup(sqlite, backupConfig(databasePath));
+  }
   migrate(db, { migrationsFolder: "./drizzle" });
   console.log("Database migrations applied.");
 } finally {

scripts/restore.ts

@@ -0,0 +1,5 @@
+import { resolve } from "node:path";
+import { restoreBackup } from "../src/ops/backups";
+const [source, destination] = process.argv.slice(2);
+if (!source || !destination || process.argv.length !== 4) throw new Error("Usage: bun scripts/restore.ts BACKUP NEW_DATABASE_PATH");
+console.log(`Restored and verified: ${restoreBackup(resolve(source), resolve(destination))}. All sessions revoked.`);

scripts/start.ts

@@ -1,9 +1,13 @@
+import { backupConfig } from "../src/ops/backups";
 import { databasePath } from "../src/db/config";
 import { fileURLToPath } from "node:url";
 
 // Bun's built HTML assets resolve from dist; keep the database path absolute.
 process.env.DATABASE_PATH = databasePath;
 process.env.NODE_ENV = "production";
+const backups = backupConfig(databasePath);
+process.env.BACKUP_DIR = backups.directory;
+if (backups.replica) process.env.BACKUP_REPLICA_DIR = backups.replica;
 process.chdir(fileURLToPath(new URL("../dist", import.meta.url)));
 const entrypoint = new URL("../dist/index.js", import.meta.url).href;
 await import(entrypoint);

src/index.ts

@@ -1,9 +1,14 @@
-import { db } from "./db";
+import { backupConfig, startBackups } from "./ops/backups";
+import { databasePath } from "./db/config";
+import { db, sqlite } from "./db";
 import { createApi } from "./api";
 import { readAuthConfig } from "./auth/config";
 import { readDiscordSharingConfig } from "./sharing/config";
 import index from "./index.html";
 
+const backups = process.env.NODE_ENV === "production" ? startBackups(sqlite, backupConfig(databasePath)) : undefined;
+if (import.meta.hot) import.meta.hot.dispose(() => backups?.stop());
+
 const app = createApi(db, readAuthConfig(), undefined, undefined, undefined, readDiscordSharingConfig());
 
 const server = Bun.serve({

src/ops/backups.test.ts

@@ -0,0 +1,50 @@
+import { expect, test } from "bun:test";
+import { Database } from "bun:sqlite";
+import { mkdtempSync, mkdirSync, readdirSync, rmSync, statSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import { fixture } from "../habits/test-fixture";
+import { backupConfig, createBackup, latestBackupTime, restoreBackup, startBackups } from "./backups";
+
+test("WAL snapshots restore committed progress, revoke sessions, replicate, and retain only owned snapshots", async () => {
+  const directory = mkdtempSync(join(tmpdir(), "minabot-backup-"));
+  const f = fixture(join(directory, "live.sqlite"));
+  try {
+    f.sqlite.exec("PRAGMA journal_mode=WAL");
+    const config = backupConfig(join(directory, "live.sqlite"), { BACKUP_RETAIN: "2", BACKUP_REPLICA_DIR: "replica" });
+    mkdirSync(config.replica!);
+    const habit = await f.json("/habits", "POST", { name: "Water", method: "count", target: 8, unit: "cups" }, 201);
+    await f.json(`/habits/${habit.id}/days/2026-09-04/progress`, "PUT", { count: 6 });
+    const first = createBackup(f.sqlite, config);
+    expect(statSync(first).mode & 0o777).toBe(0o600);
+    expect(latestBackupTime(config)).toBeGreaterThan(0);
+    const destination = join(directory, "restored.sqlite");
+    restoreBackup(first, destination);
+    const restored = new Database(destination, { readonly: true });
+    try {
+      expect(restored.query("SELECT count FROM habit_days").get()).toEqual({ count: 6 });
+      expect(restored.query("SELECT * FROM sessions").all()).toEqual([]);
+      expect(restored.query("PRAGMA integrity_check").get()).toEqual({ integrity_check: "ok" });
+    } finally { restored.close(); }
+    expect(() => restoreBackup(first, destination)).toThrow("new path");
+    writeFileSync(join(config.directory, "keep.txt"), "unrelated");
+    createBackup(f.sqlite, config); createBackup(f.sqlite, config);
+    expect(readdirSync(config.directory).filter(name => name.endsWith(".sqlite"))).toHaveLength(2);
+    expect(readdirSync(config.replica!)).toHaveLength(2);
+    expect(readdirSync(config.directory)).toContain("keep.txt");
+    const runner = startBackups(f.sqlite, config);
+    expect(runner.state.failed).toBe(false); expect(runner.state.lastSuccess).toBeGreaterThan(0); runner.stop();
+  } finally { f.close(); rmSync(directory, { recursive: true, force: true }); }
+});
+
+test("invalid backups and missing replica mounts fail without overwriting data", () => {
+  const directory = mkdtempSync(join(tmpdir(), "minabot-backup-failure-"));
+  const f = fixture();
+  try {
+    const bad = join(directory, "bad.sqlite"); writeFileSync(bad, "not a database");
+    expect(() => restoreBackup(bad, join(directory, "target.sqlite"))).toThrow();
+    const config = backupConfig(join(directory, "live.sqlite"), { BACKUP_REPLICA_DIR: "missing" });
+    expect(() => createBackup(f.sqlite, config)).toThrow();
+    expect(latestBackupTime(config)).toBe(0);
+  } finally { f.close(); rmSync(directory, { recursive: true, force: true }); }
+});

src/ops/backups.ts

@@ -0,0 +1,96 @@
+import { Database } from "bun:sqlite";
+import { chmodSync, closeSync, constants, copyFileSync, existsSync, fsyncSync, linkSync, mkdirSync, openSync, readdirSync, renameSync, rmSync, statSync, unlinkSync } from "node:fs";
+import { dirname, join, resolve } from "node:path";
+
+export type BackupConfig = { directory: string; replica?: string; retain: number; intervalMs: number; enabled: boolean };
+const filenamePattern = /^minabot-\d{4}-\d{2}-\d{2}T[\d-]+Z-[a-f0-9-]+\.sqlite$/;
+export function backupConfig(database: string, env = process.env): BackupConfig {
+  const retain = Number(env.BACKUP_RETAIN ?? 7);
+  const hours = Number(env.BACKUP_INTERVAL_HOURS ?? 24);
+  if (!Number.isInteger(retain) || retain < 2 || retain > 365 || !Number.isFinite(hours) || hours < 1 || hours > 168)
+    throw new Error("BACKUP_RETAIN must be 2–365 and BACKUP_INTERVAL_HOURS must be 1–168");
+  const base = dirname(database);
+  const directory = resolve(base, env.BACKUP_DIR ?? "backups");
+  const replica = env.BACKUP_REPLICA_DIR ? resolve(base, env.BACKUP_REPLICA_DIR) : undefined;
+  if (replica === directory) throw new Error("The backup replica must use a separate directory");
+  return { directory, replica, retain, intervalMs: hours * 3600000, enabled: env.BACKUP_ENABLED !== "false" };
+}
+function durable(path: string) {
+  chmodSync(path, 0o600);
+  const fd = openSync(path, "r"); try { fsyncSync(fd); } finally { closeSync(fd); }
+}
+export function verifyBackup(path: string) {
+  const db = new Database(path, { readonly: true, strict: true });
+  try {
+    const integrity = db.query("PRAGMA integrity_check").all() as { integrity_check: string }[];
+    if (integrity.length !== 1 || integrity[0]?.integrity_check !== "ok" || db.query("PRAGMA foreign_key_check").all().length)
+      throw new Error("Backup integrity check failed");
+    db.query("SELECT id FROM users LIMIT 1").all();
+    db.query("SELECT id FROM habits LIMIT 1").all();
+    db.query("SELECT hash FROM __drizzle_migrations LIMIT 1").all();
+  } finally { db.close(); }
+}
+function snapshots(directory: string) {
+  if (!existsSync(directory)) return [];
+  return readdirSync(directory).filter(name => filenamePattern.test(name)).sort().reverse();
+}
+export function latestBackupTime(config: BackupConfig) {
+  const latest = snapshots(config.directory)[0];
+  if (!latest || (config.replica && !existsSync(join(config.replica, latest)))) return 0;
+  return statSync(join(config.directory, latest)).mtimeMs;
+}
+export function createBackup(database: Database, config: BackupConfig) {
+  const name = `minabot-${new Date().toISOString().replace(/[:.]/g, "-")}-${crypto.randomUUID()}.sqlite`;
+  mkdirSync(config.directory, { recursive: true, mode: 0o700 });
+  const path = join(config.directory, name);
+  const staging = `${path}.partial`;
+  mkdirSync(staging, { mode: 0o700 });
+  const temporary = join(staging, "snapshot.sqlite");
+  try {
+    // VACUUM INTO includes committed WAL transactions in one consistent snapshot.
+    // SQLite creates the output inside a private staging directory.
+    database.query("VACUUM INTO ?").run(temporary);
+    verifyBackup(temporary); durable(temporary); renameSync(temporary, path);
+    if (config.replica) {
+      // Require the configured mount to exist; never silently create an absent mount.
+      if (!statSync(config.replica).isDirectory()) throw new Error("Backup replica directory unavailable");
+      const replica = join(config.replica, name);
+      copyFileSync(path, `${replica}.partial`, constants.COPYFILE_EXCL);
+      durable(`${replica}.partial`); verifyBackup(`${replica}.partial`); renameSync(`${replica}.partial`, replica);
+    }
+    for (const directory of [config.directory, ...(config.replica ? [config.replica] : [])])
+      for (const old of snapshots(directory).slice(config.retain)) unlinkSync(join(directory, old));
+    return path;
+  } finally { rmSync(staging, { recursive: true, force: true }); }
+}
+export function restoreBackup(source: string, destination: string) {
+  if (existsSync(destination) || existsSync(`${destination}-wal`) || existsSync(`${destination}-shm`))
+    throw new Error("Restore destination must be a new path with no SQLite sidecars");
+  verifyBackup(source);
+  mkdirSync(dirname(destination), { recursive: true, mode: 0o700 });
+  const temporary = `${destination}.${crypto.randomUUID()}.partial`;
+  try {
+    copyFileSync(source, temporary, constants.COPYFILE_EXCL); chmodSync(temporary, 0o600);
+    const restored = new Database(temporary);
+    try { restored.exec("PRAGMA journal_mode=DELETE; DELETE FROM sessions;"); } finally { restored.close(); }
+    verifyBackup(temporary); durable(temporary);
+    // Link is exclusive: a concurrently-created destination is never overwritten.
+    linkSync(temporary, destination);
+  } finally { if (existsSync(temporary)) unlinkSync(temporary); }
+  return destination;
+}
+export function startBackups(database: Database, config: BackupConfig) {
+  const state = { enabled: config.enabled, lastSuccess: 0, failed: false };
+  if (!config.enabled) return { state, stop() {} };
+  const tick = () => {
+    try {
+      const latest = latestBackupTime(config);
+      if (latest && Date.now() - latest < config.intervalMs) { state.lastSuccess = latest; state.failed = false; return; }
+      createBackup(database, config); state.lastSuccess = Date.now(); state.failed = false;
+      console.log(JSON.stringify({ event: "backup_completed", timestamp: new Date().toISOString() }));
+    } catch { state.failed = true; console.error(JSON.stringify({ event: "backup_failed", timestamp: new Date().toISOString() })); }
+  };
+  tick();
+  const timer = setInterval(tick, 60000); timer.unref();
+  return { state, stop: () => clearInterval(timer) };
+}