feat: add health check command and tracking

.gitignore

@@ -44,3 +44,4 @@ report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json
 src/db/data
 src/db/log
 scratchpad/
+tickets/

drizzle/0002_fancy_forge.sql

@@ -0,0 +1,8 @@
+CREATE INDEX "moderation_cases_user_id_idx" ON "moderation_cases" USING btree ("user_id");--> statement-breakpoint
+CREATE INDEX "moderation_cases_case_id_idx" ON "moderation_cases" USING btree ("case_id");--> statement-breakpoint
+CREATE INDEX "transactions_created_at_idx" ON "transactions" USING btree ("created_at");--> statement-breakpoint
+CREATE INDEX "user_timers_expires_at_idx" ON "user_timers" USING btree ("expires_at");--> statement-breakpoint
+CREATE INDEX "user_timers_lookup_idx" ON "user_timers" USING btree ("user_id","type","key");--> statement-breakpoint
+CREATE INDEX "users_username_idx" ON "users" USING btree ("username");--> statement-breakpoint
+CREATE INDEX "users_balance_idx" ON "users" USING btree ("balance");--> statement-breakpoint
+CREATE INDEX "users_level_xp_idx" ON "users" USING btree ("level","xp");

drizzle/meta/_journal.json

@@ -15,6 +15,13 @@
       "when": 1766606046050,
       "tag": "0001_heavy_thundra",
       "breakpoints": true
+    },
+    {
+      "idx": 2,
+      "version": "7",
+      "when": 1767716705797,
+      "tag": "0002_fancy_forge",
+      "breakpoints": true
     }
   ]
 }

src/commands/admin/health.test.ts

@@ -0,0 +1,84 @@
+import { describe, test, expect, mock, beforeEach } from "bun:test";
+import { health } from "./health";
+import { ChatInputCommandInteraction, Colors } from "discord.js";
+import { AuroraClient } from "@/lib/BotClient";
+
+// Mock DrizzleClient
+const executeMock = mock(() => Promise.resolve());
+mock.module("@/lib/DrizzleClient", () => ({
+    DrizzleClient: {
+        execute: executeMock
+    }
+}));
+
+// Mock BotClient (already has lastCommandTimestamp if imported, but we might want to control it)
+AuroraClient.lastCommandTimestamp = 1641481200000; // Fixed timestamp for testing
+
+describe("Health Command", () => {
+    beforeEach(() => {
+        executeMock.mockClear();
+    });
+
+    test("should execute successfully and return health embed", async () => {
+        const interaction = {
+            deferReply: mock(() => Promise.resolve()),
+            editReply: mock(() => Promise.resolve()),
+            client: {
+                ws: {
+                    ping: 42
+                }
+            },
+            user: { id: "123", username: "testuser" },
+            commandName: "health"
+        } as unknown as ChatInputCommandInteraction;
+
+        await health.execute(interaction);
+
+        expect(interaction.deferReply).toHaveBeenCalled();
+        expect(executeMock).toHaveBeenCalled();
+        expect(interaction.editReply).toHaveBeenCalled();
+
+        const editReplyCall = (interaction.editReply as any).mock.calls[0][0];
+        const embed = editReplyCall.embeds[0];
+
+        expect(embed.data.title).toBe("System Health Status");
+        expect(embed.data.color).toBe(Colors.Aqua);
+
+        // Check fields
+        const fields = embed.data.fields;
+        expect(fields).toBeDefined();
+
+        // Connectivity field
+        const connectivityField = fields.find((f: any) => f.name === "📡 Connectivity");
+        expect(connectivityField.value).toContain("42ms");
+        expect(connectivityField.value).toContain("Connected");
+
+        // Activity field
+        const activityField = fields.find((f: any) => f.name === "⌨️ Activity");
+        expect(activityField.value).toContain("R>"); // Relative Discord timestamp
+    });
+
+    test("should handle database disconnection", async () => {
+        executeMock.mockImplementationOnce(() => Promise.reject(new Error("DB Down")));
+
+        const interaction = {
+            deferReply: mock(() => Promise.resolve()),
+            editReply: mock(() => Promise.resolve()),
+            client: {
+                ws: {
+                    ping: 42
+                }
+            },
+            user: { id: "123", username: "testuser" },
+            commandName: "health"
+        } as unknown as ChatInputCommandInteraction;
+
+        await health.execute(interaction);
+
+        const editReplyCall = (interaction.editReply as any).mock.calls[0][0];
+        const embed = editReplyCall.embeds[0];
+        const connectivityField = embed.data.fields.find((f: any) => f.name === "📡 Connectivity");
+
+        expect(connectivityField.value).toContain("Disconnected");
+    });
+});

src/commands/admin/health.ts

@@ -0,0 +1,60 @@
+import { createCommand } from "@lib/utils";
+import { AuroraClient } from "@/lib/BotClient";
+import { SlashCommandBuilder, PermissionFlagsBits, EmbedBuilder, Colors } from "discord.js";
+import { DrizzleClient } from "@/lib/DrizzleClient";
+import { sql } from "drizzle-orm";
+import { createBaseEmbed } from "@lib/embeds";
+
+export const health = createCommand({
+    data: new SlashCommandBuilder()
+        .setName("health")
+        .setDescription("Check the bot's health status")
+        .setDefaultMemberPermissions(PermissionFlagsBits.Administrator),
+    execute: async (interaction) => {
+        await interaction.deferReply();
+
+        // 1. Check Discord API latency
+        const wsPing = interaction.client.ws.ping;
+
+        // 2. Verify database connection
+        let dbStatus = "Connected";
+        let dbPing = -1;
+        try {
+            const start = Date.now();
+            await DrizzleClient.execute(sql`SELECT 1`);
+            dbPing = Date.now() - start;
+        } catch (error) {
+            dbStatus = "Disconnected";
+            console.error("Health check DB error:", error);
+        }
+
+        // 3. Uptime
+        const uptime = process.uptime();
+        const days = Math.floor(uptime / 86400);
+        const hours = Math.floor((uptime % 86400) / 3600);
+        const minutes = Math.floor((uptime % 3600) / 60);
+        const seconds = Math.floor(uptime % 60);
+        const uptimeString = `${days}d ${hours}h ${minutes}m ${seconds}s`;
+
+        // 4. Memory usage
+        const memory = process.memoryUsage();
+        const heapUsed = (memory.heapUsed / 1024 / 1024).toFixed(2);
+        const heapTotal = (memory.heapTotal / 1024 / 1024).toFixed(2);
+        const rss = (memory.rss / 1024 / 1024).toFixed(2);
+
+        // 5. Last successful command
+        const lastCommand = AuroraClient.lastCommandTimestamp
+            ? `<t:${Math.floor(AuroraClient.lastCommandTimestamp / 1000)}:R>`
+            : "None since startup";
+
+        const embed = createBaseEmbed("System Health Status", undefined, Colors.Aqua)
+            .addFields(
+                { name: "📡 Connectivity", value: `**Discord WS:** ${wsPing}ms\n**Database:** ${dbStatus} ${dbPing >= 0 ? `(${dbPing}ms)` : ""}`, inline: true },
+                { name: "⏱️ Uptime", value: uptimeString, inline: true },
+                { name: "🧠 Memory Usage", value: `**RSS:** ${rss} MB\n**Heap:** ${heapUsed} / ${heapTotal} MB`, inline: false },
+                { name: "⌨️ Activity", value: `**Last Command:** ${lastCommand}`, inline: true }
+            );
+
+        await interaction.editReply({ embeds: [embed] });
+    }
+});

src/db/indexes.test.ts

@@ -0,0 +1,43 @@
+import { expect, test, describe } from "bun:test";
+import { postgres } from "../lib/DrizzleClient";
+
+describe("Database Indexes", () => {
+    test("should have indexes on users table", async () => {
+        const result = await postgres`
+            SELECT indexname FROM pg_indexes 
+            WHERE tablename = 'users'
+        `;
+        const indexNames = result.map(r => r.indexname);
+        expect(indexNames).toContain("users_balance_idx");
+        expect(indexNames).toContain("users_level_xp_idx");
+    });
+
+    test("should have index on transactions table", async () => {
+        const result = await postgres`
+            SELECT indexname FROM pg_indexes 
+            WHERE tablename = 'transactions'
+        `;
+        const indexNames = result.map(r => r.indexname);
+        expect(indexNames).toContain("transactions_created_at_idx");
+    });
+
+    test("should have indexes on moderation_cases table", async () => {
+        const result = await postgres`
+            SELECT indexname FROM pg_indexes 
+            WHERE tablename = 'moderation_cases'
+        `;
+        const indexNames = result.map(r => r.indexname);
+        expect(indexNames).toContain("moderation_cases_user_id_idx");
+        expect(indexNames).toContain("moderation_cases_case_id_idx");
+    });
+
+    test("should have indexes on user_timers table", async () => {
+        const result = await postgres`
+            SELECT indexname FROM pg_indexes 
+            WHERE tablename = 'user_timers'
+        `;
+        const indexNames = result.map(r => r.indexname);
+        expect(indexNames).toContain("user_timers_expires_at_idx");
+        expect(indexNames).toContain("user_timers_lookup_idx");
+    });
+});

src/db/schema.ts

@@ -9,6 +9,7 @@ import {
     text,
     integer,
     primaryKey,
+    index,
     bigserial,
     check
 } from 'drizzle-orm/pg-core';
@@ -41,7 +42,11 @@ export const users = pgTable('users', {
     settings: jsonb('settings').default({}),
     createdAt: timestamp('created_at', { withTimezone: true }).defaultNow(),
     updatedAt: timestamp('updated_at', { withTimezone: true }).defaultNow(),
-});
+}, (table) => [
+    index('users_username_idx').on(table.username),
+    index('users_balance_idx').on(table.balance),
+    index('users_level_xp_idx').on(table.level, table.xp),
+]);
 
 // 3. Items
 export const items = pgTable('items', {
@@ -82,7 +87,9 @@ export const transactions = pgTable('transactions', {
     type: varchar('type', { length: 50 }).notNull(),
     description: text('description'),
     createdAt: timestamp('created_at', { withTimezone: true }).defaultNow(),
-});
+}, (table) => [
+    index('transactions_created_at_idx').on(table.createdAt),
+]);
 
 export const itemTransactions = pgTable('item_transactions', {
     id: bigserial('id', { mode: 'bigint' }).primaryKey(),
@@ -129,7 +136,9 @@ export const userTimers = pgTable('user_timers', {
     expiresAt: timestamp('expires_at', { withTimezone: true }).notNull(),
     metadata: jsonb('metadata').default({}), // Store channelId, specific buff amounts, etc.
 }, (table) => [
-    primaryKey({ columns: [table.userId, table.type, table.key] })
+    primaryKey({ columns: [table.userId, table.type, table.key] }),
+    index('user_timers_expires_at_idx').on(table.expiresAt),
+    index('user_timers_lookup_idx').on(table.userId, table.type, table.key),
 ]);
 // 9. Lootdrops
 export const lootdrops = pgTable('lootdrops', {
@@ -158,7 +167,10 @@ export const moderationCases = pgTable('moderation_cases', {
     resolvedAt: timestamp('resolved_at', { withTimezone: true }),
     resolvedBy: bigint('resolved_by', { mode: 'bigint' }),
     resolvedReason: text('resolved_reason'),
-});
+}, (table) => [
+    index('moderation_cases_user_id_idx').on(table.userId),
+    index('moderation_cases_case_id_idx').on(table.caseId),
+]);
 
 
 

src/index.ts

@@ -12,3 +12,7 @@ if (!env.DISCORD_BOT_TOKEN) {
     throw new Error("❌ DISCORD_BOT_TOKEN is not set in environment variables.");
 }
 AuroraClient.login(env.DISCORD_BOT_TOKEN);
+
+// Handle graceful shutdown
+process.on("SIGINT", () => AuroraClient.shutdown());
+process.on("SIGTERM", () => AuroraClient.shutdown());

src/lib/BotClient.ts

@@ -9,6 +9,7 @@ import { logger } from "@lib/logger";
 export class Client extends DiscordClient {
 
     commands: Collection<string, Command>;
+    lastCommandTimestamp: number | null = null;
     private commandLoader: CommandLoader;
     private eventLoader: EventLoader;
 
@@ -93,6 +94,29 @@ export class Client extends DiscordClient {
             }
         }
     }
+
+    async shutdown() {
+        const { setShuttingDown, waitForTransactions } = await import("./shutdown");
+        const { closeDatabase } = await import("./DrizzleClient");
+
+        logger.info("🛑 Shutdown signal received. Starting graceful shutdown...");
+        setShuttingDown(true);
+
+        // Wait for transactions to complete
+        logger.info("⏳ Waiting for active transactions to complete...");
+        await waitForTransactions(10000);
+
+        // Destroy Discord client
+        logger.info("🔌 Disconnecting from Discord...");
+        this.destroy();
+
+        // Close database
+        logger.info("🗄️ Closing database connection...");
+        await closeDatabase();
+
+        logger.success("👋 Graceful shutdown complete. Exiting.");
+        process.exit(0);
+    }
 }
 
 export const AuroraClient = new Client({ intents: [GatewayIntentBits.Guilds, GatewayIntentBits.MessageContent, GatewayIntentBits.GuildMessages, GatewayIntentBits.GuildMembers] });

src/lib/DrizzleClient.ts

@@ -4,6 +4,10 @@ import * as schema from "@db/schema";
 import { env } from "@lib/env";
 
 const connectionString = env.DATABASE_URL;
-const postgres = new SQL(connectionString);
+export const postgres = new SQL(connectionString);
 
 export const DrizzleClient = drizzle(postgres, { schema });
+
+export const closeDatabase = async () => {
+    await postgres.close();
+};

src/lib/db.test.ts

@@ -0,0 +1,47 @@
+import { describe, it, expect, mock, beforeEach } from "bun:test";
+
+// Mock DrizzleClient
+mock.module("./DrizzleClient", () => ({
+    DrizzleClient: {
+        transaction: async (cb: any) => cb("MOCK_TX")
+    }
+}));
+
+import { withTransaction } from "./db";
+import { setShuttingDown, getActiveTransactions, decrementTransactions } from "./shutdown";
+
+describe("db withTransaction", () => {
+    beforeEach(() => {
+        setShuttingDown(false);
+        // Reset transaction count
+        while (getActiveTransactions() > 0) {
+            decrementTransactions();
+        }
+    });
+
+    it("should allow transactions when not shutting down", async () => {
+        const result = await withTransaction(async (tx) => {
+            return "success";
+        });
+        expect(result).toBe("success");
+        expect(getActiveTransactions()).toBe(0);
+    });
+
+    it("should throw error when shutting down", async () => {
+        setShuttingDown(true);
+        expect(withTransaction(async (tx) => {
+            return "success";
+        })).rejects.toThrow("System is shutting down, no new transactions allowed.");
+        expect(getActiveTransactions()).toBe(0);
+    });
+
+    it("should increment and decrement transaction count", async () => {
+        let countDuring = 0;
+        await withTransaction(async (tx) => {
+            countDuring = getActiveTransactions();
+            return "ok";
+        });
+        expect(countDuring).toBe(1);
+        expect(getActiveTransactions()).toBe(0);
+    });
+});

src/lib/db.ts

@@ -1,5 +1,6 @@
 import { DrizzleClient } from "./DrizzleClient";
 import type { Transaction } from "./types";
+import { isShuttingDown, incrementTransactions, decrementTransactions } from "./shutdown";
 
 export const withTransaction = async <T>(
     callback: (tx: Transaction) => Promise<T>,
@@ -8,8 +9,17 @@ export const withTransaction = async <T>(
     if (tx) {
         return await callback(tx);
     } else {
-        return await DrizzleClient.transaction(async (newTx) => {
-            return await callback(newTx);
-        });
+        if (isShuttingDown()) {
+            throw new Error("System is shutting down, no new transactions allowed.");
+        }
+
+        incrementTransactions();
+        try {
+            return await DrizzleClient.transaction(async (newTx) => {
+                return await callback(newTx);
+            });
+        } finally {
+            decrementTransactions();
+        }
     }
 };

src/lib/handlers/CommandHandler.test.ts

@@ -0,0 +1,59 @@
+import { describe, test, expect, mock, beforeEach } from "bun:test";
+import { CommandHandler } from "./CommandHandler";
+import { AuroraClient } from "@/lib/BotClient";
+import { ChatInputCommandInteraction } from "discord.js";
+
+// Mock UserService
+mock.module("@/modules/user/user.service", () => ({
+    userService: {
+        getOrCreateUser: mock(() => Promise.resolve())
+    }
+}));
+
+describe("CommandHandler", () => {
+    beforeEach(() => {
+        AuroraClient.commands.clear();
+        AuroraClient.lastCommandTimestamp = null;
+    });
+
+    test("should update lastCommandTimestamp on successful execution", async () => {
+        const executeSuccess = mock(() => Promise.resolve());
+        AuroraClient.commands.set("test", {
+            data: { name: "test" } as any,
+            execute: executeSuccess
+        } as any);
+
+        const interaction = {
+            commandName: "test",
+            user: { id: "123", username: "testuser" }
+        } as unknown as ChatInputCommandInteraction;
+
+        await CommandHandler.handle(interaction);
+
+        expect(executeSuccess).toHaveBeenCalled();
+        expect(AuroraClient.lastCommandTimestamp).not.toBeNull();
+        expect(AuroraClient.lastCommandTimestamp).toBeGreaterThan(0);
+    });
+
+    test("should not update lastCommandTimestamp on failed execution", async () => {
+        const executeError = mock(() => Promise.reject(new Error("Command Failed")));
+        AuroraClient.commands.set("fail", {
+            data: { name: "fail" } as any,
+            execute: executeError
+        } as any);
+
+        const interaction = {
+            commandName: "fail",
+            user: { id: "123", username: "testuser" },
+            replied: false,
+            deferred: false,
+            reply: mock(() => Promise.resolve()),
+            followUp: mock(() => Promise.resolve())
+        } as unknown as ChatInputCommandInteraction;
+
+        await CommandHandler.handle(interaction);
+
+        expect(executeError).toHaveBeenCalled();
+        expect(AuroraClient.lastCommandTimestamp).toBeNull();
+    });
+});

src/lib/handlers/CommandHandler.ts

@@ -26,6 +26,7 @@ export class CommandHandler {
 
         try {
             await command.execute(interaction);
+            AuroraClient.lastCommandTimestamp = Date.now();
         } catch (error) {
             logger.error(String(error));
             const errorEmbed = createErrorEmbed('There was an error while executing this command!');

src/lib/shutdown.test.ts

@@ -0,0 +1,56 @@
+import { describe, it, expect, beforeEach } from "bun:test";
+import { isShuttingDown, setShuttingDown, incrementTransactions, decrementTransactions, getActiveTransactions, waitForTransactions } from "./shutdown";
+
+describe("shutdown logic", () => {
+    beforeEach(() => {
+        setShuttingDown(false);
+        while (getActiveTransactions() > 0) {
+            decrementTransactions();
+        }
+    });
+
+    it("should initialize with shuttingDown as false", () => {
+        expect(isShuttingDown()).toBe(false);
+    });
+
+    it("should update shuttingDown state", () => {
+        setShuttingDown(true);
+        expect(isShuttingDown()).toBe(true);
+    });
+
+    it("should track active transactions", () => {
+        expect(getActiveTransactions()).toBe(0);
+        incrementTransactions();
+        expect(getActiveTransactions()).toBe(1);
+        decrementTransactions();
+        expect(getActiveTransactions()).toBe(0);
+    });
+
+    it("should wait for transactions to complete", async () => {
+        incrementTransactions();
+
+        const start = Date.now();
+        const waitPromise = waitForTransactions(1000);
+
+        // Simulate completion after 200ms
+        setTimeout(() => {
+            decrementTransactions();
+        }, 200);
+
+        await waitPromise;
+        const duration = Date.now() - start;
+
+        expect(duration).toBeGreaterThanOrEqual(200);
+        expect(getActiveTransactions()).toBe(0);
+    });
+
+    it("should timeout if transactions never complete", async () => {
+        incrementTransactions();
+        const start = Date.now();
+        await waitForTransactions(500);
+        const duration = Date.now() - start;
+
+        expect(duration).toBeGreaterThanOrEqual(500);
+        expect(getActiveTransactions()).toBe(1); // Still 1 because we didn't decrement
+    });
+});

src/lib/shutdown.ts

@@ -0,0 +1,30 @@
+import { logger } from "@lib/logger";
+
+let shuttingDown = false;
+let activeTransactions = 0;
+
+export const isShuttingDown = () => shuttingDown;
+export const setShuttingDown = (value: boolean) => {
+    shuttingDown = value;
+};
+
+export const incrementTransactions = () => {
+    activeTransactions++;
+};
+
+export const decrementTransactions = () => {
+    activeTransactions--;
+};
+
+export const getActiveTransactions = () => activeTransactions;
+
+export const waitForTransactions = async (timeoutMs: number = 10000) => {
+    const start = Date.now();
+    while (activeTransactions > 0) {
+        if (Date.now() - start > timeoutMs) {
+            logger.warn(`Shutdown timed out waiting for ${activeTransactions} transactions after ${timeoutMs}ms`);
+            break;
+        }
+        await new Promise(resolve => setTimeout(resolve, 100));
+    }
+};