feat: enhance layer masking functionality and brush controls

- Refactor LayersSheet component to support mask editing state and improve layer visibility handling.
- Introduce functions to collect mask layer IDs and count display layers excluding masks.
- Update BrushControls to include mask view mode options and a Done button for exiting mask editing.
- Modify brush session handling to support brush previews when editing masks.
- Implement a new BrushPreviewRenderer for rendering brush strokes with visual feedback.
- Add document geometry utilities for transforming points and resolving layer bounds.
- Ensure proper cleanup of brush preview on pointer leave and other interactions.

commands/tool.test.ts

@@ -1,11 +1,13 @@
 import { describe, expect, test } from "bun:test";
 import { createInitialAppState } from "@editor/initial-state";
-import { toolEnterTemporaryPanCommand, toolExitTemporaryPanCommand, toolSetActiveCommand, toolSetBrushSettingsCommand, toolSetMaskEditLayerCommand } from "./tool";
+import { toolEnterMaskEditCommand, toolEnterTemporaryPanCommand, toolExitMaskEditCommand, toolExitTemporaryPanCommand, toolSetActiveCommand, toolSetBrushPreviewCommand, toolSetBrushSettingsCommand, toolSetBrushStrokePreviewCommand, toolSetMaskViewModeCommand } from "./tool";
+
+const defaultBrush = { color: "#111827", size: 8, hardness: 100 };
 
 describe("tool commands", () => {
   test("sets active tool", () => {
     const next = toolSetActiveCommand.execute({ state: createInitialAppState("Test") }, { tool: "crop" });
-    expect(next.editor.tools).toEqual({ activeTool: "crop", interactionMode: { type: "tool", tool: "crop" }, brush: { color: "#111827", size: 8, hardness: 100 } });
+    expect(next.editor.tools).toEqual({ activeTool: "crop", interactionMode: { type: "tool", tool: "crop" }, brush: defaultBrush });
   });
 
   test("sets brush settings", () => {
@@ -14,10 +16,39 @@ describe("tool commands", () => {
     expect(next.editor.tools.brush).toEqual({ color: "#ff0000", size: 24, hardness: 50 });
   });
 
-  test("sets mask edit layer", () => {
-    const next = toolSetMaskEditLayerCommand.execute({ state: createInitialAppState("Test") }, { layerId: "mask-1" });
+  test("sets and clears brush preview", () => {
+    const showing = toolSetBrushPreviewCommand.execute({ state: createInitialAppState("Test") }, { position: { x: 10, y: 20 } });
+    const cleared = toolSetBrushPreviewCommand.execute({ state: showing }, undefined);
 
-    expect(next.editor.maskEditLayerId).toBe("mask-1");
+    expect(showing.editor.brushPreview).toEqual({ position: { x: 10, y: 20 } });
+    expect(cleared.editor.brushPreview).toBeUndefined();
+  });
+
+  test("sets and clears brush stroke preview", () => {
+    const showing = toolSetBrushStrokePreviewCommand.execute({ state: createInitialAppState("Test") }, { layerId: "layer", assetId: "asset", source: "preview" });
+    const cleared = toolSetBrushStrokePreviewCommand.execute({ state: showing }, undefined);
+
+    expect(showing.editor.brushStrokePreview).toEqual({ layerId: "layer", assetId: "asset", source: "preview" });
+    expect(cleared.editor.brushStrokePreview).toBeUndefined();
+  });
+
+  test("enters and exits mask edit", () => {
+    const editing = toolEnterMaskEditCommand.execute({ state: stateWithMask() }, { targetLayerId: "target", maskLayerId: "mask" });
+    const exited = toolExitMaskEditCommand.execute({ state: editing }, undefined);
+
+    expect(editing.editor.maskEdit).toEqual({ targetLayerId: "target", maskLayerId: "mask" });
+    expect(editing.editor.selection).toEqual({ artboardId: "a1", layerIds: ["target"] });
+    expect(editing.editor.tools.activeTool).toBe("brush");
+    expect(exited.editor.maskEdit).toBeUndefined();
+  });
+
+  test("sets mask view mode while editing a mask", () => {
+    const editing = toolEnterMaskEditCommand.execute({ state: stateWithMask() }, { targetLayerId: "target", maskLayerId: "mask" });
+    const alpha = toolSetMaskViewModeCommand.execute({ state: editing }, { mode: "alpha" });
+    const ignored = toolSetMaskViewModeCommand.execute({ state: createInitialAppState("Test") }, { mode: "blackWhite" });
+
+    expect(alpha.editor.maskEdit).toEqual({ targetLayerId: "target", maskLayerId: "mask", viewMode: "alpha" });
+    expect(ignored.editor.maskEdit).toBeUndefined();
   });
 
   test("enters and exits temporary pan", () => {
@@ -25,7 +56,43 @@ describe("tool commands", () => {
     const panning = toolEnterTemporaryPanCommand.execute({ state: initial }, undefined);
     const restored = toolExitTemporaryPanCommand.execute({ state: panning }, undefined);
 
-    expect(panning.editor.tools).toEqual({ activeTool: "select", interactionMode: { type: "temporary-pan", previousTool: "select" }, brush: { color: "#111827", size: 8, hardness: 100 } });
+    expect(panning.editor.tools).toEqual({ activeTool: "select", interactionMode: { type: "temporary-pan", previousTool: "select" }, brush: defaultBrush });
     expect(restored.editor.tools).toEqual(initial.editor.tools);
   });
 });
+
+function stateWithMask() {
+  return {
+    ...createInitialAppState("Test"),
+    document: {
+      ...createInitialAppState("Test").document,
+      artboards: [
+        {
+          id: "a1",
+          name: "Artboard 1",
+          bounds: { x: 0, y: 0, w: 320, h: 240 },
+          backgroundColor: "transparent" as const,
+          visible: true,
+          locked: false,
+          layers: [
+            raster("mask", "Mask"),
+            { ...raster("target", "Target"), clippingMask: { maskLayerId: "mask" } },
+          ],
+        },
+      ],
+    },
+  };
+}
+
+function raster(id: string, name: string) {
+  return {
+    id,
+    type: "raster" as const,
+    name,
+    visible: true,
+    locked: false,
+    opacity: 1,
+    assetId: `${id}-asset`,
+    transform: { position: { x: 0, y: 0 }, scale: { x: 1, y: 1 }, rotation: 0 },
+  };
+}