feat: enhance layer masking functionality and brush controls

- Refactor LayersSheet component to support mask editing state and improve layer visibility handling.
- Introduce functions to collect mask layer IDs and count display layers excluding masks.
- Update BrushControls to include mask view mode options and a Done button for exiting mask editing.
- Modify brush session handling to support brush previews when editing masks.
- Implement a new BrushPreviewRenderer for rendering brush strokes with visual feedback.
- Add document geometry utilities for transforming points and resolving layer bounds.
- Ensure proper cleanup of brush preview on pointer leave and other interactions.

commands/document.test.ts

@@ -1,15 +1,18 @@
 import { describe, expect, test } from "bun:test";
+import type { Layer } from "@core/layer";
 import { createInitialAppState } from "@editor/initial-state";
 import {
   documentAddArtboardCommand,
   documentAddAssetCommand,
   documentAddGroupLayerCommand,
   documentAddImageLayerCommand,
+  documentAddLayerMaskCommand,
   documentAddRasterLayerCommand,
   documentGroupLayersCommand,
   documentMoveLayerCommand,
   documentRemoveArtboardCommand,
   documentRemoveLayerCommand,
+  documentRemoveLayerMaskCommand,
   documentRenameArtboardCommand,
   documentRenameLayerCommand,
   documentSetArtboardBoundsCommand,
@@ -201,6 +204,53 @@ describe("document commands", () => {
     expect(masked.document.artboards[0]?.layers[2]?.clippingMask).toEqual({ maskLayerId: "mask" });
   });
 
+  test("adds editable raster masks directly before the target layer", () => {
+    const state = documentWithLayers([raster("target", "Target")]);
+
+    const masked = documentAddLayerMaskCommand.execute(
+      { state },
+      {
+        layerId: "target",
+        asset: maskAsset(),
+        maskLayer: raster("mask", "Target Mask", "mask-asset"),
+      },
+    );
+
+    expect(masked.document.assets).toContainEqual(maskAsset());
+    expect(masked.document.artboards[0]?.layers.map((layer) => layer.id)).toEqual(["mask", "target"]);
+    expect(masked.document.artboards[0]?.layers[1]?.clippingMask).toEqual({ maskLayerId: "mask" });
+    expect(masked.editor.maskEdit).toEqual({ targetLayerId: "target", maskLayerId: "mask" });
+    expect(masked.editor.tools.activeTool).toBe("brush");
+  });
+
+  test("removes attached layer masks and exits mask editing", () => {
+    const state = documentAddLayerMaskCommand.execute(
+      { state: documentWithLayers([raster("target", "Target")]) },
+      { layerId: "target", asset: maskAsset(), maskLayer: raster("mask", "Target Mask", "mask-asset") },
+    );
+
+    const unmasked = documentRemoveLayerMaskCommand.execute({ state }, { layerId: "target" });
+
+    expect(unmasked.document.artboards[0]?.layers.map((layer) => layer.id)).toEqual(["target"]);
+    expect(unmasked.document.artboards[0]?.layers[0]?.clippingMask).toBeUndefined();
+    expect(unmasked.editor.maskEdit).toBeUndefined();
+  });
+
+  test("cleans mask references when deleting targets or mask layers", () => {
+    const state = documentAddLayerMaskCommand.execute(
+      { state: documentWithLayers([raster("target", "Target")]) },
+      { layerId: "target", asset: maskAsset(), maskLayer: raster("mask", "Target Mask", "mask-asset") },
+    );
+
+    const removedTarget = documentRemoveLayerCommand.execute({ state }, { layerId: "target" });
+    const removedMask = documentRemoveLayerCommand.execute({ state }, { layerId: "mask" });
+
+    expect(removedTarget.document.artboards[0]?.layers).toEqual([]);
+    expect(removedMask.document.artboards[0]?.layers[0]?.id).toBe("target");
+    expect(removedMask.document.artboards[0]?.layers[0]?.clippingMask).toBeUndefined();
+    expect(removedMask.editor.maskEdit).toBeUndefined();
+  });
+
   test("sets artboard visibility and lock state", () => {
     const state = documentAddArtboardCommand.execute(
       { state: createInitialAppState("Test") },
@@ -236,7 +286,7 @@ describe("document commands", () => {
   });
 });
 
-function documentWithLayers(layers: ReturnType<typeof group>[]) {
+function documentWithLayers(layers: Layer[]) {
   return {
     ...createInitialAppState("Test"),
     document: {
@@ -262,3 +312,20 @@ function group(id: string, name: string) {
     children: [],
   };
 }
+
+function raster(id: string, name: string, assetId = `${id}-asset`) {
+  return {
+    id,
+    type: "raster" as const,
+    name,
+    visible: true,
+    locked: false,
+    opacity: 1,
+    assetId,
+    transform: { position: { x: 0, y: 0 }, scale: { x: 1, y: 1 }, rotation: 0 },
+  };
+}
+
+function maskAsset() {
+  return { id: "mask-asset", name: "Target Mask", mimeType: "image/svg+xml", source: "mask", intrinsicSize: { w: 100, h: 100 } };
+}