feat: enhance layer masking functionality and brush controls

- Refactor LayersSheet component to support mask editing state and improve layer visibility handling.
- Introduce functions to collect mask layer IDs and count display layers excluding masks.
- Update BrushControls to include mask view mode options and a Done button for exiting mask editing.
- Modify brush session handling to support brush previews when editing masks.
- Implement a new BrushPreviewRenderer for rendering brush strokes with visual feedback.
- Add document geometry utilities for transforming points and resolving layer bounds.
- Ensure proper cleanup of brush preview on pointer leave and other interactions.

input/document-geometry.ts

@@ -0,0 +1,107 @@
+import type { ImageDocument } from "@core/document";
+import type { Rect, Size, Vec2D } from "@core/geometry";
+import type { ArtboardId, LayerId } from "@core/id";
+import type { Layer } from "@core/layer";
+
+export type InputViewportState = {
+  center: Vec2D;
+  zoom: number;
+  rotation?: number;
+  size: Size;
+};
+
+export type InputSelectionState = {
+  artboardId?: ArtboardId;
+  layerIds: LayerId[];
+};
+
+export type InputTransformTarget =
+  | { type: "artboard"; id: ArtboardId }
+  | { type: "layer"; id: LayerId };
+
+export function selectedTransformTarget(_document: ImageDocument, selection: InputSelectionState): InputTransformTarget | undefined {
+  if (selection.layerIds.length === 1 && selection.layerIds[0]) return { type: "layer", id: selection.layerIds[0] };
+  if (selection.artboardId) return { type: "artboard", id: selection.artboardId };
+  return undefined;
+}
+
+export function resolveTransformTargetBounds(document: ImageDocument, target: InputTransformTarget): Rect | undefined {
+  switch (target.type) {
+    case "artboard":
+      return document.artboards.find((artboard) => artboard.id === target.id)?.bounds;
+    case "layer": {
+      const layer = findLayer(document, target.id);
+      return layer ? resolveLayerBounds(document, layer) : undefined;
+    }
+  }
+}
+
+export function viewportPointToDocumentPoint(point: Vec2D, viewport: InputViewportState): Vec2D {
+  return {
+    x: viewport.center.x + (point.x - viewport.size.w / 2) / viewport.zoom,
+    y: viewport.center.y + (point.y - viewport.size.h / 2) / viewport.zoom,
+  };
+}
+
+export function documentRectToViewportRect(rect: Rect, viewport: InputViewportState): Rect {
+  return {
+    x: viewport.size.w / 2 + (rect.x - viewport.center.x) * viewport.zoom,
+    y: viewport.size.h / 2 + (rect.y - viewport.center.y) * viewport.zoom,
+    w: rect.w * viewport.zoom,
+    h: rect.h * viewport.zoom,
+  };
+}
+
+function findLayer(document: ImageDocument, layerId: LayerId): Layer | undefined {
+  for (const artboard of document.artboards) {
+    const layer = findLayerInTree(artboard.layers, layerId);
+    if (layer) return layer;
+  }
+
+  return undefined;
+}
+
+function findLayerInTree(layers: readonly Layer[], layerId: LayerId): Layer | undefined {
+  for (const layer of layers) {
+    if (layer.id === layerId) return layer;
+    if (layer.type === "group") {
+      const child = findLayerInTree(layer.children, layerId);
+      if (child) return child;
+    }
+  }
+
+  return undefined;
+}
+
+function resolveLayerBounds(document: ImageDocument, layer: Layer): Rect | undefined {
+  switch (layer.type) {
+    case "group":
+      return unionRects(layer.children.flatMap((child) => {
+        const bounds = resolveLayerBounds(document, child);
+        return bounds ? [bounds] : [];
+      }));
+    case "image":
+    case "raster": {
+      const asset = document.assets.find((candidate) => candidate.id === layer.assetId);
+      if (!asset) return undefined;
+
+      return {
+        x: layer.transform.position.x,
+        y: layer.transform.position.y,
+        w: asset.intrinsicSize.w * layer.transform.scale.x,
+        h: asset.intrinsicSize.h * layer.transform.scale.y,
+      };
+    }
+  }
+}
+
+function unionRects(rects: Rect[]): Rect | undefined {
+  if (rects.length === 0) return undefined;
+
+  const minX = Math.min(...rects.map((rect) => rect.x));
+  const minY = Math.min(...rects.map((rect) => rect.y));
+  const maxX = Math.max(...rects.map((rect) => rect.x + rect.w));
+  const maxY = Math.max(...rects.map((rect) => rect.y + rect.h));
+
+  return { x: minX, y: minY, w: maxX - minX, h: maxY - minY };
+}