feat: enhance layer masking functionality and brush controls

- Refactor LayersSheet component to support mask editing state and improve layer visibility handling.
- Introduce functions to collect mask layer IDs and count display layers excluding masks.
- Update BrushControls to include mask view mode options and a Done button for exiting mask editing.
- Modify brush session handling to support brush previews when editing masks.
- Implement a new BrushPreviewRenderer for rendering brush strokes with visual feedback.
- Add document geometry utilities for transforming points and resolving layer bounds.
- Ensure proper cleanup of brush preview on pointer leave and other interactions.

renderer/layers.ts

@@ -1,6 +1,6 @@
 import type { ImageDocument } from "@core/document";
 import type { Layer } from "@core/layer";
-import type { EditorState, ViewportState } from "@editor/state";
+import type { EditorState, MaskViewMode, ViewportState } from "@editor/state";
 import { resolveTransformTargetBounds } from "@editor/transform-targets";
 import { clearScreenRect } from "./clear-rect";
 import type { ImageTextureRenderer } from "./image-textures";
@@ -9,6 +9,8 @@ import type { RgbaColor, ScreenRect, WebGlRendererContext } from "./types";
 
 const imageLayerColor: RgbaColor = [0.38, 0.42, 0.5, 1];
 const imageLayerInsetColor: RgbaColor = [0.48, 0.54, 0.64, 1];
+const hiddenMaskOverlayColor: RgbaColor = [1, 0.08, 0.08, 0.45];
+const maskRevealPreviewOpacity = 0.28;
 
 export function renderLayers(context: WebGlRendererContext, document: ImageDocument, editor: EditorState, imageTextureRenderer: ImageTextureRenderer) {
   for (const artboard of document.artboards) {
@@ -28,7 +30,9 @@ function renderLayer(
   clipRect: ScreenRect,
   maskLayerIds: ReadonlySet<string>,
 ) {
-  const editingMaskLayer = editor.maskEditLayerId === layer.id;
+  const editingMaskLayer = editor.maskEdit?.maskLayerId === layer.id;
+  const maskViewMode = editor.maskEdit?.viewMode ?? "composite";
+  const isolatedMaskView = isIsolatedMaskView(maskViewMode);
   if (!layer.visible || maskLayerIds.has(layer.id)) return;
 
   const effectiveClipRect = resolveLayerClipRect(context, document, editor.viewport, layer, clipRect);
@@ -39,16 +43,33 @@ function renderLayer(
     return;
   }
 
+  if (isolatedMaskView && layer.id !== editor.maskEdit?.targetLayerId) return;
+
   const bounds = resolveTransformTargetBounds(document, { type: "layer", id: layer.id });
   if (!bounds) return;
 
   const rect = documentRectToScreenRect(context.canvas, bounds, editor.viewport);
-  const asset = document.assets.find((candidate) => candidate.id === layer.assetId);
+  const asset = assetWithBrushStrokePreview(document.assets.find((candidate) => candidate.id === layer.assetId), editor);
   const maskLayer = !editingMaskLayer && layer.clippingMask ? findLayer(document, layer.clippingMask.maskLayerId) : undefined;
-  const maskAsset = maskLayer && maskLayer.type !== "group" ? document.assets.find((candidate) => candidate.id === maskLayer.assetId) : undefined;
+  const maskAsset = assetWithBrushStrokePreview(maskLayer && maskLayer.type !== "group" ? document.assets.find((candidate) => candidate.id === maskLayer.assetId) : undefined, editor);
   const maskBounds = maskLayer ? resolveTransformTargetBounds(document, { type: "layer", id: maskLayer.id }) : undefined;
   const maskRect = maskBounds ? documentRectToScreenRect(context.canvas, maskBounds, editor.viewport) : undefined;
-  if (asset && maskAsset && maskRect && imageTextureRenderer.renderMasked(asset, rect, maskAsset, maskRect, effectiveClipRect)) return;
+  const activeMaskTarget = Boolean(editor.maskEdit?.targetLayerId === layer.id && editor.maskEdit.maskLayerId === layer.clippingMask?.maskLayerId);
+  const showMaskRevealPreview = editor.tools.activeTool === "brush" && activeMaskTarget && maskViewMode === "composite";
+
+  if (asset && maskAsset && maskRect && activeMaskTarget) {
+    if (maskViewMode === "blackWhite" && imageTextureRenderer.renderMaskVisualization(maskAsset, maskRect, "blackWhite", undefined, effectiveClipRect)) return;
+    if (maskViewMode === "alpha" && imageTextureRenderer.renderMaskVisualization(maskAsset, maskRect, "alpha", undefined, effectiveClipRect)) return;
+    if (maskViewMode === "overlay" && imageTextureRenderer.render(asset, rect, effectiveClipRect)) {
+      imageTextureRenderer.renderMaskVisualization(maskAsset, maskRect, "hiddenOverlay", hiddenMaskOverlayColor, effectiveClipRect);
+      return;
+    }
+  }
+
+  if (asset && maskAsset && maskRect && imageTextureRenderer.renderMasked(asset, rect, maskAsset, maskRect, effectiveClipRect)) {
+    if (showMaskRevealPreview) imageTextureRenderer.renderMaskRevealPreview(asset, rect, maskAsset, maskRect, maskRevealPreviewOpacity, effectiveClipRect);
+    return;
+  }
   if (asset && imageTextureRenderer.render(asset, rect, effectiveClipRect)) return;
 
   const fallbackRect = intersectScreenRects(rect, effectiveClipRect);
@@ -68,11 +89,20 @@ function resolveLayerClipRect(
   if (!layer.clippingMask) return clipRect;
 
   const maskBounds = resolveTransformTargetBounds(document, { type: "layer", id: layer.clippingMask.maskLayerId });
-  if (!maskBounds) return undefined;
+  if (!maskBounds) return clipRect;
 
   return intersectScreenRects(clipRect, documentRectToScreenRect(context.canvas, maskBounds, viewport));
 }
 
+function isIsolatedMaskView(mode: MaskViewMode) {
+  return mode === "blackWhite" || mode === "alpha" || mode === "overlay";
+}
+
+function assetWithBrushStrokePreview<TAsset extends ImageDocument["assets"][number] | undefined>(asset: TAsset, editor: EditorState): TAsset {
+  if (!asset || editor.brushStrokePreview?.assetId !== asset.id) return asset;
+  return { ...asset, source: editor.brushStrokePreview.source } as TAsset;
+}
+
 function findLayer(document: ImageDocument, layerId: string): Layer | undefined {
   for (const artboard of document.artboards) {
     const layer = findLayerInTree(artboard.layers, layerId);