feat: enhance layer masking functionality and brush controls

- Refactor LayersSheet component to support mask editing state and improve layer visibility handling.
- Introduce functions to collect mask layer IDs and count display layers excluding masks.
- Update BrushControls to include mask view mode options and a Done button for exiting mask editing.
- Modify brush session handling to support brush previews when editing masks.
- Implement a new BrushPreviewRenderer for rendering brush strokes with visual feedback.
- Add document geometry utilities for transforming points and resolving layer bounds.
- Ensure proper cleanup of brush preview on pointer leave and other interactions.

renderer/renderer.ts

@@ -1,9 +1,9 @@
 import type { ImageDocument } from "@core/document";
 import type { EditorState } from "@editor/state";
 import { renderArtboard } from "./artboard";
+import { createBrushPreviewRenderer } from "./brush-preview";
 import { createImageTextureRenderer } from "./image-textures";
 import { renderLayers } from "./layers";
-import { renderMaskEditOverlay } from "./mask-edit-overlay";
 import { renderSelectionOverlay } from "./selection";
 import { renderTransformControls } from "./transform-controls";
 import type { WebGlRendererContext } from "./types";
@@ -31,6 +31,7 @@ export function createRenderer(canvas: HTMLCanvasElement, backend: RendererBacke
   }
 
   const rendererContext: WebGlRendererContext = { gl: context, canvas };
+  const brushPreviewRenderer = createOptionalBrushPreviewRenderer(rendererContext);
   let lastFrame: RenderFrame | undefined;
   let rerenderQueued = false;
   const imageTextureRenderer = createImageTextureRenderer(rendererContext, () => {
@@ -61,17 +62,28 @@ export function createRenderer(canvas: HTMLCanvasElement, backend: RendererBacke
       }
       imageTextureRenderer.syncAssets(frame.document.assets);
       renderLayers(rendererContext, frame.document, frame.editor, imageTextureRenderer);
-      renderMaskEditOverlay(rendererContext, frame.document, frame.editor, imageTextureRenderer);
 
-      renderSelectionOverlay(rendererContext, frame.document, frame.editor);
-      renderTransformControls(rendererContext, frame.document, frame.editor);
+      if (!frame.editor.maskEdit) {
+        renderSelectionOverlay(rendererContext, frame.document, frame.editor);
+        renderTransformControls(rendererContext, frame.document, frame.editor);
+      }
+      brushPreviewRenderer?.render(frame.document, frame.editor);
 
       context.disable(context.SCISSOR_TEST);
     },
     dispose() {
       imageTextureRenderer.dispose();
+      brushPreviewRenderer?.dispose();
     },
   };
 
   return renderer;
 }
+
+function createOptionalBrushPreviewRenderer(context: WebGlRendererContext) {
+  try {
+    return createBrushPreviewRenderer(context);
+  } catch {
+    return undefined;
+  }
+}