feat: enhance layer masking functionality and brush controls

- Refactor LayersSheet component to support mask editing state and improve layer visibility handling.
- Introduce functions to collect mask layer IDs and count display layers excluding masks.
- Update BrushControls to include mask view mode options and a Done button for exiting mask editing.
- Modify brush session handling to support brush previews when editing masks.
- Implement a new BrushPreviewRenderer for rendering brush strokes with visual feedback.
- Add document geometry utilities for transforming points and resolving layer bounds.
- Ensure proper cleanup of brush preview on pointer leave and other interactions.

view/canvas/useCanvasInput.ts

@@ -1,4 +1,5 @@
 import { useEffect, useRef, useState, type RefObject } from "react";
+import { commandIds } from "@commands/ids";
 import type { AppStore } from "@editor/store";
 import { isPanInteractionMode } from "@editor/tools";
 import type { GlobalKeybindConsumer, GlobalPointerConsumer, GlobalWheelConsumer } from "@input/index";
@@ -11,7 +12,7 @@ import {
   pointerInputEventFromPointerEvent,
   wheelInputEventFromWheelEvent,
 } from "@input/index";
-import { beginBrushSession, updateBrushSession, type BrushSession } from "./brush";
+import { beginBrushSession, canPreviewBrush, commitBrushSession, updateBrushSession, type BrushSession } from "./brush";
 
 export type CanvasInputOptions = {
   globalKeybindConsumer: GlobalKeybindConsumer;
@@ -50,9 +51,31 @@ export function useCanvasInput(
       isPanMode: () => isPanInteractionMode(store.getState().editor.tools.interactionMode),
     });
 
+    const clearBrushPreview = () => {
+      if (store.getState().editor.brushPreview) store.dispatch(commandIds.toolSetBrushPreview, undefined);
+    };
+
+    const updateBrushPreview = (position: { x: number; y: number }) => {
+      if (!pointInsideCanvas(position, canvas)) {
+        clearBrushPreview();
+        return;
+      }
+
+      const state = store.getState();
+      if (!canPreviewBrush(state.document, state.editor)) {
+        clearBrushPreview();
+        return;
+      }
+
+      store.dispatch(commandIds.toolSetBrushPreview, { position: viewportPointToDocumentPoint(position, state.editor.viewport) });
+    };
+
     const handleKeyDown = (event: KeyboardEvent) => {
       const consumed = panHandler.keyDown(keybindEventFromKeyboardEvent(event));
-      if (consumed) event.preventDefault();
+      if (consumed) {
+        clearBrushPreview();
+        event.preventDefault();
+      }
     };
 
     const handleKeyUp = (event: KeyboardEvent) => {
@@ -64,6 +87,7 @@ export function useCanvasInput(
       const inputEvent = pointerInputEventFromPointerEvent(event);
       const transformed = transformHandler.pointerDown(inputEvent);
       if (transformed) {
+        clearBrushPreview();
         canvas.setPointerCapture(event.pointerId);
         event.preventDefault();
         return;
@@ -71,6 +95,7 @@ export function useCanvasInput(
 
       const consumed = panHandler.pointerDown(inputEvent);
       if (consumed) {
+        clearBrushPreview();
         canvas.setPointerCapture(event.pointerId);
         setIsPanning(true);
         event.preventDefault();
@@ -78,10 +103,12 @@ export function useCanvasInput(
       }
 
       const state = store.getState();
+      const documentPoint = viewportPointToDocumentPoint(inputEvent.position, state.editor.viewport);
       const brush = (inputEvent.buttons & 1) === 1 && !isPanInteractionMode(state.editor.tools.interactionMode)
-        ? beginBrushSession(state.document, state.editor, viewportPointToDocumentPoint(inputEvent.position, state.editor.viewport))
+        ? beginBrushSession(state.document, state.editor, documentPoint)
         : undefined;
       if (brush) {
+        store.dispatch(commandIds.toolSetBrushPreview, { position: documentPoint });
         brushSessionId.current += 1;
         brushSession.current = brush;
         canvas.setPointerCapture(event.pointerId);
@@ -104,43 +131,53 @@ export function useCanvasInput(
       const inputEvent = pointerInputEventFromPointerEvent(event);
       if (brushSession.current) {
         if ((inputEvent.buttons & 1) !== 1 || isPanInteractionMode(store.getState().editor.tools.interactionMode)) {
+          const session = brushSession.current;
           brushSessionId.current += 1;
           brushSession.current = undefined;
+          void commitBrushSession({ store, session }).then(() => updateBrushPreview(inputEvent.position));
           event.preventDefault();
           return;
         }
 
-        const activeSessionId = brushSessionId.current;
         const point = viewportPointToDocumentPoint(inputEvent.position, store.getState().editor.viewport);
+        store.dispatch(commandIds.toolSetBrushPreview, { position: point });
         const settings = store.getState().editor.tools.brush;
-        void updateBrushSession({ store, session: brushSession.current, point, color: settings.color, size: settings.size, hardness: settings.hardness }).then((nextSession) => {
-          if (brushSessionId.current === activeSessionId) brushSession.current = nextSession;
-        });
+        brushSession.current = updateBrushSession({ store, session: brushSession.current, point, color: settings.color, size: settings.size, hardness: settings.hardness });
         event.preventDefault();
         return;
       }
 
       const transformed = transformHandler.pointerMove(inputEvent);
       if (transformed) {
+        clearBrushPreview();
         event.preventDefault();
         return;
       }
 
       const consumed = panHandler.pointerMove(inputEvent);
-      if (consumed) event.preventDefault();
+      if (consumed) {
+        clearBrushPreview();
+        event.preventDefault();
+        return;
+      }
+
+      updateBrushPreview(inputEvent.position);
     };
 
     const handlePointerUp = (event: PointerEvent) => {
       const inputEvent = pointerInputEventFromPointerEvent(event);
       if (brushSession.current) {
+        const session = brushSession.current;
         brushSessionId.current += 1;
         brushSession.current = undefined;
+        void commitBrushSession({ store, session }).then(() => updateBrushPreview(inputEvent.position));
         event.preventDefault();
         return;
       }
 
       const transformed = transformHandler.pointerUp(inputEvent);
       if (transformed) {
+        clearBrushPreview();
         event.preventDefault();
         return;
       }
@@ -149,9 +186,14 @@ export function useCanvasInput(
       if (!consumed) return;
 
       setIsPanning(false);
+      clearBrushPreview();
       event.preventDefault();
     };
 
+    const handlePointerLeave = () => {
+      if (!brushSession.current) clearBrushPreview();
+    };
+
     const handleWheel = (event: WheelEvent) => {
       const consumed = handleViewportWheel({
         event: wheelInputEventFromWheelEvent(event),
@@ -169,6 +211,7 @@ export function useCanvasInput(
     canvas.addEventListener("pointermove", handlePointerMove);
     canvas.addEventListener("pointerup", handlePointerUp);
     canvas.addEventListener("pointercancel", handlePointerUp);
+    canvas.addEventListener("pointerleave", handlePointerLeave);
     canvas.addEventListener("wheel", handleWheel, { passive: false });
 
     return () => {
@@ -178,6 +221,7 @@ export function useCanvasInput(
       canvas.removeEventListener("pointermove", handlePointerMove);
       canvas.removeEventListener("pointerup", handlePointerUp);
       canvas.removeEventListener("pointercancel", handlePointerUp);
+      canvas.removeEventListener("pointerleave", handlePointerLeave);
       canvas.removeEventListener("wheel", handleWheel);
     };
   }, [canvasRef, options, store]);
@@ -185,6 +229,10 @@ export function useCanvasInput(
   return { isPanning };
 }
 
+function pointInsideCanvas(point: { x: number; y: number }, canvas: HTMLCanvasElement) {
+  return point.x >= 0 && point.y >= 0 && point.x <= canvas.width && point.y <= canvas.height;
+}
+
 function viewportPointToDocumentPoint(point: { x: number; y: number }, viewport: { center: { x: number; y: number }; size: { w: number; h: number }; zoom: number }) {
   return {
     x: viewport.center.x + (point.x - viewport.size.w / 2) / viewport.zoom,