feat: enhance layer masking functionality and brush controls

- Refactor LayersSheet component to support mask editing state and improve layer visibility handling.
- Introduce functions to collect mask layer IDs and count display layers excluding masks.
- Update BrushControls to include mask view mode options and a Done button for exiting mask editing.
- Modify brush session handling to support brush previews when editing masks.
- Implement a new BrushPreviewRenderer for rendering brush strokes with visual feedback.
- Add document geometry utilities for transforming points and resolving layer bounds.
- Ensure proper cleanup of brush preview on pointer leave and other interactions.

commands/selection.ts

@@ -15,14 +15,19 @@ export const selectionSetCommand: Command<SelectionSetPayload> = {
   id: commandIds.selectionSet,
   name: "Set selection",
   execute({ state }, payload) {
+    const selection = {
+      artboardId: payload.artboardId,
+      layerIds: [...payload.layerIds],
+    };
+
     return {
       ...state,
       editor: {
         ...state.editor,
-        selection: {
-          artboardId: payload.artboardId,
-          layerIds: [...payload.layerIds],
-        },
+        selection,
+        maskEdit: selection.layerIds.length === 1 && selection.layerIds[0] === state.editor.maskEdit?.targetLayerId ? state.editor.maskEdit : undefined,
+        brushPreview: undefined,
+        brushStrokePreview: undefined,
       },
     };
   },
@@ -37,6 +42,9 @@ export const selectionClearCommand: Command = {
       editor: {
         ...state.editor,
         selection: { layerIds: [] },
+        maskEdit: undefined,
+        brushPreview: undefined,
+        brushStrokePreview: undefined,
       },
     };
   },
@@ -56,6 +64,9 @@ export const selectionAddLayerCommand: Command<SelectionAddLayerPayload> = {
           ...state.editor.selection,
           layerIds: [...state.editor.selection.layerIds, payload.layerId],
         },
+        maskEdit: undefined,
+        brushPreview: undefined,
+        brushStrokePreview: undefined,
       },
     };
   },