feat: implement inpainting functionality with mask handling

- Added `createMaskedPixelReplacementSource` function to handle pixel replacement using inpainting.
- Introduced `buildInpaintBundle` to prepare inpainting data including mask generation and validation.
- Created utility functions for mask operations such as `applyMaskedContentModeToRgba`, `expandRectWithinBounds`, and others for mask manipulation.
- Developed tests for inpainting preparation and mask raster utilities to ensure functionality and correctness.
- Implemented mask raster operations including inversion, feathering, blurring, and more.

commands/document.ts

@@ -113,6 +113,25 @@ export type DocumentAddLayerMaskPayload = {
   maskLayer: RasterLayer;
 };
 
+export type LayerMaskOperation =
+  | { type: "paint" }
+  | { type: "magicWand" }
+  | { type: "chromaKey" }
+  | { type: "invert" }
+  | { type: "fill"; fill: "white" | "black" | "clear" }
+  | { type: "feather"; radius: number }
+  | { type: "expand"; radius: number }
+  | { type: "contract"; radius: number }
+  | { type: "blur"; radius: number }
+  | { type: "despeckle"; strength: number };
+
+export type DocumentApplyLayerMaskOperationPayload = {
+  maskLayerId: LayerId;
+  source: string;
+  mimeType?: string;
+  operation: LayerMaskOperation;
+};
+
 export type DocumentRemoveLayerMaskPayload = {
   layerId: LayerId;
 };
@@ -490,6 +509,38 @@ export const documentAddLayerMaskCommand: Command<DocumentAddLayerMaskPayload> =
   },
 };
 
+export const documentApplyLayerMaskOperationCommand: Command<DocumentApplyLayerMaskOperationPayload> = {
+  id: commandIds.documentApplyLayerMaskOperation,
+  name: "Apply layer mask operation",
+  execute({ state }, payload) {
+    if (!payload.source.trim()) return state;
+
+    const maskLocation = findLayerLocation(state.document, payload.maskLayerId);
+    if (!maskLocation || maskLocation.layer.type === "group") return state;
+    if (!isReferencedMaskLayer(state.document, payload.maskLayerId)) return state;
+
+    return {
+      ...state,
+      document: {
+        ...state.document,
+        assets: state.document.assets.map((asset) =>
+          asset.id === maskLocation.layer.assetId
+            ? {
+                ...asset,
+                source: payload.source,
+                mimeType: payload.mimeType ?? asset.mimeType,
+              }
+            : asset,
+        ),
+      },
+      editor: {
+        ...state.editor,
+        brushStrokePreview: state.editor.brushStrokePreview?.assetId === maskLocation.layer.assetId ? undefined : state.editor.brushStrokePreview,
+      },
+    };
+  },
+};
+
 export const documentRemoveLayerMaskCommand: Command<DocumentRemoveLayerMaskPayload> = {
   id: commandIds.documentRemoveLayerMask,
   name: "Remove layer mask",
@@ -564,6 +615,7 @@ export const documentCommands = [
   documentRenameLayerCommand,
   documentSetLayerClippingMaskCommand,
   documentAddLayerMaskCommand,
+  documentApplyLayerMaskOperationCommand,
   documentRemoveLayerMaskCommand,
 ] satisfies Command<unknown>[];
 
@@ -582,6 +634,18 @@ function findLayerLocation(document: ImageDocument, layerId: LayerId): LayerLoca
   return undefined;
 }
 
+function isReferencedMaskLayer(document: ImageDocument, maskLayerId: LayerId): boolean {
+  return document.artboards.some((artboard) => isReferencedMaskLayerInTree(artboard.layers, maskLayerId));
+}
+
+function isReferencedMaskLayerInTree(layers: readonly Layer[], maskLayerId: LayerId): boolean {
+  for (const layer of layers) {
+    if (layer.clippingMask?.maskLayerId === maskLayerId) return true;
+    if (layer.type === "group" && isReferencedMaskLayerInTree(layer.children, maskLayerId)) return true;
+  }
+  return false;
+}
+
 function findLayerLocationInTree(layers: Layer[], layerId: LayerId, artboardId: ArtboardId, parentGroupId?: LayerId): LayerLocation | undefined {
   for (let index = 0; index < layers.length; index++) {
     const layer = layers[index];