feat: enhance layer mask handling and opacity management

- Introduced `getLayerMask` utility to streamline layer mask retrieval.
- Updated layer rendering logic to incorporate layer masks and opacity adjustments.
- Added functionality to prevent dropping a group into its descendants.
- Enhanced image texture rendering to support opacity parameters across various rendering functions.
- Implemented group layer bounds application for better scaling and positioning.
- Added tests to ensure correct behavior when handling layer masks and group layers.
- Created new types for asset generation provenance to track generated assets more effectively.

commands/generation.ts

@@ -1,9 +1,10 @@
 import type { Asset } from "@core/asset";
+import type { AssetGenerationProvenance, GeneratedAssetAcceptance } from "@core/asset-provenance";
 import type { ImageDocument } from "@core/document";
 import type { ArtboardId, AssetId, LayerId } from "@core/id";
 import type { ImageLayer } from "@core/image-layer";
 import type { Layer } from "@core/layer";
-import type { GenerationCandidate, GenerationState } from "@editor/state";
+import type { GenerationCandidate, GenerationCompareMode, GenerationState } from "@editor/state";
 import type { Command } from "./command";
 import { commandIds } from "./ids";
 
@@ -15,6 +16,10 @@ export type GenerationSelectCandidatePayload = {
   candidateId?: string;
 };
 
+export type GenerationSetCompareModePayload = {
+  mode: GenerationCompareMode;
+};
+
 export type GenerationRemoveCandidatePayload = {
   candidateId: string;
 };
@@ -33,6 +38,7 @@ export type GenerationReplaceCandidatePixelsPayload = {
 };
 
 const maxCandidates = 12;
+const generationCompareModes = new Set<GenerationCompareMode>(["result", "before", "split"]);
 
 export const generationAddCandidateCommand: Command<GenerationAddCandidatePayload> = {
   id: commandIds.generationAddCandidate,
@@ -47,6 +53,7 @@ export const generationAddCandidateCommand: Command<GenerationAddCandidatePayloa
         generation: {
           candidates,
           selectedCandidateId: payload.candidate.id,
+          compareMode: "result",
         },
       },
     };
@@ -73,6 +80,26 @@ export const generationSelectCandidateCommand: Command<GenerationSelectCandidate
   },
 };
 
+export const generationSetCompareModeCommand: Command<GenerationSetCompareModePayload> = {
+  id: commandIds.generationSetCompareMode,
+  name: "Set generation compare mode",
+  history: { mode: "ignore" },
+  execute({ state }, payload) {
+    if (!generationCompareModes.has(payload.mode)) return state;
+    if (state.editor.generation.compareMode === payload.mode) return state;
+    return {
+      ...state,
+      editor: {
+        ...state.editor,
+        generation: {
+          ...state.editor.generation,
+          compareMode: payload.mode,
+        },
+      },
+    };
+  },
+};
+
 export const generationRemoveCandidateCommand: Command<GenerationRemoveCandidatePayload> = {
   id: commandIds.generationRemoveCandidate,
   name: "Remove generation candidate",
@@ -88,6 +115,7 @@ export const generationRemoveCandidateCommand: Command<GenerationRemoveCandidate
         generation: {
           candidates,
           selectedCandidateId,
+          compareMode: candidates.length > 0 ? state.editor.generation.compareMode : "result",
         },
       },
     };
@@ -99,12 +127,12 @@ export const generationClearCandidatesCommand: Command = {
   name: "Clear generation candidates",
   history: { mode: "ignore" },
   execute({ state }) {
-    if (state.editor.generation.candidates.length === 0 && !state.editor.generation.selectedCandidateId) return state;
+    if (state.editor.generation.candidates.length === 0 && !state.editor.generation.selectedCandidateId && state.editor.generation.compareMode === "result") return state;
     return {
       ...state,
       editor: {
         ...state.editor,
-        generation: { candidates: [], selectedCandidateId: undefined },
+        generation: { candidates: [], selectedCandidateId: undefined, compareMode: "result" },
       },
     };
   },
@@ -125,6 +153,7 @@ export const generationApplyCandidateAsLayerCommand: Command<GenerationApplyCand
       mimeType: candidate.mimeType,
       source: candidate.source,
       intrinsicSize: { ...candidate.intrinsicSize },
+      provenance: generationProvenance(candidate, payload.variant ? "variant-layer" : "layer"),
     };
     const layer: ImageLayer = {
       id: payload.layerId,
@@ -167,7 +196,16 @@ export const generationReplaceCandidatePixelsCommand: Command<GenerationReplaceC
       ...state,
       document: {
         ...state.document,
-        assets: state.document.assets.map((asset) => asset.id === targetAsset.id ? { ...asset, source: payload.source, mimeType: payload.mimeType ?? asset.mimeType } : asset),
+        assets: state.document.assets.map((asset) =>
+          asset.id === targetAsset.id
+            ? {
+                ...asset,
+                source: payload.source,
+                mimeType: payload.mimeType ?? asset.mimeType,
+                provenance: generationProvenance(candidate, "replacement"),
+              }
+            : asset,
+        ),
       },
       editor: {
         ...state.editor,
@@ -181,6 +219,7 @@ export const generationReplaceCandidatePixelsCommand: Command<GenerationReplaceC
 export const generationCommands = [
   generationAddCandidateCommand,
   generationSelectCandidateCommand,
+  generationSetCompareModeCommand,
   generationRemoveCandidateCommand,
   generationClearCandidatesCommand,
   generationApplyCandidateAsLayerCommand,
@@ -219,6 +258,58 @@ function findLayerInTree(layers: readonly Layer[], layerId: LayerId): Layer | un
 }
 
 function clearCommittedGenerationPreview(generation: GenerationState): GenerationState {
-  if (generation.candidates.length === 0 && !generation.selectedCandidateId) return generation;
-  return { candidates: [], selectedCandidateId: undefined };
+  if (generation.candidates.length === 0 && !generation.selectedCandidateId && generation.compareMode === "result") return generation;
+  return { candidates: [], selectedCandidateId: undefined, compareMode: "result" };
+}
+
+function generationProvenance(candidate: GenerationCandidate, acceptance: GeneratedAssetAcceptance): AssetGenerationProvenance {
+  return {
+    kind: "generated",
+    candidateId: candidate.id,
+    mode: candidate.mode,
+    acceptance,
+    prompt: candidate.settings.prompt,
+    negativePrompt: candidate.settings.negativePrompt,
+    seed: candidate.seed,
+    outputSize: { ...candidate.intrinsicSize },
+    settings: {
+      architecture: candidate.settings.architecture,
+      model: candidate.settings.model,
+      textEncoder: candidate.settings.textEncoder,
+      vae: candidate.settings.vae,
+      strength: candidate.settings.strength,
+      steps: candidate.settings.steps,
+      cfg: candidate.settings.cfg,
+      sampler: candidate.settings.sampler,
+      scheduler: candidate.settings.scheduler,
+      width: candidate.settings.width,
+      height: candidate.settings.height,
+    },
+    inpaint: candidate.inpaint
+      ? {
+          targetLayerId: candidate.inpaint.targetLayerId,
+          maskLayerId: candidate.inpaint.maskLayerId,
+          sourceAssetId: candidate.inpaint.sourceAssetId,
+          maskAssetId: candidate.inpaint.maskAssetId,
+          crop: {
+            assetBounds: { ...candidate.inpaint.crop.assetBounds },
+            documentBounds: { ...candidate.inpaint.crop.documentBounds },
+            padding: candidate.inpaint.crop.padding,
+            maskedAreaOnly: candidate.inpaint.crop.maskedAreaOnly,
+          },
+          mask: {
+            polarity: candidate.inpaint.mask.polarity,
+            activeBounds: { ...candidate.inpaint.mask.activeBounds },
+          },
+          backend: {
+            growMaskBy: candidate.inpaint.backend.growMaskBy,
+            maskedContent: candidate.inpaint.backend.maskedContent,
+            maskBlur: candidate.inpaint.backend.maskBlur,
+            maskFeather: candidate.inpaint.backend.maskFeather,
+            maskExpand: candidate.inpaint.backend.maskExpand,
+            cropPadding: candidate.inpaint.backend.cropPadding,
+          },
+        }
+      : undefined,
+  };
 }