feat: enhance layer mask handling and opacity management

- Introduced `getLayerMask` utility to streamline layer mask retrieval. - Updated layer rendering logic to incorporate layer masks and opacity adjustments. - Added functionality to prevent dropping a group into its descendants. - Enhanced image texture rendering to support opacity parameters across various rendering functions. - Implemented group layer bounds application for better scaling and positioning. - Added tests to ensure correct behavior when handling layer masks and group layers. - Created new types for asset generation provenance to track generated assets more effectively.
2026-07-05 16:06:23 +02:00
parent a83024c35c
commit 5eaf37ba28
33 changed files with 635 additions and 88 deletions
--- a/view/LayersSheet.tsx
+++ b/view/LayersSheet.tsx
@@ -4,6 +4,7 @@ import { commandIds } from "@commands/ids";
 import type { Asset } from "@core/asset";
 import type { ImageDocument } from "@core/document";
 import type { Layer } from "@core/layer";
+import { getLayerMask } from "@core/layer-mask-utils";
 import type { ArtboardId } from "@core/id";
 import { createDocumentReadIndex, resolveIndexedLayerBounds, type DocumentReadIndex, type IndexedLayerInfo } from "@editor/document-indexes";
 import type { MaskEditState, SelectionState } from "@editor/state";
@@ -220,10 +221,11 @@ function LayerRow({

  const selected = selectedLayerIds.includes(layer.id);
  const layerInfo = documentIndex.layerInfoById.get(layer.id);
-  const maskLayer = layer.clippingMask ? documentIndex.layerById.get(layer.clippingMask.maskLayerId) : undefined;
+  const layerMask = getLayerMask(layer);
+  const maskLayer = layerMask ? documentIndex.layerById.get(layerMask.maskLayerId) : undefined;
  const maskAsset = maskLayer && maskLayer.type !== "group" ? documentIndex.assetById.get(maskLayer.assetId) : undefined;
-  const canAddMask = Boolean(layerInfo && layer.type !== "group" && !layer.clippingMask);
-  const editingMask = Boolean(maskEdit && layer.clippingMask && maskEdit.targetLayerId === layer.id && maskEdit.maskLayerId === layer.clippingMask.maskLayerId);
+  const canAddMask = Boolean(layerInfo && layer.type !== "group" && !layerMask);
+  const editingMask = Boolean(maskEdit && layerMask && maskEdit.targetLayerId === layer.id && maskEdit.maskLayerId === layerMask.maskLayerId);
  const rowPadding = 12 + depth * 16;

  return (
@@ -273,7 +275,7 @@ function LayerRow({
            {layer.name}
          </button>
        )}
-        {layer.clippingMask ? (
+        {layerMask ? (
          <span className={`inline-flex items-center gap-1 rounded-full px-3 py-1 text-xs ${editingMask || selected ? "bg-black/10 text-black/65" : "bg-sky-400/10 text-sky-100/70"}`}>
            <Stack size={13} weight="fill" /> Mask
          </span>
@@ -290,7 +292,7 @@ function LayerRow({
          {layer.locked ? <Lock size={24} weight="regular" /> : <LockOpen size={24} weight="regular" />}
        </button>
      </div>
-      {layer.clippingMask ? (
+      {layerMask ? (
        <div className="mt-2 flex min-h-14 flex-wrap items-center gap-2 rounded-[1.5rem] py-2 pl-4 pr-2 text-sm text-sky-100/70 hover:bg-white/[0.04]">
          <Stack size={24} weight="fill" className="shrink-0" />
          <span className="min-w-28 flex-1 truncate">{maskLayer ? "Layer mask" : "Layer mask missing"}</span>
@@ -303,7 +305,7 @@ function LayerRow({
                onClick={() =>
                  editingMask
                    ? dispatch(commandIds.toolExitMaskEdit, undefined)
-                    : dispatch(commandIds.toolEnterMaskEdit, { targetLayerId: layer.id, maskLayerId: layer.clippingMask!.maskLayerId })
+                    : dispatch(commandIds.toolEnterMaskEdit, { targetLayerId: layer.id, maskLayerId: layerMask.maskLayerId })
                }
              >
                {editingMask ? "Done" : "Edit"}
@@ -313,7 +315,7 @@ function LayerRow({
                className={maskActionButtonClass()}
                title="Paint reveal"
                onClick={() => {
-                  dispatch(commandIds.toolEnterMaskEdit, { targetLayerId: layer.id, maskLayerId: layer.clippingMask!.maskLayerId });
+                  dispatch(commandIds.toolEnterMaskEdit, { targetLayerId: layer.id, maskLayerId: layerMask.maskLayerId });
                  dispatch(commandIds.toolSetActive, { tool: "brush" });
                }}
              >
@@ -324,7 +326,7 @@ function LayerRow({
                className={maskActionButtonClass()}
                title="Paint hide"
                onClick={() => {
-                  dispatch(commandIds.toolEnterMaskEdit, { targetLayerId: layer.id, maskLayerId: layer.clippingMask!.maskLayerId });
+                  dispatch(commandIds.toolEnterMaskEdit, { targetLayerId: layer.id, maskLayerId: layerMask.maskLayerId });
                  dispatch(commandIds.toolSetActive, { tool: "eraser" });
                }}
              >
@@ -592,7 +594,8 @@ function moveLayer(documentIndex: DocumentReadIndex, info: IndexedLayerInfo, dir
  const blocks = siblings.flatMap((layer, index) => {
    if (maskLayerIds.has(layer.id)) return [];

-    const maskIndex = layer.clippingMask ? siblings.findIndex((candidate) => candidate.id === layer.clippingMask?.maskLayerId) : -1;
+    const layerMask = getLayerMask(layer);
+    const maskIndex = layerMask ? siblings.findIndex((candidate) => candidate.id === layerMask.maskLayerId) : -1;
    const start = maskIndex >= 0 ? Math.min(maskIndex, index) : index;
    const end = maskIndex >= 0 ? Math.max(maskIndex, index) : index;
    return [{ layerId: layer.id, start, end, size: end - start + 1 }];
--- a/view/bottom-controls/ChromaKeyControls.tsx
+++ b/view/bottom-controls/ChromaKeyControls.tsx
@@ -3,6 +3,7 @@ import { DropHalf } from "@phosphor-icons/react";
 import { commandIds } from "@commands/ids";
 import type { ImageDocument } from "@core/document";
 import type { Layer } from "@core/layer";
+import { getLayerMask } from "@core/layer-mask-utils";
 import { resolveTransformTargetBounds } from "@editor/transform-targets";
 import type { AppStore } from "@editor/store";
 import type { ChromaKeySettings } from "@editor/tools";
@@ -160,7 +161,8 @@ function resolveChromaKeyTarget(document: ImageDocument, selection: SelectionSta
  if (!layer || layer.type === "group") return undefined;
  const asset = document.assets.find((candidate) => candidate.id === layer.assetId);
  const bounds = resolveTransformTargetBounds(document, { type: "layer", id: layer.id });
-  const maskLayer = layer.clippingMask ? findLayer(document.artboards.flatMap((artboard) => artboard.layers), layer.clippingMask.maskLayerId) : undefined;
+  const layerMask = getLayerMask(layer);
+  const maskLayer = layerMask?.enabled ? findLayer(document.artboards.flatMap((artboard) => artboard.layers), layerMask.maskLayerId) : undefined;
  const maskAsset = maskLayer && maskLayer.type !== "group" ? document.assets.find((candidate) => candidate.id === maskLayer.assetId) : undefined;
  return asset && bounds ? { layer, asset, bounds, maskLayer, maskAsset } : undefined;
 }
--- a/view/bottom-controls/GenerateActionControls.tsx
+++ b/view/bottom-controls/GenerateActionControls.tsx
@@ -1,7 +1,7 @@
 import { useEffect, useState } from "react";
 import { commandIds } from "@commands/ids";
 import type { ImageDocument } from "@core/document";
-import type { GenerationCandidate, GenerationState, SelectionState, ViewportState } from "@editor/state";
+import type { GenerationCandidate, GenerationCompareMode, GenerationState, SelectionState, ViewportState } from "@editor/state";
 import type { GenerateSettings } from "@editor/tools";
 import type { AppStore } from "@editor/store";
 import { createMaskedPixelReplacementSource } from "../generate/candidateActions";
@@ -62,6 +62,7 @@ export function GenerateActionControls({ document, selection, viewport, settings
          <CandidateControls
            document={document}
            candidate={candidate}
+            compareMode={generation.compareMode ?? "result"}
            settings={settings}
            busy={busy}
            setBusy={setBusy}
@@ -100,6 +101,7 @@ function CandidatePicker({ generation, dispatch }: { generation: GenerationState
 function CandidateControls({
  document,
  candidate,
+  compareMode,
  settings,
  busy,
  setBusy,
@@ -108,6 +110,7 @@ function CandidateControls({
 }: {
  document: ImageDocument;
  candidate: GenerationCandidate;
+  compareMode: GenerationCompareMode;
  settings: GenerateSettings;
  busy?: string;
  setBusy: (busy: string | undefined) => void;
@@ -128,6 +131,7 @@ function CandidateControls({
    <div className="flex flex-wrap items-center justify-center gap-1 rounded-full bg-white/[0.04] px-2 py-1 ring-1 ring-white/[0.05]">
      <CandidatePreview candidate={candidate} />
      <span className="px-2 text-xs font-medium text-white/55">Seed {candidate.seed}</span>
+      <CandidateCompareControls compareMode={compareMode} disabled={disabled} dispatch={dispatch} />
      <CandidateButton disabled={disabled} label="Regenerate" title="Regenerate same mask and crop" busy={busy === "Regenerate"} onClick={() => rerun("Regenerate", candidate.settings)} />
      <CandidateButton
        disabled={disabled}
@@ -138,11 +142,11 @@ function CandidateControls({
      />
      <CandidateButton disabled={disabled} label="Reuse seed" title="Regenerate with the same seed" busy={busy === "Reuse seed"} onClick={() => rerun("Reuse seed", { ...candidate.settings, seed: candidate.seed })} />
      <CandidateButton disabled={disabled} label="New seed" title="Regenerate with a new seed" busy={busy === "New seed"} onClick={() => rerun("New seed", { ...candidate.settings, seed: -1 })} />
-      <CandidateButton disabled={disabled} label="Apply layer" title="Apply candidate as a normal layer" onClick={() => applyCandidateAsLayer(candidate, false, dispatch)} />
+      <CandidateButton disabled={disabled} label="Accept layer" title="Accept candidate as a normal layer" onClick={() => applyCandidateAsLayer(candidate, false, dispatch)} />
      <CandidateButton
        disabled={disabled}
-        label="Apply + refine"
-        title="Apply candidate as a layer with a fresh refinement mask"
+        label="Accept + mask"
+        title="Accept candidate as a layer with a fresh refinement mask"
        busy={busy === "Refine"}
        onClick={() => {
          setBusy("Refine");
@@ -152,10 +156,10 @@ function CandidateControls({
            .finally(() => setBusy(undefined));
        }}
      />
-      <CandidateButton disabled={disabled} label="Stack variant" title="Stack candidate as another variant layer" onClick={() => applyCandidateAsLayer(candidate, true, dispatch)} />
+      <CandidateButton disabled={disabled} label="Accept variant" title="Stack candidate as another variant layer" onClick={() => applyCandidateAsLayer(candidate, true, dispatch)} />
      <CandidateButton
        disabled={disabled || !candidate.inpaint}
-        label="Replace"
+        label="Accept replace"
        title={candidate.inpaint ? "Replace masked pixels and preserve unmasked pixels" : "Only inpaint candidates can replace masked pixels"}
        busy={busy === "Replace"}
        onClick={() => {
@@ -183,6 +187,36 @@ function CandidateControls({
  );
 }

+function CandidateCompareControls({ compareMode, disabled, dispatch }: { compareMode: GenerationCompareMode; disabled: boolean; dispatch: AppStore["dispatch"] }) {
+  return (
+    <span className="flex items-center gap-1 rounded-full bg-black/20 p-1" aria-label="Compare candidate">
+      {generationCompareOptions.map((option) => {
+        const active = compareMode === option.mode;
+        return (
+          <button
+            key={option.mode}
+            type="button"
+            className={`h-7 rounded-full px-2 text-[0.7rem] font-semibold transition ${
+              active ? "bg-white text-black" : "text-white/55 hover:bg-white/10 hover:text-white"
+            } disabled:pointer-events-none disabled:opacity-35`}
+            disabled={disabled}
+            title={option.title}
+            onClick={() => dispatch(commandIds.generationSetCompareMode, { mode: option.mode })}
+          >
+            {option.label}
+          </button>
+        );
+      })}
+    </span>
+  );
+}
+
+const generationCompareOptions: Array<{ mode: GenerationCompareMode; label: string; title: string }> = [
+  { mode: "result", label: "After", title: "Show the generated result over the document" },
+  { mode: "before", label: "Before", title: "Hide the generated result and show the source document" },
+  { mode: "split", label: "Split", title: "Compare source on the left with result on the right" },
+];
+
 function CandidatePreview({ candidate }: { candidate: GenerationCandidate }) {
  if (!candidate.inputImage) {
    return <img src={candidate.source} alt="" className="h-10 w-10 rounded-full bg-black/25 object-cover ring-1 ring-white/10" />;
--- a/view/canvas/magic-wand.ts
+++ b/view/canvas/magic-wand.ts
@@ -2,6 +2,7 @@ import { commandIds } from "@commands/ids";
 import type { ImageDocument } from "@core/document";
 import type { Vec2D } from "@core/geometry";
 import type { Layer } from "@core/layer";
+import { getLayerMask } from "@core/layer-mask-utils";
 import { resolveTransformTargetBounds } from "@editor/transform-targets";
 import type { AppStore } from "@editor/store";
 import type { EditorState } from "@editor/state";
@@ -41,7 +42,8 @@ function resolveTarget(document: ImageDocument, editor: EditorState) {
  if (!layer || layer.type === "group") return undefined;
  const asset = document.assets.find((candidate) => candidate.id === layer.assetId);
  const bounds = resolveTransformTargetBounds(document, { type: "layer", id: layer.id });
-  const maskLayer = layer.clippingMask ? findLayer(document.artboards.flatMap((artboard) => artboard.layers), layer.clippingMask.maskLayerId) : undefined;
+  const layerMask = getLayerMask(layer);
+  const maskLayer = layerMask?.enabled ? findLayer(document.artboards.flatMap((artboard) => artboard.layers), layerMask.maskLayerId) : undefined;
  const maskAsset = maskLayer && maskLayer.type !== "group" ? document.assets.find((candidate) => candidate.id === maskLayer.assetId) : undefined;
  return asset && bounds ? { layer, asset, bounds, maskLayer, maskAsset } : undefined;
 }
--- a/view/canvas/renderFrame.test.ts
+++ b/view/canvas/renderFrame.test.ts
@@ -177,6 +177,7 @@ const visualEditorChanges: Array<[string, (state: AppState) => AppState]> = [
            },
          ],
          selectedCandidateId: "candidate",
+          compareMode: "result",
        },
      },
    }),
--- a/view/exportArtboardPng.ts
+++ b/view/exportArtboardPng.ts
@@ -2,6 +2,7 @@ import type { Artboard } from "@core/artboard";
 import type { Asset } from "@core/asset";
 import type { Rect } from "@core/geometry";
 import type { Layer } from "@core/layer";
+import { getLayerMask } from "@core/layer-mask-utils";

 export async function downloadArtboardPng(artboard: Artboard, assets: readonly Asset[]) {
  const width = Math.max(1, Math.round(artboard.bounds.w));
@@ -41,8 +42,9 @@ async function drawLayer(
 ) {
  if (!layer.visible || (!options.ignoreOwnMask && options.maskLayerIds.has(layer.id))) return;

-  if (!options.ignoreOwnMask && layer.clippingMask) {
-    const maskLayer = findLayer(layerTree, layer.clippingMask.maskLayerId);
+  const layerMask = getLayerMask(layer);
+  if (!options.ignoreOwnMask && layerMask?.enabled) {
+    const maskLayer = findLayer(layerTree, layerMask.maskLayerId);
    if (!maskLayer) return;
    await drawMaskedLayer(context, layer, maskLayer, layerTree, assets, artboardBounds);
    return;
@@ -122,7 +124,8 @@ function translatedContext(canvas: HTMLCanvasElement, bounds: Rect) {

 function collectMaskLayerIds(layers: readonly Layer[], ids = new Set<string>()) {
  for (const layer of layers) {
-    if (layer.clippingMask) ids.add(layer.clippingMask.maskLayerId);
+    const layerMask = getLayerMask(layer);
+    if (layerMask) ids.add(layerMask.maskLayerId);
    if (layer.type === "group") collectMaskLayerIds(layer.children, ids);
  }
  return ids;
--- a/view/generate/inpaintPrep.ts
+++ b/view/generate/inpaintPrep.ts
@@ -2,6 +2,7 @@ import type { Asset } from "@core/asset";
 import type { ImageDocument } from "@core/document";
 import type { Rect } from "@core/geometry";
 import type { Layer } from "@core/layer";
+import { getLayerMask } from "@core/layer-mask-utils";
 import type { SelectionState } from "@editor/state";
 import type { GenerateSettings } from "@editor/tools";
 import { createDocumentReadIndex, resolveIndexedLayerBounds } from "@editor/document-indexes";
@@ -137,9 +138,10 @@ function resolveInpaintTarget(document: ImageDocument, selection: SelectionState

  const asset = documentIndex.assetById.get(layerInfo.layer.assetId);
  if (!asset) throw new Error("The selected layer is missing its source image.");
-  if (!layerInfo.layer.clippingMask) throw new Error("Add a layer mask before running inpaint.");
+  const layerMask = getLayerMask(layerInfo.layer);
+  if (!layerMask?.enabled) throw new Error("Add a layer mask before running inpaint.");

-  const maskLayer = documentIndex.layerById.get(layerInfo.layer.clippingMask.maskLayerId);
+  const maskLayer = documentIndex.layerById.get(layerMask.maskLayerId);
  if (!maskLayer || maskLayer.type === "group") throw new Error("The selected layer mask is missing.");

  const maskAsset = documentIndex.assetById.get(maskLayer.assetId);
--- a/view/generate/runGenerate.ts
+++ b/view/generate/runGenerate.ts
@@ -2,6 +2,7 @@ import { commandIds } from "@commands/ids";
 import type { ImageDocument } from "@core/document";
 import type { Transform } from "@core/geometry";
 import type { Layer } from "@core/layer";
+import { getLayerMask } from "@core/layer-mask-utils";
 import type { AppStore } from "@editor/store";
 import type { GenerationCandidate, SelectionState, ViewportState } from "@editor/state";
 import type { GenerateSettings } from "@editor/tools";
@@ -224,7 +225,8 @@ function resolveSelectedImage(document: ImageDocument, selection: SelectionState
  const layer = findLayer(document.artboards.find((artboard) => artboard.id === selection.artboardId)?.layers ?? [], layerId);
  if (!layer || layer.type === "group") return undefined;
  const asset = document.assets.find((candidate) => candidate.id === layer.assetId);
-  const maskLayer = layer.clippingMask ? findLayer(document.artboards.flatMap((artboard) => artboard.layers), layer.clippingMask.maskLayerId) : undefined;
+  const layerMask = getLayerMask(layer);
+  const maskLayer = layerMask?.enabled ? findLayer(document.artboards.flatMap((artboard) => artboard.layers), layerMask.maskLayerId) : undefined;
  const maskAsset = maskLayer && maskLayer.type !== "group" ? document.assets.find((candidate) => candidate.id === maskLayer.assetId) : undefined;
  return asset ? { layer, asset, maskAsset } : undefined;
 }