feat: implement generation preconditions check and integrate with generation commands

operations/generation/preconditions.test.ts

@@ -0,0 +1,72 @@
+import { describe, expect, test } from "bun:test";
+import type { ImageDocument } from "@core/document";
+import { initialToolState } from "@editor/tools";
+import { checkGenerationPreconditions } from "./preconditions";
+
+describe("generation preconditions", () => {
+  test("allows text-to-image with a prompt and artboard but no layer selection", () => {
+    expect(checkGenerationPreconditions(document(), { artboardId: "artboard", layerIds: [] }, settings("text-to-image"))).toEqual({ ready: true });
+  });
+
+  test("requires exactly one source layer for image-to-image and outpaint", () => {
+    const selection = { artboardId: "artboard", layerIds: [] };
+    expect(checkGenerationPreconditions(document(), selection, settings("image-to-image"))).toEqual({ ready: false, message: "Select exactly one image or raster layer for image-to-image generation." });
+    expect(checkGenerationPreconditions(document(), selection, settings("outpaint"))).toEqual({ ready: false, message: "Select exactly one image or raster layer to outpaint." });
+  });
+
+  test("requires an enabled mask for inpaint", () => {
+    expect(checkGenerationPreconditions(document(), selected(), settings("inpaint"))).toEqual({ ready: false, message: "Add and enable a layer mask on the selected image before inpainting." });
+  });
+
+  test("allows inpaint when the selected source has an aligned enabled mask", () => {
+    expect(checkGenerationPreconditions(document(true), selected(), settings("inpaint"))).toEqual({ ready: true });
+  });
+
+  test("explains prompt and outpaint-padding requirements", () => {
+    expect(checkGenerationPreconditions(document(), selected(), { ...settings("image-to-image"), prompt: "  " })).toEqual({ ready: false, message: "Enter a prompt to generate an image." });
+    expect(checkGenerationPreconditions(document(), selected(), { ...settings("outpaint"), outpaint: { left: 0, top: 0, right: 0, bottom: 0, feathering: 0 } })).toEqual({ ready: false, message: "Add outpaint padding on at least one side." });
+  });
+});
+
+function settings(mode: "text-to-image" | "image-to-image" | "inpaint" | "outpaint") {
+  return { ...initialToolState.generate, mode, prompt: "A lighthouse" };
+}
+
+function selected() {
+  return { artboardId: "artboard" as const, layerIds: ["source"] };
+}
+
+function document(masked = false): ImageDocument {
+  const transform = { position: { x: 0, y: 0 }, scale: { x: 1, y: 1 }, rotation: 0 };
+  return {
+    id: "document",
+    name: "Test",
+    version: 1,
+    assets: [
+      { id: "source-asset", name: "Source", mimeType: "image/png", source: "source", intrinsicSize: { w: 100, h: 100 } },
+      { id: "mask-asset", name: "Mask", mimeType: "image/png", source: "mask", intrinsicSize: { w: 100, h: 100 } },
+    ],
+    artboards: [{
+      id: "artboard",
+      name: "Artboard",
+      bounds: { x: 0, y: 0, w: 100, h: 100 },
+      backgroundColor: "transparent",
+      visible: true,
+      locked: false,
+      layers: [
+        { id: "mask", type: "raster", name: "Mask", visible: true, locked: false, opacity: 1, assetId: "mask-asset", transform },
+        {
+          id: "source",
+          type: "raster",
+          name: "Source",
+          visible: true,
+          locked: false,
+          opacity: 1,
+          assetId: "source-asset",
+          transform,
+          layerMask: masked ? { kind: "raster", maskLayerId: "mask", enabled: true, inverted: false } : undefined,
+        },
+      ],
+    }],
+  };
+}

operations/generation/preconditions.ts

@@ -0,0 +1,78 @@
+import type { ImageDocument } from "@core/document";
+import { getLayerMask } from "@core/layer-mask-utils";
+import { createDocumentReadIndex, resolveIndexedLayerBounds } from "@editor/document-indexes";
+import type { SelectionState } from "@editor/state";
+import type { GenerateSettings } from "@editor/tools";
+
+export type GenerationPrecondition =
+  | { ready: true }
+  | { ready: false; message: string };
+
+export function checkGenerationPreconditions(
+  document: ImageDocument,
+  selection: SelectionState,
+  settings: GenerateSettings,
+): GenerationPrecondition {
+  if (!settings.prompt.trim()) return missing("Enter a prompt to generate an image.");
+
+  const artboard = selection.artboardId
+    ? document.artboards.find((candidate) => candidate.id === selection.artboardId)
+    : document.artboards[0];
+  if (!artboard) return missing("Create an artboard before generating an image.");
+  if (settings.mode === "text-to-image") return { ready: true };
+
+  if (selection.layerIds.length !== 1 || !selection.layerIds[0]) {
+    return missing(modeSelectionMessage(settings.mode));
+  }
+
+  const index = createDocumentReadIndex(document);
+  const layerInfo = index.layerInfoById.get(selection.layerIds[0]);
+  if (!layerInfo || layerInfo.artboardId !== artboard.id || layerInfo.layer.type === "group") {
+    return missing(modeSelectionMessage(settings.mode));
+  }
+
+  const asset = index.assetById.get(layerInfo.layer.assetId);
+  if (!asset) return missing("The selected layer is missing its source image.");
+  if (asset.intrinsicSize.w <= 0 || asset.intrinsicSize.h <= 0) return missing("The selected image has an invalid size.");
+
+  if (settings.mode === "outpaint") {
+    const padding = settings.outpaint;
+    if (padding.left + padding.top + padding.right + padding.bottom <= 0) {
+      return missing("Add outpaint padding on at least one side.");
+    }
+  }
+
+  if (settings.mode !== "inpaint") return { ready: true };
+
+  const layerMask = getLayerMask(layerInfo.layer);
+  if (!layerMask?.enabled) return missing("Add and enable a layer mask on the selected image before inpainting.");
+  const maskLayer = index.layerById.get(layerMask.maskLayerId);
+  if (!maskLayer || maskLayer.type === "group") return missing("The selected layer mask is missing.");
+  const maskAsset = index.assetById.get(maskLayer.assetId);
+  if (!maskAsset) return missing("The selected layer mask is missing its image data.");
+  if (Math.round(asset.intrinsicSize.w) !== Math.round(maskAsset.intrinsicSize.w) || Math.round(asset.intrinsicSize.h) !== Math.round(maskAsset.intrinsicSize.h)) {
+    return missing("The selected layer and mask image sizes must match.");
+  }
+
+  const layerBounds = resolveIndexedLayerBounds(index, layerInfo.layer);
+  const maskBounds = resolveIndexedLayerBounds(index, maskLayer);
+  if (!layerBounds || !maskBounds || !rectsAligned(layerBounds, maskBounds) || Math.abs(layerInfo.layer.transform.rotation - maskLayer.transform.rotation) > 0.001) {
+    return missing("Align the selected layer and its mask before inpainting.");
+  }
+
+  return { ready: true };
+}
+
+function modeSelectionMessage(mode: GenerateSettings["mode"]): string {
+  if (mode === "image-to-image") return "Select exactly one image or raster layer for image-to-image generation.";
+  if (mode === "inpaint") return "Select exactly one masked image or raster layer to inpaint.";
+  return "Select exactly one image or raster layer to outpaint.";
+}
+
+function missing(message: string): GenerationPrecondition {
+  return { ready: false, message };
+}
+
+function rectsAligned(a: { x: number; y: number; w: number; h: number }, b: { x: number; y: number; w: number; h: number }): boolean {
+  return Math.abs(a.x - b.x) <= 0.5 && Math.abs(a.y - b.y) <= 0.5 && Math.abs(a.w - b.w) <= 0.5 && Math.abs(a.h - b.h) <= 0.5;
+}

operations/generation/runGenerate.ts

@@ -9,6 +9,7 @@ import type { GenerateSettings } from "@editor/tools";
 import { buildInpaintBundle, type InpaintBundle } from "./inpaintPrep";
 import { imageSourceToDataUrl, loadImageSize } from "@platform/browser/imageRaster";
 import { requestGeneration } from "@platform/comfy/generationClient";
+import { checkGenerationPreconditions } from "./preconditions";
 
 export async function runGenerate(options: {
   document: ImageDocument;
@@ -18,8 +19,10 @@ export async function runGenerate(options: {
   dispatch: AppStore["dispatch"];
 }) {
   const { document, selection, settings, dispatch } = options;
+  const precondition = checkGenerationPreconditions(document, selection, settings);
+  if (!precondition.ready) throw new Error(precondition.message);
   const artboard = selection.artboardId ? document.artboards.find((candidate) => candidate.id === selection.artboardId) : document.artboards[0];
-  if (!artboard) return;
+  if (!artboard) throw new Error("Create an artboard before generating an image.");
 
   const target = resolveSelectedImage(document, selection);
   const inpaintBundle = settings.mode === "inpaint" ? await buildInpaintBundle(document, selection, settings) : undefined;

view/bottom-controls/GenerateActionControls.tsx

@@ -8,6 +8,7 @@ import { runGenerate, runGenerateFromCandidate } from "@operations/generation/ru
 import { runGenerationJob } from "@operations/generation/generationJob";
 import { currentGenerationJob, GenerationJobStatus } from "../GenerationJobStatus";
 import { createRefinementMask } from "@operations/masks/rasterActions";
+import { checkGenerationPreconditions } from "@operations/generation/preconditions";
 
 export type GenerateActionControlsProps = {
   document: ImageDocument;
@@ -22,7 +23,9 @@ export function GenerateActionControls({ document, selection, viewport, settings
   const job = currentGenerationJob(generation);
   const busy = job?.status === "running";
   const candidate = selectedCandidate(generation);
-  const canGenerate = Boolean(settings.prompt.trim()) && !busy;
+  const precondition = checkGenerationPreconditions(document, selection, settings);
+  const canGenerate = precondition.ready && !busy;
+  const preconditionMessage = precondition.ready ? undefined : precondition.message;
 
   return (
     <div className="flex max-w-[calc(100vw-2rem)] flex-wrap items-center justify-center gap-2 px-2">
@@ -30,13 +33,14 @@ export function GenerateActionControls({ document, selection, viewport, settings
         type="button"
         disabled={!canGenerate}
         className="h-12 rounded-full bg-white px-7 text-base font-semibold !text-black transition hover:bg-white/90 focus:outline-none focus-visible:ring-2 focus-visible:ring-white/40 disabled:pointer-events-none disabled:opacity-35"
-        title={job?.status === "failed" ? job.error : "Generate with ComfyUI"}
+        title={job?.status === "failed" ? job.error : preconditionMessage ?? "Generate with ComfyUI"}
         onClick={() => {
           void runGenerationJob({ kind: "generate", label: "Generating", dispatch, task: () => runGenerate({ document, selection, viewport, settings, dispatch }) });
         }}
       >
         {busy && job?.kind === "generate" ? "Generating..." : "Generate"}
       </button>
+      {preconditionMessage ? <span className="max-w-72 text-xs text-white/55" role="status">{preconditionMessage}</span> : null}
       <GenerationJobStatus generation={generation} />
       {candidate ? (
         <>