From bef2fb30514fff226b018a43d7f8d168ff693128 Mon Sep 17 00:00:00 2001 From: syntaxbullet Date: Sat, 11 Jul 2026 11:58:16 +0200 Subject: [PATCH] feat: unify AI edit workflow --- commands/ids.ts | 1 + commands/index.ts | 4 +- commands/payloads.ts | 3 +- commands/tool.test.ts | 11 ++- commands/tool.ts | 23 +++++- editor/tools.ts | 1 + operations/generation/preconditions.test.ts | 4 +- operations/generation/preconditions.ts | 10 +-- operations/generation/workflow.test.ts | 16 ++++ operations/generation/workflow.ts | 40 ++++++++++ .../GenerateActionControls.tsx | 27 ++++--- view/bottom-controls/GenerateControls.tsx | 74 ++++++++++++++----- 12 files changed, 173 insertions(+), 41 deletions(-) diff --git a/commands/ids.ts b/commands/ids.ts index 5cf0a22..7cd733a 100644 --- a/commands/ids.ts +++ b/commands/ids.ts @@ -27,6 +27,7 @@ export const commandIds = { selectionAddLayer: "selection.addLayer", toolSetActive: "tool.setActive", toolSetGenerateSettings: "tool.setGenerateSettings", + toolChooseGenerateIntent: "tool.chooseGenerateIntent", toolSetBrushSettings: "tool.setBrushSettings", toolSetChromaKeySettings: "tool.setChromaKeySettings", toolSetMagicWandSettings: "tool.setMagicWandSettings", diff --git a/commands/index.ts b/commands/index.ts index f9e4f88..5e93a82 100644 --- a/commands/index.ts +++ b/commands/index.ts @@ -89,10 +89,10 @@ export type { CommandRegistry } from "./registry"; export { createCommandRegistry } from "./registry"; export { selectionAddLayerCommand, selectionClearCommand, selectionCommands, selectionSetCommand } from "./selection"; export type { SelectionAddLayerPayload, SelectionSetPayload } from "./selection"; -export { toolCommands, toolEnterMaskEditCommand, toolEnterTemporaryPanCommand, toolExitMaskEditCommand, toolExitTemporaryPanCommand, toolSetActiveCommand, toolSetBrushPreviewCommand, toolSetBrushSettingsCommand, toolSetBrushStrokePreviewCommand, toolSetChromaKeySettingsCommand, toolSetGenerateSettingsCommand, toolSetMagicWandSettingsCommand, toolSetMaskViewModeCommand } from "./tool"; +export { toolChooseGenerateIntentCommand, toolCommands, toolEnterMaskEditCommand, toolEnterTemporaryPanCommand, toolExitMaskEditCommand, toolExitTemporaryPanCommand, toolSetActiveCommand, toolSetBrushPreviewCommand, toolSetBrushSettingsCommand, toolSetBrushStrokePreviewCommand, toolSetChromaKeySettingsCommand, toolSetGenerateSettingsCommand, toolSetMagicWandSettingsCommand, toolSetMaskViewModeCommand } from "./tool"; export { transformBeginCommand, transformCommands, transformEndCommand, transformSetBoundsCommand, transformUpdateCommand } from "./transform"; export type { TransformBeginPayload, TransformSetBoundsPayload, TransformUpdatePayload } from "./transform"; -export type { ToolEnterMaskEditPayload, ToolSetActivePayload, ToolSetBrushPreviewPayload, ToolSetBrushSettingsPayload, ToolSetBrushStrokePreviewPayload, ToolSetChromaKeySettingsPayload, ToolSetGenerateSettingsPayload, ToolSetMagicWandSettingsPayload, ToolSetMaskViewModePayload } from "./tool"; +export type { ToolChooseGenerateIntentPayload, ToolEnterMaskEditPayload, ToolSetActivePayload, ToolSetBrushPreviewPayload, ToolSetBrushSettingsPayload, ToolSetBrushStrokePreviewPayload, ToolSetChromaKeySettingsPayload, ToolSetGenerateSettingsPayload, ToolSetMagicWandSettingsPayload, ToolSetMaskViewModePayload } from "./tool"; export { viewportCommands, viewportPanCommand, diff --git a/commands/payloads.ts b/commands/payloads.ts index 2f91aca..b1518a3 100644 --- a/commands/payloads.ts +++ b/commands/payloads.ts @@ -44,7 +44,7 @@ import type { CommandPaletteSetSelectedIndexPayload, } from "./palette"; import type { SelectionAddLayerPayload, SelectionSetPayload } from "./selection"; -import type { ToolEnterMaskEditPayload, ToolSetActivePayload, ToolSetBrushPreviewPayload, ToolSetBrushSettingsPayload, ToolSetBrushStrokePreviewPayload, ToolSetChromaKeySettingsPayload, ToolSetGenerateSettingsPayload, ToolSetMagicWandSettingsPayload, ToolSetMaskViewModePayload } from "./tool"; +import type { ToolChooseGenerateIntentPayload, ToolEnterMaskEditPayload, ToolSetActivePayload, ToolSetBrushPreviewPayload, ToolSetBrushSettingsPayload, ToolSetBrushStrokePreviewPayload, ToolSetChromaKeySettingsPayload, ToolSetGenerateSettingsPayload, ToolSetMagicWandSettingsPayload, ToolSetMaskViewModePayload } from "./tool"; import type { TransformBeginPayload, TransformSetBoundsPayload, TransformUpdatePayload } from "./transform"; import type { WorkspaceSetPanelPayload } from "./workspace"; import type { EditorSetPointerSessionPayload } from "./editor"; @@ -86,6 +86,7 @@ export type CommandPayloads = { [commandIds.selectionAddLayer]: SelectionAddLayerPayload; [commandIds.toolSetActive]: ToolSetActivePayload; [commandIds.toolSetGenerateSettings]: ToolSetGenerateSettingsPayload; + [commandIds.toolChooseGenerateIntent]: ToolChooseGenerateIntentPayload; [commandIds.toolSetBrushSettings]: ToolSetBrushSettingsPayload; [commandIds.toolSetChromaKeySettings]: ToolSetChromaKeySettingsPayload; [commandIds.toolSetMagicWandSettings]: ToolSetMagicWandSettingsPayload; diff --git a/commands/tool.test.ts b/commands/tool.test.ts index 6804f28..060bddf 100644 --- a/commands/tool.test.ts +++ b/commands/tool.test.ts @@ -1,6 +1,6 @@ import { describe, expect, test } from "bun:test"; import { createInitialAppState } from "@editor/initial-state"; -import { toolEnterMaskEditCommand, toolEnterTemporaryPanCommand, toolExitMaskEditCommand, toolExitTemporaryPanCommand, toolSetActiveCommand, toolSetBrushPreviewCommand, toolSetBrushSettingsCommand, toolSetBrushStrokePreviewCommand, toolSetChromaKeySettingsCommand, toolSetGenerateSettingsCommand, toolSetMaskViewModeCommand } from "./tool"; +import { toolChooseGenerateIntentCommand, toolEnterMaskEditCommand, toolEnterTemporaryPanCommand, toolExitMaskEditCommand, toolExitTemporaryPanCommand, toolSetActiveCommand, toolSetBrushPreviewCommand, toolSetBrushSettingsCommand, toolSetBrushStrokePreviewCommand, toolSetChromaKeySettingsCommand, toolSetGenerateSettingsCommand, toolSetMaskViewModeCommand } from "./tool"; const defaultBrush = { color: "#111827", size: 8, hardness: 100 }; const defaultChromaKey = { color: "#00ff00", tolerance: 32, softness: 24, feather: 0, choke: 0, despeckle: 0, spill: 50 }; @@ -60,6 +60,15 @@ describe("tool commands", () => { }); }); + test("maps user intents to supported generation modes and a compatible architecture", () => { + const zImage = toolSetGenerateSettingsCommand.execute({ state: createInitialAppState("Test") }, { architecture: "z-image" }); + const replace = toolChooseGenerateIntentCommand.execute({ state: zImage }, { intent: "replace" }); + const variations = toolChooseGenerateIntentCommand.execute({ state: replace }, { intent: "variations" }); + + expect(replace.editor.tools.generate).toMatchObject({ architecture: "sdxl", mode: "inpaint" }); + expect(variations.editor.tools.generate).toMatchObject({ architecture: "sdxl", mode: "image-to-image" }); + }); + test("sets and clears brush preview", () => { const showing = toolSetBrushPreviewCommand.execute({ state: createInitialAppState("Test") }, { position: { x: 10, y: 20 } }); const cleared = toolSetBrushPreviewCommand.execute({ state: showing }, undefined); diff --git a/commands/tool.ts b/commands/tool.ts index 7d79966..8cc4b00 100644 --- a/commands/tool.ts +++ b/commands/tool.ts @@ -5,7 +5,7 @@ import type { Layer } from "@core/layer"; import { getLayerMask } from "@core/layer-mask-utils"; import type { MaskViewMode } from "@editor/state"; import { generateArchitectureDefaults } from "@editor/tools"; -import type { BrushSettings, ChromaKeySettings, GenerateSettings, MagicWandSettings, ToolId } from "@editor/tools"; +import type { BrushSettings, ChromaKeySettings, GenerateIntent, GenerateSettings, MagicWandSettings, ToolId } from "@editor/tools"; import type { Command } from "./command"; import { commandIds } from "./ids"; @@ -16,6 +16,7 @@ export type ToolSetActivePayload = { export type ToolSetBrushSettingsPayload = Partial; export type ToolSetGenerateSettingsPayload = Partial; +export type ToolChooseGenerateIntentPayload = { intent: GenerateIntent }; export type ToolSetChromaKeySettingsPayload = Partial; @@ -120,6 +121,25 @@ export const toolSetGenerateSettingsCommand: Command = { + create: "text-to-image", + replace: "inpaint", + extend: "outpaint", + variations: "image-to-image", +}; + +export const toolChooseGenerateIntentCommand: Command = { + id: commandIds.toolChooseGenerateIntent, + name: "Choose AI edit intent", + execute({ state }, payload) { + const mode = modeByGenerateIntent[payload.intent]; + const architecture = generateArchitectureDefaults[state.editor.tools.generate.architecture].supportedModes.includes(mode) + ? state.editor.tools.generate.architecture + : "sdxl"; + return toolSetGenerateSettingsCommand.execute({ state }, { architecture, mode }); + }, +}; + export const toolSetBrushSettingsCommand: Command = { id: commandIds.toolSetBrushSettings, name: "Set brush settings", @@ -339,6 +359,7 @@ export const toolExitTemporaryPanCommand: Command = { export const toolCommands = [ toolSetActiveCommand, toolSetGenerateSettingsCommand, + toolChooseGenerateIntentCommand, toolSetBrushSettingsCommand, toolSetChromaKeySettingsCommand, toolSetMagicWandSettingsCommand, diff --git a/editor/tools.ts b/editor/tools.ts index eef2bd3..8ca0d00 100644 --- a/editor/tools.ts +++ b/editor/tools.ts @@ -28,6 +28,7 @@ export type ChromaKeySettings = { export type MagicWandMode = "replace" | "add" | "subtract"; export type GenerateMode = "text-to-image" | "image-to-image" | "inpaint" | "outpaint"; +export type GenerateIntent = "create" | "replace" | "extend" | "variations"; export const generateArchitectures = ["sdxl", "z-image", "z-image-turbo", "anima"] as const; export type GenerateArchitecture = (typeof generateArchitectures)[number]; diff --git a/operations/generation/preconditions.test.ts b/operations/generation/preconditions.test.ts index a4beafd..2d40ddf 100644 --- a/operations/generation/preconditions.test.ts +++ b/operations/generation/preconditions.test.ts @@ -15,7 +15,7 @@ describe("generation preconditions", () => { }); test("requires an enabled mask for inpaint", () => { - expect(checkGenerationPreconditions(document(), selected(), settings("inpaint"))).toEqual({ ready: false, message: "Add and enable a layer mask on the selected image before inpainting." }); + expect(checkGenerationPreconditions(document(), selected(), settings("inpaint"))).toEqual({ ready: false, message: "Paint a mask over the area you want AI to replace.", repair: "add-mask" }); }); test("allows inpaint when the selected source has an aligned enabled mask", () => { @@ -24,7 +24,7 @@ describe("generation preconditions", () => { test("explains prompt and outpaint-padding requirements", () => { expect(checkGenerationPreconditions(document(), selected(), { ...settings("image-to-image"), prompt: " " })).toEqual({ ready: false, message: "Enter a prompt to generate an image." }); - expect(checkGenerationPreconditions(document(), selected(), { ...settings("outpaint"), outpaint: { left: 0, top: 0, right: 0, bottom: 0, feathering: 0 } })).toEqual({ ready: false, message: "Add outpaint padding on at least one side." }); + expect(checkGenerationPreconditions(document(), selected(), { ...settings("outpaint"), outpaint: { left: 0, top: 0, right: 0, bottom: 0, feathering: 0 } })).toEqual({ ready: false, message: "Add outpaint padding on at least one side.", repair: "set-outpaint-padding" }); }); }); diff --git a/operations/generation/preconditions.ts b/operations/generation/preconditions.ts index 177532e..654acd9 100644 --- a/operations/generation/preconditions.ts +++ b/operations/generation/preconditions.ts @@ -6,7 +6,7 @@ import type { GenerateSettings } from "@editor/tools"; export type GenerationPrecondition = | { ready: true } - | { ready: false; message: string }; + | { ready: false; message: string; repair?: "add-mask" | "set-outpaint-padding" }; export function checkGenerationPreconditions( document: ImageDocument, @@ -38,14 +38,14 @@ export function checkGenerationPreconditions( if (settings.mode === "outpaint") { const padding = settings.outpaint; if (padding.left + padding.top + padding.right + padding.bottom <= 0) { - return missing("Add outpaint padding on at least one side."); + return missing("Add outpaint padding on at least one side.", "set-outpaint-padding"); } } if (settings.mode !== "inpaint") return { ready: true }; const layerMask = getLayerMask(layerInfo.layer); - if (!layerMask?.enabled) return missing("Add and enable a layer mask on the selected image before inpainting."); + if (!layerMask?.enabled) return missing("Paint a mask over the area you want AI to replace.", "add-mask"); const maskLayer = index.layerById.get(layerMask.maskLayerId); if (!maskLayer || maskLayer.type === "group") return missing("The selected layer mask is missing."); const maskAsset = index.assetById.get(maskLayer.assetId); @@ -69,8 +69,8 @@ function modeSelectionMessage(mode: GenerateSettings["mode"]): string { return "Select exactly one image or raster layer to outpaint."; } -function missing(message: string): GenerationPrecondition { - return { ready: false, message }; +function missing(message: string, repair?: Extract["repair"]): GenerationPrecondition { + return { ready: false, message, ...(repair ? { repair } : {}) }; } function rectsAligned(a: { x: number; y: number; w: number; h: number }, b: { x: number; y: number; w: number; h: number }): boolean { diff --git a/operations/generation/workflow.test.ts b/operations/generation/workflow.test.ts index fc27d73..f877dcb 100644 --- a/operations/generation/workflow.test.ts +++ b/operations/generation/workflow.test.ts @@ -40,6 +40,22 @@ describe("generation workflow", () => { expect(app.store.getState().editor.generation.candidates).toHaveLength(0); }); + test("repairs replace prerequisites by adding an aligned editable mask through commands", async () => { + const app = createTestApp(); + const state = app.store.getState(); + state.document.assets.push({ id: "source-asset", name: "Source", mimeType: "image/png", source: "source", intrinsicSize: { w: 80, h: 60 } }); + state.document.artboards[0]!.layers.push({ id: "source", type: "raster", name: "Source", visible: true, locked: false, opacity: 1, assetId: "source-asset", transform: { position: { x: 4, y: 8 }, scale: { x: 2, y: 2 }, rotation: 0 } }); + state.editor.selection = { artboardId: "artboard", layerIds: ["source"] }; + const ids = ["mask-asset", "mask-layer"]; + const workflow = createGenerationWorkflow(app.store, dependencies({ createId: () => ids.shift() ?? "unused" })); + + await workflow.prepareInpaintMask(); + + const next = app.store.getState(); + expect(next.document.artboards[0]?.layers[0]).toMatchObject({ id: "mask-layer", transform: { position: { x: 4, y: 8 }, scale: { x: 2, y: 2 }, rotation: 0 } }); + expect(next.editor.maskEdit).toEqual({ targetLayerId: "source", maskLayerId: "mask-layer" }); + }); + test("cancels the active operation and records a cancelled job", async () => { const app = createTestApp(); let receivedSignal: AbortSignal | undefined; diff --git a/operations/generation/workflow.ts b/operations/generation/workflow.ts index a3a01bb..4777345 100644 --- a/operations/generation/workflow.ts +++ b/operations/generation/workflow.ts @@ -1,4 +1,5 @@ import { commandIds } from "@commands/ids"; +import type { Layer } from "@core/layer"; import type { GenerationCandidate, GenerationJobKind } from "@editor/state"; import type { AppStore } from "@editor/store"; import type { GenerateSettings } from "@editor/tools"; @@ -50,6 +51,34 @@ export function createGenerationWorkflow(store: AppStore, dependencies: Generati loadResources: () => dependencies.loadGenerationResources(store), + prepareInpaintMask: async () => { + const state = store.getState(); + const layerId = state.editor.selection.layerIds.length === 1 ? state.editor.selection.layerIds[0] : undefined; + if (!layerId) return; + const artboard = state.document.artboards.find((candidate) => candidate.id === state.editor.selection.artboardId); + const layer = artboard ? findLayer(artboard.layers, layerId) : undefined; + if (!layer || layer.type === "group") return; + const asset = state.document.assets.find((candidate) => candidate.id === layer.assetId); + if (!asset) return; + const source = await dependencies.createRefinementMask(asset.intrinsicSize.w, asset.intrinsicSize.h); + const maskAssetId = dependencies.createId(); + const maskLayerId = dependencies.createId(); + store.dispatch(commandIds.documentAddLayerMask, { + layerId, + asset: { id: maskAssetId, name: `${layer.name} AI edit mask`, mimeType: "image/png", source, intrinsicSize: { ...asset.intrinsicSize } }, + maskLayer: { + id: maskLayerId, + type: "raster", + name: `${layer.name} AI edit mask`, + visible: true, + locked: false, + opacity: 1, + assetId: maskAssetId, + transform: { position: { ...layer.transform.position }, scale: { ...layer.transform.scale }, rotation: layer.transform.rotation }, + }, + }); + }, + generate: () => job("generate", "Generating", async (signal) => { const state = store.getState(); await dependencies.runGenerate({ @@ -130,6 +159,17 @@ export function createGenerationWorkflow(store: AppStore, dependencies: Generati }; } +function findLayer(layers: readonly Layer[], layerId: string): Layer | undefined { + for (const layer of layers) { + if (layer.id === layerId) return layer; + if (layer.type === "group") { + const child = findLayer(layer.children, layerId); + if (child) return child; + } + } + return undefined; +} + function findCandidate(store: AppStore, candidateId: string): GenerationCandidate { const candidate = store.getState().editor.generation.candidates.find((item) => item.id === candidateId); if (!candidate) throw new Error("This generation candidate is no longer available."); diff --git a/view/bottom-controls/GenerateActionControls.tsx b/view/bottom-controls/GenerateActionControls.tsx index 28481f5..a0263ff 100644 --- a/view/bottom-controls/GenerateActionControls.tsx +++ b/view/bottom-controls/GenerateActionControls.tsx @@ -20,6 +20,7 @@ export function GenerateActionControls({ settings, generation, dispatch, workflo const precondition = workflow.precondition(); const canGenerate = precondition.ready && !busy; const preconditionMessage = precondition.ready ? undefined : precondition.message; + const repair = precondition.ready ? undefined : precondition.repair; return (
@@ -34,7 +35,16 @@ export function GenerateActionControls({ settings, generation, dispatch, workflo > {busy && job?.kind === "generate" ? "Generating..." : "Generate"} - {preconditionMessage ? {preconditionMessage} : null} + {preconditionMessage ? ( + + {preconditionMessage} + {repair === "add-mask" ? ( + + ) : repair === "set-outpaint-padding" ? ( + + ) : null} + + ) : null} {busy ? : null} {candidate ? ( @@ -56,25 +66,26 @@ export function GenerateActionControls({ settings, generation, dispatch, workflo function CandidatePicker({ generation, dispatch }: { generation: GenerationState; dispatch: AppStore["dispatch"] }) { return ( -
+
{generation.candidates.map((candidate) => { const selected = candidate.id === (generation.selectedCandidateId ?? generation.candidates[0]?.id); return ( ); })} + ))} + +
+ +
dispatch(commandIds.toolSetGenerateSettings, { outpaint: { ...settings.outpaint, feathering } })} />
- + : null} -
+ {settings.mode === "inpaint" ?
+
: null} ); } +const intentOptions: ReadonlyArray<{ value: GenerateIntent; mode: GenerateMode; label: string; description: string }> = [ + { value: "create", mode: "text-to-image", label: "Create", description: "Make a new image from your prompt." }, + { value: "replace", mode: "inpaint", label: "Replace", description: "Regenerate the masked part of one layer." }, + { value: "extend", mode: "outpaint", label: "Extend", description: "Grow one selected image beyond its edges." }, + { value: "variations", mode: "image-to-image", label: "Variations", description: "Explore alternatives based on one image." }, +]; + function SectionTitle({ title }: { title: string }) { return
{title}
; }