syntaxbullet/image-studio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add inpaint region functionality and related tools

Browse Source 
- Enhanced cursor behavior for new tools: semantic select, mask lasso, and mask rectangle.
- Updated mask edit state to include mask asset ID and kind.
- Implemented inpaint region commands for adding, applying, and removing inpaint regions.
- Introduced new operations for lasso and semantic selection tools.
- Created UI components for candidate review and inpaint region management.
- Added tests for inpaint region commands to ensure functionality.
- Updated various components to support new inpaint features and improve user experience.
This commit is contained in:
syntaxbullet
2026-07-11 16:41:22 +02:00
parent f4e13b80e7
commit ff762b8f17
78 changed files with 1632 additions and 301 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
operations/generation/candidateActions.ts
View File

@@ -1,16 +1,24 @@
import type { ImageDocument } from "@core/document";
import type { GenerationCandidate } from "@editor/state";
import { loadImageCanvas, maskValueFromRgba } from "@platform/browser/maskRaster";
import { createContentRevision } from "./inpaintPrep";
export async function createMaskedPixelReplacementSource(document: ImageDocument, candidate: GenerationCandidate): Promise<string> {
if (!candidate.inpaint) throw new Error("Only inpaint candidates can replace masked pixels.");
const targetAsset = document.assets.find((asset) => asset.id === candidate.inpaint?.sourceAssetId);
if (!targetAsset) throw new Error("The source layer for this candidate no longer exists.");
const region = document.inpaintRegions.find((item) => item.id === candidate.inpaint?.regionId);
const maskAsset = region ? document.assets.find((asset) => asset.id === region.maskAssetId) : undefined;
if (!region || !maskAsset) throw new Error("The AI edit region for this candidate no longer exists.");
const [sourceRevision, maskRevision] = await Promise.all([createContentRevision(targetAsset.source), createContentRevision(maskAsset.source)]);
if (sourceRevision !== candidate.inpaint.revision.source || maskRevision !== candidate.inpaint.revision.mask) {
throw new Error("The source or AI edit region changed after generation. Rebuild candidates from the current edit region before replacing pixels.");
}
const targetCanvas = await loadImageCanvas(targetAsset.source, targetAsset.intrinsicSize.w, targetAsset.intrinsicSize.h);
const generatedCanvas = await loadImageCanvas(candidate.source, candidate.width, candidate.height);
const maskCanvas = await loadImageCanvas(candidate.inpaint.maskImage, candidate.width, candidate.height);
const maskCanvas = await loadImageCanvas(candidate.inpaint.blendMaskImage, candidate.width, candidate.height);
const targetContext = require2dContext(targetCanvas);
const generatedContext = require2dContext(generatedCanvas);
@@ -19,6 +27,7 @@ export async function createMaskedPixelReplacementSource(document: ImageDocument
const generatedData = generatedContext.getImageData(0, 0, generatedCanvas.width, generatedCanvas.height);
const maskData = maskContext.getImageData(0, 0, maskCanvas.width, maskCanvas.height);
const crop = candidate.inpaint.crop.assetBounds;
const colorOffset = candidate.settings.inpaint.colorMatch ? boundaryColorOffset(targetData.data, generatedData.data, maskData.data, candidate.width, candidate.height, crop, targetCanvas.width, targetCanvas.height) : [0, 0, 0];
for (let y = 0; y < candidate.height; y += 1) {
for (let x = 0; x < candidate.width; x += 1) {
@@ -33,7 +42,8 @@ export async function createMaskedPixelReplacementSource(document: ImageDocument
for (let channel = 0; channel < 4; channel += 1) {
const previous = targetData.data[targetIndex + channel] ?? 0;
const next = generatedData.data[generatedIndex + channel] ?? previous;
const rawNext = generatedData.data[generatedIndex + channel] ?? previous;
const next = channel < 3 ? Math.max(0, Math.min(255, rawNext + (colorOffset[channel] ?? 0))) : rawNext;
targetData.data[targetIndex + channel] = Math.round(previous * (1 - mask) + next * mask);
}
}
@@ -43,6 +53,28 @@ export async function createMaskedPixelReplacementSource(document: ImageDocument
return targetCanvas.toDataURL("image/png");
}
function boundaryColorOffset(target: Uint8ClampedArray, generated: Uint8ClampedArray, mask: Uint8ClampedArray, width: number, height: number, crop: { x: number; y: number }, targetWidth: number, targetHeight: number): number[] {
const targetTotal = [0, 0, 0];
const generatedTotal = [0, 0, 0];
let count = 0;
for (let y = 0; y < height; y += 1) for (let x = 0; x < width; x += 1) {
const index = (y * width + x) * 4;
const amount = maskValueFromRgba(mask, index) / 255;
if (amount <= 0.05 || amount >= 0.65) continue;
const targetX = Math.round(crop.x) + x;
const targetY = Math.round(crop.y) + y;
if (targetX < 0 || targetY < 0 || targetX >= targetWidth || targetY >= targetHeight) continue;
const targetIndex = (targetY * targetWidth + targetX) * 4;
for (let channel = 0; channel < 3; channel += 1) {
targetTotal[channel] = (targetTotal[channel] ?? 0) + (target[targetIndex + channel] ?? 0);
generatedTotal[channel] = (generatedTotal[channel] ?? 0) + (generated[index + channel] ?? 0);
}
count += 1;
}
if (count < 16) return [0, 0, 0];
return targetTotal.map((total, channel) => Math.max(-32, Math.min(32, total / count - (generatedTotal[channel] ?? 0) / count)));
}
function require2dContext(canvas: HTMLCanvasElement): CanvasRenderingContext2D {
const context = canvas.getContext("2d");
if (!context) throw new Error("Unable to prepare generated candidate");

4
operations/generation/generationJob.ts
View File

@@ -7,7 +7,7 @@ export async function runGenerationJob(options: {
label: string;
dispatch: AppStore["dispatch"];
signal: AbortSignal;
task: (signal: AbortSignal) => Promise<void>;
task: (signal: AbortSignal, report: (progress: number, detail: string) => void) => Promise<void>;
}): Promise<void> {
const jobId = crypto.randomUUID();
const startedAt = Date.now();
@@ -15,7 +15,7 @@ export async function runGenerationJob(options: {
if (!nextState.editor.generation.jobs.some((job) => job.id === jobId && job.status === "running")) return;
try {
await options.task(options.signal);
await options.task(options.signal, (progress, detail) => options.dispatch(commandIds.generationUpdateJob, { jobId, progress, detail }));
options.dispatch(commandIds.generationSucceedJob, { jobId, finishedAt: Date.now() });
} catch (reason: unknown) {
if (options.signal.aborted) {

156
operations/generation/inpaintPrep.ts
View File

@@ -1,22 +1,28 @@
import type { Asset } from "@core/asset";
import type { ImageDocument } from "@core/document";
import type { Rect } from "@core/geometry";
import type { AssetId, InpaintRegionId, LayerId } from "@core/id";
import type { Layer } from "@core/layer";
import { getLayerMask } from "@core/layer-mask-utils";
import type { SelectionState } from "@editor/state";
import type { GenerateSettings } from "@editor/tools";
import { createDocumentReadIndex, resolveIndexedLayerBounds } from "@editor/document-indexes";
import { createNormalizedMaskSource, cropCanvas, cropMaskValuesToDataUrl, expandRectWithinBounds, loadImageCanvas } from "@platform/browser/maskRaster";
import { createNormalizedMaskSource, cropCanvas, cropMaskValuesToDataUrl, expandRectWithinBounds, loadImageCanvas, sampleDocumentCanvasInLayerSpace } from "@platform/browser/maskRaster";
import { renderArtboardCanvas } from "@platform/browser/exportArtboardPng";
export type InpaintBundle = {
inputImage: string;
sourceImage: string;
contextImage: string;
maskImage: string;
editMaskImage: string;
blendMaskImage: string;
width: number;
height: number;
targetLayerId: string;
maskLayerId: string;
sourceAssetId: string;
maskAssetId: string;
targetLayerId: LayerId;
regionId: InpaintRegionId;
sourceAssetId: AssetId;
maskAssetId: AssetId;
revision: { source: string; mask: string };
crop: {
assetBounds: Rect;
documentBounds: Rect;
@@ -46,10 +52,8 @@ type InpaintTarget = {
artboardId: string;
layer: Extract<Layer, { type: "image" | "raster" }>;
asset: Asset;
bounds: Rect;
maskLayer: Extract<Layer, { type: "image" | "raster" }>;
maskAsset: Asset;
maskBounds: Rect;
regionId: InpaintRegionId;
};
const modelMultiple = 8;
@@ -62,41 +66,65 @@ export async function buildInpaintBundle(document: ImageDocument, selection: Sel
const width = Math.max(1, Math.round(target.asset.intrinsicSize.w));
const height = Math.max(1, Math.round(target.asset.intrinsicSize.h));
const normalizedMask = await createNormalizedMaskSource(target.maskAsset.source, width, height, {
polarity: settings.inpaint.maskPolarity,
const sourceLimit = target.layer.sourceRect ?? { x: 0, y: 0, w: width, h: height };
const editMask = await createNormalizedMaskSource(target.maskAsset.source, width, height, {
polarity: "revealed",
despeckle: settings.inpaint.maskDespeckle,
limit: sourceLimit,
});
const noiseMask = await createNormalizedMaskSource(target.maskAsset.source, width, height, {
polarity: "revealed",
expand: settings.inpaint.maskExpand,
feather: settings.inpaint.maskFeather,
blur: settings.inpaint.maskBlur,
despeckle: settings.inpaint.maskDespeckle,
limit: sourceLimit,
});
const blendMask = await createNormalizedMaskSource(target.maskAsset.source, width, height, {
polarity: "revealed",
feather: settings.inpaint.maskFeather,
despeckle: settings.inpaint.maskDespeckle,
limit: sourceLimit,
});
if (!normalizedMask.bounds) throw new Error("The selected layer mask has no inpaint pixels.");
if (!editMask.bounds || !noiseMask.bounds) throw new Error("Paint over the area you want AI to replace.");
const crop = settings.inpaint.maskedAreaOnly
? expandRectWithinBounds(normalizedMask.bounds, settings.inpaint.cropPadding, { w: width, h: height }, modelMultiple, minModelSize)
? expandRectWithinBounds(noiseMask.bounds, settings.inpaint.cropPadding, { w: width, h: height }, modelMultiple, minModelSize)
: { x: 0, y: 0, w: width, h: height };
const outputWidth = toModelSize(crop.w, "width");
const outputHeight = toModelSize(crop.h, "height");
const inputCanvas = prepareMaskedContentInputCanvas(await loadImageCanvas(target.asset.source, width, height), normalizedMask.values, settings.inpaint.maskedContent);
const inputImage = cropCanvas(inputCanvas, crop, outputWidth, outputHeight);
const maskImage = cropMaskValuesToDataUrl(normalizedMask.values, width, height, crop, outputWidth, outputHeight);
const scaleX = target.bounds.w / width;
const scaleY = target.bounds.h / height;
const sourceCanvas = await loadImageCanvas(target.asset.source, width, height);
const sourceImage = cropCanvas(sourceCanvas, crop, outputWidth, outputHeight);
const preparedCanvas = prepareMaskedContentInputCanvas(await loadImageCanvas(target.asset.source, width, height), editMask.values, settings.inpaint.maskedContent);
const preparedSourceImage = cropCanvas(preparedCanvas, crop, outputWidth, outputHeight);
const artboard = document.artboards.find((candidate) => candidate.id === target.artboardId);
if (!artboard) throw new Error("The target artboard no longer exists.");
const contextImage = sampleDocumentCanvasInLayerSpace(await renderArtboardCanvas(artboard, document.assets), artboard.bounds, target.layer, target.asset.intrinsicSize, crop, outputWidth, outputHeight);
const inputImage = await mergePreparedRegionIntoContext(contextImage, preparedSourceImage, cropMaskValuesToDataUrl(editMask.values, width, height, crop, outputWidth, outputHeight), outputWidth, outputHeight);
const maskImage = cropMaskValuesToDataUrl(noiseMask.values, width, height, crop, outputWidth, outputHeight);
const editMaskImage = cropMaskValuesToDataUrl(editMask.values, width, height, crop, outputWidth, outputHeight);
const blendMaskImage = cropMaskValuesToDataUrl(blendMask.values, width, height, crop, outputWidth, outputHeight);
const scaleX = target.layer.transform.scale.x;
const scaleY = target.layer.transform.scale.y;
const documentBounds = {
x: target.bounds.x + crop.x * scaleX,
y: target.bounds.y + crop.y * scaleY,
x: target.layer.transform.position.x + crop.x * scaleX,
y: target.layer.transform.position.y + crop.y * scaleY,
w: outputWidth * scaleX,
h: outputHeight * scaleY,
};
return {
inputImage,
sourceImage,
contextImage,
maskImage,
editMaskImage,
blendMaskImage,
width: outputWidth,
height: outputHeight,
targetLayerId: target.layer.id,
maskLayerId: target.maskLayer.id,
regionId: target.regionId,
sourceAssetId: target.asset.id,
maskAssetId: target.maskAsset.id,
crop: {
@@ -106,14 +134,14 @@ export async function buildInpaintBundle(document: ImageDocument, selection: Sel
maskedAreaOnly: settings.inpaint.maskedAreaOnly,
},
mask: {
polarity: settings.inpaint.maskPolarity,
activeBounds: normalizedMask.bounds,
polarity: "revealed",
activeBounds: editMask.bounds,
},
placement: {
artboardId: target.artboardId,
layerName: `${target.layer.name} inpaint`,
transform: {
position: { x: documentBounds.x, y: documentBounds.y },
position: rotatedCropPosition(target.layer, target.asset, { ...crop, w: outputWidth, h: outputHeight }),
scale: { x: documentBounds.w / outputWidth, y: documentBounds.h / outputHeight },
rotation: target.layer.transform.rotation,
},
@@ -126,6 +154,10 @@ export async function buildInpaintBundle(document: ImageDocument, selection: Sel
maskExpand: settings.inpaint.maskExpand,
cropPadding: settings.inpaint.cropPadding,
},
revision: {
source: await createContentRevision(target.asset.source),
mask: await createContentRevision(target.maskAsset.source),
},
};
}
@@ -138,20 +170,14 @@ function resolveInpaintTarget(document: ImageDocument, selection: SelectionState
const asset = documentIndex.assetById.get(layerInfo.layer.assetId);
if (!asset) throw new Error("The selected layer is missing its source image.");
const layerMask = getLayerMask(layerInfo.layer);
if (!layerMask?.enabled) throw new Error("Add a layer mask before running inpaint.");
const region = document.inpaintRegions.find((candidate) => candidate.targetLayerId === layerInfo.layer.id && candidate.enabled);
if (!region) throw new Error("Add an AI edit region before running inpaint.");
const maskAsset = documentIndex.assetById.get(region.maskAssetId);
if (!maskAsset) throw new Error("The AI edit region is missing its mask data.");
const maskLayer = documentIndex.layerById.get(layerMask.maskLayerId);
if (!maskLayer || (maskLayer.type !== "image" && maskLayer.type !== "raster")) throw new Error("The selected layer mask is missing.");
if (!resolveIndexedLayerBounds(documentIndex, layerInfo.layer)) throw new Error("Unable to resolve the selected layer bounds.");
const maskAsset = documentIndex.assetById.get(maskLayer.assetId);
if (!maskAsset) throw new Error("The selected layer mask is missing its image data.");
const bounds = resolveIndexedLayerBounds(documentIndex, layerInfo.layer);
const maskBounds = resolveIndexedLayerBounds(documentIndex, maskLayer);
if (!bounds || !maskBounds) throw new Error("Unable to resolve the selected layer and mask bounds.");
return { artboardId: layerInfo.artboardId, layer: layerInfo.layer, asset, bounds, maskLayer, maskAsset, maskBounds };
return { artboardId: layerInfo.artboardId, layer: layerInfo.layer, asset, maskAsset, regionId: region.id };
}
function validateInpaintTarget(target: InpaintTarget) {
@@ -160,13 +186,6 @@ function validateInpaintTarget(target: InpaintTarget) {
if (Math.round(target.asset.intrinsicSize.w) !== Math.round(target.maskAsset.intrinsicSize.w) || Math.round(target.asset.intrinsicSize.h) !== Math.round(target.maskAsset.intrinsicSize.h)) {
throw new Error("The selected layer and mask image sizes do not match.");
}
if (!rectsAligned(target.bounds, target.maskBounds) || Math.abs(target.layer.transform.rotation - target.maskLayer.transform.rotation) > 0.001) {
throw new Error("The selected layer and mask are not aligned.");
}
}
function rectsAligned(a: Rect, b: Rect): boolean {
return Math.abs(a.x - b.x) <= 0.5 && Math.abs(a.y - b.y) <= 0.5 && Math.abs(a.w - b.w) <= 0.5 && Math.abs(a.h - b.h) <= 0.5;
}
function toModelSize(value: number, axis: "width" | "height"): number {
@@ -258,3 +277,52 @@ function blendChannel(previous: number, next: number, amount: number): number {
function clampNumber(value: number, min: number, max: number): number {
return Math.max(min, Math.min(max, value));
}
async function mergePreparedRegionIntoContext(contextSource: string, preparedSource: string, editMaskSource: string, width: number, height: number): Promise<string> {
const [contextCanvas, preparedCanvas, maskCanvas] = await Promise.all([
loadImageCanvas(contextSource, width, height),
loadImageCanvas(preparedSource, width, height),
loadImageCanvas(editMaskSource, width, height),
]);
const context = contextCanvas.getContext("2d");
const prepared = preparedCanvas.getContext("2d");
const mask = maskCanvas.getContext("2d");
if (!context || !prepared || !mask) return contextSource;
const contextData = context.getImageData(0, 0, width, height);
const preparedData = prepared.getImageData(0, 0, width, height);
const maskData = mask.getImageData(0, 0, width, height);
for (let pixel = 0; pixel < width * height; pixel += 1) {
const amount = (maskData.data[pixel * 4] ?? 0) / 255;
if (amount <= 0) continue;
for (let channel = 0; channel < 4; channel += 1) {
const index = pixel * 4 + channel;
contextData.data[index] = blendChannel(contextData.data[index] ?? 0, preparedData.data[index] ?? 0, amount);
}
}
context.putImageData(contextData, 0, 0);
return contextCanvas.toDataURL("image/png");
}
function rotatedCropPosition(layer: InpaintTarget["layer"], asset: Asset, crop: Rect) {
const source = layer.sourceRect ?? { x: 0, y: 0, ...asset.intrinsicSize };
const scale = layer.transform.scale;
const originalCenter = {
x: layer.transform.position.x + (source.x + source.w / 2) * scale.x,
y: layer.transform.position.y + (source.y + source.h / 2) * scale.y,
};
const cropCenter = {
x: layer.transform.position.x + (crop.x + crop.w / 2) * scale.x,
y: layer.transform.position.y + (crop.y + crop.h / 2) * scale.y,
};
const dx = cropCenter.x - originalCenter.x;
const dy = cropCenter.y - originalCenter.y;
const cos = Math.cos(layer.transform.rotation);
const sin = Math.sin(layer.transform.rotation);
const rotatedCenter = { x: originalCenter.x + dx * cos - dy * sin, y: originalCenter.y + dx * sin + dy * cos };
return { x: rotatedCenter.x - crop.w * scale.x / 2, y: rotatedCenter.y - crop.h * scale.y / 2 };
}
export async function createContentRevision(source: string): Promise<string> {
const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(source));
return Array.from(new Uint8Array(digest), (byte) => byte.toString(16).padStart(2, "0")).join("");
}

6
operations/generation/outputPlacement.test.ts
View File

@@ -42,8 +42,8 @@ describe("generated output placement", () => {
settings: settings("inpaint"),
intrinsicSize: { w: 128, h: 64 },
inpaintBundle: {
inputImage: "input", maskImage: "mask", width: 256, height: 128,
targetLayerId: "source", maskLayerId: "mask", sourceAssetId: "source-asset", maskAssetId: "mask-asset",
inputImage: "input", sourceImage: "source", contextImage: "context", maskImage: "mask", editMaskImage: "edit", blendMaskImage: "blend", width: 256, height: 128,
targetLayerId: "source", regionId: "region", sourceAssetId: "source-asset", maskAssetId: "mask-asset", revision: { source: "source-revision", mask: "mask-revision" },
crop: { assetBounds: { x: 0, y: 0, w: 256, h: 128 }, documentBounds: { x: 140, y: 150, w: 384, h: 64 }, padding: 16, maskedAreaOnly: true },
mask: { polarity: "hidden", activeBounds: { x: 20, y: 20, w: 40, h: 40 } },
placement: { artboardId: "artboard", layerName: "Source inpaint", transform: { position: { x: 140, y: 150 }, scale: { x: 1.5, y: 0.5 }, rotation: 12 } },
@@ -81,7 +81,7 @@ function selected() {
function document(): ImageDocument {
return {
id: "document", name: "Test", version: 1,
id: "document", name: "Test", version: 1, inpaintRegions: [],
assets: [
{ id: "source-asset", name: "Source", mimeType: "image/png", source: "source", intrinsicSize: { w: 100, h: 500 } },
{ id: "mask-asset", name: "Mask", mimeType: "image/png", source: "mask", intrinsicSize: { w: 100, h: 500 } },

4
operations/generation/preconditions.test.ts
View File

@@ -15,7 +15,7 @@ describe("generation preconditions", () => {
});
test("requires an enabled mask for inpaint", () => {
expect(checkGenerationPreconditions(document(), selected(), settings("inpaint"))).toEqual({ ready: false, message: "Paint a mask over the area you want AI to replace.", repair: "add-mask" });
expect(checkGenerationPreconditions(document(), selected(), settings("inpaint"))).toEqual({ ready: false, message: "Paint an AI edit region over the area you want to replace.", repair: "add-mask" });
});
test("allows inpaint when the selected source has an aligned enabled mask", () => {
@@ -46,6 +46,7 @@ function document(masked = false): ImageDocument {
{ id: "source-asset", name: "Source", mimeType: "image/png", source: "source", intrinsicSize: { w: 100, h: 100 } },
{ id: "mask-asset", name: "Mask", mimeType: "image/png", source: "mask", intrinsicSize: { w: 100, h: 100 } },
],
inpaintRegions: masked ? [{ id: "region", name: "AI edit", targetLayerId: "source", maskAssetId: "mask-asset", enabled: true }] : [],
artboards: [{
id: "artboard",
name: "Artboard",
@@ -64,7 +65,6 @@ function document(masked = false): ImageDocument {
opacity: 1,
assetId: "source-asset",
transform,
layerMask: masked ? { kind: "raster", maskLayerId: "mask", enabled: true, inverted: false } : undefined,
},
],
}],

21
operations/generation/preconditions.ts
View File

@@ -1,5 +1,4 @@
import type { ImageDocument } from "@core/document";
import { getLayerMask } from "@core/layer-mask-utils";
import { createDocumentReadIndex, resolveIndexedLayerBounds } from "@editor/document-indexes";
import type { SelectionState } from "@editor/state";
import type { GenerateSettings } from "@editor/tools";
@@ -44,21 +43,15 @@ export function checkGenerationPreconditions(
if (settings.mode !== "inpaint") return { ready: true };
const layerMask = getLayerMask(layerInfo.layer);
if (!layerMask?.enabled) return missing("Paint a mask over the area you want AI to replace.", "add-mask");
const maskLayer = index.layerById.get(layerMask.maskLayerId);
if (!maskLayer || (maskLayer.type !== "image" && maskLayer.type !== "raster")) return missing("The selected layer mask is missing.");
const maskAsset = index.assetById.get(maskLayer.assetId);
if (!maskAsset) return missing("The selected layer mask is missing its image data.");
const region = document.inpaintRegions.find((candidate) => candidate.targetLayerId === layerInfo.layer.id && candidate.enabled);
if (!region) return missing("Paint an AI edit region over the area you want to replace.", "add-mask");
const maskAsset = index.assetById.get(region.maskAssetId);
if (!maskAsset) return missing("The AI edit region is missing its mask data.");
if (Math.round(asset.intrinsicSize.w) !== Math.round(maskAsset.intrinsicSize.w) || Math.round(asset.intrinsicSize.h) !== Math.round(maskAsset.intrinsicSize.h)) {
return missing("The selected layer and mask image sizes must match.");
}
const layerBounds = resolveIndexedLayerBounds(index, layerInfo.layer);
const maskBounds = resolveIndexedLayerBounds(index, maskLayer);
if (!layerBounds || !maskBounds || !rectsAligned(layerBounds, maskBounds) || Math.abs(layerInfo.layer.transform.rotation - maskLayer.transform.rotation) > 0.001) {
return missing("Align the selected layer and its mask before inpainting.");
}
if (!resolveIndexedLayerBounds(index, layerInfo.layer)) return missing("The selected layer has invalid geometry.");
return { ready: true };
}
@@ -72,7 +65,3 @@ function modeSelectionMessage(mode: GenerateSettings["mode"]): string {
function missing(message: string, repair?: Extract<GenerationPrecondition, { ready: false }>["repair"]): GenerationPrecondition {
return { ready: false, message, ...(repair ? { repair } : {}) };
}
function rectsAligned(a: { x: number; y: number; w: number; h: number }, b: { x: number; y: number; w: number; h: number }): boolean {
return Math.abs(a.x - b.x) <= 0.5 && Math.abs(a.y - b.y) <= 0.5 && Math.abs(a.w - b.w) <= 0.5 && Math.abs(a.h - b.h) <= 0.5;
}

86
operations/generation/runGenerate.ts
View File

@@ -18,6 +18,7 @@ export async function runGenerate(options: {
settings: GenerateSettings;
dispatch: AppStore["dispatch"];
signal?: AbortSignal;
onProgress?: (progress: number, detail: string) => void;
}) {
const { document, selection, settings, dispatch } = options;
const precondition = checkGenerationPreconditions(document, selection, settings);
@@ -41,25 +42,27 @@ export async function runGenerate(options: {
maskImage,
inpaintBundle,
signal: options.signal,
onProgress: options.onProgress,
});
const intrinsicSize = await loadImageSize(generated.source);
const placement = resolveGeneratedOutputPlacement({ document, selection, settings, intrinsicSize, inpaintBundle });
dispatch(commandIds.generationAddCandidate, {
candidate: createGenerationCandidate({
source: generated.source,
mimeType: generated.mimeType,
intrinsicSize,
settings: requestSettings,
seed,
width,
height,
inputImage,
maskImage,
placement,
inpaintBundle,
}),
});
for (const result of generated.results) {
const intrinsicSize = await loadImageSize(result.source);
const placement = resolveGeneratedOutputPlacement({ document, selection, settings, intrinsicSize, inpaintBundle });
dispatch(commandIds.generationAddCandidate, {
candidate: createGenerationCandidate({
source: result.source,
mimeType: result.mimeType,
intrinsicSize,
settings: requestSettings,
seed: result.seed || seed,
width,
height,
inputImage,
maskImage,
placement,
inpaintBundle,
}),
});
}
}
export async function runGenerateFromCandidate(options: {
@@ -67,6 +70,7 @@ export async function runGenerateFromCandidate(options: {
settings?: GenerateSettings;
dispatch: AppStore["dispatch"];
signal?: AbortSignal;
onProgress?: (progress: number, detail: string) => void;
}) {
const settings = options.settings ?? options.candidate.settings;
const seed = resolveSeed(settings.seed);
@@ -79,20 +83,22 @@ export async function runGenerateFromCandidate(options: {
maskImage: options.candidate.maskImage,
inpaintCandidate: options.candidate,
signal: options.signal,
onProgress: options.onProgress,
});
const intrinsicSize = await loadImageSize(generated.source);
options.dispatch(commandIds.generationAddCandidate, {
candidate: {
...options.candidate,
id: crypto.randomUUID(),
source: generated.source,
mimeType: generated.mimeType,
intrinsicSize,
settings: requestSettings,
seed,
},
});
for (const result of generated.results) {
const intrinsicSize = await loadImageSize(result.source);
options.dispatch(commandIds.generationAddCandidate, {
candidate: {
...options.candidate,
id: crypto.randomUUID(),
source: result.source,
mimeType: result.mimeType,
intrinsicSize,
settings: requestSettings,
seed: result.seed || seed,
},
});
}
}
function createGenerationCandidate(options: {
@@ -120,15 +126,19 @@ function createGenerationCandidate(options: {
height: options.height,
inputImage: options.inputImage,
maskImage: options.maskImage,
blendMaskImage: options.inpaintBundle?.blendMaskImage,
placement: options.placement,
inpaint: options.inpaintBundle
? {
targetLayerId: options.inpaintBundle.targetLayerId,
maskLayerId: options.inpaintBundle.maskLayerId,
regionId: options.inpaintBundle.regionId,
sourceAssetId: options.inpaintBundle.sourceAssetId,
maskAssetId: options.inpaintBundle.maskAssetId,
inputImage: options.inpaintBundle.inputImage,
maskImage: options.inpaintBundle.maskImage,
editMaskImage: options.inpaintBundle.editMaskImage,
blendMaskImage: options.inpaintBundle.blendMaskImage,
revision: options.inpaintBundle.revision,
crop: options.inpaintBundle.crop,
mask: options.inpaintBundle.mask,
backend: options.inpaintBundle.backend,
@@ -146,6 +156,7 @@ async function requestGenerate(options: {
inpaintBundle?: InpaintBundle;
inpaintCandidate?: GenerationCandidate;
signal?: AbortSignal;
onProgress?: (progress: number, detail: string) => void;
}) {
return requestGeneration({
architecture: options.settings.architecture,
@@ -163,11 +174,14 @@ async function requestGenerate(options: {
scheduler: options.settings.scheduler,
width: options.width,
height: options.height,
batchSize: options.settings.batchSize,
refinePass: options.settings.refinePass,
refineStrength: options.settings.refineStrength,
outpaint: options.settings.outpaint,
inpaint: resolveInpaintRequest(options.inpaintBundle, options.inpaintCandidate, options.settings),
inputImage: options.inputImage,
maskImage: options.maskImage,
}, options.signal);
}, options.signal, options.onProgress);
}
function resolveInpaintRequest(inpaintBundle: InpaintBundle | undefined, inpaintCandidate: GenerationCandidate | undefined, settings: GenerateSettings) {
@@ -182,6 +196,9 @@ function resolveInpaintRequest(inpaintBundle: InpaintBundle | undefined, inpaint
maskPolarity: inpaintBundle.mask.polarity,
crop: inpaintBundle.crop,
placement: inpaintBundle.placement,
structureControl: settings.inpaint.structureControl,
controlStrength: settings.inpaint.controlStrength,
controlModel: settings.inpaint.controlModel,
};
}
@@ -196,6 +213,9 @@ function resolveInpaintRequest(inpaintBundle: InpaintBundle | undefined, inpaint
maskPolarity: inpaintCandidate.inpaint.mask.polarity,
crop: inpaintCandidate.inpaint.crop,
placement: inpaintCandidate.placement,
structureControl: settings.inpaint.structureControl,
controlStrength: settings.inpaint.controlStrength,
controlModel: settings.inpaint.controlModel,
};
}

10
operations/generation/workflow.test.ts
View File

@@ -1,5 +1,6 @@
import { describe, expect, test } from "bun:test";
import { documentCommands } from "@commands/document";
import { inpaintRegionCommands } from "@commands/inpaint-region";
import { generationCommands } from "@commands/generation";
import { commandIds } from "@commands/ids";
import { createCommandRegistry } from "@commands/registry";
@@ -46,14 +47,14 @@ describe("generation workflow", () => {
state.document.assets.push({ id: "source-asset", name: "Source", mimeType: "image/png", source: "source", intrinsicSize: { w: 80, h: 60 } });
state.document.artboards[0]!.layers.push({ id: "source", type: "raster", name: "Source", visible: true, locked: false, opacity: 1, assetId: "source-asset", transform: { position: { x: 4, y: 8 }, scale: { x: 2, y: 2 }, rotation: 0 } });
state.editor.selection = { artboardId: "artboard", layerIds: ["source"] };
const ids = ["mask-asset", "mask-layer"];
const ids = ["mask-asset", "region"];
const workflow = createGenerationWorkflow(app.store, dependencies({ createId: () => ids.shift() ?? "unused" }));
await workflow.prepareInpaintMask();
const next = app.store.getState();
expect(next.document.artboards[0]?.layers[0]).toMatchObject({ id: "mask-layer", transform: { position: { x: 4, y: 8 }, scale: { x: 2, y: 2 }, rotation: 0 } });
expect(next.editor.maskEdit).toEqual({ targetLayerId: "source", maskLayerId: "mask-layer" });
expect(next.document.inpaintRegions).toContainEqual({ id: "region", name: "Source AI edit", targetLayerId: "source", maskAssetId: "mask-asset", enabled: true });
expect(next.editor.maskEdit).toEqual({ kind: "inpaintRegion", targetLayerId: "source", inpaintRegionId: "region", maskAssetId: "mask-asset", viewMode: "overlay" });
});
test("cancels the active operation and records a cancelled job", async () => {
@@ -82,6 +83,7 @@ function dependencies(overrides: Partial<GenerationWorkflowDependencies>): Gener
runGenerateFromCandidate: async () => undefined,
createMaskedPixelReplacementSource: async () => "replacement",
createRefinementMask: async () => "mask",
createInpaintRegionMask: async () => "mask",
loadGenerationResources: async () => undefined,
createId: () => crypto.randomUUID(),
...overrides,
@@ -118,6 +120,6 @@ function createTestApp() {
locked: false,
layers: [],
});
const registry = createCommandRegistry([...documentCommands, ...toolCommands, ...generationCommands]);
const registry = createCommandRegistry([...documentCommands, ...inpaintRegionCommands, ...toolCommands, ...generationCommands]);
return { store: createAppStore(state, registry) };
}

57
operations/generation/workflow.ts
View File

@@ -3,7 +3,7 @@ import type { Layer } from "@core/layer";
import type { GenerationCandidate, GenerationJobKind } from "@editor/state";
import type { AppStore } from "@editor/store";
import type { GenerateSettings } from "@editor/tools";
import { createRefinementMask } from "@operations/masks/rasterActions";
import { createInpaintRegionMask, createRefinementMask } from "@operations/masks/rasterActions";
import { createMaskedPixelReplacementSource } from "./candidateActions";
import { runGenerationJob } from "./generationJob";
import { loadGenerationResources } from "./loadResources";
@@ -17,6 +17,7 @@ export type GenerationWorkflowDependencies = {
runGenerateFromCandidate: typeof runGenerateFromCandidate;
createMaskedPixelReplacementSource: typeof createMaskedPixelReplacementSource;
createRefinementMask: typeof createRefinementMask;
createInpaintRegionMask: typeof createInpaintRegionMask;
loadGenerationResources: typeof loadGenerationResources;
createId(): string;
};
@@ -26,13 +27,14 @@ const defaultDependencies: GenerationWorkflowDependencies = {
runGenerateFromCandidate,
createMaskedPixelReplacementSource,
createRefinementMask,
createInpaintRegionMask,
loadGenerationResources,
createId: () => crypto.randomUUID(),
};
export function createGenerationWorkflow(store: AppStore, dependencies: GenerationWorkflowDependencies = defaultDependencies) {
let activeController: AbortController | undefined;
const job = async (kind: GenerationJobKind, label: string, task: (signal: AbortSignal) => Promise<void>) => {
const job = async (kind: GenerationJobKind, label: string, task: (signal: AbortSignal, report: (progress: number, detail: string) => void) => Promise<void>) => {
if (store.getState().editor.generation.jobs.some((candidate) => candidate.status === "running")) return;
const controller = new AbortController();
activeController = controller;
@@ -60,26 +62,22 @@ export function createGenerationWorkflow(store: AppStore, dependencies: Generati
if (!layer || (layer.type !== "image" && layer.type !== "raster")) return;
const asset = state.document.assets.find((candidate) => candidate.id === layer.assetId);
if (!asset) return;
const source = await dependencies.createRefinementMask(asset.intrinsicSize.w, asset.intrinsicSize.h);
const existing = state.document.inpaintRegions.find((region) => region.targetLayerId === layerId && region.enabled);
if (existing) {
store.dispatch(commandIds.toolEnterInpaintRegionEdit, { targetLayerId: layerId, regionId: existing.id });
return;
}
const source = await dependencies.createInpaintRegionMask(asset.intrinsicSize.w, asset.intrinsicSize.h);
const maskAssetId = dependencies.createId();
const maskLayerId = dependencies.createId();
store.dispatch(commandIds.documentAddLayerMask, {
layerId,
asset: { id: maskAssetId, name: `${layer.name} AI edit mask`, mimeType: "image/png", source, intrinsicSize: { ...asset.intrinsicSize } },
maskLayer: {
id: maskLayerId,
type: "raster",
name: `${layer.name} AI edit mask`,
visible: true,
locked: false,
opacity: 1,
assetId: maskAssetId,
transform: { position: { ...layer.transform.position }, scale: { ...layer.transform.scale }, rotation: layer.transform.rotation },
},
const regionId = dependencies.createId();
store.dispatch(commandIds.documentAddInpaintRegion, {
region: { id: regionId, name: `${layer.name} AI edit`, targetLayerId: layer.id, maskAssetId, enabled: true },
maskAsset: { id: maskAssetId, name: `${layer.name} AI edit mask`, mimeType: "image/png", source, intrinsicSize: { ...asset.intrinsicSize } },
});
store.dispatch(commandIds.toolEnterInpaintRegionEdit, { targetLayerId: layer.id, regionId });
},
generate: () => job("generate", "Generating", async (signal) => {
generate: () => job("generate", "Generating", async (signal, report) => {
const state = store.getState();
await dependencies.runGenerate({
document: state.document,
@@ -88,15 +86,34 @@ export function createGenerationWorkflow(store: AppStore, dependencies: Generati
settings: state.editor.tools.generate,
dispatch: store.dispatch,
signal,
onProgress: report,
});
}),
regenerate: (candidateId: string, settings?: GenerateSettings, label = "Regenerate") =>
job("regenerate", label, async (signal) => {
job("regenerate", label, async (signal, report) => {
const candidate = findCandidate(store, candidateId);
const nextSettings = settings ?? candidate.settings;
store.dispatch(commandIds.toolSetGenerateSettings, nextSettings);
await dependencies.runGenerateFromCandidate({ candidate, settings: nextSettings, dispatch: store.dispatch, signal });
await dependencies.runGenerateFromCandidate({ candidate, settings: nextSettings, dispatch: store.dispatch, signal, onProgress: report });
}),
rebuildFromCurrentRegion: (candidateId: string) =>
job("regenerate", "Rebuilding from current edit region", async (signal, report) => {
const candidate = findCandidate(store, candidateId);
if (!candidate.inpaint) throw new Error("Only inpaint candidates can rebuild from an edit region.");
store.dispatch(commandIds.selectionSet, { artboardId: candidate.placement.artboardId, layerIds: [candidate.inpaint.targetLayerId] });
store.dispatch(commandIds.toolSetGenerateSettings, candidate.settings);
const state = store.getState();
await dependencies.runGenerate({
document: state.document,
selection: state.editor.selection,
viewport: state.editor.viewport,
settings: candidate.settings,
dispatch: store.dispatch,
signal,
onProgress: report,
});
}),
applyCandidateAsLayer: (candidateId: string) => {