feat: implement inpainting functionality with mask handling
- Added `createMaskedPixelReplacementSource` function to handle pixel replacement using inpainting. - Introduced `buildInpaintBundle` to prepare inpainting data including mask generation and validation. - Created utility functions for mask operations such as `applyMaskedContentModeToRgba`, `expandRectWithinBounds`, and others for mask manipulation. - Developed tests for inpainting preparation and mask raster utilities to ensure functionality and correctness. - Implemented mask raster operations including inversion, feathering, blurring, and more.
This commit is contained in:
syntaxbullet
50
view/generate/candidateActions.ts
Normal file
50
view/generate/candidateActions.ts
Normal file
@@ -0,0 +1,50 @@
|
||||
import type { ImageDocument } from "@core/document";
|
||||
import type { GenerationCandidate } from "@editor/state";
|
||||
import { loadImageCanvas, maskValueFromRgba } from "../mask/maskRaster";
|
||||
|
||||
export async function createMaskedPixelReplacementSource(document: ImageDocument, candidate: GenerationCandidate): Promise<string> {
|
||||
if (!candidate.inpaint) throw new Error("Only inpaint candidates can replace masked pixels.");
|
||||
|
||||
const targetAsset = document.assets.find((asset) => asset.id === candidate.inpaint?.sourceAssetId);
|
||||
if (!targetAsset) throw new Error("The source layer for this candidate no longer exists.");
|
||||
|
||||
const targetCanvas = await loadImageCanvas(targetAsset.source, targetAsset.intrinsicSize.w, targetAsset.intrinsicSize.h);
|
||||
const generatedCanvas = await loadImageCanvas(candidate.source, candidate.width, candidate.height);
|
||||
const maskCanvas = await loadImageCanvas(candidate.inpaint.maskImage, candidate.width, candidate.height);
|
||||
|
||||
const targetContext = require2dContext(targetCanvas);
|
||||
const generatedContext = require2dContext(generatedCanvas);
|
||||
const maskContext = require2dContext(maskCanvas);
|
||||
const targetData = targetContext.getImageData(0, 0, targetCanvas.width, targetCanvas.height);
|
||||
const generatedData = generatedContext.getImageData(0, 0, generatedCanvas.width, generatedCanvas.height);
|
||||
const maskData = maskContext.getImageData(0, 0, maskCanvas.width, maskCanvas.height);
|
||||
const crop = candidate.inpaint.crop.assetBounds;
|
||||
|
||||
for (let y = 0; y < candidate.height; y += 1) {
|
||||
for (let x = 0; x < candidate.width; x += 1) {
|
||||
const targetX = Math.round(crop.x) + x;
|
||||
const targetY = Math.round(crop.y) + y;
|
||||
if (targetX < 0 || targetY < 0 || targetX >= targetCanvas.width || targetY >= targetCanvas.height) continue;
|
||||
|
||||
const generatedIndex = (y * generatedCanvas.width + x) * 4;
|
||||
const targetIndex = (targetY * targetCanvas.width + targetX) * 4;
|
||||
const mask = maskValueFromRgba(maskData.data, generatedIndex) / 255;
|
||||
if (mask <= 0) continue;
|
||||
|
||||
for (let channel = 0; channel < 4; channel += 1) {
|
||||
const previous = targetData.data[targetIndex + channel] ?? 0;
|
||||
const next = generatedData.data[generatedIndex + channel] ?? previous;
|
||||
targetData.data[targetIndex + channel] = Math.round(previous * (1 - mask) + next * mask);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
targetContext.putImageData(targetData, 0, 0);
|
||||
return targetCanvas.toDataURL("image/png");
|
||||
}
|
||||
|
||||
function require2dContext(canvas: HTMLCanvasElement): CanvasRenderingContext2D {
|
||||
const context = canvas.getContext("2d");
|
||||
if (!context) throw new Error("Unable to prepare generated candidate");
|
||||
return context;
|
||||
}
|
||||
36
view/generate/inpaintPrep.test.ts
Normal file
36
view/generate/inpaintPrep.test.ts
Normal file
@@ -0,0 +1,36 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import { applyMaskedContentModeToRgba } from "./inpaintPrep";
|
||||
|
||||
describe("inpaint prep", () => {
|
||||
test("converts only masked original pixels to grayscale", () => {
|
||||
const pixels = new Uint8ClampedArray([
|
||||
255, 0, 0, 255,
|
||||
0, 0, 255, 255,
|
||||
]);
|
||||
const mask = new Uint8ClampedArray([255, 0]);
|
||||
|
||||
const next = applyMaskedContentModeToRgba(pixels, 2, 1, mask, "original");
|
||||
|
||||
expect(next[0]).toBe(next[1]);
|
||||
expect(next[1]).toBe(next[2]);
|
||||
expect(Array.from(next.slice(4, 8))).toEqual([0, 0, 255, 255]);
|
||||
});
|
||||
|
||||
test("uses an edge map inside the mask without recoloring unmasked context", () => {
|
||||
const pixels = new Uint8ClampedArray([
|
||||
255, 0, 0, 255,
|
||||
0, 255, 0, 255,
|
||||
0, 0, 255, 255,
|
||||
255, 255, 0, 255,
|
||||
]);
|
||||
const mask = new Uint8ClampedArray([255, 0, 0, 0]);
|
||||
|
||||
const next = applyMaskedContentModeToRgba(pixels, 2, 2, mask, "edges");
|
||||
|
||||
expect(next[0]).toBe(next[1]);
|
||||
expect(next[1]).toBe(next[2]);
|
||||
expect(Array.from(next.slice(4, 8))).toEqual([0, 255, 0, 255]);
|
||||
expect(Array.from(next.slice(8, 12))).toEqual([0, 0, 255, 255]);
|
||||
expect(Array.from(next.slice(12, 16))).toEqual([255, 255, 0, 255]);
|
||||
});
|
||||
});
|
||||
258
view/generate/inpaintPrep.ts
Normal file
258
view/generate/inpaintPrep.ts
Normal file
@@ -0,0 +1,258 @@
|
||||
import type { Asset } from "@core/asset";
|
||||
import type { ImageDocument } from "@core/document";
|
||||
import type { Rect } from "@core/geometry";
|
||||
import type { Layer } from "@core/layer";
|
||||
import type { SelectionState } from "@editor/state";
|
||||
import type { GenerateSettings } from "@editor/tools";
|
||||
import { createDocumentReadIndex, resolveIndexedLayerBounds } from "@editor/document-indexes";
|
||||
import { createNormalizedMaskSource, cropCanvas, cropMaskValuesToDataUrl, expandRectWithinBounds, loadImageCanvas } from "../mask/maskRaster";
|
||||
|
||||
export type InpaintBundle = {
|
||||
inputImage: string;
|
||||
maskImage: string;
|
||||
width: number;
|
||||
height: number;
|
||||
targetLayerId: string;
|
||||
maskLayerId: string;
|
||||
sourceAssetId: string;
|
||||
maskAssetId: string;
|
||||
crop: {
|
||||
assetBounds: Rect;
|
||||
documentBounds: Rect;
|
||||
padding: number;
|
||||
maskedAreaOnly: boolean;
|
||||
};
|
||||
mask: {
|
||||
polarity: GenerateSettings["inpaint"]["maskPolarity"];
|
||||
activeBounds: Rect;
|
||||
};
|
||||
placement: {
|
||||
artboardId: string;
|
||||
layerName: string;
|
||||
transform: Extract<Layer, { type: "image" | "raster" }>["transform"];
|
||||
};
|
||||
backend: {
|
||||
growMaskBy: number;
|
||||
maskedContent: GenerateSettings["inpaint"]["maskedContent"];
|
||||
maskBlur: number;
|
||||
maskFeather: number;
|
||||
maskExpand: number;
|
||||
cropPadding: number;
|
||||
};
|
||||
};
|
||||
|
||||
type InpaintTarget = {
|
||||
artboardId: string;
|
||||
layer: Extract<Layer, { type: "image" | "raster" }>;
|
||||
asset: Asset;
|
||||
bounds: Rect;
|
||||
maskLayer: Extract<Layer, { type: "image" | "raster" }>;
|
||||
maskAsset: Asset;
|
||||
maskBounds: Rect;
|
||||
};
|
||||
|
||||
const modelMultiple = 8;
|
||||
const minModelSize = 64;
|
||||
const maxModelSize = 4096;
|
||||
|
||||
export async function buildInpaintBundle(document: ImageDocument, selection: SelectionState, settings: GenerateSettings): Promise<InpaintBundle> {
|
||||
const target = resolveInpaintTarget(document, selection);
|
||||
validateInpaintTarget(target);
|
||||
|
||||
const width = Math.max(1, Math.round(target.asset.intrinsicSize.w));
|
||||
const height = Math.max(1, Math.round(target.asset.intrinsicSize.h));
|
||||
const normalizedMask = await createNormalizedMaskSource(target.maskAsset.source, width, height, {
|
||||
polarity: settings.inpaint.maskPolarity,
|
||||
expand: settings.inpaint.maskExpand,
|
||||
feather: settings.inpaint.maskFeather,
|
||||
blur: settings.inpaint.maskBlur,
|
||||
despeckle: settings.inpaint.maskDespeckle,
|
||||
});
|
||||
|
||||
if (!normalizedMask.bounds) throw new Error("The selected layer mask has no inpaint pixels.");
|
||||
|
||||
const crop = settings.inpaint.maskedAreaOnly
|
||||
? expandRectWithinBounds(normalizedMask.bounds, settings.inpaint.cropPadding, { w: width, h: height }, modelMultiple, minModelSize)
|
||||
: { x: 0, y: 0, w: width, h: height };
|
||||
const outputWidth = toModelSize(crop.w, "width");
|
||||
const outputHeight = toModelSize(crop.h, "height");
|
||||
|
||||
const inputCanvas = prepareMaskedContentInputCanvas(await loadImageCanvas(target.asset.source, width, height), normalizedMask.values, settings.inpaint.maskedContent);
|
||||
const inputImage = cropCanvas(inputCanvas, crop, outputWidth, outputHeight);
|
||||
const maskImage = cropMaskValuesToDataUrl(normalizedMask.values, width, height, crop, outputWidth, outputHeight);
|
||||
const scaleX = target.bounds.w / width;
|
||||
const scaleY = target.bounds.h / height;
|
||||
const documentBounds = {
|
||||
x: target.bounds.x + crop.x * scaleX,
|
||||
y: target.bounds.y + crop.y * scaleY,
|
||||
w: outputWidth * scaleX,
|
||||
h: outputHeight * scaleY,
|
||||
};
|
||||
|
||||
return {
|
||||
inputImage,
|
||||
maskImage,
|
||||
width: outputWidth,
|
||||
height: outputHeight,
|
||||
targetLayerId: target.layer.id,
|
||||
maskLayerId: target.maskLayer.id,
|
||||
sourceAssetId: target.asset.id,
|
||||
maskAssetId: target.maskAsset.id,
|
||||
crop: {
|
||||
assetBounds: crop,
|
||||
documentBounds,
|
||||
padding: settings.inpaint.cropPadding,
|
||||
maskedAreaOnly: settings.inpaint.maskedAreaOnly,
|
||||
},
|
||||
mask: {
|
||||
polarity: settings.inpaint.maskPolarity,
|
||||
activeBounds: normalizedMask.bounds,
|
||||
},
|
||||
placement: {
|
||||
artboardId: target.artboardId,
|
||||
layerName: `${target.layer.name} inpaint`,
|
||||
transform: {
|
||||
position: { x: documentBounds.x, y: documentBounds.y },
|
||||
scale: { x: documentBounds.w / outputWidth, y: documentBounds.h / outputHeight },
|
||||
rotation: target.layer.transform.rotation,
|
||||
},
|
||||
},
|
||||
backend: {
|
||||
growMaskBy: settings.inpaint.growMaskBy,
|
||||
maskedContent: settings.inpaint.maskedContent,
|
||||
maskBlur: settings.inpaint.maskBlur,
|
||||
maskFeather: settings.inpaint.maskFeather,
|
||||
maskExpand: settings.inpaint.maskExpand,
|
||||
cropPadding: settings.inpaint.cropPadding,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function resolveInpaintTarget(document: ImageDocument, selection: SelectionState): InpaintTarget {
|
||||
if (selection.layerIds.length !== 1 || !selection.layerIds[0]) throw new Error("Select one image or raster layer to inpaint.");
|
||||
|
||||
const documentIndex = createDocumentReadIndex(document);
|
||||
const layerInfo = documentIndex.layerInfoById.get(selection.layerIds[0]);
|
||||
if (!layerInfo || layerInfo.layer.type === "group") throw new Error("Select one image or raster layer to inpaint.");
|
||||
|
||||
const asset = documentIndex.assetById.get(layerInfo.layer.assetId);
|
||||
if (!asset) throw new Error("The selected layer is missing its source image.");
|
||||
if (!layerInfo.layer.clippingMask) throw new Error("Add a layer mask before running inpaint.");
|
||||
|
||||
const maskLayer = documentIndex.layerById.get(layerInfo.layer.clippingMask.maskLayerId);
|
||||
if (!maskLayer || maskLayer.type === "group") throw new Error("The selected layer mask is missing.");
|
||||
|
||||
const maskAsset = documentIndex.assetById.get(maskLayer.assetId);
|
||||
if (!maskAsset) throw new Error("The selected layer mask is missing its image data.");
|
||||
|
||||
const bounds = resolveIndexedLayerBounds(documentIndex, layerInfo.layer);
|
||||
const maskBounds = resolveIndexedLayerBounds(documentIndex, maskLayer);
|
||||
if (!bounds || !maskBounds) throw new Error("Unable to resolve the selected layer and mask bounds.");
|
||||
|
||||
return { artboardId: layerInfo.artboardId, layer: layerInfo.layer, asset, bounds, maskLayer, maskAsset, maskBounds };
|
||||
}
|
||||
|
||||
function validateInpaintTarget(target: InpaintTarget) {
|
||||
if (target.asset.intrinsicSize.w <= 0 || target.asset.intrinsicSize.h <= 0) throw new Error("The selected image has an invalid size.");
|
||||
if (target.maskAsset.intrinsicSize.w <= 0 || target.maskAsset.intrinsicSize.h <= 0) throw new Error("The selected mask has an invalid size.");
|
||||
if (Math.round(target.asset.intrinsicSize.w) !== Math.round(target.maskAsset.intrinsicSize.w) || Math.round(target.asset.intrinsicSize.h) !== Math.round(target.maskAsset.intrinsicSize.h)) {
|
||||
throw new Error("The selected layer and mask image sizes do not match.");
|
||||
}
|
||||
if (!rectsAligned(target.bounds, target.maskBounds) || Math.abs(target.layer.transform.rotation - target.maskLayer.transform.rotation) > 0.001) {
|
||||
throw new Error("The selected layer and mask are not aligned.");
|
||||
}
|
||||
}
|
||||
|
||||
function rectsAligned(a: Rect, b: Rect): boolean {
|
||||
return Math.abs(a.x - b.x) <= 0.5 && Math.abs(a.y - b.y) <= 0.5 && Math.abs(a.w - b.w) <= 0.5 && Math.abs(a.h - b.h) <= 0.5;
|
||||
}
|
||||
|
||||
function toModelSize(value: number, axis: "width" | "height"): number {
|
||||
const rounded = Math.max(minModelSize, Math.ceil(value / modelMultiple) * modelMultiple);
|
||||
if (rounded > maxModelSize) throw new Error(`The inpaint ${axis} is too large for the model. Use masked-area inpaint or a smaller source.`);
|
||||
return rounded;
|
||||
}
|
||||
|
||||
function prepareMaskedContentInputCanvas(sourceCanvas: HTMLCanvasElement, maskValues: Uint8ClampedArray, mode: GenerateSettings["inpaint"]["maskedContent"]): HTMLCanvasElement {
|
||||
if (mode === "neutral" || mode === "originalColor") return sourceCanvas;
|
||||
const context = sourceCanvas.getContext("2d");
|
||||
if (!context) return sourceCanvas;
|
||||
const imageData = context.getImageData(0, 0, sourceCanvas.width, sourceCanvas.height);
|
||||
imageData.data.set(applyMaskedContentModeToRgba(imageData.data, sourceCanvas.width, sourceCanvas.height, maskValues, mode));
|
||||
context.putImageData(imageData, 0, 0);
|
||||
return sourceCanvas;
|
||||
}
|
||||
|
||||
export function applyMaskedContentModeToRgba(data: Uint8ClampedArray, width: number, height: number, maskValues: Uint8ClampedArray, mode: GenerateSettings["inpaint"]["maskedContent"]): Uint8ClampedArray {
|
||||
const next = new Uint8ClampedArray(data);
|
||||
if (mode === "neutral" || mode === "originalColor") return next;
|
||||
|
||||
const edgeValues = mode === "edges" ? createEdgeMapValues(data, width, height) : undefined;
|
||||
|
||||
for (let pixel = 0; pixel < width * height; pixel += 1) {
|
||||
const amount = (maskValues[pixel] ?? 0) / 255;
|
||||
if (amount <= 0) continue;
|
||||
|
||||
const index = pixel * 4;
|
||||
const red = data[index] ?? 0;
|
||||
const green = data[index + 1] ?? 0;
|
||||
const blue = data[index + 2] ?? 0;
|
||||
const target = edgeValues ? edgeValues[pixel] ?? 128 : luminance(red, green, blue);
|
||||
next[index] = blendChannel(red, target, amount);
|
||||
next[index + 1] = blendChannel(green, target, amount);
|
||||
next[index + 2] = blendChannel(blue, target, amount);
|
||||
}
|
||||
|
||||
return next;
|
||||
}
|
||||
|
||||
function createEdgeMapValues(data: Uint8ClampedArray, width: number, height: number): Uint8ClampedArray {
|
||||
const gray = new Float32Array(width * height);
|
||||
const edges = new Uint8ClampedArray(width * height);
|
||||
|
||||
for (let pixel = 0; pixel < width * height; pixel += 1) {
|
||||
const index = pixel * 4;
|
||||
gray[pixel] = luminance(data[index] ?? 0, data[index + 1] ?? 0, data[index + 2] ?? 0);
|
||||
}
|
||||
|
||||
for (let y = 0; y < height; y += 1) {
|
||||
for (let x = 0; x < width; x += 1) {
|
||||
const gx =
|
||||
-sampleGray(gray, width, height, x - 1, y - 1) +
|
||||
sampleGray(gray, width, height, x + 1, y - 1) -
|
||||
2 * sampleGray(gray, width, height, x - 1, y) +
|
||||
2 * sampleGray(gray, width, height, x + 1, y) -
|
||||
sampleGray(gray, width, height, x - 1, y + 1) +
|
||||
sampleGray(gray, width, height, x + 1, y + 1);
|
||||
const gy =
|
||||
-sampleGray(gray, width, height, x - 1, y - 1) -
|
||||
2 * sampleGray(gray, width, height, x, y - 1) -
|
||||
sampleGray(gray, width, height, x + 1, y - 1) +
|
||||
sampleGray(gray, width, height, x - 1, y + 1) +
|
||||
2 * sampleGray(gray, width, height, x, y + 1) +
|
||||
sampleGray(gray, width, height, x + 1, y + 1);
|
||||
const magnitude = Math.hypot(gx, gy);
|
||||
edges[y * width + x] = Math.round(clampNumber(128 + Math.max(0, magnitude - 24) * 0.75, 128, 255));
|
||||
}
|
||||
}
|
||||
|
||||
return edges;
|
||||
}
|
||||
|
||||
function sampleGray(values: Float32Array, width: number, height: number, x: number, y: number): number {
|
||||
const clampedX = Math.max(0, Math.min(width - 1, x));
|
||||
const clampedY = Math.max(0, Math.min(height - 1, y));
|
||||
return values[clampedY * width + clampedX] ?? 0;
|
||||
}
|
||||
|
||||
function luminance(red: number, green: number, blue: number): number {
|
||||
return Math.round(0.2126 * red + 0.7152 * green + 0.0722 * blue);
|
||||
}
|
||||
|
||||
function blendChannel(previous: number, next: number, amount: number): number {
|
||||
return Math.round(previous * (1 - amount) + next * amount);
|
||||
}
|
||||
|
||||
function clampNumber(value: number, min: number, max: number): number {
|
||||
return Math.max(min, Math.min(max, value));
|
||||
}
|
||||
@@ -1,9 +1,11 @@
|
||||
import { commandIds } from "@commands/ids";
|
||||
import type { ImageDocument } from "@core/document";
|
||||
import type { Transform } from "@core/geometry";
|
||||
import type { Layer } from "@core/layer";
|
||||
import type { AppStore } from "@editor/store";
|
||||
import type { SelectionState, ViewportState } from "@editor/state";
|
||||
import type { GenerationCandidate, SelectionState, ViewportState } from "@editor/state";
|
||||
import type { GenerateSettings } from "@editor/tools";
|
||||
import { buildInpaintBundle, type InpaintBundle } from "./inpaintPrep";
|
||||
|
||||
export async function runGenerate(options: {
|
||||
document: ImageDocument;
|
||||
@@ -17,50 +19,200 @@ export async function runGenerate(options: {
|
||||
if (!artboard) return;
|
||||
|
||||
const target = resolveSelectedImage(document, selection);
|
||||
const inputImage = target && settings.mode !== "text-to-image" ? await imageSourceToDataUrl(target.asset.source) : undefined;
|
||||
const maskImage = target?.maskAsset && settings.mode === "inpaint" ? await imageSourceToDataUrl(target.maskAsset.source) : undefined;
|
||||
const inpaintBundle = settings.mode === "inpaint" ? await buildInpaintBundle(document, selection, settings) : undefined;
|
||||
const inputImage = inpaintBundle?.inputImage ?? (target && settings.mode !== "text-to-image" ? await imageSourceToDataUrl(target.asset.source) : undefined);
|
||||
const maskImage = inpaintBundle?.maskImage;
|
||||
const seed = resolveSeed(settings.seed);
|
||||
const requestSettings = { ...settings, seed };
|
||||
const width = inpaintBundle?.width ?? settings.width;
|
||||
const height = inpaintBundle?.height ?? settings.height;
|
||||
const generated = await requestGenerate({
|
||||
settings: requestSettings,
|
||||
width,
|
||||
height,
|
||||
inputImage,
|
||||
maskImage,
|
||||
inpaintBundle,
|
||||
});
|
||||
const intrinsicSize = await loadImageSize(generated.source);
|
||||
const targetArtboardId = inpaintBundle?.placement.artboardId ?? artboard.id;
|
||||
const placement = {
|
||||
artboardId: targetArtboardId,
|
||||
layerName: inpaintBundle?.placement.layerName ?? "Generated image",
|
||||
transform: generatedLayerTransform(inpaintBundle, intrinsicSize),
|
||||
};
|
||||
|
||||
dispatch(commandIds.generationAddCandidate, {
|
||||
candidate: createGenerationCandidate({
|
||||
source: generated.source,
|
||||
mimeType: generated.mimeType,
|
||||
intrinsicSize,
|
||||
settings: requestSettings,
|
||||
seed,
|
||||
width,
|
||||
height,
|
||||
inputImage,
|
||||
maskImage,
|
||||
placement,
|
||||
inpaintBundle,
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
export async function runGenerateFromCandidate(options: {
|
||||
candidate: GenerationCandidate;
|
||||
settings?: GenerateSettings;
|
||||
dispatch: AppStore["dispatch"];
|
||||
}) {
|
||||
const settings = options.settings ?? options.candidate.settings;
|
||||
const seed = resolveSeed(settings.seed);
|
||||
const requestSettings = { ...settings, seed };
|
||||
const generated = await requestGenerate({
|
||||
settings: requestSettings,
|
||||
width: options.candidate.width,
|
||||
height: options.candidate.height,
|
||||
inputImage: options.candidate.inputImage,
|
||||
maskImage: options.candidate.maskImage,
|
||||
inpaintCandidate: options.candidate,
|
||||
});
|
||||
const intrinsicSize = await loadImageSize(generated.source);
|
||||
|
||||
options.dispatch(commandIds.generationAddCandidate, {
|
||||
candidate: {
|
||||
...options.candidate,
|
||||
id: crypto.randomUUID(),
|
||||
source: generated.source,
|
||||
mimeType: generated.mimeType,
|
||||
intrinsicSize,
|
||||
settings: requestSettings,
|
||||
seed,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function createGenerationCandidate(options: {
|
||||
source: string;
|
||||
mimeType: string;
|
||||
intrinsicSize: { w: number; h: number };
|
||||
settings: GenerateSettings;
|
||||
seed: number;
|
||||
width: number;
|
||||
height: number;
|
||||
inputImage?: string;
|
||||
maskImage?: string;
|
||||
placement: GenerationCandidate["placement"];
|
||||
inpaintBundle?: InpaintBundle;
|
||||
}): GenerationCandidate {
|
||||
return {
|
||||
id: crypto.randomUUID(),
|
||||
source: options.source,
|
||||
mimeType: options.mimeType,
|
||||
intrinsicSize: options.intrinsicSize,
|
||||
mode: options.settings.mode,
|
||||
settings: options.settings,
|
||||
seed: options.seed,
|
||||
width: options.width,
|
||||
height: options.height,
|
||||
inputImage: options.inputImage,
|
||||
maskImage: options.maskImage,
|
||||
placement: options.placement,
|
||||
inpaint: options.inpaintBundle
|
||||
? {
|
||||
targetLayerId: options.inpaintBundle.targetLayerId,
|
||||
maskLayerId: options.inpaintBundle.maskLayerId,
|
||||
sourceAssetId: options.inpaintBundle.sourceAssetId,
|
||||
maskAssetId: options.inpaintBundle.maskAssetId,
|
||||
inputImage: options.inpaintBundle.inputImage,
|
||||
maskImage: options.inpaintBundle.maskImage,
|
||||
crop: options.inpaintBundle.crop,
|
||||
mask: options.inpaintBundle.mask,
|
||||
backend: options.inpaintBundle.backend,
|
||||
}
|
||||
: undefined,
|
||||
};
|
||||
}
|
||||
|
||||
async function requestGenerate(options: {
|
||||
settings: GenerateSettings;
|
||||
width: number;
|
||||
height: number;
|
||||
inputImage?: string;
|
||||
maskImage?: string;
|
||||
inpaintBundle?: InpaintBundle;
|
||||
inpaintCandidate?: GenerationCandidate;
|
||||
}) {
|
||||
const response = await fetch("/api/comfy/generate", {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
mode: settings.mode,
|
||||
model: settings.model,
|
||||
prompt: settings.prompt,
|
||||
negativePrompt: settings.negativePrompt,
|
||||
strength: settings.strength,
|
||||
steps: settings.steps,
|
||||
cfg: settings.cfg,
|
||||
seed: settings.seed,
|
||||
sampler: settings.sampler,
|
||||
scheduler: settings.scheduler,
|
||||
width: settings.width,
|
||||
height: settings.height,
|
||||
outpaint: settings.outpaint,
|
||||
inputImage,
|
||||
maskImage,
|
||||
mode: options.settings.mode,
|
||||
model: options.settings.model,
|
||||
prompt: options.settings.prompt,
|
||||
negativePrompt: options.settings.negativePrompt,
|
||||
strength: options.settings.strength,
|
||||
steps: options.settings.steps,
|
||||
cfg: options.settings.cfg,
|
||||
seed: options.settings.seed,
|
||||
sampler: options.settings.sampler,
|
||||
scheduler: options.settings.scheduler,
|
||||
width: options.width,
|
||||
height: options.height,
|
||||
outpaint: options.settings.outpaint,
|
||||
inpaint: resolveInpaintRequest(options.inpaintBundle, options.inpaintCandidate, options.settings),
|
||||
inputImage: options.inputImage,
|
||||
maskImage: options.maskImage,
|
||||
}),
|
||||
});
|
||||
if (!response.ok) throw new Error(await response.text());
|
||||
const generated = await response.json() as { source: string; mimeType: string };
|
||||
const intrinsicSize = await loadImageSize(generated.source);
|
||||
const assetId = crypto.randomUUID();
|
||||
const layerId = crypto.randomUUID();
|
||||
dispatch(commandIds.documentAddAsset, { asset: { id: assetId, name: "Generated image", mimeType: generated.mimeType, source: generated.source, intrinsicSize } });
|
||||
dispatch(commandIds.documentAddImageLayer, {
|
||||
artboardId: artboard.id,
|
||||
layer: {
|
||||
id: layerId,
|
||||
type: "image",
|
||||
name: "Generated image",
|
||||
visible: true,
|
||||
locked: false,
|
||||
opacity: 1,
|
||||
assetId,
|
||||
transform: { position: { x: 0, y: 0 }, scale: { x: 1, y: 1 }, rotation: 0 },
|
||||
return await response.json() as { source: string; mimeType: string };
|
||||
}
|
||||
|
||||
function resolveInpaintRequest(inpaintBundle: InpaintBundle | undefined, inpaintCandidate: GenerationCandidate | undefined, settings: GenerateSettings) {
|
||||
if (inpaintBundle) {
|
||||
return {
|
||||
growMaskBy: inpaintBundle.backend.growMaskBy,
|
||||
maskedContent: inpaintBundle.backend.maskedContent,
|
||||
maskBlur: inpaintBundle.backend.maskBlur,
|
||||
maskFeather: inpaintBundle.backend.maskFeather,
|
||||
maskExpand: inpaintBundle.backend.maskExpand,
|
||||
cropPadding: inpaintBundle.backend.cropPadding,
|
||||
maskPolarity: inpaintBundle.mask.polarity,
|
||||
crop: inpaintBundle.crop,
|
||||
placement: inpaintBundle.placement,
|
||||
};
|
||||
}
|
||||
|
||||
if (inpaintCandidate?.inpaint) {
|
||||
return {
|
||||
growMaskBy: inpaintCandidate.inpaint.backend.growMaskBy,
|
||||
maskedContent: inpaintCandidate.inpaint.backend.maskedContent,
|
||||
maskBlur: inpaintCandidate.inpaint.backend.maskBlur,
|
||||
maskFeather: inpaintCandidate.inpaint.backend.maskFeather,
|
||||
maskExpand: inpaintCandidate.inpaint.backend.maskExpand,
|
||||
cropPadding: inpaintCandidate.inpaint.backend.cropPadding,
|
||||
maskPolarity: inpaintCandidate.inpaint.mask.polarity,
|
||||
crop: inpaintCandidate.inpaint.crop,
|
||||
placement: inpaintCandidate.placement,
|
||||
};
|
||||
}
|
||||
|
||||
return settings.inpaint;
|
||||
}
|
||||
|
||||
function generatedLayerTransform(inpaintBundle: InpaintBundle | undefined, intrinsicSize: { w: number; h: number }): Transform {
|
||||
if (!inpaintBundle) return { position: { x: 0, y: 0 }, scale: { x: 1, y: 1 }, rotation: 0 };
|
||||
return {
|
||||
position: { ...inpaintBundle.placement.transform.position },
|
||||
scale: {
|
||||
x: (inpaintBundle.placement.transform.scale.x * inpaintBundle.width) / Math.max(1, intrinsicSize.w),
|
||||
y: (inpaintBundle.placement.transform.scale.y * inpaintBundle.height) / Math.max(1, intrinsicSize.h),
|
||||
},
|
||||
});
|
||||
dispatch(commandIds.documentMoveLayer, { layerId, toArtboardId: artboard.id, toIndex: 0 });
|
||||
dispatch(commandIds.selectionSet, { artboardId: artboard.id, layerIds: [layerId] });
|
||||
rotation: inpaintBundle.placement.transform.rotation,
|
||||
};
|
||||
}
|
||||
|
||||
function resolveSeed(seed: number): number {
|
||||
return seed < 0 ? Math.floor(Math.random() * 2 ** 32) : Math.round(seed);
|
||||
}
|
||||
|
||||
function resolveSelectedImage(document: ImageDocument, selection: SelectionState) {
|
||||
|
||||
Reference in New Issue
Block a user