feat: add inpaint region functionality and related tools
- Enhanced cursor behavior for new tools: semantic select, mask lasso, and mask rectangle. - Updated mask edit state to include mask asset ID and kind. - Implemented inpaint region commands for adding, applying, and removing inpaint regions. - Introduced new operations for lasso and semantic selection tools. - Created UI components for candidate review and inpaint region management. - Added tests for inpaint region commands to ensure functionality. - Updated various components to support new inpaint features and improve user experience.
This commit is contained in:
syntaxbullet
@@ -1,22 +1,28 @@
|
||||
import type { Asset } from "@core/asset";
|
||||
import type { ImageDocument } from "@core/document";
|
||||
import type { Rect } from "@core/geometry";
|
||||
import type { AssetId, InpaintRegionId, LayerId } from "@core/id";
|
||||
import type { Layer } from "@core/layer";
|
||||
import { getLayerMask } from "@core/layer-mask-utils";
|
||||
import type { SelectionState } from "@editor/state";
|
||||
import type { GenerateSettings } from "@editor/tools";
|
||||
import { createDocumentReadIndex, resolveIndexedLayerBounds } from "@editor/document-indexes";
|
||||
import { createNormalizedMaskSource, cropCanvas, cropMaskValuesToDataUrl, expandRectWithinBounds, loadImageCanvas } from "@platform/browser/maskRaster";
|
||||
import { createNormalizedMaskSource, cropCanvas, cropMaskValuesToDataUrl, expandRectWithinBounds, loadImageCanvas, sampleDocumentCanvasInLayerSpace } from "@platform/browser/maskRaster";
|
||||
import { renderArtboardCanvas } from "@platform/browser/exportArtboardPng";
|
||||
|
||||
export type InpaintBundle = {
|
||||
inputImage: string;
|
||||
sourceImage: string;
|
||||
contextImage: string;
|
||||
maskImage: string;
|
||||
editMaskImage: string;
|
||||
blendMaskImage: string;
|
||||
width: number;
|
||||
height: number;
|
||||
targetLayerId: string;
|
||||
maskLayerId: string;
|
||||
sourceAssetId: string;
|
||||
maskAssetId: string;
|
||||
targetLayerId: LayerId;
|
||||
regionId: InpaintRegionId;
|
||||
sourceAssetId: AssetId;
|
||||
maskAssetId: AssetId;
|
||||
revision: { source: string; mask: string };
|
||||
crop: {
|
||||
assetBounds: Rect;
|
||||
documentBounds: Rect;
|
||||
@@ -46,10 +52,8 @@ type InpaintTarget = {
|
||||
artboardId: string;
|
||||
layer: Extract<Layer, { type: "image" | "raster" }>;
|
||||
asset: Asset;
|
||||
bounds: Rect;
|
||||
maskLayer: Extract<Layer, { type: "image" | "raster" }>;
|
||||
maskAsset: Asset;
|
||||
maskBounds: Rect;
|
||||
regionId: InpaintRegionId;
|
||||
};
|
||||
|
||||
const modelMultiple = 8;
|
||||
@@ -62,41 +66,65 @@ export async function buildInpaintBundle(document: ImageDocument, selection: Sel
|
||||
|
||||
const width = Math.max(1, Math.round(target.asset.intrinsicSize.w));
|
||||
const height = Math.max(1, Math.round(target.asset.intrinsicSize.h));
|
||||
const normalizedMask = await createNormalizedMaskSource(target.maskAsset.source, width, height, {
|
||||
polarity: settings.inpaint.maskPolarity,
|
||||
const sourceLimit = target.layer.sourceRect ?? { x: 0, y: 0, w: width, h: height };
|
||||
const editMask = await createNormalizedMaskSource(target.maskAsset.source, width, height, {
|
||||
polarity: "revealed",
|
||||
despeckle: settings.inpaint.maskDespeckle,
|
||||
limit: sourceLimit,
|
||||
});
|
||||
const noiseMask = await createNormalizedMaskSource(target.maskAsset.source, width, height, {
|
||||
polarity: "revealed",
|
||||
expand: settings.inpaint.maskExpand,
|
||||
feather: settings.inpaint.maskFeather,
|
||||
blur: settings.inpaint.maskBlur,
|
||||
despeckle: settings.inpaint.maskDespeckle,
|
||||
limit: sourceLimit,
|
||||
});
|
||||
const blendMask = await createNormalizedMaskSource(target.maskAsset.source, width, height, {
|
||||
polarity: "revealed",
|
||||
feather: settings.inpaint.maskFeather,
|
||||
despeckle: settings.inpaint.maskDespeckle,
|
||||
limit: sourceLimit,
|
||||
});
|
||||
|
||||
if (!normalizedMask.bounds) throw new Error("The selected layer mask has no inpaint pixels.");
|
||||
if (!editMask.bounds || !noiseMask.bounds) throw new Error("Paint over the area you want AI to replace.");
|
||||
|
||||
const crop = settings.inpaint.maskedAreaOnly
|
||||
? expandRectWithinBounds(normalizedMask.bounds, settings.inpaint.cropPadding, { w: width, h: height }, modelMultiple, minModelSize)
|
||||
? expandRectWithinBounds(noiseMask.bounds, settings.inpaint.cropPadding, { w: width, h: height }, modelMultiple, minModelSize)
|
||||
: { x: 0, y: 0, w: width, h: height };
|
||||
const outputWidth = toModelSize(crop.w, "width");
|
||||
const outputHeight = toModelSize(crop.h, "height");
|
||||
|
||||
const inputCanvas = prepareMaskedContentInputCanvas(await loadImageCanvas(target.asset.source, width, height), normalizedMask.values, settings.inpaint.maskedContent);
|
||||
const inputImage = cropCanvas(inputCanvas, crop, outputWidth, outputHeight);
|
||||
const maskImage = cropMaskValuesToDataUrl(normalizedMask.values, width, height, crop, outputWidth, outputHeight);
|
||||
const scaleX = target.bounds.w / width;
|
||||
const scaleY = target.bounds.h / height;
|
||||
const sourceCanvas = await loadImageCanvas(target.asset.source, width, height);
|
||||
const sourceImage = cropCanvas(sourceCanvas, crop, outputWidth, outputHeight);
|
||||
const preparedCanvas = prepareMaskedContentInputCanvas(await loadImageCanvas(target.asset.source, width, height), editMask.values, settings.inpaint.maskedContent);
|
||||
const preparedSourceImage = cropCanvas(preparedCanvas, crop, outputWidth, outputHeight);
|
||||
const artboard = document.artboards.find((candidate) => candidate.id === target.artboardId);
|
||||
if (!artboard) throw new Error("The target artboard no longer exists.");
|
||||
const contextImage = sampleDocumentCanvasInLayerSpace(await renderArtboardCanvas(artboard, document.assets), artboard.bounds, target.layer, target.asset.intrinsicSize, crop, outputWidth, outputHeight);
|
||||
const inputImage = await mergePreparedRegionIntoContext(contextImage, preparedSourceImage, cropMaskValuesToDataUrl(editMask.values, width, height, crop, outputWidth, outputHeight), outputWidth, outputHeight);
|
||||
const maskImage = cropMaskValuesToDataUrl(noiseMask.values, width, height, crop, outputWidth, outputHeight);
|
||||
const editMaskImage = cropMaskValuesToDataUrl(editMask.values, width, height, crop, outputWidth, outputHeight);
|
||||
const blendMaskImage = cropMaskValuesToDataUrl(blendMask.values, width, height, crop, outputWidth, outputHeight);
|
||||
const scaleX = target.layer.transform.scale.x;
|
||||
const scaleY = target.layer.transform.scale.y;
|
||||
const documentBounds = {
|
||||
x: target.bounds.x + crop.x * scaleX,
|
||||
y: target.bounds.y + crop.y * scaleY,
|
||||
x: target.layer.transform.position.x + crop.x * scaleX,
|
||||
y: target.layer.transform.position.y + crop.y * scaleY,
|
||||
w: outputWidth * scaleX,
|
||||
h: outputHeight * scaleY,
|
||||
};
|
||||
|
||||
return {
|
||||
inputImage,
|
||||
sourceImage,
|
||||
contextImage,
|
||||
maskImage,
|
||||
editMaskImage,
|
||||
blendMaskImage,
|
||||
width: outputWidth,
|
||||
height: outputHeight,
|
||||
targetLayerId: target.layer.id,
|
||||
maskLayerId: target.maskLayer.id,
|
||||
regionId: target.regionId,
|
||||
sourceAssetId: target.asset.id,
|
||||
maskAssetId: target.maskAsset.id,
|
||||
crop: {
|
||||
@@ -106,14 +134,14 @@ export async function buildInpaintBundle(document: ImageDocument, selection: Sel
|
||||
maskedAreaOnly: settings.inpaint.maskedAreaOnly,
|
||||
},
|
||||
mask: {
|
||||
polarity: settings.inpaint.maskPolarity,
|
||||
activeBounds: normalizedMask.bounds,
|
||||
polarity: "revealed",
|
||||
activeBounds: editMask.bounds,
|
||||
},
|
||||
placement: {
|
||||
artboardId: target.artboardId,
|
||||
layerName: `${target.layer.name} inpaint`,
|
||||
transform: {
|
||||
position: { x: documentBounds.x, y: documentBounds.y },
|
||||
position: rotatedCropPosition(target.layer, target.asset, { ...crop, w: outputWidth, h: outputHeight }),
|
||||
scale: { x: documentBounds.w / outputWidth, y: documentBounds.h / outputHeight },
|
||||
rotation: target.layer.transform.rotation,
|
||||
},
|
||||
@@ -126,6 +154,10 @@ export async function buildInpaintBundle(document: ImageDocument, selection: Sel
|
||||
maskExpand: settings.inpaint.maskExpand,
|
||||
cropPadding: settings.inpaint.cropPadding,
|
||||
},
|
||||
revision: {
|
||||
source: await createContentRevision(target.asset.source),
|
||||
mask: await createContentRevision(target.maskAsset.source),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -138,20 +170,14 @@ function resolveInpaintTarget(document: ImageDocument, selection: SelectionState
|
||||
|
||||
const asset = documentIndex.assetById.get(layerInfo.layer.assetId);
|
||||
if (!asset) throw new Error("The selected layer is missing its source image.");
|
||||
const layerMask = getLayerMask(layerInfo.layer);
|
||||
if (!layerMask?.enabled) throw new Error("Add a layer mask before running inpaint.");
|
||||
const region = document.inpaintRegions.find((candidate) => candidate.targetLayerId === layerInfo.layer.id && candidate.enabled);
|
||||
if (!region) throw new Error("Add an AI edit region before running inpaint.");
|
||||
const maskAsset = documentIndex.assetById.get(region.maskAssetId);
|
||||
if (!maskAsset) throw new Error("The AI edit region is missing its mask data.");
|
||||
|
||||
const maskLayer = documentIndex.layerById.get(layerMask.maskLayerId);
|
||||
if (!maskLayer || (maskLayer.type !== "image" && maskLayer.type !== "raster")) throw new Error("The selected layer mask is missing.");
|
||||
if (!resolveIndexedLayerBounds(documentIndex, layerInfo.layer)) throw new Error("Unable to resolve the selected layer bounds.");
|
||||
|
||||
const maskAsset = documentIndex.assetById.get(maskLayer.assetId);
|
||||
if (!maskAsset) throw new Error("The selected layer mask is missing its image data.");
|
||||
|
||||
const bounds = resolveIndexedLayerBounds(documentIndex, layerInfo.layer);
|
||||
const maskBounds = resolveIndexedLayerBounds(documentIndex, maskLayer);
|
||||
if (!bounds || !maskBounds) throw new Error("Unable to resolve the selected layer and mask bounds.");
|
||||
|
||||
return { artboardId: layerInfo.artboardId, layer: layerInfo.layer, asset, bounds, maskLayer, maskAsset, maskBounds };
|
||||
return { artboardId: layerInfo.artboardId, layer: layerInfo.layer, asset, maskAsset, regionId: region.id };
|
||||
}
|
||||
|
||||
function validateInpaintTarget(target: InpaintTarget) {
|
||||
@@ -160,13 +186,6 @@ function validateInpaintTarget(target: InpaintTarget) {
|
||||
if (Math.round(target.asset.intrinsicSize.w) !== Math.round(target.maskAsset.intrinsicSize.w) || Math.round(target.asset.intrinsicSize.h) !== Math.round(target.maskAsset.intrinsicSize.h)) {
|
||||
throw new Error("The selected layer and mask image sizes do not match.");
|
||||
}
|
||||
if (!rectsAligned(target.bounds, target.maskBounds) || Math.abs(target.layer.transform.rotation - target.maskLayer.transform.rotation) > 0.001) {
|
||||
throw new Error("The selected layer and mask are not aligned.");
|
||||
}
|
||||
}
|
||||
|
||||
function rectsAligned(a: Rect, b: Rect): boolean {
|
||||
return Math.abs(a.x - b.x) <= 0.5 && Math.abs(a.y - b.y) <= 0.5 && Math.abs(a.w - b.w) <= 0.5 && Math.abs(a.h - b.h) <= 0.5;
|
||||
}
|
||||
|
||||
function toModelSize(value: number, axis: "width" | "height"): number {
|
||||
@@ -258,3 +277,52 @@ function blendChannel(previous: number, next: number, amount: number): number {
|
||||
function clampNumber(value: number, min: number, max: number): number {
|
||||
return Math.max(min, Math.min(max, value));
|
||||
}
|
||||
|
||||
async function mergePreparedRegionIntoContext(contextSource: string, preparedSource: string, editMaskSource: string, width: number, height: number): Promise<string> {
|
||||
const [contextCanvas, preparedCanvas, maskCanvas] = await Promise.all([
|
||||
loadImageCanvas(contextSource, width, height),
|
||||
loadImageCanvas(preparedSource, width, height),
|
||||
loadImageCanvas(editMaskSource, width, height),
|
||||
]);
|
||||
const context = contextCanvas.getContext("2d");
|
||||
const prepared = preparedCanvas.getContext("2d");
|
||||
const mask = maskCanvas.getContext("2d");
|
||||
if (!context || !prepared || !mask) return contextSource;
|
||||
const contextData = context.getImageData(0, 0, width, height);
|
||||
const preparedData = prepared.getImageData(0, 0, width, height);
|
||||
const maskData = mask.getImageData(0, 0, width, height);
|
||||
for (let pixel = 0; pixel < width * height; pixel += 1) {
|
||||
const amount = (maskData.data[pixel * 4] ?? 0) / 255;
|
||||
if (amount <= 0) continue;
|
||||
for (let channel = 0; channel < 4; channel += 1) {
|
||||
const index = pixel * 4 + channel;
|
||||
contextData.data[index] = blendChannel(contextData.data[index] ?? 0, preparedData.data[index] ?? 0, amount);
|
||||
}
|
||||
}
|
||||
context.putImageData(contextData, 0, 0);
|
||||
return contextCanvas.toDataURL("image/png");
|
||||
}
|
||||
|
||||
function rotatedCropPosition(layer: InpaintTarget["layer"], asset: Asset, crop: Rect) {
|
||||
const source = layer.sourceRect ?? { x: 0, y: 0, ...asset.intrinsicSize };
|
||||
const scale = layer.transform.scale;
|
||||
const originalCenter = {
|
||||
x: layer.transform.position.x + (source.x + source.w / 2) * scale.x,
|
||||
y: layer.transform.position.y + (source.y + source.h / 2) * scale.y,
|
||||
};
|
||||
const cropCenter = {
|
||||
x: layer.transform.position.x + (crop.x + crop.w / 2) * scale.x,
|
||||
y: layer.transform.position.y + (crop.y + crop.h / 2) * scale.y,
|
||||
};
|
||||
const dx = cropCenter.x - originalCenter.x;
|
||||
const dy = cropCenter.y - originalCenter.y;
|
||||
const cos = Math.cos(layer.transform.rotation);
|
||||
const sin = Math.sin(layer.transform.rotation);
|
||||
const rotatedCenter = { x: originalCenter.x + dx * cos - dy * sin, y: originalCenter.y + dx * sin + dy * cos };
|
||||
return { x: rotatedCenter.x - crop.w * scale.x / 2, y: rotatedCenter.y - crop.h * scale.y / 2 };
|
||||
}
|
||||
|
||||
export async function createContentRevision(source: string): Promise<string> {
|
||||
const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(source));
|
||||
return Array.from(new Uint8Array(digest), (byte) => byte.toString(16).padStart(2, "0")).join("");
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user