feat: add inpaint region functionality and related tools
- Enhanced cursor behavior for new tools: semantic select, mask lasso, and mask rectangle. - Updated mask edit state to include mask asset ID and kind. - Implemented inpaint region commands for adding, applying, and removing inpaint regions. - Introduced new operations for lasso and semantic selection tools. - Created UI components for candidate review and inpaint region management. - Added tests for inpaint region commands to ensure functionality. - Updated various components to support new inpaint features and improve user experience.
This commit is contained in:
syntaxbullet
@@ -15,9 +15,9 @@ export function createBrushSurface(width: number, height: number, source: string
|
||||
return surface;
|
||||
}
|
||||
|
||||
export function drawBrushSegment(surface: BrushSurface, options: { from: { x: number; y: number }; to: { x: number; y: number }; color: string; size: number; hardness: number; mode: "brush" | "eraser" }) {
|
||||
export function drawBrushSegment(surface: BrushSurface, options: { from: { x: number; y: number }; to: { x: number; y: number }; color: string; size: number; hardness: number; opacity: number; flow: number; mode: "brush" | "eraser" }) {
|
||||
const context = internal(surface).context; const hardness = Math.max(0, Math.min(100, options.hardness)) / 100;
|
||||
context.save(); context.globalCompositeOperation = options.mode === "eraser" ? "destination-out" : "source-over"; context.strokeStyle = options.color; context.shadowColor = options.mode === "eraser" ? "rgba(0,0,0,1)" : options.color; context.shadowBlur = (1 - hardness) * options.size; context.lineWidth = options.size; context.lineCap = "round"; context.lineJoin = "round"; context.beginPath(); context.moveTo(options.from.x, options.from.y); context.lineTo(options.to.x, options.to.y); context.stroke(); context.restore();
|
||||
context.save(); context.globalAlpha = Math.max(0, Math.min(1, options.opacity / 100)) * Math.max(0.01, Math.min(1, options.flow / 100)); context.globalCompositeOperation = options.mode === "eraser" ? "destination-out" : "source-over"; context.strokeStyle = options.color; context.shadowColor = options.mode === "eraser" ? "rgba(0,0,0,1)" : options.color; context.shadowBlur = (1 - hardness) * options.size; context.lineWidth = options.size; context.lineCap = "round"; context.lineJoin = "round"; context.beginPath(); context.moveTo(options.from.x, options.from.y); context.lineTo(options.to.x, options.to.y); context.stroke(); context.restore();
|
||||
}
|
||||
|
||||
export function brushSurfaceDataUrl(surface: BrushSurface) { try { return internal(surface).canvas.toDataURL("image/png"); } catch { return undefined; } }
|
||||
|
||||
@@ -7,6 +7,15 @@ import { getLayerMask } from "@core/layer-mask-utils";
|
||||
import { measureTextLayer } from "@core/text-layer";
|
||||
|
||||
export async function downloadArtboardPng(artboard: Artboard, assets: readonly Asset[]) {
|
||||
const canvas = await renderArtboardCanvas(artboard, assets);
|
||||
const url = canvas.toDataURL("image/png");
|
||||
const link = document.createElement("a");
|
||||
link.href = url;
|
||||
link.download = `${safeFilename(artboard.name)}.png`;
|
||||
link.click();
|
||||
}
|
||||
|
||||
export async function renderArtboardCanvas(artboard: Artboard, assets: readonly Asset[]): Promise<HTMLCanvasElement> {
|
||||
const width = Math.max(1, Math.round(artboard.bounds.w));
|
||||
const height = Math.max(1, Math.round(artboard.bounds.h));
|
||||
const canvas = document.createElement("canvas");
|
||||
@@ -27,11 +36,7 @@ export async function downloadArtboardPng(artboard: Artboard, assets: readonly A
|
||||
for (const layer of renderStack(artboard.layers)) await drawLayer(context, layer, artboard.layers, assets, artboard.bounds, { maskLayerIds });
|
||||
context.restore();
|
||||
|
||||
const url = canvas.toDataURL("image/png");
|
||||
const link = document.createElement("a");
|
||||
link.href = url;
|
||||
link.download = `${safeFilename(artboard.name)}.png`;
|
||||
link.click();
|
||||
return canvas;
|
||||
}
|
||||
|
||||
async function drawLayer(
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { blurMaskValues, despeckleMaskValues, dilateMaskValues, erodeMaskValues, maskValueFromRgba } from "./maskRaster";
|
||||
|
||||
export type MagicWandRasterSettings = { tolerance: number; feather: number; choke: number; despeckle: number; contiguous: boolean; mode: "replace" | "add" | "subtract" };
|
||||
export type MagicWandRasterSettings = { tolerance: number; feather: number; choke: number; despeckle: number; contiguous: boolean; mode: "replace" | "add" | "subtract"; target?: "visibility" | "inpaint" };
|
||||
|
||||
export async function createWandMask(source: string, existingMaskSource: string | undefined, width: number, height: number, startX: number, startY: number, settings: MagicWandRasterSettings) {
|
||||
const canvas = document.createElement("canvas");
|
||||
@@ -10,8 +10,9 @@ export async function createWandMask(source: string, existingMaskSource: string
|
||||
context.drawImage(await loadImage(source), 0, 0, canvas.width, canvas.height);
|
||||
const data = context.getImageData(0, 0, canvas.width, canvas.height);
|
||||
const start = (startY * canvas.width + startX) * 4;
|
||||
const key = [data.data[start] ?? 0, data.data[start + 1] ?? 0, data.data[start + 2] ?? 0];
|
||||
const selected = settings.contiguous ? floodSelect(data, width, height, startX, startY, key, settings.tolerance) : globalSelect(data, key, settings.tolerance);
|
||||
const key = rgbToLab(data.data[start] ?? 0, data.data[start + 1] ?? 0, data.data[start + 2] ?? 0);
|
||||
const keyAlpha = data.data[start + 3] ?? 255;
|
||||
const selected = settings.contiguous ? floodSelect(data, width, height, startX, startY, key, keyAlpha, settings.tolerance) : globalSelect(data, key, keyAlpha, settings.tolerance);
|
||||
let values: Uint8ClampedArray<ArrayBufferLike> = toValues(selected);
|
||||
if (settings.despeckle > 0) values = despeckleMaskValues(values, width, height, Math.round(settings.despeckle));
|
||||
if (settings.choke > 0) values = erodeMaskValues(values, width, height, Math.round(settings.choke));
|
||||
@@ -19,20 +20,31 @@ export async function createWandMask(source: string, existingMaskSource: string
|
||||
if (settings.feather > 0) values = blurMaskValues(values, width, height, Math.round(settings.feather));
|
||||
const existing = existingMaskSource ? await loadMask(existingMaskSource, width, height) : undefined;
|
||||
for (let pixel = 0; pixel < values.length; pixel++) {
|
||||
const current = existing?.[pixel] ?? 255; const selectedValue = values[pixel] ?? 0;
|
||||
const alpha = settings.mode === "add" ? Math.min(current, 255 - selectedValue) : settings.mode === "subtract" ? Math.max(current, selectedValue) : 255 - selectedValue;
|
||||
const current = existing?.[pixel] ?? (settings.target === "inpaint" ? 0 : 255); const selectedValue = values[pixel] ?? 0;
|
||||
const alpha = settings.target === "inpaint"
|
||||
? settings.mode === "add" ? Math.max(current, selectedValue) : settings.mode === "subtract" ? Math.min(current, 255 - selectedValue) : selectedValue
|
||||
: settings.mode === "add" ? Math.min(current, 255 - selectedValue) : settings.mode === "subtract" ? Math.max(current, selectedValue) : 255 - selectedValue;
|
||||
const index = pixel * 4; data.data[index] = data.data[index + 1] = data.data[index + 2] = 255; data.data[index + 3] = alpha;
|
||||
}
|
||||
context.putImageData(data, 0, 0); return canvas.toDataURL("image/png");
|
||||
}
|
||||
|
||||
function floodSelect(data: ImageData, width: number, height: number, x: number, y: number, key: number[], tolerance: number) {
|
||||
function floodSelect(data: ImageData, width: number, height: number, x: number, y: number, key: readonly number[], keyAlpha: number, tolerance: number) {
|
||||
const result = new Uint8Array(width * height); const queue: Array<[number, number]> = [[x, y]];
|
||||
while (queue.length) { const [px, py] = queue.pop()!; if (px < 0 || py < 0 || px >= width || py >= height) continue; const i = py * width + px; if (result[i] || !matches(data, i, key, tolerance)) continue; result[i] = 1; queue.push([px + 1, py], [px - 1, py], [px, py + 1], [px, py - 1]); }
|
||||
while (queue.length) { const [px, py] = queue.pop()!; if (px < 0 || py < 0 || px >= width || py >= height) continue; const i = py * width + px; if (result[i] || !matches(data, i, key, keyAlpha, tolerance)) continue; result[i] = 1; queue.push([px + 1, py], [px - 1, py], [px, py + 1], [px, py - 1]); }
|
||||
return result;
|
||||
}
|
||||
function globalSelect(data: ImageData, key: number[], tolerance: number) { const result = new Uint8Array(data.width * data.height); for (let i = 0; i < result.length; i++) if (matches(data, i, key, tolerance)) result[i] = 1; return result; }
|
||||
function matches(data: ImageData, pixel: number, key: number[], tolerance: number) { const i = pixel * 4; return Math.hypot((data.data[i] ?? 0) - (key[0] ?? 0), (data.data[i + 1] ?? 0) - (key[1] ?? 0), (data.data[i + 2] ?? 0) - (key[2] ?? 0)) <= tolerance; }
|
||||
function globalSelect(data: ImageData, key: readonly number[], keyAlpha: number, tolerance: number) { const result = new Uint8Array(data.width * data.height); for (let i = 0; i < result.length; i++) if (matches(data, i, key, keyAlpha, tolerance)) result[i] = 1; return result; }
|
||||
function matches(data: ImageData, pixel: number, key: readonly number[], keyAlpha: number, tolerance: number) { const i = pixel * 4; const sample = rgbToLab(data.data[i] ?? 0, data.data[i + 1] ?? 0, data.data[i + 2] ?? 0); const colorDistance = Math.hypot((sample[0] ?? 0) - (key[0] ?? 0), (sample[1] ?? 0) - (key[1] ?? 0), (sample[2] ?? 0) - (key[2] ?? 0)); const alphaDistance = Math.abs((data.data[i + 3] ?? 255) - keyAlpha) / 2.55; return Math.hypot(colorDistance, alphaDistance) <= tolerance * 0.45; }
|
||||
|
||||
function rgbToLab(red: number, green: number, blue: number): [number, number, number] {
|
||||
const linear = [red, green, blue].map((value) => { const channel = value / 255; return channel <= 0.04045 ? channel / 12.92 : ((channel + 0.055) / 1.055) ** 2.4; });
|
||||
const x = ((linear[0] ?? 0) * 0.4124 + (linear[1] ?? 0) * 0.3576 + (linear[2] ?? 0) * 0.1805) / 0.95047;
|
||||
const y = ((linear[0] ?? 0) * 0.2126 + (linear[1] ?? 0) * 0.7152 + (linear[2] ?? 0) * 0.0722);
|
||||
const z = ((linear[0] ?? 0) * 0.0193 + (linear[1] ?? 0) * 0.1192 + (linear[2] ?? 0) * 0.9505) / 1.08883;
|
||||
const f = (value: number) => value > 0.008856 ? Math.cbrt(value) : 7.787 * value + 16 / 116;
|
||||
return [116 * f(y) - 16, 500 * (f(x) - f(y)), 200 * (f(y) - f(z))];
|
||||
}
|
||||
function toValues(selected: Uint8Array) { const values = new Uint8ClampedArray(selected.length); for (let i = 0; i < selected.length; i++) values[i] = selected[i] ? 255 : 0; return values; }
|
||||
async function loadMask(source: string, width: number, height: number) { const canvas = document.createElement("canvas"); canvas.width = width; canvas.height = height; const context = canvas.getContext("2d"); if (!context) return undefined; context.drawImage(await loadImage(source), 0, 0, width, height); const data = context.getImageData(0, 0, width, height); const values = new Uint8ClampedArray(width * height); for (let i = 0; i < values.length; i++) values[i] = maskValueFromRgba(data.data, i * 4); return values; }
|
||||
function loadImage(source: string) { return new Promise<HTMLImageElement>((resolve, reject) => { const image = new Image(); image.onload = () => resolve(image); image.onerror = () => reject(new Error("Failed to load image")); image.src = source; }); }
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import { blurMaskValues, cropMaskValuesToRgba, dilateMaskValues, erodeMaskValues, expandRectWithinBounds, invertMaskValues } from "./maskRaster";
|
||||
import { blurMaskValues, cropMaskValuesToRgba, dilateMaskValues, erodeMaskValues, expandRectWithinBounds, invertMaskValues, limitMaskValues } from "./maskRaster";
|
||||
|
||||
describe("mask raster utilities", () => {
|
||||
test("exports the normalized drawn mask without filling the whole crop", () => {
|
||||
@@ -35,6 +35,15 @@ describe("mask raster utilities", () => {
|
||||
expect([...erodeMaskValues(new Uint8ClampedArray(new Array(9).fill(255)), 3, 3, 1)]).toEqual(new Array(9).fill(255));
|
||||
expect([...blurMaskValues(values, 3, 3, 1)]).toEqual(new Array(9).fill(28));
|
||||
});
|
||||
|
||||
test("clips AI edit regions to a retained source crop", () => {
|
||||
expect([...limitMaskValues(new Uint8ClampedArray(16).fill(255), 4, 4, { x: 1, y: 1, w: 2, h: 2 })]).toEqual([
|
||||
0, 0, 0, 0,
|
||||
0, 255, 255, 0,
|
||||
0, 255, 255, 0,
|
||||
0, 0, 0, 0,
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
function activeRedPixels(rgba: Uint8ClampedArray) {
|
||||
|
||||
@@ -30,6 +30,7 @@ export type NormalizedMaskOptions = {
|
||||
feather?: number;
|
||||
blur?: number;
|
||||
despeckle?: number;
|
||||
limit?: Rect;
|
||||
};
|
||||
|
||||
export async function createSolidMaskSource(width: number, height: number, fill: MaskFill): Promise<string> {
|
||||
@@ -58,6 +59,36 @@ export async function applyMaskRasterOperation(source: string, width: number, he
|
||||
return maskValuesToDataUrl(next, mask.width, mask.height);
|
||||
}
|
||||
|
||||
export async function applyPolygonMask(source: string, width: number, height: number, points: readonly { x: number; y: number }[], mode: "replace" | "add" | "subtract"): Promise<string> {
|
||||
assertProcessingRasterSize(width, height, "Mask");
|
||||
if (points.length < 3) return source;
|
||||
const canvas = await loadImageCanvas(source, width, height);
|
||||
const context = require2dContext(canvas);
|
||||
if (mode === "replace") context.clearRect(0, 0, canvas.width, canvas.height);
|
||||
context.save();
|
||||
context.globalCompositeOperation = mode === "subtract" ? "destination-out" : "source-over";
|
||||
context.fillStyle = "#ffffff";
|
||||
context.beginPath();
|
||||
context.moveTo(points[0]?.x ?? 0, points[0]?.y ?? 0);
|
||||
for (let index = 1; index < points.length; index += 1) context.lineTo(points[index]?.x ?? 0, points[index]?.y ?? 0);
|
||||
context.closePath();
|
||||
context.fill();
|
||||
context.restore();
|
||||
return canvas.toDataURL("image/png");
|
||||
}
|
||||
|
||||
export async function mergeMaskSources(existingSource: string, selectedSource: string, width: number, height: number, mode: "replace" | "add" | "subtract"): Promise<string> {
|
||||
assertProcessingRasterSize(width, height, "Mask");
|
||||
const [existing, selected] = await Promise.all([loadMaskValues(existingSource, width, height), loadMaskValues(selectedSource, width, height)]);
|
||||
const values = new Uint8ClampedArray(width * height);
|
||||
for (let pixel = 0; pixel < values.length; pixel += 1) {
|
||||
const current = existing.values[pixel] ?? 0;
|
||||
const choice = selected.values[pixel] ?? 0;
|
||||
values[pixel] = mode === "add" ? Math.max(current, choice) : mode === "subtract" ? Math.min(current, 255 - choice) : choice;
|
||||
}
|
||||
return maskValuesToDataUrl(values, width, height);
|
||||
}
|
||||
|
||||
export async function analyzeMaskSource(source: string, width: number, height: number): Promise<MaskAnalysis> {
|
||||
assertProcessingRasterSize(width, height, "Mask");
|
||||
const mask = await loadMaskValues(source, width, height);
|
||||
@@ -81,6 +112,7 @@ export async function createNormalizedMaskSource(source: string, width: number,
|
||||
assertProcessingRasterSize(width, height, "Mask");
|
||||
const mask = await loadMaskValues(source, width, height);
|
||||
let values = options.polarity === "hidden" ? invertMaskValues(mask.values) : new Uint8ClampedArray(mask.values);
|
||||
if (options.limit) values = limitMaskValues(values, mask.width, mask.height, options.limit);
|
||||
|
||||
const despeckle = Math.round(clampNumber(options.despeckle ?? 0, 0, 64));
|
||||
const expand = Math.round(clampNumber(options.expand ?? 0, -256, 256));
|
||||
@@ -97,6 +129,16 @@ export async function createNormalizedMaskSource(source: string, width: number,
|
||||
return { source: maskValuesToDataUrl(values, mask.width, mask.height), values, bounds: analysis.bounds };
|
||||
}
|
||||
|
||||
export function limitMaskValues(values: Uint8ClampedArray, width: number, height: number, limit: Rect): Uint8ClampedArray {
|
||||
const next = new Uint8ClampedArray(values.length);
|
||||
const x1 = Math.max(0, Math.floor(limit.x));
|
||||
const y1 = Math.max(0, Math.floor(limit.y));
|
||||
const x2 = Math.min(width, Math.ceil(limit.x + limit.w));
|
||||
const y2 = Math.min(height, Math.ceil(limit.y + limit.h));
|
||||
for (let y = y1; y < y2; y += 1) for (let x = x1; x < x2; x += 1) next[y * width + x] = values[y * width + x] ?? 0;
|
||||
return next;
|
||||
}
|
||||
|
||||
export async function loadImageCanvas(source: string, width?: number, height?: number): Promise<HTMLCanvasElement> {
|
||||
const image = await loadImage(source);
|
||||
assertCanvasRasterSize(width ?? image.naturalWidth, height ?? image.naturalHeight);
|
||||
@@ -122,6 +164,33 @@ export function cropCanvas(sourceCanvas: HTMLCanvasElement, crop: Rect, outputWi
|
||||
return canvas.toDataURL("image/png");
|
||||
}
|
||||
|
||||
export function sampleDocumentCanvasInLayerSpace(
|
||||
documentCanvas: HTMLCanvasElement,
|
||||
artboardBounds: Rect,
|
||||
layer: { transform: { position: { x: number; y: number }; scale: { x: number; y: number }; rotation: number }; sourceRect?: Rect },
|
||||
intrinsicSize: { w: number; h: number },
|
||||
crop: Rect,
|
||||
outputWidth = crop.w,
|
||||
outputHeight = crop.h,
|
||||
): string {
|
||||
assertProcessingRasterSize(outputWidth, outputHeight, "Context crop");
|
||||
const canvas = createCanvas(outputWidth, outputHeight);
|
||||
const context = require2dContext(canvas);
|
||||
const source = layer.sourceRect ?? { x: 0, y: 0, ...intrinsicSize };
|
||||
const center = {
|
||||
x: layer.transform.position.x + (source.x + source.w / 2) * layer.transform.scale.x,
|
||||
y: layer.transform.position.y + (source.y + source.h / 2) * layer.transform.scale.y,
|
||||
};
|
||||
context.translate(-crop.x, -crop.y);
|
||||
context.scale(1 / Math.max(0.0001, layer.transform.scale.x), 1 / Math.max(0.0001, layer.transform.scale.y));
|
||||
context.translate(-layer.transform.position.x, -layer.transform.position.y);
|
||||
context.translate(center.x, center.y);
|
||||
context.rotate(-layer.transform.rotation);
|
||||
context.translate(-center.x, -center.y);
|
||||
context.drawImage(documentCanvas, artboardBounds.x, artboardBounds.y);
|
||||
return canvas.toDataURL("image/png");
|
||||
}
|
||||
|
||||
export function cropMaskValuesToDataUrl(values: Uint8ClampedArray, width: number, height: number, crop: Rect, outputWidth = crop.w, outputHeight = crop.h): string {
|
||||
assertProcessingRasterSize(outputWidth, outputHeight, "Mask crop");
|
||||
const canvas = createCanvas(outputWidth, outputHeight);
|
||||
|
||||
@@ -4,8 +4,40 @@ export async function fetchGenerationOptions(): Promise<unknown> {
|
||||
return response.json() as Promise<unknown>;
|
||||
}
|
||||
|
||||
export async function requestGeneration(body: unknown, signal?: AbortSignal): Promise<{ source: string; mimeType: string }> {
|
||||
const response = await fetch("/api/comfy/generate", { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(body), signal });
|
||||
export type GenerationResult = { source: string; mimeType: string; seed: number };
|
||||
|
||||
export async function requestSemanticSelection(body: { inputImage: string; x: number; y: number; model?: string }, signal?: AbortSignal): Promise<{ source: string; mimeType: string }> {
|
||||
const response = await fetch("/api/comfy/segment", { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(body), signal });
|
||||
if (!response.ok) throw new Error(await response.text());
|
||||
return response.json() as Promise<{ source: string; mimeType: string }>;
|
||||
}
|
||||
|
||||
export async function requestGeneration(body: unknown, signal?: AbortSignal, onProgress?: (progress: number, detail: string) => void): Promise<{ results: GenerationResult[] }> {
|
||||
const response = await fetch("/api/comfy/generate", { method: "POST", headers: { "content-type": "application/json", accept: "application/x-ndjson" }, body: JSON.stringify(body), signal });
|
||||
if (!response.ok) throw new Error(await response.text());
|
||||
const contentType = response.headers.get("content-type") ?? "";
|
||||
if (contentType.includes("application/x-ndjson") && response.body) {
|
||||
const reader = response.body.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
let pending = "";
|
||||
while (true) {
|
||||
const { value, done } = await reader.read();
|
||||
pending += decoder.decode(value, { stream: !done });
|
||||
const lines = pending.split("\n");
|
||||
pending = lines.pop() ?? "";
|
||||
for (const line of lines) {
|
||||
if (!line.trim()) continue;
|
||||
const event = JSON.parse(line) as { type: "progress"; progress: number; detail: string } | { type: "result"; results: GenerationResult[] } | { type: "error"; message: string };
|
||||
if (event.type === "progress") onProgress?.(event.progress, event.detail);
|
||||
if (event.type === "error") throw new Error(event.message);
|
||||
if (event.type === "result") return { results: event.results };
|
||||
}
|
||||
if (done) break;
|
||||
}
|
||||
throw new Error("Generation stream ended without results");
|
||||
}
|
||||
const payload = await response.json() as { results?: GenerationResult[]; source?: string; mimeType?: string };
|
||||
if (payload.results?.length) return { results: payload.results };
|
||||
if (payload.source) return { results: [{ source: payload.source, mimeType: payload.mimeType ?? "image/png", seed: 0 }] };
|
||||
throw new Error("Generation returned no results");
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user