syntaxbullet/image-studio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add inpaint region functionality and related tools

Browse Source 
- Enhanced cursor behavior for new tools: semantic select, mask lasso, and mask rectangle.
- Updated mask edit state to include mask asset ID and kind.
- Implemented inpaint region commands for adding, applying, and removing inpaint regions.
- Introduced new operations for lasso and semantic selection tools.
- Created UI components for candidate review and inpaint region management.
- Added tests for inpaint region commands to ensure functionality.
- Updated various components to support new inpaint features and improve user experience.
This commit is contained in:
syntaxbullet
2026-07-11 16:41:22 +02:00
parent f4e13b80e7
commit ff762b8f17
78 changed files with 1632 additions and 301 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
platform/browser/brushRaster.ts
View File

@@ -15,9 +15,9 @@ export function createBrushSurface(width: number, height: number, source: string
return surface;
}
export function drawBrushSegment(surface: BrushSurface, options: { from: { x: number; y: number }; to: { x: number; y: number }; color: string; size: number; hardness: number; mode: "brush" | "eraser" }) {
export function drawBrushSegment(surface: BrushSurface, options: { from: { x: number; y: number }; to: { x: number; y: number }; color: string; size: number; hardness: number; opacity: number; flow: number; mode: "brush" | "eraser" }) {
const context = internal(surface).context; const hardness = Math.max(0, Math.min(100, options.hardness)) / 100;
context.save(); context.globalCompositeOperation = options.mode === "eraser" ? "destination-out" : "source-over"; context.strokeStyle = options.color; context.shadowColor = options.mode === "eraser" ? "rgba(0,0,0,1)" : options.color; context.shadowBlur = (1 - hardness) * options.size; context.lineWidth = options.size; context.lineCap = "round"; context.lineJoin = "round"; context.beginPath(); context.moveTo(options.from.x, options.from.y); context.lineTo(options.to.x, options.to.y); context.stroke(); context.restore();
context.save(); context.globalAlpha = Math.max(0, Math.min(1, options.opacity / 100)) * Math.max(0.01, Math.min(1, options.flow / 100)); context.globalCompositeOperation = options.mode === "eraser" ? "destination-out" : "source-over"; context.strokeStyle = options.color; context.shadowColor = options.mode === "eraser" ? "rgba(0,0,0,1)" : options.color; context.shadowBlur = (1 - hardness) * options.size; context.lineWidth = options.size; context.lineCap = "round"; context.lineJoin = "round"; context.beginPath(); context.moveTo(options.from.x, options.from.y); context.lineTo(options.to.x, options.to.y); context.stroke(); context.restore();
}
export function brushSurfaceDataUrl(surface: BrushSurface) { try { return internal(surface).canvas.toDataURL("image/png"); } catch { return undefined; } }

15
platform/browser/exportArtboardPng.ts
View File

@@ -7,6 +7,15 @@ import { getLayerMask } from "@core/layer-mask-utils";
import { measureTextLayer } from "@core/text-layer";
export async function downloadArtboardPng(artboard: Artboard, assets: readonly Asset[]) {
const canvas = await renderArtboardCanvas(artboard, assets);
const url = canvas.toDataURL("image/png");
const link = document.createElement("a");
link.href = url;
link.download = `${safeFilename(artboard.name)}.png`;
link.click();
}
export async function renderArtboardCanvas(artboard: Artboard, assets: readonly Asset[]): Promise<HTMLCanvasElement> {
const width = Math.max(1, Math.round(artboard.bounds.w));
const height = Math.max(1, Math.round(artboard.bounds.h));
const canvas = document.createElement("canvas");
@@ -27,11 +36,7 @@ export async function downloadArtboardPng(artboard: Artboard, assets: readonly A
for (const layer of renderStack(artboard.layers)) await drawLayer(context, layer, artboard.layers, assets, artboard.bounds, { maskLayerIds });
context.restore();
const url = canvas.toDataURL("image/png");
const link = document.createElement("a");
link.href = url;
link.download = `${safeFilename(artboard.name)}.png`;
link.click();
return canvas;
}
async function drawLayer(

30
platform/browser/magicWandRaster.ts
View File

@@ -1,6 +1,6 @@
import { blurMaskValues, despeckleMaskValues, dilateMaskValues, erodeMaskValues, maskValueFromRgba } from "./maskRaster";
export type MagicWandRasterSettings = { tolerance: number; feather: number; choke: number; despeckle: number; contiguous: boolean; mode: "replace" | "add" | "subtract" };
export type MagicWandRasterSettings = { tolerance: number; feather: number; choke: number; despeckle: number; contiguous: boolean; mode: "replace" | "add" | "subtract"; target?: "visibility" | "inpaint" };
export async function createWandMask(source: string, existingMaskSource: string | undefined, width: number, height: number, startX: number, startY: number, settings: MagicWandRasterSettings) {
const canvas = document.createElement("canvas");
@@ -10,8 +10,9 @@ export async function createWandMask(source: string, existingMaskSource: string
context.drawImage(await loadImage(source), 0, 0, canvas.width, canvas.height);
const data = context.getImageData(0, 0, canvas.width, canvas.height);
const start = (startY * canvas.width + startX) * 4;
const key = [data.data[start] ?? 0, data.data[start + 1] ?? 0, data.data[start + 2] ?? 0];
const selected = settings.contiguous ? floodSelect(data, width, height, startX, startY, key, settings.tolerance) : globalSelect(data, key, settings.tolerance);
const key = rgbToLab(data.data[start] ?? 0, data.data[start + 1] ?? 0, data.data[start + 2] ?? 0);
const keyAlpha = data.data[start + 3] ?? 255;
const selected = settings.contiguous ? floodSelect(data, width, height, startX, startY, key, keyAlpha, settings.tolerance) : globalSelect(data, key, keyAlpha, settings.tolerance);
let values: Uint8ClampedArray<ArrayBufferLike> = toValues(selected);
if (settings.despeckle > 0) values = despeckleMaskValues(values, width, height, Math.round(settings.despeckle));
if (settings.choke > 0) values = erodeMaskValues(values, width, height, Math.round(settings.choke));
@@ -19,20 +20,31 @@ export async function createWandMask(source: string, existingMaskSource: string
if (settings.feather > 0) values = blurMaskValues(values, width, height, Math.round(settings.feather));
const existing = existingMaskSource ? await loadMask(existingMaskSource, width, height) : undefined;
for (let pixel = 0; pixel < values.length; pixel++) {
const current = existing?.[pixel] ?? 255; const selectedValue = values[pixel] ?? 0;
const alpha = settings.mode === "add" ? Math.min(current, 255 - selectedValue) : settings.mode === "subtract" ? Math.max(current, selectedValue) : 255 - selectedValue;
const current = existing?.[pixel] ?? (settings.target === "inpaint" ? 0 : 255); const selectedValue = values[pixel] ?? 0;
const alpha = settings.target === "inpaint"
? settings.mode === "add" ? Math.max(current, selectedValue) : settings.mode === "subtract" ? Math.min(current, 255 - selectedValue) : selectedValue
: settings.mode === "add" ? Math.min(current, 255 - selectedValue) : settings.mode === "subtract" ? Math.max(current, selectedValue) : 255 - selectedValue;
const index = pixel * 4; data.data[index] = data.data[index + 1] = data.data[index + 2] = 255; data.data[index + 3] = alpha;
}
context.putImageData(data, 0, 0); return canvas.toDataURL("image/png");
}
function floodSelect(data: ImageData, width: number, height: number, x: number, y: number, key: number[], tolerance: number) {
function floodSelect(data: ImageData, width: number, height: number, x: number, y: number, key: readonly number[], keyAlpha: number, tolerance: number) {
const result = new Uint8Array(width * height); const queue: Array<[number, number]> = [[x, y]];
while (queue.length) { const [px, py] = queue.pop()!; if (px < 0 || py < 0 || px >= width || py >= height) continue; const i = py * width + px; if (result[i] || !matches(data, i, key, tolerance)) continue; result[i] = 1; queue.push([px + 1, py], [px - 1, py], [px, py + 1], [px, py - 1]); }
while (queue.length) { const [px, py] = queue.pop()!; if (px < 0 || py < 0 || px >= width || py >= height) continue; const i = py * width + px; if (result[i] || !matches(data, i, key, keyAlpha, tolerance)) continue; result[i] = 1; queue.push([px + 1, py], [px - 1, py], [px, py + 1], [px, py - 1]); }
return result;
}
function globalSelect(data: ImageData, key: number[], tolerance: number) { const result = new Uint8Array(data.width * data.height); for (let i = 0; i < result.length; i++) if (matches(data, i, key, tolerance)) result[i] = 1; return result; }
function matches(data: ImageData, pixel: number, key: number[], tolerance: number) { const i = pixel * 4; return Math.hypot((data.data[i] ?? 0) - (key[0] ?? 0), (data.data[i + 1] ?? 0) - (key[1] ?? 0), (data.data[i + 2] ?? 0) - (key[2] ?? 0)) <= tolerance; }
function globalSelect(data: ImageData, key: readonly number[], keyAlpha: number, tolerance: number) { const result = new Uint8Array(data.width * data.height); for (let i = 0; i < result.length; i++) if (matches(data, i, key, keyAlpha, tolerance)) result[i] = 1; return result; }
function matches(data: ImageData, pixel: number, key: readonly number[], keyAlpha: number, tolerance: number) { const i = pixel * 4; const sample = rgbToLab(data.data[i] ?? 0, data.data[i + 1] ?? 0, data.data[i + 2] ?? 0); const colorDistance = Math.hypot((sample[0] ?? 0) - (key[0] ?? 0), (sample[1] ?? 0) - (key[1] ?? 0), (sample[2] ?? 0) - (key[2] ?? 0)); const alphaDistance = Math.abs((data.data[i + 3] ?? 255) - keyAlpha) / 2.55; return Math.hypot(colorDistance, alphaDistance) <= tolerance * 0.45; }
function rgbToLab(red: number, green: number, blue: number): [number, number, number] {
const linear = [red, green, blue].map((value) => { const channel = value / 255; return channel <= 0.04045 ? channel / 12.92 : ((channel + 0.055) / 1.055) ** 2.4; });
const x = ((linear[0] ?? 0) * 0.4124 + (linear[1] ?? 0) * 0.3576 + (linear[2] ?? 0) * 0.1805) / 0.95047;
const y = ((linear[0] ?? 0) * 0.2126 + (linear[1] ?? 0) * 0.7152 + (linear[2] ?? 0) * 0.0722);
const z = ((linear[0] ?? 0) * 0.0193 + (linear[1] ?? 0) * 0.1192 + (linear[2] ?? 0) * 0.9505) / 1.08883;
const f = (value: number) => value > 0.008856 ? Math.cbrt(value) : 7.787 * value + 16 / 116;
return [116 * f(y) - 16, 500 * (f(x) - f(y)), 200 * (f(y) - f(z))];
}
function toValues(selected: Uint8Array) { const values = new Uint8ClampedArray(selected.length); for (let i = 0; i < selected.length; i++) values[i] = selected[i] ? 255 : 0; return values; }
async function loadMask(source: string, width: number, height: number) { const canvas = document.createElement("canvas"); canvas.width = width; canvas.height = height; const context = canvas.getContext("2d"); if (!context) return undefined; context.drawImage(await loadImage(source), 0, 0, width, height); const data = context.getImageData(0, 0, width, height); const values = new Uint8ClampedArray(width * height); for (let i = 0; i < values.length; i++) values[i] = maskValueFromRgba(data.data, i * 4); return values; }
function loadImage(source: string) { return new Promise<HTMLImageElement>((resolve, reject) => { const image = new Image(); image.onload = () => resolve(image); image.onerror = () => reject(new Error("Failed to load image")); image.src = source; }); }

11
platform/browser/maskRaster.test.ts
View File

@@ -1,5 +1,5 @@
import { describe, expect, test } from "bun:test";
import { blurMaskValues, cropMaskValuesToRgba, dilateMaskValues, erodeMaskValues, expandRectWithinBounds, invertMaskValues } from "./maskRaster";
import { blurMaskValues, cropMaskValuesToRgba, dilateMaskValues, erodeMaskValues, expandRectWithinBounds, invertMaskValues, limitMaskValues } from "./maskRaster";
describe("mask raster utilities", () => {
test("exports the normalized drawn mask without filling the whole crop", () => {
@@ -35,6 +35,15 @@ describe("mask raster utilities", () => {
expect([...erodeMaskValues(new Uint8ClampedArray(new Array(9).fill(255)), 3, 3, 1)]).toEqual(new Array(9).fill(255));
expect([...blurMaskValues(values, 3, 3, 1)]).toEqual(new Array(9).fill(28));
});
test("clips AI edit regions to a retained source crop", () => {
expect([...limitMaskValues(new Uint8ClampedArray(16).fill(255), 4, 4, { x: 1, y: 1, w: 2, h: 2 })]).toEqual([
0, 0, 0, 0,
0, 255, 255, 0,
0, 255, 255, 0,
0, 0, 0, 0,
]);
});
});
function activeRedPixels(rgba: Uint8ClampedArray) {

69
platform/browser/maskRaster.ts
View File

@@ -30,6 +30,7 @@ export type NormalizedMaskOptions = {
feather?: number;
blur?: number;
despeckle?: number;
limit?: Rect;
};
export async function createSolidMaskSource(width: number, height: number, fill: MaskFill): Promise<string> {
@@ -58,6 +59,36 @@ export async function applyMaskRasterOperation(source: string, width: number, he
return maskValuesToDataUrl(next, mask.width, mask.height);
}
export async function applyPolygonMask(source: string, width: number, height: number, points: readonly { x: number; y: number }[], mode: "replace" | "add" | "subtract"): Promise<string> {
assertProcessingRasterSize(width, height, "Mask");
if (points.length < 3) return source;
const canvas = await loadImageCanvas(source, width, height);
const context = require2dContext(canvas);
if (mode === "replace") context.clearRect(0, 0, canvas.width, canvas.height);
context.save();
context.globalCompositeOperation = mode === "subtract" ? "destination-out" : "source-over";
context.fillStyle = "#ffffff";
context.beginPath();
context.moveTo(points[0]?.x ?? 0, points[0]?.y ?? 0);
for (let index = 1; index < points.length; index += 1) context.lineTo(points[index]?.x ?? 0, points[index]?.y ?? 0);
context.closePath();
context.fill();
context.restore();
return canvas.toDataURL("image/png");
}
export async function mergeMaskSources(existingSource: string, selectedSource: string, width: number, height: number, mode: "replace" | "add" | "subtract"): Promise<string> {
assertProcessingRasterSize(width, height, "Mask");
const [existing, selected] = await Promise.all([loadMaskValues(existingSource, width, height), loadMaskValues(selectedSource, width, height)]);
const values = new Uint8ClampedArray(width * height);
for (let pixel = 0; pixel < values.length; pixel += 1) {
const current = existing.values[pixel] ?? 0;
const choice = selected.values[pixel] ?? 0;
values[pixel] = mode === "add" ? Math.max(current, choice) : mode === "subtract" ? Math.min(current, 255 - choice) : choice;
}
return maskValuesToDataUrl(values, width, height);
}
export async function analyzeMaskSource(source: string, width: number, height: number): Promise<MaskAnalysis> {
assertProcessingRasterSize(width, height, "Mask");
const mask = await loadMaskValues(source, width, height);
@@ -81,6 +112,7 @@ export async function createNormalizedMaskSource(source: string, width: number,
assertProcessingRasterSize(width, height, "Mask");
const mask = await loadMaskValues(source, width, height);
let values = options.polarity === "hidden" ? invertMaskValues(mask.values) : new Uint8ClampedArray(mask.values);
if (options.limit) values = limitMaskValues(values, mask.width, mask.height, options.limit);
const despeckle = Math.round(clampNumber(options.despeckle ?? 0, 0, 64));
const expand = Math.round(clampNumber(options.expand ?? 0, -256, 256));
@@ -97,6 +129,16 @@ export async function createNormalizedMaskSource(source: string, width: number,
return { source: maskValuesToDataUrl(values, mask.width, mask.height), values, bounds: analysis.bounds };
}
export function limitMaskValues(values: Uint8ClampedArray, width: number, height: number, limit: Rect): Uint8ClampedArray {
const next = new Uint8ClampedArray(values.length);
const x1 = Math.max(0, Math.floor(limit.x));
const y1 = Math.max(0, Math.floor(limit.y));
const x2 = Math.min(width, Math.ceil(limit.x + limit.w));
const y2 = Math.min(height, Math.ceil(limit.y + limit.h));
for (let y = y1; y < y2; y += 1) for (let x = x1; x < x2; x += 1) next[y * width + x] = values[y * width + x] ?? 0;
return next;
}
export async function loadImageCanvas(source: string, width?: number, height?: number): Promise<HTMLCanvasElement> {
const image = await loadImage(source);
assertCanvasRasterSize(width ?? image.naturalWidth, height ?? image.naturalHeight);
@@ -122,6 +164,33 @@ export function cropCanvas(sourceCanvas: HTMLCanvasElement, crop: Rect, outputWi
return canvas.toDataURL("image/png");
}
export function sampleDocumentCanvasInLayerSpace(
documentCanvas: HTMLCanvasElement,
artboardBounds: Rect,
layer: { transform: { position: { x: number; y: number }; scale: { x: number; y: number }; rotation: number }; sourceRect?: Rect },
intrinsicSize: { w: number; h: number },
crop: Rect,
outputWidth = crop.w,
outputHeight = crop.h,
): string {
assertProcessingRasterSize(outputWidth, outputHeight, "Context crop");
const canvas = createCanvas(outputWidth, outputHeight);
const context = require2dContext(canvas);
const source = layer.sourceRect ?? { x: 0, y: 0, ...intrinsicSize };
const center = {
x: layer.transform.position.x + (source.x + source.w / 2) * layer.transform.scale.x,
y: layer.transform.position.y + (source.y + source.h / 2) * layer.transform.scale.y,
};
context.translate(-crop.x, -crop.y);
context.scale(1 / Math.max(0.0001, layer.transform.scale.x), 1 / Math.max(0.0001, layer.transform.scale.y));
context.translate(-layer.transform.position.x, -layer.transform.position.y);
context.translate(center.x, center.y);
context.rotate(-layer.transform.rotation);
context.translate(-center.x, -center.y);
context.drawImage(documentCanvas, artboardBounds.x, artboardBounds.y);
return canvas.toDataURL("image/png");
}
export function cropMaskValuesToDataUrl(values: Uint8ClampedArray, width: number, height: number, crop: Rect, outputWidth = crop.w, outputHeight = crop.h): string {
assertProcessingRasterSize(outputWidth, outputHeight, "Mask crop");
const canvas = createCanvas(outputWidth, outputHeight);