syntaxbullet/image-studio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: implement inpainting functionality with mask handling

Browse Source 
- Added `createMaskedPixelReplacementSource` function to handle pixel replacement using inpainting.
- Introduced `buildInpaintBundle` to prepare inpainting data including mask generation and validation.
- Created utility functions for mask operations such as `applyMaskedContentModeToRgba`, `expandRectWithinBounds`, and others for mask manipulation.
- Developed tests for inpainting preparation and mask raster utilities to ensure functionality and correctness.
- Implemented mask raster operations including inversion, feathering, blurring, and more.
This commit is contained in:
syntaxbullet
2026-07-05 09:35:16 +02:00
parent 6e5d58a638
commit f5c610dac5
35 changed files with 2285 additions and 242 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
view/canvas/brush.ts
View File

@@ -138,7 +138,15 @@ export async function commitBrushSession(options: { store: AppStore; session: Br
if (options.session.cancelled) return;
const source = options.session.changed ? canvasToDataUrl(options.session.canvas) : undefined;
if (source) options.store.dispatch(commandIds.documentUpdateAssetSource, { assetId: options.session.assetId, source });
if (source) {
const state = options.store.getState();
const maskEdit = state.editor.maskEdit;
if (maskEdit?.maskLayerId === options.session.layerId) {
options.store.dispatch(commandIds.documentApplyLayerMaskOperation, { maskLayerId: maskEdit.maskLayerId, source, mimeType: "image/png", operation: { type: "paint" } });
} else {
options.store.dispatch(commandIds.documentUpdateAssetSource, { assetId: options.session.assetId, source });
}
}
options.store.dispatch(commandIds.toolSetBrushStrokePreview, undefined);
closeBrushStrokePreview(options.session);
}

91
view/canvas/magic-wand.ts
View File

@@ -5,6 +5,7 @@ import type { Layer } from "@core/layer";
import { resolveTransformTargetBounds } from "@editor/transform-targets";
import type { AppStore } from "@editor/store";
import type { EditorState } from "@editor/state";
import { blurMaskValues, despeckleMaskValues, dilateMaskValues, erodeMaskValues, maskValueFromRgba } from "../mask/maskRaster";
export async function applyMagicWandAt(store: AppStore, point: Vec2D, modeOverride?: EditorState["tools"]["magicWand"]["mode"]) {
const state = store.getState();
@@ -15,8 +16,8 @@ export async function applyMagicWandAt(store: AppStore, point: Vec2D, modeOverri
const y = Math.floor((point.y - target.layer.transform.position.y) / Math.max(0.0001, target.layer.transform.scale.y));
if (x < 0 || y < 0 || x >= target.asset.intrinsicSize.w || y >= target.asset.intrinsicSize.h) return true;
const source = await createWandMask(target.asset.source, target.maskAsset?.source, Math.round(target.asset.intrinsicSize.w), Math.round(target.asset.intrinsicSize.h), x, y, { ...state.editor.tools.magicWand, mode: modeOverride ?? state.editor.tools.magicWand.mode });
if (target.maskAsset) {
store.dispatch(commandIds.documentUpdateAssetSource, { assetId: target.maskAsset.id, source });
if (target.maskAsset && target.maskLayer && target.maskLayer.type !== "group") {
store.dispatch(commandIds.documentApplyLayerMaskOperation, { maskLayerId: target.maskLayer.id, source, mimeType: "image/png", operation: { type: "magicWand" } });
return true;
}
const assetId = crypto.randomUUID();
@@ -42,7 +43,7 @@ function resolveTarget(document: ImageDocument, editor: EditorState) {
const bounds = resolveTransformTargetBounds(document, { type: "layer", id: layer.id });
const maskLayer = layer.clippingMask ? findLayer(document.artboards.flatMap((artboard) => artboard.layers), layer.clippingMask.maskLayerId) : undefined;
const maskAsset = maskLayer && maskLayer.type !== "group" ? document.assets.find((candidate) => candidate.id === maskLayer.assetId) : undefined;
return asset && bounds ? { layer, asset, bounds, maskAsset } : undefined;
return asset && bounds ? { layer, asset, bounds, maskLayer, maskAsset } : undefined;
}
async function createWandMask(source: string, existingMaskSource: string | undefined, width: number, height: number, startX: number, startY: number, settings: EditorState["tools"]["magicWand"]) {
@@ -57,10 +58,15 @@ async function createWandMask(source: string, existingMaskSource: string | undef
const start = (startY * canvas.width + startX) * 4;
const key = [imageData.data[start] ?? 0, imageData.data[start + 1] ?? 0, imageData.data[start + 2] ?? 0];
const selected = postProcessSelection(settings.contiguous ? floodSelect(imageData, canvas.width, canvas.height, startX, startY, key, settings.tolerance) : globalSelect(imageData, key, settings.tolerance), canvas.width, canvas.height, settings);
const existingAlpha = existingMaskSource ? await loadMaskAlpha(existingMaskSource, canvas.width, canvas.height) : undefined;
const existingMask = existingMaskSource ? await loadMaskValues(existingMaskSource, canvas.width, canvas.height) : undefined;
for (let pixel = 0; pixel < selected.length; pixel++) {
const current = existingAlpha?.[pixel] ?? 255;
const value = settings.mode === "add" ? (selected[pixel] ? 0 : current) : settings.mode === "subtract" ? (selected[pixel] ? 255 : current) : selected[pixel] ? 0 : 255;
const current = existingMask?.[pixel] ?? 255;
const selectionValue = selected[pixel] ?? 0;
const value = settings.mode === "add"
? Math.min(current, 255 - selectionValue)
: settings.mode === "subtract"
? Math.max(current, selectionValue)
: 255 - selectionValue;
const index = pixel * 4;
imageData.data[index] = 255;
imageData.data[index + 1] = 255;
@@ -98,67 +104,24 @@ function matches(data: ImageData, pixel: number, key: number[], tolerance: numbe
}
function postProcessSelection(selected: Uint8Array, width: number, height: number, settings: EditorState["tools"]["magicWand"]) {
let next = selected;
let next = selectionToMaskValues(selected);
const despeckle = Math.round(Math.max(0, Math.min(20, settings.despeckle)));
const choke = Math.round(Math.max(-20, Math.min(20, settings.choke)));
const feather = Math.round(Math.max(0, Math.min(20, settings.feather)));
if (despeckle > 0) next = despeckleSelection(next, width, height, despeckle);
if (choke > 0) next = erodeSelection(next, width, height, choke);
if (choke < 0) next = dilateSelection(next, width, height, -choke);
if (feather > 0) next = featherSelection(next, width, height, feather);
if (despeckle > 0) next = despeckleMaskValues(next, width, height, despeckle);
if (choke > 0) next = erodeMaskValues(next, width, height, choke);
if (choke < 0) next = dilateMaskValues(next, width, height, -choke);
if (feather > 0) next = blurMaskValues(next, width, height, feather);
return next;
}
function erodeSelection(selected: Uint8Array, width: number, height: number, radius: number) {
const next = new Uint8Array(selected.length);
for (let y = 0; y < height; y++) for (let x = 0; x < width; x++) {
let value = 1;
for (let oy = -radius; oy <= radius; oy++) for (let ox = -radius; ox <= radius; ox++) value = Math.min(value, selected[clamp(y + oy, 0, height - 1) * width + clamp(x + ox, 0, width - 1)] ?? 0);
next[y * width + x] = value;
}
return next;
function selectionToMaskValues(selected: Uint8Array) {
const values = new Uint8ClampedArray(selected.length);
for (let index = 0; index < selected.length; index += 1) values[index] = selected[index] ? 255 : 0;
return values;
}
function dilateSelection(selected: Uint8Array, width: number, height: number, radius: number) {
const next = new Uint8Array(selected.length);
for (let y = 0; y < height; y++) for (let x = 0; x < width; x++) {
let value = 0;
for (let oy = -radius; oy <= radius; oy++) for (let ox = -radius; ox <= radius; ox++) value = Math.max(value, selected[clamp(y + oy, 0, height - 1) * width + clamp(x + ox, 0, width - 1)] ?? 0);
next[y * width + x] = value;
}
return next;
}
function featherSelection(selected: Uint8Array, width: number, height: number, radius: number) {
const next = new Uint8Array(selected.length);
for (let y = 0; y < height; y++) for (let x = 0; x < width; x++) {
let total = 0;
let count = 0;
for (let oy = -radius; oy <= radius; oy++) for (let ox = -radius; ox <= radius; ox++) {
total += selected[clamp(y + oy, 0, height - 1) * width + clamp(x + ox, 0, width - 1)] ?? 0;
count += 1;
}
next[y * width + x] = Math.round(total / count);
}
return next;
}
function despeckleSelection(selected: Uint8Array, width: number, height: number, strength: number) {
const radius = Math.max(1, Math.ceil(strength / 6));
const threshold = Math.max(1, Math.round(strength / 2));
const next = new Uint8Array(selected);
for (let y = 0; y < height; y++) for (let x = 0; x < width; x++) {
const index = y * width + x;
let same = 0;
for (let oy = -radius; oy <= radius; oy++) for (let ox = -radius; ox <= radius; ox++) if (ox !== 0 || oy !== 0) {
if ((selected[clamp(y + oy, 0, height - 1) * width + clamp(x + ox, 0, width - 1)] ?? 0) === selected[index]) same += 1;
}
if (same <= threshold) next[index] = selected[index] ? 0 : 1;
}
return next;
}
async function loadMaskAlpha(source: string, width: number, height: number) {
async function loadMaskValues(source: string, width: number, height: number) {
const canvas = document.createElement("canvas");
canvas.width = width;
canvas.height = height;
@@ -167,13 +130,9 @@ async function loadMaskAlpha(source: string, width: number, height: number) {
const image = await loadImage(source);
context.drawImage(image, 0, 0, width, height);
const data = context.getImageData(0, 0, width, height);
const alpha = new Uint8ClampedArray(width * height);
for (let pixel = 0; pixel < alpha.length; pixel++) alpha[pixel] = data.data[pixel * 4 + 3] ?? 255;
return alpha;
}
function clamp(value: number, min: number, max: number) {
return Math.max(min, Math.min(max, value));
const values = new Uint8ClampedArray(width * height);
for (let pixel = 0; pixel < values.length; pixel++) values[pixel] = maskValueFromRgba(data.data, pixel * 4);
return values;
}
function findLayer(layers: readonly Layer[], layerId: string): Layer | undefined {

30
view/canvas/renderFrame.test.ts
View File

@@ -151,6 +151,36 @@ const visualEditorChanges: Array<[string, (state: AppState) => AppState]> = [
},
}),
],
[
"generation candidate preview",
(state) => ({
...state,
editor: {
...state.editor,
generation: {
candidates: [
{
id: "candidate",
source: "data:image/png;base64,candidate",
mimeType: "image/png",
intrinsicSize: { w: 64, h: 64 },
mode: "text-to-image",
settings: state.editor.tools.generate,
seed: 12,
width: 64,
height: 64,
placement: {
artboardId: "artboard",
layerName: "Candidate",
transform: { position: { x: 1, y: 2 }, scale: { x: 1, y: 1 }, rotation: 0 },
},
},
],
selectedCandidateId: "candidate",
},
},
}),
],
[
"active tool",
(state) => ({

7
view/canvas/renderFrame.ts
View File

@@ -1,6 +1,6 @@
import type { Rect, Vec2D } from "@core/geometry";
import type { RenderFrame } from "@renderer/index";
import type { AppState, BrushPreviewState, BrushStrokePreviewState, EditorState, MaskEditState, SelectionState, ViewportState } from "@editor/state";
import type { AppState, BrushPreviewState, BrushStrokePreviewState, EditorState, GenerationState, MaskEditState, SelectionState, ViewportState } from "@editor/state";
import type { BrushSettings, InteractionMode } from "@editor/tools";
import type { TransformSession, TransformTarget } from "@editor/transform";
@@ -23,6 +23,7 @@ function visualEditorStatesEqual(a: EditorState, b: EditorState): boolean {
maskEditStatesEqual(a.maskEdit, b.maskEdit) &&
brushPreviewStatesEqual(a.brushPreview, b.brushPreview) &&
brushStrokePreviewStatesEqual(a.brushStrokePreview, b.brushStrokePreview) &&
generationStatesEqual(a.generation, b.generation) &&
visualToolStatesEqual(a.tools, b.tools)
);
}
@@ -64,6 +65,10 @@ function brushStrokePreviewStatesEqual(a: BrushStrokePreviewState | undefined, b
return a.layerId === b.layerId && a.assetId === b.assetId && a.source === b.source;
}
function generationStatesEqual(a: GenerationState, b: GenerationState): boolean {
return a === b;
}
function visualToolStatesEqual(a: EditorState["tools"], b: EditorState["tools"]): boolean {
return a.activeTool === b.activeTool && interactionModesEqual(a.interactionMode, b.interactionMode) && brushSettingsEqual(a.brush, b.brush);
}