syntaxbullet/image-studio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add inpaint region functionality and related tools

Browse Source 
- Enhanced cursor behavior for new tools: semantic select, mask lasso, and mask rectangle.
- Updated mask edit state to include mask asset ID and kind.
- Implemented inpaint region commands for adding, applying, and removing inpaint regions.
- Introduced new operations for lasso and semantic selection tools.
- Created UI components for candidate review and inpaint region management.
- Added tests for inpaint region commands to ensure functionality.
- Updated various components to support new inpaint features and improve user experience.
This commit is contained in:
syntaxbullet
2026-07-11 16:41:22 +02:00
parent f4e13b80e7
commit ff762b8f17
78 changed files with 1632 additions and 301 deletions
Download Patch File Download Diff File Expand all files Collapse all files

52
operations/masks/lasso.ts Normal file
View File

@@ -0,0 +1,52 @@
import { commandIds } from "@commands/ids";
import type { Vec2D } from "@core/geometry";
import type { Layer } from "@core/layer";
import type { AppStore } from "@editor/store";
import { applyPolygonMask } from "@platform/browser/maskRaster";
export async function commitInpaintLasso(store: AppStore) {
const state = store.getState();
const edit = state.editor.maskEdit;
const session = state.editor.maskShapeSession;
const minimumPoints = session?.shape === "rectangle" ? 2 : 3;
if (edit?.kind !== "inpaintRegion" || !edit.inpaintRegionId || !session || session.points.length < minimumPoints) {
store.dispatch(commandIds.toolClearMaskShape, undefined);
return;
}
const region = state.document.inpaintRegions.find((candidate) => candidate.id === edit.inpaintRegionId);
const target = findLayer(state.document.artboards.flatMap((artboard) => artboard.layers), edit.targetLayerId);
const asset = region ? state.document.assets.find((candidate) => candidate.id === region.maskAssetId) : undefined;
if (!region || !target || (target.type !== "image" && target.type !== "raster") || !asset) {
store.dispatch(commandIds.toolClearMaskShape, undefined);
return;
}
const documentPoints = session.shape === "rectangle" ? rectanglePoints(session.points[0]!, session.points[1]!) : session.points;
const points = documentPoints.map((point) => documentPointToAssetPoint(point, target, asset.intrinsicSize));
const source = await applyPolygonMask(asset.source, asset.intrinsicSize.w, asset.intrinsicSize.h, points, session.mode);
store.dispatch(commandIds.documentApplyInpaintRegionMaskOperation, { regionId: region.id, source, mimeType: "image/png", operation: { type: "paint" } });
store.dispatch(commandIds.toolClearMaskShape, undefined);
}
function rectanglePoints(start: Vec2D, end: Vec2D): Vec2D[] {
return [start, { x: end.x, y: start.y }, end, { x: start.x, y: end.y }];
}
function documentPointToAssetPoint(point: Vec2D, layer: Extract<Layer, { type: "image" | "raster" }>, intrinsicSize: { w: number; h: number }): Vec2D {
const source = layer.sourceRect ?? { x: 0, y: 0, ...intrinsicSize };
const destination = { x: layer.transform.position.x + source.x * layer.transform.scale.x, y: layer.transform.position.y + source.y * layer.transform.scale.y, w: source.w * layer.transform.scale.x, h: source.h * layer.transform.scale.y };
const center = { x: destination.x + destination.w / 2, y: destination.y + destination.h / 2 };
const dx = point.x - center.x;
const dy = point.y - center.y;
const cos = Math.cos(-layer.transform.rotation);
const sin = Math.sin(-layer.transform.rotation);
const x = center.x + dx * cos - dy * sin;
const y = center.y + dx * sin + dy * cos;
return { x: source.x + (x - destination.x) / Math.max(0.0001, layer.transform.scale.x), y: source.y + (y - destination.y) / Math.max(0.0001, layer.transform.scale.y) };
}
function findLayer(layers: readonly Layer[], id: string): Layer | undefined {
for (const layer of layers) {
if (layer.id === id) return layer;
if (layer.type === "group") { const child = findLayer(layer.children, id); if (child) return child; }
}
}

33
operations/masks/magic-wand.ts
View File

@@ -11,12 +11,17 @@ import { createWandMask } from "@platform/browser/magicWandRaster";
export async function applyMagicWandAt(store: AppStore, point: Vec2D, modeOverride?: EditorState["tools"]["magicWand"]["mode"]) {
const state = store.getState();
if (state.editor.tools.activeTool !== "magicWand") return false;
const target = resolveTarget(state.document, state.editor);
const target = resolveMaskSelectionTarget(state.document, state.editor);
if (!target) return true;
const x = Math.floor((point.x - target.layer.transform.position.x) / Math.max(0.0001, target.layer.transform.scale.x));
const y = Math.floor((point.y - target.layer.transform.position.y) / Math.max(0.0001, target.layer.transform.scale.y));
const assetPoint = documentPointToAssetPoint(point, target.layer, target.asset.intrinsicSize);
const x = Math.floor(assetPoint.x);
const y = Math.floor(assetPoint.y);
if (x < 0 || y < 0 || x >= target.asset.intrinsicSize.w || y >= target.asset.intrinsicSize.h) return true;
const source = await createWandMask(target.asset.source, target.maskAsset?.source, Math.round(target.asset.intrinsicSize.w), Math.round(target.asset.intrinsicSize.h), x, y, { ...state.editor.tools.magicWand, mode: modeOverride ?? state.editor.tools.magicWand.mode });
const source = await createWandMask(target.asset.source, target.maskAsset?.source, Math.round(target.asset.intrinsicSize.w), Math.round(target.asset.intrinsicSize.h), x, y, { ...state.editor.tools.magicWand, mode: modeOverride ?? state.editor.tools.magicWand.mode, target: target.inpaintRegion ? "inpaint" : "visibility" });
if (target.inpaintRegion && target.maskAsset) {
store.dispatch(commandIds.documentApplyInpaintRegionMaskOperation, { regionId: target.inpaintRegion.id, source, mimeType: "image/png", operation: { type: "magicWand" } });
return true;
}
if (target.maskAsset && target.maskLayer && (target.maskLayer.type === "image" || target.maskLayer.type === "raster")) {
store.dispatch(commandIds.documentApplyLayerMaskOperation, { maskLayerId: target.maskLayer.id, source, mimeType: "image/png", operation: { type: "magicWand" } });
return true;
@@ -35,7 +40,7 @@ export async function applyMagicWandAt(store: AppStore, point: Vec2D, modeOverri
return true;
}
function resolveTarget(document: ImageDocument, editor: EditorState) {
export function resolveMaskSelectionTarget(document: ImageDocument, editor: EditorState) {
const layerId = editor.selection.layerIds[0];
if (!layerId || editor.selection.layerIds.length !== 1) return undefined;
const layer = findLayer(document.artboards.flatMap((artboard) => artboard.layers), layerId);
@@ -43,9 +48,23 @@ function resolveTarget(document: ImageDocument, editor: EditorState) {
const asset = document.assets.find((candidate) => candidate.id === layer.assetId);
const bounds = resolveTransformTargetBounds(document, { type: "layer", id: layer.id });
const layerMask = getLayerMask(layer);
const inpaintRegion = editor.maskEdit?.kind === "inpaintRegion" ? document.inpaintRegions.find((candidate) => candidate.id === editor.maskEdit?.inpaintRegionId && candidate.targetLayerId === layer.id) : undefined;
const maskLayer = layerMask?.enabled ? findLayer(document.artboards.flatMap((artboard) => artboard.layers), layerMask.maskLayerId) : undefined;
const maskAsset = maskLayer && (maskLayer.type === "image" || maskLayer.type === "raster") ? document.assets.find((candidate) => candidate.id === maskLayer.assetId) : undefined;
return asset && bounds ? { layer, asset, bounds, maskLayer, maskAsset } : undefined;
const maskAsset = inpaintRegion ? document.assets.find((candidate) => candidate.id === inpaintRegion.maskAssetId) : maskLayer && (maskLayer.type === "image" || maskLayer.type === "raster") ? document.assets.find((candidate) => candidate.id === maskLayer.assetId) : undefined;
return asset && bounds ? { layer, asset, bounds, maskLayer, maskAsset, inpaintRegion } : undefined;
}
export function documentPointToAssetPoint(point: Vec2D, layer: Extract<Layer, { type: "image" | "raster" }>, intrinsicSize: { w: number; h: number }): Vec2D {
const source = layer.sourceRect ?? { x: 0, y: 0, ...intrinsicSize };
const destination = { x: layer.transform.position.x + source.x * layer.transform.scale.x, y: layer.transform.position.y + source.y * layer.transform.scale.y, w: source.w * layer.transform.scale.x, h: source.h * layer.transform.scale.y };
const center = { x: destination.x + destination.w / 2, y: destination.y + destination.h / 2 };
const dx = point.x - center.x;
const dy = point.y - center.y;
const cos = Math.cos(-layer.transform.rotation);
const sin = Math.sin(-layer.transform.rotation);
const x = center.x + dx * cos - dy * sin;
const y = center.y + dx * sin + dy * cos;
return { x: source.x + (x - destination.x) / Math.max(0.0001, layer.transform.scale.x), y: source.y + (y - destination.y) / Math.max(0.0001, layer.transform.scale.y) };
}
function findLayer(layers: readonly Layer[], layerId: string): Layer | undefined {

3
operations/masks/rasterActions.ts
View File

@@ -1,10 +1,13 @@
import { commandIds } from "@commands/ids";
import type { Asset } from "@core/asset";
import type { LayerId } from "@core/id";
import type { InpaintRegionId } from "@core/id";
import type { AppStore } from "@editor/store";
import { analyzeMaskSource, applyMaskRasterOperation, createSolidMaskSource, type MaskAnalysis, type MaskRasterOperation } from "@platform/browser/maskRaster";
export type { MaskAnalysis, MaskRasterOperation };
export function analyzeMask(asset: Asset): Promise<MaskAnalysis> { return analyzeMaskSource(asset.source, asset.intrinsicSize.w, asset.intrinsicSize.h); }
export async function runMaskOperation(maskLayerId: LayerId, asset: Asset, operation: MaskRasterOperation, dispatch: AppStore["dispatch"]) { const source = await applyMaskRasterOperation(asset.source, asset.intrinsicSize.w, asset.intrinsicSize.h, operation); dispatch(commandIds.documentApplyLayerMaskOperation, { maskLayerId, source, mimeType: "image/png", operation }); }
export async function runInpaintRegionOperation(regionId: InpaintRegionId, asset: Asset, operation: MaskRasterOperation, dispatch: AppStore["dispatch"]) { const source = await applyMaskRasterOperation(asset.source, asset.intrinsicSize.w, asset.intrinsicSize.h, operation); dispatch(commandIds.documentApplyInpaintRegionMaskOperation, { regionId, source, mimeType: "image/png", operation }); }
export function createRefinementMask(width: number, height: number) { return createSolidMaskSource(width, height, "white"); }
export function createInpaintRegionMask(width: number, height: number) { return createSolidMaskSource(width, height, "black"); }

34
operations/masks/semantic-select.ts Normal file
View File

@@ -0,0 +1,34 @@
import { commandIds } from "@commands/ids";
import type { Vec2D } from "@core/geometry";
import type { AppStore } from "@editor/store";
import { mergeMaskSources } from "@platform/browser/maskRaster";
import { requestSemanticSelection } from "@platform/comfy/generationClient";
import { runGenerationJob } from "@operations/generation/generationJob";
import { documentPointToAssetPoint, resolveMaskSelectionTarget } from "./magic-wand";
export async function applySemanticSelectionAt(store: AppStore, point: Vec2D, mode: "replace" | "add" | "subtract") {
const state = store.getState();
if (state.editor.tools.activeTool !== "semanticSelect") return false;
const target = resolveMaskSelectionTarget(state.document, state.editor);
if (!target?.inpaintRegion || !target.maskAsset) return true;
const region = target.inpaintRegion;
const maskAsset = target.maskAsset;
const assetPoint = documentPointToAssetPoint(point, target.layer, target.asset.intrinsicSize);
if (assetPoint.x < 0 || assetPoint.y < 0 || assetPoint.x >= target.asset.intrinsicSize.w || assetPoint.y >= target.asset.intrinsicSize.h) return true;
const controller = new AbortController();
await runGenerationJob({
kind: "mask",
label: "Selecting object",
dispatch: store.dispatch,
signal: controller.signal,
task: async (signal, report) => {
report(0.1, "Sending point to SAM3");
const result = await requestSemanticSelection({ inputImage: target.asset.source, x: assetPoint.x, y: assetPoint.y }, signal);
report(0.85, "Merging object mask");
const source = await mergeMaskSources(maskAsset.source, result.source, Math.round(target.asset.intrinsicSize.w), Math.round(target.asset.intrinsicSize.h), mode);
store.dispatch(commandIds.documentApplyInpaintRegionMaskOperation, { regionId: region.id, source, mimeType: "image/png", operation: { type: "magicWand" } });
report(1, "Object selected");
},
});
return true;
}