feat: implement inpainting functionality with mask handling

- Added `createMaskedPixelReplacementSource` function to handle pixel replacement using inpainting.
- Introduced `buildInpaintBundle` to prepare inpainting data including mask generation and validation.
- Created utility functions for mask operations such as `applyMaskedContentModeToRgba`, `expandRectWithinBounds`, and others for mask manipulation.
- Developed tests for inpainting preparation and mask raster utilities to ensure functionality and correctness.
- Implemented mask raster operations including inversion, feathering, blurring, and more.
This commit is contained in:
syntaxbullet
2026-07-05 09:35:16 +02:00
parent 6e5d58a638
commit f5c610dac5
35 changed files with 2285 additions and 242 deletions

View File

@@ -1,4 +1,5 @@
import { documentCommands } from "@commands/document";
import { generationCommands } from "@commands/generation";
import { historyCommands } from "@commands/history";
import { commandIds } from "@commands/ids";
import { createCommandRegistry } from "@commands/registry";
@@ -12,7 +13,7 @@ import { createAppStore } from "@editor/store";
export type ImageStudioApp = ReturnType<typeof createImageStudioApp>;
export function createImageStudioApp(options?: { documentName?: string; createDefaultArtboard?: boolean }) {
const registry = createCommandRegistry([...viewportCommands, ...selectionCommands, ...documentCommands, ...toolCommands, ...transformCommands, ...historyCommands]);
const registry = createCommandRegistry([...viewportCommands, ...selectionCommands, ...documentCommands, ...toolCommands, ...generationCommands, ...transformCommands, ...historyCommands]);
const store = createAppStore(createInitialAppState(options?.documentName), registry);
if (options?.createDefaultArtboard !== false) {

59
app/comfy.test.ts Normal file
View File

@@ -0,0 +1,59 @@
import { describe, expect, test } from "bun:test";
import { buildSdxlWorkflow, selectGeneratedOutputImage } from "./comfy";
describe("Comfy adapter", () => {
test("selects SaveImage output instead of uploaded input or mask images", () => {
const image = selectGeneratedOutputImage({
outputs: {
"4": { images: [{ filename: "image-studio-input.png", type: "input" }] },
"9": { images: [{ filename: "image-studio-mask.png", type: "input" }] },
"8": { images: [{ filename: "image-studio-inpaint_00001_.png", subfolder: "", type: "output" }] },
},
});
expect(image).toEqual({ filename: "image-studio-inpaint_00001_.png", subfolder: "", type: "output" });
});
test("falls back to generated filename prefixes when node ids differ", () => {
const image = selectGeneratedOutputImage({
outputs: {
"12": { images: [{ filename: "image-studio-inpaint_00002_.png", type: "output" }] },
"4": { images: [{ filename: "image-studio-input.png", type: "input" }] },
},
});
expect(image?.filename).toBe("image-studio-inpaint_00002_.png");
});
test("builds neutral inpaint with VAEEncodeForInpaint", () => {
const workflow = buildSdxlWorkflow(inpaintRequest({ maskedContent: "neutral" }));
expect(workflow["5"]?.class_type).toBe("VAEEncodeForInpaint");
expect(workflow["5"]?.inputs).toMatchObject({ grow_mask_by: 6, mask: ["11", 0] });
expect(workflow["6"]?.inputs.latent_image).toEqual(["5", 0]);
});
test("builds original-content inpaint with a latent noise mask", () => {
const workflow = buildSdxlWorkflow(inpaintRequest({ maskedContent: "original", growMaskBy: 12 }));
expect(workflow["5"]?.class_type).toBe("VAEEncode");
expect(workflow["12"]?.class_type).toBe("GrowMask");
expect(workflow["12"]?.inputs).toMatchObject({ mask: ["11", 0], expand: 12 });
expect(workflow["13"]?.class_type).toBe("SetLatentNoiseMask");
expect(workflow["13"]?.inputs).toMatchObject({ samples: ["5", 0], mask: ["12", 0] });
expect(workflow["6"]?.inputs.latent_image).toEqual(["13", 0]);
});
});
function inpaintRequest(inpaint: { maskedContent: "neutral" | "original"; growMaskBy?: number }) {
return {
mode: "inpaint" as const,
model: "model.safetensors",
prompt: "replace garment",
width: 128,
height: 128,
inputImage: "input.png",
maskImage: "mask.png",
inpaint,
};
}

View File

@@ -20,6 +20,17 @@ type ComfyGenerateRequest = {
bottom?: number;
feathering?: number;
};
inpaint?: {
growMaskBy?: number;
maskBlur?: number;
maskFeather?: number;
maskExpand?: number;
cropPadding?: number;
maskPolarity?: "hidden" | "revealed";
maskedContent?: "neutral" | "original" | "originalColor" | "edges";
crop?: unknown;
placement?: unknown;
};
inputImage?: string;
maskImage?: string;
};
@@ -27,10 +38,14 @@ type ComfyGenerateRequest = {
const comfyBaseUrl = process.env.COMFYUI_URL ?? "http://127.0.0.1:8188";
export async function handleComfyApi(request: Request) {
const url = new URL(request.url);
if (url.pathname === "/api/comfy/models" && request.method === "GET") return json(await listGenerationOptions());
if (url.pathname === "/api/comfy/generate" && request.method === "POST") return json(await generate(await request.json() as ComfyGenerateRequest));
return new Response("Not found", { status: 404 });
try {
const url = new URL(request.url);
if (url.pathname === "/api/comfy/models" && request.method === "GET") return json(await listGenerationOptions());
if (url.pathname === "/api/comfy/generate" && request.method === "POST") return json(await generate(await request.json() as ComfyGenerateRequest));
return new Response("Not found", { status: 404 });
} catch (error) {
return new Response(error instanceof Error ? error.message : "ComfyUI request failed", { status: 500 });
}
}
async function listGenerationOptions() {
@@ -61,6 +76,7 @@ async function generate(request: ComfyGenerateRequest) {
const clientId = crypto.randomUUID();
const uploaded = request.inputImage ? await uploadDataUrl(request.inputImage, `image-studio-${crypto.randomUUID()}.png`) : undefined;
const mask = request.maskImage ? await uploadDataUrl(request.maskImage, `image-studio-mask-${crypto.randomUUID()}.png`) : undefined;
if (request.mode === "inpaint" && (!uploaded || !mask)) throw new Error("Inpaint requires normalized input and mask images");
const prompt = buildSdxlWorkflow({ ...request, inputImage: uploaded, maskImage: mask });
const queued = await fetch(`${comfyBaseUrl}/prompt`, {
@@ -69,9 +85,15 @@ async function generate(request: ComfyGenerateRequest) {
body: JSON.stringify({ client_id: clientId, prompt }),
});
if (!queued.ok) throw new Error(`ComfyUI prompt failed: ${queued.status} ${await queued.text()}`);
const { prompt_id } = await queued.json() as { prompt_id: string };
const queuedBody = await queued.json() as { prompt_id?: string; node_errors?: unknown };
const nodeError = nodeErrorsMessage(queuedBody.node_errors);
if (nodeError) throw new Error(`ComfyUI rejected the workflow: ${nodeError}`);
if (!queuedBody.prompt_id) throw new Error("ComfyUI did not return a prompt id");
const prompt_id = queuedBody.prompt_id;
const history = await waitForHistory(prompt_id);
const image = firstOutputImage(history);
const historyError = historyErrorMessage(history);
if (historyError) throw new Error(`ComfyUI generation failed: ${historyError}`);
const image = selectGeneratedOutputImage(history);
if (!image) throw new Error("ComfyUI did not return an image");
const imageResponse = await fetch(`${comfyBaseUrl}/view?${new URLSearchParams({ filename: image.filename, subfolder: image.subfolder ?? "", type: image.type ?? "output" })}`);
@@ -82,7 +104,7 @@ async function generate(request: ComfyGenerateRequest) {
async function uploadDataUrl(dataUrl: string, filename: string) {
const match = /^data:([^;]+);base64,(.+)$/.exec(dataUrl);
if (!match) return undefined;
if (!match) throw new Error("Expected a base64 data URL image");
const mimeType = match[1] ?? "image/png";
const base64 = match[2] ?? "";
const form = new FormData();
@@ -103,19 +125,25 @@ async function waitForHistory(promptId: string) {
}
await Bun.sleep(500);
}
throw new Error("Timed out waiting for ComfyUI");
throw new Error(`Timed out waiting for ComfyUI prompt ${promptId}`);
}
function firstOutputImage(history: unknown): { filename: string; subfolder?: string; type?: string } | undefined {
export function selectGeneratedOutputImage(history: unknown): { filename: string; subfolder?: string; type?: string } | undefined {
const outputs = (history as { outputs?: Record<string, { images?: { filename: string; subfolder?: string; type?: string }[] }> }).outputs ?? {};
for (const output of Object.values(outputs)) {
const image = output.images?.[0];
if (image) return image;
}
return undefined;
const saveImageOutput = outputs["8"]?.images?.find(isGeneratedImage);
if (saveImageOutput) return saveImageOutput;
const prefixedOutput = Object.values(outputs).flatMap((output) => output.images ?? []).find((image) => image.filename.startsWith("image-studio-") && image.type !== "input");
if (prefixedOutput) return prefixedOutput;
return Object.values(outputs).flatMap((output) => output.images ?? []).find(isGeneratedImage);
}
function buildSdxlWorkflow(request: ComfyGenerateRequest) {
function isGeneratedImage(image: { filename: string; subfolder?: string; type?: string }) {
return image.type === undefined || image.type === "output";
}
export function buildSdxlWorkflow(request: ComfyGenerateRequest) {
if (!request.width || !request.height) throw new Error("Generation width and height are required");
const width = Math.max(64, Math.round(request.width));
const height = Math.max(64, Math.round(request.height));
@@ -127,7 +155,7 @@ function buildSdxlWorkflow(request: ComfyGenerateRequest) {
const scheduler = request.scheduler ?? "normal";
const positive = request.prompt;
const negative = request.negativePrompt ?? "";
const workflow: Record<string, unknown> = {
const workflow: Record<string, { class_type: string; inputs: Record<string, unknown> }> = {
"1": { class_type: "CheckpointLoaderSimple", inputs: { ckpt_name: request.model } },
"2": { class_type: "CLIPTextEncode", inputs: { text: positive, clip: ["1", 1] } },
"3": { class_type: "CLIPTextEncode", inputs: { text: negative, clip: ["1", 1] } },
@@ -145,13 +173,21 @@ function buildSdxlWorkflow(request: ComfyGenerateRequest) {
if (request.mode === "inpaint" && request.maskImage) {
workflow["9"] = { class_type: "LoadImage", inputs: { image: request.maskImage } };
workflow["5"] = { class_type: "VAEEncodeForInpaint", inputs: { pixels: ["4", 0], vae: ["1", 2], mask: ["9", 1], grow_mask_by: 6 } };
workflow["11"] = { class_type: "ImageToMask", inputs: { image: ["9", 0], channel: "red" } };
if (usesOriginalLatentContent(request)) {
workflow["5"] = { class_type: "VAEEncode", inputs: { pixels: ["4", 0], vae: ["1", 2] } };
workflow["12"] = { class_type: "GrowMask", inputs: { mask: ["11", 0], expand: resolveGrowMaskBy(request), tapered_corners: true } };
workflow["13"] = { class_type: "SetLatentNoiseMask", inputs: { samples: ["5", 0], mask: ["12", 0] } };
workflow["6"].inputs.latent_image = ["13", 0];
} else {
workflow["5"] = { class_type: "VAEEncodeForInpaint", inputs: { pixels: ["4", 0], vae: ["1", 2], mask: ["11", 0], grow_mask_by: resolveGrowMaskBy(request) } };
}
return workflow;
}
if (request.mode === "outpaint") {
workflow["10"] = { class_type: "ImagePadForOutpaint", inputs: { image: ["4", 0], left: Math.round(request.outpaint?.left ?? 0), top: Math.round(request.outpaint?.top ?? 0), right: Math.round(request.outpaint?.right ?? 0), bottom: Math.round(request.outpaint?.bottom ?? 0), feathering: Math.round(request.outpaint?.feathering ?? 0) } };
workflow["5"] = { class_type: "VAEEncodeForInpaint", inputs: { pixels: ["10", 0], vae: ["1", 2], mask: ["10", 1], grow_mask_by: 6 } };
workflow["5"] = { class_type: "VAEEncodeForInpaint", inputs: { pixels: ["10", 0], vae: ["1", 2], mask: ["10", 1], grow_mask_by: resolveGrowMaskBy(request) } };
return workflow;
}
@@ -159,6 +195,56 @@ function buildSdxlWorkflow(request: ComfyGenerateRequest) {
return workflow;
}
function resolveGrowMaskBy(request: ComfyGenerateRequest): number {
const value = request.inpaint?.growMaskBy ?? 6;
if (!Number.isFinite(value)) return 6;
return Math.round(Math.max(0, Math.min(256, value)));
}
function usesOriginalLatentContent(request: ComfyGenerateRequest): boolean {
return request.inpaint?.maskedContent === "original" || request.inpaint?.maskedContent === "originalColor" || request.inpaint?.maskedContent === "edges";
}
function nodeErrorsMessage(nodeErrors: unknown): string | undefined {
if (!nodeErrors) return undefined;
if (Array.isArray(nodeErrors) && nodeErrors.length === 0) return undefined;
if (typeof nodeErrors === "object" && Object.keys(nodeErrors).length === 0) return undefined;
if (typeof nodeErrors === "string") return nodeErrors;
try {
return JSON.stringify(nodeErrors);
} catch {
return "Unknown node validation error";
}
}
function historyErrorMessage(history: unknown): string | undefined {
const status = (history as { status?: { status_str?: string; completed?: boolean; messages?: unknown[] } }).status;
if (!status) return undefined;
if (status.status_str && status.status_str !== "success") return statusMessage(status);
if (status.completed === false) return statusMessage(status);
return undefined;
}
function statusMessage(status: { status_str?: string; messages?: unknown[] }) {
const message = status.messages?.map(formatHistoryMessage).filter(Boolean).join("; ");
return message || status.status_str || "Unknown execution error";
}
function formatHistoryMessage(message: unknown): string | undefined {
if (!Array.isArray(message)) return undefined;
const eventName = typeof message[0] === "string" ? message[0] : undefined;
const payload = message[1];
if (payload && typeof payload === "object") {
const detail = payload as { exception_message?: string; node_type?: string; node_id?: string | number };
if (detail.exception_message) {
const node = detail.node_type ? ` in ${detail.node_type}${detail.node_id !== undefined ? ` ${detail.node_id}` : ""}` : "";
return `${eventName ?? "error"}${node}: ${detail.exception_message}`;
}
}
return eventName;
}
function json(value: unknown) {
return new Response(JSON.stringify(value), { headers: { "content-type": "application/json" } });
}

View File

@@ -8,6 +8,7 @@ import {
documentAddImageLayerCommand,
documentAddLayerMaskCommand,
documentAddRasterLayerCommand,
documentApplyLayerMaskOperationCommand,
documentGroupLayersCommand,
documentMoveLayerCommand,
documentRemoveArtboardCommand,
@@ -236,6 +237,30 @@ describe("document commands", () => {
expect(unmasked.editor.maskEdit).toBeUndefined();
});
test("applies operations only to attached layer mask assets", () => {
const state = documentAddLayerMaskCommand.execute(
{ state: documentWithLayers([raster("target", "Target"), raster("loose", "Loose", "loose-asset")]) },
{ layerId: "target", asset: maskAsset(), maskLayer: raster("mask", "Target Mask", "mask-asset") },
);
const withLooseAsset = documentAddAssetCommand.execute(
{ state },
{ asset: { id: "loose-asset", name: "Loose", mimeType: "image/png", source: "loose-source", intrinsicSize: { w: 100, h: 100 } } },
);
const updated = documentApplyLayerMaskOperationCommand.execute(
{ state: withLooseAsset },
{ maskLayerId: "mask", source: "updated-mask", mimeType: "image/png", operation: { type: "invert" } },
);
const ignored = documentApplyLayerMaskOperationCommand.execute(
{ state: updated },
{ maskLayerId: "loose", source: "wrong", operation: { type: "invert" } },
);
expect(updated.document.assets.find((asset) => asset.id === "mask-asset")?.source).toBe("updated-mask");
expect(updated.document.assets.find((asset) => asset.id === "mask-asset")?.mimeType).toBe("image/png");
expect(ignored.document.assets.find((asset) => asset.id === "loose-asset")?.source).toBe("loose-source");
});
test("cleans mask references when deleting targets or mask layers", () => {
const state = documentAddLayerMaskCommand.execute(
{ state: documentWithLayers([raster("target", "Target")]) },

View File

@@ -113,6 +113,25 @@ export type DocumentAddLayerMaskPayload = {
maskLayer: RasterLayer;
};
export type LayerMaskOperation =
| { type: "paint" }
| { type: "magicWand" }
| { type: "chromaKey" }
| { type: "invert" }
| { type: "fill"; fill: "white" | "black" | "clear" }
| { type: "feather"; radius: number }
| { type: "expand"; radius: number }
| { type: "contract"; radius: number }
| { type: "blur"; radius: number }
| { type: "despeckle"; strength: number };
export type DocumentApplyLayerMaskOperationPayload = {
maskLayerId: LayerId;
source: string;
mimeType?: string;
operation: LayerMaskOperation;
};
export type DocumentRemoveLayerMaskPayload = {
layerId: LayerId;
};
@@ -490,6 +509,38 @@ export const documentAddLayerMaskCommand: Command<DocumentAddLayerMaskPayload> =
},
};
export const documentApplyLayerMaskOperationCommand: Command<DocumentApplyLayerMaskOperationPayload> = {
id: commandIds.documentApplyLayerMaskOperation,
name: "Apply layer mask operation",
execute({ state }, payload) {
if (!payload.source.trim()) return state;
const maskLocation = findLayerLocation(state.document, payload.maskLayerId);
if (!maskLocation || maskLocation.layer.type === "group") return state;
if (!isReferencedMaskLayer(state.document, payload.maskLayerId)) return state;
return {
...state,
document: {
...state.document,
assets: state.document.assets.map((asset) =>
asset.id === maskLocation.layer.assetId
? {
...asset,
source: payload.source,
mimeType: payload.mimeType ?? asset.mimeType,
}
: asset,
),
},
editor: {
...state.editor,
brushStrokePreview: state.editor.brushStrokePreview?.assetId === maskLocation.layer.assetId ? undefined : state.editor.brushStrokePreview,
},
};
},
};
export const documentRemoveLayerMaskCommand: Command<DocumentRemoveLayerMaskPayload> = {
id: commandIds.documentRemoveLayerMask,
name: "Remove layer mask",
@@ -564,6 +615,7 @@ export const documentCommands = [
documentRenameLayerCommand,
documentSetLayerClippingMaskCommand,
documentAddLayerMaskCommand,
documentApplyLayerMaskOperationCommand,
documentRemoveLayerMaskCommand,
] satisfies Command<unknown>[];
@@ -582,6 +634,18 @@ function findLayerLocation(document: ImageDocument, layerId: LayerId): LayerLoca
return undefined;
}
function isReferencedMaskLayer(document: ImageDocument, maskLayerId: LayerId): boolean {
return document.artboards.some((artboard) => isReferencedMaskLayerInTree(artboard.layers, maskLayerId));
}
function isReferencedMaskLayerInTree(layers: readonly Layer[], maskLayerId: LayerId): boolean {
for (const layer of layers) {
if (layer.clippingMask?.maskLayerId === maskLayerId) return true;
if (layer.type === "group" && isReferencedMaskLayerInTree(layer.children, maskLayerId)) return true;
}
return false;
}
function findLayerLocationInTree(layers: Layer[], layerId: LayerId, artboardId: ArtboardId, parentGroupId?: LayerId): LayerLocation | undefined {
for (let index = 0; index < layers.length; index++) {
const layer = layers[index];

143
commands/generation.test.ts Normal file
View File

@@ -0,0 +1,143 @@
import { describe, expect, test } from "bun:test";
import type { GenerationCandidate } from "@editor/state";
import { createInitialAppState } from "@editor/initial-state";
import {
generationAddCandidateCommand,
generationApplyCandidateAsLayerCommand,
generationRemoveCandidateCommand,
generationReplaceCandidatePixelsCommand,
} from "./generation";
describe("generation commands", () => {
test("adds, selects, and removes candidates", () => {
const state = createInitialAppState("Test");
const candidate = generationCandidate("candidate-1");
const added = generationAddCandidateCommand.execute({ state }, { candidate });
const removed = generationRemoveCandidateCommand.execute({ state: added }, { candidateId: candidate.id });
expect(added.editor.generation.candidates).toEqual([candidate]);
expect(added.editor.generation.selectedCandidateId).toBe(candidate.id);
expect(removed.editor.generation.candidates).toEqual([]);
expect(removed.editor.generation.selectedCandidateId).toBeUndefined();
});
test("applies candidates as top-level layers", () => {
const state = generationAddCandidateCommand.execute({ state: documentWithSourceLayer() }, { candidate: generationCandidate("candidate-1") });
const next = generationApplyCandidateAsLayerCommand.execute(
{ state },
{ candidateId: "candidate-1", assetId: "generated-asset", layerId: "generated-layer" },
);
expect(next.document.assets.find((asset) => asset.id === "generated-asset")?.source).toBe("generated-source");
expect(next.document.artboards[0]?.layers[0]?.id).toBe("generated-layer");
expect(next.editor.selection).toEqual({ artboardId: "a1", layerIds: ["generated-layer"] });
});
test("replaces source asset pixels for inpaint candidates", () => {
const state = generationAddCandidateCommand.execute({ state: documentWithSourceLayer() }, { candidate: generationCandidate("candidate-1", true) });
const next = generationReplaceCandidatePixelsCommand.execute(
{ state },
{ candidateId: "candidate-1", source: "composited-source", mimeType: "image/png" },
);
expect(next.document.assets.find((asset) => asset.id === "source-asset")?.source).toBe("composited-source");
expect(next.document.assets.find((asset) => asset.id === "source-asset")?.mimeType).toBe("image/png");
expect(next.editor.selection).toEqual({ artboardId: "a1", layerIds: ["source-layer"] });
});
});
function documentWithSourceLayer() {
return {
...createInitialAppState("Test"),
document: {
...createInitialAppState("Test").document,
assets: [
{ id: "source-asset", name: "Source", mimeType: "image/png", source: "source", intrinsicSize: { w: 100, h: 100 } },
{ id: "mask-asset", name: "Mask", mimeType: "image/png", source: "mask", intrinsicSize: { w: 100, h: 100 } },
],
artboards: [
{
id: "a1",
name: "Artboard 1",
bounds: { x: 0, y: 0, w: 100, h: 100 },
backgroundColor: "transparent",
visible: true,
locked: false,
layers: [
raster("mask-layer", "Mask", "mask-asset"),
{ ...raster("source-layer", "Source", "source-asset"), clippingMask: { maskLayerId: "mask-layer" } },
],
},
],
},
};
}
function generationCandidate(id: string, inpaint = false): GenerationCandidate {
const candidate: GenerationCandidate = {
id,
source: "generated-source",
mimeType: "image/png",
intrinsicSize: { w: 64, h: 64 },
mode: inpaint ? "inpaint" : "text-to-image",
settings: createInitialAppState("Test").editor.tools.generate,
seed: 123,
width: 64,
height: 64,
placement: {
artboardId: "a1",
layerName: "Generated",
transform: { position: { x: 5, y: 6 }, scale: { x: 1, y: 1 }, rotation: 0 },
},
};
return inpaint
? {
...candidate,
inputImage: "input",
maskImage: "mask",
inpaint: {
targetLayerId: "source-layer",
maskLayerId: "mask-layer",
sourceAssetId: "source-asset",
maskAssetId: "mask-asset",
inputImage: "input",
maskImage: "mask",
crop: {
assetBounds: { x: 0, y: 0, w: 64, h: 64 },
documentBounds: { x: 0, y: 0, w: 64, h: 64 },
padding: 12,
maskedAreaOnly: true,
},
mask: {
polarity: "hidden",
activeBounds: { x: 10, y: 10, w: 20, h: 20 },
},
backend: {
growMaskBy: 6,
maskedContent: "neutral",
maskBlur: 0,
maskFeather: 0,
maskExpand: 0,
cropPadding: 12,
},
},
}
: candidate;
}
function raster(id: string, name: string, assetId: string) {
return {
id,
type: "raster" as const,
name,
visible: true,
locked: false,
opacity: 1,
assetId,
transform: { position: { x: 0, y: 0 }, scale: { x: 1, y: 1 }, rotation: 0 },
};
}

217
commands/generation.ts Normal file
View File

@@ -0,0 +1,217 @@
import type { Asset } from "@core/asset";
import type { ImageDocument } from "@core/document";
import type { ArtboardId, AssetId, LayerId } from "@core/id";
import type { ImageLayer } from "@core/image-layer";
import type { Layer } from "@core/layer";
import type { GenerationCandidate } from "@editor/state";
import type { Command } from "./command";
import { commandIds } from "./ids";
export type GenerationAddCandidatePayload = {
candidate: GenerationCandidate;
};
export type GenerationSelectCandidatePayload = {
candidateId?: string;
};
export type GenerationRemoveCandidatePayload = {
candidateId: string;
};
export type GenerationApplyCandidateAsLayerPayload = {
candidateId: string;
assetId: AssetId;
layerId: LayerId;
variant?: boolean;
};
export type GenerationReplaceCandidatePixelsPayload = {
candidateId: string;
source: string;
mimeType?: string;
};
const maxCandidates = 12;
export const generationAddCandidateCommand: Command<GenerationAddCandidatePayload> = {
id: commandIds.generationAddCandidate,
name: "Add generation candidate",
history: { mode: "ignore" },
execute({ state }, payload) {
const candidates = [payload.candidate, ...state.editor.generation.candidates.filter((candidate) => candidate.id !== payload.candidate.id)].slice(0, maxCandidates);
return {
...state,
editor: {
...state.editor,
generation: {
candidates,
selectedCandidateId: payload.candidate.id,
},
},
};
},
};
export const generationSelectCandidateCommand: Command<GenerationSelectCandidatePayload> = {
id: commandIds.generationSelectCandidate,
name: "Select generation candidate",
history: { mode: "ignore" },
execute({ state }, payload) {
const selectedCandidateId = payload.candidateId && state.editor.generation.candidates.some((candidate) => candidate.id === payload.candidateId) ? payload.candidateId : undefined;
if (state.editor.generation.selectedCandidateId === selectedCandidateId) return state;
return {
...state,
editor: {
...state.editor,
generation: {
...state.editor.generation,
selectedCandidateId,
},
},
};
},
};
export const generationRemoveCandidateCommand: Command<GenerationRemoveCandidatePayload> = {
id: commandIds.generationRemoveCandidate,
name: "Remove generation candidate",
history: { mode: "ignore" },
execute({ state }, payload) {
const candidates = state.editor.generation.candidates.filter((candidate) => candidate.id !== payload.candidateId);
if (candidates.length === state.editor.generation.candidates.length) return state;
const selectedCandidateId = state.editor.generation.selectedCandidateId === payload.candidateId ? candidates[0]?.id : state.editor.generation.selectedCandidateId;
return {
...state,
editor: {
...state.editor,
generation: {
candidates,
selectedCandidateId,
},
},
};
},
};
export const generationClearCandidatesCommand: Command = {
id: commandIds.generationClearCandidates,
name: "Clear generation candidates",
history: { mode: "ignore" },
execute({ state }) {
if (state.editor.generation.candidates.length === 0 && !state.editor.generation.selectedCandidateId) return state;
return {
...state,
editor: {
...state.editor,
generation: { candidates: [], selectedCandidateId: undefined },
},
};
},
};
export const generationApplyCandidateAsLayerCommand: Command<GenerationApplyCandidateAsLayerPayload> = {
id: commandIds.generationApplyCandidateAsLayer,
name: "Apply generation candidate as layer",
execute({ state }, payload) {
const candidate = state.editor.generation.candidates.find((item) => item.id === payload.candidateId);
if (!candidate) return state;
if (state.document.assets.some((asset) => asset.id === payload.assetId) || findLayerLocation(state.document, payload.layerId)) return state;
if (!state.document.artboards.some((artboard) => artboard.id === candidate.placement.artboardId)) return state;
const asset: Asset = {
id: payload.assetId,
name: payload.variant ? `${candidate.placement.layerName} variant` : candidate.placement.layerName,
mimeType: candidate.mimeType,
source: candidate.source,
intrinsicSize: { ...candidate.intrinsicSize },
};
const layer: ImageLayer = {
id: payload.layerId,
type: "image",
name: payload.variant ? `${candidate.placement.layerName} variant` : candidate.placement.layerName,
visible: true,
locked: false,
opacity: 1,
assetId: asset.id,
transform: {
position: { ...candidate.placement.transform.position },
scale: { ...candidate.placement.transform.scale },
rotation: candidate.placement.transform.rotation,
},
};
return {
...state,
document: insertLayerAtTop({ ...state.document, assets: [...state.document.assets, asset] }, candidate.placement.artboardId, layer),
editor: {
...state.editor,
selection: { artboardId: candidate.placement.artboardId, layerIds: [layer.id] },
},
};
},
};
export const generationReplaceCandidatePixelsCommand: Command<GenerationReplaceCandidatePixelsPayload> = {
id: commandIds.generationReplaceCandidatePixels,
name: "Replace masked pixels with generation candidate",
execute({ state }, payload) {
const candidate = state.editor.generation.candidates.find((item) => item.id === payload.candidateId);
if (!candidate?.inpaint || !payload.source.trim()) return state;
const targetAsset = state.document.assets.find((asset) => asset.id === candidate.inpaint?.sourceAssetId);
const targetLayerLocation = findLayerLocation(state.document, candidate.inpaint.targetLayerId);
if (!targetAsset || !targetLayerLocation) return state;
return {
...state,
document: {
...state.document,
assets: state.document.assets.map((asset) => asset.id === targetAsset.id ? { ...asset, source: payload.source, mimeType: payload.mimeType ?? asset.mimeType } : asset),
},
editor: {
...state.editor,
selection: { artboardId: targetLayerLocation.artboardId, layerIds: [candidate.inpaint.targetLayerId] },
},
};
},
};
export const generationCommands = [
generationAddCandidateCommand,
generationSelectCandidateCommand,
generationRemoveCandidateCommand,
generationClearCandidatesCommand,
generationApplyCandidateAsLayerCommand,
generationReplaceCandidatePixelsCommand,
] satisfies Command<unknown>[];
type LayerLocation = {
artboardId: ArtboardId;
layer: Layer;
};
function insertLayerAtTop(document: ImageDocument, artboardId: ArtboardId, layer: Layer): ImageDocument {
return {
...document,
artboards: document.artboards.map((artboard) => artboard.id === artboardId ? { ...artboard, layers: [layer, ...artboard.layers] } : artboard),
};
}
function findLayerLocation(document: ImageDocument, layerId: LayerId): LayerLocation | undefined {
for (const artboard of document.artboards) {
const layer = findLayerInTree(artboard.layers, layerId);
if (layer) return { artboardId: artboard.id, layer };
}
return undefined;
}
function findLayerInTree(layers: readonly Layer[], layerId: LayerId): Layer | undefined {
for (const layer of layers) {
if (layer.id === layerId) return layer;
if (layer.type === "group") {
const child = findLayerInTree(layer.children, layerId);
if (child) return child;
}
}
return undefined;
}

View File

@@ -19,6 +19,7 @@ export const commandIds = {
documentRenameLayer: "document.renameLayer",
documentSetLayerClippingMask: "document.setLayerClippingMask",
documentAddLayerMask: "document.addLayerMask",
documentApplyLayerMaskOperation: "document.applyLayerMaskOperation",
documentRemoveLayerMask: "document.removeLayerMask",
selectionSet: "selection.set",
selectionClear: "selection.clear",
@@ -35,6 +36,12 @@ export const commandIds = {
toolExitMaskEdit: "tool.exitMaskEdit",
toolEnterTemporaryPan: "tool.enterTemporaryPan",
toolExitTemporaryPan: "tool.exitTemporaryPan",
generationAddCandidate: "generation.addCandidate",
generationSelectCandidate: "generation.selectCandidate",
generationRemoveCandidate: "generation.removeCandidate",
generationClearCandidates: "generation.clearCandidates",
generationApplyCandidateAsLayer: "generation.applyCandidateAsLayer",
generationReplaceCandidatePixels: "generation.replaceCandidatePixels",
transformBegin: "transform.begin",
transformUpdate: "transform.update",
transformSetBounds: "transform.setBounds",

View File

@@ -6,6 +6,7 @@ export {
documentAddImageLayerCommand,
documentAddLayerMaskCommand,
documentAddRasterLayerCommand,
documentApplyLayerMaskOperationCommand,
documentCommands,
documentGroupLayersCommand,
documentMoveLayerCommand,
@@ -30,6 +31,7 @@ export type {
DocumentAddImageLayerPayload,
DocumentAddLayerMaskPayload,
DocumentAddRasterLayerPayload,
DocumentApplyLayerMaskOperationPayload,
DocumentGroupLayersPayload,
DocumentMoveLayerPayload,
DocumentRemoveArtboardPayload,
@@ -45,8 +47,25 @@ export type {
DocumentSetLayerVisiblePayload,
DocumentUpdateAssetSourcePayload,
DocumentUngroupLayerPayload,
LayerMaskOperation,
} from "./document";
export { historyCommands, historyRedoCommand, historyUndoCommand } from "./history";
export {
generationAddCandidateCommand,
generationApplyCandidateAsLayerCommand,
generationClearCandidatesCommand,
generationCommands,
generationRemoveCandidateCommand,
generationReplaceCandidatePixelsCommand,
generationSelectCandidateCommand,
} from "./generation";
export type {
GenerationAddCandidatePayload,
GenerationApplyCandidateAsLayerPayload,
GenerationRemoveCandidatePayload,
GenerationReplaceCandidatePixelsPayload,
GenerationSelectCandidatePayload,
} from "./generation";
export type { CommandDispatcher, Dispatch } from "./dispatcher";
export type { CommandId, CommandPayloads } from "./payloads";
export { createCommandDispatcher } from "./dispatcher";

View File

@@ -6,6 +6,7 @@ import type {
DocumentAddImageLayerPayload,
DocumentAddLayerMaskPayload,
DocumentAddRasterLayerPayload,
DocumentApplyLayerMaskOperationPayload,
DocumentGroupLayersPayload,
DocumentMoveLayerPayload,
DocumentRemoveArtboardPayload,
@@ -22,6 +23,13 @@ import type {
DocumentUpdateAssetSourcePayload,
DocumentUngroupLayerPayload,
} from "./document";
import type {
GenerationAddCandidatePayload,
GenerationApplyCandidateAsLayerPayload,
GenerationRemoveCandidatePayload,
GenerationReplaceCandidatePixelsPayload,
GenerationSelectCandidatePayload,
} from "./generation";
import type { SelectionAddLayerPayload, SelectionSetPayload } from "./selection";
import type { ToolEnterMaskEditPayload, ToolSetActivePayload, ToolSetBrushPreviewPayload, ToolSetBrushSettingsPayload, ToolSetBrushStrokePreviewPayload, ToolSetChromaKeySettingsPayload, ToolSetGenerateSettingsPayload, ToolSetMagicWandSettingsPayload, ToolSetMaskViewModePayload } from "./tool";
import type { TransformBeginPayload, TransformSetBoundsPayload, TransformUpdatePayload } from "./transform";
@@ -46,6 +54,7 @@ export type CommandPayloads = {
[commandIds.documentAddRasterLayer]: DocumentAddRasterLayerPayload;
[commandIds.documentAddGroupLayer]: DocumentAddGroupLayerPayload;
[commandIds.documentAddLayerMask]: DocumentAddLayerMaskPayload;
[commandIds.documentApplyLayerMaskOperation]: DocumentApplyLayerMaskOperationPayload;
[commandIds.documentRemoveLayerMask]: DocumentRemoveLayerMaskPayload;
[commandIds.documentMoveLayer]: DocumentMoveLayerPayload;
[commandIds.documentGroupLayers]: DocumentGroupLayersPayload;
@@ -70,6 +79,12 @@ export type CommandPayloads = {
[commandIds.toolExitMaskEdit]: void;
[commandIds.toolEnterTemporaryPan]: void;
[commandIds.toolExitTemporaryPan]: void;
[commandIds.generationAddCandidate]: GenerationAddCandidatePayload;
[commandIds.generationSelectCandidate]: GenerationSelectCandidatePayload;
[commandIds.generationRemoveCandidate]: GenerationRemoveCandidatePayload;
[commandIds.generationClearCandidates]: void;
[commandIds.generationApplyCandidateAsLayer]: GenerationApplyCandidateAsLayerPayload;
[commandIds.generationReplaceCandidatePixels]: GenerationReplaceCandidatePixelsPayload;
[commandIds.transformBegin]: TransformBeginPayload;
[commandIds.transformUpdate]: TransformUpdatePayload;
[commandIds.transformSetBounds]: TransformSetBoundsPayload;

View File

@@ -1,11 +1,11 @@
import { describe, expect, test } from "bun:test";
import { createInitialAppState } from "@editor/initial-state";
import { toolEnterMaskEditCommand, toolEnterTemporaryPanCommand, toolExitMaskEditCommand, toolExitTemporaryPanCommand, toolSetActiveCommand, toolSetBrushPreviewCommand, toolSetBrushSettingsCommand, toolSetBrushStrokePreviewCommand, toolSetChromaKeySettingsCommand, toolSetMaskViewModeCommand } from "./tool";
import { toolEnterMaskEditCommand, toolEnterTemporaryPanCommand, toolExitMaskEditCommand, toolExitTemporaryPanCommand, toolSetActiveCommand, toolSetBrushPreviewCommand, toolSetBrushSettingsCommand, toolSetBrushStrokePreviewCommand, toolSetChromaKeySettingsCommand, toolSetGenerateSettingsCommand, toolSetMaskViewModeCommand } from "./tool";
const defaultBrush = { color: "#111827", size: 8, hardness: 100 };
const defaultChromaKey = { color: "#00ff00", tolerance: 32, softness: 24, feather: 0, choke: 0, despeckle: 0, spill: 50 };
const defaultMagicWand = { tolerance: 32, feather: 0, choke: 0, despeckle: 0, contiguous: true, mode: "replace" as const };
const defaultGenerate = { mode: "text-to-image" as const, model: "auto" as const, prompt: "", negativePrompt: "", strength: 75, steps: 30, cfg: 7, seed: -1, sampler: "euler", scheduler: "normal", width: 1024, height: 1024, outpaint: { left: 128, top: 128, right: 128, bottom: 128, feathering: 32 } };
const defaultGenerate = { mode: "text-to-image" as const, model: "auto" as const, prompt: "", negativePrompt: "", strength: 75, steps: 30, cfg: 7, seed: -1, sampler: "euler", scheduler: "normal", width: 1024, height: 1024, outpaint: { left: 128, top: 128, right: 128, bottom: 128, feathering: 32 }, inpaint: { maskedAreaOnly: true, cropPadding: 96, maskPolarity: "hidden" as const, maskedContent: "neutral" as const, growMaskBy: 6, maskExpand: 0, maskFeather: 0, maskBlur: 0, maskDespeckle: 0 } };
describe("tool commands", () => {
test("sets active tool", () => {
@@ -25,6 +25,16 @@ describe("tool commands", () => {
expect(next.editor.tools.chromaKey).toEqual({ color: "#123456", tolerance: 255, softness: 24, feather: 0, choke: 0, despeckle: 0, spill: 50 });
});
test("sets inpaint generation settings", () => {
const next = toolSetGenerateSettingsCommand.execute(
{ state: createInitialAppState("Test") },
{ mode: "inpaint", inpaint: { ...defaultGenerate.inpaint, cropPadding: 5000, growMaskBy: -4, maskExpand: -500, maskBlur: 300, maskPolarity: "revealed", maskedContent: "original" } },
);
expect(next.editor.tools.generate.mode).toBe("inpaint");
expect(next.editor.tools.generate.inpaint).toEqual({ ...defaultGenerate.inpaint, cropPadding: 2048, growMaskBy: 0, maskExpand: -256, maskBlur: 256, maskPolarity: "revealed", maskedContent: "original" });
});
test("sets and clears brush preview", () => {
const showing = toolSetBrushPreviewCommand.execute({ state: createInitialAppState("Test") }, { position: { x: 10, y: 20 } });
const cleared = toolSetBrushPreviewCommand.execute({ state: showing }, undefined);

View File

@@ -89,6 +89,17 @@ export const toolSetGenerateSettingsCommand: Command<ToolSetGenerateSettingsPayl
bottom: Math.round(clampNumber(payload.outpaint?.bottom ?? state.editor.tools.generate.outpaint.bottom, 0, 2048)),
feathering: Math.round(clampNumber(payload.outpaint?.feathering ?? state.editor.tools.generate.outpaint.feathering, 0, 512)),
},
inpaint: {
maskedAreaOnly: payload.inpaint?.maskedAreaOnly ?? state.editor.tools.generate.inpaint.maskedAreaOnly,
cropPadding: Math.round(clampNumber(payload.inpaint?.cropPadding ?? state.editor.tools.generate.inpaint.cropPadding, 0, 2048)),
maskPolarity: payload.inpaint?.maskPolarity ?? state.editor.tools.generate.inpaint.maskPolarity,
maskedContent: payload.inpaint?.maskedContent ?? state.editor.tools.generate.inpaint.maskedContent,
growMaskBy: Math.round(clampNumber(payload.inpaint?.growMaskBy ?? state.editor.tools.generate.inpaint.growMaskBy, 0, 256)),
maskExpand: Math.round(clampNumber(payload.inpaint?.maskExpand ?? state.editor.tools.generate.inpaint.maskExpand, -256, 256)),
maskFeather: Math.round(clampNumber(payload.inpaint?.maskFeather ?? state.editor.tools.generate.inpaint.maskFeather, 0, 256)),
maskBlur: Math.round(clampNumber(payload.inpaint?.maskBlur ?? state.editor.tools.generate.inpaint.maskBlur, 0, 256)),
maskDespeckle: Math.round(clampNumber(payload.inpaint?.maskDespeckle ?? state.editor.tools.generate.inpaint.maskDespeckle, 0, 64)),
},
},
},
},

View File

@@ -12,6 +12,10 @@ export const initialEditorState: EditorState = {
layerIds: [],
},
tools: initialToolState,
generation: {
candidates: [],
selectedCandidateId: undefined,
},
transformSession: undefined,
maskEdit: undefined,
brushPreview: undefined,

View File

@@ -1,7 +1,7 @@
import type { ImageDocument } from "@core/document";
import type { Angle, Size, Vec2D } from "@core/geometry";
import type { Angle, Rect, Size, Transform, Vec2D } from "@core/geometry";
import type { ArtboardId, AssetId, LayerId } from "@core/id";
import type { ToolState } from "./tools";
import type { GenerateSettings, ToolState } from "./tools";
import type { TransformSession } from "./transform";
export type ViewportState = {
@@ -34,10 +34,61 @@ export type BrushStrokePreviewState = {
source: string;
};
export type GenerationCandidate = {
id: string;
source: string;
mimeType: string;
intrinsicSize: Size;
mode: GenerateSettings["mode"];
settings: GenerateSettings;
seed: number;
width: number;
height: number;
inputImage?: string;
maskImage?: string;
placement: {
artboardId: ArtboardId;
layerName: string;
transform: Transform;
};
inpaint?: {
targetLayerId: LayerId;
maskLayerId: LayerId;
sourceAssetId: AssetId;
maskAssetId: AssetId;
inputImage: string;
maskImage: string;
crop: {
assetBounds: Rect;
documentBounds: Rect;
padding: number;
maskedAreaOnly: boolean;
};
mask: {
polarity: GenerateSettings["inpaint"]["maskPolarity"];
activeBounds: Rect;
};
backend: {
growMaskBy: number;
maskedContent: GenerateSettings["inpaint"]["maskedContent"];
maskBlur: number;
maskFeather: number;
maskExpand: number;
cropPadding: number;
};
};
};
export type GenerationState = {
candidates: GenerationCandidate[];
selectedCandidateId?: string;
};
export type EditorState = {
viewport: ViewportState;
selection: SelectionState;
tools: ToolState;
generation: GenerationState;
transformSession?: TransformSession;
maskEdit?: MaskEditState;
brushPreview?: BrushPreviewState;

View File

@@ -27,6 +27,7 @@ export type MagicWandMode = "replace" | "add" | "subtract";
export type GenerateMode = "text-to-image" | "image-to-image" | "inpaint" | "outpaint";
export type GenerateModel = string;
export type InpaintMaskedContent = "neutral" | "original" | "originalColor" | "edges";
export type GenerateSettings = {
mode: GenerateMode;
@@ -48,6 +49,17 @@ export type GenerateSettings = {
bottom: number;
feathering: number;
};
inpaint: {
maskedAreaOnly: boolean;
cropPadding: number;
maskPolarity: "hidden" | "revealed";
maskedContent: InpaintMaskedContent;
growMaskBy: number;
maskExpand: number;
maskFeather: number;
maskBlur: number;
maskDespeckle: number;
};
};
export type MagicWandSettings = {
@@ -74,7 +86,22 @@ export const initialToolState: ToolState = {
brush: { color: "#111827", size: 8, hardness: 100 },
chromaKey: { color: "#00ff00", tolerance: 32, softness: 24, feather: 0, choke: 0, despeckle: 0, spill: 50 },
magicWand: { tolerance: 32, feather: 0, choke: 0, despeckle: 0, contiguous: true, mode: "replace" },
generate: { mode: "text-to-image", model: "auto", prompt: "", negativePrompt: "", strength: 75, steps: 30, cfg: 7, seed: -1, sampler: "euler", scheduler: "normal", width: 1024, height: 1024, outpaint: { left: 128, top: 128, right: 128, bottom: 128, feathering: 32 } },
generate: {
mode: "text-to-image",
model: "auto",
prompt: "",
negativePrompt: "",
strength: 75,
steps: 30,
cfg: 7,
seed: -1,
sampler: "euler",
scheduler: "normal",
width: 1024,
height: 1024,
outpaint: { left: 128, top: 128, right: 128, bottom: 128, feathering: 32 },
inpaint: { maskedAreaOnly: true, cropPadding: 96, maskPolarity: "hidden", maskedContent: "neutral", growMaskBy: 6, maskExpand: 0, maskFeather: 0, maskBlur: 0, maskDespeckle: 0 },
},
};
export function isPanInteractionMode(interactionMode: InteractionMode): boolean {

View File

@@ -520,7 +520,8 @@ function createMaskVisualizationProgram(gl: WebGL2RenderingContext): MaskVisuali
in vec2 v_texCoord;
out vec4 outColor;
void main() {
float maskAlpha = texture(u_mask, v_texCoord).a;
vec4 maskColor = texture(u_mask, v_texCoord);
float maskAlpha = maskColor.a * dot(maskColor.rgb, vec3(0.2126, 0.7152, 0.0722));
if (u_mode == 0) {
float value = step(0.5, maskAlpha);
outColor = vec4(value, value, value, 1.0);
@@ -642,7 +643,9 @@ function createMaskRevealPreviewProgram(gl: WebGL2RenderingContext) {
out vec4 outColor;
void main() {
vec4 color = texture(u_image, v_texCoord);
float hiddenMaskAlpha = 1.0 - texture(u_mask, v_maskTexCoord).a;
vec4 maskColor = texture(u_mask, v_maskTexCoord);
float maskAlpha = maskColor.a * dot(maskColor.rgb, vec3(0.2126, 0.7152, 0.0722));
float hiddenMaskAlpha = 1.0 - maskAlpha;
float previewAlpha = clamp(hiddenMaskAlpha * u_opacity, 0.0, 1.0);
outColor = vec4(color.rgb * previewAlpha, color.a * previewAlpha);
}`,
@@ -693,7 +696,8 @@ function createMaskedProgram(gl: WebGL2RenderingContext) {
out vec4 outColor;
void main() {
vec4 color = texture(u_image, v_texCoord);
float maskAlpha = texture(u_mask, v_maskTexCoord).a;
vec4 maskColor = texture(u_mask, v_maskTexCoord);
float maskAlpha = maskColor.a * dot(maskColor.rgb, vec3(0.2126, 0.7152, 0.0722));
float alpha = color.a * maskAlpha;
outColor = vec4(color.rgb * maskAlpha, alpha);
}`,

View File

@@ -1,7 +1,9 @@
import type { Asset } from "@core/asset";
import type { ImageDocument } from "@core/document";
import type { Rect } from "@core/geometry";
import type { Layer } from "@core/layer";
import { createDocumentReadIndex, forEachLayerBackToFront, resolveIndexedLayerBounds, type DocumentReadIndex } from "@editor/document-indexes";
import type { EditorState, MaskViewMode, ViewportState } from "@editor/state";
import type { EditorState, GenerationCandidate, MaskViewMode, ViewportState } from "@editor/state";
import { clearScreenRect } from "./clear-rect";
import type { ImageTextureRenderer } from "./image-textures";
import { documentRectToScreenRect } from "./screen-rect";
@@ -14,15 +16,21 @@ const maskRevealPreviewOpacity = 0.28;
export function renderLayers(context: WebGlRendererContext, document: ImageDocument, editor: EditorState, imageTextureRenderer: ImageTextureRenderer) {
const documentIndex = createDocumentReadIndex(document);
const generationCandidate = selectedGenerationCandidate(editor);
for (const artboard of document.artboards) {
if (!artboard.visible) continue;
const clipRect = documentRectToScreenRect(context.canvas, artboard.bounds, editor.viewport);
const maskLayerIds = documentIndex.maskLayerIdsByArtboardId.get(artboard.id) ?? emptyLayerIds;
forEachLayerBackToFront(artboard.layers, (layer) => renderLayer(context, documentIndex, editor, layer, imageTextureRenderer, clipRect, maskLayerIds));
if (generationCandidate?.placement.artboardId === artboard.id) renderGenerationCandidatePreview(context, editor, generationCandidate, imageTextureRenderer, clipRect);
}
}
export function generationCandidatePreviewAssets(editor: EditorState): Asset[] {
return editor.generation.candidates.map((candidate) => generationCandidateAsset(candidate));
}
function renderLayer(
context: WebGlRendererContext,
documentIndex: DocumentReadIndex,
@@ -102,6 +110,44 @@ function isIsolatedMaskView(mode: MaskViewMode) {
return mode === "blackWhite" || mode === "alpha" || mode === "overlay";
}
function renderGenerationCandidatePreview(
context: WebGlRendererContext,
editor: EditorState,
candidate: GenerationCandidate,
imageTextureRenderer: ImageTextureRenderer,
clipRect: ScreenRect,
) {
const rect = documentRectToScreenRect(context.canvas, generationCandidateBounds(candidate), editor.viewport);
imageTextureRenderer.render(generationCandidateAsset(candidate), rect, clipRect);
}
function selectedGenerationCandidate(editor: EditorState): GenerationCandidate | undefined {
return editor.generation.candidates.find((candidate) => candidate.id === editor.generation.selectedCandidateId) ?? editor.generation.candidates[0];
}
function generationCandidateAsset(candidate: GenerationCandidate): Asset {
return {
id: generationCandidateAssetId(candidate.id),
name: candidate.placement.layerName,
mimeType: candidate.mimeType,
source: candidate.source,
intrinsicSize: candidate.intrinsicSize,
};
}
function generationCandidateAssetId(candidateId: string) {
return `generation-candidate:${candidateId}`;
}
function generationCandidateBounds(candidate: GenerationCandidate): Rect {
return {
x: candidate.placement.transform.position.x,
y: candidate.placement.transform.position.y,
w: candidate.intrinsicSize.w * candidate.placement.transform.scale.x,
h: candidate.intrinsicSize.h * candidate.placement.transform.scale.y,
};
}
function assetWithBrushStrokePreview<TAsset extends ImageDocument["assets"][number] | undefined>(asset: TAsset, editor: EditorState): TAsset {
if (!asset || editor.brushStrokePreview?.assetId !== asset.id) return asset;
return { ...asset, source: editor.brushStrokePreview.source } as TAsset;

View File

@@ -4,7 +4,7 @@ import { renderArtboard } from "./artboard";
import { createBrushPreviewRenderer } from "./brush-preview";
import { createCheckerboardRenderer } from "./checkerboard";
import { createImageTextureRenderer } from "./image-textures";
import { renderLayers } from "./layers";
import { generationCandidatePreviewAssets, renderLayers } from "./layers";
import { renderSelectionOverlay } from "./selection";
import { renderTransformControls } from "./transform-controls";
import type { WebGlRendererContext } from "./types";
@@ -62,7 +62,7 @@ export function createRenderer(canvas: HTMLCanvasElement, backend: RendererBacke
for (const artboard of frame.document.artboards) {
if (artboard.visible) renderArtboard(rendererContext, artboard, frame.editor.viewport, checkerboardRenderer);
}
imageTextureRenderer.syncAssets(frame.document.assets);
imageTextureRenderer.syncAssets([...frame.document.assets, ...generationCandidatePreviewAssets(frame.editor)]);
renderLayers(rendererContext, frame.document, frame.editor, imageTextureRenderer);
if (!frame.editor.maskEdit) {

View File

@@ -25,7 +25,7 @@ export type AppProps = {
export function App({ app }: AppProps) {
const shellState = useAppState(app.store, selectAppShellState, shallowEqual);
const { document, selection, viewport, tools, transformSession, maskEdit } = shellState;
const { document, selection, viewport, tools, generation, transformSession, maskEdit } = shellState;
const viewportActivityIsland = useViewportActivityIsland(viewport);
const imageImport = useImageImport(app.store);
const [layersOpen, setLayersOpen] = useState(false);
@@ -161,6 +161,7 @@ export function App({ app }: AppProps) {
activeTool={tools.activeTool}
brushSettings={tools.brush}
generateSettings={tools.generate}
generation={generation}
chromaKeySettings={tools.chromaKey}
magicWandSettings={tools.magicWand}
editingMask={Boolean(maskEdit)}
@@ -189,6 +190,7 @@ type AppShellState = {
selection: AppState["editor"]["selection"];
viewport: AppState["editor"]["viewport"];
tools: AppState["editor"]["tools"];
generation: AppState["editor"]["generation"];
transformSession: AppState["editor"]["transformSession"];
maskEdit: AppState["editor"]["maskEdit"];
};
@@ -199,6 +201,7 @@ function selectAppShellState(state: AppState): AppShellState {
selection: state.editor.selection,
viewport: state.editor.viewport,
tools: state.editor.tools,
generation: state.editor.generation,
transformSession: state.editor.transformSession,
maskEdit: state.editor.maskEdit,
};

View File

@@ -1,6 +1,6 @@
import type { AppStore } from "@editor/store";
import type { ImageDocument } from "@core/document";
import type { MaskViewMode, SelectionState, ViewportState } from "@editor/state";
import type { GenerationState, MaskViewMode, SelectionState, ViewportState } from "@editor/state";
import type { BrushSettings, ChromaKeySettings, GenerateSettings, MagicWandSettings, ToolId } from "@editor/tools";
import { BrushControls } from "./bottom-controls/BrushControls";
import { ChromaKeyControls } from "./bottom-controls/ChromaKeyControls";
@@ -22,6 +22,7 @@ export type BottomControlsIslandProps = {
activeTool: ToolId;
brushSettings: BrushSettings;
generateSettings: GenerateSettings;
generation: GenerationState;
chromaKeySettings: ChromaKeySettings;
magicWandSettings: MagicWandSettings;
editingMask?: boolean;
@@ -32,7 +33,7 @@ export type BottomControlsIslandProps = {
dispatch: AppStore["dispatch"];
};
export function BottomControlsIsland({ document, selection, viewport, visible, action, activeTool, brushSettings, generateSettings, chromaKeySettings, magicWandSettings, editingMask = false, maskViewMode = "composite", transformBounds, transformTarget, brushHint, dispatch }: BottomControlsIslandProps) {
export function BottomControlsIsland({ document, selection, viewport, visible, action, activeTool, brushSettings, generateSettings, generation, chromaKeySettings, magicWandSettings, editingMask = false, maskViewMode = "composite", transformBounds, transformTarget, brushHint, dispatch }: BottomControlsIslandProps) {
const zoomPercent = Math.round(viewport.zoom * 100);
const x = Math.round(viewport.center.x);
const y = Math.round(viewport.center.y);
@@ -40,12 +41,12 @@ export function BottomControlsIsland({ document, selection, viewport, visible, a
return (
<div
aria-hidden={!visible}
className={`flex h-20 min-w-96 rounded-full px-4 py-2 items-center justify-center gap-4 text-sm text-white backdrop-blur transition-all duration-200 ${
className={`flex min-h-20 min-w-96 rounded-[2rem] px-4 py-2 items-center justify-center gap-4 text-sm text-white backdrop-blur transition-all duration-200 ${
visible ? "pointer-events-auto translate-y-0 opacity-100" : "pointer-events-none translate-y-3 opacity-0"
}`}
>
{activeTool === "generate" ? (
<GenerateActionControls document={document} selection={selection} viewport={viewport} settings={generateSettings} dispatch={dispatch} />
<GenerateActionControls document={document} selection={selection} viewport={viewport} settings={generateSettings} generation={generation} dispatch={dispatch} />
) : (activeTool === "brush" || activeTool === "eraser") && brushHint ? (
<BrushHint tool={activeTool} hint={brushHint} />
) : activeTool === "brush" || activeTool === "eraser" ? (

View File

@@ -1,6 +1,7 @@
import { useMemo, useRef, useState, type DragEvent, type MutableRefObject } from "react";
import { useEffect, useMemo, useRef, useState, type DragEvent, type MutableRefObject } from "react";
import { ArrowDown, ArrowUp, DownloadSimple, Eye, EyeSlash, FolderPlus, Lock, LockOpen, Plus, Stack, Trash } from "@phosphor-icons/react";
import { commandIds } from "@commands/ids";
import type { Asset } from "@core/asset";
import type { ImageDocument } from "@core/document";
import type { Layer } from "@core/layer";
import type { ArtboardId } from "@core/id";
@@ -9,6 +10,7 @@ import type { MaskEditState, SelectionState } from "@editor/state";
import type { AppStore } from "@editor/store";
import { resolveLayerDrop } from "@input/index";
import { downloadArtboardPng } from "./exportArtboardPng";
import { analyzeMaskSource, applyMaskRasterOperation, type MaskAnalysis, type MaskRasterOperation } from "./mask/maskRaster";
export type LayersSheetProps = {
document: ImageDocument;
@@ -219,6 +221,7 @@ function LayerRow({
const selected = selectedLayerIds.includes(layer.id);
const layerInfo = documentIndex.layerInfoById.get(layer.id);
const maskLayer = layer.clippingMask ? documentIndex.layerById.get(layer.clippingMask.maskLayerId) : undefined;
const maskAsset = maskLayer && maskLayer.type !== "group" ? documentIndex.assetById.get(maskLayer.assetId) : undefined;
const canAddMask = Boolean(layerInfo && layer.type !== "group" && !layer.clippingMask);
const editingMask = Boolean(maskEdit && layer.clippingMask && maskEdit.targetLayerId === layer.id && maskEdit.maskLayerId === layer.clippingMask.maskLayerId);
const rowPadding = 12 + depth * 16;
@@ -288,21 +291,49 @@ function LayerRow({
</button>
</div>
{layer.clippingMask ? (
<div className="mt-2 flex min-h-14 items-center gap-3 rounded-full py-2 pl-4 pr-2 text-sm text-sky-100/70 hover:bg-white/[0.04]">
<Stack size={24} weight="fill" />
<span className="min-w-0 flex-1 truncate">{maskLayer ? "Layer mask" : "Layer mask missing"}</span>
<div className="mt-2 flex min-h-14 flex-wrap items-center gap-2 rounded-[1.5rem] py-2 pl-4 pr-2 text-sm text-sky-100/70 hover:bg-white/[0.04]">
<Stack size={24} weight="fill" className="shrink-0" />
<span className="min-w-28 flex-1 truncate">{maskLayer ? "Layer mask" : "Layer mask missing"}</span>
{maskAsset ? <MaskStatus asset={maskAsset} /> : null}
{maskLayer ? (
<button
type="button"
className={`h-9 rounded-full px-4 text-sm font-medium transition ${editingMask ? "bg-sky-300 text-black" : "text-sky-100/75 hover:bg-white/10 hover:text-sky-50"}`}
onClick={() =>
editingMask
? dispatch(commandIds.toolExitMaskEdit, undefined)
: dispatch(commandIds.toolEnterMaskEdit, { targetLayerId: layer.id, maskLayerId: layer.clippingMask!.maskLayerId })
}
>
{editingMask ? "Done" : "Edit"}
</button>
<div className="flex flex-wrap items-center gap-1">
<button
type="button"
className={`h-9 rounded-full px-4 text-sm font-medium transition ${editingMask ? "bg-sky-300 text-black" : "text-sky-100/75 hover:bg-white/10 hover:text-sky-50"}`}
onClick={() =>
editingMask
? dispatch(commandIds.toolExitMaskEdit, undefined)
: dispatch(commandIds.toolEnterMaskEdit, { targetLayerId: layer.id, maskLayerId: layer.clippingMask!.maskLayerId })
}
>
{editingMask ? "Done" : "Edit"}
</button>
<button
type="button"
className={maskActionButtonClass()}
title="Paint reveal"
onClick={() => {
dispatch(commandIds.toolEnterMaskEdit, { targetLayerId: layer.id, maskLayerId: layer.clippingMask!.maskLayerId });
dispatch(commandIds.toolSetActive, { tool: "brush" });
}}
>
Reveal
</button>
<button
type="button"
className={maskActionButtonClass()}
title="Paint hide"
onClick={() => {
dispatch(commandIds.toolEnterMaskEdit, { targetLayerId: layer.id, maskLayerId: layer.clippingMask!.maskLayerId });
dispatch(commandIds.toolSetActive, { tool: "eraser" });
}}
>
Hide
</button>
{maskAsset && maskLayer.type !== "group" ? (
<MaskOperationButtons maskLayerId={maskLayer.id} maskAsset={maskAsset} dispatch={dispatch} />
) : null}
</div>
) : null}
<button
type="button"
@@ -336,6 +367,83 @@ function LayerRow({
);
}
function MaskStatus({ asset }: { asset: Asset }) {
const [analysis, setAnalysis] = useState<MaskAnalysis>();
useEffect(() => {
let cancelled = false;
void analyzeMaskSource(asset.source, asset.intrinsicSize.w, asset.intrinsicSize.h)
.then((nextAnalysis) => {
if (!cancelled) setAnalysis(nextAnalysis);
})
.catch(() => {
if (!cancelled) setAnalysis(undefined);
});
return () => {
cancelled = true;
};
}, [asset.source, asset.intrinsicSize.w, asset.intrinsicSize.h]);
if (!analysis) return <span className="rounded-full bg-white/5 px-3 py-1 text-xs text-sky-100/45">Reading</span>;
return (
<span className="inline-flex min-w-0 items-center gap-2 rounded-full bg-white/5 px-2 py-1 text-xs text-sky-100/65">
<img src={analysis.thumbnail} alt="" className="h-7 w-10 rounded-md bg-black/30 object-cover ring-1 ring-white/10" />
<span className="whitespace-nowrap">Reveal {formatPercent(analysis.coverage)}</span>
<span className="whitespace-nowrap text-sky-100/45">Inpaint {formatPercent(analysis.hiddenCoverage)}</span>
</span>
);
}
function MaskOperationButtons({ maskLayerId, maskAsset, dispatch }: { maskLayerId: string; maskAsset: Asset; dispatch: AppStore["dispatch"] }) {
return (
<>
<MaskOperationButton label="Invert" title="Invert mask" maskLayerId={maskLayerId} maskAsset={maskAsset} operation={{ type: "invert" }} dispatch={dispatch} />
<MaskOperationButton label="White" title="Fill mask white" maskLayerId={maskLayerId} maskAsset={maskAsset} operation={{ type: "fill", fill: "white" }} dispatch={dispatch} />
<MaskOperationButton label="Black" title="Fill mask black" maskLayerId={maskLayerId} maskAsset={maskAsset} operation={{ type: "fill", fill: "black" }} dispatch={dispatch} />
<MaskOperationButton label="Clear" title="Clear mask" maskLayerId={maskLayerId} maskAsset={maskAsset} operation={{ type: "fill", fill: "clear" }} dispatch={dispatch} />
<MaskOperationButton label="Feather" title="Feather mask edge" maskLayerId={maskLayerId} maskAsset={maskAsset} operation={{ type: "feather", radius: 3 }} dispatch={dispatch} />
<MaskOperationButton label="Expand" title="Expand mask" maskLayerId={maskLayerId} maskAsset={maskAsset} operation={{ type: "expand", radius: 3 }} dispatch={dispatch} />
<MaskOperationButton label="Contract" title="Contract mask" maskLayerId={maskLayerId} maskAsset={maskAsset} operation={{ type: "contract", radius: 3 }} dispatch={dispatch} />
<MaskOperationButton label="Blur" title="Blur mask edge" maskLayerId={maskLayerId} maskAsset={maskAsset} operation={{ type: "blur", radius: 2 }} dispatch={dispatch} />
<MaskOperationButton label="Clean" title="Despeckle mask" maskLayerId={maskLayerId} maskAsset={maskAsset} operation={{ type: "despeckle", strength: 8 }} dispatch={dispatch} />
</>
);
}
function MaskOperationButton({
label,
title,
maskLayerId,
maskAsset,
operation,
dispatch,
}: {
label: string;
title: string;
maskLayerId: string;
maskAsset: Asset;
operation: MaskRasterOperation;
dispatch: AppStore["dispatch"];
}) {
const [busy, setBusy] = useState(false);
return (
<button
type="button"
className={maskActionButtonClass()}
disabled={busy}
title={title}
onClick={() => {
setBusy(true);
void applyMaskRasterOperation(maskAsset.source, maskAsset.intrinsicSize.w, maskAsset.intrinsicSize.h, operation)
.then((source) => dispatch(commandIds.documentApplyLayerMaskOperation, { maskLayerId, source, mimeType: "image/png", operation }))
.finally(() => setBusy(false));
}}
>
{busy ? "..." : label}
</button>
);
}
type EditingTitle =
| { type: "artboard"; id: ArtboardId; draft: string }
| { type: "layer"; id: string; draft: string };
@@ -528,3 +636,11 @@ function toolbarButtonClass() {
function labeledToolbarButtonClass() {
return "inline-flex h-12 flex-1 items-center rounded-full pr-4 text-sm font-medium text-white/75 transition hover:bg-white/10 hover:text-white disabled:pointer-events-none disabled:opacity-35 focus:outline-none focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-white/30";
}
function maskActionButtonClass() {
return "h-8 rounded-full bg-white/5 px-3 text-xs font-medium text-sky-100/65 transition hover:bg-white/10 hover:text-sky-50 disabled:pointer-events-none disabled:opacity-35";
}
function formatPercent(value: number) {
return `${Math.round(Math.max(0, Math.min(1, value)) * 100)}%`;
}

View File

@@ -31,6 +31,13 @@ export function BrushControls({ tool, settings, editingMask = false, maskViewMod
{tool === "eraser" ? <Eraser size={24} weight="regular" /> : <PaintBrush size={24} weight="regular" />}
</span>
<BottomControlDivider />
{editingMask ? (
<>
<button type="button" className={maskModeButtonClass(tool === "brush")} onClick={() => dispatch(commandIds.toolSetActive, { tool: "brush" })}>Reveal</button>
<button type="button" className={maskModeButtonClass(tool === "eraser")} onClick={() => dispatch(commandIds.toolSetActive, { tool: "eraser" })}>Hide</button>
<BottomControlDivider />
</>
) : null}
{tool === "brush" && !editingMask ? (
<label className={bottomControlFieldClass()}>
<span className={bottomControlLabelClass()}>Color</span>
@@ -91,3 +98,7 @@ export function BrushControls({ tool, settings, editingMask = false, maskViewMod
</div>
);
}
function maskModeButtonClass(active: boolean) {
return `rounded-full px-4 py-2 text-base font-medium transition ${active ? "bg-white text-black" : "bg-white/10 text-white hover:bg-white/15"}`;
}

View File

@@ -7,6 +7,7 @@ import { resolveTransformTargetBounds } from "@editor/transform-targets";
import type { AppStore } from "@editor/store";
import type { ChromaKeySettings } from "@editor/tools";
import type { SelectionState } from "@editor/state";
import { blurMaskValues, despeckleMaskValues, dilateMaskValues, erodeMaskValues } from "../mask/maskRaster";
import { BottomControlColorPicker } from "./ColorPicker";
import { BottomControlDivider } from "./Divider";
import { BottomControlSlider } from "./Slider";
@@ -179,8 +180,8 @@ async function applyChromaKeyMask(target: NonNullable<ReturnType<typeof resolveC
const source = await chromaKeyMaskSource(target.asset.source, target.asset.intrinsicSize.w, target.asset.intrinsicSize.h, settings);
dispatch(commandIds.toolSetBrushStrokePreview, undefined);
if (target.maskAsset) {
dispatch(commandIds.documentUpdateAssetSource, { assetId: target.maskAsset.id, source });
if (target.maskAsset && target.maskLayer && target.maskLayer.type !== "group") {
dispatch(commandIds.documentApplyLayerMaskOperation, { maskLayerId: target.maskLayer.id, source, mimeType: "image/png", operation: { type: "chromaKey" } });
return;
}
@@ -293,85 +294,13 @@ function postProcessAlpha(alpha: Uint8ClampedArray, width: number, height: numbe
const choke = Math.round(Math.max(-20, Math.min(20, settings.choke)));
const feather = Math.round(Math.max(0, Math.min(20, settings.feather)));
if (despeckle > 0) next = despeckleAlpha(next, width, height, despeckle);
if (choke > 0) next = erodeAlpha(next, width, height, choke);
if (choke < 0) next = dilateAlpha(next, width, height, -choke);
if (feather > 0) next = blurAlpha(next, width, height, feather);
if (despeckle > 0) next = despeckleMaskValues(next, width, height, despeckle);
if (choke > 0) next = erodeMaskValues(next, width, height, choke);
if (choke < 0) next = dilateMaskValues(next, width, height, -choke);
if (feather > 0) next = blurMaskValues(next, width, height, feather);
return next;
}
function despeckleAlpha(alpha: Uint8ClampedArray, width: number, height: number, strength: number) {
const radius = Math.max(1, Math.ceil(strength / 6));
const threshold = Math.max(1, Math.round(strength / 2));
const next = new Uint8ClampedArray(alpha);
for (let y = 0; y < height; y++) {
for (let x = 0; x < width; x++) {
const index = y * width + x;
const visible = (alpha[index] ?? 0) > 127;
let same = 0;
for (let oy = -radius; oy <= radius; oy++) {
for (let ox = -radius; ox <= radius; ox++) {
if (ox === 0 && oy === 0) continue;
const sample = alpha[clamp(y + oy, 0, height - 1) * width + clamp(x + ox, 0, width - 1)] ?? 0;
if ((sample > 127) === visible) same += 1;
}
}
if (same <= threshold) next[index] = visible ? 0 : 255;
}
}
return next;
}
function erodeAlpha(alpha: Uint8ClampedArray, width: number, height: number, radius: number) {
const next = new Uint8ClampedArray(alpha.length);
for (let y = 0; y < height; y++) {
for (let x = 0; x < width; x++) {
let value = 255;
for (let oy = -radius; oy <= radius; oy++) {
for (let ox = -radius; ox <= radius; ox++) value = Math.min(value, alpha[clamp(y + oy, 0, height - 1) * width + clamp(x + ox, 0, width - 1)] ?? 0);
}
next[y * width + x] = value;
}
}
return next;
}
function dilateAlpha(alpha: Uint8ClampedArray, width: number, height: number, radius: number) {
const next = new Uint8ClampedArray(alpha.length);
for (let y = 0; y < height; y++) {
for (let x = 0; x < width; x++) {
let value = 0;
for (let oy = -radius; oy <= radius; oy++) {
for (let ox = -radius; ox <= radius; ox++) value = Math.max(value, alpha[clamp(y + oy, 0, height - 1) * width + clamp(x + ox, 0, width - 1)] ?? 0);
}
next[y * width + x] = value;
}
}
return next;
}
function blurAlpha(alpha: Uint8ClampedArray, width: number, height: number, radius: number) {
const next = new Uint8ClampedArray(alpha.length);
for (let y = 0; y < height; y++) {
for (let x = 0; x < width; x++) {
let total = 0;
let count = 0;
for (let oy = -radius; oy <= radius; oy++) {
for (let ox = -radius; ox <= radius; ox++) {
total += alpha[clamp(y + oy, 0, height - 1) * width + clamp(x + ox, 0, width - 1)] ?? 0;
count += 1;
}
}
next[y * width + x] = Math.round(total / count);
}
}
return next;
}
function clamp(value: number, min: number, max: number) {
return Math.max(min, Math.min(max, value));
}
function loadImage(source: string) {
return new Promise<HTMLImageElement>((resolve, reject) => {
const image = new Image();

View File

@@ -1,40 +1,271 @@
import { useState } from "react";
import { useEffect, useState } from "react";
import { commandIds } from "@commands/ids";
import type { ImageDocument } from "@core/document";
import type { SelectionState, ViewportState } from "@editor/state";
import type { GenerationCandidate, GenerationState, SelectionState, ViewportState } from "@editor/state";
import type { GenerateSettings } from "@editor/tools";
import type { AppStore } from "@editor/store";
import { runGenerate } from "../generate/runGenerate";
import { createMaskedPixelReplacementSource } from "../generate/candidateActions";
import { runGenerate, runGenerateFromCandidate } from "../generate/runGenerate";
import { createSolidMaskSource } from "../mask/maskRaster";
export type GenerateActionControlsProps = {
document: ImageDocument;
selection: SelectionState;
viewport: ViewportState;
settings: GenerateSettings;
generation: GenerationState;
dispatch: AppStore["dispatch"];
};
export function GenerateActionControls({ document, selection, viewport, settings, dispatch }: GenerateActionControlsProps) {
const [busy, setBusy] = useState(false);
export function GenerateActionControls({ document, selection, viewport, settings, generation, dispatch }: GenerateActionControlsProps) {
const [busy, setBusy] = useState<string>();
const [error, setError] = useState<string>();
const [elapsedSeconds, setElapsedSeconds] = useState(0);
const candidate = selectedCandidate(generation);
const canGenerate = Boolean(settings.prompt.trim()) && !busy;
useEffect(() => {
if (!busy) {
setElapsedSeconds(0);
return;
}
setElapsedSeconds(0);
const startedAt = Date.now();
const interval = window.setInterval(() => {
setElapsedSeconds(Math.floor((Date.now() - startedAt) / 1000));
}, 1000);
return () => window.clearInterval(interval);
}, [busy]);
return (
<div className="flex items-center px-2">
<div className="flex max-w-[calc(100vw-2rem)] flex-wrap items-center justify-center gap-2 px-2">
<button
type="button"
disabled={!canGenerate}
className="h-12 rounded-full bg-white px-7 text-base font-semibold !text-black transition hover:bg-white/90 focus:outline-none focus-visible:ring-2 focus-visible:ring-white/40 disabled:pointer-events-none disabled:opacity-35"
title={error ?? "Generate with ComfyUI"}
onClick={() => {
setBusy(true);
setBusy("Generating");
setError(undefined);
void runGenerate({ document, selection, viewport, settings, dispatch })
.catch((reason: unknown) => setError(reason instanceof Error ? reason.message : "Generation failed"))
.finally(() => setBusy(false));
.finally(() => setBusy(undefined));
}}
>
{busy ? "Generating" : "Generate"}
{busy === "Generating" ? `Generating ${formatElapsed(elapsedSeconds)}` : "Generate"}
</button>
{busy ? <span className="rounded-full bg-white/10 px-3 py-2 text-xs font-medium text-white/60">{busy} {formatElapsed(elapsedSeconds)}</span> : null}
{candidate ? (
<>
<CandidatePicker generation={generation} dispatch={dispatch} />
<CandidateControls
document={document}
candidate={candidate}
settings={settings}
busy={busy}
setBusy={setBusy}
setError={setError}
dispatch={dispatch}
/>
</>
) : null}
{error ? <span className="max-w-80 truncate rounded-full bg-red-500/15 px-3 py-2 text-xs font-medium text-red-100" title={error}>{error}</span> : null}
</div>
);
}
function CandidatePicker({ generation, dispatch }: { generation: GenerationState; dispatch: AppStore["dispatch"] }) {
if (generation.candidates.length < 2) return null;
return (
<div className="flex items-center gap-1 rounded-full bg-white/[0.04] px-1.5 py-1 ring-1 ring-white/[0.05]">
{generation.candidates.slice(0, 6).map((candidate) => {
const selected = candidate.id === (generation.selectedCandidateId ?? generation.candidates[0]?.id);
return (
<button
key={candidate.id}
type="button"
className={`h-9 w-9 overflow-hidden rounded-full ring-2 transition ${selected ? "ring-white" : "ring-white/10 hover:ring-white/40"}`}
title={`Candidate seed ${candidate.seed}`}
onClick={() => dispatch(commandIds.generationSelectCandidate, { candidateId: candidate.id })}
>
<img src={candidate.source} alt="" className="h-full w-full object-cover" />
</button>
);
})}
</div>
);
}
function CandidateControls({
document,
candidate,
settings,
busy,
setBusy,
setError,
dispatch,
}: {
document: ImageDocument;
candidate: GenerationCandidate;
settings: GenerateSettings;
busy?: string;
setBusy: (busy: string | undefined) => void;
setError: (error: string | undefined) => void;
dispatch: AppStore["dispatch"];
}) {
const rerun = (label: string, nextSettings: GenerateSettings) => {
setBusy(label);
setError(undefined);
dispatch(commandIds.toolSetGenerateSettings, nextSettings);
void runGenerateFromCandidate({ candidate, settings: nextSettings, dispatch })
.catch((reason: unknown) => setError(reason instanceof Error ? reason.message : `${label} failed`))
.finally(() => setBusy(undefined));
};
const disabled = Boolean(busy);
return (
<div className="flex flex-wrap items-center justify-center gap-1 rounded-full bg-white/[0.04] px-2 py-1 ring-1 ring-white/[0.05]">
<CandidatePreview candidate={candidate} />
<span className="px-2 text-xs font-medium text-white/55">Seed {candidate.seed}</span>
<CandidateButton disabled={disabled} label="Regenerate" title="Regenerate same mask and crop" busy={busy === "Regenerate"} onClick={() => rerun("Regenerate", candidate.settings)} />
<CandidateButton
disabled={disabled}
label="Lower"
title="Lower strength and regenerate same mask"
busy={busy === "Lower"}
onClick={() => rerun("Lower", { ...candidate.settings, strength: Math.max(0, candidate.settings.strength - 10), seed: candidate.seed })}
/>
<CandidateButton disabled={disabled} label="Reuse seed" title="Regenerate with the same seed" busy={busy === "Reuse seed"} onClick={() => rerun("Reuse seed", { ...candidate.settings, seed: candidate.seed })} />
<CandidateButton disabled={disabled} label="New seed" title="Regenerate with a new seed" busy={busy === "New seed"} onClick={() => rerun("New seed", { ...candidate.settings, seed: -1 })} />
<CandidateButton disabled={disabled} label="Apply layer" title="Apply candidate as a normal layer" onClick={() => applyCandidateAsLayer(candidate, false, dispatch)} />
<CandidateButton
disabled={disabled}
label="Apply + refine"
title="Apply candidate as a layer with a fresh refinement mask"
busy={busy === "Refine"}
onClick={() => {
setBusy("Refine");
setError(undefined);
void applyCandidateAsRefinementLayer(candidate, dispatch)
.catch((reason: unknown) => setError(reason instanceof Error ? reason.message : "Refine setup failed"))
.finally(() => setBusy(undefined));
}}
/>
<CandidateButton disabled={disabled} label="Stack variant" title="Stack candidate as another variant layer" onClick={() => applyCandidateAsLayer(candidate, true, dispatch)} />
<CandidateButton
disabled={disabled || !candidate.inpaint}
label="Replace"
title={candidate.inpaint ? "Replace masked pixels and preserve unmasked pixels" : "Only inpaint candidates can replace masked pixels"}
busy={busy === "Replace"}
onClick={() => {
setBusy("Replace");
setError(undefined);
void createMaskedPixelReplacementSource(document, candidate)
.then((source) => dispatch(commandIds.generationReplaceCandidatePixels, { candidateId: candidate.id, source, mimeType: "image/png" }))
.catch((reason: unknown) => setError(reason instanceof Error ? reason.message : "Replace failed"))
.finally(() => setBusy(undefined));
}}
/>
<CandidateButton
disabled={disabled || !candidate.inpaint}
label="Edit mask"
title={candidate.inpaint ? "Paint refinement mask over the source layer" : "Only inpaint candidates have a mask to edit"}
onClick={() => {
if (!candidate.inpaint) return;
dispatch(commandIds.selectionSet, { artboardId: candidate.placement.artboardId, layerIds: [candidate.inpaint.targetLayerId] });
dispatch(commandIds.toolEnterMaskEdit, { targetLayerId: candidate.inpaint.targetLayerId, maskLayerId: candidate.inpaint.maskLayerId });
}}
/>
<CandidateButton disabled={disabled} label="Dismiss" title="Remove this candidate" onClick={() => dispatch(commandIds.generationRemoveCandidate, { candidateId: candidate.id })} />
{settings.seed !== candidate.seed ? null : <span className="sr-only">Current settings reuse this seed</span>}
</div>
);
}
function CandidatePreview({ candidate }: { candidate: GenerationCandidate }) {
if (!candidate.inputImage) {
return <img src={candidate.source} alt="" className="h-10 w-10 rounded-full bg-black/25 object-cover ring-1 ring-white/10" />;
}
return (
<span className="flex items-center -space-x-2" title={candidate.maskImage ? "Input crop, normalized mask, generated candidate" : "Before and generated candidate"}>
<img src={candidate.inputImage} alt="" className="h-10 w-10 rounded-full bg-black/25 object-cover ring-1 ring-white/10" />
{candidate.maskImage ? <img src={candidate.maskImage} alt="" className="h-10 w-10 rounded-full bg-black/25 object-cover ring-1 ring-white/20" /> : null}
<img src={candidate.source} alt="" className="h-10 w-10 rounded-full bg-black/25 object-cover ring-2 ring-white/40" />
</span>
);
}
function CandidateButton({ label, title, disabled, busy, onClick }: { label: string; title: string; disabled?: boolean; busy?: boolean; onClick: () => void }) {
return (
<button
type="button"
className="h-9 rounded-full bg-white/5 px-3 text-xs font-semibold text-white/70 transition hover:bg-white/10 hover:text-white disabled:pointer-events-none disabled:opacity-35"
disabled={disabled}
title={title}
onClick={onClick}
>
{busy ? "..." : label}
</button>
);
}
function selectedCandidate(generation: GenerationState): GenerationCandidate | undefined {
return generation.candidates.find((candidate) => candidate.id === generation.selectedCandidateId) ?? generation.candidates[0];
}
function applyCandidateAsLayer(candidate: GenerationCandidate, variant: boolean, dispatch: AppStore["dispatch"]) {
applyCandidateAsLayerWithIds(candidate, { layerId: crypto.randomUUID(), assetId: crypto.randomUUID() }, variant, dispatch);
}
async function applyCandidateAsRefinementLayer(candidate: GenerationCandidate, dispatch: AppStore["dispatch"]) {
const layerId = crypto.randomUUID();
const maskLayerId = crypto.randomUUID();
const maskAssetId = crypto.randomUUID();
applyCandidateAsLayerWithIds(candidate, { layerId, assetId: crypto.randomUUID() }, true, dispatch);
const width = Math.max(1, Math.round(candidate.intrinsicSize.w));
const height = Math.max(1, Math.round(candidate.intrinsicSize.h));
const source = await createSolidMaskSource(width, height, "white");
dispatch(commandIds.documentAddLayerMask, {
layerId,
asset: {
id: maskAssetId,
name: `${candidate.placement.layerName} refinement mask`,
mimeType: "image/png",
source,
intrinsicSize: { w: width, h: height },
},
maskLayer: {
id: maskLayerId,
type: "raster",
name: `${candidate.placement.layerName} refinement mask`,
visible: true,
locked: false,
opacity: 1,
assetId: maskAssetId,
transform: {
position: { ...candidate.placement.transform.position },
scale: { ...candidate.placement.transform.scale },
rotation: candidate.placement.transform.rotation,
},
},
});
dispatch(commandIds.toolSetActive, { tool: "eraser" });
}
function applyCandidateAsLayerWithIds(candidate: GenerationCandidate, ids: { layerId: string; assetId: string }, variant: boolean, dispatch: AppStore["dispatch"]) {
dispatch(commandIds.generationApplyCandidateAsLayer, {
candidateId: candidate.id,
assetId: ids.assetId,
layerId: ids.layerId,
variant,
});
}
function formatElapsed(seconds: number) {
const minutes = Math.floor(seconds / 60);
const remainder = seconds % 60;
return `${minutes}:${remainder.toString().padStart(2, "0")}`;
}

View File

@@ -21,6 +21,18 @@ const sizePresets = [
{ label: "9:16", w: 768, h: 1344 },
] as const;
const inpaintPolarityOptions = [
{ value: "hidden", label: "Hidden / erased" },
{ value: "revealed", label: "Revealed / painted" },
] satisfies readonly BottomControlSelectOption<GenerateSettings["inpaint"]["maskPolarity"]>[];
const inpaintMaskedContentOptions = [
{ value: "neutral", label: "Neutral fill" },
{ value: "original", label: "Original gray" },
{ value: "originalColor", label: "Original color" },
{ value: "edges", label: "Edge map" },
] satisfies readonly BottomControlSelectOption<GenerateSettings["inpaint"]["maskedContent"]>[];
export type GenerateControlsProps = {
settings: GenerateSettings;
dispatch: AppStore["dispatch"];
@@ -32,6 +44,7 @@ export function GenerateControls({ settings, dispatch }: GenerateControlsProps)
const [schedulers, setSchedulers] = useState<readonly BottomControlSelectOption<string>[]>([{ value: settings.scheduler, label: settings.scheduler }]);
const [advancedOpen, setAdvancedOpen] = useState(false);
const [outpaintOpen, setOutpaintOpen] = useState(false);
const [inpaintOpen, setInpaintOpen] = useState(false);
const [sizeOpen, setSizeOpen] = useState(false);
const sizeRef = useRef<HTMLDivElement>(null);
const [error, setError] = useState<string>();
@@ -141,6 +154,44 @@ export function GenerateControls({ settings, dispatch }: GenerateControlsProps)
<PanelNumber label="Feather" aria-label="Outpaint feathering" value={settings.outpaint.feathering} onValueChange={(feathering) => dispatch(commandIds.toolSetGenerateSettings, { outpaint: { ...settings.outpaint, feathering } })} />
</div>
</section>
<section className={panelSectionClass()}>
<button type="button" className={sectionToggleClass()} aria-expanded={inpaintOpen} aria-controls="generate-inpaint-controls" onClick={() => setInpaintOpen((open) => !open)}>
<span>
<span className="block text-sm font-semibold text-white/85">Inpaint</span>
<span className="block text-xs text-white/40">Mask polarity, crop padding, edge prep, grow</span>
</span>
{inpaintOpen ? <CaretUp size={18} weight="bold" /> : <CaretDown size={18} weight="bold" />}
</button>
<div id="generate-inpaint-controls" className={`grid gap-2 overflow-hidden transition-all duration-200 ${inpaintOpen ? "max-h-[38rem] pt-2 opacity-100" : "max-h-0 opacity-0"}`}>
<PanelSelect
label="Polarity"
value={settings.inpaint.maskPolarity}
options={inpaintPolarityOptions}
ariaLabel="Inpaint mask polarity"
onValueChange={(maskPolarity) => dispatch(commandIds.toolSetGenerateSettings, { inpaint: { ...settings.inpaint, maskPolarity } })}
/>
<PanelSelect
label="Content"
value={settings.inpaint.maskedContent}
options={inpaintMaskedContentOptions}
ariaLabel="Inpaint masked content"
onValueChange={(maskedContent) => dispatch(commandIds.toolSetGenerateSettings, { inpaint: { ...settings.inpaint, maskedContent } })}
/>
<button type="button" className={compactRowButtonClass()} aria-pressed={settings.inpaint.maskedAreaOnly} onClick={() => dispatch(commandIds.toolSetGenerateSettings, { inpaint: { ...settings.inpaint, maskedAreaOnly: !settings.inpaint.maskedAreaOnly } })}>
<span className={panelLabelClass()}>Frame</span>
<span className="min-w-0 flex-1 text-right text-white">{settings.inpaint.maskedAreaOnly ? "Mask crop" : "Full layer"}</span>
</button>
<div className="grid grid-cols-2 gap-2">
<PanelNumber label="Pad" aria-label="Inpaint crop padding" value={settings.inpaint.cropPadding} max={2048} onValueChange={(cropPadding) => dispatch(commandIds.toolSetGenerateSettings, { inpaint: { ...settings.inpaint, cropPadding } })} />
<PanelNumber label="Grow" aria-label="Inpaint backend grow mask" value={settings.inpaint.growMaskBy} max={256} onValueChange={(growMaskBy) => dispatch(commandIds.toolSetGenerateSettings, { inpaint: { ...settings.inpaint, growMaskBy } })} />
<PanelNumber label="Expand" aria-label="Inpaint mask expand" value={settings.inpaint.maskExpand} min={-256} max={256} onValueChange={(maskExpand) => dispatch(commandIds.toolSetGenerateSettings, { inpaint: { ...settings.inpaint, maskExpand } })} />
<PanelNumber label="Feather" aria-label="Inpaint mask feather" value={settings.inpaint.maskFeather} max={256} onValueChange={(maskFeather) => dispatch(commandIds.toolSetGenerateSettings, { inpaint: { ...settings.inpaint, maskFeather } })} />
<PanelNumber label="Blur" aria-label="Inpaint mask blur" value={settings.inpaint.maskBlur} max={256} onValueChange={(maskBlur) => dispatch(commandIds.toolSetGenerateSettings, { inpaint: { ...settings.inpaint, maskBlur } })} />
<PanelNumber label="Clean" aria-label="Inpaint mask despeckle" value={settings.inpaint.maskDespeckle} max={64} onValueChange={(maskDespeckle) => dispatch(commandIds.toolSetGenerateSettings, { inpaint: { ...settings.inpaint, maskDespeckle } })} />
</div>
</div>
</section>
</div>
);
}

View File

@@ -138,7 +138,15 @@ export async function commitBrushSession(options: { store: AppStore; session: Br
if (options.session.cancelled) return;
const source = options.session.changed ? canvasToDataUrl(options.session.canvas) : undefined;
if (source) options.store.dispatch(commandIds.documentUpdateAssetSource, { assetId: options.session.assetId, source });
if (source) {
const state = options.store.getState();
const maskEdit = state.editor.maskEdit;
if (maskEdit?.maskLayerId === options.session.layerId) {
options.store.dispatch(commandIds.documentApplyLayerMaskOperation, { maskLayerId: maskEdit.maskLayerId, source, mimeType: "image/png", operation: { type: "paint" } });
} else {
options.store.dispatch(commandIds.documentUpdateAssetSource, { assetId: options.session.assetId, source });
}
}
options.store.dispatch(commandIds.toolSetBrushStrokePreview, undefined);
closeBrushStrokePreview(options.session);
}

View File

@@ -5,6 +5,7 @@ import type { Layer } from "@core/layer";
import { resolveTransformTargetBounds } from "@editor/transform-targets";
import type { AppStore } from "@editor/store";
import type { EditorState } from "@editor/state";
import { blurMaskValues, despeckleMaskValues, dilateMaskValues, erodeMaskValues, maskValueFromRgba } from "../mask/maskRaster";
export async function applyMagicWandAt(store: AppStore, point: Vec2D, modeOverride?: EditorState["tools"]["magicWand"]["mode"]) {
const state = store.getState();
@@ -15,8 +16,8 @@ export async function applyMagicWandAt(store: AppStore, point: Vec2D, modeOverri
const y = Math.floor((point.y - target.layer.transform.position.y) / Math.max(0.0001, target.layer.transform.scale.y));
if (x < 0 || y < 0 || x >= target.asset.intrinsicSize.w || y >= target.asset.intrinsicSize.h) return true;
const source = await createWandMask(target.asset.source, target.maskAsset?.source, Math.round(target.asset.intrinsicSize.w), Math.round(target.asset.intrinsicSize.h), x, y, { ...state.editor.tools.magicWand, mode: modeOverride ?? state.editor.tools.magicWand.mode });
if (target.maskAsset) {
store.dispatch(commandIds.documentUpdateAssetSource, { assetId: target.maskAsset.id, source });
if (target.maskAsset && target.maskLayer && target.maskLayer.type !== "group") {
store.dispatch(commandIds.documentApplyLayerMaskOperation, { maskLayerId: target.maskLayer.id, source, mimeType: "image/png", operation: { type: "magicWand" } });
return true;
}
const assetId = crypto.randomUUID();
@@ -42,7 +43,7 @@ function resolveTarget(document: ImageDocument, editor: EditorState) {
const bounds = resolveTransformTargetBounds(document, { type: "layer", id: layer.id });
const maskLayer = layer.clippingMask ? findLayer(document.artboards.flatMap((artboard) => artboard.layers), layer.clippingMask.maskLayerId) : undefined;
const maskAsset = maskLayer && maskLayer.type !== "group" ? document.assets.find((candidate) => candidate.id === maskLayer.assetId) : undefined;
return asset && bounds ? { layer, asset, bounds, maskAsset } : undefined;
return asset && bounds ? { layer, asset, bounds, maskLayer, maskAsset } : undefined;
}
async function createWandMask(source: string, existingMaskSource: string | undefined, width: number, height: number, startX: number, startY: number, settings: EditorState["tools"]["magicWand"]) {
@@ -57,10 +58,15 @@ async function createWandMask(source: string, existingMaskSource: string | undef
const start = (startY * canvas.width + startX) * 4;
const key = [imageData.data[start] ?? 0, imageData.data[start + 1] ?? 0, imageData.data[start + 2] ?? 0];
const selected = postProcessSelection(settings.contiguous ? floodSelect(imageData, canvas.width, canvas.height, startX, startY, key, settings.tolerance) : globalSelect(imageData, key, settings.tolerance), canvas.width, canvas.height, settings);
const existingAlpha = existingMaskSource ? await loadMaskAlpha(existingMaskSource, canvas.width, canvas.height) : undefined;
const existingMask = existingMaskSource ? await loadMaskValues(existingMaskSource, canvas.width, canvas.height) : undefined;
for (let pixel = 0; pixel < selected.length; pixel++) {
const current = existingAlpha?.[pixel] ?? 255;
const value = settings.mode === "add" ? (selected[pixel] ? 0 : current) : settings.mode === "subtract" ? (selected[pixel] ? 255 : current) : selected[pixel] ? 0 : 255;
const current = existingMask?.[pixel] ?? 255;
const selectionValue = selected[pixel] ?? 0;
const value = settings.mode === "add"
? Math.min(current, 255 - selectionValue)
: settings.mode === "subtract"
? Math.max(current, selectionValue)
: 255 - selectionValue;
const index = pixel * 4;
imageData.data[index] = 255;
imageData.data[index + 1] = 255;
@@ -98,67 +104,24 @@ function matches(data: ImageData, pixel: number, key: number[], tolerance: numbe
}
function postProcessSelection(selected: Uint8Array, width: number, height: number, settings: EditorState["tools"]["magicWand"]) {
let next = selected;
let next = selectionToMaskValues(selected);
const despeckle = Math.round(Math.max(0, Math.min(20, settings.despeckle)));
const choke = Math.round(Math.max(-20, Math.min(20, settings.choke)));
const feather = Math.round(Math.max(0, Math.min(20, settings.feather)));
if (despeckle > 0) next = despeckleSelection(next, width, height, despeckle);
if (choke > 0) next = erodeSelection(next, width, height, choke);
if (choke < 0) next = dilateSelection(next, width, height, -choke);
if (feather > 0) next = featherSelection(next, width, height, feather);
if (despeckle > 0) next = despeckleMaskValues(next, width, height, despeckle);
if (choke > 0) next = erodeMaskValues(next, width, height, choke);
if (choke < 0) next = dilateMaskValues(next, width, height, -choke);
if (feather > 0) next = blurMaskValues(next, width, height, feather);
return next;
}
function erodeSelection(selected: Uint8Array, width: number, height: number, radius: number) {
const next = new Uint8Array(selected.length);
for (let y = 0; y < height; y++) for (let x = 0; x < width; x++) {
let value = 1;
for (let oy = -radius; oy <= radius; oy++) for (let ox = -radius; ox <= radius; ox++) value = Math.min(value, selected[clamp(y + oy, 0, height - 1) * width + clamp(x + ox, 0, width - 1)] ?? 0);
next[y * width + x] = value;
}
return next;
function selectionToMaskValues(selected: Uint8Array) {
const values = new Uint8ClampedArray(selected.length);
for (let index = 0; index < selected.length; index += 1) values[index] = selected[index] ? 255 : 0;
return values;
}
function dilateSelection(selected: Uint8Array, width: number, height: number, radius: number) {
const next = new Uint8Array(selected.length);
for (let y = 0; y < height; y++) for (let x = 0; x < width; x++) {
let value = 0;
for (let oy = -radius; oy <= radius; oy++) for (let ox = -radius; ox <= radius; ox++) value = Math.max(value, selected[clamp(y + oy, 0, height - 1) * width + clamp(x + ox, 0, width - 1)] ?? 0);
next[y * width + x] = value;
}
return next;
}
function featherSelection(selected: Uint8Array, width: number, height: number, radius: number) {
const next = new Uint8Array(selected.length);
for (let y = 0; y < height; y++) for (let x = 0; x < width; x++) {
let total = 0;
let count = 0;
for (let oy = -radius; oy <= radius; oy++) for (let ox = -radius; ox <= radius; ox++) {
total += selected[clamp(y + oy, 0, height - 1) * width + clamp(x + ox, 0, width - 1)] ?? 0;
count += 1;
}
next[y * width + x] = Math.round(total / count);
}
return next;
}
function despeckleSelection(selected: Uint8Array, width: number, height: number, strength: number) {
const radius = Math.max(1, Math.ceil(strength / 6));
const threshold = Math.max(1, Math.round(strength / 2));
const next = new Uint8Array(selected);
for (let y = 0; y < height; y++) for (let x = 0; x < width; x++) {
const index = y * width + x;
let same = 0;
for (let oy = -radius; oy <= radius; oy++) for (let ox = -radius; ox <= radius; ox++) if (ox !== 0 || oy !== 0) {
if ((selected[clamp(y + oy, 0, height - 1) * width + clamp(x + ox, 0, width - 1)] ?? 0) === selected[index]) same += 1;
}
if (same <= threshold) next[index] = selected[index] ? 0 : 1;
}
return next;
}
async function loadMaskAlpha(source: string, width: number, height: number) {
async function loadMaskValues(source: string, width: number, height: number) {
const canvas = document.createElement("canvas");
canvas.width = width;
canvas.height = height;
@@ -167,13 +130,9 @@ async function loadMaskAlpha(source: string, width: number, height: number) {
const image = await loadImage(source);
context.drawImage(image, 0, 0, width, height);
const data = context.getImageData(0, 0, width, height);
const alpha = new Uint8ClampedArray(width * height);
for (let pixel = 0; pixel < alpha.length; pixel++) alpha[pixel] = data.data[pixel * 4 + 3] ?? 255;
return alpha;
}
function clamp(value: number, min: number, max: number) {
return Math.max(min, Math.min(max, value));
const values = new Uint8ClampedArray(width * height);
for (let pixel = 0; pixel < values.length; pixel++) values[pixel] = maskValueFromRgba(data.data, pixel * 4);
return values;
}
function findLayer(layers: readonly Layer[], layerId: string): Layer | undefined {

View File

@@ -151,6 +151,36 @@ const visualEditorChanges: Array<[string, (state: AppState) => AppState]> = [
},
}),
],
[
"generation candidate preview",
(state) => ({
...state,
editor: {
...state.editor,
generation: {
candidates: [
{
id: "candidate",
source: "data:image/png;base64,candidate",
mimeType: "image/png",
intrinsicSize: { w: 64, h: 64 },
mode: "text-to-image",
settings: state.editor.tools.generate,
seed: 12,
width: 64,
height: 64,
placement: {
artboardId: "artboard",
layerName: "Candidate",
transform: { position: { x: 1, y: 2 }, scale: { x: 1, y: 1 }, rotation: 0 },
},
},
],
selectedCandidateId: "candidate",
},
},
}),
],
[
"active tool",
(state) => ({

View File

@@ -1,6 +1,6 @@
import type { Rect, Vec2D } from "@core/geometry";
import type { RenderFrame } from "@renderer/index";
import type { AppState, BrushPreviewState, BrushStrokePreviewState, EditorState, MaskEditState, SelectionState, ViewportState } from "@editor/state";
import type { AppState, BrushPreviewState, BrushStrokePreviewState, EditorState, GenerationState, MaskEditState, SelectionState, ViewportState } from "@editor/state";
import type { BrushSettings, InteractionMode } from "@editor/tools";
import type { TransformSession, TransformTarget } from "@editor/transform";
@@ -23,6 +23,7 @@ function visualEditorStatesEqual(a: EditorState, b: EditorState): boolean {
maskEditStatesEqual(a.maskEdit, b.maskEdit) &&
brushPreviewStatesEqual(a.brushPreview, b.brushPreview) &&
brushStrokePreviewStatesEqual(a.brushStrokePreview, b.brushStrokePreview) &&
generationStatesEqual(a.generation, b.generation) &&
visualToolStatesEqual(a.tools, b.tools)
);
}
@@ -64,6 +65,10 @@ function brushStrokePreviewStatesEqual(a: BrushStrokePreviewState | undefined, b
return a.layerId === b.layerId && a.assetId === b.assetId && a.source === b.source;
}
function generationStatesEqual(a: GenerationState, b: GenerationState): boolean {
return a === b;
}
function visualToolStatesEqual(a: EditorState["tools"], b: EditorState["tools"]): boolean {
return a.activeTool === b.activeTool && interactionModesEqual(a.interactionMode, b.interactionMode) && brushSettingsEqual(a.brush, b.brush);
}

View File

@@ -0,0 +1,50 @@
import type { ImageDocument } from "@core/document";
import type { GenerationCandidate } from "@editor/state";
import { loadImageCanvas, maskValueFromRgba } from "../mask/maskRaster";
export async function createMaskedPixelReplacementSource(document: ImageDocument, candidate: GenerationCandidate): Promise<string> {
if (!candidate.inpaint) throw new Error("Only inpaint candidates can replace masked pixels.");
const targetAsset = document.assets.find((asset) => asset.id === candidate.inpaint?.sourceAssetId);
if (!targetAsset) throw new Error("The source layer for this candidate no longer exists.");
const targetCanvas = await loadImageCanvas(targetAsset.source, targetAsset.intrinsicSize.w, targetAsset.intrinsicSize.h);
const generatedCanvas = await loadImageCanvas(candidate.source, candidate.width, candidate.height);
const maskCanvas = await loadImageCanvas(candidate.inpaint.maskImage, candidate.width, candidate.height);
const targetContext = require2dContext(targetCanvas);
const generatedContext = require2dContext(generatedCanvas);
const maskContext = require2dContext(maskCanvas);
const targetData = targetContext.getImageData(0, 0, targetCanvas.width, targetCanvas.height);
const generatedData = generatedContext.getImageData(0, 0, generatedCanvas.width, generatedCanvas.height);
const maskData = maskContext.getImageData(0, 0, maskCanvas.width, maskCanvas.height);
const crop = candidate.inpaint.crop.assetBounds;
for (let y = 0; y < candidate.height; y += 1) {
for (let x = 0; x < candidate.width; x += 1) {
const targetX = Math.round(crop.x) + x;
const targetY = Math.round(crop.y) + y;
if (targetX < 0 || targetY < 0 || targetX >= targetCanvas.width || targetY >= targetCanvas.height) continue;
const generatedIndex = (y * generatedCanvas.width + x) * 4;
const targetIndex = (targetY * targetCanvas.width + targetX) * 4;
const mask = maskValueFromRgba(maskData.data, generatedIndex) / 255;
if (mask <= 0) continue;
for (let channel = 0; channel < 4; channel += 1) {
const previous = targetData.data[targetIndex + channel] ?? 0;
const next = generatedData.data[generatedIndex + channel] ?? previous;
targetData.data[targetIndex + channel] = Math.round(previous * (1 - mask) + next * mask);
}
}
}
targetContext.putImageData(targetData, 0, 0);
return targetCanvas.toDataURL("image/png");
}
function require2dContext(canvas: HTMLCanvasElement): CanvasRenderingContext2D {
const context = canvas.getContext("2d");
if (!context) throw new Error("Unable to prepare generated candidate");
return context;
}

View File

@@ -0,0 +1,36 @@
import { describe, expect, test } from "bun:test";
import { applyMaskedContentModeToRgba } from "./inpaintPrep";
describe("inpaint prep", () => {
test("converts only masked original pixels to grayscale", () => {
const pixels = new Uint8ClampedArray([
255, 0, 0, 255,
0, 0, 255, 255,
]);
const mask = new Uint8ClampedArray([255, 0]);
const next = applyMaskedContentModeToRgba(pixels, 2, 1, mask, "original");
expect(next[0]).toBe(next[1]);
expect(next[1]).toBe(next[2]);
expect(Array.from(next.slice(4, 8))).toEqual([0, 0, 255, 255]);
});
test("uses an edge map inside the mask without recoloring unmasked context", () => {
const pixels = new Uint8ClampedArray([
255, 0, 0, 255,
0, 255, 0, 255,
0, 0, 255, 255,
255, 255, 0, 255,
]);
const mask = new Uint8ClampedArray([255, 0, 0, 0]);
const next = applyMaskedContentModeToRgba(pixels, 2, 2, mask, "edges");
expect(next[0]).toBe(next[1]);
expect(next[1]).toBe(next[2]);
expect(Array.from(next.slice(4, 8))).toEqual([0, 255, 0, 255]);
expect(Array.from(next.slice(8, 12))).toEqual([0, 0, 255, 255]);
expect(Array.from(next.slice(12, 16))).toEqual([255, 255, 0, 255]);
});
});

View File

@@ -0,0 +1,258 @@
import type { Asset } from "@core/asset";
import type { ImageDocument } from "@core/document";
import type { Rect } from "@core/geometry";
import type { Layer } from "@core/layer";
import type { SelectionState } from "@editor/state";
import type { GenerateSettings } from "@editor/tools";
import { createDocumentReadIndex, resolveIndexedLayerBounds } from "@editor/document-indexes";
import { createNormalizedMaskSource, cropCanvas, cropMaskValuesToDataUrl, expandRectWithinBounds, loadImageCanvas } from "../mask/maskRaster";
export type InpaintBundle = {
inputImage: string;
maskImage: string;
width: number;
height: number;
targetLayerId: string;
maskLayerId: string;
sourceAssetId: string;
maskAssetId: string;
crop: {
assetBounds: Rect;
documentBounds: Rect;
padding: number;
maskedAreaOnly: boolean;
};
mask: {
polarity: GenerateSettings["inpaint"]["maskPolarity"];
activeBounds: Rect;
};
placement: {
artboardId: string;
layerName: string;
transform: Extract<Layer, { type: "image" | "raster" }>["transform"];
};
backend: {
growMaskBy: number;
maskedContent: GenerateSettings["inpaint"]["maskedContent"];
maskBlur: number;
maskFeather: number;
maskExpand: number;
cropPadding: number;
};
};
type InpaintTarget = {
artboardId: string;
layer: Extract<Layer, { type: "image" | "raster" }>;
asset: Asset;
bounds: Rect;
maskLayer: Extract<Layer, { type: "image" | "raster" }>;
maskAsset: Asset;
maskBounds: Rect;
};
const modelMultiple = 8;
const minModelSize = 64;
const maxModelSize = 4096;
export async function buildInpaintBundle(document: ImageDocument, selection: SelectionState, settings: GenerateSettings): Promise<InpaintBundle> {
const target = resolveInpaintTarget(document, selection);
validateInpaintTarget(target);
const width = Math.max(1, Math.round(target.asset.intrinsicSize.w));
const height = Math.max(1, Math.round(target.asset.intrinsicSize.h));
const normalizedMask = await createNormalizedMaskSource(target.maskAsset.source, width, height, {
polarity: settings.inpaint.maskPolarity,
expand: settings.inpaint.maskExpand,
feather: settings.inpaint.maskFeather,
blur: settings.inpaint.maskBlur,
despeckle: settings.inpaint.maskDespeckle,
});
if (!normalizedMask.bounds) throw new Error("The selected layer mask has no inpaint pixels.");
const crop = settings.inpaint.maskedAreaOnly
? expandRectWithinBounds(normalizedMask.bounds, settings.inpaint.cropPadding, { w: width, h: height }, modelMultiple, minModelSize)
: { x: 0, y: 0, w: width, h: height };
const outputWidth = toModelSize(crop.w, "width");
const outputHeight = toModelSize(crop.h, "height");
const inputCanvas = prepareMaskedContentInputCanvas(await loadImageCanvas(target.asset.source, width, height), normalizedMask.values, settings.inpaint.maskedContent);
const inputImage = cropCanvas(inputCanvas, crop, outputWidth, outputHeight);
const maskImage = cropMaskValuesToDataUrl(normalizedMask.values, width, height, crop, outputWidth, outputHeight);
const scaleX = target.bounds.w / width;
const scaleY = target.bounds.h / height;
const documentBounds = {
x: target.bounds.x + crop.x * scaleX,
y: target.bounds.y + crop.y * scaleY,
w: outputWidth * scaleX,
h: outputHeight * scaleY,
};
return {
inputImage,
maskImage,
width: outputWidth,
height: outputHeight,
targetLayerId: target.layer.id,
maskLayerId: target.maskLayer.id,
sourceAssetId: target.asset.id,
maskAssetId: target.maskAsset.id,
crop: {
assetBounds: crop,
documentBounds,
padding: settings.inpaint.cropPadding,
maskedAreaOnly: settings.inpaint.maskedAreaOnly,
},
mask: {
polarity: settings.inpaint.maskPolarity,
activeBounds: normalizedMask.bounds,
},
placement: {
artboardId: target.artboardId,
layerName: `${target.layer.name} inpaint`,
transform: {
position: { x: documentBounds.x, y: documentBounds.y },
scale: { x: documentBounds.w / outputWidth, y: documentBounds.h / outputHeight },
rotation: target.layer.transform.rotation,
},
},
backend: {
growMaskBy: settings.inpaint.growMaskBy,
maskedContent: settings.inpaint.maskedContent,
maskBlur: settings.inpaint.maskBlur,
maskFeather: settings.inpaint.maskFeather,
maskExpand: settings.inpaint.maskExpand,
cropPadding: settings.inpaint.cropPadding,
},
};
}
function resolveInpaintTarget(document: ImageDocument, selection: SelectionState): InpaintTarget {
if (selection.layerIds.length !== 1 || !selection.layerIds[0]) throw new Error("Select one image or raster layer to inpaint.");
const documentIndex = createDocumentReadIndex(document);
const layerInfo = documentIndex.layerInfoById.get(selection.layerIds[0]);
if (!layerInfo || layerInfo.layer.type === "group") throw new Error("Select one image or raster layer to inpaint.");
const asset = documentIndex.assetById.get(layerInfo.layer.assetId);
if (!asset) throw new Error("The selected layer is missing its source image.");
if (!layerInfo.layer.clippingMask) throw new Error("Add a layer mask before running inpaint.");
const maskLayer = documentIndex.layerById.get(layerInfo.layer.clippingMask.maskLayerId);
if (!maskLayer || maskLayer.type === "group") throw new Error("The selected layer mask is missing.");
const maskAsset = documentIndex.assetById.get(maskLayer.assetId);
if (!maskAsset) throw new Error("The selected layer mask is missing its image data.");
const bounds = resolveIndexedLayerBounds(documentIndex, layerInfo.layer);
const maskBounds = resolveIndexedLayerBounds(documentIndex, maskLayer);
if (!bounds || !maskBounds) throw new Error("Unable to resolve the selected layer and mask bounds.");
return { artboardId: layerInfo.artboardId, layer: layerInfo.layer, asset, bounds, maskLayer, maskAsset, maskBounds };
}
function validateInpaintTarget(target: InpaintTarget) {
if (target.asset.intrinsicSize.w <= 0 || target.asset.intrinsicSize.h <= 0) throw new Error("The selected image has an invalid size.");
if (target.maskAsset.intrinsicSize.w <= 0 || target.maskAsset.intrinsicSize.h <= 0) throw new Error("The selected mask has an invalid size.");
if (Math.round(target.asset.intrinsicSize.w) !== Math.round(target.maskAsset.intrinsicSize.w) || Math.round(target.asset.intrinsicSize.h) !== Math.round(target.maskAsset.intrinsicSize.h)) {
throw new Error("The selected layer and mask image sizes do not match.");
}
if (!rectsAligned(target.bounds, target.maskBounds) || Math.abs(target.layer.transform.rotation - target.maskLayer.transform.rotation) > 0.001) {
throw new Error("The selected layer and mask are not aligned.");
}
}
function rectsAligned(a: Rect, b: Rect): boolean {
return Math.abs(a.x - b.x) <= 0.5 && Math.abs(a.y - b.y) <= 0.5 && Math.abs(a.w - b.w) <= 0.5 && Math.abs(a.h - b.h) <= 0.5;
}
function toModelSize(value: number, axis: "width" | "height"): number {
const rounded = Math.max(minModelSize, Math.ceil(value / modelMultiple) * modelMultiple);
if (rounded > maxModelSize) throw new Error(`The inpaint ${axis} is too large for the model. Use masked-area inpaint or a smaller source.`);
return rounded;
}
function prepareMaskedContentInputCanvas(sourceCanvas: HTMLCanvasElement, maskValues: Uint8ClampedArray, mode: GenerateSettings["inpaint"]["maskedContent"]): HTMLCanvasElement {
if (mode === "neutral" || mode === "originalColor") return sourceCanvas;
const context = sourceCanvas.getContext("2d");
if (!context) return sourceCanvas;
const imageData = context.getImageData(0, 0, sourceCanvas.width, sourceCanvas.height);
imageData.data.set(applyMaskedContentModeToRgba(imageData.data, sourceCanvas.width, sourceCanvas.height, maskValues, mode));
context.putImageData(imageData, 0, 0);
return sourceCanvas;
}
export function applyMaskedContentModeToRgba(data: Uint8ClampedArray, width: number, height: number, maskValues: Uint8ClampedArray, mode: GenerateSettings["inpaint"]["maskedContent"]): Uint8ClampedArray {
const next = new Uint8ClampedArray(data);
if (mode === "neutral" || mode === "originalColor") return next;
const edgeValues = mode === "edges" ? createEdgeMapValues(data, width, height) : undefined;
for (let pixel = 0; pixel < width * height; pixel += 1) {
const amount = (maskValues[pixel] ?? 0) / 255;
if (amount <= 0) continue;
const index = pixel * 4;
const red = data[index] ?? 0;
const green = data[index + 1] ?? 0;
const blue = data[index + 2] ?? 0;
const target = edgeValues ? edgeValues[pixel] ?? 128 : luminance(red, green, blue);
next[index] = blendChannel(red, target, amount);
next[index + 1] = blendChannel(green, target, amount);
next[index + 2] = blendChannel(blue, target, amount);
}
return next;
}
function createEdgeMapValues(data: Uint8ClampedArray, width: number, height: number): Uint8ClampedArray {
const gray = new Float32Array(width * height);
const edges = new Uint8ClampedArray(width * height);
for (let pixel = 0; pixel < width * height; pixel += 1) {
const index = pixel * 4;
gray[pixel] = luminance(data[index] ?? 0, data[index + 1] ?? 0, data[index + 2] ?? 0);
}
for (let y = 0; y < height; y += 1) {
for (let x = 0; x < width; x += 1) {
const gx =
-sampleGray(gray, width, height, x - 1, y - 1) +
sampleGray(gray, width, height, x + 1, y - 1) -
2 * sampleGray(gray, width, height, x - 1, y) +
2 * sampleGray(gray, width, height, x + 1, y) -
sampleGray(gray, width, height, x - 1, y + 1) +
sampleGray(gray, width, height, x + 1, y + 1);
const gy =
-sampleGray(gray, width, height, x - 1, y - 1) -
2 * sampleGray(gray, width, height, x, y - 1) -
sampleGray(gray, width, height, x + 1, y - 1) +
sampleGray(gray, width, height, x - 1, y + 1) +
2 * sampleGray(gray, width, height, x, y + 1) +
sampleGray(gray, width, height, x + 1, y + 1);
const magnitude = Math.hypot(gx, gy);
edges[y * width + x] = Math.round(clampNumber(128 + Math.max(0, magnitude - 24) * 0.75, 128, 255));
}
}
return edges;
}
function sampleGray(values: Float32Array, width: number, height: number, x: number, y: number): number {
const clampedX = Math.max(0, Math.min(width - 1, x));
const clampedY = Math.max(0, Math.min(height - 1, y));
return values[clampedY * width + clampedX] ?? 0;
}
function luminance(red: number, green: number, blue: number): number {
return Math.round(0.2126 * red + 0.7152 * green + 0.0722 * blue);
}
function blendChannel(previous: number, next: number, amount: number): number {
return Math.round(previous * (1 - amount) + next * amount);
}
function clampNumber(value: number, min: number, max: number): number {
return Math.max(min, Math.min(max, value));
}

View File

@@ -1,9 +1,11 @@
import { commandIds } from "@commands/ids";
import type { ImageDocument } from "@core/document";
import type { Transform } from "@core/geometry";
import type { Layer } from "@core/layer";
import type { AppStore } from "@editor/store";
import type { SelectionState, ViewportState } from "@editor/state";
import type { GenerationCandidate, SelectionState, ViewportState } from "@editor/state";
import type { GenerateSettings } from "@editor/tools";
import { buildInpaintBundle, type InpaintBundle } from "./inpaintPrep";
export async function runGenerate(options: {
document: ImageDocument;
@@ -17,50 +19,200 @@ export async function runGenerate(options: {
if (!artboard) return;
const target = resolveSelectedImage(document, selection);
const inputImage = target && settings.mode !== "text-to-image" ? await imageSourceToDataUrl(target.asset.source) : undefined;
const maskImage = target?.maskAsset && settings.mode === "inpaint" ? await imageSourceToDataUrl(target.maskAsset.source) : undefined;
const inpaintBundle = settings.mode === "inpaint" ? await buildInpaintBundle(document, selection, settings) : undefined;
const inputImage = inpaintBundle?.inputImage ?? (target && settings.mode !== "text-to-image" ? await imageSourceToDataUrl(target.asset.source) : undefined);
const maskImage = inpaintBundle?.maskImage;
const seed = resolveSeed(settings.seed);
const requestSettings = { ...settings, seed };
const width = inpaintBundle?.width ?? settings.width;
const height = inpaintBundle?.height ?? settings.height;
const generated = await requestGenerate({
settings: requestSettings,
width,
height,
inputImage,
maskImage,
inpaintBundle,
});
const intrinsicSize = await loadImageSize(generated.source);
const targetArtboardId = inpaintBundle?.placement.artboardId ?? artboard.id;
const placement = {
artboardId: targetArtboardId,
layerName: inpaintBundle?.placement.layerName ?? "Generated image",
transform: generatedLayerTransform(inpaintBundle, intrinsicSize),
};
dispatch(commandIds.generationAddCandidate, {
candidate: createGenerationCandidate({
source: generated.source,
mimeType: generated.mimeType,
intrinsicSize,
settings: requestSettings,
seed,
width,
height,
inputImage,
maskImage,
placement,
inpaintBundle,
}),
});
}
export async function runGenerateFromCandidate(options: {
candidate: GenerationCandidate;
settings?: GenerateSettings;
dispatch: AppStore["dispatch"];
}) {
const settings = options.settings ?? options.candidate.settings;
const seed = resolveSeed(settings.seed);
const requestSettings = { ...settings, seed };
const generated = await requestGenerate({
settings: requestSettings,
width: options.candidate.width,
height: options.candidate.height,
inputImage: options.candidate.inputImage,
maskImage: options.candidate.maskImage,
inpaintCandidate: options.candidate,
});
const intrinsicSize = await loadImageSize(generated.source);
options.dispatch(commandIds.generationAddCandidate, {
candidate: {
...options.candidate,
id: crypto.randomUUID(),
source: generated.source,
mimeType: generated.mimeType,
intrinsicSize,
settings: requestSettings,
seed,
},
});
}
function createGenerationCandidate(options: {
source: string;
mimeType: string;
intrinsicSize: { w: number; h: number };
settings: GenerateSettings;
seed: number;
width: number;
height: number;
inputImage?: string;
maskImage?: string;
placement: GenerationCandidate["placement"];
inpaintBundle?: InpaintBundle;
}): GenerationCandidate {
return {
id: crypto.randomUUID(),
source: options.source,
mimeType: options.mimeType,
intrinsicSize: options.intrinsicSize,
mode: options.settings.mode,
settings: options.settings,
seed: options.seed,
width: options.width,
height: options.height,
inputImage: options.inputImage,
maskImage: options.maskImage,
placement: options.placement,
inpaint: options.inpaintBundle
? {
targetLayerId: options.inpaintBundle.targetLayerId,
maskLayerId: options.inpaintBundle.maskLayerId,
sourceAssetId: options.inpaintBundle.sourceAssetId,
maskAssetId: options.inpaintBundle.maskAssetId,
inputImage: options.inpaintBundle.inputImage,
maskImage: options.inpaintBundle.maskImage,
crop: options.inpaintBundle.crop,
mask: options.inpaintBundle.mask,
backend: options.inpaintBundle.backend,
}
: undefined,
};
}
async function requestGenerate(options: {
settings: GenerateSettings;
width: number;
height: number;
inputImage?: string;
maskImage?: string;
inpaintBundle?: InpaintBundle;
inpaintCandidate?: GenerationCandidate;
}) {
const response = await fetch("/api/comfy/generate", {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({
mode: settings.mode,
model: settings.model,
prompt: settings.prompt,
negativePrompt: settings.negativePrompt,
strength: settings.strength,
steps: settings.steps,
cfg: settings.cfg,
seed: settings.seed,
sampler: settings.sampler,
scheduler: settings.scheduler,
width: settings.width,
height: settings.height,
outpaint: settings.outpaint,
inputImage,
maskImage,
mode: options.settings.mode,
model: options.settings.model,
prompt: options.settings.prompt,
negativePrompt: options.settings.negativePrompt,
strength: options.settings.strength,
steps: options.settings.steps,
cfg: options.settings.cfg,
seed: options.settings.seed,
sampler: options.settings.sampler,
scheduler: options.settings.scheduler,
width: options.width,
height: options.height,
outpaint: options.settings.outpaint,
inpaint: resolveInpaintRequest(options.inpaintBundle, options.inpaintCandidate, options.settings),
inputImage: options.inputImage,
maskImage: options.maskImage,
}),
});
if (!response.ok) throw new Error(await response.text());
const generated = await response.json() as { source: string; mimeType: string };
const intrinsicSize = await loadImageSize(generated.source);
const assetId = crypto.randomUUID();
const layerId = crypto.randomUUID();
dispatch(commandIds.documentAddAsset, { asset: { id: assetId, name: "Generated image", mimeType: generated.mimeType, source: generated.source, intrinsicSize } });
dispatch(commandIds.documentAddImageLayer, {
artboardId: artboard.id,
layer: {
id: layerId,
type: "image",
name: "Generated image",
visible: true,
locked: false,
opacity: 1,
assetId,
transform: { position: { x: 0, y: 0 }, scale: { x: 1, y: 1 }, rotation: 0 },
return await response.json() as { source: string; mimeType: string };
}
function resolveInpaintRequest(inpaintBundle: InpaintBundle | undefined, inpaintCandidate: GenerationCandidate | undefined, settings: GenerateSettings) {
if (inpaintBundle) {
return {
growMaskBy: inpaintBundle.backend.growMaskBy,
maskedContent: inpaintBundle.backend.maskedContent,
maskBlur: inpaintBundle.backend.maskBlur,
maskFeather: inpaintBundle.backend.maskFeather,
maskExpand: inpaintBundle.backend.maskExpand,
cropPadding: inpaintBundle.backend.cropPadding,
maskPolarity: inpaintBundle.mask.polarity,
crop: inpaintBundle.crop,
placement: inpaintBundle.placement,
};
}
if (inpaintCandidate?.inpaint) {
return {
growMaskBy: inpaintCandidate.inpaint.backend.growMaskBy,
maskedContent: inpaintCandidate.inpaint.backend.maskedContent,
maskBlur: inpaintCandidate.inpaint.backend.maskBlur,
maskFeather: inpaintCandidate.inpaint.backend.maskFeather,
maskExpand: inpaintCandidate.inpaint.backend.maskExpand,
cropPadding: inpaintCandidate.inpaint.backend.cropPadding,
maskPolarity: inpaintCandidate.inpaint.mask.polarity,
crop: inpaintCandidate.inpaint.crop,
placement: inpaintCandidate.placement,
};
}
return settings.inpaint;
}
function generatedLayerTransform(inpaintBundle: InpaintBundle | undefined, intrinsicSize: { w: number; h: number }): Transform {
if (!inpaintBundle) return { position: { x: 0, y: 0 }, scale: { x: 1, y: 1 }, rotation: 0 };
return {
position: { ...inpaintBundle.placement.transform.position },
scale: {
x: (inpaintBundle.placement.transform.scale.x * inpaintBundle.width) / Math.max(1, intrinsicSize.w),
y: (inpaintBundle.placement.transform.scale.y * inpaintBundle.height) / Math.max(1, intrinsicSize.h),
},
});
dispatch(commandIds.documentMoveLayer, { layerId, toArtboardId: artboard.id, toIndex: 0 });
dispatch(commandIds.selectionSet, { artboardId: artboard.id, layerIds: [layerId] });
rotation: inpaintBundle.placement.transform.rotation,
};
}
function resolveSeed(seed: number): number {
return seed < 0 ? Math.floor(Math.random() * 2 ** 32) : Math.round(seed);
}
function resolveSelectedImage(document: ImageDocument, selection: SelectionState) {

View File

@@ -0,0 +1,40 @@
import { describe, expect, test } from "bun:test";
import { cropMaskValuesToRgba, expandRectWithinBounds, invertMaskValues } from "./maskRaster";
describe("mask raster utilities", () => {
test("exports the normalized drawn mask without filling the whole crop", () => {
const revealedMask = new Uint8ClampedArray(4 * 3).fill(255);
revealedMask[1 * 4 + 2] = 0;
const inpaintMask = invertMaskValues(revealedMask);
const rgba = cropMaskValuesToRgba(inpaintMask, 4, 3, { x: 1, y: 0, w: 3, h: 3 }, 4, 4);
const activePixels = activeRedPixels(rgba);
expect(activePixels).toEqual([{ x: 1, y: 1 }]);
for (let pixel = 0; pixel < rgba.length / 4; pixel += 1) expect(rgba[pixel * 4 + 3]).toBe(255);
});
test("expands masked-area crops to the model minimum when possible", () => {
expect(expandRectWithinBounds({ x: 20, y: 20, w: 4, h: 4 }, 4, { w: 100, h: 100 }, 8, 64)).toEqual({
x: 0,
y: 0,
w: 64,
h: 64,
});
expect(expandRectWithinBounds({ x: 80, y: 80, w: 4, h: 4 }, 4, { w: 100, h: 100 }, 8, 64)).toEqual({
x: 36,
y: 36,
w: 64,
h: 64,
});
});
});
function activeRedPixels(rgba: Uint8ClampedArray) {
const width = 4;
const pixels: Array<{ x: number; y: number }> = [];
for (let pixel = 0; pixel < rgba.length / 4; pixel += 1) {
if ((rgba[pixel * 4] ?? 0) > 127) pixels.push({ x: pixel % width, y: Math.floor(pixel / width) });
}
return pixels;
}

374
view/mask/maskRaster.ts Normal file
View File

@@ -0,0 +1,374 @@
import type { Rect } from "@core/geometry";
export type MaskFill = "white" | "black" | "clear";
export type MaskRasterOperation =
| { type: "invert" }
| { type: "fill"; fill: MaskFill }
| { type: "feather"; radius: number }
| { type: "expand"; radius: number }
| { type: "contract"; radius: number }
| { type: "blur"; radius: number }
| { type: "despeckle"; strength: number };
export type MaskAnalysis = {
width: number;
height: number;
revealedPixels: number;
hiddenPixels: number;
coverage: number;
hiddenCoverage: number;
bounds?: Rect;
hiddenBounds?: Rect;
thumbnail: string;
};
export type NormalizedMaskOptions = {
polarity: "hidden" | "revealed";
expand?: number;
feather?: number;
blur?: number;
despeckle?: number;
};
export async function createSolidMaskSource(width: number, height: number, fill: MaskFill): Promise<string> {
const canvas = createCanvas(width, height);
const context = require2dContext(canvas);
context.clearRect(0, 0, canvas.width, canvas.height);
if (fill === "white") {
context.fillStyle = "#ffffff";
context.fillRect(0, 0, canvas.width, canvas.height);
} else if (fill === "black") {
context.fillStyle = "#000000";
context.fillRect(0, 0, canvas.width, canvas.height);
}
return canvas.toDataURL("image/png");
}
export async function applyMaskRasterOperation(source: string, width: number, height: number, operation: MaskRasterOperation): Promise<string> {
if (operation.type === "fill") return createSolidMaskSource(width, height, operation.fill);
const mask = await loadMaskValues(source, width, height);
const next = applyMaskValueOperation(mask.values, mask.width, mask.height, operation);
return maskValuesToDataUrl(next, mask.width, mask.height);
}
export async function analyzeMaskSource(source: string, width: number, height: number): Promise<MaskAnalysis> {
const mask = await loadMaskValues(source, width, height);
const revealed = analyzeValues(mask.values, mask.width, mask.height, false);
const hiddenValues = invertMaskValues(mask.values);
const hidden = analyzeValues(hiddenValues, mask.width, mask.height, false);
return {
width: mask.width,
height: mask.height,
revealedPixels: revealed.pixels,
hiddenPixels: hidden.pixels,
coverage: revealed.coverage,
hiddenCoverage: hidden.coverage,
bounds: revealed.bounds,
hiddenBounds: hidden.bounds,
thumbnail: maskValuesToDataUrl(mask.values, mask.width, mask.height, 72, 48),
};
}
export async function createNormalizedMaskSource(source: string, width: number, height: number, options: NormalizedMaskOptions): Promise<{ source: string; values: Uint8ClampedArray; bounds?: Rect }> {
const mask = await loadMaskValues(source, width, height);
let values = options.polarity === "hidden" ? invertMaskValues(mask.values) : new Uint8ClampedArray(mask.values);
const despeckle = Math.round(clampNumber(options.despeckle ?? 0, 0, 64));
const expand = Math.round(clampNumber(options.expand ?? 0, -256, 256));
const feather = Math.round(clampNumber(options.feather ?? 0, 0, 256));
const blur = Math.round(clampNumber(options.blur ?? 0, 0, 256));
if (despeckle > 0) values = despeckleMaskValues(values, mask.width, mask.height, despeckle);
if (expand > 0) values = dilateMaskValues(values, mask.width, mask.height, expand);
if (expand < 0) values = erodeMaskValues(values, mask.width, mask.height, -expand);
if (feather > 0) values = blurMaskValues(values, mask.width, mask.height, feather);
if (blur > 0) values = blurMaskValues(values, mask.width, mask.height, blur);
const analysis = analyzeValues(values, mask.width, mask.height, false);
return { source: maskValuesToDataUrl(values, mask.width, mask.height), values, bounds: analysis.bounds };
}
export async function loadImageCanvas(source: string, width?: number, height?: number): Promise<HTMLCanvasElement> {
const image = await loadImage(source);
const canvas = createCanvas(width ?? image.naturalWidth, height ?? image.naturalHeight);
const context = require2dContext(canvas);
context.clearRect(0, 0, canvas.width, canvas.height);
context.drawImage(image, 0, 0, canvas.width, canvas.height);
return canvas;
}
export async function imageSourceToPngDataUrl(source: string): Promise<string> {
if (source.startsWith("data:image/png;base64,")) return source;
const canvas = await loadImageCanvas(source);
return canvas.toDataURL("image/png");
}
export function cropCanvas(sourceCanvas: HTMLCanvasElement, crop: Rect, outputWidth = crop.w, outputHeight = crop.h): string {
const canvas = createCanvas(outputWidth, outputHeight);
const context = require2dContext(canvas);
context.clearRect(0, 0, outputWidth, outputHeight);
context.drawImage(sourceCanvas, crop.x, crop.y, crop.w, crop.h, 0, 0, crop.w, crop.h);
return canvas.toDataURL("image/png");
}
export function cropMaskValuesToDataUrl(values: Uint8ClampedArray, width: number, height: number, crop: Rect, outputWidth = crop.w, outputHeight = crop.h): string {
const canvas = createCanvas(outputWidth, outputHeight);
const context = require2dContext(canvas);
const imageData = context.createImageData(outputWidth, outputHeight);
imageData.data.set(cropMaskValuesToRgba(values, width, height, crop, outputWidth, outputHeight));
context.putImageData(imageData, 0, 0);
return canvas.toDataURL("image/png");
}
export function cropMaskValuesToRgba(values: Uint8ClampedArray, width: number, height: number, crop: Rect, outputWidth = crop.w, outputHeight = crop.h): Uint8ClampedArray {
const safeOutputWidth = Math.max(1, Math.round(outputWidth));
const safeOutputHeight = Math.max(1, Math.round(outputHeight));
const data = new Uint8ClampedArray(safeOutputWidth * safeOutputHeight * 4);
for (let pixel = 0; pixel < safeOutputWidth * safeOutputHeight; pixel += 1) data[pixel * 4 + 3] = 255;
for (let y = 0; y < Math.min(crop.h, safeOutputHeight); y += 1) {
for (let x = 0; x < Math.min(crop.w, safeOutputWidth); x += 1) {
const sourceX = crop.x + x;
const sourceY = crop.y + y;
if (sourceX < 0 || sourceY < 0 || sourceX >= width || sourceY >= height) continue;
const value = values[sourceY * width + sourceX] ?? 0;
const index = (y * safeOutputWidth + x) * 4;
data[index] = value;
data[index + 1] = value;
data[index + 2] = value;
}
}
return data;
}
export function expandRectWithinBounds(rect: Rect, padding: number, bounds: { w: number; h: number }, multiple = 1, minSize = 1): Rect {
const padded = Math.max(0, Math.round(padding));
let x1 = Math.max(0, Math.floor(rect.x) - padded);
let y1 = Math.max(0, Math.floor(rect.y) - padded);
let x2 = Math.min(bounds.w, Math.ceil(rect.x + rect.w) + padded);
let y2 = Math.min(bounds.h, Math.ceil(rect.y + rect.h) + padded);
const safeMinSize = Math.max(1, Math.round(minSize));
const targetWidth = Math.min(bounds.w, roundUp(Math.max(safeMinSize, x2 - x1), multiple));
const targetHeight = Math.min(bounds.h, roundUp(Math.max(safeMinSize, y2 - y1), multiple));
const extraWidth = targetWidth - (x2 - x1);
const extraHeight = targetHeight - (y2 - y1);
x1 = Math.max(0, x1 - Math.floor(extraWidth / 2));
y1 = Math.max(0, y1 - Math.floor(extraHeight / 2));
x2 = Math.min(bounds.w, x1 + targetWidth);
y2 = Math.min(bounds.h, y1 + targetHeight);
x1 = Math.max(0, x2 - targetWidth);
y1 = Math.max(0, y2 - targetHeight);
return { x: x1, y: y1, w: Math.max(1, x2 - x1), h: Math.max(1, y2 - y1) };
}
export function maskValueFromRgba(data: Uint8ClampedArray, index: number): number {
const red = data[index] ?? 0;
const green = data[index + 1] ?? 0;
const blue = data[index + 2] ?? 0;
const alpha = data[index + 3] ?? 0;
const luminance = 0.2126 * red + 0.7152 * green + 0.0722 * blue;
return Math.round((alpha * luminance) / 255);
}
export function applyMaskValueOperation(values: Uint8ClampedArray, width: number, height: number, operation: Exclude<MaskRasterOperation, { type: "fill" }>): Uint8ClampedArray {
switch (operation.type) {
case "invert":
return invertMaskValues(values);
case "feather":
case "blur":
return blurMaskValues(values, width, height, Math.round(clampNumber(operation.radius, 0, 256)));
case "expand":
return dilateMaskValues(values, width, height, Math.round(clampNumber(operation.radius, 0, 256)));
case "contract":
return erodeMaskValues(values, width, height, Math.round(clampNumber(operation.radius, 0, 256)));
case "despeckle":
return despeckleMaskValues(values, width, height, Math.round(clampNumber(operation.strength, 0, 64)));
}
}
export function invertMaskValues(values: Uint8ClampedArray): Uint8ClampedArray {
const next = new Uint8ClampedArray(values.length);
for (let index = 0; index < values.length; index += 1) next[index] = 255 - (values[index] ?? 0);
return next;
}
export function erodeMaskValues(values: Uint8ClampedArray, width: number, height: number, radius: number): Uint8ClampedArray {
const safeRadius = Math.round(clampNumber(radius, 0, 256));
if (safeRadius <= 0) return new Uint8ClampedArray(values);
const next = new Uint8ClampedArray(values.length);
for (let y = 0; y < height; y += 1) {
for (let x = 0; x < width; x += 1) {
let value = 255;
for (let oy = -safeRadius; oy <= safeRadius; oy += 1) {
for (let ox = -safeRadius; ox <= safeRadius; ox += 1) value = Math.min(value, values[clampInt(y + oy, 0, height - 1) * width + clampInt(x + ox, 0, width - 1)] ?? 0);
}
next[y * width + x] = value;
}
}
return next;
}
export function dilateMaskValues(values: Uint8ClampedArray, width: number, height: number, radius: number): Uint8ClampedArray {
const safeRadius = Math.round(clampNumber(radius, 0, 256));
if (safeRadius <= 0) return new Uint8ClampedArray(values);
const next = new Uint8ClampedArray(values.length);
for (let y = 0; y < height; y += 1) {
for (let x = 0; x < width; x += 1) {
let value = 0;
for (let oy = -safeRadius; oy <= safeRadius; oy += 1) {
for (let ox = -safeRadius; ox <= safeRadius; ox += 1) value = Math.max(value, values[clampInt(y + oy, 0, height - 1) * width + clampInt(x + ox, 0, width - 1)] ?? 0);
}
next[y * width + x] = value;
}
}
return next;
}
export function blurMaskValues(values: Uint8ClampedArray, width: number, height: number, radius: number): Uint8ClampedArray {
const safeRadius = Math.round(clampNumber(radius, 0, 256));
if (safeRadius <= 0) return new Uint8ClampedArray(values);
const next = new Uint8ClampedArray(values.length);
for (let y = 0; y < height; y += 1) {
for (let x = 0; x < width; x += 1) {
let total = 0;
let count = 0;
for (let oy = -safeRadius; oy <= safeRadius; oy += 1) {
for (let ox = -safeRadius; ox <= safeRadius; ox += 1) {
total += values[clampInt(y + oy, 0, height - 1) * width + clampInt(x + ox, 0, width - 1)] ?? 0;
count += 1;
}
}
next[y * width + x] = Math.round(total / count);
}
}
return next;
}
export function despeckleMaskValues(values: Uint8ClampedArray, width: number, height: number, strength: number): Uint8ClampedArray {
const safeStrength = Math.round(clampNumber(strength, 0, 64));
if (safeStrength <= 0) return new Uint8ClampedArray(values);
const radius = Math.max(1, Math.ceil(safeStrength / 6));
const threshold = Math.max(1, Math.round(safeStrength / 2));
const next = new Uint8ClampedArray(values);
for (let y = 0; y < height; y += 1) {
for (let x = 0; x < width; x += 1) {
const index = y * width + x;
const visible = (values[index] ?? 0) > 127;
let same = 0;
for (let oy = -radius; oy <= radius; oy += 1) {
for (let ox = -radius; ox <= radius; ox += 1) {
if (ox === 0 && oy === 0) continue;
const sample = values[clampInt(y + oy, 0, height - 1) * width + clampInt(x + ox, 0, width - 1)] ?? 0;
if ((sample > 127) === visible) same += 1;
}
}
if (same <= threshold) next[index] = visible ? 0 : 255;
}
}
return next;
}
async function loadMaskValues(source: string, width: number, height: number): Promise<{ width: number; height: number; values: Uint8ClampedArray }> {
const canvas = await loadImageCanvas(source, width, height);
const context = require2dContext(canvas);
const data = context.getImageData(0, 0, canvas.width, canvas.height);
const values = new Uint8ClampedArray(canvas.width * canvas.height);
for (let pixel = 0; pixel < values.length; pixel += 1) values[pixel] = maskValueFromRgba(data.data, pixel * 4);
return { width: canvas.width, height: canvas.height, values };
}
function maskValuesToDataUrl(values: Uint8ClampedArray, width: number, height: number, outputWidth = width, outputHeight = height): string {
const canvas = createCanvas(outputWidth, outputHeight);
const context = require2dContext(canvas);
const imageData = context.createImageData(outputWidth, outputHeight);
for (let y = 0; y < outputHeight; y += 1) {
for (let x = 0; x < outputWidth; x += 1) {
const sourceX = Math.floor((x / outputWidth) * width);
const sourceY = Math.floor((y / outputHeight) * height);
const value = values[clampInt(sourceY, 0, height - 1) * width + clampInt(sourceX, 0, width - 1)] ?? 0;
const index = (y * outputWidth + x) * 4;
imageData.data[index] = 255;
imageData.data[index + 1] = 255;
imageData.data[index + 2] = 255;
imageData.data[index + 3] = value;
}
}
context.putImageData(imageData, 0, 0);
return canvas.toDataURL("image/png");
}
function analyzeValues(values: Uint8ClampedArray, width: number, height: number, includeSoftPixels: boolean): { pixels: number; coverage: number; bounds?: Rect } {
let pixels = 0;
let minX = Number.POSITIVE_INFINITY;
let minY = Number.POSITIVE_INFINITY;
let maxX = Number.NEGATIVE_INFINITY;
let maxY = Number.NEGATIVE_INFINITY;
for (let y = 0; y < height; y += 1) {
for (let x = 0; x < width; x += 1) {
const value = values[y * width + x] ?? 0;
const active = includeSoftPixels ? value > 0 : value > 127;
if (!active) continue;
pixels += 1;
minX = Math.min(minX, x);
minY = Math.min(minY, y);
maxX = Math.max(maxX, x + 1);
maxY = Math.max(maxY, y + 1);
}
}
return {
pixels,
coverage: pixels / Math.max(1, width * height),
bounds: pixels > 0 ? { x: minX, y: minY, w: maxX - minX, h: maxY - minY } : undefined,
};
}
function createCanvas(width: number, height: number): HTMLCanvasElement {
const canvas = document.createElement("canvas");
canvas.width = Math.max(1, Math.round(width));
canvas.height = Math.max(1, Math.round(height));
return canvas;
}
function require2dContext(canvas: HTMLCanvasElement): CanvasRenderingContext2D {
const context = canvas.getContext("2d");
if (!context) throw new Error("Unable to create mask canvas");
return context;
}
function loadImage(source: string): Promise<HTMLImageElement> {
return new Promise((resolve, reject) => {
const image = new Image();
image.onload = () => resolve(image);
image.onerror = () => reject(new Error("Failed to load image"));
image.src = source;
});
}
function roundUp(value: number, multiple: number): number {
const safeMultiple = Math.max(1, Math.round(multiple));
return Math.ceil(value / safeMultiple) * safeMultiple;
}
function clampNumber(value: number, min: number, max: number): number {
if (!Number.isFinite(value)) return min;
return Math.max(min, Math.min(max, value));
}
function clampInt(value: number, min: number, max: number): number {
return Math.round(Math.max(min, Math.min(max, value)));
}