From f5c610dac5fcd58f47f17fb7a3595347184fb804 Mon Sep 17 00:00:00 2001 From: syntaxbullet Date: Sun, 5 Jul 2026 09:35:16 +0200 Subject: [PATCH] feat: implement inpainting functionality with mask handling - Added `createMaskedPixelReplacementSource` function to handle pixel replacement using inpainting. - Introduced `buildInpaintBundle` to prepare inpainting data including mask generation and validation. - Created utility functions for mask operations such as `applyMaskedContentModeToRgba`, `expandRectWithinBounds`, and others for mask manipulation. - Developed tests for inpainting preparation and mask raster utilities to ensure functionality and correctness. - Implemented mask raster operations including inversion, feathering, blurring, and more. --- app/app.ts | 3 +- app/comfy.test.ts | 59 +++ app/comfy.ts | 122 +++++- commands/document.test.ts | 25 ++ commands/document.ts | 64 +++ commands/generation.test.ts | 143 +++++++ commands/generation.ts | 217 ++++++++++ commands/ids.ts | 7 + commands/index.ts | 19 + commands/payloads.ts | 15 + commands/tool.test.ts | 14 +- commands/tool.ts | 11 + editor/initial-state.ts | 4 + editor/state.ts | 55 ++- editor/tools.ts | 29 +- renderer/image-textures.ts | 10 +- renderer/layers.ts | 48 ++- renderer/renderer.ts | 4 +- view/App.tsx | 5 +- view/BottomControlsIsland.tsx | 9 +- view/LayersSheet.tsx | 146 ++++++- view/bottom-controls/BrushControls.tsx | 11 + view/bottom-controls/ChromaKeyControls.tsx | 85 +--- .../GenerateActionControls.tsx | 249 +++++++++++- view/bottom-controls/GenerateControls.tsx | 51 +++ view/canvas/brush.ts | 10 +- view/canvas/magic-wand.ts | 91 ++--- view/canvas/renderFrame.test.ts | 30 ++ view/canvas/renderFrame.ts | 7 +- view/generate/candidateActions.ts | 50 +++ view/generate/inpaintPrep.test.ts | 36 ++ view/generate/inpaintPrep.ts | 258 ++++++++++++ view/generate/runGenerate.ts | 226 +++++++++-- view/mask/maskRaster.test.ts | 40 ++ view/mask/maskRaster.ts | 374 ++++++++++++++++++ 35 files changed, 2285 insertions(+), 242 deletions(-) create mode 100644 app/comfy.test.ts create mode 100644 commands/generation.test.ts create mode 100644 commands/generation.ts create mode 100644 view/generate/candidateActions.ts create mode 100644 view/generate/inpaintPrep.test.ts create mode 100644 view/generate/inpaintPrep.ts create mode 100644 view/mask/maskRaster.test.ts create mode 100644 view/mask/maskRaster.ts diff --git a/app/app.ts b/app/app.ts index 6c67aab..0e30a8d 100644 --- a/app/app.ts +++ b/app/app.ts @@ -1,4 +1,5 @@ import { documentCommands } from "@commands/document"; +import { generationCommands } from "@commands/generation"; import { historyCommands } from "@commands/history"; import { commandIds } from "@commands/ids"; import { createCommandRegistry } from "@commands/registry"; @@ -12,7 +13,7 @@ import { createAppStore } from "@editor/store"; export type ImageStudioApp = ReturnType; export function createImageStudioApp(options?: { documentName?: string; createDefaultArtboard?: boolean }) { - const registry = createCommandRegistry([...viewportCommands, ...selectionCommands, ...documentCommands, ...toolCommands, ...transformCommands, ...historyCommands]); + const registry = createCommandRegistry([...viewportCommands, ...selectionCommands, ...documentCommands, ...toolCommands, ...generationCommands, ...transformCommands, ...historyCommands]); const store = createAppStore(createInitialAppState(options?.documentName), registry); if (options?.createDefaultArtboard !== false) { diff --git a/app/comfy.test.ts b/app/comfy.test.ts new file mode 100644 index 0000000..70ac3fe --- /dev/null +++ b/app/comfy.test.ts @@ -0,0 +1,59 @@ +import { describe, expect, test } from "bun:test"; +import { buildSdxlWorkflow, selectGeneratedOutputImage } from "./comfy"; + +describe("Comfy adapter", () => { + test("selects SaveImage output instead of uploaded input or mask images", () => { + const image = selectGeneratedOutputImage({ + outputs: { + "4": { images: [{ filename: "image-studio-input.png", type: "input" }] }, + "9": { images: [{ filename: "image-studio-mask.png", type: "input" }] }, + "8": { images: [{ filename: "image-studio-inpaint_00001_.png", subfolder: "", type: "output" }] }, + }, + }); + + expect(image).toEqual({ filename: "image-studio-inpaint_00001_.png", subfolder: "", type: "output" }); + }); + + test("falls back to generated filename prefixes when node ids differ", () => { + const image = selectGeneratedOutputImage({ + outputs: { + "12": { images: [{ filename: "image-studio-inpaint_00002_.png", type: "output" }] }, + "4": { images: [{ filename: "image-studio-input.png", type: "input" }] }, + }, + }); + + expect(image?.filename).toBe("image-studio-inpaint_00002_.png"); + }); + + test("builds neutral inpaint with VAEEncodeForInpaint", () => { + const workflow = buildSdxlWorkflow(inpaintRequest({ maskedContent: "neutral" })); + + expect(workflow["5"]?.class_type).toBe("VAEEncodeForInpaint"); + expect(workflow["5"]?.inputs).toMatchObject({ grow_mask_by: 6, mask: ["11", 0] }); + expect(workflow["6"]?.inputs.latent_image).toEqual(["5", 0]); + }); + + test("builds original-content inpaint with a latent noise mask", () => { + const workflow = buildSdxlWorkflow(inpaintRequest({ maskedContent: "original", growMaskBy: 12 })); + + expect(workflow["5"]?.class_type).toBe("VAEEncode"); + expect(workflow["12"]?.class_type).toBe("GrowMask"); + expect(workflow["12"]?.inputs).toMatchObject({ mask: ["11", 0], expand: 12 }); + expect(workflow["13"]?.class_type).toBe("SetLatentNoiseMask"); + expect(workflow["13"]?.inputs).toMatchObject({ samples: ["5", 0], mask: ["12", 0] }); + expect(workflow["6"]?.inputs.latent_image).toEqual(["13", 0]); + }); +}); + +function inpaintRequest(inpaint: { maskedContent: "neutral" | "original"; growMaskBy?: number }) { + return { + mode: "inpaint" as const, + model: "model.safetensors", + prompt: "replace garment", + width: 128, + height: 128, + inputImage: "input.png", + maskImage: "mask.png", + inpaint, + }; +} diff --git a/app/comfy.ts b/app/comfy.ts index 3b3bdae..3055927 100644 --- a/app/comfy.ts +++ b/app/comfy.ts @@ -20,6 +20,17 @@ type ComfyGenerateRequest = { bottom?: number; feathering?: number; }; + inpaint?: { + growMaskBy?: number; + maskBlur?: number; + maskFeather?: number; + maskExpand?: number; + cropPadding?: number; + maskPolarity?: "hidden" | "revealed"; + maskedContent?: "neutral" | "original" | "originalColor" | "edges"; + crop?: unknown; + placement?: unknown; + }; inputImage?: string; maskImage?: string; }; @@ -27,10 +38,14 @@ type ComfyGenerateRequest = { const comfyBaseUrl = process.env.COMFYUI_URL ?? "http://127.0.0.1:8188"; export async function handleComfyApi(request: Request) { - const url = new URL(request.url); - if (url.pathname === "/api/comfy/models" && request.method === "GET") return json(await listGenerationOptions()); - if (url.pathname === "/api/comfy/generate" && request.method === "POST") return json(await generate(await request.json() as ComfyGenerateRequest)); - return new Response("Not found", { status: 404 }); + try { + const url = new URL(request.url); + if (url.pathname === "/api/comfy/models" && request.method === "GET") return json(await listGenerationOptions()); + if (url.pathname === "/api/comfy/generate" && request.method === "POST") return json(await generate(await request.json() as ComfyGenerateRequest)); + return new Response("Not found", { status: 404 }); + } catch (error) { + return new Response(error instanceof Error ? error.message : "ComfyUI request failed", { status: 500 }); + } } async function listGenerationOptions() { @@ -61,6 +76,7 @@ async function generate(request: ComfyGenerateRequest) { const clientId = crypto.randomUUID(); const uploaded = request.inputImage ? await uploadDataUrl(request.inputImage, `image-studio-${crypto.randomUUID()}.png`) : undefined; const mask = request.maskImage ? await uploadDataUrl(request.maskImage, `image-studio-mask-${crypto.randomUUID()}.png`) : undefined; + if (request.mode === "inpaint" && (!uploaded || !mask)) throw new Error("Inpaint requires normalized input and mask images"); const prompt = buildSdxlWorkflow({ ...request, inputImage: uploaded, maskImage: mask }); const queued = await fetch(`${comfyBaseUrl}/prompt`, { @@ -69,9 +85,15 @@ async function generate(request: ComfyGenerateRequest) { body: JSON.stringify({ client_id: clientId, prompt }), }); if (!queued.ok) throw new Error(`ComfyUI prompt failed: ${queued.status} ${await queued.text()}`); - const { prompt_id } = await queued.json() as { prompt_id: string }; + const queuedBody = await queued.json() as { prompt_id?: string; node_errors?: unknown }; + const nodeError = nodeErrorsMessage(queuedBody.node_errors); + if (nodeError) throw new Error(`ComfyUI rejected the workflow: ${nodeError}`); + if (!queuedBody.prompt_id) throw new Error("ComfyUI did not return a prompt id"); + const prompt_id = queuedBody.prompt_id; const history = await waitForHistory(prompt_id); - const image = firstOutputImage(history); + const historyError = historyErrorMessage(history); + if (historyError) throw new Error(`ComfyUI generation failed: ${historyError}`); + const image = selectGeneratedOutputImage(history); if (!image) throw new Error("ComfyUI did not return an image"); const imageResponse = await fetch(`${comfyBaseUrl}/view?${new URLSearchParams({ filename: image.filename, subfolder: image.subfolder ?? "", type: image.type ?? "output" })}`); @@ -82,7 +104,7 @@ async function generate(request: ComfyGenerateRequest) { async function uploadDataUrl(dataUrl: string, filename: string) { const match = /^data:([^;]+);base64,(.+)$/.exec(dataUrl); - if (!match) return undefined; + if (!match) throw new Error("Expected a base64 data URL image"); const mimeType = match[1] ?? "image/png"; const base64 = match[2] ?? ""; const form = new FormData(); @@ -103,19 +125,25 @@ async function waitForHistory(promptId: string) { } await Bun.sleep(500); } - throw new Error("Timed out waiting for ComfyUI"); + throw new Error(`Timed out waiting for ComfyUI prompt ${promptId}`); } -function firstOutputImage(history: unknown): { filename: string; subfolder?: string; type?: string } | undefined { +export function selectGeneratedOutputImage(history: unknown): { filename: string; subfolder?: string; type?: string } | undefined { const outputs = (history as { outputs?: Record }).outputs ?? {}; - for (const output of Object.values(outputs)) { - const image = output.images?.[0]; - if (image) return image; - } - return undefined; + const saveImageOutput = outputs["8"]?.images?.find(isGeneratedImage); + if (saveImageOutput) return saveImageOutput; + + const prefixedOutput = Object.values(outputs).flatMap((output) => output.images ?? []).find((image) => image.filename.startsWith("image-studio-") && image.type !== "input"); + if (prefixedOutput) return prefixedOutput; + + return Object.values(outputs).flatMap((output) => output.images ?? []).find(isGeneratedImage); } -function buildSdxlWorkflow(request: ComfyGenerateRequest) { +function isGeneratedImage(image: { filename: string; subfolder?: string; type?: string }) { + return image.type === undefined || image.type === "output"; +} + +export function buildSdxlWorkflow(request: ComfyGenerateRequest) { if (!request.width || !request.height) throw new Error("Generation width and height are required"); const width = Math.max(64, Math.round(request.width)); const height = Math.max(64, Math.round(request.height)); @@ -127,7 +155,7 @@ function buildSdxlWorkflow(request: ComfyGenerateRequest) { const scheduler = request.scheduler ?? "normal"; const positive = request.prompt; const negative = request.negativePrompt ?? ""; - const workflow: Record = { + const workflow: Record }> = { "1": { class_type: "CheckpointLoaderSimple", inputs: { ckpt_name: request.model } }, "2": { class_type: "CLIPTextEncode", inputs: { text: positive, clip: ["1", 1] } }, "3": { class_type: "CLIPTextEncode", inputs: { text: negative, clip: ["1", 1] } }, @@ -145,13 +173,21 @@ function buildSdxlWorkflow(request: ComfyGenerateRequest) { if (request.mode === "inpaint" && request.maskImage) { workflow["9"] = { class_type: "LoadImage", inputs: { image: request.maskImage } }; - workflow["5"] = { class_type: "VAEEncodeForInpaint", inputs: { pixels: ["4", 0], vae: ["1", 2], mask: ["9", 1], grow_mask_by: 6 } }; + workflow["11"] = { class_type: "ImageToMask", inputs: { image: ["9", 0], channel: "red" } }; + if (usesOriginalLatentContent(request)) { + workflow["5"] = { class_type: "VAEEncode", inputs: { pixels: ["4", 0], vae: ["1", 2] } }; + workflow["12"] = { class_type: "GrowMask", inputs: { mask: ["11", 0], expand: resolveGrowMaskBy(request), tapered_corners: true } }; + workflow["13"] = { class_type: "SetLatentNoiseMask", inputs: { samples: ["5", 0], mask: ["12", 0] } }; + workflow["6"].inputs.latent_image = ["13", 0]; + } else { + workflow["5"] = { class_type: "VAEEncodeForInpaint", inputs: { pixels: ["4", 0], vae: ["1", 2], mask: ["11", 0], grow_mask_by: resolveGrowMaskBy(request) } }; + } return workflow; } if (request.mode === "outpaint") { workflow["10"] = { class_type: "ImagePadForOutpaint", inputs: { image: ["4", 0], left: Math.round(request.outpaint?.left ?? 0), top: Math.round(request.outpaint?.top ?? 0), right: Math.round(request.outpaint?.right ?? 0), bottom: Math.round(request.outpaint?.bottom ?? 0), feathering: Math.round(request.outpaint?.feathering ?? 0) } }; - workflow["5"] = { class_type: "VAEEncodeForInpaint", inputs: { pixels: ["10", 0], vae: ["1", 2], mask: ["10", 1], grow_mask_by: 6 } }; + workflow["5"] = { class_type: "VAEEncodeForInpaint", inputs: { pixels: ["10", 0], vae: ["1", 2], mask: ["10", 1], grow_mask_by: resolveGrowMaskBy(request) } }; return workflow; } @@ -159,6 +195,56 @@ function buildSdxlWorkflow(request: ComfyGenerateRequest) { return workflow; } +function resolveGrowMaskBy(request: ComfyGenerateRequest): number { + const value = request.inpaint?.growMaskBy ?? 6; + if (!Number.isFinite(value)) return 6; + return Math.round(Math.max(0, Math.min(256, value))); +} + +function usesOriginalLatentContent(request: ComfyGenerateRequest): boolean { + return request.inpaint?.maskedContent === "original" || request.inpaint?.maskedContent === "originalColor" || request.inpaint?.maskedContent === "edges"; +} + +function nodeErrorsMessage(nodeErrors: unknown): string | undefined { + if (!nodeErrors) return undefined; + if (Array.isArray(nodeErrors) && nodeErrors.length === 0) return undefined; + if (typeof nodeErrors === "object" && Object.keys(nodeErrors).length === 0) return undefined; + + if (typeof nodeErrors === "string") return nodeErrors; + try { + return JSON.stringify(nodeErrors); + } catch { + return "Unknown node validation error"; + } +} + +function historyErrorMessage(history: unknown): string | undefined { + const status = (history as { status?: { status_str?: string; completed?: boolean; messages?: unknown[] } }).status; + if (!status) return undefined; + if (status.status_str && status.status_str !== "success") return statusMessage(status); + if (status.completed === false) return statusMessage(status); + return undefined; +} + +function statusMessage(status: { status_str?: string; messages?: unknown[] }) { + const message = status.messages?.map(formatHistoryMessage).filter(Boolean).join("; "); + return message || status.status_str || "Unknown execution error"; +} + +function formatHistoryMessage(message: unknown): string | undefined { + if (!Array.isArray(message)) return undefined; + const eventName = typeof message[0] === "string" ? message[0] : undefined; + const payload = message[1]; + if (payload && typeof payload === "object") { + const detail = payload as { exception_message?: string; node_type?: string; node_id?: string | number }; + if (detail.exception_message) { + const node = detail.node_type ? ` in ${detail.node_type}${detail.node_id !== undefined ? ` ${detail.node_id}` : ""}` : ""; + return `${eventName ?? "error"}${node}: ${detail.exception_message}`; + } + } + return eventName; +} + function json(value: unknown) { return new Response(JSON.stringify(value), { headers: { "content-type": "application/json" } }); } diff --git a/commands/document.test.ts b/commands/document.test.ts index ff43e27..969b239 100644 --- a/commands/document.test.ts +++ b/commands/document.test.ts @@ -8,6 +8,7 @@ import { documentAddImageLayerCommand, documentAddLayerMaskCommand, documentAddRasterLayerCommand, + documentApplyLayerMaskOperationCommand, documentGroupLayersCommand, documentMoveLayerCommand, documentRemoveArtboardCommand, @@ -236,6 +237,30 @@ describe("document commands", () => { expect(unmasked.editor.maskEdit).toBeUndefined(); }); + test("applies operations only to attached layer mask assets", () => { + const state = documentAddLayerMaskCommand.execute( + { state: documentWithLayers([raster("target", "Target"), raster("loose", "Loose", "loose-asset")]) }, + { layerId: "target", asset: maskAsset(), maskLayer: raster("mask", "Target Mask", "mask-asset") }, + ); + const withLooseAsset = documentAddAssetCommand.execute( + { state }, + { asset: { id: "loose-asset", name: "Loose", mimeType: "image/png", source: "loose-source", intrinsicSize: { w: 100, h: 100 } } }, + ); + + const updated = documentApplyLayerMaskOperationCommand.execute( + { state: withLooseAsset }, + { maskLayerId: "mask", source: "updated-mask", mimeType: "image/png", operation: { type: "invert" } }, + ); + const ignored = documentApplyLayerMaskOperationCommand.execute( + { state: updated }, + { maskLayerId: "loose", source: "wrong", operation: { type: "invert" } }, + ); + + expect(updated.document.assets.find((asset) => asset.id === "mask-asset")?.source).toBe("updated-mask"); + expect(updated.document.assets.find((asset) => asset.id === "mask-asset")?.mimeType).toBe("image/png"); + expect(ignored.document.assets.find((asset) => asset.id === "loose-asset")?.source).toBe("loose-source"); + }); + test("cleans mask references when deleting targets or mask layers", () => { const state = documentAddLayerMaskCommand.execute( { state: documentWithLayers([raster("target", "Target")]) }, diff --git a/commands/document.ts b/commands/document.ts index b6fef46..84e10f2 100644 --- a/commands/document.ts +++ b/commands/document.ts @@ -113,6 +113,25 @@ export type DocumentAddLayerMaskPayload = { maskLayer: RasterLayer; }; +export type LayerMaskOperation = + | { type: "paint" } + | { type: "magicWand" } + | { type: "chromaKey" } + | { type: "invert" } + | { type: "fill"; fill: "white" | "black" | "clear" } + | { type: "feather"; radius: number } + | { type: "expand"; radius: number } + | { type: "contract"; radius: number } + | { type: "blur"; radius: number } + | { type: "despeckle"; strength: number }; + +export type DocumentApplyLayerMaskOperationPayload = { + maskLayerId: LayerId; + source: string; + mimeType?: string; + operation: LayerMaskOperation; +}; + export type DocumentRemoveLayerMaskPayload = { layerId: LayerId; }; @@ -490,6 +509,38 @@ export const documentAddLayerMaskCommand: Command = }, }; +export const documentApplyLayerMaskOperationCommand: Command = { + id: commandIds.documentApplyLayerMaskOperation, + name: "Apply layer mask operation", + execute({ state }, payload) { + if (!payload.source.trim()) return state; + + const maskLocation = findLayerLocation(state.document, payload.maskLayerId); + if (!maskLocation || maskLocation.layer.type === "group") return state; + if (!isReferencedMaskLayer(state.document, payload.maskLayerId)) return state; + + return { + ...state, + document: { + ...state.document, + assets: state.document.assets.map((asset) => + asset.id === maskLocation.layer.assetId + ? { + ...asset, + source: payload.source, + mimeType: payload.mimeType ?? asset.mimeType, + } + : asset, + ), + }, + editor: { + ...state.editor, + brushStrokePreview: state.editor.brushStrokePreview?.assetId === maskLocation.layer.assetId ? undefined : state.editor.brushStrokePreview, + }, + }; + }, +}; + export const documentRemoveLayerMaskCommand: Command = { id: commandIds.documentRemoveLayerMask, name: "Remove layer mask", @@ -564,6 +615,7 @@ export const documentCommands = [ documentRenameLayerCommand, documentSetLayerClippingMaskCommand, documentAddLayerMaskCommand, + documentApplyLayerMaskOperationCommand, documentRemoveLayerMaskCommand, ] satisfies Command[]; @@ -582,6 +634,18 @@ function findLayerLocation(document: ImageDocument, layerId: LayerId): LayerLoca return undefined; } +function isReferencedMaskLayer(document: ImageDocument, maskLayerId: LayerId): boolean { + return document.artboards.some((artboard) => isReferencedMaskLayerInTree(artboard.layers, maskLayerId)); +} + +function isReferencedMaskLayerInTree(layers: readonly Layer[], maskLayerId: LayerId): boolean { + for (const layer of layers) { + if (layer.clippingMask?.maskLayerId === maskLayerId) return true; + if (layer.type === "group" && isReferencedMaskLayerInTree(layer.children, maskLayerId)) return true; + } + return false; +} + function findLayerLocationInTree(layers: Layer[], layerId: LayerId, artboardId: ArtboardId, parentGroupId?: LayerId): LayerLocation | undefined { for (let index = 0; index < layers.length; index++) { const layer = layers[index]; diff --git a/commands/generation.test.ts b/commands/generation.test.ts new file mode 100644 index 0000000..7d80d40 --- /dev/null +++ b/commands/generation.test.ts @@ -0,0 +1,143 @@ +import { describe, expect, test } from "bun:test"; +import type { GenerationCandidate } from "@editor/state"; +import { createInitialAppState } from "@editor/initial-state"; +import { + generationAddCandidateCommand, + generationApplyCandidateAsLayerCommand, + generationRemoveCandidateCommand, + generationReplaceCandidatePixelsCommand, +} from "./generation"; + +describe("generation commands", () => { + test("adds, selects, and removes candidates", () => { + const state = createInitialAppState("Test"); + const candidate = generationCandidate("candidate-1"); + + const added = generationAddCandidateCommand.execute({ state }, { candidate }); + const removed = generationRemoveCandidateCommand.execute({ state: added }, { candidateId: candidate.id }); + + expect(added.editor.generation.candidates).toEqual([candidate]); + expect(added.editor.generation.selectedCandidateId).toBe(candidate.id); + expect(removed.editor.generation.candidates).toEqual([]); + expect(removed.editor.generation.selectedCandidateId).toBeUndefined(); + }); + + test("applies candidates as top-level layers", () => { + const state = generationAddCandidateCommand.execute({ state: documentWithSourceLayer() }, { candidate: generationCandidate("candidate-1") }); + + const next = generationApplyCandidateAsLayerCommand.execute( + { state }, + { candidateId: "candidate-1", assetId: "generated-asset", layerId: "generated-layer" }, + ); + + expect(next.document.assets.find((asset) => asset.id === "generated-asset")?.source).toBe("generated-source"); + expect(next.document.artboards[0]?.layers[0]?.id).toBe("generated-layer"); + expect(next.editor.selection).toEqual({ artboardId: "a1", layerIds: ["generated-layer"] }); + }); + + test("replaces source asset pixels for inpaint candidates", () => { + const state = generationAddCandidateCommand.execute({ state: documentWithSourceLayer() }, { candidate: generationCandidate("candidate-1", true) }); + + const next = generationReplaceCandidatePixelsCommand.execute( + { state }, + { candidateId: "candidate-1", source: "composited-source", mimeType: "image/png" }, + ); + + expect(next.document.assets.find((asset) => asset.id === "source-asset")?.source).toBe("composited-source"); + expect(next.document.assets.find((asset) => asset.id === "source-asset")?.mimeType).toBe("image/png"); + expect(next.editor.selection).toEqual({ artboardId: "a1", layerIds: ["source-layer"] }); + }); +}); + +function documentWithSourceLayer() { + return { + ...createInitialAppState("Test"), + document: { + ...createInitialAppState("Test").document, + assets: [ + { id: "source-asset", name: "Source", mimeType: "image/png", source: "source", intrinsicSize: { w: 100, h: 100 } }, + { id: "mask-asset", name: "Mask", mimeType: "image/png", source: "mask", intrinsicSize: { w: 100, h: 100 } }, + ], + artboards: [ + { + id: "a1", + name: "Artboard 1", + bounds: { x: 0, y: 0, w: 100, h: 100 }, + backgroundColor: "transparent", + visible: true, + locked: false, + layers: [ + raster("mask-layer", "Mask", "mask-asset"), + { ...raster("source-layer", "Source", "source-asset"), clippingMask: { maskLayerId: "mask-layer" } }, + ], + }, + ], + }, + }; +} + +function generationCandidate(id: string, inpaint = false): GenerationCandidate { + const candidate: GenerationCandidate = { + id, + source: "generated-source", + mimeType: "image/png", + intrinsicSize: { w: 64, h: 64 }, + mode: inpaint ? "inpaint" : "text-to-image", + settings: createInitialAppState("Test").editor.tools.generate, + seed: 123, + width: 64, + height: 64, + placement: { + artboardId: "a1", + layerName: "Generated", + transform: { position: { x: 5, y: 6 }, scale: { x: 1, y: 1 }, rotation: 0 }, + }, + }; + + return inpaint + ? { + ...candidate, + inputImage: "input", + maskImage: "mask", + inpaint: { + targetLayerId: "source-layer", + maskLayerId: "mask-layer", + sourceAssetId: "source-asset", + maskAssetId: "mask-asset", + inputImage: "input", + maskImage: "mask", + crop: { + assetBounds: { x: 0, y: 0, w: 64, h: 64 }, + documentBounds: { x: 0, y: 0, w: 64, h: 64 }, + padding: 12, + maskedAreaOnly: true, + }, + mask: { + polarity: "hidden", + activeBounds: { x: 10, y: 10, w: 20, h: 20 }, + }, + backend: { + growMaskBy: 6, + maskedContent: "neutral", + maskBlur: 0, + maskFeather: 0, + maskExpand: 0, + cropPadding: 12, + }, + }, + } + : candidate; +} + +function raster(id: string, name: string, assetId: string) { + return { + id, + type: "raster" as const, + name, + visible: true, + locked: false, + opacity: 1, + assetId, + transform: { position: { x: 0, y: 0 }, scale: { x: 1, y: 1 }, rotation: 0 }, + }; +} diff --git a/commands/generation.ts b/commands/generation.ts new file mode 100644 index 0000000..16c575a --- /dev/null +++ b/commands/generation.ts @@ -0,0 +1,217 @@ +import type { Asset } from "@core/asset"; +import type { ImageDocument } from "@core/document"; +import type { ArtboardId, AssetId, LayerId } from "@core/id"; +import type { ImageLayer } from "@core/image-layer"; +import type { Layer } from "@core/layer"; +import type { GenerationCandidate } from "@editor/state"; +import type { Command } from "./command"; +import { commandIds } from "./ids"; + +export type GenerationAddCandidatePayload = { + candidate: GenerationCandidate; +}; + +export type GenerationSelectCandidatePayload = { + candidateId?: string; +}; + +export type GenerationRemoveCandidatePayload = { + candidateId: string; +}; + +export type GenerationApplyCandidateAsLayerPayload = { + candidateId: string; + assetId: AssetId; + layerId: LayerId; + variant?: boolean; +}; + +export type GenerationReplaceCandidatePixelsPayload = { + candidateId: string; + source: string; + mimeType?: string; +}; + +const maxCandidates = 12; + +export const generationAddCandidateCommand: Command = { + id: commandIds.generationAddCandidate, + name: "Add generation candidate", + history: { mode: "ignore" }, + execute({ state }, payload) { + const candidates = [payload.candidate, ...state.editor.generation.candidates.filter((candidate) => candidate.id !== payload.candidate.id)].slice(0, maxCandidates); + return { + ...state, + editor: { + ...state.editor, + generation: { + candidates, + selectedCandidateId: payload.candidate.id, + }, + }, + }; + }, +}; + +export const generationSelectCandidateCommand: Command = { + id: commandIds.generationSelectCandidate, + name: "Select generation candidate", + history: { mode: "ignore" }, + execute({ state }, payload) { + const selectedCandidateId = payload.candidateId && state.editor.generation.candidates.some((candidate) => candidate.id === payload.candidateId) ? payload.candidateId : undefined; + if (state.editor.generation.selectedCandidateId === selectedCandidateId) return state; + return { + ...state, + editor: { + ...state.editor, + generation: { + ...state.editor.generation, + selectedCandidateId, + }, + }, + }; + }, +}; + +export const generationRemoveCandidateCommand: Command = { + id: commandIds.generationRemoveCandidate, + name: "Remove generation candidate", + history: { mode: "ignore" }, + execute({ state }, payload) { + const candidates = state.editor.generation.candidates.filter((candidate) => candidate.id !== payload.candidateId); + if (candidates.length === state.editor.generation.candidates.length) return state; + const selectedCandidateId = state.editor.generation.selectedCandidateId === payload.candidateId ? candidates[0]?.id : state.editor.generation.selectedCandidateId; + return { + ...state, + editor: { + ...state.editor, + generation: { + candidates, + selectedCandidateId, + }, + }, + }; + }, +}; + +export const generationClearCandidatesCommand: Command = { + id: commandIds.generationClearCandidates, + name: "Clear generation candidates", + history: { mode: "ignore" }, + execute({ state }) { + if (state.editor.generation.candidates.length === 0 && !state.editor.generation.selectedCandidateId) return state; + return { + ...state, + editor: { + ...state.editor, + generation: { candidates: [], selectedCandidateId: undefined }, + }, + }; + }, +}; + +export const generationApplyCandidateAsLayerCommand: Command = { + id: commandIds.generationApplyCandidateAsLayer, + name: "Apply generation candidate as layer", + execute({ state }, payload) { + const candidate = state.editor.generation.candidates.find((item) => item.id === payload.candidateId); + if (!candidate) return state; + if (state.document.assets.some((asset) => asset.id === payload.assetId) || findLayerLocation(state.document, payload.layerId)) return state; + if (!state.document.artboards.some((artboard) => artboard.id === candidate.placement.artboardId)) return state; + + const asset: Asset = { + id: payload.assetId, + name: payload.variant ? `${candidate.placement.layerName} variant` : candidate.placement.layerName, + mimeType: candidate.mimeType, + source: candidate.source, + intrinsicSize: { ...candidate.intrinsicSize }, + }; + const layer: ImageLayer = { + id: payload.layerId, + type: "image", + name: payload.variant ? `${candidate.placement.layerName} variant` : candidate.placement.layerName, + visible: true, + locked: false, + opacity: 1, + assetId: asset.id, + transform: { + position: { ...candidate.placement.transform.position }, + scale: { ...candidate.placement.transform.scale }, + rotation: candidate.placement.transform.rotation, + }, + }; + + return { + ...state, + document: insertLayerAtTop({ ...state.document, assets: [...state.document.assets, asset] }, candidate.placement.artboardId, layer), + editor: { + ...state.editor, + selection: { artboardId: candidate.placement.artboardId, layerIds: [layer.id] }, + }, + }; + }, +}; + +export const generationReplaceCandidatePixelsCommand: Command = { + id: commandIds.generationReplaceCandidatePixels, + name: "Replace masked pixels with generation candidate", + execute({ state }, payload) { + const candidate = state.editor.generation.candidates.find((item) => item.id === payload.candidateId); + if (!candidate?.inpaint || !payload.source.trim()) return state; + const targetAsset = state.document.assets.find((asset) => asset.id === candidate.inpaint?.sourceAssetId); + const targetLayerLocation = findLayerLocation(state.document, candidate.inpaint.targetLayerId); + if (!targetAsset || !targetLayerLocation) return state; + + return { + ...state, + document: { + ...state.document, + assets: state.document.assets.map((asset) => asset.id === targetAsset.id ? { ...asset, source: payload.source, mimeType: payload.mimeType ?? asset.mimeType } : asset), + }, + editor: { + ...state.editor, + selection: { artboardId: targetLayerLocation.artboardId, layerIds: [candidate.inpaint.targetLayerId] }, + }, + }; + }, +}; + +export const generationCommands = [ + generationAddCandidateCommand, + generationSelectCandidateCommand, + generationRemoveCandidateCommand, + generationClearCandidatesCommand, + generationApplyCandidateAsLayerCommand, + generationReplaceCandidatePixelsCommand, +] satisfies Command[]; + +type LayerLocation = { + artboardId: ArtboardId; + layer: Layer; +}; + +function insertLayerAtTop(document: ImageDocument, artboardId: ArtboardId, layer: Layer): ImageDocument { + return { + ...document, + artboards: document.artboards.map((artboard) => artboard.id === artboardId ? { ...artboard, layers: [layer, ...artboard.layers] } : artboard), + }; +} + +function findLayerLocation(document: ImageDocument, layerId: LayerId): LayerLocation | undefined { + for (const artboard of document.artboards) { + const layer = findLayerInTree(artboard.layers, layerId); + if (layer) return { artboardId: artboard.id, layer }; + } + return undefined; +} + +function findLayerInTree(layers: readonly Layer[], layerId: LayerId): Layer | undefined { + for (const layer of layers) { + if (layer.id === layerId) return layer; + if (layer.type === "group") { + const child = findLayerInTree(layer.children, layerId); + if (child) return child; + } + } + return undefined; +} diff --git a/commands/ids.ts b/commands/ids.ts index 22fbd01..9c3f531 100644 --- a/commands/ids.ts +++ b/commands/ids.ts @@ -19,6 +19,7 @@ export const commandIds = { documentRenameLayer: "document.renameLayer", documentSetLayerClippingMask: "document.setLayerClippingMask", documentAddLayerMask: "document.addLayerMask", + documentApplyLayerMaskOperation: "document.applyLayerMaskOperation", documentRemoveLayerMask: "document.removeLayerMask", selectionSet: "selection.set", selectionClear: "selection.clear", @@ -35,6 +36,12 @@ export const commandIds = { toolExitMaskEdit: "tool.exitMaskEdit", toolEnterTemporaryPan: "tool.enterTemporaryPan", toolExitTemporaryPan: "tool.exitTemporaryPan", + generationAddCandidate: "generation.addCandidate", + generationSelectCandidate: "generation.selectCandidate", + generationRemoveCandidate: "generation.removeCandidate", + generationClearCandidates: "generation.clearCandidates", + generationApplyCandidateAsLayer: "generation.applyCandidateAsLayer", + generationReplaceCandidatePixels: "generation.replaceCandidatePixels", transformBegin: "transform.begin", transformUpdate: "transform.update", transformSetBounds: "transform.setBounds", diff --git a/commands/index.ts b/commands/index.ts index fcd27f1..69fe209 100644 --- a/commands/index.ts +++ b/commands/index.ts @@ -6,6 +6,7 @@ export { documentAddImageLayerCommand, documentAddLayerMaskCommand, documentAddRasterLayerCommand, + documentApplyLayerMaskOperationCommand, documentCommands, documentGroupLayersCommand, documentMoveLayerCommand, @@ -30,6 +31,7 @@ export type { DocumentAddImageLayerPayload, DocumentAddLayerMaskPayload, DocumentAddRasterLayerPayload, + DocumentApplyLayerMaskOperationPayload, DocumentGroupLayersPayload, DocumentMoveLayerPayload, DocumentRemoveArtboardPayload, @@ -45,8 +47,25 @@ export type { DocumentSetLayerVisiblePayload, DocumentUpdateAssetSourcePayload, DocumentUngroupLayerPayload, + LayerMaskOperation, } from "./document"; export { historyCommands, historyRedoCommand, historyUndoCommand } from "./history"; +export { + generationAddCandidateCommand, + generationApplyCandidateAsLayerCommand, + generationClearCandidatesCommand, + generationCommands, + generationRemoveCandidateCommand, + generationReplaceCandidatePixelsCommand, + generationSelectCandidateCommand, +} from "./generation"; +export type { + GenerationAddCandidatePayload, + GenerationApplyCandidateAsLayerPayload, + GenerationRemoveCandidatePayload, + GenerationReplaceCandidatePixelsPayload, + GenerationSelectCandidatePayload, +} from "./generation"; export type { CommandDispatcher, Dispatch } from "./dispatcher"; export type { CommandId, CommandPayloads } from "./payloads"; export { createCommandDispatcher } from "./dispatcher"; diff --git a/commands/payloads.ts b/commands/payloads.ts index c905fd9..e23602f 100644 --- a/commands/payloads.ts +++ b/commands/payloads.ts @@ -6,6 +6,7 @@ import type { DocumentAddImageLayerPayload, DocumentAddLayerMaskPayload, DocumentAddRasterLayerPayload, + DocumentApplyLayerMaskOperationPayload, DocumentGroupLayersPayload, DocumentMoveLayerPayload, DocumentRemoveArtboardPayload, @@ -22,6 +23,13 @@ import type { DocumentUpdateAssetSourcePayload, DocumentUngroupLayerPayload, } from "./document"; +import type { + GenerationAddCandidatePayload, + GenerationApplyCandidateAsLayerPayload, + GenerationRemoveCandidatePayload, + GenerationReplaceCandidatePixelsPayload, + GenerationSelectCandidatePayload, +} from "./generation"; import type { SelectionAddLayerPayload, SelectionSetPayload } from "./selection"; import type { ToolEnterMaskEditPayload, ToolSetActivePayload, ToolSetBrushPreviewPayload, ToolSetBrushSettingsPayload, ToolSetBrushStrokePreviewPayload, ToolSetChromaKeySettingsPayload, ToolSetGenerateSettingsPayload, ToolSetMagicWandSettingsPayload, ToolSetMaskViewModePayload } from "./tool"; import type { TransformBeginPayload, TransformSetBoundsPayload, TransformUpdatePayload } from "./transform"; @@ -46,6 +54,7 @@ export type CommandPayloads = { [commandIds.documentAddRasterLayer]: DocumentAddRasterLayerPayload; [commandIds.documentAddGroupLayer]: DocumentAddGroupLayerPayload; [commandIds.documentAddLayerMask]: DocumentAddLayerMaskPayload; + [commandIds.documentApplyLayerMaskOperation]: DocumentApplyLayerMaskOperationPayload; [commandIds.documentRemoveLayerMask]: DocumentRemoveLayerMaskPayload; [commandIds.documentMoveLayer]: DocumentMoveLayerPayload; [commandIds.documentGroupLayers]: DocumentGroupLayersPayload; @@ -70,6 +79,12 @@ export type CommandPayloads = { [commandIds.toolExitMaskEdit]: void; [commandIds.toolEnterTemporaryPan]: void; [commandIds.toolExitTemporaryPan]: void; + [commandIds.generationAddCandidate]: GenerationAddCandidatePayload; + [commandIds.generationSelectCandidate]: GenerationSelectCandidatePayload; + [commandIds.generationRemoveCandidate]: GenerationRemoveCandidatePayload; + [commandIds.generationClearCandidates]: void; + [commandIds.generationApplyCandidateAsLayer]: GenerationApplyCandidateAsLayerPayload; + [commandIds.generationReplaceCandidatePixels]: GenerationReplaceCandidatePixelsPayload; [commandIds.transformBegin]: TransformBeginPayload; [commandIds.transformUpdate]: TransformUpdatePayload; [commandIds.transformSetBounds]: TransformSetBoundsPayload; diff --git a/commands/tool.test.ts b/commands/tool.test.ts index ad3c376..7406fbf 100644 --- a/commands/tool.test.ts +++ b/commands/tool.test.ts @@ -1,11 +1,11 @@ import { describe, expect, test } from "bun:test"; import { createInitialAppState } from "@editor/initial-state"; -import { toolEnterMaskEditCommand, toolEnterTemporaryPanCommand, toolExitMaskEditCommand, toolExitTemporaryPanCommand, toolSetActiveCommand, toolSetBrushPreviewCommand, toolSetBrushSettingsCommand, toolSetBrushStrokePreviewCommand, toolSetChromaKeySettingsCommand, toolSetMaskViewModeCommand } from "./tool"; +import { toolEnterMaskEditCommand, toolEnterTemporaryPanCommand, toolExitMaskEditCommand, toolExitTemporaryPanCommand, toolSetActiveCommand, toolSetBrushPreviewCommand, toolSetBrushSettingsCommand, toolSetBrushStrokePreviewCommand, toolSetChromaKeySettingsCommand, toolSetGenerateSettingsCommand, toolSetMaskViewModeCommand } from "./tool"; const defaultBrush = { color: "#111827", size: 8, hardness: 100 }; const defaultChromaKey = { color: "#00ff00", tolerance: 32, softness: 24, feather: 0, choke: 0, despeckle: 0, spill: 50 }; const defaultMagicWand = { tolerance: 32, feather: 0, choke: 0, despeckle: 0, contiguous: true, mode: "replace" as const }; -const defaultGenerate = { mode: "text-to-image" as const, model: "auto" as const, prompt: "", negativePrompt: "", strength: 75, steps: 30, cfg: 7, seed: -1, sampler: "euler", scheduler: "normal", width: 1024, height: 1024, outpaint: { left: 128, top: 128, right: 128, bottom: 128, feathering: 32 } }; +const defaultGenerate = { mode: "text-to-image" as const, model: "auto" as const, prompt: "", negativePrompt: "", strength: 75, steps: 30, cfg: 7, seed: -1, sampler: "euler", scheduler: "normal", width: 1024, height: 1024, outpaint: { left: 128, top: 128, right: 128, bottom: 128, feathering: 32 }, inpaint: { maskedAreaOnly: true, cropPadding: 96, maskPolarity: "hidden" as const, maskedContent: "neutral" as const, growMaskBy: 6, maskExpand: 0, maskFeather: 0, maskBlur: 0, maskDespeckle: 0 } }; describe("tool commands", () => { test("sets active tool", () => { @@ -25,6 +25,16 @@ describe("tool commands", () => { expect(next.editor.tools.chromaKey).toEqual({ color: "#123456", tolerance: 255, softness: 24, feather: 0, choke: 0, despeckle: 0, spill: 50 }); }); + test("sets inpaint generation settings", () => { + const next = toolSetGenerateSettingsCommand.execute( + { state: createInitialAppState("Test") }, + { mode: "inpaint", inpaint: { ...defaultGenerate.inpaint, cropPadding: 5000, growMaskBy: -4, maskExpand: -500, maskBlur: 300, maskPolarity: "revealed", maskedContent: "original" } }, + ); + + expect(next.editor.tools.generate.mode).toBe("inpaint"); + expect(next.editor.tools.generate.inpaint).toEqual({ ...defaultGenerate.inpaint, cropPadding: 2048, growMaskBy: 0, maskExpand: -256, maskBlur: 256, maskPolarity: "revealed", maskedContent: "original" }); + }); + test("sets and clears brush preview", () => { const showing = toolSetBrushPreviewCommand.execute({ state: createInitialAppState("Test") }, { position: { x: 10, y: 20 } }); const cleared = toolSetBrushPreviewCommand.execute({ state: showing }, undefined); diff --git a/commands/tool.ts b/commands/tool.ts index 9afedb0..7a406e8 100644 --- a/commands/tool.ts +++ b/commands/tool.ts @@ -89,6 +89,17 @@ export const toolSetGenerateSettingsCommand: Command renderLayer(context, documentIndex, editor, layer, imageTextureRenderer, clipRect, maskLayerIds)); + if (generationCandidate?.placement.artboardId === artboard.id) renderGenerationCandidatePreview(context, editor, generationCandidate, imageTextureRenderer, clipRect); } } +export function generationCandidatePreviewAssets(editor: EditorState): Asset[] { + return editor.generation.candidates.map((candidate) => generationCandidateAsset(candidate)); +} + function renderLayer( context: WebGlRendererContext, documentIndex: DocumentReadIndex, @@ -102,6 +110,44 @@ function isIsolatedMaskView(mode: MaskViewMode) { return mode === "blackWhite" || mode === "alpha" || mode === "overlay"; } +function renderGenerationCandidatePreview( + context: WebGlRendererContext, + editor: EditorState, + candidate: GenerationCandidate, + imageTextureRenderer: ImageTextureRenderer, + clipRect: ScreenRect, +) { + const rect = documentRectToScreenRect(context.canvas, generationCandidateBounds(candidate), editor.viewport); + imageTextureRenderer.render(generationCandidateAsset(candidate), rect, clipRect); +} + +function selectedGenerationCandidate(editor: EditorState): GenerationCandidate | undefined { + return editor.generation.candidates.find((candidate) => candidate.id === editor.generation.selectedCandidateId) ?? editor.generation.candidates[0]; +} + +function generationCandidateAsset(candidate: GenerationCandidate): Asset { + return { + id: generationCandidateAssetId(candidate.id), + name: candidate.placement.layerName, + mimeType: candidate.mimeType, + source: candidate.source, + intrinsicSize: candidate.intrinsicSize, + }; +} + +function generationCandidateAssetId(candidateId: string) { + return `generation-candidate:${candidateId}`; +} + +function generationCandidateBounds(candidate: GenerationCandidate): Rect { + return { + x: candidate.placement.transform.position.x, + y: candidate.placement.transform.position.y, + w: candidate.intrinsicSize.w * candidate.placement.transform.scale.x, + h: candidate.intrinsicSize.h * candidate.placement.transform.scale.y, + }; +} + function assetWithBrushStrokePreview(asset: TAsset, editor: EditorState): TAsset { if (!asset || editor.brushStrokePreview?.assetId !== asset.id) return asset; return { ...asset, source: editor.brushStrokePreview.source } as TAsset; diff --git a/renderer/renderer.ts b/renderer/renderer.ts index 1d0d9d4..50de897 100644 --- a/renderer/renderer.ts +++ b/renderer/renderer.ts @@ -4,7 +4,7 @@ import { renderArtboard } from "./artboard"; import { createBrushPreviewRenderer } from "./brush-preview"; import { createCheckerboardRenderer } from "./checkerboard"; import { createImageTextureRenderer } from "./image-textures"; -import { renderLayers } from "./layers"; +import { generationCandidatePreviewAssets, renderLayers } from "./layers"; import { renderSelectionOverlay } from "./selection"; import { renderTransformControls } from "./transform-controls"; import type { WebGlRendererContext } from "./types"; @@ -62,7 +62,7 @@ export function createRenderer(canvas: HTMLCanvasElement, backend: RendererBacke for (const artboard of frame.document.artboards) { if (artboard.visible) renderArtboard(rendererContext, artboard, frame.editor.viewport, checkerboardRenderer); } - imageTextureRenderer.syncAssets(frame.document.assets); + imageTextureRenderer.syncAssets([...frame.document.assets, ...generationCandidatePreviewAssets(frame.editor)]); renderLayers(rendererContext, frame.document, frame.editor, imageTextureRenderer); if (!frame.editor.maskEdit) { diff --git a/view/App.tsx b/view/App.tsx index d863302..cbfae84 100644 --- a/view/App.tsx +++ b/view/App.tsx @@ -25,7 +25,7 @@ export type AppProps = { export function App({ app }: AppProps) { const shellState = useAppState(app.store, selectAppShellState, shallowEqual); - const { document, selection, viewport, tools, transformSession, maskEdit } = shellState; + const { document, selection, viewport, tools, generation, transformSession, maskEdit } = shellState; const viewportActivityIsland = useViewportActivityIsland(viewport); const imageImport = useImageImport(app.store); const [layersOpen, setLayersOpen] = useState(false); @@ -161,6 +161,7 @@ export function App({ app }: AppProps) { activeTool={tools.activeTool} brushSettings={tools.brush} generateSettings={tools.generate} + generation={generation} chromaKeySettings={tools.chromaKey} magicWandSettings={tools.magicWand} editingMask={Boolean(maskEdit)} @@ -189,6 +190,7 @@ type AppShellState = { selection: AppState["editor"]["selection"]; viewport: AppState["editor"]["viewport"]; tools: AppState["editor"]["tools"]; + generation: AppState["editor"]["generation"]; transformSession: AppState["editor"]["transformSession"]; maskEdit: AppState["editor"]["maskEdit"]; }; @@ -199,6 +201,7 @@ function selectAppShellState(state: AppState): AppShellState { selection: state.editor.selection, viewport: state.editor.viewport, tools: state.editor.tools, + generation: state.editor.generation, transformSession: state.editor.transformSession, maskEdit: state.editor.maskEdit, }; diff --git a/view/BottomControlsIsland.tsx b/view/BottomControlsIsland.tsx index c83427f..d1886a3 100644 --- a/view/BottomControlsIsland.tsx +++ b/view/BottomControlsIsland.tsx @@ -1,6 +1,6 @@ import type { AppStore } from "@editor/store"; import type { ImageDocument } from "@core/document"; -import type { MaskViewMode, SelectionState, ViewportState } from "@editor/state"; +import type { GenerationState, MaskViewMode, SelectionState, ViewportState } from "@editor/state"; import type { BrushSettings, ChromaKeySettings, GenerateSettings, MagicWandSettings, ToolId } from "@editor/tools"; import { BrushControls } from "./bottom-controls/BrushControls"; import { ChromaKeyControls } from "./bottom-controls/ChromaKeyControls"; @@ -22,6 +22,7 @@ export type BottomControlsIslandProps = { activeTool: ToolId; brushSettings: BrushSettings; generateSettings: GenerateSettings; + generation: GenerationState; chromaKeySettings: ChromaKeySettings; magicWandSettings: MagicWandSettings; editingMask?: boolean; @@ -32,7 +33,7 @@ export type BottomControlsIslandProps = { dispatch: AppStore["dispatch"]; }; -export function BottomControlsIsland({ document, selection, viewport, visible, action, activeTool, brushSettings, generateSettings, chromaKeySettings, magicWandSettings, editingMask = false, maskViewMode = "composite", transformBounds, transformTarget, brushHint, dispatch }: BottomControlsIslandProps) { +export function BottomControlsIsland({ document, selection, viewport, visible, action, activeTool, brushSettings, generateSettings, generation, chromaKeySettings, magicWandSettings, editingMask = false, maskViewMode = "composite", transformBounds, transformTarget, brushHint, dispatch }: BottomControlsIslandProps) { const zoomPercent = Math.round(viewport.zoom * 100); const x = Math.round(viewport.center.x); const y = Math.round(viewport.center.y); @@ -40,12 +41,12 @@ export function BottomControlsIsland({ document, selection, viewport, visible, a return (
{activeTool === "generate" ? ( - + ) : (activeTool === "brush" || activeTool === "eraser") && brushHint ? ( ) : activeTool === "brush" || activeTool === "eraser" ? ( diff --git a/view/LayersSheet.tsx b/view/LayersSheet.tsx index 28cbb9a..892ea83 100644 --- a/view/LayersSheet.tsx +++ b/view/LayersSheet.tsx @@ -1,6 +1,7 @@ -import { useMemo, useRef, useState, type DragEvent, type MutableRefObject } from "react"; +import { useEffect, useMemo, useRef, useState, type DragEvent, type MutableRefObject } from "react"; import { ArrowDown, ArrowUp, DownloadSimple, Eye, EyeSlash, FolderPlus, Lock, LockOpen, Plus, Stack, Trash } from "@phosphor-icons/react"; import { commandIds } from "@commands/ids"; +import type { Asset } from "@core/asset"; import type { ImageDocument } from "@core/document"; import type { Layer } from "@core/layer"; import type { ArtboardId } from "@core/id"; @@ -9,6 +10,7 @@ import type { MaskEditState, SelectionState } from "@editor/state"; import type { AppStore } from "@editor/store"; import { resolveLayerDrop } from "@input/index"; import { downloadArtboardPng } from "./exportArtboardPng"; +import { analyzeMaskSource, applyMaskRasterOperation, type MaskAnalysis, type MaskRasterOperation } from "./mask/maskRaster"; export type LayersSheetProps = { document: ImageDocument; @@ -219,6 +221,7 @@ function LayerRow({ const selected = selectedLayerIds.includes(layer.id); const layerInfo = documentIndex.layerInfoById.get(layer.id); const maskLayer = layer.clippingMask ? documentIndex.layerById.get(layer.clippingMask.maskLayerId) : undefined; + const maskAsset = maskLayer && maskLayer.type !== "group" ? documentIndex.assetById.get(maskLayer.assetId) : undefined; const canAddMask = Boolean(layerInfo && layer.type !== "group" && !layer.clippingMask); const editingMask = Boolean(maskEdit && layer.clippingMask && maskEdit.targetLayerId === layer.id && maskEdit.maskLayerId === layer.clippingMask.maskLayerId); const rowPadding = 12 + depth * 16; @@ -288,21 +291,49 @@ function LayerRow({
{layer.clippingMask ? ( -
- - {maskLayer ? "Layer mask" : "Layer mask missing"} +
+ + {maskLayer ? "Layer mask" : "Layer mask missing"} + {maskAsset ? : null} {maskLayer ? ( - +
+ + + + {maskAsset && maskLayer.type !== "group" ? ( + + ) : null} +
) : null} + ); +} + type EditingTitle = | { type: "artboard"; id: ArtboardId; draft: string } | { type: "layer"; id: string; draft: string }; @@ -528,3 +636,11 @@ function toolbarButtonClass() { function labeledToolbarButtonClass() { return "inline-flex h-12 flex-1 items-center rounded-full pr-4 text-sm font-medium text-white/75 transition hover:bg-white/10 hover:text-white disabled:pointer-events-none disabled:opacity-35 focus:outline-none focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-white/30"; } + +function maskActionButtonClass() { + return "h-8 rounded-full bg-white/5 px-3 text-xs font-medium text-sky-100/65 transition hover:bg-white/10 hover:text-sky-50 disabled:pointer-events-none disabled:opacity-35"; +} + +function formatPercent(value: number) { + return `${Math.round(Math.max(0, Math.min(1, value)) * 100)}%`; +} diff --git a/view/bottom-controls/BrushControls.tsx b/view/bottom-controls/BrushControls.tsx index 3092edd..b66456a 100644 --- a/view/bottom-controls/BrushControls.tsx +++ b/view/bottom-controls/BrushControls.tsx @@ -31,6 +31,13 @@ export function BrushControls({ tool, settings, editingMask = false, maskViewMod {tool === "eraser" ? : } + {editingMask ? ( + <> + + + + + ) : null} {tool === "brush" && !editingMask ? (
); } + +function maskModeButtonClass(active: boolean) { + return `rounded-full px-4 py-2 text-base font-medium transition ${active ? "bg-white text-black" : "bg-white/10 text-white hover:bg-white/15"}`; +} diff --git a/view/bottom-controls/ChromaKeyControls.tsx b/view/bottom-controls/ChromaKeyControls.tsx index e2e43a5..7a36b75 100644 --- a/view/bottom-controls/ChromaKeyControls.tsx +++ b/view/bottom-controls/ChromaKeyControls.tsx @@ -7,6 +7,7 @@ import { resolveTransformTargetBounds } from "@editor/transform-targets"; import type { AppStore } from "@editor/store"; import type { ChromaKeySettings } from "@editor/tools"; import type { SelectionState } from "@editor/state"; +import { blurMaskValues, despeckleMaskValues, dilateMaskValues, erodeMaskValues } from "../mask/maskRaster"; import { BottomControlColorPicker } from "./ColorPicker"; import { BottomControlDivider } from "./Divider"; import { BottomControlSlider } from "./Slider"; @@ -179,8 +180,8 @@ async function applyChromaKeyMask(target: NonNullable 0) next = despeckleAlpha(next, width, height, despeckle); - if (choke > 0) next = erodeAlpha(next, width, height, choke); - if (choke < 0) next = dilateAlpha(next, width, height, -choke); - if (feather > 0) next = blurAlpha(next, width, height, feather); + if (despeckle > 0) next = despeckleMaskValues(next, width, height, despeckle); + if (choke > 0) next = erodeMaskValues(next, width, height, choke); + if (choke < 0) next = dilateMaskValues(next, width, height, -choke); + if (feather > 0) next = blurMaskValues(next, width, height, feather); return next; } -function despeckleAlpha(alpha: Uint8ClampedArray, width: number, height: number, strength: number) { - const radius = Math.max(1, Math.ceil(strength / 6)); - const threshold = Math.max(1, Math.round(strength / 2)); - const next = new Uint8ClampedArray(alpha); - for (let y = 0; y < height; y++) { - for (let x = 0; x < width; x++) { - const index = y * width + x; - const visible = (alpha[index] ?? 0) > 127; - let same = 0; - for (let oy = -radius; oy <= radius; oy++) { - for (let ox = -radius; ox <= radius; ox++) { - if (ox === 0 && oy === 0) continue; - const sample = alpha[clamp(y + oy, 0, height - 1) * width + clamp(x + ox, 0, width - 1)] ?? 0; - if ((sample > 127) === visible) same += 1; - } - } - if (same <= threshold) next[index] = visible ? 0 : 255; - } - } - return next; -} - -function erodeAlpha(alpha: Uint8ClampedArray, width: number, height: number, radius: number) { - const next = new Uint8ClampedArray(alpha.length); - for (let y = 0; y < height; y++) { - for (let x = 0; x < width; x++) { - let value = 255; - for (let oy = -radius; oy <= radius; oy++) { - for (let ox = -radius; ox <= radius; ox++) value = Math.min(value, alpha[clamp(y + oy, 0, height - 1) * width + clamp(x + ox, 0, width - 1)] ?? 0); - } - next[y * width + x] = value; - } - } - return next; -} - -function dilateAlpha(alpha: Uint8ClampedArray, width: number, height: number, radius: number) { - const next = new Uint8ClampedArray(alpha.length); - for (let y = 0; y < height; y++) { - for (let x = 0; x < width; x++) { - let value = 0; - for (let oy = -radius; oy <= radius; oy++) { - for (let ox = -radius; ox <= radius; ox++) value = Math.max(value, alpha[clamp(y + oy, 0, height - 1) * width + clamp(x + ox, 0, width - 1)] ?? 0); - } - next[y * width + x] = value; - } - } - return next; -} - -function blurAlpha(alpha: Uint8ClampedArray, width: number, height: number, radius: number) { - const next = new Uint8ClampedArray(alpha.length); - for (let y = 0; y < height; y++) { - for (let x = 0; x < width; x++) { - let total = 0; - let count = 0; - for (let oy = -radius; oy <= radius; oy++) { - for (let ox = -radius; ox <= radius; ox++) { - total += alpha[clamp(y + oy, 0, height - 1) * width + clamp(x + ox, 0, width - 1)] ?? 0; - count += 1; - } - } - next[y * width + x] = Math.round(total / count); - } - } - return next; -} - -function clamp(value: number, min: number, max: number) { - return Math.max(min, Math.min(max, value)); -} - function loadImage(source: string) { return new Promise((resolve, reject) => { const image = new Image(); diff --git a/view/bottom-controls/GenerateActionControls.tsx b/view/bottom-controls/GenerateActionControls.tsx index c1aca89..f14f3a5 100644 --- a/view/bottom-controls/GenerateActionControls.tsx +++ b/view/bottom-controls/GenerateActionControls.tsx @@ -1,40 +1,271 @@ -import { useState } from "react"; +import { useEffect, useState } from "react"; +import { commandIds } from "@commands/ids"; import type { ImageDocument } from "@core/document"; -import type { SelectionState, ViewportState } from "@editor/state"; +import type { GenerationCandidate, GenerationState, SelectionState, ViewportState } from "@editor/state"; import type { GenerateSettings } from "@editor/tools"; import type { AppStore } from "@editor/store"; -import { runGenerate } from "../generate/runGenerate"; +import { createMaskedPixelReplacementSource } from "../generate/candidateActions"; +import { runGenerate, runGenerateFromCandidate } from "../generate/runGenerate"; +import { createSolidMaskSource } from "../mask/maskRaster"; export type GenerateActionControlsProps = { document: ImageDocument; selection: SelectionState; viewport: ViewportState; settings: GenerateSettings; + generation: GenerationState; dispatch: AppStore["dispatch"]; }; -export function GenerateActionControls({ document, selection, viewport, settings, dispatch }: GenerateActionControlsProps) { - const [busy, setBusy] = useState(false); +export function GenerateActionControls({ document, selection, viewport, settings, generation, dispatch }: GenerateActionControlsProps) { + const [busy, setBusy] = useState(); const [error, setError] = useState(); + const [elapsedSeconds, setElapsedSeconds] = useState(0); + const candidate = selectedCandidate(generation); const canGenerate = Boolean(settings.prompt.trim()) && !busy; + useEffect(() => { + if (!busy) { + setElapsedSeconds(0); + return; + } + + setElapsedSeconds(0); + const startedAt = Date.now(); + const interval = window.setInterval(() => { + setElapsedSeconds(Math.floor((Date.now() - startedAt) / 1000)); + }, 1000); + return () => window.clearInterval(interval); + }, [busy]); + return ( -
+
+ {busy ? {busy} {formatElapsed(elapsedSeconds)} : null} + {candidate ? ( + <> + + + + ) : null} + {error ? {error} : null}
); } + +function CandidatePicker({ generation, dispatch }: { generation: GenerationState; dispatch: AppStore["dispatch"] }) { + if (generation.candidates.length < 2) return null; + return ( +
+ {generation.candidates.slice(0, 6).map((candidate) => { + const selected = candidate.id === (generation.selectedCandidateId ?? generation.candidates[0]?.id); + return ( + + ); + })} +
+ ); +} + +function CandidateControls({ + document, + candidate, + settings, + busy, + setBusy, + setError, + dispatch, +}: { + document: ImageDocument; + candidate: GenerationCandidate; + settings: GenerateSettings; + busy?: string; + setBusy: (busy: string | undefined) => void; + setError: (error: string | undefined) => void; + dispatch: AppStore["dispatch"]; +}) { + const rerun = (label: string, nextSettings: GenerateSettings) => { + setBusy(label); + setError(undefined); + dispatch(commandIds.toolSetGenerateSettings, nextSettings); + void runGenerateFromCandidate({ candidate, settings: nextSettings, dispatch }) + .catch((reason: unknown) => setError(reason instanceof Error ? reason.message : `${label} failed`)) + .finally(() => setBusy(undefined)); + }; + const disabled = Boolean(busy); + + return ( +
+ + Seed {candidate.seed} + rerun("Regenerate", candidate.settings)} /> + rerun("Lower", { ...candidate.settings, strength: Math.max(0, candidate.settings.strength - 10), seed: candidate.seed })} + /> + rerun("Reuse seed", { ...candidate.settings, seed: candidate.seed })} /> + rerun("New seed", { ...candidate.settings, seed: -1 })} /> + applyCandidateAsLayer(candidate, false, dispatch)} /> + { + setBusy("Refine"); + setError(undefined); + void applyCandidateAsRefinementLayer(candidate, dispatch) + .catch((reason: unknown) => setError(reason instanceof Error ? reason.message : "Refine setup failed")) + .finally(() => setBusy(undefined)); + }} + /> + applyCandidateAsLayer(candidate, true, dispatch)} /> + { + setBusy("Replace"); + setError(undefined); + void createMaskedPixelReplacementSource(document, candidate) + .then((source) => dispatch(commandIds.generationReplaceCandidatePixels, { candidateId: candidate.id, source, mimeType: "image/png" })) + .catch((reason: unknown) => setError(reason instanceof Error ? reason.message : "Replace failed")) + .finally(() => setBusy(undefined)); + }} + /> + { + if (!candidate.inpaint) return; + dispatch(commandIds.selectionSet, { artboardId: candidate.placement.artboardId, layerIds: [candidate.inpaint.targetLayerId] }); + dispatch(commandIds.toolEnterMaskEdit, { targetLayerId: candidate.inpaint.targetLayerId, maskLayerId: candidate.inpaint.maskLayerId }); + }} + /> + dispatch(commandIds.generationRemoveCandidate, { candidateId: candidate.id })} /> + {settings.seed !== candidate.seed ? null : Current settings reuse this seed} +
+ ); +} + +function CandidatePreview({ candidate }: { candidate: GenerationCandidate }) { + if (!candidate.inputImage) { + return ; + } + + return ( + + + {candidate.maskImage ? : null} + + + ); +} + +function CandidateButton({ label, title, disabled, busy, onClick }: { label: string; title: string; disabled?: boolean; busy?: boolean; onClick: () => void }) { + return ( + + ); +} + +function selectedCandidate(generation: GenerationState): GenerationCandidate | undefined { + return generation.candidates.find((candidate) => candidate.id === generation.selectedCandidateId) ?? generation.candidates[0]; +} + +function applyCandidateAsLayer(candidate: GenerationCandidate, variant: boolean, dispatch: AppStore["dispatch"]) { + applyCandidateAsLayerWithIds(candidate, { layerId: crypto.randomUUID(), assetId: crypto.randomUUID() }, variant, dispatch); +} + +async function applyCandidateAsRefinementLayer(candidate: GenerationCandidate, dispatch: AppStore["dispatch"]) { + const layerId = crypto.randomUUID(); + const maskLayerId = crypto.randomUUID(); + const maskAssetId = crypto.randomUUID(); + applyCandidateAsLayerWithIds(candidate, { layerId, assetId: crypto.randomUUID() }, true, dispatch); + const width = Math.max(1, Math.round(candidate.intrinsicSize.w)); + const height = Math.max(1, Math.round(candidate.intrinsicSize.h)); + const source = await createSolidMaskSource(width, height, "white"); + + dispatch(commandIds.documentAddLayerMask, { + layerId, + asset: { + id: maskAssetId, + name: `${candidate.placement.layerName} refinement mask`, + mimeType: "image/png", + source, + intrinsicSize: { w: width, h: height }, + }, + maskLayer: { + id: maskLayerId, + type: "raster", + name: `${candidate.placement.layerName} refinement mask`, + visible: true, + locked: false, + opacity: 1, + assetId: maskAssetId, + transform: { + position: { ...candidate.placement.transform.position }, + scale: { ...candidate.placement.transform.scale }, + rotation: candidate.placement.transform.rotation, + }, + }, + }); + dispatch(commandIds.toolSetActive, { tool: "eraser" }); +} + +function applyCandidateAsLayerWithIds(candidate: GenerationCandidate, ids: { layerId: string; assetId: string }, variant: boolean, dispatch: AppStore["dispatch"]) { + dispatch(commandIds.generationApplyCandidateAsLayer, { + candidateId: candidate.id, + assetId: ids.assetId, + layerId: ids.layerId, + variant, + }); +} + +function formatElapsed(seconds: number) { + const minutes = Math.floor(seconds / 60); + const remainder = seconds % 60; + return `${minutes}:${remainder.toString().padStart(2, "0")}`; +} diff --git a/view/bottom-controls/GenerateControls.tsx b/view/bottom-controls/GenerateControls.tsx index c3a1014..bd02b08 100644 --- a/view/bottom-controls/GenerateControls.tsx +++ b/view/bottom-controls/GenerateControls.tsx @@ -21,6 +21,18 @@ const sizePresets = [ { label: "9:16", w: 768, h: 1344 }, ] as const; +const inpaintPolarityOptions = [ + { value: "hidden", label: "Hidden / erased" }, + { value: "revealed", label: "Revealed / painted" }, +] satisfies readonly BottomControlSelectOption[]; + +const inpaintMaskedContentOptions = [ + { value: "neutral", label: "Neutral fill" }, + { value: "original", label: "Original gray" }, + { value: "originalColor", label: "Original color" }, + { value: "edges", label: "Edge map" }, +] satisfies readonly BottomControlSelectOption[]; + export type GenerateControlsProps = { settings: GenerateSettings; dispatch: AppStore["dispatch"]; @@ -32,6 +44,7 @@ export function GenerateControls({ settings, dispatch }: GenerateControlsProps) const [schedulers, setSchedulers] = useState[]>([{ value: settings.scheduler, label: settings.scheduler }]); const [advancedOpen, setAdvancedOpen] = useState(false); const [outpaintOpen, setOutpaintOpen] = useState(false); + const [inpaintOpen, setInpaintOpen] = useState(false); const [sizeOpen, setSizeOpen] = useState(false); const sizeRef = useRef(null); const [error, setError] = useState(); @@ -141,6 +154,44 @@ export function GenerateControls({ settings, dispatch }: GenerateControlsProps) dispatch(commandIds.toolSetGenerateSettings, { outpaint: { ...settings.outpaint, feathering } })} />
+ +
+ +
+ dispatch(commandIds.toolSetGenerateSettings, { inpaint: { ...settings.inpaint, maskPolarity } })} + /> + dispatch(commandIds.toolSetGenerateSettings, { inpaint: { ...settings.inpaint, maskedContent } })} + /> + +
+ dispatch(commandIds.toolSetGenerateSettings, { inpaint: { ...settings.inpaint, cropPadding } })} /> + dispatch(commandIds.toolSetGenerateSettings, { inpaint: { ...settings.inpaint, growMaskBy } })} /> + dispatch(commandIds.toolSetGenerateSettings, { inpaint: { ...settings.inpaint, maskExpand } })} /> + dispatch(commandIds.toolSetGenerateSettings, { inpaint: { ...settings.inpaint, maskFeather } })} /> + dispatch(commandIds.toolSetGenerateSettings, { inpaint: { ...settings.inpaint, maskBlur } })} /> + dispatch(commandIds.toolSetGenerateSettings, { inpaint: { ...settings.inpaint, maskDespeckle } })} /> +
+
+
); } diff --git a/view/canvas/brush.ts b/view/canvas/brush.ts index f5675a5..44b96cf 100644 --- a/view/canvas/brush.ts +++ b/view/canvas/brush.ts @@ -138,7 +138,15 @@ export async function commitBrushSession(options: { store: AppStore; session: Br if (options.session.cancelled) return; const source = options.session.changed ? canvasToDataUrl(options.session.canvas) : undefined; - if (source) options.store.dispatch(commandIds.documentUpdateAssetSource, { assetId: options.session.assetId, source }); + if (source) { + const state = options.store.getState(); + const maskEdit = state.editor.maskEdit; + if (maskEdit?.maskLayerId === options.session.layerId) { + options.store.dispatch(commandIds.documentApplyLayerMaskOperation, { maskLayerId: maskEdit.maskLayerId, source, mimeType: "image/png", operation: { type: "paint" } }); + } else { + options.store.dispatch(commandIds.documentUpdateAssetSource, { assetId: options.session.assetId, source }); + } + } options.store.dispatch(commandIds.toolSetBrushStrokePreview, undefined); closeBrushStrokePreview(options.session); } diff --git a/view/canvas/magic-wand.ts b/view/canvas/magic-wand.ts index b9fbb72..b3d035b 100644 --- a/view/canvas/magic-wand.ts +++ b/view/canvas/magic-wand.ts @@ -5,6 +5,7 @@ import type { Layer } from "@core/layer"; import { resolveTransformTargetBounds } from "@editor/transform-targets"; import type { AppStore } from "@editor/store"; import type { EditorState } from "@editor/state"; +import { blurMaskValues, despeckleMaskValues, dilateMaskValues, erodeMaskValues, maskValueFromRgba } from "../mask/maskRaster"; export async function applyMagicWandAt(store: AppStore, point: Vec2D, modeOverride?: EditorState["tools"]["magicWand"]["mode"]) { const state = store.getState(); @@ -15,8 +16,8 @@ export async function applyMagicWandAt(store: AppStore, point: Vec2D, modeOverri const y = Math.floor((point.y - target.layer.transform.position.y) / Math.max(0.0001, target.layer.transform.scale.y)); if (x < 0 || y < 0 || x >= target.asset.intrinsicSize.w || y >= target.asset.intrinsicSize.h) return true; const source = await createWandMask(target.asset.source, target.maskAsset?.source, Math.round(target.asset.intrinsicSize.w), Math.round(target.asset.intrinsicSize.h), x, y, { ...state.editor.tools.magicWand, mode: modeOverride ?? state.editor.tools.magicWand.mode }); - if (target.maskAsset) { - store.dispatch(commandIds.documentUpdateAssetSource, { assetId: target.maskAsset.id, source }); + if (target.maskAsset && target.maskLayer && target.maskLayer.type !== "group") { + store.dispatch(commandIds.documentApplyLayerMaskOperation, { maskLayerId: target.maskLayer.id, source, mimeType: "image/png", operation: { type: "magicWand" } }); return true; } const assetId = crypto.randomUUID(); @@ -42,7 +43,7 @@ function resolveTarget(document: ImageDocument, editor: EditorState) { const bounds = resolveTransformTargetBounds(document, { type: "layer", id: layer.id }); const maskLayer = layer.clippingMask ? findLayer(document.artboards.flatMap((artboard) => artboard.layers), layer.clippingMask.maskLayerId) : undefined; const maskAsset = maskLayer && maskLayer.type !== "group" ? document.assets.find((candidate) => candidate.id === maskLayer.assetId) : undefined; - return asset && bounds ? { layer, asset, bounds, maskAsset } : undefined; + return asset && bounds ? { layer, asset, bounds, maskLayer, maskAsset } : undefined; } async function createWandMask(source: string, existingMaskSource: string | undefined, width: number, height: number, startX: number, startY: number, settings: EditorState["tools"]["magicWand"]) { @@ -57,10 +58,15 @@ async function createWandMask(source: string, existingMaskSource: string | undef const start = (startY * canvas.width + startX) * 4; const key = [imageData.data[start] ?? 0, imageData.data[start + 1] ?? 0, imageData.data[start + 2] ?? 0]; const selected = postProcessSelection(settings.contiguous ? floodSelect(imageData, canvas.width, canvas.height, startX, startY, key, settings.tolerance) : globalSelect(imageData, key, settings.tolerance), canvas.width, canvas.height, settings); - const existingAlpha = existingMaskSource ? await loadMaskAlpha(existingMaskSource, canvas.width, canvas.height) : undefined; + const existingMask = existingMaskSource ? await loadMaskValues(existingMaskSource, canvas.width, canvas.height) : undefined; for (let pixel = 0; pixel < selected.length; pixel++) { - const current = existingAlpha?.[pixel] ?? 255; - const value = settings.mode === "add" ? (selected[pixel] ? 0 : current) : settings.mode === "subtract" ? (selected[pixel] ? 255 : current) : selected[pixel] ? 0 : 255; + const current = existingMask?.[pixel] ?? 255; + const selectionValue = selected[pixel] ?? 0; + const value = settings.mode === "add" + ? Math.min(current, 255 - selectionValue) + : settings.mode === "subtract" + ? Math.max(current, selectionValue) + : 255 - selectionValue; const index = pixel * 4; imageData.data[index] = 255; imageData.data[index + 1] = 255; @@ -98,67 +104,24 @@ function matches(data: ImageData, pixel: number, key: number[], tolerance: numbe } function postProcessSelection(selected: Uint8Array, width: number, height: number, settings: EditorState["tools"]["magicWand"]) { - let next = selected; + let next = selectionToMaskValues(selected); const despeckle = Math.round(Math.max(0, Math.min(20, settings.despeckle))); const choke = Math.round(Math.max(-20, Math.min(20, settings.choke))); const feather = Math.round(Math.max(0, Math.min(20, settings.feather))); - if (despeckle > 0) next = despeckleSelection(next, width, height, despeckle); - if (choke > 0) next = erodeSelection(next, width, height, choke); - if (choke < 0) next = dilateSelection(next, width, height, -choke); - if (feather > 0) next = featherSelection(next, width, height, feather); + if (despeckle > 0) next = despeckleMaskValues(next, width, height, despeckle); + if (choke > 0) next = erodeMaskValues(next, width, height, choke); + if (choke < 0) next = dilateMaskValues(next, width, height, -choke); + if (feather > 0) next = blurMaskValues(next, width, height, feather); return next; } -function erodeSelection(selected: Uint8Array, width: number, height: number, radius: number) { - const next = new Uint8Array(selected.length); - for (let y = 0; y < height; y++) for (let x = 0; x < width; x++) { - let value = 1; - for (let oy = -radius; oy <= radius; oy++) for (let ox = -radius; ox <= radius; ox++) value = Math.min(value, selected[clamp(y + oy, 0, height - 1) * width + clamp(x + ox, 0, width - 1)] ?? 0); - next[y * width + x] = value; - } - return next; +function selectionToMaskValues(selected: Uint8Array) { + const values = new Uint8ClampedArray(selected.length); + for (let index = 0; index < selected.length; index += 1) values[index] = selected[index] ? 255 : 0; + return values; } -function dilateSelection(selected: Uint8Array, width: number, height: number, radius: number) { - const next = new Uint8Array(selected.length); - for (let y = 0; y < height; y++) for (let x = 0; x < width; x++) { - let value = 0; - for (let oy = -radius; oy <= radius; oy++) for (let ox = -radius; ox <= radius; ox++) value = Math.max(value, selected[clamp(y + oy, 0, height - 1) * width + clamp(x + ox, 0, width - 1)] ?? 0); - next[y * width + x] = value; - } - return next; -} - -function featherSelection(selected: Uint8Array, width: number, height: number, radius: number) { - const next = new Uint8Array(selected.length); - for (let y = 0; y < height; y++) for (let x = 0; x < width; x++) { - let total = 0; - let count = 0; - for (let oy = -radius; oy <= radius; oy++) for (let ox = -radius; ox <= radius; ox++) { - total += selected[clamp(y + oy, 0, height - 1) * width + clamp(x + ox, 0, width - 1)] ?? 0; - count += 1; - } - next[y * width + x] = Math.round(total / count); - } - return next; -} - -function despeckleSelection(selected: Uint8Array, width: number, height: number, strength: number) { - const radius = Math.max(1, Math.ceil(strength / 6)); - const threshold = Math.max(1, Math.round(strength / 2)); - const next = new Uint8Array(selected); - for (let y = 0; y < height; y++) for (let x = 0; x < width; x++) { - const index = y * width + x; - let same = 0; - for (let oy = -radius; oy <= radius; oy++) for (let ox = -radius; ox <= radius; ox++) if (ox !== 0 || oy !== 0) { - if ((selected[clamp(y + oy, 0, height - 1) * width + clamp(x + ox, 0, width - 1)] ?? 0) === selected[index]) same += 1; - } - if (same <= threshold) next[index] = selected[index] ? 0 : 1; - } - return next; -} - -async function loadMaskAlpha(source: string, width: number, height: number) { +async function loadMaskValues(source: string, width: number, height: number) { const canvas = document.createElement("canvas"); canvas.width = width; canvas.height = height; @@ -167,13 +130,9 @@ async function loadMaskAlpha(source: string, width: number, height: number) { const image = await loadImage(source); context.drawImage(image, 0, 0, width, height); const data = context.getImageData(0, 0, width, height); - const alpha = new Uint8ClampedArray(width * height); - for (let pixel = 0; pixel < alpha.length; pixel++) alpha[pixel] = data.data[pixel * 4 + 3] ?? 255; - return alpha; -} - -function clamp(value: number, min: number, max: number) { - return Math.max(min, Math.min(max, value)); + const values = new Uint8ClampedArray(width * height); + for (let pixel = 0; pixel < values.length; pixel++) values[pixel] = maskValueFromRgba(data.data, pixel * 4); + return values; } function findLayer(layers: readonly Layer[], layerId: string): Layer | undefined { diff --git a/view/canvas/renderFrame.test.ts b/view/canvas/renderFrame.test.ts index 1df93a5..175f25b 100644 --- a/view/canvas/renderFrame.test.ts +++ b/view/canvas/renderFrame.test.ts @@ -151,6 +151,36 @@ const visualEditorChanges: Array<[string, (state: AppState) => AppState]> = [ }, }), ], + [ + "generation candidate preview", + (state) => ({ + ...state, + editor: { + ...state.editor, + generation: { + candidates: [ + { + id: "candidate", + source: "data:image/png;base64,candidate", + mimeType: "image/png", + intrinsicSize: { w: 64, h: 64 }, + mode: "text-to-image", + settings: state.editor.tools.generate, + seed: 12, + width: 64, + height: 64, + placement: { + artboardId: "artboard", + layerName: "Candidate", + transform: { position: { x: 1, y: 2 }, scale: { x: 1, y: 1 }, rotation: 0 }, + }, + }, + ], + selectedCandidateId: "candidate", + }, + }, + }), + ], [ "active tool", (state) => ({ diff --git a/view/canvas/renderFrame.ts b/view/canvas/renderFrame.ts index d82f58e..22a8743 100644 --- a/view/canvas/renderFrame.ts +++ b/view/canvas/renderFrame.ts @@ -1,6 +1,6 @@ import type { Rect, Vec2D } from "@core/geometry"; import type { RenderFrame } from "@renderer/index"; -import type { AppState, BrushPreviewState, BrushStrokePreviewState, EditorState, MaskEditState, SelectionState, ViewportState } from "@editor/state"; +import type { AppState, BrushPreviewState, BrushStrokePreviewState, EditorState, GenerationState, MaskEditState, SelectionState, ViewportState } from "@editor/state"; import type { BrushSettings, InteractionMode } from "@editor/tools"; import type { TransformSession, TransformTarget } from "@editor/transform"; @@ -23,6 +23,7 @@ function visualEditorStatesEqual(a: EditorState, b: EditorState): boolean { maskEditStatesEqual(a.maskEdit, b.maskEdit) && brushPreviewStatesEqual(a.brushPreview, b.brushPreview) && brushStrokePreviewStatesEqual(a.brushStrokePreview, b.brushStrokePreview) && + generationStatesEqual(a.generation, b.generation) && visualToolStatesEqual(a.tools, b.tools) ); } @@ -64,6 +65,10 @@ function brushStrokePreviewStatesEqual(a: BrushStrokePreviewState | undefined, b return a.layerId === b.layerId && a.assetId === b.assetId && a.source === b.source; } +function generationStatesEqual(a: GenerationState, b: GenerationState): boolean { + return a === b; +} + function visualToolStatesEqual(a: EditorState["tools"], b: EditorState["tools"]): boolean { return a.activeTool === b.activeTool && interactionModesEqual(a.interactionMode, b.interactionMode) && brushSettingsEqual(a.brush, b.brush); } diff --git a/view/generate/candidateActions.ts b/view/generate/candidateActions.ts new file mode 100644 index 0000000..36c43c9 --- /dev/null +++ b/view/generate/candidateActions.ts @@ -0,0 +1,50 @@ +import type { ImageDocument } from "@core/document"; +import type { GenerationCandidate } from "@editor/state"; +import { loadImageCanvas, maskValueFromRgba } from "../mask/maskRaster"; + +export async function createMaskedPixelReplacementSource(document: ImageDocument, candidate: GenerationCandidate): Promise { + if (!candidate.inpaint) throw new Error("Only inpaint candidates can replace masked pixels."); + + const targetAsset = document.assets.find((asset) => asset.id === candidate.inpaint?.sourceAssetId); + if (!targetAsset) throw new Error("The source layer for this candidate no longer exists."); + + const targetCanvas = await loadImageCanvas(targetAsset.source, targetAsset.intrinsicSize.w, targetAsset.intrinsicSize.h); + const generatedCanvas = await loadImageCanvas(candidate.source, candidate.width, candidate.height); + const maskCanvas = await loadImageCanvas(candidate.inpaint.maskImage, candidate.width, candidate.height); + + const targetContext = require2dContext(targetCanvas); + const generatedContext = require2dContext(generatedCanvas); + const maskContext = require2dContext(maskCanvas); + const targetData = targetContext.getImageData(0, 0, targetCanvas.width, targetCanvas.height); + const generatedData = generatedContext.getImageData(0, 0, generatedCanvas.width, generatedCanvas.height); + const maskData = maskContext.getImageData(0, 0, maskCanvas.width, maskCanvas.height); + const crop = candidate.inpaint.crop.assetBounds; + + for (let y = 0; y < candidate.height; y += 1) { + for (let x = 0; x < candidate.width; x += 1) { + const targetX = Math.round(crop.x) + x; + const targetY = Math.round(crop.y) + y; + if (targetX < 0 || targetY < 0 || targetX >= targetCanvas.width || targetY >= targetCanvas.height) continue; + + const generatedIndex = (y * generatedCanvas.width + x) * 4; + const targetIndex = (targetY * targetCanvas.width + targetX) * 4; + const mask = maskValueFromRgba(maskData.data, generatedIndex) / 255; + if (mask <= 0) continue; + + for (let channel = 0; channel < 4; channel += 1) { + const previous = targetData.data[targetIndex + channel] ?? 0; + const next = generatedData.data[generatedIndex + channel] ?? previous; + targetData.data[targetIndex + channel] = Math.round(previous * (1 - mask) + next * mask); + } + } + } + + targetContext.putImageData(targetData, 0, 0); + return targetCanvas.toDataURL("image/png"); +} + +function require2dContext(canvas: HTMLCanvasElement): CanvasRenderingContext2D { + const context = canvas.getContext("2d"); + if (!context) throw new Error("Unable to prepare generated candidate"); + return context; +} diff --git a/view/generate/inpaintPrep.test.ts b/view/generate/inpaintPrep.test.ts new file mode 100644 index 0000000..510da21 --- /dev/null +++ b/view/generate/inpaintPrep.test.ts @@ -0,0 +1,36 @@ +import { describe, expect, test } from "bun:test"; +import { applyMaskedContentModeToRgba } from "./inpaintPrep"; + +describe("inpaint prep", () => { + test("converts only masked original pixels to grayscale", () => { + const pixels = new Uint8ClampedArray([ + 255, 0, 0, 255, + 0, 0, 255, 255, + ]); + const mask = new Uint8ClampedArray([255, 0]); + + const next = applyMaskedContentModeToRgba(pixels, 2, 1, mask, "original"); + + expect(next[0]).toBe(next[1]); + expect(next[1]).toBe(next[2]); + expect(Array.from(next.slice(4, 8))).toEqual([0, 0, 255, 255]); + }); + + test("uses an edge map inside the mask without recoloring unmasked context", () => { + const pixels = new Uint8ClampedArray([ + 255, 0, 0, 255, + 0, 255, 0, 255, + 0, 0, 255, 255, + 255, 255, 0, 255, + ]); + const mask = new Uint8ClampedArray([255, 0, 0, 0]); + + const next = applyMaskedContentModeToRgba(pixels, 2, 2, mask, "edges"); + + expect(next[0]).toBe(next[1]); + expect(next[1]).toBe(next[2]); + expect(Array.from(next.slice(4, 8))).toEqual([0, 255, 0, 255]); + expect(Array.from(next.slice(8, 12))).toEqual([0, 0, 255, 255]); + expect(Array.from(next.slice(12, 16))).toEqual([255, 255, 0, 255]); + }); +}); diff --git a/view/generate/inpaintPrep.ts b/view/generate/inpaintPrep.ts new file mode 100644 index 0000000..99b0dc8 --- /dev/null +++ b/view/generate/inpaintPrep.ts @@ -0,0 +1,258 @@ +import type { Asset } from "@core/asset"; +import type { ImageDocument } from "@core/document"; +import type { Rect } from "@core/geometry"; +import type { Layer } from "@core/layer"; +import type { SelectionState } from "@editor/state"; +import type { GenerateSettings } from "@editor/tools"; +import { createDocumentReadIndex, resolveIndexedLayerBounds } from "@editor/document-indexes"; +import { createNormalizedMaskSource, cropCanvas, cropMaskValuesToDataUrl, expandRectWithinBounds, loadImageCanvas } from "../mask/maskRaster"; + +export type InpaintBundle = { + inputImage: string; + maskImage: string; + width: number; + height: number; + targetLayerId: string; + maskLayerId: string; + sourceAssetId: string; + maskAssetId: string; + crop: { + assetBounds: Rect; + documentBounds: Rect; + padding: number; + maskedAreaOnly: boolean; + }; + mask: { + polarity: GenerateSettings["inpaint"]["maskPolarity"]; + activeBounds: Rect; + }; + placement: { + artboardId: string; + layerName: string; + transform: Extract["transform"]; + }; + backend: { + growMaskBy: number; + maskedContent: GenerateSettings["inpaint"]["maskedContent"]; + maskBlur: number; + maskFeather: number; + maskExpand: number; + cropPadding: number; + }; +}; + +type InpaintTarget = { + artboardId: string; + layer: Extract; + asset: Asset; + bounds: Rect; + maskLayer: Extract; + maskAsset: Asset; + maskBounds: Rect; +}; + +const modelMultiple = 8; +const minModelSize = 64; +const maxModelSize = 4096; + +export async function buildInpaintBundle(document: ImageDocument, selection: SelectionState, settings: GenerateSettings): Promise { + const target = resolveInpaintTarget(document, selection); + validateInpaintTarget(target); + + const width = Math.max(1, Math.round(target.asset.intrinsicSize.w)); + const height = Math.max(1, Math.round(target.asset.intrinsicSize.h)); + const normalizedMask = await createNormalizedMaskSource(target.maskAsset.source, width, height, { + polarity: settings.inpaint.maskPolarity, + expand: settings.inpaint.maskExpand, + feather: settings.inpaint.maskFeather, + blur: settings.inpaint.maskBlur, + despeckle: settings.inpaint.maskDespeckle, + }); + + if (!normalizedMask.bounds) throw new Error("The selected layer mask has no inpaint pixels."); + + const crop = settings.inpaint.maskedAreaOnly + ? expandRectWithinBounds(normalizedMask.bounds, settings.inpaint.cropPadding, { w: width, h: height }, modelMultiple, minModelSize) + : { x: 0, y: 0, w: width, h: height }; + const outputWidth = toModelSize(crop.w, "width"); + const outputHeight = toModelSize(crop.h, "height"); + + const inputCanvas = prepareMaskedContentInputCanvas(await loadImageCanvas(target.asset.source, width, height), normalizedMask.values, settings.inpaint.maskedContent); + const inputImage = cropCanvas(inputCanvas, crop, outputWidth, outputHeight); + const maskImage = cropMaskValuesToDataUrl(normalizedMask.values, width, height, crop, outputWidth, outputHeight); + const scaleX = target.bounds.w / width; + const scaleY = target.bounds.h / height; + const documentBounds = { + x: target.bounds.x + crop.x * scaleX, + y: target.bounds.y + crop.y * scaleY, + w: outputWidth * scaleX, + h: outputHeight * scaleY, + }; + + return { + inputImage, + maskImage, + width: outputWidth, + height: outputHeight, + targetLayerId: target.layer.id, + maskLayerId: target.maskLayer.id, + sourceAssetId: target.asset.id, + maskAssetId: target.maskAsset.id, + crop: { + assetBounds: crop, + documentBounds, + padding: settings.inpaint.cropPadding, + maskedAreaOnly: settings.inpaint.maskedAreaOnly, + }, + mask: { + polarity: settings.inpaint.maskPolarity, + activeBounds: normalizedMask.bounds, + }, + placement: { + artboardId: target.artboardId, + layerName: `${target.layer.name} inpaint`, + transform: { + position: { x: documentBounds.x, y: documentBounds.y }, + scale: { x: documentBounds.w / outputWidth, y: documentBounds.h / outputHeight }, + rotation: target.layer.transform.rotation, + }, + }, + backend: { + growMaskBy: settings.inpaint.growMaskBy, + maskedContent: settings.inpaint.maskedContent, + maskBlur: settings.inpaint.maskBlur, + maskFeather: settings.inpaint.maskFeather, + maskExpand: settings.inpaint.maskExpand, + cropPadding: settings.inpaint.cropPadding, + }, + }; +} + +function resolveInpaintTarget(document: ImageDocument, selection: SelectionState): InpaintTarget { + if (selection.layerIds.length !== 1 || !selection.layerIds[0]) throw new Error("Select one image or raster layer to inpaint."); + + const documentIndex = createDocumentReadIndex(document); + const layerInfo = documentIndex.layerInfoById.get(selection.layerIds[0]); + if (!layerInfo || layerInfo.layer.type === "group") throw new Error("Select one image or raster layer to inpaint."); + + const asset = documentIndex.assetById.get(layerInfo.layer.assetId); + if (!asset) throw new Error("The selected layer is missing its source image."); + if (!layerInfo.layer.clippingMask) throw new Error("Add a layer mask before running inpaint."); + + const maskLayer = documentIndex.layerById.get(layerInfo.layer.clippingMask.maskLayerId); + if (!maskLayer || maskLayer.type === "group") throw new Error("The selected layer mask is missing."); + + const maskAsset = documentIndex.assetById.get(maskLayer.assetId); + if (!maskAsset) throw new Error("The selected layer mask is missing its image data."); + + const bounds = resolveIndexedLayerBounds(documentIndex, layerInfo.layer); + const maskBounds = resolveIndexedLayerBounds(documentIndex, maskLayer); + if (!bounds || !maskBounds) throw new Error("Unable to resolve the selected layer and mask bounds."); + + return { artboardId: layerInfo.artboardId, layer: layerInfo.layer, asset, bounds, maskLayer, maskAsset, maskBounds }; +} + +function validateInpaintTarget(target: InpaintTarget) { + if (target.asset.intrinsicSize.w <= 0 || target.asset.intrinsicSize.h <= 0) throw new Error("The selected image has an invalid size."); + if (target.maskAsset.intrinsicSize.w <= 0 || target.maskAsset.intrinsicSize.h <= 0) throw new Error("The selected mask has an invalid size."); + if (Math.round(target.asset.intrinsicSize.w) !== Math.round(target.maskAsset.intrinsicSize.w) || Math.round(target.asset.intrinsicSize.h) !== Math.round(target.maskAsset.intrinsicSize.h)) { + throw new Error("The selected layer and mask image sizes do not match."); + } + if (!rectsAligned(target.bounds, target.maskBounds) || Math.abs(target.layer.transform.rotation - target.maskLayer.transform.rotation) > 0.001) { + throw new Error("The selected layer and mask are not aligned."); + } +} + +function rectsAligned(a: Rect, b: Rect): boolean { + return Math.abs(a.x - b.x) <= 0.5 && Math.abs(a.y - b.y) <= 0.5 && Math.abs(a.w - b.w) <= 0.5 && Math.abs(a.h - b.h) <= 0.5; +} + +function toModelSize(value: number, axis: "width" | "height"): number { + const rounded = Math.max(minModelSize, Math.ceil(value / modelMultiple) * modelMultiple); + if (rounded > maxModelSize) throw new Error(`The inpaint ${axis} is too large for the model. Use masked-area inpaint or a smaller source.`); + return rounded; +} + +function prepareMaskedContentInputCanvas(sourceCanvas: HTMLCanvasElement, maskValues: Uint8ClampedArray, mode: GenerateSettings["inpaint"]["maskedContent"]): HTMLCanvasElement { + if (mode === "neutral" || mode === "originalColor") return sourceCanvas; + const context = sourceCanvas.getContext("2d"); + if (!context) return sourceCanvas; + const imageData = context.getImageData(0, 0, sourceCanvas.width, sourceCanvas.height); + imageData.data.set(applyMaskedContentModeToRgba(imageData.data, sourceCanvas.width, sourceCanvas.height, maskValues, mode)); + context.putImageData(imageData, 0, 0); + return sourceCanvas; +} + +export function applyMaskedContentModeToRgba(data: Uint8ClampedArray, width: number, height: number, maskValues: Uint8ClampedArray, mode: GenerateSettings["inpaint"]["maskedContent"]): Uint8ClampedArray { + const next = new Uint8ClampedArray(data); + if (mode === "neutral" || mode === "originalColor") return next; + + const edgeValues = mode === "edges" ? createEdgeMapValues(data, width, height) : undefined; + + for (let pixel = 0; pixel < width * height; pixel += 1) { + const amount = (maskValues[pixel] ?? 0) / 255; + if (amount <= 0) continue; + + const index = pixel * 4; + const red = data[index] ?? 0; + const green = data[index + 1] ?? 0; + const blue = data[index + 2] ?? 0; + const target = edgeValues ? edgeValues[pixel] ?? 128 : luminance(red, green, blue); + next[index] = blendChannel(red, target, amount); + next[index + 1] = blendChannel(green, target, amount); + next[index + 2] = blendChannel(blue, target, amount); + } + + return next; +} + +function createEdgeMapValues(data: Uint8ClampedArray, width: number, height: number): Uint8ClampedArray { + const gray = new Float32Array(width * height); + const edges = new Uint8ClampedArray(width * height); + + for (let pixel = 0; pixel < width * height; pixel += 1) { + const index = pixel * 4; + gray[pixel] = luminance(data[index] ?? 0, data[index + 1] ?? 0, data[index + 2] ?? 0); + } + + for (let y = 0; y < height; y += 1) { + for (let x = 0; x < width; x += 1) { + const gx = + -sampleGray(gray, width, height, x - 1, y - 1) + + sampleGray(gray, width, height, x + 1, y - 1) - + 2 * sampleGray(gray, width, height, x - 1, y) + + 2 * sampleGray(gray, width, height, x + 1, y) - + sampleGray(gray, width, height, x - 1, y + 1) + + sampleGray(gray, width, height, x + 1, y + 1); + const gy = + -sampleGray(gray, width, height, x - 1, y - 1) - + 2 * sampleGray(gray, width, height, x, y - 1) - + sampleGray(gray, width, height, x + 1, y - 1) + + sampleGray(gray, width, height, x - 1, y + 1) + + 2 * sampleGray(gray, width, height, x, y + 1) + + sampleGray(gray, width, height, x + 1, y + 1); + const magnitude = Math.hypot(gx, gy); + edges[y * width + x] = Math.round(clampNumber(128 + Math.max(0, magnitude - 24) * 0.75, 128, 255)); + } + } + + return edges; +} + +function sampleGray(values: Float32Array, width: number, height: number, x: number, y: number): number { + const clampedX = Math.max(0, Math.min(width - 1, x)); + const clampedY = Math.max(0, Math.min(height - 1, y)); + return values[clampedY * width + clampedX] ?? 0; +} + +function luminance(red: number, green: number, blue: number): number { + return Math.round(0.2126 * red + 0.7152 * green + 0.0722 * blue); +} + +function blendChannel(previous: number, next: number, amount: number): number { + return Math.round(previous * (1 - amount) + next * amount); +} + +function clampNumber(value: number, min: number, max: number): number { + return Math.max(min, Math.min(max, value)); +} diff --git a/view/generate/runGenerate.ts b/view/generate/runGenerate.ts index 852303e..09aeee1 100644 --- a/view/generate/runGenerate.ts +++ b/view/generate/runGenerate.ts @@ -1,9 +1,11 @@ import { commandIds } from "@commands/ids"; import type { ImageDocument } from "@core/document"; +import type { Transform } from "@core/geometry"; import type { Layer } from "@core/layer"; import type { AppStore } from "@editor/store"; -import type { SelectionState, ViewportState } from "@editor/state"; +import type { GenerationCandidate, SelectionState, ViewportState } from "@editor/state"; import type { GenerateSettings } from "@editor/tools"; +import { buildInpaintBundle, type InpaintBundle } from "./inpaintPrep"; export async function runGenerate(options: { document: ImageDocument; @@ -17,50 +19,200 @@ export async function runGenerate(options: { if (!artboard) return; const target = resolveSelectedImage(document, selection); - const inputImage = target && settings.mode !== "text-to-image" ? await imageSourceToDataUrl(target.asset.source) : undefined; - const maskImage = target?.maskAsset && settings.mode === "inpaint" ? await imageSourceToDataUrl(target.maskAsset.source) : undefined; + const inpaintBundle = settings.mode === "inpaint" ? await buildInpaintBundle(document, selection, settings) : undefined; + const inputImage = inpaintBundle?.inputImage ?? (target && settings.mode !== "text-to-image" ? await imageSourceToDataUrl(target.asset.source) : undefined); + const maskImage = inpaintBundle?.maskImage; + const seed = resolveSeed(settings.seed); + const requestSettings = { ...settings, seed }; + const width = inpaintBundle?.width ?? settings.width; + const height = inpaintBundle?.height ?? settings.height; + const generated = await requestGenerate({ + settings: requestSettings, + width, + height, + inputImage, + maskImage, + inpaintBundle, + }); + const intrinsicSize = await loadImageSize(generated.source); + const targetArtboardId = inpaintBundle?.placement.artboardId ?? artboard.id; + const placement = { + artboardId: targetArtboardId, + layerName: inpaintBundle?.placement.layerName ?? "Generated image", + transform: generatedLayerTransform(inpaintBundle, intrinsicSize), + }; + + dispatch(commandIds.generationAddCandidate, { + candidate: createGenerationCandidate({ + source: generated.source, + mimeType: generated.mimeType, + intrinsicSize, + settings: requestSettings, + seed, + width, + height, + inputImage, + maskImage, + placement, + inpaintBundle, + }), + }); +} + +export async function runGenerateFromCandidate(options: { + candidate: GenerationCandidate; + settings?: GenerateSettings; + dispatch: AppStore["dispatch"]; +}) { + const settings = options.settings ?? options.candidate.settings; + const seed = resolveSeed(settings.seed); + const requestSettings = { ...settings, seed }; + const generated = await requestGenerate({ + settings: requestSettings, + width: options.candidate.width, + height: options.candidate.height, + inputImage: options.candidate.inputImage, + maskImage: options.candidate.maskImage, + inpaintCandidate: options.candidate, + }); + const intrinsicSize = await loadImageSize(generated.source); + + options.dispatch(commandIds.generationAddCandidate, { + candidate: { + ...options.candidate, + id: crypto.randomUUID(), + source: generated.source, + mimeType: generated.mimeType, + intrinsicSize, + settings: requestSettings, + seed, + }, + }); +} + +function createGenerationCandidate(options: { + source: string; + mimeType: string; + intrinsicSize: { w: number; h: number }; + settings: GenerateSettings; + seed: number; + width: number; + height: number; + inputImage?: string; + maskImage?: string; + placement: GenerationCandidate["placement"]; + inpaintBundle?: InpaintBundle; +}): GenerationCandidate { + return { + id: crypto.randomUUID(), + source: options.source, + mimeType: options.mimeType, + intrinsicSize: options.intrinsicSize, + mode: options.settings.mode, + settings: options.settings, + seed: options.seed, + width: options.width, + height: options.height, + inputImage: options.inputImage, + maskImage: options.maskImage, + placement: options.placement, + inpaint: options.inpaintBundle + ? { + targetLayerId: options.inpaintBundle.targetLayerId, + maskLayerId: options.inpaintBundle.maskLayerId, + sourceAssetId: options.inpaintBundle.sourceAssetId, + maskAssetId: options.inpaintBundle.maskAssetId, + inputImage: options.inpaintBundle.inputImage, + maskImage: options.inpaintBundle.maskImage, + crop: options.inpaintBundle.crop, + mask: options.inpaintBundle.mask, + backend: options.inpaintBundle.backend, + } + : undefined, + }; +} + +async function requestGenerate(options: { + settings: GenerateSettings; + width: number; + height: number; + inputImage?: string; + maskImage?: string; + inpaintBundle?: InpaintBundle; + inpaintCandidate?: GenerationCandidate; +}) { const response = await fetch("/api/comfy/generate", { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ - mode: settings.mode, - model: settings.model, - prompt: settings.prompt, - negativePrompt: settings.negativePrompt, - strength: settings.strength, - steps: settings.steps, - cfg: settings.cfg, - seed: settings.seed, - sampler: settings.sampler, - scheduler: settings.scheduler, - width: settings.width, - height: settings.height, - outpaint: settings.outpaint, - inputImage, - maskImage, + mode: options.settings.mode, + model: options.settings.model, + prompt: options.settings.prompt, + negativePrompt: options.settings.negativePrompt, + strength: options.settings.strength, + steps: options.settings.steps, + cfg: options.settings.cfg, + seed: options.settings.seed, + sampler: options.settings.sampler, + scheduler: options.settings.scheduler, + width: options.width, + height: options.height, + outpaint: options.settings.outpaint, + inpaint: resolveInpaintRequest(options.inpaintBundle, options.inpaintCandidate, options.settings), + inputImage: options.inputImage, + maskImage: options.maskImage, }), }); if (!response.ok) throw new Error(await response.text()); - const generated = await response.json() as { source: string; mimeType: string }; - const intrinsicSize = await loadImageSize(generated.source); - const assetId = crypto.randomUUID(); - const layerId = crypto.randomUUID(); - dispatch(commandIds.documentAddAsset, { asset: { id: assetId, name: "Generated image", mimeType: generated.mimeType, source: generated.source, intrinsicSize } }); - dispatch(commandIds.documentAddImageLayer, { - artboardId: artboard.id, - layer: { - id: layerId, - type: "image", - name: "Generated image", - visible: true, - locked: false, - opacity: 1, - assetId, - transform: { position: { x: 0, y: 0 }, scale: { x: 1, y: 1 }, rotation: 0 }, + return await response.json() as { source: string; mimeType: string }; +} + +function resolveInpaintRequest(inpaintBundle: InpaintBundle | undefined, inpaintCandidate: GenerationCandidate | undefined, settings: GenerateSettings) { + if (inpaintBundle) { + return { + growMaskBy: inpaintBundle.backend.growMaskBy, + maskedContent: inpaintBundle.backend.maskedContent, + maskBlur: inpaintBundle.backend.maskBlur, + maskFeather: inpaintBundle.backend.maskFeather, + maskExpand: inpaintBundle.backend.maskExpand, + cropPadding: inpaintBundle.backend.cropPadding, + maskPolarity: inpaintBundle.mask.polarity, + crop: inpaintBundle.crop, + placement: inpaintBundle.placement, + }; + } + + if (inpaintCandidate?.inpaint) { + return { + growMaskBy: inpaintCandidate.inpaint.backend.growMaskBy, + maskedContent: inpaintCandidate.inpaint.backend.maskedContent, + maskBlur: inpaintCandidate.inpaint.backend.maskBlur, + maskFeather: inpaintCandidate.inpaint.backend.maskFeather, + maskExpand: inpaintCandidate.inpaint.backend.maskExpand, + cropPadding: inpaintCandidate.inpaint.backend.cropPadding, + maskPolarity: inpaintCandidate.inpaint.mask.polarity, + crop: inpaintCandidate.inpaint.crop, + placement: inpaintCandidate.placement, + }; + } + + return settings.inpaint; +} + +function generatedLayerTransform(inpaintBundle: InpaintBundle | undefined, intrinsicSize: { w: number; h: number }): Transform { + if (!inpaintBundle) return { position: { x: 0, y: 0 }, scale: { x: 1, y: 1 }, rotation: 0 }; + return { + position: { ...inpaintBundle.placement.transform.position }, + scale: { + x: (inpaintBundle.placement.transform.scale.x * inpaintBundle.width) / Math.max(1, intrinsicSize.w), + y: (inpaintBundle.placement.transform.scale.y * inpaintBundle.height) / Math.max(1, intrinsicSize.h), }, - }); - dispatch(commandIds.documentMoveLayer, { layerId, toArtboardId: artboard.id, toIndex: 0 }); - dispatch(commandIds.selectionSet, { artboardId: artboard.id, layerIds: [layerId] }); + rotation: inpaintBundle.placement.transform.rotation, + }; +} + +function resolveSeed(seed: number): number { + return seed < 0 ? Math.floor(Math.random() * 2 ** 32) : Math.round(seed); } function resolveSelectedImage(document: ImageDocument, selection: SelectionState) { diff --git a/view/mask/maskRaster.test.ts b/view/mask/maskRaster.test.ts new file mode 100644 index 0000000..e8ed2a4 --- /dev/null +++ b/view/mask/maskRaster.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, test } from "bun:test"; +import { cropMaskValuesToRgba, expandRectWithinBounds, invertMaskValues } from "./maskRaster"; + +describe("mask raster utilities", () => { + test("exports the normalized drawn mask without filling the whole crop", () => { + const revealedMask = new Uint8ClampedArray(4 * 3).fill(255); + revealedMask[1 * 4 + 2] = 0; + + const inpaintMask = invertMaskValues(revealedMask); + const rgba = cropMaskValuesToRgba(inpaintMask, 4, 3, { x: 1, y: 0, w: 3, h: 3 }, 4, 4); + const activePixels = activeRedPixels(rgba); + + expect(activePixels).toEqual([{ x: 1, y: 1 }]); + for (let pixel = 0; pixel < rgba.length / 4; pixel += 1) expect(rgba[pixel * 4 + 3]).toBe(255); + }); + + test("expands masked-area crops to the model minimum when possible", () => { + expect(expandRectWithinBounds({ x: 20, y: 20, w: 4, h: 4 }, 4, { w: 100, h: 100 }, 8, 64)).toEqual({ + x: 0, + y: 0, + w: 64, + h: 64, + }); + expect(expandRectWithinBounds({ x: 80, y: 80, w: 4, h: 4 }, 4, { w: 100, h: 100 }, 8, 64)).toEqual({ + x: 36, + y: 36, + w: 64, + h: 64, + }); + }); +}); + +function activeRedPixels(rgba: Uint8ClampedArray) { + const width = 4; + const pixels: Array<{ x: number; y: number }> = []; + for (let pixel = 0; pixel < rgba.length / 4; pixel += 1) { + if ((rgba[pixel * 4] ?? 0) > 127) pixels.push({ x: pixel % width, y: Math.floor(pixel / width) }); + } + return pixels; +} diff --git a/view/mask/maskRaster.ts b/view/mask/maskRaster.ts new file mode 100644 index 0000000..7a7f55b --- /dev/null +++ b/view/mask/maskRaster.ts @@ -0,0 +1,374 @@ +import type { Rect } from "@core/geometry"; + +export type MaskFill = "white" | "black" | "clear"; + +export type MaskRasterOperation = + | { type: "invert" } + | { type: "fill"; fill: MaskFill } + | { type: "feather"; radius: number } + | { type: "expand"; radius: number } + | { type: "contract"; radius: number } + | { type: "blur"; radius: number } + | { type: "despeckle"; strength: number }; + +export type MaskAnalysis = { + width: number; + height: number; + revealedPixels: number; + hiddenPixels: number; + coverage: number; + hiddenCoverage: number; + bounds?: Rect; + hiddenBounds?: Rect; + thumbnail: string; +}; + +export type NormalizedMaskOptions = { + polarity: "hidden" | "revealed"; + expand?: number; + feather?: number; + blur?: number; + despeckle?: number; +}; + +export async function createSolidMaskSource(width: number, height: number, fill: MaskFill): Promise { + const canvas = createCanvas(width, height); + const context = require2dContext(canvas); + context.clearRect(0, 0, canvas.width, canvas.height); + + if (fill === "white") { + context.fillStyle = "#ffffff"; + context.fillRect(0, 0, canvas.width, canvas.height); + } else if (fill === "black") { + context.fillStyle = "#000000"; + context.fillRect(0, 0, canvas.width, canvas.height); + } + + return canvas.toDataURL("image/png"); +} + +export async function applyMaskRasterOperation(source: string, width: number, height: number, operation: MaskRasterOperation): Promise { + if (operation.type === "fill") return createSolidMaskSource(width, height, operation.fill); + + const mask = await loadMaskValues(source, width, height); + const next = applyMaskValueOperation(mask.values, mask.width, mask.height, operation); + return maskValuesToDataUrl(next, mask.width, mask.height); +} + +export async function analyzeMaskSource(source: string, width: number, height: number): Promise { + const mask = await loadMaskValues(source, width, height); + const revealed = analyzeValues(mask.values, mask.width, mask.height, false); + const hiddenValues = invertMaskValues(mask.values); + const hidden = analyzeValues(hiddenValues, mask.width, mask.height, false); + return { + width: mask.width, + height: mask.height, + revealedPixels: revealed.pixels, + hiddenPixels: hidden.pixels, + coverage: revealed.coverage, + hiddenCoverage: hidden.coverage, + bounds: revealed.bounds, + hiddenBounds: hidden.bounds, + thumbnail: maskValuesToDataUrl(mask.values, mask.width, mask.height, 72, 48), + }; +} + +export async function createNormalizedMaskSource(source: string, width: number, height: number, options: NormalizedMaskOptions): Promise<{ source: string; values: Uint8ClampedArray; bounds?: Rect }> { + const mask = await loadMaskValues(source, width, height); + let values = options.polarity === "hidden" ? invertMaskValues(mask.values) : new Uint8ClampedArray(mask.values); + + const despeckle = Math.round(clampNumber(options.despeckle ?? 0, 0, 64)); + const expand = Math.round(clampNumber(options.expand ?? 0, -256, 256)); + const feather = Math.round(clampNumber(options.feather ?? 0, 0, 256)); + const blur = Math.round(clampNumber(options.blur ?? 0, 0, 256)); + + if (despeckle > 0) values = despeckleMaskValues(values, mask.width, mask.height, despeckle); + if (expand > 0) values = dilateMaskValues(values, mask.width, mask.height, expand); + if (expand < 0) values = erodeMaskValues(values, mask.width, mask.height, -expand); + if (feather > 0) values = blurMaskValues(values, mask.width, mask.height, feather); + if (blur > 0) values = blurMaskValues(values, mask.width, mask.height, blur); + + const analysis = analyzeValues(values, mask.width, mask.height, false); + return { source: maskValuesToDataUrl(values, mask.width, mask.height), values, bounds: analysis.bounds }; +} + +export async function loadImageCanvas(source: string, width?: number, height?: number): Promise { + const image = await loadImage(source); + const canvas = createCanvas(width ?? image.naturalWidth, height ?? image.naturalHeight); + const context = require2dContext(canvas); + context.clearRect(0, 0, canvas.width, canvas.height); + context.drawImage(image, 0, 0, canvas.width, canvas.height); + return canvas; +} + +export async function imageSourceToPngDataUrl(source: string): Promise { + if (source.startsWith("data:image/png;base64,")) return source; + const canvas = await loadImageCanvas(source); + return canvas.toDataURL("image/png"); +} + +export function cropCanvas(sourceCanvas: HTMLCanvasElement, crop: Rect, outputWidth = crop.w, outputHeight = crop.h): string { + const canvas = createCanvas(outputWidth, outputHeight); + const context = require2dContext(canvas); + context.clearRect(0, 0, outputWidth, outputHeight); + context.drawImage(sourceCanvas, crop.x, crop.y, crop.w, crop.h, 0, 0, crop.w, crop.h); + return canvas.toDataURL("image/png"); +} + +export function cropMaskValuesToDataUrl(values: Uint8ClampedArray, width: number, height: number, crop: Rect, outputWidth = crop.w, outputHeight = crop.h): string { + const canvas = createCanvas(outputWidth, outputHeight); + const context = require2dContext(canvas); + const imageData = context.createImageData(outputWidth, outputHeight); + imageData.data.set(cropMaskValuesToRgba(values, width, height, crop, outputWidth, outputHeight)); + context.putImageData(imageData, 0, 0); + return canvas.toDataURL("image/png"); +} + +export function cropMaskValuesToRgba(values: Uint8ClampedArray, width: number, height: number, crop: Rect, outputWidth = crop.w, outputHeight = crop.h): Uint8ClampedArray { + const safeOutputWidth = Math.max(1, Math.round(outputWidth)); + const safeOutputHeight = Math.max(1, Math.round(outputHeight)); + const data = new Uint8ClampedArray(safeOutputWidth * safeOutputHeight * 4); + for (let pixel = 0; pixel < safeOutputWidth * safeOutputHeight; pixel += 1) data[pixel * 4 + 3] = 255; + + for (let y = 0; y < Math.min(crop.h, safeOutputHeight); y += 1) { + for (let x = 0; x < Math.min(crop.w, safeOutputWidth); x += 1) { + const sourceX = crop.x + x; + const sourceY = crop.y + y; + if (sourceX < 0 || sourceY < 0 || sourceX >= width || sourceY >= height) continue; + const value = values[sourceY * width + sourceX] ?? 0; + const index = (y * safeOutputWidth + x) * 4; + data[index] = value; + data[index + 1] = value; + data[index + 2] = value; + } + } + + return data; +} + +export function expandRectWithinBounds(rect: Rect, padding: number, bounds: { w: number; h: number }, multiple = 1, minSize = 1): Rect { + const padded = Math.max(0, Math.round(padding)); + let x1 = Math.max(0, Math.floor(rect.x) - padded); + let y1 = Math.max(0, Math.floor(rect.y) - padded); + let x2 = Math.min(bounds.w, Math.ceil(rect.x + rect.w) + padded); + let y2 = Math.min(bounds.h, Math.ceil(rect.y + rect.h) + padded); + + const safeMinSize = Math.max(1, Math.round(minSize)); + const targetWidth = Math.min(bounds.w, roundUp(Math.max(safeMinSize, x2 - x1), multiple)); + const targetHeight = Math.min(bounds.h, roundUp(Math.max(safeMinSize, y2 - y1), multiple)); + + const extraWidth = targetWidth - (x2 - x1); + const extraHeight = targetHeight - (y2 - y1); + x1 = Math.max(0, x1 - Math.floor(extraWidth / 2)); + y1 = Math.max(0, y1 - Math.floor(extraHeight / 2)); + x2 = Math.min(bounds.w, x1 + targetWidth); + y2 = Math.min(bounds.h, y1 + targetHeight); + x1 = Math.max(0, x2 - targetWidth); + y1 = Math.max(0, y2 - targetHeight); + + return { x: x1, y: y1, w: Math.max(1, x2 - x1), h: Math.max(1, y2 - y1) }; +} + +export function maskValueFromRgba(data: Uint8ClampedArray, index: number): number { + const red = data[index] ?? 0; + const green = data[index + 1] ?? 0; + const blue = data[index + 2] ?? 0; + const alpha = data[index + 3] ?? 0; + const luminance = 0.2126 * red + 0.7152 * green + 0.0722 * blue; + return Math.round((alpha * luminance) / 255); +} + +export function applyMaskValueOperation(values: Uint8ClampedArray, width: number, height: number, operation: Exclude): Uint8ClampedArray { + switch (operation.type) { + case "invert": + return invertMaskValues(values); + case "feather": + case "blur": + return blurMaskValues(values, width, height, Math.round(clampNumber(operation.radius, 0, 256))); + case "expand": + return dilateMaskValues(values, width, height, Math.round(clampNumber(operation.radius, 0, 256))); + case "contract": + return erodeMaskValues(values, width, height, Math.round(clampNumber(operation.radius, 0, 256))); + case "despeckle": + return despeckleMaskValues(values, width, height, Math.round(clampNumber(operation.strength, 0, 64))); + } +} + +export function invertMaskValues(values: Uint8ClampedArray): Uint8ClampedArray { + const next = new Uint8ClampedArray(values.length); + for (let index = 0; index < values.length; index += 1) next[index] = 255 - (values[index] ?? 0); + return next; +} + +export function erodeMaskValues(values: Uint8ClampedArray, width: number, height: number, radius: number): Uint8ClampedArray { + const safeRadius = Math.round(clampNumber(radius, 0, 256)); + if (safeRadius <= 0) return new Uint8ClampedArray(values); + const next = new Uint8ClampedArray(values.length); + for (let y = 0; y < height; y += 1) { + for (let x = 0; x < width; x += 1) { + let value = 255; + for (let oy = -safeRadius; oy <= safeRadius; oy += 1) { + for (let ox = -safeRadius; ox <= safeRadius; ox += 1) value = Math.min(value, values[clampInt(y + oy, 0, height - 1) * width + clampInt(x + ox, 0, width - 1)] ?? 0); + } + next[y * width + x] = value; + } + } + return next; +} + +export function dilateMaskValues(values: Uint8ClampedArray, width: number, height: number, radius: number): Uint8ClampedArray { + const safeRadius = Math.round(clampNumber(radius, 0, 256)); + if (safeRadius <= 0) return new Uint8ClampedArray(values); + const next = new Uint8ClampedArray(values.length); + for (let y = 0; y < height; y += 1) { + for (let x = 0; x < width; x += 1) { + let value = 0; + for (let oy = -safeRadius; oy <= safeRadius; oy += 1) { + for (let ox = -safeRadius; ox <= safeRadius; ox += 1) value = Math.max(value, values[clampInt(y + oy, 0, height - 1) * width + clampInt(x + ox, 0, width - 1)] ?? 0); + } + next[y * width + x] = value; + } + } + return next; +} + +export function blurMaskValues(values: Uint8ClampedArray, width: number, height: number, radius: number): Uint8ClampedArray { + const safeRadius = Math.round(clampNumber(radius, 0, 256)); + if (safeRadius <= 0) return new Uint8ClampedArray(values); + const next = new Uint8ClampedArray(values.length); + for (let y = 0; y < height; y += 1) { + for (let x = 0; x < width; x += 1) { + let total = 0; + let count = 0; + for (let oy = -safeRadius; oy <= safeRadius; oy += 1) { + for (let ox = -safeRadius; ox <= safeRadius; ox += 1) { + total += values[clampInt(y + oy, 0, height - 1) * width + clampInt(x + ox, 0, width - 1)] ?? 0; + count += 1; + } + } + next[y * width + x] = Math.round(total / count); + } + } + return next; +} + +export function despeckleMaskValues(values: Uint8ClampedArray, width: number, height: number, strength: number): Uint8ClampedArray { + const safeStrength = Math.round(clampNumber(strength, 0, 64)); + if (safeStrength <= 0) return new Uint8ClampedArray(values); + + const radius = Math.max(1, Math.ceil(safeStrength / 6)); + const threshold = Math.max(1, Math.round(safeStrength / 2)); + const next = new Uint8ClampedArray(values); + + for (let y = 0; y < height; y += 1) { + for (let x = 0; x < width; x += 1) { + const index = y * width + x; + const visible = (values[index] ?? 0) > 127; + let same = 0; + for (let oy = -radius; oy <= radius; oy += 1) { + for (let ox = -radius; ox <= radius; ox += 1) { + if (ox === 0 && oy === 0) continue; + const sample = values[clampInt(y + oy, 0, height - 1) * width + clampInt(x + ox, 0, width - 1)] ?? 0; + if ((sample > 127) === visible) same += 1; + } + } + if (same <= threshold) next[index] = visible ? 0 : 255; + } + } + + return next; +} + +async function loadMaskValues(source: string, width: number, height: number): Promise<{ width: number; height: number; values: Uint8ClampedArray }> { + const canvas = await loadImageCanvas(source, width, height); + const context = require2dContext(canvas); + const data = context.getImageData(0, 0, canvas.width, canvas.height); + const values = new Uint8ClampedArray(canvas.width * canvas.height); + for (let pixel = 0; pixel < values.length; pixel += 1) values[pixel] = maskValueFromRgba(data.data, pixel * 4); + return { width: canvas.width, height: canvas.height, values }; +} + +function maskValuesToDataUrl(values: Uint8ClampedArray, width: number, height: number, outputWidth = width, outputHeight = height): string { + const canvas = createCanvas(outputWidth, outputHeight); + const context = require2dContext(canvas); + const imageData = context.createImageData(outputWidth, outputHeight); + + for (let y = 0; y < outputHeight; y += 1) { + for (let x = 0; x < outputWidth; x += 1) { + const sourceX = Math.floor((x / outputWidth) * width); + const sourceY = Math.floor((y / outputHeight) * height); + const value = values[clampInt(sourceY, 0, height - 1) * width + clampInt(sourceX, 0, width - 1)] ?? 0; + const index = (y * outputWidth + x) * 4; + imageData.data[index] = 255; + imageData.data[index + 1] = 255; + imageData.data[index + 2] = 255; + imageData.data[index + 3] = value; + } + } + + context.putImageData(imageData, 0, 0); + return canvas.toDataURL("image/png"); +} + +function analyzeValues(values: Uint8ClampedArray, width: number, height: number, includeSoftPixels: boolean): { pixels: number; coverage: number; bounds?: Rect } { + let pixels = 0; + let minX = Number.POSITIVE_INFINITY; + let minY = Number.POSITIVE_INFINITY; + let maxX = Number.NEGATIVE_INFINITY; + let maxY = Number.NEGATIVE_INFINITY; + + for (let y = 0; y < height; y += 1) { + for (let x = 0; x < width; x += 1) { + const value = values[y * width + x] ?? 0; + const active = includeSoftPixels ? value > 0 : value > 127; + if (!active) continue; + pixels += 1; + minX = Math.min(minX, x); + minY = Math.min(minY, y); + maxX = Math.max(maxX, x + 1); + maxY = Math.max(maxY, y + 1); + } + } + + return { + pixels, + coverage: pixels / Math.max(1, width * height), + bounds: pixels > 0 ? { x: minX, y: minY, w: maxX - minX, h: maxY - minY } : undefined, + }; +} + +function createCanvas(width: number, height: number): HTMLCanvasElement { + const canvas = document.createElement("canvas"); + canvas.width = Math.max(1, Math.round(width)); + canvas.height = Math.max(1, Math.round(height)); + return canvas; +} + +function require2dContext(canvas: HTMLCanvasElement): CanvasRenderingContext2D { + const context = canvas.getContext("2d"); + if (!context) throw new Error("Unable to create mask canvas"); + return context; +} + +function loadImage(source: string): Promise { + return new Promise((resolve, reject) => { + const image = new Image(); + image.onload = () => resolve(image); + image.onerror = () => reject(new Error("Failed to load image")); + image.src = source; + }); +} + +function roundUp(value: number, multiple: number): number { + const safeMultiple = Math.max(1, Math.round(multiple)); + return Math.ceil(value / safeMultiple) * safeMultiple; +} + +function clampNumber(value: number, min: number, max: number): number { + if (!Number.isFinite(value)) return min; + return Math.max(min, Math.min(max, value)); +} + +function clampInt(value: number, min: number, max: number): number { + return Math.round(Math.max(min, Math.min(max, value))); +}