feat: implement inpainting functionality with mask handling

- Added `createMaskedPixelReplacementSource` function to handle pixel replacement using inpainting.
- Introduced `buildInpaintBundle` to prepare inpainting data including mask generation and validation.
- Created utility functions for mask operations such as `applyMaskedContentModeToRgba`, `expandRectWithinBounds`, and others for mask manipulation.
- Developed tests for inpainting preparation and mask raster utilities to ensure functionality and correctness.
- Implemented mask raster operations including inversion, feathering, blurring, and more.
This commit is contained in:
syntaxbullet
2026-07-05 09:35:16 +02:00
parent 6e5d58a638
commit f5c610dac5
35 changed files with 2285 additions and 242 deletions

View File

@@ -25,7 +25,7 @@ export type AppProps = {
export function App({ app }: AppProps) {
const shellState = useAppState(app.store, selectAppShellState, shallowEqual);
const { document, selection, viewport, tools, transformSession, maskEdit } = shellState;
const { document, selection, viewport, tools, generation, transformSession, maskEdit } = shellState;
const viewportActivityIsland = useViewportActivityIsland(viewport);
const imageImport = useImageImport(app.store);
const [layersOpen, setLayersOpen] = useState(false);
@@ -161,6 +161,7 @@ export function App({ app }: AppProps) {
activeTool={tools.activeTool}
brushSettings={tools.brush}
generateSettings={tools.generate}
generation={generation}
chromaKeySettings={tools.chromaKey}
magicWandSettings={tools.magicWand}
editingMask={Boolean(maskEdit)}
@@ -189,6 +190,7 @@ type AppShellState = {
selection: AppState["editor"]["selection"];
viewport: AppState["editor"]["viewport"];
tools: AppState["editor"]["tools"];
generation: AppState["editor"]["generation"];
transformSession: AppState["editor"]["transformSession"];
maskEdit: AppState["editor"]["maskEdit"];
};
@@ -199,6 +201,7 @@ function selectAppShellState(state: AppState): AppShellState {
selection: state.editor.selection,
viewport: state.editor.viewport,
tools: state.editor.tools,
generation: state.editor.generation,
transformSession: state.editor.transformSession,
maskEdit: state.editor.maskEdit,
};

View File

@@ -1,6 +1,6 @@
import type { AppStore } from "@editor/store";
import type { ImageDocument } from "@core/document";
import type { MaskViewMode, SelectionState, ViewportState } from "@editor/state";
import type { GenerationState, MaskViewMode, SelectionState, ViewportState } from "@editor/state";
import type { BrushSettings, ChromaKeySettings, GenerateSettings, MagicWandSettings, ToolId } from "@editor/tools";
import { BrushControls } from "./bottom-controls/BrushControls";
import { ChromaKeyControls } from "./bottom-controls/ChromaKeyControls";
@@ -22,6 +22,7 @@ export type BottomControlsIslandProps = {
activeTool: ToolId;
brushSettings: BrushSettings;
generateSettings: GenerateSettings;
generation: GenerationState;
chromaKeySettings: ChromaKeySettings;
magicWandSettings: MagicWandSettings;
editingMask?: boolean;
@@ -32,7 +33,7 @@ export type BottomControlsIslandProps = {
dispatch: AppStore["dispatch"];
};
export function BottomControlsIsland({ document, selection, viewport, visible, action, activeTool, brushSettings, generateSettings, chromaKeySettings, magicWandSettings, editingMask = false, maskViewMode = "composite", transformBounds, transformTarget, brushHint, dispatch }: BottomControlsIslandProps) {
export function BottomControlsIsland({ document, selection, viewport, visible, action, activeTool, brushSettings, generateSettings, generation, chromaKeySettings, magicWandSettings, editingMask = false, maskViewMode = "composite", transformBounds, transformTarget, brushHint, dispatch }: BottomControlsIslandProps) {
const zoomPercent = Math.round(viewport.zoom * 100);
const x = Math.round(viewport.center.x);
const y = Math.round(viewport.center.y);
@@ -40,12 +41,12 @@ export function BottomControlsIsland({ document, selection, viewport, visible, a
return (
<div
aria-hidden={!visible}
className={`flex h-20 min-w-96 rounded-full px-4 py-2 items-center justify-center gap-4 text-sm text-white backdrop-blur transition-all duration-200 ${
className={`flex min-h-20 min-w-96 rounded-[2rem] px-4 py-2 items-center justify-center gap-4 text-sm text-white backdrop-blur transition-all duration-200 ${
visible ? "pointer-events-auto translate-y-0 opacity-100" : "pointer-events-none translate-y-3 opacity-0"
}`}
>
{activeTool === "generate" ? (
<GenerateActionControls document={document} selection={selection} viewport={viewport} settings={generateSettings} dispatch={dispatch} />
<GenerateActionControls document={document} selection={selection} viewport={viewport} settings={generateSettings} generation={generation} dispatch={dispatch} />
) : (activeTool === "brush" || activeTool === "eraser") && brushHint ? (
<BrushHint tool={activeTool} hint={brushHint} />
) : activeTool === "brush" || activeTool === "eraser" ? (

View File

@@ -1,6 +1,7 @@
import { useMemo, useRef, useState, type DragEvent, type MutableRefObject } from "react";
import { useEffect, useMemo, useRef, useState, type DragEvent, type MutableRefObject } from "react";
import { ArrowDown, ArrowUp, DownloadSimple, Eye, EyeSlash, FolderPlus, Lock, LockOpen, Plus, Stack, Trash } from "@phosphor-icons/react";
import { commandIds } from "@commands/ids";
import type { Asset } from "@core/asset";
import type { ImageDocument } from "@core/document";
import type { Layer } from "@core/layer";
import type { ArtboardId } from "@core/id";
@@ -9,6 +10,7 @@ import type { MaskEditState, SelectionState } from "@editor/state";
import type { AppStore } from "@editor/store";
import { resolveLayerDrop } from "@input/index";
import { downloadArtboardPng } from "./exportArtboardPng";
import { analyzeMaskSource, applyMaskRasterOperation, type MaskAnalysis, type MaskRasterOperation } from "./mask/maskRaster";
export type LayersSheetProps = {
document: ImageDocument;
@@ -219,6 +221,7 @@ function LayerRow({
const selected = selectedLayerIds.includes(layer.id);
const layerInfo = documentIndex.layerInfoById.get(layer.id);
const maskLayer = layer.clippingMask ? documentIndex.layerById.get(layer.clippingMask.maskLayerId) : undefined;
const maskAsset = maskLayer && maskLayer.type !== "group" ? documentIndex.assetById.get(maskLayer.assetId) : undefined;
const canAddMask = Boolean(layerInfo && layer.type !== "group" && !layer.clippingMask);
const editingMask = Boolean(maskEdit && layer.clippingMask && maskEdit.targetLayerId === layer.id && maskEdit.maskLayerId === layer.clippingMask.maskLayerId);
const rowPadding = 12 + depth * 16;
@@ -288,21 +291,49 @@ function LayerRow({
</button>
</div>
{layer.clippingMask ? (
<div className="mt-2 flex min-h-14 items-center gap-3 rounded-full py-2 pl-4 pr-2 text-sm text-sky-100/70 hover:bg-white/[0.04]">
<Stack size={24} weight="fill" />
<span className="min-w-0 flex-1 truncate">{maskLayer ? "Layer mask" : "Layer mask missing"}</span>
<div className="mt-2 flex min-h-14 flex-wrap items-center gap-2 rounded-[1.5rem] py-2 pl-4 pr-2 text-sm text-sky-100/70 hover:bg-white/[0.04]">
<Stack size={24} weight="fill" className="shrink-0" />
<span className="min-w-28 flex-1 truncate">{maskLayer ? "Layer mask" : "Layer mask missing"}</span>
{maskAsset ? <MaskStatus asset={maskAsset} /> : null}
{maskLayer ? (
<button
type="button"
className={`h-9 rounded-full px-4 text-sm font-medium transition ${editingMask ? "bg-sky-300 text-black" : "text-sky-100/75 hover:bg-white/10 hover:text-sky-50"}`}
onClick={() =>
editingMask
? dispatch(commandIds.toolExitMaskEdit, undefined)
: dispatch(commandIds.toolEnterMaskEdit, { targetLayerId: layer.id, maskLayerId: layer.clippingMask!.maskLayerId })
}
>
{editingMask ? "Done" : "Edit"}
</button>
<div className="flex flex-wrap items-center gap-1">
<button
type="button"
className={`h-9 rounded-full px-4 text-sm font-medium transition ${editingMask ? "bg-sky-300 text-black" : "text-sky-100/75 hover:bg-white/10 hover:text-sky-50"}`}
onClick={() =>
editingMask
? dispatch(commandIds.toolExitMaskEdit, undefined)
: dispatch(commandIds.toolEnterMaskEdit, { targetLayerId: layer.id, maskLayerId: layer.clippingMask!.maskLayerId })
}
>
{editingMask ? "Done" : "Edit"}
</button>
<button
type="button"
className={maskActionButtonClass()}
title="Paint reveal"
onClick={() => {
dispatch(commandIds.toolEnterMaskEdit, { targetLayerId: layer.id, maskLayerId: layer.clippingMask!.maskLayerId });
dispatch(commandIds.toolSetActive, { tool: "brush" });
}}
>
Reveal
</button>
<button
type="button"
className={maskActionButtonClass()}
title="Paint hide"
onClick={() => {
dispatch(commandIds.toolEnterMaskEdit, { targetLayerId: layer.id, maskLayerId: layer.clippingMask!.maskLayerId });
dispatch(commandIds.toolSetActive, { tool: "eraser" });
}}
>
Hide
</button>
{maskAsset && maskLayer.type !== "group" ? (
<MaskOperationButtons maskLayerId={maskLayer.id} maskAsset={maskAsset} dispatch={dispatch} />
) : null}
</div>
) : null}
<button
type="button"
@@ -336,6 +367,83 @@ function LayerRow({
);
}
function MaskStatus({ asset }: { asset: Asset }) {
const [analysis, setAnalysis] = useState<MaskAnalysis>();
useEffect(() => {
let cancelled = false;
void analyzeMaskSource(asset.source, asset.intrinsicSize.w, asset.intrinsicSize.h)
.then((nextAnalysis) => {
if (!cancelled) setAnalysis(nextAnalysis);
})
.catch(() => {
if (!cancelled) setAnalysis(undefined);
});
return () => {
cancelled = true;
};
}, [asset.source, asset.intrinsicSize.w, asset.intrinsicSize.h]);
if (!analysis) return <span className="rounded-full bg-white/5 px-3 py-1 text-xs text-sky-100/45">Reading</span>;
return (
<span className="inline-flex min-w-0 items-center gap-2 rounded-full bg-white/5 px-2 py-1 text-xs text-sky-100/65">
<img src={analysis.thumbnail} alt="" className="h-7 w-10 rounded-md bg-black/30 object-cover ring-1 ring-white/10" />
<span className="whitespace-nowrap">Reveal {formatPercent(analysis.coverage)}</span>
<span className="whitespace-nowrap text-sky-100/45">Inpaint {formatPercent(analysis.hiddenCoverage)}</span>
</span>
);
}
function MaskOperationButtons({ maskLayerId, maskAsset, dispatch }: { maskLayerId: string; maskAsset: Asset; dispatch: AppStore["dispatch"] }) {
return (
<>
<MaskOperationButton label="Invert" title="Invert mask" maskLayerId={maskLayerId} maskAsset={maskAsset} operation={{ type: "invert" }} dispatch={dispatch} />
<MaskOperationButton label="White" title="Fill mask white" maskLayerId={maskLayerId} maskAsset={maskAsset} operation={{ type: "fill", fill: "white" }} dispatch={dispatch} />
<MaskOperationButton label="Black" title="Fill mask black" maskLayerId={maskLayerId} maskAsset={maskAsset} operation={{ type: "fill", fill: "black" }} dispatch={dispatch} />
<MaskOperationButton label="Clear" title="Clear mask" maskLayerId={maskLayerId} maskAsset={maskAsset} operation={{ type: "fill", fill: "clear" }} dispatch={dispatch} />
<MaskOperationButton label="Feather" title="Feather mask edge" maskLayerId={maskLayerId} maskAsset={maskAsset} operation={{ type: "feather", radius: 3 }} dispatch={dispatch} />
<MaskOperationButton label="Expand" title="Expand mask" maskLayerId={maskLayerId} maskAsset={maskAsset} operation={{ type: "expand", radius: 3 }} dispatch={dispatch} />
<MaskOperationButton label="Contract" title="Contract mask" maskLayerId={maskLayerId} maskAsset={maskAsset} operation={{ type: "contract", radius: 3 }} dispatch={dispatch} />
<MaskOperationButton label="Blur" title="Blur mask edge" maskLayerId={maskLayerId} maskAsset={maskAsset} operation={{ type: "blur", radius: 2 }} dispatch={dispatch} />
<MaskOperationButton label="Clean" title="Despeckle mask" maskLayerId={maskLayerId} maskAsset={maskAsset} operation={{ type: "despeckle", strength: 8 }} dispatch={dispatch} />
</>
);
}
function MaskOperationButton({
label,
title,
maskLayerId,
maskAsset,
operation,
dispatch,
}: {
label: string;
title: string;
maskLayerId: string;
maskAsset: Asset;
operation: MaskRasterOperation;
dispatch: AppStore["dispatch"];
}) {
const [busy, setBusy] = useState(false);
return (
<button
type="button"
className={maskActionButtonClass()}
disabled={busy}
title={title}
onClick={() => {
setBusy(true);
void applyMaskRasterOperation(maskAsset.source, maskAsset.intrinsicSize.w, maskAsset.intrinsicSize.h, operation)
.then((source) => dispatch(commandIds.documentApplyLayerMaskOperation, { maskLayerId, source, mimeType: "image/png", operation }))
.finally(() => setBusy(false));
}}
>
{busy ? "..." : label}
</button>
);
}
type EditingTitle =
| { type: "artboard"; id: ArtboardId; draft: string }
| { type: "layer"; id: string; draft: string };
@@ -528,3 +636,11 @@ function toolbarButtonClass() {
function labeledToolbarButtonClass() {
return "inline-flex h-12 flex-1 items-center rounded-full pr-4 text-sm font-medium text-white/75 transition hover:bg-white/10 hover:text-white disabled:pointer-events-none disabled:opacity-35 focus:outline-none focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-white/30";
}
function maskActionButtonClass() {
return "h-8 rounded-full bg-white/5 px-3 text-xs font-medium text-sky-100/65 transition hover:bg-white/10 hover:text-sky-50 disabled:pointer-events-none disabled:opacity-35";
}
function formatPercent(value: number) {
return `${Math.round(Math.max(0, Math.min(1, value)) * 100)}%`;
}

View File

@@ -31,6 +31,13 @@ export function BrushControls({ tool, settings, editingMask = false, maskViewMod
{tool === "eraser" ? <Eraser size={24} weight="regular" /> : <PaintBrush size={24} weight="regular" />}
</span>
<BottomControlDivider />
{editingMask ? (
<>
<button type="button" className={maskModeButtonClass(tool === "brush")} onClick={() => dispatch(commandIds.toolSetActive, { tool: "brush" })}>Reveal</button>
<button type="button" className={maskModeButtonClass(tool === "eraser")} onClick={() => dispatch(commandIds.toolSetActive, { tool: "eraser" })}>Hide</button>
<BottomControlDivider />
</>
) : null}
{tool === "brush" && !editingMask ? (
<label className={bottomControlFieldClass()}>
<span className={bottomControlLabelClass()}>Color</span>
@@ -91,3 +98,7 @@ export function BrushControls({ tool, settings, editingMask = false, maskViewMod
</div>
);
}
function maskModeButtonClass(active: boolean) {
return `rounded-full px-4 py-2 text-base font-medium transition ${active ? "bg-white text-black" : "bg-white/10 text-white hover:bg-white/15"}`;
}

View File

@@ -7,6 +7,7 @@ import { resolveTransformTargetBounds } from "@editor/transform-targets";
import type { AppStore } from "@editor/store";
import type { ChromaKeySettings } from "@editor/tools";
import type { SelectionState } from "@editor/state";
import { blurMaskValues, despeckleMaskValues, dilateMaskValues, erodeMaskValues } from "../mask/maskRaster";
import { BottomControlColorPicker } from "./ColorPicker";
import { BottomControlDivider } from "./Divider";
import { BottomControlSlider } from "./Slider";
@@ -179,8 +180,8 @@ async function applyChromaKeyMask(target: NonNullable<ReturnType<typeof resolveC
const source = await chromaKeyMaskSource(target.asset.source, target.asset.intrinsicSize.w, target.asset.intrinsicSize.h, settings);
dispatch(commandIds.toolSetBrushStrokePreview, undefined);
if (target.maskAsset) {
dispatch(commandIds.documentUpdateAssetSource, { assetId: target.maskAsset.id, source });
if (target.maskAsset && target.maskLayer && target.maskLayer.type !== "group") {
dispatch(commandIds.documentApplyLayerMaskOperation, { maskLayerId: target.maskLayer.id, source, mimeType: "image/png", operation: { type: "chromaKey" } });
return;
}
@@ -293,85 +294,13 @@ function postProcessAlpha(alpha: Uint8ClampedArray, width: number, height: numbe
const choke = Math.round(Math.max(-20, Math.min(20, settings.choke)));
const feather = Math.round(Math.max(0, Math.min(20, settings.feather)));
if (despeckle > 0) next = despeckleAlpha(next, width, height, despeckle);
if (choke > 0) next = erodeAlpha(next, width, height, choke);
if (choke < 0) next = dilateAlpha(next, width, height, -choke);
if (feather > 0) next = blurAlpha(next, width, height, feather);
if (despeckle > 0) next = despeckleMaskValues(next, width, height, despeckle);
if (choke > 0) next = erodeMaskValues(next, width, height, choke);
if (choke < 0) next = dilateMaskValues(next, width, height, -choke);
if (feather > 0) next = blurMaskValues(next, width, height, feather);
return next;
}
function despeckleAlpha(alpha: Uint8ClampedArray, width: number, height: number, strength: number) {
const radius = Math.max(1, Math.ceil(strength / 6));
const threshold = Math.max(1, Math.round(strength / 2));
const next = new Uint8ClampedArray(alpha);
for (let y = 0; y < height; y++) {
for (let x = 0; x < width; x++) {
const index = y * width + x;
const visible = (alpha[index] ?? 0) > 127;
let same = 0;
for (let oy = -radius; oy <= radius; oy++) {
for (let ox = -radius; ox <= radius; ox++) {
if (ox === 0 && oy === 0) continue;
const sample = alpha[clamp(y + oy, 0, height - 1) * width + clamp(x + ox, 0, width - 1)] ?? 0;
if ((sample > 127) === visible) same += 1;
}
}
if (same <= threshold) next[index] = visible ? 0 : 255;
}
}
return next;
}
function erodeAlpha(alpha: Uint8ClampedArray, width: number, height: number, radius: number) {
const next = new Uint8ClampedArray(alpha.length);
for (let y = 0; y < height; y++) {
for (let x = 0; x < width; x++) {
let value = 255;
for (let oy = -radius; oy <= radius; oy++) {
for (let ox = -radius; ox <= radius; ox++) value = Math.min(value, alpha[clamp(y + oy, 0, height - 1) * width + clamp(x + ox, 0, width - 1)] ?? 0);
}
next[y * width + x] = value;
}
}
return next;
}
function dilateAlpha(alpha: Uint8ClampedArray, width: number, height: number, radius: number) {
const next = new Uint8ClampedArray(alpha.length);
for (let y = 0; y < height; y++) {
for (let x = 0; x < width; x++) {
let value = 0;
for (let oy = -radius; oy <= radius; oy++) {
for (let ox = -radius; ox <= radius; ox++) value = Math.max(value, alpha[clamp(y + oy, 0, height - 1) * width + clamp(x + ox, 0, width - 1)] ?? 0);
}
next[y * width + x] = value;
}
}
return next;
}
function blurAlpha(alpha: Uint8ClampedArray, width: number, height: number, radius: number) {
const next = new Uint8ClampedArray(alpha.length);
for (let y = 0; y < height; y++) {
for (let x = 0; x < width; x++) {
let total = 0;
let count = 0;
for (let oy = -radius; oy <= radius; oy++) {
for (let ox = -radius; ox <= radius; ox++) {
total += alpha[clamp(y + oy, 0, height - 1) * width + clamp(x + ox, 0, width - 1)] ?? 0;
count += 1;
}
}
next[y * width + x] = Math.round(total / count);
}
}
return next;
}
function clamp(value: number, min: number, max: number) {
return Math.max(min, Math.min(max, value));
}
function loadImage(source: string) {
return new Promise<HTMLImageElement>((resolve, reject) => {
const image = new Image();

View File

@@ -1,40 +1,271 @@
import { useState } from "react";
import { useEffect, useState } from "react";
import { commandIds } from "@commands/ids";
import type { ImageDocument } from "@core/document";
import type { SelectionState, ViewportState } from "@editor/state";
import type { GenerationCandidate, GenerationState, SelectionState, ViewportState } from "@editor/state";
import type { GenerateSettings } from "@editor/tools";
import type { AppStore } from "@editor/store";
import { runGenerate } from "../generate/runGenerate";
import { createMaskedPixelReplacementSource } from "../generate/candidateActions";
import { runGenerate, runGenerateFromCandidate } from "../generate/runGenerate";
import { createSolidMaskSource } from "../mask/maskRaster";
export type GenerateActionControlsProps = {
document: ImageDocument;
selection: SelectionState;
viewport: ViewportState;
settings: GenerateSettings;
generation: GenerationState;
dispatch: AppStore["dispatch"];
};
export function GenerateActionControls({ document, selection, viewport, settings, dispatch }: GenerateActionControlsProps) {
const [busy, setBusy] = useState(false);
export function GenerateActionControls({ document, selection, viewport, settings, generation, dispatch }: GenerateActionControlsProps) {
const [busy, setBusy] = useState<string>();
const [error, setError] = useState<string>();
const [elapsedSeconds, setElapsedSeconds] = useState(0);
const candidate = selectedCandidate(generation);
const canGenerate = Boolean(settings.prompt.trim()) && !busy;
useEffect(() => {
if (!busy) {
setElapsedSeconds(0);
return;
}
setElapsedSeconds(0);
const startedAt = Date.now();
const interval = window.setInterval(() => {
setElapsedSeconds(Math.floor((Date.now() - startedAt) / 1000));
}, 1000);
return () => window.clearInterval(interval);
}, [busy]);
return (
<div className="flex items-center px-2">
<div className="flex max-w-[calc(100vw-2rem)] flex-wrap items-center justify-center gap-2 px-2">
<button
type="button"
disabled={!canGenerate}
className="h-12 rounded-full bg-white px-7 text-base font-semibold !text-black transition hover:bg-white/90 focus:outline-none focus-visible:ring-2 focus-visible:ring-white/40 disabled:pointer-events-none disabled:opacity-35"
title={error ?? "Generate with ComfyUI"}
onClick={() => {
setBusy(true);
setBusy("Generating");
setError(undefined);
void runGenerate({ document, selection, viewport, settings, dispatch })
.catch((reason: unknown) => setError(reason instanceof Error ? reason.message : "Generation failed"))
.finally(() => setBusy(false));
.finally(() => setBusy(undefined));
}}
>
{busy ? "Generating" : "Generate"}
{busy === "Generating" ? `Generating ${formatElapsed(elapsedSeconds)}` : "Generate"}
</button>
{busy ? <span className="rounded-full bg-white/10 px-3 py-2 text-xs font-medium text-white/60">{busy} {formatElapsed(elapsedSeconds)}</span> : null}
{candidate ? (
<>
<CandidatePicker generation={generation} dispatch={dispatch} />
<CandidateControls
document={document}
candidate={candidate}
settings={settings}
busy={busy}
setBusy={setBusy}
setError={setError}
dispatch={dispatch}
/>
</>
) : null}
{error ? <span className="max-w-80 truncate rounded-full bg-red-500/15 px-3 py-2 text-xs font-medium text-red-100" title={error}>{error}</span> : null}
</div>
);
}
function CandidatePicker({ generation, dispatch }: { generation: GenerationState; dispatch: AppStore["dispatch"] }) {
if (generation.candidates.length < 2) return null;
return (
<div className="flex items-center gap-1 rounded-full bg-white/[0.04] px-1.5 py-1 ring-1 ring-white/[0.05]">
{generation.candidates.slice(0, 6).map((candidate) => {
const selected = candidate.id === (generation.selectedCandidateId ?? generation.candidates[0]?.id);
return (
<button
key={candidate.id}
type="button"
className={`h-9 w-9 overflow-hidden rounded-full ring-2 transition ${selected ? "ring-white" : "ring-white/10 hover:ring-white/40"}`}
title={`Candidate seed ${candidate.seed}`}
onClick={() => dispatch(commandIds.generationSelectCandidate, { candidateId: candidate.id })}
>
<img src={candidate.source} alt="" className="h-full w-full object-cover" />
</button>
);
})}
</div>
);
}
function CandidateControls({
document,
candidate,
settings,
busy,
setBusy,
setError,
dispatch,
}: {
document: ImageDocument;
candidate: GenerationCandidate;
settings: GenerateSettings;
busy?: string;
setBusy: (busy: string | undefined) => void;
setError: (error: string | undefined) => void;
dispatch: AppStore["dispatch"];
}) {
const rerun = (label: string, nextSettings: GenerateSettings) => {
setBusy(label);
setError(undefined);
dispatch(commandIds.toolSetGenerateSettings, nextSettings);
void runGenerateFromCandidate({ candidate, settings: nextSettings, dispatch })
.catch((reason: unknown) => setError(reason instanceof Error ? reason.message : `${label} failed`))
.finally(() => setBusy(undefined));
};
const disabled = Boolean(busy);
return (
<div className="flex flex-wrap items-center justify-center gap-1 rounded-full bg-white/[0.04] px-2 py-1 ring-1 ring-white/[0.05]">
<CandidatePreview candidate={candidate} />
<span className="px-2 text-xs font-medium text-white/55">Seed {candidate.seed}</span>
<CandidateButton disabled={disabled} label="Regenerate" title="Regenerate same mask and crop" busy={busy === "Regenerate"} onClick={() => rerun("Regenerate", candidate.settings)} />
<CandidateButton
disabled={disabled}
label="Lower"
title="Lower strength and regenerate same mask"
busy={busy === "Lower"}
onClick={() => rerun("Lower", { ...candidate.settings, strength: Math.max(0, candidate.settings.strength - 10), seed: candidate.seed })}
/>
<CandidateButton disabled={disabled} label="Reuse seed" title="Regenerate with the same seed" busy={busy === "Reuse seed"} onClick={() => rerun("Reuse seed", { ...candidate.settings, seed: candidate.seed })} />
<CandidateButton disabled={disabled} label="New seed" title="Regenerate with a new seed" busy={busy === "New seed"} onClick={() => rerun("New seed", { ...candidate.settings, seed: -1 })} />
<CandidateButton disabled={disabled} label="Apply layer" title="Apply candidate as a normal layer" onClick={() => applyCandidateAsLayer(candidate, false, dispatch)} />
<CandidateButton
disabled={disabled}
label="Apply + refine"
title="Apply candidate as a layer with a fresh refinement mask"
busy={busy === "Refine"}
onClick={() => {
setBusy("Refine");
setError(undefined);
void applyCandidateAsRefinementLayer(candidate, dispatch)
.catch((reason: unknown) => setError(reason instanceof Error ? reason.message : "Refine setup failed"))
.finally(() => setBusy(undefined));
}}
/>
<CandidateButton disabled={disabled} label="Stack variant" title="Stack candidate as another variant layer" onClick={() => applyCandidateAsLayer(candidate, true, dispatch)} />
<CandidateButton
disabled={disabled || !candidate.inpaint}
label="Replace"
title={candidate.inpaint ? "Replace masked pixels and preserve unmasked pixels" : "Only inpaint candidates can replace masked pixels"}
busy={busy === "Replace"}
onClick={() => {
setBusy("Replace");
setError(undefined);
void createMaskedPixelReplacementSource(document, candidate)
.then((source) => dispatch(commandIds.generationReplaceCandidatePixels, { candidateId: candidate.id, source, mimeType: "image/png" }))
.catch((reason: unknown) => setError(reason instanceof Error ? reason.message : "Replace failed"))
.finally(() => setBusy(undefined));
}}
/>
<CandidateButton
disabled={disabled || !candidate.inpaint}
label="Edit mask"
title={candidate.inpaint ? "Paint refinement mask over the source layer" : "Only inpaint candidates have a mask to edit"}
onClick={() => {
if (!candidate.inpaint) return;
dispatch(commandIds.selectionSet, { artboardId: candidate.placement.artboardId, layerIds: [candidate.inpaint.targetLayerId] });
dispatch(commandIds.toolEnterMaskEdit, { targetLayerId: candidate.inpaint.targetLayerId, maskLayerId: candidate.inpaint.maskLayerId });
}}
/>
<CandidateButton disabled={disabled} label="Dismiss" title="Remove this candidate" onClick={() => dispatch(commandIds.generationRemoveCandidate, { candidateId: candidate.id })} />
{settings.seed !== candidate.seed ? null : <span className="sr-only">Current settings reuse this seed</span>}
</div>
);
}
function CandidatePreview({ candidate }: { candidate: GenerationCandidate }) {
if (!candidate.inputImage) {
return <img src={candidate.source} alt="" className="h-10 w-10 rounded-full bg-black/25 object-cover ring-1 ring-white/10" />;
}
return (
<span className="flex items-center -space-x-2" title={candidate.maskImage ? "Input crop, normalized mask, generated candidate" : "Before and generated candidate"}>
<img src={candidate.inputImage} alt="" className="h-10 w-10 rounded-full bg-black/25 object-cover ring-1 ring-white/10" />
{candidate.maskImage ? <img src={candidate.maskImage} alt="" className="h-10 w-10 rounded-full bg-black/25 object-cover ring-1 ring-white/20" /> : null}
<img src={candidate.source} alt="" className="h-10 w-10 rounded-full bg-black/25 object-cover ring-2 ring-white/40" />
</span>
);
}
function CandidateButton({ label, title, disabled, busy, onClick }: { label: string; title: string; disabled?: boolean; busy?: boolean; onClick: () => void }) {
return (
<button
type="button"
className="h-9 rounded-full bg-white/5 px-3 text-xs font-semibold text-white/70 transition hover:bg-white/10 hover:text-white disabled:pointer-events-none disabled:opacity-35"
disabled={disabled}
title={title}
onClick={onClick}
>
{busy ? "..." : label}
</button>
);
}
function selectedCandidate(generation: GenerationState): GenerationCandidate | undefined {
return generation.candidates.find((candidate) => candidate.id === generation.selectedCandidateId) ?? generation.candidates[0];
}
function applyCandidateAsLayer(candidate: GenerationCandidate, variant: boolean, dispatch: AppStore["dispatch"]) {
applyCandidateAsLayerWithIds(candidate, { layerId: crypto.randomUUID(), assetId: crypto.randomUUID() }, variant, dispatch);
}
async function applyCandidateAsRefinementLayer(candidate: GenerationCandidate, dispatch: AppStore["dispatch"]) {
const layerId = crypto.randomUUID();
const maskLayerId = crypto.randomUUID();
const maskAssetId = crypto.randomUUID();
applyCandidateAsLayerWithIds(candidate, { layerId, assetId: crypto.randomUUID() }, true, dispatch);
const width = Math.max(1, Math.round(candidate.intrinsicSize.w));
const height = Math.max(1, Math.round(candidate.intrinsicSize.h));
const source = await createSolidMaskSource(width, height, "white");
dispatch(commandIds.documentAddLayerMask, {
layerId,
asset: {
id: maskAssetId,
name: `${candidate.placement.layerName} refinement mask`,
mimeType: "image/png",
source,
intrinsicSize: { w: width, h: height },
},
maskLayer: {
id: maskLayerId,
type: "raster",
name: `${candidate.placement.layerName} refinement mask`,
visible: true,
locked: false,
opacity: 1,
assetId: maskAssetId,
transform: {
position: { ...candidate.placement.transform.position },
scale: { ...candidate.placement.transform.scale },
rotation: candidate.placement.transform.rotation,
},
},
});
dispatch(commandIds.toolSetActive, { tool: "eraser" });
}
function applyCandidateAsLayerWithIds(candidate: GenerationCandidate, ids: { layerId: string; assetId: string }, variant: boolean, dispatch: AppStore["dispatch"]) {
dispatch(commandIds.generationApplyCandidateAsLayer, {
candidateId: candidate.id,
assetId: ids.assetId,
layerId: ids.layerId,
variant,
});
}
function formatElapsed(seconds: number) {
const minutes = Math.floor(seconds / 60);
const remainder = seconds % 60;
return `${minutes}:${remainder.toString().padStart(2, "0")}`;
}

View File

@@ -21,6 +21,18 @@ const sizePresets = [
{ label: "9:16", w: 768, h: 1344 },
] as const;
const inpaintPolarityOptions = [
{ value: "hidden", label: "Hidden / erased" },
{ value: "revealed", label: "Revealed / painted" },
] satisfies readonly BottomControlSelectOption<GenerateSettings["inpaint"]["maskPolarity"]>[];
const inpaintMaskedContentOptions = [
{ value: "neutral", label: "Neutral fill" },
{ value: "original", label: "Original gray" },
{ value: "originalColor", label: "Original color" },
{ value: "edges", label: "Edge map" },
] satisfies readonly BottomControlSelectOption<GenerateSettings["inpaint"]["maskedContent"]>[];
export type GenerateControlsProps = {
settings: GenerateSettings;
dispatch: AppStore["dispatch"];
@@ -32,6 +44,7 @@ export function GenerateControls({ settings, dispatch }: GenerateControlsProps)
const [schedulers, setSchedulers] = useState<readonly BottomControlSelectOption<string>[]>([{ value: settings.scheduler, label: settings.scheduler }]);
const [advancedOpen, setAdvancedOpen] = useState(false);
const [outpaintOpen, setOutpaintOpen] = useState(false);
const [inpaintOpen, setInpaintOpen] = useState(false);
const [sizeOpen, setSizeOpen] = useState(false);
const sizeRef = useRef<HTMLDivElement>(null);
const [error, setError] = useState<string>();
@@ -141,6 +154,44 @@ export function GenerateControls({ settings, dispatch }: GenerateControlsProps)
<PanelNumber label="Feather" aria-label="Outpaint feathering" value={settings.outpaint.feathering} onValueChange={(feathering) => dispatch(commandIds.toolSetGenerateSettings, { outpaint: { ...settings.outpaint, feathering } })} />
</div>
</section>
<section className={panelSectionClass()}>
<button type="button" className={sectionToggleClass()} aria-expanded={inpaintOpen} aria-controls="generate-inpaint-controls" onClick={() => setInpaintOpen((open) => !open)}>
<span>
<span className="block text-sm font-semibold text-white/85">Inpaint</span>
<span className="block text-xs text-white/40">Mask polarity, crop padding, edge prep, grow</span>
</span>
{inpaintOpen ? <CaretUp size={18} weight="bold" /> : <CaretDown size={18} weight="bold" />}
</button>
<div id="generate-inpaint-controls" className={`grid gap-2 overflow-hidden transition-all duration-200 ${inpaintOpen ? "max-h-[38rem] pt-2 opacity-100" : "max-h-0 opacity-0"}`}>
<PanelSelect
label="Polarity"
value={settings.inpaint.maskPolarity}
options={inpaintPolarityOptions}
ariaLabel="Inpaint mask polarity"
onValueChange={(maskPolarity) => dispatch(commandIds.toolSetGenerateSettings, { inpaint: { ...settings.inpaint, maskPolarity } })}
/>
<PanelSelect
label="Content"
value={settings.inpaint.maskedContent}
options={inpaintMaskedContentOptions}
ariaLabel="Inpaint masked content"
onValueChange={(maskedContent) => dispatch(commandIds.toolSetGenerateSettings, { inpaint: { ...settings.inpaint, maskedContent } })}
/>
<button type="button" className={compactRowButtonClass()} aria-pressed={settings.inpaint.maskedAreaOnly} onClick={() => dispatch(commandIds.toolSetGenerateSettings, { inpaint: { ...settings.inpaint, maskedAreaOnly: !settings.inpaint.maskedAreaOnly } })}>
<span className={panelLabelClass()}>Frame</span>
<span className="min-w-0 flex-1 text-right text-white">{settings.inpaint.maskedAreaOnly ? "Mask crop" : "Full layer"}</span>
</button>
<div className="grid grid-cols-2 gap-2">
<PanelNumber label="Pad" aria-label="Inpaint crop padding" value={settings.inpaint.cropPadding} max={2048} onValueChange={(cropPadding) => dispatch(commandIds.toolSetGenerateSettings, { inpaint: { ...settings.inpaint, cropPadding } })} />
<PanelNumber label="Grow" aria-label="Inpaint backend grow mask" value={settings.inpaint.growMaskBy} max={256} onValueChange={(growMaskBy) => dispatch(commandIds.toolSetGenerateSettings, { inpaint: { ...settings.inpaint, growMaskBy } })} />
<PanelNumber label="Expand" aria-label="Inpaint mask expand" value={settings.inpaint.maskExpand} min={-256} max={256} onValueChange={(maskExpand) => dispatch(commandIds.toolSetGenerateSettings, { inpaint: { ...settings.inpaint, maskExpand } })} />
<PanelNumber label="Feather" aria-label="Inpaint mask feather" value={settings.inpaint.maskFeather} max={256} onValueChange={(maskFeather) => dispatch(commandIds.toolSetGenerateSettings, { inpaint: { ...settings.inpaint, maskFeather } })} />
<PanelNumber label="Blur" aria-label="Inpaint mask blur" value={settings.inpaint.maskBlur} max={256} onValueChange={(maskBlur) => dispatch(commandIds.toolSetGenerateSettings, { inpaint: { ...settings.inpaint, maskBlur } })} />
<PanelNumber label="Clean" aria-label="Inpaint mask despeckle" value={settings.inpaint.maskDespeckle} max={64} onValueChange={(maskDespeckle) => dispatch(commandIds.toolSetGenerateSettings, { inpaint: { ...settings.inpaint, maskDespeckle } })} />
</div>
</div>
</section>
</div>
);
}

View File

@@ -138,7 +138,15 @@ export async function commitBrushSession(options: { store: AppStore; session: Br
if (options.session.cancelled) return;
const source = options.session.changed ? canvasToDataUrl(options.session.canvas) : undefined;
if (source) options.store.dispatch(commandIds.documentUpdateAssetSource, { assetId: options.session.assetId, source });
if (source) {
const state = options.store.getState();
const maskEdit = state.editor.maskEdit;
if (maskEdit?.maskLayerId === options.session.layerId) {
options.store.dispatch(commandIds.documentApplyLayerMaskOperation, { maskLayerId: maskEdit.maskLayerId, source, mimeType: "image/png", operation: { type: "paint" } });
} else {
options.store.dispatch(commandIds.documentUpdateAssetSource, { assetId: options.session.assetId, source });
}
}
options.store.dispatch(commandIds.toolSetBrushStrokePreview, undefined);
closeBrushStrokePreview(options.session);
}

View File

@@ -5,6 +5,7 @@ import type { Layer } from "@core/layer";
import { resolveTransformTargetBounds } from "@editor/transform-targets";
import type { AppStore } from "@editor/store";
import type { EditorState } from "@editor/state";
import { blurMaskValues, despeckleMaskValues, dilateMaskValues, erodeMaskValues, maskValueFromRgba } from "../mask/maskRaster";
export async function applyMagicWandAt(store: AppStore, point: Vec2D, modeOverride?: EditorState["tools"]["magicWand"]["mode"]) {
const state = store.getState();
@@ -15,8 +16,8 @@ export async function applyMagicWandAt(store: AppStore, point: Vec2D, modeOverri
const y = Math.floor((point.y - target.layer.transform.position.y) / Math.max(0.0001, target.layer.transform.scale.y));
if (x < 0 || y < 0 || x >= target.asset.intrinsicSize.w || y >= target.asset.intrinsicSize.h) return true;
const source = await createWandMask(target.asset.source, target.maskAsset?.source, Math.round(target.asset.intrinsicSize.w), Math.round(target.asset.intrinsicSize.h), x, y, { ...state.editor.tools.magicWand, mode: modeOverride ?? state.editor.tools.magicWand.mode });
if (target.maskAsset) {
store.dispatch(commandIds.documentUpdateAssetSource, { assetId: target.maskAsset.id, source });
if (target.maskAsset && target.maskLayer && target.maskLayer.type !== "group") {
store.dispatch(commandIds.documentApplyLayerMaskOperation, { maskLayerId: target.maskLayer.id, source, mimeType: "image/png", operation: { type: "magicWand" } });
return true;
}
const assetId = crypto.randomUUID();
@@ -42,7 +43,7 @@ function resolveTarget(document: ImageDocument, editor: EditorState) {
const bounds = resolveTransformTargetBounds(document, { type: "layer", id: layer.id });
const maskLayer = layer.clippingMask ? findLayer(document.artboards.flatMap((artboard) => artboard.layers), layer.clippingMask.maskLayerId) : undefined;
const maskAsset = maskLayer && maskLayer.type !== "group" ? document.assets.find((candidate) => candidate.id === maskLayer.assetId) : undefined;
return asset && bounds ? { layer, asset, bounds, maskAsset } : undefined;
return asset && bounds ? { layer, asset, bounds, maskLayer, maskAsset } : undefined;
}
async function createWandMask(source: string, existingMaskSource: string | undefined, width: number, height: number, startX: number, startY: number, settings: EditorState["tools"]["magicWand"]) {
@@ -57,10 +58,15 @@ async function createWandMask(source: string, existingMaskSource: string | undef
const start = (startY * canvas.width + startX) * 4;
const key = [imageData.data[start] ?? 0, imageData.data[start + 1] ?? 0, imageData.data[start + 2] ?? 0];
const selected = postProcessSelection(settings.contiguous ? floodSelect(imageData, canvas.width, canvas.height, startX, startY, key, settings.tolerance) : globalSelect(imageData, key, settings.tolerance), canvas.width, canvas.height, settings);
const existingAlpha = existingMaskSource ? await loadMaskAlpha(existingMaskSource, canvas.width, canvas.height) : undefined;
const existingMask = existingMaskSource ? await loadMaskValues(existingMaskSource, canvas.width, canvas.height) : undefined;
for (let pixel = 0; pixel < selected.length; pixel++) {
const current = existingAlpha?.[pixel] ?? 255;
const value = settings.mode === "add" ? (selected[pixel] ? 0 : current) : settings.mode === "subtract" ? (selected[pixel] ? 255 : current) : selected[pixel] ? 0 : 255;
const current = existingMask?.[pixel] ?? 255;
const selectionValue = selected[pixel] ?? 0;
const value = settings.mode === "add"
? Math.min(current, 255 - selectionValue)
: settings.mode === "subtract"
? Math.max(current, selectionValue)
: 255 - selectionValue;
const index = pixel * 4;
imageData.data[index] = 255;
imageData.data[index + 1] = 255;
@@ -98,67 +104,24 @@ function matches(data: ImageData, pixel: number, key: number[], tolerance: numbe
}
function postProcessSelection(selected: Uint8Array, width: number, height: number, settings: EditorState["tools"]["magicWand"]) {
let next = selected;
let next = selectionToMaskValues(selected);
const despeckle = Math.round(Math.max(0, Math.min(20, settings.despeckle)));
const choke = Math.round(Math.max(-20, Math.min(20, settings.choke)));
const feather = Math.round(Math.max(0, Math.min(20, settings.feather)));
if (despeckle > 0) next = despeckleSelection(next, width, height, despeckle);
if (choke > 0) next = erodeSelection(next, width, height, choke);
if (choke < 0) next = dilateSelection(next, width, height, -choke);
if (feather > 0) next = featherSelection(next, width, height, feather);
if (despeckle > 0) next = despeckleMaskValues(next, width, height, despeckle);
if (choke > 0) next = erodeMaskValues(next, width, height, choke);
if (choke < 0) next = dilateMaskValues(next, width, height, -choke);
if (feather > 0) next = blurMaskValues(next, width, height, feather);
return next;
}
function erodeSelection(selected: Uint8Array, width: number, height: number, radius: number) {
const next = new Uint8Array(selected.length);
for (let y = 0; y < height; y++) for (let x = 0; x < width; x++) {
let value = 1;
for (let oy = -radius; oy <= radius; oy++) for (let ox = -radius; ox <= radius; ox++) value = Math.min(value, selected[clamp(y + oy, 0, height - 1) * width + clamp(x + ox, 0, width - 1)] ?? 0);
next[y * width + x] = value;
}
return next;
function selectionToMaskValues(selected: Uint8Array) {
const values = new Uint8ClampedArray(selected.length);
for (let index = 0; index < selected.length; index += 1) values[index] = selected[index] ? 255 : 0;
return values;
}
function dilateSelection(selected: Uint8Array, width: number, height: number, radius: number) {
const next = new Uint8Array(selected.length);
for (let y = 0; y < height; y++) for (let x = 0; x < width; x++) {
let value = 0;
for (let oy = -radius; oy <= radius; oy++) for (let ox = -radius; ox <= radius; ox++) value = Math.max(value, selected[clamp(y + oy, 0, height - 1) * width + clamp(x + ox, 0, width - 1)] ?? 0);
next[y * width + x] = value;
}
return next;
}
function featherSelection(selected: Uint8Array, width: number, height: number, radius: number) {
const next = new Uint8Array(selected.length);
for (let y = 0; y < height; y++) for (let x = 0; x < width; x++) {
let total = 0;
let count = 0;
for (let oy = -radius; oy <= radius; oy++) for (let ox = -radius; ox <= radius; ox++) {
total += selected[clamp(y + oy, 0, height - 1) * width + clamp(x + ox, 0, width - 1)] ?? 0;
count += 1;
}
next[y * width + x] = Math.round(total / count);
}
return next;
}
function despeckleSelection(selected: Uint8Array, width: number, height: number, strength: number) {
const radius = Math.max(1, Math.ceil(strength / 6));
const threshold = Math.max(1, Math.round(strength / 2));
const next = new Uint8Array(selected);
for (let y = 0; y < height; y++) for (let x = 0; x < width; x++) {
const index = y * width + x;
let same = 0;
for (let oy = -radius; oy <= radius; oy++) for (let ox = -radius; ox <= radius; ox++) if (ox !== 0 || oy !== 0) {
if ((selected[clamp(y + oy, 0, height - 1) * width + clamp(x + ox, 0, width - 1)] ?? 0) === selected[index]) same += 1;
}
if (same <= threshold) next[index] = selected[index] ? 0 : 1;
}
return next;
}
async function loadMaskAlpha(source: string, width: number, height: number) {
async function loadMaskValues(source: string, width: number, height: number) {
const canvas = document.createElement("canvas");
canvas.width = width;
canvas.height = height;
@@ -167,13 +130,9 @@ async function loadMaskAlpha(source: string, width: number, height: number) {
const image = await loadImage(source);
context.drawImage(image, 0, 0, width, height);
const data = context.getImageData(0, 0, width, height);
const alpha = new Uint8ClampedArray(width * height);
for (let pixel = 0; pixel < alpha.length; pixel++) alpha[pixel] = data.data[pixel * 4 + 3] ?? 255;
return alpha;
}
function clamp(value: number, min: number, max: number) {
return Math.max(min, Math.min(max, value));
const values = new Uint8ClampedArray(width * height);
for (let pixel = 0; pixel < values.length; pixel++) values[pixel] = maskValueFromRgba(data.data, pixel * 4);
return values;
}
function findLayer(layers: readonly Layer[], layerId: string): Layer | undefined {

View File

@@ -151,6 +151,36 @@ const visualEditorChanges: Array<[string, (state: AppState) => AppState]> = [
},
}),
],
[
"generation candidate preview",
(state) => ({
...state,
editor: {
...state.editor,
generation: {
candidates: [
{
id: "candidate",
source: "data:image/png;base64,candidate",
mimeType: "image/png",
intrinsicSize: { w: 64, h: 64 },
mode: "text-to-image",
settings: state.editor.tools.generate,
seed: 12,
width: 64,
height: 64,
placement: {
artboardId: "artboard",
layerName: "Candidate",
transform: { position: { x: 1, y: 2 }, scale: { x: 1, y: 1 }, rotation: 0 },
},
},
],
selectedCandidateId: "candidate",
},
},
}),
],
[
"active tool",
(state) => ({

View File

@@ -1,6 +1,6 @@
import type { Rect, Vec2D } from "@core/geometry";
import type { RenderFrame } from "@renderer/index";
import type { AppState, BrushPreviewState, BrushStrokePreviewState, EditorState, MaskEditState, SelectionState, ViewportState } from "@editor/state";
import type { AppState, BrushPreviewState, BrushStrokePreviewState, EditorState, GenerationState, MaskEditState, SelectionState, ViewportState } from "@editor/state";
import type { BrushSettings, InteractionMode } from "@editor/tools";
import type { TransformSession, TransformTarget } from "@editor/transform";
@@ -23,6 +23,7 @@ function visualEditorStatesEqual(a: EditorState, b: EditorState): boolean {
maskEditStatesEqual(a.maskEdit, b.maskEdit) &&
brushPreviewStatesEqual(a.brushPreview, b.brushPreview) &&
brushStrokePreviewStatesEqual(a.brushStrokePreview, b.brushStrokePreview) &&
generationStatesEqual(a.generation, b.generation) &&
visualToolStatesEqual(a.tools, b.tools)
);
}
@@ -64,6 +65,10 @@ function brushStrokePreviewStatesEqual(a: BrushStrokePreviewState | undefined, b
return a.layerId === b.layerId && a.assetId === b.assetId && a.source === b.source;
}
function generationStatesEqual(a: GenerationState, b: GenerationState): boolean {
return a === b;
}
function visualToolStatesEqual(a: EditorState["tools"], b: EditorState["tools"]): boolean {
return a.activeTool === b.activeTool && interactionModesEqual(a.interactionMode, b.interactionMode) && brushSettingsEqual(a.brush, b.brush);
}

View File

@@ -0,0 +1,50 @@
import type { ImageDocument } from "@core/document";
import type { GenerationCandidate } from "@editor/state";
import { loadImageCanvas, maskValueFromRgba } from "../mask/maskRaster";
export async function createMaskedPixelReplacementSource(document: ImageDocument, candidate: GenerationCandidate): Promise<string> {
if (!candidate.inpaint) throw new Error("Only inpaint candidates can replace masked pixels.");
const targetAsset = document.assets.find((asset) => asset.id === candidate.inpaint?.sourceAssetId);
if (!targetAsset) throw new Error("The source layer for this candidate no longer exists.");
const targetCanvas = await loadImageCanvas(targetAsset.source, targetAsset.intrinsicSize.w, targetAsset.intrinsicSize.h);
const generatedCanvas = await loadImageCanvas(candidate.source, candidate.width, candidate.height);
const maskCanvas = await loadImageCanvas(candidate.inpaint.maskImage, candidate.width, candidate.height);
const targetContext = require2dContext(targetCanvas);
const generatedContext = require2dContext(generatedCanvas);
const maskContext = require2dContext(maskCanvas);
const targetData = targetContext.getImageData(0, 0, targetCanvas.width, targetCanvas.height);
const generatedData = generatedContext.getImageData(0, 0, generatedCanvas.width, generatedCanvas.height);
const maskData = maskContext.getImageData(0, 0, maskCanvas.width, maskCanvas.height);
const crop = candidate.inpaint.crop.assetBounds;
for (let y = 0; y < candidate.height; y += 1) {
for (let x = 0; x < candidate.width; x += 1) {
const targetX = Math.round(crop.x) + x;
const targetY = Math.round(crop.y) + y;
if (targetX < 0 || targetY < 0 || targetX >= targetCanvas.width || targetY >= targetCanvas.height) continue;
const generatedIndex = (y * generatedCanvas.width + x) * 4;
const targetIndex = (targetY * targetCanvas.width + targetX) * 4;
const mask = maskValueFromRgba(maskData.data, generatedIndex) / 255;
if (mask <= 0) continue;
for (let channel = 0; channel < 4; channel += 1) {
const previous = targetData.data[targetIndex + channel] ?? 0;
const next = generatedData.data[generatedIndex + channel] ?? previous;
targetData.data[targetIndex + channel] = Math.round(previous * (1 - mask) + next * mask);
}
}
}
targetContext.putImageData(targetData, 0, 0);
return targetCanvas.toDataURL("image/png");
}
function require2dContext(canvas: HTMLCanvasElement): CanvasRenderingContext2D {
const context = canvas.getContext("2d");
if (!context) throw new Error("Unable to prepare generated candidate");
return context;
}

View File

@@ -0,0 +1,36 @@
import { describe, expect, test } from "bun:test";
import { applyMaskedContentModeToRgba } from "./inpaintPrep";
describe("inpaint prep", () => {
test("converts only masked original pixels to grayscale", () => {
const pixels = new Uint8ClampedArray([
255, 0, 0, 255,
0, 0, 255, 255,
]);
const mask = new Uint8ClampedArray([255, 0]);
const next = applyMaskedContentModeToRgba(pixels, 2, 1, mask, "original");
expect(next[0]).toBe(next[1]);
expect(next[1]).toBe(next[2]);
expect(Array.from(next.slice(4, 8))).toEqual([0, 0, 255, 255]);
});
test("uses an edge map inside the mask without recoloring unmasked context", () => {
const pixels = new Uint8ClampedArray([
255, 0, 0, 255,
0, 255, 0, 255,
0, 0, 255, 255,
255, 255, 0, 255,
]);
const mask = new Uint8ClampedArray([255, 0, 0, 0]);
const next = applyMaskedContentModeToRgba(pixels, 2, 2, mask, "edges");
expect(next[0]).toBe(next[1]);
expect(next[1]).toBe(next[2]);
expect(Array.from(next.slice(4, 8))).toEqual([0, 255, 0, 255]);
expect(Array.from(next.slice(8, 12))).toEqual([0, 0, 255, 255]);
expect(Array.from(next.slice(12, 16))).toEqual([255, 255, 0, 255]);
});
});

View File

@@ -0,0 +1,258 @@
import type { Asset } from "@core/asset";
import type { ImageDocument } from "@core/document";
import type { Rect } from "@core/geometry";
import type { Layer } from "@core/layer";
import type { SelectionState } from "@editor/state";
import type { GenerateSettings } from "@editor/tools";
import { createDocumentReadIndex, resolveIndexedLayerBounds } from "@editor/document-indexes";
import { createNormalizedMaskSource, cropCanvas, cropMaskValuesToDataUrl, expandRectWithinBounds, loadImageCanvas } from "../mask/maskRaster";
export type InpaintBundle = {
inputImage: string;
maskImage: string;
width: number;
height: number;
targetLayerId: string;
maskLayerId: string;
sourceAssetId: string;
maskAssetId: string;
crop: {
assetBounds: Rect;
documentBounds: Rect;
padding: number;
maskedAreaOnly: boolean;
};
mask: {
polarity: GenerateSettings["inpaint"]["maskPolarity"];
activeBounds: Rect;
};
placement: {
artboardId: string;
layerName: string;
transform: Extract<Layer, { type: "image" | "raster" }>["transform"];
};
backend: {
growMaskBy: number;
maskedContent: GenerateSettings["inpaint"]["maskedContent"];
maskBlur: number;
maskFeather: number;
maskExpand: number;
cropPadding: number;
};
};
type InpaintTarget = {
artboardId: string;
layer: Extract<Layer, { type: "image" | "raster" }>;
asset: Asset;
bounds: Rect;
maskLayer: Extract<Layer, { type: "image" | "raster" }>;
maskAsset: Asset;
maskBounds: Rect;
};
const modelMultiple = 8;
const minModelSize = 64;
const maxModelSize = 4096;
export async function buildInpaintBundle(document: ImageDocument, selection: SelectionState, settings: GenerateSettings): Promise<InpaintBundle> {
const target = resolveInpaintTarget(document, selection);
validateInpaintTarget(target);
const width = Math.max(1, Math.round(target.asset.intrinsicSize.w));
const height = Math.max(1, Math.round(target.asset.intrinsicSize.h));
const normalizedMask = await createNormalizedMaskSource(target.maskAsset.source, width, height, {
polarity: settings.inpaint.maskPolarity,
expand: settings.inpaint.maskExpand,
feather: settings.inpaint.maskFeather,
blur: settings.inpaint.maskBlur,
despeckle: settings.inpaint.maskDespeckle,
});
if (!normalizedMask.bounds) throw new Error("The selected layer mask has no inpaint pixels.");
const crop = settings.inpaint.maskedAreaOnly
? expandRectWithinBounds(normalizedMask.bounds, settings.inpaint.cropPadding, { w: width, h: height }, modelMultiple, minModelSize)
: { x: 0, y: 0, w: width, h: height };
const outputWidth = toModelSize(crop.w, "width");
const outputHeight = toModelSize(crop.h, "height");
const inputCanvas = prepareMaskedContentInputCanvas(await loadImageCanvas(target.asset.source, width, height), normalizedMask.values, settings.inpaint.maskedContent);
const inputImage = cropCanvas(inputCanvas, crop, outputWidth, outputHeight);
const maskImage = cropMaskValuesToDataUrl(normalizedMask.values, width, height, crop, outputWidth, outputHeight);
const scaleX = target.bounds.w / width;
const scaleY = target.bounds.h / height;
const documentBounds = {
x: target.bounds.x + crop.x * scaleX,
y: target.bounds.y + crop.y * scaleY,
w: outputWidth * scaleX,
h: outputHeight * scaleY,
};
return {
inputImage,
maskImage,
width: outputWidth,
height: outputHeight,
targetLayerId: target.layer.id,
maskLayerId: target.maskLayer.id,
sourceAssetId: target.asset.id,
maskAssetId: target.maskAsset.id,
crop: {
assetBounds: crop,
documentBounds,
padding: settings.inpaint.cropPadding,
maskedAreaOnly: settings.inpaint.maskedAreaOnly,
},
mask: {
polarity: settings.inpaint.maskPolarity,
activeBounds: normalizedMask.bounds,
},
placement: {
artboardId: target.artboardId,
layerName: `${target.layer.name} inpaint`,
transform: {
position: { x: documentBounds.x, y: documentBounds.y },
scale: { x: documentBounds.w / outputWidth, y: documentBounds.h / outputHeight },
rotation: target.layer.transform.rotation,
},
},
backend: {
growMaskBy: settings.inpaint.growMaskBy,
maskedContent: settings.inpaint.maskedContent,
maskBlur: settings.inpaint.maskBlur,
maskFeather: settings.inpaint.maskFeather,
maskExpand: settings.inpaint.maskExpand,
cropPadding: settings.inpaint.cropPadding,
},
};
}
function resolveInpaintTarget(document: ImageDocument, selection: SelectionState): InpaintTarget {
if (selection.layerIds.length !== 1 || !selection.layerIds[0]) throw new Error("Select one image or raster layer to inpaint.");
const documentIndex = createDocumentReadIndex(document);
const layerInfo = documentIndex.layerInfoById.get(selection.layerIds[0]);
if (!layerInfo || layerInfo.layer.type === "group") throw new Error("Select one image or raster layer to inpaint.");
const asset = documentIndex.assetById.get(layerInfo.layer.assetId);
if (!asset) throw new Error("The selected layer is missing its source image.");
if (!layerInfo.layer.clippingMask) throw new Error("Add a layer mask before running inpaint.");
const maskLayer = documentIndex.layerById.get(layerInfo.layer.clippingMask.maskLayerId);
if (!maskLayer || maskLayer.type === "group") throw new Error("The selected layer mask is missing.");
const maskAsset = documentIndex.assetById.get(maskLayer.assetId);
if (!maskAsset) throw new Error("The selected layer mask is missing its image data.");
const bounds = resolveIndexedLayerBounds(documentIndex, layerInfo.layer);
const maskBounds = resolveIndexedLayerBounds(documentIndex, maskLayer);
if (!bounds || !maskBounds) throw new Error("Unable to resolve the selected layer and mask bounds.");
return { artboardId: layerInfo.artboardId, layer: layerInfo.layer, asset, bounds, maskLayer, maskAsset, maskBounds };
}
function validateInpaintTarget(target: InpaintTarget) {
if (target.asset.intrinsicSize.w <= 0 || target.asset.intrinsicSize.h <= 0) throw new Error("The selected image has an invalid size.");
if (target.maskAsset.intrinsicSize.w <= 0 || target.maskAsset.intrinsicSize.h <= 0) throw new Error("The selected mask has an invalid size.");
if (Math.round(target.asset.intrinsicSize.w) !== Math.round(target.maskAsset.intrinsicSize.w) || Math.round(target.asset.intrinsicSize.h) !== Math.round(target.maskAsset.intrinsicSize.h)) {
throw new Error("The selected layer and mask image sizes do not match.");
}
if (!rectsAligned(target.bounds, target.maskBounds) || Math.abs(target.layer.transform.rotation - target.maskLayer.transform.rotation) > 0.001) {
throw new Error("The selected layer and mask are not aligned.");
}
}
function rectsAligned(a: Rect, b: Rect): boolean {
return Math.abs(a.x - b.x) <= 0.5 && Math.abs(a.y - b.y) <= 0.5 && Math.abs(a.w - b.w) <= 0.5 && Math.abs(a.h - b.h) <= 0.5;
}
function toModelSize(value: number, axis: "width" | "height"): number {
const rounded = Math.max(minModelSize, Math.ceil(value / modelMultiple) * modelMultiple);
if (rounded > maxModelSize) throw new Error(`The inpaint ${axis} is too large for the model. Use masked-area inpaint or a smaller source.`);
return rounded;
}
function prepareMaskedContentInputCanvas(sourceCanvas: HTMLCanvasElement, maskValues: Uint8ClampedArray, mode: GenerateSettings["inpaint"]["maskedContent"]): HTMLCanvasElement {
if (mode === "neutral" || mode === "originalColor") return sourceCanvas;
const context = sourceCanvas.getContext("2d");
if (!context) return sourceCanvas;
const imageData = context.getImageData(0, 0, sourceCanvas.width, sourceCanvas.height);
imageData.data.set(applyMaskedContentModeToRgba(imageData.data, sourceCanvas.width, sourceCanvas.height, maskValues, mode));
context.putImageData(imageData, 0, 0);
return sourceCanvas;
}
export function applyMaskedContentModeToRgba(data: Uint8ClampedArray, width: number, height: number, maskValues: Uint8ClampedArray, mode: GenerateSettings["inpaint"]["maskedContent"]): Uint8ClampedArray {
const next = new Uint8ClampedArray(data);
if (mode === "neutral" || mode === "originalColor") return next;
const edgeValues = mode === "edges" ? createEdgeMapValues(data, width, height) : undefined;
for (let pixel = 0; pixel < width * height; pixel += 1) {
const amount = (maskValues[pixel] ?? 0) / 255;
if (amount <= 0) continue;
const index = pixel * 4;
const red = data[index] ?? 0;
const green = data[index + 1] ?? 0;
const blue = data[index + 2] ?? 0;
const target = edgeValues ? edgeValues[pixel] ?? 128 : luminance(red, green, blue);
next[index] = blendChannel(red, target, amount);
next[index + 1] = blendChannel(green, target, amount);
next[index + 2] = blendChannel(blue, target, amount);
}
return next;
}
function createEdgeMapValues(data: Uint8ClampedArray, width: number, height: number): Uint8ClampedArray {
const gray = new Float32Array(width * height);
const edges = new Uint8ClampedArray(width * height);
for (let pixel = 0; pixel < width * height; pixel += 1) {
const index = pixel * 4;
gray[pixel] = luminance(data[index] ?? 0, data[index + 1] ?? 0, data[index + 2] ?? 0);
}
for (let y = 0; y < height; y += 1) {
for (let x = 0; x < width; x += 1) {
const gx =
-sampleGray(gray, width, height, x - 1, y - 1) +
sampleGray(gray, width, height, x + 1, y - 1) -
2 * sampleGray(gray, width, height, x - 1, y) +
2 * sampleGray(gray, width, height, x + 1, y) -
sampleGray(gray, width, height, x - 1, y + 1) +
sampleGray(gray, width, height, x + 1, y + 1);
const gy =
-sampleGray(gray, width, height, x - 1, y - 1) -
2 * sampleGray(gray, width, height, x, y - 1) -
sampleGray(gray, width, height, x + 1, y - 1) +
sampleGray(gray, width, height, x - 1, y + 1) +
2 * sampleGray(gray, width, height, x, y + 1) +
sampleGray(gray, width, height, x + 1, y + 1);
const magnitude = Math.hypot(gx, gy);
edges[y * width + x] = Math.round(clampNumber(128 + Math.max(0, magnitude - 24) * 0.75, 128, 255));
}
}
return edges;
}
function sampleGray(values: Float32Array, width: number, height: number, x: number, y: number): number {
const clampedX = Math.max(0, Math.min(width - 1, x));
const clampedY = Math.max(0, Math.min(height - 1, y));
return values[clampedY * width + clampedX] ?? 0;
}
function luminance(red: number, green: number, blue: number): number {
return Math.round(0.2126 * red + 0.7152 * green + 0.0722 * blue);
}
function blendChannel(previous: number, next: number, amount: number): number {
return Math.round(previous * (1 - amount) + next * amount);
}
function clampNumber(value: number, min: number, max: number): number {
return Math.max(min, Math.min(max, value));
}

View File

@@ -1,9 +1,11 @@
import { commandIds } from "@commands/ids";
import type { ImageDocument } from "@core/document";
import type { Transform } from "@core/geometry";
import type { Layer } from "@core/layer";
import type { AppStore } from "@editor/store";
import type { SelectionState, ViewportState } from "@editor/state";
import type { GenerationCandidate, SelectionState, ViewportState } from "@editor/state";
import type { GenerateSettings } from "@editor/tools";
import { buildInpaintBundle, type InpaintBundle } from "./inpaintPrep";
export async function runGenerate(options: {
document: ImageDocument;
@@ -17,50 +19,200 @@ export async function runGenerate(options: {
if (!artboard) return;
const target = resolveSelectedImage(document, selection);
const inputImage = target && settings.mode !== "text-to-image" ? await imageSourceToDataUrl(target.asset.source) : undefined;
const maskImage = target?.maskAsset && settings.mode === "inpaint" ? await imageSourceToDataUrl(target.maskAsset.source) : undefined;
const inpaintBundle = settings.mode === "inpaint" ? await buildInpaintBundle(document, selection, settings) : undefined;
const inputImage = inpaintBundle?.inputImage ?? (target && settings.mode !== "text-to-image" ? await imageSourceToDataUrl(target.asset.source) : undefined);
const maskImage = inpaintBundle?.maskImage;
const seed = resolveSeed(settings.seed);
const requestSettings = { ...settings, seed };
const width = inpaintBundle?.width ?? settings.width;
const height = inpaintBundle?.height ?? settings.height;
const generated = await requestGenerate({
settings: requestSettings,
width,
height,
inputImage,
maskImage,
inpaintBundle,
});
const intrinsicSize = await loadImageSize(generated.source);
const targetArtboardId = inpaintBundle?.placement.artboardId ?? artboard.id;
const placement = {
artboardId: targetArtboardId,
layerName: inpaintBundle?.placement.layerName ?? "Generated image",
transform: generatedLayerTransform(inpaintBundle, intrinsicSize),
};
dispatch(commandIds.generationAddCandidate, {
candidate: createGenerationCandidate({
source: generated.source,
mimeType: generated.mimeType,
intrinsicSize,
settings: requestSettings,
seed,
width,
height,
inputImage,
maskImage,
placement,
inpaintBundle,
}),
});
}
export async function runGenerateFromCandidate(options: {
candidate: GenerationCandidate;
settings?: GenerateSettings;
dispatch: AppStore["dispatch"];
}) {
const settings = options.settings ?? options.candidate.settings;
const seed = resolveSeed(settings.seed);
const requestSettings = { ...settings, seed };
const generated = await requestGenerate({
settings: requestSettings,
width: options.candidate.width,
height: options.candidate.height,
inputImage: options.candidate.inputImage,
maskImage: options.candidate.maskImage,
inpaintCandidate: options.candidate,
});
const intrinsicSize = await loadImageSize(generated.source);
options.dispatch(commandIds.generationAddCandidate, {
candidate: {
...options.candidate,
id: crypto.randomUUID(),
source: generated.source,
mimeType: generated.mimeType,
intrinsicSize,
settings: requestSettings,
seed,
},
});
}
function createGenerationCandidate(options: {
source: string;
mimeType: string;
intrinsicSize: { w: number; h: number };
settings: GenerateSettings;
seed: number;
width: number;
height: number;
inputImage?: string;
maskImage?: string;
placement: GenerationCandidate["placement"];
inpaintBundle?: InpaintBundle;
}): GenerationCandidate {
return {
id: crypto.randomUUID(),
source: options.source,
mimeType: options.mimeType,
intrinsicSize: options.intrinsicSize,
mode: options.settings.mode,
settings: options.settings,
seed: options.seed,
width: options.width,
height: options.height,
inputImage: options.inputImage,
maskImage: options.maskImage,
placement: options.placement,
inpaint: options.inpaintBundle
? {
targetLayerId: options.inpaintBundle.targetLayerId,
maskLayerId: options.inpaintBundle.maskLayerId,
sourceAssetId: options.inpaintBundle.sourceAssetId,
maskAssetId: options.inpaintBundle.maskAssetId,
inputImage: options.inpaintBundle.inputImage,
maskImage: options.inpaintBundle.maskImage,
crop: options.inpaintBundle.crop,
mask: options.inpaintBundle.mask,
backend: options.inpaintBundle.backend,
}
: undefined,
};
}
async function requestGenerate(options: {
settings: GenerateSettings;
width: number;
height: number;
inputImage?: string;
maskImage?: string;
inpaintBundle?: InpaintBundle;
inpaintCandidate?: GenerationCandidate;
}) {
const response = await fetch("/api/comfy/generate", {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({
mode: settings.mode,
model: settings.model,
prompt: settings.prompt,
negativePrompt: settings.negativePrompt,
strength: settings.strength,
steps: settings.steps,
cfg: settings.cfg,
seed: settings.seed,
sampler: settings.sampler,
scheduler: settings.scheduler,
width: settings.width,
height: settings.height,
outpaint: settings.outpaint,
inputImage,
maskImage,
mode: options.settings.mode,
model: options.settings.model,
prompt: options.settings.prompt,
negativePrompt: options.settings.negativePrompt,
strength: options.settings.strength,
steps: options.settings.steps,
cfg: options.settings.cfg,
seed: options.settings.seed,
sampler: options.settings.sampler,
scheduler: options.settings.scheduler,
width: options.width,
height: options.height,
outpaint: options.settings.outpaint,
inpaint: resolveInpaintRequest(options.inpaintBundle, options.inpaintCandidate, options.settings),
inputImage: options.inputImage,
maskImage: options.maskImage,
}),
});
if (!response.ok) throw new Error(await response.text());
const generated = await response.json() as { source: string; mimeType: string };
const intrinsicSize = await loadImageSize(generated.source);
const assetId = crypto.randomUUID();
const layerId = crypto.randomUUID();
dispatch(commandIds.documentAddAsset, { asset: { id: assetId, name: "Generated image", mimeType: generated.mimeType, source: generated.source, intrinsicSize } });
dispatch(commandIds.documentAddImageLayer, {
artboardId: artboard.id,
layer: {
id: layerId,
type: "image",
name: "Generated image",
visible: true,
locked: false,
opacity: 1,
assetId,
transform: { position: { x: 0, y: 0 }, scale: { x: 1, y: 1 }, rotation: 0 },
return await response.json() as { source: string; mimeType: string };
}
function resolveInpaintRequest(inpaintBundle: InpaintBundle | undefined, inpaintCandidate: GenerationCandidate | undefined, settings: GenerateSettings) {
if (inpaintBundle) {
return {
growMaskBy: inpaintBundle.backend.growMaskBy,
maskedContent: inpaintBundle.backend.maskedContent,
maskBlur: inpaintBundle.backend.maskBlur,
maskFeather: inpaintBundle.backend.maskFeather,
maskExpand: inpaintBundle.backend.maskExpand,
cropPadding: inpaintBundle.backend.cropPadding,
maskPolarity: inpaintBundle.mask.polarity,
crop: inpaintBundle.crop,
placement: inpaintBundle.placement,
};
}
if (inpaintCandidate?.inpaint) {
return {
growMaskBy: inpaintCandidate.inpaint.backend.growMaskBy,
maskedContent: inpaintCandidate.inpaint.backend.maskedContent,
maskBlur: inpaintCandidate.inpaint.backend.maskBlur,
maskFeather: inpaintCandidate.inpaint.backend.maskFeather,
maskExpand: inpaintCandidate.inpaint.backend.maskExpand,
cropPadding: inpaintCandidate.inpaint.backend.cropPadding,
maskPolarity: inpaintCandidate.inpaint.mask.polarity,
crop: inpaintCandidate.inpaint.crop,
placement: inpaintCandidate.placement,
};
}
return settings.inpaint;
}
function generatedLayerTransform(inpaintBundle: InpaintBundle | undefined, intrinsicSize: { w: number; h: number }): Transform {
if (!inpaintBundle) return { position: { x: 0, y: 0 }, scale: { x: 1, y: 1 }, rotation: 0 };
return {
position: { ...inpaintBundle.placement.transform.position },
scale: {
x: (inpaintBundle.placement.transform.scale.x * inpaintBundle.width) / Math.max(1, intrinsicSize.w),
y: (inpaintBundle.placement.transform.scale.y * inpaintBundle.height) / Math.max(1, intrinsicSize.h),
},
});
dispatch(commandIds.documentMoveLayer, { layerId, toArtboardId: artboard.id, toIndex: 0 });
dispatch(commandIds.selectionSet, { artboardId: artboard.id, layerIds: [layerId] });
rotation: inpaintBundle.placement.transform.rotation,
};
}
function resolveSeed(seed: number): number {
return seed < 0 ? Math.floor(Math.random() * 2 ** 32) : Math.round(seed);
}
function resolveSelectedImage(document: ImageDocument, selection: SelectionState) {

View File

@@ -0,0 +1,40 @@
import { describe, expect, test } from "bun:test";
import { cropMaskValuesToRgba, expandRectWithinBounds, invertMaskValues } from "./maskRaster";
describe("mask raster utilities", () => {
test("exports the normalized drawn mask without filling the whole crop", () => {
const revealedMask = new Uint8ClampedArray(4 * 3).fill(255);
revealedMask[1 * 4 + 2] = 0;
const inpaintMask = invertMaskValues(revealedMask);
const rgba = cropMaskValuesToRgba(inpaintMask, 4, 3, { x: 1, y: 0, w: 3, h: 3 }, 4, 4);
const activePixels = activeRedPixels(rgba);
expect(activePixels).toEqual([{ x: 1, y: 1 }]);
for (let pixel = 0; pixel < rgba.length / 4; pixel += 1) expect(rgba[pixel * 4 + 3]).toBe(255);
});
test("expands masked-area crops to the model minimum when possible", () => {
expect(expandRectWithinBounds({ x: 20, y: 20, w: 4, h: 4 }, 4, { w: 100, h: 100 }, 8, 64)).toEqual({
x: 0,
y: 0,
w: 64,
h: 64,
});
expect(expandRectWithinBounds({ x: 80, y: 80, w: 4, h: 4 }, 4, { w: 100, h: 100 }, 8, 64)).toEqual({
x: 36,
y: 36,
w: 64,
h: 64,
});
});
});
function activeRedPixels(rgba: Uint8ClampedArray) {
const width = 4;
const pixels: Array<{ x: number; y: number }> = [];
for (let pixel = 0; pixel < rgba.length / 4; pixel += 1) {
if ((rgba[pixel * 4] ?? 0) > 127) pixels.push({ x: pixel % width, y: Math.floor(pixel / width) });
}
return pixels;
}

374
view/mask/maskRaster.ts Normal file
View File

@@ -0,0 +1,374 @@
import type { Rect } from "@core/geometry";
export type MaskFill = "white" | "black" | "clear";
export type MaskRasterOperation =
| { type: "invert" }
| { type: "fill"; fill: MaskFill }
| { type: "feather"; radius: number }
| { type: "expand"; radius: number }
| { type: "contract"; radius: number }
| { type: "blur"; radius: number }
| { type: "despeckle"; strength: number };
export type MaskAnalysis = {
width: number;
height: number;
revealedPixels: number;
hiddenPixels: number;
coverage: number;
hiddenCoverage: number;
bounds?: Rect;
hiddenBounds?: Rect;
thumbnail: string;
};
export type NormalizedMaskOptions = {
polarity: "hidden" | "revealed";
expand?: number;
feather?: number;
blur?: number;
despeckle?: number;
};
export async function createSolidMaskSource(width: number, height: number, fill: MaskFill): Promise<string> {
const canvas = createCanvas(width, height);
const context = require2dContext(canvas);
context.clearRect(0, 0, canvas.width, canvas.height);
if (fill === "white") {
context.fillStyle = "#ffffff";
context.fillRect(0, 0, canvas.width, canvas.height);
} else if (fill === "black") {
context.fillStyle = "#000000";
context.fillRect(0, 0, canvas.width, canvas.height);
}
return canvas.toDataURL("image/png");
}
export async function applyMaskRasterOperation(source: string, width: number, height: number, operation: MaskRasterOperation): Promise<string> {
if (operation.type === "fill") return createSolidMaskSource(width, height, operation.fill);
const mask = await loadMaskValues(source, width, height);
const next = applyMaskValueOperation(mask.values, mask.width, mask.height, operation);
return maskValuesToDataUrl(next, mask.width, mask.height);
}
export async function analyzeMaskSource(source: string, width: number, height: number): Promise<MaskAnalysis> {
const mask = await loadMaskValues(source, width, height);
const revealed = analyzeValues(mask.values, mask.width, mask.height, false);
const hiddenValues = invertMaskValues(mask.values);
const hidden = analyzeValues(hiddenValues, mask.width, mask.height, false);
return {
width: mask.width,
height: mask.height,
revealedPixels: revealed.pixels,
hiddenPixels: hidden.pixels,
coverage: revealed.coverage,
hiddenCoverage: hidden.coverage,
bounds: revealed.bounds,
hiddenBounds: hidden.bounds,
thumbnail: maskValuesToDataUrl(mask.values, mask.width, mask.height, 72, 48),
};
}
export async function createNormalizedMaskSource(source: string, width: number, height: number, options: NormalizedMaskOptions): Promise<{ source: string; values: Uint8ClampedArray; bounds?: Rect }> {
const mask = await loadMaskValues(source, width, height);
let values = options.polarity === "hidden" ? invertMaskValues(mask.values) : new Uint8ClampedArray(mask.values);
const despeckle = Math.round(clampNumber(options.despeckle ?? 0, 0, 64));
const expand = Math.round(clampNumber(options.expand ?? 0, -256, 256));
const feather = Math.round(clampNumber(options.feather ?? 0, 0, 256));
const blur = Math.round(clampNumber(options.blur ?? 0, 0, 256));
if (despeckle > 0) values = despeckleMaskValues(values, mask.width, mask.height, despeckle);
if (expand > 0) values = dilateMaskValues(values, mask.width, mask.height, expand);
if (expand < 0) values = erodeMaskValues(values, mask.width, mask.height, -expand);
if (feather > 0) values = blurMaskValues(values, mask.width, mask.height, feather);
if (blur > 0) values = blurMaskValues(values, mask.width, mask.height, blur);
const analysis = analyzeValues(values, mask.width, mask.height, false);
return { source: maskValuesToDataUrl(values, mask.width, mask.height), values, bounds: analysis.bounds };
}
export async function loadImageCanvas(source: string, width?: number, height?: number): Promise<HTMLCanvasElement> {
const image = await loadImage(source);
const canvas = createCanvas(width ?? image.naturalWidth, height ?? image.naturalHeight);
const context = require2dContext(canvas);
context.clearRect(0, 0, canvas.width, canvas.height);
context.drawImage(image, 0, 0, canvas.width, canvas.height);
return canvas;
}
export async function imageSourceToPngDataUrl(source: string): Promise<string> {
if (source.startsWith("data:image/png;base64,")) return source;
const canvas = await loadImageCanvas(source);
return canvas.toDataURL("image/png");
}
export function cropCanvas(sourceCanvas: HTMLCanvasElement, crop: Rect, outputWidth = crop.w, outputHeight = crop.h): string {
const canvas = createCanvas(outputWidth, outputHeight);
const context = require2dContext(canvas);
context.clearRect(0, 0, outputWidth, outputHeight);
context.drawImage(sourceCanvas, crop.x, crop.y, crop.w, crop.h, 0, 0, crop.w, crop.h);
return canvas.toDataURL("image/png");
}
export function cropMaskValuesToDataUrl(values: Uint8ClampedArray, width: number, height: number, crop: Rect, outputWidth = crop.w, outputHeight = crop.h): string {
const canvas = createCanvas(outputWidth, outputHeight);
const context = require2dContext(canvas);
const imageData = context.createImageData(outputWidth, outputHeight);
imageData.data.set(cropMaskValuesToRgba(values, width, height, crop, outputWidth, outputHeight));
context.putImageData(imageData, 0, 0);
return canvas.toDataURL("image/png");
}
export function cropMaskValuesToRgba(values: Uint8ClampedArray, width: number, height: number, crop: Rect, outputWidth = crop.w, outputHeight = crop.h): Uint8ClampedArray {
const safeOutputWidth = Math.max(1, Math.round(outputWidth));
const safeOutputHeight = Math.max(1, Math.round(outputHeight));
const data = new Uint8ClampedArray(safeOutputWidth * safeOutputHeight * 4);
for (let pixel = 0; pixel < safeOutputWidth * safeOutputHeight; pixel += 1) data[pixel * 4 + 3] = 255;
for (let y = 0; y < Math.min(crop.h, safeOutputHeight); y += 1) {
for (let x = 0; x < Math.min(crop.w, safeOutputWidth); x += 1) {
const sourceX = crop.x + x;
const sourceY = crop.y + y;
if (sourceX < 0 || sourceY < 0 || sourceX >= width || sourceY >= height) continue;
const value = values[sourceY * width + sourceX] ?? 0;
const index = (y * safeOutputWidth + x) * 4;
data[index] = value;
data[index + 1] = value;
data[index + 2] = value;
}
}
return data;
}
export function expandRectWithinBounds(rect: Rect, padding: number, bounds: { w: number; h: number }, multiple = 1, minSize = 1): Rect {
const padded = Math.max(0, Math.round(padding));
let x1 = Math.max(0, Math.floor(rect.x) - padded);
let y1 = Math.max(0, Math.floor(rect.y) - padded);
let x2 = Math.min(bounds.w, Math.ceil(rect.x + rect.w) + padded);
let y2 = Math.min(bounds.h, Math.ceil(rect.y + rect.h) + padded);
const safeMinSize = Math.max(1, Math.round(minSize));
const targetWidth = Math.min(bounds.w, roundUp(Math.max(safeMinSize, x2 - x1), multiple));
const targetHeight = Math.min(bounds.h, roundUp(Math.max(safeMinSize, y2 - y1), multiple));
const extraWidth = targetWidth - (x2 - x1);
const extraHeight = targetHeight - (y2 - y1);
x1 = Math.max(0, x1 - Math.floor(extraWidth / 2));
y1 = Math.max(0, y1 - Math.floor(extraHeight / 2));
x2 = Math.min(bounds.w, x1 + targetWidth);
y2 = Math.min(bounds.h, y1 + targetHeight);
x1 = Math.max(0, x2 - targetWidth);
y1 = Math.max(0, y2 - targetHeight);
return { x: x1, y: y1, w: Math.max(1, x2 - x1), h: Math.max(1, y2 - y1) };
}
export function maskValueFromRgba(data: Uint8ClampedArray, index: number): number {
const red = data[index] ?? 0;
const green = data[index + 1] ?? 0;
const blue = data[index + 2] ?? 0;
const alpha = data[index + 3] ?? 0;
const luminance = 0.2126 * red + 0.7152 * green + 0.0722 * blue;
return Math.round((alpha * luminance) / 255);
}
export function applyMaskValueOperation(values: Uint8ClampedArray, width: number, height: number, operation: Exclude<MaskRasterOperation, { type: "fill" }>): Uint8ClampedArray {
switch (operation.type) {
case "invert":
return invertMaskValues(values);
case "feather":
case "blur":
return blurMaskValues(values, width, height, Math.round(clampNumber(operation.radius, 0, 256)));
case "expand":
return dilateMaskValues(values, width, height, Math.round(clampNumber(operation.radius, 0, 256)));
case "contract":
return erodeMaskValues(values, width, height, Math.round(clampNumber(operation.radius, 0, 256)));
case "despeckle":
return despeckleMaskValues(values, width, height, Math.round(clampNumber(operation.strength, 0, 64)));
}
}
export function invertMaskValues(values: Uint8ClampedArray): Uint8ClampedArray {
const next = new Uint8ClampedArray(values.length);
for (let index = 0; index < values.length; index += 1) next[index] = 255 - (values[index] ?? 0);
return next;
}
export function erodeMaskValues(values: Uint8ClampedArray, width: number, height: number, radius: number): Uint8ClampedArray {
const safeRadius = Math.round(clampNumber(radius, 0, 256));
if (safeRadius <= 0) return new Uint8ClampedArray(values);
const next = new Uint8ClampedArray(values.length);
for (let y = 0; y < height; y += 1) {
for (let x = 0; x < width; x += 1) {
let value = 255;
for (let oy = -safeRadius; oy <= safeRadius; oy += 1) {
for (let ox = -safeRadius; ox <= safeRadius; ox += 1) value = Math.min(value, values[clampInt(y + oy, 0, height - 1) * width + clampInt(x + ox, 0, width - 1)] ?? 0);
}
next[y * width + x] = value;
}
}
return next;
}
export function dilateMaskValues(values: Uint8ClampedArray, width: number, height: number, radius: number): Uint8ClampedArray {
const safeRadius = Math.round(clampNumber(radius, 0, 256));
if (safeRadius <= 0) return new Uint8ClampedArray(values);
const next = new Uint8ClampedArray(values.length);
for (let y = 0; y < height; y += 1) {
for (let x = 0; x < width; x += 1) {
let value = 0;
for (let oy = -safeRadius; oy <= safeRadius; oy += 1) {
for (let ox = -safeRadius; ox <= safeRadius; ox += 1) value = Math.max(value, values[clampInt(y + oy, 0, height - 1) * width + clampInt(x + ox, 0, width - 1)] ?? 0);
}
next[y * width + x] = value;
}
}
return next;
}
export function blurMaskValues(values: Uint8ClampedArray, width: number, height: number, radius: number): Uint8ClampedArray {
const safeRadius = Math.round(clampNumber(radius, 0, 256));
if (safeRadius <= 0) return new Uint8ClampedArray(values);
const next = new Uint8ClampedArray(values.length);
for (let y = 0; y < height; y += 1) {
for (let x = 0; x < width; x += 1) {
let total = 0;
let count = 0;
for (let oy = -safeRadius; oy <= safeRadius; oy += 1) {
for (let ox = -safeRadius; ox <= safeRadius; ox += 1) {
total += values[clampInt(y + oy, 0, height - 1) * width + clampInt(x + ox, 0, width - 1)] ?? 0;
count += 1;
}
}
next[y * width + x] = Math.round(total / count);
}
}
return next;
}
export function despeckleMaskValues(values: Uint8ClampedArray, width: number, height: number, strength: number): Uint8ClampedArray {
const safeStrength = Math.round(clampNumber(strength, 0, 64));
if (safeStrength <= 0) return new Uint8ClampedArray(values);
const radius = Math.max(1, Math.ceil(safeStrength / 6));
const threshold = Math.max(1, Math.round(safeStrength / 2));
const next = new Uint8ClampedArray(values);
for (let y = 0; y < height; y += 1) {
for (let x = 0; x < width; x += 1) {
const index = y * width + x;
const visible = (values[index] ?? 0) > 127;
let same = 0;
for (let oy = -radius; oy <= radius; oy += 1) {
for (let ox = -radius; ox <= radius; ox += 1) {
if (ox === 0 && oy === 0) continue;
const sample = values[clampInt(y + oy, 0, height - 1) * width + clampInt(x + ox, 0, width - 1)] ?? 0;
if ((sample > 127) === visible) same += 1;
}
}
if (same <= threshold) next[index] = visible ? 0 : 255;
}
}
return next;
}
async function loadMaskValues(source: string, width: number, height: number): Promise<{ width: number; height: number; values: Uint8ClampedArray }> {
const canvas = await loadImageCanvas(source, width, height);
const context = require2dContext(canvas);
const data = context.getImageData(0, 0, canvas.width, canvas.height);
const values = new Uint8ClampedArray(canvas.width * canvas.height);
for (let pixel = 0; pixel < values.length; pixel += 1) values[pixel] = maskValueFromRgba(data.data, pixel * 4);
return { width: canvas.width, height: canvas.height, values };
}
function maskValuesToDataUrl(values: Uint8ClampedArray, width: number, height: number, outputWidth = width, outputHeight = height): string {
const canvas = createCanvas(outputWidth, outputHeight);
const context = require2dContext(canvas);
const imageData = context.createImageData(outputWidth, outputHeight);
for (let y = 0; y < outputHeight; y += 1) {
for (let x = 0; x < outputWidth; x += 1) {
const sourceX = Math.floor((x / outputWidth) * width);
const sourceY = Math.floor((y / outputHeight) * height);
const value = values[clampInt(sourceY, 0, height - 1) * width + clampInt(sourceX, 0, width - 1)] ?? 0;
const index = (y * outputWidth + x) * 4;
imageData.data[index] = 255;
imageData.data[index + 1] = 255;
imageData.data[index + 2] = 255;
imageData.data[index + 3] = value;
}
}
context.putImageData(imageData, 0, 0);
return canvas.toDataURL("image/png");
}
function analyzeValues(values: Uint8ClampedArray, width: number, height: number, includeSoftPixels: boolean): { pixels: number; coverage: number; bounds?: Rect } {
let pixels = 0;
let minX = Number.POSITIVE_INFINITY;
let minY = Number.POSITIVE_INFINITY;
let maxX = Number.NEGATIVE_INFINITY;
let maxY = Number.NEGATIVE_INFINITY;
for (let y = 0; y < height; y += 1) {
for (let x = 0; x < width; x += 1) {
const value = values[y * width + x] ?? 0;
const active = includeSoftPixels ? value > 0 : value > 127;
if (!active) continue;
pixels += 1;
minX = Math.min(minX, x);
minY = Math.min(minY, y);
maxX = Math.max(maxX, x + 1);
maxY = Math.max(maxY, y + 1);
}
}
return {
pixels,
coverage: pixels / Math.max(1, width * height),
bounds: pixels > 0 ? { x: minX, y: minY, w: maxX - minX, h: maxY - minY } : undefined,
};
}
function createCanvas(width: number, height: number): HTMLCanvasElement {
const canvas = document.createElement("canvas");
canvas.width = Math.max(1, Math.round(width));
canvas.height = Math.max(1, Math.round(height));
return canvas;
}
function require2dContext(canvas: HTMLCanvasElement): CanvasRenderingContext2D {
const context = canvas.getContext("2d");
if (!context) throw new Error("Unable to create mask canvas");
return context;
}
function loadImage(source: string): Promise<HTMLImageElement> {
return new Promise((resolve, reject) => {
const image = new Image();
image.onload = () => resolve(image);
image.onerror = () => reject(new Error("Failed to load image"));
image.src = source;
});
}
function roundUp(value: number, multiple: number): number {
const safeMultiple = Math.max(1, Math.round(multiple));
return Math.ceil(value / safeMultiple) * safeMultiple;
}
function clampNumber(value: number, min: number, max: number): number {
if (!Number.isFinite(value)) return min;
return Math.max(min, Math.min(max, value));
}
function clampInt(value: number, min: number, max: number): number {
return Math.round(Math.max(min, Math.min(max, value)));
}