feat: implement inpainting functionality with mask handling
- Added `createMaskedPixelReplacementSource` function to handle pixel replacement using inpainting. - Introduced `buildInpaintBundle` to prepare inpainting data including mask generation and validation. - Created utility functions for mask operations such as `applyMaskedContentModeToRgba`, `expandRectWithinBounds`, and others for mask manipulation. - Developed tests for inpainting preparation and mask raster utilities to ensure functionality and correctness. - Implemented mask raster operations including inversion, feathering, blurring, and more.
This commit is contained in:
@@ -25,7 +25,7 @@ export type AppProps = {
|
||||
|
||||
export function App({ app }: AppProps) {
|
||||
const shellState = useAppState(app.store, selectAppShellState, shallowEqual);
|
||||
const { document, selection, viewport, tools, transformSession, maskEdit } = shellState;
|
||||
const { document, selection, viewport, tools, generation, transformSession, maskEdit } = shellState;
|
||||
const viewportActivityIsland = useViewportActivityIsland(viewport);
|
||||
const imageImport = useImageImport(app.store);
|
||||
const [layersOpen, setLayersOpen] = useState(false);
|
||||
@@ -161,6 +161,7 @@ export function App({ app }: AppProps) {
|
||||
activeTool={tools.activeTool}
|
||||
brushSettings={tools.brush}
|
||||
generateSettings={tools.generate}
|
||||
generation={generation}
|
||||
chromaKeySettings={tools.chromaKey}
|
||||
magicWandSettings={tools.magicWand}
|
||||
editingMask={Boolean(maskEdit)}
|
||||
@@ -189,6 +190,7 @@ type AppShellState = {
|
||||
selection: AppState["editor"]["selection"];
|
||||
viewport: AppState["editor"]["viewport"];
|
||||
tools: AppState["editor"]["tools"];
|
||||
generation: AppState["editor"]["generation"];
|
||||
transformSession: AppState["editor"]["transformSession"];
|
||||
maskEdit: AppState["editor"]["maskEdit"];
|
||||
};
|
||||
@@ -199,6 +201,7 @@ function selectAppShellState(state: AppState): AppShellState {
|
||||
selection: state.editor.selection,
|
||||
viewport: state.editor.viewport,
|
||||
tools: state.editor.tools,
|
||||
generation: state.editor.generation,
|
||||
transformSession: state.editor.transformSession,
|
||||
maskEdit: state.editor.maskEdit,
|
||||
};
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import type { AppStore } from "@editor/store";
|
||||
import type { ImageDocument } from "@core/document";
|
||||
import type { MaskViewMode, SelectionState, ViewportState } from "@editor/state";
|
||||
import type { GenerationState, MaskViewMode, SelectionState, ViewportState } from "@editor/state";
|
||||
import type { BrushSettings, ChromaKeySettings, GenerateSettings, MagicWandSettings, ToolId } from "@editor/tools";
|
||||
import { BrushControls } from "./bottom-controls/BrushControls";
|
||||
import { ChromaKeyControls } from "./bottom-controls/ChromaKeyControls";
|
||||
@@ -22,6 +22,7 @@ export type BottomControlsIslandProps = {
|
||||
activeTool: ToolId;
|
||||
brushSettings: BrushSettings;
|
||||
generateSettings: GenerateSettings;
|
||||
generation: GenerationState;
|
||||
chromaKeySettings: ChromaKeySettings;
|
||||
magicWandSettings: MagicWandSettings;
|
||||
editingMask?: boolean;
|
||||
@@ -32,7 +33,7 @@ export type BottomControlsIslandProps = {
|
||||
dispatch: AppStore["dispatch"];
|
||||
};
|
||||
|
||||
export function BottomControlsIsland({ document, selection, viewport, visible, action, activeTool, brushSettings, generateSettings, chromaKeySettings, magicWandSettings, editingMask = false, maskViewMode = "composite", transformBounds, transformTarget, brushHint, dispatch }: BottomControlsIslandProps) {
|
||||
export function BottomControlsIsland({ document, selection, viewport, visible, action, activeTool, brushSettings, generateSettings, generation, chromaKeySettings, magicWandSettings, editingMask = false, maskViewMode = "composite", transformBounds, transformTarget, brushHint, dispatch }: BottomControlsIslandProps) {
|
||||
const zoomPercent = Math.round(viewport.zoom * 100);
|
||||
const x = Math.round(viewport.center.x);
|
||||
const y = Math.round(viewport.center.y);
|
||||
@@ -40,12 +41,12 @@ export function BottomControlsIsland({ document, selection, viewport, visible, a
|
||||
return (
|
||||
<div
|
||||
aria-hidden={!visible}
|
||||
className={`flex h-20 min-w-96 rounded-full px-4 py-2 items-center justify-center gap-4 text-sm text-white backdrop-blur transition-all duration-200 ${
|
||||
className={`flex min-h-20 min-w-96 rounded-[2rem] px-4 py-2 items-center justify-center gap-4 text-sm text-white backdrop-blur transition-all duration-200 ${
|
||||
visible ? "pointer-events-auto translate-y-0 opacity-100" : "pointer-events-none translate-y-3 opacity-0"
|
||||
}`}
|
||||
>
|
||||
{activeTool === "generate" ? (
|
||||
<GenerateActionControls document={document} selection={selection} viewport={viewport} settings={generateSettings} dispatch={dispatch} />
|
||||
<GenerateActionControls document={document} selection={selection} viewport={viewport} settings={generateSettings} generation={generation} dispatch={dispatch} />
|
||||
) : (activeTool === "brush" || activeTool === "eraser") && brushHint ? (
|
||||
<BrushHint tool={activeTool} hint={brushHint} />
|
||||
) : activeTool === "brush" || activeTool === "eraser" ? (
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { useMemo, useRef, useState, type DragEvent, type MutableRefObject } from "react";
|
||||
import { useEffect, useMemo, useRef, useState, type DragEvent, type MutableRefObject } from "react";
|
||||
import { ArrowDown, ArrowUp, DownloadSimple, Eye, EyeSlash, FolderPlus, Lock, LockOpen, Plus, Stack, Trash } from "@phosphor-icons/react";
|
||||
import { commandIds } from "@commands/ids";
|
||||
import type { Asset } from "@core/asset";
|
||||
import type { ImageDocument } from "@core/document";
|
||||
import type { Layer } from "@core/layer";
|
||||
import type { ArtboardId } from "@core/id";
|
||||
@@ -9,6 +10,7 @@ import type { MaskEditState, SelectionState } from "@editor/state";
|
||||
import type { AppStore } from "@editor/store";
|
||||
import { resolveLayerDrop } from "@input/index";
|
||||
import { downloadArtboardPng } from "./exportArtboardPng";
|
||||
import { analyzeMaskSource, applyMaskRasterOperation, type MaskAnalysis, type MaskRasterOperation } from "./mask/maskRaster";
|
||||
|
||||
export type LayersSheetProps = {
|
||||
document: ImageDocument;
|
||||
@@ -219,6 +221,7 @@ function LayerRow({
|
||||
const selected = selectedLayerIds.includes(layer.id);
|
||||
const layerInfo = documentIndex.layerInfoById.get(layer.id);
|
||||
const maskLayer = layer.clippingMask ? documentIndex.layerById.get(layer.clippingMask.maskLayerId) : undefined;
|
||||
const maskAsset = maskLayer && maskLayer.type !== "group" ? documentIndex.assetById.get(maskLayer.assetId) : undefined;
|
||||
const canAddMask = Boolean(layerInfo && layer.type !== "group" && !layer.clippingMask);
|
||||
const editingMask = Boolean(maskEdit && layer.clippingMask && maskEdit.targetLayerId === layer.id && maskEdit.maskLayerId === layer.clippingMask.maskLayerId);
|
||||
const rowPadding = 12 + depth * 16;
|
||||
@@ -288,21 +291,49 @@ function LayerRow({
|
||||
</button>
|
||||
</div>
|
||||
{layer.clippingMask ? (
|
||||
<div className="mt-2 flex min-h-14 items-center gap-3 rounded-full py-2 pl-4 pr-2 text-sm text-sky-100/70 hover:bg-white/[0.04]">
|
||||
<Stack size={24} weight="fill" />
|
||||
<span className="min-w-0 flex-1 truncate">{maskLayer ? "Layer mask" : "Layer mask missing"}</span>
|
||||
<div className="mt-2 flex min-h-14 flex-wrap items-center gap-2 rounded-[1.5rem] py-2 pl-4 pr-2 text-sm text-sky-100/70 hover:bg-white/[0.04]">
|
||||
<Stack size={24} weight="fill" className="shrink-0" />
|
||||
<span className="min-w-28 flex-1 truncate">{maskLayer ? "Layer mask" : "Layer mask missing"}</span>
|
||||
{maskAsset ? <MaskStatus asset={maskAsset} /> : null}
|
||||
{maskLayer ? (
|
||||
<button
|
||||
type="button"
|
||||
className={`h-9 rounded-full px-4 text-sm font-medium transition ${editingMask ? "bg-sky-300 text-black" : "text-sky-100/75 hover:bg-white/10 hover:text-sky-50"}`}
|
||||
onClick={() =>
|
||||
editingMask
|
||||
? dispatch(commandIds.toolExitMaskEdit, undefined)
|
||||
: dispatch(commandIds.toolEnterMaskEdit, { targetLayerId: layer.id, maskLayerId: layer.clippingMask!.maskLayerId })
|
||||
}
|
||||
>
|
||||
{editingMask ? "Done" : "Edit"}
|
||||
</button>
|
||||
<div className="flex flex-wrap items-center gap-1">
|
||||
<button
|
||||
type="button"
|
||||
className={`h-9 rounded-full px-4 text-sm font-medium transition ${editingMask ? "bg-sky-300 text-black" : "text-sky-100/75 hover:bg-white/10 hover:text-sky-50"}`}
|
||||
onClick={() =>
|
||||
editingMask
|
||||
? dispatch(commandIds.toolExitMaskEdit, undefined)
|
||||
: dispatch(commandIds.toolEnterMaskEdit, { targetLayerId: layer.id, maskLayerId: layer.clippingMask!.maskLayerId })
|
||||
}
|
||||
>
|
||||
{editingMask ? "Done" : "Edit"}
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className={maskActionButtonClass()}
|
||||
title="Paint reveal"
|
||||
onClick={() => {
|
||||
dispatch(commandIds.toolEnterMaskEdit, { targetLayerId: layer.id, maskLayerId: layer.clippingMask!.maskLayerId });
|
||||
dispatch(commandIds.toolSetActive, { tool: "brush" });
|
||||
}}
|
||||
>
|
||||
Reveal
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className={maskActionButtonClass()}
|
||||
title="Paint hide"
|
||||
onClick={() => {
|
||||
dispatch(commandIds.toolEnterMaskEdit, { targetLayerId: layer.id, maskLayerId: layer.clippingMask!.maskLayerId });
|
||||
dispatch(commandIds.toolSetActive, { tool: "eraser" });
|
||||
}}
|
||||
>
|
||||
Hide
|
||||
</button>
|
||||
{maskAsset && maskLayer.type !== "group" ? (
|
||||
<MaskOperationButtons maskLayerId={maskLayer.id} maskAsset={maskAsset} dispatch={dispatch} />
|
||||
) : null}
|
||||
</div>
|
||||
) : null}
|
||||
<button
|
||||
type="button"
|
||||
@@ -336,6 +367,83 @@ function LayerRow({
|
||||
);
|
||||
}
|
||||
|
||||
function MaskStatus({ asset }: { asset: Asset }) {
|
||||
const [analysis, setAnalysis] = useState<MaskAnalysis>();
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
void analyzeMaskSource(asset.source, asset.intrinsicSize.w, asset.intrinsicSize.h)
|
||||
.then((nextAnalysis) => {
|
||||
if (!cancelled) setAnalysis(nextAnalysis);
|
||||
})
|
||||
.catch(() => {
|
||||
if (!cancelled) setAnalysis(undefined);
|
||||
});
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [asset.source, asset.intrinsicSize.w, asset.intrinsicSize.h]);
|
||||
|
||||
if (!analysis) return <span className="rounded-full bg-white/5 px-3 py-1 text-xs text-sky-100/45">Reading</span>;
|
||||
return (
|
||||
<span className="inline-flex min-w-0 items-center gap-2 rounded-full bg-white/5 px-2 py-1 text-xs text-sky-100/65">
|
||||
<img src={analysis.thumbnail} alt="" className="h-7 w-10 rounded-md bg-black/30 object-cover ring-1 ring-white/10" />
|
||||
<span className="whitespace-nowrap">Reveal {formatPercent(analysis.coverage)}</span>
|
||||
<span className="whitespace-nowrap text-sky-100/45">Inpaint {formatPercent(analysis.hiddenCoverage)}</span>
|
||||
</span>
|
||||
);
|
||||
}
|
||||
|
||||
function MaskOperationButtons({ maskLayerId, maskAsset, dispatch }: { maskLayerId: string; maskAsset: Asset; dispatch: AppStore["dispatch"] }) {
|
||||
return (
|
||||
<>
|
||||
<MaskOperationButton label="Invert" title="Invert mask" maskLayerId={maskLayerId} maskAsset={maskAsset} operation={{ type: "invert" }} dispatch={dispatch} />
|
||||
<MaskOperationButton label="White" title="Fill mask white" maskLayerId={maskLayerId} maskAsset={maskAsset} operation={{ type: "fill", fill: "white" }} dispatch={dispatch} />
|
||||
<MaskOperationButton label="Black" title="Fill mask black" maskLayerId={maskLayerId} maskAsset={maskAsset} operation={{ type: "fill", fill: "black" }} dispatch={dispatch} />
|
||||
<MaskOperationButton label="Clear" title="Clear mask" maskLayerId={maskLayerId} maskAsset={maskAsset} operation={{ type: "fill", fill: "clear" }} dispatch={dispatch} />
|
||||
<MaskOperationButton label="Feather" title="Feather mask edge" maskLayerId={maskLayerId} maskAsset={maskAsset} operation={{ type: "feather", radius: 3 }} dispatch={dispatch} />
|
||||
<MaskOperationButton label="Expand" title="Expand mask" maskLayerId={maskLayerId} maskAsset={maskAsset} operation={{ type: "expand", radius: 3 }} dispatch={dispatch} />
|
||||
<MaskOperationButton label="Contract" title="Contract mask" maskLayerId={maskLayerId} maskAsset={maskAsset} operation={{ type: "contract", radius: 3 }} dispatch={dispatch} />
|
||||
<MaskOperationButton label="Blur" title="Blur mask edge" maskLayerId={maskLayerId} maskAsset={maskAsset} operation={{ type: "blur", radius: 2 }} dispatch={dispatch} />
|
||||
<MaskOperationButton label="Clean" title="Despeckle mask" maskLayerId={maskLayerId} maskAsset={maskAsset} operation={{ type: "despeckle", strength: 8 }} dispatch={dispatch} />
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
function MaskOperationButton({
|
||||
label,
|
||||
title,
|
||||
maskLayerId,
|
||||
maskAsset,
|
||||
operation,
|
||||
dispatch,
|
||||
}: {
|
||||
label: string;
|
||||
title: string;
|
||||
maskLayerId: string;
|
||||
maskAsset: Asset;
|
||||
operation: MaskRasterOperation;
|
||||
dispatch: AppStore["dispatch"];
|
||||
}) {
|
||||
const [busy, setBusy] = useState(false);
|
||||
return (
|
||||
<button
|
||||
type="button"
|
||||
className={maskActionButtonClass()}
|
||||
disabled={busy}
|
||||
title={title}
|
||||
onClick={() => {
|
||||
setBusy(true);
|
||||
void applyMaskRasterOperation(maskAsset.source, maskAsset.intrinsicSize.w, maskAsset.intrinsicSize.h, operation)
|
||||
.then((source) => dispatch(commandIds.documentApplyLayerMaskOperation, { maskLayerId, source, mimeType: "image/png", operation }))
|
||||
.finally(() => setBusy(false));
|
||||
}}
|
||||
>
|
||||
{busy ? "..." : label}
|
||||
</button>
|
||||
);
|
||||
}
|
||||
|
||||
type EditingTitle =
|
||||
| { type: "artboard"; id: ArtboardId; draft: string }
|
||||
| { type: "layer"; id: string; draft: string };
|
||||
@@ -528,3 +636,11 @@ function toolbarButtonClass() {
|
||||
function labeledToolbarButtonClass() {
|
||||
return "inline-flex h-12 flex-1 items-center rounded-full pr-4 text-sm font-medium text-white/75 transition hover:bg-white/10 hover:text-white disabled:pointer-events-none disabled:opacity-35 focus:outline-none focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-white/30";
|
||||
}
|
||||
|
||||
function maskActionButtonClass() {
|
||||
return "h-8 rounded-full bg-white/5 px-3 text-xs font-medium text-sky-100/65 transition hover:bg-white/10 hover:text-sky-50 disabled:pointer-events-none disabled:opacity-35";
|
||||
}
|
||||
|
||||
function formatPercent(value: number) {
|
||||
return `${Math.round(Math.max(0, Math.min(1, value)) * 100)}%`;
|
||||
}
|
||||
|
||||
@@ -31,6 +31,13 @@ export function BrushControls({ tool, settings, editingMask = false, maskViewMod
|
||||
{tool === "eraser" ? <Eraser size={24} weight="regular" /> : <PaintBrush size={24} weight="regular" />}
|
||||
</span>
|
||||
<BottomControlDivider />
|
||||
{editingMask ? (
|
||||
<>
|
||||
<button type="button" className={maskModeButtonClass(tool === "brush")} onClick={() => dispatch(commandIds.toolSetActive, { tool: "brush" })}>Reveal</button>
|
||||
<button type="button" className={maskModeButtonClass(tool === "eraser")} onClick={() => dispatch(commandIds.toolSetActive, { tool: "eraser" })}>Hide</button>
|
||||
<BottomControlDivider />
|
||||
</>
|
||||
) : null}
|
||||
{tool === "brush" && !editingMask ? (
|
||||
<label className={bottomControlFieldClass()}>
|
||||
<span className={bottomControlLabelClass()}>Color</span>
|
||||
@@ -91,3 +98,7 @@ export function BrushControls({ tool, settings, editingMask = false, maskViewMod
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function maskModeButtonClass(active: boolean) {
|
||||
return `rounded-full px-4 py-2 text-base font-medium transition ${active ? "bg-white text-black" : "bg-white/10 text-white hover:bg-white/15"}`;
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import { resolveTransformTargetBounds } from "@editor/transform-targets";
|
||||
import type { AppStore } from "@editor/store";
|
||||
import type { ChromaKeySettings } from "@editor/tools";
|
||||
import type { SelectionState } from "@editor/state";
|
||||
import { blurMaskValues, despeckleMaskValues, dilateMaskValues, erodeMaskValues } from "../mask/maskRaster";
|
||||
import { BottomControlColorPicker } from "./ColorPicker";
|
||||
import { BottomControlDivider } from "./Divider";
|
||||
import { BottomControlSlider } from "./Slider";
|
||||
@@ -179,8 +180,8 @@ async function applyChromaKeyMask(target: NonNullable<ReturnType<typeof resolveC
|
||||
const source = await chromaKeyMaskSource(target.asset.source, target.asset.intrinsicSize.w, target.asset.intrinsicSize.h, settings);
|
||||
dispatch(commandIds.toolSetBrushStrokePreview, undefined);
|
||||
|
||||
if (target.maskAsset) {
|
||||
dispatch(commandIds.documentUpdateAssetSource, { assetId: target.maskAsset.id, source });
|
||||
if (target.maskAsset && target.maskLayer && target.maskLayer.type !== "group") {
|
||||
dispatch(commandIds.documentApplyLayerMaskOperation, { maskLayerId: target.maskLayer.id, source, mimeType: "image/png", operation: { type: "chromaKey" } });
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -293,85 +294,13 @@ function postProcessAlpha(alpha: Uint8ClampedArray, width: number, height: numbe
|
||||
const choke = Math.round(Math.max(-20, Math.min(20, settings.choke)));
|
||||
const feather = Math.round(Math.max(0, Math.min(20, settings.feather)));
|
||||
|
||||
if (despeckle > 0) next = despeckleAlpha(next, width, height, despeckle);
|
||||
if (choke > 0) next = erodeAlpha(next, width, height, choke);
|
||||
if (choke < 0) next = dilateAlpha(next, width, height, -choke);
|
||||
if (feather > 0) next = blurAlpha(next, width, height, feather);
|
||||
if (despeckle > 0) next = despeckleMaskValues(next, width, height, despeckle);
|
||||
if (choke > 0) next = erodeMaskValues(next, width, height, choke);
|
||||
if (choke < 0) next = dilateMaskValues(next, width, height, -choke);
|
||||
if (feather > 0) next = blurMaskValues(next, width, height, feather);
|
||||
return next;
|
||||
}
|
||||
|
||||
function despeckleAlpha(alpha: Uint8ClampedArray, width: number, height: number, strength: number) {
|
||||
const radius = Math.max(1, Math.ceil(strength / 6));
|
||||
const threshold = Math.max(1, Math.round(strength / 2));
|
||||
const next = new Uint8ClampedArray(alpha);
|
||||
for (let y = 0; y < height; y++) {
|
||||
for (let x = 0; x < width; x++) {
|
||||
const index = y * width + x;
|
||||
const visible = (alpha[index] ?? 0) > 127;
|
||||
let same = 0;
|
||||
for (let oy = -radius; oy <= radius; oy++) {
|
||||
for (let ox = -radius; ox <= radius; ox++) {
|
||||
if (ox === 0 && oy === 0) continue;
|
||||
const sample = alpha[clamp(y + oy, 0, height - 1) * width + clamp(x + ox, 0, width - 1)] ?? 0;
|
||||
if ((sample > 127) === visible) same += 1;
|
||||
}
|
||||
}
|
||||
if (same <= threshold) next[index] = visible ? 0 : 255;
|
||||
}
|
||||
}
|
||||
return next;
|
||||
}
|
||||
|
||||
function erodeAlpha(alpha: Uint8ClampedArray, width: number, height: number, radius: number) {
|
||||
const next = new Uint8ClampedArray(alpha.length);
|
||||
for (let y = 0; y < height; y++) {
|
||||
for (let x = 0; x < width; x++) {
|
||||
let value = 255;
|
||||
for (let oy = -radius; oy <= radius; oy++) {
|
||||
for (let ox = -radius; ox <= radius; ox++) value = Math.min(value, alpha[clamp(y + oy, 0, height - 1) * width + clamp(x + ox, 0, width - 1)] ?? 0);
|
||||
}
|
||||
next[y * width + x] = value;
|
||||
}
|
||||
}
|
||||
return next;
|
||||
}
|
||||
|
||||
function dilateAlpha(alpha: Uint8ClampedArray, width: number, height: number, radius: number) {
|
||||
const next = new Uint8ClampedArray(alpha.length);
|
||||
for (let y = 0; y < height; y++) {
|
||||
for (let x = 0; x < width; x++) {
|
||||
let value = 0;
|
||||
for (let oy = -radius; oy <= radius; oy++) {
|
||||
for (let ox = -radius; ox <= radius; ox++) value = Math.max(value, alpha[clamp(y + oy, 0, height - 1) * width + clamp(x + ox, 0, width - 1)] ?? 0);
|
||||
}
|
||||
next[y * width + x] = value;
|
||||
}
|
||||
}
|
||||
return next;
|
||||
}
|
||||
|
||||
function blurAlpha(alpha: Uint8ClampedArray, width: number, height: number, radius: number) {
|
||||
const next = new Uint8ClampedArray(alpha.length);
|
||||
for (let y = 0; y < height; y++) {
|
||||
for (let x = 0; x < width; x++) {
|
||||
let total = 0;
|
||||
let count = 0;
|
||||
for (let oy = -radius; oy <= radius; oy++) {
|
||||
for (let ox = -radius; ox <= radius; ox++) {
|
||||
total += alpha[clamp(y + oy, 0, height - 1) * width + clamp(x + ox, 0, width - 1)] ?? 0;
|
||||
count += 1;
|
||||
}
|
||||
}
|
||||
next[y * width + x] = Math.round(total / count);
|
||||
}
|
||||
}
|
||||
return next;
|
||||
}
|
||||
|
||||
function clamp(value: number, min: number, max: number) {
|
||||
return Math.max(min, Math.min(max, value));
|
||||
}
|
||||
|
||||
function loadImage(source: string) {
|
||||
return new Promise<HTMLImageElement>((resolve, reject) => {
|
||||
const image = new Image();
|
||||
|
||||
@@ -1,40 +1,271 @@
|
||||
import { useState } from "react";
|
||||
import { useEffect, useState } from "react";
|
||||
import { commandIds } from "@commands/ids";
|
||||
import type { ImageDocument } from "@core/document";
|
||||
import type { SelectionState, ViewportState } from "@editor/state";
|
||||
import type { GenerationCandidate, GenerationState, SelectionState, ViewportState } from "@editor/state";
|
||||
import type { GenerateSettings } from "@editor/tools";
|
||||
import type { AppStore } from "@editor/store";
|
||||
import { runGenerate } from "../generate/runGenerate";
|
||||
import { createMaskedPixelReplacementSource } from "../generate/candidateActions";
|
||||
import { runGenerate, runGenerateFromCandidate } from "../generate/runGenerate";
|
||||
import { createSolidMaskSource } from "../mask/maskRaster";
|
||||
|
||||
export type GenerateActionControlsProps = {
|
||||
document: ImageDocument;
|
||||
selection: SelectionState;
|
||||
viewport: ViewportState;
|
||||
settings: GenerateSettings;
|
||||
generation: GenerationState;
|
||||
dispatch: AppStore["dispatch"];
|
||||
};
|
||||
|
||||
export function GenerateActionControls({ document, selection, viewport, settings, dispatch }: GenerateActionControlsProps) {
|
||||
const [busy, setBusy] = useState(false);
|
||||
export function GenerateActionControls({ document, selection, viewport, settings, generation, dispatch }: GenerateActionControlsProps) {
|
||||
const [busy, setBusy] = useState<string>();
|
||||
const [error, setError] = useState<string>();
|
||||
const [elapsedSeconds, setElapsedSeconds] = useState(0);
|
||||
const candidate = selectedCandidate(generation);
|
||||
const canGenerate = Boolean(settings.prompt.trim()) && !busy;
|
||||
|
||||
useEffect(() => {
|
||||
if (!busy) {
|
||||
setElapsedSeconds(0);
|
||||
return;
|
||||
}
|
||||
|
||||
setElapsedSeconds(0);
|
||||
const startedAt = Date.now();
|
||||
const interval = window.setInterval(() => {
|
||||
setElapsedSeconds(Math.floor((Date.now() - startedAt) / 1000));
|
||||
}, 1000);
|
||||
return () => window.clearInterval(interval);
|
||||
}, [busy]);
|
||||
|
||||
return (
|
||||
<div className="flex items-center px-2">
|
||||
<div className="flex max-w-[calc(100vw-2rem)] flex-wrap items-center justify-center gap-2 px-2">
|
||||
<button
|
||||
type="button"
|
||||
disabled={!canGenerate}
|
||||
className="h-12 rounded-full bg-white px-7 text-base font-semibold !text-black transition hover:bg-white/90 focus:outline-none focus-visible:ring-2 focus-visible:ring-white/40 disabled:pointer-events-none disabled:opacity-35"
|
||||
title={error ?? "Generate with ComfyUI"}
|
||||
onClick={() => {
|
||||
setBusy(true);
|
||||
setBusy("Generating");
|
||||
setError(undefined);
|
||||
void runGenerate({ document, selection, viewport, settings, dispatch })
|
||||
.catch((reason: unknown) => setError(reason instanceof Error ? reason.message : "Generation failed"))
|
||||
.finally(() => setBusy(false));
|
||||
.finally(() => setBusy(undefined));
|
||||
}}
|
||||
>
|
||||
{busy ? "Generating…" : "Generate"}
|
||||
{busy === "Generating" ? `Generating ${formatElapsed(elapsedSeconds)}` : "Generate"}
|
||||
</button>
|
||||
{busy ? <span className="rounded-full bg-white/10 px-3 py-2 text-xs font-medium text-white/60">{busy} {formatElapsed(elapsedSeconds)}</span> : null}
|
||||
{candidate ? (
|
||||
<>
|
||||
<CandidatePicker generation={generation} dispatch={dispatch} />
|
||||
<CandidateControls
|
||||
document={document}
|
||||
candidate={candidate}
|
||||
settings={settings}
|
||||
busy={busy}
|
||||
setBusy={setBusy}
|
||||
setError={setError}
|
||||
dispatch={dispatch}
|
||||
/>
|
||||
</>
|
||||
) : null}
|
||||
{error ? <span className="max-w-80 truncate rounded-full bg-red-500/15 px-3 py-2 text-xs font-medium text-red-100" title={error}>{error}</span> : null}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function CandidatePicker({ generation, dispatch }: { generation: GenerationState; dispatch: AppStore["dispatch"] }) {
|
||||
if (generation.candidates.length < 2) return null;
|
||||
return (
|
||||
<div className="flex items-center gap-1 rounded-full bg-white/[0.04] px-1.5 py-1 ring-1 ring-white/[0.05]">
|
||||
{generation.candidates.slice(0, 6).map((candidate) => {
|
||||
const selected = candidate.id === (generation.selectedCandidateId ?? generation.candidates[0]?.id);
|
||||
return (
|
||||
<button
|
||||
key={candidate.id}
|
||||
type="button"
|
||||
className={`h-9 w-9 overflow-hidden rounded-full ring-2 transition ${selected ? "ring-white" : "ring-white/10 hover:ring-white/40"}`}
|
||||
title={`Candidate seed ${candidate.seed}`}
|
||||
onClick={() => dispatch(commandIds.generationSelectCandidate, { candidateId: candidate.id })}
|
||||
>
|
||||
<img src={candidate.source} alt="" className="h-full w-full object-cover" />
|
||||
</button>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function CandidateControls({
|
||||
document,
|
||||
candidate,
|
||||
settings,
|
||||
busy,
|
||||
setBusy,
|
||||
setError,
|
||||
dispatch,
|
||||
}: {
|
||||
document: ImageDocument;
|
||||
candidate: GenerationCandidate;
|
||||
settings: GenerateSettings;
|
||||
busy?: string;
|
||||
setBusy: (busy: string | undefined) => void;
|
||||
setError: (error: string | undefined) => void;
|
||||
dispatch: AppStore["dispatch"];
|
||||
}) {
|
||||
const rerun = (label: string, nextSettings: GenerateSettings) => {
|
||||
setBusy(label);
|
||||
setError(undefined);
|
||||
dispatch(commandIds.toolSetGenerateSettings, nextSettings);
|
||||
void runGenerateFromCandidate({ candidate, settings: nextSettings, dispatch })
|
||||
.catch((reason: unknown) => setError(reason instanceof Error ? reason.message : `${label} failed`))
|
||||
.finally(() => setBusy(undefined));
|
||||
};
|
||||
const disabled = Boolean(busy);
|
||||
|
||||
return (
|
||||
<div className="flex flex-wrap items-center justify-center gap-1 rounded-full bg-white/[0.04] px-2 py-1 ring-1 ring-white/[0.05]">
|
||||
<CandidatePreview candidate={candidate} />
|
||||
<span className="px-2 text-xs font-medium text-white/55">Seed {candidate.seed}</span>
|
||||
<CandidateButton disabled={disabled} label="Regenerate" title="Regenerate same mask and crop" busy={busy === "Regenerate"} onClick={() => rerun("Regenerate", candidate.settings)} />
|
||||
<CandidateButton
|
||||
disabled={disabled}
|
||||
label="Lower"
|
||||
title="Lower strength and regenerate same mask"
|
||||
busy={busy === "Lower"}
|
||||
onClick={() => rerun("Lower", { ...candidate.settings, strength: Math.max(0, candidate.settings.strength - 10), seed: candidate.seed })}
|
||||
/>
|
||||
<CandidateButton disabled={disabled} label="Reuse seed" title="Regenerate with the same seed" busy={busy === "Reuse seed"} onClick={() => rerun("Reuse seed", { ...candidate.settings, seed: candidate.seed })} />
|
||||
<CandidateButton disabled={disabled} label="New seed" title="Regenerate with a new seed" busy={busy === "New seed"} onClick={() => rerun("New seed", { ...candidate.settings, seed: -1 })} />
|
||||
<CandidateButton disabled={disabled} label="Apply layer" title="Apply candidate as a normal layer" onClick={() => applyCandidateAsLayer(candidate, false, dispatch)} />
|
||||
<CandidateButton
|
||||
disabled={disabled}
|
||||
label="Apply + refine"
|
||||
title="Apply candidate as a layer with a fresh refinement mask"
|
||||
busy={busy === "Refine"}
|
||||
onClick={() => {
|
||||
setBusy("Refine");
|
||||
setError(undefined);
|
||||
void applyCandidateAsRefinementLayer(candidate, dispatch)
|
||||
.catch((reason: unknown) => setError(reason instanceof Error ? reason.message : "Refine setup failed"))
|
||||
.finally(() => setBusy(undefined));
|
||||
}}
|
||||
/>
|
||||
<CandidateButton disabled={disabled} label="Stack variant" title="Stack candidate as another variant layer" onClick={() => applyCandidateAsLayer(candidate, true, dispatch)} />
|
||||
<CandidateButton
|
||||
disabled={disabled || !candidate.inpaint}
|
||||
label="Replace"
|
||||
title={candidate.inpaint ? "Replace masked pixels and preserve unmasked pixels" : "Only inpaint candidates can replace masked pixels"}
|
||||
busy={busy === "Replace"}
|
||||
onClick={() => {
|
||||
setBusy("Replace");
|
||||
setError(undefined);
|
||||
void createMaskedPixelReplacementSource(document, candidate)
|
||||
.then((source) => dispatch(commandIds.generationReplaceCandidatePixels, { candidateId: candidate.id, source, mimeType: "image/png" }))
|
||||
.catch((reason: unknown) => setError(reason instanceof Error ? reason.message : "Replace failed"))
|
||||
.finally(() => setBusy(undefined));
|
||||
}}
|
||||
/>
|
||||
<CandidateButton
|
||||
disabled={disabled || !candidate.inpaint}
|
||||
label="Edit mask"
|
||||
title={candidate.inpaint ? "Paint refinement mask over the source layer" : "Only inpaint candidates have a mask to edit"}
|
||||
onClick={() => {
|
||||
if (!candidate.inpaint) return;
|
||||
dispatch(commandIds.selectionSet, { artboardId: candidate.placement.artboardId, layerIds: [candidate.inpaint.targetLayerId] });
|
||||
dispatch(commandIds.toolEnterMaskEdit, { targetLayerId: candidate.inpaint.targetLayerId, maskLayerId: candidate.inpaint.maskLayerId });
|
||||
}}
|
||||
/>
|
||||
<CandidateButton disabled={disabled} label="Dismiss" title="Remove this candidate" onClick={() => dispatch(commandIds.generationRemoveCandidate, { candidateId: candidate.id })} />
|
||||
{settings.seed !== candidate.seed ? null : <span className="sr-only">Current settings reuse this seed</span>}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function CandidatePreview({ candidate }: { candidate: GenerationCandidate }) {
|
||||
if (!candidate.inputImage) {
|
||||
return <img src={candidate.source} alt="" className="h-10 w-10 rounded-full bg-black/25 object-cover ring-1 ring-white/10" />;
|
||||
}
|
||||
|
||||
return (
|
||||
<span className="flex items-center -space-x-2" title={candidate.maskImage ? "Input crop, normalized mask, generated candidate" : "Before and generated candidate"}>
|
||||
<img src={candidate.inputImage} alt="" className="h-10 w-10 rounded-full bg-black/25 object-cover ring-1 ring-white/10" />
|
||||
{candidate.maskImage ? <img src={candidate.maskImage} alt="" className="h-10 w-10 rounded-full bg-black/25 object-cover ring-1 ring-white/20" /> : null}
|
||||
<img src={candidate.source} alt="" className="h-10 w-10 rounded-full bg-black/25 object-cover ring-2 ring-white/40" />
|
||||
</span>
|
||||
);
|
||||
}
|
||||
|
||||
function CandidateButton({ label, title, disabled, busy, onClick }: { label: string; title: string; disabled?: boolean; busy?: boolean; onClick: () => void }) {
|
||||
return (
|
||||
<button
|
||||
type="button"
|
||||
className="h-9 rounded-full bg-white/5 px-3 text-xs font-semibold text-white/70 transition hover:bg-white/10 hover:text-white disabled:pointer-events-none disabled:opacity-35"
|
||||
disabled={disabled}
|
||||
title={title}
|
||||
onClick={onClick}
|
||||
>
|
||||
{busy ? "..." : label}
|
||||
</button>
|
||||
);
|
||||
}
|
||||
|
||||
function selectedCandidate(generation: GenerationState): GenerationCandidate | undefined {
|
||||
return generation.candidates.find((candidate) => candidate.id === generation.selectedCandidateId) ?? generation.candidates[0];
|
||||
}
|
||||
|
||||
function applyCandidateAsLayer(candidate: GenerationCandidate, variant: boolean, dispatch: AppStore["dispatch"]) {
|
||||
applyCandidateAsLayerWithIds(candidate, { layerId: crypto.randomUUID(), assetId: crypto.randomUUID() }, variant, dispatch);
|
||||
}
|
||||
|
||||
async function applyCandidateAsRefinementLayer(candidate: GenerationCandidate, dispatch: AppStore["dispatch"]) {
|
||||
const layerId = crypto.randomUUID();
|
||||
const maskLayerId = crypto.randomUUID();
|
||||
const maskAssetId = crypto.randomUUID();
|
||||
applyCandidateAsLayerWithIds(candidate, { layerId, assetId: crypto.randomUUID() }, true, dispatch);
|
||||
const width = Math.max(1, Math.round(candidate.intrinsicSize.w));
|
||||
const height = Math.max(1, Math.round(candidate.intrinsicSize.h));
|
||||
const source = await createSolidMaskSource(width, height, "white");
|
||||
|
||||
dispatch(commandIds.documentAddLayerMask, {
|
||||
layerId,
|
||||
asset: {
|
||||
id: maskAssetId,
|
||||
name: `${candidate.placement.layerName} refinement mask`,
|
||||
mimeType: "image/png",
|
||||
source,
|
||||
intrinsicSize: { w: width, h: height },
|
||||
},
|
||||
maskLayer: {
|
||||
id: maskLayerId,
|
||||
type: "raster",
|
||||
name: `${candidate.placement.layerName} refinement mask`,
|
||||
visible: true,
|
||||
locked: false,
|
||||
opacity: 1,
|
||||
assetId: maskAssetId,
|
||||
transform: {
|
||||
position: { ...candidate.placement.transform.position },
|
||||
scale: { ...candidate.placement.transform.scale },
|
||||
rotation: candidate.placement.transform.rotation,
|
||||
},
|
||||
},
|
||||
});
|
||||
dispatch(commandIds.toolSetActive, { tool: "eraser" });
|
||||
}
|
||||
|
||||
function applyCandidateAsLayerWithIds(candidate: GenerationCandidate, ids: { layerId: string; assetId: string }, variant: boolean, dispatch: AppStore["dispatch"]) {
|
||||
dispatch(commandIds.generationApplyCandidateAsLayer, {
|
||||
candidateId: candidate.id,
|
||||
assetId: ids.assetId,
|
||||
layerId: ids.layerId,
|
||||
variant,
|
||||
});
|
||||
}
|
||||
|
||||
function formatElapsed(seconds: number) {
|
||||
const minutes = Math.floor(seconds / 60);
|
||||
const remainder = seconds % 60;
|
||||
return `${minutes}:${remainder.toString().padStart(2, "0")}`;
|
||||
}
|
||||
|
||||
@@ -21,6 +21,18 @@ const sizePresets = [
|
||||
{ label: "9:16", w: 768, h: 1344 },
|
||||
] as const;
|
||||
|
||||
const inpaintPolarityOptions = [
|
||||
{ value: "hidden", label: "Hidden / erased" },
|
||||
{ value: "revealed", label: "Revealed / painted" },
|
||||
] satisfies readonly BottomControlSelectOption<GenerateSettings["inpaint"]["maskPolarity"]>[];
|
||||
|
||||
const inpaintMaskedContentOptions = [
|
||||
{ value: "neutral", label: "Neutral fill" },
|
||||
{ value: "original", label: "Original gray" },
|
||||
{ value: "originalColor", label: "Original color" },
|
||||
{ value: "edges", label: "Edge map" },
|
||||
] satisfies readonly BottomControlSelectOption<GenerateSettings["inpaint"]["maskedContent"]>[];
|
||||
|
||||
export type GenerateControlsProps = {
|
||||
settings: GenerateSettings;
|
||||
dispatch: AppStore["dispatch"];
|
||||
@@ -32,6 +44,7 @@ export function GenerateControls({ settings, dispatch }: GenerateControlsProps)
|
||||
const [schedulers, setSchedulers] = useState<readonly BottomControlSelectOption<string>[]>([{ value: settings.scheduler, label: settings.scheduler }]);
|
||||
const [advancedOpen, setAdvancedOpen] = useState(false);
|
||||
const [outpaintOpen, setOutpaintOpen] = useState(false);
|
||||
const [inpaintOpen, setInpaintOpen] = useState(false);
|
||||
const [sizeOpen, setSizeOpen] = useState(false);
|
||||
const sizeRef = useRef<HTMLDivElement>(null);
|
||||
const [error, setError] = useState<string>();
|
||||
@@ -141,6 +154,44 @@ export function GenerateControls({ settings, dispatch }: GenerateControlsProps)
|
||||
<PanelNumber label="Feather" aria-label="Outpaint feathering" value={settings.outpaint.feathering} onValueChange={(feathering) => dispatch(commandIds.toolSetGenerateSettings, { outpaint: { ...settings.outpaint, feathering } })} />
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section className={panelSectionClass()}>
|
||||
<button type="button" className={sectionToggleClass()} aria-expanded={inpaintOpen} aria-controls="generate-inpaint-controls" onClick={() => setInpaintOpen((open) => !open)}>
|
||||
<span>
|
||||
<span className="block text-sm font-semibold text-white/85">Inpaint</span>
|
||||
<span className="block text-xs text-white/40">Mask polarity, crop padding, edge prep, grow</span>
|
||||
</span>
|
||||
{inpaintOpen ? <CaretUp size={18} weight="bold" /> : <CaretDown size={18} weight="bold" />}
|
||||
</button>
|
||||
<div id="generate-inpaint-controls" className={`grid gap-2 overflow-hidden transition-all duration-200 ${inpaintOpen ? "max-h-[38rem] pt-2 opacity-100" : "max-h-0 opacity-0"}`}>
|
||||
<PanelSelect
|
||||
label="Polarity"
|
||||
value={settings.inpaint.maskPolarity}
|
||||
options={inpaintPolarityOptions}
|
||||
ariaLabel="Inpaint mask polarity"
|
||||
onValueChange={(maskPolarity) => dispatch(commandIds.toolSetGenerateSettings, { inpaint: { ...settings.inpaint, maskPolarity } })}
|
||||
/>
|
||||
<PanelSelect
|
||||
label="Content"
|
||||
value={settings.inpaint.maskedContent}
|
||||
options={inpaintMaskedContentOptions}
|
||||
ariaLabel="Inpaint masked content"
|
||||
onValueChange={(maskedContent) => dispatch(commandIds.toolSetGenerateSettings, { inpaint: { ...settings.inpaint, maskedContent } })}
|
||||
/>
|
||||
<button type="button" className={compactRowButtonClass()} aria-pressed={settings.inpaint.maskedAreaOnly} onClick={() => dispatch(commandIds.toolSetGenerateSettings, { inpaint: { ...settings.inpaint, maskedAreaOnly: !settings.inpaint.maskedAreaOnly } })}>
|
||||
<span className={panelLabelClass()}>Frame</span>
|
||||
<span className="min-w-0 flex-1 text-right text-white">{settings.inpaint.maskedAreaOnly ? "Mask crop" : "Full layer"}</span>
|
||||
</button>
|
||||
<div className="grid grid-cols-2 gap-2">
|
||||
<PanelNumber label="Pad" aria-label="Inpaint crop padding" value={settings.inpaint.cropPadding} max={2048} onValueChange={(cropPadding) => dispatch(commandIds.toolSetGenerateSettings, { inpaint: { ...settings.inpaint, cropPadding } })} />
|
||||
<PanelNumber label="Grow" aria-label="Inpaint backend grow mask" value={settings.inpaint.growMaskBy} max={256} onValueChange={(growMaskBy) => dispatch(commandIds.toolSetGenerateSettings, { inpaint: { ...settings.inpaint, growMaskBy } })} />
|
||||
<PanelNumber label="Expand" aria-label="Inpaint mask expand" value={settings.inpaint.maskExpand} min={-256} max={256} onValueChange={(maskExpand) => dispatch(commandIds.toolSetGenerateSettings, { inpaint: { ...settings.inpaint, maskExpand } })} />
|
||||
<PanelNumber label="Feather" aria-label="Inpaint mask feather" value={settings.inpaint.maskFeather} max={256} onValueChange={(maskFeather) => dispatch(commandIds.toolSetGenerateSettings, { inpaint: { ...settings.inpaint, maskFeather } })} />
|
||||
<PanelNumber label="Blur" aria-label="Inpaint mask blur" value={settings.inpaint.maskBlur} max={256} onValueChange={(maskBlur) => dispatch(commandIds.toolSetGenerateSettings, { inpaint: { ...settings.inpaint, maskBlur } })} />
|
||||
<PanelNumber label="Clean" aria-label="Inpaint mask despeckle" value={settings.inpaint.maskDespeckle} max={64} onValueChange={(maskDespeckle) => dispatch(commandIds.toolSetGenerateSettings, { inpaint: { ...settings.inpaint, maskDespeckle } })} />
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -138,7 +138,15 @@ export async function commitBrushSession(options: { store: AppStore; session: Br
|
||||
if (options.session.cancelled) return;
|
||||
|
||||
const source = options.session.changed ? canvasToDataUrl(options.session.canvas) : undefined;
|
||||
if (source) options.store.dispatch(commandIds.documentUpdateAssetSource, { assetId: options.session.assetId, source });
|
||||
if (source) {
|
||||
const state = options.store.getState();
|
||||
const maskEdit = state.editor.maskEdit;
|
||||
if (maskEdit?.maskLayerId === options.session.layerId) {
|
||||
options.store.dispatch(commandIds.documentApplyLayerMaskOperation, { maskLayerId: maskEdit.maskLayerId, source, mimeType: "image/png", operation: { type: "paint" } });
|
||||
} else {
|
||||
options.store.dispatch(commandIds.documentUpdateAssetSource, { assetId: options.session.assetId, source });
|
||||
}
|
||||
}
|
||||
options.store.dispatch(commandIds.toolSetBrushStrokePreview, undefined);
|
||||
closeBrushStrokePreview(options.session);
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import type { Layer } from "@core/layer";
|
||||
import { resolveTransformTargetBounds } from "@editor/transform-targets";
|
||||
import type { AppStore } from "@editor/store";
|
||||
import type { EditorState } from "@editor/state";
|
||||
import { blurMaskValues, despeckleMaskValues, dilateMaskValues, erodeMaskValues, maskValueFromRgba } from "../mask/maskRaster";
|
||||
|
||||
export async function applyMagicWandAt(store: AppStore, point: Vec2D, modeOverride?: EditorState["tools"]["magicWand"]["mode"]) {
|
||||
const state = store.getState();
|
||||
@@ -15,8 +16,8 @@ export async function applyMagicWandAt(store: AppStore, point: Vec2D, modeOverri
|
||||
const y = Math.floor((point.y - target.layer.transform.position.y) / Math.max(0.0001, target.layer.transform.scale.y));
|
||||
if (x < 0 || y < 0 || x >= target.asset.intrinsicSize.w || y >= target.asset.intrinsicSize.h) return true;
|
||||
const source = await createWandMask(target.asset.source, target.maskAsset?.source, Math.round(target.asset.intrinsicSize.w), Math.round(target.asset.intrinsicSize.h), x, y, { ...state.editor.tools.magicWand, mode: modeOverride ?? state.editor.tools.magicWand.mode });
|
||||
if (target.maskAsset) {
|
||||
store.dispatch(commandIds.documentUpdateAssetSource, { assetId: target.maskAsset.id, source });
|
||||
if (target.maskAsset && target.maskLayer && target.maskLayer.type !== "group") {
|
||||
store.dispatch(commandIds.documentApplyLayerMaskOperation, { maskLayerId: target.maskLayer.id, source, mimeType: "image/png", operation: { type: "magicWand" } });
|
||||
return true;
|
||||
}
|
||||
const assetId = crypto.randomUUID();
|
||||
@@ -42,7 +43,7 @@ function resolveTarget(document: ImageDocument, editor: EditorState) {
|
||||
const bounds = resolveTransformTargetBounds(document, { type: "layer", id: layer.id });
|
||||
const maskLayer = layer.clippingMask ? findLayer(document.artboards.flatMap((artboard) => artboard.layers), layer.clippingMask.maskLayerId) : undefined;
|
||||
const maskAsset = maskLayer && maskLayer.type !== "group" ? document.assets.find((candidate) => candidate.id === maskLayer.assetId) : undefined;
|
||||
return asset && bounds ? { layer, asset, bounds, maskAsset } : undefined;
|
||||
return asset && bounds ? { layer, asset, bounds, maskLayer, maskAsset } : undefined;
|
||||
}
|
||||
|
||||
async function createWandMask(source: string, existingMaskSource: string | undefined, width: number, height: number, startX: number, startY: number, settings: EditorState["tools"]["magicWand"]) {
|
||||
@@ -57,10 +58,15 @@ async function createWandMask(source: string, existingMaskSource: string | undef
|
||||
const start = (startY * canvas.width + startX) * 4;
|
||||
const key = [imageData.data[start] ?? 0, imageData.data[start + 1] ?? 0, imageData.data[start + 2] ?? 0];
|
||||
const selected = postProcessSelection(settings.contiguous ? floodSelect(imageData, canvas.width, canvas.height, startX, startY, key, settings.tolerance) : globalSelect(imageData, key, settings.tolerance), canvas.width, canvas.height, settings);
|
||||
const existingAlpha = existingMaskSource ? await loadMaskAlpha(existingMaskSource, canvas.width, canvas.height) : undefined;
|
||||
const existingMask = existingMaskSource ? await loadMaskValues(existingMaskSource, canvas.width, canvas.height) : undefined;
|
||||
for (let pixel = 0; pixel < selected.length; pixel++) {
|
||||
const current = existingAlpha?.[pixel] ?? 255;
|
||||
const value = settings.mode === "add" ? (selected[pixel] ? 0 : current) : settings.mode === "subtract" ? (selected[pixel] ? 255 : current) : selected[pixel] ? 0 : 255;
|
||||
const current = existingMask?.[pixel] ?? 255;
|
||||
const selectionValue = selected[pixel] ?? 0;
|
||||
const value = settings.mode === "add"
|
||||
? Math.min(current, 255 - selectionValue)
|
||||
: settings.mode === "subtract"
|
||||
? Math.max(current, selectionValue)
|
||||
: 255 - selectionValue;
|
||||
const index = pixel * 4;
|
||||
imageData.data[index] = 255;
|
||||
imageData.data[index + 1] = 255;
|
||||
@@ -98,67 +104,24 @@ function matches(data: ImageData, pixel: number, key: number[], tolerance: numbe
|
||||
}
|
||||
|
||||
function postProcessSelection(selected: Uint8Array, width: number, height: number, settings: EditorState["tools"]["magicWand"]) {
|
||||
let next = selected;
|
||||
let next = selectionToMaskValues(selected);
|
||||
const despeckle = Math.round(Math.max(0, Math.min(20, settings.despeckle)));
|
||||
const choke = Math.round(Math.max(-20, Math.min(20, settings.choke)));
|
||||
const feather = Math.round(Math.max(0, Math.min(20, settings.feather)));
|
||||
if (despeckle > 0) next = despeckleSelection(next, width, height, despeckle);
|
||||
if (choke > 0) next = erodeSelection(next, width, height, choke);
|
||||
if (choke < 0) next = dilateSelection(next, width, height, -choke);
|
||||
if (feather > 0) next = featherSelection(next, width, height, feather);
|
||||
if (despeckle > 0) next = despeckleMaskValues(next, width, height, despeckle);
|
||||
if (choke > 0) next = erodeMaskValues(next, width, height, choke);
|
||||
if (choke < 0) next = dilateMaskValues(next, width, height, -choke);
|
||||
if (feather > 0) next = blurMaskValues(next, width, height, feather);
|
||||
return next;
|
||||
}
|
||||
|
||||
function erodeSelection(selected: Uint8Array, width: number, height: number, radius: number) {
|
||||
const next = new Uint8Array(selected.length);
|
||||
for (let y = 0; y < height; y++) for (let x = 0; x < width; x++) {
|
||||
let value = 1;
|
||||
for (let oy = -radius; oy <= radius; oy++) for (let ox = -radius; ox <= radius; ox++) value = Math.min(value, selected[clamp(y + oy, 0, height - 1) * width + clamp(x + ox, 0, width - 1)] ?? 0);
|
||||
next[y * width + x] = value;
|
||||
}
|
||||
return next;
|
||||
function selectionToMaskValues(selected: Uint8Array) {
|
||||
const values = new Uint8ClampedArray(selected.length);
|
||||
for (let index = 0; index < selected.length; index += 1) values[index] = selected[index] ? 255 : 0;
|
||||
return values;
|
||||
}
|
||||
|
||||
function dilateSelection(selected: Uint8Array, width: number, height: number, radius: number) {
|
||||
const next = new Uint8Array(selected.length);
|
||||
for (let y = 0; y < height; y++) for (let x = 0; x < width; x++) {
|
||||
let value = 0;
|
||||
for (let oy = -radius; oy <= radius; oy++) for (let ox = -radius; ox <= radius; ox++) value = Math.max(value, selected[clamp(y + oy, 0, height - 1) * width + clamp(x + ox, 0, width - 1)] ?? 0);
|
||||
next[y * width + x] = value;
|
||||
}
|
||||
return next;
|
||||
}
|
||||
|
||||
function featherSelection(selected: Uint8Array, width: number, height: number, radius: number) {
|
||||
const next = new Uint8Array(selected.length);
|
||||
for (let y = 0; y < height; y++) for (let x = 0; x < width; x++) {
|
||||
let total = 0;
|
||||
let count = 0;
|
||||
for (let oy = -radius; oy <= radius; oy++) for (let ox = -radius; ox <= radius; ox++) {
|
||||
total += selected[clamp(y + oy, 0, height - 1) * width + clamp(x + ox, 0, width - 1)] ?? 0;
|
||||
count += 1;
|
||||
}
|
||||
next[y * width + x] = Math.round(total / count);
|
||||
}
|
||||
return next;
|
||||
}
|
||||
|
||||
function despeckleSelection(selected: Uint8Array, width: number, height: number, strength: number) {
|
||||
const radius = Math.max(1, Math.ceil(strength / 6));
|
||||
const threshold = Math.max(1, Math.round(strength / 2));
|
||||
const next = new Uint8Array(selected);
|
||||
for (let y = 0; y < height; y++) for (let x = 0; x < width; x++) {
|
||||
const index = y * width + x;
|
||||
let same = 0;
|
||||
for (let oy = -radius; oy <= radius; oy++) for (let ox = -radius; ox <= radius; ox++) if (ox !== 0 || oy !== 0) {
|
||||
if ((selected[clamp(y + oy, 0, height - 1) * width + clamp(x + ox, 0, width - 1)] ?? 0) === selected[index]) same += 1;
|
||||
}
|
||||
if (same <= threshold) next[index] = selected[index] ? 0 : 1;
|
||||
}
|
||||
return next;
|
||||
}
|
||||
|
||||
async function loadMaskAlpha(source: string, width: number, height: number) {
|
||||
async function loadMaskValues(source: string, width: number, height: number) {
|
||||
const canvas = document.createElement("canvas");
|
||||
canvas.width = width;
|
||||
canvas.height = height;
|
||||
@@ -167,13 +130,9 @@ async function loadMaskAlpha(source: string, width: number, height: number) {
|
||||
const image = await loadImage(source);
|
||||
context.drawImage(image, 0, 0, width, height);
|
||||
const data = context.getImageData(0, 0, width, height);
|
||||
const alpha = new Uint8ClampedArray(width * height);
|
||||
for (let pixel = 0; pixel < alpha.length; pixel++) alpha[pixel] = data.data[pixel * 4 + 3] ?? 255;
|
||||
return alpha;
|
||||
}
|
||||
|
||||
function clamp(value: number, min: number, max: number) {
|
||||
return Math.max(min, Math.min(max, value));
|
||||
const values = new Uint8ClampedArray(width * height);
|
||||
for (let pixel = 0; pixel < values.length; pixel++) values[pixel] = maskValueFromRgba(data.data, pixel * 4);
|
||||
return values;
|
||||
}
|
||||
|
||||
function findLayer(layers: readonly Layer[], layerId: string): Layer | undefined {
|
||||
|
||||
@@ -151,6 +151,36 @@ const visualEditorChanges: Array<[string, (state: AppState) => AppState]> = [
|
||||
},
|
||||
}),
|
||||
],
|
||||
[
|
||||
"generation candidate preview",
|
||||
(state) => ({
|
||||
...state,
|
||||
editor: {
|
||||
...state.editor,
|
||||
generation: {
|
||||
candidates: [
|
||||
{
|
||||
id: "candidate",
|
||||
source: "data:image/png;base64,candidate",
|
||||
mimeType: "image/png",
|
||||
intrinsicSize: { w: 64, h: 64 },
|
||||
mode: "text-to-image",
|
||||
settings: state.editor.tools.generate,
|
||||
seed: 12,
|
||||
width: 64,
|
||||
height: 64,
|
||||
placement: {
|
||||
artboardId: "artboard",
|
||||
layerName: "Candidate",
|
||||
transform: { position: { x: 1, y: 2 }, scale: { x: 1, y: 1 }, rotation: 0 },
|
||||
},
|
||||
},
|
||||
],
|
||||
selectedCandidateId: "candidate",
|
||||
},
|
||||
},
|
||||
}),
|
||||
],
|
||||
[
|
||||
"active tool",
|
||||
(state) => ({
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import type { Rect, Vec2D } from "@core/geometry";
|
||||
import type { RenderFrame } from "@renderer/index";
|
||||
import type { AppState, BrushPreviewState, BrushStrokePreviewState, EditorState, MaskEditState, SelectionState, ViewportState } from "@editor/state";
|
||||
import type { AppState, BrushPreviewState, BrushStrokePreviewState, EditorState, GenerationState, MaskEditState, SelectionState, ViewportState } from "@editor/state";
|
||||
import type { BrushSettings, InteractionMode } from "@editor/tools";
|
||||
import type { TransformSession, TransformTarget } from "@editor/transform";
|
||||
|
||||
@@ -23,6 +23,7 @@ function visualEditorStatesEqual(a: EditorState, b: EditorState): boolean {
|
||||
maskEditStatesEqual(a.maskEdit, b.maskEdit) &&
|
||||
brushPreviewStatesEqual(a.brushPreview, b.brushPreview) &&
|
||||
brushStrokePreviewStatesEqual(a.brushStrokePreview, b.brushStrokePreview) &&
|
||||
generationStatesEqual(a.generation, b.generation) &&
|
||||
visualToolStatesEqual(a.tools, b.tools)
|
||||
);
|
||||
}
|
||||
@@ -64,6 +65,10 @@ function brushStrokePreviewStatesEqual(a: BrushStrokePreviewState | undefined, b
|
||||
return a.layerId === b.layerId && a.assetId === b.assetId && a.source === b.source;
|
||||
}
|
||||
|
||||
function generationStatesEqual(a: GenerationState, b: GenerationState): boolean {
|
||||
return a === b;
|
||||
}
|
||||
|
||||
function visualToolStatesEqual(a: EditorState["tools"], b: EditorState["tools"]): boolean {
|
||||
return a.activeTool === b.activeTool && interactionModesEqual(a.interactionMode, b.interactionMode) && brushSettingsEqual(a.brush, b.brush);
|
||||
}
|
||||
|
||||
50
view/generate/candidateActions.ts
Normal file
50
view/generate/candidateActions.ts
Normal file
@@ -0,0 +1,50 @@
|
||||
import type { ImageDocument } from "@core/document";
|
||||
import type { GenerationCandidate } from "@editor/state";
|
||||
import { loadImageCanvas, maskValueFromRgba } from "../mask/maskRaster";
|
||||
|
||||
export async function createMaskedPixelReplacementSource(document: ImageDocument, candidate: GenerationCandidate): Promise<string> {
|
||||
if (!candidate.inpaint) throw new Error("Only inpaint candidates can replace masked pixels.");
|
||||
|
||||
const targetAsset = document.assets.find((asset) => asset.id === candidate.inpaint?.sourceAssetId);
|
||||
if (!targetAsset) throw new Error("The source layer for this candidate no longer exists.");
|
||||
|
||||
const targetCanvas = await loadImageCanvas(targetAsset.source, targetAsset.intrinsicSize.w, targetAsset.intrinsicSize.h);
|
||||
const generatedCanvas = await loadImageCanvas(candidate.source, candidate.width, candidate.height);
|
||||
const maskCanvas = await loadImageCanvas(candidate.inpaint.maskImage, candidate.width, candidate.height);
|
||||
|
||||
const targetContext = require2dContext(targetCanvas);
|
||||
const generatedContext = require2dContext(generatedCanvas);
|
||||
const maskContext = require2dContext(maskCanvas);
|
||||
const targetData = targetContext.getImageData(0, 0, targetCanvas.width, targetCanvas.height);
|
||||
const generatedData = generatedContext.getImageData(0, 0, generatedCanvas.width, generatedCanvas.height);
|
||||
const maskData = maskContext.getImageData(0, 0, maskCanvas.width, maskCanvas.height);
|
||||
const crop = candidate.inpaint.crop.assetBounds;
|
||||
|
||||
for (let y = 0; y < candidate.height; y += 1) {
|
||||
for (let x = 0; x < candidate.width; x += 1) {
|
||||
const targetX = Math.round(crop.x) + x;
|
||||
const targetY = Math.round(crop.y) + y;
|
||||
if (targetX < 0 || targetY < 0 || targetX >= targetCanvas.width || targetY >= targetCanvas.height) continue;
|
||||
|
||||
const generatedIndex = (y * generatedCanvas.width + x) * 4;
|
||||
const targetIndex = (targetY * targetCanvas.width + targetX) * 4;
|
||||
const mask = maskValueFromRgba(maskData.data, generatedIndex) / 255;
|
||||
if (mask <= 0) continue;
|
||||
|
||||
for (let channel = 0; channel < 4; channel += 1) {
|
||||
const previous = targetData.data[targetIndex + channel] ?? 0;
|
||||
const next = generatedData.data[generatedIndex + channel] ?? previous;
|
||||
targetData.data[targetIndex + channel] = Math.round(previous * (1 - mask) + next * mask);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
targetContext.putImageData(targetData, 0, 0);
|
||||
return targetCanvas.toDataURL("image/png");
|
||||
}
|
||||
|
||||
function require2dContext(canvas: HTMLCanvasElement): CanvasRenderingContext2D {
|
||||
const context = canvas.getContext("2d");
|
||||
if (!context) throw new Error("Unable to prepare generated candidate");
|
||||
return context;
|
||||
}
|
||||
36
view/generate/inpaintPrep.test.ts
Normal file
36
view/generate/inpaintPrep.test.ts
Normal file
@@ -0,0 +1,36 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import { applyMaskedContentModeToRgba } from "./inpaintPrep";
|
||||
|
||||
describe("inpaint prep", () => {
|
||||
test("converts only masked original pixels to grayscale", () => {
|
||||
const pixels = new Uint8ClampedArray([
|
||||
255, 0, 0, 255,
|
||||
0, 0, 255, 255,
|
||||
]);
|
||||
const mask = new Uint8ClampedArray([255, 0]);
|
||||
|
||||
const next = applyMaskedContentModeToRgba(pixels, 2, 1, mask, "original");
|
||||
|
||||
expect(next[0]).toBe(next[1]);
|
||||
expect(next[1]).toBe(next[2]);
|
||||
expect(Array.from(next.slice(4, 8))).toEqual([0, 0, 255, 255]);
|
||||
});
|
||||
|
||||
test("uses an edge map inside the mask without recoloring unmasked context", () => {
|
||||
const pixels = new Uint8ClampedArray([
|
||||
255, 0, 0, 255,
|
||||
0, 255, 0, 255,
|
||||
0, 0, 255, 255,
|
||||
255, 255, 0, 255,
|
||||
]);
|
||||
const mask = new Uint8ClampedArray([255, 0, 0, 0]);
|
||||
|
||||
const next = applyMaskedContentModeToRgba(pixels, 2, 2, mask, "edges");
|
||||
|
||||
expect(next[0]).toBe(next[1]);
|
||||
expect(next[1]).toBe(next[2]);
|
||||
expect(Array.from(next.slice(4, 8))).toEqual([0, 255, 0, 255]);
|
||||
expect(Array.from(next.slice(8, 12))).toEqual([0, 0, 255, 255]);
|
||||
expect(Array.from(next.slice(12, 16))).toEqual([255, 255, 0, 255]);
|
||||
});
|
||||
});
|
||||
258
view/generate/inpaintPrep.ts
Normal file
258
view/generate/inpaintPrep.ts
Normal file
@@ -0,0 +1,258 @@
|
||||
import type { Asset } from "@core/asset";
|
||||
import type { ImageDocument } from "@core/document";
|
||||
import type { Rect } from "@core/geometry";
|
||||
import type { Layer } from "@core/layer";
|
||||
import type { SelectionState } from "@editor/state";
|
||||
import type { GenerateSettings } from "@editor/tools";
|
||||
import { createDocumentReadIndex, resolveIndexedLayerBounds } from "@editor/document-indexes";
|
||||
import { createNormalizedMaskSource, cropCanvas, cropMaskValuesToDataUrl, expandRectWithinBounds, loadImageCanvas } from "../mask/maskRaster";
|
||||
|
||||
export type InpaintBundle = {
|
||||
inputImage: string;
|
||||
maskImage: string;
|
||||
width: number;
|
||||
height: number;
|
||||
targetLayerId: string;
|
||||
maskLayerId: string;
|
||||
sourceAssetId: string;
|
||||
maskAssetId: string;
|
||||
crop: {
|
||||
assetBounds: Rect;
|
||||
documentBounds: Rect;
|
||||
padding: number;
|
||||
maskedAreaOnly: boolean;
|
||||
};
|
||||
mask: {
|
||||
polarity: GenerateSettings["inpaint"]["maskPolarity"];
|
||||
activeBounds: Rect;
|
||||
};
|
||||
placement: {
|
||||
artboardId: string;
|
||||
layerName: string;
|
||||
transform: Extract<Layer, { type: "image" | "raster" }>["transform"];
|
||||
};
|
||||
backend: {
|
||||
growMaskBy: number;
|
||||
maskedContent: GenerateSettings["inpaint"]["maskedContent"];
|
||||
maskBlur: number;
|
||||
maskFeather: number;
|
||||
maskExpand: number;
|
||||
cropPadding: number;
|
||||
};
|
||||
};
|
||||
|
||||
type InpaintTarget = {
|
||||
artboardId: string;
|
||||
layer: Extract<Layer, { type: "image" | "raster" }>;
|
||||
asset: Asset;
|
||||
bounds: Rect;
|
||||
maskLayer: Extract<Layer, { type: "image" | "raster" }>;
|
||||
maskAsset: Asset;
|
||||
maskBounds: Rect;
|
||||
};
|
||||
|
||||
const modelMultiple = 8;
|
||||
const minModelSize = 64;
|
||||
const maxModelSize = 4096;
|
||||
|
||||
export async function buildInpaintBundle(document: ImageDocument, selection: SelectionState, settings: GenerateSettings): Promise<InpaintBundle> {
|
||||
const target = resolveInpaintTarget(document, selection);
|
||||
validateInpaintTarget(target);
|
||||
|
||||
const width = Math.max(1, Math.round(target.asset.intrinsicSize.w));
|
||||
const height = Math.max(1, Math.round(target.asset.intrinsicSize.h));
|
||||
const normalizedMask = await createNormalizedMaskSource(target.maskAsset.source, width, height, {
|
||||
polarity: settings.inpaint.maskPolarity,
|
||||
expand: settings.inpaint.maskExpand,
|
||||
feather: settings.inpaint.maskFeather,
|
||||
blur: settings.inpaint.maskBlur,
|
||||
despeckle: settings.inpaint.maskDespeckle,
|
||||
});
|
||||
|
||||
if (!normalizedMask.bounds) throw new Error("The selected layer mask has no inpaint pixels.");
|
||||
|
||||
const crop = settings.inpaint.maskedAreaOnly
|
||||
? expandRectWithinBounds(normalizedMask.bounds, settings.inpaint.cropPadding, { w: width, h: height }, modelMultiple, minModelSize)
|
||||
: { x: 0, y: 0, w: width, h: height };
|
||||
const outputWidth = toModelSize(crop.w, "width");
|
||||
const outputHeight = toModelSize(crop.h, "height");
|
||||
|
||||
const inputCanvas = prepareMaskedContentInputCanvas(await loadImageCanvas(target.asset.source, width, height), normalizedMask.values, settings.inpaint.maskedContent);
|
||||
const inputImage = cropCanvas(inputCanvas, crop, outputWidth, outputHeight);
|
||||
const maskImage = cropMaskValuesToDataUrl(normalizedMask.values, width, height, crop, outputWidth, outputHeight);
|
||||
const scaleX = target.bounds.w / width;
|
||||
const scaleY = target.bounds.h / height;
|
||||
const documentBounds = {
|
||||
x: target.bounds.x + crop.x * scaleX,
|
||||
y: target.bounds.y + crop.y * scaleY,
|
||||
w: outputWidth * scaleX,
|
||||
h: outputHeight * scaleY,
|
||||
};
|
||||
|
||||
return {
|
||||
inputImage,
|
||||
maskImage,
|
||||
width: outputWidth,
|
||||
height: outputHeight,
|
||||
targetLayerId: target.layer.id,
|
||||
maskLayerId: target.maskLayer.id,
|
||||
sourceAssetId: target.asset.id,
|
||||
maskAssetId: target.maskAsset.id,
|
||||
crop: {
|
||||
assetBounds: crop,
|
||||
documentBounds,
|
||||
padding: settings.inpaint.cropPadding,
|
||||
maskedAreaOnly: settings.inpaint.maskedAreaOnly,
|
||||
},
|
||||
mask: {
|
||||
polarity: settings.inpaint.maskPolarity,
|
||||
activeBounds: normalizedMask.bounds,
|
||||
},
|
||||
placement: {
|
||||
artboardId: target.artboardId,
|
||||
layerName: `${target.layer.name} inpaint`,
|
||||
transform: {
|
||||
position: { x: documentBounds.x, y: documentBounds.y },
|
||||
scale: { x: documentBounds.w / outputWidth, y: documentBounds.h / outputHeight },
|
||||
rotation: target.layer.transform.rotation,
|
||||
},
|
||||
},
|
||||
backend: {
|
||||
growMaskBy: settings.inpaint.growMaskBy,
|
||||
maskedContent: settings.inpaint.maskedContent,
|
||||
maskBlur: settings.inpaint.maskBlur,
|
||||
maskFeather: settings.inpaint.maskFeather,
|
||||
maskExpand: settings.inpaint.maskExpand,
|
||||
cropPadding: settings.inpaint.cropPadding,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function resolveInpaintTarget(document: ImageDocument, selection: SelectionState): InpaintTarget {
|
||||
if (selection.layerIds.length !== 1 || !selection.layerIds[0]) throw new Error("Select one image or raster layer to inpaint.");
|
||||
|
||||
const documentIndex = createDocumentReadIndex(document);
|
||||
const layerInfo = documentIndex.layerInfoById.get(selection.layerIds[0]);
|
||||
if (!layerInfo || layerInfo.layer.type === "group") throw new Error("Select one image or raster layer to inpaint.");
|
||||
|
||||
const asset = documentIndex.assetById.get(layerInfo.layer.assetId);
|
||||
if (!asset) throw new Error("The selected layer is missing its source image.");
|
||||
if (!layerInfo.layer.clippingMask) throw new Error("Add a layer mask before running inpaint.");
|
||||
|
||||
const maskLayer = documentIndex.layerById.get(layerInfo.layer.clippingMask.maskLayerId);
|
||||
if (!maskLayer || maskLayer.type === "group") throw new Error("The selected layer mask is missing.");
|
||||
|
||||
const maskAsset = documentIndex.assetById.get(maskLayer.assetId);
|
||||
if (!maskAsset) throw new Error("The selected layer mask is missing its image data.");
|
||||
|
||||
const bounds = resolveIndexedLayerBounds(documentIndex, layerInfo.layer);
|
||||
const maskBounds = resolveIndexedLayerBounds(documentIndex, maskLayer);
|
||||
if (!bounds || !maskBounds) throw new Error("Unable to resolve the selected layer and mask bounds.");
|
||||
|
||||
return { artboardId: layerInfo.artboardId, layer: layerInfo.layer, asset, bounds, maskLayer, maskAsset, maskBounds };
|
||||
}
|
||||
|
||||
function validateInpaintTarget(target: InpaintTarget) {
|
||||
if (target.asset.intrinsicSize.w <= 0 || target.asset.intrinsicSize.h <= 0) throw new Error("The selected image has an invalid size.");
|
||||
if (target.maskAsset.intrinsicSize.w <= 0 || target.maskAsset.intrinsicSize.h <= 0) throw new Error("The selected mask has an invalid size.");
|
||||
if (Math.round(target.asset.intrinsicSize.w) !== Math.round(target.maskAsset.intrinsicSize.w) || Math.round(target.asset.intrinsicSize.h) !== Math.round(target.maskAsset.intrinsicSize.h)) {
|
||||
throw new Error("The selected layer and mask image sizes do not match.");
|
||||
}
|
||||
if (!rectsAligned(target.bounds, target.maskBounds) || Math.abs(target.layer.transform.rotation - target.maskLayer.transform.rotation) > 0.001) {
|
||||
throw new Error("The selected layer and mask are not aligned.");
|
||||
}
|
||||
}
|
||||
|
||||
function rectsAligned(a: Rect, b: Rect): boolean {
|
||||
return Math.abs(a.x - b.x) <= 0.5 && Math.abs(a.y - b.y) <= 0.5 && Math.abs(a.w - b.w) <= 0.5 && Math.abs(a.h - b.h) <= 0.5;
|
||||
}
|
||||
|
||||
function toModelSize(value: number, axis: "width" | "height"): number {
|
||||
const rounded = Math.max(minModelSize, Math.ceil(value / modelMultiple) * modelMultiple);
|
||||
if (rounded > maxModelSize) throw new Error(`The inpaint ${axis} is too large for the model. Use masked-area inpaint or a smaller source.`);
|
||||
return rounded;
|
||||
}
|
||||
|
||||
function prepareMaskedContentInputCanvas(sourceCanvas: HTMLCanvasElement, maskValues: Uint8ClampedArray, mode: GenerateSettings["inpaint"]["maskedContent"]): HTMLCanvasElement {
|
||||
if (mode === "neutral" || mode === "originalColor") return sourceCanvas;
|
||||
const context = sourceCanvas.getContext("2d");
|
||||
if (!context) return sourceCanvas;
|
||||
const imageData = context.getImageData(0, 0, sourceCanvas.width, sourceCanvas.height);
|
||||
imageData.data.set(applyMaskedContentModeToRgba(imageData.data, sourceCanvas.width, sourceCanvas.height, maskValues, mode));
|
||||
context.putImageData(imageData, 0, 0);
|
||||
return sourceCanvas;
|
||||
}
|
||||
|
||||
export function applyMaskedContentModeToRgba(data: Uint8ClampedArray, width: number, height: number, maskValues: Uint8ClampedArray, mode: GenerateSettings["inpaint"]["maskedContent"]): Uint8ClampedArray {
|
||||
const next = new Uint8ClampedArray(data);
|
||||
if (mode === "neutral" || mode === "originalColor") return next;
|
||||
|
||||
const edgeValues = mode === "edges" ? createEdgeMapValues(data, width, height) : undefined;
|
||||
|
||||
for (let pixel = 0; pixel < width * height; pixel += 1) {
|
||||
const amount = (maskValues[pixel] ?? 0) / 255;
|
||||
if (amount <= 0) continue;
|
||||
|
||||
const index = pixel * 4;
|
||||
const red = data[index] ?? 0;
|
||||
const green = data[index + 1] ?? 0;
|
||||
const blue = data[index + 2] ?? 0;
|
||||
const target = edgeValues ? edgeValues[pixel] ?? 128 : luminance(red, green, blue);
|
||||
next[index] = blendChannel(red, target, amount);
|
||||
next[index + 1] = blendChannel(green, target, amount);
|
||||
next[index + 2] = blendChannel(blue, target, amount);
|
||||
}
|
||||
|
||||
return next;
|
||||
}
|
||||
|
||||
function createEdgeMapValues(data: Uint8ClampedArray, width: number, height: number): Uint8ClampedArray {
|
||||
const gray = new Float32Array(width * height);
|
||||
const edges = new Uint8ClampedArray(width * height);
|
||||
|
||||
for (let pixel = 0; pixel < width * height; pixel += 1) {
|
||||
const index = pixel * 4;
|
||||
gray[pixel] = luminance(data[index] ?? 0, data[index + 1] ?? 0, data[index + 2] ?? 0);
|
||||
}
|
||||
|
||||
for (let y = 0; y < height; y += 1) {
|
||||
for (let x = 0; x < width; x += 1) {
|
||||
const gx =
|
||||
-sampleGray(gray, width, height, x - 1, y - 1) +
|
||||
sampleGray(gray, width, height, x + 1, y - 1) -
|
||||
2 * sampleGray(gray, width, height, x - 1, y) +
|
||||
2 * sampleGray(gray, width, height, x + 1, y) -
|
||||
sampleGray(gray, width, height, x - 1, y + 1) +
|
||||
sampleGray(gray, width, height, x + 1, y + 1);
|
||||
const gy =
|
||||
-sampleGray(gray, width, height, x - 1, y - 1) -
|
||||
2 * sampleGray(gray, width, height, x, y - 1) -
|
||||
sampleGray(gray, width, height, x + 1, y - 1) +
|
||||
sampleGray(gray, width, height, x - 1, y + 1) +
|
||||
2 * sampleGray(gray, width, height, x, y + 1) +
|
||||
sampleGray(gray, width, height, x + 1, y + 1);
|
||||
const magnitude = Math.hypot(gx, gy);
|
||||
edges[y * width + x] = Math.round(clampNumber(128 + Math.max(0, magnitude - 24) * 0.75, 128, 255));
|
||||
}
|
||||
}
|
||||
|
||||
return edges;
|
||||
}
|
||||
|
||||
function sampleGray(values: Float32Array, width: number, height: number, x: number, y: number): number {
|
||||
const clampedX = Math.max(0, Math.min(width - 1, x));
|
||||
const clampedY = Math.max(0, Math.min(height - 1, y));
|
||||
return values[clampedY * width + clampedX] ?? 0;
|
||||
}
|
||||
|
||||
function luminance(red: number, green: number, blue: number): number {
|
||||
return Math.round(0.2126 * red + 0.7152 * green + 0.0722 * blue);
|
||||
}
|
||||
|
||||
function blendChannel(previous: number, next: number, amount: number): number {
|
||||
return Math.round(previous * (1 - amount) + next * amount);
|
||||
}
|
||||
|
||||
function clampNumber(value: number, min: number, max: number): number {
|
||||
return Math.max(min, Math.min(max, value));
|
||||
}
|
||||
@@ -1,9 +1,11 @@
|
||||
import { commandIds } from "@commands/ids";
|
||||
import type { ImageDocument } from "@core/document";
|
||||
import type { Transform } from "@core/geometry";
|
||||
import type { Layer } from "@core/layer";
|
||||
import type { AppStore } from "@editor/store";
|
||||
import type { SelectionState, ViewportState } from "@editor/state";
|
||||
import type { GenerationCandidate, SelectionState, ViewportState } from "@editor/state";
|
||||
import type { GenerateSettings } from "@editor/tools";
|
||||
import { buildInpaintBundle, type InpaintBundle } from "./inpaintPrep";
|
||||
|
||||
export async function runGenerate(options: {
|
||||
document: ImageDocument;
|
||||
@@ -17,50 +19,200 @@ export async function runGenerate(options: {
|
||||
if (!artboard) return;
|
||||
|
||||
const target = resolveSelectedImage(document, selection);
|
||||
const inputImage = target && settings.mode !== "text-to-image" ? await imageSourceToDataUrl(target.asset.source) : undefined;
|
||||
const maskImage = target?.maskAsset && settings.mode === "inpaint" ? await imageSourceToDataUrl(target.maskAsset.source) : undefined;
|
||||
const inpaintBundle = settings.mode === "inpaint" ? await buildInpaintBundle(document, selection, settings) : undefined;
|
||||
const inputImage = inpaintBundle?.inputImage ?? (target && settings.mode !== "text-to-image" ? await imageSourceToDataUrl(target.asset.source) : undefined);
|
||||
const maskImage = inpaintBundle?.maskImage;
|
||||
const seed = resolveSeed(settings.seed);
|
||||
const requestSettings = { ...settings, seed };
|
||||
const width = inpaintBundle?.width ?? settings.width;
|
||||
const height = inpaintBundle?.height ?? settings.height;
|
||||
const generated = await requestGenerate({
|
||||
settings: requestSettings,
|
||||
width,
|
||||
height,
|
||||
inputImage,
|
||||
maskImage,
|
||||
inpaintBundle,
|
||||
});
|
||||
const intrinsicSize = await loadImageSize(generated.source);
|
||||
const targetArtboardId = inpaintBundle?.placement.artboardId ?? artboard.id;
|
||||
const placement = {
|
||||
artboardId: targetArtboardId,
|
||||
layerName: inpaintBundle?.placement.layerName ?? "Generated image",
|
||||
transform: generatedLayerTransform(inpaintBundle, intrinsicSize),
|
||||
};
|
||||
|
||||
dispatch(commandIds.generationAddCandidate, {
|
||||
candidate: createGenerationCandidate({
|
||||
source: generated.source,
|
||||
mimeType: generated.mimeType,
|
||||
intrinsicSize,
|
||||
settings: requestSettings,
|
||||
seed,
|
||||
width,
|
||||
height,
|
||||
inputImage,
|
||||
maskImage,
|
||||
placement,
|
||||
inpaintBundle,
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
export async function runGenerateFromCandidate(options: {
|
||||
candidate: GenerationCandidate;
|
||||
settings?: GenerateSettings;
|
||||
dispatch: AppStore["dispatch"];
|
||||
}) {
|
||||
const settings = options.settings ?? options.candidate.settings;
|
||||
const seed = resolveSeed(settings.seed);
|
||||
const requestSettings = { ...settings, seed };
|
||||
const generated = await requestGenerate({
|
||||
settings: requestSettings,
|
||||
width: options.candidate.width,
|
||||
height: options.candidate.height,
|
||||
inputImage: options.candidate.inputImage,
|
||||
maskImage: options.candidate.maskImage,
|
||||
inpaintCandidate: options.candidate,
|
||||
});
|
||||
const intrinsicSize = await loadImageSize(generated.source);
|
||||
|
||||
options.dispatch(commandIds.generationAddCandidate, {
|
||||
candidate: {
|
||||
...options.candidate,
|
||||
id: crypto.randomUUID(),
|
||||
source: generated.source,
|
||||
mimeType: generated.mimeType,
|
||||
intrinsicSize,
|
||||
settings: requestSettings,
|
||||
seed,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function createGenerationCandidate(options: {
|
||||
source: string;
|
||||
mimeType: string;
|
||||
intrinsicSize: { w: number; h: number };
|
||||
settings: GenerateSettings;
|
||||
seed: number;
|
||||
width: number;
|
||||
height: number;
|
||||
inputImage?: string;
|
||||
maskImage?: string;
|
||||
placement: GenerationCandidate["placement"];
|
||||
inpaintBundle?: InpaintBundle;
|
||||
}): GenerationCandidate {
|
||||
return {
|
||||
id: crypto.randomUUID(),
|
||||
source: options.source,
|
||||
mimeType: options.mimeType,
|
||||
intrinsicSize: options.intrinsicSize,
|
||||
mode: options.settings.mode,
|
||||
settings: options.settings,
|
||||
seed: options.seed,
|
||||
width: options.width,
|
||||
height: options.height,
|
||||
inputImage: options.inputImage,
|
||||
maskImage: options.maskImage,
|
||||
placement: options.placement,
|
||||
inpaint: options.inpaintBundle
|
||||
? {
|
||||
targetLayerId: options.inpaintBundle.targetLayerId,
|
||||
maskLayerId: options.inpaintBundle.maskLayerId,
|
||||
sourceAssetId: options.inpaintBundle.sourceAssetId,
|
||||
maskAssetId: options.inpaintBundle.maskAssetId,
|
||||
inputImage: options.inpaintBundle.inputImage,
|
||||
maskImage: options.inpaintBundle.maskImage,
|
||||
crop: options.inpaintBundle.crop,
|
||||
mask: options.inpaintBundle.mask,
|
||||
backend: options.inpaintBundle.backend,
|
||||
}
|
||||
: undefined,
|
||||
};
|
||||
}
|
||||
|
||||
async function requestGenerate(options: {
|
||||
settings: GenerateSettings;
|
||||
width: number;
|
||||
height: number;
|
||||
inputImage?: string;
|
||||
maskImage?: string;
|
||||
inpaintBundle?: InpaintBundle;
|
||||
inpaintCandidate?: GenerationCandidate;
|
||||
}) {
|
||||
const response = await fetch("/api/comfy/generate", {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
mode: settings.mode,
|
||||
model: settings.model,
|
||||
prompt: settings.prompt,
|
||||
negativePrompt: settings.negativePrompt,
|
||||
strength: settings.strength,
|
||||
steps: settings.steps,
|
||||
cfg: settings.cfg,
|
||||
seed: settings.seed,
|
||||
sampler: settings.sampler,
|
||||
scheduler: settings.scheduler,
|
||||
width: settings.width,
|
||||
height: settings.height,
|
||||
outpaint: settings.outpaint,
|
||||
inputImage,
|
||||
maskImage,
|
||||
mode: options.settings.mode,
|
||||
model: options.settings.model,
|
||||
prompt: options.settings.prompt,
|
||||
negativePrompt: options.settings.negativePrompt,
|
||||
strength: options.settings.strength,
|
||||
steps: options.settings.steps,
|
||||
cfg: options.settings.cfg,
|
||||
seed: options.settings.seed,
|
||||
sampler: options.settings.sampler,
|
||||
scheduler: options.settings.scheduler,
|
||||
width: options.width,
|
||||
height: options.height,
|
||||
outpaint: options.settings.outpaint,
|
||||
inpaint: resolveInpaintRequest(options.inpaintBundle, options.inpaintCandidate, options.settings),
|
||||
inputImage: options.inputImage,
|
||||
maskImage: options.maskImage,
|
||||
}),
|
||||
});
|
||||
if (!response.ok) throw new Error(await response.text());
|
||||
const generated = await response.json() as { source: string; mimeType: string };
|
||||
const intrinsicSize = await loadImageSize(generated.source);
|
||||
const assetId = crypto.randomUUID();
|
||||
const layerId = crypto.randomUUID();
|
||||
dispatch(commandIds.documentAddAsset, { asset: { id: assetId, name: "Generated image", mimeType: generated.mimeType, source: generated.source, intrinsicSize } });
|
||||
dispatch(commandIds.documentAddImageLayer, {
|
||||
artboardId: artboard.id,
|
||||
layer: {
|
||||
id: layerId,
|
||||
type: "image",
|
||||
name: "Generated image",
|
||||
visible: true,
|
||||
locked: false,
|
||||
opacity: 1,
|
||||
assetId,
|
||||
transform: { position: { x: 0, y: 0 }, scale: { x: 1, y: 1 }, rotation: 0 },
|
||||
return await response.json() as { source: string; mimeType: string };
|
||||
}
|
||||
|
||||
function resolveInpaintRequest(inpaintBundle: InpaintBundle | undefined, inpaintCandidate: GenerationCandidate | undefined, settings: GenerateSettings) {
|
||||
if (inpaintBundle) {
|
||||
return {
|
||||
growMaskBy: inpaintBundle.backend.growMaskBy,
|
||||
maskedContent: inpaintBundle.backend.maskedContent,
|
||||
maskBlur: inpaintBundle.backend.maskBlur,
|
||||
maskFeather: inpaintBundle.backend.maskFeather,
|
||||
maskExpand: inpaintBundle.backend.maskExpand,
|
||||
cropPadding: inpaintBundle.backend.cropPadding,
|
||||
maskPolarity: inpaintBundle.mask.polarity,
|
||||
crop: inpaintBundle.crop,
|
||||
placement: inpaintBundle.placement,
|
||||
};
|
||||
}
|
||||
|
||||
if (inpaintCandidate?.inpaint) {
|
||||
return {
|
||||
growMaskBy: inpaintCandidate.inpaint.backend.growMaskBy,
|
||||
maskedContent: inpaintCandidate.inpaint.backend.maskedContent,
|
||||
maskBlur: inpaintCandidate.inpaint.backend.maskBlur,
|
||||
maskFeather: inpaintCandidate.inpaint.backend.maskFeather,
|
||||
maskExpand: inpaintCandidate.inpaint.backend.maskExpand,
|
||||
cropPadding: inpaintCandidate.inpaint.backend.cropPadding,
|
||||
maskPolarity: inpaintCandidate.inpaint.mask.polarity,
|
||||
crop: inpaintCandidate.inpaint.crop,
|
||||
placement: inpaintCandidate.placement,
|
||||
};
|
||||
}
|
||||
|
||||
return settings.inpaint;
|
||||
}
|
||||
|
||||
function generatedLayerTransform(inpaintBundle: InpaintBundle | undefined, intrinsicSize: { w: number; h: number }): Transform {
|
||||
if (!inpaintBundle) return { position: { x: 0, y: 0 }, scale: { x: 1, y: 1 }, rotation: 0 };
|
||||
return {
|
||||
position: { ...inpaintBundle.placement.transform.position },
|
||||
scale: {
|
||||
x: (inpaintBundle.placement.transform.scale.x * inpaintBundle.width) / Math.max(1, intrinsicSize.w),
|
||||
y: (inpaintBundle.placement.transform.scale.y * inpaintBundle.height) / Math.max(1, intrinsicSize.h),
|
||||
},
|
||||
});
|
||||
dispatch(commandIds.documentMoveLayer, { layerId, toArtboardId: artboard.id, toIndex: 0 });
|
||||
dispatch(commandIds.selectionSet, { artboardId: artboard.id, layerIds: [layerId] });
|
||||
rotation: inpaintBundle.placement.transform.rotation,
|
||||
};
|
||||
}
|
||||
|
||||
function resolveSeed(seed: number): number {
|
||||
return seed < 0 ? Math.floor(Math.random() * 2 ** 32) : Math.round(seed);
|
||||
}
|
||||
|
||||
function resolveSelectedImage(document: ImageDocument, selection: SelectionState) {
|
||||
|
||||
40
view/mask/maskRaster.test.ts
Normal file
40
view/mask/maskRaster.test.ts
Normal file
@@ -0,0 +1,40 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import { cropMaskValuesToRgba, expandRectWithinBounds, invertMaskValues } from "./maskRaster";
|
||||
|
||||
describe("mask raster utilities", () => {
|
||||
test("exports the normalized drawn mask without filling the whole crop", () => {
|
||||
const revealedMask = new Uint8ClampedArray(4 * 3).fill(255);
|
||||
revealedMask[1 * 4 + 2] = 0;
|
||||
|
||||
const inpaintMask = invertMaskValues(revealedMask);
|
||||
const rgba = cropMaskValuesToRgba(inpaintMask, 4, 3, { x: 1, y: 0, w: 3, h: 3 }, 4, 4);
|
||||
const activePixels = activeRedPixels(rgba);
|
||||
|
||||
expect(activePixels).toEqual([{ x: 1, y: 1 }]);
|
||||
for (let pixel = 0; pixel < rgba.length / 4; pixel += 1) expect(rgba[pixel * 4 + 3]).toBe(255);
|
||||
});
|
||||
|
||||
test("expands masked-area crops to the model minimum when possible", () => {
|
||||
expect(expandRectWithinBounds({ x: 20, y: 20, w: 4, h: 4 }, 4, { w: 100, h: 100 }, 8, 64)).toEqual({
|
||||
x: 0,
|
||||
y: 0,
|
||||
w: 64,
|
||||
h: 64,
|
||||
});
|
||||
expect(expandRectWithinBounds({ x: 80, y: 80, w: 4, h: 4 }, 4, { w: 100, h: 100 }, 8, 64)).toEqual({
|
||||
x: 36,
|
||||
y: 36,
|
||||
w: 64,
|
||||
h: 64,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
function activeRedPixels(rgba: Uint8ClampedArray) {
|
||||
const width = 4;
|
||||
const pixels: Array<{ x: number; y: number }> = [];
|
||||
for (let pixel = 0; pixel < rgba.length / 4; pixel += 1) {
|
||||
if ((rgba[pixel * 4] ?? 0) > 127) pixels.push({ x: pixel % width, y: Math.floor(pixel / width) });
|
||||
}
|
||||
return pixels;
|
||||
}
|
||||
374
view/mask/maskRaster.ts
Normal file
374
view/mask/maskRaster.ts
Normal file
@@ -0,0 +1,374 @@
|
||||
import type { Rect } from "@core/geometry";
|
||||
|
||||
export type MaskFill = "white" | "black" | "clear";
|
||||
|
||||
export type MaskRasterOperation =
|
||||
| { type: "invert" }
|
||||
| { type: "fill"; fill: MaskFill }
|
||||
| { type: "feather"; radius: number }
|
||||
| { type: "expand"; radius: number }
|
||||
| { type: "contract"; radius: number }
|
||||
| { type: "blur"; radius: number }
|
||||
| { type: "despeckle"; strength: number };
|
||||
|
||||
export type MaskAnalysis = {
|
||||
width: number;
|
||||
height: number;
|
||||
revealedPixels: number;
|
||||
hiddenPixels: number;
|
||||
coverage: number;
|
||||
hiddenCoverage: number;
|
||||
bounds?: Rect;
|
||||
hiddenBounds?: Rect;
|
||||
thumbnail: string;
|
||||
};
|
||||
|
||||
export type NormalizedMaskOptions = {
|
||||
polarity: "hidden" | "revealed";
|
||||
expand?: number;
|
||||
feather?: number;
|
||||
blur?: number;
|
||||
despeckle?: number;
|
||||
};
|
||||
|
||||
export async function createSolidMaskSource(width: number, height: number, fill: MaskFill): Promise<string> {
|
||||
const canvas = createCanvas(width, height);
|
||||
const context = require2dContext(canvas);
|
||||
context.clearRect(0, 0, canvas.width, canvas.height);
|
||||
|
||||
if (fill === "white") {
|
||||
context.fillStyle = "#ffffff";
|
||||
context.fillRect(0, 0, canvas.width, canvas.height);
|
||||
} else if (fill === "black") {
|
||||
context.fillStyle = "#000000";
|
||||
context.fillRect(0, 0, canvas.width, canvas.height);
|
||||
}
|
||||
|
||||
return canvas.toDataURL("image/png");
|
||||
}
|
||||
|
||||
export async function applyMaskRasterOperation(source: string, width: number, height: number, operation: MaskRasterOperation): Promise<string> {
|
||||
if (operation.type === "fill") return createSolidMaskSource(width, height, operation.fill);
|
||||
|
||||
const mask = await loadMaskValues(source, width, height);
|
||||
const next = applyMaskValueOperation(mask.values, mask.width, mask.height, operation);
|
||||
return maskValuesToDataUrl(next, mask.width, mask.height);
|
||||
}
|
||||
|
||||
export async function analyzeMaskSource(source: string, width: number, height: number): Promise<MaskAnalysis> {
|
||||
const mask = await loadMaskValues(source, width, height);
|
||||
const revealed = analyzeValues(mask.values, mask.width, mask.height, false);
|
||||
const hiddenValues = invertMaskValues(mask.values);
|
||||
const hidden = analyzeValues(hiddenValues, mask.width, mask.height, false);
|
||||
return {
|
||||
width: mask.width,
|
||||
height: mask.height,
|
||||
revealedPixels: revealed.pixels,
|
||||
hiddenPixels: hidden.pixels,
|
||||
coverage: revealed.coverage,
|
||||
hiddenCoverage: hidden.coverage,
|
||||
bounds: revealed.bounds,
|
||||
hiddenBounds: hidden.bounds,
|
||||
thumbnail: maskValuesToDataUrl(mask.values, mask.width, mask.height, 72, 48),
|
||||
};
|
||||
}
|
||||
|
||||
export async function createNormalizedMaskSource(source: string, width: number, height: number, options: NormalizedMaskOptions): Promise<{ source: string; values: Uint8ClampedArray; bounds?: Rect }> {
|
||||
const mask = await loadMaskValues(source, width, height);
|
||||
let values = options.polarity === "hidden" ? invertMaskValues(mask.values) : new Uint8ClampedArray(mask.values);
|
||||
|
||||
const despeckle = Math.round(clampNumber(options.despeckle ?? 0, 0, 64));
|
||||
const expand = Math.round(clampNumber(options.expand ?? 0, -256, 256));
|
||||
const feather = Math.round(clampNumber(options.feather ?? 0, 0, 256));
|
||||
const blur = Math.round(clampNumber(options.blur ?? 0, 0, 256));
|
||||
|
||||
if (despeckle > 0) values = despeckleMaskValues(values, mask.width, mask.height, despeckle);
|
||||
if (expand > 0) values = dilateMaskValues(values, mask.width, mask.height, expand);
|
||||
if (expand < 0) values = erodeMaskValues(values, mask.width, mask.height, -expand);
|
||||
if (feather > 0) values = blurMaskValues(values, mask.width, mask.height, feather);
|
||||
if (blur > 0) values = blurMaskValues(values, mask.width, mask.height, blur);
|
||||
|
||||
const analysis = analyzeValues(values, mask.width, mask.height, false);
|
||||
return { source: maskValuesToDataUrl(values, mask.width, mask.height), values, bounds: analysis.bounds };
|
||||
}
|
||||
|
||||
export async function loadImageCanvas(source: string, width?: number, height?: number): Promise<HTMLCanvasElement> {
|
||||
const image = await loadImage(source);
|
||||
const canvas = createCanvas(width ?? image.naturalWidth, height ?? image.naturalHeight);
|
||||
const context = require2dContext(canvas);
|
||||
context.clearRect(0, 0, canvas.width, canvas.height);
|
||||
context.drawImage(image, 0, 0, canvas.width, canvas.height);
|
||||
return canvas;
|
||||
}
|
||||
|
||||
export async function imageSourceToPngDataUrl(source: string): Promise<string> {
|
||||
if (source.startsWith("data:image/png;base64,")) return source;
|
||||
const canvas = await loadImageCanvas(source);
|
||||
return canvas.toDataURL("image/png");
|
||||
}
|
||||
|
||||
export function cropCanvas(sourceCanvas: HTMLCanvasElement, crop: Rect, outputWidth = crop.w, outputHeight = crop.h): string {
|
||||
const canvas = createCanvas(outputWidth, outputHeight);
|
||||
const context = require2dContext(canvas);
|
||||
context.clearRect(0, 0, outputWidth, outputHeight);
|
||||
context.drawImage(sourceCanvas, crop.x, crop.y, crop.w, crop.h, 0, 0, crop.w, crop.h);
|
||||
return canvas.toDataURL("image/png");
|
||||
}
|
||||
|
||||
export function cropMaskValuesToDataUrl(values: Uint8ClampedArray, width: number, height: number, crop: Rect, outputWidth = crop.w, outputHeight = crop.h): string {
|
||||
const canvas = createCanvas(outputWidth, outputHeight);
|
||||
const context = require2dContext(canvas);
|
||||
const imageData = context.createImageData(outputWidth, outputHeight);
|
||||
imageData.data.set(cropMaskValuesToRgba(values, width, height, crop, outputWidth, outputHeight));
|
||||
context.putImageData(imageData, 0, 0);
|
||||
return canvas.toDataURL("image/png");
|
||||
}
|
||||
|
||||
export function cropMaskValuesToRgba(values: Uint8ClampedArray, width: number, height: number, crop: Rect, outputWidth = crop.w, outputHeight = crop.h): Uint8ClampedArray {
|
||||
const safeOutputWidth = Math.max(1, Math.round(outputWidth));
|
||||
const safeOutputHeight = Math.max(1, Math.round(outputHeight));
|
||||
const data = new Uint8ClampedArray(safeOutputWidth * safeOutputHeight * 4);
|
||||
for (let pixel = 0; pixel < safeOutputWidth * safeOutputHeight; pixel += 1) data[pixel * 4 + 3] = 255;
|
||||
|
||||
for (let y = 0; y < Math.min(crop.h, safeOutputHeight); y += 1) {
|
||||
for (let x = 0; x < Math.min(crop.w, safeOutputWidth); x += 1) {
|
||||
const sourceX = crop.x + x;
|
||||
const sourceY = crop.y + y;
|
||||
if (sourceX < 0 || sourceY < 0 || sourceX >= width || sourceY >= height) continue;
|
||||
const value = values[sourceY * width + sourceX] ?? 0;
|
||||
const index = (y * safeOutputWidth + x) * 4;
|
||||
data[index] = value;
|
||||
data[index + 1] = value;
|
||||
data[index + 2] = value;
|
||||
}
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
export function expandRectWithinBounds(rect: Rect, padding: number, bounds: { w: number; h: number }, multiple = 1, minSize = 1): Rect {
|
||||
const padded = Math.max(0, Math.round(padding));
|
||||
let x1 = Math.max(0, Math.floor(rect.x) - padded);
|
||||
let y1 = Math.max(0, Math.floor(rect.y) - padded);
|
||||
let x2 = Math.min(bounds.w, Math.ceil(rect.x + rect.w) + padded);
|
||||
let y2 = Math.min(bounds.h, Math.ceil(rect.y + rect.h) + padded);
|
||||
|
||||
const safeMinSize = Math.max(1, Math.round(minSize));
|
||||
const targetWidth = Math.min(bounds.w, roundUp(Math.max(safeMinSize, x2 - x1), multiple));
|
||||
const targetHeight = Math.min(bounds.h, roundUp(Math.max(safeMinSize, y2 - y1), multiple));
|
||||
|
||||
const extraWidth = targetWidth - (x2 - x1);
|
||||
const extraHeight = targetHeight - (y2 - y1);
|
||||
x1 = Math.max(0, x1 - Math.floor(extraWidth / 2));
|
||||
y1 = Math.max(0, y1 - Math.floor(extraHeight / 2));
|
||||
x2 = Math.min(bounds.w, x1 + targetWidth);
|
||||
y2 = Math.min(bounds.h, y1 + targetHeight);
|
||||
x1 = Math.max(0, x2 - targetWidth);
|
||||
y1 = Math.max(0, y2 - targetHeight);
|
||||
|
||||
return { x: x1, y: y1, w: Math.max(1, x2 - x1), h: Math.max(1, y2 - y1) };
|
||||
}
|
||||
|
||||
export function maskValueFromRgba(data: Uint8ClampedArray, index: number): number {
|
||||
const red = data[index] ?? 0;
|
||||
const green = data[index + 1] ?? 0;
|
||||
const blue = data[index + 2] ?? 0;
|
||||
const alpha = data[index + 3] ?? 0;
|
||||
const luminance = 0.2126 * red + 0.7152 * green + 0.0722 * blue;
|
||||
return Math.round((alpha * luminance) / 255);
|
||||
}
|
||||
|
||||
export function applyMaskValueOperation(values: Uint8ClampedArray, width: number, height: number, operation: Exclude<MaskRasterOperation, { type: "fill" }>): Uint8ClampedArray {
|
||||
switch (operation.type) {
|
||||
case "invert":
|
||||
return invertMaskValues(values);
|
||||
case "feather":
|
||||
case "blur":
|
||||
return blurMaskValues(values, width, height, Math.round(clampNumber(operation.radius, 0, 256)));
|
||||
case "expand":
|
||||
return dilateMaskValues(values, width, height, Math.round(clampNumber(operation.radius, 0, 256)));
|
||||
case "contract":
|
||||
return erodeMaskValues(values, width, height, Math.round(clampNumber(operation.radius, 0, 256)));
|
||||
case "despeckle":
|
||||
return despeckleMaskValues(values, width, height, Math.round(clampNumber(operation.strength, 0, 64)));
|
||||
}
|
||||
}
|
||||
|
||||
export function invertMaskValues(values: Uint8ClampedArray): Uint8ClampedArray {
|
||||
const next = new Uint8ClampedArray(values.length);
|
||||
for (let index = 0; index < values.length; index += 1) next[index] = 255 - (values[index] ?? 0);
|
||||
return next;
|
||||
}
|
||||
|
||||
export function erodeMaskValues(values: Uint8ClampedArray, width: number, height: number, radius: number): Uint8ClampedArray {
|
||||
const safeRadius = Math.round(clampNumber(radius, 0, 256));
|
||||
if (safeRadius <= 0) return new Uint8ClampedArray(values);
|
||||
const next = new Uint8ClampedArray(values.length);
|
||||
for (let y = 0; y < height; y += 1) {
|
||||
for (let x = 0; x < width; x += 1) {
|
||||
let value = 255;
|
||||
for (let oy = -safeRadius; oy <= safeRadius; oy += 1) {
|
||||
for (let ox = -safeRadius; ox <= safeRadius; ox += 1) value = Math.min(value, values[clampInt(y + oy, 0, height - 1) * width + clampInt(x + ox, 0, width - 1)] ?? 0);
|
||||
}
|
||||
next[y * width + x] = value;
|
||||
}
|
||||
}
|
||||
return next;
|
||||
}
|
||||
|
||||
export function dilateMaskValues(values: Uint8ClampedArray, width: number, height: number, radius: number): Uint8ClampedArray {
|
||||
const safeRadius = Math.round(clampNumber(radius, 0, 256));
|
||||
if (safeRadius <= 0) return new Uint8ClampedArray(values);
|
||||
const next = new Uint8ClampedArray(values.length);
|
||||
for (let y = 0; y < height; y += 1) {
|
||||
for (let x = 0; x < width; x += 1) {
|
||||
let value = 0;
|
||||
for (let oy = -safeRadius; oy <= safeRadius; oy += 1) {
|
||||
for (let ox = -safeRadius; ox <= safeRadius; ox += 1) value = Math.max(value, values[clampInt(y + oy, 0, height - 1) * width + clampInt(x + ox, 0, width - 1)] ?? 0);
|
||||
}
|
||||
next[y * width + x] = value;
|
||||
}
|
||||
}
|
||||
return next;
|
||||
}
|
||||
|
||||
export function blurMaskValues(values: Uint8ClampedArray, width: number, height: number, radius: number): Uint8ClampedArray {
|
||||
const safeRadius = Math.round(clampNumber(radius, 0, 256));
|
||||
if (safeRadius <= 0) return new Uint8ClampedArray(values);
|
||||
const next = new Uint8ClampedArray(values.length);
|
||||
for (let y = 0; y < height; y += 1) {
|
||||
for (let x = 0; x < width; x += 1) {
|
||||
let total = 0;
|
||||
let count = 0;
|
||||
for (let oy = -safeRadius; oy <= safeRadius; oy += 1) {
|
||||
for (let ox = -safeRadius; ox <= safeRadius; ox += 1) {
|
||||
total += values[clampInt(y + oy, 0, height - 1) * width + clampInt(x + ox, 0, width - 1)] ?? 0;
|
||||
count += 1;
|
||||
}
|
||||
}
|
||||
next[y * width + x] = Math.round(total / count);
|
||||
}
|
||||
}
|
||||
return next;
|
||||
}
|
||||
|
||||
export function despeckleMaskValues(values: Uint8ClampedArray, width: number, height: number, strength: number): Uint8ClampedArray {
|
||||
const safeStrength = Math.round(clampNumber(strength, 0, 64));
|
||||
if (safeStrength <= 0) return new Uint8ClampedArray(values);
|
||||
|
||||
const radius = Math.max(1, Math.ceil(safeStrength / 6));
|
||||
const threshold = Math.max(1, Math.round(safeStrength / 2));
|
||||
const next = new Uint8ClampedArray(values);
|
||||
|
||||
for (let y = 0; y < height; y += 1) {
|
||||
for (let x = 0; x < width; x += 1) {
|
||||
const index = y * width + x;
|
||||
const visible = (values[index] ?? 0) > 127;
|
||||
let same = 0;
|
||||
for (let oy = -radius; oy <= radius; oy += 1) {
|
||||
for (let ox = -radius; ox <= radius; ox += 1) {
|
||||
if (ox === 0 && oy === 0) continue;
|
||||
const sample = values[clampInt(y + oy, 0, height - 1) * width + clampInt(x + ox, 0, width - 1)] ?? 0;
|
||||
if ((sample > 127) === visible) same += 1;
|
||||
}
|
||||
}
|
||||
if (same <= threshold) next[index] = visible ? 0 : 255;
|
||||
}
|
||||
}
|
||||
|
||||
return next;
|
||||
}
|
||||
|
||||
async function loadMaskValues(source: string, width: number, height: number): Promise<{ width: number; height: number; values: Uint8ClampedArray }> {
|
||||
const canvas = await loadImageCanvas(source, width, height);
|
||||
const context = require2dContext(canvas);
|
||||
const data = context.getImageData(0, 0, canvas.width, canvas.height);
|
||||
const values = new Uint8ClampedArray(canvas.width * canvas.height);
|
||||
for (let pixel = 0; pixel < values.length; pixel += 1) values[pixel] = maskValueFromRgba(data.data, pixel * 4);
|
||||
return { width: canvas.width, height: canvas.height, values };
|
||||
}
|
||||
|
||||
function maskValuesToDataUrl(values: Uint8ClampedArray, width: number, height: number, outputWidth = width, outputHeight = height): string {
|
||||
const canvas = createCanvas(outputWidth, outputHeight);
|
||||
const context = require2dContext(canvas);
|
||||
const imageData = context.createImageData(outputWidth, outputHeight);
|
||||
|
||||
for (let y = 0; y < outputHeight; y += 1) {
|
||||
for (let x = 0; x < outputWidth; x += 1) {
|
||||
const sourceX = Math.floor((x / outputWidth) * width);
|
||||
const sourceY = Math.floor((y / outputHeight) * height);
|
||||
const value = values[clampInt(sourceY, 0, height - 1) * width + clampInt(sourceX, 0, width - 1)] ?? 0;
|
||||
const index = (y * outputWidth + x) * 4;
|
||||
imageData.data[index] = 255;
|
||||
imageData.data[index + 1] = 255;
|
||||
imageData.data[index + 2] = 255;
|
||||
imageData.data[index + 3] = value;
|
||||
}
|
||||
}
|
||||
|
||||
context.putImageData(imageData, 0, 0);
|
||||
return canvas.toDataURL("image/png");
|
||||
}
|
||||
|
||||
function analyzeValues(values: Uint8ClampedArray, width: number, height: number, includeSoftPixels: boolean): { pixels: number; coverage: number; bounds?: Rect } {
|
||||
let pixels = 0;
|
||||
let minX = Number.POSITIVE_INFINITY;
|
||||
let minY = Number.POSITIVE_INFINITY;
|
||||
let maxX = Number.NEGATIVE_INFINITY;
|
||||
let maxY = Number.NEGATIVE_INFINITY;
|
||||
|
||||
for (let y = 0; y < height; y += 1) {
|
||||
for (let x = 0; x < width; x += 1) {
|
||||
const value = values[y * width + x] ?? 0;
|
||||
const active = includeSoftPixels ? value > 0 : value > 127;
|
||||
if (!active) continue;
|
||||
pixels += 1;
|
||||
minX = Math.min(minX, x);
|
||||
minY = Math.min(minY, y);
|
||||
maxX = Math.max(maxX, x + 1);
|
||||
maxY = Math.max(maxY, y + 1);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
pixels,
|
||||
coverage: pixels / Math.max(1, width * height),
|
||||
bounds: pixels > 0 ? { x: minX, y: minY, w: maxX - minX, h: maxY - minY } : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
function createCanvas(width: number, height: number): HTMLCanvasElement {
|
||||
const canvas = document.createElement("canvas");
|
||||
canvas.width = Math.max(1, Math.round(width));
|
||||
canvas.height = Math.max(1, Math.round(height));
|
||||
return canvas;
|
||||
}
|
||||
|
||||
function require2dContext(canvas: HTMLCanvasElement): CanvasRenderingContext2D {
|
||||
const context = canvas.getContext("2d");
|
||||
if (!context) throw new Error("Unable to create mask canvas");
|
||||
return context;
|
||||
}
|
||||
|
||||
function loadImage(source: string): Promise<HTMLImageElement> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const image = new Image();
|
||||
image.onload = () => resolve(image);
|
||||
image.onerror = () => reject(new Error("Failed to load image"));
|
||||
image.src = source;
|
||||
});
|
||||
}
|
||||
|
||||
function roundUp(value: number, multiple: number): number {
|
||||
const safeMultiple = Math.max(1, Math.round(multiple));
|
||||
return Math.ceil(value / safeMultiple) * safeMultiple;
|
||||
}
|
||||
|
||||
function clampNumber(value: number, min: number, max: number): number {
|
||||
if (!Number.isFinite(value)) return min;
|
||||
return Math.max(min, Math.min(max, value));
|
||||
}
|
||||
|
||||
function clampInt(value: number, min: number, max: number): number {
|
||||
return Math.round(Math.max(min, Math.min(max, value)));
|
||||
}
|
||||
Reference in New Issue
Block a user