feat: add ComfyUI integration for image generation and management
- Implemented ComfyGenerateRequest type and associated functions for generating images using various architectures and modes. - Added functions for listing generation options and handling image uploads. - Created workflows for different generation modes including SDXL, Z-Image, Z-Image Turbo, and Anima. - Introduced GenerationJobStatus component to display the status of ongoing generation jobs. - Developed MaskControls for managing mask operations and displaying mask analysis. - Created palette items for tool selection, layer management, and generation settings.
This commit is contained in:
7
operations/AGENTS.md
Normal file
7
operations/AGENTS.md
Normal file
@@ -0,0 +1,7 @@
|
||||
# Application Operation Rules
|
||||
|
||||
- `operations/` coordinates asynchronous editor use cases such as generation, import, raster processing, and export.
|
||||
- Operations may read immutable state snapshots, call injected platform capabilities, and dispatch commands.
|
||||
- Operations must never mutate `ImageDocument` or `EditorState` directly.
|
||||
- Keep React, DOM elements, WebGL objects, server-only APIs, and concrete network details out of operation contracts.
|
||||
- Prefer dependency injection for platform behavior so operation control flow remains testable.
|
||||
159
operations/document/layerActions.ts
Normal file
159
operations/document/layerActions.ts
Normal file
@@ -0,0 +1,159 @@
|
||||
import { commandIds } from "@commands/ids";
|
||||
import type { ImageDocument } from "@core/document";
|
||||
import type { ArtboardId } from "@core/id";
|
||||
import type { Layer } from "@core/layer";
|
||||
import { getLayerMask } from "@core/layer-mask-utils";
|
||||
import type { DocumentReadIndex, IndexedLayerInfo } from "@editor/document-indexes";
|
||||
import { resolveIndexedLayerBounds } from "@editor/document-indexes";
|
||||
import type { SelectionState } from "@editor/state";
|
||||
import type { AppStore } from "@editor/store";
|
||||
|
||||
export function addArtboard(document: ImageDocument, dispatch: AppStore["dispatch"]) {
|
||||
const index = document.artboards.length + 1;
|
||||
dispatch(commandIds.documentAddArtboard, {
|
||||
id: crypto.randomUUID(),
|
||||
name: `Artboard ${index}`,
|
||||
bounds: { x: (index - 1) * 40, y: (index - 1) * 40, w: 800, h: 600 },
|
||||
});
|
||||
}
|
||||
|
||||
export function addEmptyLayer(
|
||||
document: ImageDocument,
|
||||
artboardId: ArtboardId,
|
||||
selectedLayer: IndexedLayerInfo | undefined,
|
||||
dispatch: AppStore["dispatch"],
|
||||
) {
|
||||
const artboard = document.artboards.find((candidate) => candidate.id === artboardId);
|
||||
if (!artboard) return;
|
||||
|
||||
const assetId = crypto.randomUUID();
|
||||
const layerId = crypto.randomUUID();
|
||||
const width = Math.max(1, Math.round(artboard.bounds.w));
|
||||
const height = Math.max(1, Math.round(artboard.bounds.h));
|
||||
const source = `data:image/svg+xml,${encodeURIComponent(`<svg xmlns="http://www.w3.org/2000/svg" width="${width}" height="${height}"></svg>`)}`;
|
||||
|
||||
dispatch(commandIds.documentAddAsset, {
|
||||
asset: {
|
||||
id: assetId,
|
||||
name: "Empty Layer",
|
||||
mimeType: "image/svg+xml",
|
||||
source,
|
||||
intrinsicSize: { w: width, h: height },
|
||||
},
|
||||
});
|
||||
dispatch(commandIds.documentAddRasterLayer, {
|
||||
artboardId,
|
||||
parentGroupId: selectedLayer?.layer.type === "group" ? selectedLayer.layer.id : undefined,
|
||||
layer: {
|
||||
id: layerId,
|
||||
type: "raster",
|
||||
name: "Layer",
|
||||
visible: true,
|
||||
locked: false,
|
||||
opacity: 1,
|
||||
assetId,
|
||||
transform: { position: { x: artboard.bounds.x, y: artboard.bounds.y }, scale: { x: 1, y: 1 }, rotation: 0 },
|
||||
},
|
||||
});
|
||||
dispatch(commandIds.selectionSet, { artboardId, layerIds: [layerId] });
|
||||
}
|
||||
|
||||
export function addGroupLayer(artboardId: ArtboardId, dispatch: AppStore["dispatch"]) {
|
||||
dispatch(commandIds.documentAddGroupLayer, { artboardId, group: createGroup("Group") });
|
||||
}
|
||||
|
||||
export function groupLayers(artboardId: ArtboardId, layerIds: string[], dispatch: AppStore["dispatch"]) {
|
||||
dispatch(commandIds.documentGroupLayers, { artboardId, layerIds, group: createGroup("Group") });
|
||||
}
|
||||
|
||||
export function deleteSelection(selection: SelectionState, selectedLayer: IndexedLayerInfo | undefined, dispatch: AppStore["dispatch"]) {
|
||||
if (selectedLayer) {
|
||||
dispatch(commandIds.documentRemoveLayer, { layerId: selectedLayer.layer.id });
|
||||
return;
|
||||
}
|
||||
if (selection.artboardId) dispatch(commandIds.documentRemoveArtboard, { id: selection.artboardId });
|
||||
}
|
||||
|
||||
export function moveLayer(documentIndex: DocumentReadIndex, info: IndexedLayerInfo, direction: -1 | 1, dispatch: AppStore["dispatch"]) {
|
||||
const siblings = info.siblings;
|
||||
const maskLayerIds = documentIndex.maskLayerIdsByLayerList.get(siblings) ?? emptyLayerIds;
|
||||
const blocks = siblings.flatMap((layer, index) => {
|
||||
if (maskLayerIds.has(layer.id)) return [];
|
||||
|
||||
const layerMask = getLayerMask(layer);
|
||||
const maskIndex = layerMask ? siblings.findIndex((candidate) => candidate.id === layerMask.maskLayerId) : -1;
|
||||
const start = maskIndex >= 0 ? Math.min(maskIndex, index) : index;
|
||||
const end = maskIndex >= 0 ? Math.max(maskIndex, index) : index;
|
||||
return [{ layerId: layer.id, start, end, size: end - start + 1 }];
|
||||
});
|
||||
|
||||
const currentBlockIndex = blocks.findIndex((block) => block.layerId === info.layer.id);
|
||||
const currentBlock = blocks[currentBlockIndex];
|
||||
const targetBlock = blocks[currentBlockIndex + direction];
|
||||
if (!currentBlock || !targetBlock) return;
|
||||
|
||||
const insertionIndex = direction === -1 ? targetBlock.start : targetBlock.end + 1;
|
||||
const removedBeforeInsertion = currentBlock.end < insertionIndex ? currentBlock.size : currentBlock.start < insertionIndex ? insertionIndex - currentBlock.start : 0;
|
||||
|
||||
dispatch(commandIds.documentMoveLayer, {
|
||||
layerId: info.layer.id,
|
||||
toArtboardId: info.artboardId,
|
||||
toParentGroupId: info.parentGroupId,
|
||||
toIndex: insertionIndex - removedBeforeInsertion,
|
||||
});
|
||||
}
|
||||
|
||||
export function addLayerMask(documentIndex: DocumentReadIndex, layerInfo: IndexedLayerInfo, dispatch: AppStore["dispatch"]) {
|
||||
const layer = layerInfo.layer;
|
||||
if (layer.type === "group") return;
|
||||
|
||||
const asset = documentIndex.assetById.get(layer.assetId);
|
||||
const bounds = resolveIndexedLayerBounds(documentIndex, layer);
|
||||
if (!asset || !bounds) return;
|
||||
|
||||
const assetId = crypto.randomUUID();
|
||||
const maskLayerId = crypto.randomUUID();
|
||||
const width = Math.max(1, Math.round(asset.intrinsicSize.w));
|
||||
const height = Math.max(1, Math.round(asset.intrinsicSize.h));
|
||||
const source = `data:image/svg+xml,${encodeURIComponent(`<svg xmlns="http://www.w3.org/2000/svg" width="${width}" height="${height}" viewBox="0 0 ${width} ${height}"><rect width="${width}" height="${height}" fill="white"/></svg>`)}`;
|
||||
|
||||
dispatch(commandIds.documentAddLayerMask, {
|
||||
layerId: layer.id,
|
||||
asset: {
|
||||
id: assetId,
|
||||
name: `${layer.name} Mask`,
|
||||
mimeType: "image/svg+xml",
|
||||
source,
|
||||
intrinsicSize: { w: width, h: height },
|
||||
},
|
||||
maskLayer: {
|
||||
id: maskLayerId,
|
||||
type: "raster",
|
||||
name: `${layer.name} Mask`,
|
||||
visible: true,
|
||||
locked: false,
|
||||
opacity: 1,
|
||||
assetId,
|
||||
transform: {
|
||||
position: { x: bounds.x, y: bounds.y },
|
||||
scale: { x: bounds.w / width, y: bounds.h / height },
|
||||
rotation: layer.transform.rotation,
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
const emptyLayerIds = new Set<string>();
|
||||
|
||||
function createGroup(name: string): Extract<Layer, { type: "group" }> {
|
||||
return {
|
||||
id: crypto.randomUUID(),
|
||||
type: "group",
|
||||
name,
|
||||
visible: true,
|
||||
locked: false,
|
||||
opacity: 1,
|
||||
transform: { position: { x: 0, y: 0 }, scale: { x: 1, y: 1 }, rotation: 0 },
|
||||
children: [],
|
||||
};
|
||||
}
|
||||
4
operations/export/downloadArtboard.ts
Normal file
4
operations/export/downloadArtboard.ts
Normal file
@@ -0,0 +1,4 @@
|
||||
import type { Artboard } from "@core/artboard";
|
||||
import type { Asset } from "@core/asset";
|
||||
import { downloadArtboardPng as download } from "@platform/browser/exportArtboardPng";
|
||||
export function downloadArtboardPng(artboard: Artboard, assets: readonly Asset[]) { return download(artboard, assets); }
|
||||
50
operations/generation/candidateActions.ts
Normal file
50
operations/generation/candidateActions.ts
Normal file
@@ -0,0 +1,50 @@
|
||||
import type { ImageDocument } from "@core/document";
|
||||
import type { GenerationCandidate } from "@editor/state";
|
||||
import { loadImageCanvas, maskValueFromRgba } from "@platform/browser/maskRaster";
|
||||
|
||||
export async function createMaskedPixelReplacementSource(document: ImageDocument, candidate: GenerationCandidate): Promise<string> {
|
||||
if (!candidate.inpaint) throw new Error("Only inpaint candidates can replace masked pixels.");
|
||||
|
||||
const targetAsset = document.assets.find((asset) => asset.id === candidate.inpaint?.sourceAssetId);
|
||||
if (!targetAsset) throw new Error("The source layer for this candidate no longer exists.");
|
||||
|
||||
const targetCanvas = await loadImageCanvas(targetAsset.source, targetAsset.intrinsicSize.w, targetAsset.intrinsicSize.h);
|
||||
const generatedCanvas = await loadImageCanvas(candidate.source, candidate.width, candidate.height);
|
||||
const maskCanvas = await loadImageCanvas(candidate.inpaint.maskImage, candidate.width, candidate.height);
|
||||
|
||||
const targetContext = require2dContext(targetCanvas);
|
||||
const generatedContext = require2dContext(generatedCanvas);
|
||||
const maskContext = require2dContext(maskCanvas);
|
||||
const targetData = targetContext.getImageData(0, 0, targetCanvas.width, targetCanvas.height);
|
||||
const generatedData = generatedContext.getImageData(0, 0, generatedCanvas.width, generatedCanvas.height);
|
||||
const maskData = maskContext.getImageData(0, 0, maskCanvas.width, maskCanvas.height);
|
||||
const crop = candidate.inpaint.crop.assetBounds;
|
||||
|
||||
for (let y = 0; y < candidate.height; y += 1) {
|
||||
for (let x = 0; x < candidate.width; x += 1) {
|
||||
const targetX = Math.round(crop.x) + x;
|
||||
const targetY = Math.round(crop.y) + y;
|
||||
if (targetX < 0 || targetY < 0 || targetX >= targetCanvas.width || targetY >= targetCanvas.height) continue;
|
||||
|
||||
const generatedIndex = (y * generatedCanvas.width + x) * 4;
|
||||
const targetIndex = (targetY * targetCanvas.width + targetX) * 4;
|
||||
const mask = maskValueFromRgba(maskData.data, generatedIndex) / 255;
|
||||
if (mask <= 0) continue;
|
||||
|
||||
for (let channel = 0; channel < 4; channel += 1) {
|
||||
const previous = targetData.data[targetIndex + channel] ?? 0;
|
||||
const next = generatedData.data[generatedIndex + channel] ?? previous;
|
||||
targetData.data[targetIndex + channel] = Math.round(previous * (1 - mask) + next * mask);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
targetContext.putImageData(targetData, 0, 0);
|
||||
return targetCanvas.toDataURL("image/png");
|
||||
}
|
||||
|
||||
function require2dContext(canvas: HTMLCanvasElement): CanvasRenderingContext2D {
|
||||
const context = canvas.getContext("2d");
|
||||
if (!context) throw new Error("Unable to prepare generated candidate");
|
||||
return context;
|
||||
}
|
||||
26
operations/generation/generationJob.ts
Normal file
26
operations/generation/generationJob.ts
Normal file
@@ -0,0 +1,26 @@
|
||||
import { commandIds } from "@commands/ids";
|
||||
import type { GenerationJobKind } from "@editor/state";
|
||||
import type { AppStore } from "@editor/store";
|
||||
|
||||
export async function runGenerationJob(options: {
|
||||
kind: GenerationJobKind;
|
||||
label: string;
|
||||
dispatch: AppStore["dispatch"];
|
||||
task: () => Promise<void>;
|
||||
}): Promise<void> {
|
||||
const jobId = crypto.randomUUID();
|
||||
const startedAt = Date.now();
|
||||
const nextState = options.dispatch(commandIds.generationStartJob, { jobId, kind: options.kind, label: options.label, startedAt });
|
||||
if (!nextState.editor.generation.jobs.some((job) => job.id === jobId && job.status === "running")) return;
|
||||
|
||||
try {
|
||||
await options.task();
|
||||
options.dispatch(commandIds.generationSucceedJob, { jobId, finishedAt: Date.now() });
|
||||
} catch (reason: unknown) {
|
||||
options.dispatch(commandIds.generationFailJob, {
|
||||
jobId,
|
||||
finishedAt: Date.now(),
|
||||
error: reason instanceof Error ? reason.message : `${options.label} failed`,
|
||||
});
|
||||
}
|
||||
}
|
||||
36
operations/generation/inpaintPrep.test.ts
Normal file
36
operations/generation/inpaintPrep.test.ts
Normal file
@@ -0,0 +1,36 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import { applyMaskedContentModeToRgba } from "./inpaintPrep";
|
||||
|
||||
describe("inpaint prep", () => {
|
||||
test("converts only masked original pixels to grayscale", () => {
|
||||
const pixels = new Uint8ClampedArray([
|
||||
255, 0, 0, 255,
|
||||
0, 0, 255, 255,
|
||||
]);
|
||||
const mask = new Uint8ClampedArray([255, 0]);
|
||||
|
||||
const next = applyMaskedContentModeToRgba(pixels, 2, 1, mask, "original");
|
||||
|
||||
expect(next[0]).toBe(next[1]);
|
||||
expect(next[1]).toBe(next[2]);
|
||||
expect(Array.from(next.slice(4, 8))).toEqual([0, 0, 255, 255]);
|
||||
});
|
||||
|
||||
test("uses an edge map inside the mask without recoloring unmasked context", () => {
|
||||
const pixels = new Uint8ClampedArray([
|
||||
255, 0, 0, 255,
|
||||
0, 255, 0, 255,
|
||||
0, 0, 255, 255,
|
||||
255, 255, 0, 255,
|
||||
]);
|
||||
const mask = new Uint8ClampedArray([255, 0, 0, 0]);
|
||||
|
||||
const next = applyMaskedContentModeToRgba(pixels, 2, 2, mask, "edges");
|
||||
|
||||
expect(next[0]).toBe(next[1]);
|
||||
expect(next[1]).toBe(next[2]);
|
||||
expect(Array.from(next.slice(4, 8))).toEqual([0, 255, 0, 255]);
|
||||
expect(Array.from(next.slice(8, 12))).toEqual([0, 0, 255, 255]);
|
||||
expect(Array.from(next.slice(12, 16))).toEqual([255, 255, 0, 255]);
|
||||
});
|
||||
});
|
||||
260
operations/generation/inpaintPrep.ts
Normal file
260
operations/generation/inpaintPrep.ts
Normal file
@@ -0,0 +1,260 @@
|
||||
import type { Asset } from "@core/asset";
|
||||
import type { ImageDocument } from "@core/document";
|
||||
import type { Rect } from "@core/geometry";
|
||||
import type { Layer } from "@core/layer";
|
||||
import { getLayerMask } from "@core/layer-mask-utils";
|
||||
import type { SelectionState } from "@editor/state";
|
||||
import type { GenerateSettings } from "@editor/tools";
|
||||
import { createDocumentReadIndex, resolveIndexedLayerBounds } from "@editor/document-indexes";
|
||||
import { createNormalizedMaskSource, cropCanvas, cropMaskValuesToDataUrl, expandRectWithinBounds, loadImageCanvas } from "@platform/browser/maskRaster";
|
||||
|
||||
export type InpaintBundle = {
|
||||
inputImage: string;
|
||||
maskImage: string;
|
||||
width: number;
|
||||
height: number;
|
||||
targetLayerId: string;
|
||||
maskLayerId: string;
|
||||
sourceAssetId: string;
|
||||
maskAssetId: string;
|
||||
crop: {
|
||||
assetBounds: Rect;
|
||||
documentBounds: Rect;
|
||||
padding: number;
|
||||
maskedAreaOnly: boolean;
|
||||
};
|
||||
mask: {
|
||||
polarity: GenerateSettings["inpaint"]["maskPolarity"];
|
||||
activeBounds: Rect;
|
||||
};
|
||||
placement: {
|
||||
artboardId: string;
|
||||
layerName: string;
|
||||
transform: Extract<Layer, { type: "image" | "raster" }>["transform"];
|
||||
};
|
||||
backend: {
|
||||
growMaskBy: number;
|
||||
maskedContent: GenerateSettings["inpaint"]["maskedContent"];
|
||||
maskBlur: number;
|
||||
maskFeather: number;
|
||||
maskExpand: number;
|
||||
cropPadding: number;
|
||||
};
|
||||
};
|
||||
|
||||
type InpaintTarget = {
|
||||
artboardId: string;
|
||||
layer: Extract<Layer, { type: "image" | "raster" }>;
|
||||
asset: Asset;
|
||||
bounds: Rect;
|
||||
maskLayer: Extract<Layer, { type: "image" | "raster" }>;
|
||||
maskAsset: Asset;
|
||||
maskBounds: Rect;
|
||||
};
|
||||
|
||||
const modelMultiple = 8;
|
||||
const minModelSize = 64;
|
||||
const maxModelSize = 4096;
|
||||
|
||||
export async function buildInpaintBundle(document: ImageDocument, selection: SelectionState, settings: GenerateSettings): Promise<InpaintBundle> {
|
||||
const target = resolveInpaintTarget(document, selection);
|
||||
validateInpaintTarget(target);
|
||||
|
||||
const width = Math.max(1, Math.round(target.asset.intrinsicSize.w));
|
||||
const height = Math.max(1, Math.round(target.asset.intrinsicSize.h));
|
||||
const normalizedMask = await createNormalizedMaskSource(target.maskAsset.source, width, height, {
|
||||
polarity: settings.inpaint.maskPolarity,
|
||||
expand: settings.inpaint.maskExpand,
|
||||
feather: settings.inpaint.maskFeather,
|
||||
blur: settings.inpaint.maskBlur,
|
||||
despeckle: settings.inpaint.maskDespeckle,
|
||||
});
|
||||
|
||||
if (!normalizedMask.bounds) throw new Error("The selected layer mask has no inpaint pixels.");
|
||||
|
||||
const crop = settings.inpaint.maskedAreaOnly
|
||||
? expandRectWithinBounds(normalizedMask.bounds, settings.inpaint.cropPadding, { w: width, h: height }, modelMultiple, minModelSize)
|
||||
: { x: 0, y: 0, w: width, h: height };
|
||||
const outputWidth = toModelSize(crop.w, "width");
|
||||
const outputHeight = toModelSize(crop.h, "height");
|
||||
|
||||
const inputCanvas = prepareMaskedContentInputCanvas(await loadImageCanvas(target.asset.source, width, height), normalizedMask.values, settings.inpaint.maskedContent);
|
||||
const inputImage = cropCanvas(inputCanvas, crop, outputWidth, outputHeight);
|
||||
const maskImage = cropMaskValuesToDataUrl(normalizedMask.values, width, height, crop, outputWidth, outputHeight);
|
||||
const scaleX = target.bounds.w / width;
|
||||
const scaleY = target.bounds.h / height;
|
||||
const documentBounds = {
|
||||
x: target.bounds.x + crop.x * scaleX,
|
||||
y: target.bounds.y + crop.y * scaleY,
|
||||
w: outputWidth * scaleX,
|
||||
h: outputHeight * scaleY,
|
||||
};
|
||||
|
||||
return {
|
||||
inputImage,
|
||||
maskImage,
|
||||
width: outputWidth,
|
||||
height: outputHeight,
|
||||
targetLayerId: target.layer.id,
|
||||
maskLayerId: target.maskLayer.id,
|
||||
sourceAssetId: target.asset.id,
|
||||
maskAssetId: target.maskAsset.id,
|
||||
crop: {
|
||||
assetBounds: crop,
|
||||
documentBounds,
|
||||
padding: settings.inpaint.cropPadding,
|
||||
maskedAreaOnly: settings.inpaint.maskedAreaOnly,
|
||||
},
|
||||
mask: {
|
||||
polarity: settings.inpaint.maskPolarity,
|
||||
activeBounds: normalizedMask.bounds,
|
||||
},
|
||||
placement: {
|
||||
artboardId: target.artboardId,
|
||||
layerName: `${target.layer.name} inpaint`,
|
||||
transform: {
|
||||
position: { x: documentBounds.x, y: documentBounds.y },
|
||||
scale: { x: documentBounds.w / outputWidth, y: documentBounds.h / outputHeight },
|
||||
rotation: target.layer.transform.rotation,
|
||||
},
|
||||
},
|
||||
backend: {
|
||||
growMaskBy: settings.inpaint.growMaskBy,
|
||||
maskedContent: settings.inpaint.maskedContent,
|
||||
maskBlur: settings.inpaint.maskBlur,
|
||||
maskFeather: settings.inpaint.maskFeather,
|
||||
maskExpand: settings.inpaint.maskExpand,
|
||||
cropPadding: settings.inpaint.cropPadding,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function resolveInpaintTarget(document: ImageDocument, selection: SelectionState): InpaintTarget {
|
||||
if (selection.layerIds.length !== 1 || !selection.layerIds[0]) throw new Error("Select one image or raster layer to inpaint.");
|
||||
|
||||
const documentIndex = createDocumentReadIndex(document);
|
||||
const layerInfo = documentIndex.layerInfoById.get(selection.layerIds[0]);
|
||||
if (!layerInfo || layerInfo.layer.type === "group") throw new Error("Select one image or raster layer to inpaint.");
|
||||
|
||||
const asset = documentIndex.assetById.get(layerInfo.layer.assetId);
|
||||
if (!asset) throw new Error("The selected layer is missing its source image.");
|
||||
const layerMask = getLayerMask(layerInfo.layer);
|
||||
if (!layerMask?.enabled) throw new Error("Add a layer mask before running inpaint.");
|
||||
|
||||
const maskLayer = documentIndex.layerById.get(layerMask.maskLayerId);
|
||||
if (!maskLayer || maskLayer.type === "group") throw new Error("The selected layer mask is missing.");
|
||||
|
||||
const maskAsset = documentIndex.assetById.get(maskLayer.assetId);
|
||||
if (!maskAsset) throw new Error("The selected layer mask is missing its image data.");
|
||||
|
||||
const bounds = resolveIndexedLayerBounds(documentIndex, layerInfo.layer);
|
||||
const maskBounds = resolveIndexedLayerBounds(documentIndex, maskLayer);
|
||||
if (!bounds || !maskBounds) throw new Error("Unable to resolve the selected layer and mask bounds.");
|
||||
|
||||
return { artboardId: layerInfo.artboardId, layer: layerInfo.layer, asset, bounds, maskLayer, maskAsset, maskBounds };
|
||||
}
|
||||
|
||||
function validateInpaintTarget(target: InpaintTarget) {
|
||||
if (target.asset.intrinsicSize.w <= 0 || target.asset.intrinsicSize.h <= 0) throw new Error("The selected image has an invalid size.");
|
||||
if (target.maskAsset.intrinsicSize.w <= 0 || target.maskAsset.intrinsicSize.h <= 0) throw new Error("The selected mask has an invalid size.");
|
||||
if (Math.round(target.asset.intrinsicSize.w) !== Math.round(target.maskAsset.intrinsicSize.w) || Math.round(target.asset.intrinsicSize.h) !== Math.round(target.maskAsset.intrinsicSize.h)) {
|
||||
throw new Error("The selected layer and mask image sizes do not match.");
|
||||
}
|
||||
if (!rectsAligned(target.bounds, target.maskBounds) || Math.abs(target.layer.transform.rotation - target.maskLayer.transform.rotation) > 0.001) {
|
||||
throw new Error("The selected layer and mask are not aligned.");
|
||||
}
|
||||
}
|
||||
|
||||
function rectsAligned(a: Rect, b: Rect): boolean {
|
||||
return Math.abs(a.x - b.x) <= 0.5 && Math.abs(a.y - b.y) <= 0.5 && Math.abs(a.w - b.w) <= 0.5 && Math.abs(a.h - b.h) <= 0.5;
|
||||
}
|
||||
|
||||
function toModelSize(value: number, axis: "width" | "height"): number {
|
||||
const rounded = Math.max(minModelSize, Math.ceil(value / modelMultiple) * modelMultiple);
|
||||
if (rounded > maxModelSize) throw new Error(`The inpaint ${axis} is too large for the model. Use masked-area inpaint or a smaller source.`);
|
||||
return rounded;
|
||||
}
|
||||
|
||||
function prepareMaskedContentInputCanvas(sourceCanvas: HTMLCanvasElement, maskValues: Uint8ClampedArray, mode: GenerateSettings["inpaint"]["maskedContent"]): HTMLCanvasElement {
|
||||
if (mode === "neutral" || mode === "originalColor") return sourceCanvas;
|
||||
const context = sourceCanvas.getContext("2d");
|
||||
if (!context) return sourceCanvas;
|
||||
const imageData = context.getImageData(0, 0, sourceCanvas.width, sourceCanvas.height);
|
||||
imageData.data.set(applyMaskedContentModeToRgba(imageData.data, sourceCanvas.width, sourceCanvas.height, maskValues, mode));
|
||||
context.putImageData(imageData, 0, 0);
|
||||
return sourceCanvas;
|
||||
}
|
||||
|
||||
export function applyMaskedContentModeToRgba(data: Uint8ClampedArray, width: number, height: number, maskValues: Uint8ClampedArray, mode: GenerateSettings["inpaint"]["maskedContent"]): Uint8ClampedArray {
|
||||
const next = new Uint8ClampedArray(data);
|
||||
if (mode === "neutral" || mode === "originalColor") return next;
|
||||
|
||||
const edgeValues = mode === "edges" ? createEdgeMapValues(data, width, height) : undefined;
|
||||
|
||||
for (let pixel = 0; pixel < width * height; pixel += 1) {
|
||||
const amount = (maskValues[pixel] ?? 0) / 255;
|
||||
if (amount <= 0) continue;
|
||||
|
||||
const index = pixel * 4;
|
||||
const red = data[index] ?? 0;
|
||||
const green = data[index + 1] ?? 0;
|
||||
const blue = data[index + 2] ?? 0;
|
||||
const target = edgeValues ? edgeValues[pixel] ?? 128 : luminance(red, green, blue);
|
||||
next[index] = blendChannel(red, target, amount);
|
||||
next[index + 1] = blendChannel(green, target, amount);
|
||||
next[index + 2] = blendChannel(blue, target, amount);
|
||||
}
|
||||
|
||||
return next;
|
||||
}
|
||||
|
||||
function createEdgeMapValues(data: Uint8ClampedArray, width: number, height: number): Uint8ClampedArray {
|
||||
const gray = new Float32Array(width * height);
|
||||
const edges = new Uint8ClampedArray(width * height);
|
||||
|
||||
for (let pixel = 0; pixel < width * height; pixel += 1) {
|
||||
const index = pixel * 4;
|
||||
gray[pixel] = luminance(data[index] ?? 0, data[index + 1] ?? 0, data[index + 2] ?? 0);
|
||||
}
|
||||
|
||||
for (let y = 0; y < height; y += 1) {
|
||||
for (let x = 0; x < width; x += 1) {
|
||||
const gx =
|
||||
-sampleGray(gray, width, height, x - 1, y - 1) +
|
||||
sampleGray(gray, width, height, x + 1, y - 1) -
|
||||
2 * sampleGray(gray, width, height, x - 1, y) +
|
||||
2 * sampleGray(gray, width, height, x + 1, y) -
|
||||
sampleGray(gray, width, height, x - 1, y + 1) +
|
||||
sampleGray(gray, width, height, x + 1, y + 1);
|
||||
const gy =
|
||||
-sampleGray(gray, width, height, x - 1, y - 1) -
|
||||
2 * sampleGray(gray, width, height, x, y - 1) -
|
||||
sampleGray(gray, width, height, x + 1, y - 1) +
|
||||
sampleGray(gray, width, height, x - 1, y + 1) +
|
||||
2 * sampleGray(gray, width, height, x, y + 1) +
|
||||
sampleGray(gray, width, height, x + 1, y + 1);
|
||||
const magnitude = Math.hypot(gx, gy);
|
||||
edges[y * width + x] = Math.round(clampNumber(128 + Math.max(0, magnitude - 24) * 0.75, 128, 255));
|
||||
}
|
||||
}
|
||||
|
||||
return edges;
|
||||
}
|
||||
|
||||
function sampleGray(values: Float32Array, width: number, height: number, x: number, y: number): number {
|
||||
const clampedX = Math.max(0, Math.min(width - 1, x));
|
||||
const clampedY = Math.max(0, Math.min(height - 1, y));
|
||||
return values[clampedY * width + clampedX] ?? 0;
|
||||
}
|
||||
|
||||
function luminance(red: number, green: number, blue: number): number {
|
||||
return Math.round(0.2126 * red + 0.7152 * green + 0.0722 * blue);
|
||||
}
|
||||
|
||||
function blendChannel(previous: number, next: number, amount: number): number {
|
||||
return Math.round(previous * (1 - amount) + next * amount);
|
||||
}
|
||||
|
||||
function clampNumber(value: number, min: number, max: number): number {
|
||||
return Math.max(min, Math.min(max, value));
|
||||
}
|
||||
16
operations/generation/loadResources.ts
Normal file
16
operations/generation/loadResources.ts
Normal file
@@ -0,0 +1,16 @@
|
||||
import { commandIds } from "@commands/ids";
|
||||
import type { AppStore } from "@editor/store";
|
||||
import { fetchGenerationOptions } from "@platform/comfy/generationClient";
|
||||
import type { GenerationOptions } from "@editor/state";
|
||||
|
||||
export async function loadGenerationResources(store: AppStore): Promise<void> {
|
||||
const current = store.getState().editor.generation.resources.status;
|
||||
if (current === "loading" || current === "ready") return;
|
||||
store.dispatch(commandIds.generationLoadResources, undefined);
|
||||
try {
|
||||
const options = await fetchGenerationOptions() as GenerationOptions;
|
||||
store.dispatch(commandIds.generationSetResources, { options });
|
||||
} catch (reason: unknown) {
|
||||
store.dispatch(commandIds.generationFailResources, { error: reason instanceof Error ? reason.message : "Unable to load ComfyUI models" });
|
||||
}
|
||||
}
|
||||
239
operations/generation/runGenerate.ts
Normal file
239
operations/generation/runGenerate.ts
Normal file
@@ -0,0 +1,239 @@
|
||||
import { commandIds } from "@commands/ids";
|
||||
import type { ImageDocument } from "@core/document";
|
||||
import type { Transform } from "@core/geometry";
|
||||
import type { Layer } from "@core/layer";
|
||||
import { getLayerMask } from "@core/layer-mask-utils";
|
||||
import type { AppStore } from "@editor/store";
|
||||
import type { GenerationCandidate, SelectionState, ViewportState } from "@editor/state";
|
||||
import type { GenerateSettings } from "@editor/tools";
|
||||
import { buildInpaintBundle, type InpaintBundle } from "./inpaintPrep";
|
||||
import { imageSourceToDataUrl, loadImageSize } from "@platform/browser/imageRaster";
|
||||
import { requestGeneration } from "@platform/comfy/generationClient";
|
||||
|
||||
export async function runGenerate(options: {
|
||||
document: ImageDocument;
|
||||
selection: SelectionState;
|
||||
viewport: ViewportState;
|
||||
settings: GenerateSettings;
|
||||
dispatch: AppStore["dispatch"];
|
||||
}) {
|
||||
const { document, selection, settings, dispatch } = options;
|
||||
const artboard = selection.artboardId ? document.artboards.find((candidate) => candidate.id === selection.artboardId) : document.artboards[0];
|
||||
if (!artboard) return;
|
||||
|
||||
const target = resolveSelectedImage(document, selection);
|
||||
const inpaintBundle = settings.mode === "inpaint" ? await buildInpaintBundle(document, selection, settings) : undefined;
|
||||
const inputImage = inpaintBundle?.inputImage ?? (target && settings.mode !== "text-to-image" ? await imageSourceToDataUrl(target.asset.source) : undefined);
|
||||
const maskImage = inpaintBundle?.maskImage;
|
||||
const seed = resolveSeed(settings.seed);
|
||||
const requestSettings = { ...settings, seed };
|
||||
const width = inpaintBundle?.width ?? settings.width;
|
||||
const height = inpaintBundle?.height ?? settings.height;
|
||||
const generated = await requestGenerate({
|
||||
settings: requestSettings,
|
||||
width,
|
||||
height,
|
||||
inputImage,
|
||||
maskImage,
|
||||
inpaintBundle,
|
||||
});
|
||||
const intrinsicSize = await loadImageSize(generated.source);
|
||||
const targetArtboardId = inpaintBundle?.placement.artboardId ?? artboard.id;
|
||||
const placement = {
|
||||
artboardId: targetArtboardId,
|
||||
layerName: inpaintBundle?.placement.layerName ?? "Generated image",
|
||||
transform: generatedLayerTransform(inpaintBundle, intrinsicSize),
|
||||
};
|
||||
|
||||
dispatch(commandIds.generationAddCandidate, {
|
||||
candidate: createGenerationCandidate({
|
||||
source: generated.source,
|
||||
mimeType: generated.mimeType,
|
||||
intrinsicSize,
|
||||
settings: requestSettings,
|
||||
seed,
|
||||
width,
|
||||
height,
|
||||
inputImage,
|
||||
maskImage,
|
||||
placement,
|
||||
inpaintBundle,
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
export async function runGenerateFromCandidate(options: {
|
||||
candidate: GenerationCandidate;
|
||||
settings?: GenerateSettings;
|
||||
dispatch: AppStore["dispatch"];
|
||||
}) {
|
||||
const settings = options.settings ?? options.candidate.settings;
|
||||
const seed = resolveSeed(settings.seed);
|
||||
const requestSettings = { ...settings, seed };
|
||||
const generated = await requestGenerate({
|
||||
settings: requestSettings,
|
||||
width: options.candidate.width,
|
||||
height: options.candidate.height,
|
||||
inputImage: options.candidate.inputImage,
|
||||
maskImage: options.candidate.maskImage,
|
||||
inpaintCandidate: options.candidate,
|
||||
});
|
||||
const intrinsicSize = await loadImageSize(generated.source);
|
||||
|
||||
options.dispatch(commandIds.generationAddCandidate, {
|
||||
candidate: {
|
||||
...options.candidate,
|
||||
id: crypto.randomUUID(),
|
||||
source: generated.source,
|
||||
mimeType: generated.mimeType,
|
||||
intrinsicSize,
|
||||
settings: requestSettings,
|
||||
seed,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function createGenerationCandidate(options: {
|
||||
source: string;
|
||||
mimeType: string;
|
||||
intrinsicSize: { w: number; h: number };
|
||||
settings: GenerateSettings;
|
||||
seed: number;
|
||||
width: number;
|
||||
height: number;
|
||||
inputImage?: string;
|
||||
maskImage?: string;
|
||||
placement: GenerationCandidate["placement"];
|
||||
inpaintBundle?: InpaintBundle;
|
||||
}): GenerationCandidate {
|
||||
return {
|
||||
id: crypto.randomUUID(),
|
||||
source: options.source,
|
||||
mimeType: options.mimeType,
|
||||
intrinsicSize: options.intrinsicSize,
|
||||
mode: options.settings.mode,
|
||||
settings: options.settings,
|
||||
seed: options.seed,
|
||||
width: options.width,
|
||||
height: options.height,
|
||||
inputImage: options.inputImage,
|
||||
maskImage: options.maskImage,
|
||||
placement: options.placement,
|
||||
inpaint: options.inpaintBundle
|
||||
? {
|
||||
targetLayerId: options.inpaintBundle.targetLayerId,
|
||||
maskLayerId: options.inpaintBundle.maskLayerId,
|
||||
sourceAssetId: options.inpaintBundle.sourceAssetId,
|
||||
maskAssetId: options.inpaintBundle.maskAssetId,
|
||||
inputImage: options.inpaintBundle.inputImage,
|
||||
maskImage: options.inpaintBundle.maskImage,
|
||||
crop: options.inpaintBundle.crop,
|
||||
mask: options.inpaintBundle.mask,
|
||||
backend: options.inpaintBundle.backend,
|
||||
}
|
||||
: undefined,
|
||||
};
|
||||
}
|
||||
|
||||
async function requestGenerate(options: {
|
||||
settings: GenerateSettings;
|
||||
width: number;
|
||||
height: number;
|
||||
inputImage?: string;
|
||||
maskImage?: string;
|
||||
inpaintBundle?: InpaintBundle;
|
||||
inpaintCandidate?: GenerationCandidate;
|
||||
}) {
|
||||
return requestGeneration({
|
||||
architecture: options.settings.architecture,
|
||||
mode: options.settings.mode,
|
||||
model: options.settings.model,
|
||||
textEncoder: options.settings.textEncoder,
|
||||
vae: options.settings.vae,
|
||||
prompt: options.settings.prompt,
|
||||
negativePrompt: options.settings.negativePrompt,
|
||||
strength: options.settings.strength,
|
||||
steps: options.settings.steps,
|
||||
cfg: options.settings.cfg,
|
||||
seed: options.settings.seed,
|
||||
sampler: options.settings.sampler,
|
||||
scheduler: options.settings.scheduler,
|
||||
width: options.width,
|
||||
height: options.height,
|
||||
outpaint: options.settings.outpaint,
|
||||
inpaint: resolveInpaintRequest(options.inpaintBundle, options.inpaintCandidate, options.settings),
|
||||
inputImage: options.inputImage,
|
||||
maskImage: options.maskImage,
|
||||
});
|
||||
}
|
||||
|
||||
function resolveInpaintRequest(inpaintBundle: InpaintBundle | undefined, inpaintCandidate: GenerationCandidate | undefined, settings: GenerateSettings) {
|
||||
if (inpaintBundle) {
|
||||
return {
|
||||
growMaskBy: inpaintBundle.backend.growMaskBy,
|
||||
maskedContent: inpaintBundle.backend.maskedContent,
|
||||
maskBlur: inpaintBundle.backend.maskBlur,
|
||||
maskFeather: inpaintBundle.backend.maskFeather,
|
||||
maskExpand: inpaintBundle.backend.maskExpand,
|
||||
cropPadding: inpaintBundle.backend.cropPadding,
|
||||
maskPolarity: inpaintBundle.mask.polarity,
|
||||
crop: inpaintBundle.crop,
|
||||
placement: inpaintBundle.placement,
|
||||
};
|
||||
}
|
||||
|
||||
if (inpaintCandidate?.inpaint) {
|
||||
return {
|
||||
growMaskBy: inpaintCandidate.inpaint.backend.growMaskBy,
|
||||
maskedContent: inpaintCandidate.inpaint.backend.maskedContent,
|
||||
maskBlur: inpaintCandidate.inpaint.backend.maskBlur,
|
||||
maskFeather: inpaintCandidate.inpaint.backend.maskFeather,
|
||||
maskExpand: inpaintCandidate.inpaint.backend.maskExpand,
|
||||
cropPadding: inpaintCandidate.inpaint.backend.cropPadding,
|
||||
maskPolarity: inpaintCandidate.inpaint.mask.polarity,
|
||||
crop: inpaintCandidate.inpaint.crop,
|
||||
placement: inpaintCandidate.placement,
|
||||
};
|
||||
}
|
||||
|
||||
return settings.inpaint;
|
||||
}
|
||||
|
||||
function generatedLayerTransform(inpaintBundle: InpaintBundle | undefined, intrinsicSize: { w: number; h: number }): Transform {
|
||||
if (!inpaintBundle) return { position: { x: 0, y: 0 }, scale: { x: 1, y: 1 }, rotation: 0 };
|
||||
return {
|
||||
position: { ...inpaintBundle.placement.transform.position },
|
||||
scale: {
|
||||
x: (inpaintBundle.placement.transform.scale.x * inpaintBundle.width) / Math.max(1, intrinsicSize.w),
|
||||
y: (inpaintBundle.placement.transform.scale.y * inpaintBundle.height) / Math.max(1, intrinsicSize.h),
|
||||
},
|
||||
rotation: inpaintBundle.placement.transform.rotation,
|
||||
};
|
||||
}
|
||||
|
||||
function resolveSeed(seed: number): number {
|
||||
return seed < 0 ? Math.floor(Math.random() * 2 ** 32) : Math.round(seed);
|
||||
}
|
||||
|
||||
function resolveSelectedImage(document: ImageDocument, selection: SelectionState) {
|
||||
const layerId = selection.layerIds[0];
|
||||
if (!layerId) return undefined;
|
||||
const layer = findLayer(document.artboards.find((artboard) => artboard.id === selection.artboardId)?.layers ?? [], layerId);
|
||||
if (!layer || layer.type === "group") return undefined;
|
||||
const asset = document.assets.find((candidate) => candidate.id === layer.assetId);
|
||||
const layerMask = getLayerMask(layer);
|
||||
const maskLayer = layerMask?.enabled ? findLayer(document.artboards.flatMap((artboard) => artboard.layers), layerMask.maskLayerId) : undefined;
|
||||
const maskAsset = maskLayer && maskLayer.type !== "group" ? document.assets.find((candidate) => candidate.id === maskLayer.assetId) : undefined;
|
||||
return asset ? { layer, asset, maskAsset } : undefined;
|
||||
}
|
||||
|
||||
function findLayer(layers: readonly Layer[], layerId: string): Layer | undefined {
|
||||
for (const layer of layers) {
|
||||
if (layer.id === layerId) return layer;
|
||||
if (layer.type === "group") {
|
||||
const found = findLayer(layer.children, layerId);
|
||||
if (found) return found;
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
34
operations/import/importImage.ts
Normal file
34
operations/import/importImage.ts
Normal file
@@ -0,0 +1,34 @@
|
||||
import { commandIds } from "@commands/ids";
|
||||
import type { AppStore } from "@editor/store";
|
||||
import type { BrowserImageFile } from "@platform/browser/imageFiles";
|
||||
|
||||
export function importImageAsLayer(store: AppStore, image: BrowserImageFile): boolean {
|
||||
const state = store.getState();
|
||||
const artboard = state.editor.selection.artboardId
|
||||
? state.document.artboards.find((candidate) => candidate.id === state.editor.selection.artboardId)
|
||||
: state.document.artboards[0];
|
||||
if (!artboard) {
|
||||
image.release();
|
||||
return false;
|
||||
}
|
||||
|
||||
const assetId = crypto.randomUUID();
|
||||
const layerId = crypto.randomUUID();
|
||||
const center = state.editor.viewport.center;
|
||||
store.dispatch(commandIds.documentAddAsset, { asset: { id: assetId, name: image.name, mimeType: image.mimeType, source: image.source, intrinsicSize: image.intrinsicSize } });
|
||||
store.dispatch(commandIds.documentAddImageLayer, {
|
||||
artboardId: artboard.id,
|
||||
layer: {
|
||||
id: layerId,
|
||||
type: "image",
|
||||
name: image.name,
|
||||
visible: true,
|
||||
locked: false,
|
||||
opacity: 1,
|
||||
assetId,
|
||||
transform: { position: { x: center.x - image.intrinsicSize.w / 2, y: center.y - image.intrinsicSize.h / 2 }, scale: { x: 1, y: 1 }, rotation: 0 },
|
||||
},
|
||||
});
|
||||
store.dispatch(commandIds.selectionSet, { artboardId: artboard.id, layerIds: [layerId] });
|
||||
return true;
|
||||
}
|
||||
51
operations/masks/chromaKey.ts
Normal file
51
operations/masks/chromaKey.ts
Normal file
@@ -0,0 +1,51 @@
|
||||
import { commandIds } from "@commands/ids";
|
||||
import type { ImageDocument } from "@core/document";
|
||||
import type { Layer } from "@core/layer";
|
||||
import { getLayerMask } from "@core/layer-mask-utils";
|
||||
import { resolveTransformTargetBounds } from "@editor/transform-targets";
|
||||
import type { SelectionState } from "@editor/state";
|
||||
import type { AppStore } from "@editor/store";
|
||||
import type { ChromaKeySettings } from "@editor/tools";
|
||||
import { createChromaKeyMask, createChromaKeyPreview } from "@platform/browser/chromaKey";
|
||||
|
||||
export function previewChromaKey(source: string, width: number, height: number, settings: ChromaKeySettings) { return createChromaKeyPreview(source, width, height, settings); }
|
||||
|
||||
export function resolveChromaKeyTarget(document: ImageDocument, selection: SelectionState) {
|
||||
const layerId = selection.layerIds[0];
|
||||
if (selection.layerIds.length !== 1 || !layerId) return undefined;
|
||||
const layer = findLayer(document.artboards.find((artboard) => artboard.id === selection.artboardId)?.layers ?? [], layerId);
|
||||
if (!layer || layer.type === "group") return undefined;
|
||||
const asset = document.assets.find((candidate) => candidate.id === layer.assetId);
|
||||
const bounds = resolveTransformTargetBounds(document, { type: "layer", id: layer.id });
|
||||
const layerMask = getLayerMask(layer);
|
||||
const maskLayer = layerMask?.enabled ? findLayer(document.artboards.flatMap((artboard) => artboard.layers), layerMask.maskLayerId) : undefined;
|
||||
const maskAsset = maskLayer && maskLayer.type !== "group" ? document.assets.find((candidate) => candidate.id === maskLayer.assetId) : undefined;
|
||||
return asset && bounds ? { layer, asset, bounds, maskLayer, maskAsset } : undefined;
|
||||
}
|
||||
|
||||
export async function applyChromaKeyMask(target: NonNullable<ReturnType<typeof resolveChromaKeyTarget>>, settings: ChromaKeySettings, dispatch: AppStore["dispatch"]) {
|
||||
const source = await createChromaKeyMask(target.asset.source, target.asset.intrinsicSize.w, target.asset.intrinsicSize.h, settings);
|
||||
dispatch(commandIds.toolSetBrushStrokePreview, undefined);
|
||||
if (target.maskAsset && target.maskLayer && target.maskLayer.type !== "group") {
|
||||
dispatch(commandIds.documentApplyLayerMaskOperation, { maskLayerId: target.maskLayer.id, source, mimeType: "image/png", operation: { type: "chromaKey" } });
|
||||
return;
|
||||
}
|
||||
const assetId = crypto.randomUUID();
|
||||
const maskLayerId = crypto.randomUUID();
|
||||
const width = Math.max(1, Math.round(target.asset.intrinsicSize.w));
|
||||
const height = Math.max(1, Math.round(target.asset.intrinsicSize.h));
|
||||
dispatch(commandIds.documentAddLayerMask, {
|
||||
layerId: target.layer.id,
|
||||
asset: { id: assetId, name: `${target.layer.name} Chroma Mask`, mimeType: "image/png", source, intrinsicSize: { w: width, h: height } },
|
||||
maskLayer: { id: maskLayerId, type: "raster", name: `${target.layer.name} Chroma Mask`, visible: true, locked: false, opacity: 1, assetId, transform: { position: { x: target.bounds.x, y: target.bounds.y }, scale: { x: target.bounds.w / width, y: target.bounds.h / height }, rotation: target.layer.transform.rotation } },
|
||||
});
|
||||
dispatch(commandIds.toolExitMaskEdit, undefined);
|
||||
dispatch(commandIds.toolSetActive, { tool: "chromaKey" });
|
||||
}
|
||||
|
||||
function findLayer(layers: readonly Layer[], layerId: string): Layer | undefined {
|
||||
for (const layer of layers) {
|
||||
if (layer.id === layerId) return layer;
|
||||
if (layer.type === "group") { const found = findLayer(layer.children, layerId); if (found) return found; }
|
||||
}
|
||||
}
|
||||
60
operations/masks/magic-wand.ts
Normal file
60
operations/masks/magic-wand.ts
Normal file
@@ -0,0 +1,60 @@
|
||||
import { commandIds } from "@commands/ids";
|
||||
import type { ImageDocument } from "@core/document";
|
||||
import type { Vec2D } from "@core/geometry";
|
||||
import type { Layer } from "@core/layer";
|
||||
import { getLayerMask } from "@core/layer-mask-utils";
|
||||
import { resolveTransformTargetBounds } from "@editor/transform-targets";
|
||||
import type { AppStore } from "@editor/store";
|
||||
import type { EditorState } from "@editor/state";
|
||||
import { createWandMask } from "@platform/browser/magicWandRaster";
|
||||
|
||||
export async function applyMagicWandAt(store: AppStore, point: Vec2D, modeOverride?: EditorState["tools"]["magicWand"]["mode"]) {
|
||||
const state = store.getState();
|
||||
if (state.editor.tools.activeTool !== "magicWand") return false;
|
||||
const target = resolveTarget(state.document, state.editor);
|
||||
if (!target) return true;
|
||||
const x = Math.floor((point.x - target.layer.transform.position.x) / Math.max(0.0001, target.layer.transform.scale.x));
|
||||
const y = Math.floor((point.y - target.layer.transform.position.y) / Math.max(0.0001, target.layer.transform.scale.y));
|
||||
if (x < 0 || y < 0 || x >= target.asset.intrinsicSize.w || y >= target.asset.intrinsicSize.h) return true;
|
||||
const source = await createWandMask(target.asset.source, target.maskAsset?.source, Math.round(target.asset.intrinsicSize.w), Math.round(target.asset.intrinsicSize.h), x, y, { ...state.editor.tools.magicWand, mode: modeOverride ?? state.editor.tools.magicWand.mode });
|
||||
if (target.maskAsset && target.maskLayer && target.maskLayer.type !== "group") {
|
||||
store.dispatch(commandIds.documentApplyLayerMaskOperation, { maskLayerId: target.maskLayer.id, source, mimeType: "image/png", operation: { type: "magicWand" } });
|
||||
return true;
|
||||
}
|
||||
const assetId = crypto.randomUUID();
|
||||
const maskLayerId = crypto.randomUUID();
|
||||
const width = Math.max(1, Math.round(target.asset.intrinsicSize.w));
|
||||
const height = Math.max(1, Math.round(target.asset.intrinsicSize.h));
|
||||
store.dispatch(commandIds.documentAddLayerMask, {
|
||||
layerId: target.layer.id,
|
||||
asset: { id: assetId, name: `${target.layer.name} Wand Mask`, mimeType: "image/png", source, intrinsicSize: { w: width, h: height } },
|
||||
maskLayer: { id: maskLayerId, type: "raster", name: `${target.layer.name} Wand Mask`, visible: true, locked: false, opacity: 1, assetId, transform: { position: { x: target.bounds.x, y: target.bounds.y }, scale: { x: target.bounds.w / width, y: target.bounds.h / height }, rotation: target.layer.transform.rotation } },
|
||||
});
|
||||
store.dispatch(commandIds.toolExitMaskEdit, undefined);
|
||||
store.dispatch(commandIds.toolSetActive, { tool: "magicWand" });
|
||||
return true;
|
||||
}
|
||||
|
||||
function resolveTarget(document: ImageDocument, editor: EditorState) {
|
||||
const layerId = editor.selection.layerIds[0];
|
||||
if (!layerId || editor.selection.layerIds.length !== 1) return undefined;
|
||||
const layer = findLayer(document.artboards.flatMap((artboard) => artboard.layers), layerId);
|
||||
if (!layer || layer.type === "group") return undefined;
|
||||
const asset = document.assets.find((candidate) => candidate.id === layer.assetId);
|
||||
const bounds = resolveTransformTargetBounds(document, { type: "layer", id: layer.id });
|
||||
const layerMask = getLayerMask(layer);
|
||||
const maskLayer = layerMask?.enabled ? findLayer(document.artboards.flatMap((artboard) => artboard.layers), layerMask.maskLayerId) : undefined;
|
||||
const maskAsset = maskLayer && maskLayer.type !== "group" ? document.assets.find((candidate) => candidate.id === maskLayer.assetId) : undefined;
|
||||
return asset && bounds ? { layer, asset, bounds, maskLayer, maskAsset } : undefined;
|
||||
}
|
||||
|
||||
function findLayer(layers: readonly Layer[], layerId: string): Layer | undefined {
|
||||
for (const layer of layers) {
|
||||
if (layer.id === layerId) return layer;
|
||||
if (layer.type === "group") {
|
||||
const found = findLayer(layer.children, layerId);
|
||||
if (found) return found;
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
10
operations/masks/rasterActions.ts
Normal file
10
operations/masks/rasterActions.ts
Normal file
@@ -0,0 +1,10 @@
|
||||
import { commandIds } from "@commands/ids";
|
||||
import type { Asset } from "@core/asset";
|
||||
import type { LayerId } from "@core/id";
|
||||
import type { AppStore } from "@editor/store";
|
||||
import { analyzeMaskSource, applyMaskRasterOperation, createSolidMaskSource, type MaskAnalysis, type MaskRasterOperation } from "@platform/browser/maskRaster";
|
||||
|
||||
export type { MaskAnalysis, MaskRasterOperation };
|
||||
export function analyzeMask(asset: Asset): Promise<MaskAnalysis> { return analyzeMaskSource(asset.source, asset.intrinsicSize.w, asset.intrinsicSize.h); }
|
||||
export async function runMaskOperation(maskLayerId: LayerId, asset: Asset, operation: MaskRasterOperation, dispatch: AppStore["dispatch"]) { const source = await applyMaskRasterOperation(asset.source, asset.intrinsicSize.w, asset.intrinsicSize.h, operation); dispatch(commandIds.documentApplyLayerMaskOperation, { maskLayerId, source, mimeType: "image/png", operation }); }
|
||||
export function createRefinementMask(width: number, height: number) { return createSolidMaskSource(width, height, "white"); }
|
||||
225
operations/paint/brush.ts
Normal file
225
operations/paint/brush.ts
Normal file
@@ -0,0 +1,225 @@
|
||||
import { commandIds } from "@commands/ids";
|
||||
import type { ImageDocument } from "@core/document";
|
||||
import type { Vec2D } from "@core/geometry";
|
||||
import type { Layer } from "@core/layer";
|
||||
import type { RasterLayer } from "@core/raster-layer";
|
||||
import type { MaskEditState, SelectionState } from "@editor/state";
|
||||
import { isPanInteractionMode, type ToolState } from "@editor/tools";
|
||||
import type { AppStore } from "@editor/store";
|
||||
import { brushSurfaceDataUrl, brushSurfaceObjectUrl, cancelFrame, createBrushSurface, drawBrushSegment, releaseObjectUrl, scheduleFrame, type BrushSurface } from "@platform/browser/brushRaster";
|
||||
|
||||
export type BrushSession = {
|
||||
layerId: string;
|
||||
assetId: string;
|
||||
width: number;
|
||||
height: number;
|
||||
surface: BrushSurface;
|
||||
ready: Promise<boolean>;
|
||||
previousPoint: Vec2D;
|
||||
mode: "brush" | "eraser";
|
||||
changed?: boolean;
|
||||
pending?: Promise<void>;
|
||||
cancelled?: boolean;
|
||||
previewClosed?: boolean;
|
||||
previewRequested?: boolean;
|
||||
previewInFlight?: boolean;
|
||||
previewFrame?: number;
|
||||
previewSource?: string;
|
||||
};
|
||||
|
||||
export type BrushTargetEditorState = {
|
||||
selection: SelectionState;
|
||||
tools: Pick<ToolState, "activeTool" | "interactionMode">;
|
||||
maskEdit?: MaskEditState;
|
||||
};
|
||||
|
||||
export function beginBrushSession(document: ImageDocument, editor: BrushTargetEditorState, point: Vec2D): BrushSession | undefined {
|
||||
const layer = resolveBrushTargetLayer(document, editor);
|
||||
if (!layer || (editor.tools.activeTool !== "brush" && editor.tools.activeTool !== "eraser")) return undefined;
|
||||
|
||||
const asset = document.assets.find((candidate) => candidate.id === layer.assetId);
|
||||
if (!asset) return undefined;
|
||||
|
||||
const surface = createBrushSurface(asset.intrinsicSize.w, asset.intrinsicSize.h, asset.source);
|
||||
if (!surface) return undefined;
|
||||
|
||||
const session: BrushSession = {
|
||||
layerId: layer.id,
|
||||
assetId: layer.assetId,
|
||||
width: surface.width,
|
||||
height: surface.height,
|
||||
surface,
|
||||
ready: surface.ready,
|
||||
previousPoint: point,
|
||||
mode: editor.tools.activeTool,
|
||||
};
|
||||
return session;
|
||||
}
|
||||
|
||||
export function canPreviewBrush(document: ImageDocument, editor: BrushTargetEditorState): boolean {
|
||||
return Boolean(resolveBrushTargetLayer(document, editor));
|
||||
}
|
||||
|
||||
export function brushUnavailableHint(document: ImageDocument, editor: BrushTargetEditorState): string | undefined {
|
||||
if (isPanInteractionMode(editor.tools.interactionMode) || (editor.tools.activeTool !== "brush" && editor.tools.activeTool !== "eraser")) return undefined;
|
||||
if (resolveBrushTargetLayer(document, editor)) return undefined;
|
||||
|
||||
const layerId = editor.maskEdit?.maskLayerId ?? editor.selection.layerIds[0];
|
||||
if (!layerId) {
|
||||
if (editor.selection.artboardId) return "Brushes paint layers, not artboards. Select or add a raster layer first.";
|
||||
return "Select a raster layer or layer mask to paint.";
|
||||
}
|
||||
|
||||
const layer = findLayer(document.artboards.flatMap((artboard) => artboard.layers), layerId);
|
||||
if (!layer) return "Select a raster layer or layer mask to paint.";
|
||||
if (layer.locked) return "Unlock this layer before painting.";
|
||||
if (layer.type === "image") return "Image layers are non-destructive. Add a layer mask to paint or erase.";
|
||||
if (layer.type === "group") return "Select a raster layer inside the group to paint.";
|
||||
if (!editor.maskEdit && !layer.visible) return "Show this layer before painting.";
|
||||
return "Select a raster layer or layer mask to paint.";
|
||||
}
|
||||
|
||||
function resolveBrushTargetLayer(document: ImageDocument, editor: BrushTargetEditorState): RasterLayer | undefined {
|
||||
if (isPanInteractionMode(editor.tools.interactionMode) || (editor.tools.activeTool !== "brush" && editor.tools.activeTool !== "eraser")) return undefined;
|
||||
const editingMask = Boolean(editor.maskEdit);
|
||||
const layerId = editor.maskEdit?.maskLayerId ?? editor.selection.layerIds[0];
|
||||
if (!layerId) return undefined;
|
||||
const layer = findRasterLayer(document.artboards.flatMap((artboard) => artboard.layers), layerId);
|
||||
if (!layer || layer.locked || (!editingMask && !layer.visible)) return undefined;
|
||||
return layer;
|
||||
}
|
||||
|
||||
export function updateBrushSession(options: {
|
||||
store: AppStore;
|
||||
session: BrushSession;
|
||||
point: Vec2D;
|
||||
color: string;
|
||||
size: number;
|
||||
hardness: number;
|
||||
}): BrushSession {
|
||||
const state = options.store.getState();
|
||||
const layer = findRasterLayer(state.document.artboards.flatMap((artboard) => artboard.layers), options.session.layerId);
|
||||
if (!layer || layer.assetId !== options.session.assetId) return options.session;
|
||||
|
||||
const from = options.session.previousPoint;
|
||||
const to = options.point;
|
||||
options.session.previousPoint = to;
|
||||
options.session.pending = (options.session.pending ?? Promise.resolve())
|
||||
.then(async () => {
|
||||
if (options.session.cancelled) return;
|
||||
if (!(await options.session.ready) || options.session.cancelled) return;
|
||||
|
||||
drawBrushSegment(options.session.surface, {
|
||||
from: documentPointToAssetPoint(from, layer, options.session.width, options.session.height),
|
||||
to: documentPointToAssetPoint(to, layer, options.session.width, options.session.height),
|
||||
color: state.editor.maskEdit ? "#ffffff" : options.color,
|
||||
size: options.size,
|
||||
hardness: options.hardness,
|
||||
mode: options.session.mode,
|
||||
});
|
||||
|
||||
if (options.session.cancelled) return;
|
||||
options.session.changed = true;
|
||||
requestBrushStrokePreview({ store: options.store, session: options.session });
|
||||
})
|
||||
.catch(() => undefined);
|
||||
|
||||
return options.session;
|
||||
}
|
||||
|
||||
export async function commitBrushSession(options: { store: AppStore; session: BrushSession }) {
|
||||
await options.session.pending;
|
||||
if (options.session.cancelled) return;
|
||||
|
||||
const source = options.session.changed ? brushSurfaceDataUrl(options.session.surface) : undefined;
|
||||
if (source) {
|
||||
const state = options.store.getState();
|
||||
const maskEdit = state.editor.maskEdit;
|
||||
if (maskEdit?.maskLayerId === options.session.layerId) {
|
||||
options.store.dispatch(commandIds.documentApplyLayerMaskOperation, { maskLayerId: maskEdit.maskLayerId, source, mimeType: "image/png", operation: { type: "paint" } });
|
||||
} else {
|
||||
options.store.dispatch(commandIds.documentUpdateAssetSource, { assetId: options.session.assetId, source });
|
||||
}
|
||||
}
|
||||
options.store.dispatch(commandIds.toolSetBrushStrokePreview, undefined);
|
||||
closeBrushStrokePreview(options.session);
|
||||
}
|
||||
|
||||
export function cancelBrushSession(options: { store: AppStore; session: BrushSession }) {
|
||||
options.session.cancelled = true;
|
||||
options.store.dispatch(commandIds.toolSetBrushStrokePreview, undefined);
|
||||
closeBrushStrokePreview(options.session);
|
||||
}
|
||||
|
||||
function documentPointToAssetPoint(point: Vec2D, layer: RasterLayer, width: number, height: number): Vec2D {
|
||||
return {
|
||||
x: ((point.x - layer.transform.position.x) / Math.max(0.0001, layer.transform.scale.x) / width) * width,
|
||||
y: ((point.y - layer.transform.position.y) / Math.max(0.0001, layer.transform.scale.y) / height) * height,
|
||||
};
|
||||
}
|
||||
|
||||
function requestBrushStrokePreview(options: { store: AppStore; session: BrushSession }) {
|
||||
if (options.session.cancelled || options.session.previewClosed) return;
|
||||
|
||||
options.session.previewRequested = true;
|
||||
if (options.session.previewFrame !== undefined || options.session.previewInFlight) return;
|
||||
|
||||
options.session.previewFrame = scheduleFrame(() => {
|
||||
options.session.previewFrame = undefined;
|
||||
void publishBrushStrokePreview(options);
|
||||
});
|
||||
}
|
||||
|
||||
async function publishBrushStrokePreview(options: { store: AppStore; session: BrushSession }) {
|
||||
if (options.session.cancelled || options.session.previewClosed || !options.session.previewRequested) return;
|
||||
|
||||
options.session.previewRequested = false;
|
||||
options.session.previewInFlight = true;
|
||||
const source = await brushSurfaceObjectUrl(options.session.surface).catch(() => undefined);
|
||||
options.session.previewInFlight = false;
|
||||
|
||||
if (!source) {
|
||||
if (options.session.previewRequested) requestBrushStrokePreview(options);
|
||||
return;
|
||||
}
|
||||
|
||||
if (options.session.cancelled || options.session.previewClosed) {
|
||||
releaseObjectUrl(source);
|
||||
return;
|
||||
}
|
||||
|
||||
const previousSource = options.session.previewSource;
|
||||
options.session.previewSource = source;
|
||||
options.store.dispatch(commandIds.toolSetBrushStrokePreview, { layerId: options.session.layerId, assetId: options.session.assetId, source });
|
||||
releaseObjectUrl(previousSource);
|
||||
|
||||
if (options.session.previewRequested) requestBrushStrokePreview(options);
|
||||
}
|
||||
|
||||
function closeBrushStrokePreview(session: BrushSession) {
|
||||
session.previewClosed = true;
|
||||
if (session.previewFrame !== undefined) {
|
||||
cancelFrame(session.previewFrame);
|
||||
session.previewFrame = undefined;
|
||||
}
|
||||
if (session.previewSource) {
|
||||
releaseObjectUrl(session.previewSource);
|
||||
session.previewSource = undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function findRasterLayer(layers: Layer[], layerId: string): RasterLayer | undefined {
|
||||
const layer = findLayer(layers, layerId);
|
||||
return layer?.type === "raster" ? layer : undefined;
|
||||
}
|
||||
|
||||
function findLayer(layers: Layer[], layerId: string): Layer | undefined {
|
||||
for (const layer of layers) {
|
||||
if (layer.id === layerId) return layer;
|
||||
if (layer.type === "group") {
|
||||
const child = findLayer(layer.children, layerId);
|
||||
if (child) return child;
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
Reference in New Issue
Block a user