feat: add ComfyUI integration for image generation and management
- Implemented ComfyGenerateRequest type and associated functions for generating images using various architectures and modes. - Added functions for listing generation options and handling image uploads. - Created workflows for different generation modes including SDXL, Z-Image, Z-Image Turbo, and Anima. - Introduced GenerationJobStatus component to display the status of ongoing generation jobs. - Developed MaskControls for managing mask operations and displaying mask analysis. - Created palette items for tool selection, layer management, and generation settings.
This commit is contained in:
6
platform/AGENTS.md
Normal file
6
platform/AGENTS.md
Normal file
@@ -0,0 +1,6 @@
|
||||
# Platform Adapter Rules
|
||||
|
||||
- `platform/` implements browser or runtime capabilities behind focused functions and interfaces.
|
||||
- Browser raster, image decoding, object URLs, downloads, and HTTP clients belong here.
|
||||
- Platform adapters do not own application state and do not dispatch commands.
|
||||
- Keep business rules, document traversal, and editor workflow decisions out of this layer.
|
||||
29
platform/browser/brushRaster.ts
Normal file
29
platform/browser/brushRaster.ts
Normal file
@@ -0,0 +1,29 @@
|
||||
export type BrushSurface = {
|
||||
readonly width: number;
|
||||
readonly height: number;
|
||||
readonly ready: Promise<boolean>;
|
||||
readonly resource: object;
|
||||
};
|
||||
|
||||
type InternalSurface = BrushSurface & { canvas: HTMLCanvasElement; context: CanvasRenderingContext2D };
|
||||
|
||||
export function createBrushSurface(width: number, height: number, source: string): BrushSurface | undefined {
|
||||
const canvas = document.createElement("canvas"); canvas.width = Math.max(1, Math.round(width)); canvas.height = Math.max(1, Math.round(height));
|
||||
const context = canvas.getContext("2d"); if (!context) return undefined;
|
||||
const surface = { width: canvas.width, height: canvas.height, canvas, context, resource: {}, ready: Promise.resolve(false) } as InternalSurface;
|
||||
(surface as { ready: Promise<boolean> }).ready = loadImage(source).then((image) => { context.clearRect(0, 0, canvas.width, canvas.height); context.drawImage(image, 0, 0, canvas.width, canvas.height); return true; }).catch(() => false);
|
||||
return surface;
|
||||
}
|
||||
|
||||
export function drawBrushSegment(surface: BrushSurface, options: { from: { x: number; y: number }; to: { x: number; y: number }; color: string; size: number; hardness: number; mode: "brush" | "eraser" }) {
|
||||
const context = internal(surface).context; const hardness = Math.max(0, Math.min(100, options.hardness)) / 100;
|
||||
context.save(); context.globalCompositeOperation = options.mode === "eraser" ? "destination-out" : "source-over"; context.strokeStyle = options.color; context.shadowColor = options.mode === "eraser" ? "rgba(0,0,0,1)" : options.color; context.shadowBlur = (1 - hardness) * options.size; context.lineWidth = options.size; context.lineCap = "round"; context.lineJoin = "round"; context.beginPath(); context.moveTo(options.from.x, options.from.y); context.lineTo(options.to.x, options.to.y); context.stroke(); context.restore();
|
||||
}
|
||||
|
||||
export function brushSurfaceDataUrl(surface: BrushSurface) { try { return internal(surface).canvas.toDataURL("image/png"); } catch { return undefined; } }
|
||||
export function brushSurfaceObjectUrl(surface: BrushSurface) { return new Promise<string | undefined>((resolve) => internal(surface).canvas.toBlob((blob) => resolve(blob ? URL.createObjectURL(blob) : undefined), "image/png")); }
|
||||
export function releaseObjectUrl(source?: string) { if (source) URL.revokeObjectURL(source); }
|
||||
export function scheduleFrame(callback: () => void) { return requestAnimationFrame(callback); }
|
||||
export function cancelFrame(id: number) { cancelAnimationFrame(id); }
|
||||
function internal(surface: BrushSurface) { return surface as InternalSurface; }
|
||||
function loadImage(source: string) { return new Promise<HTMLImageElement>((resolve, reject) => { const image = new Image(); image.onload = () => resolve(image); image.onerror = () => reject(new Error("Failed to load raster layer")); image.src = source; }); }
|
||||
66
platform/browser/chromaKey.ts
Normal file
66
platform/browser/chromaKey.ts
Normal file
@@ -0,0 +1,66 @@
|
||||
import { blurMaskValues, despeckleMaskValues, dilateMaskValues, erodeMaskValues } from "./maskRaster";
|
||||
|
||||
type ChromaKeySettings = { color: string; tolerance: number; softness: number; feather: number; choke: number; despeckle: number; spill: number };
|
||||
|
||||
export async function createChromaKeyPreview(source: string, width: number, height: number, settings: ChromaKeySettings) {
|
||||
return render(source, width, height, settings, false);
|
||||
}
|
||||
|
||||
export async function createChromaKeyMask(source: string, width: number, height: number, settings: ChromaKeySettings) {
|
||||
return render(source, width, height, settings, true);
|
||||
}
|
||||
|
||||
async function render(source: string, width: number, height: number, settings: ChromaKeySettings, maskOnly: boolean) {
|
||||
const canvas = document.createElement("canvas");
|
||||
canvas.width = Math.max(1, Math.round(width));
|
||||
canvas.height = Math.max(1, Math.round(height));
|
||||
const context = canvas.getContext("2d");
|
||||
if (!context) return source;
|
||||
context.drawImage(await loadImage(source), 0, 0, canvas.width, canvas.height);
|
||||
const data = context.getImageData(0, 0, canvas.width, canvas.height);
|
||||
const alpha = chromaKeyAlpha(data, canvas.width, canvas.height, settings);
|
||||
for (let pixel = 0; pixel < alpha.length; pixel++) {
|
||||
const index = pixel * 4;
|
||||
if (maskOnly) data.data[index] = data.data[index + 1] = data.data[index + 2] = 255;
|
||||
data.data[index + 3] = alpha[pixel] ?? 255;
|
||||
}
|
||||
context.putImageData(data, 0, 0);
|
||||
return canvas.toDataURL("image/png");
|
||||
}
|
||||
|
||||
function chromaKeyAlpha(data: ImageData, width: number, height: number, settings: ChromaKeySettings) {
|
||||
const hex = settings.color.replace("#", "");
|
||||
const key = { r: Number.parseInt(hex.slice(0, 2), 16), g: Number.parseInt(hex.slice(2, 4), 16), b: Number.parseInt(hex.slice(4, 6), 16) };
|
||||
const alpha = new Uint8ClampedArray(width * height);
|
||||
for (let pixel = 0; pixel < alpha.length; pixel++) {
|
||||
const index = pixel * 4;
|
||||
const red = data.data[index] ?? 0;
|
||||
const green = data.data[index + 1] ?? 0;
|
||||
const blue = data.data[index + 2] ?? 0;
|
||||
const distance = Math.hypot(red - key.r, green - key.g, blue - key.b);
|
||||
const tolerance = Math.max(0, Math.min(255, settings.tolerance));
|
||||
const softness = Math.max(0, Math.min(255, settings.softness));
|
||||
const edgeKeep = distance <= tolerance ? 0 : softness > 0 && distance < tolerance + softness ? (distance - tolerance) / softness : 1;
|
||||
const dominant = key.g >= key.r && key.g >= key.b ? green : key.r >= key.b ? red : blue;
|
||||
const neutral = key.g >= key.r && key.g >= key.b ? Math.max(red, blue) : key.r >= key.b ? Math.max(green, blue) : Math.max(red, green);
|
||||
const spillKeep = 1 - Math.max(0, dominant - neutral) / 255 * Math.max(0, Math.min(100, settings.spill)) / 100;
|
||||
alpha[pixel] = Math.round((data.data[index + 3] ?? 255) * Math.max(0, Math.min(edgeKeep, spillKeep)));
|
||||
}
|
||||
let next: Uint8ClampedArray<ArrayBufferLike> = alpha;
|
||||
const despeckle = Math.round(Math.max(0, Math.min(20, settings.despeckle)));
|
||||
const choke = Math.round(Math.max(-20, Math.min(20, settings.choke)));
|
||||
const feather = Math.round(Math.max(0, Math.min(20, settings.feather)));
|
||||
if (despeckle > 0) next = despeckleMaskValues(next, width, height, despeckle);
|
||||
if (choke > 0) next = erodeMaskValues(next, width, height, choke);
|
||||
if (choke < 0) next = dilateMaskValues(next, width, height, -choke);
|
||||
return feather > 0 ? blurMaskValues(next, width, height, feather) : next;
|
||||
}
|
||||
|
||||
function loadImage(source: string) {
|
||||
return new Promise<HTMLImageElement>((resolve, reject) => {
|
||||
const image = new Image();
|
||||
image.onload = () => resolve(image);
|
||||
image.onerror = () => reject(new Error("Failed to load image"));
|
||||
image.src = source;
|
||||
});
|
||||
}
|
||||
156
platform/browser/exportArtboardPng.ts
Normal file
156
platform/browser/exportArtboardPng.ts
Normal file
@@ -0,0 +1,156 @@
|
||||
import type { Artboard } from "@core/artboard";
|
||||
import type { Asset } from "@core/asset";
|
||||
import type { Rect } from "@core/geometry";
|
||||
import type { Layer } from "@core/layer";
|
||||
import { getLayerMask } from "@core/layer-mask-utils";
|
||||
|
||||
export async function downloadArtboardPng(artboard: Artboard, assets: readonly Asset[]) {
|
||||
const width = Math.max(1, Math.round(artboard.bounds.w));
|
||||
const height = Math.max(1, Math.round(artboard.bounds.h));
|
||||
const canvas = document.createElement("canvas");
|
||||
canvas.width = width;
|
||||
canvas.height = height;
|
||||
|
||||
const context = canvas.getContext("2d");
|
||||
if (!context) throw new Error("Canvas 2D is not available");
|
||||
|
||||
if (artboard.backgroundColor !== "transparent") {
|
||||
context.fillStyle = artboard.backgroundColor;
|
||||
context.fillRect(0, 0, width, height);
|
||||
}
|
||||
|
||||
const maskLayerIds = collectMaskLayerIds(artboard.layers);
|
||||
context.save();
|
||||
context.translate(-artboard.bounds.x, -artboard.bounds.y);
|
||||
for (const layer of renderStack(artboard.layers)) await drawLayer(context, layer, artboard.layers, assets, artboard.bounds, { maskLayerIds });
|
||||
context.restore();
|
||||
|
||||
const url = canvas.toDataURL("image/png");
|
||||
const link = document.createElement("a");
|
||||
link.href = url;
|
||||
link.download = `${safeFilename(artboard.name)}.png`;
|
||||
link.click();
|
||||
}
|
||||
|
||||
async function drawLayer(
|
||||
context: CanvasRenderingContext2D,
|
||||
layer: Layer,
|
||||
layerTree: readonly Layer[],
|
||||
assets: readonly Asset[],
|
||||
artboardBounds: Rect,
|
||||
options: { maskLayerIds: ReadonlySet<string>; ignoreOwnMask?: boolean },
|
||||
) {
|
||||
if (!layer.visible || (!options.ignoreOwnMask && options.maskLayerIds.has(layer.id))) return;
|
||||
|
||||
const layerMask = getLayerMask(layer);
|
||||
if (!options.ignoreOwnMask && layerMask?.enabled) {
|
||||
const maskLayer = findLayer(layerTree, layerMask.maskLayerId);
|
||||
if (!maskLayer) return;
|
||||
await drawMaskedLayer(context, layer, maskLayer, layerTree, assets, artboardBounds);
|
||||
return;
|
||||
}
|
||||
|
||||
context.save();
|
||||
context.globalAlpha *= layer.opacity;
|
||||
|
||||
if (layer.type === "group") {
|
||||
for (const child of renderStack(layer.children)) await drawLayer(context, child, layerTree, assets, artboardBounds, options);
|
||||
context.restore();
|
||||
return;
|
||||
}
|
||||
|
||||
const asset = assets.find((candidate) => candidate.id === layer.assetId);
|
||||
if (!asset) {
|
||||
context.restore();
|
||||
return;
|
||||
}
|
||||
|
||||
const image = await loadImage(asset.source);
|
||||
context.drawImage(
|
||||
image,
|
||||
layer.transform.position.x,
|
||||
layer.transform.position.y,
|
||||
asset.intrinsicSize.w * layer.transform.scale.x,
|
||||
asset.intrinsicSize.h * layer.transform.scale.y,
|
||||
);
|
||||
context.restore();
|
||||
}
|
||||
|
||||
async function drawMaskedLayer(
|
||||
context: CanvasRenderingContext2D,
|
||||
layer: Layer,
|
||||
maskLayer: Layer,
|
||||
layerTree: readonly Layer[],
|
||||
assets: readonly Asset[],
|
||||
artboardBounds: Rect,
|
||||
) {
|
||||
const layerCanvas = createArtboardCanvas(artboardBounds);
|
||||
const layerContext = translatedContext(layerCanvas, artboardBounds);
|
||||
await drawLayer(layerContext, layer, layerTree, assets, artboardBounds, { maskLayerIds: new Set(), ignoreOwnMask: true });
|
||||
layerContext.restore();
|
||||
|
||||
const maskCanvas = createArtboardCanvas(artboardBounds);
|
||||
const maskContext = translatedContext(maskCanvas, artboardBounds);
|
||||
await drawLayer(maskContext, maskLayer, layerTree, assets, artboardBounds, { maskLayerIds: new Set(), ignoreOwnMask: true });
|
||||
maskContext.restore();
|
||||
|
||||
const compositeContext = layerCanvas.getContext("2d");
|
||||
if (!compositeContext) return;
|
||||
compositeContext.globalCompositeOperation = "destination-in";
|
||||
compositeContext.drawImage(maskCanvas, 0, 0);
|
||||
compositeContext.globalCompositeOperation = "source-over";
|
||||
|
||||
context.drawImage(layerCanvas, artboardBounds.x, artboardBounds.y);
|
||||
}
|
||||
|
||||
function renderStack(layers: readonly Layer[]) {
|
||||
return [...layers].reverse();
|
||||
}
|
||||
|
||||
function createArtboardCanvas(bounds: Rect) {
|
||||
const canvas = document.createElement("canvas");
|
||||
canvas.width = Math.max(1, Math.round(bounds.w));
|
||||
canvas.height = Math.max(1, Math.round(bounds.h));
|
||||
return canvas;
|
||||
}
|
||||
|
||||
function translatedContext(canvas: HTMLCanvasElement, bounds: Rect) {
|
||||
const context = canvas.getContext("2d");
|
||||
if (!context) throw new Error("Canvas 2D is not available");
|
||||
context.save();
|
||||
context.translate(-bounds.x, -bounds.y);
|
||||
return context;
|
||||
}
|
||||
|
||||
function collectMaskLayerIds(layers: readonly Layer[], ids = new Set<string>()) {
|
||||
for (const layer of layers) {
|
||||
const layerMask = getLayerMask(layer);
|
||||
if (layerMask) ids.add(layerMask.maskLayerId);
|
||||
if (layer.type === "group") collectMaskLayerIds(layer.children, ids);
|
||||
}
|
||||
return ids;
|
||||
}
|
||||
|
||||
function findLayer(layers: readonly Layer[], layerId: string): Layer | undefined {
|
||||
for (const layer of layers) {
|
||||
if (layer.id === layerId) return layer;
|
||||
if (layer.type === "group") {
|
||||
const child = findLayer(layer.children, layerId);
|
||||
if (child) return child;
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function loadImage(source: string) {
|
||||
return new Promise<HTMLImageElement>((resolve, reject) => {
|
||||
const image = new Image();
|
||||
image.onload = () => resolve(image);
|
||||
image.onerror = () => reject(new Error("Failed to load image for export"));
|
||||
image.src = source;
|
||||
});
|
||||
}
|
||||
|
||||
function safeFilename(name: string) {
|
||||
return name.trim().replace(/[^a-z0-9-_]+/gi, "-").replace(/^-+|-+$/g, "") || "artboard";
|
||||
}
|
||||
32
platform/browser/imageFiles.ts
Normal file
32
platform/browser/imageFiles.ts
Normal file
@@ -0,0 +1,32 @@
|
||||
export type BrowserImageFile = {
|
||||
name: string;
|
||||
mimeType: string;
|
||||
source: string;
|
||||
intrinsicSize: { w: number; h: number };
|
||||
release(): void;
|
||||
};
|
||||
|
||||
export async function decodeBrowserImageFile(file: File): Promise<BrowserImageFile | undefined> {
|
||||
if (!file.type.startsWith("image/")) return undefined;
|
||||
const source = await fileToDataUrl(file);
|
||||
const intrinsicSize = await loadImageSize(source);
|
||||
return { name: file.name, mimeType: file.type, source, intrinsicSize, release: () => undefined };
|
||||
}
|
||||
|
||||
function fileToDataUrl(file: File): Promise<string> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const reader = new FileReader();
|
||||
reader.onload = () => typeof reader.result === "string" ? resolve(reader.result) : reject(new Error("Failed to read image file"));
|
||||
reader.onerror = () => reject(reader.error ?? new Error("Failed to read image file"));
|
||||
reader.readAsDataURL(file);
|
||||
});
|
||||
}
|
||||
|
||||
function loadImageSize(source: string): Promise<{ w: number; h: number }> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const image = new Image();
|
||||
image.onload = () => resolve({ w: image.naturalWidth, h: image.naturalHeight });
|
||||
image.onerror = () => reject(new Error("Failed to load image"));
|
||||
image.src = source;
|
||||
});
|
||||
}
|
||||
24
platform/browser/imageRaster.ts
Normal file
24
platform/browser/imageRaster.ts
Normal file
@@ -0,0 +1,24 @@
|
||||
export async function imageSourceToDataUrl(source: string): Promise<string> {
|
||||
if (source.startsWith("data:")) return source;
|
||||
const image = await loadImage(source);
|
||||
const canvas = document.createElement("canvas");
|
||||
canvas.width = image.naturalWidth;
|
||||
canvas.height = image.naturalHeight;
|
||||
const context = canvas.getContext("2d");
|
||||
if (!context) throw new Error("Unable to read selected image");
|
||||
context.drawImage(image, 0, 0);
|
||||
return canvas.toDataURL("image/png");
|
||||
}
|
||||
|
||||
export function loadImageSize(source: string): Promise<{ w: number; h: number }> {
|
||||
return loadImage(source).then((image) => ({ w: image.naturalWidth, h: image.naturalHeight }));
|
||||
}
|
||||
|
||||
function loadImage(source: string): Promise<HTMLImageElement> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const image = new Image();
|
||||
image.onload = () => resolve(image);
|
||||
image.onerror = () => reject(new Error("Failed to load image"));
|
||||
image.src = source;
|
||||
});
|
||||
}
|
||||
38
platform/browser/magicWandRaster.ts
Normal file
38
platform/browser/magicWandRaster.ts
Normal file
@@ -0,0 +1,38 @@
|
||||
import { blurMaskValues, despeckleMaskValues, dilateMaskValues, erodeMaskValues, maskValueFromRgba } from "./maskRaster";
|
||||
|
||||
export type MagicWandRasterSettings = { tolerance: number; feather: number; choke: number; despeckle: number; contiguous: boolean; mode: "replace" | "add" | "subtract" };
|
||||
|
||||
export async function createWandMask(source: string, existingMaskSource: string | undefined, width: number, height: number, startX: number, startY: number, settings: MagicWandRasterSettings) {
|
||||
const canvas = document.createElement("canvas");
|
||||
canvas.width = Math.max(1, width); canvas.height = Math.max(1, height);
|
||||
const context = canvas.getContext("2d");
|
||||
if (!context) return source;
|
||||
context.drawImage(await loadImage(source), 0, 0, canvas.width, canvas.height);
|
||||
const data = context.getImageData(0, 0, canvas.width, canvas.height);
|
||||
const start = (startY * canvas.width + startX) * 4;
|
||||
const key = [data.data[start] ?? 0, data.data[start + 1] ?? 0, data.data[start + 2] ?? 0];
|
||||
const selected = settings.contiguous ? floodSelect(data, width, height, startX, startY, key, settings.tolerance) : globalSelect(data, key, settings.tolerance);
|
||||
let values: Uint8ClampedArray<ArrayBufferLike> = toValues(selected);
|
||||
if (settings.despeckle > 0) values = despeckleMaskValues(values, width, height, Math.round(settings.despeckle));
|
||||
if (settings.choke > 0) values = erodeMaskValues(values, width, height, Math.round(settings.choke));
|
||||
if (settings.choke < 0) values = dilateMaskValues(values, width, height, Math.round(-settings.choke));
|
||||
if (settings.feather > 0) values = blurMaskValues(values, width, height, Math.round(settings.feather));
|
||||
const existing = existingMaskSource ? await loadMask(existingMaskSource, width, height) : undefined;
|
||||
for (let pixel = 0; pixel < values.length; pixel++) {
|
||||
const current = existing?.[pixel] ?? 255; const selectedValue = values[pixel] ?? 0;
|
||||
const alpha = settings.mode === "add" ? Math.min(current, 255 - selectedValue) : settings.mode === "subtract" ? Math.max(current, selectedValue) : 255 - selectedValue;
|
||||
const index = pixel * 4; data.data[index] = data.data[index + 1] = data.data[index + 2] = 255; data.data[index + 3] = alpha;
|
||||
}
|
||||
context.putImageData(data, 0, 0); return canvas.toDataURL("image/png");
|
||||
}
|
||||
|
||||
function floodSelect(data: ImageData, width: number, height: number, x: number, y: number, key: number[], tolerance: number) {
|
||||
const result = new Uint8Array(width * height); const queue: Array<[number, number]> = [[x, y]];
|
||||
while (queue.length) { const [px, py] = queue.pop()!; if (px < 0 || py < 0 || px >= width || py >= height) continue; const i = py * width + px; if (result[i] || !matches(data, i, key, tolerance)) continue; result[i] = 1; queue.push([px + 1, py], [px - 1, py], [px, py + 1], [px, py - 1]); }
|
||||
return result;
|
||||
}
|
||||
function globalSelect(data: ImageData, key: number[], tolerance: number) { const result = new Uint8Array(data.width * data.height); for (let i = 0; i < result.length; i++) if (matches(data, i, key, tolerance)) result[i] = 1; return result; }
|
||||
function matches(data: ImageData, pixel: number, key: number[], tolerance: number) { const i = pixel * 4; return Math.hypot((data.data[i] ?? 0) - (key[0] ?? 0), (data.data[i + 1] ?? 0) - (key[1] ?? 0), (data.data[i + 2] ?? 0) - (key[2] ?? 0)) <= tolerance; }
|
||||
function toValues(selected: Uint8Array) { const values = new Uint8ClampedArray(selected.length); for (let i = 0; i < selected.length; i++) values[i] = selected[i] ? 255 : 0; return values; }
|
||||
async function loadMask(source: string, width: number, height: number) { const canvas = document.createElement("canvas"); canvas.width = width; canvas.height = height; const context = canvas.getContext("2d"); if (!context) return undefined; context.drawImage(await loadImage(source), 0, 0, width, height); const data = context.getImageData(0, 0, width, height); const values = new Uint8ClampedArray(width * height); for (let i = 0; i < values.length; i++) values[i] = maskValueFromRgba(data.data, i * 4); return values; }
|
||||
function loadImage(source: string) { return new Promise<HTMLImageElement>((resolve, reject) => { const image = new Image(); image.onload = () => resolve(image); image.onerror = () => reject(new Error("Failed to load image")); image.src = source; }); }
|
||||
40
platform/browser/maskRaster.test.ts
Normal file
40
platform/browser/maskRaster.test.ts
Normal file
@@ -0,0 +1,40 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import { cropMaskValuesToRgba, expandRectWithinBounds, invertMaskValues } from "./maskRaster";
|
||||
|
||||
describe("mask raster utilities", () => {
|
||||
test("exports the normalized drawn mask without filling the whole crop", () => {
|
||||
const revealedMask = new Uint8ClampedArray(4 * 3).fill(255);
|
||||
revealedMask[1 * 4 + 2] = 0;
|
||||
|
||||
const inpaintMask = invertMaskValues(revealedMask);
|
||||
const rgba = cropMaskValuesToRgba(inpaintMask, 4, 3, { x: 1, y: 0, w: 3, h: 3 }, 4, 4);
|
||||
const activePixels = activeRedPixels(rgba);
|
||||
|
||||
expect(activePixels).toEqual([{ x: 1, y: 1 }]);
|
||||
for (let pixel = 0; pixel < rgba.length / 4; pixel += 1) expect(rgba[pixel * 4 + 3]).toBe(255);
|
||||
});
|
||||
|
||||
test("expands masked-area crops to the model minimum when possible", () => {
|
||||
expect(expandRectWithinBounds({ x: 20, y: 20, w: 4, h: 4 }, 4, { w: 100, h: 100 }, 8, 64)).toEqual({
|
||||
x: 0,
|
||||
y: 0,
|
||||
w: 64,
|
||||
h: 64,
|
||||
});
|
||||
expect(expandRectWithinBounds({ x: 80, y: 80, w: 4, h: 4 }, 4, { w: 100, h: 100 }, 8, 64)).toEqual({
|
||||
x: 36,
|
||||
y: 36,
|
||||
w: 64,
|
||||
h: 64,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
function activeRedPixels(rgba: Uint8ClampedArray) {
|
||||
const width = 4;
|
||||
const pixels: Array<{ x: number; y: number }> = [];
|
||||
for (let pixel = 0; pixel < rgba.length / 4; pixel += 1) {
|
||||
if ((rgba[pixel * 4] ?? 0) > 127) pixels.push({ x: pixel % width, y: Math.floor(pixel / width) });
|
||||
}
|
||||
return pixels;
|
||||
}
|
||||
374
platform/browser/maskRaster.ts
Normal file
374
platform/browser/maskRaster.ts
Normal file
@@ -0,0 +1,374 @@
|
||||
import type { Rect } from "@core/geometry";
|
||||
|
||||
export type MaskFill = "white" | "black" | "clear";
|
||||
|
||||
export type MaskRasterOperation =
|
||||
| { type: "invert" }
|
||||
| { type: "fill"; fill: MaskFill }
|
||||
| { type: "feather"; radius: number }
|
||||
| { type: "expand"; radius: number }
|
||||
| { type: "contract"; radius: number }
|
||||
| { type: "blur"; radius: number }
|
||||
| { type: "despeckle"; strength: number };
|
||||
|
||||
export type MaskAnalysis = {
|
||||
width: number;
|
||||
height: number;
|
||||
revealedPixels: number;
|
||||
hiddenPixels: number;
|
||||
coverage: number;
|
||||
hiddenCoverage: number;
|
||||
bounds?: Rect;
|
||||
hiddenBounds?: Rect;
|
||||
thumbnail: string;
|
||||
};
|
||||
|
||||
export type NormalizedMaskOptions = {
|
||||
polarity: "hidden" | "revealed";
|
||||
expand?: number;
|
||||
feather?: number;
|
||||
blur?: number;
|
||||
despeckle?: number;
|
||||
};
|
||||
|
||||
export async function createSolidMaskSource(width: number, height: number, fill: MaskFill): Promise<string> {
|
||||
const canvas = createCanvas(width, height);
|
||||
const context = require2dContext(canvas);
|
||||
context.clearRect(0, 0, canvas.width, canvas.height);
|
||||
|
||||
if (fill === "white") {
|
||||
context.fillStyle = "#ffffff";
|
||||
context.fillRect(0, 0, canvas.width, canvas.height);
|
||||
} else if (fill === "black") {
|
||||
context.fillStyle = "#000000";
|
||||
context.fillRect(0, 0, canvas.width, canvas.height);
|
||||
}
|
||||
|
||||
return canvas.toDataURL("image/png");
|
||||
}
|
||||
|
||||
export async function applyMaskRasterOperation(source: string, width: number, height: number, operation: MaskRasterOperation): Promise<string> {
|
||||
if (operation.type === "fill") return createSolidMaskSource(width, height, operation.fill);
|
||||
|
||||
const mask = await loadMaskValues(source, width, height);
|
||||
const next = applyMaskValueOperation(mask.values, mask.width, mask.height, operation);
|
||||
return maskValuesToDataUrl(next, mask.width, mask.height);
|
||||
}
|
||||
|
||||
export async function analyzeMaskSource(source: string, width: number, height: number): Promise<MaskAnalysis> {
|
||||
const mask = await loadMaskValues(source, width, height);
|
||||
const revealed = analyzeValues(mask.values, mask.width, mask.height, false);
|
||||
const hiddenValues = invertMaskValues(mask.values);
|
||||
const hidden = analyzeValues(hiddenValues, mask.width, mask.height, false);
|
||||
return {
|
||||
width: mask.width,
|
||||
height: mask.height,
|
||||
revealedPixels: revealed.pixels,
|
||||
hiddenPixels: hidden.pixels,
|
||||
coverage: revealed.coverage,
|
||||
hiddenCoverage: hidden.coverage,
|
||||
bounds: revealed.bounds,
|
||||
hiddenBounds: hidden.bounds,
|
||||
thumbnail: maskValuesToDataUrl(mask.values, mask.width, mask.height, 72, 48),
|
||||
};
|
||||
}
|
||||
|
||||
export async function createNormalizedMaskSource(source: string, width: number, height: number, options: NormalizedMaskOptions): Promise<{ source: string; values: Uint8ClampedArray; bounds?: Rect }> {
|
||||
const mask = await loadMaskValues(source, width, height);
|
||||
let values = options.polarity === "hidden" ? invertMaskValues(mask.values) : new Uint8ClampedArray(mask.values);
|
||||
|
||||
const despeckle = Math.round(clampNumber(options.despeckle ?? 0, 0, 64));
|
||||
const expand = Math.round(clampNumber(options.expand ?? 0, -256, 256));
|
||||
const feather = Math.round(clampNumber(options.feather ?? 0, 0, 256));
|
||||
const blur = Math.round(clampNumber(options.blur ?? 0, 0, 256));
|
||||
|
||||
if (despeckle > 0) values = despeckleMaskValues(values, mask.width, mask.height, despeckle);
|
||||
if (expand > 0) values = dilateMaskValues(values, mask.width, mask.height, expand);
|
||||
if (expand < 0) values = erodeMaskValues(values, mask.width, mask.height, -expand);
|
||||
if (feather > 0) values = blurMaskValues(values, mask.width, mask.height, feather);
|
||||
if (blur > 0) values = blurMaskValues(values, mask.width, mask.height, blur);
|
||||
|
||||
const analysis = analyzeValues(values, mask.width, mask.height, false);
|
||||
return { source: maskValuesToDataUrl(values, mask.width, mask.height), values, bounds: analysis.bounds };
|
||||
}
|
||||
|
||||
export async function loadImageCanvas(source: string, width?: number, height?: number): Promise<HTMLCanvasElement> {
|
||||
const image = await loadImage(source);
|
||||
const canvas = createCanvas(width ?? image.naturalWidth, height ?? image.naturalHeight);
|
||||
const context = require2dContext(canvas);
|
||||
context.clearRect(0, 0, canvas.width, canvas.height);
|
||||
context.drawImage(image, 0, 0, canvas.width, canvas.height);
|
||||
return canvas;
|
||||
}
|
||||
|
||||
export async function imageSourceToPngDataUrl(source: string): Promise<string> {
|
||||
if (source.startsWith("data:image/png;base64,")) return source;
|
||||
const canvas = await loadImageCanvas(source);
|
||||
return canvas.toDataURL("image/png");
|
||||
}
|
||||
|
||||
export function cropCanvas(sourceCanvas: HTMLCanvasElement, crop: Rect, outputWidth = crop.w, outputHeight = crop.h): string {
|
||||
const canvas = createCanvas(outputWidth, outputHeight);
|
||||
const context = require2dContext(canvas);
|
||||
context.clearRect(0, 0, outputWidth, outputHeight);
|
||||
context.drawImage(sourceCanvas, crop.x, crop.y, crop.w, crop.h, 0, 0, crop.w, crop.h);
|
||||
return canvas.toDataURL("image/png");
|
||||
}
|
||||
|
||||
export function cropMaskValuesToDataUrl(values: Uint8ClampedArray, width: number, height: number, crop: Rect, outputWidth = crop.w, outputHeight = crop.h): string {
|
||||
const canvas = createCanvas(outputWidth, outputHeight);
|
||||
const context = require2dContext(canvas);
|
||||
const imageData = context.createImageData(outputWidth, outputHeight);
|
||||
imageData.data.set(cropMaskValuesToRgba(values, width, height, crop, outputWidth, outputHeight));
|
||||
context.putImageData(imageData, 0, 0);
|
||||
return canvas.toDataURL("image/png");
|
||||
}
|
||||
|
||||
export function cropMaskValuesToRgba(values: Uint8ClampedArray, width: number, height: number, crop: Rect, outputWidth = crop.w, outputHeight = crop.h): Uint8ClampedArray {
|
||||
const safeOutputWidth = Math.max(1, Math.round(outputWidth));
|
||||
const safeOutputHeight = Math.max(1, Math.round(outputHeight));
|
||||
const data = new Uint8ClampedArray(safeOutputWidth * safeOutputHeight * 4);
|
||||
for (let pixel = 0; pixel < safeOutputWidth * safeOutputHeight; pixel += 1) data[pixel * 4 + 3] = 255;
|
||||
|
||||
for (let y = 0; y < Math.min(crop.h, safeOutputHeight); y += 1) {
|
||||
for (let x = 0; x < Math.min(crop.w, safeOutputWidth); x += 1) {
|
||||
const sourceX = crop.x + x;
|
||||
const sourceY = crop.y + y;
|
||||
if (sourceX < 0 || sourceY < 0 || sourceX >= width || sourceY >= height) continue;
|
||||
const value = values[sourceY * width + sourceX] ?? 0;
|
||||
const index = (y * safeOutputWidth + x) * 4;
|
||||
data[index] = value;
|
||||
data[index + 1] = value;
|
||||
data[index + 2] = value;
|
||||
}
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
export function expandRectWithinBounds(rect: Rect, padding: number, bounds: { w: number; h: number }, multiple = 1, minSize = 1): Rect {
|
||||
const padded = Math.max(0, Math.round(padding));
|
||||
let x1 = Math.max(0, Math.floor(rect.x) - padded);
|
||||
let y1 = Math.max(0, Math.floor(rect.y) - padded);
|
||||
let x2 = Math.min(bounds.w, Math.ceil(rect.x + rect.w) + padded);
|
||||
let y2 = Math.min(bounds.h, Math.ceil(rect.y + rect.h) + padded);
|
||||
|
||||
const safeMinSize = Math.max(1, Math.round(minSize));
|
||||
const targetWidth = Math.min(bounds.w, roundUp(Math.max(safeMinSize, x2 - x1), multiple));
|
||||
const targetHeight = Math.min(bounds.h, roundUp(Math.max(safeMinSize, y2 - y1), multiple));
|
||||
|
||||
const extraWidth = targetWidth - (x2 - x1);
|
||||
const extraHeight = targetHeight - (y2 - y1);
|
||||
x1 = Math.max(0, x1 - Math.floor(extraWidth / 2));
|
||||
y1 = Math.max(0, y1 - Math.floor(extraHeight / 2));
|
||||
x2 = Math.min(bounds.w, x1 + targetWidth);
|
||||
y2 = Math.min(bounds.h, y1 + targetHeight);
|
||||
x1 = Math.max(0, x2 - targetWidth);
|
||||
y1 = Math.max(0, y2 - targetHeight);
|
||||
|
||||
return { x: x1, y: y1, w: Math.max(1, x2 - x1), h: Math.max(1, y2 - y1) };
|
||||
}
|
||||
|
||||
export function maskValueFromRgba(data: Uint8ClampedArray, index: number): number {
|
||||
const red = data[index] ?? 0;
|
||||
const green = data[index + 1] ?? 0;
|
||||
const blue = data[index + 2] ?? 0;
|
||||
const alpha = data[index + 3] ?? 0;
|
||||
const luminance = 0.2126 * red + 0.7152 * green + 0.0722 * blue;
|
||||
return Math.round((alpha * luminance) / 255);
|
||||
}
|
||||
|
||||
export function applyMaskValueOperation(values: Uint8ClampedArray, width: number, height: number, operation: Exclude<MaskRasterOperation, { type: "fill" }>): Uint8ClampedArray {
|
||||
switch (operation.type) {
|
||||
case "invert":
|
||||
return invertMaskValues(values);
|
||||
case "feather":
|
||||
case "blur":
|
||||
return blurMaskValues(values, width, height, Math.round(clampNumber(operation.radius, 0, 256)));
|
||||
case "expand":
|
||||
return dilateMaskValues(values, width, height, Math.round(clampNumber(operation.radius, 0, 256)));
|
||||
case "contract":
|
||||
return erodeMaskValues(values, width, height, Math.round(clampNumber(operation.radius, 0, 256)));
|
||||
case "despeckle":
|
||||
return despeckleMaskValues(values, width, height, Math.round(clampNumber(operation.strength, 0, 64)));
|
||||
}
|
||||
}
|
||||
|
||||
export function invertMaskValues(values: Uint8ClampedArray): Uint8ClampedArray {
|
||||
const next = new Uint8ClampedArray(values.length);
|
||||
for (let index = 0; index < values.length; index += 1) next[index] = 255 - (values[index] ?? 0);
|
||||
return next;
|
||||
}
|
||||
|
||||
export function erodeMaskValues(values: Uint8ClampedArray, width: number, height: number, radius: number): Uint8ClampedArray {
|
||||
const safeRadius = Math.round(clampNumber(radius, 0, 256));
|
||||
if (safeRadius <= 0) return new Uint8ClampedArray(values);
|
||||
const next = new Uint8ClampedArray(values.length);
|
||||
for (let y = 0; y < height; y += 1) {
|
||||
for (let x = 0; x < width; x += 1) {
|
||||
let value = 255;
|
||||
for (let oy = -safeRadius; oy <= safeRadius; oy += 1) {
|
||||
for (let ox = -safeRadius; ox <= safeRadius; ox += 1) value = Math.min(value, values[clampInt(y + oy, 0, height - 1) * width + clampInt(x + ox, 0, width - 1)] ?? 0);
|
||||
}
|
||||
next[y * width + x] = value;
|
||||
}
|
||||
}
|
||||
return next;
|
||||
}
|
||||
|
||||
export function dilateMaskValues(values: Uint8ClampedArray, width: number, height: number, radius: number): Uint8ClampedArray {
|
||||
const safeRadius = Math.round(clampNumber(radius, 0, 256));
|
||||
if (safeRadius <= 0) return new Uint8ClampedArray(values);
|
||||
const next = new Uint8ClampedArray(values.length);
|
||||
for (let y = 0; y < height; y += 1) {
|
||||
for (let x = 0; x < width; x += 1) {
|
||||
let value = 0;
|
||||
for (let oy = -safeRadius; oy <= safeRadius; oy += 1) {
|
||||
for (let ox = -safeRadius; ox <= safeRadius; ox += 1) value = Math.max(value, values[clampInt(y + oy, 0, height - 1) * width + clampInt(x + ox, 0, width - 1)] ?? 0);
|
||||
}
|
||||
next[y * width + x] = value;
|
||||
}
|
||||
}
|
||||
return next;
|
||||
}
|
||||
|
||||
export function blurMaskValues(values: Uint8ClampedArray, width: number, height: number, radius: number): Uint8ClampedArray {
|
||||
const safeRadius = Math.round(clampNumber(radius, 0, 256));
|
||||
if (safeRadius <= 0) return new Uint8ClampedArray(values);
|
||||
const next = new Uint8ClampedArray(values.length);
|
||||
for (let y = 0; y < height; y += 1) {
|
||||
for (let x = 0; x < width; x += 1) {
|
||||
let total = 0;
|
||||
let count = 0;
|
||||
for (let oy = -safeRadius; oy <= safeRadius; oy += 1) {
|
||||
for (let ox = -safeRadius; ox <= safeRadius; ox += 1) {
|
||||
total += values[clampInt(y + oy, 0, height - 1) * width + clampInt(x + ox, 0, width - 1)] ?? 0;
|
||||
count += 1;
|
||||
}
|
||||
}
|
||||
next[y * width + x] = Math.round(total / count);
|
||||
}
|
||||
}
|
||||
return next;
|
||||
}
|
||||
|
||||
export function despeckleMaskValues(values: Uint8ClampedArray, width: number, height: number, strength: number): Uint8ClampedArray {
|
||||
const safeStrength = Math.round(clampNumber(strength, 0, 64));
|
||||
if (safeStrength <= 0) return new Uint8ClampedArray(values);
|
||||
|
||||
const radius = Math.max(1, Math.ceil(safeStrength / 6));
|
||||
const threshold = Math.max(1, Math.round(safeStrength / 2));
|
||||
const next = new Uint8ClampedArray(values);
|
||||
|
||||
for (let y = 0; y < height; y += 1) {
|
||||
for (let x = 0; x < width; x += 1) {
|
||||
const index = y * width + x;
|
||||
const visible = (values[index] ?? 0) > 127;
|
||||
let same = 0;
|
||||
for (let oy = -radius; oy <= radius; oy += 1) {
|
||||
for (let ox = -radius; ox <= radius; ox += 1) {
|
||||
if (ox === 0 && oy === 0) continue;
|
||||
const sample = values[clampInt(y + oy, 0, height - 1) * width + clampInt(x + ox, 0, width - 1)] ?? 0;
|
||||
if ((sample > 127) === visible) same += 1;
|
||||
}
|
||||
}
|
||||
if (same <= threshold) next[index] = visible ? 0 : 255;
|
||||
}
|
||||
}
|
||||
|
||||
return next;
|
||||
}
|
||||
|
||||
async function loadMaskValues(source: string, width: number, height: number): Promise<{ width: number; height: number; values: Uint8ClampedArray }> {
|
||||
const canvas = await loadImageCanvas(source, width, height);
|
||||
const context = require2dContext(canvas);
|
||||
const data = context.getImageData(0, 0, canvas.width, canvas.height);
|
||||
const values = new Uint8ClampedArray(canvas.width * canvas.height);
|
||||
for (let pixel = 0; pixel < values.length; pixel += 1) values[pixel] = maskValueFromRgba(data.data, pixel * 4);
|
||||
return { width: canvas.width, height: canvas.height, values };
|
||||
}
|
||||
|
||||
function maskValuesToDataUrl(values: Uint8ClampedArray, width: number, height: number, outputWidth = width, outputHeight = height): string {
|
||||
const canvas = createCanvas(outputWidth, outputHeight);
|
||||
const context = require2dContext(canvas);
|
||||
const imageData = context.createImageData(outputWidth, outputHeight);
|
||||
|
||||
for (let y = 0; y < outputHeight; y += 1) {
|
||||
for (let x = 0; x < outputWidth; x += 1) {
|
||||
const sourceX = Math.floor((x / outputWidth) * width);
|
||||
const sourceY = Math.floor((y / outputHeight) * height);
|
||||
const value = values[clampInt(sourceY, 0, height - 1) * width + clampInt(sourceX, 0, width - 1)] ?? 0;
|
||||
const index = (y * outputWidth + x) * 4;
|
||||
imageData.data[index] = 255;
|
||||
imageData.data[index + 1] = 255;
|
||||
imageData.data[index + 2] = 255;
|
||||
imageData.data[index + 3] = value;
|
||||
}
|
||||
}
|
||||
|
||||
context.putImageData(imageData, 0, 0);
|
||||
return canvas.toDataURL("image/png");
|
||||
}
|
||||
|
||||
function analyzeValues(values: Uint8ClampedArray, width: number, height: number, includeSoftPixels: boolean): { pixels: number; coverage: number; bounds?: Rect } {
|
||||
let pixels = 0;
|
||||
let minX = Number.POSITIVE_INFINITY;
|
||||
let minY = Number.POSITIVE_INFINITY;
|
||||
let maxX = Number.NEGATIVE_INFINITY;
|
||||
let maxY = Number.NEGATIVE_INFINITY;
|
||||
|
||||
for (let y = 0; y < height; y += 1) {
|
||||
for (let x = 0; x < width; x += 1) {
|
||||
const value = values[y * width + x] ?? 0;
|
||||
const active = includeSoftPixels ? value > 0 : value > 127;
|
||||
if (!active) continue;
|
||||
pixels += 1;
|
||||
minX = Math.min(minX, x);
|
||||
minY = Math.min(minY, y);
|
||||
maxX = Math.max(maxX, x + 1);
|
||||
maxY = Math.max(maxY, y + 1);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
pixels,
|
||||
coverage: pixels / Math.max(1, width * height),
|
||||
bounds: pixels > 0 ? { x: minX, y: minY, w: maxX - minX, h: maxY - minY } : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
function createCanvas(width: number, height: number): HTMLCanvasElement {
|
||||
const canvas = document.createElement("canvas");
|
||||
canvas.width = Math.max(1, Math.round(width));
|
||||
canvas.height = Math.max(1, Math.round(height));
|
||||
return canvas;
|
||||
}
|
||||
|
||||
function require2dContext(canvas: HTMLCanvasElement): CanvasRenderingContext2D {
|
||||
const context = canvas.getContext("2d");
|
||||
if (!context) throw new Error("Unable to create mask canvas");
|
||||
return context;
|
||||
}
|
||||
|
||||
function loadImage(source: string): Promise<HTMLImageElement> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const image = new Image();
|
||||
image.onload = () => resolve(image);
|
||||
image.onerror = () => reject(new Error("Failed to load image"));
|
||||
image.src = source;
|
||||
});
|
||||
}
|
||||
|
||||
function roundUp(value: number, multiple: number): number {
|
||||
const safeMultiple = Math.max(1, Math.round(multiple));
|
||||
return Math.ceil(value / safeMultiple) * safeMultiple;
|
||||
}
|
||||
|
||||
function clampNumber(value: number, min: number, max: number): number {
|
||||
if (!Number.isFinite(value)) return min;
|
||||
return Math.max(min, Math.min(max, value));
|
||||
}
|
||||
|
||||
function clampInt(value: number, min: number, max: number): number {
|
||||
return Math.round(Math.max(min, Math.min(max, value)));
|
||||
}
|
||||
11
platform/comfy/generationClient.ts
Normal file
11
platform/comfy/generationClient.ts
Normal file
@@ -0,0 +1,11 @@
|
||||
export async function fetchGenerationOptions(): Promise<unknown> {
|
||||
const response = await fetch("/api/comfy/models");
|
||||
if (!response.ok) throw new Error("Unable to load ComfyUI models");
|
||||
return response.json() as Promise<unknown>;
|
||||
}
|
||||
|
||||
export async function requestGeneration(body: unknown): Promise<{ source: string; mimeType: string }> {
|
||||
const response = await fetch("/api/comfy/generate", { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(body) });
|
||||
if (!response.ok) throw new Error(await response.text());
|
||||
return response.json() as Promise<{ source: string; mimeType: string }>;
|
||||
}
|
||||
Reference in New Issue
Block a user