From 7188569672a12bfb7d799a45a4bfa0e0cc1a50ba Mon Sep 17 00:00:00 2001 From: syntaxbullet Date: Sat, 4 Jul 2026 15:10:30 +0200 Subject: [PATCH] feat: add ComfyUI integration for image generation - Implemented ComfyUI API for generating images with various modes (text-to-image, image-to-image, inpaint, outpaint). - Created GenerateSheet and associated controls for user input on generation settings. - Added subtle scrollbar styles for improved UI experience. - Enhanced canvas input handling to ignore key events when focused on editable elements. - Optimized canvas resizing logic to prevent unnecessary dispatches. - Introduced error handling for generation failures and loading models. - Added functionality to upload images and masks for inpainting. --- app/comfy.ts | 164 +++++++++ commands/ids.ts | 1 + commands/index.ts | 4 +- commands/payloads.ts | 3 +- commands/tool.test.ts | 5 +- commands/tool.ts | 43 ++- editor/tools.ts | 30 +- index.ts | 3 + input/tool-keybinds.ts | 1 + input/transform-controls.ts | 2 +- view/App.tsx | 58 ++- view/BottomControlsIsland.tsx | 12 +- view/GenerateSheet.tsx | 32 ++ view/ShortcutsDisplay.tsx | 1 + view/ToolOverlay.tsx | 4 +- .../GenerateActionControls.tsx | 40 +++ view/bottom-controls/GenerateControls.tsx | 334 ++++++++++++++++++ view/bottom-controls/SelectMenu.tsx | 130 +++++-- view/canvas/useCanvasInput.ts | 11 + view/canvas/useCanvasResize.ts | 19 +- view/generate/runGenerate.ts | 111 ++++++ view/index.css | 23 ++ view/toolLabels.ts | 2 + 23 files changed, 981 insertions(+), 52 deletions(-) create mode 100644 app/comfy.ts create mode 100644 view/GenerateSheet.tsx create mode 100644 view/bottom-controls/GenerateActionControls.tsx create mode 100644 view/bottom-controls/GenerateControls.tsx create mode 100644 view/generate/runGenerate.ts diff --git a/app/comfy.ts b/app/comfy.ts new file mode 100644 index 0000000..3b3bdae --- /dev/null +++ b/app/comfy.ts @@ -0,0 +1,164 @@ +type GenerateMode = "text-to-image" | "image-to-image" | "inpaint" | "outpaint"; + +type ComfyGenerateRequest = { + mode: GenerateMode; + model: string; + prompt: string; + negativePrompt?: string; + strength?: number; + steps?: number; + cfg?: number; + seed?: number; + sampler?: string; + scheduler?: string; + width?: number; + height?: number; + outpaint?: { + left?: number; + top?: number; + right?: number; + bottom?: number; + feathering?: number; + }; + inputImage?: string; + maskImage?: string; +}; + +const comfyBaseUrl = process.env.COMFYUI_URL ?? "http://127.0.0.1:8188"; + +export async function handleComfyApi(request: Request) { + const url = new URL(request.url); + if (url.pathname === "/api/comfy/models" && request.method === "GET") return json(await listGenerationOptions()); + if (url.pathname === "/api/comfy/generate" && request.method === "POST") return json(await generate(await request.json() as ComfyGenerateRequest)); + return new Response("Not found", { status: 404 }); +} + +async function listGenerationOptions() { + const response = await fetch(`${comfyBaseUrl}/object_info`); + if (!response.ok) throw new Error(`ComfyUI option lookup failed: ${response.status}`); + const info = await response.json() as { + CheckpointLoaderSimple?: { input?: { required?: { ckpt_name?: [string[]] } } }; + KSampler?: { input?: { required?: { sampler_name?: [string[]]; scheduler?: [string[]] } } }; + }; + return { + models: info.CheckpointLoaderSimple?.input?.required?.ckpt_name?.[0] ?? [], + samplers: info.KSampler?.input?.required?.sampler_name?.[0] ?? [], + schedulers: info.KSampler?.input?.required?.scheduler?.[0] ?? [], + }; +} + +async function listCheckpointModels() { + return (await listGenerationOptions()).models; +} + +async function generate(request: ComfyGenerateRequest) { + if (!request.prompt?.trim()) throw new Error("Prompt is required"); + if (!request.model || request.model === "auto") { + const models = await listCheckpointModels(); + request.model = models[0] ?? "sd_xl_base_1.0.safetensors"; + } + + const clientId = crypto.randomUUID(); + const uploaded = request.inputImage ? await uploadDataUrl(request.inputImage, `image-studio-${crypto.randomUUID()}.png`) : undefined; + const mask = request.maskImage ? await uploadDataUrl(request.maskImage, `image-studio-mask-${crypto.randomUUID()}.png`) : undefined; + const prompt = buildSdxlWorkflow({ ...request, inputImage: uploaded, maskImage: mask }); + + const queued = await fetch(`${comfyBaseUrl}/prompt`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ client_id: clientId, prompt }), + }); + if (!queued.ok) throw new Error(`ComfyUI prompt failed: ${queued.status} ${await queued.text()}`); + const { prompt_id } = await queued.json() as { prompt_id: string }; + const history = await waitForHistory(prompt_id); + const image = firstOutputImage(history); + if (!image) throw new Error("ComfyUI did not return an image"); + + const imageResponse = await fetch(`${comfyBaseUrl}/view?${new URLSearchParams({ filename: image.filename, subfolder: image.subfolder ?? "", type: image.type ?? "output" })}`); + if (!imageResponse.ok) throw new Error(`ComfyUI image fetch failed: ${imageResponse.status}`); + const bytes = Buffer.from(await imageResponse.arrayBuffer()); + return { source: `data:image/png;base64,${bytes.toString("base64")}`, mimeType: "image/png" }; +} + +async function uploadDataUrl(dataUrl: string, filename: string) { + const match = /^data:([^;]+);base64,(.+)$/.exec(dataUrl); + if (!match) return undefined; + const mimeType = match[1] ?? "image/png"; + const base64 = match[2] ?? ""; + const form = new FormData(); + form.append("image", new File([new Uint8Array(Buffer.from(base64, "base64"))], filename, { type: mimeType })); + form.append("overwrite", "true"); + const response = await fetch(`${comfyBaseUrl}/upload/image`, { method: "POST", body: form }); + if (!response.ok) throw new Error(`ComfyUI upload failed: ${response.status}`); + const uploaded = await response.json() as { name: string }; + return uploaded.name; +} + +async function waitForHistory(promptId: string) { + for (let attempt = 0; attempt < 240; attempt++) { + const response = await fetch(`${comfyBaseUrl}/history/${promptId}`); + if (response.ok) { + const history = await response.json() as Record; + if (history[promptId]) return history[promptId]; + } + await Bun.sleep(500); + } + throw new Error("Timed out waiting for ComfyUI"); +} + +function firstOutputImage(history: unknown): { filename: string; subfolder?: string; type?: string } | undefined { + const outputs = (history as { outputs?: Record }).outputs ?? {}; + for (const output of Object.values(outputs)) { + const image = output.images?.[0]; + if (image) return image; + } + return undefined; +} + +function buildSdxlWorkflow(request: ComfyGenerateRequest) { + if (!request.width || !request.height) throw new Error("Generation width and height are required"); + const width = Math.max(64, Math.round(request.width)); + const height = Math.max(64, Math.round(request.height)); + const denoise = Math.max(0, Math.min(1, (request.strength ?? 75) / 100)); + const seed = request.seed === undefined || request.seed < 0 ? Math.floor(Math.random() * 2 ** 32) : Math.round(request.seed); + const steps = Math.max(1, Math.round(request.steps ?? 30)); + const cfg = Math.max(0, request.cfg ?? 7); + const sampler = request.sampler ?? "euler"; + const scheduler = request.scheduler ?? "normal"; + const positive = request.prompt; + const negative = request.negativePrompt ?? ""; + const workflow: Record = { + "1": { class_type: "CheckpointLoaderSimple", inputs: { ckpt_name: request.model } }, + "2": { class_type: "CLIPTextEncode", inputs: { text: positive, clip: ["1", 1] } }, + "3": { class_type: "CLIPTextEncode", inputs: { text: negative, clip: ["1", 1] } }, + "6": { class_type: "KSampler", inputs: { seed, steps, cfg, sampler_name: sampler, scheduler, denoise, model: ["1", 0], positive: ["2", 0], negative: ["3", 0], latent_image: ["5", 0] } }, + "7": { class_type: "VAEDecode", inputs: { samples: ["6", 0], vae: ["1", 2] } }, + "8": { class_type: "SaveImage", inputs: { filename_prefix: `image-studio-${request.mode}`, images: ["7", 0] } }, + }; + + if (request.mode === "text-to-image" || !request.inputImage) { + workflow["5"] = { class_type: "EmptyLatentImage", inputs: { width, height, batch_size: 1 } }; + return workflow; + } + + workflow["4"] = { class_type: "LoadImage", inputs: { image: request.inputImage } }; + + if (request.mode === "inpaint" && request.maskImage) { + workflow["9"] = { class_type: "LoadImage", inputs: { image: request.maskImage } }; + workflow["5"] = { class_type: "VAEEncodeForInpaint", inputs: { pixels: ["4", 0], vae: ["1", 2], mask: ["9", 1], grow_mask_by: 6 } }; + return workflow; + } + + if (request.mode === "outpaint") { + workflow["10"] = { class_type: "ImagePadForOutpaint", inputs: { image: ["4", 0], left: Math.round(request.outpaint?.left ?? 0), top: Math.round(request.outpaint?.top ?? 0), right: Math.round(request.outpaint?.right ?? 0), bottom: Math.round(request.outpaint?.bottom ?? 0), feathering: Math.round(request.outpaint?.feathering ?? 0) } }; + workflow["5"] = { class_type: "VAEEncodeForInpaint", inputs: { pixels: ["10", 0], vae: ["1", 2], mask: ["10", 1], grow_mask_by: 6 } }; + return workflow; + } + + workflow["5"] = { class_type: "VAEEncode", inputs: { pixels: ["4", 0], vae: ["1", 2] } }; + return workflow; +} + +function json(value: unknown) { + return new Response(JSON.stringify(value), { headers: { "content-type": "application/json" } }); +} diff --git a/commands/ids.ts b/commands/ids.ts index a7cb1c0..22fbd01 100644 --- a/commands/ids.ts +++ b/commands/ids.ts @@ -24,6 +24,7 @@ export const commandIds = { selectionClear: "selection.clear", selectionAddLayer: "selection.addLayer", toolSetActive: "tool.setActive", + toolSetGenerateSettings: "tool.setGenerateSettings", toolSetBrushSettings: "tool.setBrushSettings", toolSetChromaKeySettings: "tool.setChromaKeySettings", toolSetMagicWandSettings: "tool.setMagicWandSettings", diff --git a/commands/index.ts b/commands/index.ts index 2ddfd1c..fcd27f1 100644 --- a/commands/index.ts +++ b/commands/index.ts @@ -54,10 +54,10 @@ export type { CommandRegistry } from "./registry"; export { createCommandRegistry } from "./registry"; export { selectionAddLayerCommand, selectionClearCommand, selectionCommands, selectionSetCommand } from "./selection"; export type { SelectionAddLayerPayload, SelectionSetPayload } from "./selection"; -export { toolCommands, toolEnterMaskEditCommand, toolEnterTemporaryPanCommand, toolExitMaskEditCommand, toolExitTemporaryPanCommand, toolSetActiveCommand, toolSetBrushPreviewCommand, toolSetBrushSettingsCommand, toolSetBrushStrokePreviewCommand, toolSetChromaKeySettingsCommand, toolSetMagicWandSettingsCommand, toolSetMaskViewModeCommand } from "./tool"; +export { toolCommands, toolEnterMaskEditCommand, toolEnterTemporaryPanCommand, toolExitMaskEditCommand, toolExitTemporaryPanCommand, toolSetActiveCommand, toolSetBrushPreviewCommand, toolSetBrushSettingsCommand, toolSetBrushStrokePreviewCommand, toolSetChromaKeySettingsCommand, toolSetGenerateSettingsCommand, toolSetMagicWandSettingsCommand, toolSetMaskViewModeCommand } from "./tool"; export { transformBeginCommand, transformCommands, transformEndCommand, transformSetBoundsCommand, transformUpdateCommand } from "./transform"; export type { TransformBeginPayload, TransformSetBoundsPayload, TransformUpdatePayload } from "./transform"; -export type { ToolEnterMaskEditPayload, ToolSetActivePayload, ToolSetBrushPreviewPayload, ToolSetBrushSettingsPayload, ToolSetBrushStrokePreviewPayload, ToolSetChromaKeySettingsPayload, ToolSetMagicWandSettingsPayload, ToolSetMaskViewModePayload } from "./tool"; +export type { ToolEnterMaskEditPayload, ToolSetActivePayload, ToolSetBrushPreviewPayload, ToolSetBrushSettingsPayload, ToolSetBrushStrokePreviewPayload, ToolSetChromaKeySettingsPayload, ToolSetGenerateSettingsPayload, ToolSetMagicWandSettingsPayload, ToolSetMaskViewModePayload } from "./tool"; export { viewportCommands, viewportPanCommand, diff --git a/commands/payloads.ts b/commands/payloads.ts index 5295cde..c905fd9 100644 --- a/commands/payloads.ts +++ b/commands/payloads.ts @@ -23,7 +23,7 @@ import type { DocumentUngroupLayerPayload, } from "./document"; import type { SelectionAddLayerPayload, SelectionSetPayload } from "./selection"; -import type { ToolEnterMaskEditPayload, ToolSetActivePayload, ToolSetBrushPreviewPayload, ToolSetBrushSettingsPayload, ToolSetBrushStrokePreviewPayload, ToolSetChromaKeySettingsPayload, ToolSetMagicWandSettingsPayload, ToolSetMaskViewModePayload } from "./tool"; +import type { ToolEnterMaskEditPayload, ToolSetActivePayload, ToolSetBrushPreviewPayload, ToolSetBrushSettingsPayload, ToolSetBrushStrokePreviewPayload, ToolSetChromaKeySettingsPayload, ToolSetGenerateSettingsPayload, ToolSetMagicWandSettingsPayload, ToolSetMaskViewModePayload } from "./tool"; import type { TransformBeginPayload, TransformSetBoundsPayload, TransformUpdatePayload } from "./transform"; import type { ViewportFitArtboardPayload, @@ -59,6 +59,7 @@ export type CommandPayloads = { [commandIds.selectionClear]: void; [commandIds.selectionAddLayer]: SelectionAddLayerPayload; [commandIds.toolSetActive]: ToolSetActivePayload; + [commandIds.toolSetGenerateSettings]: ToolSetGenerateSettingsPayload; [commandIds.toolSetBrushSettings]: ToolSetBrushSettingsPayload; [commandIds.toolSetChromaKeySettings]: ToolSetChromaKeySettingsPayload; [commandIds.toolSetMagicWandSettings]: ToolSetMagicWandSettingsPayload; diff --git a/commands/tool.test.ts b/commands/tool.test.ts index 2d99138..ad3c376 100644 --- a/commands/tool.test.ts +++ b/commands/tool.test.ts @@ -5,11 +5,12 @@ import { toolEnterMaskEditCommand, toolEnterTemporaryPanCommand, toolExitMaskEdi const defaultBrush = { color: "#111827", size: 8, hardness: 100 }; const defaultChromaKey = { color: "#00ff00", tolerance: 32, softness: 24, feather: 0, choke: 0, despeckle: 0, spill: 50 }; const defaultMagicWand = { tolerance: 32, feather: 0, choke: 0, despeckle: 0, contiguous: true, mode: "replace" as const }; +const defaultGenerate = { mode: "text-to-image" as const, model: "auto" as const, prompt: "", negativePrompt: "", strength: 75, steps: 30, cfg: 7, seed: -1, sampler: "euler", scheduler: "normal", width: 1024, height: 1024, outpaint: { left: 128, top: 128, right: 128, bottom: 128, feathering: 32 } }; describe("tool commands", () => { test("sets active tool", () => { const next = toolSetActiveCommand.execute({ state: createInitialAppState("Test") }, { tool: "brush" }); - expect(next.editor.tools).toEqual({ activeTool: "brush", interactionMode: { type: "tool", tool: "brush" }, brush: defaultBrush, chromaKey: defaultChromaKey, magicWand: defaultMagicWand }); + expect(next.editor.tools).toEqual({ activeTool: "brush", interactionMode: { type: "tool", tool: "brush" }, brush: defaultBrush, generate: defaultGenerate, chromaKey: defaultChromaKey, magicWand: defaultMagicWand }); }); test("sets brush settings", () => { @@ -64,7 +65,7 @@ describe("tool commands", () => { const panning = toolEnterTemporaryPanCommand.execute({ state: initial }, undefined); const restored = toolExitTemporaryPanCommand.execute({ state: panning }, undefined); - expect(panning.editor.tools).toEqual({ activeTool: "select", interactionMode: { type: "temporary-pan", previousTool: "select" }, brush: defaultBrush, chromaKey: defaultChromaKey, magicWand: defaultMagicWand }); + expect(panning.editor.tools).toEqual({ activeTool: "select", interactionMode: { type: "temporary-pan", previousTool: "select" }, brush: defaultBrush, generate: defaultGenerate, chromaKey: defaultChromaKey, magicWand: defaultMagicWand }); expect(restored.editor.tools).toEqual(initial.editor.tools); }); }); diff --git a/commands/tool.ts b/commands/tool.ts index 24ba357..9afedb0 100644 --- a/commands/tool.ts +++ b/commands/tool.ts @@ -3,7 +3,7 @@ import type { Vec2D } from "@core/geometry"; import type { LayerId, ArtboardId, AssetId } from "@core/id"; import type { Layer } from "@core/layer"; import type { MaskViewMode } from "@editor/state"; -import type { BrushSettings, ChromaKeySettings, MagicWandSettings, ToolId } from "@editor/tools"; +import type { BrushSettings, ChromaKeySettings, GenerateSettings, MagicWandSettings, ToolId } from "@editor/tools"; import type { Command } from "./command"; import { commandIds } from "./ids"; @@ -13,6 +13,8 @@ export type ToolSetActivePayload = { export type ToolSetBrushSettingsPayload = Partial; +export type ToolSetGenerateSettingsPayload = Partial; + export type ToolSetChromaKeySettingsPayload = Partial; export type ToolSetMagicWandSettingsPayload = Partial; @@ -56,6 +58,44 @@ export const toolSetActiveCommand: Command = { }, }; +export const toolSetGenerateSettingsCommand: Command = { + id: commandIds.toolSetGenerateSettings, + name: "Set generate settings", + execute({ state }, payload) { + const mode = payload.mode ?? state.editor.tools.generate.mode; + return { + ...state, + editor: { + ...state.editor, + tools: { + ...state.editor.tools, + generate: { + mode, + model: payload.model ?? state.editor.tools.generate.model, + prompt: payload.prompt ?? state.editor.tools.generate.prompt, + negativePrompt: payload.negativePrompt ?? state.editor.tools.generate.negativePrompt, + strength: clampNumber(payload.strength ?? state.editor.tools.generate.strength, 0, 100), + steps: Math.round(clampNumber(payload.steps ?? state.editor.tools.generate.steps, 1, 150)), + cfg: clampNumber(payload.cfg ?? state.editor.tools.generate.cfg, 0, 30), + seed: Math.round(clampNumber(payload.seed ?? state.editor.tools.generate.seed, -1, Number.MAX_SAFE_INTEGER)), + sampler: payload.sampler ?? state.editor.tools.generate.sampler, + scheduler: payload.scheduler ?? state.editor.tools.generate.scheduler, + width: Math.round(clampNumber(payload.width ?? state.editor.tools.generate.width, 64, 4096)), + height: Math.round(clampNumber(payload.height ?? state.editor.tools.generate.height, 64, 4096)), + outpaint: { + left: Math.round(clampNumber(payload.outpaint?.left ?? state.editor.tools.generate.outpaint.left, 0, 2048)), + top: Math.round(clampNumber(payload.outpaint?.top ?? state.editor.tools.generate.outpaint.top, 0, 2048)), + right: Math.round(clampNumber(payload.outpaint?.right ?? state.editor.tools.generate.outpaint.right, 0, 2048)), + bottom: Math.round(clampNumber(payload.outpaint?.bottom ?? state.editor.tools.generate.outpaint.bottom, 0, 2048)), + feathering: Math.round(clampNumber(payload.outpaint?.feathering ?? state.editor.tools.generate.outpaint.feathering, 0, 512)), + }, + }, + }, + }, + }; + }, +}; + export const toolSetBrushSettingsCommand: Command = { id: commandIds.toolSetBrushSettings, name: "Set brush settings", @@ -274,6 +314,7 @@ export const toolExitTemporaryPanCommand: Command = { export const toolCommands = [ toolSetActiveCommand, + toolSetGenerateSettingsCommand, toolSetBrushSettingsCommand, toolSetChromaKeySettingsCommand, toolSetMagicWandSettingsCommand, diff --git a/editor/tools.ts b/editor/tools.ts index 565e771..6ea030b 100644 --- a/editor/tools.ts +++ b/editor/tools.ts @@ -1,4 +1,4 @@ -export const availableToolIds = ["select", "brush", "eraser", "chromaKey", "magicWand", "pan"] as const; +export const availableToolIds = ["select", "generate", "brush", "eraser", "chromaKey", "magicWand", "pan"] as const; export type ToolId = (typeof availableToolIds)[number]; @@ -24,6 +24,32 @@ export type ChromaKeySettings = { export type MagicWandMode = "replace" | "add" | "subtract"; +export type GenerateMode = "text-to-image" | "image-to-image" | "inpaint" | "outpaint"; + +export type GenerateModel = string; + +export type GenerateSettings = { + mode: GenerateMode; + model: GenerateModel; + prompt: string; + negativePrompt: string; + strength: number; + steps: number; + cfg: number; + seed: number; + sampler: string; + scheduler: string; + width: number; + height: number; + outpaint: { + left: number; + top: number; + right: number; + bottom: number; + feathering: number; + }; +}; + export type MagicWandSettings = { tolerance: number; feather: number; @@ -39,6 +65,7 @@ export type ToolState = { brush: BrushSettings; chromaKey: ChromaKeySettings; magicWand: MagicWandSettings; + generate: GenerateSettings; }; export const initialToolState: ToolState = { @@ -47,6 +74,7 @@ export const initialToolState: ToolState = { brush: { color: "#111827", size: 8, hardness: 100 }, chromaKey: { color: "#00ff00", tolerance: 32, softness: 24, feather: 0, choke: 0, despeckle: 0, spill: 50 }, magicWand: { tolerance: 32, feather: 0, choke: 0, despeckle: 0, contiguous: true, mode: "replace" }, + generate: { mode: "text-to-image", model: "auto", prompt: "", negativePrompt: "", strength: 75, steps: 30, cfg: 7, seed: -1, sampler: "euler", scheduler: "normal", width: 1024, height: 1024, outpaint: { left: 128, top: 128, right: 128, bottom: 128, feathering: 32 } }, }; export function isPanInteractionMode(interactionMode: InteractionMode): boolean { diff --git a/index.ts b/index.ts index 82b6378..e9ed970 100644 --- a/index.ts +++ b/index.ts @@ -1,8 +1,11 @@ import { serve } from "bun"; +import { handleComfyApi } from "./app/comfy"; import index from "./view/index.html"; const server = serve({ routes: { + "/api/comfy/models": handleComfyApi, + "/api/comfy/generate": handleComfyApi, "/*": index, }, diff --git a/input/tool-keybinds.ts b/input/tool-keybinds.ts index ced12f3..6314fb7 100644 --- a/input/tool-keybinds.ts +++ b/input/tool-keybinds.ts @@ -3,6 +3,7 @@ import type { Dispatch } from "@commands/dispatcher"; import type { KeybindEvent } from "./keyboard"; const toolKeybinds = { + g: "generate", b: "brush", e: "eraser", k: "chromaKey", diff --git a/input/transform-controls.ts b/input/transform-controls.ts index 4c985a4..169f5ba 100644 --- a/input/transform-controls.ts +++ b/input/transform-controls.ts @@ -16,7 +16,7 @@ import type { PointerInputEvent } from "./pointer"; type TransformHandle = "body" | "nw" | "n" | "ne" | "e" | "se" | "s" | "sw" | "w"; -type InputToolId = "select" | "brush" | "eraser" | "chromaKey" | "magicWand" | "pan"; +type InputToolId = "select" | "generate" | "brush" | "eraser" | "chromaKey" | "magicWand" | "pan"; type InputInteractionMode = | { type: "tool"; tool: InputToolId } diff --git a/view/App.tsx b/view/App.tsx index ff08ede..2405d9d 100644 --- a/view/App.tsx +++ b/view/App.tsx @@ -1,14 +1,16 @@ -import { useEffect, useState } from "react"; -import { DownloadSimple, FolderOpen, Stack } from "@phosphor-icons/react"; +import { useCallback, useEffect, useRef, useState } from "react"; +import { DownloadSimple, FolderOpen, Sparkle, Stack } from "@phosphor-icons/react"; import type { ImageStudioApp } from "@app/app"; import { commandIds } from "@commands/ids"; import { BottomControlsIsland } from "./BottomControlsIsland"; import { brushUnavailableHint } from "./canvas/brush"; import { CanvasViewport } from "./CanvasViewport"; +import { GenerateSheet } from "./GenerateSheet"; import { LayersSheet } from "./LayersSheet"; import { ShortcutsDisplay } from "./ShortcutsDisplay"; import { ToolOverlay } from "./ToolOverlay"; import { resolveTransformTargetBounds, selectedTransformTarget } from "@editor/transform-targets"; +import type { ToolId } from "@editor/tools"; import { handleDeleteSelectionKey, handleHistoryKey, handleToolKey, keybindEventFromKeyboardEvent } from "@input/index"; import { useAppState } from "./useAppState"; import { downloadArtboardPng } from "./exportArtboardPng"; @@ -25,9 +27,31 @@ export function App({ app }: AppProps) { const viewportActivityIsland = useViewportActivityIsland(state.editor.viewport); const imageImport = useImageImport(app.store); const [layersOpen, setLayersOpen] = useState(false); + const [generateOpen, setGenerateOpen] = useState(false); + const previousGenerateTool = useRef("select"); const transformTarget = state.editor.transformSession?.target ?? selectedTransformTarget(state.document, state.editor.selection); const activeArtboard = state.document.artboards.find((artboard) => artboard.id === state.editor.selection.artboardId) ?? state.document.artboards[0]; + const openGenerate = useCallback(() => { + const activeTool = app.store.getState().editor.tools.activeTool; + if (activeTool !== "generate") previousGenerateTool.current = activeTool; + app.store.dispatch(commandIds.toolSetActive, { tool: "generate" }); + setLayersOpen(false); + setGenerateOpen(true); + }, [app.store]); + + const closeGenerate = useCallback(() => { + setGenerateOpen(false); + if (app.store.getState().editor.tools.activeTool === "generate") { + app.store.dispatch(commandIds.toolSetActive, { tool: previousGenerateTool.current }); + } + }, [app.store]); + + const toggleGenerate = useCallback(() => { + if (generateOpen) closeGenerate(); + else openGenerate(); + }, [closeGenerate, generateOpen, openGenerate]); + useEffect(() => { const handleKeyDown = (event: KeyboardEvent) => { const target = event.target; @@ -51,16 +75,23 @@ export function App({ app }: AppProps) { return; } + const key = event.key.toLowerCase(); const keybindEvent = keybindEventFromKeyboardEvent(event); + + if (key === "g") { + toggleGenerate(); + event.preventDefault(); + return; + } + const toolConsumed = handleToolKey({ event: keybindEvent, dispatch: app.store.dispatch }); if (toolConsumed) { event.preventDefault(); return; } - const key = event.key.toLowerCase(); - if (key === "l") { + closeGenerate(); setLayersOpen((open) => !open); event.preventDefault(); return; @@ -76,7 +107,7 @@ export function App({ app }: AppProps) { window.addEventListener("keydown", handleKeyDown); return () => window.removeEventListener("keydown", handleKeyDown); - }, [app.store]); + }, [app.store, closeGenerate, toggleGenerate]); const transformBounds = transformTarget ? resolveTransformTargetBounds(state.document, transformTarget) : undefined; const brushHint = brushUnavailableHint(state.document, state.editor); @@ -88,10 +119,13 @@ export function App({ app }: AppProps) { + - @@ -103,6 +137,15 @@ export function App({ app }: AppProps) { dispatch={app.store.dispatch} /> + open ? openGenerate() : closeGenerate()} + dispatch={app.store.dispatch} + /> - {(activeTool === "brush" || activeTool === "eraser") && brushHint ? ( + {activeTool === "generate" ? ( + + ) : (activeTool === "brush" || activeTool === "eraser") && brushHint ? ( ) : activeTool === "brush" || activeTool === "eraser" ? ( diff --git a/view/GenerateSheet.tsx b/view/GenerateSheet.tsx new file mode 100644 index 0000000..aef19d1 --- /dev/null +++ b/view/GenerateSheet.tsx @@ -0,0 +1,32 @@ +import type { ImageDocument } from "@core/document"; +import type { SelectionState, ViewportState } from "@editor/state"; +import type { GenerateSettings } from "@editor/tools"; +import type { AppStore } from "@editor/store"; +import { GenerateControls } from "./bottom-controls/GenerateControls"; + +export type GenerateSheetProps = { + document: ImageDocument; + selection: SelectionState; + viewport: ViewportState; + settings: GenerateSettings; + open: boolean; + onOpenChange: (open: boolean) => void; + dispatch: AppStore["dispatch"]; +}; + +export function GenerateSheet({ document, selection, viewport, settings, open, onOpenChange, dispatch }: GenerateSheetProps) { + return ( + + ); +} + diff --git a/view/ShortcutsDisplay.tsx b/view/ShortcutsDisplay.tsx index 7854063..fa32911 100644 --- a/view/ShortcutsDisplay.tsx +++ b/view/ShortcutsDisplay.tsx @@ -8,6 +8,7 @@ type Shortcut = { const shortcuts: Shortcut[] = [ { keys: ["S"], label: "Select" }, + { keys: ["G"], label: "Generate" }, { keys: ["B"], label: "Brush" }, { keys: ["E"], label: "Eraser" }, { keys: ["K"], label: "Chroma key" }, diff --git a/view/ToolOverlay.tsx b/view/ToolOverlay.tsx index 307cee0..28f3833 100644 --- a/view/ToolOverlay.tsx +++ b/view/ToolOverlay.tsx @@ -1,4 +1,4 @@ -import { Cursor, Eraser, Hand, PaintBrush, DropHalf, MagicWand } from "@phosphor-icons/react"; +import { Cursor, Eraser, Hand, PaintBrush, DropHalf, MagicWand, Sparkle } from "@phosphor-icons/react"; import { commandIds } from "@commands/ids"; import type { AppStore } from "@editor/store"; import type { InteractionMode, ToolId } from "@editor/tools"; @@ -41,6 +41,8 @@ export function ToolOverlay({ activeTool, interactionMode, dispatch }: ToolOverl function iconForTool(tool: ToolId) { switch (tool) { + case "generate": + return Sparkle; case "brush": return PaintBrush; case "eraser": diff --git a/view/bottom-controls/GenerateActionControls.tsx b/view/bottom-controls/GenerateActionControls.tsx new file mode 100644 index 0000000..c1aca89 --- /dev/null +++ b/view/bottom-controls/GenerateActionControls.tsx @@ -0,0 +1,40 @@ +import { useState } from "react"; +import type { ImageDocument } from "@core/document"; +import type { SelectionState, ViewportState } from "@editor/state"; +import type { GenerateSettings } from "@editor/tools"; +import type { AppStore } from "@editor/store"; +import { runGenerate } from "../generate/runGenerate"; + +export type GenerateActionControlsProps = { + document: ImageDocument; + selection: SelectionState; + viewport: ViewportState; + settings: GenerateSettings; + dispatch: AppStore["dispatch"]; +}; + +export function GenerateActionControls({ document, selection, viewport, settings, dispatch }: GenerateActionControlsProps) { + const [busy, setBusy] = useState(false); + const [error, setError] = useState(); + const canGenerate = Boolean(settings.prompt.trim()) && !busy; + + return ( +
+ +
+ ); +} diff --git a/view/bottom-controls/GenerateControls.tsx b/view/bottom-controls/GenerateControls.tsx new file mode 100644 index 0000000..c2b2ac4 --- /dev/null +++ b/view/bottom-controls/GenerateControls.tsx @@ -0,0 +1,334 @@ +import { useEffect, useRef, useState, type RefObject } from "react"; +import { CaretDown, CaretUp } from "@phosphor-icons/react"; +import { commandIds } from "@commands/ids"; +import type { ImageDocument } from "@core/document"; +import type { Layer } from "@core/layer"; +import type { AppStore } from "@editor/store"; +import type { SelectionState, ViewportState } from "@editor/state"; +import type { GenerateMode, GenerateModel, GenerateSettings } from "@editor/tools"; +import { BottomControlSelectMenu, type BottomControlSelectOption } from "./SelectMenu"; +import { BottomControlSlider } from "./Slider"; + +const modes = [ + { value: "text-to-image", label: "Text → image" }, + { value: "image-to-image", label: "Image → image" }, + { value: "inpaint", label: "Inpaint" }, + { value: "outpaint", label: "Outpaint" }, +] satisfies readonly BottomControlSelectOption[]; + +const sizePresets = [ + { label: "1:1", w: 1024, h: 1024 }, + { label: "4:3", w: 1152, h: 896 }, + { label: "3:4", w: 896, h: 1152 }, + { label: "16:9", w: 1344, h: 768 }, + { label: "9:16", w: 768, h: 1344 }, +] as const; + +export type GenerateControlsProps = { + document: ImageDocument; + selection: SelectionState; + viewport: ViewportState; + settings: GenerateSettings; + dispatch: AppStore["dispatch"]; +}; + +export function GenerateControls({ settings, dispatch }: GenerateControlsProps) { + const [models, setModels] = useState[]>([{ value: "auto", label: "Auto" }]); + const [samplers, setSamplers] = useState[]>([{ value: settings.sampler, label: settings.sampler }]); + const [schedulers, setSchedulers] = useState[]>([{ value: settings.scheduler, label: settings.scheduler }]); + const [advancedOpen, setAdvancedOpen] = useState(false); + const [outpaintOpen, setOutpaintOpen] = useState(false); + const [sizeOpen, setSizeOpen] = useState(false); + const sizeRef = useRef(null); + const [error, setError] = useState(); + + useEffect(() => { + let cancelled = false; + void fetch("/api/comfy/models") + .then((response) => response.ok ? response.json() : Promise.reject(new Error("Unable to load ComfyUI models"))) + .then((body: { models?: string[]; samplers?: string[]; schedulers?: string[] }) => { + if (cancelled) return; + setModels([{ value: "auto", label: "Auto" }, ...(body.models ?? []).map((model) => ({ value: model, label: model }))]); + if (body.samplers?.length) setSamplers(body.samplers.map((sampler) => ({ value: sampler, label: sampler }))); + if (body.schedulers?.length) setSchedulers(body.schedulers.map((scheduler) => ({ value: scheduler, label: scheduler }))); + }) + .catch((reason: unknown) => { + if (!cancelled) setError(reason instanceof Error ? reason.message : "Unable to load ComfyUI models"); + }); + return () => { + cancelled = true; + }; + }, []); + + useEffect(() => { + if (!sizeOpen) return; + const close = (event: PointerEvent) => { + if (!sizeRef.current?.contains(event.target as Node)) setSizeOpen(false); + }; + window.addEventListener("pointerdown", close); + return () => window.removeEventListener("pointerdown", close); + }, [sizeOpen]); + + return ( +
+ {error ?

{error}

: null} + +
+