feat: enhance generate settings to support multiple architectures and their defaults
This commit is contained in:
syntaxbullet
232
app/comfy.ts
232
app/comfy.ts
@@ -1,8 +1,13 @@
|
||||
type GenerateArchitecture = "sdxl" | "z-image" | "z-image-turbo" | "anima";
|
||||
type GenerateMode = "text-to-image" | "image-to-image" | "inpaint" | "outpaint";
|
||||
type Workflow = Record<string, { class_type: string; inputs: Record<string, unknown> }>;
|
||||
|
||||
type ComfyGenerateRequest = {
|
||||
architecture?: GenerateArchitecture;
|
||||
mode: GenerateMode;
|
||||
model: string;
|
||||
textEncoder?: string;
|
||||
vae?: string;
|
||||
prompt: string;
|
||||
negativePrompt?: string;
|
||||
strength?: number;
|
||||
@@ -35,7 +40,21 @@ type ComfyGenerateRequest = {
|
||||
maskImage?: string;
|
||||
};
|
||||
|
||||
type ComfyObjectInfo = {
|
||||
CheckpointLoaderSimple?: { input?: { required?: { ckpt_name?: [string[]] } } };
|
||||
KSampler?: { input?: { required?: { sampler_name?: [string[]]; scheduler?: [string[]] } } };
|
||||
UNETLoader?: { input?: { required?: { unet_name?: [string[]] } } };
|
||||
CLIPLoader?: { input?: { required?: { clip_name?: [string[]] } } };
|
||||
VAELoader?: { input?: { required?: { vae_name?: [string[]] } } };
|
||||
};
|
||||
|
||||
const comfyBaseUrl = process.env.COMFYUI_URL ?? "http://127.0.0.1:8188";
|
||||
const defaultModels: Record<GenerateArchitecture, string> = {
|
||||
sdxl: "sd_xl_base_1.0.safetensors",
|
||||
"z-image": "z_image_bf16.safetensors",
|
||||
"z-image-turbo": "z_image_turbo_bf16.safetensors",
|
||||
anima: "anima-base-v1.0.safetensors",
|
||||
};
|
||||
|
||||
export async function handleComfyApi(request: Request) {
|
||||
try {
|
||||
@@ -51,14 +70,49 @@ export async function handleComfyApi(request: Request) {
|
||||
async function listGenerationOptions() {
|
||||
const response = await fetch(`${comfyBaseUrl}/object_info`);
|
||||
if (!response.ok) throw new Error(`ComfyUI option lookup failed: ${response.status}`);
|
||||
const info = await response.json() as {
|
||||
CheckpointLoaderSimple?: { input?: { required?: { ckpt_name?: [string[]] } } };
|
||||
KSampler?: { input?: { required?: { sampler_name?: [string[]]; scheduler?: [string[]] } } };
|
||||
};
|
||||
const info = await response.json() as ComfyObjectInfo;
|
||||
const checkpointModels = info.CheckpointLoaderSimple?.input?.required?.ckpt_name?.[0] ?? [];
|
||||
const diffusionModels = info.UNETLoader?.input?.required?.unet_name?.[0] ?? [];
|
||||
const textEncoders = info.CLIPLoader?.input?.required?.clip_name?.[0] ?? [];
|
||||
const vaes = info.VAELoader?.input?.required?.vae_name?.[0] ?? [];
|
||||
|
||||
return {
|
||||
models: info.CheckpointLoaderSimple?.input?.required?.ckpt_name?.[0] ?? [],
|
||||
models: checkpointModels,
|
||||
samplers: info.KSampler?.input?.required?.sampler_name?.[0] ?? [],
|
||||
schedulers: info.KSampler?.input?.required?.scheduler?.[0] ?? [],
|
||||
diffusionModels,
|
||||
textEncoders,
|
||||
vaes,
|
||||
architectures: [
|
||||
{
|
||||
value: "sdxl",
|
||||
label: "SDXL",
|
||||
defaultModel: checkpointModels[0] ?? defaultModels.sdxl,
|
||||
models: checkpointModels,
|
||||
supportedModes: ["text-to-image", "image-to-image", "inpaint", "outpaint"],
|
||||
},
|
||||
{
|
||||
value: "z-image",
|
||||
label: "Z-Image",
|
||||
defaultModel: defaultModels["z-image"],
|
||||
models: modelsForArchitecture(diffusionModels, defaultModels["z-image"], [/z[_-]?image(?!.*turbo)/i]),
|
||||
supportedModes: ["text-to-image"],
|
||||
},
|
||||
{
|
||||
value: "z-image-turbo",
|
||||
label: "Z-Image Turbo",
|
||||
defaultModel: defaultModels["z-image-turbo"],
|
||||
models: modelsForArchitecture(diffusionModels, defaultModels["z-image-turbo"], [/z[_-]?image.*turbo/i]),
|
||||
supportedModes: ["text-to-image"],
|
||||
},
|
||||
{
|
||||
value: "anima",
|
||||
label: "Anima",
|
||||
defaultModel: defaultModels.anima,
|
||||
models: modelsForArchitecture(diffusionModels, defaultModels.anima, [/anima/i]),
|
||||
supportedModes: ["text-to-image"],
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
@@ -68,16 +122,15 @@ async function listCheckpointModels() {
|
||||
|
||||
async function generate(request: ComfyGenerateRequest) {
|
||||
if (!request.prompt?.trim()) throw new Error("Prompt is required");
|
||||
if (!request.model || request.model === "auto") {
|
||||
const models = await listCheckpointModels();
|
||||
request.model = models[0] ?? "sd_xl_base_1.0.safetensors";
|
||||
}
|
||||
const architecture = normalizeArchitecture(request.architecture);
|
||||
if (request.mode !== "text-to-image" && architecture !== "sdxl") throw new Error(`${architectureLabel(architecture)} currently supports text-to-image only`);
|
||||
if (!request.model || request.model === "auto") request.model = await defaultModelForArchitecture(architecture);
|
||||
|
||||
const clientId = crypto.randomUUID();
|
||||
const uploaded = request.inputImage ? await uploadDataUrl(request.inputImage, `image-studio-${crypto.randomUUID()}.png`) : undefined;
|
||||
const mask = request.maskImage ? await uploadDataUrl(request.maskImage, `image-studio-mask-${crypto.randomUUID()}.png`) : undefined;
|
||||
if (request.mode === "inpaint" && (!uploaded || !mask)) throw new Error("Inpaint requires normalized input and mask images");
|
||||
const prompt = buildSdxlWorkflow({ ...request, inputImage: uploaded, maskImage: mask });
|
||||
const prompt = buildComfyWorkflow({ ...request, architecture, inputImage: uploaded, maskImage: mask });
|
||||
|
||||
const queued = await fetch(`${comfyBaseUrl}/prompt`, {
|
||||
method: "POST",
|
||||
@@ -143,7 +196,21 @@ function isGeneratedImage(image: { filename: string; subfolder?: string; type?:
|
||||
return image.type === undefined || image.type === "output";
|
||||
}
|
||||
|
||||
export function buildSdxlWorkflow(request: ComfyGenerateRequest) {
|
||||
export function buildComfyWorkflow(request: ComfyGenerateRequest): Workflow {
|
||||
switch (normalizeArchitecture(request.architecture)) {
|
||||
case "z-image":
|
||||
return buildZImageWorkflow(request);
|
||||
case "z-image-turbo":
|
||||
return buildZImageTurboWorkflow(request);
|
||||
case "anima":
|
||||
return buildAnimaWorkflow(request);
|
||||
case "sdxl":
|
||||
default:
|
||||
return buildSdxlWorkflow(request);
|
||||
}
|
||||
}
|
||||
|
||||
export function buildSdxlWorkflow(request: ComfyGenerateRequest): Workflow {
|
||||
if (!request.width || !request.height) throw new Error("Generation width and height are required");
|
||||
const width = Math.max(64, Math.round(request.width));
|
||||
const height = Math.max(64, Math.round(request.height));
|
||||
@@ -155,7 +222,7 @@ export function buildSdxlWorkflow(request: ComfyGenerateRequest) {
|
||||
const scheduler = request.scheduler ?? "normal";
|
||||
const positive = request.prompt;
|
||||
const negative = request.negativePrompt ?? "";
|
||||
const workflow: Record<string, { class_type: string; inputs: Record<string, unknown> }> = {
|
||||
const workflow: Workflow = {
|
||||
"1": { class_type: "CheckpointLoaderSimple", inputs: { ckpt_name: request.model } },
|
||||
"2": { class_type: "CLIPTextEncode", inputs: { text: positive, clip: ["1", 1] } },
|
||||
"3": { class_type: "CLIPTextEncode", inputs: { text: negative, clip: ["1", 1] } },
|
||||
@@ -195,6 +262,114 @@ export function buildSdxlWorkflow(request: ComfyGenerateRequest) {
|
||||
return workflow;
|
||||
}
|
||||
|
||||
export function buildZImageWorkflow(request: ComfyGenerateRequest): Workflow {
|
||||
return buildSeparatedTextToImageWorkflow(request, {
|
||||
architecture: "z-image",
|
||||
filenamePrefix: "image-studio-z-image",
|
||||
model: defaultModels["z-image"],
|
||||
textEncoder: "qwen_3_4b.safetensors",
|
||||
vae: "ae.safetensors",
|
||||
clipType: "lumina2",
|
||||
latentNode: "EmptySD3LatentImage",
|
||||
modelSamplingAuraFlow: true,
|
||||
negativeMode: "prompt",
|
||||
steps: 30,
|
||||
cfg: 4,
|
||||
sampler: "res_multistep",
|
||||
scheduler: "simple",
|
||||
});
|
||||
}
|
||||
|
||||
export function buildZImageTurboWorkflow(request: ComfyGenerateRequest): Workflow {
|
||||
return buildSeparatedTextToImageWorkflow(request, {
|
||||
architecture: "z-image-turbo",
|
||||
filenamePrefix: "image-studio-z-image-turbo",
|
||||
model: defaultModels["z-image-turbo"],
|
||||
textEncoder: "qwen_3_4b.safetensors",
|
||||
vae: "ae.safetensors",
|
||||
clipType: "lumina2",
|
||||
latentNode: "EmptySD3LatentImage",
|
||||
modelSamplingAuraFlow: true,
|
||||
negativeMode: "zero",
|
||||
steps: 8,
|
||||
cfg: 1,
|
||||
sampler: "res_multistep",
|
||||
scheduler: "simple",
|
||||
});
|
||||
}
|
||||
|
||||
export function buildAnimaWorkflow(request: ComfyGenerateRequest): Workflow {
|
||||
return buildSeparatedTextToImageWorkflow(request, {
|
||||
architecture: "anima",
|
||||
filenamePrefix: "image-studio-anima",
|
||||
model: defaultModels.anima,
|
||||
textEncoder: "qwen_3_06b_base.safetensors",
|
||||
vae: "qwen_image_vae.safetensors",
|
||||
clipType: "stable_diffusion",
|
||||
latentNode: "EmptyLatentImage",
|
||||
modelSamplingAuraFlow: false,
|
||||
negativeMode: "prompt",
|
||||
steps: 30,
|
||||
cfg: 4,
|
||||
sampler: "er_sde",
|
||||
scheduler: "simple",
|
||||
});
|
||||
}
|
||||
|
||||
function buildSeparatedTextToImageWorkflow(request: ComfyGenerateRequest, config: {
|
||||
architecture: GenerateArchitecture;
|
||||
filenamePrefix: string;
|
||||
model: string;
|
||||
textEncoder: string;
|
||||
vae: string;
|
||||
clipType: string;
|
||||
latentNode: "EmptyLatentImage" | "EmptySD3LatentImage";
|
||||
modelSamplingAuraFlow: boolean;
|
||||
negativeMode: "prompt" | "zero";
|
||||
steps: number;
|
||||
cfg: number;
|
||||
sampler: string;
|
||||
scheduler: string;
|
||||
}): Workflow {
|
||||
if (request.mode !== "text-to-image") throw new Error(`${architectureLabel(config.architecture)} currently supports text-to-image only`);
|
||||
const options = resolveSamplerOptions(request, config);
|
||||
const modelOutput: [string, number] = config.modelSamplingAuraFlow ? ["7", 0] : ["1", 0];
|
||||
const workflow: Workflow = {
|
||||
"1": { class_type: "UNETLoader", inputs: { unet_name: request.model && request.model !== "auto" ? request.model : config.model, weight_dtype: "default" } },
|
||||
"2": { class_type: "CLIPLoader", inputs: { clip_name: resolveSupportModelName(request.textEncoder, config.textEncoder), type: config.clipType, device: "default" } },
|
||||
"3": { class_type: "VAELoader", inputs: { vae_name: resolveSupportModelName(request.vae, config.vae) } },
|
||||
"4": { class_type: "CLIPTextEncode", inputs: { text: request.prompt, clip: ["2", 0] } },
|
||||
"6": { class_type: config.latentNode, inputs: { width: options.width, height: options.height, batch_size: 1 } },
|
||||
"8": { class_type: "KSampler", inputs: { seed: options.seed, steps: options.steps, cfg: options.cfg, sampler_name: options.sampler, scheduler: options.scheduler, denoise: 1, model: modelOutput, positive: ["4", 0], negative: ["5", 0], latent_image: ["6", 0] } },
|
||||
"9": { class_type: "VAEDecode", inputs: { samples: ["8", 0], vae: ["3", 0] } },
|
||||
"10": { class_type: "SaveImage", inputs: { filename_prefix: config.filenamePrefix, images: ["9", 0] } },
|
||||
};
|
||||
|
||||
if (config.modelSamplingAuraFlow) workflow["7"] = { class_type: "ModelSamplingAuraFlow", inputs: { model: ["1", 0], shift: 3 } };
|
||||
workflow["5"] = config.negativeMode === "zero"
|
||||
? { class_type: "ConditioningZeroOut", inputs: { conditioning: ["4", 0] } }
|
||||
: { class_type: "CLIPTextEncode", inputs: { text: request.negativePrompt ?? "", clip: ["2", 0] } };
|
||||
|
||||
return workflow;
|
||||
}
|
||||
|
||||
function resolveSamplerOptions(request: ComfyGenerateRequest, defaults: { steps: number; cfg: number; sampler: string; scheduler: string }) {
|
||||
if (!request.width || !request.height) throw new Error("Generation width and height are required");
|
||||
return {
|
||||
width: Math.max(64, Math.round(request.width)),
|
||||
height: Math.max(64, Math.round(request.height)),
|
||||
seed: request.seed === undefined || request.seed < 0 ? Math.floor(Math.random() * 2 ** 32) : Math.round(request.seed),
|
||||
steps: Math.max(1, Math.round(request.steps ?? defaults.steps)),
|
||||
cfg: Math.max(0, request.cfg ?? defaults.cfg),
|
||||
sampler: request.sampler ?? defaults.sampler,
|
||||
scheduler: request.scheduler ?? defaults.scheduler,
|
||||
};
|
||||
}
|
||||
|
||||
function resolveSupportModelName(value: string | undefined, fallback: string) {
|
||||
return value && value !== "auto" ? value : fallback;
|
||||
}
|
||||
|
||||
function resolveGrowMaskBy(request: ComfyGenerateRequest): number {
|
||||
const value = request.inpaint?.growMaskBy ?? 6;
|
||||
if (!Number.isFinite(value)) return 6;
|
||||
@@ -205,6 +380,39 @@ function usesOriginalLatentContent(request: ComfyGenerateRequest): boolean {
|
||||
return request.inpaint?.maskedContent === "original" || request.inpaint?.maskedContent === "originalColor" || request.inpaint?.maskedContent === "edges";
|
||||
}
|
||||
|
||||
async function defaultModelForArchitecture(architecture: GenerateArchitecture) {
|
||||
if (architecture !== "sdxl") return defaultModels[architecture];
|
||||
const models = await listCheckpointModels();
|
||||
return models[0] ?? defaultModels.sdxl;
|
||||
}
|
||||
|
||||
function normalizeArchitecture(architecture: ComfyGenerateRequest["architecture"]): GenerateArchitecture {
|
||||
if (architecture === "z-image" || architecture === "z-image-turbo" || architecture === "anima") return architecture;
|
||||
return "sdxl";
|
||||
}
|
||||
|
||||
function architectureLabel(architecture: GenerateArchitecture): string {
|
||||
switch (architecture) {
|
||||
case "z-image":
|
||||
return "Z-Image";
|
||||
case "z-image-turbo":
|
||||
return "Z-Image Turbo";
|
||||
case "anima":
|
||||
return "Anima";
|
||||
case "sdxl":
|
||||
default:
|
||||
return "SDXL";
|
||||
}
|
||||
}
|
||||
|
||||
function modelsForArchitecture(models: string[], defaultModel: string, matchers: RegExp[]) {
|
||||
return unique([defaultModel, ...models.filter((model) => model === defaultModel || matchers.some((matcher) => matcher.test(model)))]);
|
||||
}
|
||||
|
||||
function unique<T>(values: T[]) {
|
||||
return Array.from(new Set(values));
|
||||
}
|
||||
|
||||
function nodeErrorsMessage(nodeErrors: unknown): string | undefined {
|
||||
if (!nodeErrors) return undefined;
|
||||
if (Array.isArray(nodeErrors) && nodeErrors.length === 0) return undefined;
|
||||
|
||||
Reference in New Issue
Block a user