syntaxbullet/image-studio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: enhance ControlNet integration with Xinsir-compatible pose rendering and update model handling

Browse Source
This commit is contained in:
syntaxbullet
2026-07-11 19:46:02 +02:00
parent ff762b8f17
commit 95043dfbdd
2 changed files with 37 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
server/comfy.test.ts
View File

@@ -65,6 +65,14 @@ describe("Comfy adapter", () => {
expect(workflow["6"]?.inputs).toMatchObject({ latent_image: ["19", 0], positive: ["22", 0], negative: ["22", 1] });
});
test("uses Xinsir-compatible pose rendering for pose ControlNet", () => {
const workflow = buildSdxlWorkflow({
...inpaintRequest({ maskedContent: "neutral" }),
inpaint: { maskedContent: "neutral", structureControl: "pose", controlModel: "controlnet-openpose-sdxl-1.0.safetensors" },
});
expect(workflow["21"]).toMatchObject({ class_type: "OpenposePreprocessor", inputs: { detect_hand: "enable", detect_body: "enable", detect_face: "enable", scale_stick_for_xinsr_cn: "enable" } });
});
test("adds an optional low-denoise detail pass", () => {
const workflow = buildSdxlWorkflow({ ...inpaintRequest({ maskedContent: "neutral" }), refinePass: true, refineStrength: 18, seed: 40 });
expect(workflow["30"]).toMatchObject({ class_type: "KSampler", inputs: { seed: 41, denoise: 0.18, latent_image: ["6", 0] } });
@@ -73,7 +81,7 @@ describe("Comfy adapter", () => {
test("builds native SAM3 point selection as a mask output", () => {
const workflow = buildSemanticSelectionWorkflow({ inputImage: "input.png", model: "sam3.safetensors", x: 24.4, y: 18.6 });
expect(workflow["1"]).toMatchObject({ class_type: "UNETLoader", inputs: { unet_name: "sam3.safetensors" } });
expect(workflow["1"]).toMatchObject({ class_type: "CheckpointLoaderSimple", inputs: { ckpt_name: "sam3.safetensors" } });
expect(workflow["3"]).toMatchObject({ class_type: "SAM3_Detect", inputs: { positive_coords: '[{"x":24,"y":19}]', refine_iterations: 2 } });
expect(workflow["4"]).toMatchObject({ class_type: "MaskToImage", inputs: { mask: ["3", 0] } });
});
@@ -123,7 +131,7 @@ describe("Comfy adapter", () => {
test("lists branded non-Z diffusion models under Anima", async () => {
const originalFetch = globalThis.fetch;
const mockFetch: typeof fetch = Object.assign(async () => new Response(JSON.stringify({
CheckpointLoaderSimple: { input: { required: { ckpt_name: [["sd_xl_base_1.0.safetensors"]] } } },
CheckpointLoaderSimple: { input: { required: { ckpt_name: [["sd_xl_base_1.0.safetensors", "sam3.1_multiplex_fp16.safetensors"]] } } },
KSampler: { input: { required: { sampler_name: [["euler"]], scheduler: [["normal"]] } } },
UNETLoader: {
input: {
@@ -140,12 +148,29 @@ describe("Comfy adapter", () => {
},
CLIPLoader: { input: { required: { clip_name: [["qwen_3_06b_base.safetensors"]] } } },
VAELoader: { input: { required: { vae_name: [["qwen_image_vae.safetensors"]] } } },
ControlNetLoader: { input: { required: { control_net_name: [[
"controlnet-canny-sdxl-1.0-fp16.safetensors",
"controlnet-depth-sdxl-1.0-fp16.safetensors",
"controlnet-openpose-sdxl-1.0.safetensors",
]] } } },
Canny: {},
"MiDaS-DepthMapPreprocessor": {},
OpenposePreprocessor: {},
ControlNetApplyAdvanced: {},
SAM3_Detect: {},
MaskToImage: {},
}), { headers: { "content-type": "application/json" } }), { preconnect: originalFetch.preconnect });
globalThis.fetch = mockFetch;
try {
const response = await handleComfyApi(new Request("http://image-studio.test/api/comfy/models"));
const body = await response.json() as { architectures: { value: string; models: string[] }[] };
const body = await response.json() as {
architectures: { value: string; models: string[] }[];
controlModels: string[];
structureControls: string[];
semanticSelection: boolean;
sam3Models: string[];
};
const anima = body.architectures.find((architecture) => architecture.value === "anima");
expect(response.status).toBe(200);
@@ -153,6 +178,10 @@ describe("Comfy adapter", () => {
expect(anima?.models).toContain("miaomiaoHarem_anima13.safetensors");
expect(anima?.models).not.toContain("z_image_bf16.safetensors");
expect(anima?.models).not.toContain("z_image_turbo_bf16.safetensors");
expect(body.structureControls).toEqual(["canny", "depth", "pose"]);
expect(body.controlModels).toHaveLength(3);
expect(body.semanticSelection).toBeTrue();
expect(body.sam3Models).toEqual(["sam3.1_multiplex_fp16.safetensors"]);
} finally {
globalThis.fetch = originalFetch;
}

10
server/comfy.ts
View File

@@ -95,8 +95,8 @@ export async function listGenerationOptions() {
...(info["MiDaS-DepthMapPreprocessor"] && info.ControlNetApplyAdvanced && info.ControlNetLoader ? ["depth"] : []),
...(info.OpenposePreprocessor && info.ControlNetApplyAdvanced && info.ControlNetLoader ? ["pose"] : []),
],
semanticSelection: Boolean(info.SAM3_Detect && info.MaskToImage && diffusionModels.some((model) => /sam.?3/i.test(model))),
sam3Models: diffusionModels.filter((model) => /sam.?3/i.test(model)),
semanticSelection: Boolean(info.SAM3_Detect && info.MaskToImage && checkpointModels.some((model) => /sam.?3/i.test(model))),
sam3Models: checkpointModels.filter((model) => /sam.?3/i.test(model)),
architectures: [
{
value: "sdxl",
@@ -134,7 +134,7 @@ export async function segment(request: ComfySegmentRequest, signal?: AbortSignal
if (!request.inputImage) throw new Error("Semantic selection requires an image");
if (!Number.isFinite(request.x) || !Number.isFinite(request.y)) throw new Error("Semantic selection requires a valid point");
const info = await fetchObjectInfo();
const models = info.UNETLoader?.input?.required?.unet_name?.[0] ?? [];
const models = info.CheckpointLoaderSimple?.input?.required?.ckpt_name?.[0] ?? [];
const model = request.model && request.model !== "auto" ? request.model : models.find((candidate) => /sam.?3/i.test(candidate));
if (!info.SAM3_Detect || !info.MaskToImage || !model || !models.includes(model)) throw new Error("SAM3 semantic selection is not installed in ComfyUI. Install a SAM3 model and enable the native SAM3 nodes.");
const uploaded = await uploadDataUrl(request.inputImage, `image-studio-segment-${crypto.randomUUID()}.png`, signal);
@@ -158,7 +158,7 @@ export async function segment(request: ComfySegmentRequest, signal?: AbortSignal
export function buildSemanticSelectionWorkflow(request: ComfySegmentRequest & { model: string }): Workflow {
return {
"1": { class_type: "UNETLoader", inputs: { unet_name: request.model, weight_dtype: "default" } },
"1": { class_type: "CheckpointLoaderSimple", inputs: { ckpt_name: request.model } },
"2": { class_type: "LoadImage", inputs: { image: request.inputImage } },
"3": { class_type: "SAM3_Detect", inputs: { model: ["1", 0], image: ["2", 0], positive_coords: JSON.stringify([{ x: Math.round(request.x), y: Math.round(request.y) }]), threshold: 0.5, refine_iterations: 2, individual_masks: false } },
"4": { class_type: "MaskToImage", inputs: { mask: ["3", 0] } },
@@ -379,7 +379,7 @@ function finalizeSdxlWorkflow(workflow: Workflow, samplerInputs: Record<string,
} else if (control === "depth") {
workflow["21"] = { class_type: "MiDaS-DepthMapPreprocessor", inputs: { image: ["4", 0], a: 6.283, bg_threshold: 0.1, resolution: Math.max(request.width ?? 512, request.height ?? 512) } };
} else {
workflow["21"] = { class_type: "OpenposePreprocessor", inputs: { image: ["4", 0], detect_hand: "enable", detect_body: "enable", detect_face: "enable", resolution: Math.max(request.width ?? 512, request.height ?? 512) } };
workflow["21"] = { class_type: "OpenposePreprocessor", inputs: { image: ["4", 0], detect_hand: "enable", detect_body: "enable", detect_face: "enable", scale_stick_for_xinsr_cn: "enable", resolution: Math.max(request.width ?? 512, request.height ?? 512) } };
}
workflow["22"] = {
class_type: "ControlNetApplyAdvanced",