Files
image-studio/view/bottom-controls/GenerateControls.tsx
syntaxbullet ff762b8f17 feat: add inpaint region functionality and related tools
- Enhanced cursor behavior for new tools: semantic select, mask lasso, and mask rectangle.
- Updated mask edit state to include mask asset ID and kind.
- Implemented inpaint region commands for adding, applying, and removing inpaint regions.
- Introduced new operations for lasso and semantic selection tools.
- Created UI components for candidate review and inpaint region management.
- Added tests for inpaint region commands to ensure functionality.
- Updated various components to support new inpaint features and improve user experience.
2026-07-11 16:41:22 +02:00

345 lines
24 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import { useEffect, useRef, useState, type RefObject } from "react";
import { CaretDown, CaretUp } from "@phosphor-icons/react";
import { commandIds } from "@commands/ids";
import type { AppStore } from "@editor/store";
import type { GenerationResourcesState } from "@editor/state";
import type { GenerateArchitecture, GenerateIntent, GenerateMode, GenerateSettings } from "@editor/tools";
import { resolveGenerationModeOptions, resolveGenerationModelOptions, resolveGenerationStringOptions, resolveGenerationSupportOptions } from "@operations/generation/options";
import { BottomControlSelectMenu, type BottomControlSelectOption } from "./SelectMenu";
import { BottomControlSlider } from "./Slider";
import type { ImageDocument } from "@core/document";
import type { SelectionState } from "@editor/state";
import { InpaintRegionPanel } from "../inpaint/InpaintRegionPanel";
const architectures = [
{ value: "sdxl", label: "SDXL" },
{ value: "z-image", label: "Z-Image" },
{ value: "z-image-turbo", label: "Z-Image Turbo" },
{ value: "anima", label: "Anima" },
] satisfies readonly BottomControlSelectOption<GenerateArchitecture>[];
const modes = [
{ value: "text-to-image", label: "Text → image" },
{ value: "image-to-image", label: "Image → image" },
{ value: "inpaint", label: "Inpaint" },
{ value: "outpaint", label: "Outpaint" },
] satisfies readonly BottomControlSelectOption<GenerateMode>[];
const sizePresets = [
{ label: "1:1", w: 1024, h: 1024 },
{ label: "4:3", w: 1152, h: 896 },
{ label: "3:4", w: 896, h: 1152 },
{ label: "16:9", w: 1344, h: 768 },
{ label: "9:16", w: 768, h: 1344 },
] as const;
const inpaintMaskedContentOptions = [
{ value: "neutral", label: "Neutral fill" },
{ value: "original", label: "Original gray" },
{ value: "originalColor", label: "Original color" },
{ value: "edges", label: "Edge map" },
] satisfies readonly BottomControlSelectOption<GenerateSettings["inpaint"]["maskedContent"]>[];
const inpaintProfileOptions = [
{ value: "remove", label: "Remove object" },
{ value: "replace", label: "Replace object" },
{ value: "repair", label: "Repair detail" },
{ value: "material", label: "Change material" },
{ value: "reshape", label: "Change shape" },
{ value: "custom", label: "Custom" },
] satisfies readonly BottomControlSelectOption<GenerateSettings["inpaint"]["profile"]>[];
const structureControlOptions = [
{ value: "none", label: "None" },
{ value: "canny", label: "Canny edges" },
{ value: "depth", label: "Depth" },
{ value: "pose", label: "Pose" },
] satisfies readonly BottomControlSelectOption<GenerateSettings["inpaint"]["structureControl"]>[];
export type GenerateControlsProps = {
settings: GenerateSettings;
document: ImageDocument;
selection: SelectionState;
resources: GenerationResourcesState;
dispatch: AppStore["dispatch"];
};
export function GenerateControls({ settings, document, selection, resources, dispatch }: GenerateControlsProps) {
const comfyOptions = resources.options;
const [advancedOpen, setAdvancedOpen] = useState(false);
const [outpaintOpen, setOutpaintOpen] = useState(false);
const [inpaintOpen, setInpaintOpen] = useState(false);
const [sizeOpen, setSizeOpen] = useState(false);
const sizeRef = useRef<HTMLDivElement>(null);
const modelOptions = resolveGenerationModelOptions(settings, comfyOptions);
const supportOptions = resolveGenerationSupportOptions(settings, comfyOptions);
const samplerOptions = resolveGenerationStringOptions(comfyOptions?.samplers, settings.sampler);
const schedulerOptions = resolveGenerationStringOptions(comfyOptions?.schedulers, settings.scheduler);
const modeOptions = resolveGenerationModeOptions(settings, comfyOptions, modes);
const availableStructureControls = structureControlOptions.filter((option) => option.value === "none" || option.value === settings.inpaint.structureControl || comfyOptions?.structureControls?.includes(option.value));
useEffect(() => {
if (!sizeOpen) return;
const close = (event: PointerEvent) => {
if (!sizeRef.current?.contains(event.target as Node)) setSizeOpen(false);
};
window.addEventListener("pointerdown", close);
return () => window.removeEventListener("pointerdown", close);
}, [sizeOpen]);
return (
<div className="grid gap-5 pb-2 tabular-nums">
{resources.error ? <p className="rounded-md border border-red-400/15 bg-red-500/10 px-2.5 py-2 text-xs text-red-200">{resources.error}</p> : null}
<section className={panelSectionClass()}>
<div className="px-1">
<h2 className="text-sm font-semibold text-white/90">Edit with AI</h2>
<p className="mt-1 text-xs leading-5 text-white/45">Choose an outcome. Image Studio configures the matching generation workflow.</p>
</div>
<div className="grid grid-cols-2 gap-2">
{intentOptions.map((intent) => (
<button
key={intent.value}
type="button"
aria-pressed={isIntentActive(intent.value, intent.mode, settings)}
className={`border-l-2 px-2.5 py-1.5 text-left transition focus:outline-none focus-visible:ring-2 focus-visible:ring-sky-300/50 ${isIntentActive(intent.value, intent.mode, settings) ? "border-sky-300 bg-sky-300/10 text-sky-100" : "border-transparent text-white/60 hover:border-white/20 hover:text-white"}`}
onClick={() => dispatch(commandIds.toolChooseGenerateIntent, { intent: intent.value })}
>
<span className="block text-sm font-semibold">{intent.label}</span>
<span className={`mt-1 block text-xs leading-4 ${isIntentActive(intent.value, intent.mode, settings) ? "text-sky-100/70" : "text-white/40"}`}>{intent.description}</span>
</button>
))}
</div>
</section>
<section className={panelSectionClass()}>
<label className="grid gap-2">
<span className={panelLabelClass()}>Prompt</span>
<textarea
aria-label="Generate prompt"
value={settings.prompt}
rows={6}
placeholder="Describe the image you want to create…"
onChange={(event) => dispatch(commandIds.toolSetGenerateSettings, { prompt: event.currentTarget.value })}
className={panelTextAreaClass("min-h-36")}
/>
</label>
<label className="grid gap-2">
<span className={panelLabelClass()}>Negative prompt</span>
<textarea
aria-label="Negative prompt"
value={settings.negativePrompt}
rows={3}
placeholder="Things to avoid, e.g. bad quality, artifacts…"
onChange={(event) => dispatch(commandIds.toolSetGenerateSettings, { negativePrompt: event.currentTarget.value })}
className={panelTextAreaClass("min-h-20")}
/>
</label>
</section>
<section className={panelSectionClass()}>
<SectionTitle title="Essentials" />
<SizeControl refRoot={sizeRef} open={sizeOpen} setOpen={setSizeOpen} settings={settings} dispatch={dispatch} />
{settings.mode === "image-to-image" || settings.mode === "inpaint" ? (
<div className={panelRowClass()}>
<span className={panelLabelClass()}>Influence</span>
<BottomControlSlider min={0} max={100} value={settings.strength} className="w-32" aria-label="Generate strength" onValueChange={(strength) => dispatch(commandIds.toolSetGenerateSettings, { strength })} />
<span className="w-10 text-right text-sm text-white">{Math.round(settings.strength)}</span>
</div>
) : null}
<PanelNumber label="Results" aria-label="Generation result count" value={settings.batchSize} onValueChange={(batchSize) => dispatch(commandIds.toolSetGenerateSettings, { batchSize })} />
<button type="button" className={compactRowButtonClass()} aria-pressed={settings.refinePass} onClick={() => dispatch(commandIds.toolSetGenerateSettings, { refinePass: !settings.refinePass })}><span className={panelLabelClass()}>Detail pass</span><span className="min-w-0 flex-1 text-right text-white">{settings.refinePass ? `${Math.round(settings.refineStrength)}%` : "Off"}</span></button>
{settings.refinePass ? <PanelNumber label="Detail strength" aria-label="Generation detail pass strength" value={settings.refineStrength} onValueChange={(refineStrength) => dispatch(commandIds.toolSetGenerateSettings, { refineStrength })} /> : null}
</section>
<section className={panelSectionClass()}>
<button type="button" className={sectionToggleClass()} aria-expanded={advancedOpen} aria-controls="generate-advanced-controls" onClick={() => setAdvancedOpen((open) => !open)}>
<span>
<span className="block text-sm font-semibold text-white/85">Advanced</span>
<span className="block text-xs text-white/40">Model, backend, seed, sampling, and workflow internals</span>
</span>
{advancedOpen ? <CaretUp size={18} weight="bold" /> : <CaretDown size={18} weight="bold" />}
</button>
<div id="generate-advanced-controls" className={`grid gap-2 overflow-hidden transition-all duration-200 ${advancedOpen ? "max-h-[32rem] pt-2 opacity-100" : "max-h-0 opacity-0"}`}>
<PanelSelect label="Mode" value={settings.mode} options={modeOptions} ariaLabel="Generate mode" onValueChange={(mode) => dispatch(commandIds.toolSetGenerateSettings, { mode })} />
<PanelSelect label="Backend" value={settings.architecture} options={architectures} ariaLabel="Generate backend" onValueChange={(architecture) => dispatch(commandIds.toolSetGenerateSettings, { architecture })} />
<PanelSelect label="Model" value={settings.model} options={modelOptions} ariaLabel="Generate model" onValueChange={(model) => dispatch(commandIds.toolSetGenerateSettings, { model })} />
{settings.architecture !== "sdxl" ? (
<>
<PanelSelect label="Text enc." value={settings.textEncoder} options={supportOptions.textEncoders} ariaLabel="Generate text encoder" onValueChange={(textEncoder) => dispatch(commandIds.toolSetGenerateSettings, { textEncoder })} />
<PanelSelect label="VAE" value={settings.vae} options={supportOptions.vaes} ariaLabel="Generate VAE" onValueChange={(vae) => dispatch(commandIds.toolSetGenerateSettings, { vae })} />
</>
) : null}
<PanelNumber label="Seed" aria-label="Generate seed" value={settings.seed} onValueChange={(seed) => dispatch(commandIds.toolSetGenerateSettings, { seed })} />
<PanelSelect label="Sampler" value={settings.sampler} options={samplerOptions} ariaLabel="Generate sampler" onValueChange={(sampler) => dispatch(commandIds.toolSetGenerateSettings, { sampler })} />
<PanelSelect label="Scheduler" value={settings.scheduler} options={schedulerOptions} ariaLabel="Generate scheduler" onValueChange={(scheduler) => dispatch(commandIds.toolSetGenerateSettings, { scheduler })} />
<div className="grid grid-cols-2 gap-2">
<PanelNumber label="Steps" aria-label="Generate steps" value={settings.steps} onValueChange={(steps) => dispatch(commandIds.toolSetGenerateSettings, { steps })} />
<PanelNumber label="CFG" aria-label="Generate CFG" value={settings.cfg} onValueChange={(cfg) => dispatch(commandIds.toolSetGenerateSettings, { cfg })} />
</div>
</div>
</section>
{settings.mode === "outpaint" ? <section className={panelSectionClass()}>
<button type="button" className={sectionToggleClass()} aria-expanded={outpaintOpen} aria-controls="generate-outpaint-controls" onClick={() => setOutpaintOpen((open) => !open)}>
<span>
<span className="block text-sm font-semibold text-white/85">Outpaint</span>
<span className="block text-xs text-white/40">Padding and feathering for outpaint mode</span>
</span>
{outpaintOpen ? <CaretUp size={18} weight="bold" /> : <CaretDown size={18} weight="bold" />}
</button>
<div id="generate-outpaint-controls" className={`grid gap-2 overflow-hidden transition-all duration-200 ${outpaintOpen ? "max-h-72 pt-2 opacity-100" : "max-h-0 opacity-0"}`}>
<div className="grid grid-cols-2 gap-2">
<PanelNumber label="Left" aria-label="Outpaint left padding" value={settings.outpaint.left} onValueChange={(left) => dispatch(commandIds.toolSetGenerateSettings, { outpaint: { ...settings.outpaint, left } })} />
<PanelNumber label="Top" aria-label="Outpaint top padding" value={settings.outpaint.top} onValueChange={(top) => dispatch(commandIds.toolSetGenerateSettings, { outpaint: { ...settings.outpaint, top } })} />
<PanelNumber label="Right" aria-label="Outpaint right padding" value={settings.outpaint.right} onValueChange={(right) => dispatch(commandIds.toolSetGenerateSettings, { outpaint: { ...settings.outpaint, right } })} />
<PanelNumber label="Bottom" aria-label="Outpaint bottom padding" value={settings.outpaint.bottom} onValueChange={(bottom) => dispatch(commandIds.toolSetGenerateSettings, { outpaint: { ...settings.outpaint, bottom } })} />
</div>
<PanelNumber label="Feather" aria-label="Outpaint feathering" value={settings.outpaint.feathering} onValueChange={(feathering) => dispatch(commandIds.toolSetGenerateSettings, { outpaint: { ...settings.outpaint, feathering } })} />
</div>
</section> : null}
{settings.mode === "inpaint" ? <section className={panelSectionClass()}>
<InpaintRegionPanel document={document} selection={selection} settings={settings} dispatch={dispatch} />
<button type="button" className={sectionToggleClass()} aria-expanded={inpaintOpen} aria-controls="generate-inpaint-controls" onClick={() => setInpaintOpen((open) => !open)}>
<span>
<span className="block text-sm font-semibold text-white/85">Inpaint</span>
<span className="block text-xs text-white/40">Mask polarity, crop padding, edge prep, grow</span>
</span>
{inpaintOpen ? <CaretUp size={18} weight="bold" /> : <CaretDown size={18} weight="bold" />}
</button>
<div id="generate-inpaint-controls" className={`grid gap-2 overflow-hidden transition-all duration-200 ${inpaintOpen ? "max-h-[38rem] pt-2 opacity-100" : "max-h-0 opacity-0"}`}>
<PanelSelect label="Intent" value={settings.inpaint.profile} options={inpaintProfileOptions} ariaLabel="Inpaint intent" onValueChange={(profile) => dispatch(commandIds.toolSetGenerateSettings, { inpaint: { profile } })} />
<PanelSelect
label="Content"
value={settings.inpaint.maskedContent}
options={inpaintMaskedContentOptions}
ariaLabel="Inpaint masked content"
onValueChange={(maskedContent) => dispatch(commandIds.toolSetGenerateSettings, { inpaint: { ...settings.inpaint, maskedContent } })}
/>
<PanelSelect label="Structure" value={settings.inpaint.structureControl} options={availableStructureControls} ariaLabel="Inpaint structure control" onValueChange={(structureControl) => dispatch(commandIds.toolSetGenerateSettings, { inpaint: { structureControl, profile: "custom" } })} />
{settings.inpaint.structureControl !== "none" ? (
<div className="grid grid-cols-2 gap-2">
<PanelNumber label="Control" aria-label="Structure control strength percent" value={Math.round(settings.inpaint.controlStrength * 100)} onValueChange={(controlStrength) => dispatch(commandIds.toolSetGenerateSettings, { inpaint: { controlStrength: controlStrength / 100, profile: "custom" } })} />
<PanelSelect label="Control model" value={settings.inpaint.controlModel} options={resolveGenerationStringOptions(comfyOptions?.controlModels, settings.inpaint.controlModel)} ariaLabel="Inpaint control model" onValueChange={(controlModel) => dispatch(commandIds.toolSetGenerateSettings, { inpaint: { controlModel } })} />
</div>
) : null}
<button type="button" className={compactRowButtonClass()} aria-pressed={settings.inpaint.maskedAreaOnly} onClick={() => dispatch(commandIds.toolSetGenerateSettings, { inpaint: { ...settings.inpaint, maskedAreaOnly: !settings.inpaint.maskedAreaOnly } })}>
<span className={panelLabelClass()}>Frame</span>
<span className="min-w-0 flex-1 text-right text-white">{settings.inpaint.maskedAreaOnly ? "Mask crop" : "Full layer"}</span>
</button>
<button type="button" className={compactRowButtonClass()} aria-pressed={settings.inpaint.colorMatch} onClick={() => dispatch(commandIds.toolSetGenerateSettings, { inpaint: { colorMatch: !settings.inpaint.colorMatch, profile: "custom" } })}>
<span className={panelLabelClass()}>Seam color</span><span className="min-w-0 flex-1 text-right text-white">{settings.inpaint.colorMatch ? "Match boundary" : "Preserve result"}</span>
</button>
<div className="grid grid-cols-2 gap-2">
<PanelNumber label="Pad" aria-label="Inpaint crop padding" value={settings.inpaint.cropPadding} onValueChange={(cropPadding) => dispatch(commandIds.toolSetGenerateSettings, { inpaint: { ...settings.inpaint, cropPadding } })} />
<PanelNumber label="Grow" aria-label="Inpaint backend grow mask" value={settings.inpaint.growMaskBy} onValueChange={(growMaskBy) => dispatch(commandIds.toolSetGenerateSettings, { inpaint: { ...settings.inpaint, growMaskBy } })} />
<PanelNumber label="Expand" aria-label="Inpaint mask expand" value={settings.inpaint.maskExpand} onValueChange={(maskExpand) => dispatch(commandIds.toolSetGenerateSettings, { inpaint: { ...settings.inpaint, maskExpand } })} />
<PanelNumber label="Feather" aria-label="Inpaint mask feather" value={settings.inpaint.maskFeather} onValueChange={(maskFeather) => dispatch(commandIds.toolSetGenerateSettings, { inpaint: { ...settings.inpaint, maskFeather } })} />
<PanelNumber label="Blur" aria-label="Inpaint mask blur" value={settings.inpaint.maskBlur} onValueChange={(maskBlur) => dispatch(commandIds.toolSetGenerateSettings, { inpaint: { ...settings.inpaint, maskBlur } })} />
<PanelNumber label="Clean" aria-label="Inpaint mask despeckle" value={settings.inpaint.maskDespeckle} onValueChange={(maskDespeckle) => dispatch(commandIds.toolSetGenerateSettings, { inpaint: { ...settings.inpaint, maskDespeckle } })} />
</div>
</div>
</section> : null}
</div>
);
}
const intentOptions: ReadonlyArray<{ value: GenerateIntent; mode: GenerateMode; label: string; description: string }> = [
{ value: "create", mode: "text-to-image", label: "Create", description: "Make a new image from your prompt." },
{ value: "replace", mode: "inpaint", label: "Replace", description: "Regenerate the masked part of one layer." },
{ value: "remove", mode: "inpaint", label: "Remove", description: "Erase an object and reconstruct its background." },
{ value: "extend", mode: "outpaint", label: "Extend", description: "Grow one selected image beyond its edges." },
{ value: "variations", mode: "image-to-image", label: "Variations", description: "Explore alternatives based on one image." },
];
function isIntentActive(intent: GenerateIntent, mode: GenerateMode, settings: GenerateSettings) {
if (settings.mode !== mode) return false;
if (intent === "remove") return settings.inpaint.profile === "remove";
if (intent === "replace") return settings.inpaint.profile !== "remove";
return true;
}
function SectionTitle({ title }: { title: string }) {
return <div className="px-1 text-xs font-semibold uppercase tracking-[0.18em] text-white/35">{title}</div>;
}
function panelSectionClass() {
return "grid gap-2 border-b border-white/[0.07] px-0.5 pb-3 last:border-b-0 last:pb-0";
}
function sectionToggleClass() {
return "flex w-full items-center justify-between gap-2 rounded-md px-1 py-0.5 text-left transition hover:bg-white/[0.04] focus:outline-none focus-visible:ring-2 focus-visible:ring-sky-300/50";
}
function PanelSelect<TValue extends string>({ label, value, options, ariaLabel, onValueChange }: { label: string; value: TValue; options: readonly BottomControlSelectOption<TValue>[]; ariaLabel: string; onValueChange: (value: TValue) => void }) {
return (
<div className="p-0.5">
<BottomControlSelectMenu label={label} value={value} options={options} aria-label={ariaLabel} placement="inline" onValueChange={onValueChange} />
</div>
);
}
function PanelNumber({ label, value, onValueChange, ...props }: { label: string; "aria-label": string; value: number; onValueChange: (value: number) => void }) {
return (
<label className={panelRowClass()}>
<span className={panelLabelClass()}>{label}</span>
<NumberInput {...props} value={value} onValueChange={onValueChange} />
</label>
);
}
function SizeControl({ refRoot, open, setOpen, settings, dispatch }: { refRoot: RefObject<HTMLDivElement | null>; open: boolean; setOpen: (open: boolean) => void; settings: GenerateSettings; dispatch: AppStore["dispatch"] }) {
return (
<div ref={refRoot} className="relative">
<button type="button" className={compactRowButtonClass()} aria-label={`Generate size ${settings.width} by ${settings.height}`} aria-expanded={open} onClick={() => setOpen(!open)}>
<span className={panelLabelClass()}>Size</span>
<span className="min-w-0 flex-1 text-right text-white">{settings.width} × {settings.height}</span>
<CaretDown size={16} weight="bold" />
</button>
{open ? (
<div className="app-surface absolute right-0 top-full z-30 mt-1.5 grid w-full gap-2 rounded-lg p-2.5 text-white shadow-2xl">
<div className="grid grid-cols-2 gap-2">
<PanelNumber label="W" aria-label="Generate width" value={settings.width} onValueChange={(width) => dispatch(commandIds.toolSetGenerateSettings, { width })} />
<PanelNumber label="H" aria-label="Generate height" value={settings.height} onValueChange={(height) => dispatch(commandIds.toolSetGenerateSettings, { height })} />
</div>
<div className="grid grid-cols-5 gap-2">
{sizePresets.map((preset) => (
<button key={preset.label} type="button" className="rounded-md bg-white/[0.04] px-2 py-1.5 text-xs text-white/65 transition hover:bg-white/[0.08] hover:text-white focus:outline-none focus-visible:ring-2 focus-visible:ring-sky-300/50" onClick={() => dispatch(commandIds.toolSetGenerateSettings, { width: preset.w, height: preset.h })}>
{preset.label}
</button>
))}
</div>
</div>
) : null}
</div>
);
}
function NumberInput({ value, onValueChange, ...props }: { "aria-label": string; value: number; onValueChange: (value: number) => void }) {
const [draft, setDraft] = useState(String(Math.round(value)));
useEffect(() => setDraft(String(Math.round(value))), [value]);
const commit = () => {
const next = Number(draft.trim());
if (draft.trim() !== "" && Number.isFinite(next)) onValueChange(next);
else setDraft(String(Math.round(value)));
};
return <input {...props} type="text" inputMode="numeric" value={draft} className="h-7 w-12 rounded bg-transparent px-1.5 text-right font-mono text-xs text-white/90 outline-none transition hover:bg-white/[0.04] focus:bg-white/[0.06] focus:ring-1 focus:ring-sky-300/50" onFocus={(event) => event.currentTarget.select()} onChange={(event) => setDraft(event.currentTarget.value)} onBlur={commit} onKeyDown={(event) => { event.stopPropagation(); if (event.key === "Enter") event.currentTarget.blur(); if (event.key === "Escape") { setDraft(String(Math.round(value))); event.currentTarget.blur(); } }} />;
}
function panelRowClass() {
return "flex min-h-9 items-center justify-between gap-2 border-b border-white/[0.045] px-1 last:border-b-0";
}
function compactRowButtonClass() {
return `${panelRowClass()} w-full text-left transition hover:bg-white/[0.07] focus:outline-none focus-visible:ring-2 focus-visible:ring-white/30`;
}
function panelLabelClass() {
return "shrink-0 text-sm font-medium text-white/55";
}
function panelTextAreaClass(extra = "") {
return `${extra} resize-none rounded-lg border border-white/[0.05] bg-white/[0.035] px-3 py-2 text-xs text-white outline-none transition placeholder:text-white/25 focus:bg-white/[0.06] focus:ring-1 focus:ring-sky-300/50`;
}