image-studio/operations/generation/candidateActions.ts

import type { ImageDocument } from "@core/document";
import type { GenerationCandidate } from "@editor/state";
import { loadImageCanvas, maskValueFromRgba } from "@platform/browser/maskRaster";
import { createContentRevision } from "./inpaintPrep";

export async function createMaskedPixelReplacementSource(document: ImageDocument, candidate: GenerationCandidate): Promise<string> {
  if (!candidate.inpaint) throw new Error("Only inpaint candidates can replace masked pixels.");

  const targetAsset = document.assets.find((asset) => asset.id === candidate.inpaint?.sourceAssetId);
  if (!targetAsset) throw new Error("The source layer for this candidate no longer exists.");
  const region = document.inpaintRegions.find((item) => item.id === candidate.inpaint?.regionId);
  const maskAsset = region ? document.assets.find((asset) => asset.id === region.maskAssetId) : undefined;
  if (!region || !maskAsset) throw new Error("The AI edit region for this candidate no longer exists.");
  const [sourceRevision, maskRevision] = await Promise.all([createContentRevision(targetAsset.source), createContentRevision(maskAsset.source)]);
  if (sourceRevision !== candidate.inpaint.revision.source || maskRevision !== candidate.inpaint.revision.mask) {
    throw new Error("The source or AI edit region changed after generation. Rebuild candidates from the current edit region before replacing pixels.");
  }

  const targetCanvas = await loadImageCanvas(targetAsset.source, targetAsset.intrinsicSize.w, targetAsset.intrinsicSize.h);
  const generatedCanvas = await loadImageCanvas(candidate.source, candidate.width, candidate.height);
  const maskCanvas = await loadImageCanvas(candidate.inpaint.blendMaskImage, candidate.width, candidate.height);

  const targetContext = require2dContext(targetCanvas);
  const generatedContext = require2dContext(generatedCanvas);
  const maskContext = require2dContext(maskCanvas);
  const targetData = targetContext.getImageData(0, 0, targetCanvas.width, targetCanvas.height);
  const generatedData = generatedContext.getImageData(0, 0, generatedCanvas.width, generatedCanvas.height);
  const maskData = maskContext.getImageData(0, 0, maskCanvas.width, maskCanvas.height);
  const crop = candidate.inpaint.crop.assetBounds;
  const colorOffset = candidate.settings.inpaint.colorMatch ? boundaryColorOffset(targetData.data, generatedData.data, maskData.data, candidate.width, candidate.height, crop, targetCanvas.width, targetCanvas.height) : [0, 0, 0];

  for (let y = 0; y < candidate.height; y += 1) {
    for (let x = 0; x < candidate.width; x += 1) {
      const targetX = Math.round(crop.x) + x;
      const targetY = Math.round(crop.y) + y;
      if (targetX < 0 || targetY < 0 || targetX >= targetCanvas.width || targetY >= targetCanvas.height) continue;

      const generatedIndex = (y * generatedCanvas.width + x) * 4;
      const targetIndex = (targetY * targetCanvas.width + targetX) * 4;
      const mask = maskValueFromRgba(maskData.data, generatedIndex) / 255;
      if (mask <= 0) continue;

      for (let channel = 0; channel < 4; channel += 1) {
        const previous = targetData.data[targetIndex + channel] ?? 0;
        const rawNext = generatedData.data[generatedIndex + channel] ?? previous;
        const next = channel < 3 ? Math.max(0, Math.min(255, rawNext + (colorOffset[channel] ?? 0))) : rawNext;
        targetData.data[targetIndex + channel] = Math.round(previous * (1 - mask) + next * mask);
      }
    }
  }

  targetContext.putImageData(targetData, 0, 0);
  return targetCanvas.toDataURL("image/png");
}

function boundaryColorOffset(target: Uint8ClampedArray, generated: Uint8ClampedArray, mask: Uint8ClampedArray, width: number, height: number, crop: { x: number; y: number }, targetWidth: number, targetHeight: number): number[] {
  const targetTotal = [0, 0, 0];
  const generatedTotal = [0, 0, 0];
  let count = 0;
  for (let y = 0; y < height; y += 1) for (let x = 0; x < width; x += 1) {
    const index = (y * width + x) * 4;
    const amount = maskValueFromRgba(mask, index) / 255;
    if (amount <= 0.05 || amount >= 0.65) continue;
    const targetX = Math.round(crop.x) + x;
    const targetY = Math.round(crop.y) + y;
    if (targetX < 0 || targetY < 0 || targetX >= targetWidth || targetY >= targetHeight) continue;
    const targetIndex = (targetY * targetWidth + targetX) * 4;
    for (let channel = 0; channel < 3; channel += 1) {
      targetTotal[channel] = (targetTotal[channel] ?? 0) + (target[targetIndex + channel] ?? 0);
      generatedTotal[channel] = (generatedTotal[channel] ?? 0) + (generated[index + channel] ?? 0);
    }
    count += 1;
  }
  if (count < 16) return [0, 0, 0];
  return targetTotal.map((total, channel) => Math.max(-32, Math.min(32, total / count - (generatedTotal[channel] ?? 0) / count)));
}

function require2dContext(canvas: HTMLCanvasElement): CanvasRenderingContext2D {
  const context = canvas.getContext("2d");
  if (!context) throw new Error("Unable to prepare generated candidate");
  return context;
}