image-studio/renderer/layers.ts

import type { Asset } from "@core/asset";
import type { ImageDocument } from "@core/document";
import type { Rect } from "@core/geometry";
import type { Layer } from "@core/layer";
import { createDocumentReadIndex, forEachLayerBackToFront, resolveIndexedLayerBounds, type DocumentReadIndex } from "@editor/document-indexes";
import type { EditorState, GenerationCandidate, MaskViewMode, ViewportState } from "@editor/state";
import { clearScreenRect } from "./clear-rect";
import type { ImageTextureRenderer } from "./image-textures";
import { documentRectToScreenRect } from "./screen-rect";
import type { RgbaColor, ScreenRect, WebGlRendererContext } from "./types";

const imageLayerColor: RgbaColor = [0.38, 0.42, 0.5, 1];
const imageLayerInsetColor: RgbaColor = [0.48, 0.54, 0.64, 1];
const hiddenMaskOverlayColor: RgbaColor = [1, 0.08, 0.08, 0.45];
const maskRevealPreviewOpacity = 0.28;

export function renderLayers(context: WebGlRendererContext, document: ImageDocument, editor: EditorState, imageTextureRenderer: ImageTextureRenderer) {
  const documentIndex = createDocumentReadIndex(document);
  const generationCandidate = selectedGenerationCandidate(editor);

  for (const artboard of document.artboards) {
    if (!artboard.visible) continue;
    const clipRect = documentRectToScreenRect(context.canvas, artboard.bounds, editor.viewport);
    const maskLayerIds = documentIndex.maskLayerIdsByArtboardId.get(artboard.id) ?? emptyLayerIds;
    forEachLayerBackToFront(artboard.layers, (layer) => renderLayer(context, documentIndex, editor, layer, imageTextureRenderer, clipRect, maskLayerIds));
    if (generationCandidate?.placement.artboardId === artboard.id) renderGenerationCandidatePreview(context, editor, generationCandidate, imageTextureRenderer, clipRect);
  }
}

export function generationCandidatePreviewAssets(editor: EditorState): Asset[] {
  return editor.generation.candidates.map((candidate) => generationCandidateAsset(candidate));
}

function renderLayer(
  context: WebGlRendererContext,
  documentIndex: DocumentReadIndex,
  editor: EditorState,
  layer: Layer,
  imageTextureRenderer: ImageTextureRenderer,
  clipRect: ScreenRect,
  maskLayerIds: ReadonlySet<string>,
) {
  const editingMaskLayer = editor.maskEdit?.maskLayerId === layer.id;
  const maskViewMode = editor.maskEdit?.viewMode ?? "composite";
  const isolatedMaskView = isIsolatedMaskView(maskViewMode);
  if (!layer.visible || maskLayerIds.has(layer.id)) return;

  const effectiveClipRect = resolveLayerClipRect(context, documentIndex, editor.viewport, layer, clipRect);
  if (!effectiveClipRect) return;

  if (layer.type === "group") {
    forEachLayerBackToFront(layer.children, (child) => renderLayer(context, documentIndex, editor, child, imageTextureRenderer, effectiveClipRect, maskLayerIds));
    return;
  }

  if (isolatedMaskView && layer.id !== editor.maskEdit?.targetLayerId) return;

  const bounds = resolveIndexedLayerBounds(documentIndex, layer);
  if (!bounds) return;

  const rect = documentRectToScreenRect(context.canvas, bounds, editor.viewport);
  const asset = assetWithBrushStrokePreview(documentIndex.assetById.get(layer.assetId), editor);
  const maskLayer = !editingMaskLayer && layer.clippingMask ? documentIndex.layerById.get(layer.clippingMask.maskLayerId) : undefined;
  const maskAsset = assetWithBrushStrokePreview(maskLayer && maskLayer.type !== "group" ? documentIndex.assetById.get(maskLayer.assetId) : undefined, editor);
  const maskBounds = maskLayer ? resolveIndexedLayerBounds(documentIndex, maskLayer) : undefined;
  const maskRect = maskBounds ? documentRectToScreenRect(context.canvas, maskBounds, editor.viewport) : undefined;
  const activeMaskTarget = Boolean(editor.maskEdit?.targetLayerId === layer.id && editor.maskEdit.maskLayerId === layer.clippingMask?.maskLayerId);
  const showMaskRevealPreview = editor.tools.activeTool === "brush" && activeMaskTarget && maskViewMode === "composite";

  if (asset && maskAsset && maskRect && activeMaskTarget) {
    if (maskViewMode === "blackWhite" && imageTextureRenderer.renderMaskVisualization(maskAsset, maskRect, "blackWhite", undefined, effectiveClipRect)) return;
    if (maskViewMode === "alpha" && imageTextureRenderer.renderMaskVisualization(maskAsset, maskRect, "alpha", undefined, effectiveClipRect)) return;
    if (maskViewMode === "overlay" && imageTextureRenderer.render(asset, rect, effectiveClipRect)) {
      imageTextureRenderer.renderMaskVisualization(maskAsset, maskRect, "hiddenOverlay", hiddenMaskOverlayColor, effectiveClipRect);
      return;
    }
  }

  if (asset && maskAsset && maskRect && imageTextureRenderer.renderMasked(asset, rect, maskAsset, maskRect, effectiveClipRect)) {
    if (showMaskRevealPreview) imageTextureRenderer.renderMaskRevealPreview(asset, rect, maskAsset, maskRect, maskRevealPreviewOpacity, effectiveClipRect);
    return;
  }
  if (asset && imageTextureRenderer.render(asset, rect, effectiveClipRect)) return;

  const fallbackRect = intersectScreenRects(rect, effectiveClipRect);
  if (!fallbackRect) return;
  clearScreenRect(context, fallbackRect, imageLayerColor);
  const insetRect = intersectScreenRects({ x: rect.x + 4, y: rect.y + 4, w: Math.max(0, rect.w - 8), h: Math.max(0, rect.h - 8) }, effectiveClipRect);
  if (insetRect) clearScreenRect(context, insetRect, imageLayerInsetColor);
}

const emptyLayerIds = new Set<string>();

function resolveLayerClipRect(
  context: WebGlRendererContext,
  documentIndex: DocumentReadIndex,
  viewport: ViewportState,
  layer: Layer,
  clipRect: ScreenRect,
): ScreenRect | undefined {
  if (!layer.clippingMask) return clipRect;

  const maskBounds = resolveIndexedLayerBounds(documentIndex, layer.clippingMask.maskLayerId);
  if (!maskBounds) return clipRect;

  return intersectScreenRects(clipRect, documentRectToScreenRect(context.canvas, maskBounds, viewport));
}

function isIsolatedMaskView(mode: MaskViewMode) {
  return mode === "blackWhite" || mode === "alpha" || mode === "overlay";
}

function renderGenerationCandidatePreview(
  context: WebGlRendererContext,
  editor: EditorState,
  candidate: GenerationCandidate,
  imageTextureRenderer: ImageTextureRenderer,
  clipRect: ScreenRect,
) {
  const rect = documentRectToScreenRect(context.canvas, generationCandidateBounds(candidate), editor.viewport);
  imageTextureRenderer.render(generationCandidateAsset(candidate), rect, clipRect);
}

function selectedGenerationCandidate(editor: EditorState): GenerationCandidate | undefined {
  return editor.generation.candidates.find((candidate) => candidate.id === editor.generation.selectedCandidateId) ?? editor.generation.candidates[0];
}

function generationCandidateAsset(candidate: GenerationCandidate): Asset {
  return {
    id: generationCandidateAssetId(candidate.id),
    name: candidate.placement.layerName,
    mimeType: candidate.mimeType,
    source: candidate.source,
    intrinsicSize: candidate.intrinsicSize,
  };
}

function generationCandidateAssetId(candidateId: string) {
  return `generation-candidate:${candidateId}`;
}

function generationCandidateBounds(candidate: GenerationCandidate): Rect {
  return {
    x: candidate.placement.transform.position.x,
    y: candidate.placement.transform.position.y,
    w: candidate.intrinsicSize.w * candidate.placement.transform.scale.x,
    h: candidate.intrinsicSize.h * candidate.placement.transform.scale.y,
  };
}

function assetWithBrushStrokePreview<TAsset extends ImageDocument["assets"][number] | undefined>(asset: TAsset, editor: EditorState): TAsset {
  if (!asset || editor.brushStrokePreview?.assetId !== asset.id) return asset;
  return { ...asset, source: editor.brushStrokePreview.source } as TAsset;
}

function intersectScreenRects(a: ScreenRect, b: ScreenRect): ScreenRect | undefined {
  const x1 = Math.max(a.x, b.x);
  const y1 = Math.max(a.y, b.y);
  const x2 = Math.min(a.x + a.w, b.x + b.w);
  const y2 = Math.min(a.y + a.h, b.y + b.h);
  if (x2 <= x1 || y2 <= y1) return undefined;

  return { x: x1, y: y1, w: x2 - x1, h: y2 - y1 };
}