image-studio/renderer/layers.ts

import type { ImageDocument } from "@core/document";
import type { Layer } from "@core/layer";
import type { EditorState, MaskViewMode, ViewportState } from "@editor/state";
import { resolveTransformTargetBounds } from "@editor/transform-targets";
import { clearScreenRect } from "./clear-rect";
import type { ImageTextureRenderer } from "./image-textures";
import { documentRectToScreenRect } from "./screen-rect";
import type { RgbaColor, ScreenRect, WebGlRendererContext } from "./types";

const imageLayerColor: RgbaColor = [0.38, 0.42, 0.5, 1];
const imageLayerInsetColor: RgbaColor = [0.48, 0.54, 0.64, 1];
const hiddenMaskOverlayColor: RgbaColor = [1, 0.08, 0.08, 0.45];
const maskRevealPreviewOpacity = 0.28;

export function renderLayers(context: WebGlRendererContext, document: ImageDocument, editor: EditorState, imageTextureRenderer: ImageTextureRenderer) {
  for (const artboard of document.artboards) {
    if (!artboard.visible) continue;
    const clipRect = documentRectToScreenRect(context.canvas, artboard.bounds, editor.viewport);
    const maskLayerIds = collectMaskLayerIds(artboard.layers);
    for (const layer of artboard.layers) renderLayer(context, document, editor, layer, imageTextureRenderer, clipRect, maskLayerIds);
  }
}

function renderLayer(
  context: WebGlRendererContext,
  document: ImageDocument,
  editor: EditorState,
  layer: Layer,
  imageTextureRenderer: ImageTextureRenderer,
  clipRect: ScreenRect,
  maskLayerIds: ReadonlySet<string>,
) {
  const editingMaskLayer = editor.maskEdit?.maskLayerId === layer.id;
  const maskViewMode = editor.maskEdit?.viewMode ?? "composite";
  const isolatedMaskView = isIsolatedMaskView(maskViewMode);
  if (!layer.visible || maskLayerIds.has(layer.id)) return;

  const effectiveClipRect = resolveLayerClipRect(context, document, editor.viewport, layer, clipRect);
  if (!effectiveClipRect) return;

  if (layer.type === "group") {
    for (const child of layer.children) renderLayer(context, document, editor, child, imageTextureRenderer, effectiveClipRect, maskLayerIds);
    return;
  }

  if (isolatedMaskView && layer.id !== editor.maskEdit?.targetLayerId) return;

  const bounds = resolveTransformTargetBounds(document, { type: "layer", id: layer.id });
  if (!bounds) return;

  const rect = documentRectToScreenRect(context.canvas, bounds, editor.viewport);
  const asset = assetWithBrushStrokePreview(document.assets.find((candidate) => candidate.id === layer.assetId), editor);
  const maskLayer = !editingMaskLayer && layer.clippingMask ? findLayer(document, layer.clippingMask.maskLayerId) : undefined;
  const maskAsset = assetWithBrushStrokePreview(maskLayer && maskLayer.type !== "group" ? document.assets.find((candidate) => candidate.id === maskLayer.assetId) : undefined, editor);
  const maskBounds = maskLayer ? resolveTransformTargetBounds(document, { type: "layer", id: maskLayer.id }) : undefined;
  const maskRect = maskBounds ? documentRectToScreenRect(context.canvas, maskBounds, editor.viewport) : undefined;
  const activeMaskTarget = Boolean(editor.maskEdit?.targetLayerId === layer.id && editor.maskEdit.maskLayerId === layer.clippingMask?.maskLayerId);
  const showMaskRevealPreview = editor.tools.activeTool === "brush" && activeMaskTarget && maskViewMode === "composite";

  if (asset && maskAsset && maskRect && activeMaskTarget) {
    if (maskViewMode === "blackWhite" && imageTextureRenderer.renderMaskVisualization(maskAsset, maskRect, "blackWhite", undefined, effectiveClipRect)) return;
    if (maskViewMode === "alpha" && imageTextureRenderer.renderMaskVisualization(maskAsset, maskRect, "alpha", undefined, effectiveClipRect)) return;
    if (maskViewMode === "overlay" && imageTextureRenderer.render(asset, rect, effectiveClipRect)) {
      imageTextureRenderer.renderMaskVisualization(maskAsset, maskRect, "hiddenOverlay", hiddenMaskOverlayColor, effectiveClipRect);
      return;
    }
  }

  if (asset && maskAsset && maskRect && imageTextureRenderer.renderMasked(asset, rect, maskAsset, maskRect, effectiveClipRect)) {
    if (showMaskRevealPreview) imageTextureRenderer.renderMaskRevealPreview(asset, rect, maskAsset, maskRect, maskRevealPreviewOpacity, effectiveClipRect);
    return;
  }
  if (asset && imageTextureRenderer.render(asset, rect, effectiveClipRect)) return;

  const fallbackRect = intersectScreenRects(rect, effectiveClipRect);
  if (!fallbackRect) return;
  clearScreenRect(context, fallbackRect, imageLayerColor);
  const insetRect = intersectScreenRects({ x: rect.x + 4, y: rect.y + 4, w: Math.max(0, rect.w - 8), h: Math.max(0, rect.h - 8) }, effectiveClipRect);
  if (insetRect) clearScreenRect(context, insetRect, imageLayerInsetColor);
}

function resolveLayerClipRect(
  context: WebGlRendererContext,
  document: ImageDocument,
  viewport: ViewportState,
  layer: Layer,
  clipRect: ScreenRect,
): ScreenRect | undefined {
  if (!layer.clippingMask) return clipRect;

  const maskBounds = resolveTransformTargetBounds(document, { type: "layer", id: layer.clippingMask.maskLayerId });
  if (!maskBounds) return clipRect;

  return intersectScreenRects(clipRect, documentRectToScreenRect(context.canvas, maskBounds, viewport));
}

function isIsolatedMaskView(mode: MaskViewMode) {
  return mode === "blackWhite" || mode === "alpha" || mode === "overlay";
}

function assetWithBrushStrokePreview<TAsset extends ImageDocument["assets"][number] | undefined>(asset: TAsset, editor: EditorState): TAsset {
  if (!asset || editor.brushStrokePreview?.assetId !== asset.id) return asset;
  return { ...asset, source: editor.brushStrokePreview.source } as TAsset;
}

function findLayer(document: ImageDocument, layerId: string): Layer | undefined {
  for (const artboard of document.artboards) {
    const layer = findLayerInTree(artboard.layers, layerId);
    if (layer) return layer;
  }
  return undefined;
}

function findLayerInTree(layers: readonly Layer[], layerId: string): Layer | undefined {
  for (const layer of layers) {
    if (layer.id === layerId) return layer;
    if (layer.type === "group") {
      const child = findLayerInTree(layer.children, layerId);
      if (child) return child;
    }
  }
  return undefined;
}

function collectMaskLayerIds(layers: readonly Layer[], ids = new Set<string>()) {
  for (const layer of layers) {
    if (layer.clippingMask) ids.add(layer.clippingMask.maskLayerId);
    if (layer.type === "group") collectMaskLayerIds(layer.children, ids);
  }
  return ids;
}

function intersectScreenRects(a: ScreenRect, b: ScreenRect): ScreenRect | undefined {
  const x1 = Math.max(a.x, b.x);
  const y1 = Math.max(a.y, b.y);
  const x2 = Math.min(a.x + a.w, b.x + b.w);
  const y2 = Math.min(a.y + a.h, b.y + b.h);
  if (x2 <= x1 || y2 <= y1) return undefined;

  return { x: x1, y: y1, w: x2 - x1, h: y2 - y1 };
}