feat: enhance layer masking functionality and brush controls

- Refactor LayersSheet component to support mask editing state and improve layer visibility handling.
- Introduce functions to collect mask layer IDs and count display layers excluding masks.
- Update BrushControls to include mask view mode options and a Done button for exiting mask editing.
- Modify brush session handling to support brush previews when editing masks.
- Implement a new BrushPreviewRenderer for rendering brush strokes with visual feedback.
- Add document geometry utilities for transforming points and resolving layer bounds.
- Ensure proper cleanup of brush preview on pointer leave and other interactions.

renderer/brush-preview.ts

@@ -0,0 +1,216 @@
+import type { ImageDocument } from "@core/document";
+import type { Vec2D } from "@core/geometry";
+import type { Layer } from "@core/layer";
+import type { RasterLayer } from "@core/raster-layer";
+import type { EditorState } from "@editor/state";
+import type { RgbaColor, WebGlRendererContext } from "./types";
+
+export type BrushPreviewRenderer = {
+  render(document: ImageDocument, editor: EditorState): void;
+  dispose(): void;
+};
+
+const previewHaloColor: RgbaColor = [0, 0, 0, 0.55];
+const previewColor: RgbaColor = [1, 1, 1, 0.92];
+const previewFillOpacity = 0.08;
+
+export function createBrushPreviewRenderer(context: WebGlRendererContext): BrushPreviewRenderer {
+  const { gl } = context;
+  const program = createProgram(gl);
+  const positionLocation = gl.getAttribLocation(program, "a_position");
+  const canvasSizeLocation = gl.getUniformLocation(program, "u_canvasSize");
+  const centerLocation = gl.getUniformLocation(program, "u_center");
+  const radiusLocation = gl.getUniformLocation(program, "u_radius");
+  const hardnessLocation = gl.getUniformLocation(program, "u_hardness");
+  const colorLocation = gl.getUniformLocation(program, "u_color");
+  const ringWidthLocation = gl.getUniformLocation(program, "u_ringWidth");
+  const fillOpacityLocation = gl.getUniformLocation(program, "u_fillOpacity");
+  const positionBuffer = gl.createBuffer();
+
+  if (!canvasSizeLocation || !centerLocation || !radiusLocation || !hardnessLocation || !colorLocation || !ringWidthLocation || !fillOpacityLocation || !positionBuffer) {
+    throw new Error("Failed to create brush preview renderer");
+  }
+
+  return {
+    render(document, editor) {
+      const preview = resolveBrushPreview(document, editor, context.canvas);
+      if (!preview) return;
+
+      gl.disable(gl.SCISSOR_TEST);
+      gl.enable(gl.BLEND);
+      gl.blendFunc(gl.ONE, gl.ONE_MINUS_SRC_ALPHA);
+      gl.useProgram(program);
+
+      gl.uniform2f(canvasSizeLocation, context.canvas.width, context.canvas.height);
+      gl.uniform2f(centerLocation, preview.center.x, preview.center.y);
+      gl.uniform2f(radiusLocation, preview.radius.x, preview.radius.y);
+      gl.uniform1f(hardnessLocation, preview.hardness);
+
+      gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
+      gl.bufferData(gl.ARRAY_BUFFER, previewVertices(preview.center, preview.radius), gl.DYNAMIC_DRAW);
+      gl.enableVertexAttribArray(positionLocation);
+      gl.vertexAttribPointer(positionLocation, 2, gl.FLOAT, false, 0, 0);
+
+      drawPreviewPass(gl, colorLocation, ringWidthLocation, fillOpacityLocation, previewHaloColor, 3, 0);
+      drawPreviewPass(gl, colorLocation, ringWidthLocation, fillOpacityLocation, previewColor, 1.35, previewFillOpacity);
+
+      gl.disable(gl.BLEND);
+    },
+    dispose() {
+      gl.deleteBuffer(positionBuffer);
+      gl.deleteProgram(program);
+    },
+  };
+}
+
+function drawPreviewPass(
+  gl: WebGL2RenderingContext,
+  colorLocation: WebGLUniformLocation,
+  ringWidthLocation: WebGLUniformLocation,
+  fillOpacityLocation: WebGLUniformLocation,
+  color: RgbaColor,
+  ringWidth: number,
+  fillOpacity: number,
+) {
+  gl.uniform4fv(colorLocation, color);
+  gl.uniform1f(ringWidthLocation, ringWidth);
+  gl.uniform1f(fillOpacityLocation, fillOpacity);
+  gl.drawArrays(gl.TRIANGLES, 0, 6);
+}
+
+function resolveBrushPreview(document: ImageDocument, editor: EditorState, canvas: HTMLCanvasElement) {
+  if (!editor.brushPreview || (editor.tools.activeTool !== "brush" && editor.tools.activeTool !== "eraser")) return undefined;
+  if (editor.tools.interactionMode.type === "temporary-pan" || (editor.tools.interactionMode.type === "tool" && editor.tools.interactionMode.tool === "pan")) return undefined;
+
+  const layer = resolveBrushTargetLayer(document, editor);
+  if (!layer) return undefined;
+
+  const size = Math.max(1, editor.tools.brush.size);
+  const zoom = editor.viewport.zoom;
+  return {
+    center: documentPointToScreenPoint(canvas, editor.brushPreview.position, editor),
+    radius: {
+      x: Math.max(1, Math.abs(layer.transform.scale.x) * size * zoom * 0.5),
+      y: Math.max(1, Math.abs(layer.transform.scale.y) * size * zoom * 0.5),
+    },
+    hardness: Math.max(0, Math.min(1, editor.tools.brush.hardness / 100)),
+  };
+}
+
+function resolveBrushTargetLayer(document: ImageDocument, editor: EditorState): RasterLayer | undefined {
+  const editingMask = Boolean(editor.maskEdit);
+  const layerId = editor.maskEdit?.maskLayerId ?? editor.selection.layerIds[0];
+  if (!layerId) return undefined;
+
+  const layer = findRasterLayer(document.artboards.flatMap((artboard) => artboard.layers), layerId);
+  if (!layer || layer.locked || (!editingMask && !layer.visible)) return undefined;
+  return layer;
+}
+
+function documentPointToScreenPoint(canvas: HTMLCanvasElement, point: Vec2D, editor: EditorState): Vec2D {
+  return {
+    x: canvas.width / 2 + (point.x - editor.viewport.center.x) * editor.viewport.zoom,
+    y: canvas.height / 2 + (point.y - editor.viewport.center.y) * editor.viewport.zoom,
+  };
+}
+
+function previewVertices(center: Vec2D, radius: Vec2D) {
+  const padding = 5;
+  const x1 = center.x - radius.x - padding;
+  const x2 = center.x + radius.x + padding;
+  const y1 = center.y - radius.y - padding;
+  const y2 = center.y + radius.y + padding;
+
+  return new Float32Array([x1, y1, x2, y1, x1, y2, x1, y2, x2, y1, x2, y2]);
+}
+
+function findRasterLayer(layers: readonly Layer[], layerId: string): RasterLayer | undefined {
+  for (const layer of layers) {
+    if (layer.id === layerId && layer.type === "raster") return layer;
+    if (layer.type === "group") {
+      const child = findRasterLayer(layer.children, layerId);
+      if (child) return child;
+    }
+  }
+  return undefined;
+}
+
+function createProgram(gl: WebGL2RenderingContext) {
+  const vertexShader = compileShader(
+    gl,
+    gl.VERTEX_SHADER,
+    `#version 300 es
+    in vec2 a_position;
+    uniform vec2 u_canvasSize;
+    out vec2 v_position;
+    void main() {
+      vec2 clip = vec2((a_position.x / u_canvasSize.x) * 2.0 - 1.0, 1.0 - (a_position.y / u_canvasSize.y) * 2.0);
+      gl_Position = vec4(clip, 0.0, 1.0);
+      v_position = a_position;
+    }`,
+  );
+  const fragmentShader = compileShader(
+    gl,
+    gl.FRAGMENT_SHADER,
+    `#version 300 es
+    precision mediump float;
+    uniform vec2 u_center;
+    uniform vec2 u_radius;
+    uniform float u_hardness;
+    uniform vec4 u_color;
+    uniform float u_ringWidth;
+    uniform float u_fillOpacity;
+    in vec2 v_position;
+    out vec4 outColor;
+    void main() {
+      vec2 radius = max(u_radius, vec2(1.0));
+      float minimumRadius = max(1.0, min(radius.x, radius.y));
+      float normalizedDistance = length((v_position - u_center) / radius);
+      float outerDistance = abs(normalizedDistance - 1.0) * minimumRadius;
+      float outerRing = 1.0 - smoothstep(max(0.0, u_ringWidth - 1.0), u_ringWidth + 1.0, outerDistance);
+
+      float innerRing = 0.0;
+      if (u_hardness > 0.05 && u_hardness < 0.98) {
+        float innerDistance = abs(normalizedDistance - u_hardness) * minimumRadius;
+        innerRing = 0.45 * (1.0 - smoothstep(max(0.0, u_ringWidth - 1.0), u_ringWidth + 1.0, innerDistance));
+      }
+
+      float fillStart = min(u_hardness, 0.98);
+      float fill = normalizedDistance <= 1.0 ? 1.0 - smoothstep(fillStart, 1.0, normalizedDistance) : 0.0;
+      float alpha = max(max(outerRing, innerRing) * u_color.a, fill * u_fillOpacity);
+      if (alpha <= 0.001) discard;
+      outColor = vec4(u_color.rgb * alpha, alpha);
+    }`,
+  );
+  const program = gl.createProgram();
+  if (!program) throw new Error("Failed to create brush preview shader program");
+
+  gl.attachShader(program, vertexShader);
+  gl.attachShader(program, fragmentShader);
+  gl.linkProgram(program);
+  gl.deleteShader(vertexShader);
+  gl.deleteShader(fragmentShader);
+
+  if (!gl.getProgramParameter(program, gl.LINK_STATUS)) {
+    const message = gl.getProgramInfoLog(program) ?? "Unknown brush preview program link error";
+    gl.deleteProgram(program);
+    throw new Error(message);
+  }
+
+  return program;
+}
+
+function compileShader(gl: WebGL2RenderingContext, type: number, source: string) {
+  const shader = gl.createShader(type);
+  if (!shader) throw new Error("Failed to create shader");
+
+  gl.shaderSource(shader, source);
+  gl.compileShader(shader);
+  if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) {
+    const message = gl.getShaderInfoLog(shader) ?? "Unknown shader compile error";
+    gl.deleteShader(shader);
+    throw new Error(message);
+  }
+
+  return shader;
+}

renderer/image-textures.ts

@@ -1,10 +1,14 @@
 import type { Asset } from "@core/asset";
 import type { RgbaColor, ScreenRect, WebGlRendererContext } from "./types";
 
+export type MaskVisualizationMode = "blackWhite" | "alpha" | "hiddenOverlay";
+
 export type ImageTextureRenderer = {
   syncAssets(assets: readonly Asset[]): void;
   render(asset: Asset, rect: ScreenRect, clipRect?: ScreenRect): boolean;
   renderMasked(asset: Asset, rect: ScreenRect, maskAsset: Asset, maskRect: ScreenRect, clipRect?: ScreenRect): boolean;
+  renderMaskRevealPreview(asset: Asset, rect: ScreenRect, maskAsset: Asset, maskRect: ScreenRect, opacity: number, clipRect?: ScreenRect): boolean;
+  renderMaskVisualization(maskAsset: Asset, maskRect: ScreenRect, mode: MaskVisualizationMode, color?: RgbaColor, clipRect?: ScreenRect): boolean;
   renderTinted(asset: Asset, rect: ScreenRect, color: RgbaColor, clipRect?: ScreenRect): boolean;
   dispose(): void;
 };
@@ -20,6 +24,15 @@ type TextureEntry = {
   previousTexture?: WebGLTexture;
 };
 
+type MaskVisualizationResources = {
+  program: WebGLProgram;
+  positionLocation: number;
+  texCoordLocation: number;
+  samplerLocation: WebGLUniformLocation;
+  modeLocation: WebGLUniformLocation;
+  colorLocation: WebGLUniformLocation;
+};
+
 export function createImageTextureRenderer(context: WebGlRendererContext, invalidate: () => void): ImageTextureRenderer {
   const { gl } = context;
   const program = createProgram(gl);
@@ -32,6 +45,13 @@ export function createImageTextureRenderer(context: WebGlRendererContext, invali
   const maskedMaskTexCoordLocation = gl.getAttribLocation(maskedProgram, "a_maskTexCoord");
   const maskedSamplerLocation = gl.getUniformLocation(maskedProgram, "u_image");
   const maskedMaskSamplerLocation = gl.getUniformLocation(maskedProgram, "u_mask");
+  const maskRevealPreviewProgram = createMaskRevealPreviewProgram(gl);
+  const maskRevealPreviewPositionLocation = gl.getAttribLocation(maskRevealPreviewProgram, "a_position");
+  const maskRevealPreviewTexCoordLocation = gl.getAttribLocation(maskRevealPreviewProgram, "a_texCoord");
+  const maskRevealPreviewMaskTexCoordLocation = gl.getAttribLocation(maskRevealPreviewProgram, "a_maskTexCoord");
+  const maskRevealPreviewSamplerLocation = gl.getUniformLocation(maskRevealPreviewProgram, "u_image");
+  const maskRevealPreviewMaskSamplerLocation = gl.getUniformLocation(maskRevealPreviewProgram, "u_mask");
+  const maskRevealPreviewOpacityLocation = gl.getUniformLocation(maskRevealPreviewProgram, "u_opacity");
   const tintedProgram = createTintedProgram(gl);
   const tintedPositionLocation = gl.getAttribLocation(tintedProgram, "a_position");
   const tintedTexCoordLocation = gl.getAttribLocation(tintedProgram, "a_texCoord");
@@ -41,9 +61,22 @@ export function createImageTextureRenderer(context: WebGlRendererContext, invali
   const texCoordBuffer = gl.createBuffer();
   const maskTexCoordBuffer = gl.createBuffer();
   const textures = new Map<string, TextureEntry>();
+  let maskVisualizationResources: MaskVisualizationResources | "failed" | undefined;
   let disposed = false;
 
-  if (!positionBuffer || !texCoordBuffer || !maskTexCoordBuffer || !samplerLocation || !maskedSamplerLocation || !maskedMaskSamplerLocation || !tintedSamplerLocation || !tintedColorLocation) throw new Error("Failed to create image texture renderer");
+  if (
+    !positionBuffer ||
+    !texCoordBuffer ||
+    !maskTexCoordBuffer ||
+    !samplerLocation ||
+    !maskedSamplerLocation ||
+    !maskedMaskSamplerLocation ||
+    !maskRevealPreviewSamplerLocation ||
+    !maskRevealPreviewMaskSamplerLocation ||
+    !maskRevealPreviewOpacityLocation ||
+    !tintedSamplerLocation ||
+    !tintedColorLocation
+  ) throw new Error("Failed to create image texture renderer");
 
   gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer);
   gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([0, 0, 1, 0, 0, 1, 0, 1, 1, 0, 1, 1]), gl.STATIC_DRAW);
@@ -68,8 +101,7 @@ export function createImageTextureRenderer(context: WebGlRendererContext, invali
 
       gl.enable(gl.SCISSOR_TEST);
       gl.scissor(drawRect.x, context.canvas.height - drawRect.y - drawRect.h, drawRect.w, drawRect.h);
-      gl.enable(gl.BLEND);
-      gl.blendFunc(gl.SRC_ALPHA, gl.ONE_MINUS_SRC_ALPHA);
+      enablePremultipliedAlphaBlending(gl);
       gl.useProgram(program);
 
       gl.activeTexture(gl.TEXTURE0);
@@ -103,8 +135,7 @@ export function createImageTextureRenderer(context: WebGlRendererContext, invali
 
       gl.enable(gl.SCISSOR_TEST);
       gl.scissor(drawRect.x, context.canvas.height - drawRect.y - drawRect.h, drawRect.w, drawRect.h);
-      gl.enable(gl.BLEND);
-      gl.blendFunc(gl.SRC_ALPHA, gl.ONE_MINUS_SRC_ALPHA);
+      enablePremultipliedAlphaBlending(gl);
       gl.useProgram(maskedProgram);
 
       gl.activeTexture(gl.TEXTURE0);
@@ -133,6 +164,90 @@ export function createImageTextureRenderer(context: WebGlRendererContext, invali
       gl.disable(gl.BLEND);
       return true;
     },
+    renderMaskRevealPreview(asset, rect, maskAsset, maskRect, opacity, clipRect) {
+      const clampedOpacity = Math.max(0, Math.min(1, opacity));
+      if (clampedOpacity <= 0) return true;
+
+      const clippedRect = clipRect ? intersectScreenRects(rect, clipRect) : rect;
+      const drawRect = clippedRect ? intersectScreenRects(clippedRect, maskRect) : undefined;
+      if (!drawRect || drawRect.w <= 0 || drawRect.h <= 0) return true;
+
+      const entry = getTextureEntry(context, textures, asset, invalidate, () => disposed);
+      const maskEntry = getTextureEntry(context, textures, maskAsset, invalidate, () => disposed);
+      const texture = renderableTexture(entry);
+      const maskTexture = renderableTexture(maskEntry);
+      if (!texture || !maskTexture) return false;
+
+      gl.enable(gl.SCISSOR_TEST);
+      gl.scissor(drawRect.x, context.canvas.height - drawRect.y - drawRect.h, drawRect.w, drawRect.h);
+      enablePremultipliedAlphaBlending(gl);
+      gl.useProgram(maskRevealPreviewProgram);
+
+      gl.activeTexture(gl.TEXTURE0);
+      gl.bindTexture(gl.TEXTURE_2D, texture);
+      gl.uniform1i(maskRevealPreviewSamplerLocation, 0);
+      gl.activeTexture(gl.TEXTURE1);
+      gl.bindTexture(gl.TEXTURE_2D, maskTexture);
+      gl.uniform1i(maskRevealPreviewMaskSamplerLocation, 1);
+      gl.uniform1f(maskRevealPreviewOpacityLocation, clampedOpacity);
+
+      gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
+      gl.bufferData(gl.ARRAY_BUFFER, rectVertices(context.canvas, drawRect), gl.DYNAMIC_DRAW);
+      gl.enableVertexAttribArray(maskRevealPreviewPositionLocation);
+      gl.vertexAttribPointer(maskRevealPreviewPositionLocation, 2, gl.FLOAT, false, 0, 0);
+
+      gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer);
+      gl.bufferData(gl.ARRAY_BUFFER, texCoordsForRect(drawRect, rect), gl.DYNAMIC_DRAW);
+      gl.enableVertexAttribArray(maskRevealPreviewTexCoordLocation);
+      gl.vertexAttribPointer(maskRevealPreviewTexCoordLocation, 2, gl.FLOAT, false, 0, 0);
+
+      gl.bindBuffer(gl.ARRAY_BUFFER, maskTexCoordBuffer);
+      gl.bufferData(gl.ARRAY_BUFFER, texCoordsForRect(drawRect, maskRect), gl.DYNAMIC_DRAW);
+      gl.enableVertexAttribArray(maskRevealPreviewMaskTexCoordLocation);
+      gl.vertexAttribPointer(maskRevealPreviewMaskTexCoordLocation, 2, gl.FLOAT, false, 0, 0);
+
+      gl.drawArrays(gl.TRIANGLES, 0, 6);
+      gl.disable(gl.BLEND);
+      return true;
+    },
+    renderMaskVisualization(maskAsset, maskRect, mode, color = [1, 1, 1, 1], clipRect) {
+      const drawRect = clipRect ? intersectScreenRects(maskRect, clipRect) : maskRect;
+      if (!drawRect || drawRect.w <= 0 || drawRect.h <= 0) return true;
+
+      const resources = getMaskVisualizationResources(gl, () => maskVisualizationResources, (nextResources) => {
+        maskVisualizationResources = nextResources;
+      });
+      if (!resources) return false;
+
+      const maskEntry = getTextureEntry(context, textures, maskAsset, invalidate, () => disposed);
+      const maskTexture = renderableTexture(maskEntry);
+      if (!maskTexture) return false;
+
+      gl.enable(gl.SCISSOR_TEST);
+      gl.scissor(drawRect.x, context.canvas.height - drawRect.y - drawRect.h, drawRect.w, drawRect.h);
+      enablePremultipliedAlphaBlending(gl);
+      gl.useProgram(resources.program);
+
+      gl.activeTexture(gl.TEXTURE0);
+      gl.bindTexture(gl.TEXTURE_2D, maskTexture);
+      gl.uniform1i(resources.samplerLocation, 0);
+      gl.uniform1i(resources.modeLocation, maskVisualizationModeValue(mode));
+      gl.uniform4fv(resources.colorLocation, color);
+
+      gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
+      gl.bufferData(gl.ARRAY_BUFFER, rectVertices(context.canvas, drawRect), gl.DYNAMIC_DRAW);
+      gl.enableVertexAttribArray(resources.positionLocation);
+      gl.vertexAttribPointer(resources.positionLocation, 2, gl.FLOAT, false, 0, 0);
+
+      gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer);
+      gl.bufferData(gl.ARRAY_BUFFER, texCoordsForRect(drawRect, maskRect), gl.DYNAMIC_DRAW);
+      gl.enableVertexAttribArray(resources.texCoordLocation);
+      gl.vertexAttribPointer(resources.texCoordLocation, 2, gl.FLOAT, false, 0, 0);
+
+      gl.drawArrays(gl.TRIANGLES, 0, 6);
+      gl.disable(gl.BLEND);
+      return true;
+    },
     renderTinted(asset, rect, color, clipRect) {
       const drawRect = clipRect ? intersectScreenRects(rect, clipRect) : rect;
       if (!drawRect || drawRect.w <= 0 || drawRect.h <= 0) return true;
@@ -143,8 +258,7 @@ export function createImageTextureRenderer(context: WebGlRendererContext, invali
 
       gl.enable(gl.SCISSOR_TEST);
       gl.scissor(drawRect.x, context.canvas.height - drawRect.y - drawRect.h, drawRect.w, drawRect.h);
-      gl.enable(gl.BLEND);
-      gl.blendFunc(gl.SRC_ALPHA, gl.ONE_MINUS_SRC_ALPHA);
+      enablePremultipliedAlphaBlending(gl);
       gl.useProgram(tintedProgram);
 
       gl.activeTexture(gl.TEXTURE0);
@@ -174,6 +288,8 @@ export function createImageTextureRenderer(context: WebGlRendererContext, invali
       gl.deleteBuffer(maskTexCoordBuffer);
       gl.deleteProgram(program);
       gl.deleteProgram(maskedProgram);
+      gl.deleteProgram(maskRevealPreviewProgram);
+      if (maskVisualizationResources && maskVisualizationResources !== "failed") gl.deleteProgram(maskVisualizationResources.program);
       gl.deleteProgram(tintedProgram);
     },
   };
@@ -274,11 +390,17 @@ function createTexture(gl: WebGL2RenderingContext, image: HTMLImageElement) {
   gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
   gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
   gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, false);
+  gl.pixelStorei(gl.UNPACK_PREMULTIPLY_ALPHA_WEBGL, true);
   gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, image);
 
   return texture;
 }
 
+function enablePremultipliedAlphaBlending(gl: WebGL2RenderingContext) {
+  gl.enable(gl.BLEND);
+  gl.blendFunc(gl.ONE, gl.ONE_MINUS_SRC_ALPHA);
+}
+
 function fullTexCoords() {
   return new Float32Array([0, 0, 1, 0, 0, 1, 0, 1, 1, 0, 1, 1]);
 }
@@ -344,6 +466,108 @@ function createProgram(gl: WebGL2RenderingContext) {
   return program;
 }
 
+function getMaskVisualizationResources(
+  gl: WebGL2RenderingContext,
+  getResources: () => MaskVisualizationResources | "failed" | undefined,
+  setResources: (resources: MaskVisualizationResources | "failed") => void,
+): MaskVisualizationResources | undefined {
+  const currentResources = getResources();
+  if (currentResources === "failed") return undefined;
+  if (currentResources) return currentResources;
+
+  try {
+    const resources = createMaskVisualizationProgram(gl);
+    setResources(resources);
+    return resources;
+  } catch {
+    setResources("failed");
+    return undefined;
+  }
+}
+
+function maskVisualizationModeValue(mode: MaskVisualizationMode) {
+  switch (mode) {
+    case "blackWhite":
+      return 0;
+    case "alpha":
+      return 1;
+    case "hiddenOverlay":
+      return 2;
+  }
+}
+
+function createMaskVisualizationProgram(gl: WebGL2RenderingContext): MaskVisualizationResources {
+  const vertexShader = compileShader(
+    gl,
+    gl.VERTEX_SHADER,
+    `#version 300 es
+    in vec2 a_position;
+    in vec2 a_texCoord;
+    out vec2 v_texCoord;
+    void main() {
+      gl_Position = vec4(a_position, 0.0, 1.0);
+      v_texCoord = a_texCoord;
+    }`,
+  );
+  const fragmentShader = compileShader(
+    gl,
+    gl.FRAGMENT_SHADER,
+    `#version 300 es
+    precision mediump float;
+    uniform sampler2D u_mask;
+    uniform int u_mode;
+    uniform vec4 u_color;
+    in vec2 v_texCoord;
+    out vec4 outColor;
+    void main() {
+      float maskAlpha = texture(u_mask, v_texCoord).a;
+      if (u_mode == 0) {
+        float value = step(0.5, maskAlpha);
+        outColor = vec4(value, value, value, 1.0);
+        return;
+      }
+      if (u_mode == 1) {
+        outColor = vec4(maskAlpha, maskAlpha, maskAlpha, 1.0);
+        return;
+      }
+
+      float alpha = (1.0 - maskAlpha) * u_color.a;
+      outColor = vec4(u_color.rgb * alpha, alpha);
+    }`,
+  );
+  const program = gl.createProgram();
+  if (!program) throw new Error("Failed to create mask visualization shader program");
+
+  gl.attachShader(program, vertexShader);
+  gl.attachShader(program, fragmentShader);
+  gl.linkProgram(program);
+  gl.deleteShader(vertexShader);
+  gl.deleteShader(fragmentShader);
+
+  if (!gl.getProgramParameter(program, gl.LINK_STATUS)) {
+    const message = gl.getProgramInfoLog(program) ?? "Unknown mask visualization program link error";
+    gl.deleteProgram(program);
+    throw new Error(message);
+  }
+
+  const samplerLocation = gl.getUniformLocation(program, "u_mask");
+  const modeLocation = gl.getUniformLocation(program, "u_mode");
+  const colorLocation = gl.getUniformLocation(program, "u_color");
+  if (!samplerLocation || !modeLocation || !colorLocation) {
+    gl.deleteProgram(program);
+    throw new Error("Failed to resolve mask visualization shader uniforms");
+  }
+
+  return {
+    program,
+    positionLocation: gl.getAttribLocation(program, "a_position"),
+    texCoordLocation: gl.getAttribLocation(program, "a_texCoord"),
+    samplerLocation,
+    modeLocation,
+    colorLocation,
+  };
+}
+
 function createTintedProgram(gl: WebGL2RenderingContext) {
   const vertexShader = compileShader(
     gl,
@@ -367,8 +591,8 @@ function createTintedProgram(gl: WebGL2RenderingContext) {
     in vec2 v_texCoord;
     out vec4 outColor;
     void main() {
-      float maskAlpha = texture(u_image, v_texCoord).a;
-      outColor = vec4(u_color.rgb, u_color.a * maskAlpha);
+      float alpha = u_color.a * texture(u_image, v_texCoord).a;
+      outColor = vec4(u_color.rgb * alpha, alpha);
     }`,
   );
   const program = gl.createProgram();
@@ -389,6 +613,58 @@ function createTintedProgram(gl: WebGL2RenderingContext) {
   return program;
 }
 
+function createMaskRevealPreviewProgram(gl: WebGL2RenderingContext) {
+  const vertexShader = compileShader(
+    gl,
+    gl.VERTEX_SHADER,
+    `#version 300 es
+    in vec2 a_position;
+    in vec2 a_texCoord;
+    in vec2 a_maskTexCoord;
+    out vec2 v_texCoord;
+    out vec2 v_maskTexCoord;
+    void main() {
+      gl_Position = vec4(a_position, 0.0, 1.0);
+      v_texCoord = a_texCoord;
+      v_maskTexCoord = a_maskTexCoord;
+    }`,
+  );
+  const fragmentShader = compileShader(
+    gl,
+    gl.FRAGMENT_SHADER,
+    `#version 300 es
+    precision mediump float;
+    uniform sampler2D u_image;
+    uniform sampler2D u_mask;
+    uniform float u_opacity;
+    in vec2 v_texCoord;
+    in vec2 v_maskTexCoord;
+    out vec4 outColor;
+    void main() {
+      vec4 color = texture(u_image, v_texCoord);
+      float hiddenMaskAlpha = 1.0 - texture(u_mask, v_maskTexCoord).a;
+      float previewAlpha = clamp(hiddenMaskAlpha * u_opacity, 0.0, 1.0);
+      outColor = vec4(color.rgb * previewAlpha, color.a * previewAlpha);
+    }`,
+  );
+  const program = gl.createProgram();
+  if (!program) throw new Error("Failed to create mask reveal preview shader program");
+
+  gl.attachShader(program, vertexShader);
+  gl.attachShader(program, fragmentShader);
+  gl.linkProgram(program);
+  gl.deleteShader(vertexShader);
+  gl.deleteShader(fragmentShader);
+
+  if (!gl.getProgramParameter(program, gl.LINK_STATUS)) {
+    const message = gl.getProgramInfoLog(program) ?? "Unknown mask reveal preview program link error";
+    gl.deleteProgram(program);
+    throw new Error(message);
+  }
+
+  return program;
+}
+
 function createMaskedProgram(gl: WebGL2RenderingContext) {
   const vertexShader = compileShader(
     gl,
@@ -418,7 +694,8 @@ function createMaskedProgram(gl: WebGL2RenderingContext) {
     void main() {
       vec4 color = texture(u_image, v_texCoord);
       float maskAlpha = texture(u_mask, v_maskTexCoord).a;
-      outColor = vec4(color.rgb, color.a * maskAlpha);
+      float alpha = color.a * maskAlpha;
+      outColor = vec4(color.rgb * maskAlpha, alpha);
     }`,
   );
   const program = gl.createProgram();

renderer/layers.ts

@@ -1,6 +1,6 @@
 import type { ImageDocument } from "@core/document";
 import type { Layer } from "@core/layer";
-import type { EditorState, ViewportState } from "@editor/state";
+import type { EditorState, MaskViewMode, ViewportState } from "@editor/state";
 import { resolveTransformTargetBounds } from "@editor/transform-targets";
 import { clearScreenRect } from "./clear-rect";
 import type { ImageTextureRenderer } from "./image-textures";
@@ -9,6 +9,8 @@ import type { RgbaColor, ScreenRect, WebGlRendererContext } from "./types";
 
 const imageLayerColor: RgbaColor = [0.38, 0.42, 0.5, 1];
 const imageLayerInsetColor: RgbaColor = [0.48, 0.54, 0.64, 1];
+const hiddenMaskOverlayColor: RgbaColor = [1, 0.08, 0.08, 0.45];
+const maskRevealPreviewOpacity = 0.28;
 
 export function renderLayers(context: WebGlRendererContext, document: ImageDocument, editor: EditorState, imageTextureRenderer: ImageTextureRenderer) {
   for (const artboard of document.artboards) {
@@ -28,7 +30,9 @@ function renderLayer(
   clipRect: ScreenRect,
   maskLayerIds: ReadonlySet<string>,
 ) {
-  const editingMaskLayer = editor.maskEditLayerId === layer.id;
+  const editingMaskLayer = editor.maskEdit?.maskLayerId === layer.id;
+  const maskViewMode = editor.maskEdit?.viewMode ?? "composite";
+  const isolatedMaskView = isIsolatedMaskView(maskViewMode);
   if (!layer.visible || maskLayerIds.has(layer.id)) return;
 
   const effectiveClipRect = resolveLayerClipRect(context, document, editor.viewport, layer, clipRect);
@@ -39,16 +43,33 @@ function renderLayer(
     return;
   }
 
+  if (isolatedMaskView && layer.id !== editor.maskEdit?.targetLayerId) return;
+
   const bounds = resolveTransformTargetBounds(document, { type: "layer", id: layer.id });
   if (!bounds) return;
 
   const rect = documentRectToScreenRect(context.canvas, bounds, editor.viewport);
-  const asset = document.assets.find((candidate) => candidate.id === layer.assetId);
+  const asset = assetWithBrushStrokePreview(document.assets.find((candidate) => candidate.id === layer.assetId), editor);
   const maskLayer = !editingMaskLayer && layer.clippingMask ? findLayer(document, layer.clippingMask.maskLayerId) : undefined;
-  const maskAsset = maskLayer && maskLayer.type !== "group" ? document.assets.find((candidate) => candidate.id === maskLayer.assetId) : undefined;
+  const maskAsset = assetWithBrushStrokePreview(maskLayer && maskLayer.type !== "group" ? document.assets.find((candidate) => candidate.id === maskLayer.assetId) : undefined, editor);
   const maskBounds = maskLayer ? resolveTransformTargetBounds(document, { type: "layer", id: maskLayer.id }) : undefined;
   const maskRect = maskBounds ? documentRectToScreenRect(context.canvas, maskBounds, editor.viewport) : undefined;
-  if (asset && maskAsset && maskRect && imageTextureRenderer.renderMasked(asset, rect, maskAsset, maskRect, effectiveClipRect)) return;
+  const activeMaskTarget = Boolean(editor.maskEdit?.targetLayerId === layer.id && editor.maskEdit.maskLayerId === layer.clippingMask?.maskLayerId);
+  const showMaskRevealPreview = editor.tools.activeTool === "brush" && activeMaskTarget && maskViewMode === "composite";
+
+  if (asset && maskAsset && maskRect && activeMaskTarget) {
+    if (maskViewMode === "blackWhite" && imageTextureRenderer.renderMaskVisualization(maskAsset, maskRect, "blackWhite", undefined, effectiveClipRect)) return;
+    if (maskViewMode === "alpha" && imageTextureRenderer.renderMaskVisualization(maskAsset, maskRect, "alpha", undefined, effectiveClipRect)) return;
+    if (maskViewMode === "overlay" && imageTextureRenderer.render(asset, rect, effectiveClipRect)) {
+      imageTextureRenderer.renderMaskVisualization(maskAsset, maskRect, "hiddenOverlay", hiddenMaskOverlayColor, effectiveClipRect);
+      return;
+    }
+  }
+
+  if (asset && maskAsset && maskRect && imageTextureRenderer.renderMasked(asset, rect, maskAsset, maskRect, effectiveClipRect)) {
+    if (showMaskRevealPreview) imageTextureRenderer.renderMaskRevealPreview(asset, rect, maskAsset, maskRect, maskRevealPreviewOpacity, effectiveClipRect);
+    return;
+  }
   if (asset && imageTextureRenderer.render(asset, rect, effectiveClipRect)) return;
 
   const fallbackRect = intersectScreenRects(rect, effectiveClipRect);
@@ -68,11 +89,20 @@ function resolveLayerClipRect(
   if (!layer.clippingMask) return clipRect;
 
   const maskBounds = resolveTransformTargetBounds(document, { type: "layer", id: layer.clippingMask.maskLayerId });
-  if (!maskBounds) return undefined;
+  if (!maskBounds) return clipRect;
 
   return intersectScreenRects(clipRect, documentRectToScreenRect(context.canvas, maskBounds, viewport));
 }
 
+function isIsolatedMaskView(mode: MaskViewMode) {
+  return mode === "blackWhite" || mode === "alpha" || mode === "overlay";
+}
+
+function assetWithBrushStrokePreview<TAsset extends ImageDocument["assets"][number] | undefined>(asset: TAsset, editor: EditorState): TAsset {
+  if (!asset || editor.brushStrokePreview?.assetId !== asset.id) return asset;
+  return { ...asset, source: editor.brushStrokePreview.source } as TAsset;
+}
+
 function findLayer(document: ImageDocument, layerId: string): Layer | undefined {
   for (const artboard of document.artboards) {
     const layer = findLayerInTree(artboard.layers, layerId);

renderer/mask-edit-overlay.ts

@@ -1,40 +0,0 @@
-import type { ImageDocument } from "@core/document";
-import type { Layer } from "@core/layer";
-import type { EditorState } from "@editor/state";
-import { resolveTransformTargetBounds } from "@editor/transform-targets";
-import type { ImageTextureRenderer } from "./image-textures";
-import { documentRectToScreenRect } from "./screen-rect";
-import type { WebGlRendererContext } from "./types";
-
-const maskOverlayColor = [0.25, 0.65, 1, 0.35] as const;
-
-export function renderMaskEditOverlay(
-  context: WebGlRendererContext,
-  document: ImageDocument,
-  editor: EditorState,
-  imageTextureRenderer: ImageTextureRenderer,
-) {
-  const layerId = editor.maskEditLayerId;
-  if (!layerId) return;
-
-  const layer = findLayer(document.artboards.flatMap((artboard) => artboard.layers), layerId);
-  if (!layer || layer.type === "group") return;
-
-  const bounds = resolveTransformTargetBounds(document, { type: "layer", id: layer.id });
-  const asset = document.assets.find((candidate) => candidate.id === layer.assetId);
-  if (!bounds || !asset) return;
-
-  const rect = documentRectToScreenRect(context.canvas, bounds, editor.viewport);
-  imageTextureRenderer.renderTinted(asset, rect, maskOverlayColor);
-}
-
-function findLayer(layers: readonly Layer[], layerId: string): Layer | undefined {
-  for (const layer of layers) {
-    if (layer.id === layerId) return layer;
-    if (layer.type === "group") {
-      const child = findLayer(layer.children, layerId);
-      if (child) return child;
-    }
-  }
-  return undefined;
-}

renderer/renderer.ts

@@ -1,9 +1,9 @@
 import type { ImageDocument } from "@core/document";
 import type { EditorState } from "@editor/state";
 import { renderArtboard } from "./artboard";
+import { createBrushPreviewRenderer } from "./brush-preview";
 import { createImageTextureRenderer } from "./image-textures";
 import { renderLayers } from "./layers";
-import { renderMaskEditOverlay } from "./mask-edit-overlay";
 import { renderSelectionOverlay } from "./selection";
 import { renderTransformControls } from "./transform-controls";
 import type { WebGlRendererContext } from "./types";
@@ -31,6 +31,7 @@ export function createRenderer(canvas: HTMLCanvasElement, backend: RendererBacke
   }
 
   const rendererContext: WebGlRendererContext = { gl: context, canvas };
+  const brushPreviewRenderer = createOptionalBrushPreviewRenderer(rendererContext);
   let lastFrame: RenderFrame | undefined;
   let rerenderQueued = false;
   const imageTextureRenderer = createImageTextureRenderer(rendererContext, () => {
@@ -61,17 +62,28 @@ export function createRenderer(canvas: HTMLCanvasElement, backend: RendererBacke
       }
       imageTextureRenderer.syncAssets(frame.document.assets);
       renderLayers(rendererContext, frame.document, frame.editor, imageTextureRenderer);
-      renderMaskEditOverlay(rendererContext, frame.document, frame.editor, imageTextureRenderer);
 
-      renderSelectionOverlay(rendererContext, frame.document, frame.editor);
-      renderTransformControls(rendererContext, frame.document, frame.editor);
+      if (!frame.editor.maskEdit) {
+        renderSelectionOverlay(rendererContext, frame.document, frame.editor);
+        renderTransformControls(rendererContext, frame.document, frame.editor);
+      }
+      brushPreviewRenderer?.render(frame.document, frame.editor);
 
       context.disable(context.SCISSOR_TEST);
     },
     dispose() {
       imageTextureRenderer.dispose();
+      brushPreviewRenderer?.dispose();
     },
   };
 
   return renderer;
 }
+
+function createOptionalBrushPreviewRenderer(context: WebGlRendererContext) {
+  try {
+    return createBrushPreviewRenderer(context);
+  } catch {
+    return undefined;
+  }
+}