feat: enhance layer masking functionality and brush controls

- Refactor LayersSheet component to support mask editing state and improve layer visibility handling. - Introduce functions to collect mask layer IDs and count display layers excluding masks. - Update BrushControls to include mask view mode options and a Done button for exiting mask editing. - Modify brush session handling to support brush previews when editing masks. - Implement a new BrushPreviewRenderer for rendering brush strokes with visual feedback. - Add document geometry utilities for transforming points and resolving layer bounds. - Ensure proper cleanup of brush preview on pointer leave and other interactions.
2026-07-03 20:49:29 +02:00
parent daaa2a6667
commit 4ad0bb8b2c
33 changed files with 1889 additions and 351 deletions
--- a/renderer/image-textures.ts
+++ b/renderer/image-textures.ts
@@ -1,10 +1,14 @@
 import type { Asset } from "@core/asset";
 import type { RgbaColor, ScreenRect, WebGlRendererContext } from "./types";

+export type MaskVisualizationMode = "blackWhite" | "alpha" | "hiddenOverlay";
+
 export type ImageTextureRenderer = {
  syncAssets(assets: readonly Asset[]): void;
  render(asset: Asset, rect: ScreenRect, clipRect?: ScreenRect): boolean;
  renderMasked(asset: Asset, rect: ScreenRect, maskAsset: Asset, maskRect: ScreenRect, clipRect?: ScreenRect): boolean;
+  renderMaskRevealPreview(asset: Asset, rect: ScreenRect, maskAsset: Asset, maskRect: ScreenRect, opacity: number, clipRect?: ScreenRect): boolean;
+  renderMaskVisualization(maskAsset: Asset, maskRect: ScreenRect, mode: MaskVisualizationMode, color?: RgbaColor, clipRect?: ScreenRect): boolean;
  renderTinted(asset: Asset, rect: ScreenRect, color: RgbaColor, clipRect?: ScreenRect): boolean;
  dispose(): void;
 };
@@ -20,6 +24,15 @@ type TextureEntry = {
  previousTexture?: WebGLTexture;
 };

+type MaskVisualizationResources = {
+  program: WebGLProgram;
+  positionLocation: number;
+  texCoordLocation: number;
+  samplerLocation: WebGLUniformLocation;
+  modeLocation: WebGLUniformLocation;
+  colorLocation: WebGLUniformLocation;
+};
+
 export function createImageTextureRenderer(context: WebGlRendererContext, invalidate: () => void): ImageTextureRenderer {
  const { gl } = context;
  const program = createProgram(gl);
@@ -32,6 +45,13 @@ export function createImageTextureRenderer(context: WebGlRendererContext, invali
  const maskedMaskTexCoordLocation = gl.getAttribLocation(maskedProgram, "a_maskTexCoord");
  const maskedSamplerLocation = gl.getUniformLocation(maskedProgram, "u_image");
  const maskedMaskSamplerLocation = gl.getUniformLocation(maskedProgram, "u_mask");
+  const maskRevealPreviewProgram = createMaskRevealPreviewProgram(gl);
+  const maskRevealPreviewPositionLocation = gl.getAttribLocation(maskRevealPreviewProgram, "a_position");
+  const maskRevealPreviewTexCoordLocation = gl.getAttribLocation(maskRevealPreviewProgram, "a_texCoord");
+  const maskRevealPreviewMaskTexCoordLocation = gl.getAttribLocation(maskRevealPreviewProgram, "a_maskTexCoord");
+  const maskRevealPreviewSamplerLocation = gl.getUniformLocation(maskRevealPreviewProgram, "u_image");
+  const maskRevealPreviewMaskSamplerLocation = gl.getUniformLocation(maskRevealPreviewProgram, "u_mask");
+  const maskRevealPreviewOpacityLocation = gl.getUniformLocation(maskRevealPreviewProgram, "u_opacity");
  const tintedProgram = createTintedProgram(gl);
  const tintedPositionLocation = gl.getAttribLocation(tintedProgram, "a_position");
  const tintedTexCoordLocation = gl.getAttribLocation(tintedProgram, "a_texCoord");
@@ -41,9 +61,22 @@ export function createImageTextureRenderer(context: WebGlRendererContext, invali
  const texCoordBuffer = gl.createBuffer();
  const maskTexCoordBuffer = gl.createBuffer();
  const textures = new Map<string, TextureEntry>();
+  let maskVisualizationResources: MaskVisualizationResources | "failed" | undefined;
  let disposed = false;

-  if (!positionBuffer || !texCoordBuffer || !maskTexCoordBuffer || !samplerLocation || !maskedSamplerLocation || !maskedMaskSamplerLocation || !tintedSamplerLocation || !tintedColorLocation) throw new Error("Failed to create image texture renderer");
+  if (
+    !positionBuffer ||
+    !texCoordBuffer ||
+    !maskTexCoordBuffer ||
+    !samplerLocation ||
+    !maskedSamplerLocation ||
+    !maskedMaskSamplerLocation ||
+    !maskRevealPreviewSamplerLocation ||
+    !maskRevealPreviewMaskSamplerLocation ||
+    !maskRevealPreviewOpacityLocation ||
+    !tintedSamplerLocation ||
+    !tintedColorLocation
+  ) throw new Error("Failed to create image texture renderer");

  gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer);
  gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([0, 0, 1, 0, 0, 1, 0, 1, 1, 0, 1, 1]), gl.STATIC_DRAW);
@@ -68,8 +101,7 @@ export function createImageTextureRenderer(context: WebGlRendererContext, invali

      gl.enable(gl.SCISSOR_TEST);
      gl.scissor(drawRect.x, context.canvas.height - drawRect.y - drawRect.h, drawRect.w, drawRect.h);
-      gl.enable(gl.BLEND);
-      gl.blendFunc(gl.SRC_ALPHA, gl.ONE_MINUS_SRC_ALPHA);
+      enablePremultipliedAlphaBlending(gl);
      gl.useProgram(program);

      gl.activeTexture(gl.TEXTURE0);
@@ -103,8 +135,7 @@ export function createImageTextureRenderer(context: WebGlRendererContext, invali

      gl.enable(gl.SCISSOR_TEST);
      gl.scissor(drawRect.x, context.canvas.height - drawRect.y - drawRect.h, drawRect.w, drawRect.h);
-      gl.enable(gl.BLEND);
-      gl.blendFunc(gl.SRC_ALPHA, gl.ONE_MINUS_SRC_ALPHA);
+      enablePremultipliedAlphaBlending(gl);
      gl.useProgram(maskedProgram);

      gl.activeTexture(gl.TEXTURE0);
@@ -133,6 +164,90 @@ export function createImageTextureRenderer(context: WebGlRendererContext, invali
      gl.disable(gl.BLEND);
      return true;
    },
+    renderMaskRevealPreview(asset, rect, maskAsset, maskRect, opacity, clipRect) {
+      const clampedOpacity = Math.max(0, Math.min(1, opacity));
+      if (clampedOpacity <= 0) return true;
+
+      const clippedRect = clipRect ? intersectScreenRects(rect, clipRect) : rect;
+      const drawRect = clippedRect ? intersectScreenRects(clippedRect, maskRect) : undefined;
+      if (!drawRect || drawRect.w <= 0 || drawRect.h <= 0) return true;
+
+      const entry = getTextureEntry(context, textures, asset, invalidate, () => disposed);
+      const maskEntry = getTextureEntry(context, textures, maskAsset, invalidate, () => disposed);
+      const texture = renderableTexture(entry);
+      const maskTexture = renderableTexture(maskEntry);
+      if (!texture || !maskTexture) return false;
+
+      gl.enable(gl.SCISSOR_TEST);
+      gl.scissor(drawRect.x, context.canvas.height - drawRect.y - drawRect.h, drawRect.w, drawRect.h);
+      enablePremultipliedAlphaBlending(gl);
+      gl.useProgram(maskRevealPreviewProgram);
+
+      gl.activeTexture(gl.TEXTURE0);
+      gl.bindTexture(gl.TEXTURE_2D, texture);
+      gl.uniform1i(maskRevealPreviewSamplerLocation, 0);
+      gl.activeTexture(gl.TEXTURE1);
+      gl.bindTexture(gl.TEXTURE_2D, maskTexture);
+      gl.uniform1i(maskRevealPreviewMaskSamplerLocation, 1);
+      gl.uniform1f(maskRevealPreviewOpacityLocation, clampedOpacity);
+
+      gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
+      gl.bufferData(gl.ARRAY_BUFFER, rectVertices(context.canvas, drawRect), gl.DYNAMIC_DRAW);
+      gl.enableVertexAttribArray(maskRevealPreviewPositionLocation);
+      gl.vertexAttribPointer(maskRevealPreviewPositionLocation, 2, gl.FLOAT, false, 0, 0);
+
+      gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer);
+      gl.bufferData(gl.ARRAY_BUFFER, texCoordsForRect(drawRect, rect), gl.DYNAMIC_DRAW);
+      gl.enableVertexAttribArray(maskRevealPreviewTexCoordLocation);
+      gl.vertexAttribPointer(maskRevealPreviewTexCoordLocation, 2, gl.FLOAT, false, 0, 0);
+
+      gl.bindBuffer(gl.ARRAY_BUFFER, maskTexCoordBuffer);
+      gl.bufferData(gl.ARRAY_BUFFER, texCoordsForRect(drawRect, maskRect), gl.DYNAMIC_DRAW);
+      gl.enableVertexAttribArray(maskRevealPreviewMaskTexCoordLocation);
+      gl.vertexAttribPointer(maskRevealPreviewMaskTexCoordLocation, 2, gl.FLOAT, false, 0, 0);
+
+      gl.drawArrays(gl.TRIANGLES, 0, 6);
+      gl.disable(gl.BLEND);
+      return true;
+    },
+    renderMaskVisualization(maskAsset, maskRect, mode, color = [1, 1, 1, 1], clipRect) {
+      const drawRect = clipRect ? intersectScreenRects(maskRect, clipRect) : maskRect;
+      if (!drawRect || drawRect.w <= 0 || drawRect.h <= 0) return true;
+
+      const resources = getMaskVisualizationResources(gl, () => maskVisualizationResources, (nextResources) => {
+        maskVisualizationResources = nextResources;
+      });
+      if (!resources) return false;
+
+      const maskEntry = getTextureEntry(context, textures, maskAsset, invalidate, () => disposed);
+      const maskTexture = renderableTexture(maskEntry);
+      if (!maskTexture) return false;
+
+      gl.enable(gl.SCISSOR_TEST);
+      gl.scissor(drawRect.x, context.canvas.height - drawRect.y - drawRect.h, drawRect.w, drawRect.h);
+      enablePremultipliedAlphaBlending(gl);
+      gl.useProgram(resources.program);
+
+      gl.activeTexture(gl.TEXTURE0);
+      gl.bindTexture(gl.TEXTURE_2D, maskTexture);
+      gl.uniform1i(resources.samplerLocation, 0);
+      gl.uniform1i(resources.modeLocation, maskVisualizationModeValue(mode));
+      gl.uniform4fv(resources.colorLocation, color);
+
+      gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
+      gl.bufferData(gl.ARRAY_BUFFER, rectVertices(context.canvas, drawRect), gl.DYNAMIC_DRAW);
+      gl.enableVertexAttribArray(resources.positionLocation);
+      gl.vertexAttribPointer(resources.positionLocation, 2, gl.FLOAT, false, 0, 0);
+
+      gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer);
+      gl.bufferData(gl.ARRAY_BUFFER, texCoordsForRect(drawRect, maskRect), gl.DYNAMIC_DRAW);
+      gl.enableVertexAttribArray(resources.texCoordLocation);
+      gl.vertexAttribPointer(resources.texCoordLocation, 2, gl.FLOAT, false, 0, 0);
+
+      gl.drawArrays(gl.TRIANGLES, 0, 6);
+      gl.disable(gl.BLEND);
+      return true;
+    },
    renderTinted(asset, rect, color, clipRect) {
      const drawRect = clipRect ? intersectScreenRects(rect, clipRect) : rect;
      if (!drawRect || drawRect.w <= 0 || drawRect.h <= 0) return true;
@@ -143,8 +258,7 @@ export function createImageTextureRenderer(context: WebGlRendererContext, invali

      gl.enable(gl.SCISSOR_TEST);
      gl.scissor(drawRect.x, context.canvas.height - drawRect.y - drawRect.h, drawRect.w, drawRect.h);
-      gl.enable(gl.BLEND);
-      gl.blendFunc(gl.SRC_ALPHA, gl.ONE_MINUS_SRC_ALPHA);
+      enablePremultipliedAlphaBlending(gl);
      gl.useProgram(tintedProgram);

      gl.activeTexture(gl.TEXTURE0);
@@ -174,6 +288,8 @@ export function createImageTextureRenderer(context: WebGlRendererContext, invali
      gl.deleteBuffer(maskTexCoordBuffer);
      gl.deleteProgram(program);
      gl.deleteProgram(maskedProgram);
+      gl.deleteProgram(maskRevealPreviewProgram);
+      if (maskVisualizationResources && maskVisualizationResources !== "failed") gl.deleteProgram(maskVisualizationResources.program);
      gl.deleteProgram(tintedProgram);
    },
  };
@@ -274,11 +390,17 @@ function createTexture(gl: WebGL2RenderingContext, image: HTMLImageElement) {
  gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
  gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
  gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, false);
+  gl.pixelStorei(gl.UNPACK_PREMULTIPLY_ALPHA_WEBGL, true);
  gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, image);

  return texture;
 }

+function enablePremultipliedAlphaBlending(gl: WebGL2RenderingContext) {
+  gl.enable(gl.BLEND);
+  gl.blendFunc(gl.ONE, gl.ONE_MINUS_SRC_ALPHA);
+}
+
 function fullTexCoords() {
  return new Float32Array([0, 0, 1, 0, 0, 1, 0, 1, 1, 0, 1, 1]);
 }
@@ -344,6 +466,108 @@ function createProgram(gl: WebGL2RenderingContext) {
  return program;
 }

+function getMaskVisualizationResources(
+  gl: WebGL2RenderingContext,
+  getResources: () => MaskVisualizationResources | "failed" | undefined,
+  setResources: (resources: MaskVisualizationResources | "failed") => void,
+): MaskVisualizationResources | undefined {
+  const currentResources = getResources();
+  if (currentResources === "failed") return undefined;
+  if (currentResources) return currentResources;
+
+  try {
+    const resources = createMaskVisualizationProgram(gl);
+    setResources(resources);
+    return resources;
+  } catch {
+    setResources("failed");
+    return undefined;
+  }
+}
+
+function maskVisualizationModeValue(mode: MaskVisualizationMode) {
+  switch (mode) {
+    case "blackWhite":
+      return 0;
+    case "alpha":
+      return 1;
+    case "hiddenOverlay":
+      return 2;
+  }
+}
+
+function createMaskVisualizationProgram(gl: WebGL2RenderingContext): MaskVisualizationResources {
+  const vertexShader = compileShader(
+    gl,
+    gl.VERTEX_SHADER,
+    `#version 300 es
+    in vec2 a_position;
+    in vec2 a_texCoord;
+    out vec2 v_texCoord;
+    void main() {
+      gl_Position = vec4(a_position, 0.0, 1.0);
+      v_texCoord = a_texCoord;
+    }`,
+  );
+  const fragmentShader = compileShader(
+    gl,
+    gl.FRAGMENT_SHADER,
+    `#version 300 es
+    precision mediump float;
+    uniform sampler2D u_mask;
+    uniform int u_mode;
+    uniform vec4 u_color;
+    in vec2 v_texCoord;
+    out vec4 outColor;
+    void main() {
+      float maskAlpha = texture(u_mask, v_texCoord).a;
+      if (u_mode == 0) {
+        float value = step(0.5, maskAlpha);
+        outColor = vec4(value, value, value, 1.0);
+        return;
+      }
+      if (u_mode == 1) {
+        outColor = vec4(maskAlpha, maskAlpha, maskAlpha, 1.0);
+        return;
+      }
+
+      float alpha = (1.0 - maskAlpha) * u_color.a;
+      outColor = vec4(u_color.rgb * alpha, alpha);
+    }`,
+  );
+  const program = gl.createProgram();
+  if (!program) throw new Error("Failed to create mask visualization shader program");
+
+  gl.attachShader(program, vertexShader);
+  gl.attachShader(program, fragmentShader);
+  gl.linkProgram(program);
+  gl.deleteShader(vertexShader);
+  gl.deleteShader(fragmentShader);
+
+  if (!gl.getProgramParameter(program, gl.LINK_STATUS)) {
+    const message = gl.getProgramInfoLog(program) ?? "Unknown mask visualization program link error";
+    gl.deleteProgram(program);
+    throw new Error(message);
+  }
+
+  const samplerLocation = gl.getUniformLocation(program, "u_mask");
+  const modeLocation = gl.getUniformLocation(program, "u_mode");
+  const colorLocation = gl.getUniformLocation(program, "u_color");
+  if (!samplerLocation || !modeLocation || !colorLocation) {
+    gl.deleteProgram(program);
+    throw new Error("Failed to resolve mask visualization shader uniforms");
+  }
+
+  return {
+    program,
+    positionLocation: gl.getAttribLocation(program, "a_position"),
+    texCoordLocation: gl.getAttribLocation(program, "a_texCoord"),
+    samplerLocation,
+    modeLocation,
+    colorLocation,
+  };
+}
+
 function createTintedProgram(gl: WebGL2RenderingContext) {
  const vertexShader = compileShader(
    gl,
@@ -367,8 +591,8 @@ function createTintedProgram(gl: WebGL2RenderingContext) {
    in vec2 v_texCoord;
    out vec4 outColor;
    void main() {
-      float maskAlpha = texture(u_image, v_texCoord).a;
-      outColor = vec4(u_color.rgb, u_color.a * maskAlpha);
+      float alpha = u_color.a * texture(u_image, v_texCoord).a;
+      outColor = vec4(u_color.rgb * alpha, alpha);
    }`,
  );
  const program = gl.createProgram();
@@ -389,6 +613,58 @@ function createTintedProgram(gl: WebGL2RenderingContext) {
  return program;
 }

+function createMaskRevealPreviewProgram(gl: WebGL2RenderingContext) {
+  const vertexShader = compileShader(
+    gl,
+    gl.VERTEX_SHADER,
+    `#version 300 es
+    in vec2 a_position;
+    in vec2 a_texCoord;
+    in vec2 a_maskTexCoord;
+    out vec2 v_texCoord;
+    out vec2 v_maskTexCoord;
+    void main() {
+      gl_Position = vec4(a_position, 0.0, 1.0);
+      v_texCoord = a_texCoord;
+      v_maskTexCoord = a_maskTexCoord;
+    }`,
+  );
+  const fragmentShader = compileShader(
+    gl,
+    gl.FRAGMENT_SHADER,
+    `#version 300 es
+    precision mediump float;
+    uniform sampler2D u_image;
+    uniform sampler2D u_mask;
+    uniform float u_opacity;
+    in vec2 v_texCoord;
+    in vec2 v_maskTexCoord;
+    out vec4 outColor;
+    void main() {
+      vec4 color = texture(u_image, v_texCoord);
+      float hiddenMaskAlpha = 1.0 - texture(u_mask, v_maskTexCoord).a;
+      float previewAlpha = clamp(hiddenMaskAlpha * u_opacity, 0.0, 1.0);
+      outColor = vec4(color.rgb * previewAlpha, color.a * previewAlpha);
+    }`,
+  );
+  const program = gl.createProgram();
+  if (!program) throw new Error("Failed to create mask reveal preview shader program");
+
+  gl.attachShader(program, vertexShader);
+  gl.attachShader(program, fragmentShader);
+  gl.linkProgram(program);
+  gl.deleteShader(vertexShader);
+  gl.deleteShader(fragmentShader);
+
+  if (!gl.getProgramParameter(program, gl.LINK_STATUS)) {
+    const message = gl.getProgramInfoLog(program) ?? "Unknown mask reveal preview program link error";
+    gl.deleteProgram(program);
+    throw new Error(message);
+  }
+
+  return program;
+}
+
 function createMaskedProgram(gl: WebGL2RenderingContext) {
  const vertexShader = compileShader(
    gl,
@@ -418,7 +694,8 @@ function createMaskedProgram(gl: WebGL2RenderingContext) {
    void main() {
      vec4 color = texture(u_image, v_texCoord);
      float maskAlpha = texture(u_mask, v_maskTexCoord).a;
-      outColor = vec4(color.rgb, color.a * maskAlpha);
+      float alpha = color.a * maskAlpha;
+      outColor = vec4(color.rgb * maskAlpha, alpha);
    }`,
  );
  const program = gl.createProgram();