Files
image-studio/renderer/image-textures.ts
2026-07-11 12:16:33 +02:00

440 lines
20 KiB
TypeScript

import { createMaskedProgram, createMaskRevealPreviewProgram, createProgram, createTintedProgram, getMaskVisualizationResources, maskVisualizationModeValue, type MaskVisualizationResources } from "./image-texture-programs";
import type { Asset } from "@core/asset";
import type { RgbaColor, ScreenRect, WebGlRendererContext } from "./types";
import type { Rect } from "@core/geometry";
import { rotatedRectBounds, rotatedRectCorners } from "./rotated-rect";
import { textureCoordinatesForCrop, textureCoordinatesForRect } from "./texture-coordinates";
export type MaskVisualizationMode = "blackWhite" | "alpha" | "hiddenOverlay";
export type ImageTextureRenderer = {
syncAssets(assets: readonly Asset[]): void;
render(asset: Asset, rect: ScreenRect, clipRect?: ScreenRect, opacity?: number, rotation?: number, sourceRect?: Rect): boolean;
renderMasked(asset: Asset, rect: ScreenRect, maskAsset: Asset, maskRect: ScreenRect, clipRect?: ScreenRect, opacity?: number, rotation?: number, sourceRect?: Rect): boolean;
renderMaskRevealPreview(asset: Asset, rect: ScreenRect, maskAsset: Asset, maskRect: ScreenRect, opacity: number, clipRect?: ScreenRect, layerOpacity?: number, rotation?: number, sourceRect?: Rect): boolean;
renderMaskVisualization(maskAsset: Asset, maskRect: ScreenRect, mode: MaskVisualizationMode, color?: RgbaColor, clipRect?: ScreenRect): boolean;
renderTinted(asset: Asset, rect: ScreenRect, color: RgbaColor, clipRect?: ScreenRect, opacity?: number): boolean;
dispose(): void;
};
type TextureState =
| { status: "loading"; image: HTMLImageElement }
| { status: "ready"; texture: WebGLTexture }
| { status: "error" };
type TextureEntry = {
source: string;
current: TextureState;
previousTexture?: WebGLTexture;
};
export function createImageTextureRenderer(context: WebGlRendererContext, invalidate: () => void): ImageTextureRenderer {
const { gl } = context;
const program = createProgram(gl);
const positionLocation = gl.getAttribLocation(program, "a_position");
const texCoordLocation = gl.getAttribLocation(program, "a_texCoord");
const samplerLocation = gl.getUniformLocation(program, "u_image");
const opacityLocation = gl.getUniformLocation(program, "u_opacity");
const maskedProgram = createMaskedProgram(gl);
const maskedPositionLocation = gl.getAttribLocation(maskedProgram, "a_position");
const maskedTexCoordLocation = gl.getAttribLocation(maskedProgram, "a_texCoord");
const maskedMaskTexCoordLocation = gl.getAttribLocation(maskedProgram, "a_maskTexCoord");
const maskedSamplerLocation = gl.getUniformLocation(maskedProgram, "u_image");
const maskedMaskSamplerLocation = gl.getUniformLocation(maskedProgram, "u_mask");
const maskedOpacityLocation = gl.getUniformLocation(maskedProgram, "u_opacity");
const maskRevealPreviewProgram = createMaskRevealPreviewProgram(gl);
const maskRevealPreviewPositionLocation = gl.getAttribLocation(maskRevealPreviewProgram, "a_position");
const maskRevealPreviewTexCoordLocation = gl.getAttribLocation(maskRevealPreviewProgram, "a_texCoord");
const maskRevealPreviewMaskTexCoordLocation = gl.getAttribLocation(maskRevealPreviewProgram, "a_maskTexCoord");
const maskRevealPreviewSamplerLocation = gl.getUniformLocation(maskRevealPreviewProgram, "u_image");
const maskRevealPreviewMaskSamplerLocation = gl.getUniformLocation(maskRevealPreviewProgram, "u_mask");
const maskRevealPreviewOpacityLocation = gl.getUniformLocation(maskRevealPreviewProgram, "u_opacity");
const tintedProgram = createTintedProgram(gl);
const tintedPositionLocation = gl.getAttribLocation(tintedProgram, "a_position");
const tintedTexCoordLocation = gl.getAttribLocation(tintedProgram, "a_texCoord");
const tintedSamplerLocation = gl.getUniformLocation(tintedProgram, "u_image");
const tintedColorLocation = gl.getUniformLocation(tintedProgram, "u_color");
const positionBuffer = gl.createBuffer();
const texCoordBuffer = gl.createBuffer();
const maskTexCoordBuffer = gl.createBuffer();
const textures = new Map<string, TextureEntry>();
let maskVisualizationResources: MaskVisualizationResources | "failed" | undefined;
let disposed = false;
if (
!positionBuffer ||
!texCoordBuffer ||
!maskTexCoordBuffer ||
!samplerLocation ||
!opacityLocation ||
!maskedSamplerLocation ||
!maskedMaskSamplerLocation ||
!maskedOpacityLocation ||
!maskRevealPreviewSamplerLocation ||
!maskRevealPreviewMaskSamplerLocation ||
!maskRevealPreviewOpacityLocation ||
!tintedSamplerLocation ||
!tintedColorLocation
) throw new Error("Failed to create image texture renderer");
gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer);
gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([0, 0, 1, 0, 0, 1, 0, 1, 1, 0, 1, 1]), gl.STATIC_DRAW);
return {
syncAssets(assets) {
const activeIds = new Set(assets.map((asset) => asset.id));
for (const [assetId, entry] of textures) {
if (!activeIds.has(assetId)) {
disposeEntry(gl, entry);
textures.delete(assetId);
}
}
},
render(asset, rect, clipRect, opacity = 1, rotation = 0, sourceRect) {
const clampedOpacity = clampOpacity(opacity);
if (clampedOpacity <= 0) return true;
const rotatedRect = rotatedRectBounds(rect, rotation);
const drawRect = clipRect ? intersectScreenRects(rotatedRect, clipRect) : rotatedRect;
if (!drawRect || drawRect.w <= 0 || drawRect.h <= 0) return true;
const entry = getTextureEntry(context, textures, asset, invalidate, () => disposed);
const texture = renderableTexture(entry);
if (!texture) return false;
gl.enable(gl.SCISSOR_TEST);
gl.scissor(drawRect.x, context.canvas.height - drawRect.y - drawRect.h, drawRect.w, drawRect.h);
enablePremultipliedAlphaBlending(gl);
gl.useProgram(program);
gl.activeTexture(gl.TEXTURE0);
gl.bindTexture(gl.TEXTURE_2D, texture);
gl.uniform1i(samplerLocation, 0);
gl.uniform1f(opacityLocation, clampedOpacity);
gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
gl.bufferData(gl.ARRAY_BUFFER, rectVertices(context.canvas, rect, rotation), gl.DYNAMIC_DRAW);
gl.enableVertexAttribArray(positionLocation);
gl.vertexAttribPointer(positionLocation, 2, gl.FLOAT, false, 0, 0);
gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer);
gl.bufferData(gl.ARRAY_BUFFER, textureCoordinatesForCrop(asset.intrinsicSize, sourceRect), gl.DYNAMIC_DRAW);
gl.enableVertexAttribArray(texCoordLocation);
gl.vertexAttribPointer(texCoordLocation, 2, gl.FLOAT, false, 0, 0);
gl.drawArrays(gl.TRIANGLES, 0, 6);
gl.disable(gl.BLEND);
return true;
},
renderMasked(asset, rect, maskAsset, maskRect, clipRect, opacity = 1, rotation = 0, sourceRect) {
const clampedOpacity = clampOpacity(opacity);
if (clampedOpacity <= 0) return true;
const contentBounds = rotatedRectBounds(rect, rotation);
const maskBounds = rotatedRectBounds(maskRect, rotation);
const clippedRect = clipRect ? intersectScreenRects(contentBounds, clipRect) : contentBounds;
const drawRect = clippedRect ? intersectScreenRects(clippedRect, maskBounds) : undefined;
if (!drawRect || drawRect.w <= 0 || drawRect.h <= 0) return true;
const entry = getTextureEntry(context, textures, asset, invalidate, () => disposed);
const maskEntry = getTextureEntry(context, textures, maskAsset, invalidate, () => disposed);
const texture = renderableTexture(entry);
const maskTexture = renderableTexture(maskEntry);
if (!texture || !maskTexture) return false;
gl.enable(gl.SCISSOR_TEST);
gl.scissor(drawRect.x, context.canvas.height - drawRect.y - drawRect.h, drawRect.w, drawRect.h);
enablePremultipliedAlphaBlending(gl);
gl.useProgram(maskedProgram);
gl.activeTexture(gl.TEXTURE0);
gl.bindTexture(gl.TEXTURE_2D, texture);
gl.uniform1i(maskedSamplerLocation, 0);
gl.activeTexture(gl.TEXTURE1);
gl.bindTexture(gl.TEXTURE_2D, maskTexture);
gl.uniform1i(maskedMaskSamplerLocation, 1);
gl.uniform1f(maskedOpacityLocation, clampedOpacity);
gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
gl.bufferData(gl.ARRAY_BUFFER, rectVertices(context.canvas, rect, rotation), gl.DYNAMIC_DRAW);
gl.enableVertexAttribArray(maskedPositionLocation);
gl.vertexAttribPointer(maskedPositionLocation, 2, gl.FLOAT, false, 0, 0);
gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer);
gl.bufferData(gl.ARRAY_BUFFER, textureCoordinatesForCrop(asset.intrinsicSize, sourceRect), gl.DYNAMIC_DRAW);
gl.enableVertexAttribArray(maskedTexCoordLocation);
gl.vertexAttribPointer(maskedTexCoordLocation, 2, gl.FLOAT, false, 0, 0);
gl.bindBuffer(gl.ARRAY_BUFFER, maskTexCoordBuffer);
gl.bufferData(gl.ARRAY_BUFFER, textureCoordinatesForRect(rect, maskRect), gl.DYNAMIC_DRAW);
gl.enableVertexAttribArray(maskedMaskTexCoordLocation);
gl.vertexAttribPointer(maskedMaskTexCoordLocation, 2, gl.FLOAT, false, 0, 0);
gl.drawArrays(gl.TRIANGLES, 0, 6);
gl.disable(gl.BLEND);
return true;
},
renderMaskRevealPreview(asset, rect, maskAsset, maskRect, opacity, clipRect, layerOpacity = 1, rotation = 0, sourceRect) {
const clampedOpacity = clampOpacity(opacity) * clampOpacity(layerOpacity);
if (clampedOpacity <= 0) return true;
const contentBounds = rotatedRectBounds(rect, rotation);
const maskBounds = rotatedRectBounds(maskRect, rotation);
const clippedRect = clipRect ? intersectScreenRects(contentBounds, clipRect) : contentBounds;
const drawRect = clippedRect ? intersectScreenRects(clippedRect, maskBounds) : undefined;
if (!drawRect || drawRect.w <= 0 || drawRect.h <= 0) return true;
const entry = getTextureEntry(context, textures, asset, invalidate, () => disposed);
const maskEntry = getTextureEntry(context, textures, maskAsset, invalidate, () => disposed);
const texture = renderableTexture(entry);
const maskTexture = renderableTexture(maskEntry);
if (!texture || !maskTexture) return false;
gl.enable(gl.SCISSOR_TEST);
gl.scissor(drawRect.x, context.canvas.height - drawRect.y - drawRect.h, drawRect.w, drawRect.h);
enablePremultipliedAlphaBlending(gl);
gl.useProgram(maskRevealPreviewProgram);
gl.activeTexture(gl.TEXTURE0);
gl.bindTexture(gl.TEXTURE_2D, texture);
gl.uniform1i(maskRevealPreviewSamplerLocation, 0);
gl.activeTexture(gl.TEXTURE1);
gl.bindTexture(gl.TEXTURE_2D, maskTexture);
gl.uniform1i(maskRevealPreviewMaskSamplerLocation, 1);
gl.uniform1f(maskRevealPreviewOpacityLocation, clampedOpacity);
gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
gl.bufferData(gl.ARRAY_BUFFER, rectVertices(context.canvas, rect, rotation), gl.DYNAMIC_DRAW);
gl.enableVertexAttribArray(maskRevealPreviewPositionLocation);
gl.vertexAttribPointer(maskRevealPreviewPositionLocation, 2, gl.FLOAT, false, 0, 0);
gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer);
gl.bufferData(gl.ARRAY_BUFFER, textureCoordinatesForCrop(asset.intrinsicSize, sourceRect), gl.DYNAMIC_DRAW);
gl.enableVertexAttribArray(maskRevealPreviewTexCoordLocation);
gl.vertexAttribPointer(maskRevealPreviewTexCoordLocation, 2, gl.FLOAT, false, 0, 0);
gl.bindBuffer(gl.ARRAY_BUFFER, maskTexCoordBuffer);
gl.bufferData(gl.ARRAY_BUFFER, textureCoordinatesForRect(rect, maskRect), gl.DYNAMIC_DRAW);
gl.enableVertexAttribArray(maskRevealPreviewMaskTexCoordLocation);
gl.vertexAttribPointer(maskRevealPreviewMaskTexCoordLocation, 2, gl.FLOAT, false, 0, 0);
gl.drawArrays(gl.TRIANGLES, 0, 6);
gl.disable(gl.BLEND);
return true;
},
renderMaskVisualization(maskAsset, maskRect, mode, color = [1, 1, 1, 1], clipRect) {
const drawRect = clipRect ? intersectScreenRects(maskRect, clipRect) : maskRect;
if (!drawRect || drawRect.w <= 0 || drawRect.h <= 0) return true;
const resources = getMaskVisualizationResources(gl, () => maskVisualizationResources, (nextResources) => {
maskVisualizationResources = nextResources;
});
if (!resources) return false;
const maskEntry = getTextureEntry(context, textures, maskAsset, invalidate, () => disposed);
const maskTexture = renderableTexture(maskEntry);
if (!maskTexture) return false;
gl.enable(gl.SCISSOR_TEST);
gl.scissor(drawRect.x, context.canvas.height - drawRect.y - drawRect.h, drawRect.w, drawRect.h);
enablePremultipliedAlphaBlending(gl);
gl.useProgram(resources.program);
gl.activeTexture(gl.TEXTURE0);
gl.bindTexture(gl.TEXTURE_2D, maskTexture);
gl.uniform1i(resources.samplerLocation, 0);
gl.uniform1i(resources.modeLocation, maskVisualizationModeValue(mode));
gl.uniform4fv(resources.colorLocation, color);
gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
gl.bufferData(gl.ARRAY_BUFFER, rectVertices(context.canvas, drawRect), gl.DYNAMIC_DRAW);
gl.enableVertexAttribArray(resources.positionLocation);
gl.vertexAttribPointer(resources.positionLocation, 2, gl.FLOAT, false, 0, 0);
gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer);
gl.bufferData(gl.ARRAY_BUFFER, texCoordsForRect(drawRect, maskRect), gl.DYNAMIC_DRAW);
gl.enableVertexAttribArray(resources.texCoordLocation);
gl.vertexAttribPointer(resources.texCoordLocation, 2, gl.FLOAT, false, 0, 0);
gl.drawArrays(gl.TRIANGLES, 0, 6);
gl.disable(gl.BLEND);
return true;
},
renderTinted(asset, rect, color, clipRect, opacity = 1) {
const clampedOpacity = clampOpacity(opacity);
if (clampedOpacity <= 0) return true;
const drawRect = clipRect ? intersectScreenRects(rect, clipRect) : rect;
if (!drawRect || drawRect.w <= 0 || drawRect.h <= 0) return true;
const entry = getTextureEntry(context, textures, asset, invalidate, () => disposed);
const texture = renderableTexture(entry);
if (!texture) return false;
gl.enable(gl.SCISSOR_TEST);
gl.scissor(drawRect.x, context.canvas.height - drawRect.y - drawRect.h, drawRect.w, drawRect.h);
enablePremultipliedAlphaBlending(gl);
gl.useProgram(tintedProgram);
gl.activeTexture(gl.TEXTURE0);
gl.bindTexture(gl.TEXTURE_2D, texture);
gl.uniform1i(tintedSamplerLocation, 0);
gl.uniform4fv(tintedColorLocation, [color[0], color[1], color[2], color[3] * clampedOpacity]);
gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
gl.bufferData(gl.ARRAY_BUFFER, rectVertices(context.canvas, rect), gl.DYNAMIC_DRAW);
gl.enableVertexAttribArray(tintedPositionLocation);
gl.vertexAttribPointer(tintedPositionLocation, 2, gl.FLOAT, false, 0, 0);
gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer);
gl.bufferData(gl.ARRAY_BUFFER, fullTexCoords(), gl.DYNAMIC_DRAW);
gl.enableVertexAttribArray(tintedTexCoordLocation);
gl.vertexAttribPointer(tintedTexCoordLocation, 2, gl.FLOAT, false, 0, 0);
gl.drawArrays(gl.TRIANGLES, 0, 6);
gl.disable(gl.BLEND);
return true;
},
dispose() {
disposed = true;
for (const entry of textures.values()) disposeEntry(gl, entry);
gl.deleteBuffer(positionBuffer);
gl.deleteBuffer(texCoordBuffer);
gl.deleteBuffer(maskTexCoordBuffer);
gl.deleteProgram(program);
gl.deleteProgram(maskedProgram);
gl.deleteProgram(maskRevealPreviewProgram);
if (maskVisualizationResources && maskVisualizationResources !== "failed") gl.deleteProgram(maskVisualizationResources.program);
gl.deleteProgram(tintedProgram);
},
};
}
function getTextureEntry(
context: WebGlRendererContext,
textures: Map<string, TextureEntry>,
asset: Asset,
invalidate: () => void,
isDisposed: () => boolean,
): TextureEntry {
const cached = textures.get(asset.id);
if (cached && cached.source === asset.source) return cached;
if (cached) {
if (cached.current.status === "ready") {
if (cached.previousTexture) context.gl.deleteTexture(cached.previousTexture);
cached.previousTexture = cached.current.texture;
} else {
disposeState(context.gl, cached.current);
}
cached.source = asset.source;
cached.current = loadTextureState(context, asset, cached, invalidate, isDisposed);
return cached;
}
const entry: TextureEntry = { source: asset.source, current: { status: "error" } };
entry.current = loadTextureState(context, asset, entry, invalidate, isDisposed);
textures.set(asset.id, entry);
return entry;
}
function loadTextureState(
context: WebGlRendererContext,
asset: Asset,
entry: TextureEntry,
invalidate: () => void,
isDisposed: () => boolean,
): TextureState {
const image = new Image();
const source = asset.source;
image.onload = () => {
if (isDisposed() || entry.source !== source) return;
const texture = createTexture(context.gl, image);
if (entry.previousTexture) context.gl.deleteTexture(entry.previousTexture);
entry.previousTexture = undefined;
entry.current = { status: "ready", texture };
invalidate();
};
image.onerror = () => {
if (isDisposed() || entry.source !== source) return;
entry.current = { status: "error" };
invalidate();
};
image.src = source;
return { status: "loading", image };
}
function renderableTexture(entry: TextureEntry): WebGLTexture | undefined {
if (entry.current.status === "ready") return entry.current.texture;
return entry.previousTexture;
}
function disposeEntry(gl: WebGL2RenderingContext, entry: TextureEntry) {
disposeState(gl, entry.current);
if (entry.previousTexture) gl.deleteTexture(entry.previousTexture);
}
function disposeState(gl: WebGL2RenderingContext, state: TextureState) {
if (state.status === "ready") {
gl.deleteTexture(state.texture);
return;
}
if (state.status === "loading") {
state.image.onload = null;
state.image.onerror = null;
}
}
function intersectScreenRects(a: ScreenRect, b: ScreenRect): ScreenRect | undefined {
const x1 = Math.max(a.x, b.x);
const y1 = Math.max(a.y, b.y);
const x2 = Math.min(a.x + a.w, b.x + b.w);
const y2 = Math.min(a.y + a.h, b.y + b.h);
if (x2 <= x1 || y2 <= y1) return undefined;
return { x: x1, y: y1, w: x2 - x1, h: y2 - y1 };
}
function createTexture(gl: WebGL2RenderingContext, image: HTMLImageElement) {
const texture = gl.createTexture();
if (!texture) throw new Error("Failed to create image texture");
gl.bindTexture(gl.TEXTURE_2D, texture);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, false);
gl.pixelStorei(gl.UNPACK_PREMULTIPLY_ALPHA_WEBGL, true);
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, image);
return texture;
}
function enablePremultipliedAlphaBlending(gl: WebGL2RenderingContext) {
gl.enable(gl.BLEND);
gl.blendFunc(gl.ONE, gl.ONE_MINUS_SRC_ALPHA);
}
function clampOpacity(opacity: number) {
return Math.max(0, Math.min(1, opacity));
}
function fullTexCoords() {
return new Float32Array([0, 0, 1, 0, 0, 1, 0, 1, 1, 0, 1, 1]);
}
function texCoordsForRect(drawRect: ScreenRect, sourceRect: ScreenRect) {
const x1 = (drawRect.x - sourceRect.x) / sourceRect.w;
const x2 = (drawRect.x + drawRect.w - sourceRect.x) / sourceRect.w;
const y1 = (drawRect.y - sourceRect.y) / sourceRect.h;
const y2 = (drawRect.y + drawRect.h - sourceRect.y) / sourceRect.h;
return new Float32Array([x1, y1, x2, y1, x1, y2, x1, y2, x2, y1, x2, y2]);
}
function rectVertices(canvas: HTMLCanvasElement, rect: ScreenRect, rotation = 0) {
const [nw, ne, sw, se] = rotatedRectCorners(rect, rotation).map((point) => ({
x: (point.x / canvas.width) * 2 - 1,
y: 1 - (point.y / canvas.height) * 2,
}));
return new Float32Array([nw!.x, nw!.y, ne!.x, ne!.y, sw!.x, sw!.y, sw!.x, sw!.y, ne!.x, ne!.y, se!.x, se!.y]);
}