import { createMaskedProgram, createMaskRevealPreviewProgram, createProgram, createTintedProgram, getMaskVisualizationResources, maskVisualizationModeValue, type MaskVisualizationResources } from "./image-texture-programs"; import type { Asset } from "@core/asset"; import type { RgbaColor, ScreenRect, WebGlRendererContext } from "./types"; import type { Rect } from "@core/geometry"; import { rotatedRectBounds, rotatedRectCorners } from "./rotated-rect"; import { textureCoordinatesForCrop, textureCoordinatesForRect } from "./texture-coordinates"; export type MaskVisualizationMode = "blackWhite" | "alpha" | "hiddenOverlay" | "activeOverlay"; export type ImageTextureRenderer = { syncAssets(assets: readonly Pick[]): void; render(asset: Asset, rect: ScreenRect, clipRect?: ScreenRect, opacity?: number, rotation?: number, sourceRect?: Rect): boolean; renderMasked(asset: Asset, rect: ScreenRect, maskAsset: Asset, maskRect: ScreenRect, clipRect?: ScreenRect, opacity?: number, rotation?: number, sourceRect?: Rect): boolean; renderMaskRevealPreview(asset: Asset, rect: ScreenRect, maskAsset: Asset, maskRect: ScreenRect, opacity: number, clipRect?: ScreenRect, layerOpacity?: number, rotation?: number, sourceRect?: Rect): boolean; renderMaskVisualization(maskAsset: Asset, maskRect: ScreenRect, mode: MaskVisualizationMode, color?: RgbaColor, clipRect?: ScreenRect): boolean; renderTinted(asset: Asset, rect: ScreenRect, color: RgbaColor, clipRect?: ScreenRect, opacity?: number): boolean; dispose(): void; }; type TextureState = | { status: "loading"; image: HTMLImageElement } | { status: "ready"; texture: WebGLTexture } | { status: "error" }; type TextureEntry = { source: string; current: TextureState; previousTexture?: WebGLTexture; }; export function createImageTextureRenderer(context: WebGlRendererContext, invalidate: () => void): ImageTextureRenderer { const { gl } = context; const program = createProgram(gl); const positionLocation = gl.getAttribLocation(program, "a_position"); const texCoordLocation = gl.getAttribLocation(program, "a_texCoord"); const samplerLocation = gl.getUniformLocation(program, "u_image"); const opacityLocation = gl.getUniformLocation(program, "u_opacity"); const maskedProgram = createMaskedProgram(gl); const maskedPositionLocation = gl.getAttribLocation(maskedProgram, "a_position"); const maskedTexCoordLocation = gl.getAttribLocation(maskedProgram, "a_texCoord"); const maskedMaskTexCoordLocation = gl.getAttribLocation(maskedProgram, "a_maskTexCoord"); const maskedSamplerLocation = gl.getUniformLocation(maskedProgram, "u_image"); const maskedMaskSamplerLocation = gl.getUniformLocation(maskedProgram, "u_mask"); const maskedOpacityLocation = gl.getUniformLocation(maskedProgram, "u_opacity"); const maskRevealPreviewProgram = createMaskRevealPreviewProgram(gl); const maskRevealPreviewPositionLocation = gl.getAttribLocation(maskRevealPreviewProgram, "a_position"); const maskRevealPreviewTexCoordLocation = gl.getAttribLocation(maskRevealPreviewProgram, "a_texCoord"); const maskRevealPreviewMaskTexCoordLocation = gl.getAttribLocation(maskRevealPreviewProgram, "a_maskTexCoord"); const maskRevealPreviewSamplerLocation = gl.getUniformLocation(maskRevealPreviewProgram, "u_image"); const maskRevealPreviewMaskSamplerLocation = gl.getUniformLocation(maskRevealPreviewProgram, "u_mask"); const maskRevealPreviewOpacityLocation = gl.getUniformLocation(maskRevealPreviewProgram, "u_opacity"); const tintedProgram = createTintedProgram(gl); const tintedPositionLocation = gl.getAttribLocation(tintedProgram, "a_position"); const tintedTexCoordLocation = gl.getAttribLocation(tintedProgram, "a_texCoord"); const tintedSamplerLocation = gl.getUniformLocation(tintedProgram, "u_image"); const tintedColorLocation = gl.getUniformLocation(tintedProgram, "u_color"); const positionBuffer = gl.createBuffer(); const texCoordBuffer = gl.createBuffer(); const maskTexCoordBuffer = gl.createBuffer(); const textures = new Map(); let maskVisualizationResources: MaskVisualizationResources | "failed" | undefined; let disposed = false; if ( !positionBuffer || !texCoordBuffer || !maskTexCoordBuffer || !samplerLocation || !opacityLocation || !maskedSamplerLocation || !maskedMaskSamplerLocation || !maskedOpacityLocation || !maskRevealPreviewSamplerLocation || !maskRevealPreviewMaskSamplerLocation || !maskRevealPreviewOpacityLocation || !tintedSamplerLocation || !tintedColorLocation ) throw new Error("Failed to create image texture renderer"); gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer); gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([0, 0, 1, 0, 0, 1, 0, 1, 1, 0, 1, 1]), gl.STATIC_DRAW); return { syncAssets(assets) { const activeIds = new Set(assets.map((asset) => asset.id)); for (const [assetId, entry] of textures) { if (!activeIds.has(assetId)) { disposeEntry(gl, entry); textures.delete(assetId); } } }, render(asset, rect, clipRect, opacity = 1, rotation = 0, sourceRect) { const clampedOpacity = clampOpacity(opacity); if (clampedOpacity <= 0) return true; const rotatedRect = rotatedRectBounds(rect, rotation); const drawRect = clipRect ? intersectScreenRects(rotatedRect, clipRect) : rotatedRect; if (!drawRect || drawRect.w <= 0 || drawRect.h <= 0) return true; const entry = getTextureEntry(context, textures, asset, invalidate, () => disposed); const texture = renderableTexture(entry); if (!texture) return false; gl.enable(gl.SCISSOR_TEST); gl.scissor(drawRect.x, context.canvas.height - drawRect.y - drawRect.h, drawRect.w, drawRect.h); enablePremultipliedAlphaBlending(gl); gl.useProgram(program); gl.activeTexture(gl.TEXTURE0); gl.bindTexture(gl.TEXTURE_2D, texture); gl.uniform1i(samplerLocation, 0); gl.uniform1f(opacityLocation, clampedOpacity); gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer); gl.bufferData(gl.ARRAY_BUFFER, rectVertices(context.canvas, rect, rotation), gl.DYNAMIC_DRAW); gl.enableVertexAttribArray(positionLocation); gl.vertexAttribPointer(positionLocation, 2, gl.FLOAT, false, 0, 0); gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer); gl.bufferData(gl.ARRAY_BUFFER, textureCoordinatesForCrop(asset.intrinsicSize, sourceRect), gl.DYNAMIC_DRAW); gl.enableVertexAttribArray(texCoordLocation); gl.vertexAttribPointer(texCoordLocation, 2, gl.FLOAT, false, 0, 0); gl.drawArrays(gl.TRIANGLES, 0, 6); gl.disable(gl.BLEND); return true; }, renderMasked(asset, rect, maskAsset, maskRect, clipRect, opacity = 1, rotation = 0, sourceRect) { const clampedOpacity = clampOpacity(opacity); if (clampedOpacity <= 0) return true; const contentBounds = rotatedRectBounds(rect, rotation); const maskBounds = rotatedRectBounds(maskRect, rotation); const clippedRect = clipRect ? intersectScreenRects(contentBounds, clipRect) : contentBounds; const drawRect = clippedRect ? intersectScreenRects(clippedRect, maskBounds) : undefined; if (!drawRect || drawRect.w <= 0 || drawRect.h <= 0) return true; const entry = getTextureEntry(context, textures, asset, invalidate, () => disposed); const maskEntry = getTextureEntry(context, textures, maskAsset, invalidate, () => disposed); const texture = renderableTexture(entry); const maskTexture = renderableTexture(maskEntry); if (!texture || !maskTexture) return false; gl.enable(gl.SCISSOR_TEST); gl.scissor(drawRect.x, context.canvas.height - drawRect.y - drawRect.h, drawRect.w, drawRect.h); enablePremultipliedAlphaBlending(gl); gl.useProgram(maskedProgram); gl.activeTexture(gl.TEXTURE0); gl.bindTexture(gl.TEXTURE_2D, texture); gl.uniform1i(maskedSamplerLocation, 0); gl.activeTexture(gl.TEXTURE1); gl.bindTexture(gl.TEXTURE_2D, maskTexture); gl.uniform1i(maskedMaskSamplerLocation, 1); gl.uniform1f(maskedOpacityLocation, clampedOpacity); gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer); gl.bufferData(gl.ARRAY_BUFFER, rectVertices(context.canvas, rect, rotation), gl.DYNAMIC_DRAW); gl.enableVertexAttribArray(maskedPositionLocation); gl.vertexAttribPointer(maskedPositionLocation, 2, gl.FLOAT, false, 0, 0); gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer); gl.bufferData(gl.ARRAY_BUFFER, textureCoordinatesForCrop(asset.intrinsicSize, sourceRect), gl.DYNAMIC_DRAW); gl.enableVertexAttribArray(maskedTexCoordLocation); gl.vertexAttribPointer(maskedTexCoordLocation, 2, gl.FLOAT, false, 0, 0); gl.bindBuffer(gl.ARRAY_BUFFER, maskTexCoordBuffer); gl.bufferData(gl.ARRAY_BUFFER, textureCoordinatesForRect(rect, maskRect), gl.DYNAMIC_DRAW); gl.enableVertexAttribArray(maskedMaskTexCoordLocation); gl.vertexAttribPointer(maskedMaskTexCoordLocation, 2, gl.FLOAT, false, 0, 0); gl.drawArrays(gl.TRIANGLES, 0, 6); gl.disable(gl.BLEND); return true; }, renderMaskRevealPreview(asset, rect, maskAsset, maskRect, opacity, clipRect, layerOpacity = 1, rotation = 0, sourceRect) { const clampedOpacity = clampOpacity(opacity) * clampOpacity(layerOpacity); if (clampedOpacity <= 0) return true; const contentBounds = rotatedRectBounds(rect, rotation); const maskBounds = rotatedRectBounds(maskRect, rotation); const clippedRect = clipRect ? intersectScreenRects(contentBounds, clipRect) : contentBounds; const drawRect = clippedRect ? intersectScreenRects(clippedRect, maskBounds) : undefined; if (!drawRect || drawRect.w <= 0 || drawRect.h <= 0) return true; const entry = getTextureEntry(context, textures, asset, invalidate, () => disposed); const maskEntry = getTextureEntry(context, textures, maskAsset, invalidate, () => disposed); const texture = renderableTexture(entry); const maskTexture = renderableTexture(maskEntry); if (!texture || !maskTexture) return false; gl.enable(gl.SCISSOR_TEST); gl.scissor(drawRect.x, context.canvas.height - drawRect.y - drawRect.h, drawRect.w, drawRect.h); enablePremultipliedAlphaBlending(gl); gl.useProgram(maskRevealPreviewProgram); gl.activeTexture(gl.TEXTURE0); gl.bindTexture(gl.TEXTURE_2D, texture); gl.uniform1i(maskRevealPreviewSamplerLocation, 0); gl.activeTexture(gl.TEXTURE1); gl.bindTexture(gl.TEXTURE_2D, maskTexture); gl.uniform1i(maskRevealPreviewMaskSamplerLocation, 1); gl.uniform1f(maskRevealPreviewOpacityLocation, clampedOpacity); gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer); gl.bufferData(gl.ARRAY_BUFFER, rectVertices(context.canvas, rect, rotation), gl.DYNAMIC_DRAW); gl.enableVertexAttribArray(maskRevealPreviewPositionLocation); gl.vertexAttribPointer(maskRevealPreviewPositionLocation, 2, gl.FLOAT, false, 0, 0); gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer); gl.bufferData(gl.ARRAY_BUFFER, textureCoordinatesForCrop(asset.intrinsicSize, sourceRect), gl.DYNAMIC_DRAW); gl.enableVertexAttribArray(maskRevealPreviewTexCoordLocation); gl.vertexAttribPointer(maskRevealPreviewTexCoordLocation, 2, gl.FLOAT, false, 0, 0); gl.bindBuffer(gl.ARRAY_BUFFER, maskTexCoordBuffer); gl.bufferData(gl.ARRAY_BUFFER, textureCoordinatesForRect(rect, maskRect), gl.DYNAMIC_DRAW); gl.enableVertexAttribArray(maskRevealPreviewMaskTexCoordLocation); gl.vertexAttribPointer(maskRevealPreviewMaskTexCoordLocation, 2, gl.FLOAT, false, 0, 0); gl.drawArrays(gl.TRIANGLES, 0, 6); gl.disable(gl.BLEND); return true; }, renderMaskVisualization(maskAsset, maskRect, mode, color = [1, 1, 1, 1], clipRect) { const drawRect = clipRect ? intersectScreenRects(maskRect, clipRect) : maskRect; if (!drawRect || drawRect.w <= 0 || drawRect.h <= 0) return true; const resources = getMaskVisualizationResources(gl, () => maskVisualizationResources, (nextResources) => { maskVisualizationResources = nextResources; }); if (!resources) return false; const maskEntry = getTextureEntry(context, textures, maskAsset, invalidate, () => disposed); const maskTexture = renderableTexture(maskEntry); if (!maskTexture) return false; gl.enable(gl.SCISSOR_TEST); gl.scissor(drawRect.x, context.canvas.height - drawRect.y - drawRect.h, drawRect.w, drawRect.h); enablePremultipliedAlphaBlending(gl); gl.useProgram(resources.program); gl.activeTexture(gl.TEXTURE0); gl.bindTexture(gl.TEXTURE_2D, maskTexture); gl.uniform1i(resources.samplerLocation, 0); gl.uniform1i(resources.modeLocation, maskVisualizationModeValue(mode)); gl.uniform4fv(resources.colorLocation, color); gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer); gl.bufferData(gl.ARRAY_BUFFER, rectVertices(context.canvas, drawRect), gl.DYNAMIC_DRAW); gl.enableVertexAttribArray(resources.positionLocation); gl.vertexAttribPointer(resources.positionLocation, 2, gl.FLOAT, false, 0, 0); gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer); gl.bufferData(gl.ARRAY_BUFFER, texCoordsForRect(drawRect, maskRect), gl.DYNAMIC_DRAW); gl.enableVertexAttribArray(resources.texCoordLocation); gl.vertexAttribPointer(resources.texCoordLocation, 2, gl.FLOAT, false, 0, 0); gl.drawArrays(gl.TRIANGLES, 0, 6); gl.disable(gl.BLEND); return true; }, renderTinted(asset, rect, color, clipRect, opacity = 1) { const clampedOpacity = clampOpacity(opacity); if (clampedOpacity <= 0) return true; const drawRect = clipRect ? intersectScreenRects(rect, clipRect) : rect; if (!drawRect || drawRect.w <= 0 || drawRect.h <= 0) return true; const entry = getTextureEntry(context, textures, asset, invalidate, () => disposed); const texture = renderableTexture(entry); if (!texture) return false; gl.enable(gl.SCISSOR_TEST); gl.scissor(drawRect.x, context.canvas.height - drawRect.y - drawRect.h, drawRect.w, drawRect.h); enablePremultipliedAlphaBlending(gl); gl.useProgram(tintedProgram); gl.activeTexture(gl.TEXTURE0); gl.bindTexture(gl.TEXTURE_2D, texture); gl.uniform1i(tintedSamplerLocation, 0); gl.uniform4fv(tintedColorLocation, [color[0], color[1], color[2], color[3] * clampedOpacity]); gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer); gl.bufferData(gl.ARRAY_BUFFER, rectVertices(context.canvas, rect), gl.DYNAMIC_DRAW); gl.enableVertexAttribArray(tintedPositionLocation); gl.vertexAttribPointer(tintedPositionLocation, 2, gl.FLOAT, false, 0, 0); gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer); gl.bufferData(gl.ARRAY_BUFFER, fullTexCoords(), gl.DYNAMIC_DRAW); gl.enableVertexAttribArray(tintedTexCoordLocation); gl.vertexAttribPointer(tintedTexCoordLocation, 2, gl.FLOAT, false, 0, 0); gl.drawArrays(gl.TRIANGLES, 0, 6); gl.disable(gl.BLEND); return true; }, dispose() { disposed = true; for (const entry of textures.values()) disposeEntry(gl, entry); gl.deleteBuffer(positionBuffer); gl.deleteBuffer(texCoordBuffer); gl.deleteBuffer(maskTexCoordBuffer); gl.deleteProgram(program); gl.deleteProgram(maskedProgram); gl.deleteProgram(maskRevealPreviewProgram); if (maskVisualizationResources && maskVisualizationResources !== "failed") gl.deleteProgram(maskVisualizationResources.program); gl.deleteProgram(tintedProgram); }, }; } function getTextureEntry( context: WebGlRendererContext, textures: Map, asset: Asset, invalidate: () => void, isDisposed: () => boolean, ): TextureEntry { const cached = textures.get(asset.id); if (cached && cached.source === asset.source) return cached; if (cached) { if (cached.current.status === "ready") { if (cached.previousTexture) context.gl.deleteTexture(cached.previousTexture); cached.previousTexture = cached.current.texture; } else { disposeState(context.gl, cached.current); } cached.source = asset.source; cached.current = loadTextureState(context, asset, cached, invalidate, isDisposed); return cached; } const entry: TextureEntry = { source: asset.source, current: { status: "error" } }; entry.current = loadTextureState(context, asset, entry, invalidate, isDisposed); textures.set(asset.id, entry); return entry; } function loadTextureState( context: WebGlRendererContext, asset: Asset, entry: TextureEntry, invalidate: () => void, isDisposed: () => boolean, ): TextureState { const image = new Image(); const source = asset.source; image.onload = () => { if (isDisposed() || entry.source !== source) return; const textureSource = asset.mimeType === "image/svg+xml" ? rasterizeSvgImage(image, asset) : image; const texture = createTexture(context.gl, textureSource); if (entry.previousTexture) context.gl.deleteTexture(entry.previousTexture); entry.previousTexture = undefined; entry.current = { status: "ready", texture }; invalidate(); }; image.onerror = () => { if (isDisposed() || entry.source !== source) return; entry.current = { status: "error" }; invalidate(); }; image.src = source; return { status: "loading", image }; } function rasterizeSvgImage(image: HTMLImageElement, asset: Asset): HTMLCanvasElement { const canvas = document.createElement("canvas"); canvas.width = Math.max(1, Math.ceil(asset.intrinsicSize.w)); canvas.height = Math.max(1, Math.ceil(asset.intrinsicSize.h)); const context = canvas.getContext("2d"); if (!context) throw new Error("Failed to create SVG rasterization context"); context.clearRect(0, 0, canvas.width, canvas.height); context.drawImage(image, 0, 0, canvas.width, canvas.height); return canvas; } function renderableTexture(entry: TextureEntry): WebGLTexture | undefined { if (entry.current.status === "ready") return entry.current.texture; return entry.previousTexture; } function disposeEntry(gl: WebGL2RenderingContext, entry: TextureEntry) { disposeState(gl, entry.current); if (entry.previousTexture) gl.deleteTexture(entry.previousTexture); } function disposeState(gl: WebGL2RenderingContext, state: TextureState) { if (state.status === "ready") { gl.deleteTexture(state.texture); return; } if (state.status === "loading") { state.image.onload = null; state.image.onerror = null; } } function intersectScreenRects(a: ScreenRect, b: ScreenRect): ScreenRect | undefined { const x1 = Math.max(a.x, b.x); const y1 = Math.max(a.y, b.y); const x2 = Math.min(a.x + a.w, b.x + b.w); const y2 = Math.min(a.y + a.h, b.y + b.h); if (x2 <= x1 || y2 <= y1) return undefined; return { x: x1, y: y1, w: x2 - x1, h: y2 - y1 }; } function createTexture(gl: WebGL2RenderingContext, image: TexImageSource) { const texture = gl.createTexture(); if (!texture) throw new Error("Failed to create image texture"); gl.bindTexture(gl.TEXTURE_2D, texture); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR); gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, false); gl.pixelStorei(gl.UNPACK_PREMULTIPLY_ALPHA_WEBGL, true); gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, image); return texture; } function enablePremultipliedAlphaBlending(gl: WebGL2RenderingContext) { gl.enable(gl.BLEND); gl.blendFunc(gl.ONE, gl.ONE_MINUS_SRC_ALPHA); } function clampOpacity(opacity: number) { return Math.max(0, Math.min(1, opacity)); } function fullTexCoords() { return new Float32Array([0, 0, 1, 0, 0, 1, 0, 1, 1, 0, 1, 1]); } function texCoordsForRect(drawRect: ScreenRect, sourceRect: ScreenRect) { const x1 = (drawRect.x - sourceRect.x) / sourceRect.w; const x2 = (drawRect.x + drawRect.w - sourceRect.x) / sourceRect.w; const y1 = (drawRect.y - sourceRect.y) / sourceRect.h; const y2 = (drawRect.y + drawRect.h - sourceRect.y) / sourceRect.h; return new Float32Array([x1, y1, x2, y1, x1, y2, x1, y2, x2, y1, x2, y2]); } function rectVertices(canvas: HTMLCanvasElement, rect: ScreenRect, rotation = 0) { const [nw, ne, sw, se] = rotatedRectCorners(rect, rotation).map((point) => ({ x: (point.x / canvas.width) * 2 - 1, y: 1 - (point.y / canvas.height) * 2, })); return new Float32Array([nw!.x, nw!.y, ne!.x, ne!.y, sw!.x, sw!.y, sw!.x, sw!.y, ne!.x, ne!.y, se!.x, se!.y]); }