feat: add generation cancel job functionality and improve job handling

- Introduced `generationCancelJob` command ID in `commands/ids.ts`.
- Added `GenerationCancelJobPayload` type in `commands/payloads.ts`.
- Enhanced job status to include "cancelled" in `editor/state.ts`.
- Updated `runGenerationJob` to accept an `AbortSignal` and handle cancellation.
- Implemented cancellation logic in `runGenerate` and related functions.
- Added tests for job cancellation in `operations/generation/workflow.test.ts`.
- Improved layer rendering logic to prevent stack overflow in `editor/document-indexes.ts`.
- Added raster size assertions in `platform/browser/rasterLimits.ts` for image processing limits.
- Enhanced image file handling to check for size limits in `platform/browser/imageFiles.ts`.
- Updated UI components to reflect job cancellation state in `view/GenerationJobStatus.tsx` and `view/bottom-controls/GenerateActionControls.tsx`.

editor/document-indexes.test.ts

@@ -67,6 +67,17 @@ describe("document read indexes", () => {
     expect(visited).toEqual(["group", "target", "mask"]);
     expect(layers.map((layer) => layer.id)).toEqual(["mask", "target", "group"]);
   });
+
+  test("indexes pathological nesting without recursive stack overflow", () => {
+    let layers: Layer[] = [raster("leaf", "Leaf", "asset-nested", { x: 80, y: 10 }, { x: 2, y: 3 })];
+    for (let depth = 0; depth < 10_000; depth += 1) layers = [group(`group-${depth}`, `Group ${depth}`, layers)];
+    const deepDocument = { ...document, artboards: [{ ...document.artboards[0]!, layers }] };
+
+    const index = createDocumentReadIndex(deepDocument);
+
+    expect(index.layerById.size).toBe(10_001);
+    expect(resolveIndexedLayerBounds(index, layers[0]!)).toEqual({ x: 80, y: 10, w: 40, h: 30 });
+  });
 });
 
 function raster(

editor/document-indexes.ts

@@ -99,55 +99,58 @@ function indexLayerTree(options: {
   documentMaskLayerIds: Set<LayerId>;
   maskLayerIdsByLayerList: Map<readonly Layer[], ReadonlySet<LayerId>>;
 }): Set<LayerId> {
-  const layerListMaskLayerIds = new Set<LayerId>();
-
-  for (let index = 0; index < options.layers.length; index += 1) {
-    const layer = options.layers[index];
-    if (!layer) continue;
-
-    options.layerById.set(layer.id, layer);
-    options.layerInfoById.set(layer.id, {
-      artboardId: options.artboardId,
-      parentGroupId: options.parentGroupId,
-      layer,
-      siblings: options.layers,
-      index,
-    });
-
-    const layerMask = getLayerMask(layer);
-    if (layerMask) {
-      options.documentMaskLayerIds.add(layerMask.maskLayerId);
-      layerListMaskLayerIds.add(layerMask.maskLayerId);
+  type Frame = { layers: readonly Layer[]; parentGroupId?: LayerId; visited: boolean };
+  const stack: Frame[] = [{ layers: options.layers, parentGroupId: options.parentGroupId, visited: false }];
+  while (stack.length > 0) {
+    const frame = stack.pop();
+    if (!frame) continue;
+    if (frame.visited) {
+      const ids = new Set<LayerId>();
+      for (const layer of frame.layers) {
+        const mask = getLayerMask(layer);
+        if (mask) ids.add(mask.maskLayerId);
+        if (layer.type === "group") for (const id of options.maskLayerIdsByLayerList.get(layer.children) ?? []) ids.add(id);
+      }
+      options.maskLayerIdsByLayerList.set(frame.layers, ids);
+      continue;
     }
-
-    if (layer.type === "group") {
-      const childMaskLayerIds = indexLayerTree({
-        ...options,
-        layers: layer.children,
-        parentGroupId: layer.id,
-      });
-      for (const maskLayerId of childMaskLayerIds) layerListMaskLayerIds.add(maskLayerId);
+    stack.push({ ...frame, visited: true });
+    for (let index = frame.layers.length - 1; index >= 0; index -= 1) {
+      const layer = frame.layers[index];
+      if (!layer) continue;
+      options.layerById.set(layer.id, layer);
+      options.layerInfoById.set(layer.id, { artboardId: options.artboardId, parentGroupId: frame.parentGroupId, layer, siblings: frame.layers, index });
+      const mask = getLayerMask(layer);
+      if (mask) options.documentMaskLayerIds.add(mask.maskLayerId);
+      if (layer.type === "group") stack.push({ layers: layer.children, parentGroupId: layer.id, visited: false });
     }
   }
-
-  options.maskLayerIdsByLayerList.set(options.layers, layerListMaskLayerIds);
-  return layerListMaskLayerIds;
+  return new Set(options.maskLayerIdsByLayerList.get(options.layers) ?? []);
 }
 
 function countDisplayLayers(layers: readonly Layer[], maskLayerIds: ReadonlySet<LayerId>): number {
   let count = 0;
-  for (const layer of layers) {
+  const stack = [...layers];
+  while (stack.length > 0) {
+    const layer = stack.pop();
+    if (!layer) continue;
     if (maskLayerIds.has(layer.id)) continue;
     count += 1;
-    if (layer.type === "group") count += countDisplayLayers(layer.children, maskLayerIds);
+    if (layer.type === "group") stack.push(...layer.children);
   }
   return count;
 }
 
 function unionLayerBounds(index: DocumentReadIndex, layers: readonly Layer[]): Rect | undefined {
   let bounds: Rect | undefined;
-
-  for (const layer of layers) {
+  const stack = [...layers];
+  while (stack.length > 0) {
+    const layer = stack.pop();
+    if (!layer) continue;
+    if (layer.type === "group") {
+      stack.push(...layer.children);
+      continue;
+    }
     const layerBounds = resolveIndexedLayerBounds(index, layer);
     if (!layerBounds) continue;
     bounds = bounds ? unionRects(bounds, layerBounds) : layerBounds;

editor/state.ts

@@ -82,7 +82,7 @@ export type GenerationCandidate = {
 export type GenerationCompareMode = "result" | "before" | "split";
 
 export type GenerationJobKind = "generate" | "regenerate" | "refine" | "replace";
-export type GenerationJobStatus = "running" | "succeeded" | "failed";
+export type GenerationJobStatus = "running" | "succeeded" | "failed" | "cancelled";
 
 export type GenerationJob = {
   id: GenerationJobId;