Add summarizer observability and LLM retries

packages/summarizer/summarizer.ts

@@ -90,6 +90,8 @@ export interface LlmCallTrace {
   operation: "segmentierung" | "zusammenfassung" | "scoring" | "swot";
   label: string;
   model: string;
+  attempt: number;
+  maxAttempts: number;
   status: LlmCallStatus;
   startedAt: string;
   finishedAt?: string;
@@ -101,6 +103,8 @@ export interface LlmCallTrace {
 
 export interface SummarizeCompanyOptions {
   onLlmCall?: (trace: LlmCallTrace) => void;
+  onCompanyInput?: (stem: string, input: ExtractedData) => void;
+  onCompanyOutput?: (stem: string, output: SummaryData) => void;
 }
 
 const OUTPUTS_DIR =
@@ -111,6 +115,8 @@ const OPENROUTER_PRIVACY_PROVIDER = {
   data_collection: "deny",
   zdr: true,
 } as const;
+const MAX_LLM_RETRY_ATTEMPTS = Math.max(1, Number(process.env.BMP_LLM_RETRY_ATTEMPTS ?? 3));
+const LLM_RETRY_BASE_DELAY_MS = Math.max(0, Number(process.env.BMP_LLM_RETRY_BASE_DELAY_MS ?? 750));
 
 function createClient(): OpenAI {
   const apiKey = process.env.OPENROUTER_API_KEY;
@@ -169,12 +175,16 @@ function beginLlmTrace(
   label: string,
   model: string,
   request: Record<string, unknown>,
+  attempt: number,
+  maxAttempts: number,
 ): LlmCallTrace {
   const trace: LlmCallTrace = {
     id: crypto.randomUUID(),
     operation,
     label,
     model,
+    attempt,
+    maxAttempts,
     status: "running",
     startedAt: new Date().toISOString(),
     request,
@@ -200,6 +210,66 @@ function finishLlmTrace(
   });
 }
 
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function retryDelayMs(attempt: number): number {
+  if (LLM_RETRY_BASE_DELAY_MS <= 0) return 0;
+  const jitter = Math.floor(Math.random() * Math.min(250, LLM_RETRY_BASE_DELAY_MS));
+  return LLM_RETRY_BASE_DELAY_MS * 2 ** Math.max(0, attempt - 1) + jitter;
+}
+
+function shouldRetryLlmError(error: unknown): boolean {
+  const status = (error as { status?: unknown; code?: unknown })?.status;
+  if (typeof status === "number") {
+    return status === 408 || status === 409 || status === 425 || status === 429 || status >= 500;
+  }
+  const message = String(error).toLowerCase();
+  return [
+    "timeout",
+    "timed out",
+    "rate limit",
+    "temporarily",
+    "econnreset",
+    "fetch failed",
+    "socket",
+    "overloaded",
+    "provider returned error",
+  ].some((needle) => message.includes(needle));
+}
+
+async function createCompletionWithRetry(
+  client: OpenAI,
+  request: Record<string, unknown>,
+  trace: {
+    operation: LlmCallTrace["operation"];
+    label: string;
+    model: string;
+    onLlmCall?: SummarizeCompanyOptions["onLlmCall"];
+  } | undefined,
+) {
+  let lastError: unknown;
+  for (let attempt = 1; attempt <= MAX_LLM_RETRY_ATTEMPTS; attempt += 1) {
+    const llmTrace = trace
+      ? beginLlmTrace(trace.onLlmCall, trace.operation, trace.label, trace.model, request, attempt, MAX_LLM_RETRY_ATTEMPTS)
+      : undefined;
+    try {
+      const completion = await client.chat.completions.create(request as never);
+      if (llmTrace) finishLlmTrace(trace?.onLlmCall, llmTrace, { response: completion });
+      return completion;
+    } catch (error) {
+      lastError = error;
+      if (llmTrace) finishLlmTrace(trace?.onLlmCall, llmTrace, { error });
+      if (attempt >= MAX_LLM_RETRY_ATTEMPTS || !shouldRetryLlmError(error)) {
+        throw error;
+      }
+      await sleep(retryDelayMs(attempt));
+    }
+  }
+  throw lastError;
+}
+
 async function createStructuredCompletion(
   client: OpenAI,
   model: string,
@@ -222,19 +292,16 @@ async function createStructuredCompletion(
     session_id: sessionId,
   };
   const primaryTrace = trace
-    ? beginLlmTrace(trace.onLlmCall, trace.operation, trace.label, model, structuredRequest as Record<string, unknown>)
+    ? { ...trace, model }
     : undefined;
   try {
-    const completion = await client.chat.completions.create(structuredRequest as never);
-    if (primaryTrace) finishLlmTrace(trace?.onLlmCall, primaryTrace, { response: completion });
+    const completion = await createCompletionWithRetry(client, structuredRequest as Record<string, unknown>, primaryTrace);
     return completion;
   } catch (error) {
     const message = String(error);
     if (!message.includes("400") && !message.toLowerCase().includes("provider returned error")) {
-      if (primaryTrace) finishLlmTrace(trace?.onLlmCall, primaryTrace, { error });
       throw error;
     }
-    if (primaryTrace) finishLlmTrace(trace?.onLlmCall, primaryTrace, { error });
 
     const fallbackRequest = {
       model,
@@ -245,16 +312,9 @@ async function createStructuredCompletion(
       session_id: sessionId,
     };
     const fallbackTrace = trace
-      ? beginLlmTrace(trace.onLlmCall, trace.operation, `${trace.label} (JSON fallback)`, model, fallbackRequest as Record<string, unknown>)
+      ? { ...trace, label: `${trace.label} (JSON fallback)`, model }
       : undefined;
-    try {
-      const completion = await client.chat.completions.create(fallbackRequest as never);
-      if (fallbackTrace) finishLlmTrace(trace?.onLlmCall, fallbackTrace, { response: completion });
-      return completion;
-    } catch (fallbackError) {
-      if (fallbackTrace) finishLlmTrace(trace?.onLlmCall, fallbackTrace, { error: fallbackError });
-      throw fallbackError;
-    }
+    return createCompletionWithRetry(client, fallbackRequest as Record<string, unknown>, fallbackTrace);
   }
 }
 
@@ -287,16 +347,9 @@ async function summarizeText(
     session_id: sessionId,
   };
   const llmTrace = trace
-    ? beginLlmTrace(trace.onLlmCall, "zusammenfassung", trace.label, model, request as Record<string, unknown>)
+    ? { operation: "zusammenfassung" as const, label: trace.label, model, onLlmCall: trace.onLlmCall }
     : undefined;
-  let completion: Awaited<ReturnType<typeof client.chat.completions.create>>;
-  try {
-    completion = await client.chat.completions.create(request as never);
-    if (llmTrace) finishLlmTrace(trace?.onLlmCall, llmTrace, { response: completion });
-  } catch (error) {
-    if (llmTrace) finishLlmTrace(trace?.onLlmCall, llmTrace, { error });
-    throw error;
-  }
+  const completion = await createCompletionWithRetry(client, request as Record<string, unknown>, llmTrace);
 
   const msg = completion.choices[0]?.message as Record<string, unknown> | undefined;
   const content = (msg?.content ?? msg?.reasoning) as string | null | undefined;
@@ -1142,6 +1195,7 @@ export async function summarizeCompany(
   }
 
   const data: ExtractedData = await file.json();
+  options.onCompanyInput?.(stem, data);
   const client = createClient();
   const output: SummaryData = structuredClone(data);
   const privacy: Pseudonymizer = createPseudonymizer(data);
@@ -1292,6 +1346,7 @@ export async function summarizeCompany(
   output._privacy = privacy.audit();
   output._schemaVersion = 3;
   output._summarizedAt = new Date().toISOString();
+  options.onCompanyOutput?.(stem, output);
   return output;
 }