Validate and correct structured LLM JSON

bun.lock

@@ -11,6 +11,7 @@
         "pdfkit": "^0.18.0",
         "playwright": "^1.59.1",
         "xlsx": "^0.18.5",
+        "zod": "^4.4.3",
       },
       "devDependencies": {
         "@types/bun": "latest",
@@ -290,6 +291,8 @@
 
     "zip-stream": ["zip-stream@4.1.1", "", { "dependencies": { "archiver-utils": "^3.0.4", "compress-commons": "^4.1.2", "readable-stream": "^3.6.0" } }, "sha512-9qv4rlDiopXg4E69k+vMHjNN63YFMe9sZMrdlvKnCjlCRWeCBswPPMPUfx+ipsAWq1LXHe70RcbaHdJJpS6hyQ=="],
 
+    "zod": ["zod@4.4.3", "", {}, "sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ=="],
+
     "@fast-csv/format/@types/node": ["@types/node@14.18.63", "", {}, "sha512-fAtCfv4jJg+ExtXhvCkCqUKZ+4ok/JQk01qDKhL5BDDoS3AxKXhV5/MAVUZyQnSEd2GT92fkgZl0pz0Q0AzcIQ=="],
 
     "@fast-csv/parse/@types/node": ["@types/node@14.18.63", "", {}, "sha512-fAtCfv4jJg+ExtXhvCkCqUKZ+4ok/JQk01qDKhL5BDDoS3AxKXhV5/MAVUZyQnSEd2GT92fkgZl0pz0Q0AzcIQ=="],

package.json

@@ -22,6 +22,7 @@
     "pdfjs-dist": "^5.6.205",
     "pdfkit": "^0.18.0",
     "playwright": "^1.59.1",
-    "xlsx": "^0.18.5"
+    "xlsx": "^0.18.5",
+    "zod": "^4.4.3"
   }
 }

packages/summarizer/summarizer.ts

@@ -3,6 +3,7 @@ import { readdir } from "node:fs/promises";
 import { join } from "node:path";
 import OpenAI from "openai";
 import * as XLSX from "xlsx";
+import { z } from "zod";
 import { SCORING_MODEL, type Ampelfarbe } from "./scoring-model";
 import {
   calculateScoringResult,
@@ -118,6 +119,42 @@ const OPENROUTER_PRIVACY_PROVIDER = {
 const MAX_LLM_RETRY_ATTEMPTS = Math.max(1, Number(process.env.BMP_LLM_RETRY_ATTEMPTS ?? 3));
 const LLM_RETRY_BASE_DELAY_MS = Math.max(0, Number(process.env.BMP_LLM_RETRY_BASE_DELAY_MS ?? 750));
 
+const frageMitAntwortSchema = z.object({
+  id: z.string(),
+  text: z.string(),
+  antwort: z.string(),
+  confidence: z.number().min(0).max(1),
+}).strict();
+
+const segmentierungResponseSchema = z.object({
+  fragen: z.array(frageMitAntwortSchema),
+}).strict();
+
+const scoringSubcriterionSchema = z.object({
+  id: z.string(),
+  farbe: z.enum(["gruen", "gelb", "rot", "unbewertbar"]),
+  evidence: z.string(),
+  begruendung: z.string(),
+  confidence: z.number().min(0).max(1),
+  missingReason: z.string(),
+}).strict();
+
+const scoringResponseSchema = z.object({
+  begruendung: z.string().optional(),
+  ausschlussgruende: z.array(z.string()),
+  dimensionen: z.array(z.object({
+    id: z.string(),
+    subcriteria: z.array(scoringSubcriterionSchema),
+  }).strict()),
+}).strict();
+
+const swotResponseSchema = z.object({
+  staerken: z.array(z.string()),
+  schwaechen: z.array(z.string()),
+  chancen: z.array(z.string()),
+  risiken: z.array(z.string()),
+}).strict();
+
 function createClient(): OpenAI {
   const apiKey = process.env.OPENROUTER_API_KEY;
   if (!apiKey) throw new Error("OPENROUTER_API_KEY is not set in environment");
@@ -334,6 +371,7 @@ async function createParsedStructuredCompletion<T>(
   messages: Array<{ role: "system" | "user"; content: string }>,
   schemaName: string,
   schema: Record<string, unknown>,
+  validator: z.ZodType<T>,
   context: string,
   sessionId?: string,
   trace?: {
@@ -346,7 +384,24 @@ async function createParsedStructuredCompletion<T>(
   for (let attempt = 1; attempt <= MAX_LLM_RETRY_ATTEMPTS; attempt += 1) {
     try {
       const completion = await createStructuredCompletion(client, model, messages, schemaName, schema, sessionId, trace);
-      return parseLlmJson<T>(completionContent(completion, context), context);
+      const content = completionContent(completion, context);
+      try {
+        return parseAndValidateLlmJson(content, context, validator);
+      } catch (error) {
+        lastError = error;
+        return await correctStructuredJson(
+          client,
+          model,
+          content,
+          error,
+          schemaName,
+          schema,
+          validator,
+          context,
+          sessionId,
+          trace,
+        );
+      }
     } catch (error) {
       lastError = error;
       if (attempt >= MAX_LLM_RETRY_ATTEMPTS) {
@@ -434,6 +489,74 @@ function parseLlmJson<T>(content: string, context: string): T {
   }
 }
 
+function validationMessage(error: unknown): string {
+  if (error instanceof z.ZodError) {
+    return error.issues
+      .map((issue) => `${issue.path.join(".") || "(root)"}: ${issue.message}`)
+      .join("\n");
+  }
+  return errorMessage(error);
+}
+
+function parseAndValidateLlmJson<T>(content: string, context: string, validator: z.ZodType<T>): T {
+  const parsed = parseLlmJson<unknown>(content, context);
+  const result = validator.safeParse(parsed);
+  if (!result.success) {
+    throw new SyntaxError(`${context}: ${validationMessage(result.error)}`);
+  }
+  return result.data;
+}
+
+async function correctStructuredJson<T>(
+  client: OpenAI,
+  model: string,
+  invalidContent: string,
+  validationError: unknown,
+  schemaName: string,
+  schema: Record<string, unknown>,
+  validator: z.ZodType<T>,
+  context: string,
+  sessionId?: string,
+  trace?: {
+    operation: LlmCallTrace["operation"];
+    label: string;
+    onLlmCall?: SummarizeCompanyOptions["onLlmCall"];
+  },
+): Promise<T> {
+  const correctionContext = `${context} correction`;
+  const completion = await createStructuredCompletion(
+    client,
+    model,
+    [
+      {
+        role: "system",
+        content:
+          "Du reparierst fehlerhafte JSON-Ausgaben. Antworte ausschliesslich mit gueltigem JSON, das dem Schema entspricht. Erfinde keine neuen Informationen.",
+      },
+      {
+        role: "user",
+        content: `Die folgende JSON-Ausgabe konnte nicht verarbeitet werden.
+
+Fehler:
+${validationMessage(validationError)}
+
+Erwartetes JSON Schema:
+${JSON.stringify(schema, null, 2)}
+
+Fehlerhafte Ausgabe:
+${invalidContent}
+
+Korrigiere nur Syntax, Typen, fehlende Pflichtfelder und enum-Werte. Antworte ausschliesslich mit dem korrigierten JSON.`,
+      },
+    ],
+    `${schemaName}_correction`,
+    schema,
+    sessionId,
+    trace ? { ...trace, label: `${trace.label} (JSON-Korrektur)` } : undefined,
+  );
+  return parseAndValidateLlmJson(completionContent(completion, correctionContext), correctionContext, validator);
+}
+
 function summarizeSegmentierungsQualitaet(fragen: FrageMitAntwort[] | undefined): SegmentierungsQualitaet | undefined {
   const items = fragen ?? [];
   if (!items.length) return undefined;
@@ -916,7 +1039,7 @@ function buildScoringFallbackBegruendung(
   return `${farbe} (${meaning}). Staerkste Bereiche: ${strongestDimensions || "keine"}.${missing}`;
 }
 
-function fallbackScoringAssessment(reason: string): Partial<LlmScoringAssessment> & { begruendung?: string } {
+function fallbackScoringAssessment(reason: string): z.infer<typeof scoringResponseSchema> {
   return {
     begruendung: `Die automatische Detailbewertung konnte nicht vollstaendig ausgewertet werden: ${reason}`,
     ausschlussgruende: [],
@@ -989,11 +1112,9 @@ Bewerbungsdaten:
 ${JSON.stringify(data, null, 2)}`,
       },
     ];
-  let parsed: Partial<LlmScoringAssessment> & { begruendung?: string };
+  let parsed: z.infer<typeof scoringResponseSchema>;
   try {
-    parsed = await createParsedStructuredCompletion<Partial<LlmScoringAssessment> & {
+    parsed = await createParsedStructuredCompletion<z.infer<typeof scoringResponseSchema>>(
-      begruendung?: string;
-    }>(
       client,
       model,
       messages,
@@ -1037,6 +1158,7 @@ ${JSON.stringify(data, null, 2)}`,
       },
       required: ["begruendung", "ausschlussgruende", "dimensionen"],
     },
+    scoringResponseSchema,
     "Could not parse traffic light assessment JSON",
     sessionId,
     { operation: "scoring", label: "Ampelbewertung", onLlmCall },
@@ -1107,9 +1229,9 @@ Vorgaben:
 - Lasse keine Unterfrage aus.`,
       },
     ];
-  let parsed: { fragen?: Array<Partial<FrageMitAntwort>> };
+  let parsed: z.infer<typeof segmentierungResponseSchema>;
   try {
-    parsed = await createParsedStructuredCompletion<{ fragen?: Array<Partial<FrageMitAntwort>> }>(
+    parsed = await createParsedStructuredCompletion<z.infer<typeof segmentierungResponseSchema>>(
       client,
       model,
       messages,
@@ -1135,24 +1257,25 @@ Vorgaben:
       },
       required: ["fragen"],
     },
+    segmentierungResponseSchema,
     `Could not parse answer segmentation JSON for ${fallbackLabel}`,
     sessionId,
     { operation: "segmentierung", label: fallbackLabel, onLlmCall },
     );
   } catch {
     parsed = {
-      fragen: fragen.map((frage) => ({
+      fragen: fragen.map((frage, index) => ({
-        id: frage.id,
+        id: frage.id ?? `frage_${index + 1}`,
         text: frage.text,
         antwort: "",
         confidence: 0,
       })),
     };
   }
-  const byId = new Map((parsed.fragen ?? []).map((frage) => [String(frage.id ?? ""), frage]));
+  const byId = new Map(parsed.fragen.map((frage) => [frage.id, frage]));
 
-  return fragen.map((frage) => {
+  return fragen.map((frage, index) => {
-    const mapped = byId.get(String(frage.id ?? ""));
+    const mapped = byId.get(String(frage.id ?? `frage_${index + 1}`));
     const rawConfidence = mapped?.confidence;
     const confidence =
       typeof rawConfidence === "number"
@@ -1204,9 +1327,9 @@ Bewerbungsdaten:
 ${JSON.stringify(data, null, 2)}`,
       },
     ];
-  let parsed: Partial<SWOTAnalyse>;
+  let parsed: z.infer<typeof swotResponseSchema>;
   try {
-    parsed = await createParsedStructuredCompletion<Partial<SWOTAnalyse>>(
+    parsed = await createParsedStructuredCompletion<z.infer<typeof swotResponseSchema>>(
       client,
       model,
       messages,
@@ -1222,6 +1345,7 @@ ${JSON.stringify(data, null, 2)}`,
       },
       required: ["staerken", "schwaechen", "chancen", "risiken"],
     },
+    swotResponseSchema,
     "Could not parse SWOT analysis JSON",
     sessionId,
     { operation: "swot", label: "SWOT-Analyse", onLlmCall },