From 1957d0dffaa91ae619c738e95567dc645ce96b1b Mon Sep 17 00:00:00 2001 From: syntaxbullet Date: Wed, 17 Jun 2026 15:39:04 +0200 Subject: [PATCH] Validate and correct structured LLM JSON --- bun.lock | 3 + package.json | 3 +- packages/summarizer/summarizer.ts | 154 +++++++++++++++++++++++++++--- 3 files changed, 144 insertions(+), 16 deletions(-) diff --git a/bun.lock b/bun.lock index 43252c4..339460a 100644 --- a/bun.lock +++ b/bun.lock @@ -11,6 +11,7 @@ "pdfkit": "^0.18.0", "playwright": "^1.59.1", "xlsx": "^0.18.5", + "zod": "^4.4.3", }, "devDependencies": { "@types/bun": "latest", @@ -290,6 +291,8 @@ "zip-stream": ["zip-stream@4.1.1", "", { "dependencies": { "archiver-utils": "^3.0.4", "compress-commons": "^4.1.2", "readable-stream": "^3.6.0" } }, "sha512-9qv4rlDiopXg4E69k+vMHjNN63YFMe9sZMrdlvKnCjlCRWeCBswPPMPUfx+ipsAWq1LXHe70RcbaHdJJpS6hyQ=="], + "zod": ["zod@4.4.3", "", {}, "sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ=="], + "@fast-csv/format/@types/node": ["@types/node@14.18.63", "", {}, "sha512-fAtCfv4jJg+ExtXhvCkCqUKZ+4ok/JQk01qDKhL5BDDoS3AxKXhV5/MAVUZyQnSEd2GT92fkgZl0pz0Q0AzcIQ=="], "@fast-csv/parse/@types/node": ["@types/node@14.18.63", "", {}, "sha512-fAtCfv4jJg+ExtXhvCkCqUKZ+4ok/JQk01qDKhL5BDDoS3AxKXhV5/MAVUZyQnSEd2GT92fkgZl0pz0Q0AzcIQ=="], diff --git a/package.json b/package.json index f7aa05e..61295a0 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,7 @@ "pdfjs-dist": "^5.6.205", "pdfkit": "^0.18.0", "playwright": "^1.59.1", - "xlsx": "^0.18.5" + "xlsx": "^0.18.5", + "zod": "^4.4.3" } } diff --git a/packages/summarizer/summarizer.ts b/packages/summarizer/summarizer.ts index 354c7c5..5c23882 100644 --- a/packages/summarizer/summarizer.ts +++ b/packages/summarizer/summarizer.ts @@ -3,6 +3,7 @@ import { readdir } from "node:fs/promises"; import { join } from "node:path"; import OpenAI from "openai"; import * as XLSX from "xlsx"; +import { z } from "zod"; import { SCORING_MODEL, type Ampelfarbe } from "./scoring-model"; import { calculateScoringResult, @@ -118,6 +119,42 @@ const OPENROUTER_PRIVACY_PROVIDER = { const MAX_LLM_RETRY_ATTEMPTS = Math.max(1, Number(process.env.BMP_LLM_RETRY_ATTEMPTS ?? 3)); const LLM_RETRY_BASE_DELAY_MS = Math.max(0, Number(process.env.BMP_LLM_RETRY_BASE_DELAY_MS ?? 750)); +const frageMitAntwortSchema = z.object({ + id: z.string(), + text: z.string(), + antwort: z.string(), + confidence: z.number().min(0).max(1), +}).strict(); + +const segmentierungResponseSchema = z.object({ + fragen: z.array(frageMitAntwortSchema), +}).strict(); + +const scoringSubcriterionSchema = z.object({ + id: z.string(), + farbe: z.enum(["gruen", "gelb", "rot", "unbewertbar"]), + evidence: z.string(), + begruendung: z.string(), + confidence: z.number().min(0).max(1), + missingReason: z.string(), +}).strict(); + +const scoringResponseSchema = z.object({ + begruendung: z.string().optional(), + ausschlussgruende: z.array(z.string()), + dimensionen: z.array(z.object({ + id: z.string(), + subcriteria: z.array(scoringSubcriterionSchema), + }).strict()), +}).strict(); + +const swotResponseSchema = z.object({ + staerken: z.array(z.string()), + schwaechen: z.array(z.string()), + chancen: z.array(z.string()), + risiken: z.array(z.string()), +}).strict(); + function createClient(): OpenAI { const apiKey = process.env.OPENROUTER_API_KEY; if (!apiKey) throw new Error("OPENROUTER_API_KEY is not set in environment"); @@ -334,6 +371,7 @@ async function createParsedStructuredCompletion( messages: Array<{ role: "system" | "user"; content: string }>, schemaName: string, schema: Record, + validator: z.ZodType, context: string, sessionId?: string, trace?: { @@ -346,7 +384,24 @@ async function createParsedStructuredCompletion( for (let attempt = 1; attempt <= MAX_LLM_RETRY_ATTEMPTS; attempt += 1) { try { const completion = await createStructuredCompletion(client, model, messages, schemaName, schema, sessionId, trace); - return parseLlmJson(completionContent(completion, context), context); + const content = completionContent(completion, context); + try { + return parseAndValidateLlmJson(content, context, validator); + } catch (error) { + lastError = error; + return await correctStructuredJson( + client, + model, + content, + error, + schemaName, + schema, + validator, + context, + sessionId, + trace, + ); + } } catch (error) { lastError = error; if (attempt >= MAX_LLM_RETRY_ATTEMPTS) { @@ -434,6 +489,74 @@ function parseLlmJson(content: string, context: string): T { } } +function validationMessage(error: unknown): string { + if (error instanceof z.ZodError) { + return error.issues + .map((issue) => `${issue.path.join(".") || "(root)"}: ${issue.message}`) + .join("\n"); + } + return errorMessage(error); +} + +function parseAndValidateLlmJson(content: string, context: string, validator: z.ZodType): T { + const parsed = parseLlmJson(content, context); + const result = validator.safeParse(parsed); + if (!result.success) { + throw new SyntaxError(`${context}: ${validationMessage(result.error)}`); + } + return result.data; +} + +async function correctStructuredJson( + client: OpenAI, + model: string, + invalidContent: string, + validationError: unknown, + schemaName: string, + schema: Record, + validator: z.ZodType, + context: string, + sessionId?: string, + trace?: { + operation: LlmCallTrace["operation"]; + label: string; + onLlmCall?: SummarizeCompanyOptions["onLlmCall"]; + }, +): Promise { + const correctionContext = `${context} correction`; + const completion = await createStructuredCompletion( + client, + model, + [ + { + role: "system", + content: + "Du reparierst fehlerhafte JSON-Ausgaben. Antworte ausschliesslich mit gueltigem JSON, das dem Schema entspricht. Erfinde keine neuen Informationen.", + }, + { + role: "user", + content: `Die folgende JSON-Ausgabe konnte nicht verarbeitet werden. + +Fehler: +${validationMessage(validationError)} + +Erwartetes JSON Schema: +${JSON.stringify(schema, null, 2)} + +Fehlerhafte Ausgabe: +${invalidContent} + +Korrigiere nur Syntax, Typen, fehlende Pflichtfelder und enum-Werte. Antworte ausschliesslich mit dem korrigierten JSON.`, + }, + ], + `${schemaName}_correction`, + schema, + sessionId, + trace ? { ...trace, label: `${trace.label} (JSON-Korrektur)` } : undefined, + ); + return parseAndValidateLlmJson(completionContent(completion, correctionContext), correctionContext, validator); +} + function summarizeSegmentierungsQualitaet(fragen: FrageMitAntwort[] | undefined): SegmentierungsQualitaet | undefined { const items = fragen ?? []; if (!items.length) return undefined; @@ -916,7 +1039,7 @@ function buildScoringFallbackBegruendung( return `${farbe} (${meaning}). Staerkste Bereiche: ${strongestDimensions || "keine"}.${missing}`; } -function fallbackScoringAssessment(reason: string): Partial & { begruendung?: string } { +function fallbackScoringAssessment(reason: string): z.infer { return { begruendung: `Die automatische Detailbewertung konnte nicht vollstaendig ausgewertet werden: ${reason}`, ausschlussgruende: [], @@ -989,11 +1112,9 @@ Bewerbungsdaten: ${JSON.stringify(data, null, 2)}`, }, ]; - let parsed: Partial & { begruendung?: string }; + let parsed: z.infer; try { - parsed = await createParsedStructuredCompletion & { - begruendung?: string; - }>( + parsed = await createParsedStructuredCompletion>( client, model, messages, @@ -1037,6 +1158,7 @@ ${JSON.stringify(data, null, 2)}`, }, required: ["begruendung", "ausschlussgruende", "dimensionen"], }, + scoringResponseSchema, "Could not parse traffic light assessment JSON", sessionId, { operation: "scoring", label: "Ampelbewertung", onLlmCall }, @@ -1107,9 +1229,9 @@ Vorgaben: - Lasse keine Unterfrage aus.`, }, ]; - let parsed: { fragen?: Array> }; + let parsed: z.infer; try { - parsed = await createParsedStructuredCompletion<{ fragen?: Array> }>( + parsed = await createParsedStructuredCompletion>( client, model, messages, @@ -1135,24 +1257,25 @@ Vorgaben: }, required: ["fragen"], }, + segmentierungResponseSchema, `Could not parse answer segmentation JSON for ${fallbackLabel}`, sessionId, { operation: "segmentierung", label: fallbackLabel, onLlmCall }, ); } catch { parsed = { - fragen: fragen.map((frage) => ({ - id: frage.id, + fragen: fragen.map((frage, index) => ({ + id: frage.id ?? `frage_${index + 1}`, text: frage.text, antwort: "", confidence: 0, })), }; } - const byId = new Map((parsed.fragen ?? []).map((frage) => [String(frage.id ?? ""), frage])); + const byId = new Map(parsed.fragen.map((frage) => [frage.id, frage])); - return fragen.map((frage) => { - const mapped = byId.get(String(frage.id ?? "")); + return fragen.map((frage, index) => { + const mapped = byId.get(String(frage.id ?? `frage_${index + 1}`)); const rawConfidence = mapped?.confidence; const confidence = typeof rawConfidence === "number" @@ -1204,9 +1327,9 @@ Bewerbungsdaten: ${JSON.stringify(data, null, 2)}`, }, ]; - let parsed: Partial; + let parsed: z.infer; try { - parsed = await createParsedStructuredCompletion>( + parsed = await createParsedStructuredCompletion>( client, model, messages, @@ -1222,6 +1345,7 @@ ${JSON.stringify(data, null, 2)}`, }, required: ["staerken", "schwaechen", "chancen", "risiken"], }, + swotResponseSchema, "Could not parse SWOT analysis JSON", sessionId, { operation: "swot", label: "SWOT-Analyse", onLlmCall },