syntaxbullet/bmp-project
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Validate and correct structured LLM JSON

Browse Source
This commit is contained in:
syntaxbullet
2026-06-17 15:39:04 +02:00
parent cb518fb95d
commit 1957d0dffa
3 changed files with 144 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

154
packages/summarizer/summarizer.ts
View File

@@ -3,6 +3,7 @@ import { readdir } from "node:fs/promises";
import { join } from "node:path";
import OpenAI from "openai";
import * as XLSX from "xlsx";
import { z } from "zod";
import { SCORING_MODEL, type Ampelfarbe } from "./scoring-model";
import {
calculateScoringResult,
@@ -118,6 +119,42 @@ const OPENROUTER_PRIVACY_PROVIDER = {
const MAX_LLM_RETRY_ATTEMPTS = Math.max(1, Number(process.env.BMP_LLM_RETRY_ATTEMPTS ?? 3));
const LLM_RETRY_BASE_DELAY_MS = Math.max(0, Number(process.env.BMP_LLM_RETRY_BASE_DELAY_MS ?? 750));
const frageMitAntwortSchema = z.object({
id: z.string(),
text: z.string(),
antwort: z.string(),
confidence: z.number().min(0).max(1),
}).strict();
const segmentierungResponseSchema = z.object({
fragen: z.array(frageMitAntwortSchema),
}).strict();
const scoringSubcriterionSchema = z.object({
id: z.string(),
farbe: z.enum(["gruen", "gelb", "rot", "unbewertbar"]),
evidence: z.string(),
begruendung: z.string(),
confidence: z.number().min(0).max(1),
missingReason: z.string(),
}).strict();
const scoringResponseSchema = z.object({
begruendung: z.string().optional(),
ausschlussgruende: z.array(z.string()),
dimensionen: z.array(z.object({
id: z.string(),
subcriteria: z.array(scoringSubcriterionSchema),
}).strict()),
}).strict();
const swotResponseSchema = z.object({
staerken: z.array(z.string()),
schwaechen: z.array(z.string()),
chancen: z.array(z.string()),
risiken: z.array(z.string()),
}).strict();
function createClient(): OpenAI {
const apiKey = process.env.OPENROUTER_API_KEY;
if (!apiKey) throw new Error("OPENROUTER_API_KEY is not set in environment");
@@ -334,6 +371,7 @@ async function createParsedStructuredCompletion<T>(
messages: Array<{ role: "system" | "user"; content: string }>,
schemaName: string,
schema: Record<string, unknown>,
validator: z.ZodType<T>,
context: string,
sessionId?: string,
trace?: {
@@ -346,7 +384,24 @@ async function createParsedStructuredCompletion<T>(
for (let attempt = 1; attempt <= MAX_LLM_RETRY_ATTEMPTS; attempt += 1) {
try {
const completion = await createStructuredCompletion(client, model, messages, schemaName, schema, sessionId, trace);
return parseLlmJson<T>(completionContent(completion, context), context);
const content = completionContent(completion, context);
try {
return parseAndValidateLlmJson(content, context, validator);
} catch (error) {
lastError = error;
return await correctStructuredJson(
client,
model,
content,
error,
schemaName,
schema,
validator,
context,
sessionId,
trace,
);
}
} catch (error) {
lastError = error;
if (attempt >= MAX_LLM_RETRY_ATTEMPTS) {
@@ -434,6 +489,74 @@ function parseLlmJson<T>(content: string, context: string): T {
}
}
function validationMessage(error: unknown): string {
if (error instanceof z.ZodError) {
return error.issues
.map((issue) => `${issue.path.join(".") || "(root)"}: ${issue.message}`)
.join("\n");
}
return errorMessage(error);
}
function parseAndValidateLlmJson<T>(content: string, context: string, validator: z.ZodType<T>): T {
const parsed = parseLlmJson<unknown>(content, context);
const result = validator.safeParse(parsed);
if (!result.success) {
throw new SyntaxError(`${context}: ${validationMessage(result.error)}`);
}
return result.data;
}
async function correctStructuredJson<T>(
client: OpenAI,
model: string,
invalidContent: string,
validationError: unknown,
schemaName: string,
schema: Record<string, unknown>,
validator: z.ZodType<T>,
context: string,
sessionId?: string,
trace?: {
operation: LlmCallTrace["operation"];
label: string;
onLlmCall?: SummarizeCompanyOptions["onLlmCall"];
},
): Promise<T> {
const correctionContext = `${context} correction`;
const completion = await createStructuredCompletion(
client,
model,
[
{
role: "system",
content:
"Du reparierst fehlerhafte JSON-Ausgaben. Antworte ausschliesslich mit gueltigem JSON, das dem Schema entspricht. Erfinde keine neuen Informationen.",
},
{
role: "user",
content: `Die folgende JSON-Ausgabe konnte nicht verarbeitet werden.
Fehler:
${validationMessage(validationError)}
Erwartetes JSON Schema:
${JSON.stringify(schema, null, 2)}
Fehlerhafte Ausgabe:
${invalidContent}
Korrigiere nur Syntax, Typen, fehlende Pflichtfelder und enum-Werte. Antworte ausschliesslich mit dem korrigierten JSON.`,
},
],
`${schemaName}_correction`,
schema,
sessionId,
trace ? { ...trace, label: `${trace.label} (JSON-Korrektur)` } : undefined,
);
return parseAndValidateLlmJson(completionContent(completion, correctionContext), correctionContext, validator);
}
function summarizeSegmentierungsQualitaet(fragen: FrageMitAntwort[] | undefined): SegmentierungsQualitaet | undefined {
const items = fragen ?? [];
if (!items.length) return undefined;
@@ -916,7 +1039,7 @@ function buildScoringFallbackBegruendung(
return `${farbe} (${meaning}). Staerkste Bereiche: ${strongestDimensions || "keine"}.${missing}`;
}
function fallbackScoringAssessment(reason: string): Partial<LlmScoringAssessment> & { begruendung?: string } {
function fallbackScoringAssessment(reason: string): z.infer<typeof scoringResponseSchema> {
return {
begruendung: `Die automatische Detailbewertung konnte nicht vollstaendig ausgewertet werden: ${reason}`,
ausschlussgruende: [],
@@ -989,11 +1112,9 @@ Bewerbungsdaten:
${JSON.stringify(data, null, 2)}`,
},
];
let parsed: Partial<LlmScoringAssessment> & { begruendung?: string };
let parsed: z.infer<typeof scoringResponseSchema>;
try {
parsed = await createParsedStructuredCompletion<Partial<LlmScoringAssessment> & {
begruendung?: string;
}>(
parsed = await createParsedStructuredCompletion<z.infer<typeof scoringResponseSchema>>(
client,
model,
messages,
@@ -1037,6 +1158,7 @@ ${JSON.stringify(data, null, 2)}`,
},
required: ["begruendung", "ausschlussgruende", "dimensionen"],
},
scoringResponseSchema,
"Could not parse traffic light assessment JSON",
sessionId,
{ operation: "scoring", label: "Ampelbewertung", onLlmCall },
@@ -1107,9 +1229,9 @@ Vorgaben:
- Lasse keine Unterfrage aus.`,
},
];
let parsed: { fragen?: Array<Partial<FrageMitAntwort>> };
let parsed: z.infer<typeof segmentierungResponseSchema>;
try {
parsed = await createParsedStructuredCompletion<{ fragen?: Array<Partial<FrageMitAntwort>> }>(
parsed = await createParsedStructuredCompletion<z.infer<typeof segmentierungResponseSchema>>(
client,
model,
messages,
@@ -1135,24 +1257,25 @@ Vorgaben:
},
required: ["fragen"],
},
segmentierungResponseSchema,
`Could not parse answer segmentation JSON for ${fallbackLabel}`,
sessionId,
{ operation: "segmentierung", label: fallbackLabel, onLlmCall },
);
} catch {
parsed = {
fragen: fragen.map((frage) => ({
id: frage.id,
fragen: fragen.map((frage, index) => ({
id: frage.id ?? `frage_${index + 1}`,
text: frage.text,
antwort: "",
confidence: 0,
})),
};
}
const byId = new Map((parsed.fragen ?? []).map((frage) => [String(frage.id ?? ""), frage]));
const byId = new Map(parsed.fragen.map((frage) => [frage.id, frage]));
return fragen.map((frage) => {
const mapped = byId.get(String(frage.id ?? ""));
return fragen.map((frage, index) => {
const mapped = byId.get(String(frage.id ?? `frage_${index + 1}`));
const rawConfidence = mapped?.confidence;
const confidence =
typeof rawConfidence === "number"
@@ -1204,9 +1327,9 @@ Bewerbungsdaten:
${JSON.stringify(data, null, 2)}`,
},
];
let parsed: Partial<SWOTAnalyse>;
let parsed: z.infer<typeof swotResponseSchema>;
try {
parsed = await createParsedStructuredCompletion<Partial<SWOTAnalyse>>(
parsed = await createParsedStructuredCompletion<z.infer<typeof swotResponseSchema>>(
client,
model,
messages,
@@ -1222,6 +1345,7 @@ ${JSON.stringify(data, null, 2)}`,
},
required: ["staerken", "schwaechen", "chancen", "risiken"],
},
swotResponseSchema,
"Could not parse SWOT analysis JSON",
sessionId,
{ operation: "swot", label: "SWOT-Analyse", onLlmCall },