Add summarizer observability and LLM retries
This commit is contained in:
syntaxbullet
@@ -1,4 +1,5 @@
|
||||
import { basename, join } from "node:path";
|
||||
import { mkdir } from "node:fs/promises";
|
||||
import { listCompanies, summarizeCompany, writeSummary, type LlmCallStatus, type LlmCallTrace } from "./summarizer";
|
||||
import { dataPath } from "../../deployPaths";
|
||||
import { allowedModelFromEnv, isSafeStem, jsonError, validateRequestedModel, withAuth } from "../../security";
|
||||
@@ -10,6 +11,9 @@ const PAGE_PATH = join(import.meta.dir, "index.html");
|
||||
const MAX_PARALLEL_SUMMARIES = 2;
|
||||
const MAX_ACTIVE_JOBS = Number(process.env.BMP_MAX_ACTIVE_SUMMARY_JOBS ?? 10);
|
||||
const EXPECTED_LLM_CALLS_PER_COMPANY = 13;
|
||||
const OBSERVABILITY_DIR =
|
||||
process.env.BMP_OBSERVABILITY_DIR ??
|
||||
dataPath(join(import.meta.dir, "outputs", "_observability"), "summarizer", "outputs", "_observability");
|
||||
|
||||
type JobStatus = "queued" | "running" | "done" | "error" | "cancelled";
|
||||
|
||||
@@ -25,6 +29,8 @@ interface SummaryJob {
|
||||
downloads: SummaryDownload[];
|
||||
errors: string[];
|
||||
llmCalls: SafeLlmCallTrace[];
|
||||
observability: ObservabilityLog;
|
||||
observabilityWrite: Promise<void>;
|
||||
usage: UsageSummary;
|
||||
statusMessage?: string;
|
||||
createdAt: string;
|
||||
@@ -35,9 +41,47 @@ type SafeLlmCallTrace = Pick<
|
||||
LlmCallTrace,
|
||||
"id" | "operation" | "label" | "model" | "status" | "startedAt" | "finishedAt" | "durationMs" | "error"
|
||||
> & {
|
||||
attempt: number;
|
||||
maxAttempts: number;
|
||||
usage?: unknown;
|
||||
};
|
||||
|
||||
interface ObservabilityEvent {
|
||||
at: string;
|
||||
type: string;
|
||||
message?: string;
|
||||
data?: unknown;
|
||||
}
|
||||
|
||||
interface ObservabilityLog {
|
||||
version: 1;
|
||||
jobId: string;
|
||||
stem: string;
|
||||
model: string;
|
||||
status: JobStatus;
|
||||
createdAt: string;
|
||||
finishedAt?: string;
|
||||
total: number;
|
||||
completed: number;
|
||||
current?: string;
|
||||
companies: string[];
|
||||
files: string[];
|
||||
downloads: SummaryDownload[];
|
||||
errors: SerializedError[];
|
||||
inputs: Record<string, unknown>;
|
||||
outputs: Record<string, unknown>;
|
||||
llmCalls: LlmCallTrace[];
|
||||
events: ObservabilityEvent[];
|
||||
logPath: string;
|
||||
}
|
||||
|
||||
interface SerializedError {
|
||||
name?: string;
|
||||
message: string;
|
||||
stack?: string;
|
||||
cause?: unknown;
|
||||
}
|
||||
|
||||
interface SummaryDownload {
|
||||
company: string;
|
||||
label: string;
|
||||
@@ -77,6 +121,8 @@ function sanitizeTrace(trace: LlmCallTrace): SafeLlmCallTrace {
|
||||
operation: trace.operation,
|
||||
label: trace.label,
|
||||
model: trace.model,
|
||||
attempt: trace.attempt,
|
||||
maxAttempts: trace.maxAttempts,
|
||||
status: trace.status,
|
||||
startedAt: trace.startedAt,
|
||||
finishedAt: trace.finishedAt,
|
||||
@@ -209,9 +255,83 @@ function isCancelled(job: SummaryJob): boolean {
|
||||
|
||||
const jobs = new Map<string, SummaryJob>();
|
||||
|
||||
function serializeError(error: unknown): SerializedError {
|
||||
if (error instanceof Error) {
|
||||
return {
|
||||
name: error.name,
|
||||
message: error.message,
|
||||
stack: error.stack,
|
||||
cause: error.cause,
|
||||
};
|
||||
}
|
||||
return { message: String(error) };
|
||||
}
|
||||
|
||||
function createObservabilityLog(jobId: string, stem: string, model: string, total: number, createdAt: string): ObservabilityLog {
|
||||
return {
|
||||
version: 1,
|
||||
jobId,
|
||||
stem,
|
||||
model,
|
||||
status: "queued",
|
||||
createdAt,
|
||||
total,
|
||||
completed: 0,
|
||||
companies: [],
|
||||
files: [],
|
||||
downloads: [],
|
||||
errors: [],
|
||||
inputs: {},
|
||||
outputs: {},
|
||||
llmCalls: [],
|
||||
events: [
|
||||
{
|
||||
at: createdAt,
|
||||
type: "job.created",
|
||||
message: "Summary job was queued.",
|
||||
},
|
||||
],
|
||||
logPath: join(OBSERVABILITY_DIR, `${createdAt.replaceAll(":", "-")}-${jobId}.json`),
|
||||
};
|
||||
}
|
||||
|
||||
function recordEvent(job: SummaryJob, type: string, message?: string, data?: unknown): void {
|
||||
job.observability.events.push({
|
||||
at: new Date().toISOString(),
|
||||
type,
|
||||
message,
|
||||
data,
|
||||
});
|
||||
}
|
||||
|
||||
function syncObservabilityState(job: SummaryJob): void {
|
||||
job.observability.status = job.status;
|
||||
job.observability.finishedAt = job.finishedAt;
|
||||
job.observability.completed = job.completed;
|
||||
job.observability.current = job.current;
|
||||
job.observability.files = [...job.files];
|
||||
job.observability.downloads = [...job.downloads];
|
||||
}
|
||||
|
||||
function persistObservability(job: SummaryJob): Promise<void> {
|
||||
syncObservabilityState(job);
|
||||
job.observabilityWrite = job.observabilityWrite
|
||||
.catch(() => undefined)
|
||||
.then(async () => {
|
||||
await mkdir(OBSERVABILITY_DIR, { recursive: true });
|
||||
await Bun.write(job.observability.logPath, JSON.stringify(job.observability, null, 2));
|
||||
})
|
||||
.catch((error) => {
|
||||
console.error("Failed to write summarizer observability log:", error);
|
||||
});
|
||||
return job.observabilityWrite;
|
||||
}
|
||||
|
||||
function createJob(stem: string, model: string, total: number): SummaryJob {
|
||||
const id = crypto.randomUUID();
|
||||
const createdAt = new Date().toISOString();
|
||||
const job: SummaryJob = {
|
||||
id: crypto.randomUUID(),
|
||||
id,
|
||||
status: "queued",
|
||||
stem,
|
||||
model,
|
||||
@@ -221,10 +341,13 @@ function createJob(stem: string, model: string, total: number): SummaryJob {
|
||||
downloads: [],
|
||||
errors: [],
|
||||
llmCalls: [],
|
||||
observability: createObservabilityLog(id, stem, model, total, createdAt),
|
||||
observabilityWrite: Promise.resolve(),
|
||||
usage: emptyUsageSummary(),
|
||||
createdAt: new Date().toISOString(),
|
||||
createdAt,
|
||||
};
|
||||
jobs.set(job.id, job);
|
||||
void persistObservability(job);
|
||||
return job;
|
||||
}
|
||||
|
||||
@@ -269,14 +392,31 @@ async function executeJob(job: SummaryJob): Promise<void> {
|
||||
if (isCancelled(job)) return;
|
||||
job.status = "running";
|
||||
job.statusMessage = "Auswertung wird vorbereitet.";
|
||||
recordEvent(job, "job.started", job.statusMessage);
|
||||
await persistObservability(job);
|
||||
const companies = job.stem === "__all__" ? await listCompanies() : [job.stem];
|
||||
job.observability.companies = [...companies];
|
||||
recordEvent(job, "companies.selected", undefined, { companies });
|
||||
await persistObservability(job);
|
||||
|
||||
try {
|
||||
await runWithConcurrency(companies, MAX_PARALLEL_SUMMARIES, async (company) => {
|
||||
if (isCancelled(job)) return;
|
||||
job.current = company;
|
||||
job.statusMessage = `LLM-Auswertung für ${company} läuft.`;
|
||||
recordEvent(job, "company.started", job.statusMessage, { company });
|
||||
await persistObservability(job);
|
||||
const summary = await summarizeCompany(company, job.model, {
|
||||
onCompanyInput: (companyStem, input) => {
|
||||
job.observability.inputs[companyStem] = input;
|
||||
recordEvent(job, "company.input.loaded", undefined, { company: companyStem });
|
||||
void persistObservability(job);
|
||||
},
|
||||
onCompanyOutput: (companyStem, output) => {
|
||||
job.observability.outputs[companyStem] = output;
|
||||
recordEvent(job, "company.output.created", undefined, { company: companyStem });
|
||||
void persistObservability(job);
|
||||
},
|
||||
onLlmCall: (trace) => {
|
||||
const existingIndex = job.llmCalls.findIndex((call) => call.id === trace.id);
|
||||
const safeTrace = sanitizeTrace(trace);
|
||||
@@ -285,24 +425,48 @@ async function executeJob(job: SummaryJob): Promise<void> {
|
||||
} else {
|
||||
job.llmCalls.push(safeTrace);
|
||||
}
|
||||
const fullTraceIndex = job.observability.llmCalls.findIndex((call) => call.id === trace.id);
|
||||
if (fullTraceIndex >= 0) {
|
||||
job.observability.llmCalls[fullTraceIndex] = trace;
|
||||
} else {
|
||||
job.observability.llmCalls.push(trace);
|
||||
}
|
||||
job.usage = summarizeUsage(job.llmCalls);
|
||||
job.statusMessage = trace.status === "running"
|
||||
? `${trace.label} wird verarbeitet.`
|
||||
: `${trace.label} abgeschlossen.`;
|
||||
? `${trace.label} wird verarbeitet (Versuch ${trace.attempt}/${trace.maxAttempts}).`
|
||||
: trace.status === "error"
|
||||
? `${trace.label} fehlgeschlagen (Versuch ${trace.attempt}/${trace.maxAttempts}).`
|
||||
: `${trace.label} abgeschlossen.`;
|
||||
recordEvent(job, `llm.${trace.status}`, job.statusMessage, {
|
||||
id: trace.id,
|
||||
operation: trace.operation,
|
||||
label: trace.label,
|
||||
attempt: trace.attempt,
|
||||
maxAttempts: trace.maxAttempts,
|
||||
});
|
||||
void persistObservability(job);
|
||||
},
|
||||
});
|
||||
job.statusMessage = `Downloads für ${company} werden vorbereitet.`;
|
||||
recordEvent(job, "company.summary.created", job.statusMessage, { company });
|
||||
if (isCancelled(job)) return;
|
||||
const outputPaths = await writeSummary(company, summary, OUTPUTS_DIR);
|
||||
job.files.push(...outputPaths.map((path) => basename(path)));
|
||||
job.downloads.push(...outputPaths.map((path) => buildDownload(company, path)));
|
||||
job.completed += 1;
|
||||
recordEvent(job, "company.finished", `Auswertung für ${company} fertig.`, {
|
||||
company,
|
||||
outputFiles: outputPaths.map((path) => basename(path)),
|
||||
});
|
||||
await persistObservability(job);
|
||||
});
|
||||
|
||||
if (isCancelled(job)) {
|
||||
job.current = undefined;
|
||||
job.statusMessage = "Auswertung abgebrochen.";
|
||||
job.finishedAt = new Date().toISOString();
|
||||
recordEvent(job, "job.cancelled", job.statusMessage);
|
||||
await persistObservability(job);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -310,17 +474,25 @@ async function executeJob(job: SummaryJob): Promise<void> {
|
||||
job.current = undefined;
|
||||
job.statusMessage = "Auswertung fertig.";
|
||||
job.finishedAt = new Date().toISOString();
|
||||
recordEvent(job, "job.done", job.statusMessage);
|
||||
await persistObservability(job);
|
||||
} catch (error) {
|
||||
if (isCancelled(job)) {
|
||||
job.current = undefined;
|
||||
job.statusMessage = "Auswertung abgebrochen.";
|
||||
job.finishedAt = new Date().toISOString();
|
||||
recordEvent(job, "job.cancelled", job.statusMessage);
|
||||
await persistObservability(job);
|
||||
return;
|
||||
}
|
||||
const serializedError = serializeError(error);
|
||||
job.status = "error";
|
||||
job.errors.push(String(error));
|
||||
job.statusMessage = "Auswertung fehlgeschlagen.";
|
||||
job.errors.push(serializedError.message);
|
||||
job.observability.errors.push(serializedError);
|
||||
job.statusMessage = `Auswertung fehlgeschlagen: ${serializedError.message}`;
|
||||
job.finishedAt = new Date().toISOString();
|
||||
recordEvent(job, "job.error", job.statusMessage, serializedError);
|
||||
await persistObservability(job);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -432,7 +604,10 @@ export const routes = {
|
||||
return Response.json(publicJob(job));
|
||||
}),
|
||||
DELETE: withAuth(async (req: Request) => {
|
||||
const id = (req as Request & { params: Record<string, string> }).params.id;
|
||||
const id = (req as Request & { params: Record<string, string | undefined> }).params.id;
|
||||
if (!id) {
|
||||
return Response.json({ error: "Job not found" }, { status: 404 });
|
||||
}
|
||||
const job = jobs.get(id);
|
||||
if (!job) {
|
||||
return Response.json({ error: "Job not found" }, { status: 404 });
|
||||
|
||||
Reference in New Issue
Block a user