Add result and job management APIs
This commit is contained in:
@@ -873,6 +873,7 @@
|
|||||||
renderLlmProgress(job);
|
renderLlmProgress(job);
|
||||||
statusEl.textContent = job.statusMessage || "Die Einschätzung wird erstellt. Das kann einen Moment dauern.";
|
statusEl.textContent = job.statusMessage || "Die Einschätzung wird erstellt. Das kann einen Moment dauern.";
|
||||||
if (job.status === "done") return job;
|
if (job.status === "done") return job;
|
||||||
|
if (job.status === "cancelled") throw new Error("Job wurde abgebrochen");
|
||||||
if (job.status === "error") throw new Error(job.errors?.join("\\n") || "Die Auswertung ist fehlgeschlagen.");
|
if (job.status === "error") throw new Error(job.errors?.join("\\n") || "Die Auswertung ist fehlgeschlagen.");
|
||||||
await new Promise((resolve) => setTimeout(resolve, POLL_DELAY_MS));
|
await new Promise((resolve) => setTimeout(resolve, POLL_DELAY_MS));
|
||||||
}
|
}
|
||||||
|
|||||||
30
index.ts
30
index.ts
@@ -2,9 +2,12 @@ import { routes as templatebuilder, workerPath } from "./packages/templatebuilde
|
|||||||
import { routes as extractor } from "./packages/extractor/index";
|
import { routes as extractor } from "./packages/extractor/index";
|
||||||
import { routes as summarizer } from "./packages/summarizer/index";
|
import { routes as summarizer } from "./packages/summarizer/index";
|
||||||
import { join } from "node:path";
|
import { join } from "node:path";
|
||||||
|
import { rm } from "node:fs/promises";
|
||||||
import { dataPath } from "./deployPaths";
|
import { dataPath } from "./deployPaths";
|
||||||
import {
|
import {
|
||||||
addSecurityHeaders,
|
addSecurityHeaders,
|
||||||
|
getCsrfCookieName,
|
||||||
|
isAuthenticated,
|
||||||
isSafeStem,
|
isSafeStem,
|
||||||
login,
|
login,
|
||||||
loginPage,
|
loginPage,
|
||||||
@@ -17,6 +20,9 @@ import {
|
|||||||
const SUMMARIZER_OUTPUTS_DIR =
|
const SUMMARIZER_OUTPUTS_DIR =
|
||||||
process.env.BMP_SUMMARIZER_OUTPUTS_DIR ??
|
process.env.BMP_SUMMARIZER_OUTPUTS_DIR ??
|
||||||
dataPath(join(import.meta.dir, "packages/summarizer/outputs"), "summarizer", "outputs");
|
dataPath(join(import.meta.dir, "packages/summarizer/outputs"), "summarizer", "outputs");
|
||||||
|
const EXTRACTOR_OUTPUTS_DIR =
|
||||||
|
process.env.BMP_EXTRACTOR_OUTPUTS_DIR ??
|
||||||
|
dataPath(join(import.meta.dir, "packages/extractor/outputs"), "extractor", "outputs");
|
||||||
const ROOT_PAGE_PATH = join(import.meta.dir, "index.html");
|
const ROOT_PAGE_PATH = join(import.meta.dir, "index.html");
|
||||||
const BMP_LOGO_PATH = join(import.meta.dir, "assets/bmp-logo.png");
|
const BMP_LOGO_PATH = join(import.meta.dir, "assets/bmp-logo.png");
|
||||||
const PORT = Number(process.env.PORT ?? 3000);
|
const PORT = Number(process.env.PORT ?? 3000);
|
||||||
@@ -74,6 +80,12 @@ Bun.serve({
|
|||||||
"/logout": {
|
"/logout": {
|
||||||
POST: withAuth(async () => logout(), { csrf: true }),
|
POST: withAuth(async () => logout(), { csrf: true }),
|
||||||
},
|
},
|
||||||
|
"/api/auth/session": {
|
||||||
|
GET: async (req: Request) => addSecurityHeaders(Response.json({
|
||||||
|
authenticated: await isAuthenticated(req),
|
||||||
|
csrfCookieName: getCsrfCookieName(),
|
||||||
|
})),
|
||||||
|
},
|
||||||
"/": {
|
"/": {
|
||||||
GET: withAuth(async () => new Response(Bun.file(ROOT_PAGE_PATH), {
|
GET: withAuth(async () => new Response(Bun.file(ROOT_PAGE_PATH), {
|
||||||
headers: { "Content-Type": "text/html; charset=utf-8" },
|
headers: { "Content-Type": "text/html; charset=utf-8" },
|
||||||
@@ -84,6 +96,24 @@ Bun.serve({
|
|||||||
headers: { "Content-Type": "image/png", "Cache-Control": "private, max-age=86400" },
|
headers: { "Content-Type": "image/png", "Cache-Control": "private, max-age=86400" },
|
||||||
})),
|
})),
|
||||||
},
|
},
|
||||||
|
"/api/results/:stem": {
|
||||||
|
DELETE: withAuth(async (req: Request) => {
|
||||||
|
const stem = (req as Request & { params: Record<string, string | undefined> }).params.stem ?? "";
|
||||||
|
if (!isSafeStem(stem)) {
|
||||||
|
return Response.json({ error: "Invalid result id" }, { status: 400 });
|
||||||
|
}
|
||||||
|
|
||||||
|
const targets = [
|
||||||
|
join(EXTRACTOR_OUTPUTS_DIR, stem),
|
||||||
|
join(SUMMARIZER_OUTPUTS_DIR, stem),
|
||||||
|
];
|
||||||
|
await Promise.all(targets.map((target) => rm(target, { recursive: true, force: true })));
|
||||||
|
return Response.json({ ok: true, stem, deleted: targets.length });
|
||||||
|
}, {
|
||||||
|
csrf: true,
|
||||||
|
limit: { key: "result-delete", max: 20, windowMs: 60_000 },
|
||||||
|
}),
|
||||||
|
},
|
||||||
"/api/pipeline/result/:stem": {
|
"/api/pipeline/result/:stem": {
|
||||||
GET: withAuth(async (req: Request) => {
|
GET: withAuth(async (req: Request) => {
|
||||||
const stem = (req as Request & { params: Record<string, string | undefined> }).params.stem ?? "";
|
const stem = (req as Request & { params: Record<string, string | undefined> }).params.stem ?? "";
|
||||||
|
|||||||
@@ -1,9 +1,9 @@
|
|||||||
import { join } from "node:path";
|
import { join } from "node:path";
|
||||||
import { dataPath } from "../../deployPaths";
|
import { dataPath } from "../../deployPaths";
|
||||||
import { jsonError, normalizeStem, securityHeaders, withAuth } from "../../security";
|
import { isSafeStem, jsonError, normalizeStem, securityHeaders, withAuth } from "../../security";
|
||||||
|
|
||||||
const UPLOAD_PAGE_PATH = join(import.meta.dir, "upload.html");
|
const UPLOAD_PAGE_PATH = join(import.meta.dir, "upload.html");
|
||||||
import { mkdir } from "node:fs/promises";
|
import { mkdir, readdir } from "node:fs/promises";
|
||||||
import { flattenTemplate, extractFields, buildOutput, getNested } from "./extractor";
|
import { flattenTemplate, extractFields, buildOutput, getNested } from "./extractor";
|
||||||
|
|
||||||
const TEMPLATE_PATH =
|
const TEMPLATE_PATH =
|
||||||
@@ -59,6 +59,60 @@ function extractScalarText(value: unknown): string | undefined {
|
|||||||
return undefined;
|
return undefined;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function extractorDownloadUrl(stem: string, file: string): string {
|
||||||
|
return `/downloads/extractor/${encodeURIComponent(stem)}/${encodeURIComponent(file)}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function listExtractorResults() {
|
||||||
|
let entries: string[];
|
||||||
|
try {
|
||||||
|
entries = await readdir(OUTPUTS_DIR);
|
||||||
|
} catch {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
const results = [];
|
||||||
|
for (const stem of entries.toSorted()) {
|
||||||
|
if (!isSafeStem(stem)) continue;
|
||||||
|
const jsonFile = `${stem}.json`;
|
||||||
|
const pdfFile = `${stem}.pdf`;
|
||||||
|
const jsonPath = join(OUTPUTS_DIR, stem, jsonFile);
|
||||||
|
const pdfPath = join(OUTPUTS_DIR, stem, pdfFile);
|
||||||
|
if (!(await Bun.file(jsonPath).exists())) continue;
|
||||||
|
results.push({
|
||||||
|
stem,
|
||||||
|
files: {
|
||||||
|
json: (await Bun.file(jsonPath).exists()) ? jsonFile : undefined,
|
||||||
|
pdf: (await Bun.file(pdfPath).exists()) ? pdfFile : undefined,
|
||||||
|
},
|
||||||
|
downloads: [
|
||||||
|
{ label: "JSON herunterladen", url: extractorDownloadUrl(stem, jsonFile) },
|
||||||
|
...((await Bun.file(pdfPath).exists()) ? [{ label: "PDF herunterladen", url: extractorDownloadUrl(stem, pdfFile) }] : []),
|
||||||
|
],
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return results;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function serveExtractorDownload(stem: string, file: string): Promise<Response> {
|
||||||
|
if (!isSafeStem(stem) || (file !== `${stem}.json` && file !== `${stem}.pdf`)) {
|
||||||
|
return new Response("Not found", { status: 404, headers: securityHeaders });
|
||||||
|
}
|
||||||
|
const path = join(OUTPUTS_DIR, stem, file);
|
||||||
|
const output = Bun.file(path);
|
||||||
|
if (!(await output.exists())) {
|
||||||
|
return new Response("Not found", { status: 404, headers: securityHeaders });
|
||||||
|
}
|
||||||
|
return new Response(output, {
|
||||||
|
headers: {
|
||||||
|
...securityHeaders,
|
||||||
|
"Content-Type": file.endsWith(".json") ? "application/json; charset=utf-8" : "application/pdf",
|
||||||
|
"Cache-Control": "private, no-store",
|
||||||
|
"Content-Disposition": `attachment; filename="${file.replaceAll('"', "")}"`,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
export const routes = {
|
export const routes = {
|
||||||
"/extractor": {
|
"/extractor": {
|
||||||
GET: withAuth(async () => new Response(Bun.file(UPLOAD_PAGE_PATH), {
|
GET: withAuth(async () => new Response(Bun.file(UPLOAD_PAGE_PATH), {
|
||||||
@@ -67,6 +121,41 @@ export const routes = {
|
|||||||
),
|
),
|
||||||
},
|
},
|
||||||
|
|
||||||
|
"/api/extractor/results": {
|
||||||
|
GET: withAuth(async () => Response.json({ results: await listExtractorResults() })),
|
||||||
|
},
|
||||||
|
|
||||||
|
"/api/extractor/results/:stem": {
|
||||||
|
GET: withAuth(async (req: Request) => {
|
||||||
|
const stem = (req as Request & { params: Record<string, string | undefined> }).params.stem ?? "";
|
||||||
|
if (!isSafeStem(stem)) {
|
||||||
|
return jsonError("Invalid result id", 400);
|
||||||
|
}
|
||||||
|
const jsonPath = join(OUTPUTS_DIR, stem, `${stem}.json`);
|
||||||
|
const pdfPath = join(OUTPUTS_DIR, stem, `${stem}.pdf`);
|
||||||
|
const jsonFile = Bun.file(jsonPath);
|
||||||
|
if (!(await jsonFile.exists())) {
|
||||||
|
return Response.json({ error: "Extractor result not found" }, { status: 404 });
|
||||||
|
}
|
||||||
|
return Response.json({
|
||||||
|
ok: true,
|
||||||
|
stem,
|
||||||
|
result: await jsonFile.json(),
|
||||||
|
downloads: [
|
||||||
|
{ label: "JSON herunterladen", url: extractorDownloadUrl(stem, `${stem}.json`) },
|
||||||
|
...((await Bun.file(pdfPath).exists()) ? [{ label: "PDF herunterladen", url: extractorDownloadUrl(stem, `${stem}.pdf`) }] : []),
|
||||||
|
],
|
||||||
|
});
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
|
||||||
|
"/downloads/extractor/:stem/:file": {
|
||||||
|
GET: withAuth(async (req: Request) => {
|
||||||
|
const params = (req as Request & { params: Record<string, string | undefined> }).params;
|
||||||
|
return serveExtractorDownload(params.stem ?? "", params.file ?? "");
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
|
||||||
"/extract": {
|
"/extract": {
|
||||||
POST: withAuth(async (req: Request) => {
|
POST: withAuth(async (req: Request) => {
|
||||||
const contentLength = Number(req.headers.get("content-length") ?? 0);
|
const contentLength = Number(req.headers.get("content-length") ?? 0);
|
||||||
|
|||||||
@@ -185,6 +185,7 @@
|
|||||||
renderProgress(job);
|
renderProgress(job);
|
||||||
|
|
||||||
if (job.status === "done") return job;
|
if (job.status === "done") return job;
|
||||||
|
if (job.status === "cancelled") throw new Error("Job wurde abgebrochen");
|
||||||
if (job.status === "error") throw new Error(job.errors?.join("\n") || "Job fehlgeschlagen");
|
if (job.status === "error") throw new Error(job.errors?.join("\n") || "Job fehlgeschlagen");
|
||||||
|
|
||||||
await new Promise((resolve) => setTimeout(resolve, 700));
|
await new Promise((resolve) => setTimeout(resolve, 700));
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ const MAX_PARALLEL_SUMMARIES = 2;
|
|||||||
const MAX_ACTIVE_JOBS = Number(process.env.BMP_MAX_ACTIVE_SUMMARY_JOBS ?? 10);
|
const MAX_ACTIVE_JOBS = Number(process.env.BMP_MAX_ACTIVE_SUMMARY_JOBS ?? 10);
|
||||||
const EXPECTED_LLM_CALLS_PER_COMPANY = 13;
|
const EXPECTED_LLM_CALLS_PER_COMPANY = 13;
|
||||||
|
|
||||||
type JobStatus = "queued" | "running" | "done" | "error";
|
type JobStatus = "queued" | "running" | "done" | "error" | "cancelled";
|
||||||
|
|
||||||
interface SummaryJob {
|
interface SummaryJob {
|
||||||
id: string;
|
id: string;
|
||||||
@@ -203,6 +203,10 @@ function activeJobCount(): number {
|
|||||||
return [...jobs.values()].filter((job) => job.status === "queued" || job.status === "running").length;
|
return [...jobs.values()].filter((job) => job.status === "queued" || job.status === "running").length;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function isCancelled(job: SummaryJob): boolean {
|
||||||
|
return job.status === "cancelled";
|
||||||
|
}
|
||||||
|
|
||||||
const jobs = new Map<string, SummaryJob>();
|
const jobs = new Map<string, SummaryJob>();
|
||||||
|
|
||||||
function createJob(stem: string, model: string, total: number): SummaryJob {
|
function createJob(stem: string, model: string, total: number): SummaryJob {
|
||||||
@@ -262,12 +266,14 @@ async function runWithConcurrency<T>(
|
|||||||
}
|
}
|
||||||
|
|
||||||
async function executeJob(job: SummaryJob): Promise<void> {
|
async function executeJob(job: SummaryJob): Promise<void> {
|
||||||
|
if (isCancelled(job)) return;
|
||||||
job.status = "running";
|
job.status = "running";
|
||||||
job.statusMessage = "Auswertung wird vorbereitet.";
|
job.statusMessage = "Auswertung wird vorbereitet.";
|
||||||
const companies = job.stem === "__all__" ? await listCompanies() : [job.stem];
|
const companies = job.stem === "__all__" ? await listCompanies() : [job.stem];
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await runWithConcurrency(companies, MAX_PARALLEL_SUMMARIES, async (company) => {
|
await runWithConcurrency(companies, MAX_PARALLEL_SUMMARIES, async (company) => {
|
||||||
|
if (isCancelled(job)) return;
|
||||||
job.current = company;
|
job.current = company;
|
||||||
job.statusMessage = `LLM-Auswertung für ${company} läuft.`;
|
job.statusMessage = `LLM-Auswertung für ${company} läuft.`;
|
||||||
const summary = await summarizeCompany(company, job.model, {
|
const summary = await summarizeCompany(company, job.model, {
|
||||||
@@ -286,17 +292,31 @@ async function executeJob(job: SummaryJob): Promise<void> {
|
|||||||
},
|
},
|
||||||
});
|
});
|
||||||
job.statusMessage = `Downloads für ${company} werden vorbereitet.`;
|
job.statusMessage = `Downloads für ${company} werden vorbereitet.`;
|
||||||
|
if (isCancelled(job)) return;
|
||||||
const outputPaths = await writeSummary(company, summary, OUTPUTS_DIR);
|
const outputPaths = await writeSummary(company, summary, OUTPUTS_DIR);
|
||||||
job.files.push(...outputPaths.map((path) => basename(path)));
|
job.files.push(...outputPaths.map((path) => basename(path)));
|
||||||
job.downloads.push(...outputPaths.map((path) => buildDownload(company, path)));
|
job.downloads.push(...outputPaths.map((path) => buildDownload(company, path)));
|
||||||
job.completed += 1;
|
job.completed += 1;
|
||||||
});
|
});
|
||||||
|
|
||||||
|
if (isCancelled(job)) {
|
||||||
|
job.current = undefined;
|
||||||
|
job.statusMessage = "Auswertung abgebrochen.";
|
||||||
|
job.finishedAt = new Date().toISOString();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
job.status = "done";
|
job.status = "done";
|
||||||
job.current = undefined;
|
job.current = undefined;
|
||||||
job.statusMessage = "Auswertung fertig.";
|
job.statusMessage = "Auswertung fertig.";
|
||||||
job.finishedAt = new Date().toISOString();
|
job.finishedAt = new Date().toISOString();
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
if (isCancelled(job)) {
|
||||||
|
job.current = undefined;
|
||||||
|
job.statusMessage = "Auswertung abgebrochen.";
|
||||||
|
job.finishedAt = new Date().toISOString();
|
||||||
|
return;
|
||||||
|
}
|
||||||
job.status = "error";
|
job.status = "error";
|
||||||
job.errors.push(String(error));
|
job.errors.push(String(error));
|
||||||
job.statusMessage = "Auswertung fehlgeschlagen.";
|
job.statusMessage = "Auswertung fehlgeschlagen.";
|
||||||
@@ -368,6 +388,15 @@ export const routes = {
|
|||||||
}),
|
}),
|
||||||
},
|
},
|
||||||
|
|
||||||
|
"/api/summarizer/jobs": {
|
||||||
|
GET: withAuth(async () => {
|
||||||
|
const summaries = [...jobs.values()]
|
||||||
|
.toSorted((a, b) => Date.parse(b.createdAt) - Date.parse(a.createdAt))
|
||||||
|
.map(publicJob);
|
||||||
|
return Response.json({ jobs: summaries });
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
|
||||||
"/api/summarizer/jobs/:id/llm-calls": {
|
"/api/summarizer/jobs/:id/llm-calls": {
|
||||||
GET: withAuth(async (req: Request) => {
|
GET: withAuth(async (req: Request) => {
|
||||||
const id = (req as Request & { params: Record<string, string> }).params.id;
|
const id = (req as Request & { params: Record<string, string> }).params.id;
|
||||||
@@ -402,5 +431,24 @@ export const routes = {
|
|||||||
}
|
}
|
||||||
return Response.json(publicJob(job));
|
return Response.json(publicJob(job));
|
||||||
}),
|
}),
|
||||||
|
DELETE: withAuth(async (req: Request) => {
|
||||||
|
const id = (req as Request & { params: Record<string, string> }).params.id;
|
||||||
|
const job = jobs.get(id);
|
||||||
|
if (!job) {
|
||||||
|
return Response.json({ error: "Job not found" }, { status: 404 });
|
||||||
|
}
|
||||||
|
if (job.status === "queued" || job.status === "running") {
|
||||||
|
job.status = "cancelled";
|
||||||
|
job.current = undefined;
|
||||||
|
job.statusMessage = "Auswertung abgebrochen.";
|
||||||
|
job.finishedAt = new Date().toISOString();
|
||||||
|
return Response.json({ ok: true, job: publicJob(job) });
|
||||||
|
}
|
||||||
|
jobs.delete(id);
|
||||||
|
return Response.json({ ok: true, deleted: true });
|
||||||
|
}, {
|
||||||
|
csrf: true,
|
||||||
|
limit: { key: "summarizer-job-delete", max: 30, windowMs: 60_000 },
|
||||||
|
}),
|
||||||
},
|
},
|
||||||
} as const;
|
} as const;
|
||||||
|
|||||||
@@ -21,7 +21,9 @@ async function loadTemplate(): Promise<Record<string, unknown>> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export const routes = {
|
export const routes = {
|
||||||
"/builder": index,
|
"/builder": {
|
||||||
|
GET: withAuth(async () => index),
|
||||||
|
},
|
||||||
|
|
||||||
"/api/save": {
|
"/api/save": {
|
||||||
POST: withAuth(async (req: Request) => {
|
POST: withAuth(async (req: Request) => {
|
||||||
|
|||||||
@@ -245,7 +245,7 @@ function hasValidCsrf(req: Request): boolean {
|
|||||||
return Boolean(header && cookie && timingSafeEqual(header, cookie));
|
return Boolean(header && cookie && timingSafeEqual(header, cookie));
|
||||||
}
|
}
|
||||||
|
|
||||||
async function isAuthenticated(req: Request): Promise<boolean> {
|
export async function isAuthenticated(req: Request): Promise<boolean> {
|
||||||
const auth = parseCookies(req.headers.get("cookie")).get(AUTH_COOKIE);
|
const auth = parseCookies(req.headers.get("cookie")).get(AUTH_COOKIE);
|
||||||
if (auth && await verifyAuthCookie(auth)) return true;
|
if (auth && await verifyAuthCookie(auth)) return true;
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user