initial commit

2026-05-13 17:26:13 +02:00
commit 99569d2cf3
43 changed files with 8725 additions and 0 deletions
--- a/packages/extractor/extractor.ts
+++ b/packages/extractor/extractor.ts
@@ -0,0 +1,176 @@
+// Use the legacy build — the main build relies on browser-only APIs (DOMMatrix
+// etc.) that aren't available in a Bun/Node server context.
+// @ts-ignore — legacy path has no separate .d.ts; types are compatible
+import * as pdfjsLib from "pdfjs-dist/legacy/build/pdf.mjs";
+
+// Point at the legacy worker. import.meta.resolve returns a file:// URL that
+// Bun can load as a Worker, which pdfjs requires even in server contexts.
+pdfjsLib.GlobalWorkerOptions.workerSrc = import.meta.resolve(
+  "pdfjs-dist/legacy/build/pdf.worker.mjs",
+);
+
+// ─── Types ────────────────────────────────────────────────────────────────────
+
+/** Raw extracted field values keyed by PDF field name. */
+export type FieldValues = Record<string, string>;
+
+export interface FrageVorlage {
+  id?: string;
+  text: string;
+}
+
+export interface TemplateLeafObject {
+  fieldName: string;
+  label?: string;
+  antwortFormat?: "einzelfrage" | "mehrfachfrage_ein_antwortfeld";
+  fragen?: FrageVorlage[];
+}
+
+export type TemplateLeaf = string | TemplateLeafObject;
+
+export interface FieldIndexEntry {
+  path: string;
+  fieldName: string;
+  label?: string;
+  antwortFormat?: "einzelfrage" | "mehrfachfrage_ein_antwortfeld";
+  fragen?: FrageVorlage[];
+  isRich?: boolean;
+}
+
+/**
+ * Flattened view of a template: PDF field name → metadata about the target path.
+ */
+export type FieldIndex = Record<string, FieldIndexEntry>;
+
+// ─── Template helpers ─────────────────────────────────────────────────────────
+
+function isTemplateLeafObject(value: unknown): value is TemplateLeafObject {
+  return typeof value === "object" && value !== null && typeof (value as Record<string, unknown>).fieldName === "string";
+}
+
+/**
+ * Recursively flattens the nested template produced by templatebuilder.
+ */
+export function flattenTemplate(
+  node: Record<string, unknown>,
+  prefix = "",
+): FieldIndex {
+  const index: FieldIndex = {};
+
+  for (const [key, value] of Object.entries(node)) {
+    const path = prefix ? `${prefix}.${key}` : key;
+
+    if (typeof value === "string") {
+      index[value] = {
+        path,
+        fieldName: value,
+        antwortFormat: "einzelfrage",
+        isRich: false,
+      };
+    } else if (isTemplateLeafObject(value)) {
+      index[value.fieldName] = {
+        path,
+        fieldName: value.fieldName,
+        label: value.label,
+        antwortFormat: value.antwortFormat ?? "einzelfrage",
+        fragen: value.fragen,
+        isRich: true,
+      };
+    } else if (typeof value === "object" && value !== null) {
+      Object.assign(index, flattenTemplate(value as Record<string, unknown>, path));
+    }
+  }
+
+  return index;
+}
+
+// ─── PDF extraction ───────────────────────────────────────────────────────────
+
+export async function extractFields(
+  data: ArrayBuffer,
+  options: { maxPages?: number } = {},
+): Promise<FieldValues> {
+  const pdf = await pdfjsLib.getDocument({ data }).promise;
+  if (options.maxPages && pdf.numPages > options.maxPages) {
+    throw new Error(`PDF has ${pdf.numPages} pages; the configured limit is ${options.maxPages}.`);
+  }
+  const values: FieldValues = {};
+
+  for (let p = 1; p <= pdf.numPages; p++) {
+    const page = await pdf.getPage(p);
+    const annotations = await page.getAnnotations();
+
+    for (const ann of annotations) {
+      if (ann.subtype !== "Widget" || !ann.fieldName) continue;
+
+      const raw = ann.fieldValue;
+      if (raw === undefined || raw === null) continue;
+
+      values[ann.fieldName] = Array.isArray(raw) ? raw.join(", ") : String(raw);
+    }
+  }
+
+  return values;
+}
+
+// ─── Output builder ───────────────────────────────────────────────────────────
+
+export function buildOutput(
+  values: FieldValues,
+  index: FieldIndex,
+): Record<string, unknown> {
+  const output: Record<string, unknown> = {};
+
+  for (const entry of Object.values(index)) {
+    const value = values[entry.fieldName];
+    if (value === undefined) continue;
+
+    if (!entry.isRich) {
+      setNested(output, entry.path, value);
+      continue;
+    }
+
+    const antwortFormat = entry.antwortFormat ?? "einzelfrage";
+    const fragen = (entry.fragen ?? []).map((frage, i) => ({
+      id: frage.id ?? `${entry.path.split(".").at(-1) ?? "frage"}_${i + 1}`,
+      text: frage.text,
+      ...(antwortFormat === "einzelfrage" ? { antwort: value } : {}),
+    }));
+
+    setNested(output, entry.path, {
+      fieldName: entry.fieldName,
+      label: entry.label,
+      antwortFormat,
+      zuordnungsmodus:
+        antwortFormat === "mehrfachfrage_ein_antwortfeld"
+          ? "gemeinsame_antwort"
+          : "einzelfrage",
+      fragen,
+      antwort: value,
+    });
+  }
+
+  return output;
+}
+
+export function getNested(obj: Record<string, unknown>, path: string): unknown {
+  let cur: unknown = obj;
+  for (const key of path.split(".")) {
+    if (typeof cur !== "object" || cur === null) return undefined;
+    cur = (cur as Record<string, unknown>)[key];
+  }
+  return cur;
+}
+
+// ─── Internal helpers ─────────────────────────────────────────────────────────
+
+function setNested(obj: Record<string, unknown>, path: string, value: unknown): void {
+  const parts = path.split(".");
+  let cur = obj;
+  for (let i = 0; i < parts.length - 1; i++) {
+    const key = parts[i]!;
+    if (typeof cur[key] !== "object" || cur[key] === null) cur[key] = {};
+    cur = cur[key] as Record<string, unknown>;
+  }
+  cur[parts[parts.length - 1]!] = value;
+}
--- a/packages/extractor/index.ts
+++ b/packages/extractor/index.ts
@@ -0,0 +1,154 @@
+import { join } from "node:path";
+import { dataPath } from "../../deployPaths";
+import { jsonError, normalizeStem, securityHeaders, withAuth } from "../../security";
+
+const UPLOAD_PAGE_PATH = join(import.meta.dir, "upload.html");
+import { mkdir } from "node:fs/promises";
+import { flattenTemplate, extractFields, buildOutput, getNested } from "./extractor";
+
+const TEMPLATE_PATH =
+  process.env.BMP_TEMPLATE_PATH ??
+  dataPath(join(import.meta.dir, "../templatebuilder/template.json"), "templatebuilder", "template.json");
+const OUTPUTS_DIR =
+  process.env.BMP_EXTRACTOR_OUTPUTS_DIR ??
+  dataPath(join(import.meta.dir, "outputs"), "extractor", "outputs");
+const MAX_UPLOAD_BYTES = Number(process.env.BMP_MAX_UPLOAD_BYTES ?? 15 * 1024 * 1024);
+const MAX_PDF_PAGES = Number(process.env.BMP_MAX_PDF_PAGES ?? 35);
+
+await mkdir(OUTPUTS_DIR, { recursive: true });
+
+async function loadTemplate(): Promise<Record<string, unknown>> {
+  const file = Bun.file(TEMPLATE_PATH);
+  if (!(await file.exists())) {
+    throw new Error(
+      `template.json not found at ${TEMPLATE_PATH}. Run the templatebuilder first.`,
+    );
+  }
+  return file.json();
+}
+
+function normalizeFilenamePart(value: string): string {
+  return normalizeStem(value);
+}
+
+function toSafeFilename(...parts: unknown[]): string {
+  const normalizedParts: string[] = [];
+
+  for (const part of parts) {
+    if (typeof part !== "string") continue;
+    const trimmed = part.trim();
+    if (!trimmed) continue;
+
+    const normalized = normalizeFilenamePart(trimmed);
+    const previous = normalizedParts[normalizedParts.length - 1];
+
+    if (normalized && normalized !== previous) {
+      normalizedParts.push(normalized);
+    }
+  }
+
+  return normalizedParts.join("_");
+}
+
+function extractScalarText(value: unknown): string | undefined {
+  if (typeof value === "string") return value.trim() || undefined;
+  if (typeof value === "object" && value !== null) {
+    const antwort = (value as Record<string, unknown>).antwort;
+    if (typeof antwort === "string") return antwort.trim() || undefined;
+  }
+  return undefined;
+}
+
+export const routes = {
+  "/extractor": {
+    GET: withAuth(async () => new Response(Bun.file(UPLOAD_PAGE_PATH), {
+      headers: { ...securityHeaders, "Content-Type": "text/html" },
+    }),
+    ),
+  },
+
+  "/extract": {
+    POST: withAuth(async (req: Request) => {
+      const contentLength = Number(req.headers.get("content-length") ?? 0);
+      if (contentLength > MAX_UPLOAD_BYTES + 4096) {
+        return jsonError(`PDF is too large. Maximum size is ${Math.floor(MAX_UPLOAD_BYTES / 1024 / 1024)} MB.`, 413);
+      }
+
+      let form;
+      try {
+        form = await req.formData();
+      } catch {
+        return Response.json({ error: "Expected multipart/form-data" }, { status: 400 });
+      }
+
+      const entry = form.get("pdf");
+      if (!(entry instanceof File)) {
+        return Response.json({ error: 'Missing "pdf" file field' }, { status: 400 });
+      }
+      if (!entry.name.toLowerCase().endsWith(".pdf") || entry.size > MAX_UPLOAD_BYTES) {
+        return jsonError(`Only PDF uploads up to ${Math.floor(MAX_UPLOAD_BYTES / 1024 / 1024)} MB are allowed.`, 400);
+      }
+
+      let template: Record<string, unknown>;
+      try {
+        template = await loadTemplate();
+      } catch (err) {
+        return Response.json({ error: String(err) }, { status: 500 });
+      }
+
+      const index  = flattenTemplate(template);
+      const data   = await entry.arrayBuffer();
+      if (!isPdfMagic(data)) {
+        return jsonError("Uploaded file is not a valid PDF.", 400);
+      }
+      // pdfjs transfers the ArrayBuffer to its Worker, detaching it (byteLength → 0).
+      // Slice a copy first so we still have the original bytes to write to disk.
+      const pdfBytes = data.slice(0);
+      let values;
+      try {
+        values = await extractFields(data, { maxPages: MAX_PDF_PAGES });
+      } catch (error) {
+        return jsonError(`PDF could not be processed: ${String(error).replace(/^Error:\s*/, "")}`, 400);
+      }
+      const output = buildOutput(values, index);
+
+      const name = extractScalarText(getNested(output, "kontakt.unternehmen.name"));
+      const rechtsform = extractScalarText(getNested(output, "kontakt.unternehmen.rechtsform"));
+      const stem =
+        name && rechtsform && normalizeFilenamePart(name).endsWith(normalizeFilenamePart(rechtsform))
+          ? toSafeFilename(name)
+          : toSafeFilename(name, rechtsform) || "output";
+
+      const outDir  = join(OUTPUTS_DIR, stem);
+      await mkdir(outDir, { recursive: true });
+
+      const jsonPath = join(outDir, `${stem}.json`);
+      const pdfPath  = join(outDir, `${stem}.pdf`);
+
+      await Promise.all([
+        Bun.write(jsonPath, JSON.stringify(output, null, 2)),
+        Bun.write(pdfPath, pdfBytes),
+      ]);
+
+      return Response.json({ ok: true, stem, files: { json: `${stem}.json`, pdf: `${stem}.pdf` } });
+    }, {
+      csrf: true,
+      limit: { key: "extract", max: 12, windowMs: 60_000 },
+    }),
+  },
+
+  "/template": {
+    GET: withAuth(async () => {
+      try {
+        return Response.json(await loadTemplate());
+      } catch (err) {
+        return Response.json({ error: String(err) }, { status: 500 });
+      }
+    }),
+  },
+} as const;
+
+function isPdfMagic(data: ArrayBuffer): boolean {
+  const header = new TextDecoder().decode(new Uint8Array(data.slice(0, 5)));
+  return header === "%PDF-";
+}
--- a/packages/extractor/package.json
+++ b/packages/extractor/package.json
@@ -0,0 +1,7 @@
+{
+  "name": "extractor",
+  "version": "0.1.0",
+  "main": "index.ts",
+  "type": "module",
+  "private": true
+}
--- a/packages/extractor/tsconfig.json
+++ b/packages/extractor/tsconfig.json
@@ -0,0 +1,3 @@
+{
+  "extends": "../../tsconfig.json"
+}
--- a/packages/extractor/upload.html
+++ b/packages/extractor/upload.html
@@ -0,0 +1,131 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8">
+  <title>Extractor — Test Upload</title>
+  <style>
+    *, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; }
+    :root {
+      --bg: #0d0d0d; --surface: #161616; --border: #2a2a2a;
+      --text: #e0e0e0; --muted: #666; --accent: #2563eb; --green: #16a34a; --red: #dc2626;
+      font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
+      font-size: 14px;
+    }
+    body { background: var(--bg); color: var(--text); min-height: 100vh; display: flex; align-items: center; justify-content: center; padding: 24px; }
+    .card { background: var(--surface); border: 1px solid var(--border); border-radius: 10px; padding: 28px; width: 100%; max-width: 520px; display: flex; flex-direction: column; gap: 18px; }
+    h1 { font-size: 15px; font-weight: 600; }
+    .drop-zone {
+      border: 2px dashed var(--border); border-radius: 7px; padding: 36px 20px;
+      text-align: center; cursor: pointer; transition: border-color 0.15s, background 0.15s;
+      color: var(--muted); font-size: 13px;
+    }
+    .drop-zone:hover, .drop-zone.over { border-color: var(--accent); background: rgba(37,99,235,0.05); color: var(--text); }
+    .drop-zone.has-file { border-color: var(--green); color: var(--green); }
+    input[type=file] { display: none; }
+    button {
+      padding: 8px 16px; border-radius: 6px; border: none; background: var(--accent);
+      color: #fff; font-size: 13px; cursor: pointer; transition: background 0.12s;
+    }
+    button:hover:not(:disabled) { background: #1d4ed8; }
+    button:disabled { opacity: 0.4; cursor: not-allowed; }
+    .result {
+      background: var(--bg); border: 1px solid var(--border); border-radius: 6px;
+      padding: 14px; font-family: "SF Mono", "Fira Mono", monospace; font-size: 11px;
+      white-space: pre-wrap; word-break: break-all; max-height: 320px; overflow-y: auto;
+      color: var(--text); display: none;
+    }
+    .result.error { border-color: var(--red); color: #f87171; }
+    .result.success { border-color: var(--green); }
+    .status { font-size: 12px; color: var(--muted); min-height: 16px; }
+  </style>
+</head>
+<body>
+  <div class="card">
+    <h1>Extractor — Test Upload</h1>
+
+    <div class="drop-zone" id="drop-zone">
+      <input type="file" id="file-input" accept=".pdf">
+      <span id="drop-label">Drop a PDF here, or click to select</span>
+    </div>
+
+    <button id="submit-btn" disabled>Extract</button>
+
+    <p class="status" id="status"></p>
+    <pre class="result" id="result"></pre>
+  </div>
+
+  <script>
+    const dropZone   = document.getElementById("drop-zone");
+    const fileInput  = document.getElementById("file-input");
+    const dropLabel  = document.getElementById("drop-label");
+    const submitBtn  = document.getElementById("submit-btn");
+    const statusEl   = document.getElementById("status");
+    const resultEl   = document.getElementById("result");
+
+    let selectedFile = null;
+
+    function csrfHeaders(extra = {}) {
+      const csrf = document.cookie
+        .split("; ")
+        .find((part) => part.startsWith("bmp_demo_csrf="))
+        ?.slice("bmp_demo_csrf=".length);
+      return csrf ? { ...extra, "X-BMP-CSRF": decodeURIComponent(csrf) } : extra;
+    }
+
+    dropZone.addEventListener("click", () => fileInput.click());
+
+    fileInput.addEventListener("change", () => setFile(fileInput.files[0]));
+
+    dropZone.addEventListener("dragover", e => { e.preventDefault(); dropZone.classList.add("over"); });
+    dropZone.addEventListener("dragleave", () => dropZone.classList.remove("over"));
+    dropZone.addEventListener("drop", e => {
+      e.preventDefault();
+      dropZone.classList.remove("over");
+      setFile(e.dataTransfer.files[0]);
+    });
+
+    function setFile(file) {
+      if (!file || file.type !== "application/pdf") {
+        statusEl.textContent = "Please select a PDF file.";
+        return;
+      }
+      selectedFile = file;
+      dropLabel.textContent = file.name;
+      dropZone.classList.add("has-file");
+      submitBtn.disabled = false;
+      statusEl.textContent = "";
+      resultEl.style.display = "none";
+    }
+
+    submitBtn.addEventListener("click", async () => {
+      if (!selectedFile) return;
+
+      submitBtn.disabled = true;
+      statusEl.textContent = "Extracting…";
+      resultEl.style.display = "none";
+
+      const form = new FormData();
+      form.append("pdf", selectedFile);
+
+      try {
+        const res  = await fetch("/extract", { method: "POST", headers: csrfHeaders(), body: form });
+        const json = await res.json();
+
+        resultEl.textContent   = JSON.stringify(json, null, 2);
+        resultEl.className     = "result " + (res.ok ? "success" : "error");
+        resultEl.style.display = "block";
+        statusEl.textContent   = res.ok
+          ? `Written → ${json.files?.json || json.stem}`
+          : `Error ${res.status}`;
+      } catch (err) {
+        resultEl.textContent   = String(err);
+        resultEl.className     = "result error";
+        resultEl.style.display = "block";
+        statusEl.textContent   = "Request failed.";
+      } finally {
+        submitBtn.disabled = false;
+      }
+    });
+  </script>
+</body>
+</html>