import type { AntwortEintrag, ExtractedData, FrageMitAntwort, SummaryData } from "./summarizer"; export type { SummaryData }; export interface LlmFrage { id?: string; text: string; antwort?: string; confidence?: number; abgedeckt?: boolean; } export interface LlmAnswer { label?: string; antwortFormat?: AntwortEintrag["antwortFormat"]; fragen?: LlmFrage[]; antwort?: string; zusammenfassung?: string; segmentierung?: AntwortEintrag["segmentierung"]; } export interface LlmDossier { unternehmen?: Record; fragen?: Record; kriterium?: Record; } export interface PseudonymReplacement { type: "company" | "person" | "email" | "phone" | "url" | "address" | "known"; placeholder: string; } export interface PrivacyAudit { mode: "minimized+pseudonymized"; companyName?: string; removedFields: string[]; replacements: PseudonymReplacement[]; } export interface PreparedLlmInput { safe: LlmDossier; audit: PrivacyAudit; } export interface Pseudonymizer { pseudonymizeText(text: string): string; reversePseudonyms(text: string): string; pseudonymizeEntry(entry: T): T; pseudonymizeDossier(dossier: LlmDossier): PreparedLlmInput; audit(): PrivacyAudit; } const REMOVED_FIELDS = [ "kontakt", "kontakt.unternehmen.name", "kontakt.unternehmen.adresse", "kontakt.unternehmen.plz", "kontakt.unternehmen.telefon", "kontakt.unternehmen.email", "kontakt.unternehmen.web", "kontakt.ansprechpartner", ]; const COMPANY_FACT_ALLOWLIST = [ "branche", "rechtsform", "gruendungsjahr", "anzahl_mitarbeiter", "anzahl_azubi", "standorte_deutschland", "standorte_ausland", "umsatzvolumen1", "umsatzvolumen2", "umsatzvolumen3", "referenzen", ]; interface KnownReplacement { value: string; type: PseudonymReplacement["type"]; placeholder: string; } const COMPANY_LEGAL_SUFFIX_PATTERN = /\b(?:gmbh|ag|kg|ohg|ug|eg|ev|e\.v\.|gbr|mbh|co\.?|kgaa|se|ltd\.?|inc\.?|corp\.?)\b/gi; function isRecord(value: unknown): value is Record { return Boolean(value) && typeof value === "object" && !Array.isArray(value); } function normalize(value: string): string { return value.trim().replace(/\s+/g, " "); } function escapeRegExp(value: string): string { return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); } function addKnown( replacements: KnownReplacement[], value: unknown, type: PseudonymReplacement["type"], placeholder: string, ): void { if (typeof value !== "string") return; const normalized = normalize(value); if (normalized.length < 3) return; if (replacements.some((item) => item.value.toLowerCase() === normalized.toLowerCase())) return; replacements.push({ value: normalized, type, placeholder }); } function companyAliasCandidates(companyName: string): string[] { const normalized = normalize(companyName); const withoutSuffix = normalize( normalized .replace(/&/g, " ") .replace(COMPANY_LEGAL_SUFFIX_PATTERN, " ") .replace(/\s+/g, " "), ); const firstToken = withoutSuffix.split(/\s+/).find((token) => token.length >= 4); return [normalized, withoutSuffix, firstToken] .filter((value): value is string => Boolean(value && value.length >= 4)) .filter((value, index, values) => values.findIndex((item) => item.toLowerCase() === value.toLowerCase()) === index); } function getCompanyName(data: ExtractedData): string | undefined { const kontakt = isRecord(data.kontakt) ? data.kontakt : {}; const kontaktUnternehmen = isRecord(kontakt.unternehmen) ? kontakt.unternehmen : {}; return typeof kontaktUnternehmen.name === "string" && kontaktUnternehmen.name.trim() ? normalize(kontaktUnternehmen.name) : undefined; } function collectKnownReplacements(data: ExtractedData, companyName?: string): KnownReplacement[] { const kontakt = isRecord(data.kontakt) ? data.kontakt : {}; const kontaktUnternehmen = isRecord(kontakt.unternehmen) ? kontakt.unternehmen : {}; const ansprechpartner = isRecord(kontakt.ansprechpartner) ? kontakt.ansprechpartner : {}; const replacements: KnownReplacement[] = []; for (const alias of companyName ? companyAliasCandidates(companyName) : []) { addKnown(replacements, alias, "company", "[UNTERNEHMEN]"); } addKnown(replacements, kontaktUnternehmen.email, "email", "[EMAIL_1]"); addKnown(replacements, kontaktUnternehmen.telefon, "phone", "[TELEFON_1]"); addKnown(replacements, kontaktUnternehmen.web, "url", "[URL_1]"); addKnown(replacements, kontaktUnternehmen.adresse, "address", "[ADRESSE_1]"); addKnown(replacements, kontaktUnternehmen.plz, "address", "[ORT_1]"); addKnown(replacements, ansprechpartner.name, "person", "[PERSON_1]"); addKnown(replacements, ansprechpartner.email, "email", "[EMAIL_2]"); addKnown(replacements, ansprechpartner.telefon, "phone", "[TELEFON_2]"); return replacements.toSorted((a, b) => b.value.length - a.value.length); } const PLACEHOLDER_RE = /\[([A-Z][A-Z0-9_]*)_(\d+)\]/g; const PLACEHOLDER_ALL_RE = /\[([A-Z][A-Z0-9_]*)(?:_\d+)?\]/g; function recordReplacement( seen: Map, type: PseudonymReplacement["type"], placeholder: string, ): void { const key = `${type}:${placeholder}`; if (!seen.has(key)) seen.set(key, { type, placeholder }); } function pseudonymizeUnknownPatterns( text: string, seen: Map, ): string { let emailIndex = 10; let phoneIndex = 10; let urlIndex = 10; return text .replace(/\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/gi, () => { const placeholder = `[EMAIL_${emailIndex++}]`; recordReplacement(seen, "email", placeholder); return placeholder; }) .replace(/\b(?:https?:\/\/)?(?:www\.)[^\s<>"')]+/gi, () => { const placeholder = `[URL_${urlIndex++}]`; recordReplacement(seen, "url", placeholder); return placeholder; }) .replace(/(?:\+49|0049|0)[\d\s()/.-]{6,}\d/g, (match) => { const digits = match.replace(/\D/g, ""); if (digits.length < 7) return match; const placeholder = `[TELEFON_${phoneIndex++}]`; recordReplacement(seen, "phone", placeholder); return placeholder; }); } export function reversePseudonymsInText( text: string, knownReplacements: KnownReplacement[], ): string { let output = text; for (const replacement of knownReplacements) { output = output.replaceAll(replacement.placeholder, replacement.value); } return output; } function pseudonymizeValue(value: unknown, pseudonymizeText: (text: string) => string): unknown { if (typeof value === "string") return pseudonymizeText(value); if (Array.isArray(value)) return value.map((item) => pseudonymizeValue(item, pseudonymizeText)); if (isRecord(value)) { return Object.fromEntries( Object.entries(value).map(([key, item]) => [key, pseudonymizeValue(item, pseudonymizeText)]), ); } return value; } function minimizeQuestion(frage: FrageMitAntwort, pseudonymizeText?: (text: string) => string): LlmFrage { const map = pseudonymizeText ?? ((text: string) => text); return { id: frage.id, text: map(frage.text), antwort: frage.antwort ? map(frage.antwort) : frage.antwort, confidence: frage.confidence, abgedeckt: frage.abgedeckt, }; } function minimizeAnswer(entry: AntwortEintrag, pseudonymizeText?: (text: string) => string): LlmAnswer { const map = pseudonymizeText ?? ((text: string) => text); const hasSummary = Boolean(entry.zusammenfassung?.trim()); return { label: entry.label ? map(entry.label) : entry.label, antwortFormat: entry.antwortFormat, fragen: entry.fragen?.map((frage) => minimizeQuestion(frage, map)), antwort: !hasSummary && entry.antwort ? map(entry.antwort) : undefined, zusammenfassung: entry.zusammenfassung ? map(entry.zusammenfassung) : entry.zusammenfassung, segmentierung: entry.segmentierung, }; } export function reversePseudonymsInDossier( dossier: LlmDossier, knownReplacements: KnownReplacement[], ): LlmDossier { return { unternehmen: dossier.unternehmen ? pseudonymizeValue(dossier.unternehmen, (text) => reversePseudonymsInText(text, knownReplacements)) as Record : undefined, fragen: dossier.fragen ? Object.fromEntries( Object.entries(dossier.fragen).map(([key, entry]) => { const reverse = (text: string) => reversePseudonymsInText(text, knownReplacements); return [ key, { ...entry, label: entry.label ? reverse(entry.label) : entry.label, zusammenfassung: entry.zusammenfassung ? reverse(entry.zusammenfassung) : entry.zusammenfassung, fragen: entry.fragen?.map((frage) => ({ ...frage, text: reverse(frage.text), antwort: frage.antwort ? reverse(frage.antwort) : undefined, })) ?? entry.fragen, }, ]; }), ) : undefined, kriterium: dossier.kriterium ? Object.fromEntries( Object.entries(dossier.kriterium).map(([key, entry]) => { const reverse = (text: string) => reversePseudonymsInText(text, knownReplacements); return [ key, { ...entry, label: entry.label ? reverse(entry.label) : entry.label, zusammenfassung: entry.zusammenfassung ? reverse(entry.zusammenfassung) : entry.zusammenfassung, fragen: entry.fragen?.map((frage) => ({ ...frage, text: reverse(frage.text), antwort: frage.antwort ? reverse(frage.antwort) : undefined, })) ?? entry.fragen, }, ]; }), ) : undefined, }; } export function buildLlmDossier(data: SummaryData, pseudonymizeText?: (text: string) => string): LlmDossier { const map = pseudonymizeText ?? ((text: string) => text); const unternehmen = isRecord(data.unternehmen) ? Object.fromEntries( COMPANY_FACT_ALLOWLIST .filter((key) => key in data.unternehmen!) .map((key) => [key, pseudonymizeValue(data.unternehmen![key], map)]), ) : undefined; return { unternehmen, fragen: data.fragen ? Object.fromEntries(Object.entries(data.fragen).map(([key, entry]) => [key, minimizeAnswer(entry, map)])) : undefined, kriterium: data.kriterium ? Object.fromEntries(Object.entries(data.kriterium).map(([key, entry]) => [key, minimizeAnswer(entry, map)])) : undefined, }; } export function createPseudonymizer(data: ExtractedData): Pseudonymizer { const companyName = getCompanyName(data); const known = collectKnownReplacements(data, companyName); const seen = new Map(); function pseudonymizeText(text: string): string { let output = text; for (const replacement of known) { const before = output; output = output.replace(new RegExp(escapeRegExp(replacement.value), "gi"), replacement.placeholder); if (output !== before) { recordReplacement(seen, replacement.type, replacement.placeholder); } } return pseudonymizeUnknownPatterns(output, seen); } function pseudonymizeEntry(entry: T): T { return { ...entry, label: entry.label ? pseudonymizeText(entry.label) : entry.label, fragen: entry.fragen?.map((frage) => minimizeQuestion(frage, pseudonymizeText)), antwort: entry.antwort ? pseudonymizeText(entry.antwort) : entry.antwort, zusammenfassung: entry.zusammenfassung ? pseudonymizeText(entry.zusammenfassung) : entry.zusammenfassung, } as T; } function audit(): PrivacyAudit { return { mode: "minimized+pseudonymized", companyName, removedFields: REMOVED_FIELDS, replacements: Array.from(seen.values()), }; } function pseudonymizeDossier(dossier: LlmDossier): PreparedLlmInput { const safe = pseudonymizeValue(dossier, pseudonymizeText) as LlmDossier; return { safe, audit: audit() }; } function reversePseudonyms(text: string): string { return reversePseudonymsInText(text, known); } return { pseudonymizeText, reversePseudonyms, pseudonymizeEntry, pseudonymizeDossier, audit }; }