bmp-project/packages/summarizer/privacy.ts

import type { AntwortEintrag, ExtractedData, FrageMitAntwort, SummaryData } from "./summarizer";

export type { SummaryData };

export interface LlmFrage {
  id?: string;
  text: string;
  antwort?: string;
  confidence?: number;
  abgedeckt?: boolean;
}

export interface LlmAnswer {
  label?: string;
  antwortFormat?: AntwortEintrag["antwortFormat"];
  fragen?: LlmFrage[];
  antwort?: string;
  zusammenfassung?: string;
  segmentierung?: AntwortEintrag["segmentierung"];
}

export interface LlmDossier {
  unternehmen?: Record<string, unknown>;
  fragen?: Record<string, LlmAnswer>;
  kriterium?: Record<string, LlmAnswer>;
}

export interface PseudonymReplacement {
  type: "company" | "person" | "email" | "phone" | "url" | "address" | "known";
  placeholder: string;
}

export interface PrivacyAudit {
  mode: "minimized+pseudonymized";
  companyName?: string;
  removedFields: string[];
  replacements: PseudonymReplacement[];
}

export interface PreparedLlmInput {
  safe: LlmDossier;
  audit: PrivacyAudit;
}

export interface Pseudonymizer {
  pseudonymizeText(text: string): string;
  reversePseudonyms(text: string): string;
  pseudonymizeEntry<T extends AntwortEintrag>(entry: T): T;
  pseudonymizeDossier(dossier: LlmDossier): PreparedLlmInput;
  audit(): PrivacyAudit;
}

const REMOVED_FIELDS = [
  "kontakt",
  "kontakt.unternehmen.name",
  "kontakt.unternehmen.adresse",
  "kontakt.unternehmen.plz",
  "kontakt.unternehmen.telefon",
  "kontakt.unternehmen.email",
  "kontakt.unternehmen.web",
  "kontakt.ansprechpartner",
];

const COMPANY_FACT_ALLOWLIST = [
  "branche",
  "rechtsform",
  "gruendungsjahr",
  "anzahl_mitarbeiter",
  "anzahl_azubi",
  "standorte_deutschland",
  "standorte_ausland",
  "umsatzvolumen1",
  "umsatzvolumen2",
  "umsatzvolumen3",
  "referenzen",
];

interface KnownReplacement {
  value: string;
  type: PseudonymReplacement["type"];
  placeholder: string;
}

const COMPANY_LEGAL_SUFFIX_PATTERN =
  /\b(?:gmbh|ag|kg|ohg|ug|eg|ev|e\.v\.|gbr|mbh|co\.?|kgaa|se|ltd\.?|inc\.?|corp\.?)\b/gi;

function isRecord(value: unknown): value is Record<string, unknown> {
  return Boolean(value) && typeof value === "object" && !Array.isArray(value);
}

function normalize(value: string): string {
  return value.trim().replace(/\s+/g, " ");
}

function escapeRegExp(value: string): string {
  return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
}

function addKnown(
  replacements: KnownReplacement[],
  value: unknown,
  type: PseudonymReplacement["type"],
  placeholder: string,
): void {
  if (typeof value !== "string") return;
  const normalized = normalize(value);
  if (normalized.length < 3) return;
  if (replacements.some((item) => item.value.toLowerCase() === normalized.toLowerCase())) return;
  replacements.push({ value: normalized, type, placeholder });
}

function companyAliasCandidates(companyName: string): string[] {
  const normalized = normalize(companyName);
  const withoutSuffix = normalize(
    normalized
      .replace(/&/g, " ")
      .replace(COMPANY_LEGAL_SUFFIX_PATTERN, " ")
      .replace(/\s+/g, " "),
  );
  const firstToken = withoutSuffix.split(/\s+/).find((token) => token.length >= 4);

  return [normalized, withoutSuffix, firstToken]
    .filter((value): value is string => Boolean(value && value.length >= 4))
    .filter((value, index, values) => values.findIndex((item) => item.toLowerCase() === value.toLowerCase()) === index);
}

function getCompanyName(data: ExtractedData): string | undefined {
  const kontakt = isRecord(data.kontakt) ? data.kontakt : {};
  const kontaktUnternehmen = isRecord(kontakt.unternehmen) ? kontakt.unternehmen : {};
  return typeof kontaktUnternehmen.name === "string" && kontaktUnternehmen.name.trim()
    ? normalize(kontaktUnternehmen.name)
    : undefined;
}

function collectKnownReplacements(data: ExtractedData, companyName?: string): KnownReplacement[] {
  const kontakt = isRecord(data.kontakt) ? data.kontakt : {};
  const kontaktUnternehmen = isRecord(kontakt.unternehmen) ? kontakt.unternehmen : {};
  const ansprechpartner = isRecord(kontakt.ansprechpartner) ? kontakt.ansprechpartner : {};
  const replacements: KnownReplacement[] = [];

  for (const alias of companyName ? companyAliasCandidates(companyName) : []) {
    addKnown(replacements, alias, "company", "[UNTERNEHMEN]");
  }
  addKnown(replacements, kontaktUnternehmen.email, "email", "[EMAIL_1]");
  addKnown(replacements, kontaktUnternehmen.telefon, "phone", "[TELEFON_1]");
  addKnown(replacements, kontaktUnternehmen.web, "url", "[URL_1]");
  addKnown(replacements, kontaktUnternehmen.adresse, "address", "[ADRESSE_1]");
  addKnown(replacements, kontaktUnternehmen.plz, "address", "[ORT_1]");
  addKnown(replacements, ansprechpartner.name, "person", "[PERSON_1]");
  addKnown(replacements, ansprechpartner.email, "email", "[EMAIL_2]");
  addKnown(replacements, ansprechpartner.telefon, "phone", "[TELEFON_2]");

  return replacements.toSorted((a, b) => b.value.length - a.value.length);
}

const PLACEHOLDER_RE = /\[([A-Z][A-Z0-9_]*)_(\d+)\]/g;
const PLACEHOLDER_ALL_RE = /\[([A-Z][A-Z0-9_]*)(?:_\d+)?\]/g;

function recordReplacement(
  seen: Map<string, PseudonymReplacement>,
  type: PseudonymReplacement["type"],
  placeholder: string,
): void {
  const key = `${type}:${placeholder}`;
  if (!seen.has(key)) seen.set(key, { type, placeholder });
}

function pseudonymizeUnknownPatterns(
  text: string,
  seen: Map<string, PseudonymReplacement>,
): string {
  let emailIndex = 10;
  let phoneIndex = 10;
  let urlIndex = 10;

  return text
    .replace(/\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/gi, () => {
      const placeholder = `[EMAIL_${emailIndex++}]`;
      recordReplacement(seen, "email", placeholder);
      return placeholder;
    })
    .replace(/\b(?:https?:\/\/)?(?:www\.)[^\s<>"')]+/gi, () => {
      const placeholder = `[URL_${urlIndex++}]`;
      recordReplacement(seen, "url", placeholder);
      return placeholder;
    })
    .replace(/(?:\+49|0049|0)[\d\s()/.-]{6,}\d/g, (match) => {
      const digits = match.replace(/\D/g, "");
      if (digits.length < 7) return match;
      const placeholder = `[TELEFON_${phoneIndex++}]`;
      recordReplacement(seen, "phone", placeholder);
      return placeholder;
    });
}

export function reversePseudonymsInText(
  text: string,
  knownReplacements: KnownReplacement[],
): string {
  let output = text;
  for (const replacement of knownReplacements) {
    output = output.replaceAll(replacement.placeholder, replacement.value);
  }
  return output;
}

function pseudonymizeValue(value: unknown, pseudonymizeText: (text: string) => string): unknown {
  if (typeof value === "string") return pseudonymizeText(value);
  if (Array.isArray(value)) return value.map((item) => pseudonymizeValue(item, pseudonymizeText));
  if (isRecord(value)) {
    return Object.fromEntries(
      Object.entries(value).map(([key, item]) => [key, pseudonymizeValue(item, pseudonymizeText)]),
    );
  }
  return value;
}

function minimizeQuestion(frage: FrageMitAntwort, pseudonymizeText?: (text: string) => string): LlmFrage {
  const map = pseudonymizeText ?? ((text: string) => text);
  return {
    id: frage.id,
    text: map(frage.text),
    antwort: frage.antwort ? map(frage.antwort) : frage.antwort,
    confidence: frage.confidence,
    abgedeckt: frage.abgedeckt,
  };
}

function minimizeAnswer(entry: AntwortEintrag, pseudonymizeText?: (text: string) => string): LlmAnswer {
  const map = pseudonymizeText ?? ((text: string) => text);
  const hasSummary = Boolean(entry.zusammenfassung?.trim());
  return {
    label: entry.label ? map(entry.label) : entry.label,
    antwortFormat: entry.antwortFormat,
    fragen: entry.fragen?.map((frage) => minimizeQuestion(frage, map)),
    antwort: !hasSummary && entry.antwort ? map(entry.antwort) : undefined,
    zusammenfassung: entry.zusammenfassung ? map(entry.zusammenfassung) : entry.zusammenfassung,
    segmentierung: entry.segmentierung,
  };
}

export function reversePseudonymsInDossier(
  dossier: LlmDossier,
  knownReplacements: KnownReplacement[],
): LlmDossier {
  return {
    unternehmen: dossier.unternehmen
      ? pseudonymizeValue(dossier.unternehmen, (text) => reversePseudonymsInText(text, knownReplacements)) as Record<string, unknown>
      : undefined,
    fragen: dossier.fragen
      ? Object.fromEntries(
        Object.entries(dossier.fragen).map(([key, entry]) => {
          const reverse = (text: string) => reversePseudonymsInText(text, knownReplacements);
          return [
            key,
            {
              ...entry,
              label: entry.label ? reverse(entry.label) : entry.label,
              zusammenfassung: entry.zusammenfassung ? reverse(entry.zusammenfassung) : entry.zusammenfassung,
              fragen: entry.fragen?.map((frage) => ({
                ...frage,
                text: reverse(frage.text),
                antwort: frage.antwort ? reverse(frage.antwort) : undefined,
              })) ?? entry.fragen,
            },
          ];
        }),
      )
      : undefined,
    kriterium: dossier.kriterium
      ? Object.fromEntries(
        Object.entries(dossier.kriterium).map(([key, entry]) => {
          const reverse = (text: string) => reversePseudonymsInText(text, knownReplacements);
          return [
            key,
            {
              ...entry,
              label: entry.label ? reverse(entry.label) : entry.label,
              zusammenfassung: entry.zusammenfassung ? reverse(entry.zusammenfassung) : entry.zusammenfassung,
              fragen: entry.fragen?.map((frage) => ({
                ...frage,
                text: reverse(frage.text),
                antwort: frage.antwort ? reverse(frage.antwort) : undefined,
              })) ?? entry.fragen,
            },
          ];
        }),
      )
      : undefined,
  };
}

export function buildLlmDossier(data: SummaryData, pseudonymizeText?: (text: string) => string): LlmDossier {
  const map = pseudonymizeText ?? ((text: string) => text);
  const unternehmen = isRecord(data.unternehmen)
    ? Object.fromEntries(
      COMPANY_FACT_ALLOWLIST
        .filter((key) => key in data.unternehmen!)
        .map((key) => [key, pseudonymizeValue(data.unternehmen![key], map)]),
    )
    : undefined;

  return {
    unternehmen,
    fragen: data.fragen
      ? Object.fromEntries(Object.entries(data.fragen).map(([key, entry]) => [key, minimizeAnswer(entry, map)]))
      : undefined,
    kriterium: data.kriterium
      ? Object.fromEntries(Object.entries(data.kriterium).map(([key, entry]) => [key, minimizeAnswer(entry, map)]))
      : undefined,
  };
}

export function createPseudonymizer(data: ExtractedData): Pseudonymizer {
  const companyName = getCompanyName(data);
  const known = collectKnownReplacements(data, companyName);
  const seen = new Map<string, PseudonymReplacement>();

  function pseudonymizeText(text: string): string {
    let output = text;
    for (const replacement of known) {
      const before = output;
      output = output.replace(new RegExp(escapeRegExp(replacement.value), "gi"), replacement.placeholder);
      if (output !== before) {
        recordReplacement(seen, replacement.type, replacement.placeholder);
      }
    }
    return pseudonymizeUnknownPatterns(output, seen);
  }

  function pseudonymizeEntry<T extends AntwortEintrag>(entry: T): T {
    return {
      ...entry,
      label: entry.label ? pseudonymizeText(entry.label) : entry.label,
      fragen: entry.fragen?.map((frage) => minimizeQuestion(frage, pseudonymizeText)),
      antwort: entry.antwort ? pseudonymizeText(entry.antwort) : entry.antwort,
      zusammenfassung: entry.zusammenfassung ? pseudonymizeText(entry.zusammenfassung) : entry.zusammenfassung,
    } as T;
  }

  function audit(): PrivacyAudit {
    return {
      mode: "minimized+pseudonymized",
      companyName,
      removedFields: REMOVED_FIELDS,
      replacements: Array.from(seen.values()),
    };
  }

  function pseudonymizeDossier(dossier: LlmDossier): PreparedLlmInput {
    const safe = pseudonymizeValue(dossier, pseudonymizeText) as LlmDossier;
    return { safe, audit: audit() };
  }

  function reversePseudonyms(text: string): string {
    return reversePseudonymsInText(text, known);
  }

  return { pseudonymizeText, reversePseudonyms, pseudonymizeEntry, pseudonymizeDossier, audit };
}