361 lines
12 KiB
TypeScript
361 lines
12 KiB
TypeScript
import type { AntwortEintrag, ExtractedData, FrageMitAntwort, SummaryData } from "./summarizer";
|
|
|
|
export type { SummaryData };
|
|
|
|
export interface LlmFrage {
|
|
id?: string;
|
|
text: string;
|
|
antwort?: string;
|
|
confidence?: number;
|
|
abgedeckt?: boolean;
|
|
}
|
|
|
|
export interface LlmAnswer {
|
|
label?: string;
|
|
antwortFormat?: AntwortEintrag["antwortFormat"];
|
|
fragen?: LlmFrage[];
|
|
antwort?: string;
|
|
zusammenfassung?: string;
|
|
segmentierung?: AntwortEintrag["segmentierung"];
|
|
}
|
|
|
|
export interface LlmDossier {
|
|
unternehmen?: Record<string, unknown>;
|
|
fragen?: Record<string, LlmAnswer>;
|
|
kriterium?: Record<string, LlmAnswer>;
|
|
}
|
|
|
|
export interface PseudonymReplacement {
|
|
type: "company" | "person" | "email" | "phone" | "url" | "address" | "known";
|
|
placeholder: string;
|
|
}
|
|
|
|
export interface PrivacyAudit {
|
|
mode: "minimized+pseudonymized";
|
|
companyName?: string;
|
|
removedFields: string[];
|
|
replacements: PseudonymReplacement[];
|
|
}
|
|
|
|
export interface PreparedLlmInput {
|
|
safe: LlmDossier;
|
|
audit: PrivacyAudit;
|
|
}
|
|
|
|
export interface Pseudonymizer {
|
|
pseudonymizeText(text: string): string;
|
|
reversePseudonyms(text: string): string;
|
|
pseudonymizeEntry<T extends AntwortEintrag>(entry: T): T;
|
|
pseudonymizeDossier(dossier: LlmDossier): PreparedLlmInput;
|
|
audit(): PrivacyAudit;
|
|
}
|
|
|
|
const REMOVED_FIELDS = [
|
|
"kontakt",
|
|
"kontakt.unternehmen.name",
|
|
"kontakt.unternehmen.adresse",
|
|
"kontakt.unternehmen.plz",
|
|
"kontakt.unternehmen.telefon",
|
|
"kontakt.unternehmen.email",
|
|
"kontakt.unternehmen.web",
|
|
"kontakt.ansprechpartner",
|
|
];
|
|
|
|
const COMPANY_FACT_ALLOWLIST = [
|
|
"branche",
|
|
"rechtsform",
|
|
"gruendungsjahr",
|
|
"anzahl_mitarbeiter",
|
|
"anzahl_azubi",
|
|
"standorte_deutschland",
|
|
"standorte_ausland",
|
|
"umsatzvolumen1",
|
|
"umsatzvolumen2",
|
|
"umsatzvolumen3",
|
|
"referenzen",
|
|
];
|
|
|
|
interface KnownReplacement {
|
|
value: string;
|
|
type: PseudonymReplacement["type"];
|
|
placeholder: string;
|
|
}
|
|
|
|
const COMPANY_LEGAL_SUFFIX_PATTERN =
|
|
/\b(?:gmbh|ag|kg|ohg|ug|eg|ev|e\.v\.|gbr|mbh|co\.?|kgaa|se|ltd\.?|inc\.?|corp\.?)\b/gi;
|
|
|
|
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
return Boolean(value) && typeof value === "object" && !Array.isArray(value);
|
|
}
|
|
|
|
function normalize(value: string): string {
|
|
return value.trim().replace(/\s+/g, " ");
|
|
}
|
|
|
|
function escapeRegExp(value: string): string {
|
|
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
}
|
|
|
|
function addKnown(
|
|
replacements: KnownReplacement[],
|
|
value: unknown,
|
|
type: PseudonymReplacement["type"],
|
|
placeholder: string,
|
|
): void {
|
|
if (typeof value !== "string") return;
|
|
const normalized = normalize(value);
|
|
if (normalized.length < 3) return;
|
|
if (replacements.some((item) => item.value.toLowerCase() === normalized.toLowerCase())) return;
|
|
replacements.push({ value: normalized, type, placeholder });
|
|
}
|
|
|
|
function companyAliasCandidates(companyName: string): string[] {
|
|
const normalized = normalize(companyName);
|
|
const withoutSuffix = normalize(
|
|
normalized
|
|
.replace(/&/g, " ")
|
|
.replace(COMPANY_LEGAL_SUFFIX_PATTERN, " ")
|
|
.replace(/\s+/g, " "),
|
|
);
|
|
const firstToken = withoutSuffix.split(/\s+/).find((token) => token.length >= 4);
|
|
|
|
return [normalized, withoutSuffix, firstToken]
|
|
.filter((value): value is string => Boolean(value && value.length >= 4))
|
|
.filter((value, index, values) => values.findIndex((item) => item.toLowerCase() === value.toLowerCase()) === index);
|
|
}
|
|
|
|
function getCompanyName(data: ExtractedData): string | undefined {
|
|
const kontakt = isRecord(data.kontakt) ? data.kontakt : {};
|
|
const kontaktUnternehmen = isRecord(kontakt.unternehmen) ? kontakt.unternehmen : {};
|
|
return typeof kontaktUnternehmen.name === "string" && kontaktUnternehmen.name.trim()
|
|
? normalize(kontaktUnternehmen.name)
|
|
: undefined;
|
|
}
|
|
|
|
function collectKnownReplacements(data: ExtractedData, companyName?: string): KnownReplacement[] {
|
|
const kontakt = isRecord(data.kontakt) ? data.kontakt : {};
|
|
const kontaktUnternehmen = isRecord(kontakt.unternehmen) ? kontakt.unternehmen : {};
|
|
const ansprechpartner = isRecord(kontakt.ansprechpartner) ? kontakt.ansprechpartner : {};
|
|
const replacements: KnownReplacement[] = [];
|
|
|
|
for (const alias of companyName ? companyAliasCandidates(companyName) : []) {
|
|
addKnown(replacements, alias, "company", "[UNTERNEHMEN]");
|
|
}
|
|
addKnown(replacements, kontaktUnternehmen.email, "email", "[EMAIL_1]");
|
|
addKnown(replacements, kontaktUnternehmen.telefon, "phone", "[TELEFON_1]");
|
|
addKnown(replacements, kontaktUnternehmen.web, "url", "[URL_1]");
|
|
addKnown(replacements, kontaktUnternehmen.adresse, "address", "[ADRESSE_1]");
|
|
addKnown(replacements, kontaktUnternehmen.plz, "address", "[ORT_1]");
|
|
addKnown(replacements, ansprechpartner.name, "person", "[PERSON_1]");
|
|
addKnown(replacements, ansprechpartner.email, "email", "[EMAIL_2]");
|
|
addKnown(replacements, ansprechpartner.telefon, "phone", "[TELEFON_2]");
|
|
|
|
return replacements.toSorted((a, b) => b.value.length - a.value.length);
|
|
}
|
|
|
|
const PLACEHOLDER_RE = /\[([A-Z][A-Z0-9_]*)_(\d+)\]/g;
|
|
const PLACEHOLDER_ALL_RE = /\[([A-Z][A-Z0-9_]*)(?:_\d+)?\]/g;
|
|
|
|
function recordReplacement(
|
|
seen: Map<string, PseudonymReplacement>,
|
|
type: PseudonymReplacement["type"],
|
|
placeholder: string,
|
|
): void {
|
|
const key = `${type}:${placeholder}`;
|
|
if (!seen.has(key)) seen.set(key, { type, placeholder });
|
|
}
|
|
|
|
function pseudonymizeUnknownPatterns(
|
|
text: string,
|
|
seen: Map<string, PseudonymReplacement>,
|
|
): string {
|
|
let emailIndex = 10;
|
|
let phoneIndex = 10;
|
|
let urlIndex = 10;
|
|
|
|
return text
|
|
.replace(/\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/gi, () => {
|
|
const placeholder = `[EMAIL_${emailIndex++}]`;
|
|
recordReplacement(seen, "email", placeholder);
|
|
return placeholder;
|
|
})
|
|
.replace(/\b(?:https?:\/\/)?(?:www\.)[^\s<>"')]+/gi, () => {
|
|
const placeholder = `[URL_${urlIndex++}]`;
|
|
recordReplacement(seen, "url", placeholder);
|
|
return placeholder;
|
|
})
|
|
.replace(/(?:\+49|0049|0)[\d\s()/.-]{6,}\d/g, (match) => {
|
|
const digits = match.replace(/\D/g, "");
|
|
if (digits.length < 7) return match;
|
|
const placeholder = `[TELEFON_${phoneIndex++}]`;
|
|
recordReplacement(seen, "phone", placeholder);
|
|
return placeholder;
|
|
});
|
|
}
|
|
|
|
export function reversePseudonymsInText(
|
|
text: string,
|
|
knownReplacements: KnownReplacement[],
|
|
): string {
|
|
let output = text;
|
|
for (const replacement of knownReplacements) {
|
|
output = output.replaceAll(replacement.placeholder, replacement.value);
|
|
}
|
|
return output;
|
|
}
|
|
|
|
function pseudonymizeValue(value: unknown, pseudonymizeText: (text: string) => string): unknown {
|
|
if (typeof value === "string") return pseudonymizeText(value);
|
|
if (Array.isArray(value)) return value.map((item) => pseudonymizeValue(item, pseudonymizeText));
|
|
if (isRecord(value)) {
|
|
return Object.fromEntries(
|
|
Object.entries(value).map(([key, item]) => [key, pseudonymizeValue(item, pseudonymizeText)]),
|
|
);
|
|
}
|
|
return value;
|
|
}
|
|
|
|
function minimizeQuestion(frage: FrageMitAntwort, pseudonymizeText?: (text: string) => string): LlmFrage {
|
|
const map = pseudonymizeText ?? ((text: string) => text);
|
|
return {
|
|
id: frage.id,
|
|
text: map(frage.text),
|
|
antwort: frage.antwort ? map(frage.antwort) : frage.antwort,
|
|
confidence: frage.confidence,
|
|
abgedeckt: frage.abgedeckt,
|
|
};
|
|
}
|
|
|
|
function minimizeAnswer(entry: AntwortEintrag, pseudonymizeText?: (text: string) => string): LlmAnswer {
|
|
const map = pseudonymizeText ?? ((text: string) => text);
|
|
const hasSummary = Boolean(entry.zusammenfassung?.trim());
|
|
return {
|
|
label: entry.label ? map(entry.label) : entry.label,
|
|
antwortFormat: entry.antwortFormat,
|
|
fragen: entry.fragen?.map((frage) => minimizeQuestion(frage, map)),
|
|
antwort: !hasSummary && entry.antwort ? map(entry.antwort) : undefined,
|
|
zusammenfassung: entry.zusammenfassung ? map(entry.zusammenfassung) : entry.zusammenfassung,
|
|
segmentierung: entry.segmentierung,
|
|
};
|
|
}
|
|
|
|
export function reversePseudonymsInDossier(
|
|
dossier: LlmDossier,
|
|
knownReplacements: KnownReplacement[],
|
|
): LlmDossier {
|
|
return {
|
|
unternehmen: dossier.unternehmen
|
|
? pseudonymizeValue(dossier.unternehmen, (text) => reversePseudonymsInText(text, knownReplacements)) as Record<string, unknown>
|
|
: undefined,
|
|
fragen: dossier.fragen
|
|
? Object.fromEntries(
|
|
Object.entries(dossier.fragen).map(([key, entry]) => {
|
|
const reverse = (text: string) => reversePseudonymsInText(text, knownReplacements);
|
|
return [
|
|
key,
|
|
{
|
|
...entry,
|
|
label: entry.label ? reverse(entry.label) : entry.label,
|
|
zusammenfassung: entry.zusammenfassung ? reverse(entry.zusammenfassung) : entry.zusammenfassung,
|
|
fragen: entry.fragen?.map((frage) => ({
|
|
...frage,
|
|
text: reverse(frage.text),
|
|
antwort: frage.antwort ? reverse(frage.antwort) : undefined,
|
|
})) ?? entry.fragen,
|
|
},
|
|
];
|
|
}),
|
|
)
|
|
: undefined,
|
|
kriterium: dossier.kriterium
|
|
? Object.fromEntries(
|
|
Object.entries(dossier.kriterium).map(([key, entry]) => {
|
|
const reverse = (text: string) => reversePseudonymsInText(text, knownReplacements);
|
|
return [
|
|
key,
|
|
{
|
|
...entry,
|
|
label: entry.label ? reverse(entry.label) : entry.label,
|
|
zusammenfassung: entry.zusammenfassung ? reverse(entry.zusammenfassung) : entry.zusammenfassung,
|
|
fragen: entry.fragen?.map((frage) => ({
|
|
...frage,
|
|
text: reverse(frage.text),
|
|
antwort: frage.antwort ? reverse(frage.antwort) : undefined,
|
|
})) ?? entry.fragen,
|
|
},
|
|
];
|
|
}),
|
|
)
|
|
: undefined,
|
|
};
|
|
}
|
|
|
|
export function buildLlmDossier(data: SummaryData, pseudonymizeText?: (text: string) => string): LlmDossier {
|
|
const map = pseudonymizeText ?? ((text: string) => text);
|
|
const unternehmen = isRecord(data.unternehmen)
|
|
? Object.fromEntries(
|
|
COMPANY_FACT_ALLOWLIST
|
|
.filter((key) => key in data.unternehmen!)
|
|
.map((key) => [key, pseudonymizeValue(data.unternehmen![key], map)]),
|
|
)
|
|
: undefined;
|
|
|
|
return {
|
|
unternehmen,
|
|
fragen: data.fragen
|
|
? Object.fromEntries(Object.entries(data.fragen).map(([key, entry]) => [key, minimizeAnswer(entry, map)]))
|
|
: undefined,
|
|
kriterium: data.kriterium
|
|
? Object.fromEntries(Object.entries(data.kriterium).map(([key, entry]) => [key, minimizeAnswer(entry, map)]))
|
|
: undefined,
|
|
};
|
|
}
|
|
|
|
export function createPseudonymizer(data: ExtractedData): Pseudonymizer {
|
|
const companyName = getCompanyName(data);
|
|
const known = collectKnownReplacements(data, companyName);
|
|
const seen = new Map<string, PseudonymReplacement>();
|
|
|
|
function pseudonymizeText(text: string): string {
|
|
let output = text;
|
|
for (const replacement of known) {
|
|
const before = output;
|
|
output = output.replace(new RegExp(escapeRegExp(replacement.value), "gi"), replacement.placeholder);
|
|
if (output !== before) {
|
|
recordReplacement(seen, replacement.type, replacement.placeholder);
|
|
}
|
|
}
|
|
return pseudonymizeUnknownPatterns(output, seen);
|
|
}
|
|
|
|
function pseudonymizeEntry<T extends AntwortEintrag>(entry: T): T {
|
|
return {
|
|
...entry,
|
|
label: entry.label ? pseudonymizeText(entry.label) : entry.label,
|
|
fragen: entry.fragen?.map((frage) => minimizeQuestion(frage, pseudonymizeText)),
|
|
antwort: entry.antwort ? pseudonymizeText(entry.antwort) : entry.antwort,
|
|
zusammenfassung: entry.zusammenfassung ? pseudonymizeText(entry.zusammenfassung) : entry.zusammenfassung,
|
|
} as T;
|
|
}
|
|
|
|
function audit(): PrivacyAudit {
|
|
return {
|
|
mode: "minimized+pseudonymized",
|
|
companyName,
|
|
removedFields: REMOVED_FIELDS,
|
|
replacements: Array.from(seen.values()),
|
|
};
|
|
}
|
|
|
|
function pseudonymizeDossier(dossier: LlmDossier): PreparedLlmInput {
|
|
const safe = pseudonymizeValue(dossier, pseudonymizeText) as LlmDossier;
|
|
return { safe, audit: audit() };
|
|
}
|
|
|
|
function reversePseudonyms(text: string): string {
|
|
return reversePseudonymsInText(text, known);
|
|
}
|
|
|
|
return { pseudonymizeText, reversePseudonyms, pseudonymizeEntry, pseudonymizeDossier, audit };
|
|
}
|