initial commit
This commit is contained in:
syntaxbullet
360
packages/summarizer/privacy.ts
Normal file
360
packages/summarizer/privacy.ts
Normal file
@@ -0,0 +1,360 @@
|
||||
import type { AntwortEintrag, ExtractedData, FrageMitAntwort, SummaryData } from "./summarizer";
|
||||
|
||||
export type { SummaryData };
|
||||
|
||||
export interface LlmFrage {
|
||||
id?: string;
|
||||
text: string;
|
||||
antwort?: string;
|
||||
confidence?: number;
|
||||
abgedeckt?: boolean;
|
||||
}
|
||||
|
||||
export interface LlmAnswer {
|
||||
label?: string;
|
||||
antwortFormat?: AntwortEintrag["antwortFormat"];
|
||||
fragen?: LlmFrage[];
|
||||
antwort?: string;
|
||||
zusammenfassung?: string;
|
||||
segmentierung?: AntwortEintrag["segmentierung"];
|
||||
}
|
||||
|
||||
export interface LlmDossier {
|
||||
unternehmen?: Record<string, unknown>;
|
||||
fragen?: Record<string, LlmAnswer>;
|
||||
kriterium?: Record<string, LlmAnswer>;
|
||||
}
|
||||
|
||||
export interface PseudonymReplacement {
|
||||
type: "company" | "person" | "email" | "phone" | "url" | "address" | "known";
|
||||
placeholder: string;
|
||||
}
|
||||
|
||||
export interface PrivacyAudit {
|
||||
mode: "minimized+pseudonymized";
|
||||
companyName?: string;
|
||||
removedFields: string[];
|
||||
replacements: PseudonymReplacement[];
|
||||
}
|
||||
|
||||
export interface PreparedLlmInput {
|
||||
safe: LlmDossier;
|
||||
audit: PrivacyAudit;
|
||||
}
|
||||
|
||||
export interface Pseudonymizer {
|
||||
pseudonymizeText(text: string): string;
|
||||
reversePseudonyms(text: string): string;
|
||||
pseudonymizeEntry<T extends AntwortEintrag>(entry: T): T;
|
||||
pseudonymizeDossier(dossier: LlmDossier): PreparedLlmInput;
|
||||
audit(): PrivacyAudit;
|
||||
}
|
||||
|
||||
const REMOVED_FIELDS = [
|
||||
"kontakt",
|
||||
"kontakt.unternehmen.name",
|
||||
"kontakt.unternehmen.adresse",
|
||||
"kontakt.unternehmen.plz",
|
||||
"kontakt.unternehmen.telefon",
|
||||
"kontakt.unternehmen.email",
|
||||
"kontakt.unternehmen.web",
|
||||
"kontakt.ansprechpartner",
|
||||
];
|
||||
|
||||
const COMPANY_FACT_ALLOWLIST = [
|
||||
"branche",
|
||||
"rechtsform",
|
||||
"gruendungsjahr",
|
||||
"anzahl_mitarbeiter",
|
||||
"anzahl_azubi",
|
||||
"standorte_deutschland",
|
||||
"standorte_ausland",
|
||||
"umsatzvolumen1",
|
||||
"umsatzvolumen2",
|
||||
"umsatzvolumen3",
|
||||
"referenzen",
|
||||
];
|
||||
|
||||
interface KnownReplacement {
|
||||
value: string;
|
||||
type: PseudonymReplacement["type"];
|
||||
placeholder: string;
|
||||
}
|
||||
|
||||
const COMPANY_LEGAL_SUFFIX_PATTERN =
|
||||
/\b(?:gmbh|ag|kg|ohg|ug|eg|ev|e\.v\.|gbr|mbh|co\.?|kgaa|se|ltd\.?|inc\.?|corp\.?)\b/gi;
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return Boolean(value) && typeof value === "object" && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function normalize(value: string): string {
|
||||
return value.trim().replace(/\s+/g, " ");
|
||||
}
|
||||
|
||||
function escapeRegExp(value: string): string {
|
||||
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||
}
|
||||
|
||||
function addKnown(
|
||||
replacements: KnownReplacement[],
|
||||
value: unknown,
|
||||
type: PseudonymReplacement["type"],
|
||||
placeholder: string,
|
||||
): void {
|
||||
if (typeof value !== "string") return;
|
||||
const normalized = normalize(value);
|
||||
if (normalized.length < 3) return;
|
||||
if (replacements.some((item) => item.value.toLowerCase() === normalized.toLowerCase())) return;
|
||||
replacements.push({ value: normalized, type, placeholder });
|
||||
}
|
||||
|
||||
function companyAliasCandidates(companyName: string): string[] {
|
||||
const normalized = normalize(companyName);
|
||||
const withoutSuffix = normalize(
|
||||
normalized
|
||||
.replace(/&/g, " ")
|
||||
.replace(COMPANY_LEGAL_SUFFIX_PATTERN, " ")
|
||||
.replace(/\s+/g, " "),
|
||||
);
|
||||
const firstToken = withoutSuffix.split(/\s+/).find((token) => token.length >= 4);
|
||||
|
||||
return [normalized, withoutSuffix, firstToken]
|
||||
.filter((value): value is string => Boolean(value && value.length >= 4))
|
||||
.filter((value, index, values) => values.findIndex((item) => item.toLowerCase() === value.toLowerCase()) === index);
|
||||
}
|
||||
|
||||
function getCompanyName(data: ExtractedData): string | undefined {
|
||||
const kontakt = isRecord(data.kontakt) ? data.kontakt : {};
|
||||
const kontaktUnternehmen = isRecord(kontakt.unternehmen) ? kontakt.unternehmen : {};
|
||||
return typeof kontaktUnternehmen.name === "string" && kontaktUnternehmen.name.trim()
|
||||
? normalize(kontaktUnternehmen.name)
|
||||
: undefined;
|
||||
}
|
||||
|
||||
function collectKnownReplacements(data: ExtractedData, companyName?: string): KnownReplacement[] {
|
||||
const kontakt = isRecord(data.kontakt) ? data.kontakt : {};
|
||||
const kontaktUnternehmen = isRecord(kontakt.unternehmen) ? kontakt.unternehmen : {};
|
||||
const ansprechpartner = isRecord(kontakt.ansprechpartner) ? kontakt.ansprechpartner : {};
|
||||
const replacements: KnownReplacement[] = [];
|
||||
|
||||
for (const alias of companyName ? companyAliasCandidates(companyName) : []) {
|
||||
addKnown(replacements, alias, "company", "[UNTERNEHMEN]");
|
||||
}
|
||||
addKnown(replacements, kontaktUnternehmen.email, "email", "[EMAIL_1]");
|
||||
addKnown(replacements, kontaktUnternehmen.telefon, "phone", "[TELEFON_1]");
|
||||
addKnown(replacements, kontaktUnternehmen.web, "url", "[URL_1]");
|
||||
addKnown(replacements, kontaktUnternehmen.adresse, "address", "[ADRESSE_1]");
|
||||
addKnown(replacements, kontaktUnternehmen.plz, "address", "[ORT_1]");
|
||||
addKnown(replacements, ansprechpartner.name, "person", "[PERSON_1]");
|
||||
addKnown(replacements, ansprechpartner.email, "email", "[EMAIL_2]");
|
||||
addKnown(replacements, ansprechpartner.telefon, "phone", "[TELEFON_2]");
|
||||
|
||||
return replacements.toSorted((a, b) => b.value.length - a.value.length);
|
||||
}
|
||||
|
||||
const PLACEHOLDER_RE = /\[([A-Z][A-Z0-9_]*)_(\d+)\]/g;
|
||||
const PLACEHOLDER_ALL_RE = /\[([A-Z][A-Z0-9_]*)(?:_\d+)?\]/g;
|
||||
|
||||
function recordReplacement(
|
||||
seen: Map<string, PseudonymReplacement>,
|
||||
type: PseudonymReplacement["type"],
|
||||
placeholder: string,
|
||||
): void {
|
||||
const key = `${type}:${placeholder}`;
|
||||
if (!seen.has(key)) seen.set(key, { type, placeholder });
|
||||
}
|
||||
|
||||
function pseudonymizeUnknownPatterns(
|
||||
text: string,
|
||||
seen: Map<string, PseudonymReplacement>,
|
||||
): string {
|
||||
let emailIndex = 10;
|
||||
let phoneIndex = 10;
|
||||
let urlIndex = 10;
|
||||
|
||||
return text
|
||||
.replace(/\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/gi, () => {
|
||||
const placeholder = `[EMAIL_${emailIndex++}]`;
|
||||
recordReplacement(seen, "email", placeholder);
|
||||
return placeholder;
|
||||
})
|
||||
.replace(/\b(?:https?:\/\/)?(?:www\.)[^\s<>"')]+/gi, () => {
|
||||
const placeholder = `[URL_${urlIndex++}]`;
|
||||
recordReplacement(seen, "url", placeholder);
|
||||
return placeholder;
|
||||
})
|
||||
.replace(/(?:\+49|0049|0)[\d\s()/.-]{6,}\d/g, (match) => {
|
||||
const digits = match.replace(/\D/g, "");
|
||||
if (digits.length < 7) return match;
|
||||
const placeholder = `[TELEFON_${phoneIndex++}]`;
|
||||
recordReplacement(seen, "phone", placeholder);
|
||||
return placeholder;
|
||||
});
|
||||
}
|
||||
|
||||
export function reversePseudonymsInText(
|
||||
text: string,
|
||||
knownReplacements: KnownReplacement[],
|
||||
): string {
|
||||
let output = text;
|
||||
for (const replacement of knownReplacements) {
|
||||
output = output.replaceAll(replacement.placeholder, replacement.value);
|
||||
}
|
||||
return output;
|
||||
}
|
||||
|
||||
function pseudonymizeValue(value: unknown, pseudonymizeText: (text: string) => string): unknown {
|
||||
if (typeof value === "string") return pseudonymizeText(value);
|
||||
if (Array.isArray(value)) return value.map((item) => pseudonymizeValue(item, pseudonymizeText));
|
||||
if (isRecord(value)) {
|
||||
return Object.fromEntries(
|
||||
Object.entries(value).map(([key, item]) => [key, pseudonymizeValue(item, pseudonymizeText)]),
|
||||
);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function minimizeQuestion(frage: FrageMitAntwort, pseudonymizeText?: (text: string) => string): LlmFrage {
|
||||
const map = pseudonymizeText ?? ((text: string) => text);
|
||||
return {
|
||||
id: frage.id,
|
||||
text: map(frage.text),
|
||||
antwort: frage.antwort ? map(frage.antwort) : frage.antwort,
|
||||
confidence: frage.confidence,
|
||||
abgedeckt: frage.abgedeckt,
|
||||
};
|
||||
}
|
||||
|
||||
function minimizeAnswer(entry: AntwortEintrag, pseudonymizeText?: (text: string) => string): LlmAnswer {
|
||||
const map = pseudonymizeText ?? ((text: string) => text);
|
||||
const hasSummary = Boolean(entry.zusammenfassung?.trim());
|
||||
return {
|
||||
label: entry.label ? map(entry.label) : entry.label,
|
||||
antwortFormat: entry.antwortFormat,
|
||||
fragen: entry.fragen?.map((frage) => minimizeQuestion(frage, map)),
|
||||
antwort: !hasSummary && entry.antwort ? map(entry.antwort) : undefined,
|
||||
zusammenfassung: entry.zusammenfassung ? map(entry.zusammenfassung) : entry.zusammenfassung,
|
||||
segmentierung: entry.segmentierung,
|
||||
};
|
||||
}
|
||||
|
||||
export function reversePseudonymsInDossier(
|
||||
dossier: LlmDossier,
|
||||
knownReplacements: KnownReplacement[],
|
||||
): LlmDossier {
|
||||
return {
|
||||
unternehmen: dossier.unternehmen
|
||||
? pseudonymizeValue(dossier.unternehmen, (text) => reversePseudonymsInText(text, knownReplacements)) as Record<string, unknown>
|
||||
: undefined,
|
||||
fragen: dossier.fragen
|
||||
? Object.fromEntries(
|
||||
Object.entries(dossier.fragen).map(([key, entry]) => {
|
||||
const reverse = (text: string) => reversePseudonymsInText(text, knownReplacements);
|
||||
return [
|
||||
key,
|
||||
{
|
||||
...entry,
|
||||
label: entry.label ? reverse(entry.label) : entry.label,
|
||||
zusammenfassung: entry.zusammenfassung ? reverse(entry.zusammenfassung) : entry.zusammenfassung,
|
||||
fragen: entry.fragen?.map((frage) => ({
|
||||
...frage,
|
||||
text: reverse(frage.text),
|
||||
antwort: frage.antwort ? reverse(frage.antwort) : undefined,
|
||||
})) ?? entry.fragen,
|
||||
},
|
||||
];
|
||||
}),
|
||||
)
|
||||
: undefined,
|
||||
kriterium: dossier.kriterium
|
||||
? Object.fromEntries(
|
||||
Object.entries(dossier.kriterium).map(([key, entry]) => {
|
||||
const reverse = (text: string) => reversePseudonymsInText(text, knownReplacements);
|
||||
return [
|
||||
key,
|
||||
{
|
||||
...entry,
|
||||
label: entry.label ? reverse(entry.label) : entry.label,
|
||||
zusammenfassung: entry.zusammenfassung ? reverse(entry.zusammenfassung) : entry.zusammenfassung,
|
||||
fragen: entry.fragen?.map((frage) => ({
|
||||
...frage,
|
||||
text: reverse(frage.text),
|
||||
antwort: frage.antwort ? reverse(frage.antwort) : undefined,
|
||||
})) ?? entry.fragen,
|
||||
},
|
||||
];
|
||||
}),
|
||||
)
|
||||
: undefined,
|
||||
};
|
||||
}
|
||||
|
||||
export function buildLlmDossier(data: SummaryData, pseudonymizeText?: (text: string) => string): LlmDossier {
|
||||
const map = pseudonymizeText ?? ((text: string) => text);
|
||||
const unternehmen = isRecord(data.unternehmen)
|
||||
? Object.fromEntries(
|
||||
COMPANY_FACT_ALLOWLIST
|
||||
.filter((key) => key in data.unternehmen!)
|
||||
.map((key) => [key, pseudonymizeValue(data.unternehmen![key], map)]),
|
||||
)
|
||||
: undefined;
|
||||
|
||||
return {
|
||||
unternehmen,
|
||||
fragen: data.fragen
|
||||
? Object.fromEntries(Object.entries(data.fragen).map(([key, entry]) => [key, minimizeAnswer(entry, map)]))
|
||||
: undefined,
|
||||
kriterium: data.kriterium
|
||||
? Object.fromEntries(Object.entries(data.kriterium).map(([key, entry]) => [key, minimizeAnswer(entry, map)]))
|
||||
: undefined,
|
||||
};
|
||||
}
|
||||
|
||||
export function createPseudonymizer(data: ExtractedData): Pseudonymizer {
|
||||
const companyName = getCompanyName(data);
|
||||
const known = collectKnownReplacements(data, companyName);
|
||||
const seen = new Map<string, PseudonymReplacement>();
|
||||
|
||||
function pseudonymizeText(text: string): string {
|
||||
let output = text;
|
||||
for (const replacement of known) {
|
||||
const before = output;
|
||||
output = output.replace(new RegExp(escapeRegExp(replacement.value), "gi"), replacement.placeholder);
|
||||
if (output !== before) {
|
||||
recordReplacement(seen, replacement.type, replacement.placeholder);
|
||||
}
|
||||
}
|
||||
return pseudonymizeUnknownPatterns(output, seen);
|
||||
}
|
||||
|
||||
function pseudonymizeEntry<T extends AntwortEintrag>(entry: T): T {
|
||||
return {
|
||||
...entry,
|
||||
label: entry.label ? pseudonymizeText(entry.label) : entry.label,
|
||||
fragen: entry.fragen?.map((frage) => minimizeQuestion(frage, pseudonymizeText)),
|
||||
antwort: entry.antwort ? pseudonymizeText(entry.antwort) : entry.antwort,
|
||||
zusammenfassung: entry.zusammenfassung ? pseudonymizeText(entry.zusammenfassung) : entry.zusammenfassung,
|
||||
} as T;
|
||||
}
|
||||
|
||||
function audit(): PrivacyAudit {
|
||||
return {
|
||||
mode: "minimized+pseudonymized",
|
||||
companyName,
|
||||
removedFields: REMOVED_FIELDS,
|
||||
replacements: Array.from(seen.values()),
|
||||
};
|
||||
}
|
||||
|
||||
function pseudonymizeDossier(dossier: LlmDossier): PreparedLlmInput {
|
||||
const safe = pseudonymizeValue(dossier, pseudonymizeText) as LlmDossier;
|
||||
return { safe, audit: audit() };
|
||||
}
|
||||
|
||||
function reversePseudonyms(text: string): string {
|
||||
return reversePseudonymsInText(text, known);
|
||||
}
|
||||
|
||||
return { pseudonymizeText, reversePseudonyms, pseudonymizeEntry, pseudonymizeDossier, audit };
|
||||
}
|
||||
Reference in New Issue
Block a user