91e5f44cc8
- Implemented MemoryStore class for managing concepts with YAML frontmatter. - Added methods for creating, reading, updating, and deleting concepts. - Introduced locking mechanism for concurrent access. - Developed search and hinting capabilities for concept retrieval. - Created tests for access control and store functionality. - Added long-term memory skill documentation and operational guidelines. - Included package.json and package-lock.json for dependency management.
696 lines
28 KiB
JavaScript
696 lines
28 KiB
JavaScript
import { open, mkdir, readFile, readdir, rename, rm, stat, unlink, writeFile } from "node:fs/promises";
|
|
import { randomUUID } from "node:crypto";
|
|
import { basename, dirname, join, posix, relative, resolve, sep } from "node:path";
|
|
import YAML from "yaml";
|
|
|
|
const RESERVED = new Set(["index", "log"]);
|
|
const INDEX_START = "<!-- kimiko-memory:index:start -->";
|
|
const INDEX_END = "<!-- kimiko-memory:index:end -->";
|
|
const LOCAL_REQUIRED_KEYS = ["type", "timestamp"];
|
|
|
|
function sleep(ms) {
|
|
return new Promise((resolvePromise) => setTimeout(resolvePromise, ms));
|
|
}
|
|
|
|
function normalizeText(value) {
|
|
return value.replace(/^\uFEFF/, "").replace(/\r\n?/g, "\n");
|
|
}
|
|
|
|
function normalizeBody(value) {
|
|
const normalized = normalizeText(value).trim();
|
|
return normalized ? `${normalized}\n` : "";
|
|
}
|
|
|
|
function isPlainObject(value) {
|
|
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
}
|
|
|
|
function stableValue(value) {
|
|
if (Array.isArray(value)) return value.map(stableValue);
|
|
if (!isPlainObject(value)) return value;
|
|
return Object.fromEntries(Object.keys(value).sort().map((key) => [key, stableValue(value[key])]));
|
|
}
|
|
|
|
function fingerprint(metadata, body) {
|
|
const meaningful = { ...metadata };
|
|
delete meaningful.timestamp;
|
|
return JSON.stringify({ metadata: stableValue(meaningful), body: normalizeBody(body) });
|
|
}
|
|
|
|
function validTimestamp(value) {
|
|
return typeof value === "string" && /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,3})?Z$/.test(value)
|
|
&& Number.isFinite(Date.parse(value));
|
|
}
|
|
|
|
function parseFrontmatter(raw) {
|
|
const text = normalizeText(raw);
|
|
if (!text.startsWith("---\n")) {
|
|
throw new Error("missing YAML frontmatter");
|
|
}
|
|
|
|
const lines = text.split("\n");
|
|
let closing = -1;
|
|
for (let index = 1; index < lines.length; index += 1) {
|
|
if (lines[index] === "---" || lines[index] === "...") {
|
|
closing = index;
|
|
break;
|
|
}
|
|
}
|
|
if (closing < 0) throw new Error("unterminated YAML frontmatter");
|
|
|
|
let metadata;
|
|
try {
|
|
metadata = YAML.parse(lines.slice(1, closing).join("\n")) ?? {};
|
|
} catch (error) {
|
|
throw new Error(`invalid YAML frontmatter: ${error instanceof Error ? error.message : String(error)}`);
|
|
}
|
|
if (!isPlainObject(metadata)) throw new Error("frontmatter must be a YAML mapping");
|
|
|
|
return { metadata, body: normalizeBody(lines.slice(closing + 1).join("\n")) };
|
|
}
|
|
|
|
function serializeConcept(metadata, body) {
|
|
const yaml = YAML.stringify(metadata, { lineWidth: 0 }).trimEnd();
|
|
return `---\n${yaml}\n---\n\n${normalizeBody(body)}`;
|
|
}
|
|
|
|
function extractRecoverableBody(raw) {
|
|
const text = normalizeText(raw);
|
|
if (!text.startsWith("---\n")) return normalizeBody(text);
|
|
const match = text.match(/^---\n[\s\S]*?\n(?:---|\.\.\.)\n?/);
|
|
if (!match) return normalizeBody(text);
|
|
return normalizeBody(text.slice(match[0].length));
|
|
}
|
|
|
|
function titleFromId(id) {
|
|
const leaf = id.split("/").at(-1) ?? id;
|
|
return leaf
|
|
.split(/[-_.]+/)
|
|
.filter(Boolean)
|
|
.map((part) => part.charAt(0).toUpperCase() + part.slice(1))
|
|
.join(" ");
|
|
}
|
|
|
|
function normalizeTags(tags) {
|
|
if (tags === undefined) return undefined;
|
|
const result = [];
|
|
const seen = new Set();
|
|
for (const raw of tags) {
|
|
const tag = raw.trim();
|
|
if (!tag || seen.has(tag)) continue;
|
|
if (tag.length > 64) throw new Error(`tag exceeds 64 characters: ${tag.slice(0, 20)}…`);
|
|
seen.add(tag);
|
|
result.push(tag);
|
|
}
|
|
if (result.length > 20) throw new Error("at most 20 tags are allowed");
|
|
return result;
|
|
}
|
|
|
|
function assertShortSingleLine(name, value, max, required = false) {
|
|
if (value === undefined) {
|
|
if (required) throw new Error(`${name} is required`);
|
|
return;
|
|
}
|
|
if (!value.trim()) throw new Error(`${name} must not be empty`);
|
|
if (value.length > max) throw new Error(`${name} exceeds ${max} characters`);
|
|
if (/[\r\n]/.test(value)) throw new Error(`${name} must be one line`);
|
|
}
|
|
|
|
function escapeMarkdown(value) {
|
|
return value.replace(/([\\[\]])/g, "\\$1");
|
|
}
|
|
|
|
function replaceGeneratedSection(text, section) {
|
|
const start = text.indexOf(INDEX_START);
|
|
const end = text.indexOf(INDEX_END);
|
|
if (start >= 0 && end >= start) {
|
|
return `${text.slice(0, start).trimEnd()}\n\n${section}\n${text.slice(end + INDEX_END.length).trimStart()}`.trimEnd() + "\n";
|
|
}
|
|
return `${text.trimEnd()}\n\n${section}\n`;
|
|
}
|
|
|
|
const SEARCH_STOP_WORDS = new Set([
|
|
"about", "after", "again", "also", "been", "before", "being", "could", "does", "from", "have", "into",
|
|
"just", "know", "like", "more", "most", "much", "need", "only", "other", "should", "some", "such", "than",
|
|
"that", "their", "them", "then", "there", "these", "they", "this", "those", "through", "user", "very", "want",
|
|
"what", "when", "where", "which", "while", "with", "would", "your", "youre", "remember", "memory",
|
|
]);
|
|
const HINT_GENERIC_TERMS = new Set(["decision", "fact", "person", "preference", "project", "remember", "memory"]);
|
|
|
|
function tokenize(text) {
|
|
return [...text.toLocaleLowerCase().matchAll(/[\p{L}\p{N}]+/gu)]
|
|
.map((match) => match[0])
|
|
.filter((term) => term.length > 2 && !SEARCH_STOP_WORDS.has(term));
|
|
}
|
|
|
|
export class MemoryStore {
|
|
constructor(root, options = {}) {
|
|
this.root = resolve(root);
|
|
this.clock = options.clock ?? (() => new Date());
|
|
this.afterWriteAttempt = options.afterWriteAttempt;
|
|
this.lockPath = join(this.root, ".kimiko-memory.lock");
|
|
}
|
|
|
|
normalizeId(input) {
|
|
if (typeof input !== "string") throw new Error("concept id must be a string");
|
|
let id = input.trim().replace(/^@/, "");
|
|
if (id.endsWith(".md")) id = id.slice(0, -3);
|
|
if (!id || id.length > 240) throw new Error("concept id must contain 1-240 characters");
|
|
if (id.startsWith("/") || id.includes("\\") || id.split("/").some((part) => part === "" || part === "." || part === "..")) {
|
|
throw new Error("concept id must be a safe bundle-relative path");
|
|
}
|
|
if (posix.normalize(id) !== id || !id.split("/").every((part) => /^[a-z0-9][a-z0-9._-]*$/.test(part))) {
|
|
throw new Error("concept id segments may contain lowercase letters, numbers, dots, underscores, and hyphens");
|
|
}
|
|
if (RESERVED.has(id.split("/").at(-1))) throw new Error("index and log are reserved filenames");
|
|
return id;
|
|
}
|
|
|
|
pathForId(id) {
|
|
const normalized = this.normalizeId(id);
|
|
const path = resolve(this.root, ...normalized.split("/")) + ".md";
|
|
if (relative(this.root, path).startsWith(`..${sep}`)) throw new Error("concept path escapes bundle root");
|
|
return { id: normalized, path };
|
|
}
|
|
|
|
async initialize() {
|
|
await mkdir(this.root, { recursive: true });
|
|
return this.withLock(() => this.auditUnlocked({ repair: true }));
|
|
}
|
|
|
|
async withLock(task) {
|
|
await mkdir(this.root, { recursive: true });
|
|
let handle;
|
|
for (let attempt = 0; attempt < 20; attempt += 1) {
|
|
try {
|
|
handle = await open(this.lockPath, "wx", 0o600);
|
|
await handle.writeFile(`${process.pid}\n${this.clock().toISOString()}\n`, "utf8");
|
|
break;
|
|
} catch (error) {
|
|
if (error?.code !== "EEXIST") throw error;
|
|
try {
|
|
const info = await stat(this.lockPath);
|
|
if (Date.now() - info.mtimeMs > 30_000) await unlink(this.lockPath);
|
|
} catch (statError) {
|
|
if (statError?.code !== "ENOENT") throw statError;
|
|
}
|
|
await sleep(Math.min(50 * (attempt + 1), 250));
|
|
}
|
|
}
|
|
if (!handle) throw new Error("memory bundle is locked by another process");
|
|
|
|
try {
|
|
return await task();
|
|
} finally {
|
|
await handle.close().catch(() => {});
|
|
await unlink(this.lockPath).catch(() => {});
|
|
}
|
|
}
|
|
|
|
async atomicWrite(path, content) {
|
|
await mkdir(dirname(path), { recursive: true });
|
|
const temporary = join(dirname(path), `.${basename(path)}.${process.pid}.${randomUUID()}.tmp`);
|
|
const handle = await open(temporary, "wx", 0o600);
|
|
try {
|
|
await handle.writeFile(content, "utf8");
|
|
await handle.sync();
|
|
} finally {
|
|
await handle.close();
|
|
}
|
|
try {
|
|
await rename(temporary, path);
|
|
} catch (error) {
|
|
await rm(temporary, { force: true });
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
async verifiedWrite(path, content, verify, kind = "file") {
|
|
let lastError;
|
|
for (let attempt = 1; attempt <= 2; attempt += 1) {
|
|
try {
|
|
await this.atomicWrite(path, content);
|
|
await this.afterWriteAttempt?.({ path, attempt, kind });
|
|
const persisted = await readFile(path, "utf8");
|
|
await verify(persisted);
|
|
return attempt;
|
|
} catch (error) {
|
|
lastError = error;
|
|
}
|
|
}
|
|
throw new Error(`${kind} verification failed after deterministic retry: ${lastError instanceof Error ? lastError.message : String(lastError)}`);
|
|
}
|
|
|
|
async backup(path, raw) {
|
|
const rel = relative(this.root, path).split(sep).join("/");
|
|
const stamp = this.clock().toISOString().replace(/[:.]/g, "-");
|
|
const backupPath = join(this.root, ".recovery", `${rel}.${stamp}.bak`);
|
|
await mkdir(dirname(backupPath), { recursive: true });
|
|
await writeFile(backupPath, raw, { encoding: "utf8", mode: 0o600 });
|
|
return relative(this.root, backupPath).split(sep).join("/");
|
|
}
|
|
|
|
async conceptFiles(directory = this.root) {
|
|
const found = [];
|
|
let entries = [];
|
|
try {
|
|
entries = await readdir(directory, { withFileTypes: true });
|
|
} catch (error) {
|
|
if (error?.code === "ENOENT") return found;
|
|
throw error;
|
|
}
|
|
for (const entry of entries) {
|
|
if (entry.name.startsWith(".")) continue;
|
|
const path = join(directory, entry.name);
|
|
if (entry.isDirectory()) found.push(...await this.conceptFiles(path));
|
|
else if (entry.isFile() && entry.name.endsWith(".md") && !["index.md", "log.md"].includes(entry.name)) found.push(path);
|
|
}
|
|
return found.sort();
|
|
}
|
|
|
|
async bundleDirectories(directory = this.root) {
|
|
const found = [directory];
|
|
let entries = [];
|
|
try {
|
|
entries = await readdir(directory, { withFileTypes: true });
|
|
} catch (error) {
|
|
if (error?.code === "ENOENT") return found;
|
|
throw error;
|
|
}
|
|
for (const entry of entries) {
|
|
if (entry.isDirectory() && !entry.name.startsWith(".")) {
|
|
found.push(...await this.bundleDirectories(join(directory, entry.name)));
|
|
}
|
|
}
|
|
return found;
|
|
}
|
|
|
|
async inspect(path) {
|
|
const raw = await readFile(path, "utf8");
|
|
const { metadata, body } = parseFrontmatter(raw);
|
|
const errors = [];
|
|
if (typeof metadata.type !== "string" || !metadata.type.trim()) errors.push("missing non-empty type");
|
|
if (!validTimestamp(metadata.timestamp)) errors.push("missing or invalid ISO-8601 UTC timestamp (local producer profile)");
|
|
return { raw, metadata, body, errors };
|
|
}
|
|
|
|
async search(query, limit = 8) {
|
|
const boundedLimit = Math.max(1, Math.min(20, limit));
|
|
const terms = [...new Set(tokenize(query))];
|
|
const normalizedQuery = query.trim().toLocaleLowerCase();
|
|
const results = [];
|
|
const warnings = [];
|
|
|
|
for (const path of await this.conceptFiles()) {
|
|
const id = relative(this.root, path).split(sep).join("/").slice(0, -3);
|
|
try {
|
|
const { metadata, body, errors } = await this.inspect(path);
|
|
if (errors.length) warnings.push(`${id}: ${errors.join(", ")}`);
|
|
const title = typeof metadata.title === "string" ? metadata.title : titleFromId(id);
|
|
const description = typeof metadata.description === "string" ? metadata.description : "";
|
|
const tags = Array.isArray(metadata.tags) ? metadata.tags.filter((tag) => typeof tag === "string") : [];
|
|
const weighted = [
|
|
["id", id, 8], ["title", title, 7], ["description", description, 5], ["tags", tags.join(" "), 5],
|
|
["type", String(metadata.type ?? ""), 3], ["body", body, 1],
|
|
];
|
|
const matchedFields = new Set();
|
|
const metadataTerms = new Set();
|
|
let score = terms.length === 0 ? 0 : terms.reduce((total, term) => total + weighted.reduce(
|
|
(subtotal, [field, value, weight]) => {
|
|
if (!String(value).toLocaleLowerCase().includes(term)) return subtotal;
|
|
matchedFields.add(String(field));
|
|
if (field !== "body") metadataTerms.add(term);
|
|
return subtotal + Number(weight);
|
|
}, 0,
|
|
), 0);
|
|
if (normalizedQuery && weighted.some(([, value]) => String(value).toLocaleLowerCase().includes(normalizedQuery))) score += 10;
|
|
if (terms.length > 0 && score === 0) continue;
|
|
results.push({
|
|
id,
|
|
type: String(metadata.type ?? "Unknown"),
|
|
title,
|
|
description,
|
|
tags,
|
|
timestamp: validTimestamp(metadata.timestamp) ? metadata.timestamp : undefined,
|
|
score,
|
|
matches: { fields: [...matchedFields], metadataTerms: [...metadataTerms] },
|
|
});
|
|
} catch (error) {
|
|
warnings.push(`${id}: ${error instanceof Error ? error.message : String(error)}`);
|
|
}
|
|
}
|
|
|
|
results.sort((left, right) => right.score - left.score
|
|
|| String(right.timestamp ?? "").localeCompare(String(left.timestamp ?? ""))
|
|
|| left.id.localeCompare(right.id));
|
|
return { results: results.slice(0, boundedLimit), warnings };
|
|
}
|
|
|
|
async hints(prompt, limit = 2) {
|
|
const { results } = await this.search(prompt.slice(0, 2_000), 20);
|
|
return results.filter((result) => {
|
|
const distinctive = result.matches.metadataTerms.filter((term) => !HINT_GENERIC_TERMS.has(term));
|
|
if (distinctive.length === 0) return false;
|
|
const direct = result.matches.fields.some((field) => ["id", "title", "tags"].includes(field));
|
|
return direct || (distinctive.length >= 2 && result.matches.fields.includes("description"));
|
|
}).slice(0, Math.max(1, Math.min(3, limit)));
|
|
}
|
|
|
|
async exists(id) {
|
|
const { path } = this.pathForId(id);
|
|
try {
|
|
await stat(path);
|
|
return true;
|
|
} catch (error) {
|
|
if (error?.code === "ENOENT") return false;
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
async read(ids, maxCharacters = 45_000) {
|
|
const unique = [...new Set(ids.map((id) => this.normalizeId(id)))];
|
|
if (unique.length > 10) throw new Error("at most 10 concepts may be read at once");
|
|
const documents = [];
|
|
let text = "";
|
|
let truncated = false;
|
|
|
|
for (const id of unique) {
|
|
const { path } = this.pathForId(id);
|
|
let inspected;
|
|
try {
|
|
inspected = await this.inspect(path);
|
|
} catch (error) {
|
|
if (error?.code === "ENOENT") throw new Error(`memory concept not found: ${id}`);
|
|
throw error;
|
|
}
|
|
if (inspected.errors.length) throw new Error(`${id} is invalid; run /memory-check --repair`);
|
|
const heading = `## ${id}\n\nType: ${inspected.metadata.type}\nTimestamp: ${inspected.metadata.timestamp}\n\n`;
|
|
const block = `${heading}${inspected.body}`;
|
|
if (text.length + block.length > maxCharacters) {
|
|
const remaining = Math.max(0, maxCharacters - text.length);
|
|
text += block.slice(0, remaining);
|
|
truncated = true;
|
|
break;
|
|
}
|
|
text += `${block}\n`;
|
|
documents.push({ id, metadata: inspected.metadata });
|
|
}
|
|
if (truncated) text += "\n[Memory output truncated; read fewer concept IDs.]";
|
|
return { text: text.trimEnd(), documents, truncated };
|
|
}
|
|
|
|
async upsert(input) {
|
|
return this.withLock(async () => {
|
|
const { id, path } = this.pathForId(input.id);
|
|
assertShortSingleLine("type", input.type, 100, true);
|
|
assertShortSingleLine("title", input.title, 200);
|
|
assertShortSingleLine("description", input.description, 500);
|
|
if (typeof input.body !== "string") throw new Error("body is required");
|
|
if (input.body.length > 100_000) throw new Error("body exceeds 100,000 characters");
|
|
const tags = normalizeTags(input.tags);
|
|
|
|
let existing;
|
|
let existingRaw;
|
|
try {
|
|
existingRaw = await readFile(path, "utf8");
|
|
existing = parseFrontmatter(existingRaw);
|
|
} catch (error) {
|
|
if (error?.code !== "ENOENT" && existingRaw !== undefined) {
|
|
await this.backup(path, existingRaw);
|
|
existing = { metadata: {}, body: extractRecoverableBody(existingRaw) };
|
|
} else if (error?.code !== "ENOENT") {
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
const metadata = { ...(existing?.metadata ?? {}) };
|
|
metadata.type = input.type.trim();
|
|
if (input.title !== undefined) metadata.title = input.title.trim();
|
|
if (input.description !== undefined) metadata.description = input.description.trim();
|
|
if (tags !== undefined) metadata.tags = tags;
|
|
const body = normalizeBody(input.body);
|
|
const changed = !existing || fingerprint(metadata, body) !== fingerprint(existing.metadata, existing.body);
|
|
|
|
if (!changed && validTimestamp(existing.metadata.timestamp)) {
|
|
return { id, action: "unchanged", timestamp: existing.metadata.timestamp, attempts: 0 };
|
|
}
|
|
|
|
if (changed) metadata.timestamp = this.clock().toISOString();
|
|
else {
|
|
const info = await stat(path);
|
|
metadata.timestamp = info.mtime.toISOString();
|
|
}
|
|
const serialized = serializeConcept(metadata, body);
|
|
const expectedFingerprint = fingerprint(metadata, body);
|
|
const attempts = await this.verifiedWrite(path, serialized, (persisted) => {
|
|
const parsed = parseFrontmatter(persisted);
|
|
if (parsed.metadata.timestamp !== metadata.timestamp || fingerprint(parsed.metadata, parsed.body) !== expectedFingerprint) {
|
|
throw new Error("persisted concept differs from requested concept");
|
|
}
|
|
if (!LOCAL_REQUIRED_KEYS.every((key) => parsed.metadata[key])) throw new Error("persisted concept lacks required producer metadata");
|
|
}, "concept");
|
|
|
|
await this.updateIndexes();
|
|
await this.appendLog(changed && existing ? "Update" : existing ? "Repair" : "Creation", id, metadata.title ?? titleFromId(id));
|
|
return { id, action: existing ? (changed ? "updated" : "repaired") : "created", timestamp: metadata.timestamp, attempts };
|
|
});
|
|
}
|
|
|
|
async forget(input) {
|
|
return this.withLock(async () => {
|
|
const { id, path } = this.pathForId(input.id);
|
|
if (input.confirm !== id) throw new Error("confirm must exactly match the concept id");
|
|
let title = titleFromId(id);
|
|
try {
|
|
const inspected = await this.inspect(path);
|
|
if (typeof inspected.metadata.title === "string") title = inspected.metadata.title;
|
|
} catch (error) {
|
|
if (error?.code === "ENOENT") throw new Error(`memory concept not found: ${id}`);
|
|
}
|
|
await unlink(path);
|
|
|
|
const recoveryBase = join(this.root, ".recovery", `${id}.md`);
|
|
const recoveryDirectory = dirname(recoveryBase);
|
|
const recoveryPrefix = `${basename(recoveryBase)}.`;
|
|
let removedBackups = 0;
|
|
try {
|
|
for (const entry of await readdir(recoveryDirectory, { withFileTypes: true })) {
|
|
if (entry.isFile() && entry.name.startsWith(recoveryPrefix) && entry.name.endsWith(".bak")) {
|
|
await unlink(join(recoveryDirectory, entry.name));
|
|
removedBackups += 1;
|
|
}
|
|
}
|
|
} catch (error) {
|
|
if (error?.code !== "ENOENT") throw error;
|
|
}
|
|
|
|
await this.redactLogReferences(id);
|
|
await this.updateIndexes();
|
|
await this.appendLog("Deletion", id, title);
|
|
return { id, action: "deleted", removedBackups };
|
|
});
|
|
}
|
|
|
|
async updateIndexes() {
|
|
const files = await this.conceptFiles();
|
|
const directories = new Set(await this.bundleDirectories());
|
|
for (const file of files) {
|
|
let current = dirname(file);
|
|
while (current.startsWith(this.root)) {
|
|
directories.add(current);
|
|
if (current === this.root) break;
|
|
current = dirname(current);
|
|
}
|
|
}
|
|
|
|
for (const directory of [...directories].sort((a, b) => b.length - a.length)) {
|
|
const immediateFiles = files.filter((file) => dirname(file) === directory);
|
|
const childDirs = [...new Set(files
|
|
.filter((file) => dirname(file) !== directory && relative(directory, file) && !relative(directory, file).startsWith(".."))
|
|
.map((file) => relative(directory, file).split(sep)[0]))].sort();
|
|
const lines = [INDEX_START, "_Generated inventory; text outside this section is preserved._"];
|
|
if (childDirs.length) {
|
|
lines.push("", "## Directories");
|
|
for (const child of childDirs) {
|
|
const count = files.filter((file) => relative(join(directory, child), file) && !relative(join(directory, child), file).startsWith("..")).length;
|
|
lines.push(`- [${escapeMarkdown(titleFromId(child))}](${child}/) - ${count} concept${count === 1 ? "" : "s"}.`);
|
|
}
|
|
}
|
|
if (immediateFiles.length) {
|
|
lines.push("", "## Concepts");
|
|
for (const file of immediateFiles) {
|
|
const id = relative(this.root, file).split(sep).join("/").slice(0, -3);
|
|
let title = titleFromId(id);
|
|
let description = "";
|
|
try {
|
|
const { metadata } = await this.inspect(file);
|
|
if (typeof metadata.title === "string") title = metadata.title;
|
|
if (typeof metadata.description === "string") description = metadata.description;
|
|
} catch {}
|
|
lines.push(`- [${escapeMarkdown(title)}](${basename(file)})${description ? ` - ${description}` : ""}`);
|
|
}
|
|
}
|
|
if (!childDirs.length && !immediateFiles.length) lines.push("", "_No concepts._");
|
|
lines.push(INDEX_END);
|
|
const section = lines.join("\n");
|
|
const indexPath = join(directory, "index.md");
|
|
let current = "";
|
|
try { current = await readFile(indexPath, "utf8"); } catch (error) { if (error?.code !== "ENOENT") throw error; }
|
|
|
|
if (directory === this.root) {
|
|
let body = current;
|
|
let rootMetadata = {};
|
|
if (current.startsWith("---\n")) {
|
|
try {
|
|
const parsed = parseFrontmatter(current);
|
|
rootMetadata = parsed.metadata;
|
|
body = parsed.body;
|
|
} catch {
|
|
await this.backup(indexPath, current);
|
|
body = current;
|
|
}
|
|
}
|
|
rootMetadata.okf_version = "0.1";
|
|
if (!/^#\s+/m.test(body)) body = `# Memory\n\n${body}`;
|
|
current = `---\n${YAML.stringify(rootMetadata, { lineWidth: 0 }).trimEnd()}\n---\n\n${body}`;
|
|
} else if (!/^#\s+/m.test(current)) {
|
|
current = `# ${titleFromId(basename(directory))}\n\n${current}`;
|
|
}
|
|
|
|
const next = replaceGeneratedSection(current, section);
|
|
if (normalizeText(current) !== next) {
|
|
await this.verifiedWrite(indexPath, next, (persisted) => {
|
|
if (!persisted.includes(INDEX_START) || !persisted.includes(INDEX_END) || !/^#\s+/m.test(persisted)) {
|
|
throw new Error("generated index markers or heading missing");
|
|
}
|
|
}, "index");
|
|
}
|
|
}
|
|
}
|
|
|
|
async ensureLog() {
|
|
const path = join(this.root, "log.md");
|
|
let current;
|
|
try {
|
|
current = normalizeText(await readFile(path, "utf8"));
|
|
} catch (error) {
|
|
if (error?.code !== "ENOENT") throw error;
|
|
current = "";
|
|
}
|
|
if (current.startsWith("# Update Log")) return;
|
|
if (current) await this.backup(path, current);
|
|
const next = `# Update Log\n${current ? `\n${current.trim()}\n` : ""}`;
|
|
await this.verifiedWrite(path, next, (persisted) => {
|
|
if (!persisted.startsWith("# Update Log")) throw new Error("log heading was not persisted");
|
|
}, "log repair");
|
|
}
|
|
|
|
async redactLogReferences(id) {
|
|
const path = join(this.root, "log.md");
|
|
let current;
|
|
try {
|
|
current = normalizeText(await readFile(path, "utf8"));
|
|
} catch (error) {
|
|
if (error?.code === "ENOENT") return;
|
|
throw error;
|
|
}
|
|
const target = `](/${id}.md)`;
|
|
const next = current.split("\n").filter((line) => !line.includes(target)).join("\n").replace(/\n{3,}/g, "\n\n").trimEnd() + "\n";
|
|
if (next !== current) {
|
|
await this.verifiedWrite(path, next, (persisted) => {
|
|
if (persisted.includes(target)) throw new Error("deleted concept remains in log");
|
|
}, "log redaction");
|
|
}
|
|
}
|
|
|
|
async appendLog(action, id, title) {
|
|
const path = join(this.root, "log.md");
|
|
let current = "# Update Log\n";
|
|
try { current = normalizeText(await readFile(path, "utf8")); } catch (error) { if (error?.code !== "ENOENT") throw error; }
|
|
if (!current.startsWith("# Update Log")) {
|
|
await this.backup(path, current);
|
|
current = `# Update Log\n\n${current.trim()}\n`;
|
|
}
|
|
const date = this.clock().toISOString().slice(0, 10);
|
|
const entry = action === "Deletion"
|
|
? "- **Deletion**: Removed a concept."
|
|
: `- **${action}**: ${action === "Creation" ? "Added" : "Changed"} [${escapeMarkdown(String(title))}](/${id}.md).`;
|
|
if (current.includes(entry)) return;
|
|
const heading = `## ${date}`;
|
|
let next;
|
|
const headingAt = current.indexOf(heading);
|
|
if (headingAt >= 0) {
|
|
const insertion = headingAt + heading.length;
|
|
next = `${current.slice(0, insertion)}\n\n${entry}${current.slice(insertion)}`;
|
|
} else {
|
|
const firstLineEnd = current.indexOf("\n");
|
|
next = `${current.slice(0, firstLineEnd + 1)}\n${heading}\n\n${entry}\n${current.slice(firstLineEnd + 1).trimStart()}`;
|
|
}
|
|
next = next.trimEnd() + "\n";
|
|
await this.verifiedWrite(path, next, (persisted) => {
|
|
if (!persisted.startsWith("# Update Log") || !persisted.includes(entry)) throw new Error("log entry was not persisted");
|
|
}, "log");
|
|
}
|
|
|
|
async audit(options = {}) {
|
|
return this.withLock(() => this.auditUnlocked(options));
|
|
}
|
|
|
|
async auditUnlocked({ repair = false } = {}) {
|
|
const issues = [];
|
|
let repaired = 0;
|
|
for (const path of await this.conceptFiles()) {
|
|
const id = relative(this.root, path).split(sep).join("/").slice(0, -3);
|
|
let raw;
|
|
try {
|
|
raw = await readFile(path, "utf8");
|
|
const parsed = parseFrontmatter(raw);
|
|
const errors = [];
|
|
if (typeof parsed.metadata.type !== "string" || !parsed.metadata.type.trim()) errors.push("missing type");
|
|
if (!validTimestamp(parsed.metadata.timestamp)) errors.push("missing/invalid timestamp");
|
|
if (!errors.length) continue;
|
|
issues.push({ id, errors, repaired: repair });
|
|
if (!repair) continue;
|
|
const info = await stat(path);
|
|
const metadata = { ...parsed.metadata };
|
|
if (typeof metadata.type !== "string" || !metadata.type.trim()) metadata.type = "Recovered Concept";
|
|
if (!validTimestamp(metadata.timestamp)) metadata.timestamp = info.mtime.toISOString();
|
|
const content = serializeConcept(metadata, parsed.body);
|
|
await this.verifiedWrite(path, content, (persisted) => {
|
|
const checked = parseFrontmatter(persisted);
|
|
if (!checked.metadata.type || !validTimestamp(checked.metadata.timestamp)) throw new Error("safe repair did not restore required metadata");
|
|
}, "concept repair");
|
|
repaired += 1;
|
|
} catch (error) {
|
|
const message = error instanceof Error ? error.message : String(error);
|
|
issues.push({ id, errors: [message], repaired: repair });
|
|
if (!repair || raw === undefined) continue;
|
|
const info = await stat(path);
|
|
const backup = await this.backup(path, raw);
|
|
const metadata = {
|
|
type: "Recovered Concept",
|
|
title: titleFromId(id),
|
|
description: "Recovered deterministically from an invalid concept document; review recommended.",
|
|
timestamp: info.mtime.toISOString(),
|
|
repair_backup: backup,
|
|
};
|
|
const content = serializeConcept(metadata, extractRecoverableBody(raw));
|
|
await this.verifiedWrite(path, content, (persisted) => {
|
|
const checked = parseFrontmatter(persisted);
|
|
if (!checked.metadata.type || !validTimestamp(checked.metadata.timestamp)) throw new Error("recovery did not restore required metadata");
|
|
}, "concept recovery");
|
|
repaired += 1;
|
|
}
|
|
}
|
|
|
|
if (repair) {
|
|
await this.updateIndexes();
|
|
await this.ensureLog();
|
|
}
|
|
const files = await this.conceptFiles();
|
|
return { concepts: files.length, issues, repaired, valid: issues.length === 0 || issues.every((issue) => issue.repaired) };
|
|
}
|
|
}
|
|
|
|
export { parseFrontmatter, serializeConcept, validTimestamp };
|