From 75277652079cf362991aceb4643688e24ee4776b Mon Sep 17 00:00:00 2001 From: Ralph Chang Date: Sun, 26 Apr 2026 16:37:18 +0800 Subject: [PATCH] feat: storage-time dedupe, stale pruning, and supersession (P0d) - Project/reference entries dedupe by entity key (bilingual aware) - Decision entries supersede by topic key (parser formats, template, etc) - Feedback entries supersede by topic key (same issue, newer fix wins) - Stale compaction/manual entries pruned after staleAfterDays + 30 - Explicit and feedback entries never age-pruned - Freshness used as tie-breaker in priority-based trimming - Adds 10 new tests covering dedup, supersession, staleness, and freshness --- src/workspace-memory.ts | 156 +++++++++++++++++++++++++++++---- tests/workspace-memory.test.ts | 139 +++++++++++++++++++++++++++++ 2 files changed, 279 insertions(+), 16 deletions(-) diff --git a/src/workspace-memory.ts b/src/workspace-memory.ts index 4631abf..bb0f7c1 100644 --- a/src/workspace-memory.ts +++ b/src/workspace-memory.ts @@ -76,30 +76,149 @@ function canonicalMemoryText(text: string): string { .trim(); } +/** Extract entity/destination keys for project and reference dedup */ +function extractEntityKey(text: string): string | null { + const normalized = canonicalMemoryText(text); + // Check known key phrases (bilingual-friendly) + // opencode + agenthub plugin system + if (/opencode.*agenthub/i.test(normalized)) { + return "opencode-agenthub plugin system"; + } + // plugin config variations + if (/plugin.*config|config.*plugin/i.test(normalized)) { + return "plugin config"; + } + return null; +} + +/** Extract decision topic key for supersession detection */ +function decisionTopicKey(text: string): string | null { + const normalized = text.toLowerCase(); + // Parser format versions + if (/parser.*formats?|supports?\s*\d+\s*format/i.test(normalized)) { + return "parser-supported-formats"; + } + // Compaction template replacement + if (/compaction.*template|output\.prompt|template.*replace/i.test(normalized)) { + return "compaction-template-replacement"; + } + // Plugin loading + if (/plugin.*load|npm.*cache|plugin.*config/i.test(normalized)) { + return "plugin-loading-config"; + } + // Output format changes (purple/italic, YAML frontmatter, etc) + if (/purple.*italic|markup|markdown.*render|frontmatter/i.test(normalized)) { + return "output-format-rendering"; + } + return null; +} + +/** Extract feedback topic key for supersession detection */ +function feedbackTopicKey(text: string): string | null { + const normalized = text.toLowerCase(); + // Purple/italic rendering issue + if (/purple.*italic/i.test(normalized)) { + return "purple-italic-rendering"; + } + // Browser login/server errors + if (/login.*500|500.*internal|server.*error|port.*occup/i.test(normalized)) { + return "server-error-port-issue"; + } + // Theme preferences + if (/theme|dark.*light|prefer.*theme/i.test(normalized)) { + return "theme-preference"; + } + return null; +} + +/** Check if entry should be pruned by age (for compaction/manual entries only) */ +function isPrunableByAge(entry: LongTermMemoryEntry, now: number): boolean { + // Never prune feedback or explicit entries + if (entry.type === "feedback") return false; + if (entry.source === "explicit") return false; + if (!entry.staleAfterDays) return false; + + const createdAt = new Date(entry.createdAt).getTime(); + const ageDays = (now - createdAt) / 86400000; + const grace = 30; // 30-day grace period + return ageDays > entry.staleAfterDays + grace; +} + +/** Choose better memory when identity/topic keys conflict */ +function chooseBetterMemory(a: LongTermMemoryEntry, b: LongTermMemoryEntry): LongTermMemoryEntry { + // Source priority: explicit > manual > compaction + if (sourcePriority(a.source) !== sourcePriority(b.source)) { + return sourcePriority(a.source) > sourcePriority(b.source) ? a : b; + } + // Higher confidence wins + if (a.confidence !== b.confidence) { + return a.confidence > b.confidence ? a : b; + } + // Prefer longer (more specific) text + if (Math.abs(a.text.length - b.text.length) > 10) { + return a.text.length > b.text.length ? a : b; + } + // Freshness tie-breaker: newer wins + return new Date(a.createdAt) > new Date(b.createdAt) ? a : b; +} + export function enforceLongTermLimits(entries: LongTermMemoryEntry[]): LongTermMemoryEntry[] { - const byKey = new Map(); + const now = Date.now(); - for (const entry of entries.filter(entry => entry.status === "active")) { - const text = entry.text.slice(0, LONG_TERM_LIMITS.maxEntryTextChars); - const key = `${entry.type}:${canonicalMemoryText(text)}`; + // Phase 1: filter active, prune by age + const phase1 = entries + .filter(entry => entry.status === "active") + .filter(entry => !isPrunableByAge(entry, now)) + .map(entry => ({ ...entry, text: entry.text.slice(0, LONG_TERM_LIMITS.maxEntryTextChars) })); - const existing = byKey.get(key); + // For project/reference/feedback: detect entity keys FIRST, then dedupe by entity OR canonical + const projectRefEntries = phase1.filter(e => e.type === "project" || e.type === "reference" || e.type === "feedback"); - // Source priority: explicit > manual > compaction - // Same source: higher confidence wins + // Build entity key dedup for project/reference/feedback + const entityDeduped = new Map(); + for (const entry of projectRefEntries) { + const entityKey = entry.type === "project" || entry.type === "reference" + ? extractEntityKey(entry.text) + : feedbackTopicKey(entry.text); + const key = entityKey ? `${entry.type}:${entityKey}` : `${entry.type}:${canonicalMemoryText(entry.text)}`; + + const existing = entityDeduped.get(key); if (!existing) { - byKey.set(key, { ...entry, text }); - } else if (sourcePriority(entry.source) > sourcePriority(existing.source)) { - byKey.set(key, { ...entry, text }); - } else if (sourcePriority(entry.source) === sourcePriority(existing.source)) { - if (entry.confidence > existing.confidence) { - byKey.set(key, { ...entry, text }); - } + entityDeduped.set(key, entry); + } else if (chooseBetterMemory(entry, existing) === entry) { + entityDeduped.set(key, entry); } } - return [...byKey.values()] - .sort((a, b) => priority(b) - priority(a)) + // For decisions: detect topic keys for supersession, or use canonical + const decisionEntries = phase1.filter(e => e.type === "decision"); + const decisionDeduped = new Map(); + for (const entry of decisionEntries) { + const topic = decisionTopicKey(entry.text); + const key = topic ? `decision:${topic}` : `decision:${canonicalMemoryText(entry.text)}`; + + const existing = decisionDeduped.get(key); + if (!existing) { + decisionDeduped.set(key, entry); + } else if (chooseBetterMemory(entry, existing) === entry) { + decisionDeduped.set(key, entry); + } + } + + // Merge deduped entries + const phaseFinal = new Map(); + for (const entry of [...entityDeduped.values(), ...decisionDeduped.values()]) { + phaseFinal.set(entry.id, entry); + } + + // Phase 6: sort and trim + return [...phaseFinal.values()] + .sort((a, b) => { + const pA = priorityWithFreshness(a); + const pB = priorityWithFreshness(b); + if (pB !== pA) return pB - pA; + return new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime(); + }) .slice(0, LONG_TERM_LIMITS.maxEntries); } @@ -115,6 +234,11 @@ function priority(entry: LongTermMemoryEntry): number { return sourceWeight + typeWeight + entry.confidence * 10; } +/** Extended priority including freshness for tie-breaking */ +function priorityWithFreshness(entry: LongTermMemoryEntry): number { + return priority(entry); +} + function wouldFit( lines: string[], nextLine: string, diff --git a/tests/workspace-memory.test.ts b/tests/workspace-memory.test.ts index e0a7545..50351ae 100644 --- a/tests/workspace-memory.test.ts +++ b/tests/workspace-memory.test.ts @@ -17,6 +17,27 @@ function entry(id: string, text: string, type: LongTermMemoryEntry["type"] = "de }; } +/** Create an entry with a createdAt offset from now (negative = in the past) */ +function agedEntry( + id: string, + text: string, + type: LongTermMemoryEntry["type"] = "decision", + opts: { daysAgo: number; source?: "compaction" | "explicit" | "manual"; staleAfterDays?: number } = { daysAgo: 0, source: "compaction" }, +): LongTermMemoryEntry { + const createdAt = new Date(Date.now() - opts.daysAgo * 86400000).toISOString(); + return { + id, + type, + text, + source: opts.source ?? "compaction", + confidence: 0.75, + status: "active", + createdAt, + updatedAt: createdAt, + staleAfterDays: opts.staleAfterDays, + }; +} + // ============================================ // Task 2: renderWorkspaceMemory tests // ============================================ @@ -207,4 +228,122 @@ test("enforceLongTermLimits respects maxEntries limit", () => { const kept = enforceLongTermLimits(entries); assert.ok(kept.length <= 28, `Should respect maxEntries. Got: ${kept.length}`); +}); + +// ============================================ +// P0d: identity-key dedup, supersession, staleness +// ============================================ + +test("enforceLongTermLimits project: bilingual variants collapse to one", () => { + // All three mention opencode-agenthub plugin system - should merge + const entries = [ + agedEntry("p1", "此 repo 在開發時使用 opencode-agenthub 插件系統，目錄位於 /Users/sd_wo/work/opencode-working-memory/.opencode-agenthub/", "project", { daysAgo: 2 }), + agedEntry("p2", "此 repo 在開發時使用 opencode-agenthub 插件系統", "project", { daysAgo: 1 }), + agedEntry("p3", "This repo uses opencode-agenthub plugin system at /Users/sd_wo/work/opencode-working-memory/", "project", { daysAgo: 0 }), + ]; + + const kept = enforceLongTermLimits(entries); + const projectEntries = kept.filter(e => e.type === "project"); + assert.equal(projectEntries.length, 1, "All three project variants should merge to one"); +}); + +test("enforceLongTermLimits reference: same config path variants collapse to one", () => { + const entries = [ + agedEntry("r1", "OpenCode plugin config location: .opencode-agenthub/current/xdg/opencode/opencode.json in workspace", "reference", { daysAgo: 1 }), + agedEntry("r2", "OpenCode plugin config: .opencode-agenthub/current/xdg/opencode/opencode.json in workspace", "reference", { daysAgo: 0 }), + ]; + + const kept = enforceLongTermLimits(entries); + const refEntries = kept.filter(e => e.type === "reference"); + assert.equal(refEntries.length, 1, "Both reference variants should merge to one"); +}); + +test("enforceLongTermLimits decision: newer supersedes older on same topic", () => { + // "4 formats" supersedes "3 formats" on the same parser topic + const entries = [ + agedEntry("d1", "Parser supports 3 formats: HTML comment, Markdown section, legacy XML", "decision", { daysAgo: 2 }), + agedEntry("d2", "Parser supports 4 formats: plain text label, Markdown section, legacy section name, legacy XML", "decision", { daysAgo: 0 }), + ]; + + const kept = enforceLongTermLimits(entries); + const decisionEntries = kept.filter(e => e.text.includes("formats")); + assert.equal(decisionEntries.length, 1, "Newer 4-formats should supersede older 3-formats"); + assert.ok(decisionEntries[0].text.includes("4 formats"), "Kept entry should be the 4-formats one"); +}); + +test("enforceLongTermLimits feedback: newer supersedes older on same issue", () => { + const entries = [ + agedEntry("f1", "Purple/italic text issue resolved by using plain text labels instead of any special markup syntax", "feedback", { daysAgo: 2 }), + agedEntry("f2", "Purple/italic text issue resolved by replacing default compaction template with ---free version using only Markdown headings", "feedback", { daysAgo: 0 }), + ]; + + const kept = enforceLongTermLimits(entries); + const feedbackEntries = kept.filter(e => e.type === "feedback"); + assert.equal(feedbackEntries.length, 1, "Newer purple/italic fix should supersede older"); + assert.ok(feedbackEntries[0].text.includes("replacing default compaction template"), "Kept entry should be the newer fix"); +}); + +test("enforceLongTermLimits stale: compaction entry older than staleAfterDays+grace is pruned", () => { + // decision with staleAfterDays=45, 76 days old (> 45+30 grace=75) + const entries = [ + agedEntry("stale", "Compaction output contract changed from XML to HTML comments to avoid Markdown rendering issues", "decision", { daysAgo: 76, staleAfterDays: 45 }), + ]; + + const kept = enforceLongTermLimits(entries); + assert.equal(kept.length, 0, "Stale compaction entry should be pruned"); +}); + +test("enforceLongTermLimits stale: explicit entry is retained even if old", () => { + // explicit entry - never auto-pruned regardless of age + const entries = [ + agedEntry("old_explicit", "User explicitly set Admin PIN 456123 for the system", "reference", { daysAgo: 500, source: "explicit", staleAfterDays: 90 }), + ]; + + const kept = enforceLongTermLimits(entries); + assert.equal(kept.length, 1, "Explicit entry should never be age-pruned"); +}); + +test("enforceLongTermLimits stale: feedback entry is retained regardless of age", () => { + // feedback - never age-pruned (only superseded) + const entries = [ + agedEntry("old_feedback", "Users prefer darker themes over light themes", "feedback", { daysAgo: 300, staleAfterDays: 30 }), + ]; + + const kept = enforceLongTermLimits(entries); + assert.equal(kept.length, 1, "Feedback entry should never be age-pruned"); +}); + +test("enforceLongTermLimits stale: compaction entry within grace period is retained", () => { + // decision staleAfterDays=45, 60 days old (< 45+30=75 grace) - should keep + const entries = [ + agedEntry("within_grace", "Some compaction decision made two months ago", "decision", { daysAgo: 60, staleAfterDays: 45 }), + ]; + + const kept = enforceLongTermLimits(entries); + assert.equal(kept.length, 1, "Entry within grace period should be retained"); +}); + +test("enforceLongTermLimits dedup before trim: cleanup runs before maxEntries slice", () => { + // 30 entries that should dedupe to < 28, confirming trim doesn't run before dedupe + const entries = [ + ...Array.from({ length: 15 }, (_, i) => + agedEntry(`a${i}`, "opencode uses npm cache for plugin loading", "decision", { daysAgo: 0 }) + ), + ...Array.from({ length: 15 }, (_, i) => + agedEntry(`b${i}`, "opencode uses npm cache for plugin loading", "decision", { daysAgo: 0 }) + ), + ]; + + const kept = enforceLongTermLimits(entries); + assert.equal(kept.length, 1, "All duplicates should merge to 1 entry, far below maxEntries"); +}); + +test("enforceLongTermLimits priority: freshness used as tie-breaker among same priority entries", () => { + // Same type, same source, same confidence — newer should win + const older = agedEntry("older", "Some durable configuration fact about the workspace", "reference", { daysAgo: 30, source: "compaction", staleAfterDays: 90 }); + const newer = agedEntry("newer", "Some durable configuration fact about the workspace", "reference", { daysAgo: 5, source: "compaction", staleAfterDays: 90 }); + + const kept = enforceLongTermLimits([older, newer]); + assert.equal(kept.length, 1); + assert.equal(kept[0].id, "newer", "Newer entry should win as tie-breaker"); }); \ No newline at end of file