mirror of
https://github.com/sdwolf4103/opencode-working-memory.git
synced 2026-07-17 12:56:42 +02:00
feat: storage-time dedupe, stale pruning, and supersession (P0d)
- Project/reference entries dedupe by entity key (bilingual aware) - Decision entries supersede by topic key (parser formats, template, etc) - Feedback entries supersede by topic key (same issue, newer fix wins) - Stale compaction/manual entries pruned after staleAfterDays + 30 - Explicit and feedback entries never age-pruned - Freshness used as tie-breaker in priority-based trimming - Adds 10 new tests covering dedup, supersession, staleness, and freshness
This commit is contained in:
Ralph Chang
+140
-16
@@ -76,30 +76,149 @@ function canonicalMemoryText(text: string): string {
|
||||
.trim();
|
||||
}
|
||||
|
||||
/** Extract entity/destination keys for project and reference dedup */
|
||||
function extractEntityKey(text: string): string | null {
|
||||
const normalized = canonicalMemoryText(text);
|
||||
// Check known key phrases (bilingual-friendly)
|
||||
// opencode + agenthub plugin system
|
||||
if (/opencode.*agenthub/i.test(normalized)) {
|
||||
return "opencode-agenthub plugin system";
|
||||
}
|
||||
// plugin config variations
|
||||
if (/plugin.*config|config.*plugin/i.test(normalized)) {
|
||||
return "plugin config";
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Extract decision topic key for supersession detection */
|
||||
function decisionTopicKey(text: string): string | null {
|
||||
const normalized = text.toLowerCase();
|
||||
// Parser format versions
|
||||
if (/parser.*formats?|supports?\s*\d+\s*format/i.test(normalized)) {
|
||||
return "parser-supported-formats";
|
||||
}
|
||||
// Compaction template replacement
|
||||
if (/compaction.*template|output\.prompt|template.*replace/i.test(normalized)) {
|
||||
return "compaction-template-replacement";
|
||||
}
|
||||
// Plugin loading
|
||||
if (/plugin.*load|npm.*cache|plugin.*config/i.test(normalized)) {
|
||||
return "plugin-loading-config";
|
||||
}
|
||||
// Output format changes (purple/italic, YAML frontmatter, etc)
|
||||
if (/purple.*italic|markup|markdown.*render|frontmatter/i.test(normalized)) {
|
||||
return "output-format-rendering";
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Extract feedback topic key for supersession detection */
|
||||
function feedbackTopicKey(text: string): string | null {
|
||||
const normalized = text.toLowerCase();
|
||||
// Purple/italic rendering issue
|
||||
if (/purple.*italic/i.test(normalized)) {
|
||||
return "purple-italic-rendering";
|
||||
}
|
||||
// Browser login/server errors
|
||||
if (/login.*500|500.*internal|server.*error|port.*occup/i.test(normalized)) {
|
||||
return "server-error-port-issue";
|
||||
}
|
||||
// Theme preferences
|
||||
if (/theme|dark.*light|prefer.*theme/i.test(normalized)) {
|
||||
return "theme-preference";
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Check if entry should be pruned by age (for compaction/manual entries only) */
|
||||
function isPrunableByAge(entry: LongTermMemoryEntry, now: number): boolean {
|
||||
// Never prune feedback or explicit entries
|
||||
if (entry.type === "feedback") return false;
|
||||
if (entry.source === "explicit") return false;
|
||||
if (!entry.staleAfterDays) return false;
|
||||
|
||||
const createdAt = new Date(entry.createdAt).getTime();
|
||||
const ageDays = (now - createdAt) / 86400000;
|
||||
const grace = 30; // 30-day grace period
|
||||
return ageDays > entry.staleAfterDays + grace;
|
||||
}
|
||||
|
||||
/** Choose better memory when identity/topic keys conflict */
|
||||
function chooseBetterMemory(a: LongTermMemoryEntry, b: LongTermMemoryEntry): LongTermMemoryEntry {
|
||||
// Source priority: explicit > manual > compaction
|
||||
if (sourcePriority(a.source) !== sourcePriority(b.source)) {
|
||||
return sourcePriority(a.source) > sourcePriority(b.source) ? a : b;
|
||||
}
|
||||
// Higher confidence wins
|
||||
if (a.confidence !== b.confidence) {
|
||||
return a.confidence > b.confidence ? a : b;
|
||||
}
|
||||
// Prefer longer (more specific) text
|
||||
if (Math.abs(a.text.length - b.text.length) > 10) {
|
||||
return a.text.length > b.text.length ? a : b;
|
||||
}
|
||||
// Freshness tie-breaker: newer wins
|
||||
return new Date(a.createdAt) > new Date(b.createdAt) ? a : b;
|
||||
}
|
||||
|
||||
export function enforceLongTermLimits(entries: LongTermMemoryEntry[]): LongTermMemoryEntry[] {
|
||||
const byKey = new Map<string, LongTermMemoryEntry>();
|
||||
const now = Date.now();
|
||||
|
||||
for (const entry of entries.filter(entry => entry.status === "active")) {
|
||||
const text = entry.text.slice(0, LONG_TERM_LIMITS.maxEntryTextChars);
|
||||
const key = `${entry.type}:${canonicalMemoryText(text)}`;
|
||||
// Phase 1: filter active, prune by age
|
||||
const phase1 = entries
|
||||
.filter(entry => entry.status === "active")
|
||||
.filter(entry => !isPrunableByAge(entry, now))
|
||||
.map(entry => ({ ...entry, text: entry.text.slice(0, LONG_TERM_LIMITS.maxEntryTextChars) }));
|
||||
|
||||
const existing = byKey.get(key);
|
||||
// For project/reference/feedback: detect entity keys FIRST, then dedupe by entity OR canonical
|
||||
const projectRefEntries = phase1.filter(e => e.type === "project" || e.type === "reference" || e.type === "feedback");
|
||||
|
||||
// Source priority: explicit > manual > compaction
|
||||
// Same source: higher confidence wins
|
||||
// Build entity key dedup for project/reference/feedback
|
||||
const entityDeduped = new Map<string, LongTermMemoryEntry>();
|
||||
for (const entry of projectRefEntries) {
|
||||
const entityKey = entry.type === "project" || entry.type === "reference"
|
||||
? extractEntityKey(entry.text)
|
||||
: feedbackTopicKey(entry.text);
|
||||
const key = entityKey ? `${entry.type}:${entityKey}` : `${entry.type}:${canonicalMemoryText(entry.text)}`;
|
||||
|
||||
const existing = entityDeduped.get(key);
|
||||
if (!existing) {
|
||||
byKey.set(key, { ...entry, text });
|
||||
} else if (sourcePriority(entry.source) > sourcePriority(existing.source)) {
|
||||
byKey.set(key, { ...entry, text });
|
||||
} else if (sourcePriority(entry.source) === sourcePriority(existing.source)) {
|
||||
if (entry.confidence > existing.confidence) {
|
||||
byKey.set(key, { ...entry, text });
|
||||
}
|
||||
entityDeduped.set(key, entry);
|
||||
} else if (chooseBetterMemory(entry, existing) === entry) {
|
||||
entityDeduped.set(key, entry);
|
||||
}
|
||||
}
|
||||
|
||||
return [...byKey.values()]
|
||||
.sort((a, b) => priority(b) - priority(a))
|
||||
// For decisions: detect topic keys for supersession, or use canonical
|
||||
const decisionEntries = phase1.filter(e => e.type === "decision");
|
||||
const decisionDeduped = new Map<string, LongTermMemoryEntry>();
|
||||
for (const entry of decisionEntries) {
|
||||
const topic = decisionTopicKey(entry.text);
|
||||
const key = topic ? `decision:${topic}` : `decision:${canonicalMemoryText(entry.text)}`;
|
||||
|
||||
const existing = decisionDeduped.get(key);
|
||||
if (!existing) {
|
||||
decisionDeduped.set(key, entry);
|
||||
} else if (chooseBetterMemory(entry, existing) === entry) {
|
||||
decisionDeduped.set(key, entry);
|
||||
}
|
||||
}
|
||||
|
||||
// Merge deduped entries
|
||||
const phaseFinal = new Map<string, LongTermMemoryEntry>();
|
||||
for (const entry of [...entityDeduped.values(), ...decisionDeduped.values()]) {
|
||||
phaseFinal.set(entry.id, entry);
|
||||
}
|
||||
|
||||
// Phase 6: sort and trim
|
||||
return [...phaseFinal.values()]
|
||||
.sort((a, b) => {
|
||||
const pA = priorityWithFreshness(a);
|
||||
const pB = priorityWithFreshness(b);
|
||||
if (pB !== pA) return pB - pA;
|
||||
return new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime();
|
||||
})
|
||||
.slice(0, LONG_TERM_LIMITS.maxEntries);
|
||||
}
|
||||
|
||||
@@ -115,6 +234,11 @@ function priority(entry: LongTermMemoryEntry): number {
|
||||
return sourceWeight + typeWeight + entry.confidence * 10;
|
||||
}
|
||||
|
||||
/** Extended priority including freshness for tie-breaking */
|
||||
function priorityWithFreshness(entry: LongTermMemoryEntry): number {
|
||||
return priority(entry);
|
||||
}
|
||||
|
||||
function wouldFit(
|
||||
lines: string[],
|
||||
nextLine: string,
|
||||
|
||||
@@ -17,6 +17,27 @@ function entry(id: string, text: string, type: LongTermMemoryEntry["type"] = "de
|
||||
};
|
||||
}
|
||||
|
||||
/** Create an entry with a createdAt offset from now (negative = in the past) */
|
||||
function agedEntry(
|
||||
id: string,
|
||||
text: string,
|
||||
type: LongTermMemoryEntry["type"] = "decision",
|
||||
opts: { daysAgo: number; source?: "compaction" | "explicit" | "manual"; staleAfterDays?: number } = { daysAgo: 0, source: "compaction" },
|
||||
): LongTermMemoryEntry {
|
||||
const createdAt = new Date(Date.now() - opts.daysAgo * 86400000).toISOString();
|
||||
return {
|
||||
id,
|
||||
type,
|
||||
text,
|
||||
source: opts.source ?? "compaction",
|
||||
confidence: 0.75,
|
||||
status: "active",
|
||||
createdAt,
|
||||
updatedAt: createdAt,
|
||||
staleAfterDays: opts.staleAfterDays,
|
||||
};
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// Task 2: renderWorkspaceMemory tests
|
||||
// ============================================
|
||||
@@ -207,4 +228,122 @@ test("enforceLongTermLimits respects maxEntries limit", () => {
|
||||
|
||||
const kept = enforceLongTermLimits(entries);
|
||||
assert.ok(kept.length <= 28, `Should respect maxEntries. Got: ${kept.length}`);
|
||||
});
|
||||
|
||||
// ============================================
|
||||
// P0d: identity-key dedup, supersession, staleness
|
||||
// ============================================
|
||||
|
||||
test("enforceLongTermLimits project: bilingual variants collapse to one", () => {
|
||||
// All three mention opencode-agenthub plugin system - should merge
|
||||
const entries = [
|
||||
agedEntry("p1", "此 repo 在開發時使用 opencode-agenthub 插件系統，目錄位於 /Users/sd_wo/work/opencode-working-memory/.opencode-agenthub/", "project", { daysAgo: 2 }),
|
||||
agedEntry("p2", "此 repo 在開發時使用 opencode-agenthub 插件系統", "project", { daysAgo: 1 }),
|
||||
agedEntry("p3", "This repo uses opencode-agenthub plugin system at /Users/sd_wo/work/opencode-working-memory/", "project", { daysAgo: 0 }),
|
||||
];
|
||||
|
||||
const kept = enforceLongTermLimits(entries);
|
||||
const projectEntries = kept.filter(e => e.type === "project");
|
||||
assert.equal(projectEntries.length, 1, "All three project variants should merge to one");
|
||||
});
|
||||
|
||||
test("enforceLongTermLimits reference: same config path variants collapse to one", () => {
|
||||
const entries = [
|
||||
agedEntry("r1", "OpenCode plugin config location: .opencode-agenthub/current/xdg/opencode/opencode.json in workspace", "reference", { daysAgo: 1 }),
|
||||
agedEntry("r2", "OpenCode plugin config: .opencode-agenthub/current/xdg/opencode/opencode.json in workspace", "reference", { daysAgo: 0 }),
|
||||
];
|
||||
|
||||
const kept = enforceLongTermLimits(entries);
|
||||
const refEntries = kept.filter(e => e.type === "reference");
|
||||
assert.equal(refEntries.length, 1, "Both reference variants should merge to one");
|
||||
});
|
||||
|
||||
test("enforceLongTermLimits decision: newer supersedes older on same topic", () => {
|
||||
// "4 formats" supersedes "3 formats" on the same parser topic
|
||||
const entries = [
|
||||
agedEntry("d1", "Parser supports 3 formats: HTML comment, Markdown section, legacy XML", "decision", { daysAgo: 2 }),
|
||||
agedEntry("d2", "Parser supports 4 formats: plain text label, Markdown section, legacy section name, legacy XML", "decision", { daysAgo: 0 }),
|
||||
];
|
||||
|
||||
const kept = enforceLongTermLimits(entries);
|
||||
const decisionEntries = kept.filter(e => e.text.includes("formats"));
|
||||
assert.equal(decisionEntries.length, 1, "Newer 4-formats should supersede older 3-formats");
|
||||
assert.ok(decisionEntries[0].text.includes("4 formats"), "Kept entry should be the 4-formats one");
|
||||
});
|
||||
|
||||
test("enforceLongTermLimits feedback: newer supersedes older on same issue", () => {
|
||||
const entries = [
|
||||
agedEntry("f1", "Purple/italic text issue resolved by using plain text labels instead of any special markup syntax", "feedback", { daysAgo: 2 }),
|
||||
agedEntry("f2", "Purple/italic text issue resolved by replacing default compaction template with ---free version using only Markdown headings", "feedback", { daysAgo: 0 }),
|
||||
];
|
||||
|
||||
const kept = enforceLongTermLimits(entries);
|
||||
const feedbackEntries = kept.filter(e => e.type === "feedback");
|
||||
assert.equal(feedbackEntries.length, 1, "Newer purple/italic fix should supersede older");
|
||||
assert.ok(feedbackEntries[0].text.includes("replacing default compaction template"), "Kept entry should be the newer fix");
|
||||
});
|
||||
|
||||
test("enforceLongTermLimits stale: compaction entry older than staleAfterDays+grace is pruned", () => {
|
||||
// decision with staleAfterDays=45, 76 days old (> 45+30 grace=75)
|
||||
const entries = [
|
||||
agedEntry("stale", "Compaction output contract changed from XML to HTML comments to avoid Markdown rendering issues", "decision", { daysAgo: 76, staleAfterDays: 45 }),
|
||||
];
|
||||
|
||||
const kept = enforceLongTermLimits(entries);
|
||||
assert.equal(kept.length, 0, "Stale compaction entry should be pruned");
|
||||
});
|
||||
|
||||
test("enforceLongTermLimits stale: explicit entry is retained even if old", () => {
|
||||
// explicit entry - never auto-pruned regardless of age
|
||||
const entries = [
|
||||
agedEntry("old_explicit", "User explicitly set Admin PIN 456123 for the system", "reference", { daysAgo: 500, source: "explicit", staleAfterDays: 90 }),
|
||||
];
|
||||
|
||||
const kept = enforceLongTermLimits(entries);
|
||||
assert.equal(kept.length, 1, "Explicit entry should never be age-pruned");
|
||||
});
|
||||
|
||||
test("enforceLongTermLimits stale: feedback entry is retained regardless of age", () => {
|
||||
// feedback - never age-pruned (only superseded)
|
||||
const entries = [
|
||||
agedEntry("old_feedback", "Users prefer darker themes over light themes", "feedback", { daysAgo: 300, staleAfterDays: 30 }),
|
||||
];
|
||||
|
||||
const kept = enforceLongTermLimits(entries);
|
||||
assert.equal(kept.length, 1, "Feedback entry should never be age-pruned");
|
||||
});
|
||||
|
||||
test("enforceLongTermLimits stale: compaction entry within grace period is retained", () => {
|
||||
// decision staleAfterDays=45, 60 days old (< 45+30=75 grace) - should keep
|
||||
const entries = [
|
||||
agedEntry("within_grace", "Some compaction decision made two months ago", "decision", { daysAgo: 60, staleAfterDays: 45 }),
|
||||
];
|
||||
|
||||
const kept = enforceLongTermLimits(entries);
|
||||
assert.equal(kept.length, 1, "Entry within grace period should be retained");
|
||||
});
|
||||
|
||||
test("enforceLongTermLimits dedup before trim: cleanup runs before maxEntries slice", () => {
|
||||
// 30 entries that should dedupe to < 28, confirming trim doesn't run before dedupe
|
||||
const entries = [
|
||||
...Array.from({ length: 15 }, (_, i) =>
|
||||
agedEntry(`a${i}`, "opencode uses npm cache for plugin loading", "decision", { daysAgo: 0 })
|
||||
),
|
||||
...Array.from({ length: 15 }, (_, i) =>
|
||||
agedEntry(`b${i}`, "opencode uses npm cache for plugin loading", "decision", { daysAgo: 0 })
|
||||
),
|
||||
];
|
||||
|
||||
const kept = enforceLongTermLimits(entries);
|
||||
assert.equal(kept.length, 1, "All duplicates should merge to 1 entry, far below maxEntries");
|
||||
});
|
||||
|
||||
test("enforceLongTermLimits priority: freshness used as tie-breaker among same priority entries", () => {
|
||||
// Same type, same source, same confidence — newer should win
|
||||
const older = agedEntry("older", "Some durable configuration fact about the workspace", "reference", { daysAgo: 30, source: "compaction", staleAfterDays: 90 });
|
||||
const newer = agedEntry("newer", "Some durable configuration fact about the workspace", "reference", { daysAgo: 5, source: "compaction", staleAfterDays: 90 });
|
||||
|
||||
const kept = enforceLongTermLimits([older, newer]);
|
||||
assert.equal(kept.length, 1);
|
||||
assert.equal(kept[0].id, "newer", "Newer entry should win as tie-breaker");
|
||||
});
|
||||
Reference in New Issue
Block a user