mirror of
https://github.com/sdwolf4103/opencode-working-memory.git
synced 2026-07-17 12:56:42 +02:00
fix(memory): isolate test workspace cleanup
This commit is contained in:
Ralph Chang
+2
-10
@@ -5,6 +5,7 @@ import type { ActiveFile, LongTermMemoryEntry, LongTermType, OpenError } from ".
|
||||
import { LONG_TERM_LIMITS } from "./types.ts";
|
||||
import { assessMemoryQuality } from "./memory-quality.ts";
|
||||
import { extractionRejectionLogPath } from "./paths.ts";
|
||||
import { redactCredentials } from "./redaction.ts";
|
||||
|
||||
function id(prefix: string): string {
|
||||
return `${prefix}_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
|
||||
@@ -248,15 +249,6 @@ async function logExtractionRejection(entry: ExtractionRejectionLogEntry): Promi
|
||||
}
|
||||
}
|
||||
|
||||
function redactSensitiveText(text: string): string {
|
||||
return text
|
||||
.replace(/bearer\s+[a-zA-Z0-9._-]+/gi, "bearer [REDACTED]")
|
||||
.replace(/token[=:]\s*[a-zA-Z0-9._-]+/gi, "token=[REDACTED]")
|
||||
.replace(/password[=:]\s*[a-zA-Z0-9._-]+/gi, "password=[REDACTED]")
|
||||
.replace(/secret[=:]\s*[a-zA-Z0-9._-]+/gi, "secret=[REDACTED]")
|
||||
.replace(/api[-_]?key[=:]\s*[a-zA-Z0-9._-]+/gi, "api_key=[REDACTED]");
|
||||
}
|
||||
|
||||
function shouldAcceptWorkspaceMemoryCandidate(
|
||||
entry: {
|
||||
type: LongTermType;
|
||||
@@ -287,7 +279,7 @@ function shouldAcceptWorkspaceMemoryCandidate(
|
||||
void logExtractionRejection({
|
||||
timestamp: new Date().toISOString(),
|
||||
type: entry.type,
|
||||
text: redactSensitiveText(text),
|
||||
text: redactCredentials(text),
|
||||
reasons: quality.reasons,
|
||||
source: "compaction",
|
||||
});
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
/**
|
||||
* Shared redaction utilities for sensitive credential patterns.
|
||||
* Used by both workspace memory normalization and extraction rejection logging.
|
||||
*/
|
||||
|
||||
// Password labels in multiple languages
|
||||
const PASSWORD_LABELS = /password|passwd|pwd|密碼|密码|パスワード|비밀번호|contraseña|mot de passe|passwort/i;
|
||||
|
||||
// Username labels in multiple languages
|
||||
const USERNAME_LABELS = /username|user name|用戶名|用户名|ユーザー名|사용자명|usuario|utilisateur|benutzer/i;
|
||||
|
||||
// Sensitive key labels
|
||||
const SENSITIVE_LABELS = /api[_-]?key|token|bearer|secret|credential|auth|auth[_-]?key|private[_-]?key/i;
|
||||
|
||||
// Secret value pattern (excludes common delimiters and brackets)
|
||||
const SECRET_VALUE = String.raw`[^` + "`" + String.raw`'",，,\s\[]+`;
|
||||
|
||||
// Prefix patterns for different credential types
|
||||
const PIN_PREFIX = String.raw`(\bPIN\b(?:\s*(?:是|=|:|：)\s*|\s+(?![是=:：])))`;
|
||||
const PASSWORD_PREFIX = String.raw`((?:${PASSWORD_LABELS.source})(?:\s*(?:是|=|:|：)\s*|\s+(?![是=:：])))`;
|
||||
const USERNAME_PREFIX = String.raw`((?:${USERNAME_LABELS.source})(?:\s*(?:是|=|:|：)\s*|\s+(?![是=:：])))`;
|
||||
const SENSITIVE_PREFIX = String.raw`((?:${SENSITIVE_LABELS.source})(?:\s*(?:推|是|=|:|：)\s*|[:：]\s*))`;
|
||||
const BEARER_PREFIX = String.raw`(Bearer\s+)`;
|
||||
|
||||
/**
|
||||
* Redacts sensitive credentials from text.
|
||||
* Handles:
|
||||
* - PINs in multiple formats
|
||||
* - Username/password pairs
|
||||
* - Standalone passwords
|
||||
* - Bearer tokens
|
||||
* - API keys, secrets, credentials, auth tokens, private keys
|
||||
*
|
||||
* Supports multiple languages and delimiters (ASCII and CJK).
|
||||
*/
|
||||
export function redactCredentials(text: string): string {
|
||||
let result = text;
|
||||
|
||||
// 1. PIN
|
||||
result = result.replace(
|
||||
new RegExp(String.raw`${PIN_PREFIX}[\`'"]?(${SECRET_VALUE})`, "gi"),
|
||||
"$1[REDACTED]",
|
||||
);
|
||||
|
||||
// 2. Username+password pair
|
||||
result = result.replace(
|
||||
new RegExp(
|
||||
String.raw`${USERNAME_PREFIX}[\`'"]?(${SECRET_VALUE})((?:，|,)\s*)${PASSWORD_PREFIX}[\`'"]?(${SECRET_VALUE})`,
|
||||
"gi",
|
||||
),
|
||||
"$1[REDACTED]$3$4[REDACTED]",
|
||||
);
|
||||
|
||||
// 3. Standalone password
|
||||
result = result.replace(
|
||||
new RegExp(String.raw`${PASSWORD_PREFIX}[\`'"]?(${SECRET_VALUE})`, "gi"),
|
||||
"$1[REDACTED]",
|
||||
);
|
||||
|
||||
// 4. Bearer tokens (but not "bearer token:" labels)
|
||||
result = result.replace(
|
||||
new RegExp(String.raw`${BEARER_PREFIX}(?!token\s*[:=：])[\`'"]?(${SECRET_VALUE})`, "gi"),
|
||||
"$1[REDACTED]",
|
||||
);
|
||||
|
||||
// 5. Sensitive keys/tokens
|
||||
result = result.replace(
|
||||
new RegExp(String.raw`${SENSITIVE_PREFIX}[\`'"]?(${SECRET_VALUE})`, "gi"),
|
||||
"$1[REDACTED]",
|
||||
);
|
||||
|
||||
return result;
|
||||
}
|
||||
@@ -0,0 +1,282 @@
|
||||
import { existsSync } from "node:fs";
|
||||
import { appendFile, mkdir, readFile, readdir, rename, stat } from "node:fs/promises";
|
||||
import { basename, dirname, join, resolve } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dataHome as defaultDataHome } from "./paths.ts";
|
||||
|
||||
export type WorkspaceCleanupClassification =
|
||||
| "test_temp_definite"
|
||||
| "orphan_unknown"
|
||||
| "live_or_existing"
|
||||
| "invalid_or_unreadable";
|
||||
|
||||
export type WorkspaceCleanupResult = {
|
||||
workspaceKey: string;
|
||||
workspaceDir: string;
|
||||
root?: string;
|
||||
rootExists: boolean;
|
||||
classification: WorkspaceCleanupClassification;
|
||||
reasons: string[];
|
||||
entryCount?: number;
|
||||
migrations?: string[];
|
||||
updatedAt?: string;
|
||||
};
|
||||
|
||||
export type WorkspaceCleanupScanOptions = {
|
||||
dataHome?: string;
|
||||
nowMs?: number;
|
||||
minAgeMs?: number;
|
||||
includeOrphans?: boolean;
|
||||
};
|
||||
|
||||
export type WorkspaceCleanupScan = {
|
||||
results: WorkspaceCleanupResult[];
|
||||
candidates: WorkspaceCleanupResult[];
|
||||
};
|
||||
|
||||
export type WorkspaceCleanupMode = "dry-run" | "quarantine";
|
||||
|
||||
export type WorkspaceCleanupOptions = WorkspaceCleanupScanOptions & {
|
||||
mode?: WorkspaceCleanupMode;
|
||||
now?: Date;
|
||||
};
|
||||
|
||||
export type WorkspaceCleanupQuarantineEvent = WorkspaceCleanupResult & {
|
||||
from: string;
|
||||
to: string;
|
||||
quarantinedAt: string;
|
||||
};
|
||||
|
||||
export type WorkspaceCleanupRunResult = WorkspaceCleanupScan & {
|
||||
mode: WorkspaceCleanupMode;
|
||||
quarantined: WorkspaceCleanupQuarantineEvent[];
|
||||
quarantineDir?: string;
|
||||
};
|
||||
|
||||
type WorkspaceMemoryShape = {
|
||||
workspace?: {
|
||||
root?: unknown;
|
||||
key?: unknown;
|
||||
};
|
||||
entries?: unknown[];
|
||||
migrations?: unknown[];
|
||||
updatedAt?: unknown;
|
||||
};
|
||||
|
||||
const DEFAULT_MIN_AGE_MS = 10 * 60 * 1_000;
|
||||
|
||||
const KNOWN_TEST_ROOT_PREFIXES = [
|
||||
"memory-plugin-test-",
|
||||
"memory-plugin-prompt-",
|
||||
"wm-",
|
||||
"wm-quality-",
|
||||
"wm-accounting-",
|
||||
"wm-redact-",
|
||||
"wm-normalized-",
|
||||
"wm-ordering-",
|
||||
"wm-extraction-",
|
||||
];
|
||||
|
||||
function normalizePathForComparison(path: string): string {
|
||||
return resolve(path).replace(/\/+$/, "");
|
||||
}
|
||||
|
||||
function isInsidePath(path: string, parent: string): boolean {
|
||||
const normalizedPath = normalizePathForComparison(path);
|
||||
const normalizedParent = normalizePathForComparison(parent);
|
||||
return normalizedPath === normalizedParent || normalizedPath.startsWith(`${normalizedParent}/`);
|
||||
}
|
||||
|
||||
export function isTempRoot(root: string, osTmpdir = tmpdir()): boolean {
|
||||
const normalized = normalizePathForComparison(root);
|
||||
const normalizedTmp = normalizePathForComparison(osTmpdir);
|
||||
|
||||
if (isInsidePath(normalized, normalizedTmp)) return true;
|
||||
if (isInsidePath(normalized, "/tmp")) return true;
|
||||
if (isInsidePath(normalized, "/private/tmp")) return true;
|
||||
|
||||
return /^\/(?:private\/)?var\/folders\/[^/]+\/[^/]+\/T(?:\/|$)/.test(normalized);
|
||||
}
|
||||
|
||||
export function isKnownTestWorkspaceRoot(root: string): boolean {
|
||||
const name = basename(root);
|
||||
return KNOWN_TEST_ROOT_PREFIXES.some(prefix => name.startsWith(prefix));
|
||||
}
|
||||
|
||||
function classifyCandidate(result: WorkspaceCleanupResult, includeOrphans: boolean): boolean {
|
||||
if (result.reasons.includes("recent_workspace_dir")) return false;
|
||||
if (result.reasons.includes("lock_present")) return false;
|
||||
if (result.classification === "test_temp_definite") return true;
|
||||
return includeOrphans && result.classification === "orphan_unknown";
|
||||
}
|
||||
|
||||
export async function classifyWorkspaceDir(
|
||||
workspaceDir: string,
|
||||
options: { nowMs?: number; minAgeMs?: number } = {},
|
||||
): Promise<WorkspaceCleanupResult> {
|
||||
const workspaceKey = basename(workspaceDir);
|
||||
const reasons: string[] = [];
|
||||
const memoryPath = join(workspaceDir, "workspace-memory.json");
|
||||
|
||||
if (existsSync(`${memoryPath}.lock`)) {
|
||||
reasons.push("lock_present");
|
||||
}
|
||||
|
||||
let stats;
|
||||
try {
|
||||
stats = await stat(workspaceDir);
|
||||
} catch {
|
||||
return {
|
||||
workspaceKey,
|
||||
workspaceDir,
|
||||
rootExists: false,
|
||||
classification: "invalid_or_unreadable",
|
||||
reasons: ["workspace_dir_unreadable"],
|
||||
};
|
||||
}
|
||||
|
||||
const minAgeMs = options.minAgeMs ?? DEFAULT_MIN_AGE_MS;
|
||||
const nowMs = options.nowMs ?? Date.now();
|
||||
if (minAgeMs > 0 && nowMs - stats.mtimeMs < minAgeMs) {
|
||||
reasons.push("recent_workspace_dir");
|
||||
}
|
||||
|
||||
let store: WorkspaceMemoryShape;
|
||||
try {
|
||||
store = JSON.parse(await readFile(memoryPath, "utf8")) as WorkspaceMemoryShape;
|
||||
} catch {
|
||||
return {
|
||||
workspaceKey,
|
||||
workspaceDir,
|
||||
rootExists: false,
|
||||
classification: "invalid_or_unreadable",
|
||||
reasons: [...reasons, "invalid_json"],
|
||||
};
|
||||
}
|
||||
|
||||
const root = typeof store.workspace?.root === "string" ? store.workspace.root : undefined;
|
||||
const key = typeof store.workspace?.key === "string" ? store.workspace.key : workspaceKey;
|
||||
const entryCount = Array.isArray(store.entries) ? store.entries.length : undefined;
|
||||
const migrations = Array.isArray(store.migrations) ? store.migrations.filter((item): item is string => typeof item === "string") : undefined;
|
||||
const updatedAt = typeof store.updatedAt === "string" ? store.updatedAt : undefined;
|
||||
|
||||
if (!root) {
|
||||
return {
|
||||
workspaceKey: key,
|
||||
workspaceDir,
|
||||
rootExists: false,
|
||||
classification: "invalid_or_unreadable",
|
||||
reasons: [...reasons, "missing_workspace_root"],
|
||||
entryCount,
|
||||
migrations,
|
||||
updatedAt,
|
||||
};
|
||||
}
|
||||
|
||||
const rootExists = existsSync(root);
|
||||
if (rootExists) {
|
||||
return {
|
||||
workspaceKey: key,
|
||||
workspaceDir,
|
||||
root,
|
||||
rootExists,
|
||||
classification: "live_or_existing",
|
||||
reasons: [...reasons, "root_exists"],
|
||||
entryCount,
|
||||
migrations,
|
||||
updatedAt,
|
||||
};
|
||||
}
|
||||
|
||||
reasons.push("root_missing");
|
||||
const tempRoot = isTempRoot(root);
|
||||
const testRoot = isKnownTestWorkspaceRoot(root);
|
||||
if (tempRoot) reasons.push("root_under_temp");
|
||||
if (testRoot) reasons.push(`test_prefix_${KNOWN_TEST_ROOT_PREFIXES.find(prefix => basename(root).startsWith(prefix))?.replace(/-$/, "") ?? basename(root)}`);
|
||||
|
||||
return {
|
||||
workspaceKey: key,
|
||||
workspaceDir,
|
||||
root,
|
||||
rootExists,
|
||||
classification: tempRoot || testRoot ? "test_temp_definite" : "orphan_unknown",
|
||||
reasons,
|
||||
entryCount,
|
||||
migrations,
|
||||
updatedAt,
|
||||
};
|
||||
}
|
||||
|
||||
function workspacesDir(dataHome: string): string {
|
||||
return join(dataHome, "opencode-working-memory", "workspaces");
|
||||
}
|
||||
|
||||
export async function scanWorkspaceResidues(options: WorkspaceCleanupScanOptions = {}): Promise<WorkspaceCleanupScan> {
|
||||
const root = workspacesDir(options.dataHome ?? defaultDataHome());
|
||||
const results: WorkspaceCleanupResult[] = [];
|
||||
|
||||
let entries: string[];
|
||||
try {
|
||||
entries = await readdir(root);
|
||||
} catch {
|
||||
return { results, candidates: [] };
|
||||
}
|
||||
|
||||
for (const entry of entries) {
|
||||
const workspaceDir = join(root, entry);
|
||||
const stats = await stat(workspaceDir).catch(() => undefined);
|
||||
if (!stats?.isDirectory()) continue;
|
||||
|
||||
results.push(await classifyWorkspaceDir(workspaceDir, {
|
||||
nowMs: options.nowMs,
|
||||
minAgeMs: options.minAgeMs,
|
||||
}));
|
||||
}
|
||||
|
||||
return {
|
||||
results,
|
||||
candidates: results.filter(result => classifyCandidate(result, options.includeOrphans ?? false)),
|
||||
};
|
||||
}
|
||||
|
||||
function quarantineName(now: Date): string {
|
||||
return `workspace-cleanup-${now.toISOString().replace(/[-:]/g, "").replace(/\.\d{3}Z$/, "Z")}`;
|
||||
}
|
||||
|
||||
export async function cleanupWorkspaceResidues(options: WorkspaceCleanupOptions = {}): Promise<WorkspaceCleanupRunResult> {
|
||||
const mode = options.mode ?? "dry-run";
|
||||
const now = options.now ?? new Date();
|
||||
const scan = await scanWorkspaceResidues({
|
||||
dataHome: options.dataHome,
|
||||
nowMs: options.nowMs,
|
||||
minAgeMs: options.minAgeMs,
|
||||
includeOrphans: options.includeOrphans,
|
||||
});
|
||||
|
||||
if (mode === "dry-run" || scan.candidates.length === 0) {
|
||||
return { ...scan, mode, quarantined: [] };
|
||||
}
|
||||
|
||||
const dataHome = options.dataHome ?? defaultDataHome();
|
||||
const quarantineDir = join(dataHome, "opencode-working-memory", "quarantine", quarantineName(now));
|
||||
const quarantined: WorkspaceCleanupQuarantineEvent[] = [];
|
||||
|
||||
for (const candidate of scan.candidates) {
|
||||
const destination = join(quarantineDir, "workspaces", candidate.workspaceKey);
|
||||
await mkdir(dirname(destination), { recursive: true });
|
||||
await rename(candidate.workspaceDir, destination);
|
||||
|
||||
const event: WorkspaceCleanupQuarantineEvent = {
|
||||
...candidate,
|
||||
from: candidate.workspaceDir,
|
||||
to: destination,
|
||||
quarantinedAt: now.toISOString(),
|
||||
};
|
||||
quarantined.push(event);
|
||||
|
||||
await mkdir(quarantineDir, { recursive: true });
|
||||
await appendFile(join(quarantineDir, "manifest.jsonl"), JSON.stringify(event) + "\n", "utf8");
|
||||
}
|
||||
|
||||
return { ...scan, mode, quarantined, quarantineDir };
|
||||
}
|
||||
+1
-50
@@ -5,24 +5,13 @@ import { LONG_TERM_LIMITS } from "./types.ts";
|
||||
import { migrationLogPath, workspaceKey, workspaceMemoryPath } from "./paths.ts";
|
||||
import { atomicWriteJSON, readJSON, updateJSON } from "./storage.ts";
|
||||
import { assessMemoryQuality, isHardQualityReason, isProgressSnapshotViolation } from "./memory-quality.ts";
|
||||
import { redactCredentials } from "./redaction.ts";
|
||||
|
||||
// Minimum length for workspace_memory envelope: <workspace_memory>\n...\n</workspace_memory>
|
||||
const MIN_ENVELOPE_LENGTH = 80;
|
||||
const MIGRATION_ID = "2026-04-26-p0-cleanup";
|
||||
const QUALITY_CLEANUP_MIGRATION_ID = "2026-04-28-quality-cleanup";
|
||||
|
||||
const SECRET_VALUE = String.raw`[^` + "`" + String.raw`'",，,\s\[]+`;
|
||||
|
||||
const PASSWORD_LABELS = /password|passwd|pwd|密碼|密码|パスワード|비밀번호|contraseña|mot de passe|passwort/i;
|
||||
const USERNAME_LABELS = /username|user name|用戶名|用户名|ユーザー名|사용자명|usuario|utilisateur|benutzer/i;
|
||||
const SENSITIVE_LABELS = /api[_-]?key|token|bearer|secret|credential|auth|auth[_-]?key|private[_-]?key/i;
|
||||
|
||||
const PIN_PREFIX = String.raw`(\bPIN\b(?:\s*(?:是|=|:|：)\s*|\s+(?![是=:：])))`;
|
||||
const PASSWORD_PREFIX = String.raw`((?:${PASSWORD_LABELS.source})(?:\s*(?:是|=|:|：)\s*|\s+(?![是=:：])))`;
|
||||
const USERNAME_PREFIX = String.raw`((?:${USERNAME_LABELS.source})(?:\s*(?:是|=|:|：)\s*|\s+(?![是=:：])))`;
|
||||
const SENSITIVE_PREFIX = String.raw`((?:${SENSITIVE_LABELS.source})(?:\s*(?:推|是|=|:|：)\s*|[:：]\s*))`;
|
||||
const BEARER_PREFIX = String.raw`(Bearer\s+)`;
|
||||
|
||||
export type MemoryConsolidationReason =
|
||||
| "promoted"
|
||||
| "absorbed_exact"
|
||||
@@ -250,44 +239,6 @@ export async function normalizeWorkspaceMemoryWithAccounting(
|
||||
};
|
||||
}
|
||||
|
||||
export function redactCredentials(text: string): string {
|
||||
let result = text;
|
||||
|
||||
// 1. PIN
|
||||
result = result.replace(
|
||||
new RegExp(String.raw`${PIN_PREFIX}[\`'"]?(${SECRET_VALUE})`, "gi"),
|
||||
"$1[REDACTED]",
|
||||
);
|
||||
|
||||
// 2. Username+password pair
|
||||
result = result.replace(
|
||||
new RegExp(
|
||||
String.raw`${USERNAME_PREFIX}[\`'"]?(${SECRET_VALUE})((?:，|,)\s*)${PASSWORD_PREFIX}[\`'"]?(${SECRET_VALUE})`,
|
||||
"gi",
|
||||
),
|
||||
"$1[REDACTED]$3$4[REDACTED]",
|
||||
);
|
||||
|
||||
// 3. Standalone password
|
||||
result = result.replace(
|
||||
new RegExp(String.raw`${PASSWORD_PREFIX}[\`'"]?(${SECRET_VALUE})`, "gi"),
|
||||
"$1[REDACTED]",
|
||||
);
|
||||
|
||||
// 4. Standalone sensitive keys/tokens
|
||||
result = result.replace(
|
||||
new RegExp(String.raw`${BEARER_PREFIX}(?!token\s*[:=：])[\`'"]?(${SECRET_VALUE})`, "gi"),
|
||||
"$1[REDACTED]",
|
||||
);
|
||||
|
||||
result = result.replace(
|
||||
new RegExp(String.raw`${SENSITIVE_PREFIX}[\`'"]?(${SECRET_VALUE})`, "gi"),
|
||||
"$1[REDACTED]",
|
||||
);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
export function runMigrationP0Cleanup(
|
||||
store: WorkspaceMemoryStore,
|
||||
nowIso: string,
|
||||
|
||||
Reference in New Issue
Block a user