From 2f04612e8dddc13f8bd2d9c8cf8b2377adf16d8c Mon Sep 17 00:00:00 2001 From: Pawan Osman Date: Wed, 15 Jul 2026 13:56:26 +0300 Subject: [PATCH] Implement incremental indexing and file watching for semantic updates - Introduced a new `indexWatch` module to handle debounced file watching for incremental updates to the semantic index. - Updated `activate` function to initialize the index watcher and replace the previous build index call with a warm disk and incremental sync. - Enhanced the `semanticIndex` module to support enabling/disabling indexing and improved index path normalization. - Modified the settings panel and UI components to include an option for enabling/disabling indexing, reflecting changes in the feature store. - Ensured that indexing operations respect the new `indexingEnabled` flag across various functionalities, including search and document indexing. --- src/agent/indexWatch.ts | 115 ++++++++++++++++++ src/agent/semanticIndex.ts | 204 ++++++++++++++++++++++++++++---- src/agent/tools/search.ts | 4 +- src/extension.ts | 11 +- src/stores/featureStore.ts | 3 + src/ui/settingsPanel.ts | 6 +- webview-ui/settings/App.tsx | 34 ++++-- webview-ui/settings/features.ts | 2 + 8 files changed, 332 insertions(+), 47 deletions(-) create mode 100644 src/agent/indexWatch.ts diff --git a/src/agent/indexWatch.ts b/src/agent/indexWatch.ts new file mode 100644 index 0000000..0a50afa --- /dev/null +++ b/src/agent/indexWatch.ts @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2026 Pawan Osman + * + * This file is part of OpenCursor — AI coding agent chat inside VS Code. + * https://github.com/PawanOsman/OpenCursor + * + * Licensed under the MIT License. See LICENSE file in the project root. + */ + +// Debounced workspace file watcher → incremental semantic index updates. + +import * as vscode from "vscode"; +import * as path from "path"; +import { upsertFile, removeFile, buildIndex, setIndexingEnabled, warmIndex, isIndexingEnabled } from "./semanticIndex"; +import { getWorkspaceRoot } from "../context/workspaceUtils"; +import type { FeatureStore } from "../stores/featureStore"; + +const DEBOUNCE_MS = 800; +const pending = new Map(); // abs path -> action +let timer: ReturnType | null = null; +let featureStore: FeatureStore | null = null; +let flushing = false; + +function scheduleFlush(): void { + if (timer) clearTimeout(timer); + timer = setTimeout(() => { + timer = null; + void flush(); + }, DEBOUNCE_MS); +} + +async function flush(): Promise { + if (flushing) { + scheduleFlush(); + return; + } + if (!isIndexingEnabled()) { + pending.clear(); + return; + } + const root = getWorkspaceRoot(); + if (!root || !pending.size) return; + flushing = true; + try { + const batch = new Map(pending); + pending.clear(); + for (const [abs, action] of batch) { + try { + if (action === "del") await removeFile(root, abs); + else await upsertFile(root, abs); + } catch { + /* ignore single-file failures */ + } + } + } finally { + flushing = false; + if (pending.size) scheduleFlush(); + } +} + +function onFs(uri: vscode.Uri, action: "up" | "del"): void { + if (!isIndexingEnabled()) return; + if (uri.scheme !== "file") return; + const root = getWorkspaceRoot(); + if (!root) return; + const abs = uri.fsPath; + if (!abs.startsWith(root) && !abs.toLowerCase().startsWith(root.toLowerCase())) return; + // Skip heavy/vendor trees (walk already ignores some; watcher still sees them). + const rel = path.relative(root, abs).split(path.sep).join("/"); + if (!rel || rel.startsWith("..")) return; + if (/(^|\/)(node_modules|\.git|dist|out|build|\.next|coverage)(\/|$)/.test(rel)) return; + pending.set(abs, action); + scheduleFlush(); +} + +/** Wire indexing enable flag, warm disk index, incremental sync, file watcher. */ +export function initIndexWatch(context: vscode.ExtensionContext, store: FeatureStore): void { + featureStore = store; + let lastEnabled = store.get().indexingEnabled !== false; + setIndexingEnabled(lastEnabled); + + const root = getWorkspaceRoot(); + void warmIndex(root).then(() => { + if (lastEnabled) void buildIndex(root).catch(() => {}); + }); + + const watcher = vscode.workspace.createFileSystemWatcher("**/*"); + context.subscriptions.push( + watcher, + watcher.onDidCreate((u) => onFs(u, "up")), + watcher.onDidChange((u) => onFs(u, "up")), + watcher.onDidDelete((u) => onFs(u, "del")), + vscode.workspace.onDidSaveTextDocument((doc) => onFs(doc.uri, "up")), + store.onDidChange(() => { + const on = store.get().indexingEnabled !== false; + if (on === lastEnabled) return; + lastEnabled = on; + setIndexingEnabled(on); + if (on) void buildIndex(getWorkspaceRoot()).catch(() => {}); + }), + vscode.workspace.onDidChangeWorkspaceFolders(() => { + if (!isIndexingEnabled()) return; + const f = featureStore?.get(); + if (f && f.indexNewFolders === false) return; + const r = getWorkspaceRoot(); + void warmIndex(r).then(() => buildIndex(r).catch(() => {})); + }), + { + dispose: () => { + if (timer) clearTimeout(timer); + pending.clear(); + }, + }, + ); +} diff --git a/src/agent/semanticIndex.ts b/src/agent/semanticIndex.ts index 38ecfe3..0fa6c61 100644 --- a/src/agent/semanticIndex.ts +++ b/src/agent/semanticIndex.ts @@ -169,34 +169,61 @@ export async function embedTexts(texts: string[]): Promise { } // ---- Index lifecycle ---- +/** Stable workspace key so Windows drive-letter case / trailing slashes don't orphan the index. */ +function normRoot(root: string): string { + const r = path.resolve(root); + return process.platform === "win32" ? r.toLowerCase() : r; +} + function indexPath(root: string): string { - const id = crypto.createHash("sha1").update(root).digest("hex").slice(0, 16); + const id = crypto.createHash("sha1").update(normRoot(root)).digest("hex").slice(0, 16); const mid = getEmbedModelId().replace(/[^\w.-]+/g, "_"); return path.join(storageDir!, `index-${id}-${mid}.json`); } let memIndex: IndexFile | null = null; -let memRoot: string | null = null; +let memRoot: string | null = null; // normRoot key +let indexingEnabled = true; + +export function setIndexingEnabled(on: boolean): void { + indexingEnabled = on; +} + +export function isIndexingEnabled(): boolean { + return indexingEnabled; +} async function load(root: string): Promise { - if (memIndex && memRoot === root) return memIndex; + const key = normRoot(root); + if (memIndex && memRoot === key) return memIndex; try { const raw = await fs.readFile(indexPath(root), "utf8"); const parsed = JSON.parse(raw) as IndexFile; - if (parsed.model === getEmbedModelId()) { + if (parsed.model === getEmbedModelId() && parsed.files && Array.isArray(parsed.chunks)) { memIndex = parsed; - memRoot = root; + memRoot = key; return parsed; } } catch {} memIndex = { model: getEmbedModelId(), files: {}, chunks: [] }; - memRoot = root; + memRoot = key; return memIndex; } async function save(root: string, idx: IndexFile): Promise { - await fs.mkdir(storageDir!, { recursive: true }); + if (!storageDir) return; + await fs.mkdir(storageDir, { recursive: true }); await fs.writeFile(indexPath(root), JSON.stringify(idx), "utf8"); + memIndex = idx; + memRoot = normRoot(root); +} + +/** Load persisted index into memory (no embed work). Call on activate so status/UI show prior work. */ +export async function warmIndex(root: string): Promise { + if (!storageDir || !root) return getStatus(root); + await load(root); + emitStatus(root); + return getStatus(root); } function chunkFile(rel: string, text: string): { start: number; end: number; text: string }[] { @@ -233,7 +260,7 @@ export function onIndexStatus(fn: (s: IndexStatus) => void): () => void { return () => statusSubs.delete(fn); } export function getStatus(root: string): IndexStatus { - const idx = memRoot === root ? memIndex : null; + const idx = memRoot === normRoot(root) ? memIndex : null; return { indexing, done: progress.done, @@ -251,15 +278,138 @@ function emitStatus(root: string): void { /** Delete the persisted index for a workspace. */ export async function deleteIndex(root: string): Promise { memIndex = { model: getEmbedModelId(), files: {}, chunks: [] }; - memRoot = root; + memRoot = normRoot(root); progress = { done: 0, total: 0 }; try { await fs.unlink(indexPath(root)); } catch {} emitStatus(root); } -/** (Re)build the index incrementally. Safe to call repeatedly; no-op if busy. */ +function isIndexableRel(rel: string): boolean { + return EMBED_EXTS.has(path.extname(rel).toLowerCase()); +} + +async function embedFileInto(idx: IndexFile, root: string, rel: string): Promise { + const abs = path.join(root, rel); + let st; + try { st = await fs.stat(abs); } catch { return false; } + if (st.size > MAX_FILE_BYTES) return false; + let text: string; + try { text = await fs.readFile(abs, "utf8"); } catch { return false; } + // Drop old chunks for this path first. + idx.chunks = idx.chunks.filter((c) => c.path !== rel); + const pieces = chunkFile(rel, text); + if (pieces.length) { + const vecs = await embed(pieces.map((p) => p.text)); + if (vecs) { + pieces.forEach((p, i) => idx.chunks.push({ path: rel, start: p.start, end: p.end, text: p.text, vec: vecs[i] })); + } + } + idx.files[rel] = `${Math.round(st.mtimeMs)}:${st.size}`; + return true; +} + +/** Pending single-file updates while a full build runs (or coalesced watcher queue). */ +const pendingUpserts = new Map>(); // rootKey -> rel paths +const pendingDeletes = new Map>(); +let drainRunning = false; + +function queueKey(root: string): string { + return normRoot(root); +} + +/** Index/update one file immediately (or queue if full build busy). */ +export async function upsertFile(root: string, absOrRel: string): Promise { + if (!storageDir || !indexingEnabled || !root) return; + const abs = path.isAbsolute(absOrRel) ? absOrRel : path.join(root, absOrRel); + const rel = path.relative(root, abs).split(path.sep).join("/"); + if (!rel || rel.startsWith("..") || !isIndexableRel(rel)) return; + const key = queueKey(root); + if (indexing || drainRunning) { + if (!pendingUpserts.has(key)) pendingUpserts.set(key, new Set()); + pendingUpserts.get(key)!.add(rel); + pendingDeletes.get(key)?.delete(rel); + return; + } + const idx = await load(root); + let st; + try { st = await fs.stat(abs); } catch { + await removeFile(root, abs); + return; + } + const hash = `${Math.round(st.mtimeMs)}:${st.size}`; + if (idx.files[rel] === hash || idx.files[rel] === `${st.mtimeMs}:${st.size}`) return; + await embedFileInto(idx, root, rel); + await save(root, idx); + emitStatus(root); +} + +/** Remove a file from the index (delete/rename). */ +export async function removeFile(root: string, absOrRel: string): Promise { + if (!storageDir || !indexingEnabled || !root) return; + const abs = path.isAbsolute(absOrRel) ? absOrRel : path.join(root, absOrRel); + const rel = path.relative(root, abs).split(path.sep).join("/"); + if (!rel || rel.startsWith("..")) return; + const key = queueKey(root); + if (indexing || drainRunning) { + if (!pendingDeletes.has(key)) pendingDeletes.set(key, new Set()); + pendingDeletes.get(key)!.add(rel); + pendingUpserts.get(key)?.delete(rel); + return; + } + const idx = await load(root); + if (!idx.files[rel] && !idx.chunks.some((c) => c.path === rel)) return; + idx.chunks = idx.chunks.filter((c) => c.path !== rel); + delete idx.files[rel]; + await save(root, idx); + emitStatus(root); +} + +async function drainPending(root: string): Promise { + if (drainRunning || indexing || !indexingEnabled) return; + const key = queueKey(root); + const ups = pendingUpserts.get(key); + const dels = pendingDeletes.get(key); + if ((!ups || !ups.size) && (!dels || !dels.size)) return; + drainRunning = true; + try { + const idx = await load(root); + if (dels?.size) { + for (const rel of dels) { + idx.chunks = idx.chunks.filter((c) => c.path !== rel); + delete idx.files[rel]; + } + dels.clear(); + } + if (ups?.size) { + const list = [...ups]; + ups.clear(); + progress = { done: 0, total: list.length }; + indexing = true; + emitStatus(root); + let done = 0; + for (const rel of list) { + await embedFileInto(idx, root, rel); + done++; + progress = { done, total: list.length }; + emitStatus(root); + } + indexing = false; + } + await save(root, idx); + emitStatus(root); + } finally { + drainRunning = false; + indexing = false; + // More events may have arrived. + if ((pendingUpserts.get(key)?.size || 0) + (pendingDeletes.get(key)?.size || 0) > 0) { + void drainPending(root); + } + } +} + +/** (Re)build the index incrementally. Only re-embeds changed files. No-op if disabled/busy. */ export async function buildIndex(root: string, onProgress?: (done: number, total: number) => void): Promise { - if (!storageDir || indexing) return; + if (!storageDir || !root || indexing || !indexingEnabled) return; indexing = true; progress = { done: 0, total: 0 }; emitStatus(root); @@ -276,41 +426,43 @@ export async function buildIndex(root: string, onProgress?: (done: number, total try { st = await fs.stat(f); } catch { continue; } if (st.size > MAX_FILE_BYTES) continue; seen.add(rel); - const hash = `${st.mtimeMs}:${st.size}`; - if (idx.files[rel] !== hash) targets.push(rel); + const hash = `${Math.round(st.mtimeMs)}:${st.size}`; + const prev = idx.files[rel]; + // Accept either rounded or raw mtime strings from older indexes. + if (prev !== hash && prev !== `${st.mtimeMs}:${st.size}`) targets.push(rel); } - // Drop chunks for deleted/changed files. const changed = new Set(targets); idx.chunks = idx.chunks.filter((c) => seen.has(c.path) && !changed.has(c.path)); for (const rel of Object.keys(idx.files)) { if (!seen.has(rel)) delete idx.files[rel]; } + // Nothing to do — still save cleaned deletions if any, emit status. + if (!targets.length) { + await save(root, idx); + return; + } + let done = 0; progress = { done: 0, total: targets.length }; emitStatus(root); for (const rel of targets) { - const abs = path.join(root, rel); - let text: string; - try { text = await fs.readFile(abs, "utf8"); } catch { done++; continue; } - const pieces = chunkFile(rel, text); - if (pieces.length) { - const vecs = await embed(pieces.map((p) => p.text)); - if (vecs) { - pieces.forEach((p, i) => idx.chunks.push({ path: rel, start: p.start, end: p.end, text: p.text, vec: vecs[i] })); - } - } - const st = await fs.stat(abs).catch(() => null); - if (st) idx.files[rel] = `${st.mtimeMs}:${st.size}`; + await embedFileInto(idx, root, rel); + // Prefer stable rounded hash going forward. + const st = await fs.stat(path.join(root, rel)).catch(() => null); + if (st) idx.files[rel] = `${Math.round(st.mtimeMs)}:${st.size}`; done++; progress = { done, total: targets.length }; onProgress?.(done, targets.length); emitStatus(root); + // Persist periodically so reopen mid-index keeps progress. + if (done % 25 === 0) await save(root, idx); } await save(root, idx); } finally { indexing = false; emitStatus(root); + void drainPending(root); } } diff --git a/src/agent/tools/search.ts b/src/agent/tools/search.ts index 775ab7f..732d4ab 100644 --- a/src/agent/tools/search.ts +++ b/src/agent/tools/search.ts @@ -13,7 +13,7 @@ import { spawn } from "child_process"; import { safePath, getWorkspaceRoot } from "../../context/workspaceUtils"; import { defineTool } from "./types"; import { STOP, walk, globToRe, rgAvailable } from "./shared"; -import { search as semanticIndexSearch, buildIndex, isIndexing } from "../semanticIndex"; +import { search as semanticIndexSearch, buildIndex, isIndexing, isIndexingEnabled } from "../semanticIndex"; import { searchDocs, listDocSources } from "../docsIndex"; // Minimal ripgrep --type -> file-extension map for the node fallback. @@ -170,7 +170,7 @@ export const semanticSearchTool = defineTool("SemanticSearch", false, async (inp const root = getWorkspaceRoot(); // Build/refresh index on demand (incremental; cheap if already fresh). - if (!isIndexing()) buildIndex(root).catch(() => {}); + if (isIndexingEnabled() && !isIndexing()) buildIndex(root).catch(() => {}); // Scope by target_directories (prefix match on workspace-relative paths). const dirs: string[] = Array.isArray(input.target_directories) ? input.target_directories : []; diff --git a/src/extension.ts b/src/extension.ts index 25e4a79..22c8584 100644 --- a/src/extension.ts +++ b/src/extension.ts @@ -14,9 +14,9 @@ import { registerInlineReview } from './ui/inlineReview'; import { SettingsPanel } from './ui/settingsPanel'; import { FeatureStore } from './stores/featureStore'; import { mcpManager } from './integrations/mcpClient'; -import { setIndexStorageDir, buildIndex } from './agent/semanticIndex'; +import { setIndexStorageDir } from './agent/semanticIndex'; import { setDocsStorageDir, setDocSourcesProvider } from './agent/docsIndex'; -import { getWorkspaceRoot } from './context/workspaceUtils'; +import { initIndexWatch } from './agent/indexWatch'; import { initLlamacpp, checkInstalled, loadModel, disposeLlamacpp } from './agent/llamacpp'; import { initOAuth } from './agent/oauth'; import { initUsage } from './stores/usageStore'; @@ -38,14 +38,13 @@ export function activate(context: vscode.ExtensionContext) { // renders instantly from the backend cache. initModelRegistry(featureStore, settingsManager); - // Local semantic index: model + vectors live in extension globalStorage. - // Kick off an initial background build (incremental; no-op if already fresh). + // Local semantic index: vectors in globalStorage; warm disk + incremental sync. setIndexStorageDir(context.globalStorageUri.fsPath); setDocsStorageDir(context.globalStorageUri.fsPath); setDocSourcesProvider(() => featureStore.get().docSources ?? []); applyEmbedModel(featureStore.get().embedModel || "minilm") - .then(() => buildIndex(getWorkspaceRoot())) - .catch(() => {}); + .then(() => initIndexWatch(context, featureStore)) + .catch(() => initIndexWatch(context, featureStore)); // Connect any enabled MCP servers in the background. mcpManager.sync(featureStore.get().mcpServers).catch(() => {}); diff --git a/src/stores/featureStore.ts b/src/stores/featureStore.ts index 6ab4b14..f494437 100644 --- a/src/stores/featureStore.ts +++ b/src/stores/featureStore.ts @@ -221,6 +221,8 @@ export interface FeatureConfig { approvalPolicy: ApprovalPolicy; /** External documentation sources indexed for @Docs mentions. */ docSources: DocSource[]; + /** Master switch for semantic codebase indexing. */ + indexingEnabled: boolean; /** Automatically index newly added workspace folders. */ indexNewFolders: boolean; /** Index repositories to speed up grep searches (all data local). */ @@ -262,6 +264,7 @@ const DEFAULTS: FeatureConfig = { webFetchEnabled: true, approvalPolicy: DEFAULT_APPROVAL, docSources: [], + indexingEnabled: true, indexNewFolders: true, indexForGrep: true, }; diff --git a/src/ui/settingsPanel.ts b/src/ui/settingsPanel.ts index 84c4a20..98332a7 100644 --- a/src/ui/settingsPanel.ts +++ b/src/ui/settingsPanel.ts @@ -15,7 +15,7 @@ import { FeatureStore, MODEL_CATALOG } from "../stores/featureStore"; import { listRules, listSkills } from "../context/workspaceContext"; import { mcpManager } from "../integrations/mcpClient"; import { BUILTIN_PERSONAS } from "../agent/personas"; -import { getStatus, onIndexStatus, buildIndex, deleteIndex, EMBED_MODELS } from "../agent/semanticIndex"; +import { getStatus, onIndexStatus, buildIndex, deleteIndex, warmIndex, EMBED_MODELS } from "../agent/semanticIndex"; import { indexDocSource, deleteDocIndex, onDocsStatus, getDocsStatus, getDocLogs, type DocSource } from "../agent/docsIndex"; import { getWorkspaceRoot } from "../context/workspaceUtils"; import * as llama from "../agent/llamacpp"; @@ -274,6 +274,7 @@ export class SettingsPanel { await this._sendFeatures(); break; case "getIndexStatus": + await warmIndex(getWorkspaceRoot()); this._panel.webview.postMessage({ type: "indexStatus", status: getStatus(getWorkspaceRoot()), models: EMBED_MODELS }); this._sendDocs(); break; @@ -327,6 +328,7 @@ export class SettingsPanel { break; } case "syncIndex": + if (this.featureStore.get().indexingEnabled === false) break; buildIndex(getWorkspaceRoot()).catch(() => {}); break; case "deleteIndex": @@ -337,7 +339,7 @@ export class SettingsPanel { await this.featureStore.set({ ...f, embedModel: message.modelId }); await applyEmbedModel(message.modelId); this._panel.webview.postMessage({ type: "indexStatus", status: getStatus(getWorkspaceRoot()), models: EMBED_MODELS }); - buildIndex(getWorkspaceRoot()).catch(() => {}); // re-embed with new model + if (f.indexingEnabled !== false) buildIndex(getWorkspaceRoot()).catch(() => {}); break; } case "llamacppGet": diff --git a/webview-ui/settings/App.tsx b/webview-ui/settings/App.tsx index c7ddb45..892fe57 100644 --- a/webview-ui/settings/App.tsx +++ b/webview-ui/settings/App.tsx @@ -120,10 +120,10 @@ function NumInput({ ); } -function Toggle({ checked, onChange }: { checked: boolean; onChange: (v: boolean) => void }) { +function Toggle({ checked, onChange, disabled }: { checked: boolean; onChange: (v: boolean) => void; disabled?: boolean }) { return ( -