feat: Implement initial personal knowledge management system with vector search and MCP server integration.

2026-07-18 08:05:59 +02:00 · 2025-12-14 12:31:01 -08:00
commit 22aeab952d
10 changed files with 1532 additions and 0 deletions
@@ -0,0 +1,141 @@
+/**
+ * Knowledge Service
+ * 
+ * Business logic coordinating database and vector operations.
+ */
+import {
+  saveKnowledgeEntry,
+  getKnowledgeEntry,
+  updateKnowledgeEntry,
+  deleteKnowledgeEntry,
+  listKnowledgeEntries,
+  searchKnowledgeByText,
+  getStats as getDbStats,
+  type KnowledgeEntry,
+} from "../database/index.js";
+import {
+  queryVectors,
+  updateVector,
+  deleteVector,
+  getVectorStats,
+  type SearchResult,
+} from "./vectorService.js";
+
+export { type KnowledgeEntry };
+
+/**
+ * Add a new knowledge entry with automatic vector indexing.
+ */
+export async function addKnowledge(entry: {
+  title: string;
+  content: string;
+  source?: string;
+  tags?: string[];
+}): Promise<{ id: number; vectorized: boolean }> {
+  // Save to SQLite
+  const id = saveKnowledgeEntry(entry);
+  
+  // Index in vector DB
+  let vectorized = false;
+  try {
+    const savedEntry = getKnowledgeEntry(id);
+    if (savedEntry) {
+      await updateVector(savedEntry);
+      vectorized = true;
+    }
+  } catch {
+    // Vector indexing failed, but entry is saved
+    console.error("Vector indexing failed, entry saved to database only");
+  }
+
+  return { id, vectorized };
+}
+
+/**
+ * Search knowledge using semantic similarity.
+ */
+export async function searchKnowledge(
+  query: string,
+  options: { limit?: number; minScore?: number } = {}
+): Promise<SearchResult[]> {
+  return queryVectors(query, options);
+}
+
+/**
+ * Search knowledge using text matching.
+ */
+export function searchKnowledgeText(query: string, limit = 10): KnowledgeEntry[] {
+  return searchKnowledgeByText(query, limit);
+}
+
+/**
+ * Get a knowledge entry by ID.
+ */
+export function getKnowledge(id: number): KnowledgeEntry | null {
+  return getKnowledgeEntry(id);
+}
+
+/**
+ * Update a knowledge entry with automatic vector re-indexing.
+ */
+export async function updateKnowledge(
+  id: number,
+  updates: Partial<Pick<KnowledgeEntry, "title" | "content" | "source" | "tags">>
+): Promise<{ success: boolean; vectorized: boolean }> {
+  const success = updateKnowledgeEntry(id, updates);
+  
+  if (!success) {
+    return { success: false, vectorized: false };
+  }
+
+  // Re-index in vector DB
+  let vectorized = false;
+  try {
+    const updatedEntry = getKnowledgeEntry(id);
+    if (updatedEntry) {
+      await updateVector(updatedEntry);
+      vectorized = true;
+    }
+  } catch {
+    console.error("Vector re-indexing failed");
+  }
+
+  return { success, vectorized };
+}
+
+/**
+ * Delete a knowledge entry and its vector.
+ */
+export async function deleteKnowledge(id: number): Promise<boolean> {
+  // Delete vector first
+  try {
+    await deleteVector(id);
+  } catch {
+    // Continue even if vector deletion fails
+  }
+
+  return deleteKnowledgeEntry(id);
+}
+
+/**
+ * List knowledge entries with optional filters.
+ */
+export function listKnowledge(options: {
+  limit?: number;
+  offset?: number;
+  tags?: string[];
+}): KnowledgeEntry[] {
+  return listKnowledgeEntries(options);
+}
+
+/**
+ * Get combined statistics.
+ */
+export async function getKnowledgeStats(): Promise<{
+  database: ReturnType<typeof getDbStats>;
+  vectors: Awaited<ReturnType<typeof getVectorStats>>;
+}> {
+  const database = getDbStats();
+  const vectors = await getVectorStats();
+  return { database, vectors };
+}
@@ -0,0 +1,290 @@
+/**
+ * Vector Database Service
+ * 
+ * Provides semantic search over knowledge entries using LanceDB and Transformers.js embeddings.
+ * Uses all-MiniLM-L6-v2 model which auto-downloads on first use (~22MB).
+ */
+import lancedb from "@lancedb/lancedb";
+import { pipeline, type FeatureExtractionPipeline } from "@xenova/transformers";
+import { join } from "path";
+import { existsSync, mkdirSync } from "fs";
+import { homedir } from "os";
+import { getAllEntries, type KnowledgeEntry } from "../database/index.js";
+
+// Use persistent user data directory (XDG-compliant on Linux)
+const DATA_DIR = process.env.OPENCODE_PK_DATA_DIR || join(homedir(), ".local", "share", "opencode-personal-knowledge");
+const VECTOR_DB_PATH = join(DATA_DIR, "vectors");
+
+// Embedding model (auto-downloads on first use)
+const EMBEDDING_MODEL = "Xenova/all-MiniLM-L6-v2";
+
+// Singleton embedding pipeline
+let embeddingPipeline: FeatureExtractionPipeline | null = null;
+
+/**
+ * Get or initialize the embedding pipeline.
+ */
+async function getEmbeddingPipeline(): Promise<FeatureExtractionPipeline> {
+  if (!embeddingPipeline) {
+    console.error("Loading embedding model (first run may download ~22MB)...");
+    embeddingPipeline = await pipeline("feature-extraction", EMBEDDING_MODEL);
+    console.error("Embedding model loaded.");
+  }
+  return embeddingPipeline;
+}
+
+/**
+ * Generate embedding for text.
+ */
+export async function embed(text: string): Promise<number[]> {
+  const extractor = await getEmbeddingPipeline();
+  const output = await extractor(text, { pooling: "mean", normalize: true });
+  return Array.from(output.data as Float32Array);
+}
+
+/**
+ * Vector record with embedding.
+ */
+export interface VectorRecord {
+  [key: string]: unknown;
+  id: number;
+  title: string;
+  content_preview: string;
+  tags: string | null;
+  vector: number[];
+}
+
+/**
+ * Search result from vector query.
+ */
+export interface SearchResult {
+  id: number;
+  title: string;
+  content_preview: string;
+  tags: string[];
+  score: number;
+}
+
+/**
+ * Ensure data directory exists.
+ */
+function ensureDataDir(): void {
+  if (!existsSync(VECTOR_DB_PATH)) {
+    mkdirSync(VECTOR_DB_PATH, { recursive: true });
+  }
+}
+
+/**
+ * Get or create LanceDB connection.
+ */
+async function getVectorDB() {
+  ensureDataDir();
+  return await lancedb.connect(VECTOR_DB_PATH);
+}
+
+/**
+ * Convert all knowledge entries to vector database.
+ */
+export async function convertToVectorDB(options: {
+  batchSize?: number;
+  onProgress?: (current: number, total: number) => void;
+} = {}): Promise<{ converted: number; skipped: number }> {
+  const { batchSize = 50, onProgress } = options;
+
+  // Get all entries from SQLite
+  const entries = getAllEntries();
+
+  if (entries.length === 0) {
+    return { converted: 0, skipped: 0 };
+  }
+
+  const db = await getVectorDB();
+  
+  // Check for existing table
+  const tables = await db.tableNames();
+  let existingIds = new Set<number>();
+  
+  if (tables.includes("knowledge_vectors")) {
+    const table = await db.openTable("knowledge_vectors");
+    const existing = await table.query().select(["id"]).toArray();
+    existingIds = new Set(existing.map((r: { id: number }) => r.id));
+  }
+
+  // Filter out already converted entries
+  const toConvert = entries.filter((e) => e.id && !existingIds.has(e.id));
+  
+  if (toConvert.length === 0) {
+    return { converted: 0, skipped: entries.length };
+  }
+
+  // Process in batches
+  const vectorRecords: VectorRecord[] = [];
+  
+  for (let i = 0; i < toConvert.length; i += batchSize) {
+    const batch = toConvert.slice(i, i + batchSize);
+    
+    for (const entry of batch) {
+      // Combine title and content for embedding
+      const text = `${entry.title}\n${entry.content.slice(0, 1000)}`;
+      const vector = await embed(text);
+      
+      vectorRecords.push({
+        id: entry.id!,
+        title: entry.title,
+        content_preview: entry.content.slice(0, 500),
+        tags: entry.tags ? JSON.stringify(entry.tags) : null,
+        vector,
+      });
+    }
+    
+    onProgress?.(Math.min(i + batchSize, toConvert.length), toConvert.length);
+  }
+
+  // Create or append to table
+  if (tables.includes("knowledge_vectors")) {
+    const table = await db.openTable("knowledge_vectors");
+    await table.add(vectorRecords);
+  } else {
+    await db.createTable("knowledge_vectors", vectorRecords);
+  }
+
+  return { converted: vectorRecords.length, skipped: existingIds.size };
+}
+
+/**
+ * Query the vector database for similar entries.
+ */
+export async function queryVectors(
+  query: string,
+  options: {
+    limit?: number;
+    minScore?: number;
+  } = {}
+): Promise<SearchResult[]> {
+  const { limit = 5, minScore = 0.3 } = options;
+
+  const db = await getVectorDB();
+  const tables = await db.tableNames();
+  
+  if (!tables.includes("knowledge_vectors")) {
+    throw new Error("Vector database not initialized. Run 'bun start vectors convert' first.");
+  }
+
+  // Generate query embedding
+  const queryVector = await embed(query);
+
+  // Search
+  const table = await db.openTable("knowledge_vectors");
+  const results = await table
+    .vectorSearch(queryVector)
+    .limit(limit)
+    .toArray();
+
+  // Format and filter results
+  return results
+    .map((r: Record<string, unknown>) => ({
+      id: r.id as number,
+      title: r.title as string,
+      content_preview: r.content_preview as string,
+      tags: r.tags ? JSON.parse(r.tags as string) : [],
+      score: 1 - (r._distance as number), // Convert distance to similarity score
+    }))
+    .filter((r) => r.score >= minScore);
+}
+
+/**
+ * Delete a vector by entry ID.
+ */
+export async function deleteVector(id: number): Promise<boolean> {
+  const db = await getVectorDB();
+  const tables = await db.tableNames();
+  
+  if (!tables.includes("knowledge_vectors")) {
+    return false;
+  }
+
+  const table = await db.openTable("knowledge_vectors");
+  await table.delete(`id = ${id}`);
+  return true;
+}
+
+/**
+ * Update vector for a single entry.
+ */
+export async function updateVector(entry: KnowledgeEntry): Promise<boolean> {
+  if (!entry.id) return false;
+
+  // Delete old vector
+  await deleteVector(entry.id);
+
+  // Create new vector
+  const db = await getVectorDB();
+  const tables = await db.tableNames();
+  
+  const text = `${entry.title}\n${entry.content.slice(0, 1000)}`;
+  const vector = await embed(text);
+  
+  const record: VectorRecord = {
+    id: entry.id,
+    title: entry.title,
+    content_preview: entry.content.slice(0, 500),
+    tags: entry.tags ? JSON.stringify(entry.tags) : null,
+    vector,
+  };
+
+  if (tables.includes("knowledge_vectors")) {
+    const table = await db.openTable("knowledge_vectors");
+    await table.add([record]);
+  } else {
+    await db.createTable("knowledge_vectors", [record]);
+  }
+
+  return true;
+}
+
+/**
+ * Get vector database statistics.
+ */
+export async function getVectorStats(): Promise<{
+  totalVectors: number;
+  tagCounts: Record<string, number>;
+}> {
+  const db = await getVectorDB();
+  const tables = await db.tableNames();
+  
+  if (!tables.includes("knowledge_vectors")) {
+    return { totalVectors: 0, tagCounts: {} };
+  }
+
+  const table = await db.openTable("knowledge_vectors");
+  const all = await table.query().select(["tags"]).toArray();
+  
+  const tagCounts: Record<string, number> = {};
+  
+  for (const r of all) {
+    const record = r as { tags: string | null };
+    if (record.tags) {
+      const tags = JSON.parse(record.tags) as string[];
+      for (const tag of tags) {
+        tagCounts[tag] = (tagCounts[tag] || 0) + 1;
+      }
+    }
+  }
+
+  return {
+    totalVectors: all.length,
+    tagCounts,
+  };
+}
+
+/**
+ * Clear the vector database.
+ */
+export async function clearVectorDB(): Promise<void> {
+  const db = await getVectorDB();
+  const tables = await db.tableNames();
+  
+  if (tables.includes("knowledge_vectors")) {
+    await db.dropTable("knowledge_vectors");
+  }
+}