chore: scaffold project with editorconfig, ci workflows, gitignore, pre-commit, changelog, contributing guide, license, and makefile

2025-11-04 04:17:27 +00:00
commit 5d42e8d377
77 changed files with 10179 additions and 0 deletions
@@ -0,0 +1,7 @@
+from .knowledge_store import KnowledgeStore, KnowledgeEntry
+from .semantic_index import SemanticIndex
+from .conversation_memory import ConversationMemory
+from .fact_extractor import FactExtractor
+
+__all__ = ['KnowledgeStore', 'KnowledgeEntry', 'SemanticIndex',
+           'ConversationMemory', 'FactExtractor']
@@ -0,0 +1,259 @@
+import json
+import sqlite3
+import time
+from typing import List, Dict, Any, Optional
+
+class ConversationMemory:
+    def __init__(self, db_path: str):
+        self.db_path = db_path
+        self._initialize_memory()
+
+    def _initialize_memory(self):
+        conn = sqlite3.connect(self.db_path, check_same_thread=False)
+        cursor = conn.cursor()
+
+        cursor.execute('''
+            CREATE TABLE IF NOT EXISTS conversation_history (
+                conversation_id TEXT PRIMARY KEY,
+                session_id TEXT,
+                started_at REAL NOT NULL,
+                ended_at REAL,
+                message_count INTEGER DEFAULT 0,
+                summary TEXT,
+                topics TEXT,
+                metadata TEXT
+            )
+        ''')
+
+        cursor.execute('''
+            CREATE TABLE IF NOT EXISTS conversation_messages (
+                message_id TEXT PRIMARY KEY,
+                conversation_id TEXT NOT NULL,
+                role TEXT NOT NULL,
+                content TEXT NOT NULL,
+                timestamp REAL NOT NULL,
+                tool_calls TEXT,
+                metadata TEXT,
+                FOREIGN KEY (conversation_id) REFERENCES conversation_history(conversation_id)
+            )
+        ''')
+
+        cursor.execute('''
+            CREATE INDEX IF NOT EXISTS idx_conv_session ON conversation_history(session_id)
+        ''')
+        cursor.execute('''
+            CREATE INDEX IF NOT EXISTS idx_conv_started ON conversation_history(started_at DESC)
+        ''')
+        cursor.execute('''
+            CREATE INDEX IF NOT EXISTS idx_msg_conversation ON conversation_messages(conversation_id)
+        ''')
+        cursor.execute('''
+            CREATE INDEX IF NOT EXISTS idx_msg_timestamp ON conversation_messages(timestamp)
+        ''')
+
+        conn.commit()
+        conn.close()
+
+    def create_conversation(self, conversation_id: str, session_id: Optional[str] = None,
+                           metadata: Optional[Dict[str, Any]] = None):
+        conn = sqlite3.connect(self.db_path, check_same_thread=False)
+        cursor = conn.cursor()
+
+        cursor.execute('''
+            INSERT INTO conversation_history
+            (conversation_id, session_id, started_at, metadata)
+            VALUES (?, ?, ?, ?)
+        ''', (
+            conversation_id,
+            session_id,
+            time.time(),
+            json.dumps(metadata) if metadata else None
+        ))
+
+        conn.commit()
+        conn.close()
+
+    def add_message(self, conversation_id: str, message_id: str, role: str,
+                   content: str, tool_calls: Optional[List[Dict[str, Any]]] = None,
+                   metadata: Optional[Dict[str, Any]] = None):
+        conn = sqlite3.connect(self.db_path, check_same_thread=False)
+        cursor = conn.cursor()
+
+        cursor.execute('''
+            INSERT INTO conversation_messages
+            (message_id, conversation_id, role, content, timestamp, tool_calls, metadata)
+            VALUES (?, ?, ?, ?, ?, ?, ?)
+        ''', (
+            message_id,
+            conversation_id,
+            role,
+            content,
+            time.time(),
+            json.dumps(tool_calls) if tool_calls else None,
+            json.dumps(metadata) if metadata else None
+        ))
+
+        cursor.execute('''
+            UPDATE conversation_history
+            SET message_count = message_count + 1
+            WHERE conversation_id = ?
+        ''', (conversation_id,))
+
+        conn.commit()
+        conn.close()
+
+    def get_conversation_messages(self, conversation_id: str,
+                                 limit: Optional[int] = None) -> List[Dict[str, Any]]:
+        conn = sqlite3.connect(self.db_path, check_same_thread=False)
+        cursor = conn.cursor()
+
+        if limit:
+            cursor.execute('''
+                SELECT message_id, role, content, timestamp, tool_calls, metadata
+                FROM conversation_messages
+                WHERE conversation_id = ?
+                ORDER BY timestamp DESC
+                LIMIT ?
+            ''', (conversation_id, limit))
+        else:
+            cursor.execute('''
+                SELECT message_id, role, content, timestamp, tool_calls, metadata
+                FROM conversation_messages
+                WHERE conversation_id = ?
+                ORDER BY timestamp ASC
+            ''', (conversation_id,))
+
+        messages = []
+        for row in cursor.fetchall():
+            messages.append({
+                'message_id': row[0],
+                'role': row[1],
+                'content': row[2],
+                'timestamp': row[3],
+                'tool_calls': json.loads(row[4]) if row[4] else None,
+                'metadata': json.loads(row[5]) if row[5] else None
+            })
+
+        conn.close()
+        return messages
+
+    def update_conversation_summary(self, conversation_id: str, summary: str,
+                                   topics: Optional[List[str]] = None):
+        conn = sqlite3.connect(self.db_path, check_same_thread=False)
+        cursor = conn.cursor()
+
+        cursor.execute('''
+            UPDATE conversation_history
+            SET summary = ?, topics = ?, ended_at = ?
+            WHERE conversation_id = ?
+        ''', (summary, json.dumps(topics) if topics else None, time.time(), conversation_id))
+
+        conn.commit()
+        conn.close()
+
+    def search_conversations(self, query: str, limit: int = 10) -> List[Dict[str, Any]]:
+        conn = sqlite3.connect(self.db_path, check_same_thread=False)
+        cursor = conn.cursor()
+
+        cursor.execute('''
+            SELECT DISTINCT h.conversation_id, h.session_id, h.started_at,
+                   h.message_count, h.summary, h.topics
+            FROM conversation_history h
+            LEFT JOIN conversation_messages m ON h.conversation_id = m.conversation_id
+            WHERE h.summary LIKE ? OR h.topics LIKE ? OR m.content LIKE ?
+            ORDER BY h.started_at DESC
+            LIMIT ?
+        ''', (f'%{query}%', f'%{query}%', f'%{query}%', limit))
+
+        conversations = []
+        for row in cursor.fetchall():
+            conversations.append({
+                'conversation_id': row[0],
+                'session_id': row[1],
+                'started_at': row[2],
+                'message_count': row[3],
+                'summary': row[4],
+                'topics': json.loads(row[5]) if row[5] else []
+            })
+
+        conn.close()
+        return conversations
+
+    def get_recent_conversations(self, limit: int = 10,
+                                session_id: Optional[str] = None) -> List[Dict[str, Any]]:
+        conn = sqlite3.connect(self.db_path, check_same_thread=False)
+        cursor = conn.cursor()
+
+        if session_id:
+            cursor.execute('''
+                SELECT conversation_id, session_id, started_at, ended_at,
+                       message_count, summary, topics
+                FROM conversation_history
+                WHERE session_id = ?
+                ORDER BY started_at DESC
+                LIMIT ?
+            ''', (session_id, limit))
+        else:
+            cursor.execute('''
+                SELECT conversation_id, session_id, started_at, ended_at,
+                       message_count, summary, topics
+                FROM conversation_history
+                ORDER BY started_at DESC
+                LIMIT ?
+            ''', (limit,))
+
+        conversations = []
+        for row in cursor.fetchall():
+            conversations.append({
+                'conversation_id': row[0],
+                'session_id': row[1],
+                'started_at': row[2],
+                'ended_at': row[3],
+                'message_count': row[4],
+                'summary': row[5],
+                'topics': json.loads(row[6]) if row[6] else []
+            })
+
+        conn.close()
+        return conversations
+
+    def delete_conversation(self, conversation_id: str) -> bool:
+        conn = sqlite3.connect(self.db_path, check_same_thread=False)
+        cursor = conn.cursor()
+
+        cursor.execute('DELETE FROM conversation_messages WHERE conversation_id = ?',
+                      (conversation_id,))
+        cursor.execute('DELETE FROM conversation_history WHERE conversation_id = ?',
+                      (conversation_id,))
+
+        deleted = cursor.rowcount > 0
+        conn.commit()
+        conn.close()
+
+        return deleted
+
+    def get_statistics(self) -> Dict[str, Any]:
+        conn = sqlite3.connect(self.db_path, check_same_thread=False)
+        cursor = conn.cursor()
+
+        cursor.execute('SELECT COUNT(*) FROM conversation_history')
+        total_conversations = cursor.fetchone()[0]
+
+        cursor.execute('SELECT COUNT(*) FROM conversation_messages')
+        total_messages = cursor.fetchone()[0]
+
+        cursor.execute('SELECT SUM(message_count) FROM conversation_history')
+        total_message_count = cursor.fetchone()[0] or 0
+
+        cursor.execute('''
+            SELECT AVG(message_count) FROM conversation_history WHERE message_count > 0
+        ''')
+        avg_messages = cursor.fetchone()[0] or 0
+
+        conn.close()
+
+        return {
+            'total_conversations': total_conversations,
+            'total_messages': total_messages,
+            'average_messages_per_conversation': round(avg_messages, 2)
+        }
@@ -0,0 +1,146 @@
+import re
+import json
+from typing import List, Dict, Any, Set
+from collections import defaultdict
+
+class FactExtractor:
+    def __init__(self):
+        self.fact_patterns = [
+            (r'([A-Z][a-z]+ [A-Z][a-z]+) is (a|an) ([^.]+)', 'definition'),
+            (r'([A-Z][a-z]+) (was|is) (born|created|founded) in (\d{4})', 'temporal'),
+            (r'([A-Z][a-z]+) (invented|created|developed) ([^.]+)', 'attribution'),
+            (r'([^.]+) (costs?|worth) (\$[\d,]+)', 'numeric'),
+            (r'([A-Z][a-z]+) (lives?|works?|located) in ([A-Z][a-z]+)', 'location'),
+        ]
+
+    def extract_facts(self, text: str) -> List[Dict[str, Any]]:
+        facts = []
+
+        for pattern, fact_type in self.fact_patterns:
+            matches = re.finditer(pattern, text)
+            for match in matches:
+                facts.append({
+                    'type': fact_type,
+                    'text': match.group(0),
+                    'components': match.groups(),
+                    'confidence': 0.7
+                })
+
+        noun_phrases = self._extract_noun_phrases(text)
+        for phrase in noun_phrases:
+            if len(phrase.split()) >= 2:
+                facts.append({
+                    'type': 'entity',
+                    'text': phrase,
+                    'components': [phrase],
+                    'confidence': 0.5
+                })
+
+        return facts
+
+    def _extract_noun_phrases(self, text: str) -> List[str]:
+        sentences = re.split(r'[.!?]', text)
+        phrases = []
+
+        for sentence in sentences:
+            words = sentence.split()
+            current_phrase = []
+
+            for word in words:
+                if word and word[0].isupper() and len(word) > 1:
+                    current_phrase.append(word)
+                else:
+                    if len(current_phrase) >= 2:
+                        phrases.append(' '.join(current_phrase))
+                    current_phrase = []
+
+            if len(current_phrase) >= 2:
+                phrases.append(' '.join(current_phrase))
+
+        return list(set(phrases))
+
+    def extract_key_terms(self, text: str, top_k: int = 10) -> List[tuple]:
+        words = re.findall(r'\b[a-z]{4,}\b', text.lower())
+
+        stopwords = {
+            'this', 'that', 'these', 'those', 'what', 'which', 'where', 'when',
+            'with', 'from', 'have', 'been', 'were', 'will', 'would', 'could',
+            'should', 'about', 'their', 'there', 'other', 'than', 'then', 'them',
+            'some', 'more', 'very', 'such', 'into', 'through', 'during', 'before',
+            'after', 'above', 'below', 'between', 'under', 'again', 'further',
+            'once', 'here', 'both', 'each', 'doing', 'only', 'over', 'same',
+            'being', 'does', 'just', 'also', 'make', 'made', 'know', 'like'
+        }
+
+        filtered_words = [w for w in words if w not in stopwords]
+
+        word_freq = defaultdict(int)
+        for word in filtered_words:
+            word_freq[word] += 1
+
+        sorted_terms = sorted(word_freq.items(), key=lambda x: x[1], reverse=True)
+        return sorted_terms[:top_k]
+
+    def extract_relationships(self, text: str) -> List[Dict[str, Any]]:
+        relationships = []
+
+        relationship_patterns = [
+            (r'([A-Z][a-z]+) (works for|employed by|member of) ([A-Z][a-z]+)', 'employment'),
+            (r'([A-Z][a-z]+) (owns|has|possesses) ([^.]+)', 'ownership'),
+            (r'([A-Z][a-z]+) (located in|part of|belongs to) ([A-Z][a-z]+)', 'location'),
+            (r'([A-Z][a-z]+) (uses|utilizes|implements) ([^.]+)', 'usage'),
+        ]
+
+        for pattern, rel_type in relationship_patterns:
+            matches = re.finditer(pattern, text)
+            for match in matches:
+                relationships.append({
+                    'type': rel_type,
+                    'subject': match.group(1),
+                    'predicate': match.group(2),
+                    'object': match.group(3),
+                    'confidence': 0.6
+                })
+
+        return relationships
+
+    def extract_metadata(self, text: str) -> Dict[str, Any]:
+        word_count = len(text.split())
+        sentence_count = len(re.split(r'[.!?]', text))
+
+        urls = re.findall(r'https?://[^\s]+', text)
+        email_addresses = re.findall(r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b', text)
+        dates = re.findall(r'\b\d{1,2}[-/]\d{1,2}[-/]\d{2,4}\b|\b\d{4}[-/]\d{1,2}[-/]\d{1,2}\b', text)
+        numbers = re.findall(r'\b\d+(?:,\d{3})*(?:\.\d+)?\b', text)
+
+        return {
+            'word_count': word_count,
+            'sentence_count': sentence_count,
+            'avg_words_per_sentence': round(word_count / max(sentence_count, 1), 2),
+            'urls': urls,
+            'email_addresses': email_addresses,
+            'dates': dates,
+            'numeric_values': numbers,
+            'has_code': bool(re.search(r'```|def |class |import |function ', text)),
+            'has_questions': bool(re.search(r'\?', text))
+        }
+
+    def categorize_content(self, text: str) -> List[str]:
+        categories = []
+
+        category_keywords = {
+            'programming': ['code', 'function', 'class', 'variable', 'programming', 'software', 'debug'],
+            'data': ['data', 'database', 'query', 'table', 'record', 'statistics', 'analysis'],
+            'documentation': ['documentation', 'guide', 'tutorial', 'manual', 'readme', 'explain'],
+            'configuration': ['config', 'settings', 'configuration', 'setup', 'install', 'deployment'],
+            'testing': ['test', 'testing', 'validate', 'verification', 'quality', 'assertion'],
+            'research': ['research', 'study', 'analysis', 'investigation', 'findings', 'results'],
+            'planning': ['plan', 'planning', 'schedule', 'roadmap', 'milestone', 'timeline'],
+        }
+
+        text_lower = text.lower()
+        for category, keywords in category_keywords.items():
+            if any(keyword in text_lower for keyword in keywords):
+                categories.append(category)
+
+        return categories if categories else ['general']
@@ -0,0 +1,265 @@
+import json
+import sqlite3
+import time
+from typing import List, Dict, Any, Optional
+from dataclasses import dataclass
+from .semantic_index import SemanticIndex
+
+@dataclass
+class KnowledgeEntry:
+    entry_id: str
+    category: str
+    content: str
+    metadata: Dict[str, Any]
+    created_at: float
+    updated_at: float
+    access_count: int = 0
+    importance_score: float = 1.0
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            'entry_id': self.entry_id,
+            'category': self.category,
+            'content': self.content,
+            'metadata': self.metadata,
+            'created_at': self.created_at,
+            'updated_at': self.updated_at,
+            'access_count': self.access_count,
+            'importance_score': self.importance_score
+        }
+
+class KnowledgeStore:
+    def __init__(self, db_path: str):
+        self.db_path = db_path
+        self.semantic_index = SemanticIndex()
+        self._initialize_store()
+        self._load_index()
+
+    def _initialize_store(self):
+        conn = sqlite3.connect(self.db_path, check_same_thread=False)
+        cursor = conn.cursor()
+
+        cursor.execute('''
+            CREATE TABLE IF NOT EXISTS knowledge_entries (
+                entry_id TEXT PRIMARY KEY,
+                category TEXT NOT NULL,
+                content TEXT NOT NULL,
+                metadata TEXT,
+                created_at REAL NOT NULL,
+                updated_at REAL NOT NULL,
+                access_count INTEGER DEFAULT 0,
+                importance_score REAL DEFAULT 1.0
+            )
+        ''')
+
+        cursor.execute('''
+            CREATE INDEX IF NOT EXISTS idx_category ON knowledge_entries(category)
+        ''')
+        cursor.execute('''
+            CREATE INDEX IF NOT EXISTS idx_importance ON knowledge_entries(importance_score DESC)
+        ''')
+        cursor.execute('''
+            CREATE INDEX IF NOT EXISTS idx_created ON knowledge_entries(created_at DESC)
+        ''')
+
+        conn.commit()
+        conn.close()
+
+    def _load_index(self):
+        conn = sqlite3.connect(self.db_path, check_same_thread=False)
+        cursor = conn.cursor()
+
+        cursor.execute('SELECT entry_id, content FROM knowledge_entries')
+        for row in cursor.fetchall():
+            self.semantic_index.add_document(row[0], row[1])
+
+        conn.close()
+
+    def add_entry(self, entry: KnowledgeEntry):
+        conn = sqlite3.connect(self.db_path, check_same_thread=False)
+        cursor = conn.cursor()
+
+        cursor.execute('''
+            INSERT OR REPLACE INTO knowledge_entries
+            (entry_id, category, content, metadata, created_at, updated_at, access_count, importance_score)
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+        ''', (
+            entry.entry_id,
+            entry.category,
+            entry.content,
+            json.dumps(entry.metadata),
+            entry.created_at,
+            entry.updated_at,
+            entry.access_count,
+            entry.importance_score
+        ))
+
+        conn.commit()
+        conn.close()
+
+        self.semantic_index.add_document(entry.entry_id, entry.content)
+
+    def get_entry(self, entry_id: str) -> Optional[KnowledgeEntry]:
+        conn = sqlite3.connect(self.db_path, check_same_thread=False)
+        cursor = conn.cursor()
+
+        cursor.execute('''
+            SELECT entry_id, category, content, metadata, created_at, updated_at, access_count, importance_score
+            FROM knowledge_entries
+            WHERE entry_id = ?
+        ''', (entry_id,))
+
+        row = cursor.fetchone()
+
+        if row:
+            cursor.execute('''
+                UPDATE knowledge_entries
+                SET access_count = access_count + 1
+                WHERE entry_id = ?
+            ''', (entry_id,))
+            conn.commit()
+
+            conn.close()
+
+            return KnowledgeEntry(
+                entry_id=row[0],
+                category=row[1],
+                content=row[2],
+                metadata=json.loads(row[3]) if row[3] else {},
+                created_at=row[4],
+                updated_at=row[5],
+                access_count=row[6] + 1,
+                importance_score=row[7]
+            )
+
+        conn.close()
+        return None
+
+    def search_entries(self, query: str, category: Optional[str] = None,
+                      top_k: int = 5) -> List[KnowledgeEntry]:
+        search_results = self.semantic_index.search(query, top_k * 2)
+
+        conn = sqlite3.connect(self.db_path, check_same_thread=False)
+        cursor = conn.cursor()
+
+        entries = []
+        for entry_id, score in search_results:
+            if category:
+                cursor.execute('''
+                    SELECT entry_id, category, content, metadata, created_at, updated_at, access_count, importance_score
+                    FROM knowledge_entries
+                    WHERE entry_id = ? AND category = ?
+                ''', (entry_id, category))
+            else:
+                cursor.execute('''
+                    SELECT entry_id, category, content, metadata, created_at, updated_at, access_count, importance_score
+                    FROM knowledge_entries
+                    WHERE entry_id = ?
+                ''', (entry_id,))
+
+            row = cursor.fetchone()
+            if row:
+                entry = KnowledgeEntry(
+                    entry_id=row[0],
+                    category=row[1],
+                    content=row[2],
+                    metadata=json.loads(row[3]) if row[3] else {},
+                    created_at=row[4],
+                    updated_at=row[5],
+                    access_count=row[6],
+                    importance_score=row[7]
+                )
+                entries.append(entry)
+
+            if len(entries) >= top_k:
+                break
+
+        conn.close()
+        return entries
+
+    def get_by_category(self, category: str, limit: int = 20) -> List[KnowledgeEntry]:
+        conn = sqlite3.connect(self.db_path, check_same_thread=False)
+        cursor = conn.cursor()
+
+        cursor.execute('''
+            SELECT entry_id, category, content, metadata, created_at, updated_at, access_count, importance_score
+            FROM knowledge_entries
+            WHERE category = ?
+            ORDER BY importance_score DESC, created_at DESC
+            LIMIT ?
+        ''', (category, limit))
+
+        entries = []
+        for row in cursor.fetchall():
+            entries.append(KnowledgeEntry(
+                entry_id=row[0],
+                category=row[1],
+                content=row[2],
+                metadata=json.loads(row[3]) if row[3] else {},
+                created_at=row[4],
+                updated_at=row[5],
+                access_count=row[6],
+                importance_score=row[7]
+            ))
+
+        conn.close()
+        return entries
+
+    def update_importance(self, entry_id: str, importance_score: float):
+        conn = sqlite3.connect(self.db_path, check_same_thread=False)
+        cursor = conn.cursor()
+
+        cursor.execute('''
+            UPDATE knowledge_entries
+            SET importance_score = ?, updated_at = ?
+            WHERE entry_id = ?
+        ''', (importance_score, time.time(), entry_id))
+
+        conn.commit()
+        conn.close()
+
+    def delete_entry(self, entry_id: str) -> bool:
+        conn = sqlite3.connect(self.db_path, check_same_thread=False)
+        cursor = conn.cursor()
+
+        cursor.execute('DELETE FROM knowledge_entries WHERE entry_id = ?', (entry_id,))
+        deleted = cursor.rowcount > 0
+
+        conn.commit()
+        conn.close()
+
+        if deleted:
+            self.semantic_index.remove_document(entry_id)
+
+        return deleted
+
+    def get_statistics(self) -> Dict[str, Any]:
+        conn = sqlite3.connect(self.db_path, check_same_thread=False)
+        cursor = conn.cursor()
+
+        cursor.execute('SELECT COUNT(*) FROM knowledge_entries')
+        total_entries = cursor.fetchone()[0]
+
+        cursor.execute('SELECT COUNT(DISTINCT category) FROM knowledge_entries')
+        total_categories = cursor.fetchone()[0]
+
+        cursor.execute('''
+            SELECT category, COUNT(*) as count
+            FROM knowledge_entries
+            GROUP BY category
+            ORDER BY count DESC
+        ''')
+        category_counts = {row[0]: row[1] for row in cursor.fetchall()}
+
+        cursor.execute('SELECT SUM(access_count) FROM knowledge_entries')
+        total_accesses = cursor.fetchone()[0] or 0
+
+        conn.close()
+
+        return {
+            'total_entries': total_entries,
+            'total_categories': total_categories,
+            'category_distribution': category_counts,
+            'total_accesses': total_accesses,
+            'vocabulary_size': len(self.semantic_index.vocabulary)
+        }
@@ -0,0 +1,87 @@
+import math
+import re
+from collections import Counter, defaultdict
+from typing import List, Dict, Tuple, Set
+
+class SemanticIndex:
+    def __init__(self):
+        self.documents: Dict[str, str] = {}
+        self.vocabulary: Set[str] = set()
+        self.idf_scores: Dict[str, float] = {}
+        self.doc_vectors: Dict[str, Dict[str, float]] = {}
+
+    def _tokenize(self, text: str) -> List[str]:
+        text = text.lower()
+        text = re.sub(r'[^a-z0-9\s]', ' ', text)
+        tokens = text.split()
+        return tokens
+
+    def _compute_tf(self, tokens: List[str]) -> Dict[str, float]:
+        term_count = Counter(tokens)
+        total_terms = len(tokens)
+        return {term: count / total_terms for term, count in term_count.items()}
+
+    def _compute_idf(self):
+        doc_count = len(self.documents)
+        if doc_count == 0:
+            return
+
+        token_doc_count = defaultdict(int)
+
+        for doc_id, doc_text in self.documents.items():
+            tokens = set(self._tokenize(doc_text))
+            for token in tokens:
+                token_doc_count[token] += 1
+
+        if doc_count == 1:
+            self.idf_scores = {token: 1.0 for token in token_doc_count}
+        else:
+            self.idf_scores = {
+                token: math.log(doc_count / count)
+                for token, count in token_doc_count.items()
+            }
+
+    def add_document(self, doc_id: str, text: str):
+        self.documents[doc_id] = text
+        tokens = self._tokenize(text)
+        self.vocabulary.update(tokens)
+
+        self._compute_idf()
+
+        tf_scores = self._compute_tf(tokens)
+        self.doc_vectors[doc_id] = {
+            token: tf_scores.get(token, 0) * self.idf_scores.get(token, 0)
+            for token in tokens
+        }
+
+    def remove_document(self, doc_id: str):
+        if doc_id in self.documents:
+            del self.documents[doc_id]
+            if doc_id in self.doc_vectors:
+                del self.doc_vectors[doc_id]
+            self._compute_idf()
+
+    def search(self, query: str, top_k: int = 5) -> List[Tuple[str, float]]:
+        query_tokens = self._tokenize(query)
+        query_tf = self._compute_tf(query_tokens)
+
+        query_vector = {
+            token: query_tf.get(token, 0) * self.idf_scores.get(token, 0)
+            for token in query_tokens
+        }
+
+        scores = []
+        for doc_id, doc_vector in self.doc_vectors.items():
+            similarity = self._cosine_similarity(query_vector, doc_vector)
+            scores.append((doc_id, similarity))
+
+        scores.sort(key=lambda x: x[1], reverse=True)
+        return scores[:top_k]
+
+    def _cosine_similarity(self, vec1: Dict[str, float], vec2: Dict[str, float]) -> float:
+        dot_product = sum(vec1.get(token, 0) * vec2.get(token, 0) for token in set(vec1) | set(vec2))
+        norm1 = math.sqrt(sum(val**2 for val in vec1.values()))
+        norm2 = math.sqrt(sum(val**2 for val in vec2.values()))
+        if norm1 == 0 or norm2 == 0:
+            return 0
+        return dot_product / (norm1 * norm2)