retoor/rp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: standardize string quotes and fix import ordering across multiple modules

Browse Source
This commit is contained in:
retoor
2025-11-04 07:09:12 +00:00
parent ea29bdc403
commit e9ced4a493
82 changed files with 4963 additions and 3094 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pr/memory/__init__.py
+9 -4
View File
@@ -1,7 +1,12 @@
from .knowledge_store import KnowledgeStore, KnowledgeEntry
from .semantic_index import SemanticIndex
from .conversation_memory import ConversationMemory
from .fact_extractor import FactExtractor
from .knowledge_store import KnowledgeEntry, KnowledgeStore
from .semantic_index import SemanticIndex
__all__ = ['KnowledgeStore', 'KnowledgeEntry', 'SemanticIndex',
'ConversationMemory', 'FactExtractor']
__all__ = [
"KnowledgeStore",
"KnowledgeEntry",
"SemanticIndex",
"ConversationMemory",
"FactExtractor",
]
pr/memory/conversation_memory.py
+163 -93
View File
@@ -1,7 +1,8 @@
import json
import sqlite3
import time
from typing import List, Dict, Any, Optional
from typing import Any, Dict, List, Optional
class ConversationMemory:
def __init__(self, db_path: str):
@@ -12,7 +13,8 @@ class ConversationMemory:
conn = sqlite3.connect(self.db_path, check_same_thread=False)
cursor = conn.cursor()
cursor.execute('''
cursor.execute(
"""
CREATE TABLE IF NOT EXISTS conversation_history (
conversation_id TEXT PRIMARY KEY,
session_id TEXT,
@@ -23,9 +25,11 @@ class ConversationMemory:
topics TEXT,
metadata TEXT
)
''')
"""
)
cursor.execute('''
cursor.execute(
"""
CREATE TABLE IF NOT EXISTS conversation_messages (
message_id TEXT PRIMARY KEY,
conversation_id TEXT NOT NULL,
@@ -36,117 +40,163 @@ class ConversationMemory:
metadata TEXT,
FOREIGN KEY (conversation_id) REFERENCES conversation_history(conversation_id)
)
''')
"""
)
cursor.execute('''
cursor.execute(
"""
CREATE INDEX IF NOT EXISTS idx_conv_session ON conversation_history(session_id)
''')
cursor.execute('''
"""
)
cursor.execute(
"""
CREATE INDEX IF NOT EXISTS idx_conv_started ON conversation_history(started_at DESC)
''')
cursor.execute('''
"""
)
cursor.execute(
"""
CREATE INDEX IF NOT EXISTS idx_msg_conversation ON conversation_messages(conversation_id)
''')
cursor.execute('''
"""
)
cursor.execute(
"""
CREATE INDEX IF NOT EXISTS idx_msg_timestamp ON conversation_messages(timestamp)
''')
"""
)
conn.commit()
conn.close()
def create_conversation(self, conversation_id: str, session_id: Optional[str] = None,
metadata: Optional[Dict[str, Any]] = None):
def create_conversation(
self,
conversation_id: str,
session_id: Optional[str] = None,
metadata: Optional[Dict[str, Any]] = None,
):
conn = sqlite3.connect(self.db_path, check_same_thread=False)
cursor = conn.cursor()
cursor.execute('''
cursor.execute(
"""
INSERT INTO conversation_history
(conversation_id, session_id, started_at, metadata)
VALUES (?, ?, ?, ?)
''', (
conversation_id,
session_id,
time.time(),
json.dumps(metadata) if metadata else None
))
""",
(
conversation_id,
session_id,
time.time(),
json.dumps(metadata) if metadata else None,
),
)
conn.commit()
conn.close()
def add_message(self, conversation_id: str, message_id: str, role: str,
content: str, tool_calls: Optional[List[Dict[str, Any]]] = None,
metadata: Optional[Dict[str, Any]] = None):
def add_message(
self,
conversation_id: str,
message_id: str,
role: str,
content: str,
tool_calls: Optional[List[Dict[str, Any]]] = None,
metadata: Optional[Dict[str, Any]] = None,
):
conn = sqlite3.connect(self.db_path, check_same_thread=False)
cursor = conn.cursor()
cursor.execute('''
cursor.execute(
"""
INSERT INTO conversation_messages
(message_id, conversation_id, role, content, timestamp, tool_calls, metadata)
VALUES (?, ?, ?, ?, ?, ?, ?)
''', (
message_id,
conversation_id,
role,
content,
time.time(),
json.dumps(tool_calls) if tool_calls else None,
json.dumps(metadata) if metadata else None
))
""",
(
message_id,
conversation_id,
role,
content,
time.time(),
json.dumps(tool_calls) if tool_calls else None,
json.dumps(metadata) if metadata else None,
),
)
cursor.execute('''
cursor.execute(
"""
UPDATE conversation_history
SET message_count = message_count + 1
WHERE conversation_id = ?
''', (conversation_id,))
""",
(conversation_id,),
)
conn.commit()
conn.close()
def get_conversation_messages(self, conversation_id: str,
limit: Optional[int] = None) -> List[Dict[str, Any]]:
def get_conversation_messages(
self, conversation_id: str, limit: Optional[int] = None
) -> List[Dict[str, Any]]:
conn = sqlite3.connect(self.db_path, check_same_thread=False)
cursor = conn.cursor()
if limit:
cursor.execute('''
cursor.execute(
"""
SELECT message_id, role, content, timestamp, tool_calls, metadata
FROM conversation_messages
WHERE conversation_id = ?
ORDER BY timestamp DESC
LIMIT ?
''', (conversation_id, limit))
""",
(conversation_id, limit),
)
else:
cursor.execute('''
cursor.execute(
"""
SELECT message_id, role, content, timestamp, tool_calls, metadata
FROM conversation_messages
WHERE conversation_id = ?
ORDER BY timestamp ASC
''', (conversation_id,))
""",
(conversation_id,),
)
messages = []
for row in cursor.fetchall():
messages.append({
'message_id': row[0],
'role': row[1],
'content': row[2],
'timestamp': row[3],
'tool_calls': json.loads(row[4]) if row[4] else None,
'metadata': json.loads(row[5]) if row[5] else None
})
messages.append(
{
"message_id": row[0],
"role": row[1],
"content": row[2],
"timestamp": row[3],
"tool_calls": json.loads(row[4]) if row[4] else None,
"metadata": json.loads(row[5]) if row[5] else None,
}
)
conn.close()
return messages
def update_conversation_summary(self, conversation_id: str, summary: str,
topics: Optional[List[str]] = None):
def update_conversation_summary(
self, conversation_id: str, summary: str, topics: Optional[List[str]] = None
):
conn = sqlite3.connect(self.db_path, check_same_thread=False)
cursor = conn.cursor()
cursor.execute('''
cursor.execute(
"""
UPDATE conversation_history
SET summary = ?, topics = ?, ended_at = ?
WHERE conversation_id = ?
''', (summary, json.dumps(topics) if topics else None, time.time(), conversation_id))
""",
(
summary,
json.dumps(topics) if topics else None,
time.time(),
conversation_id,
),
)
conn.commit()
conn.close()
@@ -155,7 +205,8 @@ class ConversationMemory:
conn = sqlite3.connect(self.db_path, check_same_thread=False)
cursor = conn.cursor()
cursor.execute('''
cursor.execute(
"""
SELECT DISTINCT h.conversation_id, h.session_id, h.started_at,
h.message_count, h.summary, h.topics
FROM conversation_history h
@@ -163,56 +214,69 @@ class ConversationMemory:
WHERE h.summary LIKE ? OR h.topics LIKE ? OR m.content LIKE ?
ORDER BY h.started_at DESC
LIMIT ?
''', (f'%{query}%', f'%{query}%', f'%{query}%', limit))
""",
(f"%{query}%", f"%{query}%", f"%{query}%", limit),
)
conversations = []
for row in cursor.fetchall():
conversations.append({
'conversation_id': row[0],
'session_id': row[1],
'started_at': row[2],
'message_count': row[3],
'summary': row[4],
'topics': json.loads(row[5]) if row[5] else []
})
conversations.append(
{
"conversation_id": row[0],
"session_id": row[1],
"started_at": row[2],
"message_count": row[3],
"summary": row[4],
"topics": json.loads(row[5]) if row[5] else [],
}
)
conn.close()
return conversations
def get_recent_conversations(self, limit: int = 10,
session_id: Optional[str] = None) -> List[Dict[str, Any]]:
def get_recent_conversations(
self, limit: int = 10, session_id: Optional[str] = None
) -> List[Dict[str, Any]]:
conn = sqlite3.connect(self.db_path, check_same_thread=False)
cursor = conn.cursor()
if session_id:
cursor.execute('''
cursor.execute(
"""
SELECT conversation_id, session_id, started_at, ended_at,
message_count, summary, topics
FROM conversation_history
WHERE session_id = ?
ORDER BY started_at DESC
LIMIT ?
''', (session_id, limit))
""",
(session_id, limit),
)
else:
cursor.execute('''
cursor.execute(
"""
SELECT conversation_id, session_id, started_at, ended_at,
message_count, summary, topics
FROM conversation_history
ORDER BY started_at DESC
LIMIT ?
''', (limit,))
""",
(limit,),
)
conversations = []
for row in cursor.fetchall():
conversations.append({
'conversation_id': row[0],
'session_id': row[1],
'started_at': row[2],
'ended_at': row[3],
'message_count': row[4],
'summary': row[5],
'topics': json.loads(row[6]) if row[6] else []
})
conversations.append(
{
"conversation_id": row[0],
"session_id": row[1],
"started_at": row[2],
"ended_at": row[3],
"message_count": row[4],
"summary": row[5],
"topics": json.loads(row[6]) if row[6] else [],
}
)
conn.close()
return conversations
@@ -221,10 +285,14 @@ class ConversationMemory:
conn = sqlite3.connect(self.db_path, check_same_thread=False)
cursor = conn.cursor()
cursor.execute('DELETE FROM conversation_messages WHERE conversation_id = ?',
(conversation_id,))
cursor.execute('DELETE FROM conversation_history WHERE conversation_id = ?',
(conversation_id,))
cursor.execute(
"DELETE FROM conversation_messages WHERE conversation_id = ?",
(conversation_id,),
)
cursor.execute(
"DELETE FROM conversation_history WHERE conversation_id = ?",
(conversation_id,),
)
deleted = cursor.rowcount > 0
conn.commit()
@@ -236,24 +304,26 @@ class ConversationMemory:
conn = sqlite3.connect(self.db_path, check_same_thread=False)
cursor = conn.cursor()
cursor.execute('SELECT COUNT(*) FROM conversation_history')
cursor.execute("SELECT COUNT(*) FROM conversation_history")
total_conversations = cursor.fetchone()[0]
cursor.execute('SELECT COUNT(*) FROM conversation_messages')
cursor.execute("SELECT COUNT(*) FROM conversation_messages")
total_messages = cursor.fetchone()[0]
cursor.execute('SELECT SUM(message_count) FROM conversation_history')
total_message_count = cursor.fetchone()[0] or 0
cursor.execute("SELECT SUM(message_count) FROM conversation_history")
cursor.fetchone()[0] or 0
cursor.execute('''
cursor.execute(
"""
SELECT AVG(message_count) FROM conversation_history WHERE message_count > 0
''')
"""
)
avg_messages = cursor.fetchone()[0] or 0
conn.close()
return {
'total_conversations': total_conversations,
'total_messages': total_messages,
'average_messages_per_conversation': round(avg_messages, 2)
"total_conversations": total_conversations,
"total_messages": total_messages,
"average_messages_per_conversation": round(avg_messages, 2),
}
pr/memory/fact_extractor.py
+178 -63
View File
@@ -1,16 +1,16 @@
import re
import json
from typing import List, Dict, Any, Set
from collections import defaultdict
from typing import Any, Dict, List
class FactExtractor:
def __init__(self):
self.fact_patterns = [
(r'([A-Z][a-z]+ [A-Z][a-z]+) is (a|an) ([^.]+)', 'definition'),
(r'([A-Z][a-z]+) (was|is) (born|created|founded) in (\d{4})', 'temporal'),
(r'([A-Z][a-z]+) (invented|created|developed) ([^.]+)', 'attribution'),
(r'([^.]+) (costs?|worth) (\$[\d,]+)', 'numeric'),
(r'([A-Z][a-z]+) (lives?|works?|located) in ([A-Z][a-z]+)', 'location'),
(r"([A-Z][a-z]+ [A-Z][a-z]+) is (a|an) ([^.]+)", "definition"),
(r"([A-Z][a-z]+) (was|is) (born|created|founded) in (\d{4})", "temporal"),
(r"([A-Z][a-z]+) (invented|created|developed) ([^.]+)", "attribution"),
(r"([^.]+) (costs?|worth) (\$[\d,]+)", "numeric"),
(r"([A-Z][a-z]+) (lives?|works?|located) in ([A-Z][a-z]+)", "location"),
]
def extract_facts(self, text: str) -> List[Dict[str, Any]]:
@@ -19,27 +19,31 @@ class FactExtractor:
for pattern, fact_type in self.fact_patterns:
matches = re.finditer(pattern, text)
for match in matches:
facts.append({
'type': fact_type,
'text': match.group(0),
'components': match.groups(),
'confidence': 0.7
})
facts.append(
{
"type": fact_type,
"text": match.group(0),
"components": match.groups(),
"confidence": 0.7,
}
)
noun_phrases = self._extract_noun_phrases(text)
for phrase in noun_phrases:
if len(phrase.split()) >= 2:
facts.append({
'type': 'entity',
'text': phrase,
'components': [phrase],
'confidence': 0.5
})
facts.append(
{
"type": "entity",
"text": phrase,
"components": [phrase],
"confidence": 0.5,
}
)
return facts
def _extract_noun_phrases(self, text: str) -> List[str]:
sentences = re.split(r'[.!?]', text)
sentences = re.split(r"[.!?]", text)
phrases = []
for sentence in sentences:
@@ -51,25 +55,73 @@ class FactExtractor:
current_phrase.append(word)
else:
if len(current_phrase) >= 2:
phrases.append(' '.join(current_phrase))
phrases.append(" ".join(current_phrase))
current_phrase = []
if len(current_phrase) >= 2:
phrases.append(' '.join(current_phrase))
phrases.append(" ".join(current_phrase))
return list(set(phrases))
def extract_key_terms(self, text: str, top_k: int = 10) -> List[tuple]:
words = re.findall(r'\b[a-z]{4,}\b', text.lower())
words = re.findall(r"\b[a-z]{4,}\b", text.lower())
stopwords = {
'this', 'that', 'these', 'those', 'what', 'which', 'where', 'when',
'with', 'from', 'have', 'been', 'were', 'will', 'would', 'could',
'should', 'about', 'their', 'there', 'other', 'than', 'then', 'them',
'some', 'more', 'very', 'such', 'into', 'through', 'during', 'before',
'after', 'above', 'below', 'between', 'under', 'again', 'further',
'once', 'here', 'both', 'each', 'doing', 'only', 'over', 'same',
'being', 'does', 'just', 'also', 'make', 'made', 'know', 'like'
"this",
"that",
"these",
"those",
"what",
"which",
"where",
"when",
"with",
"from",
"have",
"been",
"were",
"will",
"would",
"could",
"should",
"about",
"their",
"there",
"other",
"than",
"then",
"them",
"some",
"more",
"very",
"such",
"into",
"through",
"during",
"before",
"after",
"above",
"below",
"between",
"under",
"again",
"further",
"once",
"here",
"both",
"each",
"doing",
"only",
"over",
"same",
"being",
"does",
"just",
"also",
"make",
"made",
"know",
"like",
}
filtered_words = [w for w in words if w not in stopwords]
@@ -85,57 +137,120 @@ class FactExtractor:
relationships = []
relationship_patterns = [
(r'([A-Z][a-z]+) (works for|employed by|member of) ([A-Z][a-z]+)', 'employment'),
(r'([A-Z][a-z]+) (owns|has|possesses) ([^.]+)', 'ownership'),
(r'([A-Z][a-z]+) (located in|part of|belongs to) ([A-Z][a-z]+)', 'location'),
(r'([A-Z][a-z]+) (uses|utilizes|implements) ([^.]+)', 'usage'),
(
r"([A-Z][a-z]+) (works for|employed by|member of) ([A-Z][a-z]+)",
"employment",
),
(r"([A-Z][a-z]+) (owns|has|possesses) ([^.]+)", "ownership"),
(
r"([A-Z][a-z]+) (located in|part of|belongs to) ([A-Z][a-z]+)",
"location",
),
(r"([A-Z][a-z]+) (uses|utilizes|implements) ([^.]+)", "usage"),
]
for pattern, rel_type in relationship_patterns:
matches = re.finditer(pattern, text)
for match in matches:
relationships.append({
'type': rel_type,
'subject': match.group(1),
'predicate': match.group(2),
'object': match.group(3),
'confidence': 0.6
})
relationships.append(
{
"type": rel_type,
"subject": match.group(1),
"predicate": match.group(2),
"object": match.group(3),
"confidence": 0.6,
}
)
return relationships
def extract_metadata(self, text: str) -> Dict[str, Any]:
word_count = len(text.split())
sentence_count = len(re.split(r'[.!?]', text))
sentence_count = len(re.split(r"[.!?]", text))
urls = re.findall(r'https?://[^\s]+', text)
email_addresses = re.findall(r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b', text)
dates = re.findall(r'\b\d{1,2}[-/]\d{1,2}[-/]\d{2,4}\b|\b\d{4}[-/]\d{1,2}[-/]\d{1,2}\b', text)
numbers = re.findall(r'\b\d+(?:,\d{3})*(?:\.\d+)?\b', text)
urls = re.findall(r"https?://[^\s]+", text)
email_addresses = re.findall(
r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b", text
)
dates = re.findall(
r"\b\d{1,2}[-/]\d{1,2}[-/]\d{2,4}\b|\b\d{4}[-/]\d{1,2}[-/]\d{1,2}\b", text
)
numbers = re.findall(r"\b\d+(?:,\d{3})*(?:\.\d+)?\b", text)
return {
'word_count': word_count,
'sentence_count': sentence_count,
'avg_words_per_sentence': round(word_count / max(sentence_count, 1), 2),
'urls': urls,
'email_addresses': email_addresses,
'dates': dates,
'numeric_values': numbers,
'has_code': bool(re.search(r'```|def |class |import |function ', text)),
'has_questions': bool(re.search(r'\?', text))
"word_count": word_count,
"sentence_count": sentence_count,
"avg_words_per_sentence": round(word_count / max(sentence_count, 1), 2),
"urls": urls,
"email_addresses": email_addresses,
"dates": dates,
"numeric_values": numbers,
"has_code": bool(re.search(r"```|def |class |import |function ", text)),
"has_questions": bool(re.search(r"\?", text)),
}
def categorize_content(self, text: str) -> List[str]:
categories = []
category_keywords = {
'programming': ['code', 'function', 'class', 'variable', 'programming', 'software', 'debug'],
'data': ['data', 'database', 'query', 'table', 'record', 'statistics', 'analysis'],
'documentation': ['documentation', 'guide', 'tutorial', 'manual', 'readme', 'explain'],
'configuration': ['config', 'settings', 'configuration', 'setup', 'install', 'deployment'],
'testing': ['test', 'testing', 'validate', 'verification', 'quality', 'assertion'],
'research': ['research', 'study', 'analysis', 'investigation', 'findings', 'results'],
'planning': ['plan', 'planning', 'schedule', 'roadmap', 'milestone', 'timeline'],
"programming": [
"code",
"function",
"class",
"variable",
"programming",
"software",
"debug",
],
"data": [
"data",
"database",
"query",
"table",
"record",
"statistics",
"analysis",
],
"documentation": [
"documentation",
"guide",
"tutorial",
"manual",
"readme",
"explain",
],
"configuration": [
"config",
"settings",
"configuration",
"setup",
"install",
"deployment",
],
"testing": [
"test",
"testing",
"validate",
"verification",
"quality",
"assertion",
],
"research": [
"research",
"study",
"analysis",
"investigation",
"findings",
"results",
],
"planning": [
"plan",
"planning",
"schedule",
"roadmap",
"milestone",
"timeline",
],
}
text_lower = text.lower()
@@ -143,4 +258,4 @@ class FactExtractor:
if any(keyword in text_lower for keyword in keywords):
categories.append(category)
return categories if categories else ['general']
return categories if categories else ["general"]
pr/memory/knowledge_store.py
+103 -66
View File
@@ -1,10 +1,12 @@
import json
import sqlite3
import time
from typing import List, Dict, Any, Optional
from dataclasses import dataclass
from typing import Any, Dict, List, Optional
from .semantic_index import SemanticIndex
@dataclass
class KnowledgeEntry:
entry_id: str
@@ -18,16 +20,17 @@ class KnowledgeEntry:
def to_dict(self) -> Dict[str, Any]:
return {
'entry_id': self.entry_id,
'category': self.category,
'content': self.content,
'metadata': self.metadata,
'created_at': self.created_at,
'updated_at': self.updated_at,
'access_count': self.access_count,
'importance_score': self.importance_score
"entry_id": self.entry_id,
"category": self.category,
"content": self.content,
"metadata": self.metadata,
"created_at": self.created_at,
"updated_at": self.updated_at,
"access_count": self.access_count,
"importance_score": self.importance_score,
}
class KnowledgeStore:
def __init__(self, db_path: str):
self.db_path = db_path
@@ -39,7 +42,8 @@ class KnowledgeStore:
def _initialize_store(self):
cursor = self.conn.cursor()
cursor.execute('''
cursor.execute(
"""
CREATE TABLE IF NOT EXISTS knowledge_entries (
entry_id TEXT PRIMARY KEY,
category TEXT NOT NULL,
@@ -50,44 +54,54 @@ class KnowledgeStore:
access_count INTEGER DEFAULT 0,
importance_score REAL DEFAULT 1.0
)
''')
"""
)
cursor.execute('''
cursor.execute(
"""
CREATE INDEX IF NOT EXISTS idx_category ON knowledge_entries(category)
''')
cursor.execute('''
"""
)
cursor.execute(
"""
CREATE INDEX IF NOT EXISTS idx_importance ON knowledge_entries(importance_score DESC)
''')
cursor.execute('''
"""
)
cursor.execute(
"""
CREATE INDEX IF NOT EXISTS idx_created ON knowledge_entries(created_at DESC)
''')
"""
)
self.conn.commit()
def _load_index(self):
cursor = self.conn.cursor()
cursor.execute('SELECT entry_id, content FROM knowledge_entries')
cursor.execute("SELECT entry_id, content FROM knowledge_entries")
for row in cursor.fetchall():
self.semantic_index.add_document(row[0], row[1])
def add_entry(self, entry: KnowledgeEntry):
cursor = self.conn.cursor()
cursor.execute('''
cursor.execute(
"""
INSERT OR REPLACE INTO knowledge_entries
(entry_id, category, content, metadata, created_at, updated_at, access_count, importance_score)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
''', (
entry.entry_id,
entry.category,
entry.content,
json.dumps(entry.metadata),
entry.created_at,
entry.updated_at,
entry.access_count,
entry.importance_score
))
""",
(
entry.entry_id,
entry.category,
entry.content,
json.dumps(entry.metadata),
entry.created_at,
entry.updated_at,
entry.access_count,
entry.importance_score,
),
)
self.conn.commit()
@@ -96,20 +110,26 @@ class KnowledgeStore:
def get_entry(self, entry_id: str) -> Optional[KnowledgeEntry]:
cursor = self.conn.cursor()
cursor.execute('''
cursor.execute(
"""
SELECT entry_id, category, content, metadata, created_at, updated_at, access_count, importance_score
FROM knowledge_entries
WHERE entry_id = ?
''', (entry_id,))
""",
(entry_id,),
)
row = cursor.fetchone()
if row:
cursor.execute('''
cursor.execute(
"""
UPDATE knowledge_entries
SET access_count = access_count + 1
WHERE entry_id = ?
''', (entry_id,))
""",
(entry_id,),
)
self.conn.commit()
return KnowledgeEntry(
@@ -120,13 +140,14 @@ class KnowledgeStore:
created_at=row[4],
updated_at=row[5],
access_count=row[6] + 1,
importance_score=row[7]
importance_score=row[7],
)
return None
def search_entries(self, query: str, category: Optional[str] = None,
top_k: int = 5) -> List[KnowledgeEntry]:
def search_entries(
self, query: str, category: Optional[str] = None, top_k: int = 5
) -> List[KnowledgeEntry]:
search_results = self.semantic_index.search(query, top_k * 2)
cursor = self.conn.cursor()
@@ -134,17 +155,23 @@ class KnowledgeStore:
entries = []
for entry_id, score in search_results:
if category:
cursor.execute('''
cursor.execute(
"""
SELECT entry_id, category, content, metadata, created_at, updated_at, access_count, importance_score
FROM knowledge_entries
WHERE entry_id = ? AND category = ?
''', (entry_id, category))
""",
(entry_id, category),
)
else:
cursor.execute('''
cursor.execute(
"""
SELECT entry_id, category, content, metadata, created_at, updated_at, access_count, importance_score
FROM knowledge_entries
WHERE entry_id = ?
''', (entry_id,))
""",
(entry_id,),
)
row = cursor.fetchone()
if row:
@@ -156,7 +183,7 @@ class KnowledgeStore:
created_at=row[4],
updated_at=row[5],
access_count=row[6],
importance_score=row[7]
importance_score=row[7],
)
entries.append(entry)
@@ -168,44 +195,52 @@ class KnowledgeStore:
def get_by_category(self, category: str, limit: int = 20) -> List[KnowledgeEntry]:
cursor = self.conn.cursor()
cursor.execute('''
cursor.execute(
"""
SELECT entry_id, category, content, metadata, created_at, updated_at, access_count, importance_score
FROM knowledge_entries
WHERE category = ?
ORDER BY importance_score DESC, created_at DESC
LIMIT ?
''', (category, limit))
""",
(category, limit),
)
entries = []
for row in cursor.fetchall():
entries.append(KnowledgeEntry(
entry_id=row[0],
category=row[1],
content=row[2],
metadata=json.loads(row[3]) if row[3] else {},
created_at=row[4],
updated_at=row[5],
access_count=row[6],
importance_score=row[7]
))
entries.append(
KnowledgeEntry(
entry_id=row[0],
category=row[1],
content=row[2],
metadata=json.loads(row[3]) if row[3] else {},
created_at=row[4],
updated_at=row[5],
access_count=row[6],
importance_score=row[7],
)
)
return entries
def update_importance(self, entry_id: str, importance_score: float):
cursor = self.conn.cursor()
cursor.execute('''
cursor.execute(
"""
UPDATE knowledge_entries
SET importance_score = ?, updated_at = ?
WHERE entry_id = ?
''', (importance_score, time.time(), entry_id))
""",
(importance_score, time.time(), entry_id),
)
self.conn.commit()
def delete_entry(self, entry_id: str) -> bool:
cursor = self.conn.cursor()
cursor.execute('DELETE FROM knowledge_entries WHERE entry_id = ?', (entry_id,))
cursor.execute("DELETE FROM knowledge_entries WHERE entry_id = ?", (entry_id,))
deleted = cursor.rowcount > 0
self.conn.commit()
@@ -218,27 +253,29 @@ class KnowledgeStore:
def get_statistics(self) -> Dict[str, Any]:
cursor = self.conn.cursor()
cursor.execute('SELECT COUNT(*) FROM knowledge_entries')
cursor.execute("SELECT COUNT(*) FROM knowledge_entries")
total_entries = cursor.fetchone()[0]
cursor.execute('SELECT COUNT(DISTINCT category) FROM knowledge_entries')
cursor.execute("SELECT COUNT(DISTINCT category) FROM knowledge_entries")
total_categories = cursor.fetchone()[0]
cursor.execute('''
cursor.execute(
"""
SELECT category, COUNT(*) as count
FROM knowledge_entries
GROUP BY category
ORDER BY count DESC
''')
"""
)
category_counts = {row[0]: row[1] for row in cursor.fetchall()}
cursor.execute('SELECT SUM(access_count) FROM knowledge_entries')
cursor.execute("SELECT SUM(access_count) FROM knowledge_entries")
total_accesses = cursor.fetchone()[0] or 0
return {
'total_entries': total_entries,
'total_categories': total_categories,
'category_distribution': category_counts,
'total_accesses': total_accesses,
'vocabulary_size': len(self.semantic_index.vocabulary)
"total_entries": total_entries,
"total_categories": total_categories,
"category_distribution": category_counts,
"total_accesses": total_accesses,
"vocabulary_size": len(self.semantic_index.vocabulary),
}
pr/memory/semantic_index.py
+9 -4
View File
@@ -1,7 +1,8 @@
import math
import re
from collections import Counter, defaultdict
from typing import List, Dict, Tuple, Set
from typing import Dict, List, Set, Tuple
class SemanticIndex:
def __init__(self):
@@ -12,7 +13,7 @@ class SemanticIndex:
def _tokenize(self, text: str) -> List[str]:
text = text.lower()
text = re.sub(r'[^a-z0-9\s]', ' ', text)
text = re.sub(r"[^a-z0-9\s]", " ", text)
tokens = text.split()
return tokens
@@ -78,8 +79,12 @@ class SemanticIndex:
scores.sort(key=lambda x: x[1], reverse=True)
return scores[:top_k]
def _cosine_similarity(self, vec1: Dict[str, float], vec2: Dict[str, float]) -> float:
dot_product = sum(vec1.get(token, 0) * vec2.get(token, 0) for token in set(vec1) | set(vec2))
def _cosine_similarity(
self, vec1: Dict[str, float], vec2: Dict[str, float]
) -> float:
dot_product = sum(
vec1.get(token, 0) * vec2.get(token, 0) for token in set(vec1) | set(vec2)
)
norm1 = math.sqrt(sum(val**2 for val in vec1.values()))
norm2 = math.sqrt(sum(val**2 for val in vec2.values()))
if norm1 == 0 or norm2 == 0: