feat: add distributed async dataset with unix socket server and refactor agent communication bus

Implement AsyncDataSet class supporting client-server model over Unix sockets with SQLite backend, including KV store, table management, and concurrent query handling. Rename `get_messages` to `receive_messages` in AgentCommunicationBus and update all callers. Remove deprecated `get_recommended_agent` function from agent_roles, `invalidate_tool` from tool_cache, and legacy `receive_messages` wrapper. Add multiplexer command routing in handlers with `/prompt` command support. Introduce comprehensive help documentation system for workflows. Update default API URLs to production endpoints and refactor adaptive context window calculation in AdvancedContextManager.

pr/core/advanced_context.py

@@ -7,44 +7,45 @@ class AdvancedContextManager:
         self.knowledge_store = knowledge_store
         self.conversation_memory = conversation_memory
 
-    def adaptive_context_window(
-        self, messages: List[Dict[str, Any]], task_complexity: str = "medium"
-    ) -> int:
-        complexity_thresholds = {
-            "simple": 10,
-            "medium": 20,
-            "complex": 35,
-            "very_complex": 50,
+    def adaptive_context_window(self, messages: List[Dict[str, Any]], complexity: str) -> int:
+        """Calculate adaptive context window size based on message complexity."""
+        base_window = 10
+
+        complexity_multipliers = {
+            "simple": 1.0,
+            "medium": 2.0,
+            "complex": 3.5,
+            "very_complex": 5.0,
         }
 
-        base_threshold = complexity_thresholds.get(task_complexity, 20)
-
-        message_complexity_score = self._analyze_message_complexity(messages)
-
-        if message_complexity_score > 0.7:
-            adjusted = int(base_threshold * 1.5)
-        elif message_complexity_score < 0.3:
-            adjusted = int(base_threshold * 0.7)
-        else:
-            adjusted = base_threshold
-
-        return max(base_threshold, adjusted)
+        multiplier = complexity_multipliers.get(complexity, 2.0)
+        return int(base_window * multiplier)
 
     def _analyze_message_complexity(self, messages: List[Dict[str, Any]]) -> float:
-        total_length = sum(len(msg.get("content", "")) for msg in messages)
-        avg_length = total_length / len(messages) if messages else 0
+        """Analyze the complexity of messages and return a score between 0.0 and 1.0."""
+        if not messages:
+            return 0.0
 
-        unique_words = set()
-        for msg in messages:
-            content = msg.get("content", "")
-            words = re.findall(r"\b\w+\b", content.lower())
-            unique_words.update(words)
+        total_complexity = 0.0
+        for message in messages:
+            content = message.get("content", "")
+            if not content:
+                continue
 
-        vocabulary_richness = len(unique_words) / total_length if total_length > 0 else 0
+            # Calculate complexity based on various factors
+            word_count = len(content.split())
+            sentence_count = len(re.split(r"[.!?]+", content))
+            avg_word_length = sum(len(word) for word in content.split()) / max(word_count, 1)
 
-        # Simple complexity score based on length and richness
-        complexity = min(1.0, (avg_length / 100) + vocabulary_richness)
-        return complexity
+            # Complexity score based on length, vocabulary, and structure
+            length_score = min(1.0, word_count / 100)  # Normalize to 0-1
+            structure_score = min(1.0, sentence_count / 10)
+            vocabulary_score = min(1.0, avg_word_length / 8)
+
+            message_complexity = (length_score + structure_score + vocabulary_score) / 3
+            total_complexity += message_complexity
+
+        return min(1.0, total_complexity / len(messages))
 
     def extract_key_sentences(self, text: str, top_k: int = 5) -> List[str]:
         if not text.strip():
@@ -82,3 +83,38 @@ class AdvancedContextManager:
             return 0.0
 
         return len(intersection) / len(union)
+
+    def create_enhanced_context(
+        self, messages: List[Dict[str, Any]], user_message: str, include_knowledge: bool = True
+    ) -> tuple:
+        """Create enhanced context with knowledge base integration."""
+        working_messages = messages.copy()
+
+        if include_knowledge and self.knowledge_store:
+            # Search knowledge base for relevant information
+            search_results = self.knowledge_store.search_entries(user_message, top_k=3)
+
+            if search_results:
+                knowledge_parts = []
+                for idx, entry in enumerate(search_results, 1):
+                    content = entry.content
+                    if len(content) > 2000:
+                        content = content[:2000] + "..."
+
+                    knowledge_parts.append(f"Match {idx} (Category: {entry.category}):\n{content}")
+
+                knowledge_message_content = (
+                    "[KNOWLEDGE_BASE_CONTEXT]\nRelevant knowledge base entries:\n\n"
+                    + "\n\n".join(knowledge_parts)
+                )
+
+                knowledge_message = {"role": "user", "content": knowledge_message_content}
+                working_messages.append(knowledge_message)
+
+                context_info = f"Added {len(search_results)} knowledge base entries"
+            else:
+                context_info = "No relevant knowledge base entries found"
+        else:
+            context_info = "Knowledge base integration disabled"
+
+        return working_messages, context_info

pr/core/api.py

@@ -67,6 +67,15 @@ def call_api(messages, model, api_url, api_key, use_tools, tools_definition, ver
                     if "tool_calls" in msg:
                         logger.debug(f"Response contains {len(msg['tool_calls'])} tool call(s)")
 
+            if verbose and "usage" in result:
+                from pr.core.usage_tracker import UsageTracker
+
+                usage = result["usage"]
+                input_t = usage.get("prompt_tokens", 0)
+                output_t = usage.get("completion_tokens", 0)
+                cost = UsageTracker._calculate_cost(model, input_t, output_t)
+                print(f"API call cost: €{cost:.4f}")
+
         logger.debug("=== API CALL END ===")
         return result
 

pr/core/assistant.py

@@ -29,34 +29,28 @@ from pr.core.background_monitor import (
     stop_global_monitor,
 )
 from pr.core.context import init_system_message, truncate_tool_result
-from pr.tools import (
-    apply_patch,
-    chdir,
-    create_diff,
-    db_get,
-    db_query,
-    db_set,
-    getpwd,
-    http_fetch,
-    index_source_directory,
-    kill_process,
-    list_directory,
-    mkdir,
-    python_exec,
-    read_file,
-    run_command,
-    search_replace,
-    tail_process,
-    web_search,
-    web_search_news,
-    write_file,
-    post_image,
+from pr.tools import get_tools_definition
+from pr.tools.agents import (
+    collaborate_agents,
+    create_agent,
+    execute_agent_task,
+    list_agents,
+    remove_agent,
 )
-from pr.tools.base import get_tools_definition
+from pr.tools.command import kill_process, run_command, tail_process
+from pr.tools.database import db_get, db_query, db_set
 from pr.tools.filesystem import (
+    chdir,
     clear_edit_tracker,
     display_edit_summary,
     display_edit_timeline,
+    getpwd,
+    index_source_directory,
+    list_directory,
+    mkdir,
+    read_file,
+    search_replace,
+    write_file,
 )
 from pr.tools.interactive_control import (
     close_interactive_session,
@@ -65,7 +59,18 @@ from pr.tools.interactive_control import (
     send_input_to_session,
     start_interactive_session,
 )
-from pr.tools.patch import display_file_diff
+from pr.tools.memory import (
+    add_knowledge_entry,
+    delete_knowledge_entry,
+    get_knowledge_by_category,
+    get_knowledge_entry,
+    get_knowledge_statistics,
+    search_knowledge,
+    update_knowledge_importance,
+)
+from pr.tools.patch import apply_patch, create_diff, display_file_diff
+from pr.tools.python_exec import python_exec
+from pr.tools.web import http_fetch, web_search, web_search_news
 from pr.ui import Colors, render_markdown
 
 logger = logging.getLogger("pr")
@@ -97,7 +102,7 @@ class Assistant:
             "MODEL_LIST_URL", MODEL_LIST_URL
         )
         self.use_tools = os.environ.get("USE_TOOLS", "1") == "1"
-        self.strict_mode = os.environ.get("STRICT_MODE", "0") == "1"
+
         self.interrupt_count = 0
         self.python_globals = {}
         self.db_conn = None
@@ -245,7 +250,6 @@ class Assistant:
                 logger.debug(f"Tool call: {func_name} with arguments: {arguments}")
 
                 func_map = {
-                    "post_image": lambda **kw: post_image(**kw),
                     "http_fetch": lambda **kw: http_fetch(**kw),
                     "run_command": lambda **kw: run_command(**kw),
                     "tail_process": lambda **kw: tail_process(**kw),

pr/core/background_monitor.py

@@ -27,15 +27,6 @@ class BackgroundMonitor:
         if self.monitor_thread:
             self.monitor_thread.join(timeout=2)
 
-    def add_event_callback(self, callback):
-        """Add a callback function to be called when events are detected."""
-        self.event_callbacks.append(callback)
-
-    def remove_event_callback(self, callback):
-        """Remove an event callback."""
-        if callback in self.event_callbacks:
-            self.event_callbacks.remove(callback)
-
     def get_pending_events(self):
         """Get all pending events from the queue."""
         events = []
@@ -224,30 +215,3 @@ def stop_global_monitor():
     global _global_monitor
     if _global_monitor:
         _global_monitor.stop()
-
-
-# Global monitor instance
-_global_monitor = None
-
-
-def start_global_monitor():
-    """Start the global background monitor."""
-    global _global_monitor
-    if _global_monitor is None:
-        _global_monitor = BackgroundMonitor()
-        _global_monitor.start()
-    return _global_monitor
-
-
-def stop_global_monitor():
-    """Stop the global background monitor."""
-    global _global_monitor
-    if _global_monitor:
-        _global_monitor.stop()
-        _global_monitor = None
-
-
-def get_global_monitor():
-    """Get the global background monitor instance."""
-    global _global_monitor
-    return _global_monitor

pr/core/config_loader.py

@@ -1,7 +1,6 @@
 import configparser
 import os
 from typing import Any, Dict
-import uuid
 from pr.core.logging import get_logger
 
 logger = get_logger("config")
@@ -11,29 +10,6 @@ CONFIG_FILE = os.path.join(CONFIG_DIRECTORY, ".prrc")
 LOCAL_CONFIG_FILE = ".prrc"
 
 
-def load_config() -> Dict[str, Any]:
-    os.makedirs(CONFIG_DIRECTORY, exist_ok=True)
-    config = {
-        "api": {},
-        "autonomous": {},
-        "ui": {},
-        "output": {},
-        "session": {},
-        "api_key": "rp-" + str(uuid.uuid4()),
-    }
-
-    global_config = _load_config_file(CONFIG_FILE)
-    local_config = _load_config_file(LOCAL_CONFIG_FILE)
-
-    for section in config.keys():
-        if section in global_config:
-            config[section].update(global_config[section])
-        if section in local_config:
-            config[section].update(local_config[section])
-
-    return config
-
-
 def _load_config_file(filepath: str) -> Dict[str, Dict[str, Any]]:
     if not os.path.exists(filepath):
         return {}

pr/core/enhanced_assistant.py

@@ -12,7 +12,6 @@ from pr.config import (
     CONVERSATION_SUMMARY_THRESHOLD,
     DB_PATH,
     KNOWLEDGE_SEARCH_LIMIT,
-    MEMORY_AUTO_SUMMARIZE,
     TOOL_CACHE_TTL,
     WORKFLOW_EXECUTOR_MAX_WORKERS,
 )
@@ -169,24 +168,28 @@ class EnhancedAssistant:
             self.current_conversation_id, str(uuid.uuid4())[:16], "user", user_message
         )
 
-        if MEMORY_AUTO_SUMMARIZE and len(self.base.messages) % 5 == 0:
-            facts = self.fact_extractor.extract_facts(user_message)
-            for fact in facts[:3]:
-                entry_id = str(uuid.uuid4())[:16]
-                import time
+        # Automatically extract and store facts from every user message
+        facts = self.fact_extractor.extract_facts(user_message)
+        for fact in facts[:5]:  # Store up to 5 facts per message
+            entry_id = str(uuid.uuid4())[:16]
+            import time
 
-                from pr.memory import KnowledgeEntry
+            from pr.memory import KnowledgeEntry
 
-                categories = self.fact_extractor.categorize_content(fact["text"])
-                entry = KnowledgeEntry(
-                    entry_id=entry_id,
-                    category=categories[0] if categories else "general",
-                    content=fact["text"],
-                    metadata={"type": fact["type"], "confidence": fact["confidence"]},
-                    created_at=time.time(),
-                    updated_at=time.time(),
-                )
-                self.knowledge_store.add_entry(entry)
+            categories = self.fact_extractor.categorize_content(fact["text"])
+            entry = KnowledgeEntry(
+                entry_id=entry_id,
+                category=categories[0] if categories else "general",
+                content=fact["text"],
+                metadata={
+                    "type": fact["type"],
+                    "confidence": fact["confidence"],
+                    "source": "user_message",
+                },
+                created_at=time.time(),
+                updated_at=time.time(),
+            )
+            self.knowledge_store.add_entry(entry)
 
         if self.context_manager and ADVANCED_CONTEXT_ENABLED:
             enhanced_messages, context_info = self.context_manager.create_enhanced_context(

pr/core/knowledge_context.py

@@ -0,0 +1,148 @@
+import logging
+
+logger = logging.getLogger("pr")
+
+KNOWLEDGE_MESSAGE_MARKER = "[KNOWLEDGE_BASE_CONTEXT]"
+
+
+def inject_knowledge_context(assistant, user_message):
+    if not hasattr(assistant, "enhanced") or not assistant.enhanced:
+        return
+
+    messages = assistant.messages
+
+    # Remove any existing knowledge context messages
+    for i in range(len(messages) - 1, -1, -1):
+        if messages[i].get("role") == "user" and KNOWLEDGE_MESSAGE_MARKER in messages[i].get(
+            "content", ""
+        ):
+            del messages[i]
+            logger.debug(f"Removed existing knowledge base message at index {i}")
+            break
+
+    try:
+        # Search knowledge base with enhanced FTS + semantic search
+        knowledge_results = assistant.enhanced.knowledge_store.search_entries(user_message, top_k=5)
+
+        # Search conversation history for related content
+        conversation_results = []
+        if hasattr(assistant.enhanced, "conversation_memory"):
+            history_results = assistant.enhanced.conversation_memory.search_conversations(
+                user_message, limit=3
+            )
+            for conv in history_results:
+                # Extract relevant messages from conversation
+                conv_messages = assistant.enhanced.conversation_memory.get_conversation_messages(
+                    conv["conversation_id"]
+                )
+                for msg in conv_messages[-5:]:  # Last 5 messages from each conversation
+                    if msg["role"] == "user" and msg["content"] != user_message:
+                        # Calculate relevance score
+                        relevance = calculate_text_similarity(user_message, msg["content"])
+                        if relevance > 0.3:  # Only include relevant matches
+                            conversation_results.append(
+                                {
+                                    "content": msg["content"],
+                                    "score": relevance,
+                                    "source": f"Previous conversation: {conv['conversation_id'][:8]}",
+                                }
+                            )
+
+        # Combine and sort results by relevance score
+        all_results = []
+
+        # Add knowledge base results
+        for entry in knowledge_results:
+            score = entry.metadata.get("search_score", 0.5)
+            all_results.append(
+                {
+                    "content": entry.content,
+                    "score": score,
+                    "source": f"Knowledge Base ({entry.category})",
+                    "type": "knowledge",
+                }
+            )
+
+        # Add conversation results
+        for conv in conversation_results:
+            all_results.append(
+                {
+                    "content": conv["content"],
+                    "score": conv["score"],
+                    "source": conv["source"],
+                    "type": "conversation",
+                }
+            )
+
+        # Sort by score and take top 5
+        all_results.sort(key=lambda x: x["score"], reverse=True)
+        top_results = all_results[:5]
+
+        if not top_results:
+            logger.debug("No relevant knowledge or conversation matches found")
+            return
+
+        # Format context for LLM
+        knowledge_parts = []
+        for idx, result in enumerate(top_results, 1):
+            content = result["content"]
+            if len(content) > 1500:  # Shorter limit for multiple results
+                content = content[:1500] + "..."
+
+            score_indicator = f"({result['score']:.2f})" if result["score"] < 1.0 else "(exact)"
+            knowledge_parts.append(
+                f"Match {idx} {score_indicator} - {result['source']}:\n{content}"
+            )
+
+        knowledge_message_content = (
+            f"{KNOWLEDGE_MESSAGE_MARKER}\nRelevant information from knowledge base and conversation history:\n\n"
+            + "\n\n".join(knowledge_parts)
+        )
+
+        knowledge_message = {"role": "user", "content": knowledge_message_content}
+
+        messages.append(knowledge_message)
+        logger.debug(f"Injected enhanced context message with {len(top_results)} matches")
+
+    except Exception as e:
+        logger.error(f"Error injecting knowledge context: {e}")
+
+
+def calculate_text_similarity(text1: str, text2: str) -> float:
+    """Calculate similarity between two texts using word overlap and sequence matching."""
+    import re
+
+    # Normalize texts
+    text1_lower = text1.lower()
+    text2_lower = text2.lower()
+
+    # Exact substring match gets highest score
+    if text1_lower in text2_lower or text2_lower in text1_lower:
+        return 1.0
+
+    # Word-level similarity
+    words1 = set(re.findall(r"\b\w+\b", text1_lower))
+    words2 = set(re.findall(r"\b\w+\b", text2_lower))
+
+    if not words1 or not words2:
+        return 0.0
+
+    intersection = words1 & words2
+    union = words1 | words2
+
+    word_similarity = len(intersection) / len(union)
+
+    # Bonus for consecutive word sequences (partial sentences)
+    consecutive_bonus = 0.0
+    words1_list = list(words1)
+    list(words2)
+
+    for i in range(len(words1_list) - 1):
+        for j in range(i + 2, min(i + 5, len(words1_list) + 1)):
+            phrase = " ".join(words1_list[i:j])
+            if phrase in text2_lower:
+                consecutive_bonus += 0.1 * (j - i)
+
+    total_similarity = min(1.0, word_similarity + consecutive_bonus)
+
+    return total_similarity

pr/core/usage_tracker.py

@@ -9,8 +9,10 @@ logger = get_logger("usage")
 
 USAGE_DB_FILE = os.path.expanduser("~/.assistant_usage.json")
 
+EXCHANGE_RATE = 1.0  # Keep in USD
+
 MODEL_COSTS = {
-    "x-ai/grok-code-fast-1": {"input": 0.0, "output": 0.0},
+    "x-ai/grok-code-fast-1": {"input": 0.0002, "output": 0.0015},  # per 1000 tokens in USD
     "gpt-4": {"input": 0.03, "output": 0.06},
     "gpt-4-turbo": {"input": 0.01, "output": 0.03},
     "gpt-3.5-turbo": {"input": 0.0005, "output": 0.0015},
@@ -64,9 +66,10 @@ class UsageTracker:
 
         self._save_to_history(model, input_tokens, output_tokens, cost)
 
-        logger.debug(f"Tracked request: {model}, tokens: {total_tokens}, cost: ${cost:.4f}")
+        logger.debug(f"Tracked request: {model}, tokens: {total_tokens}, cost: €{cost:.4f}")
 
-    def _calculate_cost(self, model: str, input_tokens: int, output_tokens: int) -> float:
+    @staticmethod
+    def _calculate_cost(model: str, input_tokens: int, output_tokens: int) -> float:
         if model not in MODEL_COSTS:
             base_model = model.split("/")[0] if "/" in model else model
             if base_model not in MODEL_COSTS:
@@ -76,8 +79,8 @@ class UsageTracker:
         else:
             costs = MODEL_COSTS[model]
 
-        input_cost = (input_tokens / 1000) * costs["input"]
-        output_cost = (output_tokens / 1000) * costs["output"]
+        input_cost = (input_tokens / 1000) * costs["input"] * EXCHANGE_RATE
+        output_cost = (output_tokens / 1000) * costs["output"] * EXCHANGE_RATE
 
         return input_cost + output_cost