290 lines
8.9 KiB
Python
290 lines
8.9 KiB
Python
# retoor <retoor@molodetz.nl>
|
|
|
|
import dataclasses
|
|
import json
|
|
|
|
from devplacepy.services.devii.agentic.compaction import context_size
|
|
from devplacepy.services.devii.agentic.loop import (
|
|
MAX_CONTEXT_OVERFLOW_RETRIES,
|
|
_run_tool_call,
|
|
react_loop,
|
|
)
|
|
from devplacepy.services.devii.agentic.state import AgentState
|
|
from devplacepy.services.devii.config import load_settings
|
|
from devplacepy.services.devii.errors import LLMError
|
|
from tests.conftest import run_async
|
|
class _FakeDispatcher:
|
|
def __init__(self):
|
|
self.calls = []
|
|
|
|
async def dispatch(self, name, arguments):
|
|
self.calls.append((name, arguments))
|
|
return json.dumps({"status": "ok"})
|
|
def _run(call, dispatcher=None):
|
|
return json.loads(run_async(_run_tool_call(dispatcher, call)))
|
|
|
|
|
|
def test_truncated_arguments_reported_not_dispatched():
|
|
dispatcher = _FakeDispatcher()
|
|
call = {
|
|
"function": {
|
|
"name": "project_write_file",
|
|
"arguments": '{"path":"a.md","content":"# hi',
|
|
}
|
|
}
|
|
out = _run(call, dispatcher)
|
|
assert out["error"] == "tool_input_truncated"
|
|
assert "one write tool call per turn" in out["message"]
|
|
assert dispatcher.calls == []
|
|
|
|
|
|
def test_non_object_arguments_rejected():
|
|
dispatcher = _FakeDispatcher()
|
|
out = _run({"function": {"name": "x", "arguments": '"a string"'}}, dispatcher)
|
|
assert out["error"] == "tool_input_error"
|
|
assert dispatcher.calls == []
|
|
|
|
|
|
def test_valid_string_arguments_dispatched():
|
|
dispatcher = _FakeDispatcher()
|
|
out = _run({"function": {"name": "vote", "arguments": '{"value":1}'}}, dispatcher)
|
|
assert out["status"] == "ok"
|
|
assert dispatcher.calls == [("vote", {"value": 1})]
|
|
|
|
|
|
def test_valid_dict_arguments_dispatched():
|
|
dispatcher = _FakeDispatcher()
|
|
out = _run({"function": {"name": "vote", "arguments": {"value": -1}}}, dispatcher)
|
|
assert out["status"] == "ok"
|
|
assert dispatcher.calls == [("vote", {"value": -1})]
|
|
|
|
|
|
def test_missing_arguments_defaults_to_empty_object():
|
|
dispatcher = _FakeDispatcher()
|
|
out = _run({"function": {"name": "auth_status"}}, dispatcher)
|
|
assert out["status"] == "ok"
|
|
assert dispatcher.calls == [("auth_status", {})]
|
|
|
|
|
|
_CONTEXT_LENGTH_BODY = json.dumps(
|
|
{
|
|
"error": {
|
|
"message": (
|
|
"This endpoint's maximum context length is 131072 tokens. "
|
|
"However, you requested about 403355 tokens. Please reduce "
|
|
"the length."
|
|
),
|
|
"code": 400,
|
|
}
|
|
}
|
|
)
|
|
|
|
|
|
def _context_length_error():
|
|
return LLMError(
|
|
"Model endpoint returned 400: over limit", status=400, body=_CONTEXT_LENGTH_BODY
|
|
)
|
|
|
|
|
|
def _settings_for_test(keep_tail=4, threshold=10**9):
|
|
return dataclasses.replace(
|
|
load_settings(),
|
|
context_compact_threshold=threshold,
|
|
context_keep_tail=keep_tail,
|
|
)
|
|
|
|
|
|
def _long_message_history(count=10):
|
|
messages = [{"role": "system", "content": "system prompt"}]
|
|
for i in range(count):
|
|
role = "user" if i % 2 == 0 else "assistant"
|
|
messages.append({"role": role, "content": f"turn {i}"})
|
|
return messages
|
|
|
|
|
|
class _FakeLLM:
|
|
def __init__(self, complete_results):
|
|
self._complete_results = list(complete_results)
|
|
self.complete_calls = 0
|
|
self.summarize_calls = 0
|
|
|
|
async def complete(self, messages, tools):
|
|
self.complete_calls += 1
|
|
result = self._complete_results[
|
|
min(self.complete_calls, len(self._complete_results)) - 1
|
|
]
|
|
if isinstance(result, Exception):
|
|
raise result
|
|
return result
|
|
|
|
async def summarize(self, text):
|
|
self.summarize_calls += 1
|
|
return "compacted summary"
|
|
|
|
|
|
def test_context_overflow_triggers_compaction_and_retries():
|
|
llm = _FakeLLM(
|
|
[_context_length_error(), {"role": "assistant", "content": "Recovered answer"}]
|
|
)
|
|
messages = _long_message_history()
|
|
result = run_async(
|
|
react_loop(
|
|
llm,
|
|
_FakeDispatcher(),
|
|
messages,
|
|
tools=[],
|
|
state=AgentState(),
|
|
settings=_settings_for_test(),
|
|
max_iterations=5,
|
|
plan_required=False,
|
|
verify_required=False,
|
|
)
|
|
)
|
|
assert result == "Recovered answer"
|
|
assert llm.complete_calls == 2
|
|
assert llm.summarize_calls == 1
|
|
assert not result.startswith("[model error]")
|
|
|
|
|
|
def test_context_overflow_gives_up_after_max_retries_with_clear_message():
|
|
llm = _FakeLLM([_context_length_error()])
|
|
messages = _long_message_history()
|
|
result = run_async(
|
|
react_loop(
|
|
llm,
|
|
_FakeDispatcher(),
|
|
messages,
|
|
tools=[],
|
|
state=AgentState(),
|
|
settings=_settings_for_test(),
|
|
max_iterations=10,
|
|
plan_required=False,
|
|
verify_required=False,
|
|
)
|
|
)
|
|
assert result.startswith("[model error]")
|
|
assert "context length" in result.lower() or "400" in result
|
|
assert llm.complete_calls == MAX_CONTEXT_OVERFLOW_RETRIES + 1
|
|
assert 0 < llm.summarize_calls <= MAX_CONTEXT_OVERFLOW_RETRIES
|
|
|
|
|
|
def test_non_context_length_error_never_triggers_compaction():
|
|
llm = _FakeLLM([LLMError("Model endpoint returned 500: boom", status=500, body="{}")])
|
|
messages = _long_message_history()
|
|
result = run_async(
|
|
react_loop(
|
|
llm,
|
|
_FakeDispatcher(),
|
|
messages,
|
|
tools=[],
|
|
state=AgentState(),
|
|
settings=_settings_for_test(),
|
|
max_iterations=5,
|
|
plan_required=False,
|
|
verify_required=False,
|
|
)
|
|
)
|
|
assert result == "[model error] Model endpoint returned 500: boom"
|
|
assert llm.complete_calls == 1
|
|
assert llm.summarize_calls == 0
|
|
|
|
|
|
class _FakeLLMWithRealLimit:
|
|
def __init__(self, simulated_limit_chars):
|
|
self.simulated_limit_chars = simulated_limit_chars
|
|
self.complete_calls = 0
|
|
self.summarize_calls = 0
|
|
|
|
async def complete(self, messages, tools):
|
|
self.complete_calls += 1
|
|
if context_size(messages) > self.simulated_limit_chars:
|
|
raise _context_length_error()
|
|
return {"role": "assistant", "content": "Recovered answer"}
|
|
|
|
async def summarize(self, text):
|
|
self.summarize_calls += 1
|
|
return "short summary"
|
|
|
|
|
|
def test_proactive_compaction_targets_40_percent_of_threshold():
|
|
messages = _long_message_history(100)
|
|
threshold = context_size(messages) - 50
|
|
llm = _FakeLLM([{"role": "assistant", "content": "done"}])
|
|
trace_events = []
|
|
result = run_async(
|
|
react_loop(
|
|
llm,
|
|
_FakeDispatcher(),
|
|
messages,
|
|
tools=[],
|
|
state=AgentState(),
|
|
settings=_settings_for_test(keep_tail=4, threshold=threshold),
|
|
max_iterations=5,
|
|
plan_required=False,
|
|
verify_required=False,
|
|
on_trace=lambda event, name, detail: trace_events.append(event),
|
|
)
|
|
)
|
|
assert result == "done"
|
|
assert "compact" in trace_events
|
|
assert context_size(messages) <= int(threshold * 0.4)
|
|
|
|
|
|
def test_proactive_compaction_retries_when_first_pass_is_not_enough():
|
|
giant = "X" * 300_000
|
|
messages = _long_message_history(20)
|
|
messages.append(
|
|
{"role": "tool", "tool_call_id": "1", "name": "big_tool", "content": giant}
|
|
)
|
|
messages.append({"role": "user", "content": "please continue"})
|
|
|
|
llm = _FakeLLM([{"role": "assistant", "content": "done"}])
|
|
result = run_async(
|
|
react_loop(
|
|
llm,
|
|
_FakeDispatcher(),
|
|
messages,
|
|
tools=[],
|
|
state=AgentState(),
|
|
settings=_settings_for_test(keep_tail=2, threshold=50_000),
|
|
max_iterations=5,
|
|
plan_required=False,
|
|
verify_required=False,
|
|
)
|
|
)
|
|
|
|
assert result == "done"
|
|
assert llm.summarize_calls > 1
|
|
giant_message = next(m for m in messages if m.get("name") == "big_tool")
|
|
assert len(giant_message["content"]) < len(giant)
|
|
|
|
|
|
def test_one_oversized_tail_message_alone_still_recovers():
|
|
giant = "X" * 300_000
|
|
messages = _long_message_history(16)
|
|
messages.append(
|
|
{"role": "tool", "tool_call_id": "1", "name": "big_tool", "content": giant}
|
|
)
|
|
messages.append({"role": "user", "content": "please continue"})
|
|
|
|
llm = _FakeLLMWithRealLimit(simulated_limit_chars=30_000)
|
|
result = run_async(
|
|
react_loop(
|
|
llm,
|
|
_FakeDispatcher(),
|
|
messages,
|
|
tools=[],
|
|
state=AgentState(),
|
|
settings=_settings_for_test(keep_tail=4),
|
|
max_iterations=10,
|
|
plan_required=False,
|
|
verify_required=False,
|
|
)
|
|
)
|
|
|
|
assert result == "Recovered answer"
|
|
assert llm.complete_calls > MAX_CONTEXT_OVERFLOW_RETRIES - 1
|
|
giant_message = next(m for m in messages if m.get("name") == "big_tool")
|
|
assert len(giant_message["content"]) < len(giant)
|
|
assert "truncated" in giant_message["content"]
|