From c4f7d01b2db490cfe6e598bdcecca1d0e35422f1 Mon Sep 17 00:00:00 2001 From: retoor Date: Thu, 10 Sep 2026 15:09:22 +0200 Subject: [PATCH] Update werkend --- devplacepy/services/devii/CLAUDE.md | 2 + .../services/devii/agentic/compaction.py | 12 +++++ devplacepy/services/devii/agentic/loop.py | 37 +++++++++---- devplacepy/services/devii/llm.py | 40 ++++++++++++++ pyproject.toml | 2 +- tests/unit/services/devii/agentic/loop.py | 53 +++++++++++++++++++ tests/unit/services/devii/llm.py | 46 ++++++++++++++++ 7 files changed, 180 insertions(+), 12 deletions(-) create mode 100644 tests/unit/services/devii/llm.py diff --git a/devplacepy/services/devii/CLAUDE.md b/devplacepy/services/devii/CLAUDE.md index 12cc692..f7629eb 100644 --- a/devplacepy/services/devii/CLAUDE.md +++ b/devplacepy/services/devii/CLAUDE.md @@ -108,6 +108,8 @@ The upstream model `deepseek-v4-flash` (what `deepseek-chat` routes to; default **Reactive recovery when the char-based budget above is wrong for the model actually serving the request (hard rule).** The proactive `context_compact_threshold` check is only ever as good as the assumed `CONTEXT_WINDOW_TOKENS` for whatever model answers a given call - a gateway reroute/fallback to a smaller-context backend silently invalidates it, and the agent would keep sending oversized requests until the provider itself rejects one with `400` (a real production case: provider capped at 131,072 tokens against a budget tuned for 1,048,576). `react_loop` (`agentic/loop.py`) does NOT just surface that error as `[model error] ...` and give up - `agentic/compaction.py` `is_context_length_error(exc)` recognizes it (OpenAI-style `error.code == "context_length_exceeded"`, or a phrase match on "maximum context length"/"reduce the length"/etc., since provider error shapes vary), and on a match the loop retries up to `MAX_CONTEXT_OVERFLOW_RETRIES` (5) times, geometrically halving two independent knobs each attempt: `context_keep_tail` (how many recent messages survive compaction verbatim) and a per-message character cap fed to `shrink_large_messages` (starting at `CONTEXT_OVERFLOW_MESSAGE_CAP_START`=200,000, floor `CONTEXT_OVERFLOW_MESSAGE_CAP_FLOOR`=4,000). **The shrink step is load-bearing and runs BEFORE `compact_messages` on every retry**: `compact_messages` only ever decides which whole messages survive vs. get summarized - it never touches an individual message's own size, so one oversized message (a big tool result) sitting in the always-kept tail defeats every retry no matter how far `keep_tail` drops, which is exactly the failure this fixes (a real incident: two retries of keep_tail-only reduction left the request essentially unchanged, 349,784 -> 350,050 input tokens, because the offending message never left the tail). Shrinking first also bounds what `compact_messages`'s own `summarize()` call has to ingest, so the summarizer itself is less likely to hit the same wall. Only after exhausting all 5 attempts does it fail closed with the provider's own message. This is the same reactive-at-the-chokepoint philosophy as the 401 stale-API-key self-healing above - never try to guess the right model beforehand, react to what the provider actually says, and don't stop at "fewer messages" when the real problem is "one message too big." +**Proactive compaction targets 40% headroom, not just "under the threshold" (a third real incident of this same failure class).** The proactive check (`context_size(messages) > settings.context_compact_threshold`, top of every `react_loop` iteration) used to run a single `compact_messages` pass and stop the moment the result was merely under the threshold - which, for a conversation with a large tool result sitting in the always-kept tail (a fetched web page, a doc dump), left the context only barely under it. The very next `fetch_url`/`search_docs` round trip pushed it back over, so compaction re-fired almost every turn - visible in a trace as `! context compaction` repeating every 1-2 tool calls with no real progress. The fix: the proactive step now loops (bounded by `MAX_PROACTIVE_COMPACT_ATTEMPTS`=5, `agentic/loop.py`), geometrically halving `keep_tail` and a per-message shrink cap exactly like the reactive retry above, until `context_size(messages)` drops to `CONTEXT_COMPACT_TARGET_RATIO` (`agentic/compaction.py`, 0.4) of `context_compact_threshold` - or until an attempt makes no further progress (`size_after >= size_before`), which stops the loop instead of spinning uselessly. Both the proactive loop and the reactive retry now share one helper, `compaction.shrink_and_compact(llm, messages, keep_tail, message_cap, summary_max_chars)` (shrink oversized messages, then compact) - the reactive branch no longer calls `shrink_large_messages`/`compact_messages` separately. The 40% target is deliberately well below 100%: it buys several turns of headroom before the next proactive compaction has to fire at all, rather than living turn-to-turn at the edge of the threshold. Regression-guarded by `tests/unit/services/devii/agentic/loop.py` (`test_proactive_compaction_targets_40_percent_of_threshold`, `test_proactive_compaction_retries_when_first_pass_is_not_enough`). + **`find_compaction_split` must accept any safe boundary, not only a `user`-role message (a second real incident of this same failure class).** `compact_messages` needs a split point where `messages[1:split]` (summarized away) and `messages[split:]` (kept verbatim) each stay internally self-contained - a tool result can never be separated from the assistant `tool_calls` message that produced it, or the provider rejects the request as malformed on the very next call. The original `find_compaction_split` enforced this by scanning backward from `len(messages) - keep_tail` for a `role == "user"` message and returning `1` (a **total no-op** - `compact_messages` then returns `messages` completely unchanged) if none was found before reaching index 1. That guard is correct for ordinary multi-turn chat, but a long single-turn agentic run (many consecutive `assistant` tool_calls + `tool` result pairs with no interleaved `user` message, e.g. a big scheduled task or a chat turn that just keeps calling tools) has **no `user`-role message anywhere in the scan range**, so both the proactive compaction (`context_size(messages) > context_compact_threshold`, checked every iteration) and the reactive overflow retry above silently did nothing at all, turn after turn, while `context_overflow_attempts` still climbed to `MAX_CONTEXT_OVERFLOW_RETRIES` and the loop still failed closed with the provider's raw 400 - "compaction" fired (visibly, as repeated `compact-overflow` trace events) but never actually shrank anything. The fix: track the closest safe fallback boundary (`role != "tool"`, i.e. `user` **or** a non-tool-calling point) seen during the same backward scan, and use it when no `user` message turns up - `assistant` is just as safe a split point as `user` since neither leaves an orphaned tool result on either side of the cut. Given the conversation always starts `[system, user, assistant, ...]`, this fallback is always found once `len(messages) >= keep_tail + 3` (the precondition `compact_messages` already checks before calling it), so the function can no longer degenerate to a permanent no-op. Regression-guarded by `tests/unit/services/devii/agentic/compaction.py` (`test_find_compaction_split_falls_back_to_a_non_tool_boundary_without_a_recent_user_message`, `test_find_compaction_split_never_lands_inside_a_tool_result_run`, `test_compact_messages_shrinks_a_tool_heavy_conversation_with_no_recent_user_message`) - each one fails against the pre-fix code with a synthetic tool-heavy, user-message-free conversation. ## Multi-worker service-lock routing diff --git a/devplacepy/services/devii/agentic/compaction.py b/devplacepy/services/devii/agentic/compaction.py index ea79567..23b3493 100644 --- a/devplacepy/services/devii/agentic/compaction.py +++ b/devplacepy/services/devii/agentic/compaction.py @@ -12,6 +12,7 @@ from ..text import normalize_newlines logger = logging.getLogger("devii.agentic.compaction") SUMMARY_INPUT_CAP = 600_000 +CONTEXT_COMPACT_TARGET_RATIO = 0.4 CONTEXT_LENGTH_ERROR_CODES = {"context_length_exceeded"} CONTEXT_LENGTH_ERROR_PHRASES = ( "maximum context length", @@ -149,3 +150,14 @@ async def compact_messages( }, *tail, ] + + +async def shrink_and_compact( + llm: Any, + messages: list[dict[str, Any]], + keep_tail: int, + message_cap: int, + summary_max_chars: int, +) -> list[dict[str, Any]]: + shrink_large_messages(messages, message_cap) + return await compact_messages(llm, messages, keep_tail, summary_max_chars) diff --git a/devplacepy/services/devii/agentic/loop.py b/devplacepy/services/devii/agentic/loop.py index 768803b..c6ffb1c 100644 --- a/devplacepy/services/devii/agentic/loop.py +++ b/devplacepy/services/devii/agentic/loop.py @@ -12,10 +12,10 @@ from ..errors import LLMError from ..chunks import reset_store, set_store from ..cost import reset_tracker, set_tracker from .compaction import ( - compact_messages, + CONTEXT_COMPACT_TARGET_RATIO, context_size, is_context_length_error, - shrink_large_messages, + shrink_and_compact, ) from .state import AgentState, reset_state, set_state @@ -24,6 +24,7 @@ logger = logging.getLogger("devii.agentic.loop") TraceCallback = Callable[[str, str, str], None] OUTPUT_CAP_CHARS = 400_000 MAX_CONTEXT_OVERFLOW_RETRIES = 5 +MAX_PROACTIVE_COMPACT_ATTEMPTS = 5 CONTEXT_OVERFLOW_MESSAGE_CAP_START = 200_000 CONTEXT_OVERFLOW_MESSAGE_CAP_FLOOR = 4_000 @@ -214,13 +215,28 @@ async def react_loop( state.iteration += 1 if context_size(messages) > settings.context_compact_threshold: - trace("compact") - messages[:] = await compact_messages( - llm, - messages, - settings.context_keep_tail, - settings.context_summary_max_chars, + target_size = int( + settings.context_compact_threshold * CONTEXT_COMPACT_TARGET_RATIO ) + for compact_attempt in range(1, MAX_PROACTIVE_COMPACT_ATTEMPTS + 1): + trace("compact") + shrink_factor = 2 ** (compact_attempt - 1) + message_cap = max( + CONTEXT_OVERFLOW_MESSAGE_CAP_FLOOR, + CONTEXT_OVERFLOW_MESSAGE_CAP_START // shrink_factor, + ) + keep_tail = max(2, settings.context_keep_tail // shrink_factor) + summary_max_chars = max( + CONTEXT_OVERFLOW_MESSAGE_CAP_FLOOR, + settings.context_summary_max_chars // shrink_factor, + ) + size_before = context_size(messages) + messages[:] = await shrink_and_compact( + llm, messages, keep_tail, message_cap, summary_max_chars + ) + size_after = context_size(messages) + if size_after <= target_size or size_after >= size_before: + break try: message = await llm.complete(messages, tools) @@ -250,9 +266,8 @@ async def react_loop( message_cap, keep_tail, ) - shrink_large_messages(messages, message_cap) - messages[:] = await compact_messages( - llm, messages, keep_tail, summary_max_chars + messages[:] = await shrink_and_compact( + llm, messages, keep_tail, message_cap, summary_max_chars ) continue logger.info("LLM error: %s", exc.message) diff --git a/devplacepy/services/devii/llm.py b/devplacepy/services/devii/llm.py index 78da7da..e8a0290 100644 --- a/devplacepy/services/devii/llm.py +++ b/devplacepy/services/devii/llm.py @@ -3,6 +3,7 @@ from __future__ import annotations import logging +import re from typing import Any, Callable import httpx @@ -14,6 +15,44 @@ from .errors import LLMError logger = logging.getLogger("devii.llm") +# Some upstreams (observed live with openai/gpt-oss-20b via OpenRouter) use +# OpenAI's "Harmony" response format internally and occasionally fail to +# strip its channel-routing special tokens (<|channel|>commentary, +# <|message|>, <|end|>, ...) before filling tool_calls[].function.name - +# the model then tries to call e.g. "plan<|channel|>commentary", gets an +# unrecognized-tool error, and spirals trying to guess a fixed call syntax +# instead of just retrying "plan" cleanly. Sanitized once here, the single +# choke point every completion (main loop, delegate, eval) goes through, so +# neither the dispatcher nor the model's own context ever sees the leak. +_SPECIAL_TOKEN_RE = re.compile(r"<\|[^|>]*\|>") + + +def _sanitize_tool_name(name: str) -> str: + match = _SPECIAL_TOKEN_RE.search(name) + if match is None: + return name + cleaned = name[: match.start()].strip() + return cleaned or name + + +def _sanitize_tool_calls(message: dict[str, Any]) -> None: + for call in message.get("tool_calls") or []: + function = call.get("function") + if not isinstance(function, dict): + continue + raw_name = function.get("name", "") + if not isinstance(raw_name, str): + continue + clean_name = _sanitize_tool_name(raw_name) + if clean_name != raw_name: + logger.warning( + "Upstream tool call name contained a leaked special token, " + "sanitized %r -> %r", + raw_name, + clean_name, + ) + function["name"] = clean_name + class LLMClient: def __init__( @@ -91,6 +130,7 @@ class LLMClient: if message is None: raise LLMError("Model response contained no message.", body=str(data)[:500]) + _sanitize_tool_calls(message) record_usage(data.get("usage")) self._record_native_cost(response) logger.debug( diff --git a/pyproject.toml b/pyproject.toml index d1851b0..19dbf41 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "devplacepy" -version = "1.0.14" +version = "1.0.15" description = "DevPlace - The Developer Social Network" requires-python = ">=3.12" dependencies = [ diff --git a/tests/unit/services/devii/agentic/loop.py b/tests/unit/services/devii/agentic/loop.py index c469841..1ed2cd3 100644 --- a/tests/unit/services/devii/agentic/loop.py +++ b/tests/unit/services/devii/agentic/loop.py @@ -206,6 +206,59 @@ class _FakeLLMWithRealLimit: return "short summary" +def test_proactive_compaction_targets_40_percent_of_threshold(): + messages = _long_message_history(100) + threshold = context_size(messages) - 50 + llm = _FakeLLM([{"role": "assistant", "content": "done"}]) + trace_events = [] + result = run_async( + react_loop( + llm, + _FakeDispatcher(), + messages, + tools=[], + state=AgentState(), + settings=_settings_for_test(keep_tail=4, threshold=threshold), + max_iterations=5, + plan_required=False, + verify_required=False, + on_trace=lambda event, name, detail: trace_events.append(event), + ) + ) + assert result == "done" + assert "compact" in trace_events + assert context_size(messages) <= int(threshold * 0.4) + + +def test_proactive_compaction_retries_when_first_pass_is_not_enough(): + giant = "X" * 300_000 + messages = _long_message_history(20) + messages.append( + {"role": "tool", "tool_call_id": "1", "name": "big_tool", "content": giant} + ) + messages.append({"role": "user", "content": "please continue"}) + + llm = _FakeLLM([{"role": "assistant", "content": "done"}]) + result = run_async( + react_loop( + llm, + _FakeDispatcher(), + messages, + tools=[], + state=AgentState(), + settings=_settings_for_test(keep_tail=2, threshold=50_000), + max_iterations=5, + plan_required=False, + verify_required=False, + ) + ) + + assert result == "done" + assert llm.summarize_calls > 1 + giant_message = next(m for m in messages if m.get("name") == "big_tool") + assert len(giant_message["content"]) < len(giant) + + def test_one_oversized_tail_message_alone_still_recovers(): giant = "X" * 300_000 messages = _long_message_history(16) diff --git a/tests/unit/services/devii/llm.py b/tests/unit/services/devii/llm.py new file mode 100644 index 0000000..648adc3 --- /dev/null +++ b/tests/unit/services/devii/llm.py @@ -0,0 +1,46 @@ +# retoor + +from devplacepy.services.devii.llm import _sanitize_tool_calls, _sanitize_tool_name + + +def test_sanitize_tool_name_leaves_clean_names_untouched(): + assert _sanitize_tool_name("plan") == "plan" + assert _sanitize_tool_name("project_write_file") == "project_write_file" + + +def test_sanitize_tool_name_strips_a_harmony_channel_leak(): + assert _sanitize_tool_name("plan<|channel|>commentary") == "plan" + + +def test_sanitize_tool_name_strips_any_special_token_style_suffix(): + assert _sanitize_tool_name("verify<|message|>") == "verify" + assert _sanitize_tool_name("recall<|end|>") == "recall" + + +def test_sanitize_tool_name_falls_back_to_original_if_nothing_survives(): + assert _sanitize_tool_name("<|channel|>commentary") == "<|channel|>commentary" + + +def test_sanitize_tool_calls_mutates_leaked_names_in_place(): + message = { + "role": "assistant", + "tool_calls": [ + {"id": "1", "function": {"name": "plan<|channel|>commentary", "arguments": "{}"}}, + {"id": "2", "function": {"name": "verify", "arguments": "{}"}}, + ], + } + _sanitize_tool_calls(message) + assert message["tool_calls"][0]["function"]["name"] == "plan" + assert message["tool_calls"][1]["function"]["name"] == "verify" + + +def test_sanitize_tool_calls_handles_no_tool_calls(): + message = {"role": "assistant", "content": "hi"} + _sanitize_tool_calls(message) + assert message == {"role": "assistant", "content": "hi"} + + +def test_sanitize_tool_calls_ignores_malformed_function_entries(): + message = {"tool_calls": [{"id": "1", "function": "not-a-dict"}]} + _sanitize_tool_calls(message) + assert message["tool_calls"][0]["function"] == "not-a-dict"