forked from retoor/devplacepy
Comment and post layouts stay full width on small screens. Secondary actions live in the More menu. Notifications can be grouped and throttled, and Enter can submit when that profile setting is on. Emulated-tool models accept large catalogs, with reserved names aliased on the wire and the catalog cached per owner. A dropped service-worker navigation is retried once. The test server waits 120s to finish importing before it is stopped.
353 lines
13 KiB
Python
353 lines
13 KiB
Python
# retoor <retoor@molodetz.nl>
|
|
|
|
import json
|
|
|
|
import pytest
|
|
|
|
from devplacepy.services.openai_gateway import tool_emulation as te
|
|
|
|
BASH = {
|
|
"type": "function",
|
|
"function": {
|
|
"name": "bash",
|
|
"description": "Run a shell command",
|
|
"parameters": {
|
|
"type": "object",
|
|
"properties": {"command": {"type": "string"}},
|
|
"required": ["command"],
|
|
},
|
|
},
|
|
}
|
|
|
|
|
|
def _payload(**over):
|
|
body = {
|
|
"model": "m",
|
|
"messages": [
|
|
{"role": "system", "content": "Be terse."},
|
|
{"role": "user", "content": "list files"},
|
|
],
|
|
"tools": [BASH],
|
|
}
|
|
body.update(over)
|
|
return body
|
|
|
|
|
|
def _block(name="bash", arguments=None):
|
|
arguments = {"command": "ls"} if arguments is None else arguments
|
|
return f'{te.CALL_OPEN}\n{json.dumps({"name": name, "arguments": arguments})}\n{te.CALL_CLOSE}'
|
|
|
|
|
|
def test_request_strips_native_fields_and_injects_catalog():
|
|
out = te.emulate_request(_payload(tool_choice="auto", parallel_tool_calls=False))
|
|
for key in te.REMOVED_PAYLOAD_KEYS:
|
|
assert key not in out
|
|
system = out["messages"][0]["content"]
|
|
assert system.startswith("Be terse.")
|
|
assert '"name": "bash"' in system
|
|
assert te.CALL_OPEN in system
|
|
assert "at most one tool call" in system
|
|
assert out["stop"] == [te.RESPONSE_OPEN]
|
|
|
|
|
|
def test_request_does_not_mutate_the_input():
|
|
original = _payload()
|
|
snapshot = json.dumps(original, sort_keys=True)
|
|
te.emulate_request(original)
|
|
assert json.dumps(original, sort_keys=True) == snapshot
|
|
|
|
|
|
def test_system_message_is_created_when_absent():
|
|
out = te.emulate_request(_payload(messages=[{"role": "user", "content": "hi"}]))
|
|
assert out["messages"][0]["role"] == "system"
|
|
assert "<tools>" in out["messages"][0]["content"]
|
|
|
|
|
|
def test_tool_choice_none_leaves_the_catalog_out():
|
|
out = te.emulate_request(_payload(tool_choice="none"))
|
|
assert "<tools>" not in out["messages"][0]["content"]
|
|
|
|
|
|
def test_tool_choice_required_and_named_add_a_directive():
|
|
required = te.emulate_request(_payload(tool_choice="required"))
|
|
assert "MUST call at least one tool" in required["messages"][0]["content"]
|
|
named = te.emulate_request(
|
|
_payload(tool_choice={"type": "function", "function": {"name": "bash"}})
|
|
)
|
|
assert 'MUST call the tool "bash"' in named["messages"][0]["content"]
|
|
|
|
|
|
def test_history_roundtrips_into_the_same_format_the_model_emits():
|
|
history = [
|
|
{"role": "user", "content": "go"},
|
|
{
|
|
"role": "assistant",
|
|
"content": "ok",
|
|
"tool_calls": [
|
|
{
|
|
"id": "call_1",
|
|
"type": "function",
|
|
"function": {"name": "bash", "arguments": '{"command": "ls"}'},
|
|
},
|
|
{
|
|
"id": "call_2",
|
|
"type": "function",
|
|
"function": {"name": "bash", "arguments": '{"command": "pwd"}'},
|
|
},
|
|
],
|
|
},
|
|
{"role": "tool", "tool_call_id": "call_1", "content": "a.txt"},
|
|
{"role": "tool", "tool_call_id": "call_2", "content": "/tmp"},
|
|
]
|
|
out = te.emulate_request(_payload(messages=history))
|
|
roles = [m["role"] for m in out["messages"]]
|
|
assert roles == ["system", "user", "assistant", "user"]
|
|
assistant = out["messages"][2]
|
|
assert "tool_calls" not in assistant
|
|
assert assistant["content"].count(te.CALL_OPEN) == 2
|
|
results = out["messages"][3]["content"]
|
|
assert results.count(te.RESPONSE_OPEN) == 2
|
|
assert '"name": "bash"' in results and "a.txt" in results
|
|
|
|
|
|
def _many_tools(count):
|
|
return [
|
|
{
|
|
"type": "function",
|
|
"function": {
|
|
"name": f"tool_{index}",
|
|
"description": f"tool number {index}",
|
|
"parameters": {"type": "object", "properties": {}, "required": []},
|
|
},
|
|
}
|
|
for index in range(count)
|
|
]
|
|
|
|
|
|
def test_large_catalogs_are_served_with_no_cap():
|
|
out = te.emulate_request(_payload(tools=_many_tools(424)))
|
|
system = out["messages"][0]["content"]
|
|
assert '"name": "tool_0"' in system
|
|
assert '"name": "tool_423"' in system
|
|
assert "tools" not in out and "tool_choice" not in out
|
|
|
|
|
|
def test_malformed_tools_still_reject_loudly():
|
|
with pytest.raises(te.ToolEmulationError):
|
|
te.emulate_request(_payload(tools=[{"type": "retrieval"}]))
|
|
with pytest.raises(te.ToolEmulationError):
|
|
te.emulate_request(_payload(tools=[{"type": "function", "function": {}}]))
|
|
with pytest.raises(te.ToolEmulationError):
|
|
te.emulate_request(_payload(tools="bash"))
|
|
with pytest.raises(te.ToolEmulationError):
|
|
te.cached_emulated_tools(_payload(tools="bash"))
|
|
|
|
|
|
def test_catalog_cache_reuses_the_built_instruction_per_owner():
|
|
cache = te.CatalogCache()
|
|
owner = ("user", "uid-1")
|
|
first = te.emulate_request(_payload(tools=_many_tools(424)), cache, owner)
|
|
assert len(cache._entries) == 1
|
|
second = te.emulate_request(_payload(tools=_many_tools(424)), cache, owner)
|
|
assert len(cache._entries) == 1
|
|
assert second["messages"][0]["content"] == first["messages"][0]["content"]
|
|
emulated = te.cached_emulated_tools(_payload(tools=_many_tools(424)), cache, owner)
|
|
assert emulated is not None and len(emulated[0]) == 424
|
|
|
|
|
|
def test_catalog_cache_is_namespaced_per_owner_and_toolset():
|
|
cache = te.CatalogCache()
|
|
payload = _payload(tools=_many_tools(10))
|
|
te.emulate_request(payload, cache, ("user", "uid-1"))
|
|
te.emulate_request(payload, cache, ("user", "uid-2"))
|
|
assert len(cache._entries) == 2
|
|
assert cache.lookup(payload, ("user", "uid-3")) is None
|
|
changed = _payload(tools=_many_tools(11))
|
|
assert cache.lookup(changed, ("user", "uid-1")) is None
|
|
renamed_choice = _payload(tools=_many_tools(10), tool_choice="required")
|
|
assert cache.lookup(renamed_choice, ("user", "uid-1")) is None
|
|
|
|
|
|
def test_catalog_cache_evicts_oldest_first_within_bounds():
|
|
cache = te.CatalogCache()
|
|
cache.MAX_ENTRIES = 3
|
|
for index in range(5):
|
|
te.emulate_request(
|
|
_payload(tools=_many_tools(index + 1)), cache, ("user", "uid-1")
|
|
)
|
|
assert len(cache._entries) == 3
|
|
assert cache.lookup(_payload(tools=_many_tools(1)), ("user", "uid-1")) is None
|
|
assert cache.lookup(_payload(tools=_many_tools(5)), ("user", "uid-1")) is not None
|
|
|
|
|
|
def test_catalog_cache_never_caches_a_shape_error():
|
|
cache = te.CatalogCache()
|
|
with pytest.raises(te.ToolEmulationError):
|
|
te.emulate_request(_payload(tools=[{"type": "retrieval"}]), cache, ("user", "uid-1"))
|
|
assert len(cache._entries) == 0
|
|
|
|
|
|
def test_existing_stop_sequences_are_kept_and_capped():
|
|
out = te.emulate_request(_payload(stop="END"))
|
|
assert out["stop"] == ["END", te.RESPONSE_OPEN]
|
|
full = te.emulate_request(_payload(stop=["a", "b", "c", "d"]))
|
|
assert full["stop"] == ["a", "b", "c", "d"]
|
|
|
|
|
|
def test_request_without_tools_or_tool_history_is_untouched():
|
|
plain = {"model": "m", "messages": [{"role": "user", "content": "hi"}]}
|
|
assert te.emulate_request(plain) is plain
|
|
|
|
|
|
def test_parse_extracts_calls_and_remaining_text():
|
|
declared = te.declared_tools(_payload())
|
|
text = "Sure.\n" + _block() + "\n" + _block(arguments={"command": "pwd"})
|
|
remaining, calls, warnings = te.parse_tool_calls(text, declared)
|
|
assert remaining == "Sure."
|
|
assert [json.loads(c["function"]["arguments"])["command"] for c in calls] == ["ls", "pwd"]
|
|
assert all(c["id"].startswith("call_") and c["type"] == "function" for c in calls)
|
|
assert warnings == []
|
|
|
|
|
|
def test_parse_is_tolerant_of_fences_arrays_and_unterminated_blocks():
|
|
declared = te.declared_tools(_payload())
|
|
fenced = f'{te.CALL_OPEN}\n```json\n{{"name": "bash", "arguments": {{"command": "ls"}}}}\n```\n{te.CALL_CLOSE}'
|
|
assert len(te.parse_tool_calls(fenced, declared)[1]) == 1
|
|
array = f'{te.CALL_OPEN}[{{"name":"bash","arguments":{{"command":"a"}}}},{{"name":"bash","arguments":{{"command":"b"}}}}]{te.CALL_CLOSE}'
|
|
assert len(te.parse_tool_calls(array, declared)[1]) == 2
|
|
unterminated = f'{te.CALL_OPEN}{{"name": "bash", "arguments": {{"command": "ls"}}}}'
|
|
assert len(te.parse_tool_calls(unterminated, declared)[1]) == 1
|
|
string_args = f'{te.CALL_OPEN}{{"name":"bash","arguments":"{{\\"command\\": \\"ls\\"}}"}}{te.CALL_CLOSE}'
|
|
assert len(te.parse_tool_calls(string_args, declared)[1]) == 1
|
|
|
|
|
|
def test_parse_rejects_invalid_calls_loudly_and_keeps_the_text():
|
|
declared = te.declared_tools(_payload())
|
|
for bad in (
|
|
_block(name="nope"),
|
|
_block(arguments={}),
|
|
f"{te.CALL_OPEN}not json{te.CALL_CLOSE}",
|
|
):
|
|
remaining, calls, warnings = te.parse_tool_calls(bad, declared)
|
|
assert calls == []
|
|
assert warnings
|
|
assert te.CALL_OPEN in remaining
|
|
|
|
|
|
def test_parallel_and_per_turn_caps():
|
|
declared = te.declared_tools(_payload())
|
|
text = "".join(_block() for _ in range(3))
|
|
_, calls, warnings = te.parse_tool_calls(text, declared, allow_parallel=False)
|
|
assert len(calls) == 1 and warnings
|
|
many = "".join(_block() for _ in range(te.MAX_CALLS_PER_TURN + 3))
|
|
assert len(te.parse_tool_calls(many, declared)[1]) == te.MAX_CALLS_PER_TURN
|
|
|
|
|
|
def test_convert_completion_builds_openai_tool_calls():
|
|
declared = te.declared_tools(_payload())
|
|
completion = {"choices": [{"message": {"role": "assistant", "content": "x\n" + _block()}, "finish_reason": "stop"}]}
|
|
out, warnings = te.convert_completion(completion, declared)
|
|
choice = out["choices"][0]
|
|
assert choice["finish_reason"] == "tool_calls"
|
|
assert choice["message"]["content"] == "x"
|
|
assert choice["message"]["tool_calls"][0]["function"]["name"] == "bash"
|
|
assert warnings == []
|
|
|
|
|
|
def test_convert_completion_with_only_a_call_has_null_content():
|
|
declared = te.declared_tools(_payload())
|
|
completion = {"choices": [{"message": {"content": _block()}, "finish_reason": "stop"}]}
|
|
out, _ = te.convert_completion(completion, declared)
|
|
assert out["choices"][0]["message"]["content"] is None
|
|
|
|
|
|
def _chunk(content=None, finish=None, **extra):
|
|
delta = {} if content is None else {"content": content}
|
|
return {
|
|
"id": "c",
|
|
"object": "chat.completion.chunk",
|
|
"model": "m",
|
|
"choices": [{"index": 0, "delta": delta, "finish_reason": finish}],
|
|
**extra,
|
|
}
|
|
|
|
|
|
def _run_stream(pieces, declared=None, finish="stop", allow_parallel=True):
|
|
stream = te.ToolCallStream(declared or te.declared_tools(_payload()), allow_parallel)
|
|
out = []
|
|
for piece in pieces:
|
|
out.extend(stream.feed(_chunk(piece)))
|
|
out.extend(stream.feed(_chunk(None, finish)))
|
|
return out
|
|
|
|
|
|
def _texts(chunks):
|
|
return "".join(
|
|
c["choices"][0]["delta"].get("content", "") for c in chunks if c.get("choices")
|
|
)
|
|
|
|
|
|
def _calls(chunks):
|
|
found = []
|
|
for c in chunks:
|
|
for choice in c.get("choices") or []:
|
|
found.extend(choice["delta"].get("tool_calls") or [])
|
|
return found
|
|
|
|
|
|
def test_stream_passes_plain_text_through_unchanged():
|
|
out = _run_stream(["hel", "lo ", "world"])
|
|
assert _texts(out) == "hello world"
|
|
assert _calls(out) == []
|
|
assert out[-1]["choices"][0]["finish_reason"] == "stop"
|
|
|
|
|
|
def test_stream_converts_a_call_split_across_arbitrary_chunk_boundaries():
|
|
full = "Sure.\n" + _block()
|
|
for size in (1, 2, 3, 5, 7, 50):
|
|
pieces = [full[i : i + size] for i in range(0, len(full), size)]
|
|
out = _run_stream(pieces)
|
|
assert _texts(out).strip() == "Sure."
|
|
calls = _calls(out)
|
|
assert len(calls) == 1
|
|
assert calls[0]["function"]["name"] == "bash"
|
|
assert json.loads(calls[0]["function"]["arguments"]) == {"command": "ls"}
|
|
assert calls[0]["index"] == 0
|
|
assert out[-1]["choices"][0]["finish_reason"] == "tool_calls"
|
|
|
|
|
|
def test_stream_never_leaks_call_markup_into_content():
|
|
out = _run_stream(["a <tool", "_call>", '{"name":"bash","arguments":{"command":"ls"}}', "</tool_call>"])
|
|
assert te.CALL_OPEN not in _texts(out)
|
|
|
|
|
|
def test_stream_emits_a_lookalike_prefix_that_never_became_a_call():
|
|
out = _run_stream(["use <tool", " carefully"])
|
|
assert _texts(out) == "use <tool carefully"
|
|
|
|
|
|
def test_stream_invalid_call_is_flushed_as_text_with_a_warning():
|
|
stream = te.ToolCallStream(te.declared_tools(_payload()))
|
|
out = []
|
|
for piece in [_block(name="nope")]:
|
|
out.extend(stream.feed(_chunk(piece)))
|
|
out.extend(stream.feed(_chunk(None, "stop")))
|
|
assert _calls(out) == []
|
|
assert te.CALL_OPEN in _texts(out)
|
|
assert stream.warnings
|
|
|
|
|
|
def test_stream_without_a_finish_chunk_flushes_on_finish():
|
|
stream = te.ToolCallStream(te.declared_tools(_payload()))
|
|
out = stream.feed(_chunk(_block()))
|
|
out.extend(stream.finish())
|
|
assert len(_calls(out)) == 1
|
|
|
|
|
|
def test_stream_keeps_usage_chunks_and_other_deltas():
|
|
stream = te.ToolCallStream(te.declared_tools(_payload()))
|
|
usage = {"id": "c", "choices": [], "usage": {"total_tokens": 3}}
|
|
assert stream.feed(usage) == [usage]
|
|
out = stream.feed({"id": "c", "choices": [{"index": 0, "delta": {"role": "assistant"}, "finish_reason": None}]})
|
|
assert out[0]["choices"][0]["delta"] == {"role": "assistant"}
|