forked from retoor/devplacepy
287 lines
10 KiB
Python
287 lines
10 KiB
Python
# retoor <retoor@molodetz.nl>
|
|
|
|
import json
|
|
|
|
import pytest
|
|
|
|
from devplacepy.services.openai_gateway import tool_emulation as te
|
|
|
|
BASH = {
|
|
"type": "function",
|
|
"function": {
|
|
"name": "bash",
|
|
"description": "Run a shell command",
|
|
"parameters": {
|
|
"type": "object",
|
|
"properties": {"command": {"type": "string"}},
|
|
"required": ["command"],
|
|
},
|
|
},
|
|
}
|
|
|
|
|
|
def _payload(**over):
|
|
body = {
|
|
"model": "m",
|
|
"messages": [
|
|
{"role": "system", "content": "Be terse."},
|
|
{"role": "user", "content": "list files"},
|
|
],
|
|
"tools": [BASH],
|
|
}
|
|
body.update(over)
|
|
return body
|
|
|
|
|
|
def _block(name="bash", arguments=None):
|
|
arguments = {"command": "ls"} if arguments is None else arguments
|
|
return f'{te.CALL_OPEN}\n{json.dumps({"name": name, "arguments": arguments})}\n{te.CALL_CLOSE}'
|
|
|
|
|
|
def test_request_strips_native_fields_and_injects_catalog():
|
|
out = te.emulate_request(_payload(tool_choice="auto", parallel_tool_calls=False))
|
|
for key in te.REMOVED_PAYLOAD_KEYS:
|
|
assert key not in out
|
|
system = out["messages"][0]["content"]
|
|
assert system.startswith("Be terse.")
|
|
assert '"name": "bash"' in system
|
|
assert te.CALL_OPEN in system
|
|
assert "at most one tool call" in system
|
|
assert out["stop"] == [te.RESPONSE_OPEN]
|
|
|
|
|
|
def test_request_does_not_mutate_the_input():
|
|
original = _payload()
|
|
snapshot = json.dumps(original, sort_keys=True)
|
|
te.emulate_request(original)
|
|
assert json.dumps(original, sort_keys=True) == snapshot
|
|
|
|
|
|
def test_system_message_is_created_when_absent():
|
|
out = te.emulate_request(_payload(messages=[{"role": "user", "content": "hi"}]))
|
|
assert out["messages"][0]["role"] == "system"
|
|
assert "<tools>" in out["messages"][0]["content"]
|
|
|
|
|
|
def test_tool_choice_none_leaves_the_catalog_out():
|
|
out = te.emulate_request(_payload(tool_choice="none"))
|
|
assert "<tools>" not in out["messages"][0]["content"]
|
|
|
|
|
|
def test_tool_choice_required_and_named_add_a_directive():
|
|
required = te.emulate_request(_payload(tool_choice="required"))
|
|
assert "MUST call at least one tool" in required["messages"][0]["content"]
|
|
named = te.emulate_request(
|
|
_payload(tool_choice={"type": "function", "function": {"name": "bash"}})
|
|
)
|
|
assert 'MUST call the tool "bash"' in named["messages"][0]["content"]
|
|
|
|
|
|
def test_history_roundtrips_into_the_same_format_the_model_emits():
|
|
history = [
|
|
{"role": "user", "content": "go"},
|
|
{
|
|
"role": "assistant",
|
|
"content": "ok",
|
|
"tool_calls": [
|
|
{
|
|
"id": "call_1",
|
|
"type": "function",
|
|
"function": {"name": "bash", "arguments": '{"command": "ls"}'},
|
|
},
|
|
{
|
|
"id": "call_2",
|
|
"type": "function",
|
|
"function": {"name": "bash", "arguments": '{"command": "pwd"}'},
|
|
},
|
|
],
|
|
},
|
|
{"role": "tool", "tool_call_id": "call_1", "content": "a.txt"},
|
|
{"role": "tool", "tool_call_id": "call_2", "content": "/tmp"},
|
|
]
|
|
out = te.emulate_request(_payload(messages=history))
|
|
roles = [m["role"] for m in out["messages"]]
|
|
assert roles == ["system", "user", "assistant", "user"]
|
|
assistant = out["messages"][2]
|
|
assert "tool_calls" not in assistant
|
|
assert assistant["content"].count(te.CALL_OPEN) == 2
|
|
results = out["messages"][3]["content"]
|
|
assert results.count(te.RESPONSE_OPEN) == 2
|
|
assert '"name": "bash"' in results and "a.txt" in results
|
|
|
|
|
|
def test_caps_reject_loudly():
|
|
with pytest.raises(te.ToolEmulationError):
|
|
te.emulate_request(_payload(tools=[BASH] * (te.MAX_TOOLS + 1)))
|
|
huge = json.loads(json.dumps(BASH))
|
|
huge["function"]["description"] = "x" * (te.MAX_CATALOG_CHARS + 1)
|
|
with pytest.raises(te.ToolEmulationError):
|
|
te.emulate_request(_payload(tools=[huge]))
|
|
with pytest.raises(te.ToolEmulationError):
|
|
te.emulate_request(_payload(tools=[{"type": "retrieval"}]))
|
|
|
|
|
|
def test_existing_stop_sequences_are_kept_and_capped():
|
|
out = te.emulate_request(_payload(stop="END"))
|
|
assert out["stop"] == ["END", te.RESPONSE_OPEN]
|
|
full = te.emulate_request(_payload(stop=["a", "b", "c", "d"]))
|
|
assert full["stop"] == ["a", "b", "c", "d"]
|
|
|
|
|
|
def test_request_without_tools_or_tool_history_is_untouched():
|
|
plain = {"model": "m", "messages": [{"role": "user", "content": "hi"}]}
|
|
assert te.emulate_request(plain) is plain
|
|
|
|
|
|
def test_parse_extracts_calls_and_remaining_text():
|
|
declared = te.declared_tools(_payload())
|
|
text = "Sure.\n" + _block() + "\n" + _block(arguments={"command": "pwd"})
|
|
remaining, calls, warnings = te.parse_tool_calls(text, declared)
|
|
assert remaining == "Sure."
|
|
assert [json.loads(c["function"]["arguments"])["command"] for c in calls] == ["ls", "pwd"]
|
|
assert all(c["id"].startswith("call_") and c["type"] == "function" for c in calls)
|
|
assert warnings == []
|
|
|
|
|
|
def test_parse_is_tolerant_of_fences_arrays_and_unterminated_blocks():
|
|
declared = te.declared_tools(_payload())
|
|
fenced = f'{te.CALL_OPEN}\n```json\n{{"name": "bash", "arguments": {{"command": "ls"}}}}\n```\n{te.CALL_CLOSE}'
|
|
assert len(te.parse_tool_calls(fenced, declared)[1]) == 1
|
|
array = f'{te.CALL_OPEN}[{{"name":"bash","arguments":{{"command":"a"}}}},{{"name":"bash","arguments":{{"command":"b"}}}}]{te.CALL_CLOSE}'
|
|
assert len(te.parse_tool_calls(array, declared)[1]) == 2
|
|
unterminated = f'{te.CALL_OPEN}{{"name": "bash", "arguments": {{"command": "ls"}}}}'
|
|
assert len(te.parse_tool_calls(unterminated, declared)[1]) == 1
|
|
string_args = f'{te.CALL_OPEN}{{"name":"bash","arguments":"{{\\"command\\": \\"ls\\"}}"}}{te.CALL_CLOSE}'
|
|
assert len(te.parse_tool_calls(string_args, declared)[1]) == 1
|
|
|
|
|
|
def test_parse_rejects_invalid_calls_loudly_and_keeps_the_text():
|
|
declared = te.declared_tools(_payload())
|
|
for bad in (
|
|
_block(name="nope"),
|
|
_block(arguments={}),
|
|
f"{te.CALL_OPEN}not json{te.CALL_CLOSE}",
|
|
):
|
|
remaining, calls, warnings = te.parse_tool_calls(bad, declared)
|
|
assert calls == []
|
|
assert warnings
|
|
assert te.CALL_OPEN in remaining
|
|
|
|
|
|
def test_parallel_and_per_turn_caps():
|
|
declared = te.declared_tools(_payload())
|
|
text = "".join(_block() for _ in range(3))
|
|
_, calls, warnings = te.parse_tool_calls(text, declared, allow_parallel=False)
|
|
assert len(calls) == 1 and warnings
|
|
many = "".join(_block() for _ in range(te.MAX_CALLS_PER_TURN + 3))
|
|
assert len(te.parse_tool_calls(many, declared)[1]) == te.MAX_CALLS_PER_TURN
|
|
|
|
|
|
def test_convert_completion_builds_openai_tool_calls():
|
|
declared = te.declared_tools(_payload())
|
|
completion = {"choices": [{"message": {"role": "assistant", "content": "x\n" + _block()}, "finish_reason": "stop"}]}
|
|
out, warnings = te.convert_completion(completion, declared)
|
|
choice = out["choices"][0]
|
|
assert choice["finish_reason"] == "tool_calls"
|
|
assert choice["message"]["content"] == "x"
|
|
assert choice["message"]["tool_calls"][0]["function"]["name"] == "bash"
|
|
assert warnings == []
|
|
|
|
|
|
def test_convert_completion_with_only_a_call_has_null_content():
|
|
declared = te.declared_tools(_payload())
|
|
completion = {"choices": [{"message": {"content": _block()}, "finish_reason": "stop"}]}
|
|
out, _ = te.convert_completion(completion, declared)
|
|
assert out["choices"][0]["message"]["content"] is None
|
|
|
|
|
|
def _chunk(content=None, finish=None, **extra):
|
|
delta = {} if content is None else {"content": content}
|
|
return {
|
|
"id": "c",
|
|
"object": "chat.completion.chunk",
|
|
"model": "m",
|
|
"choices": [{"index": 0, "delta": delta, "finish_reason": finish}],
|
|
**extra,
|
|
}
|
|
|
|
|
|
def _run_stream(pieces, declared=None, finish="stop", allow_parallel=True):
|
|
stream = te.ToolCallStream(declared or te.declared_tools(_payload()), allow_parallel)
|
|
out = []
|
|
for piece in pieces:
|
|
out.extend(stream.feed(_chunk(piece)))
|
|
out.extend(stream.feed(_chunk(None, finish)))
|
|
return out
|
|
|
|
|
|
def _texts(chunks):
|
|
return "".join(
|
|
c["choices"][0]["delta"].get("content", "") for c in chunks if c.get("choices")
|
|
)
|
|
|
|
|
|
def _calls(chunks):
|
|
found = []
|
|
for c in chunks:
|
|
for choice in c.get("choices") or []:
|
|
found.extend(choice["delta"].get("tool_calls") or [])
|
|
return found
|
|
|
|
|
|
def test_stream_passes_plain_text_through_unchanged():
|
|
out = _run_stream(["hel", "lo ", "world"])
|
|
assert _texts(out) == "hello world"
|
|
assert _calls(out) == []
|
|
assert out[-1]["choices"][0]["finish_reason"] == "stop"
|
|
|
|
|
|
def test_stream_converts_a_call_split_across_arbitrary_chunk_boundaries():
|
|
full = "Sure.\n" + _block()
|
|
for size in (1, 2, 3, 5, 7, 50):
|
|
pieces = [full[i : i + size] for i in range(0, len(full), size)]
|
|
out = _run_stream(pieces)
|
|
assert _texts(out).strip() == "Sure."
|
|
calls = _calls(out)
|
|
assert len(calls) == 1
|
|
assert calls[0]["function"]["name"] == "bash"
|
|
assert json.loads(calls[0]["function"]["arguments"]) == {"command": "ls"}
|
|
assert calls[0]["index"] == 0
|
|
assert out[-1]["choices"][0]["finish_reason"] == "tool_calls"
|
|
|
|
|
|
def test_stream_never_leaks_call_markup_into_content():
|
|
out = _run_stream(["a <tool", "_call>", '{"name":"bash","arguments":{"command":"ls"}}', "</tool_call>"])
|
|
assert te.CALL_OPEN not in _texts(out)
|
|
|
|
|
|
def test_stream_emits_a_lookalike_prefix_that_never_became_a_call():
|
|
out = _run_stream(["use <tool", " carefully"])
|
|
assert _texts(out) == "use <tool carefully"
|
|
|
|
|
|
def test_stream_invalid_call_is_flushed_as_text_with_a_warning():
|
|
stream = te.ToolCallStream(te.declared_tools(_payload()))
|
|
out = []
|
|
for piece in [_block(name="nope")]:
|
|
out.extend(stream.feed(_chunk(piece)))
|
|
out.extend(stream.feed(_chunk(None, "stop")))
|
|
assert _calls(out) == []
|
|
assert te.CALL_OPEN in _texts(out)
|
|
assert stream.warnings
|
|
|
|
|
|
def test_stream_without_a_finish_chunk_flushes_on_finish():
|
|
stream = te.ToolCallStream(te.declared_tools(_payload()))
|
|
out = stream.feed(_chunk(_block()))
|
|
out.extend(stream.finish())
|
|
assert len(_calls(out)) == 1
|
|
|
|
|
|
def test_stream_keeps_usage_chunks_and_other_deltas():
|
|
stream = te.ToolCallStream(te.declared_tools(_payload()))
|
|
usage = {"id": "c", "choices": [], "usage": {"total_tokens": 3}}
|
|
assert stream.feed(usage) == [usage]
|
|
out = stream.feed({"id": "c", "choices": [{"index": 0, "delta": {"role": "assistant"}, "finish_reason": None}]})
|
|
assert out[0]["choices"][0]["delta"] == {"role": "assistant"}
|