forked from retoor/devplacepy
Add community node protocol spec, Agent Skills, and make the failing tests hermetic
Community nodes (Phase 1 of plan.md): the protocol specification in services/nodes/spec.py as the single source of truth, GET /nodes/spec.json, a generated protocol docs page, the Community nodes API docs group, and the implementation plan plus the residential proxy research. The no-open-proxy rule is enforced in the spec (open_proxy cannot be set). Agent Skills: register the Skills API docs group, cascade skills on account deletion, add the nested CLAUDE.md, README and routes docs, and cover the parser, the fetcher (end to end against a mock origin), the data layer and the routes with 73 unit and 16 api tests. Tests made hermetic with the repo's existing patterns instead of ambient environment: embeddings tests set OPENROUTER_API_KEY via monkeypatch, the workspace tests install FakeBackend via runtime.set_backend, and the skills install tests use run_async. DockerCliBackend now returns 127 (command not found) when the CLI is missing instead of raising FileNotFoundError, so a host without docker answers with a clean error rather than a 500. The Providers tab empty state no longer names the Model routes tab. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
@@ -0,0 +1,44 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from devplacepy.services.containers.backend.docker_cli import (
|
||||
COMMAND_NOT_FOUND,
|
||||
DockerCliBackend,
|
||||
)
|
||||
from tests.conftest import run_async
|
||||
|
||||
MISSING = "devplace-no-such-container-cli"
|
||||
|
||||
|
||||
def test_a_missing_cli_is_a_command_not_found_result_not_an_exception():
|
||||
backend = DockerCliBackend(binary=MISSING)
|
||||
code, out, err = run_async(backend._run([MISSING, "ps"]))
|
||||
assert code == COMMAND_NOT_FOUND == 127
|
||||
assert out == ""
|
||||
assert MISSING in err and "not found" in err
|
||||
|
||||
|
||||
def test_a_missing_cli_means_the_image_does_not_exist():
|
||||
backend = DockerCliBackend(binary=MISSING)
|
||||
assert run_async(backend.image_exists("ppy:latest")) is False
|
||||
|
||||
|
||||
def test_a_missing_cli_reports_through_the_stream_callback():
|
||||
backend = DockerCliBackend(binary=MISSING)
|
||||
lines = []
|
||||
|
||||
async def on_log(line):
|
||||
lines.append(line)
|
||||
|
||||
code = run_async(backend._stream([MISSING, "run"], on_log))
|
||||
assert code == COMMAND_NOT_FOUND
|
||||
assert lines == [f"{MISSING}: command not found"]
|
||||
|
||||
|
||||
def test_a_failing_check_names_the_missing_cli():
|
||||
backend = DockerCliBackend(binary=MISSING)
|
||||
try:
|
||||
run_async(backend._check([MISSING, "info"]))
|
||||
except RuntimeError as exc:
|
||||
assert "127" in str(exc) and "not found" in str(exc)
|
||||
else:
|
||||
raise AssertionError("expected a RuntimeError")
|
||||
@@ -0,0 +1 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
@@ -0,0 +1,73 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
import pytest
|
||||
|
||||
from devplacepy.services.nodes import spec
|
||||
from devplacepy.services.nodes.spec import SPEC
|
||||
|
||||
|
||||
def test_open_proxy_is_never_enabled():
|
||||
assert SPEC.open_proxy is False
|
||||
assert SPEC.describe()["open_proxy"] is False
|
||||
|
||||
|
||||
def test_ports_are_web_only():
|
||||
assert spec.ALLOWED_PORTS == (80, 443)
|
||||
for destination in SPEC.destinations:
|
||||
assert set(destination.ports) <= {80, 443}
|
||||
|
||||
|
||||
def test_frame_types_unique_per_direction():
|
||||
seen = [(item.type, item.direction) for item in SPEC.frames]
|
||||
assert len(seen) == len(set(seen))
|
||||
|
||||
|
||||
def test_every_frame_fits_the_wire_cap():
|
||||
for item in SPEC.frames:
|
||||
assert 0 < item.max_bytes <= 3 * 1024 * 1024
|
||||
assert item.direction in (spec.CLIENT_TO_SERVER, spec.SERVER_TO_CLIENT)
|
||||
|
||||
|
||||
def test_frame_lookup():
|
||||
assert SPEC.frame("hello", spec.CLIENT_TO_SERVER).required()[0] == "protocol"
|
||||
assert SPEC.frame("hello", spec.SERVER_TO_CLIENT) is None
|
||||
assert "type" in SPEC.frame("ping", spec.SERVER_TO_CLIENT).allowed()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"host,expected",
|
||||
[
|
||||
("reddit.com", True),
|
||||
("www.reddit.com", True),
|
||||
("old.reddit.com", True),
|
||||
("REDDIT.COM.", True),
|
||||
("evilreddit.com", False),
|
||||
("reddit.com.evil.net", False),
|
||||
("localhost", False),
|
||||
("127.0.0.1", False),
|
||||
],
|
||||
)
|
||||
def test_destination_matching_is_suffix_safe(host, expected):
|
||||
assert (SPEC.destination(host) is not None) is expected
|
||||
|
||||
|
||||
def test_limits_are_positive_and_ordered():
|
||||
limits = SPEC.limits
|
||||
for value in limits.describe().values():
|
||||
assert value >= 0
|
||||
assert limits.default_wait_seconds <= limits.max_wait_seconds
|
||||
assert limits.reconnect_min_seconds <= limits.reconnect_max_seconds
|
||||
assert limits.cache_entry_max_bytes <= limits.cache_max_bytes
|
||||
assert limits.default_donor_mb_per_day <= limits.max_donor_mb_per_day
|
||||
assert limits.job_timeout_seconds <= limits.max_wait_seconds
|
||||
assert limits.default_daily_requests_guest == 0
|
||||
|
||||
|
||||
def test_protocol_versions_consistent():
|
||||
assert SPEC.min_supported <= SPEC.version
|
||||
|
||||
|
||||
def test_describe_is_json_serialisable():
|
||||
import json
|
||||
|
||||
json.dumps(SPEC.describe())
|
||||
@@ -439,6 +439,7 @@ class FakeEmbedClient_openai_gateway:
|
||||
|
||||
def test_embeddings_remaps_alias_to_configured_model(local_db, monkeypatch):
|
||||
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeEmbedClient_openai_gateway)
|
||||
monkeypatch.setenv("OPENROUTER_API_KEY", "test-embed-key")
|
||||
svc = GatewayService()
|
||||
cfg = svc.effective_config()
|
||||
cfg["gateway_force_model"] = False
|
||||
@@ -459,6 +460,7 @@ def test_embeddings_remaps_alias_to_configured_model(local_db, monkeypatch):
|
||||
|
||||
def test_embeddings_success_records_ledger(local_db, monkeypatch):
|
||||
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeEmbedClient_openai_gateway)
|
||||
monkeypatch.setenv("OPENROUTER_API_KEY", "test-embed-key")
|
||||
svc = GatewayService()
|
||||
cfg = svc.effective_config()
|
||||
cfg["gateway_force_model"] = True
|
||||
@@ -740,6 +742,7 @@ def test_app_reference_defaults_when_not_passed(local_db, monkeypatch):
|
||||
|
||||
def test_app_reference_stored_in_ledger_for_embeddings(local_db, monkeypatch):
|
||||
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeEmbedClient_openai_gateway)
|
||||
monkeypatch.setenv("OPENROUTER_API_KEY", "test-embed-key")
|
||||
svc = GatewayService()
|
||||
cfg = svc.effective_config()
|
||||
cfg["gateway_embed_enabled"] = True
|
||||
@@ -1228,6 +1231,7 @@ def test_chat_without_a_fallback_never_makes_a_second_call(local_db, monkeypatch
|
||||
|
||||
def test_embeddings_unknown_model_falls_back_to_default(local_db, monkeypatch):
|
||||
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeEmbedClient_openai_gateway)
|
||||
monkeypatch.setenv("OPENROUTER_API_KEY", "test-embed-key")
|
||||
svc = GatewayService()
|
||||
cfg = svc.effective_config()
|
||||
cfg["gateway_force_model"] = False
|
||||
@@ -1834,6 +1838,7 @@ def test_upstream_x_gateway_model_header_is_forwarded_for_embeddings(
|
||||
monkeypatch.setattr(
|
||||
gwmod.httpx, "AsyncClient", FakeEmbedClientUpstreamModelHeader_openai_gateway
|
||||
)
|
||||
monkeypatch.setenv("OPENROUTER_API_KEY", "test-embed-key")
|
||||
svc = GatewayService()
|
||||
cfg = svc.effective_config()
|
||||
cfg["gateway_embed_enabled"] = True
|
||||
@@ -2211,3 +2216,325 @@ def test_internal_bypass_allowed_requires_the_exact_internal_key(local_db):
|
||||
assert svc.internal_bypass_allowed(_make_request_openai_gateway(), cfg) is False
|
||||
finally:
|
||||
set_setting("gateway_internal_key", "")
|
||||
|
||||
|
||||
class FakeResponsesResp_openai_gateway(FakeResp_openai_gateway):
|
||||
def __init__(self, status=200, payload=None, events=None, **kw):
|
||||
super().__init__(status=status, payload=payload, **kw)
|
||||
self._events = events or []
|
||||
|
||||
async def aiter_lines(self):
|
||||
for event in self._events:
|
||||
yield f"data: {json.dumps(event)}"
|
||||
yield "data: [DONE]"
|
||||
|
||||
|
||||
def _resp_obj_openai_gateway(model):
|
||||
return {
|
||||
"id": "resp_1",
|
||||
"object": "response",
|
||||
"created_at": 7,
|
||||
"status": "completed",
|
||||
"model": model,
|
||||
"output": [
|
||||
{
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"content": [{"type": "output_text", "text": "hi there"}],
|
||||
}
|
||||
],
|
||||
"usage": {"input_tokens": 5, "output_tokens": 7, "total_tokens": 12},
|
||||
}
|
||||
|
||||
|
||||
def _resp_events_openai_gateway(model):
|
||||
return [
|
||||
{"type": "response.created", "response": {"id": "resp_1"}},
|
||||
{"type": "response.output_text.delta", "delta": "hi "},
|
||||
{"type": "response.output_text.delta", "delta": "there"},
|
||||
{"type": "response.completed", "response": _resp_obj_openai_gateway(model)},
|
||||
]
|
||||
|
||||
|
||||
class FakeResponsesClient_openai_gateway:
|
||||
behavior = "ok"
|
||||
|
||||
def __init__(self, *a, **k):
|
||||
self.calls = []
|
||||
|
||||
def build_request(self, method, url, headers=None, json=None, content=None):
|
||||
return FakeRequest(method, url, json)
|
||||
|
||||
async def send(self, request, stream=False):
|
||||
self.calls.append((request.url, request.json_body))
|
||||
body = request.json_body or {}
|
||||
if self.behavior == "error-envelope":
|
||||
return FakeResp_openai_gateway(
|
||||
status=429,
|
||||
payload={
|
||||
"error": {
|
||||
"message": "slow down",
|
||||
"type": "rate_limit",
|
||||
"code": "rate_limit_exceeded",
|
||||
}
|
||||
},
|
||||
)
|
||||
if self.behavior == "failed-object":
|
||||
return FakeResponsesResp_openai_gateway(
|
||||
payload={
|
||||
"object": "response",
|
||||
"status": "failed",
|
||||
"error": {"message": "boom"},
|
||||
}
|
||||
)
|
||||
model = body.get("model", "")
|
||||
return FakeResponsesResp_openai_gateway(
|
||||
payload=_resp_obj_openai_gateway(model),
|
||||
events=_resp_events_openai_gateway(model),
|
||||
)
|
||||
|
||||
async def post(self, url, headers=None, json=None, timeout=None):
|
||||
return await self.send(FakeRequest("POST", url, json), stream=False)
|
||||
|
||||
async def aclose(self):
|
||||
pass
|
||||
|
||||
|
||||
def _setup_responses_route_openai_gateway():
|
||||
from devplacepy.services.openai_gateway import routing
|
||||
|
||||
routing.provider_store.set(
|
||||
routing.ProviderIn(
|
||||
name="gwmeta",
|
||||
base_url="https://api.meta.ai/v1/responses",
|
||||
api_key="sk-test",
|
||||
protocol="responses",
|
||||
)
|
||||
)
|
||||
routing.model_store.set(
|
||||
routing.ModelRouteIn(
|
||||
source_model="meta-chat",
|
||||
provider="gwmeta",
|
||||
target_model="muse-spark-1.3-contributor",
|
||||
kind="chat",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _teardown_responses_route_openai_gateway():
|
||||
from devplacepy.services.openai_gateway import routing
|
||||
|
||||
routing.model_store.remove("meta-chat")
|
||||
routing.provider_store.remove("gwmeta")
|
||||
|
||||
|
||||
def _chat_body_openai_gateway(**over):
|
||||
body = {
|
||||
"model": "meta-chat",
|
||||
"messages": [
|
||||
{"role": "system", "content": "Be terse."},
|
||||
{"role": "user", "content": "hi"},
|
||||
],
|
||||
}
|
||||
body.update(over)
|
||||
return body
|
||||
|
||||
|
||||
def test_responses_protocol_transpiles_request_and_response(local_db, monkeypatch):
|
||||
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeResponsesClient_openai_gateway)
|
||||
FakeResponsesClient_openai_gateway.behavior = "ok"
|
||||
_setup_responses_route_openai_gateway()
|
||||
try:
|
||||
svc = GatewayService()
|
||||
cfg = svc.effective_config()
|
||||
cfg["gateway_vision_enabled"] = False
|
||||
rt = svc.runtime()
|
||||
resp = run_async(
|
||||
rt.handle_chat(
|
||||
_chat_body_openai_gateway(), cfg, ("guest", "test"), "test", "default"
|
||||
)
|
||||
)
|
||||
url, sent = rt._client.calls[-1]
|
||||
assert str(url) == "https://api.meta.ai/v1/responses"
|
||||
assert sent["model"] == "muse-spark-1.3-contributor"
|
||||
assert "messages" not in sent
|
||||
assert "Be terse." in sent["instructions"]
|
||||
assert sent["input"] == [
|
||||
{"type": "message", "role": "user", "content": "hi"}
|
||||
]
|
||||
assert sent["reasoning"] == {"effort": "minimal"}
|
||||
assert resp.status_code == 200
|
||||
payload = json.loads(bytes(resp.body).decode())
|
||||
assert payload["object"] == "chat.completion"
|
||||
assert payload["choices"][0]["message"]["content"] == "hi there"
|
||||
assert payload["choices"][0]["finish_reason"] == "stop"
|
||||
assert payload["usage"]["prompt_tokens"] == 5
|
||||
assert resp.headers["X-Gateway-Model"] == "muse-spark-1.3-contributor"
|
||||
finally:
|
||||
_teardown_responses_route_openai_gateway()
|
||||
|
||||
|
||||
def test_responses_protocol_stream_relays_chat_chunks(local_db, monkeypatch):
|
||||
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeResponsesClient_openai_gateway)
|
||||
FakeResponsesClient_openai_gateway.behavior = "ok"
|
||||
_setup_responses_route_openai_gateway()
|
||||
try:
|
||||
svc = GatewayService()
|
||||
cfg = svc.effective_config()
|
||||
cfg["gateway_vision_enabled"] = False
|
||||
rt = svc.runtime()
|
||||
resp = run_async(
|
||||
rt.handle_chat(
|
||||
_chat_body_openai_gateway(stream=True),
|
||||
cfg,
|
||||
("guest", "test"),
|
||||
"test",
|
||||
"default",
|
||||
)
|
||||
)
|
||||
|
||||
async def drain():
|
||||
out = []
|
||||
async for chunk in resp.body_iterator:
|
||||
out.append(chunk if isinstance(chunk, str) else chunk.decode())
|
||||
return "".join(out)
|
||||
|
||||
body = run_async(drain())
|
||||
assert "chat.completion.chunk" in body
|
||||
assert "hi " in body and "there" in body
|
||||
assert '"usage"' not in body
|
||||
assert "[DONE]" in body
|
||||
finally:
|
||||
_teardown_responses_route_openai_gateway()
|
||||
|
||||
|
||||
def test_responses_protocol_stream_relays_usage_when_asked(local_db, monkeypatch):
|
||||
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeResponsesClient_openai_gateway)
|
||||
FakeResponsesClient_openai_gateway.behavior = "ok"
|
||||
_setup_responses_route_openai_gateway()
|
||||
try:
|
||||
svc = GatewayService()
|
||||
cfg = svc.effective_config()
|
||||
cfg["gateway_vision_enabled"] = False
|
||||
rt = svc.runtime()
|
||||
resp = run_async(
|
||||
rt.handle_chat(
|
||||
_chat_body_openai_gateway(
|
||||
stream=True, stream_options={"include_usage": True}
|
||||
),
|
||||
cfg,
|
||||
("guest", "test"),
|
||||
"test",
|
||||
"default",
|
||||
)
|
||||
)
|
||||
_, sent = rt._client.calls[-1]
|
||||
assert "stream_options" not in sent
|
||||
|
||||
async def drain():
|
||||
out = []
|
||||
async for chunk in resp.body_iterator:
|
||||
out.append(chunk if isinstance(chunk, str) else chunk.decode())
|
||||
return "".join(out)
|
||||
|
||||
body = run_async(drain())
|
||||
assert '"usage"' in body
|
||||
assert '"prompt_tokens": 5' in body
|
||||
finally:
|
||||
_teardown_responses_route_openai_gateway()
|
||||
|
||||
|
||||
def test_responses_protocol_relays_error_envelope(local_db, monkeypatch):
|
||||
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeResponsesClient_openai_gateway)
|
||||
FakeResponsesClient_openai_gateway.behavior = "error-envelope"
|
||||
_setup_responses_route_openai_gateway()
|
||||
try:
|
||||
svc = GatewayService()
|
||||
cfg = svc.effective_config()
|
||||
cfg["gateway_vision_enabled"] = False
|
||||
rt = svc.runtime()
|
||||
resp = run_async(
|
||||
rt.handle_chat(
|
||||
_chat_body_openai_gateway(), cfg, ("guest", "test"), "test", "default"
|
||||
)
|
||||
)
|
||||
assert resp.status_code == 429
|
||||
payload = json.loads(bytes(resp.body).decode())
|
||||
assert payload["error"]["message"] == "slow down"
|
||||
assert payload["error"]["code"] == "rate_limit_exceeded"
|
||||
finally:
|
||||
FakeResponsesClient_openai_gateway.behavior = "ok"
|
||||
_teardown_responses_route_openai_gateway()
|
||||
|
||||
|
||||
def test_responses_protocol_failed_object_is_502(local_db, monkeypatch):
|
||||
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeResponsesClient_openai_gateway)
|
||||
FakeResponsesClient_openai_gateway.behavior = "failed-object"
|
||||
_setup_responses_route_openai_gateway()
|
||||
try:
|
||||
svc = GatewayService()
|
||||
cfg = svc.effective_config()
|
||||
cfg["gateway_vision_enabled"] = False
|
||||
rt = svc.runtime()
|
||||
resp = run_async(
|
||||
rt.handle_chat(
|
||||
_chat_body_openai_gateway(), cfg, ("guest", "test"), "test", "default"
|
||||
)
|
||||
)
|
||||
assert resp.status_code == 502
|
||||
finally:
|
||||
FakeResponsesClient_openai_gateway.behavior = "ok"
|
||||
_teardown_responses_route_openai_gateway()
|
||||
|
||||
|
||||
def test_responses_protocol_client_reasoning_effort_passes_through(
|
||||
local_db, monkeypatch
|
||||
):
|
||||
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeResponsesClient_openai_gateway)
|
||||
FakeResponsesClient_openai_gateway.behavior = "ok"
|
||||
_setup_responses_route_openai_gateway()
|
||||
try:
|
||||
svc = GatewayService()
|
||||
cfg = svc.effective_config()
|
||||
cfg["gateway_vision_enabled"] = False
|
||||
rt = svc.runtime()
|
||||
run_async(
|
||||
rt.handle_chat(
|
||||
_chat_body_openai_gateway(reasoning_effort="high"),
|
||||
cfg,
|
||||
("guest", "test"),
|
||||
"test",
|
||||
"default",
|
||||
)
|
||||
)
|
||||
_, sent = rt._client.calls[-1]
|
||||
assert sent["reasoning"] == {"effort": "high"}
|
||||
finally:
|
||||
_teardown_responses_route_openai_gateway()
|
||||
|
||||
|
||||
def test_responses_protocol_default_provider(local_db, monkeypatch):
|
||||
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeResponsesClient_openai_gateway)
|
||||
FakeResponsesClient_openai_gateway.behavior = "ok"
|
||||
svc = GatewayService()
|
||||
cfg = svc.effective_config()
|
||||
cfg["gateway_vision_enabled"] = False
|
||||
cfg["gateway_upstream_url"] = "https://api.meta.ai/v1/responses"
|
||||
cfg["gateway_upstream_protocol"] = "responses"
|
||||
cfg["gateway_model"] = "muse-spark-1.3-contributor"
|
||||
rt = svc.runtime()
|
||||
resp = run_async(
|
||||
rt.handle_chat(
|
||||
{"messages": [{"role": "user", "content": "hi"}]},
|
||||
cfg,
|
||||
("guest", "test"),
|
||||
"test",
|
||||
"default",
|
||||
)
|
||||
)
|
||||
url, sent = rt._client.calls[-1]
|
||||
assert str(url) == "https://api.meta.ai/v1/responses"
|
||||
assert sent["input"] == [{"type": "message", "role": "user", "content": "hi"}]
|
||||
assert resp.status_code == 200
|
||||
payload = json.loads(bytes(resp.body).decode())
|
||||
assert payload["object"] == "chat.completion"
|
||||
|
||||
@@ -0,0 +1,520 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from devplacepy.services.openai_gateway.responses import (
|
||||
ChatToResponsesStream,
|
||||
ResponsesToChatStream,
|
||||
chat_completion_to_responses_object,
|
||||
chat_to_responses,
|
||||
responses_error_object,
|
||||
responses_failure_message,
|
||||
responses_object_to_chat_completion,
|
||||
responses_to_chat,
|
||||
)
|
||||
|
||||
|
||||
def _chat_body(**over):
|
||||
body = {
|
||||
"model": "m",
|
||||
"messages": [
|
||||
{"role": "system", "content": "Be terse."},
|
||||
{"role": "user", "content": [{"type": "text", "text": "hi"}]},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_1",
|
||||
"type": "function",
|
||||
"function": {"name": "get_weather", "arguments": "{}"},
|
||||
}
|
||||
],
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "call_1", "content": "sunny"},
|
||||
],
|
||||
"tools": [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"parameters": {"type": "object"},
|
||||
},
|
||||
}
|
||||
],
|
||||
"tool_choice": "auto",
|
||||
"response_format": {
|
||||
"type": "json_schema",
|
||||
"json_schema": {"name": "r", "schema": {"type": "object"}},
|
||||
},
|
||||
"max_tokens": 10,
|
||||
}
|
||||
body.update(over)
|
||||
return body
|
||||
|
||||
|
||||
def test_chat_to_responses_core_mapping():
|
||||
resp, warn = chat_to_responses(_chat_body())
|
||||
assert warn == []
|
||||
assert resp["model"] == "m"
|
||||
assert resp["instructions"] == "Be terse."
|
||||
assert [i.get("type") for i in resp["input"]] == [
|
||||
"message",
|
||||
"function_call",
|
||||
"function_call_output",
|
||||
]
|
||||
assert resp["input"][0]["content"] == [
|
||||
{"type": "input_text", "text": "hi"}
|
||||
]
|
||||
assert resp["input"][1]["call_id"] == "call_1"
|
||||
assert resp["input"][2]["output"] == "sunny"
|
||||
assert resp["tools"][0]["name"] == "get_weather"
|
||||
assert resp["tools"][0]["strict"] is False
|
||||
assert resp["text"]["format"]["type"] == "json_schema"
|
||||
assert resp["max_output_tokens"] == 10
|
||||
|
||||
|
||||
def test_chat_to_responses_pins_strict_explicitly():
|
||||
resp, _ = chat_to_responses(
|
||||
{
|
||||
"model": "m",
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"tools": [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "f",
|
||||
"strict": True,
|
||||
"parameters": {"type": "object"},
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
assert resp["tools"][0]["strict"] is True
|
||||
|
||||
|
||||
def test_chat_to_responses_warns_on_drops_and_unknowns():
|
||||
resp, warn = chat_to_responses(
|
||||
_chat_body(n=2, stop=["x"], seed=1, wat_field=True)
|
||||
)
|
||||
assert "n" not in resp
|
||||
assert "stop" not in resp
|
||||
assert "seed" not in resp
|
||||
assert any("field 'n'" in w for w in warn)
|
||||
assert any("field 'stop'" in w for w in warn)
|
||||
assert any("field 'seed'" in w for w in warn)
|
||||
assert any("unknown chat field 'wat_field'" in w for w in warn)
|
||||
|
||||
|
||||
def test_chat_to_responses_drops_custom_tools_with_warning():
|
||||
resp, warn = chat_to_responses(
|
||||
{
|
||||
"model": "m",
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"tools": [{"type": "custom", "custom": {"name": "x"}}],
|
||||
}
|
||||
)
|
||||
assert "tools" not in resp
|
||||
assert any("custom" in w for w in warn)
|
||||
|
||||
|
||||
def test_chat_to_responses_named_tool_choice():
|
||||
resp, warn = chat_to_responses(
|
||||
{
|
||||
"model": "m",
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"tool_choice": {
|
||||
"type": "function",
|
||||
"function": {"name": "get_weather"},
|
||||
},
|
||||
}
|
||||
)
|
||||
assert warn == []
|
||||
assert resp["tool_choice"] == {"type": "function", "name": "get_weather"}
|
||||
|
||||
|
||||
def test_chat_to_responses_response_format_variants():
|
||||
resp, _ = chat_to_responses(
|
||||
{
|
||||
"model": "m",
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"response_format": {"type": "json_object"},
|
||||
}
|
||||
)
|
||||
assert resp["text"] == {"format": {"type": "json_object"}}
|
||||
resp, _ = chat_to_responses(
|
||||
{
|
||||
"model": "m",
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"response_format": {"type": "text"},
|
||||
}
|
||||
)
|
||||
assert resp["text"] == {"format": {"type": "text"}}
|
||||
|
||||
|
||||
def test_chat_to_responses_reasoning_effort_and_image():
|
||||
resp, warn = chat_to_responses(
|
||||
{
|
||||
"model": "m",
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": "https://x/y.png",
|
||||
"detail": "low",
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
],
|
||||
"reasoning_effort": "high",
|
||||
}
|
||||
)
|
||||
assert warn == []
|
||||
assert resp["reasoning"] == {"effort": "high"}
|
||||
assert resp["input"][0]["content"] == [
|
||||
{
|
||||
"type": "input_image",
|
||||
"image_url": "https://x/y.png",
|
||||
"detail": "low",
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def test_chat_to_responses_thinking_flags_map_to_reasoning():
|
||||
base = {"model": "m", "messages": [{"role": "user", "content": "hi"}]}
|
||||
resp, warn = chat_to_responses({**base, "think": False})
|
||||
assert warn == []
|
||||
assert resp["reasoning"] == {"effort": "minimal"}
|
||||
resp, warn = chat_to_responses(
|
||||
{**base, "thinking": {"type": "disabled"}}
|
||||
)
|
||||
assert warn == []
|
||||
assert resp["reasoning"] == {"effort": "minimal"}
|
||||
resp, warn = chat_to_responses({**base, "think": True})
|
||||
assert warn == []
|
||||
assert "reasoning" not in resp
|
||||
resp, warn = chat_to_responses(
|
||||
{**base, "reasoning": {"effort": "low", "max_tokens": 5}}
|
||||
)
|
||||
assert resp["reasoning"] == {"effort": "low"}
|
||||
assert any("reasoning keys" in w for w in warn)
|
||||
resp, warn = chat_to_responses(
|
||||
{**base, "reasoning_effort": "high", "think": False}
|
||||
)
|
||||
assert warn == []
|
||||
assert resp["reasoning"] == {"effort": "high"}
|
||||
|
||||
|
||||
def test_chat_to_responses_audio_part_warns_and_drops():
|
||||
resp, warn = chat_to_responses(
|
||||
{
|
||||
"model": "m",
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "input_audio", "input_audio": {}}],
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
assert resp["input"][0]["content"] == []
|
||||
assert any("input_audio" in w for w in warn)
|
||||
|
||||
|
||||
def test_responses_to_chat_round_trip():
|
||||
resp, _ = chat_to_responses(_chat_body())
|
||||
back, warn = responses_to_chat(resp)
|
||||
assert warn == []
|
||||
assert back["messages"][0] == {"role": "system", "content": "Be terse."}
|
||||
assert back["messages"][1]["role"] == "user"
|
||||
assert back["messages"][2]["tool_calls"][0]["id"] == "call_1"
|
||||
assert back["messages"][3] == {
|
||||
"role": "tool",
|
||||
"tool_call_id": "call_1",
|
||||
"content": "sunny",
|
||||
}
|
||||
assert back["tools"][0]["function"]["name"] == "get_weather"
|
||||
assert back["response_format"]["type"] == "json_schema"
|
||||
assert back["max_tokens"] == 10
|
||||
|
||||
|
||||
def test_responses_to_chat_drops_namespace_and_state():
|
||||
back, warn = responses_to_chat(
|
||||
{
|
||||
"model": "m",
|
||||
"instructions": "sys",
|
||||
"input": [{"type": "message", "role": "user", "content": "hi"}],
|
||||
"tools": [{"type": "namespace", "name": "muse", "tools": []}],
|
||||
"store": False,
|
||||
"previous_response_id": "resp_1",
|
||||
"include": ["reasoning.encrypted_content"],
|
||||
}
|
||||
)
|
||||
assert back["messages"][0] == {"role": "system", "content": "sys"}
|
||||
assert "tools" not in back
|
||||
assert back["store"] is False
|
||||
assert any("namespace" in w for w in warn)
|
||||
assert any("previous_response_id" in w for w in warn)
|
||||
assert any("'include'" in w for w in warn)
|
||||
|
||||
|
||||
def test_responses_to_chat_drops_reasoning_items():
|
||||
back, warn = responses_to_chat(
|
||||
{
|
||||
"model": "m",
|
||||
"input": [
|
||||
{"type": "reasoning", "id": "rs_1"},
|
||||
{"type": "message", "role": "user", "content": "hi"},
|
||||
],
|
||||
}
|
||||
)
|
||||
assert [m["role"] for m in back["messages"]] == ["user"]
|
||||
assert any("reasoning" in w for w in warn)
|
||||
|
||||
|
||||
def test_responses_to_chat_string_input_and_instructions_list():
|
||||
back, warn = responses_to_chat({"model": "m", "input": "hello"})
|
||||
assert warn == []
|
||||
assert back["messages"] == [{"role": "user", "content": "hello"}]
|
||||
back, warn = responses_to_chat(
|
||||
{
|
||||
"model": "m",
|
||||
"input": [{"type": "message", "role": "user", "content": "hi"}],
|
||||
"instructions": [
|
||||
{"type": "message", "role": "system", "content": "a"},
|
||||
{"type": "message", "role": "developer", "content": "b"},
|
||||
],
|
||||
}
|
||||
)
|
||||
assert warn == []
|
||||
assert [m["role"] for m in back["messages"]] == [
|
||||
"system",
|
||||
"developer",
|
||||
"user",
|
||||
]
|
||||
|
||||
|
||||
def test_response_object_to_chat_completion():
|
||||
comp, warn = responses_object_to_chat_completion(
|
||||
{
|
||||
"id": "resp_1",
|
||||
"created_at": 7,
|
||||
"model": "m",
|
||||
"status": "completed",
|
||||
"output": [
|
||||
{
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"content": [{"type": "output_text", "text": "ok"}],
|
||||
},
|
||||
{
|
||||
"type": "function_call",
|
||||
"call_id": "call_9",
|
||||
"name": "f",
|
||||
"arguments": "{}",
|
||||
},
|
||||
],
|
||||
"usage": {"input_tokens": 3, "output_tokens": 1, "total_tokens": 4},
|
||||
}
|
||||
)
|
||||
assert warn == []
|
||||
assert comp["object"] == "chat.completion"
|
||||
assert comp["choices"][0]["message"]["content"] == "ok"
|
||||
assert comp["choices"][0]["message"]["tool_calls"][0]["id"] == "call_9"
|
||||
assert comp["choices"][0]["finish_reason"] == "tool_calls"
|
||||
assert comp["usage"] == {
|
||||
"prompt_tokens": 3,
|
||||
"completion_tokens": 1,
|
||||
"total_tokens": 4,
|
||||
}
|
||||
|
||||
|
||||
def test_response_object_finish_reasons():
|
||||
comp, _ = responses_object_to_chat_completion(
|
||||
{
|
||||
"id": "r",
|
||||
"output": [],
|
||||
"incomplete_details": {"reason": "max_output_tokens"},
|
||||
}
|
||||
)
|
||||
assert comp["choices"][0]["finish_reason"] == "length"
|
||||
comp, _ = responses_object_to_chat_completion(
|
||||
{
|
||||
"id": "r",
|
||||
"output": [],
|
||||
"incomplete_details": {"reason": "content_filter"},
|
||||
}
|
||||
)
|
||||
assert comp["choices"][0]["finish_reason"] == "content_filter"
|
||||
comp, _ = responses_object_to_chat_completion(
|
||||
{
|
||||
"id": "r",
|
||||
"output": [
|
||||
{
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"content": [{"type": "refusal", "refusal": "no"}],
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
assert comp["choices"][0]["message"]["refusal"] == "no"
|
||||
assert comp["choices"][0]["finish_reason"] == "stop"
|
||||
|
||||
|
||||
def test_chat_completion_to_response_object_keeps_choice_zero():
|
||||
obj, warn = chat_completion_to_responses_object(
|
||||
{
|
||||
"id": "c",
|
||||
"created": 1,
|
||||
"model": "m",
|
||||
"choices": [
|
||||
{"message": {"role": "assistant", "content": "a"}},
|
||||
{"message": {"role": "assistant", "content": "b"}},
|
||||
],
|
||||
"usage": {"prompt_tokens": 1, "completion_tokens": 1,
|
||||
"total_tokens": 2},
|
||||
}
|
||||
)
|
||||
assert any("choice 0 of 2" in w for w in warn)
|
||||
assert obj["output"][0]["content"][0]["text"] == "a"
|
||||
assert obj["usage"]["input_tokens"] == 1
|
||||
|
||||
|
||||
def test_responses_failure_and_error_shapes():
|
||||
assert responses_failure_message({"a": 1}) is None
|
||||
assert responses_failure_message("x") is None
|
||||
assert (
|
||||
responses_failure_message({"error": {"message": "bad", "type": "t"}})
|
||||
== "bad"
|
||||
)
|
||||
assert responses_failure_message(
|
||||
{"object": "response", "status": "failed",
|
||||
"incomplete_details": {"reason": "r"}}
|
||||
) == "response failed: r"
|
||||
assert responses_error_object({"error": {"message": "m", "code": "c"}}) == {
|
||||
"error": {"message": "m", "code": "c"}
|
||||
}
|
||||
assert responses_error_object({"a": 1}) is None
|
||||
|
||||
|
||||
def test_stream_responses_to_chat_text_and_done():
|
||||
stream = ResponsesToChatStream(model="m")
|
||||
chunks = stream.feed({"type": "response.output_text.delta", "delta": "Hi"})
|
||||
assert chunks[0]["choices"][0]["delta"] == {
|
||||
"role": "assistant",
|
||||
"content": "Hi",
|
||||
}
|
||||
chunks = stream.feed({"type": "response.output_text.delta", "delta": "!"})
|
||||
assert chunks[0]["choices"][0]["delta"] == {"content": "!"}
|
||||
done = stream.feed(
|
||||
{
|
||||
"type": "response.completed",
|
||||
"response": {
|
||||
"id": "r",
|
||||
"output": [],
|
||||
"usage": {"input_tokens": 1, "output_tokens": 1,
|
||||
"total_tokens": 2},
|
||||
},
|
||||
}
|
||||
)
|
||||
assert done[0]["choices"][0]["finish_reason"] == "stop"
|
||||
assert done[0]["usage"]["prompt_tokens"] == 1
|
||||
|
||||
|
||||
def test_stream_responses_to_chat_tool_calls():
|
||||
stream = ResponsesToChatStream(model="m")
|
||||
chunks = stream.feed(
|
||||
{
|
||||
"type": "response.output_item.added",
|
||||
"item": {
|
||||
"type": "function_call",
|
||||
"call_id": "call_1",
|
||||
"name": "f",
|
||||
},
|
||||
}
|
||||
)
|
||||
call = chunks[0]["choices"][0]["delta"]["tool_calls"][0]
|
||||
assert call["id"] == "call_1"
|
||||
assert call["function"]["name"] == "f"
|
||||
chunks = stream.feed(
|
||||
{
|
||||
"type": "response.function_call_arguments.delta",
|
||||
"item_id": "call_1",
|
||||
"delta": '{"a":',
|
||||
}
|
||||
)
|
||||
call = chunks[0]["choices"][0]["delta"]["tool_calls"][0]
|
||||
assert call["index"] == 0
|
||||
assert call["function"] == {"arguments": '{"a":'}
|
||||
|
||||
|
||||
def test_stream_responses_to_chat_error_and_unknown():
|
||||
stream = ResponsesToChatStream(model="m")
|
||||
assert stream.feed({"type": "response.in_progress"}) == []
|
||||
assert stream.feed("nope") == []
|
||||
out = stream.feed(
|
||||
{
|
||||
"type": "response.failed",
|
||||
"response": {
|
||||
"object": "response",
|
||||
"status": "failed",
|
||||
"error": {"message": "boom", "type": "server_error"},
|
||||
},
|
||||
}
|
||||
)
|
||||
assert out == [{"error": {"message": "boom", "type": "server_error"}}]
|
||||
assert stream.failed is not None
|
||||
|
||||
|
||||
def test_stream_chat_to_responses_round_trip():
|
||||
stream = ChatToResponsesStream()
|
||||
events = stream.feed(
|
||||
{
|
||||
"model": "m",
|
||||
"choices": [
|
||||
{
|
||||
"delta": {
|
||||
"content": "Hi",
|
||||
"tool_calls": [
|
||||
{
|
||||
"index": 0,
|
||||
"id": "call_1",
|
||||
"function": {"name": "f", "arguments": ""},
|
||||
}
|
||||
],
|
||||
},
|
||||
"finish_reason": None,
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
assert [e["type"] for e in events] == [
|
||||
"response.output_item.added",
|
||||
"response.output_text.delta",
|
||||
]
|
||||
events = stream.feed(
|
||||
{
|
||||
"model": "m",
|
||||
"choices": [
|
||||
{
|
||||
"delta": {
|
||||
"tool_calls": [
|
||||
{"index": 0, "function": {"arguments": "{}"}}
|
||||
]
|
||||
},
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
assert [e["type"] for e in events] == [
|
||||
"response.function_call_arguments.delta",
|
||||
"response.completed",
|
||||
]
|
||||
assert events[0]["item_id"] == "call_1"
|
||||
assert events[1]["response"]["output"][0]["content"][0]["text"] == "Hi"
|
||||
assert stream.warnings == []
|
||||
@@ -595,3 +595,106 @@ def test_fetch_provider_models_none_when_provider_has_no_base_url(local_db):
|
||||
assert run_async(r.fetch_provider_models("probeprovnourl")) is None
|
||||
finally:
|
||||
r.provider_store.remove("probeprovnourl")
|
||||
|
||||
|
||||
def test_provider_protocol_defaults_and_validates(local_db):
|
||||
assert r.ProviderIn(name="p1").protocol == "chat_completions"
|
||||
assert r.ProviderIn(name="p1", protocol="responses").protocol == "responses"
|
||||
assert r.ProviderIn(name="p1", protocol=" Responses ").protocol == "responses"
|
||||
with pytest.raises(ValidationError):
|
||||
r.ProviderIn(name="p1", protocol="soap")
|
||||
|
||||
|
||||
def test_provider_protocol_round_trips_through_store(local_db):
|
||||
r.provider_store.set(
|
||||
r.ProviderIn(
|
||||
name="protometa",
|
||||
base_url="https://api.meta.ai/v1/responses",
|
||||
api_key="sk-test",
|
||||
protocol="responses",
|
||||
)
|
||||
)
|
||||
try:
|
||||
saved = r.provider_store.get("protometa")
|
||||
assert saved is not None
|
||||
assert saved["protocol"] == "responses"
|
||||
assert saved["base_url"] == "https://api.meta.ai/v1/responses"
|
||||
finally:
|
||||
r.provider_store.remove("protometa")
|
||||
|
||||
|
||||
def test_chat_overlay_carries_provider_protocol(local_db):
|
||||
r.provider_store.set(
|
||||
r.ProviderIn(
|
||||
name="protoovl",
|
||||
base_url="https://api.meta.ai/v1/responses",
|
||||
api_key="sk-test",
|
||||
protocol="responses",
|
||||
)
|
||||
)
|
||||
r.model_store.set(
|
||||
r.ModelRouteIn(
|
||||
source_model="meta-route",
|
||||
provider="protoovl",
|
||||
target_model="muse-spark-1.3-contributor",
|
||||
kind="chat",
|
||||
)
|
||||
)
|
||||
try:
|
||||
overlay = r.chat_overlay("meta-route", {"gateway_model": "x"})
|
||||
assert overlay["gateway_upstream_protocol"] == "responses"
|
||||
assert (
|
||||
overlay["gateway_upstream_url"]
|
||||
== "https://api.meta.ai/v1/responses"
|
||||
)
|
||||
finally:
|
||||
_cleanup(["protoovl"], ["meta-route"])
|
||||
|
||||
|
||||
def test_responses_models_urls_probe_both_shapes():
|
||||
assert r._responses_models_urls("https://api.meta.ai/v1/responses") == [
|
||||
"https://api.meta.ai/v1/models",
|
||||
"https://api.meta.ai/v1/muse-code/models",
|
||||
]
|
||||
assert r._responses_models_urls("") == []
|
||||
|
||||
|
||||
def test_fetch_provider_models_tries_meta_listing_second(local_db, monkeypatch):
|
||||
r.provider_store.set(
|
||||
r.ProviderIn(
|
||||
name="probeprovmeta",
|
||||
base_url="https://api.meta.ai/v1/responses",
|
||||
protocol="responses",
|
||||
)
|
||||
)
|
||||
|
||||
class _SeqClient:
|
||||
def __init__(self):
|
||||
self.requested_urls = []
|
||||
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *exc_info):
|
||||
return False
|
||||
|
||||
async def get(self, url):
|
||||
self.requested_urls.append(url)
|
||||
if url == "https://api.meta.ai/v1/models":
|
||||
return _FakeModelsResponse(404, {})
|
||||
return _FakeModelsResponse(
|
||||
200, {"data": [{"id": "muse-spark-1.3-contributor"}]}
|
||||
)
|
||||
|
||||
fake_client = _SeqClient()
|
||||
_patch_stealth_client(monkeypatch, fake_client)
|
||||
try:
|
||||
assert run_async(r.fetch_provider_models("probeprovmeta")) == [
|
||||
"muse-spark-1.3-contributor"
|
||||
]
|
||||
assert fake_client.requested_urls == [
|
||||
"https://api.meta.ai/v1/models",
|
||||
"https://api.meta.ai/v1/muse-code/models",
|
||||
]
|
||||
finally:
|
||||
r.provider_store.remove("probeprovmeta")
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
@@ -0,0 +1,94 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
import pytest
|
||||
|
||||
from devplacepy.services.skills import limits
|
||||
from devplacepy.services.skills.fetcher import (
|
||||
SkillFetchError,
|
||||
classify_candidate,
|
||||
decode_text,
|
||||
directory_url,
|
||||
extract_candidates,
|
||||
normalize_input_url,
|
||||
origin_of,
|
||||
)
|
||||
from devplacepy.services.skills.parser import SkillFormatError
|
||||
|
||||
|
||||
def test_normalize_adds_scheme_and_strips_fragment():
|
||||
assert normalize_input_url("example.org/skill#frag") == "https://example.org/skill"
|
||||
assert normalize_input_url(" http://example.org/a/SKILL.md ") == "http://example.org/a/SKILL.md"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"raw",
|
||||
["", " ", "ftp://example.org/x", "https://user:pw@example.org/x", "x" * (limits.URL_MAX_CHARS + 1)],
|
||||
)
|
||||
def test_normalize_rejects_bad_urls(raw):
|
||||
with pytest.raises(SkillFetchError):
|
||||
normalize_input_url(raw)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"url,expected",
|
||||
[
|
||||
("https://e.org/a/SKILL.md", "https://e.org/a/"),
|
||||
("https://e.org/a/index.html", "https://e.org/a/"),
|
||||
("https://e.org/a", "https://e.org/a/"),
|
||||
("https://e.org/a/", "https://e.org/a/"),
|
||||
("https://e.org", "https://e.org/"),
|
||||
],
|
||||
)
|
||||
def test_directory_url(url, expected):
|
||||
assert directory_url(url) == expected
|
||||
|
||||
|
||||
def test_origin_uses_default_ports():
|
||||
assert origin_of("https://Example.org/x") == ("https", "example.org", 443)
|
||||
assert origin_of("http://example.org:8080/x") == ("http", "example.org", 8080)
|
||||
|
||||
|
||||
def test_candidates_come_from_links_and_code_spans_once():
|
||||
body = (
|
||||
"See [the script](scripts/run.py) and `references/guide.md`.\n"
|
||||
"Again [dup](scripts/run.py#anchor) and [other](docs/readme.md) and `not a path`."
|
||||
)
|
||||
assert extract_candidates(body) == ["scripts/run.py", "references/guide.md"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"candidate,accepted",
|
||||
[
|
||||
("scripts/run.py", True),
|
||||
("./references/guide.md", True),
|
||||
("assets/templates/SPEC.md", True),
|
||||
("assets/img/hero.png", False),
|
||||
("../scripts/run.py", False),
|
||||
("scripts/../../etc/passwd", False),
|
||||
("scripts/%2e%2e/x.py", False),
|
||||
("scripts/a.py?x=1", False),
|
||||
("/scripts/run.py", False),
|
||||
("https://evil.org/scripts/run.py", False),
|
||||
("scripts\\run.py", False),
|
||||
("other/run.py", False),
|
||||
("scripts", False),
|
||||
("scripts/sp ace.py", False),
|
||||
],
|
||||
)
|
||||
def test_classify_candidate_only_accepts_safe_relative_text_paths(candidate, accepted):
|
||||
path, reason = classify_candidate(candidate)
|
||||
assert (path is not None) is accepted
|
||||
assert (reason == "") is accepted
|
||||
|
||||
|
||||
def test_overlong_path_is_rejected():
|
||||
path, reason = classify_candidate("scripts/" + "a" * limits.BUNDLED_PATH_MAX_CHARS + ".py")
|
||||
assert path is None and reason
|
||||
|
||||
|
||||
def test_decode_text():
|
||||
assert decode_text("héllo".encode("utf-8")) == "héllo"
|
||||
with pytest.raises(SkillFormatError):
|
||||
decode_text(b"a\x00b")
|
||||
with pytest.raises(SkillFormatError):
|
||||
decode_text(b"\xff\xfe\xfa")
|
||||
@@ -0,0 +1,179 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
import asyncio
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from devplacepy.net_guard import BlockedAddressError
|
||||
from devplacepy.services.skills import fetcher, limits
|
||||
from devplacepy.services.skills.fetcher import SkillFetchError, fetch_skill
|
||||
from tests.conftest import run_async as run
|
||||
|
||||
SKILL = """---
|
||||
name: demo-skill
|
||||
description: Demonstrates an install.
|
||||
license: MIT
|
||||
---
|
||||
# Demo
|
||||
|
||||
Run `scripts/run.py`, read [the guide](references/guide.md), see `./assets/logo.png`,
|
||||
[outside](https://other.example/scripts/x.py), [missing](references/missing.md)
|
||||
and [escape](scripts/../../etc/passwd).
|
||||
"""
|
||||
|
||||
|
||||
def serve(routes):
|
||||
seen = []
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
seen.append(str(request.url))
|
||||
entry = routes.get(request.url.path)
|
||||
if entry is None:
|
||||
return httpx.Response(404, text="missing")
|
||||
if callable(entry):
|
||||
return entry(request)
|
||||
status, body, content_type = entry
|
||||
return httpx.Response(status, content=body, headers={"content-type": content_type})
|
||||
|
||||
return httpx.MockTransport(handler), seen
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_origin(monkeypatch):
|
||||
def install(routes):
|
||||
transport, seen = serve(routes)
|
||||
|
||||
def client_factory(**kwargs):
|
||||
return httpx.AsyncClient(
|
||||
transport=transport,
|
||||
follow_redirects=kwargs.get("follow_redirects", True),
|
||||
max_redirects=kwargs.get("max_redirects", 20),
|
||||
headers=kwargs.get("headers"),
|
||||
)
|
||||
|
||||
monkeypatch.setattr(fetcher, "guarded_async_client", client_factory)
|
||||
return seen
|
||||
|
||||
return install
|
||||
|
||||
|
||||
def text(body, content_type="text/markdown"):
|
||||
return (200, body.encode("utf-8"), content_type)
|
||||
|
||||
|
||||
def test_direct_skill_url_installs_with_bundled_files_and_reasons(mock_origin):
|
||||
seen = mock_origin(
|
||||
{
|
||||
"/pack/SKILL.md": text(SKILL),
|
||||
"/pack/scripts/run.py": text("print('hi')", "text/x-python"),
|
||||
"/pack/references/guide.md": text("# Guide"),
|
||||
}
|
||||
)
|
||||
result = run(fetch_skill("https://skills.example/pack/SKILL.md"))
|
||||
assert result.parsed.name == "demo-skill"
|
||||
assert result.parsed.license == "MIT"
|
||||
assert result.source_url == "https://skills.example/pack/SKILL.md"
|
||||
assert result.files == {"scripts/run.py": "print('hi')", "references/guide.md": "# Guide"}
|
||||
reasons = {item.path: item.reason for item in result.skipped}
|
||||
assert "binary" in reasons["./assets/logo.png"]
|
||||
assert "references/missing.md" in reasons and "404" in reasons["references/missing.md"]
|
||||
assert not any("other.example" in url for url in seen)
|
||||
assert not any("etc/passwd" in url for url in seen)
|
||||
|
||||
|
||||
def test_directory_url_resolves_to_skill_md(mock_origin):
|
||||
mock_origin(
|
||||
{
|
||||
"/pack": (200, b"<html>index</html>", "text/html"),
|
||||
"/pack/SKILL.md": text(SKILL),
|
||||
}
|
||||
)
|
||||
result = run(fetch_skill("https://skills.example/pack"))
|
||||
assert result.source_url == "https://skills.example/pack/SKILL.md"
|
||||
assert result.parsed.name == "demo-skill"
|
||||
|
||||
|
||||
def test_index_page_redirect_resolves_to_its_directory(mock_origin):
|
||||
def redirect(request):
|
||||
return httpx.Response(302, headers={"location": "/pack/index.html"})
|
||||
|
||||
mock_origin(
|
||||
{
|
||||
"/start": redirect,
|
||||
"/pack/index.html": (200, b"<html></html>", "text/html"),
|
||||
"/pack/SKILL.md": text(SKILL),
|
||||
}
|
||||
)
|
||||
result = run(fetch_skill("https://skills.example/start"))
|
||||
assert result.source_url == "https://skills.example/pack/SKILL.md"
|
||||
|
||||
|
||||
def test_html_where_skill_md_was_expected_is_an_error(mock_origin):
|
||||
mock_origin({"/SKILL.md": (200, b"<html>login</html>", "text/html; charset=utf-8")})
|
||||
with pytest.raises(SkillFetchError, match="HTML page"):
|
||||
run(fetch_skill("https://skills.example/SKILL.md"))
|
||||
|
||||
|
||||
def test_http_error_and_invalid_skill_are_reported(mock_origin):
|
||||
mock_origin({"/gone/SKILL.md": (500, b"boom", "text/plain"), "/bad/SKILL.md": text("no frontmatter")})
|
||||
with pytest.raises(SkillFetchError, match="500"):
|
||||
run(fetch_skill("https://skills.example/gone/SKILL.md"))
|
||||
with pytest.raises(SkillFetchError, match="frontmatter"):
|
||||
run(fetch_skill("https://skills.example/bad/SKILL.md"))
|
||||
|
||||
|
||||
def test_oversize_skill_md_is_refused(mock_origin):
|
||||
mock_origin({"/big/SKILL.md": text("x" * (limits.SKILL_MD_MAX_BYTES + 10))})
|
||||
with pytest.raises(SkillFetchError, match="exceeds"):
|
||||
run(fetch_skill("https://skills.example/big/SKILL.md"))
|
||||
|
||||
|
||||
def test_bundled_file_count_and_size_limits(mock_origin):
|
||||
links = " ".join(f"`scripts/f{index}.py`" for index in range(limits.BUNDLED_MAX_FILES + 3))
|
||||
routes = {"/many/SKILL.md": text(f"---\nname: many\ndescription: d\n---\n{links}")}
|
||||
for index in range(limits.BUNDLED_MAX_FILES + 3):
|
||||
routes[f"/many/scripts/f{index}.py"] = text("x = 1")
|
||||
mock_origin(routes)
|
||||
result = run(fetch_skill("https://skills.example/many/SKILL.md"))
|
||||
assert len(result.files) == limits.BUNDLED_MAX_FILES
|
||||
assert sum(1 for item in result.skipped if "count limit" in item.reason) == 3
|
||||
|
||||
big = "y" * limits.BUNDLED_MAX_FILE_BYTES
|
||||
routes = {"/size/SKILL.md": text("---\nname: size\ndescription: d\n---\n`scripts/a.py` `scripts/b.py` `scripts/c.py` `scripts/d.py` `scripts/e.py`")}
|
||||
for letter in "abcde":
|
||||
routes[f"/size/scripts/{letter}.py"] = text(big)
|
||||
mock_origin(routes)
|
||||
result = run(fetch_skill("https://skills.example/size/SKILL.md"))
|
||||
assert sum(len(content) for content in result.files.values()) <= limits.BUNDLED_MAX_TOTAL_BYTES
|
||||
assert any("total size limit" in item.reason for item in result.skipped)
|
||||
|
||||
|
||||
def test_a_binary_bundled_file_with_a_text_extension_is_skipped(mock_origin):
|
||||
mock_origin(
|
||||
{
|
||||
"/bin/SKILL.md": text("---\nname: bin\ndescription: d\n---\n`scripts/blob.py`"),
|
||||
"/bin/scripts/blob.py": (200, b"abc\x00def", "text/x-python"),
|
||||
}
|
||||
)
|
||||
result = run(fetch_skill("https://skills.example/bin/SKILL.md"))
|
||||
assert result.files == {}
|
||||
assert result.skipped[0].path == "scripts/blob.py"
|
||||
|
||||
|
||||
def test_a_blocked_address_is_a_fetch_error(monkeypatch):
|
||||
def refuse(**kwargs):
|
||||
raise BlockedAddressError("Refusing to reach a private address.")
|
||||
|
||||
monkeypatch.setattr(fetcher, "guarded_async_client", refuse)
|
||||
with pytest.raises(SkillFetchError, match="private address"):
|
||||
run(fetch_skill("http://10.0.0.1/SKILL.md"))
|
||||
|
||||
|
||||
def test_the_overall_deadline_is_enforced(monkeypatch):
|
||||
async def slow_fetch(url):
|
||||
await asyncio.sleep(5)
|
||||
|
||||
monkeypatch.setattr(fetcher, "_fetch", slow_fetch)
|
||||
with pytest.raises(SkillFetchError, match="deadline"):
|
||||
run(fetch_skill("https://skills.example/slow/SKILL.md", deadline_seconds=0.05))
|
||||
@@ -0,0 +1,107 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
import pytest
|
||||
|
||||
from devplacepy.services.skills import limits
|
||||
from devplacepy.services.skills.parser import SkillFormatError, parse_skill_md
|
||||
|
||||
|
||||
def skill(frontmatter: str, body: str = "# Body\n\nDo the thing.") -> str:
|
||||
return f"---\n{frontmatter}\n---\n{body}"
|
||||
|
||||
|
||||
def test_minimal_skill_parses():
|
||||
parsed = parse_skill_md(skill("name: my-skill\ndescription: Does a thing."))
|
||||
assert parsed.name == "my-skill"
|
||||
assert parsed.description == "Does a thing."
|
||||
assert parsed.body.startswith("# Body")
|
||||
assert parsed.metadata == {}
|
||||
assert parsed.warnings == ()
|
||||
|
||||
|
||||
def test_all_known_fields_parse():
|
||||
parsed = parse_skill_md(
|
||||
skill(
|
||||
"name: full\n"
|
||||
"description: Full skill\n"
|
||||
"license: MIT\n"
|
||||
"compatibility: any agent\n"
|
||||
"allowed-tools: Read Write\n"
|
||||
"metadata:\n"
|
||||
" author: someone\n"
|
||||
" version: '1.0'"
|
||||
)
|
||||
)
|
||||
assert parsed.license == "MIT"
|
||||
assert parsed.compatibility == "any agent"
|
||||
assert parsed.allowed_tools == "Read Write"
|
||||
assert parsed.metadata == {"author": "someone", "version": "1.0"}
|
||||
|
||||
|
||||
def test_block_scalars():
|
||||
folded = parse_skill_md(skill("name: a\ndescription: >\n first line\n second line"))
|
||||
assert folded.description == "first line second line"
|
||||
literal = parse_skill_md(skill("name: a\ndescription: |\n first\n second"))
|
||||
assert literal.description == "first\nsecond"
|
||||
|
||||
|
||||
def test_quoted_values_and_inline_comments():
|
||||
parsed = parse_skill_md(skill('name: a\ndescription: "Quoted: value" '))
|
||||
assert parsed.description == "Quoted: value"
|
||||
parsed = parse_skill_md(skill("name: a\ndescription: plain # a comment"))
|
||||
assert parsed.description == "plain"
|
||||
|
||||
|
||||
def test_unknown_key_is_ignored_with_a_warning():
|
||||
parsed = parse_skill_md(skill("name: a\ndescription: d\nfuture-key: x"))
|
||||
assert parsed.warnings == ("Ignored unknown frontmatter key 'future-key'.",)
|
||||
|
||||
|
||||
def test_bom_and_crlf_are_accepted():
|
||||
text = "---\r\nname: a\r\ndescription: d\r\n---\r\nBody"
|
||||
assert parse_skill_md(text).body == "Body"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"text,fragment",
|
||||
[
|
||||
("name: a\ndescription: d", "no YAML frontmatter"),
|
||||
("---\nname: a\ndescription: d\n", "not closed"),
|
||||
(skill("description: d"), "'name' is required"),
|
||||
(skill("name: a"), "'description' is required"),
|
||||
(skill("name: A_B\ndescription: d"), "lowercase letters"),
|
||||
(skill("name: -a\ndescription: d"), "lowercase letters"),
|
||||
(skill("name: a--b\ndescription: d"), "lowercase letters"),
|
||||
(skill("name: a\ndescription: d\nname: b"), "duplicate frontmatter key"),
|
||||
(skill("name: a\ndescription: [x]"), "flow collections"),
|
||||
(skill("name: a\ndescription: d\nmetadata: x"), "indented map"),
|
||||
(skill("name: a\ndescription: d\nmetadata:\n a: 1\n b: 2"), "flat map"),
|
||||
(skill("name: a\ndescription: d\nmetadata:\n a: 1\n a: 2"), "duplicate metadata key"),
|
||||
(skill("name: a\ndescription: \"unterminated"), "unterminated"),
|
||||
(skill("name: a\ndescription: d\n\tbad: x"), "tab"),
|
||||
(skill("name: a\ndescription: d") + "\x00", "NUL"),
|
||||
],
|
||||
)
|
||||
def test_malformed_skills_are_rejected_with_a_reason(text, fragment):
|
||||
with pytest.raises(SkillFormatError) as error:
|
||||
parse_skill_md(text)
|
||||
assert fragment in str(error.value)
|
||||
|
||||
|
||||
def test_overlong_fields_are_rejected():
|
||||
with pytest.raises(SkillFormatError):
|
||||
parse_skill_md(skill(f"name: {'a' * (limits.NAME_MAX_CHARS + 1)}\ndescription: d"))
|
||||
with pytest.raises(SkillFormatError):
|
||||
parse_skill_md(skill(f"name: a\ndescription: {'d' * (limits.DESCRIPTION_MAX_CHARS + 1)}"))
|
||||
|
||||
|
||||
def test_metadata_entry_cap():
|
||||
entries = "\n".join(f" k{i}: v" for i in range(limits.METADATA_MAX_ENTRIES + 1))
|
||||
with pytest.raises(SkillFormatError):
|
||||
parse_skill_md(skill(f"name: a\ndescription: d\nmetadata:\n{entries}"))
|
||||
|
||||
|
||||
def test_frontmatter_scan_is_bounded():
|
||||
padding = "\n".join("# c" for _ in range(limits.FRONTMATTER_MAX_LINES + 5))
|
||||
with pytest.raises(SkillFormatError):
|
||||
parse_skill_md(f"---\nname: a\ndescription: d\n{padding}\n---\nBody")
|
||||
Reference in New Issue
Block a user