36 lines
1.2 KiB
Python
36 lines
1.2 KiB
Python
|
# retoor <retoor@molodetz.nl>
|
||
|
|
|
||
|
|
from devplacepy.services.devii.agents.actions import AGENT_ACTIONS
|
||
|
|
from devplacepy.services.devii.agents.evals import EVALS
|
||
|
|
from devplacepy.services.devii.agents.policy import CATEGORIES
|
||
|
|
from devplacepy.services.devii.registry import CATALOG
|
||
|
|
|
||
|
|
|
||
|
|
def test_twenty_unique_evals():
|
||
|
|
assert len(EVALS) == 20
|
||
|
|
assert len({item["uid"] for item in EVALS}) == 20
|
||
|
|
|
||
|
|
|
||
|
|
def test_evals_cover_channels_and_kinds():
|
||
|
|
channels = {item["channel"] for item in EVALS}
|
||
|
|
assert channels == {"main", "telegram"}
|
||
|
|
kinds = {item["expect"] for item in EVALS}
|
||
|
|
assert "direct" in kinds
|
||
|
|
assert kinds - {"direct"} <= set(CATEGORIES)
|
||
|
|
|
||
|
|
|
||
|
|
def test_evals_are_well_formed_and_grounded():
|
||
|
|
known = set(CATALOG.by_name()) | {a.name for a in AGENT_ACTIONS}
|
||
|
|
for item in EVALS:
|
||
|
|
assert item["prompt"].strip()
|
||
|
|
assert item["end_state"].strip()
|
||
|
|
assert item["channel"] in ("main", "telegram")
|
||
|
|
assert item["expect"] == "direct" or item["expect"] in CATEGORIES
|
||
|
|
for tool in item["tools"]:
|
||
|
|
assert tool in known
|
||
|
|
if item["expect"] != "direct":
|
||
|
|
assert "agent_start" in item["tools"]
|
||
|
|
assert {"agent_result", "agent_status", "agent_cancel"} & set(
|
||
|
|
item["tools"]
|
||
|
|
)
|