94 lines
4.7 KiB
C++
94 lines
4.7 KiB
C++
// CapModelCatalog: agent checks, model protocol (manifest list + probe), matching, reports.
|
|
#include <cstdio>
|
|
#include <cstdlib>
|
|
#include <fstream>
|
|
#include <sstream>
|
|
#include <string>
|
|
|
|
#include "cap_model_catalog.h"
|
|
#include "wire_os.h"
|
|
|
|
using namespace detz::cap;
|
|
|
|
static int failures = 0;
|
|
#define CHECK(c) do { if (!(c)) { std::printf("FAIL %s:%d %s\n", __FILE__, __LINE__, #c); ++failures; } } while (0)
|
|
|
|
static void put(const std::string& dir, const std::string& name, const std::string& binary, const std::string& extra) {
|
|
std::ofstream(dir + "/" + name + ".toml") << "[agent]\nname = \"" << name << "\"\nbinary = \"" << binary
|
|
<< "\"\nprofiles = [\"coding\"]\n[startup]\ncommand = [\"" << binary << "\"]\nready_when = {pattern = \"x\"}\n"
|
|
<< "[pty]\ncols = 80\nrows = 24\n[capabilities]\nstreaming_output = true\n" << extra;
|
|
}
|
|
|
|
int main() {
|
|
char tmpl[] = "/tmp/capcatXXXXXX";
|
|
std::string dir = mkdtemp(tmpl);
|
|
put(dir, "alpha", "sh", "[models]\nlist = [\"alpha-large\", \"alpha-small\", \"shared-1\"]\ndefault = \"alpha-small\"\n");
|
|
put(dir, "beta", "sh", "[models]\nprobe = [\"beta\", \"--cap-models\"]\n");
|
|
put(dir, "gone", "no-such-binary-xyz", "");
|
|
std::ofstream(dir + "/bad.toml") << "garbage\n";
|
|
int probes = 0;
|
|
CapModelCatalog cat(CapRegistry(dir, dir + "/none"), [&](const std::vector<std::string>& argv, double) -> std::string {
|
|
++probes;
|
|
if (argv[0] != "beta") throw std::runtime_error("unexpected probe");
|
|
return "{\"models\":[{\"id\":\"beta-one\"},\"shared-1\"],\"default\":\"beta-one\"}";
|
|
});
|
|
|
|
CHECK(cat.agentNames().size() == 4);
|
|
CHECK(cat.entry("alpha").ready() && cat.entry("alpha").models.size() == 3);
|
|
CHECK(cat.entry("beta").modelSource == "probe" && cat.entry("beta").defaultModel == "beta-one");
|
|
CHECK(cat.entry("gone").status == "no_binary");
|
|
CHECK(cat.entry("bad").status == "bad_manifest");
|
|
CHECK(probes == 1); // cached
|
|
|
|
CapSelection s = cat.resolve("alpha");
|
|
CHECK(s.ok && s.agent == "alpha" && s.model.empty());
|
|
s = cat.resolve("ALPHA:alpha-s");
|
|
CHECK(s.ok && s.model == "alpha-small"); // case-insensitive agent, unique prefix
|
|
s = cat.resolve("alpha:alpha");
|
|
CHECK(s.ok && s.model == "alpha-large"); // equal scores: list order, never an error
|
|
s = cat.resolve("alpha:lrg");
|
|
CHECK(s.ok && s.model == "alpha-large" && !s.note.empty());
|
|
s = cat.resolve("alpah:small"); // typo in the agent, partial model
|
|
CHECK(s.ok && s.agent == "alpha" && s.model == "alpha-small");
|
|
s = cat.resolve("alpha:zzz");
|
|
CHECK(!s.ok && s.error.find("alpha [ready]") != std::string::npos && s.error.find("beta-one") != std::string::npos);
|
|
s = cat.resolve("gone");
|
|
CHECK(!s.ok && s.error.find("not on PATH") != std::string::npos && s.error.find("bad [bad_manifest]") != std::string::npos);
|
|
s = cat.resolve("beta-one"); // bare model, unique agent
|
|
CHECK(s.ok && s.agent == "beta" && s.model == "beta-one");
|
|
s = cat.resolve("shared-1"); // on two agents: first agent wins
|
|
CHECK(s.ok && s.agent == "alpha" && s.model == "shared-1");
|
|
s = cat.resolve("bet");
|
|
CHECK(s.ok && s.agent == "beta" && s.model.empty());
|
|
s = cat.resolve("betaone");
|
|
CHECK(s.ok && s.agent == "beta" && s.model == "beta-one");
|
|
s = cat.resolve("qqzzxx");
|
|
CHECK(!s.ok && s.error.find("resembles") != std::string::npos && s.error.find("cap agents (4)") != std::string::npos);
|
|
CHECK(cat.isAgent("alpha:x") && !cat.isAgent("x"));
|
|
|
|
put(dir, "open", "sh", ""); // no list: pass through with a note
|
|
cat.invalidate();
|
|
s = cat.resolve("open:anything");
|
|
CHECK(s.ok && s.model == "anything" && !s.note.empty());
|
|
|
|
CapModelCatalog failing(CapRegistry(dir, dir + "/none"), [](const std::vector<std::string>&, double) -> std::string {
|
|
throw std::runtime_error("timed out");
|
|
});
|
|
CapAgentEntry b = failing.entry("beta");
|
|
CHECK(b.ready() && b.models.empty() && b.modelError.find("timed out") != std::string::npos);
|
|
|
|
{ // docs/cap-agents.md lists every constant and status the code defines
|
|
std::ifstream in("docs/cap-agents.md");
|
|
std::stringstream ss;
|
|
ss << in.rdbuf();
|
|
const std::string doc = ss.str();
|
|
CHECK(!doc.empty());
|
|
for (const char* name : {"CAP_AGENT_MAX", "CAP_MODEL_MAX", "CAP_MODEL_PROBE_TIMEOUT", "CAP_MODEL_PROBE_MAX_BYTES", "CAP_CATALOG_TTL",
|
|
"CAP_FUZZY_FLOOR", "CAP_TURN_TIMEOUT", "ready", "no_binary", "bad_manifest", "no_manifest"})
|
|
if (doc.find(name) == std::string::npos) { std::printf("FAIL docs/cap-agents.md lacks %s\n", name); ++failures; }
|
|
}
|
|
if (std::system(("rm -rf " + dir).c_str()) != 0) std::printf("cleanup failed\n");
|
|
std::printf(failures ? "FAILED %d\n" : "cap_model_catalog_test OK\n", failures);
|
|
return failures ? 1 : 0;
|
|
}
|