taicpp: integrate build and tests (libdetz.a, per-module test binaries, tests/run_all.py, real lint, parity targets)
make test builds and runs 32 steps; not green: misc.diff and mplex.e2e fail on real module bugs. test_main.cpp replaced by util_config_test.cpp; stale *.mk and store_test rule removed. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Sonnet 5.5
@@ -1,36 +0,0 @@
|
||||
# Foundation tests (store, config, session, plexer, repl).
|
||||
# Run from taicpp/: make -f tests/foundation.mk test
|
||||
# Outputs stay under build/. Independent of the main Makefile on purpose: the
|
||||
# foundation area builds and tests itself with only sqlite3 as a link dependency.
|
||||
CXX ?= g++
|
||||
CXXFLAGS ?= -std=c++23 -Wall -Wextra -O1
|
||||
INC = -Isrc -Isrc/store -Isrc/session -Isrc/config -Isrc/plexer -Isrc/repl
|
||||
FB = build/foundation
|
||||
FSRCS = $(wildcard src/store/*.cpp) $(wildcard src/session/*.cpp) $(wildcard src/config/*.cpp) $(wildcard src/plexer/*.cpp) $(wildcard src/repl/*.cpp)
|
||||
FOBJS = $(patsubst src/%.cpp,$(FB)/%.o,$(FSRCS))
|
||||
LIBS = -lsqlite3 -pthread
|
||||
|
||||
.PHONY: all test clean-foundation
|
||||
all: build/foundation_probe build/repl_probe build/foundation_test
|
||||
|
||||
$(FB)/%.o: src/%.cpp
|
||||
@mkdir -p $(dir $@)
|
||||
$(CXX) $(CXXFLAGS) $(INC) -c $< -o $@
|
||||
|
||||
build/foundation_probe: tests/foundation_probe.cpp $(FOBJS)
|
||||
$(CXX) $(CXXFLAGS) $(INC) $< $(FOBJS) $(LIBS) -o $@
|
||||
|
||||
build/repl_probe: tests/repl_probe.cpp $(FOBJS)
|
||||
$(CXX) $(CXXFLAGS) $(INC) $< $(FOBJS) $(LIBS) -o $@
|
||||
|
||||
build/foundation_test: tests/foundation_test.cpp $(FOBJS)
|
||||
$(CXX) $(CXXFLAGS) $(INC) $< $(FOBJS) $(LIBS) -o $@
|
||||
|
||||
test: all
|
||||
build/foundation_test
|
||||
python3 tests/foundation_parity.py
|
||||
python3 tests/repl_persist.py
|
||||
python3 tests/tmux_ask.py
|
||||
|
||||
clean-foundation:
|
||||
rm -rf $(FB) build/foundation_probe build/repl_probe build/foundation_test
|
||||
@@ -1,34 +0,0 @@
|
||||
# Standalone build of the misc modules (skills, sched, telegram, rag, install) and their tests.
|
||||
# Usage (from taicpp/): make -f tests/misc.mk test
|
||||
# The integrator can fold these into the main Makefile; see docs/wiring/misc.md.
|
||||
CXX ?= g++
|
||||
CXXFLAGS ?= -std=c++23 -Wall -Wextra -O1
|
||||
B = build/misc
|
||||
DIRS = src/skills src/sched src/telegram src/rag src/install
|
||||
INC = $(foreach d,$(DIRS),-I$d)
|
||||
SRCS = $(foreach d,$(DIRS),$(wildcard $(d)/*.cpp))
|
||||
OBJS = $(patsubst %.cpp,$(B)/%.o,$(SRCS))
|
||||
LIBS = -lsqlite3 -lcurl -lz -lpthread
|
||||
|
||||
all: $(B)/misc_test $(B)/misc_driver
|
||||
|
||||
$(B)/%.o: %.cpp
|
||||
@mkdir -p $(dir $@)
|
||||
$(CXX) $(CXXFLAGS) $(INC) -MMD -MP -c $< -o $@
|
||||
|
||||
$(B)/misc_test: tests/misc_test.cpp $(OBJS)
|
||||
@mkdir -p $(B)
|
||||
$(CXX) $(CXXFLAGS) $(INC) tests/misc_test.cpp $(OBJS) $(LIBS) -o $@
|
||||
|
||||
$(B)/misc_driver: tests/misc_driver.cpp $(OBJS)
|
||||
@mkdir -p $(B)
|
||||
$(CXX) $(CXXFLAGS) $(INC) tests/misc_driver.cpp $(OBJS) $(LIBS) -lz -o $@
|
||||
|
||||
test: $(B)/misc_test
|
||||
$(B)/misc_test
|
||||
|
||||
objs: $(OBJS)
|
||||
|
||||
.PHONY: all test objs
|
||||
|
||||
-include $(OBJS:.o=.d)
|
||||
@@ -0,0 +1,209 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Runs every taicpp test (C++ unit binaries and python differential drivers) and prints one
|
||||
summary table. Exit 1 if any step fails; nothing is skipped silently: a step whose prerequisite is
|
||||
missing (binary not built, tmux absent) is reported as FAIL or SKIP with the reason.
|
||||
|
||||
usage (from taicpp/, after `make test-build`; `make test` does both):
|
||||
python3 tests/run_all.py [--only SUBSTR ...] [--list] [--timeout SECONDS] [-v]
|
||||
|
||||
Output of every step goes to build/test-logs/<step>.log. Nothing is written outside build/.
|
||||
A step is a list of commands that all must exit 0; ("diff", A, B) runs two commands and requires
|
||||
identical stdout. Counts in the table are numbers the step itself printed (checks, cases,
|
||||
scenarios, tests); "-" means the step prints no count.
|
||||
"""
|
||||
import argparse
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
ROOT = os.path.dirname(HERE) # taicpp/
|
||||
REPO = os.path.dirname(ROOT) # repository root (tai.py lives here)
|
||||
BUILD = os.path.join(ROOT, "build")
|
||||
LOGS = os.path.join(BUILD, "test-logs")
|
||||
TAI = os.path.join(REPO, "tai.py")
|
||||
PY = sys.executable
|
||||
STEP_TIMEOUT = 1200 # hard cap per step, seconds
|
||||
|
||||
|
||||
def b(name):
|
||||
return os.path.join(BUILD, name)
|
||||
|
||||
|
||||
def t(name):
|
||||
return os.path.join(HERE, name)
|
||||
|
||||
|
||||
# (name, module, [commands], env or None). A command is an argv list or ("diff", argv, argv).
|
||||
STEPS = [
|
||||
# foundation
|
||||
("foundation.unit", "foundation", [[b("foundation_test")]], None),
|
||||
("foundation.parity", "foundation", [[PY, t("foundation_parity.py")]], None),
|
||||
("foundation.repl_persist", "foundation", [[PY, t("repl_persist.py")]], None),
|
||||
("foundation.tmux_ask", "foundation", [[PY, t("tmux_ask.py")]], None),
|
||||
("store.smoke", "foundation", [[b("store_test")]], None),
|
||||
("cli.unit", "cli", [[b("cli_test")]], None),
|
||||
("util_config.unit", "util/config", [[b("util_config_test")]], None),
|
||||
# agent
|
||||
("agent.diff", "agent", [[PY, t("agent_diff.py")]], None),
|
||||
("agent.e2e", "agent", [[PY, t("agent_e2e.py")]], None),
|
||||
("agent.sched", "agent", [[PY, t("agent_sched.py")]], None),
|
||||
# shell and util
|
||||
("shell.unit", "shell", [[b("shell_test")]], None),
|
||||
("shell.parity", "shell", [[PY, t("shell_parity.py")]], None),
|
||||
# cap and mcp
|
||||
("capmcp.unit", "cap/mcp", [[b("capmcp/capmcp_unit")]], None),
|
||||
("capmcp.wire_diff", "cap/mcp", [[PY, "-I", t("capmcp_diff.py"), b("capmcp/capmcp_probe")]], None),
|
||||
("capmcp.cli_diff", "cap/mcp", [[PY, "-I", t("capmcp_cli_diff.py"), b("capmcp/capmcp_cli"), "--tai", TAI]], None),
|
||||
# misc
|
||||
("misc.unit", "misc", [[b("misc/misc_test")]], None),
|
||||
("misc.diff", "misc", [[PY, t("misc_diff.py")]], None),
|
||||
# tui and grr
|
||||
("tui.spec", "tui", [[b("tui_spec")]], None),
|
||||
("tui.demo_shot", "tui", [("diff", [PY, TAI, "--demo-shot"], [b("tui_shot")])], None),
|
||||
# mplex
|
||||
("mplex.foundation", "mplex", [[b("mplex_foundation_test")]], {"TZ": "UTC"}),
|
||||
("mplex.core", "mplex", [[PY, "-I", t("mplex_core_gen.py")], [b("mplex_core_test")]], None),
|
||||
("mplex.model", "mplex", [[PY, "-I", t("mplex_model_gen.py"), b("mplex_model_cases.jsonl"), b("mplex_model_scratch")],
|
||||
[b("mplex_model_test"), b("mplex_model_cases.jsonl"), b("mplex_model_scratch")]], None),
|
||||
("mplex.copy", "mplex", [[PY, "-I", t("mplex_copy_gen.py"), REPO, b("mplex_copy_scenarios.txt"), "3600", "20251025"],
|
||||
[b("mplex_copy_test"), b("mplex_copy_scenarios.txt")]], None),
|
||||
("mplex.vt", "mplex", [[PY, "-I", t("mplex_vt_oracle.py"), TAI, b("mplex_vt_cases.txt")],
|
||||
[b("mplex_vt_test"), TAI, b("mplex_vt_cases.txt")]], None),
|
||||
("mplex.flow", "mplex", [[PY, "-I", t("mplex_flow_gen.py"), REPO, b("mplex_flow_corpus.jsonl")],
|
||||
[b("mplex_flow_test"), b("mplex_flow_corpus.jsonl")]], None),
|
||||
("mplex.flow_tmux", "mplex", [("diff", [b("mplex_flow_test"), "--tmux"], [PY, "-I", t("mplex_tmux_oracle.py"), REPO])], None),
|
||||
("mplex.regress", "mplex", [[b("mplex_regress_test")]], None),
|
||||
("mplex.spec", "mplex", [[b("mplex_spec_test"), REPO]], None),
|
||||
("mplex.server", "mplex", [[b("mplex_server_test"), "--tai", TAI]], None),
|
||||
("mplex.commands", "mplex", [[b("mplex_commands_test"), b("mplex_cli"), REPO]], None),
|
||||
("mplex.commands_pty", "mplex", [[PY, "-I", t("mplex_commands_pty.py"), b("mplex_cli"), REPO, b("mplex_commands_pty_work")]], None),
|
||||
("mplex.e2e", "mplex", [[PY, "-I", t("mplex_e2e.py"), REPO, "--cpp", b("mplex_cli")]], None),
|
||||
]
|
||||
|
||||
COUNT_RE = re.compile(r"(\d[\d,]*)\s+(checks?|cases?|tests?|scenarios?|assertions?|passed|ok|probes?|runs?)\b", re.I)
|
||||
SLASH_RE = re.compile(r"\b(\d+)\s*/\s*(\d+)\b")
|
||||
|
||||
|
||||
def count_of(text):
|
||||
"""Best effort: the biggest number the step reported next to a unit word. None if none."""
|
||||
best = None
|
||||
for m in COUNT_RE.finditer(text):
|
||||
n = int(m.group(1).replace(",", ""))
|
||||
if best is None or n > best:
|
||||
best = n
|
||||
return best or None
|
||||
|
||||
|
||||
def run_cmd(argv, env, log):
|
||||
start = time.time()
|
||||
try:
|
||||
done = subprocess.run(argv, cwd=ROOT, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
|
||||
stdin=subprocess.DEVNULL, timeout=STEP_TIMEOUT)
|
||||
out, code = done.stdout.decode("utf-8", "replace"), done.returncode
|
||||
except subprocess.TimeoutExpired as exc:
|
||||
out, code = (exc.stdout or b"").decode("utf-8", "replace") + "\nTIMEOUT after %ds\n" % STEP_TIMEOUT, 124
|
||||
except OSError as exc:
|
||||
out, code = "cannot start %s: %s\n" % (argv[0], exc), 127
|
||||
log.write("$ %s (rc=%d, %.1fs)\n%s\n" % (" ".join(argv), code, time.time() - start, out))
|
||||
return code, out
|
||||
|
||||
|
||||
def run_step(step):
|
||||
name, module, cmds, extra = step
|
||||
env = dict(os.environ)
|
||||
for key in [k for k in env if k.startswith("TAI_")]:
|
||||
env.pop(key, None)
|
||||
env.update(extra or {})
|
||||
start = time.time()
|
||||
status, note, text = "PASS", "", ""
|
||||
with open(os.path.join(LOGS, name + ".log"), "w") as log:
|
||||
for cmd in cmds:
|
||||
if isinstance(cmd, tuple):
|
||||
c1, o1 = run_cmd(cmd[1], env, log)
|
||||
c2, o2 = run_cmd(cmd[2], env, log)
|
||||
text += o1 + o2
|
||||
if c1 or c2:
|
||||
status, note = "FAIL", "rc %d/%d" % (c1, c2)
|
||||
elif o1 != o2:
|
||||
status, note = "FAIL", "outputs differ (%d vs %d bytes)" % (len(o1), len(o2))
|
||||
log.write("DIFF: outputs differ\n")
|
||||
else:
|
||||
text += "%d bytes identical\n" % len(o1)
|
||||
else:
|
||||
code, out = run_cmd(cmd, env, log)
|
||||
text += out
|
||||
if code:
|
||||
status, note = "FAIL", "rc %d" % code
|
||||
if status == "FAIL":
|
||||
break
|
||||
if status == "PASS" and re.search(r"^SKIP", text, re.M):
|
||||
status, note = "SKIP", "reported SKIP (see log)"
|
||||
if status == "PASS" and re.search(r"\bFAIL\b|\b[1-9]\d* fail(ed|ures?)\b(?!=)", text):
|
||||
status, note = "FAIL", "exit 0 but output reports failures"
|
||||
return {"name": name, "module": module, "status": status, "note": note, "secs": time.time() - start,
|
||||
"count": count_of(text), "tail": text.strip().splitlines()[-12:]}
|
||||
|
||||
|
||||
def snapshot():
|
||||
"""Files under taicpp/ outside build/ and third_party/, to flag tests that leak outputs."""
|
||||
seen = set()
|
||||
for dirpath, dirnames, filenames in os.walk(ROOT):
|
||||
dirnames[:] = [d for d in dirnames if d not in ("build", "third_party", "__pycache__")]
|
||||
for name in filenames:
|
||||
seen.add(os.path.relpath(os.path.join(dirpath, name), ROOT))
|
||||
return seen
|
||||
|
||||
|
||||
def main():
|
||||
global STEP_TIMEOUT
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--only", action="append", default=[])
|
||||
ap.add_argument("--list", action="store_true")
|
||||
ap.add_argument("--timeout", type=int, default=STEP_TIMEOUT)
|
||||
ap.add_argument("-v", action="store_true")
|
||||
opts = ap.parse_args()
|
||||
STEP_TIMEOUT = opts.timeout
|
||||
steps = [s for s in STEPS if not opts.only or any(o in s[0] for o in opts.only)]
|
||||
if opts.list:
|
||||
for s in steps:
|
||||
print(s[0])
|
||||
return 0
|
||||
os.makedirs(LOGS, exist_ok=True)
|
||||
before = snapshot()
|
||||
results = []
|
||||
for s in steps:
|
||||
print("running %-26s" % s[0], end=" ", flush=True)
|
||||
r = run_step(s)
|
||||
print("%s %.1fs" % (r["status"], r["secs"]), flush=True)
|
||||
if r["status"] != "PASS":
|
||||
print("\n".join(" | " + line for line in r["tail"]))
|
||||
results.append(r)
|
||||
print()
|
||||
print("%-26s %-12s %-6s %8s %8s %s" % ("step", "module", "status", "count", "secs", "note"))
|
||||
print("-" * 90)
|
||||
for r in results:
|
||||
print("%-26s %-12s %-6s %8s %8.1f %s" % (r["name"], r["module"], r["status"],
|
||||
"-" if r["count"] is None else r["count"], r["secs"], r["note"]))
|
||||
print("-" * 90)
|
||||
npass = sum(r["status"] == "PASS" for r in results)
|
||||
nfail = sum(r["status"] == "FAIL" for r in results)
|
||||
nskip = sum(r["status"] == "SKIP" for r in results)
|
||||
total = sum(r["count"] or 0 for r in results)
|
||||
leaked = sorted(snapshot() - before)
|
||||
if leaked:
|
||||
print("WARNING: tests wrote outside build/: " + ", ".join(leaked[:10]))
|
||||
print("steps: %d pass: %d fail: %d skip: %d reported counts summed: %d logs: %s" %
|
||||
(len(results), npass, nfail, nskip, total, os.path.relpath(LOGS, ROOT)))
|
||||
if nfail:
|
||||
print("RESULT: FAILED: " + ", ".join(r["name"] for r in results if r["status"] == "FAIL"))
|
||||
return 1
|
||||
print("RESULT: ok" + (" (with skips)" if nskip else ""))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -1,65 +1,27 @@
|
||||
// Unit checks for src/util (text helpers, ansi) and src/config (consts, Config, tag resolution),
|
||||
// plus the CLI help smoke. This is the surviving part of the old tests/test_main.cpp: its shell
|
||||
// section is covered by tests/shell_test.cpp, its cli section by tests/cli_test.cpp, and its llm
|
||||
// section exercised the placeholder llm API that the agent port removed (ChatClient("openai", ...),
|
||||
// ModelHealth, Opencode, provider_info); the agent tests (agent_diff, agent_e2e) replace it.
|
||||
// Plain asserts, so NDEBUG must stay off.
|
||||
#undef NDEBUG
|
||||
#include "../src/cli/cli.h"
|
||||
#include "../src/shell/sandbox.h"
|
||||
#include "../src/shell/session.h"
|
||||
#include "../src/shell/pty.h"
|
||||
#include "../src/shell/spinner.h"
|
||||
#include "../src/shell/livestream.h"
|
||||
#include "../src/shell/lanes.h"
|
||||
#include "../src/shell/runlive.h"
|
||||
#include "../src/shell/targets.h"
|
||||
#include "../src/shell/glow.h"
|
||||
#include "../src/llm/llm.h"
|
||||
#include "../src/util/ansi.h"
|
||||
#include "../src/util/text.h"
|
||||
#include "../src/config/config.h"
|
||||
#include "../src/config/consts.h"
|
||||
#include <cassert>
|
||||
#include <filesystem>
|
||||
#include <cstdio>
|
||||
using detz::llm::ChatClient;
|
||||
using detz::llm::ChatRequest;
|
||||
#include <filesystem>
|
||||
#include <map>
|
||||
|
||||
int main() {
|
||||
detz::cli::Parser p;
|
||||
const char* a0 = "detz";
|
||||
auto c = p.parse(1, (char**)&a0);
|
||||
assert(c.name == "--help");
|
||||
assert(c.run() == 0);
|
||||
// shell (full coverage: tests/shell_test.cpp, differential: tests/shell_parity.py)
|
||||
assert(!detz::shell::CurlHint("curl http://x").empty());
|
||||
assert(detz::shell::CurlHint("echo hi").empty());
|
||||
assert(detz::shell::ShellTargets::Segment({"rm", "a"}).size() == 1);
|
||||
{ detz::shell::RunLiveOptions o; auto r = detz::shell::RunLive::Run(detz::shell::Command::Shell("echo hi"), o);
|
||||
assert(r.code == 0 && r.output == "hi\n" && !r.timedOut); }
|
||||
{ detz::shell::LiveStream ls(4, 40, nullptr, false); ls.Feed("x"); assert(ls.shown() == 0); }
|
||||
assert(detz::shell::Glow::Ellipsis("short", 60) == "short");
|
||||
assert(detz::shell::Sandbox::kName == std::string("tai-box"));
|
||||
// llm mocked transport
|
||||
auto mock = [](const std::string&, const std::string&, const std::map<std::string,std::string>&) {
|
||||
return std::string("{\"content\":\"hello\"}");
|
||||
};
|
||||
ChatClient cc("openai", "m", mock);
|
||||
auto cr = cc.chat(ChatRequest{"m", {{"user", "hi"}}, false});
|
||||
assert(cr.text == "hello");
|
||||
auto smock = [](const std::string&, const std::string&, const std::map<std::string,std::string>&) {
|
||||
return std::string("data: {\"content\":\"a\"}\n\ndata: [DONE]\n\n");
|
||||
};
|
||||
ChatClient cs("anthropic", "m", smock);
|
||||
std::vector<std::string> got;
|
||||
cs.chat_stream(ChatRequest{"m", {}, true}, [&](const std::string& d) { got.push_back(d); return true; });
|
||||
assert(got.size() == 1);
|
||||
bool threw = false;
|
||||
try { ChatClient bad("openai", "m"); bad.chat(ChatRequest{}); } catch (const detz::llm::BackendError&) { threw = true; }
|
||||
assert(threw);
|
||||
detz::llm::ModelHealth h; h.record("a", false, 5); h.record("b", true, 9);
|
||||
assert(h.size() == 2 && h.best()->model == "b");
|
||||
assert(detz::llm::provider_names().find("openrouter") != std::string::npos);
|
||||
assert(detz::llm::provider_info("xai").base_url.find("x.ai") != std::string::npos);
|
||||
detz::llm::Opencode oc("http://localhost:9", mock);
|
||||
assert(oc.complete("m", {}).text == "hello");
|
||||
bool be = false;
|
||||
try { detz::llm::Opencode no; no.complete("m", {}); } catch (const detz::llm::BackendError&) { be = true; }
|
||||
assert(be);
|
||||
// M2 util+config
|
||||
// util + config
|
||||
assert(detz::Consts::kTotalStepCap == 500 && detz::Consts::kAuditMaxChars == 20000);
|
||||
assert(std::string(detz::kDefaultModel) == "muse");
|
||||
assert(!detz::ZenFreeModels().empty() && !detz::PollinationsModels().empty());
|
||||
@@ -90,7 +52,6 @@ int main() {
|
||||
auto ru = detz::ResolveTag("zzz-unknown-xyz", {}, ids, "/tmp"); assert(ru.first == "unknown");
|
||||
std::map<std::string,std::string> bots = {{"bob", "BotBob"}};
|
||||
auto rb = detz::ResolveTag("bob", bots, ids, "/tmp"); assert(rb.first == "bot"); }
|
||||
std::puts("M2 OK");
|
||||
std::puts("M4+M5 OK");
|
||||
std::puts("util+config OK");
|
||||
return 0;
|
||||
}
|
||||
Reference in New Issue
Block a user