267 lines
13 KiB
Python
267 lines
13 KiB
Python
#!/usr/bin/env python3
|
|
"""tests/glue_persist.py: integration tests of the real build/detz binary (docs/glue.md).
|
|
|
|
Built on ../../tai_test.py. Every case runs the binary as a child of a long-lived bash that stands for one
|
|
terminal pane, because a conversation is keyed by the nearest shell ancestor: same shell and restart means
|
|
the same history, another shell means its own history.
|
|
|
|
These cases make real LLM calls through the real binary (the project rule, tests/README.md at the repository
|
|
root): a free, rotating backend makes the model-dependent checks flaky, which is accepted. Assertions read
|
|
vault files (profiles/*.session.*.json, *.turn.*.json, memory.db), never the rendered terminal output, except
|
|
for the deterministic --version/--help case.
|
|
|
|
GlueRestartPersistence start, a message the model answers, quit, start again in the same shell: history
|
|
resumed and extended; another shell gets a separate history
|
|
GlueProfileSwitch /profile and /clear act on the vault history the agent uses
|
|
GlueOneShotShell one-shot prompt that must make a shell tool call; the tool call and its result are in the vault
|
|
GlueFlagsSmoke --version and --help (no model)
|
|
|
|
Usage: python3 tests/glue_persist.py [--filter NAME] [--timeout SECONDS] (DETZ_BIN overrides build/detz)
|
|
"""
|
|
import glob
|
|
import json
|
|
import os
|
|
import re
|
|
import shlex
|
|
import shutil
|
|
import sqlite3
|
|
import sys
|
|
import tempfile
|
|
import uuid
|
|
from pathlib import Path
|
|
|
|
HERE = Path(__file__).resolve().parent
|
|
TAICPP = HERE.parent
|
|
sys.path.insert(0, str(TAICPP.parent))
|
|
from tai_test import TestCase, TestAssertionFailed, main_cli # noqa: E402
|
|
|
|
DETZ = os.environ.get("DETZ_BIN") or str(TAICPP / "build" / "detz")
|
|
RC_RE = re.compile(r"__RC=\d+\n")
|
|
RC_CAPTURE = re.compile(r"__RC=(\d+)")
|
|
CODEWORD = "ZEBRA-4471"
|
|
|
|
|
|
class PaneCase(TestCase):
|
|
"""A TestCase whose processes are bash shells (panes) with a throwaway TAI_HOME."""
|
|
|
|
timeout = 420.0
|
|
|
|
async def setup(self):
|
|
self.home = tempfile.mkdtemp(prefix="detz-glue-")
|
|
|
|
async def teardown(self):
|
|
shutil.rmtree(self.home, ignore_errors=True)
|
|
|
|
def build_env(self):
|
|
env = {k: v for k, v in os.environ.items() if not k.startswith(("TAI_", "TMUX"))}
|
|
env.update({"TAI_HOME": self.home, "HOME": self.home, "TAI_PASSPHRASE": "", "NO_COLOR": "1"})
|
|
return env
|
|
|
|
def build_command(self, extra_args=()):
|
|
return ["bash", "--norc", "--noprofile"]
|
|
|
|
async def pane_run(self, pane, lines, flags=()):
|
|
"""Feed `lines` on stdin of the binary in this pane; returns (exit code, output)."""
|
|
feed = "".join(line + "\n" for line in lines)
|
|
command = "printf %%s %s | %s %s; echo __RC=$?" % (shlex.quote(feed), shlex.quote(DETZ), " ".join(shlex.quote(f) for f in flags))
|
|
before = len(pane.transcript)
|
|
await pane.send_line(command)
|
|
output = await pane.wait_for_pattern(RC_RE, timeout=self.timeout - 30)
|
|
text = "".join(item.text for item in pane.transcript[before:])
|
|
return int(RC_CAPTURE.findall(text)[-1]), output
|
|
|
|
async def oneshot(self, pane, prompt):
|
|
command = "%s %s; echo __RC=$?" % (shlex.quote(DETZ), shlex.quote(prompt))
|
|
before = len(pane.transcript)
|
|
await pane.send_line(command)
|
|
output = await pane.wait_for_pattern(RC_RE, timeout=self.timeout - 30)
|
|
text = "".join(item.text for item in pane.transcript[before:])
|
|
return int(RC_CAPTURE.findall(text)[-1]), output
|
|
|
|
# ---- vault readers ----
|
|
def sessions(self, profile="default"):
|
|
return sorted(glob.glob(os.path.join(self.home, "profiles", profile + ".session.*.json")))
|
|
|
|
@staticmethod
|
|
def load(path):
|
|
with open(path, "r", encoding="utf-8") as handle:
|
|
return json.load(handle)
|
|
|
|
|
|
class GlueRestartPersistence(PaneCase):
|
|
name = "GlueRestartPersistence"
|
|
|
|
async def run(self):
|
|
pane_a = await self.spawn()
|
|
pane_b = await self.spawn()
|
|
rcs = []
|
|
rcs.append((await self.pane_run(pane_a, ["Remember this codeword: %s. Reply with just OK." % CODEWORD, "/quit"]))[0])
|
|
files_after_first = self.sessions()
|
|
first_snapshot = self.load(files_after_first[0]) if len(files_after_first) == 1 else None
|
|
rcs.append((await self.pane_run(pane_a, ["What codeword did I tell you earlier? Reply with just the codeword.", "/quit"]))[0])
|
|
rcs.append((await self.pane_run(pane_b, ["What codeword did I tell you earlier? If you were told none, reply NONE.", "/quit"]))[0])
|
|
return {"rcs": rcs, "files_after_first": files_after_first, "first_snapshot": first_snapshot}
|
|
|
|
async def verify(self, result):
|
|
problems = []
|
|
if any(rc != 0 for rc in result["rcs"]):
|
|
problems.append("exit codes %s" % result["rcs"])
|
|
if len(result["files_after_first"]) != 1:
|
|
problems.append("first start did not leave exactly one session file: %s" % result["files_after_first"])
|
|
files = self.sessions()
|
|
if len(files) != 2:
|
|
problems.append("expected 2 session files (pane A, pane B), found %s" % files)
|
|
raise TestAssertionFailed("; ".join(problems))
|
|
# The file written by the first start is pane A's; it must still be the same file after the restart.
|
|
pane_a = result["files_after_first"][0]
|
|
if pane_a not in files:
|
|
problems.append("pane A's session file disappeared after the restart")
|
|
pane_b = [path for path in files if path != pane_a][0]
|
|
a = self.load(pane_a)
|
|
b = self.load(pane_b)
|
|
users_a = [m["content"] for m in a if m["role"] == "user"]
|
|
users_b = [m["content"] for m in b if m["role"] == "user"]
|
|
first = result["first_snapshot"] or []
|
|
if not (len(users_a) == 2 and CODEWORD in users_a[0] and "earlier" in users_a[1]):
|
|
problems.append("pane A should hold both turns in order, user messages were %s" % users_a)
|
|
if a[:len(first)] != first:
|
|
problems.append("the restart rewrote the earlier history instead of extending it")
|
|
if not any(m["role"] == "assistant" for m in a[len(first):]):
|
|
problems.append("no assistant message after the restart in pane A")
|
|
if len(users_b) != 1 or any(CODEWORD in text for text in users_b):
|
|
problems.append("pane B must have its own single-turn history, user messages were %s" % users_b)
|
|
# The restored history actually reached the model: its answer in pane A carries the codeword.
|
|
answers_a = [m.get("content") or "" for m in a[len(first):] if m["role"] == "assistant"]
|
|
recalled = any(CODEWORD in text for text in answers_a)
|
|
answers_b = [m.get("content") or "" for m in b if m["role"] == "assistant"]
|
|
leaked = any(CODEWORD in text for text in answers_b)
|
|
if not recalled:
|
|
problems.append("model did not recall %s in pane A (history not delivered or model flake): %s" % (CODEWORD, answers_a))
|
|
if leaked:
|
|
problems.append("pane B's model saw pane A's codeword: %s" % answers_b)
|
|
# turn files closed, input history and shell rows in the vault
|
|
for path in glob.glob(os.path.join(self.home, "profiles", "default.turn.*.json")):
|
|
if self.load(path).get("open"):
|
|
problems.append("turn left open: %s" % path)
|
|
db = sqlite3.connect(os.path.join(self.home, "memory.db"))
|
|
try:
|
|
typed = [row[0] for row in db.execute("SELECT text FROM input_history WHERE profile = 'default' ORDER BY id")]
|
|
sids = {row[0] for row in db.execute("SELECT sid FROM shell_sessions")}
|
|
finally:
|
|
db.close()
|
|
if sum(1 for text in typed if CODEWORD in text) != 1:
|
|
problems.append("input history in the vault is missing the first message: %s" % typed)
|
|
for path in (pane_a, pane_b):
|
|
sid = os.path.basename(path)[len("default.session."):-len(".json")]
|
|
if sid not in sids:
|
|
problems.append("no shell_sessions row for %s" % sid)
|
|
if problems:
|
|
raise TestAssertionFailed("; ".join(problems))
|
|
print("restart persistence ok: pane A %d messages, recalled=%s, pane B %d messages" % (len(a), recalled, len(b)))
|
|
|
|
|
|
class GlueProfileSwitch(PaneCase):
|
|
name = "GlueProfileSwitch"
|
|
|
|
async def run(self):
|
|
pane = await self.spawn()
|
|
rc, output = await self.pane_run(pane, ["Remember this codeword: %s. Reply with just OK." % CODEWORD, "/profile alt",
|
|
"What codeword did I tell you? If none, reply NONE.", "/profile default", "/clear", "/quit"])
|
|
return {"rc": rc}
|
|
|
|
async def verify(self, result):
|
|
problems = []
|
|
default = self.sessions("default")
|
|
alt = self.sessions("alt")
|
|
if len(alt) != 1:
|
|
raise TestAssertionFailed("profile alt should have its own session file, found %s" % alt)
|
|
a = self.load(alt[0])
|
|
users_alt = [m["content"] for m in a if m["role"] == "user"]
|
|
if users_alt != ["What codeword did I tell you? If none, reply NONE."]:
|
|
problems.append("alt profile history should hold only its own turn: %s" % users_alt)
|
|
if any(CODEWORD in (m.get("content") or "") for m in a if m["role"] == "assistant"):
|
|
problems.append("alt profile saw the default profile's codeword")
|
|
# /clear in the default profile emptied the saved history that /profile default had restored
|
|
if default:
|
|
left = [m for m in self.load(default[0]) if m["role"] == "user"]
|
|
if left:
|
|
problems.append("/clear did not reach the vault, default session still has %s" % [m["content"][:30] for m in left])
|
|
if result["rc"] != 0:
|
|
problems.append("exit code %d" % result["rc"])
|
|
if problems:
|
|
raise TestAssertionFailed("; ".join(problems))
|
|
print("profile switch ok: alt has %d messages, default session cleared" % len(a))
|
|
|
|
|
|
class GlueOneShotShell(PaneCase):
|
|
name = "GlueOneShotShell"
|
|
|
|
async def run(self):
|
|
self.marker = "glue-marker-" + uuid.uuid4().hex[:8]
|
|
pane = await self.spawn()
|
|
rc, output = await self.oneshot(pane, "Use the shell tool to run: echo %s and tell me what it printed." % self.marker)
|
|
return {"rc": rc}
|
|
|
|
async def verify(self, result):
|
|
problems = []
|
|
files = self.sessions()
|
|
if len(files) != 1:
|
|
raise TestAssertionFailed("expected one session file, found %s" % files)
|
|
session = self.load(files[0])
|
|
calls = [c for m in session for c in (m.get("tool_calls") or []) if c.get("function", {}).get("name") == "shell"]
|
|
if not any(self.marker in (c["function"].get("arguments") or "") for c in calls):
|
|
problems.append("no shell tool call carrying the marker in the vault session")
|
|
tool_msgs = [m for m in session if m["role"] == "tool" and self.marker in (m.get("content") or "")]
|
|
if not tool_msgs:
|
|
problems.append("no tool result with the marker in the vault session")
|
|
elif not tool_msgs[0]["content"].startswith("exit 0"):
|
|
problems.append("shell result did not start with 'exit 0': %r" % tool_msgs[0]["content"][:80])
|
|
final = [m for m in session if m["role"] == "assistant" and not m.get("tool_calls")]
|
|
if not final or self.marker not in (final[-1].get("content") or ""):
|
|
problems.append("final assistant answer does not carry the marker")
|
|
if result["rc"] != 0:
|
|
problems.append("exit code %d (0 means the goal verifier accepted the answer)" % result["rc"])
|
|
if problems:
|
|
raise TestAssertionFailed("; ".join(problems))
|
|
print("one-shot shell ok: %d shell call(s), session %d messages" % (len(calls), len(session)))
|
|
|
|
|
|
class GlueFlagsSmoke(PaneCase):
|
|
name = "GlueFlagsSmoke"
|
|
timeout = 60.0
|
|
|
|
async def run(self):
|
|
pane = await self.spawn()
|
|
outputs = {}
|
|
for flag in ("--version", "--help", "--cap-manifest"):
|
|
command = "%s %s; echo __RC=$?" % (shlex.quote(DETZ), flag)
|
|
before = len(pane.transcript)
|
|
await pane.send_line(command)
|
|
outputs[flag] = await pane.wait_for_pattern(RC_RE, timeout=30)
|
|
text = "".join(item.text for item in pane.transcript[before:])
|
|
outputs[flag + "_rc"] = int(RC_CAPTURE.findall(text)[-1])
|
|
return outputs
|
|
|
|
async def verify(self, result):
|
|
version = (TAICPP / "VERSION").read_text().strip()
|
|
problems = []
|
|
if result["--version"].strip() != "tai " + version:
|
|
problems.append("--version printed %r, VERSION is %s" % (result["--version"], version))
|
|
if not result["--help"].lstrip().startswith("usage: tai"):
|
|
problems.append("--help does not start with the usage line: %r" % result["--help"][:60])
|
|
if "# tai CAP agent manifest" in result["--cap-manifest"] and len(result["--cap-manifest"]) < 200:
|
|
problems.append("--cap-manifest is still the placeholder")
|
|
if "[agent]" not in result["--cap-manifest"] and "name" not in result["--cap-manifest"]:
|
|
problems.append("--cap-manifest printed no manifest: %r" % result["--cap-manifest"][:80])
|
|
for flag in ("--version", "--help", "--cap-manifest"):
|
|
if result[flag + "_rc"] != 0:
|
|
problems.append("%s exit code %d" % (flag, result[flag + "_rc"]))
|
|
if problems:
|
|
raise TestAssertionFailed("; ".join(problems))
|
|
|
|
|
|
ALL_CASES = [GlueFlagsSmoke(), GlueRestartPersistence(), GlueProfileSwitch(), GlueOneShotShell()]
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main_cli(ALL_CASES, default_log_dir=str(TAICPP / "build" / "test-logs")))
|