Strict goal verification, approval hardening, retrieval eval, work limits, sandbox caps (1.38.0)
- Approval: re-prompt unknown answers (max 3), shell-only affirmative guidance recovery; destructive actions stay strict - GoalVerifier subagent with strict JSON verdicts, bounded follow-up rounds, nonzero exit plus open turn on unverified one-shots - Retrieval eval (Hit@K/MRR/nDCG/Context Precision), FTS cascade with title weighting, query/read event logging, --eval-retrieval CLI - Supervisor caps (8 parallel, 64 records) and scheduler hardening (4 fires/tick, per-schedule lease, stated misfire policy) - Sandbox box resource trio: 2g/2.0/256 at create, best-effort update for existing boxes, rm -f recovery pointer - Research archive: 12 subjects, 667 sources, 27 conclusions; suites 979 + 16 green
This commit is contained in:
+109
-2
@@ -11,7 +11,8 @@ the whole point of this file is that it exercises the actual
|
||||
testing a pty wrapper by driving it through another pty), not mocked
|
||||
termios/signal calls. The multiplexer's shortcuts and config live in tests/mplex.py;
|
||||
this file covers what the agent tools rely on: the control socket, per-pane
|
||||
targeting, file permissions and the session log.
|
||||
targeting, file permissions, the session log, and the UI flow dialogs
|
||||
(show/result/cancel) with a real attached client answering real keystrokes.
|
||||
|
||||
A real bug was caught by actually running the original version of this test, not
|
||||
by reading the code: an external SIGTERM (and SIGHUP) does not terminate an
|
||||
@@ -380,7 +381,113 @@ class PlexerPaneTargetingTest(TestCase):
|
||||
shutil.rmtree(self.tmp_home, ignore_errors=True)
|
||||
|
||||
|
||||
ALL_CASES = [NativePlexerBackendTest(), NativePlexerBackendRobustnessTest(), PlexerPaneTargetingTest()]
|
||||
class PlexerFlowNativeTest(TestCase):
|
||||
"""The UI flow path end to end over a real server and a real attached
|
||||
client: show renders a dialog, keystrokes written to the pty master answer
|
||||
it, result collects the answer. Deterministic plumbing like the rest of
|
||||
this file: no model anywhere, the mechanics are what's under test."""
|
||||
|
||||
name = "plexer-flow-native"
|
||||
mode = "pty"
|
||||
timeout = 60.0
|
||||
|
||||
async def setup(self):
|
||||
self.tmp_home = tempfile.mkdtemp(prefix="tai-test-home-")
|
||||
self.env_overrides = {"TAI_HOME": self.tmp_home, "TAI_PASSPHRASE": "", "TERM": "xterm-256color", "SHELL": "/bin/bash"}
|
||||
|
||||
async def collect(self, controller, flow_id, budget=4.0):
|
||||
for _ in range(int(budget / 0.1)):
|
||||
reply = controller._call({"cmd": "result", "id": flow_id})
|
||||
if reply.get("done"):
|
||||
return reply
|
||||
await asyncio.sleep(0.1)
|
||||
raise TestAssertionFailed("flow %r never settled" % (flow_id,))
|
||||
|
||||
async def run(self):
|
||||
proc = await self.spawn(extra_args=["--plexer-wrap"])
|
||||
self._drain_task = asyncio.ensure_future(_drain_forever(proc))
|
||||
sock_path = os.path.join(self.tmp_home, "plexer", tai.MPLEX_SOCKET_NAME)
|
||||
for _ in range(50):
|
||||
if os.path.exists(sock_path):
|
||||
break
|
||||
await asyncio.sleep(0.2)
|
||||
else:
|
||||
raise TestAssertionFailed("the mplex server never created its control socket")
|
||||
controller = tai.NativePlexerController(sock_path)
|
||||
await asyncio.sleep(0.8)
|
||||
keys = tai.MplexKeys.to_bytes
|
||||
|
||||
shown = controller._call({"cmd": "show", "screen": {"kind": "choice", "title": "T", "text": "Pick one",
|
||||
"options": ["alpha", "beta"]}, "timeout": 10})
|
||||
pending = controller._call({"cmd": "result", "id": shown["id"]})
|
||||
await asyncio.sleep(0.3)
|
||||
os.write(proc._master_fd, keys("Down"))
|
||||
await asyncio.sleep(0.3)
|
||||
os.write(proc._master_fd, keys("Enter"))
|
||||
choice = await self.collect(controller, shown["id"])
|
||||
|
||||
shown_text = controller._call({"cmd": "show", "screen": {"kind": "text", "title": "T", "text": "Name?"}, "timeout": 10})
|
||||
await asyncio.sleep(0.3)
|
||||
os.write(proc._master_fd, b"hi" + keys("Enter"))
|
||||
text = await self.collect(controller, shown_text["id"])
|
||||
|
||||
shown_msg = controller._call({"cmd": "show", "screen": {"kind": "message", "title": "", "text": "hi"}, "timeout": 10})
|
||||
await asyncio.sleep(0.3)
|
||||
os.write(proc._master_fd, b"x")
|
||||
message = await self.collect(controller, shown_msg["id"])
|
||||
|
||||
shown_cancel = controller._call({"cmd": "show", "screen": {"kind": "message", "title": "", "text": "hi"}, "timeout": 10})
|
||||
await asyncio.sleep(0.3)
|
||||
os.write(proc._master_fd, keys("Escape"))
|
||||
cancelled = await self.collect(controller, shown_cancel["id"])
|
||||
|
||||
stuck = controller._call({"cmd": "show", "screen": {"kind": "message", "title": "", "text": "stuck"}, "timeout": 10})
|
||||
busy = None
|
||||
try:
|
||||
controller._call({"cmd": "show", "screen": {"kind": "message", "title": "", "text": "second"}, "timeout": 10})
|
||||
except tai.PlexerError as exc:
|
||||
busy = str(exc)
|
||||
controller._call({"cmd": "cancel", "id": stuck["id"]})
|
||||
dropped = None
|
||||
try:
|
||||
controller._call({"cmd": "result", "id": stuck["id"]})
|
||||
except tai.PlexerError as exc:
|
||||
dropped = str(exc)
|
||||
|
||||
asked = asyncio.ensure_future(asyncio.to_thread(
|
||||
controller.ask, {"kind": "choice", "title": "", "text": "P", "options": ["a"]}, 10))
|
||||
await asyncio.sleep(0.5)
|
||||
os.write(proc._master_fd, keys("Enter"))
|
||||
ask_outcome = await asyncio.wait_for(asked, timeout=10.0)
|
||||
|
||||
return {"pending": pending, "choice": choice, "text": text, "message": message, "cancelled": cancelled,
|
||||
"busy": busy, "dropped": dropped, "ask_outcome": ask_outcome}
|
||||
|
||||
async def verify(self, result):
|
||||
if result["pending"].get("done") is not False:
|
||||
raise TestAssertionFailed("a fresh show must poll as not done: %r" % result["pending"])
|
||||
if result["choice"] != {"done": True, "status": "ok", "value": 1}:
|
||||
raise TestAssertionFailed("Down+Enter on [alpha, beta] must answer index 1: %r" % result["choice"])
|
||||
if result["text"] != {"done": True, "status": "ok", "value": "hi"}:
|
||||
raise TestAssertionFailed("typing hi+Enter must answer 'hi': %r" % result["text"])
|
||||
if result["message"] != {"done": True, "status": "ok", "value": True}:
|
||||
raise TestAssertionFailed("any key must ack a message: %r" % result["message"])
|
||||
if result["cancelled"].get("status") != "cancelled":
|
||||
raise TestAssertionFailed("Escape must cancel the dialog: %r" % result["cancelled"])
|
||||
if not result["busy"] or "busy" not in result["busy"]:
|
||||
raise TestAssertionFailed("a second show on a busy client must refuse loudly: %r" % result["busy"])
|
||||
if not result["dropped"] or "unknown flow" not in result["dropped"]:
|
||||
raise TestAssertionFailed("result after cancel must report an unknown flow: %r" % result["dropped"])
|
||||
if result["ask_outcome"] != {"status": "ok", "value": 0}:
|
||||
raise TestAssertionFailed("controller.ask must poll to the same answer: %r" % result["ask_outcome"])
|
||||
|
||||
async def teardown(self):
|
||||
self._drain_task.cancel()
|
||||
_mplex_cli(self, "kill-server")
|
||||
shutil.rmtree(self.tmp_home, ignore_errors=True)
|
||||
|
||||
|
||||
ALL_CASES = [NativePlexerBackendTest(), NativePlexerBackendRobustnessTest(), PlexerPaneTargetingTest(), PlexerFlowNativeTest()]
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main_cli(ALL_CASES, default_log_dir=str(Path(__file__).resolve().parent / "logs")))
|
||||
|
||||
Reference in New Issue
Block a user