tai/tests/container.py

#!/usr/bin/env python3
"""tests/container.py — run the full test suite inside a throwaway container.

Two things can only be tested honestly inside a disposable environment, not
on the machine actually running Claude Code: installing the plexer hook into
a real ~/.bashrc and watching a real `bash -i` actually get wrapped by it
(tests/bashrc.py), and driving TmuxPlexerController against a real tmux
session. This is the harness for that -- not a replacement for test_seal.py,
test_tai.py, or tests/plexer.py, which also run inside the same container
first, so "did it work in a clean environment" is answered for everything,
not just the two new container-only tests. tests/rag.py runs too, but as a
best-effort check, not a required one -- see BEST_EFFORT_SUITE below for why.

Engine detection deliberately reuses tai.container_engine() (podman first,
then docker, same as the project's own sandbox feature) rather than
reimplementing "which one is installed" a second way. docker-compose.yml at
the repo root exists for a human who wants `docker compose run --rm
tai-test` / `podman compose run --rm tai-test`; this script drives plain
build/run directly so the automated path never depends on whether a compose
implementation happens to be installed on top of the engine itself.

Usage:
    python3 tests/container.py              # build the image, run everything inside it
    python3 tests/container.py --inside      # (used by the container itself) actually run the suite
"""
import argparse
import os
import subprocess
import sys
from pathlib import Path

ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT))
import tai  # noqa: E402

IMAGE_TAG = "tai-test:latest"

# Run in this order: cheapest/most-fundamental first, so a basic break fails
# fast instead of burning time on tmux/bashrc tests that assume the basics
# already work. These must all pass for the container run to count as green.
REQUIRED_SUITE = [
    ("test_seal.py", ["python3", "test_seal.py"]),
    ("test_tai.py", ["python3", "-m", "unittest", "test_tai", "-v"]),
    ("tests/plexer.py", ["python3", "tests/plexer.py"]),
    ("tests/bashrc.py", ["python3", "tests/bashrc.py"]),
]

# tests/rag.py's RagIndexAndSearchTest needs a real, working LLM backend.
# Confirmed by direct investigation (not assumed): inside this disposable
# container there is no opencode/claude/codex/gemini/grok CLI (the host's
# own successful run used a locally authenticated `opencode` CLI that has
# no business being baked into a throwaway test image), and the remaining
# free HTTP backends fail for reasons that are about the shared sandbox
# network, not this code -- openrouter's free tier returned "HTTP 401: No
# cookie auth credentials found" for every model (anonymous free-tier auth
# needs a browser cookie session this container doesn't have), and
# pollinations returned "HTTP 429: Queue full for IP: ...: 1 requests
# already queued" (its free tier's per-IP concurrency cap, already
# exhausted by other traffic sharing this sandbox's egress address). Both
# reproduced with a bare urllib POST outside of tai.py entirely, so this is
# not a tai.py bug to fix. It's run and reported like everything else, but
# never fails the overall container result -- tests/rag.py already runs
# (and passes) against the host directly, which is where a real LLM
# backend actually exists; see tests/README.md's "test with the real AI,
# accept the flakes" rule, which this is really just a container-flavored
# instance of.
BEST_EFFORT_SUITE = [
    ("tests/rag.py", ["python3", "tests/rag.py"]),
]


def run_outside():
    engine = tai.container_engine()
    if not engine:
        print("no container engine found (checked podman, then docker) -- refusing to run the "
              "container-only test suite. Install either one to run tests/bashrc.py and the "
              "TmuxPlexerController live test; everything else already ran on the host directly.")
        return 1

    print("using container engine: %s" % engine, flush=True)
    build = subprocess.run([engine, "build", "-t", IMAGE_TAG, "-f", "tests/docker/Dockerfile", "."], cwd=str(ROOT))
    if build.returncode != 0:
        print("image build failed", flush=True)
        return build.returncode

    run = subprocess.run([engine, "run", "--rm", IMAGE_TAG, "python3", "tests/container.py", "--inside"], cwd=str(ROOT))
    return run.returncode


def _run_suite(suite):
    results = []
    for name, cmd in suite:
        print("\n=== %s ===" % name)
        done = subprocess.run(cmd, cwd=str(ROOT))
        results.append((name, done.returncode == 0))
        print("--- %s: %s ---" % (name, "PASS" if done.returncode == 0 else "FAIL"))
    return results


def run_inside():
    required = _run_suite(REQUIRED_SUITE)
    best_effort = _run_suite(BEST_EFFORT_SUITE)

    print("\n=== container test summary ===")
    for name, passed in required:
        print("%-6s %s" % ("PASS" if passed else "FAIL", name))
    for name, passed in best_effort:
        print("%-6s %s  (best-effort: needs a real LLM backend, see comment in this file)" % ("PASS" if passed else "FAIL", name))

    failed_required = [name for name, passed in required if not passed]
    failed_best_effort = [name for name, passed in best_effort if not passed]
    total = len(required) + len(best_effort)
    passed_count = total - len(failed_required) - len(failed_best_effort)
    print("%d/%d passed" % (passed_count, total), end="")
    if failed_best_effort:
        print(" (ignoring best-effort failures: %s)" % ", ".join(failed_best_effort))
    else:
        print("")

    if failed_required:
        print("REQUIRED tests failed: %s" % ", ".join(failed_required))
        return 1
    return 0


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--inside", action="store_true", help="internal: actually run the suite (used by the container itself)")
    args = parser.parse_args()
    if args.inside:
        return run_inside()
    return run_outside()


if __name__ == "__main__":
    sys.exit(main())