87 lines
3.5 KiB
Python
87 lines
3.5 KiB
Python
#!/usr/bin/env python3
|
|
"""Parser parity: tai.py build_parser() (in process) versus build/cli/cli_driver.
|
|
|
|
Compares stdout, stderr and exit code for every argv in tests/cli_cases.txt (one shlex line
|
|
each) at COLUMNS=80 and 40, and --help plus the usage printed by an error for every terminal
|
|
width from 12 to 200. The Python side is the real argparse of the installed interpreter.
|
|
Usage: tests/cli_parity.py --driver build/cli/cli_driver [--verbose]
|
|
"""
|
|
import argparse
|
|
import contextlib
|
|
import io
|
|
import json
|
|
import os
|
|
import random
|
|
import shlex
|
|
import subprocess
|
|
import sys
|
|
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
ROOT = os.path.dirname(HERE)
|
|
sys.path.insert(0, os.path.dirname(ROOT))
|
|
import tai # noqa: E402 (the tai.py next to taicpp/)
|
|
|
|
|
|
def reference(argv, columns):
|
|
os.environ["COLUMNS"] = str(columns)
|
|
out, err = io.StringIO(), io.StringIO()
|
|
code, text = 0, ""
|
|
try:
|
|
with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
|
|
ns = tai.build_parser().parse_args(argv)
|
|
keep = {}
|
|
for key, value in vars(ns).items():
|
|
if value is None or value is False or (key == "prompt" and value == []):
|
|
continue
|
|
keep[key] = value
|
|
text = json.dumps(keep, sort_keys=True) + "\n"
|
|
except SystemExit as exc:
|
|
code = exc.code or 0
|
|
return out.getvalue() + text, err.getvalue(), code
|
|
|
|
|
|
def candidate(driver, argv, columns):
|
|
env = dict(os.environ, COLUMNS=str(columns))
|
|
p = subprocess.run([driver] + argv, capture_output=True, text=True, env=env, timeout=30)
|
|
return p.stdout, p.stderr, p.returncode
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--driver", default=os.path.join(ROOT, "build", "cli", "cli_driver"))
|
|
ap.add_argument("--verbose", action="store_true")
|
|
ap.add_argument("--fuzz", type=int, default=1500, help="random argv count (seeded, deterministic)")
|
|
args = ap.parse_args()
|
|
cases = []
|
|
with open(os.path.join(HERE, "cli_cases.txt"), encoding="utf-8") as fh:
|
|
for line in fh.read().split("\n"):
|
|
if line.strip():
|
|
cases.append(shlex.split(line))
|
|
jobs = [(a, c) for a in cases for c in (80, 40)]
|
|
for c in range(12, 201):
|
|
jobs.append((["--help"], c))
|
|
jobs.append((["--bogus"], c))
|
|
rng = random.Random(20260101)
|
|
vocab = ["--profile", "--yes", "--debug", "--debug-format", "--cap", "--cap-m", "--cap-max-turns", "--cap-permission-mode",
|
|
"--mcp-port", "--mplex", "--ins", "--tui", "--tui-", "--vers", "--version", "-h", "-hx", "-z", "--", "-", "-3",
|
|
"-3x", "--x=1", "--yes=1", "--debug=stdout", "--profile=p", "a", "b c", "file", "stdout", "json", "none", "7",
|
|
"abc", "", "--theme", "--env", "-1.5", "--y", "--u"]
|
|
for _ in range(max(args.fuzz, 0)):
|
|
jobs.append(([rng.choice(vocab) for _ in range(rng.randint(1, 5))], rng.choice((80, 50, 100))))
|
|
bad = 0
|
|
for argv, columns in jobs:
|
|
want, got = reference(argv, columns), candidate(args.driver, argv, columns)
|
|
if want != got:
|
|
bad += 1
|
|
print("FAIL argv=%r COLUMNS=%d" % (argv, columns))
|
|
if args.verbose or bad <= 3:
|
|
for name, w, g in zip(("stdout", "stderr", "exit"), want, got):
|
|
if w != g:
|
|
print(" %s\n --- python\n%s\n --- detz\n%s" % (name, w, g))
|
|
print("cli parity: %d jobs, %d failed" % (len(jobs), bad))
|
|
return 1 if bad else 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|