tai/taicpp/tests/capmcp_diff.py

#!/usr/bin/env python3
"""Differential test of the C++ wire layer (JSON, TOML, str(), float repr,
regex) against CPython. Builds nothing: pass the compiled probe.

    python3 -I tests/capmcp_diff.py PROBE_BINARY [--seed N] [--fuzz N]

Exit 0 only when every case prints the same line in both programs."""
import json
import os
import random
import re
import subprocess
import sys
import tempfile
import tomllib

JSON_CASES = [
    "", " ", "x", "{", "{\"a\"", "{\"a\":", "{\"a\":1", "{\"a\":1,", "{\"a\":1,}", "[1,]", "[1", "[", "[1 2]",
    "{\"a\" 1}", "{a:1}", "{'a':1}", "\"abc", "\"a\\x\"", "\"a\\u12\"", "\"a\nb\"", "01", "-", "-x", "1.", "1e",
    "1.5e3", "nul", "tru", "NaN", "-Infinity", "Infinity", "{\"a\":1} x", "[] []", "\"\\ud83d\\ude00\"",
    "123456789012345678901234567890", "  {\"a\": [1, 2.5, \"x\", null, true, false]}  ", "{\"a\":1,\"a\":2}",
    "\"\\u00e9\"", "[1,\n2,\n x]", "\ufeff{}", "{\"a\":1 \"b\":2}", "{,}", "[,1]", "{\"a\":}", "\"\\", "\"\\u",
    "\"\\u12", "{\"a\":1,,}", "tru e", "-0", "-0.0", "1E5", "1e400", "-1e400", "\"é\"", "\"日本語\"", "[\"😀\"]",
    "{\"b\":1,\"a\":{\"z\":[],\"y\":{}}}", "0.1", "1e-7", "1e22", "123456789012345680000.0", "5e-324", "1.7976931348623157e308",
    "9223372036854775807", "9223372036854775808", "-9223372036854775808", "-9223372036854775809",
    "\"\\/\\b\\f\\n\\r\\t\\\"\\\\\"", "\"\x7f\"", "\"\\u007f\"", "\"\\u2028\"", "{\"a\":\n\n  [\n1\n,\n2] }",
]

TOML_CASES = [
    "", "a = 1", "a = 1\nb = 2.5\nc = \"x\"\nd = true\ne = [1, 2, 3]\nf = { g = 1, h = \"i\" }",
    "[t]\na = 1\n[t.u]\nb = 2", "[[s]]\nname = \"a\"\n[[s]]\nname = \"b\"",
    "a = ", "a", "= 1", "a = 1 b = 2", "a = 1\na = 2", "[t]\n[t]", "a = [1, 2", "a = {x = 1", "a = \"abc",
    "a = 'abc'\nb = '''multi\nline'''\nc = \"\"\"x\ny\"\"\"", "a = \"\"\"line \\\n   continued\"\"\"",
    "a = 0x1F\nb = 0o17\nc = 0b101\nd = 1_000\ne = +5\nf = -7", "a = 1e3\nb = 1.5e-3\nc = inf\nd = -inf\ne = nan",
    "a = 01", "a = 1__0", "a = 0x", "a = .5", "a = 5.", "a = 1979-05-27", "a = 1979-05-27T07:32:00Z",
    "a = 1979-05-27 07:32:00.5-08:00", "a = 07:32:00", "a = 1979-02-30", "a = 2024-02-29",
    "a.b.c = 1\na.b.d = 2", "a.b = 1\n[a]\nc = 2", "[a]\nb = 1\n[a.b]", "[a.b]\nc = 1\n[a]\nd = 2",
    "a = [[1], [2, 3], []]", "a = [\n  1, # one\n  2,\n]\n", "a = { b = 1, }", "a = [ ,]",
    "# comment only\n", "a = 1 # trailing\n", "a = \"x\" y", "\"quoted key\" = 1\n'lit key' = 2",
    "a = \"\\u00e9\\U0001F600\"", "a = \"\\x\"", "a = \"\\ud800\"", "a = \"tab\there\"", "a = \"nl\nx\"",
    "[ a . b ]\nc = 1", "[a]\nb = 1\n[[a.c]]\nd = 1\n[[a.c]]\nd = 2", "x = {a.b = 1, a.c = 2}",
    "x = {a = 1}\n[x.b]", "a = [1, \"two\", 3.0]", "[a]\n[a.b]\n[a]", "a = 1\n[a]", "[a]\nb = 1\n[a]\nc = 2",
    "\r\na = 1\r\nb = 2\r\n", "a = 1\rb = 2", "a = \"\"\"\"x\"\"\"\"", "a = '''x'''''", "a=\"\"\"\\  \n  x\"\"\"",
    "a = tru", "a = True", "[]", "[a", "[[a]", "[[a]]\n[a]", "[a]\n[[a]]", "a = [\n", "key = 'unterminated",
    "bad key = 1", "a = \"\x01\"", "# \x01", "a = 1_000.000_1", "a = 1e+3_0", "a = 0e0", "a = -0.0",
    "a = 123456789012345678", "a = 9223372036854775807", "a = 9223372036854775808",
    "[agent]\nname = \"tai\"\nprofiles = [\"coding\"]\n[probe]\ncommand = [\"tai.py\", \"--version\"]\nversion_pattern = \"tai (\\\\d+)\"",
]

REGEX_CASES = [
    ("^tai\\[.*\\]› $", "tai[main]› "), ("^tai\\[.*\\]› $", "tai[main]›"), ("^>\\s*$", ">  "), ("^❯\\s*$", "❯"),
    ("(?P<tool>\\w+)\\((?P<args>.*)\\)", "run(ls -l)"), ("(?P<status>ok|failed)?x", "x"),
    ("github\\.com/([^/\\s?#]+)/([^/\\s?#]+?)(?:\\.git)?(?:[/?#]|$)", "https://github.com/o/r.git"),
    ("github\\.com/([^/\\s?#]+)/([^/\\s?#]+?)(?:\\.git)?(?:[/?#]|$)", "https://github.com/o/r/tree/x"),
    ("(npx|uvx)\\s+([^\\s\"'`]+)", "run: npx -y pkg-name now"), ("@(cap://[A-Za-z0-9_.-]+)", "hi @cap://a-1 and @cap://b"),
    ("\\$\\{([A-Za-z_][A-Za-z0-9_]*)(?::-([^}]*))?\\}", "x ${FOO:-bar} y"),
    ("\\$\\{([A-Za-z_][A-Za-z0-9_]*)(?::-([^}]*))?\\}", "x ${FOO} y"),
    ("resource_metadata=\"([^\"]+)\"", "Bearer resource_metadata=\"https://x/y\", realm=x"),
    ("\\bno\\b", "say no way"), ("\\byes\\b", "yes."), ("seal error:", "seal error: boom"),
    ("^seal error:", "x\nseal error:"), ("end$", "the end\n"), ("end$", "the end\nmore"), ("a{2}", "caab aab"),
    ("a{,2}b", "aab"), ("x{", "x{"), ("}", "a}"), ("[]a]+", "]]a"), ("[^]a]+", "bc]"), ("\\d+\\.\\d+", "v 1.44.2"),
    ("tai (\\d+\\.\\d+\\.\\d+)", "tai 1.44.2"), ("(a)|(b)", "b"), ("é+", "ééé"), ("\\w+", "日本語 x"),
    ("(?:ab)+", "ababab"), ("a(?=b)", "ab"), ("a(?!b)", "ac"), ("\\Aab", "ab"), ("ab\\Z", "ab"),
]

REPR_CASES = [
    "{\"a\":\"it's\",\"b\":[1,null,true,1.5,\"x\\n\"]}", "[\"a\\\"b'c\"]", "[\"\\u00e9\\u2028\\u00a0\\u0007\"]",
    "{}", "[]", "[1e22, 1e-7, 0.1, 100.0, 1e16, 123456.789]", "\"plain\"", "null", "[true,false]", "{\"k\":{\"n\":[{}]}}",
    "\"日本語\"", "\"\\ud83d\\ude00\"", "[NaN, Infinity, -Infinity]", "\"\\\\\"",
]

FLOAT_CASES = ["0", "1", "0.1", "1e22", "1e-7", "100", "1e16", "1e15", "123456789.123456789", "5e-324", "1.7976931348623157e308",
               "0.0001", "0.00001", "12345678901234567890", "-2.5", "3.14159", "1e300", "2.5e-5", "0.30000000000000004"]


def write_cases(path, cases):
    with open(path, "wb") as handle:
        handle.write(b"\0".join(c.encode("utf-8", "surrogatepass") for c in cases))


def py_json(i, text):
    try:
        return "%d OK %s" % (i, json.dumps(json.loads(text)))
    except ValueError as exc:
        return "%d ERR %s" % (i, exc)


def py_toml(i, text):
    try:
        return "%d OK %s" % (i, json.dumps(tomllib.loads(text), default=str))
    except ValueError as exc:
        return "%d ERR %s" % (i, exc)


def py_repr(i, text):
    try:
        return "%d OK %s" % (i, repr(json.loads(text)))
    except ValueError as exc:
        return "%d ERR %s" % (i, exc)


def py_float(i, text):
    return "%d OK %s" % (i, repr(float(text)))


def py_regex(i, text):
    pattern, subject = text.split("\x01", 1)
    try:
        found = re.search(pattern, subject)
    except re.error as exc:
        return "%d ERR %s" % (i, exc)
    if not found:
        return "%d NONE" % i
    groups = [found.group(0)] + list(found.groups())
    return "%d OK %d %d %s %s" % (i, found.start(), found.end(), json.dumps(groups), json.dumps(found.groupdict()))


def fuzz_json(rng, count):
    out = []
    atoms = ["null", "true", "false", "0", "-1", "12", "3.5", "-2.5e3", "\"s\"", "\"\\u00e9\"", "\"a b\"", "[]", "{}", "NaN", "\"\\n\""]
    for _ in range(count):
        depth = rng.randint(0, 3)

        def build(d):
            if d == 0:
                return rng.choice(atoms)
            if rng.random() < 0.5:
                return "[" + ", ".join(build(d - 1) for _ in range(rng.randint(0, 3))) + "]"
            return "{" + ", ".join("\"k%d\": %s" % (rng.randint(0, 5), build(d - 1)) for _ in range(rng.randint(0, 3))) + "}"

        text = build(depth)
        out.append(text)
        if text:
            cut = rng.randint(0, len(text))
            out.append(text[:cut])
            pos = rng.randint(0, len(text) - 1) if len(text) > 1 else 0
            out.append(text[:pos] + rng.choice([",", "]", "}", "\"", "\\", " ", "x", ":"]) + text[pos + 1:])
    return out


def run(probe, mode, cases, pyfn, label):
    with tempfile.TemporaryDirectory() as tmp:
        path = os.path.join(tmp, "cases.bin")
        write_cases(path, cases)
        done = subprocess.run([probe, mode, path], stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=120)
    got = done.stdout.decode("utf-8", "replace").splitlines()
    want = [pyfn(i, c) for i, c in enumerate(cases)]
    bad = 0
    for i, (g, w) in enumerate(zip(got, want)):
        if g != w:
            bad += 1
            if bad <= 8:
                print("MISMATCH %s case %d %r\n  cpp: %s\n  py : %s" % (label, i, cases[i][:120], g[:300], w[:300]))
    if len(got) != len(want):
        bad += 1
        print("MISMATCH %s: %d lines from probe, %d cases (stderr: %s)" % (label, len(got), len(want), done.stderr.decode()[:300]))
    print("%-8s %4d cases, %d mismatches" % (label, len(cases), bad))
    return bad


def main():
    args = sys.argv[1:]
    if not args:
        print(__doc__)
        return 2
    probe = args[0]
    seed = int(args[args.index("--seed") + 1]) if "--seed" in args else 7
    fuzz = int(args[args.index("--fuzz") + 1]) if "--fuzz" in args else 300
    rng = random.Random(seed)
    bad = 0
    jcases = JSON_CASES + fuzz_json(rng, fuzz)
    bad += run(probe, "json", jcases, py_json, "json")
    bad += run(probe, "repr", REPR_CASES, py_repr, "repr")
    bad += run(probe, "float", FLOAT_CASES, py_float, "float")
    bad += run(probe, "toml", TOML_CASES, py_toml, "toml")
    bad += run(probe, "regex", ["%s\x01%s" % c for c in REGEX_CASES], py_regex, "regex")
    print("TOTAL mismatches: %d" % bad)
    return 1 if bad else 0


if __name__ == "__main__":
    sys.exit(main())