#!/usr/bin/env python3 """Differential test of the C++ wire layer (JSON, TOML, str(), float repr, regex) against CPython. Builds nothing: pass the compiled probe. python3 -I tests/capmcp_diff.py PROBE_BINARY [--seed N] [--fuzz N] Exit 0 only when every case prints the same line in both programs.""" import json import os import random import re import subprocess import sys import tempfile import tomllib JSON_CASES = [ "", " ", "x", "{", "{\"a\"", "{\"a\":", "{\"a\":1", "{\"a\":1,", "{\"a\":1,}", "[1,]", "[1", "[", "[1 2]", "{\"a\" 1}", "{a:1}", "{'a':1}", "\"abc", "\"a\\x\"", "\"a\\u12\"", "\"a\nb\"", "01", "-", "-x", "1.", "1e", "1.5e3", "nul", "tru", "NaN", "-Infinity", "Infinity", "{\"a\":1} x", "[] []", "\"\\ud83d\\ude00\"", "123456789012345678901234567890", " {\"a\": [1, 2.5, \"x\", null, true, false]} ", "{\"a\":1,\"a\":2}", "\"\\u00e9\"", "[1,\n2,\n x]", "\ufeff{}", "{\"a\":1 \"b\":2}", "{,}", "[,1]", "{\"a\":}", "\"\\", "\"\\u", "\"\\u12", "{\"a\":1,,}", "tru e", "-0", "-0.0", "1E5", "1e400", "-1e400", "\"é\"", "\"日本語\"", "[\"😀\"]", "{\"b\":1,\"a\":{\"z\":[],\"y\":{}}}", "0.1", "1e-7", "1e22", "123456789012345680000.0", "5e-324", "1.7976931348623157e308", "9223372036854775807", "9223372036854775808", "-9223372036854775808", "-9223372036854775809", "\"\\/\\b\\f\\n\\r\\t\\\"\\\\\"", "\"\x7f\"", "\"\\u007f\"", "\"\\u2028\"", "{\"a\":\n\n [\n1\n,\n2] }", ] TOML_CASES = [ "", "a = 1", "a = 1\nb = 2.5\nc = \"x\"\nd = true\ne = [1, 2, 3]\nf = { g = 1, h = \"i\" }", "[t]\na = 1\n[t.u]\nb = 2", "[[s]]\nname = \"a\"\n[[s]]\nname = \"b\"", "a = ", "a", "= 1", "a = 1 b = 2", "a = 1\na = 2", "[t]\n[t]", "a = [1, 2", "a = {x = 1", "a = \"abc", "a = 'abc'\nb = '''multi\nline'''\nc = \"\"\"x\ny\"\"\"", "a = \"\"\"line \\\n continued\"\"\"", "a = 0x1F\nb = 0o17\nc = 0b101\nd = 1_000\ne = +5\nf = -7", "a = 1e3\nb = 1.5e-3\nc = inf\nd = -inf\ne = nan", "a = 01", "a = 1__0", "a = 0x", "a = .5", "a = 5.", "a = 1979-05-27", "a = 1979-05-27T07:32:00Z", "a = 1979-05-27 07:32:00.5-08:00", "a = 07:32:00", "a = 1979-02-30", "a = 2024-02-29", "a.b.c = 1\na.b.d = 2", "a.b = 1\n[a]\nc = 2", "[a]\nb = 1\n[a.b]", "[a.b]\nc = 1\n[a]\nd = 2", "a = [[1], [2, 3], []]", "a = [\n 1, # one\n 2,\n]\n", "a = { b = 1, }", "a = [ ,]", "# comment only\n", "a = 1 # trailing\n", "a = \"x\" y", "\"quoted key\" = 1\n'lit key' = 2", "a = \"\\u00e9\\U0001F600\"", "a = \"\\x\"", "a = \"\\ud800\"", "a = \"tab\there\"", "a = \"nl\nx\"", "[ a . b ]\nc = 1", "[a]\nb = 1\n[[a.c]]\nd = 1\n[[a.c]]\nd = 2", "x = {a.b = 1, a.c = 2}", "x = {a = 1}\n[x.b]", "a = [1, \"two\", 3.0]", "[a]\n[a.b]\n[a]", "a = 1\n[a]", "[a]\nb = 1\n[a]\nc = 2", "\r\na = 1\r\nb = 2\r\n", "a = 1\rb = 2", "a = \"\"\"\"x\"\"\"\"", "a = '''x'''''", "a=\"\"\"\\ \n x\"\"\"", "a = tru", "a = True", "[]", "[a", "[[a]", "[[a]]\n[a]", "[a]\n[[a]]", "a = [\n", "key = 'unterminated", "bad key = 1", "a = \"\x01\"", "# \x01", "a = 1_000.000_1", "a = 1e+3_0", "a = 0e0", "a = -0.0", "a = 123456789012345678", "a = 9223372036854775807", "a = 9223372036854775808", "[agent]\nname = \"tai\"\nprofiles = [\"coding\"]\n[probe]\ncommand = [\"tai.py\", \"--version\"]\nversion_pattern = \"tai (\\\\d+)\"", ] REGEX_CASES = [ ("^tai\\[.*\\]› $", "tai[main]› "), ("^tai\\[.*\\]› $", "tai[main]›"), ("^>\\s*$", "> "), ("^❯\\s*$", "❯"), ("(?P\\w+)\\((?P.*)\\)", "run(ls -l)"), ("(?Pok|failed)?x", "x"), ("github\\.com/([^/\\s?#]+)/([^/\\s?#]+?)(?:\\.git)?(?:[/?#]|$)", "https://github.com/o/r.git"), ("github\\.com/([^/\\s?#]+)/([^/\\s?#]+?)(?:\\.git)?(?:[/?#]|$)", "https://github.com/o/r/tree/x"), ("(npx|uvx)\\s+([^\\s\"'`]+)", "run: npx -y pkg-name now"), ("@(cap://[A-Za-z0-9_.-]+)", "hi @cap://a-1 and @cap://b"), ("\\$\\{([A-Za-z_][A-Za-z0-9_]*)(?::-([^}]*))?\\}", "x ${FOO:-bar} y"), ("\\$\\{([A-Za-z_][A-Za-z0-9_]*)(?::-([^}]*))?\\}", "x ${FOO} y"), ("resource_metadata=\"([^\"]+)\"", "Bearer resource_metadata=\"https://x/y\", realm=x"), ("\\bno\\b", "say no way"), ("\\byes\\b", "yes."), ("seal error:", "seal error: boom"), ("^seal error:", "x\nseal error:"), ("end$", "the end\n"), ("end$", "the end\nmore"), ("a{2}", "caab aab"), ("a{,2}b", "aab"), ("x{", "x{"), ("}", "a}"), ("[]a]+", "]]a"), ("[^]a]+", "bc]"), ("\\d+\\.\\d+", "v 1.44.2"), ("tai (\\d+\\.\\d+\\.\\d+)", "tai 1.44.2"), ("(a)|(b)", "b"), ("é+", "ééé"), ("\\w+", "日本語 x"), ("(?:ab)+", "ababab"), ("a(?=b)", "ab"), ("a(?!b)", "ac"), ("\\Aab", "ab"), ("ab\\Z", "ab"), ] REPR_CASES = [ "{\"a\":\"it's\",\"b\":[1,null,true,1.5,\"x\\n\"]}", "[\"a\\\"b'c\"]", "[\"\\u00e9\\u2028\\u00a0\\u0007\"]", "{}", "[]", "[1e22, 1e-7, 0.1, 100.0, 1e16, 123456.789]", "\"plain\"", "null", "[true,false]", "{\"k\":{\"n\":[{}]}}", "\"日本語\"", "\"\\ud83d\\ude00\"", "[NaN, Infinity, -Infinity]", "\"\\\\\"", ] FLOAT_CASES = ["0", "1", "0.1", "1e22", "1e-7", "100", "1e16", "1e15", "123456789.123456789", "5e-324", "1.7976931348623157e308", "0.0001", "0.00001", "12345678901234567890", "-2.5", "3.14159", "1e300", "2.5e-5", "0.30000000000000004"] def write_cases(path, cases): with open(path, "wb") as handle: handle.write(b"\0".join(c.encode("utf-8", "surrogatepass") for c in cases)) def py_json(i, text): try: return "%d OK %s" % (i, json.dumps(json.loads(text))) except ValueError as exc: return "%d ERR %s" % (i, exc) def py_toml(i, text): try: return "%d OK %s" % (i, json.dumps(tomllib.loads(text), default=str)) except ValueError as exc: return "%d ERR %s" % (i, exc) def py_repr(i, text): try: return "%d OK %s" % (i, repr(json.loads(text))) except ValueError as exc: return "%d ERR %s" % (i, exc) def py_float(i, text): return "%d OK %s" % (i, repr(float(text))) def py_regex(i, text): pattern, subject = text.split("\x01", 1) try: found = re.search(pattern, subject) except re.error as exc: return "%d ERR %s" % (i, exc) if not found: return "%d NONE" % i groups = [found.group(0)] + list(found.groups()) return "%d OK %d %d %s %s" % (i, found.start(), found.end(), json.dumps(groups), json.dumps(found.groupdict())) def fuzz_json(rng, count): out = [] atoms = ["null", "true", "false", "0", "-1", "12", "3.5", "-2.5e3", "\"s\"", "\"\\u00e9\"", "\"a b\"", "[]", "{}", "NaN", "\"\\n\""] for _ in range(count): depth = rng.randint(0, 3) def build(d): if d == 0: return rng.choice(atoms) if rng.random() < 0.5: return "[" + ", ".join(build(d - 1) for _ in range(rng.randint(0, 3))) + "]" return "{" + ", ".join("\"k%d\": %s" % (rng.randint(0, 5), build(d - 1)) for _ in range(rng.randint(0, 3))) + "}" text = build(depth) out.append(text) if text: cut = rng.randint(0, len(text)) out.append(text[:cut]) pos = rng.randint(0, len(text) - 1) if len(text) > 1 else 0 out.append(text[:pos] + rng.choice([",", "]", "}", "\"", "\\", " ", "x", ":"]) + text[pos + 1:]) return out def run(probe, mode, cases, pyfn, label): with tempfile.TemporaryDirectory() as tmp: path = os.path.join(tmp, "cases.bin") write_cases(path, cases) done = subprocess.run([probe, mode, path], stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=120) got = done.stdout.decode("utf-8", "replace").splitlines() want = [pyfn(i, c) for i, c in enumerate(cases)] bad = 0 for i, (g, w) in enumerate(zip(got, want)): if g != w: bad += 1 if bad <= 8: print("MISMATCH %s case %d %r\n cpp: %s\n py : %s" % (label, i, cases[i][:120], g[:300], w[:300])) if len(got) != len(want): bad += 1 print("MISMATCH %s: %d lines from probe, %d cases (stderr: %s)" % (label, len(got), len(want), done.stderr.decode()[:300])) print("%-8s %4d cases, %d mismatches" % (label, len(cases), bad)) return bad def main(): args = sys.argv[1:] if not args: print(__doc__) return 2 probe = args[0] seed = int(args[args.index("--seed") + 1]) if "--seed" in args else 7 fuzz = int(args[args.index("--fuzz") + 1]) if "--fuzz" in args else 300 rng = random.Random(seed) bad = 0 jcases = JSON_CASES + fuzz_json(rng, fuzz) bad += run(probe, "json", jcases, py_json, "json") bad += run(probe, "repr", REPR_CASES, py_repr, "repr") bad += run(probe, "float", FLOAT_CASES, py_float, "float") bad += run(probe, "toml", TOML_CASES, py_toml, "toml") bad += run(probe, "regex", ["%s\x01%s" % c for c in REGEX_CASES], py_regex, "regex") print("TOTAL mismatches: %d" % bad) return 1 if bad else 0 if __name__ == "__main__": sys.exit(main())