tai/taicpp/tests/foundation_parity.py

#!/usr/bin/env python3
"""Vault parity: replay one scenario through tai.py and through the C++ Store
(build/foundation_probe) against two throwaway TAI_HOMEs, then diff the
operation results, every table, the FTS index, the schema and every file under
profiles/. Timestamps and random record ids are masked; everything else must
be identical.

Usage: python3 tests/foundation_parity.py [--probe PATH] [--keep]
"""
import argparse
import json
import os
import re
import sqlite3
import subprocess
import sys
import tempfile

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import foundation_cases as cases  # noqa: E402
import foundation_fixtures as fixtures  # noqa: E402

HERE = os.path.dirname(os.path.abspath(__file__))
ROOT = os.path.dirname(os.path.dirname(HERE))
SESSION = "pane-parity-ctx"
BIG = ("Zażółć gęślą jaźń 😀 line %d fox cats\n" * 300) % tuple(range(300))
BIG = "".join("Zażółć gęślą jaźń 😀 line %d fox cats\n" % n for n in range(300))

SCENARIO = [
    ["add_record", ["note", "Alpha plan", "The quick brown fox jumps over the lazy dogs. cats and mice", ["Animals", "fox"]]],  # 0
    ["add_record", ["note", "Second note", "foxes and cats share tags with animals", ["animal"]]],  # 1
    ["add_record", ["file", "/tmp/x.py", "print('héllo wörld 😀')", ["python"]]],  # 2
    ["add_record", ["output", "Big output", BIG, []]],  # 3
    ["add_chunks", ["record", "$R3", ["chunk a fox", "chunk b"]]],  # 4
    ["get_record", ["$R0"]],  # 5
    ["read_record", ["$R3", 4, 25]],  # 6
    ["search_records", ["fox"]],  # 7
    ["search_records", ["fox animals"]],  # 8
    ["search_records", ["", "note"]],  # 9
    ["search_records", ["zzzz"]],  # 10
    ["search_records", ["", None, ["animals"]]],  # 11
    ["tag_counts", []],  # 12
    ["add_edge", ["$R0", "$R1", "Relates To!"]],  # 13
    ["add_edge", ["$R0", "mem:deadbeefdeadbeef"]],  # 14
    ["edges_for", ["$R0"]],  # 15
    ["traverse", ["$R0"]],  # 16
    ["save_secret", ["api_key", "sekrit-value-123", {"k": 1, "z": "ü"}, ["svc"]]],  # 17
    ["secret_meta", ["api_key"]],  # 18
    ["list_secret_infos", []],  # 19
    ["load_secret", ["api_key"]],  # 20
    ["redact", ["my token sekrit-value-123 here"]],  # 21
    ["add_edge", ["$R0", "secret:api_key", "uses"]],  # 22
    ["add_schedule", ["daily", "do the thing", "default", 3600, "2026-01-01T00:00:00+00:00", 60]],  # 23
    ["list_schedules", []],  # 24
    ["add_edge", ["$R0", "sched:1"]],  # 25
    ["traverse", ["$R0", 3]],  # 26
    ["audit_event", ["main", "write", "/tmp/x.py", "msg about fox", "old text", "new text", ["Edit", "py"]]],  # 27
    ["audit_history", ["/tmp/x.py"]],  # 28
    ["audit_get", ["$R27"]],  # 29
    ["audit_search", ["fox"]],  # 30
    ["upsert_file_record", ["/tmp/y.md", "# Title\nfox content"]],  # 31
    ["file_record_id", ["/tmp/y.md"]],  # 32
    ["upsert_file_record", ["/tmp/y.md", "# Title 2\nupdated fox", None]],  # 33
    ["delete_file_record", ["/tmp/y.md"]],  # 34
    ["log_event", ["default", "user", "say", "hello fox world", ["a"]]],  # 35
    ["log_event", ["default", "assistant", "say", "bye"]],  # 36
    ["search_events", ["default", "fox"]],  # 37
    ["events_like", ["default", "bye"]],  # 38
    ["history_add", ["default", "main", "first input"]],  # 39
    ["history_add", ["default", "main", "first input"]],  # 40
    ["history_add", ["default", "main", "  padded ünï  "]],  # 41
    ["history_load", ["default"]],  # 42
    ["fts_search", ["fox"]],  # 43
    ["fts_search", ["fox", ["record"]]],  # 44
    ["save_system", ["default", "You are a test."]],  # 45
    ["load_system", ["default"]],  # 46
    ["save_session", ["default", [{"role": "system", "content": "S"}, {"role": "user", "content": "hi ü 😀"},
                                    {"role": "assistant", "content": "yo", "tool_calls": [{"id": "1", "function": {"name": "shell", "arguments": "{}"}}]},
                                    {"role": "tool", "content": "x", "tool_call_id": "1"}], "main", SESSION]],  # 47
    ["load_session", ["default", "main", SESSION]],  # 48
    ["load_session", ["default", "main", "other-ctx"]],  # 49
    ["save_bots", ["default", {"helper": {"nicknames": ["h"]}, "zed": {}}]],  # 50
    ["list_bots", ["default"]],  # 51
    ["resolve_bot", ["default", "H"]],  # 52
    ["resolve_bot", ["default", "main"]],  # 53
    ["resolve_bot", ["default", "nope"]],  # 54
    ["save_bot_system", ["default", "helper", "I am helper"]],  # 55
    ["load_bot_system", ["default", "helper"]],  # 56
    ["save_session", ["default", [{"role": "user", "content": "to helper"}], "helper", SESSION]],  # 57
    ["load_session", ["default", "helper", SESSION]],  # 58
    ["save_turn_state", ["default", "main", {"open": True, "goal": "ship it", "steps": 3, "tools": 2, "updated": "2026-01-01T00:00:00+00:00"}, SESSION]],  # 59
    ["load_turn_state", ["default", "main", SESSION]],  # 60
    ["get_or_create_settings", []],  # 61
    ["upsert_settings", [{"model": "foo", "voice": "v", "retry_seconds": 2.5, "zen_enabled": "yes", "theme": "nord", "openrouter_key": "abc", "gemini_plan": True}]],  # 62
    ["upsert_settings", [{"bogus": 1}]],  # 63
    ["upsert_settings", [{"theme": "neon"}]],  # 64
    ["upsert_settings", [{"model": None}]],  # 65
    ["upsert_settings", [{"retry_seconds": -1}]],  # 66
    ["upsert_settings", [{"zen_enabled": "maybe"}]],  # 67
    ["upsert_settings", [{"muse_bin": None, "opencode_models": "a,b"}], {"scope": "work"}],  # 68
    ["get_all_settings", [], {"scope": "work"}],  # 69
    ["get_or_create_settings", [["model", "voice"]], {"scope": "bad scope!"}],  # 70
    ["mcp_save_server", [{"name": "srv", "transport": "stdio", "command": "x", "args": ["a", "b"], "env": {"K": "v"}, "headers": {"Authorization": "Bearer t"}, "allow": ["t1"], "source": "test"}]],  # 71
    ["mcp_save_server", [{"name": "bad name!", "transport": "stdio"}]],  # 72
    ["mcp_get_server", ["srv"]],  # 73
    ["mcp_store_tools", ["srv", [{"name": "t1", "description": "d1", "inputSchema": {"type": "object", "b": 1, "a": 2}}, {"name": "t2"}, {"nope": 1}]]],  # 74
    ["mcp_cached_tools", ["srv"]],  # 75
    ["mcp_tool_pins", ["srv"]],  # 76
    ["mcp_set_enabled", ["srv", "default", True]],  # 77
    ["mcp_enabled_servers", ["default"]],  # 78
    ["mcp_list_servers", []],  # 79
    ["save_dir_defaults", ["/tmp/dd", "prov", "mod", "bot1"]],  # 80
    ["load_dir_defaults", ["/tmp/dd"]],  # 81
    ["save_dir_defaults", ["/tmp/dd", None, None, None, ["model"]]],  # 82
    ["load_dir_defaults", ["/tmp/dd"]],  # 83
    ["spill_or_truncate", ["output", "spilled", BIG]],  # 84
    ["spill_or_truncate", ["output", "small", "short text"]],  # 85
    ["known_tags", []],  # 86
    ["canonical_tags", [["Animals", "new tag", "cats"]]],  # 87
    ["tagged_items", [["record", "fox"]]],  # 88
    ["item_tags", ["$R0"]],  # 89
    ["delete_record", ["$R1"]],  # 90
    ["delete_secret", ["api_key"]],  # 91
    ["remove_schedule", [1]],  # 92
    ["search_records", ["fox"]],  # 93
    ["add_record", ["note", "Work note", "work profile fox", ["w"], "work"]],  # 94
    ["search_records", ["fox", None, [], 8, "work"]],  # 95
    ["mcp_remove_server", ["srv"]],  # 96
    ["list_profiles", []],  # 97
    ["get_chunks_by_ids", [[1, 2, 999]]],  # 98
]

UNORDERED = {"tagged_items"}
TS = re.compile(r"\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d(?:\.\d+)?\+00:00")
MEM = re.compile(r"mem:[0-9a-f]{16}")
EPHEMERAL = re.compile(r"(\d{8}-\d{6}-e\d+)-[0-9a-f]{4}")


PURE_OPS = {"prompt", "capability_block", "orientation_block", "settings_catalog", "json_dumps", "json_roundtrip", "float_repr", "normalize_tag", "normalize_tags",
            "singular_noun", "tag_variants", "parse_node_id", "fts_match_variants", "fts_rank", "profile_re", "bot_re", "secret_re", "mcp_server_re",
            "normalize_setting_value", "parse_setting_env", "parse_setting_stored", "parse_backends", "split_models", "chunk_lines", "truncate", "sanitize_sid",
            "shell_current", "validate_flow_spec", "run_flow", "sleep", "native_ctl", "tmux_ctl",
            "statusbar", "strip_ansi", "truncate_ansi", "glow_prompt", "glow_title"}


def special(tai, op, pos):
    """Module-level functions of tai.py that have no Store: the python side of the pure cases."""
    if op == "prompt":
        return getattr(tai, pos[0])
    if op == "capability_block":
        return tai.capability_block(*pos)
    if op == "orientation_block":
        return tai.orientation_block(*pos)
    if op == "settings_catalog":
        return {key: {"key": spec["key"], "env": spec["env"], "type": spec["type"], "default": spec["default"], "desc": spec["desc"], "required": spec["required"],
                      "nullable": spec["nullable"], "encrypted": spec["encrypted"], "options": spec["options"], "preferred": spec["preferred"],
                      "true_words": spec["true_words"], "false_words": spec["false_words"]} for key, spec in tai.SETTINGS_CATALOG.items()}
    if op == "json_dumps":
        return [json.dumps(pos[0]), json.dumps(pos[0], sort_keys=True)]
    if op == "json_roundtrip":
        try:
            return json.dumps(json.loads(pos[0]), sort_keys=True)
        except ValueError:
            return {"ValueError": "x"}
    if op == "float_repr":
        return repr(float(pos[0]))
    if op == "normalize_tag":
        return tai.normalize_tag(pos[0])
    if op == "normalize_tags":
        return tai.normalize_tags(pos[0])
    if op == "singular_noun":
        return tai.singular_noun(pos[0])
    if op == "tag_variants":
        return tai.tag_variants(pos[0])
    if op == "parse_node_id":
        return tai.parse_node_id(pos[0])
    if op == "fts_match_variants":
        terms = tai.fts_escape(pos[0])
        return None if terms is None else tai.fts_match_variants(terms)
    if op == "fts_rank":
        return tai.fts_rank(pos[0])
    if op in ("profile_re", "bot_re", "secret_re", "mcp_server_re"):
        regex = {"profile_re": tai.PROFILE_RE, "bot_re": tai.BOT_RE, "secret_re": tai.SECRET_RE, "mcp_server_re": tai.MCP_SERVER_RE}[op]
        return regex.match(pos[0]) is not None
    if op == "normalize_setting_value":
        try:
            return tai.normalize_setting_value(pos[0], tai.SETTINGS_CATALOG[pos[0]], pos[1])
        except ValueError as exc:
            return {"ValueError": str(exc)}
    if op == "parse_setting_env":
        try:
            return tai.parse_setting_env(tai.SETTINGS_CATALOG[pos[0]], pos[1].strip())
        except ValueError as exc:
            return {"ValueError": str(exc)}
    if op == "parse_setting_stored":
        try:
            return tai.parse_setting_stored(tai.SETTINGS_CATALOG[pos[0]], pos[1])
        except ValueError:
            return {"ValueError": "could not convert"}
    if op == "parse_backends":
        return tai.parse_backends(pos[0])
    if op == "split_models":
        return tai.split_models(pos[0])
    if op == "chunk_lines":
        return tai.chunk_lines(*pos)
    if op == "truncate":
        return tai.truncate(*pos)
    if op == "sanitize_sid":
        return tai.ShellSession.sanitize(pos[0])
    if op == "shell_current":
        env, pid, root, now, tick = pos
        c = tai.ShellSession.current(env=env, pid=pid, proc_root=root, now=now, clk_tck=tick)
        return [c.sid, c.shell, c.pid, c.started, c.parent, c.chain]
    if op == "statusbar":
        return tai.Glow.statusbar(pos[0], pos[1], pos[2], yolo=pos[3], agents=pos[4], width=pos[5])
    if op == "strip_ansi":
        return tai.strip_ansi(pos[0])
    if op == "truncate_ansi":
        return tai.truncate_ansi(pos[0], pos[1])
    if op == "glow_prompt":
        return tai.Glow.prompt_text()
    if op == "glow_title":
        A, version, profile = tai.Ansi, pos[0], pos[1]
        return "%s◆%s %stai%s %s%s%s %s· profile %s%s" % (A.BOLD, A.RESET, A.BOLD, A.RESET, tai.FG_FILE, version, A.RESET, A.DIM, profile, A.RESET)
    if op == "sleep":
        import time
        time.sleep(pos[0])
        return None
    if op in ("native_ctl", "tmux_ctl"):
        if op == "native_ctl":
            sock, pane, method, args = pos
            ctl = tai.NativePlexerController(sock, pane or None)
        else:
            pane, method, args = pos
            ctl = tai.TmuxPlexerController(pane)
        try:
            if method == "type_text":
                return ctl.type_text(args[0], 0.0, args[1])
            if method == "call":
                return ctl._call(args[0])
            return getattr(ctl, method)(*args)
        except tai.PlexerError as exc:
            return {"PlexerError": str(exc)}
    if op == "validate_flow_spec":
        app, error = tai.validate_flow_spec(pos[0])
        return [app if not error else None, error]
    if op == "run_flow":
        app, error = tai.validate_flow_spec(pos[0])
        if error:
            return [None, error]

        class Fake(tai.PlexerController):
            def __init__(self, replies):
                self.replies = list(replies)
                self.asked = []

            def ask(self, screen, timeout):
                self.asked.append(screen)
                return self.replies.pop(0)
        fake = Fake(pos[1])
        outcome = tai.PlexerFlow(app).run(fake)
        return [outcome, fake.asked]
    raise KeyError(op)


STORE_OPS = {"debug_log", "repl_script", "log_query"}


def special_store(tai, op, pos, store, config):
    """tai.py behaviour that needs the live store: DebugLog sinks, REPL commands, log queries."""
    import contextlib
    import io
    if op == "log_query":
        return store.log_query(**pos[0])
    if op == "debug_log":
        sink, fmt, events = pos
        tai.LOG.configure(sink, fmt, config.home, store=store)
        for level, component, event, message, kwargs in events:
            tai.LOG.event(level, component, event, message, **kwargs)
        run = tai.LOG.run
        tai.LOG.close()
        tai.LOG.reset()
        with open(os.path.join(config.home, "debug.log"), "r", encoding="utf-8") as handle:
            return [run, handle.read()]
    if op == "repl_script":
        agent = tai.Agent(config, store, ctx=os.environ.get("TAI_SESSION"))
        outputs = []
        for text in pos[0]:
            buf = io.StringIO()
            with contextlib.redirect_stdout(buf):
                try:
                    cont = tai.handle_command(agent, text)
                except IndexError:
                    print("unknown command, try /help")  # python crashes on a bare "/"; the C++ port reports it as unknown
                    cont = True
            outputs.append([buf.getvalue(), cont])
        return [outputs, [agent.profile, agent.bot, agent.messages[1:]]]
    raise KeyError(op)


def run_python(home, scenario_path):
    """Executed in a child interpreter: replay through tai.py."""
    sys.path.insert(0, ROOT)
    import tai
    import argparse as ap
    args = ap.Namespace(profile="default", yes=False, no_opencode=False)
    config = tai.Config(args)
    seal = tai.Seal(config.home, "")
    store = tai.Store(config, seal)
    with open(scenario_path, "r", encoding="utf-8") as handle:
        scenario = json.load(handle)
    results = []

    def sub(value):
        if isinstance(value, str) and value.startswith("$R") and value[2:].isdigit():
            return results[int(value[2:])]
        if isinstance(value, list):
            return [sub(item) for item in value]
        if isinstance(value, dict):
            return {key: sub(item) for key, item in value.items()}
        return value

    def plain(value):
        if isinstance(value, tuple):
            return [plain(item) for item in value]
        if isinstance(value, list):
            return [plain(item) for item in value]
        if isinstance(value, set):
            return sorted(value)
        if isinstance(value, dict):
            return {str(key): plain(item) for key, item in value.items()}
        return value

    for step in scenario:
        op, pos = step[0], sub(step[1]) if len(step) > 1 else []
        kw = sub(step[2]) if len(step) > 2 else {}
        try:
            if op in STORE_OPS:
                out = special_store(tai, op, pos, store, config)
            elif op in PURE_OPS:
                out = special(tai, op, pos)
            elif op == "spill_or_truncate":
                out = tai.spill_or_truncate(store, *pos, **kw)
            elif op == "mcp_set_enabled":
                out = store.mcp_set_enabled(*pos, **kw)
            else:
                out = getattr(store, op)(*pos, **kw)
        except ValueError as exc:
            out = {"ValueError": str(exc)}
        results.append(plain(out))
    store.close()
    print(json.dumps(results, sort_keys=True))


def dump_vault(home):
    out = {"tables": {}, "schema": [], "files": {}}
    db = sqlite3.connect(os.path.join(home, "memory.db"))
    for name, sql in db.execute("SELECT name, sql FROM sqlite_master WHERE name NOT LIKE 'fts_docs_%' AND name NOT LIKE 'sqlite_%' ORDER BY type, name").fetchall():
        out["schema"].append([name, " ".join((sql or "").split())])
    tables = [row[0] for row in db.execute("SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'fts_docs_%' AND name NOT LIKE 'sqlite_%' ORDER BY name").fetchall()]
    for table in tables:
        cols = [row[1] for row in db.execute("PRAGMA table_info(%s)" % table).fetchall()]
        if table == "fts_docs":
            rows = db.execute("SELECT item, kind, sub, title, body, profile FROM fts_docs").fetchall()
            cols = ["item", "kind", "sub", "title", "body", "profile"]
        else:
            rows = db.execute("SELECT * FROM %s" % table).fetchall()
        out["tables"][table] = {"columns": cols, "rows": [list(row) for row in rows]}
    db.close()
    prof = os.path.join(home, "profiles")
    for entry in sorted(os.listdir(prof)):
        with open(os.path.join(prof, entry), "r", encoding="utf-8") as handle:
            out["files"][entry] = handle.read()
    return out


def normalizer(results, scenario):
    runs = [results[pos][0] for pos, step in enumerate(scenario) if step[0] == "debug_log"]
    order = []
    for blob in (json.dumps(results, sort_keys=True),):
        for match in MEM.finditer(blob):
            if match.group(0) not in order:
                order.append(match.group(0))
    names = {mem: "<MEM%d>" % pos for pos, mem in enumerate(order)}

    def norm(value):
        if isinstance(value, str):
            for run in runs:
                value = value.replace(run, "<RUN>")
            value = TS.sub("<TS>", value)
            value = EPHEMERAL.sub(r"\1-<RAND>", value)
            return MEM.sub(lambda m: names.get(m.group(0), "<MEM?>"), value)
        if isinstance(value, float):
            return float("%.9g" % value)
        if isinstance(value, list):
            return [norm(item) for item in value]
        if isinstance(value, dict):
            return {norm(key): norm(item) for key, item in value.items()}
        return value
    return norm


def settle(dump, norm):
    dump = norm(dump)
    for table in dump["tables"].values():
        table["rows"].sort(key=lambda row: json.dumps(row, sort_keys=True, default=str))
    return dump


def first_diff(a, b, path="$"):
    if type(a) is not type(b):
        return "%s: type %s vs %s: %r vs %r" % (path, type(a).__name__, type(b).__name__, str(a)[:200], str(b)[:200])
    if isinstance(a, dict):
        for key in sorted(set(a) | set(b)):
            if key not in a or key not in b:
                return "%s.%s: only in %s" % (path, key, "python" if key in a else "cpp")
            found = first_diff(a[key], b[key], "%s.%s" % (path, key))
            if found:
                return found
        return None
    if isinstance(a, list):
        if len(a) != len(b):
            return "%s: length %d vs %d" % (path, len(a), len(b))
        for pos, (x, y) in enumerate(zip(a, b)):
            found = first_diff(x, y, "%s[%d]" % (path, pos))
            if found:
                return found
        return None
    if a != b:
        return "%s: %r vs %r" % (path, str(a)[:300], str(b)[:300])
    return None


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--probe", default=os.path.join(os.path.dirname(HERE), "build", "foundation_probe"))
    parser.add_argument("--keep", action="store_true")
    parser.add_argument("--show", nargs="*", default=[], metavar="OP", help="print the C++ result of every step with this op")
    parser.add_argument("--py-runner", nargs=2, metavar=("HOME", "SCENARIO"))
    opts = parser.parse_args()
    if opts.py_runner:
        run_python(*opts.py_runner)
        return 0
    work = tempfile.mkdtemp(prefix="detz-parity-")
    proc = os.path.join(work, "proc")
    cases.build_fake_proc(proc)
    cases.build_fake_proc(proc + "_noboot", boot_id=False)
    cases.build_fake_proc(proc + "_nobtime", btime=False)
    cases.build_fake_proc(proc + "_bare", boot_id=False, btime=False)
    scenario = SCENARIO + cases.debug_cases() + cases.repl_cases() + cases.pure_cases() + cases.flow_cases() + cases.proc_cases(proc) + cases.style_cases() + cases.prune_cases() + cases.plexer_cases()
    runs = {}
    for label in ("python", "cpp"):
        home = os.path.join(work, label)
        os.makedirs(home)
        backend = fixtures.FakeBackend(os.path.join(work, label + ".sock"))
        tmux_server = fixtures.TmuxServer(os.path.join(work, label + "-tmux-root"))
        scenario_path = os.path.join(work, label + "-scenario.json")
        with open(scenario_path, "w", encoding="utf-8") as handle:
            json.dump(json.loads(json.dumps(scenario).replace("$SOCK", backend.path).replace("@@HOME@@", work)), handle)
        env = {key: value for key, value in os.environ.items() if not key.startswith(("TAI_", "TMUX"))}
        env.update({"TAI_HOME": home, "TAI_SESSION": SESSION, "TAI_PASSPHRASE": "", "HOME": work, "TMUX_TMPDIR": tmux_server.tmpdir})
        if label == "python":
            cmd = [sys.executable, "-I", os.path.abspath(__file__), "--py-runner", home, scenario_path]
        else:
            cmd = [opts.probe, scenario_path]
        try:
            done = subprocess.run(cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, timeout=300)
        finally:
            backend.close()
            tmux_server.close()
        if done.returncode != 0:
            print("%s run failed (%d):\n%s" % (label, done.returncode, done.stderr[-2000:]))
            return 1
        runs[label] = {"results": json.loads(done.stdout), "dump": dump_vault(home), "requests": backend.log}
    # vault portability: each implementation reads the vault the other one wrote
    read_path = os.path.join(work, "readback.json")
    with open(read_path, "w", encoding="utf-8") as handle:
        json.dump(cases.readback_cases(SESSION), handle)

    def read_vault(impl, home):
        env = {key: value for key, value in os.environ.items() if not key.startswith(("TAI_", "TMUX"))}
        env.update({"TAI_HOME": home, "TAI_SESSION": SESSION, "TAI_PASSPHRASE": "", "HOME": work})
        cmd = [sys.executable, "-I", os.path.abspath(__file__), "--py-runner", home, read_path] if impl == "python" else [opts.probe, read_path]
        done = subprocess.run(cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, timeout=120)
        if done.returncode != 0:
            raise RuntimeError("%s readback failed:\n%s" % (impl, done.stderr[-1500:]))
        return json.loads(done.stdout)

    portability = 0
    for home_label in ("python", "cpp"):
        home = os.path.join(work, home_label)
        by_python, by_cpp = read_vault("python", home), read_vault("cpp", home)
        for pos, (a, b) in enumerate(zip(by_python, by_cpp)):
            diff = first_diff(a, b)
            if diff:
                portability += 1
                print("PORTABILITY %s vault, step %d (%s): %s" % (home_label, pos, cases.readback_cases(SESSION)[pos][0], diff))
    py_norm, cpp_norm = normalizer(runs["python"]["results"], scenario), normalizer(runs["cpp"]["results"], scenario)
    py_res, cpp_res = py_norm(runs["python"]["results"]), cpp_norm(runs["cpp"]["results"])
    failures = portability
    for pos, step in enumerate(scenario):
        if step[0] in opts.show:
            print("%d %s -> %s" % (pos, step[0], json.dumps(cpp_res[pos], ensure_ascii=False)[:300]))
    for pos, step in enumerate(scenario):
        if step[0] in UNORDERED:  # SQL group order follows the random record ids
            py_res[pos] = sorted(py_res[pos], key=str)
            cpp_res[pos] = sorted(cpp_res[pos], key=str)
    for pos, (a, b) in enumerate(zip(py_res, cpp_res)):
        diff = first_diff(a, b)
        if diff:
            failures += 1
            print("RESULT %d (%s) differs: %s" % (pos, scenario[pos][0], diff))
    if len(py_res) != len(cpp_res):
        failures += 1
        print("result count %d vs %d" % (len(py_res), len(cpp_res)))
    py_dump, cpp_dump = settle(runs["python"]["dump"], py_norm), settle(runs["cpp"]["dump"], cpp_norm)
    diff = first_diff(py_dump["schema"], cpp_dump["schema"])
    if diff:
        failures += 1
        print("SCHEMA differs: %s" % diff)
    for table in sorted(set(py_dump["tables"]) | set(cpp_dump["tables"])):
        diff = first_diff(py_dump["tables"].get(table), cpp_dump["tables"].get(table))
        if diff:
            failures += 1
            print("TABLE %s differs: %s" % (table, diff))
    diff = first_diff(py_dump["files"], cpp_dump["files"])
    if diff:
        failures += 1
        print("FILES differ: %s" % diff)
    diff = first_diff(runs["python"]["requests"], runs["cpp"]["requests"])
    if diff:
        failures += 1
        print("NATIVE BACKEND REQUESTS differ: %s" % diff)
    total_rows = sum(len(t["rows"]) for t in py_dump["tables"].values())
    print("%d operations, %d tables, %d rows, %d profile files compared, %d differences" % (len(py_res), len(py_dump["tables"]), total_rows, len(py_dump["files"]), failures))
    if opts.keep:
        print("kept %s" % work)
    return 1 if failures else 0


if __name__ == "__main__":
    sys.exit(main())