tai/taicpp/tests/misc_diff.py

#!/usr/bin/env python3
"""Differential test of the misc modules: runs each scenario through the real tai.py (tests/misc_py.py)
and through the C++ driver (build/misc/misc_driver) on identical state, then diffs the transcripts and
the resulting file trees.

Usage (from taicpp/): python3 tests/misc_diff.py [scenario ...]   (build first: make -f tests/misc.mk all)
Exit code 0 only when every selected scenario matches byte for byte after path/timestamp normalising.
"""
import difflib
import http.server
import json
import threading
import hashlib
import os
import re
import shutil
import subprocess
import sys
import tempfile

HERE = os.path.dirname(os.path.abspath(__file__))
TAICPP = os.path.dirname(HERE)
DRIVER = os.path.join(TAICPP, "build", "misc", "misc_driver")
PYDRV = os.path.join(HERE, "misc_py.py")
DATA = os.path.join(HERE, "misc_data")

ORDER = ["json", "datetime", "chunk", "fts", "skills_data", "skills_files", "skills_ops", "install_hooks", "install_hooks_noshell", "install_ops",
         "install_noengine", "vault_line", "sched", "rag", "telegram", "telegram_install", "release", "audit"]
EXTRA_BIN = {"install_hooks": ("zsh", "fish")}  # scenario -> extra executables to create in the restricted PATH dir
TZ = "Europe/Amsterdam"

TS_RE = re.compile(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:[+-]\d{2}:\d{2})?")
STAMP_RE = re.compile(r"\d{8}T\d{6}Z")
PID_RE = re.compile(r"\.tmp-\d+")


TG_UPDATES = [
    {"update_id": 10, "message": {"chat": {"id": 111}, "text": "/start"}},
    {"update_id": 11, "message": {"chat": {"id": 111}, "text": "hello there"}},
    {"update_id": 12, "message": {"chat": {"id": 222}, "voice": {"file_id": "f1"}}},
    {"update_id": 13, "message": {"chat": {"id": 111}, "text": "explode now"}},
    {"update_id": 14},
    {"update_id": 15, "message": {"chat": {"id": 333}, "text": "/new"}},
    {"update_id": 16, "message": {"chat": {"id": 111}, "text": "long"}},
    {"message": {"chat": {"id": 5}, "text": "no update id"}},
]


class TgServer:
    """Scripted Bot API on localhost; records every request so both sides can be compared on the wire."""

    def __init__(self):
        self.log = []
        self.polls = {}
        outer = self

        class H(http.server.BaseHTTPRequestHandler):
            def log_message(self, *a):
                pass

            def reply(self, code, body, reason=None):
                data = body if isinstance(body, bytes) else body.encode()
                self.send_response(code, reason)
                self.send_header("Content-Type", "application/json")
                self.send_header("Content-Length", str(len(data)))
                self.end_headers()
                self.wfile.write(data)

            def handle_any(self, method):
                n = int(self.headers.get("Content-Length") or 0)
                body = self.rfile.read(n).decode() if n else ""
                outer.log.append("%s %s ct=%s body=%s" % (method, self.path, self.headers.get("Content-Type"), body))
                path = self.path
                if path.startswith("/file/bot"):
                    if path.endswith("voice/f1.oga"):
                        return self.reply(200, b"AUDIO")
                    if path.endswith("voice/fmodel.oga"):
                        return self.reply(200, b"OTHER")
                    return self.reply(404, "{}", "Not Found")
                token, _, name = path[len("/bot"):].partition("/")
                if token == "bad":
                    return self.reply(401, json.dumps({"ok": False, "description": "Unauthorized"}), "Unauthorized")
                if token == "nojson" and name == "getMe":
                    return self.reply(200, "not json")
                if name == "getMe":
                    return self.reply(200, json.dumps({"ok": True, "result": {} if token == "noname" else {"username": "tai_test_bot"}}))
                if name == "getUpdates":
                    count = outer.polls[token] = outer.polls.get(token, 0) + 1
                    if count == 1:
                        return self.reply(200, json.dumps({"ok": True, "result": TG_UPDATES}))
                    return self.reply(502, "{}", "Bad Gateway")
                if name == "getFile":
                    fid = json.loads(body).get("file_id")
                    if fid == "nofile":
                        return self.reply(200, json.dumps({"ok": True, "result": {}}))
                    if fid == "fbad":
                        return self.reply(500, "{}", "Internal Server Error")
                    if fid == "fmodel":
                        return self.reply(200, json.dumps({"ok": True, "result": {"file_path": "voice/fmodel.oga"}}))
                    if fid == "fmissingfile":
                        return self.reply(200, json.dumps({"ok": True, "result": {"file_path": "voice/gone.oga"}}))
                    return self.reply(200, json.dumps({"ok": True, "result": {"file_path": "voice/f1.oga"}}))
                return self.reply(200, json.dumps({"ok": True, "result": {}}))

            def do_POST(self):
                self.handle_any("POST")

            def do_GET(self):
                self.handle_any("GET")

        self.httpd = http.server.ThreadingHTTPServer(("127.0.0.1", 0), H)
        self.port = self.httpd.server_address[1]
        threading.Thread(target=self.httpd.serve_forever, daemon=True).start()

    def stop(self):
        self.httpd.shutdown()
        self.httpd.server_close()


SERVER_SCENARIOS = {"telegram", "telegram_install"}


def normalize(text, root):
    text = text.replace(root, "<ROOT>")
    text = TS_RE.sub("<TS>", text)
    text = STAMP_RE.sub("<STAMP>", text)
    text = PID_RE.sub(".tmp-<PID>", text)
    return text


def make_root(base, name, extra_bins):
    root = os.path.join(base, name)
    for sub in ("home/.tai", "proj", "bin"):
        os.makedirs(os.path.join(root, sub), exist_ok=True)
    os.symlink(sys.executable, os.path.join(root, "bin", "python3"))
    for tool in extra_bins:
        path = os.path.join(root, "bin", tool)
        with open(path, "w") as handle:
            handle.write("#!/bin/sh\nexit 0\n")
        os.chmod(path, 0o755)
    return root


def env_for(root):
    return {
        "HOME": os.path.join(root, "home"),
        "TAI_HOME": os.path.join(root, "home", ".tai"),
        "PATH": os.path.join(root, "bin"),
        "TZ": TZ,
        "LANG": "C.UTF-8",
        "MISC_DATA_DIR": DATA,
        "PYTHONIOENCODING": "utf-8",
        "SHELL": "/bin/bash",
        "MISC_TAI_PY": os.path.join(os.path.dirname(TAICPP), "tai.py"),
    }


def tree(root):
    found = {}
    for dirpath, dirs, files in os.walk(root):
        dirs.sort()
        rel_dir = os.path.relpath(dirpath, root)
        if rel_dir == "bin" or rel_dir.startswith("bin" + os.sep):
            continue
        if rel_dir.startswith(os.path.join("home", ".tai", "venv")):
            continue  # venv contents embed the invoking interpreter path; the report lines are compared instead
        for d in dirs:
            full = os.path.join(dirpath, d)
            if os.path.islink(full):
                found[os.path.relpath(full, root)] = "symlink -> " + normalize(os.readlink(full), root)
        for f in files:
            full = os.path.join(dirpath, f)
            rel = os.path.relpath(full, root)
            if os.path.islink(full):
                found[rel] = "symlink -> " + normalize(os.readlink(full), root)
                continue
            with open(full, "rb") as handle:
                raw = handle.read()
            mode = oct(os.stat(full).st_mode & 0o777)
            try:
                body = normalize(raw.decode("utf-8"), root)
                digest = hashlib.sha256(body.encode("utf-8")).hexdigest()[:16]
            except UnicodeDecodeError:
                digest = "bin-" + hashlib.sha256(raw).hexdigest()[:16]
            if rel.endswith((".db", ".db-wal", ".db-shm")):
                digest = "db"
            found[normalize(rel, root)] = "%s %s" % (mode, digest)
    return found


def run(cmd, root, extra_env=None):
    env = env_for(root)
    if extra_env:
        env.update(extra_env)
    done = subprocess.run(cmd, cwd=os.path.join(root, "proj"), env=env, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                          start_new_session=True, timeout=600)
    return done.returncode, done.stdout.decode("utf-8", "replace"), done.stderr.decode("utf-8", "replace")


def check(name, base, verbose):
    root_py = make_root(base, name + "-py", EXTRA_BIN.get(name, ()))
    code, _out, err = run([sys.executable, "-I", PYDRV, name, root_py, "prepare"], root_py)
    if code != 0:
        return False, "python prepare failed (%d): %s" % (code, err[-2000:])
    root_cpp = os.path.join(base, name + "-cpp")
    shutil.copytree(root_py, root_cpp, symlinks=True)
    # copytree keeps absolute symlink targets; point the bin symlink at the same interpreter (already absolute)
    def one(cmd, root):
        if name not in SERVER_SCENARIOS:
            return run(cmd, root)
        srv = TgServer()
        try:
            extra = {"MISC_TG_API": "http://127.0.0.1:%d/bot" % srv.port, "MISC_TG_FILE": "http://127.0.0.1:%d/file/bot" % srv.port}
            code, o, e = run(cmd, root, extra)
        finally:
            srv.stop()
        o = o + "---- wire log\n" + "\n".join(l.replace(str(srv.port), "PORT") for l in srv.log) 
        if name == "telegram":
            o += "\n---- stderr\n" + e
        return code, o, ""

    code_p, out_p, err_p = one([sys.executable, "-I", PYDRV, name, root_py], root_py)
    code_c, out_c, err_c = one([DRIVER, name, root_cpp], root_cpp)
    out_p = normalize(out_p, root_py)
    out_c = normalize(out_c, root_cpp)
    problems = []
    if code_p != 0:
        problems.append("python exit %d: %s" % (code_p, err_p[-2000:]))
    if code_c != 0:
        problems.append("cpp exit %d: %s" % (code_c, err_c[-2000:]))
    if out_p != out_c:
        diff = list(difflib.unified_diff(out_p.splitlines(), out_c.splitlines(), "python", "cpp", lineterm="", n=1))
        problems.append("transcript differs (%d diff lines):\n%s" % (len(diff), "\n".join(diff[:60])))
    tp, tc = tree(root_py), tree(root_cpp)
    if tp != tc:
        lines = []
        for key in sorted(set(tp) | set(tc)):
            if tp.get(key) != tc.get(key):
                lines.append("  %s: py=%s cpp=%s" % (key, tp.get(key), tc.get(key)))
        problems.append("file tree differs:\n%s" % "\n".join(lines[:60]))
    if verbose:
        sys.stdout.write("---- python transcript (%d lines)\n%s\n" % (len(out_p.splitlines()), out_p))
    return not problems, "\n".join(problems), len(out_p.splitlines()), len(tp)


def main():
    args = [a for a in sys.argv[1:] if not a.startswith("-")]
    verbose = "-v" in sys.argv
    if not os.path.isdir(DATA):
        subprocess.run([sys.executable, "-I", os.path.join(HERE, "gen_misc_data.py")], check=True)
    names = args or ORDER
    base = tempfile.mkdtemp(prefix="misc-diff-")
    failed = 0
    try:
        for name in names:
            result = check(name, base, verbose)
            ok, why = result[0], result[1]
            if ok:
                print("PASS %-14s transcript=%d lines, tree=%d files" % (name, result[2], result[3]))
            else:
                failed += 1
                print("FAIL %s\n%s" % (name, why))
    finally:
        if "--keep" not in sys.argv:
            shutil.rmtree(base, ignore_errors=True)
        else:
            print("kept %s" % base)
    print("%d scenario(s), %d failed" % (len(names), failed))
    return 1 if failed else 0


if __name__ == "__main__":
    sys.exit(main())