tai/taicpp/tools/parity_tests.py

#!/usr/bin/env python3
"""Mine ../test_tai.py into a coverage table: test class -> tai.py symbols -> diff case or `uncovered`.

Usage: tools/parity_tests.py     (writes build/parity/tests-map.json and tests-map.md)

For every unittest class in test_tai.py (ast, never imported): the number of tests, and the
tai.py symbols it touches (`tai.X`, `tai.X.y`, and `"tai.X.y"` mock.patch strings), resolved
against the ledger ids (class:X, function:X, method:X.y; module data shows as data:X).

A test (or whole class) maps to a differential case ONLY when the case lists `Class` or `Class.test_name` in its `tests` field
AND that case passed in the last unfiltered run. Everything else is `uncovered`. Listing a
class in a case means the case reproduces what those tests assert through the real binaries;
it is a claim the reviewer can read in the case file, not something inferred here.
"""
import ast
import json
import os
import re
import sys

HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, HERE)
import parity_ledger as pl  # noqa: E402

TEST_FILE = os.path.join(pl.REPO, "test_tai.py")


class TestMiner:
    def __init__(self, path, symbol_ids):
        with open(path, encoding="utf-8") as fh:
            self.tree = ast.parse(fh.read())
        self.ids = symbol_ids

    def classes(self):
        out = []
        for node in self.tree.body:
            if isinstance(node, ast.ClassDef):
                tests = [n for n in node.body if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)) and n.name.startswith("test")]
                if tests:
                    out.append({"class": node.name, "line": node.lineno, "tests": len(tests), "names": [t.name for t in tests], "symbols": self.symbols(node)})
        return out

    def symbols(self, cls):
        found = set()
        for node in ast.walk(cls):
            if isinstance(node, ast.Attribute):
                inner = node.value
                if isinstance(inner, ast.Name) and inner.id == "tai":
                    found.add(node.attr)
                elif isinstance(inner, ast.Attribute) and isinstance(inner.value, ast.Name) and inner.value.id == "tai":
                    found.add("%s.%s" % (inner.attr, node.attr))
            elif isinstance(node, ast.Constant) and isinstance(node.value, str):
                for m in re.finditer(r"\btai\.([A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)?)\b", node.value):
                    found.add(m.group(1))
        return sorted({self.resolve(n) for n in found})

    def resolve(self, name):
        for kind in ("method", "class", "function"):
            sid = "%s:%s" % (kind, name)
            if sid in self.ids:
                return sid
        head = name.split(".")[0]
        if "." in name and "class:%s" % head in self.ids:
            return "class:%s" % head  # attribute or nested name on a known class
        return "data:%s" % name


def main():
    ledger = pl.Ledger()
    ids = {r["id"] for r in ledger.rows}
    classes = TestMiner(TEST_FILE, ids).classes()
    ev = ledger.evidence
    by_test = {}  # "Class" or "Class.test" -> passing case ids
    for cid, res in ev.cases.items():
        if res["status"] == "pass":
            for tc in res["tests"]:
                by_test.setdefault(tc, []).append(cid)
    errors = []
    names = {c["class"]: set(c["names"]) for c in classes}
    for cid, res in ev.cases.items():
        for tc in res["tests"]:
            cls, _, test = tc.partition(".")
            if cls not in names or (test and test not in names[cls]):
                errors.append("case %s names unknown test %s" % (cid, tc))
    for c in classes:
        whole = by_test.get(c["class"], [])
        per = {n: sorted(set(whole + by_test.get("%s.%s" % (c["class"], n), []))) for n in c["names"]}
        c["covered_tests"] = sum(1 for v in per.values() if v)
        c["cases"] = sorted({cid for v in per.values() for cid in v})
        c["status"] = "uncovered" if not c["covered_tests"] else ("covered" if c["covered_tests"] == c["tests"] else "partial")
    total_tests = sum(c["tests"] for c in classes)
    cov = [c for c in classes if c["covered_tests"]]
    summary = {"classes": len(classes), "tests": total_tests, "classes_covered": len(cov),
               "tests_covered": sum(c["covered_tests"] for c in classes), "classes_uncovered": len(classes) - len(cov)}
    os.makedirs(pl.OUT, exist_ok=True)
    with open(os.path.join(pl.OUT, "tests-map.json"), "w") as fh:
        json.dump({"summary": summary, "classes": classes, "errors": sorted(set(errors))}, fh, indent=1)
    lines = ["# test_tai.py coverage map (generated)", "",
             "%(classes)d classes, %(tests)d tests. Tests reproduced by a passing diff case: %(tests_covered)d, in %(classes_covered)d classes. Fully uncovered classes: %(classes_uncovered)d." % summary, "",
             "| test class | tests | covered | status | diff cases | tai.py symbols |", "|---|---|---|---|---|---|"]
    for c in sorted(classes, key=lambda x: (x["status"], -x["tests"], x["class"])):
        syms = ", ".join(c["symbols"][:8]) + (" ... +%d" % (len(c["symbols"]) - 8) if len(c["symbols"]) > 8 else "")
        lines.append("| %s | %d | %d | %s | %s | %s |" % (c["class"], c["tests"], c["covered_tests"], c["status"], ",".join(c["cases"]) or "uncovered", syms))
    if errors:
        lines += ["", "## Errors", ""] + ["- " + e for e in sorted(set(errors))]
    with open(os.path.join(pl.OUT, "tests-map.md"), "w") as fh:
        fh.write("\n".join(lines) + "\n")
    print("test classes %(classes)d, tests %(tests)d, covered classes %(classes_covered)d (tests %(tests_covered)d), uncovered classes %(classes_uncovered)d" % summary)
    for e in sorted(set(errors)):
        print("ERROR:", e)
    return 1 if errors else 0


if __name__ == "__main__":
    sys.exit(main())