regexx/tests/gen.py

#!/usr/bin/env python3
"""Generate tests/generated_tests.c from tests/cases.py, using CPython's
own `re` module as ground truth (concept.md Section 11)."""
import re
import sys
import os

sys.path.insert(0, os.path.dirname(__file__))
from cases import CASES  # noqa: E402

FLAGMAP = {
    "IGNORECASE": re.IGNORECASE,
    "MULTILINE": re.MULTILINE,
    "DOTALL": re.DOTALL,
    "VERBOSE": re.VERBOSE,
    "LOCALE": re.LOCALE,
    "ASCII": re.ASCII,
}


def c_str(b):
    if isinstance(b, str):
        b = b.encode("utf-8")
    out = ['"']
    for byte in b:
        c = chr(byte)
        if c == '"':
            out.append('\\"')
        elif c == "\\":
            out.append("\\\\")
        elif 32 <= byte < 127:
            out.append(c)
        else:
            out.append("\\%03o" % byte)
    out.append('"')
    return "".join(out)


def py_flags(case):
    f = 0
    if case["mode"] == "ascii":
        f |= re.ASCII
    for name in case["flags"]:
        f |= FLAGMAP[name]
    return f


def c_flags(case):
    names = []
    if case["mode"] == "ascii":
        names.append("ASCII")
    elif case["mode"] == "utf8":
        names.append("UTF8")
    elif case["mode"] == "binary":
        names.append("BINARY")
    for name in case["flags"]:
        names.append(name)
    return "|".join(names) if names else "0"


def encode(case, s):
    if case["mode"] == "binary":
        return s.encode("utf-8")
    return s


def blen(case, s):
    """Byte length of the encoded subject, as the C API always wants a
    byte count (UTF8 mode still stores a byte buffer; only Match_start
    reports code points), never a Python character count."""
    e = encode(case, s)
    return len(e) if isinstance(e, bytes) else len(e.encode("utf-8"))


def desc(case, i):
    return "#%d %s /%s/ on %r" % (i, case["op"], case["pattern"], case["subject"])


def gen_search_family(case, i, out):
    kind = {"search": 0, "match": 1, "fullmatch": 2}[case["op"]]
    flags = py_flags(case)
    subj = encode(case, case["subject"])
    pat = encode(case, case["pattern"])
    compiled = re.compile(pat, flags)
    fn = {"search": compiled.search, "match": compiled.match, "fullmatch": compiled.fullmatch}[case["op"]]
    m = fn(subj)
    d = desc(case, i)
    if m is None:
        out.append('    tc_search_family(%s, %s, %s, %s, %d, %d, 0, 0, NULL);' % (
            c_str(d), c_str(case["pattern"]), c_flags(case), c_str(case["subject"]), blen(case, case["subject"]), kind))
        return
    spans = [m.start(0), m.end(0)]
    for g in range(1, compiled.groups + 1):
        try:
            spans += [m.start(g), m.end(g)]
        except IndexError:
            spans += [-1, -1]
    n_spans = compiled.groups + 1
    arr = ",".join(str(v) for v in spans)
    out.append('    { static const long long sp[] = {%s}; tc_search_family(%s, %s, %s, %s, %d, %d, 1, %d, sp); }' % (
        arr, c_str(d), c_str(case["pattern"]), c_flags(case), c_str(case["subject"]), blen(case, case["subject"]), kind, n_spans))


def gen_finditer(case, i, out):
    flags = py_flags(case)
    subj = encode(case, case["subject"])
    pat = encode(case, case["pattern"])
    ms = list(re.finditer(pat, subj, flags))
    spans = []
    for m in ms:
        spans += [m.start(0), m.end(0)]
    d = desc(case, i)
    if not spans:
        out.append('    tc_finditer(%s, %s, %s, %s, %d, 0, NULL);' % (
            c_str(d), c_str(case["pattern"]), c_flags(case), c_str(case["subject"]), blen(case, case["subject"])))
        return
    arr = ",".join(str(v) for v in spans)
    out.append('    { static const long long sp[] = {%s}; tc_finditer(%s, %s, %s, %s, %d, %d, sp); }' % (
        arr, c_str(d), c_str(case["pattern"]), c_flags(case), c_str(case["subject"]), blen(case, case["subject"]), len(ms)))


def gen_sub(case, i, out):
    flags = py_flags(case)
    subj = encode(case, case["subject"])
    pat = encode(case, case["pattern"])
    repl = encode(case, case["repl"])
    count = case.get("count", 0)
    result, n = re.subn(pat, repl, subj, count=count, flags=flags)
    d = desc(case, i)
    result_str = result.decode("utf-8") if isinstance(result, bytes) else result
    out.append('    tc_sub(%s, %s, %s, %s, %d, %s, %d, %s, %d);' % (
        c_str(d), c_str(case["pattern"]), c_flags(case), c_str(case["subject"]), blen(case, case["subject"]),
        c_str(case["repl"]), count, c_str(result_str), n))


def gen_split(case, i, out):
    flags = py_flags(case)
    subj = encode(case, case["subject"])
    pat = encode(case, case["pattern"])
    maxsplit = case.get("maxsplit", 0)
    items = re.split(pat, subj, maxsplit=maxsplit, flags=flags)
    d = desc(case, i)
    parts = []
    for it in items:
        if it is None:
            parts.append("NULL")
        else:
            s = it.decode("utf-8") if isinstance(it, bytes) else it
            parts.append(c_str(s))
    arr = ",".join(parts) if parts else "0"
    out.append('    { static const char *const items[] = {%s}; tc_split(%s, %s, %s, %s, %d, %d, %d, items); }' % (
        arr, c_str(d), c_str(case["pattern"]), c_flags(case), c_str(case["subject"]), blen(case, case["subject"]),
        maxsplit, len(items)))


def main():
    out = []
    out.append('/* GENERATED by tests/gen.py from tests/cases.py. Do not edit by hand. */')
    out.append('#include "harness.h"')
    out.append('void run_generated_tests(void) {')
    for i, case in enumerate(CASES):
        op = case["op"]
        if op in ("search", "match", "fullmatch"):
            gen_search_family(case, i, out)
        elif op == "finditer":
            gen_finditer(case, i, out)
        elif op == "sub":
            gen_sub(case, i, out)
        elif op == "split":
            gen_split(case, i, out)
        else:
            raise ValueError("unknown op %r" % op)
    out.append('}')
    dest = os.path.join(os.path.dirname(__file__), "generated_tests.c")
    with open(dest, "w") as f:
        f.write("\n".join(out) + "\n")
    print("wrote %s (%d cases)" % (dest, len(CASES)))


if __name__ == "__main__":
    main()