regexx/tests/gen.py

#!/usr/bin/env python3
"""Generate tests/generated_tests.c from tests/cases.py, using CPython's
own `re` module as ground truth (concept.md Section 11)."""
import re
import sys
import os

sys.path.insert(0, os.path.dirname(__file__))
from cases import CASES  # noqa: E402

FLAGMAP = {
    "IGNORECASE": re.IGNORECASE,
    "MULTILINE": re.MULTILINE,
    "DOTALL": re.DOTALL,
    "VERBOSE": re.VERBOSE,
    "LOCALE": re.LOCALE,
    "ASCII": re.ASCII,
}


def c_str(b):
    """Encode bytes/str as a C string literal. Escapes '?' as '\\?' (a
    standard C escape for a literal '?', otherwise unnecessary) so that
    no two-or-more-'?' run can ever form a trigraph sequence (??=, ??),
    ??!, and friends), which a strict-mode C compiler silently rewrites
    INSIDE string literals before the string is ever parsed as string
    content: "(?:.??)" (7 bytes) became "(?:.]" (5 bytes) under this
    project's own build flags before this fix, corrupting every
    generated pattern containing that sequence. Confirmed necessary,
    not theoretical, by compiling and printing the corrupted string."""
    if isinstance(b, str):
        b = b.encode("utf-8")
    out = ['"']
    for byte in b:
        c = chr(byte)
        if c == '"':
            out.append('\\"')
        elif c == "\\":
            out.append("\\\\")
        elif c == "?":
            out.append("\\?")
        elif 32 <= byte < 127:
            out.append(c)
        else:
            out.append("\\%03o" % byte)
    out.append('"')
    return "".join(out)


def py_flags(case):
    # re.ASCII is not added here for mode == "ascii": ground truth for
    # that mode is already a `bytes` pattern against a `bytes` subject
    # (encode(), above), and a bytes pattern is inherently ASCII-only
    # for \w/\s/\d regardless of re.ASCII, so adding it would be purely
    # redundant, and actively wrong whenever a case also uses LOCALE,
    # since Python rejects ASCII and LOCALE combined (confirmed
    # directly: ValueError: ASCII and LOCALE flags are incompatible).
    f = 0
    for name in case["flags"]:
        f |= FLAGMAP[name]
    return f


def c_flags(case):
    names = []
    if case["mode"] == "ascii":
        names.append("ASCII")
    elif case["mode"] == "utf8":
        names.append("UTF8")
    elif case["mode"] == "binary":
        names.append("BINARY")
    for name in case["flags"]:
        names.append(name)
    return "|".join(names) if names else "0"


def encode(case, s):
    # ASCII mode is byte oriented, exactly like BINARY (regexx.c's
    # resolve_mode/build_matbuf treat them identically: raw bytes, no
    # UTF-8 decoding); it differs from BINARY only in which bytes \w/\s/
    # \d treat as word/space/digit, not in the character model. Python's
    # re.ASCII on a str is not the right ground truth for that: it still
    # operates on code points, only restricting \w/\s/\d's *definition*,
    # so it silently diverges from a byte-oriented match the moment a
    # subject contains a multi-byte UTF-8 character (confirmed directly:
    # str+re.ASCII on "naïve" gives (3,5) for the second \w+, the actual
    # byte-oriented match is (4,6), and regexx's ASCII mode reports the
    # latter, correctly). A `bytes` pattern against a `bytes` subject is
    # always byte oriented with ASCII-only \w/\s/\d in Python too, so it
    # is the correct ground truth for both ASCII and BINARY mode here.
    if isinstance(s, bytes):
        return s  # already raw bytes (arbitrary, not necessarily valid UTF-8): pass through
    if case["mode"] in ("binary", "ascii"):
        return s.encode("utf-8")
    return s


def blen(case, s):
    """Byte length of the encoded subject, as the C API always wants a
    byte count (UTF8 mode still stores a byte buffer; only Match_start
    reports code points), never a Python character count."""
    e = encode(case, s)
    return len(e) if isinstance(e, bytes) else len(e.encode("utf-8"))


def desc(case, i):
    return "#%d %s /%s/ on %r" % (i, case["op"], case["pattern"], case["subject"])


def gen_search_family(case, i, out):
    kind = {"search": 0, "match": 1, "fullmatch": 2}[case["op"]]
    flags = py_flags(case)
    subj = encode(case, case["subject"])
    pat = encode(case, case["pattern"])
    compiled = re.compile(pat, flags)
    fn = {"search": compiled.search, "match": compiled.match, "fullmatch": compiled.fullmatch}[case["op"]]
    m = fn(subj)
    d = desc(case, i)
    if m is None:
        out.append('    tc_search_family(%s, %s, %s, %s, %d, %d, 0, 0, NULL);' % (
            c_str(d), c_str(case["pattern"]), c_flags(case), c_str(case["subject"]), blen(case, case["subject"]), kind))
        return
    spans = [m.start(0), m.end(0)]
    for g in range(1, compiled.groups + 1):
        try:
            spans += [m.start(g), m.end(g)]
        except IndexError:
            spans += [-1, -1]
    n_spans = compiled.groups + 1
    arr = ",".join(str(v) for v in spans)
    out.append('    { static const long long sp[] = {%s}; tc_search_family(%s, %s, %s, %s, %d, %d, 1, %d, sp); }' % (
        arr, c_str(d), c_str(case["pattern"]), c_flags(case), c_str(case["subject"]), blen(case, case["subject"]), kind, n_spans))


def gen_finditer(case, i, out):
    flags = py_flags(case)
    subj = encode(case, case["subject"])
    pat = encode(case, case["pattern"])
    ms = list(re.finditer(pat, subj, flags))
    spans = []
    for m in ms:
        spans += [m.start(0), m.end(0)]
    d = desc(case, i)
    if not spans:
        out.append('    tc_finditer(%s, %s, %s, %s, %d, 0, NULL);' % (
            c_str(d), c_str(case["pattern"]), c_flags(case), c_str(case["subject"]), blen(case, case["subject"])))
        return
    arr = ",".join(str(v) for v in spans)
    out.append('    { static const long long sp[] = {%s}; tc_finditer(%s, %s, %s, %s, %d, %d, sp); }' % (
        arr, c_str(d), c_str(case["pattern"]), c_flags(case), c_str(case["subject"]), blen(case, case["subject"]), len(ms)))


def gen_sub(case, i, out):
    flags = py_flags(case)
    subj = encode(case, case["subject"])
    pat = encode(case, case["pattern"])
    repl = encode(case, case["repl"])
    count = case.get("count", 0)
    result, n = re.subn(pat, repl, subj, count=count, flags=flags)
    d = desc(case, i)
    result_str = result.decode("utf-8") if isinstance(result, bytes) else result
    out.append('    tc_sub(%s, %s, %s, %s, %d, %s, %d, %s, %d);' % (
        c_str(d), c_str(case["pattern"]), c_flags(case), c_str(case["subject"]), blen(case, case["subject"]),
        c_str(case["repl"]), count, c_str(result_str), n))


def gen_split(case, i, out):
    flags = py_flags(case)
    subj = encode(case, case["subject"])
    pat = encode(case, case["pattern"])
    maxsplit = case.get("maxsplit", 0)
    items = re.split(pat, subj, maxsplit=maxsplit, flags=flags)
    d = desc(case, i)
    parts = []
    for it in items:
        if it is None:
            parts.append("NULL")
        else:
            s = it.decode("utf-8") if isinstance(it, bytes) else it
            parts.append(c_str(s))
    arr = ",".join(parts) if parts else "0"
    out.append('    { static const char *const items[] = {%s}; tc_split(%s, %s, %s, %s, %d, %d, %d, items); }' % (
        arr, c_str(d), c_str(case["pattern"]), c_flags(case), c_str(case["subject"]), blen(case, case["subject"]),
        maxsplit, len(items)))


def main():
    out = []
    out.append('/* GENERATED by tests/gen.py from tests/cases.py. Do not edit by hand. */')
    out.append('#include "harness.h"')
    out.append('void run_generated_tests(void) {')
    emitted = 0
    skipped = 0
    for i, case in enumerate(CASES):
        op = case["op"]
        if op not in ("search", "match", "fullmatch", "finditer", "sub", "split"):
            raise ValueError("unknown op %r" % op)  # a typo in a hand-written case, not a skippable combination
        try:
            if op in ("search", "match", "fullmatch"):
                gen_search_family(case, i, out)
            elif op == "finditer":
                gen_finditer(case, i, out)
            elif op == "sub":
                gen_sub(case, i, out)
            elif op == "split":
                gen_split(case, i, out)
            emitted += 1
        except (re.error, ValueError) as e:
            # A combinatorially generated pattern that CPython itself
            # cannot compile is not a useful ground-truth case (there is
            # nothing to check regexx against); skip it rather than
            # aborting the whole generation run. Hand-written cases are
            # never expected to hit this path, so seeing it fire on one
            # is a signal to look at the offending combination, not to
            # silently rely on the skip.
            skipped += 1
            print("skip #%d (%s %r): %s" % (i, op, case["pattern"], e), file=sys.stderr)
    out.append('}')
    dest = os.path.join(os.path.dirname(__file__), "generated_tests.c")
    with open(dest, "w") as f:
        f.write("\n".join(out) + "\n")
    print("wrote %s (%d cases emitted, %d skipped)" % (dest, emitted, skipped))


if __name__ == "__main__":
    main()