regexx/examples/bench_vs_posix.c

/* bench_vs_posix - a direct, measured comparison between this library and
 * the C standard library's own regex engine (POSIX <regex.h>,
 * regcomp/regexec, glibc's implementation on the platform this is built
 * on). Every other example in this directory demonstrates a feature; this
 * one instead demonstrates a limitation, honestly, at scale: regexx is a
 * roughly 2000-line single-file backtracking interpreter (concept.md
 * Section 7.3) built for Python `re` compatibility and convenience,
 * competing against a DFA-backed engine that has been the default regex
 * implementation on every Linux system for decades. Nothing here is
 * rigged to make either engine look better; every pattern used is valid
 * POSIX ERE syntax AND valid regexx syntax (no \d/\w/\s, no POSIX bracket
 * classes, no lazy quantifiers), so both engines run the identical
 * pattern text, and every scenario reports both engines' actual timing,
 * including the ones where POSIX wins by a wide margin.
 *
 * Six scenarios, each at multi-megabyte or multi-hundred-thousand-line
 * scale:
 *   A. A literal needle near the end of a large non-matching haystack
 *      (tests raw linear-scan throughput).
 *   B. Extracting every number from a large text (tests iterated-match
 *      throughput, not just a single search).
 *   C. `a*b` against a large non-matching run of 'a' characters (the
 *      pattern shape README.md documents this library's precomputed
 *      skip-ahead tables were built specifically to fix).
 *   D. `(a+)+b` against a non-matching run of 'a' characters (the textbook
 *      ReDoS shape): the plain form, and, for regexx only, since POSIX ERE
 *      has no equivalent syntax at all, the atomic-group form that avoids
 *      it (examples/redos_atomic.c covers this one feature in isolation).
 *   E. Case-insensitive alternation over a large text with many matches.
 *   F. Anchored, per-line matching over several hundred thousand lines
 *      (the shape examples/ascii_logparse.c and rxgrep.c's line mode
 *      both use for real).
 *
 * A POSIX regex operates on a NUL-terminated C string with no notion of
 * "here is how many bytes, keep going past an embedded NUL if there are
 * any"; regexx's BINARY mode (examples/binary_scan.c) exists specifically
 * because POSIX <regex.h> cannot do that at all, at any speed, which is a
 * capability difference this file does not attempt to benchmark, since
 * there is no POSIX baseline to measure against for it.
 */
#define _POSIX_C_SOURCE 199309L
#include "regexx.h"
#include <regex.h>
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <time.h>

/* Iterated matching against POSIX <regex.h> is done with REG_STARTEND
 * (a widely available GNU/BSD extension to POSIX regexec, unconditionally
 * declared by glibc's regex.h with no feature-test macro needed, unlike
 * the rest of that header's GNU-only surface) rather than by advancing a
 * `const char *` pointer and calling plain regexec() repeatedly. The
 * pointer-advancing approach was tried first and measured to be quadratic
 * in practice: without REG_STARTEND, regexec() has no way to know where
 * the searchable string ends except by scanning for a NUL byte, so every
 * call after the first rescans however much of the buffer remains,
 * turning "run this pattern m times over an n-byte buffer" into O(n*m)
 * instead of O(n). REG_STARTEND instead takes explicit [rm_so, rm_eo)
 * byte offsets into the original, fixed buffer pointer, so no rescan
 * happens and no NUL terminator is consulted at all (this is also,
 * incidentally, the mechanism a POSIX-based program would use to search a
 * buffer containing embedded NUL bytes, though this file does not
 * otherwise exercise that). Deliberately not built with _GNU_SOURCE: that
 * macro additionally exposes glibc's older re_search/re_match API, whose
 * declarations collide with this project's own re_search/re_match
 * (regexx.h, Python-exact naming, concept.md Section 9), so this file
 * only enables what REG_STARTEND itself needs. */

static double now_seconds(void) {
    struct timespec ts;
    clock_gettime(CLOCK_MONOTONIC, &ts);
    return (double)ts.tv_sec + (double)ts.tv_nsec / 1e9;
}

typedef struct {
    const char *label;
    double regexx_time;
    double posix_time;
    int have_posix;      /* 0 for the one regexx-only row (atomic group) */
    long regexx_count;   /* matches found, or 0/1 for a single search */
    long posix_count;
    const char *unit;    /* what regexx_time/posix_time are measuring, for the table */
} Result;

#define MAX_RESULTS 16
static Result results[MAX_RESULTS];
static int nresults = 0;

static Result *add_result(const char *label, const char *unit) {
    Result *r = &results[nresults++];
    memset(r, 0, sizeof *r);
    r->label = label;
    r->unit = unit;
    r->have_posix = 1;
    return r;
}

/* ---- Scenario A: literal needle near the end of a large haystack ---- */
static void scenario_a(void) {
    size_t n = 40u * 1024 * 1024;
    char *buf = malloc(n + 1);
    memset(buf, 'x', n);
    const char *needle = "needle_in_a_haystack_42";
    size_t nl = strlen(needle);
    memcpy(buf + n - nl - 100, needle, nl); /* near, not at, the very end */
    buf[n] = '\0';

    Result *r = add_result("A: literal search, 40MB haystack", "search");

    PatternError err; memset(&err, 0, sizeof err);
    Pattern *pat = re_compile(needle, nl, ASCII, &err);
    Input *in = Input_from_buffer((const uint8_t *)buf, n);
    Match m; memset(&m, 0, sizeof m);
    double t0 = now_seconds();
    int rc = Pattern_search(pat, in, 0, -1, &m);
    r->regexx_time = now_seconds() - t0;
    r->regexx_count = (rc == 1);
    if (rc == 1) Match_free(&m);
    Input_free(in);
    Pattern_free(pat);

    regex_t re;
    regcomp(&re, needle, REG_EXTENDED | REG_NOSUB);
    regmatch_t pm;
    t0 = now_seconds();
    int prc = regexec(&re, buf, 0, &pm, 0);
    r->posix_time = now_seconds() - t0;
    r->posix_count = (prc == 0);
    regfree(&re);

    free(buf);
}

/* ---- Scenario B: extract every number from a large text ---- */
static void print_count_cb(void *ctx, const Match *m) {
    (void)m;
    long *n = ctx;
    (*n)++;
}
static void scenario_b(void) {
    size_t target = 20u * 1024 * 1024;
    size_t cap = target + 4096;
    char *buf = malloc(cap);
    size_t len = 0;
    unsigned long seed = 1; /* unsigned: wraps well-defined on overflow, unlike signed long */
    while (len < target) {
        seed = seed * 1103515245u + 12345u;
        int v = (int)((seed >> 16) & 0xFFFF);
        len += (size_t)snprintf(buf + len, cap - len, "id%d val%d code%d ", v, v * 3, v % 97);
    }
    buf[len] = '\0';

    Result *r = add_result("B: extract all numbers, 20MB text", "matches");
    const char *pattern = "[0-9]+";

    PatternError err; memset(&err, 0, sizeof err);
    Pattern *pat = re_compile(pattern, strlen(pattern), ASCII, &err);
    Input *in = Input_from_buffer((const uint8_t *)buf, len);
    long count = 0;
    double t0 = now_seconds();
    Pattern_finditer(pat, in, 0, -1, print_count_cb, &count);
    r->regexx_time = now_seconds() - t0;
    r->regexx_count = count;
    Input_free(in);
    Pattern_free(pat);

    regex_t re;
    regcomp(&re, pattern, REG_EXTENDED);
    long pcount = 0;
    size_t start = 0;
    regmatch_t pm;
    t0 = now_seconds();
    while (start <= len) {
        pm.rm_so = (regoff_t)start;
        pm.rm_eo = (regoff_t)len;
        if (regexec(&re, buf, 1, &pm, REG_STARTEND | (start == 0 ? 0 : REG_NOTBOL)) != 0) break;
        pcount++;
        size_t mso = (size_t)pm.rm_so, meo = (size_t)pm.rm_eo;
        start = (meo > mso) ? meo : meo + 1;
    }
    r->posix_time = now_seconds() - t0;
    r->posix_count = pcount;
    regfree(&re);

    free(buf);
}

/* ---- Scenario C: a*b, non-matching, over a large run of 'a' ---- */
static void scenario_c(void) {
    size_t n = 20u * 1024 * 1024;
    char *buf = malloc(n + 1);
    memset(buf, 'a', n);
    buf[n] = '\0';

    Result *r = add_result("C: a*b non-match, 20MB of 'a'", "search");
    const char *pattern = "a*b";

    PatternError err; memset(&err, 0, sizeof err);
    Pattern *pat = re_compile(pattern, strlen(pattern), ASCII, &err);
    Input *in = Input_from_buffer((const uint8_t *)buf, n);
    Match m; memset(&m, 0, sizeof m);
    double t0 = now_seconds();
    int rc = Pattern_search(pat, in, 0, -1, &m);
    r->regexx_time = now_seconds() - t0;
    r->regexx_count = (rc == 1);
    if (rc == 1) Match_free(&m);
    Input_free(in);
    Pattern_free(pat);

    regex_t re;
    regcomp(&re, pattern, REG_EXTENDED | REG_NOSUB);
    regmatch_t pm;
    t0 = now_seconds();
    int prc = regexec(&re, buf, 0, &pm, 0);
    r->posix_time = now_seconds() - t0;
    r->posix_count = (prc == 0);
    regfree(&re);

    free(buf);
}

/* ---- Scenario D: (a+)+b, the textbook ReDoS shape, non-matching ---- */
static void scenario_d(void) {
    size_t n = 22000;
    char *buf = malloc(n + 1);
    memset(buf, 'a', n);
    buf[n] = '\0';

    Result *r = add_result("D1: (a+)+b non-match, plain, 22000 'a'", "search");
    const char *pattern = "(a+)+b";

    PatternError err; memset(&err, 0, sizeof err);
    Pattern *pat = re_compile(pattern, strlen(pattern), ASCII, &err);
    Input *in = Input_from_buffer((const uint8_t *)buf, n);
    Match m; memset(&m, 0, sizeof m);
    double t0 = now_seconds();
    int rc = Pattern_search(pat, in, 0, -1, &m);
    r->regexx_time = now_seconds() - t0;
    r->regexx_count = (rc == 1);
    if (rc == 1) Match_free(&m);
    Input_free(in);
    Pattern_free(pat);

    regex_t re;
    regcomp(&re, pattern, REG_EXTENDED | REG_NOSUB);
    regmatch_t pm;
    t0 = now_seconds();
    int prc = regexec(&re, buf, 0, &pm, 0);
    r->posix_time = now_seconds() - t0;
    r->posix_count = (prc == 0);
    regfree(&re);

    /* Second row: the atomic-group rewrite. POSIX ERE has no atomic
     * group or possessive quantifier syntax at all (not "slower", simply
     * inexpressible), so this row has no POSIX column. */
    Result *r2 = add_result("D2: (?>a+)+b non-match, atomic, same input", "search");
    r2->have_posix = 0;
    const char *pattern2 = "(?>a+)+b";
    PatternError err2; memset(&err2, 0, sizeof err2);
    Pattern *pat2 = re_compile(pattern2, strlen(pattern2), ASCII, &err2);
    Input *in2 = Input_from_buffer((const uint8_t *)buf, n);
    Match m2; memset(&m2, 0, sizeof m2);
    t0 = now_seconds();
    int rc2 = Pattern_search(pat2, in2, 0, -1, &m2);
    r2->regexx_time = now_seconds() - t0;
    r2->regexx_count = (rc2 == 1);
    if (rc2 == 1) Match_free(&m2);
    Input_free(in2);
    Pattern_free(pat2);

    free(buf);
}

/* ---- Scenario E: case-insensitive alternation, many matches ---- */
static void scenario_e(void) {
    const char *animals[] = { "Cat", "DOG", "bird", "FiSh", "snake" };
    size_t target = 12u * 1024 * 1024;
    size_t cap = target + 4096;
    char *buf = malloc(cap);
    size_t len = 0;
    int i = 0;
    while (len < target) {
        len += (size_t)snprintf(buf + len, cap - len, "%s and some filler text here. ", animals[i % 5]);
        i++;
    }
    buf[len] = '\0';

    Result *r = add_result("E: case-insensitive alternation, 12MB", "matches");
    const char *pattern = "(cat|dog|bird|fish|snake)";

    PatternError err; memset(&err, 0, sizeof err);
    Pattern *pat = re_compile(pattern, strlen(pattern), ASCII | IGNORECASE, &err);
    Input *in = Input_from_buffer((const uint8_t *)buf, len);
    long count = 0;
    double t0 = now_seconds();
    Pattern_finditer(pat, in, 0, -1, print_count_cb, &count);
    r->regexx_time = now_seconds() - t0;
    r->regexx_count = count;
    Input_free(in);
    Pattern_free(pat);

    regex_t re;
    regcomp(&re, pattern, REG_EXTENDED | REG_ICASE);
    long pcount = 0;
    size_t start = 0;
    regmatch_t pm;
    t0 = now_seconds();
    while (start <= len) {
        pm.rm_so = (regoff_t)start;
        pm.rm_eo = (regoff_t)len;
        if (regexec(&re, buf, 1, &pm, REG_STARTEND | (start == 0 ? 0 : REG_NOTBOL)) != 0) break;
        pcount++;
        size_t mso = (size_t)pm.rm_so, meo = (size_t)pm.rm_eo;
        start = (meo > mso) ? meo : meo + 1;
    }
    r->posix_time = now_seconds() - t0;
    r->posix_count = pcount;
    regfree(&re);

    free(buf);
}

/* ---- Scenario F: anchored per-line matching, many lines ---- */
static void scenario_f(void) {
    long nlines = 400000;
    size_t cap = (size_t)nlines * 64;
    char *buf = malloc(cap);
    size_t len = 0;
    for (long i = 0; i < nlines; i++) {
        const char *level = (i % 10 == 0) ? "ERROR" : (i % 3 == 0) ? "WARN" : "INFO";
        len += (size_t)snprintf(buf + len, cap - len, "%s message number %ld here\n", level, i);
    }
    buf[len] = '\0';

    Result *r = add_result("F: '^ERROR', 400000 lines", "matches");
    const char *pattern = "^ERROR";

    PatternError err; memset(&err, 0, sizeof err);
    Pattern *pat = re_compile(pattern, strlen(pattern), ASCII, &err);
    long rcount = 0;
    double t0 = now_seconds();
    {
        size_t start = 0;
        for (size_t pos = 0; pos <= len; pos++) {
            if (pos == len || buf[pos] == '\n') {
                size_t linelen = pos - start;
                Input *lin = Input_from_buffer((const uint8_t *)(buf + start), linelen);
                Match m; memset(&m, 0, sizeof m);
                if (Pattern_search(pat, lin, 0, -1, &m) == 1) { rcount++; Match_free(&m); }
                Input_free(lin);
                start = pos + 1;
            }
        }
    }
    r->regexx_time = now_seconds() - t0;
    r->regexx_count = rcount;
    Pattern_free(pat);

    regex_t re;
    regcomp(&re, pattern, REG_EXTENDED | REG_NOSUB);
    long pcount = 0;
    t0 = now_seconds();
    {
        char *line = malloc(256);
        size_t start = 0;
        for (size_t pos = 0; pos <= len; pos++) {
            if (pos == len || buf[pos] == '\n') {
                size_t linelen = pos - start;
                if (linelen < 256) {
                    memcpy(line, buf + start, linelen);
                    line[linelen] = '\0';
                    regmatch_t pm;
                    if (regexec(&re, line, 0, &pm, 0) == 0) pcount++;
                }
                start = pos + 1;
            }
        }
        free(line);
    }
    r->posix_time = now_seconds() - t0;
    r->posix_count = pcount;
    regfree(&re);

    free(buf);
}

static void bar(double frac, char *out, int width) {
    int filled = (int)(frac * width);
    if (filled > width) filled = width;
    if (filled < 0) filled = 0;
    for (int i = 0; i < width; i++) out[i] = (i < filled) ? '#' : '.';
    out[width] = '\0';
}

int main(void) {
    printf("regexx vs POSIX <regex.h>: measured, not estimated.\n");
    printf("Every pattern below is valid syntax for both engines; both run\n");
    printf("against the identical subject text. Times are wall-clock seconds\n");
    printf("(CLOCK_MONOTONIC), one run each, on whatever machine this runs on.\n\n");

    scenario_a();
    scenario_b();
    scenario_c();
    scenario_d();
    scenario_e();
    scenario_f();

    printf("%-42s %12s %12s %10s %8s\n", "scenario", "regexx", "POSIX", "ratio", "match?");
    for (int i = 0; i < nresults; i++) {
        Result *r = &results[i];
        char barbuf[41];
        if (r->have_posix) {
            double ratio = r->posix_time > 0 ? r->regexx_time / r->posix_time : 0;
            printf("%-42s %10.4fs %10.4fs %9.1fx %8s\n",
                   r->label, r->regexx_time, r->posix_time, ratio,
                   (r->regexx_count == r->posix_count) ? "agree" : "DIFFER");
            bar(r->regexx_time / (r->regexx_time + r->posix_time), barbuf, 40);
            printf("    regexx [%s] POSIX [", barbuf);
            bar(r->posix_time / (r->regexx_time + r->posix_time), barbuf, 40);
            printf("%s]\n", barbuf);
        } else {
            printf("%-42s %10.4fs %12s %9s %8s\n",
                   r->label, r->regexx_time, "n/a", "n/a", "n/a");
            printf("    (no POSIX ERE equivalent syntax exists for an atomic group)\n");
        }
    }

    printf("\nWhat this measures and does not measure: regexx is Python `re`\n"
           "compatible and dispatches every pattern here (all backreference-,\n"
           "lookaround-, and atomic-group-free) to its Pike VM (README.md \"The\n"
           "Pike VM\", concept.md 7.2/7.7/7.9), a Thompson-NFA simulation with a\n"
           "literal prefilter (added after this benchmark first found scenario A\n"
           "regressed the most, concept.md 7.9) but still no lazy DFA state\n"
           "caching or allocation pooling; glibc's <regex.h> is a mature,\n"
           "DFA-backed engine with decades of that exact kind of optimization\n"
           "behind it and a much smaller feature set (no named groups, no lazy\n"
           "quantifiers, no lookaround, no atomic groups, and, structurally, no\n"
           "way to search past an embedded NUL byte at all, docs/API.md Section\n"
           "1.3 vs examples/binary_scan.c). Scenarios A-C measure what remains of\n"
           "that missing optimization layer: this build was previously measured\n"
           "on this engine's backtracking predecessor plus its own precomputed\n"
           "skip-ahead tables (README.md \"Implementation status\") at roughly\n"
           "2x-25x against POSIX on these same three scenarios; the Pike VM\n"
           "without a prefilter regressed that to roughly 8x-70x, and the\n"
           "prefilter closed most of that back up for A (a plain literal, its\n"
           "best case) and part of it for B (starts with a class, not a single\n"
           "literal); C's a*b starts with a nullable loop no prefilter can help,\n"
           "and is unchanged. What remains (roughly 7x-27x here) is genuinely no\n"
           "lazy DFA caching or allocation pooling yet, not a defect.\n\n"
           "Scenario D inverts entirely, and is the actual point of building this\n"
           "engine at all: (a+)+b has no backreference, so it is Pike VM eligible,\n"
           "and a Thompson-NFA simulation has no notion of \"try one split, then\n"
           "backtrack and try another\" to begin with, so the classic nested-\n"
           "quantifier ambiguity this pattern is famous for causing simply does\n"
           "not exist for this engine; row D1 now runs this pattern faster than\n"
           "POSIX <regex.h> itself, with no atomic group, not merely fast enough\n"
           "to stop being the slowest thing on this page. Row D2 (still with no\n"
           "POSIX equivalent) is retained for comparison, not because D1 needs\n"
           "fixing anymore: examples/redos_atomic.c's Part 2 shows a pattern this\n"
           "specific improvement cannot reach (a backreference forces the older\n"
           "engine regardless of shape), which is where an atomic group remains\n"
           "the pattern author's own necessary tool, not something automatic.\n");

    return 0;
}