/* regexx.c - a single-file C regex interpreter with Python `re` semantics. * * See concept.md for the full design rationale. This file is the v1 * implementation: it covers the public API and pattern syntax listed in * README.md "Implementation status" in full, executed by a single * recursive backtracking engine (concept.md Section 7.3) over a fully * materialized copy of the input. It does not yet implement the * streaming, bounded-memory regular engine of concept.md Section 7.2; * that remains future work, tracked in README.md. * * That backtracking engine is memoized (run_memo) and, for quantifiers * over a single character/class/./ (OP_REPEAT1), backed by precomputed * run-length and skip-ahead tables (compute_maxrun, * compute_next_prevmatch), so that Pattern_search/finditer/split/sub * run in linear rather than quadratic time for the common, * backreference-free case; see README.md "Implementation status" for * the measured numbers, the precise remaining limitations, and * citations to the published techniques this matches. * * Naming convention (concept.md Section 9.0): every identifier with a * direct Python `re` counterpart uses that counterpart's exact spelling. * * Not reentrant: re_compile() uses one process-wide parser scratch pool * and the module level functions share one process-wide pattern cache * (concept.md 13.6); do not call regexx functions from more than one * thread without external synchronization. */ #define _DEFAULT_SOURCE #include "regexx.h" #include #include #include #include #include #include #include #include #include #include #include #include #include /* ================================================================ * 0. Small utilities: dynamic array, allocation helpers * ================================================================ */ typedef struct { void *data; size_t len, cap, elemsize; } DArr; static void da_init(DArr *a, size_t elemsize) { a->data = NULL; a->len = 0; a->cap = 0; a->elemsize = elemsize; } /* Returns NULL on allocation failure, leaving `a` exactly as it was * (never loses or corrupts the existing elements): most callers in * this file grow bounded-by-pattern-size structures (the AST pool, * class items, the instruction array), where a failure here is * already effectively unreachable in practice and is not separately * checked; the two callers that grow a structure sized to the * *input* (build_matbuf's UTF-8 decode arrays, the only other * uses that can plausibly reach real-world allocation limits) do * check it. */ static void *da_push(DArr *a) { if (a->len == a->cap) { size_t nc = a->cap ? a->cap * 2 : 8; void *nd = realloc(a->data, nc * a->elemsize); if (!nd) return NULL; a->data = nd; a->cap = nc; } void *p = (char *)a->data + a->len * a->elemsize; a->len++; return p; } static void da_free(DArr *a) { free(a->data); a->data = NULL; a->len = a->cap = 0; } static char *xstrndup(const char *s, size_t n) { char *r = malloc(n + 1); memcpy(r, s, n); r[n] = 0; return r; } static void ensure_unicode_locale(void) { static int done = 0; if (!done) { setlocale(LC_CTYPE, "C.utf8"); done = 1; } } /* ================================================================ * 1. UTF-8 decoding (concept.md Section 7.2's continuation handling is * not needed here since v1 materializes the whole input up front, * see README.md "Implementation status") * ================================================================ */ static int utf8_decode(const uint8_t *buf, size_t len, size_t i, uint32_t *cp) { uint8_t b0 = buf[i]; if (b0 < 0x80) { *cp = b0; return 1; } if ((b0 & 0xE0) == 0xC0) { if (i + 1 >= len || (buf[i+1] & 0xC0) != 0x80) return 0; uint32_t v = (uint32_t)((b0 & 0x1F) << 6) | (buf[i+1] & 0x3F); if (v < 0x80) return 0; *cp = v; return 2; } if ((b0 & 0xF0) == 0xE0) { if (i + 2 >= len || (buf[i+1] & 0xC0) != 0x80 || (buf[i+2] & 0xC0) != 0x80) return 0; uint32_t v = ((uint32_t)(b0 & 0x0F) << 12) | ((uint32_t)(buf[i+1] & 0x3F) << 6) | (buf[i+2] & 0x3F); if (v < 0x800 || (v >= 0xD800 && v <= 0xDFFF)) return 0; *cp = v; return 3; } if ((b0 & 0xF8) == 0xF0) { if (i + 3 >= len || (buf[i+1] & 0xC0) != 0x80 || (buf[i+2] & 0xC0) != 0x80 || (buf[i+3] & 0xC0) != 0x80) return 0; uint32_t v = ((uint32_t)(b0 & 0x07) << 18) | ((uint32_t)(buf[i+1] & 0x3F) << 12) | ((uint32_t)(buf[i+2] & 0x3F) << 6) | (buf[i+3] & 0x3F); if (v < 0x10000 || v > 0x10FFFF) return 0; *cp = v; return 4; } return 0; } /* ================================================================ * 2. Character classification predicates (mode aware, Section 2.2/14.4) * ================================================================ */ #define MODE_BINARY 0 #define MODE_ASCII 1 #define MODE_UTF8 2 static int is_ascii_word(uint32_t cp) { return (cp >= 'a' && cp <= 'z') || (cp >= 'A' && cp <= 'Z') || (cp >= '0' && cp <= '9') || cp == '_'; } static int is_ascii_space(uint32_t cp) { return cp == ' ' || cp == '\t' || cp == '\n' || cp == '\r' || cp == '\f' || cp == '\v'; } static int is_ascii_digit(uint32_t cp) { return cp >= '0' && cp <= '9'; } static int cls_is_digit(uint32_t cp, int mode, int flags) { if (mode == MODE_UTF8 && !(flags & ASCII)) { ensure_unicode_locale(); return iswdigit((wint_t)cp) ? 1 : 0; } return is_ascii_digit(cp); } static int cls_is_space(uint32_t cp, int mode, int flags) { if (mode == MODE_UTF8 && !(flags & ASCII)) { ensure_unicode_locale(); return iswspace((wint_t)cp) ? 1 : 0; } return is_ascii_space(cp); } static int cls_is_word(uint32_t cp, int mode, int flags) { if (mode == MODE_UTF8 && !(flags & ASCII)) { ensure_unicode_locale(); return (iswalnum((wint_t)cp) || cp == '_') ? 1 : 0; } return is_ascii_word(cp); /* LOCALE has no additional effect here, see below */ /* In BINARY/ASCII mode, LOCALE (Section 2.6) is intentionally a * no-op relative to plain ASCII classification: the C standard * guarantees the "C" locale's isalnum()/isalpha() classify only * the ASCII letters/digits, nothing in 0x80-0xFF, and CPython's own * re.LOCALE, checked directly under an explicit "C" locale, agrees * (rb'\w+' against b"abc\x80\x90def" still stops at "abc"). An * earlier revision of this function instead treated every byte in * 0x80-0xFF as a word character under LOCALE, based on an * unverified assumption about what "the C locale" does; the large * combinatorial test expansion (tests/cases.py Category H) caught * the resulting mismatch against a real CPython interpreter, which * is what prompted checking the premise directly (see above) and * finding it false. README.md "Known deviations" records this. */ } static uint32_t cls_swapcase(uint32_t cp, int mode, int flags) { if (mode == MODE_UTF8 && !(flags & ASCII)) { ensure_unicode_locale(); wint_t lo = towlower((wint_t)cp), up = towupper((wint_t)cp); if ((uint32_t)lo != cp) return (uint32_t)lo; if ((uint32_t)up != cp) return (uint32_t)up; return cp; } if (cp >= 'a' && cp <= 'z') return cp - 32; if (cp >= 'A' && cp <= 'Z') return cp + 32; return cp; } static uint32_t cls_fold(uint32_t cp, int mode, int flags) { if (mode == MODE_UTF8 && !(flags & ASCII)) { ensure_unicode_locale(); return (uint32_t)towlower((wint_t)cp); } if (cp >= 'A' && cp <= 'Z') return cp + 32; return cp; } /* ================================================================ * 3. AST * ================================================================ */ enum { N_CHAR, N_ANY, N_CLASS, N_CONCAT, N_ALT, N_REPEAT, N_GROUP, N_BACKREF, N_ANCHOR, N_LOOKAROUND, N_ATOMIC, N_EMPTY }; enum { A_BOL, A_EOL, A_BOS, A_EOS, A_WB, A_NWB }; enum { LK_AHEAD, LK_BEHIND }; enum { CI_RANGE, CI_D, CI_W, CI_S }; typedef struct { int kind; uint32_t lo, hi; int neg; } ClassItem; typedef struct Node { int kind; struct Node *a, *b; int32_t lo, hi; int greedy; uint32_t ch; DArr items; int class_negate; int group_index; char *group_name; int backref_index; char *backref_name; int anchor_kind; int lookaround_kind; int lookaround_negate; int width; } Node; typedef struct { Node **data; size_t len, cap; } NodePool; static NodePool g_pool; static Node *node_new(int kind) { if (g_pool.len == g_pool.cap) { g_pool.cap = g_pool.cap ? g_pool.cap * 2 : 64; g_pool.data = realloc(g_pool.data, g_pool.cap * sizeof(Node *)); } Node *n = calloc(1, sizeof(Node)); n->kind = kind; n->group_index = -1; n->backref_index = -1; n->greedy = 1; n->width = -1; da_init(&n->items, sizeof(ClassItem)); g_pool.data[g_pool.len++] = n; return n; } static void nodepool_free_all(NodePool *pool) { for (size_t i = 0; i < pool->len; i++) { Node *n = pool->data[i]; da_free(&n->items); free(n->group_name); free(n->backref_name); free(n); } free(pool->data); pool->data = NULL; pool->len = pool->cap = 0; } static int node_nullable(Node *n) { if (!n) return 1; switch (n->kind) { case N_CHAR: case N_ANY: case N_CLASS: case N_BACKREF: return 0; case N_CONCAT: return node_nullable(n->a) && node_nullable(n->b); case N_ALT: return node_nullable(n->a) || node_nullable(n->b); case N_REPEAT: return n->lo == 0 || node_nullable(n->a); case N_GROUP: return node_nullable(n->a); case N_ANCHOR: case N_LOOKAROUND: return 1; case N_ATOMIC: return node_nullable(n->a); default: return 1; } } static int node_width(Node *n) { if (!n) return 0; switch (n->kind) { case N_CHAR: case N_ANY: case N_CLASS: return 1; case N_CONCAT: { int wa = node_width(n->a), wb = node_width(n->b); return (wa < 0 || wb < 0) ? -1 : wa + wb; } case N_ALT: { int wa = node_width(n->a), wb = node_width(n->b); return (wa < 0 || wb < 0 || wa != wb) ? -1 : wa; } case N_REPEAT: { if (n->lo != n->hi) return -1; int w = node_width(n->a); return w < 0 ? -1 : w * n->lo; } case N_GROUP: return node_width(n->a); case N_BACKREF: return -1; case N_ANCHOR: case N_LOOKAROUND: return 0; case N_ATOMIC: return node_width(n->a); default: return 0; } } static int is_simple_atom(Node *n) { return n->kind == N_CHAR || n->kind == N_ANY || n->kind == N_CLASS; } /* ================================================================ * 4. Parser * ================================================================ */ typedef struct { char *name; int index; } GroupName; typedef struct { const char *s; size_t len, i; int flags; int mode; int ngroups; DArr backrefs; /* Node* */ DArr groupnames; /* GroupName */ char errmsg[256]; int64_t errpos; int failed; } Parser; static void perr(Parser *p, int64_t pos, const char *fmt, ...) { if (p->failed) return; p->failed = 1; p->errpos = pos; va_list ap; va_start(ap, fmt); vsnprintf(p->errmsg, sizeof p->errmsg, fmt, ap); va_end(ap); } static int peek(Parser *p) { return p->i < p->len ? (unsigned char)p->s[p->i] : -1; } static int peek2(Parser *p) { return p->i + 1 < p->len ? (unsigned char)p->s[p->i + 1] : -1; } static int at_end(Parser *p) { return p->i >= p->len; } static int hexval(int c) { if (c >= '0' && c <= '9') return c - '0'; if (c >= 'a' && c <= 'f') return c - 'a' + 10; if (c >= 'A' && c <= 'F') return c - 'A' + 10; return -1; } static Node *parse_alt(Parser *p); static Node *parse_escape(Parser *p, int in_class, uint32_t *literal_cp, int *is_literal) { *is_literal = 0; if (at_end(p)) { perr(p, (int64_t)p->i, "bad escape (end of pattern)"); return NULL; } int c = peek(p); p->i++; switch (c) { case 'n': *literal_cp = '\n'; *is_literal = 1; return NULL; case 'r': *literal_cp = '\r'; *is_literal = 1; return NULL; case 't': *literal_cp = '\t'; *is_literal = 1; return NULL; case 'f': *literal_cp = '\f'; *is_literal = 1; return NULL; case 'v': *literal_cp = '\v'; *is_literal = 1; return NULL; case 'a': *literal_cp = '\a'; *is_literal = 1; return NULL; case '0': { uint32_t v = 0; int n = 0; while (n < 2 && peek(p) >= '0' && peek(p) <= '7') { v = v * 8 + (uint32_t)(peek(p) - '0'); p->i++; n++; } *literal_cp = v; *is_literal = 1; return NULL; } case 'x': { int h1 = hexval(peek(p)); if (h1 < 0) { perr(p, (int64_t)p->i, "incomplete escape \\x"); return NULL; } p->i++; int h2 = hexval(peek(p)); if (h2 < 0) { perr(p, (int64_t)p->i, "incomplete escape \\x"); return NULL; } p->i++; *literal_cp = (uint32_t)(h1 * 16 + h2); *is_literal = 1; return NULL; } case 'u': case 'U': { int ndig = (c == 'u') ? 4 : 8; uint32_t v = 0; for (int k = 0; k < ndig; k++) { int h = hexval(peek(p)); if (h < 0) { perr(p, (int64_t)p->i, "incomplete escape \\%c", c); return NULL; } v = v * 16 + (uint32_t)h; p->i++; } *literal_cp = v; *is_literal = 1; return NULL; } case 'N': perr(p, (int64_t)p->i, "\\N{...} named code points are not implemented in this build"); return NULL; case 'd': case 'D': case 'w': case 'W': case 's': case 'S': { Node *n = node_new(N_CLASS); ClassItem *it = da_push(&n->items); it->kind = (c == 'd' || c == 'D') ? CI_D : (c == 'w' || c == 'W') ? CI_W : CI_S; it->neg = (c == 'D' || c == 'W' || c == 'S') ? 1 : 0; it->lo = it->hi = 0; return n; } case 'b': if (in_class) { *literal_cp = '\b'; *is_literal = 1; return NULL; } { Node *n = node_new(N_ANCHOR); n->anchor_kind = A_WB; return n; } case 'B': { Node *n = node_new(N_ANCHOR); n->anchor_kind = A_NWB; return n; } case 'A': { Node *n = node_new(N_ANCHOR); n->anchor_kind = A_BOS; return n; } case 'Z': { Node *n = node_new(N_ANCHOR); n->anchor_kind = A_EOS; return n; } case 'g': { if (in_class) { *literal_cp = 'g'; *is_literal = 1; return NULL; } if (peek(p) != '<') { perr(p, (int64_t)p->i, "missing < in \\g<...>"); return NULL; } p->i++; size_t start = p->i; while (!at_end(p) && peek(p) != '>') p->i++; if (at_end(p)) { perr(p, (int64_t)p->i, "missing > in \\g<...>"); return NULL; } char *ref = xstrndup(p->s + start, p->i - start); p->i++; Node *n = node_new(N_BACKREF); size_t rl = strlen(ref); int all_digit = rl > 0; for (size_t k = 0; k < rl; k++) if (!isdigit((unsigned char)ref[k])) all_digit = 0; if (all_digit) { n->backref_index = atoi(ref); free(ref); } else n->backref_name = ref; Node **slot = da_push(&p->backrefs); *slot = n; return n; } case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': { if (in_class) { *literal_cp = (uint32_t)c; *is_literal = 1; return NULL; } uint32_t v = (uint32_t)(c - '0'); while (peek(p) >= '0' && peek(p) <= '9') { v = v * 10 + (uint32_t)(peek(p) - '0'); p->i++; } Node *n = node_new(N_BACKREF); n->backref_index = (int)v; Node **slot = da_push(&p->backrefs); *slot = n; return n; } default: *literal_cp = (uint32_t)c; *is_literal = 1; return NULL; } } static uint32_t read_raw_char(Parser *p) { if (p->mode == MODE_UTF8) { uint32_t cp; int n = utf8_decode((const uint8_t *)p->s, p->len, p->i, &cp); if (n == 0) { perr(p, (int64_t)p->i, "invalid UTF-8 in pattern"); p->i++; return 0; } p->i += (size_t)n; return cp; } return (unsigned char)p->s[p->i++]; } static Node *parse_class(Parser *p) { Node *n = node_new(N_CLASS); p->i++; if (peek(p) == '^') { n->class_negate = 1; p->i++; } int first = 1; while (1) { if (at_end(p)) { perr(p, (int64_t)p->i, "unterminated character set"); return n; } if (peek(p) == ']' && !first) { p->i++; break; } first = 0; uint32_t lo; if (peek(p) == '\\') { p->i++; uint32_t litcp; int is_lit; Node *sub = parse_escape(p, 1, &litcp, &is_lit); if (p->failed) return n; if (!is_lit && sub) { ClassItem *it = da_push(&n->items); *it = ((ClassItem *)sub->items.data)[0]; continue; } lo = litcp; } else if (peek(p) == '[' && peek2(p) == ':') { perr(p, (int64_t)p->i, "POSIX named classes ([:alpha:] etc) are not part of Python re, see concept.md 14.3"); return n; } else { lo = read_raw_char(p); if (p->failed) return n; } uint32_t hi = lo; if (peek(p) == '-' && peek2(p) != ']' && peek2(p) != -1) { p->i++; if (peek(p) == '\\') { p->i++; uint32_t litcp; int is_lit; parse_escape(p, 1, &litcp, &is_lit); if (p->failed) return n; if (!is_lit) { perr(p, (int64_t)p->i, "bad character range"); return n; } hi = litcp; } else { hi = read_raw_char(p); if (p->failed) return n; } if (hi < lo) { perr(p, (int64_t)p->i, "bad character range"); return n; } } ClassItem *it = da_push(&n->items); it->kind = CI_RANGE; it->lo = lo; it->hi = hi; it->neg = 0; } return n; } static Node *make_concat(Node *a, Node *b) { if (!a) return b; if (!b) return a; Node *n = node_new(N_CONCAT); n->a = a; n->b = b; return n; } static Node *parse_repeat_suffix(Parser *p, Node *atom) { int lo = -1, hi = -1; if (peek(p) == '*') { lo = 0; hi = -1; p->i++; } else if (peek(p) == '+') { lo = 1; hi = -1; p->i++; } else if (peek(p) == '?') { lo = 0; hi = 1; p->i++; } else if (peek(p) == '{') { size_t save = p->i; p->i++; int have_lo = 0, have_hi = 0; long vlo = 0, vhi = 0; while (isdigit(peek(p))) { vlo = vlo * 10 + (peek(p) - '0'); p->i++; have_lo = 1; } int has_comma = 0; if (peek(p) == ',') { has_comma = 1; p->i++; while (isdigit(peek(p))) { vhi = vhi * 10 + (peek(p) - '0'); p->i++; have_hi = 1; } } if (peek(p) != '}' || (!have_lo && !has_comma)) { p->i = save; return atom; } p->i++; lo = have_lo ? (int)vlo : 0; hi = has_comma ? (have_hi ? (int)vhi : -1) : lo; if (hi != -1 && hi < lo) { perr(p, (int64_t)save, "min repeat greater than max repeat"); return atom; } } else { return atom; } Node *rep = node_new(N_REPEAT); rep->a = atom; rep->lo = lo; rep->hi = hi; rep->greedy = 1; if (peek(p) == '?') { rep->greedy = 0; p->i++; return rep; } if (peek(p) == '+') { p->i++; Node *at = node_new(N_ATOMIC); at->a = rep; return at; } return rep; } static Node *parse_group_or_special(Parser *p) { size_t open_pos = p->i; p->i++; if (peek(p) == '?') { p->i++; int c = peek(p); if (c == ':') { p->i++; Node *body = parse_alt(p); if (peek(p) != ')') { perr(p, (int64_t)p->i, "missing ), unterminated subpattern"); return body; } p->i++; Node *g = node_new(N_GROUP); g->a = body; g->group_index = -1; return g; } if (c == '#') { p->i++; while (!at_end(p) && peek(p) != ')') p->i++; if (at_end(p)) { perr(p, (int64_t)p->i, "missing ), unterminated comment"); return node_new(N_EMPTY); } p->i++; return node_new(N_EMPTY); } if (c == '=' || c == '!') { p->i++; Node *body = parse_alt(p); if (peek(p) != ')') { perr(p, (int64_t)p->i, "missing ), unterminated subpattern"); return body; } p->i++; Node *n = node_new(N_LOOKAROUND); n->a = body; n->lookaround_kind = LK_AHEAD; n->lookaround_negate = (c == '!'); return n; } if (c == '<' && (peek2(p) == '=' || peek2(p) == '!')) { int neg = (peek2(p) == '!'); p->i += 2; Node *body = parse_alt(p); if (peek(p) != ')') { perr(p, (int64_t)p->i, "missing ), unterminated subpattern"); return body; } p->i++; int w = node_width(body); if (w < 0) { perr(p, (int64_t)open_pos, "look-behind requires fixed-width pattern"); return node_new(N_EMPTY); } Node *n = node_new(N_LOOKAROUND); n->a = body; n->lookaround_kind = LK_BEHIND; n->lookaround_negate = neg; n->width = w; return n; } if (c == '>') { p->i++; Node *body = parse_alt(p); if (peek(p) != ')') { perr(p, (int64_t)p->i, "missing ), unterminated subpattern"); return body; } p->i++; Node *n = node_new(N_ATOMIC); n->a = body; return n; } if (c == 'P') { p->i++; if (peek(p) == '<') { p->i++; size_t start = p->i; while (!at_end(p) && peek(p) != '>') p->i++; if (at_end(p)) { perr(p, (int64_t)p->i, "missing >, unterminated name"); return node_new(N_EMPTY); } char *name = xstrndup(p->s + start, p->i - start); p->i++; int idx = ++p->ngroups; Node *body = parse_alt(p); if (peek(p) != ')') { perr(p, (int64_t)p->i, "missing ), unterminated subpattern"); free(name); return body; } p->i++; Node *g = node_new(N_GROUP); g->a = body; g->group_index = idx; g->group_name = name; GroupName *gn = da_push(&p->groupnames); gn->name = xstrndup(name, strlen(name)); gn->index = idx; return g; } if (peek(p) == '=') { p->i++; size_t start = p->i; while (!at_end(p) && peek(p) != ')') p->i++; if (at_end(p)) { perr(p, (int64_t)p->i, "missing ), unterminated name"); return node_new(N_EMPTY); } char *name = xstrndup(p->s + start, p->i - start); p->i++; Node *n = node_new(N_BACKREF); n->backref_name = name; Node **slot = da_push(&p->backrefs); *slot = n; return n; } perr(p, (int64_t)p->i, "unknown extension ?P"); return node_new(N_EMPTY); } if (c == '(') { perr(p, (int64_t)open_pos, "conditional groups (?(id)yes|no) are not implemented in this build"); return node_new(N_EMPTY); } { size_t save = p->i; int newflags = 0, any = 0; while (!at_end(p) && strchr("aiLmsux", peek(p))) { switch (peek(p)) { case 'i': newflags |= IGNORECASE; break; case 'm': newflags |= MULTILINE; break; case 's': newflags |= DOTALL; break; case 'x': newflags |= VERBOSE; break; case 'a': newflags |= ASCII; break; case 'L': newflags |= LOCALE; break; default: break; } p->i++; any = 1; } if (any && peek(p) == ')') { p->i++; if (open_pos != 0) { perr(p, (int64_t)open_pos, "global flags not at the start of the expression"); return node_new(N_EMPTY); } p->flags |= newflags; return node_new(N_EMPTY); } p->i = save; perr(p, (int64_t)open_pos, "scoped inline flags (?flags:...) are not implemented in this build"); return node_new(N_EMPTY); } } int idx = ++p->ngroups; Node *body = parse_alt(p); if (peek(p) != ')') { perr(p, (int64_t)p->i, "missing ), unterminated subpattern"); return body; } p->i++; Node *g = node_new(N_GROUP); g->a = body; g->group_index = idx; return g; } static Node *parse_atom(Parser *p) { int c = peek(p); if (c == '(') return parse_group_or_special(p); if (c == '[') return parse_class(p); if (c == '.') { p->i++; return node_new(N_ANY); } if (c == '^') { p->i++; Node *n = node_new(N_ANCHOR); n->anchor_kind = A_BOL; return n; } if (c == '$') { p->i++; Node *n = node_new(N_ANCHOR); n->anchor_kind = A_EOL; return n; } if (c == '\\') { p->i++; uint32_t litcp; int is_lit; Node *n = parse_escape(p, 0, &litcp, &is_lit); if (p->failed) return node_new(N_EMPTY); if (is_lit) { Node *cn = node_new(N_CHAR); cn->ch = litcp; return cn; } return n; } if (c == '*' || c == '+' || c == '?') { perr(p, (int64_t)p->i, "nothing to repeat"); p->i++; return node_new(N_EMPTY); } uint32_t cp = read_raw_char(p); if (p->failed) return node_new(N_EMPTY); Node *cn = node_new(N_CHAR); cn->ch = cp; return cn; } static Node *parse_concat(Parser *p) { Node *result = NULL; while (!at_end(p) && peek(p) != '|' && peek(p) != ')' && !p->failed) { Node *atom = parse_atom(p); if (p->failed) break; atom = parse_repeat_suffix(p, atom); result = make_concat(result, atom); } if (!result) result = node_new(N_EMPTY); return result; } static Node *parse_alt(Parser *p) { Node *first = parse_concat(p); if (peek(p) == '|') { p->i++; Node *rest = parse_alt(p); Node *n = node_new(N_ALT); n->a = first; n->b = rest; return n; } return first; } /* Strip VERBOSE-mode whitespace and #-comments, respecting [...] classes * and backslash escapes. Only called once the VERBOSE flag is known. */ static char *strip_verbose(const char *s, size_t len, size_t *outlen) { char *out = malloc(len + 1); size_t o = 0, i = 0; int in_class = 0; while (i < len) { char c = s[i]; if (c == '\\' && i + 1 < len) { out[o++] = s[i++]; out[o++] = s[i++]; continue; } if (c == '[' && !in_class) { in_class = 1; out[o++] = s[i++]; continue; } if (c == ']' && in_class) { in_class = 0; out[o++] = s[i++]; continue; } if (!in_class && isspace((unsigned char)c)) { i++; continue; } if (!in_class && c == '#') { while (i < len && s[i] != '\n') i++; continue; } out[o++] = s[i++]; } out[o] = 0; *outlen = o; return out; } /* ================================================================ * 5. Bytecode * ================================================================ */ enum { OP_CHAR, OP_CLASS, OP_ANY, OP_SPLIT, OP_JMP, OP_SAVE, OP_MATCH, OP_ASSERT, OP_BACKREF, OP_LOOKAHEAD, OP_LOOKBEHIND, OP_ATOMIC, OP_RETURN, OP_REPEAT1 }; typedef struct { int op; int32_t x, y; uint32_t data; int width; int neg; int is_loop; int loop_close; int32_t lo, hi; int greedy; int atomkind; /* OP_REPEAT1: 0=CHAR 1=CLASS 2=ANY */ } Inst; typedef struct { Inst *insts; int n, cap; DArr classnodes; /* Node* */ int ngroups; int mode; int flags; int has_backref; /* computed once after compiling; gates memoized search, see run_memo */ int no_repeat1; /* compile time only: when set, compile_repeat never emits the * OP_REPEAT1 fast path (Section 6), even for a simple-atom * repeat, so the result contains only instructions the Pike * VM understands (concept.md 7.7). Never set on the ordinary * backtracking Prog every Pattern already compiles; set only * on the second, NFA-only Prog compiled from the same AST for * an eligible pattern, see PatternImpl.nfa_prog. */ } Prog; static int emit(Prog *pr, int op) { if (pr->n == pr->cap) { pr->cap = pr->cap ? pr->cap * 2 : 64; pr->insts = realloc(pr->insts, (size_t)pr->cap * sizeof(Inst)); } Inst *i = &pr->insts[pr->n]; memset(i, 0, sizeof(*i)); i->op = op; i->x = i->y = -1; return pr->n++; } static void compile_node(Prog *pr, Node *n); static void compile_repeat(Prog *pr, Node *n) { if (!pr->no_repeat1 && is_simple_atom(n->a)) { int pc = emit(pr, OP_REPEAT1); Inst *in = &pr->insts[pc]; in->lo = n->lo; in->hi = n->hi; in->greedy = n->greedy; if (n->a->kind == N_CHAR) { in->atomkind = 0; in->data = n->a->ch; } else if (n->a->kind == N_ANY) { in->atomkind = 2; } else { in->atomkind = 1; int idx = (int)pr->classnodes.len; Node **slot = da_push(&pr->classnodes); *slot = n->a; in->data = (uint32_t)idx; } return; } int lo = n->lo, hi = n->hi; for (int k = 0; k < lo; k++) compile_node(pr, n->a); if (hi == -1) { int nullable = node_nullable(n->a); int split_pc = emit(pr, OP_SPLIT); pr->insts[split_pc].is_loop = nullable; int body_start = pr->n; compile_node(pr, n->a); int jmp_pc = emit(pr, OP_JMP); pr->insts[jmp_pc].x = split_pc; pr->insts[jmp_pc].loop_close = nullable; int end = pr->n; if (n->greedy) { pr->insts[split_pc].x = body_start; pr->insts[split_pc].y = end; } else { pr->insts[split_pc].x = end; pr->insts[split_pc].y = body_start; } } else if (hi > lo) { DArr splits; da_init(&splits, sizeof(int)); for (int k = 0; k < hi - lo; k++) { int sp = emit(pr, OP_SPLIT); int *slot = da_push(&splits); *slot = sp; int body_start = pr->n; compile_node(pr, n->a); if (n->greedy) pr->insts[sp].x = body_start; else pr->insts[sp].y = body_start; } int end = pr->n; for (size_t k = 0; k < splits.len; k++) { int sp = ((int *)splits.data)[k]; if (n->greedy) pr->insts[sp].y = end; else pr->insts[sp].x = end; } da_free(&splits); } } static void compile_node(Prog *pr, Node *n) { switch (n->kind) { case N_EMPTY: return; case N_CHAR: { int pc = emit(pr, OP_CHAR); pr->insts[pc].data = n->ch; return; } case N_ANY: emit(pr, OP_ANY); return; case N_CLASS: { int idx = (int)pr->classnodes.len; Node **slot = da_push(&pr->classnodes); *slot = n; int pc = emit(pr, OP_CLASS); pr->insts[pc].data = (uint32_t)idx; return; } case N_CONCAT: compile_node(pr, n->a); compile_node(pr, n->b); return; case N_ALT: { int split_pc = emit(pr, OP_SPLIT); int x0 = pr->n; compile_node(pr, n->a); int jmp_pc = emit(pr, OP_JMP); int y0 = pr->n; compile_node(pr, n->b); int end = pr->n; pr->insts[split_pc].x = x0; pr->insts[split_pc].y = y0; pr->insts[jmp_pc].x = end; return; } case N_REPEAT: compile_repeat(pr, n); return; case N_GROUP: { if (n->group_index >= 0) { int pc = emit(pr, OP_SAVE); pr->insts[pc].data = (uint32_t)(2 * n->group_index); } compile_node(pr, n->a); if (n->group_index >= 0) { int pc = emit(pr, OP_SAVE); pr->insts[pc].data = (uint32_t)(2 * n->group_index + 1); } return; } case N_BACKREF: { int pc = emit(pr, OP_BACKREF); pr->insts[pc].data = (uint32_t)n->backref_index; return; } case N_ANCHOR: { int pc = emit(pr, OP_ASSERT); pr->insts[pc].data = (uint32_t)n->anchor_kind; return; } case N_LOOKAROUND: { int op = (n->lookaround_kind == LK_AHEAD) ? OP_LOOKAHEAD : OP_LOOKBEHIND; int pc = emit(pr, op); pr->insts[pc].neg = n->lookaround_negate; pr->insts[pc].width = n->width; int sub_start = pr->n; compile_node(pr, n->a); emit(pr, OP_RETURN); int after = pr->n; pr->insts[pc].x = sub_start; pr->insts[pc].y = after; return; } case N_ATOMIC: { int pc = emit(pr, OP_ATOMIC); int sub_start = pr->n; compile_node(pr, n->a); emit(pr, OP_RETURN); int after = pr->n; pr->insts[pc].x = sub_start; pr->insts[pc].y = after; return; } } } /* ================================================================ * 6. Matcher (recursive backtracking VM, concept.md Section 7.3/7.1) * ================================================================ */ #define MAX_DEPTH 60000 /* An earlier revision of this file capped compute_maxrun/ * compute_next_prevmatch's table size and fell back to the plain * O(remaining length) scan above the cap, intending a bounded-memory * degradation for huge inputs. Measured directly (a real 200MB search * with no match) and reverted: the fallback does not degrade * gracefully, it reintroduces the O(n^2) behavior these tables exist * to fix, for exactly the pattern shape (a simple-atom quantifier * followed by a required literal/class that never occurs) they matter * most for, and O(n^2) at n in the hundreds of millions does not * finish in any practical amount of time. A fast, diagnosable * allocation failure is a better failure mode than a silent, * effectively-unbounded hang, so these tables are allocated * unconditionally again; README.md "Implementation status" states the * resulting memory-per-input-byte multiplier plainly instead. There is * no way to get both bounded memory and linear time out of this * technique; only concept.md Section 7.2's actual streaming automaton * (still unimplemented) gets both at once, by construction. */ typedef struct { Prog *pr; const uint32_t *text; /* UTF8 mode: real code points. NULL in BINARY/ASCII mode: use text8/text_at() instead. */ const uint8_t *text8; /* BINARY/ASCII mode: raw bytes, no widening copy, see MatBuf.text8 */ int64_t len; int64_t *caps; /* 2*(ngroups+1) */ int64_t require_end; /* -1 unconstrained, -2 "sub-program, report on OP_RETURN", >=0 exact end required */ int64_t sub_end; /* scratch: end sp reported by a sub-program's OP_RETURN */ int depth_exceeded; uint8_t *memo; /* NULL, or a (ninsts * (len+1))-bit "known to fail" cache, see run_memo */ int32_t **repeat_maxrun; /* NULL, or per-OP_REPEAT1-instruction run-length tables, see compute_maxrun */ int32_t **repeat_nextpm; /* NULL, or per-OP_REPEAT1-instruction "rightmost position <= X where * the following atom matches" tables, see compute_next_prevmatch */ int forbid_empty; /* 1: OP_MATCH must not end at the attempt's own start, see * Pattern_finditer/Pattern_split's same-position retry */ } MCtx; /* Root-cause fix for the search-family quadratic behavior documented in * README.md "Implementation status": a plain backtracking matcher has * no memory that "starting at instruction pc, text position sp is * hopeless", so re-trying nearby start positions (do_one's loop) or * nearby alternative partitions of the same run of characters * ((a+)+b-style catastrophic backtracking) re-derives the same failure * over and over. For a pattern with no backreference, whether * execution starting at (pc, sp) can ever reach OP_MATCH is a pure * function of (pc, sp) alone whenever it is checked outside any active * nullable-loop guard (guard_pc == -1) and outside any nested * sub-program's constrained context (require_end == -1); this is * exactly the "memoized backtracking" technique, provably the same * complexity class as simulating the compiled program as an NFA * (concept.md Section 7.2's Pike VM), just expressed recursively with * a memo table instead of iteratively over an explicit thread list. * Memoization never caches a *success*, only a proven failure, so it * cannot change which match (or which captures) is found, only skip * re-deriving failures already known. This is why it is always safe to * enable, and it is enabled automatically whenever the compiled * program has no OP_BACKREF (Prog.has_backref), regardless of which * Pattern_ function is used. */ static int memo_get(MCtx *c, int pc, int64_t sp) { if (!c->memo) return 0; int64_t bit = (int64_t)pc * (c->len + 1) + sp; return (c->memo[bit >> 3] >> (bit & 7)) & 1; } static void memo_set(MCtx *c, int pc, int64_t sp) { if (!c->memo) return; int64_t bit = (int64_t)pc * (c->len + 1) + sp; c->memo[bit >> 3] |= (uint8_t)(1u << (bit & 7)); } static int class_match(Node *cls, uint32_t cp, int mode, int flags) { int hit = 0; for (size_t k = 0; k < cls->items.len && !hit; k++) { ClassItem *it = &((ClassItem *)cls->items.data)[k]; int m; switch (it->kind) { case CI_RANGE: m = (cp >= it->lo && cp <= it->hi); if (!m && (flags & IGNORECASE)) { uint32_t alt = cls_swapcase(cp, mode, flags); m = (alt >= it->lo && alt <= it->hi); } break; case CI_D: m = cls_is_digit(cp, mode, flags); break; case CI_W: m = cls_is_word(cp, mode, flags); break; case CI_S: m = cls_is_space(cp, mode, flags); break; default: m = 0; } if (it->kind != CI_RANGE && it->neg) m = !m; if (m) hit = 1; } if (cls->class_negate) hit = !hit; return hit; } static int char_eq(uint32_t a, uint32_t b, int mode, int flags) { if (a == b) return 1; if (flags & IGNORECASE) return cls_fold(a, mode, flags) == cls_fold(b, mode, flags); return 0; } /* The one place BINARY/ASCII mode's "no widening copy" (MatBuf.text8, * MCtx.text8) and UTF8 mode's real decoded code points (text) are * reconciled into a single per-position value: c->text is NULL in * BINARY/ASCII mode, so this reads c->text8 instead there. */ static inline uint32_t text_at(const MCtx *c, int64_t i) { return c->text ? c->text[i] : c->text8[i]; } static int is_word_at(MCtx *c, int64_t pos) { if (pos < 0 || pos >= c->len) return 0; return cls_is_word(text_at(c, pos), c->pr->mode, c->pr->flags); } static int run_memo(MCtx *c, int pc, int64_t sp, int64_t guard_sp, int32_t guard_pc, int depth); static int run(MCtx *c, int pc, int64_t sp, int64_t guard_sp, int32_t guard_pc, int depth) { if (depth > MAX_DEPTH) { c->depth_exceeded = 1; return 0; } for (;;) { Inst *in = &c->pr->insts[pc]; switch (in->op) { case OP_CHAR: if (sp >= c->len || !char_eq(text_at(c, sp), in->data, c->pr->mode, c->pr->flags)) return 0; pc++; sp++; continue; case OP_ANY: if (sp >= c->len) return 0; if (text_at(c, sp) == '\n' && !(c->pr->flags & DOTALL)) return 0; pc++; sp++; continue; case OP_CLASS: { if (sp >= c->len) return 0; Node *cls = ((Node **)c->pr->classnodes.data)[in->data]; if (!class_match(cls, text_at(c, sp), c->pr->mode, c->pr->flags)) return 0; pc++; sp++; continue; } case OP_REPEAT1: { int64_t maxc = (in->hi == -1) ? (c->len - sp) : in->hi; if (maxc > c->len - sp) maxc = c->len - sp; int64_t count; if (c->repeat_maxrun && c->repeat_maxrun[pc]) { /* O(1): see compute_maxrun. Without this, counting * how many atoms match from sp is an O(remaining * length) scan repeated at every start position * do_one/finditer/split try, which is exactly the * quadratic-time defect documented in README.md * "Implementation status"; this is its fix. */ count = c->repeat_maxrun[pc][sp]; } else { count = 0; Node *cls = (in->atomkind == 1) ? ((Node **)c->pr->classnodes.data)[in->data] : NULL; while (count < maxc) { uint32_t ch = text_at(c, sp + count); int m; if (in->atomkind == 0) m = char_eq(ch, in->data, c->pr->mode, c->pr->flags); else if (in->atomkind == 2) m = !(ch == '\n' && !(c->pr->flags & DOTALL)); else m = class_match(cls, ch, c->pr->mode, c->pr->flags); if (!m) break; count++; } } if (count > maxc) count = maxc; if (count < in->lo) return 0; if (in->greedy && c->repeat_nextpm && c->repeat_nextpm[pc]) { /* O(candidates actually worth trying), not O(count): * see compute_next_prevmatch. Trying every k from * count down to lo one at a time costs O(count) * loop iterations even though each individual * OP_CHAR/OP_CLASS/OP_ANY check at pc+1 is O(1), * because the *number* of iterations, not the cost * of any one of them, is what stayed quadratic * across do_one/finditer/split's outer position * loop; jumping straight to positions where pc+1 * can actually succeed is what fixes that. */ int32_t *pm = c->repeat_nextpm[pc]; int64_t hi = sp + count; if (hi > c->len - 1) hi = c->len - 1; int64_t lo_bound = sp + in->lo; int64_t p = (hi >= lo_bound && hi >= 0) ? pm[hi] : -1; while (p >= lo_bound) { if (run_memo(c, pc + 1, p, guard_sp, guard_pc, depth + 1)) return 1; p = (p > 0) ? pm[p - 1] : -1; } return 0; } if (in->greedy) { for (int64_t k = count; k >= in->lo; k--) if (run_memo(c, pc + 1, sp + k, guard_sp, guard_pc, depth + 1)) return 1; } else { for (int64_t k = in->lo; k <= count; k++) if (run_memo(c, pc + 1, sp + k, guard_sp, guard_pc, depth + 1)) return 1; } return 0; } case OP_ASSERT: { int ok; switch (in->data) { case A_BOL: ok = (sp == 0) || ((c->pr->flags & MULTILINE) && sp > 0 && text_at(c, sp - 1) == '\n'); break; case A_EOL: ok = (sp == c->len) || (text_at(c, sp) == '\n' && ((c->pr->flags & MULTILINE) || sp == c->len - 1)); break; case A_BOS: ok = (sp == 0); break; case A_EOS: ok = (sp == c->len); break; case A_WB: ok = is_word_at(c, sp - 1) != is_word_at(c, sp); break; case A_NWB: ok = is_word_at(c, sp - 1) == is_word_at(c, sp); break; default: ok = 0; } if (!ok) return 0; pc++; continue; } case OP_SAVE: { int64_t old = c->caps[in->data]; c->caps[in->data] = sp; if (run_memo(c, pc + 1, sp, guard_sp, guard_pc, depth + 1)) return 1; c->caps[in->data] = old; return 0; } case OP_SPLIT: if (in->is_loop) { if (run_memo(c, in->x, sp, sp, pc, depth + 1)) return 1; pc = in->y; continue; } else { if (run_memo(c, in->x, sp, guard_sp, guard_pc, depth + 1)) return 1; pc = in->y; continue; } case OP_JMP: if (in->loop_close && in->x == guard_pc && sp == guard_sp) return 0; pc = in->x; continue; case OP_BACKREF: { int gi = (int)in->data; if (gi < 0 || gi > c->pr->ngroups) return 0; int64_t s = c->caps[2 * gi], e = c->caps[2 * gi + 1]; if (s < 0 || e < 0) return 0; int64_t rl = e - s; if (sp + rl > c->len) return 0; for (int64_t k = 0; k < rl; k++) if (!char_eq(text_at(c, sp + k), text_at(c, s + k), c->pr->mode, c->pr->flags)) return 0; pc++; sp += rl; continue; } case OP_LOOKAHEAD: { size_t ncaps = 2 * (size_t)(c->pr->ngroups + 1); int64_t *snap = malloc(ncaps * sizeof(int64_t)); memcpy(snap, c->caps, ncaps * sizeof(int64_t)); int64_t save_req = c->require_end; c->require_end = -2; int ok = run_memo(c, in->x, sp, -1, -1, depth + 1); c->require_end = save_req; int accept = in->neg ? !ok : ok; if (!accept) { memcpy(c->caps, snap, ncaps * sizeof(int64_t)); free(snap); return 0; } free(snap); pc = in->y; continue; } case OP_LOOKBEHIND: { size_t ncaps = 2 * (size_t)(c->pr->ngroups + 1); int64_t *snap = malloc(ncaps * sizeof(int64_t)); memcpy(snap, c->caps, ncaps * sizeof(int64_t)); int64_t start = sp - in->width; int ok = 0; if (start >= 0) { int64_t save_req = c->require_end; c->require_end = sp; ok = run_memo(c, in->x, start, -1, -1, depth + 1); c->require_end = save_req; } int accept = in->neg ? !ok : ok; if (!accept) { memcpy(c->caps, snap, ncaps * sizeof(int64_t)); free(snap); return 0; } free(snap); pc = in->y; continue; } case OP_ATOMIC: { int64_t save_req = c->require_end; c->require_end = -2; int ok = run_memo(c, in->x, sp, -1, -1, depth + 1); c->require_end = save_req; if (!ok) return 0; sp = c->sub_end; pc = in->y; continue; } case OP_RETURN: if (c->require_end == -2) { c->sub_end = sp; return 1; } if (c->require_end >= 0) return sp == c->require_end; return 1; case OP_MATCH: if (c->require_end >= 0 && sp != c->require_end) return 0; if (c->forbid_empty && sp == c->caps[0]) return 0; c->caps[1] = sp; return 1; default: return 0; } } } /* The only call site allowed to consult/populate the memo (see the * comment on MCtx.memo above): every recursive call inside run(), and * every external entry point, goes through here instead of run() * directly. Caching is restricted to guard_pc == -1 (not inside an * active nullable-loop guard) and c->require_end == -1 (not inside a * fullmatch's exact-end constraint, nor inside a nested lookaround/ * atomic sub-program, which always sets require_end elsewhere first); * outside that combination, whether (pc, sp) succeeds can depend on * context beyond (pc, sp) itself, so it is never cached or consulted * there, only computed directly by run(), exactly as before this * optimization existed. */ static int run_memo(MCtx *c, int pc, int64_t sp, int64_t guard_sp, int32_t guard_pc, int depth) { int cacheable = c->memo && guard_pc == -1 && c->require_end == -1 && !c->forbid_empty; if (cacheable && memo_get(c, pc, sp)) return 0; int result = run(c, pc, sp, guard_sp, guard_pc, depth); if (cacheable && !result && !c->depth_exceeded) memo_set(c, pc, sp); return result; } /* ================================================================ * 6b. Pike VM (regular engine over an NFA-only Prog, concept.md 7.7) * * Matches a Prog compiled with no_repeat1 set and known, by * construction (the eligibility scan in re_compile), to contain none * of OP_BACKREF/OP_LOOKAHEAD/OP_LOOKBEHIND/OP_ATOMIC/OP_REPEAT1: only * OP_CHAR, OP_CLASS, OP_ANY, OP_SPLIT, OP_JMP, OP_SAVE, OP_MATCH, * OP_ASSERT. Reuses MCtx purely as a bundle of the four fields that * matter here (pr/text/text8/len); its other fields (memo, caps, * repeat_maxrun/nextpm, depth_exceeded, sub_end) are backtracking * engine state this VM never reads or writes. * ================================================================ */ typedef struct { int32_t pc; int64_t *saved; /* owned by whichever thread list currently holds it; * 2*(ngroups+1) entries */ } PikeThread; typedef struct { PikeThread *threads; int count; int cap; /* == nfa_prog->n; fixed for the whole search, see the * "no growth logic, nothing can overflow it" argument * in concept.md 7.7 (Cox's one-thread-per-PC bound) */ uint8_t *seen; /* size cap; which PCs already have a thread this step */ } PikeList; static int pikelist_init(PikeList *l, int n) { size_t cap = (size_t)(n > 0 ? n : 1); l->threads = malloc(cap * sizeof(PikeThread)); l->seen = calloc(cap, 1); l->count = 0; l->cap = n; return l->threads != NULL && l->seen != NULL; } /* Frees every currently-listed thread's saved array first: correct only * when called at a point where l->count accurately reflects threads this * list still owns (never right after a pike_step() call without first * resetting count to 0, since pike_step() itself always resolves every * entry it was given, by transferring, accepting, or freeing it). */ static void pikelist_free(PikeList *l) { for (int i = 0; i < l->count; i++) free(l->threads[i].saved); free(l->threads); free(l->seen); } static void pikelist_clear(PikeList *l) { l->count = 0; memset(l->seen, 0, (size_t)(l->cap > 0 ? l->cap : 1)); } typedef struct { int32_t pc; int64_t *saved; } PikeWork; /* Epsilon closure from (pc0, saved0) at text position sp, added into * list l (deduplicated by pc, first writer per step wins, exactly Cox's * stated invariant). Specified recursively by the source material * (concept.md 7.7) but implemented with an explicit heap stack instead * of C call recursion, so a pattern with many alternations cannot * recurse the C stack as deep as the instruction count. * * Contract: always resolves ownership of saved0 (frees it, or stores it * into l, possibly after being duplicated further along the way) before * returning, on every path, including failure. Returns 0 only on * allocation failure, having freed everything it touched; l's existing * entries (from before this call) are left valid and untouched. */ static int pike_addthread(PikeList *l, MCtx *sctx, int32_t pc0, int64_t *saved0, int64_t sp) { int ncaps = 2 * (sctx->pr->ngroups + 1); DArr stack; da_init(&stack, sizeof(PikeWork)); PikeWork *w0 = da_push(&stack); if (!w0) { free(saved0); da_free(&stack); return 0; } w0->pc = pc0; w0->saved = saved0; while (stack.len > 0) { PikeWork w = ((PikeWork *)stack.data)[stack.len - 1]; stack.len--; if (l->seen[w.pc]) { free(w.saved); continue; } l->seen[w.pc] = 1; Inst *in = &sctx->pr->insts[w.pc]; switch (in->op) { case OP_JMP: { PikeWork *nw = da_push(&stack); if (!nw) { free(w.saved); goto fail; } nw->pc = in->x; nw->saved = w.saved; break; } case OP_SPLIT: { /* x is higher priority than y (compile_node already chose * x/y per the quantifier's greedy/lazy sense, Section 6/ * 7.1); pushing y then x means x is popped next, giving * the same DFS pre-order a recursive addthread(x) then * addthread(y) would, so list order still encodes * priority exactly as it does in the backtracking engine. */ int64_t *saved_y = malloc((size_t)ncaps * sizeof(int64_t)); if (!saved_y) { free(w.saved); goto fail; } memcpy(saved_y, w.saved, (size_t)ncaps * sizeof(int64_t)); PikeWork *wy = da_push(&stack); if (!wy) { free(saved_y); free(w.saved); goto fail; } wy->pc = in->y; wy->saved = saved_y; PikeWork *wx = da_push(&stack); if (!wx) { free(w.saved); goto fail; } /* wy already on the stack; the fail unwind below frees it along with every other still-pending entry */ wx->pc = in->x; wx->saved = w.saved; break; } case OP_SAVE: { /* Safe to mutate w.saved in place, not duplicate: by * construction the only place a saved array is ever * shared between two in-flight branches is the instant * OP_SPLIT above duplicates it, so past that point this * copy is exclusively owned along this one path. */ w.saved[in->data] = sp; PikeWork *nw = da_push(&stack); if (!nw) { free(w.saved); goto fail; } nw->pc = w.pc + 1; nw->saved = w.saved; break; } case OP_ASSERT: { int ok; switch (in->data) { case A_BOL: ok = (sp == 0) || ((sctx->pr->flags & MULTILINE) && sp > 0 && text_at(sctx, sp - 1) == '\n'); break; case A_EOL: ok = (sp == sctx->len) || (text_at(sctx, sp) == '\n' && ((sctx->pr->flags & MULTILINE) || sp == sctx->len - 1)); break; case A_BOS: ok = (sp == 0); break; case A_EOS: ok = (sp == sctx->len); break; case A_WB: ok = is_word_at(sctx, sp - 1) != is_word_at(sctx, sp); break; case A_NWB: ok = is_word_at(sctx, sp - 1) == is_word_at(sctx, sp); break; default: ok = 0; } if (!ok) { free(w.saved); break; } PikeWork *nw = da_push(&stack); if (!nw) { free(w.saved); goto fail; } nw->pc = w.pc + 1; nw->saved = w.saved; break; } case OP_CHAR: case OP_CLASS: case OP_ANY: case OP_MATCH: if (l->count >= l->cap) { free(w.saved); goto fail; } /* unreachable in practice, see the fixed-bound argument above; guarded rather than assumed */ l->threads[l->count].pc = w.pc; l->threads[l->count].saved = w.saved; l->count++; break; default: free(w.saved); /* unreachable: an eligible Prog contains no other op */ break; } } da_free(&stack); return 1; fail: for (size_t k = 0; k < stack.len; k++) free(((PikeWork *)stack.data)[k].saved); da_free(&stack); return 0; } /* Processes every thread in clist at position sp: a char/class/any * thread that matches is carried into nlist at pc+1; a match thread is * recorded as the best answer so far (subject to require_end and * forbid_empty, reusing MCtx's exact same-named semantics) and every * strictly lower priority thread remaining in clist this step is * discarded without being run (the leftmost-first rule, concept.md * 7.7, verified against rust-lang/regex's PikeVM). Always resolves * ownership of every clist entry before returning, success or not. * Returns 0 only on allocation failure. */ static int pike_step(PikeList *clist, PikeList *nlist, MCtx *sctx, int64_t sp, int64_t require_end, int forbid_empty, int64_t **best_saved, int *matched) { int i; for (i = 0; i < clist->count; i++) { PikeThread th = clist->threads[i]; Inst *in = &sctx->pr->insts[th.pc]; int consume_ok; switch (in->op) { case OP_CHAR: consume_ok = (sp < sctx->len) && char_eq(text_at(sctx, sp), in->data, sctx->pr->mode, sctx->pr->flags); break; case OP_ANY: consume_ok = (sp < sctx->len) && !(text_at(sctx, sp) == '\n' && !(sctx->pr->flags & DOTALL)); break; case OP_CLASS: { Node *cls = ((Node **)sctx->pr->classnodes.data)[in->data]; consume_ok = (sp < sctx->len) && class_match(cls, text_at(sctx, sp), sctx->pr->mode, sctx->pr->flags); break; } case OP_MATCH: /* Group 0's end has no OP_SAVE (only explicit user groups * do, Section 7.1's N_GROUP case); run()'s own OP_MATCH * handler sets it directly the same way (Section 6), and * this VM must too. */ if ((require_end < 0 || sp == require_end) && !(forbid_empty && sp == th.saved[0])) { th.saved[1] = sp; free(*best_saved); *best_saved = th.saved; *matched = 1; i++; goto discard_rest; } free(th.saved); continue; default: free(th.saved); /* unreachable for an eligible Prog */ continue; } if (consume_ok) { if (!pike_addthread(nlist, sctx, th.pc + 1, th.saved, sp + 1)) { for (int j = i + 1; j < clist->count; j++) free(clist->threads[j].saved); return 0; } } else { free(th.saved); } } return 1; discard_rest: for (; i < clist->count; i++) free(clist->threads[i].saved); return 1; } /* Finds the single leftmost, highest priority match for an NFA-only * Prog between pos and endpos (inclusive of trying a start at endpos * itself, matching do_one's existing <= convention). anchored seeds * exactly one start thread, at pos, and never injects another * (Pattern_match/fullmatch); unanchored search (anchored == 0) injects * a fresh start thread, at lowest priority, at every position up to * endpos, until a match has been recorded (Pattern_search/finditer/ * split), giving one linear pass over [pos, endpos] rather than one * attempt per candidate start position. * * require_end (-1 unconstrained, else the exact end position required) * and forbid_empty mirror MCtx's identically named fields exactly * (Section 6), so fullmatch and the empty-match retry rule (Pattern_ * finditer/Pattern_split) behave identically on both engines. * * Returns 1 (match found, caps_out filled with 2*(ngroups+1) entries), * 0 (no match anywhere in range), or -1 (allocation failure partway * through; mirrors run_memo's depth_exceeded signalling an * unrecoverable condition through the same -1 every Pattern_/re_ * matching function already documents, docs/API.md Section 3). */ static int pike_find(Prog *nfa_pr, const uint32_t *text, const uint8_t *text8, int64_t pos, int64_t endpos, int anchored, int64_t require_end, int forbid_empty, int64_t *caps_out) { MCtx sctx; memset(&sctx, 0, sizeof sctx); sctx.pr = nfa_pr; sctx.text = text; sctx.text8 = text8; sctx.len = endpos; int ncaps = 2 * (nfa_pr->ngroups + 1); PikeList clist, nlist; if (!pikelist_init(&clist, nfa_pr->n)) return -1; if (!pikelist_init(&nlist, nfa_pr->n)) { pikelist_free(&clist); return -1; } int64_t *best_saved = NULL; int matched = 0, oom = 0; int64_t sp = pos; /* Literal prefilter, added after v1 (concept.md 7.9): when the very * first instruction is a mandatory OP_CHAR or OP_CLASS (a pattern * that begins with a required literal or class, not a nullable loop * or a leading assertion), a freshly injected start thread at a * position that instruction does not accept is certain to die on * the very next pike_step call, exactly the way an un-prefiltered * one already does, just after paying for a malloc and a full * pike_addthread call first. Checking the identical condition * pike_step's own OP_CHAR/OP_CLASS case checks, before injecting * rather than after, changes nothing about which threads ever exist * (a position this skips would have produced a thread that dies * unobserved on the next step regardless), only how much work is * spent finding that out; this is the literal prefilter technique * concept.md 7.5/README.md "Implementation status" already cite for * the backtracking engine's own compute_next_prevmatch, applied * here in its simplest form (per position, not precomputed). A * pattern starting with `^`, a group, an alternation, or a nullable * repeat gets no benefit and none of the risk: should_inject stays * 1 and every position is injected exactly as in v1. */ int first_op = nfa_pr->n > 0 ? nfa_pr->insts[0].op : -1; Node *first_cls = (first_op == OP_CLASS) ? ((Node **)nfa_pr->classnodes.data)[nfa_pr->insts[0].data] : NULL; #define PIKE_SHOULD_INJECT(atpos) \ (first_op == OP_CHAR ? ((atpos) < endpos && char_eq(text_at(&sctx, (atpos)), nfa_pr->insts[0].data, nfa_pr->mode, nfa_pr->flags)) : \ first_op == OP_CLASS ? ((atpos) < endpos && class_match(first_cls, text_at(&sctx, (atpos)), nfa_pr->mode, nfa_pr->flags)) : \ 1) if (PIKE_SHOULD_INJECT(pos)) { int64_t *seed = malloc((size_t)ncaps * sizeof(int64_t)); if (!seed) oom = 1; else { for (int k = 0; k < ncaps; k++) seed[k] = -1; seed[0] = pos; if (!pike_addthread(&clist, &sctx, 0, seed, sp)) oom = 1; } } while (!oom) { int step_ok = pike_step(&clist, &nlist, &sctx, sp, require_end, forbid_empty, &best_saved, &matched); clist.count = 0; /* pike_step resolved every entry's ownership either way */ if (!step_ok) { oom = 1; break; } if (sp >= endpos) break; sp++; if (!anchored && !matched && PIKE_SHOULD_INJECT(sp)) { int64_t *s2 = malloc((size_t)ncaps * sizeof(int64_t)); if (!s2) { oom = 1; break; } for (int k = 0; k < ncaps; k++) s2[k] = -1; s2[0] = sp; if (!pike_addthread(&nlist, &sctx, 0, s2, sp)) { oom = 1; break; } } { PikeList tmp = clist; clist = nlist; nlist = tmp; } pikelist_clear(&nlist); /* An empty list only means "give up" when nothing will ever be * injected into it again: once anchored (a single seed, never * renewed) or matched (new starts stop being seeded, see above), * an empty list can never become non-empty again. Otherwise, a * momentarily empty list is not the same as "no more positions * are worth trying": a fresh start seeded at some later position * can die immediately during its own epsilon closure (its very * first instruction, commonly \b, failing outright, exactly what * happens repeatedly inside a longer word like "catalog" for the * pattern \bcat\b) without that meaning every later position * will too. Breaking here unconditionally was a real bug, found * by the existing test suite: it stopped the whole search the * first time this happened, before ever reaching a later * position that would have matched. */ if (clist.count == 0 && (anchored || matched)) break; } #undef PIKE_SHOULD_INJECT pikelist_free(&clist); pikelist_free(&nlist); if (oom) { free(best_saved); return -1; } if (!matched) return 0; memcpy(caps_out, best_saved, (size_t)ncaps * sizeof(int64_t)); free(best_saved); return 1; } /* ================================================================ * 7. Pattern / Match / Input structures * ================================================================ */ typedef struct { char **names; int *idx; int n; } GroupIndex; typedef struct { Prog prog; GroupIndex gidx; NodePool pool; char *pattern_copy; int has_nfa; /* 1 if nfa_prog below is valid and should be preferred, concept.md 7.7 */ Prog nfa_prog; /* compiled only when has_nfa; contains no OP_BACKREF/OP_LOOKAHEAD/ * OP_LOOKBEHIND/OP_ATOMIC/OP_REPEAT1, matched by pike_find() instead * of run_memo(). Shares classnodes' underlying Node* with prog (both * point into the one AST pool, freed once via PatternImpl.pool). */ } PatternImpl; struct Input { uint8_t *buf; size_t len; int owns_buf; /* free(buf) on Input_free */ int is_mmap; /* munmap(buf, len) on Input_free instead of free() */ }; typedef struct { int64_t *pos; /* 2*(ngroups+1) */ int ngroups; int64_t *cp_to_byte; /* NULL unless UTF8 mode; length ngroups-independent, sized to text length+1 */ } MatchSlots; static int groupindex_lookup(GroupIndex *gi, const char *name) { for (int k = 0; k < gi->n; k++) if (strcmp(gi->names[k], name) == 0) return gi->idx[k]; return -1; } static int resolve_mode(int flags) { if (flags & BINARY) return MODE_BINARY; if (flags & UTF8) return MODE_UTF8; return MODE_ASCII; } static void pattern_error_fill(PatternError *err, const char *pattern, const char *msg, int64_t pos) { if (!err) return; err->msg = strdup(msg ? msg : ""); err->pattern = pattern ? strdup(pattern) : NULL; err->pos = pos; err->lineno = 1; err->colno = 1; if (pattern) { for (int64_t k = 0; k < pos && pattern[k]; k++) { if (pattern[k] == '\n') { err->lineno++; err->colno = 1; } else err->colno++; } } } void PatternError_free(PatternError *err) { if (!err) return; free((void *)err->msg); free((void *)err->pattern); err->msg = NULL; err->pattern = NULL; } /* ================================================================ * 8. Compile driver * ================================================================ */ Pattern *re_compile(const char *pattern, size_t len, int flags, PatternError *err) { /* Section 9.3: BINARY/UTF8 select their mode; otherwise ASCII. */ int mode = resolve_mode(flags); /* Step 1: leading global inline flags (?aiLmsux), Section 2.5/2.6 */ int extra_flags = 0; size_t body_off = 0; if (len >= 3 && pattern[0] == '(' && pattern[1] == '?') { size_t k = 2; int any = 0, ok = 1; while (k < len && strchr("aiLmsux", (unsigned char)pattern[k])) { switch (pattern[k]) { case 'i': extra_flags |= IGNORECASE; break; case 'm': extra_flags |= MULTILINE; break; case 's': extra_flags |= DOTALL; break; case 'x': extra_flags |= VERBOSE; break; case 'a': extra_flags |= ASCII; break; case 'L': extra_flags |= LOCALE; break; default: break; } k++; any = 1; } if (any && k < len && pattern[k] == ')') { body_off = k + 1; } else { extra_flags = 0; ok = 0; (void)ok; } } int all_flags = flags | extra_flags; const char *body = pattern + body_off; size_t bodylen = len - body_off; char *stripped = NULL; if (all_flags & VERBOSE) { size_t nl; stripped = strip_verbose(body, bodylen, &nl); body = stripped; bodylen = nl; } Parser p; memset(&p, 0, sizeof p); p.s = body; p.len = bodylen; p.i = 0; p.flags = all_flags; p.mode = mode; da_init(&p.backrefs, sizeof(Node *)); da_init(&p.groupnames, sizeof(GroupName)); g_pool.data = NULL; g_pool.len = 0; g_pool.cap = 0; Node *ast = parse_alt(&p); if (!p.failed && !at_end(&p)) { if (peek(&p) == ')') perr(&p, (int64_t)p.i, "unbalanced parenthesis"); else perr(&p, (int64_t)p.i, "unexpected character"); } if (!p.failed) { for (size_t k = 0; k < p.backrefs.len; k++) { Node *b = ((Node **)p.backrefs.data)[k]; if (b->backref_name) { int found = -1; for (size_t j = 0; j < p.groupnames.len; j++) { GroupName *gn = &((GroupName *)p.groupnames.data)[j]; if (strcmp(gn->name, b->backref_name) == 0) { found = gn->index; break; } } if (found < 0) { perr(&p, 0, "unknown group name '%s'", b->backref_name); break; } b->backref_index = found; } else if (b->backref_index < 1 || b->backref_index > p.ngroups) { perr(&p, 0, "invalid group reference %d", b->backref_index); break; } } } if (p.failed) { pattern_error_fill(err, pattern, p.errmsg, p.errpos + (int64_t)body_off); nodepool_free_all(&g_pool); da_free(&p.backrefs); for (size_t k = 0; k < p.groupnames.len; k++) free(((GroupName *)p.groupnames.data)[k].name); da_free(&p.groupnames); free(stripped); return NULL; } Prog prog; memset(&prog, 0, sizeof prog); prog.ngroups = p.ngroups; prog.mode = mode; prog.flags = all_flags; da_init(&prog.classnodes, sizeof(Node *)); compile_node(&prog, ast); emit(&prog, OP_MATCH); /* Pike VM eligibility (concept.md 7.7): a pattern is ineligible iff * its ordinary Prog contains a construct with no pure-NFA execution * (a backreference, or a sub-program instruction: lookahead, * lookbehind, atomic group; a possessive quantifier is already * OP_ATOMIC by the time it reaches here, Section 4, so it needs no * separate check). One scan serves both this and has_backref. */ int nfa_eligible = 1; for (int k = 0; k < prog.n; k++) { switch (prog.insts[k].op) { case OP_BACKREF: prog.has_backref = 1; nfa_eligible = 0; break; case OP_LOOKAHEAD: case OP_LOOKBEHIND: case OP_ATOMIC: nfa_eligible = 0; break; default: break; } } PatternImpl *impl = calloc(1, sizeof(PatternImpl)); impl->prog = prog; impl->pool = g_pool; if (nfa_eligible) { Prog nprog; memset(&nprog, 0, sizeof nprog); nprog.ngroups = p.ngroups; nprog.mode = mode; nprog.flags = all_flags; nprog.no_repeat1 = 1; da_init(&nprog.classnodes, sizeof(Node *)); compile_node(&nprog, ast); emit(&nprog, OP_MATCH); impl->nfa_prog = nprog; impl->has_nfa = 1; } impl->pattern_copy = xstrndup(pattern, len); impl->gidx.n = (int)p.groupnames.len; impl->gidx.names = malloc(sizeof(char *) * (size_t)(impl->gidx.n ? impl->gidx.n : 1)); impl->gidx.idx = malloc(sizeof(int) * (size_t)(impl->gidx.n ? impl->gidx.n : 1)); for (int k = 0; k < impl->gidx.n; k++) { GroupName *gn = &((GroupName *)p.groupnames.data)[k]; impl->gidx.names[k] = gn->name; /* transfer ownership */ impl->gidx.idx[k] = gn->index; } da_free(&p.groupnames); da_free(&p.backrefs); free(stripped); Pattern *pat = calloc(1, sizeof(Pattern)); pat->pattern = impl->pattern_copy; pat->flags = all_flags; pat->groups = prog.ngroups; pat->groupindex = &impl->gidx; pat->program = impl; return pat; } void Pattern_free(Pattern *self) { if (!self) return; PatternImpl *impl = (PatternImpl *)self->program; if (impl) { free(impl->prog.insts); da_free(&impl->prog.classnodes); if (impl->has_nfa) { free(impl->nfa_prog.insts); da_free(&impl->nfa_prog.classnodes); } nodepool_free_all(&impl->pool); for (int k = 0; k < impl->gidx.n; k++) free(impl->gidx.names[k]); free(impl->gidx.names); free(impl->gidx.idx); free(impl->pattern_copy); free(impl); } free(self); } /* ================================================================ * 9. Input * ================================================================ */ Input *Input_from_buffer(const uint8_t *buf, size_t len) { Input *in = calloc(1, sizeof(Input)); in->buf = (uint8_t *)buf; in->len = len; in->owns_buf = 0; return in; } /* Reads the rest of an already-open fd (positioned at 0) into a fresh, * owned, growable buffer, used both as Input_from_file's fallback when * mmap is not applicable or fails, and for non-regular files (pipes, * FIFOs, process substitution, stdin) whose size cannot be known * upfront. Takes ownership of fd (always closes it). */ static Input *read_fd_incrementally(int fd, PatternError *err) { size_t cap = 1 << 16, len = 0; uint8_t *buf = malloc(cap); if (!buf) { close(fd); pattern_error_fill(err, NULL, "out of memory reading file", 0); return NULL; } ssize_t n; while ((n = read(fd, buf + len, cap - len)) > 0) { len += (size_t)n; if (len == cap) { size_t ncap = cap * 2; uint8_t *nbuf = realloc(buf, ncap); if (!nbuf) { free(buf); close(fd); pattern_error_fill(err, NULL, "out of memory reading file", (int64_t)len); return NULL; } buf = nbuf; cap = ncap; } } close(fd); Input *in = calloc(1, sizeof(Input)); if (!in) { free(buf); pattern_error_fill(err, NULL, "out of memory reading file", 0); return NULL; } in->buf = buf; in->len = len; in->owns_buf = 1; return in; } /* Prefers mmap() over reading the file into a malloc'd copy: the * mapped pages are backed directly by the file and stay clean (never * written), so the kernel can reclaim them under memory pressure and * page them back in from disk later, instead of them being pinned for * the whole match attempt the way a malloc'd copy would be; it also * skips one whole redundant copy of the file's bytes (README.md * "Implementation status" records the memory-per-input-byte * multiplier this and the other fixes around it were measured * against). Only applies to regular, non-empty, seekable files; * anything else (a pipe, a FIFO, an empty file) falls back to * read_fd_incrementally, exactly as before mmap support existed. */ Input *Input_from_file(const char *path, PatternError *err) { int fd = open(path, O_RDONLY); if (fd < 0) { pattern_error_fill(err, NULL, strerror(errno), 0); return NULL; } struct stat st; if (fstat(fd, &st) == 0 && S_ISREG(st.st_mode) && st.st_size > 0) { void *addr = mmap(NULL, (size_t)st.st_size, PROT_READ, MAP_PRIVATE, fd, 0); if (addr != MAP_FAILED) { close(fd); Input *in = calloc(1, sizeof(Input)); if (!in) { munmap(addr, (size_t)st.st_size); pattern_error_fill(err, NULL, "out of memory reading file", 0); return NULL; } in->buf = addr; in->len = (size_t)st.st_size; in->owns_buf = 0; in->is_mmap = 1; return in; } /* mmap failed (unusual: an overcommit-restricted system, a * filesystem that does not support it, and so on); the fd is * still open and positioned at 0, so fall back to reading it * the ordinary way instead of failing outright. */ return read_fd_incrementally(fd, err); } /* Empty regular file, or not a regular file at all: size is 0 or * unknowable upfront, and mmap does not apply either way. */ return read_fd_incrementally(fd, err); } void Input_free(Input *in) { if (!in) return; if (in->is_mmap) munmap(in->buf, in->len); else if (in->owns_buf) free(in->buf); free(in); } /* ================================================================ * 10. Match buffer construction (UTF-8 pre-decode, Section 8.4/9.3) * ================================================================ */ typedef struct { uint32_t *text; /* UTF8 mode only: owned, decoded code points */ const uint8_t *text8; /* BINARY/ASCII mode only: borrowed, points directly * into the Input's own buffer, no widening copy; * see text_at() and README.md "Implementation * status" for why this matters at real scale. */ int64_t len; int64_t *cp_to_byte; /* len+1 entries, NULL for non-UTF8 */ } MatBuf; static int build_matbuf(Pattern *pat, Input *in, MatBuf *mb, PatternError *err) { int mode = resolve_mode(pat->flags); if (mode == MODE_UTF8) { DArr cps; da_init(&cps, sizeof(uint32_t)); DArr offs; da_init(&offs, sizeof(int64_t)); size_t i = 0; while (i < in->len) { uint32_t cp; int n = utf8_decode(in->buf, in->len, i, &cp); if (n == 0) { pattern_error_fill(err, NULL, "invalid UTF-8 in subject", (int64_t)i); da_free(&cps); da_free(&offs); return 0; } uint32_t *cpp = da_push(&cps); int64_t *op = da_push(&offs); if (!cpp || !op) { pattern_error_fill(err, NULL, "out of memory decoding UTF-8 subject", (int64_t)i); da_free(&cps); da_free(&offs); return 0; } *cpp = cp; *op = (int64_t)i; i += (size_t)n; } int64_t *sentinel = da_push(&offs); if (!sentinel) { pattern_error_fill(err, NULL, "out of memory decoding UTF-8 subject", (int64_t)in->len); da_free(&cps); da_free(&offs); return 0; } *sentinel = (int64_t)in->len; mb->text = (uint32_t *)cps.data; mb->text8 = NULL; mb->cp_to_byte = (int64_t *)offs.data; mb->len = (int64_t)cps.len; } else { /* No widening copy: a byte never exceeds 255, so BINARY/ASCII * mode reads the Input's own buffer directly. This is the * single biggest lever found profiling a real search with * Massif (README.md "Implementation status"): the widened * uint32_t copy this replaces cost as much memory as the * input itself, four times over, for every mode that never * actually needed code points wider than a byte. mb->text8 is * a borrowed pointer (into Input, which outlives the MatBuf * that borrows it), never freed here. */ mb->text = NULL; mb->text8 = in->buf; mb->len = (int64_t)in->len; mb->cp_to_byte = NULL; } return 1; } static void free_matbuf(MatBuf *mb) { free(mb->text); free(mb->cp_to_byte); } /* ================================================================ * 11. Matching driver and Pattern_ methods * ================================================================ */ static void alloc_caps(int64_t **caps, int ngroups) { *caps = malloc(2 * (size_t)(ngroups + 1) * sizeof(int64_t)); } static void reset_caps(int64_t *caps, int ngroups) { for (int k = 0; k < 2 * (ngroups + 1); k++) caps[k] = -1; } static void fill_match_from_caps(Match *out, Pattern *self, Input *string, int64_t pos, int64_t endpos, int64_t *caps, int ngroups, MatBuf *mb) { MatchSlots *ms = calloc(1, sizeof(MatchSlots)); ms->pos = caps; ms->ngroups = ngroups; if (mb->cp_to_byte) { ms->cp_to_byte = malloc((size_t)(mb->len + 1) * sizeof(int64_t)); memcpy(ms->cp_to_byte, mb->cp_to_byte, (size_t)(mb->len + 1) * sizeof(int64_t)); } out->re = self; out->string = string; out->pos = pos; out->endpos = endpos; out->slots = ms; out->lastindex = -1; out->lastgroup = NULL; PatternImpl *impl = (PatternImpl *)self->program; for (int g = ngroups; g >= 1; g--) { if (caps[2 * g] >= 0) { out->lastindex = g; for (int k = 0; k < impl->gidx.n; k++) if (impl->gidx.idx[k] == g) { out->lastgroup = impl->gidx.names[k]; break; } break; } } } static int64_t clamp(int64_t v, int64_t lo, int64_t hi) { return v < lo ? lo : (v > hi ? hi : v); } /* Allocated once per top-level Pattern_/re_ call (not once per start * position tried, and, for Pattern_finditer/Pattern_split, not once * per match found either): see the comment on MCtx.memo. NULL, with no * behavior change beyond the missing optimization, whenever the * pattern contains a backreference anywhere. */ static uint8_t *alloc_memo(Prog *pr, int64_t textlen) { if (pr->has_backref) return NULL; int64_t bits = (int64_t)pr->n * (textlen + 1); int64_t bytes = (bits + 7) / 8; return calloc((size_t)(bytes > 0 ? bytes : 1), 1); } /* mr[i] = how many consecutive positions starting at text[i] satisfy * in's atom predicate (0 if text[i] itself does not), computed in one * backward O(textlen) pass instead of redone forward, from scratch, at * every position a caller asks about it. This is what makes * OP_REPEAT1 O(1) per position instead of O(remaining run length), * which otherwise stays quadratic across do_one/finditer/split's outer * position loop even with run_memo's (pc, sp) memoization, because the * counting scan is an internal C loop, not expressed as (pc, sp) * recursive calls at all. int32_t bounds a single repeat's run length * to ~2 billion, far past any input this build's fully-materializing * Input can hold in memory in the first place. */ /* Shared by compute_maxrun/compute_next_prevmatch, exactly like * text_at() above them for run() itself: text32 is non-NULL only in * UTF8 mode, in which case it holds real decoded code points; in * BINARY/ASCII mode text32 is NULL and text8 (the Input's own buffer, * no widening copy) is used instead. */ static inline uint32_t buf_at(const uint32_t *text32, const uint8_t *text8, int64_t i) { return text32 ? text32[i] : text8[i]; } static int32_t *compute_maxrun(Prog *pr, Inst *in, const uint32_t *text32, const uint8_t *text8, int64_t len) { int32_t *mr = malloc((size_t)(len + 1) * sizeof(int32_t)); if (!mr) return NULL; mr[len] = 0; Node *cls = (in->atomkind == 1) ? ((Node **)pr->classnodes.data)[in->data] : NULL; for (int64_t i = len - 1; i >= 0; i--) { uint32_t ch = buf_at(text32, text8, i); int m; if (in->atomkind == 0) m = char_eq(ch, in->data, pr->mode, pr->flags); else if (in->atomkind == 2) m = !(ch == '\n' && !(pr->flags & DOTALL)); else m = class_match(cls, ch, pr->mode, pr->flags); int64_t next = mr[i + 1]; mr[i] = m ? (int32_t)(next < INT32_MAX ? next + 1 : INT32_MAX) : 0; } return mr; } /* One table per OP_REPEAT1 instruction actually present in the * program, indexed by instruction number; only worth the O(text * length) memory per instruction for the multi-position operations * (search/finditer/split) that would otherwise redo the scan at every * position, so match/fullmatch (a single attempt) do not allocate it. */ static int32_t **alloc_repeat_maxrun(Prog *pr, const uint32_t *text32, const uint8_t *text8, int64_t len) { int32_t **arr = calloc((size_t)pr->n, sizeof(int32_t *)); if (!arr) return NULL; for (int i = 0; i < pr->n; i++) if (pr->insts[i].op == OP_REPEAT1) arr[i] = compute_maxrun(pr, &pr->insts[i], text32, text8, len); return arr; } static void free_repeat_maxrun(Prog *pr, int32_t **arr) { if (!arr) return; for (int i = 0; i < pr->n; i++) free(arr[i]); free(arr); } static int inst_atom_match(Prog *pr, Inst *in, uint32_t ch) { switch (in->op) { case OP_CHAR: return char_eq(ch, in->data, pr->mode, pr->flags); case OP_ANY: return !(ch == '\n' && !(pr->flags & DOTALL)); case OP_CLASS: return class_match(((Node **)pr->classnodes.data)[in->data], ch, pr->mode, pr->flags); default: return 0; } } /* pm[i] = the largest position <= i where the instruction right after * an OP_REPEAT1 matches, or -1 if none exists in [0, i]. Only computed * when that next instruction is itself a single simple atom (OP_CHAR/ * OP_CLASS/OP_ANY); this is what lets OP_REPEAT1's greedy backtrack * jump straight from "the largest k worth trying" to "the next * smaller one worth trying" instead of visiting every k in between * (see the comment at its one call site). Built in one forward * O(textlen) pass instead of walked freshly, backward, from every * position a caller asks about it. */ static int32_t *compute_next_prevmatch(Prog *pr, Inst *next, const uint32_t *text32, const uint8_t *text8, int64_t len) { if (len <= 0) return NULL; int32_t *pm = malloc((size_t)len * sizeof(int32_t)); if (!pm) return NULL; int32_t last = -1; for (int64_t i = 0; i < len; i++) { if (inst_atom_match(pr, next, buf_at(text32, text8, i))) last = (int32_t)i; pm[i] = last; } return pm; } static int32_t **alloc_repeat_nextpm(Prog *pr, const uint32_t *text32, const uint8_t *text8, int64_t len) { int32_t **arr = calloc((size_t)pr->n, sizeof(int32_t *)); if (!arr) return NULL; for (int i = 0; i < pr->n; i++) { if (pr->insts[i].op != OP_REPEAT1 || !pr->insts[i].greedy) continue; if (i + 1 >= pr->n) continue; int nextop = pr->insts[i + 1].op; if (nextop == OP_CHAR || nextop == OP_ANY || nextop == OP_CLASS) arr[i] = compute_next_prevmatch(pr, &pr->insts[i + 1], text32, text8, len); } return arr; } static void free_repeat_nextpm(Prog *pr, int32_t **arr) { if (!arr) return; for (int i = 0; i < pr->n; i++) free(arr[i]); free(arr); } static int do_one(Pattern *self, Input *string, int64_t pos, int64_t endpos, int anchored, int fullmatch, Match *out) { MatBuf mb; if (!build_matbuf(self, string, &mb, NULL)) return -1; int64_t ep = (endpos < 0) ? mb.len : clamp(endpos, 0, mb.len); int64_t p0 = clamp(pos, 0, mb.len); PatternImpl *impl = (PatternImpl *)self->program; int ng = impl->prog.ngroups; /* concept.md 7.7: an eligible pattern is matched by the Pike VM * instead, in one call rather than the per-position loop below (that * loop's entire job, finding the next match starting no earlier than * a given position, is pike_find's job too, done in a single linear * pass instead of one attempt per candidate start). Skips memo/ * maxrun/nextpm entirely: those exist only to make the loop below * linear (Section 7.5), and this path was never quadratic in the * first place. */ if (impl->has_nfa) { int64_t *ncaps; alloc_caps(&ncaps, ng); int r = pike_find(&impl->nfa_prog, mb.text, mb.text8, p0, ep, anchored, fullmatch ? ep : -1, 0, ncaps); if (r < 0) { free(ncaps); free_matbuf(&mb); return -1; } if (r == 0) { free(ncaps); free_matbuf(&mb); return 0; } fill_match_from_caps(out, self, string, p0, ep, ncaps, ng, &mb); free_matbuf(&mb); return 1; } int64_t *caps; alloc_caps(&caps, ng); uint8_t *memo = alloc_memo(&impl->prog, ep); /* Only search (anchored == 0) tries more than one position, so * only search pays for the run-length precompute (see * compute_maxrun); match/fullmatch's single attempt gets no * benefit from it and skips the O(text length) memory. */ int32_t **maxrun = anchored ? NULL : alloc_repeat_maxrun(&impl->prog, mb.text, mb.text8, ep); int32_t **nextpm = anchored ? NULL : alloc_repeat_nextpm(&impl->prog, mb.text, mb.text8, ep); int found = 0; int64_t last_start = anchored ? p0 : ep; for (int64_t start = p0; start <= last_start && !found; start++) { reset_caps(caps, ng); caps[0] = start; MCtx c; c.pr = &impl->prog; c.text = mb.text; c.text8 = mb.text8; c.len = ep; c.caps = caps; c.depth_exceeded = 0; c.require_end = fullmatch ? ep : -1; c.sub_end = -1; c.memo = memo; c.repeat_maxrun = maxrun; c.repeat_nextpm = nextpm; c.forbid_empty = 0; found = run_memo(&c, 0, start, -1, -1, 0); if (!found && c.depth_exceeded) { free(caps); free(memo); free_repeat_maxrun(&impl->prog, maxrun); free_repeat_nextpm(&impl->prog, nextpm); free_matbuf(&mb); return -1; } } free(memo); free_repeat_maxrun(&impl->prog, maxrun); free_repeat_nextpm(&impl->prog, nextpm); if (!found) { free(caps); free_matbuf(&mb); return 0; } fill_match_from_caps(out, self, string, p0, ep, caps, ng, &mb); free_matbuf(&mb); return 1; } int Pattern_match(Pattern *self, Input *string, int64_t pos, int64_t endpos, Match *out) { return do_one(self, string, pos, endpos, 1, 0, out); } int Pattern_fullmatch(Pattern *self, Input *string, int64_t pos, int64_t endpos, Match *out) { return do_one(self, string, pos, endpos, 1, 1, out); } int Pattern_search(Pattern *self, Input *string, int64_t pos, int64_t endpos, Match *out) { return do_one(self, string, pos, endpos, 0, 0, out); } /* Shared by Pattern_finditer/Pattern_split's two find points each (the * primary "next match starting no earlier than `start`" search, and the * "anchored, non-empty" empty-match retry at the exact same position, * concept.md 7.7): dispatches to whichever engine this Pattern uses. * anchored=0 is an unanchored scan of [start, ep]; anchored=1 is a * single attempt at exactly `start` (the retry's actual shape: it is * not a forward scan, matching the original backtracking call it * replaces). memo/maxrun/nextpm are the backtracking engine's per-scan * state and are ignored when impl->has_nfa (Section 7.7: the Pike VM * needs none of them, having never been quadratic in the first place). * Returns 1 (found, caps filled), 0 (no match), or -1 (unrecoverable; * caller must clean up and return -1, exactly as a depth_exceeded * backtracking failure already required before this engine existed). */ static int find_next(PatternImpl *impl, MatBuf *mb, int64_t ep, int64_t start, int anchored, int forbid_empty, uint8_t *memo, int32_t **maxrun, int32_t **nextpm, int64_t *caps, int ng) { if (impl->has_nfa) return pike_find(&impl->nfa_prog, mb->text, mb->text8, start, ep, anchored, -1, forbid_empty, caps); int found = 0; int64_t last = anchored ? start : ep; for (int64_t s = start; s <= last && !found; s++) { reset_caps(caps, ng); caps[0] = s; MCtx c; c.pr = &impl->prog; c.text = mb->text; c.text8 = mb->text8; c.len = ep; c.caps = caps; c.depth_exceeded = 0; c.require_end = -1; c.sub_end = -1; c.memo = memo; c.repeat_maxrun = maxrun; c.repeat_nextpm = nextpm; c.forbid_empty = forbid_empty; found = run_memo(&c, 0, s, -1, -1, 0); if (!found && c.depth_exceeded) return -1; } return found ? 1 : 0; } int Pattern_finditer(Pattern *self, Input *string, int64_t pos, int64_t endpos, MatchIterCb cb, void *ctx) { MatBuf mb; if (!build_matbuf(self, string, &mb, NULL)) return -1; int64_t ep = (endpos < 0) ? mb.len : clamp(endpos, 0, mb.len); int64_t start = clamp(pos, 0, mb.len); PatternImpl *impl = (PatternImpl *)self->program; int ng = impl->prog.ngroups; /* One memo table for the whole scan: every position tried, across * every match found, shares it (see the comment on MCtx.memo). A * fact it records ("(pc, sp) cannot reach OP_MATCH") never becomes * false later in the same scan, so nothing here ever needs to * invalidate or reset it between matches. */ uint8_t *memo = alloc_memo(&impl->prog, ep); int32_t **maxrun = alloc_repeat_maxrun(&impl->prog, mb.text, mb.text8, ep); /* see compute_maxrun */ int32_t **nextpm = alloc_repeat_nextpm(&impl->prog, mb.text, mb.text8, ep); /* see compute_next_prevmatch */ int count = 0; while (start <= ep) { int64_t *caps; alloc_caps(&caps, ng); int found = find_next(impl, &mb, ep, start, 0, 0, memo, maxrun, nextpm, caps, ng); if (found < 0) { free(caps); free(memo); free_repeat_maxrun(&impl->prog, maxrun); free_repeat_nextpm(&impl->prog, nextpm); free_matbuf(&mb); return -1; } if (!found) { free(caps); break; } Match m; memset(&m, 0, sizeof m); MatBuf shallow = mb; shallow.cp_to_byte = mb.cp_to_byte; /* share for lookup, copied inside fill */ fill_match_from_caps(&m, self, string, start, ep, caps, ng, &shallow); cb(ctx, &m); count++; int64_t mstart = caps[0], mend = caps[1]; Match_free(&m); /* CPython's finditer, reverse engineered against a real * interpreter (not documented): if the match just reported was * empty, it additionally looks for a second, non-empty match at * that exact same start position before moving on, and reports * that one too if it exists (README.md/docs/API.md carry the * probe cases this was derived from). OP_MATCH's forbid_empty * check forces exactly that search: the same attempt, with the * empty solution excluded, so ordinary backtracking finds the * next (longer) alternative on its own if one exists. */ if (mend == mstart) { int64_t *caps2; alloc_caps(&caps2, ng); int found2 = find_next(impl, &mb, ep, mstart, 1, 1, memo, maxrun, nextpm, caps2, ng); if (found2 < 0) { free(caps2); free(memo); free_repeat_maxrun(&impl->prog, maxrun); free_repeat_nextpm(&impl->prog, nextpm); free_matbuf(&mb); return -1; } if (found2) { Match m2; memset(&m2, 0, sizeof m2); fill_match_from_caps(&m2, self, string, mstart, ep, caps2, ng, &shallow); cb(ctx, &m2); count++; mend = caps2[1]; Match_free(&m2); } else { free(caps2); } } start = (mend > mstart) ? mend : mstart + 1; } free(memo); free_repeat_maxrun(&impl->prog, maxrun); free_repeat_nextpm(&impl->prog, nextpm); free_matbuf(&mb); return count; } int Pattern_findall(Pattern *self, Input *string, int64_t pos, int64_t endpos, MatchIterCb cb, void *ctx) { return Pattern_finditer(self, string, pos, endpos, cb, ctx); } /* Pattern_split's contract: cb is invoked once per element of the list * Python's re.split() would return, in order, each time as a Match * whose group 0 (via Match_group(m, NULL, 0, ...)) is that element's * text; a "None" element (an unparticipated capturing group between * two matches) reports via Match_group returning 0, matching how an * unparticipated group reports on any other Match. */ static void emit_split_item(MatchIterCb cb, void *ctx, Pattern *self, Input *string, MatBuf *mb, int64_t s, int64_t e) { int64_t *caps = malloc(2 * sizeof(int64_t)); caps[0] = s; caps[1] = e; Match m; memset(&m, 0, sizeof m); fill_match_from_caps(&m, self, string, 0, mb->len, caps, 0, mb); cb(ctx, &m); Match_free(&m); } int Pattern_split(Pattern *self, Input *string, int maxsplit, MatchIterCb cb, void *ctx) { MatBuf mb; if (!build_matbuf(self, string, &mb, NULL)) return -1; int64_t ep = mb.len; int64_t start = 0, seg_start = 0; PatternImpl *impl = (PatternImpl *)self->program; int ng = impl->prog.ngroups; uint8_t *memo = alloc_memo(&impl->prog, ep); /* one table for the whole scan, see MCtx.memo */ int32_t **maxrun = alloc_repeat_maxrun(&impl->prog, mb.text, mb.text8, ep); /* see compute_maxrun */ int32_t **nextpm = alloc_repeat_nextpm(&impl->prog, mb.text, mb.text8, ep); /* see compute_next_prevmatch */ int n = 0; while (start <= ep) { if (maxsplit > 0 && n >= maxsplit) break; int64_t *caps; alloc_caps(&caps, ng); int found = find_next(impl, &mb, ep, start, 0, 0, memo, maxrun, nextpm, caps, ng); if (found < 0) { free(caps); free(memo); free_repeat_maxrun(&impl->prog, maxrun); free_repeat_nextpm(&impl->prog, nextpm); free_matbuf(&mb); return -1; } if (!found) { free(caps); break; } emit_split_item(cb, ctx, self, string, &mb, seg_start, caps[0]); for (int g = 1; g <= ng; g++) emit_split_item(cb, ctx, self, string, &mb, caps[2 * g], caps[2 * g + 1]); n++; seg_start = caps[1]; int64_t mstart = caps[0], mend = caps[1]; free(caps); /* Same same-position retry as Pattern_finditer; see its comment. */ if (mend == mstart) { if (maxsplit <= 0 || n < maxsplit) { int64_t *caps2; alloc_caps(&caps2, ng); int found2 = find_next(impl, &mb, ep, mstart, 1, 1, memo, maxrun, nextpm, caps2, ng); if (found2 < 0) { free(caps2); free(memo); free_repeat_maxrun(&impl->prog, maxrun); free_repeat_nextpm(&impl->prog, nextpm); free_matbuf(&mb); return -1; } if (found2) { emit_split_item(cb, ctx, self, string, &mb, mstart, caps2[0]); for (int g = 1; g <= ng; g++) emit_split_item(cb, ctx, self, string, &mb, caps2[2 * g], caps2[2 * g + 1]); n++; seg_start = caps2[1]; mend = caps2[1]; } free(caps2); } } start = (mend > mstart) ? mend : mstart + 1; } free(memo); free_repeat_maxrun(&impl->prog, maxrun); free_repeat_nextpm(&impl->prog, nextpm); emit_split_item(cb, ctx, self, string, &mb, seg_start, ep); free_matbuf(&mb); return n; } static char *expand_template(const char *repl, Match *m, size_t *outlen) { size_t cap = 64, len = 0; char *out = malloc(cap); size_t rl = strlen(repl); for (size_t i = 0; i < rl; i++) { char c = repl[i]; const char *piece = NULL; size_t piecelen = 0; int consumed_one_char = 1; if (c == '\\' && i + 1 < rl) { char nc = repl[i + 1]; if (nc == 'g' && i + 2 < rl && repl[i + 2] == '<') { size_t j = i + 3, start = j; while (j < rl && repl[j] != '>') j++; char *name = xstrndup(repl + start, j - start); int idx = -1; if (strspn(name, "0123456789") == strlen(name) && name[0]) idx = atoi(name); else idx = groupindex_lookup((GroupIndex *)m->re->groupindex, name); free(name); Match_group(m, NULL, idx, &piece, &piecelen); i = j; } else if (isdigit((unsigned char)nc)) { size_t j = i + 1, start = j; while (j < rl && isdigit((unsigned char)repl[j]) && j - start < 2) j++; char *ns = xstrndup(repl + start, j - start); int idx = atoi(ns); free(ns); Match_group(m, NULL, idx, &piece, &piecelen); i = j - 1; } else if (nc == 'n') { char cc = '\n'; if (len+1>=cap){cap*=2;out=realloc(out,cap);} out[len++]=cc; i++; continue; } else if (nc == 't') { char cc = '\t'; if (len+1>=cap){cap*=2;out=realloc(out,cap);} out[len++]=cc; i++; continue; } else if (nc == '\\') { char cc = '\\'; if (len+1>=cap){cap*=2;out=realloc(out,cap);} out[len++]=cc; i++; continue; } else { piece = repl + i + 1; piecelen = 1; i++; } } else { if (len + 1 >= cap) { cap *= 2; out = realloc(out, cap); } out[len++] = c; consumed_one_char = 1; (void)consumed_one_char; continue; } if (piece && piecelen) { while (len + piecelen >= cap) { cap *= 2; out = realloc(out, cap); } memcpy(out + len, piece, piecelen); len += piecelen; } } out[len] = 0; *outlen = len; return out; } typedef struct { char *buf; size_t len, cap; int64_t last_end; Input *string; const char *repl; MatchSubCb cb; void *cbctx; int n; int count_limit; } SubCtx; static void subctx_append(SubCtx *sc, const void *data, size_t len) { while (sc->len + len + 1 > sc->cap) { sc->cap = sc->cap ? sc->cap * 2 : 256; sc->buf = realloc(sc->buf, sc->cap); } memcpy(sc->buf + sc->len, data, len); sc->len += len; } static void sub_cb(void *ctx, const Match *m_const) { SubCtx *sc = ctx; Match *m = (Match *)m_const; if (sc->count_limit > 0 && sc->n >= sc->count_limit) return; int64_t mstart_byte = Match_start_byte(m, 0); subctx_append(sc, sc->string->buf + sc->last_end, (size_t)(mstart_byte - sc->last_end)); if (sc->cb) { char *piece = NULL; size_t plen = 0; sc->cb(sc->cbctx, m, &piece, &plen); if (piece) { subctx_append(sc, piece, plen); free(piece); } } else { size_t plen; char *piece = expand_template(sc->repl, m, &plen); subctx_append(sc, piece, plen); free(piece); } sc->last_end = Match_end_byte(m, 0); sc->n++; } int Pattern_subn(Pattern *self, Input *string, const char *repl, MatchSubCb cb, void *ctx, int count, char **out, size_t *outlen, int *n) { SubCtx sc; memset(&sc, 0, sizeof sc); sc.string = string; sc.repl = repl; sc.cb = cb; sc.cbctx = ctx; sc.count_limit = count; sc.last_end = 0; int total = Pattern_finditer(self, string, 0, -1, sub_cb, &sc); if (total < 0) { free(sc.buf); return -1; } subctx_append(&sc, string->buf + sc.last_end, (size_t)((int64_t)string->len - sc.last_end)); if (!sc.buf) sc.buf = malloc(1); sc.buf[sc.len] = 0; *out = sc.buf; *outlen = sc.len; if (n) *n = sc.n; return 1; } int Pattern_sub(Pattern *self, Input *string, const char *repl, MatchSubCb cb, void *ctx, int count, char **out, size_t *outlen) { int n; return Pattern_subn(self, string, repl, cb, ctx, count, out, outlen, &n); } /* ================================================================ * 12. Match accessors * ================================================================ */ int Pattern_groupindex_lookup(Pattern *self, const char *name) { PatternImpl *impl = (PatternImpl *)self->program; return groupindex_lookup(&impl->gidx, name); } int Match_group(Match *self, const char *name_or_null, int index, const char **out, size_t *outlen) { MatchSlots *ms = self->slots; int g = index; if (name_or_null) { PatternImpl *impl = (PatternImpl *)self->re->program; g = groupindex_lookup(&impl->gidx, name_or_null); if (g < 0) return -1; } if (g < 0 || g > ms->ngroups) return -1; int64_t s = ms->pos[2 * g], e = ms->pos[2 * g + 1]; if (s < 0 || e < 0) { *out = NULL; *outlen = 0; return 0; } int64_t bs = ms->cp_to_byte ? ms->cp_to_byte[s] : s; int64_t be = ms->cp_to_byte ? ms->cp_to_byte[e] : e; *out = (const char *)(self->string->buf + bs); *outlen = (size_t)(be - bs); return 1; } int64_t Match_start(Match *self, int group) { MatchSlots *ms = self->slots; if (group < 0 || group > ms->ngroups) return -1; return ms->pos[2 * group]; } int64_t Match_end(Match *self, int group) { MatchSlots *ms = self->slots; if (group < 0 || group > ms->ngroups) return -1; return ms->pos[2 * group + 1]; } void Match_span(Match *self, int group, int64_t *start, int64_t *end) { *start = Match_start(self, group); *end = Match_end(self, group); } int64_t Match_start_byte(Match *self, int group) { MatchSlots *ms = self->slots; if (group < 0 || group > ms->ngroups) return -1; int64_t v = ms->pos[2 * group]; if (v < 0) return -1; return ms->cp_to_byte ? ms->cp_to_byte[v] : v; } int64_t Match_end_byte(Match *self, int group) { MatchSlots *ms = self->slots; if (group < 0 || group > ms->ngroups) return -1; int64_t v = ms->pos[2 * group + 1]; if (v < 0) return -1; return ms->cp_to_byte ? ms->cp_to_byte[v] : v; } void Match_span_byte(Match *self, int group, int64_t *start, int64_t *end) { *start = Match_start_byte(self, group); *end = Match_end_byte(self, group); } void Match_free(Match *self) { if (!self || !self->slots) return; MatchSlots *ms = self->slots; free(ms->pos); free(ms->cp_to_byte); free(ms); self->slots = NULL; } /* ================================================================ * 13. Module level functions, cache, escape (Section 9.1) * ================================================================ */ #define CACHE_CAP 512 typedef struct { char *key; int keylen; int flags; Pattern *pat; } CacheEnt; static CacheEnt g_cache[CACHE_CAP]; static int g_cache_n = 0; void re_purge(void) { for (int i = 0; i < g_cache_n; i++) { free(g_cache[i].key); Pattern_free(g_cache[i].pat); } g_cache_n = 0; } static Pattern *cache_compile(const char *pattern, size_t len, int flags, PatternError *err) { for (int i = 0; i < g_cache_n; i++) if (g_cache[i].flags == flags && (size_t)g_cache[i].keylen == len && memcmp(g_cache[i].key, pattern, len) == 0) return g_cache[i].pat; Pattern *pat = re_compile(pattern, len, flags, err); if (!pat) return NULL; if (g_cache_n >= CACHE_CAP) re_purge(); g_cache[g_cache_n].key = xstrndup(pattern, len); g_cache[g_cache_n].keylen = (int)len; g_cache[g_cache_n].flags = flags; g_cache[g_cache_n].pat = pat; g_cache_n++; return pat; } int re_match(const char *pattern, size_t len, int flags, Input *string, Match *out) { Pattern *pat = cache_compile(pattern, len, flags, NULL); if (!pat) return -1; return Pattern_match(pat, string, 0, -1, out); } int re_fullmatch(const char *pattern, size_t len, int flags, Input *string, Match *out) { Pattern *pat = cache_compile(pattern, len, flags, NULL); if (!pat) return -1; return Pattern_fullmatch(pat, string, 0, -1, out); } int re_search(const char *pattern, size_t len, int flags, Input *string, Match *out) { Pattern *pat = cache_compile(pattern, len, flags, NULL); if (!pat) return -1; return Pattern_search(pat, string, 0, -1, out); } int re_finditer(const char *pattern, size_t len, int flags, Input *string, MatchIterCb cb, void *ctx) { Pattern *pat = cache_compile(pattern, len, flags, NULL); if (!pat) return -1; return Pattern_finditer(pat, string, 0, -1, cb, ctx); } int re_findall(const char *pattern, size_t len, int flags, Input *string, MatchIterCb cb, void *ctx) { return re_finditer(pattern, len, flags, string, cb, ctx); } int re_split(const char *pattern, size_t len, int flags, Input *string, int maxsplit, MatchIterCb cb, void *ctx) { Pattern *pat = cache_compile(pattern, len, flags, NULL); if (!pat) return -1; return Pattern_split(pat, string, maxsplit, cb, ctx); } int re_sub(const char *pattern, size_t len, int flags, Input *string, const char *repl, MatchSubCb cb, void *ctx, int count, char **out, size_t *outlen) { Pattern *pat = cache_compile(pattern, len, flags, NULL); if (!pat) return -1; return Pattern_sub(pat, string, repl, cb, ctx, count, out, outlen); } int re_subn(const char *pattern, size_t len, int flags, Input *string, const char *repl, MatchSubCb cb, void *ctx, int count, char **out, size_t *outlen, int *n) { Pattern *pat = cache_compile(pattern, len, flags, NULL); if (!pat) return -1; return Pattern_subn(pat, string, repl, cb, ctx, count, out, outlen, n); } void re_escape(const char *in, size_t len, char **out, size_t *outlen) { char *buf = malloc(len * 2 + 1); size_t o = 0; for (size_t i = 0; i < len; i++) { unsigned char ch = (unsigned char)in[i]; int is_word = (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || (ch >= '0' && ch <= '9') || ch == '_'; if (!is_word && ch < 0x80) buf[o++] = '\\'; buf[o++] = (char)ch; } buf[o] = 0; *out = buf; *outlen = o; }