feat: add extended language validators and fix tokenizer crash bugs

Expand nimcheck with validators and tokenizers for C, C++, C#, Go, Rust,
Ruby, CSS, SQL, Markdown, Dockerfile, Makefile, Kotlin, Lua, Swift,
TypeScript, and XML. Register all flavors in the validator factory and
improve auto-detection scoring for the new languages.

Fix infinite tokenizer loops that caused OOM kills: closeBracket now
advances position, finishTokenizeStep guards stalled tokenization, and
Jinja/JS tokenizers no longer double-advance on brackets.

Fix block-balance false positives in Lua (for/do) and Ruby (postfix
unless), SQL trailing-comma detection across whitespace, and Makefile
tab literals in test fixtures.
This commit is contained in:
2026-07-15 06:49:45 +02:00
parent df2b327a5d
commit 5a7c24f936
109 changed files with 8811 additions and 121 deletions
+40
View File
@@ -0,0 +1,40 @@
#include <stdio.h>
#include <string.h>
#include <wchar.h>
#include <locale.h>
/* Unicode in comments: Привет, 世界, 🌍 */
int main(void) {
setlocale(LC_ALL, "");
/* Unicode string literals */
wchar_t *greeting = L"こんにちは";
wprintf(L"%ls\n", greeting);
/* Null byte embedded */
char buf[] = "Hello\x00World";
printf("Length: %zu\n", strlen(buf)); /* stops at null byte */
/* Deeply nested structs */
struct a { int x; };
struct b { struct a a; };
struct c { struct b b; };
struct d { struct c c; };
struct e { struct d d; };
struct f { struct e e; };
struct f val = {{{{{{42}}}}}};
printf("%d\n", val.e.d.c.b.a.x);
/* Very long function name */
void this_is_an_extremely_long_function_name_that_goes_on_and_on_and_on(void) {
printf("deep\n");
}
this_is_an_extremely_long_function_name_that_goes_on_and_on_and_on();
/* Encoding: BOM-like byte sequences */
unsigned char raw[] = {0xEF, 0xBB, 0xBF, 'A', 'B', 'C', 0};
printf("%s\n", raw);
return 0;
}
+26
View File
@@ -0,0 +1,26 @@
#include <stdio.h>
#include <stdlib.h>
#define MAX 10
int main(void) {
int x = 5
printf("x is %d\n", x)
char *s = "unclosed string;
printf("%s\n", s);
int arr[3] = {1, 2, 3;
printf("%d\n", arr[1]);
if (x > 0 {
printf("positive\n");
}
char *p = malloc(10;
free(p);
struct { int a; int b; } s = {1, 2};
s.a = ;
return 0;
+36
View File
@@ -0,0 +1,36 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#define MAX_NAME 64
#define GREETING "Hello, World!"
typedef struct {
int id;
char name[MAX_NAME];
double score;
} Player;
int add(int a, int b) {
return a + b;
}
void greet(const char *name) {
printf("%s, %s!\n", GREETING, name);
}
int main(void) {
Player p = {1, "Alice", 95.5};
greet(p.name);
int result = add(3, 4);
printf("3 + 4 = %d\n", result);
FILE *fp = fopen("test.txt", "w");
if (fp) {
fprintf(fp, "id=%d,name=%s,score=%.1f\n", p.id, p.name, p.score);
fclose(fp);
}
return 0;
}