feat: add extended language validators and fix tokenizer crash bugs
Expand nimcheck with validators and tokenizers for C, C++, C#, Go, Rust, Ruby, CSS, SQL, Markdown, Dockerfile, Makefile, Kotlin, Lua, Swift, TypeScript, and XML. Register all flavors in the validator factory and improve auto-detection scoring for the new languages. Fix infinite tokenizer loops that caused OOM kills: closeBracket now advances position, finishTokenizeStep guards stalled tokenization, and Jinja/JS tokenizers no longer double-advance on brackets. Fix block-balance false positives in Lua (for/do) and Ruby (postfix unless), SQL trailing-comma detection across whitespace, and Makefile tab literals in test fixtures.
This commit is contained in:
Vendored
+40
@@ -0,0 +1,40 @@
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
#include <wchar.h>
|
||||
#include <locale.h>
|
||||
|
||||
/* Unicode in comments: Привет, 世界, 🌍 */
|
||||
|
||||
int main(void) {
|
||||
setlocale(LC_ALL, "");
|
||||
|
||||
/* Unicode string literals */
|
||||
wchar_t *greeting = L"こんにちは";
|
||||
wprintf(L"%ls\n", greeting);
|
||||
|
||||
/* Null byte embedded */
|
||||
char buf[] = "Hello\x00World";
|
||||
printf("Length: %zu\n", strlen(buf)); /* stops at null byte */
|
||||
|
||||
/* Deeply nested structs */
|
||||
struct a { int x; };
|
||||
struct b { struct a a; };
|
||||
struct c { struct b b; };
|
||||
struct d { struct c c; };
|
||||
struct e { struct d d; };
|
||||
struct f { struct e e; };
|
||||
struct f val = {{{{{{42}}}}}};
|
||||
printf("%d\n", val.e.d.c.b.a.x);
|
||||
|
||||
/* Very long function name */
|
||||
void this_is_an_extremely_long_function_name_that_goes_on_and_on_and_on(void) {
|
||||
printf("deep\n");
|
||||
}
|
||||
this_is_an_extremely_long_function_name_that_goes_on_and_on_and_on();
|
||||
|
||||
/* Encoding: BOM-like byte sequences */
|
||||
unsigned char raw[] = {0xEF, 0xBB, 0xBF, 'A', 'B', 'C', 0};
|
||||
printf("%s\n", raw);
|
||||
|
||||
return 0;
|
||||
}
|
||||
Vendored
+26
@@ -0,0 +1,26 @@
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
#define MAX 10
|
||||
|
||||
int main(void) {
|
||||
int x = 5
|
||||
printf("x is %d\n", x)
|
||||
|
||||
char *s = "unclosed string;
|
||||
printf("%s\n", s);
|
||||
|
||||
int arr[3] = {1, 2, 3;
|
||||
printf("%d\n", arr[1]);
|
||||
|
||||
if (x > 0 {
|
||||
printf("positive\n");
|
||||
}
|
||||
|
||||
char *p = malloc(10;
|
||||
free(p);
|
||||
|
||||
struct { int a; int b; } s = {1, 2};
|
||||
s.a = ;
|
||||
|
||||
return 0;
|
||||
Vendored
+36
@@ -0,0 +1,36 @@
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#define MAX_NAME 64
|
||||
#define GREETING "Hello, World!"
|
||||
|
||||
typedef struct {
|
||||
int id;
|
||||
char name[MAX_NAME];
|
||||
double score;
|
||||
} Player;
|
||||
|
||||
int add(int a, int b) {
|
||||
return a + b;
|
||||
}
|
||||
|
||||
void greet(const char *name) {
|
||||
printf("%s, %s!\n", GREETING, name);
|
||||
}
|
||||
|
||||
int main(void) {
|
||||
Player p = {1, "Alice", 95.5};
|
||||
greet(p.name);
|
||||
|
||||
int result = add(3, 4);
|
||||
printf("3 + 4 = %d\n", result);
|
||||
|
||||
FILE *fp = fopen("test.txt", "w");
|
||||
if (fp) {
|
||||
fprintf(fp, "id=%d,name=%s,score=%.1f\n", p.id, p.name, p.score);
|
||||
fclose(fp);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
Reference in New Issue
Block a user