feat: add extended language validators and fix tokenizer crash bugs

Expand nimcheck with validators and tokenizers for C, C++, C#, Go, Rust,
Ruby, CSS, SQL, Markdown, Dockerfile, Makefile, Kotlin, Lua, Swift,
TypeScript, and XML. Register all flavors in the validator factory and
improve auto-detection scoring for the new languages.

Fix infinite tokenizer loops that caused OOM kills: closeBracket now
advances position, finishTokenizeStep guards stalled tokenization, and
Jinja/JS tokenizers no longer double-advance on brackets.

Fix block-balance false positives in Lua (for/do) and Ruby (postfix
unless), SQL trailing-comma detection across whitespace, and Makefile
tab literals in test fixtures.
This commit is contained in:
2026-07-15 06:49:45 +02:00
parent df2b327a5d
commit 5a7c24f936
109 changed files with 8811 additions and 121 deletions
+61
View File
@@ -0,0 +1,61 @@
package fixtures;
import java.util.*;
import java.nio.charset.*;
// Unicode comment: 你好世界 🌍🚀
public class EdgeCases {
public static void main(String[] args) throws Exception {
// Unicode string literals
String unicode = "こんにちは世界";
System.out.println(unicode);
// Null byte in string
String nullStr = "hello\u0000world";
System.out.println("Null-str length: " + nullStr.length());
// Deeply nested classes
Map<Integer, Map<Integer, Map<Integer, Map<Integer, Map<Integer, String>>>>> deep =
new HashMap<>();
Map<Integer, Map<Integer, Map<Integer, Map<Integer, String>>>> l4 = new HashMap<>();
Map<Integer, Map<Integer, Map<Integer, String>>> l3 = new HashMap<>();
Map<Integer, Map<Integer, String>> l2 = new HashMap<>();
Map<Integer, String> l1 = new HashMap<>();
l1.put(5, "deep");
l2.put(4, l1);
l3.put(3, l2);
l4.put(2, l3);
deep.put(1, l4);
System.out.println(deep.get(1).get(2).get(3).get(4).get(5));
// Very long string
StringBuilder sb = new StringBuilder(10000);
for (int i = 0; i < 10000; i++) sb.append('x');
System.out.println("Long string length: " + sb.length());
// BOM prefix
byte[] bom = {(byte)0xEF, (byte)0xBB, (byte)0xBF, (byte)'H', (byte)'i'};
System.out.println(new String(bom, StandardCharsets.UTF_8));
// Deep exception chaining
try {
try {
try {
throw new RuntimeException("inner");
} catch (Exception e) {
throw new RuntimeException("middle", e);
}
} catch (Exception e) {
throw new RuntimeException("outer", e);
}
} catch (Exception e) {
System.out.println("Chained: " + e.getMessage());
}
// Deep array nesting
int[][][][][][] deepArray = new int[2][2][2][2][2][2];
deepArray[0][0][0][0][0][0] = 42;
System.out.println("Deep array: " + deepArray[0][0][0][0][0][0]);
}
}
+44
View File
@@ -0,0 +1,44 @@
package fixtures;
import java.util.*;
public class Invalid {
private int x;
public Invalid(int x) {
this.x = x
}
public void show() {
System.out.println(x)
}
public String broken() {
String s = "unclosed string;
return s;
public int missingBody()
public void badIf() {
if (x > 0 {
System.out.println("positive");
}
}
public void badArray() {
int[] arr = {1, 2, 3;
System.out.println(arr[0]);
}
public void missingSemicolon() {
int a = 5
int b = 10;
int c = a + b
System.out.println(c);
}
public static void main(String[] args) {
Invalid obj = new Invalid(42);
obj.show()
}
}
+59
View File
@@ -0,0 +1,59 @@
package fixtures;
import java.util.*;
import java.util.stream.*;
public class Valid {
public static class Person {
private final int id;
private final String name;
private final String email;
public Person(int id, String name, String email) {
this.id = id;
this.name = name;
this.email = email;
}
public int getId() { return id; }
public String getName() { return name; }
public String getEmail() { return email; }
@Override
public String toString() {
return name + " <" + email + ">";
}
}
public static class Repository<T> {
private final Map<Integer, T> items = new HashMap<>();
public void add(int key, T item) {
items.put(key, item);
}
public Optional<T> find(int key) {
return Optional.ofNullable(items.get(key));
}
public List<T> getAll() {
return new ArrayList<>(items.values());
}
}
public static int add(int a, int b) {
return a + b;
}
public static void main(String[] args) {
Repository<Person> repo = new Repository<>();
repo.add(1, new Person(1, "Alice", "alice@example.com"));
repo.add(2, new Person(2, "Bob", "bob@example.com"));
repo.getAll().stream()
.map(Person::getName)
.forEach(System.out::println);
System.out.println("3 + 4 = " + add(3, 4));
}
}