feat: add extended language validators and fix tokenizer crash bugs
Expand nimcheck with validators and tokenizers for C, C++, C#, Go, Rust, Ruby, CSS, SQL, Markdown, Dockerfile, Makefile, Kotlin, Lua, Swift, TypeScript, and XML. Register all flavors in the validator factory and improve auto-detection scoring for the new languages. Fix infinite tokenizer loops that caused OOM kills: closeBracket now advances position, finishTokenizeStep guards stalled tokenization, and Jinja/JS tokenizers no longer double-advance on brackets. Fix block-balance false positives in Lua (for/do) and Ruby (postfix unless), SQL trailing-comma detection across whitespace, and Makefile tab literals in test fixtures.
This commit is contained in:
Vendored
+61
@@ -0,0 +1,61 @@
|
||||
package fixtures;
|
||||
|
||||
import java.util.*;
|
||||
import java.nio.charset.*;
|
||||
|
||||
// Unicode comment: 你好世界 🌍🚀
|
||||
|
||||
public class EdgeCases {
|
||||
public static void main(String[] args) throws Exception {
|
||||
// Unicode string literals
|
||||
String unicode = "こんにちは世界";
|
||||
System.out.println(unicode);
|
||||
|
||||
// Null byte in string
|
||||
String nullStr = "hello\u0000world";
|
||||
System.out.println("Null-str length: " + nullStr.length());
|
||||
|
||||
// Deeply nested classes
|
||||
Map<Integer, Map<Integer, Map<Integer, Map<Integer, Map<Integer, String>>>>> deep =
|
||||
new HashMap<>();
|
||||
Map<Integer, Map<Integer, Map<Integer, Map<Integer, String>>>> l4 = new HashMap<>();
|
||||
Map<Integer, Map<Integer, Map<Integer, String>>> l3 = new HashMap<>();
|
||||
Map<Integer, Map<Integer, String>> l2 = new HashMap<>();
|
||||
Map<Integer, String> l1 = new HashMap<>();
|
||||
l1.put(5, "deep");
|
||||
l2.put(4, l1);
|
||||
l3.put(3, l2);
|
||||
l4.put(2, l3);
|
||||
deep.put(1, l4);
|
||||
System.out.println(deep.get(1).get(2).get(3).get(4).get(5));
|
||||
|
||||
// Very long string
|
||||
StringBuilder sb = new StringBuilder(10000);
|
||||
for (int i = 0; i < 10000; i++) sb.append('x');
|
||||
System.out.println("Long string length: " + sb.length());
|
||||
|
||||
// BOM prefix
|
||||
byte[] bom = {(byte)0xEF, (byte)0xBB, (byte)0xBF, (byte)'H', (byte)'i'};
|
||||
System.out.println(new String(bom, StandardCharsets.UTF_8));
|
||||
|
||||
// Deep exception chaining
|
||||
try {
|
||||
try {
|
||||
try {
|
||||
throw new RuntimeException("inner");
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException("middle", e);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException("outer", e);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
System.out.println("Chained: " + e.getMessage());
|
||||
}
|
||||
|
||||
// Deep array nesting
|
||||
int[][][][][][] deepArray = new int[2][2][2][2][2][2];
|
||||
deepArray[0][0][0][0][0][0] = 42;
|
||||
System.out.println("Deep array: " + deepArray[0][0][0][0][0][0]);
|
||||
}
|
||||
}
|
||||
Vendored
+44
@@ -0,0 +1,44 @@
|
||||
package fixtures;
|
||||
|
||||
import java.util.*;
|
||||
|
||||
public class Invalid {
|
||||
private int x;
|
||||
|
||||
public Invalid(int x) {
|
||||
this.x = x
|
||||
}
|
||||
|
||||
public void show() {
|
||||
System.out.println(x)
|
||||
}
|
||||
|
||||
public String broken() {
|
||||
String s = "unclosed string;
|
||||
return s;
|
||||
|
||||
public int missingBody()
|
||||
|
||||
public void badIf() {
|
||||
if (x > 0 {
|
||||
System.out.println("positive");
|
||||
}
|
||||
}
|
||||
|
||||
public void badArray() {
|
||||
int[] arr = {1, 2, 3;
|
||||
System.out.println(arr[0]);
|
||||
}
|
||||
|
||||
public void missingSemicolon() {
|
||||
int a = 5
|
||||
int b = 10;
|
||||
int c = a + b
|
||||
System.out.println(c);
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
Invalid obj = new Invalid(42);
|
||||
obj.show()
|
||||
}
|
||||
}
|
||||
Vendored
+59
@@ -0,0 +1,59 @@
|
||||
package fixtures;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.stream.*;
|
||||
|
||||
public class Valid {
|
||||
public static class Person {
|
||||
private final int id;
|
||||
private final String name;
|
||||
private final String email;
|
||||
|
||||
public Person(int id, String name, String email) {
|
||||
this.id = id;
|
||||
this.name = name;
|
||||
this.email = email;
|
||||
}
|
||||
|
||||
public int getId() { return id; }
|
||||
public String getName() { return name; }
|
||||
public String getEmail() { return email; }
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return name + " <" + email + ">";
|
||||
}
|
||||
}
|
||||
|
||||
public static class Repository<T> {
|
||||
private final Map<Integer, T> items = new HashMap<>();
|
||||
|
||||
public void add(int key, T item) {
|
||||
items.put(key, item);
|
||||
}
|
||||
|
||||
public Optional<T> find(int key) {
|
||||
return Optional.ofNullable(items.get(key));
|
||||
}
|
||||
|
||||
public List<T> getAll() {
|
||||
return new ArrayList<>(items.values());
|
||||
}
|
||||
}
|
||||
|
||||
public static int add(int a, int b) {
|
||||
return a + b;
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
Repository<Person> repo = new Repository<>();
|
||||
repo.add(1, new Person(1, "Alice", "alice@example.com"));
|
||||
repo.add(2, new Person(2, "Bob", "bob@example.com"));
|
||||
|
||||
repo.getAll().stream()
|
||||
.map(Person::getName)
|
||||
.forEach(System.out::println);
|
||||
|
||||
System.out.println("3 + 4 = " + add(3, 4));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user