feat: add extended language validators and fix tokenizer crash bugs

Expand nimcheck with validators and tokenizers for C, C++, C#, Go, Rust,
Ruby, CSS, SQL, Markdown, Dockerfile, Makefile, Kotlin, Lua, Swift,
TypeScript, and XML. Register all flavors in the validator factory and
improve auto-detection scoring for the new languages.

Fix infinite tokenizer loops that caused OOM kills: closeBracket now
advances position, finishTokenizeStep guards stalled tokenization, and
Jinja/JS tokenizers no longer double-advance on brackets.

Fix block-balance false positives in Lua (for/do) and Ruby (postfix
unless), SQL trailing-comma detection across whitespace, and Makefile
tab literals in test fixtures.
This commit is contained in:
2026-07-15 06:49:45 +02:00
parent df2b327a5d
commit 5a7c24f936
109 changed files with 8811 additions and 121 deletions
+59
View File
@@ -0,0 +1,59 @@
package fixtures
// Unicode comment: 你好世界 🌍🚀
fun main() {
// Unicode string literals
val unicode = "こんにちは世界"
println(unicode)
// Null byte in string
val nullStr = "hello\u0000world"
println("Null-str length: ${nullStr.length}")
// Deeply nested generics
val deep = mapOf(
1 to mapOf(
2 to mapOf(
3 to mapOf(
4 to mapOf(
5 to "deep"
)
)
)
)
)
println(deep[1]?.get(2)?.get(3)?.get(4)?.get(5))
// Very long string
val longStr = "x".repeat(10000)
println("Long string length: ${longStr.length}")
// BOM bytes
val bom = byteArrayOf(0xEF.toByte(), 0xBB.toByte(), 0xBF.toByte(), 'H'.code.toByte(), 'i'.code.toByte())
println(String(bom))
// Deep lambda nesting
val f1: (Int) -> (Int) -> (Int) -> (Int) -> Int = { x ->
{ y ->
{ z ->
{ w -> x + y + z + w }
}
}
}
println("Nested lambdas: ${f1(1)(2)(3)(4)}")
// Infinite sequence (lazy)
val naturals = generateSequence(0) { it + 1 }
println("First 5 naturals: ${naturals.take(5).toList()}")
// Deep data class nesting
data class A(val x: Int)
data class B(val a: A)
data class C(val b: B)
data class D(val c: C)
data class E(val d: D)
data class F(val e: E)
val deepData = F(E(D(C(B(A(42))))))
println("Deep data: ${deepData.e.d.c.b.a.x}")
}
+44
View File
@@ -0,0 +1,44 @@
package fixtures
data class Broken(
val x: Int,
val name: String
)
fun show() {
println("hello")
val x = 5
println(x)
}
class BadRepo<T> {
private val items = mutableMapOf<Int, T>()
fun add(key: Int, item: T) {
items[key] = item
}
fun find(key: Int): T? = items[key]
}
fun main() {
val repo = BadRepo<String>()
repo.add(1, "test")
val s = "unclosed string;
println(s)
val x = 5
if (x > 0 {
println("positive")
}
val list = listOf(1, 2, 3
println(list)
fun missingReturn(): Int {
val a = 5
}
println("done")
}
+38
View File
@@ -0,0 +1,38 @@
package fixtures
data class Person(
val id: Int,
val name: String,
val email: String
)
class Repository<T> {
private val items = mutableMapOf<Int, T>()
fun add(key: Int, item: T) {
items[key] = item
}
fun find(key: Int): T? = items[key]
fun getAll(): List<T> = items.values.toList()
}
fun add(a: Int, b: Int): Int = a + b
fun <T> filter(items: List<T>, predicate: (T) -> Boolean): List<T> {
return items.filter(predicate)
}
fun main() {
val repo = Repository<Person>()
repo.add(1, Person(1, "Alice", "alice@example.com"))
repo.add(2, Person(2, "Bob", "bob@example.com"))
repo.getAll().forEach { println(it) }
println("3 + 4 = ${add(3, 4)}")
val numbers = listOf(1, 2, 3, 4, 5)
val evens = filter(numbers) { it % 2 == 0 }
println("Evens: $evens")
}