feat: add extended language validators and fix tokenizer crash bugs
Expand nimcheck with validators and tokenizers for C, C++, C#, Go, Rust, Ruby, CSS, SQL, Markdown, Dockerfile, Makefile, Kotlin, Lua, Swift, TypeScript, and XML. Register all flavors in the validator factory and improve auto-detection scoring for the new languages. Fix infinite tokenizer loops that caused OOM kills: closeBracket now advances position, finishTokenizeStep guards stalled tokenization, and Jinja/JS tokenizers no longer double-advance on brackets. Fix block-balance false positives in Lua (for/do) and Ruby (postfix unless), SQL trailing-comma detection across whitespace, and Makefile tab literals in test fixtures.
This commit is contained in:
Vendored
+59
@@ -0,0 +1,59 @@
|
||||
package fixtures
|
||||
|
||||
// Unicode comment: 你好世界 🌍🚀
|
||||
|
||||
fun main() {
|
||||
// Unicode string literals
|
||||
val unicode = "こんにちは世界"
|
||||
println(unicode)
|
||||
|
||||
// Null byte in string
|
||||
val nullStr = "hello\u0000world"
|
||||
println("Null-str length: ${nullStr.length}")
|
||||
|
||||
// Deeply nested generics
|
||||
val deep = mapOf(
|
||||
1 to mapOf(
|
||||
2 to mapOf(
|
||||
3 to mapOf(
|
||||
4 to mapOf(
|
||||
5 to "deep"
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
println(deep[1]?.get(2)?.get(3)?.get(4)?.get(5))
|
||||
|
||||
// Very long string
|
||||
val longStr = "x".repeat(10000)
|
||||
println("Long string length: ${longStr.length}")
|
||||
|
||||
// BOM bytes
|
||||
val bom = byteArrayOf(0xEF.toByte(), 0xBB.toByte(), 0xBF.toByte(), 'H'.code.toByte(), 'i'.code.toByte())
|
||||
println(String(bom))
|
||||
|
||||
// Deep lambda nesting
|
||||
val f1: (Int) -> (Int) -> (Int) -> (Int) -> Int = { x ->
|
||||
{ y ->
|
||||
{ z ->
|
||||
{ w -> x + y + z + w }
|
||||
}
|
||||
}
|
||||
}
|
||||
println("Nested lambdas: ${f1(1)(2)(3)(4)}")
|
||||
|
||||
// Infinite sequence (lazy)
|
||||
val naturals = generateSequence(0) { it + 1 }
|
||||
println("First 5 naturals: ${naturals.take(5).toList()}")
|
||||
|
||||
// Deep data class nesting
|
||||
data class A(val x: Int)
|
||||
data class B(val a: A)
|
||||
data class C(val b: B)
|
||||
data class D(val c: C)
|
||||
data class E(val d: D)
|
||||
data class F(val e: E)
|
||||
val deepData = F(E(D(C(B(A(42))))))
|
||||
println("Deep data: ${deepData.e.d.c.b.a.x}")
|
||||
}
|
||||
Vendored
+44
@@ -0,0 +1,44 @@
|
||||
package fixtures
|
||||
|
||||
data class Broken(
|
||||
val x: Int,
|
||||
val name: String
|
||||
)
|
||||
|
||||
fun show() {
|
||||
println("hello")
|
||||
val x = 5
|
||||
println(x)
|
||||
}
|
||||
|
||||
class BadRepo<T> {
|
||||
private val items = mutableMapOf<Int, T>()
|
||||
|
||||
fun add(key: Int, item: T) {
|
||||
items[key] = item
|
||||
}
|
||||
|
||||
fun find(key: Int): T? = items[key]
|
||||
}
|
||||
|
||||
fun main() {
|
||||
val repo = BadRepo<String>()
|
||||
repo.add(1, "test")
|
||||
|
||||
val s = "unclosed string;
|
||||
println(s)
|
||||
|
||||
val x = 5
|
||||
if (x > 0 {
|
||||
println("positive")
|
||||
}
|
||||
|
||||
val list = listOf(1, 2, 3
|
||||
println(list)
|
||||
|
||||
fun missingReturn(): Int {
|
||||
val a = 5
|
||||
}
|
||||
|
||||
println("done")
|
||||
}
|
||||
Vendored
+38
@@ -0,0 +1,38 @@
|
||||
package fixtures
|
||||
|
||||
data class Person(
|
||||
val id: Int,
|
||||
val name: String,
|
||||
val email: String
|
||||
)
|
||||
|
||||
class Repository<T> {
|
||||
private val items = mutableMapOf<Int, T>()
|
||||
|
||||
fun add(key: Int, item: T) {
|
||||
items[key] = item
|
||||
}
|
||||
|
||||
fun find(key: Int): T? = items[key]
|
||||
|
||||
fun getAll(): List<T> = items.values.toList()
|
||||
}
|
||||
|
||||
fun add(a: Int, b: Int): Int = a + b
|
||||
|
||||
fun <T> filter(items: List<T>, predicate: (T) -> Boolean): List<T> {
|
||||
return items.filter(predicate)
|
||||
}
|
||||
|
||||
fun main() {
|
||||
val repo = Repository<Person>()
|
||||
repo.add(1, Person(1, "Alice", "alice@example.com"))
|
||||
repo.add(2, Person(2, "Bob", "bob@example.com"))
|
||||
|
||||
repo.getAll().forEach { println(it) }
|
||||
println("3 + 4 = ${add(3, 4)}")
|
||||
|
||||
val numbers = listOf(1, 2, 3, 4, 5)
|
||||
val evens = filter(numbers) { it % 2 == 0 }
|
||||
println("Evens: $evens")
|
||||
}
|
||||
Reference in New Issue
Block a user