feat: add extended language validators and fix tokenizer crash bugs

Expand nimcheck with validators and tokenizers for C, C++, C#, Go, Rust,
Ruby, CSS, SQL, Markdown, Dockerfile, Makefile, Kotlin, Lua, Swift,
TypeScript, and XML. Register all flavors in the validator factory and
improve auto-detection scoring for the new languages.

Fix infinite tokenizer loops that caused OOM kills: closeBracket now
advances position, finishTokenizeStep guards stalled tokenization, and
Jinja/JS tokenizers no longer double-advance on brackets.

Fix block-balance false positives in Lua (for/do) and Ruby (postfix
unless), SQL trailing-comma detection across whitespace, and Makefile
tab literals in test fixtures.
This commit is contained in:
2026-07-15 06:49:45 +02:00
parent df2b327a5d
commit 5a7c24f936
109 changed files with 8811 additions and 121 deletions
+64
View File
@@ -0,0 +1,64 @@
-- Unicode comment: 你好世界 🌍🚀
-- Unicode string literals
local unicode = "こんにちは世界"
print(unicode)
-- Null byte in string
local null_str = "hello\0world"
print("Null-str length: " .. #null_str)
-- Deeply nested tables
local deep = {
a = {
b = {
c = {
d = {
e = "deep"
}
}
}
}
}
print(deep.a.b.c.d.e)
-- Very long string
local long_str = string.rep("x", 10000)
print("Long string length: " .. #long_str)
-- Metatable chains
local a = {}
local b = {}
local c = {}
setmetatable(a, {__index = b})
setmetatable(b, {__index = c})
c.value = 42
print("Metatable chain: " .. a.value)
-- Deep function nesting
local function level1()
return function()
return function()
return function()
return function()
return 42
end
end
end
end
end
print("Deep functions: " .. level1()()()()())
-- Coroutine with deep stack
local co = coroutine.create(function()
local function recurse(n)
if n == 0 then
coroutine.yield("done")
else
return recurse(n - 1)
end
end
recurse(100)
end)
local _, msg = coroutine.resume(co)
print("Coroutine: " .. msg)
+31
View File
@@ -0,0 +1,31 @@
local function broken()
local x = 5
print(x)
end
local s = "unclosed string
print(s)
local function bad_if(x)
if x > 0
print("positive")
end
end
local function missing_end(x)
if x > 0 then
print("positive")
end
local t = {1, 2, 3
print(t[1])
local function nested()
local function inner()
local function deeper()
return 42
end
end
end
print("done"
+57
View File
@@ -0,0 +1,57 @@
local function add(a, b)
return a + b
end
local function greet(name)
print("Hello, " .. name .. "!")
end
local Person = {}
Person.__index = Person
function Person:new(id, name, email)
local obj = {
id = id,
name = name,
email = email
}
setmetatable(obj, self)
return obj
end
function Person:__tostring()
return self.name .. " <" .. self.email .. ">"
end
local Repository = {}
Repository.__index = Repository
function Repository:new()
return setmetatable({items = {}}, self)
end
function Repository:add(key, item)
self.items[key] = item
end
function Repository:find(key)
return self.items[key]
end
function Repository:getAll()
local result = {}
for _, v in pairs(self.items) do
table.insert(result, v)
end
return result
end
local repo = Repository:new()
repo:add(1, Person:new(1, "Alice", "alice@example.com"))
repo:add(2, Person:new(2, "Bob", "bob@example.com"))
for _, p in ipairs(repo:getAll()) do
print(p)
end
print("3 + 4 = " .. add(3, 4))