feat: add extended language validators and fix tokenizer crash bugs

Expand nimcheck with validators and tokenizers for C, C++, C#, Go, Rust,
Ruby, CSS, SQL, Markdown, Dockerfile, Makefile, Kotlin, Lua, Swift,
TypeScript, and XML. Register all flavors in the validator factory and
improve auto-detection scoring for the new languages.

Fix infinite tokenizer loops that caused OOM kills: closeBracket now
advances position, finishTokenizeStep guards stalled tokenization, and
Jinja/JS tokenizers no longer double-advance on brackets.

Fix block-balance false positives in Lua (for/do) and Ruby (postfix
unless), SQL trailing-comma detection across whitespace, and Makefile
tab literals in test fixtures.
This commit is contained in:
2026-07-15 06:49:45 +02:00
parent df2b327a5d
commit 5a7c24f936
109 changed files with 8811 additions and 121 deletions
+46
View File
@@ -0,0 +1,46 @@
#include <iostream>
#include <string>
#include <vector>
#include <locale>
#include <codecvt>
/* Unicode comment: 你好, 世界, 🌍🚀 */
int main() {
// Unicode string literals
std::string utf8 = "こんにちは世界";
std::cout << utf8 << std::endl;
// Null byte in string
std::string null_str = std::string("hello\x00world", 11);
std::cout << "Null-str size: " << null_str.size() << std::endl;
// Deeply nested templates
template <typename T>
struct Wrap { T value; };
Wrap<Wrap<Wrap<Wrap<Wrap<Wrap<Wrap<Wrap<int>>>>>>>> deep = {{{{{{{{42}}}}}}}};
std::cout << deep.value.value.value.value.value.value.value.value << std::endl;
// Extremely long string
std::string long_str(10000, 'x');
std::cout << "Long string length: " << long_str.length() << std::endl;
// BOM-like bytes
const unsigned char bom[] = {0xEF, 0xBB, 0xBF, 'H', 'i', 0};
std::cout << reinterpret_cast<const char*>(bom) << std::endl;
// Deep lambda nesting
auto f1 = [](int x) {
return [x](int y) {
return [x, y](int z) {
return [x, y, z](int w) {
return x + y + z + w;
};
};
};
};
std::cout << "Nested lambdas: " << f1(1)(2)(3)(4) << std::endl;
return 0;
}
+33
View File
@@ -0,0 +1,33 @@
#include <iostream>
#include <string>
class Broken {
public:
Broken(int x) : x_(x) {}
void show() {
std::cout << x_ << std::endl;
int missing_semicolon()
return x_;
}
private:
int x_
};
int main() {
Broken b(42);
b.show()
char *s = "unclosed string;
std::cout << s << std::endl;
if (b.x_ > 0 {
std::cout << "positive" << std::endl;
}
std::vector<int> v = {1, 2, 3};
std::cout << v[5] << std::endl;
return 0;
+58
View File
@@ -0,0 +1,58 @@
#include <iostream>
#include <vector>
#include <string>
#include <memory>
#include <algorithm>
class Person {
public:
Person(int id, std::string name)
: id_(id), name_(std::move(name)) {}
int id() const { return id_; }
const std::string& name() const { return name_; }
virtual void greet() const {
std::cout << "Hello, I'm " << name_ << std::endl;
}
virtual ~Person() = default;
private:
int id_;
std::string name_;
};
class Student : public Person {
public:
Student(int id, std::string name, double gpa)
: Person(id, std::move(name)), gpa_(gpa) {}
void greet() const override {
std::cout << "Hi, I'm " << name() << " (GPA: " << gpa_ << ")" << std::endl;
}
private:
double gpa_;
};
template <typename T>
T max_value(T a, T b) {
return (a > b) ? a : b;
}
int main() {
auto p = std::make_unique<Student>(1, "Alice", 3.9);
p->greet();
std::vector<int> nums = {3, 1, 4, 1, 5, 9};
std::sort(nums.begin(), nums.end());
for (int n : nums) {
std::cout << n << " ";
}
std::cout << std::endl;
std::cout << "Max of 10 and 20: " << max_value(10, 20) << std::endl;
return 0;
}