feat: add extended language validators and fix tokenizer crash bugs
Expand nimcheck with validators and tokenizers for C, C++, C#, Go, Rust, Ruby, CSS, SQL, Markdown, Dockerfile, Makefile, Kotlin, Lua, Swift, TypeScript, and XML. Register all flavors in the validator factory and improve auto-detection scoring for the new languages. Fix infinite tokenizer loops that caused OOM kills: closeBracket now advances position, finishTokenizeStep guards stalled tokenization, and Jinja/JS tokenizers no longer double-advance on brackets. Fix block-balance false positives in Lua (for/do) and Ruby (postfix unless), SQL trailing-comma detection across whitespace, and Makefile tab literals in test fixtures.
This commit is contained in:
Vendored
+46
@@ -0,0 +1,46 @@
|
||||
#include <iostream>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <locale>
|
||||
#include <codecvt>
|
||||
|
||||
/* Unicode comment: 你好, 世界, 🌍🚀 */
|
||||
|
||||
int main() {
|
||||
// Unicode string literals
|
||||
std::string utf8 = "こんにちは世界";
|
||||
std::cout << utf8 << std::endl;
|
||||
|
||||
// Null byte in string
|
||||
std::string null_str = std::string("hello\x00world", 11);
|
||||
std::cout << "Null-str size: " << null_str.size() << std::endl;
|
||||
|
||||
// Deeply nested templates
|
||||
template <typename T>
|
||||
struct Wrap { T value; };
|
||||
|
||||
Wrap<Wrap<Wrap<Wrap<Wrap<Wrap<Wrap<Wrap<int>>>>>>>> deep = {{{{{{{{42}}}}}}}};
|
||||
std::cout << deep.value.value.value.value.value.value.value.value << std::endl;
|
||||
|
||||
// Extremely long string
|
||||
std::string long_str(10000, 'x');
|
||||
std::cout << "Long string length: " << long_str.length() << std::endl;
|
||||
|
||||
// BOM-like bytes
|
||||
const unsigned char bom[] = {0xEF, 0xBB, 0xBF, 'H', 'i', 0};
|
||||
std::cout << reinterpret_cast<const char*>(bom) << std::endl;
|
||||
|
||||
// Deep lambda nesting
|
||||
auto f1 = [](int x) {
|
||||
return [x](int y) {
|
||||
return [x, y](int z) {
|
||||
return [x, y, z](int w) {
|
||||
return x + y + z + w;
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
std::cout << "Nested lambdas: " << f1(1)(2)(3)(4) << std::endl;
|
||||
|
||||
return 0;
|
||||
}
|
||||
Vendored
+33
@@ -0,0 +1,33 @@
|
||||
#include <iostream>
|
||||
#include <string>
|
||||
|
||||
class Broken {
|
||||
public:
|
||||
Broken(int x) : x_(x) {}
|
||||
|
||||
void show() {
|
||||
std::cout << x_ << std::endl;
|
||||
|
||||
int missing_semicolon()
|
||||
return x_;
|
||||
}
|
||||
|
||||
private:
|
||||
int x_
|
||||
};
|
||||
|
||||
int main() {
|
||||
Broken b(42);
|
||||
b.show()
|
||||
|
||||
char *s = "unclosed string;
|
||||
std::cout << s << std::endl;
|
||||
|
||||
if (b.x_ > 0 {
|
||||
std::cout << "positive" << std::endl;
|
||||
}
|
||||
|
||||
std::vector<int> v = {1, 2, 3};
|
||||
std::cout << v[5] << std::endl;
|
||||
|
||||
return 0;
|
||||
Vendored
+58
@@ -0,0 +1,58 @@
|
||||
#include <iostream>
|
||||
#include <vector>
|
||||
#include <string>
|
||||
#include <memory>
|
||||
#include <algorithm>
|
||||
|
||||
class Person {
|
||||
public:
|
||||
Person(int id, std::string name)
|
||||
: id_(id), name_(std::move(name)) {}
|
||||
|
||||
int id() const { return id_; }
|
||||
const std::string& name() const { return name_; }
|
||||
|
||||
virtual void greet() const {
|
||||
std::cout << "Hello, I'm " << name_ << std::endl;
|
||||
}
|
||||
|
||||
virtual ~Person() = default;
|
||||
|
||||
private:
|
||||
int id_;
|
||||
std::string name_;
|
||||
};
|
||||
|
||||
class Student : public Person {
|
||||
public:
|
||||
Student(int id, std::string name, double gpa)
|
||||
: Person(id, std::move(name)), gpa_(gpa) {}
|
||||
|
||||
void greet() const override {
|
||||
std::cout << "Hi, I'm " << name() << " (GPA: " << gpa_ << ")" << std::endl;
|
||||
}
|
||||
|
||||
private:
|
||||
double gpa_;
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
T max_value(T a, T b) {
|
||||
return (a > b) ? a : b;
|
||||
}
|
||||
|
||||
int main() {
|
||||
auto p = std::make_unique<Student>(1, "Alice", 3.9);
|
||||
p->greet();
|
||||
|
||||
std::vector<int> nums = {3, 1, 4, 1, 5, 9};
|
||||
std::sort(nums.begin(), nums.end());
|
||||
|
||||
for (int n : nums) {
|
||||
std::cout << n << " ";
|
||||
}
|
||||
std::cout << std::endl;
|
||||
|
||||
std::cout << "Max of 10 and 20: " << max_value(10, 20) << std::endl;
|
||||
return 0;
|
||||
}
|
||||
Reference in New Issue
Block a user