feat: add extended language validators and fix tokenizer crash bugs
Expand nimcheck with validators and tokenizers for C, C++, C#, Go, Rust, Ruby, CSS, SQL, Markdown, Dockerfile, Makefile, Kotlin, Lua, Swift, TypeScript, and XML. Register all flavors in the validator factory and improve auto-detection scoring for the new languages. Fix infinite tokenizer loops that caused OOM kills: closeBracket now advances position, finishTokenizeStep guards stalled tokenization, and Jinja/JS tokenizers no longer double-advance on brackets. Fix block-balance false positives in Lua (for/do) and Ruby (postfix unless), SQL trailing-comma detection across whitespace, and Makefile tab literals in test fixtures.
This commit is contained in:
Vendored
+70
@@ -0,0 +1,70 @@
|
||||
using System;
|
||||
using System.Text;
|
||||
using System.Collections.Generic;
|
||||
|
||||
// Unicode comment: 你好世界 🌍🚀
|
||||
|
||||
public class EdgeCases
|
||||
{
|
||||
public static void Main()
|
||||
{
|
||||
// Unicode string literals
|
||||
string unicode = "こんにちは世界";
|
||||
Console.WriteLine(unicode);
|
||||
|
||||
// Null byte embedded
|
||||
string nullStr = "hello\x00world";
|
||||
Console.WriteLine($"Null-str length: {nullStr.Length}");
|
||||
|
||||
// Deeply nested generics
|
||||
var deep = new Dictionary<int, Dictionary<int, Dictionary<int, Dictionary<int, Dictionary<int, string>>>>>
|
||||
{
|
||||
{ 1, new Dictionary<int, Dictionary<int, Dictionary<int, Dictionary<int, string>>>>
|
||||
{
|
||||
{ 2, new Dictionary<int, Dictionary<int, Dictionary<int, string>>>
|
||||
{
|
||||
{ 3, new Dictionary<int, Dictionary<int, string>>
|
||||
{
|
||||
{ 4, new Dictionary<int, string> { { 5, "deep" } } }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
Console.WriteLine(deep[1][2][3][4][5]);
|
||||
|
||||
// Very long string
|
||||
string longStr = new string('x', 10000);
|
||||
Console.WriteLine($"Long string length: {longStr.Length}");
|
||||
|
||||
// BOM prefix
|
||||
byte[] bom = { 0xEF, 0xBB, 0xBF, (byte)'H', (byte)'i' };
|
||||
Console.WriteLine(Encoding.UTF8.GetString(bom));
|
||||
|
||||
// Deep exception nesting
|
||||
try
|
||||
{
|
||||
try
|
||||
{
|
||||
try
|
||||
{
|
||||
throw new InvalidOperationException("inner");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
throw new ApplicationException("middle", ex);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
throw new Exception("outer", ex);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Console.WriteLine($"Nested exception: {ex.Message}");
|
||||
}
|
||||
}
|
||||
}
|
||||
Vendored
+53
@@ -0,0 +1,53 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace Broken
|
||||
{
|
||||
public class BrokenClass
|
||||
{
|
||||
public int X { get; set; }
|
||||
public string Name;
|
||||
|
||||
public BrokenClass(int x, string name)
|
||||
{
|
||||
X = x;
|
||||
Name = name
|
||||
}
|
||||
|
||||
public void Show()
|
||||
{
|
||||
Console.WriteLine($"{Name}: {X}")
|
||||
}
|
||||
|
||||
public int MissingBody()
|
||||
}
|
||||
|
||||
public class Derived : BrokenClass
|
||||
{
|
||||
public Derived() : base(42, "test")
|
||||
{
|
||||
}
|
||||
|
||||
public void BrokenMethod()
|
||||
{
|
||||
string s = "unclosed string;
|
||||
Console.WriteLine(s);
|
||||
|
||||
if (X > 0
|
||||
Console.WriteLine("positive");
|
||||
}
|
||||
|
||||
int[] arr = {1, 2, 3;
|
||||
Console.WriteLine(arr[0]);
|
||||
}
|
||||
}
|
||||
|
||||
public class Program
|
||||
{
|
||||
public static void Main()
|
||||
{
|
||||
var obj = new Derived();
|
||||
obj.Show()
|
||||
}
|
||||
}
|
||||
}
|
||||
Vendored
+73
@@ -0,0 +1,73 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
|
||||
namespace Fixtures
|
||||
{
|
||||
public class Person
|
||||
{
|
||||
public int Id { get; set; }
|
||||
public string Name { get; set; }
|
||||
public string Email { get; set; }
|
||||
|
||||
public Person(int id, string name, string email)
|
||||
{
|
||||
Id = id;
|
||||
Name = name;
|
||||
Email = email;
|
||||
}
|
||||
|
||||
public override string ToString()
|
||||
{
|
||||
return $"{Name} <{Email}>";
|
||||
}
|
||||
}
|
||||
|
||||
public class Repository<T>
|
||||
{
|
||||
private readonly Dictionary<int, T> _items = new();
|
||||
|
||||
public void Add(int key, T item)
|
||||
{
|
||||
_items[key] = item;
|
||||
}
|
||||
|
||||
public T? Find(int key)
|
||||
{
|
||||
return _items.TryGetValue(key, out var item) ? item : default;
|
||||
}
|
||||
|
||||
public IEnumerable<T> GetAll()
|
||||
{
|
||||
return _items.Values.ToList();
|
||||
}
|
||||
}
|
||||
|
||||
public static class Helpers
|
||||
{
|
||||
public static int Add(int a, int b) => a + b;
|
||||
|
||||
public static IEnumerable<T> Filter<T>(IEnumerable<T> items, Func<T, bool> predicate)
|
||||
{
|
||||
return items.Where(predicate);
|
||||
}
|
||||
}
|
||||
|
||||
public class Program
|
||||
{
|
||||
public static void Main(string[] args)
|
||||
{
|
||||
var repo = new Repository<Person>();
|
||||
repo.Add(1, new Person(1, "Alice", "alice@example.com"));
|
||||
repo.Add(2, new Person(2, "Bob", "bob@example.com"));
|
||||
|
||||
var active = repo.GetAll();
|
||||
foreach (var p in active)
|
||||
{
|
||||
Console.WriteLine(p);
|
||||
}
|
||||
|
||||
Console.WriteLine($"3 + 4 = {Helpers.Add(3, 4)}");
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user