retoor/nimcheck
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add extended language validators and fix tokenizer crash bugs

Browse Source 
Expand nimcheck with validators and tokenizers for C, C++, C#, Go, Rust,
Ruby, CSS, SQL, Markdown, Dockerfile, Makefile, Kotlin, Lua, Swift,
TypeScript, and XML. Register all flavors in the validator factory and
improve auto-detection scoring for the new languages.

Fix infinite tokenizer loops that caused OOM kills: closeBracket now
advances position, finishTokenizeStep guards stalled tokenization, and
Jinja/JS tokenizers no longer double-advance on brackets.

Fix block-balance false positives in Lua (for/do) and Ruby (postfix
unless), SQL trailing-comma detection across whitespace, and Makefile
tab literals in test fixtures.
This commit is contained in:
retoor
2026-07-15 06:49:45 +02:00
parent df2b327a5d
commit 5a7c24f936
109 changed files with 8811 additions and 121 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/fixtures/xml/edge_cases.xml
Vendored
+46
View File
@@ -0,0 +1,46 @@
<?xml version="1.0" encoding="UTF-8"?>
<!-- Unicode comment: 你好世界 🌍🚀 -->
<edge-cases>
<!-- Unicode content -->
<unicode>こんにちは世界🌍🚀</unicode>
<!-- Null byte in content (will be truncated by XML parser) -->
<null-bytes>hello&#x0;world</null-bytes>
<!-- Deeply nested elements -->
<level1>
<level2>
<level3>
<level4>
<level5>
<level6>
<level7>
<level8>
<value>deep</value>
</level8>
</level7>
</level6>
</level5>
</level4>
</level3>
</level2>
</level1>
<!-- Very long string content -->
<long-string>xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx</long-string>
<!-- BOM prefix (encoded) -->
<bom>&#xEF;&#xBB;&#xBF;Hi</bom>
<!-- Special characters -->
<special-chars>&amp;&lt;&gt;&quot;&apos;</special-chars>
<!-- CDATA with special content -->
<cdata-example><![CDATA[<script>alert("xss")</script>]]></cdata-example>
<!-- Deep attribute nesting (flat but many attributes) -->
<multi-attrs a="1" b="2" c="3" d="4" e="5" f="6" g="7" h="8" i="9" j="10" k="11" l="12" m="13" n="14" o="15" p="16" q="17" r="18" s="19" t="20">many-attrs</multi-attrs>
<!-- Mixed content -->
<mixed>Some text <child>with child</child> and more text <another/> trailing.</mixed>
</edge-cases>
tests/fixtures/xml/invalid.xml
Vendored
+18
View File
@@ -0,0 +1,18 @@
<?xml version="1.0" encoding="UTF-8"?>
<fixtures>
<person id=1>
<name>Alice</name>
<email>alice@example.com
</person>
<person id="2">
<name>Bob</name>
<email>bob@example.com</email>
<active>true</active>
</persn>
<unclosed>
<inner>value
<bad>&unknown;</bad>
<nested>
<unclosedTag>
</metadata>
</fixtures>
tests/fixtures/xml/valid.xml
Vendored
+25
View File
@@ -0,0 +1,25 @@
<?xml version="1.0" encoding="UTF-8"?>
<fixtures>
<person id="1">
<name>Alice</name>
<email>alice@example.com</email>
<active>true</active>
<tags>
<tag>admin</tag>
<tag>backend</tag>
</tags>
</person>
<person id="2">
<name>Bob</name>
<email>bob@example.com</email>
<active>false</active>
<tags>
<tag>user</tag>
</tags>
</person>
<metadata>
<source>test-fixture</source>
<version>1.0</version>
<generated>2026-07-13</generated>
</metadata>
</fixtures>