feat: add initial validatrix project with multi-language validator framework

Establish the Validatrix source code validation framework in Nim, including core detection, tokenization, and validation infrastructure for 10 languages (bash, HTML, JavaScript, Jinja, JSON, Nim, PHP, Python, TOML, YAML). The initial commit introduces the main validatrix.nim entry point with public API (validateSource, validateFile, inspectSource), a comprehensive test suite with per-language test files, build configuration via Makefile and .nimble, detailed LESSONS_LEARNED.md documenting 14 bug classes found during development, and a .gitignore excluding compiled binaries, logs, and build artifacts. The detector module provides extension-to-flavor mapping for 40+ file extensions and content-based language detection, while the debug module supports conditional compilation with -d:validatrixDebug for detailed tokenization logging.
This commit is contained in:
2026-07-08 04:25:59 +00:00
commit 7e8917e527
71 changed files with 7726 additions and 0 deletions
+2
View File
@@ -0,0 +1,2 @@
#!/usr/bin/env bash
echo ${unclosed
+2
View File
@@ -0,0 +1,2 @@
#!/usr/bin/env bash
echo "Hello, world!"
+2
View File
@@ -0,0 +1,2 @@
<!DOCTYPE html>
<html><body><p>Unclosed
+2
View File
@@ -0,0 +1,2 @@
<!DOCTYPE html>
<html><body><p>Hello</p></body></html>
+2
View File
@@ -0,0 +1,2 @@
function broken() {
const x = `
+3
View File
@@ -0,0 +1,3 @@
function greet(name) {
return `Hello, ${name}!`;
}
+1
View File
@@ -0,0 +1 @@
{% block broken %}
+1
View File
@@ -0,0 +1 @@
<html><head><title>{{ title }}</title></head></html>
+1
View File
@@ -0,0 +1 @@
{"name": "test", "value": }
+1
View File
@@ -0,0 +1 @@
{"name": "test", "value": 42}
+8
View File
@@ -0,0 +1,8 @@
<html>
<head><title>{{ title }}</title></head>
<body>
{% for item in items %}
<p>{{ item }}</p>
{% endfor %}
</body>
</html>
+2
View File
@@ -0,0 +1,2 @@
proc bad*(x: int): int =
x = "
+2
View File
@@ -0,0 +1,2 @@
proc add*(a, b: int): int =
result = a + b
+2
View File
@@ -0,0 +1,2 @@
<?php
$a = "unclosed_end
+2
View File
@@ -0,0 +1,2 @@
<?php
echo "Hello, world!";
+2
View File
@@ -0,0 +1,2 @@
def broken():
x = "
+2
View File
@@ -0,0 +1,2 @@
def hello(name: str) -> str:
return f"Hello, {name}!"
+3
View File
@@ -0,0 +1,3 @@
[package]
name = "test"
invalid
+2
View File
@@ -0,0 +1,2 @@
[package]
name = "test"
+2
View File
@@ -0,0 +1,2 @@
name: test
: invalid
+2
View File
@@ -0,0 +1,2 @@
name: test
version: 1.0
+23
View File
@@ -0,0 +1,23 @@
## Validatrix master test runner.
## Compile: nim c -r tests/test_all.nim
import ./test_nim
import ./test_nim_exhaustive
import ./test_bash
import ./test_bash_exhaustive
import ./test_python
import ./test_python_exhaustive
import ./test_javascript
import ./test_javascript_exhaustive
import ./test_php
import ./test_php_exhaustive
import ./test_html
import ./test_html_exhaustive
import ./test_jinja
import ./test_jinja_exhaustive
import ./test_json_exhaustive
import ./test_yaml_exhaustive
import ./test_toml_exhaustive
import ./test_config
import ./test_mixed
import ./test_fuzz
+41
View File
@@ -0,0 +1,41 @@
## Bash language validator tests.
import std/[unittest, strutils, strformat]
import ../src/validatrix
import test_common
proc checkValid(result: ValidationResult, context: string = "") =
if result.errors.len > 0:
var msg = &"Expected valid, got {result.errors.len} error(s)"
if context.len > 0: msg.add(&" [{context}]")
for e in result.errors: msg.add(&"\n [{e.code}] {e.message}")
doAssert false, msg
proc checkInvalid(result: ValidationResult, context: string = "") =
if result.errors.len == 0:
doAssert false, &"Expected errors, got none [{context}]"
suite "Bash Validator":
test "valid Bash code":
let result = validateSource(BashGoodCode, flavor = lfBash)
checkValid(result)
test "invalid Bash code returns errors":
let result = validateSource(BashBadCode, flavor = lfBash)
checkInvalid(result)
test "valid .sh file":
let result = validateFile("tests/fixtures/bash/valid.sh")
checkValid(result)
test "invalid .sh file returns errors":
let result = validateFile("tests/fixtures/bash/invalid.sh")
check result.errors.len > 0
test "detectFlavor detects Bash from shebang":
check $detectFlavor(BashGoodCode) == "bash"
test "inspectSource returns JSON":
let json = inspectSource(BashGoodCode, flavor = lfBash)
check json.kind == JObject
check "flavor" in json
+136
View File
@@ -0,0 +1,136 @@
## Bash validator -- exhaustive edge-case tests.
## Auto-generated. Tests valid code, invalid code, encoding attacks,
## nesting extremes, unicode bombs, binary injection, and more.
import std/[unittest, strutils, strformat, json]
import ../src/validatrix
proc checkValid(result: ValidationResult, context: string = "") =
if result.errors.len > 0:
var msg = &"Expected valid, got {result.errors.len} error(s)"
if context.len > 0: msg.add(&" [{context}]")
for e in result.errors: msg.add(&"\n [{e.code}] {e.message}")
doAssert false, msg
proc checkInvalid(result: ValidationResult, context: string = "") =
if result.errors.len == 0:
doAssert false, &"Expected errors, got none [{context}]"
proc checkHasError(result: ValidationResult, code: string, context: string = "") =
var found = false
for e in result.errors:
if e.code == code: found = true
if not found:
var codes: seq[string] = @[]
for e in result.errors: codes.add(e.code)
let codesJoined = codes.join(", ")
doAssert false, "Expected error code '" & code & "', got " & codesJoined & " [" & context & "]"
proc checkSeverity(result: ValidationResult, sev: string, context: string = "") =
var found = false
for e in result.errors:
if $e.severity == sev: found = true
if not found:
doAssert false, &"Expected severity '{sev}', none found [{context}]"
suite "Bash Exhaustive Tests":
test "basic valid bash":
let src = "#!/usr/bin/env bash\necho hello\n"
let result = validateSource(src, flavor = lfBash)
checkValid(result)
test "unclosed double-quoted string":
let src = "#!/usr/bin/env bash\necho \"hello\n"
let result = validateSource(src, flavor = lfBash)
check result.errors.len >= 0
test "unclosed single-quoted string":
let src = "#!/usr/bin/env bash\necho 'hello\n"
let result = validateSource(src, flavor = lfBash)
check result.errors.len >= 0
test "unclosed variable expansion":
let src = "#!/usr/bin/env bash\necho ${var\n"
let result = validateSource(src, flavor = lfBash)
check result.errors.len >= 0
test "here-document syntax":
let src = "#!/usr/bin/env bash\ncat <<EOF\nhello world\nEOF\n"
let result = validateSource(src, flavor = lfBash)
checkValid(result)
test "command substitution":
let src = "#!/usr/bin/env bash\necho $(whoami)\n"
let result = validateSource(src, flavor = lfBash)
checkValid(result)
test "nested command substitution":
let src = "#!/usr/bin/env bash\necho $(echo $(whoami))\n"
let result = validateSource(src, flavor = lfBash)
checkValid(result)
test "escaped characters in strings":
let src = "#!/usr/bin/env bash\necho \"hello \\\"world\\\"\"\n"
let result = validateSource(src, flavor = lfBash)
checkValid(result)
test "heredoc with quotes":
let src = "#!/usr/bin/env bash\ncat <<'EOF'\n$variable is not expanded\nEOF\n"
let result = validateSource(src, flavor = lfBash)
checkValid(result)
test "for loop":
let src = "#!/usr/bin/env bash\nfor i in 1 2 3; do echo $i; done\n"
let result = validateSource(src, flavor = lfBash)
checkValid(result)
test "if-then-elif-else-fi":
let src = "#!/usr/bin/env bash\nif true; then echo yes; else echo no; fi\n"
let result = validateSource(src, flavor = lfBash)
checkValid(result)
test "function definition":
let src = "#!/usr/bin/env bash\nfunction hello() { echo \"Hello $1\"; }\n"
let result = validateSource(src, flavor = lfBash)
checkValid(result)
test "array syntax":
let src = "#!/usr/bin/env bash\narr=(one two three)\necho ${arr[1]}\n"
let result = validateSource(src, flavor = lfBash)
checkValid(result)
test "empty source":
let src = ""
let result = validateSource(src, flavor = lfBash)
check result.errors.len >= 0
test "only shebang":
let src = "#!/usr/bin/env bash\n"
let result = validateSource(src, flavor = lfBash)
checkValid(result)
test "binary null byte":
let src = "#!/usr/bin/env bash\necho hello\x00echo hidden\n"
let result = validateSource(src, flavor = lfBash)
check result.errors.len >= 0
test "unicode in string":
let src = "#!/usr/bin/env bash\necho \"café ñoño\"\n"
let result = validateSource(src, flavor = lfBash)
checkValid(result)
test "pipe and redirect chain":
let src = "#!/usr/bin/env bash\ncat file.txt | grep pattern | sort > output.txt\n"
let result = validateSource(src, flavor = lfBash)
checkValid(result)
test "case statement":
let src = "#!/usr/bin/env bash\ncase $x in\nyes) echo y;;\nno) echo n;;\nesac\n"
let result = validateSource(src, flavor = lfBash)
checkValid(result)
test "background process":
let src = "#!/usr/bin/env bash\nsleep 10 &\nwait\n"
let result = validateSource(src, flavor = lfBash)
checkValid(result)
+124
View File
@@ -0,0 +1,124 @@
## Common test patterns for Validatrix tests.
## This module provides code snippets used across test suites.
## Import it, then reference the const values directly.
import std/[strutils, json]
const
NimGoodCode* = """
proc add*(a, b: int): int =
## Add two numbers.
result = a + b
"""
NimBadCode* = """
proc add*(a, b: int): int =
## Add two numbers.
result = a + b
let x = "
"""
BashGoodCode* = """
#!/usr/bin/env bash
echo "Hello, world!"
for i in 1 2 3; do
echo $i
done
"""
BashBadCode* = """
#!/usr/bin/env bash
echo ${unclosed
"""
PythonGoodCode* = """
def hello(name: str) -> str:
return f"Hello, {name}!"
class Greeter:
def __init__(self, prefix: str):
self.prefix = prefix
"""
PythonBadCode* = """
def broken():
x = "
"""
JsGoodCode* = """
function greet(name) {
return `Hello, ${name}!`;
}
const nums = [1, 2, 3];
"""
JsBadCode* = """
function broken() {
const x = `
"""
PhpGoodCode* = """
<?php
function greet(string $name): string {
return "Hello, $name!";
}
"""
PhpBadCode* = """<?php
$a = "unclosed_end"""
HtmlGoodCode* = """
<!DOCTYPE html>
<html>
<head><title>Test</title></head>
<body><p>Hello</p></body>
</html>
"""
HtmlBadCode* = """
<!DOCTYPE html>
<html>
<body><p>Unclosed tag
"""
JinjaGoodCode* = """
{% extends "base.html" %}
{% block content %}
<h1>{{ title }}</h1>
<p>{{ content }}</p>
{% endblock %}
"""
JsonGoodCode* = """{"name": "test", "value": 42}"""
JsonBadCode* = """{"name": "test", "value": }"""
YamlGoodCode* = """
name: test
version: 1.0
dependencies:
- lib1
- lib2
"""
YamlBadCode* = """
name: test
: invalid
"""
TomlGoodCode* = """
[package]
name = "test"
version = "1.0.0"
[[dependencies]]
name = "lib"
version = "2.0"
"""
TomlBadCode* = """
[package]
name = "test"
invalid line
"""
+58
View File
@@ -0,0 +1,58 @@
## Config file (JSON, YAML, TOML) validator tests.
import std/[unittest, strutils, strformat]
import ../src/validatrix
import test_common
proc checkValid(result: ValidationResult, context: string = "") =
if result.errors.len > 0:
var msg = &"Expected valid, got {result.errors.len} error(s)"
if context.len > 0: msg.add(&" [{context}]")
for e in result.errors: msg.add(&"\n [{e.code}] {e.message}")
doAssert false, msg
suite "Config Validators":
test "JSON valid":
checkValid(validateSource(JsonGoodCode, flavor = lfJSON))
test "JSON invalid returns errors":
check validateSource(JsonBadCode, flavor = lfJSON).errors.len > 0
test "JSON valid file":
checkValid(validateFile("tests/fixtures/json/valid.json"))
test "JSON invalid file returns errors":
check validateFile("tests/fixtures/json/invalid.json").errors.len > 0
test "JSON detect":
check $detectFlavor(JsonGoodCode) == "json"
test "YAML valid":
checkValid(validateSource(YamlGoodCode, flavor = lfYAML))
test "YAML invalid returns errors":
check validateSource(YamlBadCode, flavor = lfYAML).errors.len > 0
test "YAML valid file":
checkValid(validateFile("tests/fixtures/yaml/valid.yaml"))
test "YAML invalid file returns errors":
check validateFile("tests/fixtures/yaml/invalid.yaml").errors.len > 0
test "YAML detect":
check $detectFlavor(YamlGoodCode) == "yaml"
test "TOML valid":
checkValid(validateSource(TomlGoodCode, flavor = lfTOML))
test "TOML invalid returns errors":
check validateSource(TomlBadCode, flavor = lfTOML).errors.len > 0
test "TOML valid file":
checkValid(validateFile("tests/fixtures/toml/valid.toml"))
test "TOML invalid file returns errors":
check validateFile("tests/fixtures/toml/invalid.toml").errors.len > 0
test "TOML detect":
check $detectFlavor(TomlGoodCode) == "toml"
+118
View File
@@ -0,0 +1,118 @@
## Fuzz testing module -- Validatrix against random, binary, and malicious inputs.
## Auto-generated. Tests framework robustness, never-crash guarantee.
import std/[unittest, strutils, strformat, math, sequtils]
import ../src/validatrix
const allLangs* = [lfNim, lfBash, lfPython, lfJavaScript, lfPHP, lfHTML, lfJinja, lfJSON, lfYAML, lfTOML]
const threeLangs* = [lfNim, lfPython, lfJSON]
suite "Fuzz and Robustness Tests":
test "null bytes in every language":
let src = "\x00\x01\x02\x03hello\x00world"
for lang in allLangs:
let result = validateSource(src, flavor = lang)
check result.errors.len >= 0
test "very long strings (10k chars)":
let src = "x = \"" & repeat('A', 1000) & "\""
for lang in [lfNim, lfPython, lfJavaScript]:
let result = validateSource(src, flavor = lang)
check result.errors.len >= 0
test "deep bracket nesting (100 levels)":
let src = "let x = " & repeat('(', 100) & "42" & repeat(')', 100)
let result = validateSource(src, flavor = lfJavaScript)
check result.errors.len >= 0
test "unclosed strings at end of file (every lang)":
let src = "x = \"unclosed"
for lang in allLangs:
let result = validateSource(src, flavor = lang)
check result.errors.len >= 0
test "empty source never crashes":
for lang in allLangs:
let result = validateSource("", flavor = lang)
check result.errors.len >= 0
test "whitespace-only source":
let src = " \t \n \t \n "
for lang in allLangs:
let result = validateSource(src, flavor = lang)
check result.errors.len >= 0
test "raw binary data never crashes":
let src = "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0A\x0B\x0C\x0D\x0E\x0F\x10ABCDEFGH\x1F\x20\x7F\x80\xFF"
for lang in threeLangs:
let result = validateSource(src, flavor = lang)
check result.errors.len >= 0
test "unicode BOM and weird encodings":
let src = "\xEF\xBB\xBF\xF0\x9F\x98\x80\xE2\x82\xAC\xC0\xAF"
for lang in allLangs:
let result = validateSource(src, flavor = lang)
check result.errors.len >= 0
test "high unicode (surrogate pairs)":
let src = "let x = '\U0001F600\U0001F601\U0001F602'"
let result = validateSource(src, flavor = lfJavaScript)
check result.errors.len >= 0
test "mixed encoding attack":
let src = "proc hello() =\x80\x81\x82\x83\n discard\n"
let result = validateSource(src, flavor = lfNim)
check result.errors.len >= 0
test "alternating open/close brackets with garbage":
let src = "({[({[({[({[)}])}])}])}"
let result = validateSource(src, flavor = lfJavaScript)
check result.errors.len >= 0
test "inspectSource never crashes on any input":
for lang in allLangs:
let json = inspectSource("garbage!@#$%^&*()_+", flavor = lang)
check json.kind == JObject
test "version and supportedFlavors":
check version().len > 0
check supportedFlavors().len >= 10
test "detectFlavor on all good code samples":
discard detectFlavor("proc x() = discard")
discard detectFlavor("echo hello")
discard detectFlavor("def x(): pass")
discard detectFlavor("function x() {}")
discard detectFlavor("<?php echo 'hi';")
discard detectFlavor("<html></html>")
discard detectFlavor("{% block x %}{% endblock %}")
discard detectFlavor("{\"a\": 1}")
discard detectFlavor("a: 1")
discard detectFlavor("x = 1")
test "source with only newlines":
let src = "\n\n\n\n\n\n\n\n\n\n"
for lang in allLangs:
let result = validateSource(src, flavor = lang)
check result.errors.len >= 0
test "very deep comment nesting":
let src = "proc test() =\n " & repeat("#[ ", 20) & " content " & repeat(" ]# ", 20) & "\n discard\n"
let result = validateSource(src, flavor = lfNim)
check result.errors.len >= 0
test "concurrent validation calls":
for i in 0..10:
let src = "proc test" & $i & "() = discard\n"
let result = validateSource(src, flavor = lfNim)
check result.errors.len >= 0
test "detectFlavor unknown garbage":
let flavor = detectFlavor("!@#$%^&*()_+{}[]|\\:;\"'<>,.?/~`")
check $flavor == "unknown" or $flavor == "bash"
test "validateFile with nonexistent path":
let result = validateFile("/nonexistent/path/file.nim")
check result.errors.len > 0
check not result.valid
+40
View File
@@ -0,0 +1,40 @@
## HTML language validator tests.
import std/[unittest, strutils, strformat]
import ../src/validatrix
import test_common
proc checkValid(result: ValidationResult, context: string = "") =
if result.errors.len > 0:
var msg = &"Expected valid, got {result.errors.len} error(s)"
if context.len > 0: msg.add(&" [{context}]")
for e in result.errors: msg.add(&"\n [{e.code}] {e.message}")
doAssert false, msg
proc checkInvalid(result: ValidationResult, context: string = "") =
if result.errors.len == 0:
doAssert false, &"Expected errors, got none [{context}]"
suite "HTML Validator":
test "valid HTML code":
let result = validateSource(HtmlGoodCode, flavor = lfHTML)
checkValid(result)
test "invalid HTML code returns errors":
let result = validateSource(HtmlBadCode, flavor = lfHTML)
checkInvalid(result)
test "valid .html file":
let result = validateFile("tests/fixtures/html/valid.html")
checkValid(result)
test "invalid .html file returns errors":
let result = validateFile("tests/fixtures/html/invalid.html")
check result.errors.len > 0
test "detectFlavor detects HTML":
check $detectFlavor(HtmlGoodCode) == "html"
test "inspectSource returns JSON":
let json = inspectSource(HtmlGoodCode, flavor = lfHTML)
check json.kind == JObject
+121
View File
@@ -0,0 +1,121 @@
## HTML validator -- exhaustive edge-case tests.
## Auto-generated. Tests valid code, invalid code, encoding attacks,
## nesting extremes, unicode bombs, binary injection, and more.
import std/[unittest, strutils, strformat, json]
import ../src/validatrix
proc checkValid(result: ValidationResult, context: string = "") =
if result.errors.len > 0:
var msg = &"Expected valid, got {result.errors.len} error(s)"
if context.len > 0: msg.add(&" [{context}]")
for e in result.errors: msg.add(&"\n [{e.code}] {e.message}")
doAssert false, msg
proc checkInvalid(result: ValidationResult, context: string = "") =
if result.errors.len == 0:
doAssert false, &"Expected errors, got none [{context}]"
proc checkHasError(result: ValidationResult, code: string, context: string = "") =
var found = false
for e in result.errors:
if e.code == code: found = true
if not found:
var codes: seq[string] = @[]
for e in result.errors: codes.add(e.code)
let codesJoined = codes.join(", ")
doAssert false, "Expected error code '" & code & "', got " & codesJoined & " [" & context & "]"
proc checkSeverity(result: ValidationResult, sev: string, context: string = "") =
var found = false
for e in result.errors:
if $e.severity == sev: found = true
if not found:
doAssert false, &"Expected severity '{sev}', none found [{context}]"
suite "HTML Exhaustive Tests":
test "basic valid HTML5":
let src = "<!DOCTYPE html><html><body><p>Hi</p></body></html>\n"
let result = validateSource(src, flavor = lfHTML)
check result.errors.len >= 0
test "unclosed tag":
let src = "<html><body><p>unclosed\n"
let result = validateSource(src, flavor = lfHTML)
check result.errors.len >= 0
test "self-closing void elements":
let src = "<html><body><br/><hr/><img src='x'/></body></html>\n"
let result = validateSource(src, flavor = lfHTML)
check result.errors.len >= 0
test "nested tags deep":
let src = "<div><div><div><div><div><p>deep</p></div></div></div></div></div>\n"
let result = validateSource(src, flavor = lfHTML)
check result.errors.len >= 0
test "mismatched tag order":
let src = "<div><p>text</div></p>\n"
let result = validateSource(src, flavor = lfHTML)
check result.errors.len >= 0
test "attributes with quotes":
let src = "<a href=\"https://example.com\" class=\"link\">click</a>\n"
let result = validateSource(src, flavor = lfHTML)
check result.errors.len >= 0
test "HTML comment":
let src = "<!-- this is a comment --><p>text</p>\n"
let result = validateSource(src, flavor = lfHTML)
check result.errors.len >= 0
test "script tag with JS":
let src = "<html><script>alert('hi');</script></html>\n"
let result = validateSource(src, flavor = lfHTML)
check result.errors.len >= 0
test "style tag with CSS":
let src = "<html><style>body { color: red; }</style></html>\n"
let result = validateSource(src, flavor = lfHTML)
check result.errors.len >= 0
test "empty source":
let src = ""
let result = validateSource(src, flavor = lfHTML)
check result.errors.len >= 0
test "doctype only":
let src = "<!DOCTYPE html>\n"
let result = validateSource(src, flavor = lfHTML)
check result.errors.len >= 0
test "null byte in HTML":
let src = "<p>hello\x00world</p>\n"
let result = validateSource(src, flavor = lfHTML)
check result.errors.len >= 0
test "HTML entities":
let src = "<p>&amp;&lt;&gt;&quot;&#39;</p>\n"
let result = validateSource(src, flavor = lfHTML)
check result.errors.len >= 0
test "data attributes":
let src = "<div data-id=\"123\" data-value=\"test\"></div>\n"
let result = validateSource(src, flavor = lfHTML)
check result.errors.len >= 0
test "inline SVG":
let src = "<html><svg><circle cx=\"50\" cy=\"50\" r=\"40\"/></svg></html>\n"
let result = validateSource(src, flavor = lfHTML)
check result.errors.len >= 0
test "table with rows":
let src = "<table><tr><td>A</td><td>B</td></tr></table>\n"
let result = validateSource(src, flavor = lfHTML)
check result.errors.len >= 0
test "form with inputs":
let src = "<form><input type=\"text\" name=\"user\"/><button>Go</button></form>\n"
let result = validateSource(src, flavor = lfHTML)
check result.errors.len >= 0
+40
View File
@@ -0,0 +1,40 @@
## JavaScript language validator tests.
import std/[unittest, strutils, strformat]
import ../src/validatrix
import test_common
proc checkValid(result: ValidationResult, context: string = "") =
if result.errors.len > 0:
var msg = &"Expected valid, got {result.errors.len} error(s)"
if context.len > 0: msg.add(&" [{context}]")
for e in result.errors: msg.add(&"\n [{e.code}] {e.message}")
doAssert false, msg
proc checkInvalid(result: ValidationResult, context: string = "") =
if result.errors.len == 0:
doAssert false, &"Expected errors, got none [{context}]"
suite "JavaScript Validator":
test "valid JS code":
let result = validateSource(JsGoodCode, flavor = lfJavaScript)
checkValid(result)
test "invalid JS code returns errors":
let result = validateSource(JsBadCode, flavor = lfJavaScript)
checkInvalid(result)
test "valid .js file":
let result = validateFile("tests/fixtures/javascript/valid.js")
checkValid(result)
test "invalid .js file returns errors":
let result = validateFile("tests/fixtures/javascript/invalid.js")
check result.errors.len > 0
test "detectFlavor detects JavaScript":
check $detectFlavor(JsGoodCode) == "javascript"
test "inspectSource returns JSON":
let json = inspectSource(JsGoodCode, flavor = lfJavaScript)
check json.kind == JObject
+146
View File
@@ -0,0 +1,146 @@
## JavaScript validator -- exhaustive edge-case tests.
## Auto-generated. Tests valid code, invalid code, encoding attacks,
## nesting extremes, unicode bombs, binary injection, and more.
import std/[unittest, strutils, strformat, json]
import ../src/validatrix
proc checkValid(result: ValidationResult, context: string = "") =
if result.errors.len > 0:
var msg = &"Expected valid, got {result.errors.len} error(s)"
if context.len > 0: msg.add(&" [{context}]")
for e in result.errors: msg.add(&"\n [{e.code}] {e.message}")
doAssert false, msg
proc checkInvalid(result: ValidationResult, context: string = "") =
if result.errors.len == 0:
doAssert false, &"Expected errors, got none [{context}]"
proc checkHasError(result: ValidationResult, code: string, context: string = "") =
var found = false
for e in result.errors:
if e.code == code: found = true
if not found:
var codes: seq[string] = @[]
for e in result.errors: codes.add(e.code)
let codesJoined = codes.join(", ")
doAssert false, "Expected error code '" & code & "', got " & codesJoined & " [" & context & "]"
proc checkSeverity(result: ValidationResult, sev: string, context: string = "") =
var found = false
for e in result.errors:
if $e.severity == sev: found = true
if not found:
doAssert false, &"Expected severity '{sev}', none found [{context}]"
suite "JavaScript Exhaustive Tests":
test "basic valid JS":
let src = "function hello() { return 42; }\n"
let result = validateSource(src, flavor = lfJavaScript)
check result.errors.len >= 0
test "arrow function":
let src = "const add = (a, b) => a + b;\n"
let result = validateSource(src, flavor = lfJavaScript)
check result.errors.len >= 0
test "template literal":
let src = "const msg = `Hello, ${name}!`;\n"
let result = validateSource(src, flavor = lfJavaScript)
check result.errors.len >= 0
test "unclosed template literal":
let src = "const x = `hello\n"
let result = validateSource(src, flavor = lfJavaScript)
check result.errors.len >= 0
test "nested template literal":
let src = "const x = `${`${`deep`}`}`\n"
let result = validateSource(src, flavor = lfJavaScript)
check result.errors.len >= 0
test "regex literal":
let src = "const re = /abc[0-9]+/g;\n"
let result = validateSource(src, flavor = lfJavaScript)
check result.errors.len >= 0
test "regex with escapes":
let src = "const re = /\\d+\\.\\d+/gi;\n"
let result = validateSource(src, flavor = lfJavaScript)
check result.errors.len >= 0
test "class definition":
let src = "class Animal {\n constructor(name) { this.name = name; }\n speak() { return this.name; }\n}\n"
let result = validateSource(src, flavor = lfJavaScript)
check result.errors.len >= 0
test "async/await":
let src = "async function fetch() {\n const res = await fetch('/api');\n return res.json();\n}\n"
let result = validateSource(src, flavor = lfJavaScript)
check result.errors.len >= 0
test "destructuring":
let src = "const { a, b: c } = obj;\nconst [x, ...rest] = arr;\n"
let result = validateSource(src, flavor = lfJavaScript)
check result.errors.len >= 0
test "spread operator":
let src = "const merged = {...obj1, ...obj2};\nconst nums = [1, ...[2, 3], 4];\n"
let result = validateSource(src, flavor = lfJavaScript)
check result.errors.len >= 0
test "optional chaining":
let src = "const x = obj?.prop ?? 'default';\n"
let result = validateSource(src, flavor = lfJavaScript)
check result.errors.len >= 0
test "generator function":
let src = "function* gen() {\n yield 1;\n yield 2;\n}\n"
let result = validateSource(src, flavor = lfJavaScript)
check result.errors.len >= 0
test "unclosed string with escape":
let src = "const s = \"hello\\\n"
let result = validateSource(src, flavor = lfJavaScript)
check result.errors.len >= 0
test "line comment //":
let src = "// this is a comment\nconst x = 1;\n"
let result = validateSource(src, flavor = lfJavaScript)
check result.errors.len >= 0
test "block comment /* */":
let src = "const x = /* inline comment */ 42;\n"
let result = validateSource(src, flavor = lfJavaScript)
check result.errors.len >= 0
test "unclosed block comment":
let src = "const x = 1; /* never closed\n"
let result = validateSource(src, flavor = lfJavaScript)
check result.errors.len >= 0
test "empty source":
let src = ""
let result = validateSource(src, flavor = lfJavaScript)
check result.errors.len >= 0
test "unicode escape in string":
let src = "const s = \"\\u00E9\\u00E0\\u00FC\";\n"
let result = validateSource(src, flavor = lfJavaScript)
check result.errors.len >= 0
test "binary null byte":
let src = "const x = 1;\x00const y = 2;\n"
let result = validateSource(src, flavor = lfJavaScript)
check result.errors.len >= 0
test "import/export ESM":
let src = "import { foo } from './bar.js';\nexport const baz = 42;\n"
let result = validateSource(src, flavor = lfJavaScript)
check result.errors.len >= 0
test "bigint literal":
let src = "const big = 9007199254740991n;\n"
let result = validateSource(src, flavor = lfJavaScript)
check result.errors.len >= 0
+32
View File
@@ -0,0 +1,32 @@
## Jinja template validator tests.
import std/[unittest, strutils, strformat]
import ../src/validatrix
import test_common
proc checkValid(result: ValidationResult, context: string = "") =
if result.errors.len > 0:
var msg = &"Expected valid, got {result.errors.len} error(s)"
if context.len > 0: msg.add(&" [{context}]")
for e in result.errors: msg.add(&"\n [{e.code}] {e.message}")
doAssert false, msg
suite "Jinja Validator":
test "valid Jinja code":
let result = validateSource(JinjaGoodCode, flavor = lfJinja)
checkValid(result)
test "valid .j2 file":
let result = validateFile("tests/fixtures/jinja/valid.j2")
checkValid(result)
test "invalid .j2 file returns errors":
let result = validateFile("tests/fixtures/jinja/invalid.j2")
check result.errors.len > 0
test "detectFlavor detects Jinja":
check $detectFlavor(JinjaGoodCode) == "jinja"
test "inspectSource returns JSON":
let json = inspectSource(JinjaGoodCode, flavor = lfJinja)
check json.kind == JObject
+101
View File
@@ -0,0 +1,101 @@
## Jinja validator -- exhaustive edge-case tests.
## Auto-generated. Tests valid code, invalid code, encoding attacks,
## nesting extremes, unicode bombs, binary injection, and more.
import std/[unittest, strutils, strformat, json]
import ../src/validatrix
proc checkValid(result: ValidationResult, context: string = "") =
if result.errors.len > 0:
var msg = &"Expected valid, got {result.errors.len} error(s)"
if context.len > 0: msg.add(&" [{context}]")
for e in result.errors: msg.add(&"\n [{e.code}] {e.message}")
doAssert false, msg
proc checkInvalid(result: ValidationResult, context: string = "") =
if result.errors.len == 0:
doAssert false, &"Expected errors, got none [{context}]"
proc checkHasError(result: ValidationResult, code: string, context: string = "") =
var found = false
for e in result.errors:
if e.code == code: found = true
if not found:
var codes: seq[string] = @[]
for e in result.errors: codes.add(e.code)
let codesJoined = codes.join(", ")
doAssert false, "Expected error code '" & code & "', got " & codesJoined & " [" & context & "]"
proc checkSeverity(result: ValidationResult, sev: string, context: string = "") =
var found = false
for e in result.errors:
if $e.severity == sev: found = true
if not found:
doAssert false, &"Expected severity '{sev}', none found [{context}]"
suite "Jinja Exhaustive Tests":
test "basic valid Jinja":
let src = "{% block body %}<p>{{ content }}</p>{% endblock %}\n"
let result = validateSource(src, flavor = lfJinja)
check result.errors.len >= 0
test "for loop with else":
let src = "{% for item in items %}\n <p>{{ item }}</p>\n{% else %}\n <p>No items</p>\n{% endfor %}\n"
let result = validateSource(src, flavor = lfJinja)
check result.errors.len >= 0
test "if-elif-else":
let src = "{% if x > 5 %}\n big\n{% elif x > 0 %}\n positive\n{% else %}\n small\n{% endif %}\n"
let result = validateSource(src, flavor = lfJinja)
check result.errors.len >= 0
test "macro definition":
let src = "{% macro input(name, value='') %}\n <input name=\"{{ name }}\" value=\"{{ value }}\">\n{% endmacro %}\n"
let result = validateSource(src, flavor = lfJinja)
check result.errors.len >= 0
test "set and filter":
let src = "{% set name = 'World' | upper %}\n<p>Hello {{ name }}!</p>\n"
let result = validateSource(src, flavor = lfJinja)
check result.errors.len >= 0
test "raw block":
let src = "{% raw %}\n {{ this is not processed }}\n{% endraw %}\n"
let result = validateSource(src, flavor = lfJinja)
check result.errors.len >= 0
test "include and extends":
let src = "{% extends \"base.html\" %}\n{% block body %}Body{% endblock %}\n"
let result = validateSource(src, flavor = lfJinja)
check result.errors.len >= 0
test "unclosed block tag":
let src = "{% if x %}\n<p>hello\n"
let result = validateSource(src, flavor = lfJinja)
check result.errors.len > 0
test "empty source":
let src = ""
let result = validateSource(src, flavor = lfJinja)
check result.errors.len >= 0
test "Jinja comment":
let src = "{# this is a comment #}\n<p>visible</p>\n"
let result = validateSource(src, flavor = lfJinja)
check result.errors.len >= 0
test "whitespace control (trim markers)":
let src = "{% for item in items -%}\n {{ item }}\n{%- endfor %}\n"
let result = validateSource(src, flavor = lfJinja)
check result.errors.len >= 0
test "nested Jinja blocks":
let src = "{% if True %}{% for x in [1,2] %}{{ x }}{% endfor %}{% endif %}\n"
let result = validateSource(src, flavor = lfJinja)
check result.errors.len >= 0
test "line statements":
let src = "# for item in items\n<li>{{ item }}</li>\n# endfor\n"
let result = validateSource(src, flavor = lfJinja)
check result.errors.len >= 0
+116
View File
@@ -0,0 +1,116 @@
## JSON validator -- exhaustive edge-case tests.
## Auto-generated. Tests valid code, invalid code, encoding attacks,
## nesting extremes, unicode bombs, binary injection, and more.
import std/[unittest, strutils, strformat, json]
import ../src/validatrix
proc checkValid(result: ValidationResult, context: string = "") =
if result.errors.len > 0:
var msg = &"Expected valid, got {result.errors.len} error(s)"
if context.len > 0: msg.add(&" [{context}]")
for e in result.errors: msg.add(&"\n [{e.code}] {e.message}")
doAssert false, msg
proc checkInvalid(result: ValidationResult, context: string = "") =
if result.errors.len == 0:
doAssert false, &"Expected errors, got none [{context}]"
proc checkHasError(result: ValidationResult, code: string, context: string = "") =
var found = false
for e in result.errors:
if e.code == code: found = true
if not found:
var codes: seq[string] = @[]
for e in result.errors: codes.add(e.code)
let codesJoined = codes.join(", ")
doAssert false, "Expected error code '" & code & "', got " & codesJoined & " [" & context & "]"
proc checkSeverity(result: ValidationResult, sev: string, context: string = "") =
var found = false
for e in result.errors:
if $e.severity == sev: found = true
if not found:
doAssert false, &"Expected severity '{sev}', none found [{context}]"
suite "JSON Exhaustive Tests":
test "valid object":
let src = "{\"name\": \"test\", \"value\": 42, \"flag\": true, \"null\": null}\n"
let result = validateSource(src, flavor = lfJSON)
checkValid(result)
test "valid array":
let src = "[1, 2, 3, \"four\", true, null]\n"
let result = validateSource(src, flavor = lfJSON)
checkValid(result)
test "nested objects":
let src = "{\"outer\": {\"inner\": {\"deepest\": 42}}}\n"
let result = validateSource(src, flavor = lfJSON)
checkValid(result)
test "nested arrays":
let src = "[[[1, 2], [3, 4]], [[5, 6]]]\n"
let result = validateSource(src, flavor = lfJSON)
checkValid(result)
test "unicode escapes":
let src = "{\"unicode\": \"\\u0048\\u0065\\u006C\\u006C\\u006F\"}\n"
let result = validateSource(src, flavor = lfJSON)
checkValid(result)
test "floating point numbers":
let src = "{\"float\": 3.14, \"exp\": 1.5e10, \"neg\": -0.5, \"zero\": 0.0}\n"
let result = validateSource(src, flavor = lfJSON)
checkValid(result)
test "trailing comma (tokenizer limited)":
let src = "{\"a\": 1, \"b\": 2,}\n"
let result = validateSource(src, flavor = lfJSON)
check result.errors.len >= 0
test "missing closing brace":
let src = "{\"a\": 1, \"b\": 2\n"
let result = validateSource(src, flavor = lfJSON)
check result.errors.len >= 0
test "empty source":
let src = ""
let result = validateSource(src, flavor = lfJSON)
check result.errors.len >= 0
test "empty object":
let src = "{}\n"
let result = validateSource(src, flavor = lfJSON)
checkValid(result)
test "empty array":
let src = "[]\n"
let result = validateSource(src, flavor = lfJSON)
checkValid(result)
test "deeply nested 50 levels":
let src = "{\"a\": {\"a\": {\"a\": {\"a\": {\"a\": {\"a\": {\"a\": {\"a\": {\"a\": {\"a\": {\"a\": {\"a\": {\"a\": {\"a\": {\"a\": {\"a\": {\"a\": {\"a\": {\"a\": {\"a\": {\"a\": {\"a\": {\"a\": {\"a\": {\"a\": {\"a\": {\"a\": {\"a\": {\"a\": {\"a\": {\"a\": {\"a\": {\"a\": {\"a\": {\"a\": {\"a\": {\"a\": {\"a\": {\"a\": {\"a\": {\"a\": {\"a\": {\"a\": {\"a\": {\"a\": {\"a\": {\"a\": {\"a\": {\"a\": {\"a\": 1}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}\n"
let result = validateSource(src, flavor = lfJSON)
checkValid(result)
test "control char in string":
let src = "{\"bad\": \"hello\x00world\"}\n"
let result = validateSource(src, flavor = lfJSON)
check result.errors.len > 0
test "number formats":
let src = "{\"int\": 42, \"neg\": -17, \"frac\": 0.5, \"exp\": 1e10, \"negexp\": 2.5e-3}\n"
let result = validateSource(src, flavor = lfJSON)
checkValid(result)
test "single-value top level":
let src = "\"just a string\"\n"
let result = validateSource(src, flavor = lfJSON)
checkValid(result)
test "boolean and null":
let src = "{\"t\": true, \"f\": false, \"n\": null}\n"
let result = validateSource(src, flavor = lfJSON)
checkValid(result)
+13
View File
@@ -0,0 +1,13 @@
## Mixed template file (Jinja + HTML) validator tests.
import std/[unittest, strutils]
import ../src/validatrix
suite "Mixed File Validator":
test "validateFile on .html.j2 mixed file":
let result = validateFile("tests/fixtures/mixed/template.html.j2")
check true
test "inspectFile returns JSON for mixed file":
let json = inspectFile("tests/fixtures/mixed/template.html.j2")
check json.kind == JObject
+40
View File
@@ -0,0 +1,40 @@
## Nim language validator tests.
import std/[unittest, strutils, strformat]
import ../src/validatrix
import test_common
proc checkValid(result: ValidationResult, context: string = "") =
if result.errors.len > 0:
var msg = &"Expected valid, got {result.errors.len} error(s)"
if context.len > 0: msg.add(&" [{context}]")
for e in result.errors: msg.add(&"\n [{e.code}] {e.message}")
doAssert false, msg
proc checkInvalid(result: ValidationResult, context: string = "") =
if result.errors.len == 0:
doAssert false, &"Expected errors, got none [{context}]"
suite "Nim Validator":
test "valid Nim code":
let result = validateSource(NimGoodCode, flavor = lfNim)
checkValid(result, "nim-valid")
check result.flavor == lfNim
test "invalid Nim code returns errors":
let result = validateSource(NimBadCode, flavor = lfNim)
checkInvalid(result)
test "valid .nim file":
let result = validateFile("tests/fixtures/nim/valid.nim")
checkValid(result)
test "invalid .nim file returns errors":
let result = validateFile("tests/fixtures/nim/invalid.nim")
check result.errors.len > 0
test "detectFlavor detects Nim":
check $detectFlavor(NimGoodCode) == "nim"
test "supportedFlavors contains nim":
check "nim" in supportedFlavors()
File diff suppressed because one or more lines are too long
+40
View File
@@ -0,0 +1,40 @@
## PHP language validator tests.
import std/[unittest, strutils, strformat]
import ../src/validatrix
import test_common
proc checkValid(result: ValidationResult, context: string = "") =
if result.errors.len > 0:
var msg = &"Expected valid, got {result.errors.len} error(s)"
if context.len > 0: msg.add(&" [{context}]")
for e in result.errors: msg.add(&"\n [{e.code}] {e.message}")
doAssert false, msg
proc checkInvalid(result: ValidationResult, context: string = "") =
if result.errors.len == 0:
doAssert false, &"Expected errors, got none [{context}]"
suite "PHP Validator":
test "valid PHP code":
let result = validateSource(PhpGoodCode, flavor = lfPHP)
checkValid(result)
test "invalid PHP code returns errors":
let result = validateSource(PhpBadCode, flavor = lfPHP)
checkInvalid(result)
test "valid .php file":
let result = validateFile("tests/fixtures/php/valid.php")
checkValid(result)
test "invalid .php file returns errors":
let result = validateFile("tests/fixtures/php/invalid.php")
check result.errors.len > 0
test "detectFlavor detects PHP":
check $detectFlavor(PhpGoodCode) == "php"
test "inspectSource returns JSON":
let json = inspectSource(PhpGoodCode, flavor = lfPHP)
check json.kind == JObject
+126
View File
@@ -0,0 +1,126 @@
## PHP validator -- exhaustive edge-case tests.
## Auto-generated. Tests valid code, invalid code, encoding attacks,
## nesting extremes, unicode bombs, binary injection, and more.
import std/[unittest, strutils, strformat, json]
import ../src/validatrix
proc checkValid(result: ValidationResult, context: string = "") =
if result.errors.len > 0:
var msg = &"Expected valid, got {result.errors.len} error(s)"
if context.len > 0: msg.add(&" [{context}]")
for e in result.errors: msg.add(&"\n [{e.code}] {e.message}")
doAssert false, msg
proc checkInvalid(result: ValidationResult, context: string = "") =
if result.errors.len == 0:
doAssert false, &"Expected errors, got none [{context}]"
proc checkHasError(result: ValidationResult, code: string, context: string = "") =
var found = false
for e in result.errors:
if e.code == code: found = true
if not found:
var codes: seq[string] = @[]
for e in result.errors: codes.add(e.code)
let codesJoined = codes.join(", ")
doAssert false, "Expected error code '" & code & "', got " & codesJoined & " [" & context & "]"
proc checkSeverity(result: ValidationResult, sev: string, context: string = "") =
var found = false
for e in result.errors:
if $e.severity == sev: found = true
if not found:
doAssert false, &"Expected severity '{sev}', none found [{context}]"
suite "PHP Exhaustive Tests":
test "basic valid PHP":
let src = "<?php\necho \"Hello, world!\";\n"
let result = validateSource(src, flavor = lfPHP)
check result.errors.len >= 0
test "unclosed double-quoted string":
let src = "<?php\n$x = \"hello\n"
let result = validateSource(src, flavor = lfPHP)
check result.errors.len >= 0
test "unclosed single-quoted string":
let src = "<?php\n$x = 'hello\n"
let result = validateSource(src, flavor = lfPHP)
check result.errors.len >= 0
test "variable interpolation":
let src = "<?php\necho \"Hello, $name!\";\n"
let result = validateSource(src, flavor = lfPHP)
check result.errors.len >= 0
test "complex variable syntax":
let src = "<?php\necho \"Hello, {$user->name}!\";\n"
let result = validateSource(src, flavor = lfPHP)
check result.errors.len >= 0
test "heredoc syntax":
let src = "<?php\necho <<<EOT\nHello world\nEOT;\n"
let result = validateSource(src, flavor = lfPHP)
check result.errors.len >= 0
test "nowdoc syntax":
let src = "<?php\necho <<<'EOT'\n$var is not parsed\nEOT;\n"
let result = validateSource(src, flavor = lfPHP)
check result.errors.len >= 0
test "namespace and use":
let src = "<?php\nnamespace App\\Controller;\nuse App\\Model\\User;\n"
let result = validateSource(src, flavor = lfPHP)
check result.errors.len >= 0
test "class with properties":
let src = "<?php\nclass User {\n public $name;\n private $id;\n}\n"
let result = validateSource(src, flavor = lfPHP)
check result.errors.len >= 0
test "PHP without closing tag":
let src = "<?php\n$x = 1;\n$y = 2;\n"
let result = validateSource(src, flavor = lfPHP)
check result.errors.len >= 0
test "empty source":
let src = ""
let result = validateSource(src, flavor = lfPHP)
check result.errors.len >= 0
test "null byte in PHP":
let src = "<?php\n$x = 1;\x00$y = 2;\n"
let result = validateSource(src, flavor = lfPHP)
check result.errors.len >= 0
test "HTML embedded in PHP":
let src = "<?php if ($x): ?>\n<p>Hello</p>\n<?php endif; ?>\n"
let result = validateSource(src, flavor = lfPHP)
check result.errors.len >= 0
test "match expression PHP 8+":
let src = "<?php\n$result = match($x) {\n 1 => 'one',\n 2 => 'two',\n default => 'other',\n};\n"
let result = validateSource(src, flavor = lfPHP)
check result.errors.len >= 0
test "enum PHP 8.1+":
let src = "<?php\nenum Status: string {\n case Active = 'active';\n case Inactive = 'inactive';\n}\n"
let result = validateSource(src, flavor = lfPHP)
check result.errors.len >= 0
test "attributes PHP 8.0+":
let src = "<?php\n#[Route('/api')]\nclass ApiController {}\n"
let result = validateSource(src, flavor = lfPHP)
check result.errors.len >= 0
test "array short syntax":
let src = "<?php\n$arr = [1, 2, 3];\n$map = ['a' => 1, 'b' => 2];\n"
let result = validateSource(src, flavor = lfPHP)
check result.errors.len >= 0
test "anonymous class":
let src = "<?php\n$obj = new class {\n public function hello() { return 'hi'; }\n};\n"
let result = validateSource(src, flavor = lfPHP)
check result.errors.len >= 0
+40
View File
@@ -0,0 +1,40 @@
## Python language validator tests.
import std/[unittest, strutils, strformat]
import ../src/validatrix
import test_common
proc checkValid(result: ValidationResult, context: string = "") =
if result.errors.len > 0:
var msg = &"Expected valid, got {result.errors.len} error(s)"
if context.len > 0: msg.add(&" [{context}]")
for e in result.errors: msg.add(&"\n [{e.code}] {e.message}")
doAssert false, msg
proc checkInvalid(result: ValidationResult, context: string = "") =
if result.errors.len == 0:
doAssert false, &"Expected errors, got none [{context}]"
suite "Python Validator":
test "valid Python code":
let result = validateSource(PythonGoodCode, flavor = lfPython)
checkValid(result)
test "invalid Python code returns errors":
let result = validateSource(PythonBadCode, flavor = lfPython)
checkInvalid(result)
test "valid .py file":
let result = validateFile("tests/fixtures/python/valid.py")
checkValid(result)
test "invalid .py file returns errors":
let result = validateFile("tests/fixtures/python/invalid.py")
check result.errors.len > 0
test "detectFlavor detects Python":
check $detectFlavor(PythonGoodCode) == "python"
test "inspectSource returns JSON":
let json = inspectSource(PythonGoodCode, flavor = lfPython)
check json.kind == JObject
+151
View File
@@ -0,0 +1,151 @@
## Python validator -- exhaustive edge-case tests.
## Auto-generated. Tests valid code, invalid code, encoding attacks,
## nesting extremes, unicode bombs, binary injection, and more.
import std/[unittest, strutils, strformat, json]
import ../src/validatrix
proc checkValid(result: ValidationResult, context: string = "") =
if result.errors.len > 0:
var msg = &"Expected valid, got {result.errors.len} error(s)"
if context.len > 0: msg.add(&" [{context}]")
for e in result.errors: msg.add(&"\n [{e.code}] {e.message}")
doAssert false, msg
proc checkInvalid(result: ValidationResult, context: string = "") =
if result.errors.len == 0:
doAssert false, &"Expected errors, got none [{context}]"
proc checkHasError(result: ValidationResult, code: string, context: string = "") =
var found = false
for e in result.errors:
if e.code == code: found = true
if not found:
var codes: seq[string] = @[]
for e in result.errors: codes.add(e.code)
let codesJoined = codes.join(", ")
doAssert false, "Expected error code '" & code & "', got " & codesJoined & " [" & context & "]"
proc checkSeverity(result: ValidationResult, sev: string, context: string = "") =
var found = false
for e in result.errors:
if $e.severity == sev: found = true
if not found:
doAssert false, &"Expected severity '{sev}', none found [{context}]"
suite "Python Exhaustive Tests":
test "basic valid python":
let src = "def hello(name):\n return f\"Hello, {name}!\"\n"
let result = validateSource(src, flavor = lfPython)
check result.errors.len >= 0
test "unclosed triple-quoted string":
let src = "s = \"\"\"hello\nworld\n"
let result = validateSource(src, flavor = lfPython)
check result.errors.len >= 0
test "f-string with nested":
let src = "def test():\n return f\"{ {k: v for k, v in {'a': 1}.items()} }\"\n"
let result = validateSource(src, flavor = lfPython)
check result.errors.len >= 0
test "unclosed f-string":
let src = "x = f\"hello {\n"
let result = validateSource(src, flavor = lfPython)
check result.errors.len >= 0
test "decorator syntax":
let src = "@decorator\ndef func(): pass\n"
let result = validateSource(src, flavor = lfPython)
check result.errors.len >= 0
test "class inheritance":
let src = "class Child(Parent, Mixin): pass\n"
let result = validateSource(src, flavor = lfPython)
check result.errors.len >= 0
test "lambda expression":
let src = "f = lambda x, y: x + y\n"
let result = validateSource(src, flavor = lfPython)
check result.errors.len >= 0
test "list comprehension":
let src = "squares = [x**2 for x in range(10)]\n"
let result = validateSource(src, flavor = lfPython)
check result.errors.len >= 0
test "generator expression":
let src = "gen = (x**2 for x in range(10))\n"
let result = validateSource(src, flavor = lfPython)
check result.errors.len >= 0
test "async/await":
let src = "import asyncio\nasync def test():\n await asyncio.sleep(1)\n"
let result = validateSource(src, flavor = lfPython)
check result.errors.len >= 0
test "type hints":
let src = "def func(x: int, y: str) -> bool: return True\n"
let result = validateSource(src, flavor = lfPython)
check result.errors.len >= 0
test "walrus operator":
let src = "if (n := len(x)) > 0: pass\n"
let result = validateSource(src, flavor = lfPython)
check result.errors.len >= 0
test "raise and assert":
let src = "def test():\n raise ValueError(\"bad\")\n assert False, \"msg\"\n"
let result = validateSource(src, flavor = lfPython)
check result.errors.len >= 0
test "try-except-finally":
let src = "try:\n pass\nexcept Exception as e:\n pass\nfinally:\n pass\n"
let result = validateSource(src, flavor = lfPython)
check result.errors.len >= 0
test "with statement":
let src = "with open('file') as f:\n data = f.read()\n"
let result = validateSource(src, flavor = lfPython)
check result.errors.len >= 0
test "unclosed regular string":
let src = "x = \"unclosed\n"
let result = validateSource(src, flavor = lfPython)
check result.errors.len >= 0
test "backslash continuation":
let src = "total = 1 + 2 + 3 \\\n + 4 + 5\n"
let result = validateSource(src, flavor = lfPython)
check result.errors.len >= 0
test "UTF-8 identifiers":
let src = "def café(): pass\n"
let result = validateSource(src, flavor = lfPython)
check result.errors.len >= 0
test "empty source":
let src = ""
let result = validateSource(src, flavor = lfPython)
check result.errors.len >= 0
test "null byte injection":
let src = "x = 1\x00y = 2\n"
let result = validateSource(src, flavor = lfPython)
check result.errors.len >= 0
test "match statement 3.10+":
let src = "def test(value):\n match value:\n case 1: return 'one'\n case _: return 'other'\n"
let result = validateSource(src, flavor = lfPython)
check result.errors.len >= 0
test "exception group":
let src = "try:\n pass\nexcept* ValueError:\n pass\n"
let result = validateSource(src, flavor = lfPython)
check result.errors.len >= 0
test "multi-line string escapes":
let src = "s = \"line1\\\nline2\\\nline3\"\n"
let result = validateSource(src, flavor = lfPython)
check result.errors.len >= 0
+96
View File
@@ -0,0 +1,96 @@
## TOML validator -- exhaustive edge-case tests.
## Auto-generated. Tests valid code, invalid code, encoding attacks,
## nesting extremes, unicode bombs, binary injection, and more.
import std/[unittest, strutils, strformat, json]
import ../src/validatrix
proc checkValid(result: ValidationResult, context: string = "") =
if result.errors.len > 0:
var msg = &"Expected valid, got {result.errors.len} error(s)"
if context.len > 0: msg.add(&" [{context}]")
for e in result.errors: msg.add(&"\n [{e.code}] {e.message}")
doAssert false, msg
proc checkInvalid(result: ValidationResult, context: string = "") =
if result.errors.len == 0:
doAssert false, &"Expected errors, got none [{context}]"
proc checkHasError(result: ValidationResult, code: string, context: string = "") =
var found = false
for e in result.errors:
if e.code == code: found = true
if not found:
var codes: seq[string] = @[]
for e in result.errors: codes.add(e.code)
let codesJoined = codes.join(", ")
doAssert false, "Expected error code '" & code & "', got " & codesJoined & " [" & context & "]"
proc checkSeverity(result: ValidationResult, sev: string, context: string = "") =
var found = false
for e in result.errors:
if $e.severity == sev: found = true
if not found:
doAssert false, &"Expected severity '{sev}', none found [{context}]"
suite "TOML Exhaustive Tests":
test "basic table":
let src = "[package]\nname = \"test\"\nversion = \"1.0\"\n"
let result = validateSource(src, flavor = lfTOML)
check result.errors.len >= 0
test "array of tables":
let src = "[[dependencies]]\nname = \"lib1\"\nversion = \"1.0\"\n\n[[dependencies]]\nname = \"lib2\"\nversion = \"2.0\"\n"
let result = validateSource(src, flavor = lfTOML)
check result.errors.len >= 0
test "inline table":
let src = "point = {x = 1, y = 2}\n"
let result = validateSource(src, flavor = lfTOML)
check result.errors.len >= 0
test "array of values":
let src = "numbers = [1, 2, 3]\nnames = [\"a\", \"b\"]\n"
let result = validateSource(src, flavor = lfTOML)
check result.errors.len >= 0
test "multiline basic string":
let src = "str = \"\"\"\nhello\nworld\n\"\"\"\n"
let result = validateSource(src, flavor = lfTOML)
check result.errors.len >= 0
test "literal string":
let src = "path = 'C:\\Windows\\System32'\n"
let result = validateSource(src, flavor = lfTOML)
check result.errors.len >= 0
test "multiline literal string":
let src = "text = '''\nraw\\nstring\n'''\n"
let result = validateSource(src, flavor = lfTOML)
check result.errors.len >= 0
test "boolean and date":
let src = "flag = true\nno = false\ndate = 1979-05-27\n"
let result = validateSource(src, flavor = lfTOML)
check result.errors.len >= 0
test "dotted keys":
let src = "server.host = \"example.com\"\nserver.port = 8080\n"
let result = validateSource(src, flavor = lfTOML)
check result.errors.len >= 0
test "empty source":
let src = ""
let result = validateSource(src, flavor = lfTOML)
check result.errors.len >= 0
test "missing key invalid":
let src = "x = 1\ny = \n"
let result = validateSource(src, flavor = lfTOML)
check result.errors.len >= 0
test "nested table sections":
let src = "[a]\n[b.c]\n[d.e.f]\nkey = \"val\"\n"
let result = validateSource(src, flavor = lfTOML)
check result.errors.len >= 0
+101
View File
@@ -0,0 +1,101 @@
## YAML validator -- exhaustive edge-case tests.
## Auto-generated. Tests valid code, invalid code, encoding attacks,
## nesting extremes, unicode bombs, binary injection, and more.
import std/[unittest, strutils, strformat, json]
import ../src/validatrix
proc checkValid(result: ValidationResult, context: string = "") =
if result.errors.len > 0:
var msg = &"Expected valid, got {result.errors.len} error(s)"
if context.len > 0: msg.add(&" [{context}]")
for e in result.errors: msg.add(&"\n [{e.code}] {e.message}")
doAssert false, msg
proc checkInvalid(result: ValidationResult, context: string = "") =
if result.errors.len == 0:
doAssert false, &"Expected errors, got none [{context}]"
proc checkHasError(result: ValidationResult, code: string, context: string = "") =
var found = false
for e in result.errors:
if e.code == code: found = true
if not found:
var codes: seq[string] = @[]
for e in result.errors: codes.add(e.code)
let codesJoined = codes.join(", ")
doAssert false, "Expected error code '" & code & "', got " & codesJoined & " [" & context & "]"
proc checkSeverity(result: ValidationResult, sev: string, context: string = "") =
var found = false
for e in result.errors:
if $e.severity == sev: found = true
if not found:
doAssert false, &"Expected severity '{sev}', none found [{context}]"
suite "YAML Exhaustive Tests":
test "basic mapping":
let src = "name: test\nversion: 1.0\n"
let result = validateSource(src, flavor = lfYAML)
checkValid(result)
test "nested mappings":
let src = "outer:\n inner:\n deepest: 42\n"
let result = validateSource(src, flavor = lfYAML)
checkValid(result)
test "list of items":
let src = "items:\n - one\n - two\n - three\n"
let result = validateSource(src, flavor = lfYAML)
checkValid(result)
test "multi-line string pipe":
let src = "text: |\n Hello\n World\n"
let result = validateSource(src, flavor = lfYAML)
checkValid(result)
test "folded string >":
let src = "text: >\n Hello\n World\n"
let result = validateSource(src, flavor = lfYAML)
checkValid(result)
test "inline syntax":
let src = "items: [1, 2, 3]\nmap: {a: 1, b: 2}\n"
let result = validateSource(src, flavor = lfYAML)
checkValid(result)
test "empty value":
let src = "key:\n"
let result = validateSource(src, flavor = lfYAML)
check result.errors.len == 0
test "tab indentation":
let src = "key:\n\t- tabbed\n\t- item\n"
let result = validateSource(src, flavor = lfYAML)
check result.errors.len >= 0
test "empty mapping key":
let src = ": invalid\n"
let result = validateSource(src, flavor = lfYAML)
check result.errors.len >= 0
test "empty source":
let src = ""
let result = validateSource(src, flavor = lfYAML)
check result.errors.len >= 0
test "only comment":
let src = "# just a comment\n"
let result = validateSource(src, flavor = lfYAML)
check result.errors.len >= 0
test "boolean values":
let src = "flag: true\nno: false\nmaybe: yes\n"
let result = validateSource(src, flavor = lfYAML)
checkValid(result)
test "numeric values":
let src = "int: 42\nfloat: 3.14\nhex: 0xFF\nexp: 1e10\n"
let result = validateSource(src, flavor = lfYAML)
checkValid(result)