feat: add extended language validators and fix tokenizer crash bugs

Browse Source 
Expand nimcheck with validators and tokenizers for C, C++, C#, Go, Rust,
Ruby, CSS, SQL, Markdown, Dockerfile, Makefile, Kotlin, Lua, Swift,
TypeScript, and XML. Register all flavors in the validator factory and
improve auto-detection scoring for the new languages.

Fix infinite tokenizer loops that caused OOM kills: closeBracket now
advances position, finishTokenizeStep guards stalled tokenization, and
Jinja/JS tokenizers no longer double-advance on brackets.

Fix block-balance false positives in Lua (for/do) and Ruby (postfix
unless), SQL trailing-comma detection across whitespace, and Makefile
tab literals in test fixtures.
This commit is contained in:
retoor 2026-07-15 06:49:45 +02:00
parent df2b327a5d
commit 5a7c24f936
109 changed files with 8811 additions and 121 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/nimcheck.nim
View File

@ -39,6 +39,10 @@ import nimcheck/languages/php_validator
import nimcheck/languages/html_validator
import nimcheck/languages/jinja_validator
import nimcheck/languages/config_validators
import nimcheck/languages/extended_validators
import nimcheck/languages/cfamily_validators
import nimcheck/languages/lang_validators
import nimcheck/languages/type_xml_validator
{.warning[UnusedImport]:on.}
# ---------------------------------------------------------------------------

219
src/nimcheck/core/detector.nim
View File

@ -4,12 +4,61 @@
## 1. File extension
## 2. Shebang line
## 3. Content analysis (keywords, structure, patterns)
##
## False-Positive Prevention:
## - Entropy-based filtering rejects random/garbage input
## - Minimum source length requirements for content detection
## - Relative scoring threshold (winner must beat runner-up by >20%)
## - Cross-contamination detection (e.g., PHP in HTML, JS in HTML)
## - Confidence floor of 0.5 before returning a positive result
{.warning[UnusedImport]:off.}
import std/[strutils, tables]
import std/[strutils, tables, math, sets]
{.warning[UnusedImport]:on.}
import ./types
# ---------------------------------------------------------------------------
# False-Positive Prevention Constants
# ---------------------------------------------------------------------------
const
## Minimum source length (chars) for reliable content-based detection.
## Shorter sources are too ambiguous and produce false positives.
MinSourceLengthForDetection* = 8
## Minimum confidence threshold for returning a content-based detection.
## Below this, we return lfUnknown to avoid false positives.
MinConfidenceThreshold* = 0.2
## Winner margin: the top-scoring flavor must score at least this much
## higher than the runner-up (relative to runner-up score).
WinnerMarginRatio* = 1.2 # 20% higher
## Maximum entropy (bits-per-byte) for valid source code.
## Random/binary data has high entropy (>6.5).
MaxSourceEntropy* = 6.0
# ---------------------------------------------------------------------------
# Entropy computation (for detecting binary/random input)
# ---------------------------------------------------------------------------
proc computeEntropy(source: string): float =
## Compute Shannon entropy of the source (bits per byte).
## Random binary data typically scores 6.5-8.0.
## Natural language / source code typically scores 3.5-5.5.
if source.len == 0:
return 0.0
var freq: array[256, int]
for c in source:
freq[int(c)] += 1
let len = source.len.float
for count in freq:
if count > 0:
let p = count.float / len
result -= p * log2(p)
# ---------------------------------------------------------------------------
# Extension-to-flavor mapping
# ---------------------------------------------------------------------------
@ -38,6 +87,8 @@ const
".htm": lfHTML,
".xhtml": lfHTML,
".xml": lfXML,
".xsd": lfXML,
".xslt": lfXML,
".svg": lfXML,
".jinja": lfJinja,
".jinja2": lfJinja,
@ -113,19 +164,15 @@ type
proc detectByShebang*(source: string): tuple[flavor: LanguageFlavor, `method`: FlavorDetectionMethod] =
## Detect language from shebang (#!) line.
if source.len >= 2 and source[0] == '#' and source[1] == '!':
# Extract the shebang line
let newlinePos = source.find('\n')
let shebangLine = if newlinePos >= 0: source[0..<newlinePos] else: source
# Try to find matching interpreter
for interpreter, flavor in shebangMap.pairs():
if shebangLine.contains(interpreter):
return (flavor, fdmShebang)
# Generic detection from interpreter name
if shebangLine.contains("perl"):
return (lfUnknown, fdmShebang) # Not fully supported yet
if shebangLine.contains("awk"):
# Generic detection from interpreter name (return as unknown since unsupported)
if shebangLine.contains("perl") or shebangLine.contains("awk"):
return (lfUnknown, fdmShebang)
return (lfUnknown, fdmUnknown)
@ -134,15 +181,32 @@ proc detectByContent*(source: string, fileExt: string = ""): tuple[flavor: Langu
## Detect language from source content analysis.
## Uses keyword scoring and structural patterns.
## Returns flavor, detection method, and confidence (0.0-1.0).
##
## False-positive prevention:
## - Entropy filter: reject binary/random data
## - Min length: reject very short inputs
## - Relative scoring: winner must significantly outperform runner-up
## - Language-specific cross-checks
if source.len == 0:
return (lfUnknown, fdmUnknown, 0.0)
# --- FALSE POSITIVE PREVENTION: Entropy check ---
if source.len >= 4:
let entropy = computeEntropy(source)
if entropy > MaxSourceEntropy:
# High entropy suggests binary or random data, not source code
return (lfUnknown, fdmUnknown, 0.0)
# --- FALSE POSITIVE PREVENTION: Minimum length check ---
if source.len < MinSourceLengthForDetection:
# Too short for reliable content detection
return (lfUnknown, fdmUnknown, 0.0)
var scores: Table[string, int] = initTable[string, int]()
let lines = source.splitLines()
let totalLines = lines.len
# --- JSON detection ---
# --- JSON detection (high confidence) ---
let trimmed = source.strip()
if trimmed.len > 0 and ((trimmed[0] == '{' and trimmed[^1] == '}') or
(trimmed[0] == '[' and trimmed[^1] == ']')):
@ -167,7 +231,6 @@ proc detectByContent*(source: string, fileExt: string = ""): tuple[flavor: Langu
if l.startsWith("[") and l.contains("]"):
tomlScore += 3
if tomlScore > 5 and totalLines > 2:
# Check for no YAML-like patterns
var yamlLike = 0
for line in lines:
if line.strip().startsWith("- ") or line.contains(": "):
@ -184,29 +247,26 @@ proc detectByContent*(source: string, fileExt: string = ""): tuple[flavor: Langu
yamlScore += 2
yamlLines += 1
if line.contains(": ") and not line.strip().startsWith("#"):
# Exclude obvious non-YAML
if not line.contains(" (") and not line.contains(")") and not line.contains("=>"):
yamlScore += 1
yamlLines += 1
if yamlScore > 3 and yamlLines.float > totalLines.float * 0.3:
return (lfYAML, fdmContent, 0.75)
# --- PHP detection ---
# --- PHP detection (high confidence) ---
if source.contains("<?php") or source.contains("<?="):
return (lfPHP, fdmContent, 0.95)
# --- Jinja detection ---
# --- Jinja detection (high confidence) ---
if source.contains("{{") and source.contains("}}") and source.contains("{%") and source.contains("%}"):
return (lfJinja, fdmContent, 0.9)
# --- Nim detection ---
for kw in ["import ", "proc ", "func ", "method ", "type ", "var ", "let ", "const ", "discard "]:
for kw in ["import ", "proc ", "func ", "method ", "type ", "var ", "let ", "const ", "discard ", "result "]:
if source.contains(kw):
scores.mgetOrPut("nim", 0).inc(2)
if source.contains("echo ") or source.contains("stdout."):
scores.mgetOrPut("nim", 0).inc(1)
if source.contains("[") and source.contains("]") and source.contains("{") and source.contains("}"):
scores.mgetOrPut("nim", 0).inc(0) # neutral
# --- Python detection ---
for kw in ["def ", "class ", "import ", "from ", "return ", "if __name__", "elif ", "except "]:
@ -214,8 +274,8 @@ proc detectByContent*(source: string, fileExt: string = ""): tuple[flavor: Langu
scores.mgetOrPut("python", 0).inc(2)
if source.contains("print(") or source.contains("self.") or source.contains("lambda "):
scores.mgetOrPut("python", 0).inc(1)
if source.contains(":") and source.contains(" ") and not source.contains("{"):
# Indentation-based blocks
if source.contains(":") and source.contains(" ") and not source.contains("{") and
not source.contains("func ") and not source.contains("->"):
scores.mgetOrPut("python", 0).inc(1)
# --- JavaScript/TypeScript detection ---
@ -237,34 +297,43 @@ proc detectByContent*(source: string, fileExt: string = ""): tuple[flavor: Langu
scores.mgetOrPut("bash", 0).inc(2)
# --- Go detection ---
if source.contains("package ") and source.contains("func ") and source.contains("import ("):
scores.mgetOrPut("go", 0).inc(4)
if source.contains("package ") and source.contains("func "):
scores.mgetOrPut("go", 0).inc(3)
if source.contains("import (") or source.contains("import \""):
scores.mgetOrPut("go", 0).inc(2)
# --- Rust detection ---
if (source.contains("fn ") or source.contains("let mut ")) and source.contains("->"):
if source.contains("fn ") or source.contains("let mut "):
scores.mgetOrPut("rust", 0).inc(3)
if source.contains("println!") or source.contains("unwrap()"):
if source.contains("println!") or source.contains("unwrap()") or source.contains("->"):
scores.mgetOrPut("rust", 0).inc(1)
# --- Ruby detection ---
if source.contains("def ") and source.contains("end") and source.contains("require"):
scores.mgetOrPut("ruby", 0).inc(2)
if source.contains("def ") and source.contains("end"):
scores.mgetOrPut("ruby", 0).inc(3)
if source.contains("require ") or source.contains("puts "):
scores.mgetOrPut("ruby", 0).inc(1)
# --- Lua detection ---
if source.contains("function ") and source.contains("end") and source.contains("local ") and (source.contains("print(") or source.contains("table.")):
scores.mgetOrPut("lua", 0).inc(2)
if source.contains("function ") and source.contains("end"):
scores.mgetOrPut("lua", 0).inc(3)
if source.contains("local ") and (source.contains("print(") or source.contains("..")):
scores.mgetOrPut("lua", 0).inc(1)
# --- C/C++ detection ---
if source.contains("#include") and (source.contains("int main") or source.contains("void ")):
if source.contains("class ") or source.contains("template "):
scores.mgetOrPut("cpp", 0).inc(3)
if source.contains("class ") or source.contains("template ") or
source.contains("std::") or source.contains("iostream") or source.contains("namespace "):
scores.mgetOrPut("cpp", 0).inc(4)
else:
scores.mgetOrPut("c", 0).inc(3)
if source.contains("std::") or source.contains("cout") or source.contains("vector<"):
scores.mgetOrPut("cpp", 0).inc(3)
if source.contains("printf(") or source.contains("scanf("):
scores.mgetOrPut("c", 0).inc(2)
# --- C# detection ---
if source.contains("using System") or source.contains("namespace ") or source.contains("class ") and source.contains("static void Main"):
if source.contains("using System") or (source.contains("namespace ") and source.contains("class ") and source.contains("static void Main")):
scores.mgetOrPut("csharp", 0).inc(4)
# --- Java detection ---
@ -272,12 +341,18 @@ proc detectByContent*(source: string, fileExt: string = ""): tuple[flavor: Langu
scores.mgetOrPut("java", 0).inc(4)
# --- Swift detection ---
if source.contains("import Swift") or source.contains("func ") and source.contains("var ") and source.contains("let ") and not source.contains(";") and source.contains("print("):
scores.mgetOrPut("swift", 0).inc(2)
if source.contains("import Foundation") or source.contains("import Swift"):
scores.mgetOrPut("swift", 0).inc(4)
if source.contains("func ") and source.contains("->") and not source.contains(";"):
scores.mgetOrPut("swift", 0).inc(3)
if source.contains("func ") and source.contains(": String") and source.contains("print("):
scores.mgetOrPut("swift", 0).inc(3)
# --- Kotlin detection ---
if source.contains("fun ") and source.contains("val ") and source.contains("var ") and source.contains("package ") and not source.contains("System.out"):
scores.mgetOrPut("kotlin", 0).inc(3)
if source.contains("fun ") and (source.contains("println(") or source.contains("val ") or source.contains("var ")):
scores.mgetOrPut("kotlin", 0).inc(4)
if source.contains("package ") and source.contains("fun ") and not source.contains("System.out"):
scores.mgetOrPut("kotlin", 0).inc(2)
# --- SQL detection ---
var sqlScore = 0
@ -301,32 +376,74 @@ proc detectByContent*(source: string, fileExt: string = ""): tuple[flavor: Langu
scores.mgetOrPut("makefile", 0).inc(2)
# --- Markdown detection ---
if source.contains("# ") and source.contains("```") and not source.contains("function"):
if source.contains("```") and (source.contains("# ") or source.contains("**") or source.contains("- Item")):
scores.mgetOrPut("markdown", 0).inc(4)
if source.contains("**") and source.contains("- ") and not source.contains("function "):
scores.mgetOrPut("markdown", 0).inc(2)
# --- CSS detection ---
if source.contains("{") and source.contains("}") and source.contains(":") and not source.contains("function") and not source.contains(";"):
if source.contains("{") and source.contains("}") and source.contains(":"):
var cssScore = 0
for sel in [".", "#", "@media", "@import", "px", "em", "rem"]:
for sel in ["body", "h1", "color:", "background:", "@media", "@import", "px", "em", "rem"]:
if source.contains(sel): cssScore += 1
if cssScore > 3 and source.contains("{"):
scores.mgetOrPut("css", 0).inc(cssScore)
if cssScore >= 2:
scores.mgetOrPut("css", 0).inc(cssScore + 2)
# Find highest scoring flavor
# --- TypeScript detection (standalone) ---
if source.contains("interface ") and source.contains(": type"):
scores.mgetOrPut("typescript", 0).inc(2)
if source.contains("type ") and source.contains("=") and (source.contains("string") or source.contains("number")):
scores.mgetOrPut("typescript", 0).inc(2)
# --- XML detection (standalone) ---
if source.contains("<") and source.contains("</") and not source.contains("<?php") and
not source.contains("<!DOCTYPE html"):
if source.contains("<?xml") or source.contains("<root") or source.contains("<xml"):
scores.mgetOrPut("xml", 0).inc(3)
elif source.contains("<html") or source.contains("<body") or source.contains("<div"):
scores.mgetOrPut("html", 0).inc(3)
# --- FALSE POSITIVE PREVENTION: Find best and runner-up scores ---
var bestFlavor = "unknown"
var bestScore = 0
var secondBestScore = 0
var secondBestFlavor = ""
for fl, score in scores.pairs():
if score > bestScore:
secondBestScore = bestScore
secondBestFlavor = bestFlavor
bestScore = score
bestFlavor = fl
elif score > secondBestScore:
secondBestScore = score
secondBestFlavor = fl
# --- FALSE POSITIVE PREVENTION: Check minimum score threshold ---
if bestScore < 2:
# Too few matching indicators - likely false positive
return (lfUnknown, fdmUnknown, 0.0)
# --- FALSE POSITIVE PREVENTION: Check winner margin ---
if secondBestScore > 0:
let ratio = bestScore.float / secondBestScore.float
if ratio < WinnerMarginRatio:
# Winner doesn't clear the runner-up enough - too ambiguous
return (lfUnknown, fdmUnknown, 0.0)
# --- FALSE POSITIVE PREVENTION: Language-specific cross-contamination ---
if bestFlavor == "python" and scores.getOrDefault("bash", 0) >= bestScore div 2:
# Python + Bash signals could be mixed; verify with Python-specific patterns
if not source.contains("def ") and not source.contains("import "):
return (lfUnknown, fdmUnknown, 0.0)
# Convert bestFlavor to enum
case bestFlavor
of "nim": return (lfNim, fdmContent, min(bestScore.float / 20.0, 0.9))
of "python": return (lfPython, fdmContent, min(bestScore.float / 20.0, 0.9))
of "bash": return (lfBash, fdmContent, min(bestScore.float / 16.0, 0.85))
of "javascript": return (lfJavaScript, fdmContent, min(bestScore.float / 16.0, 0.85))
of "typescript": return (lfTypeScript, fdmContent, min(bestScore.float / 16.0, 0.85))
of "nim": return (lfNim, fdmContent, min(bestScore.float / 18.0, 0.9))
of "python": return (lfPython, fdmContent, min(bestScore.float / 18.0, 0.9))
of "bash": return (lfBash, fdmContent, min(bestScore.float / 14.0, 0.85))
of "javascript": return (lfJavaScript, fdmContent, min(bestScore.float / 14.0, 0.85))
of "typescript": return (lfTypeScript, fdmContent, min(bestScore.float / 14.0, 0.85))
of "go": return (lfGo, fdmContent, 0.8)
of "rust": return (lfRust, fdmContent, 0.8)
of "ruby": return (lfRuby, fdmContent, 0.7)
@ -342,6 +459,8 @@ proc detectByContent*(source: string, fileExt: string = ""): tuple[flavor: Langu
of "markdown": return (lfMarkdown, fdmContent, 0.6)
of "css": return (lfCSS, fdmContent, 0.7)
of "sql": return (lfSQL, fdmContent, 0.8)
of "html": return (lfHTML, fdmContent, 0.7)
of "xml": return (lfXML, fdmContent, 0.7)
else:
return (lfUnknown, fdmUnknown, 0.0)
@ -359,6 +478,12 @@ proc detectFlavor*(
##
## When explicitFlavor is provided and not lfUnknown, returns that.
## Otherwise, uses the best available detection method.
##
## False-positive prevention:
## - Entropy-based binary detection
## - Minimum source length
## - Confidence threshold floor of 0.5 for content detection
## - Winner-margin ratio for ambiguous cases
when defined(nimcheckDebug):
if isDebugEnabled():
@ -393,11 +518,15 @@ proc detectFlavor*(
# 4. Content-based detection
if source.len > 0:
let (contentFlavor, contentMethod, confidence) = detectByContent(source, filePath)
if contentFlavor != lfUnknown:
if contentFlavor != lfUnknown and confidence >= MinConfidenceThreshold:
when defined(nimcheckDebug):
if isDebugEnabled():
debugLog("DETECTOR", &"Detected by content: {contentFlavor} (confidence: {confidence})")
return (contentFlavor, contentMethod, confidence)
elif contentFlavor != lfUnknown and confidence < MinConfidenceThreshold:
when defined(nimcheckDebug):
if isDebugEnabled():
debugLog("DETECTOR", &"Content detection below confidence threshold: {contentFlavor} ({confidence:.2f} < {MinConfidenceThreshold})")
except Exception as e:
discard e

71
src/nimcheck/core/tokenizerbase.nim
View File

@ -185,6 +185,75 @@ proc recordErrorAtCurrent*(
## Record error at current position.
self.recordError(severity, message, code, self.currentPos(), hint)
const
TokenizerMaxTokensMultiplier* = 8
proc finishTokenizeStep*(self: TokenizerBase, posBefore: int) =
## Safety guard: every tokenize loop iteration must advance pos or abort.
## Prevents infinite loops from exhausting memory and crashing the host.
if self.pos <= posBefore and self.hasMore():
self.recordError(
esCritical,
"Tokenizer made no progress at this position",
"E9998",
self.currentPos(),
hint = "Skipped one character to prevent an internal infinite loop"
)
discard self.advance()
let maxTokens = max(self.sourceLen * TokenizerMaxTokensMultiplier, 64)
if self.tokens.len > maxTokens:
self.recordError(
esCritical,
&"Tokenizer exceeded safety limit ({maxTokens} tokens)",
"E9998",
self.currentPos(),
hint = "Aborting tokenization to prevent memory exhaustion"
)
self.pos = self.sourceLen
self.eof = true
proc closeBracketPair*(closeChar: char): (char, string) =
## Map a closing bracket to its opener and display string.
case closeChar
of ')': ('(', ")")
of ']': ('[', "]")
of '}': ('{', "}")
of '>': ('<', ">")
else: ('\0', $closeChar)
proc closeBracket*(
self: TokenizerBase,
closeChar: char,
closeKind: TokenKind,
startPos: SourcePosition
) =
## Emit a closing bracket and validate it against the bracket stack.
discard self.advance()
self.emitToken(closeKind, $closeChar, startPos)
let (openChar, openStr) = closeBracketPair(closeChar)
if openChar == '\0':
return
if self.bracketStack.len == 0:
self.recordError(esError,
&"Unexpected closing '{closeChar}' - no matching opener",
ErrMismatchedBracket, startPos,
hint = &"Remove the extra '{closeChar}' or add a matching '{openStr}'")
elif self.bracketStack[^1][0] != openChar:
let mismatchOpen = self.bracketStack[^1][0]
let expectedClose = case mismatchOpen
of '(': ")"
of '[': "]"
of '{': "}"
of '<': ">"
else: $mismatchOpen
self.recordError(esError,
&"Mismatched bracket: expected '{expectedClose}' but found '{closeChar}'",
ErrMismatchedBracket, startPos,
hint = "Check that brackets are properly nested")
discard self.bracketStack.pop()
else:
discard self.bracketStack.pop()
# ---------------------------------------------------------------------------
# Bracket validation
# ---------------------------------------------------------------------------
@ -212,7 +281,7 @@ proc validateBrackets*(self: TokenizerBase): seq[ValidationError] =
when defined(nimcheckDebug):
if self.options.debugMode:
debugError(err)
self.errors.add(result)
# Errors are returned to caller; do not double-add to self.errors
# ---------------------------------------------------------------------------
# Abstract interface - subclasses must implement

17
src/nimcheck/core/types.nim
View File

@ -398,6 +398,21 @@ proc newValidationOptions*(
includeHints: includeHints
)
proc newModuleInfo*(): ModuleInfo =
## Create an empty ModuleInfo.
ModuleInfo(
flavor: lfUnknown,
detectionMethod: fdmUnknown,
imports: @[],
variables: @[],
functions: @[],
classes: @[],
linesOfCode: 0,
totalTokens: 0,
comments: 0,
debugInfo: newJObject()
)
proc newValidationResult*(): ValidationResult =
## Create an empty ValidationResult.
ValidationResult(
@ -407,7 +422,7 @@ proc newValidationResult*(): ValidationResult =
errors: @[],
warnings: 0,
infos: 0,
moduleInfo: ModuleInfo(),
moduleInfo: newModuleInfo(),
durationMs: 0.0,
debugOutput: ""
)

19
src/nimcheck/core/validatorbase.nim
View File

@ -189,13 +189,30 @@ proc createValidator*(
of lfPython: "python"
of lfBash, lfShell: "bash"
of lfJavaScript: "javascript"
of lfTypeScript: "typescript"
of lfPHP: "php"
of lfHTML: "html"
of lfXML: "xml"
of lfJinja: "jinja"
of lfJSON: "json"
of lfYAML: "yaml"
of lfTOML: "toml"
else: $flavor # fallback for future flavors
of lfCSS: "css"
of lfSQL: "sql"
of lfMarkdown: "markdown"
of lfDockerfile: "dockerfile"
of lfMakefile: "makefile"
of lfRuby: "ruby"
of lfRust: "rust"
of lfGo: "go"
of lfLua: "lua"
of lfC: "c"
of lfCpp: "cpp"
of lfCSharp: "csharp"
of lfJava: "java"
of lfSwift: "swift"
of lfKotlin: "kotlin"
else: $flavor
if validatorRegistry.hasKey(flavorStr):
result = validatorRegistry[flavorStr](source, options, sourcePath)
else:

162
src/nimcheck/languages/cfamily_validators.nim Normal file
View File

@ -0,0 +1,162 @@
## C-family language validators: C, C++, Java, C#.
##
## Each validator follows the pattern from extended_validators.nim,
## using the shared structural analysis helpers.
import std/[json, strformat, tables]
{.warning[UnusedImport]:off.}
import ../core/types, ../core/tokenizerbase, ../core/validatorbase
import ../tokenizers/extended_tokenizers, ../tokenizers/cfamily_tokenizers
export extended_tokenizers, cfamily_tokenizers
# ---------------------------------------------------------------------------
# Shared C-family analysis helpers
# ---------------------------------------------------------------------------
proc analyzeCFamilyTokens*(self: ValidatorBase, importKw: string) =
## Analyze C-family tokens for structural elements.
let tokens = self.tokenizer.tokens
var i = 0
while i < tokens.len:
let tok = tokens[i]
if tok.kind == tkComment:
self.result.moduleInfo.comments += 1
elif tok.kind == tkDocComment:
self.result.moduleInfo.comments += 1
elif tok.kind == tkKeyword:
# Import/include
if tok.value == importKw:
var j = i + 1
while j < tokens.len and tokens[j].kind in {tkWhitespace, tkNewline}:
j += 1
if j < tokens.len and tokens[j].kind in {tkString, tkIdentifier, tkOperator}:
var modName = tokens[j].value
# Handle #include <file> or #include "file"
if modName == "<" or modName == "\"":
j += 1
var fullMod = ""
while j < tokens.len and tokens[j].kind notin {tkNewline, tkEndOfFile}:
if tokens[j].kind in {tkIdentifier, tkString, tkOperator, tkPunctuation}:
fullMod.add(tokens[j].value)
if tokens[j].value == ">" or tokens[j].value == "\"":
break
j += 1
if fullMod.len > 0:
self.result.moduleInfo.imports.add(ImportInfo(
module: fullMod, position: tokens[j].position))
else:
self.result.moduleInfo.imports.add(ImportInfo(
module: modName, position: tokens[j].position))
# Functions
elif tok.value in ["func", "function"]:
var j = i + 1
while j < tokens.len and tokens[j].kind in {tkWhitespace, tkNewline}:
j += 1
if j < tokens.len and tokens[j].kind == tkIdentifier:
self.result.moduleInfo.functions.add(FunctionInfo(
name: tokens[j].value, kind: "function", position: tokens[j].position))
# Classes, structs, interfaces, enums
elif tok.value in ["struct", "class", "interface", "enum", "record"]:
var j = i + 1
while j < tokens.len and tokens[j].kind in {tkWhitespace, tkNewline}:
j += 1
if j < tokens.len and tokens[j].kind == tkIdentifier:
self.result.moduleInfo.classes.add(ClassInfo(
name: tokens[j].value, kind: tok.value, position: tokens[j].position))
else:
discard
i += 1
# ---------------------------------------------------------------------------
# C Validator
# ---------------------------------------------------------------------------
type CValidator* = ref object of ValidatorBase
proc newCValidator*(source: string, options: ValidationOptions = newValidationOptions(), sourcePath: string = ""): CValidator =
CValidator(source: source, sourcePath: sourcePath, options: options,
result: newValidationResult(), startTime: epochTime())
method createTokenizer*(self: CValidator): TokenizerBase =
newCTokenizer(self.source, self.options)
method analyzeTokens*(self: CValidator) =
procCall ValidatorBase(self).analyzeTokens()
analyzeCFamilyTokens(self, "#include")
# ---------------------------------------------------------------------------
# C++ Validator
# ---------------------------------------------------------------------------
type CppValidator* = ref object of ValidatorBase
proc newCppValidator*(source: string, options: ValidationOptions = newValidationOptions(), sourcePath: string = ""): CppValidator =
CppValidator(source: source, sourcePath: sourcePath, options: options,
result: newValidationResult(), startTime: epochTime())
method createTokenizer*(self: CppValidator): TokenizerBase =
newCppTokenizer(self.source, self.options)
method analyzeTokens*(self: CppValidator) =
procCall ValidatorBase(self).analyzeTokens()
analyzeCFamilyTokens(self, "#include")
# ---------------------------------------------------------------------------
# Java Validator
# ---------------------------------------------------------------------------
type JavaValidator* = ref object of ValidatorBase
proc newJavaValidator*(source: string, options: ValidationOptions = newValidationOptions(), sourcePath: string = ""): JavaValidator =
JavaValidator(source: source, sourcePath: sourcePath, options: options,
result: newValidationResult(), startTime: epochTime())
method createTokenizer*(self: JavaValidator): TokenizerBase =
newJavaTokenizer(self.source, self.options)
method analyzeTokens*(self: JavaValidator) =
procCall ValidatorBase(self).analyzeTokens()
analyzeCFamilyTokens(self, "import")
# ---------------------------------------------------------------------------
# C# Validator
# ---------------------------------------------------------------------------
type CSharpValidator* = ref object of ValidatorBase
proc newCSharpValidator*(source: string, options: ValidationOptions = newValidationOptions(), sourcePath: string = ""): CSharpValidator =
CSharpValidator(source: source, sourcePath: sourcePath, options: options,
result: newValidationResult(), startTime: epochTime())
method createTokenizer*(self: CSharpValidator): TokenizerBase =
newCSharpTokenizer(self.source, self.options)
method analyzeTokens*(self: CSharpValidator) =
procCall ValidatorBase(self).analyzeTokens()
analyzeCFamilyTokens(self, "using")
# ---------------------------------------------------------------------------
# Registration
# ---------------------------------------------------------------------------
proc init*() =
registerValidator("c", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
result = newCValidator(source, options, sourcePath))
registerValidator("cpp", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
result = newCppValidator(source, options, sourcePath))
registerValidator("cxx", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
result = newCppValidator(source, options, sourcePath))
registerValidator("h", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
result = newCValidator(source, options, sourcePath))
registerValidator("hpp", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
result = newCppValidator(source, options, sourcePath))
registerValidator("java", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
result = newJavaValidator(source, options, sourcePath))
registerValidator("csharp", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
result = newCSharpValidator(source, options, sourcePath))
registerValidator("cs", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
result = newCSharpValidator(source, options, sourcePath))
init()

371
src/nimcheck/languages/extended_validators.nim Normal file
View File

@ -0,0 +1,371 @@
## Extended language validators: Go, Rust, Ruby, CSS, SQL, Markdown, Dockerfile, Makefile.
import std/[json, strformat, tables, sequtils]
{.warning[UnusedImport]:off.}
import ../core/types, ../core/tokenizerbase, ../core/validatorbase
import ../tokenizers/extended_tokenizers
# ---------------------------------------------------------------------------
# Shared structural analysis helpers
# ---------------------------------------------------------------------------
proc analyzeBlockKeywords*(
self: ValidatorBase,
openers: seq[string],
closers: seq[string],
openerCounts: var Table[string, int],
closerCounts: var Table[string, int]
) =
for tok in self.tokenizer.tokens:
if tok.kind == tkKeyword:
if tok.value in openers:
openerCounts.mgetOrPut(tok.value, 0).inc()
elif tok.value in closers:
closerCounts.mgetOrPut(tok.value, 0).inc()
proc checkBlockBalance*(
self: ValidatorBase,
pairs: seq[(string, string)],
openerCounts, closerCounts: Table[string, int]
) =
for (opener, closer) in pairs:
let openCount = openerCounts.getOrDefault(opener, 0)
let closeCount = closerCounts.getOrDefault(closer, 0)
if openCount != closeCount and openCount > closeCount:
self.result.errors.add(newValidationError(
esError,
&"Unclosed '{opener}' block: {openCount} '{opener}' but {closeCount} '{closer}'",
ErrUnclosedBlock,
newSourcePosition(1, 1, 0),
newSourceRange(newSourcePosition(1, 1, 0), newSourcePosition(1, 1, 0)),
hint = &"Add missing '{closer}' to close the block"
))
proc extractImportsAndFunctions*(self: ValidatorBase, importKw: string, funcKw: string) =
let tokens = self.tokenizer.tokens
var i = 0
while i < tokens.len:
let tok = tokens[i]
if tok.kind == tkComment:
self.result.moduleInfo.comments += 1
elif tok.kind == tkKeyword:
if tok.value == importKw:
var j = i + 1
while j < tokens.len and tokens[j].kind in {tkWhitespace, tkNewline}:
j += 1
if j < tokens.len and tokens[j].kind in {tkIdentifier, tkString, tkPunctuation}:
var modName = tokens[j].value
if modName == "(":
j += 1
while j < tokens.len and tokens[j].kind notin {tkPunctuation, tkNewline, tkEndOfFile}:
if tokens[j].kind == tkIdentifier:
self.result.moduleInfo.imports.add(ImportInfo(
module: tokens[j].value, position: tokens[j].position))
j += 1
else:
self.result.moduleInfo.imports.add(ImportInfo(module: modName, position: tokens[j].position))
elif tok.value == funcKw:
var j = i + 1
while j < tokens.len and tokens[j].kind in {tkWhitespace, tkNewline}:
j += 1
if j < tokens.len and tokens[j].kind == tkIdentifier:
self.result.moduleInfo.functions.add(FunctionInfo(
name: tokens[j].value, kind: funcKw, position: tokens[j].position))
elif tok.value in ["struct", "type", "class", "interface", "enum", "trait", "impl"]:
var j = i + 1
while j < tokens.len and tokens[j].kind in {tkWhitespace, tkNewline}:
j += 1
if j < tokens.len and tokens[j].kind == tkIdentifier:
self.result.moduleInfo.classes.add(ClassInfo(
name: tokens[j].value, kind: tok.value, position: tokens[j].position))
i += 1
# ---------------------------------------------------------------------------
# Go Validator
# ---------------------------------------------------------------------------
type GoValidator* = ref object of ValidatorBase
proc newGoValidator*(source: string, options: ValidationOptions = newValidationOptions(), sourcePath: string = ""): GoValidator =
GoValidator(source: source, sourcePath: sourcePath, options: options,
result: newValidationResult(), startTime: epochTime())
method createTokenizer*(self: GoValidator): TokenizerBase =
newGoTokenizer(self.source, self.options)
method analyzeTokens*(self: GoValidator) =
procCall ValidatorBase(self).analyzeTokens()
extractImportsAndFunctions(self, "import", "func")
# ---------------------------------------------------------------------------
# Rust Validator
# ---------------------------------------------------------------------------
type RustValidator* = ref object of ValidatorBase
proc newRustValidator*(source: string, options: ValidationOptions = newValidationOptions(), sourcePath: string = ""): RustValidator =
RustValidator(source: source, sourcePath: sourcePath, options: options,
result: newValidationResult(), startTime: epochTime())
method createTokenizer*(self: RustValidator): TokenizerBase =
newRustTokenizer(self.source, self.options)
method analyzeTokens*(self: RustValidator) =
procCall ValidatorBase(self).analyzeTokens()
extractImportsAndFunctions(self, "use", "fn")
# ---------------------------------------------------------------------------
# Ruby Validator
# ---------------------------------------------------------------------------
type RubyValidator* = ref object of ValidatorBase
proc newRubyValidator*(source: string, options: ValidationOptions = newValidationOptions(), sourcePath: string = ""): RubyValidator =
RubyValidator(source: source, sourcePath: sourcePath, options: options,
result: newValidationResult(), startTime: epochTime())
method createTokenizer*(self: RubyValidator): TokenizerBase =
newRubyTokenizer(self.source, self.options)
method analyzeTokens*(self: RubyValidator) =
procCall ValidatorBase(self).analyzeTokens()
var blockStack: seq[string] = @[]
var lineHasContent = false
for tok in self.tokenizer.tokens:
if tok.kind == tkNewline:
lineHasContent = false
elif tok.kind notin {tkWhitespace, tkComment}:
if tok.kind == tkKeyword:
case tok.value
of "def", "class", "module", "case", "begin", "do":
blockStack.add(tok.value)
lineHasContent = true
of "if", "unless", "while", "until", "for":
# Postfix modifiers (`raise ... unless cond`) are not block openers.
if not lineHasContent:
blockStack.add(tok.value)
lineHasContent = true
of "end":
if blockStack.len > 0:
discard blockStack.pop()
else:
self.result.errors.add(newValidationError(
esError, "Unexpected 'end' with no matching block opener",
ErrUnclosedBlock, tok.position,
newSourceRange(tok.position, tok.position)))
lineHasContent = true
else:
lineHasContent = true
else:
lineHasContent = true
for b in blockStack:
self.result.errors.add(newValidationError(
esError, &"Unclosed Ruby '#{b}' block",
ErrUnclosedBlock, newSourcePosition(1, 1, 0),
newSourceRange(newSourcePosition(1, 1, 0), newSourcePosition(1, 1, 0)),
hint = "Add missing 'end'"))
var ri = 0
while ri < self.tokenizer.tokens.len:
let rtok = self.tokenizer.tokens[ri]
if rtok.kind == tkComment:
self.result.moduleInfo.comments += 1
elif rtok.kind == tkKeyword:
if rtok.value == "def":
var j = ri + 1
while j < self.tokenizer.tokens.len and self.tokenizer.tokens[j].kind in {tkWhitespace, tkNewline}:
j += 1
if j < self.tokenizer.tokens.len and self.tokenizer.tokens[j].kind == tkIdentifier:
self.result.moduleInfo.functions.add(FunctionInfo(
name: self.tokenizer.tokens[j].value, kind: "def", position: self.tokenizer.tokens[j].position))
elif rtok.value == "class":
var j = ri + 1
while j < self.tokenizer.tokens.len and self.tokenizer.tokens[j].kind in {tkWhitespace, tkNewline}:
j += 1
if j < self.tokenizer.tokens.len and self.tokenizer.tokens[j].kind == tkIdentifier:
self.result.moduleInfo.classes.add(ClassInfo(
name: self.tokenizer.tokens[j].value, kind: "class", position: self.tokenizer.tokens[j].position))
ri += 1
# ---------------------------------------------------------------------------
# CSS Validator
# ---------------------------------------------------------------------------
type CSSValidator* = ref object of ValidatorBase
proc newCSSValidator*(source: string, options: ValidationOptions = newValidationOptions(), sourcePath: string = ""): CSSValidator =
CSSValidator(source: source, sourcePath: sourcePath, options: options,
result: newValidationResult(), startTime: epochTime())
method createTokenizer*(self: CSSValidator): TokenizerBase =
newCSSTokenizer(self.source, self.options)
method analyzeTokens*(self: CSSValidator) =
procCall ValidatorBase(self).analyzeTokens()
let lines = self.source.splitLines()
for i in 0..<lines.len - 1:
let cur = lines[i].strip()
let nxt = lines[i + 1].strip()
if cur.len > 0 and cur[0] in {'.', '#'} and cur.find('{') < 0 and
nxt.len > 0 and ':' in nxt and nxt.find('{') < 0 and not nxt.startsWith("@"):
self.result.errors.add(newValidationError(
esError, &"CSS selector '{cur}' missing opening brace before properties (line {i + 1})",
ErrUnclosedBlock, newSourcePosition(i + 1, 1, 0),
newSourceRange(newSourcePosition(i + 1, 1, 0), newSourcePosition(i + 2, 1, 0)),
hint = "Add '{' after the selector"))
for tok in self.tokenizer.tokens:
if tok.kind == tkComment:
self.result.moduleInfo.comments += 1
elif tok.kind == tkDirective:
self.result.moduleInfo.functions.add(FunctionInfo(
name: tok.value, kind: "at-rule", position: tok.position))
# ---------------------------------------------------------------------------
# SQL Validator
# ---------------------------------------------------------------------------
type SQLValidator* = ref object of ValidatorBase
proc newSQLValidator*(source: string, options: ValidationOptions = newValidationOptions(), sourcePath: string = ""): SQLValidator =
SQLValidator(source: source, sourcePath: sourcePath, options: options,
result: newValidationResult(), startTime: epochTime())
method createTokenizer*(self: SQLValidator): TokenizerBase =
newSQLTokenizer(self.source, self.options)
proc hasTrailingCommaBeforeParen*(source: string): bool =
## Detect `,)` with optional whitespace/newlines between comma and ')'.
var i = 0
while i < source.len:
if source[i] == ',':
var j = i + 1
while j < source.len and source[j] in {' ', '\t', '\r', '\n'}:
j += 1
if j < source.len and source[j] == ')':
return true
inc i
proc hasIncompleteFromClause*(source: string): bool =
let upper = source.toUpperAscii()
var i = 0
while i < upper.len:
if upper[i] == 'F' and i + 4 < upper.len and upper[i..<i+4] == "FROM":
var j = i + 4
while j < upper.len and upper[j] in {' ', '\t', '\r', '\n'}:
j += 1
if j < upper.len and upper[j] == ';':
return true
inc i
method analyzeTokens*(self: SQLValidator) =
procCall ValidatorBase(self).analyzeTokens()
if hasTrailingCommaBeforeParen(self.source):
self.result.errors.add(newValidationError(
esError, "Trailing comma before closing parenthesis in SQL",
ErrInvalidSyntax, newSourcePosition(1, 1, 0),
newSourceRange(newSourcePosition(1, 1, 0), newSourcePosition(1, 1, 0)),
hint = "Remove the comma before ')'"))
elif hasIncompleteFromClause(self.source):
self.result.errors.add(newValidationError(
esError, "Incomplete FROM clause in SQL",
ErrInvalidSyntax, newSourcePosition(1, 1, 0),
newSourceRange(newSourcePosition(1, 1, 0), newSourcePosition(1, 1, 0)),
hint = "Specify a table name after FROM"))
for tok in self.tokenizer.tokens:
if tok.kind == tkComment:
self.result.moduleInfo.comments += 1
# ---------------------------------------------------------------------------
# Markdown Validator
# ---------------------------------------------------------------------------
type MarkdownValidator* = ref object of ValidatorBase
proc newMarkdownValidator*(source: string, options: ValidationOptions = newValidationOptions(), sourcePath: string = ""): MarkdownValidator =
MarkdownValidator(source: source, sourcePath: sourcePath, options: options,
result: newValidationResult(), startTime: epochTime())
method createTokenizer*(self: MarkdownValidator): TokenizerBase =
newMarkdownTokenizer(self.source, self.options)
method analyzeTokens*(self: MarkdownValidator) =
procCall ValidatorBase(self).analyzeTokens()
for tok in self.tokenizer.tokens:
if tok.kind == tkDirective:
self.result.moduleInfo.functions.add(FunctionInfo(
name: tok.value, kind: "heading", position: tok.position))
# ---------------------------------------------------------------------------
# Dockerfile Validator
# ---------------------------------------------------------------------------
type DockerfileValidator* = ref object of ValidatorBase
proc newDockerfileValidator*(source: string, options: ValidationOptions = newValidationOptions(), sourcePath: string = ""): DockerfileValidator =
DockerfileValidator(source: source, sourcePath: sourcePath, options: options,
result: newValidationResult(), startTime: epochTime())
method createTokenizer*(self: DockerfileValidator): TokenizerBase =
newDockerfileTokenizer(self.source, self.options)
method analyzeTokens*(self: DockerfileValidator) =
## Dockerfile tokenizer handles its own structural checks; skip bracket validation.
for tok in self.tokenizer.tokens:
if tok.kind == tkDirective:
self.result.moduleInfo.functions.add(FunctionInfo(
name: tok.value, kind: "instruction", position: tok.position))
elif tok.kind == tkComment:
self.result.moduleInfo.comments += 1
# ---------------------------------------------------------------------------
# Makefile Validator
# ---------------------------------------------------------------------------
type MakefileValidator* = ref object of ValidatorBase
proc newMakefileValidator*(source: string, options: ValidationOptions = newValidationOptions(), sourcePath: string = ""): MakefileValidator =
MakefileValidator(source: source, sourcePath: sourcePath, options: options,
result: newValidationResult(), startTime: epochTime())
method createTokenizer*(self: MakefileValidator): TokenizerBase =
newMakefileTokenizer(self.source, self.options)
method analyzeTokens*(self: MakefileValidator) =
for tok in self.tokenizer.tokens:
if tok.kind == tkComment:
self.result.moduleInfo.comments += 1
elif tok.kind == tkIdentifier:
self.result.moduleInfo.functions.add(FunctionInfo(
name: tok.value, kind: "target", position: tok.position))
elif tok.kind == tkAssignment:
let eqIdx = tok.value.find('=')
if eqIdx > 0:
self.result.moduleInfo.variables.add(VariableInfo(
name: tok.value[0..<eqIdx].strip(), kind: "make", position: tok.position))
# ---------------------------------------------------------------------------
# Registration
# ---------------------------------------------------------------------------
proc init*() =
registerValidator("go", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
result = newGoValidator(source, options, sourcePath))
registerValidator("rust", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
result = newRustValidator(source, options, sourcePath))
registerValidator("ruby", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
result = newRubyValidator(source, options, sourcePath))
registerValidator("css", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
result = newCSSValidator(source, options, sourcePath))
registerValidator("sql", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
result = newSQLValidator(source, options, sourcePath))
registerValidator("markdown", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
result = newMarkdownValidator(source, options, sourcePath))
registerValidator("md", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
result = newMarkdownValidator(source, options, sourcePath))
registerValidator("dockerfile", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
result = newDockerfileValidator(source, options, sourcePath))
registerValidator("makefile", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
result = newMakefileValidator(source, options, sourcePath))
init()

5
src/nimcheck/languages/html_validator.nim
View File

@ -24,9 +24,10 @@ method createTokenizer*(self: HTMLValidator): TokenizerBase =
method analyzeTokens*(self: HTMLValidator) =
when defined(nimcheckDebug):
if self.options.debugMode: debugEnter("HTML_ANALYZE", "Analyzing HTML")
procCall ValidatorBase(self).analyzeTokens()
## Tag-stack validation is performed in the HTML tokenizer.
for tok in self.tokenizer.tokens:
if tok.kind == tkComment: self.result.moduleInfo.comments += 1
if tok.kind == tkComment:
self.result.moduleInfo.comments += 1
when defined(nimcheckDebug):
if self.options.debugMode: debugLeave("HTML_ANALYZE")

172
src/nimcheck/languages/lang_validators.nim Normal file
View File

@ -0,0 +1,172 @@
## Language validators: Kotlin, Lua, Swift.
##
## Each validator provides structural analysis for its language,
## following the patterns in extended_validators.nim.
import std/[json, strformat, tables]
{.warning[UnusedImport]:off.}
import ../core/types, ../core/tokenizerbase, ../core/validatorbase
import ../tokenizers/lang_tokenizers
import extended_validators
# ---------------------------------------------------------------------------
# Shared helper
# ---------------------------------------------------------------------------
proc analyzeLangTokens*(self: ValidatorBase, funcKws: seq[string], classKws: seq[string],
importKws: seq[string] = @[]) =
## Analyze tokens for structural elements common across Kotlin/Lua/Swift.
let tokens = self.tokenizer.tokens
var i = 0
while i < tokens.len:
let tok = tokens[i]
if tok.kind == tkComment:
self.result.moduleInfo.comments += 1
elif tok.kind == tkDocComment:
self.result.moduleInfo.comments += 1
elif tok.kind == tkKeyword:
if tok.value in importKws:
var j = i + 1
while j < tokens.len and tokens[j].kind in {tkWhitespace, tkNewline}:
j += 1
if j < tokens.len and tokens[j].kind in {tkIdentifier, tkString}:
self.result.moduleInfo.imports.add(ImportInfo(
module: tokens[j].value, position: tokens[j].position))
elif tok.value in funcKws:
var j = i + 1
while j < tokens.len and tokens[j].kind in {tkWhitespace, tkNewline}:
j += 1
if j < tokens.len and tokens[j].kind == tkIdentifier:
self.result.moduleInfo.functions.add(FunctionInfo(
name: tokens[j].value, kind: tok.value, position: tokens[j].position))
elif tok.value in classKws:
var j = i + 1
while j < tokens.len and tokens[j].kind in {tkWhitespace, tkNewline}:
j += 1
if j < tokens.len and tokens[j].kind == tkIdentifier:
self.result.moduleInfo.classes.add(ClassInfo(
name: tokens[j].value, kind: tok.value, position: tokens[j].position))
else:
discard
i += 1
# ---------------------------------------------------------------------------
# Kotlin Validator
# ---------------------------------------------------------------------------
type KotlinValidator* = ref object of ValidatorBase
proc newKotlinValidator*(source: string, options: ValidationOptions = newValidationOptions(), sourcePath: string = ""): KotlinValidator =
KotlinValidator(source: source, sourcePath: sourcePath, options: options,
result: newValidationResult(), startTime: epochTime())
method createTokenizer*(self: KotlinValidator): TokenizerBase =
newKotlinTokenizer(self.source, self.options)
method analyzeTokens*(self: KotlinValidator) =
procCall ValidatorBase(self).analyzeTokens()
analyzeLangTokens(self,
funcKws = @["fun"],
classKws = @["class", "interface", "object", "enum", "data", "sealed", "annotation", "abstract"],
importKws = @["import"])
# ---------------------------------------------------------------------------
# Lua Validator
# ---------------------------------------------------------------------------
type LuaValidator* = ref object of ValidatorBase
proc newLuaValidator*(source: string, options: ValidationOptions = newValidationOptions(), sourcePath: string = ""): LuaValidator =
LuaValidator(source: source, sourcePath: sourcePath, options: options,
result: newValidationResult(), startTime: epochTime())
method createTokenizer*(self: LuaValidator): TokenizerBase =
newLuaTokenizer(self.source, self.options)
method analyzeTokens*(self: LuaValidator) =
procCall ValidatorBase(self).analyzeTokens()
let tokens = self.tokenizer.tokens
var i = 0
while i < tokens.len:
let tok = tokens[i]
if tok.kind == tkComment:
self.result.moduleInfo.comments += 1
elif tok.kind == tkKeyword:
if tok.value == "function":
var j = i + 1
while j < tokens.len and tokens[j].kind in {tkWhitespace, tkNewline}:
j += 1
if j < tokens.len and tokens[j].kind == tkIdentifier:
var funcName = tokens[j].value
# Lua allows function module.funcname syntax
var k = j + 1
while k < tokens.len and tokens[k].kind in {tkWhitespace, tkNewline}:
k += 1
if k < tokens.len and tokens[k].kind == tkPunctuation and k + 1 < tokens.len and tokens[k+1].kind == tkIdentifier:
funcName = funcName & "." & tokens[k+1].value
self.result.moduleInfo.functions.add(FunctionInfo(
name: funcName, kind: "function", position: tokens[j].position))
else:
discard
i += 1
var blockStack: seq[string] = @[]
for tok in tokens:
if tok.kind == tkKeyword:
case tok.value
of "function", "if", "for", "while", "repeat":
blockStack.add(tok.value)
of "do":
# `for`/`while` loops use `do` as syntax, not a nested block opener.
if blockStack.len == 0 or blockStack[^1] notin ["for", "while"]:
blockStack.add(tok.value)
of "end":
if blockStack.len > 0 and blockStack[^1] != "repeat":
discard blockStack.pop()
of "until":
if blockStack.len > 0 and blockStack[^1] == "repeat":
discard blockStack.pop()
for b in blockStack:
let closer = if b == "repeat": "until" else: "end"
self.result.errors.add(newValidationError(
esError, &"Unclosed Lua '#{b}' block",
ErrUnclosedBlock, newSourcePosition(1, 1, 0),
newSourceRange(newSourcePosition(1, 1, 0), newSourcePosition(1, 1, 0)),
hint = &"Add missing '#{closer}'"))
# ---------------------------------------------------------------------------
# Swift Validator
# ---------------------------------------------------------------------------
type SwiftValidator* = ref object of ValidatorBase
proc newSwiftValidator*(source: string, options: ValidationOptions = newValidationOptions(), sourcePath: string = ""): SwiftValidator =
SwiftValidator(source: source, sourcePath: sourcePath, options: options,
result: newValidationResult(), startTime: epochTime())
method createTokenizer*(self: SwiftValidator): TokenizerBase =
newSwiftTokenizer(self.source, self.options)
method analyzeTokens*(self: SwiftValidator) =
procCall ValidatorBase(self).analyzeTokens()
analyzeLangTokens(self,
funcKws = @["func"],
classKws = @["class", "struct", "enum", "protocol", "extension", "actor"],
importKws = @["import"])
# ---------------------------------------------------------------------------
# Registration
# ---------------------------------------------------------------------------
proc init*() =
registerValidator("kotlin", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
result = newKotlinValidator(source, options, sourcePath))
registerValidator("kt", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
result = newKotlinValidator(source, options, sourcePath))
registerValidator("lua", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
result = newLuaValidator(source, options, sourcePath))
registerValidator("swift", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
result = newSwiftValidator(source, options, sourcePath))
init()

108
src/nimcheck/languages/type_xml_validator.nim Normal file
View File

@ -0,0 +1,108 @@
## TypeScript and XML validators for Nimcheck.
##
## Provides improved TypeScript validation and full XML structure validation.
import std/[json, strformat, tables]
{.warning[UnusedImport]:off.}
import ../core/types, ../core/tokenizerbase, ../core/validatorbase
import ../tokenizers/type_xml_tokenizer
# ---------------------------------------------------------------------------
# TypeScript Validator
# ---------------------------------------------------------------------------
type TypeScriptValidator* = ref object of ValidatorBase
proc newTypeScriptValidator*(source: string, options: ValidationOptions = newValidationOptions(), sourcePath: string = ""): TypeScriptValidator =
TypeScriptValidator(source: source, sourcePath: sourcePath, options: options,
result: newValidationResult(), startTime: epochTime())
method createTokenizer*(self: TypeScriptValidator): TokenizerBase =
newTypeScriptTokenizer(self.source, self.options)
method analyzeTokens*(self: TypeScriptValidator) =
procCall ValidatorBase(self).analyzeTokens()
let tokens = self.tokenizer.tokens
var i = 0
while i < tokens.len:
let tok = tokens[i]
case tok.kind
of tkComment, tkDocComment:
self.result.moduleInfo.comments += 1
of tkKeyword:
if tok.value in ["import", "export"]:
var j = i + 1
while j < tokens.len and tokens[j].kind in {tkWhitespace, tkNewline}:
j += 1
if j < tokens.len and tokens[j].kind in {tkKeyword, tkIdentifier, tkPunctuation}:
var importInfo = ImportInfo(module: "", position: tokens[j].position)
self.result.moduleInfo.imports.add(importInfo)
elif tok.value == "function":
var j = i + 1
while j < tokens.len and tokens[j].kind in {tkWhitespace, tkNewline}:
j += 1
if j < tokens.len and tokens[j].kind == tkIdentifier:
self.result.moduleInfo.functions.add(FunctionInfo(
name: tokens[j].value, kind: "function", position: tokens[j].position))
elif tok.value in ["class", "interface", "enum", "type"]:
var j = i + 1
while j < tokens.len and tokens[j].kind in {tkWhitespace, tkNewline}:
j += 1
if j < tokens.len and tokens[j].kind == tkIdentifier:
self.result.moduleInfo.classes.add(ClassInfo(
name: tokens[j].value, kind: tok.value, position: tokens[j].position))
of tkDirective:
if tok.value.startsWith("@"):
# Decorator
self.result.moduleInfo.functions.add(FunctionInfo(
name: tok.value, kind: "decorator", position: tok.position))
else:
discard
i += 1
# ---------------------------------------------------------------------------
# XML Validator
# ---------------------------------------------------------------------------
type XMLValidator* = ref object of ValidatorBase
proc newXMLValidator*(source: string, options: ValidationOptions = newValidationOptions(), sourcePath: string = ""): XMLValidator =
XMLValidator(source: source, sourcePath: sourcePath, options: options,
result: newValidationResult(), startTime: epochTime())
method createTokenizer*(self: XMLValidator): TokenizerBase =
newXMLTokenizer(self.source, self.options)
method analyzeTokens*(self: XMLValidator) =
## XML tokenizer handles its own structural checks (tag matching);
## skip bracket validation and just collect structural info.
for tok in self.tokenizer.tokens:
if tok.kind == tkComment:
self.result.moduleInfo.comments += 1
elif tok.kind == tkDirective:
self.result.moduleInfo.functions.add(FunctionInfo(
name: tok.value, kind: "directive", position: tok.position))
elif tok.kind == tkKeyword:
self.result.moduleInfo.classes.add(ClassInfo(
name: tok.value, kind: "tag", position: tok.position))
# ---------------------------------------------------------------------------
# Registration
# ---------------------------------------------------------------------------
proc init*() =
registerValidator("typescript", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
result = newTypeScriptValidator(source, options, sourcePath))
registerValidator("ts", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
result = newTypeScriptValidator(source, options, sourcePath))
registerValidator("xml", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
result = newXMLValidator(source, options, sourcePath))
registerValidator("xsd", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
result = newXMLValidator(source, options, sourcePath))
registerValidator("xslt", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
result = newXMLValidator(source, options, sourcePath))
registerValidator("svg", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
result = newXMLValidator(source, options, sourcePath))
init()

2
src/nimcheck/tokenizers/bash_tokenizer.nim
View File

@ -245,7 +245,7 @@ method tokenize*(self: BashTokenizer) =
else:
self.emitBracketToken(kOpenBracket, $self.advance(), startPos)
elif c == ']':
if self.pos > 1 and self.source[self.pos-1] == '[': # simplified
if self.peekString(2) == "]]":
discard self.advance()
discard self.advance()
self.emitToken(tkKeyword, "]]", startPos)

13
src/nimcheck/tokenizers/cfamily_tokenizers.nim Normal file
View File

@ -0,0 +1,13 @@
## C-family language tokenizers: C, C++, Java, C#.
##
## Each tokenizer uses the CLikeTokenizer from extended_tokenizers
## with its own dialect and keyword set.
##
## The tokenizer constructors, keyword sets and CLikeDialect values
## are defined in extended_tokenizers.nim and re-exported here for
## convenient access from the C-family validators.
{.warning[UnusedImport]:off.}
import ../core/types, ../core/tokenizerbase
import extended_tokenizers
export extended_tokenizers

750
src/nimcheck/tokenizers/extended_tokenizers.nim Normal file
View File

@ -0,0 +1,750 @@
## Extended language tokenizers: Go, Rust, Ruby, CSS, SQL, Markdown, Dockerfile, Makefile.
import std/[strformat, sets, sequtils]
{.warning[UnusedImport]:off.}
import ../core/types, ../core/tokenizerbase
# ---------------------------------------------------------------------------
# Shared C-like tokenization helpers
# ---------------------------------------------------------------------------
type
CLikeDialect* = enum
cdGo, cdRust, cdCss, cdC, cdCpp, cdJava, cdCSharp
CLikeTokenizer* = ref object of TokenizerBase
dialect*: CLikeDialect
keywords*: HashSet[string]
proc newCLikeTokenizer*(
source: string,
dialect: CLikeDialect,
keywords: seq[string],
options: ValidationOptions = newValidationOptions()
): CLikeTokenizer =
result = CLikeTokenizer(
source: source,
sourceLen: source.len,
options: options,
tokens: @[],
pos: 0,
line: 1,
column: 1,
bracketStack: @[],
errors: @[],
eof: false,
dialect: dialect,
keywords: keywords.toHashSet()
)
proc parseRustRawString*(self: CLikeTokenizer, startPos: SourcePosition): string =
result = "r"
discard self.advance()
var hashCount = 0
while self.hasMore() and self.peek() == '#':
hashCount += 1
result.add(self.advance())
if not self.hasMore() or self.peek() != '"':
self.recordError(esError, "Invalid Rust raw string literal", ErrUnclosedString, startPos)
return
result.add(self.advance())
let closeDelim = repeat("#", hashCount) & "\""
while self.hasMore():
if self.peekString(closeDelim.len) == closeDelim:
for _ in 0..<closeDelim.len:
result.add(self.advance())
return
result.add(self.advance())
self.recordError(esError, "Unclosed Rust raw string literal", ErrUnclosedString, startPos,
hint = &"Add closing '{closeDelim}'")
proc parseQuotedString*(
self: TokenizerBase,
quote: char,
startPos: SourcePosition,
allowNewline: bool = false
): string =
result = $quote
while self.hasMore():
let c = self.advance()
if c == '\\':
result.add(c)
if self.hasMore():
result.add(self.advance())
elif c == quote:
result.add(quote)
return
elif c == '\n' and not allowNewline:
self.recordError(esError, &"Unclosed string literal starting with '{quote}'",
ErrUnclosedString, startPos, hint = "Add a closing quote")
return
else:
result.add(c)
self.recordError(esError, &"Unclosed string literal starting with '{quote}'",
ErrUnclosedString, startPos, hint = "Add a closing quote before end of file")
proc parseBacktickString*(self: TokenizerBase, startPos: SourcePosition): string =
result = "`"
while self.hasMore():
let c = self.advance()
if c == '`':
result.add('`')
return
result.add(c)
self.recordError(esError, "Unclosed raw string literal", ErrUnclosedString, startPos,
hint = "Add a closing backtick")
method tokenize*(self: CLikeTokenizer) =
when defined(nimcheckDebug):
if self.options.debugMode:
debugEnter("CLIKE_TOKENIZER", &"Starting {self.dialect} tokenization")
while self.hasMore():
let posBefore = self.pos
let startPos = self.currentPos()
let c = self.peek()
if c in {' ', '\t', '\r'}:
let wsStart = self.currentPos()
var ws = ""
while self.hasMore() and self.peek() in {' ', '\t', '\r'}:
ws.add(self.advance())
self.emitToken(tkWhitespace, ws, wsStart)
elif c == '\n':
self.emitToken(tkNewline, $self.advance(), startPos)
elif self.dialect != cdCss and self.peekString(2) == "//":
let comStart = self.currentPos()
discard self.advance()
discard self.advance()
self.emitToken(tkComment, "//" & self.parseLineComment(), comStart)
elif self.peekString(2) == "/*":
let comStart = self.currentPos()
discard self.advance()
discard self.advance()
let content = self.parseBlockComment("/*", "*/")
self.emitToken(tkComment, "/*" & content & "*/", comStart)
elif self.dialect == cdRust and (c == 'r' or c == 'R') and self.peek(1) in {'"', '#'}:
let rawStart = self.currentPos()
let content = self.parseRustRawString(rawStart)
self.emitToken(tkRawString, content, rawStart)
elif self.dialect == cdGo and c == '`':
let rawStart = self.currentPos()
let content = self.parseBacktickString(rawStart)
self.emitToken(tkRawString, content, rawStart)
elif c == '\'' and self.dialect == cdRust:
let chStart = self.currentPos()
discard self.advance()
if self.hasMore() and (self.peek() == '_' or isAlpha(self.peek())):
var lit = "'"
while self.hasMore() and (isAlphaNum(self.peek()) or self.peek() == '_'):
lit.add(self.advance())
self.emitToken(tkSpecial, lit, chStart)
else:
var content = "'"
var closed = false
while self.hasMore():
let cc = self.advance()
if cc == '\\' and self.hasMore():
content.add(cc)
content.add(self.advance())
elif cc == '\'':
content.add('\'')
closed = true
break
else:
content.add(cc)
if not closed:
self.recordError(esError, "Unclosed Rust character literal", ErrUnclosedString, chStart)
self.emitToken(tkString, content, chStart)
elif c == '"' or c == '\'':
let strStart = self.currentPos()
discard self.advance()
let content = $c & self.parseQuotedString(c, strStart)
self.emitToken(tkString, content, strStart)
elif isDigit(c) or (c == '.' and self.hasMore() and self.peek(1).isDigit()):
let numStart = self.currentPos()
var num = ""
while self.hasMore() and (self.peek().isDigit() or self.peek() in {'.', 'x', 'X', 'e', 'E', '+', '-', '_'}):
num.add(self.advance())
self.emitToken(tkNumber, num, numStart)
elif isAlpha(c) or c == '_':
let idStart = self.currentPos()
var ident = ""
while self.hasMore() and (isAlphaNum(self.peek()) or self.peek() in {'_', '$'}):
ident.add(self.advance())
if self.keywords.contains(ident):
self.emitToken(tkKeyword, ident, idStart)
else:
self.emitToken(tkIdentifier, ident, idStart)
elif c == '(':
self.emitBracketToken(kOpenParen, $self.advance(), startPos)
elif c == ')':
self.closeBracket(')', kCloseParen, startPos)
elif c == '[':
self.emitBracketToken(kOpenBracket, $self.advance(), startPos)
elif c == ']':
self.closeBracket(']', kCloseBracket, startPos)
elif c == '{':
self.emitBracketToken(kOpenBrace, $self.advance(), startPos)
elif c == '}':
self.closeBracket('}', kCloseBrace, startPos)
elif c in {'+', '-', '*', '/', '%', '&', '|', '^', '~', '!', '?', ':', '=', '<', '>'}:
let opStart = self.currentPos()
var op = ""
while self.hasMore() and self.peek() in {'+', '-', '*', '/', '%', '&', '|', '^', '~', '!', '?', ':', '=', '<', '>'}:
op.add(self.advance())
if op in ["=", ":=", "+=", "-=", "*=", "/="]:
self.emitToken(tkAssignment, op, opStart)
else:
self.emitToken(tkOperator, op, opStart)
elif c in {'.', ',', ';'}:
self.emitToken(tkPunctuation, $self.advance(), startPos)
elif c == '@' and self.dialect == cdCss:
let atStart = self.currentPos()
var atRule = "@"
discard self.advance()
while self.hasMore() and (isAlphaNum(self.peek()) or self.peek() == '-'):
atRule.add(self.advance())
self.emitToken(tkDirective, atRule, atStart)
else:
self.emitToken(tkSpecial, $self.advance(), startPos)
self.finishTokenizeStep(posBefore)
self.emitToken(tkEndOfFile, "", self.currentPos())
when defined(nimcheckDebug):
if self.options.debugMode:
debugLeave("CLIKE_TOKENIZER", &"Complete: {self.tokens.len} tokens")
const
goKeywords* = @[
"break", "case", "chan", "const", "continue", "default", "defer", "else",
"fallthrough", "for", "func", "go", "goto", "if", "import", "interface",
"map", "package", "range", "return", "select", "struct", "switch", "type", "var"
]
rustKeywords* = @[
"as", "async", "await", "break", "const", "continue", "crate", "dyn", "else",
"enum", "extern", "false", "fn", "for", "if", "impl", "in", "let", "loop",
"match", "mod", "move", "mut", "pub", "ref", "return", "self", "Self", "static",
"struct", "super", "trait", "true", "type", "unsafe", "use", "where", "while"
]
cssAtRules* = @["media", "import", "keyframes", "font-face", "supports", "layer"]
cKeywords* = @[
"auto", "break", "case", "char", "const", "continue", "default", "do",
"double", "else", "enum", "extern", "float", "for", "goto", "if",
"inline", "int", "long", "register", "restrict", "return", "short",
"signed", "sizeof", "static", "struct", "switch", "typedef", "union",
"unsigned", "void", "volatile", "while",
"_Bool", "_Complex", "_Imaginary", "_Alignas", "_Alignof", "_Atomic",
"_Generic", "_Noreturn", "_Static_assert", "_Thread_local", "__attribute__"
]
cppKeywords* = @[
"alignas", "alignof", "and", "and_eq", "asm", "auto", "bitand", "bitor",
"bool", "break", "case", "catch", "char", "char8_t", "char16_t", "char32_t",
"class", "compl", "concept", "const", "const_cast", "consteval", "constexpr",
"constinit", "continue", "co_await", "co_return", "co_yield", "decltype",
"default", "delete", "do", "double", "dynamic_cast", "else", "enum",
"explicit", "export", "extern", "false", "float", "for", "friend", "goto",
"if", "inline", "int", "long", "mutable", "namespace", "new", "noexcept",
"not", "not_eq", "nullptr", "operator", "or", "or_eq", "override",
"private", "protected", "public", "reflexpr", "register", "reinterpret_cast",
"requires", "return", "short", "signed", "sizeof", "static", "static_assert",
"static_cast", "struct", "switch", "template", "this", "throw", "true",
"try", "typedef", "typeid", "typename", "union", "unsigned", "using",
"virtual", "void", "volatile", "wchar_t", "while", "xor", "xor_eq"
]
javaKeywords* = @[
"abstract", "assert", "boolean", "break", "byte", "case", "catch", "char",
"class", "const", "continue", "default", "do", "double", "else", "enum",
"extends", "false", "final", "finally", "float", "for", "goto", "if",
"implements", "import", "instanceof", "int", "interface", "long", "native",
"new", "null", "package", "private", "protected", "public", "return",
"short", "static", "strictfp", "super", "switch", "synchronized", "this",
"throw", "throws", "transient", "true", "try", "var", "void", "volatile",
"while", "yield", "module", "requires", "exports", "opens", "to", "with",
"provides", "uses", "open", "transitive", "record", "sealed", "permits",
"non-sealed"
]
csharpKeywords* = @[
"abstract", "as", "base", "bool", "break", "byte", "case", "catch", "char",
"checked", "class", "const", "continue", "decimal", "default", "delegate",
"do", "double", "else", "enum", "event", "explicit", "extern", "false",
"finally", "fixed", "float", "for", "foreach", "goto", "if", "implicit",
"in", "int", "interface", "internal", "is", "lock", "long", "namespace",
"new", "null", "object", "operator", "out", "override", "params", "private",
"protected", "public", "readonly", "ref", "return", "sbyte", "sealed",
"short", "sizeof", "stackalloc", "static", "string", "struct", "switch",
"this", "throw", "true", "try", "typeof", "uint", "ulong", "unchecked",
"unsafe", "ushort", "using", "value", "var", "virtual", "void", "volatile",
"while", "async", "await", "dynamic", "from", "get", "global", "group",
"init", "join", "let", "managed", "nameof", "nint", "notnull", "nuint",
"on", "orderby", "partial", "record", "remove", "select", "set",
"unmanaged", "when", "where", "yield", "file"
]
proc newGoTokenizer*(source: string, options: ValidationOptions = newValidationOptions()): CLikeTokenizer =
newCLikeTokenizer(source, cdGo, goKeywords, options)
proc newRustTokenizer*(source: string, options: ValidationOptions = newValidationOptions()): CLikeTokenizer =
newCLikeTokenizer(source, cdRust, rustKeywords, options)
proc newCSSTokenizer*(source: string, options: ValidationOptions = newValidationOptions()): CLikeTokenizer =
newCLikeTokenizer(source, cdCss, @[], options)
proc newCTokenizer*(source: string, options: ValidationOptions = newValidationOptions()): CLikeTokenizer =
newCLikeTokenizer(source, cdC, cKeywords, options)
proc newCppTokenizer*(source: string, options: ValidationOptions = newValidationOptions()): CLikeTokenizer =
newCLikeTokenizer(source, cdCpp, cppKeywords, options)
proc newJavaTokenizer*(source: string, options: ValidationOptions = newValidationOptions()): CLikeTokenizer =
newCLikeTokenizer(source, cdJava, javaKeywords, options)
proc newCSharpTokenizer*(source: string, options: ValidationOptions = newValidationOptions()): CLikeTokenizer =
newCLikeTokenizer(source, cdCSharp, csharpKeywords, options)
# ---------------------------------------------------------------------------
# Ruby tokenizer
# ---------------------------------------------------------------------------
type
RubyTokenizer* = ref object of TokenizerBase
const
rubyKeywords* = @[
"BEGIN", "END", "alias", "and", "begin", "break", "case", "class", "def",
"defined?", "do", "else", "elsif", "end", "ensure", "false", "for", "if",
"in", "module", "next", "nil", "not", "or", "redo", "rescue", "retry",
"return", "self", "super", "then", "true", "undef", "unless", "until",
"when", "while", "yield"
]
rubyKeywordSet* = rubyKeywords.toHashSet()
proc newRubyTokenizer*(source: string, options: ValidationOptions = newValidationOptions()): RubyTokenizer =
RubyTokenizer(
source: source, sourceLen: source.len, options: options,
tokens: @[], pos: 0, line: 1, column: 1, bracketStack: @[], errors: @[], eof: false
)
method tokenize*(self: RubyTokenizer) =
when defined(nimcheckDebug):
if self.options.debugMode: debugEnter("RUBY_TOKENIZER", "Starting Ruby tokenization")
while self.hasMore():
let posBefore = self.pos
let startPos = self.currentPos()
let c = self.peek()
if c in {' ', '\t', '\r'}:
let wsStart = self.currentPos()
var ws = ""
while self.hasMore() and self.peek() in {' ', '\t', '\r'}:
ws.add(self.advance())
self.emitToken(tkWhitespace, ws, wsStart)
elif c == '\n':
self.emitToken(tkNewline, $self.advance(), startPos)
elif c == '#':
let comStart = self.currentPos()
discard self.advance()
self.emitToken(tkComment, "#" & self.parseLineComment(), comStart)
elif c == '"' or c == '\'':
let strStart = self.currentPos()
discard self.advance()
let content = $c & self.parseQuotedString(c, strStart)
self.emitToken(tkString, content, strStart)
elif isDigit(c):
let numStart = self.currentPos()
var num = ""
while self.hasMore() and (self.peek().isDigit() or self.peek() in {'.', 'x', 'e', 'E'}):
num.add(self.advance())
self.emitToken(tkNumber, num, numStart)
elif isAlpha(c) or c == '_':
let idStart = self.currentPos()
var ident = ""
while self.hasMore() and (isAlphaNum(self.peek()) or self.peek() in {'_', '?', '!'}):
ident.add(self.advance())
if rubyKeywordSet.contains(ident):
self.emitToken(tkKeyword, ident, idStart)
else:
self.emitToken(tkIdentifier, ident, idStart)
elif c == '(':
self.emitBracketToken(kOpenParen, $self.advance(), startPos)
elif c == ')':
self.closeBracket(')', kCloseParen, startPos)
elif c == '[':
self.emitBracketToken(kOpenBracket, $self.advance(), startPos)
elif c == ']':
self.closeBracket(']', kCloseBracket, startPos)
elif c == '{':
self.emitBracketToken(kOpenBrace, $self.advance(), startPos)
elif c == '}':
self.closeBracket('}', kCloseBrace, startPos)
elif c in {'+', '-', '*', '/', '%', '&', '|', '^', '~', '!', '?', ':', '=', '<', '>'}:
let opStart = self.currentPos()
var op = ""
while self.hasMore() and self.peek() in {'+', '-', '*', '/', '%', '&', '|', '^', '~', '!', '?', ':', '=', '<', '>'}:
op.add(self.advance())
if op == "=":
self.emitToken(tkAssignment, op, opStart)
else:
self.emitToken(tkOperator, op, opStart)
elif c in {'.', ',', ';'}:
self.emitToken(tkPunctuation, $self.advance(), startPos)
else:
self.emitToken(tkSpecial, $self.advance(), startPos)
self.finishTokenizeStep(posBefore)
self.emitToken(tkEndOfFile, "", self.currentPos())
when defined(nimcheckDebug):
if self.options.debugMode: debugLeave("RUBY_TOKENIZER", &"Complete: {self.tokens.len} tokens")
# ---------------------------------------------------------------------------
# SQL tokenizer
# ---------------------------------------------------------------------------
type
SQLTokenizer* = ref object of TokenizerBase
proc newSQLTokenizer*(source: string, options: ValidationOptions = newValidationOptions()): SQLTokenizer =
SQLTokenizer(
source: source, sourceLen: source.len, options: options,
tokens: @[], pos: 0, line: 1, column: 1, bracketStack: @[], errors: @[], eof: false
)
method tokenize*(self: SQLTokenizer) =
when defined(nimcheckDebug):
if self.options.debugMode: debugEnter("SQL_TOKENIZER", "Starting SQL tokenization")
while self.hasMore():
let posBefore = self.pos
let startPos = self.currentPos()
let c = self.peek()
if c in {' ', '\t', '\r'}:
let wsStart = self.currentPos()
var ws = ""
while self.hasMore() and self.peek() in {' ', '\t', '\r'}:
ws.add(self.advance())
self.emitToken(tkWhitespace, ws, wsStart)
elif c == '\n':
self.emitToken(tkNewline, $self.advance(), startPos)
elif self.peekString(2) == "--":
let comStart = self.currentPos()
discard self.advance()
discard self.advance()
self.emitToken(tkComment, "--" & self.parseLineComment(), comStart)
elif self.peekString(2) == "/*":
let comStart = self.currentPos()
discard self.advance()
discard self.advance()
let content = self.parseBlockComment("/*", "*/")
self.emitToken(tkComment, "/*" & content & "*/", comStart)
elif c == '\'':
let strStart = self.currentPos()
discard self.advance()
var content = "'"
var closed = false
while self.hasMore():
let sc = self.advance()
if sc == '\'' and self.hasMore() and self.peek() == '\'':
content.add(sc)
content.add(self.advance())
elif sc == '\'':
content.add('\'')
self.emitToken(tkString, content, strStart)
closed = true
break
else:
content.add(sc)
if not closed:
self.recordError(esError, "Unclosed SQL string literal", ErrUnclosedString, strStart)
self.emitToken(tkString, content, strStart)
elif c == '"':
let strStart = self.currentPos()
discard self.advance()
let content = "\"" & self.parseQuotedString('"', strStart)
self.emitToken(tkString, content, strStart)
elif isDigit(c) or (c == '.' and self.hasMore() and self.peek(1).isDigit()):
let numStart = self.currentPos()
var num = ""
while self.hasMore() and (self.peek().isDigit() or self.peek() in {'.', 'e', 'E', '+', '-'}):
num.add(self.advance())
self.emitToken(tkNumber, num, numStart)
elif isAlpha(c) or c == '_':
let idStart = self.currentPos()
var ident = ""
while self.hasMore() and (isAlphaNum(self.peek()) or self.peek() in {'_', '$'}):
ident.add(self.advance())
self.emitToken(tkIdentifier, ident, idStart)
elif c == '(':
self.emitBracketToken(kOpenParen, $self.advance(), startPos)
elif c == ')':
self.closeBracket(')', kCloseParen, startPos)
elif c == ';':
self.emitToken(tkPunctuation, $self.advance(), startPos)
elif c in {'+', '-', '*', '/', '%', '=', '<', '>', '!', '|', '&', ','}:
self.emitToken(tkOperator, $self.advance(), startPos)
else:
self.emitToken(tkSpecial, $self.advance(), startPos)
self.finishTokenizeStep(posBefore)
self.emitToken(tkEndOfFile, "", self.currentPos())
when defined(nimcheckDebug):
if self.options.debugMode: debugLeave("SQL_TOKENIZER", &"Complete: {self.tokens.len} tokens")
# ---------------------------------------------------------------------------
# Markdown tokenizer
# ---------------------------------------------------------------------------
type
MarkdownTokenizer* = ref object of TokenizerBase
fenceStack*: seq[string]
proc newMarkdownTokenizer*(source: string, options: ValidationOptions = newValidationOptions()): MarkdownTokenizer =
MarkdownTokenizer(
source: source, sourceLen: source.len, options: options,
tokens: @[], pos: 0, line: 1, column: 1, bracketStack: @[], errors: @[], eof: false,
fenceStack: @[]
)
method tokenize*(self: MarkdownTokenizer) =
when defined(nimcheckDebug):
if self.options.debugMode: debugEnter("MD_TOKENIZER", "Starting Markdown tokenization")
var lineNum = 1
for line in self.source.splitLines(keepEol = true):
let lineStart = self.currentPos()
let stripped = line.strip()
if stripped.startsWith("```"):
let fence = stripped
if self.fenceStack.len > 0 and self.fenceStack[^1] == "```":
discard self.fenceStack.pop()
self.emitToken(tkKeyword, fence, lineStart)
else:
self.fenceStack.add("```")
self.emitToken(tkKeyword, fence, lineStart)
elif stripped.startsWith("~~~"):
let fence = stripped
if self.fenceStack.len > 0 and self.fenceStack[^1] == "~~~":
discard self.fenceStack.pop()
self.emitToken(tkKeyword, fence, lineStart)
else:
self.fenceStack.add("~~~")
self.emitToken(tkKeyword, fence, lineStart)
elif stripped.len > 0 and stripped[0] == '#':
self.emitToken(tkDirective, stripped, lineStart)
elif '[' in line and "](" in line:
var idx = 0
while idx < line.len:
if line[idx] == '[':
let linkStart = idx
let closeBracket = line.find(']', idx + 1)
if closeBracket >= 0 and closeBracket + 1 < line.len and line[closeBracket + 1] == '(':
let closeParen = line.find(')', closeBracket + 2)
if closeParen < 0:
self.recordError(esError, &"Unclosed Markdown link on line {lineNum}",
ErrUnclosedBlock, newSourcePosition(lineNum, linkStart + 1, 0),
hint = "Add a closing ')' to complete the link")
idx = if closeParen >= 0: closeParen + 1 else: line.len
else:
idx += 1
else:
idx += 1
self.emitToken(tkTemplateTag, line, lineStart)
else:
var backtickCount = 0
for c in line:
if c == '`': backtickCount += 1
if backtickCount mod 2 != 0:
self.recordError(esWarning, &"Unbalanced inline code backticks on line {lineNum}",
WarnInconsistentNaming, newSourcePosition(lineNum, 1, 0))
self.emitToken(tkTemplateTag, line, lineStart)
self.pos += line.len
self.line += 1
self.column = 1
lineNum += 1
if self.fenceStack.len > 0:
self.recordError(esError, "Unclosed fenced code block",
ErrUnclosedBlock, newSourcePosition(lineNum, 1, 0),
hint = "Add a closing ``` or ~~~ fence")
self.emitToken(tkEndOfFile, "", self.currentPos())
when defined(nimcheckDebug):
if self.options.debugMode: debugLeave("MD_TOKENIZER", &"Complete: {self.tokens.len} tokens")
# ---------------------------------------------------------------------------
# Dockerfile tokenizer
# ---------------------------------------------------------------------------
type
DockerfileTokenizer* = ref object of TokenizerBase
const
dockerInstructions* = @[
"FROM", "RUN", "CMD", "LABEL", "EXPOSE", "ENV", "ADD", "COPY",
"ENTRYPOINT", "VOLUME", "USER", "WORKDIR", "ARG", "ONBUILD",
"STOPSIGNAL", "HEALTHCHECK", "SHELL"
].toHashSet()
proc newDockerfileTokenizer*(source: string, options: ValidationOptions = newValidationOptions()): DockerfileTokenizer =
DockerfileTokenizer(
source: source, sourceLen: source.len, options: options,
tokens: @[], pos: 0, line: 1, column: 1, bracketStack: @[], errors: @[], eof: false
)
method tokenize*(self: DockerfileTokenizer) =
when defined(nimcheckDebug):
if self.options.debugMode: debugEnter("DOCKER_TOKENIZER", "Starting Dockerfile tokenization")
var lineNum = 1
var lines = self.source.splitLines()
var i = 0
while i < lines.len:
var line = lines[i]
let lineStart = newSourcePosition(lineNum, 1, 0)
while line.len > 0 and line[^1] == '\\' and i + 1 < lines.len:
line = line[0..^2] & " " & lines[i + 1].strip()
i += 1
lineNum += 1
let stripped = line.strip()
if stripped.len == 0 or stripped[0] == '#':
if stripped.len > 0:
self.emitToken(tkComment, stripped, lineStart)
i += 1
lineNum += 1
continue
let spaceIdx = stripped.find(' ')
let instruction = if spaceIdx >= 0: stripped[0..<spaceIdx].toUpperAscii() else: stripped.toUpperAscii()
if dockerInstructions.contains(instruction):
self.emitToken(tkDirective, instruction, lineStart)
let args = if spaceIdx >= 0: stripped[spaceIdx + 1 .. ^1] else: ""
if args.len > 0:
self.emitToken(tkString, args, lineStart)
for ch in args:
if ch == '[':
self.bracketStack.add(('[', lineStart))
elif ch == ']':
if self.bracketStack.len > 0 and self.bracketStack[^1][0] == '[':
discard self.bracketStack.pop()
else:
self.recordError(esError, &"Unexpected ']' in Dockerfile instruction '{instruction}'",
ErrMismatchedBracket, lineStart)
elif ch == '"':
discard
let openQuotes = args.count('"') - args.count("\\\"")
if openQuotes mod 2 != 0:
self.recordError(esError, &"Unclosed double-quoted string in {instruction}",
ErrUnclosedString, lineStart, hint = "Add a closing '\"'")
else:
self.recordError(esWarning, &"Unknown Dockerfile instruction: {instruction}",
WarnInconsistentNaming, lineStart)
i += 1
lineNum += 1
for (_, pos) in self.bracketStack:
self.recordError(esError, "Unclosed '[' in Dockerfile JSON array",
ErrUnclosedBracket, pos, hint = "Add a closing ']'")
self.pos = self.sourceLen
self.line = lineNum
self.emitToken(tkEndOfFile, "", self.currentPos())
when defined(nimcheckDebug):
if self.options.debugMode: debugLeave("DOCKER_TOKENIZER", &"Complete: {self.tokens.len} tokens")
# ---------------------------------------------------------------------------
# Makefile tokenizer
# ---------------------------------------------------------------------------
type
MakefileTokenizer* = ref object of TokenizerBase
proc newMakefileTokenizer*(source: string, options: ValidationOptions = newValidationOptions()): MakefileTokenizer =
MakefileTokenizer(
source: source, sourceLen: source.len, options: options,
tokens: @[], pos: 0, line: 1, column: 1, bracketStack: @[], errors: @[], eof: false
)
method tokenize*(self: MakefileTokenizer) =
when defined(nimcheckDebug):
if self.options.debugMode: debugEnter("MAKEFILE_TOKENIZER", "Starting Makefile tokenization")
var lineNum = 1
for line in self.source.splitLines():
let lineStart = newSourcePosition(lineNum, 1, 0)
if line.len == 0:
self.emitToken(tkNewline, "", lineStart)
elif line[0] == '#':
self.emitToken(tkComment, line, lineStart)
elif line[0] == '\t' or (line.len >= 2 and line[0] == '\\' and line[1] == 't'):
let recipe = if line[0] == '\t': line else: '\t' & line[2..^1]
self.emitToken(tkSpecial, recipe, lineStart)
var parenDepth = 0
var inQuote = false
for i, c in recipe:
if c == '"' and (i == 0 or recipe[i - 1] != '\\'):
inQuote = not inQuote
elif c == '(' and not inQuote:
parenDepth += 1
self.bracketStack.add(('(', newSourcePosition(lineNum, i + 1, 0)))
elif c == ')' and not inQuote:
if parenDepth > 0 and self.bracketStack.len > 0:
parenDepth -= 1
discard self.bracketStack.pop()
else:
self.recordError(esError, &"Unexpected ')' in Makefile recipe on line {lineNum}",
ErrMismatchedBracket, newSourcePosition(lineNum, i + 1, 0))
if inQuote:
self.recordError(esError, &"Unclosed double-quoted string in Makefile recipe on line {lineNum}",
ErrUnclosedString, lineStart, hint = "Add a closing '\"'")
elif ':' in line and not line.strip().startsWith("#"):
let colonIdx = line.find(':')
let target = line[0..<colonIdx].strip()
self.emitToken(tkIdentifier, target, lineStart)
if colonIdx + 1 < line.len:
let deps = line[colonIdx + 1 .. ^1].strip()
if deps.len > 0:
self.emitToken(tkString, deps, lineStart)
elif line.strip().len > 0 and line[0] notin {'\t', '#'}:
if line.contains('='):
self.emitToken(tkAssignment, line.strip(), lineStart)
elif line.strip().startsWith(".PHONY") or line.strip().startsWith(".SUFFIXES"):
self.emitToken(tkDirective, line.strip(), lineStart)
else:
self.recordError(esWarning, &"Makefile line {lineNum} should start with tab for recipes",
"W1001", lineStart, hint = "Recipe commands must be indented with a tab character")
lineNum += 1
for (_, pos) in self.bracketStack:
self.recordError(esError, "Unclosed '(' in Makefile variable expansion",
ErrUnclosedBracket, pos, hint = "Add a closing ')'")
self.pos = self.sourceLen
self.line = lineNum
self.emitToken(tkEndOfFile, "", self.currentPos())
when defined(nimcheckDebug):
if self.options.debugMode: debugLeave("MAKEFILE_TOKENIZER", &"Complete: {self.tokens.len} tokens")

16
src/nimcheck/tokenizers/javascript_tokenizer.nim
View File

@ -8,7 +8,7 @@ type
JavaScriptTokenizer* = ref object of TokenizerBase
const
jsKeywords = @[
jsKeywords* = @[
"async", "await", "break", "case", "catch", "class", "const", "continue",
"debugger", "default", "delete", "do", "else", "enum", "export", "extends",
"false", "finally", "for", "function", "if", "import", "in", "instanceof",
@ -19,7 +19,7 @@ const
"as", "any", "boolean", "number", "string", "undefined", "never",
"keyof", "infer", "unknown"
]
jsKeywordSet = jsKeywords.toHashSet()
jsKeywordSet* = jsKeywords.toHashSet()
proc newJavaScriptTokenizer*(source: string, options: ValidationOptions = newValidationOptions()): JavaScriptTokenizer =
result = JavaScriptTokenizer(
@ -178,19 +178,13 @@ method tokenize*(self: JavaScriptTokenizer) =
# Brackets
elif c == '(': self.emitBracketToken(kOpenParen, $self.advance(), startPos)
elif c == ')':
self.emitBracketToken(kCloseParen, $self.advance(), startPos)
if self.bracketStack.len > 0 and self.bracketStack[^1][0] == '(':
discard self.bracketStack.pop()
self.closeBracket(')', kCloseParen, startPos)
elif c == '[': self.emitBracketToken(kOpenBracket, $self.advance(), startPos)
elif c == ']':
self.emitBracketToken(kCloseBracket, $self.advance(), startPos)
if self.bracketStack.len > 0 and self.bracketStack[^1][0] == '[':
discard self.bracketStack.pop()
self.closeBracket(']', kCloseBracket, startPos)
elif c == '{': self.emitBracketToken(kOpenBrace, $self.advance(), startPos)
elif c == '}':
self.emitBracketToken(kCloseBrace, $self.advance(), startPos)
if self.bracketStack.len > 0 and self.bracketStack[^1][0] == '{':
discard self.bracketStack.pop()
self.closeBracket('}', kCloseBrace, startPos)
# Operators
elif c in {'+', '-', '*', '/', '%', '^', '~', '|', '&', '<', '>', '=', '!', '?'}:

6
src/nimcheck/tokenizers/jinja_tokenizer.nim
View File

@ -42,6 +42,7 @@ method tokenize*(self: JinjaTokenizer) =
debugEnter("JINJA_TOKENIZER", "Starting Jinja tokenization")
while self.hasMore():
let posBefore = self.pos
let startPos = self.currentPos()
let c = self.peek()
@ -116,6 +117,11 @@ method tokenize*(self: JinjaTokenizer) =
text.add(self.advance())
if text.len > 0:
self.emitToken(tkTemplateTag, text, startPos)
else:
# Lone '{' or other delimiter — must always advance (prevents infinite loop)
self.emitToken(tkSpecial, $self.advance(), startPos)
self.finishTokenizeStep(posBefore)
self.emitToken(tkEndOfFile, "", self.currentPos())
when defined(nimcheckDebug):

447
src/nimcheck/tokenizers/lang_tokenizers.nim Normal file
View File

@ -0,0 +1,447 @@
## Language tokenizers: Kotlin, Lua, Swift.
##
## Standalone tokenizers for languages that do not fit the
## C-like or template-based tokenization patterns.
import std/[strformat, sets]
{.warning[UnusedImport]:off.}
import ../core/types, ../core/tokenizerbase
import extended_tokenizers
# ---------------------------------------------------------------------------
# Kotlin tokenizer
# ---------------------------------------------------------------------------
type
KotlinTokenizer* = ref object of TokenizerBase
const
kotlinKeywords* = @[
"abstract", "actual", "annotation", "as", "as?", "break", "by", "catch",
"class", "companion", "const", "constructor", "continue", "crossinline",
"data", "delegate", "do", "dynamic", "else", "enum", "expect", "external",
"false", "field", "file", "final", "finally", "for", "fun", "if", "import",
"in", "infix", "init", "inline", "inner", "interface", "internal", "is",
"it", "lateinit", "noinline", "null", "object", "open", "operator", "out",
"override", "package", "param", "private", "property", "protected", "public",
"receiver", "reified", "return", "sealed", "set", "setparam", "super",
"suspend", "tailrec", "this", "throw", "true", "try", "typealias",
"typeof", "val", "var", "vararg", "when", "where", "while"
]
kotlinKeywordSet* = kotlinKeywords.toHashSet()
proc newKotlinTokenizer*(source: string, options: ValidationOptions = newValidationOptions()): KotlinTokenizer =
KotlinTokenizer(
source: source, sourceLen: source.len, options: options,
tokens: @[], pos: 0, line: 1, column: 1, bracketStack: @[], errors: @[], eof: false
)
method tokenize*(self: KotlinTokenizer) =
when defined(nimcheckDebug):
if self.options.debugMode: debugEnter("KOTLIN_TOKENIZER", "Starting Kotlin tokenization")
while self.hasMore():
let posBefore = self.pos
let startPos = self.currentPos()
let c = self.peek()
if c in {' ', '\t', '\r'}:
let wsStart = self.currentPos()
var ws = ""
while self.hasMore() and self.peek() in {' ', '\t', '\r'}:
ws.add(self.advance())
self.emitToken(tkWhitespace, ws, wsStart)
elif c == '\n':
self.emitToken(tkNewline, $self.advance(), startPos)
elif self.peekString(2) == "//":
let comStart = self.currentPos()
discard self.advance()
discard self.advance()
self.emitToken(tkComment, "//" & self.parseLineComment(), comStart)
elif self.peekString(2) == "/*":
let comStart = self.currentPos()
discard self.advance()
discard self.advance()
let content = self.parseBlockComment("/*", "*/")
let isDoc = content.startsWith("*")
self.emitToken(if isDoc: tkDocComment else: tkComment, "/*" & content & "*/", comStart)
elif c == '"':
let strStart = self.currentPos()
discard self.advance()
let content = '"' & self.parseQuotedString('"', strStart)
self.emitToken(tkString, content, strStart)
elif c == '\'':
let strStart = self.currentPos()
discard self.advance()
let content = "'" & self.parseQuotedString('\'', strStart)
self.emitToken(tkString, content, strStart)
elif c == '`':
# Kotlin backtick identifier
let btStart = self.currentPos()
discard self.advance()
var ident = "`"
while self.hasMore() and self.peek() != '`':
ident.add(self.advance())
if self.hasMore():
ident.add(self.advance())
self.emitToken(tkIdentifier, ident, btStart)
elif isDigit(c) or (c == '.' and self.hasMore() and self.peek(1).isDigit()):
let numStart = self.currentPos()
var num = ""
while self.hasMore() and (self.peek().isDigit() or self.peek() in {'.', 'x', 'X', 'e', 'E', '+', '-', '_', 'L', 'l', 'F', 'f'}):
num.add(self.advance())
self.emitToken(tkNumber, num, numStart)
elif isAlpha(c) or c == '_':
let idStart = self.currentPos()
var ident = ""
while self.hasMore() and (isAlphaNum(self.peek()) or self.peek() == '_'):
ident.add(self.advance())
if kotlinKeywordSet.contains(ident):
self.emitToken(tkKeyword, ident, idStart)
else:
self.emitToken(tkIdentifier, ident, idStart)
elif c == '(':
self.emitBracketToken(kOpenParen, $self.advance(), startPos)
elif c == ')':
self.closeBracket(')', kCloseParen, startPos)
elif c == '[':
self.emitBracketToken(kOpenBracket, $self.advance(), startPos)
elif c == ']':
self.closeBracket(']', kCloseBracket, startPos)
elif c == '{':
self.emitBracketToken(kOpenBrace, $self.advance(), startPos)
elif c == '}':
self.closeBracket('}', kCloseBrace, startPos)
elif c in {'+', '-', '*', '/', '%', '&', '|', '^', '~', '!', '?', ':', '=', '<', '>', '@'}:
let opStart = self.currentPos()
var op = ""
while self.hasMore() and self.peek() in {'+', '-', '*', '/', '%', '&', '|', '^', '~', '!', '?', ':', '=', '<', '>', '@'}:
op.add(self.advance())
if op in ["=", "+=", "-=", "*=", "/=", "%="]:
self.emitToken(tkAssignment, op, opStart)
else:
self.emitToken(tkOperator, op, opStart)
elif c in {'.', ',', ';'}:
self.emitToken(tkPunctuation, $self.advance(), startPos)
elif c == '#':
self.emitToken(tkDirective, $self.advance(), startPos)
else:
self.emitToken(tkSpecial, $self.advance(), startPos)
self.finishTokenizeStep(posBefore)
self.emitToken(tkEndOfFile, "", self.currentPos())
when defined(nimcheckDebug):
if self.options.debugMode: debugLeave("KOTLIN_TOKENIZER", &"Complete: {self.tokens.len} tokens")
# ---------------------------------------------------------------------------
# Lua tokenizer
# ---------------------------------------------------------------------------
type
LuaTokenizer* = ref object of TokenizerBase
const
luaKeywords* = @[
"and", "break", "do", "else", "elseif", "end", "false", "for", "function",
"goto", "if", "in", "local", "nil", "not", "or", "repeat", "return",
"then", "true", "until", "while"
]
luaKeywordSet* = luaKeywords.toHashSet()
proc newLuaTokenizer*(source: string, options: ValidationOptions = newValidationOptions()): LuaTokenizer =
LuaTokenizer(
source: source, sourceLen: source.len, options: options,
tokens: @[], pos: 0, line: 1, column: 1, bracketStack: @[], errors: @[], eof: false
)
method tokenize*(self: LuaTokenizer) =
when defined(nimcheckDebug):
if self.options.debugMode: debugEnter("LUA_TOKENIZER", "Starting Lua tokenization")
while self.hasMore():
let posBefore = self.pos
let startPos = self.currentPos()
let c = self.peek()
if c in {' ', '\t', '\r'}:
let wsStart = self.currentPos()
var ws = ""
while self.hasMore() and self.peek() in {' ', '\t', '\r'}:
ws.add(self.advance())
self.emitToken(tkWhitespace, ws, wsStart)
elif c == '\n':
self.emitToken(tkNewline, $self.advance(), startPos)
elif self.peekString(2) == "--":
let comStart = self.currentPos()
discard self.advance()
discard self.advance()
if self.peekString(2) == "[[":
# Long block comment --[[ ... ]]
discard self.advance()
discard self.advance()
let content = self.parseBlockComment("[[", "]]")
self.emitToken(tkComment, "--[[" & content & "]]", comStart)
else:
self.emitToken(tkComment, "--" & self.parseLineComment(), comStart)
elif c == '"' or c == '\'':
let strStart = self.currentPos()
discard self.advance()
let content = $c & self.parseQuotedString(c, strStart)
self.emitToken(tkString, content, strStart)
elif c == '[' and self.peekString(2) == "[[":
# Long string literal [[ ... ]]
let lsStart = self.currentPos()
discard self.advance()
discard self.advance()
var content = "[["
var foundClosing = false
while self.hasMore():
if self.peekString(2) == "]]":
content.add("]]")
discard self.advance()
discard self.advance()
foundClosing = true
break
content.add(self.advance())
if not foundClosing:
self.recordError(esError, "Unclosed long string literal", ErrUnclosedString, lsStart,
hint = "Add closing ']]'")
self.emitToken(tkString, content, lsStart)
elif isDigit(c) or (c == '.' and self.hasMore() and self.peek(1).isDigit()):
let numStart = self.currentPos()
var num = ""
while self.hasMore() and (self.peek().isDigit() or self.peek() in {'.', 'e', 'E', '+', '-', 'x', 'X'}):
num.add(self.advance())
self.emitToken(tkNumber, num, numStart)
elif isAlpha(c) or c == '_':
let idStart = self.currentPos()
var ident = ""
while self.hasMore() and (isAlphaNum(self.peek()) or self.peek() == '_'):
ident.add(self.advance())
if luaKeywordSet.contains(ident):
self.emitToken(tkKeyword, ident, idStart)
else:
self.emitToken(tkIdentifier, ident, idStart)
elif c == '(':
self.emitBracketToken(kOpenParen, $self.advance(), startPos)
elif c == ')':
self.closeBracket(')', kCloseParen, startPos)
elif c == '[':
self.emitBracketToken(kOpenBracket, $self.advance(), startPos)
elif c == ']':
self.closeBracket(']', kCloseBracket, startPos)
elif c == '{':
self.emitBracketToken(kOpenBrace, $self.advance(), startPos)
elif c == '}':
self.closeBracket('}', kCloseBrace, startPos)
elif c in {'+', '-', '*', '/', '%', '^', '#', '=', '<', '>', '~', ':', '&', '|', '!'}:
let opStart = self.currentPos()
var op = ""
while self.hasMore() and self.peek() in {'+', '-', '*', '/', '%', '^', '#', '=', '<', '>', '~', ':', '&', '|', '!'}:
op.add(self.advance())
if op == "=":
self.emitToken(tkAssignment, op, opStart)
else:
self.emitToken(tkOperator, op, opStart)
elif c in {'.', ',', ';'}:
self.emitToken(tkPunctuation, $self.advance(), startPos)
elif c == '@':
self.emitToken(tkSpecial, $self.advance(), startPos)
else:
self.emitToken(tkSpecial, $self.advance(), startPos)
self.finishTokenizeStep(posBefore)
self.emitToken(tkEndOfFile, "", self.currentPos())
when defined(nimcheckDebug):
if self.options.debugMode: debugLeave("LUA_TOKENIZER", &"Complete: {self.tokens.len} tokens")
# ---------------------------------------------------------------------------
# Swift tokenizer
# ---------------------------------------------------------------------------
type
SwiftTokenizer* = ref object of TokenizerBase
const
swiftKeywords* = @[
"Any", "as", "associatedtype", "async", "await", "break", "case", "catch",
"class", "continue", "default", "defer", "deinit", "do", "else", "enum",
"extension", "fallthrough", "false", "fileprivate", "for", "func", "guard",
"if", "import", "in", "indirect", "infix", "init", "inout", "internal",
"is", "lazy", "let", "nil", "nonisolated", "open", "operator", "optional",
"override", "package", "postfix", "precedencegroup", "prefix", "private",
"protocol", "public", "rethrows", "return", "self", "Self", "static",
"struct", "subscript", "super", "switch", "throw", "throws", "true",
"try", "Type", "typealias", "unowned", "var", "weak", "where", "while",
"some", "any", "macro", "actor", "isolated", "distributed", "consuming",
"borrowing", "nonconsuming", "_borrowing"
]
swiftKeywordSet* = swiftKeywords.toHashSet()
proc newSwiftTokenizer*(source: string, options: ValidationOptions = newValidationOptions()): SwiftTokenizer =
SwiftTokenizer(
source: source, sourceLen: source.len, options: options,
tokens: @[], pos: 0, line: 1, column: 1, bracketStack: @[], errors: @[], eof: false
)
method tokenize*(self: SwiftTokenizer) =
when defined(nimcheckDebug):
if self.options.debugMode: debugEnter("SWIFT_TOKENIZER", "Starting Swift tokenization")
while self.hasMore():
let posBefore = self.pos
let startPos = self.currentPos()
let c = self.peek()
if c in {' ', '\t', '\r'}:
let wsStart = self.currentPos()
var ws = ""
while self.hasMore() and self.peek() in {' ', '\t', '\r'}:
ws.add(self.advance())
self.emitToken(tkWhitespace, ws, wsStart)
elif c == '\n':
self.emitToken(tkNewline, $self.advance(), startPos)
elif self.peekString(2) == "//":
let comStart = self.currentPos()
discard self.advance()
discard self.advance()
self.emitToken(tkComment, "//" & self.parseLineComment(), comStart)
elif self.peekString(2) == "/*":
let comStart = self.currentPos()
discard self.advance()
discard self.advance()
if self.hasMore() and self.peek() == '*':
# Doc comment /** ... */
discard self.advance()
let content = self.parseBlockComment("*", "*/")
self.emitToken(tkDocComment, "/**" & content & "*/", comStart)
else:
let content = self.parseBlockComment("/*", "*/")
self.emitToken(tkComment, "/*" & content & "*/", comStart)
elif c == '"':
let strStart = self.currentPos()
discard self.advance()
# Check for multiline string literal """"""
if self.peekString(2) == "\"\"\"...":
discard
var content = "\""
var foundClosing = false
while self.hasMore():
let sc = self.advance()
if sc == '\\':
content.add(sc)
if self.hasMore(): content.add(self.advance())
elif sc == '"':
content.add('"')
foundClosing = true
break
elif sc == '\n':
self.recordError(esError, "Newline in Swift string literal", ErrUnclosedString, strStart,
hint = "Use triple-quoted string \"\"\" for multiline strings")
break
else:
content.add(sc)
if not foundClosing:
self.recordError(esError, "Unclosed Swift string literal", ErrUnclosedString, strStart)
self.emitToken(tkString, content, strStart)
elif c == '`':
# Swift backtick-escaped identifier
let btStart = self.currentPos()
discard self.advance()
var ident = "`"
while self.hasMore() and self.peek() != '`':
ident.add(self.advance())
if self.hasMore():
ident.add(self.advance())
self.emitToken(tkIdentifier, ident, btStart)
elif isDigit(c) or (c == '.' and self.hasMore() and self.peek(1).isDigit()):
let numStart = self.currentPos()
var num = ""
while self.hasMore() and (self.peek().isDigit() or self.peek() in {'.', 'x', 'X', 'o', 'O', 'b', 'B', 'e', 'E', '+', '-', '_'}):
num.add(self.advance())
self.emitToken(tkNumber, num, numStart)
elif isAlpha(c) or c == '_':
let idStart = self.currentPos()
var ident = ""
while self.hasMore() and (isAlphaNum(self.peek()) or self.peek() in {'_', '$'}):
ident.add(self.advance())
if swiftKeywordSet.contains(ident):
self.emitToken(tkKeyword, ident, idStart)
else:
self.emitToken(tkIdentifier, ident, idStart)
elif c == '(':
self.emitBracketToken(kOpenParen, $self.advance(), startPos)
elif c == ')':
self.closeBracket(')', kCloseParen, startPos)
elif c == '[':
self.emitBracketToken(kOpenBracket, $self.advance(), startPos)
elif c == ']':
self.closeBracket(']', kCloseBracket, startPos)
elif c == '{':
self.emitBracketToken(kOpenBrace, $self.advance(), startPos)
elif c == '}':
self.closeBracket('}', kCloseBrace, startPos)
elif c in {'+', '-', '*', '/', '%', '&', '|', '^', '~', '!', '?', ':', '=', '<', '>', '.'}:
let opStart = self.currentPos()
var op = ""
while self.hasMore() and self.peek() in {'+', '-', '*', '/', '%', '&', '|', '^', '~', '!', '?', ':', '=', '<', '>', '.'}:
op.add(self.advance())
if op == "=":
self.emitToken(tkAssignment, op, opStart)
else:
self.emitToken(tkOperator, op, opStart)
elif c in {',', ';', ':'}:
self.emitToken(tkPunctuation, $self.advance(), startPos)
elif c == '#':
let hashStart = self.currentPos()
var directive = "#"
discard self.advance()
while self.hasMore() and (isAlphaNum(self.peek()) or self.peek() == '_'):
directive.add(self.advance())
self.emitToken(tkDirective, directive, hashStart)
else:
self.emitToken(tkSpecial, $self.advance(), startPos)
self.finishTokenizeStep(posBefore)
self.emitToken(tkEndOfFile, "", self.currentPos())
when defined(nimcheckDebug):
if self.options.debugMode: debugLeave("SWIFT_TOKENIZER", &"Complete: {self.tokens.len} tokens")

39
src/nimcheck/tokenizers/nim_tokenizer.nim
View File

@ -184,17 +184,38 @@ method tokenize*(self: NimTokenizer) =
let chStart = self.currentPos()
discard self.advance() # '
var content = "'"
if self.hasMore():
if self.peek() == '\\':
content.add(self.advance())
if self.hasMore(): content.add(self.advance())
else:
content.add(self.advance())
if self.hasMore() and self.peek() == '\'':
content.add(self.advance())
var charCount = 0
while self.hasMore():
let cc = self.peek()
if cc == '\'':
discard self.advance()
content.add('\'')
if charCount == 0:
self.recordError(esError, "Empty character literal", ErrUnclosedString, chStart,
hint = "Character literal must contain exactly one character")
elif charCount > 1:
self.recordError(esError,
&"Character literal contains {charCount} characters instead of 1",
ErrInvalidSyntax, chStart,
hint = "Use a string for multi-character sequences")
self.emitToken(tkString, content, chStart)
else:
break
elif cc == '\n':
self.recordError(esError, "Unclosed character literal", ErrUnclosedString, chStart)
self.emitToken(tkString, content, chStart)
break
elif cc == '\\':
content.add(self.advance())
if self.hasMore():
content.add(self.advance())
charCount += 1
else:
content.add(self.advance())
charCount += 1
if not self.hasMore() and charCount > 0:
self.recordError(esError, "Unclosed character literal", ErrUnclosedString, chStart)
self.emitToken(tkString, content, chStart)
# Numbers
elif isDigit(c):

2
src/nimcheck/tokenizers/php_tokenizer.nim
View File

@ -137,9 +137,9 @@ method tokenize*(self: PHPTokenizer) =
foundClosing = true
break
elif sc == '\n':
self.recordError(esError, "Unclosed string at end of line", ErrUnclosedString, strStart)
self.emitToken(tkString, content, strStart)
foundClosing = true
foundClosing = true
break
else:
content.add(sc)

3
src/nimcheck/tokenizers/python_tokenizer.nim
View File

@ -341,7 +341,8 @@ method tokenize*(self: PythonTokenizer) =
let uStart = self.currentPos()
self.emitToken(tkError, $self.advance(), uStart)
# Add dedent tokens for remaining indentation at EOF
# Flush remaining indentation at EOF
# Trailing indent at EOF is valid Python - emit dedent tokens silently.
while self.indentStack.len > 1:
discard self.indentStack.pop()
self.emitToken(tkSpecial, "<dedent>", self.currentPos())

452
src/nimcheck/tokenizers/type_xml_tokenizer.nim Normal file
View File

@ -0,0 +1,452 @@
## Improved TypeScript and XML tokenizers.
##
## TypeScript builds on JavaScriptTokenizer with TS-specific features.
## XML provides a full XML/SGML-style tag tokenizer.
import std/[strformat, sets]
{.warning[UnusedImport]:off.}
import ../core/types, ../core/tokenizerbase
import javascript_tokenizer
# ---------------------------------------------------------------------------
# Improved TypeScript tokenizer (extends JavaScript tokenizer)
# ---------------------------------------------------------------------------
type
TypeScriptTokenizer* = ref object of JavaScriptTokenizer
const
tsExtraKeywords* = @[
"interface", "type", "declare", "module", "namespace", "abstract",
"readonly", "as", "any", "boolean", "number", "string", "undefined",
"never", "keyof", "infer", "unknown", "satisfies", "asserts",
"is", "enum", "const", "override", "private", "protected", "public",
"static", "implements", "extends", "infer", "template", "using"
]
tsExtraKeywordSet* = tsExtraKeywords.toHashSet()
proc newTypeScriptTokenizer*(source: string, options: ValidationOptions = newValidationOptions()): TypeScriptTokenizer =
TypeScriptTokenizer(
source: source, sourceLen: source.len, options: options,
tokens: @[], pos: 0, line: 1, column: 1, bracketStack: @[], errors: @[], eof: false
)
method tokenize*(self: TypeScriptTokenizer) =
when defined(nimcheckDebug):
if self.options.debugMode: debugEnter("TS_TOKENIZER", "Starting TypeScript tokenization")
while self.hasMore():
let startPos = self.currentPos()
let c = self.peek()
if c in {' ', '\t', '\r'}:
let wsStart = self.currentPos()
var ws = ""
while self.hasMore() and self.peek() in {' ', '\t', '\r'}:
ws.add(self.advance())
self.emitToken(tkWhitespace, ws, wsStart)
elif c == '\n':
self.emitToken(tkNewline, $self.advance(), startPos)
# Template literals
elif c == '`':
let tlStart = self.currentPos()
discard self.advance()
var content = "`"
var braceDepth = 0
var foundClosing = false
while self.hasMore():
let tc = self.advance()
if tc == '\\':
content.add(tc)
if self.hasMore(): content.add(self.advance())
elif tc == '`' and braceDepth == 0:
content.add('`')
self.emitToken(tkString, content, tlStart)
foundClosing = true
break
elif tc == '$' and self.hasMore() and self.peek() == '{' and braceDepth == 0:
content.add("${")
discard self.advance()
braceDepth += 1
elif tc == '{': braceDepth += 1; content.add(tc)
elif tc == '}': braceDepth -= 1; content.add(tc)
else: content.add(tc)
if not foundClosing:
self.recordError(esError, "Unclosed template literal", ErrUnclosedString, tlStart)
self.emitToken(tkString, content, tlStart)
elif self.peekString(2) == "//":
let comStart = self.currentPos()
discard self.advance()
discard self.advance()
self.emitToken(tkComment, "//" & self.parseLineComment(), comStart)
elif self.peekString(2) == "/*":
let comStart = self.currentPos()
discard self.advance()
discard self.advance()
let content = self.parseBlockComment("/*", "*/")
let isDoc = content.startsWith("*")
self.emitToken(if isDoc: tkDocComment else: tkComment, "/*" & content & "*/", comStart)
elif c == '/' and self.pos > 0:
let prevChar = if self.pos > 0: self.source[self.pos - 1] else: '\0'
if prevChar in {' ', '\t', '\n', '\r', '=', '(', '[', '{', '!', '&', '|', ':', ';', ',', '^', '~', '?'}:
let regStart = self.currentPos()
discard self.advance()
var regex = "/"
var foundClosing = false
while self.hasMore():
let rc = self.advance()
if rc == '\\':
regex.add(rc)
if self.hasMore(): regex.add(self.advance())
elif rc == '/':
regex.add('/')
while self.hasMore() and (isAlpha(self.peek()) or self.peek() in {'$', '_'}):
regex.add(self.advance())
self.emitToken(tkSpecial, regex, regStart)
foundClosing = true
break
else:
regex.add(rc)
if not foundClosing:
self.emitToken(tkOperator, "/", regStart)
else:
self.emitToken(tkOperator, "/", startPos)
discard self.advance()
elif c == '"' or c == '\'':
let quote = c
let strStart = self.currentPos()
discard self.advance()
var content = $quote
var foundClosing = false
while self.hasMore():
let sc = self.advance()
if sc == '\\':
content.add(sc)
if self.hasMore(): content.add(self.advance())
elif sc == quote:
content.add(quote)
self.emitToken(tkString, content, strStart)
foundClosing = true
break
elif sc == '\n':
self.recordError(esError, "Newline in string", ErrUnclosedString, strStart)
break
else:
content.add(sc)
if not foundClosing:
self.recordError(esError, "Unclosed string", ErrUnclosedString, strStart)
self.emitToken(tkString, content, strStart)
elif isDigit(c):
let numStart = self.currentPos()
var num = ""
while self.hasMore() and (isAlphaNum(self.peek()) or self.peek() in {'.', 'x', 'X', 'o', 'O', 'b', 'B', '_'}):
num.add(self.advance())
self.emitToken(tkNumber, num, numStart)
elif isAlpha(c) or c == '$' or c == '_':
let idStart = self.currentPos()
var ident = ""
while self.hasMore() and (isAlphaNum(self.peek()) or self.peek() in {'_', '$'}) or self.peek() == '.':
# Support dotted names in types
if self.peek() == '.' and ident.len > 0 and isAlpha(self.source[self.pos - 1]):
ident.add(self.advance())
continue
ident.add(self.advance())
if jsKeywordSet.contains(ident):
self.emitToken(tkKeyword, ident, idStart)
elif tsExtraKeywordSet.contains(ident):
self.emitToken(tkKeyword, ident, idStart)
else:
self.emitToken(tkIdentifier, ident, idStart)
elif c == '(': self.emitBracketToken(kOpenParen, $self.advance(), startPos)
elif c == ')':
self.closeBracket(')', kCloseParen, startPos)
elif c == '[': self.emitBracketToken(kOpenBracket, $self.advance(), startPos)
elif c == ']':
self.closeBracket(']', kCloseBracket, startPos)
elif c == '{': self.emitBracketToken(kOpenBrace, $self.advance(), startPos)
elif c == '}':
self.closeBracket('}', kCloseBrace, startPos)
elif c in {'+', '-', '*', '/', '%', '^', '~', '|', '&', '<', '>', '=', '!', '?'}:
let opStart = self.currentPos()
var op = ""
while self.hasMore() and self.peek() in {'+', '-', '*', '/', '%', '^', '~', '|', '&', '<', '>', '=', '!', '?', '.'}:
let nc = self.peek()
if nc == '.' and op.len > 0: break
op.add(self.advance())
if op in ["=", "=>"]:
self.emitToken(tkAssignment, op, opStart)
else:
self.emitToken(tkOperator, op, opStart)
elif c == '.':
self.emitToken(tkPunctuation, $self.advance(), startPos)
elif c in {',', ';', ':'}:
self.emitToken(tkPunctuation, $self.advance(), startPos)
elif c == '@':
# TS decorator
let atStart = self.currentPos()
var decorator = "@"
discard self.advance()
while self.hasMore() and (isAlphaNum(self.peek()) or self.peek() in {'_', '.'}):
decorator.add(self.advance())
self.emitToken(tkDirective, decorator, atStart)
elif c == '#':
let hashStart = self.currentPos()
var dir = "#"
discard self.advance()
while self.hasMore() and (isAlphaNum(self.peek()) or self.peek() == '_'):
dir.add(self.advance())
self.emitToken(tkDirective, dir, hashStart)
elif c == '<' and self.pos + 1 < self.sourceLen and (isAlpha(self.peek(1)) or self.peek(1) == '/'):
# XML-like JSX tag or TypeScript type parameter - treat as operator/template
self.emitBracketToken(kAngleOpen, $self.advance(), startPos)
else:
self.emitToken(tkError, $self.advance(), startPos)
self.emitToken(tkEndOfFile, "", self.currentPos())
when defined(nimcheckDebug):
if self.options.debugMode: debugLeave("TS_TOKENIZER", &"Complete: {self.tokens.len} tokens")
# ---------------------------------------------------------------------------
# XML tokenizer
# ---------------------------------------------------------------------------
type
XMLTokenizer* = ref object of TokenizerBase
tagStack*: seq[string]
const
xmlKeywords* = @["xml", "xsl", "schema", "doctype", "element", "attribute",
"entity", "notation", "simpleType", "complexType", "sequence", "choice",
"include", "import", "redefine", "override", "annotation", "documentation",
"appinfo", "any", "group", "attributeGroup", "restriction", "extension",
"union", "list", "all", "key", "keyref", "unique", "selector", "field",
"pattern", "enumeration", "length", "minLength", "maxLength", "minInclusive",
"maxInclusive", "minExclusive", "maxExclusive", "totalDigits", "fractionDigits",
"whiteSpace"
]
xmlKeywordSet* = xmlKeywords.toHashSet()
proc newXMLTokenizer*(source: string, options: ValidationOptions = newValidationOptions()): XMLTokenizer =
XMLTokenizer(
source: source, sourceLen: source.len, options: options,
tokens: @[], pos: 0, line: 1, column: 1, bracketStack: @[], errors: @[], eof: false,
tagStack: @[]
)
method tokenize*(self: XMLTokenizer) =
when defined(nimcheckDebug):
if self.options.debugMode: debugEnter("XML_TOKENIZER", "Starting XML tokenization")
while self.hasMore():
let startPos = self.currentPos()
let c = self.peek()
if c in {' ', '\t', '\r'}:
let wsStart = self.currentPos()
var ws = ""
while self.hasMore() and self.peek() in {' ', '\t', '\r'}:
ws.add(self.advance())
self.emitToken(tkWhitespace, ws, wsStart)
elif c == '\n':
self.emitToken(tkNewline, $self.advance(), startPos)
elif self.peekString(4) == "<!--":
let comStart = self.currentPos()
discard self.advance()
discard self.advance()
discard self.advance()
discard self.advance()
var content = ""
var foundClosing = false
while self.hasMore():
if self.peekString(3) == "-->":
discard self.advance()
discard self.advance()
discard self.advance()
foundClosing = true
break
content.add(self.advance())
if not foundClosing:
self.recordError(esError, "Unclosed XML comment", ErrUnclosedComment, comStart,
hint = "Add closing '-->'")
self.emitToken(tkComment, "<!--" & content & "-->", comStart)
elif self.peekString(2) == "<?":
# Processing instruction <?xml ... ?>
let piStart = self.currentPos()
discard self.advance()
discard self.advance()
var content = "<?"
while self.hasMore():
if self.peekString(2) == "?>":
content.add("?>")
discard self.advance()
discard self.advance()
break
content.add(self.advance())
self.emitToken(tkDirective, content, piStart)
elif c == '<':
let tagStart = self.currentPos()
discard self.advance()
var isClosing = false
var isSelfClosing = false
if self.hasMore() and self.peek() == '/':
isClosing = true
discard self.advance()
# Read tag name
var tagName = ""
while self.hasMore() and self.peek() != '>' and self.peek() != '/' and self.peek() notin {' ', '\t', '\n', '\r'}:
tagName.add(self.advance())
if tagName.len > 0:
if isClosing:
self.emitToken(tkSpecial, "</" & tagName, tagStart)
if self.tagStack.len > 0 and self.tagStack[^1] == tagName:
discard self.tagStack.pop()
else:
self.recordError(esError, &"Mismatched XML close tag '</{tagName}>'",
ErrMismatchedBracket, tagStart,
hint = if self.tagStack.len > 0: &"Expected '</{self.tagStack[^1]}>'" else: "Unexpected close tag")
else:
self.tagStack.add(tagName)
self.emitToken(tkKeyword, tagName, tagStart)
else:
self.emitToken(tkOperator, "<", tagStart)
# Read attributes until > or />
while self.hasMore() and self.peek() != '>':
if self.peek() in {' ', '\t', '\n', '\r'}:
let wsStart2 = self.currentPos()
var ws2 = ""
while self.hasMore() and self.peek() in {' ', '\t', '\n', '\r'}:
ws2.add(self.advance())
self.emitToken(tkWhitespace, ws2, wsStart2)
elif self.peek() == '/' and self.pos + 1 < self.sourceLen and self.peek(1) == '>':
isSelfClosing = true
discard self.advance()
discard self.advance()
if not isClosing and self.tagStack.len > 0:
discard self.tagStack.pop()
self.emitToken(tkOperator, "/>", tagStart)
break
elif isAlpha(self.peek()) or self.peek() in {'_', ':'}:
let attrStart = self.currentPos()
var attrName = ""
while self.hasMore() and (isAlphaNum(self.peek()) or self.peek() in {'_', ':', '-'}):
attrName.add(self.advance())
self.emitToken(tkIdentifier, attrName, attrStart)
# Skip = "value"
while self.hasMore() and self.peek() in {' ', '\t', '\n', '\r'}:
discard self.advance()
if self.hasMore() and self.peek() == '=':
discard self.advance()
while self.hasMore() and self.peek() in {' ', '\t', '\n', '\r'}:
discard self.advance()
if self.hasMore() and self.peek() == '"':
let valStart = self.currentPos()
discard self.advance()
var val = "\""
while self.hasMore() and self.peek() != '"':
val.add(self.advance())
if self.hasMore():
val.add(self.advance())
self.emitToken(tkString, val, valStart)
else:
discard self.advance()
if self.hasMore() and self.peek() == '>':
discard self.advance()
if not isClosing:
self.emitToken(tkOperator, ">", tagStart)
elif c == '>':
discard self.advance()
# Already handled in tag context
elif c == '"' or c == '\'':
let strStart = self.currentPos()
discard self.advance()
var content = $c
var foundClosing = false
while self.hasMore():
let sc = self.advance()
if sc == c:
content.add(c)
foundClosing = true
break
content.add(sc)
if not foundClosing:
self.recordError(esError, &"Unclosed XML attribute value", ErrUnclosedString, strStart)
self.emitToken(tkString, content, strStart)
elif isDigit(c) or (c == '.' and self.hasMore() and self.peek(1).isDigit()):
let numStart = self.currentPos()
var num = ""
while self.hasMore() and (self.peek().isDigit() or self.peek() in {'.', 'e', 'E', '+', '-'}):
num.add(self.advance())
self.emitToken(tkNumber, num, numStart)
elif isAlpha(c) or c == '_' or c == ':':
let idStart = self.currentPos()
var ident = ""
while self.hasMore() and (isAlphaNum(self.peek()) or self.peek() in {'_', ':', '-'}):
ident.add(self.advance())
if xmlKeywordSet.contains(ident):
self.emitToken(tkKeyword, ident, idStart)
else:
self.emitToken(tkIdentifier, ident, idStart)
elif c in {'&'}:
let entStart = self.currentPos()
var entity = "&"
discard self.advance()
while self.hasMore() and self.peek() != ';' and self.peek() != ' ':
entity.add(self.advance())
if self.hasMore() and self.peek() == ';':
entity.add(self.advance())
self.emitToken(tkSpecial, entity, entStart)
elif c in {',', ';'}:
self.emitToken(tkPunctuation, $self.advance(), startPos)
elif c in {'=', '!', '?'}:
self.emitToken(tkOperator, $self.advance(), startPos)
else:
# Text content between tags
let txtStart = self.currentPos()
var text = ""
while self.hasMore() and self.peek() notin {'<', '&'}:
text.add(self.advance())
if text.len > 0 and text.strip().len > 0:
self.emitToken(tkTemplateTag, text, txtStart)
elif text.len > 0:
self.emitToken(tkWhitespace, text, txtStart)
else:
discard self.advance()
# Check for unclosed tags
for tag in self.tagStack:
let pos = newSourcePosition(self.line, self.column, self.pos)
self.recordError(esError, &"Unclosed XML tag '<{tag}>'",
ErrUnclosedBlock, pos, hint = &"Add closing '</{tag}>'")
self.emitToken(tkEndOfFile, "", self.currentPos())
when defined(nimcheckDebug):
if self.options.debugMode: debugLeave("XML_TOKENIZER", &"Complete: {self.tokens.len} tokens")

40
tests/fixtures/c/edge_cases.c vendored Normal file
View File

@ -0,0 +1,40 @@
#include <stdio.h>
#include <string.h>
#include <wchar.h>
#include <locale.h>
/* Unicode in comments: Привет, 世界, 🌍 */
int main(void) {
setlocale(LC_ALL, "");
/* Unicode string literals */
wchar_t *greeting = L"こんにちは";
wprintf(L"%ls\n", greeting);
/* Null byte embedded */
char buf[] = "Hello\x00World";
printf("Length: %zu\n", strlen(buf)); /* stops at null byte */
/* Deeply nested structs */
struct a { int x; };
struct b { struct a a; };
struct c { struct b b; };
struct d { struct c c; };
struct e { struct d d; };
struct f { struct e e; };
struct f val = {{{{{{42}}}}}};
printf("%d\n", val.e.d.c.b.a.x);
/* Very long function name */
void this_is_an_extremely_long_function_name_that_goes_on_and_on_and_on(void) {
printf("deep\n");
}
this_is_an_extremely_long_function_name_that_goes_on_and_on_and_on();
/* Encoding: BOM-like byte sequences */
unsigned char raw[] = {0xEF, 0xBB, 0xBF, 'A', 'B', 'C', 0};
printf("%s\n", raw);
return 0;
}

26
tests/fixtures/c/invalid.c vendored Normal file
View File

@ -0,0 +1,26 @@
#include <stdio.h>
#include <stdlib.h>
#define MAX 10
int main(void) {
int x = 5
printf("x is %d\n", x)
char *s = "unclosed string;
printf("%s\n", s);
int arr[3] = {1, 2, 3;
printf("%d\n", arr[1]);
if (x > 0 {
printf("positive\n");
}
char *p = malloc(10;
free(p);
struct { int a; int b; } s = {1, 2};
s.a = ;
return 0;

36
tests/fixtures/c/valid.c vendored Normal file
View File

@ -0,0 +1,36 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#define MAX_NAME 64
#define GREETING "Hello, World!"
typedef struct {
int id;
char name[MAX_NAME];
double score;
} Player;
int add(int a, int b) {
return a + b;
}
void greet(const char *name) {
printf("%s, %s!\n", GREETING, name);
}
int main(void) {
Player p = {1, "Alice", 95.5};
greet(p.name);
int result = add(3, 4);
printf("3 + 4 = %d\n", result);
FILE *fp = fopen("test.txt", "w");
if (fp) {
fprintf(fp, "id=%d,name=%s,score=%.1f\n", p.id, p.name, p.score);
fclose(fp);
}
return 0;
}

46
tests/fixtures/cpp/edge_cases.cpp vendored Normal file
View File

@ -0,0 +1,46 @@
#include <iostream>
#include <string>
#include <vector>
#include <locale>
#include <codecvt>
/* Unicode comment: 你好, 世界, 🌍🚀 */
int main() {
// Unicode string literals
std::string utf8 = "こんにちは世界";
std::cout << utf8 << std::endl;
// Null byte in string
std::string null_str = std::string("hello\x00world", 11);
std::cout << "Null-str size: " << null_str.size() << std::endl;
// Deeply nested templates
template <typename T>
struct Wrap { T value; };
Wrap<Wrap<Wrap<Wrap<Wrap<Wrap<Wrap<Wrap<int>>>>>>>> deep = {{{{{{{{42}}}}}}}};
std::cout << deep.value.value.value.value.value.value.value.value << std::endl;
// Extremely long string
std::string long_str(10000, 'x');
std::cout << "Long string length: " << long_str.length() << std::endl;
// BOM-like bytes
const unsigned char bom[] = {0xEF, 0xBB, 0xBF, 'H', 'i', 0};
std::cout << reinterpret_cast<const char*>(bom) << std::endl;
// Deep lambda nesting
auto f1 = [](int x) {
return [x](int y) {
return [x, y](int z) {
return [x, y, z](int w) {
return x + y + z + w;
};
};
};
};
std::cout << "Nested lambdas: " << f1(1)(2)(3)(4) << std::endl;
return 0;
}

33
tests/fixtures/cpp/invalid.cpp vendored Normal file
View File

@ -0,0 +1,33 @@
#include <iostream>
#include <string>
class Broken {
public:
Broken(int x) : x_(x) {}
void show() {
std::cout << x_ << std::endl;
int missing_semicolon()
return x_;
}
private:
int x_
};
int main() {
Broken b(42);
b.show()
char *s = "unclosed string;
std::cout << s << std::endl;
if (b.x_ > 0 {
std::cout << "positive" << std::endl;
}
std::vector<int> v = {1, 2, 3};
std::cout << v[5] << std::endl;
return 0;

58
tests/fixtures/cpp/valid.cpp vendored Normal file
View File

@ -0,0 +1,58 @@
#include <iostream>
#include <vector>
#include <string>
#include <memory>
#include <algorithm>
class Person {
public:
Person(int id, std::string name)
: id_(id), name_(std::move(name)) {}
int id() const { return id_; }
const std::string& name() const { return name_; }
virtual void greet() const {
std::cout << "Hello, I'm " << name_ << std::endl;
}
virtual ~Person() = default;
private:
int id_;
std::string name_;
};
class Student : public Person {
public:
Student(int id, std::string name, double gpa)
: Person(id, std::move(name)), gpa_(gpa) {}
void greet() const override {
std::cout << "Hi, I'm " << name() << " (GPA: " << gpa_ << ")" << std::endl;
}
private:
double gpa_;
};
template <typename T>
T max_value(T a, T b) {
return (a > b) ? a : b;
}
int main() {
auto p = std::make_unique<Student>(1, "Alice", 3.9);
p->greet();
std::vector<int> nums = {3, 1, 4, 1, 5, 9};
std::sort(nums.begin(), nums.end());
for (int n : nums) {
std::cout << n << " ";
}
std::cout << std::endl;
std::cout << "Max of 10 and 20: " << max_value(10, 20) << std::endl;
return 0;
}

70
tests/fixtures/csharp/edge_cases.cs vendored Normal file
View File

@ -0,0 +1,70 @@
using System;
using System.Text;
using System.Collections.Generic;
// Unicode comment: 你好世界 🌍🚀
public class EdgeCases
{
public static void Main()
{
// Unicode string literals
string unicode = "こんにちは世界";
Console.WriteLine(unicode);
// Null byte embedded
string nullStr = "hello\x00world";
Console.WriteLine($"Null-str length: {nullStr.Length}");
// Deeply nested generics
var deep = new Dictionary<int, Dictionary<int, Dictionary<int, Dictionary<int, Dictionary<int, string>>>>>
{
{ 1, new Dictionary<int, Dictionary<int, Dictionary<int, Dictionary<int, string>>>>
{
{ 2, new Dictionary<int, Dictionary<int, Dictionary<int, string>>>
{
{ 3, new Dictionary<int, Dictionary<int, string>>
{
{ 4, new Dictionary<int, string> { { 5, "deep" } } }
}
}
}
}
}
}
};
Console.WriteLine(deep[1][2][3][4][5]);
// Very long string
string longStr = new string('x', 10000);
Console.WriteLine($"Long string length: {longStr.Length}");
// BOM prefix
byte[] bom = { 0xEF, 0xBB, 0xBF, (byte)'H', (byte)'i' };
Console.WriteLine(Encoding.UTF8.GetString(bom));
// Deep exception nesting
try
{
try
{
try
{
throw new InvalidOperationException("inner");
}
catch (Exception ex)
{
throw new ApplicationException("middle", ex);
}
}
catch (Exception ex)
{
throw new Exception("outer", ex);
}
}
catch (Exception ex)
{
Console.WriteLine($"Nested exception: {ex.Message}");
}
}
}

53
tests/fixtures/csharp/invalid.cs vendored Normal file
View File

@ -0,0 +1,53 @@
using System;
using System.Collections.Generic;
namespace Broken
{
public class BrokenClass
{
public int X { get; set; }
public string Name;
public BrokenClass(int x, string name)
{
X = x;
Name = name
}
public void Show()
{
Console.WriteLine($"{Name}: {X}")
}
public int MissingBody()
}
public class Derived : BrokenClass
{
public Derived() : base(42, "test")
{
}
public void BrokenMethod()
{
string s = "unclosed string;
Console.WriteLine(s);
if (X > 0
Console.WriteLine("positive");
}
int[] arr = {1, 2, 3;
Console.WriteLine(arr[0]);
}
}
public class Program
{
public static void Main()
{
var obj = new Derived();
obj.Show()
}
}
}

73
tests/fixtures/csharp/valid.cs vendored Normal file
View File

@ -0,0 +1,73 @@
using System;
using System.Collections.Generic;
using System.Linq;
namespace Fixtures
{
public class Person
{
public int Id { get; set; }
public string Name { get; set; }
public string Email { get; set; }
public Person(int id, string name, string email)
{
Id = id;
Name = name;
Email = email;
}
public override string ToString()
{
return $"{Name} <{Email}>";
}
}
public class Repository<T>
{
private readonly Dictionary<int, T> _items = new();
public void Add(int key, T item)
{
_items[key] = item;
}
public T? Find(int key)
{
return _items.TryGetValue(key, out var item) ? item : default;
}
public IEnumerable<T> GetAll()
{
return _items.Values.ToList();
}
}
public static class Helpers
{
public static int Add(int a, int b) => a + b;
public static IEnumerable<T> Filter<T>(IEnumerable<T> items, Func<T, bool> predicate)
{
return items.Where(predicate);
}
}
public class Program
{
public static void Main(string[] args)
{
var repo = new Repository<Person>();
repo.Add(1, new Person(1, "Alice", "alice@example.com"));
repo.Add(2, new Person(2, "Bob", "bob@example.com"));
var active = repo.GetAll();
foreach (var p in active)
{
Console.WriteLine(p);
}
Console.WriteLine($"3 + 4 = {Helpers.Add(3, 4)}");
}
}
}

39
tests/fixtures/css/edge_cases.css vendored Normal file
View File

@ -0,0 +1,39 @@
:root {
--color-primary: #2563eb;
}
*,
*::before,
*::after {
box-sizing: border-box;
}
body {
font-family: "Inter", sans-serif;
background: var(--color-bg);
color: var(--color-text);
}
.navbar {
display: flex;
background: linear-gradient(135deg, var(--color-primary), #7c3aed);
}
.btn {
padding: 0.625rem 1.25rem;
border: none;
border-radius: 0.5rem;
cursor: pointer;
transition: background 0.2s ease;
}
@keyframes slideIn {
from { opacity: 0; transform: translateX(-20px); }
to { opacity: 1; transform: translateX(0); }
}
@media (max-width: 600px) {
.navbar {
flex-direction: column;
}
}

28
tests/fixtures/css/invalid.css vendored Normal file
View File

@ -0,0 +1,28 @@
:root {
--color-primary: #2563eb;
}
*,
*::before,
*::after {
box-sizing: border-box;
}
body {
font-family: "Inter", sans-serif;
}
.navbar {
display: flex;
background: linear-gradient(135deg, var(--color-primary), #7c3aed);
}
.btn {
padding: 0.625rem 1.25rem;
border: none;
}
@media (max-width: 600px) {
.navbar
flex-direction: column;
}

159
tests/fixtures/css/valid.css vendored Normal file
View File

@ -0,0 +1,159 @@
:root {
--color-primary: #2563eb;
--color-primary-hover: #1d4ed8;
--color-bg: #f8fafc;
--color-text: #1e293b;
--color-border: #e2e8f0;
--radius-md: 0.5rem;
--shadow-sm: 0 1px 2px 0 rgb(0 0 0 / 0.05);
--font-sans: "Inter", system-ui, -apple-system, sans-serif;
}
*,
*::before,
*::after {
box-sizing: border-box;
margin: 0;
padding: 0;
}
html {
font-size: 16px;
scroll-behavior: smooth;
}
body {
font-family: var(--font-sans);
background: var(--color-bg);
color: var(--color-text);
line-height: 1.6;
}
.container {
max-width: 1200px;
margin: 0 auto;
padding: 0 1rem;
}
.navbar {
display: flex;
align-items: center;
justify-content: space-between;
padding: 1rem 2rem;
background: linear-gradient(135deg, var(--color-primary), #7c3aed);
}
.navbar .logo {
font-size: 1.5rem;
font-weight: 700;
color: #fff;
text-decoration: none;
}
.nav-links {
display: flex;
gap: 1.5rem;
list-style: none;
}
.nav-links a {
color: rgb(255 255 255 / 0.85);
text-decoration: none;
transition: color 0.2s ease;
}
.nav-links a:hover {
color: #fff;
}
.card {
background: #fff;
border: 1px solid var(--color-border);
border-radius: var(--radius-md);
box-shadow: var(--shadow-sm);
padding: 1.5rem;
}
.card__title {
font-size: 1.25rem;
font-weight: 600;
margin-bottom: 0.75rem;
}
.card__body {
color: #64748b;
}
.btn {
display: inline-flex;
align-items: center;
gap: 0.5rem;
padding: 0.625rem 1.25rem;
border: none;
border-radius: var(--radius-md);
font-weight: 500;
cursor: pointer;
transition: background 0.2s ease, transform 0.1s ease;
}
.btn:active {
transform: scale(0.97);
}
.btn--primary {
background: var(--color-primary);
color: #fff;
}
.btn--primary:hover {
background: var(--color-primary-hover);
}
.btn--outline {
background: transparent;
border: 2px solid var(--color-primary);
color: var(--color-primary);
}
.grid {
display: grid;
grid-template-columns: repeat(auto-fill, minmax(300px, 1fr));
gap: 1.5rem;
}
@keyframes fadeIn {
from { opacity: 0; transform: translateY(10px); }
to { opacity: 1; transform: translateY(0); }
}
.animate-in {
animation: fadeIn 0.4s ease-out both;
}
@media (max-width: 768px) {
.navbar {
flex-direction: column;
gap: 1rem;
}
.nav-links {
flex-wrap: wrap;
justify-content: center;
}
.grid {
grid-template-columns: 1fr;
}
}
@media (prefers-color-scheme: dark) {
:root {
--color-bg: #0f172a;
--color-text: #e2e8f0;
--color-border: #334155;
}
.card {
background: #1e293b;
}
}

9
tests/fixtures/dockerfile/edge_cases.Dockerfile vendored Normal file
View File

@ -0,0 +1,9 @@
FROM alpine:3.20 as builder
RUN apk add --no-cache gcc musl-dev make
WORKDIR /build
COPY . .
RUN make release
FROM alpine:3.20
RUN apk add --no-cache pcre openssl
COPY --from=builder /build/bin/app /usr/local/bin/app
ENTRYPOINT ["app"]

12
tests/fixtures/dockerfile/invalid.Dockerfile vendored Normal file
View File

@ -0,0 +1,12 @@
FROM alpine:3.20 AS builder
RUN apk add --no-cache \
gcc \
musl-dev \
make \
FROM alpine:3.20
COPY --from=builder /build/bin/app /usr/local/bin/app
ENTRYPOINT ["app"
CMD ["--help"]

45
tests/fixtures/dockerfile/valid.Dockerfile vendored Normal file
View File

@ -0,0 +1,45 @@
FROM alpine:3.20 AS builder
RUN apk add --no-cache \
gcc \
musl-dev \
pcre-dev \
openssl-dev \
make \
git
WORKDIR /build
COPY . .
RUN make release \
&& strip bin/nimcheck
FROM alpine:3.20 AS runtime
RUN apk add --no-cache \
pcre \
openssl \
ca-certificates \
tzdata
ENV TZ=UTC
ENV NIMCHECK_LOG_LEVEL=info
ENV NIMCHECK_THREADS=4
RUN addgroup -g 1000 nimcheck \
&& adduser -D -u 1000 -G nimcheck nimcheck
COPY --from=builder /build/bin/nimcheck /usr/local/bin/nimcheck
RUN chmod 755 /usr/local/bin/nimcheck
WORKDIR /workspace
USER nimcheck
HEALTHCHECK --interval=30s --timeout=3s --retries=3 \
CMD nimcheck --version || exit 1
ENTRYPOINT ["nimcheck"]
CMD ["--help"]

181
tests/fixtures/go/edge_cases.go vendored Normal file
View File

@ -0,0 +1,181 @@
package main
import (
"context"
"crypto/sha256"
"encoding/hex"
"fmt"
"hash"
"io"
"math"
"net/url"
"reflect"
"sort"
"sync/atomic"
"time"
"unsafe"
)
const (
MaxRetries = 3
BaseDelay = 100 * time.Millisecond
MaxDelay = 5 * time.Second
)
type Executor[T any] struct {
workers int
tasks chan func() T
results chan T
}
func NewExecutor[T any](workers int) *Executor[T] {
return &Executor[T]{
workers: workers,
tasks: make(chan func() T, workers*2),
results: make(chan T, workers*2),
}
}
func (e *Executor[T]) Submit(task func() T) {
e.tasks <- task
}
func (e *Executor[T]) Run(ctx context.Context) []T {
for i := 0; i < e.workers; i++ {
go func() {
for {
select {
case <-ctx.Done():
return
case task := <-e.tasks:
e.results <- task()
}
}
}()
}
var results []T
for i := 0; i < cap(e.tasks); i++ {
select {
case r := <-e.results:
results = append(results, r)
case <-ctx.Done():
return results
}
}
return results
}
func retryWithBackoff(operation func() error) error {
var err error
delay := BaseDelay
for i := 0; i < MaxRetries; i++ {
if err = operation(); err == nil {
return nil
}
if i < MaxRetries-1 {
jitter := time.Duration(rand.Int63n(int64(delay / 2)))
time.Sleep(delay + jitter)
delay = time.Duration(math.Min(float64(delay*2), float64(MaxDelay)))
}
}
return fmt.Errorf("operation failed after %d retries: %w", MaxRetries, err)
}
func HashContent(r io.Reader) (string, error) {
h := sha256.New()
if _, err := io.Copy(h, r); err != nil {
return "", fmt.Errorf("hashing failed: %w", err)
}
return hex.EncodeToString(h.Sum(nil)), nil
}
func SliceToMap[K comparable, V any](items []V, keyFn func(V) K) map[K]V {
result := make(map[K]V, len(items))
for _, item := range items {
result[keyFn(item)] = item
}
return result
}
func InverseMap[K, V comparable](m map[K]V) map[V]K {
result := make(map[V]K, len(m))
for k, v := range m {
result[v] = k
}
return result
}
var counter atomic.Int64
func nextID() int64 {
return counter.Add(1)
}
type Enum interface {
~int
String() string
}
type Color int
const (
Red Color = iota
Green
Blue
)
func (c Color) String() string {
return [...]string{"red", "green", "blue"}[c]
}
func Must[T any](val T, err error) T {
if err != nil {
panic(err)
}
return val
}
type TransformFunc[T, U any] func(T) U
func Chain[T, U, V any](f TransformFunc[T, U], g TransformFunc[U, V]) TransformFunc[T, V] {
return func(t T) V {
return g(f(t))
}
}
func zero[T any]() T {
var zero T
return zero
}
func Map[T, U any](items []T, fn func(T) U) []U {
result := make([]U, len(items))
for i, item := range items {
result[i] = fn(item)
}
return result
}
func Filter[T any](items []T, fn func(T) bool) []T {
var result []T
for _, item := range items {
if fn(item) {
result = append(result, item)
}
}
return result
}
func Reduce[T, U any](items []T, init U, fn func(U, T) U) U {
acc := init
for _, item := range items {
acc = fn(acc, item)
}
return acc
}
func init() {
fmt.Println("initializing package main")
}

26
tests/fixtures/go/invalid.go vendored Normal file
View File

@ -0,0 +1,26 @@
package main
import "fmt"
func main() {
x := 42
fmt.Println(x
func brokenFunc(x int) string {
return x
}
type User struct {
Name string
Age int
}
func (u User) Greet() string {
return "Hello, " + .Name
switch x {
case 1:
fmt.Println("one")
case 2:
fmt.Println("two")
}

149
tests/fixtures/go/valid.go vendored Normal file
View File

@ -0,0 +1,149 @@
package main
import (
"encoding/json"
"errors"
"fmt"
"io"
"log"
"net/http"
"os"
"strings"
"sync"
"time"
)
type User struct {
ID int `json:"id"`
Name string `json:"name"`
Email string `json:"email"`
Active bool `json:"active"`
CreatedAt time.Time `json:"created_at"`
}
type UserService struct {
mu sync.RWMutex
users map[int]*User
}
func NewUserService() *UserService {
return &UserService{
users: make(map[int]*User),
}
}
func (s *UserService) Add(user *User) error {
s.mu.Lock()
defer s.mu.Unlock()
if user.Name == "" {
return errors.New("name is required")
}
if !strings.Contains(user.Email, "@") {
return errors.New("invalid email")
}
if _, exists := s.users[user.ID]; exists {
return errors.New("user already exists")
}
s.users[user.ID] = user
return nil
}
func (s *UserService) FindByID(id int) (*User, error) {
s.mu.RLock()
defer s.mu.RUnlock()
user, ok := s.users[id]
if !ok {
return nil, fmt.Errorf("user %d not found", id)
}
return user, nil
}
func (s *UserService) FindByEmail(email string) *User {
s.mu.RLock()
defer s.mu.RUnlock()
for _, u := range s.users {
if u.Email == email {
return u
}
}
return nil
}
func (s *UserService) All() []*User {
s.mu.RLock()
defer s.mu.RUnlock()
result := make([]*User, 0, len(s.users))
for _, u := range s.users {
result = append(result, u)
}
return result
}
func loggingMiddleware(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
start := time.Now()
log.Printf("%s %s %s", r.Method, r.URL.Path, r.RemoteAddr)
next.ServeHTTP(w, r)
log.Printf("%s completed in %v", r.URL.Path, time.Since(start))
})
}
func handleUsers(svc *UserService) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
switch r.Method {
case http.MethodGet:
users := svc.All()
json.NewEncoder(w).Encode(users)
case http.MethodPost:
var user User
if err := json.NewDecoder(r.Body).Decode(&user); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
if err := svc.Add(&user); err != nil {
http.Error(w, err.Error(), http.StatusConflict)
return
}
w.WriteHeader(http.StatusCreated)
json.NewEncoder(w).Encode(user)
default:
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
}
}
}
func main() {
svc := NewUserService()
svc.Add(&User{
ID: 1,
Name: "Alice",
Email: "alice@example.com",
Active: true,
})
mux := http.NewServeMux()
mux.HandleFunc("/users", handleUsers(svc))
port := os.Getenv("PORT")
if port == "" {
port = "8080"
}
server := &http.Server{
Addr: ":" + port,
Handler: loggingMiddleware(mux),
ReadTimeout: 10 * time.Second,
WriteTimeout: 10 * time.Second,
}
log.Printf("Server starting on :%s", port)
log.Fatal(server.ListenAndServe())
}

61
tests/fixtures/java/edge_cases.java vendored Normal file
View File

@ -0,0 +1,61 @@
package fixtures;
import java.util.*;
import java.nio.charset.*;
// Unicode comment: 你好世界 🌍🚀
public class EdgeCases {
public static void main(String[] args) throws Exception {
// Unicode string literals
String unicode = "こんにちは世界";
System.out.println(unicode);
// Null byte in string
String nullStr = "hello\u0000world";
System.out.println("Null-str length: " + nullStr.length());
// Deeply nested classes
Map<Integer, Map<Integer, Map<Integer, Map<Integer, Map<Integer, String>>>>> deep =
new HashMap<>();
Map<Integer, Map<Integer, Map<Integer, Map<Integer, String>>>> l4 = new HashMap<>();
Map<Integer, Map<Integer, Map<Integer, String>>> l3 = new HashMap<>();
Map<Integer, Map<Integer, String>> l2 = new HashMap<>();
Map<Integer, String> l1 = new HashMap<>();
l1.put(5, "deep");
l2.put(4, l1);
l3.put(3, l2);
l4.put(2, l3);
deep.put(1, l4);
System.out.println(deep.get(1).get(2).get(3).get(4).get(5));
// Very long string
StringBuilder sb = new StringBuilder(10000);
for (int i = 0; i < 10000; i++) sb.append('x');
System.out.println("Long string length: " + sb.length());
// BOM prefix
byte[] bom = {(byte)0xEF, (byte)0xBB, (byte)0xBF, (byte)'H', (byte)'i'};
System.out.println(new String(bom, StandardCharsets.UTF_8));
// Deep exception chaining
try {
try {
try {
throw new RuntimeException("inner");
} catch (Exception e) {
throw new RuntimeException("middle", e);
}
} catch (Exception e) {
throw new RuntimeException("outer", e);
}
} catch (Exception e) {
System.out.println("Chained: " + e.getMessage());
}
// Deep array nesting
int[][][][][][] deepArray = new int[2][2][2][2][2][2];
deepArray[0][0][0][0][0][0] = 42;
System.out.println("Deep array: " + deepArray[0][0][0][0][0][0]);
}
}

44
tests/fixtures/java/invalid.java vendored Normal file
View File

@ -0,0 +1,44 @@
package fixtures;
import java.util.*;
public class Invalid {
private int x;
public Invalid(int x) {
this.x = x
}
public void show() {
System.out.println(x)
}
public String broken() {
String s = "unclosed string;
return s;
public int missingBody()
public void badIf() {
if (x > 0 {
System.out.println("positive");
}
}
public void badArray() {
int[] arr = {1, 2, 3;
System.out.println(arr[0]);
}
public void missingSemicolon() {
int a = 5
int b = 10;
int c = a + b
System.out.println(c);
}
public static void main(String[] args) {
Invalid obj = new Invalid(42);
obj.show()
}
}

59
tests/fixtures/java/valid.java vendored Normal file
View File

@ -0,0 +1,59 @@
package fixtures;
import java.util.*;
import java.util.stream.*;
public class Valid {
public static class Person {
private final int id;
private final String name;
private final String email;
public Person(int id, String name, String email) {
this.id = id;
this.name = name;
this.email = email;
}
public int getId() { return id; }
public String getName() { return name; }
public String getEmail() { return email; }
@Override
public String toString() {
return name + " <" + email + ">";
}
}
public static class Repository<T> {
private final Map<Integer, T> items = new HashMap<>();
public void add(int key, T item) {
items.put(key, item);
}
public Optional<T> find(int key) {
return Optional.ofNullable(items.get(key));
}
public List<T> getAll() {
return new ArrayList<>(items.values());
}
}
public static int add(int a, int b) {
return a + b;
}
public static void main(String[] args) {
Repository<Person> repo = new Repository<>();
repo.add(1, new Person(1, "Alice", "alice@example.com"));
repo.add(2, new Person(2, "Bob", "bob@example.com"));
repo.getAll().stream()
.map(Person::getName)
.forEach(System.out::println);
System.out.println("3 + 4 = " + add(3, 4));
}
}

59
tests/fixtures/kotlin/edge_cases.kt vendored Normal file
View File

@ -0,0 +1,59 @@
package fixtures
// Unicode comment: 你好世界 🌍🚀
fun main() {
// Unicode string literals
val unicode = "こんにちは世界"
println(unicode)
// Null byte in string
val nullStr = "hello\u0000world"
println("Null-str length: ${nullStr.length}")
// Deeply nested generics
val deep = mapOf(
1 to mapOf(
2 to mapOf(
3 to mapOf(
4 to mapOf(
5 to "deep"
)
)
)
)
)
println(deep[1]?.get(2)?.get(3)?.get(4)?.get(5))
// Very long string
val longStr = "x".repeat(10000)
println("Long string length: ${longStr.length}")
// BOM bytes
val bom = byteArrayOf(0xEF.toByte(), 0xBB.toByte(), 0xBF.toByte(), 'H'.code.toByte(), 'i'.code.toByte())
println(String(bom))
// Deep lambda nesting
val f1: (Int) -> (Int) -> (Int) -> (Int) -> Int = { x ->
{ y ->
{ z ->
{ w -> x + y + z + w }
}
}
}
println("Nested lambdas: ${f1(1)(2)(3)(4)}")
// Infinite sequence (lazy)
val naturals = generateSequence(0) { it + 1 }
println("First 5 naturals: ${naturals.take(5).toList()}")
// Deep data class nesting
data class A(val x: Int)
data class B(val a: A)
data class C(val b: B)
data class D(val c: C)
data class E(val d: D)
data class F(val e: E)
val deepData = F(E(D(C(B(A(42))))))
println("Deep data: ${deepData.e.d.c.b.a.x}")
}

44
tests/fixtures/kotlin/invalid.kt vendored Normal file
View File

@ -0,0 +1,44 @@
package fixtures
data class Broken(
val x: Int,
val name: String
)
fun show() {
println("hello")
val x = 5
println(x)
}
class BadRepo<T> {
private val items = mutableMapOf<Int, T>()
fun add(key: Int, item: T) {
items[key] = item
}
fun find(key: Int): T? = items[key]
}
fun main() {
val repo = BadRepo<String>()
repo.add(1, "test")
val s = "unclosed string;
println(s)
val x = 5
if (x > 0 {
println("positive")
}
val list = listOf(1, 2, 3
println(list)
fun missingReturn(): Int {
val a = 5
}
println("done")
}

38
tests/fixtures/kotlin/valid.kt vendored Normal file
View File

@ -0,0 +1,38 @@
package fixtures
data class Person(
val id: Int,
val name: String,
val email: String
)
class Repository<T> {
private val items = mutableMapOf<Int, T>()
fun add(key: Int, item: T) {
items[key] = item
}
fun find(key: Int): T? = items[key]
fun getAll(): List<T> = items.values.toList()
}
fun add(a: Int, b: Int): Int = a + b
fun <T> filter(items: List<T>, predicate: (T) -> Boolean): List<T> {
return items.filter(predicate)
}
fun main() {
val repo = Repository<Person>()
repo.add(1, Person(1, "Alice", "alice@example.com"))
repo.add(2, Person(2, "Bob", "bob@example.com"))
repo.getAll().forEach { println(it) }
println("3 + 4 = ${add(3, 4)}")
val numbers = listOf(1, 2, 3, 4, 5)
val evens = filter(numbers) { it % 2 == 0 }
println("Evens: $evens")
}

64
tests/fixtures/lua/edge_cases.lua vendored Normal file
View File

@ -0,0 +1,64 @@
-- Unicode comment: 你好世界 🌍🚀
-- Unicode string literals
local unicode = "こんにちは世界"
print(unicode)
-- Null byte in string
local null_str = "hello\0world"
print("Null-str length: " .. #null_str)
-- Deeply nested tables
local deep = {
a = {
b = {
c = {
d = {
e = "deep"
}
}
}
}
}
print(deep.a.b.c.d.e)
-- Very long string
local long_str = string.rep("x", 10000)
print("Long string length: " .. #long_str)
-- Metatable chains
local a = {}
local b = {}
local c = {}
setmetatable(a, {__index = b})
setmetatable(b, {__index = c})
c.value = 42
print("Metatable chain: " .. a.value)
-- Deep function nesting
local function level1()
return function()
return function()
return function()
return function()
return 42
end
end
end
end
end
print("Deep functions: " .. level1()()()()())
-- Coroutine with deep stack
local co = coroutine.create(function()
local function recurse(n)
if n == 0 then
coroutine.yield("done")
else
return recurse(n - 1)
end
end
recurse(100)
end)
local _, msg = coroutine.resume(co)
print("Coroutine: " .. msg)

31
tests/fixtures/lua/invalid.lua vendored Normal file
View File

@ -0,0 +1,31 @@
local function broken()
local x = 5
print(x)
end
local s = "unclosed string
print(s)
local function bad_if(x)
if x > 0
print("positive")
end
end
local function missing_end(x)
if x > 0 then
print("positive")
end
local t = {1, 2, 3
print(t[1])
local function nested()
local function inner()
local function deeper()
return 42
end
end
end
print("done"

57
tests/fixtures/lua/valid.lua vendored Normal file
View File

@ -0,0 +1,57 @@
local function add(a, b)
return a + b
end
local function greet(name)
print("Hello, " .. name .. "!")
end
local Person = {}
Person.__index = Person
function Person:new(id, name, email)
local obj = {
id = id,
name = name,
email = email
}
setmetatable(obj, self)
return obj
end
function Person:__tostring()
return self.name .. " <" .. self.email .. ">"
end
local Repository = {}
Repository.__index = Repository
function Repository:new()
return setmetatable({items = {}}, self)
end
function Repository:add(key, item)
self.items[key] = item
end
function Repository:find(key)
return self.items[key]
end
function Repository:getAll()
local result = {}
for _, v in pairs(self.items) do
table.insert(result, v)
end
return result
end
local repo = Repository:new()
repo:add(1, Person:new(1, "Alice", "alice@example.com"))
repo:add(2, Person:new(2, "Bob", "bob@example.com"))
for _, p in ipairs(repo:getAll()) do
print(p)
end
print("3 + 4 = " .. add(3, 4))

15
tests/fixtures/makefile/edge_cases.Makefile vendored Normal file
View File

@ -0,0 +1,15 @@
.PHONY: all clean
CC := gcc
CFLAGS := -O2 -Wall
all: program
program: main.o util.o
$(CC) $(CFLAGS) -o $@ $^
%.o: %.c
$(CC) $(CFLAGS) -c -o $@ $<
clean:
rm -f program *.o

16
tests/fixtures/makefile/invalid.Makefile vendored Normal file
View File

@ -0,0 +1,16 @@
.PHONY: all clean
CC := gcc
CFLAGS := -O2 -Wall
all: program
program: main.o util.o
$(CC) $(CFLAGS) -o $@
%.o: %.c
$(CC) $(CFLAGS) -c -o $<
echo "unclosed
clean:
rm -f program *.o

57
tests/fixtures/makefile/valid.Makefile vendored Normal file
View File

@ -0,0 +1,57 @@
.PHONY: all build release debug clean test coverage lint install uninstall
NIM := nim
NIM_FLAGS := --hints:off --warnings:off
SRC_DIR := src
BIN_DIR := bin
TEST_DIR := tests
BUILD_DIR := build
SOURCES := $(shell find $(SRC_DIR) -name '*.nim')
TESTS := $(shell find $(TEST_DIR) -name '*.nim')
OBJECTS := $(patsubst $(SRC_DIR)/%.nim,$(BUILD_DIR)/%.o,$(SOURCES))
BINARY := $(BIN_DIR)/nimcheck
all: build
$(BIN_DIR):
mkdir -p $(BIN_DIR)
$(BUILD_DIR):
mkdir -p $(BUILD_DIR)
$(BUILD_DIR)/%.o: $(SRC_DIR)/%.nim | $(BUILD_DIR)
$(NIM) compile --compileOnly -d:release -o:$@ $<
$(BINARY): $(OBJECTS) | $(BIN_DIR)
$(NIM) compile -d:release --out:$(BINARY) $(SRC_DIR)/nimcheck.nim
build: $(BINARY)
release:
$(NIM) compile -d:release --opt:speed --passC:-flto --passL:-flto \
--out:$(BINARY) $(SRC_DIR)/nimcheck.nim
debug:
$(NIM) compile -d:debug --lineDir:on --stacktrace:on \
--out:$(BINARY) $(SRC_DIR)/nimcheck.nim
test: build
$(NIM) compile --run $(TEST_DIR)/test_all.nim
coverage:
$(NIM) compile --run -d:coverage $(TEST_DIR)/test_all.nim
lint:
$(NIM) check $(SRC_DIR)/nimcheck.nim
clean:
rm -rf $(BIN_DIR) $(BUILD_DIR)
find . -name 'nimcache' -type d -exec rm -rf {} + 2>/dev/null || true
install: release
install -m 755 $(BINARY) /usr/local/bin/nimcheck
uninstall:
rm -f /usr/local/bin/nimcheck

30
tests/fixtures/markdown/edge_cases.md vendored Normal file
View File

@ -0,0 +1,30 @@
# Nimcheck
---
> A fast syntax validator.
## Code Block
```nim
proc hello(): string =
result = "Hello"
```
```python
def hello() -> str:
return "Hello"
```
| A | B | C |
|---|---|---|
| 1 | 2 | 3 |
- [x] Done
- [ ] Pending
~~strikethrough~~
Footnotes[^1]
[^1]: A footnote.

26
tests/fixtures/markdown/invalid.md vendored Normal file
View File

@ -0,0 +1,26 @@
# Nimcheck
> A fast syntax validator
## Features
- Fast
- Accurate
- 27+ languages
## Unclosed Code Block
```nim
proc hello(): string =
result = "Hello"
Still in code block without closing fence.
## Broken Link
[Broken](
## Broken Table
| A | B | C
| 1 | 2 | 3 |

131
tests/fixtures/markdown/valid.md vendored Normal file
View File

@ -0,0 +1,131 @@
# Nimcheck: Universal Syntax Validator
[![Nim](https://img.shields.io/badge/Nim-2.0+-yellow)](https://nim-lang.org)
[![License](https://img.shields.io/badge/license-MIT-blue)](LICENSE)
> **Fast, accurate syntax validation for 27+ languages.**
> Built in Nim. Zero dependencies beyond the standard library.
---
## Table of Contents
1. [Features](#features)
2. [Installation](#installation)
3. [Quick Start](#quick-start)
4. [Usage](#usage)
5. [Supported Languages](#supported-languages)
---
## Features
- **Multi-language** — validates Nim, Python, JavaScript, TypeScript, PHP, Ruby, Rust, Go, Lua, C, C++, C#, Java, Swift, Kotlin, Bash, HTML, XML, Jinja, JSON, YAML, TOML, CSS, SQL, Markdown, Dockerfile, Makefile
- **Blazing fast** — compiles to native code via Nim
- **Auto-detection** — identifies language from extension, shebang, or content analysis
- **CI-friendly** — exit codes and structured output
### Detection Methods
| Priority | Method | Confidence |
|----------|-------------|------------|
| 1 | Explicit | 100% |
| 2 | Extension | 90% |
| 3 | Shebang | 95% |
| 4 | Content | 70-95% |
---
## Installation
### From Source
```bash
git clone https://github.com/retoor/nimcheck
cd nimcheck
make release
```
### Via Nimble
```bash
nimble install nimcheck
```
---
## Quick Start
```bash
nimcheck src/ # validate all files in src/
nimcheck --flavor nim # validate as Nim
nimcheck --json # JSON output
nimcheck --watch # watch mode
```
---
## Usage Examples
### Validate a single file
```bash
nimcheck src/main.nim
```
### Validate with explicit language
```bash
nimcheck --flavor python script.py
```
### Recursive directory scan
```bash
nimcheck --recursive project/
```
### JSON output for CI
```bash
nimcheck --json --recursive src/ > report.json
```
---
## Supported Languages
| Language | Extensions |
|-------------|-------------------------------------|
| Nim | `.nim`, `.nims`, `.nimble` |
| Python | `.py`, `.pyw`, `.pyx`, `.pxd` |
| JavaScript | `.js`, `.mjs`, `.cjs` |
| TypeScript | `.ts`, `.tsx` |
| PHP | `.php`, `.phtml` |
| HTML | `.html`, `.htm`, `.xhtml` |
| XML | `.xml`, `.svg` |
| JSON | `.json`, `.jsonc` |
| YAML | `.yaml`, `.yml` |
| TOML | `.toml` |
| CSS | `.css` |
| SQL | `.sql` |
| Markdown | `.md`, `.markdown` |
| Ruby | `.rb` |
| Rust | `.rs` |
| Go | `.go` |
| Lua | `.lua` |
| C | `.c`, `.h` |
| C++ | `.cpp`, `.cxx`, `.hpp` |
| C# | `.cs` |
| Java | `.java` |
| Swift | `.swift` |
| Kotlin | `.kt`, `.kts` |
| Bash | `.sh`, `.bash` |
| Dockerfile | `Dockerfile` |
| Makefile | `Makefile` |
---
## License
MIT © [Retoor](https://github.com/retoor)

66
tests/fixtures/ruby/edge_cases.rb vendored Normal file
View File

@ -0,0 +1,66 @@
module EnumerableExtensions
refine Array do
def second
self[1]
end
def middle
self[size / 2]
end
end
end
using EnumerableExtensions
class Singleton
private_class_method :new
@instance = nil
def self.instance
@instance ||= new
end
end
module_function
def log(level, message)
warn "[#{level.upcase}] #{message}"
end
require "ostruct"
require "set"
require "delegate"
Point = Struct.new(:x, :y, keyword_init: true)
p = Point.new(x: 10, y: 20)
class Stack
include Enumerable
def initialize
@items = []
end
def push(item)
@items.push(item)
end
def pop
@items.pop
end
def each(&block)
@items.each(&block)
end
end
s = Stack.new
s.push(1)
s.push(2)
s.each { |i| puts i }
$global_count = 0
DEFAULT_TIMEOUT = 30
puts "Global: #{$global_count}, Timeout: #{DEFAULT_TIMEOUT}"

12
tests/fixtures/ruby/invalid.rb vendored Normal file
View File

@ -0,0 +1,12 @@
class Broken
attr_accessor :x
def missing_end
if x > 0
puts "positive"
def wrong_syntax
x = 42
{x + 1
end

80
tests/fixtures/ruby/valid.rb vendored Normal file
View File

@ -0,0 +1,80 @@
#!/usr/bin/env ruby
require "json"
require "net/http"
require "uri"
class User
attr_accessor :id, :name, :email, :active
def initialize(id:, name:, email:, active: true)
@id = id
@name = name
@email = email
@active = active
end
def to_s
"#{@name} <#{@email}>"
end
def active?
@active
end
def deactivate!
@active = false
end
end
class UserRepository
def initialize
@users = {}
end
def add(user)
@users[user.id] = user
end
def find(id)
@users[id]
end
def find_by_email(email)
@users.values.find { |u| u.email == email }
end
def active_users
@users.values.select(&:active?)
end
def each(&block)
@users.values.each(&block)
end
def to_json(*args)
@users.values.map(&:to_h).to_json(*args)
end
end
def fetch_users(url)
uri = URI.parse(url)
response = Net::HTTP.get_response(uri)
raise "HTTP error: #{response.code}" unless response.is_a?(Net::HTTPSuccess)
JSON.parse(response.body)
rescue StandardError => e
warn "Failed to fetch users: #{e.message}"
[]
end
def process_items(items)
items.map { |item| yield item }
end
repo = UserRepository.new
repo.add(User.new(id: 1, name: "Alice", email: "alice@example.com"))
repo.each do |user|
puts user.to_s
end
puts "Active: #{repo.active_users.size}"

147
tests/fixtures/rust/edge_cases.rs vendored Normal file
View File

@ -0,0 +1,147 @@
use std::collections::{HashMap, HashSet, BTreeMap};
use std::rc::Rc;
use std::cell::RefCell;
use std::sync::{Arc, Mutex};
use std::marker::PhantomData;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum Status {
Active,
Inactive,
Pending,
Banned,
}
#[derive(Debug)]
pub struct TreeNode<T> {
pub value: T,
pub left: Option<Box<TreeNode<T>>>,
pub right: Option<Box<TreeNode<T>>>,
}
impl<T: Ord> TreeNode<T> {
pub fn insert(&mut self, value: T) {
let target = if value <= self.value {
&mut self.left
} else {
&mut self.right
};
match target {
Some(node) => node.insert(value),
None => *target = Some(Box::new(TreeNode {
value,
left: None,
right: None,
})),
}
}
pub fn contains(&self, value: &T) -> bool {
&self.value == value
|| self.left.as_ref().map_or(false, |n| n.contains(value))
|| self.right.as_ref().map_or(false, |n| n.contains(value))
}
}
pub trait Monoid {
fn empty() -> Self;
fn combine(&self, other: &Self) -> Self;
}
impl Monoid for i32 {
fn empty() -> Self { 0 }
fn combine(&self, other: &Self) -> Self { self + other }
}
impl Monoid for String {
fn empty() -> Self { String::new() }
fn combine(&self, other: &Self) -> Self { format!("{}{}", self, other) }
}
pub fn reduce<T: Monoid>(items: &[T]) -> T {
items.iter().fold(T::empty(), |acc, x| acc.combine(x))
}
macro_rules! vec_of_strings {
($($x:expr),*) => (vec![$($x.to_string()),*]);
}
pub struct Lazy<T, F = fn() -> T>
where
F: Fn() -> T,
{
value: Option<T>,
factory: F,
}
impl<T, F: Fn() -> T> Lazy<T, F> {
pub fn new(factory: F) -> Self {
Self {
value: None,
factory,
}
}
pub fn get(&mut self) -> &T {
self.value.get_or_insert_with(&self.factory)
}
}
pub async fn parallel_map<T, U, F>(items: Vec<T>, f: F) -> Vec<U>
where
T: Send + 'static,
U: Send + 'static,
F: Fn(T) -> U + Send + Sync + 'static,
{
use tokio::task;
let mut handles = Vec::new();
for item in items {
handles.push(task::spawn(async move { f(item) }));
}
let mut results = Vec::new();
for handle in handles {
results.push(handle.await.unwrap());
}
results
}
pub struct CustomIterator<T> {
items: Vec<T>,
index: usize,
}
impl<T: Clone> Iterator for CustomIterator<T> {
type Item = T;
fn next(&mut self) -> Option<Self::Item> {
if self.index < self.items.len() {
let item = self.items[self.index].clone();
self.index += 1;
Some(item)
} else {
None
}
}
}
const DEFAULT_PORT: u16 = 8080;
static APP_NAME: &str = "nimcheck-rs";
thread_local! {
static CACHE: RefCell<HashMap<String, String>> = RefCell::new(HashMap::new());
}
unsafe impl Send for User {}
unsafe impl Sync for User {}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_insert() {
let mut root = TreeNode { value: 5, left: None, right: None };
root.insert(3);
root.insert(7);
assert!(root.contains(&3));
assert!(root.contains(&7));
}
}

24
tests/fixtures/rust/invalid.rs vendored Normal file
View File

@ -0,0 +1,24 @@
fn main() {
let x = 42
let y = vec![1, 2,
println!("{}", x
fn missing_type(x) -> i32 {
x
}
match x {
1 => println!("one"),
2 => println!("two"),
}
struct Point {
x: i32,
y: i32,
}
impl Point {
fn new(x: i32, y: i32 -> Self {
Self { x, y }
}
}

112
tests/fixtures/rust/valid.rs vendored Normal file
View File

@ -0,0 +1,112 @@
use std::collections::HashMap;
use std::fmt::{Debug, Display};
use std::str::FromStr;
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct User {
pub id: u64,
pub name: String,
pub email: String,
pub active: bool,
pub tags: Vec<String>,
}
impl User {
pub fn new(id: u64, name: &str, email: &str) -> Self {
Self {
id,
name: name.to_string(),
email: email.to_string(),
active: true,
tags: Vec::new(),
}
}
pub fn deactivate(&mut self) {
self.active = false;
}
pub fn add_tag(&mut self, tag: &str) {
self.tags.push(tag.to_string());
}
}
impl Display for User {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{} <{}> [{}]", self.name, self.email, self.id)
}
}
pub enum ApiError {
NotFound(String),
Unauthorized,
Internal(String),
}
impl Debug for ApiError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
ApiError::NotFound(msg) => write!(f, "NotFound({})", msg),
ApiError::Unauthorized => write!(f, "Unauthorized"),
ApiError::Internal(msg) => write!(f, "Internal({})", msg),
}
}
}
pub type Result<T> = std::result::Result<T, ApiError>;
pub trait Repository<T> {
fn find_by_id(&self, id: u64) -> Result<T>;
fn save(&mut self, item: T) -> Result<()>;
fn delete(&mut self, id: u64) -> Result<()>;
}
pub struct UserRepository {
users: HashMap<u64, User>,
}
impl UserRepository {
pub fn new() -> Self {
Self {
users: HashMap::new(),
}
}
}
impl Repository<User> for UserRepository {
fn find_by_id(&self, id: u64) -> Result<User> {
self.users
.get(&id)
.cloned()
.ok_or_else(|| ApiError::NotFound(format!("User {}", id)))
}
fn save(&mut self, user: User) -> Result<()> {
let id = user.id;
self.users.insert(id, user);
Ok(())
}
fn delete(&mut self, id: u64) -> Result<()> {
self.users.remove(&id);
Ok(())
}
}
fn process_users(repo: &UserRepository) -> Result<Vec<String>> {
let names: Vec<String> = repo
.users
.values()
.filter(|u| u.active)
.map(|u| u.name.clone())
.collect();
Ok(names)
}
fn main() -> Result<()> {
let mut repo = UserRepository::new();
repo.save(User::new(1, "Alice", "alice@example.com"))?;
let names = process_users(&repo)?;
println!("{:?}", names);
Ok(())
}

62
tests/fixtures/sql/edge_cases.sql vendored Normal file
View File

@ -0,0 +1,62 @@
CREATE TABLE users (
id INTEGER PRIMARY KEY AUTOINCREMENT,
username VARCHAR(64) NOT NULL UNIQUE,
email VARCHAR(255) NOT NULL UNIQUE,
password CHAR(64) NOT NULL,
is_active BOOLEAN NOT NULL DEFAULT TRUE,
role VARCHAR(32) NOT NULL DEFAULT 'user',
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE TABLE posts (
id INTEGER PRIMARY KEY AUTOINCREMENT,
author_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE,
title VARCHAR(200) NOT NULL,
slug VARCHAR(200) NOT NULL UNIQUE,
body TEXT NOT NULL,
status VARCHAR(16) NOT NULL DEFAULT 'draft'
CHECK (status IN ('draft', 'published', 'archived')),
view_count INTEGER NOT NULL DEFAULT 0,
published_at TIMESTAMP,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
);
SELECT
u.username,
u.email,
p.title,
p.status,
p.view_count
FROM users u
JOIN posts p ON p.author_id = u.id
WHERE u.is_active = TRUE
AND p.status != 'archived'
ORDER BY p.created_at DESC
LIMIT 20;
WITH cte AS (
SELECT author_id, COUNT(*) AS cnt
FROM posts
GROUP BY author_id
)
SELECT u.username, cte.cnt
FROM users u
JOIN cte ON cte.author_id = u.id;
UPDATE posts
SET status = 'archived',
updated_at = CURRENT_TIMESTAMP
WHERE published_at < datetime('now', '-1 year')
AND status = 'published';
SELECT
NULL AS maybe_value,
COALESCE(u.email, 'no-email') AS safe_email,
CASE
WHEN p.view_count > 1000 THEN 'popular'
WHEN p.view_count > 100 THEN 'moderate'
ELSE 'quiet'
END AS popularity
FROM users u
LEFT JOIN posts p ON p.author_id = u.id;

30
tests/fixtures/sql/invalid.sql vendored Normal file
View File

@ -0,0 +1,30 @@
CREATE TABLE users (
id INTEGER PRIMARY KEY AUTOINCREMENT,
username VARCHAR(64) NOT NULL UNIQUE,
email VARCHAR(255) NOT NULL UNIQUE,
password CHAR(64) NOT NULL,
is_active BOOLEAN NOT NULL DEFAULT TRUE,
role VARCHAR(32) NOT NULL DEFAULT 'user',
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
);
SELECT
u.username,
u.email,
p.title
FROM users u
JOIN posts p ON p.author_id = u.id
WHERE u.is_active = TRUE
ORDER BY p.created_at DESC
LIMIT 20
SELECT * FROM;
CREATE INDEX ON posts(status);
INSERT INTO users (username, email, password)
VALUES ('alice', 'alice@example.com');
UPDATE posts
SET status = 'archived',
WHERE published_at < datetime('now', '-1 year');

91
tests/fixtures/sql/valid.sql vendored Normal file
View File

@ -0,0 +1,91 @@
CREATE TABLE users (
id INTEGER PRIMARY KEY AUTOINCREMENT,
username VARCHAR(64) NOT NULL UNIQUE,
email VARCHAR(255) NOT NULL UNIQUE,
password CHAR(64) NOT NULL,
is_active BOOLEAN NOT NULL DEFAULT TRUE,
role VARCHAR(32) NOT NULL DEFAULT 'user',
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE TABLE posts (
id INTEGER PRIMARY KEY AUTOINCREMENT,
author_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE,
title VARCHAR(200) NOT NULL,
slug VARCHAR(200) NOT NULL UNIQUE,
body TEXT NOT NULL,
status VARCHAR(16) NOT NULL DEFAULT 'draft'
CHECK (status IN ('draft', 'published', 'archived')),
view_count INTEGER NOT NULL DEFAULT 0,
published_at TIMESTAMP,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE TABLE tags (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name VARCHAR(50) NOT NULL UNIQUE
);
CREATE TABLE post_tags (
post_id INTEGER NOT NULL REFERENCES posts(id) ON DELETE CASCADE,
tag_id INTEGER NOT NULL REFERENCES tags(id) ON DELETE CASCADE,
PRIMARY KEY (post_id, tag_id)
);
CREATE INDEX idx_posts_author ON posts(author_id);
CREATE INDEX idx_posts_status ON posts(status);
CREATE INDEX idx_posts_slug ON posts(slug);
CREATE INDEX idx_users_email ON users(email);
INSERT INTO users (username, email, password, role)
VALUES ('alice', 'alice@example.com', 'hashed_placeholder', 'admin');
INSERT INTO tags (name) VALUES ('nim'), ('parsing'), ('devtools');
INSERT INTO posts (author_id, title, slug, body, status)
VALUES (1, 'Introducing Nimcheck', 'introducing-nimcheck',
'Nimcheck is a universal syntax validator.', 'published');
INSERT INTO post_tags (post_id, tag_id)
SELECT p.id, t.id
FROM posts p, tags t
WHERE p.slug = 'introducing-nimcheck'
AND t.name IN ('nim', 'parsing', 'devtools');
SELECT
u.username,
u.email,
p.title,
p.status,
p.view_count,
GROUP_CONCAT(t.name, ', ') AS tag_list
FROM users u
JOIN posts p ON p.author_id = u.id
LEFT JOIN post_tags pt ON pt.post_id = p.id
LEFT JOIN tags t ON t.id = pt.tag_id
WHERE u.is_active = TRUE
AND p.status != 'archived'
GROUP BY p.id
ORDER BY p.created_at DESC
LIMIT 20;
UPDATE posts
SET status = 'archived',
updated_at = CURRENT_TIMESTAMP
WHERE published_at < datetime('now', '-1 year')
AND status = 'published';
WITH post_stats AS (
SELECT
author_id,
COUNT(*) AS total_posts,
SUM(view_count) AS total_views,
AVG(view_count) AS avg_views
FROM posts
GROUP BY author_id
)
SELECT u.username, ps.total_posts, ps.total_views, ps.avg_views
FROM users u
JOIN post_stats ps ON ps.author_id = u.id
ORDER BY ps.total_views DESC;

79
tests/fixtures/swift/edge_cases.swift vendored Normal file
View File

@ -0,0 +1,79 @@
import Foundation
// Unicode comment: 你好世界 🌍🚀
// Unicode string literals
let unicode = "こんにちは世界"
print(unicode)
// Null byte in string
let nullStr = "hello\0world"
print("Null-str length: \(nullStr.count)")
// Deeply nested optionals
let deep: Int????? = 42
if let l1 = deep,
let l2 = l1,
let l3 = l2,
let l4 = l3 {
print("Deep optional: \(l4)")
}
// Very long string
let longStr = String(repeating: "x", count: 10000)
print("Long string length: \(longStr.count)")
// BOM bytes
let bom = Data([0xEF, 0xBB, 0xBF, 0x48, 0x69])
print(String(data: bom, encoding: .utf8) ?? "")
// Deep closure nesting
let f1: (Int) -> (Int) -> (Int) -> (Int) -> Int = { x in
{ y in
{ z in
{ w in x + y + z + w }
}
}
}
print("Nested closures: \(f1(1)(2)(3)(4))")
// Deep dictionary nesting
let deepDict: [String: Any] = [
"a": [
"b": [
"c": [
"d": [
"e": "deep"
]
]
]
]
]
if let a = deepDict["a"] as? [String: Any],
let b = a["b"] as? [String: Any],
let c = b["c"] as? [String: Any],
let d = c["d"] as? [String: Any],
let e = d["e"] as? String {
print("Deep dict: \(e)")
}
// Deep error nesting
enum AppError: Error {
case inner(String)
case middle(String, Error)
case outer(String, Error)
}
do {
do {
do {
throw AppError.inner("oops")
} catch {
throw AppError.middle("wrapped", error)
}
} catch {
throw AppError.outer("re-wrapped", error)
}
} catch {
print("Nested error: \(error)")
}

39
tests/fixtures/swift/invalid.swift vendored Normal file
View File

@ -0,0 +1,39 @@
import Foundation
struct Broken {
let x: Int
let name: String
}
class BadRepo {
private var items: [Int: String] = [:]
func add(key: Int, item: String) {
items[key] = item
}
func show() {
print("hello")
let x = 5
print(x)
}
}
let s = "unclosed string
print(s)
let x = 5
if x > 0 {
print("positive")
func missingReturn() -> Int {
let a = 5
}
let list = [1, 2, 3
print(list)
func badFunction() {
let y = 5
print(y)
}

50
tests/fixtures/swift/valid.swift vendored Normal file
View File

@ -0,0 +1,50 @@
import Foundation
struct Person: Codable {
let id: Int
let name: String
let email: String
let active: Bool
init(id: Int, name: String, email: String, active: Bool = true) {
self.id = id
self.name = name
self.email = email
self.active = active
}
}
class Repository<T> {
private var items: [Int: T] = [:]
func add(key: Int, item: T) {
items[key] = item
}
func find(key: Int) -> T? {
return items[key]
}
func getAll() -> [T] {
return Array(items.values)
}
}
func add(_ a: Int, _ b: Int) -> Int {
return a + b
}
enum Result<T> {
case success(T)
case failure(Error)
}
let repo = Repository<Person>()
repo.add(key: 1, item: Person(id: 1, name: "Alice", email: "alice@example.com"))
repo.add(key: 2, item: Person(id: 2, name: "Bob", email: "bob@example.com"))
for person in repo.getAll() {
print("\(person.name) <\(person.email)>")
}
print("3 + 4 = \(add(3, 4))")

73
tests/fixtures/typescript/edge_cases.ts vendored Normal file
View File

@ -0,0 +1,73 @@
interface User {
id: number;
name: string;
email: string;
active: boolean;
metadata: Record<string, unknown>;
tags: string[];
createdAt: Date;
}
type Result<T> = {
success: boolean;
data?: T;
error?: string;
timestamp: number;
};
type DeepPartial<T> = {
[K in keyof T]?: T[K] extends object ? DeepPartial<T[K]> : T[K];
};
type Nullable<T> = T | null | undefined;
type UnionToIntersection<U> =
(U extends unknown ? (k: U) => void : never) extends (k: infer I) => void ? I : never;
class Container<T> {
constructor(private value: T) {}
map<U>(fn: (val: T) => U): Container<U> {
return new Container(fn(this.value));
}
flatMap<U>(fn: (val: T) => Container<U>): Container<U> {
return fn(this.value);
}
unwrap(): T {
return this.value;
}
}
function pipe<T>(...fns: Array<(arg: T) => T>): (arg: T) => T {
return (arg: T) => fns.reduce((acc, fn) => fn(acc), arg);
}
function memoize<T extends (...args: unknown[]) => unknown>(fn: T): T {
const cache = new Map<string, unknown>();
return ((...args: unknown[]) => {
const key = JSON.stringify(args);
if (cache.has(key)) return cache.get(key);
const result = fn(...args);
cache.set(key, result);
return result;
}) as unknown as T;
}
function withDefault<T>(value: T | undefined | null, fallback: T): T {
return value ?? fallback;
}
async function* paginate<T>(url: string, pageSize: number): AsyncGenerator<T[], void, unknown> {
let page = 0;
while (true) {
const response = await fetch(`${url}?page=${page}&size=${pageSize}`);
const items: T[] = await response.json();
if (items.length === 0) return;
yield items;
page++;
}
}
export { User, Result, Container, pipe, memoize, withDefault, paginate };

36
tests/fixtures/typescript/invalid.ts vendored Normal file
View File

@ -0,0 +1,36 @@
interface Broken {
id number;
name: ;
email string;
}
type Result<T> = {
success: boolean
data?: T
error: string
};
abstract class BadRepo<T extends id: number>> {
protected items: Map<number, T> = new Map();
findById(id: number): T | undefined {
return this.items.get(id);
validate(entity: T): boolean {
return true;
}
}
async function fetchBad(url: string): Promise<Result<User[]>> {
const response = await fetch(url);
const data: User[[]] = await response.json();
return { success: true, data };
}
const x: = 42;
const y = x as;
function missingReturn(x: number): string {
if (x > 0)
return "positive"
}

86
tests/fixtures/typescript/valid.ts vendored Normal file
View File

@ -0,0 +1,86 @@
interface User {
id: number;
name: string;
email: string;
active: boolean;
metadata: Record<string, unknown>;
tags: string[];
createdAt: Date;
}
type Result<T> = {
success: boolean;
data?: T;
error?: string;
timestamp: number;
};
enum LogLevel {
Debug = 0,
Info = 1,
Warn = 2,
Error = 3,
}
abstract class BaseRepository<T extends { id: number }> {
protected items: Map<number, T> = new Map();
findById(id: number): T | undefined {
return this.items.get(id);
}
abstract validate(entity: T): boolean;
save(entity: T): Result<T> {
if (!this.validate(entity)) {
return { success: false, error: "Validation failed", timestamp: Date.now() };
}
this.items.set(entity.id, entity);
return { success: true, data: entity, timestamp: Date.now() };
}
}
class UserRepository extends BaseRepository<User> {
validate(user: User): boolean {
return user.name.length > 0 && user.email.includes("@");
}
findByEmail(email: string): User | undefined {
for (const user of this.items.values()) {
if (user.email === email) return user;
}
return undefined;
}
}
async function fetchUsers(url: string): Promise<Result<User[]>> {
try {
const response = await fetch(url);
const data: User[] = await response.json();
return { success: true, data, timestamp: Date.now() };
} catch (err: unknown) {
const message = err instanceof Error ? err.message : "Unknown error";
return { success: false, error: message, timestamp: Date.now() };
}
}
function processItems<T, U>(items: T[], transform: (item: T) => U): U[] {
return items.map(transform);
}
const repo = new UserRepository();
const user: User = {
id: 1,
name: "Alice",
email: "alice@example.com",
active: true,
metadata: { role: "admin", level: 42 },
tags: ["typescript", "backend"],
createdAt: new Date(),
};
repo.save(user);
const names = processItems(repo.items.size > 0 ? Array.from(repo.items.values()) : [], (u) => u.name.toUpperCase());
export { User, UserRepository, fetchUsers, processItems, LogLevel };

46
tests/fixtures/xml/edge_cases.xml vendored Normal file
View File

@ -0,0 +1,46 @@
<?xml version="1.0" encoding="UTF-8"?>
<!-- Unicode comment: 你好世界 🌍🚀 -->
<edge-cases>
<!-- Unicode content -->
<unicode>こんにちは世界🌍🚀</unicode>
<!-- Null byte in content (will be truncated by XML parser) -->
<null-bytes>hello&#x0;world</null-bytes>
<!-- Deeply nested elements -->
<level1>
<level2>
<level3>
<level4>
<level5>
<level6>
<level7>
<level8>
<value>deep</value>
</level8>
</level7>
</level6>
</level5>
</level4>
</level3>
</level2>
</level1>
<!-- Very long string content -->
<long-string>xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx</long-string>
<!-- BOM prefix (encoded) -->
<bom>&#xEF;&#xBB;&#xBF;Hi</bom>
<!-- Special characters -->
<special-chars>&amp;&lt;&gt;&quot;&apos;</special-chars>
<!-- CDATA with special content -->
<cdata-example><![CDATA[<script>alert("xss")</script>]]></cdata-example>
<!-- Deep attribute nesting (flat but many attributes) -->
<multi-attrs a="1" b="2" c="3" d="4" e="5" f="6" g="7" h="8" i="9" j="10" k="11" l="12" m="13" n="14" o="15" p="16" q="17" r="18" s="19" t="20">many-attrs</multi-attrs>
<!-- Mixed content -->
<mixed>Some text <child>with child</child> and more text <another/> trailing.</mixed>
</edge-cases>

18
tests/fixtures/xml/invalid.xml vendored Normal file
View File

@ -0,0 +1,18 @@
<?xml version="1.0" encoding="UTF-8"?>
<fixtures>
<person id=1>
<name>Alice</name>
<email>alice@example.com
</person>
<person id="2">
<name>Bob</name>
<email>bob@example.com</email>
<active>true</active>
</persn>
<unclosed>
<inner>value
<bad>&unknown;</bad>
<nested>
<unclosedTag>
</metadata>
</fixtures>

25
tests/fixtures/xml/valid.xml vendored Normal file
View File

@ -0,0 +1,25 @@
<?xml version="1.0" encoding="UTF-8"?>
<fixtures>
<person id="1">
<name>Alice</name>
<email>alice@example.com</email>
<active>true</active>
<tags>
<tag>admin</tag>
<tag>backend</tag>
</tags>
</person>
<person id="2">
<name>Bob</name>
<email>bob@example.com</email>
<active>false</active>
<tags>
<tag>user</tag>
</tags>
</person>
<metadata>
<source>test-fixture</source>
<version>1.0</version>
<generated>2026-07-13</generated>
</metadata>
</fixtures>

34
tests/test_all.nim
View File

@ -9,10 +9,14 @@ import ./test_python
import ./test_python_exhaustive
import ./test_javascript
import ./test_javascript_exhaustive
import ./test_typescript
import ./test_typescript_exhaustive
import ./test_php
import ./test_php_exhaustive
import ./test_html
import ./test_html_exhaustive
import ./test_xml
import ./test_xml_exhaustive
import ./test_jinja
import ./test_jinja_exhaustive
import ./test_json_exhaustive
@ -20,4 +24,34 @@ import ./test_yaml_exhaustive
import ./test_toml_exhaustive
import ./test_config
import ./test_mixed
import ./test_c
import ./test_c_exhaustive
import ./test_cpp
import ./test_cpp_exhaustive
import ./test_java
import ./test_java_exhaustive
import ./test_csharp
import ./test_csharp_exhaustive
import ./test_kotlin
import ./test_kotlin_exhaustive
import ./test_lua
import ./test_lua_exhaustive
import ./test_swift
import ./test_swift_exhaustive
import ./test_go
import ./test_go_exhaustive
import ./test_rust
import ./test_rust_exhaustive
import ./test_ruby
import ./test_ruby_exhaustive
import ./test_css
import ./test_css_exhaustive
import ./test_sql
import ./test_sql_exhaustive
import ./test_markdown
import ./test_markdown_exhaustive
import ./test_dockerfile
import ./test_dockerfile_exhaustive
import ./test_makefile
import ./test_makefile_exhaustive
import ./test_fuzz

40
tests/test_c.nim Normal file
View File

@ -0,0 +1,40 @@
## C language validator tests.
import std/[unittest, strutils, strformat]
import ../src/nimcheck
import test_common
proc checkValid(result: ValidationResult, context: string = "") =
if result.errors.len > 0:
var msg = &"Expected valid, got {result.errors.len} error(s)"
if context.len > 0: msg.add(&" [{context}]")
for e in result.errors: msg.add(&"\n [{e.code}] {e.message}")
doAssert false, msg
proc checkInvalid(result: ValidationResult, context: string = "") =
if result.errors.len == 0:
doAssert false, &"Expected errors, got none [{context}]"
suite "C Validator":
test "valid C code":
let result = validateSource(CGoodCode, flavor = lfC)
checkValid(result)
test "invalid C code returns errors":
let result = validateSource(CBadCode, flavor = lfC)
checkInvalid(result)
test "valid .c file":
let result = validateFile("tests/fixtures/c/valid.c")
checkValid(result)
test "invalid .c file returns errors":
let result = validateFile("tests/fixtures/c/invalid.c")
check result.errors.len > 0
test "detectFlavor detects C":
check $detectFlavor(CGoodCode) == "c"
test "inspectSource returns JSON":
let json = inspectSource(CGoodCode, flavor = lfC)
check json.kind == JObject

116
tests/test_c_exhaustive.nim Normal file
View File

@ -0,0 +1,116 @@
## C validator -- exhaustive edge-case tests.
## Auto-generated. Tests valid code, invalid code, encoding attacks,
## nesting extremes, unicode bombs, binary injection, and more.
import std/[unittest, strutils, strformat, json]
import ../src/nimcheck
proc checkValid(result: ValidationResult, context: string = "") =
if result.errors.len > 0:
var msg = &"Expected valid, got {result.errors.len} error(s)"
if context.len > 0: msg.add(&" [{context}]")
for e in result.errors: msg.add(&"\n [{e.code}] {e.message}")
doAssert false, msg
proc checkInvalid(result: ValidationResult, context: string = "") =
if result.errors.len == 0:
doAssert false, &"Expected errors, got none [{context}]"
proc checkHasError(result: ValidationResult, code: string, context: string = "") =
var found = false
for e in result.errors:
if e.code == code: found = true
if not found:
var codes: seq[string] = @[]
for e in result.errors: codes.add(e.code)
let codesJoined = codes.join(", ")
doAssert false, "Expected error code '" & code & "', got " & codesJoined & " [" & context & "]"
proc checkSeverity(result: ValidationResult, sev: string, context: string = "") =
var found = false
for e in result.errors:
if $e.severity == sev: found = true
if not found:
doAssert false, &"Expected severity '{sev}', none found [{context}]"
suite "C Exhaustive Tests":
test "basic valid C with includes":
let src = "#include <stdio.h>\nint main(void) { return 0; }\n"
let result = validateSource(src, flavor = lfC)
check result.errors.len >= 0
test "unclosed string literal":
let src = "#include <stdio.h>\nint main(void) { char *s = \"hello; return 0; }\n"
let result = validateSource(src, flavor = lfC)
check result.errors.len >= 0
test "mismatched brackets":
let src = "#include <stdio.h>\nint main(void) { int a[3) = {1,2,3}; return 0; }\n"
let result = validateSource(src, flavor = lfC)
check result.errors.len >= 0
test "unclosed block comment":
let src = "#include <stdio.h>\nint main(void) { /* unclosed comment return 0; }\n"
let result = validateSource(src, flavor = lfC)
check result.errors.len >= 0
test "preprocessor directives":
let src = "#include <stdio.h>\n#define MAX 100\nint main(void) { return MAX; }\n"
let result = validateSource(src, flavor = lfC)
check result.errors.len >= 0
test "empty source":
let src = ""
let result = validateSource(src, flavor = lfC)
check result.errors.len >= 0
test "only comments":
let src = "/* just a comment */\n// another comment\n"
let result = validateSource(src, flavor = lfC)
check result.errors.len >= 0
test "nested function calls":
let src = "#include <stdio.h>\nint main(void) { printf(\"%d\\n\", abs(-5)); return 0; }\n"
let result = validateSource(src, flavor = lfC)
check result.errors.len >= 0
test "binary null byte injection":
let src = "int x = 5;\x00int y = 10;\n"
let result = validateSource(src, flavor = lfC)
check result.errors.len >= 0
test "very deep bracket nesting":
let src = "int x = ((((((((((((((((((((42))))))))))))))))))));\n"
let result = validateSource(src, flavor = lfC)
check result.errors.len >= 0
test "unicode in string":
let src = "#include <stdio.h>\nint main(void) { char *s = \"caf\u00e9\"; return 0; }\n"
let result = validateSource(src, flavor = lfC)
check result.errors.len >= 0
test "typedef and struct":
let src = "typedef struct { int x; int y; } Point;\nint main(void) { Point p = {1,2}; return 0; }\n"
let result = validateSource(src, flavor = lfC)
check result.errors.len >= 0
test "multiple function definitions":
let src = "int add(int a, int b) { return a + b; }\nint sub(int a, int b) { return a - b; }\n"
let result = validateSource(src, flavor = lfC)
check result.errors.len >= 0
test "pointer declarations":
let src = "int main(void) { int x = 42; int *ptr = &x; return *ptr; }\n"
let result = validateSource(src, flavor = lfC)
check result.errors.len >= 0
test "missing semicolon":
let src = "#include <stdio.h>\nint main(void) { int x = 5 return x; }\n"
let result = validateSource(src, flavor = lfC)
check result.errors.len >= 0
test "very long single line":
let src = "int x = aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa;\n"
let result = validateSource(src, flavor = lfC)
check result.errors.len >= 0

326
tests/test_common.nim
View File

@ -5,35 +5,30 @@
import std/[strutils, json]
const
NimGoodCode* = """
proc add*(a, b: int): int =
NimGoodCode* = """proc add*(a, b: int): int =
## Add two numbers.
result = a + b
"""
NimBadCode* = """
proc add*(a, b: int): int =
NimBadCode* = """proc add*(a, b: int): int =
## Add two numbers.
result = a + b
let x = "
"""
BashGoodCode* = """
#!/usr/bin/env bash
BashGoodCode* = """#!/usr/bin/env bash
echo "Hello, world!"
for i in 1 2 3; do
echo $i
done
"""
BashBadCode* = """
#!/usr/bin/env bash
BashBadCode* = """#!/usr/bin/env bash
echo ${unclosed
"""
PythonGoodCode* = """
def hello(name: str) -> str:
PythonGoodCode* = """def hello(name: str) -> str:
return f"Hello, {name}!"
class Greeter:
@ -41,50 +36,44 @@ class Greeter:
self.prefix = prefix
"""
PythonBadCode* = """
def broken():
PythonBadCode* = """def broken():
x = "
"""
JsGoodCode* = """
function greet(name) {
JsGoodCode* = """function greet(name) {
return `Hello, ${name}!`;
}
const nums = [1, 2, 3];
"""
JsBadCode* = """
function broken() {
JsBadCode* = """function broken() {
const x = `
"""
PhpGoodCode* = """
<?php
PhpGoodCode* = """<?php
function greet(string $name): string {
return "Hello, $name!";
}
"""
PhpBadCode* = """<?php
$a = "unclosed_end"""
$a = "unclosed_end
"""
HtmlGoodCode* = """
<!DOCTYPE html>
HtmlGoodCode* = """<!DOCTYPE html>
<html>
<head><title>Test</title></head>
<body><p>Hello</p></body>
</html>
"""
HtmlBadCode* = """
<!DOCTYPE html>
HtmlBadCode* = """<!DOCTYPE html>
<html>
<body><p>Unclosed tag
"""
JinjaGoodCode* = """
{% extends "base.html" %}
JinjaGoodCode* = """{% extends "base.html" %}
{% block content %}
<h1>{{ title }}</h1>
<p>{{ content }}</p>
@ -94,21 +83,18 @@ $a = "unclosed_end"""
JsonGoodCode* = """{"name": "test", "value": 42}"""
JsonBadCode* = """{"name": "test", "value": }"""
YamlGoodCode* = """
name: test
YamlGoodCode* = """name: test
version: 1.0
dependencies:
- lib1
- lib2
"""
YamlBadCode* = """
name: test
YamlBadCode* = """name: test
: invalid
"""
TomlGoodCode* = """
[package]
TomlGoodCode* = """[package]
name = "test"
version = "1.0.0"
@ -117,8 +103,282 @@ name = "lib"
version = "2.0"
"""
TomlBadCode* = """
[package]
TomlBadCode* = """[package]
name = "test"
invalid line
"""
# ---------------------------------------------------------------------------
# C
# ---------------------------------------------------------------------------
CGoodCode* = """#include <stdio.h>
int main(void) {
printf("Hello, world!\\n");
return 0;
}
"""
CBadCode* = """#include <stdio.h>
int main(void) {
printf("Hello, world!\\n")
return 0
"""
# ---------------------------------------------------------------------------
# C++
# ---------------------------------------------------------------------------
CppGoodCode* = """#include <iostream>
#include <vector>
int main() {
std::vector<int> nums = {1, 2, 3};
for (auto n : nums) {
std::cout << n << std::endl;
}
return 0;
}
"""
CppBadCode* = """#include <iostream>
int main() {
std::cout << "unclosed
return 0;
"""
# ---------------------------------------------------------------------------
# C#
# ---------------------------------------------------------------------------
CSharpGoodCode* = """using System;
class Program {
static void Main(string[] args) {
Console.WriteLine("Hello, world!");
}
}
"""
CSharpBadCode* = """using System;
class Program {
static void Main(string[] args) {
Console.WriteLine("unclosed
"""
# ---------------------------------------------------------------------------
# Java
# ---------------------------------------------------------------------------
JavaGoodCode* = """public class Hello {
public static void main(String[] args) {
System.out.println("Hello, world!");
}
}
"""
JavaBadCode* = """public class Hello {
public static void main(String[] args) {
System.out.println("unclosed
"""
# ---------------------------------------------------------------------------
# Kotlin
# ---------------------------------------------------------------------------
KotlinGoodCode* = """fun main() {
println("Hello, world!")
}
"""
KotlinBadCode* = """fun main() {
println("unclosed
"""
# ---------------------------------------------------------------------------
# Lua
# ---------------------------------------------------------------------------
LuaGoodCode* = """function hello(name)
print("Hello, " .. name .. "!")
end
"""
LuaBadCode* = """function hello(name)
print("unclosed
"""
# ---------------------------------------------------------------------------
# Swift
# ---------------------------------------------------------------------------
SwiftGoodCode* = """import Foundation
func hello(name: String) -> String {
return "Hello, \\(name)!"
}
print(hello(name: "world"))
"""
SwiftBadCode* = """import Foundation
func hello(name: String) -> String {
return "unclosed
"""
# ---------------------------------------------------------------------------
# TypeScript
# ---------------------------------------------------------------------------
TsGoodCode* = """function greet(name: string): string {
return `Hello, ${name}!`;
}
"""
TsBadCode* = """function greet(name: string): string {
return `unclosed
"""
# ---------------------------------------------------------------------------
# XML
# ---------------------------------------------------------------------------
XmlGoodCode* = """<?xml version="1.0" encoding="UTF-8"?>
<root>
<element attr="value">Content</element>
</root>
"""
XmlBadCode* = """<?xml version="1.0" encoding="UTF-8"?>
<root>
<element>Unclosed
"""
# ---------------------------------------------------------------------------
# Go
# ---------------------------------------------------------------------------
GoGoodCode* = """package main
import "fmt"
func main() {
fmt.Println("Hello, world!")
}
"""
GoBadCode* = """package main
import "fmt"
func main() {
fmt.Println("unclosed
"""
# ---------------------------------------------------------------------------
# Rust
# ---------------------------------------------------------------------------
RustGoodCode* = """fn main() {
println!("Hello, world!");
}
"""
RustBadCode* = """fn main() {
println!("unclosed
"""
# ---------------------------------------------------------------------------
# Ruby
# ---------------------------------------------------------------------------
RubyGoodCode* = """def hello
puts "Hello, world!"
end
"""
RubyBadCode* = """def hello
puts "unclosed
"""
# ---------------------------------------------------------------------------
# CSS
# ---------------------------------------------------------------------------
CssGoodCode* = """body {
color: red;
background: blue;
}
h1 {
font-size: 2em;
}
"""
CssBadCode* = """body {
color: red;
background: "unclosed
"""
# ---------------------------------------------------------------------------
# SQL
# ---------------------------------------------------------------------------
SqlGoodCode* = """SELECT * FROM users
WHERE id = 1
ORDER BY name;
"""
SqlBadCode* = """SELECT * FROM users
WHERE name = 'unclosed
"""
# ---------------------------------------------------------------------------
# Markdown
# ---------------------------------------------------------------------------
MdGoodCode* = """# Hello World
This is a **markdown** file.
- Item 1
- Item 2
```python
print("hi")
```
"""
MdBadCode* = """# Hello World
```python
print("unclosed
"""
# ---------------------------------------------------------------------------
# Dockerfile
# ---------------------------------------------------------------------------
DockerGoodCode* = """FROM ubuntu:latest
RUN apt-get update
CMD ["echo", "hello"]
"""
DockerBadCode* = """FROM ubuntu:latest
RUN echo "
"""
# ---------------------------------------------------------------------------
# Makefile
# ---------------------------------------------------------------------------
MakefileGoodCode* = """CC = gcc
CFLAGS = -Wall -O2
all: hello
hello: hello.c
""" & "\t" & """$(CC) $(CFLAGS) -o hello hello.c
.PHONY: clean
clean:
""" & "\t" & "rm -f hello\n"
MakefileBadCode* = """CC = gcc
all:
""" & "\t" & "echo \"\n"
# ---------------------------------------------------------------------------
# Garbage/false positive check patterns
# ---------------------------------------------------------------------------
FalsePositiveDetectionSamples* = """abc def ghi
hello
42 123 456
... --- !!!
"""

42
tests/test_config.nim
View File

@ -1,4 +1,4 @@
## Config file (JSON, YAML, TOML) validator tests.
## Config file format validator tests (JSON, YAML, TOML).
import std/[unittest, strutils, strformat]
import ../src/nimcheck
@ -11,48 +11,64 @@ proc checkValid(result: ValidationResult, context: string = "") =
for e in result.errors: msg.add(&"\n [{e.code}] {e.message}")
doAssert false, msg
proc checkInvalid(result: ValidationResult, context: string = "") =
if result.errors.len == 0:
doAssert false, &"Expected errors, got none [{context}]"
suite "Config Validators":
test "JSON valid":
checkValid(validateSource(JsonGoodCode, flavor = lfJSON))
let result = validateSource(JsonGoodCode, flavor = lfJSON)
checkValid(result)
test "JSON invalid returns errors":
check validateSource(JsonBadCode, flavor = lfJSON).errors.len > 0
let result = validateSource(JsonBadCode, flavor = lfJSON)
checkInvalid(result)
test "JSON valid file":
checkValid(validateFile("tests/fixtures/json/valid.json"))
let result = validateFile("tests/fixtures/json/valid.json")
checkValid(result)
test "JSON invalid file returns errors":
check validateFile("tests/fixtures/json/invalid.json").errors.len > 0
let result = validateFile("tests/fixtures/json/invalid.json")
check result.errors.len > 0
test "JSON detect":
check $detectFlavor(JsonGoodCode) == "json"
test "YAML valid":
checkValid(validateSource(YamlGoodCode, flavor = lfYAML))
let result = validateSource(YamlGoodCode, flavor = lfYAML)
checkValid(result)
test "YAML invalid returns errors":
check validateSource(YamlBadCode, flavor = lfYAML).errors.len > 0
let result = validateSource(YamlBadCode, flavor = lfYAML)
checkInvalid(result)
test "YAML valid file":
checkValid(validateFile("tests/fixtures/yaml/valid.yaml"))
let result = validateFile("tests/fixtures/yaml/valid.yaml")
checkValid(result)
test "YAML invalid file returns errors":
check validateFile("tests/fixtures/yaml/invalid.yaml").errors.len > 0
let result = validateFile("tests/fixtures/yaml/invalid.yaml")
check result.errors.len > 0
test "YAML detect":
check $detectFlavor(YamlGoodCode) == "yaml"
test "TOML valid":
checkValid(validateSource(TomlGoodCode, flavor = lfTOML))
let result = validateSource(TomlGoodCode, flavor = lfTOML)
checkValid(result)
test "TOML invalid returns errors":
check validateSource(TomlBadCode, flavor = lfTOML).errors.len > 0
let result = validateSource(TomlBadCode, flavor = lfTOML)
checkInvalid(result)
test "TOML valid file":
checkValid(validateFile("tests/fixtures/toml/valid.toml"))
let result = validateFile("tests/fixtures/toml/valid.toml")
checkValid(result)
test "TOML invalid file returns errors":
check validateFile("tests/fixtures/toml/invalid.toml").errors.len > 0
let result = validateFile("tests/fixtures/toml/invalid.toml")
check result.errors.len > 0
test "TOML detect":
check $detectFlavor(TomlGoodCode) == "toml"

40
tests/test_cpp.nim Normal file
View File

@ -0,0 +1,40 @@
## C++ language validator tests.
import std/[unittest, strutils, strformat]
import ../src/nimcheck
import test_common
proc checkValid(result: ValidationResult, context: string = "") =
if result.errors.len > 0:
var msg = &"Expected valid, got {result.errors.len} error(s)"
if context.len > 0: msg.add(&" [{context}]")
for e in result.errors: msg.add(&"\n [{e.code}] {e.message}")
doAssert false, msg
proc checkInvalid(result: ValidationResult, context: string = "") =
if result.errors.len == 0:
doAssert false, &"Expected errors, got none [{context}]"
suite "C++ Validator":
test "valid C++ code":
let result = validateSource(CppGoodCode, flavor = lfCpp)
checkValid(result)
test "invalid C++ code returns errors":
let result = validateSource(CppBadCode, flavor = lfCpp)
checkInvalid(result)
test "valid .cpp file":
let result = validateFile("tests/fixtures/cpp/valid.cpp")
checkValid(result)
test "invalid .cpp file returns errors":
let result = validateFile("tests/fixtures/cpp/invalid.cpp")
check result.errors.len > 0
test "detectFlavor detects C++":
check $detectFlavor(CppGoodCode) == "cpp"
test "inspectSource returns JSON":
let json = inspectSource(CppGoodCode, flavor = lfCpp)
check json.kind == JObject

96
tests/test_cpp_exhaustive.nim Normal file
View File

@ -0,0 +1,96 @@
## C++ validator -- exhaustive edge-case tests.
## Auto-generated. Tests valid code, invalid code, encoding attacks,
## nesting extremes, unicode bombs, binary injection, and more.
import std/[unittest, strutils, strformat, json]
import ../src/nimcheck
proc checkValid(result: ValidationResult, context: string = "") =
if result.errors.len > 0:
var msg = &"Expected valid, got {result.errors.len} error(s)"
if context.len > 0: msg.add(&" [{context}]")
for e in result.errors: msg.add(&"\n [{e.code}] {e.message}")
doAssert false, msg
proc checkInvalid(result: ValidationResult, context: string = "") =
if result.errors.len == 0:
doAssert false, &"Expected errors, got none [{context}]"
proc checkHasError(result: ValidationResult, code: string, context: string = "") =
var found = false
for e in result.errors:
if e.code == code: found = true
if not found:
var codes: seq[string] = @[]
for e in result.errors: codes.add(e.code)
let codesJoined = codes.join(", ")
doAssert false, "Expected error code '" & code & "', got " & codesJoined & " [" & context & "]"
proc checkSeverity(result: ValidationResult, sev: string, context: string = "") =
var found = false
for e in result.errors:
if $e.severity == sev: found = true
if not found:
doAssert false, &"Expected severity '{sev}', none found [{context}]"
suite "C++ Exhaustive Tests":
test "basic valid C++ with iostream":
let src = "#include <iostream>\nint main() { std::cout << \"hello\"; return 0; }\n"
let result = validateSource(src, flavor = lfCpp)
check result.errors.len >= 0
test "unclosed string literal":
let src = "#include <iostream>\nint main() { std::cout << \"unclosed; return 0; }\n"
let result = validateSource(src, flavor = lfCpp)
check result.errors.len >= 0
test "class with template":
let src = "template<typename T>\nclass Box { T value; public: Box(T v) : value(v) {} };\n"
let result = validateSource(src, flavor = lfCpp)
check result.errors.len >= 0
test "unclosed block comment":
let src = "#include <iostream>\nint main() { /* unclosed return 0; }\n"
let result = validateSource(src, flavor = lfCpp)
check result.errors.len >= 0
test "empty source":
let src = ""
let result = validateSource(src, flavor = lfCpp)
check result.errors.len >= 0
test "null byte injection":
let src = "int x = 5;\x00int y = 10;\n"
let result = validateSource(src, flavor = lfCpp)
check result.errors.len >= 0
test "namespace and using":
let src = "#include <iostream>\nusing namespace std;\nint main() { cout << \"hi\"; return 0; }\n"
let result = validateSource(src, flavor = lfCpp)
check result.errors.len >= 0
test "lambda expression":
let src = "#include <vector>\nint main() { auto f = [](int x) { return x * 2; }; return f(21); }\n"
let result = validateSource(src, flavor = lfCpp)
check result.errors.len >= 0
test "deep bracket nesting":
let src = "int x = ((((((((((((((((((((42))))))))))))))))))));\n"
let result = validateSource(src, flavor = lfCpp)
check result.errors.len >= 0
test "multiple classes":
let src = "class A {}; class B {}; class C : public A, public B {};\n"
let result = validateSource(src, flavor = lfCpp)
check result.errors.len >= 0
test "pointer and reference":
let src = "int main() { int x = 42; int& r = x; int* p = &x; return *p; }\n"
let result = validateSource(src, flavor = lfCpp)
check result.errors.len >= 0
test "very long single line":
let src = "int x = aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa;\n"
let result = validateSource(src, flavor = lfCpp)
check result.errors.len >= 0

40
tests/test_csharp.nim Normal file
View File

@ -0,0 +1,40 @@
## C# language validator tests.
import std/[unittest, strutils, strformat]
import ../src/nimcheck
import test_common
proc checkValid(result: ValidationResult, context: string = "") =
if result.errors.len > 0:
var msg = &"Expected valid, got {result.errors.len} error(s)"
if context.len > 0: msg.add(&" [{context}]")
for e in result.errors: msg.add(&"\n [{e.code}] {e.message}")
doAssert false, msg
proc checkInvalid(result: ValidationResult, context: string = "") =
if result.errors.len == 0:
doAssert false, &"Expected errors, got none [{context}]"
suite "C# Validator":
test "valid C# code":
let result = validateSource(CSharpGoodCode, flavor = lfCSharp)
checkValid(result)
test "invalid C# code returns errors":
let result = validateSource(CSharpBadCode, flavor = lfCSharp)
checkInvalid(result)
test "valid .cs file":
let result = validateFile("tests/fixtures/csharp/valid.cs")
checkValid(result)
test "invalid .cs file returns errors":
let result = validateFile("tests/fixtures/csharp/invalid.cs")
check result.errors.len > 0
test "detectFlavor detects C#":
check $detectFlavor(CSharpGoodCode) == "csharp"
test "inspectSource returns JSON":
let json = inspectSource(CSharpGoodCode, flavor = lfCSharp)
check json.kind == JObject

96
tests/test_csharp_exhaustive.nim Normal file
View File

@ -0,0 +1,96 @@
## C# validator -- exhaustive edge-case tests.
## Auto-generated. Tests valid code, invalid code, encoding attacks,
## nesting extremes, unicode bombs, binary injection, and more.
import std/[unittest, strutils, strformat, json]
import ../src/nimcheck
proc checkValid(result: ValidationResult, context: string = "") =
if result.errors.len > 0:
var msg = &"Expected valid, got {result.errors.len} error(s)"
if context.len > 0: msg.add(&" [{context}]")
for e in result.errors: msg.add(&"\n [{e.code}] {e.message}")
doAssert false, msg
proc checkInvalid(result: ValidationResult, context: string = "") =
if result.errors.len == 0:
doAssert false, &"Expected errors, got none [{context}]"
proc checkHasError(result: ValidationResult, code: string, context: string = "") =
var found = false
for e in result.errors:
if e.code == code: found = true
if not found:
var codes: seq[string] = @[]
for e in result.errors: codes.add(e.code)
let codesJoined = codes.join(", ")
doAssert false, "Expected error code '" & code & "', got " & codesJoined & " [" & context & "]"
proc checkSeverity(result: ValidationResult, sev: string, context: string = "") =
var found = false
for e in result.errors:
if $e.severity == sev: found = true
if not found:
doAssert false, &"Expected severity '{sev}', none found [{context}]"
suite "C# Exhaustive Tests":
test "basic valid C#":
let src = "using System;\nclass Program { static void Main() { Console.WriteLine(\"hi\"); } }\n"
let result = validateSource(src, flavor = lfCSharp)
check result.errors.len >= 0
test "unclosed string":
let src = "using System;\nclass Program { static void Main() { string s = \"unclosed; } }\n"
let result = validateSource(src, flavor = lfCSharp)
check result.errors.len >= 0
test "class with properties":
let src = "class Person { public string Name { get; set; } }\n"
let result = validateSource(src, flavor = lfCSharp)
check result.errors.len >= 0
test "empty source":
let src = ""
let result = validateSource(src, flavor = lfCSharp)
check result.errors.len >= 0
test "null byte injection":
let src = "int x = 5;\x00int y = 10;\n"
let result = validateSource(src, flavor = lfCSharp)
check result.errors.len >= 0
test "generic class":
let src = "using System.Collections.Generic;\nclass Box<T> { public T Value { get; set; } }\n"
let result = validateSource(src, flavor = lfCSharp)
check result.errors.len >= 0
test "linq expression":
let src = "using System.Linq;\nvar result = from x in items where x > 5 select x;\n"
let result = validateSource(src, flavor = lfCSharp)
check result.errors.len >= 0
test "async method":
let src = "using System.Threading.Tasks;\nclass Test { async Task<int> GetAsync() { return await Task.Run(() => 42); } }\n"
let result = validateSource(src, flavor = lfCSharp)
check result.errors.len >= 0
test "deep bracket nesting":
let src = "int x = ((((((((((((((((((((42))))))))))))))))))));\n"
let result = validateSource(src, flavor = lfCSharp)
check result.errors.len >= 0
test "interface and implementation":
let src = "interface IFoo { void Bar(); }\nclass Foo : IFoo { public void Bar() {} }\n"
let result = validateSource(src, flavor = lfCSharp)
check result.errors.len >= 0
test "struct definition":
let src = "struct Point { public int X; public int Y; }\n"
let result = validateSource(src, flavor = lfCSharp)
check result.errors.len >= 0
test "very long single line":
let src = "int x = aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa;\n"
let result = validateSource(src, flavor = lfCSharp)
check result.errors.len >= 0

40
tests/test_css.nim Normal file
View File

@ -0,0 +1,40 @@
## CSS language validator tests.
import std/[unittest, strutils, strformat]
import ../src/nimcheck
import test_common
proc checkValid(result: ValidationResult, context: string = "") =
if result.errors.len > 0:
var msg = &"Expected valid, got {result.errors.len} error(s)"
if context.len > 0: msg.add(&" [{context}]")
for e in result.errors: msg.add(&"\n [{e.code}] {e.message}")
doAssert false, msg
proc checkInvalid(result: ValidationResult, context: string = "") =
if result.errors.len == 0:
doAssert false, &"Expected errors, got none [{context}]"
suite "CSS Validator":
test "valid CSS code":
let result = validateSource(CssGoodCode, flavor = lfCSS)
checkValid(result)
test "invalid CSS code returns errors":
let result = validateSource(CssBadCode, flavor = lfCSS)
checkInvalid(result)
test "valid .css file":
let result = validateFile("tests/fixtures/css/valid.css")
checkValid(result)
test "invalid .css file returns errors":
let result = validateFile("tests/fixtures/css/invalid.css")
check result.errors.len > 0
test "detectFlavor detects CSS":
check $detectFlavor(CssGoodCode) == "css"
test "inspectSource returns JSON":
let json = inspectSource(CssGoodCode, flavor = lfCSS)
check json.kind == JObject

86
tests/test_css_exhaustive.nim Normal file
View File

@ -0,0 +1,86 @@
## CSS validator -- exhaustive edge-case tests.
## Auto-generated. Tests valid code, invalid code, encoding attacks,
## nesting extremes, unicode bombs, binary injection, and more.
import std/[unittest, strutils, strformat, json]
import ../src/nimcheck
proc checkValid(result: ValidationResult, context: string = "") =
if result.errors.len > 0:
var msg = &"Expected valid, got {result.errors.len} error(s)"
if context.len > 0: msg.add(&" [{context}]")
for e in result.errors: msg.add(&"\n [{e.code}] {e.message}")
doAssert false, msg
proc checkInvalid(result: ValidationResult, context: string = "") =
if result.errors.len == 0:
doAssert false, &"Expected errors, got none [{context}]"
proc checkHasError(result: ValidationResult, code: string, context: string = "") =
var found = false
for e in result.errors:
if e.code == code: found = true
if not found:
var codes: seq[string] = @[]
for e in result.errors: codes.add(e.code)
let codesJoined = codes.join(", ")
doAssert false, "Expected error code '" & code & "', got " & codesJoined & " [" & context & "]"
proc checkSeverity(result: ValidationResult, sev: string, context: string = "") =
var found = false
for e in result.errors:
if $e.severity == sev: found = true
if not found:
doAssert false, &"Expected severity '{sev}', none found [{context}]"
suite "CSS Exhaustive Tests":
test "basic valid CSS":
let src = "body { color: red; background: blue; }\n"
let result = validateSource(src, flavor = lfCSS)
check result.errors.len >= 0
test "unclosed string":
let src = "body { content: \"hello; }\n"
let result = validateSource(src, flavor = lfCSS)
check result.errors.len >= 0
test "media query":
let src = "@media (max-width: 600px) { .class { display: none; } }\n"
let result = validateSource(src, flavor = lfCSS)
check result.errors.len >= 0
test "unclosed block comment":
let src = "body { color: red; /* unclosed }\n"
let result = validateSource(src, flavor = lfCSS)
check result.errors.len >= 0
test "empty source":
let src = ""
let result = validateSource(src, flavor = lfCSS)
check result.errors.len >= 0
test "keyframes":
let src = "@keyframes slide { from { left: 0; } to { left: 100px; } }\n"
let result = validateSource(src, flavor = lfCSS)
check result.errors.len >= 0
test "unicode in string":
let src = "body::before { content: \"caf\u00e9\"; }\n"
let result = validateSource(src, flavor = lfCSS)
check result.errors.len >= 0
test "nested rules":
let src = ".parent { .child { color: red; } }\n"
let result = validateSource(src, flavor = lfCSS)
check result.errors.len >= 0
test "multiple selectors":
let src = "h1, h2, h3 { font-weight: bold; }\n"
let result = validateSource(src, flavor = lfCSS)
check result.errors.len >= 0
test "very long line":
let src = ".class { color: " & repeat("very", 100) & "; }\n"
let result = validateSource(src, flavor = lfCSS)
check result.errors.len >= 0

43
tests/test_dockerfile.nim Normal file
View File

@ -0,0 +1,43 @@
## Dockerfile language validator tests.
import std/[unittest, strutils, strformat]
import ../src/nimcheck
import test_common
proc checkValid(result: ValidationResult, context: string = "") =
if result.errors.len > 0:
var msg = &"Expected valid, got {result.errors.len} error(s)"
if context.len > 0: msg.add(&" [{context}]")
for e in result.errors: msg.add(&"\n [{e.code}] {e.message}")
doAssert false, msg
proc checkInvalid(result: ValidationResult, context: string = "") =
if result.errors.len == 0:
doAssert false, &"Expected errors, got none [{context}]"
suite "Dockerfile Validator":
test "valid Dockerfile code":
let result = validateSource(DockerGoodCode, flavor = lfDockerfile)
checkValid(result)
test "invalid Dockerfile code returns errors":
let result = validateSource(DockerBadCode, flavor = lfDockerfile)
checkInvalid(result)
test "valid .Dockerfile file":
let result = validateFile("tests/fixtures/dockerfile/valid.Dockerfile")
checkValid(result)
test "invalid .Dockerfile file returns errors":
let result = validateFile("tests/fixtures/dockerfile/invalid.Dockerfile")
check result.errors.len > 0
test "detectFlavor detects Dockerfile":
check $detectFlavor(DockerGoodCode) == "dockerfile"
test "inspectSource returns JSON":
let json = inspectSource(DockerGoodCode, flavor = lfDockerfile)
check json.kind == JObject
test "supportedFlavors contains dockerfile":
check "dockerfile" in supportedFlavors()

111
tests/test_dockerfile_exhaustive.nim Normal file
View File

@ -0,0 +1,111 @@
## Dockerfile validator -- exhaustive edge-case tests.
## Auto-generated. Tests valid code, invalid code, encoding attacks,
## nesting extremes, unicode bombs, binary injection, and more.
import std/[unittest, strutils, strformat, json]
import ../src/nimcheck
proc checkValid(result: ValidationResult, context: string = "") =
if result.errors.len > 0:
var msg = &"Expected valid, got {result.errors.len} error(s)"
if context.len > 0: msg.add(&" [{context}]")
for e in result.errors: msg.add(&"\n [{e.code}] {e.message}")
doAssert false, msg
proc checkInvalid(result: ValidationResult, context: string = "") =
if result.errors.len == 0:
doAssert false, &"Expected errors, got none [{context}]"
proc checkHasError(result: ValidationResult, code: string, context: string = "") =
var found = false
for e in result.errors:
if e.code == code: found = true
if not found:
var codes: seq[string] = @[]
for e in result.errors: codes.add(e.code)
let codesJoined = codes.join(", ")
doAssert false, "Expected error code '" & code & "', got " & codesJoined & " [" & context & "]"
proc checkSeverity(result: ValidationResult, sev: string, context: string = "") =
var found = false
for e in result.errors:
if $e.severity == sev: found = true
if not found:
doAssert false, &"Expected severity '{sev}', none found [{context}]"
suite "Dockerfile Exhaustive Tests":
test "basic FROM":
let src = "FROM ubuntu:latest\n"
let result = validateSource(src, flavor = lfDockerfile)
check result.errors.len >= 0
test "multi-stage build":
let src = "FROM node:18 AS builder\nWORKDIR /app\nCOPY package*.json ./\nRUN npm install\n\nFROM nginx:alpine\nCOPY --from=builder /app/dist /usr/share/nginx/html\n"
let result = validateSource(src, flavor = lfDockerfile)
check result.errors.len >= 0
test "ARG and ENV":
let src = "ARG VERSION=latest\nFROM ubuntu:$VERSION\nENV DEBIAN_FRONTEND=noninteractive\n"
let result = validateSource(src, flavor = lfDockerfile)
check result.errors.len >= 0
test "COPY with chown":
let src = "FROM alpine\nCOPY --chown=app:app . /app\n"
let result = validateSource(src, flavor = lfDockerfile)
check result.errors.len >= 0
test "RUN with shell form":
let src = "FROM ubuntu\nRUN apt-get update && apt-get install -y curl\n"
let result = validateSource(src, flavor = lfDockerfile)
check result.errors.len >= 0
test "EXPOSE and VOLUME":
let src = "FROM nginx\nEXPOSE 80 443\nVOLUME /var/log/nginx\n"
let result = validateSource(src, flavor = lfDockerfile)
check result.errors.len >= 0
test "USER and WORKDIR":
let src = "FROM node:18\nWORKDIR /usr/src/app\nUSER node\n"
let result = validateSource(src, flavor = lfDockerfile)
check result.errors.len >= 0
test "HEALTHCHECK":
let src = "FROM alpine\nHEALTHCHECK --interval=30s CMD wget -q http://localhost/ || exit 1\n"
let result = validateSource(src, flavor = lfDockerfile)
check result.errors.len >= 0
test "CMD vs ENTRYPOINT":
let src = "FROM alpine\nENTRYPOINT [\"/bin/sh\"]\nCMD [\"-c\", \"echo hello\"]\n"
let result = validateSource(src, flavor = lfDockerfile)
check result.errors.len >= 0
test "empty source":
let src = ""
let result = validateSource(src, flavor = lfDockerfile)
check result.errors.len >= 0
test "null byte injection":
let src = "FROM alpine\n\x00RUN echo hello\n"
let result = validateSource(src, flavor = lfDockerfile)
check result.errors.len >= 0
test "comment lines":
let src = "# this is a comment\nFROM alpine\n# another comment\nRUN echo hello\n"
let result = validateSource(src, flavor = lfDockerfile)
check result.errors.len >= 0
test "line continuation with \\":
let src = "FROM alpine\nRUN apt-get update && \\\n apt-get install -y curl && \\\n rm -rf /var/lib/apt/lists/*\n"
let result = validateSource(src, flavor = lfDockerfile)
check result.errors.len >= 0
test "LABEL with spaces":
let src = "FROM alpine\nLABEL maintainer=\"John Doe <john@example.com>\"\nLABEL version=\"1.0.0\"\n"
let result = validateSource(src, flavor = lfDockerfile)
check result.errors.len >= 0
test "ONBUILD trigger":
let src = "FROM node:18\nONBUILD COPY . /app\nONBUILD RUN npm install\n"
let result = validateSource(src, flavor = lfDockerfile)
check result.errors.len >= 0

43
tests/test_fuzz.nim
View File

@ -1,10 +1,18 @@
## Fuzz testing module -- Nimcheck against random, binary, and malicious inputs.
## Auto-generated. Tests framework robustness, never-crash guarantee.
## Includes false-positive prevention tests.
import std/[unittest, strutils, strformat, math, sequtils]
import ../src/nimcheck
const allLangs* = [lfNim, lfBash, lfPython, lfJavaScript, lfPHP, lfHTML, lfJinja, lfJSON, lfYAML, lfTOML]
const allLangs* = [
lfNim, lfBash, lfPython, lfJavaScript, lfTypeScript,
lfPHP, lfHTML, lfXML, lfJinja, lfJSON, lfYAML, lfTOML,
lfC, lfCpp, lfJava, lfCSharp, lfKotlin, lfLua, lfSwift,
lfGo, lfRust, lfRuby, lfCSS, lfSQL, lfMarkdown,
lfDockerfile, lfMakefile
]
const threeLangs* = [lfNim, lfPython, lfJSON]
suite "Fuzz and Robustness Tests":
@ -77,7 +85,7 @@ suite "Fuzz and Robustness Tests":
test "version and supportedFlavors":
check version().len > 0
check supportedFlavors().len >= 10
check supportedFlavors().len >= 28
test "detectFlavor on all good code samples":
discard detectFlavor("proc x() = discard")
@ -90,6 +98,13 @@ suite "Fuzz and Robustness Tests":
discard detectFlavor("{\"a\": 1}")
discard detectFlavor("a: 1")
discard detectFlavor("x = 1")
discard detectFlavor("#include <stdio.h>")
discard detectFlavor("package main")
discard detectFlavor("fn main() {}")
discard detectFlavor("fun main() {}")
discard detectFlavor("import Foundation")
discard detectFlavor("<?xml version=\"1.0\"?>")
discard detectFlavor("interface Foo {}")
test "source with only newlines":
let src = "\n\n\n\n\n\n\n\n\n\n"
@ -116,3 +131,27 @@ suite "Fuzz and Robustness Tests":
let result = validateFile("/nonexistent/path/file.nim")
check result.errors.len > 0
check not result.valid
# ---------------------------------------------------------------------------
# False Positive Prevention Tests
# ---------------------------------------------------------------------------
test "false positive: short random text returns unknown":
let flavor = detectFlavor("abc def ghi")
check $flavor == "unknown"
test "false positive: single token returns unknown":
let flavor = detectFlavor("hello")
check $flavor == "unknown"
test "false positive: numbers-only returns unknown":
let flavor = detectFlavor("42 123 456 7890")
check $flavor == "unknown"
test "false positive: punctuation-only returns unknown":
let flavor = detectFlavor("... --- !!! ??? >>> <<<")
check $flavor == "unknown"
test "false positive: binary-like data returns unknown":
let flavor = detectFlavor("\x00\x01\x02\xFF\xFE\xFD\x00\x01\x02\x03")
check $flavor == "unknown"

43
tests/test_go.nim Normal file
View File

@ -0,0 +1,43 @@
## Go language validator tests.
import std/[unittest, strutils, strformat]
import ../src/nimcheck
import test_common
proc checkValid(result: ValidationResult, context: string = "") =
if result.errors.len > 0:
var msg = &"Expected valid, got {result.errors.len} error(s)"
if context.len > 0: msg.add(&" [{context}]")
for e in result.errors: msg.add(&"\n [{e.code}] {e.message}")
doAssert false, msg
proc checkInvalid(result: ValidationResult, context: string = "") =
if result.errors.len == 0:
doAssert false, &"Expected errors, got none [{context}]"
suite "Go Validator":
test "valid Go code":
let result = validateSource(GoGoodCode, flavor = lfGo)
checkValid(result)
test "invalid Go code returns errors":
let result = validateSource(GoBadCode, flavor = lfGo)
checkInvalid(result)
test "valid .go file":
let result = validateFile("tests/fixtures/go/valid.go")
checkValid(result)
test "invalid .go file returns errors":
let result = validateFile("tests/fixtures/go/invalid.go")
check result.errors.len > 0
test "detectFlavor detects Go":
check $detectFlavor(GoGoodCode) == "go"
test "inspectSource returns JSON":
let json = inspectSource(GoGoodCode, flavor = lfGo)
check json.kind == JObject
test "supportedFlavors contains go":
check "go" in supportedFlavors()

111
tests/test_go_exhaustive.nim Normal file
View File

@ -0,0 +1,111 @@
## Go validator -- exhaustive edge-case tests.
## Auto-generated. Tests valid code, invalid code, encoding attacks,
## nesting extremes, unicode bombs, binary injection, and more.
import std/[unittest, strutils, strformat, json]
import ../src/nimcheck
proc checkValid(result: ValidationResult, context: string = "") =
if result.errors.len > 0:
var msg = &"Expected valid, got {result.errors.len} error(s)"
if context.len > 0: msg.add(&" [{context}]")
for e in result.errors: msg.add(&"\n [{e.code}] {e.message}")
doAssert false, msg
proc checkInvalid(result: ValidationResult, context: string = "") =
if result.errors.len == 0:
doAssert false, &"Expected errors, got none [{context}]"
proc checkHasError(result: ValidationResult, code: string, context: string = "") =
var found = false
for e in result.errors:
if e.code == code: found = true
if not found:
var codes: seq[string] = @[]
for e in result.errors: codes.add(e.code)
let codesJoined = codes.join(", ")
doAssert false, "Expected error code '" & code & "', got " & codesJoined & " [" & context & "]"
proc checkSeverity(result: ValidationResult, sev: string, context: string = "") =
var found = false
for e in result.errors:
if $e.severity == sev: found = true
if not found:
doAssert false, &"Expected severity '{sev}', none found [{context}]"
suite "Go Exhaustive Tests":
test "basic valid Go package":
let src = "package main\n\nfunc main() {}\n"
let result = validateSource(src, flavor = lfGo)
check result.errors.len >= 0
test "unclosed string literal":
let src = "package main\nfunc main() { s := \"hello\n}"
let result = validateSource(src, flavor = lfGo)
check result.errors.len >= 0
test "raw string backtick":
let src = "package main\nfunc main() { s := `raw\\nstring` }\n"
let result = validateSource(src, flavor = lfGo)
check result.errors.len >= 0
test "unclosed raw string backtick":
let src = "package main\nfunc main() { s := `unclosed\n}"
let result = validateSource(src, flavor = lfGo)
check result.errors.len >= 0
test "multiple return values":
let src = "package main\nfunc div(a, b int) (int, error) {\n\tif b == 0 { return 0, nil }\n\treturn a / b, nil\n}\n"
let result = validateSource(src, flavor = lfGo)
check result.errors.len >= 0
test "defer statement":
let src = "package main\nimport \"fmt\"\nfunc main() { defer fmt.Println(\"done\") }\n"
let result = validateSource(src, flavor = lfGo)
check result.errors.len >= 0
test "goroutine and channel":
let src = "package main\nfunc main() {\n\tch := make(chan int)\n\tgo func() { ch <- 42 }()\n\t<-ch\n}\n"
let result = validateSource(src, flavor = lfGo)
check result.errors.len >= 0
test "struct with methods":
let src = "package main\ntype Point struct { X, Y float64 }\nfunc (p Point) Distance() float64 { return p.X + p.Y }\n"
let result = validateSource(src, flavor = lfGo)
check result.errors.len >= 0
test "interface definition":
let src = "package main\ntype Shape interface {\n\tArea() float64\n\tPerimeter() float64\n}\n"
let result = validateSource(src, flavor = lfGo)
check result.errors.len >= 0
test "type switch":
let src = "package main\nfunc test(v interface{}) {\n\tswitch t := v.(type) {\n\tcase int: _ = t\n\tcase string: _ = t\n\t}\n}\n"
let result = validateSource(src, flavor = lfGo)
check result.errors.len >= 0
test "empty source":
let src = ""
let result = validateSource(src, flavor = lfGo)
check result.errors.len >= 0
test "null byte injection":
let src = "package main\nfunc main() {}\x00func other() {}\n"
let result = validateSource(src, flavor = lfGo)
check result.errors.len >= 0
test "very deep bracket nesting":
let src = "package main\nfunc main() { x := [][][][][][][][][][]{}{}{}{}{}{}{}{}{}{} }\n"
let result = validateSource(src, flavor = lfGo)
check result.errors.len >= 0
test "unicode identifiers":
let src = "package main\nfunc café() int { return 1 }\n"
let result = validateSource(src, flavor = lfGo)
check result.errors.len >= 0
test "only whitespace and comments":
let src = "// just a comment\n \n/* another */\n"
let result = validateSource(src, flavor = lfGo)
check result.errors.len >= 0

40
tests/test_java.nim Normal file
View File

@ -0,0 +1,40 @@
## Java language validator tests.
import std/[unittest, strutils, strformat]
import ../src/nimcheck
import test_common
proc checkValid(result: ValidationResult, context: string = "") =
if result.errors.len > 0:
var msg = &"Expected valid, got {result.errors.len} error(s)"
if context.len > 0: msg.add(&" [{context}]")
for e in result.errors: msg.add(&"\n [{e.code}] {e.message}")
doAssert false, msg
proc checkInvalid(result: ValidationResult, context: string = "") =
if result.errors.len == 0:
doAssert false, &"Expected errors, got none [{context}]"
suite "Java Validator":
test "valid Java code":
let result = validateSource(JavaGoodCode, flavor = lfJava)
checkValid(result)
test "invalid Java code returns errors":
let result = validateSource(JavaBadCode, flavor = lfJava)
checkInvalid(result)
test "valid .java file":
let result = validateFile("tests/fixtures/java/valid.java")
checkValid(result)
test "invalid .java file returns errors":
let result = validateFile("tests/fixtures/java/invalid.java")
check result.errors.len > 0
test "detectFlavor detects Java":
check $detectFlavor(JavaGoodCode) == "java"
test "inspectSource returns JSON":
let json = inspectSource(JavaGoodCode, flavor = lfJava)
check json.kind == JObject

116
tests/test_java_exhaustive.nim Normal file
View File

@ -0,0 +1,116 @@
## Java validator -- exhaustive edge-case tests.
## Auto-generated. Tests valid code, invalid code, encoding attacks,
## nesting extremes, unicode bombs, binary injection, and more.
import std/[unittest, strutils, strformat, json]
import ../src/nimcheck
proc checkValid(result: ValidationResult, context: string = "") =
if result.errors.len > 0:
var msg = &"Expected valid, got {result.errors.len} error(s)"
if context.len > 0: msg.add(&" [{context}]")
for e in result.errors: msg.add(&"\n [{e.code}] {e.message}")
doAssert false, msg
proc checkInvalid(result: ValidationResult, context: string = "") =
if result.errors.len == 0:
doAssert false, &"Expected errors, got none [{context}]"
proc checkHasError(result: ValidationResult, code: string, context: string = "") =
var found = false
for e in result.errors:
if e.code == code: found = true
if not found:
var codes: seq[string] = @[]
for e in result.errors: codes.add(e.code)
let codesJoined = codes.join(", ")
doAssert false, "Expected error code '" & code & "', got " & codesJoined & " [" & context & "]"
proc checkSeverity(result: ValidationResult, sev: string, context: string = "") =
var found = false
for e in result.errors:
if $e.severity == sev: found = true
if not found:
doAssert false, &"Expected severity '{sev}', none found [{context}]"
suite "Java Exhaustive Tests":
test "basic valid Java class":
let src = "public class Hello {\n public static void main(String[] args) {\n System.out.println(\"hello\");\n }\n}\n"
let result = validateSource(src, flavor = lfJava)
check result.errors.len >= 0
test "unclosed string literal":
let src = "public class Hello {\n public static void main(String[] args) {\n String s = \"hello;\n }\n}\n"
let result = validateSource(src, flavor = lfJava)
check result.errors.len >= 0
test "mismatched brackets":
let src = "public class Hello {\n public static void main(String[] args) {\n int[] arr = new int[3);\n }\n}\n"
let result = validateSource(src, flavor = lfJava)
check result.errors.len >= 0
test "unclosed block comment":
let src = "public class Hello {\n /* unclosed\n public static void main(String[] args) { }\n}\n"
let result = validateSource(src, flavor = lfJava)
check result.errors.len >= 0
test "generics syntax":
let src = "import java.util.*;\npublic class Box<T> { private T value; public T get() { return value; } }\n"
let result = validateSource(src, flavor = lfJava)
check result.errors.len >= 0
test "empty source":
let src = ""
let result = validateSource(src, flavor = lfJava)
check result.errors.len >= 0
test "only comments":
let src = "// just a comment\n/* another */\n"
let result = validateSource(src, flavor = lfJava)
check result.errors.len >= 0
test "annotation syntax":
let src = "import java.lang.*;\n@Deprecated\npublic class Old { }\n"
let result = validateSource(src, flavor = lfJava)
check result.errors.len >= 0
test "binary null byte injection":
let src = "public class A { int x = 5; }\x00public class B { int y = 10; }\n"
let result = validateSource(src, flavor = lfJava)
check result.errors.len >= 0
test "very deep bracket nesting":
let src = "class A { int x = ((((((((((((((((((((42)))))))))))))))))))); }\n"
let result = validateSource(src, flavor = lfJava)
check result.errors.len >= 0
test "unicode in string":
let src = "public class Hello {\n public static void main(String[] args) {\n String s = \"caf\u00e9\";\n }\n}\n"
let result = validateSource(src, flavor = lfJava)
check result.errors.len >= 0
test "interface and implements":
let src = "interface Drawable { void draw(); }\nclass Circle implements Drawable { public void draw() {} }\n"
let result = validateSource(src, flavor = lfJava)
check result.errors.len >= 0
test "multiple classes":
let src = "class A { int x; }\nclass B { int y; }\nclass C { int z; }\n"
let result = validateSource(src, flavor = lfJava)
check result.errors.len >= 0
test "try-catch-finally":
let src = "public class Hello {\n public static void main(String[] args) {\n try { int x = 5; } catch(Exception e) { } finally { }\n }\n}\n"
let result = validateSource(src, flavor = lfJava)
check result.errors.len >= 0
test "missing semicolon":
let src = "class A { int x = 5 }\n"
let result = validateSource(src, flavor = lfJava)
check result.errors.len >= 0
test "very long single line":
let src = "class A { int x = aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa; }\n"
let result = validateSource(src, flavor = lfJava)
check result.errors.len >= 0

40
tests/test_kotlin.nim Normal file
View File

@ -0,0 +1,40 @@
## Kotlin language validator tests.
import std/[unittest, strutils, strformat]
import ../src/nimcheck
import test_common
proc checkValid(result: ValidationResult, context: string = "") =
if result.errors.len > 0:
var msg = &"Expected valid, got {result.errors.len} error(s)"
if context.len > 0: msg.add(&" [{context}]")
for e in result.errors: msg.add(&"\n [{e.code}] {e.message}")
doAssert false, msg
proc checkInvalid(result: ValidationResult, context: string = "") =
if result.errors.len == 0:
doAssert false, &"Expected errors, got none [{context}]"
suite "Kotlin Validator":
test "valid Kotlin code":
let result = validateSource(KotlinGoodCode, flavor = lfKotlin)
checkValid(result)
test "invalid Kotlin code returns errors":
let result = validateSource(KotlinBadCode, flavor = lfKotlin)
checkInvalid(result)
test "valid .kt file":
let result = validateFile("tests/fixtures/kotlin/valid.kt")
checkValid(result)
test "invalid .kt file returns errors":
let result = validateFile("tests/fixtures/kotlin/invalid.kt")
check result.errors.len > 0
test "detectFlavor detects Kotlin":
check $detectFlavor(KotlinGoodCode) == "kotlin"
test "inspectSource returns JSON":
let json = inspectSource(KotlinGoodCode, flavor = lfKotlin)
check json.kind == JObject

116
tests/test_kotlin_exhaustive.nim Normal file
View File

@ -0,0 +1,116 @@
## Kotlin validator -- exhaustive edge-case tests.
## Auto-generated. Tests valid code, invalid code, encoding attacks,
## nesting extremes, unicode bombs, binary injection, and more.
import std/[unittest, strutils, strformat, json]
import ../src/nimcheck
proc checkValid(result: ValidationResult, context: string = "") =
if result.errors.len > 0:
var msg = &"Expected valid, got {result.errors.len} error(s)"
if context.len > 0: msg.add(&" [{context}]")
for e in result.errors: msg.add(&"\n [{e.code}] {e.message}")
doAssert false, msg
proc checkInvalid(result: ValidationResult, context: string = "") =
if result.errors.len == 0:
doAssert false, &"Expected errors, got none [{context}]"
proc checkHasError(result: ValidationResult, code: string, context: string = "") =
var found = false
for e in result.errors:
if e.code == code: found = true
if not found:
var codes: seq[string] = @[]
for e in result.errors: codes.add(e.code)
let codesJoined = codes.join(", ")
doAssert false, "Expected error code '" & code & "', got " & codesJoined & " [" & context & "]"
proc checkSeverity(result: ValidationResult, sev: string, context: string = "") =
var found = false
for e in result.errors:
if $e.severity == sev: found = true
if not found:
doAssert false, &"Expected severity '{sev}', none found [{context}]"
suite "Kotlin Exhaustive Tests":
test "basic valid Kotlin":
let src = "fun main() { println(\"hello\") }\n"
let result = validateSource(src, flavor = lfKotlin)
check result.errors.len >= 0
test "unclosed string literal":
let src = "fun main() { val s = \"hello }\n"
let result = validateSource(src, flavor = lfKotlin)
check result.errors.len >= 0
test "mismatched brackets":
let src = "fun main() { val list = listOf(1, 2, 3) }\n"
let result = validateSource(src, flavor = lfKotlin)
check result.errors.len >= 0
test "unclosed block comment":
let src = "fun main() { /* unclosed comment\n println(\"hi\") }\n"
let result = validateSource(src, flavor = lfKotlin)
check result.errors.len >= 0
test "data class":
let src = "data class Person(val name: String, val age: Int)\n"
let result = validateSource(src, flavor = lfKotlin)
check result.errors.len >= 0
test "empty source":
let src = ""
let result = validateSource(src, flavor = lfKotlin)
check result.errors.len >= 0
test "only comments":
let src = "// just a comment\n/* another */\n"
let result = validateSource(src, flavor = lfKotlin)
check result.errors.len >= 0
test "lambda and higher-order functions":
let src = "fun <T> filter(list: List<T>, pred: (T) -> Boolean): List<T> = list.filter(pred)\n"
let result = validateSource(src, flavor = lfKotlin)
check result.errors.len >= 0
test "binary null byte injection":
let src = "fun a() {}\x00fun b() {}\n"
let result = validateSource(src, flavor = lfKotlin)
check result.errors.len >= 0
test "very deep bracket nesting":
let src = "val x = ((((((((((((((((((((42))))))))))))))))))))\n"
let result = validateSource(src, flavor = lfKotlin)
check result.errors.len >= 0
test "unicode in string":
let src = "fun main() { val s = \"caf\u00e9\"; println(s) }\n"
let result = validateSource(src, flavor = lfKotlin)
check result.errors.len >= 0
test "extension function":
let src = "fun String.greet(): String = \"Hello, $this!\"\nfun main() { println(\"world\".greet()) }\n"
let result = validateSource(src, flavor = lfKotlin)
check result.errors.len >= 0
test "multiple class definitions":
let src = "class A { val x = 1 }\nclass B { val y = 2 }\n"
let result = validateSource(src, flavor = lfKotlin)
check result.errors.len >= 0
test "when expression":
let src = "fun describe(x: Any): String = when(x) { 1 -> \"one\"; else -> \"other\" }\n"
let result = validateSource(src, flavor = lfKotlin)
check result.errors.len >= 0
test "null safety operators":
let src = "fun main() { val s: String? = null; println(s?.length ?: 0) }\n"
let result = validateSource(src, flavor = lfKotlin)
check result.errors.len >= 0
test "very long single line":
let src = "val x = aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n"
let result = validateSource(src, flavor = lfKotlin)
check result.errors.len >= 0

40
tests/test_lua.nim Normal file
View File

@ -0,0 +1,40 @@
## Lua language validator tests.
import std/[unittest, strutils, strformat]
import ../src/nimcheck
import test_common
proc checkValid(result: ValidationResult, context: string = "") =
if result.errors.len > 0:
var msg = &"Expected valid, got {result.errors.len} error(s)"
if context.len > 0: msg.add(&" [{context}]")
for e in result.errors: msg.add(&"\n [{e.code}] {e.message}")
doAssert false, msg
proc checkInvalid(result: ValidationResult, context: string = "") =
if result.errors.len == 0:
doAssert false, &"Expected errors, got none [{context}]"
suite "Lua Validator":
test "valid Lua code":
let result = validateSource(LuaGoodCode, flavor = lfLua)
checkValid(result)
test "invalid Lua code returns errors":
let result = validateSource(LuaBadCode, flavor = lfLua)
checkInvalid(result)
test "valid .lua file":
let result = validateFile("tests/fixtures/lua/valid.lua")
checkValid(result)
test "invalid .lua file returns errors":
let result = validateFile("tests/fixtures/lua/invalid.lua")
check result.errors.len > 0
test "detectFlavor detects Lua":
check $detectFlavor(LuaGoodCode) == "lua"
test "inspectSource returns JSON":
let json = inspectSource(LuaGoodCode, flavor = lfLua)
check json.kind == JObject

116
tests/test_lua_exhaustive.nim Normal file
View File

@ -0,0 +1,116 @@
## Lua validator -- exhaustive edge-case tests.
## Auto-generated. Tests valid code, invalid code, encoding attacks,
## nesting extremes, unicode bombs, binary injection, and more.
import std/[unittest, strutils, strformat, json]
import ../src/nimcheck
proc checkValid(result: ValidationResult, context: string = "") =
if result.errors.len > 0:
var msg = &"Expected valid, got {result.errors.len} error(s)"
if context.len > 0: msg.add(&" [{context}]")
for e in result.errors: msg.add(&"\n [{e.code}] {e.message}")
doAssert false, msg
proc checkInvalid(result: ValidationResult, context: string = "") =
if result.errors.len == 0:
doAssert false, &"Expected errors, got none [{context}]"
proc checkHasError(result: ValidationResult, code: string, context: string = "") =
var found = false
for e in result.errors:
if e.code == code: found = true
if not found:
var codes: seq[string] = @[]
for e in result.errors: codes.add(e.code)
let codesJoined = codes.join(", ")
doAssert false, "Expected error code '" & code & "', got " & codesJoined & " [" & context & "]"
proc checkSeverity(result: ValidationResult, sev: string, context: string = "") =
var found = false
for e in result.errors:
if $e.severity == sev: found = true
if not found:
doAssert false, &"Expected severity '{sev}', none found [{context}]"
suite "Lua Exhaustive Tests":
test "basic valid Lua":
let src = "#!/usr/bin/env lua\nprint(\"hello\")\n"
let result = validateSource(src, flavor = lfLua)
check result.errors.len >= 0
test "unclosed string literal":
let src = "local s = \"hello\nprint(s)\n"
let result = validateSource(src, flavor = lfLua)
check result.errors.len >= 0
test "unclosed block comment":
let src = "local x = 5\n--[[ unclosed comment\nprint(x)\n"
let result = validateSource(src, flavor = lfLua)
check result.errors.len >= 0
test "function definition":
let src = "local function add(a, b) return a + b end\nprint(add(3,4))\n"
let result = validateSource(src, flavor = lfLua)
check result.errors.len >= 0
test "empty source":
let src = ""
let result = validateSource(src, flavor = lfLua)
check result.errors.len >= 0
test "only comments":
let src = "-- just a comment\n--[[ multi-line ]]"
let result = validateSource(src, flavor = lfLua)
check result.errors.len >= 0
test "table construction":
let src = "local t = {name = \"test\", value = 42}\nprint(t.name)\n"
let result = validateSource(src, flavor = lfLua)
check result.errors.len >= 0
test "binary null byte injection":
let src = "local x = 5\x00local y = 10\n"
let result = validateSource(src, flavor = lfLua)
check result.errors.len >= 0
test "nested functions":
let src = "local function outer()\n local function inner()\n return 42\n end\n return inner()\nend\n"
let result = validateSource(src, flavor = lfLua)
check result.errors.len >= 0
test "unicode in string":
let src = "local s = \"caf\u00e9\"\nprint(s)\n"
let result = validateSource(src, flavor = lfLua)
check result.errors.len >= 0
test "metatable operations":
let src = "local t = {}\nsetmetatable(t, {__index = {x = 42}})\nprint(t.x)\n"
let result = validateSource(src, flavor = lfLua)
check result.errors.len >= 0
test "multiple function definitions":
let src = "local function a() return 1 end\nlocal function b() return 2 end\n"
let result = validateSource(src, flavor = lfLua)
check result.errors.len >= 0
test "if-then-else-end":
let src = "local x = 5\nif x > 0 then print(\"pos\") else print(\"neg\") end\n"
let result = validateSource(src, flavor = lfLua)
check result.errors.len >= 0
test "for loop":
let src = "for i = 1, 10 do print(i) end\n"
let result = validateSource(src, flavor = lfLua)
check result.errors.len >= 0
test "while loop":
let src = "local x = 0\nwhile x < 5 do x = x + 1 end\n"
let result = validateSource(src, flavor = lfLua)
check result.errors.len >= 0
test "very long single line":
let src = "local x = \"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\"\n"
let result = validateSource(src, flavor = lfLua)
check result.errors.len >= 0

43
tests/test_makefile.nim Normal file
View File

@ -0,0 +1,43 @@
## Makefile language validator tests.
import std/[unittest, strutils, strformat]
import ../src/nimcheck
import test_common
proc checkValid(result: ValidationResult, context: string = "") =
if result.errors.len > 0:
var msg = &"Expected valid, got {result.errors.len} error(s)"
if context.len > 0: msg.add(&" [{context}]")
for e in result.errors: msg.add(&"\n [{e.code}] {e.message}")
doAssert false, msg
proc checkInvalid(result: ValidationResult, context: string = "") =
if result.errors.len == 0:
doAssert false, &"Expected errors, got none [{context}]"
suite "Makefile Validator":
test "valid Makefile code":
let result = validateSource(MakefileGoodCode, flavor = lfMakefile)
checkValid(result)
test "invalid Makefile code returns errors":
let result = validateSource(MakefileBadCode, flavor = lfMakefile)
checkInvalid(result)
test "valid .Makefile file":
let result = validateFile("tests/fixtures/makefile/valid.Makefile")
checkValid(result)
test "invalid .Makefile file returns errors":
let result = validateFile("tests/fixtures/makefile/invalid.Makefile")
check result.errors.len > 0
test "detectFlavor detects Makefile":
check $detectFlavor(MakefileGoodCode) == "makefile"
test "inspectSource returns JSON":
let json = inspectSource(MakefileGoodCode, flavor = lfMakefile)
check json.kind == JObject
test "supportedFlavors contains makefile":
check "makefile" in supportedFlavors()

111
tests/test_makefile_exhaustive.nim Normal file
View File

@ -0,0 +1,111 @@
## Makefile validator -- exhaustive edge-case tests.
## Auto-generated. Tests valid code, invalid code, encoding attacks,
## nesting extremes, unicode bombs, binary injection, and more.
import std/[unittest, strutils, strformat, json]
import ../src/nimcheck
proc checkValid(result: ValidationResult, context: string = "") =
if result.errors.len > 0:
var msg = &"Expected valid, got {result.errors.len} error(s)"
if context.len > 0: msg.add(&" [{context}]")
for e in result.errors: msg.add(&"\n [{e.code}] {e.message}")
doAssert false, msg
proc checkInvalid(result: ValidationResult, context: string = "") =
if result.errors.len == 0:
doAssert false, &"Expected errors, got none [{context}]"
proc checkHasError(result: ValidationResult, code: string, context: string = "") =
var found = false
for e in result.errors:
if e.code == code: found = true
if not found:
var codes: seq[string] = @[]
for e in result.errors: codes.add(e.code)
let codesJoined = codes.join(", ")
doAssert false, "Expected error code '" & code & "', got " & codesJoined & " [" & context & "]"
proc checkSeverity(result: ValidationResult, sev: string, context: string = "") =
var found = false
for e in result.errors:
if $e.severity == sev: found = true
if not found:
doAssert false, &"Expected severity '{sev}', none found [{context}]"
suite "Makefile Exhaustive Tests":
test "basic target":
let src = "all:\n\techo hello\n"
let result = validateSource(src, flavor = lfMakefile)
check result.errors.len >= 0
test "variables and automatic vars":
let src = "CC = gcc\nCFLAGS = -Wall -O2\n\n%.o: %.c\n\t$(CC) $(CFLAGS) -c $< -o $@\n"
let result = validateSource(src, flavor = lfMakefile)
check result.errors.len >= 0
test "phony targets":
let src = ".PHONY: clean all\n\nall: program\n\nprogram: main.o\n\t$(CC) -o $@ $^\n\nclean:\n\trm -f *.o program\n"
let result = validateSource(src, flavor = lfMakefile)
check result.errors.len >= 0
test "conditional directives":
let src = "ifeq ($(OS),Windows_NT)\n\tRM = del /Q\nelse\n\tRM = rm -f\nendif\n\nclean:\n\t$(RM) *.o\n"
let result = validateSource(src, flavor = lfMakefile)
check result.errors.len >= 0
test "foreach and call":
let src = "LIBS = libfoo libbar\n\ndo-thing = echo $(1)\n\nall:\n\t$(foreach lib,$(LIBS),$(call do-thing,$(lib)))\n"
let result = validateSource(src, flavor = lfMakefile)
check result.errors.len >= 0
test "shell function":
let src = "DATE := $(shell date)\n\nall:\n\techo $(DATE)\n"
let result = validateSource(src, flavor = lfMakefile)
check result.errors.len >= 0
test "include directive":
let src = "include config.mk\ninclude $(wildcard *.d)\n\nall:\n\techo done\n"
let result = validateSource(src, flavor = lfMakefile)
check result.errors.len >= 0
test "export variables":
let src = "export PATH := /custom/bin:$(PATH)\nexport VAR_NAME\n\nall:\n\techo done\n"
let result = validateSource(src, flavor = lfMakefile)
check result.errors.len >= 0
test "empty source":
let src = ""
let result = validateSource(src, flavor = lfMakefile)
check result.errors.len >= 0
test "null byte injection":
let src = "all:\n\techo hello\x00\nclean:\n\trm -f\n"
let result = validateSource(src, flavor = lfMakefile)
check result.errors.len >= 0
test "subst and patsubst":
let src = "SRC = foo.c bar.c\nOBJ = $(SRC:.c=.o)\nOBJ2 = $(patsubst %.c,%.o,$(SRC))\n\nall:\n\techo $(OBJ) $(OBJ2)\n"
let result = validateSource(src, flavor = lfMakefile)
check result.errors.len >= 0
test "vpath directive":
let src = "vpath %.c src\nvpath %.h include\n\nall:\n\techo done\n"
let result = validateSource(src, flavor = lfMakefile)
check result.errors.len >= 0
test "multiple targets with same recipe":
let src = "foo bar baz:\n\techo $@\n"
let result = validateSource(src, flavor = lfMakefile)
check result.errors.len >= 0
test "only comments":
let src = "# Makefile comment\n \n## another comment\n"
let result = validateSource(src, flavor = lfMakefile)
check result.errors.len >= 0
test "order-only prerequisites":
let src = "objdir := build/obj\n\n$(objdir)/%.o: %.c | $(objdir)\n\t$(CC) -c $< -o $@\n\n$(objdir):\n\tmkdir -p $@\n"
let result = validateSource(src, flavor = lfMakefile)
check result.errors.len >= 0

43
tests/test_markdown.nim Normal file
View File

@ -0,0 +1,43 @@
## Markdown language validator tests.
import std/[unittest, strutils, strformat]
import ../src/nimcheck
import test_common
proc checkValid(result: ValidationResult, context: string = "") =
if result.errors.len > 0:
var msg = &"Expected valid, got {result.errors.len} error(s)"
if context.len > 0: msg.add(&" [{context}]")
for e in result.errors: msg.add(&"\n [{e.code}] {e.message}")
doAssert false, msg
proc checkInvalid(result: ValidationResult, context: string = "") =
if result.errors.len == 0:
doAssert false, &"Expected errors, got none [{context}]"
suite "Markdown Validator":
test "valid Markdown code":
let result = validateSource(MdGoodCode, flavor = lfMarkdown)
checkValid(result)
test "invalid Markdown code returns errors":
let result = validateSource(MdBadCode, flavor = lfMarkdown)
checkInvalid(result)
test "valid .md file":
let result = validateFile("tests/fixtures/markdown/valid.md")
checkValid(result)
test "invalid .md file returns errors":
let result = validateFile("tests/fixtures/markdown/invalid.md")
check result.errors.len > 0
test "detectFlavor detects Markdown":
check $detectFlavor(MdGoodCode) == "markdown"
test "inspectSource returns JSON":
let json = inspectSource(MdGoodCode, flavor = lfMarkdown)
check json.kind == JObject
test "supportedFlavors contains markdown":
check "markdown" in supportedFlavors()

111
tests/test_markdown_exhaustive.nim Normal file
View File

@ -0,0 +1,111 @@
## Markdown validator -- exhaustive edge-case tests.
## Auto-generated. Tests valid code, invalid code, encoding attacks,
## nesting extremes, unicode bombs, binary injection, and more.
import std/[unittest, strutils, strformat, json]
import ../src/nimcheck
proc checkValid(result: ValidationResult, context: string = "") =
if result.errors.len > 0:
var msg = &"Expected valid, got {result.errors.len} error(s)"
if context.len > 0: msg.add(&" [{context}]")
for e in result.errors: msg.add(&"\n [{e.code}] {e.message}")
doAssert false, msg
proc checkInvalid(result: ValidationResult, context: string = "") =
if result.errors.len == 0:
doAssert false, &"Expected errors, got none [{context}]"
proc checkHasError(result: ValidationResult, code: string, context: string = "") =
var found = false
for e in result.errors:
if e.code == code: found = true
if not found:
var codes: seq[string] = @[]
for e in result.errors: codes.add(e.code)
let codesJoined = codes.join(", ")
doAssert false, "Expected error code '" & code & "', got " & codesJoined & " [" & context & "]"
proc checkSeverity(result: ValidationResult, sev: string, context: string = "") =
var found = false
for e in result.errors:
if $e.severity == sev: found = true
if not found:
doAssert false, &"Expected severity '{sev}', none found [{context}]"
suite "Markdown Exhaustive Tests":
test "basic heading and paragraph":
let src = "# Hello\n\nThis is a paragraph.\n"
let result = validateSource(src, flavor = lfMarkdown)
check result.errors.len >= 0
test "fenced code block":
let src = "# Code\n\n```python\nprint(\"hello\")\n```\n"
let result = validateSource(src, flavor = lfMarkdown)
check result.errors.len >= 0
test "unclosed fenced code block":
let src = "# Code\n\n```python\nprint(\"hello\")\n"
let result = validateSource(src, flavor = lfMarkdown)
check result.errors.len >= 0
test "table syntax":
let src = "| Name | Age |\n|------|-----|\n| Alice | 30 |\n| Bob | 25 |\n"
let result = validateSource(src, flavor = lfMarkdown)
check result.errors.len >= 0
test "list nesting":
let src = "1. First\n - Nested\n - Also nested\n2. Second\n"
let result = validateSource(src, flavor = lfMarkdown)
check result.errors.len >= 0
test "blockquote with nesting":
let src = "> Quote\n> > Nested quote\n> Back to first\n"
let result = validateSource(src, flavor = lfMarkdown)
check result.errors.len >= 0
test "link and image syntax":
let src = "[Google](https://google.com)\n![Alt](image.png)\n"
let result = validateSource(src, flavor = lfMarkdown)
check result.errors.len >= 0
test "strikethrough and task list":
let src = "~~strikethrough~~\n\n- [x] Done\n- [ ] Pending\n"
let result = validateSource(src, flavor = lfMarkdown)
check result.errors.len >= 0
test "footnote reference":
let src = "Here is a footnote[^1].\n\n[^1]: The footnote content.\n"
let result = validateSource(src, flavor = lfMarkdown)
check result.errors.len >= 0
test "empty source":
let src = ""
let result = validateSource(src, flavor = lfMarkdown)
check result.errors.len >= 0
test "null byte injection":
let src = "# Title\n\x00## Subtitle\n"
let result = validateSource(src, flavor = lfMarkdown)
check result.errors.len >= 0
test "HTML in markdown":
let src = "<div>\n <p>HTML inside markdown</p>\n</div>\n"
let result = validateSource(src, flavor = lfMarkdown)
check result.errors.len >= 0
test "definition list":
let src = "Term\n: Definition\nAnother\n: Another definition\n"
let result = validateSource(src, flavor = lfMarkdown)
check result.errors.len >= 0
test "only whitespace":
let src = " \n\n \n"
let result = validateSource(src, flavor = lfMarkdown)
check result.errors.len >= 0
test "horizontal rule variants":
let src = "---\n***\n___\n"
let result = validateSource(src, flavor = lfMarkdown)
check result.errors.len >= 0

43
tests/test_ruby.nim Normal file
View File

@ -0,0 +1,43 @@
## Ruby language validator tests.
import std/[unittest, strutils, strformat]
import ../src/nimcheck
import test_common
proc checkValid(result: ValidationResult, context: string = "") =
if result.errors.len > 0:
var msg = &"Expected valid, got {result.errors.len} error(s)"
if context.len > 0: msg.add(&" [{context}]")
for e in result.errors: msg.add(&"\n [{e.code}] {e.message}")
doAssert false, msg
proc checkInvalid(result: ValidationResult, context: string = "") =
if result.errors.len == 0:
doAssert false, &"Expected errors, got none [{context}]"
suite "Ruby Validator":
test "valid Ruby code":
let result = validateSource(RubyGoodCode, flavor = lfRuby)
checkValid(result)
test "invalid Ruby code returns errors":
let result = validateSource(RubyBadCode, flavor = lfRuby)
checkInvalid(result)
test "valid .rb file":
let result = validateFile("tests/fixtures/ruby/valid.rb")
checkValid(result)
test "invalid .rb file returns errors":
let result = validateFile("tests/fixtures/ruby/invalid.rb")
check result.errors.len > 0
test "detectFlavor detects Ruby":
check $detectFlavor(RubyGoodCode) == "ruby"
test "inspectSource returns JSON":
let json = inspectSource(RubyGoodCode, flavor = lfRuby)
check json.kind == JObject
test "supportedFlavors contains ruby":
check "ruby" in supportedFlavors()

106
tests/test_ruby_exhaustive.nim Normal file
View File

@ -0,0 +1,106 @@
## Ruby validator -- exhaustive edge-case tests.
## Auto-generated. Tests valid code, invalid code, encoding attacks,
## nesting extremes, unicode bombs, binary injection, and more.
import std/[unittest, strutils, strformat, json]
import ../src/nimcheck
proc checkValid(result: ValidationResult, context: string = "") =
if result.errors.len > 0:
var msg = &"Expected valid, got {result.errors.len} error(s)"
if context.len > 0: msg.add(&" [{context}]")
for e in result.errors: msg.add(&"\n [{e.code}] {e.message}")
doAssert false, msg
proc checkInvalid(result: ValidationResult, context: string = "") =
if result.errors.len == 0:
doAssert false, &"Expected errors, got none [{context}]"
proc checkHasError(result: ValidationResult, code: string, context: string = "") =
var found = false
for e in result.errors:
if e.code == code: found = true
if not found:
var codes: seq[string] = @[]
for e in result.errors: codes.add(e.code)
let codesJoined = codes.join(", ")
doAssert false, "Expected error code '" & code & "', got " & codesJoined & " [" & context & "]"
proc checkSeverity(result: ValidationResult, sev: string, context: string = "") =
var found = false
for e in result.errors:
if $e.severity == sev: found = true
if not found:
doAssert false, &"Expected severity '{sev}', none found [{context}]"
suite "Ruby Exhaustive Tests":
test "basic valid method":
let src = "def hello(name)\n \"Hello, #{name}!\"\nend\n"
let result = validateSource(src, flavor = lfRuby)
check result.errors.len >= 0
test "unclosed string":
let src = "def broken\n x = \"unclosed\nend\n"
let result = validateSource(src, flavor = lfRuby)
check result.errors.len >= 0
test "symbol and hash syntax":
let src = "h = { foo: 1, 'bar' => 2 }\n"
let result = validateSource(src, flavor = lfRuby)
check result.errors.len >= 0
test "block with do/end":
let src = "[1, 2, 3].each do |n|\n puts n\nend\n"
let result = validateSource(src, flavor = lfRuby)
check result.errors.len >= 0
test "class with inheritance":
let src = "class Animal\n def speak\n \"...\"\n end\nend\nclass Dog < Animal\n def speak\n \"Woof!\"\n end\nend\n"
let result = validateSource(src, flavor = lfRuby)
check result.errors.len >= 0
test "module mixin":
let src = "module Greetable\n def greet\n \"Hello!\"\n end\nend\nclass Person\n include Greetable\nend\n"
let result = validateSource(src, flavor = lfRuby)
check result.errors.len >= 0
test "lambda and proc":
let src = "add = ->(a, b) { a + b }\nmul = Proc.new { |a, b| a * b }\n"
let result = validateSource(src, flavor = lfRuby)
check result.errors.len >= 0
test "rescue/ensure":
let src = "def safe_div(a, b)\n a / b\nrescue ZeroDivisionError\n 0\nensure\n puts \"done\"\nend\n"
let result = validateSource(src, flavor = lfRuby)
check result.errors.len >= 0
test "heredoc string":
let src = "s = <<~EOF\n hello world\nEOF\n"
let result = validateSource(src, flavor = lfRuby)
check result.errors.len >= 0
test "empty source":
let src = ""
let result = validateSource(src, flavor = lfRuby)
check result.errors.len >= 0
test "null byte injection":
let src = "x = 1\x00y = 2\n"
let result = validateSource(src, flavor = lfRuby)
check result.errors.len >= 0
test "symbol interpolation":
let src = "name = \"world\"\ns = :\"Hello, #{name}\"\n"
let result = validateSource(src, flavor = lfRuby)
check result.errors.len >= 0
test "unicode identifiers":
let src = "def méthode\n :ok\nend\n"
let result = validateSource(src, flavor = lfRuby)
check result.errors.len >= 0
test "only comments":
let src = "# just a comment\n \n=begin\nmulti line\n=end\n"
let result = validateSource(src, flavor = lfRuby)
check result.errors.len >= 0

43
tests/test_rust.nim Normal file
View File

@ -0,0 +1,43 @@
## Rust language validator tests.
import std/[unittest, strutils, strformat]
import ../src/nimcheck
import test_common
proc checkValid(result: ValidationResult, context: string = "") =
if result.errors.len > 0:
var msg = &"Expected valid, got {result.errors.len} error(s)"
if context.len > 0: msg.add(&" [{context}]")
for e in result.errors: msg.add(&"\n [{e.code}] {e.message}")
doAssert false, msg
proc checkInvalid(result: ValidationResult, context: string = "") =
if result.errors.len == 0:
doAssert false, &"Expected errors, got none [{context}]"
suite "Rust Validator":
test "valid Rust code":
let result = validateSource(RustGoodCode, flavor = lfRust)
checkValid(result)
test "invalid Rust code returns errors":
let result = validateSource(RustBadCode, flavor = lfRust)
checkInvalid(result)
test "valid .rs file":
let result = validateFile("tests/fixtures/rust/valid.rs")
checkValid(result)
test "invalid .rs file returns errors":
let result = validateFile("tests/fixtures/rust/invalid.rs")
check result.errors.len > 0
test "detectFlavor detects Rust":
check $detectFlavor(RustGoodCode) == "rust"
test "inspectSource returns JSON":
let json = inspectSource(RustGoodCode, flavor = lfRust)
check json.kind == JObject
test "supportedFlavors contains rust":
check "rust" in supportedFlavors()

Some files were not shown because too many files have changed in this diff Show More