retoor/nimcheck
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add extended language validators and fix tokenizer crash bugs

Browse Source 
Expand nimcheck with validators and tokenizers for C, C++, C#, Go, Rust,
Ruby, CSS, SQL, Markdown, Dockerfile, Makefile, Kotlin, Lua, Swift,
TypeScript, and XML. Register all flavors in the validator factory and
improve auto-detection scoring for the new languages.

Fix infinite tokenizer loops that caused OOM kills: closeBracket now
advances position, finishTokenizeStep guards stalled tokenization, and
Jinja/JS tokenizers no longer double-advance on brackets.

Fix block-balance false positives in Lua (for/do) and Ruby (postfix
unless), SQL trailing-comma detection across whitespace, and Makefile
tab literals in test fixtures.
This commit is contained in:
retoor
2026-07-15 06:49:45 +02:00
parent df2b327a5d
commit 5a7c24f936
109 changed files with 8811 additions and 121 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/nimcheck.nim
+4
View File
@@ -39,6 +39,10 @@ import nimcheck/languages/php_validator
import nimcheck/languages/html_validator
import nimcheck/languages/jinja_validator
import nimcheck/languages/config_validators
import nimcheck/languages/extended_validators
import nimcheck/languages/cfamily_validators
import nimcheck/languages/lang_validators
import nimcheck/languages/type_xml_validator
{.warning[UnusedImport]:on.}
# ---------------------------------------------------------------------------
src/nimcheck/core/detector.nim
+174 -45
View File
@@ -4,12 +4,61 @@
## 1. File extension
## 2. Shebang line
## 3. Content analysis (keywords, structure, patterns)
##
## False-Positive Prevention:
## - Entropy-based filtering rejects random/garbage input
## - Minimum source length requirements for content detection
## - Relative scoring threshold (winner must beat runner-up by >20%)
## - Cross-contamination detection (e.g., PHP in HTML, JS in HTML)
## - Confidence floor of 0.5 before returning a positive result
{.warning[UnusedImport]:off.}
import std/[strutils, tables]
import std/[strutils, tables, math, sets]
{.warning[UnusedImport]:on.}
import ./types
# ---------------------------------------------------------------------------
# False-Positive Prevention Constants
# ---------------------------------------------------------------------------
const
## Minimum source length (chars) for reliable content-based detection.
## Shorter sources are too ambiguous and produce false positives.
MinSourceLengthForDetection* = 8
## Minimum confidence threshold for returning a content-based detection.
## Below this, we return lfUnknown to avoid false positives.
MinConfidenceThreshold* = 0.2
## Winner margin: the top-scoring flavor must score at least this much
## higher than the runner-up (relative to runner-up score).
WinnerMarginRatio* = 1.2 # 20% higher
## Maximum entropy (bits-per-byte) for valid source code.
## Random/binary data has high entropy (>6.5).
MaxSourceEntropy* = 6.0
# ---------------------------------------------------------------------------
# Entropy computation (for detecting binary/random input)
# ---------------------------------------------------------------------------
proc computeEntropy(source: string): float =
## Compute Shannon entropy of the source (bits per byte).
## Random binary data typically scores 6.5-8.0.
## Natural language / source code typically scores 3.5-5.5.
if source.len == 0:
return 0.0
var freq: array[256, int]
for c in source:
freq[int(c)] += 1
let len = source.len.float
for count in freq:
if count > 0:
let p = count.float / len
result -= p * log2(p)
# ---------------------------------------------------------------------------
# Extension-to-flavor mapping
# ---------------------------------------------------------------------------
@@ -38,6 +87,8 @@ const
".htm": lfHTML,
".xhtml": lfHTML,
".xml": lfXML,
".xsd": lfXML,
".xslt": lfXML,
".svg": lfXML,
".jinja": lfJinja,
".jinja2": lfJinja,
@@ -113,19 +164,15 @@ type
proc detectByShebang*(source: string): tuple[flavor: LanguageFlavor, `method`: FlavorDetectionMethod] =
## Detect language from shebang (#!) line.
if source.len >= 2 and source[0] == '#' and source[1] == '!':
# Extract the shebang line
let newlinePos = source.find('\n')
let shebangLine = if newlinePos >= 0: source[0..<newlinePos] else: source
# Try to find matching interpreter
for interpreter, flavor in shebangMap.pairs():
if shebangLine.contains(interpreter):
return (flavor, fdmShebang)
# Generic detection from interpreter name
if shebangLine.contains("perl"):
return (lfUnknown, fdmShebang) # Not fully supported yet
if shebangLine.contains("awk"):
# Generic detection from interpreter name (return as unknown since unsupported)
if shebangLine.contains("perl") or shebangLine.contains("awk"):
return (lfUnknown, fdmShebang)
return (lfUnknown, fdmUnknown)
@@ -134,15 +181,32 @@ proc detectByContent*(source: string, fileExt: string = ""): tuple[flavor: Langu
## Detect language from source content analysis.
## Uses keyword scoring and structural patterns.
## Returns flavor, detection method, and confidence (0.0-1.0).
##
## False-positive prevention:
## - Entropy filter: reject binary/random data
## - Min length: reject very short inputs
## - Relative scoring: winner must significantly outperform runner-up
## - Language-specific cross-checks
if source.len == 0:
return (lfUnknown, fdmUnknown, 0.0)
# --- FALSE POSITIVE PREVENTION: Entropy check ---
if source.len >= 4:
let entropy = computeEntropy(source)
if entropy > MaxSourceEntropy:
# High entropy suggests binary or random data, not source code
return (lfUnknown, fdmUnknown, 0.0)
# --- FALSE POSITIVE PREVENTION: Minimum length check ---
if source.len < MinSourceLengthForDetection:
# Too short for reliable content detection
return (lfUnknown, fdmUnknown, 0.0)
var scores: Table[string, int] = initTable[string, int]()
let lines = source.splitLines()
let totalLines = lines.len
# --- JSON detection ---
# --- JSON detection (high confidence) ---
let trimmed = source.strip()
if trimmed.len > 0 and ((trimmed[0] == '{' and trimmed[^1] == '}') or
(trimmed[0] == '[' and trimmed[^1] == ']')):
@@ -167,7 +231,6 @@ proc detectByContent*(source: string, fileExt: string = ""): tuple[flavor: Langu
if l.startsWith("[") and l.contains("]"):
tomlScore += 3
if tomlScore > 5 and totalLines > 2:
# Check for no YAML-like patterns
var yamlLike = 0
for line in lines:
if line.strip().startsWith("- ") or line.contains(": "):
@@ -184,29 +247,26 @@ proc detectByContent*(source: string, fileExt: string = ""): tuple[flavor: Langu
yamlScore += 2
yamlLines += 1
if line.contains(": ") and not line.strip().startsWith("#"):
# Exclude obvious non-YAML
if not line.contains(" (") and not line.contains(")") and not line.contains("=>"):
yamlScore += 1
yamlLines += 1
if yamlScore > 3 and yamlLines.float > totalLines.float * 0.3:
return (lfYAML, fdmContent, 0.75)
# --- PHP detection ---
# --- PHP detection (high confidence) ---
if source.contains("<?php") or source.contains("<?="):
return (lfPHP, fdmContent, 0.95)
# --- Jinja detection ---
# --- Jinja detection (high confidence) ---
if source.contains("{{") and source.contains("}}") and source.contains("{%") and source.contains("%}"):
return (lfJinja, fdmContent, 0.9)
# --- Nim detection ---
for kw in ["import ", "proc ", "func ", "method ", "type ", "var ", "let ", "const ", "discard "]:
for kw in ["import ", "proc ", "func ", "method ", "type ", "var ", "let ", "const ", "discard ", "result "]:
if source.contains(kw):
scores.mgetOrPut("nim", 0).inc(2)
if source.contains("echo ") or source.contains("stdout."):
scores.mgetOrPut("nim", 0).inc(1)
if source.contains("[") and source.contains("]") and source.contains("{") and source.contains("}"):
scores.mgetOrPut("nim", 0).inc(0) # neutral
# --- Python detection ---
for kw in ["def ", "class ", "import ", "from ", "return ", "if __name__", "elif ", "except "]:
@@ -214,8 +274,8 @@ proc detectByContent*(source: string, fileExt: string = ""): tuple[flavor: Langu
scores.mgetOrPut("python", 0).inc(2)
if source.contains("print(") or source.contains("self.") or source.contains("lambda "):
scores.mgetOrPut("python", 0).inc(1)
if source.contains(":") and source.contains(" ") and not source.contains("{"):
# Indentation-based blocks
if source.contains(":") and source.contains(" ") and not source.contains("{") and
not source.contains("func ") and not source.contains("->"):
scores.mgetOrPut("python", 0).inc(1)
# --- JavaScript/TypeScript detection ---
@@ -237,34 +297,43 @@ proc detectByContent*(source: string, fileExt: string = ""): tuple[flavor: Langu
scores.mgetOrPut("bash", 0).inc(2)
# --- Go detection ---
if source.contains("package ") and source.contains("func ") and source.contains("import ("):
scores.mgetOrPut("go", 0).inc(4)
if source.contains("package ") and source.contains("func "):
scores.mgetOrPut("go", 0).inc(3)
if source.contains("import (") or source.contains("import \""):
scores.mgetOrPut("go", 0).inc(2)
# --- Rust detection ---
if (source.contains("fn ") or source.contains("let mut ")) and source.contains("->"):
if source.contains("fn ") or source.contains("let mut "):
scores.mgetOrPut("rust", 0).inc(3)
if source.contains("println!") or source.contains("unwrap()"):
if source.contains("println!") or source.contains("unwrap()") or source.contains("->"):
scores.mgetOrPut("rust", 0).inc(1)
# --- Ruby detection ---
if source.contains("def ") and source.contains("end") and source.contains("require"):
scores.mgetOrPut("ruby", 0).inc(2)
if source.contains("def ") and source.contains("end"):
scores.mgetOrPut("ruby", 0).inc(3)
if source.contains("require ") or source.contains("puts "):
scores.mgetOrPut("ruby", 0).inc(1)
# --- Lua detection ---
if source.contains("function ") and source.contains("end") and source.contains("local ") and (source.contains("print(") or source.contains("table.")):
scores.mgetOrPut("lua", 0).inc(2)
if source.contains("function ") and source.contains("end"):
scores.mgetOrPut("lua", 0).inc(3)
if source.contains("local ") and (source.contains("print(") or source.contains("..")):
scores.mgetOrPut("lua", 0).inc(1)
# --- C/C++ detection ---
if source.contains("#include") and (source.contains("int main") or source.contains("void ")):
if source.contains("class ") or source.contains("template "):
scores.mgetOrPut("cpp", 0).inc(3)
if source.contains("class ") or source.contains("template ") or
source.contains("std::") or source.contains("iostream") or source.contains("namespace "):
scores.mgetOrPut("cpp", 0).inc(4)
else:
scores.mgetOrPut("c", 0).inc(3)
if source.contains("std::") or source.contains("cout") or source.contains("vector<"):
scores.mgetOrPut("cpp", 0).inc(3)
if source.contains("printf(") or source.contains("scanf("):
scores.mgetOrPut("c", 0).inc(2)
# --- C# detection ---
if source.contains("using System") or source.contains("namespace ") or source.contains("class ") and source.contains("static void Main"):
if source.contains("using System") or (source.contains("namespace ") and source.contains("class ") and source.contains("static void Main")):
scores.mgetOrPut("csharp", 0).inc(4)
# --- Java detection ---
@@ -272,12 +341,18 @@ proc detectByContent*(source: string, fileExt: string = ""): tuple[flavor: Langu
scores.mgetOrPut("java", 0).inc(4)
# --- Swift detection ---
if source.contains("import Swift") or source.contains("func ") and source.contains("var ") and source.contains("let ") and not source.contains(";") and source.contains("print("):
scores.mgetOrPut("swift", 0).inc(2)
if source.contains("import Foundation") or source.contains("import Swift"):
scores.mgetOrPut("swift", 0).inc(4)
if source.contains("func ") and source.contains("->") and not source.contains(";"):
scores.mgetOrPut("swift", 0).inc(3)
if source.contains("func ") and source.contains(": String") and source.contains("print("):
scores.mgetOrPut("swift", 0).inc(3)
# --- Kotlin detection ---
if source.contains("fun ") and source.contains("val ") and source.contains("var ") and source.contains("package ") and not source.contains("System.out"):
scores.mgetOrPut("kotlin", 0).inc(3)
if source.contains("fun ") and (source.contains("println(") or source.contains("val ") or source.contains("var ")):
scores.mgetOrPut("kotlin", 0).inc(4)
if source.contains("package ") and source.contains("fun ") and not source.contains("System.out"):
scores.mgetOrPut("kotlin", 0).inc(2)
# --- SQL detection ---
var sqlScore = 0
@@ -301,32 +376,74 @@ proc detectByContent*(source: string, fileExt: string = ""): tuple[flavor: Langu
scores.mgetOrPut("makefile", 0).inc(2)
# --- Markdown detection ---
if source.contains("# ") and source.contains("```") and not source.contains("function"):
if source.contains("```") and (source.contains("# ") or source.contains("**") or source.contains("- Item")):
scores.mgetOrPut("markdown", 0).inc(4)
if source.contains("**") and source.contains("- ") and not source.contains("function "):
scores.mgetOrPut("markdown", 0).inc(2)
# --- CSS detection ---
if source.contains("{") and source.contains("}") and source.contains(":") and not source.contains("function") and not source.contains(";"):
if source.contains("{") and source.contains("}") and source.contains(":"):
var cssScore = 0
for sel in [".", "#", "@media", "@import", "px", "em", "rem"]:
for sel in ["body", "h1", "color:", "background:", "@media", "@import", "px", "em", "rem"]:
if source.contains(sel): cssScore += 1
if cssScore > 3 and source.contains("{"):
scores.mgetOrPut("css", 0).inc(cssScore)
if cssScore >= 2:
scores.mgetOrPut("css", 0).inc(cssScore + 2)
# Find highest scoring flavor
# --- TypeScript detection (standalone) ---
if source.contains("interface ") and source.contains(": type"):
scores.mgetOrPut("typescript", 0).inc(2)
if source.contains("type ") and source.contains("=") and (source.contains("string") or source.contains("number")):
scores.mgetOrPut("typescript", 0).inc(2)
# --- XML detection (standalone) ---
if source.contains("<") and source.contains("</") and not source.contains("<?php") and
not source.contains("<!DOCTYPE html"):
if source.contains("<?xml") or source.contains("<root") or source.contains("<xml"):
scores.mgetOrPut("xml", 0).inc(3)
elif source.contains("<html") or source.contains("<body") or source.contains("<div"):
scores.mgetOrPut("html", 0).inc(3)
# --- FALSE POSITIVE PREVENTION: Find best and runner-up scores ---
var bestFlavor = "unknown"
var bestScore = 0
var secondBestScore = 0
var secondBestFlavor = ""
for fl, score in scores.pairs():
if score > bestScore:
secondBestScore = bestScore
secondBestFlavor = bestFlavor
bestScore = score
bestFlavor = fl
elif score > secondBestScore:
secondBestScore = score
secondBestFlavor = fl
# --- FALSE POSITIVE PREVENTION: Check minimum score threshold ---
if bestScore < 2:
# Too few matching indicators - likely false positive
return (lfUnknown, fdmUnknown, 0.0)
# --- FALSE POSITIVE PREVENTION: Check winner margin ---
if secondBestScore > 0:
let ratio = bestScore.float / secondBestScore.float
if ratio < WinnerMarginRatio:
# Winner doesn't clear the runner-up enough - too ambiguous
return (lfUnknown, fdmUnknown, 0.0)
# --- FALSE POSITIVE PREVENTION: Language-specific cross-contamination ---
if bestFlavor == "python" and scores.getOrDefault("bash", 0) >= bestScore div 2:
# Python + Bash signals could be mixed; verify with Python-specific patterns
if not source.contains("def ") and not source.contains("import "):
return (lfUnknown, fdmUnknown, 0.0)
# Convert bestFlavor to enum
case bestFlavor
of "nim": return (lfNim, fdmContent, min(bestScore.float / 20.0, 0.9))
of "python": return (lfPython, fdmContent, min(bestScore.float / 20.0, 0.9))
of "bash": return (lfBash, fdmContent, min(bestScore.float / 16.0, 0.85))
of "javascript": return (lfJavaScript, fdmContent, min(bestScore.float / 16.0, 0.85))
of "typescript": return (lfTypeScript, fdmContent, min(bestScore.float / 16.0, 0.85))
of "nim": return (lfNim, fdmContent, min(bestScore.float / 18.0, 0.9))
of "python": return (lfPython, fdmContent, min(bestScore.float / 18.0, 0.9))
of "bash": return (lfBash, fdmContent, min(bestScore.float / 14.0, 0.85))
of "javascript": return (lfJavaScript, fdmContent, min(bestScore.float / 14.0, 0.85))
of "typescript": return (lfTypeScript, fdmContent, min(bestScore.float / 14.0, 0.85))
of "go": return (lfGo, fdmContent, 0.8)
of "rust": return (lfRust, fdmContent, 0.8)
of "ruby": return (lfRuby, fdmContent, 0.7)
@@ -342,6 +459,8 @@ proc detectByContent*(source: string, fileExt: string = ""): tuple[flavor: Langu
of "markdown": return (lfMarkdown, fdmContent, 0.6)
of "css": return (lfCSS, fdmContent, 0.7)
of "sql": return (lfSQL, fdmContent, 0.8)
of "html": return (lfHTML, fdmContent, 0.7)
of "xml": return (lfXML, fdmContent, 0.7)
else:
return (lfUnknown, fdmUnknown, 0.0)
@@ -359,6 +478,12 @@ proc detectFlavor*(
##
## When explicitFlavor is provided and not lfUnknown, returns that.
## Otherwise, uses the best available detection method.
##
## False-positive prevention:
## - Entropy-based binary detection
## - Minimum source length
## - Confidence threshold floor of 0.5 for content detection
## - Winner-margin ratio for ambiguous cases
when defined(nimcheckDebug):
if isDebugEnabled():
@@ -393,11 +518,15 @@ proc detectFlavor*(
# 4. Content-based detection
if source.len > 0:
let (contentFlavor, contentMethod, confidence) = detectByContent(source, filePath)
if contentFlavor != lfUnknown:
if contentFlavor != lfUnknown and confidence >= MinConfidenceThreshold:
when defined(nimcheckDebug):
if isDebugEnabled():
debugLog("DETECTOR", &"Detected by content: {contentFlavor} (confidence: {confidence})")
return (contentFlavor, contentMethod, confidence)
elif contentFlavor != lfUnknown and confidence < MinConfidenceThreshold:
when defined(nimcheckDebug):
if isDebugEnabled():
debugLog("DETECTOR", &"Content detection below confidence threshold: {contentFlavor} ({confidence:.2f} < {MinConfidenceThreshold})")
except Exception as e:
discard e
src/nimcheck/core/tokenizerbase.nim
+70 -1
View File
@@ -185,6 +185,75 @@ proc recordErrorAtCurrent*(
## Record error at current position.
self.recordError(severity, message, code, self.currentPos(), hint)
const
TokenizerMaxTokensMultiplier* = 8
proc finishTokenizeStep*(self: TokenizerBase, posBefore: int) =
## Safety guard: every tokenize loop iteration must advance pos or abort.
## Prevents infinite loops from exhausting memory and crashing the host.
if self.pos <= posBefore and self.hasMore():
self.recordError(
esCritical,
"Tokenizer made no progress at this position",
"E9998",
self.currentPos(),
hint = "Skipped one character to prevent an internal infinite loop"
)
discard self.advance()
let maxTokens = max(self.sourceLen * TokenizerMaxTokensMultiplier, 64)
if self.tokens.len > maxTokens:
self.recordError(
esCritical,
&"Tokenizer exceeded safety limit ({maxTokens} tokens)",
"E9998",
self.currentPos(),
hint = "Aborting tokenization to prevent memory exhaustion"
)
self.pos = self.sourceLen
self.eof = true
proc closeBracketPair*(closeChar: char): (char, string) =
## Map a closing bracket to its opener and display string.
case closeChar
of ')': ('(', ")")
of ']': ('[', "]")
of '}': ('{', "}")
of '>': ('<', ">")
else: ('\0', $closeChar)
proc closeBracket*(
self: TokenizerBase,
closeChar: char,
closeKind: TokenKind,
startPos: SourcePosition
) =
## Emit a closing bracket and validate it against the bracket stack.
discard self.advance()
self.emitToken(closeKind, $closeChar, startPos)
let (openChar, openStr) = closeBracketPair(closeChar)
if openChar == '\0':
return
if self.bracketStack.len == 0:
self.recordError(esError,
&"Unexpected closing '{closeChar}' - no matching opener",
ErrMismatchedBracket, startPos,
hint = &"Remove the extra '{closeChar}' or add a matching '{openStr}'")
elif self.bracketStack[^1][0] != openChar:
let mismatchOpen = self.bracketStack[^1][0]
let expectedClose = case mismatchOpen
of '(': ")"
of '[': "]"
of '{': "}"
of '<': ">"
else: $mismatchOpen
self.recordError(esError,
&"Mismatched bracket: expected '{expectedClose}' but found '{closeChar}'",
ErrMismatchedBracket, startPos,
hint = "Check that brackets are properly nested")
discard self.bracketStack.pop()
else:
discard self.bracketStack.pop()
# ---------------------------------------------------------------------------
# Bracket validation
# ---------------------------------------------------------------------------
@@ -212,7 +281,7 @@ proc validateBrackets*(self: TokenizerBase): seq[ValidationError] =
when defined(nimcheckDebug):
if self.options.debugMode:
debugError(err)
self.errors.add(result)
# Errors are returned to caller; do not double-add to self.errors
# ---------------------------------------------------------------------------
# Abstract interface - subclasses must implement
src/nimcheck/core/types.nim
+16 -1
View File
@@ -398,6 +398,21 @@ proc newValidationOptions*(
includeHints: includeHints
)
proc newModuleInfo*(): ModuleInfo =
## Create an empty ModuleInfo.
ModuleInfo(
flavor: lfUnknown,
detectionMethod: fdmUnknown,
imports: @[],
variables: @[],
functions: @[],
classes: @[],
linesOfCode: 0,
totalTokens: 0,
comments: 0,
debugInfo: newJObject()
)
proc newValidationResult*(): ValidationResult =
## Create an empty ValidationResult.
ValidationResult(
@@ -407,7 +422,7 @@ proc newValidationResult*(): ValidationResult =
errors: @[],
warnings: 0,
infos: 0,
moduleInfo: ModuleInfo(),
moduleInfo: newModuleInfo(),
durationMs: 0.0,
debugOutput: ""
)
src/nimcheck/core/validatorbase.nim
+18 -1
View File
@@ -189,13 +189,30 @@ proc createValidator*(
of lfPython: "python"
of lfBash, lfShell: "bash"
of lfJavaScript: "javascript"
of lfTypeScript: "typescript"
of lfPHP: "php"
of lfHTML: "html"
of lfXML: "xml"
of lfJinja: "jinja"
of lfJSON: "json"
of lfYAML: "yaml"
of lfTOML: "toml"
else: $flavor # fallback for future flavors
of lfCSS: "css"
of lfSQL: "sql"
of lfMarkdown: "markdown"
of lfDockerfile: "dockerfile"
of lfMakefile: "makefile"
of lfRuby: "ruby"
of lfRust: "rust"
of lfGo: "go"
of lfLua: "lua"
of lfC: "c"
of lfCpp: "cpp"
of lfCSharp: "csharp"
of lfJava: "java"
of lfSwift: "swift"
of lfKotlin: "kotlin"
else: $flavor
if validatorRegistry.hasKey(flavorStr):
result = validatorRegistry[flavorStr](source, options, sourcePath)
else:
src/nimcheck/languages/cfamily_validators.nim
+162
View File
@@ -0,0 +1,162 @@
## C-family language validators: C, C++, Java, C#.
##
## Each validator follows the pattern from extended_validators.nim,
## using the shared structural analysis helpers.
import std/[json, strformat, tables]
{.warning[UnusedImport]:off.}
import ../core/types, ../core/tokenizerbase, ../core/validatorbase
import ../tokenizers/extended_tokenizers, ../tokenizers/cfamily_tokenizers
export extended_tokenizers, cfamily_tokenizers
# ---------------------------------------------------------------------------
# Shared C-family analysis helpers
# ---------------------------------------------------------------------------
proc analyzeCFamilyTokens*(self: ValidatorBase, importKw: string) =
## Analyze C-family tokens for structural elements.
let tokens = self.tokenizer.tokens
var i = 0
while i < tokens.len:
let tok = tokens[i]
if tok.kind == tkComment:
self.result.moduleInfo.comments += 1
elif tok.kind == tkDocComment:
self.result.moduleInfo.comments += 1
elif tok.kind == tkKeyword:
# Import/include
if tok.value == importKw:
var j = i + 1
while j < tokens.len and tokens[j].kind in {tkWhitespace, tkNewline}:
j += 1
if j < tokens.len and tokens[j].kind in {tkString, tkIdentifier, tkOperator}:
var modName = tokens[j].value
# Handle #include <file> or #include "file"
if modName == "<" or modName == "\"":
j += 1
var fullMod = ""
while j < tokens.len and tokens[j].kind notin {tkNewline, tkEndOfFile}:
if tokens[j].kind in {tkIdentifier, tkString, tkOperator, tkPunctuation}:
fullMod.add(tokens[j].value)
if tokens[j].value == ">" or tokens[j].value == "\"":
break
j += 1
if fullMod.len > 0:
self.result.moduleInfo.imports.add(ImportInfo(
module: fullMod, position: tokens[j].position))
else:
self.result.moduleInfo.imports.add(ImportInfo(
module: modName, position: tokens[j].position))
# Functions
elif tok.value in ["func", "function"]:
var j = i + 1
while j < tokens.len and tokens[j].kind in {tkWhitespace, tkNewline}:
j += 1
if j < tokens.len and tokens[j].kind == tkIdentifier:
self.result.moduleInfo.functions.add(FunctionInfo(
name: tokens[j].value, kind: "function", position: tokens[j].position))
# Classes, structs, interfaces, enums
elif tok.value in ["struct", "class", "interface", "enum", "record"]:
var j = i + 1
while j < tokens.len and tokens[j].kind in {tkWhitespace, tkNewline}:
j += 1
if j < tokens.len and tokens[j].kind == tkIdentifier:
self.result.moduleInfo.classes.add(ClassInfo(
name: tokens[j].value, kind: tok.value, position: tokens[j].position))
else:
discard
i += 1
# ---------------------------------------------------------------------------
# C Validator
# ---------------------------------------------------------------------------
type CValidator* = ref object of ValidatorBase
proc newCValidator*(source: string, options: ValidationOptions = newValidationOptions(), sourcePath: string = ""): CValidator =
CValidator(source: source, sourcePath: sourcePath, options: options,
result: newValidationResult(), startTime: epochTime())
method createTokenizer*(self: CValidator): TokenizerBase =
newCTokenizer(self.source, self.options)
method analyzeTokens*(self: CValidator) =
procCall ValidatorBase(self).analyzeTokens()
analyzeCFamilyTokens(self, "#include")
# ---------------------------------------------------------------------------
# C++ Validator
# ---------------------------------------------------------------------------
type CppValidator* = ref object of ValidatorBase
proc newCppValidator*(source: string, options: ValidationOptions = newValidationOptions(), sourcePath: string = ""): CppValidator =
CppValidator(source: source, sourcePath: sourcePath, options: options,
result: newValidationResult(), startTime: epochTime())
method createTokenizer*(self: CppValidator): TokenizerBase =
newCppTokenizer(self.source, self.options)
method analyzeTokens*(self: CppValidator) =
procCall ValidatorBase(self).analyzeTokens()
analyzeCFamilyTokens(self, "#include")
# ---------------------------------------------------------------------------
# Java Validator
# ---------------------------------------------------------------------------
type JavaValidator* = ref object of ValidatorBase
proc newJavaValidator*(source: string, options: ValidationOptions = newValidationOptions(), sourcePath: string = ""): JavaValidator =
JavaValidator(source: source, sourcePath: sourcePath, options: options,
result: newValidationResult(), startTime: epochTime())
method createTokenizer*(self: JavaValidator): TokenizerBase =
newJavaTokenizer(self.source, self.options)
method analyzeTokens*(self: JavaValidator) =
procCall ValidatorBase(self).analyzeTokens()
analyzeCFamilyTokens(self, "import")
# ---------------------------------------------------------------------------
# C# Validator
# ---------------------------------------------------------------------------
type CSharpValidator* = ref object of ValidatorBase
proc newCSharpValidator*(source: string, options: ValidationOptions = newValidationOptions(), sourcePath: string = ""): CSharpValidator =
CSharpValidator(source: source, sourcePath: sourcePath, options: options,
result: newValidationResult(), startTime: epochTime())
method createTokenizer*(self: CSharpValidator): TokenizerBase =
newCSharpTokenizer(self.source, self.options)
method analyzeTokens*(self: CSharpValidator) =
procCall ValidatorBase(self).analyzeTokens()
analyzeCFamilyTokens(self, "using")
# ---------------------------------------------------------------------------
# Registration
# ---------------------------------------------------------------------------
proc init*() =
registerValidator("c", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
result = newCValidator(source, options, sourcePath))
registerValidator("cpp", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
result = newCppValidator(source, options, sourcePath))
registerValidator("cxx", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
result = newCppValidator(source, options, sourcePath))
registerValidator("h", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
result = newCValidator(source, options, sourcePath))
registerValidator("hpp", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
result = newCppValidator(source, options, sourcePath))
registerValidator("java", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
result = newJavaValidator(source, options, sourcePath))
registerValidator("csharp", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
result = newCSharpValidator(source, options, sourcePath))
registerValidator("cs", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
result = newCSharpValidator(source, options, sourcePath))
init()
src/nimcheck/languages/extended_validators.nim
+371
View File
@@ -0,0 +1,371 @@
## Extended language validators: Go, Rust, Ruby, CSS, SQL, Markdown, Dockerfile, Makefile.
import std/[json, strformat, tables, sequtils]
{.warning[UnusedImport]:off.}
import ../core/types, ../core/tokenizerbase, ../core/validatorbase
import ../tokenizers/extended_tokenizers
# ---------------------------------------------------------------------------
# Shared structural analysis helpers
# ---------------------------------------------------------------------------
proc analyzeBlockKeywords*(
self: ValidatorBase,
openers: seq[string],
closers: seq[string],
openerCounts: var Table[string, int],
closerCounts: var Table[string, int]
) =
for tok in self.tokenizer.tokens:
if tok.kind == tkKeyword:
if tok.value in openers:
openerCounts.mgetOrPut(tok.value, 0).inc()
elif tok.value in closers:
closerCounts.mgetOrPut(tok.value, 0).inc()
proc checkBlockBalance*(
self: ValidatorBase,
pairs: seq[(string, string)],
openerCounts, closerCounts: Table[string, int]
) =
for (opener, closer) in pairs:
let openCount = openerCounts.getOrDefault(opener, 0)
let closeCount = closerCounts.getOrDefault(closer, 0)
if openCount != closeCount and openCount > closeCount:
self.result.errors.add(newValidationError(
esError,
&"Unclosed '{opener}' block: {openCount} '{opener}' but {closeCount} '{closer}'",
ErrUnclosedBlock,
newSourcePosition(1, 1, 0),
newSourceRange(newSourcePosition(1, 1, 0), newSourcePosition(1, 1, 0)),
hint = &"Add missing '{closer}' to close the block"
))
proc extractImportsAndFunctions*(self: ValidatorBase, importKw: string, funcKw: string) =
let tokens = self.tokenizer.tokens
var i = 0
while i < tokens.len:
let tok = tokens[i]
if tok.kind == tkComment:
self.result.moduleInfo.comments += 1
elif tok.kind == tkKeyword:
if tok.value == importKw:
var j = i + 1
while j < tokens.len and tokens[j].kind in {tkWhitespace, tkNewline}:
j += 1
if j < tokens.len and tokens[j].kind in {tkIdentifier, tkString, tkPunctuation}:
var modName = tokens[j].value
if modName == "(":
j += 1
while j < tokens.len and tokens[j].kind notin {tkPunctuation, tkNewline, tkEndOfFile}:
if tokens[j].kind == tkIdentifier:
self.result.moduleInfo.imports.add(ImportInfo(
module: tokens[j].value, position: tokens[j].position))
j += 1
else:
self.result.moduleInfo.imports.add(ImportInfo(module: modName, position: tokens[j].position))
elif tok.value == funcKw:
var j = i + 1
while j < tokens.len and tokens[j].kind in {tkWhitespace, tkNewline}:
j += 1
if j < tokens.len and tokens[j].kind == tkIdentifier:
self.result.moduleInfo.functions.add(FunctionInfo(
name: tokens[j].value, kind: funcKw, position: tokens[j].position))
elif tok.value in ["struct", "type", "class", "interface", "enum", "trait", "impl"]:
var j = i + 1
while j < tokens.len and tokens[j].kind in {tkWhitespace, tkNewline}:
j += 1
if j < tokens.len and tokens[j].kind == tkIdentifier:
self.result.moduleInfo.classes.add(ClassInfo(
name: tokens[j].value, kind: tok.value, position: tokens[j].position))
i += 1
# ---------------------------------------------------------------------------
# Go Validator
# ---------------------------------------------------------------------------
type GoValidator* = ref object of ValidatorBase
proc newGoValidator*(source: string, options: ValidationOptions = newValidationOptions(), sourcePath: string = ""): GoValidator =
GoValidator(source: source, sourcePath: sourcePath, options: options,
result: newValidationResult(), startTime: epochTime())
method createTokenizer*(self: GoValidator): TokenizerBase =
newGoTokenizer(self.source, self.options)
method analyzeTokens*(self: GoValidator) =
procCall ValidatorBase(self).analyzeTokens()
extractImportsAndFunctions(self, "import", "func")
# ---------------------------------------------------------------------------
# Rust Validator
# ---------------------------------------------------------------------------
type RustValidator* = ref object of ValidatorBase
proc newRustValidator*(source: string, options: ValidationOptions = newValidationOptions(), sourcePath: string = ""): RustValidator =
RustValidator(source: source, sourcePath: sourcePath, options: options,
result: newValidationResult(), startTime: epochTime())
method createTokenizer*(self: RustValidator): TokenizerBase =
newRustTokenizer(self.source, self.options)
method analyzeTokens*(self: RustValidator) =
procCall ValidatorBase(self).analyzeTokens()
extractImportsAndFunctions(self, "use", "fn")
# ---------------------------------------------------------------------------
# Ruby Validator
# ---------------------------------------------------------------------------
type RubyValidator* = ref object of ValidatorBase
proc newRubyValidator*(source: string, options: ValidationOptions = newValidationOptions(), sourcePath: string = ""): RubyValidator =
RubyValidator(source: source, sourcePath: sourcePath, options: options,
result: newValidationResult(), startTime: epochTime())
method createTokenizer*(self: RubyValidator): TokenizerBase =
newRubyTokenizer(self.source, self.options)
method analyzeTokens*(self: RubyValidator) =
procCall ValidatorBase(self).analyzeTokens()
var blockStack: seq[string] = @[]
var lineHasContent = false
for tok in self.tokenizer.tokens:
if tok.kind == tkNewline:
lineHasContent = false
elif tok.kind notin {tkWhitespace, tkComment}:
if tok.kind == tkKeyword:
case tok.value
of "def", "class", "module", "case", "begin", "do":
blockStack.add(tok.value)
lineHasContent = true
of "if", "unless", "while", "until", "for":
# Postfix modifiers (`raise ... unless cond`) are not block openers.
if not lineHasContent:
blockStack.add(tok.value)
lineHasContent = true
of "end":
if blockStack.len > 0:
discard blockStack.pop()
else:
self.result.errors.add(newValidationError(
esError, "Unexpected 'end' with no matching block opener",
ErrUnclosedBlock, tok.position,
newSourceRange(tok.position, tok.position)))
lineHasContent = true
else:
lineHasContent = true
else:
lineHasContent = true
for b in blockStack:
self.result.errors.add(newValidationError(
esError, &"Unclosed Ruby '#{b}' block",
ErrUnclosedBlock, newSourcePosition(1, 1, 0),
newSourceRange(newSourcePosition(1, 1, 0), newSourcePosition(1, 1, 0)),
hint = "Add missing 'end'"))
var ri = 0
while ri < self.tokenizer.tokens.len:
let rtok = self.tokenizer.tokens[ri]
if rtok.kind == tkComment:
self.result.moduleInfo.comments += 1
elif rtok.kind == tkKeyword:
if rtok.value == "def":
var j = ri + 1
while j < self.tokenizer.tokens.len and self.tokenizer.tokens[j].kind in {tkWhitespace, tkNewline}:
j += 1
if j < self.tokenizer.tokens.len and self.tokenizer.tokens[j].kind == tkIdentifier:
self.result.moduleInfo.functions.add(FunctionInfo(
name: self.tokenizer.tokens[j].value, kind: "def", position: self.tokenizer.tokens[j].position))
elif rtok.value == "class":
var j = ri + 1
while j < self.tokenizer.tokens.len and self.tokenizer.tokens[j].kind in {tkWhitespace, tkNewline}:
j += 1
if j < self.tokenizer.tokens.len and self.tokenizer.tokens[j].kind == tkIdentifier:
self.result.moduleInfo.classes.add(ClassInfo(
name: self.tokenizer.tokens[j].value, kind: "class", position: self.tokenizer.tokens[j].position))
ri += 1
# ---------------------------------------------------------------------------
# CSS Validator
# ---------------------------------------------------------------------------
type CSSValidator* = ref object of ValidatorBase
proc newCSSValidator*(source: string, options: ValidationOptions = newValidationOptions(), sourcePath: string = ""): CSSValidator =
CSSValidator(source: source, sourcePath: sourcePath, options: options,
result: newValidationResult(), startTime: epochTime())
method createTokenizer*(self: CSSValidator): TokenizerBase =
newCSSTokenizer(self.source, self.options)
method analyzeTokens*(self: CSSValidator) =
procCall ValidatorBase(self).analyzeTokens()
let lines = self.source.splitLines()
for i in 0..<lines.len - 1:
let cur = lines[i].strip()
let nxt = lines[i + 1].strip()
if cur.len > 0 and cur[0] in {'.', '#'} and cur.find('{') < 0 and
nxt.len > 0 and ':' in nxt and nxt.find('{') < 0 and not nxt.startsWith("@"):
self.result.errors.add(newValidationError(
esError, &"CSS selector '{cur}' missing opening brace before properties (line {i + 1})",
ErrUnclosedBlock, newSourcePosition(i + 1, 1, 0),
newSourceRange(newSourcePosition(i + 1, 1, 0), newSourcePosition(i + 2, 1, 0)),
hint = "Add '{' after the selector"))
for tok in self.tokenizer.tokens:
if tok.kind == tkComment:
self.result.moduleInfo.comments += 1
elif tok.kind == tkDirective:
self.result.moduleInfo.functions.add(FunctionInfo(
name: tok.value, kind: "at-rule", position: tok.position))
# ---------------------------------------------------------------------------
# SQL Validator
# ---------------------------------------------------------------------------
type SQLValidator* = ref object of ValidatorBase
proc newSQLValidator*(source: string, options: ValidationOptions = newValidationOptions(), sourcePath: string = ""): SQLValidator =
SQLValidator(source: source, sourcePath: sourcePath, options: options,
result: newValidationResult(), startTime: epochTime())
method createTokenizer*(self: SQLValidator): TokenizerBase =
newSQLTokenizer(self.source, self.options)
proc hasTrailingCommaBeforeParen*(source: string): bool =
## Detect `,)` with optional whitespace/newlines between comma and ')'.
var i = 0
while i < source.len:
if source[i] == ',':
var j = i + 1
while j < source.len and source[j] in {' ', '\t', '\r', '\n'}:
j += 1
if j < source.len and source[j] == ')':
return true
inc i
proc hasIncompleteFromClause*(source: string): bool =
let upper = source.toUpperAscii()
var i = 0
while i < upper.len:
if upper[i] == 'F' and i + 4 < upper.len and upper[i..<i+4] == "FROM":
var j = i + 4
while j < upper.len and upper[j] in {' ', '\t', '\r', '\n'}:
j += 1
if j < upper.len and upper[j] == ';':
return true
inc i
method analyzeTokens*(self: SQLValidator) =
procCall ValidatorBase(self).analyzeTokens()
if hasTrailingCommaBeforeParen(self.source):
self.result.errors.add(newValidationError(
esError, "Trailing comma before closing parenthesis in SQL",
ErrInvalidSyntax, newSourcePosition(1, 1, 0),
newSourceRange(newSourcePosition(1, 1, 0), newSourcePosition(1, 1, 0)),
hint = "Remove the comma before ')'"))
elif hasIncompleteFromClause(self.source):
self.result.errors.add(newValidationError(
esError, "Incomplete FROM clause in SQL",
ErrInvalidSyntax, newSourcePosition(1, 1, 0),
newSourceRange(newSourcePosition(1, 1, 0), newSourcePosition(1, 1, 0)),
hint = "Specify a table name after FROM"))
for tok in self.tokenizer.tokens:
if tok.kind == tkComment:
self.result.moduleInfo.comments += 1
# ---------------------------------------------------------------------------
# Markdown Validator
# ---------------------------------------------------------------------------
type MarkdownValidator* = ref object of ValidatorBase
proc newMarkdownValidator*(source: string, options: ValidationOptions = newValidationOptions(), sourcePath: string = ""): MarkdownValidator =
MarkdownValidator(source: source, sourcePath: sourcePath, options: options,
result: newValidationResult(), startTime: epochTime())
method createTokenizer*(self: MarkdownValidator): TokenizerBase =
newMarkdownTokenizer(self.source, self.options)
method analyzeTokens*(self: MarkdownValidator) =
procCall ValidatorBase(self).analyzeTokens()
for tok in self.tokenizer.tokens:
if tok.kind == tkDirective:
self.result.moduleInfo.functions.add(FunctionInfo(
name: tok.value, kind: "heading", position: tok.position))
# ---------------------------------------------------------------------------
# Dockerfile Validator
# ---------------------------------------------------------------------------
type DockerfileValidator* = ref object of ValidatorBase
proc newDockerfileValidator*(source: string, options: ValidationOptions = newValidationOptions(), sourcePath: string = ""): DockerfileValidator =
DockerfileValidator(source: source, sourcePath: sourcePath, options: options,
result: newValidationResult(), startTime: epochTime())
method createTokenizer*(self: DockerfileValidator): TokenizerBase =
newDockerfileTokenizer(self.source, self.options)
method analyzeTokens*(self: DockerfileValidator) =
## Dockerfile tokenizer handles its own structural checks; skip bracket validation.
for tok in self.tokenizer.tokens:
if tok.kind == tkDirective:
self.result.moduleInfo.functions.add(FunctionInfo(
name: tok.value, kind: "instruction", position: tok.position))
elif tok.kind == tkComment:
self.result.moduleInfo.comments += 1
# ---------------------------------------------------------------------------
# Makefile Validator
# ---------------------------------------------------------------------------
type MakefileValidator* = ref object of ValidatorBase
proc newMakefileValidator*(source: string, options: ValidationOptions = newValidationOptions(), sourcePath: string = ""): MakefileValidator =
MakefileValidator(source: source, sourcePath: sourcePath, options: options,
result: newValidationResult(), startTime: epochTime())
method createTokenizer*(self: MakefileValidator): TokenizerBase =
newMakefileTokenizer(self.source, self.options)
method analyzeTokens*(self: MakefileValidator) =
for tok in self.tokenizer.tokens:
if tok.kind == tkComment:
self.result.moduleInfo.comments += 1
elif tok.kind == tkIdentifier:
self.result.moduleInfo.functions.add(FunctionInfo(
name: tok.value, kind: "target", position: tok.position))
elif tok.kind == tkAssignment:
let eqIdx = tok.value.find('=')
if eqIdx > 0:
self.result.moduleInfo.variables.add(VariableInfo(
name: tok.value[0..<eqIdx].strip(), kind: "make", position: tok.position))
# ---------------------------------------------------------------------------
# Registration
# ---------------------------------------------------------------------------
proc init*() =
registerValidator("go", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
result = newGoValidator(source, options, sourcePath))
registerValidator("rust", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
result = newRustValidator(source, options, sourcePath))
registerValidator("ruby", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
result = newRubyValidator(source, options, sourcePath))
registerValidator("css", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
result = newCSSValidator(source, options, sourcePath))
registerValidator("sql", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
result = newSQLValidator(source, options, sourcePath))
registerValidator("markdown", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
result = newMarkdownValidator(source, options, sourcePath))
registerValidator("md", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
result = newMarkdownValidator(source, options, sourcePath))
registerValidator("dockerfile", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
result = newDockerfileValidator(source, options, sourcePath))
registerValidator("makefile", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
result = newMakefileValidator(source, options, sourcePath))
init()
src/nimcheck/languages/html_validator.nim
+3 -2
View File
@@ -24,9 +24,10 @@ method createTokenizer*(self: HTMLValidator): TokenizerBase =
method analyzeTokens*(self: HTMLValidator) =
when defined(nimcheckDebug):
if self.options.debugMode: debugEnter("HTML_ANALYZE", "Analyzing HTML")
procCall ValidatorBase(self).analyzeTokens()
## Tag-stack validation is performed in the HTML tokenizer.
for tok in self.tokenizer.tokens:
if tok.kind == tkComment: self.result.moduleInfo.comments += 1
if tok.kind == tkComment:
self.result.moduleInfo.comments += 1
when defined(nimcheckDebug):
if self.options.debugMode: debugLeave("HTML_ANALYZE")
src/nimcheck/languages/lang_validators.nim
+172
View File
@@ -0,0 +1,172 @@
## Language validators: Kotlin, Lua, Swift.
##
## Each validator provides structural analysis for its language,
## following the patterns in extended_validators.nim.
import std/[json, strformat, tables]
{.warning[UnusedImport]:off.}
import ../core/types, ../core/tokenizerbase, ../core/validatorbase
import ../tokenizers/lang_tokenizers
import extended_validators
# ---------------------------------------------------------------------------
# Shared helper
# ---------------------------------------------------------------------------
proc analyzeLangTokens*(self: ValidatorBase, funcKws: seq[string], classKws: seq[string],
importKws: seq[string] = @[]) =
## Analyze tokens for structural elements common across Kotlin/Lua/Swift.
let tokens = self.tokenizer.tokens
var i = 0
while i < tokens.len:
let tok = tokens[i]
if tok.kind == tkComment:
self.result.moduleInfo.comments += 1
elif tok.kind == tkDocComment:
self.result.moduleInfo.comments += 1
elif tok.kind == tkKeyword:
if tok.value in importKws:
var j = i + 1
while j < tokens.len and tokens[j].kind in {tkWhitespace, tkNewline}:
j += 1
if j < tokens.len and tokens[j].kind in {tkIdentifier, tkString}:
self.result.moduleInfo.imports.add(ImportInfo(
module: tokens[j].value, position: tokens[j].position))
elif tok.value in funcKws:
var j = i + 1
while j < tokens.len and tokens[j].kind in {tkWhitespace, tkNewline}:
j += 1
if j < tokens.len and tokens[j].kind == tkIdentifier:
self.result.moduleInfo.functions.add(FunctionInfo(
name: tokens[j].value, kind: tok.value, position: tokens[j].position))
elif tok.value in classKws:
var j = i + 1
while j < tokens.len and tokens[j].kind in {tkWhitespace, tkNewline}:
j += 1
if j < tokens.len and tokens[j].kind == tkIdentifier:
self.result.moduleInfo.classes.add(ClassInfo(
name: tokens[j].value, kind: tok.value, position: tokens[j].position))
else:
discard
i += 1
# ---------------------------------------------------------------------------
# Kotlin Validator
# ---------------------------------------------------------------------------
type KotlinValidator* = ref object of ValidatorBase
proc newKotlinValidator*(source: string, options: ValidationOptions = newValidationOptions(), sourcePath: string = ""): KotlinValidator =
KotlinValidator(source: source, sourcePath: sourcePath, options: options,
result: newValidationResult(), startTime: epochTime())
method createTokenizer*(self: KotlinValidator): TokenizerBase =
newKotlinTokenizer(self.source, self.options)
method analyzeTokens*(self: KotlinValidator) =
procCall ValidatorBase(self).analyzeTokens()
analyzeLangTokens(self,
funcKws = @["fun"],
classKws = @["class", "interface", "object", "enum", "data", "sealed", "annotation", "abstract"],
importKws = @["import"])
# ---------------------------------------------------------------------------
# Lua Validator
# ---------------------------------------------------------------------------
type LuaValidator* = ref object of ValidatorBase
proc newLuaValidator*(source: string, options: ValidationOptions = newValidationOptions(), sourcePath: string = ""): LuaValidator =
LuaValidator(source: source, sourcePath: sourcePath, options: options,
result: newValidationResult(), startTime: epochTime())
method createTokenizer*(self: LuaValidator): TokenizerBase =
newLuaTokenizer(self.source, self.options)
method analyzeTokens*(self: LuaValidator) =
procCall ValidatorBase(self).analyzeTokens()
let tokens = self.tokenizer.tokens
var i = 0
while i < tokens.len:
let tok = tokens[i]
if tok.kind == tkComment:
self.result.moduleInfo.comments += 1
elif tok.kind == tkKeyword:
if tok.value == "function":
var j = i + 1
while j < tokens.len and tokens[j].kind in {tkWhitespace, tkNewline}:
j += 1
if j < tokens.len and tokens[j].kind == tkIdentifier:
var funcName = tokens[j].value
# Lua allows function module.funcname syntax
var k = j + 1
while k < tokens.len and tokens[k].kind in {tkWhitespace, tkNewline}:
k += 1
if k < tokens.len and tokens[k].kind == tkPunctuation and k + 1 < tokens.len and tokens[k+1].kind == tkIdentifier:
funcName = funcName & "." & tokens[k+1].value
self.result.moduleInfo.functions.add(FunctionInfo(
name: funcName, kind: "function", position: tokens[j].position))
else:
discard
i += 1
var blockStack: seq[string] = @[]
for tok in tokens:
if tok.kind == tkKeyword:
case tok.value
of "function", "if", "for", "while", "repeat":
blockStack.add(tok.value)
of "do":
# `for`/`while` loops use `do` as syntax, not a nested block opener.
if blockStack.len == 0 or blockStack[^1] notin ["for", "while"]:
blockStack.add(tok.value)
of "end":
if blockStack.len > 0 and blockStack[^1] != "repeat":
discard blockStack.pop()
of "until":
if blockStack.len > 0 and blockStack[^1] == "repeat":
discard blockStack.pop()
for b in blockStack:
let closer = if b == "repeat": "until" else: "end"
self.result.errors.add(newValidationError(
esError, &"Unclosed Lua '#{b}' block",
ErrUnclosedBlock, newSourcePosition(1, 1, 0),
newSourceRange(newSourcePosition(1, 1, 0), newSourcePosition(1, 1, 0)),
hint = &"Add missing '#{closer}'"))
# ---------------------------------------------------------------------------
# Swift Validator
# ---------------------------------------------------------------------------
type SwiftValidator* = ref object of ValidatorBase
proc newSwiftValidator*(source: string, options: ValidationOptions = newValidationOptions(), sourcePath: string = ""): SwiftValidator =
SwiftValidator(source: source, sourcePath: sourcePath, options: options,
result: newValidationResult(), startTime: epochTime())
method createTokenizer*(self: SwiftValidator): TokenizerBase =
newSwiftTokenizer(self.source, self.options)
method analyzeTokens*(self: SwiftValidator) =
procCall ValidatorBase(self).analyzeTokens()
analyzeLangTokens(self,
funcKws = @["func"],
classKws = @["class", "struct", "enum", "protocol", "extension", "actor"],
importKws = @["import"])
# ---------------------------------------------------------------------------
# Registration
# ---------------------------------------------------------------------------
proc init*() =
registerValidator("kotlin", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
result = newKotlinValidator(source, options, sourcePath))
registerValidator("kt", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
result = newKotlinValidator(source, options, sourcePath))
registerValidator("lua", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
result = newLuaValidator(source, options, sourcePath))
registerValidator("swift", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
result = newSwiftValidator(source, options, sourcePath))
init()
src/nimcheck/languages/type_xml_validator.nim
+108
View File
@@ -0,0 +1,108 @@
## TypeScript and XML validators for Nimcheck.
##
## Provides improved TypeScript validation and full XML structure validation.
import std/[json, strformat, tables]
{.warning[UnusedImport]:off.}
import ../core/types, ../core/tokenizerbase, ../core/validatorbase
import ../tokenizers/type_xml_tokenizer
# ---------------------------------------------------------------------------
# TypeScript Validator
# ---------------------------------------------------------------------------
type TypeScriptValidator* = ref object of ValidatorBase
proc newTypeScriptValidator*(source: string, options: ValidationOptions = newValidationOptions(), sourcePath: string = ""): TypeScriptValidator =
TypeScriptValidator(source: source, sourcePath: sourcePath, options: options,
result: newValidationResult(), startTime: epochTime())
method createTokenizer*(self: TypeScriptValidator): TokenizerBase =
newTypeScriptTokenizer(self.source, self.options)
method analyzeTokens*(self: TypeScriptValidator) =
procCall ValidatorBase(self).analyzeTokens()
let tokens = self.tokenizer.tokens
var i = 0
while i < tokens.len:
let tok = tokens[i]
case tok.kind
of tkComment, tkDocComment:
self.result.moduleInfo.comments += 1
of tkKeyword:
if tok.value in ["import", "export"]:
var j = i + 1
while j < tokens.len and tokens[j].kind in {tkWhitespace, tkNewline}:
j += 1
if j < tokens.len and tokens[j].kind in {tkKeyword, tkIdentifier, tkPunctuation}:
var importInfo = ImportInfo(module: "", position: tokens[j].position)
self.result.moduleInfo.imports.add(importInfo)
elif tok.value == "function":
var j = i + 1
while j < tokens.len and tokens[j].kind in {tkWhitespace, tkNewline}:
j += 1
if j < tokens.len and tokens[j].kind == tkIdentifier:
self.result.moduleInfo.functions.add(FunctionInfo(
name: tokens[j].value, kind: "function", position: tokens[j].position))
elif tok.value in ["class", "interface", "enum", "type"]:
var j = i + 1
while j < tokens.len and tokens[j].kind in {tkWhitespace, tkNewline}:
j += 1
if j < tokens.len and tokens[j].kind == tkIdentifier:
self.result.moduleInfo.classes.add(ClassInfo(
name: tokens[j].value, kind: tok.value, position: tokens[j].position))
of tkDirective:
if tok.value.startsWith("@"):
# Decorator
self.result.moduleInfo.functions.add(FunctionInfo(
name: tok.value, kind: "decorator", position: tok.position))
else:
discard
i += 1
# ---------------------------------------------------------------------------
# XML Validator
# ---------------------------------------------------------------------------
type XMLValidator* = ref object of ValidatorBase
proc newXMLValidator*(source: string, options: ValidationOptions = newValidationOptions(), sourcePath: string = ""): XMLValidator =
XMLValidator(source: source, sourcePath: sourcePath, options: options,
result: newValidationResult(), startTime: epochTime())
method createTokenizer*(self: XMLValidator): TokenizerBase =
newXMLTokenizer(self.source, self.options)
method analyzeTokens*(self: XMLValidator) =
## XML tokenizer handles its own structural checks (tag matching);
## skip bracket validation and just collect structural info.
for tok in self.tokenizer.tokens:
if tok.kind == tkComment:
self.result.moduleInfo.comments += 1
elif tok.kind == tkDirective:
self.result.moduleInfo.functions.add(FunctionInfo(
name: tok.value, kind: "directive", position: tok.position))
elif tok.kind == tkKeyword:
self.result.moduleInfo.classes.add(ClassInfo(
name: tok.value, kind: "tag", position: tok.position))
# ---------------------------------------------------------------------------
# Registration
# ---------------------------------------------------------------------------
proc init*() =
registerValidator("typescript", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
result = newTypeScriptValidator(source, options, sourcePath))
registerValidator("ts", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
result = newTypeScriptValidator(source, options, sourcePath))
registerValidator("xml", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
result = newXMLValidator(source, options, sourcePath))
registerValidator("xsd", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
result = newXMLValidator(source, options, sourcePath))
registerValidator("xslt", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
result = newXMLValidator(source, options, sourcePath))
registerValidator("svg", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
result = newXMLValidator(source, options, sourcePath))
init()
src/nimcheck/tokenizers/bash_tokenizer.nim
+1 -1
View File
@@ -245,7 +245,7 @@ method tokenize*(self: BashTokenizer) =
else:
self.emitBracketToken(kOpenBracket, $self.advance(), startPos)
elif c == ']':
if self.pos > 1 and self.source[self.pos-1] == '[': # simplified
if self.peekString(2) == "]]":
discard self.advance()
discard self.advance()
self.emitToken(tkKeyword, "]]", startPos)
src/nimcheck/tokenizers/cfamily_tokenizers.nim
+13
View File
@@ -0,0 +1,13 @@
## C-family language tokenizers: C, C++, Java, C#.
##
## Each tokenizer uses the CLikeTokenizer from extended_tokenizers
## with its own dialect and keyword set.
##
## The tokenizer constructors, keyword sets and CLikeDialect values
## are defined in extended_tokenizers.nim and re-exported here for
## convenient access from the C-family validators.
{.warning[UnusedImport]:off.}
import ../core/types, ../core/tokenizerbase
import extended_tokenizers
export extended_tokenizers
src/nimcheck/tokenizers/extended_tokenizers.nim
+750
View File
@@ -0,0 +1,750 @@
## Extended language tokenizers: Go, Rust, Ruby, CSS, SQL, Markdown, Dockerfile, Makefile.
import std/[strformat, sets, sequtils]
{.warning[UnusedImport]:off.}
import ../core/types, ../core/tokenizerbase
# ---------------------------------------------------------------------------
# Shared C-like tokenization helpers
# ---------------------------------------------------------------------------
type
CLikeDialect* = enum
cdGo, cdRust, cdCss, cdC, cdCpp, cdJava, cdCSharp
CLikeTokenizer* = ref object of TokenizerBase
dialect*: CLikeDialect
keywords*: HashSet[string]
proc newCLikeTokenizer*(
source: string,
dialect: CLikeDialect,
keywords: seq[string],
options: ValidationOptions = newValidationOptions()
): CLikeTokenizer =
result = CLikeTokenizer(
source: source,
sourceLen: source.len,
options: options,
tokens: @[],
pos: 0,
line: 1,
column: 1,
bracketStack: @[],
errors: @[],
eof: false,
dialect: dialect,
keywords: keywords.toHashSet()
)
proc parseRustRawString*(self: CLikeTokenizer, startPos: SourcePosition): string =
result = "r"
discard self.advance()
var hashCount = 0
while self.hasMore() and self.peek() == '#':
hashCount += 1
result.add(self.advance())
if not self.hasMore() or self.peek() != '"':
self.recordError(esError, "Invalid Rust raw string literal", ErrUnclosedString, startPos)
return
result.add(self.advance())
let closeDelim = repeat("#", hashCount) & "\""
while self.hasMore():
if self.peekString(closeDelim.len) == closeDelim:
for _ in 0..<closeDelim.len:
result.add(self.advance())
return
result.add(self.advance())
self.recordError(esError, "Unclosed Rust raw string literal", ErrUnclosedString, startPos,
hint = &"Add closing '{closeDelim}'")
proc parseQuotedString*(
self: TokenizerBase,
quote: char,
startPos: SourcePosition,
allowNewline: bool = false
): string =
result = $quote
while self.hasMore():
let c = self.advance()
if c == '\\':
result.add(c)
if self.hasMore():
result.add(self.advance())
elif c == quote:
result.add(quote)
return
elif c == '\n' and not allowNewline:
self.recordError(esError, &"Unclosed string literal starting with '{quote}'",
ErrUnclosedString, startPos, hint = "Add a closing quote")
return
else:
result.add(c)
self.recordError(esError, &"Unclosed string literal starting with '{quote}'",
ErrUnclosedString, startPos, hint = "Add a closing quote before end of file")
proc parseBacktickString*(self: TokenizerBase, startPos: SourcePosition): string =
result = "`"
while self.hasMore():
let c = self.advance()
if c == '`':
result.add('`')
return
result.add(c)
self.recordError(esError, "Unclosed raw string literal", ErrUnclosedString, startPos,
hint = "Add a closing backtick")
method tokenize*(self: CLikeTokenizer) =
when defined(nimcheckDebug):
if self.options.debugMode:
debugEnter("CLIKE_TOKENIZER", &"Starting {self.dialect} tokenization")
while self.hasMore():
let posBefore = self.pos
let startPos = self.currentPos()
let c = self.peek()
if c in {' ', '\t', '\r'}:
let wsStart = self.currentPos()
var ws = ""
while self.hasMore() and self.peek() in {' ', '\t', '\r'}:
ws.add(self.advance())
self.emitToken(tkWhitespace, ws, wsStart)
elif c == '\n':
self.emitToken(tkNewline, $self.advance(), startPos)
elif self.dialect != cdCss and self.peekString(2) == "//":
let comStart = self.currentPos()
discard self.advance()
discard self.advance()
self.emitToken(tkComment, "//" & self.parseLineComment(), comStart)
elif self.peekString(2) == "/*":
let comStart = self.currentPos()
discard self.advance()
discard self.advance()
let content = self.parseBlockComment("/*", "*/")
self.emitToken(tkComment, "/*" & content & "*/", comStart)
elif self.dialect == cdRust and (c == 'r' or c == 'R') and self.peek(1) in {'"', '#'}:
let rawStart = self.currentPos()
let content = self.parseRustRawString(rawStart)
self.emitToken(tkRawString, content, rawStart)
elif self.dialect == cdGo and c == '`':
let rawStart = self.currentPos()
let content = self.parseBacktickString(rawStart)
self.emitToken(tkRawString, content, rawStart)
elif c == '\'' and self.dialect == cdRust:
let chStart = self.currentPos()
discard self.advance()
if self.hasMore() and (self.peek() == '_' or isAlpha(self.peek())):
var lit = "'"
while self.hasMore() and (isAlphaNum(self.peek()) or self.peek() == '_'):
lit.add(self.advance())
self.emitToken(tkSpecial, lit, chStart)
else:
var content = "'"
var closed = false
while self.hasMore():
let cc = self.advance()
if cc == '\\' and self.hasMore():
content.add(cc)
content.add(self.advance())
elif cc == '\'':
content.add('\'')
closed = true
break
else:
content.add(cc)
if not closed:
self.recordError(esError, "Unclosed Rust character literal", ErrUnclosedString, chStart)
self.emitToken(tkString, content, chStart)
elif c == '"' or c == '\'':
let strStart = self.currentPos()
discard self.advance()
let content = $c & self.parseQuotedString(c, strStart)
self.emitToken(tkString, content, strStart)
elif isDigit(c) or (c == '.' and self.hasMore() and self.peek(1).isDigit()):
let numStart = self.currentPos()
var num = ""
while self.hasMore() and (self.peek().isDigit() or self.peek() in {'.', 'x', 'X', 'e', 'E', '+', '-', '_'}):
num.add(self.advance())
self.emitToken(tkNumber, num, numStart)
elif isAlpha(c) or c == '_':
let idStart = self.currentPos()
var ident = ""
while self.hasMore() and (isAlphaNum(self.peek()) or self.peek() in {'_', '$'}):
ident.add(self.advance())
if self.keywords.contains(ident):
self.emitToken(tkKeyword, ident, idStart)
else:
self.emitToken(tkIdentifier, ident, idStart)
elif c == '(':
self.emitBracketToken(kOpenParen, $self.advance(), startPos)
elif c == ')':
self.closeBracket(')', kCloseParen, startPos)
elif c == '[':
self.emitBracketToken(kOpenBracket, $self.advance(), startPos)
elif c == ']':
self.closeBracket(']', kCloseBracket, startPos)
elif c == '{':
self.emitBracketToken(kOpenBrace, $self.advance(), startPos)
elif c == '}':
self.closeBracket('}', kCloseBrace, startPos)
elif c in {'+', '-', '*', '/', '%', '&', '|', '^', '~', '!', '?', ':', '=', '<', '>'}:
let opStart = self.currentPos()
var op = ""
while self.hasMore() and self.peek() in {'+', '-', '*', '/', '%', '&', '|', '^', '~', '!', '?', ':', '=', '<', '>'}:
op.add(self.advance())
if op in ["=", ":=", "+=", "-=", "*=", "/="]:
self.emitToken(tkAssignment, op, opStart)
else:
self.emitToken(tkOperator, op, opStart)
elif c in {'.', ',', ';'}:
self.emitToken(tkPunctuation, $self.advance(), startPos)
elif c == '@' and self.dialect == cdCss:
let atStart = self.currentPos()
var atRule = "@"
discard self.advance()
while self.hasMore() and (isAlphaNum(self.peek()) or self.peek() == '-'):
atRule.add(self.advance())
self.emitToken(tkDirective, atRule, atStart)
else:
self.emitToken(tkSpecial, $self.advance(), startPos)
self.finishTokenizeStep(posBefore)
self.emitToken(tkEndOfFile, "", self.currentPos())
when defined(nimcheckDebug):
if self.options.debugMode:
debugLeave("CLIKE_TOKENIZER", &"Complete: {self.tokens.len} tokens")
const
goKeywords* = @[
"break", "case", "chan", "const", "continue", "default", "defer", "else",
"fallthrough", "for", "func", "go", "goto", "if", "import", "interface",
"map", "package", "range", "return", "select", "struct", "switch", "type", "var"
]
rustKeywords* = @[
"as", "async", "await", "break", "const", "continue", "crate", "dyn", "else",
"enum", "extern", "false", "fn", "for", "if", "impl", "in", "let", "loop",
"match", "mod", "move", "mut", "pub", "ref", "return", "self", "Self", "static",
"struct", "super", "trait", "true", "type", "unsafe", "use", "where", "while"
]
cssAtRules* = @["media", "import", "keyframes", "font-face", "supports", "layer"]
cKeywords* = @[
"auto", "break", "case", "char", "const", "continue", "default", "do",
"double", "else", "enum", "extern", "float", "for", "goto", "if",
"inline", "int", "long", "register", "restrict", "return", "short",
"signed", "sizeof", "static", "struct", "switch", "typedef", "union",
"unsigned", "void", "volatile", "while",
"_Bool", "_Complex", "_Imaginary", "_Alignas", "_Alignof", "_Atomic",
"_Generic", "_Noreturn", "_Static_assert", "_Thread_local", "__attribute__"
]
cppKeywords* = @[
"alignas", "alignof", "and", "and_eq", "asm", "auto", "bitand", "bitor",
"bool", "break", "case", "catch", "char", "char8_t", "char16_t", "char32_t",
"class", "compl", "concept", "const", "const_cast", "consteval", "constexpr",
"constinit", "continue", "co_await", "co_return", "co_yield", "decltype",
"default", "delete", "do", "double", "dynamic_cast", "else", "enum",
"explicit", "export", "extern", "false", "float", "for", "friend", "goto",
"if", "inline", "int", "long", "mutable", "namespace", "new", "noexcept",
"not", "not_eq", "nullptr", "operator", "or", "or_eq", "override",
"private", "protected", "public", "reflexpr", "register", "reinterpret_cast",
"requires", "return", "short", "signed", "sizeof", "static", "static_assert",
"static_cast", "struct", "switch", "template", "this", "throw", "true",
"try", "typedef", "typeid", "typename", "union", "unsigned", "using",
"virtual", "void", "volatile", "wchar_t", "while", "xor", "xor_eq"
]
javaKeywords* = @[
"abstract", "assert", "boolean", "break", "byte", "case", "catch", "char",
"class", "const", "continue", "default", "do", "double", "else", "enum",
"extends", "false", "final", "finally", "float", "for", "goto", "if",
"implements", "import", "instanceof", "int", "interface", "long", "native",
"new", "null", "package", "private", "protected", "public", "return",
"short", "static", "strictfp", "super", "switch", "synchronized", "this",
"throw", "throws", "transient", "true", "try", "var", "void", "volatile",
"while", "yield", "module", "requires", "exports", "opens", "to", "with",
"provides", "uses", "open", "transitive", "record", "sealed", "permits",
"non-sealed"
]
csharpKeywords* = @[
"abstract", "as", "base", "bool", "break", "byte", "case", "catch", "char",
"checked", "class", "const", "continue", "decimal", "default", "delegate",
"do", "double", "else", "enum", "event", "explicit", "extern", "false",
"finally", "fixed", "float", "for", "foreach", "goto", "if", "implicit",
"in", "int", "interface", "internal", "is", "lock", "long", "namespace",
"new", "null", "object", "operator", "out", "override", "params", "private",
"protected", "public", "readonly", "ref", "return", "sbyte", "sealed",
"short", "sizeof", "stackalloc", "static", "string", "struct", "switch",
"this", "throw", "true", "try", "typeof", "uint", "ulong", "unchecked",
"unsafe", "ushort", "using", "value", "var", "virtual", "void", "volatile",
"while", "async", "await", "dynamic", "from", "get", "global", "group",
"init", "join", "let", "managed", "nameof", "nint", "notnull", "nuint",
"on", "orderby", "partial", "record", "remove", "select", "set",
"unmanaged", "when", "where", "yield", "file"
]
proc newGoTokenizer*(source: string, options: ValidationOptions = newValidationOptions()): CLikeTokenizer =
newCLikeTokenizer(source, cdGo, goKeywords, options)
proc newRustTokenizer*(source: string, options: ValidationOptions = newValidationOptions()): CLikeTokenizer =
newCLikeTokenizer(source, cdRust, rustKeywords, options)
proc newCSSTokenizer*(source: string, options: ValidationOptions = newValidationOptions()): CLikeTokenizer =
newCLikeTokenizer(source, cdCss, @[], options)
proc newCTokenizer*(source: string, options: ValidationOptions = newValidationOptions()): CLikeTokenizer =
newCLikeTokenizer(source, cdC, cKeywords, options)
proc newCppTokenizer*(source: string, options: ValidationOptions = newValidationOptions()): CLikeTokenizer =
newCLikeTokenizer(source, cdCpp, cppKeywords, options)
proc newJavaTokenizer*(source: string, options: ValidationOptions = newValidationOptions()): CLikeTokenizer =
newCLikeTokenizer(source, cdJava, javaKeywords, options)
proc newCSharpTokenizer*(source: string, options: ValidationOptions = newValidationOptions()): CLikeTokenizer =
newCLikeTokenizer(source, cdCSharp, csharpKeywords, options)
# ---------------------------------------------------------------------------
# Ruby tokenizer
# ---------------------------------------------------------------------------
type
RubyTokenizer* = ref object of TokenizerBase
const
rubyKeywords* = @[
"BEGIN", "END", "alias", "and", "begin", "break", "case", "class", "def",
"defined?", "do", "else", "elsif", "end", "ensure", "false", "for", "if",
"in", "module", "next", "nil", "not", "or", "redo", "rescue", "retry",
"return", "self", "super", "then", "true", "undef", "unless", "until",
"when", "while", "yield"
]
rubyKeywordSet* = rubyKeywords.toHashSet()
proc newRubyTokenizer*(source: string, options: ValidationOptions = newValidationOptions()): RubyTokenizer =
RubyTokenizer(
source: source, sourceLen: source.len, options: options,
tokens: @[], pos: 0, line: 1, column: 1, bracketStack: @[], errors: @[], eof: false
)
method tokenize*(self: RubyTokenizer) =
when defined(nimcheckDebug):
if self.options.debugMode: debugEnter("RUBY_TOKENIZER", "Starting Ruby tokenization")
while self.hasMore():
let posBefore = self.pos
let startPos = self.currentPos()
let c = self.peek()
if c in {' ', '\t', '\r'}:
let wsStart = self.currentPos()
var ws = ""
while self.hasMore() and self.peek() in {' ', '\t', '\r'}:
ws.add(self.advance())
self.emitToken(tkWhitespace, ws, wsStart)
elif c == '\n':
self.emitToken(tkNewline, $self.advance(), startPos)
elif c == '#':
let comStart = self.currentPos()
discard self.advance()
self.emitToken(tkComment, "#" & self.parseLineComment(), comStart)
elif c == '"' or c == '\'':
let strStart = self.currentPos()
discard self.advance()
let content = $c & self.parseQuotedString(c, strStart)
self.emitToken(tkString, content, strStart)
elif isDigit(c):
let numStart = self.currentPos()
var num = ""
while self.hasMore() and (self.peek().isDigit() or self.peek() in {'.', 'x', 'e', 'E'}):
num.add(self.advance())
self.emitToken(tkNumber, num, numStart)
elif isAlpha(c) or c == '_':
let idStart = self.currentPos()
var ident = ""
while self.hasMore() and (isAlphaNum(self.peek()) or self.peek() in {'_', '?', '!'}):
ident.add(self.advance())
if rubyKeywordSet.contains(ident):
self.emitToken(tkKeyword, ident, idStart)
else:
self.emitToken(tkIdentifier, ident, idStart)
elif c == '(':
self.emitBracketToken(kOpenParen, $self.advance(), startPos)
elif c == ')':
self.closeBracket(')', kCloseParen, startPos)
elif c == '[':
self.emitBracketToken(kOpenBracket, $self.advance(), startPos)
elif c == ']':
self.closeBracket(']', kCloseBracket, startPos)
elif c == '{':
self.emitBracketToken(kOpenBrace, $self.advance(), startPos)
elif c == '}':
self.closeBracket('}', kCloseBrace, startPos)
elif c in {'+', '-', '*', '/', '%', '&', '|', '^', '~', '!', '?', ':', '=', '<', '>'}:
let opStart = self.currentPos()
var op = ""
while self.hasMore() and self.peek() in {'+', '-', '*', '/', '%', '&', '|', '^', '~', '!', '?', ':', '=', '<', '>'}:
op.add(self.advance())
if op == "=":
self.emitToken(tkAssignment, op, opStart)
else:
self.emitToken(tkOperator, op, opStart)
elif c in {'.', ',', ';'}:
self.emitToken(tkPunctuation, $self.advance(), startPos)
else:
self.emitToken(tkSpecial, $self.advance(), startPos)
self.finishTokenizeStep(posBefore)
self.emitToken(tkEndOfFile, "", self.currentPos())
when defined(nimcheckDebug):
if self.options.debugMode: debugLeave("RUBY_TOKENIZER", &"Complete: {self.tokens.len} tokens")
# ---------------------------------------------------------------------------
# SQL tokenizer
# ---------------------------------------------------------------------------
type
SQLTokenizer* = ref object of TokenizerBase
proc newSQLTokenizer*(source: string, options: ValidationOptions = newValidationOptions()): SQLTokenizer =
SQLTokenizer(
source: source, sourceLen: source.len, options: options,
tokens: @[], pos: 0, line: 1, column: 1, bracketStack: @[], errors: @[], eof: false
)
method tokenize*(self: SQLTokenizer) =
when defined(nimcheckDebug):
if self.options.debugMode: debugEnter("SQL_TOKENIZER", "Starting SQL tokenization")
while self.hasMore():
let posBefore = self.pos
let startPos = self.currentPos()
let c = self.peek()
if c in {' ', '\t', '\r'}:
let wsStart = self.currentPos()
var ws = ""
while self.hasMore() and self.peek() in {' ', '\t', '\r'}:
ws.add(self.advance())
self.emitToken(tkWhitespace, ws, wsStart)
elif c == '\n':
self.emitToken(tkNewline, $self.advance(), startPos)
elif self.peekString(2) == "--":
let comStart = self.currentPos()
discard self.advance()
discard self.advance()
self.emitToken(tkComment, "--" & self.parseLineComment(), comStart)
elif self.peekString(2) == "/*":
let comStart = self.currentPos()
discard self.advance()
discard self.advance()
let content = self.parseBlockComment("/*", "*/")
self.emitToken(tkComment, "/*" & content & "*/", comStart)
elif c == '\'':
let strStart = self.currentPos()
discard self.advance()
var content = "'"
var closed = false
while self.hasMore():
let sc = self.advance()
if sc == '\'' and self.hasMore() and self.peek() == '\'':
content.add(sc)
content.add(self.advance())
elif sc == '\'':
content.add('\'')
self.emitToken(tkString, content, strStart)
closed = true
break
else:
content.add(sc)
if not closed:
self.recordError(esError, "Unclosed SQL string literal", ErrUnclosedString, strStart)
self.emitToken(tkString, content, strStart)
elif c == '"':
let strStart = self.currentPos()
discard self.advance()
let content = "\"" & self.parseQuotedString('"', strStart)
self.emitToken(tkString, content, strStart)
elif isDigit(c) or (c == '.' and self.hasMore() and self.peek(1).isDigit()):
let numStart = self.currentPos()
var num = ""
while self.hasMore() and (self.peek().isDigit() or self.peek() in {'.', 'e', 'E', '+', '-'}):
num.add(self.advance())
self.emitToken(tkNumber, num, numStart)
elif isAlpha(c) or c == '_':
let idStart = self.currentPos()
var ident = ""
while self.hasMore() and (isAlphaNum(self.peek()) or self.peek() in {'_', '$'}):
ident.add(self.advance())
self.emitToken(tkIdentifier, ident, idStart)
elif c == '(':
self.emitBracketToken(kOpenParen, $self.advance(), startPos)
elif c == ')':
self.closeBracket(')', kCloseParen, startPos)
elif c == ';':
self.emitToken(tkPunctuation, $self.advance(), startPos)
elif c in {'+', '-', '*', '/', '%', '=', '<', '>', '!', '|', '&', ','}:
self.emitToken(tkOperator, $self.advance(), startPos)
else:
self.emitToken(tkSpecial, $self.advance(), startPos)
self.finishTokenizeStep(posBefore)
self.emitToken(tkEndOfFile, "", self.currentPos())
when defined(nimcheckDebug):
if self.options.debugMode: debugLeave("SQL_TOKENIZER", &"Complete: {self.tokens.len} tokens")
# ---------------------------------------------------------------------------
# Markdown tokenizer
# ---------------------------------------------------------------------------
type
MarkdownTokenizer* = ref object of TokenizerBase
fenceStack*: seq[string]
proc newMarkdownTokenizer*(source: string, options: ValidationOptions = newValidationOptions()): MarkdownTokenizer =
MarkdownTokenizer(
source: source, sourceLen: source.len, options: options,
tokens: @[], pos: 0, line: 1, column: 1, bracketStack: @[], errors: @[], eof: false,
fenceStack: @[]
)
method tokenize*(self: MarkdownTokenizer) =
when defined(nimcheckDebug):
if self.options.debugMode: debugEnter("MD_TOKENIZER", "Starting Markdown tokenization")
var lineNum = 1
for line in self.source.splitLines(keepEol = true):
let lineStart = self.currentPos()
let stripped = line.strip()
if stripped.startsWith("```"):
let fence = stripped
if self.fenceStack.len > 0 and self.fenceStack[^1] == "```":
discard self.fenceStack.pop()
self.emitToken(tkKeyword, fence, lineStart)
else:
self.fenceStack.add("```")
self.emitToken(tkKeyword, fence, lineStart)
elif stripped.startsWith("~~~"):
let fence = stripped
if self.fenceStack.len > 0 and self.fenceStack[^1] == "~~~":
discard self.fenceStack.pop()
self.emitToken(tkKeyword, fence, lineStart)
else:
self.fenceStack.add("~~~")
self.emitToken(tkKeyword, fence, lineStart)
elif stripped.len > 0 and stripped[0] == '#':
self.emitToken(tkDirective, stripped, lineStart)
elif '[' in line and "](" in line:
var idx = 0
while idx < line.len:
if line[idx] == '[':
let linkStart = idx
let closeBracket = line.find(']', idx + 1)
if closeBracket >= 0 and closeBracket + 1 < line.len and line[closeBracket + 1] == '(':
let closeParen = line.find(')', closeBracket + 2)
if closeParen < 0:
self.recordError(esError, &"Unclosed Markdown link on line {lineNum}",
ErrUnclosedBlock, newSourcePosition(lineNum, linkStart + 1, 0),
hint = "Add a closing ')' to complete the link")
idx = if closeParen >= 0: closeParen + 1 else: line.len
else:
idx += 1
else:
idx += 1
self.emitToken(tkTemplateTag, line, lineStart)
else:
var backtickCount = 0
for c in line:
if c == '`': backtickCount += 1
if backtickCount mod 2 != 0:
self.recordError(esWarning, &"Unbalanced inline code backticks on line {lineNum}",
WarnInconsistentNaming, newSourcePosition(lineNum, 1, 0))
self.emitToken(tkTemplateTag, line, lineStart)
self.pos += line.len
self.line += 1
self.column = 1
lineNum += 1
if self.fenceStack.len > 0:
self.recordError(esError, "Unclosed fenced code block",
ErrUnclosedBlock, newSourcePosition(lineNum, 1, 0),
hint = "Add a closing ``` or ~~~ fence")
self.emitToken(tkEndOfFile, "", self.currentPos())
when defined(nimcheckDebug):
if self.options.debugMode: debugLeave("MD_TOKENIZER", &"Complete: {self.tokens.len} tokens")
# ---------------------------------------------------------------------------
# Dockerfile tokenizer
# ---------------------------------------------------------------------------
type
DockerfileTokenizer* = ref object of TokenizerBase
const
dockerInstructions* = @[
"FROM", "RUN", "CMD", "LABEL", "EXPOSE", "ENV", "ADD", "COPY",
"ENTRYPOINT", "VOLUME", "USER", "WORKDIR", "ARG", "ONBUILD",
"STOPSIGNAL", "HEALTHCHECK", "SHELL"
].toHashSet()
proc newDockerfileTokenizer*(source: string, options: ValidationOptions = newValidationOptions()): DockerfileTokenizer =
DockerfileTokenizer(
source: source, sourceLen: source.len, options: options,
tokens: @[], pos: 0, line: 1, column: 1, bracketStack: @[], errors: @[], eof: false
)
method tokenize*(self: DockerfileTokenizer) =
when defined(nimcheckDebug):
if self.options.debugMode: debugEnter("DOCKER_TOKENIZER", "Starting Dockerfile tokenization")
var lineNum = 1
var lines = self.source.splitLines()
var i = 0
while i < lines.len:
var line = lines[i]
let lineStart = newSourcePosition(lineNum, 1, 0)
while line.len > 0 and line[^1] == '\\' and i + 1 < lines.len:
line = line[0..^2] & " " & lines[i + 1].strip()
i += 1
lineNum += 1
let stripped = line.strip()
if stripped.len == 0 or stripped[0] == '#':
if stripped.len > 0:
self.emitToken(tkComment, stripped, lineStart)
i += 1
lineNum += 1
continue
let spaceIdx = stripped.find(' ')
let instruction = if spaceIdx >= 0: stripped[0..<spaceIdx].toUpperAscii() else: stripped.toUpperAscii()
if dockerInstructions.contains(instruction):
self.emitToken(tkDirective, instruction, lineStart)
let args = if spaceIdx >= 0: stripped[spaceIdx + 1 .. ^1] else: ""
if args.len > 0:
self.emitToken(tkString, args, lineStart)
for ch in args:
if ch == '[':
self.bracketStack.add(('[', lineStart))
elif ch == ']':
if self.bracketStack.len > 0 and self.bracketStack[^1][0] == '[':
discard self.bracketStack.pop()
else:
self.recordError(esError, &"Unexpected ']' in Dockerfile instruction '{instruction}'",
ErrMismatchedBracket, lineStart)
elif ch == '"':
discard
let openQuotes = args.count('"') - args.count("\\\"")
if openQuotes mod 2 != 0:
self.recordError(esError, &"Unclosed double-quoted string in {instruction}",
ErrUnclosedString, lineStart, hint = "Add a closing '\"'")
else:
self.recordError(esWarning, &"Unknown Dockerfile instruction: {instruction}",
WarnInconsistentNaming, lineStart)
i += 1
lineNum += 1
for (_, pos) in self.bracketStack:
self.recordError(esError, "Unclosed '[' in Dockerfile JSON array",
ErrUnclosedBracket, pos, hint = "Add a closing ']'")
self.pos = self.sourceLen
self.line = lineNum
self.emitToken(tkEndOfFile, "", self.currentPos())
when defined(nimcheckDebug):
if self.options.debugMode: debugLeave("DOCKER_TOKENIZER", &"Complete: {self.tokens.len} tokens")
# ---------------------------------------------------------------------------
# Makefile tokenizer
# ---------------------------------------------------------------------------
type
MakefileTokenizer* = ref object of TokenizerBase
proc newMakefileTokenizer*(source: string, options: ValidationOptions = newValidationOptions()): MakefileTokenizer =
MakefileTokenizer(
source: source, sourceLen: source.len, options: options,
tokens: @[], pos: 0, line: 1, column: 1, bracketStack: @[], errors: @[], eof: false
)
method tokenize*(self: MakefileTokenizer) =
when defined(nimcheckDebug):
if self.options.debugMode: debugEnter("MAKEFILE_TOKENIZER", "Starting Makefile tokenization")
var lineNum = 1
for line in self.source.splitLines():
let lineStart = newSourcePosition(lineNum, 1, 0)
if line.len == 0:
self.emitToken(tkNewline, "", lineStart)
elif line[0] == '#':
self.emitToken(tkComment, line, lineStart)
elif line[0] == '\t' or (line.len >= 2 and line[0] == '\\' and line[1] == 't'):
let recipe = if line[0] == '\t': line else: '\t' & line[2..^1]
self.emitToken(tkSpecial, recipe, lineStart)
var parenDepth = 0
var inQuote = false
for i, c in recipe:
if c == '"' and (i == 0 or recipe[i - 1] != '\\'):
inQuote = not inQuote
elif c == '(' and not inQuote:
parenDepth += 1
self.bracketStack.add(('(', newSourcePosition(lineNum, i + 1, 0)))
elif c == ')' and not inQuote:
if parenDepth > 0 and self.bracketStack.len > 0:
parenDepth -= 1
discard self.bracketStack.pop()
else:
self.recordError(esError, &"Unexpected ')' in Makefile recipe on line {lineNum}",
ErrMismatchedBracket, newSourcePosition(lineNum, i + 1, 0))
if inQuote:
self.recordError(esError, &"Unclosed double-quoted string in Makefile recipe on line {lineNum}",
ErrUnclosedString, lineStart, hint = "Add a closing '\"'")
elif ':' in line and not line.strip().startsWith("#"):
let colonIdx = line.find(':')
let target = line[0..<colonIdx].strip()
self.emitToken(tkIdentifier, target, lineStart)
if colonIdx + 1 < line.len:
let deps = line[colonIdx + 1 .. ^1].strip()
if deps.len > 0:
self.emitToken(tkString, deps, lineStart)
elif line.strip().len > 0 and line[0] notin {'\t', '#'}:
if line.contains('='):
self.emitToken(tkAssignment, line.strip(), lineStart)
elif line.strip().startsWith(".PHONY") or line.strip().startsWith(".SUFFIXES"):
self.emitToken(tkDirective, line.strip(), lineStart)
else:
self.recordError(esWarning, &"Makefile line {lineNum} should start with tab for recipes",
"W1001", lineStart, hint = "Recipe commands must be indented with a tab character")
lineNum += 1
for (_, pos) in self.bracketStack:
self.recordError(esError, "Unclosed '(' in Makefile variable expansion",
ErrUnclosedBracket, pos, hint = "Add a closing ')'")
self.pos = self.sourceLen
self.line = lineNum
self.emitToken(tkEndOfFile, "", self.currentPos())
when defined(nimcheckDebug):
if self.options.debugMode: debugLeave("MAKEFILE_TOKENIZER", &"Complete: {self.tokens.len} tokens")
src/nimcheck/tokenizers/javascript_tokenizer.nim
+5 -11
View File
@@ -8,7 +8,7 @@ type
JavaScriptTokenizer* = ref object of TokenizerBase
const
jsKeywords = @[
jsKeywords* = @[
"async", "await", "break", "case", "catch", "class", "const", "continue",
"debugger", "default", "delete", "do", "else", "enum", "export", "extends",
"false", "finally", "for", "function", "if", "import", "in", "instanceof",
@@ -19,7 +19,7 @@ const
"as", "any", "boolean", "number", "string", "undefined", "never",
"keyof", "infer", "unknown"
]
jsKeywordSet = jsKeywords.toHashSet()
jsKeywordSet* = jsKeywords.toHashSet()
proc newJavaScriptTokenizer*(source: string, options: ValidationOptions = newValidationOptions()): JavaScriptTokenizer =
result = JavaScriptTokenizer(
@@ -178,19 +178,13 @@ method tokenize*(self: JavaScriptTokenizer) =
# Brackets
elif c == '(': self.emitBracketToken(kOpenParen, $self.advance(), startPos)
elif c == ')':
self.emitBracketToken(kCloseParen, $self.advance(), startPos)
if self.bracketStack.len > 0 and self.bracketStack[^1][0] == '(':
discard self.bracketStack.pop()
self.closeBracket(')', kCloseParen, startPos)
elif c == '[': self.emitBracketToken(kOpenBracket, $self.advance(), startPos)
elif c == ']':
self.emitBracketToken(kCloseBracket, $self.advance(), startPos)
if self.bracketStack.len > 0 and self.bracketStack[^1][0] == '[':
discard self.bracketStack.pop()
self.closeBracket(']', kCloseBracket, startPos)
elif c == '{': self.emitBracketToken(kOpenBrace, $self.advance(), startPos)
elif c == '}':
self.emitBracketToken(kCloseBrace, $self.advance(), startPos)
if self.bracketStack.len > 0 and self.bracketStack[^1][0] == '{':
discard self.bracketStack.pop()
self.closeBracket('}', kCloseBrace, startPos)
# Operators
elif c in {'+', '-', '*', '/', '%', '^', '~', '|', '&', '<', '>', '=', '!', '?'}:
src/nimcheck/tokenizers/jinja_tokenizer.nim
+6
View File
@@ -42,6 +42,7 @@ method tokenize*(self: JinjaTokenizer) =
debugEnter("JINJA_TOKENIZER", "Starting Jinja tokenization")
while self.hasMore():
let posBefore = self.pos
let startPos = self.currentPos()
let c = self.peek()
@@ -116,6 +117,11 @@ method tokenize*(self: JinjaTokenizer) =
text.add(self.advance())
if text.len > 0:
self.emitToken(tkTemplateTag, text, startPos)
else:
# Lone '{' or other delimiter — must always advance (prevents infinite loop)
self.emitToken(tkSpecial, $self.advance(), startPos)
self.finishTokenizeStep(posBefore)
self.emitToken(tkEndOfFile, "", self.currentPos())
when defined(nimcheckDebug):
src/nimcheck/tokenizers/lang_tokenizers.nim
+447
View File
@@ -0,0 +1,447 @@
## Language tokenizers: Kotlin, Lua, Swift.
##
## Standalone tokenizers for languages that do not fit the
## C-like or template-based tokenization patterns.
import std/[strformat, sets]
{.warning[UnusedImport]:off.}
import ../core/types, ../core/tokenizerbase
import extended_tokenizers
# ---------------------------------------------------------------------------
# Kotlin tokenizer
# ---------------------------------------------------------------------------
type
KotlinTokenizer* = ref object of TokenizerBase
const
kotlinKeywords* = @[
"abstract", "actual", "annotation", "as", "as?", "break", "by", "catch",
"class", "companion", "const", "constructor", "continue", "crossinline",
"data", "delegate", "do", "dynamic", "else", "enum", "expect", "external",
"false", "field", "file", "final", "finally", "for", "fun", "if", "import",
"in", "infix", "init", "inline", "inner", "interface", "internal", "is",
"it", "lateinit", "noinline", "null", "object", "open", "operator", "out",
"override", "package", "param", "private", "property", "protected", "public",
"receiver", "reified", "return", "sealed", "set", "setparam", "super",
"suspend", "tailrec", "this", "throw", "true", "try", "typealias",
"typeof", "val", "var", "vararg", "when", "where", "while"
]
kotlinKeywordSet* = kotlinKeywords.toHashSet()
proc newKotlinTokenizer*(source: string, options: ValidationOptions = newValidationOptions()): KotlinTokenizer =
KotlinTokenizer(
source: source, sourceLen: source.len, options: options,
tokens: @[], pos: 0, line: 1, column: 1, bracketStack: @[], errors: @[], eof: false
)
method tokenize*(self: KotlinTokenizer) =
when defined(nimcheckDebug):
if self.options.debugMode: debugEnter("KOTLIN_TOKENIZER", "Starting Kotlin tokenization")
while self.hasMore():
let posBefore = self.pos
let startPos = self.currentPos()
let c = self.peek()
if c in {' ', '\t', '\r'}:
let wsStart = self.currentPos()
var ws = ""
while self.hasMore() and self.peek() in {' ', '\t', '\r'}:
ws.add(self.advance())
self.emitToken(tkWhitespace, ws, wsStart)
elif c == '\n':
self.emitToken(tkNewline, $self.advance(), startPos)
elif self.peekString(2) == "//":
let comStart = self.currentPos()
discard self.advance()
discard self.advance()
self.emitToken(tkComment, "//" & self.parseLineComment(), comStart)
elif self.peekString(2) == "/*":
let comStart = self.currentPos()
discard self.advance()
discard self.advance()
let content = self.parseBlockComment("/*", "*/")
let isDoc = content.startsWith("*")
self.emitToken(if isDoc: tkDocComment else: tkComment, "/*" & content & "*/", comStart)
elif c == '"':
let strStart = self.currentPos()
discard self.advance()
let content = '"' & self.parseQuotedString('"', strStart)
self.emitToken(tkString, content, strStart)
elif c == '\'':
let strStart = self.currentPos()
discard self.advance()
let content = "'" & self.parseQuotedString('\'', strStart)
self.emitToken(tkString, content, strStart)
elif c == '`':
# Kotlin backtick identifier
let btStart = self.currentPos()
discard self.advance()
var ident = "`"
while self.hasMore() and self.peek() != '`':
ident.add(self.advance())
if self.hasMore():
ident.add(self.advance())
self.emitToken(tkIdentifier, ident, btStart)
elif isDigit(c) or (c == '.' and self.hasMore() and self.peek(1).isDigit()):
let numStart = self.currentPos()
var num = ""
while self.hasMore() and (self.peek().isDigit() or self.peek() in {'.', 'x', 'X', 'e', 'E', '+', '-', '_', 'L', 'l', 'F', 'f'}):
num.add(self.advance())
self.emitToken(tkNumber, num, numStart)
elif isAlpha(c) or c == '_':
let idStart = self.currentPos()
var ident = ""
while self.hasMore() and (isAlphaNum(self.peek()) or self.peek() == '_'):
ident.add(self.advance())
if kotlinKeywordSet.contains(ident):
self.emitToken(tkKeyword, ident, idStart)
else:
self.emitToken(tkIdentifier, ident, idStart)
elif c == '(':
self.emitBracketToken(kOpenParen, $self.advance(), startPos)
elif c == ')':
self.closeBracket(')', kCloseParen, startPos)
elif c == '[':
self.emitBracketToken(kOpenBracket, $self.advance(), startPos)
elif c == ']':
self.closeBracket(']', kCloseBracket, startPos)
elif c == '{':
self.emitBracketToken(kOpenBrace, $self.advance(), startPos)
elif c == '}':
self.closeBracket('}', kCloseBrace, startPos)
elif c in {'+', '-', '*', '/', '%', '&', '|', '^', '~', '!', '?', ':', '=', '<', '>', '@'}:
let opStart = self.currentPos()
var op = ""
while self.hasMore() and self.peek() in {'+', '-', '*', '/', '%', '&', '|', '^', '~', '!', '?', ':', '=', '<', '>', '@'}:
op.add(self.advance())
if op in ["=", "+=", "-=", "*=", "/=", "%="]:
self.emitToken(tkAssignment, op, opStart)
else:
self.emitToken(tkOperator, op, opStart)
elif c in {'.', ',', ';'}:
self.emitToken(tkPunctuation, $self.advance(), startPos)
elif c == '#':
self.emitToken(tkDirective, $self.advance(), startPos)
else:
self.emitToken(tkSpecial, $self.advance(), startPos)
self.finishTokenizeStep(posBefore)
self.emitToken(tkEndOfFile, "", self.currentPos())
when defined(nimcheckDebug):
if self.options.debugMode: debugLeave("KOTLIN_TOKENIZER", &"Complete: {self.tokens.len} tokens")
# ---------------------------------------------------------------------------
# Lua tokenizer
# ---------------------------------------------------------------------------
type
LuaTokenizer* = ref object of TokenizerBase
const
luaKeywords* = @[
"and", "break", "do", "else", "elseif", "end", "false", "for", "function",
"goto", "if", "in", "local", "nil", "not", "or", "repeat", "return",
"then", "true", "until", "while"
]
luaKeywordSet* = luaKeywords.toHashSet()
proc newLuaTokenizer*(source: string, options: ValidationOptions = newValidationOptions()): LuaTokenizer =
LuaTokenizer(
source: source, sourceLen: source.len, options: options,
tokens: @[], pos: 0, line: 1, column: 1, bracketStack: @[], errors: @[], eof: false
)
method tokenize*(self: LuaTokenizer) =
when defined(nimcheckDebug):
if self.options.debugMode: debugEnter("LUA_TOKENIZER", "Starting Lua tokenization")
while self.hasMore():
let posBefore = self.pos
let startPos = self.currentPos()
let c = self.peek()
if c in {' ', '\t', '\r'}:
let wsStart = self.currentPos()
var ws = ""
while self.hasMore() and self.peek() in {' ', '\t', '\r'}:
ws.add(self.advance())
self.emitToken(tkWhitespace, ws, wsStart)
elif c == '\n':
self.emitToken(tkNewline, $self.advance(), startPos)
elif self.peekString(2) == "--":
let comStart = self.currentPos()
discard self.advance()
discard self.advance()
if self.peekString(2) == "[[":
# Long block comment --[[ ... ]]
discard self.advance()
discard self.advance()
let content = self.parseBlockComment("[[", "]]")
self.emitToken(tkComment, "--[[" & content & "]]", comStart)
else:
self.emitToken(tkComment, "--" & self.parseLineComment(), comStart)
elif c == '"' or c == '\'':
let strStart = self.currentPos()
discard self.advance()
let content = $c & self.parseQuotedString(c, strStart)
self.emitToken(tkString, content, strStart)
elif c == '[' and self.peekString(2) == "[[":
# Long string literal [[ ... ]]
let lsStart = self.currentPos()
discard self.advance()
discard self.advance()
var content = "[["
var foundClosing = false
while self.hasMore():
if self.peekString(2) == "]]":
content.add("]]")
discard self.advance()
discard self.advance()
foundClosing = true
break
content.add(self.advance())
if not foundClosing:
self.recordError(esError, "Unclosed long string literal", ErrUnclosedString, lsStart,
hint = "Add closing ']]'")
self.emitToken(tkString, content, lsStart)
elif isDigit(c) or (c == '.' and self.hasMore() and self.peek(1).isDigit()):
let numStart = self.currentPos()
var num = ""
while self.hasMore() and (self.peek().isDigit() or self.peek() in {'.', 'e', 'E', '+', '-', 'x', 'X'}):
num.add(self.advance())
self.emitToken(tkNumber, num, numStart)
elif isAlpha(c) or c == '_':
let idStart = self.currentPos()
var ident = ""
while self.hasMore() and (isAlphaNum(self.peek()) or self.peek() == '_'):
ident.add(self.advance())
if luaKeywordSet.contains(ident):
self.emitToken(tkKeyword, ident, idStart)
else:
self.emitToken(tkIdentifier, ident, idStart)
elif c == '(':
self.emitBracketToken(kOpenParen, $self.advance(), startPos)
elif c == ')':
self.closeBracket(')', kCloseParen, startPos)
elif c == '[':
self.emitBracketToken(kOpenBracket, $self.advance(), startPos)
elif c == ']':
self.closeBracket(']', kCloseBracket, startPos)
elif c == '{':
self.emitBracketToken(kOpenBrace, $self.advance(), startPos)
elif c == '}':
self.closeBracket('}', kCloseBrace, startPos)
elif c in {'+', '-', '*', '/', '%', '^', '#', '=', '<', '>', '~', ':', '&', '|', '!'}:
let opStart = self.currentPos()
var op = ""
while self.hasMore() and self.peek() in {'+', '-', '*', '/', '%', '^', '#', '=', '<', '>', '~', ':', '&', '|', '!'}:
op.add(self.advance())
if op == "=":
self.emitToken(tkAssignment, op, opStart)
else:
self.emitToken(tkOperator, op, opStart)
elif c in {'.', ',', ';'}:
self.emitToken(tkPunctuation, $self.advance(), startPos)
elif c == '@':
self.emitToken(tkSpecial, $self.advance(), startPos)
else:
self.emitToken(tkSpecial, $self.advance(), startPos)
self.finishTokenizeStep(posBefore)
self.emitToken(tkEndOfFile, "", self.currentPos())
when defined(nimcheckDebug):
if self.options.debugMode: debugLeave("LUA_TOKENIZER", &"Complete: {self.tokens.len} tokens")
# ---------------------------------------------------------------------------
# Swift tokenizer
# ---------------------------------------------------------------------------
type
SwiftTokenizer* = ref object of TokenizerBase
const
swiftKeywords* = @[
"Any", "as", "associatedtype", "async", "await", "break", "case", "catch",
"class", "continue", "default", "defer", "deinit", "do", "else", "enum",
"extension", "fallthrough", "false", "fileprivate", "for", "func", "guard",
"if", "import", "in", "indirect", "infix", "init", "inout", "internal",
"is", "lazy", "let", "nil", "nonisolated", "open", "operator", "optional",
"override", "package", "postfix", "precedencegroup", "prefix", "private",
"protocol", "public", "rethrows", "return", "self", "Self", "static",
"struct", "subscript", "super", "switch", "throw", "throws", "true",
"try", "Type", "typealias", "unowned", "var", "weak", "where", "while",
"some", "any", "macro", "actor", "isolated", "distributed", "consuming",
"borrowing", "nonconsuming", "_borrowing"
]
swiftKeywordSet* = swiftKeywords.toHashSet()
proc newSwiftTokenizer*(source: string, options: ValidationOptions = newValidationOptions()): SwiftTokenizer =
SwiftTokenizer(
source: source, sourceLen: source.len, options: options,
tokens: @[], pos: 0, line: 1, column: 1, bracketStack: @[], errors: @[], eof: false
)
method tokenize*(self: SwiftTokenizer) =
when defined(nimcheckDebug):
if self.options.debugMode: debugEnter("SWIFT_TOKENIZER", "Starting Swift tokenization")
while self.hasMore():
let posBefore = self.pos
let startPos = self.currentPos()
let c = self.peek()
if c in {' ', '\t', '\r'}:
let wsStart = self.currentPos()
var ws = ""
while self.hasMore() and self.peek() in {' ', '\t', '\r'}:
ws.add(self.advance())
self.emitToken(tkWhitespace, ws, wsStart)
elif c == '\n':
self.emitToken(tkNewline, $self.advance(), startPos)
elif self.peekString(2) == "//":
let comStart = self.currentPos()
discard self.advance()
discard self.advance()
self.emitToken(tkComment, "//" & self.parseLineComment(), comStart)
elif self.peekString(2) == "/*":
let comStart = self.currentPos()
discard self.advance()
discard self.advance()
if self.hasMore() and self.peek() == '*':
# Doc comment /** ... */
discard self.advance()
let content = self.parseBlockComment("*", "*/")
self.emitToken(tkDocComment, "/**" & content & "*/", comStart)
else:
let content = self.parseBlockComment("/*", "*/")
self.emitToken(tkComment, "/*" & content & "*/", comStart)
elif c == '"':
let strStart = self.currentPos()
discard self.advance()
# Check for multiline string literal """"""
if self.peekString(2) == "\"\"\"...":
discard
var content = "\""
var foundClosing = false
while self.hasMore():
let sc = self.advance()
if sc == '\\':
content.add(sc)
if self.hasMore(): content.add(self.advance())
elif sc == '"':
content.add('"')
foundClosing = true
break
elif sc == '\n':
self.recordError(esError, "Newline in Swift string literal", ErrUnclosedString, strStart,
hint = "Use triple-quoted string \"\"\" for multiline strings")
break
else:
content.add(sc)
if not foundClosing:
self.recordError(esError, "Unclosed Swift string literal", ErrUnclosedString, strStart)
self.emitToken(tkString, content, strStart)
elif c == '`':
# Swift backtick-escaped identifier
let btStart = self.currentPos()
discard self.advance()
var ident = "`"
while self.hasMore() and self.peek() != '`':
ident.add(self.advance())
if self.hasMore():
ident.add(self.advance())
self.emitToken(tkIdentifier, ident, btStart)
elif isDigit(c) or (c == '.' and self.hasMore() and self.peek(1).isDigit()):
let numStart = self.currentPos()
var num = ""
while self.hasMore() and (self.peek().isDigit() or self.peek() in {'.', 'x', 'X', 'o', 'O', 'b', 'B', 'e', 'E', '+', '-', '_'}):
num.add(self.advance())
self.emitToken(tkNumber, num, numStart)
elif isAlpha(c) or c == '_':
let idStart = self.currentPos()
var ident = ""
while self.hasMore() and (isAlphaNum(self.peek()) or self.peek() in {'_', '$'}):
ident.add(self.advance())
if swiftKeywordSet.contains(ident):
self.emitToken(tkKeyword, ident, idStart)
else:
self.emitToken(tkIdentifier, ident, idStart)
elif c == '(':
self.emitBracketToken(kOpenParen, $self.advance(), startPos)
elif c == ')':
self.closeBracket(')', kCloseParen, startPos)
elif c == '[':
self.emitBracketToken(kOpenBracket, $self.advance(), startPos)
elif c == ']':
self.closeBracket(']', kCloseBracket, startPos)
elif c == '{':
self.emitBracketToken(kOpenBrace, $self.advance(), startPos)
elif c == '}':
self.closeBracket('}', kCloseBrace, startPos)
elif c in {'+', '-', '*', '/', '%', '&', '|', '^', '~', '!', '?', ':', '=', '<', '>', '.'}:
let opStart = self.currentPos()
var op = ""
while self.hasMore() and self.peek() in {'+', '-', '*', '/', '%', '&', '|', '^', '~', '!', '?', ':', '=', '<', '>', '.'}:
op.add(self.advance())
if op == "=":
self.emitToken(tkAssignment, op, opStart)
else:
self.emitToken(tkOperator, op, opStart)
elif c in {',', ';', ':'}:
self.emitToken(tkPunctuation, $self.advance(), startPos)
elif c == '#':
let hashStart = self.currentPos()
var directive = "#"
discard self.advance()
while self.hasMore() and (isAlphaNum(self.peek()) or self.peek() == '_'):
directive.add(self.advance())
self.emitToken(tkDirective, directive, hashStart)
else:
self.emitToken(tkSpecial, $self.advance(), startPos)
self.finishTokenizeStep(posBefore)
self.emitToken(tkEndOfFile, "", self.currentPos())
when defined(nimcheckDebug):
if self.options.debugMode: debugLeave("SWIFT_TOKENIZER", &"Complete: {self.tokens.len} tokens")
src/nimcheck/tokenizers/nim_tokenizer.nim
+30 -9
View File
@@ -184,17 +184,38 @@ method tokenize*(self: NimTokenizer) =
let chStart = self.currentPos()
discard self.advance() # '
var content = "'"
if self.hasMore():
if self.peek() == '\\':
content.add(self.advance())
if self.hasMore(): content.add(self.advance())
else:
content.add(self.advance())
if self.hasMore() and self.peek() == '\'':
content.add(self.advance())
var charCount = 0
while self.hasMore():
let cc = self.peek()
if cc == '\'':
discard self.advance()
content.add('\'')
if charCount == 0:
self.recordError(esError, "Empty character literal", ErrUnclosedString, chStart,
hint = "Character literal must contain exactly one character")
elif charCount > 1:
self.recordError(esError,
&"Character literal contains {charCount} characters instead of 1",
ErrInvalidSyntax, chStart,
hint = "Use a string for multi-character sequences")
self.emitToken(tkString, content, chStart)
else:
break
elif cc == '\n':
self.recordError(esError, "Unclosed character literal", ErrUnclosedString, chStart)
self.emitToken(tkString, content, chStart)
break
elif cc == '\\':
content.add(self.advance())
if self.hasMore():
content.add(self.advance())
charCount += 1
else:
content.add(self.advance())
charCount += 1
if not self.hasMore() and charCount > 0:
self.recordError(esError, "Unclosed character literal", ErrUnclosedString, chStart)
self.emitToken(tkString, content, chStart)
# Numbers
elif isDigit(c):
src/nimcheck/tokenizers/php_tokenizer.nim
+1 -1
View File
@@ -137,9 +137,9 @@ method tokenize*(self: PHPTokenizer) =
foundClosing = true
break
elif sc == '\n':
self.recordError(esError, "Unclosed string at end of line", ErrUnclosedString, strStart)
self.emitToken(tkString, content, strStart)
foundClosing = true
foundClosing = true
break
else:
content.add(sc)
src/nimcheck/tokenizers/python_tokenizer.nim
+2 -1
View File
@@ -341,7 +341,8 @@ method tokenize*(self: PythonTokenizer) =
let uStart = self.currentPos()
self.emitToken(tkError, $self.advance(), uStart)
# Add dedent tokens for remaining indentation at EOF
# Flush remaining indentation at EOF
# Trailing indent at EOF is valid Python - emit dedent tokens silently.
while self.indentStack.len > 1:
discard self.indentStack.pop()
self.emitToken(tkSpecial, "<dedent>", self.currentPos())
src/nimcheck/tokenizers/type_xml_tokenizer.nim
+452
View File
@@ -0,0 +1,452 @@
## Improved TypeScript and XML tokenizers.
##
## TypeScript builds on JavaScriptTokenizer with TS-specific features.
## XML provides a full XML/SGML-style tag tokenizer.
import std/[strformat, sets]
{.warning[UnusedImport]:off.}
import ../core/types, ../core/tokenizerbase
import javascript_tokenizer
# ---------------------------------------------------------------------------
# Improved TypeScript tokenizer (extends JavaScript tokenizer)
# ---------------------------------------------------------------------------
type
TypeScriptTokenizer* = ref object of JavaScriptTokenizer
const
tsExtraKeywords* = @[
"interface", "type", "declare", "module", "namespace", "abstract",
"readonly", "as", "any", "boolean", "number", "string", "undefined",
"never", "keyof", "infer", "unknown", "satisfies", "asserts",
"is", "enum", "const", "override", "private", "protected", "public",
"static", "implements", "extends", "infer", "template", "using"
]
tsExtraKeywordSet* = tsExtraKeywords.toHashSet()
proc newTypeScriptTokenizer*(source: string, options: ValidationOptions = newValidationOptions()): TypeScriptTokenizer =
TypeScriptTokenizer(
source: source, sourceLen: source.len, options: options,
tokens: @[], pos: 0, line: 1, column: 1, bracketStack: @[], errors: @[], eof: false
)
method tokenize*(self: TypeScriptTokenizer) =
when defined(nimcheckDebug):
if self.options.debugMode: debugEnter("TS_TOKENIZER", "Starting TypeScript tokenization")
while self.hasMore():
let startPos = self.currentPos()
let c = self.peek()
if c in {' ', '\t', '\r'}:
let wsStart = self.currentPos()
var ws = ""
while self.hasMore() and self.peek() in {' ', '\t', '\r'}:
ws.add(self.advance())
self.emitToken(tkWhitespace, ws, wsStart)
elif c == '\n':
self.emitToken(tkNewline, $self.advance(), startPos)
# Template literals
elif c == '`':
let tlStart = self.currentPos()
discard self.advance()
var content = "`"
var braceDepth = 0
var foundClosing = false
while self.hasMore():
let tc = self.advance()
if tc == '\\':
content.add(tc)
if self.hasMore(): content.add(self.advance())
elif tc == '`' and braceDepth == 0:
content.add('`')
self.emitToken(tkString, content, tlStart)
foundClosing = true
break
elif tc == '$' and self.hasMore() and self.peek() == '{' and braceDepth == 0:
content.add("${")
discard self.advance()
braceDepth += 1
elif tc == '{': braceDepth += 1; content.add(tc)
elif tc == '}': braceDepth -= 1; content.add(tc)
else: content.add(tc)
if not foundClosing:
self.recordError(esError, "Unclosed template literal", ErrUnclosedString, tlStart)
self.emitToken(tkString, content, tlStart)
elif self.peekString(2) == "//":
let comStart = self.currentPos()
discard self.advance()
discard self.advance()
self.emitToken(tkComment, "//" & self.parseLineComment(), comStart)
elif self.peekString(2) == "/*":
let comStart = self.currentPos()
discard self.advance()
discard self.advance()
let content = self.parseBlockComment("/*", "*/")
let isDoc = content.startsWith("*")
self.emitToken(if isDoc: tkDocComment else: tkComment, "/*" & content & "*/", comStart)
elif c == '/' and self.pos > 0:
let prevChar = if self.pos > 0: self.source[self.pos - 1] else: '\0'
if prevChar in {' ', '\t', '\n', '\r', '=', '(', '[', '{', '!', '&', '|', ':', ';', ',', '^', '~', '?'}:
let regStart = self.currentPos()
discard self.advance()
var regex = "/"
var foundClosing = false
while self.hasMore():
let rc = self.advance()
if rc == '\\':
regex.add(rc)
if self.hasMore(): regex.add(self.advance())
elif rc == '/':
regex.add('/')
while self.hasMore() and (isAlpha(self.peek()) or self.peek() in {'$', '_'}):
regex.add(self.advance())
self.emitToken(tkSpecial, regex, regStart)
foundClosing = true
break
else:
regex.add(rc)
if not foundClosing:
self.emitToken(tkOperator, "/", regStart)
else:
self.emitToken(tkOperator, "/", startPos)
discard self.advance()
elif c == '"' or c == '\'':
let quote = c
let strStart = self.currentPos()
discard self.advance()
var content = $quote
var foundClosing = false
while self.hasMore():
let sc = self.advance()
if sc == '\\':
content.add(sc)
if self.hasMore(): content.add(self.advance())
elif sc == quote:
content.add(quote)
self.emitToken(tkString, content, strStart)
foundClosing = true
break
elif sc == '\n':
self.recordError(esError, "Newline in string", ErrUnclosedString, strStart)
break
else:
content.add(sc)
if not foundClosing:
self.recordError(esError, "Unclosed string", ErrUnclosedString, strStart)
self.emitToken(tkString, content, strStart)
elif isDigit(c):
let numStart = self.currentPos()
var num = ""
while self.hasMore() and (isAlphaNum(self.peek()) or self.peek() in {'.', 'x', 'X', 'o', 'O', 'b', 'B', '_'}):
num.add(self.advance())
self.emitToken(tkNumber, num, numStart)
elif isAlpha(c) or c == '$' or c == '_':
let idStart = self.currentPos()
var ident = ""
while self.hasMore() and (isAlphaNum(self.peek()) or self.peek() in {'_', '$'}) or self.peek() == '.':
# Support dotted names in types
if self.peek() == '.' and ident.len > 0 and isAlpha(self.source[self.pos - 1]):
ident.add(self.advance())
continue
ident.add(self.advance())
if jsKeywordSet.contains(ident):
self.emitToken(tkKeyword, ident, idStart)
elif tsExtraKeywordSet.contains(ident):
self.emitToken(tkKeyword, ident, idStart)
else:
self.emitToken(tkIdentifier, ident, idStart)
elif c == '(': self.emitBracketToken(kOpenParen, $self.advance(), startPos)
elif c == ')':
self.closeBracket(')', kCloseParen, startPos)
elif c == '[': self.emitBracketToken(kOpenBracket, $self.advance(), startPos)
elif c == ']':
self.closeBracket(']', kCloseBracket, startPos)
elif c == '{': self.emitBracketToken(kOpenBrace, $self.advance(), startPos)
elif c == '}':
self.closeBracket('}', kCloseBrace, startPos)
elif c in {'+', '-', '*', '/', '%', '^', '~', '|', '&', '<', '>', '=', '!', '?'}:
let opStart = self.currentPos()
var op = ""
while self.hasMore() and self.peek() in {'+', '-', '*', '/', '%', '^', '~', '|', '&', '<', '>', '=', '!', '?', '.'}:
let nc = self.peek()
if nc == '.' and op.len > 0: break
op.add(self.advance())
if op in ["=", "=>"]:
self.emitToken(tkAssignment, op, opStart)
else:
self.emitToken(tkOperator, op, opStart)
elif c == '.':
self.emitToken(tkPunctuation, $self.advance(), startPos)
elif c in {',', ';', ':'}:
self.emitToken(tkPunctuation, $self.advance(), startPos)
elif c == '@':
# TS decorator
let atStart = self.currentPos()
var decorator = "@"
discard self.advance()
while self.hasMore() and (isAlphaNum(self.peek()) or self.peek() in {'_', '.'}):
decorator.add(self.advance())
self.emitToken(tkDirective, decorator, atStart)
elif c == '#':
let hashStart = self.currentPos()
var dir = "#"
discard self.advance()
while self.hasMore() and (isAlphaNum(self.peek()) or self.peek() == '_'):
dir.add(self.advance())
self.emitToken(tkDirective, dir, hashStart)
elif c == '<' and self.pos + 1 < self.sourceLen and (isAlpha(self.peek(1)) or self.peek(1) == '/'):
# XML-like JSX tag or TypeScript type parameter - treat as operator/template
self.emitBracketToken(kAngleOpen, $self.advance(), startPos)
else:
self.emitToken(tkError, $self.advance(), startPos)
self.emitToken(tkEndOfFile, "", self.currentPos())
when defined(nimcheckDebug):
if self.options.debugMode: debugLeave("TS_TOKENIZER", &"Complete: {self.tokens.len} tokens")
# ---------------------------------------------------------------------------
# XML tokenizer
# ---------------------------------------------------------------------------
type
XMLTokenizer* = ref object of TokenizerBase
tagStack*: seq[string]
const
xmlKeywords* = @["xml", "xsl", "schema", "doctype", "element", "attribute",
"entity", "notation", "simpleType", "complexType", "sequence", "choice",
"include", "import", "redefine", "override", "annotation", "documentation",
"appinfo", "any", "group", "attributeGroup", "restriction", "extension",
"union", "list", "all", "key", "keyref", "unique", "selector", "field",
"pattern", "enumeration", "length", "minLength", "maxLength", "minInclusive",
"maxInclusive", "minExclusive", "maxExclusive", "totalDigits", "fractionDigits",
"whiteSpace"
]
xmlKeywordSet* = xmlKeywords.toHashSet()
proc newXMLTokenizer*(source: string, options: ValidationOptions = newValidationOptions()): XMLTokenizer =
XMLTokenizer(
source: source, sourceLen: source.len, options: options,
tokens: @[], pos: 0, line: 1, column: 1, bracketStack: @[], errors: @[], eof: false,
tagStack: @[]
)
method tokenize*(self: XMLTokenizer) =
when defined(nimcheckDebug):
if self.options.debugMode: debugEnter("XML_TOKENIZER", "Starting XML tokenization")
while self.hasMore():
let startPos = self.currentPos()
let c = self.peek()
if c in {' ', '\t', '\r'}:
let wsStart = self.currentPos()
var ws = ""
while self.hasMore() and self.peek() in {' ', '\t', '\r'}:
ws.add(self.advance())
self.emitToken(tkWhitespace, ws, wsStart)
elif c == '\n':
self.emitToken(tkNewline, $self.advance(), startPos)
elif self.peekString(4) == "<!--":
let comStart = self.currentPos()
discard self.advance()
discard self.advance()
discard self.advance()
discard self.advance()
var content = ""
var foundClosing = false
while self.hasMore():
if self.peekString(3) == "-->":
discard self.advance()
discard self.advance()
discard self.advance()
foundClosing = true
break
content.add(self.advance())
if not foundClosing:
self.recordError(esError, "Unclosed XML comment", ErrUnclosedComment, comStart,
hint = "Add closing '-->'")
self.emitToken(tkComment, "<!--" & content & "-->", comStart)
elif self.peekString(2) == "<?":
# Processing instruction <?xml ... ?>
let piStart = self.currentPos()
discard self.advance()
discard self.advance()
var content = "<?"
while self.hasMore():
if self.peekString(2) == "?>":
content.add("?>")
discard self.advance()
discard self.advance()
break
content.add(self.advance())
self.emitToken(tkDirective, content, piStart)
elif c == '<':
let tagStart = self.currentPos()
discard self.advance()
var isClosing = false
var isSelfClosing = false
if self.hasMore() and self.peek() == '/':
isClosing = true
discard self.advance()
# Read tag name
var tagName = ""
while self.hasMore() and self.peek() != '>' and self.peek() != '/' and self.peek() notin {' ', '\t', '\n', '\r'}:
tagName.add(self.advance())
if tagName.len > 0:
if isClosing:
self.emitToken(tkSpecial, "</" & tagName, tagStart)
if self.tagStack.len > 0 and self.tagStack[^1] == tagName:
discard self.tagStack.pop()
else:
self.recordError(esError, &"Mismatched XML close tag '</{tagName}>'",
ErrMismatchedBracket, tagStart,
hint = if self.tagStack.len > 0: &"Expected '</{self.tagStack[^1]}>'" else: "Unexpected close tag")
else:
self.tagStack.add(tagName)
self.emitToken(tkKeyword, tagName, tagStart)
else:
self.emitToken(tkOperator, "<", tagStart)
# Read attributes until > or />
while self.hasMore() and self.peek() != '>':
if self.peek() in {' ', '\t', '\n', '\r'}:
let wsStart2 = self.currentPos()
var ws2 = ""
while self.hasMore() and self.peek() in {' ', '\t', '\n', '\r'}:
ws2.add(self.advance())
self.emitToken(tkWhitespace, ws2, wsStart2)
elif self.peek() == '/' and self.pos + 1 < self.sourceLen and self.peek(1) == '>':
isSelfClosing = true
discard self.advance()
discard self.advance()
if not isClosing and self.tagStack.len > 0:
discard self.tagStack.pop()
self.emitToken(tkOperator, "/>", tagStart)
break
elif isAlpha(self.peek()) or self.peek() in {'_', ':'}:
let attrStart = self.currentPos()
var attrName = ""
while self.hasMore() and (isAlphaNum(self.peek()) or self.peek() in {'_', ':', '-'}):
attrName.add(self.advance())
self.emitToken(tkIdentifier, attrName, attrStart)
# Skip = "value"
while self.hasMore() and self.peek() in {' ', '\t', '\n', '\r'}:
discard self.advance()
if self.hasMore() and self.peek() == '=':
discard self.advance()
while self.hasMore() and self.peek() in {' ', '\t', '\n', '\r'}:
discard self.advance()
if self.hasMore() and self.peek() == '"':
let valStart = self.currentPos()
discard self.advance()
var val = "\""
while self.hasMore() and self.peek() != '"':
val.add(self.advance())
if self.hasMore():
val.add(self.advance())
self.emitToken(tkString, val, valStart)
else:
discard self.advance()
if self.hasMore() and self.peek() == '>':
discard self.advance()
if not isClosing:
self.emitToken(tkOperator, ">", tagStart)
elif c == '>':
discard self.advance()
# Already handled in tag context
elif c == '"' or c == '\'':
let strStart = self.currentPos()
discard self.advance()
var content = $c
var foundClosing = false
while self.hasMore():
let sc = self.advance()
if sc == c:
content.add(c)
foundClosing = true
break
content.add(sc)
if not foundClosing:
self.recordError(esError, &"Unclosed XML attribute value", ErrUnclosedString, strStart)
self.emitToken(tkString, content, strStart)
elif isDigit(c) or (c == '.' and self.hasMore() and self.peek(1).isDigit()):
let numStart = self.currentPos()
var num = ""
while self.hasMore() and (self.peek().isDigit() or self.peek() in {'.', 'e', 'E', '+', '-'}):
num.add(self.advance())
self.emitToken(tkNumber, num, numStart)
elif isAlpha(c) or c == '_' or c == ':':
let idStart = self.currentPos()
var ident = ""
while self.hasMore() and (isAlphaNum(self.peek()) or self.peek() in {'_', ':', '-'}):
ident.add(self.advance())
if xmlKeywordSet.contains(ident):
self.emitToken(tkKeyword, ident, idStart)
else:
self.emitToken(tkIdentifier, ident, idStart)
elif c in {'&'}:
let entStart = self.currentPos()
var entity = "&"
discard self.advance()
while self.hasMore() and self.peek() != ';' and self.peek() != ' ':
entity.add(self.advance())
if self.hasMore() and self.peek() == ';':
entity.add(self.advance())
self.emitToken(tkSpecial, entity, entStart)
elif c in {',', ';'}:
self.emitToken(tkPunctuation, $self.advance(), startPos)
elif c in {'=', '!', '?'}:
self.emitToken(tkOperator, $self.advance(), startPos)
else:
# Text content between tags
let txtStart = self.currentPos()
var text = ""
while self.hasMore() and self.peek() notin {'<', '&'}:
text.add(self.advance())
if text.len > 0 and text.strip().len > 0:
self.emitToken(tkTemplateTag, text, txtStart)
elif text.len > 0:
self.emitToken(tkWhitespace, text, txtStart)
else:
discard self.advance()
# Check for unclosed tags
for tag in self.tagStack:
let pos = newSourcePosition(self.line, self.column, self.pos)
self.recordError(esError, &"Unclosed XML tag '<{tag}>'",
ErrUnclosedBlock, pos, hint = &"Add closing '</{tag}>'")
self.emitToken(tkEndOfFile, "", self.currentPos())
when defined(nimcheckDebug):
if self.options.debugMode: debugLeave("XML_TOKENIZER", &"Complete: {self.tokens.len} tokens")