feat: add initial validatrix project with multi-language validator framework
Establish the Validatrix source code validation framework in Nim, including core detection, tokenization, and validation infrastructure for 10 languages (bash, HTML, JavaScript, Jinja, JSON, Nim, PHP, Python, TOML, YAML). The initial commit introduces the main validatrix.nim entry point with public API (validateSource, validateFile, inspectSource), a comprehensive test suite with per-language test files, build configuration via Makefile and .nimble, detailed LESSONS_LEARNED.md documenting 14 bug classes found during development, and a .gitignore excluding compiled binaries, logs, and build artifacts. The detector module provides extension-to-flavor mapping for 40+ file extensions and content-based language detection, while the debug module supports conditional compilation with -d:validatrixDebug for detailed tokenization logging.
This commit is contained in:
@@ -0,0 +1,406 @@
|
||||
## Validatrix - Universal Source Code Validation Framework
|
||||
##
|
||||
## A comprehensive, extensible validation framework for all common
|
||||
## programming languages and file formats. Provides tokenizer-based
|
||||
## analysis, auto-detection of file flavor, JSON diagnostics, and
|
||||
## consistent error reporting.
|
||||
##
|
||||
## Usage:
|
||||
## import validatrix
|
||||
## let r = validateSource("print('hello')", flavor=lfPython)
|
||||
## let jsonResult = result.toJson()
|
||||
##
|
||||
## Or with auto-detection:
|
||||
## let r = validateFile("script.py")
|
||||
##
|
||||
## See README.md for full documentation.
|
||||
|
||||
import std/[json, strutils, os, strformat, sequtils, algorithm]
|
||||
|
||||
# Core types and base classes — imported and re-exported for public API consumers
|
||||
import validatrix/core/types
|
||||
import validatrix/core/tokenizerbase
|
||||
import validatrix/core/validatorbase
|
||||
import validatrix/core/detector
|
||||
import validatrix/core/debug
|
||||
import validatrix/reporting/errors
|
||||
export types
|
||||
export tokenizerbase
|
||||
export validatorbase
|
||||
export errors
|
||||
|
||||
# Import all language validators (they register themselves on import)
|
||||
{.warning[UnusedImport]:off.}
|
||||
import validatrix/languages/nim_validator
|
||||
import validatrix/languages/bash_validator
|
||||
import validatrix/languages/python_validator
|
||||
import validatrix/languages/javascript_validator
|
||||
import validatrix/languages/php_validator
|
||||
import validatrix/languages/html_validator
|
||||
import validatrix/languages/jinja_validator
|
||||
import validatrix/languages/config_validators
|
||||
{.warning[UnusedImport]:on.}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Public API - Validate from source string
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
proc validateSource*(
|
||||
source: string,
|
||||
flavor: LanguageFlavor = lfUnknown,
|
||||
options: ValidationOptions = newValidationOptions(),
|
||||
sourcePath: string = ""
|
||||
): ValidationResult =
|
||||
## Validate source code from a string.
|
||||
##
|
||||
## If flavor is lfUnknown (default), auto-detection is performed.
|
||||
## Returns a ValidationResult with errors, diagnostics, and metadata.
|
||||
##
|
||||
## Example:
|
||||
## let r = validateSource("proc hello() = echo 'hi'", flavor=lfNim)
|
||||
## let r = validateSource("var x = 1", flavor=lfUnknown) # auto-detect
|
||||
|
||||
when defined(validatrixDebug):
|
||||
if options.debugMode and not debugEnabled:
|
||||
enableDebug()
|
||||
debugEnter("VALIDATRIX", "validateSource called")
|
||||
|
||||
try:
|
||||
var opts = options
|
||||
if flavor != lfUnknown:
|
||||
opts.flavor = flavor
|
||||
|
||||
# Detect flavor if not explicitly set
|
||||
var detectedFlavor = flavor
|
||||
var detectionMethod = fdmExplicit
|
||||
|
||||
if detectedFlavor == lfUnknown:
|
||||
let (df, dm, _) = detectFlavor(source, sourcePath, lfUnknown)
|
||||
detectedFlavor = df
|
||||
detectionMethod = dm
|
||||
opts.flavor = detectedFlavor
|
||||
|
||||
if detectedFlavor == lfUnknown:
|
||||
# Cannot determine language
|
||||
result = newValidationResult()
|
||||
result.valid = false
|
||||
result.flavor = lfUnknown
|
||||
result.detectionMethod = fdmUnknown
|
||||
result.errors.add(newValidationError(
|
||||
esError,
|
||||
"Could not detect language flavor from source content",
|
||||
ErrInvalidSyntax,
|
||||
newSourcePosition(1, 1, 0),
|
||||
newSourceRange(newSourcePosition(1, 1, 0), newSourcePosition(1, 1, 0)),
|
||||
hint = "Specify the flavor explicitly, or add a shebang/extension"
|
||||
))
|
||||
result.durationMs = 0.0
|
||||
return result
|
||||
|
||||
# Create validator and run validation
|
||||
let validator = createValidator(detectedFlavor, source, opts, sourcePath)
|
||||
result = validator.validate()
|
||||
result.flavor = detectedFlavor
|
||||
result.detectionMethod = detectionMethod
|
||||
|
||||
# Ensure module info is populated
|
||||
result.moduleInfo.flavor = detectedFlavor
|
||||
result.moduleInfo.detectionMethod = detectionMethod
|
||||
|
||||
except Exception as e:
|
||||
result = newValidationResult()
|
||||
result.valid = false
|
||||
result.errors.add(newValidationError(
|
||||
esCritical,
|
||||
&"Validation framework error: {e.msg}",
|
||||
"E9999",
|
||||
newSourcePosition(0, 0, 0),
|
||||
newSourceRange(newSourcePosition(0, 0, 0), newSourcePosition(0, 0, 0))
|
||||
))
|
||||
|
||||
when defined(validatrixDebug):
|
||||
if options.debugMode:
|
||||
debugLeave("VALIDATRIX", &"Validation complete: valid={result.valid}, errors={result.errors.len}")
|
||||
result.debugOutput = getDebugOutput()
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Public API - Validate from file
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
proc validateFile*(
|
||||
filePath: string,
|
||||
flavor: LanguageFlavor = lfUnknown,
|
||||
options: ValidationOptions = newValidationOptions()
|
||||
): ValidationResult =
|
||||
## Validate a source file from disk.
|
||||
##
|
||||
## Auto-detects flavor from file extension (and content if needed).
|
||||
## Returns ValidationResult with errors, diagnostics, and metadata.
|
||||
##
|
||||
## Example:
|
||||
## let r = validateFile("src/main.nim")
|
||||
## let r = validateFile("template.html.jinja", flavor=lfJinja)
|
||||
|
||||
when defined(validatrixDebug):
|
||||
if options.debugMode:
|
||||
debugEnter("VALIDATRIX", &"validateFile: {filePath}")
|
||||
|
||||
try:
|
||||
if not fileExists(filePath):
|
||||
result = newValidationResult()
|
||||
result.valid = false
|
||||
result.errors.add(newValidationError(
|
||||
esError,
|
||||
&"File not found: {filePath}",
|
||||
"E0011",
|
||||
newSourcePosition(0, 0, 0),
|
||||
newSourceRange(newSourcePosition(0, 0, 0), newSourcePosition(0, 0, 0))
|
||||
))
|
||||
return result
|
||||
|
||||
let source = readFile(filePath)
|
||||
|
||||
# If flavor is not set, detect from extension
|
||||
var detectedFlavor = flavor
|
||||
if detectedFlavor == lfUnknown:
|
||||
let ext {.used.} = splitFile(filePath).ext.toLowerAscii()
|
||||
let (df, _, _) = detectFlavor(source, filePath, lfUnknown)
|
||||
detectedFlavor = df
|
||||
|
||||
result = validateSource(source, detectedFlavor, options, filePath)
|
||||
|
||||
when defined(validatrixDebug):
|
||||
if options.debugMode:
|
||||
debugLeave("VALIDATRIX", &"File validation complete: {result.errors.len} errors")
|
||||
|
||||
except Exception as e:
|
||||
result = newValidationResult()
|
||||
result.valid = false
|
||||
result.errors.add(newValidationError(
|
||||
esCritical, &"File validation error: {e.msg}", "E9999",
|
||||
newSourcePosition(0, 0, 0), newSourceRange(newSourcePosition(0, 0, 0), newSourcePosition(0, 0, 0))
|
||||
))
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Public API - Inspection / Diagnostics
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
proc inspectSource*(
|
||||
source: string,
|
||||
flavor: LanguageFlavor = lfUnknown,
|
||||
options: ValidationOptions = newValidationOptions(),
|
||||
sourcePath: string = ""
|
||||
): JsonNode =
|
||||
## Inspect source code and return structural diagnostics as JSON.
|
||||
##
|
||||
## Returns a JSON object with all variables, functions, classes,
|
||||
## imports, and other structural information found in the source.
|
||||
##
|
||||
## Example:
|
||||
## let diag = inspectSource("class Foo: pass", lfPython)
|
||||
## echo $diag # JSON with class/function info
|
||||
|
||||
let r = validateSource(source, flavor, options, sourcePath)
|
||||
if r.valid or r.errors.len > 0:
|
||||
r.moduleInfo.toJson()
|
||||
else:
|
||||
%*{"error": "Validation failed", "details": r.errors.mapIt(it.toJson())}
|
||||
|
||||
proc inspectFile*(
|
||||
filePath: string,
|
||||
flavor: LanguageFlavor = lfUnknown,
|
||||
options: ValidationOptions = newValidationOptions()
|
||||
): JsonNode =
|
||||
## Inspect a source file and return structural diagnostics as JSON.
|
||||
let r = validateFile(filePath, flavor, options)
|
||||
if r.valid or r.errors.len > 0:
|
||||
r.moduleInfo.toJson()
|
||||
else:
|
||||
%*{"error": "Validation failed", "details": r.errors.mapIt(it.toJson())}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Public API - Additional helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
proc validateSourceJson*(source: string, flavor: LanguageFlavor = lfUnknown): string =
|
||||
## Validate source and return result as JSON string.
|
||||
validateSource(source, flavor).toJson().pretty()
|
||||
|
||||
proc validateFileJson*(filePath: string, flavor: LanguageFlavor = lfUnknown): string =
|
||||
## Validate file and return result as JSON string.
|
||||
validateFile(filePath, flavor).toJson().pretty()
|
||||
|
||||
proc reportSource*(source: string, flavor: LanguageFlavor = lfUnknown): string =
|
||||
## Get a human-readable validation report for source code.
|
||||
let r = validateSource(source, flavor)
|
||||
formatReport(r, source)
|
||||
|
||||
proc reportFile*(filePath: string, flavor: LanguageFlavor = lfUnknown): string =
|
||||
## Get a human-readable validation report for a file.
|
||||
let source = if fileExists(filePath): readFile(filePath) else: ""
|
||||
let r = validateFile(filePath, flavor)
|
||||
formatReport(r, source)
|
||||
|
||||
proc detectFlavor*(source: string, filePath: string = ""): LanguageFlavor =
|
||||
## Detect language flavor of source code.
|
||||
## Convenience wrapper around core detector.
|
||||
let (flavor, _, _) = detector.detectFlavor(source, filePath)
|
||||
return flavor
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Version information
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
const
|
||||
validatrixVersion* = "0.1.0"
|
||||
validatrixDescription* = "Universal Source Code Validation Framework"
|
||||
|
||||
proc version*(): string =
|
||||
## Get the Validatrix version string.
|
||||
validatrixVersion
|
||||
|
||||
proc supportedFlavors*(): seq[string] =
|
||||
## Get list of all supported language flavors.
|
||||
result = @[]
|
||||
for key in validatorRegistry.keys:
|
||||
result.add(key)
|
||||
result.sort(system.cmp[string])
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Debug API
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
proc enableDebug*() =
|
||||
## Enable debug mode globally.
|
||||
debug.enableDebug()
|
||||
|
||||
proc disableDebug*() =
|
||||
## Disable debug mode globally.
|
||||
debug.disableDebug()
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# When compiled as standalone executable
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
when isMainModule:
|
||||
|
||||
proc printUsage() =
|
||||
echo &"Validatrix v{validatrixVersion} - {validatrixDescription}"
|
||||
echo ""
|
||||
echo "Usage:"
|
||||
echo " validatrix validate [--file=<path> | --source=<code>] [--flavor=<lang>]"
|
||||
echo " validatrix inspect [--file=<path> | --source=<code>] [--flavor=<lang>]"
|
||||
echo " validatrix detect [--file=<path> | --source=<code>]"
|
||||
echo " validatrix help"
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " --file=<path> Source file to process"
|
||||
echo " --source=<code> Source code string to process"
|
||||
echo " --flavor=<lang> Language flavor (auto-detect if omitted)"
|
||||
echo " --debug Enable debug mode"
|
||||
echo " --json Output as JSON (default for inspect)"
|
||||
echo ""
|
||||
echo "Supported flavors:"
|
||||
for f in supportedFlavors():
|
||||
echo &" - {f}"
|
||||
|
||||
proc main() =
|
||||
let args = commandLineParams()
|
||||
if args.len == 0:
|
||||
printUsage()
|
||||
return
|
||||
|
||||
let cmd = args[0].toLowerAscii()
|
||||
case cmd
|
||||
of "validate", "val":
|
||||
var filePath, source, flavor = ""
|
||||
var debugMode = false
|
||||
var i = 1
|
||||
while i < args.len:
|
||||
let a = args[i]
|
||||
if a.startsWith("--file="): filePath = a[7..^1]
|
||||
elif a.startsWith("--source="): source = a[9..^1]
|
||||
elif a.startsWith("--flavor="): flavor = a[9..^1]
|
||||
elif a == "--debug": debugMode = true
|
||||
i += 1
|
||||
|
||||
var opts = newValidationOptions()
|
||||
if debugMode: opts.debugMode = true
|
||||
|
||||
var fl = lfUnknown
|
||||
if flavor.len > 0:
|
||||
try: fl = parseEnum[LanguageFlavor](flavor)
|
||||
except: discard
|
||||
|
||||
var r: ValidationResult
|
||||
if filePath.len > 0:
|
||||
r = validateFile(filePath, fl, opts)
|
||||
elif source.len > 0:
|
||||
r = validateSource(source, fl, opts, "stdin")
|
||||
else:
|
||||
# Read from stdin
|
||||
let input = readAll(stdin).strip()
|
||||
r = validateSource(input, fl, opts, "stdin")
|
||||
|
||||
echo formatReport(r, r.moduleInfo.toJson().pretty())
|
||||
|
||||
of "inspect", "ins":
|
||||
var filePath, source, flavor = ""
|
||||
var debugMode = false
|
||||
var i = 1
|
||||
while i < args.len:
|
||||
let a = args[i]
|
||||
if a.startsWith("--file="): filePath = a[7..^1]
|
||||
elif a.startsWith("--source="): source = a[9..^1]
|
||||
elif a.startsWith("--flavor="): flavor = a[9..^1]
|
||||
elif a == "--debug": debugMode = true
|
||||
i += 1
|
||||
|
||||
var opts = newValidationOptions()
|
||||
if debugMode: opts.debugMode = true
|
||||
|
||||
var fl = lfUnknown
|
||||
if flavor.len > 0:
|
||||
try: fl = parseEnum[LanguageFlavor](flavor)
|
||||
except: discard
|
||||
|
||||
var jsonResult: JsonNode
|
||||
if filePath.len > 0:
|
||||
jsonResult = inspectFile(filePath, fl, opts)
|
||||
elif source.len > 0:
|
||||
jsonResult = inspectSource(source, fl, opts, "stdin")
|
||||
else:
|
||||
let input = readAll(stdin).strip()
|
||||
jsonResult = inspectSource(input, fl, opts, "stdin")
|
||||
|
||||
echo jsonResult.pretty()
|
||||
|
||||
of "detect":
|
||||
var filePath, source = ""
|
||||
var i = 1
|
||||
while i < args.len:
|
||||
let a = args[i]
|
||||
if a.startsWith("--file="): filePath = a[7..^1]
|
||||
elif a.startsWith("--source="): source = a[9..^1]
|
||||
i += 1
|
||||
|
||||
var src = source
|
||||
if filePath.len > 0 and fileExists(filePath):
|
||||
src = readFile(filePath)
|
||||
|
||||
let (dflavor, dmethod, dconfidence) = detector.detectFlavor(src, filePath)
|
||||
echo &"Detected flavor: {dflavor}"
|
||||
echo &" Method: {dmethod}"
|
||||
echo &" Confidence: {dconfidence * 100:.2f}%"
|
||||
|
||||
of "version", "--version", "-v":
|
||||
echo &"Validatrix v{validatrixVersion}"
|
||||
|
||||
of "help", "--help", "-h":
|
||||
printUsage()
|
||||
|
||||
else:
|
||||
echo &"Unknown command: {cmd}"
|
||||
printUsage()
|
||||
|
||||
main()
|
||||
@@ -0,0 +1,126 @@
|
||||
## Debug mode support for Validatrix.
|
||||
##
|
||||
## When compiled with -d:validatrixDebug, all validators and tokenizers
|
||||
## produce detailed debug output that helps an LLM or developer trace
|
||||
## exactly what the validator is doing.
|
||||
|
||||
{.warning[UnusedImport]:off.}
|
||||
{.warning[UnusedImport]:off.}
|
||||
import std/[strutils, strformat, times, tables]
|
||||
{.warning[UnusedImport]:on.}
|
||||
{.warning[UnusedImport]:on.}
|
||||
import ./types
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Debug state
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
var
|
||||
debugEnabled*: bool = false
|
||||
debugBuffer*: seq[string] = @[]
|
||||
debugIndent*: int = 0
|
||||
debugTimers*: Table[string, float] = initTable[string, float]()
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Enable / disable
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
proc enableDebug*() =
|
||||
## Enable debug output collection.
|
||||
when defined(validatrixDebug):
|
||||
debugEnabled = true
|
||||
debugBuffer = @[]
|
||||
debugIndent = 0
|
||||
debugTimers.clear()
|
||||
debugLog("DEBUG", "Debug mode enabled")
|
||||
|
||||
proc disableDebug*() =
|
||||
## Disable debug output collection.
|
||||
when defined(validatrixDebug):
|
||||
debugLog("DEBUG", "Debug mode disabled")
|
||||
debugEnabled = false
|
||||
|
||||
proc isDebugEnabled*(): bool =
|
||||
## Check if debug mode is active.
|
||||
when defined(validatrixDebug):
|
||||
result = debugEnabled
|
||||
else:
|
||||
result = false
|
||||
|
||||
proc clearDebug*() =
|
||||
## Clear the debug buffer.
|
||||
debugBuffer = @[]
|
||||
debugTimers.clear()
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Logging
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
proc debugLog*(category: string, message: string) =
|
||||
## Log a debug message with category.
|
||||
when defined(validatrixDebug):
|
||||
if debugEnabled:
|
||||
let indent = repeat(" "Indent)
|
||||
let timestamp = now().format("HH:mm:ss.fff")
|
||||
debugBuffer.add(&"[{timestamp}] {indent}[{category}] {message}")
|
||||
|
||||
proc debugEnter*(category: string, message: string) =
|
||||
## Log entry into a scope and increase indent.
|
||||
when defined(validatrixDebug):
|
||||
debugLog(category, "--> " & message)
|
||||
debugIndent += 1
|
||||
|
||||
proc debugLeave*(category: string, message: string = "") =
|
||||
## Decrease indent and log exit from a scope.
|
||||
when defined(validatrixDebug):
|
||||
if debugIndent > 0:
|
||||
debugIndent -= 1
|
||||
let indentMsg = if message.len > 0: "<-- " & message else: "<--"
|
||||
debugLog(category, indentMsg)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Timers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
proc debugTimerStart*(name: string) =
|
||||
## Start a named timer.
|
||||
when defined(validatrixDebug):
|
||||
debugTimers[name] = epochTime()
|
||||
|
||||
proc debugTimerStop*(name: string): float =
|
||||
## Stop a named timer and return elapsed ms.
|
||||
when defined(validatrixDebug):
|
||||
let start = debugTimers.getOrDefault(name, 0.0)
|
||||
if start > 0.0:
|
||||
result = (epochTime() - start) * 1000.0
|
||||
debugLog("TIMER", &"{name}: {result:.3f}ms")
|
||||
debugTimers.del(name)
|
||||
else:
|
||||
debugLog("TIMER", &"{name}: timer not started")
|
||||
|
||||
proc debugTimerStopAndLog*(name: string) =
|
||||
## Stop and log a named timer.
|
||||
discard debugTimerStop(name)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Output
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
proc getDebugOutput*(): string =
|
||||
## Get the complete debug output as a string.
|
||||
when defined(validatrixDebug):
|
||||
result = debugBuffer.join("\n")
|
||||
else:
|
||||
result = ""
|
||||
|
||||
proc debugToken*(token: Token) =
|
||||
## Log a token.
|
||||
when defined(validatrixDebug):
|
||||
debugLog("TOKEN", &"[{token.kind}] '{token.value}' @ L{token.position.line}:{token.position.column}")
|
||||
|
||||
proc debugError*(err: ValidationError) =
|
||||
## Log a validation error.
|
||||
when defined(validatrixDebug):
|
||||
debugLog("ERROR", &"[{err.code}] {err.severity}: {err.message} @ L{err.position.line}:{err.position.column}")
|
||||
if err.hint.len > 0:
|
||||
debugLog("HINT", err.hint)
|
||||
@@ -0,0 +1,413 @@
|
||||
## Language (flavor) auto-detection for Validatrix.
|
||||
##
|
||||
## Accurately detects programming language from:
|
||||
## 1. File extension
|
||||
## 2. Shebang line
|
||||
## 3. Content analysis (keywords, structure, patterns)
|
||||
|
||||
{.warning[UnusedImport]:off.}
|
||||
import std/[strutils, tables]
|
||||
{.warning[UnusedImport]:on.}
|
||||
import ./types
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Extension-to-flavor mapping
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
const
|
||||
extensionMap* = {
|
||||
".nim": lfNim,
|
||||
".nims": lfNim,
|
||||
".nimble": lfNim,
|
||||
".py": lfPython,
|
||||
".pyw": lfPython,
|
||||
".pyx": lfPython,
|
||||
".pxd": lfPython,
|
||||
".sh": lfBash,
|
||||
".bash": lfBash,
|
||||
".zsh": lfShell,
|
||||
".fish": lfShell,
|
||||
".js": lfJavaScript,
|
||||
".mjs": lfJavaScript,
|
||||
".cjs": lfJavaScript,
|
||||
".ts": lfTypeScript,
|
||||
".tsx": lfTypeScript,
|
||||
".php": lfPHP,
|
||||
".phtml": lfPHP,
|
||||
".html": lfHTML,
|
||||
".htm": lfHTML,
|
||||
".xhtml": lfHTML,
|
||||
".xml": lfXML,
|
||||
".svg": lfXML,
|
||||
".jinja": lfJinja,
|
||||
".jinja2": lfJinja,
|
||||
".j2": lfJinja,
|
||||
".json": lfJSON,
|
||||
".jsonc": lfJSON,
|
||||
".yaml": lfYAML,
|
||||
".yml": lfYAML,
|
||||
".toml": lfTOML,
|
||||
".css": lfCSS,
|
||||
".sql": lfSQL,
|
||||
".md": lfMarkdown,
|
||||
".markdown": lfMarkdown,
|
||||
".rb": lfRuby,
|
||||
".rs": lfRust,
|
||||
".go": lfGo,
|
||||
".lua": lfLua,
|
||||
".c": lfC,
|
||||
".h": lfC,
|
||||
".cpp": lfCpp,
|
||||
".cxx": lfCpp,
|
||||
".hpp": lfCpp,
|
||||
".cs": lfCSharp,
|
||||
".java": lfJava,
|
||||
".swift": lfSwift,
|
||||
".kt": lfKotlin,
|
||||
".kts": lfKotlin,
|
||||
"Dockerfile": lfDockerfile,
|
||||
"Makefile": lfMakefile,
|
||||
"makefile": lfMakefile
|
||||
}.toTable()
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Shebang detection
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
const
|
||||
shebangMap* = {
|
||||
"/bin/sh": lfBash,
|
||||
"/bin/bash": lfBash,
|
||||
"/usr/bin/sh": lfBash,
|
||||
"/usr/bin/bash": lfBash,
|
||||
"/usr/bin/env bash": lfBash,
|
||||
"/bin/zsh": lfShell,
|
||||
"/usr/bin/zsh": lfShell,
|
||||
"/usr/bin/env zsh": lfShell,
|
||||
"/usr/bin/python": lfPython,
|
||||
"/usr/bin/python3": lfPython,
|
||||
"/usr/bin/python2": lfPython,
|
||||
"/usr/bin/env python": lfPython,
|
||||
"/usr/bin/env python3": lfPython,
|
||||
"/usr/bin/env python2": lfPython,
|
||||
"/usr/bin/node": lfJavaScript,
|
||||
"/usr/bin/env node": lfJavaScript,
|
||||
"/usr/bin/lua": lfLua,
|
||||
"/usr/bin/env lua": lfLua,
|
||||
"/usr/bin/ruby": lfRuby,
|
||||
"/usr/bin/env ruby": lfRuby,
|
||||
"/usr/bin/swift": lfSwift,
|
||||
"/usr/bin/env swift": lfSwift
|
||||
}.toTable()
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Content-based detection
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
type
|
||||
ContentSignature* = object
|
||||
flavor: LanguageFlavor
|
||||
score: int
|
||||
description: string
|
||||
|
||||
proc detectByShebang*(source: string): tuple[flavor: LanguageFlavor, `method`: FlavorDetectionMethod] =
|
||||
## Detect language from shebang (#!) line.
|
||||
if source.len >= 2 and source[0] == '#' and source[1] == '!':
|
||||
# Extract the shebang line
|
||||
let newlinePos = source.find('\n')
|
||||
let shebangLine = if newlinePos >= 0: source[0..<newlinePos] else: source
|
||||
|
||||
# Try to find matching interpreter
|
||||
for interpreter, flavor in shebangMap.pairs():
|
||||
if shebangLine.contains(interpreter):
|
||||
return (flavor, fdmShebang)
|
||||
|
||||
# Generic detection from interpreter name
|
||||
if shebangLine.contains("perl"):
|
||||
return (lfUnknown, fdmShebang) # Not fully supported yet
|
||||
if shebangLine.contains("awk"):
|
||||
return (lfUnknown, fdmShebang)
|
||||
|
||||
return (lfUnknown, fdmUnknown)
|
||||
|
||||
proc detectByContent*(source: string, fileExt: string = ""): tuple[flavor: LanguageFlavor, `method`: FlavorDetectionMethod, confidence: float] =
|
||||
## Detect language from source content analysis.
|
||||
## Uses keyword scoring and structural patterns.
|
||||
## Returns flavor, detection method, and confidence (0.0-1.0).
|
||||
|
||||
if source.len == 0:
|
||||
return (lfUnknown, fdmUnknown, 0.0)
|
||||
|
||||
var scores: Table[string, int] = initTable[string, int]()
|
||||
let lines = source.splitLines()
|
||||
let totalLines = lines.len
|
||||
|
||||
# --- JSON detection ---
|
||||
let trimmed = source.strip()
|
||||
if trimmed.len > 0 and ((trimmed[0] == '{' and trimmed[^1] == '}') or
|
||||
(trimmed[0] == '[' and trimmed[^1] == ']')):
|
||||
try:
|
||||
let _ = parseJson(trimmed)
|
||||
return (lfJSON, fdmContent, 0.95)
|
||||
except:
|
||||
discard
|
||||
|
||||
# --- XML/HTML detection ---
|
||||
if trimmed.startsWith("<?xml") or trimmed.startsWith("<html") or trimmed.startsWith("<!DOCTYPE html"):
|
||||
let xmlFlavor = if trimmed.startsWith("<?xml"): lfXML else: lfHTML
|
||||
return (xmlFlavor, fdmContent, 0.95)
|
||||
|
||||
# --- TOML detection ---
|
||||
var tomlScore = 0
|
||||
for line in lines:
|
||||
let l = line.strip()
|
||||
if l.len == 0 or l[0] == '#': continue
|
||||
if l.contains('=') and not l.contains(':') and not l.contains("=>"):
|
||||
tomlScore += 1
|
||||
if l.startsWith("[") and l.contains("]"):
|
||||
tomlScore += 3
|
||||
if tomlScore > 5 and totalLines > 2:
|
||||
# Check for no YAML-like patterns
|
||||
var yamlLike = 0
|
||||
for line in lines:
|
||||
if line.strip().startsWith("- ") or line.contains(": "):
|
||||
yamlLike += 1
|
||||
if yamlLike < tomlScore div 2:
|
||||
return (lfTOML, fdmContent, 0.8)
|
||||
|
||||
# --- YAML detection ---
|
||||
var yamlScore = 0
|
||||
var yamlLines = 0
|
||||
for line in lines:
|
||||
if line.strip().len == 0 or line.strip()[0] == '#': continue
|
||||
if line.strip().startsWith("- ") or line.strip().startsWith("- ["):
|
||||
yamlScore += 2
|
||||
yamlLines += 1
|
||||
if line.contains(": ") and not line.strip().startsWith("#"):
|
||||
# Exclude obvious non-YAML
|
||||
if not line.contains(" (") and not line.contains(")") and not line.contains("=>"):
|
||||
yamlScore += 1
|
||||
yamlLines += 1
|
||||
if yamlScore > 3 and yamlLines.float > totalLines.float * 0.3:
|
||||
return (lfYAML, fdmContent, 0.75)
|
||||
|
||||
# --- PHP detection ---
|
||||
if source.contains("<?php") or source.contains("<?="):
|
||||
return (lfPHP, fdmContent, 0.95)
|
||||
|
||||
# --- Jinja detection ---
|
||||
if source.contains("{{") and source.contains("}}") and source.contains("{%") and source.contains("%}"):
|
||||
return (lfJinja, fdmContent, 0.9)
|
||||
|
||||
# --- Nim detection ---
|
||||
for kw in ["import ", "proc ", "func ", "method ", "type ", "var ", "let ", "const ", "discard "]:
|
||||
if source.contains(kw):
|
||||
scores.mgetOrPut("nim", 0).inc(2)
|
||||
if source.contains("echo ") or source.contains("stdout."):
|
||||
scores.mgetOrPut("nim", 0).inc(1)
|
||||
if source.contains("[") and source.contains("]") and source.contains("{") and source.contains("}"):
|
||||
scores.mgetOrPut("nim", 0).inc(0) # neutral
|
||||
|
||||
# --- Python detection ---
|
||||
for kw in ["def ", "class ", "import ", "from ", "return ", "if __name__", "elif ", "except "]:
|
||||
if source.contains(kw):
|
||||
scores.mgetOrPut("python", 0).inc(2)
|
||||
if source.contains("print(") or source.contains("self.") or source.contains("lambda "):
|
||||
scores.mgetOrPut("python", 0).inc(1)
|
||||
if source.contains(":") and source.contains(" ") and not source.contains("{"):
|
||||
# Indentation-based blocks
|
||||
scores.mgetOrPut("python", 0).inc(1)
|
||||
|
||||
# --- JavaScript/TypeScript detection ---
|
||||
for kw in ["function ", "const ", "let ", "var ", "class ", "=>", "export ", "import "]:
|
||||
if source.contains(kw):
|
||||
scores.mgetOrPut("javascript", 0).inc(2)
|
||||
if source.contains("console.") or source.contains("document.") or source.contains("window."):
|
||||
scores.mgetOrPut("javascript", 0).inc(1)
|
||||
if source.contains(": string") or source.contains(": number") or source.contains(": boolean") or source.contains("interface "):
|
||||
scores.mgetOrPut("typescript", 0).inc(3)
|
||||
|
||||
# --- Bash detection ---
|
||||
if source.contains("if [") or source.contains("fi") or source.contains("then") or source.contains("esac"):
|
||||
scores.mgetOrPut("bash", 0).inc(2)
|
||||
for kw in ["function ", "local ", "export ", "echo ", "$", "do", "done"]:
|
||||
if source.contains(kw):
|
||||
scores.mgetOrPut("bash", 0).inc(1)
|
||||
if source.count("$") > 3:
|
||||
scores.mgetOrPut("bash", 0).inc(2)
|
||||
|
||||
# --- Go detection ---
|
||||
if source.contains("package ") and source.contains("func ") and source.contains("import ("):
|
||||
scores.mgetOrPut("go", 0).inc(4)
|
||||
|
||||
# --- Rust detection ---
|
||||
if (source.contains("fn ") or source.contains("let mut ")) and source.contains("->"):
|
||||
scores.mgetOrPut("rust", 0).inc(3)
|
||||
if source.contains("println!") or source.contains("unwrap()"):
|
||||
scores.mgetOrPut("rust", 0).inc(1)
|
||||
|
||||
# --- Ruby detection ---
|
||||
if source.contains("def ") and source.contains("end") and source.contains("require"):
|
||||
scores.mgetOrPut("ruby", 0).inc(2)
|
||||
|
||||
# --- Lua detection ---
|
||||
if source.contains("function ") and source.contains("end") and source.contains("local ") and (source.contains("print(") or source.contains("table.")):
|
||||
scores.mgetOrPut("lua", 0).inc(2)
|
||||
|
||||
# --- C/C++ detection ---
|
||||
if source.contains("#include") and (source.contains("int main") or source.contains("void ")):
|
||||
if source.contains("class ") or source.contains("template "):
|
||||
scores.mgetOrPut("cpp", 0).inc(3)
|
||||
else:
|
||||
scores.mgetOrPut("c", 0).inc(3)
|
||||
if source.contains("printf(") or source.contains("scanf("):
|
||||
scores.mgetOrPut("c", 0).inc(2)
|
||||
|
||||
# --- C# detection ---
|
||||
if source.contains("using System") or source.contains("namespace ") or source.contains("class ") and source.contains("static void Main"):
|
||||
scores.mgetOrPut("csharp", 0).inc(4)
|
||||
|
||||
# --- Java detection ---
|
||||
if source.contains("public class ") or source.contains("public static void main") or source.contains("System.out."):
|
||||
scores.mgetOrPut("java", 0).inc(4)
|
||||
|
||||
# --- Swift detection ---
|
||||
if source.contains("import Swift") or source.contains("func ") and source.contains("var ") and source.contains("let ") and not source.contains(";") and source.contains("print("):
|
||||
scores.mgetOrPut("swift", 0).inc(2)
|
||||
|
||||
# --- Kotlin detection ---
|
||||
if source.contains("fun ") and source.contains("val ") and source.contains("var ") and source.contains("package ") and not source.contains("System.out"):
|
||||
scores.mgetOrPut("kotlin", 0).inc(3)
|
||||
|
||||
# --- SQL detection ---
|
||||
var sqlScore = 0
|
||||
let sourceUpper = source.toUpperAscii()
|
||||
for kw in ["SELECT ", "FROM ", "WHERE ", "INSERT ", "CREATE TABLE ", "ALTER ", "DROP "]:
|
||||
if sourceUpper.contains(kw):
|
||||
sqlScore += 2
|
||||
if sqlScore > 5 and sourceUpper.contains("JOIN"):
|
||||
sqlScore += 2
|
||||
if sqlScore > 5:
|
||||
return (lfSQL, fdmContent, 0.8)
|
||||
|
||||
# --- Dockerfile detection ---
|
||||
for kw in ["FROM ", "RUN ", "CMD ", "ENV ", "COPY ", "WORKDIR "]:
|
||||
if source.contains(kw):
|
||||
scores.mgetOrPut("dockerfile", 0).inc(2)
|
||||
|
||||
# --- Makefile detection ---
|
||||
for kw in [".PHONY:", "CC=", "CFLAGS=", "$@", "$<", "$^"]:
|
||||
if source.contains(kw):
|
||||
scores.mgetOrPut("makefile", 0).inc(2)
|
||||
|
||||
# --- Markdown detection ---
|
||||
if source.contains("# ") and source.contains("```") and not source.contains("function"):
|
||||
scores.mgetOrPut("markdown", 0).inc(2)
|
||||
|
||||
# --- CSS detection ---
|
||||
if source.contains("{") and source.contains("}") and source.contains(":") and not source.contains("function") and not source.contains(";"):
|
||||
var cssScore = 0
|
||||
for sel in [".", "#", "@media", "@import", "px", "em", "rem"]:
|
||||
if source.contains(sel): cssScore += 1
|
||||
if cssScore > 3 and source.contains("{"):
|
||||
scores.mgetOrPut("css", 0).inc(cssScore)
|
||||
|
||||
# Find highest scoring flavor
|
||||
var bestFlavor = "unknown"
|
||||
var bestScore = 0
|
||||
for fl, score in scores.pairs():
|
||||
if score > bestScore:
|
||||
bestScore = score
|
||||
bestFlavor = fl
|
||||
|
||||
# Convert bestFlavor to enum
|
||||
case bestFlavor
|
||||
of "nim": return (lfNim, fdmContent, min(bestScore.float / 20.0, 0.9))
|
||||
of "python": return (lfPython, fdmContent, min(bestScore.float / 20.0, 0.9))
|
||||
of "bash": return (lfBash, fdmContent, min(bestScore.float / 16.0, 0.85))
|
||||
of "javascript": return (lfJavaScript, fdmContent, min(bestScore.float / 16.0, 0.85))
|
||||
of "typescript": return (lfTypeScript, fdmContent, min(bestScore.float / 16.0, 0.85))
|
||||
of "go": return (lfGo, fdmContent, 0.8)
|
||||
of "rust": return (lfRust, fdmContent, 0.8)
|
||||
of "ruby": return (lfRuby, fdmContent, 0.7)
|
||||
of "lua": return (lfLua, fdmContent, 0.7)
|
||||
of "c": return (lfC, fdmContent, 0.7)
|
||||
of "cpp": return (lfCpp, fdmContent, 0.7)
|
||||
of "csharp": return (lfCSharp, fdmContent, 0.8)
|
||||
of "java": return (lfJava, fdmContent, 0.8)
|
||||
of "swift": return (lfSwift, fdmContent, 0.7)
|
||||
of "kotlin": return (lfKotlin, fdmContent, 0.7)
|
||||
of "dockerfile": return (lfDockerfile, fdmContent, 0.7)
|
||||
of "makefile": return (lfMakefile, fdmContent, 0.7)
|
||||
of "markdown": return (lfMarkdown, fdmContent, 0.6)
|
||||
of "css": return (lfCSS, fdmContent, 0.7)
|
||||
of "sql": return (lfSQL, fdmContent, 0.8)
|
||||
else:
|
||||
return (lfUnknown, fdmUnknown, 0.0)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main detection API
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
proc detectFlavor*(
|
||||
source: string,
|
||||
filePath: string = "",
|
||||
explicitFlavor: LanguageFlavor = lfUnknown
|
||||
): tuple[flavor: LanguageFlavor, `method`: FlavorDetectionMethod, confidence: float] =
|
||||
## Detect language flavor from source and/or filepath.
|
||||
## Priority: explicit > extension > shebang > content analysis.
|
||||
##
|
||||
## When explicitFlavor is provided and not lfUnknown, returns that.
|
||||
## Otherwise, uses the best available detection method.
|
||||
|
||||
when defined(validatrixDebug):
|
||||
if isDebugEnabled():
|
||||
debugEnter("DETECTOR", "Detecting language flavor")
|
||||
|
||||
try:
|
||||
# 1. Explicit flavor
|
||||
if explicitFlavor != lfUnknown:
|
||||
when defined(validatrixDebug):
|
||||
if isDebugEnabled():
|
||||
debugLog("DETECTOR", &"Using explicit flavor: {explicitFlavor}")
|
||||
return (explicitFlavor, fdmExplicit, 1.0)
|
||||
|
||||
# 2. Extension-based detection
|
||||
if filePath.len > 0:
|
||||
let ext = filePath.toLowerAscii()
|
||||
for extension, flavor in extensionMap.pairs():
|
||||
if ext.endsWith(extension) or ext == extension:
|
||||
when defined(validatrixDebug):
|
||||
if isDebugEnabled():
|
||||
debugLog("DETECTOR", &"Detected by extension '{extension}': {flavor}")
|
||||
return (flavor, fdmExtension, 0.9)
|
||||
|
||||
# 3. Shebang detection
|
||||
let (shebangFlavor, shebangMethod) = detectByShebang(source)
|
||||
if shebangFlavor != lfUnknown:
|
||||
when defined(validatrixDebug):
|
||||
if isDebugEnabled():
|
||||
debugLog("DETECTOR", &"Detected by shebang: {shebangFlavor}")
|
||||
return (shebangFlavor, shebangMethod, 0.95)
|
||||
|
||||
# 4. Content-based detection
|
||||
if source.len > 0:
|
||||
let (contentFlavor, contentMethod, confidence) = detectByContent(source, filePath)
|
||||
if contentFlavor != lfUnknown:
|
||||
when defined(validatrixDebug):
|
||||
if isDebugEnabled():
|
||||
debugLog("DETECTOR", &"Detected by content: {contentFlavor} (confidence: {confidence})")
|
||||
return (contentFlavor, contentMethod, confidence)
|
||||
|
||||
except Exception as e:
|
||||
discard e
|
||||
when defined(validatrixDebug):
|
||||
if isDebugEnabled():
|
||||
debugLog("DETECTOR", &"Detection error: {e.msg}")
|
||||
|
||||
when defined(validatrixDebug):
|
||||
if isDebugEnabled():
|
||||
debugLog("DETECTOR", "Could not determine flavor")
|
||||
debugLeave("DETECTOR")
|
||||
|
||||
return (lfUnknown, fdmUnknown, 0.0)
|
||||
@@ -0,0 +1,15 @@
|
||||
## Validator registration — called at module load time.
|
||||
##
|
||||
## Each language validator registers itself here.
|
||||
## This file is imported by the main validatrix.nim module.
|
||||
|
||||
import ./types, ./validatorbase
|
||||
|
||||
# Forward declarations from language modules
|
||||
# Each validator module has an init() proc that registers itself
|
||||
|
||||
proc initAllValidators*() =
|
||||
## Initialize all validators by registering them.
|
||||
## This is called automatically when the module loads.
|
||||
# Language validators register themselves on import
|
||||
discard
|
||||
@@ -0,0 +1,333 @@
|
||||
## Abstract base tokenizer for Validatrix.
|
||||
##
|
||||
## All language-specific tokenizers inherit from this base class.
|
||||
## Provides position tracking, error recording, and common tokenization helpers.
|
||||
|
||||
{.warning[UnusedImport]:off.}
|
||||
import std/[strformat]
|
||||
{.warning[UnusedImport]:on.}
|
||||
import ./types
|
||||
|
||||
type
|
||||
TokenizerBase* = ref object of RootObj
|
||||
## Abstract base tokenizer.
|
||||
source*: string
|
||||
sourceLen*: int
|
||||
pos*: int # Current byte offset
|
||||
line*: int # Current line (1-indexed)
|
||||
column*: int # Current column (1-indexed)
|
||||
tokens*: seq[Token]
|
||||
errors*: seq[ValidationError]
|
||||
options*: ValidationOptions
|
||||
bracketStack*: seq[(char, SourcePosition)] # For bracket matching
|
||||
eof*: bool
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Construction
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
proc newTokenizerBase*(
|
||||
source: string,
|
||||
options: ValidationOptions = newValidationOptions()
|
||||
): TokenizerBase =
|
||||
## Create a new tokenizer base instance.
|
||||
result = TokenizerBase(
|
||||
source: source,
|
||||
sourceLen: source.len,
|
||||
pos: 0,
|
||||
line: 1,
|
||||
column: 1,
|
||||
tokens: @[],
|
||||
errors: @[],
|
||||
options: options,
|
||||
bracketStack: @[],
|
||||
eof: false
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Position helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
proc currentPos*(self: TokenizerBase): SourcePosition =
|
||||
## Get current position.
|
||||
SourcePosition(line: self.line, column: self.column, offset: self.pos)
|
||||
|
||||
proc posAt*(self: TokenizerBase, offset: int): SourcePosition =
|
||||
## Get position at a given offset by scanning from current pos.
|
||||
## Note: for accuracy, use calculatePosition for arbitrary offsets.
|
||||
self.currentPos()
|
||||
|
||||
proc rangeBetween*(self: TokenizerBase, startPos: SourcePosition): SourceRange =
|
||||
## Create a range from startPos to current position.
|
||||
SourceRange(startPos: startPos, endPos: self.currentPos())
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Character inspection
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
proc peek*(self: TokenizerBase, ahead: int = 0): char =
|
||||
## Look at character ahead without consuming.
|
||||
let idx = self.pos + ahead
|
||||
if idx < self.sourceLen:
|
||||
result = self.source[idx]
|
||||
else:
|
||||
result = '\0'
|
||||
|
||||
proc peekString*(self: TokenizerBase, length: int): string =
|
||||
## Look ahead multiple characters without consuming.
|
||||
let endIdx = min(self.pos + length, self.sourceLen)
|
||||
if endIdx > self.pos:
|
||||
result = self.source[self.pos ..< endIdx]
|
||||
else:
|
||||
result = ""
|
||||
|
||||
proc hasMore*(self: TokenizerBase): bool =
|
||||
## Check if there are more characters to consume.
|
||||
self.pos < self.sourceLen
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Consumption
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
proc advance*(self: TokenizerBase): char =
|
||||
## Consume one character and advance position.
|
||||
if self.pos < self.sourceLen:
|
||||
result = self.source[self.pos]
|
||||
self.pos += 1
|
||||
if result == '\n':
|
||||
self.line += 1
|
||||
self.column = 1
|
||||
else:
|
||||
self.column += 1
|
||||
else:
|
||||
self.eof = true
|
||||
result = '\0'
|
||||
|
||||
proc skipWhitespace*(self: TokenizerBase, skipNewlines: bool = true) =
|
||||
## Skip whitespace characters.
|
||||
while self.hasMore():
|
||||
let c = self.peek()
|
||||
if c == ' ' or c == '\t':
|
||||
discard self.advance()
|
||||
elif skipNewlines and c == '\n':
|
||||
discard self.advance()
|
||||
else:
|
||||
break
|
||||
|
||||
proc expect*(self: TokenizerBase, expected: char): bool =
|
||||
## Expect and consume a specific character. Returns false if not found.
|
||||
if self.hasMore() and self.peek() == expected:
|
||||
discard self.advance()
|
||||
return true
|
||||
return false
|
||||
|
||||
proc expectString*(self: TokenizerBase, expected: string): bool =
|
||||
## Expect and consume a specific string. Returns false if not found.
|
||||
if self.peekString(expected.len) == expected:
|
||||
for _ in expected:
|
||||
discard self.advance()
|
||||
return true
|
||||
return false
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Token emission
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
proc emitToken*(self: TokenizerBase, kind: TokenKind, value: string, startPos: SourcePosition) =
|
||||
## Emit a token and add it to the token list.
|
||||
let endPos = self.currentPos()
|
||||
let range = SourceRange(startPos: startPos, endPos: endPos)
|
||||
let token = Token(
|
||||
kind: kind,
|
||||
value: value,
|
||||
position: startPos,
|
||||
range: range
|
||||
)
|
||||
self.tokens.add(token)
|
||||
when defined(validatrixDebug):
|
||||
if self.options.debugMode:
|
||||
debugToken(token)
|
||||
|
||||
proc emitBracketToken*(self: TokenizerBase, kind: TokenKind, value: string, startPos: SourcePosition) =
|
||||
## Emit a bracket token and track it on the bracket stack.
|
||||
self.emitToken(kind, value, startPos)
|
||||
if kind in {kOpenParen, kOpenBracket, kOpenBrace, kAngleOpen}:
|
||||
self.bracketStack.add((value[0], startPos))
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Error recording
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
proc recordError*(
|
||||
self: TokenizerBase,
|
||||
severity: ErrorSeverity,
|
||||
message: string,
|
||||
code: string,
|
||||
position: SourcePosition,
|
||||
hint: string = ""
|
||||
) =
|
||||
## Record a validation error.
|
||||
let range = SourceRange(startPos: position, endPos: position)
|
||||
let hintVal = hint
|
||||
let err = newValidationError(severity, message, code, position, range, hint = hintVal)
|
||||
self.errors.add(err)
|
||||
when defined(validatrixDebug):
|
||||
if self.options.debugMode:
|
||||
debugError(err)
|
||||
|
||||
proc recordErrorAtCurrent*(
|
||||
self: TokenizerBase,
|
||||
severity: ErrorSeverity,
|
||||
message: string,
|
||||
code: string,
|
||||
hint: string = ""
|
||||
) =
|
||||
## Record error at current position.
|
||||
self.recordError(severity, message, code, self.currentPos(), hint)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Bracket validation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
proc validateBrackets*(self: TokenizerBase): seq[ValidationError] =
|
||||
## Check the bracket stack for unclosed brackets at end of file.
|
||||
result = @[]
|
||||
for (ch, pos) in self.bracketStack:
|
||||
let pairStr = case ch:
|
||||
of '(': ")"
|
||||
of '[': "]"
|
||||
of '{': "}"
|
||||
of '<': ">"
|
||||
else: $ch
|
||||
let hintStr = &"Add a closing '{pairStr}' before this point"
|
||||
let err = newValidationError(
|
||||
esError,
|
||||
&"Unclosed '{ch}' — expected matching '{pairStr}'",
|
||||
ErrUnclosedBracket,
|
||||
pos,
|
||||
SourceRange(startPos: pos, endPos: pos),
|
||||
hint = hintStr
|
||||
)
|
||||
result.add(err)
|
||||
when defined(validatrixDebug):
|
||||
if self.options.debugMode:
|
||||
debugError(err)
|
||||
self.errors.add(result)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Abstract interface — subclasses must implement
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
method tokenize*(self: TokenizerBase) {.base, gcsafe.} =
|
||||
## Tokenize the entire source. Must be overridden.
|
||||
raise newException(ValueError, "tokenize() not implemented — subclass must override")
|
||||
|
||||
proc getAllTokens*(self: TokenizerBase): seq[Token] =
|
||||
## Get all tokens after tokenization.
|
||||
self.tokens
|
||||
|
||||
proc getErrors*(self: TokenizerBase): seq[ValidationError] =
|
||||
## Get all errors recorded during tokenization.
|
||||
self.errors
|
||||
|
||||
proc hasErrors*(self: TokenizerBase): bool =
|
||||
## Check if any errors were recorded.
|
||||
self.errors.len > 0
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Utility
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
proc isAlpha*(c: char): bool =
|
||||
## Check if character is alphabetic.
|
||||
c in 'a'..'z' or c in 'A'..'Z' or c == '_'
|
||||
|
||||
proc isAlphaNum*(c: char): bool =
|
||||
## Check if character is alphanumeric.
|
||||
isAlpha(c) or c in '0'..'9'
|
||||
|
||||
proc isHexDigit*(c: char): bool =
|
||||
## Check if character is a hex digit.
|
||||
c in '0'..'9' or c in 'a'..'f' or c in 'A'..'F'
|
||||
|
||||
proc isOctalDigit*(c: char): bool =
|
||||
## Check if character is an octal digit.
|
||||
c in '0'..'7'
|
||||
|
||||
proc isBinaryDigit*(c: char): bool =
|
||||
## Check if character is a binary digit.
|
||||
c == '0' or c == '1'
|
||||
|
||||
proc isWhitespace*(c: char, includeNewlines: bool = true): bool =
|
||||
## Check if character is whitespace.
|
||||
result = c == ' ' or c == '\t'
|
||||
if includeNewlines:
|
||||
result = result or c == '\n' or c == '\r'
|
||||
|
||||
proc isQuote*(c: char): bool =
|
||||
## Check if character is a string quote.
|
||||
c == '"' or c == '\''
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# String parsing helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
proc parseString*(self: TokenizerBase, quote: char): string =
|
||||
## Parse a string literal until matching unescaped quote.
|
||||
## Returns the string content (without quotes).
|
||||
## Records errors for unclosed strings.
|
||||
let startPos = self.currentPos()
|
||||
result = ""
|
||||
while self.hasMore():
|
||||
let c = self.advance()
|
||||
if c == '\\':
|
||||
# Escape sequence
|
||||
if self.hasMore():
|
||||
result.add(c)
|
||||
result.add(self.advance())
|
||||
elif c == quote:
|
||||
# End of string
|
||||
return
|
||||
else:
|
||||
result.add(c)
|
||||
# Unclosed string
|
||||
self.recordError(
|
||||
esError,
|
||||
&"Unclosed string literal starting with '{quote}'",
|
||||
ErrUnclosedString,
|
||||
startPos,
|
||||
hint = "Add a closing quote before end of file"
|
||||
)
|
||||
|
||||
proc parseLineComment*(self: TokenizerBase): string =
|
||||
## Parse from // or # to end of line.
|
||||
result = ""
|
||||
while self.hasMore() and self.peek() != '\n':
|
||||
result.add(self.advance())
|
||||
|
||||
proc parseBlockComment*(self: TokenizerBase, openToken, closeToken: string): string =
|
||||
## Parse a block comment delimited by openToken/closeToken.
|
||||
let startPos = self.currentPos()
|
||||
result = ""
|
||||
var depth = 1
|
||||
while self.hasMore() and depth > 0:
|
||||
if self.peekString(openToken.len) == openToken:
|
||||
depth += 1
|
||||
for _ in openToken: discard self.advance()
|
||||
elif self.peekString(closeToken.len) == closeToken:
|
||||
depth -= 1
|
||||
if depth > 0:
|
||||
for _ in closeToken: discard self.advance()
|
||||
else:
|
||||
for _ in closeToken: discard self.advance()
|
||||
return
|
||||
else:
|
||||
result.add(self.advance())
|
||||
# Unclosed comment
|
||||
self.recordError(
|
||||
esError,
|
||||
&"Unclosed block comment starting with '{openToken}'",
|
||||
ErrUnclosedComment,
|
||||
startPos,
|
||||
hint = &"Add a closing '{closeToken}'"
|
||||
)
|
||||
@@ -0,0 +1,456 @@
|
||||
## Validatrix Core Types
|
||||
##
|
||||
## Defines all foundational types used throughout the validation framework.
|
||||
## Every validator, tokenizer, and reporter shares these types for consistency.
|
||||
|
||||
import std/[json, strutils, tables, times, sequtils]
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Language Flavor -- auto-detected or explicitly set
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
type
|
||||
LanguageFlavor* {.pure.} = enum
|
||||
## Programming language / file flavor enumeration.
|
||||
## Used for both explicit specification and auto-detection results.
|
||||
lfUnknown = "unknown"
|
||||
lfNim = "nim"
|
||||
lfPython = "python"
|
||||
lfBash = "bash"
|
||||
lfShell = "shell" # generic POSIX shell
|
||||
lfJavaScript = "javascript"
|
||||
lfTypeScript = "typescript"
|
||||
lfPHP = "php"
|
||||
lfHTML = "html"
|
||||
lfXML = "xml"
|
||||
lfJinja = "jinja" # Jinja2 templates (mixed Python/HTML)
|
||||
lfJSON = "json"
|
||||
lfYAML = "yaml"
|
||||
lfTOML = "toml"
|
||||
lfCSS = "css"
|
||||
lfSQL = "sql"
|
||||
lfMarkdown = "markdown"
|
||||
lfDockerfile = "dockerfile"
|
||||
lfMakefile = "makefile"
|
||||
lfRuby = "ruby"
|
||||
lfRust = "rust"
|
||||
lfGo = "go"
|
||||
lfLua = "lua"
|
||||
lfC = "c"
|
||||
lfCpp = "cpp"
|
||||
lfCSharp = "csharp"
|
||||
lfJava = "java"
|
||||
lfSwift = "swift"
|
||||
lfKotlin = "kotlin"
|
||||
|
||||
FlavorDetectionMethod* {.pure.} = enum
|
||||
## How the flavor was determined.
|
||||
fdmExplicit = "explicit" # User-provided
|
||||
fdmExtension = "by_extension" # From file extension
|
||||
fdmShebang = "by_shebang" # From #! line
|
||||
fdmContent = "by_content" # From content analysis
|
||||
fdmUnknown = "unknown"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Source position tracking
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
SourcePosition* = object
|
||||
## Exact position in source code.
|
||||
line*: int # 1-indexed line
|
||||
column*: int # 1-indexed column
|
||||
offset*: int # 0-indexed byte offset from start
|
||||
|
||||
SourceRange* = object
|
||||
## A range between two positions.
|
||||
startPos*: SourcePosition
|
||||
endPos*: SourcePosition
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tokens
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
TokenKind* {.pure.} = enum
|
||||
## Classification of a single token.
|
||||
tkWhitespace
|
||||
tkNewline
|
||||
tkComment # Line or block comment
|
||||
tkDocComment # Documentation comment (##, ///, """)
|
||||
tkString # "..." or '...'
|
||||
tkRawString # r"..." or similar
|
||||
tkMultilineString # """...""" or similar
|
||||
tkNumber # integer, float, hex, octal, binary
|
||||
tkIdentifier # variable, function, type names
|
||||
tkKeyword # Language-specific keyword
|
||||
tkOperator # +, -, *, /, etc.
|
||||
tkAssignment # =, :=, +=, etc.
|
||||
tkPunctuation # . , ; : etc.
|
||||
kOpenParen # (
|
||||
kCloseParen # )
|
||||
kOpenBracket # [
|
||||
kCloseBracket # ]
|
||||
kOpenBrace # {
|
||||
kCloseBrace # }
|
||||
kAngleOpen # <
|
||||
kAngleClose # >
|
||||
tkInterpolation # String interpolation (e.g., {var} in f-strings)
|
||||
tkDirective # Preprocessor / compiler directive
|
||||
tkTemplateTag # Template language tag ({{ }}, {% %}, etc.)
|
||||
tkSpecial # Special token (language-specific)
|
||||
tkError # Malformed / unrecognizable
|
||||
tkEndOfFile # End of input
|
||||
|
||||
Token* = object
|
||||
## A single lexical token with position and value.
|
||||
kind*: TokenKind
|
||||
value*: string
|
||||
position*: SourcePosition
|
||||
range*: SourceRange
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Validation errors & severity
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
ErrorSeverity* {.pure.} = enum
|
||||
## How severe a validation finding is.
|
||||
esInfo = "info"
|
||||
esWarning = "warning"
|
||||
esError = "error"
|
||||
esCritical = "critical"
|
||||
|
||||
ValidationError* = object
|
||||
## A single validation finding.
|
||||
severity*: ErrorSeverity
|
||||
message*: string
|
||||
code*: string # Machine-readable error code, e.g. "E0001"
|
||||
position*: SourcePosition
|
||||
range*: SourceRange
|
||||
context*: string # Surrounding source context (optional)
|
||||
hint*: string # Suggestion for fixing (optional)
|
||||
|
||||
ValidationErrors* = seq[ValidationError]
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Structural diagnostics
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
VariableInfo* = object
|
||||
## Information about a declared variable or constant.
|
||||
name*: string
|
||||
kind*: string # "var", "let", "const", "global", "local"
|
||||
typeAnnotation*: string # Type if annotated
|
||||
defaultValue*: string # Default value as string
|
||||
position*: SourcePosition
|
||||
mutable*: bool
|
||||
exported*: bool
|
||||
|
||||
FunctionInfo* = object
|
||||
## Information about a function/method/procedure.
|
||||
name*: string
|
||||
kind*: string # "function", "method", "proc", "func", "lambda"
|
||||
parameters*: seq[ParameterInfo]
|
||||
returnType*: string
|
||||
isAsync*: bool
|
||||
isExported*: bool
|
||||
isGenerator*: bool
|
||||
visibility*: string # "public", "private", "protected"
|
||||
position*: SourcePosition
|
||||
docComment*: string
|
||||
|
||||
ParameterInfo* = object
|
||||
## A single function parameter.
|
||||
name*: string
|
||||
typeAnnotation*: string
|
||||
defaultValue*: string
|
||||
isOptional*: bool
|
||||
isVarargs*: bool
|
||||
|
||||
ClassInfo* = object
|
||||
## Information about a class/type/object definition.
|
||||
name*: string
|
||||
kind*: string # "class", "object", "struct", "interface"
|
||||
baseTypes*: seq[string] # Inheritance / mixins
|
||||
fields*: seq[VariableInfo]
|
||||
methods*: seq[FunctionInfo]
|
||||
generics*: seq[string]
|
||||
visibility*: string
|
||||
position*: SourcePosition
|
||||
docComment*: string
|
||||
|
||||
ImportInfo* = object
|
||||
## Information about an import/include/use statement.
|
||||
module*: string
|
||||
symbols*: seq[string] # Specific imported symbols (if any)
|
||||
isRelative*: bool
|
||||
alias*: string # Import alias (as/rename)
|
||||
position*: SourcePosition
|
||||
|
||||
ModuleInfo* = object
|
||||
## Top-level structural information about the validated source.
|
||||
flavor*: LanguageFlavor
|
||||
detectionMethod*: FlavorDetectionMethod
|
||||
imports*: seq[ImportInfo]
|
||||
variables*: seq[VariableInfo]
|
||||
functions*: seq[FunctionInfo]
|
||||
classes*: seq[ClassInfo]
|
||||
linesOfCode*: int
|
||||
totalTokens*: int
|
||||
comments*: int
|
||||
debugInfo*: JsonNode # Additional language-specific info
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Validation result
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
ValidationResult* = object
|
||||
## Complete result of validating a source file or string.
|
||||
valid*: bool
|
||||
flavor*: LanguageFlavor
|
||||
detectionMethod*: FlavorDetectionMethod
|
||||
errors*: ValidationErrors
|
||||
warnings*: int
|
||||
infos*: int
|
||||
moduleInfo*: ModuleInfo
|
||||
durationMs*: float
|
||||
debugOutput*: string # Debug mode output (empty when debug off)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Validation options
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
ValidationOptions* = object
|
||||
## Options controlling validation behaviour.
|
||||
flavor*: LanguageFlavor # lfUnknown = auto-detect
|
||||
debugMode*: bool
|
||||
strictMode*: bool # Warnings become errors
|
||||
maxErrors*: int # Stop after N errors (0 = unlimited)
|
||||
includeContext*: bool # Include surrounding source in errors
|
||||
includeHints*: bool # Include fix suggestions
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Converter procs for JSON output
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
proc toJson*(pos: SourcePosition): JsonNode =
|
||||
## Convert SourcePosition to JSON.
|
||||
result = %*{
|
||||
"line": pos.line,
|
||||
"column": pos.column,
|
||||
"offset": pos.offset
|
||||
}
|
||||
|
||||
proc toJson*(range: SourceRange): JsonNode =
|
||||
## Convert SourceRange to JSON.
|
||||
result = %*{
|
||||
"start": range.startPos.toJson(),
|
||||
"end": range.endPos.toJson()
|
||||
}
|
||||
|
||||
proc toJson*(err: ValidationError): JsonNode =
|
||||
## Convert ValidationError to JSON.
|
||||
result = %*{
|
||||
"severity": $err.severity,
|
||||
"message": err.message,
|
||||
"code": err.code,
|
||||
"position": err.position.toJson(),
|
||||
"range": err.range.toJson(),
|
||||
"context": err.context,
|
||||
"hint": err.hint
|
||||
}
|
||||
|
||||
proc toJson*(vi: VariableInfo): JsonNode =
|
||||
## Convert VariableInfo to JSON.
|
||||
result = %*{
|
||||
"name": vi.name,
|
||||
"kind": vi.kind,
|
||||
"typeAnnotation": vi.typeAnnotation,
|
||||
"defaultValue": vi.defaultValue,
|
||||
"position": vi.position.toJson(),
|
||||
"mutable": vi.mutable,
|
||||
"exported": vi.exported
|
||||
}
|
||||
|
||||
proc toJson*(pi: ParameterInfo): JsonNode =
|
||||
## Convert ParameterInfo to JSON.
|
||||
result = %*{
|
||||
"name": pi.name,
|
||||
"typeAnnotation": pi.typeAnnotation,
|
||||
"defaultValue": pi.defaultValue,
|
||||
"isOptional": pi.isOptional,
|
||||
"isVarargs": pi.isVarargs
|
||||
}
|
||||
|
||||
proc toJson*(fi: FunctionInfo): JsonNode =
|
||||
## Convert FunctionInfo to JSON.
|
||||
result = %*{
|
||||
"name": fi.name,
|
||||
"kind": fi.kind,
|
||||
"parameters": %*(fi.parameters.mapIt(it.toJson())),
|
||||
"returnType": fi.returnType,
|
||||
"isAsync": fi.isAsync,
|
||||
"isExported": fi.isExported,
|
||||
"isGenerator": fi.isGenerator,
|
||||
"visibility": fi.visibility,
|
||||
"position": fi.position.toJson(),
|
||||
"docComment": fi.docComment
|
||||
}
|
||||
|
||||
proc toJson*(ci: ClassInfo): JsonNode =
|
||||
## Convert ClassInfo to JSON.
|
||||
result = %*{
|
||||
"name": ci.name,
|
||||
"kind": ci.kind,
|
||||
"baseTypes": %*(ci.baseTypes),
|
||||
"fields": %*(ci.fields.mapIt(it.toJson())),
|
||||
"methods": %*(ci.methods.mapIt(it.toJson())),
|
||||
"generics": %*(ci.generics),
|
||||
"visibility": ci.visibility,
|
||||
"position": ci.position.toJson(),
|
||||
"docComment": ci.docComment
|
||||
}
|
||||
|
||||
proc toJson*(ii: ImportInfo): JsonNode =
|
||||
## Convert ImportInfo to JSON.
|
||||
result = %*{
|
||||
"module": ii.module,
|
||||
"symbols": %*(ii.symbols),
|
||||
"isRelative": ii.isRelative,
|
||||
"alias": ii.alias,
|
||||
"position": ii.position.toJson()
|
||||
}
|
||||
|
||||
proc toJson*(mi: ModuleInfo): JsonNode =
|
||||
## Convert ModuleInfo to JSON.
|
||||
result = %*{
|
||||
"flavor": $mi.flavor,
|
||||
"detectionMethod": $mi.detectionMethod,
|
||||
"imports": %*(mi.imports.mapIt(it.toJson())),
|
||||
"variables": %*(mi.variables.mapIt(it.toJson())),
|
||||
"functions": %*(mi.functions.mapIt(it.toJson())),
|
||||
"classes": %*(mi.classes.mapIt(it.toJson())),
|
||||
"linesOfCode": %mi.linesOfCode,
|
||||
"totalTokens": %mi.totalTokens,
|
||||
"comments": %mi.comments,
|
||||
"debugInfo": mi.debugInfo
|
||||
}
|
||||
|
||||
proc toJson*(vr: ValidationResult): JsonNode =
|
||||
## Convert ValidationResult to JSON.
|
||||
result = %*{
|
||||
"valid": vr.valid,
|
||||
"flavor": $vr.flavor,
|
||||
"detectionMethod": $vr.detectionMethod,
|
||||
"errors": %*(vr.errors.mapIt(it.toJson())),
|
||||
"warnings": %vr.warnings,
|
||||
"infos": %vr.infos,
|
||||
"moduleInfo": vr.moduleInfo.toJson(),
|
||||
"durationMs": %vr.durationMs,
|
||||
"debugOutput": vr.debugOutput
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helper procs
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
proc newSourcePosition*(line, column, offset: int): SourcePosition =
|
||||
## Create a new SourcePosition.
|
||||
SourcePosition(line: line, column: column, offset: offset)
|
||||
|
||||
proc newSourceRange*(startPos, endPos: SourcePosition): SourceRange =
|
||||
## Create a new SourceRange.
|
||||
SourceRange(startPos: startPos, endPos: endPos)
|
||||
|
||||
proc newValidationError*(
|
||||
severity: ErrorSeverity,
|
||||
message: string,
|
||||
code: string,
|
||||
position: SourcePosition,
|
||||
range: SourceRange,
|
||||
context: string = "",
|
||||
hint: string = ""
|
||||
): ValidationError =
|
||||
## Create a new ValidationError.
|
||||
ValidationError(
|
||||
severity: severity,
|
||||
message: message,
|
||||
code: code,
|
||||
position: position,
|
||||
range: range,
|
||||
context: context,
|
||||
hint: hint
|
||||
)
|
||||
|
||||
proc newValidationOptions*(
|
||||
flavor: LanguageFlavor = lfUnknown,
|
||||
debugMode: bool = false,
|
||||
strictMode: bool = false,
|
||||
maxErrors: int = 0,
|
||||
includeContext: bool = true,
|
||||
includeHints: bool = true
|
||||
): ValidationOptions =
|
||||
## Create default ValidationOptions.
|
||||
ValidationOptions(
|
||||
flavor: flavor,
|
||||
debugMode: debugMode,
|
||||
strictMode: strictMode,
|
||||
maxErrors: maxErrors,
|
||||
includeContext: includeContext,
|
||||
includeHints: includeHints
|
||||
)
|
||||
|
||||
proc newValidationResult*(): ValidationResult =
|
||||
## Create an empty ValidationResult.
|
||||
ValidationResult(
|
||||
valid: true,
|
||||
flavor: lfUnknown,
|
||||
detectionMethod: fdmUnknown,
|
||||
errors: @[],
|
||||
warnings: 0,
|
||||
infos: 0,
|
||||
moduleInfo: ModuleInfo(),
|
||||
durationMs: 0.0,
|
||||
debugOutput: ""
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Error code constants
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
const
|
||||
# General errors (E0xxx)
|
||||
ErrUnexpectedToken* = "E0001"
|
||||
ErrUnclosedString* = "E0002"
|
||||
ErrUnclosedComment* = "E0003"
|
||||
ErrUnclosedBlock* = "E0004"
|
||||
ErrMismatchedBracket* = "E0005"
|
||||
ErrUnclosedBracket* = "E0006"
|
||||
ErrInvalidSyntax* = "E0007"
|
||||
ErrUnexpectedEOF* = "E0008"
|
||||
ErrEmptySource* = "E0009"
|
||||
ErrEncodingError* = "E0010"
|
||||
|
||||
# Language-specific errors (E1xxx)
|
||||
ErrUnexpectedIndent* = "E1001"
|
||||
ErrMissingIndent* = "E1002"
|
||||
ErrDuplicateParam* = "E1003"
|
||||
ErrUndefinedVariable* = "E1004"
|
||||
ErrTypeMismatch* = "E1005"
|
||||
ErrInvalidAssignment* = "E1006"
|
||||
ErrMissingReturn* = "E1007"
|
||||
ErrInvalidDecorator* = "E1008"
|
||||
ErrInvalidImport* = "E1009"
|
||||
|
||||
# Template errors (E2xxx)
|
||||
ErrUnclosedTemplateTag* = "E2001"
|
||||
ErrUnclosedBlockTag* = "E2002"
|
||||
ErrInvalidFilter* = "E2003"
|
||||
|
||||
# Info/Warning codes (I0xxx / W0xxx)
|
||||
InfoLongLine* = "I0001"
|
||||
InfoTrailingWhitespace* = "I0002"
|
||||
WarnUnusedVariable* = "W0001"
|
||||
WarnInconsistentNaming* = "W0002"
|
||||
WarnMissingDoc* = "W0003"
|
||||
|
||||
# Export everything
|
||||
export json, strutils, tables, times
|
||||
@@ -0,0 +1,202 @@
|
||||
## Abstract base validator for Validatrix.
|
||||
##
|
||||
## All language-specific validators inherit from this base class.
|
||||
## Provides the common validation pipeline: tokenize -> analyze -> report.
|
||||
|
||||
import std/[json, times, strutils, strformat]
|
||||
import ./types, ./tokenizerbase
|
||||
|
||||
type
|
||||
ValidatorBase* = ref object of RootObj
|
||||
## Abstract validator that orchestrates tokenization and structural analysis.
|
||||
source*: string
|
||||
sourcePath*: string
|
||||
options*: ValidationOptions
|
||||
tokenizer*: TokenizerBase
|
||||
result*: ValidationResult
|
||||
startTime*: float
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Construction
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
proc newValidatorBase*(
|
||||
source: string,
|
||||
options: ValidationOptions = newValidationOptions(),
|
||||
sourcePath: string = ""
|
||||
): ValidatorBase =
|
||||
## Create a new base validator.
|
||||
result = ValidatorBase(
|
||||
source: source,
|
||||
sourcePath: sourcePath,
|
||||
options: options,
|
||||
result: newValidationResult(),
|
||||
startTime: epochTime()
|
||||
)
|
||||
result.result.flavor = options.flavor
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Template methods — subclasses override
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
method createTokenizer*(self: ValidatorBase): TokenizerBase {.base, gcsafe.} =
|
||||
## Create the appropriate tokenizer for this language.
|
||||
raise newException(ValueError, "createTokenizer() not implemented")
|
||||
|
||||
method analyzeTokens*(self: ValidatorBase) {.base, gcsafe.} =
|
||||
## Analyze tokenized output for structural issues (brackets, etc).
|
||||
## Default: check bracket balance.
|
||||
let bracketErrors = self.tokenizer.validateBrackets()
|
||||
for err in bracketErrors:
|
||||
self.result.errors.add(err)
|
||||
|
||||
method buildModuleInfo*(self: ValidatorBase) {.base, gcsafe.} =
|
||||
## Build structural information (variables, functions, classes).
|
||||
## Default: populate basic info from token list.
|
||||
discard
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Validation pipeline
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
method validate*(self: ValidatorBase): ValidationResult {.base, gcsafe.} =
|
||||
## Run the full validation pipeline.
|
||||
when defined(validatrixDebug):
|
||||
if self.options.debugMode:
|
||||
if not debugEnabled:
|
||||
enableDebug()
|
||||
debugEnter("VALIDATE", &"Starting validation for {$self.options.flavor}")
|
||||
debugTimerStart("total")
|
||||
|
||||
try:
|
||||
# 1. Check for empty source
|
||||
if self.source.len == 0:
|
||||
self.result.valid = false
|
||||
self.result.errors.add(newValidationError(
|
||||
esError, "Source is empty", ErrEmptySource,
|
||||
newSourcePosition(1, 1, 0),
|
||||
newSourceRange(newSourcePosition(1, 1, 0), newSourcePosition(1, 1, 0))
|
||||
))
|
||||
return self.result
|
||||
|
||||
# 2. Create and run tokenizer
|
||||
self.tokenizer = self.createTokenizer()
|
||||
when defined(validatrixDebug):
|
||||
if self.options.debugMode:
|
||||
debugTimerStart("tokenize")
|
||||
self.tokenizer.tokenize()
|
||||
when defined(validatrixDebug):
|
||||
if self.options.debugMode:
|
||||
discard debugTimerStop("tokenize")
|
||||
debugLog("VALIDATE", &"Tokenized: {self.tokenizer.tokens.len} tokens, {self.tokenizer.errors.len} errors")
|
||||
|
||||
# 3. Collect tokenizer errors
|
||||
self.result.errors.add(self.tokenizer.getErrors())
|
||||
|
||||
# 4. Structural analysis
|
||||
when defined(validatrixDebug):
|
||||
if self.options.debugMode:
|
||||
debugEnter("ANALYZE", "Running structural analysis")
|
||||
|
||||
self.analyzeTokens()
|
||||
self.buildModuleInfo()
|
||||
|
||||
when defined(validatrixDebug):
|
||||
if self.options.debugMode:
|
||||
debugLeave("ANALYZE")
|
||||
|
||||
# 5. Populate result metadata
|
||||
self.result.warnings = 0
|
||||
self.result.infos = 0
|
||||
for e in self.result.errors:
|
||||
if e.severity == esWarning: self.result.warnings += 1
|
||||
if e.severity == esInfo: self.result.infos += 1
|
||||
|
||||
self.result.valid = self.result.errors.len == 0
|
||||
if self.options.strictMode and self.result.warnings > 0:
|
||||
self.result.valid = false
|
||||
|
||||
# 6. Populate module info counts
|
||||
self.result.moduleInfo.totalTokens = self.tokenizer.tokens.len
|
||||
self.result.moduleInfo.linesOfCode = self.source.count('\n') + 1
|
||||
|
||||
except Exception as e:
|
||||
# Never crash — always report useful error
|
||||
self.result.valid = false
|
||||
self.result.errors.add(newValidationError(
|
||||
esCritical,
|
||||
&"Internal validation error: {e.msg}",
|
||||
"E9999",
|
||||
newSourcePosition(0, 0, 0),
|
||||
newSourceRange(newSourcePosition(0, 0, 0), newSourcePosition(0, 0, 0)),
|
||||
hint = "This is a bug in the validator — please report it"
|
||||
))
|
||||
when defined(validatrixDebug):
|
||||
if self.options.debugMode:
|
||||
debugLog("EXCEPTION", &"Unhandled exception: {e.msg}")
|
||||
|
||||
# 7. Duration
|
||||
self.result.durationMs = (epochTime() - self.startTime) * 1000.0
|
||||
|
||||
when defined(validatrixDebug):
|
||||
if self.options.debugMode:
|
||||
debugTimerStopAndLog("total")
|
||||
self.result.debugOutput = getDebugOutput()
|
||||
|
||||
return self.result
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Result helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
proc getJsonResult*(self: ValidatorBase): JsonNode =
|
||||
## Get validation result as JSON.
|
||||
self.result.toJson()
|
||||
|
||||
proc getJsonResultString*(self: ValidatorBase): string =
|
||||
## Get validation result as pretty-printed JSON string.
|
||||
self.result.toJson().pretty()
|
||||
|
||||
proc isValid*(self: ValidatorBase): bool =
|
||||
## Quick validity check.
|
||||
self.result.valid
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Factory — create correct validator for a flavor
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
type
|
||||
ValidatorConstructor* = proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase {.gcsafe.}
|
||||
|
||||
var validatorRegistry*: Table[string, ValidatorConstructor] = initTable[string, ValidatorConstructor]()
|
||||
|
||||
proc registerValidator*(flavor: string, constructor: ValidatorConstructor) =
|
||||
## Register a validator constructor for a given flavor name.
|
||||
validatorRegistry[flavor] = constructor
|
||||
|
||||
proc createValidator*(
|
||||
flavor: LanguageFlavor,
|
||||
source: string,
|
||||
options: ValidationOptions = newValidationOptions(),
|
||||
sourcePath: string = ""
|
||||
): ValidatorBase =
|
||||
## Factory: create the appropriate validator for the given flavor.
|
||||
# Map the pure enum value to the registry string key.
|
||||
# Since LanguageFlavor uses {.pure.}, $flavor returns the enum name ("lfNim")
|
||||
# but the registry uses the custom string value ("nim").
|
||||
let flavorStr = case flavor
|
||||
of lfNim: "nim"
|
||||
of lfPython: "python"
|
||||
of lfBash, lfShell: "bash"
|
||||
of lfJavaScript: "javascript"
|
||||
of lfPHP: "php"
|
||||
of lfHTML: "html"
|
||||
of lfJinja: "jinja"
|
||||
of lfJSON: "json"
|
||||
of lfYAML: "yaml"
|
||||
of lfTOML: "toml"
|
||||
else: $flavor # fallback for future flavors
|
||||
if validatorRegistry.hasKey(flavorStr):
|
||||
result = validatorRegistry[flavorStr](source, options, sourcePath)
|
||||
else:
|
||||
raise newException(ValueError, &"No validator registered for flavor: {flavorStr}")
|
||||
@@ -0,0 +1,130 @@
|
||||
## Bash/Shell validator for Validatrix.
|
||||
##
|
||||
## Validates Bash/shell scripts, checking structural balance and common issues.
|
||||
|
||||
import std/[json, strformat, tables]
|
||||
{.warning[UnusedImport]:off.}
|
||||
import ../core/types, ../core/tokenizerbase, ../core/validatorbase
|
||||
import ../tokenizers/bash_tokenizer
|
||||
|
||||
type
|
||||
BashValidator* = ref object of ValidatorBase
|
||||
## Validator for Bash/shell scripts.
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Construction
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
proc newBashValidator*(source: string, options: ValidationOptions = newValidationOptions(), sourcePath: string = ""): BashValidator =
|
||||
## Create a new Bash validator.
|
||||
result = BashValidator(
|
||||
source: source,
|
||||
sourcePath: sourcePath,
|
||||
options: options,
|
||||
result: newValidationResult(),
|
||||
startTime: epochTime()
|
||||
)
|
||||
result.result.flavor = options.flavor
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tokenizer creation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
method createTokenizer*(self: BashValidator): TokenizerBase =
|
||||
## Create the Bash tokenizer.
|
||||
result = newBashTokenizer(self.source, self.options)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Structural analysis
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
method analyzeTokens*(self: BashValidator) =
|
||||
## Analyze Bash-specific patterns.
|
||||
when defined(validatrixDebug):
|
||||
if self.options.debugMode:
|
||||
debugEnter("BASH_ANALYZE", "Analyzing Bash structure")
|
||||
|
||||
procCall ValidatorBase(self).analyzeTokens()
|
||||
|
||||
let tokens = self.tokenizer.tokens
|
||||
|
||||
# Check for unclosed if/fi, do/done, case/esac
|
||||
var keywords: seq[string] = @[]
|
||||
for tok in tokens:
|
||||
if tok.kind == tkKeyword:
|
||||
keywords.add(tok.value)
|
||||
if tok.value in ["if", "case", "for", "while", "until"]:
|
||||
self.result.moduleInfo.functions.add(FunctionInfo(
|
||||
name: tok.value & "_block",
|
||||
kind: "block",
|
||||
position: tok.position
|
||||
))
|
||||
elif tok.kind == tkComment:
|
||||
self.result.moduleInfo.comments += 1
|
||||
|
||||
# Check for fi/done/esac matching
|
||||
var ifCount, forCount, caseCount = 0
|
||||
var fiCount, doneCount, esacCount = 0
|
||||
for kw in keywords:
|
||||
case kw
|
||||
of "if": ifCount += 1
|
||||
of "fi": fiCount += 1
|
||||
of "for", "while", "until": forCount += 1
|
||||
of "done": doneCount += 1
|
||||
of "case": caseCount += 1
|
||||
of "esac": esacCount += 1
|
||||
else: discard
|
||||
|
||||
if ifCount != fiCount:
|
||||
if ifCount > fiCount:
|
||||
self.result.errors.add(newValidationError(
|
||||
esError, &"Unclosed 'if' block: {ifCount} 'if' but {fiCount} 'fi'",
|
||||
ErrUnclosedBlock,
|
||||
newSourcePosition(1, 1, 0),
|
||||
newSourceRange(newSourcePosition(1, 1, 0), newSourcePosition(1, 1, 0)),
|
||||
hint = "Add missing 'fi' to close the if block"
|
||||
))
|
||||
|
||||
if forCount != doneCount:
|
||||
if forCount > doneCount:
|
||||
self.result.errors.add(newValidationError(
|
||||
esError, &"Unclosed loop: {forCount} loop starters but {doneCount} 'done'",
|
||||
ErrUnclosedBlock,
|
||||
newSourcePosition(1, 1, 0),
|
||||
newSourceRange(newSourcePosition(1, 1, 0), newSourcePosition(1, 1, 0)),
|
||||
hint = "Add missing 'done' to close the loop"
|
||||
))
|
||||
|
||||
if caseCount != esacCount:
|
||||
if caseCount > esacCount:
|
||||
self.result.errors.add(newValidationError(
|
||||
esError, &"Unclosed 'case' block: {caseCount} 'case' but {esacCount} 'esac'",
|
||||
ErrUnclosedBlock,
|
||||
newSourcePosition(1, 1, 0),
|
||||
newSourceRange(newSourcePosition(1, 1, 0), newSourcePosition(1, 1, 0)),
|
||||
hint = "Add missing 'esac' to close the case block"
|
||||
))
|
||||
|
||||
when defined(validatrixDebug):
|
||||
if self.options.debugMode:
|
||||
debugLog("BASH_ANALYZE", &"if/fi: {ifCount}/{fiCount}, for/done: {forCount}/{doneCount}, case/esac: {caseCount}/{esacCount}")
|
||||
debugLeave("BASH_ANALYZE")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Registration
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
proc init*() =
|
||||
## Register the Bash/Shell validator.
|
||||
registerValidator("bash", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
|
||||
result = newBashValidator(source, options, sourcePath)
|
||||
)
|
||||
registerValidator("shell", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
|
||||
result = newBashValidator(source, options, sourcePath)
|
||||
)
|
||||
registerValidator("sh", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
|
||||
result = newBashValidator(source, options, sourcePath)
|
||||
)
|
||||
|
||||
# Auto-init
|
||||
init()
|
||||
@@ -0,0 +1,258 @@
|
||||
## JSON, YAML, and TOML config validators for Validatrix.
|
||||
##
|
||||
## These use built-in parsers where possible for accuracy.
|
||||
|
||||
import std/[json, strformat, strutils]
|
||||
|
||||
import ../core/types, ../core/tokenizerbase, ../core/validatorbase
|
||||
|
||||
# =====================================================================
|
||||
# JSON Validator
|
||||
# =====================================================================
|
||||
|
||||
type
|
||||
JSONValidator* = ref object of ValidatorBase
|
||||
|
||||
proc newJSONValidator*(source: string, options: ValidationOptions = newValidationOptions(), sourcePath: string = ""): JSONValidator =
|
||||
result = JSONValidator(
|
||||
source: source,
|
||||
sourcePath: sourcePath,
|
||||
options: options,
|
||||
result: newValidationResult(),
|
||||
startTime: epochTime()
|
||||
)
|
||||
result.result.flavor = options.flavor
|
||||
|
||||
method createTokenizer*(self: JSONValidator): TokenizerBase =
|
||||
# We use a minimal tokenizer for JSON since the main validation is via parseJson
|
||||
result = newTokenizerBase(self.source, self.options)
|
||||
|
||||
method validate*(self: JSONValidator): ValidationResult =
|
||||
## Override validate to use native JSON parser.
|
||||
when defined(validatrixDebug):
|
||||
if self.options.debugMode:
|
||||
debugEnter("JSON_VALIDATE", "Validating JSON")
|
||||
|
||||
try:
|
||||
self.startTime = epochTime()
|
||||
if self.source.strip().len == 0:
|
||||
self.result.valid = false
|
||||
self.result.errors.add(newValidationError(esError, "Empty JSON source", ErrEmptySource,
|
||||
newSourcePosition(1,1,0), newSourceRange(newSourcePosition(1,1,0), newSourcePosition(1,1,0))))
|
||||
return self.result
|
||||
|
||||
let parsed = parseJson(self.source)
|
||||
self.result.valid = true
|
||||
self.result.moduleInfo.totalTokens = self.source.len
|
||||
self.result.moduleInfo.linesOfCode = self.source.count('\n') + 1
|
||||
|
||||
# Extract structural info
|
||||
case parsed.kind
|
||||
of JObject:
|
||||
for key, val in parsed:
|
||||
discard
|
||||
of JArray:
|
||||
for item in parsed:
|
||||
discard
|
||||
else: discard
|
||||
|
||||
except JsonParsingError as e:
|
||||
self.result.valid = false
|
||||
# Extract position from error message
|
||||
let msg = e.msg
|
||||
self.result.errors.add(newValidationError(esError, msg, ErrInvalidSyntax,
|
||||
newSourcePosition(1, 1, 0), newSourceRange(newSourcePosition(1, 1, 0), newSourcePosition(1, 1, 0))))
|
||||
|
||||
except Exception as e:
|
||||
self.result.valid = false
|
||||
self.result.errors.add(newValidationError(esError, &"JSON error: {e.msg}", ErrInvalidSyntax,
|
||||
newSourcePosition(1, 1, 0), newSourceRange(newSourcePosition(1, 1, 0), newSourcePosition(1, 1, 0))))
|
||||
|
||||
self.result.durationMs = (epochTime() - self.startTime) * 1000.0
|
||||
when defined(validatrixDebug):
|
||||
if self.options.debugMode: debugLeave("JSON_VALIDATE")
|
||||
|
||||
return self.result
|
||||
|
||||
# =====================================================================
|
||||
# YAML Validator
|
||||
# =====================================================================
|
||||
|
||||
type
|
||||
YAMLValidator* = ref object of ValidatorBase
|
||||
|
||||
proc newYAMLValidator*(source: string, options: ValidationOptions = newValidationOptions(), sourcePath: string = ""): YAMLValidator =
|
||||
result = YAMLValidator(
|
||||
source: source,
|
||||
sourcePath: sourcePath,
|
||||
options: options,
|
||||
result: newValidationResult(),
|
||||
startTime: epochTime()
|
||||
)
|
||||
result.result.flavor = options.flavor
|
||||
|
||||
method createTokenizer*(self: YAMLValidator): TokenizerBase =
|
||||
result = newTokenizerBase(self.source, self.options)
|
||||
|
||||
method validate*(self: YAMLValidator): ValidationResult =
|
||||
when defined(validatrixDebug):
|
||||
if self.options.debugMode:
|
||||
debugEnter("YAML_VALIDATE", "Validating YAML")
|
||||
|
||||
try:
|
||||
self.startTime = epochTime()
|
||||
if self.source.strip().len == 0:
|
||||
self.result.valid = true # Empty YAML is valid
|
||||
return self.result
|
||||
|
||||
# Simple YAML validation - check for basic structural issues
|
||||
var lines = self.source.splitLines()
|
||||
var indentStack {.used.}: seq[int] = @[]
|
||||
var hadContent = false
|
||||
|
||||
for i, line in lines:
|
||||
if line.strip().len == 0 or line.strip()[0] == '#':
|
||||
continue
|
||||
hadContent = true
|
||||
|
||||
# Check indentation
|
||||
var indent = 0
|
||||
for c in line:
|
||||
if c == ' ': indent += 1
|
||||
elif c == '\t':
|
||||
self.result.errors.add(newValidationError(esWarning,
|
||||
&"Line {i+1}: Tab characters in YAML indentation", "W1001",
|
||||
newSourcePosition(i+1, 1, 0),
|
||||
newSourceRange(newSourcePosition(i+1, 1, 0), newSourcePosition(i+1, 1, 0)),
|
||||
hint = "Use spaces for YAML indentation"))
|
||||
indent += 8
|
||||
else: break
|
||||
|
||||
# Check for mapping key:
|
||||
if line.contains(":") and not line.strip().startsWith("-"):
|
||||
let colonIdx = line.find(':')
|
||||
let key = line[0..<colonIdx].strip()
|
||||
if key.len == 0:
|
||||
self.result.errors.add(newValidationError(esError,
|
||||
&"Line {i+1}: Empty mapping key", ErrInvalidSyntax,
|
||||
newSourcePosition(i+1, 1, 0),
|
||||
newSourceRange(newSourcePosition(i+1, 1, 0), newSourcePosition(i+1, 1, 0))))
|
||||
|
||||
if not hadContent:
|
||||
self.result.valid = false
|
||||
self.result.errors.add(newValidationError(esError, "YAML file has no content", ErrEmptySource,
|
||||
newSourcePosition(1,1,0), newSourceRange(newSourcePosition(1,1,0), newSourcePosition(1,1,0))))
|
||||
|
||||
self.result.valid = self.result.errors.len == 0
|
||||
self.result.moduleInfo.linesOfCode = lines.len
|
||||
self.result.moduleInfo.totalTokens = self.source.len
|
||||
|
||||
except Exception as e:
|
||||
self.result.valid = false
|
||||
self.result.errors.add(newValidationError(esError, &"YAML error: {e.msg}", ErrInvalidSyntax,
|
||||
newSourcePosition(1,1,0), newSourceRange(newSourcePosition(1,1,0), newSourcePosition(1,1,0))))
|
||||
|
||||
self.result.durationMs = (epochTime() - self.startTime) * 1000.0
|
||||
when defined(validatrixDebug):
|
||||
if self.options.debugMode: debugLeave("YAML_VALIDATE")
|
||||
return self.result
|
||||
|
||||
# =====================================================================
|
||||
# TOML Validator
|
||||
# =====================================================================
|
||||
|
||||
type
|
||||
TOMLValidator* = ref object of ValidatorBase
|
||||
|
||||
proc newTOMLValidator*(source: string, options: ValidationOptions = newValidationOptions(), sourcePath: string = ""): TOMLValidator =
|
||||
result = TOMLValidator(
|
||||
source: source,
|
||||
sourcePath: sourcePath,
|
||||
options: options,
|
||||
result: newValidationResult(),
|
||||
startTime: epochTime()
|
||||
)
|
||||
result.result.flavor = options.flavor
|
||||
|
||||
method createTokenizer*(self: TOMLValidator): TokenizerBase =
|
||||
result = newTokenizerBase(self.source, self.options)
|
||||
|
||||
method validate*(self: TOMLValidator): ValidationResult =
|
||||
when defined(validatrixDebug):
|
||||
if self.options.debugMode:
|
||||
debugEnter("TOML_VALIDATE", "Validating TOML")
|
||||
|
||||
try:
|
||||
self.startTime = epochTime()
|
||||
if self.source.strip().len == 0:
|
||||
self.result.valid = true
|
||||
return self.result
|
||||
|
||||
var lines = self.source.splitLines()
|
||||
var currentSection = ""
|
||||
var hadContent = false
|
||||
|
||||
for i, line in lines:
|
||||
let stripped = line.strip()
|
||||
if stripped.len == 0 or stripped[0] == '#':
|
||||
continue
|
||||
hadContent = true
|
||||
|
||||
# Check for [section]
|
||||
if stripped[0] == '[' and stripped[^1] == ']':
|
||||
currentSection = stripped[1..^2].strip()
|
||||
continue
|
||||
|
||||
# Check key = value
|
||||
if not stripped.contains('='):
|
||||
self.result.errors.add(newValidationError(esError,
|
||||
&"Line {i+1}: Expected key=value pair, got '{stripped}'", ErrInvalidSyntax,
|
||||
newSourcePosition(i+1, 1, 0),
|
||||
newSourceRange(newSourcePosition(i+1, 1, 0), newSourcePosition(i+1, 1, 0)),
|
||||
hint = "TOML uses 'key = value' format"))
|
||||
continue
|
||||
|
||||
let eqIdx = stripped.find('=')
|
||||
let key = stripped[0..<eqIdx].strip()
|
||||
if key.len == 0:
|
||||
self.result.errors.add(newValidationError(esError,
|
||||
&"Line {i+1}: Missing key before '='", ErrInvalidSyntax,
|
||||
newSourcePosition(i+1, 1, 0),
|
||||
newSourceRange(newSourcePosition(i+1, 1, 0), newSourcePosition(i+1, 1, 0))))
|
||||
|
||||
if not hadContent:
|
||||
self.result.valid = false
|
||||
self.result.errors.add(newValidationError(esError, "TOML file has no content", ErrEmptySource,
|
||||
newSourcePosition(1,1,0), newSourceRange(newSourcePosition(1,1,0), newSourcePosition(1,1,0))))
|
||||
|
||||
self.result.valid = self.result.errors.len == 0
|
||||
self.result.moduleInfo.linesOfCode = lines.len
|
||||
self.result.moduleInfo.totalTokens = self.source.len
|
||||
|
||||
except Exception as e:
|
||||
self.result.valid = false
|
||||
self.result.errors.add(newValidationError(esError, &"TOML error: {e.msg}", ErrInvalidSyntax,
|
||||
newSourcePosition(1,1,0), newSourceRange(newSourcePosition(1,1,0), newSourcePosition(1,1,0))))
|
||||
|
||||
self.result.durationMs = (epochTime() - self.startTime) * 1000.0
|
||||
when defined(validatrixDebug):
|
||||
if self.options.debugMode: debugLeave("TOML_VALIDATE")
|
||||
return self.result
|
||||
|
||||
# =====================================================================
|
||||
# Registration
|
||||
# =====================================================================
|
||||
|
||||
proc init*() =
|
||||
registerValidator("json", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
|
||||
result = newJSONValidator(source, options, sourcePath))
|
||||
registerValidator("jsonc", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
|
||||
result = newJSONValidator(source, options, sourcePath))
|
||||
registerValidator("yaml", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
|
||||
result = newYAMLValidator(source, options, sourcePath))
|
||||
registerValidator("yml", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
|
||||
result = newYAMLValidator(source, options, sourcePath))
|
||||
registerValidator("toml", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
|
||||
result = newTOMLValidator(source, options, sourcePath))
|
||||
|
||||
init()
|
||||
@@ -0,0 +1,41 @@
|
||||
## HTML/XML validator for Validatrix.
|
||||
|
||||
import std/[json, tables]
|
||||
{.warning[UnusedImport]:off.}
|
||||
import ../core/types, ../core/tokenizerbase, ../core/validatorbase
|
||||
import ../tokenizers/html_tokenizer
|
||||
|
||||
type
|
||||
HTMLValidator* = ref object of ValidatorBase
|
||||
|
||||
proc newHTMLValidator*(source: string, options: ValidationOptions = newValidationOptions(), sourcePath: string = ""): HTMLValidator =
|
||||
result = HTMLValidator(
|
||||
source: source,
|
||||
sourcePath: sourcePath,
|
||||
options: options,
|
||||
result: newValidationResult(),
|
||||
startTime: epochTime()
|
||||
)
|
||||
result.result.flavor = options.flavor
|
||||
|
||||
method createTokenizer*(self: HTMLValidator): TokenizerBase =
|
||||
result = newHTMLTokenizer(self.source, self.options)
|
||||
|
||||
method analyzeTokens*(self: HTMLValidator) =
|
||||
when defined(validatrixDebug):
|
||||
if self.options.debugMode: debugEnter("HTML_ANALYZE", "Analyzing HTML")
|
||||
procCall ValidatorBase(self).analyzeTokens()
|
||||
for tok in self.tokenizer.tokens:
|
||||
if tok.kind == tkComment: self.result.moduleInfo.comments += 1
|
||||
when defined(validatrixDebug):
|
||||
if self.options.debugMode: debugLeave("HTML_ANALYZE")
|
||||
|
||||
proc init*() =
|
||||
registerValidator("html", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
|
||||
result = newHTMLValidator(source, options, sourcePath))
|
||||
registerValidator("htm", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
|
||||
result = newHTMLValidator(source, options, sourcePath))
|
||||
registerValidator("xml", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
|
||||
result = newHTMLValidator(source, options, sourcePath))
|
||||
|
||||
init()
|
||||
@@ -0,0 +1,82 @@
|
||||
## JavaScript/TypeScript validator for Validatrix.
|
||||
|
||||
import std/[json, tables]
|
||||
{.warning[UnusedImport]:off.}
|
||||
import ../core/types, ../core/tokenizerbase, ../core/validatorbase
|
||||
import ../tokenizers/javascript_tokenizer
|
||||
|
||||
type
|
||||
JavaScriptValidator* = ref object of ValidatorBase
|
||||
|
||||
proc newJavaScriptValidator*(source: string, options: ValidationOptions = newValidationOptions(), sourcePath: string = ""): JavaScriptValidator =
|
||||
result = JavaScriptValidator(
|
||||
source: source,
|
||||
sourcePath: sourcePath,
|
||||
options: options,
|
||||
result: newValidationResult(),
|
||||
startTime: epochTime()
|
||||
)
|
||||
result.result.flavor = options.flavor
|
||||
|
||||
method createTokenizer*(self: JavaScriptValidator): TokenizerBase =
|
||||
result = newJavaScriptTokenizer(self.source, self.options)
|
||||
|
||||
method analyzeTokens*(self: JavaScriptValidator) =
|
||||
when defined(validatrixDebug):
|
||||
if self.options.debugMode:
|
||||
debugEnter("JS_ANALYZE", "Analyzing JS structure")
|
||||
|
||||
procCall ValidatorBase(self).analyzeTokens()
|
||||
|
||||
let tokens = self.tokenizer.tokens
|
||||
var i = 0
|
||||
while i < tokens.len:
|
||||
let tok = tokens[i]
|
||||
case tok.kind
|
||||
of tkComment, tkDocComment:
|
||||
self.result.moduleInfo.comments += 1
|
||||
of tkKeyword:
|
||||
if tok.value in ["import", "export"]:
|
||||
var j = i + 1
|
||||
while j < tokens.len and tokens[j].kind in {tkWhitespace, tkNewline}:
|
||||
j += 1
|
||||
if j < tokens.len and tokens[j].kind in {tkKeyword, tkIdentifier, tkPunctuation}:
|
||||
if tok.value == "import":
|
||||
var importInfo = ImportInfo(module: "", position: tokens[j].position)
|
||||
if tokens[j].kind == tkKeyword and tokens[j].value == "default":
|
||||
j += 1
|
||||
# Find the actual imported name
|
||||
self.result.moduleInfo.imports.add(importInfo)
|
||||
elif tok.value in ["function", "async"]:
|
||||
var j = i + 1
|
||||
while j < tokens.len and tokens[j].kind in {tkWhitespace, tkNewline}:
|
||||
j += 1
|
||||
if j < tokens.len and tokens[j].kind == tkIdentifier:
|
||||
self.result.moduleInfo.functions.add(FunctionInfo(
|
||||
name: tokens[j].value, kind: "function", position: tokens[j].position))
|
||||
elif tok.value == "class":
|
||||
var j = i + 1
|
||||
while j < tokens.len and tokens[j].kind in {tkWhitespace, tkNewline}:
|
||||
j += 1
|
||||
if j < tokens.len and tokens[j].kind == tkIdentifier:
|
||||
self.result.moduleInfo.classes.add(ClassInfo(
|
||||
name: tokens[j].value, kind: "class", position: tokens[j].position))
|
||||
else:
|
||||
discard
|
||||
i += 1
|
||||
|
||||
when defined(validatrixDebug):
|
||||
if self.options.debugMode:
|
||||
debugLeave("JS_ANALYZE")
|
||||
|
||||
proc init*() =
|
||||
registerValidator("javascript", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
|
||||
result = newJavaScriptValidator(source, options, sourcePath))
|
||||
registerValidator("typescript", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
|
||||
result = newJavaScriptValidator(source, options, sourcePath))
|
||||
registerValidator("js", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
|
||||
result = newJavaScriptValidator(source, options, sourcePath))
|
||||
registerValidator("ts", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
|
||||
result = newJavaScriptValidator(source, options, sourcePath))
|
||||
|
||||
init()
|
||||
@@ -0,0 +1,104 @@
|
||||
## Jinja2 template validator for Validatrix.
|
||||
|
||||
import std/[json, strformat, tables]
|
||||
{.warning[UnusedImport]:off.}
|
||||
import ../core/types, ../core/tokenizerbase, ../core/validatorbase
|
||||
import ../tokenizers/jinja_tokenizer
|
||||
|
||||
type
|
||||
JinjaValidator* = ref object of ValidatorBase
|
||||
|
||||
proc newJinjaValidator*(source: string, options: ValidationOptions = newValidationOptions(), sourcePath: string = ""): JinjaValidator =
|
||||
result = JinjaValidator(
|
||||
source: source,
|
||||
sourcePath: sourcePath,
|
||||
options: options,
|
||||
result: newValidationResult(),
|
||||
startTime: epochTime()
|
||||
)
|
||||
result.result.flavor = options.flavor
|
||||
|
||||
method createTokenizer*(self: JinjaValidator): TokenizerBase =
|
||||
result = newJinjaTokenizer(self.source, self.options)
|
||||
|
||||
method analyzeTokens*(self: JinjaValidator) =
|
||||
when defined(validatrixDebug):
|
||||
if self.options.debugMode: debugEnter("JINJA_ANALYZE", "Analyzing Jinja structure")
|
||||
procCall ValidatorBase(self).analyzeTokens()
|
||||
|
||||
# Check matching block tags (for/endfor, if/endif, block/endblock, etc.)
|
||||
var blockStack: seq[string] = @[]
|
||||
for tok in self.tokenizer.tokens:
|
||||
if tok.kind == tkComment:
|
||||
self.result.moduleInfo.comments += 1
|
||||
elif tok.kind == tkKeyword:
|
||||
let val = tok.value.strip()
|
||||
# Strip {% and %}
|
||||
let inner = if val.startsWith("{%") and val.endsWith("%}"):
|
||||
val[2..^3].strip()
|
||||
else: val.strip()
|
||||
|
||||
let spaceIdx = inner.find(' ')
|
||||
let tagName = if spaceIdx >= 0: inner[0..<spaceIdx] else: inner
|
||||
|
||||
if tagName in ["for", "if", "block", "macro", "filter", "raw", "autoescape", "call", "with"]:
|
||||
blockStack.add(tagName)
|
||||
elif tagName == "endfor":
|
||||
if blockStack.len > 0 and blockStack[^1] == "for":
|
||||
discard blockStack.pop()
|
||||
else:
|
||||
self.result.errors.add(newValidationError(esError,
|
||||
"Unexpected 'endfor' - no matching 'for'", ErrUnclosedBlock, tok.position,
|
||||
newSourceRange(tok.position, tok.position)))
|
||||
elif tagName == "endif":
|
||||
if blockStack.len > 0 and blockStack[^1] == "if":
|
||||
discard blockStack.pop()
|
||||
else:
|
||||
self.result.errors.add(newValidationError(esError,
|
||||
"Unexpected 'endif' - no matching 'if'", ErrUnclosedBlock, tok.position,
|
||||
newSourceRange(tok.position, tok.position)))
|
||||
elif tagName == "endblock":
|
||||
if blockStack.len > 0 and blockStack[^1] == "block":
|
||||
discard blockStack.pop()
|
||||
else:
|
||||
self.result.errors.add(newValidationError(esError,
|
||||
"Unexpected 'endblock'", ErrUnclosedBlock, tok.position,
|
||||
newSourceRange(tok.position, tok.position)))
|
||||
elif tagName == "endmacro":
|
||||
if blockStack.len > 0 and blockStack[^1] == "macro":
|
||||
discard blockStack.pop()
|
||||
elif tagName == "endfilter":
|
||||
if blockStack.len > 0 and blockStack[^1] == "filter":
|
||||
discard blockStack.pop()
|
||||
elif tagName == "endraw":
|
||||
if blockStack.len > 0 and blockStack[^1] == "raw":
|
||||
discard blockStack.pop()
|
||||
elif tagName == "endautoescape":
|
||||
if blockStack.len > 0 and blockStack[^1] == "autoescape":
|
||||
discard blockStack.pop()
|
||||
elif tagName == "endcall":
|
||||
if blockStack.len > 0 and blockStack[^1] == "call":
|
||||
discard blockStack.pop()
|
||||
elif tagName == "endwith":
|
||||
if blockStack.len > 0 and blockStack[^1] == "with":
|
||||
discard blockStack.pop()
|
||||
|
||||
# Report unclosed blocks
|
||||
for b in blockStack:
|
||||
self.result.errors.add(newValidationError(esError,
|
||||
&"Unclosed '{b}' block tag", ErrUnclosedBlock, newSourcePosition(1, 1, 0),
|
||||
newSourceRange(newSourcePosition(1, 1, 0), newSourcePosition(1, 1, 0)),
|
||||
hint = &"Add a matching 'end{b}' tag"))
|
||||
|
||||
when defined(validatrixDebug):
|
||||
if self.options.debugMode: debugLeave("JINJA_ANALYZE")
|
||||
|
||||
proc init*() =
|
||||
registerValidator("jinja", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
|
||||
result = newJinjaValidator(source, options, sourcePath))
|
||||
registerValidator("jinja2", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
|
||||
result = newJinjaValidator(source, options, sourcePath))
|
||||
registerValidator("j2", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
|
||||
result = newJinjaValidator(source, options, sourcePath))
|
||||
|
||||
init()
|
||||
@@ -0,0 +1,149 @@
|
||||
## Nim language validator for Validatrix.
|
||||
##
|
||||
## Validates Nim source code using the Nim tokenizer.
|
||||
## Handles common structural patterns and reports errors consistently.
|
||||
|
||||
import std/[json, strformat, tables, sets]
|
||||
{.warning[UnusedImport]:off.}
|
||||
import ../core/types, ../core/tokenizerbase, ../core/validatorbase
|
||||
import ../tokenizers/nim_tokenizer
|
||||
|
||||
type
|
||||
NimValidator* = ref object of ValidatorBase
|
||||
## Validator for Nim source code.
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Construction
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
proc newNimValidator*(source: string, options: ValidationOptions = newValidationOptions(), sourcePath: string = ""): NimValidator =
|
||||
## Create a new Nim validator.
|
||||
result = NimValidator(
|
||||
source: source,
|
||||
sourcePath: sourcePath,
|
||||
options: options,
|
||||
result: newValidationResult(),
|
||||
startTime: epochTime()
|
||||
)
|
||||
result.result.flavor = options.flavor
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tokenizer creation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
method createTokenizer*(self: NimValidator): TokenizerBase =
|
||||
## Create the Nim tokenizer.
|
||||
result = newNimTokenizer(self.source, self.options)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Structural analysis
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
method analyzeTokens*(self: NimValidator) =
|
||||
## Analyze Nim-specific structural patterns.
|
||||
when defined(validatrixDebug):
|
||||
if self.options.debugMode:
|
||||
debugEnter("NIM_ANALYZE", "Analyzing Nim structure")
|
||||
|
||||
# Get parent's bracket validation
|
||||
procCall ValidatorBase(self).analyzeTokens()
|
||||
|
||||
# Nim-specific checks
|
||||
let tokens = self.tokenizer.tokens
|
||||
|
||||
# Check for common Nim patterns
|
||||
var i = 0
|
||||
while i < tokens.len:
|
||||
let tok = tokens[i]
|
||||
case tok.kind
|
||||
of tkComment, tkDocComment:
|
||||
self.result.moduleInfo.comments += 1
|
||||
of tkString, tkMultilineString, tkRawString:
|
||||
# Check for unclosed strings (already handled in tokenizer)
|
||||
discard
|
||||
of tkKeyword:
|
||||
# Track imports
|
||||
if tok.value in ["import", "include"]:
|
||||
# Look ahead for imported module name
|
||||
var j = i + 1
|
||||
while j < tokens.len and tokens[j].kind in {tkWhitespace, tkNewline}:
|
||||
j += 1
|
||||
if j < tokens.len and tokens[j].kind == tkIdentifier:
|
||||
var importInfo = ImportInfo(module: tokens[j].value, position: tokens[j].position)
|
||||
if tok.value == "from":
|
||||
importInfo.module = tokens[j].value
|
||||
# Find 'import' keyword after 'from'
|
||||
var k = j + 1
|
||||
while k < tokens.len and tokens[k].kind in {tkWhitespace, tkNewline, tkPunctuation}:
|
||||
k += 1
|
||||
if k < tokens.len and tokens[k].kind == tkKeyword and tokens[k].value == "import":
|
||||
k += 1
|
||||
while k < tokens.len and tokens[k].kind in {tkWhitespace, tkNewline}:
|
||||
k += 1
|
||||
if k < tokens.len and tokens[k].kind == tkIdentifier:
|
||||
importInfo.symbols.add(tokens[k].value)
|
||||
self.result.moduleInfo.imports.add(importInfo)
|
||||
of tkIdentifier:
|
||||
# Check for proc/func/method/template/macro/keyword name
|
||||
if i > 0 and tokens[i-1].kind == tkKeyword and
|
||||
tokens[i-1].value in ["proc", "func", "method", "template", "macro", "iterator", "converter"]:
|
||||
var funcInfo = FunctionInfo(
|
||||
name: tok.value,
|
||||
kind: tokens[i-1].value,
|
||||
position: tok.position
|
||||
)
|
||||
self.result.moduleInfo.functions.add(funcInfo)
|
||||
# Check for type definitions
|
||||
elif i > 0 and tokens[i-1].kind == tkKeyword and tokens[i-1].value == "type":
|
||||
var classInfo = ClassInfo(
|
||||
name: tok.value,
|
||||
kind: "type",
|
||||
position: tok.position
|
||||
)
|
||||
self.result.moduleInfo.classes.add(classInfo)
|
||||
else:
|
||||
discard
|
||||
i += 1
|
||||
|
||||
when defined(validatrixDebug):
|
||||
if self.options.debugMode:
|
||||
debugLeave("NIM_ANALYZE", &"Found {self.result.moduleInfo.functions.len} functions, {self.result.moduleInfo.classes.len} types")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Module info building
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
method buildModuleInfo*(self: NimValidator) =
|
||||
## Build Nim-specific module information.
|
||||
# Count lines
|
||||
self.result.moduleInfo.linesOfCode = self.source.count('\n') + 1
|
||||
self.result.moduleInfo.totalTokens = self.tokenizer.tokens.len
|
||||
|
||||
# Debug info
|
||||
when defined(validatrixDebug):
|
||||
if self.options.debugMode:
|
||||
self.result.moduleInfo.debugInfo = %*{
|
||||
"language": "Nim",
|
||||
"version": "2.0+",
|
||||
"tokenCount": self.tokenizer.tokens.len,
|
||||
"keywords": nimKeywords.len
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Registration
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
proc init*() =
|
||||
## Register the Nim validator.
|
||||
registerValidator("nim", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
|
||||
result = newNimValidator(source, options, sourcePath)
|
||||
)
|
||||
registerValidator("nims", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
|
||||
result = newNimValidator(source, options, sourcePath)
|
||||
)
|
||||
registerValidator("nimble", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
|
||||
result = newNimValidator(source, options, sourcePath)
|
||||
)
|
||||
|
||||
# Auto-init
|
||||
init()
|
||||
@@ -0,0 +1,74 @@
|
||||
## PHP validator for Validatrix.
|
||||
|
||||
import std/[json, tables]
|
||||
{.warning[UnusedImport]:off.}
|
||||
import ../core/types, ../core/tokenizerbase, ../core/validatorbase
|
||||
import ../tokenizers/php_tokenizer
|
||||
|
||||
type
|
||||
PHPValidator* = ref object of ValidatorBase
|
||||
|
||||
proc newPHPValidator*(source: string, options: ValidationOptions = newValidationOptions(), sourcePath: string = ""): PHPValidator =
|
||||
result = PHPValidator(
|
||||
source: source,
|
||||
sourcePath: sourcePath,
|
||||
options: options,
|
||||
result: newValidationResult(),
|
||||
startTime: epochTime()
|
||||
)
|
||||
result.result.flavor = options.flavor
|
||||
|
||||
method createTokenizer*(self: PHPValidator): TokenizerBase =
|
||||
result = newPHPTokenizer(self.source, self.options)
|
||||
|
||||
method analyzeTokens*(self: PHPValidator) =
|
||||
when defined(validatrixDebug):
|
||||
if self.options.debugMode:
|
||||
debugEnter("PHP_ANALYZE", "Analyzing PHP structure")
|
||||
|
||||
procCall ValidatorBase(self).analyzeTokens()
|
||||
|
||||
let tokens = self.tokenizer.tokens
|
||||
var i = 0
|
||||
while i < tokens.len:
|
||||
let tok = tokens[i]
|
||||
case tok.kind
|
||||
of tkComment: self.result.moduleInfo.comments += 1
|
||||
of tkKeyword:
|
||||
if tok.value in ["function", "fn"]:
|
||||
var j = i + 1
|
||||
while j < tokens.len and tokens[j].kind in {tkWhitespace, tkNewline}:
|
||||
j += 1
|
||||
if j < tokens.len and tokens[j].kind == tkIdentifier:
|
||||
self.result.moduleInfo.functions.add(FunctionInfo(
|
||||
name: tokens[j].value, kind: "function", position: tokens[j].position))
|
||||
elif tok.value == "class":
|
||||
var j = i + 1
|
||||
while j < tokens.len and tokens[j].kind in {tkWhitespace, tkNewline}:
|
||||
j += 1
|
||||
if j < tokens.len and tokens[j].kind == tkIdentifier:
|
||||
self.result.moduleInfo.classes.add(ClassInfo(
|
||||
name: tokens[j].value, kind: "class", position: tokens[j].position))
|
||||
elif tok.value in ["include", "require", "include_once", "require_once", "use", "namespace"]:
|
||||
var j = i + 1
|
||||
while j < tokens.len and tokens[j].kind in {tkWhitespace, tkNewline}:
|
||||
j += 1
|
||||
if j < tokens.len and tokens[j].kind in {tkIdentifier, tkString}:
|
||||
var importInfo = ImportInfo(module: tokens[j].value, position: tokens[j].position)
|
||||
if tok.value == "use":
|
||||
importInfo.isRelative = false
|
||||
self.result.moduleInfo.imports.add(importInfo)
|
||||
else: discard
|
||||
i += 1
|
||||
|
||||
when defined(validatrixDebug):
|
||||
if self.options.debugMode:
|
||||
debugLeave("PHP_ANALYZE")
|
||||
|
||||
proc init*() =
|
||||
registerValidator("php", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
|
||||
result = newPHPValidator(source, options, sourcePath))
|
||||
registerValidator("phtml", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
|
||||
result = newPHPValidator(source, options, sourcePath))
|
||||
|
||||
init()
|
||||
@@ -0,0 +1,90 @@
|
||||
## Python language validator for Validatrix.
|
||||
|
||||
import std/[json, tables]
|
||||
{.warning[UnusedImport]:off.}
|
||||
import ../core/types, ../core/tokenizerbase, ../core/validatorbase
|
||||
import ../tokenizers/python_tokenizer
|
||||
|
||||
type
|
||||
PythonValidator* = ref object of ValidatorBase
|
||||
|
||||
proc newPythonValidator*(source: string, options: ValidationOptions = newValidationOptions(), sourcePath: string = ""): PythonValidator =
|
||||
result = PythonValidator(
|
||||
source: source,
|
||||
sourcePath: sourcePath,
|
||||
options: options,
|
||||
result: newValidationResult(),
|
||||
startTime: epochTime()
|
||||
)
|
||||
result.result.flavor = options.flavor
|
||||
|
||||
method createTokenizer*(self: PythonValidator): TokenizerBase =
|
||||
result = newPythonTokenizer(self.source, self.options)
|
||||
|
||||
method analyzeTokens*(self: PythonValidator) =
|
||||
when defined(validatrixDebug):
|
||||
if self.options.debugMode:
|
||||
debugEnter("PYTHON_ANALYZE", "Analyzing Python structure")
|
||||
|
||||
procCall ValidatorBase(self).analyzeTokens()
|
||||
|
||||
let tokens = self.tokenizer.tokens
|
||||
var i = 0
|
||||
while i < tokens.len:
|
||||
let tok = tokens[i]
|
||||
case tok.kind
|
||||
of tkComment, tkDocComment:
|
||||
self.result.moduleInfo.comments += 1
|
||||
of tkKeyword:
|
||||
if tok.value in ["import", "from"]:
|
||||
var j = i + 1
|
||||
while j < tokens.len and tokens[j].kind in {tkWhitespace, tkNewline}:
|
||||
j += 1
|
||||
if j < tokens.len and tokens[j].kind == tkIdentifier:
|
||||
var importInfo = ImportInfo(module: tokens[j].value, position: tokens[j].position)
|
||||
if tok.value == "from":
|
||||
# Find 'import' keyword
|
||||
var k = j + 1
|
||||
while k < tokens.len and tokens[k].kind notin {tkKeyword, tkNewline, tkEndOfFile}:
|
||||
k += 1
|
||||
if k < tokens.len and tokens[k].kind == tkKeyword and tokens[k].value == "import":
|
||||
k += 1
|
||||
while k < tokens.len and tokens[k].kind in {tkWhitespace, tkNewline}:
|
||||
k += 1
|
||||
while k < tokens.len and tokens[k].kind notin {tkNewline, tkEndOfFile}:
|
||||
if tokens[k].kind == tkIdentifier:
|
||||
importInfo.symbols.add(tokens[k].value)
|
||||
k += 1
|
||||
elif i > 0 and tokens[i-1].kind == tkKeyword and tokens[i-1].value == "from":
|
||||
discard # handled above
|
||||
self.result.moduleInfo.imports.add(importInfo)
|
||||
elif tok.value == "def":
|
||||
# Function definition
|
||||
var j = i + 1
|
||||
while j < tokens.len and tokens[j].kind in {tkWhitespace, tkNewline}:
|
||||
j += 1
|
||||
if j < tokens.len and tokens[j].kind == tkIdentifier:
|
||||
var funcInfo = FunctionInfo(name: tokens[j].value, kind: "def", position: tokens[j].position)
|
||||
self.result.moduleInfo.functions.add(funcInfo)
|
||||
elif tok.value == "class":
|
||||
var j = i + 1
|
||||
while j < tokens.len and tokens[j].kind in {tkWhitespace, tkNewline}:
|
||||
j += 1
|
||||
if j < tokens.len and tokens[j].kind == tkIdentifier:
|
||||
var classInfo = ClassInfo(name: tokens[j].value, kind: "class", position: tokens[j].position)
|
||||
self.result.moduleInfo.classes.add(classInfo)
|
||||
else:
|
||||
discard
|
||||
i += 1
|
||||
|
||||
when defined(validatrixDebug):
|
||||
if self.options.debugMode:
|
||||
debugLeave("PYTHON_ANALYZE")
|
||||
|
||||
proc init*() =
|
||||
registerValidator("python", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
|
||||
result = newPythonValidator(source, options, sourcePath))
|
||||
registerValidator("py", proc(source: string, options: ValidationOptions, sourcePath: string): ValidatorBase =
|
||||
result = newPythonValidator(source, options, sourcePath))
|
||||
|
||||
init()
|
||||
@@ -0,0 +1,187 @@
|
||||
## Error reporting and formatting for Validatrix.
|
||||
##
|
||||
## Provides consistent error formatting across all language validators,
|
||||
## including human-readable and machine-readable (JSON) output.
|
||||
|
||||
import std/[strformat, json, tables]
|
||||
import ../core/types
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Error formatter
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
type
|
||||
ErrorFormatter* = object
|
||||
## Formats validation errors consistently.
|
||||
includeSourceContext*: bool
|
||||
includeColorHint*: bool
|
||||
maxErrors*: int
|
||||
|
||||
proc newErrorFormatter*(
|
||||
srcCtx: bool = true,
|
||||
colorHint: bool = false,
|
||||
maxErr: int = 0
|
||||
): ErrorFormatter =
|
||||
## Create a new error formatter.
|
||||
result.includeSourceContext = srcCtx
|
||||
result.includeColorHint = colorHint
|
||||
result.maxErrors = maxErr
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Format single error as human-readable string
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
proc formatError*(self: ErrorFormatter, err: ValidationError, source: string = ""): string =
|
||||
## Format a single validation error as a human-readable string.
|
||||
let posStr = &"L{err.position.line}:{err.position.column}"
|
||||
let codeStr = &"[{err.code}]"
|
||||
let sevStr = &"({$err.severity})"
|
||||
let msgStr = err.message
|
||||
|
||||
result = &" {codeStr} {sevStr} {posStr}: {msgStr}"
|
||||
|
||||
if self.includeSourceContext and source.len > 0 and err.position.line > 0:
|
||||
let lines = source.splitLines()
|
||||
let lineIdx = err.position.line - 1
|
||||
if lineIdx >= 0 and lineIdx < lines.len:
|
||||
let sourceLine = lines[lineIdx]
|
||||
result.add("\n" & " | " & sourceLine)
|
||||
if err.position.column > 0:
|
||||
let caretCol = max(err.position.column - 1, 0)
|
||||
let caret = " ".repeat(caretCol) & "^"
|
||||
result.add("\n" & " | " & caret)
|
||||
|
||||
if self.includeColorHint and err.hint.len > 0:
|
||||
result.add("\n" & " hint: " & err.hint)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Format all errors
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
proc formatAllErrors*(self: ErrorFormatter, errors: seq[ValidationError], source: string = ""): string =
|
||||
## Format all errors as a human-readable multi-line string.
|
||||
if errors.len == 0:
|
||||
return " No errors found."
|
||||
|
||||
var parts: seq[string] = @[]
|
||||
let count = if self.maxErrors > 0: min(errors.len, self.maxErrors) else: errors.len
|
||||
for i in 0..<count:
|
||||
parts.add(self.formatError(errors[i], source))
|
||||
|
||||
result = parts.join("\n")
|
||||
|
||||
if self.maxErrors > 0 and errors.len > self.maxErrors:
|
||||
result.add(&"\n ... and {errors.len - self.maxErrors} more errors (use maxErrors=0 to show all)")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Format validation result as human-readable report
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
proc formatReport*(vr: ValidationResult, source: string = ""): string =
|
||||
## Format the complete validation result as a human-readable report.
|
||||
let formatter = newErrorFormatter(srcCtx = true, colorHint = true)
|
||||
|
||||
var lines: seq[string] = @[]
|
||||
|
||||
lines.add(&"=== Validation Report ===")
|
||||
lines.add(&" Flavor: {$vr.flavor}")
|
||||
lines.add(&" Detection: {$vr.detectionMethod}")
|
||||
lines.add(&" Valid: {vr.valid}")
|
||||
lines.add(&" Errors: {vr.errors.len}")
|
||||
lines.add(&" Warnings: {vr.warnings}")
|
||||
lines.add(&" Duration: {vr.durationMs:.2f}ms")
|
||||
lines.add(&" Lines of code: {vr.moduleInfo.linesOfCode}")
|
||||
lines.add(&" Tokens: {vr.moduleInfo.totalTokens}")
|
||||
lines.add("")
|
||||
|
||||
if vr.errors.len > 0:
|
||||
lines.add(" --- Findings ---")
|
||||
for err in vr.errors:
|
||||
lines.add(formatter.formatError(err, source))
|
||||
else:
|
||||
lines.add(" No issues found.")
|
||||
|
||||
lines.add("")
|
||||
lines.add(" --- Summary ---")
|
||||
if vr.valid:
|
||||
lines.add(" VALID - source passed all checks.")
|
||||
else:
|
||||
lines.add(" INVALID - source has issues that need attention.")
|
||||
|
||||
if vr.debugOutput.len > 0:
|
||||
lines.add("")
|
||||
lines.add(" --- Debug Output ---")
|
||||
for line in vr.debugOutput.splitLines():
|
||||
lines.add(" " & line)
|
||||
|
||||
result = lines.join("\n")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# JSON formatting (comprehensive)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
proc toJsonReport*(vr: ValidationResult): JsonNode =
|
||||
## Full validation result as JSON (same as result.toJson()).
|
||||
vr.toJson()
|
||||
|
||||
proc toJsonReportString*(vr: ValidationResult, pretty: bool = true): string =
|
||||
## Validation result as JSON string.
|
||||
let j = vr.toJson()
|
||||
if pretty:
|
||||
result = j.pretty()
|
||||
else:
|
||||
result = $j
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Error counting / categorization
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
proc countBySeverity*(errors: seq[ValidationError]): Table[string, int] =
|
||||
## Count errors by severity.
|
||||
result = initTable[string, int]()
|
||||
for err in errors:
|
||||
let key = $err.severity
|
||||
result.mgetOrPut(key, 0).inc
|
||||
|
||||
proc countByCode*(errors: seq[ValidationError]): Table[string, int] =
|
||||
## Count errors by error code.
|
||||
result = initTable[string, int]()
|
||||
for err in errors:
|
||||
result.mgetOrPut(err.code, 0).inc
|
||||
|
||||
proc filterBySeverity*(errors: seq[ValidationError], minSeverity: ErrorSeverity): seq[ValidationError] =
|
||||
## Filter errors by minimum severity level.
|
||||
let severities: array[4, ErrorSeverity] = [esInfo, esWarning, esError, esCritical]
|
||||
var minIdx = 0
|
||||
for i, s in severities:
|
||||
if s == minSeverity:
|
||||
minIdx = i
|
||||
break
|
||||
|
||||
for err in errors:
|
||||
var errIdx = 0
|
||||
for i, s in severities:
|
||||
if s == err.severity:
|
||||
errIdx = i
|
||||
break
|
||||
if errIdx >= minIdx:
|
||||
result.add(err)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Error severity ordering for filtering
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
proc `>`*(a, b: ErrorSeverity): bool =
|
||||
## Compare severities: critical > error > warning > info.
|
||||
const order: array[4, int] = [0, 1, 2, 3]
|
||||
order[ord(a)] > order[ord(b)]
|
||||
|
||||
proc `<`*(a, b: ErrorSeverity): bool =
|
||||
const order: array[4, int] = [0, 1, 2, 3]
|
||||
order[ord(a)] < order[ord(b)]
|
||||
|
||||
proc `<=`*(a, b: ErrorSeverity): bool =
|
||||
a < b or a == b
|
||||
|
||||
proc `>=`*(a, b: ErrorSeverity): bool =
|
||||
a > b or a == b
|
||||
@@ -0,0 +1,317 @@
|
||||
## Bash/Shell tokenizer for Validatrix.
|
||||
##
|
||||
## Tokenizes Bash/Sh/Zsh source code including:
|
||||
## - Here-documents (<<EOF)
|
||||
## - Variable expansion ($var, ${var})
|
||||
## - Command substitution ($(), ``)
|
||||
## - Arrays and brace expansion
|
||||
|
||||
import std/[strformat, sets, json, sequtils]
|
||||
{.warning[UnusedImport]:off.}
|
||||
import ../core/types, ../core/tokenizerbase
|
||||
|
||||
type
|
||||
BashTokenizer* = ref object of TokenizerBase
|
||||
## Tokenizer for Bash/shell scripts.
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Bash keywords
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
const
|
||||
bashKeywords* = @[
|
||||
"if", "then", "else", "elif", "fi", "case", "esac", "for", "while",
|
||||
"until", "do", "done", "in", "function", "select", "time", "until",
|
||||
"return", "break", "continue", "exit", "declare", "local", "export",
|
||||
"readonly", "unset", "eval", "exec", "let", "source", ".", "shift",
|
||||
"trap", "type", "typeset", "ulimit", "umask", "alias", "unalias",
|
||||
"read", "echo", "printf", "test", "[", "]]", "set", "unset"
|
||||
]
|
||||
|
||||
bashKeywordSet* = bashKeywords.toHashSet()
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Here-document tracking
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
var heredocDelimiters* {.used.}: seq[string] = @[]
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Construction
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
proc newBashTokenizer*(source: string, options: ValidationOptions = newValidationOptions()): BashTokenizer =
|
||||
## Create a new Bash tokenizer.
|
||||
result = BashTokenizer(
|
||||
source: source,
|
||||
sourceLen: source.len,
|
||||
options: options,
|
||||
tokens: @[],
|
||||
pos: 0,
|
||||
line: 1,
|
||||
column: 1,
|
||||
bracketStack: @[],
|
||||
errors: @[],
|
||||
eof: false
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tokenization
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
method tokenize*(self: BashTokenizer) =
|
||||
## Tokenize Bash source code.
|
||||
when defined(validatrixDebug):
|
||||
if self.options.debugMode:
|
||||
debugEnter("BASH_TOKENIZER", "Starting Bash tokenization")
|
||||
|
||||
while self.hasMore():
|
||||
let startPos = self.currentPos()
|
||||
let c = self.peek()
|
||||
|
||||
# Whitespace
|
||||
if c == ' ' or c == '\t' or c == '\r':
|
||||
let wsStart = self.currentPos()
|
||||
var ws = ""
|
||||
while self.hasMore() and (self.peek() == ' ' or self.peek() == '\t' or self.peek() == '\r'):
|
||||
ws.add(self.advance())
|
||||
self.emitToken(tkWhitespace, ws, wsStart)
|
||||
|
||||
# Newline
|
||||
elif c == '\n':
|
||||
let nlPos = self.currentPos()
|
||||
discard self.advance()
|
||||
self.emitToken(tkNewline, "\n", nlPos)
|
||||
|
||||
# Line comment (#)
|
||||
elif c == '#':
|
||||
# Check for shebang
|
||||
if self.pos == 0 and self.hasMore() and self.peek(1) == '!':
|
||||
let shStart = self.currentPos()
|
||||
var line = ""
|
||||
while self.hasMore() and self.peek() != '\n':
|
||||
line.add(self.advance())
|
||||
self.emitToken(tkDirective, line, shStart)
|
||||
else:
|
||||
let comStart = self.currentPos()
|
||||
discard self.advance()
|
||||
var content = "#" & self.parseLineComment()
|
||||
self.emitToken(tkComment, content, comStart)
|
||||
|
||||
# Single-quoted string (no escaping in bash)
|
||||
elif c == '\'':
|
||||
let strStart = self.currentPos()
|
||||
discard self.advance() # '
|
||||
var content = "'"
|
||||
while self.hasMore() and self.peek() != '\'':
|
||||
content.add(self.advance())
|
||||
if self.hasMore():
|
||||
discard self.advance() # closing '
|
||||
content.add('\'')
|
||||
self.emitToken(tkString, content, strStart)
|
||||
else:
|
||||
self.recordError(esError, "Unclosed single-quoted string", ErrUnclosedString, strStart)
|
||||
self.emitToken(tkString, content, strStart)
|
||||
|
||||
# Double-quoted string
|
||||
elif c == '"':
|
||||
let strStart = self.currentPos()
|
||||
discard self.advance() # "
|
||||
var content = "\""
|
||||
var foundClosing = false
|
||||
while self.hasMore():
|
||||
let sc = self.advance()
|
||||
if sc == '\\':
|
||||
content.add(sc)
|
||||
if self.hasMore():
|
||||
content.add(self.advance())
|
||||
elif sc == '"':
|
||||
content.add('"')
|
||||
self.emitToken(tkString, content, strStart)
|
||||
foundClosing = true
|
||||
break
|
||||
elif sc == '\n':
|
||||
self.emitToken(tkString, "\"", strStart)
|
||||
foundClosing = true
|
||||
break
|
||||
else:
|
||||
content.add(sc)
|
||||
if not foundClosing:
|
||||
self.recordError(esError, "Unclosed double-quoted string", ErrUnclosedString, strStart)
|
||||
self.emitToken(tkString, content, strStart)
|
||||
|
||||
# Here-document detection (<<EOF, <<-EOF, <<'EOF', etc.)
|
||||
elif self.peekString(2) == "<<" and self.pos > 0:
|
||||
# Check if preceded by whitespace/newline (simple check)
|
||||
let hdStart = self.currentPos()
|
||||
discard self.advance() # <
|
||||
discard self.advance() # <
|
||||
var heredocContent = "<<"
|
||||
# Skip optional -
|
||||
if self.hasMore() and self.peek() == '-':
|
||||
heredocContent.add(self.advance())
|
||||
# Read delimiter
|
||||
self.skipWhitespace(false)
|
||||
var delim = ""
|
||||
while self.hasMore() and self.peek() != '\n' and self.peek() != ' ' and self.peek() != '\t':
|
||||
delim.add(self.advance())
|
||||
# Mark that we've seen heredoc (simplified -- we don't track content)
|
||||
heredocContent.add(delim)
|
||||
self.emitToken(tkDirective, heredocContent, hdStart)
|
||||
|
||||
# Variable expansion
|
||||
elif c == '$':
|
||||
let varStart = self.currentPos()
|
||||
discard self.advance() # $
|
||||
if self.hasMore() and self.peek() == '{':
|
||||
discard self.advance() # {
|
||||
var content = "${"
|
||||
while self.hasMore() and self.peek() != '}' and self.peek() != '\n':
|
||||
content.add(self.advance())
|
||||
if self.hasMore() and self.peek() == '}':
|
||||
content.add(self.advance()) # }
|
||||
self.emitToken(tkInterpolation, content, varStart)
|
||||
else:
|
||||
self.recordError(esError, "Unclosed variable expansion '${}'", ErrUnclosedBlock, varStart)
|
||||
self.emitToken(tkInterpolation, content, varStart)
|
||||
elif self.hasMore() and self.peek() == '(':
|
||||
discard self.advance() # (
|
||||
var content = "$("
|
||||
# Find matching )
|
||||
var parenDepth = 1
|
||||
while self.hasMore() and parenDepth > 0:
|
||||
let nc = self.advance()
|
||||
if nc == '(':
|
||||
parenDepth += 1
|
||||
elif nc == ')':
|
||||
parenDepth -= 1
|
||||
if parenDepth > 0:
|
||||
content.add(nc)
|
||||
if parenDepth == 0:
|
||||
content.add(')')
|
||||
else:
|
||||
self.recordError(esError, "Unclosed command substitution '$('", ErrUnclosedBlock, varStart)
|
||||
self.emitToken(tkInterpolation, content, varStart)
|
||||
else:
|
||||
# Simple variable $var
|
||||
var varName = "$"
|
||||
while self.hasMore() and (isAlphaNum(self.peek()) or self.peek() == '_'):
|
||||
varName.add(self.advance())
|
||||
self.emitToken(tkInterpolation, varName, varStart)
|
||||
|
||||
# Numbers
|
||||
elif isDigit(c):
|
||||
let numStart = self.currentPos()
|
||||
var num = ""
|
||||
while self.hasMore() and (self.peek().isDigit() or self.peek() == '.'):
|
||||
num.add(self.advance())
|
||||
self.emitToken(tkNumber, num, numStart)
|
||||
|
||||
# Identifiers and keywords
|
||||
elif isAlpha(c):
|
||||
let idStart = self.currentPos()
|
||||
var ident = ""
|
||||
while self.hasMore() and (isAlphaNum(self.peek()) or self.peek() == '_'):
|
||||
ident.add(self.advance())
|
||||
if bashKeywordSet.contains(ident):
|
||||
self.emitToken(tkKeyword, ident, idStart)
|
||||
else:
|
||||
self.emitToken(tkIdentifier, ident, idStart)
|
||||
|
||||
# Backtick command substitution
|
||||
elif c == '`':
|
||||
let btStart = self.currentPos()
|
||||
discard self.advance()
|
||||
var content = "`"
|
||||
while self.hasMore() and self.peek() != '`':
|
||||
content.add(self.advance())
|
||||
if self.hasMore():
|
||||
discard self.advance()
|
||||
content.add('`')
|
||||
self.emitToken(tkInterpolation, content, btStart)
|
||||
|
||||
# Brackets
|
||||
elif c == '(':
|
||||
self.emitBracketToken(kOpenParen, $self.advance(), startPos)
|
||||
elif c == ')':
|
||||
self.emitBracketToken(kCloseParen, $self.advance(), startPos)
|
||||
if self.bracketStack.len > 0 and self.bracketStack[^1][0] == '(':
|
||||
discard self.bracketStack.pop()
|
||||
elif c == '[':
|
||||
if self.peekString(2) == "[[": # [[ test
|
||||
self.emitToken(tkKeyword, "[[", startPos)
|
||||
discard self.advance()
|
||||
discard self.advance()
|
||||
else:
|
||||
self.emitBracketToken(kOpenBracket, $self.advance(), startPos)
|
||||
elif c == ']':
|
||||
if self.pos > 1 and self.source[self.pos-1] == '[': # simplified
|
||||
discard self.advance()
|
||||
discard self.advance()
|
||||
self.emitToken(tkKeyword, "]]", startPos)
|
||||
else:
|
||||
self.emitBracketToken(kCloseBracket, $self.advance(), startPos)
|
||||
if self.bracketStack.len > 0 and self.bracketStack[^1][0] == '[':
|
||||
discard self.bracketStack.pop()
|
||||
elif c == '{':
|
||||
self.emitBracketToken(kOpenBrace, $self.advance(), startPos)
|
||||
elif c == '}':
|
||||
self.emitBracketToken(kCloseBrace, $self.advance(), startPos)
|
||||
if self.bracketStack.len > 0 and self.bracketStack[^1][0] == '{':
|
||||
discard self.bracketStack.pop()
|
||||
|
||||
# Operators and special chars
|
||||
elif c in {'=', '<', '>', '|', '&', ';', '!'}:
|
||||
let opStart = self.currentPos()
|
||||
var op = ""
|
||||
while self.hasMore() and self.peek() in {'=', '>', '<', '|', '&', ';', '!'}:
|
||||
op.add(self.advance())
|
||||
if op == "=":
|
||||
self.emitToken(tkAssignment, op, opStart)
|
||||
elif op == "<<" or op == ">>" or op == "<" or op == ">":
|
||||
let bracketKind = if op[0] == '<': kAngleOpen else: kAngleClose
|
||||
self.emitBracketToken(bracketKind, op, opStart)
|
||||
elif op == "||" or op == "&&" or op == "|":
|
||||
self.emitToken(tkOperator, op, opStart)
|
||||
elif op == ";" or op == ";;":
|
||||
self.emitToken(tkPunctuation, op, opStart)
|
||||
else:
|
||||
self.emitToken(tkOperator, op, opStart)
|
||||
|
||||
# Backslash continuation
|
||||
elif c == '\\':
|
||||
let bsStart = self.currentPos()
|
||||
discard self.advance() # \
|
||||
if self.hasMore() and self.peek() == '\n':
|
||||
discard self.advance() # newline continuation
|
||||
self.emitToken(tkSpecial, "\\\n", bsStart)
|
||||
else:
|
||||
self.emitToken(tkOperator, "\\", bsStart)
|
||||
|
||||
# Arithmetic expansion $((...))
|
||||
elif self.peekString(3) == "((":
|
||||
let arStart = self.currentPos()
|
||||
discard self.advance()
|
||||
discard self.advance()
|
||||
var parenDepth = 2
|
||||
var content = "(("
|
||||
while self.hasMore() and parenDepth > 0:
|
||||
let nc = self.advance()
|
||||
if nc == '(': parenDepth += 1
|
||||
elif nc == ')': parenDepth -= 1
|
||||
if parenDepth > 0: content.add(nc)
|
||||
if parenDepth == 0:
|
||||
content.add("))")
|
||||
self.emitToken(tkInterpolation, content, arStart)
|
||||
|
||||
# Unknown character
|
||||
else:
|
||||
let uStart = self.currentPos()
|
||||
self.emitToken(tkSpecial, $self.advance(), uStart)
|
||||
|
||||
# End of file
|
||||
self.emitToken(tkEndOfFile, "", self.currentPos())
|
||||
|
||||
when defined(validatrixDebug):
|
||||
if self.options.debugMode:
|
||||
debugLeave("BASH_TOKENIZER", &"Tokenization complete: {self.tokens.len} tokens")
|
||||
@@ -0,0 +1,162 @@
|
||||
## HTML/XML tokenizer for Validatrix.
|
||||
|
||||
import std/[strformat, sets, json, sequtils, unicode]
|
||||
{.warning[UnusedImport]:off.}
|
||||
import ../core/types, ../core/tokenizerbase
|
||||
|
||||
type
|
||||
HTMLTokenizer* = ref object of TokenizerBase
|
||||
isXml*: bool
|
||||
|
||||
const
|
||||
voidElements* = @[
|
||||
"area", "base", "br", "col", "embed", "hr", "img", "input",
|
||||
"link", "meta", "param", "source", "track", "wbr"
|
||||
]
|
||||
voidElementSet* = voidElements.toHashSet()
|
||||
|
||||
proc newHTMLTokenizer*(source: string, options: ValidationOptions = newValidationOptions(), isXml: bool = false): HTMLTokenizer =
|
||||
result = HTMLTokenizer(
|
||||
source: source,
|
||||
sourceLen: source.len,
|
||||
options: options,
|
||||
tokens: @[],
|
||||
pos: 0,
|
||||
line: 1,
|
||||
column: 1,
|
||||
bracketStack: @[],
|
||||
errors: @[],
|
||||
eof: false
|
||||
)
|
||||
result.isXml = isXml
|
||||
|
||||
method tokenize*(self: HTMLTokenizer) =
|
||||
when defined(validatrixDebug):
|
||||
if self.options.debugMode:
|
||||
debugEnter("HTML_TOKENIZER", "Starting HTML tokenization")
|
||||
|
||||
var tagStack: seq[string] = @[]
|
||||
|
||||
while self.hasMore():
|
||||
let startPos = self.currentPos()
|
||||
let c = self.peek()
|
||||
|
||||
if c == '\n':
|
||||
self.emitToken(tkNewline, $self.advance(), startPos)
|
||||
continue
|
||||
|
||||
if c == ' ' or c == '\t' or c == '\r':
|
||||
var ws = ""
|
||||
while self.hasMore() and (self.peek() == ' ' or self.peek() == '\t' or self.peek() == '\r'):
|
||||
ws.add(self.advance())
|
||||
self.emitToken(tkWhitespace, ws, startPos)
|
||||
continue
|
||||
|
||||
# HTML comment
|
||||
if self.peekString(4) == "<!--":
|
||||
let comStart = self.currentPos()
|
||||
for _ in 1..4: discard self.advance()
|
||||
var content = "<!--"
|
||||
while self.hasMore() and self.peekString(3) != "-->":
|
||||
content.add(self.advance())
|
||||
if self.peekString(3) == "-->":
|
||||
for _ in 1..3: discard self.advance()
|
||||
content.add("-->")
|
||||
self.emitToken(tkComment, content, comStart)
|
||||
else:
|
||||
self.recordError(esError, "Unclosed HTML comment", ErrUnclosedComment, comStart)
|
||||
continue
|
||||
|
||||
# Opening tag
|
||||
if c == '<':
|
||||
let tagStart = self.currentPos()
|
||||
discard self.advance() # <
|
||||
|
||||
# Check for closing tag </
|
||||
var isClosing = false
|
||||
if self.hasMore() and self.peek() == '/':
|
||||
isClosing = true
|
||||
discard self.advance()
|
||||
|
||||
# Read tag name
|
||||
if self.hasMore() and isAlpha(self.peek()):
|
||||
var tagName = ""
|
||||
while self.hasMore() and (isAlphaNum(self.peek()) or self.peek() == '-' or self.peek() == ':'):
|
||||
tagName.add(self.advance())
|
||||
|
||||
if isClosing:
|
||||
# Closing tag
|
||||
# Check if matches last opened
|
||||
if tagStack.len > 0 and tagStack[^1] == tagName:
|
||||
discard tagStack.pop()
|
||||
elif not voidElementSet.contains(tagName):
|
||||
self.recordError(esWarning,
|
||||
&"Unexpected closing tag '</{tagName}>' (expected '</{tagStack[^1]}>' if stack not empty)",
|
||||
ErrMismatchedBracket, tagStart,
|
||||
hint = "Check that tags are properly nested")
|
||||
# Read rest of closing tag
|
||||
while self.hasMore() and self.peek() != '>':
|
||||
discard self.advance()
|
||||
if self.hasMore(): discard self.advance() # >
|
||||
self.emitToken(tkPunctuation, &"</{tagName}>", tagStart)
|
||||
else:
|
||||
# Opening tag
|
||||
if not voidElementSet.contains(tagName) and not self.isXml:
|
||||
tagStack.add(tagName)
|
||||
# Read attributes until >
|
||||
var tagContent = "<" & tagName
|
||||
while self.hasMore():
|
||||
let nc = self.peek()
|
||||
if nc == '>':
|
||||
tagContent.add(self.advance())
|
||||
break
|
||||
elif nc == '/' and self.hasMore() and self.peek(1) == '>':
|
||||
tagContent.add(self.advance())
|
||||
tagContent.add(self.advance())
|
||||
if not voidElementSet.contains(tagName) and not self.isXml:
|
||||
discard tagStack.pop()
|
||||
break
|
||||
elif nc == '"' or nc == '\'':
|
||||
let quote = nc
|
||||
tagContent.add(self.advance())
|
||||
while self.hasMore() and self.peek() != quote:
|
||||
tagContent.add(self.advance())
|
||||
if self.hasMore():
|
||||
tagContent.add(self.advance())
|
||||
else:
|
||||
tagContent.add(self.advance())
|
||||
self.emitToken(tkTemplateTag, tagContent, tagStart)
|
||||
else:
|
||||
# Not a real tag (could be <!DOCTYPE, <?xml, etc.)
|
||||
var rest = "<"
|
||||
while self.hasMore() and self.peek() != '>':
|
||||
rest.add(self.advance())
|
||||
if self.hasMore(): rest.add(self.advance())
|
||||
self.emitToken(tkDirective, rest, tagStart)
|
||||
continue
|
||||
|
||||
# Plain text content
|
||||
var text = ""
|
||||
while self.hasMore():
|
||||
let nc = self.peek()
|
||||
if nc == '<' or nc == '\n': break
|
||||
text.add(self.advance())
|
||||
if text.len > 0:
|
||||
self.emitToken(tkTemplateTag, text, startPos)
|
||||
continue
|
||||
|
||||
# Single char fallback
|
||||
self.emitToken(tkSpecial, $self.advance(), startPos)
|
||||
|
||||
# Check for unclosed tags
|
||||
for tag in tagStack:
|
||||
self.recordError(esError,
|
||||
&"Unclosed HTML tag '<{tag}>'",
|
||||
ErrUnclosedBlock, newSourcePosition(1, 1, 0),
|
||||
hint = &"Add a closing '</{tag}>' tag"
|
||||
)
|
||||
|
||||
self.emitToken(tkEndOfFile, "", self.currentPos())
|
||||
when defined(validatrixDebug):
|
||||
if self.options.debugMode:
|
||||
debugLeave("HTML_TOKENIZER", &"Complete: {self.tokens.len} tokens")
|
||||
@@ -0,0 +1,228 @@
|
||||
## JavaScript/TypeScript tokenizer for Validatrix.
|
||||
|
||||
import std/[strformat, sets, json, sequtils, unicode]
|
||||
{.warning[UnusedImport]:off.}
|
||||
import ../core/types, ../core/tokenizerbase
|
||||
|
||||
type
|
||||
JavaScriptTokenizer* = ref object of TokenizerBase
|
||||
|
||||
const
|
||||
jsKeywords = @[
|
||||
"async", "await", "break", "case", "catch", "class", "const", "continue",
|
||||
"debugger", "default", "delete", "do", "else", "enum", "export", "extends",
|
||||
"false", "finally", "for", "function", "if", "import", "in", "instanceof",
|
||||
"let", "new", "null", "of", "return", "super", "switch", "static", "this",
|
||||
"throw", "true", "try", "typeof", "var", "void", "while", "with", "yield",
|
||||
"interface", "implements", "package", "private", "protected", "public",
|
||||
"type", "declare", "module", "namespace", "abstract", "readonly",
|
||||
"as", "any", "boolean", "number", "string", "undefined", "never",
|
||||
"keyof", "infer", "unknown"
|
||||
]
|
||||
jsKeywordSet = jsKeywords.toHashSet()
|
||||
|
||||
proc newJavaScriptTokenizer*(source: string, options: ValidationOptions = newValidationOptions()): JavaScriptTokenizer =
|
||||
result = JavaScriptTokenizer(
|
||||
source: source,
|
||||
sourceLen: source.len,
|
||||
options: options,
|
||||
tokens: @[],
|
||||
pos: 0,
|
||||
line: 1,
|
||||
column: 1,
|
||||
bracketStack: @[],
|
||||
errors: @[],
|
||||
eof: false
|
||||
)
|
||||
|
||||
method tokenize*(self: JavaScriptTokenizer) =
|
||||
when defined(validatrixDebug):
|
||||
if self.options.debugMode:
|
||||
debugEnter("JS_TOKENIZER", "Starting JS/TS tokenization")
|
||||
|
||||
while self.hasMore():
|
||||
let startPos = self.currentPos()
|
||||
let c = self.peek()
|
||||
|
||||
if c == ' ' or c == '\t' or c == '\r':
|
||||
let wsStart = self.currentPos()
|
||||
var ws = ""
|
||||
while self.hasMore() and (self.peek() == ' ' or self.peek() == '\t' or self.peek() == '\r'):
|
||||
ws.add(self.advance())
|
||||
self.emitToken(tkWhitespace, ws, wsStart)
|
||||
|
||||
elif c == '\n':
|
||||
self.emitToken(tkNewline, $self.advance(), startPos)
|
||||
|
||||
# Template literals
|
||||
elif c == '`':
|
||||
let tlStart = self.currentPos()
|
||||
discard self.advance()
|
||||
var content = "`"
|
||||
var braceDepth = 0
|
||||
var foundClosing = false
|
||||
while self.hasMore():
|
||||
let tc = self.advance()
|
||||
if tc == '\\':
|
||||
content.add(tc)
|
||||
if self.hasMore(): content.add(self.advance())
|
||||
elif tc == '`' and braceDepth == 0:
|
||||
content.add('`')
|
||||
self.emitToken(tkString, content, tlStart)
|
||||
foundClosing = true
|
||||
break
|
||||
elif tc == '$' and self.hasMore() and self.peek() == '{' and braceDepth == 0:
|
||||
content.add("${")
|
||||
discard self.advance()
|
||||
braceDepth += 1
|
||||
elif tc == '{': braceDepth += 1; content.add(tc)
|
||||
elif tc == '}': braceDepth -= 1; content.add(tc)
|
||||
else: content.add(tc)
|
||||
if not foundClosing:
|
||||
self.recordError(esError, "Unclosed template literal", ErrUnclosedString, tlStart)
|
||||
self.emitToken(tkString, content, tlStart)
|
||||
|
||||
# Line comment
|
||||
elif self.peekString(2) == "//":
|
||||
let comStart = self.currentPos()
|
||||
discard self.advance()
|
||||
discard self.advance()
|
||||
self.emitToken(tkComment, "//" & self.parseLineComment(), comStart)
|
||||
|
||||
# Block comment
|
||||
elif self.peekString(2) == "/*":
|
||||
let comStart = self.currentPos()
|
||||
discard self.advance()
|
||||
discard self.advance()
|
||||
let content = self.parseBlockComment("/*", "*/")
|
||||
let isDoc = content.startsWith("*")
|
||||
let comKind = if isDoc: tkDocComment else: tkComment
|
||||
self.emitToken(comKind, "/*" & content & "*/", comStart)
|
||||
|
||||
# Regex literal (simplified: /.../ not at start of expression)
|
||||
elif c == '/' and self.pos > 0:
|
||||
let prevChar = if self.pos > 0: self.source[self.pos - 1] else: '\0'
|
||||
if prevChar in {' ', '\t', '\n', '\r', '=', '(', '[', '{', '!', '&', '|', ':', ';', ',', '^', '~'}:
|
||||
let regStart = self.currentPos()
|
||||
discard self.advance()
|
||||
var regex = "/"
|
||||
var foundClosing = false
|
||||
while self.hasMore():
|
||||
let rc = self.advance()
|
||||
if rc == '\\':
|
||||
regex.add(rc)
|
||||
if self.hasMore(): regex.add(self.advance())
|
||||
elif rc == '/':
|
||||
regex.add('/')
|
||||
# Flags
|
||||
while self.hasMore() and (isAlpha(self.peek())):
|
||||
regex.add(self.advance())
|
||||
self.emitToken(tkSpecial, regex, regStart)
|
||||
foundClosing = true
|
||||
foundClosing = true
|
||||
break
|
||||
else:
|
||||
regex.add(rc)
|
||||
if not foundClosing:
|
||||
self.emitToken(tkOperator, "/", regStart)
|
||||
else:
|
||||
self.emitToken(tkOperator, "/", startPos)
|
||||
discard self.advance()
|
||||
|
||||
# Strings
|
||||
elif c == '"' or c == '\'':
|
||||
let quote = c
|
||||
let strStart = self.currentPos()
|
||||
discard self.advance()
|
||||
var content = $quote
|
||||
var foundClosing = false
|
||||
while self.hasMore():
|
||||
let sc = self.advance()
|
||||
if sc == '\\':
|
||||
content.add(sc)
|
||||
if self.hasMore(): content.add(self.advance())
|
||||
elif sc == quote:
|
||||
content.add(quote)
|
||||
self.emitToken(tkString, content, strStart)
|
||||
foundClosing = true
|
||||
foundClosing = true
|
||||
break
|
||||
elif sc == '\n':
|
||||
self.recordError(esError, "Newline in string", ErrUnclosedString, strStart)
|
||||
foundClosing = true
|
||||
foundClosing = true
|
||||
break
|
||||
else:
|
||||
content.add(sc)
|
||||
if not foundClosing:
|
||||
self.recordError(esError, "Unclosed string", ErrUnclosedString, strStart)
|
||||
self.emitToken(tkString, content, strStart)
|
||||
|
||||
elif isDigit(c):
|
||||
let numStart = self.currentPos()
|
||||
var num = ""
|
||||
while self.hasMore() and (isAlphaNum(self.peek()) or self.peek() == '.' or self.peek() == 'x' or self.peek() == 'X' or self.peek() == 'o' or self.peek() == 'O' or self.peek() == 'b' or self.peek() == 'B'):
|
||||
num.add(self.advance())
|
||||
self.emitToken(tkNumber, num, numStart)
|
||||
|
||||
elif isAlpha(c) or c == '$' or c == '_':
|
||||
let idStart = self.currentPos()
|
||||
var ident = ""
|
||||
while self.hasMore() and (isAlphaNum(self.peek()) or self.peek() == '_' or self.peek() == '$'):
|
||||
ident.add(self.advance())
|
||||
if jsKeywordSet.contains(ident):
|
||||
self.emitToken(tkKeyword, ident, idStart)
|
||||
else:
|
||||
self.emitToken(tkIdentifier, ident, idStart)
|
||||
|
||||
# Brackets
|
||||
elif c == '(': self.emitBracketToken(kOpenParen, $self.advance(), startPos)
|
||||
elif c == ')':
|
||||
self.emitBracketToken(kCloseParen, $self.advance(), startPos)
|
||||
if self.bracketStack.len > 0 and self.bracketStack[^1][0] == '(':
|
||||
discard self.bracketStack.pop()
|
||||
elif c == '[': self.emitBracketToken(kOpenBracket, $self.advance(), startPos)
|
||||
elif c == ']':
|
||||
self.emitBracketToken(kCloseBracket, $self.advance(), startPos)
|
||||
if self.bracketStack.len > 0 and self.bracketStack[^1][0] == '[':
|
||||
discard self.bracketStack.pop()
|
||||
elif c == '{': self.emitBracketToken(kOpenBrace, $self.advance(), startPos)
|
||||
elif c == '}':
|
||||
self.emitBracketToken(kCloseBrace, $self.advance(), startPos)
|
||||
if self.bracketStack.len > 0 and self.bracketStack[^1][0] == '{':
|
||||
discard self.bracketStack.pop()
|
||||
|
||||
# Operators
|
||||
elif c in {'+', '-', '*', '/', '%', '^', '~', '|', '&', '<', '>', '=', '!', '?'}:
|
||||
let opStart = self.currentPos()
|
||||
var op = ""
|
||||
while self.hasMore() and self.peek() in {'+', '-', '*', '/', '%', '^', '~', '|', '&', '<', '>', '=', '!', '?', '.'}:
|
||||
let nc = self.peek()
|
||||
# Handle . operator specially (not in multi-char)
|
||||
if nc == '.' and op.len > 0: break
|
||||
op.add(self.advance())
|
||||
if op == "=" or op == "=>":
|
||||
self.emitToken(tkAssignment, op, opStart)
|
||||
elif op == "..." or op == "..":
|
||||
self.emitToken(tkOperator, op, opStart)
|
||||
else:
|
||||
self.emitToken(tkOperator, op, opStart)
|
||||
|
||||
elif c == '.':
|
||||
self.emitToken(tkPunctuation, $self.advance(), startPos)
|
||||
elif c in {',', ';', ':'}:
|
||||
let pStart = self.currentPos()
|
||||
if c == ':' and self.hasMore() and self.peek(1) == '=':
|
||||
discard self.advance()
|
||||
discard self.advance()
|
||||
self.emitToken(tkAssignment, ":=", pStart)
|
||||
else:
|
||||
self.emitToken(tkPunctuation, $self.advance(), pStart)
|
||||
|
||||
else:
|
||||
self.emitToken(tkError, $self.advance(), startPos)
|
||||
|
||||
self.emitToken(tkEndOfFile, "", self.currentPos())
|
||||
when defined(validatrixDebug):
|
||||
if self.options.debugMode:
|
||||
debugLeave("JS_TOKENIZER", &"Complete: {self.tokens.len} tokens")
|
||||
@@ -0,0 +1,123 @@
|
||||
## Jinja2 template tokenizer for Validatrix.
|
||||
##
|
||||
## Handles mixed Jinja2 template syntax ({{ }}, {% %}, {# #})
|
||||
## embedded inside HTML/text content.
|
||||
|
||||
import std/[strformat, sets, json, sequtils]
|
||||
{.warning[UnusedImport]:off.}
|
||||
import ../core/types, ../core/tokenizerbase
|
||||
|
||||
type
|
||||
JinjaTokenizer* = ref object of TokenizerBase
|
||||
inBlockTag*: bool
|
||||
|
||||
const
|
||||
jinjaKeywords = @[
|
||||
"for", "endfor", "if", "endif", "else", "elif", "block", "endblock",
|
||||
"extends", "include", "import", "from", "macro", "endmacro", "set",
|
||||
"endset", "filter", "endfilter", "raw", "endraw", "autoescape",
|
||||
"endautoescape", "call", "endcall", "with", "endwith", "scoped",
|
||||
"recursive", "ignore missing", "without context"
|
||||
]
|
||||
jinjaKeywordSet = jinjaKeywords.toHashSet()
|
||||
|
||||
proc newJinjaTokenizer*(source: string, options: ValidationOptions = newValidationOptions()): JinjaTokenizer =
|
||||
result = JinjaTokenizer(
|
||||
source: source,
|
||||
sourceLen: source.len,
|
||||
options: options,
|
||||
tokens: @[],
|
||||
pos: 0,
|
||||
line: 1,
|
||||
column: 1,
|
||||
bracketStack: @[],
|
||||
errors: @[],
|
||||
eof: false
|
||||
)
|
||||
result.inBlockTag = false
|
||||
|
||||
method tokenize*(self: JinjaTokenizer) =
|
||||
when defined(validatrixDebug):
|
||||
if self.options.debugMode:
|
||||
debugEnter("JINJA_TOKENIZER", "Starting Jinja tokenization")
|
||||
|
||||
while self.hasMore():
|
||||
let startPos = self.currentPos()
|
||||
let c = self.peek()
|
||||
|
||||
if c == '\n':
|
||||
self.emitToken(tkNewline, $self.advance(), startPos)
|
||||
continue
|
||||
|
||||
# Comment tag {# ... #}
|
||||
if self.peekString(2) == "{#":
|
||||
let comStart = self.currentPos()
|
||||
for _ in 1..2: discard self.advance()
|
||||
var content = "{#"
|
||||
while self.hasMore() and self.peekString(2) != "#}":
|
||||
content.add(self.advance())
|
||||
if self.peekString(2) == "#}":
|
||||
for _ in 1..2: discard self.advance()
|
||||
content.add("#}")
|
||||
else:
|
||||
self.recordError(esError, "Unclosed Jinja comment {#", ErrUnclosedComment, comStart)
|
||||
self.emitToken(tkComment, content, comStart)
|
||||
continue
|
||||
|
||||
# Expression tag {{ ... }}
|
||||
if self.peekString(2) == "{{":
|
||||
let exprStart = self.currentPos()
|
||||
for _ in 1..2: discard self.advance()
|
||||
var content = "{{"
|
||||
while self.hasMore():
|
||||
if self.peekString(2) == "}}":
|
||||
for _ in 1..2: discard self.advance()
|
||||
content.add("}}")
|
||||
break
|
||||
content.add(self.advance())
|
||||
self.emitToken(tkInterpolation, content, exprStart)
|
||||
continue
|
||||
|
||||
# Block tag {% ... %}
|
||||
if self.peekString(2) == "{%":
|
||||
let blockStart = self.currentPos()
|
||||
for _ in 1..2: discard self.advance()
|
||||
var content = "{%"
|
||||
while self.hasMore():
|
||||
if self.peekString(2) == "%}":
|
||||
for _ in 1..2: discard self.advance()
|
||||
content.add("%}")
|
||||
break
|
||||
content.add(self.advance())
|
||||
|
||||
# Extract tag name from content for keyword detection
|
||||
let stripped = content[2..^1].strip()
|
||||
let spaceIdx = stripped.find(' ')
|
||||
let tagName = if spaceIdx >= 0: stripped[0..<spaceIdx] else: stripped
|
||||
if jinjaKeywordSet.contains(tagName):
|
||||
self.emitToken(tkKeyword, content, blockStart)
|
||||
else:
|
||||
self.emitToken(tkTemplateTag, content, blockStart)
|
||||
continue
|
||||
|
||||
# Whitespace
|
||||
if c == ' ' or c == '\t' or c == '\r':
|
||||
var ws = ""
|
||||
while self.hasMore() and (self.peek() == ' ' or self.peek() == '\t' or self.peek() == '\r'):
|
||||
ws.add(self.advance())
|
||||
self.emitToken(tkWhitespace, ws, startPos)
|
||||
continue
|
||||
|
||||
# Regular text content
|
||||
var text = ""
|
||||
while self.hasMore():
|
||||
let nc = self.peek()
|
||||
if nc == '\n' or nc == '{': break
|
||||
text.add(self.advance())
|
||||
if text.len > 0:
|
||||
self.emitToken(tkTemplateTag, text, startPos)
|
||||
|
||||
self.emitToken(tkEndOfFile, "", self.currentPos())
|
||||
when defined(validatrixDebug):
|
||||
if self.options.debugMode:
|
||||
debugLeave("JINJA_TOKENIZER", &"Complete: {self.tokens.len} tokens")
|
||||
@@ -0,0 +1,346 @@
|
||||
## Nim language tokenizer for Validatrix.
|
||||
##
|
||||
## Tokenizes Nim source code following Nim's lexical rules including:
|
||||
## - Multi-line strings (""")
|
||||
## - Raw strings (r"...")
|
||||
## - Doc comments (##, ##[ ##])
|
||||
## - Template/macro AST
|
||||
## - Block comments with nesting (#[ ]#)
|
||||
|
||||
import std/[strformat, sets, json, sequtils, unicode]
|
||||
{.warning[UnusedImport]:off.}
|
||||
import ../core/types, ../core/tokenizerbase
|
||||
|
||||
type
|
||||
NimTokenizer* = ref object of TokenizerBase
|
||||
## Tokenizer for Nim source code.
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Nim keywords
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
const
|
||||
nimKeywords* = @[
|
||||
"addr", "and", "as", "asm", "atomic", "bind", "block", "break",
|
||||
"case", "cast", "concept", "const", "continue", "converter", "defer",
|
||||
"discard", "distinct", "div", "do", "elif", "else", "end", "enum",
|
||||
"except", "export", "finally", "for", "from", "func", "if",
|
||||
"import", "in", "include", "interface", "is", "isnot", "iterator",
|
||||
"let", "macro", "method", "mixin", "mod", "nil", "not", "notin",
|
||||
"object", "of", "or", "out", "overridable", "proc", "ptr", "raise",
|
||||
"ref", "result", "return", "self", "shl", "shr", "static", "template",
|
||||
"try", "tuple", "type", "using", "var", "when", "while", "with",
|
||||
"without", "xor", "yield",
|
||||
# Pragmas
|
||||
"compileTime", "noSideEffect", "sideEffect", "gcsafe",
|
||||
"inline", "noinline", "exportc", "importc", "header", "dynlib",
|
||||
"threadVar", "global", "shallow", "borrow", "discardable",
|
||||
"base", "requires", "ensures", "tags", "raises", "locks"
|
||||
]
|
||||
|
||||
nimKeywordSet* = nimKeywords.toHashSet()
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Construction
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
proc newNimTokenizer*(source: string, options: ValidationOptions = newValidationOptions()): NimTokenizer =
|
||||
## Create a new Nim tokenizer.
|
||||
result = NimTokenizer(
|
||||
source: source,
|
||||
sourceLen: source.len,
|
||||
options: options,
|
||||
tokens: @[],
|
||||
pos: 0,
|
||||
line: 1,
|
||||
column: 1,
|
||||
bracketStack: @[],
|
||||
errors: @[],
|
||||
eof: false
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tokenization
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
method tokenize*(self: NimTokenizer) =
|
||||
## Tokenize Nim source code.
|
||||
when defined(validatrixDebug):
|
||||
if self.options.debugMode:
|
||||
debugEnter("NIM_TOKENIZER", "Starting Nim tokenization")
|
||||
|
||||
while self.hasMore():
|
||||
let startPos = self.currentPos()
|
||||
let c = self.peek()
|
||||
|
||||
# Whitespace
|
||||
if c == ' ' or c == '\t' or c == '\r':
|
||||
let wsStart = self.currentPos()
|
||||
var ws = ""
|
||||
while self.hasMore() and (self.peek() == ' ' or self.peek() == '\t' or self.peek() == '\r'):
|
||||
ws.add(self.advance())
|
||||
self.emitToken(tkWhitespace, ws, wsStart)
|
||||
|
||||
# Newline
|
||||
elif c == '\n':
|
||||
let nlPos = self.currentPos()
|
||||
discard self.advance()
|
||||
self.emitToken(tkNewline, "\n", nlPos)
|
||||
|
||||
# Block comment with nesting
|
||||
elif self.peekString(2) == "#[":
|
||||
let comStart = self.currentPos()
|
||||
discard self.advance() # #
|
||||
discard self.advance() # [
|
||||
let content = self.parseBlockComment("#[", "]#")
|
||||
let isDoc = content.len > 0 and content[0] == '#'
|
||||
let kind = if isDoc: tkDocComment else: tkComment
|
||||
self.emitToken(kind, "#[" & content & "]#", comStart)
|
||||
|
||||
# Line comment or doc comment
|
||||
elif c == '#':
|
||||
let comStart = self.currentPos()
|
||||
discard self.advance() # #
|
||||
if self.hasMore() and self.peek() == '#':
|
||||
discard self.advance() # second #
|
||||
var content = "#" & "#" & self.parseLineComment()
|
||||
self.emitToken(tkDocComment, content, comStart)
|
||||
else:
|
||||
var content = "#" & self.parseLineComment()
|
||||
self.emitToken(tkComment, content, comStart)
|
||||
|
||||
# Triple-quoted string
|
||||
elif self.peekString(3) == "\"\"\"":
|
||||
let strStart = self.currentPos()
|
||||
discard self.advance() # "
|
||||
discard self.advance() # "
|
||||
discard self.advance() # "
|
||||
var content = ""
|
||||
var foundClosing = false
|
||||
while self.hasMore():
|
||||
if self.peekString(3) == "\"\"\"":
|
||||
discard self.advance()
|
||||
discard self.advance()
|
||||
discard self.advance()
|
||||
self.emitToken(tkMultilineString, "\"\"\"" & content & "\"\"\"", strStart)
|
||||
foundClosing = true
|
||||
break
|
||||
else:
|
||||
content.add(self.advance())
|
||||
if not foundClosing:
|
||||
self.recordError(esError, "Unclosed multi-line string", ErrUnclosedString, strStart)
|
||||
self.emitToken(tkMultilineString, "\"\"\"" & content, strStart)
|
||||
|
||||
# Raw string
|
||||
elif c == 'r' or c == 'R':
|
||||
let nextStr = self.peekString(2)
|
||||
if nextStr == "r\"" or nextStr == "R\"":
|
||||
let strStart = self.currentPos()
|
||||
discard self.advance() # r
|
||||
discard self.advance() # "
|
||||
var content = "r\""
|
||||
while self.hasMore() and self.peek() != '"':
|
||||
content.add(self.advance())
|
||||
if self.hasMore():
|
||||
discard self.advance() # closing "
|
||||
content.add('"')
|
||||
self.emitToken(tkRawString, content, strStart)
|
||||
else:
|
||||
self.recordError(esError, "Unclosed raw string", ErrUnclosedString, strStart)
|
||||
else:
|
||||
# Regular identifier starting with r
|
||||
self.emitToken(tkIdentifier, $self.advance(), startPos)
|
||||
|
||||
# Regular string
|
||||
elif c == '"':
|
||||
let strStart = self.currentPos()
|
||||
discard self.advance() # opening "
|
||||
var content = "\""
|
||||
var foundClosing = false
|
||||
while self.hasMore():
|
||||
let sc = self.advance()
|
||||
if sc == '\\':
|
||||
content.add(sc)
|
||||
if self.hasMore():
|
||||
content.add(self.advance())
|
||||
elif sc == '"':
|
||||
content.add('"')
|
||||
self.emitToken(tkString, content, strStart)
|
||||
foundClosing = true
|
||||
break
|
||||
elif sc == '\n':
|
||||
self.recordError(esError, "Newline in string literal", ErrUnclosedString, strStart)
|
||||
self.emitToken(tkString, content, strStart)
|
||||
foundClosing = true
|
||||
break
|
||||
else:
|
||||
content.add(sc)
|
||||
if not foundClosing:
|
||||
self.recordError(esError, "Unclosed string", ErrUnclosedString, strStart)
|
||||
self.emitToken(tkString, content, strStart)
|
||||
|
||||
# Character literal
|
||||
elif c == '\'':
|
||||
let chStart = self.currentPos()
|
||||
discard self.advance() # '
|
||||
var content = "'"
|
||||
if self.hasMore():
|
||||
if self.peek() == '\\':
|
||||
content.add(self.advance())
|
||||
if self.hasMore(): content.add(self.advance())
|
||||
else:
|
||||
content.add(self.advance())
|
||||
if self.hasMore() and self.peek() == '\'':
|
||||
content.add(self.advance())
|
||||
self.emitToken(tkString, content, chStart)
|
||||
else:
|
||||
self.recordError(esError, "Unclosed character literal", ErrUnclosedString, chStart)
|
||||
|
||||
# Numbers
|
||||
elif isDigit(c):
|
||||
let numStart = self.currentPos()
|
||||
var num = ""
|
||||
# Check prefix
|
||||
if c == '0' and self.hasMore():
|
||||
let nxt = self.peek(1)
|
||||
if nxt == 'x' or nxt == 'X':
|
||||
num.add(self.advance()) # 0
|
||||
num.add(self.advance()) # x
|
||||
while self.hasMore() and isHexDigit(self.peek()):
|
||||
num.add(self.advance())
|
||||
self.emitToken(tkNumber, num, numStart)
|
||||
continue
|
||||
elif nxt == 'b' or nxt == 'B':
|
||||
num.add(self.advance()) # 0
|
||||
num.add(self.advance()) # b
|
||||
while self.hasMore() and isBinaryDigit(self.peek()):
|
||||
num.add(self.advance())
|
||||
self.emitToken(tkNumber, num, numStart)
|
||||
continue
|
||||
elif nxt == 'o' or nxt == 'O':
|
||||
num.add(self.advance()) # 0
|
||||
num.add(self.advance()) # o
|
||||
while self.hasMore() and isOctalDigit(self.peek()):
|
||||
num.add(self.advance())
|
||||
self.emitToken(tkNumber, num, numStart)
|
||||
continue
|
||||
|
||||
# Regular number
|
||||
var hadDot = false
|
||||
var hadExp = false
|
||||
while self.hasMore():
|
||||
let nc = self.peek()
|
||||
if isDigit(nc):
|
||||
num.add(self.advance())
|
||||
elif nc == '.' and not hadDot and not hadExp:
|
||||
let lookahead = self.peek(1)
|
||||
if lookahead.isDigit() or lookahead == '.' or lookahead == ' ' or lookahead == '\t' or lookahead == '\n':
|
||||
num.add(self.advance())
|
||||
hadDot = true
|
||||
else:
|
||||
break
|
||||
elif (nc == 'e' or nc == 'E') and not hadExp:
|
||||
num.add(self.advance())
|
||||
hadExp = true
|
||||
if self.hasMore() and (self.peek() == '+' or self.peek() == '-'):
|
||||
num.add(self.advance())
|
||||
elif nc == '\'':
|
||||
# Nim digit separator
|
||||
num.add(self.advance())
|
||||
else:
|
||||
break
|
||||
self.emitToken(tkNumber, num, numStart)
|
||||
|
||||
# Identifiers and keywords
|
||||
elif isAlpha(c):
|
||||
let idStart = self.currentPos()
|
||||
var ident = ""
|
||||
while self.hasMore() and (isAlphaNum(self.peek()) or self.peek() == '\''):
|
||||
ident.add(self.advance())
|
||||
if nimKeywordSet.contains(ident):
|
||||
self.emitToken(tkKeyword, ident, idStart)
|
||||
else:
|
||||
self.emitToken(tkIdentifier, ident, idStart)
|
||||
|
||||
# Brackets
|
||||
elif c == '(':
|
||||
self.emitBracketToken(kOpenParen, $self.advance(), startPos)
|
||||
elif c == ')':
|
||||
self.emitBracketToken(kCloseParen, $self.advance(), startPos)
|
||||
# Check matching
|
||||
if self.bracketStack.len > 0 and self.bracketStack[^1][0] == '(':
|
||||
discard self.bracketStack.pop()
|
||||
elif c == '[':
|
||||
self.emitBracketToken(kOpenBracket, $self.advance(), startPos)
|
||||
elif c == ']':
|
||||
self.emitBracketToken(kCloseBracket, $self.advance(), startPos)
|
||||
if self.bracketStack.len > 0 and self.bracketStack[^1][0] == '[':
|
||||
discard self.bracketStack.pop()
|
||||
elif c == '{':
|
||||
self.emitBracketToken(kOpenBrace, $self.advance(), startPos)
|
||||
elif c == '}':
|
||||
self.emitBracketToken(kCloseBrace, $self.advance(), startPos)
|
||||
if self.bracketStack.len > 0 and self.bracketStack[^1][0] == '{':
|
||||
discard self.bracketStack.pop()
|
||||
|
||||
# Operators
|
||||
elif c in {'+', '-', '*', '/', '%', '\\', '^', '~', '|', '&', '=', '<', '>', '!', '@', '$', '?'}:
|
||||
let opStart = self.currentPos()
|
||||
var op = ""
|
||||
# Multi-character operators
|
||||
while self.hasMore() and self.peek() in {'+', '-', '*', '/', '%', '\\', '^', '~', '|', '&', '=', '<', '>', '!', '@', '$', '?'}:
|
||||
op.add(self.advance())
|
||||
if op == "=":
|
||||
self.emitToken(tkAssignment, op, opStart)
|
||||
elif op == ":=" or op == "+=" or op == "-=" or op == "*=" or op == "/=" or op == "%=":
|
||||
self.emitToken(tkAssignment, op, opStart)
|
||||
elif op == "<":
|
||||
self.emitBracketToken(kAngleOpen, op, opStart)
|
||||
elif op == ">":
|
||||
self.emitBracketToken(kAngleClose, op, opStart)
|
||||
else:
|
||||
self.emitToken(tkOperator, op, opStart)
|
||||
|
||||
# Punctuation
|
||||
elif c in {'.', ',', ';', ':'}:
|
||||
let pStart = self.currentPos()
|
||||
if c == ':' and self.hasMore() and self.peek(1) == '=':
|
||||
discard self.advance()
|
||||
discard self.advance()
|
||||
self.emitToken(tkAssignment, ":=", pStart)
|
||||
elif c == '.' and self.peekString(3) == "..<":
|
||||
discard self.advance()
|
||||
discard self.advance()
|
||||
discard self.advance()
|
||||
self.emitToken(tkOperator, "..<", pStart)
|
||||
elif c == '.' and self.peekString(2) == "..":
|
||||
discard self.advance()
|
||||
discard self.advance()
|
||||
self.emitToken(tkOperator, "..", pStart)
|
||||
else:
|
||||
self.emitToken(tkPunctuation, $self.advance(), pStart)
|
||||
|
||||
# Backtick (Nim identifier quoting)
|
||||
elif c == '`':
|
||||
let btStart = self.currentPos()
|
||||
discard self.advance() # `
|
||||
var bt = "`"
|
||||
while self.hasMore() and self.peek() != '`':
|
||||
bt.add(self.advance())
|
||||
if self.hasMore():
|
||||
discard self.advance() # closing `
|
||||
bt.add('`')
|
||||
self.emitToken(tkIdentifier, bt, btStart)
|
||||
|
||||
# Unknown character
|
||||
else:
|
||||
let uStart = self.currentPos()
|
||||
self.recordError(esWarning, &"Unexpected character: '{c}' (U+{int(c):04X})", ErrUnexpectedToken, uStart)
|
||||
self.emitToken(tkError, $self.advance(), uStart)
|
||||
|
||||
# End of file
|
||||
self.emitToken(tkEndOfFile, "", self.currentPos())
|
||||
|
||||
when defined(validatrixDebug):
|
||||
if self.options.debugMode:
|
||||
debugLeave("NIM_TOKENIZER", &"Tokenization complete: {self.tokens.len} tokens")
|
||||
@@ -0,0 +1,226 @@
|
||||
## PHP tokenizer for Validatrix.
|
||||
##
|
||||
## Handles mixed PHP/HTML content (template mode).
|
||||
|
||||
import std/[strformat, sets, json, sequtils, unicode]
|
||||
{.warning[UnusedImport]:off.}
|
||||
import ../core/types, ../core/tokenizerbase
|
||||
|
||||
type
|
||||
PHPTokenizer* = ref object of TokenizerBase
|
||||
inPhpMode*: bool
|
||||
|
||||
const
|
||||
phpKeywords = @[
|
||||
"abstract", "and", "array", "as", "break", "callable", "case", "catch",
|
||||
"class", "clone", "const", "continue", "declare", "default", "die",
|
||||
"do", "echo", "else", "elseif", "empty", "enddeclare", "endfor",
|
||||
"endforeach", "endif", "endswitch", "endwhile", "eval", "exit",
|
||||
"extends", "final", "finally", "fn", "for", "foreach", "function",
|
||||
"global", "goto", "if", "implements", "include", "include_once",
|
||||
"instanceof", "insteadof", "interface", "isset", "list", "match",
|
||||
"namespace", "new", "or", "print", "private", "protected", "public",
|
||||
"readonly", "require", "require_once", "return", "static", "switch",
|
||||
"throw", "trait", "try", "unset", "use", "var", "while", "xor",
|
||||
"yield", "true", "false", "null", "self", "parent", "int", "float",
|
||||
"string", "bool", "void", "never", "mixed", "iterable", "object"
|
||||
]
|
||||
phpKeywordSet = phpKeywords.toHashSet()
|
||||
|
||||
proc newPHPTokenizer*(source: string, options: ValidationOptions = newValidationOptions()): PHPTokenizer =
|
||||
result = PHPTokenizer(
|
||||
source: source,
|
||||
sourceLen: source.len,
|
||||
options: options,
|
||||
tokens: @[],
|
||||
pos: 0,
|
||||
line: 1,
|
||||
column: 1,
|
||||
bracketStack: @[],
|
||||
errors: @[],
|
||||
eof: false
|
||||
)
|
||||
result.inPhpMode = false
|
||||
|
||||
method tokenize*(self: PHPTokenizer) =
|
||||
when defined(validatrixDebug):
|
||||
if self.options.debugMode:
|
||||
debugEnter("PHP_TOKENIZER", "Starting PHP tokenization")
|
||||
|
||||
while self.hasMore():
|
||||
let startPos = self.currentPos()
|
||||
let c = self.peek()
|
||||
|
||||
# Detect PHP mode entry/exit
|
||||
if not self.inPhpMode:
|
||||
# Check for <?php or <?= or short open tag
|
||||
if self.peekString(5) == "<?php" or self.peekString(2) == "<?":
|
||||
if self.peekString(5) == "<?php":
|
||||
for _ in 1..5: discard self.advance()
|
||||
self.emitToken(tkDirective, "<?php", startPos)
|
||||
else:
|
||||
for _ in 1..2: discard self.advance()
|
||||
self.emitToken(tkDirective, "<?", startPos)
|
||||
if self.hasMore() and self.peek() == '=':
|
||||
discard self.advance()
|
||||
self.emitToken(tkDirective, "=", startPos)
|
||||
self.inPhpMode = true
|
||||
else:
|
||||
# Plain HTML content
|
||||
var html = ""
|
||||
while self.hasMore() and not (self.peekString(5) == "<?php" or self.peekString(2) == "<?") and self.peek() != '\n':
|
||||
html.add(self.advance())
|
||||
if html.len > 0:
|
||||
self.emitToken(tkTemplateTag, html, startPos)
|
||||
elif self.hasMore() and self.peek() == '\n':
|
||||
discard self.advance()
|
||||
self.emitToken(tkNewline, "\n", startPos)
|
||||
continue
|
||||
|
||||
# Inside PHP mode — check for exit
|
||||
if self.peekString(2) == "?>" and self.inPhpMode:
|
||||
for _ in 1..2: discard self.advance()
|
||||
self.emitToken(tkDirective, "?>", startPos)
|
||||
self.inPhpMode = false
|
||||
continue
|
||||
|
||||
# Newline
|
||||
if c == '\n':
|
||||
self.emitToken(tkNewline, $self.advance(), startPos)
|
||||
continue
|
||||
|
||||
# Whitespace
|
||||
if c == ' ' or c == '\t' or c == '\r':
|
||||
var ws = ""
|
||||
while self.hasMore() and (self.peek() == ' ' or self.peek() == '\t' or self.peek() == '\r'):
|
||||
ws.add(self.advance())
|
||||
self.emitToken(tkWhitespace, ws, startPos)
|
||||
continue
|
||||
|
||||
# Line comment (# or //)
|
||||
if c == '#' or self.peekString(2) == "//":
|
||||
let comStart = self.currentPos()
|
||||
if c == '#':
|
||||
discard self.advance()
|
||||
self.emitToken(tkComment, "#" & self.parseLineComment(), comStart)
|
||||
else:
|
||||
discard self.advance()
|
||||
discard self.advance()
|
||||
self.emitToken(tkComment, "//" & self.parseLineComment(), comStart)
|
||||
continue
|
||||
|
||||
# Block comment
|
||||
if self.peekString(2) == "/*":
|
||||
let comStart = self.currentPos()
|
||||
discard self.advance()
|
||||
discard self.advance()
|
||||
let content = self.parseBlockComment("/*", "*/")
|
||||
self.emitToken(tkComment, "/*" & content & "*/", comStart)
|
||||
continue
|
||||
|
||||
# Strings
|
||||
if c == '"' or c == '\'':
|
||||
let quote = c
|
||||
let strStart = self.currentPos()
|
||||
discard self.advance()
|
||||
var content = $quote
|
||||
var foundClosing = false
|
||||
while self.hasMore():
|
||||
let sc = self.advance()
|
||||
if sc == '\\':
|
||||
content.add(sc)
|
||||
if self.hasMore(): content.add(self.advance())
|
||||
elif sc == quote:
|
||||
content.add(quote)
|
||||
self.emitToken(tkString, content, strStart)
|
||||
foundClosing = true
|
||||
foundClosing = true
|
||||
break
|
||||
elif sc == '\n':
|
||||
self.emitToken(tkString, content, strStart)
|
||||
foundClosing = true
|
||||
foundClosing = true
|
||||
break
|
||||
else:
|
||||
content.add(sc)
|
||||
if not foundClosing:
|
||||
self.recordError(esError, "Unclosed string", ErrUnclosedString, strStart)
|
||||
self.emitToken(tkString, content, strStart)
|
||||
continue
|
||||
|
||||
# Numbers
|
||||
if isDigit(c):
|
||||
var num = ""
|
||||
while self.hasMore() and (isAlphaNum(self.peek()) or self.peek() == '.'):
|
||||
num.add(self.advance())
|
||||
self.emitToken(tkNumber, num, startPos)
|
||||
continue
|
||||
|
||||
# Identifiers and keywords
|
||||
if isAlpha(c) or c == '_':
|
||||
var ident = ""
|
||||
while self.hasMore() and (isAlphaNum(self.peek()) or self.peek() == '_'):
|
||||
ident.add(self.advance())
|
||||
if phpKeywordSet.contains(ident):
|
||||
self.emitToken(tkKeyword, ident, startPos)
|
||||
else:
|
||||
self.emitToken(tkIdentifier, ident, startPos)
|
||||
continue
|
||||
|
||||
# Variable names ($var)
|
||||
if c == '$':
|
||||
let varStart = self.currentPos()
|
||||
discard self.advance()
|
||||
var varName = "$"
|
||||
# Property access $obj->prop
|
||||
while self.hasMore() and (isAlphaNum(self.peek()) or self.peek() == '_' or self.peek() == '-' or self.peek() == '>'):
|
||||
let nc = self.peek()
|
||||
if nc == '-' or nc == '>':
|
||||
varName.add(self.advance())
|
||||
else:
|
||||
varName.add(self.advance())
|
||||
self.emitToken(tkIdentifier, varName, varStart)
|
||||
continue
|
||||
|
||||
# Brackets
|
||||
if c == '(': self.emitBracketToken(kOpenParen, $self.advance(), startPos); continue
|
||||
if c == ')':
|
||||
self.emitBracketToken(kCloseParen, $self.advance(), startPos)
|
||||
if self.bracketStack.len > 0 and self.bracketStack[^1][0] == '(': discard self.bracketStack.pop()
|
||||
continue
|
||||
if c == '[': self.emitBracketToken(kOpenBracket, $self.advance(), startPos); continue
|
||||
if c == ']':
|
||||
self.emitBracketToken(kCloseBracket, $self.advance(), startPos)
|
||||
if self.bracketStack.len > 0 and self.bracketStack[^1][0] == '[': discard self.bracketStack.pop()
|
||||
continue
|
||||
if c == '{': self.emitBracketToken(kOpenBrace, $self.advance(), startPos); continue
|
||||
if c == '}':
|
||||
self.emitBracketToken(kCloseBrace, $self.advance(), startPos)
|
||||
if self.bracketStack.len > 0 and self.bracketStack[^1][0] == '{': discard self.bracketStack.pop()
|
||||
continue
|
||||
|
||||
# Operators
|
||||
if c in {'+', '-', '*', '/', '%', '^', '~', '|', '&', '<', '>', '=', '!', '.', ':'}:
|
||||
var op = ""
|
||||
while self.hasMore() and self.peek() in {'+', '-', '*', '/', '%', '^', '~', '|', '&', '<', '>', '=', '!', '.', ':'}:
|
||||
op.add(self.advance())
|
||||
if op == "=" or op == "+=" or op == "-=" or op == "*=" or op == "/=" or op == "%=" or op == ".=" or op == "&=" or op == "|=" or op == "^=" or op == "<<=" or op == ">>=":
|
||||
self.emitToken(tkAssignment, op, startPos)
|
||||
elif op == "->" or op == "=>":
|
||||
self.emitToken(tkOperator, op, startPos)
|
||||
else:
|
||||
self.emitToken(tkOperator, op, startPos)
|
||||
continue
|
||||
|
||||
# Punctuation
|
||||
if c in {'.', ',', ';', ':'}:
|
||||
self.emitToken(tkPunctuation, $self.advance(), startPos)
|
||||
continue
|
||||
|
||||
# Unknown
|
||||
self.emitToken(tkError, $self.advance(), startPos)
|
||||
|
||||
self.emitToken(tkEndOfFile, "", self.currentPos())
|
||||
when defined(validatrixDebug):
|
||||
if self.options.debugMode:
|
||||
debugLeave("PHP_TOKENIZER", &"Complete: {self.tokens.len} tokens (PHP mode: {self.inPhpMode})")
|
||||
@@ -0,0 +1,354 @@
|
||||
## Python tokenizer for Validatrix.
|
||||
##
|
||||
## Tokenizes Python source code including:
|
||||
## - Single/double quoted strings
|
||||
## - Triple-quoted strings (""" ''')
|
||||
## - f-strings with interpolation
|
||||
## - Line continuation with backslash
|
||||
## - Indentation tracking
|
||||
|
||||
import std/[strformat, sets, json, sequtils, unicode]
|
||||
{.warning[UnusedImport]:off.}
|
||||
import ../core/types, ../core/tokenizerbase
|
||||
|
||||
type
|
||||
PythonTokenizer* = ref object of TokenizerBase
|
||||
## Tokenizer for Python source code.
|
||||
indentStack*: seq[int] # Track indentation levels
|
||||
parenDepth*: int # Track parens for implicit continuation
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Python keywords
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
const
|
||||
pythonKeywords* = @[
|
||||
"False", "None", "True", "and", "as", "assert", "async", "await",
|
||||
"break", "class", "continue", "def", "del", "elif", "else", "except",
|
||||
"finally", "for", "from", "global", "if", "import", "in", "is",
|
||||
"lambda", "nonlocal", "not", "or", "pass", "raise", "return",
|
||||
"try", "while", "with", "yield"
|
||||
]
|
||||
|
||||
pythonKeywordSet* = pythonKeywords.toHashSet()
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Construction
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
proc newPythonTokenizer*(source: string, options: ValidationOptions = newValidationOptions()): PythonTokenizer =
|
||||
## Create a new Python tokenizer.
|
||||
result = PythonTokenizer(
|
||||
source: source,
|
||||
sourceLen: source.len,
|
||||
options: options,
|
||||
tokens: @[],
|
||||
pos: 0,
|
||||
line: 1,
|
||||
column: 1,
|
||||
bracketStack: @[],
|
||||
errors: @[],
|
||||
eof: false
|
||||
)
|
||||
result.indentStack = @[0]
|
||||
result.parenDepth = 0
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Indentation helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
proc getIndentLevel*(line: string): int =
|
||||
## Count leading spaces.
|
||||
for c in line:
|
||||
if c == ' ': result += 1
|
||||
elif c == '\t': result += 8 # Standard tab = 8
|
||||
else: break
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tokenization
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
method tokenize*(self: PythonTokenizer) =
|
||||
## Tokenize Python source code.
|
||||
when defined(validatrixDebug):
|
||||
if self.options.debugMode:
|
||||
debugEnter("PYTHON_TOKENIZER", "Starting Python tokenization")
|
||||
|
||||
var currentLine: string = ""
|
||||
var atLineStart = true
|
||||
|
||||
while self.hasMore():
|
||||
let startPos = self.currentPos()
|
||||
let c = self.peek()
|
||||
|
||||
# Newline handling with indentation tracking
|
||||
if c == '\n':
|
||||
if atLineStart and currentLine.len == 0:
|
||||
# Empty line, just emit newline
|
||||
discard self.advance()
|
||||
self.emitToken(tkNewline, "\n", startPos)
|
||||
else:
|
||||
# End of line - parse indentation of next line
|
||||
discard self.advance()
|
||||
self.emitToken(tkNewline, "\n", startPos)
|
||||
atLineStart = true
|
||||
currentLine = ""
|
||||
|
||||
# Read next line for indentation tracking (peek ahead)
|
||||
var nextLineStart = self.pos
|
||||
var indentCount = 0
|
||||
while nextLineStart < self.sourceLen and self.source[nextLineStart] == ' ':
|
||||
indentCount += 1
|
||||
nextLineStart += 1
|
||||
# Skip blank/comment lines
|
||||
if nextLineStart < self.sourceLen and self.source[nextLineStart] != '\n' and self.source[nextLineStart] != '#':
|
||||
if self.parenDepth == 0:
|
||||
if indentCount > self.indentStack[^1]:
|
||||
self.indentStack.add(indentCount)
|
||||
self.emitToken(tkSpecial, "<indent>", startPos)
|
||||
elif indentCount < self.indentStack[^1]:
|
||||
while self.indentStack.len > 1 and indentCount < self.indentStack[^1]:
|
||||
discard self.indentStack.pop()
|
||||
self.emitToken(tkSpecial, "<dedent>", startPos)
|
||||
if indentCount != self.indentStack[^1]:
|
||||
self.recordError(esError, "Indentation mismatch", ErrUnexpectedIndent, startPos)
|
||||
continue
|
||||
|
||||
atLineStart = false
|
||||
|
||||
# Whitespace (non-newline)
|
||||
if c == ' ' or c == '\t' or c == '\r':
|
||||
let wsStart = self.currentPos()
|
||||
var ws = ""
|
||||
while self.hasMore() and (self.peek() == ' ' or self.peek() == '\t' or self.peek() == '\r'):
|
||||
ws.add(self.advance())
|
||||
self.emitToken(tkWhitespace, ws, wsStart)
|
||||
|
||||
# Line comment
|
||||
elif c == '#':
|
||||
let comStart = self.currentPos()
|
||||
discard self.advance()
|
||||
var content = "#" & self.parseLineComment()
|
||||
self.emitToken(tkComment, content, comStart)
|
||||
|
||||
# Triple-quoted strings
|
||||
elif self.peekString(3) == "\"\"\"" or self.peekString(3) == "'''":
|
||||
let quoteChar = if self.peek() == '"': '"' else: '\''
|
||||
let strStart = self.currentPos()
|
||||
discard self.advance()
|
||||
discard self.advance()
|
||||
discard self.advance()
|
||||
var content = repeat($quoteChar, 3)
|
||||
var foundClosing = false
|
||||
while self.hasMore():
|
||||
if self.peekString(3) == repeat($quoteChar, 3):
|
||||
discard self.advance()
|
||||
discard self.advance()
|
||||
discard self.advance()
|
||||
content.add(repeat($quoteChar, 3))
|
||||
self.emitToken(tkMultilineString, content, strStart)
|
||||
foundClosing = true
|
||||
break
|
||||
else:
|
||||
content.add(self.advance())
|
||||
if not foundClosing:
|
||||
self.recordError(esError, "Unclosed triple-quoted string", ErrUnclosedString, strStart)
|
||||
self.emitToken(tkMultilineString, content, strStart)
|
||||
|
||||
# f-strings
|
||||
elif c == 'f' or c == 'F':
|
||||
let peekStr = self.peekString(2)
|
||||
if peekStr.len >= 2 and peekStr[1] == '"' or peekStr[1] == '\'':
|
||||
let fStart = self.currentPos()
|
||||
discard self.advance() # f
|
||||
let quote = self.advance() # " or '
|
||||
var content = "f" & quote
|
||||
var braceDepth = 0
|
||||
var foundClosing = false
|
||||
while self.hasMore():
|
||||
let fc = self.advance()
|
||||
if fc == '{' and braceDepth == 0:
|
||||
braceDepth += 1
|
||||
content.add(fc)
|
||||
elif fc == '}' and braceDepth == 1:
|
||||
braceDepth -= 1
|
||||
content.add(fc)
|
||||
elif fc == '\\':
|
||||
content.add(fc)
|
||||
if self.hasMore(): content.add(self.advance())
|
||||
elif fc == quote and braceDepth == 0:
|
||||
content.add(fc)
|
||||
self.emitToken(tkString, content, fStart)
|
||||
foundClosing = true
|
||||
break
|
||||
else:
|
||||
content.add(fc)
|
||||
if fc == '{': braceDepth += 1
|
||||
elif fc == '}': braceDepth -= 1
|
||||
if not foundClosing:
|
||||
self.recordError(esError, "Unclosed f-string", ErrUnclosedString, fStart)
|
||||
self.emitToken(tkString, content, fStart)
|
||||
else:
|
||||
# Regular identifier
|
||||
var id = "f"
|
||||
while self.hasMore() and (isAlphaNum(self.peek()) or self.peek() == '_'):
|
||||
id.add(self.advance())
|
||||
self.emitToken(tkIdentifier, id, startPos)
|
||||
|
||||
# Regular string (single or double quote)
|
||||
elif c == '"' or c == '\'':
|
||||
let quote = c
|
||||
let strStart = self.currentPos()
|
||||
discard self.advance() # opening quote
|
||||
var content = $quote
|
||||
var foundClosing = false
|
||||
while self.hasMore():
|
||||
let sc = self.advance()
|
||||
if sc == '\\':
|
||||
content.add(sc)
|
||||
if self.hasMore(): content.add(self.advance())
|
||||
elif sc == quote:
|
||||
content.add(quote)
|
||||
self.emitToken(tkString, content, strStart)
|
||||
foundClosing = true
|
||||
break
|
||||
elif sc == '\n':
|
||||
self.recordError(esError, "Newline in string literal", ErrUnclosedString, strStart)
|
||||
content.add('\n')
|
||||
foundClosing = true
|
||||
break
|
||||
else:
|
||||
content.add(sc)
|
||||
if not foundClosing:
|
||||
self.recordError(esError, "Unclosed string", ErrUnclosedString, strStart)
|
||||
self.emitToken(tkString, content, strStart)
|
||||
|
||||
# Numbers
|
||||
elif isDigit(c):
|
||||
let numStart = self.currentPos()
|
||||
var num = ""
|
||||
# Check prefix
|
||||
if c == '0' and self.hasMore():
|
||||
let nxt = self.peek(1)
|
||||
if nxt == 'x' or nxt == 'X':
|
||||
num.add(self.advance())
|
||||
num.add(self.advance())
|
||||
while self.hasMore() and isHexDigit(self.peek()):
|
||||
num.add(self.advance())
|
||||
self.emitToken(tkNumber, num, numStart)
|
||||
continue
|
||||
elif nxt == 'b' or nxt == 'B':
|
||||
num.add(self.advance())
|
||||
num.add(self.advance())
|
||||
while self.hasMore() and isBinaryDigit(self.peek()):
|
||||
num.add(self.advance())
|
||||
self.emitToken(tkNumber, num, numStart)
|
||||
continue
|
||||
elif nxt == 'o' or nxt == 'O':
|
||||
num.add(self.advance())
|
||||
num.add(self.advance())
|
||||
while self.hasMore() and isOctalDigit(self.peek()):
|
||||
num.add(self.advance())
|
||||
self.emitToken(tkNumber, num, numStart)
|
||||
continue
|
||||
|
||||
while self.hasMore() and (self.peek().isDigit() or self.peek() == '.' or self.peek() == 'e' or self.peek() == 'E' or self.peek() == 'j' or self.peek() == 'J'):
|
||||
num.add(self.advance())
|
||||
self.emitToken(tkNumber, num, numStart)
|
||||
|
||||
# Identifiers and keywords
|
||||
elif isAlpha(c):
|
||||
let idStart = self.currentPos()
|
||||
var ident = ""
|
||||
while self.hasMore() and (isAlphaNum(self.peek()) or self.peek() == '_'):
|
||||
ident.add(self.advance())
|
||||
if pythonKeywordSet.contains(ident):
|
||||
self.emitToken(tkKeyword, ident, idStart)
|
||||
else:
|
||||
self.emitToken(tkIdentifier, ident, idStart)
|
||||
|
||||
# Brackets
|
||||
elif c == '(':
|
||||
self.parenDepth += 1
|
||||
self.emitBracketToken(kOpenParen, $self.advance(), startPos)
|
||||
elif c == ')':
|
||||
if self.parenDepth > 0: self.parenDepth -= 1
|
||||
self.emitBracketToken(kCloseParen, $self.advance(), startPos)
|
||||
if self.bracketStack.len > 0 and self.bracketStack[^1][0] == '(':
|
||||
discard self.bracketStack.pop()
|
||||
elif c == '[':
|
||||
self.parenDepth += 1
|
||||
self.emitBracketToken(kOpenBracket, $self.advance(), startPos)
|
||||
elif c == ']':
|
||||
if self.parenDepth > 0: self.parenDepth -= 1
|
||||
self.emitBracketToken(kCloseBracket, $self.advance(), startPos)
|
||||
if self.bracketStack.len > 0 and self.bracketStack[^1][0] == '[':
|
||||
discard self.bracketStack.pop()
|
||||
elif c == '{':
|
||||
self.parenDepth += 1
|
||||
self.emitBracketToken(kOpenBrace, $self.advance(), startPos)
|
||||
elif c == '}':
|
||||
if self.parenDepth > 0: self.parenDepth -= 1
|
||||
self.emitBracketToken(kCloseBrace, $self.advance(), startPos)
|
||||
if self.bracketStack.len > 0 and self.bracketStack[^1][0] == '{':
|
||||
discard self.bracketStack.pop()
|
||||
|
||||
# Operators
|
||||
elif c in {'+', '-', '*', '/', '%', '^', '~', '|', '&', '<', '>', '='}:
|
||||
let opStart = self.currentPos()
|
||||
var op = ""
|
||||
# Multi-char operators
|
||||
while self.hasMore() and self.peek() in {'+', '-', '*', '/', '%', '^', '~', '|', '&', '<', '>', '=', '!'}:
|
||||
op.add(self.advance())
|
||||
if op == "=" or op == "+=" or op == "-=" or op == "*=" or op == "/=" or op == "%=" or op == "//=" or op == "**=" or op == "&=" or op == "|=" or op == "^=" or op == "<<=" or op == ">>=":
|
||||
self.emitToken(tkAssignment, op, opStart)
|
||||
elif op == ":":
|
||||
self.emitToken(tkPunctuation, op, opStart)
|
||||
else:
|
||||
self.emitToken(tkOperator, op, opStart)
|
||||
|
||||
# Punctuation
|
||||
elif c in {'.', ',', ';', ':'}:
|
||||
let pStart = self.currentPos()
|
||||
if c == ':' and self.hasMore() and self.peek(1) == '=':
|
||||
discard self.advance()
|
||||
discard self.advance()
|
||||
self.emitToken(tkAssignment, ":=", pStart)
|
||||
else:
|
||||
self.emitToken(tkPunctuation, $self.advance(), pStart)
|
||||
|
||||
# Backslash continuation
|
||||
elif c == '\\':
|
||||
let bsStart = self.currentPos()
|
||||
discard self.advance()
|
||||
if self.hasMore() and self.peek() == '\n':
|
||||
discard self.advance()
|
||||
self.emitToken(tkSpecial, "\\\n", bsStart)
|
||||
else:
|
||||
self.emitToken(tkOperator, "\\", bsStart)
|
||||
|
||||
# Decorator
|
||||
elif c == '@':
|
||||
let decStart = self.currentPos()
|
||||
discard self.advance()
|
||||
var decorator = "@"
|
||||
while self.hasMore() and (isAlphaNum(self.peek()) or self.peek() == '_' or self.peek() == '.'):
|
||||
decorator.add(self.advance())
|
||||
self.emitToken(tkDirective, decorator, decStart)
|
||||
|
||||
# Unknown
|
||||
else:
|
||||
let uStart = self.currentPos()
|
||||
self.emitToken(tkError, $self.advance(), uStart)
|
||||
|
||||
# Add dedent tokens for remaining indentation at EOF
|
||||
while self.indentStack.len > 1:
|
||||
discard self.indentStack.pop()
|
||||
self.emitToken(tkSpecial, "<dedent>", self.currentPos())
|
||||
|
||||
# End of file
|
||||
self.emitToken(tkEndOfFile, "", self.currentPos())
|
||||
|
||||
when defined(validatrixDebug):
|
||||
if self.options.debugMode:
|
||||
debugLeave("PYTHON_TOKENIZER", &"Tokenization complete: {self.tokens.len} tokens")
|
||||
Reference in New Issue
Block a user