diff --git a/.gitignore b/.gitignore index d4b1479..583e647 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,8 @@ + + +tests/test_json_exhaustive +tests/test_toml_exhaustive +tests/test_yaml_exhaustive # ── Nim ────────────────────────────────────────────────────────────────── nimcache/ nimblecache/ diff --git a/Makefile b/Makefile index bba3e83..bdd02a5 100644 --- a/Makefile +++ b/Makefile @@ -52,13 +52,13 @@ test-jinja: $(NIMC) -r $(TEST_DIR)/test_jinja.nim test-json: - $(NIMC) -r $(TEST_DIR)/test_json.nim + $(NIMC) -r $(TEST_DIR)/test_json_exhaustive.nim test-yaml: - $(NIMC) -r $(TEST_DIR)/test_yaml.nim + $(NIMC) -r $(TEST_DIR)/test_yaml_exhaustive.nim test-toml: - $(NIMC) -r $(TEST_DIR)/test_toml.nim + $(NIMC) -r $(TEST_DIR)/test_toml_exhaustive.nim test-mixed: $(NIMC) -r $(TEST_DIR)/test_mixed.nim diff --git a/README.md b/README.md index df34ea5..abf31e9 100644 --- a/README.md +++ b/README.md @@ -1,118 +1,152 @@ + # Nimcheck -A multi-language source code validation framework written in Nim. Nimcheck uses tokenizer-based validation to check syntax correctness across 10 languages and config formats, with auto-detection, comprehensive diagnostics, and a never-crash guarantee. +Universal source code validation framework written in Nim. Tokenizer-based syntax validation across 10 languages and config formats, with auto-detection, structured JSON diagnostics, structural inspection, and a never-crash guarantee. -## Features +Version: **0.1.0** | Author: **molodetz** | License: **MIT** | Requires: **Nim >= 2.0.0** -- **10 languages supported**: Nim, Bash, Python, JavaScript, PHP, HTML, Jinja, JSON, YAML, TOML -- **Tokenizer-based validation** -- real tokenizers that track bracket depth, string context, comments, and nesting -- **Auto-detection** -- detect language from file extension or content analysis -- **Comprehensive diagnostics** -- structured error reports with position, severity, hints, and debug output -- **Never-crash guarantee** -- every validation path is wrapped in try/except; invalid input returns errors gracefully, never panics -- **JSON output** -- machine-readable diagnostics via `toJson()` on any validation result -- **Inspection API** -- extract module structure (imports, definitions, functions/classes) as JSON -- **Debug mode** -- compile with `-d:nimcheckDebug` for detailed tokenization logging -- **No external dependencies** -- pure Nim standard library +--- + +## Table of Contents + +1. [Architecture](#architecture) +2. [Quick Start](#quick-start) +3. [API Reference](#api-reference) +4. [Data Types](#data-types) +5. [JSON Output Structures](#json-output-structures) +6. [Error Code Reference](#error-code-reference) +7. [IO Table: Supported Languages and Per-Language Errors](#io-table-supported-languages-and-per-language-errors) +8. [CLI Usage](#cli-usage) +9. [Building and Testing](#building-and-testing) +10. [Project Structure](#project-structure) +11. [Contributing](#contributing) + +--- + +## Architecture + +Nimcheck uses a four-layer pipeline architecture. Each layer is independent and communicates through well-defined interfaces. + +``` +Source Code (string or file) + | + v ++-------------------+ +| Detector | LanguageFlavor from extension, shebang, or content analysis ++-------------------+ + | + v ++-------------------+ +| Tokenizer | Converts raw source into a seq[Token] stream ++-------------------+ + | + v ++-------------------+ +| Validator | Analyzes token stream for syntax errors and structure ++-------------------+ + | + v ++-------------------+ +| Reporting | Formats errors as human-readable text or structured JSON ++-------------------+ + | + v + ValidationResult +``` + +### Layer 1: Detector (`src/nimcheck/core/detector.nim`) + +Language auto-detection with cascading priority: + +| Priority | Method | Source | Confidence | +|----------|--------|--------|------------| +| 1 (highest) | Explicit | User-provided `LanguageFlavor` | 1.0 | +| 2 | File extension | 70+ extension-to-flavor mappings | 0.9 | +| 3 | Shebang line | `#!` line interpreter matching (bash, python, node, ruby, lua, swift) | 0.95 | +| 4 | Content analysis | Keyword scoring, structural patterns, bracket usage | 0.6-0.9 | + +The `detectFlavor()` function returns a `(LanguageFlavor, FlavorDetectionMethod, confidence: float)` tuple. + +### Layer 2: Tokenizer (`src/nimcheck/core/tokenizerbase.nim`, per-language impls) + +Each tokenizer converts raw source into a `seq[Token]`. Tokenizers track: + +- **Position**: line, column, and byte offset for every token +- **Bracket stack**: `(`, `[`, `{`, `<` pushed and popped with position tracking +- **String context**: escape sequences, multi-line delimiters, interpolation +- **Comment context**: line comments, block comments with nesting support + +Tokenizers emit errors for: +- Unclosed strings (`ErrUnclosedString`) +- Unclosed comments (`ErrUnclosedComment`) +- Unexpected characters (`ErrUnexpectedToken`) +- Unclosed block constructs (language-specific, e.g., `${}`, `$()`) + +### Layer 3: Validator (`src/nimcheck/core/validatorbase.nim`, per-language impls) + +Each validator: + +1. Creates its tokenizer and runs `tokenize()` +2. Collects tokenizer errors +3. Runs `analyzeTokens()` — language-specific structural analysis +4. Runs `buildModuleInfo()` — populates `ModuleInfo` with imports, functions, classes +5. Computes validity, warning/info counts, duration + +Validators register themselves via a factory pattern at import time. The registry maps flavor strings (e.g., `"python"`, `"bash"`) to constructors. + +### Layer 4: Reporting (`src/nimcheck/reporting/errors.nim`) + +Produces: +- **Human-readable reports**: `formatReport()` with position, severity, code, message, source context with carets, and hints +- **JSON output**: every type has a `toJson()` method producing `JsonNode` +- **Error categorization**: `countBySeverity()`, `countByCode()`, `filterBySeverity()` + +--- ## Quick Start +### Nim library usage + ```nim import nimcheck # Validate source code from a string let r = validateSource("proc hello() = echo 'hi'", lfNim) -echo r.errors # seq[ValidationError] +echo r.valid # bool +echo r.errors # seq[ValidationError] +echo r.toJson().pretty() # Full JSON output + +# Validate a file (auto-detects language from extension) +let r2 = validateFile("src/main.nim") # Auto-detect language -let r2 = validateSource("print('hello')", lfUnknown) # detects Python +let r3 = validateSource("print('hello')", lfUnknown) +echo r3.flavor # lfPython -# Validate a file -let r3 = validateFile("path/to/file.nim", lfNim) +# Inspect source structure (returns JsonNode with functions, classes, imports) +let info = inspectSource("class Foo:\n def bar(self): pass", lfPython) +echo info.pretty() -# Get JSON diagnostics -echo r.toJson().pretty() - -# Inspect source structure (returns JsonNode) -let info = inspectSource("class Foo: ...", lfPython) -echo info.pretty() # formatted JSON with class/function info - -# Get human-readable report -let report = reportSource("proc foo() =", lfNim) -echo report # formatted error summary +# Human-readable report +echo reportSource("proc foo() =\n let x = \"unclosed", lfNim) ``` -## Building - -### Using Makefile (recommended) +### CLI usage ```bash -# Build the library binary +# Build the binary make build -# Run all tests -make test - -# Build with debug mode -make build-debug -make debug # build debug + run all tests - -# Run a specific language test -make test-nim -make test-python -make test-bash - -# Lint with full warnings -make lint - -# Clean build artifacts -make clean - -# Show all available targets -make help -``` - -### Using nimble - -```bash -# Run all tests -nimble test - -# Run per-language tests -nimble test_nim -nimble test_python - -# Build the library -nimble build -``` - -### Manual compilation - -```bash -# Build the library -nim c src/nimcheck.nim - -# Build with debug mode -nim c -d:nimcheckDebug src/nimcheck.nim - -# Run all tests -nim c -r tests/test_all.nim -``` - -## CLI Usage - -When compiled as a standalone binary (`make build` produces `bin/nimcheck`): - -```bash # Validate a file bin/nimcheck validate --file=source.nim -# Validate source code directly +# Validate source directly bin/nimcheck validate --source="print('hi')" --flavor=python # Read from stdin echo "var x = 1" | bin/nimcheck validate -# Inspect source structure (returns JSON) +# Inspect structure (JSON output) bin/nimcheck inspect --file=source.py # Detect language @@ -124,174 +158,944 @@ bin/nimcheck version bin/nimcheck help ``` +--- + ## API Reference ### Core Functions -| Function | Description | -|---|---| -| `validateSource(source, flavor, options, sourcePath)` | Validate source code string; returns `ValidationResult` | -| `validateFile(filePath, flavor, options)` | Validate a file; returns `ValidationResult` | -| `inspectSource(source, flavor, options, sourcePath)` | Return structural diagnostics as `JsonNode` | -| `inspectFile(filePath, flavor, options)` | Return structural diagnostics for a file as `JsonNode` | -| `validateSourceJson(source, flavor)` | Validate source and return result as JSON string | -| `validateFileJson(filePath, flavor)` | Validate file and return result as JSON string | -| `reportSource(source, flavor)` | Get a human-readable validation report (string) | -| `reportFile(filePath, flavor)` | Get a human-readable validation report for a file (string) | -| `detectFlavor(source, filePath)` | Auto-detect language flavor | -| `supportedFlavors()` | List all registered language flavors (returns `seq[string]`) | -| `version()` | Get the Nimcheck version string | -| `enableDebug()` / `disableDebug()` | Toggle debug mode globally | +All functions are exported from `nimcheck.nim`. Import once with `import nimcheck`. -### Types +#### Validation -- **`ValidationResult`** -- contains `errors: seq[ValidationError]`, `valid: bool`, `moduleInfo`, `debugOutput`, `detectionInfo` -- **`ValidationError`** -- contains `code`, `message`, `severity`, `position`, `hint`, `context` -- **`ValidationOptions`** -- contains `flavor`, `debugMode`, `maxErrors`, `validateAll` -- **`LanguageFlavor`** -- enum: `lfUnknown`, `lfNim`, `lfBash`, `lfPython`, `lfJavaScript`, `lfPHP`, `lfHTML`, `lfJinja`, `lfJSON`, `lfYAML`, `lfTOML` +| Function | Signature | Returns | Description | +|----------|-----------|---------|-------------| +| `validateSource` | `(source: string, flavor: LanguageFlavor = lfUnknown, options: ValidationOptions = newValidationOptions(), sourcePath: string = "")` | `ValidationResult` | Validate source code string. Auto-detects flavor if `lfUnknown`. | +| `validateFile` | `(filePath: string, flavor: LanguageFlavor = lfUnknown, options: ValidationOptions = newValidationOptions())` | `ValidationResult` | Validate a file from disk. Auto-detects flavor from file extension. | +| `validateSourceJson` | `(source: string, flavor: LanguageFlavor = lfUnknown)` | `string` | Validate and return pretty-printed JSON string. | +| `validateFileJson` | `(filePath: string, flavor: LanguageFlavor = lfUnknown)` | `string` | Validate file and return pretty-printed JSON string. | -### JSON Output +#### Inspection -Both `ValidationResult` and `ValidationError` have a `.toJson()` method returning a `JsonNode`. Use `.pretty()` for formatted output. +| Function | Signature | Returns | Description | +|----------|-----------|---------|-------------| +| `inspectSource` | `(source: string, flavor: LanguageFlavor = lfUnknown, options: ValidationOptions = newValidationOptions(), sourcePath: string = "")` | `JsonNode` | Validate and return structural diagnostics (functions, classes, imports). | +| `inspectFile` | `(filePath: string, flavor: LanguageFlavor = lfUnknown, options: ValidationOptions = newValidationOptions())` | `JsonNode` | Inspect a file and return structural diagnostics. | -## Error Codes +#### Reporting -| Code | Meaning | -|---|---| -| `E1000` | Unexpected/unmatched bracket | -| `E1001` | Unclosed bracket | -| `E1002` | Unmatched closing bracket | -| `E2000` | Unclosed string literal | -| `E2001` | Invalid string escape | -| `E2002` | Unclosed multi-line string | -| `E3000` | Unclosed comment | -| `E3001` | Invalid comment syntax | -| `E4000` | Invalid syntax | -| `E4001` | Reserved word misuse | -| `E5000` | Unclosed block/control structure | -| `E9999` | Internal error (never-crash fallback) | +| Function | Signature | Returns | Description | +|----------|-----------|---------|-------------| +| `reportSource` | `(source: string, flavor: LanguageFlavor = lfUnknown)` | `string` | Human-readable validation report for source code. | +| `reportFile` | `(filePath: string, flavor: LanguageFlavor = lfUnknown)` | `string` | Human-readable validation report for a file. | -## Supported Languages +#### Detection and Utility -| Language | File Extensions | Tokenizer Features | -|---|---|---| -| Nim | `.nim`, `.nims`, `.nimble` | Block comments `#[ ]#` with nesting, triple-quoted strings, raw strings, doc comments | -| Bash | `.sh`, `.bash` | Shebang detection, heredocs, command substitution, variable expansion, fi/done/esac matching | -| Python | `.py` | F-strings with nesting, triple-quoted strings, decorators, async/await, match statements | -| JavaScript | `.js`, `.jsx`, `.mjs`, `.cjs` | Template literals with nesting, regex literals, arrow functions, ESM imports | -| PHP | `.php` | `` | +| `tkInterpolation` | String interpolation (`${var}`, f-string `{expr}`) | +| `tkDirective` | Preprocessor / compiler directive (`#include`, shebang) | +| `tkTemplateTag` | Template language tag (`{{ }}`, `{% %}`) | +| `tkSpecial` | Language-specific special token | +| `tkError` | Malformed or unrecognizable character | +| `tkEndOfFile` | End of input marker | + +### Token + +``` +Token = object + kind: TokenKind + value: string + position: SourcePosition + range: SourceRange ``` -## Development +### SourcePosition + +``` +SourcePosition = object + line: int # 1-indexed line number + column: int # 1-indexed column number + offset: int # 0-indexed byte offset from start +``` + +### SourceRange + +``` +SourceRange = object + startPos: SourcePosition + endPos: SourcePosition +``` + +### ErrorSeverity (enum) + +| Value | String | Description | +|-------|--------|-------------| +| `esInfo` | `"info"` | Informational note | +| `esWarning` | `"warning"` | Non-fatal issue | +| `esError` | `"error"` | Validation failure | +| `esCritical` | `"critical"` | Internal framework failure | + +### ValidationError + +``` +ValidationError = object + severity: ErrorSeverity + message: string + code: string # Error code, e.g. "E0002" + position: SourcePosition + range: SourceRange + context: string # Surrounding source context (optional) + hint: string # Fix suggestion (optional) +``` + +### ValidationResult + +``` +ValidationResult = object + valid: bool + flavor: LanguageFlavor + detectionMethod: FlavorDetectionMethod + errors: seq[ValidationError] + warnings: int + infos: int + moduleInfo: ModuleInfo + durationMs: float + debugOutput: string # Empty unless compiled with -d:nimcheckDebug +``` + +### ValidationOptions + +``` +ValidationOptions = object + flavor: LanguageFlavor # lfUnknown = auto-detect (default) + debugMode: bool # Enable per-validation debug logging (default: false) + strictMode: bool # Warnings become errors (default: false) + maxErrors: int # Stop after N errors; 0 = unlimited (default: 0) + includeContext: bool # Include surrounding source lines in error output (default: true) + includeHints: bool # Include fix suggestions (default: true) +``` + +### Structural Types (populated by `analyzeTokens()`) + +``` +VariableInfo = object + name: string + kind: string # "var", "let", "const", "global", "local" + typeAnnotation: string + defaultValue: string + position: SourcePosition + mutable: bool + exported: bool + +FunctionInfo = object + name: string + kind: string # "function", "method", "proc", "func", "lambda" + parameters: seq[ParameterInfo] + returnType: string + isAsync: bool + isExported: bool + isGenerator: bool + visibility: string # "public", "private", "protected" + position: SourcePosition + docComment: string + +ParameterInfo = object + name: string + typeAnnotation: string + defaultValue: string + isOptional: bool + isVarargs: bool + +ClassInfo = object + name: string + kind: string # "class", "object", "struct", "interface" + baseTypes: seq[string] + fields: seq[VariableInfo] + methods: seq[FunctionInfo] + generics: seq[string] + visibility: string + position: SourcePosition + docComment: string + +ImportInfo = object + module: string + symbols: seq[string] # Specific imported symbols + isRelative: bool + alias: string + position: SourcePosition + +ModuleInfo = object + flavor: LanguageFlavor + detectionMethod: FlavorDetectionMethod + imports: seq[ImportInfo] + variables: seq[VariableInfo] + functions: seq[FunctionInfo] + classes: seq[ClassInfo] + linesOfCode: int + totalTokens: int + comments: int + debugInfo: JsonNode # Language-specific extra info +``` + +--- + +## JSON Output Structures + +Every type has a `.toJson()` method. The `ValidationResult.toJson()` aggregates all nested structures. + +### ValidationResult JSON + +```json +{ + "valid": true, + "flavor": "python", + "detectionMethod": "by_extension", + "errors": [ + { + "severity": "error", + "message": "Unclosed string literal starting with '\"'", + "code": "E0002", + "position": {"line": 2, "column": 13, "offset": 42}, + "range": { + "start": {"line": 2, "column": 13, "offset": 42}, + "end": {"line": 2, "column": 13, "offset": 42} + }, + "context": "", + "hint": "Add a closing quote before end of file" + } + ], + "warnings": 0, + "infos": 0, + "moduleInfo": { + "flavor": "python", + "detectionMethod": "by_extension", + "imports": [], + "variables": [], + "functions": [ + { + "name": "hello", + "kind": "def", + "parameters": [], + "returnType": "", + "isAsync": false, + "isExported": false, + "isGenerator": false, + "visibility": "", + "position": {"line": 1, "column": 5, "offset": 4}, + "docComment": "" + } + ], + "classes": [], + "linesOfCode": 3, + "totalTokens": 12, + "comments": 0, + "debugInfo": null + }, + "durationMs": 0.342, + "debugOutput": "" +} +``` + +### ValidationError JSON + +```json +{ + "severity": "error", + "message": "Unclosed string literal starting with '\"'", + "code": "E0002", + "position": {"line": 2, "column": 13, "offset": 42}, + "range": { + "start": {"line": 2, "column": 13, "offset": 42}, + "end": {"line": 2, "column": 13, "offset": 42} + }, + "context": "", + "hint": "Add a closing quote before end of file" +} +``` + +### ModuleInfo JSON + +```json +{ + "flavor": "nim", + "detectionMethod": "by_extension", + "imports": [ + { + "module": "strutils", + "symbols": [], + "isRelative": false, + "alias": "", + "position": {"line": 1, "column": 8, "offset": 7} + } + ], + "variables": [], + "functions": [ + { + "name": "add", + "kind": "proc", + "parameters": [], + "returnType": "", + "isAsync": false, + "isExported": false, + "isGenerator": false, + "visibility": "", + "position": {"line": 3, "column": 6, "offset": 28}, + "docComment": "" + } + ], + "classes": [ + { + "name": "MyType", + "kind": "type", + "baseTypes": [], + "fields": [], + "methods": [], + "generics": [], + "visibility": "", + "position": {"line": 5, "column": 6, "offset": 52}, + "docComment": "" + } + ], + "linesOfCode": 8, + "totalTokens": 45, + "comments": 2, + "debugInfo": null +} +``` + +### Human-Readable Report Format + +``` +=== Validation Report === + Flavor: python + Detection: by_extension + Valid: false + Errors: 1 + Warnings: 0 + Duration: 0.34ms + Lines of code: 3 + Tokens: 12 + + --- Findings --- + [E0002] (error) L2:13: Unclosed string literal starting with '"' + | x = " + | ^ + hint: Add a closing quote before end of file + + --- Summary --- + INVALID - source has issues that need attention. +``` + +### Error Formatter Output (single error) + +``` + [E0002] (error) L2:13: Unclosed string literal starting with '"' + | x = " + | ^ + hint: Add a closing quote before end of file +``` + +--- + +## Error Code Reference + +Error codes follow a prefixed numbering scheme. The prefix encodes the category and severity. + +### General Errors (E0xxx) + +| Code | Constant | Description | Triggered By | +|------|----------|-------------|-------------| +| `E0001` | `ErrUnexpectedToken` | Unexpected or unrecognizable character | Tokenizer encounter of non-ASCII, non-operator char | +| `E0002` | `ErrUnclosedString` | Unclosed string literal | Missing closing quote at EOF | +| `E0003` | `ErrUnclosedComment` | Unclosed block comment | Missing `*/`, `]#`, `#}` etc. at EOF | +| `E0004` | `ErrUnclosedBlock` | Unclosed block/control structure | Language-specific: unclosed `if`/`for`/`class`/etc. | +| `E0005` | `ErrMismatchedBracket` | Mismatched or unexpected closing bracket | `)` without `(`, `` without `
` | +| `E0006` | `ErrUnclosedBracket` | Unclosed bracket at EOF | `validateBrackets()` finds remaining items on bracket stack | +| `E0007` | `ErrInvalidSyntax` | General invalid syntax | JSON parse error, TOML missing `=`, YAML empty key | +| `E0008` | `ErrUnexpectedEOF` | Unexpected end of file | Reserved for incomplete constructs not covered by other codes | +| `E0009` | `ErrEmptySource` | Source is empty (zero-length string) | `validate()` with `source.len == 0` | +| `E0010` | `ErrEncodingError` | Encoding error | Reserved for non-UTF-8 input | +| `E0011` | -- | File not found | `validateFile()` with non-existent path | + +### Language-Specific Errors (E1xxx) + +| Code | Constant | Description | Languages | +|------|----------|-------------|-----------| +| `E1001` | `ErrUnexpectedIndent` | Unexpected or inconsistent indentation | Python | +| `E1002` | `ErrMissingIndent` | Missing indentation after block opener | Python | +| `E1003` | `ErrDuplicateParam` | Duplicate parameter name | (reserved) | +| `E1004` | `ErrUndefinedVariable` | Reference to undefined variable | (reserved) | +| `E1005` | `ErrTypeMismatch` | Type mismatch | (reserved) | +| `E1006` | `ErrInvalidAssignment` | Invalid assignment target | (reserved) | +| `E1007` | `ErrMissingReturn` | Function missing return statement | (reserved) | +| `E1008` | `ErrInvalidDecorator` | Invalid decorator usage | Python | +| `E1009` | `ErrInvalidImport` | Invalid import statement | (reserved) | + +### Template Errors (E2xxx) + +| Code | Constant | Description | Languages | +|------|----------|-------------|-----------| +| `E2001` | `ErrUnclosedTemplateTag` | Unclosed template expression/block tag | Jinja | +| `E2002` | `ErrUnclosedBlockTag` | Unclosed block tag with no matching end tag | Jinja | +| `E2003` | `ErrInvalidFilter` | Invalid filter in template expression | Jinja | + +### Info Codes (I0xxx) + +| Code | Constant | Description | +|------|----------|-------------| +| `I0001` | `InfoLongLine` | Line exceeds recommended length | +| `I0002` | `InfoTrailingWhitespace` | Trailing whitespace detected | + +### Warning Codes (W0xxx / W1xxx) + +| Code | Constant | Description | +|------|----------|-------------| +| `W0001` | `WarnUnusedVariable` | Variable declared but never used | +| `W0002` | `WarnInconsistentNaming` | Naming convention inconsistency | +| `W0003` | `WarnMissingDoc` | Public symbol missing documentation | +| `W1001` | -- | Tab characters in YAML indentation | + +### Internal Error + +| Code | Description | Triggered By | +|------|-------------|-------------| +| `E9999` | Internal framework error (never-crash fallback) | Any unhandled `Exception` in `validate()` or `validateSource()` | + +--- + +## IO Table: Supported Languages and Per-Language Errors + +### Input/Output Contract + +Every validation function follows the same contract: + +| Input | Output | +|-------|--------| +| `source: string` (required) | `ValidationResult` with `valid: bool` | +| `flavor: LanguageFlavor = lfUnknown` | `ValidationResult.errors: seq[ValidationError]` | +| `options: ValidationOptions` (optional) | `ValidationResult.moduleInfo: ModuleInfo` | +| `sourcePath: string = ""` (optional) | `ValidationResult.durationMs: float` | + +**Guarantees**: +- Never throws — all exceptions are caught and returned as `E9999` errors +- Empty source returns `E0009` (or `valid: true` for YAML/TOML where empty is valid) +- File not found returns `E0011` +- Unknown flavor with no detection possible returns an `E0007` error +- Every error has a `code`, `message`, `severity`, `position`, and optional `hint` + +### Per-Language Tokenizer, Validator, and Error Table + +#### Nim (`.nim`, `.nims`, `.nimble`) + +| Aspect | Details | +|--------|---------| +| Tokenizer | `nim_tokenizer.nim` — `NimTokenizer` | +| Validator | `nim_validator.nim` — `NimValidator` | +| Keywords | 80+ (addr, and, as, block, break, case, const, converter, defer, discard, elif, else, enum, except, export, finally, for, from, func, if, import, in, include, iterator, let, macro, method, mixin, mod, nil, not, object, of, or, proc, ptr, raise, ref, return, template, try, tuple, type, using, var, when, while, with, yield, etc.) | + +| Error Code | Condition | Severity | Hint | +|------------|-----------|----------|------| +| `E0001` | Unexpected character (non-ASCII, unrecognizable) | `esWarning` | Shows hex `U+XXXX` value | +| `E0002` | Unclosed string (`"..."` at EOF) | `esError` | "Add a closing quote" | +| `E0002` | Unclosed multi-line string (`"""` at EOF) | `esError` | -- | +| `E0002` | Newline inside string literal | `esError` | -- | +| `E0002` | Unclosed raw string (`r"..."` at EOF) | `esError` | -- | +| `E0002` | Unclosed character literal (`'x` at EOF) | `esError` | -- | +| `E0003` | Unclosed block comment (`#[` without `]#`) | `esError` | "Add closing `]#`" | +| `E0006` | Unclosed bracket (`(`, `[`, `{` at EOF) | `esError` | "Add matching closing bracket" | + +Structural extraction: imports (`from X import Y`), functions (`proc`, `func`, `method`, `template`, `macro`, `iterator`, `converter`), types (`type Foo`). + +#### Bash / Shell (`.sh`, `.bash`, `.zsh`, `.fish`) + +| Aspect | Details | +|--------|---------| +| Tokenizer | `bash_tokenizer.nim` — `BashTokenizer` | +| Validator | `bash_validator.nim` — `BashValidator` | +| Keywords | 38+ (if, then, else, elif, fi, case, esac, for, while, until, do, done, in, function, select, time, return, break, continue, exit, declare, local, export, readonly, unset, eval, exec, let, source, shift, trap, type, typeset, ulimit, umask, alias, read, echo) | + +| Error Code | Condition | Severity | Hint | +|------------|-----------|----------|------| +| `E0002` | Unclosed single-quoted string | `esError` | -- | +| `E0002` | Unclosed double-quoted string | `esError` | -- | +| `E0004` | Unclosed `${var}` expansion | `esError` | -- | +| `E0004` | Unclosed `$()` command substitution | `esError` | -- | +| `E0004` | Unclosed `if` block (`if` without matching `fi`) | `esError` | "Add missing 'fi'" | +| `E0004` | Unclosed loop (`for`/`while`/`until` without `done`) | `esError` | "Add missing 'done'" | +| `E0004` | Unclosed `case` block (`case` without `esac`) | `esError` | "Add missing 'esac'" | +| `E0006` | Unclosed bracket (`(`, `[`, `{` at EOF) | `esError` | -- | + +Special features: shebang detection (`#!`), heredoc detection (`<`), ESM import/export. + +Structural extraction: imports/exports, functions (`function`, `async`), classes. + +#### PHP (`.php`, `.phtml`) + +| Aspect | Details | +|--------|---------| +| Tokenizer | `php_tokenizer.nim` — `PHPTokenizer` | +| Validator | `php_validator.nim` — `PHPValidator` | +| Mode switching | Tracks `inPhpMode` flag — toggles on `` | + +| Error Code | Condition | Severity | Hint | +|------------|-----------|----------|------| +| `E0002` | Unclosed string | `esError` | -- | +| `E0006` | Unclosed bracket (`(`, `[`, `{` at EOF) | `esError` | -- | + +Special features: PHP/HTML mode switching, `` detection, variable names (`$var`, `$obj->prop`). + +Structural extraction: functions (`function`, `fn`), classes (`class`), imports/requires (`include`, `require`, `use`, `namespace`). + +#### HTML / XML (`.html`, `.htm`, `.xhtml`, `.xml`, `.svg`) + +| Aspect | Details | +|--------|---------| +| Tokenizer | `html_tokenizer.nim` — `HTMLTokenizer` | +| Validator | `html_validator.nim` — `HTMLValidator` | +| Void elements | 14: area, base, br, col, embed, hr, img, input, link, meta, param, source, track, wbr | + +| Error Code | Condition | Severity | Hint | +|------------|-----------|----------|------| +| `E0003` | Unclosed HTML comment (``) | `esError` | -- | +| `E0004` | Unclosed HTML tag (e.g., `
` without `
`) | `esError` | "Add closing ``" | +| `E0005` | Unexpected closing tag (mismatched nesting) | `esWarning` | "Check that tags are properly nested" | + +Special features: tag stack balancing, void element awareness (no closing tag needed for self-closing elements), attribute parsing with quoted values, ``, `` directives. + +#### Jinja2 (`.jinja`, `.jinja2`, `.j2`, `.html.j2`) + +| Aspect | Details | +|--------|---------| +| Tokenizer | `jinja_tokenizer.nim` — `JinjaTokenizer` | +| Validator | `jinja_validator.nim` — `JinjaValidator` | + +| Error Code | Condition | Severity | Hint | +|------------|-----------|----------|------| +| `E0003` | Unclosed Jinja comment (`{#` without `#}`) | `esError` | -- | +| `E2001` | Unclosed expression tag (`{{` without `}}`) | Implicit via tokenizer | -- | +| `E2001` | Unclosed block tag (`{%` without `%}`) | Implicit via tokenizer | -- | +| `E2002` | Unclosed `for` block (no matching `endfor`) | `esError` | "Add `{% endfor %}`" | +| `E2002` | Unclosed `if` block (no matching `endif`) | `esError` | "Add `{% endif %}`" | +| `E2002` | Unclosed `block` (no matching `endblock`) | `esError` | "Add `{% endblock %}`" | +| `E2002` | Unclosed `macro`, `filter`, `raw`, `autoescape`, `call`, `with` | `esError` | "Add matching `end{tag}`" | + +Special features: tracks 12 block tag types (`for`, `if`, `block`, `macro`, `filter`, `raw`, `autoescape`, `call`, `with`), validates matching end tags, handles content between template syntax and plain text. + +#### JSON (`.json`, `.jsonc`) + +| Aspect | Details | +|--------|---------| +| Validator | `config_validators.nim` — `JSONValidator` | +| Method | Uses Nim's built-in `parseJson()` for full validation | + +| Error Code | Condition | Severity | Hint | +|------------|-----------|----------|------| +| `E0007` | JSON parse error (any `JsonParsingError`) | `esError` | Error message from parser | +| `E0007` | Any other exception during parse | `esError` | "JSON error: {message}" | +| `E0009` | Empty JSON source | `esError` | -- | + +#### YAML (`.yaml`, `.yml`) + +| Aspect | Details | +|--------|---------| +| Validator | `config_validators.nim` — `YAMLValidator` | +| Method | Line-by-line structural analysis | + +| Error Code | Condition | Severity | Hint | +|------------|-----------|----------|------| +| `E0007` | Empty mapping key (no key before `:`) | `esError` | -- | +| `E0007` | General YAML structural error | `esError` | -- | +| `E0009` | No content after stripping comments/blanks | `esError` | -- | +| `W1001` | Tab characters used for indentation | `esWarning` | "Use spaces for YAML indentation" | + +Note: Empty YAML (only whitespace/comments) is `valid: true`. + +#### TOML (`.toml`) + +| Aspect | Details | +|--------|---------| +| Validator | `config_validators.nim` — `TOMLValidator` | +| Method | Line-by-line structural analysis | + +| Error Code | Condition | Severity | Hint | +|------------|-----------|----------|------| +| `E0007` | Missing `=` in key-value pair | `esError` | "TOML uses `key = value` format" | +| `E0007` | Missing key before `=` | `esError` | -- | +| `E0007` | General TOML structural error | `esError` | -- | +| `E0009` | No content after stripping comments/blanks | `esError` | -- | + +--- + +## CLI Usage + +### Full CLI Reference + +``` +Nimcheck v0.1.0 - Universal Source Code Validation Framework + +Usage: + nimcheck validate [--file= | --source=] [--flavor=] + nimcheck inspect [--file= | --source=] [--flavor=] + nimcheck detect [--file= | --source=] + nimcheck help + +Options: + --file= Source file to process + --source= Source code string to process + --flavor= Language flavor (auto-detect if omitted) + --debug Enable debug mode + --json Output as JSON (default for inspect) + +Supported flavors: + - bash + - html + - jinja + - jinja2 + - js + - json + - jsonc + - nim + - nimble + - nims + - php + - phtml + - python + - py + - shell + - sh + - toml + - ts + - typescript + - yaml + - yml +``` + +### Exit Codes + +| Exit Code | Meaning | +|-----------|---------| +| 0 | Validation successful (or help/version requested) | +| Non-zero (Nim default) | Runtime error | + +Note: The `validate` command always exits 0 on the Nim level; the `valid` field in the output indicates pass/fail. This is by design — Nimcheck never crashes. + +--- + +## Building and Testing ### Prerequisites - Nim >= 2.0.0 -- `make` (optional, for Makefile targets) +- `make` (optional for Makefile targets) -### Project structure +### Build -``` -src/ - nimcheck.nim Public API entry point and CLI (isMainModule) - nimcheck/ - core/ - types.nim Core types: ValidationResult, ValidationError, LanguageFlavor - tokenizerbase.nim Abstract tokenizer base class - validatorbase.nim Abstract validator base class - detector.nim Language auto-detection - debug.nim Debug logging infrastructure - tokenizers/ - nim_tokenizer.nim - bash_tokenizer.nim - python_tokenizer.nim - javascript_tokenizer.nim - php_tokenizer.nim - html_tokenizer.nim - jinja_tokenizer.nim - languages/ - nim_validator.nim - bash_validator.nim - python_validator.nim - javascript_validator.nim - php_validator.nim - html_validator.nim - jinja_validator.nim - config_validators.nim JSON, YAML, TOML validators - reporting/ - errors.nim Error formatting and JSON output -tests/ - test_all.nim Master test runner (imports all test modules) - test_.nim Per-language tests - test__exhaustive.nim Exhaustive edge-case tests per language - test_config.nim JSON/YAML/TOML tests - test_fuzz.nim Fuzz and boundary tests - test_common.nim Shared test utilities - test_mixed.nim Mixed-language file tests - fixtures/ Test data files per language +```bash +# Using Makefile +make build # Release build -> bin/nimcheck +make build-debug # Debug build with -d:nimcheckDebug + +# Using nimble +nimble build + +# Manual +nim c src/nimcheck.nim +nim c -d:nimcheckDebug src/nimcheck.nim # With debug output ``` -### Building with warnings +### Test + +The test suite covers **250+ test cases** across all supported languages. + +```bash +make test # Run all tests +make test-nim # Nim-specific tests only +make test-python # Python-specific tests only +make test-bash # Bash-specific tests only +make test-js # JavaScript/TypeScript tests +make test-php # PHP tests +make test-html # HTML/XML tests +make test-jinja # Jinja2 tests +make test-json # JSON tests +make test-yaml # YAML tests +make test-toml # TOML tests +make test-mixed # Mixed-language file tests +make debug # Build debug + run all tests +``` + +### Test Categories + +| Suite | Files | Coverage | +|-------|-------|----------| +| Per-language basic | `test_.nim` | Valid + invalid source for each language | +| Per-language exhaustive | `test__exhaustive.nim` | Edge cases: unclosed strings, unclosed comments, mismatched brackets, binary data, Unicode | +| Config formats | `test_config.nim` + `test__exhaustive.nim` | JSON/YAML/TOML structural validation | +| Fuzz tests | `test_fuzz.nim` | Null bytes, binary data, deep nesting (100 levels), empty sources, concurrent validation | +| Mixed files | `test_mixed.nim` | HTML+Jinja combined, other mixed-format files | +| Common | `test_common.nim` | Shared test fixtures for valid and invalid code per language | + +### Lint ```bash make lint -# or manually: +# or nim c --hints:on --warnings:on src/nimcheck.nim ``` -### Adding a new language +### Clean -See the detailed checklist in [LESSONS_LEARNED.md](LESSONS_LEARNED.md) (Section 14). +```bash +make clean +make rebuild # clean + build +``` -## Architecture +--- -The framework uses a layered architecture: +## Project Structure -1. **Tokenizers** -- convert raw source into a stream of tokens, tracking bracket depth, string context, and comments -2. **Validators** -- analyze token streams for language-specific syntax rules -3. **Detector** -- identifies language from file extension, shebang, or content heuristics -4. **Reporting** -- formats errors as human-readable strings or structured JSON +``` +. +|-- Makefile +|-- nimcheck.nimble +|-- README.md +|-- LESSONS_LEARNED.md +|-- CONTRIBUTING.md +|-- LICENSE +|-- bin/ # Build output +| `-- nimcheck # Compiled binary +|-- src/ +| |-- nimcheck.nim # Public API entry point + CLI (isMainModule) +| `-- nimcheck/ +| |-- core/ +| | |-- types.nim # All core types, toJson() methods, error code constants +| | |-- tokenizerbase.nim # Abstract TokenizerBase with position tracking, bracket stack, error recording +| | |-- validatorbase.nim # Abstract ValidatorBase with pipeline, factory, registry +| | |-- detector.nim # Language auto-detection (extension, shebang, content scoring) +| | |-- debug.nim # Debug logging (compiled only with -d:nimcheckDebug) +| | `-- init_validators.nim +| |-- tokenizers/ +| | |-- nim_tokenizer.nim # Nim: """ strings, r"..." raw strings, #[ ]# nested comments, ## doc comments +| | |-- bash_tokenizer.nim # Bash: heredocs, $var/${var}, $(), backticks, (( )) +| | |-- python_tokenizer.nim # Python: f-strings, """ strings, decorators, indentation tracking +| | |-- javascript_tokenizer.nim # JS/TS: template literals, regex literals, arrow functions +| | |-- php_tokenizer.nim # PHP: mode switching, $variables +| | |-- html_tokenizer.nim # HTML/XML: tag balancing, void elements, attributes +| | `-- jinja_tokenizer.nim # Jinja2: {{ }}, {% %}, {# #} block detection +| |-- languages/ +| | |-- nim_validator.nim # Analyzes Nim: imports, procs, types +| | |-- bash_validator.nim # Analyzes Bash: if/fi, for/done, case/esac matching +| | |-- python_validator.nim # Analyzes Python: imports, def, class +| | |-- javascript_validator.nim # Analyzes JS/TS: imports, functions, classes +| | |-- php_validator.nim # Analyzes PHP: functions, classes, includes +| | |-- html_validator.nim # Analyzes HTML/XML: tag balancing +| | |-- jinja_validator.nim # Analyzes Jinja2: block tag matching (12 tag types) +| | `-- config_validators.nim # JSON (via parseJson), YAML, TOML validators +| `-- reporting/ +| `-- errors.nim # Human-readable and JSON error formatting +`-- tests/ + |-- test_all.nim # Master test runner (imports all) + |-- test_nim.nim # Nim basic tests + |-- test_nim_exhaustive.nim # Nim edge cases + |-- test_bash.nim + |-- test_bash_exhaustive.nim + |-- test_python.nim + |-- test_python_exhaustive.nim + |-- test_javascript.nim + |-- test_javascript_exhaustive.nim + |-- test_php.nim + |-- test_php_exhaustive.nim + |-- test_html.nim + |-- test_html_exhaustive.nim + |-- test_jinja.nim + |-- test_jinja_exhaustive.nim + |-- test_json_exhaustive.nim + |-- test_yaml_exhaustive.nim + |-- test_toml_exhaustive.nim + |-- test_config.nim # Config format tests + |-- test_mixed.nim # Mixed-language file tests + |-- test_fuzz.nim # Fuzz and boundary tests + |-- test_common.nim # Shared test fixtures + `-- fixtures/ # Per-language test data files + |-- nim/ + |-- bash/ + |-- python/ + |-- javascript/ + |-- php/ + |-- html/ + |-- jinja/ + |-- json/ + |-- yaml/ + |-- toml/ + `-- mixed/ +``` -Validators register themselves at import time via a side-effect factory pattern. The public API (`nimcheck.nim`) imports all validators and re-exports core types for consumer convenience. - -## Versioning - -Current version: **0.1.0** +--- ## Contributing -Contributions are welcome. Please open an issue to discuss your idea before -submitting a pull request. For detailed instructions on adding support for a new -language, see [LESSONS_LEARNED.md](LESSONS_LEARNED.md) Section 16. +Contributions are welcome. Open an issue to discuss your idea before submitting a pull request. + +### Adding a New Language + +See `LESSONS_LEARNED.md` for the complete checklist (Section 14). The minimum steps are: + +1. Create `src/nimcheck/tokenizers/_tokenizer.nim` inheriting from `TokenizerBase` +2. Implement the `tokenize()` method with language-specific lexing +3. Create `src/nimcheck/languages/_validator.nim` inheriting from `ValidatorBase` +4. Implement `createTokenizer()`, `analyzeTokens()`, and `buildModuleInfo()` +5. Add `registerValidator("flavor", constructor)` in the validator's `init()` proc +6. Import the validator in `src/nimcheck.nim` +7. Add extension mappings to `detector.nim` +8. Add test fixtures and test files under `tests/` +9. Update the test master runner `tests/test_all.nim` + +### Design Principles + +- **KISS**: Keep validators simple — tokenize, analyze brackets, report +- **Never crash**: Every validation path in a try/except +- **No external deps**: Pure Nim standard library +- **DRY**: Common logic in `tokenizerbase.nim` and `validatorbase.nim` + +--- ## License -This project is licensed under the MIT License -- see the [LICENSE](LICENSE) file -for details. +MIT -- see [LICENSE](LICENSE) for details. ## Author diff --git a/src/nimcheck.nim b/src/nimcheck.nim index 0cd58a0..641b94d 100644 --- a/src/nimcheck.nim +++ b/src/nimcheck.nim @@ -315,6 +315,7 @@ when isMainModule: of "validate", "val": var filePath, source, flavor = "" var debugMode = false + var jsonOut = false var i = 1 while i < args.len: let a = args[i] @@ -322,6 +323,7 @@ when isMainModule: elif a.startsWith("--source="): source = a[9..^1] elif a.startsWith("--flavor="): flavor = a[9..^1] elif a == "--debug": debugMode = true + elif a == "--json": jsonOut = true i += 1 var opts = newValidationOptions() @@ -333,20 +335,27 @@ when isMainModule: except: discard var r: ValidationResult + var src: string = "" if filePath.len > 0: r = validateFile(filePath, fl, opts) + if fileExists(filePath): src = readFile(filePath) elif source.len > 0: + src = source r = validateSource(source, fl, opts, "stdin") else: # Read from stdin - let input = readAll(stdin).strip() - r = validateSource(input, fl, opts, "stdin") + src = readAll(stdin).strip() + r = validateSource(src, fl, opts, "stdin") - echo formatReport(r, r.moduleInfo.toJson().pretty()) + if jsonOut: + echo r.toJson().pretty() + else: + echo formatReport(r, src) of "inspect", "ins": var filePath, source, flavor = "" var debugMode = false + var jsonOut = false var i = 1 while i < args.len: let a = args[i] @@ -354,6 +363,7 @@ when isMainModule: elif a.startsWith("--source="): source = a[9..^1] elif a.startsWith("--flavor="): flavor = a[9..^1] elif a == "--debug": debugMode = true + elif a == "--json": jsonOut = true i += 1 var opts = newValidationOptions()