diff --git a/.gitignore b/.gitignore index d4b1479..583e647 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,8 @@ + + +tests/test_json_exhaustive +tests/test_toml_exhaustive +tests/test_yaml_exhaustive # ── Nim ────────────────────────────────────────────────────────────────── nimcache/ nimblecache/ diff --git a/Makefile b/Makefile index bba3e83..bdd02a5 100644 --- a/Makefile +++ b/Makefile @@ -52,13 +52,13 @@ test-jinja: $(NIMC) -r $(TEST_DIR)/test_jinja.nim test-json: - $(NIMC) -r $(TEST_DIR)/test_json.nim + $(NIMC) -r $(TEST_DIR)/test_json_exhaustive.nim test-yaml: - $(NIMC) -r $(TEST_DIR)/test_yaml.nim + $(NIMC) -r $(TEST_DIR)/test_yaml_exhaustive.nim test-toml: - $(NIMC) -r $(TEST_DIR)/test_toml.nim + $(NIMC) -r $(TEST_DIR)/test_toml_exhaustive.nim test-mixed: $(NIMC) -r $(TEST_DIR)/test_mixed.nim diff --git a/README.md b/README.md index df34ea5..abf31e9 100644 --- a/README.md +++ b/README.md @@ -1,118 +1,152 @@ + # Nimcheck -A multi-language source code validation framework written in Nim. Nimcheck uses tokenizer-based validation to check syntax correctness across 10 languages and config formats, with auto-detection, comprehensive diagnostics, and a never-crash guarantee. +Universal source code validation framework written in Nim. Tokenizer-based syntax validation across 10 languages and config formats, with auto-detection, structured JSON diagnostics, structural inspection, and a never-crash guarantee. -## Features +Version: **0.1.0** | Author: **molodetz** | License: **MIT** | Requires: **Nim >= 2.0.0** -- **10 languages supported**: Nim, Bash, Python, JavaScript, PHP, HTML, Jinja, JSON, YAML, TOML -- **Tokenizer-based validation** -- real tokenizers that track bracket depth, string context, comments, and nesting -- **Auto-detection** -- detect language from file extension or content analysis -- **Comprehensive diagnostics** -- structured error reports with position, severity, hints, and debug output -- **Never-crash guarantee** -- every validation path is wrapped in try/except; invalid input returns errors gracefully, never panics -- **JSON output** -- machine-readable diagnostics via `toJson()` on any validation result -- **Inspection API** -- extract module structure (imports, definitions, functions/classes) as JSON -- **Debug mode** -- compile with `-d:nimcheckDebug` for detailed tokenization logging -- **No external dependencies** -- pure Nim standard library +--- + +## Table of Contents + +1. [Architecture](#architecture) +2. [Quick Start](#quick-start) +3. [API Reference](#api-reference) +4. [Data Types](#data-types) +5. [JSON Output Structures](#json-output-structures) +6. [Error Code Reference](#error-code-reference) +7. [IO Table: Supported Languages and Per-Language Errors](#io-table-supported-languages-and-per-language-errors) +8. [CLI Usage](#cli-usage) +9. [Building and Testing](#building-and-testing) +10. [Project Structure](#project-structure) +11. [Contributing](#contributing) + +--- + +## Architecture + +Nimcheck uses a four-layer pipeline architecture. Each layer is independent and communicates through well-defined interfaces. + +``` +Source Code (string or file) + | + v ++-------------------+ +| Detector | LanguageFlavor from extension, shebang, or content analysis ++-------------------+ + | + v ++-------------------+ +| Tokenizer | Converts raw source into a seq[Token] stream ++-------------------+ + | + v ++-------------------+ +| Validator | Analyzes token stream for syntax errors and structure ++-------------------+ + | + v ++-------------------+ +| Reporting | Formats errors as human-readable text or structured JSON ++-------------------+ + | + v + ValidationResult +``` + +### Layer 1: Detector (`src/nimcheck/core/detector.nim`) + +Language auto-detection with cascading priority: + +| Priority | Method | Source | Confidence | +|----------|--------|--------|------------| +| 1 (highest) | Explicit | User-provided `LanguageFlavor` | 1.0 | +| 2 | File extension | 70+ extension-to-flavor mappings | 0.9 | +| 3 | Shebang line | `#!` line interpreter matching (bash, python, node, ruby, lua, swift) | 0.95 | +| 4 | Content analysis | Keyword scoring, structural patterns, bracket usage | 0.6-0.9 | + +The `detectFlavor()` function returns a `(LanguageFlavor, FlavorDetectionMethod, confidence: float)` tuple. + +### Layer 2: Tokenizer (`src/nimcheck/core/tokenizerbase.nim`, per-language impls) + +Each tokenizer converts raw source into a `seq[Token]`. Tokenizers track: + +- **Position**: line, column, and byte offset for every token +- **Bracket stack**: `(`, `[`, `{`, `<` pushed and popped with position tracking +- **String context**: escape sequences, multi-line delimiters, interpolation +- **Comment context**: line comments, block comments with nesting support + +Tokenizers emit errors for: +- Unclosed strings (`ErrUnclosedString`) +- Unclosed comments (`ErrUnclosedComment`) +- Unexpected characters (`ErrUnexpectedToken`) +- Unclosed block constructs (language-specific, e.g., `${}`, `$()`) + +### Layer 3: Validator (`src/nimcheck/core/validatorbase.nim`, per-language impls) + +Each validator: + +1. Creates its tokenizer and runs `tokenize()` +2. Collects tokenizer errors +3. Runs `analyzeTokens()` — language-specific structural analysis +4. Runs `buildModuleInfo()` — populates `ModuleInfo` with imports, functions, classes +5. Computes validity, warning/info counts, duration + +Validators register themselves via a factory pattern at import time. The registry maps flavor strings (e.g., `"python"`, `"bash"`) to constructors. + +### Layer 4: Reporting (`src/nimcheck/reporting/errors.nim`) + +Produces: +- **Human-readable reports**: `formatReport()` with position, severity, code, message, source context with carets, and hints +- **JSON output**: every type has a `toJson()` method producing `JsonNode` +- **Error categorization**: `countBySeverity()`, `countByCode()`, `filterBySeverity()` + +--- ## Quick Start +### Nim library usage + ```nim import nimcheck # Validate source code from a string let r = validateSource("proc hello() = echo 'hi'", lfNim) -echo r.errors # seq[ValidationError] +echo r.valid # bool +echo r.errors # seq[ValidationError] +echo r.toJson().pretty() # Full JSON output + +# Validate a file (auto-detects language from extension) +let r2 = validateFile("src/main.nim") # Auto-detect language -let r2 = validateSource("print('hello')", lfUnknown) # detects Python +let r3 = validateSource("print('hello')", lfUnknown) +echo r3.flavor # lfPython -# Validate a file -let r3 = validateFile("path/to/file.nim", lfNim) +# Inspect source structure (returns JsonNode with functions, classes, imports) +let info = inspectSource("class Foo:\n def bar(self): pass", lfPython) +echo info.pretty() -# Get JSON diagnostics -echo r.toJson().pretty() - -# Inspect source structure (returns JsonNode) -let info = inspectSource("class Foo: ...", lfPython) -echo info.pretty() # formatted JSON with class/function info - -# Get human-readable report -let report = reportSource("proc foo() =", lfNim) -echo report # formatted error summary +# Human-readable report +echo reportSource("proc foo() =\n let x = \"unclosed", lfNim) ``` -## Building - -### Using Makefile (recommended) +### CLI usage ```bash -# Build the library binary +# Build the binary make build -# Run all tests -make test - -# Build with debug mode -make build-debug -make debug # build debug + run all tests - -# Run a specific language test -make test-nim -make test-python -make test-bash - -# Lint with full warnings -make lint - -# Clean build artifacts -make clean - -# Show all available targets -make help -``` - -### Using nimble - -```bash -# Run all tests -nimble test - -# Run per-language tests -nimble test_nim -nimble test_python - -# Build the library -nimble build -``` - -### Manual compilation - -```bash -# Build the library -nim c src/nimcheck.nim - -# Build with debug mode -nim c -d:nimcheckDebug src/nimcheck.nim - -# Run all tests -nim c -r tests/test_all.nim -``` - -## CLI Usage - -When compiled as a standalone binary (`make build` produces `bin/nimcheck`): - -```bash # Validate a file bin/nimcheck validate --file=source.nim -# Validate source code directly +# Validate source directly bin/nimcheck validate --source="print('hi')" --flavor=python # Read from stdin echo "var x = 1" | bin/nimcheck validate -# Inspect source structure (returns JSON) +# Inspect structure (JSON output) bin/nimcheck inspect --file=source.py # Detect language @@ -124,174 +158,944 @@ bin/nimcheck version bin/nimcheck help ``` +--- + ## API Reference ### Core Functions -| Function | Description | -|---|---| -| `validateSource(source, flavor, options, sourcePath)` | Validate source code string; returns `ValidationResult` | -| `validateFile(filePath, flavor, options)` | Validate a file; returns `ValidationResult` | -| `inspectSource(source, flavor, options, sourcePath)` | Return structural diagnostics as `JsonNode` | -| `inspectFile(filePath, flavor, options)` | Return structural diagnostics for a file as `JsonNode` | -| `validateSourceJson(source, flavor)` | Validate source and return result as JSON string | -| `validateFileJson(filePath, flavor)` | Validate file and return result as JSON string | -| `reportSource(source, flavor)` | Get a human-readable validation report (string) | -| `reportFile(filePath, flavor)` | Get a human-readable validation report for a file (string) | -| `detectFlavor(source, filePath)` | Auto-detect language flavor | -| `supportedFlavors()` | List all registered language flavors (returns `seq[string]`) | -| `version()` | Get the Nimcheck version string | -| `enableDebug()` / `disableDebug()` | Toggle debug mode globally | +All functions are exported from `nimcheck.nim`. Import once with `import nimcheck`. -### Types +#### Validation -- **`ValidationResult`** -- contains `errors: seq[ValidationError]`, `valid: bool`, `moduleInfo`, `debugOutput`, `detectionInfo` -- **`ValidationError`** -- contains `code`, `message`, `severity`, `position`, `hint`, `context` -- **`ValidationOptions`** -- contains `flavor`, `debugMode`, `maxErrors`, `validateAll` -- **`LanguageFlavor`** -- enum: `lfUnknown`, `lfNim`, `lfBash`, `lfPython`, `lfJavaScript`, `lfPHP`, `lfHTML`, `lfJinja`, `lfJSON`, `lfYAML`, `lfTOML` +| Function | Signature | Returns | Description | +|----------|-----------|---------|-------------| +| `validateSource` | `(source: string, flavor: LanguageFlavor = lfUnknown, options: ValidationOptions = newValidationOptions(), sourcePath: string = "")` | `ValidationResult` | Validate source code string. Auto-detects flavor if `lfUnknown`. | +| `validateFile` | `(filePath: string, flavor: LanguageFlavor = lfUnknown, options: ValidationOptions = newValidationOptions())` | `ValidationResult` | Validate a file from disk. Auto-detects flavor from file extension. | +| `validateSourceJson` | `(source: string, flavor: LanguageFlavor = lfUnknown)` | `string` | Validate and return pretty-printed JSON string. | +| `validateFileJson` | `(filePath: string, flavor: LanguageFlavor = lfUnknown)` | `string` | Validate file and return pretty-printed JSON string. | -### JSON Output +#### Inspection -Both `ValidationResult` and `ValidationError` have a `.toJson()` method returning a `JsonNode`. Use `.pretty()` for formatted output. +| Function | Signature | Returns | Description | +|----------|-----------|---------|-------------| +| `inspectSource` | `(source: string, flavor: LanguageFlavor = lfUnknown, options: ValidationOptions = newValidationOptions(), sourcePath: string = "")` | `JsonNode` | Validate and return structural diagnostics (functions, classes, imports). | +| `inspectFile` | `(filePath: string, flavor: LanguageFlavor = lfUnknown, options: ValidationOptions = newValidationOptions())` | `JsonNode` | Inspect a file and return structural diagnostics. | -## Error Codes +#### Reporting -| Code | Meaning | -|---|---| -| `E1000` | Unexpected/unmatched bracket | -| `E1001` | Unclosed bracket | -| `E1002` | Unmatched closing bracket | -| `E2000` | Unclosed string literal | -| `E2001` | Invalid string escape | -| `E2002` | Unclosed multi-line string | -| `E3000` | Unclosed comment | -| `E3001` | Invalid comment syntax | -| `E4000` | Invalid syntax | -| `E4001` | Reserved word misuse | -| `E5000` | Unclosed block/control structure | -| `E9999` | Internal error (never-crash fallback) | +| Function | Signature | Returns | Description | +|----------|-----------|---------|-------------| +| `reportSource` | `(source: string, flavor: LanguageFlavor = lfUnknown)` | `string` | Human-readable validation report for source code. | +| `reportFile` | `(filePath: string, flavor: LanguageFlavor = lfUnknown)` | `string` | Human-readable validation report for a file. | -## Supported Languages +#### Detection and Utility -| Language | File Extensions | Tokenizer Features | -|---|---|---| -| Nim | `.nim`, `.nims`, `.nimble` | Block comments `#[ ]#` with nesting, triple-quoted strings, raw strings, doc comments | -| Bash | `.sh`, `.bash` | Shebang detection, heredocs, command substitution, variable expansion, fi/done/esac matching | -| Python | `.py` | F-strings with nesting, triple-quoted strings, decorators, async/await, match statements | -| JavaScript | `.js`, `.jsx`, `.mjs`, `.cjs` | Template literals with nesting, regex literals, arrow functions, ESM imports | -| PHP | `.php` | `` | +| `tkInterpolation` | String interpolation (`${var}`, f-string `{expr}`) | +| `tkDirective` | Preprocessor / compiler directive (`#include`, shebang) | +| `tkTemplateTag` | Template language tag (`{{ }}`, `{% %}`) | +| `tkSpecial` | Language-specific special token | +| `tkError` | Malformed or unrecognizable character | +| `tkEndOfFile` | End of input marker | + +### Token + +``` +Token = object + kind: TokenKind + value: string + position: SourcePosition + range: SourceRange ``` -## Development +### SourcePosition + +``` +SourcePosition = object + line: int # 1-indexed line number + column: int # 1-indexed column number + offset: int # 0-indexed byte offset from start +``` + +### SourceRange + +``` +SourceRange = object + startPos: SourcePosition + endPos: SourcePosition +``` + +### ErrorSeverity (enum) + +| Value | String | Description | +|-------|--------|-------------| +| `esInfo` | `"info"` | Informational note | +| `esWarning` | `"warning"` | Non-fatal issue | +| `esError` | `"error"` | Validation failure | +| `esCritical` | `"critical"` | Internal framework failure | + +### ValidationError + +``` +ValidationError = object + severity: ErrorSeverity + message: string + code: string # Error code, e.g. "E0002" + position: SourcePosition + range: SourceRange + context: string # Surrounding source context (optional) + hint: string # Fix suggestion (optional) +``` + +### ValidationResult + +``` +ValidationResult = object + valid: bool + flavor: LanguageFlavor + detectionMethod: FlavorDetectionMethod + errors: seq[ValidationError] + warnings: int + infos: int + moduleInfo: ModuleInfo + durationMs: float + debugOutput: string # Empty unless compiled with -d:nimcheckDebug +``` + +### ValidationOptions + +``` +ValidationOptions = object + flavor: LanguageFlavor # lfUnknown = auto-detect (default) + debugMode: bool # Enable per-validation debug logging (default: false) + strictMode: bool # Warnings become errors (default: false) + maxErrors: int # Stop after N errors; 0 = unlimited (default: 0) + includeContext: bool # Include surrounding source lines in error output (default: true) + includeHints: bool # Include fix suggestions (default: true) +``` + +### Structural Types (populated by `analyzeTokens()`) + +``` +VariableInfo = object + name: string + kind: string # "var", "let", "const", "global", "local" + typeAnnotation: string + defaultValue: string + position: SourcePosition + mutable: bool + exported: bool + +FunctionInfo = object + name: string + kind: string # "function", "method", "proc", "func", "lambda" + parameters: seq[ParameterInfo] + returnType: string + isAsync: bool + isExported: bool + isGenerator: bool + visibility: string # "public", "private", "protected" + position: SourcePosition + docComment: string + +ParameterInfo = object + name: string + typeAnnotation: string + defaultValue: string + isOptional: bool + isVarargs: bool + +ClassInfo = object + name: string + kind: string # "class", "object", "struct", "interface" + baseTypes: seq[string] + fields: seq[VariableInfo] + methods: seq[FunctionInfo] + generics: seq[string] + visibility: string + position: SourcePosition + docComment: string + +ImportInfo = object + module: string + symbols: seq[string] # Specific imported symbols + isRelative: bool + alias: string + position: SourcePosition + +ModuleInfo = object + flavor: LanguageFlavor + detectionMethod: FlavorDetectionMethod + imports: seq[ImportInfo] + variables: seq[VariableInfo] + functions: seq[FunctionInfo] + classes: seq[ClassInfo] + linesOfCode: int + totalTokens: int + comments: int + debugInfo: JsonNode # Language-specific extra info +``` + +--- + +## JSON Output Structures + +Every type has a `.toJson()` method. The `ValidationResult.toJson()` aggregates all nested structures. + +### ValidationResult JSON + +```json +{ + "valid": true, + "flavor": "python", + "detectionMethod": "by_extension", + "errors": [ + { + "severity": "error", + "message": "Unclosed string literal starting with '\"'", + "code": "E0002", + "position": {"line": 2, "column": 13, "offset": 42}, + "range": { + "start": {"line": 2, "column": 13, "offset": 42}, + "end": {"line": 2, "column": 13, "offset": 42} + }, + "context": "", + "hint": "Add a closing quote before end of file" + } + ], + "warnings": 0, + "infos": 0, + "moduleInfo": { + "flavor": "python", + "detectionMethod": "by_extension", + "imports": [], + "variables": [], + "functions": [ + { + "name": "hello", + "kind": "def", + "parameters": [], + "returnType": "", + "isAsync": false, + "isExported": false, + "isGenerator": false, + "visibility": "", + "position": {"line": 1, "column": 5, "offset": 4}, + "docComment": "" + } + ], + "classes": [], + "linesOfCode": 3, + "totalTokens": 12, + "comments": 0, + "debugInfo": null + }, + "durationMs": 0.342, + "debugOutput": "" +} +``` + +### ValidationError JSON + +```json +{ + "severity": "error", + "message": "Unclosed string literal starting with '\"'", + "code": "E0002", + "position": {"line": 2, "column": 13, "offset": 42}, + "range": { + "start": {"line": 2, "column": 13, "offset": 42}, + "end": {"line": 2, "column": 13, "offset": 42} + }, + "context": "", + "hint": "Add a closing quote before end of file" +} +``` + +### ModuleInfo JSON + +```json +{ + "flavor": "nim", + "detectionMethod": "by_extension", + "imports": [ + { + "module": "strutils", + "symbols": [], + "isRelative": false, + "alias": "", + "position": {"line": 1, "column": 8, "offset": 7} + } + ], + "variables": [], + "functions": [ + { + "name": "add", + "kind": "proc", + "parameters": [], + "returnType": "", + "isAsync": false, + "isExported": false, + "isGenerator": false, + "visibility": "", + "position": {"line": 3, "column": 6, "offset": 28}, + "docComment": "" + } + ], + "classes": [ + { + "name": "MyType", + "kind": "type", + "baseTypes": [], + "fields": [], + "methods": [], + "generics": [], + "visibility": "", + "position": {"line": 5, "column": 6, "offset": 52}, + "docComment": "" + } + ], + "linesOfCode": 8, + "totalTokens": 45, + "comments": 2, + "debugInfo": null +} +``` + +### Human-Readable Report Format + +``` +=== Validation Report === + Flavor: python + Detection: by_extension + Valid: false + Errors: 1 + Warnings: 0 + Duration: 0.34ms + Lines of code: 3 + Tokens: 12 + + --- Findings --- + [E0002] (error) L2:13: Unclosed string literal starting with '"' + | x = " + | ^ + hint: Add a closing quote before end of file + + --- Summary --- + INVALID - source has issues that need attention. +``` + +### Error Formatter Output (single error) + +``` + [E0002] (error) L2:13: Unclosed string literal starting with '"' + | x = " + | ^ + hint: Add a closing quote before end of file +``` + +--- + +## Error Code Reference + +Error codes follow a prefixed numbering scheme. The prefix encodes the category and severity. + +### General Errors (E0xxx) + +| Code | Constant | Description | Triggered By | +|------|----------|-------------|-------------| +| `E0001` | `ErrUnexpectedToken` | Unexpected or unrecognizable character | Tokenizer encounter of non-ASCII, non-operator char | +| `E0002` | `ErrUnclosedString` | Unclosed string literal | Missing closing quote at EOF | +| `E0003` | `ErrUnclosedComment` | Unclosed block comment | Missing `*/`, `]#`, `#}` etc. at EOF | +| `E0004` | `ErrUnclosedBlock` | Unclosed block/control structure | Language-specific: unclosed `if`/`for`/`class`/etc. | +| `E0005` | `ErrMismatchedBracket` | Mismatched or unexpected closing bracket | `)` without `(`, `` without `
] [--flavor=]
+ nimcheck inspect [--file= | --source=] [--flavor=]
+ nimcheck detect [--file= | --source=]
+ nimcheck help
+
+Options:
+ --file= Source file to process
+ --source= Source code string to process
+ --flavor= Language flavor (auto-detect if omitted)
+ --debug Enable debug mode
+ --json Output as JSON (default for inspect)
+
+Supported flavors:
+ - bash
+ - html
+ - jinja
+ - jinja2
+ - js
+ - json
+ - jsonc
+ - nim
+ - nimble
+ - nims
+ - php
+ - phtml
+ - python
+ - py
+ - shell
+ - sh
+ - toml
+ - ts
+ - typescript
+ - yaml
+ - yml
+```
+
+### Exit Codes
+
+| Exit Code | Meaning |
+|-----------|---------|
+| 0 | Validation successful (or help/version requested) |
+| Non-zero (Nim default) | Runtime error |
+
+Note: The `validate` command always exits 0 on the Nim level; the `valid` field in the output indicates pass/fail. This is by design — Nimcheck never crashes.
+
+---
+
+## Building and Testing
### Prerequisites
- Nim >= 2.0.0
-- `make` (optional, for Makefile targets)
+- `make` (optional for Makefile targets)
-### Project structure
+### Build
-```
-src/
- nimcheck.nim Public API entry point and CLI (isMainModule)
- nimcheck/
- core/
- types.nim Core types: ValidationResult, ValidationError, LanguageFlavor
- tokenizerbase.nim Abstract tokenizer base class
- validatorbase.nim Abstract validator base class
- detector.nim Language auto-detection
- debug.nim Debug logging infrastructure
- tokenizers/
- nim_tokenizer.nim
- bash_tokenizer.nim
- python_tokenizer.nim
- javascript_tokenizer.nim
- php_tokenizer.nim
- html_tokenizer.nim
- jinja_tokenizer.nim
- languages/
- nim_validator.nim
- bash_validator.nim
- python_validator.nim
- javascript_validator.nim
- php_validator.nim
- html_validator.nim
- jinja_validator.nim
- config_validators.nim JSON, YAML, TOML validators
- reporting/
- errors.nim Error formatting and JSON output
-tests/
- test_all.nim Master test runner (imports all test modules)
- test_.nim Per-language tests
- test__exhaustive.nim Exhaustive edge-case tests per language
- test_config.nim JSON/YAML/TOML tests
- test_fuzz.nim Fuzz and boundary tests
- test_common.nim Shared test utilities
- test_mixed.nim Mixed-language file tests
- fixtures/ Test data files per language
+```bash
+# Using Makefile
+make build # Release build -> bin/nimcheck
+make build-debug # Debug build with -d:nimcheckDebug
+
+# Using nimble
+nimble build
+
+# Manual
+nim c src/nimcheck.nim
+nim c -d:nimcheckDebug src/nimcheck.nim # With debug output
```
-### Building with warnings
+### Test
+
+The test suite covers **250+ test cases** across all supported languages.
+
+```bash
+make test # Run all tests
+make test-nim # Nim-specific tests only
+make test-python # Python-specific tests only
+make test-bash # Bash-specific tests only
+make test-js # JavaScript/TypeScript tests
+make test-php # PHP tests
+make test-html # HTML/XML tests
+make test-jinja # Jinja2 tests
+make test-json # JSON tests
+make test-yaml # YAML tests
+make test-toml # TOML tests
+make test-mixed # Mixed-language file tests
+make debug # Build debug + run all tests
+```
+
+### Test Categories
+
+| Suite | Files | Coverage |
+|-------|-------|----------|
+| Per-language basic | `test_.nim` | Valid + invalid source for each language |
+| Per-language exhaustive | `test__exhaustive.nim` | Edge cases: unclosed strings, unclosed comments, mismatched brackets, binary data, Unicode |
+| Config formats | `test_config.nim` + `test__exhaustive.nim` | JSON/YAML/TOML structural validation |
+| Fuzz tests | `test_fuzz.nim` | Null bytes, binary data, deep nesting (100 levels), empty sources, concurrent validation |
+| Mixed files | `test_mixed.nim` | HTML+Jinja combined, other mixed-format files |
+| Common | `test_common.nim` | Shared test fixtures for valid and invalid code per language |
+
+### Lint
```bash
make lint
-# or manually:
+# or
nim c --hints:on --warnings:on src/nimcheck.nim
```
-### Adding a new language
+### Clean
-See the detailed checklist in [LESSONS_LEARNED.md](LESSONS_LEARNED.md) (Section 14).
+```bash
+make clean
+make rebuild # clean + build
+```
-## Architecture
+---
-The framework uses a layered architecture:
+## Project Structure
-1. **Tokenizers** -- convert raw source into a stream of tokens, tracking bracket depth, string context, and comments
-2. **Validators** -- analyze token streams for language-specific syntax rules
-3. **Detector** -- identifies language from file extension, shebang, or content heuristics
-4. **Reporting** -- formats errors as human-readable strings or structured JSON
+```
+.
+|-- Makefile
+|-- nimcheck.nimble
+|-- README.md
+|-- LESSONS_LEARNED.md
+|-- CONTRIBUTING.md
+|-- LICENSE
+|-- bin/ # Build output
+| `-- nimcheck # Compiled binary
+|-- src/
+| |-- nimcheck.nim # Public API entry point + CLI (isMainModule)
+| `-- nimcheck/
+| |-- core/
+| | |-- types.nim # All core types, toJson() methods, error code constants
+| | |-- tokenizerbase.nim # Abstract TokenizerBase with position tracking, bracket stack, error recording
+| | |-- validatorbase.nim # Abstract ValidatorBase with pipeline, factory, registry
+| | |-- detector.nim # Language auto-detection (extension, shebang, content scoring)
+| | |-- debug.nim # Debug logging (compiled only with -d:nimcheckDebug)
+| | `-- init_validators.nim
+| |-- tokenizers/
+| | |-- nim_tokenizer.nim # Nim: """ strings, r"..." raw strings, #[ ]# nested comments, ## doc comments
+| | |-- bash_tokenizer.nim # Bash: heredocs, $var/${var}, $(), backticks, (( ))
+| | |-- python_tokenizer.nim # Python: f-strings, """ strings, decorators, indentation tracking
+| | |-- javascript_tokenizer.nim # JS/TS: template literals, regex literals, arrow functions
+| | |-- php_tokenizer.nim # PHP: mode switching, $variables
+| | |-- html_tokenizer.nim # HTML/XML: tag balancing, void elements, attributes
+| | `-- jinja_tokenizer.nim # Jinja2: {{ }}, {% %}, {# #} block detection
+| |-- languages/
+| | |-- nim_validator.nim # Analyzes Nim: imports, procs, types
+| | |-- bash_validator.nim # Analyzes Bash: if/fi, for/done, case/esac matching
+| | |-- python_validator.nim # Analyzes Python: imports, def, class
+| | |-- javascript_validator.nim # Analyzes JS/TS: imports, functions, classes
+| | |-- php_validator.nim # Analyzes PHP: functions, classes, includes
+| | |-- html_validator.nim # Analyzes HTML/XML: tag balancing
+| | |-- jinja_validator.nim # Analyzes Jinja2: block tag matching (12 tag types)
+| | `-- config_validators.nim # JSON (via parseJson), YAML, TOML validators
+| `-- reporting/
+| `-- errors.nim # Human-readable and JSON error formatting
+`-- tests/
+ |-- test_all.nim # Master test runner (imports all)
+ |-- test_nim.nim # Nim basic tests
+ |-- test_nim_exhaustive.nim # Nim edge cases
+ |-- test_bash.nim
+ |-- test_bash_exhaustive.nim
+ |-- test_python.nim
+ |-- test_python_exhaustive.nim
+ |-- test_javascript.nim
+ |-- test_javascript_exhaustive.nim
+ |-- test_php.nim
+ |-- test_php_exhaustive.nim
+ |-- test_html.nim
+ |-- test_html_exhaustive.nim
+ |-- test_jinja.nim
+ |-- test_jinja_exhaustive.nim
+ |-- test_json_exhaustive.nim
+ |-- test_yaml_exhaustive.nim
+ |-- test_toml_exhaustive.nim
+ |-- test_config.nim # Config format tests
+ |-- test_mixed.nim # Mixed-language file tests
+ |-- test_fuzz.nim # Fuzz and boundary tests
+ |-- test_common.nim # Shared test fixtures
+ `-- fixtures/ # Per-language test data files
+ |-- nim/
+ |-- bash/
+ |-- python/
+ |-- javascript/
+ |-- php/
+ |-- html/
+ |-- jinja/
+ |-- json/
+ |-- yaml/
+ |-- toml/
+ `-- mixed/
+```
-Validators register themselves at import time via a side-effect factory pattern. The public API (`nimcheck.nim`) imports all validators and re-exports core types for consumer convenience.
-
-## Versioning
-
-Current version: **0.1.0**
+---
## Contributing
-Contributions are welcome. Please open an issue to discuss your idea before
-submitting a pull request. For detailed instructions on adding support for a new
-language, see [LESSONS_LEARNED.md](LESSONS_LEARNED.md) Section 16.
+Contributions are welcome. Open an issue to discuss your idea before submitting a pull request.
+
+### Adding a New Language
+
+See `LESSONS_LEARNED.md` for the complete checklist (Section 14). The minimum steps are:
+
+1. Create `src/nimcheck/tokenizers/_tokenizer.nim` inheriting from `TokenizerBase`
+2. Implement the `tokenize()` method with language-specific lexing
+3. Create `src/nimcheck/languages/_validator.nim` inheriting from `ValidatorBase`
+4. Implement `createTokenizer()`, `analyzeTokens()`, and `buildModuleInfo()`
+5. Add `registerValidator("flavor", constructor)` in the validator's `init()` proc
+6. Import the validator in `src/nimcheck.nim`
+7. Add extension mappings to `detector.nim`
+8. Add test fixtures and test files under `tests/`
+9. Update the test master runner `tests/test_all.nim`
+
+### Design Principles
+
+- **KISS**: Keep validators simple — tokenize, analyze brackets, report
+- **Never crash**: Every validation path in a try/except
+- **No external deps**: Pure Nim standard library
+- **DRY**: Common logic in `tokenizerbase.nim` and `validatorbase.nim`
+
+---
## License
-This project is licensed under the MIT License -- see the [LICENSE](LICENSE) file
-for details.
+MIT -- see [LICENSE](LICENSE) for details.
## Author
diff --git a/src/nimcheck.nim b/src/nimcheck.nim
index 0cd58a0..641b94d 100644
--- a/src/nimcheck.nim
+++ b/src/nimcheck.nim
@@ -315,6 +315,7 @@ when isMainModule:
of "validate", "val":
var filePath, source, flavor = ""
var debugMode = false
+ var jsonOut = false
var i = 1
while i < args.len:
let a = args[i]
@@ -322,6 +323,7 @@ when isMainModule:
elif a.startsWith("--source="): source = a[9..^1]
elif a.startsWith("--flavor="): flavor = a[9..^1]
elif a == "--debug": debugMode = true
+ elif a == "--json": jsonOut = true
i += 1
var opts = newValidationOptions()
@@ -333,20 +335,27 @@ when isMainModule:
except: discard
var r: ValidationResult
+ var src: string = ""
if filePath.len > 0:
r = validateFile(filePath, fl, opts)
+ if fileExists(filePath): src = readFile(filePath)
elif source.len > 0:
+ src = source
r = validateSource(source, fl, opts, "stdin")
else:
# Read from stdin
- let input = readAll(stdin).strip()
- r = validateSource(input, fl, opts, "stdin")
+ src = readAll(stdin).strip()
+ r = validateSource(src, fl, opts, "stdin")
- echo formatReport(r, r.moduleInfo.toJson().pretty())
+ if jsonOut:
+ echo r.toJson().pretty()
+ else:
+ echo formatReport(r, src)
of "inspect", "ins":
var filePath, source, flavor = ""
var debugMode = false
+ var jsonOut = false
var i = 1
while i < args.len:
let a = args[i]
@@ -354,6 +363,7 @@ when isMainModule:
elif a.startsWith("--source="): source = a[9..^1]
elif a.startsWith("--flavor="): flavor = a[9..^1]
elif a == "--debug": debugMode = true
+ elif a == "--json": jsonOut = true
i += 1
var opts = newValidationOptions()