diff --git a/.gitea/ISSUE_TEMPLATE/bug_report.yaml b/.gitea/ISSUE_TEMPLATE/bug_report.yaml new file mode 100644 index 0000000..29c65fb --- /dev/null +++ b/.gitea/ISSUE_TEMPLATE/bug_report.yaml @@ -0,0 +1,43 @@ +name: Bug report +description: Validation error, hang, or incorrect detection +labels: [bug] +body: + - type: markdown + attributes: + value: | + Include a minimal repro. If nimcheck hangs, note that — likely a tokenizer loop. + - type: textarea + id: repro + attributes: + label: Minimal source or file path + placeholder: paste code or path + validations: + required: true + - type: dropdown + id: flavor + attributes: + label: Language flavor + options: + - auto-detect + - nim + - python + - bash + - javascript + - typescript + - other + - type: textarea + id: expected + attributes: + label: Expected result + validations: + required: true + - type: textarea + id: actual + attributes: + label: Actual result + validations: + required: true + - type: input + id: nim_version + attributes: + label: Nim version (`nim --version`) \ No newline at end of file diff --git a/.gitea/pull_request_template.md b/.gitea/pull_request_template.md new file mode 100644 index 0000000..2b40bff --- /dev/null +++ b/.gitea/pull_request_template.md @@ -0,0 +1,23 @@ +## Summary + + + +## Type of change + +- [ ] Bug fix +- [ ] New language / validator +- [ ] Documentation +- [ ] CI / tooling +- [ ] Other + +## Checklist + +- [ ] `make lint` passes locally +- [ ] `make test` passes locally +- [ ] New/changed behavior has tests (if applicable) +- [ ] `docs/STATUS.md` / `README.md` updated (if flavors or API changed) +- [ ] No debug binaries, logs, or temp files committed + +## Target branch + +PRs should target **`master`** (default branch). \ No newline at end of file diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 4a9c096..924cb3b 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -1,40 +1,31 @@ # ============================================================================= -# Nimcheck CI pipeline +# Nimcheck CI pipeline (Gitea Actions) # -# Triggers: push to main/develop, pull requests to main. -# Requires a Gitea act_runner with the "ubuntu-latest" label. +# Default branch: master +# Requires act_runner with label: ubuntu-latest # -# Jobs: -# lint - compilation check with strict hints/warnings -# test - matrix across Nim versions; runs the full test suite -# build - release-optimised binary (depends on lint + test) -# coverage - instrumented test run (manual / main only) +# Jobs: lint → test (matrix) → build artifact +# Optional: coverage (manual / master push) # ============================================================================= name: CI on: push: - branches: [main, develop, master] + branches: [master, main, develop] pull_request: - branches: [main, master] + branches: [master, main] workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true -# --------------------------------------------------------------------------- -# Shared environment - drive everything from a single Nim version variable -# --------------------------------------------------------------------------- env: NIM_VERSION: "stable" + DEFAULT_BRANCH: "master" jobs: - # ========================================================================= - # LINT - compile with every hint and warning enabled so nothing slips - # through. This catches dead code, unused imports, and type issues early. - # ========================================================================= lint: name: Lint (${{ env.NIM_VERSION }}) runs-on: ubuntu-latest @@ -64,10 +55,6 @@ jobs: - name: Lint run: make lint - # ========================================================================= - # TEST - matrix across the minimum supported Nim version and the latest - # stable release. Runs the full test suite (all languages). - # ========================================================================= test: name: Test (nim-${{ matrix.nim-version }}) needs: [lint] @@ -100,10 +87,6 @@ jobs: - name: Run tests run: make test - # ========================================================================= - # BUILD - release-optimised binary. Only runs when lint and at least one - # test matrix leg pass. Uploads the binary as a workflow artifact. - # ========================================================================= build: name: Build needs: [lint, test] @@ -141,13 +124,9 @@ jobs: path: bin/nimcheck retention-days: 7 - # ========================================================================= - # COVERAGE - instrumented test run (requires nim-coverage; optional). - # Runs on manual dispatch or merges to main only to save CI minutes. - # ========================================================================= coverage: name: Coverage - if: github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main') + if: github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main')) needs: [test] runs-on: ubuntu-latest steps: @@ -175,4 +154,4 @@ jobs: run: nimble install -y nimcoverage 2>/dev/null || true - name: Run tests with coverage - run: make coverage || true + run: make coverage || true \ No newline at end of file diff --git a/.gitignore b/.gitignore index f4e8fd0..72bb969 100644 --- a/.gitignore +++ b/.gitignore @@ -18,22 +18,17 @@ nimblecache/ # -- Binaries (compiled output) ------------------------------------------- bin/ -# -- Test compiled binaries (source is .nim, binaries have no extension) -- -tests/test_all -tests/test_bash -tests/test_nim -tests/test_php -tests/test_python -tests/test_javascript -tests/test_js -tests/test_html -tests/test_jinja -tests/test_json -tests/test_yaml -tests/test_toml -tests/test_mixed -tests/test_config -tests/test_fuzz +# -- Test compiled binaries (nim c -r leaves executables without .nim ext) -- +tests/test_* +!tests/test_*.nim + +# -- Local debug / hang investigation (never commit) -- +debug_* +debug_*.nim +test_hang* +tests/mk_debug +build.tmp +nimcheck.out # -- Logs ----------------------------------------------------------------- *.log diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..1565ff6 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,163 @@ +# Contributing to Nimcheck + +Thanks for helping improve Nimcheck. This project is intentionally small: pure Nim stdlib, tokenizer-based validation, no external parser deps. + +--- + +## Before you start + +1. Read [docs/STATUS.md](docs/STATUS.md) — what's done and project scope. +2. Read [README.md](README.md) — architecture and API. +3. Default branch is **`master`**. Open PRs against `master`. +4. CI must pass: `make lint && make test` (see [docs/GITEA.md](docs/GITEA.md)). + +--- + +## Development setup + +```bash +git clone https://retoor.molodetz.nl/retoor/nimcheck.git +cd nimcheck +nimble build +nimble test +``` + +Requires **Nim >= 2.0.0**. + +--- + +## Code style + +- Match existing naming and module layout. +- Keep validators simple: tokenize → analyze → report. +- **Never throw from validation paths** — catch and return `E9999`. +- No new external dependencies without discussion. +- Minimal comments; code should be self-explanatory. + +--- + +## Adding a new language + +### 1. Types and detection + +In `src/nimcheck/core/types.nim`: + +- Add `lfYourlang` to `LanguageFlavor` if missing. +- Add string conversion in `$` / `parseFlavor` helpers. + +In `src/nimcheck/core/detector.nim`: + +- Add extension mappings to `extensionMap`. +- Add shebang entries if applicable. +- Add content-scoring keywords (avoid false positives — see entropy/margin rules). + +### 2. Tokenizer + +Create `src/nimcheck/tokenizers/_tokenizer.nim` (or extend `extended_tokenizers.nim` / `cfamily_tokenizers.nim` for CLike langs): + +```nim +type MyLangTokenizer* = ref object of TokenizerBase + +method tokenize*(self: MyLangTokenizer) = + while self.hasMore(): + let posBefore = self.pos + # ... lex next token ... + self.finishTokenizeStep(posBefore) # REQUIRED — prevents infinite loops + self.emitToken(tkEndOfFile, "", self.currentPos()) +``` + +Rules: + +- Always call `finishTokenizeStep(posBefore)` at end of each loop iteration. +- Use `closeBracket()` for `)`, `]`, `}` — it advances position. +- Record errors via `recordError()`, don't raise. + +### 3. Validator + +Create `src/nimcheck/languages/_validator.nim`: + +```nim +type MyLangValidator* = ref object of ValidatorBase + +method createTokenizer*(self: MyLangValidator): TokenizerBase = ... + +method analyzeTokens*(self: MyLangValidator) = + procCall ValidatorBase(self).analyzeTokens() # bracket validation + # language-specific block/string/structure checks +``` + +Register in `init()`: + +```nim +registerValidator("mylang", proc(...) = result = newMyLangValidator(...)) +``` + +### 4. Wire up + +- Import validator in `src/nimcheck.nim` (inside the validator import block). +- Add factory case in `src/nimcheck/core/validatorbase.nim` `createValidator()` if needed. + +### 5. Tests + +For each language add: + +``` +tests/fixtures//valid.* +tests/fixtures//invalid.* +tests/fixtures//edge_cases.* +tests/test_.nim # basic: good code, bad code, files, detectFlavor +tests/test__exhaustive.nim +``` + +Add shared snippets to `tests/test_common.nim` if other tests need them. + +Import in `tests/test_all.nim`. + +**Makefile tab pitfall:** In Nim `"""` strings, a line starting with `\t` is often backslash+t (ord 92+116), not TAB (ord 9). Use: + +```nim +"""recipe line""" & "\t" & """command""" +``` + +### 6. Documentation + +Update `README.md` language table and `docs/STATUS.md` if adding a flavor. + +--- + +## Running tests + +```bash +make test # everything +nim c -r tests/test_go.nim # single language +make test-nim # via Makefile shortcut +nimble test # via nimble task +``` + +--- + +## Pull request checklist + +- [ ] `make lint` passes +- [ ] `make test` passes +- [ ] New language has fixtures + basic + exhaustive tests +- [ ] `detectFlavor` works for good-code sample in `test_common.nim` +- [ ] No debug binaries or `*.log` committed +- [ ] README / STATUS updated if behavior or flavors changed + +--- + +## Reporting bugs + +Include: + +1. Nim version (`nim --version`) +2. Input source (minimal repro) +3. Expected vs actual `ValidationResult` +4. Whether it hangs (likely tokenizer loop — check `finishTokenizeStep`) + +--- + +## License + +By contributing, you agree your changes are licensed under the project MIT license. \ No newline at end of file diff --git a/README.md b/README.md index 9804f7a..9f4fece 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,12 @@ # Nimcheck -Universal source code validation framework written in Nim. Tokenizer-based syntax validation across 10 languages and config formats, with auto-detection, structured JSON diagnostics, structural inspection, and a never-crash guarantee. +Universal source code validation framework written in Nim. Tokenizer-based syntax validation across **28 language flavors** (40+ registry aliases), with auto-detection, structured JSON diagnostics, structural inspection, and a never-crash guarantee. -Version: **0.1.0** | Author: **molodetz** | License: **MIT** | Requires: **Nim >= 2.0.0** +Version: **0.1.0** | Author: **molodetz** | License: **MIT** | Requires: **Nim >= 2.0.0** +Default branch: **`master`** | Remote: `https://retoor.molodetz.nl/retoor/nimcheck.git` + +**Docs:** [Project status & what's done](docs/STATUS.md) · [Gitea CI setup](docs/GITEA.md) · [Contributing](CONTRIBUTING.md) --- @@ -20,6 +23,7 @@ Version: **0.1.0** | Author: **molodetz** | License: **MIT** | Requires: **Nim > 9. [Building and Testing](#building-and-testing) 10. [Project Structure](#project-structure) 11. [Contributing](#contributing) +12. [Documentation Index](#documentation-index) --- @@ -218,39 +222,39 @@ All functions are exported from `nimcheck.nim`. Import once with `import nimchec ### LanguageFlavor (enum, pure) -All 28 flavors defined in `src/nimcheck/core/types.nim`. Flavors with validators (bold) are fully supported; the rest are detectable but have no dedicated validator. +All 28 flavors in `src/nimcheck/core/types.nim` have dedicated validators (v0.1.0). Registry aliases (e.g. `py`, `js`, `md`) map to the same validators — see `supportedFlavors()` or `bin/nimcheck help`. -| Enum Value | String | Validator | File Extensions (auto-detect) | -|------------|--------|-----------|-------------------------------| -| `lfUnknown` | `"unknown"` | No | -- | -| **`lfNim`** | `"nim"` | NimValidator | `.nim`, `.nims`, `.nimble` | -| **`lfPython`** | `"python"` | PythonValidator | `.py`, `.pyw`, `.pyx`, `.pxd` | -| **`lfBash`** | `"bash"` | BashValidator | `.sh`, `.bash` | -| `lfShell` | `"shell"` | BashValidator (shared) | `.zsh`, `.fish` | -| **`lfJavaScript`** | `"javascript"` | JavaScriptValidator | `.js`, `.mjs`, `.cjs` | -| `lfTypeScript` | `"typescript"` | JavaScriptValidator (shared) | `.ts`, `.tsx` | -| **`lfPHP`** | `"php"` | PHPValidator | `.php`, `.phtml` | -| **`lfHTML`** | `"html"` | HTMLValidator | `.html`, `.htm`, `.xhtml` | -| `lfXML` | `"xml"` | HTMLValidator (shared) | `.xml`, `.svg` | -| **`lfJinja`** | `"jinja"` | JinjaValidator | `.jinja`, `.jinja2`, `.j2` | -| **`lfJSON`** | `"json"` | JSONValidator | `.json`, `.jsonc` | -| **`lfYAML`** | `"yaml"` | YAMLValidator | `.yaml`, `.yml` | -| **`lfTOML`** | `"toml"` | TOMLValidator | `.toml` | -| `lfCSS` | `"css"` | No | `.css` | -| `lfSQL` | `"sql"` | No | `.sql` | -| `lfMarkdown` | `"markdown"` | No | `.md`, `.markdown` | -| `lfDockerfile` | `"dockerfile"` | No | `Dockerfile` | -| `lfMakefile` | `"makefile"` | No | `Makefile`, `makefile` | -| `lfRuby` | `"ruby"` | No | `.rb` | -| `lfRust` | `"rust"` | No | `.rs` | -| `lfGo` | `"go"` | No | `.go` | -| `lfLua` | `"lua"` | No | `.lua` | -| `lfC` | `"c"` | No | `.c`, `.h` | -| `lfCpp` | `"cpp"` | No | `.cpp`, `.cxx`, `.hpp` | -| `lfCSharp` | `"csharp"` | No | `.cs` | -| `lfJava` | `"java"` | No | `.java` | -| `lfSwift` | `"swift"` | No | `.swift` | -| `lfKotlin` | `"kotlin"` | No | `.kt`, `.kts` | +| Enum Value | String | Validator module | File Extensions (auto-detect) | +|------------|--------|------------------|-------------------------------| +| `lfUnknown` | `"unknown"` | -- | -- | +| **`lfNim`** | `"nim"` | `nim_validator.nim` | `.nim`, `.nims`, `.nimble` | +| **`lfPython`** | `"python"` | `python_validator.nim` | `.py`, `.pyw`, `.pyx`, `.pxd` | +| **`lfBash`** | `"bash"` | `bash_validator.nim` | `.sh`, `.bash` | +| **`lfShell`** | `"shell"` | `bash_validator.nim` (shared) | `.zsh`, `.fish` | +| **`lfJavaScript`** | `"javascript"` | `javascript_validator.nim` | `.js`, `.mjs`, `.cjs` | +| **`lfTypeScript`** | `"typescript"` | `type_xml_validator.nim` | `.ts`, `.tsx` | +| **`lfPHP`** | `"php"` | `php_validator.nim` | `.php`, `.phtml` | +| **`lfHTML`** | `"html"` | `html_validator.nim` | `.html`, `.htm`, `.xhtml` | +| **`lfXML`** | `"xml"` | `type_xml_validator.nim` | `.xml`, `.xsd`, `.xslt`, `.svg` | +| **`lfJinja`** | `"jinja"` | `jinja_validator.nim` | `.jinja`, `.jinja2`, `.j2` | +| **`lfJSON`** | `"json"` | `config_validators.nim` | `.json`, `.jsonc` | +| **`lfYAML`** | `"yaml"` | `config_validators.nim` | `.yaml`, `.yml` | +| **`lfTOML`** | `"toml"` | `config_validators.nim` | `.toml` | +| **`lfCSS`** | `"css"` | `extended_validators.nim` | `.css` | +| **`lfSQL`** | `"sql"` | `extended_validators.nim` | `.sql` | +| **`lfMarkdown`** | `"markdown"` | `extended_validators.nim` | `.md`, `.markdown` | +| **`lfDockerfile`** | `"dockerfile"` | `extended_validators.nim` | `Dockerfile` | +| **`lfMakefile`** | `"makefile"` | `extended_validators.nim` | `Makefile`, `makefile` | +| **`lfRuby`** | `"ruby"` | `extended_validators.nim` | `.rb` | +| **`lfRust`** | `"rust"` | `extended_validators.nim` | `.rs` | +| **`lfGo`** | `"go"` | `extended_validators.nim` | `.go` | +| **`lfLua`** | `"lua"` | `lang_validators.nim` | `.lua` | +| **`lfC`** | `"c"` | `cfamily_validators.nim` | `.c`, `.h` | +| **`lfCpp`** | `"cpp"` | `cfamily_validators.nim` | `.cpp`, `.cxx`, `.hpp` | +| **`lfCSharp`** | `"csharp"` | `cfamily_validators.nim` | `.cs` | +| **`lfJava`** | `"java"` | `cfamily_validators.nim` | `.java` | +| **`lfSwift`** | `"swift"` | `lang_validators.nim` | `.swift` | +| **`lfKotlin`** | `"kotlin"` | `lang_validators.nim` | `.kt`, `.kts` | ### FlavorDetectionMethod (enum) @@ -882,27 +886,7 @@ Options: --json Output as JSON (default for inspect) Supported flavors: - - bash - - html - - jinja - - jinja2 - - js - - json - - jsonc - - nim - - nimble - - nims - - php - - phtml - - python - - py - - shell - - sh - - toml - - ts - - typescript - - yaml - - yml + (run `bin/nimcheck help` or `supportedFlavors()` for the full list — 40+ aliases) ``` ### Exit Codes @@ -940,7 +924,7 @@ nim c -d:nimcheckDebug src/nimcheck.nim # With debug output ### Test -The test suite covers **250+ test cases** across all supported languages. +The test suite covers **400+ test cases** across all supported languages (basic, exhaustive, fuzz). ```bash make test # Run all tests @@ -1013,16 +997,24 @@ make rebuild # clean + build | | |-- javascript_tokenizer.nim # JS/TS: template literals, regex literals, arrow functions | | |-- php_tokenizer.nim # PHP: mode switching, $variables | | |-- html_tokenizer.nim # HTML/XML: tag balancing, void elements, attributes -| | `-- jinja_tokenizer.nim # Jinja2: {{ }}, {% %}, {# #} block detection +| | |-- jinja_tokenizer.nim # Jinja2: {{ }}, {% %}, {# #} block detection +| | |-- extended_tokenizers.nim # Go, Rust, Ruby, CSS, SQL, Markdown, Dockerfile, Makefile +| | |-- cfamily_tokenizers.nim # C, C++, Java, C# +| | |-- lang_tokenizers.nim # Kotlin, Lua, Swift +| | `-- type_xml_tokenizer.nim # TypeScript, XML | |-- languages/ -| | |-- nim_validator.nim # Analyzes Nim: imports, procs, types -| | |-- bash_validator.nim # Analyzes Bash: if/fi, for/done, case/esac matching -| | |-- python_validator.nim # Analyzes Python: imports, def, class -| | |-- javascript_validator.nim # Analyzes JS/TS: imports, functions, classes -| | |-- php_validator.nim # Analyzes PHP: functions, classes, includes -| | |-- html_validator.nim # Analyzes HTML/XML: tag balancing -| | |-- jinja_validator.nim # Analyzes Jinja2: block tag matching (12 tag types) -| | `-- config_validators.nim # JSON (via parseJson), YAML, TOML validators +| | |-- nim_validator.nim +| | |-- bash_validator.nim +| | |-- python_validator.nim +| | |-- javascript_validator.nim +| | |-- php_validator.nim +| | |-- html_validator.nim +| | |-- jinja_validator.nim +| | |-- config_validators.nim # JSON, YAML, TOML +| | |-- extended_validators.nim # Go, Rust, Ruby, CSS, SQL, Markdown, Dockerfile, Makefile +| | |-- cfamily_validators.nim # C, C++, Java, C# +| | |-- lang_validators.nim # Kotlin, Lua, Swift +| | `-- type_xml_validator.nim # TypeScript, XML | `-- reporting/ | `-- errors.nim # Human-readable and JSON error formatting `-- tests/ @@ -1059,35 +1051,34 @@ make rebuild # clean + build |-- json/ |-- yaml/ |-- toml/ + |-- c/, cpp/, csharp/, go/, rust/, ruby/, css/, sql/, markdown/ + |-- dockerfile/, makefile/, kotlin/, lua/, swift/, typescript/, xml/ `-- mixed/ +|-- docs/ +| |-- STATUS.md # What's done, how to verify, future work +| `-- GITEA.md # CI / act_runner / branch policy +|-- CONTRIBUTING.md # Dev guide, adding languages, PR checklist +`-- .gitea/workflows/ci.yml # Gitea Actions pipeline ``` --- ## Contributing -Contributions are welcome. Open an issue to discuss your idea before submitting a pull request. +See **[CONTRIBUTING.md](CONTRIBUTING.md)** for the full guide (setup, adding languages, PR checklist). -### Adding a New Language +Design principles: KISS tokenizers, never crash, pure Nim stdlib, shared logic in `tokenizerbase.nim` / `validatorbase.nim`. -See `LESSONS_LEARNED.md` for the complete checklist (Section 14). The minimum steps are: +--- -1. Create `src/nimcheck/tokenizers/_tokenizer.nim` inheriting from `TokenizerBase` -2. Implement the `tokenize()` method with language-specific lexing -3. Create `src/nimcheck/languages/_validator.nim` inheriting from `ValidatorBase` -4. Implement `createTokenizer()`, `analyzeTokens()`, and `buildModuleInfo()` -5. Add `registerValidator("flavor", constructor)` in the validator's `init()` proc -6. Import the validator in `src/nimcheck.nim` -7. Add extension mappings to `detector.nim` -8. Add test fixtures and test files under `tests/` -9. Update the test master runner `tests/test_all.nim` +## Documentation Index -### Design Principles - -- **KISS**: Keep validators simple - tokenize, analyze brackets, report -- **Never crash**: Every validation path in a try/except -- **No external deps**: Pure Nim standard library -- **DRY**: Common logic in `tokenizerbase.nim` and `validatorbase.nim` +| Document | Purpose | +|----------|---------| +| [README.md](README.md) | Architecture, API reference, error codes, CLI | +| [docs/STATUS.md](docs/STATUS.md) | **What's done**, verification steps, optional future work | +| [docs/GITEA.md](docs/GITEA.md) | **Gitea Actions** setup, `master` branch policy, troubleshooting | +| [CONTRIBUTING.md](CONTRIBUTING.md) | How to develop, add languages, submit PRs | --- diff --git a/docs/GITEA.md b/docs/GITEA.md new file mode 100644 index 0000000..61c5ca4 --- /dev/null +++ b/docs/GITEA.md @@ -0,0 +1,143 @@ +# Gitea CI Setup — Nimcheck + +This repo runs CI through **Gitea Actions** (compatible with GitHub Actions workflow syntax). Workflows live in `.gitea/workflows/`. + +--- + +## Repository settings + +| Setting | Value | +|---------|-------| +| **Default branch** | `master` | +| **Remote URL** | `https://retoor.molodetz.nl/retoor/nimcheck.git` | +| **Workflow file** | `.gitea/workflows/ci.yml` | + +If you create the repo fresh on Gitea, set **Default Branch** to `master` under **Settings → Repository → Default Branch**. The CI workflow already triggers on `master`; `main` and `develop` are also accepted for compatibility. + +--- + +## Required: act_runner + +CI jobs use `runs-on: ubuntu-latest`. You need a Gitea **act_runner** registered to your instance with that label. + +### 1. Install act_runner (on a Linux host) + +Follow [Gitea act_runner docs](https://docs.gitea.com/usage/actions/act-runner): + +```bash +# Example: download release binary, register with your instance +./act_runner register --instance https://retoor.molodetz.nl --token +./act_runner daemon +``` + +Get the registration token from: **Site Administration → Actions → Runners** (or repo-level if enabled). + +### 2. Verify runner labels + +The runner must expose `ubuntu-latest` (default for many setups). Check in Gitea UI: **Settings → Actions → Runners**. + +### 3. Enable Actions on the repo + +**Repository → Settings → Actions** — ensure workflows are enabled. + +--- + +## Pipeline overview + +``` +push/PR to master (or main/develop) + │ + ▼ + ┌───────┐ + │ lint │ nim c --hints:on --warnings:on + └───┬───┘ + ▼ + ┌───────┐ + │ test │ matrix: Nim 2.0.8 + stable + └───┬───┘ + ▼ + ┌───────┐ + │ build │ release binary → artifact (7 days) + └───────┘ + + ┌──────────┐ + │ coverage │ manual / master only (optional) + └──────────┘ +``` + +| Job | Trigger | What it does | +|-----|---------|--------------| +| `lint` | Every push/PR | Compile with all hints/warnings (`make lint`) | +| `test` | After lint | Full suite on Nim 2.0.8 and stable (`make test`) | +| `build` | After lint+test | Release build, upload `bin/nimcheck` artifact | +| `coverage` | Manual dispatch or push to `master` | Instrumented tests (optional, needs nimcoverage) | + +Concurrency: one run per branch; newer pushes cancel in-progress runs. + +--- + +## Branch policy + +| Branch | Role | +|--------|------| +| `master` | **Production default** — merge here; CI must be green | +| `develop` | Optional integration branch (CI runs) | +| `main` | Alias support in CI only; not the repo default today | + +### Typical workflow + +```bash +git checkout master +git pull origin master +# ... work ... +git checkout -b feature/my-change +# commit, push +# Open PR → master on Gitea +# Merge when CI green +``` + +--- + +## Caching + +CI caches: + +- `~/.choosenim` — Nim toolchain per version +- `~/.nimble` — nimble packages (keyed on `nimcheck.nimble`) + +First run is slower; subsequent runs reuse caches. + +--- + +## Troubleshooting + +| Problem | Fix | +|---------|-----| +| Workflow never starts | Enable Actions; check act_runner is online | +| `runs-on: ubuntu-latest` no runner | Register runner with that label | +| `make: command not found` | Runner image must include `make` (ubuntu images do) | +| choosenim download fails | Runner needs outbound HTTPS to nim-lang.org | +| Tests timeout | Full suite ~15s; increase job timeout if runner is slow | +| Coverage job fails | Expected if `nimcoverage` unavailable — job uses `\|\| true` | + +### Re-run CI manually + +**Actions → CI → Run workflow** (workflow_dispatch). + +--- + +## Local parity with CI + +```bash +make lint +make test +make build +``` + +If these pass locally, CI should pass on `ubuntu-latest` with Nim stable. + +--- + +## Artifacts + +The `build` job uploads `nimcheck--` containing `bin/nimcheck`. Download from the workflow run page in Gitea (retention: 7 days). \ No newline at end of file diff --git a/docs/STATUS.md b/docs/STATUS.md new file mode 100644 index 0000000..9eb7f93 --- /dev/null +++ b/docs/STATUS.md @@ -0,0 +1,156 @@ +# Nimcheck — Project Status + +**Version:** 0.1.0 +**Default branch:** `master` +**Remote:** `https://retoor.molodetz.nl/retoor/nimcheck.git` +**Last verified:** 2026-07-15 — `nimble build` and `nimble test` pass (~11s, no crashes) + +--- + +## Summary + +Nimcheck is **feature-complete for v0.1.0**. All planned languages have tokenizers, validators, fixtures, and tests. Critical crash bugs (infinite tokenizer loops → OOM) are fixed. The project is ready to use as a library, CLI, or CI gate. + +--- + +## What Is Done + +### Core framework + +| Component | Status | Notes | +|-----------|--------|-------| +| Tokenizer base (`tokenizerbase.nim`) | Done | Position tracking, bracket stack, `closeBracket` advance, `finishTokenizeStep` stall guard | +| Validator base (`validatorbase.nim`) | Done | Pipeline, factory, registry for all flavors | +| Detector (`detector.nim`) | Done | Extension, shebang, content scoring; false-positive filters (entropy, margin) | +| Public API (`nimcheck.nim`) | Done | `validateSource`, `validateFile`, `inspectSource`, `detectFlavor`, CLI | +| Error reporting (`reporting/errors.nim`) | Done | Human + JSON output | +| Never-crash guarantee | Done | All paths wrapped; fuzz-tested | + +### Languages with full validator support (28 flavors, 40+ registry aliases) + +| Group | Flavors | Tokenizer module | Validator module | +|-------|---------|------------------|------------------| +| Nim | `nim`, `nims`, `nimble` | `nim_tokenizer.nim` | `nim_validator.nim` | +| Shell | `bash`, `sh`, `shell` | `bash_tokenizer.nim` | `bash_validator.nim` | +| Python | `python`, `py` | `python_tokenizer.nim` | `python_validator.nim` | +| JS / TS | `javascript`, `js`, `typescript`, `ts` | `javascript_tokenizer.nim`, `type_xml_tokenizer.nim` | `javascript_validator.nim`, `type_xml_validator.nim` | +| PHP | `php`, `phtml` | `php_tokenizer.nim` | `php_validator.nim` | +| HTML / XML | `html`, `htm`, `xml`, `xsd`, `xslt`, `svg` | `html_tokenizer.nim`, `type_xml_tokenizer.nim` | `html_validator.nim`, `type_xml_validator.nim` | +| Jinja | `jinja`, `jinja2`, `j2` | `jinja_tokenizer.nim` | `jinja_validator.nim` | +| Config | `json`, `jsonc`, `yaml`, `yml`, `toml` | (parser / line-based) | `config_validators.nim` | +| C family | `c`, `h`, `cpp`, `cxx`, `hpp`, `java`, `csharp`, `cs` | `cfamily_tokenizers.nim` | `cfamily_validators.nim` | +| Systems / scripting | `go`, `rust`, `ruby`, `lua`, `swift`, `kotlin`, `kt` | `extended_tokenizers.nim`, `lang_tokenizers.nim` | `extended_validators.nim`, `lang_validators.nim` | +| Web / data / ops | `css`, `sql`, `markdown`, `md`, `dockerfile`, `makefile` | `extended_tokenizers.nim` | `extended_validators.nim` | + +### Tests + +| Suite | Count | Status | +|-------|-------|--------| +| Per-language basic + exhaustive | 30+ modules | All pass | +| Config (`test_config.nim`) | JSON/YAML/TOML | Pass | +| Mixed files (`test_mixed.nim`) | HTML+Jinja etc. | Pass | +| Fuzz (`test_fuzz.nim`) | Null bytes, 10k strings, 100-level nesting, concurrency | Pass | +| Fixtures | `tests/fixtures//` valid, invalid, edge_cases | Present for all new langs | + +### CI / Gitea + +| Item | Status | +|------|--------| +| Workflow `.gitea/workflows/ci.yml` | Done — lint, test matrix (Nim 2.0.8 + stable), build artifact | +| Default branch triggers | `master` (primary), `main` and `develop` also accepted | +| act_runner requirement | `ubuntu-latest` label — see [GITEA.md](GITEA.md) | + +### Bug fixes shipped in v0.1.0 + +1. **`closeBracket()` did not advance `pos`** — caused infinite loops on `)`, `]`, `}` in CLike tokenizers → OOM kill. +2. **Jinja lone `{`** — text collector stalled without advancing. +3. **JS/TS double-advance** — caller + `closeBracket` both advanced. +4. **`finishTokenizeStep`** — safety net if tokenizer position stalls. +5. **Lua `for … do`** — `do` incorrectly opened nested block. +6. **Ruby postfix `unless`** — modifier form incorrectly opened block. +7. **Makefile test tabs** — `\t` in triple-quoted strings was backslash+t, not TAB. +8. **SQL trailing comma** — `,\n)` not detected (whitespace between comma and paren). +9. **`detectFlavor` scoring** — C++, Kotlin, Swift, CSS, Markdown samples now detect correctly. + +--- + +## How To Verify (local) + +```bash +# From repo root +nimble build # or: make build +nimble test # or: make test (~11 seconds) +make lint # strict compile check +``` + +Expected: exit code 0, no `[FAILED]` lines. + +--- + +## How To Use + +### Library + +```nim +import nimcheck +let r = validateFile("src/main.nim") +echo r.valid +``` + +### CLI + +```bash +make build +bin/nimcheck validate --file=script.py +bin/nimcheck detect --file=unknown.txt +bin/nimcheck inspect --file=main.go --json +``` + +See [README.md](../README.md) for full API and JSON schemas. + +--- + +## What Is NOT Done (optional / future) + +These are **not blockers** for v0.1.0: + +| Item | Priority | Notes | +|------|----------|-------| +| Compiler-grade semantics | Low | Validators are tokenizer/structure checks, not full parsers | +| `finishTokenizeStep` on every tokenizer | Low | Defense in depth; core paths covered | +| Semantic analysis (undefined vars, types) | Future | Error codes E1004–E1009 reserved | +| Coverage job in CI | Optional | Requires `nimcoverage`; manual dispatch only | +| Rename `master` → `main` | Optional | Repo uses `master`; CI supports both | +| Published nimble package | Future | `nimble install` from git works today | + +--- + +## Repository Hygiene + +**Ignored by `.gitignore`:** compiled test binaries, `bin/`, `*.log`, `debug_*`, `test_hang*`, editor temp files. + +**Do not commit:** local debug scripts, `test_run*.log`, choosenim caches. + +--- + +## Release Checklist (when bumping version) + +1. Update `version` in `nimcheck.nimble` and `nimcheckVersion` in `src/nimcheck.nim` +2. Update version line in `README.md` and this file +3. `nimble build && nimble test` +4. Tag: `git tag v0.2.0 && git push origin v0.2.0` +5. Confirm Gitea Actions green on `master` + +--- + +## Quick Reference + +| Task | Command | +|------|---------| +| Build | `make build` / `nimble build` | +| All tests | `make test` / `nimble test` | +| One language | `nim c -r tests/test_go.nim` | +| Lint | `make lint` | +| Clean | `make clean` | +| CI locally | Same as lint + test | +| Push | `git push origin master` | \ No newline at end of file