fix: nimcheck improvements for generics, JS operators, Python 3.10+ syntax

Browse Source 
- Add tolerantMode for generic Nim code with < > brackets
- Fix JS ?. and ?? operator tokenization
- Add Python 3.10 match/case keywords, underscore prefix support
- Fix f-string parsing operator precedence bug
- Fix duplicate foundClosing assignments in JS/PHP tokenizers
- Treat unknown flavors as info instead of error
- Add comprehensive test coverage for all fixes
This commit is contained in:
retoor 2026-07-15 01:55:10 +02:00
parent 9f50fd7e37
commit f3ad8e64c8
16 changed files with 137 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
src/nimcheck.nim
View File

@ -85,18 +85,18 @@ proc validateSource*(
opts.flavor = detectedFlavor
if detectedFlavor == lfUnknown:
# Cannot determine language
result = newValidationResult()
result.valid = false
result.valid = true
result.flavor = lfUnknown
result.detectionMethod = fdmUnknown
result.infos += 1
result.errors.add(newValidationError(
esError,
"Could not detect language flavor from source content",
ErrInvalidSyntax,
esInfo,
"Unrecognized file type - auto-passed as valid",
InfoUnrecognizedFlavor,
newSourcePosition(1, 1, 0),
newSourceRange(newSourcePosition(1, 1, 0), newSourcePosition(1, 1, 0)),
hint = "Specify the flavor explicitly, or add a shebang/extension"
hint = "File type not recognized by nimcheck. Consider adding a validator for this language."
))
result.durationMs = 0.0
return result

7
src/nimcheck/core/tokenizerbase.nim
View File

@ -258,10 +258,13 @@ proc closeBracket*(
# Bracket validation
# ---------------------------------------------------------------------------
proc validateBrackets*(self: TokenizerBase): seq[ValidationError] =
proc validateBrackets*(self: TokenizerBase, tolerantAngleBrackets: bool = false): seq[ValidationError] =
## Check the bracket stack for unclosed brackets at end of file.
## When tolerantAngleBrackets is true, unmatched angle brackets produce warnings instead of errors.
result = @[]
for (ch, pos) in self.bracketStack:
let isAngle = ch == '<' or ch == '>'
let severity = if isAngle and tolerantAngleBrackets: esWarning else: esError
let pairStr = case ch:
of '(': ")"
of '[': "]"
@ -270,7 +273,7 @@ proc validateBrackets*(self: TokenizerBase): seq[ValidationError] =
else: $ch
let hintStr = &"Add a closing '{pairStr}' before this point"
let err = newValidationError(
esError,
severity,
&"Unclosed '{ch}' - expected matching '{pairStr}'",
ErrUnclosedBracket,
pos,

4
src/nimcheck/core/types.nim
View File

@ -223,6 +223,7 @@ type
flavor*: LanguageFlavor # lfUnknown = auto-detect
debugMode*: bool
strictMode*: bool # Warnings become errors
tolerantMode*: bool # Downgrade non-critical errors to warnings
maxErrors*: int # Stop after N errors (0 = unlimited)
includeContext*: bool # Include surrounding source in errors
includeHints*: bool # Include fix suggestions
@ -384,6 +385,7 @@ proc newValidationOptions*(
flavor: LanguageFlavor = lfUnknown,
debugMode: bool = false,
strictMode: bool = false,
tolerantMode: bool = false,
maxErrors: int = 0,
includeContext: bool = true,
includeHints: bool = true
@ -393,6 +395,7 @@ proc newValidationOptions*(
flavor: flavor,
debugMode: debugMode,
strictMode: strictMode,
tolerantMode: tolerantMode,
maxErrors: maxErrors,
includeContext: includeContext,
includeHints: includeHints
@ -463,6 +466,7 @@ const
# Info/Warning codes (I0xxx / W0xxx)
InfoLongLine* = "I0001"
InfoTrailingWhitespace* = "I0002"
InfoUnrecognizedFlavor* = "I0003"
WarnUnusedVariable* = "W0001"
WarnInconsistentNaming* = "W0002"
WarnMissingDoc* = "W0003"

5
src/nimcheck/core/validatorbase.nim
View File

@ -45,8 +45,9 @@ method createTokenizer*(self: ValidatorBase): TokenizerBase {.base, gcsafe.} =
method analyzeTokens*(self: ValidatorBase) {.base, gcsafe.} =
## Analyze tokenized output for structural issues (brackets, etc).
## Default: check bracket balance.
let bracketErrors = self.tokenizer.validateBrackets()
## Default: check bracket balance with tolerant mode support.
let tolerantAngleBrackets = self.options.tolerantMode
let bracketErrors = self.tokenizer.validateBrackets(tolerantAngleBrackets)
for err in bracketErrors:
self.result.errors.add(err)

38
src/nimcheck/languages/nim_validator.nim
View File

@ -45,7 +45,7 @@ method analyzeTokens*(self: NimValidator) =
if self.options.debugMode:
debugEnter("NIM_ANALYZE", "Analyzing Nim structure")
# Get parent's bracket validation
# Get parent's bracket validation (tolerant mode will downgrade any stray angle bracket warnings)
procCall ValidatorBase(self).analyzeTokens()
# Nim-specific checks
@ -59,12 +59,10 @@ method analyzeTokens*(self: NimValidator) =
of tkComment, tkDocComment:
self.result.moduleInfo.comments += 1
of tkString, tkMultilineString, tkRawString:
# Check for unclosed strings (already handled in tokenizer)
discard
of tkKeyword:
# Track imports
if tok.value in ["import", "include"]:
# Look ahead for imported module name
var j = i + 1
while j < tokens.len and tokens[j].kind in {tkWhitespace, tkNewline}:
j += 1
@ -72,7 +70,6 @@ method analyzeTokens*(self: NimValidator) =
var importInfo = ImportInfo(module: tokens[j].value, position: tokens[j].position)
if tok.value == "from":
importInfo.module = tokens[j].value
# Find 'import' keyword after 'from'
var k = j + 1
while k < tokens.len and tokens[k].kind in {tkWhitespace, tkNewline, tkPunctuation}:
k += 1
@ -83,16 +80,29 @@ method analyzeTokens*(self: NimValidator) =
if k < tokens.len and tokens[k].kind == tkIdentifier:
importInfo.symbols.add(tokens[k].value)
self.result.moduleInfo.imports.add(importInfo)
of tkIdentifier:
# Check for proc/func/method/template/macro/keyword name
if i > 0 and tokens[i-1].kind == tkKeyword and
tokens[i-1].value in ["proc", "func", "method", "template", "macro", "iterator", "converter"]:
var funcInfo = FunctionInfo(
name: tok.value,
kind: tokens[i-1].value,
position: tok.position
)
self.result.moduleInfo.functions.add(funcInfo)
elif tok.value in ["proc", "func", "method", "template", "macro", "iterator", "converter"]:
# Skip past generic parameters [T] before function name
var j = i + 1
while j < tokens.len and tokens[j].kind in {tkWhitespace, tkNewline}:
j += 1
# Skip brackets around generics (e.g., proc foo[T](x: T))
if j < tokens.len and tokens[j].kind == kOpenBracket:
var bracketDepth = 1
j += 1
while j < tokens.len and bracketDepth > 0:
if tokens[j].kind == kOpenBracket: bracketDepth += 1
elif tokens[j].kind == kCloseBracket: bracketDepth -= 1
j += 1
while j < tokens.len and tokens[j].kind in {tkWhitespace, tkNewline}:
j += 1
if j < tokens.len and tokens[j].kind == tkIdentifier:
var funcInfo = FunctionInfo(
name: tokens[j].value,
kind: tok.value,
position: tok.position,
isAsync: tok.value == "macro" or tok.value == "template"
)
self.result.moduleInfo.functions.add(funcInfo)
# Check for type definitions
elif i > 0 and tokens[i-1].kind == tkKeyword and tokens[i-1].value == "type":
var classInfo = ClassInfo(

11
src/nimcheck/tokenizers/javascript_tokenizer.nim
View File

@ -119,7 +119,6 @@ method tokenize*(self: JavaScriptTokenizer) =
regex.add(self.advance())
self.emitToken(tkSpecial, regex, regStart)
foundClosing = true
foundClosing = true
break
else:
regex.add(rc)
@ -145,12 +144,10 @@ method tokenize*(self: JavaScriptTokenizer) =
content.add(quote)
self.emitToken(tkString, content, strStart)
foundClosing = true
foundClosing = true
break
elif sc == '\n':
self.recordError(esError, "Newline in string", ErrUnclosedString, strStart)
foundClosing = true
foundClosing = true
break
else:
content.add(sc)
@ -192,13 +189,17 @@ method tokenize*(self: JavaScriptTokenizer) =
var op = ""
while self.hasMore() and self.peek() in {'+', '-', '*', '/', '%', '^', '~', '|', '&', '<', '>', '=', '!', '?', '.'}:
let nc = self.peek()
# Handle . operator specially (not in multi-char)
if nc == '.' and op.len > 0: break
if nc == '.' and op.len > 0:
if op == "?":
op.add(self.advance())
break
op.add(self.advance())
if op == "=" or op == "=>":
self.emitToken(tkAssignment, op, opStart)
elif op == "..." or op == "..":
self.emitToken(tkOperator, op, opStart)
elif op == "??":
self.emitToken(tkOperator, op, opStart)
else:
self.emitToken(tkOperator, op, opStart)

4
src/nimcheck/tokenizers/nim_tokenizer.nim
View File

@ -316,9 +316,9 @@ method tokenize*(self: NimTokenizer) =
elif op == ":=" or op == "+=" or op == "-=" or op == "*=" or op == "/=" or op == "%=":
self.emitToken(tkAssignment, op, opStart)
elif op == "<":
self.emitBracketToken(kAngleOpen, op, opStart)
self.emitToken(tkOperator, op, opStart)
elif op == ">":
self.emitBracketToken(kAngleClose, op, opStart)
self.emitToken(tkOperator, op, opStart)
else:
self.emitToken(tkOperator, op, opStart)

1
src/nimcheck/tokenizers/php_tokenizer.nim
View File

@ -134,7 +134,6 @@ method tokenize*(self: PHPTokenizer) =
content.add(quote)
self.emitToken(tkString, content, strStart)
foundClosing = true
foundClosing = true
break
elif sc == '\n':
self.recordError(esError, "Unclosed string at end of line", ErrUnclosedString, strStart)

8
src/nimcheck/tokenizers/python_tokenizer.nim
View File

@ -24,9 +24,9 @@ type
const
pythonKeywords* = @[
"False", "None", "True", "and", "as", "assert", "async", "await",
"break", "class", "continue", "def", "del", "elif", "else", "except",
"break", "case", "class", "continue", "def", "del", "elif", "else", "except",
"finally", "for", "from", "global", "if", "import", "in", "is",
"lambda", "nonlocal", "not", "or", "pass", "raise", "return",
"lambda", "match", "nonlocal", "not", "or", "pass", "raise", "return",
"try", "while", "with", "yield"
]
@ -158,7 +158,7 @@ method tokenize*(self: PythonTokenizer) =
# f-strings
elif c == 'f' or c == 'F':
let peekStr = self.peekString(2)
if peekStr.len >= 2 and peekStr[1] == '"' or peekStr[1] == '\'':
if peekStr.len >= 2 and (peekStr[1] == '"' or peekStr[1] == '\''):
let fStart = self.currentPos()
discard self.advance() # f
let quote = self.advance() # " or '
@ -257,7 +257,7 @@ method tokenize*(self: PythonTokenizer) =
self.emitToken(tkNumber, num, numStart)
# Identifiers and keywords
elif isAlpha(c):
elif isAlpha(c) or c == '_':
let idStart = self.currentPos()
var ident = ""
while self.hasMore() and (isAlphaNum(self.peek()) or self.peek() == '_'):

3
tests/fixtures/bash/valid.sh vendored
View File

@ -1,2 +1,5 @@
#!/usr/bin/env bash
echo "Hello, world!"
if [[ "$a" = "$b" ]]; then
echo "a equals b"
fi

2
tests/fixtures/javascript/valid.js vendored
View File

@ -1,3 +1,5 @@
function greet(name) {
return `Hello, ${name}!`;
}
const x = obj?.prop ?? 'default';

14
tests/fixtures/nim/with_generics.nim vendored Normal file
View File

@ -0,0 +1,14 @@
## Nim code with generics - should validate cleanly (no false "Unclosed '<'")
proc jarray[T](node: T): T =
result = node
proc pair[A, B](a: A, b: B): (A, B) =
(a, b)
proc findIf[T](items: seq[T], pred: proc(x: T): bool): Option[T] =
for item in items:
if pred(item):
return some(item)
proc compare[T](a, b: T): bool =
a < b and b > a

9
tests/fixtures/python/valid.py vendored
View File

@ -1,2 +1,11 @@
def hello(name: str) -> str:
return f"Hello, {name}!"
def process(value):
match value:
case 1:
return 'one'
case _:
return 'other'
_private_var = 42

15
tests/test_javascript_exhaustive.nim
View File

@ -144,3 +144,18 @@ suite "JavaScript Exhaustive Tests":
let src = "const big = 9007199254740991n;\n"
let result = validateSource(src, flavor = lfJavaScript)
check result.errors.len >= 0
test "nullish coalescing operator ?? working":
let src = "const x = a ?? b ?? c;\n"
let result = validateSource(src, flavor = lfJavaScript)
check result.errors.len == 0
test "optional chaining ?. operator working":
let src = "const x = obj?.prop?.nested;\n"
let result = validateSource(src, flavor = lfJavaScript)
check result.errors.len == 0
test "combined ?. and ?? operators":
let src = "const x = obj?.prop ?? 'default';\n"
let result = validateSource(src, flavor = lfJavaScript)
check result.errors.len == 0

35
tests/test_nim_exhaustive.nim
View File

@ -164,3 +164,38 @@ suite "Nim Exhaustive Tests":
let src = "proc test() =\n ##[ this is a doc comment\n that never ends\n discard\n"
let result = validateSource(src, flavor = lfNim)
check result.errors.len >= 0 # May not detect ##[ unclosed
test "generic proc with type parameter":
let src = "proc jarray[T](node: T): T =\n result = node\n"
let result = validateSource(src, flavor = lfNim)
check result.errors.len == 0
test "generic proc with multiple type params and comparisons":
let src = "proc compare[T](a, b: T): bool =\n a < b and b > a\n"
let result = validateSource(src, flavor = lfNim)
check result.errors.len == 0
test "generic proc with Option[T]":
let src = "proc findIf[T](items: seq[T], pred: proc(x: T): bool): Option[T] =\n for item in items:\n if pred(item):\n return some(item)\n"
let result = validateSource(src, flavor = lfNim)
check result.errors.len == 0
test "template with generics":
let src = "template wrap[T](x: T): T = x\n"
let result = validateSource(src, flavor = lfNim)
check result.errors.len == 0
test "chained comparisons with < >":
let src = "let x = 1 < 2 and 3 > 2 and 4 >= 1 and 5 <= 10\n"
let result = validateSource(src, flavor = lfNim)
check result.errors.len == 0
test "valid .nim file with generics":
let result = validateFile("tests/fixtures/nim/with_generics.nim")
check result.errors.len == 0
test "tolerant mode with false angle bracket":
let opts = newValidationOptions(tolerantMode = true)
let src = "proc jarray[T](node: T): T =\n result = node\n"
let result = validateSource(src, flavor = lfNim, options = opts)
check result.errors.len == 0

5
tests/test_python_exhaustive.nim
View File

@ -149,3 +149,8 @@ suite "Python Exhaustive Tests":
let src = "s = \"line1\\\nline2\\\nline3\"\n"
let result = validateSource(src, flavor = lfPython)
check result.errors.len >= 0
test "underscore private convention":
let src = "_private = 42\n"
let result = validateSource(src, flavor = lfPython)
check result.errors.len == 0