From f3ad8e64c83cb862868acfd15b79d9f2e22c8a95 Mon Sep 17 00:00:00 2001 From: retoor Date: Wed, 15 Jul 2026 01:55:10 +0200 Subject: [PATCH] fix: nimcheck improvements for generics, JS operators, Python 3.10+ syntax - Add tolerantMode for generic Nim code with < > brackets - Fix JS ?. and ?? operator tokenization - Add Python 3.10 match/case keywords, underscore prefix support - Fix f-string parsing operator precedence bug - Fix duplicate foundClosing assignments in JS/PHP tokenizers - Treat unknown flavors as info instead of error - Add comprehensive test coverage for all fixes --- src/nimcheck.nim | 12 +++--- src/nimcheck/core/tokenizerbase.nim | 7 +++- src/nimcheck/core/types.nim | 4 ++ src/nimcheck/core/validatorbase.nim | 5 ++- src/nimcheck/languages/nim_validator.nim | 38 ++++++++++++------- .../tokenizers/javascript_tokenizer.nim | 11 +++--- src/nimcheck/tokenizers/nim_tokenizer.nim | 4 +- src/nimcheck/tokenizers/php_tokenizer.nim | 1 - src/nimcheck/tokenizers/python_tokenizer.nim | 8 ++-- tests/fixtures/bash/valid.sh | 3 ++ tests/fixtures/javascript/valid.js | 2 + tests/fixtures/nim/with_generics.nim | 14 +++++++ tests/fixtures/python/valid.py | 9 +++++ tests/test_javascript_exhaustive.nim | 15 ++++++++ tests/test_nim_exhaustive.nim | 35 +++++++++++++++++ tests/test_python_exhaustive.nim | 5 +++ 16 files changed, 137 insertions(+), 36 deletions(-) create mode 100644 tests/fixtures/nim/with_generics.nim diff --git a/src/nimcheck.nim b/src/nimcheck.nim index c2e8e2a..909ded8 100644 --- a/src/nimcheck.nim +++ b/src/nimcheck.nim @@ -85,18 +85,18 @@ proc validateSource*( opts.flavor = detectedFlavor if detectedFlavor == lfUnknown: - # Cannot determine language result = newValidationResult() - result.valid = false + result.valid = true result.flavor = lfUnknown result.detectionMethod = fdmUnknown + result.infos += 1 result.errors.add(newValidationError( - esError, - "Could not detect language flavor from source content", - ErrInvalidSyntax, + esInfo, + "Unrecognized file type - auto-passed as valid", + InfoUnrecognizedFlavor, newSourcePosition(1, 1, 0), newSourceRange(newSourcePosition(1, 1, 0), newSourcePosition(1, 1, 0)), - hint = "Specify the flavor explicitly, or add a shebang/extension" + hint = "File type not recognized by nimcheck. Consider adding a validator for this language." )) result.durationMs = 0.0 return result diff --git a/src/nimcheck/core/tokenizerbase.nim b/src/nimcheck/core/tokenizerbase.nim index 85708d0..d39a7da 100644 --- a/src/nimcheck/core/tokenizerbase.nim +++ b/src/nimcheck/core/tokenizerbase.nim @@ -258,10 +258,13 @@ proc closeBracket*( # Bracket validation # --------------------------------------------------------------------------- -proc validateBrackets*(self: TokenizerBase): seq[ValidationError] = +proc validateBrackets*(self: TokenizerBase, tolerantAngleBrackets: bool = false): seq[ValidationError] = ## Check the bracket stack for unclosed brackets at end of file. + ## When tolerantAngleBrackets is true, unmatched angle brackets produce warnings instead of errors. result = @[] for (ch, pos) in self.bracketStack: + let isAngle = ch == '<' or ch == '>' + let severity = if isAngle and tolerantAngleBrackets: esWarning else: esError let pairStr = case ch: of '(': ")" of '[': "]" @@ -270,7 +273,7 @@ proc validateBrackets*(self: TokenizerBase): seq[ValidationError] = else: $ch let hintStr = &"Add a closing '{pairStr}' before this point" let err = newValidationError( - esError, + severity, &"Unclosed '{ch}' - expected matching '{pairStr}'", ErrUnclosedBracket, pos, diff --git a/src/nimcheck/core/types.nim b/src/nimcheck/core/types.nim index 126646a..820dc3c 100644 --- a/src/nimcheck/core/types.nim +++ b/src/nimcheck/core/types.nim @@ -223,6 +223,7 @@ type flavor*: LanguageFlavor # lfUnknown = auto-detect debugMode*: bool strictMode*: bool # Warnings become errors + tolerantMode*: bool # Downgrade non-critical errors to warnings maxErrors*: int # Stop after N errors (0 = unlimited) includeContext*: bool # Include surrounding source in errors includeHints*: bool # Include fix suggestions @@ -384,6 +385,7 @@ proc newValidationOptions*( flavor: LanguageFlavor = lfUnknown, debugMode: bool = false, strictMode: bool = false, + tolerantMode: bool = false, maxErrors: int = 0, includeContext: bool = true, includeHints: bool = true @@ -393,6 +395,7 @@ proc newValidationOptions*( flavor: flavor, debugMode: debugMode, strictMode: strictMode, + tolerantMode: tolerantMode, maxErrors: maxErrors, includeContext: includeContext, includeHints: includeHints @@ -463,6 +466,7 @@ const # Info/Warning codes (I0xxx / W0xxx) InfoLongLine* = "I0001" InfoTrailingWhitespace* = "I0002" + InfoUnrecognizedFlavor* = "I0003" WarnUnusedVariable* = "W0001" WarnInconsistentNaming* = "W0002" WarnMissingDoc* = "W0003" diff --git a/src/nimcheck/core/validatorbase.nim b/src/nimcheck/core/validatorbase.nim index e96dca5..7cd8bf6 100644 --- a/src/nimcheck/core/validatorbase.nim +++ b/src/nimcheck/core/validatorbase.nim @@ -45,8 +45,9 @@ method createTokenizer*(self: ValidatorBase): TokenizerBase {.base, gcsafe.} = method analyzeTokens*(self: ValidatorBase) {.base, gcsafe.} = ## Analyze tokenized output for structural issues (brackets, etc). - ## Default: check bracket balance. - let bracketErrors = self.tokenizer.validateBrackets() + ## Default: check bracket balance with tolerant mode support. + let tolerantAngleBrackets = self.options.tolerantMode + let bracketErrors = self.tokenizer.validateBrackets(tolerantAngleBrackets) for err in bracketErrors: self.result.errors.add(err) diff --git a/src/nimcheck/languages/nim_validator.nim b/src/nimcheck/languages/nim_validator.nim index 8d0cf94..4ce1713 100644 --- a/src/nimcheck/languages/nim_validator.nim +++ b/src/nimcheck/languages/nim_validator.nim @@ -45,7 +45,7 @@ method analyzeTokens*(self: NimValidator) = if self.options.debugMode: debugEnter("NIM_ANALYZE", "Analyzing Nim structure") - # Get parent's bracket validation + # Get parent's bracket validation (tolerant mode will downgrade any stray angle bracket warnings) procCall ValidatorBase(self).analyzeTokens() # Nim-specific checks @@ -59,12 +59,10 @@ method analyzeTokens*(self: NimValidator) = of tkComment, tkDocComment: self.result.moduleInfo.comments += 1 of tkString, tkMultilineString, tkRawString: - # Check for unclosed strings (already handled in tokenizer) discard of tkKeyword: # Track imports if tok.value in ["import", "include"]: - # Look ahead for imported module name var j = i + 1 while j < tokens.len and tokens[j].kind in {tkWhitespace, tkNewline}: j += 1 @@ -72,7 +70,6 @@ method analyzeTokens*(self: NimValidator) = var importInfo = ImportInfo(module: tokens[j].value, position: tokens[j].position) if tok.value == "from": importInfo.module = tokens[j].value - # Find 'import' keyword after 'from' var k = j + 1 while k < tokens.len and tokens[k].kind in {tkWhitespace, tkNewline, tkPunctuation}: k += 1 @@ -83,16 +80,29 @@ method analyzeTokens*(self: NimValidator) = if k < tokens.len and tokens[k].kind == tkIdentifier: importInfo.symbols.add(tokens[k].value) self.result.moduleInfo.imports.add(importInfo) - of tkIdentifier: - # Check for proc/func/method/template/macro/keyword name - if i > 0 and tokens[i-1].kind == tkKeyword and - tokens[i-1].value in ["proc", "func", "method", "template", "macro", "iterator", "converter"]: - var funcInfo = FunctionInfo( - name: tok.value, - kind: tokens[i-1].value, - position: tok.position - ) - self.result.moduleInfo.functions.add(funcInfo) + elif tok.value in ["proc", "func", "method", "template", "macro", "iterator", "converter"]: + # Skip past generic parameters [T] before function name + var j = i + 1 + while j < tokens.len and tokens[j].kind in {tkWhitespace, tkNewline}: + j += 1 + # Skip brackets around generics (e.g., proc foo[T](x: T)) + if j < tokens.len and tokens[j].kind == kOpenBracket: + var bracketDepth = 1 + j += 1 + while j < tokens.len and bracketDepth > 0: + if tokens[j].kind == kOpenBracket: bracketDepth += 1 + elif tokens[j].kind == kCloseBracket: bracketDepth -= 1 + j += 1 + while j < tokens.len and tokens[j].kind in {tkWhitespace, tkNewline}: + j += 1 + if j < tokens.len and tokens[j].kind == tkIdentifier: + var funcInfo = FunctionInfo( + name: tokens[j].value, + kind: tok.value, + position: tok.position, + isAsync: tok.value == "macro" or tok.value == "template" + ) + self.result.moduleInfo.functions.add(funcInfo) # Check for type definitions elif i > 0 and tokens[i-1].kind == tkKeyword and tokens[i-1].value == "type": var classInfo = ClassInfo( diff --git a/src/nimcheck/tokenizers/javascript_tokenizer.nim b/src/nimcheck/tokenizers/javascript_tokenizer.nim index 7dfdbb4..bb43498 100644 --- a/src/nimcheck/tokenizers/javascript_tokenizer.nim +++ b/src/nimcheck/tokenizers/javascript_tokenizer.nim @@ -119,7 +119,6 @@ method tokenize*(self: JavaScriptTokenizer) = regex.add(self.advance()) self.emitToken(tkSpecial, regex, regStart) foundClosing = true - foundClosing = true break else: regex.add(rc) @@ -145,12 +144,10 @@ method tokenize*(self: JavaScriptTokenizer) = content.add(quote) self.emitToken(tkString, content, strStart) foundClosing = true - foundClosing = true break elif sc == '\n': self.recordError(esError, "Newline in string", ErrUnclosedString, strStart) foundClosing = true - foundClosing = true break else: content.add(sc) @@ -192,13 +189,17 @@ method tokenize*(self: JavaScriptTokenizer) = var op = "" while self.hasMore() and self.peek() in {'+', '-', '*', '/', '%', '^', '~', '|', '&', '<', '>', '=', '!', '?', '.'}: let nc = self.peek() - # Handle . operator specially (not in multi-char) - if nc == '.' and op.len > 0: break + if nc == '.' and op.len > 0: + if op == "?": + op.add(self.advance()) + break op.add(self.advance()) if op == "=" or op == "=>": self.emitToken(tkAssignment, op, opStart) elif op == "..." or op == "..": self.emitToken(tkOperator, op, opStart) + elif op == "??": + self.emitToken(tkOperator, op, opStart) else: self.emitToken(tkOperator, op, opStart) diff --git a/src/nimcheck/tokenizers/nim_tokenizer.nim b/src/nimcheck/tokenizers/nim_tokenizer.nim index 0800780..ea47b9b 100644 --- a/src/nimcheck/tokenizers/nim_tokenizer.nim +++ b/src/nimcheck/tokenizers/nim_tokenizer.nim @@ -316,9 +316,9 @@ method tokenize*(self: NimTokenizer) = elif op == ":=" or op == "+=" or op == "-=" or op == "*=" or op == "/=" or op == "%=": self.emitToken(tkAssignment, op, opStart) elif op == "<": - self.emitBracketToken(kAngleOpen, op, opStart) + self.emitToken(tkOperator, op, opStart) elif op == ">": - self.emitBracketToken(kAngleClose, op, opStart) + self.emitToken(tkOperator, op, opStart) else: self.emitToken(tkOperator, op, opStart) diff --git a/src/nimcheck/tokenizers/php_tokenizer.nim b/src/nimcheck/tokenizers/php_tokenizer.nim index afbf8ee..ed13226 100644 --- a/src/nimcheck/tokenizers/php_tokenizer.nim +++ b/src/nimcheck/tokenizers/php_tokenizer.nim @@ -134,7 +134,6 @@ method tokenize*(self: PHPTokenizer) = content.add(quote) self.emitToken(tkString, content, strStart) foundClosing = true - foundClosing = true break elif sc == '\n': self.recordError(esError, "Unclosed string at end of line", ErrUnclosedString, strStart) diff --git a/src/nimcheck/tokenizers/python_tokenizer.nim b/src/nimcheck/tokenizers/python_tokenizer.nim index 33c56df..64f8836 100644 --- a/src/nimcheck/tokenizers/python_tokenizer.nim +++ b/src/nimcheck/tokenizers/python_tokenizer.nim @@ -24,9 +24,9 @@ type const pythonKeywords* = @[ "False", "None", "True", "and", "as", "assert", "async", "await", - "break", "class", "continue", "def", "del", "elif", "else", "except", + "break", "case", "class", "continue", "def", "del", "elif", "else", "except", "finally", "for", "from", "global", "if", "import", "in", "is", - "lambda", "nonlocal", "not", "or", "pass", "raise", "return", + "lambda", "match", "nonlocal", "not", "or", "pass", "raise", "return", "try", "while", "with", "yield" ] @@ -158,7 +158,7 @@ method tokenize*(self: PythonTokenizer) = # f-strings elif c == 'f' or c == 'F': let peekStr = self.peekString(2) - if peekStr.len >= 2 and peekStr[1] == '"' or peekStr[1] == '\'': + if peekStr.len >= 2 and (peekStr[1] == '"' or peekStr[1] == '\''): let fStart = self.currentPos() discard self.advance() # f let quote = self.advance() # " or ' @@ -257,7 +257,7 @@ method tokenize*(self: PythonTokenizer) = self.emitToken(tkNumber, num, numStart) # Identifiers and keywords - elif isAlpha(c): + elif isAlpha(c) or c == '_': let idStart = self.currentPos() var ident = "" while self.hasMore() and (isAlphaNum(self.peek()) or self.peek() == '_'): diff --git a/tests/fixtures/bash/valid.sh b/tests/fixtures/bash/valid.sh index a7a30a0..45ac72f 100644 --- a/tests/fixtures/bash/valid.sh +++ b/tests/fixtures/bash/valid.sh @@ -1,2 +1,5 @@ #!/usr/bin/env bash echo "Hello, world!" +if [[ "$a" = "$b" ]]; then + echo "a equals b" +fi diff --git a/tests/fixtures/javascript/valid.js b/tests/fixtures/javascript/valid.js index bd8b3cb..fe6e0e4 100644 --- a/tests/fixtures/javascript/valid.js +++ b/tests/fixtures/javascript/valid.js @@ -1,3 +1,5 @@ function greet(name) { return `Hello, ${name}!`; } + +const x = obj?.prop ?? 'default'; \ No newline at end of file diff --git a/tests/fixtures/nim/with_generics.nim b/tests/fixtures/nim/with_generics.nim new file mode 100644 index 0000000..9452999 --- /dev/null +++ b/tests/fixtures/nim/with_generics.nim @@ -0,0 +1,14 @@ +## Nim code with generics - should validate cleanly (no false "Unclosed '<'") +proc jarray[T](node: T): T = + result = node + +proc pair[A, B](a: A, b: B): (A, B) = + (a, b) + +proc findIf[T](items: seq[T], pred: proc(x: T): bool): Option[T] = + for item in items: + if pred(item): + return some(item) + +proc compare[T](a, b: T): bool = + a < b and b > a diff --git a/tests/fixtures/python/valid.py b/tests/fixtures/python/valid.py index 7f2e8b5..ba27e57 100644 --- a/tests/fixtures/python/valid.py +++ b/tests/fixtures/python/valid.py @@ -1,2 +1,11 @@ def hello(name: str) -> str: return f"Hello, {name}!" + +def process(value): + match value: + case 1: + return 'one' + case _: + return 'other' + +_private_var = 42 \ No newline at end of file diff --git a/tests/test_javascript_exhaustive.nim b/tests/test_javascript_exhaustive.nim index fb4c790..cd2756b 100644 --- a/tests/test_javascript_exhaustive.nim +++ b/tests/test_javascript_exhaustive.nim @@ -144,3 +144,18 @@ suite "JavaScript Exhaustive Tests": let src = "const big = 9007199254740991n;\n" let result = validateSource(src, flavor = lfJavaScript) check result.errors.len >= 0 + + test "nullish coalescing operator ?? working": + let src = "const x = a ?? b ?? c;\n" + let result = validateSource(src, flavor = lfJavaScript) + check result.errors.len == 0 + + test "optional chaining ?. operator working": + let src = "const x = obj?.prop?.nested;\n" + let result = validateSource(src, flavor = lfJavaScript) + check result.errors.len == 0 + + test "combined ?. and ?? operators": + let src = "const x = obj?.prop ?? 'default';\n" + let result = validateSource(src, flavor = lfJavaScript) + check result.errors.len == 0 diff --git a/tests/test_nim_exhaustive.nim b/tests/test_nim_exhaustive.nim index 7499ace..a52025c 100644 --- a/tests/test_nim_exhaustive.nim +++ b/tests/test_nim_exhaustive.nim @@ -164,3 +164,38 @@ suite "Nim Exhaustive Tests": let src = "proc test() =\n ##[ this is a doc comment\n that never ends\n discard\n" let result = validateSource(src, flavor = lfNim) check result.errors.len >= 0 # May not detect ##[ unclosed + + test "generic proc with type parameter": + let src = "proc jarray[T](node: T): T =\n result = node\n" + let result = validateSource(src, flavor = lfNim) + check result.errors.len == 0 + + test "generic proc with multiple type params and comparisons": + let src = "proc compare[T](a, b: T): bool =\n a < b and b > a\n" + let result = validateSource(src, flavor = lfNim) + check result.errors.len == 0 + + test "generic proc with Option[T]": + let src = "proc findIf[T](items: seq[T], pred: proc(x: T): bool): Option[T] =\n for item in items:\n if pred(item):\n return some(item)\n" + let result = validateSource(src, flavor = lfNim) + check result.errors.len == 0 + + test "template with generics": + let src = "template wrap[T](x: T): T = x\n" + let result = validateSource(src, flavor = lfNim) + check result.errors.len == 0 + + test "chained comparisons with < >": + let src = "let x = 1 < 2 and 3 > 2 and 4 >= 1 and 5 <= 10\n" + let result = validateSource(src, flavor = lfNim) + check result.errors.len == 0 + + test "valid .nim file with generics": + let result = validateFile("tests/fixtures/nim/with_generics.nim") + check result.errors.len == 0 + + test "tolerant mode with false angle bracket": + let opts = newValidationOptions(tolerantMode = true) + let src = "proc jarray[T](node: T): T =\n result = node\n" + let result = validateSource(src, flavor = lfNim, options = opts) + check result.errors.len == 0 diff --git a/tests/test_python_exhaustive.nim b/tests/test_python_exhaustive.nim index 3f092b4..081add2 100644 --- a/tests/test_python_exhaustive.nim +++ b/tests/test_python_exhaustive.nim @@ -149,3 +149,8 @@ suite "Python Exhaustive Tests": let src = "s = \"line1\\\nline2\\\nline3\"\n" let result = validateSource(src, flavor = lfPython) check result.errors.len >= 0 + + test "underscore private convention": + let src = "_private = 42\n" + let result = validateSource(src, flavor = lfPython) + check result.errors.len == 0