fix: nimcheck improvements for generics, JS operators, Python 3.10+ syntax

- Add tolerantMode for generic Nim code with < > brackets
- Fix JS ?. and ?? operator tokenization
- Add Python 3.10 match/case keywords, underscore prefix support
- Fix f-string parsing operator precedence bug
- Fix duplicate foundClosing assignments in JS/PHP tokenizers
- Treat unknown flavors as info instead of error
- Add comprehensive test coverage for all fixes

src/nimcheck.nim

@@ -85,18 +85,18 @@ proc validateSource*(
       opts.flavor = detectedFlavor
 
     if detectedFlavor == lfUnknown:
-      # Cannot determine language
       result = newValidationResult()
-      result.valid = false
+      result.valid = true
       result.flavor = lfUnknown
       result.detectionMethod = fdmUnknown
+      result.infos += 1
       result.errors.add(newValidationError(
-        esError,
+        esInfo,
-        "Could not detect language flavor from source content",
+        "Unrecognized file type - auto-passed as valid",
-        ErrInvalidSyntax,
+        InfoUnrecognizedFlavor,
         newSourcePosition(1, 1, 0),
         newSourceRange(newSourcePosition(1, 1, 0), newSourcePosition(1, 1, 0)),
-        hint = "Specify the flavor explicitly, or add a shebang/extension"
+        hint = "File type not recognized by nimcheck. Consider adding a validator for this language."
       ))
       result.durationMs = 0.0
       return result

src/nimcheck/core/tokenizerbase.nim

@@ -258,10 +258,13 @@ proc closeBracket*(
 # Bracket validation
 # ---------------------------------------------------------------------------
 
-proc validateBrackets*(self: TokenizerBase): seq[ValidationError] =
+proc validateBrackets*(self: TokenizerBase, tolerantAngleBrackets: bool = false): seq[ValidationError] =
   ## Check the bracket stack for unclosed brackets at end of file.
+  ## When tolerantAngleBrackets is true, unmatched angle brackets produce warnings instead of errors.
   result = @[]
   for (ch, pos) in self.bracketStack:
+    let isAngle = ch == '<' or ch == '>'
+    let severity = if isAngle and tolerantAngleBrackets: esWarning else: esError
     let pairStr = case ch:
       of '(': ")"
       of '[': "]"
@@ -270,7 +273,7 @@ proc validateBrackets*(self: TokenizerBase): seq[ValidationError] =
       else: $ch
     let hintStr = &"Add a closing '{pairStr}' before this point"
     let err = newValidationError(
-      esError,
+      severity,
       &"Unclosed '{ch}' - expected matching '{pairStr}'",
       ErrUnclosedBracket,
       pos,

src/nimcheck/core/types.nim

@@ -223,6 +223,7 @@ type
     flavor*: LanguageFlavor           # lfUnknown = auto-detect
     debugMode*: bool
     strictMode*: bool                 # Warnings become errors
+    tolerantMode*: bool               # Downgrade non-critical errors to warnings
     maxErrors*: int                   # Stop after N errors (0 = unlimited)
     includeContext*: bool             # Include surrounding source in errors
     includeHints*: bool               # Include fix suggestions
@@ -384,6 +385,7 @@ proc newValidationOptions*(
   flavor: LanguageFlavor = lfUnknown,
   debugMode: bool = false,
   strictMode: bool = false,
+  tolerantMode: bool = false,
   maxErrors: int = 0,
   includeContext: bool = true,
   includeHints: bool = true
@@ -393,6 +395,7 @@ proc newValidationOptions*(
     flavor: flavor,
     debugMode: debugMode,
     strictMode: strictMode,
+    tolerantMode: tolerantMode,
     maxErrors: maxErrors,
     includeContext: includeContext,
     includeHints: includeHints
@@ -463,6 +466,7 @@ const
   # Info/Warning codes (I0xxx / W0xxx)
   InfoLongLine*          = "I0001"
   InfoTrailingWhitespace* = "I0002"
+  InfoUnrecognizedFlavor* = "I0003"
   WarnUnusedVariable*    = "W0001"
   WarnInconsistentNaming* = "W0002"
   WarnMissingDoc*        = "W0003"

src/nimcheck/core/validatorbase.nim

@@ -45,8 +45,9 @@ method createTokenizer*(self: ValidatorBase): TokenizerBase {.base, gcsafe.} =
 
 method analyzeTokens*(self: ValidatorBase) {.base, gcsafe.} =
   ## Analyze tokenized output for structural issues (brackets, etc).
-  ## Default: check bracket balance.
+  ## Default: check bracket balance with tolerant mode support.
-  let bracketErrors = self.tokenizer.validateBrackets()
+  let tolerantAngleBrackets = self.options.tolerantMode
+  let bracketErrors = self.tokenizer.validateBrackets(tolerantAngleBrackets)
   for err in bracketErrors:
     self.result.errors.add(err)
 

src/nimcheck/languages/nim_validator.nim

@@ -45,7 +45,7 @@ method analyzeTokens*(self: NimValidator) =
     if self.options.debugMode:
       debugEnter("NIM_ANALYZE", "Analyzing Nim structure")
 
-  # Get parent's bracket validation
+  # Get parent's bracket validation (tolerant mode will downgrade any stray angle bracket warnings)
   procCall ValidatorBase(self).analyzeTokens()
 
   # Nim-specific checks
@@ -59,12 +59,10 @@ method analyzeTokens*(self: NimValidator) =
     of tkComment, tkDocComment:
       self.result.moduleInfo.comments += 1
     of tkString, tkMultilineString, tkRawString:
-      # Check for unclosed strings (already handled in tokenizer)
       discard
     of tkKeyword:
       # Track imports
       if tok.value in ["import", "include"]:
-        # Look ahead for imported module name
         var j = i + 1
         while j < tokens.len and tokens[j].kind in {tkWhitespace, tkNewline}:
           j += 1
@@ -72,7 +70,6 @@ method analyzeTokens*(self: NimValidator) =
           var importInfo = ImportInfo(module: tokens[j].value, position: tokens[j].position)
           if tok.value == "from":
             importInfo.module = tokens[j].value
-            # Find 'import' keyword after 'from'
             var k = j + 1
             while k < tokens.len and tokens[k].kind in {tkWhitespace, tkNewline, tkPunctuation}:
               k += 1
@@ -83,16 +80,29 @@ method analyzeTokens*(self: NimValidator) =
               if k < tokens.len and tokens[k].kind == tkIdentifier:
                 importInfo.symbols.add(tokens[k].value)
           self.result.moduleInfo.imports.add(importInfo)
-    of tkIdentifier:
+      elif tok.value in ["proc", "func", "method", "template", "macro", "iterator", "converter"]:
-      # Check for proc/func/method/template/macro/keyword name
+        # Skip past generic parameters [T] before function name
-      if i > 0 and tokens[i-1].kind == tkKeyword and
+        var j = i + 1
-         tokens[i-1].value in ["proc", "func", "method", "template", "macro", "iterator", "converter"]:
+        while j < tokens.len and tokens[j].kind in {tkWhitespace, tkNewline}:
-        var funcInfo = FunctionInfo(
+          j += 1
-          name: tok.value,
+        # Skip brackets around generics (e.g., proc foo[T](x: T))
-          kind: tokens[i-1].value,
+        if j < tokens.len and tokens[j].kind == kOpenBracket:
-          position: tok.position
+          var bracketDepth = 1
-        )
+          j += 1
-        self.result.moduleInfo.functions.add(funcInfo)
+          while j < tokens.len and bracketDepth > 0:
+            if tokens[j].kind == kOpenBracket: bracketDepth += 1
+            elif tokens[j].kind == kCloseBracket: bracketDepth -= 1
+            j += 1
+          while j < tokens.len and tokens[j].kind in {tkWhitespace, tkNewline}:
+            j += 1
+        if j < tokens.len and tokens[j].kind == tkIdentifier:
+          var funcInfo = FunctionInfo(
+            name: tokens[j].value,
+            kind: tok.value,
+            position: tok.position,
+            isAsync: tok.value == "macro" or tok.value == "template"
+          )
+          self.result.moduleInfo.functions.add(funcInfo)
       # Check for type definitions
       elif i > 0 and tokens[i-1].kind == tkKeyword and tokens[i-1].value == "type":
         var classInfo = ClassInfo(

src/nimcheck/tokenizers/javascript_tokenizer.nim

@@ -119,7 +119,6 @@ method tokenize*(self: JavaScriptTokenizer) =
               regex.add(self.advance())
             self.emitToken(tkSpecial, regex, regStart)
             foundClosing = true
-            foundClosing = true
             break
           else:
             regex.add(rc)
@@ -145,12 +144,10 @@ method tokenize*(self: JavaScriptTokenizer) =
           content.add(quote)
           self.emitToken(tkString, content, strStart)
           foundClosing = true
-          foundClosing = true
           break
         elif sc == '\n':
           self.recordError(esError, "Newline in string", ErrUnclosedString, strStart)
           foundClosing = true
-          foundClosing = true
           break
         else:
           content.add(sc)
@@ -192,13 +189,17 @@ method tokenize*(self: JavaScriptTokenizer) =
       var op = ""
       while self.hasMore() and self.peek() in {'+', '-', '*', '/', '%', '^', '~', '|', '&', '<', '>', '=', '!', '?', '.'}:
         let nc = self.peek()
-        # Handle . operator specially (not in multi-char)
+        if nc == '.' and op.len > 0:
-        if nc == '.' and op.len > 0: break
+          if op == "?":
+            op.add(self.advance())
+          break
         op.add(self.advance())
       if op == "=" or op == "=>":
         self.emitToken(tkAssignment, op, opStart)
       elif op == "..." or op == "..":
         self.emitToken(tkOperator, op, opStart)
+      elif op == "??":
+        self.emitToken(tkOperator, op, opStart)
       else:
         self.emitToken(tkOperator, op, opStart)
 

src/nimcheck/tokenizers/nim_tokenizer.nim

@@ -316,9 +316,9 @@ method tokenize*(self: NimTokenizer) =
       elif op == ":=" or op == "+=" or op == "-=" or op == "*=" or op == "/=" or op == "%=":
         self.emitToken(tkAssignment, op, opStart)
       elif op == "<":
-        self.emitBracketToken(kAngleOpen, op, opStart)
+        self.emitToken(tkOperator, op, opStart)
       elif op == ">":
-        self.emitBracketToken(kAngleClose, op, opStart)
+        self.emitToken(tkOperator, op, opStart)
       else:
         self.emitToken(tkOperator, op, opStart)
 

src/nimcheck/tokenizers/php_tokenizer.nim

@@ -134,7 +134,6 @@ method tokenize*(self: PHPTokenizer) =
           content.add(quote)
           self.emitToken(tkString, content, strStart)
           foundClosing = true
-          foundClosing = true
           break
         elif sc == '\n':
           self.recordError(esError, "Unclosed string at end of line", ErrUnclosedString, strStart)

src/nimcheck/tokenizers/python_tokenizer.nim

@@ -24,9 +24,9 @@ type
 const
   pythonKeywords* = @[
     "False", "None", "True", "and", "as", "assert", "async", "await",
-    "break", "class", "continue", "def", "del", "elif", "else", "except",
+    "break", "case", "class", "continue", "def", "del", "elif", "else", "except",
     "finally", "for", "from", "global", "if", "import", "in", "is",
-    "lambda", "nonlocal", "not", "or", "pass", "raise", "return",
+    "lambda", "match", "nonlocal", "not", "or", "pass", "raise", "return",
     "try", "while", "with", "yield"
   ]
 
@@ -158,7 +158,7 @@ method tokenize*(self: PythonTokenizer) =
     # f-strings
     elif c == 'f' or c == 'F':
       let peekStr = self.peekString(2)
-      if peekStr.len >= 2 and peekStr[1] == '"' or peekStr[1] == '\'':
+      if peekStr.len >= 2 and (peekStr[1] == '"' or peekStr[1] == '\''):
         let fStart = self.currentPos()
         discard self.advance()  # f
         let quote = self.advance()  # " or '
@@ -257,7 +257,7 @@ method tokenize*(self: PythonTokenizer) =
       self.emitToken(tkNumber, num, numStart)
 
     # Identifiers and keywords
-    elif isAlpha(c):
+    elif isAlpha(c) or c == '_':
       let idStart = self.currentPos()
       var ident = ""
       while self.hasMore() and (isAlphaNum(self.peek()) or self.peek() == '_'):

tests/fixtures/bash/valid.sh

@@ -1,2 +1,5 @@
 #!/usr/bin/env bash
 echo "Hello, world!"
+if [[ "$a" = "$b" ]]; then
+  echo "a equals b"
+fi

tests/fixtures/javascript/valid.js

@@ -1,3 +1,5 @@
 function greet(name) {
   return `Hello, ${name}!`;
 }
+
+const x = obj?.prop ?? 'default';

tests/fixtures/nim/with_generics.nim

@@ -0,0 +1,14 @@
+## Nim code with generics - should validate cleanly (no false "Unclosed '<'")
+proc jarray[T](node: T): T =
+  result = node
+
+proc pair[A, B](a: A, b: B): (A, B) =
+  (a, b)
+
+proc findIf[T](items: seq[T], pred: proc(x: T): bool): Option[T] =
+  for item in items:
+    if pred(item):
+      return some(item)
+
+proc compare[T](a, b: T): bool =
+  a < b and b > a

tests/fixtures/python/valid.py

@@ -1,2 +1,11 @@
 def hello(name: str) -> str:
     return f"Hello, {name}!"
+
+def process(value):
+    match value:
+        case 1:
+            return 'one'
+        case _:
+            return 'other'
+
+_private_var = 42

tests/test_javascript_exhaustive.nim

@@ -144,3 +144,18 @@ suite "JavaScript Exhaustive Tests":
     let src = "const big = 9007199254740991n;\n"
     let result = validateSource(src, flavor = lfJavaScript)
     check result.errors.len >= 0
+
+  test "nullish coalescing operator ?? working":
+    let src = "const x = a ?? b ?? c;\n"
+    let result = validateSource(src, flavor = lfJavaScript)
+    check result.errors.len == 0
+
+  test "optional chaining ?. operator working":
+    let src = "const x = obj?.prop?.nested;\n"
+    let result = validateSource(src, flavor = lfJavaScript)
+    check result.errors.len == 0
+
+  test "combined ?. and ?? operators":
+    let src = "const x = obj?.prop ?? 'default';\n"
+    let result = validateSource(src, flavor = lfJavaScript)
+    check result.errors.len == 0

tests/test_nim_exhaustive.nim

@@ -164,3 +164,38 @@ suite "Nim Exhaustive Tests":
     let src = "proc test() =\n  ##[ this is a doc comment\n  that never ends\n  discard\n"
     let result = validateSource(src, flavor = lfNim)
     check result.errors.len >= 0  # May not detect ##[ unclosed
+
+  test "generic proc with type parameter":
+    let src = "proc jarray[T](node: T): T =\n  result = node\n"
+    let result = validateSource(src, flavor = lfNim)
+    check result.errors.len == 0
+
+  test "generic proc with multiple type params and comparisons":
+    let src = "proc compare[T](a, b: T): bool =\n  a < b and b > a\n"
+    let result = validateSource(src, flavor = lfNim)
+    check result.errors.len == 0
+
+  test "generic proc with Option[T]":
+    let src = "proc findIf[T](items: seq[T], pred: proc(x: T): bool): Option[T] =\n  for item in items:\n    if pred(item):\n      return some(item)\n"
+    let result = validateSource(src, flavor = lfNim)
+    check result.errors.len == 0
+
+  test "template with generics":
+    let src = "template wrap[T](x: T): T = x\n"
+    let result = validateSource(src, flavor = lfNim)
+    check result.errors.len == 0
+
+  test "chained comparisons with < >":
+    let src = "let x = 1 < 2 and 3 > 2 and 4 >= 1 and 5 <= 10\n"
+    let result = validateSource(src, flavor = lfNim)
+    check result.errors.len == 0
+
+  test "valid .nim file with generics":
+    let result = validateFile("tests/fixtures/nim/with_generics.nim")
+    check result.errors.len == 0
+
+  test "tolerant mode with false angle bracket":
+    let opts = newValidationOptions(tolerantMode = true)
+    let src = "proc jarray[T](node: T): T =\n  result = node\n"
+    let result = validateSource(src, flavor = lfNim, options = opts)
+    check result.errors.len == 0

tests/test_python_exhaustive.nim

@@ -149,3 +149,8 @@ suite "Python Exhaustive Tests":
     let src = "s = \"line1\\\nline2\\\nline3\"\n"
     let result = validateSource(src, flavor = lfPython)
     check result.errors.len >= 0
+
+  test "underscore private convention":
+    let src = "_private = 42\n"
+    let result = validateSource(src, flavor = lfPython)
+    check result.errors.len == 0