## Python tokenizer for Nimcheck.
##
## Tokenizes Python source code including:
## - Single/double quoted strings
## - Triple-quoted strings (""" ''')
## - f-strings with interpolation
## - Line continuation with backslash
## - Indentation tracking
import std/[strformat, sets, json, sequtils, unicode]
{.warning[UnusedImport]:off.}
import ../core/types, ../core/tokenizerbase
type
PythonTokenizer* = ref object of TokenizerBase
## Tokenizer for Python source code.
indentStack*: seq[int] # Track indentation levels
parenDepth*: int # Track parens for implicit continuation
# ---------------------------------------------------------------------------
# Python keywords
# ---------------------------------------------------------------------------
const
pythonKeywords* = @[
"False", "None", "True", "and", "as", "assert", "async", "await",
"break", "case", "class", "continue", "def", "del", "elif", "else", "except",
"finally", "for", "from", "global", "if", "import", "in", "is",
"lambda", "match", "nonlocal", "not", "or", "pass", "raise", "return",
"try", "while", "with", "yield"
]
pythonKeywordSet* = pythonKeywords.toHashSet()
# ---------------------------------------------------------------------------
# Construction
# ---------------------------------------------------------------------------
proc newPythonTokenizer*(source: string, options: ValidationOptions = newValidationOptions()): PythonTokenizer =
## Create a new Python tokenizer.
result = PythonTokenizer(
source: source,
sourceLen: source.len,
options: options,
tokens: @[],
pos: 0,
line: 1,
column: 1,
bracketStack: @[],
errors: @[],
eof: false
)
result.indentStack = @[0]
result.parenDepth = 0
# ---------------------------------------------------------------------------
# Indentation helpers
# ---------------------------------------------------------------------------
proc getIndentLevel*(line: string): int =
## Count leading spaces.
for c in line:
if c == ' ': result += 1
elif c == '\t': result += 8 # Standard tab = 8
else: break
# ---------------------------------------------------------------------------
# Tokenization
# ---------------------------------------------------------------------------
method tokenize*(self: PythonTokenizer) =
## Tokenize Python source code.
when defined(nimcheckDebug):
if self.options.debugMode:
debugEnter("PYTHON_TOKENIZER", "Starting Python tokenization")
var currentLine: string = ""
var atLineStart = true
while self.hasMore():
let startPos = self.currentPos()
let c = self.peek()
# Newline handling with indentation tracking
if c == '\n':
if atLineStart and currentLine.len == 0:
# Empty line, just emit newline
discard self.advance()
self.emitToken(tkNewline, "\n", startPos)
else:
# End of line - parse indentation of next line
discard self.advance()
self.emitToken(tkNewline, "\n", startPos)
atLineStart = true
currentLine = ""
# Read next line for indentation tracking (peek ahead)
var nextLineStart = self.pos
var indentCount = 0
while nextLineStart < self.sourceLen and self.source[nextLineStart] == ' ':
indentCount += 1
nextLineStart += 1
# Skip blank/comment lines
if nextLineStart < self.sourceLen and self.source[nextLineStart] != '\n' and self.source[nextLineStart] != '#':
if self.parenDepth == 0:
if indentCount > self.indentStack[^1]:
self.indentStack.add(indentCount)
self.emitToken(tkSpecial, "<indent>", startPos)
elif indentCount < self.indentStack[^1]:
while self.indentStack.len > 1 and indentCount < self.indentStack[^1]:
discard self.indentStack.pop()
self.emitToken(tkSpecial, "<dedent>", startPos)
if indentCount != self.indentStack[^1]:
self.recordError(esError, "Indentation mismatch", ErrUnexpectedIndent, startPos)
continue
atLineStart = false
# Whitespace (non-newline)
if c == ' ' or c == '\t' or c == '\r':
let wsStart = self.currentPos()
var ws = ""
while self.hasMore() and (self.peek() == ' ' or self.peek() == '\t' or self.peek() == '\r'):
ws.add(self.advance())
self.emitToken(tkWhitespace, ws, wsStart)
# Line comment
elif c == '#':
let comStart = self.currentPos()
discard self.advance()
var content = "#" & self.parseLineComment()
self.emitToken(tkComment, content, comStart)
# Triple-quoted strings
elif self.peekString(3) == "\"\"\"" or self.peekString(3) == "'''":
let quoteChar = if self.peek() == '"': '"' else: '\''
let strStart = self.currentPos()
discard self.advance()
discard self.advance()
discard self.advance()
var content = repeat($quoteChar, 3)
var foundClosing = false
while self.hasMore():
if self.peekString(3) == repeat($quoteChar, 3):
discard self.advance()
discard self.advance()
discard self.advance()
content.add(repeat($quoteChar, 3))
self.emitToken(tkMultilineString, content, strStart)
foundClosing = true
break
else:
content.add(self.advance())
if not foundClosing:
self.recordError(esError, "Unclosed triple-quoted string", ErrUnclosedString, strStart)
self.emitToken(tkMultilineString, content, strStart)
# f-strings
elif c == 'f' or c == 'F':
let peekStr = self.peekString(2)
if peekStr.len >= 2 and (peekStr[1] == '"' or peekStr[1] == '\''):
let fStart = self.currentPos()
discard self.advance() # f
let quote = self.advance() # " or '
var content = "f" & quote
var braceDepth = 0
var foundClosing = false
while self.hasMore():
let fc = self.advance()
if fc == '{' and braceDepth == 0:
braceDepth += 1
content.add(fc)
elif fc == '}' and braceDepth == 1:
braceDepth -= 1
content.add(fc)
elif fc == '\\':
content.add(fc)
if self.hasMore(): content.add(self.advance())
elif fc == quote and braceDepth == 0:
content.add(fc)
self.emitToken(tkString, content, fStart)
foundClosing = true
break
else:
content.add(fc)
if fc == '{': braceDepth += 1
elif fc == '}': braceDepth -= 1
if not foundClosing:
self.recordError(esError, "Unclosed f-string", ErrUnclosedString, fStart)
self.emitToken(tkString, content, fStart)
else:
# Regular identifier
var id = "f"
while self.hasMore() and (isAlphaNum(self.peek()) or self.peek() == '_'):
id.add(self.advance())
self.emitToken(tkIdentifier, id, startPos)
# Regular string (single or double quote)
elif c == '"' or c == '\'':
let quote = c
let strStart = self.currentPos()
discard self.advance() # opening quote
var content = $quote
var foundClosing = false
while self.hasMore():
let sc = self.advance()
if sc == '\\':
content.add(sc)
if self.hasMore(): content.add(self.advance())
elif sc == quote:
content.add(quote)
self.emitToken(tkString, content, strStart)
foundClosing = true
break
elif sc == '\n':
self.recordError(esError, "Newline in string literal", ErrUnclosedString, strStart)
content.add('\n')
foundClosing = true
break
else:
content.add(sc)
if not foundClosing:
self.recordError(esError, "Unclosed string", ErrUnclosedString, strStart)
self.emitToken(tkString, content, strStart)
# Numbers
elif isDigit(c):
let numStart = self.currentPos()
var num = ""
# Check prefix
if c == '0' and self.hasMore():
let nxt = self.peek(1)
if nxt == 'x' or nxt == 'X':
num.add(self.advance())
num.add(self.advance())
while self.hasMore() and isHexDigit(self.peek()):
num.add(self.advance())
self.emitToken(tkNumber, num, numStart)
continue
elif nxt == 'b' or nxt == 'B':
num.add(self.advance())
num.add(self.advance())
while self.hasMore() and isBinaryDigit(self.peek()):
num.add(self.advance())
self.emitToken(tkNumber, num, numStart)
continue
elif nxt == 'o' or nxt == 'O':
num.add(self.advance())
num.add(self.advance())
while self.hasMore() and isOctalDigit(self.peek()):
num.add(self.advance())
self.emitToken(tkNumber, num, numStart)
continue
while self.hasMore() and (self.peek().isDigit() or self.peek() == '.' or self.peek() == 'e' or self.peek() == 'E' or self.peek() == 'j' or self.peek() == 'J'):
num.add(self.advance())
self.emitToken(tkNumber, num, numStart)
# Identifiers and keywords
elif isAlpha(c) or c == '_':
let idStart = self.currentPos()
var ident = ""
while self.hasMore() and (isAlphaNum(self.peek()) or self.peek() == '_'):
ident.add(self.advance())
if pythonKeywordSet.contains(ident):
self.emitToken(tkKeyword, ident, idStart)
else:
self.emitToken(tkIdentifier, ident, idStart)
# Brackets
elif c == '(':
self.parenDepth += 1
self.emitBracketToken(kOpenParen, $self.advance(), startPos)
elif c == ')':
if self.parenDepth > 0: self.parenDepth -= 1
self.emitBracketToken(kCloseParen, $self.advance(), startPos)
if self.bracketStack.len > 0 and self.bracketStack[^1][0] == '(':
discard self.bracketStack.pop()
elif c == '[':
self.parenDepth += 1
self.emitBracketToken(kOpenBracket, $self.advance(), startPos)
elif c == ']':
if self.parenDepth > 0: self.parenDepth -= 1
self.emitBracketToken(kCloseBracket, $self.advance(), startPos)
if self.bracketStack.len > 0 and self.bracketStack[^1][0] == '[':
discard self.bracketStack.pop()
elif c == '{':
self.parenDepth += 1
self.emitBracketToken(kOpenBrace, $self.advance(), startPos)
elif c == '}':
if self.parenDepth > 0: self.parenDepth -= 1
self.emitBracketToken(kCloseBrace, $self.advance(), startPos)
if self.bracketStack.len > 0 and self.bracketStack[^1][0] == '{':
discard self.bracketStack.pop()
# Operators
elif c in {'+', '-', '*', '/', '%', '^', '~', '|', '&', '<', '>', '='}:
let opStart = self.currentPos()
var op = ""
# Multi-char operators
while self.hasMore() and self.peek() in {'+', '-', '*', '/', '%', '^', '~', '|', '&', '<', '>', '=', '!'}:
op.add(self.advance())
if op == "=" or op == "+=" or op == "-=" or op == "*=" or op == "/=" or op == "%=" or op == "//=" or op == "**=" or op == "&=" or op == "|=" or op == "^=" or op == "<<=" or op == ">>=":
self.emitToken(tkAssignment, op, opStart)
elif op == ":":
self.emitToken(tkPunctuation, op, opStart)
else:
self.emitToken(tkOperator, op, opStart)
# Punctuation
elif c in {'.', ',', ';', ':'}:
let pStart = self.currentPos()
if c == ':' and self.hasMore() and self.peek(1) == '=':
discard self.advance()
discard self.advance()
self.emitToken(tkAssignment, ":=", pStart)
else:
self.emitToken(tkPunctuation, $self.advance(), pStart)
# Backslash continuation
elif c == '\\':
let bsStart = self.currentPos()
discard self.advance()
if self.hasMore() and self.peek() == '\n':
discard self.advance()
self.emitToken(tkSpecial, "\\\n", bsStart)
else:
self.emitToken(tkOperator, "\\", bsStart)
# Decorator
elif c == '@':
let decStart = self.currentPos()
discard self.advance()
var decorator = "@"
while self.hasMore() and (isAlphaNum(self.peek()) or self.peek() == '_' or self.peek() == '.'):
decorator.add(self.advance())
self.emitToken(tkDirective, decorator, decStart)
# Unknown
else:
let uStart = self.currentPos()
self.emitToken(tkError, $self.advance(), uStart)
# Flush remaining indentation at EOF
# Trailing indent at EOF is valid Python - emit dedent tokens silently.
while self.indentStack.len > 1:
discard self.indentStack.pop()
self.emitToken(tkSpecial, "<dedent>", self.currentPos())
# End of file
self.emitToken(tkEndOfFile, "", self.currentPos())
when defined(nimcheckDebug):
if self.options.debugMode:
debugLeave("PYTHON_TOKENIZER", &"Tokenization complete: {self.tokens.len} tokens")
Reference in New Issue View Git Blame Copy Permalink