Initial commit of dem beast./plot.py

2025-12-02 06:54:32 +01:00
commit c9cf356cbd
58 changed files with 9092 additions and 0 deletions
@@ -0,0 +1,160 @@
+#ifndef RAVA_LEXER_H
+#define RAVA_LEXER_H
+
+#include <stddef.h>
+#include <stdint.h>
+#include <stdbool.h>
+
+typedef enum {
+    RAVA_TOKEN_EOF,
+
+    RAVA_TOKEN_KEYWORD_ABSTRACT,
+    RAVA_TOKEN_KEYWORD_ASSERT,
+    RAVA_TOKEN_KEYWORD_BOOLEAN,
+    RAVA_TOKEN_KEYWORD_BREAK,
+    RAVA_TOKEN_KEYWORD_BYTE,
+    RAVA_TOKEN_KEYWORD_CASE,
+    RAVA_TOKEN_KEYWORD_CATCH,
+    RAVA_TOKEN_KEYWORD_CHAR,
+    RAVA_TOKEN_KEYWORD_CLASS,
+    RAVA_TOKEN_KEYWORD_CONST,
+    RAVA_TOKEN_KEYWORD_CONTINUE,
+    RAVA_TOKEN_KEYWORD_DEFAULT,
+    RAVA_TOKEN_KEYWORD_DO,
+    RAVA_TOKEN_KEYWORD_DOUBLE,
+    RAVA_TOKEN_KEYWORD_ELSE,
+    RAVA_TOKEN_KEYWORD_ENUM,
+    RAVA_TOKEN_KEYWORD_EXTENDS,
+    RAVA_TOKEN_KEYWORD_FINAL,
+    RAVA_TOKEN_KEYWORD_FINALLY,
+    RAVA_TOKEN_KEYWORD_FLOAT,
+    RAVA_TOKEN_KEYWORD_FOR,
+    RAVA_TOKEN_KEYWORD_GOTO,
+    RAVA_TOKEN_KEYWORD_IF,
+    RAVA_TOKEN_KEYWORD_IMPLEMENTS,
+    RAVA_TOKEN_KEYWORD_IMPORT,
+    RAVA_TOKEN_KEYWORD_INSTANCEOF,
+    RAVA_TOKEN_KEYWORD_INT,
+    RAVA_TOKEN_KEYWORD_INTERFACE,
+    RAVA_TOKEN_KEYWORD_LONG,
+    RAVA_TOKEN_KEYWORD_NATIVE,
+    RAVA_TOKEN_KEYWORD_NEW,
+    RAVA_TOKEN_KEYWORD_PACKAGE,
+    RAVA_TOKEN_KEYWORD_PRIVATE,
+    RAVA_TOKEN_KEYWORD_PROTECTED,
+    RAVA_TOKEN_KEYWORD_PUBLIC,
+    RAVA_TOKEN_KEYWORD_RETURN,
+    RAVA_TOKEN_KEYWORD_SHORT,
+    RAVA_TOKEN_KEYWORD_STATIC,
+    RAVA_TOKEN_KEYWORD_STRICTFP,
+    RAVA_TOKEN_KEYWORD_SUPER,
+    RAVA_TOKEN_KEYWORD_SWITCH,
+    RAVA_TOKEN_KEYWORD_SYNCHRONIZED,
+    RAVA_TOKEN_KEYWORD_THIS,
+    RAVA_TOKEN_KEYWORD_THROW,
+    RAVA_TOKEN_KEYWORD_THROWS,
+    RAVA_TOKEN_KEYWORD_TRANSIENT,
+    RAVA_TOKEN_KEYWORD_TRY,
+    RAVA_TOKEN_KEYWORD_VOID,
+    RAVA_TOKEN_KEYWORD_VOLATILE,
+    RAVA_TOKEN_KEYWORD_WHILE,
+
+    RAVA_TOKEN_LITERAL_TRUE,
+    RAVA_TOKEN_LITERAL_FALSE,
+    RAVA_TOKEN_LITERAL_NULL,
+    RAVA_TOKEN_LITERAL_INTEGER,
+    RAVA_TOKEN_LITERAL_LONG,
+    RAVA_TOKEN_LITERAL_FLOAT,
+    RAVA_TOKEN_LITERAL_DOUBLE,
+    RAVA_TOKEN_LITERAL_CHARACTER,
+    RAVA_TOKEN_LITERAL_STRING,
+
+    RAVA_TOKEN_IDENTIFIER,
+
+    RAVA_TOKEN_LPAREN,
+    RAVA_TOKEN_RPAREN,
+    RAVA_TOKEN_LBRACE,
+    RAVA_TOKEN_RBRACE,
+    RAVA_TOKEN_LBRACKET,
+    RAVA_TOKEN_RBRACKET,
+    RAVA_TOKEN_SEMICOLON,
+    RAVA_TOKEN_COMMA,
+    RAVA_TOKEN_DOT,
+    RAVA_TOKEN_ELLIPSIS,
+    RAVA_TOKEN_AT,
+    RAVA_TOKEN_COLONCOLON,
+
+    RAVA_TOKEN_ASSIGN,
+    RAVA_TOKEN_GT,
+    RAVA_TOKEN_LT,
+    RAVA_TOKEN_BANG,
+    RAVA_TOKEN_TILDE,
+    RAVA_TOKEN_QUESTION,
+    RAVA_TOKEN_COLON,
+    RAVA_TOKEN_ARROW,
+    RAVA_TOKEN_EQUAL,
+    RAVA_TOKEN_GE,
+    RAVA_TOKEN_LE,
+    RAVA_TOKEN_NE,
+    RAVA_TOKEN_AND,
+    RAVA_TOKEN_OR,
+    RAVA_TOKEN_INC,
+    RAVA_TOKEN_DEC,
+    RAVA_TOKEN_PLUS,
+    RAVA_TOKEN_MINUS,
+    RAVA_TOKEN_STAR,
+    RAVA_TOKEN_SLASH,
+    RAVA_TOKEN_AMP,
+    RAVA_TOKEN_PIPE,
+    RAVA_TOKEN_CARET,
+    RAVA_TOKEN_PERCENT,
+    RAVA_TOKEN_LSHIFT,
+    RAVA_TOKEN_RSHIFT,
+    RAVA_TOKEN_URSHIFT,
+    RAVA_TOKEN_PLUSASSIGN,
+    RAVA_TOKEN_MINUSASSIGN,
+    RAVA_TOKEN_STARASSIGN,
+    RAVA_TOKEN_SLASHASSIGN,
+    RAVA_TOKEN_ANDASSIGN,
+    RAVA_TOKEN_ORASSIGN,
+    RAVA_TOKEN_CARETASSIGN,
+    RAVA_TOKEN_PERCENTASSIGN,
+    RAVA_TOKEN_LSHIFTASSIGN,
+    RAVA_TOKEN_RSHIFTASSIGN,
+    RAVA_TOKEN_URSHIFTASSIGN,
+
+    RAVA_TOKEN_ERROR
+} RavaTokenType_e;
+
+typedef union {
+    int64_t int_value;
+    double float_value;
+    char *string_value;
+    char char_value;
+} RavaLiteralValue_u;
+
+typedef struct {
+    RavaTokenType_e type;
+    char *lexeme;
+    int line;
+    int column;
+    RavaLiteralValue_u value;
+} RavaToken_t;
+
+typedef struct {
+    const char *source;
+    size_t source_length;
+    size_t current;
+    size_t start;
+    int line;
+    int column;
+    int start_column;
+    char *error_message;
+} RavaLexer_t;
+
+RavaLexer_t* rava_lexer_create(const char *source);
+void rava_lexer_destroy(RavaLexer_t *lexer);
+RavaToken_t* rava_lexer_next_token(RavaLexer_t *lexer);
+void rava_token_destroy(RavaToken_t *token);
+
+#endif
@@ -0,0 +1,74 @@
+#include "lexer.h"
+#include <string.h>
+
+typedef struct {
+    const char *keyword;
+    RavaTokenType_e type;
+} RavaKeyword_t;
+
+static const RavaKeyword_t KEYWORDS[] = {
+    {"abstract", RAVA_TOKEN_KEYWORD_ABSTRACT},
+    {"assert", RAVA_TOKEN_KEYWORD_ASSERT},
+    {"boolean", RAVA_TOKEN_KEYWORD_BOOLEAN},
+    {"break", RAVA_TOKEN_KEYWORD_BREAK},
+    {"byte", RAVA_TOKEN_KEYWORD_BYTE},
+    {"case", RAVA_TOKEN_KEYWORD_CASE},
+    {"catch", RAVA_TOKEN_KEYWORD_CATCH},
+    {"char", RAVA_TOKEN_KEYWORD_CHAR},
+    {"class", RAVA_TOKEN_KEYWORD_CLASS},
+    {"const", RAVA_TOKEN_KEYWORD_CONST},
+    {"continue", RAVA_TOKEN_KEYWORD_CONTINUE},
+    {"default", RAVA_TOKEN_KEYWORD_DEFAULT},
+    {"do", RAVA_TOKEN_KEYWORD_DO},
+    {"double", RAVA_TOKEN_KEYWORD_DOUBLE},
+    {"else", RAVA_TOKEN_KEYWORD_ELSE},
+    {"enum", RAVA_TOKEN_KEYWORD_ENUM},
+    {"extends", RAVA_TOKEN_KEYWORD_EXTENDS},
+    {"false", RAVA_TOKEN_LITERAL_FALSE},
+    {"final", RAVA_TOKEN_KEYWORD_FINAL},
+    {"finally", RAVA_TOKEN_KEYWORD_FINALLY},
+    {"float", RAVA_TOKEN_KEYWORD_FLOAT},
+    {"for", RAVA_TOKEN_KEYWORD_FOR},
+    {"goto", RAVA_TOKEN_KEYWORD_GOTO},
+    {"if", RAVA_TOKEN_KEYWORD_IF},
+    {"implements", RAVA_TOKEN_KEYWORD_IMPLEMENTS},
+    {"import", RAVA_TOKEN_KEYWORD_IMPORT},
+    {"instanceof", RAVA_TOKEN_KEYWORD_INSTANCEOF},
+    {"int", RAVA_TOKEN_KEYWORD_INT},
+    {"interface", RAVA_TOKEN_KEYWORD_INTERFACE},
+    {"long", RAVA_TOKEN_KEYWORD_LONG},
+    {"native", RAVA_TOKEN_KEYWORD_NATIVE},
+    {"new", RAVA_TOKEN_KEYWORD_NEW},
+    {"null", RAVA_TOKEN_LITERAL_NULL},
+    {"package", RAVA_TOKEN_KEYWORD_PACKAGE},
+    {"private", RAVA_TOKEN_KEYWORD_PRIVATE},
+    {"protected", RAVA_TOKEN_KEYWORD_PROTECTED},
+    {"public", RAVA_TOKEN_KEYWORD_PUBLIC},
+    {"return", RAVA_TOKEN_KEYWORD_RETURN},
+    {"short", RAVA_TOKEN_KEYWORD_SHORT},
+    {"static", RAVA_TOKEN_KEYWORD_STATIC},
+    {"strictfp", RAVA_TOKEN_KEYWORD_STRICTFP},
+    {"super", RAVA_TOKEN_KEYWORD_SUPER},
+    {"switch", RAVA_TOKEN_KEYWORD_SWITCH},
+    {"synchronized", RAVA_TOKEN_KEYWORD_SYNCHRONIZED},
+    {"this", RAVA_TOKEN_KEYWORD_THIS},
+    {"throw", RAVA_TOKEN_KEYWORD_THROW},
+    {"throws", RAVA_TOKEN_KEYWORD_THROWS},
+    {"transient", RAVA_TOKEN_KEYWORD_TRANSIENT},
+    {"true", RAVA_TOKEN_LITERAL_TRUE},
+    {"try", RAVA_TOKEN_KEYWORD_TRY},
+    {"void", RAVA_TOKEN_KEYWORD_VOID},
+    {"volatile", RAVA_TOKEN_KEYWORD_VOLATILE},
+    {"while", RAVA_TOKEN_KEYWORD_WHILE},
+};
+
+static const size_t KEYWORDS_COUNT = sizeof(KEYWORDS) / sizeof(KEYWORDS[0]);
+
+RavaTokenType_e rava_lexer_lookup_keyword(const char *identifier) {
+    for (size_t i = 0; i < KEYWORDS_COUNT; i++) {
+        if (strcmp(identifier, KEYWORDS[i].keyword) == 0) {
+            return KEYWORDS[i].type;
+        }
+    }
+    return RAVA_TOKEN_IDENTIFIER;
+}
@@ -0,0 +1,216 @@
+#define _POSIX_C_SOURCE 200809L
+#include "lexer.h"
+#include <stdlib.h>
+#include <string.h>
+#include <ctype.h>
+#include <stdio.h>
+
+extern RavaToken_t* _rava_lexer_create_token(RavaLexer_t *lexer, RavaTokenType_e type);
+extern char _rava_lexer_peek(RavaLexer_t *lexer);
+extern char _rava_lexer_peek_next(RavaLexer_t *lexer);
+extern char _rava_lexer_advance(RavaLexer_t *lexer);
+extern bool _rava_lexer_is_at_end(RavaLexer_t *lexer);
+
+static char _rava_unescape_char(char c) {
+    switch (c) {
+        case 'n': return '\n';
+        case 't': return '\t';
+        case 'r': return '\r';
+        case 'b': return '\b';
+        case 'f': return '\f';
+        case '\\': return '\\';
+        case '\'': return '\'';
+        case '"': return '"';
+        default: return c;
+    }
+}
+
+RavaToken_t* rava_lexer_parse_string(RavaLexer_t *lexer) {
+    size_t capacity = 128;
+    size_t length = 0;
+    char *str = malloc(capacity);
+
+    while (!_rava_lexer_is_at_end(lexer) && _rava_lexer_peek(lexer) != '"') {
+        char c = _rava_lexer_advance(lexer);
+
+        if (c == '\\') {
+            if (_rava_lexer_is_at_end(lexer)) {
+                free(str);
+                lexer->error_message = strdup("Unterminated string escape");
+                return _rava_lexer_create_token(lexer, RAVA_TOKEN_ERROR);
+            }
+
+            char next = _rava_lexer_advance(lexer);
+            if (next == 'u') {
+                int codepoint = 0;
+                for (int i = 0; i < 4; i++) {
+                    if (_rava_lexer_is_at_end(lexer) || !isxdigit(_rava_lexer_peek(lexer))) {
+                        free(str);
+                        lexer->error_message = strdup("Invalid Unicode escape");
+                        return _rava_lexer_create_token(lexer, RAVA_TOKEN_ERROR);
+                    }
+                    char hex = _rava_lexer_advance(lexer);
+                    codepoint = codepoint * 16 + (isdigit(hex) ? hex - '0' : tolower(hex) - 'a' + 10);
+                }
+                if (codepoint < 128) {
+                    c = (char)codepoint;
+                } else {
+                    c = '?';
+                }
+            } else if (next >= '0' && next <= '7') {
+                int octal = next - '0';
+                if (!_rava_lexer_is_at_end(lexer) && _rava_lexer_peek(lexer) >= '0' && _rava_lexer_peek(lexer) <= '7') {
+                    octal = octal * 8 + (_rava_lexer_advance(lexer) - '0');
+                    if (!_rava_lexer_is_at_end(lexer) && _rava_lexer_peek(lexer) >= '0' && _rava_lexer_peek(lexer) <= '7') {
+                        octal = octal * 8 + (_rava_lexer_advance(lexer) - '0');
+                    }
+                }
+                c = (char)octal;
+            } else {
+                c = _rava_unescape_char(next);
+            }
+        }
+
+        if (length + 1 >= capacity) {
+            capacity *= 2;
+            str = realloc(str, capacity);
+        }
+        str[length++] = c;
+    }
+
+    if (_rava_lexer_is_at_end(lexer)) {
+        free(str);
+        lexer->error_message = strdup("Unterminated string");
+        return _rava_lexer_create_token(lexer, RAVA_TOKEN_ERROR);
+    }
+
+    _rava_lexer_advance(lexer);
+
+    str[length] = '\0';
+    RavaToken_t *token = _rava_lexer_create_token(lexer, RAVA_TOKEN_LITERAL_STRING);
+    token->value.string_value = str;
+    return token;
+}
+
+RavaToken_t* rava_lexer_parse_character(RavaLexer_t *lexer) {
+    if (_rava_lexer_is_at_end(lexer)) {
+        lexer->error_message = strdup("Unterminated character literal");
+        return _rava_lexer_create_token(lexer, RAVA_TOKEN_ERROR);
+    }
+
+    char c = _rava_lexer_advance(lexer);
+
+    if (c == '\\') {
+        if (_rava_lexer_is_at_end(lexer)) {
+            lexer->error_message = strdup("Unterminated character escape");
+            return _rava_lexer_create_token(lexer, RAVA_TOKEN_ERROR);
+        }
+        char next = _rava_lexer_advance(lexer);
+        c = _rava_unescape_char(next);
+    }
+
+    if (_rava_lexer_is_at_end(lexer) || _rava_lexer_peek(lexer) != '\'') {
+        lexer->error_message = strdup("Unterminated character literal");
+        return _rava_lexer_create_token(lexer, RAVA_TOKEN_ERROR);
+    }
+
+    _rava_lexer_advance(lexer);
+
+    RavaToken_t *token = _rava_lexer_create_token(lexer, RAVA_TOKEN_LITERAL_CHARACTER);
+    token->value.char_value = c;
+    return token;
+}
+
+RavaToken_t* rava_lexer_parse_number(RavaLexer_t *lexer) {
+    bool is_hex = false;
+    bool is_octal = false;
+    bool is_binary = false;
+    bool is_float = false;
+    bool has_exponent = false;
+
+    char first_char = lexer->source[lexer->start];
+    if (first_char == '0' && !_rava_lexer_is_at_end(lexer)) {
+        char next = _rava_lexer_peek(lexer);
+        if (next == 'x' || next == 'X') {
+            is_hex = true;
+            _rava_lexer_advance(lexer);
+            _rava_lexer_advance(lexer);
+        } else if (next == 'b' || next == 'B') {
+            is_binary = true;
+            _rava_lexer_advance(lexer);
+            _rava_lexer_advance(lexer);
+        } else if (next >= '0' && next <= '7') {
+            is_octal = true;
+            _rava_lexer_advance(lexer);
+        }
+    }
+
+    while (!_rava_lexer_is_at_end(lexer)) {
+        char c = _rava_lexer_peek(lexer);
+
+        if (is_hex && isxdigit(c)) {
+            _rava_lexer_advance(lexer);
+        } else if (is_binary && (c == '0' || c == '1')) {
+            _rava_lexer_advance(lexer);
+        } else if (is_octal && c >= '0' && c <= '7') {
+            _rava_lexer_advance(lexer);
+        } else if (!is_hex && !is_binary && !is_octal && isdigit(c)) {
+            _rava_lexer_advance(lexer);
+        } else if (c == '.' && !is_float && !is_hex && !is_binary && !is_octal) {
+            is_float = true;
+            _rava_lexer_advance(lexer);
+        } else if ((c == 'e' || c == 'E') && !has_exponent && !is_hex && !is_binary && !is_octal) {
+            is_float = true;
+            has_exponent = true;
+            _rava_lexer_advance(lexer);
+            if (!_rava_lexer_is_at_end(lexer) && (_rava_lexer_peek(lexer) == '+' || _rava_lexer_peek(lexer) == '-')) {
+                _rava_lexer_advance(lexer);
+            }
+        } else if (c == '_') {
+            _rava_lexer_advance(lexer);
+        } else {
+            break;
+        }
+    }
+
+    RavaTokenType_e type = RAVA_TOKEN_LITERAL_INTEGER;
+
+    if (!_rava_lexer_is_at_end(lexer)) {
+        char suffix = _rava_lexer_peek(lexer);
+        if (suffix == 'L' || suffix == 'l') {
+            type = RAVA_TOKEN_LITERAL_LONG;
+            _rava_lexer_advance(lexer);
+        } else if (suffix == 'F' || suffix == 'f') {
+            is_float = true;
+            type = RAVA_TOKEN_LITERAL_FLOAT;
+            _rava_lexer_advance(lexer);
+        } else if (suffix == 'D' || suffix == 'd') {
+            is_float = true;
+            type = RAVA_TOKEN_LITERAL_DOUBLE;
+            _rava_lexer_advance(lexer);
+        }
+    }
+
+    if (is_float && type == RAVA_TOKEN_LITERAL_INTEGER) {
+        type = RAVA_TOKEN_LITERAL_DOUBLE;
+    }
+
+    RavaToken_t *token = _rava_lexer_create_token(lexer, type);
+
+    if (is_float) {
+        token->value.float_value = strtod(token->lexeme, NULL);
+    } else {
+        char *endptr;
+        if (is_hex) {
+            token->value.int_value = strtoll(token->lexeme, &endptr, 16);
+        } else if (is_octal) {
+            token->value.int_value = strtoll(token->lexeme, &endptr, 8);
+        } else if (is_binary) {
+            token->value.int_value = strtoll(token->lexeme + 2, &endptr, 2);
+        } else {
+            token->value.int_value = strtoll(token->lexeme, &endptr, 10);
+        }
+    }
+
+    return token;
+}
@@ -0,0 +1,319 @@
+#include "lexer.h"
+#include <stdlib.h>
+#include <string.h>
+#include <ctype.h>
+#include <stdio.h>
+
+extern RavaTokenType_e rava_lexer_lookup_keyword(const char *identifier);
+extern RavaToken_t* rava_lexer_parse_string(RavaLexer_t *lexer);
+extern RavaToken_t* rava_lexer_parse_character(RavaLexer_t *lexer);
+extern RavaToken_t* rava_lexer_parse_number(RavaLexer_t *lexer);
+
+RavaLexer_t* rava_lexer_create(const char *source) {
+    RavaLexer_t *lexer = malloc(sizeof(RavaLexer_t));
+    lexer->source = source;
+    lexer->source_length = strlen(source);
+    lexer->current = 0;
+    lexer->start = 0;
+    lexer->line = 1;
+    lexer->column = 1;
+    lexer->start_column = 1;
+    lexer->error_message = NULL;
+    return lexer;
+}
+
+void rava_lexer_destroy(RavaLexer_t *lexer) {
+    if (lexer) {
+        if (lexer->error_message) {
+            free(lexer->error_message);
+        }
+        free(lexer);
+    }
+}
+
+void rava_token_destroy(RavaToken_t *token) {
+    if (token) {
+        if (token->lexeme) {
+            free(token->lexeme);
+        }
+        if (token->type == RAVA_TOKEN_LITERAL_STRING && token->value.string_value) {
+            free(token->value.string_value);
+        }
+        free(token);
+    }
+}
+
+bool _rava_lexer_is_at_end(RavaLexer_t *lexer) {
+    return lexer->current >= lexer->source_length;
+}
+
+char _rava_lexer_peek(RavaLexer_t *lexer) {
+    if (_rava_lexer_is_at_end(lexer)) return '\0';
+    return lexer->source[lexer->current];
+}
+
+char _rava_lexer_peek_next(RavaLexer_t *lexer) {
+    if (lexer->current + 1 >= lexer->source_length) return '\0';
+    return lexer->source[lexer->current + 1];
+}
+
+char _rava_lexer_advance(RavaLexer_t *lexer) {
+    if (_rava_lexer_is_at_end(lexer)) return '\0';
+    char c = lexer->source[lexer->current++];
+    if (c == '\n') {
+        lexer->line++;
+        lexer->column = 1;
+    } else {
+        lexer->column++;
+    }
+    return c;
+}
+
+bool _rava_lexer_match(RavaLexer_t *lexer, char expected) {
+    if (_rava_lexer_is_at_end(lexer)) return false;
+    if (lexer->source[lexer->current] != expected) return false;
+    lexer->current++;
+    lexer->column++;
+    return true;
+}
+
+void _rava_lexer_skip_whitespace(RavaLexer_t *lexer) {
+    while (!_rava_lexer_is_at_end(lexer)) {
+        char c = _rava_lexer_peek(lexer);
+        switch (c) {
+            case ' ':
+            case '\t':
+            case '\r':
+            case '\n':
+            case '\f':
+                _rava_lexer_advance(lexer);
+                break;
+            case '/':
+                if (_rava_lexer_peek_next(lexer) == '/') {
+                    while (!_rava_lexer_is_at_end(lexer) && _rava_lexer_peek(lexer) != '\n') {
+                        _rava_lexer_advance(lexer);
+                    }
+                } else if (_rava_lexer_peek_next(lexer) == '*') {
+                    _rava_lexer_advance(lexer);
+                    _rava_lexer_advance(lexer);
+                    while (!_rava_lexer_is_at_end(lexer)) {
+                        if (_rava_lexer_peek(lexer) == '*' && _rava_lexer_peek_next(lexer) == '/') {
+                            _rava_lexer_advance(lexer);
+                            _rava_lexer_advance(lexer);
+                            break;
+                        }
+                        _rava_lexer_advance(lexer);
+                    }
+                } else {
+                    return;
+                }
+                break;
+            default:
+                return;
+        }
+    }
+}
+
+RavaToken_t* _rava_lexer_create_token(RavaLexer_t *lexer, RavaTokenType_e type) {
+    RavaToken_t *token = malloc(sizeof(RavaToken_t));
+    token->type = type;
+    size_t length = lexer->current - lexer->start;
+    token->lexeme = malloc(length + 1);
+    memcpy(token->lexeme, lexer->source + lexer->start, length);
+    token->lexeme[length] = '\0';
+    token->line = lexer->line;
+    token->column = lexer->start_column;
+    token->value.int_value = 0;
+    return token;
+}
+
+static bool _rava_is_alpha(char c) {
+    return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || c == '$';
+}
+
+static bool _rava_is_alnum(char c) {
+    return _rava_is_alpha(c) || (c >= '0' && c <= '9');
+}
+
+static RavaToken_t* _rava_lexer_identifier(RavaLexer_t *lexer) {
+    while (_rava_is_alnum(_rava_lexer_peek(lexer))) {
+        _rava_lexer_advance(lexer);
+    }
+
+    RavaToken_t *token = _rava_lexer_create_token(lexer, RAVA_TOKEN_IDENTIFIER);
+    RavaTokenType_e keyword_type = rava_lexer_lookup_keyword(token->lexeme);
+    token->type = keyword_type;
+
+    if (keyword_type == RAVA_TOKEN_LITERAL_TRUE) {
+        token->value.int_value = 1;
+    } else if (keyword_type == RAVA_TOKEN_LITERAL_FALSE) {
+        token->value.int_value = 0;
+    }
+
+    return token;
+}
+
+RavaToken_t* rava_lexer_next_token(RavaLexer_t *lexer) {
+    _rava_lexer_skip_whitespace(lexer);
+
+    lexer->start = lexer->current;
+    lexer->start_column = lexer->column;
+
+    if (_rava_lexer_is_at_end(lexer)) {
+        return _rava_lexer_create_token(lexer, RAVA_TOKEN_EOF);
+    }
+
+    char c = _rava_lexer_advance(lexer);
+
+    if (_rava_is_alpha(c)) {
+        return _rava_lexer_identifier(lexer);
+    }
+
+    if (isdigit(c)) {
+        return rava_lexer_parse_number(lexer);
+    }
+
+    switch (c) {
+        case '(': return _rava_lexer_create_token(lexer, RAVA_TOKEN_LPAREN);
+        case ')': return _rava_lexer_create_token(lexer, RAVA_TOKEN_RPAREN);
+        case '{': return _rava_lexer_create_token(lexer, RAVA_TOKEN_LBRACE);
+        case '}': return _rava_lexer_create_token(lexer, RAVA_TOKEN_RBRACE);
+        case '[': return _rava_lexer_create_token(lexer, RAVA_TOKEN_LBRACKET);
+        case ']': return _rava_lexer_create_token(lexer, RAVA_TOKEN_RBRACKET);
+        case ';': return _rava_lexer_create_token(lexer, RAVA_TOKEN_SEMICOLON);
+        case ',': return _rava_lexer_create_token(lexer, RAVA_TOKEN_COMMA);
+        case '~': return _rava_lexer_create_token(lexer, RAVA_TOKEN_TILDE);
+        case '?': return _rava_lexer_create_token(lexer, RAVA_TOKEN_QUESTION);
+        case '@': return _rava_lexer_create_token(lexer, RAVA_TOKEN_AT);
+
+        case '.':
+            if (_rava_lexer_peek(lexer) == '.' && _rava_lexer_peek_next(lexer) == '.') {
+                _rava_lexer_advance(lexer);
+                _rava_lexer_advance(lexer);
+                return _rava_lexer_create_token(lexer, RAVA_TOKEN_ELLIPSIS);
+            }
+            return _rava_lexer_create_token(lexer, RAVA_TOKEN_DOT);
+
+        case ':':
+            if (_rava_lexer_match(lexer, ':')) {
+                return _rava_lexer_create_token(lexer, RAVA_TOKEN_COLONCOLON);
+            }
+            return _rava_lexer_create_token(lexer, RAVA_TOKEN_COLON);
+
+        case '+':
+            if (_rava_lexer_match(lexer, '+')) {
+                return _rava_lexer_create_token(lexer, RAVA_TOKEN_INC);
+            }
+            if (_rava_lexer_match(lexer, '=')) {
+                return _rava_lexer_create_token(lexer, RAVA_TOKEN_PLUSASSIGN);
+            }
+            return _rava_lexer_create_token(lexer, RAVA_TOKEN_PLUS);
+
+        case '-':
+            if (_rava_lexer_match(lexer, '-')) {
+                return _rava_lexer_create_token(lexer, RAVA_TOKEN_DEC);
+            }
+            if (_rava_lexer_match(lexer, '=')) {
+                return _rava_lexer_create_token(lexer, RAVA_TOKEN_MINUSASSIGN);
+            }
+            if (_rava_lexer_match(lexer, '>')) {
+                return _rava_lexer_create_token(lexer, RAVA_TOKEN_ARROW);
+            }
+            return _rava_lexer_create_token(lexer, RAVA_TOKEN_MINUS);
+
+        case '*':
+            if (_rava_lexer_match(lexer, '=')) {
+                return _rava_lexer_create_token(lexer, RAVA_TOKEN_STARASSIGN);
+            }
+            return _rava_lexer_create_token(lexer, RAVA_TOKEN_STAR);
+
+        case '/':
+            if (_rava_lexer_match(lexer, '=')) {
+                return _rava_lexer_create_token(lexer, RAVA_TOKEN_SLASHASSIGN);
+            }
+            return _rava_lexer_create_token(lexer, RAVA_TOKEN_SLASH);
+
+        case '%':
+            if (_rava_lexer_match(lexer, '=')) {
+                return _rava_lexer_create_token(lexer, RAVA_TOKEN_PERCENTASSIGN);
+            }
+            return _rava_lexer_create_token(lexer, RAVA_TOKEN_PERCENT);
+
+        case '&':
+            if (_rava_lexer_match(lexer, '&')) {
+                return _rava_lexer_create_token(lexer, RAVA_TOKEN_AND);
+            }
+            if (_rava_lexer_match(lexer, '=')) {
+                return _rava_lexer_create_token(lexer, RAVA_TOKEN_ANDASSIGN);
+            }
+            return _rava_lexer_create_token(lexer, RAVA_TOKEN_AMP);
+
+        case '|':
+            if (_rava_lexer_match(lexer, '|')) {
+                return _rava_lexer_create_token(lexer, RAVA_TOKEN_OR);
+            }
+            if (_rava_lexer_match(lexer, '=')) {
+                return _rava_lexer_create_token(lexer, RAVA_TOKEN_ORASSIGN);
+            }
+            return _rava_lexer_create_token(lexer, RAVA_TOKEN_PIPE);
+
+        case '^':
+            if (_rava_lexer_match(lexer, '=')) {
+                return _rava_lexer_create_token(lexer, RAVA_TOKEN_CARETASSIGN);
+            }
+            return _rava_lexer_create_token(lexer, RAVA_TOKEN_CARET);
+
+        case '!':
+            if (_rava_lexer_match(lexer, '=')) {
+                return _rava_lexer_create_token(lexer, RAVA_TOKEN_NE);
+            }
+            return _rava_lexer_create_token(lexer, RAVA_TOKEN_BANG);
+
+        case '=':
+            if (_rava_lexer_match(lexer, '=')) {
+                return _rava_lexer_create_token(lexer, RAVA_TOKEN_EQUAL);
+            }
+            return _rava_lexer_create_token(lexer, RAVA_TOKEN_ASSIGN);
+
+        case '<':
+            if (_rava_lexer_match(lexer, '<')) {
+                if (_rava_lexer_match(lexer, '=')) {
+                    return _rava_lexer_create_token(lexer, RAVA_TOKEN_LSHIFTASSIGN);
+                }
+                return _rava_lexer_create_token(lexer, RAVA_TOKEN_LSHIFT);
+            }
+            if (_rava_lexer_match(lexer, '=')) {
+                return _rava_lexer_create_token(lexer, RAVA_TOKEN_LE);
+            }
+            return _rava_lexer_create_token(lexer, RAVA_TOKEN_LT);
+
+        case '>':
+            if (_rava_lexer_match(lexer, '>')) {
+                if (_rava_lexer_match(lexer, '>')) {
+                    if (_rava_lexer_match(lexer, '=')) {
+                        return _rava_lexer_create_token(lexer, RAVA_TOKEN_URSHIFTASSIGN);
+                    }
+                    return _rava_lexer_create_token(lexer, RAVA_TOKEN_URSHIFT);
+                }
+                if (_rava_lexer_match(lexer, '=')) {
+                    return _rava_lexer_create_token(lexer, RAVA_TOKEN_RSHIFTASSIGN);
+                }
+                return _rava_lexer_create_token(lexer, RAVA_TOKEN_RSHIFT);
+            }
+            if (_rava_lexer_match(lexer, '=')) {
+                return _rava_lexer_create_token(lexer, RAVA_TOKEN_GE);
+            }
+            return _rava_lexer_create_token(lexer, RAVA_TOKEN_GT);
+
+        case '"':
+            return rava_lexer_parse_string(lexer);
+
+        case '\'':
+            return rava_lexer_parse_character(lexer);
+
+        default:
+            lexer->error_message = malloc(100);
+            snprintf(lexer->error_message, 100, "Unexpected character '%c'", c);
+            return _rava_lexer_create_token(lexer, RAVA_TOKEN_ERROR);
+    }
+}