145 lines
3.3 KiB
C++
145 lines
3.3 KiB
C++
/**************************************************************************/
|
|
/* aethex_tokenizer.h */
|
|
/**************************************************************************/
|
|
/* This file is part of: */
|
|
/* AETHEX ENGINE */
|
|
/* https://aethex.foundation */
|
|
/**************************************************************************/
|
|
/* Copyright (c) 2026-present AeThex Labs. */
|
|
/**************************************************************************/
|
|
|
|
#pragma once
|
|
|
|
#include "core/string/ustring.h"
|
|
#include "core/templates/vector.h"
|
|
|
|
class AeThexTokenizer {
|
|
public:
|
|
enum TokenType {
|
|
// Literals
|
|
TK_IDENTIFIER,
|
|
TK_NUMBER,
|
|
TK_STRING,
|
|
TK_TEMPLATE_STRING,
|
|
|
|
// Keywords - Core constructs
|
|
TK_REALITY, // Module/namespace
|
|
TK_JOURNEY, // Cross-platform function
|
|
TK_PORTAL, // Import/export
|
|
TK_BEACON, // Event/signal
|
|
TK_ARTIFACT, // Class
|
|
TK_ESSENCE, // Interface
|
|
TK_CHRONICLE, // Enum
|
|
|
|
// Keywords - Control flow
|
|
TK_WHEN, // If
|
|
TK_OTHERWISE, // Else
|
|
TK_TRAVERSE, // For
|
|
TK_WHILE,
|
|
TK_BREAK,
|
|
TK_CONTINUE,
|
|
TK_RETURN,
|
|
TK_YIELD,
|
|
|
|
// Keywords - Data
|
|
TK_LET,
|
|
TK_CONST,
|
|
TK_MUT,
|
|
TK_NEW,
|
|
|
|
// Keywords - Platform
|
|
TK_PLATFORM,
|
|
TK_SYNC,
|
|
TK_ASYNC,
|
|
TK_AWAIT,
|
|
TK_ACROSS,
|
|
TK_ALL,
|
|
|
|
// Keywords - Actions
|
|
TK_NOTIFY,
|
|
TK_REVEAL,
|
|
TK_SUMMON,
|
|
TK_BANISH,
|
|
|
|
// Keywords - Literals
|
|
TK_TRUE,
|
|
TK_FALSE,
|
|
TK_NULL,
|
|
TK_SELF,
|
|
TK_SUPER,
|
|
|
|
// Operators
|
|
TK_PLUS, // +
|
|
TK_MINUS, // -
|
|
TK_STAR, // *
|
|
TK_SLASH, // /
|
|
TK_PERCENT, // %
|
|
TK_CARET, // ^
|
|
TK_EQUAL, // =
|
|
TK_EQUAL_EQUAL, // ==
|
|
TK_NOT_EQUAL, // !=
|
|
TK_LESS, // <
|
|
TK_LESS_EQUAL, // <=
|
|
TK_GREATER, // >
|
|
TK_GREATER_EQUAL, // >=
|
|
TK_AND, // and / &&
|
|
TK_OR, // or / ||
|
|
TK_NOT, // not / !
|
|
TK_PLUS_EQUAL, // +=
|
|
TK_MINUS_EQUAL, // -=
|
|
TK_ARROW, // ->
|
|
TK_FAT_ARROW, // =>
|
|
|
|
// Punctuation
|
|
TK_COLON, // :
|
|
TK_SEMICOLON, // ;
|
|
TK_COMMA, // ,
|
|
TK_DOT, // .
|
|
TK_PAREN_OPEN, // (
|
|
TK_PAREN_CLOSE, // )
|
|
TK_BRACKET_OPEN, // [
|
|
TK_BRACKET_CLOSE, // ]
|
|
TK_BRACE_OPEN, // {
|
|
TK_BRACE_CLOSE, // }
|
|
|
|
// Special
|
|
TK_NEWLINE,
|
|
TK_INDENT,
|
|
TK_DEDENT,
|
|
TK_EOF,
|
|
TK_ERROR,
|
|
};
|
|
|
|
struct Token {
|
|
TokenType type = TK_ERROR;
|
|
String value;
|
|
int line = 0;
|
|
int column = 0;
|
|
};
|
|
|
|
private:
|
|
String source;
|
|
int pos = 0;
|
|
int line = 1;
|
|
int column = 1;
|
|
Vector<Token> tokens;
|
|
int indent_stack_count = 0;
|
|
|
|
char peek(int offset = 0) const;
|
|
char advance();
|
|
bool match(char expected);
|
|
void skip_whitespace();
|
|
void skip_comment();
|
|
|
|
Token make_token(TokenType type, const String &value = "");
|
|
Token scan_string(char quote);
|
|
Token scan_number();
|
|
Token scan_identifier();
|
|
TokenType check_keyword(const String &identifier);
|
|
|
|
public:
|
|
Error tokenize(const String &p_source);
|
|
const Vector<Token> &get_tokens() const { return tokens; }
|
|
|
|
static String token_type_to_string(TokenType type);
|
|
};
|