"use strict"; /** * AeThex Compiler - Lexer (Tokenizer) * Converts source code into tokens */ Object.defineProperty(exports, "__esModule", { value: true }); exports.Lexer = exports.TokenType = void 0; var TokenType; (function (TokenType) { // Keywords TokenType["REALITY"] = "REALITY"; TokenType["JOURNEY"] = "JOURNEY"; TokenType["LET"] = "LET"; TokenType["IMPORT"] = "IMPORT"; TokenType["FROM"] = "FROM"; TokenType["WHEN"] = "WHEN"; TokenType["OTHERWISE"] = "OTHERWISE"; TokenType["RETURN"] = "RETURN"; TokenType["PLATFORM"] = "PLATFORM"; TokenType["NOTIFY"] = "NOTIFY"; TokenType["REVEAL"] = "REVEAL"; TokenType["SYNC"] = "SYNC"; TokenType["ACROSS"] = "ACROSS"; TokenType["NEW"] = "NEW"; // Literals TokenType["IDENTIFIER"] = "IDENTIFIER"; TokenType["STRING"] = "STRING"; TokenType["NUMBER"] = "NUMBER"; // Operators TokenType["PLUS"] = "PLUS"; TokenType["MINUS"] = "MINUS"; TokenType["STAR"] = "STAR"; TokenType["SLASH"] = "SLASH"; TokenType["EQUALS"] = "EQUALS"; TokenType["EQUALS_EQUALS"] = "EQUALS_EQUALS"; TokenType["NOT_EQUALS"] = "NOT_EQUALS"; TokenType["BANG"] = "BANG"; TokenType["LESS_THAN"] = "LESS_THAN"; TokenType["GREATER_THAN"] = "GREATER_THAN"; TokenType["LESS_EQUALS"] = "LESS_EQUALS"; TokenType["GREATER_EQUALS"] = "GREATER_EQUALS"; // Punctuation TokenType["LEFT_PAREN"] = "LEFT_PAREN"; TokenType["RIGHT_PAREN"] = "RIGHT_PAREN"; TokenType["LEFT_BRACE"] = "LEFT_BRACE"; TokenType["RIGHT_BRACE"] = "RIGHT_BRACE"; TokenType["LEFT_BRACKET"] = "LEFT_BRACKET"; TokenType["RIGHT_BRACKET"] = "RIGHT_BRACKET"; TokenType["COMMA"] = "COMMA"; TokenType["DOT"] = "DOT"; TokenType["COLON"] = "COLON"; TokenType["SEMICOLON"] = "SEMICOLON"; // Special TokenType["NEWLINE"] = "NEWLINE"; TokenType["EOF"] = "EOF"; TokenType["COMMENT"] = "COMMENT"; })(TokenType || (exports.TokenType = TokenType = {})); class Lexer { constructor(source) { this.position = 0; this.line = 1; this.column = 1; this.current = ''; this.keywords = new Map([ ['reality', TokenType.REALITY], ['journey', TokenType.JOURNEY], ['let', TokenType.LET], ['import', TokenType.IMPORT], ['from', TokenType.FROM], ['when', TokenType.WHEN], ['otherwise', TokenType.OTHERWISE], ['return', TokenType.RETURN], ['platform', TokenType.PLATFORM], ['notify', TokenType.NOTIFY], ['reveal', TokenType.REVEAL], ['sync', TokenType.SYNC], ['across', TokenType.ACROSS], ['new', TokenType.NEW] ]); this.source = source; this.current = source[0] || ''; } tokenize() { const tokens = []; while (!this.isAtEnd()) { const token = this.nextToken(); if (token && token.type !== TokenType.COMMENT) { tokens.push(token); } } tokens.push(this.makeToken(TokenType.EOF, '')); return tokens; } nextToken() { this.skipWhitespace(); if (this.isAtEnd()) { return null; } const char = this.current; // Comments if (char === '#') { return this.readComment(); } // Strings if (char === '"' || char === "'") { return this.readString(); } // Numbers if (this.isDigit(char)) { return this.readNumber(); } // Identifiers and keywords if (this.isAlpha(char)) { return this.readIdentifier(); } // Operators and punctuation return this.readOperator(); } readComment() { const start = this.column; this.advance(); // skip # let value = ''; while (!this.isAtEnd() && this.current !== '\n') { value += this.current; this.advance(); } return this.makeToken(TokenType.COMMENT, value, start); } readString() { const quote = this.current; const start = this.column; this.advance(); // skip opening quote let value = ''; while (!this.isAtEnd() && this.current !== quote) { if (this.current === '\\') { this.advance(); // Handle escape sequences if (!this.isAtEnd()) { value += this.current; this.advance(); } } else { value += this.current; this.advance(); } } if (!this.isAtEnd()) { this.advance(); // skip closing quote } return this.makeToken(TokenType.STRING, value, start); } readNumber() { const start = this.column; let value = ''; while (!this.isAtEnd() && (this.isDigit(this.current) || this.current === '.')) { value += this.current; this.advance(); } return this.makeToken(TokenType.NUMBER, value, start); } readIdentifier() { const start = this.column; let value = ''; while (!this.isAtEnd() && (this.isAlphaNumeric(this.current) || this.current === '_')) { value += this.current; this.advance(); } const tokenType = this.keywords.get(value) || TokenType.IDENTIFIER; return this.makeToken(tokenType, value, start); } readOperator() { const start = this.column; const char = this.current; this.advance(); switch (char) { case '+': return this.makeToken(TokenType.PLUS, char, start); case '-': return this.makeToken(TokenType.MINUS, char, start); case '*': return this.makeToken(TokenType.STAR, char, start); case '/': return this.makeToken(TokenType.SLASH, char, start); case '(': return this.makeToken(TokenType.LEFT_PAREN, char, start); case ')': return this.makeToken(TokenType.RIGHT_PAREN, char, start); case '{': return this.makeToken(TokenType.LEFT_BRACE, char, start); case '}': return this.makeToken(TokenType.RIGHT_BRACE, char, start); case '[': return this.makeToken(TokenType.LEFT_BRACKET, char, start); case ']': return this.makeToken(TokenType.RIGHT_BRACKET, char, start); case ',': return this.makeToken(TokenType.COMMA, char, start); case '.': return this.makeToken(TokenType.DOT, char, start); case ':': return this.makeToken(TokenType.COLON, char, start); case ';': return this.makeToken(TokenType.SEMICOLON, char, start); case '=': if (this.current === '=') { this.advance(); return this.makeToken(TokenType.EQUALS_EQUALS, '==', start); } return this.makeToken(TokenType.EQUALS, char, start); case '!': if (this.current === '=') { this.advance(); return this.makeToken(TokenType.NOT_EQUALS, '!=', start); } return this.makeToken(TokenType.BANG, char, start); case '<': if (this.current === '=') { this.advance(); return this.makeToken(TokenType.LESS_EQUALS, '<=', start); } return this.makeToken(TokenType.LESS_THAN, char, start); case '>': if (this.current === '=') { this.advance(); return this.makeToken(TokenType.GREATER_EQUALS, '>=', start); } return this.makeToken(TokenType.GREATER_THAN, char, start); default: throw new Error(`Unexpected character: ${char} at line ${this.line}, column ${start}`); } } skipWhitespace() { while (!this.isAtEnd()) { const char = this.current; if (char === ' ' || char === '\t' || char === '\r') { this.advance(); } else if (char === '\n') { this.line++; this.column = 0; this.advance(); } else { break; } } } advance() { this.position++; this.column++; this.current = this.source[this.position] || ''; } isAtEnd() { return this.position >= this.source.length; } isDigit(char) { return char >= '0' && char <= '9'; } isAlpha(char) { return (char >= 'a' && char <= 'z') || (char >= 'A' && char <= 'Z') || char === '_'; } isAlphaNumeric(char) { return this.isAlpha(char) || this.isDigit(char); } makeToken(type, value, column) { return { type, value, line: this.line, column: column || this.column }; } } exports.Lexer = Lexer; //# sourceMappingURL=Lexer.js.map