lexer.h

Go to the documentation of this file.

 
#pragma once
 
#include <cstdint>
#include <fstream>
#include <ostream>
#include <string>
#include <utility>
#include <vector>
 
#include "error.h"
 
namespace picceler {

struct Token {
  enum class Type : uint8_t {
    IDENTIFIER, // Represents identifiers (user defined names)
    NUMBER,     // Represents numeric literals (e.g., integers, floats)
    PLUS,       // Represents binary operation '+'
    MINUS,      // Represents binary operation '-'
    MULTIPLY,   // Represents binary operation '*'
    DIVIDE,     // Represents binary operation '/'
    STRING,     // Represents string literals
    L_PAREN,    // Represents the left parenthesis '('
    R_PAREN,    // Represents the right parenthesis ')'
    L_BRACKET,  // Represents the left bracket '['
    R_BRACKET,  // Represents the right bracket ']'
    L_BRACE,    // Represents the left brace '{'
    R_BRACE,    // Represents the right brace '}'
    COMMA,      // Represents the comma ','
    COLON,      // Represents the colon ':'
    ARROW,      // Represents the arrow '->'
    ASSIGN,     // Represents the assignment operator '='
    EQ,         // Represents the assignment operator '=='
    NE,         // Represents the assignment operator '!='
    LT,         // Represents the assignment operator '<'
    GT,         // Represents the assignment operator '>'
    LE,         // Represents the assignment operator '<='
    GE,         // Represents the assignment operator '>='
    KW_FOR,     // Represents the keyword 'for'
    KW_STEP,    // Represents the keyword 'step'
    DOT_DOT,    // Represents the operator '..'
    TYPE,       // Represents type annotations (e.g., int, float, string)
    KW_DEF,     // Represents the keyword 'def'
    KW_RETURN,  // Represents the keyword 'return'
    KW_IF,      // Represents the keyword 'if'
    KW_ELSE,    // Represents the keyword 'else'
    EOF_TOKEN,  // Represents the end of file
    UNKNOWN     // Represents unknown tokens
  };
 
  Token() : _type(Type::UNKNOWN), _value(""), _location() {}
 
  Token(Type type, std::string value, Location location) : _type(type), _value(std::move(value)), _location(location) {}
 
  Token(const Token &other) : _type(other._type), _value(other._value), _location(other._location) {}
  Token(Token &&other) noexcept : _type(other._type), _value(std::move(other._value)), _location(other._location) {}
  ~Token() = default;
  Token &operator=(const Token &other) {
    if (this != &other) {
      _type = other._type;
      _value = other._value;
      _location = other._location;
    }
    return *this;
  }
  Token &operator=(Token &&other) noexcept {
    if (this != &other) {
      _type = other._type;
      _value = std::move(other._value);
      _location = other._location;
    }
    return *this;
  }

  std::string typeToString() const;

  std::string toString() const {
    return std::format("Token(type: {}, value: '{}', line: {}, column: {})", typeToString(), _value, _location.line(),
                       _location.column());
  }
 
  Type type() const { return _type; }
  const std::string &value() const { return _value; }
  size_t line() const { return _location.line(); }
  size_t column() const { return _location.column(); }
  Location location() const { return _location; }
  /*
   * @brief Compares a token to a token type for easy checking in parsing.
   * @param lhs The token to compare.
   * @param rhs The token type to compare against.
   * @return True if the token's type matches the token type, false otherwise.
   */
  friend constexpr bool operator==(const Token &lhs, const Token::Type &rhs) { return lhs._type == rhs; }
 
private:
  Type _type;
  std::string _value;
  Location _location;
};

class Lexer {
public:
  Lexer();

  void setSourceString(std::string_view source);

  Result<void> setSource(const std::string &filepath);

  Result<Token> nextToken();

  Result<Token> peekToken();

  void skipWhitespaceAndComment();

  Result<std::vector<Token>> getTokens();
 
private:
  void resetState();

  bool eof() const;

  char peek() const;

  char get();

 
  bool isIdentifier(char ch) const;
  bool isSymbol(char ch) const;

  Result<Token::Type> isKeyword(const std::string &value) const;

  Result<Token::Type> isType(const std::string &value) const;

  Result<Token> readIdentifierOrKeywordOrType(std::pair<size_t, size_t> start);

  Result<Token> readNumber(std::pair<size_t, size_t> start);

  Result<Token> readString(std::pair<size_t, size_t> start);

  Result<Token> readSymbol(std::pair<size_t, size_t> start);
 
private:
  std::string unescapeString(std::string &&string) const;
 
private:
  std::ifstream _file;
  std::string _buffer;
  size_t _position;
  size_t _line;
  size_t _column;
};

std::ostream &operator<<(std::ostream &os, const Token &token);
 
} // namespace picceler