finished transpilling on lexer

src/ast.rs

@@ -0,0 +1,11 @@
+pub mod AST {
+    pub enum NodeType {
+        Program,
+        NumericLiteral,
+        Identifier,
+        BinaryExpression,
+        CallExperssion,
+        UnaryExpression,
+        FunctionDeclaration,
+    }
+}

src/lexer.rs

@@ -11,9 +11,119 @@ mod checking {
     }
 
     pub fn isskippable(str: &str) -> bool {
-        match str {
-            " " | "\n" | "\t" => true,
-            _ => false,
-        }
+        matches!(str, " " | "\n" | "\t")
+    }
+}
+
+pub mod interpreter {
+    use crate::lexer::checking;
+    use std::{fmt, process};
+
+    #[derive(Clone)]
+    pub enum Token {
+        Number(String),
+        Identifier(String),
+        Equals,
+        Let,
+        OpenParen,
+        CloseParen,
+        BinaryOperator(String),
+    }
+
+    static KEYWORDS: phf::Map<&'static str, Token> = phf::phf_map! {
+        "let" => Token::Let,
+    };
+
+    impl Token {
+        pub fn from(token: String) -> Option<Token> {
+            match token.as_str() {
+                "(" => Some(Token::OpenParen),
+                ")" => Some(Token::CloseParen),
+                "+" | "-" | "*" | "/" => Some(Token::BinaryOperator(token)),
+                "=" => Some(Token::Equals),
+
+                _ => {
+                    if checking::isint(&token) {
+                        let mut num = String::new();
+                        token.split("").for_each(|t| num.push_str(t));
+
+                        Some(Token::Number(num))
+                    } else if checking::isalpha(&token) {
+                        let mut ident = String::new();
+                        token.split("").for_each(|t| ident.push_str(t));
+
+                        match KEYWORDS.get(ident.as_str()) {
+                            Some(i) => Some(i.clone()),
+                            None => Some(Token::Identifier(ident)),
+                        }
+                    } else if checking::isskippable(&token) {
+                        None
+                    } else {
+                        eprintln!("Unrecognized character found in source code: {token}");
+                        process::exit(1);
+                    }
+                }
+            }
+        }
+    }
+
+    impl std::fmt::Display for Token {
+        fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+            match self {
+                Token::Number(s) => write!(f, "Token::Number {{{}}}", s),
+                Token::Identifier(s) => write!(f, "Token::Identifier {{{}}}", s),
+                Token::Equals => write!(f, "Token::Equals"),
+                Token::BinaryOperator(s) => write!(f, "Token::BinaryOperator {{{}}}", s),
+                Token::Let => write!(f, "Token::Let"),
+                Token::OpenParen => write!(f, "Token::OpenParen"),
+                Token::CloseParen => write!(f, "Token::CloseParen"),
+            }
+        }
+    }
+
+    pub fn tokenize(source_code: String) -> Vec<Token> {
+        let mut tokens: Vec<Token> = Vec::new();
+        let mut current_chunk = String::new();
+
+        // We convert everything to a character stream to add spaces around structural items
+        for c in source_code.chars() {
+            match c {
+                '(' | ')' | '+' | '-' | '*' | '/' | '=' => {
+                    // If we have a pending number or keyword, flush it first
+                    if !current_chunk.is_empty() {
+                        if let Some(token) = Token::from(current_chunk.clone()) {
+                            tokens.push(token);
+                        }
+                        current_chunk.clear();
+                    }
+                    // Now pass the operator/symbol directly to Token::from
+                    if let Some(token) = Token::from(c.to_string()) {
+                        tokens.push(token);
+                    }
+                }
+                ' ' | '\n' | '\t' | '\r' => {
+                    // Whitespace means we hit the end of a word or number chunk
+                    if !current_chunk.is_empty() {
+                        if let Some(token) = Token::from(current_chunk.clone()) {
+                            tokens.push(token);
+                        }
+                        current_chunk.clear();
+                    }
+                }
+                _ => {
+                    // Keep building up an identifier or a number
+                    current_chunk.push(c);
+                }
+            }
+        }
+
+        // Flush any leftover text at the very end of the file
+        if !current_chunk.is_empty() {
+            if let Some(token) = Token::from(current_chunk) {
+                tokens.push(token);
+            }
+        }
+
+        tokens
     }
 }

src/main.rs

@@ -1,6 +1,14 @@
 mod ast;
 mod lexer;
 
-fn main() {
-    println!("Hello, world!");
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    let source_code = std::fs::read_to_string("./test.riv")?;
+
+    let tokens = lexer::interpreter::tokenize(source_code);
+
+    for t in tokens {
+        println!("{t}");
+    }
+
+    Ok(())
 }