finished transpilling on lexer

Cargo.lock

@@ -3,5 +3,127 @@
 version = 4
 
 [[package]]
-name = "rust-starter"
+name = "fastrand"
+version = "2.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6"
+
+[[package]]
+name = "phf"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "010378780309880b08997fae13be7834dba947d36393bd372f2b1556deb2a2f6"
+dependencies = [
+ "phf_macros",
+ "phf_shared",
+ "serde",
+]
+
+[[package]]
+name = "phf_generator"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aeb62e0959d5a1bebc965f4d15d9e2b7cea002b6b0f5ba8cde6cc26738467100"
+dependencies = [
+ "fastrand",
+ "phf_shared",
+]
+
+[[package]]
+name = "phf_macros"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5fa8d0ca26d424d27630da600c6624696e7dec8bf7b3b492b383c5dc49e5e085"
+dependencies = [
+ "phf_generator",
+ "phf_shared",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "phf_shared"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c6fd9027e2d9319be6349febd1db4e8d02aa544921200c9b777720ac34a3aa89"
+dependencies = [
+ "siphasher",
+]
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.106"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.46"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "river-interpreter"
 version = "0.1.0"
+dependencies = [
+ "phf",
+]
+
+[[package]]
+name = "serde"
+version = "1.0.228"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e"
+dependencies = [
+ "serde_core",
+]
+
+[[package]]
+name = "serde_core"
+version = "1.0.228"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad"
+dependencies = [
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.228"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "siphasher"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649"
+
+[[package]]
+name = "syn"
+version = "2.0.118"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.24"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"

Cargo.toml

@@ -1,6 +1,7 @@
 [package]
-name = "rust-starter"
+name = "river-interpreter"
 version = "0.1.0"
 edition = "2024"
 
 [dependencies]
+phf = { version = "0.14.0", features = ["macros"] }

src/ast.rs

@@ -0,0 +1,11 @@
+pub mod AST {
+    pub enum NodeType {
+        Program,
+        NumericLiteral,
+        Identifier,
+        BinaryExpression,
+        CallExperssion,
+        UnaryExpression,
+        FunctionDeclaration,
+    }
+}

src/lexer.rs

@@ -11,9 +11,119 @@ mod checking {
     }
 
     pub fn isskippable(str: &str) -> bool {
-        match str {
+        matches!(str, " " | "\n" | "\t")
-            " " | "\n" | "\t" => true,
+    }
-            _ => false,
+}
-        }
+
+pub mod interpreter {
+    use crate::lexer::checking;
+    use std::{fmt, process};
+
+    #[derive(Clone)]
+    pub enum Token {
+        Number(String),
+        Identifier(String),
+        Equals,
+        Let,
+        OpenParen,
+        CloseParen,
+        BinaryOperator(String),
+    }
+
+    static KEYWORDS: phf::Map<&'static str, Token> = phf::phf_map! {
+        "let" => Token::Let,
+    };
+
+    impl Token {
+        pub fn from(token: String) -> Option<Token> {
+            match token.as_str() {
+                "(" => Some(Token::OpenParen),
+                ")" => Some(Token::CloseParen),
+                "+" | "-" | "*" | "/" => Some(Token::BinaryOperator(token)),
+                "=" => Some(Token::Equals),
+
+                _ => {
+                    if checking::isint(&token) {
+                        let mut num = String::new();
+                        token.split("").for_each(|t| num.push_str(t));
+
+                        Some(Token::Number(num))
+                    } else if checking::isalpha(&token) {
+                        let mut ident = String::new();
+                        token.split("").for_each(|t| ident.push_str(t));
+
+                        match KEYWORDS.get(ident.as_str()) {
+                            Some(i) => Some(i.clone()),
+                            None => Some(Token::Identifier(ident)),
+                        }
+                    } else if checking::isskippable(&token) {
+                        None
+                    } else {
+                        eprintln!("Unrecognized character found in source code: {token}");
+                        process::exit(1);
+                    }
+                }
+            }
+        }
+    }
+
+    impl std::fmt::Display for Token {
+        fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+            match self {
+                Token::Number(s) => write!(f, "Token::Number {{{}}}", s),
+                Token::Identifier(s) => write!(f, "Token::Identifier {{{}}}", s),
+                Token::Equals => write!(f, "Token::Equals"),
+                Token::BinaryOperator(s) => write!(f, "Token::BinaryOperator {{{}}}", s),
+                Token::Let => write!(f, "Token::Let"),
+                Token::OpenParen => write!(f, "Token::OpenParen"),
+                Token::CloseParen => write!(f, "Token::CloseParen"),
+            }
+        }
+    }
+
+    pub fn tokenize(source_code: String) -> Vec<Token> {
+        let mut tokens: Vec<Token> = Vec::new();
+        let mut current_chunk = String::new();
+
+        // We convert everything to a character stream to add spaces around structural items
+        for c in source_code.chars() {
+            match c {
+                '(' | ')' | '+' | '-' | '*' | '/' | '=' => {
+                    // If we have a pending number or keyword, flush it first
+                    if !current_chunk.is_empty() {
+                        if let Some(token) = Token::from(current_chunk.clone()) {
+                            tokens.push(token);
+                        }
+                        current_chunk.clear();
+                    }
+                    // Now pass the operator/symbol directly to Token::from
+                    if let Some(token) = Token::from(c.to_string()) {
+                        tokens.push(token);
+                    }
+                }
+                ' ' | '\n' | '\t' | '\r' => {
+                    // Whitespace means we hit the end of a word or number chunk
+                    if !current_chunk.is_empty() {
+                        if let Some(token) = Token::from(current_chunk.clone()) {
+                            tokens.push(token);
+                        }
+                        current_chunk.clear();
+                    }
+                }
+                _ => {
+                    // Keep building up an identifier or a number
+                    current_chunk.push(c);
+                }
+            }
+        }
+
+        // Flush any leftover text at the very end of the file
+        if !current_chunk.is_empty() {
+            if let Some(token) = Token::from(current_chunk) {
+                tokens.push(token);
+            }
+        }
+
+        tokens
     }
 }

src/main.rs

@@ -1,6 +1,14 @@
 mod ast;
 mod lexer;
 
-fn main() {
+fn main() -> Result<(), Box<dyn std::error::Error>> {
-    println!("Hello, world!");
+    let source_code = std::fs::read_to_string("./test.riv")?;
+
+    let tokens = lexer::interpreter::tokenize(source_code);
+
+    for t in tokens {
+        println!("{t}");
+    }
+
+    Ok(())
 }

test.riv

@@ -0,0 +1 @@
+let x = 4 * (3 / 5)