diff --git a/Cargo.lock b/Cargo.lock index 4993dcd..b0dfc7a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3,5 +3,127 @@ version = 4 [[package]] -name = "rust-starter" +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + +[[package]] +name = "phf" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "010378780309880b08997fae13be7834dba947d36393bd372f2b1556deb2a2f6" +dependencies = [ + "phf_macros", + "phf_shared", + "serde", +] + +[[package]] +name = "phf_generator" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aeb62e0959d5a1bebc965f4d15d9e2b7cea002b6b0f5ba8cde6cc26738467100" +dependencies = [ + "fastrand", + "phf_shared", +] + +[[package]] +name = "phf_macros" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fa8d0ca26d424d27630da600c6624696e7dec8bf7b3b492b383c5dc49e5e085" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "phf_shared" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6fd9027e2d9319be6349febd1db4e8d02aa544921200c9b777720ac34a3aa89" +dependencies = [ + "siphasher", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "river-interpreter" version = "0.1.0" +dependencies = [ + "phf", +] + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "siphasher" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" + +[[package]] +name = "syn" +version = "2.0.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" diff --git a/Cargo.toml b/Cargo.toml index 5cba28f..17b31e8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,7 @@ [package] -name = "rust-starter" +name = "river-interpreter" version = "0.1.0" edition = "2024" [dependencies] +phf = { version = "0.14.0", features = ["macros"] } diff --git a/src/ast.rs b/src/ast.rs index e69de29..d79b436 100644 --- a/src/ast.rs +++ b/src/ast.rs @@ -0,0 +1,11 @@ +pub mod AST { + pub enum NodeType { + Program, + NumericLiteral, + Identifier, + BinaryExpression, + CallExperssion, + UnaryExpression, + FunctionDeclaration, + } +} diff --git a/src/lexer.rs b/src/lexer.rs index 4db2814..060f194 100644 --- a/src/lexer.rs +++ b/src/lexer.rs @@ -11,9 +11,119 @@ mod checking { } pub fn isskippable(str: &str) -> bool { - match str { - " " | "\n" | "\t" => true, - _ => false, - } + matches!(str, " " | "\n" | "\t") + } +} + +pub mod interpreter { + use crate::lexer::checking; + use std::{fmt, process}; + + #[derive(Clone)] + pub enum Token { + Number(String), + Identifier(String), + Equals, + Let, + OpenParen, + CloseParen, + BinaryOperator(String), + } + + static KEYWORDS: phf::Map<&'static str, Token> = phf::phf_map! { + "let" => Token::Let, + }; + + impl Token { + pub fn from(token: String) -> Option { + match token.as_str() { + "(" => Some(Token::OpenParen), + ")" => Some(Token::CloseParen), + "+" | "-" | "*" | "/" => Some(Token::BinaryOperator(token)), + "=" => Some(Token::Equals), + + _ => { + if checking::isint(&token) { + let mut num = String::new(); + token.split("").for_each(|t| num.push_str(t)); + + Some(Token::Number(num)) + } else if checking::isalpha(&token) { + let mut ident = String::new(); + token.split("").for_each(|t| ident.push_str(t)); + + match KEYWORDS.get(ident.as_str()) { + Some(i) => Some(i.clone()), + None => Some(Token::Identifier(ident)), + } + } else if checking::isskippable(&token) { + None + } else { + eprintln!("Unrecognized character found in source code: {token}"); + process::exit(1); + } + } + } + } + } + + impl std::fmt::Display for Token { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Token::Number(s) => write!(f, "Token::Number {{{}}}", s), + Token::Identifier(s) => write!(f, "Token::Identifier {{{}}}", s), + Token::Equals => write!(f, "Token::Equals"), + Token::BinaryOperator(s) => write!(f, "Token::BinaryOperator {{{}}}", s), + Token::Let => write!(f, "Token::Let"), + Token::OpenParen => write!(f, "Token::OpenParen"), + Token::CloseParen => write!(f, "Token::CloseParen"), + } + } + } + + pub fn tokenize(source_code: String) -> Vec { + let mut tokens: Vec = Vec::new(); + let mut current_chunk = String::new(); + + // We convert everything to a character stream to add spaces around structural items + for c in source_code.chars() { + match c { + '(' | ')' | '+' | '-' | '*' | '/' | '=' => { + // If we have a pending number or keyword, flush it first + if !current_chunk.is_empty() { + if let Some(token) = Token::from(current_chunk.clone()) { + tokens.push(token); + } + current_chunk.clear(); + } + // Now pass the operator/symbol directly to Token::from + if let Some(token) = Token::from(c.to_string()) { + tokens.push(token); + } + } + ' ' | '\n' | '\t' | '\r' => { + // Whitespace means we hit the end of a word or number chunk + if !current_chunk.is_empty() { + if let Some(token) = Token::from(current_chunk.clone()) { + tokens.push(token); + } + current_chunk.clear(); + } + } + _ => { + // Keep building up an identifier or a number + current_chunk.push(c); + } + } + } + + // Flush any leftover text at the very end of the file + if !current_chunk.is_empty() { + if let Some(token) = Token::from(current_chunk) { + tokens.push(token); + } + } + + tokens } } diff --git a/src/main.rs b/src/main.rs index 2e74423..c3ddbe5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,6 +1,14 @@ mod ast; mod lexer; -fn main() { - println!("Hello, world!"); +fn main() -> Result<(), Box> { + let source_code = std::fs::read_to_string("./test.riv")?; + + let tokens = lexer::interpreter::tokenize(source_code); + + for t in tokens { + println!("{t}"); + } + + Ok(()) } diff --git a/test.riv b/test.riv new file mode 100644 index 0000000..59649ac --- /dev/null +++ b/test.riv @@ -0,0 +1 @@ +let x = 4 * (3 / 5)