added repl, eof token, and a parser with a AST producer

Cargo.lock

@@ -2,12 +2,126 @@
 # It is not intended for manual editing.
 version = 4
 
+[[package]]
+name = "anstream"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d"
+dependencies = [
+ "anstyle",
+ "anstyle-parse",
+ "anstyle-query",
+ "anstyle-wincon",
+ "colorchoice",
+ "is_terminal_polyfill",
+ "utf8parse",
+]
+
+[[package]]
+name = "anstyle"
+version = "1.0.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000"
+
+[[package]]
+name = "anstyle-parse"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e"
+dependencies = [
+ "utf8parse",
+]
+
+[[package]]
+name = "anstyle-query"
+version = "1.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
+dependencies = [
+ "windows-sys",
+]
+
+[[package]]
+name = "anstyle-wincon"
+version = "3.0.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
+dependencies = [
+ "anstyle",
+ "once_cell_polyfill",
+ "windows-sys",
+]
+
+[[package]]
+name = "clap"
+version = "4.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51"
+dependencies = [
+ "clap_builder",
+ "clap_derive",
+]
+
+[[package]]
+name = "clap_builder"
+version = "4.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f"
+dependencies = [
+ "anstream",
+ "anstyle",
+ "clap_lex",
+ "strsim",
+]
+
+[[package]]
+name = "clap_derive"
+version = "4.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9"
+dependencies = [
+ "heck",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "clap_lex"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9"
+
+[[package]]
+name = "colorchoice"
+version = "1.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570"
+
 [[package]]
 name = "fastrand"
 version = "2.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6"
 
+[[package]]
+name = "heck"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
+
+[[package]]
+name = "is_terminal_polyfill"
+version = "1.70.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695"
+
+[[package]]
+name = "once_cell_polyfill"
+version = "1.70.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
+
 [[package]]
 name = "phf"
 version = "0.14.0"
@@ -73,6 +187,7 @@ dependencies = [
 name = "river-interpreter"
 version = "0.1.0"
 dependencies = [
+ "clap",
  "phf",
 ]
 
@@ -111,6 +226,12 @@ version = "1.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649"
 
+[[package]]
+name = "strsim"
+version = "0.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
+
 [[package]]
 name = "syn"
 version = "2.0.118"
@@ -127,3 +248,24 @@ name = "unicode-ident"
 version = "1.0.24"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
+
+[[package]]
+name = "utf8parse"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
+
+[[package]]
+name = "windows-link"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
+
+[[package]]
+name = "windows-sys"
+version = "0.61.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc"
+dependencies = [
+ "windows-link",
+]

Cargo.toml

@@ -5,3 +5,4 @@ edition = "2024"
 
 [dependencies]
 phf = { version = "0.14.0", features = ["macros"] }
+clap = { version = "4.2.7", features = ["derive"] }

src/ast.rs

@@ -1,27 +1,33 @@
 pub trait Stmt {}
 pub trait Expr: Stmt {}
 
+#[derive(Debug)]
 pub enum NodeType {
     Program(Program),
     NumericLiteral(NumericLiteral),
     Identifier(Identifier),
     BinaryExpr(BinaryExpr),
+    EOF,
 }
 
+#[derive(Debug)]
 pub struct Program {
     pub body: Vec<NodeType>,
 }
 
+#[derive(Debug)]
 pub struct BinaryExpr {
     pub left: Box<NodeType>,
     pub right: Box<NodeType>,
     pub operator: String,
 }
 
+#[derive(Debug)]
 pub struct Identifier {
     pub symbol: String,
 }
 
+#[derive(Debug)]
 pub struct NumericLiteral {
     pub value: f64,
 }

src/frontend.rs

@@ -0,0 +1,28 @@
+use std::process;
+
+use crate::parser::{self, Parser};
+
+pub fn repl() {
+    // Read-Eval-Print Loop
+    use std::io::{self, Write};
+
+    println!("Welcome to the Riv Lang REPL! Type 'exit' to quit.");
+
+    loop {
+        print!("> ");
+        io::stdout().flush().unwrap();
+
+        let mut input = String::new();
+        io::stdin().read_line(&mut input).unwrap();
+
+        let input = input.trim().to_string();
+
+        if input == "exit" {
+            process::exit(0);
+        }
+
+        let program = Parser::new(input).produce_ast();
+
+        println!("{:#?}", program);
+    }
+}

src/lexer.rs

@@ -28,6 +28,7 @@ pub mod interpreter {
         OpenParen,
         CloseParen,
         BinaryOperator(String),
+        EOF,
     }
 
     static KEYWORDS: phf::Map<&'static str, Token> = phf::phf_map! {
@@ -77,6 +78,7 @@ pub mod interpreter {
                 Token::Let => write!(f, "Token::Let"),
                 Token::OpenParen => write!(f, "Token::OpenParen"),
                 Token::CloseParen => write!(f, "Token::CloseParen"),
+                Token::EOF => write!(f, "Token::EOF"),
             }
         }
     }
@@ -124,6 +126,9 @@ pub mod interpreter {
             }
         }
 
+        // EOF Token
+        tokens.push(Token::EOF);
+
         tokens
     }
 }

src/main.rs

@@ -1,13 +1,34 @@
+use clap::Parser;
+
 mod ast;
+mod frontend;
 mod lexer;
+mod parser;
+
+#[derive(clap::Parser)]
+struct CLI {
+    /// Path to the source code file
+    #[arg(short, long)]
+    file: Option<String>,
+
+    /// Run the REPL (Read-Eval-Print Loop)
+    #[arg(short, long)]
+    repl: bool,
+}
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
-    let source_code = std::fs::read_to_string("./test.riv")?;
+    let cli = CLI::parse();
 
-    let tokens = lexer::interpreter::tokenize(source_code);
+    if cli.repl {
-
+        frontend::repl();
-    for t in tokens {
+    } else if let Some(file_path) = cli.file {
-        println!("{t}");
+        let source_code = std::fs::read_to_string(file_path)?;
+        let mut parser = parser::Parser::new(source_code);
+        let program = parser.produce_ast();
+        println!("{:#?}", program);
+    } else {
+        eprintln!("Please provide a file path or use the --repl flag.");
+        std::process::exit(1);
     }
 
     Ok(())

src/parser.rs

@@ -0,0 +1,80 @@
+use std::process;
+
+use crate::ast::{BinaryExpr, Expr, Identifier, NodeType, NumericLiteral, Program, Stmt};
+use crate::lexer::interpreter;
+
+pub struct Parser {
+    tokens: std::iter::Peekable<std::vec::IntoIter<interpreter::Token>>,
+}
+
+impl Parser {
+    pub fn new(source_code: String) -> Self {
+        Self {
+            tokens: interpreter::tokenize(source_code).into_iter().peekable(),
+        }
+    }
+
+    pub fn produce_ast(&mut self) -> Program {
+        let mut program = Program { body: vec![] };
+
+        while let Some(t) = self.tokens.next() {
+            program.body.push(self.parse_stmt(t));
+        }
+
+        program
+    }
+
+    fn parse_stmt(&mut self, t: interpreter::Token) -> NodeType {
+        // skip to parse_expr
+        self.parse_expr(t)
+    }
+
+    fn parse_expr(&mut self, t: interpreter::Token) -> NodeType {
+        self.parse_additive_expr(t)
+    }
+
+    fn parse_primary_expr(&mut self, t: interpreter::Token) -> NodeType {
+        use crate::ast::NodeType;
+        use interpreter::Token;
+
+        match t {
+            Token::Identifier(i) => NodeType::Identifier(Identifier { symbol: i.clone() }),
+            Token::Number(n) => NodeType::NumericLiteral(NumericLiteral {
+                value: n.parse().expect(
+                    format!("Failed to parse source code, {} was not a number", n).as_str(),
+                ),
+            }),
+            Token::EOF => NodeType::EOF,
+            _ => {
+                eprintln!("Unexpected token: {}", t);
+                process::exit(1);
+            }
+        }
+    }
+
+    fn parse_additive_expr(&mut self, t: interpreter::Token) -> NodeType {
+        let mut left = self.parse_primary_expr(t);
+
+        while let Some(interpreter::Token::BinaryOperator(operator)) = self.tokens.peek() {
+            if operator != "+" && operator != "-" {
+                break;
+            }
+
+            let operator = operator.clone();
+            self.tokens.next();
+
+            let right = match self.tokens.next() {
+                Some(token) => self.parse_primary_expr(token),
+                None => break,
+            };
+
+            left = NodeType::BinaryExpr(BinaryExpr {
+                left: Box::new(left),
+                right: Box::new(right),
+                operator,
+            });
+        }
+
+        left
+    }
+}