From d568653a17bbf60e4867d33590b8ddb4b2bc7c3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kha=C3=AFs=20COLIN?= Date: Sun, 4 May 2025 14:09:33 +0200 Subject: [PATCH 1/6] feat(parser): parse tokens into commands --- notes.org | 8 ++ src/lib.rs | 1 + src/parser.rs | 83 +++++++++++++++++++ .../osdb__parser__tests__parse_exit.snap | 11 +++ ...parser__tests__parse_multiple_correct.snap | 23 +++++ ...rser__tests__parse_multiple_incorrect.snap | 68 +++++++++++++++ ...__parser__tests__parse_multiple_mixed.snap | 32 +++++++ ...parser__tests__parse_single_correct-2.snap | 11 +++ ...parser__tests__parse_single_correct-3.snap | 11 +++ ...parser__tests__parse_single_correct-4.snap | 11 +++ ...parser__tests__parse_single_correct-5.snap | 11 +++ ...__parser__tests__parse_single_correct.snap | 11 +++ ...rser__tests__parse_single_incorrect-2.snap | 20 +++++ ...rser__tests__parse_single_incorrect-3.snap | 20 +++++ ...rser__tests__parse_single_incorrect-4.snap | 20 +++++ ...rser__tests__parse_single_incorrect-5.snap | 20 +++++ ...parser__tests__parse_single_incorrect.snap | 20 +++++ 17 files changed, 381 insertions(+) create mode 100644 src/parser.rs create mode 100644 src/snapshots/osdb__parser__tests__parse_exit.snap create mode 100644 src/snapshots/osdb__parser__tests__parse_multiple_correct.snap create mode 100644 src/snapshots/osdb__parser__tests__parse_multiple_incorrect.snap create mode 100644 src/snapshots/osdb__parser__tests__parse_multiple_mixed.snap create mode 100644 src/snapshots/osdb__parser__tests__parse_single_correct-2.snap create mode 100644 src/snapshots/osdb__parser__tests__parse_single_correct-3.snap create mode 100644 src/snapshots/osdb__parser__tests__parse_single_correct-4.snap create mode 100644 src/snapshots/osdb__parser__tests__parse_single_correct-5.snap create mode 100644 src/snapshots/osdb__parser__tests__parse_single_correct.snap create mode 100644 src/snapshots/osdb__parser__tests__parse_single_incorrect-2.snap create mode 100644 src/snapshots/osdb__parser__tests__parse_single_incorrect-3.snap create mode 100644 src/snapshots/osdb__parser__tests__parse_single_incorrect-4.snap create mode 100644 src/snapshots/osdb__parser__tests__parse_single_incorrect-5.snap create mode 100644 src/snapshots/osdb__parser__tests__parse_single_incorrect.snap diff --git a/notes.org b/notes.org index e059314..a6233fd 100644 --- a/notes.org +++ b/notes.org @@ -179,6 +179,14 @@ CLOCK: [2025-05-04 dim. 13:45]--[2025-05-04 dim. 13:56] => 0:11 :EFFORT: 10 :END: +*** DONE create a generic parse command that parses string into tokens into Command +:PROPERTIES: +:EFFORT: 10 +:END: +:LOGBOOK: +CLOCK: [2025-05-04 dim. 14:01]--[2025-05-04 dim. 14:14] => 0:13 +:END: + *** TODO parse tokens into meta-commands :PROPERTIES: :EFFORT: 10 diff --git a/src/lib.rs b/src/lib.rs index 5117e84..6683a24 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3,5 +3,6 @@ pub mod cli; pub mod command; pub mod error_display; pub mod meta_commands; +pub mod parser; pub mod statements; pub mod tokens; diff --git a/src/parser.rs b/src/parser.rs new file mode 100644 index 0000000..949a1bc --- /dev/null +++ b/src/parser.rs @@ -0,0 +1,83 @@ +use std::collections::VecDeque; + +use crate::{ + command::{Command, CommandParseError}, + statements::Statement, + tokens::tokenize, +}; + +pub fn parse(file: String, input: String) -> Result, Vec> { + let mut tokens: VecDeque<_> = tokenize(input, file) + .map_err(|x| x.into_iter().map(|x| x.into()).collect::>())? + .into(); + let mut cmds = Vec::new(); + let errs = Vec::new(); + while let Some(token) = tokens.pop_front() { + match token.data { + crate::tokens::TokenData::Insert => cmds.push(Command::Statement(Statement::Insert)), + crate::tokens::TokenData::Select => cmds.push(Command::Statement(Statement::Select)), + crate::tokens::TokenData::MetaCommand(meta_command) => { + cmds.push(Command::MetaCommand(meta_command)) + } + crate::tokens::TokenData::EndOfFile => (), + } + } + if errs.is_empty() { + Ok(cmds) + } else { + Err(errs) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use insta::assert_debug_snapshot; + + #[test] + fn test_parse_single_correct() { + let file = String::from(""); + assert_debug_snapshot!(parse(file.clone(), String::from(".exit"))); + assert_debug_snapshot!(parse(file.clone(), String::from("select"))); + assert_debug_snapshot!(parse(file.clone(), String::from("sElEcT"))); + assert_debug_snapshot!(parse(file.clone(), String::from("INSERT"))); + assert_debug_snapshot!(parse(file.clone(), String::from("InSErT"))); + } + + #[test] + fn test_parse_single_incorrect() { + let file = String::from(""); + assert_debug_snapshot!(parse(file.clone(), String::from(".halp"))); + assert_debug_snapshot!(parse(file.clone(), String::from("salect"))); + assert_debug_snapshot!(parse(file.clone(), String::from("sAlEcT"))); + assert_debug_snapshot!(parse(file.clone(), String::from("INSART"))); + assert_debug_snapshot!(parse(file.clone(), String::from("InSArT"))); + } + + #[test] + fn test_parse_multiple_correct() { + let file = String::from(""); + assert_debug_snapshot!(parse( + file.clone(), + String::from(".exit select select insert select") + )); + } + + #[test] + fn test_parse_multiple_incorrect() { + let file = String::from(""); + assert_debug_snapshot!(parse( + file.clone(), + String::from(".halp salect selact inset seiect") + )); + } + + #[test] + fn test_parse_multiple_mixed() { + let file = String::from(""); + assert_debug_snapshot!(parse( + file.clone(), + String::from(".exit selct select nsert select") + )); + } +} diff --git a/src/snapshots/osdb__parser__tests__parse_exit.snap b/src/snapshots/osdb__parser__tests__parse_exit.snap new file mode 100644 index 0000000..4ca1f6e --- /dev/null +++ b/src/snapshots/osdb__parser__tests__parse_exit.snap @@ -0,0 +1,11 @@ +--- +source: src/parser.rs +expression: "parse(file, String::from(\".exit\"))" +--- +Ok( + [ + MetaCommand( + Exit, + ), + ], +) diff --git a/src/snapshots/osdb__parser__tests__parse_multiple_correct.snap b/src/snapshots/osdb__parser__tests__parse_multiple_correct.snap new file mode 100644 index 0000000..a90d0ae --- /dev/null +++ b/src/snapshots/osdb__parser__tests__parse_multiple_correct.snap @@ -0,0 +1,23 @@ +--- +source: src/parser.rs +expression: "parse(file.clone(), String::from(\".exit select select insert select\"))" +--- +Ok( + [ + MetaCommand( + Exit, + ), + Statement( + Select, + ), + Statement( + Select, + ), + Statement( + Insert, + ), + Statement( + Select, + ), + ], +) diff --git a/src/snapshots/osdb__parser__tests__parse_multiple_incorrect.snap b/src/snapshots/osdb__parser__tests__parse_multiple_incorrect.snap new file mode 100644 index 0000000..2e4ba21 --- /dev/null +++ b/src/snapshots/osdb__parser__tests__parse_multiple_incorrect.snap @@ -0,0 +1,68 @@ +--- +source: src/parser.rs +expression: "parse(file.clone(), String::from(\".halp salect selact inset seiect\"))" +--- +Err( + [ + Scan( + ScanError { + location: Location { + file: "", + offset: 0, + length: 5, + }, + kind: UnknownMetaCommand( + ".halp", + ), + }, + ), + Scan( + ScanError { + location: Location { + file: "", + offset: 6, + length: 6, + }, + kind: UnknownKeyword( + "salect", + ), + }, + ), + Scan( + ScanError { + location: Location { + file: "", + offset: 13, + length: 6, + }, + kind: UnknownKeyword( + "selact", + ), + }, + ), + Scan( + ScanError { + location: Location { + file: "", + offset: 20, + length: 5, + }, + kind: UnknownKeyword( + "inset", + ), + }, + ), + Scan( + ScanError { + location: Location { + file: "", + offset: 26, + length: 6, + }, + kind: UnknownKeyword( + "seiect", + ), + }, + ), + ], +) diff --git a/src/snapshots/osdb__parser__tests__parse_multiple_mixed.snap b/src/snapshots/osdb__parser__tests__parse_multiple_mixed.snap new file mode 100644 index 0000000..9580a02 --- /dev/null +++ b/src/snapshots/osdb__parser__tests__parse_multiple_mixed.snap @@ -0,0 +1,32 @@ +--- +source: src/parser.rs +expression: "parse(file.clone(), String::from(\".exit selct select nsert select\"))" +--- +Err( + [ + Scan( + ScanError { + location: Location { + file: "", + offset: 6, + length: 5, + }, + kind: UnknownKeyword( + "selct", + ), + }, + ), + Scan( + ScanError { + location: Location { + file: "", + offset: 19, + length: 5, + }, + kind: UnknownKeyword( + "nsert", + ), + }, + ), + ], +) diff --git a/src/snapshots/osdb__parser__tests__parse_single_correct-2.snap b/src/snapshots/osdb__parser__tests__parse_single_correct-2.snap new file mode 100644 index 0000000..94cc7e1 --- /dev/null +++ b/src/snapshots/osdb__parser__tests__parse_single_correct-2.snap @@ -0,0 +1,11 @@ +--- +source: src/parser.rs +expression: "parse(file.clone(), String::from(\"select\"))" +--- +Ok( + [ + Statement( + Select, + ), + ], +) diff --git a/src/snapshots/osdb__parser__tests__parse_single_correct-3.snap b/src/snapshots/osdb__parser__tests__parse_single_correct-3.snap new file mode 100644 index 0000000..26db6c7 --- /dev/null +++ b/src/snapshots/osdb__parser__tests__parse_single_correct-3.snap @@ -0,0 +1,11 @@ +--- +source: src/parser.rs +expression: "parse(file.clone(), String::from(\"sElEcT\"))" +--- +Ok( + [ + Statement( + Select, + ), + ], +) diff --git a/src/snapshots/osdb__parser__tests__parse_single_correct-4.snap b/src/snapshots/osdb__parser__tests__parse_single_correct-4.snap new file mode 100644 index 0000000..d3ba596 --- /dev/null +++ b/src/snapshots/osdb__parser__tests__parse_single_correct-4.snap @@ -0,0 +1,11 @@ +--- +source: src/parser.rs +expression: "parse(file.clone(), String::from(\"INSERT\"))" +--- +Ok( + [ + Statement( + Insert, + ), + ], +) diff --git a/src/snapshots/osdb__parser__tests__parse_single_correct-5.snap b/src/snapshots/osdb__parser__tests__parse_single_correct-5.snap new file mode 100644 index 0000000..7bc8e93 --- /dev/null +++ b/src/snapshots/osdb__parser__tests__parse_single_correct-5.snap @@ -0,0 +1,11 @@ +--- +source: src/parser.rs +expression: "parse(file.clone(), String::from(\"InSErT\"))" +--- +Ok( + [ + Statement( + Insert, + ), + ], +) diff --git a/src/snapshots/osdb__parser__tests__parse_single_correct.snap b/src/snapshots/osdb__parser__tests__parse_single_correct.snap new file mode 100644 index 0000000..fda989a --- /dev/null +++ b/src/snapshots/osdb__parser__tests__parse_single_correct.snap @@ -0,0 +1,11 @@ +--- +source: src/parser.rs +expression: "parse(file.clone(), String::from(\".exit\"))" +--- +Ok( + [ + MetaCommand( + Exit, + ), + ], +) diff --git a/src/snapshots/osdb__parser__tests__parse_single_incorrect-2.snap b/src/snapshots/osdb__parser__tests__parse_single_incorrect-2.snap new file mode 100644 index 0000000..4e8a5f1 --- /dev/null +++ b/src/snapshots/osdb__parser__tests__parse_single_incorrect-2.snap @@ -0,0 +1,20 @@ +--- +source: src/parser.rs +expression: "parse(file.clone(), String::from(\"salect\"))" +--- +Err( + [ + Scan( + ScanError { + location: Location { + file: "", + offset: 0, + length: 6, + }, + kind: UnknownKeyword( + "salect", + ), + }, + ), + ], +) diff --git a/src/snapshots/osdb__parser__tests__parse_single_incorrect-3.snap b/src/snapshots/osdb__parser__tests__parse_single_incorrect-3.snap new file mode 100644 index 0000000..6aae7a1 --- /dev/null +++ b/src/snapshots/osdb__parser__tests__parse_single_incorrect-3.snap @@ -0,0 +1,20 @@ +--- +source: src/parser.rs +expression: "parse(file.clone(), String::from(\"sAlEcT\"))" +--- +Err( + [ + Scan( + ScanError { + location: Location { + file: "", + offset: 0, + length: 6, + }, + kind: UnknownKeyword( + "sAlEcT", + ), + }, + ), + ], +) diff --git a/src/snapshots/osdb__parser__tests__parse_single_incorrect-4.snap b/src/snapshots/osdb__parser__tests__parse_single_incorrect-4.snap new file mode 100644 index 0000000..1087e1a --- /dev/null +++ b/src/snapshots/osdb__parser__tests__parse_single_incorrect-4.snap @@ -0,0 +1,20 @@ +--- +source: src/parser.rs +expression: "parse(file.clone(), String::from(\"INSART\"))" +--- +Err( + [ + Scan( + ScanError { + location: Location { + file: "", + offset: 0, + length: 6, + }, + kind: UnknownKeyword( + "INSART", + ), + }, + ), + ], +) diff --git a/src/snapshots/osdb__parser__tests__parse_single_incorrect-5.snap b/src/snapshots/osdb__parser__tests__parse_single_incorrect-5.snap new file mode 100644 index 0000000..978fd2e --- /dev/null +++ b/src/snapshots/osdb__parser__tests__parse_single_incorrect-5.snap @@ -0,0 +1,20 @@ +--- +source: src/parser.rs +expression: "parse(file.clone(), String::from(\"InSArT\"))" +--- +Err( + [ + Scan( + ScanError { + location: Location { + file: "", + offset: 0, + length: 6, + }, + kind: UnknownKeyword( + "InSArT", + ), + }, + ), + ], +) diff --git a/src/snapshots/osdb__parser__tests__parse_single_incorrect.snap b/src/snapshots/osdb__parser__tests__parse_single_incorrect.snap new file mode 100644 index 0000000..9f0ada8 --- /dev/null +++ b/src/snapshots/osdb__parser__tests__parse_single_incorrect.snap @@ -0,0 +1,20 @@ +--- +source: src/parser.rs +expression: "parse(file.clone(), String::from(\".halp\"))" +--- +Err( + [ + Scan( + ScanError { + location: Location { + file: "", + offset: 0, + length: 5, + }, + kind: UnknownMetaCommand( + ".halp", + ), + }, + ), + ], +) From a0869b1b664948567a3e48ca7814c3d7ad54ab2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kha=C3=AFs=20COLIN?= Date: Sun, 4 May 2025 14:22:19 +0200 Subject: [PATCH 2/6] feat(parser): use token-based parser --- notes.org | 34 +++++++++++++--------------------- src/main.rs | 25 ++++++++++++++++--------- src/tokens.rs | 17 ++++++++--------- 3 files changed, 37 insertions(+), 39 deletions(-) diff --git a/notes.org b/notes.org index a6233fd..2162255 100644 --- a/notes.org +++ b/notes.org @@ -1,4 +1,5 @@ #+title: Notes +#+property: EFFORT_ALL 0 10 * DONE show errors with ariadne :PROPERTIES: @@ -120,23 +121,15 @@ CLOCK: [2025-05-03 sam. 21:21]--[2025-05-03 sam. 21:22] => 0:01 CLOCK: [2025-05-03 sam. 19:06]--[2025-05-03 sam. 19:07] => 0:01 :END: -* TODO switch statement parsing to more extensible token-based algorithm +* DONE switch statement parsing to more extensible token-based algorithm :PROPERTIES: :EFFORT: 10 :END: :LOGBOOK: CLOCK: [2025-05-04 dim. 12:07]--[2025-05-04 dim. 12:10] => 0:03 -:END: - -** TODO use tokens to parse meta-commands -:PROPERTIES: -:EFFORT: 10 -:END: -:LOGBOOK: CLOCK: [2025-05-04 dim. 12:10]--[2025-05-04 dim. 12:22] => 0:12 :END: - -*** DONE recognize meta-commands as tokens +** DONE recognize meta-commands as tokens :PROPERTIES: :EFFORT: 10 :END: @@ -145,7 +138,7 @@ CLOCK: [2025-05-04 dim. 13:32]--[2025-05-04 dim. 13:35] => 0:03 CLOCK: [2025-05-04 dim. 13:27]--[2025-05-04 dim. 13:32] => 0:05 :END: -*** DONE CommandParseError must have a ScanError variant with an Into impl +** DONE CommandParseError must have a ScanError variant with an Into impl :PROPERTIES: :EFFORT: 10 :END: @@ -153,12 +146,12 @@ CLOCK: [2025-05-04 dim. 13:27]--[2025-05-04 dim. 13:32] => 0:05 CLOCK: [2025-05-04 dim. 13:35]--[2025-05-04 dim. 13:38] => 0:03 :END: -*** DONE ScanErrors must be convertible to ariadne reports +** DONE ScanErrors must be convertible to ariadne reports :PROPERTIES: :EFFORT: 10 :END: -**** DONE Remove the CommandParseError Display implementation +*** DONE Remove the CommandParseError Display implementation :PROPERTIES: :EFFORT: 10 :END: @@ -166,7 +159,7 @@ CLOCK: [2025-05-04 dim. 13:35]--[2025-05-04 dim. 13:38] => 0:03 CLOCK: [2025-05-04 dim. 13:38]--[2025-05-04 dim. 13:44] => 0:06 :END: -**** DONE implement OSDBError for ScanError +*** DONE implement OSDBError for ScanError :PROPERTIES: :EFFORT: 10 :END: @@ -174,12 +167,12 @@ CLOCK: [2025-05-04 dim. 13:38]--[2025-05-04 dim. 13:44] => 0:06 CLOCK: [2025-05-04 dim. 13:45]--[2025-05-04 dim. 13:56] => 0:11 :END: -*** DONE remove token types which are not recognized at all +** DONE remove token types which are not recognized at all :PROPERTIES: :EFFORT: 10 :END: -*** DONE create a generic parse command that parses string into tokens into Command +** DONE create a generic parse command that parses string into tokens into Command :PROPERTIES: :EFFORT: 10 :END: @@ -187,12 +180,11 @@ CLOCK: [2025-05-04 dim. 13:45]--[2025-05-04 dim. 13:56] => 0:11 CLOCK: [2025-05-04 dim. 14:01]--[2025-05-04 dim. 14:14] => 0:13 :END: -*** TODO parse tokens into meta-commands +** DONE parse tokens into meta-commands :PROPERTIES: :EFFORT: 10 :END: -** TODO use tokens to parse statements -:PROPERTIES: -:EFFORT: -:END: +* TODO error offsets are incorrect + +* TODO remove old FromStr parser implementation diff --git a/src/main.rs b/src/main.rs index 5757d01..bac5cd6 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,20 +1,27 @@ use osdb::branding::startup_msg; use osdb::cli::read_input; -use osdb::command::Command; use osdb::error_display::OSDBError as _; +use osdb::parser::parse; fn main() { println!("{}", startup_msg()); - while let Some(input) = read_input() { - match input.parse::() { - Ok(cmd) => { - let result = cmd.execute(); - println!("{}", result.display()); - if result.should_exit { - break; + 'main: while let Some(input) = read_input() { + let file = String::from(""); + match parse(file.clone(), input.clone()) { + Ok(cmds) => { + for cmd in cmds { + let result = cmd.execute(); + if result.should_exit { + break 'main; + } + println!("{}", result.display()); + } + } + Err(errs) => { + for err in errs { + err.display(&file, &input) } } - Err(err) => err.display("", &input), } } println!("Good-bye"); diff --git a/src/tokens.rs b/src/tokens.rs index 4ecc87d..5177532 100644 --- a/src/tokens.rs +++ b/src/tokens.rs @@ -206,13 +206,13 @@ impl Tokenizer { c.is_alphanumeric() || c == '_' } - fn scan_token(&mut self) -> Result { + fn scan_token(&mut self) -> Result, ScanError> { loop { if let Some(c) = self.peek() { if Self::ident_or_keyword_start(c) { - return self.scan_identifier_or_keyword(); + return self.scan_identifier_or_keyword().map(Some); } else if c == '.' { - return self.scan_meta_command(); + return self.scan_meta_command().map(Some); } else if c.is_whitespace() { self.advance(); } else { @@ -223,10 +223,7 @@ impl Tokenizer { }); } } else { - return Err(ScanError { - location: self.current_location(0), - kind: ScanErrorKind::UnexpectedEndOfInput, - }); + return Ok(None); } } } @@ -244,8 +241,10 @@ pub fn tokenize(input: String, file: String) -> Result, Vec tokenizer.tokens.push(token), + let token = tokenizer.scan_token(); + match token { + Ok(Some(token)) => tokenizer.tokens.push(token), + Ok(None) => break, Err(err) => errors.push(err), } } From 106c2547b556186d7776f14834723a59b3e3f5e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kha=C3=AFs=20COLIN?= Date: Sun, 4 May 2025 18:10:50 +0200 Subject: [PATCH 3/6] fix(errors): unexpected char errors were pointing one char too far --- notes.org | 6 +++++- src/tokens.rs | 5 +++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/notes.org b/notes.org index 2162255..54bdf33 100644 --- a/notes.org +++ b/notes.org @@ -185,6 +185,10 @@ CLOCK: [2025-05-04 dim. 14:01]--[2025-05-04 dim. 14:14] => 0:13 :EFFORT: 10 :END: -* TODO error offsets are incorrect +* DONE error offsets are incorrect * TODO remove old FromStr parser implementation + +* TODO use a better readline impl + +* TODO handle non-interactive input better diff --git a/src/tokens.rs b/src/tokens.rs index 5177532..86180fd 100644 --- a/src/tokens.rs +++ b/src/tokens.rs @@ -216,11 +216,12 @@ impl Tokenizer { } else if c.is_whitespace() { self.advance(); } else { - self.advance(); - return Err(ScanError { + let result = Err(ScanError { location: self.current_location(1), kind: ScanErrorKind::UnexpectedChar(c), }); + self.advance(); + return result; } } else { return Ok(None); From 6b49d3ca1445c389b2ecec52ae95b878fef5dbb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kha=C3=AFs=20COLIN?= Date: Sun, 4 May 2025 18:17:36 +0200 Subject: [PATCH 4/6] refactor(parsing): remove old FromStr-based parser implementation --- notes.org | 5 +++- src/command.rs | 24 ++++------------- src/meta_commands.rs | 13 --------- ...mand__tests__parse_wrong_meta_command.snap | 21 ++++++++++----- ...command__tests__parse_wrong_statement.snap | 21 ++++++++++----- .../osdb__parser__tests__parse_exit.snap | 11 -------- ...osdb__tokens__tests__tokenizer_errors.snap | 26 ++++++++++++++++++ src/statements.rs | 18 ------------- src/tokens.rs | 27 ++----------------- 9 files changed, 67 insertions(+), 99 deletions(-) delete mode 100644 src/snapshots/osdb__parser__tests__parse_exit.snap create mode 100644 src/snapshots/osdb__tokens__tests__tokenizer_errors.snap diff --git a/notes.org b/notes.org index 54bdf33..439dc86 100644 --- a/notes.org +++ b/notes.org @@ -187,8 +187,11 @@ CLOCK: [2025-05-04 dim. 14:01]--[2025-05-04 dim. 14:14] => 0:13 * DONE error offsets are incorrect -* TODO remove old FromStr parser implementation +* DONE remove old FromStr parser implementation * TODO use a better readline impl * TODO handle non-interactive input better + +* TODO cli tests using insta-cmd +https://insta.rs/docs/cmd/ diff --git a/src/command.rs b/src/command.rs index d016549..156b15b 100644 --- a/src/command.rs +++ b/src/command.rs @@ -94,25 +94,11 @@ impl From for CommandParseError { } } -impl std::str::FromStr for Command { - type Err = CommandParseError; - - fn from_str(s: &str) -> Result { - if s.starts_with(".") { - s.parse::() - .map(|x| x.into()) - .map_err(|x| x.into()) - } else { - s.parse::() - .map(|x| x.into()) - .map_err(|x| x.into()) - } - } -} - #[cfg(test)] mod tests { - use crate::{command::Command, meta_commands::MetaCommand, statements::Statement}; + use crate::{ + command::Command, meta_commands::MetaCommand, parser::parse, statements::Statement, + }; use insta::{assert_debug_snapshot, assert_snapshot}; #[test] @@ -136,11 +122,11 @@ mod tests { #[test] fn test_parse_wrong_statement() { - assert_debug_snapshot!("salact".parse::()); + assert_debug_snapshot!(parse("".to_string(), "salact".to_string())); } #[test] fn test_parse_wrong_meta_command() { - assert_debug_snapshot!(".halp".parse::()); + assert_debug_snapshot!(parse("".to_string(), ".halp".to_string())); } } diff --git a/src/meta_commands.rs b/src/meta_commands.rs index e0d42c4..4c7b0f8 100644 --- a/src/meta_commands.rs +++ b/src/meta_commands.rs @@ -29,16 +29,3 @@ impl std::fmt::Display for MetaCommandParseError { } } } - -impl std::str::FromStr for MetaCommand { - type Err = MetaCommandParseError; - - fn from_str(s: &str) -> Result { - match s.trim() { - ".exit" => Ok(MetaCommand::Exit), - cmd => Err(MetaCommandParseError::Unrecognized { - cmd: cmd.to_string(), - }), - } - } -} diff --git a/src/snapshots/osdb__command__tests__parse_wrong_meta_command.snap b/src/snapshots/osdb__command__tests__parse_wrong_meta_command.snap index 556ca53..3bb3484 100644 --- a/src/snapshots/osdb__command__tests__parse_wrong_meta_command.snap +++ b/src/snapshots/osdb__command__tests__parse_wrong_meta_command.snap @@ -1,11 +1,20 @@ --- source: src/command.rs -expression: "\".halp\".parse::()" +expression: "parse(\"\".to_string(), \".halp\".to_string())" --- Err( - MetaCommand( - Unrecognized { - cmd: ".halp", - }, - ), + [ + Scan( + ScanError { + location: Location { + file: "", + offset: 0, + length: 5, + }, + kind: UnknownMetaCommand( + ".halp", + ), + }, + ), + ], ) diff --git a/src/snapshots/osdb__command__tests__parse_wrong_statement.snap b/src/snapshots/osdb__command__tests__parse_wrong_statement.snap index 90bd81a..25edfbe 100644 --- a/src/snapshots/osdb__command__tests__parse_wrong_statement.snap +++ b/src/snapshots/osdb__command__tests__parse_wrong_statement.snap @@ -1,11 +1,20 @@ --- source: src/command.rs -expression: "\"salact\".parse::()" +expression: "parse(\"\".to_string(), \"salact\".to_string())" --- Err( - Statement( - Unrecognized { - stmt: "salact", - }, - ), + [ + Scan( + ScanError { + location: Location { + file: "", + offset: 0, + length: 6, + }, + kind: UnknownKeyword( + "salact", + ), + }, + ), + ], ) diff --git a/src/snapshots/osdb__parser__tests__parse_exit.snap b/src/snapshots/osdb__parser__tests__parse_exit.snap deleted file mode 100644 index 4ca1f6e..0000000 --- a/src/snapshots/osdb__parser__tests__parse_exit.snap +++ /dev/null @@ -1,11 +0,0 @@ ---- -source: src/parser.rs -expression: "parse(file, String::from(\".exit\"))" ---- -Ok( - [ - MetaCommand( - Exit, - ), - ], -) diff --git a/src/snapshots/osdb__tokens__tests__tokenizer_errors.snap b/src/snapshots/osdb__tokens__tests__tokenizer_errors.snap new file mode 100644 index 0000000..b5886f1 --- /dev/null +++ b/src/snapshots/osdb__tokens__tests__tokenizer_errors.snap @@ -0,0 +1,26 @@ +--- +source: src/tokens.rs +expression: scanerrors +--- +[ + ScanError { + location: Location { + file: "src/statement.sql", + offset: 0, + length: 6, + }, + kind: UnknownKeyword( + "salact", + ), + }, + ScanError { + location: Location { + file: "src/statement.sql", + offset: 7, + length: 1, + }, + kind: UnexpectedChar( + '+', + ), + }, +] diff --git a/src/statements.rs b/src/statements.rs index 0e58d5f..238c02a 100644 --- a/src/statements.rs +++ b/src/statements.rs @@ -19,24 +19,6 @@ impl std::fmt::Display for StatementParseError { } } -impl std::str::FromStr for Statement { - type Err = StatementParseError; - - fn from_str(s: &str) -> Result { - let s = s.trim(); - let lower = s.to_lowercase(); - if lower.starts_with("insert") { - Ok(Statement::Insert) - } else if lower.starts_with("select") { - Ok(Statement::Select) - } else { - Err(StatementParseError::Unrecognized { - stmt: s.to_string(), - }) - } - } -} - pub struct StatementExecuteResult { pub msg: String, } diff --git a/src/tokens.rs b/src/tokens.rs index 86180fd..0c5c1f7 100644 --- a/src/tokens.rs +++ b/src/tokens.rs @@ -307,32 +307,9 @@ mod tests { #[test] fn test_tokenizer_errors() { - let mut scanerrors = tokenize("salact +".to_string(), "src/statement.sql".to_string()) + let scanerrors = tokenize("salact +".to_string(), "src/statement.sql".to_string()) .err() .unwrap(); - scanerrors.reverse(); - assert_eq!( - scanerrors.pop(), - Some(ScanError { - location: Location { - file: "src/statement.sql".to_string(), - offset: 0, - length: 6, - }, - kind: ScanErrorKind::UnknownKeyword("salact".to_string()), - }) - ); - assert_eq!( - scanerrors.pop(), - Some(ScanError { - location: Location { - file: "src/statement.sql".to_string(), - offset: 8, - length: 1, - }, - kind: ScanErrorKind::UnexpectedChar('+'), - }) - ); - assert!(scanerrors.is_empty()); + assert_debug_snapshot!(scanerrors); } } From 3d4ab2e2e4a6f2a600e655b5f347d07ec2d7c22a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kha=C3=AFs=20COLIN?= Date: Sun, 4 May 2025 18:36:43 +0200 Subject: [PATCH 5/6] feat(cli): use rustyline for cmd entry, which allows for richer editing --- Cargo.lock | 138 ++++++++++++++++++++++++++++++++++++++++++++++++++++- Cargo.toml | 1 + notes.org | 13 +++++ src/cli.rs | 21 +++----- 4 files changed, 158 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 35cd7e3..b904a80 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,10 +8,37 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "36f5e3dca4e09a6f340a61a0e9c7b61e030c69fc27bf29d73218f7e5e3b7638f" dependencies = [ - "unicode-width", + "unicode-width 0.1.14", "yansi", ] +[[package]] +name = "bitflags" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "clipboard-win" +version = "5.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15efe7a882b08f34e38556b14f2fb3daa98769d06c7f0c1b076dfd0d983bc892" +dependencies = [ + "error-code", +] + [[package]] name = "console" version = "0.15.11" @@ -30,6 +57,33 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" +[[package]] +name = "errno" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "976dd42dc7e85965fe702eb8164f21f450704bdde31faefd6471dba214cb594e" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "error-code" +version = "3.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d9305ccc6942a704f4335694ecd3de2ea531b114ac2d51f5f843750787a92f" + +[[package]] +name = "fd-lock" +version = "4.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce92ff622d6dadf7349484f42c93271a0d49b7cc4d466a936405bacbe10aa78" +dependencies = [ + "cfg-if", + "rustix", + "windows-sys", +] + [[package]] name = "insta" version = "1.43.1" @@ -47,6 +101,36 @@ version = "0.2.172" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" +[[package]] +name = "linux-raw-sys" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" + +[[package]] +name = "log" +version = "0.4.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" + +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + +[[package]] +name = "nix" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" +dependencies = [ + "bitflags", + "cfg-if", + "cfg_aliases", + "libc", +] + [[package]] name = "once_cell" version = "1.21.3" @@ -59,6 +143,40 @@ version = "0.1.0" dependencies = [ "ariadne", "insta", + "rustyline", +] + +[[package]] +name = "rustix" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c71e83d6afe7ff64890ec6b71d6a69bb8a610ab78ce364b3352876bb4c801266" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "rustyline" +version = "15.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ee1e066dc922e513bda599c6ccb5f3bb2b0ea5870a579448f2622993f0a9a2f" +dependencies = [ + "bitflags", + "cfg-if", + "clipboard-win", + "fd-lock", + "libc", + "log", + "memchr", + "nix", + "unicode-segmentation", + "unicode-width 0.2.0", + "utf8parse", + "windows-sys", ] [[package]] @@ -67,12 +185,30 @@ version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" +[[package]] +name = "unicode-segmentation" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" + [[package]] name = "unicode-width" version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" +[[package]] +name = "unicode-width" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "windows-sys" version = "0.59.0" diff --git a/Cargo.toml b/Cargo.toml index 6ba1a65..c5b0dbb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,7 @@ authors = ["Khaïs COLIN"] [dependencies] ariadne = "0.5.1" +rustyline = { version = "15.0.0", default-features = false, features = ["with-file-history"] } [dev-dependencies] insta = "1.43.1" diff --git a/notes.org b/notes.org index 439dc86..7420bf0 100644 --- a/notes.org +++ b/notes.org @@ -191,7 +191,20 @@ CLOCK: [2025-05-04 dim. 14:01]--[2025-05-04 dim. 14:14] => 0:13 * TODO use a better readline impl +** DONE inform myself on the different alternatives and decide on one +i will use rustyline, since it seems like the most feature-complete + +** DONE do the impl + +** TODO tweak it to make history work + * TODO handle non-interactive input better * TODO cli tests using insta-cmd https://insta.rs/docs/cmd/ + +* WAIT autocompletion +needs a more complicated parser for that to make sense + +* WAIT tweak rustyline it to make multiline entry work +need to terminate commands with semicolons for that to make sense diff --git a/src/cli.rs b/src/cli.rs index 4c8ee91..1323240 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -1,18 +1,11 @@ +use rustyline::DefaultEditor; + pub fn read_input() -> Option { - use std::io::{BufRead, Write}; + let mut rl = DefaultEditor::new().ok()?; - print!("osdb > "); - std::io::stdout().flush().expect("failed to flush stdout"); - - let mut input = String::new(); - let len = std::io::stdin() - .lock() - .read_line(&mut input) - .expect("failed to read input from stdin"); - - if len == 0 { - None - } else { - Some(input) + let readline = rl.readline("osdb> "); + match readline { + Ok(line) => Some(line), + Err(_) => None, } } From fe66326956c84a45c81987ff3f1a58f457e0ce86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kha=C3=AFs=20COLIN?= Date: Fri, 9 May 2025 17:16:05 +0200 Subject: [PATCH 6/6] refactor(get_command): spawn rustyline instance from main() --- notes.org | 10 +++++++++- src/cli.rs | 12 +++--------- src/main.rs | 7 ++++++- src/parser.rs | 6 +----- 4 files changed, 19 insertions(+), 16 deletions(-) diff --git a/notes.org b/notes.org index 7420bf0..6db9d88 100644 --- a/notes.org +++ b/notes.org @@ -196,7 +196,15 @@ i will use rustyline, since it seems like the most feature-complete ** DONE do the impl -** TODO tweak it to make history work +** TODO make history work + +*** DONE have the rl instance be spawned from main + +*** TODO figure out how to locate the app data directory on linux + +*** TODO create our own app data directory + +*** TODO load and save the history from a file in this directory * TODO handle non-interactive input better diff --git a/src/cli.rs b/src/cli.rs index 1323240..e44b26f 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -1,11 +1,5 @@ -use rustyline::DefaultEditor; +use rustyline::{Editor, history::FileHistory}; -pub fn read_input() -> Option { - let mut rl = DefaultEditor::new().ok()?; - - let readline = rl.readline("osdb> "); - match readline { - Ok(line) => Some(line), - Err(_) => None, - } +pub fn read_input(rl: &mut Editor<(), FileHistory>) -> Option { + rl.readline("osdb> ").ok() } diff --git a/src/main.rs b/src/main.rs index bac5cd6..6691c8f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -4,9 +4,13 @@ use osdb::error_display::OSDBError as _; use osdb::parser::parse; fn main() { + let mut rl = rustyline::DefaultEditor::new().expect("failed to create stdin reader"); + println!("{}", startup_msg()); - 'main: while let Some(input) = read_input() { + + 'main: while let Some(input) = read_input(&mut rl) { let file = String::from(""); + match parse(file.clone(), input.clone()) { Ok(cmds) => { for cmd in cmds { @@ -24,5 +28,6 @@ fn main() { } } } + println!("Good-bye"); } diff --git a/src/parser.rs b/src/parser.rs index 949a1bc..98325d8 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -22,11 +22,7 @@ pub fn parse(file: String, input: String) -> Result, Vec (), } } - if errs.is_empty() { - Ok(cmds) - } else { - Err(errs) - } + if errs.is_empty() { Ok(cmds) } else { Err(errs) } } #[cfg(test)]