refactor(get_command): spawn rustyline instance from main()

Cargo.lock

@@ -8,10 +8,37 @@ version = "0.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "36f5e3dca4e09a6f340a61a0e9c7b61e030c69fc27bf29d73218f7e5e3b7638f"
 dependencies = [
- "unicode-width",
+ "unicode-width 0.1.14",
  "yansi",
 ]
 
+[[package]]
+name = "bitflags"
+version = "2.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd"
+
+[[package]]
+name = "cfg-if"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
+
+[[package]]
+name = "cfg_aliases"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
+
+[[package]]
+name = "clipboard-win"
+version = "5.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "15efe7a882b08f34e38556b14f2fb3daa98769d06c7f0c1b076dfd0d983bc892"
+dependencies = [
+ "error-code",
+]
+
 [[package]]
 name = "console"
 version = "0.15.11"
@@ -30,6 +57,33 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0"
 
+[[package]]
+name = "errno"
+version = "0.3.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "976dd42dc7e85965fe702eb8164f21f450704bdde31faefd6471dba214cb594e"
+dependencies = [
+ "libc",
+ "windows-sys",
+]
+
+[[package]]
+name = "error-code"
+version = "3.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a5d9305ccc6942a704f4335694ecd3de2ea531b114ac2d51f5f843750787a92f"
+
+[[package]]
+name = "fd-lock"
+version = "4.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0ce92ff622d6dadf7349484f42c93271a0d49b7cc4d466a936405bacbe10aa78"
+dependencies = [
+ "cfg-if",
+ "rustix",
+ "windows-sys",
+]
+
 [[package]]
 name = "insta"
 version = "1.43.1"
@@ -47,6 +101,36 @@ version = "0.2.172"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa"
 
+[[package]]
+name = "linux-raw-sys"
+version = "0.9.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12"
+
+[[package]]
+name = "log"
+version = "0.4.27"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94"
+
+[[package]]
+name = "memchr"
+version = "2.7.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
+
+[[package]]
+name = "nix"
+version = "0.29.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46"
+dependencies = [
+ "bitflags",
+ "cfg-if",
+ "cfg_aliases",
+ "libc",
+]
+
 [[package]]
 name = "once_cell"
 version = "1.21.3"
@@ -59,6 +143,40 @@ version = "0.1.0"
 dependencies = [
  "ariadne",
  "insta",
+ "rustyline",
+]
+
+[[package]]
+name = "rustix"
+version = "1.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c71e83d6afe7ff64890ec6b71d6a69bb8a610ab78ce364b3352876bb4c801266"
+dependencies = [
+ "bitflags",
+ "errno",
+ "libc",
+ "linux-raw-sys",
+ "windows-sys",
+]
+
+[[package]]
+name = "rustyline"
+version = "15.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2ee1e066dc922e513bda599c6ccb5f3bb2b0ea5870a579448f2622993f0a9a2f"
+dependencies = [
+ "bitflags",
+ "cfg-if",
+ "clipboard-win",
+ "fd-lock",
+ "libc",
+ "log",
+ "memchr",
+ "nix",
+ "unicode-segmentation",
+ "unicode-width 0.2.0",
+ "utf8parse",
+ "windows-sys",
 ]
 
 [[package]]
@@ -67,12 +185,30 @@ version = "2.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa"
 
+[[package]]
+name = "unicode-segmentation"
+version = "1.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493"
+
 [[package]]
 name = "unicode-width"
 version = "0.1.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af"
 
+[[package]]
+name = "unicode-width"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd"
+
+[[package]]
+name = "utf8parse"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
+
 [[package]]
 name = "windows-sys"
 version = "0.59.0"

Cargo.toml

@@ -6,6 +6,7 @@ authors = ["Khaïs COLIN"]
 
 [dependencies]
 ariadne = "0.5.1"
+rustyline = { version = "15.0.0", default-features = false, features = ["with-file-history"] }
 
 [dev-dependencies]
 insta = "1.43.1"

notes.org

@@ -1,4 +1,5 @@
 #+title: Notes
+#+property: EFFORT_ALL 0 10
 
 * DONE show errors with ariadne
 :PROPERTIES:
@@ -120,23 +121,15 @@ CLOCK: [2025-05-03 sam. 21:21]--[2025-05-03 sam. 21:22] =>  0:01
 CLOCK: [2025-05-03 sam. 19:06]--[2025-05-03 sam. 19:07] =>  0:01
 :END:
 
-* TODO switch statement parsing to more extensible token-based algorithm
+* DONE switch statement parsing to more extensible token-based algorithm
 :PROPERTIES:
 :EFFORT:   10
 :END:
 :LOGBOOK:
 CLOCK: [2025-05-04 dim. 12:07]--[2025-05-04 dim. 12:10] =>  0:03
-:END:
-
-** TODO use tokens to parse meta-commands
-:PROPERTIES:
-:EFFORT:   10
-:END:
-:LOGBOOK:
 CLOCK: [2025-05-04 dim. 12:10]--[2025-05-04 dim. 12:22] =>  0:12
 :END:
-
+** DONE recognize meta-commands as tokens
-*** DONE recognize meta-commands as tokens
 :PROPERTIES:
 :EFFORT:   10
 :END:
@@ -145,7 +138,7 @@ CLOCK: [2025-05-04 dim. 13:32]--[2025-05-04 dim. 13:35] =>  0:03
 CLOCK: [2025-05-04 dim. 13:27]--[2025-05-04 dim. 13:32] =>  0:05
 :END:
 
-*** DONE CommandParseError must have a ScanError variant with an Into impl
+** DONE CommandParseError must have a ScanError variant with an Into impl
 :PROPERTIES:
 :EFFORT:   10
 :END:
@@ -153,12 +146,12 @@ CLOCK: [2025-05-04 dim. 13:27]--[2025-05-04 dim. 13:32] =>  0:05
 CLOCK: [2025-05-04 dim. 13:35]--[2025-05-04 dim. 13:38] =>  0:03
 :END:
 
-*** DONE ScanErrors must be convertible to ariadne reports
+** DONE ScanErrors must be convertible to ariadne reports
 :PROPERTIES:
 :EFFORT:   10
 :END:
 
-**** DONE Remove the CommandParseError Display implementation
+*** DONE Remove the CommandParseError Display implementation
 :PROPERTIES:
 :EFFORT:   10
 :END:
@@ -166,7 +159,7 @@ CLOCK: [2025-05-04 dim. 13:35]--[2025-05-04 dim. 13:38] =>  0:03
 CLOCK: [2025-05-04 dim. 13:38]--[2025-05-04 dim. 13:44] =>  0:06
 :END:
 
-**** DONE implement OSDBError for ScanError
+*** DONE implement OSDBError for ScanError
 :PROPERTIES:
 :EFFORT:   10
 :END:
@@ -174,17 +167,52 @@ CLOCK: [2025-05-04 dim. 13:38]--[2025-05-04 dim. 13:44] =>  0:06
 CLOCK: [2025-05-04 dim. 13:45]--[2025-05-04 dim. 13:56] =>  0:11
 :END:
 
-*** DONE remove token types which are not recognized at all
+** DONE remove token types which are not recognized at all
 :PROPERTIES:
 :EFFORT:   10
 :END:
 
-*** TODO parse tokens into meta-commands
+** DONE create a generic parse command that parses string into tokens into Command
+:PROPERTIES:
+:EFFORT:   10
+:END:
+:LOGBOOK:
+CLOCK: [2025-05-04 dim. 14:01]--[2025-05-04 dim. 14:14] =>  0:13
+:END:
+
+** DONE parse tokens into meta-commands
 :PROPERTIES:
 :EFFORT:   10
 :END:
 
-** TODO use tokens to parse statements
+* DONE error offsets are incorrect
-:PROPERTIES:
+
-:EFFORT:
+* DONE remove old FromStr parser implementation
-:END:
+
+* TODO use a better readline impl
+
+** DONE inform myself on the different alternatives and decide on one
+i will use rustyline, since it seems like the most feature-complete
+
+** DONE do the impl
+
+** TODO make history work
+
+*** DONE have the rl instance be spawned from main
+
+*** TODO figure out how to locate the app data directory on linux
+
+*** TODO create our own app data directory
+
+*** TODO load and save the history from a file in this directory
+
+* TODO handle non-interactive input better
+
+* TODO cli tests using insta-cmd
+https://insta.rs/docs/cmd/
+
+* WAIT autocompletion
+needs a more complicated parser for that to make sense
+
+* WAIT tweak rustyline it to make multiline entry work
+need to terminate commands with semicolons for that to make sense

src/cli.rs

@@ -1,18 +1,5 @@
-pub fn read_input() -> Option<String> {
+use rustyline::{Editor, history::FileHistory};
-    use std::io::{BufRead, Write};
 
-    print!("osdb > ");
+pub fn read_input(rl: &mut Editor<(), FileHistory>) -> Option<String> {
-    std::io::stdout().flush().expect("failed to flush stdout");
+    rl.readline("osdb> ").ok()
-
-    let mut input = String::new();
-    let len = std::io::stdin()
-        .lock()
-        .read_line(&mut input)
-        .expect("failed to read input from stdin");
-
-    if len == 0 {
-        None
-    } else {
-        Some(input)
-    }
 }

src/command.rs

@@ -94,25 +94,11 @@ impl From<ScanError> for CommandParseError {
     }
 }
 
-impl std::str::FromStr for Command {
-    type Err = CommandParseError;
-
-    fn from_str(s: &str) -> Result<Self, Self::Err> {
-        if s.starts_with(".") {
-            s.parse::<MetaCommand>()
-                .map(|x| x.into())
-                .map_err(|x| x.into())
-        } else {
-            s.parse::<Statement>()
-                .map(|x| x.into())
-                .map_err(|x| x.into())
-        }
-    }
-}
-
 #[cfg(test)]
 mod tests {
-    use crate::{command::Command, meta_commands::MetaCommand, statements::Statement};
+    use crate::{
+        command::Command, meta_commands::MetaCommand, parser::parse, statements::Statement,
+    };
     use insta::{assert_debug_snapshot, assert_snapshot};
 
     #[test]
@@ -136,11 +122,11 @@ mod tests {
 
     #[test]
     fn test_parse_wrong_statement() {
-        assert_debug_snapshot!("salact".parse::<Command>());
+        assert_debug_snapshot!(parse("<stdin>".to_string(), "salact".to_string()));
     }
 
     #[test]
     fn test_parse_wrong_meta_command() {
-        assert_debug_snapshot!(".halp".parse::<Command>());
+        assert_debug_snapshot!(parse("<stdin>".to_string(), ".halp".to_string()));
     }
 }

src/lib.rs

@@ -3,5 +3,6 @@ pub mod cli;
 pub mod command;
 pub mod error_display;
 pub mod meta_commands;
+pub mod parser;
 pub mod statements;
 pub mod tokens;

src/main.rs

@@ -1,21 +1,33 @@
 use osdb::branding::startup_msg;
 use osdb::cli::read_input;
-use osdb::command::Command;
 use osdb::error_display::OSDBError as _;
+use osdb::parser::parse;
 
 fn main() {
+    let mut rl = rustyline::DefaultEditor::new().expect("failed to create stdin reader");
+
     println!("{}", startup_msg());
-    while let Some(input) = read_input() {
+
-        match input.parse::<Command>() {
+    'main: while let Some(input) = read_input(&mut rl) {
-            Ok(cmd) => {
+        let file = String::from("<stdin>");
-                let result = cmd.execute();
+
-                println!("{}", result.display());
+        match parse(file.clone(), input.clone()) {
-                if result.should_exit {
+            Ok(cmds) => {
-                    break;
+                for cmd in cmds {
+                    let result = cmd.execute();
+                    if result.should_exit {
+                        break 'main;
+                    }
+                    println!("{}", result.display());
+                }
+            }
+            Err(errs) => {
+                for err in errs {
+                    err.display(&file, &input)
                 }
             }
-            Err(err) => err.display("<stdin>", &input),
         }
     }
+
     println!("Good-bye");
 }

src/meta_commands.rs

@@ -29,16 +29,3 @@ impl std::fmt::Display for MetaCommandParseError {
         }
     }
 }
-
-impl std::str::FromStr for MetaCommand {
-    type Err = MetaCommandParseError;
-
-    fn from_str(s: &str) -> Result<Self, Self::Err> {
-        match s.trim() {
-            ".exit" => Ok(MetaCommand::Exit),
-            cmd => Err(MetaCommandParseError::Unrecognized {
-                cmd: cmd.to_string(),
-            }),
-        }
-    }
-}

src/parser.rs

@@ -0,0 +1,79 @@
+use std::collections::VecDeque;
+
+use crate::{
+    command::{Command, CommandParseError},
+    statements::Statement,
+    tokens::tokenize,
+};
+
+pub fn parse(file: String, input: String) -> Result<Vec<Command>, Vec<CommandParseError>> {
+    let mut tokens: VecDeque<_> = tokenize(input, file)
+        .map_err(|x| x.into_iter().map(|x| x.into()).collect::<Vec<_>>())?
+        .into();
+    let mut cmds = Vec::new();
+    let errs = Vec::new();
+    while let Some(token) = tokens.pop_front() {
+        match token.data {
+            crate::tokens::TokenData::Insert => cmds.push(Command::Statement(Statement::Insert)),
+            crate::tokens::TokenData::Select => cmds.push(Command::Statement(Statement::Select)),
+            crate::tokens::TokenData::MetaCommand(meta_command) => {
+                cmds.push(Command::MetaCommand(meta_command))
+            }
+            crate::tokens::TokenData::EndOfFile => (),
+        }
+    }
+    if errs.is_empty() { Ok(cmds) } else { Err(errs) }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use insta::assert_debug_snapshot;
+
+    #[test]
+    fn test_parse_single_correct() {
+        let file = String::from("<stdin>");
+        assert_debug_snapshot!(parse(file.clone(), String::from(".exit")));
+        assert_debug_snapshot!(parse(file.clone(), String::from("select")));
+        assert_debug_snapshot!(parse(file.clone(), String::from("sElEcT")));
+        assert_debug_snapshot!(parse(file.clone(), String::from("INSERT")));
+        assert_debug_snapshot!(parse(file.clone(), String::from("InSErT")));
+    }
+
+    #[test]
+    fn test_parse_single_incorrect() {
+        let file = String::from("<stdin>");
+        assert_debug_snapshot!(parse(file.clone(), String::from(".halp")));
+        assert_debug_snapshot!(parse(file.clone(), String::from("salect")));
+        assert_debug_snapshot!(parse(file.clone(), String::from("sAlEcT")));
+        assert_debug_snapshot!(parse(file.clone(), String::from("INSART")));
+        assert_debug_snapshot!(parse(file.clone(), String::from("InSArT")));
+    }
+
+    #[test]
+    fn test_parse_multiple_correct() {
+        let file = String::from("<stdin>");
+        assert_debug_snapshot!(parse(
+            file.clone(),
+            String::from(".exit select select insert select")
+        ));
+    }
+
+    #[test]
+    fn test_parse_multiple_incorrect() {
+        let file = String::from("<stdin>");
+        assert_debug_snapshot!(parse(
+            file.clone(),
+            String::from(".halp salect selact inset seiect")
+        ));
+    }
+
+    #[test]
+    fn test_parse_multiple_mixed() {
+        let file = String::from("<stdin>");
+        assert_debug_snapshot!(parse(
+            file.clone(),
+            String::from(".exit selct select nsert select")
+        ));
+    }
+}

src/snapshots/osdb__command__tests__parse_wrong_meta_command.snap

@@ -1,11 +1,20 @@
 ---
 source: src/command.rs
-expression: "\".halp\".parse::<Command>()"
+expression: "parse(\"<stdin>\".to_string(), \".halp\".to_string())"
 ---
 Err(
-    MetaCommand(
+    [
-        Unrecognized {
+        Scan(
-            cmd: ".halp",
+            ScanError {
-        },
+                location: Location {
-    ),
+                    file: "<stdin>",
+                    offset: 0,
+                    length: 5,
+                },
+                kind: UnknownMetaCommand(
+                    ".halp",
+                ),
+            },
+        ),
+    ],
 )

src/snapshots/osdb__command__tests__parse_wrong_statement.snap

@@ -1,11 +1,20 @@
 ---
 source: src/command.rs
-expression: "\"salact\".parse::<Command>()"
+expression: "parse(\"<stdin>\".to_string(), \"salact\".to_string())"
 ---
 Err(
-    Statement(
+    [
-        Unrecognized {
+        Scan(
-            stmt: "salact",
+            ScanError {
-        },
+                location: Location {
-    ),
+                    file: "<stdin>",
+                    offset: 0,
+                    length: 6,
+                },
+                kind: UnknownKeyword(
+                    "salact",
+                ),
+            },
+        ),
+    ],
 )

src/snapshots/osdb__parser__tests__parse_multiple_correct.snap

@@ -0,0 +1,23 @@
+---
+source: src/parser.rs
+expression: "parse(file.clone(), String::from(\".exit select select insert select\"))"
+---
+Ok(
+    [
+        MetaCommand(
+            Exit,
+        ),
+        Statement(
+            Select,
+        ),
+        Statement(
+            Select,
+        ),
+        Statement(
+            Insert,
+        ),
+        Statement(
+            Select,
+        ),
+    ],
+)

src/snapshots/osdb__parser__tests__parse_multiple_incorrect.snap

@@ -0,0 +1,68 @@
+---
+source: src/parser.rs
+expression: "parse(file.clone(), String::from(\".halp salect selact inset seiect\"))"
+---
+Err(
+    [
+        Scan(
+            ScanError {
+                location: Location {
+                    file: "<stdin>",
+                    offset: 0,
+                    length: 5,
+                },
+                kind: UnknownMetaCommand(
+                    ".halp",
+                ),
+            },
+        ),
+        Scan(
+            ScanError {
+                location: Location {
+                    file: "<stdin>",
+                    offset: 6,
+                    length: 6,
+                },
+                kind: UnknownKeyword(
+                    "salect",
+                ),
+            },
+        ),
+        Scan(
+            ScanError {
+                location: Location {
+                    file: "<stdin>",
+                    offset: 13,
+                    length: 6,
+                },
+                kind: UnknownKeyword(
+                    "selact",
+                ),
+            },
+        ),
+        Scan(
+            ScanError {
+                location: Location {
+                    file: "<stdin>",
+                    offset: 20,
+                    length: 5,
+                },
+                kind: UnknownKeyword(
+                    "inset",
+                ),
+            },
+        ),
+        Scan(
+            ScanError {
+                location: Location {
+                    file: "<stdin>",
+                    offset: 26,
+                    length: 6,
+                },
+                kind: UnknownKeyword(
+                    "seiect",
+                ),
+            },
+        ),
+    ],
+)

src/snapshots/osdb__parser__tests__parse_multiple_mixed.snap

@@ -0,0 +1,32 @@
+---
+source: src/parser.rs
+expression: "parse(file.clone(), String::from(\".exit selct select nsert select\"))"
+---
+Err(
+    [
+        Scan(
+            ScanError {
+                location: Location {
+                    file: "<stdin>",
+                    offset: 6,
+                    length: 5,
+                },
+                kind: UnknownKeyword(
+                    "selct",
+                ),
+            },
+        ),
+        Scan(
+            ScanError {
+                location: Location {
+                    file: "<stdin>",
+                    offset: 19,
+                    length: 5,
+                },
+                kind: UnknownKeyword(
+                    "nsert",
+                ),
+            },
+        ),
+    ],
+)

src/snapshots/osdb__parser__tests__parse_single_correct-2.snap

@@ -0,0 +1,11 @@
+---
+source: src/parser.rs
+expression: "parse(file.clone(), String::from(\"select\"))"
+---
+Ok(
+    [
+        Statement(
+            Select,
+        ),
+    ],
+)

src/snapshots/osdb__parser__tests__parse_single_correct-3.snap

@@ -0,0 +1,11 @@
+---
+source: src/parser.rs
+expression: "parse(file.clone(), String::from(\"sElEcT\"))"
+---
+Ok(
+    [
+        Statement(
+            Select,
+        ),
+    ],
+)

src/snapshots/osdb__parser__tests__parse_single_correct-4.snap

@@ -0,0 +1,11 @@
+---
+source: src/parser.rs
+expression: "parse(file.clone(), String::from(\"INSERT\"))"
+---
+Ok(
+    [
+        Statement(
+            Insert,
+        ),
+    ],
+)

src/snapshots/osdb__parser__tests__parse_single_correct-5.snap

@@ -0,0 +1,11 @@
+---
+source: src/parser.rs
+expression: "parse(file.clone(), String::from(\"InSErT\"))"
+---
+Ok(
+    [
+        Statement(
+            Insert,
+        ),
+    ],
+)

src/snapshots/osdb__parser__tests__parse_single_correct.snap

@@ -0,0 +1,11 @@
+---
+source: src/parser.rs
+expression: "parse(file.clone(), String::from(\".exit\"))"
+---
+Ok(
+    [
+        MetaCommand(
+            Exit,
+        ),
+    ],
+)

src/snapshots/osdb__parser__tests__parse_single_incorrect-2.snap

@@ -0,0 +1,20 @@
+---
+source: src/parser.rs
+expression: "parse(file.clone(), String::from(\"salect\"))"
+---
+Err(
+    [
+        Scan(
+            ScanError {
+                location: Location {
+                    file: "<stdin>",
+                    offset: 0,
+                    length: 6,
+                },
+                kind: UnknownKeyword(
+                    "salect",
+                ),
+            },
+        ),
+    ],
+)

src/snapshots/osdb__parser__tests__parse_single_incorrect-3.snap

@@ -0,0 +1,20 @@
+---
+source: src/parser.rs
+expression: "parse(file.clone(), String::from(\"sAlEcT\"))"
+---
+Err(
+    [
+        Scan(
+            ScanError {
+                location: Location {
+                    file: "<stdin>",
+                    offset: 0,
+                    length: 6,
+                },
+                kind: UnknownKeyword(
+                    "sAlEcT",
+                ),
+            },
+        ),
+    ],
+)

src/snapshots/osdb__parser__tests__parse_single_incorrect-4.snap

@@ -0,0 +1,20 @@
+---
+source: src/parser.rs
+expression: "parse(file.clone(), String::from(\"INSART\"))"
+---
+Err(
+    [
+        Scan(
+            ScanError {
+                location: Location {
+                    file: "<stdin>",
+                    offset: 0,
+                    length: 6,
+                },
+                kind: UnknownKeyword(
+                    "INSART",
+                ),
+            },
+        ),
+    ],
+)

src/snapshots/osdb__parser__tests__parse_single_incorrect-5.snap

@@ -0,0 +1,20 @@
+---
+source: src/parser.rs
+expression: "parse(file.clone(), String::from(\"InSArT\"))"
+---
+Err(
+    [
+        Scan(
+            ScanError {
+                location: Location {
+                    file: "<stdin>",
+                    offset: 0,
+                    length: 6,
+                },
+                kind: UnknownKeyword(
+                    "InSArT",
+                ),
+            },
+        ),
+    ],
+)

src/snapshots/osdb__parser__tests__parse_single_incorrect.snap

@@ -0,0 +1,20 @@
+---
+source: src/parser.rs
+expression: "parse(file.clone(), String::from(\".halp\"))"
+---
+Err(
+    [
+        Scan(
+            ScanError {
+                location: Location {
+                    file: "<stdin>",
+                    offset: 0,
+                    length: 5,
+                },
+                kind: UnknownMetaCommand(
+                    ".halp",
+                ),
+            },
+        ),
+    ],
+)

src/snapshots/osdb__tokens__tests__tokenizer_errors.snap

@@ -0,0 +1,26 @@
+---
+source: src/tokens.rs
+expression: scanerrors
+---
+[
+    ScanError {
+        location: Location {
+            file: "src/statement.sql",
+            offset: 0,
+            length: 6,
+        },
+        kind: UnknownKeyword(
+            "salact",
+        ),
+    },
+    ScanError {
+        location: Location {
+            file: "src/statement.sql",
+            offset: 7,
+            length: 1,
+        },
+        kind: UnexpectedChar(
+            '+',
+        ),
+    },
+]

src/statements.rs

@@ -19,24 +19,6 @@ impl std::fmt::Display for StatementParseError {
     }
 }
 
-impl std::str::FromStr for Statement {
-    type Err = StatementParseError;
-
-    fn from_str(s: &str) -> Result<Self, Self::Err> {
-        let s = s.trim();
-        let lower = s.to_lowercase();
-        if lower.starts_with("insert") {
-            Ok(Statement::Insert)
-        } else if lower.starts_with("select") {
-            Ok(Statement::Select)
-        } else {
-            Err(StatementParseError::Unrecognized {
-                stmt: s.to_string(),
-            })
-        }
-    }
-}
-
 pub struct StatementExecuteResult {
     pub msg: String,
 }

src/tokens.rs

@@ -206,27 +206,25 @@ impl Tokenizer {
         c.is_alphanumeric() || c == '_'
     }
 
-    fn scan_token(&mut self) -> Result<Token, ScanError> {
+    fn scan_token(&mut self) -> Result<Option<Token>, ScanError> {
         loop {
             if let Some(c) = self.peek() {
                 if Self::ident_or_keyword_start(c) {
-                    return self.scan_identifier_or_keyword();
+                    return self.scan_identifier_or_keyword().map(Some);
                 } else if c == '.' {
-                    return self.scan_meta_command();
+                    return self.scan_meta_command().map(Some);
                 } else if c.is_whitespace() {
                     self.advance();
                 } else {
-                    self.advance();
+                    let result = Err(ScanError {
-                    return Err(ScanError {
                         location: self.current_location(1),
                         kind: ScanErrorKind::UnexpectedChar(c),
                     });
+                    self.advance();
+                    return result;
                 }
             } else {
-                return Err(ScanError {
+                return Ok(None);
-                    location: self.current_location(0),
-                    kind: ScanErrorKind::UnexpectedEndOfInput,
-                });
             }
         }
     }
@@ -244,8 +242,10 @@ pub fn tokenize(input: String, file: String) -> Result<Vec<Token>, Vec<ScanError
     let mut tokenizer = Tokenizer::new(input, file);
     let mut errors = Vec::new();
     while !tokenizer.is_at_end() {
-        match tokenizer.scan_token() {
+        let token = tokenizer.scan_token();
-            Ok(token) => tokenizer.tokens.push(token),
+        match token {
+            Ok(Some(token)) => tokenizer.tokens.push(token),
+            Ok(None) => break,
             Err(err) => errors.push(err),
         }
     }
@@ -307,32 +307,9 @@ mod tests {
 
     #[test]
     fn test_tokenizer_errors() {
-        let mut scanerrors = tokenize("salact +".to_string(), "src/statement.sql".to_string())
+        let scanerrors = tokenize("salact +".to_string(), "src/statement.sql".to_string())
             .err()
             .unwrap();
-        scanerrors.reverse();
+        assert_debug_snapshot!(scanerrors);
-        assert_eq!(
-            scanerrors.pop(),
-            Some(ScanError {
-                location: Location {
-                    file: "src/statement.sql".to_string(),
-                    offset: 0,
-                    length: 6,
-                },
-                kind: ScanErrorKind::UnknownKeyword("salact".to_string()),
-            })
-        );
-        assert_eq!(
-            scanerrors.pop(),
-            Some(ScanError {
-                location: Location {
-                    file: "src/statement.sql".to_string(),
-                    offset: 8,
-                    length: 1,
-                },
-                kind: ScanErrorKind::UnexpectedChar('+'),
-            })
-        );
-        assert!(scanerrors.is_empty());
     }
 }