feat(parser): use token-based parser

notes.org

@@ -1,4 +1,5 @@
 #+title: Notes
+#+property: EFFORT_ALL 0 10
 
 * DONE show errors with ariadne
 :PROPERTIES:
@@ -120,23 +121,15 @@ CLOCK: [2025-05-03 sam. 21:21]--[2025-05-03 sam. 21:22] =>  0:01
 CLOCK: [2025-05-03 sam. 19:06]--[2025-05-03 sam. 19:07] =>  0:01
 :END:
 
-* TODO switch statement parsing to more extensible token-based algorithm
+* DONE switch statement parsing to more extensible token-based algorithm
 :PROPERTIES:
 :EFFORT:   10
 :END:
 :LOGBOOK:
 CLOCK: [2025-05-04 dim. 12:07]--[2025-05-04 dim. 12:10] =>  0:03
-:END:
-
-** TODO use tokens to parse meta-commands
-:PROPERTIES:
-:EFFORT:   10
-:END:
-:LOGBOOK:
 CLOCK: [2025-05-04 dim. 12:10]--[2025-05-04 dim. 12:22] =>  0:12
 :END:
-
+** DONE recognize meta-commands as tokens
-*** DONE recognize meta-commands as tokens
 :PROPERTIES:
 :EFFORT:   10
 :END:
@@ -145,7 +138,7 @@ CLOCK: [2025-05-04 dim. 13:32]--[2025-05-04 dim. 13:35] =>  0:03
 CLOCK: [2025-05-04 dim. 13:27]--[2025-05-04 dim. 13:32] =>  0:05
 :END:
 
-*** DONE CommandParseError must have a ScanError variant with an Into impl
+** DONE CommandParseError must have a ScanError variant with an Into impl
 :PROPERTIES:
 :EFFORT:   10
 :END:
@@ -153,12 +146,12 @@ CLOCK: [2025-05-04 dim. 13:27]--[2025-05-04 dim. 13:32] =>  0:05
 CLOCK: [2025-05-04 dim. 13:35]--[2025-05-04 dim. 13:38] =>  0:03
 :END:
 
-*** DONE ScanErrors must be convertible to ariadne reports
+** DONE ScanErrors must be convertible to ariadne reports
 :PROPERTIES:
 :EFFORT:   10
 :END:
 
-**** DONE Remove the CommandParseError Display implementation
+*** DONE Remove the CommandParseError Display implementation
 :PROPERTIES:
 :EFFORT:   10
 :END:
@@ -166,7 +159,7 @@ CLOCK: [2025-05-04 dim. 13:35]--[2025-05-04 dim. 13:38] =>  0:03
 CLOCK: [2025-05-04 dim. 13:38]--[2025-05-04 dim. 13:44] =>  0:06
 :END:
 
-**** DONE implement OSDBError for ScanError
+*** DONE implement OSDBError for ScanError
 :PROPERTIES:
 :EFFORT:   10
 :END:
@@ -174,12 +167,12 @@ CLOCK: [2025-05-04 dim. 13:38]--[2025-05-04 dim. 13:44] =>  0:06
 CLOCK: [2025-05-04 dim. 13:45]--[2025-05-04 dim. 13:56] =>  0:11
 :END:
 
-*** DONE remove token types which are not recognized at all
+** DONE remove token types which are not recognized at all
 :PROPERTIES:
 :EFFORT:   10
 :END:
 
-*** DONE create a generic parse command that parses string into tokens into Command
+** DONE create a generic parse command that parses string into tokens into Command
 :PROPERTIES:
 :EFFORT:   10
 :END:
@@ -187,12 +180,11 @@ CLOCK: [2025-05-04 dim. 13:45]--[2025-05-04 dim. 13:56] =>  0:11
 CLOCK: [2025-05-04 dim. 14:01]--[2025-05-04 dim. 14:14] =>  0:13
 :END:
 
-*** TODO parse tokens into meta-commands
+** DONE parse tokens into meta-commands
 :PROPERTIES:
 :EFFORT:   10
 :END:
 
-** TODO use tokens to parse statements
+* TODO error offsets are incorrect
-:PROPERTIES:
+
-:EFFORT:
+* TODO remove old FromStr parser implementation
-:END:

src/main.rs

@@ -1,20 +1,27 @@
 use osdb::branding::startup_msg;
 use osdb::cli::read_input;
-use osdb::command::Command;
 use osdb::error_display::OSDBError as _;
+use osdb::parser::parse;
 
 fn main() {
     println!("{}", startup_msg());
-    while let Some(input) = read_input() {
+    'main: while let Some(input) = read_input() {
-        match input.parse::<Command>() {
+        let file = String::from("<stdin>");
-            Ok(cmd) => {
+        match parse(file.clone(), input.clone()) {
-                let result = cmd.execute();
+            Ok(cmds) => {
-                println!("{}", result.display());
+                for cmd in cmds {
-                if result.should_exit {
+                    let result = cmd.execute();
-                    break;
+                    if result.should_exit {
+                        break 'main;
+                    }
+                    println!("{}", result.display());
+                }
+            }
+            Err(errs) => {
+                for err in errs {
+                    err.display(&file, &input)
                 }
             }
-            Err(err) => err.display("<stdin>", &input),
         }
     }
     println!("Good-bye");

src/tokens.rs

@@ -206,13 +206,13 @@ impl Tokenizer {
         c.is_alphanumeric() || c == '_'
     }
 
-    fn scan_token(&mut self) -> Result<Token, ScanError> {
+    fn scan_token(&mut self) -> Result<Option<Token>, ScanError> {
         loop {
             if let Some(c) = self.peek() {
                 if Self::ident_or_keyword_start(c) {
-                    return self.scan_identifier_or_keyword();
+                    return self.scan_identifier_or_keyword().map(Some);
                 } else if c == '.' {
-                    return self.scan_meta_command();
+                    return self.scan_meta_command().map(Some);
                 } else if c.is_whitespace() {
                     self.advance();
                 } else {
@@ -223,10 +223,7 @@ impl Tokenizer {
                     });
                 }
             } else {
-                return Err(ScanError {
+                return Ok(None);
-                    location: self.current_location(0),
-                    kind: ScanErrorKind::UnexpectedEndOfInput,
-                });
             }
         }
     }
@@ -244,8 +241,10 @@ pub fn tokenize(input: String, file: String) -> Result<Vec<Token>, Vec<ScanError
     let mut tokenizer = Tokenizer::new(input, file);
     let mut errors = Vec::new();
     while !tokenizer.is_at_end() {
-        match tokenizer.scan_token() {
+        let token = tokenizer.scan_token();
-            Ok(token) => tokenizer.tokens.push(token),
+        match token {
+            Ok(Some(token)) => tokenizer.tokens.push(token),
+            Ok(None) => break,
             Err(err) => errors.push(err),
         }
     }