feat(parser): use token-based parser

src/main.rs

@@ -1,20 +1,27 @@
 use osdb::branding::startup_msg;
 use osdb::cli::read_input;
-use osdb::command::Command;
 use osdb::error_display::OSDBError as _;
+use osdb::parser::parse;
 
 fn main() {
     println!("{}", startup_msg());
-    while let Some(input) = read_input() {
-        match input.parse::<Command>() {
-            Ok(cmd) => {
-                let result = cmd.execute();
-                println!("{}", result.display());
-                if result.should_exit {
-                    break;
+    'main: while let Some(input) = read_input() {
+        let file = String::from("<stdin>");
+        match parse(file.clone(), input.clone()) {
+            Ok(cmds) => {
+                for cmd in cmds {
+                    let result = cmd.execute();
+                    if result.should_exit {
+                        break 'main;
+                    }
+                    println!("{}", result.display());
+                }
+            }
+            Err(errs) => {
+                for err in errs {
+                    err.display(&file, &input)
                 }
             }
-            Err(err) => err.display("<stdin>", &input),
         }
     }
     println!("Good-bye");

src/tokens.rs

@@ -206,13 +206,13 @@ impl Tokenizer {
         c.is_alphanumeric() || c == '_'
     }
 
-    fn scan_token(&mut self) -> Result<Token, ScanError> {
+    fn scan_token(&mut self) -> Result<Option<Token>, ScanError> {
         loop {
             if let Some(c) = self.peek() {
                 if Self::ident_or_keyword_start(c) {
-                    return self.scan_identifier_or_keyword();
+                    return self.scan_identifier_or_keyword().map(Some);
                 } else if c == '.' {
-                    return self.scan_meta_command();
+                    return self.scan_meta_command().map(Some);
                 } else if c.is_whitespace() {
                     self.advance();
                 } else {
@@ -223,10 +223,7 @@ impl Tokenizer {
                     });
                 }
             } else {
-                return Err(ScanError {
-                    location: self.current_location(0),
-                    kind: ScanErrorKind::UnexpectedEndOfInput,
-                });
+                return Ok(None);
             }
         }
     }
@@ -244,8 +241,10 @@ pub fn tokenize(input: String, file: String) -> Result<Vec<Token>, Vec<ScanError
     let mut tokenizer = Tokenizer::new(input, file);
     let mut errors = Vec::new();
     while !tokenizer.is_at_end() {
-        match tokenizer.scan_token() {
-            Ok(token) => tokenizer.tokens.push(token),
+        let token = tokenizer.scan_token();
+        match token {
+            Ok(Some(token)) => tokenizer.tokens.push(token),
+            Ok(None) => break,
             Err(err) => errors.push(err),
         }
     }