logistic-bot/osdb
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(parser): use token-based parser

Browse source
This commit is contained in:
Khaïs COLIN 2025-05-04 14:22:19 +02:00
parent d568653a17
commit a0869b1b66
3 changed files with 37 additions and 39 deletions
Download patch file Download diff file Expand all files Collapse all files

34
notes.org
View file

@ -1,4 +1,5 @@
#+title: Notes
#+property: EFFORT_ALL 0 10
* DONE show errors with ariadne
:PROPERTIES:
@ -120,23 +121,15 @@ CLOCK: [2025-05-03 sam. 21:21]--[2025-05-03 sam. 21:22] => 0:01
CLOCK: [2025-05-03 sam. 19:06]--[2025-05-03 sam. 19:07] => 0:01
:END:
* TODO switch statement parsing to more extensible token-based algorithm
* DONE switch statement parsing to more extensible token-based algorithm
:PROPERTIES:
:EFFORT: 10
:END:
:LOGBOOK:
CLOCK: [2025-05-04 dim. 12:07]--[2025-05-04 dim. 12:10] => 0:03
:END:
** TODO use tokens to parse meta-commands
:PROPERTIES:
:EFFORT: 10
:END:
:LOGBOOK:
CLOCK: [2025-05-04 dim. 12:10]--[2025-05-04 dim. 12:22] => 0:12
:END:
*** DONE recognize meta-commands as tokens
** DONE recognize meta-commands as tokens
:PROPERTIES:
:EFFORT: 10
:END:
@ -145,7 +138,7 @@ CLOCK: [2025-05-04 dim. 13:32]--[2025-05-04 dim. 13:35] => 0:03
CLOCK: [2025-05-04 dim. 13:27]--[2025-05-04 dim. 13:32] => 0:05
:END:
*** DONE CommandParseError must have a ScanError variant with an Into impl
** DONE CommandParseError must have a ScanError variant with an Into impl
:PROPERTIES:
:EFFORT: 10
:END:
@ -153,12 +146,12 @@ CLOCK: [2025-05-04 dim. 13:27]--[2025-05-04 dim. 13:32] => 0:05
CLOCK: [2025-05-04 dim. 13:35]--[2025-05-04 dim. 13:38] => 0:03
:END:
*** DONE ScanErrors must be convertible to ariadne reports
** DONE ScanErrors must be convertible to ariadne reports
:PROPERTIES:
:EFFORT: 10
:END:
**** DONE Remove the CommandParseError Display implementation
*** DONE Remove the CommandParseError Display implementation
:PROPERTIES:
:EFFORT: 10
:END:
@ -166,7 +159,7 @@ CLOCK: [2025-05-04 dim. 13:35]--[2025-05-04 dim. 13:38] => 0:03
CLOCK: [2025-05-04 dim. 13:38]--[2025-05-04 dim. 13:44] => 0:06
:END:
**** DONE implement OSDBError for ScanError
*** DONE implement OSDBError for ScanError
:PROPERTIES:
:EFFORT: 10
:END:
@ -174,12 +167,12 @@ CLOCK: [2025-05-04 dim. 13:38]--[2025-05-04 dim. 13:44] => 0:06
CLOCK: [2025-05-04 dim. 13:45]--[2025-05-04 dim. 13:56] => 0:11
:END:
*** DONE remove token types which are not recognized at all
** DONE remove token types which are not recognized at all
:PROPERTIES:
:EFFORT: 10
:END:
*** DONE create a generic parse command that parses string into tokens into Command
** DONE create a generic parse command that parses string into tokens into Command
:PROPERTIES:
:EFFORT: 10
:END:
@ -187,12 +180,11 @@ CLOCK: [2025-05-04 dim. 13:45]--[2025-05-04 dim. 13:56] => 0:11
CLOCK: [2025-05-04 dim. 14:01]--[2025-05-04 dim. 14:14] => 0:13
:END:
*** TODO parse tokens into meta-commands
** DONE parse tokens into meta-commands
:PROPERTIES:
:EFFORT: 10
:END:
** TODO use tokens to parse statements
:PROPERTIES:
:EFFORT:
:END:
* TODO error offsets are incorrect
* TODO remove old FromStr parser implementation

25
src/main.rs
View file

@ -1,20 +1,27 @@
use osdb::branding::startup_msg;
use osdb::cli::read_input;
use osdb::command::Command;
use osdb::error_display::OSDBError as _;
use osdb::parser::parse;
fn main() {
println!("{}", startup_msg());
while let Some(input) = read_input() {
match input.parse::<Command>() {
Ok(cmd) => {
let result = cmd.execute();
println!("{}", result.display());
if result.should_exit {
break;
'main: while let Some(input) = read_input() {
let file = String::from("<stdin>");
match parse(file.clone(), input.clone()) {
Ok(cmds) => {
for cmd in cmds {
let result = cmd.execute();
if result.should_exit {
break 'main;
}
println!("{}", result.display());
}
}
Err(errs) => {
for err in errs {
err.display(&file, &input)
}
}
Err(err) => err.display("<stdin>", &input),
}
}
println!("Good-bye");

17
src/tokens.rs
View file

@ -206,13 +206,13 @@ impl Tokenizer {
c.is_alphanumeric() || c == '_'
}
fn scan_token(&mut self) -> Result<Token, ScanError> {
fn scan_token(&mut self) -> Result<Option<Token>, ScanError> {
loop {
if let Some(c) = self.peek() {
if Self::ident_or_keyword_start(c) {
return self.scan_identifier_or_keyword();
return self.scan_identifier_or_keyword().map(Some);
} else if c == '.' {
return self.scan_meta_command();
return self.scan_meta_command().map(Some);
} else if c.is_whitespace() {
self.advance();
} else {
@ -223,10 +223,7 @@ impl Tokenizer {
});
}
} else {
return Err(ScanError {
location: self.current_location(0),
kind: ScanErrorKind::UnexpectedEndOfInput,
});
return Ok(None);
}
}
}
@ -244,8 +241,10 @@ pub fn tokenize(input: String, file: String) -> Result<Vec<Token>, Vec<ScanError
let mut tokenizer = Tokenizer::new(input, file);
let mut errors = Vec::new();
while !tokenizer.is_at_end() {
match tokenizer.scan_token() {
Ok(token) => tokenizer.tokens.push(token),
let token = tokenizer.scan_token();
match token {
Ok(Some(token)) => tokenizer.tokens.push(token),
Ok(None) => break,
Err(err) => errors.push(err),
}
}