logistic-bot/osdb
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Compare commits

base: logistic-bot:33c4edf91df882a3ef9e9065f75646092063eced
Branches Tags
logistic-bot:main
..
compare: logistic-bot:2dead7de0aec7ad236ceafd4c002ddd5fec75493
Branches Tags
logistic-bot:main

No commits in common. "33c4edf91df882a3ef9e9065f75646092063eced" and "2dead7de0aec7ad236ceafd4c002ddd5fec75493" have entirely different histories.
33c4edf91d ... 2dead7de0a

18 changed files with 80 additions and 647 deletions
Download patch file Download diff file Expand all files Collapse all files

63
grammar.ebnf
View file

@ -1,43 +1,32 @@
/* token is first stage of parsing */
token ::= insert
| select
| meta-command
| int
| string
| semicolon
| end-of-file
token ::= insert
| select
| meta-command
| int
| string
| end-of-file
/* command is second stage of parsing */
command ::= cmd-insert semicolon
| cmd-select semicolon
cmd-insert ::= insert int string string
cmd-select ::= select
insert ::= "insert"
select ::= "select"
insert ::= "insert"
select ::= "select"
semicolon ::= ";"
meta-command ::= "." "exit"
| "about"
meta-command ::= meta-command-verb end-of-file
meta-command-verb ::= ".exit"
| ".about"
| ".version"
int ::= sign? digit+
sign ::= "+"
| "-"
digit ::= "0"
| "1"
| "2"
| "3"
| "4"
| "5"
| "6"
| "7"
| "8"
| "9"
int ::= sign? digit+
sign ::= "+"
| "-"
digit ::= "0"
| "1"
| "2"
| "3"
| "4"
| "5"
| "6"
| "7"
| "8"
| "9"
string ::= '"' string-char* '"'
string-char ::= '\' utf8-char
| utf8-char-not-dbl-quote
string ::= '"' string-char* '"'
string-char ::= '\' utf8-char
| utf8-char-not-dbl-quote

50
notes.org
View file

@ -226,53 +226,17 @@ i will use rustyline, since it seems like the most feature-complete
* DONE write a proper grammar
* DONE .about meta-command
* DONE .version meta-command
* TODO .about meta-command
* TODO .version meta-command
* TODO .license meta-command
* TODO .help meta-command
* DONE parse insert statements in the form
* TODO parse insert statements in the form
insert <id:int> <username:string> <email:string>
** DONE parse row insert
* DONE separate statements with semicolons
* DONE this error message could be better
#+begin example
Error: unexpected token
╭─[ <stdin>:1:24 ]
│
1 │ insert 0 "user" "email"
│ │
│ ╰─ found end of file ""
│
│ Note: expected token type to be one of ["semicolon"]
───╯
#+end example
** plan
1. Create an example mapping system
- Define a mapping of token types to example values
- Example: "integer" → "42", "string" → "example", "semicolon" → ";"
2. Enhance CommandParseError
- Add a method to generate user-friendly error messages
- Include both the expected token type and concrete examples
3. Implementation approach
- Create a static lookup table or function that returns examples
- Extend existing error handling to include examples in messages
- Make sure the examples follow SQL syntax conventions
4. Error display refinement
- Update error_display.rs to include these examples
- Format error messages to show both what was expected and example syntax
5. Testing
- Add tests that verify the error messages include helpful examples
- Ensure examples are contextually appropriate
This will make errors like "expected semicolon" more helpful by showing "expected semicolon (example: ;)".
* DONE correct all instances of <unknown> in locations
* DONE meta-commands must be followed by end-of-file
* TODO project code documentation
* TODO project usage documentation
* DONE in case of parse error, skip until next semicolon to better recover
* TODO serialize/deserialize row to/from raw bytes
** TODO look for best practices for creating binary formats
** TODO Row struct
** TODO parse row insert
** TODO serialize/deserialize row to/from raw bytes
*** TODO look for best practices for creating binary formats
* WAIT cli tests using insta-cmd
https://insta.rs/docs/cmd/

10
src/branding.rs
View file

@ -13,18 +13,12 @@ pub fn startup_msg() -> String {
)
}
pub fn version_msg() -> String {
pub fn about_msg() -> String {
let name = env!("CARGO_PKG_NAME");
let version = env!("CARGO_PKG_VERSION");
format!("{name} v{version}")
}
pub fn about_msg() -> String {
let version = version_msg();
format!(
"{version} -- A database engine\n\
"{name} v{version} -- A database engine\n\
Note: This is experimental software. No maintenance is intendend."
)
}

49
src/command.rs
View file

@ -1,6 +1,6 @@
use crate::meta_commands::{MetaCommand, MetaCommandExecuteResult};
use crate::statements::{Statement, StatementExecuteResult};
use crate::tokens::{Location, ScanError, Token};
use crate::tokens::{ScanError, Token};
#[derive(Debug)]
pub enum Command {
@ -47,48 +47,10 @@ impl Command {
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ExpectedToken {
Integer,
String,
Semicolon,
Statement,
MetaCommand,
EndOfFile,
}
impl ExpectedToken {
/// Returns an example value for this token type
pub fn example(&self) -> &'static str {
match self {
ExpectedToken::Integer => "42",
ExpectedToken::String => "\"example\"",
ExpectedToken::Semicolon => ";",
ExpectedToken::Statement => "select",
ExpectedToken::MetaCommand => ".exit",
ExpectedToken::EndOfFile => "<end of input>",
}
}
}
impl std::fmt::Display for ExpectedToken {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
ExpectedToken::Integer => write!(f, "integer"),
ExpectedToken::String => write!(f, "string"),
ExpectedToken::Semicolon => write!(f, "semicolon"),
ExpectedToken::Statement => write!(f, "statement"),
ExpectedToken::MetaCommand => write!(f, "meta command"),
ExpectedToken::EndOfFile => write!(f, "end of file"),
}
}
}
#[derive(Debug)]
pub enum CommandParseError {
Scan(ScanError),
UnexpectedToken(Token, &'static [ExpectedToken]),
UnexpectedEndOfFile(Location, &'static [ExpectedToken]),
UnexpectedToken(Token, &'static [&'static str]),
}
impl From<MetaCommand> for Command {
@ -118,12 +80,7 @@ mod tests {
#[test]
fn test_execute_insert_statement() {
let statement: Command = Statement::Insert {
id: 45,
username: String::from("user"),
email: String::from("user@example.org"),
}
.into();
let statement: Command = Statement::Insert.into();
let result = statement.execute().display();
assert_snapshot!(result);
}

74
src/error_display.rs
View file

@ -1,7 +1,4 @@
use crate::{
command::{CommandParseError, ExpectedToken},
tokens::ScanError,
};
use crate::{command::CommandParseError, tokens::ScanError};
use ariadne::{Color, Label, Report, ReportKind, Source};
pub trait OSDBError {
@ -14,79 +11,24 @@ impl OSDBError for CommandParseError {
CommandParseError::Scan(x) => {
x.display(file, input);
}
CommandParseError::UnexpectedToken(token, expected_tokens) => {
CommandParseError::UnexpectedToken(token, items) => {
let location = (file, Into::<std::ops::Range<usize>>::into(&token.location));
let mut report = Report::build(ReportKind::Error, location.clone())
Report::build(ReportKind::Error, location.clone())
.with_message("unexpected token")
.with_label(
Label::new(location.clone())
.with_color(Color::Red)
.with_message(format!("found {token}")),
);
report = add_expected_tokens_to_report(report, expected_tokens);
report.finish().eprint((file, Source::from(input))).unwrap()
}
CommandParseError::UnexpectedEndOfFile(location, expected_tokens) => {
let location = (file, Into::<std::ops::Range<usize>>::into(location));
let report = Report::build(ReportKind::Error, location)
.with_message("unexpected end of file");
let report = add_expected_tokens_to_report(report, expected_tokens);
report.finish().eprint((file, Source::from(input))).unwrap()
)
.with_note(format!("expected token type to be one of {items:?}"))
.finish()
.eprint((file, Source::from(input)))
.unwrap()
}
}
}
}
type OSDBReport<'a> = ariadne::ReportBuilder<'a, (&'a str, std::ops::Range<usize>)>;
fn add_expected_tokens_to_report<'a>(
mut report: OSDBReport<'a>,
expected_tokens: &'static [ExpectedToken],
) -> OSDBReport<'a> {
// If we have expected tokens, show an example for the first one
if let Some(help) = expected_token_example_msg(expected_tokens) {
report = report.with_help(help)
}
// If we have at least one expected token, show a message showing what type was expected
if let Some(note) = expected_token_msg(expected_tokens) {
report = report.with_note(note);
}
report
}
fn expected_token_msg(expected_tokens: &'static [ExpectedToken]) -> Option<String> {
if !expected_tokens.is_empty() {
// Add a note with all expected types
let expected_types: Vec<_> = expected_tokens.iter().map(|t| format!("{}", t)).collect();
// Use singular form when there's only one expected token type
Some(match expected_types.as_slice() {
[single_type] => {
format!("expected: {}", single_type)
}
_ => {
format!("expected one of: {}", expected_types.join(", "))
}
})
} else {
// there are no expected tokens
None
}
}
fn expected_token_example_msg(expected_tokens: &'static [ExpectedToken]) -> Option<String> {
if let Some(first_expected) = expected_tokens.first() {
let example = first_expected.example();
Some(format!("try a token of the expected type: {example}"))
} else {
None
}
}
impl OSDBError for ScanError {
fn display(&self, file: &str, input: &str) {
let location = (file, Into::<std::ops::Range<usize>>::into(&self.location));

8
src/meta_commands.rs
View file

@ -1,10 +1,9 @@
use crate::branding;
#[derive(Debug, Eq, PartialEq, Clone)]
#[derive(Debug, Eq, PartialEq)]
pub enum MetaCommand {
Exit,
About,
Version,
}
impl std::fmt::Display for MetaCommand {
@ -12,7 +11,6 @@ impl std::fmt::Display for MetaCommand {
match self {
MetaCommand::Exit => write!(f, "exit"),
MetaCommand::About => write!(f, "about"),
MetaCommand::Version => write!(f, "version"),
}
}
}
@ -29,10 +27,6 @@ impl MetaCommand {
print!("{}", branding::about_msg());
MetaCommandExecuteResult { should_exit: false }
}
MetaCommand::Version => {
print!("{}", branding::version_msg());
MetaCommandExecuteResult { should_exit: false }
}
}
}
}

275
src/parser.rs
View file

@ -1,239 +1,36 @@
use std::collections::VecDeque;
use crate::{
command::{Command, CommandParseError, ExpectedToken},
command::{Command, CommandParseError},
statements::Statement,
tokens::{Location, Token, TokenData, tokenize},
tokens::tokenize,
};
// Helper function to skip tokens until reaching a semicolon or end of file
// This helps with error recovery when a statement has a syntax error
fn skip_to_next_statement(tokens: &mut VecDeque<Token>) {
while let Some(token) = tokens.front() {
match token.data {
TokenData::Semicolon | TokenData::EndOfFile => break,
_ => {
tokens.pop_front();
}
}
}
// Consume the semicolon if that's what we stopped at
if tokens
.front()
.is_some_and(|t| matches!(t.data, TokenData::Semicolon))
{
tokens.pop_front();
}
}
// Helper function to check for a semicolon after a statement
fn expect_semicolon(tokens: &mut VecDeque<crate::tokens::Token>) -> Result<(), CommandParseError> {
if let Some(next_token) = tokens.front() {
match next_token.data {
TokenData::Semicolon => {
tokens.pop_front(); // Consume the semicolon
Ok(())
}
_ => Err(CommandParseError::UnexpectedToken(
next_token.clone(),
&[ExpectedToken::Semicolon],
)),
}
} else {
// Even at the end of input, we need a semicolon
Err(CommandParseError::UnexpectedToken(
Token {
location: tokens
.back()
.map_or_else(Location::default, |t| t.location.clone()),
data: TokenData::EndOfFile,
lexeme: String::new(),
},
&[ExpectedToken::Semicolon],
))
}
}
fn parse_select_command(
tokens: &mut VecDeque<crate::tokens::Token>,
) -> Result<Command, CommandParseError> {
// Parse the select command (currently doesn't require additional tokens)
let cmd = Command::Statement(Statement::Select);
// Check for semicolon after select command
expect_semicolon(tokens)?;
Ok(cmd)
}
pub fn parse(file: String, input: String) -> Result<Vec<Command>, Vec<CommandParseError>> {
let mut tokens: VecDeque<_> = tokenize(input, file)
.map_err(|x| x.into_iter().map(|x| x.into()).collect::<Vec<_>>())?
.into();
let mut cmds = Vec::new();
let mut errs = Vec::new();
while let Some(token) = tokens.pop_front() {
match token.data {
TokenData::Insert => match parse_insert_command(&mut tokens) {
Ok(cmd) => cmds.push(cmd),
Err(err) => {
errs.push(err);
skip_to_next_statement(&mut tokens); // Skip to next statement for error recovery
}
},
TokenData::Select => match parse_select_command(&mut tokens) {
Ok(cmd) => cmds.push(cmd),
Err(err) => {
errs.push(err);
skip_to_next_statement(&mut tokens); // Skip to next statement for error recovery
}
},
TokenData::MetaCommand(meta_command) => {
match parse_meta_command(meta_command, &mut tokens) {
Ok(cmd) => cmds.push(cmd),
Err(err) => {
errs.push(err);
skip_to_next_statement(&mut tokens); // Skip to next statement for error recovery
}
}
crate::tokens::TokenData::Insert => cmds.push(Command::Statement(Statement::Insert)),
crate::tokens::TokenData::Select => cmds.push(Command::Statement(Statement::Select)),
crate::tokens::TokenData::MetaCommand(meta_command) => {
cmds.push(Command::MetaCommand(meta_command))
}
TokenData::Semicolon => {
// Empty statement (just a semicolon) - ignore it
}
TokenData::Int(_) => {
errs.push(CommandParseError::UnexpectedToken(
token,
&[
ExpectedToken::Statement,
ExpectedToken::MetaCommand,
ExpectedToken::EndOfFile,
],
));
skip_to_next_statement(&mut tokens);
}
TokenData::String(_) => {
errs.push(CommandParseError::UnexpectedToken(
token,
&[
ExpectedToken::Statement,
ExpectedToken::MetaCommand,
ExpectedToken::EndOfFile,
],
));
skip_to_next_statement(&mut tokens);
}
TokenData::EndOfFile => (), // End of parsing
}
}
if errs.is_empty() { Ok(cmds) } else { Err(errs) }
}
fn parse_meta_command(
meta_command: crate::meta_commands::MetaCommand,
tokens: &mut VecDeque<Token>,
) -> Result<Command, CommandParseError> {
if let Some(token) = tokens.pop_front() {
if matches!(token.data, TokenData::EndOfFile) {
Ok(Command::MetaCommand(meta_command))
} else {
Err(CommandParseError::UnexpectedToken(
crate::tokens::TokenData::Int(_) => errs.push(CommandParseError::UnexpectedToken(
token,
&[ExpectedToken::EndOfFile],
))
&["statement", "meta command", "eof"],
)),
crate::tokens::TokenData::String(_) => errs.push(CommandParseError::UnexpectedToken(
token,
&["statement", "meta command", "eof"],
)),
crate::tokens::TokenData::EndOfFile => (),
}
} else {
Ok(Command::MetaCommand(meta_command))
}
}
fn parse_insert_command(
tokens: &mut VecDeque<crate::tokens::Token>,
) -> Result<Command, CommandParseError> {
// According to grammar.ebnf, insert command should be: insert int string string semicolon
// Parse the id (integer)
let id_token = tokens.pop_front().ok_or_else(|| {
CommandParseError::UnexpectedToken(
Token {
location: tokens
.back()
.map_or_else(Location::default, |t| t.location.clone()),
data: TokenData::EndOfFile,
lexeme: String::new(),
},
&[ExpectedToken::Integer],
)
})?;
let id = match id_token.data {
TokenData::Int(id) => id,
_ => {
return Err(CommandParseError::UnexpectedToken(
id_token,
&[ExpectedToken::Integer],
));
}
};
// Parse the username (string)
let username_token = tokens.pop_front().ok_or_else(|| {
CommandParseError::UnexpectedToken(
Token {
location: tokens
.back()
.map_or_else(Location::default, |t| t.location.clone()),
data: TokenData::EndOfFile,
lexeme: String::new(),
},
&[ExpectedToken::String],
)
})?;
let username = match username_token.data {
TokenData::String(username) => username,
_ => {
return Err(CommandParseError::UnexpectedToken(
username_token,
&[ExpectedToken::String],
));
}
};
// Parse the email (string)
let email_token = tokens.pop_front().ok_or_else(|| {
CommandParseError::UnexpectedToken(
Token {
location: tokens
.back()
.map_or_else(Location::default, |t| t.location.clone()),
data: TokenData::EndOfFile,
lexeme: String::new(),
},
&[ExpectedToken::String],
)
})?;
let email = match email_token.data {
TokenData::String(email) => email,
_ => {
return Err(CommandParseError::UnexpectedToken(
email_token,
&[ExpectedToken::String],
));
}
};
// Check for semicolon after the insert command
expect_semicolon(tokens)?;
Ok(Command::Statement(Statement::Insert {
id,
username,
email,
}))
if errs.is_empty() { Ok(cmds) } else { Err(errs) }
}
#[cfg(test)]
@ -245,32 +42,10 @@ mod tests {
fn test_parse_single_correct() {
let file = String::from("<stdin>");
assert_debug_snapshot!(parse(file.clone(), String::from(".exit")));
assert_debug_snapshot!(parse(file.clone(), String::from("select;")));
assert_debug_snapshot!(parse(file.clone(), String::from("sElEcT;")));
}
#[test]
fn test_parse_insert_command() {
let file = String::from("<stdin>");
assert_debug_snapshot!(parse(
file.clone(),
String::from(r#"insert 1 "username" "email@example.com";"#)
));
assert_debug_snapshot!(parse(
file.clone(),
String::from(r#"insert "not_an_id" "username" "email@example.com";"#)
));
assert_debug_snapshot!(parse(file.clone(), String::from(r#"insert 1 "username";"#)));
}
#[test]
fn test_parse_missing_semicolon() {
let file = String::from("<stdin>");
assert_debug_snapshot!(parse(file.clone(), String::from("select")));
assert_debug_snapshot!(parse(
file.clone(),
String::from(r#"insert 1 "username" "email@example.com""#)
));
assert_debug_snapshot!(parse(file.clone(), String::from("sElEcT")));
assert_debug_snapshot!(parse(file.clone(), String::from("INSERT")));
assert_debug_snapshot!(parse(file.clone(), String::from("InSErT")));
}
#[test]
@ -285,22 +60,10 @@ mod tests {
#[test]
fn test_parse_multiple_correct() {
let file = String::from("<stdin>");
assert_debug_snapshot!(parse(file.clone(), String::from("select; select; select;")));
}
#[test]
fn test_meta_command_require_eof() {
let file = String::from("<stdin>");
assert_debug_snapshot!(parse(file.clone(), String::from(".exit select; select;")));
}
#[test]
fn test_parse_multiple_statements_with_insert() {
let file = String::from("<stdin>");
assert_debug_snapshot!(parse(
file.clone(),
String::from(r#"select; insert 1 "user" "email@test.com"; select;"#)
String::from(".exit select select insert select")
));
}

2
src/snapshots/osdb__command__tests__execute_insert_statement.snap
View file

@ -2,4 +2,4 @@
source: src/command.rs
expression: result
---
insert 45 "user" "user@example.org"
insert

22
src/snapshots/osdb__parser__tests__meta_command_require_eof.snap
View file

@ -1,22 +0,0 @@
---
source: src/parser.rs
expression: "parse(file.clone(), String::from(\".exit select; select;\"))"
---
Err(
[
UnexpectedToken(
Token {
location: Location {
file: "<stdin>",
offset: 6,
length: 6,
},
data: Select,
lexeme: "select",
},
[
EndOfFile,
],
),
],
)

24
src/snapshots/osdb__parser__tests__parse_insert_command-2.snap
View file

@ -1,24 +0,0 @@
---
source: src/parser.rs
expression: "parse(file.clone(),\nString::from(r#\"insert \"not_an_id\" \"username\" \"email@example.com\";\"#))"
---
Err(
[
UnexpectedToken(
Token {
location: Location {
file: "<stdin>",
offset: 7,
length: 11,
},
data: String(
"not_an_id",
),
lexeme: "\"not_an_id\"",
},
[
Integer,
],
),
],
)

22
src/snapshots/osdb__parser__tests__parse_insert_command-3.snap
View file

@ -1,22 +0,0 @@
---
source: src/parser.rs
expression: "parse(file.clone(), String::from(r#\"insert 1 \"username\";\"#))"
---
Err(
[
UnexpectedToken(
Token {
location: Location {
file: "<stdin>",
offset: 19,
length: 1,
},
data: Semicolon,
lexeme: ";",
},
[
String,
],
),
],
)

15
src/snapshots/osdb__parser__tests__parse_insert_command.snap
View file

@ -1,15 +0,0 @@
---
source: src/parser.rs
expression: "parse(file.clone(),\nString::from(r#\"insert 1 \"username\" \"email@example.com\"\"#))"
---
Ok(
[
Statement(
Insert {
id: 1,
username: "username",
email: "email@example.com",
},
),
],
)

22
src/snapshots/osdb__parser__tests__parse_missing_semicolon-2.snap
View file

@ -1,22 +0,0 @@
---
source: src/parser.rs
expression: "parse(file.clone(),\nString::from(r#\"insert 1 \"username\" \"email@example.com\"\"#))"
---
Err(
[
UnexpectedToken(
Token {
location: Location {
file: "<stdin>",
offset: 39,
length: 0,
},
data: EndOfFile,
lexeme: "",
},
[
Semicolon,
],
),
],
)

22
src/snapshots/osdb__parser__tests__parse_missing_semicolon.snap
View file

@ -1,22 +0,0 @@
---
source: src/parser.rs
expression: "parse(file.clone(), String::from(\"select\"))"
---
Err(
[
UnexpectedToken(
Token {
location: Location {
file: "<stdin>",
offset: 6,
length: 0,
},
data: EndOfFile,
lexeme: "",
},
[
Semicolon,
],
),
],
)

8
src/snapshots/osdb__parser__tests__parse_multiple_correct.snap
View file

@ -1,15 +1,21 @@
---
source: src/parser.rs
expression: "parse(file.clone(), String::from(\"select; select; select;\"))"
expression: "parse(file.clone(), String::from(\".exit select select insert select\"))"
---
Ok(
[
MetaCommand(
Exit,
),
Statement(
Select,
),
Statement(
Select,
),
Statement(
Insert,
),
Statement(
Select,
),

21
src/snapshots/osdb__parser__tests__parse_multiple_statements_with_insert.snap
View file

@ -1,21 +0,0 @@
---
source: src/parser.rs
expression: "parse(file.clone(),\nString::from(r#\"select; insert 1 \"user\" \"email@test.com\"; select;\"#))"
---
Ok(
[
Statement(
Select,
),
Statement(
Insert {
id: 1,
username: "user",
email: "email@test.com",
},
),
Statement(
Select,
),
],
)

14
src/statements.rs
View file

@ -1,10 +1,6 @@
#[derive(Debug)]
pub enum Statement {
Insert {
id: i64,
username: String,
email: String,
},
Insert,
Select,
}
@ -15,12 +11,8 @@ pub struct StatementExecuteResult {
impl Statement {
pub fn execute(&self) -> StatementExecuteResult {
match self {
Statement::Insert {
id,
username,
email,
} => StatementExecuteResult {
msg: format!("insert {id:?} {username:?} {email:?}"),
Statement::Insert => StatementExecuteResult {
msg: String::from("insert"),
},
Statement::Select => StatementExecuteResult {
msg: String::from("select"),

26
src/tokens.rs
View file

@ -1,6 +1,6 @@
use crate::meta_commands::MetaCommand;
#[derive(Debug, Eq, PartialEq, Clone)]
#[derive(Debug, Eq, PartialEq)]
pub enum TokenData {
Insert,
Select,
@ -8,10 +8,9 @@ pub enum TokenData {
EndOfFile,
Int(i64),
String(String),
Semicolon,
}
#[derive(Debug, Eq, PartialEq, Clone)]
#[derive(Debug, Eq, PartialEq)]
pub struct Location {
/// file name
pub file: String,
@ -30,12 +29,6 @@ impl From<&Location> for std::ops::Range<usize> {
}
}
impl Default for Location {
fn default() -> Self {
Self::new(String::from("<unknown>"), 0, 0)
}
}
impl Location {
/// ```
/// use osdb::tokens::Location;
@ -53,7 +46,7 @@ impl Location {
}
}
#[derive(Debug, Eq, PartialEq, Clone)]
#[derive(Debug, Eq, PartialEq)]
pub struct Token {
/// Where in the input was this token found?
pub location: Location,
@ -72,7 +65,6 @@ impl std::fmt::Display for Token {
TokenData::EndOfFile => write!(f, "end of file"),
TokenData::Int(x) => write!(f, "integer {x}"),
TokenData::String(x) => write!(f, "string {x:?}"),
TokenData::Semicolon => write!(f, "semicolon"),
}?;
let lexeme = &self.lexeme;
write!(f, " {lexeme:?}")
@ -169,7 +161,6 @@ impl Tokenizer {
match word.to_lowercase().as_str() {
".exit" => Some(TokenData::MetaCommand(MetaCommand::Exit)),
".about" => Some(TokenData::MetaCommand(MetaCommand::About)),
".version" => Some(TokenData::MetaCommand(MetaCommand::Version)),
_ => None,
}
}
@ -315,15 +306,6 @@ impl Tokenizer {
}
}
fn scan_semicolon(&mut self) -> Result<Token, ScanError> {
self.advance();
Ok(Token {
location: self.previous_location(1),
data: TokenData::Semicolon,
lexeme: String::from(";"),
})
}
fn scan_token(&mut self) -> Result<Option<Token>, ScanError> {
loop {
if let Some(c) = self.peek() {
@ -335,8 +317,6 @@ impl Tokenizer {
return self.scan_integer().map(Some);
} else if c == '"' {
return self.scan_string().map(Some);
} else if c == ';' {
return self.scan_semicolon().map(Some);
} else if c.is_whitespace() {
self.advance();
} else {