From 39f9c5c11768309156cdada33900333a1ba91ef4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emilio=20Cobos=20=C3=81lvarez?= Date: Sun, 30 Aug 2026 15:11:58 +0200 Subject: [PATCH] parser: Avoid some redundant EOF checks during tokenization. We already need to check for EOF for the delimiter check. --- src/parser.rs | 51 ++++++++++++++++++++++-------------------------- src/tests.rs | 2 +- src/tokenizer.rs | 18 +++++++++++------ 3 files changed, 36 insertions(+), 35 deletions(-) diff --git a/src/parser.rs b/src/parser.rs index 0e924299..2fe741f5 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -354,7 +354,7 @@ impl Delimiters { } #[inline] - pub(crate) fn from_byte(byte: Option) -> Delimiters { + pub(crate) fn from_byte(byte: u8) -> Delimiters { const TABLE: [Delimiters; 256] = { let mut table = [Delimiter::None; 256]; table[b';' as usize] = Delimiter::Semicolon; @@ -367,8 +367,7 @@ impl Delimiters { table }; - assert_eq!(TABLE[0], Delimiter::None); - TABLE[byte.unwrap_or(0) as usize] + TABLE[byte as usize] } } @@ -506,11 +505,11 @@ impl<'i: 't, 't> Parser<'i, 't> { #[inline] pub(crate) fn next_byte(&self) -> Option { - let byte = self.input.tokenizer.next_byte(); + let byte = self.input.tokenizer.next_byte()?; if self.stop_before.contains(Delimiters::from_byte(byte)) { return None; } - byte + Some(byte) } /// Restore the internal state of the parser (including position within the input) @@ -618,7 +617,10 @@ impl<'i: 't, 't> Parser<'i, 't> { consume_until_end_of_block(block_type, &mut self.input.tokenizer); } - let byte = self.input.tokenizer.next_byte(); + let Some(byte) = self.input.tokenizer.next_byte() else { + return Err(BasicParseError::new(BasicParseErrorKind::EndOfInput)); + }; + if self.stop_before.contains(Delimiters::from_byte(byte)) { return Err(BasicParseError::new(BasicParseErrorKind::EndOfInput)); } @@ -633,9 +635,7 @@ impl<'i: 't, 't> Parser<'i, 't> { } &cached_token.token } else { - let Ok(new_token) = self.input.tokenizer.next() else { - return Err(BasicParseError::new(BasicParseErrorKind::EndOfInput)); - }; + let new_token = self.input.tokenizer.next_unchecked(); self.input.cached_token = CachedToken { token: new_token, start_position: token_start_position, @@ -1027,16 +1027,13 @@ where } } // FIXME: have a special-purpose tokenizer method for this that does less work. - loop { - if delimiters.contains(Delimiters::from_byte(parser.input.tokenizer.next_byte())) { + while let Some(next_byte) = parser.input.tokenizer.next_byte() { + if delimiters.contains(Delimiters::from_byte(next_byte)) { break; } - if let Ok(token) = parser.input.tokenizer.next() { - if let Some(block_type) = BlockType::opening(&token) { - consume_until_end_of_block(block_type, &mut parser.input.tokenizer); - } - } else { - break; + let token = parser.input.tokenizer.next_unchecked(); + if let Some(block_type) = BlockType::opening(&token) { + consume_until_end_of_block(block_type, &mut parser.input.tokenizer); } } result @@ -1055,17 +1052,15 @@ where if error_behavior == ParseUntilErrorBehavior::Stop && result.is_err() { return result; } - let next_byte = parser.input.tokenizer.next_byte(); - if next_byte.is_some() - && !parser - .stop_before - .contains(Delimiters::from_byte(next_byte)) - { - debug_assert!(delimiters.contains(Delimiters::from_byte(next_byte))); - // We know this byte is ASCII. - parser.input.tokenizer.advance(1); - if next_byte == Some(b'{') { - consume_until_end_of_block(BlockType::CurlyBracket, &mut parser.input.tokenizer); + if let Some(next_byte) = parser.input.tokenizer.next_byte() { + let delimiter = Delimiters::from_byte(next_byte); + if !parser.stop_before.contains(delimiter) { + debug_assert!(delimiters.contains(delimiter)); + // We know this byte is ASCII. + parser.input.tokenizer.advance(1); + if next_byte == b'{' { + consume_until_end_of_block(BlockType::CurlyBracket, &mut parser.input.tokenizer); + } } } result diff --git a/src/tests.rs b/src/tests.rs index fd5127d9..b7de5d24 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -786,7 +786,7 @@ fn delimiter_from_byte(b: &mut Bencher) { b.iter(|| { for _ in 0..1000 { for i in 0..256 { - std::hint::black_box(Delimiters::from_byte(Some(i as u8))); + std::hint::black_box(Delimiters::from_byte(i as u8)); } } }) diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 8c3c158b..9de5eea5 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -256,7 +256,15 @@ impl<'a> Tokenizer<'a> { #[inline] pub fn next(&mut self) -> Result, ()> { - next_token(self) + if self.is_eof() { + return Err(()); + } + Ok(self.next_unchecked()) + } + + #[inline] + pub fn next_unchecked(&mut self) -> Token<'a> { + next_token_unchecked(self) } #[inline] @@ -580,10 +588,8 @@ pub struct SourceLocation { #[cfg(feature = "malloc_size_of")] malloc_size_of::malloc_size_of_is_0!(SourceLocation); -fn next_token<'a>(tokenizer: &mut Tokenizer<'a>) -> Result, ()> { - if tokenizer.is_eof() { - return Err(()); - } +fn next_token_unchecked<'a>(tokenizer: &mut Tokenizer<'a>) -> Token<'a> { + debug_assert!(!tokenizer.is_eof()); let b = tokenizer.next_byte_unchecked(); let token = match_byte! { b, b' ' | b'\t' => { @@ -712,7 +718,7 @@ fn next_token<'a>(tokenizer: &mut Tokenizer<'a>) -> Result, ()> { } }, }; - Ok(token) + token } fn consume_whitespace<'a>(tokenizer: &mut Tokenizer<'a>, newline: bool) -> Token<'a> {