[Rust Frontend] Simplify reasoning parser initialization and test organization (#55411)

Co-authored-by: Codex <noreply@openai.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-09-09 10:57:15 +08:00
committed by GitHub
co-authored by Codex
parent c6ba5e89d6
commit 050f29caf1
10 changed files with 167 additions and 172 deletions
@@ -19,7 +19,6 @@ impl CohereCmdReasoningParser {
tokenizer,
"<|START_THINKING|>",
"<|END_THINKING|>",
false,
)?,
})
}
+19 -5
View File
@@ -6,10 +6,6 @@ use vllm_tokenizer::{DecodedText, DynTokenizer};
use super::{DelimitedReasoningParser, ReasoningDelta, ReasoningParser, Result};
/// Reasoning parser for DeepSeek R1 style outputs.
///
/// DeepSeek R1 may begin generating directly inside a reasoning span and only
/// emit the closing `</think>` delimiter, so the no-boundary fallback defaults
/// to `in_reasoning = true`.
pub struct DeepSeekR1ReasoningParser {
inner: DelimitedReasoningParser,
}
@@ -19,7 +15,7 @@ impl DeepSeekR1ReasoningParser {
/// machine.
pub fn new(tokenizer: DynTokenizer) -> Result<Self> {
Ok(Self {
inner: DelimitedReasoningParser::new(tokenizer, "<think>", "</think>", true)?,
inner: DelimitedReasoningParser::new(tokenizer, "<think>", "</think>")?,
})
}
}
@@ -45,3 +41,21 @@ impl ReasoningParser for DeepSeekR1ReasoningParser {
Ok(self.inner.finish())
}
}
#[cfg(test)]
mod tests {
use std::sync::Arc;
use super::DeepSeekR1ReasoningParser;
use crate::reasoning::tests::{content_str, fake_tokenizer, push_str};
#[test]
fn deepseek_r1_without_prompt_markers_expects_start_token() {
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = DeepSeekR1ReasoningParser::new(tokenizer).unwrap();
let delta = push_str(&mut parser, "reason</think>answer");
assert_eq!(delta.reasoning, None);
assert_eq!(content_str(&delta), Some("reason</think>answer"));
}
}
+2 -9
View File
@@ -24,20 +24,14 @@ pub(crate) struct DelimitedReasoningParser {
end_token: String,
start_token_id: u32,
end_token_id: u32,
default_in_reasoning: bool,
}
impl DelimitedReasoningParser {
/// Create one delimited parser state machine.
///
/// `default_in_reasoning` is only used when prompt initialization sees no
/// reasoning boundary token at all. If the prompt contains either the
/// start or end delimiter, that prompt boundary always wins.
pub(crate) fn new(
tokenizer: DynTokenizer,
start_token: impl Into<String>,
end_token: impl Into<String>,
default_in_reasoning: bool,
) -> Result<Self> {
let start_token = start_token.into();
let end_token = end_token.into();
@@ -54,13 +48,12 @@ impl DelimitedReasoningParser {
Ok(Self {
tokenizer,
current_in_reasoning: default_in_reasoning,
current_in_reasoning: false,
buffer: DecodedText::default(),
start_token,
end_token,
start_token_id,
end_token_id,
default_in_reasoning,
})
}
@@ -72,7 +65,7 @@ impl DelimitedReasoningParser {
self.end_token_id,
self.tokenizer.as_ref(),
)
.unwrap_or(self.default_in_reasoning);
.unwrap_or(false);
}
/// Return whether the parser is currently inside a reasoning section.
-1
View File
@@ -18,7 +18,6 @@ impl HyReasoningParser {
tokenizer,
format!("<think{suffix}>"),
format!("</think{suffix}>"),
false,
)?,
})
}
+1 -1
View File
@@ -14,7 +14,7 @@ impl KimiReasoningParser {
/// Create a Kimi parser backed by the shared delimited state machine.
pub fn new(tokenizer: DynTokenizer) -> Result<Self> {
Ok(Self {
inner: DelimitedReasoningParser::new(tokenizer, "◁think▷", "◁/think▷", false)?,
inner: DelimitedReasoningParser::new(tokenizer, "◁think▷", "◁/think▷")?,
})
}
}
+75 -1
View File
@@ -27,7 +27,7 @@ impl MiniMaxM3ReasoningParser {
/// Create a MiniMax M3 parser backed by the shared delimited state machine.
pub fn new(tokenizer: DynTokenizer) -> Result<Self> {
Ok(Self {
inner: DelimitedReasoningParser::new(tokenizer, M3_THINK_START, M3_THINK_END, false)?,
inner: DelimitedReasoningParser::new(tokenizer, M3_THINK_START, M3_THINK_END)?,
at_response_start: true,
leading_end_buffer: DecodedText::default(),
})
@@ -98,3 +98,77 @@ impl ReasoningParser for MiniMaxM3ReasoningParser {
Ok(delta)
}
}
#[cfg(test)]
mod tests {
use std::sync::Arc;
use super::MiniMaxM3ReasoningParser;
use crate::reasoning::ReasoningParser;
use crate::reasoning::tests::{
MM_THINK_END_ID, MM_THINK_START_ID, content_str, fake_tokenizer, push_str, reasoning_str,
};
#[test]
fn minimax_m3_handles_explicit_think_delimiters() {
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap();
let delta = push_str(&mut parser, "<mm:think>reason</mm:think>answer");
assert_eq!(reasoning_str(&delta), Some("reason"));
assert_eq!(content_str(&delta), Some("answer"));
}
#[test]
fn minimax_m3_drops_leading_end_marker() {
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap();
let delta = push_str(&mut parser, "</mm:think>answer");
assert_eq!(delta.reasoning, None);
assert_eq!(content_str(&delta), Some("answer"));
}
#[test]
fn minimax_m3_preserves_non_leading_end_marker() {
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap();
let delta = push_str(&mut parser, "XXX</mm:think>YYY");
assert_eq!(delta.reasoning, None);
assert_eq!(content_str(&delta), Some("XXX</mm:think>YYY"));
}
#[test]
fn minimax_m3_drops_split_leading_end_marker() {
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap();
assert!(push_str(&mut parser, "</mm").is_empty());
let delta = push_str(&mut parser, ":think>answer");
assert_eq!(delta.reasoning, None);
assert_eq!(content_str(&delta), Some("answer"));
}
#[test]
fn minimax_m3_uses_prompt_prefilled_start_marker() {
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap();
parser.initialize(&[MM_THINK_START_ID]).unwrap();
let delta = push_str(&mut parser, "reason</mm:think>answer");
assert_eq!(reasoning_str(&delta), Some("reason"));
assert_eq!(content_str(&delta), Some("answer"));
}
#[test]
fn minimax_m3_uses_prompt_prefilled_end_marker() {
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap();
parser.initialize(&[MM_THINK_END_ID]).unwrap();
let delta = push_str(&mut parser, "answer");
assert_eq!(delta.reasoning, None);
assert_eq!(content_str(&delta), Some("answer"));
}
}
+62 -1
View File
@@ -18,7 +18,7 @@ impl Qwen3ReasoningParser {
/// Create a Qwen3 parser backed by the shared delimited state machine.
pub fn new(tokenizer: DynTokenizer) -> Result<Self> {
Ok(Self {
inner: DelimitedReasoningParser::new(tokenizer, "<think>", "</think>", false)?,
inner: DelimitedReasoningParser::new(tokenizer, "<think>", "</think>")?,
})
}
}
@@ -44,3 +44,64 @@ impl ReasoningParser for Qwen3ReasoningParser {
Ok(self.inner.finish())
}
}
#[cfg(test)]
mod tests {
use std::sync::Arc;
use super::Qwen3ReasoningParser;
use crate::reasoning::ReasoningParser;
use crate::reasoning::tests::{
SPECIAL_BOUNDARY_ID, THINK_END_ID, THINK_START_ID, content_str, fake_tokenizer, push_str,
reasoning_str,
};
#[test]
fn qwen3_without_prompt_markers_expects_start_token() {
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = Qwen3ReasoningParser::new(tokenizer).unwrap();
let delta = push_str(&mut parser, "reason</think>answer");
assert_eq!(delta.reasoning, None);
assert_eq!(content_str(&delta), Some("reason</think>answer"));
}
#[test]
fn qwen3_prompt_end_marker_starts_in_content() {
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = Qwen3ReasoningParser::new(tokenizer).unwrap();
parser.initialize(&[THINK_END_ID]).unwrap();
let delta = push_str(&mut parser, "answer");
assert_eq!(delta.reasoning, None);
assert_eq!(content_str(&delta), Some("answer"));
}
#[test]
fn qwen3_tolerates_old_and_new_formats() {
let tokenizer = Arc::new(fake_tokenizer());
let mut old_parser = Qwen3ReasoningParser::new(tokenizer.clone()).unwrap();
let old = push_str(&mut old_parser, "<think>reason</think>answer");
assert_eq!(reasoning_str(&old), Some("reason"));
assert_eq!(content_str(&old), Some("answer"));
let mut new_parser = Qwen3ReasoningParser::new(tokenizer).unwrap();
new_parser.initialize(&[THINK_START_ID]).unwrap();
let new = push_str(&mut new_parser, "reason</think>answer");
assert_eq!(reasoning_str(&new), Some("reason"));
assert_eq!(content_str(&new), Some("answer"));
}
#[test]
fn qwen3_stops_scanning_at_last_special_token() {
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = Qwen3ReasoningParser::new(tokenizer).unwrap();
parser.initialize(&[THINK_START_ID, SPECIAL_BOUNDARY_ID]).unwrap();
let delta = push_str(&mut parser, "answer");
assert_eq!(delta.reasoning, None);
assert_eq!(content_str(&delta), Some("answer"));
}
}
+1 -6
View File
@@ -15,12 +15,7 @@ impl SeedOssReasoningParser {
/// Create a SeedOSS parser backed by the shared delimited state machine.
pub fn new(tokenizer: DynTokenizer) -> Result<Self> {
Ok(Self {
inner: DelimitedReasoningParser::new(
tokenizer,
"<seed:think>",
"</seed:think>",
false,
)?,
inner: DelimitedReasoningParser::new(tokenizer, "<seed:think>", "</seed:think>")?,
})
}
}
+1 -1
View File
@@ -26,7 +26,7 @@ impl Step3p5ReasoningParser {
/// Create a Step3p5 parser backed by the shared delimited state machine.
pub fn new(tokenizer: DynTokenizer) -> Result<Self> {
Ok(Self {
inner: DelimitedReasoningParser::new(tokenizer, "<think>", "</think>", false)?,
inner: DelimitedReasoningParser::new(tokenizer, "<think>", "</think>")?,
pending_reasoning_newline: None,
just_ended_reasoning: false,
})
+6 -146
View File
@@ -57,8 +57,7 @@ pub(crate) fn content_str(delta: &ReasoningDelta) -> Option<&str> {
#[test]
fn delimited_content_only_stream() {
let tokenizer = Arc::new(fake_tokenizer());
let mut parser =
DelimitedReasoningParser::new(tokenizer, "<think>", "</think>", false).unwrap();
let mut parser = DelimitedReasoningParser::new(tokenizer, "<think>", "</think>").unwrap();
let delta = parser.push(DecodedText::unattributed("plain content"));
assert_eq!(content_str(&delta), Some("plain content"));
@@ -67,8 +66,7 @@ fn delimited_content_only_stream() {
#[test]
fn delimited_single_chunk_with_reasoning_and_content() {
let tokenizer = Arc::new(fake_tokenizer());
let mut parser =
DelimitedReasoningParser::new(tokenizer, "<think>", "</think>", false).unwrap();
let mut parser = DelimitedReasoningParser::new(tokenizer, "<think>", "</think>").unwrap();
let delta = parser.push(DecodedText::unattributed("<think>reason</think>answer"));
assert_eq!(reasoning_str(&delta), Some("reason"));
@@ -78,8 +76,7 @@ fn delimited_single_chunk_with_reasoning_and_content() {
#[test]
fn delimited_partial_tokens_across_chunks() {
let tokenizer = Arc::new(fake_tokenizer());
let mut parser =
DelimitedReasoningParser::new(tokenizer, "<think>", "</think>", false).unwrap();
let mut parser = DelimitedReasoningParser::new(tokenizer, "<think>", "</think>").unwrap();
assert!(parser.push(DecodedText::unattributed("<thi")).is_empty());
let delta = parser.push(DecodedText::unattributed("nk>reason</think>answer"));
@@ -90,8 +87,7 @@ fn delimited_partial_tokens_across_chunks() {
#[test]
fn delimited_finish_flushes_buffer() {
let tokenizer = Arc::new(fake_tokenizer());
let mut parser =
DelimitedReasoningParser::new(tokenizer, "<think>", "</think>", false).unwrap();
let mut parser = DelimitedReasoningParser::new(tokenizer, "<think>", "</think>").unwrap();
parser.initialize(&[THINK_START_ID]);
let delta = parser.push(DecodedText::unattributed("unfinished</thi"));
@@ -100,145 +96,10 @@ fn delimited_finish_flushes_buffer() {
assert_eq!(reasoning_str(&final_delta), Some("</thi"));
}
#[test]
fn qwen3_without_prompt_markers_expects_start_token() {
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = Qwen3ReasoningParser::new(tokenizer).unwrap();
let delta = push_str(&mut parser, "reason</think>answer");
assert_eq!(delta.reasoning, None);
assert_eq!(content_str(&delta), Some("reason</think>answer"));
}
#[test]
fn qwen3_prompt_end_marker_starts_in_content() {
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = Qwen3ReasoningParser::new(tokenizer).unwrap();
parser.initialize(&[THINK_END_ID]).unwrap();
let delta = push_str(&mut parser, "answer");
assert_eq!(delta.reasoning, None);
assert_eq!(content_str(&delta), Some("answer"));
}
#[test]
fn qwen3_tolerates_old_and_new_formats() {
let tokenizer = Arc::new(fake_tokenizer());
let mut old_parser = Qwen3ReasoningParser::new(tokenizer.clone()).unwrap();
let old = push_str(&mut old_parser, "<think>reason</think>answer");
assert_eq!(reasoning_str(&old), Some("reason"));
assert_eq!(content_str(&old), Some("answer"));
let mut new_parser = Qwen3ReasoningParser::new(tokenizer).unwrap();
new_parser.initialize(&[THINK_START_ID]).unwrap();
let new = push_str(&mut new_parser, "reason</think>answer");
assert_eq!(reasoning_str(&new), Some("reason"));
assert_eq!(content_str(&new), Some("answer"));
}
#[test]
fn qwen3_stops_scanning_at_last_special_token() {
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = Qwen3ReasoningParser::new(tokenizer).unwrap();
parser.initialize(&[THINK_START_ID, SPECIAL_BOUNDARY_ID]).unwrap();
let delta = push_str(&mut parser, "answer");
assert_eq!(delta.reasoning, None);
assert_eq!(content_str(&delta), Some("answer"));
}
#[test]
fn deepseek_r1_defaults_to_reasoning_without_prompt_boundary() {
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = DeepSeekR1ReasoningParser::new(tokenizer).unwrap();
let delta = push_str(&mut parser, "reason</think>answer");
assert_eq!(reasoning_str(&delta), Some("reason"));
assert_eq!(content_str(&delta), Some("answer"));
}
#[test]
fn deepseek_r1_stops_scanning_at_last_special_token() {
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = DeepSeekR1ReasoningParser::new(tokenizer).unwrap();
parser.initialize(&[THINK_END_ID, SPECIAL_BOUNDARY_ID]).unwrap();
let delta = push_str(&mut parser, "reason</think>answer");
assert_eq!(reasoning_str(&delta), Some("reason"));
assert_eq!(content_str(&delta), Some("answer"));
}
#[test]
fn minimax_m3_handles_explicit_think_delimiters() {
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap();
let delta = push_str(&mut parser, "<mm:think>reason</mm:think>answer");
assert_eq!(reasoning_str(&delta), Some("reason"));
assert_eq!(content_str(&delta), Some("answer"));
}
#[test]
fn minimax_m3_drops_leading_end_marker() {
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap();
let delta = push_str(&mut parser, "</mm:think>answer");
assert_eq!(delta.reasoning, None);
assert_eq!(content_str(&delta), Some("answer"));
}
#[test]
fn minimax_m3_preserves_non_leading_end_marker() {
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap();
let delta = push_str(&mut parser, "XXX</mm:think>YYY");
assert_eq!(delta.reasoning, None);
assert_eq!(content_str(&delta), Some("XXX</mm:think>YYY"));
}
#[test]
fn minimax_m3_drops_split_leading_end_marker() {
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap();
assert!(push_str(&mut parser, "</mm").is_empty());
let delta = push_str(&mut parser, ":think>answer");
assert_eq!(delta.reasoning, None);
assert_eq!(content_str(&delta), Some("answer"));
}
#[test]
fn minimax_m3_uses_prompt_prefilled_start_marker() {
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap();
parser.initialize(&[MM_THINK_START_ID]).unwrap();
let delta = push_str(&mut parser, "reason</mm:think>answer");
assert_eq!(reasoning_str(&delta), Some("reason"));
assert_eq!(content_str(&delta), Some("answer"));
}
#[test]
fn minimax_m3_uses_prompt_prefilled_end_marker() {
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap();
parser.initialize(&[MM_THINK_END_ID]).unwrap();
let delta = push_str(&mut parser, "answer");
assert_eq!(delta.reasoning, None);
assert_eq!(content_str(&delta), Some("answer"));
}
#[test]
fn delimited_zero_width_only_piece_is_attributed_to_current_state() {
let tokenizer = Arc::new(fake_tokenizer());
let mut parser =
DelimitedReasoningParser::new(tokenizer, "<think>", "</think>", false).unwrap();
let mut parser = DelimitedReasoningParser::new(tokenizer, "<think>", "</think>").unwrap();
parser.initialize(&[THINK_START_ID]);
// A filtered special token produces a zero-width attribution with no text;
@@ -269,8 +130,7 @@ fn delimited_zero_width_only_piece_is_attributed_to_current_state() {
#[test]
fn delimited_marker_tokens_are_dropped_from_attributions() {
let tokenizer = Arc::new(fake_tokenizer());
let mut parser =
DelimitedReasoningParser::new(tokenizer, "<think>", "</think>", false).unwrap();
let mut parser = DelimitedReasoningParser::new(tokenizer, "<think>", "</think>").unwrap();
let mut collected = CollectedAttributions::default();
for chunk in attributed_chunks(&[