[Rust Frontend] Simplify reasoning parser initialization and test organization (#55411)
Co-authored-by: Codex <noreply@openai.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
@@ -19,7 +19,6 @@ impl CohereCmdReasoningParser {
|
||||
tokenizer,
|
||||
"<|START_THINKING|>",
|
||||
"<|END_THINKING|>",
|
||||
false,
|
||||
)?,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -6,10 +6,6 @@ use vllm_tokenizer::{DecodedText, DynTokenizer};
|
||||
use super::{DelimitedReasoningParser, ReasoningDelta, ReasoningParser, Result};
|
||||
|
||||
/// Reasoning parser for DeepSeek R1 style outputs.
|
||||
///
|
||||
/// DeepSeek R1 may begin generating directly inside a reasoning span and only
|
||||
/// emit the closing `</think>` delimiter, so the no-boundary fallback defaults
|
||||
/// to `in_reasoning = true`.
|
||||
pub struct DeepSeekR1ReasoningParser {
|
||||
inner: DelimitedReasoningParser,
|
||||
}
|
||||
@@ -19,7 +15,7 @@ impl DeepSeekR1ReasoningParser {
|
||||
/// machine.
|
||||
pub fn new(tokenizer: DynTokenizer) -> Result<Self> {
|
||||
Ok(Self {
|
||||
inner: DelimitedReasoningParser::new(tokenizer, "<think>", "</think>", true)?,
|
||||
inner: DelimitedReasoningParser::new(tokenizer, "<think>", "</think>")?,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -45,3 +41,21 @@ impl ReasoningParser for DeepSeekR1ReasoningParser {
|
||||
Ok(self.inner.finish())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::Arc;
|
||||
|
||||
use super::DeepSeekR1ReasoningParser;
|
||||
use crate::reasoning::tests::{content_str, fake_tokenizer, push_str};
|
||||
|
||||
#[test]
|
||||
fn deepseek_r1_without_prompt_markers_expects_start_token() {
|
||||
let tokenizer = Arc::new(fake_tokenizer());
|
||||
let mut parser = DeepSeekR1ReasoningParser::new(tokenizer).unwrap();
|
||||
|
||||
let delta = push_str(&mut parser, "reason</think>answer");
|
||||
assert_eq!(delta.reasoning, None);
|
||||
assert_eq!(content_str(&delta), Some("reason</think>answer"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -24,20 +24,14 @@ pub(crate) struct DelimitedReasoningParser {
|
||||
end_token: String,
|
||||
start_token_id: u32,
|
||||
end_token_id: u32,
|
||||
default_in_reasoning: bool,
|
||||
}
|
||||
|
||||
impl DelimitedReasoningParser {
|
||||
/// Create one delimited parser state machine.
|
||||
///
|
||||
/// `default_in_reasoning` is only used when prompt initialization sees no
|
||||
/// reasoning boundary token at all. If the prompt contains either the
|
||||
/// start or end delimiter, that prompt boundary always wins.
|
||||
pub(crate) fn new(
|
||||
tokenizer: DynTokenizer,
|
||||
start_token: impl Into<String>,
|
||||
end_token: impl Into<String>,
|
||||
default_in_reasoning: bool,
|
||||
) -> Result<Self> {
|
||||
let start_token = start_token.into();
|
||||
let end_token = end_token.into();
|
||||
@@ -54,13 +48,12 @@ impl DelimitedReasoningParser {
|
||||
|
||||
Ok(Self {
|
||||
tokenizer,
|
||||
current_in_reasoning: default_in_reasoning,
|
||||
current_in_reasoning: false,
|
||||
buffer: DecodedText::default(),
|
||||
start_token,
|
||||
end_token,
|
||||
start_token_id,
|
||||
end_token_id,
|
||||
default_in_reasoning,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -72,7 +65,7 @@ impl DelimitedReasoningParser {
|
||||
self.end_token_id,
|
||||
self.tokenizer.as_ref(),
|
||||
)
|
||||
.unwrap_or(self.default_in_reasoning);
|
||||
.unwrap_or(false);
|
||||
}
|
||||
|
||||
/// Return whether the parser is currently inside a reasoning section.
|
||||
|
||||
@@ -18,7 +18,6 @@ impl HyReasoningParser {
|
||||
tokenizer,
|
||||
format!("<think{suffix}>"),
|
||||
format!("</think{suffix}>"),
|
||||
false,
|
||||
)?,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -14,7 +14,7 @@ impl KimiReasoningParser {
|
||||
/// Create a Kimi parser backed by the shared delimited state machine.
|
||||
pub fn new(tokenizer: DynTokenizer) -> Result<Self> {
|
||||
Ok(Self {
|
||||
inner: DelimitedReasoningParser::new(tokenizer, "◁think▷", "◁/think▷", false)?,
|
||||
inner: DelimitedReasoningParser::new(tokenizer, "◁think▷", "◁/think▷")?,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -27,7 +27,7 @@ impl MiniMaxM3ReasoningParser {
|
||||
/// Create a MiniMax M3 parser backed by the shared delimited state machine.
|
||||
pub fn new(tokenizer: DynTokenizer) -> Result<Self> {
|
||||
Ok(Self {
|
||||
inner: DelimitedReasoningParser::new(tokenizer, M3_THINK_START, M3_THINK_END, false)?,
|
||||
inner: DelimitedReasoningParser::new(tokenizer, M3_THINK_START, M3_THINK_END)?,
|
||||
at_response_start: true,
|
||||
leading_end_buffer: DecodedText::default(),
|
||||
})
|
||||
@@ -98,3 +98,77 @@ impl ReasoningParser for MiniMaxM3ReasoningParser {
|
||||
Ok(delta)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::Arc;
|
||||
|
||||
use super::MiniMaxM3ReasoningParser;
|
||||
use crate::reasoning::ReasoningParser;
|
||||
use crate::reasoning::tests::{
|
||||
MM_THINK_END_ID, MM_THINK_START_ID, content_str, fake_tokenizer, push_str, reasoning_str,
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn minimax_m3_handles_explicit_think_delimiters() {
|
||||
let tokenizer = Arc::new(fake_tokenizer());
|
||||
let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap();
|
||||
|
||||
let delta = push_str(&mut parser, "<mm:think>reason</mm:think>answer");
|
||||
assert_eq!(reasoning_str(&delta), Some("reason"));
|
||||
assert_eq!(content_str(&delta), Some("answer"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn minimax_m3_drops_leading_end_marker() {
|
||||
let tokenizer = Arc::new(fake_tokenizer());
|
||||
let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap();
|
||||
|
||||
let delta = push_str(&mut parser, "</mm:think>answer");
|
||||
assert_eq!(delta.reasoning, None);
|
||||
assert_eq!(content_str(&delta), Some("answer"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn minimax_m3_preserves_non_leading_end_marker() {
|
||||
let tokenizer = Arc::new(fake_tokenizer());
|
||||
let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap();
|
||||
|
||||
let delta = push_str(&mut parser, "XXX</mm:think>YYY");
|
||||
assert_eq!(delta.reasoning, None);
|
||||
assert_eq!(content_str(&delta), Some("XXX</mm:think>YYY"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn minimax_m3_drops_split_leading_end_marker() {
|
||||
let tokenizer = Arc::new(fake_tokenizer());
|
||||
let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap();
|
||||
|
||||
assert!(push_str(&mut parser, "</mm").is_empty());
|
||||
let delta = push_str(&mut parser, ":think>answer");
|
||||
assert_eq!(delta.reasoning, None);
|
||||
assert_eq!(content_str(&delta), Some("answer"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn minimax_m3_uses_prompt_prefilled_start_marker() {
|
||||
let tokenizer = Arc::new(fake_tokenizer());
|
||||
let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap();
|
||||
parser.initialize(&[MM_THINK_START_ID]).unwrap();
|
||||
|
||||
let delta = push_str(&mut parser, "reason</mm:think>answer");
|
||||
assert_eq!(reasoning_str(&delta), Some("reason"));
|
||||
assert_eq!(content_str(&delta), Some("answer"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn minimax_m3_uses_prompt_prefilled_end_marker() {
|
||||
let tokenizer = Arc::new(fake_tokenizer());
|
||||
let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap();
|
||||
parser.initialize(&[MM_THINK_END_ID]).unwrap();
|
||||
|
||||
let delta = push_str(&mut parser, "answer");
|
||||
assert_eq!(delta.reasoning, None);
|
||||
assert_eq!(content_str(&delta), Some("answer"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,7 +18,7 @@ impl Qwen3ReasoningParser {
|
||||
/// Create a Qwen3 parser backed by the shared delimited state machine.
|
||||
pub fn new(tokenizer: DynTokenizer) -> Result<Self> {
|
||||
Ok(Self {
|
||||
inner: DelimitedReasoningParser::new(tokenizer, "<think>", "</think>", false)?,
|
||||
inner: DelimitedReasoningParser::new(tokenizer, "<think>", "</think>")?,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -44,3 +44,64 @@ impl ReasoningParser for Qwen3ReasoningParser {
|
||||
Ok(self.inner.finish())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::Arc;
|
||||
|
||||
use super::Qwen3ReasoningParser;
|
||||
use crate::reasoning::ReasoningParser;
|
||||
use crate::reasoning::tests::{
|
||||
SPECIAL_BOUNDARY_ID, THINK_END_ID, THINK_START_ID, content_str, fake_tokenizer, push_str,
|
||||
reasoning_str,
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn qwen3_without_prompt_markers_expects_start_token() {
|
||||
let tokenizer = Arc::new(fake_tokenizer());
|
||||
let mut parser = Qwen3ReasoningParser::new(tokenizer).unwrap();
|
||||
|
||||
let delta = push_str(&mut parser, "reason</think>answer");
|
||||
assert_eq!(delta.reasoning, None);
|
||||
assert_eq!(content_str(&delta), Some("reason</think>answer"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn qwen3_prompt_end_marker_starts_in_content() {
|
||||
let tokenizer = Arc::new(fake_tokenizer());
|
||||
let mut parser = Qwen3ReasoningParser::new(tokenizer).unwrap();
|
||||
parser.initialize(&[THINK_END_ID]).unwrap();
|
||||
|
||||
let delta = push_str(&mut parser, "answer");
|
||||
assert_eq!(delta.reasoning, None);
|
||||
assert_eq!(content_str(&delta), Some("answer"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn qwen3_tolerates_old_and_new_formats() {
|
||||
let tokenizer = Arc::new(fake_tokenizer());
|
||||
|
||||
let mut old_parser = Qwen3ReasoningParser::new(tokenizer.clone()).unwrap();
|
||||
let old = push_str(&mut old_parser, "<think>reason</think>answer");
|
||||
assert_eq!(reasoning_str(&old), Some("reason"));
|
||||
assert_eq!(content_str(&old), Some("answer"));
|
||||
|
||||
let mut new_parser = Qwen3ReasoningParser::new(tokenizer).unwrap();
|
||||
new_parser.initialize(&[THINK_START_ID]).unwrap();
|
||||
let new = push_str(&mut new_parser, "reason</think>answer");
|
||||
assert_eq!(reasoning_str(&new), Some("reason"));
|
||||
assert_eq!(content_str(&new), Some("answer"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn qwen3_stops_scanning_at_last_special_token() {
|
||||
let tokenizer = Arc::new(fake_tokenizer());
|
||||
let mut parser = Qwen3ReasoningParser::new(tokenizer).unwrap();
|
||||
|
||||
parser.initialize(&[THINK_START_ID, SPECIAL_BOUNDARY_ID]).unwrap();
|
||||
|
||||
let delta = push_str(&mut parser, "answer");
|
||||
assert_eq!(delta.reasoning, None);
|
||||
assert_eq!(content_str(&delta), Some("answer"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,12 +15,7 @@ impl SeedOssReasoningParser {
|
||||
/// Create a SeedOSS parser backed by the shared delimited state machine.
|
||||
pub fn new(tokenizer: DynTokenizer) -> Result<Self> {
|
||||
Ok(Self {
|
||||
inner: DelimitedReasoningParser::new(
|
||||
tokenizer,
|
||||
"<seed:think>",
|
||||
"</seed:think>",
|
||||
false,
|
||||
)?,
|
||||
inner: DelimitedReasoningParser::new(tokenizer, "<seed:think>", "</seed:think>")?,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -26,7 +26,7 @@ impl Step3p5ReasoningParser {
|
||||
/// Create a Step3p5 parser backed by the shared delimited state machine.
|
||||
pub fn new(tokenizer: DynTokenizer) -> Result<Self> {
|
||||
Ok(Self {
|
||||
inner: DelimitedReasoningParser::new(tokenizer, "<think>", "</think>", false)?,
|
||||
inner: DelimitedReasoningParser::new(tokenizer, "<think>", "</think>")?,
|
||||
pending_reasoning_newline: None,
|
||||
just_ended_reasoning: false,
|
||||
})
|
||||
|
||||
@@ -57,8 +57,7 @@ pub(crate) fn content_str(delta: &ReasoningDelta) -> Option<&str> {
|
||||
#[test]
|
||||
fn delimited_content_only_stream() {
|
||||
let tokenizer = Arc::new(fake_tokenizer());
|
||||
let mut parser =
|
||||
DelimitedReasoningParser::new(tokenizer, "<think>", "</think>", false).unwrap();
|
||||
let mut parser = DelimitedReasoningParser::new(tokenizer, "<think>", "</think>").unwrap();
|
||||
|
||||
let delta = parser.push(DecodedText::unattributed("plain content"));
|
||||
assert_eq!(content_str(&delta), Some("plain content"));
|
||||
@@ -67,8 +66,7 @@ fn delimited_content_only_stream() {
|
||||
#[test]
|
||||
fn delimited_single_chunk_with_reasoning_and_content() {
|
||||
let tokenizer = Arc::new(fake_tokenizer());
|
||||
let mut parser =
|
||||
DelimitedReasoningParser::new(tokenizer, "<think>", "</think>", false).unwrap();
|
||||
let mut parser = DelimitedReasoningParser::new(tokenizer, "<think>", "</think>").unwrap();
|
||||
|
||||
let delta = parser.push(DecodedText::unattributed("<think>reason</think>answer"));
|
||||
assert_eq!(reasoning_str(&delta), Some("reason"));
|
||||
@@ -78,8 +76,7 @@ fn delimited_single_chunk_with_reasoning_and_content() {
|
||||
#[test]
|
||||
fn delimited_partial_tokens_across_chunks() {
|
||||
let tokenizer = Arc::new(fake_tokenizer());
|
||||
let mut parser =
|
||||
DelimitedReasoningParser::new(tokenizer, "<think>", "</think>", false).unwrap();
|
||||
let mut parser = DelimitedReasoningParser::new(tokenizer, "<think>", "</think>").unwrap();
|
||||
|
||||
assert!(parser.push(DecodedText::unattributed("<thi")).is_empty());
|
||||
let delta = parser.push(DecodedText::unattributed("nk>reason</think>answer"));
|
||||
@@ -90,8 +87,7 @@ fn delimited_partial_tokens_across_chunks() {
|
||||
#[test]
|
||||
fn delimited_finish_flushes_buffer() {
|
||||
let tokenizer = Arc::new(fake_tokenizer());
|
||||
let mut parser =
|
||||
DelimitedReasoningParser::new(tokenizer, "<think>", "</think>", false).unwrap();
|
||||
let mut parser = DelimitedReasoningParser::new(tokenizer, "<think>", "</think>").unwrap();
|
||||
parser.initialize(&[THINK_START_ID]);
|
||||
|
||||
let delta = parser.push(DecodedText::unattributed("unfinished</thi"));
|
||||
@@ -100,145 +96,10 @@ fn delimited_finish_flushes_buffer() {
|
||||
assert_eq!(reasoning_str(&final_delta), Some("</thi"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn qwen3_without_prompt_markers_expects_start_token() {
|
||||
let tokenizer = Arc::new(fake_tokenizer());
|
||||
let mut parser = Qwen3ReasoningParser::new(tokenizer).unwrap();
|
||||
|
||||
let delta = push_str(&mut parser, "reason</think>answer");
|
||||
assert_eq!(delta.reasoning, None);
|
||||
assert_eq!(content_str(&delta), Some("reason</think>answer"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn qwen3_prompt_end_marker_starts_in_content() {
|
||||
let tokenizer = Arc::new(fake_tokenizer());
|
||||
let mut parser = Qwen3ReasoningParser::new(tokenizer).unwrap();
|
||||
parser.initialize(&[THINK_END_ID]).unwrap();
|
||||
|
||||
let delta = push_str(&mut parser, "answer");
|
||||
assert_eq!(delta.reasoning, None);
|
||||
assert_eq!(content_str(&delta), Some("answer"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn qwen3_tolerates_old_and_new_formats() {
|
||||
let tokenizer = Arc::new(fake_tokenizer());
|
||||
|
||||
let mut old_parser = Qwen3ReasoningParser::new(tokenizer.clone()).unwrap();
|
||||
let old = push_str(&mut old_parser, "<think>reason</think>answer");
|
||||
assert_eq!(reasoning_str(&old), Some("reason"));
|
||||
assert_eq!(content_str(&old), Some("answer"));
|
||||
|
||||
let mut new_parser = Qwen3ReasoningParser::new(tokenizer).unwrap();
|
||||
new_parser.initialize(&[THINK_START_ID]).unwrap();
|
||||
let new = push_str(&mut new_parser, "reason</think>answer");
|
||||
assert_eq!(reasoning_str(&new), Some("reason"));
|
||||
assert_eq!(content_str(&new), Some("answer"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn qwen3_stops_scanning_at_last_special_token() {
|
||||
let tokenizer = Arc::new(fake_tokenizer());
|
||||
let mut parser = Qwen3ReasoningParser::new(tokenizer).unwrap();
|
||||
|
||||
parser.initialize(&[THINK_START_ID, SPECIAL_BOUNDARY_ID]).unwrap();
|
||||
|
||||
let delta = push_str(&mut parser, "answer");
|
||||
assert_eq!(delta.reasoning, None);
|
||||
assert_eq!(content_str(&delta), Some("answer"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deepseek_r1_defaults_to_reasoning_without_prompt_boundary() {
|
||||
let tokenizer = Arc::new(fake_tokenizer());
|
||||
let mut parser = DeepSeekR1ReasoningParser::new(tokenizer).unwrap();
|
||||
|
||||
let delta = push_str(&mut parser, "reason</think>answer");
|
||||
assert_eq!(reasoning_str(&delta), Some("reason"));
|
||||
assert_eq!(content_str(&delta), Some("answer"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deepseek_r1_stops_scanning_at_last_special_token() {
|
||||
let tokenizer = Arc::new(fake_tokenizer());
|
||||
let mut parser = DeepSeekR1ReasoningParser::new(tokenizer).unwrap();
|
||||
|
||||
parser.initialize(&[THINK_END_ID, SPECIAL_BOUNDARY_ID]).unwrap();
|
||||
|
||||
let delta = push_str(&mut parser, "reason</think>answer");
|
||||
assert_eq!(reasoning_str(&delta), Some("reason"));
|
||||
assert_eq!(content_str(&delta), Some("answer"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn minimax_m3_handles_explicit_think_delimiters() {
|
||||
let tokenizer = Arc::new(fake_tokenizer());
|
||||
let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap();
|
||||
|
||||
let delta = push_str(&mut parser, "<mm:think>reason</mm:think>answer");
|
||||
assert_eq!(reasoning_str(&delta), Some("reason"));
|
||||
assert_eq!(content_str(&delta), Some("answer"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn minimax_m3_drops_leading_end_marker() {
|
||||
let tokenizer = Arc::new(fake_tokenizer());
|
||||
let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap();
|
||||
|
||||
let delta = push_str(&mut parser, "</mm:think>answer");
|
||||
assert_eq!(delta.reasoning, None);
|
||||
assert_eq!(content_str(&delta), Some("answer"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn minimax_m3_preserves_non_leading_end_marker() {
|
||||
let tokenizer = Arc::new(fake_tokenizer());
|
||||
let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap();
|
||||
|
||||
let delta = push_str(&mut parser, "XXX</mm:think>YYY");
|
||||
assert_eq!(delta.reasoning, None);
|
||||
assert_eq!(content_str(&delta), Some("XXX</mm:think>YYY"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn minimax_m3_drops_split_leading_end_marker() {
|
||||
let tokenizer = Arc::new(fake_tokenizer());
|
||||
let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap();
|
||||
|
||||
assert!(push_str(&mut parser, "</mm").is_empty());
|
||||
let delta = push_str(&mut parser, ":think>answer");
|
||||
assert_eq!(delta.reasoning, None);
|
||||
assert_eq!(content_str(&delta), Some("answer"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn minimax_m3_uses_prompt_prefilled_start_marker() {
|
||||
let tokenizer = Arc::new(fake_tokenizer());
|
||||
let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap();
|
||||
parser.initialize(&[MM_THINK_START_ID]).unwrap();
|
||||
|
||||
let delta = push_str(&mut parser, "reason</mm:think>answer");
|
||||
assert_eq!(reasoning_str(&delta), Some("reason"));
|
||||
assert_eq!(content_str(&delta), Some("answer"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn minimax_m3_uses_prompt_prefilled_end_marker() {
|
||||
let tokenizer = Arc::new(fake_tokenizer());
|
||||
let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap();
|
||||
parser.initialize(&[MM_THINK_END_ID]).unwrap();
|
||||
|
||||
let delta = push_str(&mut parser, "answer");
|
||||
assert_eq!(delta.reasoning, None);
|
||||
assert_eq!(content_str(&delta), Some("answer"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delimited_zero_width_only_piece_is_attributed_to_current_state() {
|
||||
let tokenizer = Arc::new(fake_tokenizer());
|
||||
let mut parser =
|
||||
DelimitedReasoningParser::new(tokenizer, "<think>", "</think>", false).unwrap();
|
||||
let mut parser = DelimitedReasoningParser::new(tokenizer, "<think>", "</think>").unwrap();
|
||||
parser.initialize(&[THINK_START_ID]);
|
||||
|
||||
// A filtered special token produces a zero-width attribution with no text;
|
||||
@@ -269,8 +130,7 @@ fn delimited_zero_width_only_piece_is_attributed_to_current_state() {
|
||||
#[test]
|
||||
fn delimited_marker_tokens_are_dropped_from_attributions() {
|
||||
let tokenizer = Arc::new(fake_tokenizer());
|
||||
let mut parser =
|
||||
DelimitedReasoningParser::new(tokenizer, "<think>", "</think>", false).unwrap();
|
||||
let mut parser = DelimitedReasoningParser::new(tokenizer, "<think>", "</think>").unwrap();
|
||||
|
||||
let mut collected = CollectedAttributions::default();
|
||||
for chunk in attributed_chunks(&[
|
||||
|
||||
Reference in New Issue
Block a user