diff --git a/rust/src/chat/src/lib.rs b/rust/src/chat/src/lib.rs index 85b98c4a19..756a1851ab 100644 --- a/rust/src/chat/src/lib.rs +++ b/rust/src/chat/src/lib.rs @@ -416,6 +416,6 @@ mod tests { ) .unwrap_err(); - expect_test::expect!["reasoning parser `definitely_missing_reasoning_parser` is not registered (choose from: cohere_cmd, deepseek_r1, deepseek_v3, deepseek_v4, gemma4, glm45, hy_v3, hy_v4, inkling, kimi, kimi_k2, kimi_k3, minimax_m2, minimax_m3, nemotron_v3, qwen3, seed_oss, step3, step3p5)"].assert_eq(&error.to_report_string()); + expect_test::expect!["reasoning parser `definitely_missing_reasoning_parser` is not registered (choose from: cohere_cmd, deepseek_r1, deepseek_v3, deepseek_v4, gemma4, glm45, glm47, hy_v3, hy_v4, inkling, kimi, kimi_k2, kimi_k3, minimax_m2, minimax_m3, nemotron_v3, qwen3, seed_oss, step3, step3p5)"].assert_eq(&error.to_report_string()); } } diff --git a/rust/src/chat/src/parser/reasoning/mod.rs b/rust/src/chat/src/parser/reasoning/mod.rs index 6942d05371..461e34b22f 100644 --- a/rust/src/chat/src/parser/reasoning/mod.rs +++ b/rust/src/chat/src/parser/reasoning/mod.rs @@ -7,10 +7,10 @@ use std::sync::{Arc, LazyLock}; pub use vllm_parser::reasoning::{ CohereCmdReasoningParser, DeepSeekR1ReasoningParser, DeepSeekV3ReasoningParser, - DeepSeekV4ReasoningParser, Glm45ReasoningParser, KimiK2ReasoningParser, KimiReasoningParser, - MiniMaxM2ReasoningParser, MiniMaxM3ReasoningParser, NemotronV3ReasoningParser, - Qwen3ReasoningParser, ReasoningDelta, ReasoningError, ReasoningParser, SeedOssReasoningParser, - Step3ReasoningParser, Step3p5ReasoningParser, + DeepSeekV4ReasoningParser, Glm45ReasoningParser, Glm47ReasoningParser, KimiK2ReasoningParser, + KimiReasoningParser, MiniMaxM2ReasoningParser, MiniMaxM3ReasoningParser, + NemotronV3ReasoningParser, Qwen3ReasoningParser, ReasoningDelta, ReasoningError, + ReasoningParser, SeedOssReasoningParser, Step3ReasoningParser, Step3p5ReasoningParser, }; use vllm_tokenizer::DynTokenizer; @@ -25,6 +25,7 @@ pub mod names { pub const GEMMA4: &str = "gemma4"; pub const INKLING: &str = "inkling"; pub const GLM45: &str = "glm45"; + pub const GLM47: &str = "glm47"; pub const HY_V3: &str = "hy_v3"; pub const HY_V4: &str = "hy_v4"; pub const KIMI: &str = "kimi"; @@ -69,6 +70,7 @@ impl ReasoningParserFactory { .register_unified_dummy(names::GEMMA4) .register_unified_dummy(names::INKLING) .register_parser::(names::GLM45) + .register_parser::(names::GLM47) .register_unified_dummy(names::HY_V3) .register_unified_dummy(names::HY_V4) .register_parser::(names::KIMI) @@ -91,8 +93,8 @@ impl ReasoningParserFactory { .register_pattern("gemma4", names::GEMMA4) .register_pattern("qwq", names::DEEPSEEK_R1) .register_pattern("qwen3", names::QWEN3) - .register_pattern("glm-5", names::GLM45) - .register_pattern("glm-4.7", names::GLM45) + .register_pattern("glm-5", names::GLM47) + .register_pattern("glm-4.7", names::GLM47) .register_pattern("glm-4.6", names::GLM45) .register_pattern("glm-4.5", names::GLM45) .register_pattern("hy3", names::HY_V3) diff --git a/rust/src/chat/src/parser/reasoning/tests.rs b/rust/src/chat/src/parser/reasoning/tests.rs index 37a20f8e4f..e392f40679 100644 --- a/rust/src/chat/src/parser/reasoning/tests.rs +++ b/rust/src/chat/src/parser/reasoning/tests.rs @@ -25,7 +25,7 @@ fn factory_contains_and_lists_registered_parsers() { } #[test] -fn factory_resolves_deepseek_v4_to_qwen3_alias() { +fn factory_resolves_deepseek_v4() { let factory = ReasoningParserFactory::new(); assert_eq!( factory.resolve_name_for_model("deepseek-ai/DeepSeek-V4"), @@ -112,3 +112,14 @@ fn factory_rejects_unknown_parser_names() { }; assert!(error.to_string().contains("choose from")); } + +#[test] +fn factory_distinguishes_glm_reasoning_framing() { + let factory = ReasoningParserFactory::new(); + for model in ["zai-org/GLM-4.5", "zai-org/GLM-4.6"] { + assert_eq!(factory.resolve_name_for_model(model), Some(names::GLM45)); + } + for model in ["zai-org/GLM-4.7-Flash", "zai-org/GLM-5.2-FP8"] { + assert_eq!(factory.resolve_name_for_model(model), Some(names::GLM47)); + } +} diff --git a/rust/src/chat/tests/roundtrip.rs b/rust/src/chat/tests/roundtrip.rs index 19aec82e45..9f5577da64 100644 --- a/rust/src/chat/tests/roundtrip.rs +++ b/rust/src/chat/tests/roundtrip.rs @@ -445,10 +445,12 @@ async fn run_roundtrip_reasoning_and_content_inner( let result = run_roundtrip(case, backends, &request, assistant).await?; assert_eq!( - result.parsed_message.reasoning().as_deref().map(str::trim), - effective_thinking.then_some(expected_reasoning) + result.parsed_message.reasoning().as_deref(), + effective_thinking.then_some(expected_reasoning), + "parsed message: {:#?}", + result.parsed_message ); - assert_eq!(result.parsed_message.text().trim(), expected_text); + assert_eq!(result.parsed_message.text(), expected_text); assert_eq!(result.parsed_message.tool_calls().count(), 0); assert_eq!( @@ -508,10 +510,12 @@ async fn run_roundtrip_tool_call_mix( .await?; assert_eq!( - result.parsed_message.reasoning().as_deref().map(str::trim), - Some(expected_reasoning) + result.parsed_message.reasoning().as_deref(), + Some(expected_reasoning), + "parsed message: {:#?}", + result.parsed_message ); - assert_eq!(result.parsed_message.text().trim(), expected_text); + assert_eq!(result.parsed_message.text(), expected_text); let tool_calls = result.parsed_message.tool_calls().collect::>(); assert_eq!( @@ -714,11 +718,7 @@ async fn parse_completion( while let Some(event) = events.next().await { if let ChatEvent::Done { message, .. } = event? { - // TODO: currently our parsers are not very strict about preserving or trimming - // whitespace, so we trim here to avoid roundtrip failures due to - // insignificant whitespace differences. However, this may hurt token-level - // fidelity so we should consider improving them. - return Ok(message.trim()); + return Ok(message); } } diff --git a/rust/src/parser/src/reasoning/cohere_cmd.rs b/rust/src/parser/src/reasoning/cohere_cmd.rs index 071b7be8f2..ce1173953e 100644 --- a/rust/src/parser/src/reasoning/cohere_cmd.rs +++ b/rust/src/parser/src/reasoning/cohere_cmd.rs @@ -3,7 +3,10 @@ use vllm_tokenizer::{DecodedText, DynTokenizer}; -use super::{DelimitedReasoningParser, ReasoningDelta, ReasoningParser, Result}; +use super::{ + DelimitedReasoningParser, DelimitedReasoningParserBuilder, ReasoningDelta, ReasoningParser, + Result, +}; /// Reasoning parser for Cohere Command models that use explicit START/END tags. pub struct CohereCmdReasoningParser { @@ -15,11 +18,12 @@ impl CohereCmdReasoningParser { /// machine. pub fn new(tokenizer: DynTokenizer) -> Result { Ok(Self { - inner: DelimitedReasoningParser::new( + inner: DelimitedReasoningParserBuilder::new( tokenizer, "<|START_THINKING|>", "<|END_THINKING|>", - )?, + ) + .build()?, }) } } @@ -33,8 +37,7 @@ impl ReasoningParser for CohereCmdReasoningParser { } fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> { - self.inner.initialize(prompt_token_ids); - Ok(()) + self.inner.initialize(prompt_token_ids) } fn push(&mut self, delta: DecodedText) -> Result { diff --git a/rust/src/parser/src/reasoning/deepseek_r1.rs b/rust/src/parser/src/reasoning/deepseek_r1.rs index 3ef7c58da0..16c2e8c8dc 100644 --- a/rust/src/parser/src/reasoning/deepseek_r1.rs +++ b/rust/src/parser/src/reasoning/deepseek_r1.rs @@ -3,7 +3,10 @@ use vllm_tokenizer::{DecodedText, DynTokenizer}; -use super::{DelimitedReasoningParser, ReasoningDelta, ReasoningParser, Result}; +use super::{ + DelimitedReasoningParser, DelimitedReasoningParserBuilder, ReasoningDelta, ReasoningParser, + Result, +}; /// Reasoning parser for DeepSeek R1 style outputs. pub struct DeepSeekR1ReasoningParser { @@ -11,11 +14,12 @@ pub struct DeepSeekR1ReasoningParser { } impl DeepSeekR1ReasoningParser { - /// Create a DeepSeek R1 parser backed by the shared delimited state - /// machine. + /// Create a DeepSeek R1 parser backed by the shared delimited state machine. pub fn new(tokenizer: DynTokenizer) -> Result { Ok(Self { - inner: DelimitedReasoningParser::new(tokenizer, "", "")?, + inner: DelimitedReasoningParserBuilder::new(tokenizer, "", "") + .with_after_start("\n") + .build()?, }) } } @@ -29,8 +33,7 @@ impl ReasoningParser for DeepSeekR1ReasoningParser { } fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> { - self.inner.initialize(prompt_token_ids); - Ok(()) + self.inner.initialize(prompt_token_ids) } fn push(&mut self, delta: DecodedText) -> Result { diff --git a/rust/src/parser/src/reasoning/deepseek_v3.rs b/rust/src/parser/src/reasoning/deepseek_v3.rs new file mode 100644 index 0000000000..8b053c89f4 --- /dev/null +++ b/rust/src/parser/src/reasoning/deepseek_v3.rs @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +use vllm_tokenizer::{DecodedText, DynTokenizer}; + +use super::{ + DelimitedReasoningParser, DelimitedReasoningParserBuilder, ReasoningDelta, ReasoningParser, + Result, +}; + +/// Reasoning parser for DeepSeek V3 style outputs. +pub struct DeepSeekV3ReasoningParser { + inner: DelimitedReasoningParser, +} + +impl DeepSeekV3ReasoningParser { + /// Create a DeepSeek V3 parser backed by the shared delimited state machine. + pub fn new(tokenizer: DynTokenizer) -> Result { + Ok(Self { + inner: DelimitedReasoningParserBuilder::new(tokenizer, "", "") + .build()?, + }) + } +} + +impl ReasoningParser for DeepSeekV3ReasoningParser { + fn create(tokenizer: DynTokenizer) -> Result> + where + Self: Sized + 'static, + { + Ok(Box::new(Self::new(tokenizer)?)) + } + + fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> { + self.inner.initialize(prompt_token_ids) + } + + fn push(&mut self, delta: DecodedText) -> Result { + Ok(self.inner.push(delta)) + } + + fn finish(&mut self) -> Result { + Ok(self.inner.finish()) + } +} diff --git a/rust/src/parser/src/reasoning/delimited.rs b/rust/src/parser/src/reasoning/delimited.rs index b6065afc52..986ce2cc25 100644 --- a/rust/src/parser/src/reasoning/delimited.rs +++ b/rust/src/parser/src/reasoning/delimited.rs @@ -4,6 +4,90 @@ use vllm_tokenizer::{DecodedText, DynTokenizer, Tokenizer}; use super::{ReasoningDelta, ReasoningError, Result}; +use crate::utils::partial_prefix_len; + +/// Build a delimited reasoning parser with fixed text at each boundary. +pub(crate) struct DelimitedReasoningParserBuilder { + tokenizer: DynTokenizer, + start_token: String, + end_token: String, + before_start: &'static str, + after_start: &'static str, + before_end: &'static str, + after_end: &'static str, +} + +impl DelimitedReasoningParserBuilder { + /// Configure the tokenizer and reasoning delimiters. + pub(crate) fn new( + tokenizer: DynTokenizer, + start_token: impl Into, + end_token: impl Into, + ) -> Self { + Self { + tokenizer, + start_token: start_token.into(), + end_token: end_token.into(), + before_start: "", + after_start: "", + before_end: "", + after_end: "", + } + } + + /// Include a fixed prefix when it immediately precedes the start marker. + pub(crate) fn with_before_start(mut self, before_start: &'static str) -> Self { + self.before_start = before_start; + self + } + + /// Consume fixed text immediately after the start marker. + pub(crate) fn with_after_start(mut self, after_start: &'static str) -> Self { + self.after_start = after_start; + self + } + + /// Include a fixed prefix when it immediately precedes the end marker. + pub(crate) fn with_before_end(mut self, before_end: &'static str) -> Self { + self.before_end = before_end; + self + } + + /// Consume fixed text immediately after the end marker. + pub(crate) fn with_after_end(mut self, after_end: &'static str) -> Self { + self.after_end = after_end; + self + } + + /// Create one delimited parser state machine. + pub(crate) fn build(self) -> Result { + let start_token_id = self.tokenizer.token_to_id(&self.start_token).ok_or_else(|| { + ReasoningError::MissingToken { + token: self.start_token.clone(), + } + })?; + let end_token_id = self.tokenizer.token_to_id(&self.end_token).ok_or_else(|| { + ReasoningError::MissingToken { + token: self.end_token.clone(), + } + })?; + + Ok(DelimitedReasoningParser { + tokenizer: self.tokenizer, + framed_start_token: format!("{}{}", self.before_start, self.start_token), + start_token: self.start_token, + framed_end_token: format!("{}{}", self.before_end, self.end_token), + end_token: self.end_token, + start_token_id, + end_token_id, + after_start: self.after_start, + after_end: self.after_end, + current_in_reasoning: false, + buffer: DecodedText::default(), + pending_after: "", + }) + } +} /// Shared incremental state machine for tag-delimited reasoning protocols. /// @@ -18,54 +102,45 @@ use super::{ReasoningDelta, ReasoningError, Result}; /// prefill different prompts. pub(crate) struct DelimitedReasoningParser { tokenizer: DynTokenizer, - current_in_reasoning: bool, - buffer: DecodedText, start_token: String, + framed_start_token: String, end_token: String, start_token_id: u32, end_token_id: u32, + after_start: &'static str, + after_end: &'static str, + framed_end_token: String, + + // Mutable state. + current_in_reasoning: bool, + buffer: DecodedText, + pending_after: &'static str, } impl DelimitedReasoningParser { - /// Create one delimited parser state machine. - pub(crate) fn new( - tokenizer: DynTokenizer, - start_token: impl Into, - end_token: impl Into, - ) -> Result { - let start_token = start_token.into(); - let end_token = end_token.into(); - let start_token_id = - tokenizer - .token_to_id(&start_token) - .ok_or_else(|| ReasoningError::MissingToken { - token: start_token.clone(), - })?; - let end_token_id = - tokenizer.token_to_id(&end_token).ok_or_else(|| ReasoningError::MissingToken { - token: end_token.clone(), - })?; - - Ok(Self { - tokenizer, - current_in_reasoning: false, - buffer: DecodedText::default(), - start_token, - end_token, - start_token_id, - end_token_id, - }) - } - - /// Initialize the starting state from prompt token IDs. - pub(crate) fn initialize(&mut self, prompt_token_ids: &[u32]) { - self.current_in_reasoning = last_reasoning_boundary( + /// Initialize the starting state and remaining framing from prompt tokens. + pub(crate) fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> { + self.buffer = DecodedText::default(); + self.current_in_reasoning = false; + self.pending_after = ""; + if let Some((index, in_reasoning)) = last_reasoning_boundary_position( prompt_token_ids, self.start_token_id, self.end_token_id, self.tokenizer.as_ref(), - ) - .unwrap_or(false); + ) { + self.current_in_reasoning = in_reasoning; + self.pending_after = if in_reasoning { + self.after_start + } else { + self.after_end + }; + if !self.pending_after.is_empty() && index + 1 < prompt_token_ids.len() { + let tail = self.tokenizer.decode(&prompt_token_ids[index + 1..], false)?; + self.pending_after = self.pending_after.strip_prefix(&tail).unwrap_or(""); + } + } + Ok(()) } /// Return whether the parser is currently inside a reasoning section. @@ -76,84 +151,74 @@ impl DelimitedReasoningParser { /// Parse one decoded text delta and return its reasoning/content split. pub(crate) fn push(&mut self, delta: DecodedText) -> ReasoningDelta { self.buffer.append(delta); - - let partial_suffix_len = self.partial_suffix_len(&self.buffer.text); - let stable_len = self.buffer.text.len() - partial_suffix_len; - let stable = self.buffer.drain_prefix(stable_len); - - self.parse_stable_text(stable) + self.parse_buffer(false) } - /// Flush any buffered partial delimiter suffix at end of stream. + /// Flush partial framing and delimiters as body text at end of stream. pub(crate) fn finish(&mut self) -> ReasoningDelta { - // `drain_prefix(text.len())` takes trailing zero-width tokens too. - let stable = self.buffer.drain_prefix(self.buffer.text.len()); - self.parse_stable_text(stable) + self.parse_buffer(true) } - /// Parse text that is known not to end with a partial delimiter suffix. - /// /// Reasoning and content pieces keep the attributions of the tokens that /// produced them; delimiter marker spans are drained and dropped, keeping /// marker tokens out of any count. - fn parse_stable_text(&mut self, mut stable: DecodedText) -> ReasoningDelta { + fn parse_buffer(&mut self, finishing: bool) -> ReasoningDelta { let mut delta = ReasoningDelta::default(); - - while !stable.text.is_empty() { - if self.current_in_reasoning { - if let Some(end_idx) = stable.text.find(&self.end_token) { - delta.push_reasoning(stable.drain_prefix(end_idx)); - let _ = stable.drain_prefix(self.end_token.len()); - self.current_in_reasoning = false; - } else { - delta.push_reasoning(stable); - return delta; + loop { + if !self.pending_after.is_empty() { + if self.buffer.text.starts_with(self.pending_after) { + let _ = self.buffer.drain_prefix(self.pending_after.len()); + } else if !finishing && self.pending_after.starts_with(&self.buffer.text) { + break; } - } else if let Some(start_idx) = stable.text.find(&self.start_token) { - delta.push_content(stable.drain_prefix(start_idx)); - let _ = stable.drain_prefix(self.start_token.len()); - self.current_in_reasoning = true; - } else { - delta.push_content(stable); - return delta; + // A mismatch preserves the candidate as body text. + self.pending_after = ""; } - } - // A remainder with empty text may still carry zero-width tokens; - // attribute them to the current state. - if !stable.attributions.is_empty() { - if self.current_in_reasoning { - delta.push_reasoning(stable); + let (marker, framed_marker) = if self.current_in_reasoning { + (&self.end_token, &self.framed_end_token) } else { - delta.push_content(stable); + (&self.start_token, &self.framed_start_token) + }; + if let Some(index) = self.buffer.text.find(marker) { + let before = &framed_marker[..framed_marker.len() - marker.len()]; + let body_len = if self.buffer.text[..index].ends_with(before) { + index - before.len() + } else { + index + }; + let boundary_len = index + marker.len() - body_len; + let body = self.buffer.drain_prefix(body_len); + self.push_body(&mut delta, body); + let _ = self.buffer.drain_prefix(boundary_len); + self.current_in_reasoning = !self.current_in_reasoning; + self.pending_after = if self.current_in_reasoning { + self.after_start + } else { + self.after_end + }; + continue; } - } + let keep_len = if finishing { + 0 + } else { + partial_prefix_len(&self.buffer.text, marker) + .max(partial_prefix_len(&self.buffer.text, framed_marker)) + }; + let body = self.buffer.drain_prefix(self.buffer.text.len() - keep_len); + self.push_body(&mut delta, body); + break; + } delta } - /// Return the longest trailing suffix that could still complete a - /// delimiter. - fn partial_suffix_len(&self, text: &str) -> usize { - let mut best = 0; - for idx in text.char_indices().map(|(idx, _)| idx).skip(1) { - let suffix = &text[idx..]; - if self.start_token.starts_with(suffix) && self.start_token != suffix { - best = best.max(text.len() - idx); - } - if self.end_token.starts_with(suffix) && self.end_token != suffix { - best = best.max(text.len() - idx); - } + fn push_body(&self, delta: &mut ReasoningDelta, body: DecodedText) { + if self.current_in_reasoning { + delta.push_reasoning(body); + } else { + delta.push_content(body); } - - if self.start_token.starts_with(text) && self.start_token != text { - best = best.max(text.len()); - } - if self.end_token.starts_with(text) && self.end_token != text { - best = best.max(text.len()); - } - - best } } @@ -164,12 +229,22 @@ pub(crate) fn last_reasoning_boundary( end_token_id: u32, tokenizer: &dyn Tokenizer, ) -> Option { - for token_id in prompt_token_ids.iter().rev().copied() { + last_reasoning_boundary_position(prompt_token_ids, start_token_id, end_token_id, tokenizer) + .map(|(_, in_reasoning)| in_reasoning) +} + +fn last_reasoning_boundary_position( + prompt_token_ids: &[u32], + start_token_id: u32, + end_token_id: u32, + tokenizer: &dyn Tokenizer, +) -> Option<(usize, bool)> { + for (index, &token_id) in prompt_token_ids.iter().enumerate().rev() { if token_id == start_token_id { - return Some(true); + return Some((index, true)); } if token_id == end_token_id { - return Some(false); + return Some((index, false)); } if tokenizer.is_special_id(token_id) { return None; diff --git a/rust/src/parser/src/reasoning/glm45.rs b/rust/src/parser/src/reasoning/glm45.rs new file mode 100644 index 0000000000..557e628b26 --- /dev/null +++ b/rust/src/parser/src/reasoning/glm45.rs @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +use vllm_tokenizer::{DecodedText, DynTokenizer}; + +use super::{ + DelimitedReasoningParser, DelimitedReasoningParserBuilder, ReasoningDelta, ReasoningParser, + Result, +}; + +/// Reasoning parser for GLM-4.5/4.6 models using ``/`` +/// delimiters. +pub struct Glm45ReasoningParser { + inner: DelimitedReasoningParser, +} + +impl Glm45ReasoningParser { + /// Create a GLM-4.5/4.6 parser backed by the shared delimited state machine. + pub fn new(tokenizer: DynTokenizer) -> Result { + Ok(Self { + inner: DelimitedReasoningParserBuilder::new(tokenizer, "", "") + .with_before_start("\n") + .with_after_end("\n") + .build()?, + }) + } +} + +impl ReasoningParser for Glm45ReasoningParser { + fn create(tokenizer: DynTokenizer) -> Result> + where + Self: Sized + 'static, + { + Ok(Box::new(Self::new(tokenizer)?)) + } + + fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> { + self.inner.initialize(prompt_token_ids) + } + + fn push(&mut self, delta: DecodedText) -> Result { + Ok(self.inner.push(delta)) + } + + fn finish(&mut self) -> Result { + Ok(self.inner.finish()) + } +} diff --git a/rust/src/parser/src/reasoning/hy.rs b/rust/src/parser/src/reasoning/hy.rs index b4c56c3bb0..4a0ae12232 100644 --- a/rust/src/parser/src/reasoning/hy.rs +++ b/rust/src/parser/src/reasoning/hy.rs @@ -3,7 +3,10 @@ use vllm_tokenizer::{DecodedText, DynTokenizer}; -use super::{DelimitedReasoningParser, ReasoningDelta, ReasoningError, ReasoningParser, Result}; +use super::{ + DelimitedReasoningParser, DelimitedReasoningParserBuilder, ReasoningDelta, ReasoningError, + ReasoningParser, Result, +}; /// Internal HY reasoning stage used by the unified HY parsers. pub(crate) struct HyReasoningParser { @@ -14,11 +17,12 @@ impl HyReasoningParser { /// Create a HY reasoning parser for the tokenizer-specific marker suffix. pub(crate) fn new(tokenizer: DynTokenizer, suffix: &str) -> Result { Ok(Self { - inner: DelimitedReasoningParser::new( + inner: DelimitedReasoningParserBuilder::new( tokenizer, format!(""), format!(""), - )?, + ) + .build()?, }) } } @@ -36,8 +40,7 @@ impl ReasoningParser for HyReasoningParser { } fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> { - self.inner.initialize(prompt_token_ids); - Ok(()) + self.inner.initialize(prompt_token_ids) } fn push(&mut self, delta: DecodedText) -> Result { diff --git a/rust/src/parser/src/reasoning/kimi.rs b/rust/src/parser/src/reasoning/kimi.rs index 769ab81c62..58de57691f 100644 --- a/rust/src/parser/src/reasoning/kimi.rs +++ b/rust/src/parser/src/reasoning/kimi.rs @@ -3,7 +3,10 @@ use vllm_tokenizer::{DecodedText, DynTokenizer}; -use super::{DelimitedReasoningParser, ReasoningDelta, ReasoningParser, Result}; +use super::{ + DelimitedReasoningParser, DelimitedReasoningParserBuilder, ReasoningDelta, ReasoningParser, + Result, +}; /// Reasoning parser for legacy Kimi models that use Unicode thinking tags. pub struct KimiReasoningParser { @@ -14,7 +17,8 @@ impl KimiReasoningParser { /// Create a Kimi parser backed by the shared delimited state machine. pub fn new(tokenizer: DynTokenizer) -> Result { Ok(Self { - inner: DelimitedReasoningParser::new(tokenizer, "◁think▷", "◁/think▷")?, + inner: DelimitedReasoningParserBuilder::new(tokenizer, "◁think▷", "◁/think▷") + .build()?, }) } } @@ -28,8 +32,7 @@ impl ReasoningParser for KimiReasoningParser { } fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> { - self.inner.initialize(prompt_token_ids); - Ok(()) + self.inner.initialize(prompt_token_ids) } fn push(&mut self, delta: DecodedText) -> Result { diff --git a/rust/src/parser/src/reasoning/minimax_m3.rs b/rust/src/parser/src/reasoning/minimax_m3.rs index 14b8ef7fbd..bb0bd5b2a9 100644 --- a/rust/src/parser/src/reasoning/minimax_m3.rs +++ b/rust/src/parser/src/reasoning/minimax_m3.rs @@ -3,7 +3,10 @@ use vllm_tokenizer::{DecodedText, DynTokenizer}; -use super::{DelimitedReasoningParser, ReasoningDelta, ReasoningParser, Result}; +use super::{ + DelimitedReasoningParser, DelimitedReasoningParserBuilder, ReasoningDelta, ReasoningParser, + Result, +}; const M3_THINK_START: &str = ""; const M3_THINK_END: &str = ""; @@ -27,7 +30,8 @@ impl MiniMaxM3ReasoningParser { /// Create a MiniMax M3 parser backed by the shared delimited state machine. pub fn new(tokenizer: DynTokenizer) -> Result { Ok(Self { - inner: DelimitedReasoningParser::new(tokenizer, M3_THINK_START, M3_THINK_END)?, + inner: DelimitedReasoningParserBuilder::new(tokenizer, M3_THINK_START, M3_THINK_END) + .build()?, at_response_start: true, leading_end_buffer: DecodedText::default(), }) @@ -77,7 +81,7 @@ impl ReasoningParser for MiniMaxM3ReasoningParser { } fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> { - self.inner.initialize(prompt_token_ids); + self.inner.initialize(prompt_token_ids)?; self.at_response_start = true; self.leading_end_buffer.clear(); Ok(()) diff --git a/rust/src/parser/src/reasoning/mod.rs b/rust/src/parser/src/reasoning/mod.rs index e5ff9081f7..9a430f1996 100644 --- a/rust/src/parser/src/reasoning/mod.rs +++ b/rust/src/parser/src/reasoning/mod.rs @@ -19,7 +19,9 @@ mod cohere_cmd; mod deepseek_r1; +mod deepseek_v3; mod delimited; +mod glm45; mod hy; mod kimi; mod minimax_m3; @@ -32,7 +34,11 @@ use vllm_tokenizer::{DecodedText, DynTokenizer}; pub use self::cohere_cmd::CohereCmdReasoningParser; pub use self::deepseek_r1::DeepSeekR1ReasoningParser; -pub(crate) use self::delimited::{DelimitedReasoningParser, last_reasoning_boundary}; +pub use self::deepseek_v3::DeepSeekV3ReasoningParser; +pub(crate) use self::delimited::{ + DelimitedReasoningParser, DelimitedReasoningParserBuilder, last_reasoning_boundary, +}; +pub use self::glm45::Glm45ReasoningParser; pub(crate) use self::hy::HyReasoningParser; pub use self::kimi::KimiReasoningParser; pub use self::minimax_m3::MiniMaxM3ReasoningParser; @@ -40,22 +46,21 @@ pub use self::qwen3::Qwen3ReasoningParser; pub use self::seed_oss::SeedOssReasoningParser; pub use self::step3p5::Step3p5ReasoningParser; -/// DeepSeek V3 currently shares the standard `...` parser. -pub type DeepSeekV3ReasoningParser = Qwen3ReasoningParser; -/// DeepSeek V4 currently shares the standard `...` parser. -pub type DeepSeekV4ReasoningParser = Qwen3ReasoningParser; -/// GLM45 currently shares the standard `...` parser. -pub type Glm45ReasoningParser = Qwen3ReasoningParser; -/// Kimi K2 currently shares the standard `...` parser. +/// Nemotron V3 and Step3p5 frame all three reasoning boundaries with one newline. +pub type NemotronV3ReasoningParser = Step3p5ReasoningParser; +/// Step3 and DeepSeek R1 prefill a newline after the reasoning opener. +pub type Step3ReasoningParser = DeepSeekR1ReasoningParser; + +/// DeepSeek V4 currently shares the DeepSeek V3 reasoning parser. +pub type DeepSeekV4ReasoningParser = DeepSeekV3ReasoningParser; +/// GLM-4.7 and GLM-5 use bare reasoning delimiters. +pub type Glm47ReasoningParser = DeepSeekV3ReasoningParser; +/// Kimi K2 currently shares the DeepSeek V3 reasoning parser. // TODO: kimi k2 may implicitly end reasoning by starting a tool call section // using <|tool_calls_section_begin|>, we should support that. -pub type KimiK2ReasoningParser = Qwen3ReasoningParser; -/// MiniMax M2 currently shares the standard `...` parser. +pub type KimiK2ReasoningParser = DeepSeekV3ReasoningParser; +/// MiniMax M2 currently shares the Qwen3 reasoning parser. pub type MiniMaxM2ReasoningParser = Qwen3ReasoningParser; -/// Nemotron V3 currently shares the standard `...` parser. -pub type NemotronV3ReasoningParser = Qwen3ReasoningParser; -/// Step3 currently shares the standard `...` parser. -pub type Step3ReasoningParser = Qwen3ReasoningParser; /// Result alias for reasoning parser operations. pub type Result = std::result::Result; @@ -140,6 +145,8 @@ pub trait ReasoningParser: Send { /// Errors produced while creating or running reasoning parsers. #[derive(Debug, Error)] pub enum ReasoningError { + #[error(transparent)] + Tokenizer(#[from] vllm_tokenizer::TokenizerError), #[error("tokenizer is missing reasoning delimiter token `{token}`")] MissingToken { token: String }, #[error( diff --git a/rust/src/parser/src/reasoning/qwen3.rs b/rust/src/parser/src/reasoning/qwen3.rs index fbde8a5c59..9a3f298c4b 100644 --- a/rust/src/parser/src/reasoning/qwen3.rs +++ b/rust/src/parser/src/reasoning/qwen3.rs @@ -3,13 +3,14 @@ use vllm_tokenizer::{DecodedText, DynTokenizer}; -use super::{DelimitedReasoningParser, ReasoningDelta, ReasoningParser, Result}; +use super::{ + DelimitedReasoningParser, DelimitedReasoningParserBuilder, ReasoningDelta, ReasoningParser, + Result, +}; /// Reasoning parser for the Qwen3/Qwen3.5 family. /// -/// This parser uses standard `...` delimiters and defaults to -/// waiting for an explicit start token when prompt initialization finds no -/// reasoning boundary. +/// This parser uses standard `...` delimiters. pub struct Qwen3ReasoningParser { inner: DelimitedReasoningParser, } @@ -18,7 +19,11 @@ impl Qwen3ReasoningParser { /// Create a Qwen3 parser backed by the shared delimited state machine. pub fn new(tokenizer: DynTokenizer) -> Result { Ok(Self { - inner: DelimitedReasoningParser::new(tokenizer, "", "")?, + inner: DelimitedReasoningParserBuilder::new(tokenizer, "", "") + .with_after_start("\n") + .with_before_end("\n") + .with_after_end("\n\n") + .build()?, }) } } @@ -32,8 +37,7 @@ impl ReasoningParser for Qwen3ReasoningParser { } fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> { - self.inner.initialize(prompt_token_ids); - Ok(()) + self.inner.initialize(prompt_token_ids) } fn push(&mut self, delta: DecodedText) -> Result { diff --git a/rust/src/parser/src/reasoning/seed_oss.rs b/rust/src/parser/src/reasoning/seed_oss.rs index bb1dcc91ce..44fc0c7232 100644 --- a/rust/src/parser/src/reasoning/seed_oss.rs +++ b/rust/src/parser/src/reasoning/seed_oss.rs @@ -3,7 +3,10 @@ use vllm_tokenizer::{DecodedText, DynTokenizer}; -use super::{DelimitedReasoningParser, ReasoningDelta, ReasoningParser, Result}; +use super::{ + DelimitedReasoningParser, DelimitedReasoningParserBuilder, ReasoningDelta, ReasoningParser, + Result, +}; /// Reasoning parser for SeedOSS models using ``/`` /// delimiters. @@ -15,7 +18,9 @@ impl SeedOssReasoningParser { /// Create a SeedOSS parser backed by the shared delimited state machine. pub fn new(tokenizer: DynTokenizer) -> Result { Ok(Self { - inner: DelimitedReasoningParser::new(tokenizer, "", "")?, + inner: DelimitedReasoningParserBuilder::new(tokenizer, "", "") + .with_after_end("\n") + .build()?, }) } } @@ -29,8 +34,7 @@ impl ReasoningParser for SeedOssReasoningParser { } fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> { - self.inner.initialize(prompt_token_ids); - Ok(()) + self.inner.initialize(prompt_token_ids) } fn push(&mut self, delta: DecodedText) -> Result { diff --git a/rust/src/parser/src/reasoning/step3p5.rs b/rust/src/parser/src/reasoning/step3p5.rs index 9294bd9690..7f94faf998 100644 --- a/rust/src/parser/src/reasoning/step3p5.rs +++ b/rust/src/parser/src/reasoning/step3p5.rs @@ -3,89 +3,27 @@ use vllm_tokenizer::{DecodedText, DynTokenizer}; -use super::{DelimitedReasoningParser, ReasoningDelta, ReasoningParser, Result}; +use super::{ + DelimitedReasoningParser, DelimitedReasoningParserBuilder, ReasoningDelta, ReasoningParser, + Result, +}; -/// Reasoning parser for Step3p5 outputs. -/// -/// Step3p5 uses standard ``/`` delimiters but emits a `\n` -/// immediately before and/or after ``. The parser drops these framing -/// newlines on both sides of the boundary, holding a trailing `\n` from -/// reasoning across pushes until either more reasoning text or `` -/// arrives, and dropping a leading `\n` from the first content delta after -/// the boundary. +/// Reasoning parser for Step3p5 style outputs. pub struct Step3p5ReasoningParser { inner: DelimitedReasoningParser, - /// `\n` at end of last reasoning delta, held in case `` follows. - pending_reasoning_newline: Option, - /// Last push ended on `` without emitting content; the next - /// content delta's leading `\n` should be dropped. - just_ended_reasoning: bool, } impl Step3p5ReasoningParser { /// Create a Step3p5 parser backed by the shared delimited state machine. pub fn new(tokenizer: DynTokenizer) -> Result { Ok(Self { - inner: DelimitedReasoningParser::new(tokenizer, "", "")?, - pending_reasoning_newline: None, - just_ended_reasoning: false, + inner: DelimitedReasoningParserBuilder::new(tokenizer, "", "") + .with_after_start("\n") + .with_before_end("\n") + .with_after_end("\n") + .build()?, }) } - - /// Drop framing newlines around `` and track held-newline state. - fn process( - &mut self, - mut inner_delta: ReasoningDelta, - was_in_reasoning: bool, - now_in_reasoning: bool, - ) -> ReasoningDelta { - // A `...` round-trip in one push still counts as a - // transition: the inner emits reasoning while ending in content mode. - let transitioned = - !now_in_reasoning && (was_in_reasoning || inner_delta.reasoning.is_some()); - - // Replay or drop a previously-held trailing reasoning newline. - if self.pending_reasoning_newline.is_some() { - if let Some(reasoning) = inner_delta.reasoning.as_mut() { - let mut held = self.pending_reasoning_newline.take().unwrap(); - held.append(std::mem::take(reasoning)); - *reasoning = held; - } else if transitioned { - // The held `\n` was the one right before ``: drop it. - self.pending_reasoning_newline = None; - } - } - - // Hold back a trailing reasoning `\n` until we know if `` follows. - if let Some(reasoning) = inner_delta.reasoning.as_mut() - && reasoning.text.ends_with('\n') - { - let kept = reasoning.drain_prefix(reasoning.text.len() - 1); - let newline = std::mem::replace(reasoning, kept); - if !transitioned { - self.pending_reasoning_newline = Some(newline); - } - } - - // Drop a leading `\n` of content emitted right after ``. - if let Some(content) = inner_delta.content.as_mut() - && (transitioned || self.just_ended_reasoning) - && content.text.starts_with('\n') - { - let _ = content.drain_prefix(1); - } - - self.just_ended_reasoning = transitioned && inner_delta.content.is_none(); - - if inner_delta.reasoning.as_ref().is_some_and(DecodedText::is_empty) { - inner_delta.reasoning = None; - } - if inner_delta.content.as_ref().is_some_and(DecodedText::is_empty) { - inner_delta.content = None; - } - - inner_delta - } } impl ReasoningParser for Step3p5ReasoningParser { @@ -97,32 +35,15 @@ impl ReasoningParser for Step3p5ReasoningParser { } fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> { - self.inner.initialize(prompt_token_ids); - Ok(()) + self.inner.initialize(prompt_token_ids) } fn push(&mut self, delta: DecodedText) -> Result { - let was = self.inner.in_reasoning(); - let inner_delta = self.inner.push(delta); - let now = self.inner.in_reasoning(); - Ok(self.process(inner_delta, was, now)) + Ok(self.inner.push(delta)) } fn finish(&mut self) -> Result { - let was = self.inner.in_reasoning(); - let inner_delta = self.inner.finish(); - let now = self.inner.in_reasoning(); - let mut delta = self.process(inner_delta, was, now); - - // Emit a still-held newline rather than silently dropping it. - if let Some(held) = self.pending_reasoning_newline.take() { - match delta.reasoning.as_mut() { - Some(existing) => existing.append(held), - None => delta.reasoning = Some(held), - } - } - - Ok(delta) + Ok(self.inner.finish()) } } @@ -181,7 +102,7 @@ mod tests { // ``; surrounding newlines remain part of reasoning/content. let tokenizer = Arc::new(fake_tokenizer()); let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap(); - parser.initialize(&[THINK_START_ID]).unwrap(); + parser.initialize(&[THINK_START_ID, u32::from(b'\n')]).unwrap(); let delta = push_str( &mut parser, diff --git a/rust/src/parser/src/reasoning/tests.rs b/rust/src/parser/src/reasoning/tests.rs index c81a9c8c63..a67277e35d 100644 --- a/rust/src/parser/src/reasoning/tests.rs +++ b/rust/src/parser/src/reasoning/tests.rs @@ -8,8 +8,9 @@ use vllm_tokenizer::{DecodedText, DynTokenizer, TokenAnchor, TokenAttribution}; use super::{ CohereCmdReasoningParser, DeepSeekR1ReasoningParser, DelimitedReasoningParser, - KimiReasoningParser, MiniMaxM3ReasoningParser, Qwen3ReasoningParser, ReasoningDelta, - ReasoningParser, Result, SeedOssReasoningParser, Step3p5ReasoningParser, + DelimitedReasoningParserBuilder, KimiReasoningParser, MiniMaxM3ReasoningParser, + Qwen3ReasoningParser, ReasoningDelta, ReasoningParser, Result, SeedOssReasoningParser, + Step3p5ReasoningParser, }; pub(crate) const THINK_START_ID: u32 = 256; @@ -57,7 +58,9 @@ pub(crate) fn content_str(delta: &ReasoningDelta) -> Option<&str> { #[test] fn delimited_content_only_stream() { let tokenizer = Arc::new(fake_tokenizer()); - let mut parser = DelimitedReasoningParser::new(tokenizer, "", "").unwrap(); + let mut parser = DelimitedReasoningParserBuilder::new(tokenizer, "", "") + .build() + .unwrap(); let delta = parser.push(DecodedText::unattributed("plain content")); assert_eq!(content_str(&delta), Some("plain content")); @@ -66,7 +69,9 @@ fn delimited_content_only_stream() { #[test] fn delimited_single_chunk_with_reasoning_and_content() { let tokenizer = Arc::new(fake_tokenizer()); - let mut parser = DelimitedReasoningParser::new(tokenizer, "", "").unwrap(); + let mut parser = DelimitedReasoningParserBuilder::new(tokenizer, "", "") + .build() + .unwrap(); let delta = parser.push(DecodedText::unattributed("reasonanswer")); assert_eq!(reasoning_str(&delta), Some("reason")); @@ -76,7 +81,9 @@ fn delimited_single_chunk_with_reasoning_and_content() { #[test] fn delimited_partial_tokens_across_chunks() { let tokenizer = Arc::new(fake_tokenizer()); - let mut parser = DelimitedReasoningParser::new(tokenizer, "", "").unwrap(); + let mut parser = DelimitedReasoningParserBuilder::new(tokenizer, "", "") + .build() + .unwrap(); assert!(parser.push(DecodedText::unattributed("reasonanswer")); @@ -87,8 +94,10 @@ fn delimited_partial_tokens_across_chunks() { #[test] fn delimited_finish_flushes_buffer() { let tokenizer = Arc::new(fake_tokenizer()); - let mut parser = DelimitedReasoningParser::new(tokenizer, "", "").unwrap(); - parser.initialize(&[THINK_START_ID]); + let mut parser = DelimitedReasoningParserBuilder::new(tokenizer, "", "") + .build() + .unwrap(); + parser.initialize(&[THINK_START_ID]).unwrap(); let delta = parser.push(DecodedText::unattributed("unfinished", "").unwrap(); - parser.initialize(&[THINK_START_ID]); + let mut parser = DelimitedReasoningParserBuilder::new(tokenizer, "", "") + .build() + .unwrap(); + parser.initialize(&[THINK_START_ID]).unwrap(); // A filtered special token produces a zero-width attribution with no text; // it must survive as a reasoning piece rather than being dropped. @@ -130,7 +141,9 @@ fn delimited_zero_width_only_piece_is_attributed_to_current_state() { #[test] fn delimited_marker_tokens_are_dropped_from_attributions() { let tokenizer = Arc::new(fake_tokenizer()); - let mut parser = DelimitedReasoningParser::new(tokenizer, "", "").unwrap(); + let mut parser = DelimitedReasoningParserBuilder::new(tokenizer, "", "") + .build() + .unwrap(); let mut collected = CollectedAttributions::default(); for chunk in attributed_chunks(&[ @@ -283,3 +296,170 @@ pub(crate) fn collect_attributed( collected.record(parser.finish().unwrap()); collected } + +fn framed_parser() -> DelimitedReasoningParser { + DelimitedReasoningParserBuilder::new(Arc::new(fake_tokenizer()), "", "") + .with_after_start("\n") + .with_before_end("\n") + .with_after_end("\n\n") + .build() + .unwrap() +} + +fn collect_framed(parser: &mut DelimitedReasoningParser, chunks: &[&str]) -> (String, String) { + let mut result = (String::new(), String::new()); + for chunk in chunks.iter().copied().map(Some).chain(std::iter::once(None)) { + let delta = match chunk { + Some(chunk) => parser.push(DecodedText::unattributed(chunk)), + None => parser.finish(), + }; + if let Some(reasoning) = delta.reasoning { + result.0.push_str(&reasoning.text); + } + if let Some(content) = delta.content { + result.1.push_str(&content.text); + } + } + result +} + +#[test] +fn delimited_framing_preserves_body_whitespace_at_every_split() { + let wire = "\n\n reason\t\n\n\n\n\n answer\n"; + let expected = ("\n reason\t\n".to_string(), "\n answer\n".to_string()); + for split in 0..=wire.len() { + let mut parser = framed_parser(); + assert_eq!( + collect_framed(&mut parser, &[&wire[..split], &wire[split..]]), + expected, + "split at {split}" + ); + } + let chars = wire + .as_bytes() + .chunks(1) + .map(|c| std::str::from_utf8(c).unwrap()) + .collect::>(); + assert_eq!(collect_framed(&mut framed_parser(), &chars), expected); +} + +#[test] +fn delimited_partial_framing_is_preserved_on_mismatch_or_finish() { + for (wire, reasoning, content) in [ + ("reason\nanswer", "reason", "\nanswer"), + ("\nreason\nmore\n", "reason\nmore\n", ""), + ("\nreason\n\nreason\n\n", "reason", "\n"), + (" plain\n", "", " plain\n"), + ] { + for split in 0..=wire.len() { + assert_eq!( + collect_framed(&mut framed_parser(), &[&wire[..split], &wire[split..]]), + (reasoning.to_string(), content.to_string()), + "wire {wire:?}, split {split}" + ); + } + } +} + +#[test] +fn delimited_prompt_initialization_consumes_only_remaining_framing() { + use vllm_tokenizer::Tokenizer; + let tokenizer = fake_tokenizer(); + for (prompt, wire, reasoning, content) in [ + ( + "", + "\nreason\n\n\nanswer", + "reason", + "answer", + ), + ( + "\n", + "\nreason\n\n\nanswer", + "\nreason", + "answer", + ), + ( + "\nprefill", + "\nreason\n\n\nanswer", + "\nreason", + "answer", + ), + ("", "\n\n\nanswer", "", "\nanswer"), + ("\n", "\n\nanswer", "", "\nanswer"), + ("\n\n", "\nanswer", "", "\nanswer"), + ("\n\nprefill", "\nanswer", "", "\nanswer"), + ] { + let mut parser = framed_parser(); + parser.initialize(&tokenizer.encode(prompt, false).unwrap()).unwrap(); + assert_eq!( + collect_framed(&mut parser, &[wire]), + (reasoning.to_string(), content.to_string()), + "prompt {prompt:?}" + ); + } +} + +#[test] +fn delimited_framing_holds_only_boundary_candidates() { + let mut parser = framed_parser(); + let first = parser.push(DecodedText::unattributed("\nreason\n")); + assert_eq!(reasoning_str(&first), Some("reason")); + let more = parser.push(DecodedText::unattributed("more\n")); + assert_eq!(reasoning_str(&more), Some("\nmore")); + assert!(parser.push(DecodedText::unattributed("\n")).is_empty()); + let answer = parser.push(DecodedText::unattributed("\n answer")); + assert_eq!(content_str(&answer), Some(" answer")); +} + +#[test] +fn delimited_start_prefix_is_consumed_only_before_the_marker() { + for wire in [ + "\nreasonanswer", + "\n\nreasonanswer", + ] { + for split in 0..=wire.len() { + let mut parser = DelimitedReasoningParserBuilder::new( + Arc::new(fake_tokenizer()), + "", + "", + ) + .with_before_start("\n") + .build() + .unwrap(); + let expected_content = if wire.starts_with("\n\n") { + "\nanswer" + } else { + "answer" + }; + assert_eq!( + collect_framed(&mut parser, &[&wire[..split], &wire[split..]]), + ("reason".to_string(), expected_content.to_string()) + ); + } + } +} + +#[test] +fn delimited_framing_preserves_body_token_attributions() { + let mut parser = framed_parser(); + let mut collected = CollectedAttributions::default(); + for chunk in attributed_chunks(&[ + Some(""), + Some("\n"), + Some("reason"), + None, + Some("\n"), + Some(""), + Some("\n"), + Some("\n"), + Some(" answer"), + ]) { + collected.record(parser.push(chunk)); + } + collected.record(parser.finish()); + assert_eq!(collected.reasoning_text, "reason"); + assert_eq!(collected.content_text, " answer"); + assert_eq!(collected.reasoning_ids, [3, 4]); + assert_eq!(collected.content_ids, [9]); +} diff --git a/rust/src/parser/src/tool/deepseek_dsml/deepseek_v32.rs b/rust/src/parser/src/tool/deepseek_dsml/deepseek_v32.rs index 09bb0f189b..632eda600f 100644 --- a/rust/src/parser/src/tool/deepseek_dsml/deepseek_v32.rs +++ b/rust/src/parser/src/tool/deepseek_dsml/deepseek_v32.rs @@ -71,6 +71,7 @@ mod tests { use super::DeepSeekV32ToolParser; use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools}; + use crate::tool::tests::assert_tool_framing_preserves_body_whitespace; use crate::tool::{ToolParser, ToolParserTestExt as _}; fn build_tool_call(function_name: &str, params: &[(&str, &str)]) -> String { @@ -445,4 +446,12 @@ mod tests { assert_eq!(streamed.normal_text(), complete.normal_text()); assert_eq!(streamed.calls(), complete.calls()); } + + #[test] + fn tool_framing_preserves_body_whitespace_across_chunk_boundaries() { + assert_tool_framing_preserves_body_whitespace::( + "\n\n", + "<｜DSML｜function_calls><｜DSML｜invoke name=\"get_weather\">", + ); + } } diff --git a/rust/src/parser/src/tool/deepseek_dsml/deepseek_v4.rs b/rust/src/parser/src/tool/deepseek_dsml/deepseek_v4.rs index f4540ea47c..b65df235ca 100644 --- a/rust/src/parser/src/tool/deepseek_dsml/deepseek_v4.rs +++ b/rust/src/parser/src/tool/deepseek_dsml/deepseek_v4.rs @@ -73,6 +73,7 @@ mod tests { use super::DeepSeekV4ToolParser; use crate::tool::test_utils::{collect_stream, test_tools}; + use crate::tool::tests::assert_tool_framing_preserves_body_whitespace; use crate::tool::{ToolParser, ToolParserTestExt as _}; fn build_tool_call(function_name: &str, params: &[(&str, &str)]) -> String { @@ -145,4 +146,12 @@ mod tests { json!({ "location": "Beijing" }) ); } + + #[test] + fn tool_framing_preserves_body_whitespace_across_chunk_boundaries() { + assert_tool_framing_preserves_body_whitespace::( + "\n\n", + "<｜DSML｜tool_calls><｜DSML｜invoke name=\"get_weather\">", + ); + } } diff --git a/rust/src/parser/src/tool/deepseek_dsml/mod.rs b/rust/src/parser/src/tool/deepseek_dsml/mod.rs index dfb45d72ca..6ef5e78d4e 100644 --- a/rust/src/parser/src/tool/deepseek_dsml/mod.rs +++ b/rust/src/parser/src/tool/deepseek_dsml/mod.rs @@ -8,7 +8,7 @@ use winnow::stream::Partial; use winnow::token::{literal, rest, take_until}; use super::parameters::ToolSchemas; -use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len, take_until_marker}; +use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len_mul, take_until_marker}; use super::{Result, ToolCallDelta, ToolParserOutput}; use crate::tool::Tool; @@ -28,16 +28,19 @@ type DsmlInput<'i> = Partial<&'i str>; #[derive(Debug, Clone, Copy)] struct DsmlTokens { tool_calls_start: &'static str, + framed_tool_calls_start: &'static str, tool_calls_end: &'static str, } impl DsmlTokens { const V32: Self = Self { tool_calls_start: "<｜DSML｜function_calls>", + framed_tool_calls_start: "\n\n<｜DSML｜function_calls>", tool_calls_end: "", }; const V4: Self = Self { tool_calls_start: "<｜DSML｜tool_calls>", + framed_tool_calls_start: "\n\n<｜DSML｜tool_calls>", tool_calls_end: "", }; } @@ -210,9 +213,12 @@ fn parse_tool_block_event( /// Parse a DSML function-calls start marker. fn tool_calls_start_event(input: &mut DsmlInput<'_>, tokens: DsmlTokens) -> ModalResult { - literal(tokens.tool_calls_start) - .value(DsmlEvent::ToolCallsStart) - .parse_next(input) + alt(( + literal(tokens.framed_tool_calls_start), + literal(tokens.tool_calls_start), + )) + .value(DsmlEvent::ToolCallsStart) + .parse_next(input) } /// Parse a DSML function-calls end marker. @@ -227,7 +233,11 @@ fn ignored_rest_event(input: &mut DsmlInput<'_>) -> ModalResult { /// Parse a safe text run before the next DSML marker. fn safe_text_event(input: &mut DsmlInput<'_>, tokens: DsmlTokens) -> ModalResult { - safe_text_len(input, tokens.tool_calls_start).map(|len| DsmlEvent::Text { len }) + safe_text_len_mul( + input, + &[tokens.framed_tool_calls_start, tokens.tool_calls_start], + ) + .map(|len| DsmlEvent::Text { len }) } /// Parse a DSML invoke block. diff --git a/rust/src/parser/src/tool/glm_xml/glm45_moe.rs b/rust/src/parser/src/tool/glm_xml/glm45_moe.rs index 06a26fae47..733174b368 100644 --- a/rust/src/parser/src/tool/glm_xml/glm45_moe.rs +++ b/rust/src/parser/src/tool/glm_xml/glm45_moe.rs @@ -45,3 +45,17 @@ impl ToolParser for Glm45MoeToolParser { self.0.reset() } } + +#[cfg(test)] +mod tests { + use super::Glm45MoeToolParser; + use crate::tool::tests::assert_tool_framing_preserves_body_whitespace; + + #[test] + fn tool_framing_preserves_body_whitespace_across_chunk_boundaries() { + assert_tool_framing_preserves_body_whitespace::( + "\n", + "get_weather\n", + ); + } +} diff --git a/rust/src/parser/src/tool/glm_xml/mod.rs b/rust/src/parser/src/tool/glm_xml/mod.rs index dc942a3891..58ef4a0045 100644 --- a/rust/src/parser/src/tool/glm_xml/mod.rs +++ b/rust/src/parser/src/tool/glm_xml/mod.rs @@ -8,7 +8,7 @@ use winnow::stream::Partial; use winnow::token::{literal, rest, take_until, take_while}; use super::parameters::ToolSchemas; -use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len, take_until_marker}; +use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len_mul, take_until_marker}; use super::{Result, ToolCallDelta, ToolParserOutput}; use crate::tool::Tool; @@ -19,6 +19,7 @@ pub use glm45_moe::Glm45MoeToolParser; pub use glm47_moe::Glm47MoeToolParser; const TOOL_CALL_START: &str = ""; +const FRAMED_TOOL_CALL_START: &str = "\n"; const TOOL_CALL_END: &str = ""; const ARG_KEY_START: &str = ""; const ARG_KEY_END: &str = ""; @@ -149,7 +150,7 @@ fn parse_next_glm_event( separator: Separator, ) -> ModalResult { match mode { - GlmMode::Text => parse_text_event(input), + GlmMode::Text => parse_text_event(input, separator), GlmMode::ToolCall { tool_call_end_scan } => { tool_call_event(input, separator, tool_call_end_scan) } @@ -158,8 +159,20 @@ fn parse_next_glm_event( } /// Parse a text-mode GLM event. -fn parse_text_event(input: &mut GlmInput<'_>) -> ModalResult { - alt((tool_call_start_event, safe_text_event)).parse_next(input) +fn parse_text_event(input: &mut GlmInput<'_>, separator: Separator) -> ModalResult { + let framed_start = match separator { + Separator::Newline => FRAMED_TOOL_CALL_START, + Separator::Flexible => TOOL_CALL_START, + }; + alt(( + literal(framed_start).value(GlmEvent::ToolCallStart), + tool_call_start_event, + |input: &mut GlmInput<'_>| { + safe_text_len_mul(input, &[framed_start, TOOL_CALL_START]) + .map(|len| GlmEvent::Text { len }) + }, + )) + .parse_next(input) } /// Parse a GLM tool-call start marker. @@ -167,11 +180,6 @@ fn tool_call_start_event(input: &mut GlmInput<'_>) -> ModalResult { literal(TOOL_CALL_START).value(GlmEvent::ToolCallStart).parse_next(input) } -/// Parse a safe text run before the next GLM marker. -fn safe_text_event(input: &mut GlmInput<'_>) -> ModalResult { - safe_text_len(input, TOOL_CALL_START).map(|len| GlmEvent::Text { len }) -} - /// Parse text after a completed GLM tool call. fn after_tool_call_event(input: &mut GlmInput<'_>) -> ModalResult { ws0.void().parse_next(input)?; @@ -303,7 +311,7 @@ mod tests { let output = parser.parse_complete(&output).unwrap(); - assert_eq!(output.normal_text(), "Let me search for that.\n"); + assert_eq!(output.normal_text(), "Let me search for that."); assert_eq!(output.calls().len(), 1); assert_eq!(output.calls()[0].name.as_deref(), Some("get_weather")); assert_eq!( diff --git a/rust/src/parser/src/tool/json/granite4.rs b/rust/src/parser/src/tool/json/granite4.rs index e50b457780..484ac3d39f 100644 --- a/rust/src/parser/src/tool/json/granite4.rs +++ b/rust/src/parser/src/tool/json/granite4.rs @@ -222,6 +222,7 @@ fn header_event(input: &mut JsonToolInput<'_>) -> ModalResult { const CONFIG: JsonToolCallConfig = JsonToolCallConfig { parser_name: "Granite4", start_marker: "", + framed_start_marker: None, end_marker: "", marker_whitespace: JsonToolCallWhitespace::Optional, delimiter: None, diff --git a/rust/src/parser/src/tool/json/hermes.rs b/rust/src/parser/src/tool/json/hermes.rs index aed27a8cd4..acb2a6b695 100644 --- a/rust/src/parser/src/tool/json/hermes.rs +++ b/rust/src/parser/src/tool/json/hermes.rs @@ -7,6 +7,7 @@ use crate::tool::{Result, StructuralTagBuilder, Tool, ToolParser, ToolParserOutp const HERMES_CONFIG: JsonToolCallConfig = JsonToolCallConfig { parser_name: "Hermes", start_marker: "", + framed_start_marker: None, end_marker: "", marker_whitespace: JsonToolCallWhitespace::Optional, delimiter: None, diff --git a/rust/src/parser/src/tool/json/internlm2.rs b/rust/src/parser/src/tool/json/internlm2.rs index 9600fbf01b..33ddc5b1b3 100644 --- a/rust/src/parser/src/tool/json/internlm2.rs +++ b/rust/src/parser/src/tool/json/internlm2.rs @@ -7,6 +7,7 @@ use crate::tool::{Result, Tool, ToolParser, ToolParserOutput}; const INTERNLM2_CONFIG: JsonToolCallConfig = JsonToolCallConfig { parser_name: "InternLM2", start_marker: "<|action_start|><|plugin|>", + framed_start_marker: None, end_marker: "<|action_end|>", marker_whitespace: JsonToolCallWhitespace::Optional, delimiter: None, diff --git a/rust/src/parser/src/tool/json/llama.rs b/rust/src/parser/src/tool/json/llama.rs index 262340a9e8..bba3650916 100644 --- a/rust/src/parser/src/tool/json/llama.rs +++ b/rust/src/parser/src/tool/json/llama.rs @@ -208,6 +208,7 @@ fn llama_tool_call_header_event(input: &mut JsonToolInput<'_>) -> ModalResult = Partial<&'i str>; pub(crate) struct JsonToolCallConfig { pub parser_name: &'static str, pub start_marker: &'static str, + /// Start marker with its fixed preceding framing, matched alongside the + /// bare start marker. + pub framed_start_marker: Option<&'static str>, pub end_marker: &'static str, + /// Whitespace inside the tool-call markers: after the start marker and + /// before the end marker. pub marker_whitespace: JsonToolCallWhitespace, pub delimiter: Option<&'static str>, pub name_key: &'static str, @@ -211,7 +217,10 @@ fn tool_call_start_event( config: JsonToolCallConfig, ) -> ModalResult { seq!( - _: literal(config.start_marker), + _: |input: &mut JsonToolInput<'_>| match config.framed_start_marker { + Some(marker) => alt((literal(marker), literal(config.start_marker))).void().parse_next(input), + None => literal(config.start_marker).void().parse_next(input), + }, _: |input: &mut JsonToolInput<'_>| marker_whitespace(input, config), ) .value(JsonToolCallEvent::ToolCallStart) @@ -369,7 +378,11 @@ fn safe_text_event( input: &mut JsonToolInput<'_>, config: JsonToolCallConfig, ) -> ModalResult { - safe_text_len(input, config.start_marker).map(|len| JsonToolCallEvent::Text { len }) + match config.framed_start_marker { + Some(marker) => safe_text_len_mul(input, &[marker, config.start_marker]), + None => safe_text_len(input, config.start_marker), + } + .map(|len| JsonToolCallEvent::Text { len }) } #[cfg(test)] @@ -382,6 +395,7 @@ mod tests { const DELIMITED_CONFIG: JsonToolCallConfig = JsonToolCallConfig { parser_name: "Delimited JSON", start_marker: "", + framed_start_marker: None, end_marker: "", marker_whitespace: JsonToolCallWhitespace::Optional, delimiter: Some("<"), diff --git a/rust/src/parser/src/tool/json/phi4mini.rs b/rust/src/parser/src/tool/json/phi4mini.rs index e2ef065905..7740ec460a 100644 --- a/rust/src/parser/src/tool/json/phi4mini.rs +++ b/rust/src/parser/src/tool/json/phi4mini.rs @@ -7,6 +7,7 @@ use crate::tool::{Result, Tool, ToolParser, ToolParserOutput}; const PHI4MINI_CONFIG: JsonToolCallConfig = JsonToolCallConfig { parser_name: "Phi4Mini", start_marker: "functools[", + framed_start_marker: None, end_marker: "]", marker_whitespace: JsonToolCallWhitespace::Optional, delimiter: Some(","), diff --git a/rust/src/parser/src/tool/json/qwen.rs b/rust/src/parser/src/tool/json/qwen.rs index 7d5b71078a..ca7c5919cf 100644 --- a/rust/src/parser/src/tool/json/qwen.rs +++ b/rust/src/parser/src/tool/json/qwen.rs @@ -7,6 +7,7 @@ use crate::tool::{Result, StructuralTagBuilder, Tool, ToolParser, ToolParserOutp const QWEN_XML_CONFIG: JsonToolCallConfig = JsonToolCallConfig { parser_name: "Qwen XML", start_marker: "", + framed_start_marker: Some("\n"), end_marker: "", marker_whitespace: JsonToolCallWhitespace::Exact("\n"), delimiter: None, @@ -74,6 +75,7 @@ mod tests { use super::Qwen3XmlToolParser; use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools}; + use crate::tool::tests::assert_tool_framing_preserves_body_whitespace; use crate::tool::{ToolParser, ToolParserOutput, ToolParserTestExt as _}; fn build_tool_call(function_name: &str, arguments: &str) -> String { @@ -102,7 +104,7 @@ mod tests { )) .unwrap(); - assert_eq!(output.normal_text(), "Let me check.\n"); + assert_eq!(output.normal_text(), "Let me check."); assert_eq!(output.calls().len(), 1); assert_eq!(output.calls()[0].tool_index, 0); assert_eq!(output.calls()[0].name.as_deref(), Some("get_weather")); @@ -287,4 +289,12 @@ mod tests { expected `name`"#]] .assert_eq(&error.to_report_string()); } + + #[test] + fn tool_framing_preserves_body_whitespace_across_chunk_boundaries() { + assert_tool_framing_preserves_body_whitespace::( + "\n", + "\n{\"name\":\"get_weather\",\"arguments\":{}}\n", + ); + } } diff --git a/rust/src/parser/src/tool/minimax_m2.rs b/rust/src/parser/src/tool/minimax_m2.rs index d90e6676f0..ee48bfe8bc 100644 --- a/rust/src/parser/src/tool/minimax_m2.rs +++ b/rust/src/parser/src/tool/minimax_m2.rs @@ -8,11 +8,12 @@ use winnow::stream::Partial; use winnow::token::{literal, rest, take_until}; use super::parameters::ToolSchemas; -use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len, take_until_marker}; +use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len_mul, take_until_marker}; use super::{Result, ToolCallDelta, ToolParser, ToolParserOutput}; use crate::tool::{StructuralTagBuilder, Tool}; const TOOL_CALL_START: &str = ""; +const FRAMED_TOOL_CALL_START: &str = "\n"; const TOOL_CALL_END: &str = ""; const INVOKE_START: &str = ") -> ModalResult) -> ModalResult { - literal(TOOL_CALL_START).value(MinimaxM2Event::ToolBlockStart).parse_next(input) + alt((literal(FRAMED_TOOL_CALL_START), literal(TOOL_CALL_START))) + .value(MinimaxM2Event::ToolBlockStart) + .parse_next(input) } /// Parse a safe text run before the next MiniMax M2 marker. fn safe_text_event(input: &mut MinimaxM2Input<'_>) -> ModalResult { - safe_text_len(input, TOOL_CALL_START).map(|len| MinimaxM2Event::Text { len }) + safe_text_len_mul(input, &[FRAMED_TOOL_CALL_START, TOOL_CALL_START]) + .map(|len| MinimaxM2Event::Text { len }) } /// Parse one event inside a MiniMax M2 tool block. @@ -277,6 +281,7 @@ mod tests { use super::{MinimaxM2ToolParser, TOOL_CALL_END, TOOL_CALL_START, ToolParser}; use crate::tool::ToolParserTestExt as _; use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools}; + use crate::tool::tests::assert_tool_framing_preserves_body_whitespace; fn build_tool_block(invokes: &[(&str, Vec<(&str, &str)>)]) -> String { let invokes = invokes @@ -543,7 +548,7 @@ mod tests { let mut parser = MinimaxM2ToolParser::new(&test_tools()); let result = collect_stream(&mut parser, &chunks); - assert_eq!(result.normal_text(), "I will call the tools.\n"); + assert_eq!(result.normal_text(), "I will call the tools."); assert_eq!(result.calls().len(), 2); assert_eq!(result.calls()[0].tool_index, 0); assert_eq!(result.calls()[0].name.as_deref(), Some("get_weather")); @@ -599,4 +604,12 @@ mod tests { expect![[r#"tool parser parsing failed: near "": "#]] .assert_eq(&error.to_report_string()); } + + #[test] + fn tool_framing_preserves_body_whitespace_across_chunk_boundaries() { + assert_tool_framing_preserves_body_whitespace::( + "\n", + "", + ); + } } diff --git a/rust/src/parser/src/tool/qwen_coder.rs b/rust/src/parser/src/tool/qwen_coder.rs index 0a9385a85c..5bfaa764f6 100644 --- a/rust/src/parser/src/tool/qwen_coder.rs +++ b/rust/src/parser/src/tool/qwen_coder.rs @@ -8,7 +8,7 @@ use winnow::stream::Partial; use winnow::token::{literal, take_until}; use super::parameters::ToolSchemas; -use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len, take_until_marker}; +use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len_mul, take_until_marker}; use super::{Result, ToolCallDelta, ToolParser, ToolParserOutput}; use crate::tool::{StructuralTagBuilder, Tool}; @@ -31,6 +31,8 @@ pub(crate) struct QwenCoderConfig { pub(crate) parser_name: &'static str, /// Marker that opens a tool-call block. pub(crate) tool_call_start: &'static str, + pub(crate) first_tool_call_start: &'static str, + pub(crate) next_tool_call_start: &'static str, /// Marker that closes a tool-call block. pub(crate) tool_call_end: &'static str, } @@ -38,6 +40,8 @@ pub(crate) struct QwenCoderConfig { const QWEN_CODER_CONFIG: QwenCoderConfig = QwenCoderConfig { parser_name: "Qwen Coder", tool_call_start: TOOL_CALL_START, + first_tool_call_start: "\n\n", + next_tool_call_start: "\n", tool_call_end: TOOL_CALL_END, }; @@ -155,7 +159,7 @@ impl ToolParser for Qwen3CoderToolParser { let config = self.config; while let Some((event, consumed_len)) = parse_buffered_event(&self.buffer, |input| { - parse_next_qwen_coder_event(input, &mut self.mode, config) + parse_next_qwen_coder_event(input, &mut self.mode, config, self.emitted_tool_count > 0) })? { self.apply_event(event, output)?; self.buffer.drain(..consumed_len); @@ -191,9 +195,10 @@ fn parse_next_qwen_coder_event( input: &mut QwenCoderInput<'_>, mode: &mut QwenCoderMode, config: QwenCoderConfig, + after_tool_call: bool, ) -> ModalResult { match mode { - QwenCoderMode::Text => parse_text_event(input, config), + QwenCoderMode::Text => parse_text_event(input, config, after_tool_call), QwenCoderMode::ToolCall { end_marker_scan } => { tool_call_event(input, end_marker_scan, config.tool_call_end) } @@ -204,30 +209,24 @@ fn parse_next_qwen_coder_event( fn parse_text_event( input: &mut QwenCoderInput<'_>, config: QwenCoderConfig, + after_tool_call: bool, ) -> ModalResult { + let framed_start = if after_tool_call { + config.next_tool_call_start + } else { + config.first_tool_call_start + }; alt(( - |input: &mut QwenCoderInput<'_>| tool_call_start_event(input, config.tool_call_start), - |input: &mut QwenCoderInput<'_>| safe_text_event(input, config.tool_call_start), + alt((literal(framed_start), literal(config.tool_call_start))) + .value(QwenCoderEvent::ToolCallStart), + |input: &mut QwenCoderInput<'_>| { + safe_text_len_mul(input, &[framed_start, config.tool_call_start]) + .map(|len| QwenCoderEvent::Text { len }) + }, )) .parse_next(input) } -/// Parse a Qwen Coder tool-call start marker. -fn tool_call_start_event( - input: &mut QwenCoderInput<'_>, - tool_call_start: &'static str, -) -> ModalResult { - literal(tool_call_start).value(QwenCoderEvent::ToolCallStart).parse_next(input) -} - -/// Parse a safe text run before the next Qwen Coder marker. -fn safe_text_event( - input: &mut QwenCoderInput<'_>, - tool_call_start: &'static str, -) -> ModalResult { - safe_text_len(input, tool_call_start).map(|len| QwenCoderEvent::Text { len }) -} - /// Parse a complete Qwen Coder tool call. fn tool_call_event( input: &mut QwenCoderInput<'_>, @@ -296,6 +295,7 @@ mod tests { use super::{Qwen3CoderToolParser, ToolParser}; use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools}; + use crate::tool::tests::assert_tool_framing_preserves_body_whitespace; use crate::tool::{ToolParserOutput, ToolParserTestExt as _}; fn build_tool_call(function_name: &str, params: &[(&str, &str)]) -> String { @@ -766,4 +766,12 @@ mod tests { json!({ "location": "Hangzhou" }) ); } + + #[test] + fn tool_framing_preserves_body_whitespace_across_chunk_boundaries() { + assert_tool_framing_preserves_body_whitespace::( + "\n\n", + "\n\n\n", + ); + } } diff --git a/rust/src/parser/src/tool/seed_oss.rs b/rust/src/parser/src/tool/seed_oss.rs index dae7c5b2c5..160297523e 100644 --- a/rust/src/parser/src/tool/seed_oss.rs +++ b/rust/src/parser/src/tool/seed_oss.rs @@ -7,6 +7,8 @@ use crate::tool::{Result, Tool, ToolParser, ToolParserOutput}; const SEED_OSS_CONFIG: QwenCoderConfig = QwenCoderConfig { parser_name: "Seed-OSS", tool_call_start: "", + first_tool_call_start: "\n\n", + next_tool_call_start: "\n", tool_call_end: "", }; @@ -79,6 +81,7 @@ mod tests { use super::SeedOssToolParser; use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools}; + use crate::tool::tests::assert_tool_framing_preserves_body_whitespace; use crate::tool::{ToolParser, ToolParserOutput, ToolParserTestExt as _}; fn build_tool_call(function_name: &str, params: &[(&str, &str)]) -> String { @@ -250,4 +253,12 @@ mod tests { "tool parser parsing failed: incomplete Seed-OSS tool call" ); } + + #[test] + fn tool_framing_preserves_body_whitespace_across_chunk_boundaries() { + assert_tool_framing_preserves_body_whitespace::( + "\n\n", + "\n\n\n", + ); + } } diff --git a/rust/src/parser/src/tool/tests.rs b/rust/src/parser/src/tool/tests.rs index bb6348ac36..921c36119e 100644 --- a/rust/src/parser/src/tool/tests.rs +++ b/rust/src/parser/src/tool/tests.rs @@ -170,3 +170,26 @@ fn default_parse_complete_delegates_through_parse_chunk_and_finish() { ] ); } + +pub(super) fn assert_tool_framing_preserves_body_whitespace( + framing: &str, + call: &str, +) { + use super::test_utils::{collect_stream, test_tools}; + + let body = " before\n"; + let wire = format!("{body}{framing}{call}"); + for split in wire.char_indices().map(|(i, _)| i).chain(std::iter::once(wire.len())) { + let mut parser = P::create(&test_tools()).unwrap(); + let output = collect_stream(parser.as_mut(), &[&wire[..split], &wire[split..]]); + assert_eq!(output.normal_text(), body, "split {split}"); + assert_eq!(output.calls().len(), 1, "split {split}"); + assert_eq!(output.calls()[0].name.as_deref(), Some("get_weather")); + } + let plain = " no tool\n\n"; @@ -259,7 +260,9 @@ fn reasoning_start_event(input: &mut Gemma4Input<'_>) -> ModalResult) -> ModalResult { - literal(CHANNEL_END).value(Gemma4Event::ReasoningEnd).parse_next(input) + alt((literal(FRAMED_CHANNEL_END), literal(CHANNEL_END))) + .value(Gemma4Event::ReasoningEnd) + .parse_next(input) } /// Parse a Gemma4 tool-call start marker. @@ -308,7 +311,8 @@ fn safe_text_event(input: &mut Gemma4Input<'_>) -> ModalResult { /// Parse a safe reasoning run before the next Gemma4 marker. fn safe_reasoning_event(input: &mut Gemma4Input<'_>) -> ModalResult { - safe_text_len_mul(input, &[CHANNEL_END, TOOL_CALL_START]).map(|_| Gemma4Event::Reasoning) + safe_text_len_mul(input, &[FRAMED_CHANNEL_END, CHANNEL_END, TOOL_CALL_START]) + .map(|_| Gemma4Event::Reasoning) } /// Parse raw Gemma4 arguments through the first end marker outside a Gemma string. @@ -686,6 +690,18 @@ mod tests { output.coalesce() } + #[test] + fn gemma4_reasoning_framing_preserves_extra_newlines_at_every_split() { + let wire = "<|channel>thought\n\n reason\n\n answer\n"; + for split in 0..=wire.len() { + let output = collect_stream(&[&wire[..split], &wire[split..]]); + assert_eq!(output.reasoning_text(), "\n reason\n", "split {split}"); + assert_eq!(output.normal_text(), " answer\n", "split {split}"); + } + let output = collect_stream(&["<|channel>thought\nreason\n ToolCallDelta { (*output.calls().first().expect("expected one tool call")).clone() } diff --git a/rust/src/parser/src/unified/inkling.rs b/rust/src/parser/src/unified/inkling.rs index d1ed783671..4372cafb24 100644 --- a/rust/src/parser/src/unified/inkling.rs +++ b/rust/src/parser/src/unified/inkling.rs @@ -45,6 +45,7 @@ const BLOCK_END_MARKERS: &[&str] = &[END_MESSAGE, CONTENT_MODEL_END_SAMPLING]; const INKLING_TOOL_CONFIG: JsonToolCallConfig = JsonToolCallConfig { parser_name: "Inkling", start_marker: CONTENT_INVOKE_TOOL_JSON, + framed_start_marker: None, end_marker: END_MESSAGE, marker_whitespace: JsonToolCallWhitespace::Optional, delimiter: None,