iHore/vLLM
8
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[Rust Frontend] Correctly parse whitespace framing in model output (#55417)

Browse Source 
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Bugen Zhao
2026-09-09 13:40:13 +10:00
committed by GitHub
co-authored by Codex
parent 27b63fb8f8
commit fc6b6e1feb
36 changed files with 780 additions and 315 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rust/src/chat/src/lib.rs
+1 -1
View File
@@ -416,6 +416,6 @@ mod tests {
)
.unwrap_err();
expect_test::expect!["reasoning parser `definitely_missing_reasoning_parser` is not registered (choose from: cohere_cmd, deepseek_r1, deepseek_v3, deepseek_v4, gemma4, glm45, hy_v3, hy_v4, inkling, kimi, kimi_k2, kimi_k3, minimax_m2, minimax_m3, nemotron_v3, qwen3, seed_oss, step3, step3p5)"].assert_eq(&error.to_report_string());
expect_test::expect!["reasoning parser `definitely_missing_reasoning_parser` is not registered (choose from: cohere_cmd, deepseek_r1, deepseek_v3, deepseek_v4, gemma4, glm45, glm47, hy_v3, hy_v4, inkling, kimi, kimi_k2, kimi_k3, minimax_m2, minimax_m3, nemotron_v3, qwen3, seed_oss, step3, step3p5)"].assert_eq(&error.to_report_string());
}
}
rust/src/chat/src/parser/reasoning/mod.rs
+8 -6
View File
@@ -7,10 +7,10 @@ use std::sync::{Arc, LazyLock};
pub use vllm_parser::reasoning::{
CohereCmdReasoningParser, DeepSeekR1ReasoningParser, DeepSeekV3ReasoningParser,
DeepSeekV4ReasoningParser, Glm45ReasoningParser, KimiK2ReasoningParser, KimiReasoningParser,
MiniMaxM2ReasoningParser, MiniMaxM3ReasoningParser, NemotronV3ReasoningParser,
Qwen3ReasoningParser, ReasoningDelta, ReasoningError, ReasoningParser, SeedOssReasoningParser,
Step3ReasoningParser, Step3p5ReasoningParser,
DeepSeekV4ReasoningParser, Glm45ReasoningParser, Glm47ReasoningParser, KimiK2ReasoningParser,
KimiReasoningParser, MiniMaxM2ReasoningParser, MiniMaxM3ReasoningParser,
NemotronV3ReasoningParser, Qwen3ReasoningParser, ReasoningDelta, ReasoningError,
ReasoningParser, SeedOssReasoningParser, Step3ReasoningParser, Step3p5ReasoningParser,
};
use vllm_tokenizer::DynTokenizer;
@@ -25,6 +25,7 @@ pub mod names {
pub const GEMMA4: &str = "gemma4";
pub const INKLING: &str = "inkling";
pub const GLM45: &str = "glm45";
pub const GLM47: &str = "glm47";
pub const HY_V3: &str = "hy_v3";
pub const HY_V4: &str = "hy_v4";
pub const KIMI: &str = "kimi";
@@ -69,6 +70,7 @@ impl ReasoningParserFactory {
.register_unified_dummy(names::GEMMA4)
.register_unified_dummy(names::INKLING)
.register_parser::<Glm45ReasoningParser>(names::GLM45)
.register_parser::<Glm47ReasoningParser>(names::GLM47)
.register_unified_dummy(names::HY_V3)
.register_unified_dummy(names::HY_V4)
.register_parser::<KimiReasoningParser>(names::KIMI)
@@ -91,8 +93,8 @@ impl ReasoningParserFactory {
.register_pattern("gemma4", names::GEMMA4)
.register_pattern("qwq", names::DEEPSEEK_R1)
.register_pattern("qwen3", names::QWEN3)
.register_pattern("glm-5", names::GLM45)
.register_pattern("glm-4.7", names::GLM45)
.register_pattern("glm-5", names::GLM47)
.register_pattern("glm-4.7", names::GLM47)
.register_pattern("glm-4.6", names::GLM45)
.register_pattern("glm-4.5", names::GLM45)
.register_pattern("hy3", names::HY_V3)
rust/src/chat/src/parser/reasoning/tests.rs
+12 -1
View File
@@ -25,7 +25,7 @@ fn factory_contains_and_lists_registered_parsers() {
}
#[test]
fn factory_resolves_deepseek_v4_to_qwen3_alias() {
fn factory_resolves_deepseek_v4() {
let factory = ReasoningParserFactory::new();
assert_eq!(
factory.resolve_name_for_model("deepseek-ai/DeepSeek-V4"),
@@ -112,3 +112,14 @@ fn factory_rejects_unknown_parser_names() {
};
assert!(error.to_string().contains("choose from"));
}
#[test]
fn factory_distinguishes_glm_reasoning_framing() {
let factory = ReasoningParserFactory::new();
for model in ["zai-org/GLM-4.5", "zai-org/GLM-4.6"] {
assert_eq!(factory.resolve_name_for_model(model), Some(names::GLM45));
}
for model in ["zai-org/GLM-4.7-Flash", "zai-org/GLM-5.2-FP8"] {
assert_eq!(factory.resolve_name_for_model(model), Some(names::GLM47));
}
}
rust/src/chat/tests/roundtrip.rs
+11 -11
View File
@@ -445,10 +445,12 @@ async fn run_roundtrip_reasoning_and_content_inner(
let result = run_roundtrip(case, backends, &request, assistant).await?;
assert_eq!(
result.parsed_message.reasoning().as_deref().map(str::trim),
effective_thinking.then_some(expected_reasoning)
result.parsed_message.reasoning().as_deref(),
effective_thinking.then_some(expected_reasoning),
"parsed message: {:#?}",
result.parsed_message
);
assert_eq!(result.parsed_message.text().trim(), expected_text);
assert_eq!(result.parsed_message.text(), expected_text);
assert_eq!(result.parsed_message.tool_calls().count(), 0);
assert_eq!(
@@ -508,10 +510,12 @@ async fn run_roundtrip_tool_call_mix(
.await?;
assert_eq!(
result.parsed_message.reasoning().as_deref().map(str::trim),
Some(expected_reasoning)
result.parsed_message.reasoning().as_deref(),
Some(expected_reasoning),
"parsed message: {:#?}",
result.parsed_message
);
assert_eq!(result.parsed_message.text().trim(), expected_text);
assert_eq!(result.parsed_message.text(), expected_text);
let tool_calls = result.parsed_message.tool_calls().collect::<Vec<_>>();
assert_eq!(
@@ -714,11 +718,7 @@ async fn parse_completion(
while let Some(event) = events.next().await {
if let ChatEvent::Done { message, .. } = event? {
// TODO: currently our parsers are not very strict about preserving or trimming
// whitespace, so we trim here to avoid roundtrip failures due to
// insignificant whitespace differences. However, this may hurt token-level
// fidelity so we should consider improving them.
return Ok(message.trim());
return Ok(message);
}
}
rust/src/parser/src/reasoning/cohere_cmd.rs
+8 -5
View File
@@ -3,7 +3,10 @@
use vllm_tokenizer::{DecodedText, DynTokenizer};
use super::{DelimitedReasoningParser, ReasoningDelta, ReasoningParser, Result};
use super::{
DelimitedReasoningParser, DelimitedReasoningParserBuilder, ReasoningDelta, ReasoningParser,
Result,
};
/// Reasoning parser for Cohere Command models that use explicit START/END tags.
pub struct CohereCmdReasoningParser {
@@ -15,11 +18,12 @@ impl CohereCmdReasoningParser {
/// machine.
pub fn new(tokenizer: DynTokenizer) -> Result<Self> {
Ok(Self {
inner: DelimitedReasoningParser::new(
inner: DelimitedReasoningParserBuilder::new(
tokenizer,
"<|START_THINKING|>",
"<|END_THINKING|>",
)?,
)
.build()?,
})
}
}
@@ -33,8 +37,7 @@ impl ReasoningParser for CohereCmdReasoningParser {
}
fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> {
self.inner.initialize(prompt_token_ids);
Ok(())
self.inner.initialize(prompt_token_ids)
}
fn push(&mut self, delta: DecodedText) -> Result<ReasoningDelta> {
rust/src/parser/src/reasoning/deepseek_r1.rs
+9 -6
View File
@@ -3,7 +3,10 @@
use vllm_tokenizer::{DecodedText, DynTokenizer};
use super::{DelimitedReasoningParser, ReasoningDelta, ReasoningParser, Result};
use super::{
DelimitedReasoningParser, DelimitedReasoningParserBuilder, ReasoningDelta, ReasoningParser,
Result,
};
/// Reasoning parser for DeepSeek R1 style outputs.
pub struct DeepSeekR1ReasoningParser {
@@ -11,11 +14,12 @@ pub struct DeepSeekR1ReasoningParser {
}
impl DeepSeekR1ReasoningParser {
/// Create a DeepSeek R1 parser backed by the shared delimited state
/// machine.
/// Create a DeepSeek R1 parser backed by the shared delimited state machine.
pub fn new(tokenizer: DynTokenizer) -> Result<Self> {
Ok(Self {
inner: DelimitedReasoningParser::new(tokenizer, "<think>", "</think>")?,
inner: DelimitedReasoningParserBuilder::new(tokenizer, "<think>", "</think>")
.with_after_start("\n")
.build()?,
})
}
}
@@ -29,8 +33,7 @@ impl ReasoningParser for DeepSeekR1ReasoningParser {
}
fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> {
self.inner.initialize(prompt_token_ids);
Ok(())
self.inner.initialize(prompt_token_ids)
}
fn push(&mut self, delta: DecodedText) -> Result<ReasoningDelta> {
rust/src/parser/src/reasoning/deepseek_v3.rs
+45
View File
@@ -0,0 +1,45 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
use vllm_tokenizer::{DecodedText, DynTokenizer};
use super::{
DelimitedReasoningParser, DelimitedReasoningParserBuilder, ReasoningDelta, ReasoningParser,
Result,
};
/// Reasoning parser for DeepSeek V3 style outputs.
pub struct DeepSeekV3ReasoningParser {
inner: DelimitedReasoningParser,
}
impl DeepSeekV3ReasoningParser {
/// Create a DeepSeek V3 parser backed by the shared delimited state machine.
pub fn new(tokenizer: DynTokenizer) -> Result<Self> {
Ok(Self {
inner: DelimitedReasoningParserBuilder::new(tokenizer, "<think>", "</think>")
.build()?,
})
}
}
impl ReasoningParser for DeepSeekV3ReasoningParser {
fn create(tokenizer: DynTokenizer) -> Result<Box<dyn ReasoningParser>>
where
Self: Sized + 'static,
{
Ok(Box::new(Self::new(tokenizer)?))
}
fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> {
self.inner.initialize(prompt_token_ids)
}
fn push(&mut self, delta: DecodedText) -> Result<ReasoningDelta> {
Ok(self.inner.push(delta))
}
fn finish(&mut self) -> Result<ReasoningDelta> {
Ok(self.inner.finish())
}
}
rust/src/parser/src/reasoning/delimited.rs
+174 -99
View File
@@ -4,6 +4,90 @@
use vllm_tokenizer::{DecodedText, DynTokenizer, Tokenizer};
use super::{ReasoningDelta, ReasoningError, Result};
use crate::utils::partial_prefix_len;
/// Build a delimited reasoning parser with fixed text at each boundary.
pub(crate) struct DelimitedReasoningParserBuilder {
tokenizer: DynTokenizer,
start_token: String,
end_token: String,
before_start: &'static str,
after_start: &'static str,
before_end: &'static str,
after_end: &'static str,
}
impl DelimitedReasoningParserBuilder {
/// Configure the tokenizer and reasoning delimiters.
pub(crate) fn new(
tokenizer: DynTokenizer,
start_token: impl Into<String>,
end_token: impl Into<String>,
) -> Self {
Self {
tokenizer,
start_token: start_token.into(),
end_token: end_token.into(),
before_start: "",
after_start: "",
before_end: "",
after_end: "",
}
}
/// Include a fixed prefix when it immediately precedes the start marker.
pub(crate) fn with_before_start(mut self, before_start: &'static str) -> Self {
self.before_start = before_start;
self
}
/// Consume fixed text immediately after the start marker.
pub(crate) fn with_after_start(mut self, after_start: &'static str) -> Self {
self.after_start = after_start;
self
}
/// Include a fixed prefix when it immediately precedes the end marker.
pub(crate) fn with_before_end(mut self, before_end: &'static str) -> Self {
self.before_end = before_end;
self
}
/// Consume fixed text immediately after the end marker.
pub(crate) fn with_after_end(mut self, after_end: &'static str) -> Self {
self.after_end = after_end;
self
}
/// Create one delimited parser state machine.
pub(crate) fn build(self) -> Result<DelimitedReasoningParser> {
let start_token_id = self.tokenizer.token_to_id(&self.start_token).ok_or_else(|| {
ReasoningError::MissingToken {
token: self.start_token.clone(),
}
})?;
let end_token_id = self.tokenizer.token_to_id(&self.end_token).ok_or_else(|| {
ReasoningError::MissingToken {
token: self.end_token.clone(),
}
})?;
Ok(DelimitedReasoningParser {
tokenizer: self.tokenizer,
framed_start_token: format!("{}{}", self.before_start, self.start_token),
start_token: self.start_token,
framed_end_token: format!("{}{}", self.before_end, self.end_token),
end_token: self.end_token,
start_token_id,
end_token_id,
after_start: self.after_start,
after_end: self.after_end,
current_in_reasoning: false,
buffer: DecodedText::default(),
pending_after: "",
})
}
}
/// Shared incremental state machine for tag-delimited reasoning protocols.
///
@@ -18,54 +102,45 @@ use super::{ReasoningDelta, ReasoningError, Result};
/// prefill different prompts.
pub(crate) struct DelimitedReasoningParser {
tokenizer: DynTokenizer,
current_in_reasoning: bool,
buffer: DecodedText,
start_token: String,
framed_start_token: String,
end_token: String,
start_token_id: u32,
end_token_id: u32,
after_start: &'static str,
after_end: &'static str,
framed_end_token: String,
// Mutable state.
current_in_reasoning: bool,
buffer: DecodedText,
pending_after: &'static str,
}
impl DelimitedReasoningParser {
/// Create one delimited parser state machine.
pub(crate) fn new(
tokenizer: DynTokenizer,
start_token: impl Into<String>,
end_token: impl Into<String>,
) -> Result<Self> {
let start_token = start_token.into();
let end_token = end_token.into();
let start_token_id =
tokenizer
.token_to_id(&start_token)
.ok_or_else(|| ReasoningError::MissingToken {
token: start_token.clone(),
})?;
let end_token_id =
tokenizer.token_to_id(&end_token).ok_or_else(|| ReasoningError::MissingToken {
token: end_token.clone(),
})?;
Ok(Self {
tokenizer,
current_in_reasoning: false,
buffer: DecodedText::default(),
start_token,
end_token,
start_token_id,
end_token_id,
})
}
/// Initialize the starting state from prompt token IDs.
pub(crate) fn initialize(&mut self, prompt_token_ids: &[u32]) {
self.current_in_reasoning = last_reasoning_boundary(
/// Initialize the starting state and remaining framing from prompt tokens.
pub(crate) fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> {
self.buffer = DecodedText::default();
self.current_in_reasoning = false;
self.pending_after = "";
if let Some((index, in_reasoning)) = last_reasoning_boundary_position(
prompt_token_ids,
self.start_token_id,
self.end_token_id,
self.tokenizer.as_ref(),
)
.unwrap_or(false);
) {
self.current_in_reasoning = in_reasoning;
self.pending_after = if in_reasoning {
self.after_start
} else {
self.after_end
};
if !self.pending_after.is_empty() && index + 1 < prompt_token_ids.len() {
let tail = self.tokenizer.decode(&prompt_token_ids[index + 1..], false)?;
self.pending_after = self.pending_after.strip_prefix(&tail).unwrap_or("");
}
}
Ok(())
}
/// Return whether the parser is currently inside a reasoning section.
@@ -76,84 +151,74 @@ impl DelimitedReasoningParser {
/// Parse one decoded text delta and return its reasoning/content split.
pub(crate) fn push(&mut self, delta: DecodedText) -> ReasoningDelta {
self.buffer.append(delta);
let partial_suffix_len = self.partial_suffix_len(&self.buffer.text);
let stable_len = self.buffer.text.len() - partial_suffix_len;
let stable = self.buffer.drain_prefix(stable_len);
self.parse_stable_text(stable)
self.parse_buffer(false)
}
/// Flush any buffered partial delimiter suffix at end of stream.
/// Flush partial framing and delimiters as body text at end of stream.
pub(crate) fn finish(&mut self) -> ReasoningDelta {
// `drain_prefix(text.len())` takes trailing zero-width tokens too.
let stable = self.buffer.drain_prefix(self.buffer.text.len());
self.parse_stable_text(stable)
self.parse_buffer(true)
}
/// Parse text that is known not to end with a partial delimiter suffix.
///
/// Reasoning and content pieces keep the attributions of the tokens that
/// produced them; delimiter marker spans are drained and dropped, keeping
/// marker tokens out of any count.
fn parse_stable_text(&mut self, mut stable: DecodedText) -> ReasoningDelta {
fn parse_buffer(&mut self, finishing: bool) -> ReasoningDelta {
let mut delta = ReasoningDelta::default();
while !stable.text.is_empty() {
if self.current_in_reasoning {
if let Some(end_idx) = stable.text.find(&self.end_token) {
delta.push_reasoning(stable.drain_prefix(end_idx));
let _ = stable.drain_prefix(self.end_token.len());
self.current_in_reasoning = false;
} else {
delta.push_reasoning(stable);
return delta;
loop {
if !self.pending_after.is_empty() {
if self.buffer.text.starts_with(self.pending_after) {
let _ = self.buffer.drain_prefix(self.pending_after.len());
} else if !finishing && self.pending_after.starts_with(&self.buffer.text) {
break;
}
} else if let Some(start_idx) = stable.text.find(&self.start_token) {
delta.push_content(stable.drain_prefix(start_idx));
let _ = stable.drain_prefix(self.start_token.len());
self.current_in_reasoning = true;
} else {
delta.push_content(stable);
return delta;
// A mismatch preserves the candidate as body text.
self.pending_after = "";
}
}
// A remainder with empty text may still carry zero-width tokens;
// attribute them to the current state.
if !stable.attributions.is_empty() {
if self.current_in_reasoning {
delta.push_reasoning(stable);
let (marker, framed_marker) = if self.current_in_reasoning {
(&self.end_token, &self.framed_end_token)
} else {
delta.push_content(stable);
(&self.start_token, &self.framed_start_token)
};
if let Some(index) = self.buffer.text.find(marker) {
let before = &framed_marker[..framed_marker.len() - marker.len()];
let body_len = if self.buffer.text[..index].ends_with(before) {
index - before.len()
} else {
index
};
let boundary_len = index + marker.len() - body_len;
let body = self.buffer.drain_prefix(body_len);
self.push_body(&mut delta, body);
let _ = self.buffer.drain_prefix(boundary_len);
self.current_in_reasoning = !self.current_in_reasoning;
self.pending_after = if self.current_in_reasoning {
self.after_start
} else {
self.after_end
};
continue;
}
}
let keep_len = if finishing {
0
} else {
partial_prefix_len(&self.buffer.text, marker)
.max(partial_prefix_len(&self.buffer.text, framed_marker))
};
let body = self.buffer.drain_prefix(self.buffer.text.len() - keep_len);
self.push_body(&mut delta, body);
break;
}
delta
}
/// Return the longest trailing suffix that could still complete a
/// delimiter.
fn partial_suffix_len(&self, text: &str) -> usize {
let mut best = 0;
for idx in text.char_indices().map(|(idx, _)| idx).skip(1) {
let suffix = &text[idx..];
if self.start_token.starts_with(suffix) && self.start_token != suffix {
best = best.max(text.len() - idx);
}
if self.end_token.starts_with(suffix) && self.end_token != suffix {
best = best.max(text.len() - idx);
}
fn push_body(&self, delta: &mut ReasoningDelta, body: DecodedText) {
if self.current_in_reasoning {
delta.push_reasoning(body);
} else {
delta.push_content(body);
}
if self.start_token.starts_with(text) && self.start_token != text {
best = best.max(text.len());
}
if self.end_token.starts_with(text) && self.end_token != text {
best = best.max(text.len());
}
best
}
}
@@ -164,12 +229,22 @@ pub(crate) fn last_reasoning_boundary(
end_token_id: u32,
tokenizer: &dyn Tokenizer,
) -> Option<bool> {
for token_id in prompt_token_ids.iter().rev().copied() {
last_reasoning_boundary_position(prompt_token_ids, start_token_id, end_token_id, tokenizer)
.map(|(_, in_reasoning)| in_reasoning)
}
fn last_reasoning_boundary_position(
prompt_token_ids: &[u32],
start_token_id: u32,
end_token_id: u32,
tokenizer: &dyn Tokenizer,
) -> Option<(usize, bool)> {
for (index, &token_id) in prompt_token_ids.iter().enumerate().rev() {
if token_id == start_token_id {
return Some(true);
return Some((index, true));
}
if token_id == end_token_id {
return Some(false);
return Some((index, false));
}
if tokenizer.is_special_id(token_id) {
return None;
rust/src/parser/src/reasoning/glm45.rs
+48
View File
@@ -0,0 +1,48 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
use vllm_tokenizer::{DecodedText, DynTokenizer};
use super::{
DelimitedReasoningParser, DelimitedReasoningParserBuilder, ReasoningDelta, ReasoningParser,
Result,
};
/// Reasoning parser for GLM-4.5/4.6 models using `<think>`/`</think>`
/// delimiters.
pub struct Glm45ReasoningParser {
inner: DelimitedReasoningParser,
}
impl Glm45ReasoningParser {
/// Create a GLM-4.5/4.6 parser backed by the shared delimited state machine.
pub fn new(tokenizer: DynTokenizer) -> Result<Self> {
Ok(Self {
inner: DelimitedReasoningParserBuilder::new(tokenizer, "<think>", "</think>")
.with_before_start("\n")
.with_after_end("\n")
.build()?,
})
}
}
impl ReasoningParser for Glm45ReasoningParser {
fn create(tokenizer: DynTokenizer) -> Result<Box<dyn ReasoningParser>>
where
Self: Sized + 'static,
{
Ok(Box::new(Self::new(tokenizer)?))
}
fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> {
self.inner.initialize(prompt_token_ids)
}
fn push(&mut self, delta: DecodedText) -> Result<ReasoningDelta> {
Ok(self.inner.push(delta))
}
fn finish(&mut self) -> Result<ReasoningDelta> {
Ok(self.inner.finish())
}
}
rust/src/parser/src/reasoning/hy.rs
+8 -5
View File
@@ -3,7 +3,10 @@
use vllm_tokenizer::{DecodedText, DynTokenizer};
use super::{DelimitedReasoningParser, ReasoningDelta, ReasoningError, ReasoningParser, Result};
use super::{
DelimitedReasoningParser, DelimitedReasoningParserBuilder, ReasoningDelta, ReasoningError,
ReasoningParser, Result,
};
/// Internal HY reasoning stage used by the unified HY parsers.
pub(crate) struct HyReasoningParser {
@@ -14,11 +17,12 @@ impl HyReasoningParser {
/// Create a HY reasoning parser for the tokenizer-specific marker suffix.
pub(crate) fn new(tokenizer: DynTokenizer, suffix: &str) -> Result<Self> {
Ok(Self {
inner: DelimitedReasoningParser::new(
inner: DelimitedReasoningParserBuilder::new(
tokenizer,
format!("<think{suffix}>"),
format!("</think{suffix}>"),
)?,
)
.build()?,
})
}
}
@@ -36,8 +40,7 @@ impl ReasoningParser for HyReasoningParser {
}
fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> {
self.inner.initialize(prompt_token_ids);
Ok(())
self.inner.initialize(prompt_token_ids)
}
fn push(&mut self, delta: DecodedText) -> Result<ReasoningDelta> {
rust/src/parser/src/reasoning/kimi.rs
+7 -4
View File
@@ -3,7 +3,10 @@
use vllm_tokenizer::{DecodedText, DynTokenizer};
use super::{DelimitedReasoningParser, ReasoningDelta, ReasoningParser, Result};
use super::{
DelimitedReasoningParser, DelimitedReasoningParserBuilder, ReasoningDelta, ReasoningParser,
Result,
};
/// Reasoning parser for legacy Kimi models that use Unicode thinking tags.
pub struct KimiReasoningParser {
@@ -14,7 +17,8 @@ impl KimiReasoningParser {
/// Create a Kimi parser backed by the shared delimited state machine.
pub fn new(tokenizer: DynTokenizer) -> Result<Self> {
Ok(Self {
inner: DelimitedReasoningParser::new(tokenizer, "◁think▷", "◁/think▷")?,
inner: DelimitedReasoningParserBuilder::new(tokenizer, "◁think▷", "◁/think▷")
.build()?,
})
}
}
@@ -28,8 +32,7 @@ impl ReasoningParser for KimiReasoningParser {
}
fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> {
self.inner.initialize(prompt_token_ids);
Ok(())
self.inner.initialize(prompt_token_ids)
}
fn push(&mut self, delta: DecodedText) -> Result<ReasoningDelta> {
rust/src/parser/src/reasoning/minimax_m3.rs
+7 -3
View File
@@ -3,7 +3,10 @@
use vllm_tokenizer::{DecodedText, DynTokenizer};
use super::{DelimitedReasoningParser, ReasoningDelta, ReasoningParser, Result};
use super::{
DelimitedReasoningParser, DelimitedReasoningParserBuilder, ReasoningDelta, ReasoningParser,
Result,
};
const M3_THINK_START: &str = "<mm:think>";
const M3_THINK_END: &str = "</mm:think>";
@@ -27,7 +30,8 @@ impl MiniMaxM3ReasoningParser {
/// Create a MiniMax M3 parser backed by the shared delimited state machine.
pub fn new(tokenizer: DynTokenizer) -> Result<Self> {
Ok(Self {
inner: DelimitedReasoningParser::new(tokenizer, M3_THINK_START, M3_THINK_END)?,
inner: DelimitedReasoningParserBuilder::new(tokenizer, M3_THINK_START, M3_THINK_END)
.build()?,
at_response_start: true,
leading_end_buffer: DecodedText::default(),
})
@@ -77,7 +81,7 @@ impl ReasoningParser for MiniMaxM3ReasoningParser {
}
fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> {
self.inner.initialize(prompt_token_ids);
self.inner.initialize(prompt_token_ids)?;
self.at_response_start = true;
self.leading_end_buffer.clear();
Ok(())
rust/src/parser/src/reasoning/mod.rs
+21 -14
View File
@@ -19,7 +19,9 @@
mod cohere_cmd;
mod deepseek_r1;
mod deepseek_v3;
mod delimited;
mod glm45;
mod hy;
mod kimi;
mod minimax_m3;
@@ -32,7 +34,11 @@ use vllm_tokenizer::{DecodedText, DynTokenizer};
pub use self::cohere_cmd::CohereCmdReasoningParser;
pub use self::deepseek_r1::DeepSeekR1ReasoningParser;
pub(crate) use self::delimited::{DelimitedReasoningParser, last_reasoning_boundary};
pub use self::deepseek_v3::DeepSeekV3ReasoningParser;
pub(crate) use self::delimited::{
DelimitedReasoningParser, DelimitedReasoningParserBuilder, last_reasoning_boundary,
};
pub use self::glm45::Glm45ReasoningParser;
pub(crate) use self::hy::HyReasoningParser;
pub use self::kimi::KimiReasoningParser;
pub use self::minimax_m3::MiniMaxM3ReasoningParser;
@@ -40,22 +46,21 @@ pub use self::qwen3::Qwen3ReasoningParser;
pub use self::seed_oss::SeedOssReasoningParser;
pub use self::step3p5::Step3p5ReasoningParser;
/// DeepSeek V3 currently shares the standard `<think>...</think>` parser.
pub type DeepSeekV3ReasoningParser = Qwen3ReasoningParser;
/// DeepSeek V4 currently shares the standard `<think>...</think>` parser.
pub type DeepSeekV4ReasoningParser = Qwen3ReasoningParser;
/// GLM45 currently shares the standard `<think>...</think>` parser.
pub type Glm45ReasoningParser = Qwen3ReasoningParser;
/// Kimi K2 currently shares the standard `<think>...</think>` parser.
/// Nemotron V3 and Step3p5 frame all three reasoning boundaries with one newline.
pub type NemotronV3ReasoningParser = Step3p5ReasoningParser;
/// Step3 and DeepSeek R1 prefill a newline after the reasoning opener.
pub type Step3ReasoningParser = DeepSeekR1ReasoningParser;
/// DeepSeek V4 currently shares the DeepSeek V3 reasoning parser.
pub type DeepSeekV4ReasoningParser = DeepSeekV3ReasoningParser;
/// GLM-4.7 and GLM-5 use bare reasoning delimiters.
pub type Glm47ReasoningParser = DeepSeekV3ReasoningParser;
/// Kimi K2 currently shares the DeepSeek V3 reasoning parser.
// TODO: kimi k2 may implicitly end reasoning by starting a tool call section
// using <|tool_calls_section_begin|>, we should support that.
pub type KimiK2ReasoningParser = Qwen3ReasoningParser;
/// MiniMax M2 currently shares the standard `<think>...</think>` parser.
pub type KimiK2ReasoningParser = DeepSeekV3ReasoningParser;
/// MiniMax M2 currently shares the Qwen3 reasoning parser.
pub type MiniMaxM2ReasoningParser = Qwen3ReasoningParser;
/// Nemotron V3 currently shares the standard `<think>...</think>` parser.
pub type NemotronV3ReasoningParser = Qwen3ReasoningParser;
/// Step3 currently shares the standard `<think>...</think>` parser.
pub type Step3ReasoningParser = Qwen3ReasoningParser;
/// Result alias for reasoning parser operations.
pub type Result<T> = std::result::Result<T, ReasoningError>;
@@ -140,6 +145,8 @@ pub trait ReasoningParser: Send {
/// Errors produced while creating or running reasoning parsers.
#[derive(Debug, Error)]
pub enum ReasoningError {
#[error(transparent)]
Tokenizer(#[from] vllm_tokenizer::TokenizerError),
#[error("tokenizer is missing reasoning delimiter token `{token}`")]
MissingToken { token: String },
#[error(
rust/src/parser/src/reasoning/qwen3.rs
+11 -7
View File
@@ -3,13 +3,14 @@
use vllm_tokenizer::{DecodedText, DynTokenizer};
use super::{DelimitedReasoningParser, ReasoningDelta, ReasoningParser, Result};
use super::{
DelimitedReasoningParser, DelimitedReasoningParserBuilder, ReasoningDelta, ReasoningParser,
Result,
};
/// Reasoning parser for the Qwen3/Qwen3.5 family.
///
/// This parser uses standard `<think>...</think>` delimiters and defaults to
/// waiting for an explicit start token when prompt initialization finds no
/// reasoning boundary.
/// This parser uses standard `<think>...</think>` delimiters.
pub struct Qwen3ReasoningParser {
inner: DelimitedReasoningParser,
}
@@ -18,7 +19,11 @@ impl Qwen3ReasoningParser {
/// Create a Qwen3 parser backed by the shared delimited state machine.
pub fn new(tokenizer: DynTokenizer) -> Result<Self> {
Ok(Self {
inner: DelimitedReasoningParser::new(tokenizer, "<think>", "</think>")?,
inner: DelimitedReasoningParserBuilder::new(tokenizer, "<think>", "</think>")
.with_after_start("\n")
.with_before_end("\n")
.with_after_end("\n\n")
.build()?,
})
}
}
@@ -32,8 +37,7 @@ impl ReasoningParser for Qwen3ReasoningParser {
}
fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> {
self.inner.initialize(prompt_token_ids);
Ok(())
self.inner.initialize(prompt_token_ids)
}
fn push(&mut self, delta: DecodedText) -> Result<ReasoningDelta> {
rust/src/parser/src/reasoning/seed_oss.rs
+8 -4
View File
@@ -3,7 +3,10 @@
use vllm_tokenizer::{DecodedText, DynTokenizer};
use super::{DelimitedReasoningParser, ReasoningDelta, ReasoningParser, Result};
use super::{
DelimitedReasoningParser, DelimitedReasoningParserBuilder, ReasoningDelta, ReasoningParser,
Result,
};
/// Reasoning parser for SeedOSS models using `<seed:think>`/`</seed:think>`
/// delimiters.
@@ -15,7 +18,9 @@ impl SeedOssReasoningParser {
/// Create a SeedOSS parser backed by the shared delimited state machine.
pub fn new(tokenizer: DynTokenizer) -> Result<Self> {
Ok(Self {
inner: DelimitedReasoningParser::new(tokenizer, "<seed:think>", "</seed:think>")?,
inner: DelimitedReasoningParserBuilder::new(tokenizer, "<seed:think>", "</seed:think>")
.with_after_end("\n")
.build()?,
})
}
}
@@ -29,8 +34,7 @@ impl ReasoningParser for SeedOssReasoningParser {
}
fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> {
self.inner.initialize(prompt_token_ids);
Ok(())
self.inner.initialize(prompt_token_ids)
}
fn push(&mut self, delta: DecodedText) -> Result<ReasoningDelta> {
rust/src/parser/src/reasoning/step3p5.rs
+14 -93
View File
@@ -3,89 +3,27 @@
use vllm_tokenizer::{DecodedText, DynTokenizer};
use super::{DelimitedReasoningParser, ReasoningDelta, ReasoningParser, Result};
use super::{
DelimitedReasoningParser, DelimitedReasoningParserBuilder, ReasoningDelta, ReasoningParser,
Result,
};
/// Reasoning parser for Step3p5 outputs.
///
/// Step3p5 uses standard `<think>`/`</think>` delimiters but emits a `\n`
/// immediately before and/or after `</think>`. The parser drops these framing
/// newlines on both sides of the boundary, holding a trailing `\n` from
/// reasoning across pushes until either more reasoning text or `</think>`
/// arrives, and dropping a leading `\n` from the first content delta after
/// the boundary.
/// Reasoning parser for Step3p5 style outputs.
pub struct Step3p5ReasoningParser {
inner: DelimitedReasoningParser,
/// `\n` at end of last reasoning delta, held in case `</think>` follows.
pending_reasoning_newline: Option<DecodedText>,
/// Last push ended on `</think>` without emitting content; the next
/// content delta's leading `\n` should be dropped.
just_ended_reasoning: bool,
}
impl Step3p5ReasoningParser {
/// Create a Step3p5 parser backed by the shared delimited state machine.
pub fn new(tokenizer: DynTokenizer) -> Result<Self> {
Ok(Self {
inner: DelimitedReasoningParser::new(tokenizer, "<think>", "</think>")?,
pending_reasoning_newline: None,
just_ended_reasoning: false,
inner: DelimitedReasoningParserBuilder::new(tokenizer, "<think>", "</think>")
.with_after_start("\n")
.with_before_end("\n")
.with_after_end("\n")
.build()?,
})
}
/// Drop framing newlines around `</think>` and track held-newline state.
fn process(
&mut self,
mut inner_delta: ReasoningDelta,
was_in_reasoning: bool,
now_in_reasoning: bool,
) -> ReasoningDelta {
// A `<think>...</think>` round-trip in one push still counts as a
// transition: the inner emits reasoning while ending in content mode.
let transitioned =
!now_in_reasoning && (was_in_reasoning || inner_delta.reasoning.is_some());
// Replay or drop a previously-held trailing reasoning newline.
if self.pending_reasoning_newline.is_some() {
if let Some(reasoning) = inner_delta.reasoning.as_mut() {
let mut held = self.pending_reasoning_newline.take().unwrap();
held.append(std::mem::take(reasoning));
*reasoning = held;
} else if transitioned {
// The held `\n` was the one right before `</think>`: drop it.
self.pending_reasoning_newline = None;
}
}
// Hold back a trailing reasoning `\n` until we know if `</think>` follows.
if let Some(reasoning) = inner_delta.reasoning.as_mut()
&& reasoning.text.ends_with('\n')
{
let kept = reasoning.drain_prefix(reasoning.text.len() - 1);
let newline = std::mem::replace(reasoning, kept);
if !transitioned {
self.pending_reasoning_newline = Some(newline);
}
}
// Drop a leading `\n` of content emitted right after `</think>`.
if let Some(content) = inner_delta.content.as_mut()
&& (transitioned || self.just_ended_reasoning)
&& content.text.starts_with('\n')
{
let _ = content.drain_prefix(1);
}
self.just_ended_reasoning = transitioned && inner_delta.content.is_none();
if inner_delta.reasoning.as_ref().is_some_and(DecodedText::is_empty) {
inner_delta.reasoning = None;
}
if inner_delta.content.as_ref().is_some_and(DecodedText::is_empty) {
inner_delta.content = None;
}
inner_delta
}
}
impl ReasoningParser for Step3p5ReasoningParser {
@@ -97,32 +35,15 @@ impl ReasoningParser for Step3p5ReasoningParser {
}
fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> {
self.inner.initialize(prompt_token_ids);
Ok(())
self.inner.initialize(prompt_token_ids)
}
fn push(&mut self, delta: DecodedText) -> Result<ReasoningDelta> {
let was = self.inner.in_reasoning();
let inner_delta = self.inner.push(delta);
let now = self.inner.in_reasoning();
Ok(self.process(inner_delta, was, now))
Ok(self.inner.push(delta))
}
fn finish(&mut self) -> Result<ReasoningDelta> {
let was = self.inner.in_reasoning();
let inner_delta = self.inner.finish();
let now = self.inner.in_reasoning();
let mut delta = self.process(inner_delta, was, now);
// Emit a still-held newline rather than silently dropping it.
if let Some(held) = self.pending_reasoning_newline.take() {
match delta.reasoning.as_mut() {
Some(existing) => existing.append(held),
None => delta.reasoning = Some(held),
}
}
Ok(delta)
Ok(self.inner.finish())
}
}
@@ -181,7 +102,7 @@ mod tests {
// `</think>`; surrounding newlines remain part of reasoning/content.
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap();
parser.initialize(&[THINK_START_ID]).unwrap();
parser.initialize(&[THINK_START_ID, u32::from(b'\n')]).unwrap();
let delta = push_str(
&mut parser,
rust/src/parser/src/reasoning/tests.rs
+190 -10
View File
@@ -8,8 +8,9 @@ use vllm_tokenizer::{DecodedText, DynTokenizer, TokenAnchor, TokenAttribution};
use super::{
CohereCmdReasoningParser, DeepSeekR1ReasoningParser, DelimitedReasoningParser,
KimiReasoningParser, MiniMaxM3ReasoningParser, Qwen3ReasoningParser, ReasoningDelta,
ReasoningParser, Result, SeedOssReasoningParser, Step3p5ReasoningParser,
DelimitedReasoningParserBuilder, KimiReasoningParser, MiniMaxM3ReasoningParser,
Qwen3ReasoningParser, ReasoningDelta, ReasoningParser, Result, SeedOssReasoningParser,
Step3p5ReasoningParser,
};
pub(crate) const THINK_START_ID: u32 = 256;
@@ -57,7 +58,9 @@ pub(crate) fn content_str(delta: &ReasoningDelta) -> Option<&str> {
#[test]
fn delimited_content_only_stream() {
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = DelimitedReasoningParser::new(tokenizer, "<think>", "</think>").unwrap();
let mut parser = DelimitedReasoningParserBuilder::new(tokenizer, "<think>", "</think>")
.build()
.unwrap();
let delta = parser.push(DecodedText::unattributed("plain content"));
assert_eq!(content_str(&delta), Some("plain content"));
@@ -66,7 +69,9 @@ fn delimited_content_only_stream() {
#[test]
fn delimited_single_chunk_with_reasoning_and_content() {
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = DelimitedReasoningParser::new(tokenizer, "<think>", "</think>").unwrap();
let mut parser = DelimitedReasoningParserBuilder::new(tokenizer, "<think>", "</think>")
.build()
.unwrap();
let delta = parser.push(DecodedText::unattributed("<think>reason</think>answer"));
assert_eq!(reasoning_str(&delta), Some("reason"));
@@ -76,7 +81,9 @@ fn delimited_single_chunk_with_reasoning_and_content() {
#[test]
fn delimited_partial_tokens_across_chunks() {
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = DelimitedReasoningParser::new(tokenizer, "<think>", "</think>").unwrap();
let mut parser = DelimitedReasoningParserBuilder::new(tokenizer, "<think>", "</think>")
.build()
.unwrap();
assert!(parser.push(DecodedText::unattributed("<thi")).is_empty());
let delta = parser.push(DecodedText::unattributed("nk>reason</think>answer"));
@@ -87,8 +94,10 @@ fn delimited_partial_tokens_across_chunks() {
#[test]
fn delimited_finish_flushes_buffer() {
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = DelimitedReasoningParser::new(tokenizer, "<think>", "</think>").unwrap();
parser.initialize(&[THINK_START_ID]);
let mut parser = DelimitedReasoningParserBuilder::new(tokenizer, "<think>", "</think>")
.build()
.unwrap();
parser.initialize(&[THINK_START_ID]).unwrap();
let delta = parser.push(DecodedText::unattributed("unfinished</thi"));
assert_eq!(reasoning_str(&delta), Some("unfinished"));
@@ -99,8 +108,10 @@ fn delimited_finish_flushes_buffer() {
#[test]
fn delimited_zero_width_only_piece_is_attributed_to_current_state() {
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = DelimitedReasoningParser::new(tokenizer, "<think>", "</think>").unwrap();
parser.initialize(&[THINK_START_ID]);
let mut parser = DelimitedReasoningParserBuilder::new(tokenizer, "<think>", "</think>")
.build()
.unwrap();
parser.initialize(&[THINK_START_ID]).unwrap();
// A filtered special token produces a zero-width attribution with no text;
// it must survive as a reasoning piece rather than being dropped.
@@ -130,7 +141,9 @@ fn delimited_zero_width_only_piece_is_attributed_to_current_state() {
#[test]
fn delimited_marker_tokens_are_dropped_from_attributions() {
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = DelimitedReasoningParser::new(tokenizer, "<think>", "</think>").unwrap();
let mut parser = DelimitedReasoningParserBuilder::new(tokenizer, "<think>", "</think>")
.build()
.unwrap();
let mut collected = CollectedAttributions::default();
for chunk in attributed_chunks(&[
@@ -283,3 +296,170 @@ pub(crate) fn collect_attributed(
collected.record(parser.finish().unwrap());
collected
}
fn framed_parser() -> DelimitedReasoningParser {
DelimitedReasoningParserBuilder::new(Arc::new(fake_tokenizer()), "<think>", "</think>")
.with_after_start("\n")
.with_before_end("\n")
.with_after_end("\n\n")
.build()
.unwrap()
}
fn collect_framed(parser: &mut DelimitedReasoningParser, chunks: &[&str]) -> (String, String) {
let mut result = (String::new(), String::new());
for chunk in chunks.iter().copied().map(Some).chain(std::iter::once(None)) {
let delta = match chunk {
Some(chunk) => parser.push(DecodedText::unattributed(chunk)),
None => parser.finish(),
};
if let Some(reasoning) = delta.reasoning {
result.0.push_str(&reasoning.text);
}
if let Some(content) = delta.content {
result.1.push_str(&content.text);
}
}
result
}
#[test]
fn delimited_framing_preserves_body_whitespace_at_every_split() {
let wire = "<think>\n\n reason\t\n\n</think>\n\n\n answer\n";
let expected = ("\n reason\t\n".to_string(), "\n answer\n".to_string());
for split in 0..=wire.len() {
let mut parser = framed_parser();
assert_eq!(
collect_framed(&mut parser, &[&wire[..split], &wire[split..]]),
expected,
"split at {split}"
);
}
let chars = wire
.as_bytes()
.chunks(1)
.map(|c| std::str::from_utf8(c).unwrap())
.collect::<Vec<_>>();
assert_eq!(collect_framed(&mut framed_parser(), &chars), expected);
}
#[test]
fn delimited_partial_framing_is_preserved_on_mismatch_or_finish() {
for (wire, reasoning, content) in [
("<think>reason</think>\nanswer", "reason", "\nanswer"),
("<think>\nreason\nmore\n", "reason\nmore\n", ""),
("<think>\nreason\n</thi", "reason\n</thi", ""),
("<think>\nreason\n</think>\n", "reason", "\n"),
(" plain\n", "", " plain\n"),
] {
for split in 0..=wire.len() {
assert_eq!(
collect_framed(&mut framed_parser(), &[&wire[..split], &wire[split..]]),
(reasoning.to_string(), content.to_string()),
"wire {wire:?}, split {split}"
);
}
}
}
#[test]
fn delimited_prompt_initialization_consumes_only_remaining_framing() {
use vllm_tokenizer::Tokenizer;
let tokenizer = fake_tokenizer();
for (prompt, wire, reasoning, content) in [
(
"<think>",
"\nreason\n</think>\n\nanswer",
"reason",
"answer",
),
(
"<think>\n",
"\nreason\n</think>\n\nanswer",
"\nreason",
"answer",
),
(
"<think>\nprefill",
"\nreason\n</think>\n\nanswer",
"\nreason",
"answer",
),
("</think>", "\n\n\nanswer", "", "\nanswer"),
("</think>\n", "\n\nanswer", "", "\nanswer"),
("</think>\n\n", "\nanswer", "", "\nanswer"),
("</think>\n\nprefill", "\nanswer", "", "\nanswer"),
] {
let mut parser = framed_parser();
parser.initialize(&tokenizer.encode(prompt, false).unwrap()).unwrap();
assert_eq!(
collect_framed(&mut parser, &[wire]),
(reasoning.to_string(), content.to_string()),
"prompt {prompt:?}"
);
}
}
#[test]
fn delimited_framing_holds_only_boundary_candidates() {
let mut parser = framed_parser();
let first = parser.push(DecodedText::unattributed("<think>\nreason\n"));
assert_eq!(reasoning_str(&first), Some("reason"));
let more = parser.push(DecodedText::unattributed("more\n"));
assert_eq!(reasoning_str(&more), Some("\nmore"));
assert!(parser.push(DecodedText::unattributed("</think>\n")).is_empty());
let answer = parser.push(DecodedText::unattributed("\n answer"));
assert_eq!(content_str(&answer), Some(" answer"));
}
#[test]
fn delimited_start_prefix_is_consumed_only_before_the_marker() {
for wire in [
"\n<think>reason</think>answer",
"\n\n<think>reason</think>answer",
] {
for split in 0..=wire.len() {
let mut parser = DelimitedReasoningParserBuilder::new(
Arc::new(fake_tokenizer()),
"<think>",
"</think>",
)
.with_before_start("\n")
.build()
.unwrap();
let expected_content = if wire.starts_with("\n\n") {
"\nanswer"
} else {
"answer"
};
assert_eq!(
collect_framed(&mut parser, &[&wire[..split], &wire[split..]]),
("reason".to_string(), expected_content.to_string())
);
}
}
}
#[test]
fn delimited_framing_preserves_body_token_attributions() {
let mut parser = framed_parser();
let mut collected = CollectedAttributions::default();
for chunk in attributed_chunks(&[
Some("<think>"),
Some("\n"),
Some("reason"),
None,
Some("\n"),
Some("</think>"),
Some("\n"),
Some("\n"),
Some(" answer"),
]) {
collected.record(parser.push(chunk));
}
collected.record(parser.finish());
assert_eq!(collected.reasoning_text, "reason");
assert_eq!(collected.content_text, " answer");
assert_eq!(collected.reasoning_ids, [3, 4]);
assert_eq!(collected.content_ids, [9]);
}
rust/src/parser/src/tool/deepseek_dsml/deepseek_v32.rs
+9
View File
@@ -71,6 +71,7 @@ mod tests {
use super::DeepSeekV32ToolParser;
use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools};
use crate::tool::tests::assert_tool_framing_preserves_body_whitespace;
use crate::tool::{ToolParser, ToolParserTestExt as _};
fn build_tool_call(function_name: &str, params: &[(&str, &str)]) -> String {
@@ -445,4 +446,12 @@ mod tests {
assert_eq!(streamed.normal_text(), complete.normal_text());
assert_eq!(streamed.calls(), complete.calls());
}
#[test]
fn tool_framing_preserves_body_whitespace_across_chunk_boundaries() {
assert_tool_framing_preserves_body_whitespace::<DeepSeekV32ToolParser>(
"\n\n",
"<｜DSML｜function_calls><｜DSML｜invoke name=\"get_weather\"></｜DSML｜invoke></｜DSML｜function_calls>",
);
}
}
rust/src/parser/src/tool/deepseek_dsml/deepseek_v4.rs
+9
View File
@@ -73,6 +73,7 @@ mod tests {
use super::DeepSeekV4ToolParser;
use crate::tool::test_utils::{collect_stream, test_tools};
use crate::tool::tests::assert_tool_framing_preserves_body_whitespace;
use crate::tool::{ToolParser, ToolParserTestExt as _};
fn build_tool_call(function_name: &str, params: &[(&str, &str)]) -> String {
@@ -145,4 +146,12 @@ mod tests {
json!({ "location": "Beijing" })
);
}
#[test]
fn tool_framing_preserves_body_whitespace_across_chunk_boundaries() {
assert_tool_framing_preserves_body_whitespace::<DeepSeekV4ToolParser>(
"\n\n",
"<｜DSML｜tool_calls><｜DSML｜invoke name=\"get_weather\"></｜DSML｜invoke></｜DSML｜tool_calls>",
);
}
}
rust/src/parser/src/tool/deepseek_dsml/mod.rs
+15 -5
View File
@@ -8,7 +8,7 @@ use winnow::stream::Partial;
use winnow::token::{literal, rest, take_until};
use super::parameters::ToolSchemas;
use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len, take_until_marker};
use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len_mul, take_until_marker};
use super::{Result, ToolCallDelta, ToolParserOutput};
use crate::tool::Tool;
@@ -28,16 +28,19 @@ type DsmlInput<'i> = Partial<&'i str>;
#[derive(Debug, Clone, Copy)]
struct DsmlTokens {
tool_calls_start: &'static str,
framed_tool_calls_start: &'static str,
tool_calls_end: &'static str,
}
impl DsmlTokens {
const V32: Self = Self {
tool_calls_start: "<｜DSML｜function_calls>",
framed_tool_calls_start: "\n\n<｜DSML｜function_calls>",
tool_calls_end: "</｜DSML｜function_calls>",
};
const V4: Self = Self {
tool_calls_start: "<｜DSML｜tool_calls>",
framed_tool_calls_start: "\n\n<｜DSML｜tool_calls>",
tool_calls_end: "</｜DSML｜tool_calls>",
};
}
@@ -210,9 +213,12 @@ fn parse_tool_block_event(
/// Parse a DSML function-calls start marker.
fn tool_calls_start_event(input: &mut DsmlInput<'_>, tokens: DsmlTokens) -> ModalResult<DsmlEvent> {
literal(tokens.tool_calls_start)
.value(DsmlEvent::ToolCallsStart)
.parse_next(input)
alt((
literal(tokens.framed_tool_calls_start),
literal(tokens.tool_calls_start),
))
.value(DsmlEvent::ToolCallsStart)
.parse_next(input)
}
/// Parse a DSML function-calls end marker.
@@ -227,7 +233,11 @@ fn ignored_rest_event(input: &mut DsmlInput<'_>) -> ModalResult<DsmlEvent> {
/// Parse a safe text run before the next DSML marker.
fn safe_text_event(input: &mut DsmlInput<'_>, tokens: DsmlTokens) -> ModalResult<DsmlEvent> {
safe_text_len(input, tokens.tool_calls_start).map(|len| DsmlEvent::Text { len })
safe_text_len_mul(
input,
&[tokens.framed_tool_calls_start, tokens.tool_calls_start],
)
.map(|len| DsmlEvent::Text { len })
}
/// Parse a DSML invoke block.
rust/src/parser/src/tool/glm_xml/glm45_moe.rs
+14
View File
@@ -45,3 +45,17 @@ impl ToolParser for Glm45MoeToolParser {
self.0.reset()
}
}
#[cfg(test)]
mod tests {
use super::Glm45MoeToolParser;
use crate::tool::tests::assert_tool_framing_preserves_body_whitespace;
#[test]
fn tool_framing_preserves_body_whitespace_across_chunk_boundaries() {
assert_tool_framing_preserves_body_whitespace::<Glm45MoeToolParser>(
"\n",
"<tool_call>get_weather\n</tool_call>",
);
}
}
rust/src/parser/src/tool/glm_xml/mod.rs
+18 -10
View File
@@ -8,7 +8,7 @@ use winnow::stream::Partial;
use winnow::token::{literal, rest, take_until, take_while};
use super::parameters::ToolSchemas;
use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len, take_until_marker};
use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len_mul, take_until_marker};
use super::{Result, ToolCallDelta, ToolParserOutput};
use crate::tool::Tool;
@@ -19,6 +19,7 @@ pub use glm45_moe::Glm45MoeToolParser;
pub use glm47_moe::Glm47MoeToolParser;
const TOOL_CALL_START: &str = "<tool_call>";
const FRAMED_TOOL_CALL_START: &str = "\n<tool_call>";
const TOOL_CALL_END: &str = "</tool_call>";
const ARG_KEY_START: &str = "<arg_key>";
const ARG_KEY_END: &str = "</arg_key>";
@@ -149,7 +150,7 @@ fn parse_next_glm_event(
separator: Separator,
) -> ModalResult<GlmEvent> {
match mode {
GlmMode::Text => parse_text_event(input),
GlmMode::Text => parse_text_event(input, separator),
GlmMode::ToolCall { tool_call_end_scan } => {
tool_call_event(input, separator, tool_call_end_scan)
}
@@ -158,8 +159,20 @@ fn parse_next_glm_event(
}
/// Parse a text-mode GLM event.
fn parse_text_event(input: &mut GlmInput<'_>) -> ModalResult<GlmEvent> {
alt((tool_call_start_event, safe_text_event)).parse_next(input)
fn parse_text_event(input: &mut GlmInput<'_>, separator: Separator) -> ModalResult<GlmEvent> {
let framed_start = match separator {
Separator::Newline => FRAMED_TOOL_CALL_START,
Separator::Flexible => TOOL_CALL_START,
};
alt((
literal(framed_start).value(GlmEvent::ToolCallStart),
tool_call_start_event,
|input: &mut GlmInput<'_>| {
safe_text_len_mul(input, &[framed_start, TOOL_CALL_START])
.map(|len| GlmEvent::Text { len })
},
))
.parse_next(input)
}
/// Parse a GLM tool-call start marker.
@@ -167,11 +180,6 @@ fn tool_call_start_event(input: &mut GlmInput<'_>) -> ModalResult<GlmEvent> {
literal(TOOL_CALL_START).value(GlmEvent::ToolCallStart).parse_next(input)
}
/// Parse a safe text run before the next GLM marker.
fn safe_text_event(input: &mut GlmInput<'_>) -> ModalResult<GlmEvent> {
safe_text_len(input, TOOL_CALL_START).map(|len| GlmEvent::Text { len })
}
/// Parse text after a completed GLM tool call.
fn after_tool_call_event(input: &mut GlmInput<'_>) -> ModalResult<GlmEvent> {
ws0.void().parse_next(input)?;
@@ -303,7 +311,7 @@ mod tests {
let output = parser.parse_complete(&output).unwrap();
assert_eq!(output.normal_text(), "Let me search for that.\n");
assert_eq!(output.normal_text(), "Let me search for that.");
assert_eq!(output.calls().len(), 1);
assert_eq!(output.calls()[0].name.as_deref(), Some("get_weather"));
assert_eq!(
rust/src/parser/src/tool/json/granite4.rs
+1
View File
@@ -222,6 +222,7 @@ fn header_event(input: &mut JsonToolInput<'_>) -> ModalResult<Granite4Event> {
const CONFIG: JsonToolCallConfig = JsonToolCallConfig {
parser_name: "Granite4",
start_marker: "",
framed_start_marker: None,
end_marker: "",
marker_whitespace: JsonToolCallWhitespace::Optional,
delimiter: None,
rust/src/parser/src/tool/json/hermes.rs
+1
View File
@@ -7,6 +7,7 @@ use crate::tool::{Result, StructuralTagBuilder, Tool, ToolParser, ToolParserOutp
const HERMES_CONFIG: JsonToolCallConfig = JsonToolCallConfig {
parser_name: "Hermes",
start_marker: "<tool_call>",
framed_start_marker: None,
end_marker: "</tool_call>",
marker_whitespace: JsonToolCallWhitespace::Optional,
delimiter: None,
rust/src/parser/src/tool/json/internlm2.rs
+1
View File
@@ -7,6 +7,7 @@ use crate::tool::{Result, Tool, ToolParser, ToolParserOutput};
const INTERNLM2_CONFIG: JsonToolCallConfig = JsonToolCallConfig {
parser_name: "InternLM2",
start_marker: "<|action_start|><|plugin|>",
framed_start_marker: None,
end_marker: "<|action_end|>",
marker_whitespace: JsonToolCallWhitespace::Optional,
delimiter: None,
rust/src/parser/src/tool/json/llama.rs
+1
View File
@@ -208,6 +208,7 @@ fn llama_tool_call_header_event(input: &mut JsonToolInput<'_>) -> ModalResult<Ll
const CONFIG: JsonToolCallConfig = JsonToolCallConfig {
parser_name: "Llama JSON",
start_marker: "",
framed_start_marker: None,
end_marker: "",
marker_whitespace: JsonToolCallWhitespace::Optional,
delimiter: Some(";"),
rust/src/parser/src/tool/json/mistral.rs
+1
View File
@@ -7,6 +7,7 @@ use crate::tool::{Result, Tool, ToolParser, ToolParserOutput};
const MISTRAL_CONFIG: JsonToolCallConfig = JsonToolCallConfig {
parser_name: "Mistral",
start_marker: "[TOOL_CALLS] [",
framed_start_marker: None,
end_marker: "]",
marker_whitespace: JsonToolCallWhitespace::Optional,
delimiter: Some(","),
rust/src/parser/src/tool/json/mod.rs
+17 -3
View File
@@ -27,7 +27,8 @@ use winnow::stream::{Partial, Stream};
use winnow::token::literal;
use super::utils::{
JsonObjectScanState, json_str, parse_buffered_event, safe_text_len, take_json_object,
JsonObjectScanState, json_str, parse_buffered_event, safe_text_len, safe_text_len_mul,
take_json_object,
};
use super::{Result, ToolCallDelta, ToolParserOutput};
@@ -37,7 +38,12 @@ pub(crate) type JsonToolInput<'i> = Partial<&'i str>;
pub(crate) struct JsonToolCallConfig {
pub parser_name: &'static str,
pub start_marker: &'static str,
/// Start marker with its fixed preceding framing, matched alongside the
/// bare start marker.
pub framed_start_marker: Option<&'static str>,
pub end_marker: &'static str,
/// Whitespace inside the tool-call markers: after the start marker and
/// before the end marker.
pub marker_whitespace: JsonToolCallWhitespace,
pub delimiter: Option<&'static str>,
pub name_key: &'static str,
@@ -211,7 +217,10 @@ fn tool_call_start_event(
config: JsonToolCallConfig,
) -> ModalResult<JsonToolCallEvent> {
seq!(
_: literal(config.start_marker),
_: |input: &mut JsonToolInput<'_>| match config.framed_start_marker {
Some(marker) => alt((literal(marker), literal(config.start_marker))).void().parse_next(input),
None => literal(config.start_marker).void().parse_next(input),
},
_: |input: &mut JsonToolInput<'_>| marker_whitespace(input, config),
)
.value(JsonToolCallEvent::ToolCallStart)
@@ -369,7 +378,11 @@ fn safe_text_event(
input: &mut JsonToolInput<'_>,
config: JsonToolCallConfig,
) -> ModalResult<JsonToolCallEvent> {
safe_text_len(input, config.start_marker).map(|len| JsonToolCallEvent::Text { len })
match config.framed_start_marker {
Some(marker) => safe_text_len_mul(input, &[marker, config.start_marker]),
None => safe_text_len(input, config.start_marker),
}
.map(|len| JsonToolCallEvent::Text { len })
}
#[cfg(test)]
@@ -382,6 +395,7 @@ mod tests {
const DELIMITED_CONFIG: JsonToolCallConfig = JsonToolCallConfig {
parser_name: "Delimited JSON",
start_marker: "<tool_calls>",
framed_start_marker: None,
end_marker: "</tool_calls>",
marker_whitespace: JsonToolCallWhitespace::Optional,
delimiter: Some("<"),
rust/src/parser/src/tool/json/phi4mini.rs
+1
View File
@@ -7,6 +7,7 @@ use crate::tool::{Result, Tool, ToolParser, ToolParserOutput};
const PHI4MINI_CONFIG: JsonToolCallConfig = JsonToolCallConfig {
parser_name: "Phi4Mini",
start_marker: "functools[",
framed_start_marker: None,
end_marker: "]",
marker_whitespace: JsonToolCallWhitespace::Optional,
delimiter: Some(","),
rust/src/parser/src/tool/json/qwen.rs
+11 -1
View File
@@ -7,6 +7,7 @@ use crate::tool::{Result, StructuralTagBuilder, Tool, ToolParser, ToolParserOutp
const QWEN_XML_CONFIG: JsonToolCallConfig = JsonToolCallConfig {
parser_name: "Qwen XML",
start_marker: "<tool_call>",
framed_start_marker: Some("\n<tool_call>"),
end_marker: "</tool_call>",
marker_whitespace: JsonToolCallWhitespace::Exact("\n"),
delimiter: None,
@@ -74,6 +75,7 @@ mod tests {
use super::Qwen3XmlToolParser;
use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools};
use crate::tool::tests::assert_tool_framing_preserves_body_whitespace;
use crate::tool::{ToolParser, ToolParserOutput, ToolParserTestExt as _};
fn build_tool_call(function_name: &str, arguments: &str) -> String {
@@ -102,7 +104,7 @@ mod tests {
))
.unwrap();
assert_eq!(output.normal_text(), "Let me check.\n");
assert_eq!(output.normal_text(), "Let me check.");
assert_eq!(output.calls().len(), 1);
assert_eq!(output.calls()[0].tool_index, 0);
assert_eq!(output.calls()[0].name.as_deref(), Some("get_weather"));
@@ -287,4 +289,12 @@ mod tests {
expected `name`"#]]
.assert_eq(&error.to_report_string());
}
#[test]
fn tool_framing_preserves_body_whitespace_across_chunk_boundaries() {
assert_tool_framing_preserves_body_whitespace::<Qwen3XmlToolParser>(
"\n",
"<tool_call>\n{\"name\":\"get_weather\",\"arguments\":{}}\n</tool_call>",
);
}
}
rust/src/parser/src/tool/minimax_m2.rs
+17 -4
View File
@@ -8,11 +8,12 @@ use winnow::stream::Partial;
use winnow::token::{literal, rest, take_until};
use super::parameters::ToolSchemas;
use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len, take_until_marker};
use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len_mul, take_until_marker};
use super::{Result, ToolCallDelta, ToolParser, ToolParserOutput};
use crate::tool::{StructuralTagBuilder, Tool};
const TOOL_CALL_START: &str = "<minimax:tool_call>";
const FRAMED_TOOL_CALL_START: &str = "\n<minimax:tool_call>";
const TOOL_CALL_END: &str = "</minimax:tool_call>";
const INVOKE_START: &str = "<invoke";
const INVOKE_END: &str = "</invoke>";
@@ -173,12 +174,15 @@ fn parse_text_event(input: &mut MinimaxM2Input<'_>) -> ModalResult<MinimaxM2Even
/// Parse a MiniMax M2 tool-block start marker.
fn tool_block_start_event(input: &mut MinimaxM2Input<'_>) -> ModalResult<MinimaxM2Event> {
literal(TOOL_CALL_START).value(MinimaxM2Event::ToolBlockStart).parse_next(input)
alt((literal(FRAMED_TOOL_CALL_START), literal(TOOL_CALL_START)))
.value(MinimaxM2Event::ToolBlockStart)
.parse_next(input)
}
/// Parse a safe text run before the next MiniMax M2 marker.
fn safe_text_event(input: &mut MinimaxM2Input<'_>) -> ModalResult<MinimaxM2Event> {
safe_text_len(input, TOOL_CALL_START).map(|len| MinimaxM2Event::Text { len })
safe_text_len_mul(input, &[FRAMED_TOOL_CALL_START, TOOL_CALL_START])
.map(|len| MinimaxM2Event::Text { len })
}
/// Parse one event inside a MiniMax M2 tool block.
@@ -277,6 +281,7 @@ mod tests {
use super::{MinimaxM2ToolParser, TOOL_CALL_END, TOOL_CALL_START, ToolParser};
use crate::tool::ToolParserTestExt as _;
use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools};
use crate::tool::tests::assert_tool_framing_preserves_body_whitespace;
fn build_tool_block(invokes: &[(&str, Vec<(&str, &str)>)]) -> String {
let invokes = invokes
@@ -543,7 +548,7 @@ mod tests {
let mut parser = MinimaxM2ToolParser::new(&test_tools());
let result = collect_stream(&mut parser, &chunks);
assert_eq!(result.normal_text(), "I will call the tools.\n");
assert_eq!(result.normal_text(), "I will call the tools.");
assert_eq!(result.calls().len(), 2);
assert_eq!(result.calls()[0].tool_index, 0);
assert_eq!(result.calls()[0].name.as_deref(), Some("get_weather"));
@@ -599,4 +604,12 @@ mod tests {
expect![[r#"tool parser parsing failed: near "<bad></minimax:tool_call>": "#]]
.assert_eq(&error.to_report_string());
}
#[test]
fn tool_framing_preserves_body_whitespace_across_chunk_boundaries() {
assert_tool_framing_preserves_body_whitespace::<MinimaxM2ToolParser>(
"\n",
"<minimax:tool_call><invoke name=\"get_weather\"></invoke></minimax:tool_call>",
);
}
}
rust/src/parser/src/tool/qwen_coder.rs
+29 -21
View File
@@ -8,7 +8,7 @@ use winnow::stream::Partial;
use winnow::token::{literal, take_until};
use super::parameters::ToolSchemas;
use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len, take_until_marker};
use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len_mul, take_until_marker};
use super::{Result, ToolCallDelta, ToolParser, ToolParserOutput};
use crate::tool::{StructuralTagBuilder, Tool};
@@ -31,6 +31,8 @@ pub(crate) struct QwenCoderConfig {
pub(crate) parser_name: &'static str,
/// Marker that opens a tool-call block.
pub(crate) tool_call_start: &'static str,
pub(crate) first_tool_call_start: &'static str,
pub(crate) next_tool_call_start: &'static str,
/// Marker that closes a tool-call block.
pub(crate) tool_call_end: &'static str,
}
@@ -38,6 +40,8 @@ pub(crate) struct QwenCoderConfig {
const QWEN_CODER_CONFIG: QwenCoderConfig = QwenCoderConfig {
parser_name: "Qwen Coder",
tool_call_start: TOOL_CALL_START,
first_tool_call_start: "\n\n<tool_call>",
next_tool_call_start: "\n<tool_call>",
tool_call_end: TOOL_CALL_END,
};
@@ -155,7 +159,7 @@ impl ToolParser for Qwen3CoderToolParser {
let config = self.config;
while let Some((event, consumed_len)) = parse_buffered_event(&self.buffer, |input| {
parse_next_qwen_coder_event(input, &mut self.mode, config)
parse_next_qwen_coder_event(input, &mut self.mode, config, self.emitted_tool_count > 0)
})? {
self.apply_event(event, output)?;
self.buffer.drain(..consumed_len);
@@ -191,9 +195,10 @@ fn parse_next_qwen_coder_event(
input: &mut QwenCoderInput<'_>,
mode: &mut QwenCoderMode,
config: QwenCoderConfig,
after_tool_call: bool,
) -> ModalResult<QwenCoderEvent> {
match mode {
QwenCoderMode::Text => parse_text_event(input, config),
QwenCoderMode::Text => parse_text_event(input, config, after_tool_call),
QwenCoderMode::ToolCall { end_marker_scan } => {
tool_call_event(input, end_marker_scan, config.tool_call_end)
}
@@ -204,30 +209,24 @@ fn parse_next_qwen_coder_event(
fn parse_text_event(
input: &mut QwenCoderInput<'_>,
config: QwenCoderConfig,
after_tool_call: bool,
) -> ModalResult<QwenCoderEvent> {
let framed_start = if after_tool_call {
config.next_tool_call_start
} else {
config.first_tool_call_start
};
alt((
|input: &mut QwenCoderInput<'_>| tool_call_start_event(input, config.tool_call_start),
|input: &mut QwenCoderInput<'_>| safe_text_event(input, config.tool_call_start),
alt((literal(framed_start), literal(config.tool_call_start)))
.value(QwenCoderEvent::ToolCallStart),
|input: &mut QwenCoderInput<'_>| {
safe_text_len_mul(input, &[framed_start, config.tool_call_start])
.map(|len| QwenCoderEvent::Text { len })
},
))
.parse_next(input)
}
/// Parse a Qwen Coder tool-call start marker.
fn tool_call_start_event(
input: &mut QwenCoderInput<'_>,
tool_call_start: &'static str,
) -> ModalResult<QwenCoderEvent> {
literal(tool_call_start).value(QwenCoderEvent::ToolCallStart).parse_next(input)
}
/// Parse a safe text run before the next Qwen Coder marker.
fn safe_text_event(
input: &mut QwenCoderInput<'_>,
tool_call_start: &'static str,
) -> ModalResult<QwenCoderEvent> {
safe_text_len(input, tool_call_start).map(|len| QwenCoderEvent::Text { len })
}
/// Parse a complete Qwen Coder tool call.
fn tool_call_event(
input: &mut QwenCoderInput<'_>,
@@ -296,6 +295,7 @@ mod tests {
use super::{Qwen3CoderToolParser, ToolParser};
use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools};
use crate::tool::tests::assert_tool_framing_preserves_body_whitespace;
use crate::tool::{ToolParserOutput, ToolParserTestExt as _};
fn build_tool_call(function_name: &str, params: &[(&str, &str)]) -> String {
@@ -766,4 +766,12 @@ mod tests {
json!({ "location": "Hangzhou" })
);
}
#[test]
fn tool_framing_preserves_body_whitespace_across_chunk_boundaries() {
assert_tool_framing_preserves_body_whitespace::<Qwen3CoderToolParser>(
"\n\n",
"<tool_call>\n<function=get_weather>\n</function>\n</tool_call>",
);
}
}
rust/src/parser/src/tool/seed_oss.rs
+11
View File
@@ -7,6 +7,8 @@ use crate::tool::{Result, Tool, ToolParser, ToolParserOutput};
const SEED_OSS_CONFIG: QwenCoderConfig = QwenCoderConfig {
parser_name: "Seed-OSS",
tool_call_start: "<seed:tool_call>",
first_tool_call_start: "\n\n<seed:tool_call>",
next_tool_call_start: "\n<seed:tool_call>",
tool_call_end: "</seed:tool_call>",
};
@@ -79,6 +81,7 @@ mod tests {
use super::SeedOssToolParser;
use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools};
use crate::tool::tests::assert_tool_framing_preserves_body_whitespace;
use crate::tool::{ToolParser, ToolParserOutput, ToolParserTestExt as _};
fn build_tool_call(function_name: &str, params: &[(&str, &str)]) -> String {
@@ -250,4 +253,12 @@ mod tests {
"tool parser parsing failed: incomplete Seed-OSS tool call"
);
}
#[test]
fn tool_framing_preserves_body_whitespace_across_chunk_boundaries() {
assert_tool_framing_preserves_body_whitespace::<SeedOssToolParser>(
"\n\n",
"<seed:tool_call>\n<function=get_weather>\n</function>\n</seed:tool_call>",
);
}
}
rust/src/parser/src/tool/tests.rs
+23
View File
@@ -170,3 +170,26 @@ fn default_parse_complete_delegates_through_parse_chunk_and_finish() {
]
);
}
pub(super) fn assert_tool_framing_preserves_body_whitespace<P: ToolParser + 'static>(
framing: &str,
call: &str,
) {
use super::test_utils::{collect_stream, test_tools};
let body = " before\n";
let wire = format!("{body}{framing}{call}");
for split in wire.char_indices().map(|(i, _)| i).chain(std::iter::once(wire.len())) {
let mut parser = P::create(&test_tools()).unwrap();
let output = collect_stream(parser.as_mut(), &[&wire[..split], &wire[split..]]);
assert_eq!(output.normal_text(), body, "split {split}");
assert_eq!(output.calls().len(), 1, "split {split}");
assert_eq!(output.calls()[0].name.as_deref(), Some("get_weather"));
}
let plain = " no tool\n\n<to";
let mut parser = P::create(&test_tools()).unwrap();
assert_eq!(
collect_stream(parser.as_mut(), &[plain]).normal_text(),
plain
);
}
rust/src/parser/src/unified/gemma4.rs
+18 -2
View File
@@ -20,6 +20,7 @@ use crate::utils::{incomplete, parse_buffered_event, partial_prefix_len, safe_te
const REASONING_START: &str = "<|channel>thought\n";
const CHANNEL_START: &str = "<|channel>";
const CHANNEL_END: &str = "<channel|>";
const FRAMED_CHANNEL_END: &str = "\n<channel|>";
const TOOL_CALL_START: &str = "<|tool_call>";
const TOOL_CALL_END: &str = "<tool_call|>";
const STRING_DELIM: &str = "<|\"|>";
@@ -259,7 +260,9 @@ fn reasoning_start_event(input: &mut Gemma4Input<'_>) -> ModalResult<Gemma4Event
/// Parse a Gemma4 reasoning end marker.
fn reasoning_end_event(input: &mut Gemma4Input<'_>) -> ModalResult<Gemma4Event> {
literal(CHANNEL_END).value(Gemma4Event::ReasoningEnd).parse_next(input)
alt((literal(FRAMED_CHANNEL_END), literal(CHANNEL_END)))
.value(Gemma4Event::ReasoningEnd)
.parse_next(input)
}
/// Parse a Gemma4 tool-call start marker.
@@ -308,7 +311,8 @@ fn safe_text_event(input: &mut Gemma4Input<'_>) -> ModalResult<Gemma4Event> {
/// Parse a safe reasoning run before the next Gemma4 marker.
fn safe_reasoning_event(input: &mut Gemma4Input<'_>) -> ModalResult<Gemma4Event> {
safe_text_len_mul(input, &[CHANNEL_END, TOOL_CALL_START]).map(|_| Gemma4Event::Reasoning)
safe_text_len_mul(input, &[FRAMED_CHANNEL_END, CHANNEL_END, TOOL_CALL_START])
.map(|_| Gemma4Event::Reasoning)
}
/// Parse raw Gemma4 arguments through the first end marker outside a Gemma string.
@@ -686,6 +690,18 @@ mod tests {
output.coalesce()
}
#[test]
fn gemma4_reasoning_framing_preserves_extra_newlines_at_every_split() {
let wire = "<|channel>thought\n\n reason\n\n<channel|> answer\n";
for split in 0..=wire.len() {
let output = collect_stream(&[&wire[..split], &wire[split..]]);
assert_eq!(output.reasoning_text(), "\n reason\n", "split {split}");
assert_eq!(output.normal_text(), " answer\n", "split {split}");
}
let output = collect_stream(&["<|channel>thought\nreason\n<channel"]);
assert_eq!(output.reasoning_text(), "reason\n<channel");
}
fn first_call(output: &UnifiedParserOutput) -> ToolCallDelta {
(*output.calls().first().expect("expected one tool call")).clone()
}
rust/src/parser/src/unified/inkling.rs
+1
View File
@@ -45,6 +45,7 @@ const BLOCK_END_MARKERS: &[&str] = &[END_MESSAGE, CONTENT_MODEL_END_SAMPLING];
const INKLING_TOOL_CONFIG: JsonToolCallConfig = JsonToolCallConfig {
parser_name: "Inkling",
start_marker: CONTENT_INVOKE_TOOL_JSON,
framed_start_marker: None,
end_marker: END_MESSAGE,
marker_whitespace: JsonToolCallWhitespace::Optional,
delimiter: None,