[Rust Frontend] Align tool parser fallback behavior between streaming & non-streaming paths (#43662)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-05-27 10:13:12 +00:00
committed by GitHub
parent ad464e16c0
commit 396c8fee50
26 changed files with 1187 additions and 899 deletions
+289 -55
View File
@@ -15,7 +15,7 @@ use crate::Result;
use crate::error::Error;
use crate::event::AssistantBlockKind;
use crate::output::generate_tool_call_id;
use crate::parser::tool::{ToolCallDelta, ToolParseResult, ToolParser};
use crate::parser::tool::{ToolCallDelta, ToolParser, ToolParserOutput};
/// Per-stream tool parsing state.
struct ToolState {
@@ -57,46 +57,52 @@ impl ToolState {
return Ok(events);
}
let parse_result = self.parser.push(&delta);
let mut output = ToolParserOutput::default();
let parse_result = self.parser.parse_into(&delta, &mut output);
match parse_result {
Ok(result) => self.process_parse_result(kind, result, &mut events)?,
Ok(()) => self.process_parser_output(kind, output, &mut events)?,
Err(error) => {
if !self.parser_failed {
warn!(
error = %error.as_report(),
"tool parser failed; falling back to plain text deltas"
);
self.parser_failed = true;
}
warn!(
error = %error.as_report(),
"tool parser failed; falling back to plain text deltas"
);
// Permanently mark this parser as failed.
// TODO: we may consider recovering from parsing errors in the future.
self.parser_failed = true;
// On parsing failure, we still apply the partial parser output if any, but we close
// any open tool calls and emit the remaining buffered text as a plain-text delta to
// preserve as much of the output as possible.
self.process_parser_output(kind, output, &mut events)?;
self.open_call_index = None;
events.push(AssistantEvent::TextDelta { kind, delta });
push_text_delta(&mut events, kind, self.parser.reset());
}
}
Ok(events)
}
/// Apply one parsed tool result to the current stream state.
fn process_parse_result(
/// Apply one parsed tool output to the current stream state.
fn process_parser_output(
&mut self,
kind: AssistantBlockKind,
result: ToolParseResult,
output: ToolParserOutput,
events: &mut Vec<AssistantEvent>,
) -> Result<()> {
// When we are not currently streaming a tool call, preserve plain
// text first and then surface any new tool call items.
if self.open_call_index.is_none() {
push_text_delta(events, kind, result.normal_text);
self.process_tool_items(result.calls, events)?;
push_text_delta(events, kind, output.normal_text);
self.process_tool_items(output.calls, events)?;
} else {
// Once a tool call is open, prioritize tool deltas first. If the
// parser emits normal text again, close the tool call and resume
// plain text output.
self.process_tool_items(result.calls, events)?;
if !result.normal_text.is_empty() {
self.process_tool_items(output.calls, events)?;
if !output.normal_text.is_empty() {
self.open_call_index = None;
push_text_delta(events, kind, result.normal_text);
push_text_delta(events, kind, output.normal_text);
}
}
Ok(())
@@ -158,8 +164,8 @@ impl ToolState {
}
match self.parser.finish() {
Ok(result) => {
self.process_parse_result(AssistantBlockKind::Text, result, &mut events)?
Ok(output) => {
self.process_parser_output(AssistantBlockKind::Text, output, &mut events)?
}
Err(error) => {
warn!(
@@ -265,17 +271,24 @@ mod tests {
use crate::error::Error;
use crate::event::{AssistantBlockKind, AssistantMessageExt as _};
use crate::output::structured::structured_chat_event_stream;
use crate::parser::tool::{ToolParseResult, ToolParser, ToolParserError};
use crate::parser::tool::{
DeepSeekV4ToolParser, ToolParser, ToolParserError, ToolParserOutput,
};
use crate::request::ChatTool;
use crate::stream::ChatEventStream;
use crate::stream::{ChatEventStream, CollectedAssistantMessage};
struct FailingParser {
fail_next: bool,
buffered: String,
}
struct ScriptedParser {
push_results: Vec<ToolParseResult>,
finish_result: ToolParseResult,
push_outputs: Vec<ToolParserOutput>,
finish_output: ToolParserOutput,
}
struct PartialThenFailParser {
buffered: String,
}
impl ToolParser for FailingParser {
@@ -283,10 +296,14 @@ mod tests {
where
Self: Sized + 'static,
{
Ok(Box::new(Self { fail_next: false }))
Ok(Box::new(Self {
fail_next: false,
buffered: String::new(),
}))
}
fn push(&mut self, _chunk: &str) -> Result<ToolParseResult> {
fn parse_into(&mut self, chunk: &str, _output: &mut ToolParserOutput) -> Result<()> {
self.buffered.push_str(chunk);
if self.fail_next {
self.fail_next = false;
return Err(ToolParserError::ParsingFailed {
@@ -294,7 +311,16 @@ mod tests {
});
}
Ok(ToolParseResult::default())
self.buffered.clear();
Ok(())
}
fn finish(&mut self) -> Result<ToolParserOutput> {
Ok(ToolParserOutput::default())
}
fn reset(&mut self) -> String {
std::mem::take(&mut self.buffered)
}
}
@@ -304,18 +330,215 @@ mod tests {
Self: Sized + 'static,
{
Ok(Box::new(Self {
push_results: Vec::new(),
finish_result: ToolParseResult::default(),
push_outputs: Vec::new(),
finish_output: ToolParserOutput::default(),
}))
}
fn push(&mut self, _chunk: &str) -> Result<ToolParseResult> {
Ok(self.push_results.pop().unwrap_or_default())
fn parse_into(&mut self, _chunk: &str, output: &mut ToolParserOutput) -> Result<()> {
let mut next = self.push_outputs.pop().unwrap_or_default();
output.normal_text.push_str(&next.normal_text);
output.calls.append(&mut next.calls);
Ok(())
}
fn finish(&mut self) -> Result<ToolParseResult> {
Ok(std::mem::take(&mut self.finish_result))
fn finish(&mut self) -> Result<ToolParserOutput> {
Ok(std::mem::take(&mut self.finish_output))
}
fn reset(&mut self) -> String {
String::new()
}
}
impl ToolParser for PartialThenFailParser {
fn create(_tools: &[ChatTool]) -> vllm_tool_parser::Result<Box<dyn ToolParser>>
where
Self: Sized + 'static,
{
Ok(Box::new(Self {
buffered: String::new(),
}))
}
fn parse_into(&mut self, _chunk: &str, output: &mut ToolParserOutput) -> Result<()> {
output.calls.extend([
crate::parser::tool::ToolCallDelta {
tool_index: 0,
name: Some("get_weather".to_string()),
arguments: String::new(),
},
crate::parser::tool::ToolCallDelta {
tool_index: 0,
name: None,
arguments: r#"{"location":"SF"}"#.to_string(),
},
]);
self.buffered.push_str(" trailing text");
Err(ToolParserError::ParsingFailed {
message: "boom".to_string(),
})
}
fn finish(&mut self) -> Result<ToolParserOutput> {
Ok(ToolParserOutput::default())
}
fn reset(&mut self) -> String {
std::mem::take(&mut self.buffered)
}
}
fn deepseek_v4_test_tools() -> Vec<ChatTool> {
vec![
ChatTool {
name: "get_weather".to_string(),
description: None,
parameters: serde_json::json!({
"type": "object",
"properties": {
"location": { "type": "string" }
}
}),
strict: None,
},
ChatTool {
name: "add".to_string(),
description: None,
parameters: serde_json::json!({
"type": "object",
"properties": {
"x": { "type": "integer" },
"y": { "type": "integer" }
}
}),
strict: None,
},
]
}
async fn collect_deepseek_v4_message(chunks: Vec<String>) -> CollectedAssistantMessage {
let events = chunks
.into_iter()
.map(|delta| {
Ok(ContentEvent::TextDelta {
kind: AssistantBlockKind::Text,
delta,
})
})
.chain(std::iter::once(Ok(ContentEvent::Done {
prompt_token_count: 1,
output_token_count: 1,
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
})));
let parser = DeepSeekV4ToolParser::create(&deepseek_v4_test_tools()).unwrap();
let assistant_events = tool_event_stream(stream::iter(events), Some(parser));
let chat_events = structured_chat_event_stream(assistant_events);
ChatEventStream::new("req_deepseek_v4".to_string(), Box::pin(chat_events))
.collect_message()
.await
.unwrap()
}
fn message_tool_projection(
message: &CollectedAssistantMessage,
) -> (String, Vec<(String, serde_json::Value)>) {
(
message.message.text(),
message
.message
.tool_calls()
.map(|call| {
(
call.name.clone(),
serde_json::from_str(&call.arguments).unwrap(),
)
})
.collect(),
)
}
#[tokio::test]
async fn tool_parser_error_preserves_partial_output_and_flushes_buffer() {
let events = stream::iter(vec![
Ok(ContentEvent::TextDelta {
kind: AssistantBlockKind::Text,
delta: "ignored".to_string(),
}),
Ok(ContentEvent::Done {
prompt_token_count: 1,
output_token_count: 1,
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
}),
]);
let events = tool_event_stream(
events,
Some(Box::new(PartialThenFailParser {
buffered: String::new(),
})),
)
.collect::<Vec<_>>()
.await
.into_iter()
.collect::<crate::Result<Vec<_>>>()
.unwrap();
assert!(matches!(
&events[0],
AssistantEvent::ToolCallStart { name, .. } if name == "get_weather"
));
assert!(matches!(
&events[1],
AssistantEvent::ToolCallArgumentsDelta { delta } if delta == r#"{"location":"SF"}"#
));
assert_eq!(
events[2],
AssistantEvent::TextDelta {
kind: AssistantBlockKind::Text,
delta: " trailing text".to_string(),
}
);
assert!(matches!(events[3], AssistantEvent::Done { .. }));
}
#[tokio::test]
async fn real_buffered_parser_error_matches_streaming_and_non_streaming() {
let prefix = "I will check both.\n";
let first_tool_call = concat!(
"<|DSML|tool_calls>\n",
"<|DSML|invoke name=\"get_weather\">\n",
"<|DSML|parameter name=\"location\" string=\"true\">Tokyo</|DSML|parameter>\n",
"</|DSML|invoke>",
);
let malformed_second_tool_call = concat!(
"\n<|DSML|invoke name=\"add\">\n",
"not a parameter\n",
"</|DSML|invoke>\n",
"</|DSML|tool_calls>",
);
let streaming_chunks = vec![
prefix.to_string(),
first_tool_call.to_string(),
malformed_second_tool_call.to_string(),
];
let full_output = streaming_chunks.concat();
let streaming = collect_deepseek_v4_message(streaming_chunks).await;
let non_streaming = collect_deepseek_v4_message(vec![full_output]).await;
let expected = (
format!("{prefix}{malformed_second_tool_call}"),
vec![(
"get_weather".to_string(),
serde_json::json!({ "location": "Tokyo" }),
)],
);
assert_eq!(message_tool_projection(&streaming), expected);
assert_eq!(message_tool_projection(&non_streaming), expected);
}
#[tokio::test]
@@ -341,10 +564,15 @@ mod tests {
}),
]);
let collected =
tool_event_stream(events, Some(Box::new(FailingParser { fail_next: true })))
.collect::<Vec<_>>()
.await;
let collected = tool_event_stream(
events,
Some(Box::new(FailingParser {
fail_next: true,
buffered: String::new(),
})),
)
.collect::<Vec<_>>()
.await;
let events = collected
.into_iter()
@@ -415,12 +643,18 @@ mod tests {
kv_transfer_params: None,
}),
]);
let events = tool_event_stream(events, Some(Box::new(FailingParser { fail_next: false })))
.collect::<Vec<_>>()
.await
.into_iter()
.collect::<crate::Result<Vec<_>>>()
.unwrap();
let events = tool_event_stream(
events,
Some(Box::new(FailingParser {
fail_next: false,
buffered: String::new(),
})),
)
.collect::<Vec<_>>()
.await
.into_iter()
.collect::<crate::Result<Vec<_>>>()
.unwrap();
assert_eq!(
events,
@@ -468,7 +702,7 @@ mod tests {
]);
let parser = ScriptedParser {
push_results: vec![ToolParseResult {
push_outputs: vec![ToolParserOutput {
normal_text: String::new(),
calls: vec![
crate::parser::tool::ToolCallDelta {
@@ -483,14 +717,14 @@ mod tests {
},
],
}],
finish_result: ToolParseResult::default(),
finish_output: ToolParserOutput::default(),
};
let err = tool_event_stream(events, Some(Box::new(parser)))
.collect::<Vec<_>>()
.await
.into_iter()
.find_map(|result| result.err())
.find_map(|output| output.err())
.expect("expected invariant error");
assert!(matches!(err, Error::ToolCallStreamInvariant { .. }));
@@ -514,8 +748,8 @@ mod tests {
]);
let parser = ScriptedParser {
push_results: vec![
ToolParseResult {
push_outputs: vec![
ToolParserOutput {
normal_text: String::new(),
calls: vec![crate::parser::tool::ToolCallDelta {
tool_index: 0,
@@ -523,11 +757,11 @@ mod tests {
arguments: "}".to_string(),
}],
},
ToolParseResult {
ToolParserOutput {
normal_text: "plain text".to_string(),
calls: Vec::new(),
},
ToolParseResult {
ToolParserOutput {
normal_text: String::new(),
calls: vec![crate::parser::tool::ToolCallDelta {
tool_index: 0,
@@ -536,14 +770,14 @@ mod tests {
}],
},
],
finish_result: ToolParseResult::default(),
finish_output: ToolParserOutput::default(),
};
let err = tool_event_stream(events, Some(Box::new(parser)))
.collect::<Vec<_>>()
.await
.into_iter()
.find_map(|result| result.err())
.find_map(|output| output.err())
.expect("expected invariant error");
assert!(matches!(
@@ -573,7 +807,7 @@ mod tests {
]);
let parser = ScriptedParser {
push_results: vec![ToolParseResult {
push_outputs: vec![ToolParserOutput {
normal_text: String::new(),
calls: vec![
crate::parser::tool::ToolCallDelta {
@@ -588,7 +822,7 @@ mod tests {
},
],
}],
finish_result: ToolParseResult::default(),
finish_output: ToolParserOutput::default(),
};
let events = tool_event_stream(events, Some(Box::new(parser)))
+1 -1
View File
@@ -6,7 +6,7 @@ pub use vllm_tool_parser::{
DeepSeekV3ToolParser, DeepSeekV4ToolParser, DeepSeekV31ToolParser, DeepSeekV32ToolParser,
Gemma4ToolParser, Glm45MoeToolParser, Glm47MoeToolParser, HermesToolParser, KimiK2ToolParser,
Llama3JsonToolParser, MinimaxM2ToolParser, MistralToolParser, Qwen3CoderToolParser,
Qwen3XmlToolParser, ToolCallDelta, ToolParseResult, ToolParser, ToolParserError,
Qwen3XmlToolParser, ToolCallDelta, ToolParser, ToolParserError, ToolParserOutput,
};
use crate::parser::ParserFactory;
+11 -3
View File
@@ -1,6 +1,6 @@
use vllm_tool_parser::Result;
use super::{ToolParseResult, ToolParser, ToolParserFactory, names};
use super::{ToolParser, ToolParserFactory, ToolParserOutput, names};
use crate::Error;
use crate::request::ChatTool;
@@ -18,8 +18,16 @@ impl ToolParser for FakeToolParser {
true
}
fn push(&mut self, _chunk: &str) -> Result<ToolParseResult> {
Ok(ToolParseResult::default())
fn parse_into(&mut self, _chunk: &str, _output: &mut ToolParserOutput) -> Result<()> {
Ok(())
}
fn finish(&mut self) -> Result<ToolParserOutput> {
Ok(ToolParserOutput::default())
}
fn reset(&mut self) -> String {
String::new()
}
}