[Rust Frontend] Support parallel_tool_calls = false (#44760)

Signed-off-by: zhoujinyu <2319109590@qq.com>
This commit is contained in:
FAUST
2026-06-15 07:50:48 +00:00
committed by GitHub
parent b5adb027ad
commit 40eac9a9d9
8 changed files with 135 additions and 23 deletions
+4 -1
View File
@@ -37,6 +37,7 @@ trait_set! {
pub struct DefaultChatOutputProcessor {
reasoning_parser: Option<Box<dyn ReasoningParser>>,
tool_parser: Option<Box<dyn ToolParser>>,
parallel_tool_calls: bool,
}
impl DefaultChatOutputProcessor {
@@ -74,6 +75,7 @@ impl DefaultChatOutputProcessor {
Ok(Self {
reasoning_parser,
tool_parser,
parallel_tool_calls: request.parallel_tool_calls,
})
}
@@ -86,6 +88,7 @@ impl DefaultChatOutputProcessor {
Self {
reasoning_parser: None,
tool_parser: None,
parallel_tool_calls: true,
}
}
@@ -159,7 +162,7 @@ impl ChatOutputProcessor for DefaultChatOutputProcessor {
fn process(self: Box<Self>, decoded: DynDecodedTextEventStream) -> Result<DynChatEventStream> {
let reasoning = reasoning_event_stream(decoded, self.reasoning_parser);
let tool = tool_event_stream(reasoning, self.tool_parser);
let structured = structured_chat_event_stream(tool);
let structured = structured_chat_event_stream(tool, self.parallel_tool_calls);
Ok(structured.boxed())
}
+9 -7
View File
@@ -473,7 +473,7 @@ mod tests {
})));
let parser = DeepSeekV4ToolParser::create(&deepseek_v4_test_tools()).unwrap();
let assistant_events = tool_event_stream(stream::iter(events), Some(parser));
let chat_events = structured_chat_event_stream(assistant_events);
let chat_events = structured_chat_event_stream(assistant_events, true);
ChatEventStream::new("req_deepseek_v4".to_string(), Box::pin(chat_events))
.collect_message()
@@ -717,9 +717,10 @@ mod tests {
let message = ChatEventStream::new(
"req_fallback".to_string(),
Box::pin(structured_chat_event_stream(stream::iter(
events.into_iter().map(Ok),
))),
Box::pin(structured_chat_event_stream(
stream::iter(events.into_iter().map(Ok)),
true,
)),
)
.collect_message()
.await
@@ -968,9 +969,10 @@ mod tests {
));
let collected = ChatEventStream::new(
"req_final_only".to_string(),
Box::pin(structured_chat_event_stream(stream::iter(
events.into_iter().map(Ok),
))),
Box::pin(structured_chat_event_stream(
stream::iter(events.into_iter().map(Ok)),
true,
)),
)
.collect_message()
.await
+7 -1
View File
@@ -35,6 +35,7 @@ use crate::request::ChatRequest;
pub struct HarmonyChatOutputProcessor {
encoding: &'static HarmonyEncoding,
tool_calls_enabled: bool,
parallel_tool_calls: bool,
}
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
@@ -76,6 +77,7 @@ impl HarmonyChatOutputProcessor {
Ok(Self {
encoding: harmony_encoding()?,
tool_calls_enabled: request.tool_parsing_enabled(),
parallel_tool_calls: request.parallel_tool_calls,
})
}
}
@@ -110,7 +112,11 @@ impl ChatOutputProcessor for HarmonyChatOutputProcessor {
fn process(self: Box<Self>, decoded: DynDecodedTextEventStream) -> Result<DynChatEventStream> {
let assistant =
harmony_assistant_event_stream(decoded, self.encoding, self.tool_calls_enabled);
Ok(crate::output::structured::structured_chat_event_stream(assistant).boxed())
Ok(crate::output::structured::structured_chat_event_stream(
assistant,
self.parallel_tool_calls,
)
.boxed())
}
}
+81 -7
View File
@@ -53,16 +53,22 @@ struct StructuredEventState {
open_tool_call: Option<OpenToolCall>,
/// Next OpenAI-compatible tool-call ordinal.
next_tool_call_index: usize,
/// Whether more than one tool call may be surfaced northbound.
parallel_tool_calls: bool,
/// Whether the current tool-call parse is being suppressed.
suppressing_tool_call: bool,
}
impl StructuredEventState {
/// Create one fresh assembly state for a new streamed response.
fn new() -> Self {
fn new(parallel_tool_calls: bool) -> Self {
Self {
message: AssistantMessage::default(),
open_text_block: None,
open_tool_call: None,
next_tool_call_index: 0,
parallel_tool_calls,
suppressing_tool_call: false,
}
}
@@ -98,6 +104,12 @@ impl StructuredEventState {
let index = self.next_tool_call_index;
self.next_tool_call_index += 1;
if !self.parallel_tool_calls && index >= 1 {
self.suppressing_tool_call = true;
return Ok(events);
}
self.suppressing_tool_call = false;
self.open_tool_call = Some(OpenToolCall {
index,
id: id.clone(),
@@ -110,6 +122,10 @@ impl StructuredEventState {
/// Append one incremental tool-call arguments delta.
fn push_tool_call_arguments(&mut self, delta: String) -> Result<Vec<ChatEvent>> {
if self.suppressing_tool_call {
return Ok(Vec::new());
}
let mut events = Vec::new();
let Some(open_tool_call) = self.open_tool_call.as_mut() else {
return Err(Error::ToolCallStreamInvariant {
@@ -207,6 +223,11 @@ impl StructuredEventState {
/// Finalize the currently open tool call, if present.
fn close_open_tool_call(&mut self, events: &mut Vec<ChatEvent>) {
if self.suppressing_tool_call {
self.suppressing_tool_call = false;
return;
}
let Some(open_tool_call) = self.open_tool_call.take() else {
return;
};
@@ -229,11 +250,12 @@ impl StructuredEventState {
#[try_stream]
pub(crate) async fn structured_chat_event_stream(
stream: impl AssistantEventStream,
parallel_tool_calls: bool,
mut y: TryYielder<ChatEvent, Error>,
) -> Result<()> {
pin_mut!(stream);
let mut state = StructuredEventState::new();
let mut state = StructuredEventState::new(parallel_tool_calls);
while let Some(event) = stream.next().await.transpose()? {
match event {
@@ -315,7 +337,7 @@ mod tests {
}),
]);
let events = structured_chat_event_stream(events)
let events = structured_chat_event_stream(events, true)
.collect::<Vec<_>>()
.await
.into_iter()
@@ -369,7 +391,7 @@ mod tests {
}),
]);
let events = structured_chat_event_stream(events)
let events = structured_chat_event_stream(events, true)
.collect::<Vec<_>>()
.await
.into_iter()
@@ -420,7 +442,7 @@ mod tests {
}),
]);
let events = structured_chat_event_stream(events)
let events = structured_chat_event_stream(events, true)
.collect::<Vec<_>>()
.await
.into_iter()
@@ -471,7 +493,7 @@ mod tests {
}),
]);
let events = structured_chat_event_stream(events)
let events = structured_chat_event_stream(events, true)
.collect::<Vec<_>>()
.await
.into_iter()
@@ -499,7 +521,7 @@ mod tests {
delta: "{}".to_string(),
})]);
let err = structured_chat_event_stream(events)
let err = structured_chat_event_stream(events, true)
.collect::<Vec<_>>()
.await
.into_iter()
@@ -509,4 +531,56 @@ mod tests {
assert!(matches!(err, Error::ToolCallStreamInvariant { .. }));
}
#[tokio::test]
async fn structured_stream_suppresses_later_tool_calls_when_parallel_disabled() {
let events = stream::iter(vec![
Ok(AssistantEvent::ToolCallStart {
id: "call_1".to_string(),
name: "first".to_string(),
}),
Ok(AssistantEvent::ToolCallArgumentsDelta {
delta: r#"{"a":1}"#.to_string(),
}),
Ok(AssistantEvent::ToolCallStart {
id: "call_2".to_string(),
name: "second".to_string(),
}),
Ok(AssistantEvent::ToolCallArgumentsDelta {
delta: r#"{"b":2}"#.to_string(),
}),
Ok(AssistantEvent::Done {
usage: vllm_llm::TokenUsage {
prompt_token_count: 1,
output_token_count: 1,
cached_token_count: 0,
},
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
}),
]);
let events = structured_chat_event_stream(events, false)
.collect::<Vec<_>>()
.await
.into_iter()
.collect::<crate::Result<Vec<_>>>()
.unwrap();
assert!(matches!(
events[0],
ChatEvent::ToolCallStart { index: 0, .. }
));
assert!(matches!(
events[1],
ChatEvent::ToolCallArgumentsDelta { index: 0, .. }
));
assert!(matches!(events[2], ChatEvent::ToolCallEnd { index: 0, .. }));
let ChatEvent::Done { message, .. } = &events[3] else {
panic!("expected done");
};
let tool_calls = message.tool_calls().collect::<Vec<_>>();
assert_eq!(tool_calls.len(), 1);
assert_eq!(tool_calls[0].name, "first");
}
}
+5
View File
@@ -406,6 +406,10 @@ pub struct ChatRequest {
pub tools: Vec<ChatTool>,
/// Tool-choice behavior for this request.
pub tool_choice: ChatToolChoice,
/// Whether the model may return more than one tool call per response.
///
/// When `false`, only the first parsed tool call is surfaced northbound.
pub parallel_tool_calls: bool,
/// Text decode options for incremental detokenization.
pub decode_options: TextDecodeOptions,
/// Whether to emit intermediate northbound content deltas before the
@@ -442,6 +446,7 @@ impl ChatRequest {
chat_options: ChatOptions::default(),
tools: Vec::new(),
tool_choice: ChatToolChoice::None,
parallel_tool_calls: true,
decode_options: TextDecodeOptions::default(),
intermediate: true,
priority: 0,
@@ -142,6 +142,7 @@ pub(super) fn prepare_chat_request(
},
tools: convert_tools(request.tools)?,
tool_choice: convert_tool_choice(request.tool_choice.as_ref())?,
parallel_tool_calls: request.parallel_tool_calls.unwrap_or(true),
decode_options: vllm_text::output::TextDecodeOptions {
skip_special_tokens: request.skip_special_tokens,
include_stop_str_in_output: request.include_stop_str_in_output,
@@ -412,6 +413,33 @@ mod tests {
}
}
#[test]
fn prepare_chat_request_maps_parallel_tool_calls() {
let mut request = base_request();
request.parallel_tool_calls = Some(false);
let prepared = prepare_chat_request(
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
)
.expect("request is valid");
assert!(!prepared.chat_request.parallel_tool_calls);
}
#[test]
fn prepare_chat_request_defaults_parallel_tool_calls_to_true() {
let prepared = prepare_chat_request(
base_request(),
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
)
.expect("request is valid");
assert!(prepared.chat_request.parallel_tool_calls);
}
#[test]
fn prepare_chat_request_maps_text_parts() {
let mut request = base_request();
@@ -93,13 +93,6 @@ pub(super) fn validate_request_compat(
// ---- Reject parameters that are accepted for deserialization but not yet
// implemented ----
if request.parallel_tool_calls.is_some() {
bail_invalid_request!(
param = "parallel_tool_calls",
"parallel_tool_calls is not supported."
);
}
reject_non_default(
request.length_penalty.as_ref(),
"length_penalty",
@@ -83,6 +83,7 @@ impl TokenizeChatRequest {
},
tools: convert_tools(self.tools)?,
tool_choice: ChatToolChoice::Auto,
parallel_tool_calls: true,
decode_options: TextDecodeOptions::default(),
intermediate: false,
priority: 0,