diff --git a/rust/src/chat/src/output/default/mod.rs b/rust/src/chat/src/output/default/mod.rs index 40526a9e84..bebcf8839d 100644 --- a/rust/src/chat/src/output/default/mod.rs +++ b/rust/src/chat/src/output/default/mod.rs @@ -37,6 +37,7 @@ trait_set! { pub struct DefaultChatOutputProcessor { reasoning_parser: Option>, tool_parser: Option>, + parallel_tool_calls: bool, } impl DefaultChatOutputProcessor { @@ -74,6 +75,7 @@ impl DefaultChatOutputProcessor { Ok(Self { reasoning_parser, tool_parser, + parallel_tool_calls: request.parallel_tool_calls, }) } @@ -86,6 +88,7 @@ impl DefaultChatOutputProcessor { Self { reasoning_parser: None, tool_parser: None, + parallel_tool_calls: true, } } @@ -159,7 +162,7 @@ impl ChatOutputProcessor for DefaultChatOutputProcessor { fn process(self: Box, decoded: DynDecodedTextEventStream) -> Result { let reasoning = reasoning_event_stream(decoded, self.reasoning_parser); let tool = tool_event_stream(reasoning, self.tool_parser); - let structured = structured_chat_event_stream(tool); + let structured = structured_chat_event_stream(tool, self.parallel_tool_calls); Ok(structured.boxed()) } diff --git a/rust/src/chat/src/output/default/tool.rs b/rust/src/chat/src/output/default/tool.rs index c216b93f74..665972f148 100644 --- a/rust/src/chat/src/output/default/tool.rs +++ b/rust/src/chat/src/output/default/tool.rs @@ -473,7 +473,7 @@ mod tests { }))); let parser = DeepSeekV4ToolParser::create(&deepseek_v4_test_tools()).unwrap(); let assistant_events = tool_event_stream(stream::iter(events), Some(parser)); - let chat_events = structured_chat_event_stream(assistant_events); + let chat_events = structured_chat_event_stream(assistant_events, true); ChatEventStream::new("req_deepseek_v4".to_string(), Box::pin(chat_events)) .collect_message() @@ -717,9 +717,10 @@ mod tests { let message = ChatEventStream::new( "req_fallback".to_string(), - Box::pin(structured_chat_event_stream(stream::iter( - events.into_iter().map(Ok), - ))), + Box::pin(structured_chat_event_stream( + stream::iter(events.into_iter().map(Ok)), + true, + )), ) .collect_message() .await @@ -968,9 +969,10 @@ mod tests { )); let collected = ChatEventStream::new( "req_final_only".to_string(), - Box::pin(structured_chat_event_stream(stream::iter( - events.into_iter().map(Ok), - ))), + Box::pin(structured_chat_event_stream( + stream::iter(events.into_iter().map(Ok)), + true, + )), ) .collect_message() .await diff --git a/rust/src/chat/src/output/harmony/mod.rs b/rust/src/chat/src/output/harmony/mod.rs index 7a043374e5..4209dc0735 100644 --- a/rust/src/chat/src/output/harmony/mod.rs +++ b/rust/src/chat/src/output/harmony/mod.rs @@ -35,6 +35,7 @@ use crate::request::ChatRequest; pub struct HarmonyChatOutputProcessor { encoding: &'static HarmonyEncoding, tool_calls_enabled: bool, + parallel_tool_calls: bool, } #[derive(Debug, Clone, PartialEq, Eq, Hash)] @@ -76,6 +77,7 @@ impl HarmonyChatOutputProcessor { Ok(Self { encoding: harmony_encoding()?, tool_calls_enabled: request.tool_parsing_enabled(), + parallel_tool_calls: request.parallel_tool_calls, }) } } @@ -110,7 +112,11 @@ impl ChatOutputProcessor for HarmonyChatOutputProcessor { fn process(self: Box, decoded: DynDecodedTextEventStream) -> Result { let assistant = harmony_assistant_event_stream(decoded, self.encoding, self.tool_calls_enabled); - Ok(crate::output::structured::structured_chat_event_stream(assistant).boxed()) + Ok(crate::output::structured::structured_chat_event_stream( + assistant, + self.parallel_tool_calls, + ) + .boxed()) } } diff --git a/rust/src/chat/src/output/structured.rs b/rust/src/chat/src/output/structured.rs index 5cbb9f8093..4be7425d90 100644 --- a/rust/src/chat/src/output/structured.rs +++ b/rust/src/chat/src/output/structured.rs @@ -53,16 +53,22 @@ struct StructuredEventState { open_tool_call: Option, /// Next OpenAI-compatible tool-call ordinal. next_tool_call_index: usize, + /// Whether more than one tool call may be surfaced northbound. + parallel_tool_calls: bool, + /// Whether the current tool-call parse is being suppressed. + suppressing_tool_call: bool, } impl StructuredEventState { /// Create one fresh assembly state for a new streamed response. - fn new() -> Self { + fn new(parallel_tool_calls: bool) -> Self { Self { message: AssistantMessage::default(), open_text_block: None, open_tool_call: None, next_tool_call_index: 0, + parallel_tool_calls, + suppressing_tool_call: false, } } @@ -98,6 +104,12 @@ impl StructuredEventState { let index = self.next_tool_call_index; self.next_tool_call_index += 1; + if !self.parallel_tool_calls && index >= 1 { + self.suppressing_tool_call = true; + return Ok(events); + } + + self.suppressing_tool_call = false; self.open_tool_call = Some(OpenToolCall { index, id: id.clone(), @@ -110,6 +122,10 @@ impl StructuredEventState { /// Append one incremental tool-call arguments delta. fn push_tool_call_arguments(&mut self, delta: String) -> Result> { + if self.suppressing_tool_call { + return Ok(Vec::new()); + } + let mut events = Vec::new(); let Some(open_tool_call) = self.open_tool_call.as_mut() else { return Err(Error::ToolCallStreamInvariant { @@ -207,6 +223,11 @@ impl StructuredEventState { /// Finalize the currently open tool call, if present. fn close_open_tool_call(&mut self, events: &mut Vec) { + if self.suppressing_tool_call { + self.suppressing_tool_call = false; + return; + } + let Some(open_tool_call) = self.open_tool_call.take() else { return; }; @@ -229,11 +250,12 @@ impl StructuredEventState { #[try_stream] pub(crate) async fn structured_chat_event_stream( stream: impl AssistantEventStream, + parallel_tool_calls: bool, mut y: TryYielder, ) -> Result<()> { pin_mut!(stream); - let mut state = StructuredEventState::new(); + let mut state = StructuredEventState::new(parallel_tool_calls); while let Some(event) = stream.next().await.transpose()? { match event { @@ -315,7 +337,7 @@ mod tests { }), ]); - let events = structured_chat_event_stream(events) + let events = structured_chat_event_stream(events, true) .collect::>() .await .into_iter() @@ -369,7 +391,7 @@ mod tests { }), ]); - let events = structured_chat_event_stream(events) + let events = structured_chat_event_stream(events, true) .collect::>() .await .into_iter() @@ -420,7 +442,7 @@ mod tests { }), ]); - let events = structured_chat_event_stream(events) + let events = structured_chat_event_stream(events, true) .collect::>() .await .into_iter() @@ -471,7 +493,7 @@ mod tests { }), ]); - let events = structured_chat_event_stream(events) + let events = structured_chat_event_stream(events, true) .collect::>() .await .into_iter() @@ -499,7 +521,7 @@ mod tests { delta: "{}".to_string(), })]); - let err = structured_chat_event_stream(events) + let err = structured_chat_event_stream(events, true) .collect::>() .await .into_iter() @@ -509,4 +531,56 @@ mod tests { assert!(matches!(err, Error::ToolCallStreamInvariant { .. })); } + + #[tokio::test] + async fn structured_stream_suppresses_later_tool_calls_when_parallel_disabled() { + let events = stream::iter(vec![ + Ok(AssistantEvent::ToolCallStart { + id: "call_1".to_string(), + name: "first".to_string(), + }), + Ok(AssistantEvent::ToolCallArgumentsDelta { + delta: r#"{"a":1}"#.to_string(), + }), + Ok(AssistantEvent::ToolCallStart { + id: "call_2".to_string(), + name: "second".to_string(), + }), + Ok(AssistantEvent::ToolCallArgumentsDelta { + delta: r#"{"b":2}"#.to_string(), + }), + Ok(AssistantEvent::Done { + usage: vllm_llm::TokenUsage { + prompt_token_count: 1, + output_token_count: 1, + cached_token_count: 0, + }, + finish_reason: FinishReason::stop_eos(), + kv_transfer_params: None, + }), + ]); + + let events = structured_chat_event_stream(events, false) + .collect::>() + .await + .into_iter() + .collect::>>() + .unwrap(); + + assert!(matches!( + events[0], + ChatEvent::ToolCallStart { index: 0, .. } + )); + assert!(matches!( + events[1], + ChatEvent::ToolCallArgumentsDelta { index: 0, .. } + )); + assert!(matches!(events[2], ChatEvent::ToolCallEnd { index: 0, .. })); + let ChatEvent::Done { message, .. } = &events[3] else { + panic!("expected done"); + }; + let tool_calls = message.tool_calls().collect::>(); + assert_eq!(tool_calls.len(), 1); + assert_eq!(tool_calls[0].name, "first"); + } } diff --git a/rust/src/chat/src/request.rs b/rust/src/chat/src/request.rs index 842c941a6c..7b9ae5f663 100644 --- a/rust/src/chat/src/request.rs +++ b/rust/src/chat/src/request.rs @@ -406,6 +406,10 @@ pub struct ChatRequest { pub tools: Vec, /// Tool-choice behavior for this request. pub tool_choice: ChatToolChoice, + /// Whether the model may return more than one tool call per response. + /// + /// When `false`, only the first parsed tool call is surfaced northbound. + pub parallel_tool_calls: bool, /// Text decode options for incremental detokenization. pub decode_options: TextDecodeOptions, /// Whether to emit intermediate northbound content deltas before the @@ -442,6 +446,7 @@ impl ChatRequest { chat_options: ChatOptions::default(), tools: Vec::new(), tool_choice: ChatToolChoice::None, + parallel_tool_calls: true, decode_options: TextDecodeOptions::default(), intermediate: true, priority: 0, diff --git a/rust/src/server/src/routes/openai/chat_completions/convert.rs b/rust/src/server/src/routes/openai/chat_completions/convert.rs index aa430db76c..bc581842da 100644 --- a/rust/src/server/src/routes/openai/chat_completions/convert.rs +++ b/rust/src/server/src/routes/openai/chat_completions/convert.rs @@ -142,6 +142,7 @@ pub(super) fn prepare_chat_request( }, tools: convert_tools(request.tools)?, tool_choice: convert_tool_choice(request.tool_choice.as_ref())?, + parallel_tool_calls: request.parallel_tool_calls.unwrap_or(true), decode_options: vllm_text::output::TextDecodeOptions { skip_special_tokens: request.skip_special_tokens, include_stop_str_in_output: request.include_stop_str_in_output, @@ -412,6 +413,33 @@ mod tests { } } + #[test] + fn prepare_chat_request_maps_parallel_tool_calls() { + let mut request = base_request(); + request.parallel_tool_calls = Some(false); + + let prepared = prepare_chat_request( + request, + &served(&["Qwen/Qwen1.5-0.5B-Chat"]), + ResolvedRequestContext::default(), + ) + .expect("request is valid"); + + assert!(!prepared.chat_request.parallel_tool_calls); + } + + #[test] + fn prepare_chat_request_defaults_parallel_tool_calls_to_true() { + let prepared = prepare_chat_request( + base_request(), + &served(&["Qwen/Qwen1.5-0.5B-Chat"]), + ResolvedRequestContext::default(), + ) + .expect("request is valid"); + + assert!(prepared.chat_request.parallel_tool_calls); + } + #[test] fn prepare_chat_request_maps_text_parts() { let mut request = base_request(); diff --git a/rust/src/server/src/routes/openai/chat_completions/validate.rs b/rust/src/server/src/routes/openai/chat_completions/validate.rs index a623925e64..b83d9035a0 100644 --- a/rust/src/server/src/routes/openai/chat_completions/validate.rs +++ b/rust/src/server/src/routes/openai/chat_completions/validate.rs @@ -93,13 +93,6 @@ pub(super) fn validate_request_compat( // ---- Reject parameters that are accepted for deserialization but not yet // implemented ---- - if request.parallel_tool_calls.is_some() { - bail_invalid_request!( - param = "parallel_tool_calls", - "parallel_tool_calls is not supported." - ); - } - reject_non_default( request.length_penalty.as_ref(), "length_penalty", diff --git a/rust/src/server/src/routes/tokenize/types.rs b/rust/src/server/src/routes/tokenize/types.rs index 9a5977b318..987e0e23f3 100644 --- a/rust/src/server/src/routes/tokenize/types.rs +++ b/rust/src/server/src/routes/tokenize/types.rs @@ -83,6 +83,7 @@ impl TokenizeChatRequest { }, tools: convert_tools(self.tools)?, tool_choice: ChatToolChoice::Auto, + parallel_tool_calls: true, decode_options: TextDecodeOptions::default(), intermediate: false, priority: 0,