[Rust Frontend] Support parallel_tool_calls = false (#44760)
Signed-off-by: zhoujinyu <2319109590@qq.com>
This commit is contained in:
@@ -37,6 +37,7 @@ trait_set! {
|
||||
pub struct DefaultChatOutputProcessor {
|
||||
reasoning_parser: Option<Box<dyn ReasoningParser>>,
|
||||
tool_parser: Option<Box<dyn ToolParser>>,
|
||||
parallel_tool_calls: bool,
|
||||
}
|
||||
|
||||
impl DefaultChatOutputProcessor {
|
||||
@@ -74,6 +75,7 @@ impl DefaultChatOutputProcessor {
|
||||
Ok(Self {
|
||||
reasoning_parser,
|
||||
tool_parser,
|
||||
parallel_tool_calls: request.parallel_tool_calls,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -86,6 +88,7 @@ impl DefaultChatOutputProcessor {
|
||||
Self {
|
||||
reasoning_parser: None,
|
||||
tool_parser: None,
|
||||
parallel_tool_calls: true,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -159,7 +162,7 @@ impl ChatOutputProcessor for DefaultChatOutputProcessor {
|
||||
fn process(self: Box<Self>, decoded: DynDecodedTextEventStream) -> Result<DynChatEventStream> {
|
||||
let reasoning = reasoning_event_stream(decoded, self.reasoning_parser);
|
||||
let tool = tool_event_stream(reasoning, self.tool_parser);
|
||||
let structured = structured_chat_event_stream(tool);
|
||||
let structured = structured_chat_event_stream(tool, self.parallel_tool_calls);
|
||||
|
||||
Ok(structured.boxed())
|
||||
}
|
||||
|
||||
@@ -473,7 +473,7 @@ mod tests {
|
||||
})));
|
||||
let parser = DeepSeekV4ToolParser::create(&deepseek_v4_test_tools()).unwrap();
|
||||
let assistant_events = tool_event_stream(stream::iter(events), Some(parser));
|
||||
let chat_events = structured_chat_event_stream(assistant_events);
|
||||
let chat_events = structured_chat_event_stream(assistant_events, true);
|
||||
|
||||
ChatEventStream::new("req_deepseek_v4".to_string(), Box::pin(chat_events))
|
||||
.collect_message()
|
||||
@@ -717,9 +717,10 @@ mod tests {
|
||||
|
||||
let message = ChatEventStream::new(
|
||||
"req_fallback".to_string(),
|
||||
Box::pin(structured_chat_event_stream(stream::iter(
|
||||
events.into_iter().map(Ok),
|
||||
))),
|
||||
Box::pin(structured_chat_event_stream(
|
||||
stream::iter(events.into_iter().map(Ok)),
|
||||
true,
|
||||
)),
|
||||
)
|
||||
.collect_message()
|
||||
.await
|
||||
@@ -968,9 +969,10 @@ mod tests {
|
||||
));
|
||||
let collected = ChatEventStream::new(
|
||||
"req_final_only".to_string(),
|
||||
Box::pin(structured_chat_event_stream(stream::iter(
|
||||
events.into_iter().map(Ok),
|
||||
))),
|
||||
Box::pin(structured_chat_event_stream(
|
||||
stream::iter(events.into_iter().map(Ok)),
|
||||
true,
|
||||
)),
|
||||
)
|
||||
.collect_message()
|
||||
.await
|
||||
|
||||
@@ -35,6 +35,7 @@ use crate::request::ChatRequest;
|
||||
pub struct HarmonyChatOutputProcessor {
|
||||
encoding: &'static HarmonyEncoding,
|
||||
tool_calls_enabled: bool,
|
||||
parallel_tool_calls: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
|
||||
@@ -76,6 +77,7 @@ impl HarmonyChatOutputProcessor {
|
||||
Ok(Self {
|
||||
encoding: harmony_encoding()?,
|
||||
tool_calls_enabled: request.tool_parsing_enabled(),
|
||||
parallel_tool_calls: request.parallel_tool_calls,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -110,7 +112,11 @@ impl ChatOutputProcessor for HarmonyChatOutputProcessor {
|
||||
fn process(self: Box<Self>, decoded: DynDecodedTextEventStream) -> Result<DynChatEventStream> {
|
||||
let assistant =
|
||||
harmony_assistant_event_stream(decoded, self.encoding, self.tool_calls_enabled);
|
||||
Ok(crate::output::structured::structured_chat_event_stream(assistant).boxed())
|
||||
Ok(crate::output::structured::structured_chat_event_stream(
|
||||
assistant,
|
||||
self.parallel_tool_calls,
|
||||
)
|
||||
.boxed())
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -53,16 +53,22 @@ struct StructuredEventState {
|
||||
open_tool_call: Option<OpenToolCall>,
|
||||
/// Next OpenAI-compatible tool-call ordinal.
|
||||
next_tool_call_index: usize,
|
||||
/// Whether more than one tool call may be surfaced northbound.
|
||||
parallel_tool_calls: bool,
|
||||
/// Whether the current tool-call parse is being suppressed.
|
||||
suppressing_tool_call: bool,
|
||||
}
|
||||
|
||||
impl StructuredEventState {
|
||||
/// Create one fresh assembly state for a new streamed response.
|
||||
fn new() -> Self {
|
||||
fn new(parallel_tool_calls: bool) -> Self {
|
||||
Self {
|
||||
message: AssistantMessage::default(),
|
||||
open_text_block: None,
|
||||
open_tool_call: None,
|
||||
next_tool_call_index: 0,
|
||||
parallel_tool_calls,
|
||||
suppressing_tool_call: false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -98,6 +104,12 @@ impl StructuredEventState {
|
||||
|
||||
let index = self.next_tool_call_index;
|
||||
self.next_tool_call_index += 1;
|
||||
if !self.parallel_tool_calls && index >= 1 {
|
||||
self.suppressing_tool_call = true;
|
||||
return Ok(events);
|
||||
}
|
||||
|
||||
self.suppressing_tool_call = false;
|
||||
self.open_tool_call = Some(OpenToolCall {
|
||||
index,
|
||||
id: id.clone(),
|
||||
@@ -110,6 +122,10 @@ impl StructuredEventState {
|
||||
|
||||
/// Append one incremental tool-call arguments delta.
|
||||
fn push_tool_call_arguments(&mut self, delta: String) -> Result<Vec<ChatEvent>> {
|
||||
if self.suppressing_tool_call {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
let mut events = Vec::new();
|
||||
let Some(open_tool_call) = self.open_tool_call.as_mut() else {
|
||||
return Err(Error::ToolCallStreamInvariant {
|
||||
@@ -207,6 +223,11 @@ impl StructuredEventState {
|
||||
|
||||
/// Finalize the currently open tool call, if present.
|
||||
fn close_open_tool_call(&mut self, events: &mut Vec<ChatEvent>) {
|
||||
if self.suppressing_tool_call {
|
||||
self.suppressing_tool_call = false;
|
||||
return;
|
||||
}
|
||||
|
||||
let Some(open_tool_call) = self.open_tool_call.take() else {
|
||||
return;
|
||||
};
|
||||
@@ -229,11 +250,12 @@ impl StructuredEventState {
|
||||
#[try_stream]
|
||||
pub(crate) async fn structured_chat_event_stream(
|
||||
stream: impl AssistantEventStream,
|
||||
parallel_tool_calls: bool,
|
||||
mut y: TryYielder<ChatEvent, Error>,
|
||||
) -> Result<()> {
|
||||
pin_mut!(stream);
|
||||
|
||||
let mut state = StructuredEventState::new();
|
||||
let mut state = StructuredEventState::new(parallel_tool_calls);
|
||||
|
||||
while let Some(event) = stream.next().await.transpose()? {
|
||||
match event {
|
||||
@@ -315,7 +337,7 @@ mod tests {
|
||||
}),
|
||||
]);
|
||||
|
||||
let events = structured_chat_event_stream(events)
|
||||
let events = structured_chat_event_stream(events, true)
|
||||
.collect::<Vec<_>>()
|
||||
.await
|
||||
.into_iter()
|
||||
@@ -369,7 +391,7 @@ mod tests {
|
||||
}),
|
||||
]);
|
||||
|
||||
let events = structured_chat_event_stream(events)
|
||||
let events = structured_chat_event_stream(events, true)
|
||||
.collect::<Vec<_>>()
|
||||
.await
|
||||
.into_iter()
|
||||
@@ -420,7 +442,7 @@ mod tests {
|
||||
}),
|
||||
]);
|
||||
|
||||
let events = structured_chat_event_stream(events)
|
||||
let events = structured_chat_event_stream(events, true)
|
||||
.collect::<Vec<_>>()
|
||||
.await
|
||||
.into_iter()
|
||||
@@ -471,7 +493,7 @@ mod tests {
|
||||
}),
|
||||
]);
|
||||
|
||||
let events = structured_chat_event_stream(events)
|
||||
let events = structured_chat_event_stream(events, true)
|
||||
.collect::<Vec<_>>()
|
||||
.await
|
||||
.into_iter()
|
||||
@@ -499,7 +521,7 @@ mod tests {
|
||||
delta: "{}".to_string(),
|
||||
})]);
|
||||
|
||||
let err = structured_chat_event_stream(events)
|
||||
let err = structured_chat_event_stream(events, true)
|
||||
.collect::<Vec<_>>()
|
||||
.await
|
||||
.into_iter()
|
||||
@@ -509,4 +531,56 @@ mod tests {
|
||||
|
||||
assert!(matches!(err, Error::ToolCallStreamInvariant { .. }));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn structured_stream_suppresses_later_tool_calls_when_parallel_disabled() {
|
||||
let events = stream::iter(vec![
|
||||
Ok(AssistantEvent::ToolCallStart {
|
||||
id: "call_1".to_string(),
|
||||
name: "first".to_string(),
|
||||
}),
|
||||
Ok(AssistantEvent::ToolCallArgumentsDelta {
|
||||
delta: r#"{"a":1}"#.to_string(),
|
||||
}),
|
||||
Ok(AssistantEvent::ToolCallStart {
|
||||
id: "call_2".to_string(),
|
||||
name: "second".to_string(),
|
||||
}),
|
||||
Ok(AssistantEvent::ToolCallArgumentsDelta {
|
||||
delta: r#"{"b":2}"#.to_string(),
|
||||
}),
|
||||
Ok(AssistantEvent::Done {
|
||||
usage: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 1,
|
||||
output_token_count: 1,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
}),
|
||||
]);
|
||||
|
||||
let events = structured_chat_event_stream(events, false)
|
||||
.collect::<Vec<_>>()
|
||||
.await
|
||||
.into_iter()
|
||||
.collect::<crate::Result<Vec<_>>>()
|
||||
.unwrap();
|
||||
|
||||
assert!(matches!(
|
||||
events[0],
|
||||
ChatEvent::ToolCallStart { index: 0, .. }
|
||||
));
|
||||
assert!(matches!(
|
||||
events[1],
|
||||
ChatEvent::ToolCallArgumentsDelta { index: 0, .. }
|
||||
));
|
||||
assert!(matches!(events[2], ChatEvent::ToolCallEnd { index: 0, .. }));
|
||||
let ChatEvent::Done { message, .. } = &events[3] else {
|
||||
panic!("expected done");
|
||||
};
|
||||
let tool_calls = message.tool_calls().collect::<Vec<_>>();
|
||||
assert_eq!(tool_calls.len(), 1);
|
||||
assert_eq!(tool_calls[0].name, "first");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -406,6 +406,10 @@ pub struct ChatRequest {
|
||||
pub tools: Vec<ChatTool>,
|
||||
/// Tool-choice behavior for this request.
|
||||
pub tool_choice: ChatToolChoice,
|
||||
/// Whether the model may return more than one tool call per response.
|
||||
///
|
||||
/// When `false`, only the first parsed tool call is surfaced northbound.
|
||||
pub parallel_tool_calls: bool,
|
||||
/// Text decode options for incremental detokenization.
|
||||
pub decode_options: TextDecodeOptions,
|
||||
/// Whether to emit intermediate northbound content deltas before the
|
||||
@@ -442,6 +446,7 @@ impl ChatRequest {
|
||||
chat_options: ChatOptions::default(),
|
||||
tools: Vec::new(),
|
||||
tool_choice: ChatToolChoice::None,
|
||||
parallel_tool_calls: true,
|
||||
decode_options: TextDecodeOptions::default(),
|
||||
intermediate: true,
|
||||
priority: 0,
|
||||
|
||||
@@ -142,6 +142,7 @@ pub(super) fn prepare_chat_request(
|
||||
},
|
||||
tools: convert_tools(request.tools)?,
|
||||
tool_choice: convert_tool_choice(request.tool_choice.as_ref())?,
|
||||
parallel_tool_calls: request.parallel_tool_calls.unwrap_or(true),
|
||||
decode_options: vllm_text::output::TextDecodeOptions {
|
||||
skip_special_tokens: request.skip_special_tokens,
|
||||
include_stop_str_in_output: request.include_stop_str_in_output,
|
||||
@@ -412,6 +413,33 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_chat_request_maps_parallel_tool_calls() {
|
||||
let mut request = base_request();
|
||||
request.parallel_tool_calls = Some(false);
|
||||
|
||||
let prepared = prepare_chat_request(
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
)
|
||||
.expect("request is valid");
|
||||
|
||||
assert!(!prepared.chat_request.parallel_tool_calls);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_chat_request_defaults_parallel_tool_calls_to_true() {
|
||||
let prepared = prepare_chat_request(
|
||||
base_request(),
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
)
|
||||
.expect("request is valid");
|
||||
|
||||
assert!(prepared.chat_request.parallel_tool_calls);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_chat_request_maps_text_parts() {
|
||||
let mut request = base_request();
|
||||
|
||||
@@ -93,13 +93,6 @@ pub(super) fn validate_request_compat(
|
||||
// ---- Reject parameters that are accepted for deserialization but not yet
|
||||
// implemented ----
|
||||
|
||||
if request.parallel_tool_calls.is_some() {
|
||||
bail_invalid_request!(
|
||||
param = "parallel_tool_calls",
|
||||
"parallel_tool_calls is not supported."
|
||||
);
|
||||
}
|
||||
|
||||
reject_non_default(
|
||||
request.length_penalty.as_ref(),
|
||||
"length_penalty",
|
||||
|
||||
@@ -83,6 +83,7 @@ impl TokenizeChatRequest {
|
||||
},
|
||||
tools: convert_tools(self.tools)?,
|
||||
tool_choice: ChatToolChoice::Auto,
|
||||
parallel_tool_calls: true,
|
||||
decode_options: TextDecodeOptions::default(),
|
||||
intermediate: false,
|
||||
priority: 0,
|
||||
|
||||
Reference in New Issue
Block a user