[Rust Frontend] Support dynamic tools from developer messages (#51144)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Bugen Zhao
2026-08-10 18:30:38 -07:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent b2506d62ae
commit f1e921b6d6
28 changed files with 686 additions and 258 deletions
@@ -5,16 +5,16 @@ use itertools::Itertools as _;
use vllm_chat::{
AssistantContentBlock, AssistantToolCall, ChatContent, ChatContentPart,
ChatMessage as VllmChatMessage, ChatOptions, ChatRequest, ChatTool, ChatToolChoice,
GenerationPromptMode, SamplingParams,
GenerationPromptMode, ResolvedToolContext, SamplingParams,
};
use super::types::ChatCompletionRequest;
use super::validate;
use crate::error::{ApiError, bail_invalid_request};
use crate::error::{ApiError, bail_invalid_request, chat_submit_error};
use crate::lora::LoraModelResolution;
use crate::routes::openai::utils::structured_outputs::convert_from_response_format;
use crate::routes::openai::utils::types::{
ChatMessage, ContentPart, MessageContent, Tool, ToolChoice, ToolChoiceValue,
ChatMessage, ContentPart, MessageContent, Tool, ToolChoice, ToolChoiceValue, ToolReference,
};
use crate::utils::{
ResolvedRequestContext, convert_logit_bias, merge_ec_transfer_params, merge_kv_transfer_params,
@@ -130,6 +130,14 @@ pub(super) fn prepare_chat_request(
request.vllm_xargs.as_ref(),
);
let tool_context = ResolvedToolContext::new(
&messages,
convert_tools(request.tools)?,
request.tool_choice.as_ref().map(convert_tool_choice).transpose()?,
request.parallel_tool_calls.unwrap_or(true),
)
.map_err(|error| chat_submit_error("failed to resolve request tools", error))?;
let chat_request = ChatRequest {
request_id: request_id.clone(),
messages,
@@ -168,9 +176,7 @@ pub(super) fn prepare_chat_request(
response_format,
template_kwargs,
},
tools: convert_tools(request.tools)?,
tool_choice: convert_tool_choice(request.tool_choice.as_ref())?,
parallel_tool_calls: request.parallel_tool_calls.unwrap_or(true),
tool_context,
decode_options: vllm_text::output::TextDecodeOptions {
skip_special_tokens: request.skip_special_tokens,
include_stop_str_in_output: request.include_stop_str_in_output,
@@ -399,17 +405,21 @@ fn convert_message_tools(tools: Option<Vec<Tool>>) -> Result<Option<Vec<ChatTool
Ok((!tools.is_empty()).then_some(tools))
}
fn convert_tool_choice(tool_choice: Option<&ToolChoice>) -> Result<ChatToolChoice, ApiError> {
fn convert_tool_choice(tool_choice: &ToolChoice) -> Result<ChatToolChoice, ApiError> {
match tool_choice {
None | Some(ToolChoice::Value(ToolChoiceValue::Auto)) => Ok(ChatToolChoice::Auto),
Some(ToolChoice::Value(ToolChoiceValue::None)) => Ok(ChatToolChoice::None),
Some(ToolChoice::Value(ToolChoiceValue::Required)) => Ok(ChatToolChoice::Required),
Some(ToolChoice::Function {
ToolChoice::Value(ToolChoiceValue::Auto) => Ok(ChatToolChoice::Auto),
ToolChoice::Value(ToolChoiceValue::None) => Ok(ChatToolChoice::None),
ToolChoice::Value(ToolChoiceValue::Required) => Ok(ChatToolChoice::Required),
ToolChoice::Function {
tool_type,
function,
}) if tool_type == "function" => Ok(ChatToolChoice::Function {
} if tool_type == "function" => Ok(ChatToolChoice::Function {
name: function.name.clone(),
}),
ToolChoice::AllowedTools { tools, .. } => bail_invalid_request!(
"allowed_tools tool_choice is not supported yet: {}.",
tools.iter().map(ToolReference::identifier).join(", ")
),
_ => bail_invalid_request!("tool_choice={:?} is not supported yet.", tool_choice),
}
}
@@ -478,7 +488,7 @@ mod tests {
)
.expect("request is valid");
assert!(!prepared.chat_request.parallel_tool_calls);
assert!(!prepared.chat_request.parallel_tool_calls());
}
#[test]
@@ -490,7 +500,7 @@ mod tests {
)
.expect("request is valid");
assert!(prepared.chat_request.parallel_tool_calls);
assert!(prepared.chat_request.parallel_tool_calls());
}
#[test]
@@ -596,8 +606,8 @@ mod tests {
min_tokens: 0,
}
);
assert!(prepared.chat_request.tools.is_empty());
assert_eq!(prepared.chat_request.tool_choice, ChatToolChoice::Auto);
assert!(prepared.chat_request.initial_tools().is_empty());
assert_eq!(prepared.chat_request.tool_choice(), &ChatToolChoice::None);
}
#[test]
@@ -671,8 +681,8 @@ mod tests {
min_tokens: 0,
}
);
assert!(prepared.chat_request.tools.is_empty());
assert_eq!(prepared.chat_request.tool_choice, ChatToolChoice::Auto);
assert!(prepared.chat_request.initial_tools().is_empty());
assert_eq!(prepared.chat_request.tool_choice(), &ChatToolChoice::None);
}
#[test]
@@ -1009,8 +1019,8 @@ mod tests {
},
])]
);
assert!(prepared.chat_request.tools.is_empty());
assert_eq!(prepared.chat_request.tool_choice, ChatToolChoice::Auto);
assert!(prepared.chat_request.initial_tools().is_empty());
assert_eq!(prepared.chat_request.tool_choice(), &ChatToolChoice::None);
}
#[test]
@@ -1105,7 +1115,7 @@ mod tests {
]
);
assert_eq!(
prepared.chat_request.tools,
prepared.chat_request.initial_tools(),
vec![VllmChatTool {
name: "get_weather".to_string(),
description: Some("Get weather".to_string()),
@@ -1116,7 +1126,7 @@ mod tests {
strict: None,
}]
);
assert_eq!(prepared.chat_request.tool_choice, ChatToolChoice::None);
assert_eq!(prepared.chat_request.tool_choice(), &ChatToolChoice::None);
}
#[test]
@@ -1145,7 +1155,10 @@ mod tests {
)
.expect("request is valid");
assert_eq!(prepared.chat_request.tool_choice, ChatToolChoice::Required);
assert_eq!(
prepared.chat_request.tool_choice(),
&ChatToolChoice::Required
);
assert!(!prepared.options.is_named_tool_choice);
}
@@ -1181,8 +1194,8 @@ mod tests {
.expect("request is valid");
assert_eq!(
prepared.chat_request.tool_choice,
ChatToolChoice::Function {
prepared.chat_request.tool_choice(),
&ChatToolChoice::Function {
name: "get_weather".to_string(),
}
);
@@ -14,8 +14,8 @@ use vllm_engine_core_client::protocol::sampling::RepetitionDetectionParams;
use crate::routes::openai::utils::structured_outputs::ResponseFormat;
use crate::routes::openai::utils::types::{
ChatLogProbs, ChatMessage, Normalizable, StreamOptions, StringOrArray, Tool, ToolCall,
ToolCallDelta, ToolChoice, ToolChoiceValue, ToolReference, UNKNOWN_MODEL_ID, Usage,
default_true, validate_messages, validate_stop, validate_top_p_value,
ToolCallDelta, ToolChoice, UNKNOWN_MODEL_ID, Usage, default_true, validate_messages,
validate_stop, validate_top_p_value,
};
/// vLLM-compatible request type for the Chat Completions API.
@@ -325,19 +325,6 @@ impl Normalizable for ChatCompletionRequest {
self.max_completion_tokens = self.max_tokens;
self.max_tokens = None;
}
// Apply tool_choice defaults
// If tools is None, leave tool_choice as None (don't set it)
if self.tool_choice.is_none()
&& let Some(tools) = &self.tools
{
let choice_value = if tools.is_empty() {
ToolChoiceValue::None
} else {
ToolChoiceValue::Auto
};
self.tool_choice = Some(ToolChoice::Value(choice_value));
}
}
}
@@ -492,99 +479,5 @@ fn validate_chat_cross_parameters(
return Err(e);
}
// 5. Validate tool_choice requires tools (except for "none")
if let Some(ref tool_choice) = req.tool_choice {
let has_tools = req.tools.as_ref().is_some_and(|t| !t.is_empty());
// Check if tool_choice is anything other than "none"
let is_some_choice = !matches!(tool_choice, ToolChoice::Value(ToolChoiceValue::None));
if is_some_choice && !has_tools {
let mut e = validator::ValidationError::new("tool_choice_requires_tools");
e.message = Some("Invalid value for 'tool_choice': 'tool_choice' is only allowed when 'tools' are specified.".into());
return Err(e);
}
// Additional validation when tools are present
if let Some(tools) = req.tools.as_ref().filter(|t| !t.is_empty()) {
match tool_choice {
ToolChoice::Function { function, .. } => {
// Validate that the specified function name exists in tools
let function_exists = tools.iter().any(|tool| {
tool.tool_type == "function" && tool.function.name == function.name
});
if !function_exists {
let mut e =
validator::ValidationError::new("tool_choice_function_not_found");
e.message = Some(
format!(
"Invalid value for 'tool_choice': function '{}' not found in 'tools'.",
function.name
)
.into(),
);
return Err(e);
}
}
ToolChoice::AllowedTools {
mode,
tools: allowed_tools,
..
} => {
// Validate mode is "auto" or "required"
if mode != "auto" && mode != "required" {
let mut e = validator::ValidationError::new("tool_choice_invalid_mode");
e.message = Some(format!(
"Invalid value for 'tool_choice.mode': must be 'auto' or 'required', got '{mode}'."
).into());
return Err(e);
}
// Validate that all ToolReferences are Function type (Chat API only supports
// function tools)
for tool_ref in allowed_tools {
match tool_ref {
ToolReference::Function { name } => {
// Validate that the function exists in tools array
let tool_exists = tools.iter().any(|tool| {
tool.tool_type == "function" && tool.function.name == *name
});
if !tool_exists {
let mut e = validator::ValidationError::new(
"tool_choice_tool_not_found",
);
e.message = Some(
format!(
"Invalid value for 'tool_choice.tools': tool '{name}' not found in 'tools'."
)
.into(),
);
return Err(e);
}
}
_ => {
// Chat Completion API only supports function tools in tool_choice
let mut e = validator::ValidationError::new(
"tool_choice_invalid_tool_type",
);
e.message = Some(
format!(
"Invalid value for 'tool_choice.tools': Chat Completion API only supports function tools, got '{}'.",
tool_ref.identifier()
)
.into(),
);
return Err(e);
}
}
}
}
ToolChoice::Value(_) => {}
}
}
}
Ok(())
}
+8 -5
View File
@@ -7,7 +7,7 @@ use itertools::Itertools as _;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use validator::{Validate, ValidationErrors};
use vllm_chat::{ChatOptions, ChatRequest, ChatToolChoice, SamplingParams};
use vllm_chat::{ChatOptions, ChatRequest, ResolvedToolContext, SamplingParams};
use vllm_text::output::TextDecodeOptions;
use vllm_text::{Prompt, TextRequest};
@@ -97,6 +97,11 @@ impl TokenizeChatRequest {
&messages,
)?;
let tool_context =
ResolvedToolContext::new(&messages, convert_tools(self.tools)?, None, true).map_err(
|error| crate::error::chat_submit_error("failed to resolve request tools", error),
)?;
Ok(ChatRequest {
request_id,
messages,
@@ -108,9 +113,7 @@ impl TokenizeChatRequest {
response_format: None,
template_kwargs: self.chat_template_kwargs.unwrap_or_default(),
},
tools: convert_tools(self.tools)?,
tool_choice: ChatToolChoice::Auto,
parallel_tool_calls: true,
tool_context,
decode_options: TextDecodeOptions::default(),
intermediate: false,
priority: 0,
@@ -199,7 +202,7 @@ mod tests {
req.into_chat_request("tokenize-test".to_string()).expect("request is valid");
assert_eq!(
chat_request.tools,
chat_request.initial_tools(),
vec![ChatTool {
name: "get_weather".to_string(),
description: Some("Get weather".to_string()),