[Rust Frontend] Support dynamic tools from developer messages (#51144)
Signed-off-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
b2506d62ae
commit
f1e921b6d6
@@ -5,16 +5,16 @@ use itertools::Itertools as _;
|
||||
use vllm_chat::{
|
||||
AssistantContentBlock, AssistantToolCall, ChatContent, ChatContentPart,
|
||||
ChatMessage as VllmChatMessage, ChatOptions, ChatRequest, ChatTool, ChatToolChoice,
|
||||
GenerationPromptMode, SamplingParams,
|
||||
GenerationPromptMode, ResolvedToolContext, SamplingParams,
|
||||
};
|
||||
|
||||
use super::types::ChatCompletionRequest;
|
||||
use super::validate;
|
||||
use crate::error::{ApiError, bail_invalid_request};
|
||||
use crate::error::{ApiError, bail_invalid_request, chat_submit_error};
|
||||
use crate::lora::LoraModelResolution;
|
||||
use crate::routes::openai::utils::structured_outputs::convert_from_response_format;
|
||||
use crate::routes::openai::utils::types::{
|
||||
ChatMessage, ContentPart, MessageContent, Tool, ToolChoice, ToolChoiceValue,
|
||||
ChatMessage, ContentPart, MessageContent, Tool, ToolChoice, ToolChoiceValue, ToolReference,
|
||||
};
|
||||
use crate::utils::{
|
||||
ResolvedRequestContext, convert_logit_bias, merge_ec_transfer_params, merge_kv_transfer_params,
|
||||
@@ -130,6 +130,14 @@ pub(super) fn prepare_chat_request(
|
||||
request.vllm_xargs.as_ref(),
|
||||
);
|
||||
|
||||
let tool_context = ResolvedToolContext::new(
|
||||
&messages,
|
||||
convert_tools(request.tools)?,
|
||||
request.tool_choice.as_ref().map(convert_tool_choice).transpose()?,
|
||||
request.parallel_tool_calls.unwrap_or(true),
|
||||
)
|
||||
.map_err(|error| chat_submit_error("failed to resolve request tools", error))?;
|
||||
|
||||
let chat_request = ChatRequest {
|
||||
request_id: request_id.clone(),
|
||||
messages,
|
||||
@@ -168,9 +176,7 @@ pub(super) fn prepare_chat_request(
|
||||
response_format,
|
||||
template_kwargs,
|
||||
},
|
||||
tools: convert_tools(request.tools)?,
|
||||
tool_choice: convert_tool_choice(request.tool_choice.as_ref())?,
|
||||
parallel_tool_calls: request.parallel_tool_calls.unwrap_or(true),
|
||||
tool_context,
|
||||
decode_options: vllm_text::output::TextDecodeOptions {
|
||||
skip_special_tokens: request.skip_special_tokens,
|
||||
include_stop_str_in_output: request.include_stop_str_in_output,
|
||||
@@ -399,17 +405,21 @@ fn convert_message_tools(tools: Option<Vec<Tool>>) -> Result<Option<Vec<ChatTool
|
||||
Ok((!tools.is_empty()).then_some(tools))
|
||||
}
|
||||
|
||||
fn convert_tool_choice(tool_choice: Option<&ToolChoice>) -> Result<ChatToolChoice, ApiError> {
|
||||
fn convert_tool_choice(tool_choice: &ToolChoice) -> Result<ChatToolChoice, ApiError> {
|
||||
match tool_choice {
|
||||
None | Some(ToolChoice::Value(ToolChoiceValue::Auto)) => Ok(ChatToolChoice::Auto),
|
||||
Some(ToolChoice::Value(ToolChoiceValue::None)) => Ok(ChatToolChoice::None),
|
||||
Some(ToolChoice::Value(ToolChoiceValue::Required)) => Ok(ChatToolChoice::Required),
|
||||
Some(ToolChoice::Function {
|
||||
ToolChoice::Value(ToolChoiceValue::Auto) => Ok(ChatToolChoice::Auto),
|
||||
ToolChoice::Value(ToolChoiceValue::None) => Ok(ChatToolChoice::None),
|
||||
ToolChoice::Value(ToolChoiceValue::Required) => Ok(ChatToolChoice::Required),
|
||||
ToolChoice::Function {
|
||||
tool_type,
|
||||
function,
|
||||
}) if tool_type == "function" => Ok(ChatToolChoice::Function {
|
||||
} if tool_type == "function" => Ok(ChatToolChoice::Function {
|
||||
name: function.name.clone(),
|
||||
}),
|
||||
ToolChoice::AllowedTools { tools, .. } => bail_invalid_request!(
|
||||
"allowed_tools tool_choice is not supported yet: {}.",
|
||||
tools.iter().map(ToolReference::identifier).join(", ")
|
||||
),
|
||||
_ => bail_invalid_request!("tool_choice={:?} is not supported yet.", tool_choice),
|
||||
}
|
||||
}
|
||||
@@ -478,7 +488,7 @@ mod tests {
|
||||
)
|
||||
.expect("request is valid");
|
||||
|
||||
assert!(!prepared.chat_request.parallel_tool_calls);
|
||||
assert!(!prepared.chat_request.parallel_tool_calls());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -490,7 +500,7 @@ mod tests {
|
||||
)
|
||||
.expect("request is valid");
|
||||
|
||||
assert!(prepared.chat_request.parallel_tool_calls);
|
||||
assert!(prepared.chat_request.parallel_tool_calls());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -596,8 +606,8 @@ mod tests {
|
||||
min_tokens: 0,
|
||||
}
|
||||
);
|
||||
assert!(prepared.chat_request.tools.is_empty());
|
||||
assert_eq!(prepared.chat_request.tool_choice, ChatToolChoice::Auto);
|
||||
assert!(prepared.chat_request.initial_tools().is_empty());
|
||||
assert_eq!(prepared.chat_request.tool_choice(), &ChatToolChoice::None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -671,8 +681,8 @@ mod tests {
|
||||
min_tokens: 0,
|
||||
}
|
||||
);
|
||||
assert!(prepared.chat_request.tools.is_empty());
|
||||
assert_eq!(prepared.chat_request.tool_choice, ChatToolChoice::Auto);
|
||||
assert!(prepared.chat_request.initial_tools().is_empty());
|
||||
assert_eq!(prepared.chat_request.tool_choice(), &ChatToolChoice::None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1009,8 +1019,8 @@ mod tests {
|
||||
},
|
||||
])]
|
||||
);
|
||||
assert!(prepared.chat_request.tools.is_empty());
|
||||
assert_eq!(prepared.chat_request.tool_choice, ChatToolChoice::Auto);
|
||||
assert!(prepared.chat_request.initial_tools().is_empty());
|
||||
assert_eq!(prepared.chat_request.tool_choice(), &ChatToolChoice::None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1105,7 +1115,7 @@ mod tests {
|
||||
]
|
||||
);
|
||||
assert_eq!(
|
||||
prepared.chat_request.tools,
|
||||
prepared.chat_request.initial_tools(),
|
||||
vec![VllmChatTool {
|
||||
name: "get_weather".to_string(),
|
||||
description: Some("Get weather".to_string()),
|
||||
@@ -1116,7 +1126,7 @@ mod tests {
|
||||
strict: None,
|
||||
}]
|
||||
);
|
||||
assert_eq!(prepared.chat_request.tool_choice, ChatToolChoice::None);
|
||||
assert_eq!(prepared.chat_request.tool_choice(), &ChatToolChoice::None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1145,7 +1155,10 @@ mod tests {
|
||||
)
|
||||
.expect("request is valid");
|
||||
|
||||
assert_eq!(prepared.chat_request.tool_choice, ChatToolChoice::Required);
|
||||
assert_eq!(
|
||||
prepared.chat_request.tool_choice(),
|
||||
&ChatToolChoice::Required
|
||||
);
|
||||
assert!(!prepared.options.is_named_tool_choice);
|
||||
}
|
||||
|
||||
@@ -1181,8 +1194,8 @@ mod tests {
|
||||
.expect("request is valid");
|
||||
|
||||
assert_eq!(
|
||||
prepared.chat_request.tool_choice,
|
||||
ChatToolChoice::Function {
|
||||
prepared.chat_request.tool_choice(),
|
||||
&ChatToolChoice::Function {
|
||||
name: "get_weather".to_string(),
|
||||
}
|
||||
);
|
||||
|
||||
@@ -14,8 +14,8 @@ use vllm_engine_core_client::protocol::sampling::RepetitionDetectionParams;
|
||||
use crate::routes::openai::utils::structured_outputs::ResponseFormat;
|
||||
use crate::routes::openai::utils::types::{
|
||||
ChatLogProbs, ChatMessage, Normalizable, StreamOptions, StringOrArray, Tool, ToolCall,
|
||||
ToolCallDelta, ToolChoice, ToolChoiceValue, ToolReference, UNKNOWN_MODEL_ID, Usage,
|
||||
default_true, validate_messages, validate_stop, validate_top_p_value,
|
||||
ToolCallDelta, ToolChoice, UNKNOWN_MODEL_ID, Usage, default_true, validate_messages,
|
||||
validate_stop, validate_top_p_value,
|
||||
};
|
||||
|
||||
/// vLLM-compatible request type for the Chat Completions API.
|
||||
@@ -325,19 +325,6 @@ impl Normalizable for ChatCompletionRequest {
|
||||
self.max_completion_tokens = self.max_tokens;
|
||||
self.max_tokens = None;
|
||||
}
|
||||
|
||||
// Apply tool_choice defaults
|
||||
// If tools is None, leave tool_choice as None (don't set it)
|
||||
if self.tool_choice.is_none()
|
||||
&& let Some(tools) = &self.tools
|
||||
{
|
||||
let choice_value = if tools.is_empty() {
|
||||
ToolChoiceValue::None
|
||||
} else {
|
||||
ToolChoiceValue::Auto
|
||||
};
|
||||
self.tool_choice = Some(ToolChoice::Value(choice_value));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -492,99 +479,5 @@ fn validate_chat_cross_parameters(
|
||||
return Err(e);
|
||||
}
|
||||
|
||||
// 5. Validate tool_choice requires tools (except for "none")
|
||||
if let Some(ref tool_choice) = req.tool_choice {
|
||||
let has_tools = req.tools.as_ref().is_some_and(|t| !t.is_empty());
|
||||
|
||||
// Check if tool_choice is anything other than "none"
|
||||
let is_some_choice = !matches!(tool_choice, ToolChoice::Value(ToolChoiceValue::None));
|
||||
|
||||
if is_some_choice && !has_tools {
|
||||
let mut e = validator::ValidationError::new("tool_choice_requires_tools");
|
||||
e.message = Some("Invalid value for 'tool_choice': 'tool_choice' is only allowed when 'tools' are specified.".into());
|
||||
return Err(e);
|
||||
}
|
||||
|
||||
// Additional validation when tools are present
|
||||
if let Some(tools) = req.tools.as_ref().filter(|t| !t.is_empty()) {
|
||||
match tool_choice {
|
||||
ToolChoice::Function { function, .. } => {
|
||||
// Validate that the specified function name exists in tools
|
||||
let function_exists = tools.iter().any(|tool| {
|
||||
tool.tool_type == "function" && tool.function.name == function.name
|
||||
});
|
||||
|
||||
if !function_exists {
|
||||
let mut e =
|
||||
validator::ValidationError::new("tool_choice_function_not_found");
|
||||
e.message = Some(
|
||||
format!(
|
||||
"Invalid value for 'tool_choice': function '{}' not found in 'tools'.",
|
||||
function.name
|
||||
)
|
||||
.into(),
|
||||
);
|
||||
return Err(e);
|
||||
}
|
||||
}
|
||||
ToolChoice::AllowedTools {
|
||||
mode,
|
||||
tools: allowed_tools,
|
||||
..
|
||||
} => {
|
||||
// Validate mode is "auto" or "required"
|
||||
if mode != "auto" && mode != "required" {
|
||||
let mut e = validator::ValidationError::new("tool_choice_invalid_mode");
|
||||
e.message = Some(format!(
|
||||
"Invalid value for 'tool_choice.mode': must be 'auto' or 'required', got '{mode}'."
|
||||
).into());
|
||||
return Err(e);
|
||||
}
|
||||
|
||||
// Validate that all ToolReferences are Function type (Chat API only supports
|
||||
// function tools)
|
||||
for tool_ref in allowed_tools {
|
||||
match tool_ref {
|
||||
ToolReference::Function { name } => {
|
||||
// Validate that the function exists in tools array
|
||||
let tool_exists = tools.iter().any(|tool| {
|
||||
tool.tool_type == "function" && tool.function.name == *name
|
||||
});
|
||||
|
||||
if !tool_exists {
|
||||
let mut e = validator::ValidationError::new(
|
||||
"tool_choice_tool_not_found",
|
||||
);
|
||||
e.message = Some(
|
||||
format!(
|
||||
"Invalid value for 'tool_choice.tools': tool '{name}' not found in 'tools'."
|
||||
)
|
||||
.into(),
|
||||
);
|
||||
return Err(e);
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
// Chat Completion API only supports function tools in tool_choice
|
||||
let mut e = validator::ValidationError::new(
|
||||
"tool_choice_invalid_tool_type",
|
||||
);
|
||||
e.message = Some(
|
||||
format!(
|
||||
"Invalid value for 'tool_choice.tools': Chat Completion API only supports function tools, got '{}'.",
|
||||
tool_ref.identifier()
|
||||
)
|
||||
.into(),
|
||||
);
|
||||
return Err(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
ToolChoice::Value(_) => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -7,7 +7,7 @@ use itertools::Itertools as _;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use validator::{Validate, ValidationErrors};
|
||||
use vllm_chat::{ChatOptions, ChatRequest, ChatToolChoice, SamplingParams};
|
||||
use vllm_chat::{ChatOptions, ChatRequest, ResolvedToolContext, SamplingParams};
|
||||
use vllm_text::output::TextDecodeOptions;
|
||||
use vllm_text::{Prompt, TextRequest};
|
||||
|
||||
@@ -97,6 +97,11 @@ impl TokenizeChatRequest {
|
||||
&messages,
|
||||
)?;
|
||||
|
||||
let tool_context =
|
||||
ResolvedToolContext::new(&messages, convert_tools(self.tools)?, None, true).map_err(
|
||||
|error| crate::error::chat_submit_error("failed to resolve request tools", error),
|
||||
)?;
|
||||
|
||||
Ok(ChatRequest {
|
||||
request_id,
|
||||
messages,
|
||||
@@ -108,9 +113,7 @@ impl TokenizeChatRequest {
|
||||
response_format: None,
|
||||
template_kwargs: self.chat_template_kwargs.unwrap_or_default(),
|
||||
},
|
||||
tools: convert_tools(self.tools)?,
|
||||
tool_choice: ChatToolChoice::Auto,
|
||||
parallel_tool_calls: true,
|
||||
tool_context,
|
||||
decode_options: TextDecodeOptions::default(),
|
||||
intermediate: false,
|
||||
priority: 0,
|
||||
@@ -199,7 +202,7 @@ mod tests {
|
||||
req.into_chat_request("tokenize-test".to_string()).expect("request is valid");
|
||||
|
||||
assert_eq!(
|
||||
chat_request.tools,
|
||||
chat_request.initial_tools(),
|
||||
vec![ChatTool {
|
||||
name: "get_weather".to_string(),
|
||||
description: Some("Get weather".to_string()),
|
||||
|
||||
Reference in New Issue
Block a user