iHore/vLLM
8
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[Rust Frontend] Support dynamic tools from developer messages (#51144)

Browse Source 
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Bugen Zhao
2026-08-10 18:30:38 -07:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent b2506d62ae
commit f1e921b6d6
28 changed files with 686 additions and 258 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rust/src/chat/src/error.rs
+10 -1
View File
@@ -71,6 +71,12 @@ pub enum Error {
StreamClosedBeforeTerminalOutput { request_id: String },
#[error("tool call stream state is inconsistent: {message}")]
ToolCallStreamInvariant { message: String },
#[error("duplicate tool name `{name}`")]
DuplicateToolName { name: String },
#[error("tool_choice requires at least one available tool")]
ToolChoiceRequiresTools,
#[error("tool_choice function `{name}` was not found in the available tools")]
ToolChoiceFunctionNotFound { name: String },
#[error("failed to build structural tag: {message}")]
StructuralTag { message: String },
#[error(transparent)]
@@ -85,7 +91,10 @@ impl Error {
/// Whether this error represents invalid user request parameters.
pub fn is_request_validation_error(&self) -> bool {
match self {
Self::PromptTooLong { .. } => true,
Self::PromptTooLong { .. }
| Self::DuplicateToolName { .. }
| Self::ToolChoiceRequiresTools
| Self::ToolChoiceFunctionNotFound { .. } => true,
Self::Text(error) => error.is_request_validation_error(),
Self::UnsupportedMultimodalRenderer
| Self::UnsupportedMultimodalContent(_)
rust/src/chat/src/lib.rs
+1 -1
View File
@@ -39,7 +39,7 @@ pub use renderer::{
};
pub use request::{
ChatContent, ChatContentPart, ChatMessage, ChatOptions, ChatRequest, ChatRole, ChatTool,
ChatToolChoice, GenerationPromptMode, ReasoningEffort, SamplingParams,
ChatToolChoice, GenerationPromptMode, ReasoningEffort, ResolvedToolContext, SamplingParams,
};
pub use stream::{ChatEventStream, ChatEventStreamTrait, CollectedAssistantMessage};
pub use vllm_engine_core_client::protocol::multimodal::MmFeatures;
rust/src/chat/src/output/default/mod.rs
+3 -3
View File
@@ -52,7 +52,7 @@ impl DefaultChatOutputProcessor {
) -> ChatResult<Self> {
let parser = if tool_call_parser == reasoning_parser
&& let Some(parser) = Self::resolve_optional_unified_parser(
&request.tools,
request.tools(),
model_id,
tokenizer.clone(),
tool_call_parser,
@@ -62,7 +62,7 @@ impl DefaultChatOutputProcessor {
let tool_parsing_enabled = request.tool_parsing_enabled();
let tool_parser = if tool_parsing_enabled {
Some(Self::resolve_tool_parser(
&request.tools,
request.tools(),
model_id,
tool_call_parser,
)?)
@@ -82,7 +82,7 @@ impl DefaultChatOutputProcessor {
Ok(Self {
parser,
parallel_tool_calls: request.parallel_tool_calls,
parallel_tool_calls: request.parallel_tool_calls(),
})
}
rust/src/chat/src/output/default/structural_tag.rs
+66 -14
View File
@@ -31,7 +31,7 @@ pub(super) fn apply_structural_tag_constraint(
};
let tools = request
.tools
.tools()
.iter()
.map(|tool| {
ToolParam::Function(FunctionToolParam::new(FunctionDefinition {
@@ -67,13 +67,13 @@ pub(super) fn apply_structural_tag_constraint(
///
/// Returns `None` if no structural tag constraints should be applied.
fn structural_tag_tool_choice(request: &ChatRequest) -> Option<StructuralTagToolChoice> {
if request.tools.is_empty() {
if request.tools().is_empty() {
return None;
}
match &request.tool_choice {
match request.tool_choice() {
// For `Auto`, only apply the structural tag if there's at least one strict tool.
ChatToolChoice::Auto if request.tools.iter().any(|tool| tool.strict == Some(true)) => {
ChatToolChoice::Auto if request.tools().iter().any(|tool| tool.strict == Some(true)) => {
Some(StructuralTagToolChoice::auto())
}
ChatToolChoice::Auto | ChatToolChoice::None => None,
@@ -92,6 +92,7 @@ mod tests {
use vllm_parser::tool::{Qwen3CoderToolParser, Tool, ToolParser};
use super::*;
use crate::request::{ChatMessage, ResolvedToolContext};
fn chat_tool(name: &str, strict: Option<bool>) -> Tool {
Tool {
@@ -114,8 +115,8 @@ mod tests {
fn request(tool_choice: ChatToolChoice, tools: Vec<Tool>) -> ChatRequest {
ChatRequest {
tool_choice,
tools,
tool_context: ResolvedToolContext::new(&[], tools, Some(tool_choice), true)
.expect("tool context should resolve"),
..ChatRequest::for_test()
}
}
@@ -145,7 +146,7 @@ mod tests {
#[test]
fn auto_strict_tool_choice_builds_structural_tag() {
let mut request = request(ChatToolChoice::Auto, vec![chat_tool("search", Some(true))]);
let parser = qwen3_coder_parser(&request.tools);
let parser = qwen3_coder_parser(request.tools());
apply_structural_tag_constraint(&mut request, parser.structural_tag_builder())
.expect("structural tag should build");
@@ -155,10 +156,61 @@ mod tests {
assert!(tag.to_string().contains("search"));
}
#[test]
fn required_dynamic_tool_builds_structural_tag_from_effective_tools() {
let messages = vec![ChatMessage::developer(
"",
Some(vec![chat_tool("lookup", None)]),
)];
let tool_context =
ResolvedToolContext::new(&messages, Vec::new(), Some(ChatToolChoice::Required), true)
.expect("dynamic tool context should resolve");
let mut request = ChatRequest {
messages,
tool_context,
..ChatRequest::for_test()
};
let parser = qwen3_coder_parser(request.tools());
apply_structural_tag_constraint(&mut request, parser.structural_tag_builder())
.expect("structural tag should build");
let tag = structural_tag_value(&request);
assert!(tag.to_string().contains("lookup"));
}
#[test]
fn required_initial_and_dynamic_tools_build_one_structural_tag() {
let messages = vec![ChatMessage::developer(
"",
Some(vec![chat_tool("lookup", None)]),
)];
let tool_context = ResolvedToolContext::new(
&messages,
vec![chat_tool("search", None)],
Some(ChatToolChoice::Required),
true,
)
.expect("tool context should resolve");
let mut request = ChatRequest {
messages,
tool_context,
..ChatRequest::for_test()
};
let parser = qwen3_coder_parser(request.tools());
apply_structural_tag_constraint(&mut request, parser.structural_tag_builder())
.expect("structural tag should build");
let tag = structural_tag_value(&request).to_string();
assert!(tag.contains("search"));
assert!(tag.contains("lookup"));
}
#[test]
fn auto_non_strict_tool_choice_skips_structural_tag() {
let mut request = request(ChatToolChoice::Auto, vec![chat_tool("search", None)]);
let parser = qwen3_coder_parser(&request.tools);
let parser = qwen3_coder_parser(request.tools());
apply_structural_tag_constraint(&mut request, parser.structural_tag_builder())
.expect("structural tag decision should succeed");
@@ -173,7 +225,7 @@ mod tests {
backend: StructuredOutputBackend::Xgrammar,
..StructuredOutputsParams::json(json!({"type": "object"}))
});
let parser = qwen3_coder_parser(&request.tools);
let parser = qwen3_coder_parser(request.tools());
apply_structural_tag_constraint(&mut request, parser.structural_tag_builder())
.expect("structural tag should build");
@@ -188,7 +240,7 @@ mod tests {
#[test]
fn required_tool_choice_builds_structural_tag_without_strict_tools() {
let mut request = request(ChatToolChoice::Required, vec![chat_tool("search", None)]);
let parser = qwen3_coder_parser(&request.tools);
let parser = qwen3_coder_parser(request.tools());
apply_structural_tag_constraint(&mut request, parser.structural_tag_builder())
.expect("structural tag should build");
@@ -205,7 +257,7 @@ mod tests {
backend: StructuredOutputBackend::Xgrammar,
..StructuredOutputsParams::json_object()
});
let parser = qwen3_coder_parser(&request.tools);
let parser = qwen3_coder_parser(request.tools());
apply_structural_tag_constraint(&mut request, parser.structural_tag_builder())
.expect("structural tag should build");
@@ -225,7 +277,7 @@ mod tests {
},
vec![chat_tool("search", None), chat_tool("lookup", None)],
);
let parser = qwen3_coder_parser(&request.tools);
let parser = qwen3_coder_parser(request.tools());
apply_structural_tag_constraint(&mut request, parser.structural_tag_builder())
.expect("structural tag should build");
@@ -238,7 +290,7 @@ mod tests {
#[test]
fn none_tool_choice_skips_structural_tag() {
let mut request = request(ChatToolChoice::None, vec![chat_tool("search", Some(true))]);
let parser = qwen3_coder_parser(&request.tools);
let parser = qwen3_coder_parser(request.tools());
apply_structural_tag_constraint(&mut request, parser.structural_tag_builder())
.expect("structural tag decision should succeed");
@@ -253,7 +305,7 @@ mod tests {
backend: StructuredOutputBackend::Xgrammar,
..StructuredOutputsParams::json_object()
});
let parser = qwen3_coder_parser(&request.tools);
let parser = qwen3_coder_parser(request.tools());
apply_structural_tag_constraint(&mut request, parser.structural_tag_builder())
.expect("structural tag decision should succeed");
rust/src/chat/src/output/harmony/mod.rs
+1 -1
View File
@@ -75,7 +75,7 @@ impl HarmonyChatOutputProcessor {
Ok(Self {
encoding: harmony_encoding()?,
tool_calls_enabled: request.tool_parsing_enabled(),
parallel_tool_calls: request.parallel_tool_calls,
parallel_tool_calls: request.parallel_tool_calls(),
})
}
}
rust/src/chat/src/output/harmony/tests.rs
+13 -12
View File
@@ -10,7 +10,7 @@ use vllm_text::output::{DecodedLogprobs, DecodedPositionLogprobs, DecodedTextEve
use super::*;
use crate::output::ChatOutputProcessor;
use crate::request::{ChatRequest, ChatTool, ChatToolChoice};
use crate::request::{ChatRequest, ChatTool, ChatToolChoice, ResolvedToolContext};
use crate::{AssistantMessageExt, ChatEvent, FinishReason};
fn assistant_prefix() -> Vec<u32> {
@@ -72,18 +72,19 @@ async fn collect_events(
}
fn request_with_tools() -> ChatRequest {
let tools = vec![ChatTool {
name: "get_weather".to_string(),
description: Some("Get weather".to_string()),
parameters: serde_json::json!({
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"]
}),
strict: None,
}];
ChatRequest {
tool_choice: ChatToolChoice::Auto,
tools: vec![ChatTool {
name: "get_weather".to_string(),
description: Some("Get weather".to_string()),
parameters: serde_json::json!({
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"]
}),
strict: None,
}],
tool_context: ResolvedToolContext::new(&[], tools, Some(ChatToolChoice::Auto), true)
.expect("tool context should resolve"),
..ChatRequest::for_test()
}
}
rust/src/chat/src/renderer/deepseek_v32/encoding.rs
+11 -3
View File
@@ -2,6 +2,9 @@
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
//! DeepSeek V3.2 prompt renderer.
//!
//! Official Python reference:
//! <https://huggingface.co/deepseek-ai/DeepSeek-V3.2/blob/main/encoding/encoding_dsv32.py>
use std::collections::{HashMap, HashSet};
use std::fmt::Write as _;
@@ -48,15 +51,20 @@ pub(super) fn render_request(request: &ChatRequest) -> Result<String> {
request.messages.last().map(ChatMessage::role),
Some(ChatRole::User | ChatRole::Developer)
);
let render_offset = isize::from(request.tool_parsing_enabled());
let initial_tools = if request.tool_parsing_enabled() {
request.initial_tools()
} else {
&[]
};
let render_offset = isize::from(!initial_tools.is_empty());
let last_user_render_index =
find_last_user_render_index(request.messages.as_slice(), render_offset);
let last_user_actual_index = find_last_user_actual_index(request.messages.as_slice());
let continue_final_message = request.chat_options.continue_final_message();
let mut prompt = String::from(BOS_TOKEN);
if request.tool_parsing_enabled() {
render_system_message(&mut prompt, None, &request.tools)?;
if !initial_tools.is_empty() {
render_system_message(&mut prompt, None, initial_tools)?;
}
for (message_index, message) in request.messages.iter().enumerate() {
rust/src/chat/src/renderer/deepseek_v32/fixtures/test_input_developer_tools.json
Generated
+32
View File
@@ -0,0 +1,32 @@
{
"messages": [
{
"role": "user",
"content": "question"
},
{
"role": "developer",
"content": "policy",
"tools": [
{
"type": "function",
"function": {
"name": "lookup",
"parameters": {
"type": "object"
}
}
}
]
},
{
"role": "assistant",
"reasoning_content": "thought",
"content": "answer"
}
],
"add_generation_prompt": false,
"template_kwargs": {
"thinking": true
}
}
rust/src/chat/src/renderer/deepseek_v32/fixtures/test_output_developer_tools.txt
Generated
+37
View File
@@ -0,0 +1,37 @@
<｜begin▁of▁sentence｜><｜User｜>question<｜Assistant｜></think><｜User｜>
## Tools
You have access to a set of tools you can use to answer the user's question.
You can invoke functions by writing a "<｜DSML｜function_calls>" block like the following as part of your reply to the user:
<｜DSML｜function_calls>
<｜DSML｜invoke name="$FUNCTION_NAME">
<｜DSML｜parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE</｜DSML｜parameter>
...
</｜DSML｜invoke>
<｜DSML｜invoke name="$FUNCTION_NAME2">
...
</｜DSML｜invoke>
</｜DSML｜function_calls>
String and scalar parameters should be specified as is without any escaping or quotes, while lists and objects should use JSON format. The "string" attribute should be set to "true" for string type parameters and "false" for other types (numbers, booleans, arrays, objects).
If the thinking_mode is enabled, then after function results you should strongly consider outputting a thinking block. Here is an example:
<｜DSML｜function_calls>
...
</｜DSML｜function_calls>
<function_results>
...
</function_results>
<think>...thinking about results</think>
Here are the functions available in JSONSchema format:
<functions>
{"name": "lookup", "parameters": {"type": "object"}}
</functions>
# The user's message is: policy<｜Assistant｜><think>thought</think>answer<｜end▁of▁sentence｜>
rust/src/chat/src/renderer/deepseek_v32/tests.rs
+30 -19
View File
@@ -13,6 +13,7 @@ use crate::event::{AssistantContentBlock, AssistantToolCall};
use crate::renderer::test_utils::{FixtureRequestOptions, fixture_chat_request};
use crate::request::{
ChatContentPart, ChatMessage, ChatRequest, ChatTool, ChatToolChoice, GenerationPromptMode,
ResolvedToolContext,
};
use crate::{ChatRenderer, ChatRole};
@@ -85,6 +86,14 @@ fn renders_vllm_parity_prompt_for_request_level_tools_fixture() {
);
}
#[test]
fn renders_developer_tools_like_hf_python() {
assert_fixture(
"test_input_developer_tools.json",
expect_file!["fixtures/test_output_developer_tools.txt"],
);
}
#[test]
fn renders_official_search_fixture_without_date() {
assert_fixture(
@@ -103,27 +112,29 @@ fn renders_official_search_fixture_with_date() {
#[test]
fn request_level_tools_are_lowered_as_synthetic_leading_system_message() {
let messages = vec![
ChatMessage::system("System prompt."),
ChatMessage::text(ChatRole::User, "Hello"),
];
let tools = vec![ChatTool {
name: "lookup".to_string(),
description: Some("Look things up".to_string()),
parameters: json!({
"type": "object",
"properties": {
"query": {
"type": "string"
}
},
"required": ["query"]
}),
strict: None,
}];
let mut request = ChatRequest {
request_id: "deepseek-v32-tools".to_string(),
messages: vec![
ChatMessage::system("System prompt."),
ChatMessage::text(ChatRole::User, "Hello"),
],
tools: vec![ChatTool {
name: "lookup".to_string(),
description: Some("Look things up".to_string()),
parameters: json!({
"type": "object",
"properties": {
"query": {
"type": "string"
}
},
"required": ["query"]
}),
strict: None,
}],
tool_choice: ChatToolChoice::Auto,
tool_context: ResolvedToolContext::new(&messages, tools, Some(ChatToolChoice::Auto), true)
.expect("tool context should resolve"),
messages,
..ChatRequest::for_test()
};
request
rust/src/chat/src/renderer/deepseek_v4/encoding.rs
+3 -3
View File
@@ -3,8 +3,8 @@
//! DeepSeek V4 prompt renderer.
//!
//! Original Python implementation:
//! <https://github.com/vllm-project/vllm/blob/main/vllm/tokenizers/deepseek_v4_encoding.py>
//! Official Python reference:
//! <https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash/blob/main/encoding/encoding_dsv4.py>
use std::collections::HashMap;
use std::fmt::Write as _;
@@ -156,7 +156,7 @@ fn resolve_thinking_options(request: &ChatRequest) -> Result<(ThinkingMode, &'st
/// Return request-level tools only when native tool parsing is enabled.
fn request_tools(request: &ChatRequest) -> &[ChatTool] {
if request.tool_parsing_enabled() {
request.tools.as_slice()
request.initial_tools()
} else {
&[]
}
rust/src/chat/src/renderer/deepseek_v4/fixtures/test_input_developer_tools.json
+31
View File
@@ -0,0 +1,31 @@
{
"messages": [
{
"role": "user",
"content": "question"
},
{
"role": "developer",
"content": "policy",
"tools": [
{
"type": "function",
"function": {
"name": "lookup",
"parameters": {
"type": "object"
}
}
}
]
},
{
"role": "assistant",
"content": "answer"
}
],
"add_generation_prompt": false,
"template_kwargs": {
"thinking": true
}
}
rust/src/chat/src/renderer/deepseek_v4/fixtures/test_output_developer_tools.txt
+28
View File
@@ -0,0 +1,28 @@
<｜begin▁of▁sentence｜><｜User｜>question<｜User｜>policy
## Tools
You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<｜DSML｜tool_calls>" block like the following:
<｜DSML｜tool_calls>
<｜DSML｜invoke name="$TOOL_NAME">
<｜DSML｜parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE</｜DSML｜parameter>
...
</｜DSML｜invoke>
<｜DSML｜invoke name="$TOOL_NAME2">
...
</｜DSML｜invoke>
</｜DSML｜tool_calls>
String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`.
If thinking_mode is enabled (triggered by <think>), you MUST output your complete reasoning inside <think>...</think> BEFORE any tool calls or final response.
Otherwise, output directly after </think> with tool calls or final response.
### Available Tool Schemas
{"name": "lookup", "parameters": {"type": "object"}}
You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.
<｜Assistant｜><think></think>answer<｜end▁of▁sentence｜>
rust/src/chat/src/renderer/deepseek_v4/tests.rs
+8
View File
@@ -63,6 +63,14 @@ fn renders_v4_fixture_2_multi_turn_drop_thinking() {
);
}
#[test]
fn renders_developer_tools_like_hf_python() {
assert_fixture(
"test_input_developer_tools.json",
expect_file!["fixtures/test_output_developer_tools.txt"],
);
}
#[test]
fn reasoning_effort_max_adds_prefix_when_thinking_is_enabled() {
let mut request = ChatRequest {
rust/src/chat/src/renderer/harmony/mod.rs
+1 -1
View File
@@ -216,7 +216,7 @@ fn preamble_tool_descriptions(
) -> Vec<ToolDescription> {
let mut tools = Vec::new();
if request.tool_parsing_enabled() {
tools.extend(to_tool_descriptions(&request.tools));
tools.extend(to_tool_descriptions(request.initial_tools()));
}
if let Some(leading_developer_tools) = leading_developer_tools {
tools.extend(to_tool_descriptions(leading_developer_tools));
rust/src/chat/src/renderer/hf/mod.rs
+46 -5
View File
@@ -179,7 +179,7 @@ impl HfChatRenderer {
None
};
let tools = request.tool_parsing_enabled().then(|| to_template_tools(&request.tools));
let tools = request.tool_parsing_enabled().then(|| to_template_tools(request.tools()));
trace!(
message_count = messages.len(),
content_format = ?effective_template.content_format(),
@@ -579,7 +579,7 @@ mod tests {
use super::{ChatTemplateContentFormatOption, HfChatRenderer, MultimodalRenderInfo};
use crate::request::{
ChatContentPart, ChatMessage, ChatRequest, ChatRole, ChatTool, ChatToolChoice,
GenerationPromptMode, ReasoningEffort,
GenerationPromptMode, ReasoningEffort, ResolvedToolContext,
};
use crate::{AssistantContentBlock, ChatRenderer, Error, Result};
@@ -587,13 +587,48 @@ mod tests {
const QWEN3_5_0_8B_TEMPLATE: &str = include_str!("../../../tests/templates/qwen35.jinja");
fn sample_request(messages: Vec<ChatMessage>) -> ChatRequest {
let tool_context = ResolvedToolContext::new(&messages, Vec::new(), None, true).unwrap();
ChatRequest {
messages,
tool_context,
request_id: "render-test".to_string(),
..ChatRequest::for_test()
}
}
#[test]
fn dynamic_tools_are_exposed_as_effective_template_tools() {
let messages = vec![
ChatMessage::developer(
"",
Some(vec![ChatTool {
name: "lookup".to_string(),
description: None,
parameters: serde_json::json!({"type": "object"}),
strict: None,
}]),
),
ChatMessage::user("hello"),
];
let tool_context = ResolvedToolContext::new(&messages, Vec::new(), None, true).unwrap();
let request = ChatRequest {
messages,
tool_context,
request_id: "render-test".to_string(),
..ChatRequest::for_test()
};
let rendered = render(
Some("{{ messages|length }}:{{ messages[0].role }}:{{ messages[0].tools[0].function.name }}:{{ tools[0].function.name }}"),
&request,
)
.unwrap();
assert_eq!(rendered, "2:developer:lookup:lookup");
assert_eq!(request.tool_choice(), &ChatToolChoice::Auto);
assert_eq!(request.tools().len(), 1);
}
fn render(template: Option<&str>, request: &ChatRequest) -> Result<String> {
HfChatRenderer::new(
template.map(str::to_owned),
@@ -897,7 +932,7 @@ mod tests {
}
#[test]
fn chat_template_exposes_developer_tools() {
fn chat_template_preserves_developer_tools() {
let request = sample_request(vec![ChatMessage::developer(
"policy",
Some(vec![ChatTool {
@@ -1218,7 +1253,7 @@ mod tests {
#[test]
fn chat_template_exposes_tools_to_templates_when_auto_enabled() {
let mut request = sample_request(vec![ChatMessage::text(ChatRole::User, "hello")]);
request.tools = vec![ChatTool {
let tools = vec![ChatTool {
name: "get_weather".to_string(),
description: Some("Get weather".to_string()),
parameters: serde_json::json!({
@@ -1228,7 +1263,13 @@ mod tests {
}),
strict: None,
}];
request.tool_choice = ChatToolChoice::Auto;
request.tool_context = crate::request::ResolvedToolContext::new(
&request.messages,
tools,
Some(ChatToolChoice::Auto),
true,
)
.expect("tool context should resolve");
let rendered = render(
Some("{{ tools[0].function.name }}|{{ tools[0].function.parameters.required[0] }}"),
rust/src/chat/src/renderer/inkling/mod.rs
+1 -12
View File
@@ -383,18 +383,7 @@ fn rendered_tools(request: &ChatRequest) -> Vec<&ChatTool> {
return Vec::new();
}
let mut tools = Vec::with_capacity(request.tools.len());
tools.extend(request.tools.iter());
for message in &request.messages {
if let ChatMessage::Developer {
tools: Some(local_tools),
..
} = message
{
tools.extend(local_tools.iter());
}
}
tools
request.tools().iter().collect()
}
fn tool_call_json(tool_call: &AssistantToolCall) -> Result<String> {
rust/src/chat/src/renderer/kimi_k3/encoding.rs
+3 -3
View File
@@ -191,7 +191,7 @@ pub(super) fn render_request(request: &ChatRequest, tokenizer: &dyn Tokenizer) -
}
flush_tool_run(&mut out, &mut pending_tool_run, &tool_call_id_index)?;
match &request.tool_choice {
match request.tool_choice() {
ChatToolChoice::Required => {
write_internal_system(
&mut out,
@@ -202,7 +202,7 @@ pub(super) fn render_request(request: &ChatRequest, tokenizer: &dyn Tokenizer) -
}
// Emit only when tools are present: Rust defaults tool_choice to None
// for tool-free requests, which must not inject a tool-choice message.
ChatToolChoice::None if !request.tools.is_empty() => {
ChatToolChoice::None if !request.tools().is_empty() => {
write_internal_system(
&mut out,
"tool-choice",
@@ -226,7 +226,7 @@ pub(super) fn render_request(request: &ChatRequest, tokenizer: &dyn Tokenizer) -
fn request_tools(request: &ChatRequest) -> &[ChatTool] {
// Declare tools whenever the request carries them. K3 tool-declare is
// independent of tool_choice; tool_choice only injects control messages.
request.tools.as_slice()
request.initial_tools()
}
fn thinking_enabled(request: &ChatRequest) -> Result<bool> {
rust/src/chat/src/renderer/kimi_k3/fixtures/dynamic_only_tool_declare_input.json
+40
View File
@@ -0,0 +1,40 @@
{
"messages": [
{
"role": "user",
"content": "Load the calculator."
},
{
"role": "developer",
"tools": [
{
"type": "function",
"function": {
"name": "calc",
"description": "Evaluate an expression",
"parameters": {
"type": "object",
"properties": {
"expression": {
"type": "string"
}
},
"required": [
"expression"
]
}
}
}
]
},
{
"role": "user",
"content": "What is 2 + 2?"
}
],
"add_generation_prompt": true,
"tool_choice": "none",
"template_kwargs": {
"thinking": true
}
}
rust/src/chat/src/renderer/kimi_k3/fixtures/dynamic_only_tool_declare_output.txt
+10
View File
@@ -0,0 +1,10 @@
<|open|>message role="system" type="thinking-effort"<|sep|>`thinking_effort` guides on how much to think in your thinking channel (not including the response channel), supported values include `low`, `medium`, `high`, and `max`.
Now the system is invoked with `thinking_effort=max`.<|close|>message<|sep|><|end_of_msg|><|open|>message role="user"<|sep|>Load the calculator.<|close|>message<|sep|><|end_of_msg|><|open|>message role="system" type="tool-declare"<|sep|>## New Tools Available
The system dynamically extends the toolset via lazy-loading.
You have access to all existing and extended tools.
Here are the specs for the extended tools.
```json
[{"function":{"description":"Evaluate an expression","name":"calc","parameters":{"properties":{"expression":{"type":"string"}},"required":["expression"],"type":"object"}},"type":"function"}]
```<|close|>message<|sep|><|end_of_msg|><|open|>message role="user"<|sep|>What is 2 + 2?<|close|>message<|sep|><|end_of_msg|><|open|>message role="system" type="tool-choice"<|sep|>The system is invoked with `tool_choice=none`.
You MUST NOT call any tools in the next message.<|close|>message<|sep|><|end_of_msg|><|open|>message role="assistant"<|sep|><|open|>think<|sep|>
rust/src/chat/src/renderer/kimi_k3/tests.rs
+13 -2
View File
@@ -92,10 +92,15 @@ fn golden_dynamic_system_tool_declare() {
assert_golden("dynamic_system_tool_declare");
}
#[test]
fn golden_dynamic_only_tool_declare() {
assert_golden("dynamic_only_tool_declare");
}
#[test]
fn tool_declare_omits_absent_fields_and_preserves_parameter_nulls() {
let mut request = crate::request::ChatRequest::for_test();
request.tools = vec![ChatTool {
let tools = vec![ChatTool {
name: "lookup".to_string(),
description: None,
parameters: json!({
@@ -109,6 +114,9 @@ fn tool_declare_omits_absent_fields_and_preserves_parameter_nulls() {
}),
strict: None,
}];
request.tool_context =
crate::request::ResolvedToolContext::new(&request.messages, tools, None, true)
.expect("tool context should resolve");
let rendered = render_request(&request);
@@ -122,12 +130,15 @@ fn tool_declare_omits_absent_fields_and_preserves_parameter_nulls() {
#[test]
fn tool_declare_preserves_present_optional_fields() {
let mut request = crate::request::ChatRequest::for_test();
request.tools = vec![ChatTool {
let tools = vec![ChatTool {
name: "lookup".to_string(),
description: Some("Look up a record".to_string()),
parameters: json!({"type": "object"}),
strict: Some(false),
}];
request.tool_context =
crate::request::ResolvedToolContext::new(&request.messages, tools, None, true)
.expect("tool context should resolve");
let rendered = render_request(&request);
rust/src/chat/src/renderer/test_utils.rs
+12 -14
View File
@@ -11,7 +11,7 @@ use serde_json::Value;
use crate::event::{AssistantContentBlock, AssistantToolCall};
use crate::request::{
ChatContent, ChatContentPart, ChatMessage, ChatRequest, ChatTool, ChatToolChoice,
GenerationPromptMode, ReasoningEffort,
GenerationPromptMode, ReasoningEffort, ResolvedToolContext,
};
/// Options for constructing a [`ChatRequest`] from a fixture file.
@@ -158,22 +158,20 @@ struct FixtureToolCallFunction {
impl FixtureRequest {
fn into_chat_request(self, options: FixtureRequestOptions) -> ChatRequest {
let tools = to_chat_tools(&self.tools);
let tool_choice = self.tool_choice.unwrap_or(if tools.is_empty() {
ChatToolChoice::None
} else {
ChatToolChoice::Auto
});
let messages = self
.messages
.into_iter()
.enumerate()
.map(|(index, message)| fixture_message_to_chat_message(index, message))
.collect::<Vec<_>>();
let tool_context = ResolvedToolContext::new(&messages, tools, self.tool_choice, true)
.expect("fixture tool context should resolve");
let mut request = ChatRequest {
request_id: "renderer-fixture".to_string(),
messages: self
.messages
.into_iter()
.enumerate()
.map(|(index, message)| fixture_message_to_chat_message(index, message))
.collect(),
tools,
tool_choice,
messages,
tool_context,
..ChatRequest::for_test()
};
rust/src/chat/src/request.rs
+224 -14
View File
@@ -1,7 +1,7 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
use std::collections::HashMap;
use std::collections::{HashMap, HashSet};
use llm_multimodal::ImageDetail;
use serde::{Deserialize, Serialize};
@@ -248,6 +248,16 @@ impl ChatMessage {
}
}
/// Return message-scoped tool declarations carried by this message.
pub fn declared_tools(&self) -> Option<&[ChatTool]> {
match self {
Self::Developer {
tools: Some(tools), ..
} if !tools.is_empty() => Some(tools),
_ => None,
}
}
/// Construct one chat message with user role.
pub fn user(content: impl Into<ChatContent>) -> Self {
Self::User {
@@ -450,6 +460,93 @@ pub enum ChatToolChoice {
},
}
/// Resolved tool state shared by rendering, output parsing, and constraints.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct ResolvedToolContext {
/// Request-level tools rendered before the ordered chat history.
pub initial_tools: Vec<ChatTool>,
/// Request-level and message-scoped tools available for this generation.
pub effective_tools: Vec<ChatTool>,
/// Resolved tool-choice behavior for this request.
pub tool_choice: ChatToolChoice,
/// Whether more than one parsed tool call may be surfaced northbound.
pub parallel_tool_calls: bool,
}
impl ResolvedToolContext {
/// Resolve one complete tool context from request and message declarations.
pub fn new(
messages: &[ChatMessage],
initial_tools: Vec<ChatTool>,
requested_tool_choice: Option<ChatToolChoice>,
parallel_tool_calls: bool,
) -> Result<Self> {
let dynamic_tool_count = messages
.iter()
.filter_map(ChatMessage::declared_tools)
.map(<[ChatTool]>::len)
.sum::<usize>();
let mut effective_tools = Vec::with_capacity(initial_tools.len() + dynamic_tool_count);
let mut names = HashSet::with_capacity(effective_tools.capacity());
for tool in initial_tools
.iter()
.chain(messages.iter().filter_map(ChatMessage::declared_tools).flatten())
{
if !names.insert(tool.name.clone()) {
return Err(Error::DuplicateToolName {
name: tool.name.clone(),
});
}
effective_tools.push(tool.clone());
}
let tool_choice = requested_tool_choice.unwrap_or({
if effective_tools.is_empty() {
ChatToolChoice::None
} else {
ChatToolChoice::Auto
}
});
match &tool_choice {
ChatToolChoice::Auto | ChatToolChoice::Required if effective_tools.is_empty() => {
return Err(Error::ToolChoiceRequiresTools);
}
ChatToolChoice::Function { name } if !names.contains(name) => {
return Err(Error::ToolChoiceFunctionNotFound { name: name.clone() });
}
ChatToolChoice::None
| ChatToolChoice::Auto
| ChatToolChoice::Required
| ChatToolChoice::Function { .. } => {}
}
Ok(Self {
initial_tools,
effective_tools,
tool_choice,
parallel_tool_calls,
})
}
/// Return whether tool output parsing is active for this request.
pub fn parsing_enabled(&self) -> bool {
!matches!(self.tool_choice, ChatToolChoice::None) && !self.effective_tools.is_empty()
}
}
impl Default for ResolvedToolContext {
fn default() -> Self {
Self {
initial_tools: Vec::new(),
effective_tools: Vec::new(),
tool_choice: ChatToolChoice::None,
parallel_tool_calls: true,
}
}
}
/// One chat request ready to be rendered into a prompt and lowered into a
/// generate request.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
@@ -462,14 +559,8 @@ pub struct ChatRequest {
pub sampling_params: SamplingParams,
/// Chat-specific rendering options.
pub chat_options: ChatOptions,
/// Function tools made available to the model for this request.
pub tools: Vec<ChatTool>,
/// Tool-choice behavior for this request.
pub tool_choice: ChatToolChoice,
/// Whether the model may return more than one tool call per response.
///
/// When `false`, only the first parsed tool call is surfaced northbound.
pub parallel_tool_calls: bool,
/// Resolved request-level and message-scoped tool behavior.
pub tool_context: ResolvedToolContext,
/// Text decode options for incremental detokenization.
pub decode_options: TextDecodeOptions,
/// Whether to emit intermediate northbound content deltas before the
@@ -507,9 +598,7 @@ impl ChatRequest {
messages: vec![ChatMessage::text(ChatRole::User, "test")],
sampling_params: SamplingParams::default(),
chat_options: ChatOptions::default(),
tools: Vec::new(),
tool_choice: ChatToolChoice::None,
parallel_tool_calls: true,
tool_context: ResolvedToolContext::default(),
decode_options: TextDecodeOptions::default(),
intermediate: true,
priority: 0,
@@ -547,10 +636,33 @@ impl ChatRequest {
self.messages.iter().any(ChatMessage::has_multimodal)
}
/// Return every tool available for this generation.
pub fn tools(&self) -> &[ChatTool] {
&self.tool_context.effective_tools
}
/// Return request-level tools before message-scoped declarations.
///
/// Renderers should use [`Self::tools`] unless their prompt format places
/// initial and message-scoped declarations differently.
pub fn initial_tools(&self) -> &[ChatTool] {
&self.tool_context.initial_tools
}
/// Return the resolved tool-choice behavior.
pub fn tool_choice(&self) -> &ChatToolChoice {
&self.tool_context.tool_choice
}
/// Return whether more than one parsed tool call may be surfaced.
pub fn parallel_tool_calls(&self) -> bool {
self.tool_context.parallel_tool_calls
}
/// Return true if this request should enable tool parsing based on the tool
/// choice and tool list.
pub(crate) fn tool_parsing_enabled(&self) -> bool {
!matches!(self.tool_choice, ChatToolChoice::None) && !self.tools.is_empty()
self.tool_context.parsing_enabled()
}
/// Return the request-level thinking toggle when explicitly requested.
@@ -605,7 +717,10 @@ impl ChatRole {
mod tests {
use serde_json::{json, to_value};
use super::{ChatContent, ChatContentPart, ChatMessage, ChatRequest, ChatRole, ChatTool};
use super::{
ChatContent, ChatContentPart, ChatMessage, ChatRequest, ChatRole, ChatTool, ChatToolChoice,
ResolvedToolContext,
};
use crate::Error;
use crate::event::AssistantContentBlock;
@@ -702,6 +817,101 @@ mod tests {
assert_eq!(decoded, message);
}
fn chat_tool(name: &str) -> ChatTool {
ChatTool {
name: name.to_string(),
description: None,
parameters: json!({"type": "object"}),
strict: None,
}
}
#[test]
fn resolved_tool_context_defaults_dynamic_only_tools_to_auto() {
let messages = vec![
ChatMessage::user("hello"),
ChatMessage::developer("", Some(vec![chat_tool("lookup")])),
];
let context = ResolvedToolContext::new(&messages, Vec::new(), None, false).unwrap();
assert!(context.initial_tools.is_empty());
assert_eq!(context.effective_tools, vec![chat_tool("lookup")]);
assert_eq!(context.tool_choice, ChatToolChoice::Auto);
assert!(!context.parallel_tool_calls);
assert!(context.parsing_enabled());
}
#[test]
fn resolved_tool_context_preserves_initial_then_message_order() {
let messages = vec![
ChatMessage::developer("", Some(vec![chat_tool("middle")])),
ChatMessage::developer("policy", Some(vec![chat_tool("last")])),
];
let context = ResolvedToolContext::new(
&messages,
vec![chat_tool("first")],
Some(ChatToolChoice::Required),
true,
)
.unwrap();
assert_eq!(
context
.effective_tools
.iter()
.map(|tool| tool.name.as_str())
.collect::<Vec<_>>(),
vec!["first", "middle", "last"]
);
assert_eq!(context.tool_choice, ChatToolChoice::Required);
}
#[test]
fn resolved_tool_context_rejects_duplicate_names() {
let messages = vec![ChatMessage::developer("", Some(vec![chat_tool("lookup")]))];
let error =
ResolvedToolContext::new(&messages, vec![chat_tool("lookup")], None, true).unwrap_err();
assert!(matches!(error, Error::DuplicateToolName { name } if name == "lookup"));
}
#[test]
fn resolved_tool_context_validates_named_choice_against_effective_tools() {
let messages = vec![ChatMessage::developer("", Some(vec![chat_tool("lookup")]))];
let context = ResolvedToolContext::new(
&messages,
Vec::new(),
Some(ChatToolChoice::Function {
name: "lookup".to_string(),
}),
true,
)
.unwrap();
assert_eq!(
context.tool_choice,
ChatToolChoice::Function {
name: "lookup".to_string()
}
);
let error = ResolvedToolContext::new(
&messages,
Vec::new(),
Some(ChatToolChoice::Function {
name: "missing".to_string(),
}),
true,
)
.unwrap_err();
assert!(matches!(
error,
Error::ToolChoiceFunctionNotFound { name } if name == "missing"
));
}
#[test]
fn enable_thinking_is_none_when_no_kwargs_are_present() {
let request = ChatRequest::for_test();
rust/src/chat/tests/chat.rs
+8 -2
View File
@@ -302,7 +302,7 @@ fn sample_request(request_id: &str) -> ChatRequest {
fn sample_tool_request(request_id: &str) -> ChatRequest {
let mut request = sample_request(request_id);
request.tools = vec![ChatTool {
let tools = vec![ChatTool {
name: "get_weather".to_string(),
description: Some("Get weather".to_string()),
parameters: serde_json::json!({
@@ -312,7 +312,13 @@ fn sample_tool_request(request_id: &str) -> ChatRequest {
}),
strict: None,
}];
request.tool_choice = ChatToolChoice::Auto;
request.tool_context = vllm_chat::ResolvedToolContext::new(
&request.messages,
tools,
Some(ChatToolChoice::Auto),
true,
)
.expect("tool context should resolve");
request
}
rust/src/chat/tests/roundtrip.rs
+6 -9
View File
@@ -16,9 +16,9 @@ use serde_json_fmt::JsonFormat as JsonFmt;
use serial_test::file_serial;
use vllm_chat::{
AssistantContentBlock, AssistantMessage, AssistantMessageExt as _, AssistantToolCall,
ChatEvent, ChatMessage, ChatRequest, ChatRole, ChatTool, ChatToolChoice, FinishReason,
GenerationPromptMode, LoadModelBackendsOptions, NewChatOutputProcessorOptions, ParserSelection,
RendererSelection, load_model_backends,
ChatEvent, ChatMessage, ChatRequest, ChatRole, ChatTool, FinishReason, GenerationPromptMode,
LoadModelBackendsOptions, NewChatOutputProcessorOptions, ParserSelection, RendererSelection,
load_model_backends,
};
use vllm_text::{DecodedTextEvent, Finished, Prompt};
use vllm_tokenizer::Tokenizer;
@@ -825,15 +825,12 @@ fn roundtrip_request(
tools: Vec<ChatTool>,
thinking: Option<bool>,
) -> ChatRequest {
let tool_context = vllm_chat::ResolvedToolContext::new(&messages, tools, None, true)
.expect("tool context should resolve");
let mut request = ChatRequest {
request_id: request_id.into(),
messages,
tool_choice: if tools.is_empty() {
ChatToolChoice::None
} else {
ChatToolChoice::Auto
},
tools,
tool_context,
..ChatRequest::for_test()
};
rust/src/server/src/routes/openai/chat_completions/convert.rs
+38 -25
View File
@@ -5,16 +5,16 @@ use itertools::Itertools as _;
use vllm_chat::{
AssistantContentBlock, AssistantToolCall, ChatContent, ChatContentPart,
ChatMessage as VllmChatMessage, ChatOptions, ChatRequest, ChatTool, ChatToolChoice,
GenerationPromptMode, SamplingParams,
GenerationPromptMode, ResolvedToolContext, SamplingParams,
};
use super::types::ChatCompletionRequest;
use super::validate;
use crate::error::{ApiError, bail_invalid_request};
use crate::error::{ApiError, bail_invalid_request, chat_submit_error};
use crate::lora::LoraModelResolution;
use crate::routes::openai::utils::structured_outputs::convert_from_response_format;
use crate::routes::openai::utils::types::{
ChatMessage, ContentPart, MessageContent, Tool, ToolChoice, ToolChoiceValue,
ChatMessage, ContentPart, MessageContent, Tool, ToolChoice, ToolChoiceValue, ToolReference,
};
use crate::utils::{
ResolvedRequestContext, convert_logit_bias, merge_ec_transfer_params, merge_kv_transfer_params,
@@ -130,6 +130,14 @@ pub(super) fn prepare_chat_request(
request.vllm_xargs.as_ref(),
);
let tool_context = ResolvedToolContext::new(
&messages,
convert_tools(request.tools)?,
request.tool_choice.as_ref().map(convert_tool_choice).transpose()?,
request.parallel_tool_calls.unwrap_or(true),
)
.map_err(|error| chat_submit_error("failed to resolve request tools", error))?;
let chat_request = ChatRequest {
request_id: request_id.clone(),
messages,
@@ -168,9 +176,7 @@ pub(super) fn prepare_chat_request(
response_format,
template_kwargs,
},
tools: convert_tools(request.tools)?,
tool_choice: convert_tool_choice(request.tool_choice.as_ref())?,
parallel_tool_calls: request.parallel_tool_calls.unwrap_or(true),
tool_context,
decode_options: vllm_text::output::TextDecodeOptions {
skip_special_tokens: request.skip_special_tokens,
include_stop_str_in_output: request.include_stop_str_in_output,
@@ -399,17 +405,21 @@ fn convert_message_tools(tools: Option<Vec<Tool>>) -> Result<Option<Vec<ChatTool
Ok((!tools.is_empty()).then_some(tools))
}
fn convert_tool_choice(tool_choice: Option<&ToolChoice>) -> Result<ChatToolChoice, ApiError> {
fn convert_tool_choice(tool_choice: &ToolChoice) -> Result<ChatToolChoice, ApiError> {
match tool_choice {
None | Some(ToolChoice::Value(ToolChoiceValue::Auto)) => Ok(ChatToolChoice::Auto),
Some(ToolChoice::Value(ToolChoiceValue::None)) => Ok(ChatToolChoice::None),
Some(ToolChoice::Value(ToolChoiceValue::Required)) => Ok(ChatToolChoice::Required),
Some(ToolChoice::Function {
ToolChoice::Value(ToolChoiceValue::Auto) => Ok(ChatToolChoice::Auto),
ToolChoice::Value(ToolChoiceValue::None) => Ok(ChatToolChoice::None),
ToolChoice::Value(ToolChoiceValue::Required) => Ok(ChatToolChoice::Required),
ToolChoice::Function {
tool_type,
function,
}) if tool_type == "function" => Ok(ChatToolChoice::Function {
} if tool_type == "function" => Ok(ChatToolChoice::Function {
name: function.name.clone(),
}),
ToolChoice::AllowedTools { tools, .. } => bail_invalid_request!(
"allowed_tools tool_choice is not supported yet: {}.",
tools.iter().map(ToolReference::identifier).join(", ")
),
_ => bail_invalid_request!("tool_choice={:?} is not supported yet.", tool_choice),
}
}
@@ -478,7 +488,7 @@ mod tests {
)
.expect("request is valid");
assert!(!prepared.chat_request.parallel_tool_calls);
assert!(!prepared.chat_request.parallel_tool_calls());
}
#[test]
@@ -490,7 +500,7 @@ mod tests {
)
.expect("request is valid");
assert!(prepared.chat_request.parallel_tool_calls);
assert!(prepared.chat_request.parallel_tool_calls());
}
#[test]
@@ -596,8 +606,8 @@ mod tests {
min_tokens: 0,
}
);
assert!(prepared.chat_request.tools.is_empty());
assert_eq!(prepared.chat_request.tool_choice, ChatToolChoice::Auto);
assert!(prepared.chat_request.initial_tools().is_empty());
assert_eq!(prepared.chat_request.tool_choice(), &ChatToolChoice::None);
}
#[test]
@@ -671,8 +681,8 @@ mod tests {
min_tokens: 0,
}
);
assert!(prepared.chat_request.tools.is_empty());
assert_eq!(prepared.chat_request.tool_choice, ChatToolChoice::Auto);
assert!(prepared.chat_request.initial_tools().is_empty());
assert_eq!(prepared.chat_request.tool_choice(), &ChatToolChoice::None);
}
#[test]
@@ -1009,8 +1019,8 @@ mod tests {
},
])]
);
assert!(prepared.chat_request.tools.is_empty());
assert_eq!(prepared.chat_request.tool_choice, ChatToolChoice::Auto);
assert!(prepared.chat_request.initial_tools().is_empty());
assert_eq!(prepared.chat_request.tool_choice(), &ChatToolChoice::None);
}
#[test]
@@ -1105,7 +1115,7 @@ mod tests {
]
);
assert_eq!(
prepared.chat_request.tools,
prepared.chat_request.initial_tools(),
vec![VllmChatTool {
name: "get_weather".to_string(),
description: Some("Get weather".to_string()),
@@ -1116,7 +1126,7 @@ mod tests {
strict: None,
}]
);
assert_eq!(prepared.chat_request.tool_choice, ChatToolChoice::None);
assert_eq!(prepared.chat_request.tool_choice(), &ChatToolChoice::None);
}
#[test]
@@ -1145,7 +1155,10 @@ mod tests {
)
.expect("request is valid");
assert_eq!(prepared.chat_request.tool_choice, ChatToolChoice::Required);
assert_eq!(
prepared.chat_request.tool_choice(),
&ChatToolChoice::Required
);
assert!(!prepared.options.is_named_tool_choice);
}
@@ -1181,8 +1194,8 @@ mod tests {
.expect("request is valid");
assert_eq!(
prepared.chat_request.tool_choice,
ChatToolChoice::Function {
prepared.chat_request.tool_choice(),
&ChatToolChoice::Function {
name: "get_weather".to_string(),
}
);
rust/src/server/src/routes/openai/chat_completions/types.rs
+2 -109
View File
@@ -14,8 +14,8 @@ use vllm_engine_core_client::protocol::sampling::RepetitionDetectionParams;
use crate::routes::openai::utils::structured_outputs::ResponseFormat;
use crate::routes::openai::utils::types::{
ChatLogProbs, ChatMessage, Normalizable, StreamOptions, StringOrArray, Tool, ToolCall,
ToolCallDelta, ToolChoice, ToolChoiceValue, ToolReference, UNKNOWN_MODEL_ID, Usage,
default_true, validate_messages, validate_stop, validate_top_p_value,
ToolCallDelta, ToolChoice, UNKNOWN_MODEL_ID, Usage, default_true, validate_messages,
validate_stop, validate_top_p_value,
};
/// vLLM-compatible request type for the Chat Completions API.
@@ -325,19 +325,6 @@ impl Normalizable for ChatCompletionRequest {
self.max_completion_tokens = self.max_tokens;
self.max_tokens = None;
}
// Apply tool_choice defaults
// If tools is None, leave tool_choice as None (don't set it)
if self.tool_choice.is_none()
&& let Some(tools) = &self.tools
{
let choice_value = if tools.is_empty() {
ToolChoiceValue::None
} else {
ToolChoiceValue::Auto
};
self.tool_choice = Some(ToolChoice::Value(choice_value));
}
}
}
@@ -492,99 +479,5 @@ fn validate_chat_cross_parameters(
return Err(e);
}
// 5. Validate tool_choice requires tools (except for "none")
if let Some(ref tool_choice) = req.tool_choice {
let has_tools = req.tools.as_ref().is_some_and(|t| !t.is_empty());
// Check if tool_choice is anything other than "none"
let is_some_choice = !matches!(tool_choice, ToolChoice::Value(ToolChoiceValue::None));
if is_some_choice && !has_tools {
let mut e = validator::ValidationError::new("tool_choice_requires_tools");
e.message = Some("Invalid value for 'tool_choice': 'tool_choice' is only allowed when 'tools' are specified.".into());
return Err(e);
}
// Additional validation when tools are present
if let Some(tools) = req.tools.as_ref().filter(|t| !t.is_empty()) {
match tool_choice {
ToolChoice::Function { function, .. } => {
// Validate that the specified function name exists in tools
let function_exists = tools.iter().any(|tool| {
tool.tool_type == "function" && tool.function.name == function.name
});
if !function_exists {
let mut e =
validator::ValidationError::new("tool_choice_function_not_found");
e.message = Some(
format!(
"Invalid value for 'tool_choice': function '{}' not found in 'tools'.",
function.name
)
.into(),
);
return Err(e);
}
}
ToolChoice::AllowedTools {
mode,
tools: allowed_tools,
..
} => {
// Validate mode is "auto" or "required"
if mode != "auto" && mode != "required" {
let mut e = validator::ValidationError::new("tool_choice_invalid_mode");
e.message = Some(format!(
"Invalid value for 'tool_choice.mode': must be 'auto' or 'required', got '{mode}'."
).into());
return Err(e);
}
// Validate that all ToolReferences are Function type (Chat API only supports
// function tools)
for tool_ref in allowed_tools {
match tool_ref {
ToolReference::Function { name } => {
// Validate that the function exists in tools array
let tool_exists = tools.iter().any(|tool| {
tool.tool_type == "function" && tool.function.name == *name
});
if !tool_exists {
let mut e = validator::ValidationError::new(
"tool_choice_tool_not_found",
);
e.message = Some(
format!(
"Invalid value for 'tool_choice.tools': tool '{name}' not found in 'tools'."
)
.into(),
);
return Err(e);
}
}
_ => {
// Chat Completion API only supports function tools in tool_choice
let mut e = validator::ValidationError::new(
"tool_choice_invalid_tool_type",
);
e.message = Some(
format!(
"Invalid value for 'tool_choice.tools': Chat Completion API only supports function tools, got '{}'.",
tool_ref.identifier()
)
.into(),
);
return Err(e);
}
}
}
}
ToolChoice::Value(_) => {}
}
}
}
Ok(())
}
rust/src/server/src/routes/tokenize/types.rs
+8 -5
View File
@@ -7,7 +7,7 @@ use itertools::Itertools as _;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use validator::{Validate, ValidationErrors};
use vllm_chat::{ChatOptions, ChatRequest, ChatToolChoice, SamplingParams};
use vllm_chat::{ChatOptions, ChatRequest, ResolvedToolContext, SamplingParams};
use vllm_text::output::TextDecodeOptions;
use vllm_text::{Prompt, TextRequest};
@@ -97,6 +97,11 @@ impl TokenizeChatRequest {
&messages,
)?;
let tool_context =
ResolvedToolContext::new(&messages, convert_tools(self.tools)?, None, true).map_err(
|error| crate::error::chat_submit_error("failed to resolve request tools", error),
)?;
Ok(ChatRequest {
request_id,
messages,
@@ -108,9 +113,7 @@ impl TokenizeChatRequest {
response_format: None,
template_kwargs: self.chat_template_kwargs.unwrap_or_default(),
},
tools: convert_tools(self.tools)?,
tool_choice: ChatToolChoice::Auto,
parallel_tool_calls: true,
tool_context,
decode_options: TextDecodeOptions::default(),
intermediate: false,
priority: 0,
@@ -199,7 +202,7 @@ mod tests {
req.into_chat_request("tokenize-test".to_string()).expect("request is valid");
assert_eq!(
chat_request.tools,
chat_request.initial_tools(),
vec![ChatTool {
name: "get_weather".to_string(),
description: Some("Get weather".to_string()),