[Rust Frontend] Pass tool defer_loading through to chat templates (#60203)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
31e2443c90
commit
6bbad6acdf
@@ -409,6 +409,7 @@ pub(crate) fn convert_tools(tools: Option<Vec<Tool>>) -> Result<Vec<ChatTool>, A
|
||||
description: tool.function.description,
|
||||
parameters: tool.function.parameters,
|
||||
strict: tool.function.strict,
|
||||
defer_loading: tool.function.defer_loading.or(tool.defer_loading),
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
@@ -712,10 +713,50 @@ mod tests {
|
||||
description: None,
|
||||
parameters: serde_json::Value::Null,
|
||||
strict: None,
|
||||
defer_loading: None,
|
||||
}]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_chat_request_maps_tool_defer_loading() {
|
||||
let request: ChatCompletionRequest = serde_json::from_value(json!({
|
||||
"model": "Qwen/Qwen1.5-0.5B-Chat",
|
||||
"messages": [{"role": "user", "content": "hello"}],
|
||||
"tools": [
|
||||
{"function": {"name": "tool_level"}, "defer_loading": true},
|
||||
{"function": {"name": "function_level", "defer_loading": true}},
|
||||
{
|
||||
"function": {"name": "function_wins", "defer_loading": false},
|
||||
"defer_loading": true,
|
||||
},
|
||||
],
|
||||
}))
|
||||
.expect("parse defer_loading");
|
||||
|
||||
let prepared = prepare_chat_request(
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
)
|
||||
.expect("prepare defer_loading");
|
||||
|
||||
let defer_loading = prepared
|
||||
.chat_request
|
||||
.tools()
|
||||
.iter()
|
||||
.map(|tool| (tool.name.as_str(), tool.defer_loading))
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(
|
||||
defer_loading,
|
||||
[
|
||||
("tool_level", Some(true)),
|
||||
("function_level", Some(true)),
|
||||
("function_wins", Some(false)),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chat_http_request_rejects_empty_cache_salt() {
|
||||
let request: ChatCompletionRequest = serde_json::from_value(json!({
|
||||
@@ -1022,7 +1063,9 @@ mod tests {
|
||||
"properties": {"city": {"type": "string"}},
|
||||
}),
|
||||
strict: Some(true),
|
||||
defer_loading: None,
|
||||
},
|
||||
defer_loading: None,
|
||||
}]),
|
||||
name: None,
|
||||
}],
|
||||
@@ -1048,6 +1091,7 @@ mod tests {
|
||||
"properties": {"city": {"type": "string"}},
|
||||
}),
|
||||
strict: Some(true),
|
||||
defer_loading: None,
|
||||
}]),
|
||||
)]
|
||||
);
|
||||
@@ -1378,7 +1422,9 @@ mod tests {
|
||||
"properties": {"city": {"type": "string"}},
|
||||
}),
|
||||
strict: None,
|
||||
defer_loading: None,
|
||||
},
|
||||
defer_loading: None,
|
||||
}]),
|
||||
tool_choice: Some(ToolChoice::Value(ToolChoiceValue::None)),
|
||||
..base_request()
|
||||
@@ -1413,6 +1459,7 @@ mod tests {
|
||||
"properties": {"city": {"type": "string"}},
|
||||
}),
|
||||
strict: None,
|
||||
defer_loading: None,
|
||||
}]
|
||||
);
|
||||
assert_eq!(prepared.chat_request.tool_choice(), &ChatToolChoice::None);
|
||||
@@ -1431,7 +1478,9 @@ mod tests {
|
||||
"properties": {"city": {"type": "string"}},
|
||||
}),
|
||||
strict: None,
|
||||
defer_loading: None,
|
||||
},
|
||||
defer_loading: None,
|
||||
}]),
|
||||
tool_choice: Some(ToolChoice::Value(ToolChoiceValue::Required)),
|
||||
..base_request()
|
||||
@@ -1464,7 +1513,9 @@ mod tests {
|
||||
"properties": {"city": {"type": "string"}},
|
||||
}),
|
||||
strict: None,
|
||||
defer_loading: None,
|
||||
},
|
||||
defer_loading: None,
|
||||
}]),
|
||||
tool_choice: Some(ToolChoice::Function {
|
||||
tool_type: "function".to_string(),
|
||||
|
||||
@@ -195,7 +195,9 @@ mod tests {
|
||||
description: None,
|
||||
parameters: json!({}),
|
||||
strict: None,
|
||||
defer_loading: None,
|
||||
},
|
||||
defer_loading: None,
|
||||
}]),
|
||||
..base_request()
|
||||
};
|
||||
@@ -212,7 +214,9 @@ mod tests {
|
||||
description: None,
|
||||
parameters: json!({}),
|
||||
strict: None,
|
||||
defer_loading: None,
|
||||
},
|
||||
defer_loading: None,
|
||||
}]),
|
||||
name: None,
|
||||
}],
|
||||
@@ -234,7 +238,9 @@ mod tests {
|
||||
description: None,
|
||||
parameters: json!({}),
|
||||
strict: None,
|
||||
defer_loading: None,
|
||||
},
|
||||
defer_loading: None,
|
||||
}]),
|
||||
name: None,
|
||||
}],
|
||||
|
||||
@@ -271,6 +271,10 @@ pub struct Tool {
|
||||
#[serde(default = "default_function_tool_type")]
|
||||
pub tool_type: String,
|
||||
pub function: Function,
|
||||
/// Tool-level form of [`Function::defer_loading`]; the function-level value
|
||||
/// takes precedence.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub defer_loading: Option<bool>,
|
||||
}
|
||||
|
||||
fn default_function_tool_type() -> String {
|
||||
@@ -286,6 +290,9 @@ pub struct Function {
|
||||
pub parameters: Value,
|
||||
/// Whether to enable strict schema adherence (OpenAI structured outputs).
|
||||
pub strict: Option<bool>,
|
||||
/// Whether the schema is loaded on demand via tool search (vLLM extension,
|
||||
/// mirroring the OpenAI Responses and Anthropic Messages field).
|
||||
pub defer_loading: Option<bool>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
|
||||
@@ -215,6 +215,7 @@ mod tests {
|
||||
"properties": {"city": {"type": "string"}},
|
||||
}),
|
||||
strict: None,
|
||||
defer_loading: None,
|
||||
}]
|
||||
);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user