[Rust Frontend] Pass tool defer_loading through to chat templates (#60203)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-10-06 18:00:15 +08:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 31e2443c90
commit 6bbad6acdf
28 changed files with 124 additions and 1 deletions
@@ -409,6 +409,7 @@ pub(crate) fn convert_tools(tools: Option<Vec<Tool>>) -> Result<Vec<ChatTool>, A
description: tool.function.description,
parameters: tool.function.parameters,
strict: tool.function.strict,
defer_loading: tool.function.defer_loading.or(tool.defer_loading),
})
})
.collect()
@@ -712,10 +713,50 @@ mod tests {
description: None,
parameters: serde_json::Value::Null,
strict: None,
defer_loading: None,
}]
);
}
#[test]
fn prepare_chat_request_maps_tool_defer_loading() {
let request: ChatCompletionRequest = serde_json::from_value(json!({
"model": "Qwen/Qwen1.5-0.5B-Chat",
"messages": [{"role": "user", "content": "hello"}],
"tools": [
{"function": {"name": "tool_level"}, "defer_loading": true},
{"function": {"name": "function_level", "defer_loading": true}},
{
"function": {"name": "function_wins", "defer_loading": false},
"defer_loading": true,
},
],
}))
.expect("parse defer_loading");
let prepared = prepare_chat_request(
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
)
.expect("prepare defer_loading");
let defer_loading = prepared
.chat_request
.tools()
.iter()
.map(|tool| (tool.name.as_str(), tool.defer_loading))
.collect::<Vec<_>>();
assert_eq!(
defer_loading,
[
("tool_level", Some(true)),
("function_level", Some(true)),
("function_wins", Some(false)),
]
);
}
#[test]
fn chat_http_request_rejects_empty_cache_salt() {
let request: ChatCompletionRequest = serde_json::from_value(json!({
@@ -1022,7 +1063,9 @@ mod tests {
"properties": {"city": {"type": "string"}},
}),
strict: Some(true),
defer_loading: None,
},
defer_loading: None,
}]),
name: None,
}],
@@ -1048,6 +1091,7 @@ mod tests {
"properties": {"city": {"type": "string"}},
}),
strict: Some(true),
defer_loading: None,
}]),
)]
);
@@ -1378,7 +1422,9 @@ mod tests {
"properties": {"city": {"type": "string"}},
}),
strict: None,
defer_loading: None,
},
defer_loading: None,
}]),
tool_choice: Some(ToolChoice::Value(ToolChoiceValue::None)),
..base_request()
@@ -1413,6 +1459,7 @@ mod tests {
"properties": {"city": {"type": "string"}},
}),
strict: None,
defer_loading: None,
}]
);
assert_eq!(prepared.chat_request.tool_choice(), &ChatToolChoice::None);
@@ -1431,7 +1478,9 @@ mod tests {
"properties": {"city": {"type": "string"}},
}),
strict: None,
defer_loading: None,
},
defer_loading: None,
}]),
tool_choice: Some(ToolChoice::Value(ToolChoiceValue::Required)),
..base_request()
@@ -1464,7 +1513,9 @@ mod tests {
"properties": {"city": {"type": "string"}},
}),
strict: None,
defer_loading: None,
},
defer_loading: None,
}]),
tool_choice: Some(ToolChoice::Function {
tool_type: "function".to_string(),
@@ -195,7 +195,9 @@ mod tests {
description: None,
parameters: json!({}),
strict: None,
defer_loading: None,
},
defer_loading: None,
}]),
..base_request()
};
@@ -212,7 +214,9 @@ mod tests {
description: None,
parameters: json!({}),
strict: None,
defer_loading: None,
},
defer_loading: None,
}]),
name: None,
}],
@@ -234,7 +238,9 @@ mod tests {
description: None,
parameters: json!({}),
strict: None,
defer_loading: None,
},
defer_loading: None,
}]),
name: None,
}],
@@ -271,6 +271,10 @@ pub struct Tool {
#[serde(default = "default_function_tool_type")]
pub tool_type: String,
pub function: Function,
/// Tool-level form of [`Function::defer_loading`]; the function-level value
/// takes precedence.
#[serde(skip_serializing_if = "Option::is_none")]
pub defer_loading: Option<bool>,
}
fn default_function_tool_type() -> String {
@@ -286,6 +290,9 @@ pub struct Function {
pub parameters: Value,
/// Whether to enable strict schema adherence (OpenAI structured outputs).
pub strict: Option<bool>,
/// Whether the schema is loaded on demand via tool search (vLLM extension,
/// mirroring the OpenAI Responses and Anthropic Messages field).
pub defer_loading: Option<bool>,
}
#[derive(Debug, Clone, Deserialize, Serialize)]
@@ -215,6 +215,7 @@ mod tests {
"properties": {"city": {"type": "string"}},
}),
strict: None,
defer_loading: None,
}]
);
}