[Rust Frontend] Pass tool defer_loading through to chat templates (#60203)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-10-06 18:00:15 +08:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 31e2443c90
commit 6bbad6acdf
28 changed files with 124 additions and 1 deletions
+2
View File
@@ -311,6 +311,7 @@ mod tests {
"required": ["query"]
}),
strict,
defer_loading: None,
}];
let mut request = ChatRequest {
tool_context: ResolvedToolContext::new(&[], tools, Some(choice), true).unwrap(),
@@ -346,6 +347,7 @@ mod tests {
description: None,
parameters: serde_json::json!({"type": "object"}),
strict: Some(true),
defer_loading: None,
}];
let mut request = ChatRequest {
tool_context: ResolvedToolContext::new(
@@ -884,6 +884,7 @@ mod tests {
description: None,
parameters: serde_json::json!({ "type": "object" }),
strict: None,
defer_loading: None,
}];
let parser = Gemma4UnifiedParser::new(&tools, Arc::new(tokenizer)).unwrap();
let events = vec![
@@ -99,6 +99,7 @@ fn request_with_tools() -> ChatRequest {
"required": ["city"]
}),
strict: None,
defer_loading: None,
}];
ChatRequest {
tool_context: ResolvedToolContext::new(&[], tools, Some(ChatToolChoice::Auto), true)
@@ -152,6 +152,7 @@ fn request_level_tools_are_lowered_as_synthetic_leading_system_message() {
"required": ["query"]
}),
strict: None,
defer_loading: None,
}];
let mut request = ChatRequest {
request_id: "deepseek-v32-tools".to_string(),
@@ -183,6 +183,7 @@ fn tools_keep_historical_developer() {
description: None,
parameters: Value::Object(Default::default()),
strict: None,
defer_loading: None,
};
let request = thinking_request(vec![
ChatMessage::developer("old instruction", Some(vec![tool])),
+17 -1
View File
@@ -324,6 +324,8 @@ struct TemplateToolDefinition {
parameters: JsonValue,
#[serde(skip_serializing_if = "Option::is_none")]
strict: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
defer_loading: Option<bool>,
}
/// Convert chat messages into the JSON shape expected by Jinja chat templates.
@@ -604,6 +606,7 @@ fn to_template_tools(tools: &[ChatTool]) -> Vec<TemplateTool> {
description: tool.description.clone(),
parameters: tool.parameters.clone(),
strict: tool.strict,
defer_loading: tool.defer_loading,
},
})
.collect()
@@ -649,6 +652,7 @@ mod tests {
description: None,
parameters: serde_json::json!({"type": "object"}),
strict: None,
defer_loading: None,
}]),
),
ChatMessage::user("hello"),
@@ -1020,6 +1024,7 @@ mod tests {
"required": ["city"],
}),
strict: Some(true),
defer_loading: None,
}]),
)]);
@@ -1419,6 +1424,7 @@ mod tests {
"required": ["city"],
}),
strict: None,
defer_loading: None,
}];
request.tool_context = crate::request::ResolvedToolContext::new(
&request.messages,
@@ -1445,6 +1451,7 @@ mod tests {
description: Some("Get weather".to_string()),
parameters: serde_json::json!({"type": "object"}),
strict: None,
defer_loading: None,
}];
request.tool_context = crate::request::ResolvedToolContext::new(
&request.messages,
@@ -1472,12 +1479,21 @@ mod tests {
description: None,
parameters: Value::Null,
strict: None,
defer_loading: None,
},
ChatTool {
name: "with_strict".to_string(),
description: Some("description".to_string()),
parameters: serde_json::json!({"type": "object"}),
strict: Some(false),
defer_loading: None,
},
ChatTool {
name: "deferred".to_string(),
description: None,
parameters: Value::Null,
strict: None,
defer_loading: Some(true),
},
];
request.tool_context = crate::request::ResolvedToolContext::new(
@@ -1498,7 +1514,7 @@ mod tests {
assert_eq!(
rendered,
"name=\"without_strict\"|description=null|parameters=null|;name=\"with_strict\"|description=\"description\"|parameters={\"type\": \"object\"}|strict=false|;"
"name=\"without_strict\"|description=null|parameters=null|;name=\"with_strict\"|description=\"description\"|parameters={\"type\": \"object\"}|strict=false|;name=\"deferred\"|description=null|parameters=null|defer_loading=true|;"
);
}
@@ -178,6 +178,7 @@ fn tool_declare_omits_absent_fields_and_preserves_parameter_nulls() {
}
}),
strict: None,
defer_loading: None,
}];
request.tool_context =
crate::request::ResolvedToolContext::new(&request.messages, tools, None, true)
@@ -200,6 +201,7 @@ fn tool_declare_preserves_present_optional_fields() {
description: Some("Look up a record".to_string()),
parameters: json!({"type": "object"}),
strict: Some(false),
defer_loading: None,
}];
request.tool_context =
crate::request::ResolvedToolContext::new(&request.messages, tools, None, true)
+1
View File
@@ -280,6 +280,7 @@ fn to_chat_tools(tools: &[FixtureTool]) -> Vec<ChatTool> {
description: tool.function.description.clone(),
parameters: tool.function.parameters.clone(),
strict: tool.function.strict,
defer_loading: None,
})
.collect()
}
+2
View File
@@ -801,6 +801,7 @@ mod tests {
"properties": {"city": {"type": "string"}},
}),
strict: Some(true),
defer_loading: None,
}]),
);
@@ -815,6 +816,7 @@ mod tests {
description: None,
parameters: json!({"type": "object"}),
strict: None,
defer_loading: None,
}
}
+1
View File
@@ -297,6 +297,7 @@ fn sample_tool_request(request_id: &str) -> ChatRequest {
"required": ["city"],
}),
strict: None,
defer_loading: None,
}];
request.tool_context = vllm_chat::ResolvedToolContext::new(
&request.messages,
+2
View File
@@ -1289,6 +1289,7 @@ fn test_tools() -> Vec<ChatTool> {
"required": ["location"]
}),
strict: None,
defer_loading: None,
},
ChatTool {
name: "add".to_string(),
@@ -1306,6 +1307,7 @@ fn test_tools() -> Vec<ChatTool> {
"required": ["y", "x", "items"]
}),
strict: None,
defer_loading: None,
},
]
}
+1
View File
@@ -255,6 +255,7 @@ mod tests {
"required": ["query"]
}),
strict,
defer_loading: None,
}
}
@@ -28,6 +28,7 @@ fn tools() -> Vec<Tool> {
}
}),
strict: None,
defer_loading: None,
}]
}
+1
View File
@@ -65,6 +65,7 @@ mod tests {
name: "convert".into(),
description: None,
strict: None,
defer_loading: None,
parameters: json!({"type": "object", "properties": {
"flag": {"type": "boolean"}, "empty": {"type": ["string", "null"]},
"text": {"type": "string"}, "literal": {"type": "string"}, "payload": {"type": "object"}
+1
View File
@@ -459,6 +459,7 @@ mod tests {
}
}),
strict: None,
defer_loading: None,
});
tools
}
+3
View File
@@ -52,6 +52,9 @@ pub struct Tool {
pub description: Option<String>,
pub parameters: Value,
pub strict: Option<bool>,
/// Whether the tool's schema is loaded on demand (tool search). Only chat
/// templates consume it; the tool stays callable for parsing and grammars.
pub defer_loading: Option<bool>,
}
/// One tool-call update emitted while parsing assistant text.
+1
View File
@@ -502,6 +502,7 @@ mod tests {
description: None,
parameters,
strict: None,
defer_loading: None,
}
}
+5
View File
@@ -24,6 +24,7 @@ pub fn test_tools() -> Vec<Tool> {
}
}),
strict: None,
defer_loading: None,
},
Tool {
name: "add".to_string(),
@@ -36,6 +37,7 @@ pub fn test_tools() -> Vec<Tool> {
}
}),
strict: None,
defer_loading: None,
},
Tool {
name: "convert".to_string(),
@@ -51,6 +53,7 @@ pub fn test_tools() -> Vec<Tool> {
}
}),
strict: None,
defer_loading: None,
},
Tool {
name: "calculate_area".to_string(),
@@ -64,6 +67,7 @@ pub fn test_tools() -> Vec<Tool> {
}
}),
strict: None,
defer_loading: None,
},
Tool {
name: "update_record".to_string(),
@@ -80,6 +84,7 @@ pub fn test_tools() -> Vec<Tool> {
}
}),
strict: None,
defer_loading: None,
},
]
}
+1
View File
@@ -197,6 +197,7 @@ mod tests {
},
}),
strict: None,
defer_loading: None,
}]
}
+8
View File
@@ -617,48 +617,56 @@ mod tests {
description: None,
parameters: json!({ "type": "object" }),
strict: None,
defer_loading: None,
},
Tool {
name: "get_time".to_string(),
description: None,
parameters: json!({ "type": "object" }),
strict: None,
defer_loading: None,
},
Tool {
name: "write_file".to_string(),
description: None,
parameters: json!({ "type": "object" }),
strict: None,
defer_loading: None,
},
Tool {
name: "Edit".to_string(),
description: None,
parameters: json!({ "type": "object" }),
strict: None,
defer_loading: None,
},
Tool {
name: "search".to_string(),
description: None,
parameters: json!({ "type": "object" }),
strict: None,
defer_loading: None,
},
Tool {
name: "set".to_string(),
description: None,
parameters: json!({ "type": "object" }),
strict: None,
defer_loading: None,
},
Tool {
name: "get_status".to_string(),
description: None,
parameters: json!({ "type": "object" }),
strict: None,
defer_loading: None,
},
Tool {
name: "todowrite".to_string(),
description: None,
parameters: json!({ "type": "object" }),
strict: None,
defer_loading: None,
},
]
}
+2
View File
@@ -686,6 +686,7 @@ fn tools_convert_string_arguments() {
},
}),
strict: None,
defer_loading: None,
}];
let text = "<tool_call>\n<function=set_alarm>\n<parameter=hour>\n7\n</parameter>\n<parameter=enabled>\ntrue\n</parameter>\n<parameter=label>\nwake up\n</parameter>\n<parameter=weekday>1</parameter><parameter=weekday>2</parameter>\n</function>\n</tool_call>";
assert_eq!(
@@ -888,6 +889,7 @@ fn differential_against_transformers() {
description: None,
parameters: tool["function"]["parameters"].clone(),
strict: None,
defer_loading: None,
})
.collect();
let expected = match DIVERGENCES.iter().find(|(divergent, _)| *divergent == text) {
+1
View File
@@ -114,6 +114,7 @@ mod tests {
"properties": { "city": { "type": "string" } },
}),
strict: None,
defer_loading: None,
}]
}
+2
View File
@@ -114,12 +114,14 @@ mod tests {
"properties": { "city": { "type": "string" } },
}),
strict: None,
defer_loading: None,
},
Tool {
name: "get_current_date".to_string(),
description: None,
parameters: json!({ "type": "object", "properties": {} }),
strict: None,
defer_loading: None,
},
]
}
+1
View File
@@ -557,6 +557,7 @@ mod tests {
},
}),
strict: None,
defer_loading: None,
}]
}
@@ -409,6 +409,7 @@ pub(crate) fn convert_tools(tools: Option<Vec<Tool>>) -> Result<Vec<ChatTool>, A
description: tool.function.description,
parameters: tool.function.parameters,
strict: tool.function.strict,
defer_loading: tool.function.defer_loading.or(tool.defer_loading),
})
})
.collect()
@@ -712,10 +713,50 @@ mod tests {
description: None,
parameters: serde_json::Value::Null,
strict: None,
defer_loading: None,
}]
);
}
#[test]
fn prepare_chat_request_maps_tool_defer_loading() {
let request: ChatCompletionRequest = serde_json::from_value(json!({
"model": "Qwen/Qwen1.5-0.5B-Chat",
"messages": [{"role": "user", "content": "hello"}],
"tools": [
{"function": {"name": "tool_level"}, "defer_loading": true},
{"function": {"name": "function_level", "defer_loading": true}},
{
"function": {"name": "function_wins", "defer_loading": false},
"defer_loading": true,
},
],
}))
.expect("parse defer_loading");
let prepared = prepare_chat_request(
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
)
.expect("prepare defer_loading");
let defer_loading = prepared
.chat_request
.tools()
.iter()
.map(|tool| (tool.name.as_str(), tool.defer_loading))
.collect::<Vec<_>>();
assert_eq!(
defer_loading,
[
("tool_level", Some(true)),
("function_level", Some(true)),
("function_wins", Some(false)),
]
);
}
#[test]
fn chat_http_request_rejects_empty_cache_salt() {
let request: ChatCompletionRequest = serde_json::from_value(json!({
@@ -1022,7 +1063,9 @@ mod tests {
"properties": {"city": {"type": "string"}},
}),
strict: Some(true),
defer_loading: None,
},
defer_loading: None,
}]),
name: None,
}],
@@ -1048,6 +1091,7 @@ mod tests {
"properties": {"city": {"type": "string"}},
}),
strict: Some(true),
defer_loading: None,
}]),
)]
);
@@ -1378,7 +1422,9 @@ mod tests {
"properties": {"city": {"type": "string"}},
}),
strict: None,
defer_loading: None,
},
defer_loading: None,
}]),
tool_choice: Some(ToolChoice::Value(ToolChoiceValue::None)),
..base_request()
@@ -1413,6 +1459,7 @@ mod tests {
"properties": {"city": {"type": "string"}},
}),
strict: None,
defer_loading: None,
}]
);
assert_eq!(prepared.chat_request.tool_choice(), &ChatToolChoice::None);
@@ -1431,7 +1478,9 @@ mod tests {
"properties": {"city": {"type": "string"}},
}),
strict: None,
defer_loading: None,
},
defer_loading: None,
}]),
tool_choice: Some(ToolChoice::Value(ToolChoiceValue::Required)),
..base_request()
@@ -1464,7 +1513,9 @@ mod tests {
"properties": {"city": {"type": "string"}},
}),
strict: None,
defer_loading: None,
},
defer_loading: None,
}]),
tool_choice: Some(ToolChoice::Function {
tool_type: "function".to_string(),
@@ -195,7 +195,9 @@ mod tests {
description: None,
parameters: json!({}),
strict: None,
defer_loading: None,
},
defer_loading: None,
}]),
..base_request()
};
@@ -212,7 +214,9 @@ mod tests {
description: None,
parameters: json!({}),
strict: None,
defer_loading: None,
},
defer_loading: None,
}]),
name: None,
}],
@@ -234,7 +238,9 @@ mod tests {
description: None,
parameters: json!({}),
strict: None,
defer_loading: None,
},
defer_loading: None,
}]),
name: None,
}],
@@ -271,6 +271,10 @@ pub struct Tool {
#[serde(default = "default_function_tool_type")]
pub tool_type: String,
pub function: Function,
/// Tool-level form of [`Function::defer_loading`]; the function-level value
/// takes precedence.
#[serde(skip_serializing_if = "Option::is_none")]
pub defer_loading: Option<bool>,
}
fn default_function_tool_type() -> String {
@@ -286,6 +290,9 @@ pub struct Function {
pub parameters: Value,
/// Whether to enable strict schema adherence (OpenAI structured outputs).
pub strict: Option<bool>,
/// Whether the schema is loaded on demand via tool search (vLLM extension,
/// mirroring the OpenAI Responses and Anthropic Messages field).
pub defer_loading: Option<bool>,
}
#[derive(Debug, Clone, Deserialize, Serialize)]
@@ -215,6 +215,7 @@ mod tests {
"properties": {"city": {"type": "string"}},
}),
strict: None,
defer_loading: None,
}]
);
}