From 6bbad6acdffd58b63037829f9fe27b92c6650071 Mon Sep 17 00:00:00 2001 From: Bugen Zhao Date: Tue, 6 Oct 2026 18:00:15 +0800 Subject: [PATCH] [Rust Frontend] Pass tool defer_loading through to chat templates (#60203) Co-authored-by: Claude Opus 5.5 Signed-off-by: Bugen Zhao --- rust/src/chat/src/output/default/mod.rs | 2 + rust/src/chat/src/output/default/unified.rs | 1 + rust/src/chat/src/output/harmony/tests.rs | 1 + .../chat/src/renderer/deepseek_v32/tests.rs | 1 + .../chat/src/renderer/deepseek_v4/tests.rs | 1 + rust/src/chat/src/renderer/hf/mod.rs | 18 ++++++- rust/src/chat/src/renderer/kimi_k3/tests.rs | 2 + rust/src/chat/src/renderer/test_utils.rs | 1 + rust/src/chat/src/request.rs | 2 + rust/src/chat/tests/chat.rs | 1 + rust/src/chat/tests/roundtrip.rs | 2 + rust/src/parser/src/output_grammar.rs | 1 + .../tool/deepseek_dsml/deepseek_v41/tests.rs | 1 + rust/src/parser/src/tool/mimo.rs | 1 + rust/src/parser/src/tool/minimax_m3.rs | 1 + rust/src/parser/src/tool/mod.rs | 3 ++ rust/src/parser/src/tool/parameters.rs | 1 + rust/src/parser/src/tool/test_utils.rs | 5 ++ rust/src/parser/src/unified/combined.rs | 1 + rust/src/parser/src/unified/gemma4.rs | 8 +++ rust/src/parser/src/unified/hf/tests.rs | 2 + rust/src/parser/src/unified/hy/hy_v3.rs | 1 + rust/src/parser/src/unified/hy/hy_v4.rs | 2 + rust/src/parser/src/unified/inkling.rs | 1 + .../routes/openai/chat_completions/convert.rs | 51 +++++++++++++++++++ .../openai/chat_completions/validate.rs | 6 +++ .../server/src/routes/openai/utils/types.rs | 7 +++ rust/src/server/src/routes/tokenize/types.rs | 1 + 28 files changed, 124 insertions(+), 1 deletion(-) diff --git a/rust/src/chat/src/output/default/mod.rs b/rust/src/chat/src/output/default/mod.rs index 3f75aed63d..b31c5e9e4c 100644 --- a/rust/src/chat/src/output/default/mod.rs +++ b/rust/src/chat/src/output/default/mod.rs @@ -311,6 +311,7 @@ mod tests { "required": ["query"] }), strict, + defer_loading: None, }]; let mut request = ChatRequest { tool_context: ResolvedToolContext::new(&[], tools, Some(choice), true).unwrap(), @@ -346,6 +347,7 @@ mod tests { description: None, parameters: serde_json::json!({"type": "object"}), strict: Some(true), + defer_loading: None, }]; let mut request = ChatRequest { tool_context: ResolvedToolContext::new( diff --git a/rust/src/chat/src/output/default/unified.rs b/rust/src/chat/src/output/default/unified.rs index b4a2dcbe5d..9f872d9794 100644 --- a/rust/src/chat/src/output/default/unified.rs +++ b/rust/src/chat/src/output/default/unified.rs @@ -884,6 +884,7 @@ mod tests { description: None, parameters: serde_json::json!({ "type": "object" }), strict: None, + defer_loading: None, }]; let parser = Gemma4UnifiedParser::new(&tools, Arc::new(tokenizer)).unwrap(); let events = vec![ diff --git a/rust/src/chat/src/output/harmony/tests.rs b/rust/src/chat/src/output/harmony/tests.rs index 035aef3cd4..5b89e53bb9 100644 --- a/rust/src/chat/src/output/harmony/tests.rs +++ b/rust/src/chat/src/output/harmony/tests.rs @@ -99,6 +99,7 @@ fn request_with_tools() -> ChatRequest { "required": ["city"] }), strict: None, + defer_loading: None, }]; ChatRequest { tool_context: ResolvedToolContext::new(&[], tools, Some(ChatToolChoice::Auto), true) diff --git a/rust/src/chat/src/renderer/deepseek_v32/tests.rs b/rust/src/chat/src/renderer/deepseek_v32/tests.rs index dd9ec13ddc..804e4c2bc9 100644 --- a/rust/src/chat/src/renderer/deepseek_v32/tests.rs +++ b/rust/src/chat/src/renderer/deepseek_v32/tests.rs @@ -152,6 +152,7 @@ fn request_level_tools_are_lowered_as_synthetic_leading_system_message() { "required": ["query"] }), strict: None, + defer_loading: None, }]; let mut request = ChatRequest { request_id: "deepseek-v32-tools".to_string(), diff --git a/rust/src/chat/src/renderer/deepseek_v4/tests.rs b/rust/src/chat/src/renderer/deepseek_v4/tests.rs index 8702e8d73c..78e9044a3b 100644 --- a/rust/src/chat/src/renderer/deepseek_v4/tests.rs +++ b/rust/src/chat/src/renderer/deepseek_v4/tests.rs @@ -183,6 +183,7 @@ fn tools_keep_historical_developer() { description: None, parameters: Value::Object(Default::default()), strict: None, + defer_loading: None, }; let request = thinking_request(vec![ ChatMessage::developer("old instruction", Some(vec![tool])), diff --git a/rust/src/chat/src/renderer/hf/mod.rs b/rust/src/chat/src/renderer/hf/mod.rs index 5cb0262191..06b58e27ee 100644 --- a/rust/src/chat/src/renderer/hf/mod.rs +++ b/rust/src/chat/src/renderer/hf/mod.rs @@ -324,6 +324,8 @@ struct TemplateToolDefinition { parameters: JsonValue, #[serde(skip_serializing_if = "Option::is_none")] strict: Option, + #[serde(skip_serializing_if = "Option::is_none")] + defer_loading: Option, } /// Convert chat messages into the JSON shape expected by Jinja chat templates. @@ -604,6 +606,7 @@ fn to_template_tools(tools: &[ChatTool]) -> Vec { description: tool.description.clone(), parameters: tool.parameters.clone(), strict: tool.strict, + defer_loading: tool.defer_loading, }, }) .collect() @@ -649,6 +652,7 @@ mod tests { description: None, parameters: serde_json::json!({"type": "object"}), strict: None, + defer_loading: None, }]), ), ChatMessage::user("hello"), @@ -1020,6 +1024,7 @@ mod tests { "required": ["city"], }), strict: Some(true), + defer_loading: None, }]), )]); @@ -1419,6 +1424,7 @@ mod tests { "required": ["city"], }), strict: None, + defer_loading: None, }]; request.tool_context = crate::request::ResolvedToolContext::new( &request.messages, @@ -1445,6 +1451,7 @@ mod tests { description: Some("Get weather".to_string()), parameters: serde_json::json!({"type": "object"}), strict: None, + defer_loading: None, }]; request.tool_context = crate::request::ResolvedToolContext::new( &request.messages, @@ -1472,12 +1479,21 @@ mod tests { description: None, parameters: Value::Null, strict: None, + defer_loading: None, }, ChatTool { name: "with_strict".to_string(), description: Some("description".to_string()), parameters: serde_json::json!({"type": "object"}), strict: Some(false), + defer_loading: None, + }, + ChatTool { + name: "deferred".to_string(), + description: None, + parameters: Value::Null, + strict: None, + defer_loading: Some(true), }, ]; request.tool_context = crate::request::ResolvedToolContext::new( @@ -1498,7 +1514,7 @@ mod tests { assert_eq!( rendered, - "name=\"without_strict\"|description=null|parameters=null|;name=\"with_strict\"|description=\"description\"|parameters={\"type\": \"object\"}|strict=false|;" + "name=\"without_strict\"|description=null|parameters=null|;name=\"with_strict\"|description=\"description\"|parameters={\"type\": \"object\"}|strict=false|;name=\"deferred\"|description=null|parameters=null|defer_loading=true|;" ); } diff --git a/rust/src/chat/src/renderer/kimi_k3/tests.rs b/rust/src/chat/src/renderer/kimi_k3/tests.rs index 5aadb3d403..29a6ecb473 100644 --- a/rust/src/chat/src/renderer/kimi_k3/tests.rs +++ b/rust/src/chat/src/renderer/kimi_k3/tests.rs @@ -178,6 +178,7 @@ fn tool_declare_omits_absent_fields_and_preserves_parameter_nulls() { } }), strict: None, + defer_loading: None, }]; request.tool_context = crate::request::ResolvedToolContext::new(&request.messages, tools, None, true) @@ -200,6 +201,7 @@ fn tool_declare_preserves_present_optional_fields() { description: Some("Look up a record".to_string()), parameters: json!({"type": "object"}), strict: Some(false), + defer_loading: None, }]; request.tool_context = crate::request::ResolvedToolContext::new(&request.messages, tools, None, true) diff --git a/rust/src/chat/src/renderer/test_utils.rs b/rust/src/chat/src/renderer/test_utils.rs index 2c59ed7a22..e42dba76a5 100644 --- a/rust/src/chat/src/renderer/test_utils.rs +++ b/rust/src/chat/src/renderer/test_utils.rs @@ -280,6 +280,7 @@ fn to_chat_tools(tools: &[FixtureTool]) -> Vec { description: tool.function.description.clone(), parameters: tool.function.parameters.clone(), strict: tool.function.strict, + defer_loading: None, }) .collect() } diff --git a/rust/src/chat/src/request.rs b/rust/src/chat/src/request.rs index f7ddd9b128..b52e9fbc0b 100644 --- a/rust/src/chat/src/request.rs +++ b/rust/src/chat/src/request.rs @@ -801,6 +801,7 @@ mod tests { "properties": {"city": {"type": "string"}}, }), strict: Some(true), + defer_loading: None, }]), ); @@ -815,6 +816,7 @@ mod tests { description: None, parameters: json!({"type": "object"}), strict: None, + defer_loading: None, } } diff --git a/rust/src/chat/tests/chat.rs b/rust/src/chat/tests/chat.rs index ab366b8827..b920b5f0ed 100644 --- a/rust/src/chat/tests/chat.rs +++ b/rust/src/chat/tests/chat.rs @@ -297,6 +297,7 @@ fn sample_tool_request(request_id: &str) -> ChatRequest { "required": ["city"], }), strict: None, + defer_loading: None, }]; request.tool_context = vllm_chat::ResolvedToolContext::new( &request.messages, diff --git a/rust/src/chat/tests/roundtrip.rs b/rust/src/chat/tests/roundtrip.rs index b1a6505cf8..079ce3e1a7 100644 --- a/rust/src/chat/tests/roundtrip.rs +++ b/rust/src/chat/tests/roundtrip.rs @@ -1289,6 +1289,7 @@ fn test_tools() -> Vec { "required": ["location"] }), strict: None, + defer_loading: None, }, ChatTool { name: "add".to_string(), @@ -1306,6 +1307,7 @@ fn test_tools() -> Vec { "required": ["y", "x", "items"] }), strict: None, + defer_loading: None, }, ] } diff --git a/rust/src/parser/src/output_grammar.rs b/rust/src/parser/src/output_grammar.rs index 9058b4a58b..53c52538ee 100644 --- a/rust/src/parser/src/output_grammar.rs +++ b/rust/src/parser/src/output_grammar.rs @@ -255,6 +255,7 @@ mod tests { "required": ["query"] }), strict, + defer_loading: None, } } diff --git a/rust/src/parser/src/tool/deepseek_dsml/deepseek_v41/tests.rs b/rust/src/parser/src/tool/deepseek_dsml/deepseek_v41/tests.rs index 208d88ea33..58043cc95c 100644 --- a/rust/src/parser/src/tool/deepseek_dsml/deepseek_v41/tests.rs +++ b/rust/src/parser/src/tool/deepseek_dsml/deepseek_v41/tests.rs @@ -28,6 +28,7 @@ fn tools() -> Vec { } }), strict: None, + defer_loading: None, }] } diff --git a/rust/src/parser/src/tool/mimo.rs b/rust/src/parser/src/tool/mimo.rs index f6491e9caa..f7ba06fb06 100644 --- a/rust/src/parser/src/tool/mimo.rs +++ b/rust/src/parser/src/tool/mimo.rs @@ -65,6 +65,7 @@ mod tests { name: "convert".into(), description: None, strict: None, + defer_loading: None, parameters: json!({"type": "object", "properties": { "flag": {"type": "boolean"}, "empty": {"type": ["string", "null"]}, "text": {"type": "string"}, "literal": {"type": "string"}, "payload": {"type": "object"} diff --git a/rust/src/parser/src/tool/minimax_m3.rs b/rust/src/parser/src/tool/minimax_m3.rs index b0643dd1a1..b6e179917e 100644 --- a/rust/src/parser/src/tool/minimax_m3.rs +++ b/rust/src/parser/src/tool/minimax_m3.rs @@ -459,6 +459,7 @@ mod tests { } }), strict: None, + defer_loading: None, }); tools } diff --git a/rust/src/parser/src/tool/mod.rs b/rust/src/parser/src/tool/mod.rs index 7789752626..e4afbda3e9 100644 --- a/rust/src/parser/src/tool/mod.rs +++ b/rust/src/parser/src/tool/mod.rs @@ -52,6 +52,9 @@ pub struct Tool { pub description: Option, pub parameters: Value, pub strict: Option, + /// Whether the tool's schema is loaded on demand (tool search). Only chat + /// templates consume it; the tool stays callable for parsing and grammars. + pub defer_loading: Option, } /// One tool-call update emitted while parsing assistant text. diff --git a/rust/src/parser/src/tool/parameters.rs b/rust/src/parser/src/tool/parameters.rs index 19d700830b..ef59ce0875 100644 --- a/rust/src/parser/src/tool/parameters.rs +++ b/rust/src/parser/src/tool/parameters.rs @@ -502,6 +502,7 @@ mod tests { description: None, parameters, strict: None, + defer_loading: None, } } diff --git a/rust/src/parser/src/tool/test_utils.rs b/rust/src/parser/src/tool/test_utils.rs index ef3143183f..ec45a31ec1 100644 --- a/rust/src/parser/src/tool/test_utils.rs +++ b/rust/src/parser/src/tool/test_utils.rs @@ -24,6 +24,7 @@ pub fn test_tools() -> Vec { } }), strict: None, + defer_loading: None, }, Tool { name: "add".to_string(), @@ -36,6 +37,7 @@ pub fn test_tools() -> Vec { } }), strict: None, + defer_loading: None, }, Tool { name: "convert".to_string(), @@ -51,6 +53,7 @@ pub fn test_tools() -> Vec { } }), strict: None, + defer_loading: None, }, Tool { name: "calculate_area".to_string(), @@ -64,6 +67,7 @@ pub fn test_tools() -> Vec { } }), strict: None, + defer_loading: None, }, Tool { name: "update_record".to_string(), @@ -80,6 +84,7 @@ pub fn test_tools() -> Vec { } }), strict: None, + defer_loading: None, }, ] } diff --git a/rust/src/parser/src/unified/combined.rs b/rust/src/parser/src/unified/combined.rs index 6f6849ec95..ba44e7fb96 100644 --- a/rust/src/parser/src/unified/combined.rs +++ b/rust/src/parser/src/unified/combined.rs @@ -197,6 +197,7 @@ mod tests { }, }), strict: None, + defer_loading: None, }] } diff --git a/rust/src/parser/src/unified/gemma4.rs b/rust/src/parser/src/unified/gemma4.rs index 2f90555b26..1c65a7171c 100644 --- a/rust/src/parser/src/unified/gemma4.rs +++ b/rust/src/parser/src/unified/gemma4.rs @@ -617,48 +617,56 @@ mod tests { description: None, parameters: json!({ "type": "object" }), strict: None, + defer_loading: None, }, Tool { name: "get_time".to_string(), description: None, parameters: json!({ "type": "object" }), strict: None, + defer_loading: None, }, Tool { name: "write_file".to_string(), description: None, parameters: json!({ "type": "object" }), strict: None, + defer_loading: None, }, Tool { name: "Edit".to_string(), description: None, parameters: json!({ "type": "object" }), strict: None, + defer_loading: None, }, Tool { name: "search".to_string(), description: None, parameters: json!({ "type": "object" }), strict: None, + defer_loading: None, }, Tool { name: "set".to_string(), description: None, parameters: json!({ "type": "object" }), strict: None, + defer_loading: None, }, Tool { name: "get_status".to_string(), description: None, parameters: json!({ "type": "object" }), strict: None, + defer_loading: None, }, Tool { name: "todowrite".to_string(), description: None, parameters: json!({ "type": "object" }), strict: None, + defer_loading: None, }, ] } diff --git a/rust/src/parser/src/unified/hf/tests.rs b/rust/src/parser/src/unified/hf/tests.rs index 7e27a42805..ed2efa6f39 100644 --- a/rust/src/parser/src/unified/hf/tests.rs +++ b/rust/src/parser/src/unified/hf/tests.rs @@ -686,6 +686,7 @@ fn tools_convert_string_arguments() { }, }), strict: None, + defer_loading: None, }]; let text = "\n\n\n7\n\n\ntrue\n\n\nwake up\n\n12\n\n"; assert_eq!( @@ -888,6 +889,7 @@ fn differential_against_transformers() { description: None, parameters: tool["function"]["parameters"].clone(), strict: None, + defer_loading: None, }) .collect(); let expected = match DIVERGENCES.iter().find(|(divergent, _)| *divergent == text) { diff --git a/rust/src/parser/src/unified/hy/hy_v3.rs b/rust/src/parser/src/unified/hy/hy_v3.rs index 30e021b5bc..3599ce2cb9 100644 --- a/rust/src/parser/src/unified/hy/hy_v3.rs +++ b/rust/src/parser/src/unified/hy/hy_v3.rs @@ -114,6 +114,7 @@ mod tests { "properties": { "city": { "type": "string" } }, }), strict: None, + defer_loading: None, }] } diff --git a/rust/src/parser/src/unified/hy/hy_v4.rs b/rust/src/parser/src/unified/hy/hy_v4.rs index 777688bcaa..a5d81e0f2a 100644 --- a/rust/src/parser/src/unified/hy/hy_v4.rs +++ b/rust/src/parser/src/unified/hy/hy_v4.rs @@ -114,12 +114,14 @@ mod tests { "properties": { "city": { "type": "string" } }, }), strict: None, + defer_loading: None, }, Tool { name: "get_current_date".to_string(), description: None, parameters: json!({ "type": "object", "properties": {} }), strict: None, + defer_loading: None, }, ] } diff --git a/rust/src/parser/src/unified/inkling.rs b/rust/src/parser/src/unified/inkling.rs index 4372cafb24..7d819b0640 100644 --- a/rust/src/parser/src/unified/inkling.rs +++ b/rust/src/parser/src/unified/inkling.rs @@ -557,6 +557,7 @@ mod tests { }, }), strict: None, + defer_loading: None, }] } diff --git a/rust/src/server/src/routes/openai/chat_completions/convert.rs b/rust/src/server/src/routes/openai/chat_completions/convert.rs index 418ab23ce8..998ca4c939 100644 --- a/rust/src/server/src/routes/openai/chat_completions/convert.rs +++ b/rust/src/server/src/routes/openai/chat_completions/convert.rs @@ -409,6 +409,7 @@ pub(crate) fn convert_tools(tools: Option>) -> Result, A description: tool.function.description, parameters: tool.function.parameters, strict: tool.function.strict, + defer_loading: tool.function.defer_loading.or(tool.defer_loading), }) }) .collect() @@ -712,10 +713,50 @@ mod tests { description: None, parameters: serde_json::Value::Null, strict: None, + defer_loading: None, }] ); } + #[test] + fn prepare_chat_request_maps_tool_defer_loading() { + let request: ChatCompletionRequest = serde_json::from_value(json!({ + "model": "Qwen/Qwen1.5-0.5B-Chat", + "messages": [{"role": "user", "content": "hello"}], + "tools": [ + {"function": {"name": "tool_level"}, "defer_loading": true}, + {"function": {"name": "function_level", "defer_loading": true}}, + { + "function": {"name": "function_wins", "defer_loading": false}, + "defer_loading": true, + }, + ], + })) + .expect("parse defer_loading"); + + let prepared = prepare_chat_request( + request, + &served(&["Qwen/Qwen1.5-0.5B-Chat"]), + ResolvedRequestContext::default(), + ) + .expect("prepare defer_loading"); + + let defer_loading = prepared + .chat_request + .tools() + .iter() + .map(|tool| (tool.name.as_str(), tool.defer_loading)) + .collect::>(); + assert_eq!( + defer_loading, + [ + ("tool_level", Some(true)), + ("function_level", Some(true)), + ("function_wins", Some(false)), + ] + ); + } + #[test] fn chat_http_request_rejects_empty_cache_salt() { let request: ChatCompletionRequest = serde_json::from_value(json!({ @@ -1022,7 +1063,9 @@ mod tests { "properties": {"city": {"type": "string"}}, }), strict: Some(true), + defer_loading: None, }, + defer_loading: None, }]), name: None, }], @@ -1048,6 +1091,7 @@ mod tests { "properties": {"city": {"type": "string"}}, }), strict: Some(true), + defer_loading: None, }]), )] ); @@ -1378,7 +1422,9 @@ mod tests { "properties": {"city": {"type": "string"}}, }), strict: None, + defer_loading: None, }, + defer_loading: None, }]), tool_choice: Some(ToolChoice::Value(ToolChoiceValue::None)), ..base_request() @@ -1413,6 +1459,7 @@ mod tests { "properties": {"city": {"type": "string"}}, }), strict: None, + defer_loading: None, }] ); assert_eq!(prepared.chat_request.tool_choice(), &ChatToolChoice::None); @@ -1431,7 +1478,9 @@ mod tests { "properties": {"city": {"type": "string"}}, }), strict: None, + defer_loading: None, }, + defer_loading: None, }]), tool_choice: Some(ToolChoice::Value(ToolChoiceValue::Required)), ..base_request() @@ -1464,7 +1513,9 @@ mod tests { "properties": {"city": {"type": "string"}}, }), strict: None, + defer_loading: None, }, + defer_loading: None, }]), tool_choice: Some(ToolChoice::Function { tool_type: "function".to_string(), diff --git a/rust/src/server/src/routes/openai/chat_completions/validate.rs b/rust/src/server/src/routes/openai/chat_completions/validate.rs index 1bfc1d9a09..579b815ba1 100644 --- a/rust/src/server/src/routes/openai/chat_completions/validate.rs +++ b/rust/src/server/src/routes/openai/chat_completions/validate.rs @@ -195,7 +195,9 @@ mod tests { description: None, parameters: json!({}), strict: None, + defer_loading: None, }, + defer_loading: None, }]), ..base_request() }; @@ -212,7 +214,9 @@ mod tests { description: None, parameters: json!({}), strict: None, + defer_loading: None, }, + defer_loading: None, }]), name: None, }], @@ -234,7 +238,9 @@ mod tests { description: None, parameters: json!({}), strict: None, + defer_loading: None, }, + defer_loading: None, }]), name: None, }], diff --git a/rust/src/server/src/routes/openai/utils/types.rs b/rust/src/server/src/routes/openai/utils/types.rs index 85a891341c..e2187bae67 100644 --- a/rust/src/server/src/routes/openai/utils/types.rs +++ b/rust/src/server/src/routes/openai/utils/types.rs @@ -271,6 +271,10 @@ pub struct Tool { #[serde(default = "default_function_tool_type")] pub tool_type: String, pub function: Function, + /// Tool-level form of [`Function::defer_loading`]; the function-level value + /// takes precedence. + #[serde(skip_serializing_if = "Option::is_none")] + pub defer_loading: Option, } fn default_function_tool_type() -> String { @@ -286,6 +290,9 @@ pub struct Function { pub parameters: Value, /// Whether to enable strict schema adherence (OpenAI structured outputs). pub strict: Option, + /// Whether the schema is loaded on demand via tool search (vLLM extension, + /// mirroring the OpenAI Responses and Anthropic Messages field). + pub defer_loading: Option, } #[derive(Debug, Clone, Deserialize, Serialize)] diff --git a/rust/src/server/src/routes/tokenize/types.rs b/rust/src/server/src/routes/tokenize/types.rs index d907073c87..ed3fd11d1d 100644 --- a/rust/src/server/src/routes/tokenize/types.rs +++ b/rust/src/server/src/routes/tokenize/types.rs @@ -215,6 +215,7 @@ mod tests { "properties": {"city": {"type": "string"}}, }), strict: None, + defer_loading: None, }] ); }