diff --git a/rust/src/chat/src/lib.rs b/rust/src/chat/src/lib.rs index 012307758c..06e46f64b8 100644 --- a/rust/src/chat/src/lib.rs +++ b/rust/src/chat/src/lib.rs @@ -50,7 +50,7 @@ mod request; mod stream; use vllm_engine_core_client::EngineCoreClient; -use vllm_engine_core_client::protocol::ModelDtype; +use vllm_engine_core_client::protocol::{ModelDtype, ReasoningParserKwargs}; use vllm_llm::Llm; use vllm_text::{Prompt, TextLlm, TextRequest}; @@ -179,6 +179,14 @@ impl ChatLlm { }, )?; let rendered = self.backend.chat_renderer().render(&request)?; + let reasoning_parser_kwargs = + request + .sampling_params + .structured_outputs + .is_some() + .then(|| ReasoningParserKwargs { + chat_template_kwargs: rendered.effective_template_kwargs.clone(), + }); let (prompt, mm_features) = multimodal::finalize_rendered_prompt( &request, @@ -199,6 +207,7 @@ impl ChatLlm { cache_salt: request.cache_salt, add_special_tokens: request.add_special_tokens, data_parallel_rank: request.data_parallel_rank, + reasoning_parser_kwargs, lora_request: request.lora_request, }; let decoded_stream = self.text.generate(text_request).await?.map_err(Error::from).boxed(); diff --git a/rust/src/chat/src/renderer/deepseek_v32/mod.rs b/rust/src/chat/src/renderer/deepseek_v32/mod.rs index 97225bbab0..9da2423389 100644 --- a/rust/src/chat/src/renderer/deepseek_v32/mod.rs +++ b/rust/src/chat/src/renderer/deepseek_v32/mod.rs @@ -2,7 +2,7 @@ mod encoding; use vllm_text::Prompt; -use super::{ChatRenderer, RenderedPrompt}; +use super::{ChatRenderer, RenderedPrompt, request_template_kwargs}; use crate::Result; use crate::request::ChatRequest; @@ -23,6 +23,7 @@ impl ChatRenderer for DeepSeekV32ChatRenderer { Ok(RenderedPrompt { prompt: Prompt::Text(encoding::render_request(request)?), + effective_template_kwargs: request_template_kwargs(request), }) } } diff --git a/rust/src/chat/src/renderer/deepseek_v4/mod.rs b/rust/src/chat/src/renderer/deepseek_v4/mod.rs index 7c3f4631d2..78047c9dbe 100644 --- a/rust/src/chat/src/renderer/deepseek_v4/mod.rs +++ b/rust/src/chat/src/renderer/deepseek_v4/mod.rs @@ -2,7 +2,7 @@ mod encoding; use vllm_text::Prompt; -use super::{ChatRenderer, RenderedPrompt}; +use super::{ChatRenderer, RenderedPrompt, request_template_kwargs}; use crate::Result; use crate::request::ChatRequest; @@ -22,6 +22,7 @@ impl ChatRenderer for DeepSeekV4ChatRenderer { Ok(RenderedPrompt { prompt: Prompt::Text(encoding::render_request(request)?), + effective_template_kwargs: request_template_kwargs(request), }) } } diff --git a/rust/src/chat/src/renderer/hf/mod.rs b/rust/src/chat/src/renderer/hf/mod.rs index 47c10c0219..3ad6a5c7c7 100644 --- a/rust/src/chat/src/renderer/hf/mod.rs +++ b/rust/src/chat/src/renderer/hf/mod.rs @@ -14,7 +14,7 @@ use self::format::{ }; use self::template::{CompiledChatTemplate, TemplateContext}; use self::value::{TemplateValue, to_template_value}; -use super::{ChatRenderer, RenderedPrompt}; +use super::{ChatRenderer, RenderedPrompt, effective_template_kwargs}; use crate::error::Result; use crate::request::{ChatContent, ChatContentPart, ChatMessage, ChatRequest}; use crate::{ @@ -169,8 +169,8 @@ impl HfChatRenderer { "applying chat template" ); - let mut merged_template_kwargs = self.default_template_kwargs.clone(); - merged_template_kwargs.extend(request.chat_options.template_kwargs.clone()); + let effective_template_kwargs = + effective_template_kwargs(&self.default_template_kwargs, request); let prompt = effective_template .apply(TemplateContext { messages: &messages, @@ -178,9 +178,8 @@ impl HfChatRenderer { continue_final_message: request.chat_options.continue_final_message(), tools: tools.as_deref(), documents: request.documents.as_deref(), - template_kwargs: Some(&merged_template_kwargs), + template_kwargs: Some(&effective_template_kwargs), special_tokens: self.special_tokens.as_ref(), - reasoning_effort: request.chat_options.reasoning_effort, }) .map_err(|error| Error::ChatTemplate(error.to_report_string()))?; @@ -191,6 +190,7 @@ impl HfChatRenderer { Ok(RenderedPrompt { prompt: Prompt::Text(prompt), + effective_template_kwargs, }) } } @@ -797,9 +797,46 @@ mod tests { ) .unwrap(); - let rendered = renderer.render(&request).unwrap().prompt; + let rendered = renderer.render(&request).unwrap(); - assert_eq!(rendered, Prompt::Text("max".to_string())); + assert_eq!(rendered.prompt, Prompt::Text("max".to_string())); + assert_eq!( + rendered.effective_template_kwargs.get("reasoning_effort"), + Some(&Value::String("max".to_string())) + ); + assert_eq!( + rendered.effective_template_kwargs.get("enable_thinking"), + Some(&Value::Bool(true)) + ); + } + + #[test] + fn chat_template_reasoning_effort_preserves_request_enable_thinking() { + let mut request = sample_request(vec![ChatMessage::text(ChatRole::User, "hello")]); + request.chat_options.reasoning_effort = Some(ReasoningEffort::None); + request + .chat_options + .template_kwargs + .insert("enable_thinking".to_string(), Value::Bool(true)); + + let renderer = HfChatRenderer::new( + Some("{{ reasoning_effort }}|{{ enable_thinking }}".to_string()), + HashMap::new(), + ChatTemplateContentFormatOption::Auto, + ) + .unwrap(); + + let rendered = renderer.render(&request).unwrap(); + + assert_eq!(rendered.prompt, Prompt::Text("none|true".to_string())); + assert_eq!( + rendered.effective_template_kwargs.get("reasoning_effort"), + Some(&Value::String("none".to_string())) + ); + assert_eq!( + rendered.effective_template_kwargs.get("enable_thinking"), + Some(&Value::Bool(true)) + ); } #[test] diff --git a/rust/src/chat/src/renderer/hf/template.rs b/rust/src/chat/src/renderer/hf/template.rs index b71efc1e53..c04df0165d 100644 --- a/rust/src/chat/src/renderer/hf/template.rs +++ b/rust/src/chat/src/renderer/hf/template.rs @@ -19,7 +19,6 @@ use super::format::{ }; use super::tojson::hf_tojson_filter; use crate::renderer::hf::{TemplateMessage, TemplateTool}; -use crate::request::ReasoningEffort; type Result = std::result::Result; @@ -50,9 +49,6 @@ pub(super) struct TemplateContext<'a> { pub(super) special_tokens: Option<&'a HfSpecialTokens>, #[serde(flatten)] pub(super) template_kwargs: Option<&'a HashMap>, - // By putting top-level `reasoning_effort` after `template_kwargs`, this overrides any - // `reasoning_effort` value that might be present there. - pub(super) reasoning_effort: Option, } /// Load chat template from a file (`.jinja` or `.json` containing Jinja). diff --git a/rust/src/chat/src/renderer/mod.rs b/rust/src/chat/src/renderer/mod.rs index 07ff5d0b6d..29e64821cf 100644 --- a/rust/src/chat/src/renderer/mod.rs +++ b/rust/src/chat/src/renderer/mod.rs @@ -1,9 +1,11 @@ +use std::collections::HashMap; use std::sync::Arc; +use serde_json::{Value, json}; use vllm_text::Prompt; use crate::error::Result; -use crate::request::ChatRequest; +use crate::request::{ChatRequest, ReasoningEffort}; pub mod deepseek_v32; pub mod deepseek_v4; @@ -15,9 +17,13 @@ pub use deepseek_v32::DeepSeekV32ChatRenderer; pub use selection::RendererSelection; /// Rendered chat prompt submitted to the text backend. -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq)] pub struct RenderedPrompt { + /// The rendered prompt, either as text or already tokenized. pub prompt: Prompt, + /// Effective chat-template kwargs visible to the renderer after applying + /// server defaults, request overrides, and typed reasoning controls. + pub effective_template_kwargs: HashMap, } /// Minimal chat-prompt renderer used by `vllm-chat`. @@ -29,3 +35,33 @@ pub trait ChatRenderer: Send + Sync { /// Shared trait-object form of [`ChatRenderer`]. pub type DynChatRenderer = Arc; + +/// Extract the effective chat-template kwargs visible to the renderer from the request, +/// using the provided defaults as the base. +pub(crate) fn effective_template_kwargs( + default_template_kwargs: &HashMap, + request: &ChatRequest, +) -> HashMap { + let mut kwargs = default_template_kwargs.clone(); + kwargs.extend(request.chat_options.template_kwargs.clone()); + + if let Some(reasoning_effort) = request.chat_options.reasoning_effort { + kwargs.insert( + "reasoning_effort".to_string(), + Value::String(reasoning_effort.as_str().to_string()), + ); + if !request.chat_options.template_kwargs.contains_key("enable_thinking") { + kwargs.insert( + "enable_thinking".to_string(), + json!(reasoning_effort != ReasoningEffort::None), + ); + } + } + + kwargs +} + +/// Extract the effective chat-template kwargs visible to the renderer from the request. +pub(crate) fn request_template_kwargs(request: &ChatRequest) -> HashMap { + effective_template_kwargs(&HashMap::new(), request) +} diff --git a/rust/src/chat/tests/chat.rs b/rust/src/chat/tests/chat.rs index 07aa304af0..611dbe2397 100644 --- a/rust/src/chat/tests/chat.rs +++ b/rust/src/chat/tests/chat.rs @@ -277,6 +277,7 @@ impl ChatRenderer for FakeChatBackend { Ok(RenderedPrompt { prompt: Prompt::Text(prompt), + effective_template_kwargs: request.chat_options.template_kwargs.clone(), }) } } diff --git a/rust/src/engine-core-client/src/protocol/mod.rs b/rust/src/engine-core-client/src/protocol/mod.rs index 8862b077c9..d750261533 100644 --- a/rust/src/engine-core-client/src/protocol/mod.rs +++ b/rust/src/engine-core-client/src/protocol/mod.rs @@ -372,6 +372,16 @@ impl EngineCoreSamplingParams { } } +/// Extra kwargs consumed by engine-side reasoning parsers. +/// +/// Original Python construction point: +/// +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ReasoningParserKwargs { + /// Effective kwargs visible to the chat template for this request. + pub chat_template_kwargs: HashMap, +} + /// Engine-core add-request payload sent from frontend to engine. /// /// Original Python definition: @@ -421,10 +431,10 @@ pub struct EngineCoreRequest { pub external_req_id: Option, #[serde(default)] pub reasoning_ended: Option, - /// Opaque reasoning-parser kwargs forwarded from the frontend to the + /// Reasoning-parser kwargs forwarded from the frontend to the /// structured-output backend. #[serde(default)] - pub reasoning_parser_kwargs: Option, + pub reasoning_parser_kwargs: Option, /// If `true`, the request should be added to the scheduler's waiting queue /// and immediately aborted, so connector-side cleanup runs via the /// standard `request_finished` hook. diff --git a/rust/src/llm/examples/external_engine_smoke.rs b/rust/src/llm/examples/external_engine_smoke.rs index c2d0e6bdfa..83a22d7dbd 100644 --- a/rust/src/llm/examples/external_engine_smoke.rs +++ b/rust/src/llm/examples/external_engine_smoke.rs @@ -56,7 +56,7 @@ fn build_request(request_id: String, max_tokens: u32) -> GenerateRequest { trace_headers: None, priority: 0, data_parallel_rank: None, - reasoning_ended: None, + reasoning_parser_kwargs: None, lora_request: None, } } diff --git a/rust/src/llm/src/request.rs b/rust/src/llm/src/request.rs index af5d257774..bbb1d60fc6 100644 --- a/rust/src/llm/src/request.rs +++ b/rust/src/llm/src/request.rs @@ -4,7 +4,9 @@ use std::time::{SystemTime, UNIX_EPOCH}; use uuid::Uuid; use vllm_engine_core_client::protocol::lora::LoraRequest; use vllm_engine_core_client::protocol::multimodal::MmFeatures; -use vllm_engine_core_client::protocol::{EngineCoreRequest, EngineCoreSamplingParams}; +use vllm_engine_core_client::protocol::{ + EngineCoreRequest, EngineCoreSamplingParams, ReasoningParserKwargs, +}; use crate::error::{Error, Result}; @@ -27,14 +29,24 @@ pub struct GenerateRequest { pub sampling_params: EngineCoreSamplingParams, /// Optional multimodal features already prepared by `vllm-chat`. pub mm_features: Option, - - // Fields below are currently likely unused by callers. + /// Unix timestamp, in seconds, when this request arrived at the frontend. + /// + /// When omitted, the Rust frontend fills it immediately before sending the + /// request to engine-core, matching Python's default arrival-time behavior. pub arrival_time: Option, + /// Optional salt used to partition prefix-cache entries for this request. pub cache_salt: Option, + /// Optional tracing headers to forward to engine-core and downstream + /// observability hooks. pub trace_headers: Option>, + /// Request scheduling priority. Lower values are scheduled earlier. pub priority: i32, + /// Optional data-parallel rank override for routing this request. pub data_parallel_rank: Option, - pub reasoning_ended: Option, + /// Optional reasoning-parser kwargs forwarded to engine-side structured + /// output logic. + pub reasoning_parser_kwargs: Option, + /// Optional LoRA adapter request applied to this generation. pub lora_request: Option, } @@ -61,7 +73,7 @@ impl GenerateRequest { trace_headers, priority, data_parallel_rank, - reasoning_ended, + reasoning_parser_kwargs, lora_request, } = self; @@ -72,7 +84,6 @@ impl GenerateRequest { } else { external_request_id.clone() }; - Ok(PreparedGenerateRequest { engine_request: EngineCoreRequest { request_id: engine_request_id, @@ -92,8 +103,10 @@ impl GenerateRequest { trace_headers, resumable: false, external_req_id: Some(external_request_id), - reasoning_ended, - reasoning_parser_kwargs: None, + // Rust parser doesn't expose this information, leave it unset and let the + // reasoning logic in engine-sided structured output manager handle it. + reasoning_ended: None, + reasoning_parser_kwargs, abort_immediately: false, }, }) @@ -121,7 +134,7 @@ fn current_unix_timestamp_secs() -> f64 { mod tests { use std::collections::BTreeMap; - use vllm_engine_core_client::protocol::EngineCoreSamplingParams; + use vllm_engine_core_client::protocol::{EngineCoreSamplingParams, ReasoningParserKwargs}; use super::GenerateRequest; use crate::error::Error; @@ -140,7 +153,15 @@ mod tests { )])), priority: 3, data_parallel_rank: Some(2), - reasoning_ended: Some(true), + reasoning_parser_kwargs: Some(ReasoningParserKwargs { + chat_template_kwargs: [( + "chat_template_kwargs".to_string(), + serde_json::json!({ + "enable_thinking": true, + }), + )] + .into(), + }), lora_request: None, } } @@ -166,7 +187,16 @@ mod tests { "abc".to_string(), )])) ); - assert_eq!(request.reasoning_ended, Some(true)); + assert_eq!(request.reasoning_ended, None); + assert_eq!( + request + .reasoning_parser_kwargs + .as_ref() + .and_then(|kwargs| kwargs.chat_template_kwargs.get("chat_template_kwargs")), + Some(&serde_json::json!({ + "enable_thinking": true + })) + ); } #[test] diff --git a/rust/src/llm/tests/generate.rs b/rust/src/llm/tests/generate.rs index cc7e7f820f..9810833473 100644 --- a/rust/src/llm/tests/generate.rs +++ b/rust/src/llm/tests/generate.rs @@ -179,7 +179,7 @@ fn sample_generate_request(request_id: &str, max_tokens: u32) -> GenerateRequest trace_headers: None, priority: 0, data_parallel_rank: None, - reasoning_ended: None, + reasoning_parser_kwargs: None, lora_request: None, } } diff --git a/rust/src/server/src/grpc/convert.rs b/rust/src/server/src/grpc/convert.rs index 0bfe7a63be..3ebe6b31fd 100644 --- a/rust/src/server/src/grpc/convert.rs +++ b/rust/src/server/src/grpc/convert.rs @@ -91,6 +91,7 @@ pub fn to_text_request( cache_salt: kv.map(|k| &k.cache_salt).filter(|s| !s.is_empty()).cloned(), add_special_tokens: true, data_parallel_rank: None, + reasoning_parser_kwargs: None, lora_request: None, }) } diff --git a/rust/src/server/src/grpc/tests.rs b/rust/src/server/src/grpc/tests.rs index 17361ae0e8..58bf894c92 100644 --- a/rust/src/server/src/grpc/tests.rs +++ b/rust/src/server/src/grpc/tests.rs @@ -206,6 +206,7 @@ impl ChatRenderer for FakeTextBackend { fn render(&self, _request: &ChatRequest) -> vllm_chat::Result { Ok(RenderedPrompt { prompt: Prompt::Text(String::new()), + effective_template_kwargs: Default::default(), }) } } diff --git a/rust/src/server/src/routes/http_client_tests.rs b/rust/src/server/src/routes/http_client_tests.rs index 8055ada979..b23c95fa6d 100644 --- a/rust/src/server/src/routes/http_client_tests.rs +++ b/rust/src/server/src/routes/http_client_tests.rs @@ -223,6 +223,7 @@ impl ChatRenderer for FakeChatBackend { } Ok(RenderedPrompt { prompt: Prompt::Text(prompt), + effective_template_kwargs: Default::default(), }) } } diff --git a/rust/src/server/src/routes/inference/generate/convert.rs b/rust/src/server/src/routes/inference/generate/convert.rs index df3ba33735..7a6dcdfc45 100644 --- a/rust/src/server/src/routes/inference/generate/convert.rs +++ b/rust/src/server/src/routes/inference/generate/convert.rs @@ -68,6 +68,7 @@ pub(super) fn prepare_generate_request( cache_salt: request.cache_salt, add_special_tokens: false, data_parallel_rank: ctx.data_parallel_rank, + reasoning_parser_kwargs: None, lora_request: lora_resolution.lora_request.clone(), }; diff --git a/rust/src/server/src/routes/openai/completions/convert.rs b/rust/src/server/src/routes/openai/completions/convert.rs index 46a91cb05d..0541346438 100644 --- a/rust/src/server/src/routes/openai/completions/convert.rs +++ b/rust/src/server/src/routes/openai/completions/convert.rs @@ -141,6 +141,7 @@ pub(super) fn prepare_completion_request( cache_salt: request.cache_salt, add_special_tokens: request.add_special_tokens, data_parallel_rank: ctx.data_parallel_rank, + reasoning_parser_kwargs: None, lora_request: lora_resolution.lora_request.clone(), }; diff --git a/rust/src/server/src/routes/tests.rs b/rust/src/server/src/routes/tests.rs index 062b4047b4..88c6835139 100644 --- a/rust/src/server/src/routes/tests.rs +++ b/rust/src/server/src/routes/tests.rs @@ -581,6 +581,7 @@ impl ChatRenderer for FakeChatBackend { } Ok(vllm_chat::RenderedPrompt { prompt: Prompt::Text(prompt), + effective_template_kwargs: Default::default(), }) } } diff --git a/rust/src/text/src/lower.rs b/rust/src/text/src/lower.rs index 7ba2fedfdb..38528114e5 100644 --- a/rust/src/text/src/lower.rs +++ b/rust/src/text/src/lower.rs @@ -49,11 +49,10 @@ pub fn lower_text_request( cache_salt: request.cache_salt.clone(), priority: request.priority, data_parallel_rank: request.data_parallel_rank, + reasoning_parser_kwargs: request.reasoning_parser_kwargs.clone(), lora_request: request.lora_request.clone(), - // Fields below are currently placeholders. arrival_time: None, trace_headers: None, - reasoning_ended: None, }; Ok(PreparedTextRequest { diff --git a/rust/src/text/src/request.rs b/rust/src/text/src/request.rs index 682d85390d..621da75ad5 100644 --- a/rust/src/text/src/request.rs +++ b/rust/src/text/src/request.rs @@ -3,9 +3,9 @@ use std::collections::HashMap; use enum_as_inner::EnumAsInner; use serde::{Deserialize, Serialize}; use serde_json::Value; -use vllm_engine_core_client::protocol::StructuredOutputsParams; use vllm_engine_core_client::protocol::lora::LoraRequest; use vllm_engine_core_client::protocol::multimodal::MmFeatures; +use vllm_engine_core_client::protocol::{ReasoningParserKwargs, StructuredOutputsParams}; use crate::error::{Error, Result}; use crate::output::TextDecodeOptions; @@ -174,6 +174,10 @@ pub struct TextRequest { /// Override data parallel rank. #[serde(default)] pub data_parallel_rank: Option, + /// Optional reasoning-parser kwargs forwarded to engine-side structured + /// output logic. + #[serde(default)] + pub reasoning_parser_kwargs: Option, /// LoRA adapter selected for this request. #[serde(default)] pub lora_request: Option, @@ -193,6 +197,7 @@ impl TextRequest { cache_salt: None, add_special_tokens: false, data_parallel_rank: None, + reasoning_parser_kwargs: None, lora_request: None, } }