[Rust Frontend] Pass effective reasoning_parser_kwargs for structured output (#46360)

Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-06-24 03:13:44 +00:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 4ed8eaafb0
commit ce9f64020b
19 changed files with 165 additions and 34 deletions
+10 -1
View File
@@ -50,7 +50,7 @@ mod request;
mod stream;
use vllm_engine_core_client::EngineCoreClient;
use vllm_engine_core_client::protocol::ModelDtype;
use vllm_engine_core_client::protocol::{ModelDtype, ReasoningParserKwargs};
use vllm_llm::Llm;
use vllm_text::{Prompt, TextLlm, TextRequest};
@@ -179,6 +179,14 @@ impl ChatLlm {
},
)?;
let rendered = self.backend.chat_renderer().render(&request)?;
let reasoning_parser_kwargs =
request
.sampling_params
.structured_outputs
.is_some()
.then(|| ReasoningParserKwargs {
chat_template_kwargs: rendered.effective_template_kwargs.clone(),
});
let (prompt, mm_features) = multimodal::finalize_rendered_prompt(
&request,
@@ -199,6 +207,7 @@ impl ChatLlm {
cache_salt: request.cache_salt,
add_special_tokens: request.add_special_tokens,
data_parallel_rank: request.data_parallel_rank,
reasoning_parser_kwargs,
lora_request: request.lora_request,
};
let decoded_stream = self.text.generate(text_request).await?.map_err(Error::from).boxed();
@@ -2,7 +2,7 @@ mod encoding;
use vllm_text::Prompt;
use super::{ChatRenderer, RenderedPrompt};
use super::{ChatRenderer, RenderedPrompt, request_template_kwargs};
use crate::Result;
use crate::request::ChatRequest;
@@ -23,6 +23,7 @@ impl ChatRenderer for DeepSeekV32ChatRenderer {
Ok(RenderedPrompt {
prompt: Prompt::Text(encoding::render_request(request)?),
effective_template_kwargs: request_template_kwargs(request),
})
}
}
@@ -2,7 +2,7 @@ mod encoding;
use vllm_text::Prompt;
use super::{ChatRenderer, RenderedPrompt};
use super::{ChatRenderer, RenderedPrompt, request_template_kwargs};
use crate::Result;
use crate::request::ChatRequest;
@@ -22,6 +22,7 @@ impl ChatRenderer for DeepSeekV4ChatRenderer {
Ok(RenderedPrompt {
prompt: Prompt::Text(encoding::render_request(request)?),
effective_template_kwargs: request_template_kwargs(request),
})
}
}
+44 -7
View File
@@ -14,7 +14,7 @@ use self::format::{
};
use self::template::{CompiledChatTemplate, TemplateContext};
use self::value::{TemplateValue, to_template_value};
use super::{ChatRenderer, RenderedPrompt};
use super::{ChatRenderer, RenderedPrompt, effective_template_kwargs};
use crate::error::Result;
use crate::request::{ChatContent, ChatContentPart, ChatMessage, ChatRequest};
use crate::{
@@ -169,8 +169,8 @@ impl HfChatRenderer {
"applying chat template"
);
let mut merged_template_kwargs = self.default_template_kwargs.clone();
merged_template_kwargs.extend(request.chat_options.template_kwargs.clone());
let effective_template_kwargs =
effective_template_kwargs(&self.default_template_kwargs, request);
let prompt = effective_template
.apply(TemplateContext {
messages: &messages,
@@ -178,9 +178,8 @@ impl HfChatRenderer {
continue_final_message: request.chat_options.continue_final_message(),
tools: tools.as_deref(),
documents: request.documents.as_deref(),
template_kwargs: Some(&merged_template_kwargs),
template_kwargs: Some(&effective_template_kwargs),
special_tokens: self.special_tokens.as_ref(),
reasoning_effort: request.chat_options.reasoning_effort,
})
.map_err(|error| Error::ChatTemplate(error.to_report_string()))?;
@@ -191,6 +190,7 @@ impl HfChatRenderer {
Ok(RenderedPrompt {
prompt: Prompt::Text(prompt),
effective_template_kwargs,
})
}
}
@@ -797,9 +797,46 @@ mod tests {
)
.unwrap();
let rendered = renderer.render(&request).unwrap().prompt;
let rendered = renderer.render(&request).unwrap();
assert_eq!(rendered, Prompt::Text("max".to_string()));
assert_eq!(rendered.prompt, Prompt::Text("max".to_string()));
assert_eq!(
rendered.effective_template_kwargs.get("reasoning_effort"),
Some(&Value::String("max".to_string()))
);
assert_eq!(
rendered.effective_template_kwargs.get("enable_thinking"),
Some(&Value::Bool(true))
);
}
#[test]
fn chat_template_reasoning_effort_preserves_request_enable_thinking() {
let mut request = sample_request(vec![ChatMessage::text(ChatRole::User, "hello")]);
request.chat_options.reasoning_effort = Some(ReasoningEffort::None);
request
.chat_options
.template_kwargs
.insert("enable_thinking".to_string(), Value::Bool(true));
let renderer = HfChatRenderer::new(
Some("{{ reasoning_effort }}|{{ enable_thinking }}".to_string()),
HashMap::new(),
ChatTemplateContentFormatOption::Auto,
)
.unwrap();
let rendered = renderer.render(&request).unwrap();
assert_eq!(rendered.prompt, Prompt::Text("none|true".to_string()));
assert_eq!(
rendered.effective_template_kwargs.get("reasoning_effort"),
Some(&Value::String("none".to_string()))
);
assert_eq!(
rendered.effective_template_kwargs.get("enable_thinking"),
Some(&Value::Bool(true))
);
}
#[test]
@@ -19,7 +19,6 @@ use super::format::{
};
use super::tojson::hf_tojson_filter;
use crate::renderer::hf::{TemplateMessage, TemplateTool};
use crate::request::ReasoningEffort;
type Result<T> = std::result::Result<T, TemplateError>;
@@ -50,9 +49,6 @@ pub(super) struct TemplateContext<'a> {
pub(super) special_tokens: Option<&'a HfSpecialTokens>,
#[serde(flatten)]
pub(super) template_kwargs: Option<&'a HashMap<String, serde_json::Value>>,
// By putting top-level `reasoning_effort` after `template_kwargs`, this overrides any
// `reasoning_effort` value that might be present there.
pub(super) reasoning_effort: Option<ReasoningEffort>,
}
/// Load chat template from a file (`.jinja` or `.json` containing Jinja).
+38 -2
View File
@@ -1,9 +1,11 @@
use std::collections::HashMap;
use std::sync::Arc;
use serde_json::{Value, json};
use vllm_text::Prompt;
use crate::error::Result;
use crate::request::ChatRequest;
use crate::request::{ChatRequest, ReasoningEffort};
pub mod deepseek_v32;
pub mod deepseek_v4;
@@ -15,9 +17,13 @@ pub use deepseek_v32::DeepSeekV32ChatRenderer;
pub use selection::RendererSelection;
/// Rendered chat prompt submitted to the text backend.
#[derive(Debug, Clone, PartialEq, Eq)]
#[derive(Debug, Clone, PartialEq)]
pub struct RenderedPrompt {
/// The rendered prompt, either as text or already tokenized.
pub prompt: Prompt,
/// Effective chat-template kwargs visible to the renderer after applying
/// server defaults, request overrides, and typed reasoning controls.
pub effective_template_kwargs: HashMap<String, Value>,
}
/// Minimal chat-prompt renderer used by `vllm-chat`.
@@ -29,3 +35,33 @@ pub trait ChatRenderer: Send + Sync {
/// Shared trait-object form of [`ChatRenderer`].
pub type DynChatRenderer = Arc<dyn ChatRenderer>;
/// Extract the effective chat-template kwargs visible to the renderer from the request,
/// using the provided defaults as the base.
pub(crate) fn effective_template_kwargs(
default_template_kwargs: &HashMap<String, Value>,
request: &ChatRequest,
) -> HashMap<String, Value> {
let mut kwargs = default_template_kwargs.clone();
kwargs.extend(request.chat_options.template_kwargs.clone());
if let Some(reasoning_effort) = request.chat_options.reasoning_effort {
kwargs.insert(
"reasoning_effort".to_string(),
Value::String(reasoning_effort.as_str().to_string()),
);
if !request.chat_options.template_kwargs.contains_key("enable_thinking") {
kwargs.insert(
"enable_thinking".to_string(),
json!(reasoning_effort != ReasoningEffort::None),
);
}
}
kwargs
}
/// Extract the effective chat-template kwargs visible to the renderer from the request.
pub(crate) fn request_template_kwargs(request: &ChatRequest) -> HashMap<String, Value> {
effective_template_kwargs(&HashMap::new(), request)
}
+1
View File
@@ -277,6 +277,7 @@ impl ChatRenderer for FakeChatBackend {
Ok(RenderedPrompt {
prompt: Prompt::Text(prompt),
effective_template_kwargs: request.chat_options.template_kwargs.clone(),
})
}
}
@@ -372,6 +372,16 @@ impl EngineCoreSamplingParams {
}
}
/// Extra kwargs consumed by engine-side reasoning parsers.
///
/// Original Python construction point:
/// <https://github.com/vllm-project/vllm/blob/cec2ec11760f9f3beabd4c90451936078bf91533/vllm/entrypoints/openai/chat_completion/serving.py#L367-L369>
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct ReasoningParserKwargs {
/// Effective kwargs visible to the chat template for this request.
pub chat_template_kwargs: HashMap<String, serde_json::Value>,
}
/// Engine-core add-request payload sent from frontend to engine.
///
/// Original Python definition:
@@ -421,10 +431,10 @@ pub struct EngineCoreRequest {
pub external_req_id: Option<String>,
#[serde(default)]
pub reasoning_ended: Option<bool>,
/// Opaque reasoning-parser kwargs forwarded from the frontend to the
/// Reasoning-parser kwargs forwarded from the frontend to the
/// structured-output backend.
#[serde(default)]
pub reasoning_parser_kwargs: Option<OpaqueValue>,
pub reasoning_parser_kwargs: Option<ReasoningParserKwargs>,
/// If `true`, the request should be added to the scheduler's waiting queue
/// and immediately aborted, so connector-side cleanup runs via the
/// standard `request_finished` hook.
@@ -56,7 +56,7 @@ fn build_request(request_id: String, max_tokens: u32) -> GenerateRequest {
trace_headers: None,
priority: 0,
data_parallel_rank: None,
reasoning_ended: None,
reasoning_parser_kwargs: None,
lora_request: None,
}
}
+41 -11
View File
@@ -4,7 +4,9 @@ use std::time::{SystemTime, UNIX_EPOCH};
use uuid::Uuid;
use vllm_engine_core_client::protocol::lora::LoraRequest;
use vllm_engine_core_client::protocol::multimodal::MmFeatures;
use vllm_engine_core_client::protocol::{EngineCoreRequest, EngineCoreSamplingParams};
use vllm_engine_core_client::protocol::{
EngineCoreRequest, EngineCoreSamplingParams, ReasoningParserKwargs,
};
use crate::error::{Error, Result};
@@ -27,14 +29,24 @@ pub struct GenerateRequest {
pub sampling_params: EngineCoreSamplingParams,
/// Optional multimodal features already prepared by `vllm-chat`.
pub mm_features: Option<MmFeatures>,
// Fields below are currently likely unused by callers.
/// Unix timestamp, in seconds, when this request arrived at the frontend.
///
/// When omitted, the Rust frontend fills it immediately before sending the
/// request to engine-core, matching Python's default arrival-time behavior.
pub arrival_time: Option<f64>,
/// Optional salt used to partition prefix-cache entries for this request.
pub cache_salt: Option<String>,
/// Optional tracing headers to forward to engine-core and downstream
/// observability hooks.
pub trace_headers: Option<BTreeMap<String, String>>,
/// Request scheduling priority. Lower values are scheduled earlier.
pub priority: i32,
/// Optional data-parallel rank override for routing this request.
pub data_parallel_rank: Option<u32>,
pub reasoning_ended: Option<bool>,
/// Optional reasoning-parser kwargs forwarded to engine-side structured
/// output logic.
pub reasoning_parser_kwargs: Option<ReasoningParserKwargs>,
/// Optional LoRA adapter request applied to this generation.
pub lora_request: Option<LoraRequest>,
}
@@ -61,7 +73,7 @@ impl GenerateRequest {
trace_headers,
priority,
data_parallel_rank,
reasoning_ended,
reasoning_parser_kwargs,
lora_request,
} = self;
@@ -72,7 +84,6 @@ impl GenerateRequest {
} else {
external_request_id.clone()
};
Ok(PreparedGenerateRequest {
engine_request: EngineCoreRequest {
request_id: engine_request_id,
@@ -92,8 +103,10 @@ impl GenerateRequest {
trace_headers,
resumable: false,
external_req_id: Some(external_request_id),
reasoning_ended,
reasoning_parser_kwargs: None,
// Rust parser doesn't expose this information, leave it unset and let the
// reasoning logic in engine-sided structured output manager handle it.
reasoning_ended: None,
reasoning_parser_kwargs,
abort_immediately: false,
},
})
@@ -121,7 +134,7 @@ fn current_unix_timestamp_secs() -> f64 {
mod tests {
use std::collections::BTreeMap;
use vllm_engine_core_client::protocol::EngineCoreSamplingParams;
use vllm_engine_core_client::protocol::{EngineCoreSamplingParams, ReasoningParserKwargs};
use super::GenerateRequest;
use crate::error::Error;
@@ -140,7 +153,15 @@ mod tests {
)])),
priority: 3,
data_parallel_rank: Some(2),
reasoning_ended: Some(true),
reasoning_parser_kwargs: Some(ReasoningParserKwargs {
chat_template_kwargs: [(
"chat_template_kwargs".to_string(),
serde_json::json!({
"enable_thinking": true,
}),
)]
.into(),
}),
lora_request: None,
}
}
@@ -166,7 +187,16 @@ mod tests {
"abc".to_string(),
)]))
);
assert_eq!(request.reasoning_ended, Some(true));
assert_eq!(request.reasoning_ended, None);
assert_eq!(
request
.reasoning_parser_kwargs
.as_ref()
.and_then(|kwargs| kwargs.chat_template_kwargs.get("chat_template_kwargs")),
Some(&serde_json::json!({
"enable_thinking": true
}))
);
}
#[test]
+1 -1
View File
@@ -179,7 +179,7 @@ fn sample_generate_request(request_id: &str, max_tokens: u32) -> GenerateRequest
trace_headers: None,
priority: 0,
data_parallel_rank: None,
reasoning_ended: None,
reasoning_parser_kwargs: None,
lora_request: None,
}
}
+1
View File
@@ -91,6 +91,7 @@ pub fn to_text_request(
cache_salt: kv.map(|k| &k.cache_salt).filter(|s| !s.is_empty()).cloned(),
add_special_tokens: true,
data_parallel_rank: None,
reasoning_parser_kwargs: None,
lora_request: None,
})
}
+1
View File
@@ -206,6 +206,7 @@ impl ChatRenderer for FakeTextBackend {
fn render(&self, _request: &ChatRequest) -> vllm_chat::Result<RenderedPrompt> {
Ok(RenderedPrompt {
prompt: Prompt::Text(String::new()),
effective_template_kwargs: Default::default(),
})
}
}
@@ -223,6 +223,7 @@ impl ChatRenderer for FakeChatBackend {
}
Ok(RenderedPrompt {
prompt: Prompt::Text(prompt),
effective_template_kwargs: Default::default(),
})
}
}
@@ -68,6 +68,7 @@ pub(super) fn prepare_generate_request(
cache_salt: request.cache_salt,
add_special_tokens: false,
data_parallel_rank: ctx.data_parallel_rank,
reasoning_parser_kwargs: None,
lora_request: lora_resolution.lora_request.clone(),
};
@@ -141,6 +141,7 @@ pub(super) fn prepare_completion_request(
cache_salt: request.cache_salt,
add_special_tokens: request.add_special_tokens,
data_parallel_rank: ctx.data_parallel_rank,
reasoning_parser_kwargs: None,
lora_request: lora_resolution.lora_request.clone(),
};
+1
View File
@@ -581,6 +581,7 @@ impl ChatRenderer for FakeChatBackend {
}
Ok(vllm_chat::RenderedPrompt {
prompt: Prompt::Text(prompt),
effective_template_kwargs: Default::default(),
})
}
}
+1 -2
View File
@@ -49,11 +49,10 @@ pub fn lower_text_request(
cache_salt: request.cache_salt.clone(),
priority: request.priority,
data_parallel_rank: request.data_parallel_rank,
reasoning_parser_kwargs: request.reasoning_parser_kwargs.clone(),
lora_request: request.lora_request.clone(),
// Fields below are currently placeholders.
arrival_time: None,
trace_headers: None,
reasoning_ended: None,
};
Ok(PreparedTextRequest {
+6 -1
View File
@@ -3,9 +3,9 @@ use std::collections::HashMap;
use enum_as_inner::EnumAsInner;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use vllm_engine_core_client::protocol::StructuredOutputsParams;
use vllm_engine_core_client::protocol::lora::LoraRequest;
use vllm_engine_core_client::protocol::multimodal::MmFeatures;
use vllm_engine_core_client::protocol::{ReasoningParserKwargs, StructuredOutputsParams};
use crate::error::{Error, Result};
use crate::output::TextDecodeOptions;
@@ -174,6 +174,10 @@ pub struct TextRequest {
/// Override data parallel rank.
#[serde(default)]
pub data_parallel_rank: Option<u32>,
/// Optional reasoning-parser kwargs forwarded to engine-side structured
/// output logic.
#[serde(default)]
pub reasoning_parser_kwargs: Option<ReasoningParserKwargs>,
/// LoRA adapter selected for this request.
#[serde(default)]
pub lora_request: Option<LoraRequest>,
@@ -193,6 +197,7 @@ impl TextRequest {
cache_salt: None,
add_special_tokens: false,
data_parallel_rank: None,
reasoning_parser_kwargs: None,
lora_request: None,
}
}