[Rust Frontend] Pass effective reasoning_parser_kwargs for structured output (#46360)

Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-06-24 03:13:44 +00:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 4ed8eaafb0
commit ce9f64020b
19 changed files with 165 additions and 34 deletions
+10 -1
View File
@@ -50,7 +50,7 @@ mod request;
mod stream;
use vllm_engine_core_client::EngineCoreClient;
use vllm_engine_core_client::protocol::ModelDtype;
use vllm_engine_core_client::protocol::{ModelDtype, ReasoningParserKwargs};
use vllm_llm::Llm;
use vllm_text::{Prompt, TextLlm, TextRequest};
@@ -179,6 +179,14 @@ impl ChatLlm {
},
)?;
let rendered = self.backend.chat_renderer().render(&request)?;
let reasoning_parser_kwargs =
request
.sampling_params
.structured_outputs
.is_some()
.then(|| ReasoningParserKwargs {
chat_template_kwargs: rendered.effective_template_kwargs.clone(),
});
let (prompt, mm_features) = multimodal::finalize_rendered_prompt(
&request,
@@ -199,6 +207,7 @@ impl ChatLlm {
cache_salt: request.cache_salt,
add_special_tokens: request.add_special_tokens,
data_parallel_rank: request.data_parallel_rank,
reasoning_parser_kwargs,
lora_request: request.lora_request,
};
let decoded_stream = self.text.generate(text_request).await?.map_err(Error::from).boxed();
@@ -2,7 +2,7 @@ mod encoding;
use vllm_text::Prompt;
use super::{ChatRenderer, RenderedPrompt};
use super::{ChatRenderer, RenderedPrompt, request_template_kwargs};
use crate::Result;
use crate::request::ChatRequest;
@@ -23,6 +23,7 @@ impl ChatRenderer for DeepSeekV32ChatRenderer {
Ok(RenderedPrompt {
prompt: Prompt::Text(encoding::render_request(request)?),
effective_template_kwargs: request_template_kwargs(request),
})
}
}
@@ -2,7 +2,7 @@ mod encoding;
use vllm_text::Prompt;
use super::{ChatRenderer, RenderedPrompt};
use super::{ChatRenderer, RenderedPrompt, request_template_kwargs};
use crate::Result;
use crate::request::ChatRequest;
@@ -22,6 +22,7 @@ impl ChatRenderer for DeepSeekV4ChatRenderer {
Ok(RenderedPrompt {
prompt: Prompt::Text(encoding::render_request(request)?),
effective_template_kwargs: request_template_kwargs(request),
})
}
}
+44 -7
View File
@@ -14,7 +14,7 @@ use self::format::{
};
use self::template::{CompiledChatTemplate, TemplateContext};
use self::value::{TemplateValue, to_template_value};
use super::{ChatRenderer, RenderedPrompt};
use super::{ChatRenderer, RenderedPrompt, effective_template_kwargs};
use crate::error::Result;
use crate::request::{ChatContent, ChatContentPart, ChatMessage, ChatRequest};
use crate::{
@@ -169,8 +169,8 @@ impl HfChatRenderer {
"applying chat template"
);
let mut merged_template_kwargs = self.default_template_kwargs.clone();
merged_template_kwargs.extend(request.chat_options.template_kwargs.clone());
let effective_template_kwargs =
effective_template_kwargs(&self.default_template_kwargs, request);
let prompt = effective_template
.apply(TemplateContext {
messages: &messages,
@@ -178,9 +178,8 @@ impl HfChatRenderer {
continue_final_message: request.chat_options.continue_final_message(),
tools: tools.as_deref(),
documents: request.documents.as_deref(),
template_kwargs: Some(&merged_template_kwargs),
template_kwargs: Some(&effective_template_kwargs),
special_tokens: self.special_tokens.as_ref(),
reasoning_effort: request.chat_options.reasoning_effort,
})
.map_err(|error| Error::ChatTemplate(error.to_report_string()))?;
@@ -191,6 +190,7 @@ impl HfChatRenderer {
Ok(RenderedPrompt {
prompt: Prompt::Text(prompt),
effective_template_kwargs,
})
}
}
@@ -797,9 +797,46 @@ mod tests {
)
.unwrap();
let rendered = renderer.render(&request).unwrap().prompt;
let rendered = renderer.render(&request).unwrap();
assert_eq!(rendered, Prompt::Text("max".to_string()));
assert_eq!(rendered.prompt, Prompt::Text("max".to_string()));
assert_eq!(
rendered.effective_template_kwargs.get("reasoning_effort"),
Some(&Value::String("max".to_string()))
);
assert_eq!(
rendered.effective_template_kwargs.get("enable_thinking"),
Some(&Value::Bool(true))
);
}
#[test]
fn chat_template_reasoning_effort_preserves_request_enable_thinking() {
let mut request = sample_request(vec![ChatMessage::text(ChatRole::User, "hello")]);
request.chat_options.reasoning_effort = Some(ReasoningEffort::None);
request
.chat_options
.template_kwargs
.insert("enable_thinking".to_string(), Value::Bool(true));
let renderer = HfChatRenderer::new(
Some("{{ reasoning_effort }}|{{ enable_thinking }}".to_string()),
HashMap::new(),
ChatTemplateContentFormatOption::Auto,
)
.unwrap();
let rendered = renderer.render(&request).unwrap();
assert_eq!(rendered.prompt, Prompt::Text("none|true".to_string()));
assert_eq!(
rendered.effective_template_kwargs.get("reasoning_effort"),
Some(&Value::String("none".to_string()))
);
assert_eq!(
rendered.effective_template_kwargs.get("enable_thinking"),
Some(&Value::Bool(true))
);
}
#[test]
@@ -19,7 +19,6 @@ use super::format::{
};
use super::tojson::hf_tojson_filter;
use crate::renderer::hf::{TemplateMessage, TemplateTool};
use crate::request::ReasoningEffort;
type Result<T> = std::result::Result<T, TemplateError>;
@@ -50,9 +49,6 @@ pub(super) struct TemplateContext<'a> {
pub(super) special_tokens: Option<&'a HfSpecialTokens>,
#[serde(flatten)]
pub(super) template_kwargs: Option<&'a HashMap<String, serde_json::Value>>,
// By putting top-level `reasoning_effort` after `template_kwargs`, this overrides any
// `reasoning_effort` value that might be present there.
pub(super) reasoning_effort: Option<ReasoningEffort>,
}
/// Load chat template from a file (`.jinja` or `.json` containing Jinja).
+38 -2
View File
@@ -1,9 +1,11 @@
use std::collections::HashMap;
use std::sync::Arc;
use serde_json::{Value, json};
use vllm_text::Prompt;
use crate::error::Result;
use crate::request::ChatRequest;
use crate::request::{ChatRequest, ReasoningEffort};
pub mod deepseek_v32;
pub mod deepseek_v4;
@@ -15,9 +17,13 @@ pub use deepseek_v32::DeepSeekV32ChatRenderer;
pub use selection::RendererSelection;
/// Rendered chat prompt submitted to the text backend.
#[derive(Debug, Clone, PartialEq, Eq)]
#[derive(Debug, Clone, PartialEq)]
pub struct RenderedPrompt {
/// The rendered prompt, either as text or already tokenized.
pub prompt: Prompt,
/// Effective chat-template kwargs visible to the renderer after applying
/// server defaults, request overrides, and typed reasoning controls.
pub effective_template_kwargs: HashMap<String, Value>,
}
/// Minimal chat-prompt renderer used by `vllm-chat`.
@@ -29,3 +35,33 @@ pub trait ChatRenderer: Send + Sync {
/// Shared trait-object form of [`ChatRenderer`].
pub type DynChatRenderer = Arc<dyn ChatRenderer>;
/// Extract the effective chat-template kwargs visible to the renderer from the request,
/// using the provided defaults as the base.
pub(crate) fn effective_template_kwargs(
default_template_kwargs: &HashMap<String, Value>,
request: &ChatRequest,
) -> HashMap<String, Value> {
let mut kwargs = default_template_kwargs.clone();
kwargs.extend(request.chat_options.template_kwargs.clone());
if let Some(reasoning_effort) = request.chat_options.reasoning_effort {
kwargs.insert(
"reasoning_effort".to_string(),
Value::String(reasoning_effort.as_str().to_string()),
);
if !request.chat_options.template_kwargs.contains_key("enable_thinking") {
kwargs.insert(
"enable_thinking".to_string(),
json!(reasoning_effort != ReasoningEffort::None),
);
}
}
kwargs
}
/// Extract the effective chat-template kwargs visible to the renderer from the request.
pub(crate) fn request_template_kwargs(request: &ChatRequest) -> HashMap<String, Value> {
effective_template_kwargs(&HashMap::new(), request)
}
+1
View File
@@ -277,6 +277,7 @@ impl ChatRenderer for FakeChatBackend {
Ok(RenderedPrompt {
prompt: Prompt::Text(prompt),
effective_template_kwargs: request.chat_options.template_kwargs.clone(),
})
}
}