[Rust Frontend] Pass effective reasoning_parser_kwargs for structured output (#46360)
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
4ed8eaafb0
commit
ce9f64020b
@@ -50,7 +50,7 @@ mod request;
|
||||
mod stream;
|
||||
|
||||
use vllm_engine_core_client::EngineCoreClient;
|
||||
use vllm_engine_core_client::protocol::ModelDtype;
|
||||
use vllm_engine_core_client::protocol::{ModelDtype, ReasoningParserKwargs};
|
||||
use vllm_llm::Llm;
|
||||
use vllm_text::{Prompt, TextLlm, TextRequest};
|
||||
|
||||
@@ -179,6 +179,14 @@ impl ChatLlm {
|
||||
},
|
||||
)?;
|
||||
let rendered = self.backend.chat_renderer().render(&request)?;
|
||||
let reasoning_parser_kwargs =
|
||||
request
|
||||
.sampling_params
|
||||
.structured_outputs
|
||||
.is_some()
|
||||
.then(|| ReasoningParserKwargs {
|
||||
chat_template_kwargs: rendered.effective_template_kwargs.clone(),
|
||||
});
|
||||
|
||||
let (prompt, mm_features) = multimodal::finalize_rendered_prompt(
|
||||
&request,
|
||||
@@ -199,6 +207,7 @@ impl ChatLlm {
|
||||
cache_salt: request.cache_salt,
|
||||
add_special_tokens: request.add_special_tokens,
|
||||
data_parallel_rank: request.data_parallel_rank,
|
||||
reasoning_parser_kwargs,
|
||||
lora_request: request.lora_request,
|
||||
};
|
||||
let decoded_stream = self.text.generate(text_request).await?.map_err(Error::from).boxed();
|
||||
|
||||
@@ -2,7 +2,7 @@ mod encoding;
|
||||
|
||||
use vllm_text::Prompt;
|
||||
|
||||
use super::{ChatRenderer, RenderedPrompt};
|
||||
use super::{ChatRenderer, RenderedPrompt, request_template_kwargs};
|
||||
use crate::Result;
|
||||
use crate::request::ChatRequest;
|
||||
|
||||
@@ -23,6 +23,7 @@ impl ChatRenderer for DeepSeekV32ChatRenderer {
|
||||
|
||||
Ok(RenderedPrompt {
|
||||
prompt: Prompt::Text(encoding::render_request(request)?),
|
||||
effective_template_kwargs: request_template_kwargs(request),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,7 +2,7 @@ mod encoding;
|
||||
|
||||
use vllm_text::Prompt;
|
||||
|
||||
use super::{ChatRenderer, RenderedPrompt};
|
||||
use super::{ChatRenderer, RenderedPrompt, request_template_kwargs};
|
||||
use crate::Result;
|
||||
use crate::request::ChatRequest;
|
||||
|
||||
@@ -22,6 +22,7 @@ impl ChatRenderer for DeepSeekV4ChatRenderer {
|
||||
|
||||
Ok(RenderedPrompt {
|
||||
prompt: Prompt::Text(encoding::render_request(request)?),
|
||||
effective_template_kwargs: request_template_kwargs(request),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,7 +14,7 @@ use self::format::{
|
||||
};
|
||||
use self::template::{CompiledChatTemplate, TemplateContext};
|
||||
use self::value::{TemplateValue, to_template_value};
|
||||
use super::{ChatRenderer, RenderedPrompt};
|
||||
use super::{ChatRenderer, RenderedPrompt, effective_template_kwargs};
|
||||
use crate::error::Result;
|
||||
use crate::request::{ChatContent, ChatContentPart, ChatMessage, ChatRequest};
|
||||
use crate::{
|
||||
@@ -169,8 +169,8 @@ impl HfChatRenderer {
|
||||
"applying chat template"
|
||||
);
|
||||
|
||||
let mut merged_template_kwargs = self.default_template_kwargs.clone();
|
||||
merged_template_kwargs.extend(request.chat_options.template_kwargs.clone());
|
||||
let effective_template_kwargs =
|
||||
effective_template_kwargs(&self.default_template_kwargs, request);
|
||||
let prompt = effective_template
|
||||
.apply(TemplateContext {
|
||||
messages: &messages,
|
||||
@@ -178,9 +178,8 @@ impl HfChatRenderer {
|
||||
continue_final_message: request.chat_options.continue_final_message(),
|
||||
tools: tools.as_deref(),
|
||||
documents: request.documents.as_deref(),
|
||||
template_kwargs: Some(&merged_template_kwargs),
|
||||
template_kwargs: Some(&effective_template_kwargs),
|
||||
special_tokens: self.special_tokens.as_ref(),
|
||||
reasoning_effort: request.chat_options.reasoning_effort,
|
||||
})
|
||||
.map_err(|error| Error::ChatTemplate(error.to_report_string()))?;
|
||||
|
||||
@@ -191,6 +190,7 @@ impl HfChatRenderer {
|
||||
|
||||
Ok(RenderedPrompt {
|
||||
prompt: Prompt::Text(prompt),
|
||||
effective_template_kwargs,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -797,9 +797,46 @@ mod tests {
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let rendered = renderer.render(&request).unwrap().prompt;
|
||||
let rendered = renderer.render(&request).unwrap();
|
||||
|
||||
assert_eq!(rendered, Prompt::Text("max".to_string()));
|
||||
assert_eq!(rendered.prompt, Prompt::Text("max".to_string()));
|
||||
assert_eq!(
|
||||
rendered.effective_template_kwargs.get("reasoning_effort"),
|
||||
Some(&Value::String("max".to_string()))
|
||||
);
|
||||
assert_eq!(
|
||||
rendered.effective_template_kwargs.get("enable_thinking"),
|
||||
Some(&Value::Bool(true))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chat_template_reasoning_effort_preserves_request_enable_thinking() {
|
||||
let mut request = sample_request(vec![ChatMessage::text(ChatRole::User, "hello")]);
|
||||
request.chat_options.reasoning_effort = Some(ReasoningEffort::None);
|
||||
request
|
||||
.chat_options
|
||||
.template_kwargs
|
||||
.insert("enable_thinking".to_string(), Value::Bool(true));
|
||||
|
||||
let renderer = HfChatRenderer::new(
|
||||
Some("{{ reasoning_effort }}|{{ enable_thinking }}".to_string()),
|
||||
HashMap::new(),
|
||||
ChatTemplateContentFormatOption::Auto,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let rendered = renderer.render(&request).unwrap();
|
||||
|
||||
assert_eq!(rendered.prompt, Prompt::Text("none|true".to_string()));
|
||||
assert_eq!(
|
||||
rendered.effective_template_kwargs.get("reasoning_effort"),
|
||||
Some(&Value::String("none".to_string()))
|
||||
);
|
||||
assert_eq!(
|
||||
rendered.effective_template_kwargs.get("enable_thinking"),
|
||||
Some(&Value::Bool(true))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -19,7 +19,6 @@ use super::format::{
|
||||
};
|
||||
use super::tojson::hf_tojson_filter;
|
||||
use crate::renderer::hf::{TemplateMessage, TemplateTool};
|
||||
use crate::request::ReasoningEffort;
|
||||
|
||||
type Result<T> = std::result::Result<T, TemplateError>;
|
||||
|
||||
@@ -50,9 +49,6 @@ pub(super) struct TemplateContext<'a> {
|
||||
pub(super) special_tokens: Option<&'a HfSpecialTokens>,
|
||||
#[serde(flatten)]
|
||||
pub(super) template_kwargs: Option<&'a HashMap<String, serde_json::Value>>,
|
||||
// By putting top-level `reasoning_effort` after `template_kwargs`, this overrides any
|
||||
// `reasoning_effort` value that might be present there.
|
||||
pub(super) reasoning_effort: Option<ReasoningEffort>,
|
||||
}
|
||||
|
||||
/// Load chat template from a file (`.jinja` or `.json` containing Jinja).
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use serde_json::{Value, json};
|
||||
use vllm_text::Prompt;
|
||||
|
||||
use crate::error::Result;
|
||||
use crate::request::ChatRequest;
|
||||
use crate::request::{ChatRequest, ReasoningEffort};
|
||||
|
||||
pub mod deepseek_v32;
|
||||
pub mod deepseek_v4;
|
||||
@@ -15,9 +17,13 @@ pub use deepseek_v32::DeepSeekV32ChatRenderer;
|
||||
pub use selection::RendererSelection;
|
||||
|
||||
/// Rendered chat prompt submitted to the text backend.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct RenderedPrompt {
|
||||
/// The rendered prompt, either as text or already tokenized.
|
||||
pub prompt: Prompt,
|
||||
/// Effective chat-template kwargs visible to the renderer after applying
|
||||
/// server defaults, request overrides, and typed reasoning controls.
|
||||
pub effective_template_kwargs: HashMap<String, Value>,
|
||||
}
|
||||
|
||||
/// Minimal chat-prompt renderer used by `vllm-chat`.
|
||||
@@ -29,3 +35,33 @@ pub trait ChatRenderer: Send + Sync {
|
||||
|
||||
/// Shared trait-object form of [`ChatRenderer`].
|
||||
pub type DynChatRenderer = Arc<dyn ChatRenderer>;
|
||||
|
||||
/// Extract the effective chat-template kwargs visible to the renderer from the request,
|
||||
/// using the provided defaults as the base.
|
||||
pub(crate) fn effective_template_kwargs(
|
||||
default_template_kwargs: &HashMap<String, Value>,
|
||||
request: &ChatRequest,
|
||||
) -> HashMap<String, Value> {
|
||||
let mut kwargs = default_template_kwargs.clone();
|
||||
kwargs.extend(request.chat_options.template_kwargs.clone());
|
||||
|
||||
if let Some(reasoning_effort) = request.chat_options.reasoning_effort {
|
||||
kwargs.insert(
|
||||
"reasoning_effort".to_string(),
|
||||
Value::String(reasoning_effort.as_str().to_string()),
|
||||
);
|
||||
if !request.chat_options.template_kwargs.contains_key("enable_thinking") {
|
||||
kwargs.insert(
|
||||
"enable_thinking".to_string(),
|
||||
json!(reasoning_effort != ReasoningEffort::None),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
kwargs
|
||||
}
|
||||
|
||||
/// Extract the effective chat-template kwargs visible to the renderer from the request.
|
||||
pub(crate) fn request_template_kwargs(request: &ChatRequest) -> HashMap<String, Value> {
|
||||
effective_template_kwargs(&HashMap::new(), request)
|
||||
}
|
||||
|
||||
@@ -277,6 +277,7 @@ impl ChatRenderer for FakeChatBackend {
|
||||
|
||||
Ok(RenderedPrompt {
|
||||
prompt: Prompt::Text(prompt),
|
||||
effective_template_kwargs: request.chat_options.template_kwargs.clone(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user