[Rust Frontend] Pass effective reasoning_parser_kwargs for structured output (#46360)
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
4ed8eaafb0
commit
ce9f64020b
@@ -50,7 +50,7 @@ mod request;
|
||||
mod stream;
|
||||
|
||||
use vllm_engine_core_client::EngineCoreClient;
|
||||
use vllm_engine_core_client::protocol::ModelDtype;
|
||||
use vllm_engine_core_client::protocol::{ModelDtype, ReasoningParserKwargs};
|
||||
use vllm_llm::Llm;
|
||||
use vllm_text::{Prompt, TextLlm, TextRequest};
|
||||
|
||||
@@ -179,6 +179,14 @@ impl ChatLlm {
|
||||
},
|
||||
)?;
|
||||
let rendered = self.backend.chat_renderer().render(&request)?;
|
||||
let reasoning_parser_kwargs =
|
||||
request
|
||||
.sampling_params
|
||||
.structured_outputs
|
||||
.is_some()
|
||||
.then(|| ReasoningParserKwargs {
|
||||
chat_template_kwargs: rendered.effective_template_kwargs.clone(),
|
||||
});
|
||||
|
||||
let (prompt, mm_features) = multimodal::finalize_rendered_prompt(
|
||||
&request,
|
||||
@@ -199,6 +207,7 @@ impl ChatLlm {
|
||||
cache_salt: request.cache_salt,
|
||||
add_special_tokens: request.add_special_tokens,
|
||||
data_parallel_rank: request.data_parallel_rank,
|
||||
reasoning_parser_kwargs,
|
||||
lora_request: request.lora_request,
|
||||
};
|
||||
let decoded_stream = self.text.generate(text_request).await?.map_err(Error::from).boxed();
|
||||
|
||||
@@ -2,7 +2,7 @@ mod encoding;
|
||||
|
||||
use vllm_text::Prompt;
|
||||
|
||||
use super::{ChatRenderer, RenderedPrompt};
|
||||
use super::{ChatRenderer, RenderedPrompt, request_template_kwargs};
|
||||
use crate::Result;
|
||||
use crate::request::ChatRequest;
|
||||
|
||||
@@ -23,6 +23,7 @@ impl ChatRenderer for DeepSeekV32ChatRenderer {
|
||||
|
||||
Ok(RenderedPrompt {
|
||||
prompt: Prompt::Text(encoding::render_request(request)?),
|
||||
effective_template_kwargs: request_template_kwargs(request),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,7 +2,7 @@ mod encoding;
|
||||
|
||||
use vllm_text::Prompt;
|
||||
|
||||
use super::{ChatRenderer, RenderedPrompt};
|
||||
use super::{ChatRenderer, RenderedPrompt, request_template_kwargs};
|
||||
use crate::Result;
|
||||
use crate::request::ChatRequest;
|
||||
|
||||
@@ -22,6 +22,7 @@ impl ChatRenderer for DeepSeekV4ChatRenderer {
|
||||
|
||||
Ok(RenderedPrompt {
|
||||
prompt: Prompt::Text(encoding::render_request(request)?),
|
||||
effective_template_kwargs: request_template_kwargs(request),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,7 +14,7 @@ use self::format::{
|
||||
};
|
||||
use self::template::{CompiledChatTemplate, TemplateContext};
|
||||
use self::value::{TemplateValue, to_template_value};
|
||||
use super::{ChatRenderer, RenderedPrompt};
|
||||
use super::{ChatRenderer, RenderedPrompt, effective_template_kwargs};
|
||||
use crate::error::Result;
|
||||
use crate::request::{ChatContent, ChatContentPart, ChatMessage, ChatRequest};
|
||||
use crate::{
|
||||
@@ -169,8 +169,8 @@ impl HfChatRenderer {
|
||||
"applying chat template"
|
||||
);
|
||||
|
||||
let mut merged_template_kwargs = self.default_template_kwargs.clone();
|
||||
merged_template_kwargs.extend(request.chat_options.template_kwargs.clone());
|
||||
let effective_template_kwargs =
|
||||
effective_template_kwargs(&self.default_template_kwargs, request);
|
||||
let prompt = effective_template
|
||||
.apply(TemplateContext {
|
||||
messages: &messages,
|
||||
@@ -178,9 +178,8 @@ impl HfChatRenderer {
|
||||
continue_final_message: request.chat_options.continue_final_message(),
|
||||
tools: tools.as_deref(),
|
||||
documents: request.documents.as_deref(),
|
||||
template_kwargs: Some(&merged_template_kwargs),
|
||||
template_kwargs: Some(&effective_template_kwargs),
|
||||
special_tokens: self.special_tokens.as_ref(),
|
||||
reasoning_effort: request.chat_options.reasoning_effort,
|
||||
})
|
||||
.map_err(|error| Error::ChatTemplate(error.to_report_string()))?;
|
||||
|
||||
@@ -191,6 +190,7 @@ impl HfChatRenderer {
|
||||
|
||||
Ok(RenderedPrompt {
|
||||
prompt: Prompt::Text(prompt),
|
||||
effective_template_kwargs,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -797,9 +797,46 @@ mod tests {
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let rendered = renderer.render(&request).unwrap().prompt;
|
||||
let rendered = renderer.render(&request).unwrap();
|
||||
|
||||
assert_eq!(rendered, Prompt::Text("max".to_string()));
|
||||
assert_eq!(rendered.prompt, Prompt::Text("max".to_string()));
|
||||
assert_eq!(
|
||||
rendered.effective_template_kwargs.get("reasoning_effort"),
|
||||
Some(&Value::String("max".to_string()))
|
||||
);
|
||||
assert_eq!(
|
||||
rendered.effective_template_kwargs.get("enable_thinking"),
|
||||
Some(&Value::Bool(true))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chat_template_reasoning_effort_preserves_request_enable_thinking() {
|
||||
let mut request = sample_request(vec![ChatMessage::text(ChatRole::User, "hello")]);
|
||||
request.chat_options.reasoning_effort = Some(ReasoningEffort::None);
|
||||
request
|
||||
.chat_options
|
||||
.template_kwargs
|
||||
.insert("enable_thinking".to_string(), Value::Bool(true));
|
||||
|
||||
let renderer = HfChatRenderer::new(
|
||||
Some("{{ reasoning_effort }}|{{ enable_thinking }}".to_string()),
|
||||
HashMap::new(),
|
||||
ChatTemplateContentFormatOption::Auto,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let rendered = renderer.render(&request).unwrap();
|
||||
|
||||
assert_eq!(rendered.prompt, Prompt::Text("none|true".to_string()));
|
||||
assert_eq!(
|
||||
rendered.effective_template_kwargs.get("reasoning_effort"),
|
||||
Some(&Value::String("none".to_string()))
|
||||
);
|
||||
assert_eq!(
|
||||
rendered.effective_template_kwargs.get("enable_thinking"),
|
||||
Some(&Value::Bool(true))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -19,7 +19,6 @@ use super::format::{
|
||||
};
|
||||
use super::tojson::hf_tojson_filter;
|
||||
use crate::renderer::hf::{TemplateMessage, TemplateTool};
|
||||
use crate::request::ReasoningEffort;
|
||||
|
||||
type Result<T> = std::result::Result<T, TemplateError>;
|
||||
|
||||
@@ -50,9 +49,6 @@ pub(super) struct TemplateContext<'a> {
|
||||
pub(super) special_tokens: Option<&'a HfSpecialTokens>,
|
||||
#[serde(flatten)]
|
||||
pub(super) template_kwargs: Option<&'a HashMap<String, serde_json::Value>>,
|
||||
// By putting top-level `reasoning_effort` after `template_kwargs`, this overrides any
|
||||
// `reasoning_effort` value that might be present there.
|
||||
pub(super) reasoning_effort: Option<ReasoningEffort>,
|
||||
}
|
||||
|
||||
/// Load chat template from a file (`.jinja` or `.json` containing Jinja).
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use serde_json::{Value, json};
|
||||
use vllm_text::Prompt;
|
||||
|
||||
use crate::error::Result;
|
||||
use crate::request::ChatRequest;
|
||||
use crate::request::{ChatRequest, ReasoningEffort};
|
||||
|
||||
pub mod deepseek_v32;
|
||||
pub mod deepseek_v4;
|
||||
@@ -15,9 +17,13 @@ pub use deepseek_v32::DeepSeekV32ChatRenderer;
|
||||
pub use selection::RendererSelection;
|
||||
|
||||
/// Rendered chat prompt submitted to the text backend.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct RenderedPrompt {
|
||||
/// The rendered prompt, either as text or already tokenized.
|
||||
pub prompt: Prompt,
|
||||
/// Effective chat-template kwargs visible to the renderer after applying
|
||||
/// server defaults, request overrides, and typed reasoning controls.
|
||||
pub effective_template_kwargs: HashMap<String, Value>,
|
||||
}
|
||||
|
||||
/// Minimal chat-prompt renderer used by `vllm-chat`.
|
||||
@@ -29,3 +35,33 @@ pub trait ChatRenderer: Send + Sync {
|
||||
|
||||
/// Shared trait-object form of [`ChatRenderer`].
|
||||
pub type DynChatRenderer = Arc<dyn ChatRenderer>;
|
||||
|
||||
/// Extract the effective chat-template kwargs visible to the renderer from the request,
|
||||
/// using the provided defaults as the base.
|
||||
pub(crate) fn effective_template_kwargs(
|
||||
default_template_kwargs: &HashMap<String, Value>,
|
||||
request: &ChatRequest,
|
||||
) -> HashMap<String, Value> {
|
||||
let mut kwargs = default_template_kwargs.clone();
|
||||
kwargs.extend(request.chat_options.template_kwargs.clone());
|
||||
|
||||
if let Some(reasoning_effort) = request.chat_options.reasoning_effort {
|
||||
kwargs.insert(
|
||||
"reasoning_effort".to_string(),
|
||||
Value::String(reasoning_effort.as_str().to_string()),
|
||||
);
|
||||
if !request.chat_options.template_kwargs.contains_key("enable_thinking") {
|
||||
kwargs.insert(
|
||||
"enable_thinking".to_string(),
|
||||
json!(reasoning_effort != ReasoningEffort::None),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
kwargs
|
||||
}
|
||||
|
||||
/// Extract the effective chat-template kwargs visible to the renderer from the request.
|
||||
pub(crate) fn request_template_kwargs(request: &ChatRequest) -> HashMap<String, Value> {
|
||||
effective_template_kwargs(&HashMap::new(), request)
|
||||
}
|
||||
|
||||
@@ -277,6 +277,7 @@ impl ChatRenderer for FakeChatBackend {
|
||||
|
||||
Ok(RenderedPrompt {
|
||||
prompt: Prompt::Text(prompt),
|
||||
effective_template_kwargs: request.chat_options.template_kwargs.clone(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -372,6 +372,16 @@ impl EngineCoreSamplingParams {
|
||||
}
|
||||
}
|
||||
|
||||
/// Extra kwargs consumed by engine-side reasoning parsers.
|
||||
///
|
||||
/// Original Python construction point:
|
||||
/// <https://github.com/vllm-project/vllm/blob/cec2ec11760f9f3beabd4c90451936078bf91533/vllm/entrypoints/openai/chat_completion/serving.py#L367-L369>
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct ReasoningParserKwargs {
|
||||
/// Effective kwargs visible to the chat template for this request.
|
||||
pub chat_template_kwargs: HashMap<String, serde_json::Value>,
|
||||
}
|
||||
|
||||
/// Engine-core add-request payload sent from frontend to engine.
|
||||
///
|
||||
/// Original Python definition:
|
||||
@@ -421,10 +431,10 @@ pub struct EngineCoreRequest {
|
||||
pub external_req_id: Option<String>,
|
||||
#[serde(default)]
|
||||
pub reasoning_ended: Option<bool>,
|
||||
/// Opaque reasoning-parser kwargs forwarded from the frontend to the
|
||||
/// Reasoning-parser kwargs forwarded from the frontend to the
|
||||
/// structured-output backend.
|
||||
#[serde(default)]
|
||||
pub reasoning_parser_kwargs: Option<OpaqueValue>,
|
||||
pub reasoning_parser_kwargs: Option<ReasoningParserKwargs>,
|
||||
/// If `true`, the request should be added to the scheduler's waiting queue
|
||||
/// and immediately aborted, so connector-side cleanup runs via the
|
||||
/// standard `request_finished` hook.
|
||||
|
||||
@@ -56,7 +56,7 @@ fn build_request(request_id: String, max_tokens: u32) -> GenerateRequest {
|
||||
trace_headers: None,
|
||||
priority: 0,
|
||||
data_parallel_rank: None,
|
||||
reasoning_ended: None,
|
||||
reasoning_parser_kwargs: None,
|
||||
lora_request: None,
|
||||
}
|
||||
}
|
||||
|
||||
+41
-11
@@ -4,7 +4,9 @@ use std::time::{SystemTime, UNIX_EPOCH};
|
||||
use uuid::Uuid;
|
||||
use vllm_engine_core_client::protocol::lora::LoraRequest;
|
||||
use vllm_engine_core_client::protocol::multimodal::MmFeatures;
|
||||
use vllm_engine_core_client::protocol::{EngineCoreRequest, EngineCoreSamplingParams};
|
||||
use vllm_engine_core_client::protocol::{
|
||||
EngineCoreRequest, EngineCoreSamplingParams, ReasoningParserKwargs,
|
||||
};
|
||||
|
||||
use crate::error::{Error, Result};
|
||||
|
||||
@@ -27,14 +29,24 @@ pub struct GenerateRequest {
|
||||
pub sampling_params: EngineCoreSamplingParams,
|
||||
/// Optional multimodal features already prepared by `vllm-chat`.
|
||||
pub mm_features: Option<MmFeatures>,
|
||||
|
||||
// Fields below are currently likely unused by callers.
|
||||
/// Unix timestamp, in seconds, when this request arrived at the frontend.
|
||||
///
|
||||
/// When omitted, the Rust frontend fills it immediately before sending the
|
||||
/// request to engine-core, matching Python's default arrival-time behavior.
|
||||
pub arrival_time: Option<f64>,
|
||||
/// Optional salt used to partition prefix-cache entries for this request.
|
||||
pub cache_salt: Option<String>,
|
||||
/// Optional tracing headers to forward to engine-core and downstream
|
||||
/// observability hooks.
|
||||
pub trace_headers: Option<BTreeMap<String, String>>,
|
||||
/// Request scheduling priority. Lower values are scheduled earlier.
|
||||
pub priority: i32,
|
||||
/// Optional data-parallel rank override for routing this request.
|
||||
pub data_parallel_rank: Option<u32>,
|
||||
pub reasoning_ended: Option<bool>,
|
||||
/// Optional reasoning-parser kwargs forwarded to engine-side structured
|
||||
/// output logic.
|
||||
pub reasoning_parser_kwargs: Option<ReasoningParserKwargs>,
|
||||
/// Optional LoRA adapter request applied to this generation.
|
||||
pub lora_request: Option<LoraRequest>,
|
||||
}
|
||||
|
||||
@@ -61,7 +73,7 @@ impl GenerateRequest {
|
||||
trace_headers,
|
||||
priority,
|
||||
data_parallel_rank,
|
||||
reasoning_ended,
|
||||
reasoning_parser_kwargs,
|
||||
lora_request,
|
||||
} = self;
|
||||
|
||||
@@ -72,7 +84,6 @@ impl GenerateRequest {
|
||||
} else {
|
||||
external_request_id.clone()
|
||||
};
|
||||
|
||||
Ok(PreparedGenerateRequest {
|
||||
engine_request: EngineCoreRequest {
|
||||
request_id: engine_request_id,
|
||||
@@ -92,8 +103,10 @@ impl GenerateRequest {
|
||||
trace_headers,
|
||||
resumable: false,
|
||||
external_req_id: Some(external_request_id),
|
||||
reasoning_ended,
|
||||
reasoning_parser_kwargs: None,
|
||||
// Rust parser doesn't expose this information, leave it unset and let the
|
||||
// reasoning logic in engine-sided structured output manager handle it.
|
||||
reasoning_ended: None,
|
||||
reasoning_parser_kwargs,
|
||||
abort_immediately: false,
|
||||
},
|
||||
})
|
||||
@@ -121,7 +134,7 @@ fn current_unix_timestamp_secs() -> f64 {
|
||||
mod tests {
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use vllm_engine_core_client::protocol::EngineCoreSamplingParams;
|
||||
use vllm_engine_core_client::protocol::{EngineCoreSamplingParams, ReasoningParserKwargs};
|
||||
|
||||
use super::GenerateRequest;
|
||||
use crate::error::Error;
|
||||
@@ -140,7 +153,15 @@ mod tests {
|
||||
)])),
|
||||
priority: 3,
|
||||
data_parallel_rank: Some(2),
|
||||
reasoning_ended: Some(true),
|
||||
reasoning_parser_kwargs: Some(ReasoningParserKwargs {
|
||||
chat_template_kwargs: [(
|
||||
"chat_template_kwargs".to_string(),
|
||||
serde_json::json!({
|
||||
"enable_thinking": true,
|
||||
}),
|
||||
)]
|
||||
.into(),
|
||||
}),
|
||||
lora_request: None,
|
||||
}
|
||||
}
|
||||
@@ -166,7 +187,16 @@ mod tests {
|
||||
"abc".to_string(),
|
||||
)]))
|
||||
);
|
||||
assert_eq!(request.reasoning_ended, Some(true));
|
||||
assert_eq!(request.reasoning_ended, None);
|
||||
assert_eq!(
|
||||
request
|
||||
.reasoning_parser_kwargs
|
||||
.as_ref()
|
||||
.and_then(|kwargs| kwargs.chat_template_kwargs.get("chat_template_kwargs")),
|
||||
Some(&serde_json::json!({
|
||||
"enable_thinking": true
|
||||
}))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -179,7 +179,7 @@ fn sample_generate_request(request_id: &str, max_tokens: u32) -> GenerateRequest
|
||||
trace_headers: None,
|
||||
priority: 0,
|
||||
data_parallel_rank: None,
|
||||
reasoning_ended: None,
|
||||
reasoning_parser_kwargs: None,
|
||||
lora_request: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -91,6 +91,7 @@ pub fn to_text_request(
|
||||
cache_salt: kv.map(|k| &k.cache_salt).filter(|s| !s.is_empty()).cloned(),
|
||||
add_special_tokens: true,
|
||||
data_parallel_rank: None,
|
||||
reasoning_parser_kwargs: None,
|
||||
lora_request: None,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -206,6 +206,7 @@ impl ChatRenderer for FakeTextBackend {
|
||||
fn render(&self, _request: &ChatRequest) -> vllm_chat::Result<RenderedPrompt> {
|
||||
Ok(RenderedPrompt {
|
||||
prompt: Prompt::Text(String::new()),
|
||||
effective_template_kwargs: Default::default(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -223,6 +223,7 @@ impl ChatRenderer for FakeChatBackend {
|
||||
}
|
||||
Ok(RenderedPrompt {
|
||||
prompt: Prompt::Text(prompt),
|
||||
effective_template_kwargs: Default::default(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -68,6 +68,7 @@ pub(super) fn prepare_generate_request(
|
||||
cache_salt: request.cache_salt,
|
||||
add_special_tokens: false,
|
||||
data_parallel_rank: ctx.data_parallel_rank,
|
||||
reasoning_parser_kwargs: None,
|
||||
lora_request: lora_resolution.lora_request.clone(),
|
||||
};
|
||||
|
||||
|
||||
@@ -141,6 +141,7 @@ pub(super) fn prepare_completion_request(
|
||||
cache_salt: request.cache_salt,
|
||||
add_special_tokens: request.add_special_tokens,
|
||||
data_parallel_rank: ctx.data_parallel_rank,
|
||||
reasoning_parser_kwargs: None,
|
||||
lora_request: lora_resolution.lora_request.clone(),
|
||||
};
|
||||
|
||||
|
||||
@@ -581,6 +581,7 @@ impl ChatRenderer for FakeChatBackend {
|
||||
}
|
||||
Ok(vllm_chat::RenderedPrompt {
|
||||
prompt: Prompt::Text(prompt),
|
||||
effective_template_kwargs: Default::default(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -49,11 +49,10 @@ pub fn lower_text_request(
|
||||
cache_salt: request.cache_salt.clone(),
|
||||
priority: request.priority,
|
||||
data_parallel_rank: request.data_parallel_rank,
|
||||
reasoning_parser_kwargs: request.reasoning_parser_kwargs.clone(),
|
||||
lora_request: request.lora_request.clone(),
|
||||
// Fields below are currently placeholders.
|
||||
arrival_time: None,
|
||||
trace_headers: None,
|
||||
reasoning_ended: None,
|
||||
};
|
||||
|
||||
Ok(PreparedTextRequest {
|
||||
|
||||
@@ -3,9 +3,9 @@ use std::collections::HashMap;
|
||||
use enum_as_inner::EnumAsInner;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use vllm_engine_core_client::protocol::StructuredOutputsParams;
|
||||
use vllm_engine_core_client::protocol::lora::LoraRequest;
|
||||
use vllm_engine_core_client::protocol::multimodal::MmFeatures;
|
||||
use vllm_engine_core_client::protocol::{ReasoningParserKwargs, StructuredOutputsParams};
|
||||
|
||||
use crate::error::{Error, Result};
|
||||
use crate::output::TextDecodeOptions;
|
||||
@@ -174,6 +174,10 @@ pub struct TextRequest {
|
||||
/// Override data parallel rank.
|
||||
#[serde(default)]
|
||||
pub data_parallel_rank: Option<u32>,
|
||||
/// Optional reasoning-parser kwargs forwarded to engine-side structured
|
||||
/// output logic.
|
||||
#[serde(default)]
|
||||
pub reasoning_parser_kwargs: Option<ReasoningParserKwargs>,
|
||||
/// LoRA adapter selected for this request.
|
||||
#[serde(default)]
|
||||
pub lora_request: Option<LoraRequest>,
|
||||
@@ -193,6 +197,7 @@ impl TextRequest {
|
||||
cache_salt: None,
|
||||
add_special_tokens: false,
|
||||
data_parallel_rank: None,
|
||||
reasoning_parser_kwargs: None,
|
||||
lora_request: None,
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user