[Rust Frontend] Pass reasoning controls through render -> generate (#60546)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Bugen Zhao
2026-10-09 11:04:49 -07:00
committed by GitHub
co-authored by Claude Opus 5.5
parent ce4603cad6
commit 5dc9bcd8cd
5 changed files with 92 additions and 13 deletions
@@ -64,6 +64,7 @@ impl EngineCoreRequestType {
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
pub struct ReasoningParserKwargs {
/// Effective kwargs visible to the chat template for this request.
#[serde(default)]
pub chat_template_kwargs: HashMap<String, serde_json::Value>,
}
@@ -85,8 +85,8 @@ pub(super) fn prepare_generate_request(
data_parallel_rank: ctx.data_parallel_rank,
session_id: ctx.session_id,
kv_hints: None,
reasoning_parser_kwargs: Default::default(),
reasoning_ended: None,
reasoning_parser_kwargs: request.reasoning_parser_kwargs.unwrap_or_default(),
reasoning_ended: request.reasoning_ended,
lora_request: lora_resolution.lora_request.clone(),
arrival_time: None,
};
@@ -224,6 +224,54 @@ mod tests {
);
}
#[test]
fn prepare_generate_request_forwards_reasoning_controls() {
for (controls, expected) in [
(json!({}), (json!({"chat_template_kwargs": {}}), None)),
(
json!({
"reasoning_ended": false,
"reasoning_parser_kwargs": {
"chat_template_kwargs": {"enable_thinking": true}
}
}),
(
json!({"chat_template_kwargs": {"enable_thinking": true}}),
Some(false),
),
),
(
json!({"reasoning_ended": true, "reasoning_parser_kwargs": {}}),
(json!({"chat_template_kwargs": {}}), Some(true)),
),
] {
let mut body = json!({
"model": "Qwen/Qwen1.5-0.5B-Chat",
"token_ids": [11, 22, 33],
"sampling_params": {}
});
body.as_object_mut().unwrap().extend(controls.as_object().unwrap().clone());
let request: GenerateRequest = serde_json::from_value(body).expect("parse request");
let prepared = prepare_generate_request(
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
None,
)
.expect("prepare");
assert_eq!(
(
serde_json::to_value(&prepared.text_request.reasoning_parser_kwargs).unwrap(),
prepared.text_request.reasoning_ended,
),
expected,
"controls: {controls}"
);
}
}
#[test]
fn prepare_generate_request_gates_continuous_usage_on_include_usage() {
let request: GenerateRequest = serde_json::from_value(json!({
@@ -8,6 +8,7 @@ use serde::{Deserialize, Serialize};
use serde_json::{Map, Value};
use validator::Validate;
use vllm_engine_core_client::protocol::output::RequestSpecDecodeMetrics;
use vllm_engine_core_client::protocol::request::ReasoningParserKwargs;
use vllm_text::SamplingParams;
use crate::routes::openai::utils::types::{Normalizable, StreamOptions, Usage};
@@ -48,6 +49,13 @@ pub struct GenerateRequest {
/// Raw multimodal input; server resolves media. Mutually exclusive with `features`.
pub content_parts: Option<Vec<MediaContentPart>>,
pub return_token_ids: Option<bool>,
/// Whether reasoning has ended before the first generated token, as
/// resolved by `/render`. `true` applies structured outputs from the first
/// token; `None` lets the engine check the prompt with its reasoning parser.
pub reasoning_ended: Option<bool>,
/// Set by `/render` so the engine-side reasoning parser agrees with the
/// frontend on flags such as `enable_thinking`.
pub reasoning_parser_kwargs: Option<ReasoningParserKwargs>,
#[serde(flatten)]
pub other: Map<String, Value>,
}
+7
View File
@@ -10,6 +10,7 @@ use axum::extract::State;
use axum::http::{HeaderMap, StatusCode};
use axum::routing::{get, post};
use thiserror_ext::AsReport as _;
use vllm_engine_core_client::protocol::request::ReasoningParserKwargs;
use vllm_text::TextRequest;
use crate::DEFAULT_REQUEST_BODY_LIMIT_BYTES;
@@ -106,6 +107,12 @@ fn lower_render_request(
ec_transfer_params: None,
content_parts: None,
return_token_ids: None,
// Carry what the chat route would pass to the engine, so the engine
// gates structured outputs the same way on `/inference/v1/generate`.
reasoning_ended: text_request.reasoning_ended,
reasoning_parser_kwargs: (text_request.reasoning_parser_kwargs
!= ReasoningParserKwargs::default())
.then_some(text_request.reasoning_parser_kwargs),
other: Default::default(),
};
validate_generate_request(&request, &state.served_model_names)?;
+26 -11
View File
@@ -542,7 +542,7 @@ impl ChatRenderer for FakeChatBackend {
Ok(vllm_chat::RenderedPrompt {
prompt: Prompt::Text(prompt),
media_order: None,
effective_template_kwargs: Default::default(),
effective_template_kwargs: request.chat_options.template_kwargs.clone(),
})
}
}
@@ -615,7 +615,7 @@ fn qwen_multimodal_model_info_with_limits(
async fn test_models_with_engine_outputs_and_backend_inner(
engine_id: impl Into<EngineId>,
output_specs: Vec<(Vec<u32>, Option<EngineCoreFinishReason>)>,
expected_prompt_token_ids: Option<Vec<u32>>,
check_request: impl FnOnce(&EngineCoreRequest) + Send + 'static,
backend: Arc<dyn ChatTextBackend>,
) -> (ChatLlm, MockEngineTask) {
let ipc = IpcNamespace::new().expect("create ipc namespace");
@@ -630,12 +630,7 @@ async fn test_models_with_engine_outputs_and_backend_inner(
let add = recv_engine_message(dealer).await;
let request: EngineCoreRequest =
rmp_serde::from_slice(&add[1]).expect("decode request");
if let Some(expected_prompt_token_ids) = expected_prompt_token_ids {
assert_eq!(
request.prompt_token_ids.as_deref(),
Some(expected_prompt_token_ids.as_slice())
);
}
check_request(&request);
send_outputs(
push,
engine_outputs_for_request(&request.request_id, output_specs),
@@ -667,7 +662,8 @@ async fn test_models_with_engine_outputs_and_backend(
output_specs: Vec<(Vec<u32>, Option<EngineCoreFinishReason>)>,
backend: Arc<dyn ChatTextBackend>,
) -> (ChatLlm, MockEngineTask) {
test_models_with_engine_outputs_and_backend_inner(engine_id, output_specs, None, backend).await
test_models_with_engine_outputs_and_backend_inner(engine_id, output_specs, |_| {}, backend)
.await
}
async fn test_chat_with_engine_outputs(
@@ -1440,6 +1436,8 @@ async fn render_completion_returns_generate_request_with_body_request_id() {
assert!(json[0].get("stream_options").is_none());
assert!(json[0].get("prompt_token_ids").is_none());
assert!(json[0].get("mm_features").is_none());
assert!(json[0].get("reasoning_ended").is_none());
assert!(json[0].get("reasoning_parser_kwargs").is_none());
}
#[tokio::test]
@@ -1479,7 +1477,8 @@ async fn render_chat_response_can_be_submitted_to_generate_unchanged() {
"messages": [{"role": "user", "content": "hello"}],
"max_completion_tokens": 8,
"bad_words": ["blocked"],
"vllm_xargs": {"custom": 1}
"vllm_xargs": {"custom": 1},
"chat_template_kwargs": {"enable_thinking": false}
})
.to_string(),
))
@@ -1502,11 +1501,27 @@ async fn render_chat_response_can_be_submitted_to_generate_unchanged() {
json!(["blocked"])
);
assert_eq!(render_json["sampling_params"]["vllm_xargs"]["custom"], 1);
assert_eq!(
render_json["reasoning_parser_kwargs"],
json!({"chat_template_kwargs": {"enable_thinking": false}})
);
let (chat, engine_task) = test_models_with_engine_outputs_and_backend_inner(
b"engine-render-generate-round-trip",
default_stream_output_specs(),
Some(token_ids),
move |request| {
assert_eq!(
request.prompt_token_ids.as_deref(),
Some(token_ids.as_slice())
);
assert_eq!(
request
.reasoning_parser_kwargs
.as_ref()
.map(|kwargs| &kwargs.chat_template_kwargs["enable_thinking"]),
Some(&json!(false))
);
},
Arc::new(FakeChatBackend::new()),
)
.await;