From 5dc9bcd8cd9656b852edfd6c5b4a8c9c46a34530 Mon Sep 17 00:00:00 2001 From: Bugen Zhao Date: Sat, 10 Oct 2026 02:04:49 +0800 Subject: [PATCH] [Rust Frontend] Pass reasoning controls through render -> generate (#60546) Signed-off-by: Bugen Zhao Co-authored-by: Claude Opus 5.5 --- .../src/protocol/request.rs | 1 + .../src/routes/inference/generate/convert.rs | 52 ++++++++++++++++++- .../src/routes/inference/generate/types.rs | 8 +++ rust/src/server/src/routes/render.rs | 7 +++ rust/src/server/src/routes/tests.rs | 37 +++++++++---- 5 files changed, 92 insertions(+), 13 deletions(-) diff --git a/rust/src/engine-core-client/src/protocol/request.rs b/rust/src/engine-core-client/src/protocol/request.rs index 02fc1c30ca..5390eed16f 100644 --- a/rust/src/engine-core-client/src/protocol/request.rs +++ b/rust/src/engine-core-client/src/protocol/request.rs @@ -64,6 +64,7 @@ impl EngineCoreRequestType { #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] pub struct ReasoningParserKwargs { /// Effective kwargs visible to the chat template for this request. + #[serde(default)] pub chat_template_kwargs: HashMap, } diff --git a/rust/src/server/src/routes/inference/generate/convert.rs b/rust/src/server/src/routes/inference/generate/convert.rs index 36a497f6dd..313e9fa1f9 100644 --- a/rust/src/server/src/routes/inference/generate/convert.rs +++ b/rust/src/server/src/routes/inference/generate/convert.rs @@ -85,8 +85,8 @@ pub(super) fn prepare_generate_request( data_parallel_rank: ctx.data_parallel_rank, session_id: ctx.session_id, kv_hints: None, - reasoning_parser_kwargs: Default::default(), - reasoning_ended: None, + reasoning_parser_kwargs: request.reasoning_parser_kwargs.unwrap_or_default(), + reasoning_ended: request.reasoning_ended, lora_request: lora_resolution.lora_request.clone(), arrival_time: None, }; @@ -224,6 +224,54 @@ mod tests { ); } + #[test] + fn prepare_generate_request_forwards_reasoning_controls() { + for (controls, expected) in [ + (json!({}), (json!({"chat_template_kwargs": {}}), None)), + ( + json!({ + "reasoning_ended": false, + "reasoning_parser_kwargs": { + "chat_template_kwargs": {"enable_thinking": true} + } + }), + ( + json!({"chat_template_kwargs": {"enable_thinking": true}}), + Some(false), + ), + ), + ( + json!({"reasoning_ended": true, "reasoning_parser_kwargs": {}}), + (json!({"chat_template_kwargs": {}}), Some(true)), + ), + ] { + let mut body = json!({ + "model": "Qwen/Qwen1.5-0.5B-Chat", + "token_ids": [11, 22, 33], + "sampling_params": {} + }); + body.as_object_mut().unwrap().extend(controls.as_object().unwrap().clone()); + let request: GenerateRequest = serde_json::from_value(body).expect("parse request"); + + let prepared = prepare_generate_request( + request, + &served(&["Qwen/Qwen1.5-0.5B-Chat"]), + ResolvedRequestContext::default(), + None, + ) + .expect("prepare"); + + assert_eq!( + ( + serde_json::to_value(&prepared.text_request.reasoning_parser_kwargs).unwrap(), + prepared.text_request.reasoning_ended, + ), + expected, + "controls: {controls}" + ); + } + } + #[test] fn prepare_generate_request_gates_continuous_usage_on_include_usage() { let request: GenerateRequest = serde_json::from_value(json!({ diff --git a/rust/src/server/src/routes/inference/generate/types.rs b/rust/src/server/src/routes/inference/generate/types.rs index bd277ef34a..0a37640d63 100644 --- a/rust/src/server/src/routes/inference/generate/types.rs +++ b/rust/src/server/src/routes/inference/generate/types.rs @@ -8,6 +8,7 @@ use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; use validator::Validate; use vllm_engine_core_client::protocol::output::RequestSpecDecodeMetrics; +use vllm_engine_core_client::protocol::request::ReasoningParserKwargs; use vllm_text::SamplingParams; use crate::routes::openai::utils::types::{Normalizable, StreamOptions, Usage}; @@ -48,6 +49,13 @@ pub struct GenerateRequest { /// Raw multimodal input; server resolves media. Mutually exclusive with `features`. pub content_parts: Option>, pub return_token_ids: Option, + /// Whether reasoning has ended before the first generated token, as + /// resolved by `/render`. `true` applies structured outputs from the first + /// token; `None` lets the engine check the prompt with its reasoning parser. + pub reasoning_ended: Option, + /// Set by `/render` so the engine-side reasoning parser agrees with the + /// frontend on flags such as `enable_thinking`. + pub reasoning_parser_kwargs: Option, #[serde(flatten)] pub other: Map, } diff --git a/rust/src/server/src/routes/render.rs b/rust/src/server/src/routes/render.rs index 2a6f70ec4b..aceb28f499 100644 --- a/rust/src/server/src/routes/render.rs +++ b/rust/src/server/src/routes/render.rs @@ -10,6 +10,7 @@ use axum::extract::State; use axum::http::{HeaderMap, StatusCode}; use axum::routing::{get, post}; use thiserror_ext::AsReport as _; +use vllm_engine_core_client::protocol::request::ReasoningParserKwargs; use vllm_text::TextRequest; use crate::DEFAULT_REQUEST_BODY_LIMIT_BYTES; @@ -106,6 +107,12 @@ fn lower_render_request( ec_transfer_params: None, content_parts: None, return_token_ids: None, + // Carry what the chat route would pass to the engine, so the engine + // gates structured outputs the same way on `/inference/v1/generate`. + reasoning_ended: text_request.reasoning_ended, + reasoning_parser_kwargs: (text_request.reasoning_parser_kwargs + != ReasoningParserKwargs::default()) + .then_some(text_request.reasoning_parser_kwargs), other: Default::default(), }; validate_generate_request(&request, &state.served_model_names)?; diff --git a/rust/src/server/src/routes/tests.rs b/rust/src/server/src/routes/tests.rs index a9c9831bbd..118e412b45 100644 --- a/rust/src/server/src/routes/tests.rs +++ b/rust/src/server/src/routes/tests.rs @@ -542,7 +542,7 @@ impl ChatRenderer for FakeChatBackend { Ok(vllm_chat::RenderedPrompt { prompt: Prompt::Text(prompt), media_order: None, - effective_template_kwargs: Default::default(), + effective_template_kwargs: request.chat_options.template_kwargs.clone(), }) } } @@ -615,7 +615,7 @@ fn qwen_multimodal_model_info_with_limits( async fn test_models_with_engine_outputs_and_backend_inner( engine_id: impl Into, output_specs: Vec<(Vec, Option)>, - expected_prompt_token_ids: Option>, + check_request: impl FnOnce(&EngineCoreRequest) + Send + 'static, backend: Arc, ) -> (ChatLlm, MockEngineTask) { let ipc = IpcNamespace::new().expect("create ipc namespace"); @@ -630,12 +630,7 @@ async fn test_models_with_engine_outputs_and_backend_inner( let add = recv_engine_message(dealer).await; let request: EngineCoreRequest = rmp_serde::from_slice(&add[1]).expect("decode request"); - if let Some(expected_prompt_token_ids) = expected_prompt_token_ids { - assert_eq!( - request.prompt_token_ids.as_deref(), - Some(expected_prompt_token_ids.as_slice()) - ); - } + check_request(&request); send_outputs( push, engine_outputs_for_request(&request.request_id, output_specs), @@ -667,7 +662,8 @@ async fn test_models_with_engine_outputs_and_backend( output_specs: Vec<(Vec, Option)>, backend: Arc, ) -> (ChatLlm, MockEngineTask) { - test_models_with_engine_outputs_and_backend_inner(engine_id, output_specs, None, backend).await + test_models_with_engine_outputs_and_backend_inner(engine_id, output_specs, |_| {}, backend) + .await } async fn test_chat_with_engine_outputs( @@ -1440,6 +1436,8 @@ async fn render_completion_returns_generate_request_with_body_request_id() { assert!(json[0].get("stream_options").is_none()); assert!(json[0].get("prompt_token_ids").is_none()); assert!(json[0].get("mm_features").is_none()); + assert!(json[0].get("reasoning_ended").is_none()); + assert!(json[0].get("reasoning_parser_kwargs").is_none()); } #[tokio::test] @@ -1479,7 +1477,8 @@ async fn render_chat_response_can_be_submitted_to_generate_unchanged() { "messages": [{"role": "user", "content": "hello"}], "max_completion_tokens": 8, "bad_words": ["blocked"], - "vllm_xargs": {"custom": 1} + "vllm_xargs": {"custom": 1}, + "chat_template_kwargs": {"enable_thinking": false} }) .to_string(), )) @@ -1502,11 +1501,27 @@ async fn render_chat_response_can_be_submitted_to_generate_unchanged() { json!(["blocked"]) ); assert_eq!(render_json["sampling_params"]["vllm_xargs"]["custom"], 1); + assert_eq!( + render_json["reasoning_parser_kwargs"], + json!({"chat_template_kwargs": {"enable_thinking": false}}) + ); let (chat, engine_task) = test_models_with_engine_outputs_and_backend_inner( b"engine-render-generate-round-trip", default_stream_output_specs(), - Some(token_ids), + move |request| { + assert_eq!( + request.prompt_token_ids.as_deref(), + Some(token_ids.as_slice()) + ); + assert_eq!( + request + .reasoning_parser_kwargs + .as_ref() + .map(|kwargs| &kwargs.chat_template_kwargs["enable_thinking"]), + Some(&json!(false)) + ); + }, Arc::new(FakeChatBackend::new()), ) .await;