[Rust Frontend] Pass reasoning controls through render -> generate (#60546)
Signed-off-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
ce4603cad6
commit
5dc9bcd8cd
@@ -64,6 +64,7 @@ impl EngineCoreRequestType {
|
||||
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
|
||||
pub struct ReasoningParserKwargs {
|
||||
/// Effective kwargs visible to the chat template for this request.
|
||||
#[serde(default)]
|
||||
pub chat_template_kwargs: HashMap<String, serde_json::Value>,
|
||||
}
|
||||
|
||||
|
||||
@@ -85,8 +85,8 @@ pub(super) fn prepare_generate_request(
|
||||
data_parallel_rank: ctx.data_parallel_rank,
|
||||
session_id: ctx.session_id,
|
||||
kv_hints: None,
|
||||
reasoning_parser_kwargs: Default::default(),
|
||||
reasoning_ended: None,
|
||||
reasoning_parser_kwargs: request.reasoning_parser_kwargs.unwrap_or_default(),
|
||||
reasoning_ended: request.reasoning_ended,
|
||||
lora_request: lora_resolution.lora_request.clone(),
|
||||
arrival_time: None,
|
||||
};
|
||||
@@ -224,6 +224,54 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_generate_request_forwards_reasoning_controls() {
|
||||
for (controls, expected) in [
|
||||
(json!({}), (json!({"chat_template_kwargs": {}}), None)),
|
||||
(
|
||||
json!({
|
||||
"reasoning_ended": false,
|
||||
"reasoning_parser_kwargs": {
|
||||
"chat_template_kwargs": {"enable_thinking": true}
|
||||
}
|
||||
}),
|
||||
(
|
||||
json!({"chat_template_kwargs": {"enable_thinking": true}}),
|
||||
Some(false),
|
||||
),
|
||||
),
|
||||
(
|
||||
json!({"reasoning_ended": true, "reasoning_parser_kwargs": {}}),
|
||||
(json!({"chat_template_kwargs": {}}), Some(true)),
|
||||
),
|
||||
] {
|
||||
let mut body = json!({
|
||||
"model": "Qwen/Qwen1.5-0.5B-Chat",
|
||||
"token_ids": [11, 22, 33],
|
||||
"sampling_params": {}
|
||||
});
|
||||
body.as_object_mut().unwrap().extend(controls.as_object().unwrap().clone());
|
||||
let request: GenerateRequest = serde_json::from_value(body).expect("parse request");
|
||||
|
||||
let prepared = prepare_generate_request(
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
None,
|
||||
)
|
||||
.expect("prepare");
|
||||
|
||||
assert_eq!(
|
||||
(
|
||||
serde_json::to_value(&prepared.text_request.reasoning_parser_kwargs).unwrap(),
|
||||
prepared.text_request.reasoning_ended,
|
||||
),
|
||||
expected,
|
||||
"controls: {controls}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_generate_request_gates_continuous_usage_on_include_usage() {
|
||||
let request: GenerateRequest = serde_json::from_value(json!({
|
||||
|
||||
@@ -8,6 +8,7 @@ use serde::{Deserialize, Serialize};
|
||||
use serde_json::{Map, Value};
|
||||
use validator::Validate;
|
||||
use vllm_engine_core_client::protocol::output::RequestSpecDecodeMetrics;
|
||||
use vllm_engine_core_client::protocol::request::ReasoningParserKwargs;
|
||||
use vllm_text::SamplingParams;
|
||||
|
||||
use crate::routes::openai::utils::types::{Normalizable, StreamOptions, Usage};
|
||||
@@ -48,6 +49,13 @@ pub struct GenerateRequest {
|
||||
/// Raw multimodal input; server resolves media. Mutually exclusive with `features`.
|
||||
pub content_parts: Option<Vec<MediaContentPart>>,
|
||||
pub return_token_ids: Option<bool>,
|
||||
/// Whether reasoning has ended before the first generated token, as
|
||||
/// resolved by `/render`. `true` applies structured outputs from the first
|
||||
/// token; `None` lets the engine check the prompt with its reasoning parser.
|
||||
pub reasoning_ended: Option<bool>,
|
||||
/// Set by `/render` so the engine-side reasoning parser agrees with the
|
||||
/// frontend on flags such as `enable_thinking`.
|
||||
pub reasoning_parser_kwargs: Option<ReasoningParserKwargs>,
|
||||
#[serde(flatten)]
|
||||
pub other: Map<String, Value>,
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@ use axum::extract::State;
|
||||
use axum::http::{HeaderMap, StatusCode};
|
||||
use axum::routing::{get, post};
|
||||
use thiserror_ext::AsReport as _;
|
||||
use vllm_engine_core_client::protocol::request::ReasoningParserKwargs;
|
||||
use vllm_text::TextRequest;
|
||||
|
||||
use crate::DEFAULT_REQUEST_BODY_LIMIT_BYTES;
|
||||
@@ -106,6 +107,12 @@ fn lower_render_request(
|
||||
ec_transfer_params: None,
|
||||
content_parts: None,
|
||||
return_token_ids: None,
|
||||
// Carry what the chat route would pass to the engine, so the engine
|
||||
// gates structured outputs the same way on `/inference/v1/generate`.
|
||||
reasoning_ended: text_request.reasoning_ended,
|
||||
reasoning_parser_kwargs: (text_request.reasoning_parser_kwargs
|
||||
!= ReasoningParserKwargs::default())
|
||||
.then_some(text_request.reasoning_parser_kwargs),
|
||||
other: Default::default(),
|
||||
};
|
||||
validate_generate_request(&request, &state.served_model_names)?;
|
||||
|
||||
@@ -542,7 +542,7 @@ impl ChatRenderer for FakeChatBackend {
|
||||
Ok(vllm_chat::RenderedPrompt {
|
||||
prompt: Prompt::Text(prompt),
|
||||
media_order: None,
|
||||
effective_template_kwargs: Default::default(),
|
||||
effective_template_kwargs: request.chat_options.template_kwargs.clone(),
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -615,7 +615,7 @@ fn qwen_multimodal_model_info_with_limits(
|
||||
async fn test_models_with_engine_outputs_and_backend_inner(
|
||||
engine_id: impl Into<EngineId>,
|
||||
output_specs: Vec<(Vec<u32>, Option<EngineCoreFinishReason>)>,
|
||||
expected_prompt_token_ids: Option<Vec<u32>>,
|
||||
check_request: impl FnOnce(&EngineCoreRequest) + Send + 'static,
|
||||
backend: Arc<dyn ChatTextBackend>,
|
||||
) -> (ChatLlm, MockEngineTask) {
|
||||
let ipc = IpcNamespace::new().expect("create ipc namespace");
|
||||
@@ -630,12 +630,7 @@ async fn test_models_with_engine_outputs_and_backend_inner(
|
||||
let add = recv_engine_message(dealer).await;
|
||||
let request: EngineCoreRequest =
|
||||
rmp_serde::from_slice(&add[1]).expect("decode request");
|
||||
if let Some(expected_prompt_token_ids) = expected_prompt_token_ids {
|
||||
assert_eq!(
|
||||
request.prompt_token_ids.as_deref(),
|
||||
Some(expected_prompt_token_ids.as_slice())
|
||||
);
|
||||
}
|
||||
check_request(&request);
|
||||
send_outputs(
|
||||
push,
|
||||
engine_outputs_for_request(&request.request_id, output_specs),
|
||||
@@ -667,7 +662,8 @@ async fn test_models_with_engine_outputs_and_backend(
|
||||
output_specs: Vec<(Vec<u32>, Option<EngineCoreFinishReason>)>,
|
||||
backend: Arc<dyn ChatTextBackend>,
|
||||
) -> (ChatLlm, MockEngineTask) {
|
||||
test_models_with_engine_outputs_and_backend_inner(engine_id, output_specs, None, backend).await
|
||||
test_models_with_engine_outputs_and_backend_inner(engine_id, output_specs, |_| {}, backend)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn test_chat_with_engine_outputs(
|
||||
@@ -1440,6 +1436,8 @@ async fn render_completion_returns_generate_request_with_body_request_id() {
|
||||
assert!(json[0].get("stream_options").is_none());
|
||||
assert!(json[0].get("prompt_token_ids").is_none());
|
||||
assert!(json[0].get("mm_features").is_none());
|
||||
assert!(json[0].get("reasoning_ended").is_none());
|
||||
assert!(json[0].get("reasoning_parser_kwargs").is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -1479,7 +1477,8 @@ async fn render_chat_response_can_be_submitted_to_generate_unchanged() {
|
||||
"messages": [{"role": "user", "content": "hello"}],
|
||||
"max_completion_tokens": 8,
|
||||
"bad_words": ["blocked"],
|
||||
"vllm_xargs": {"custom": 1}
|
||||
"vllm_xargs": {"custom": 1},
|
||||
"chat_template_kwargs": {"enable_thinking": false}
|
||||
})
|
||||
.to_string(),
|
||||
))
|
||||
@@ -1502,11 +1501,27 @@ async fn render_chat_response_can_be_submitted_to_generate_unchanged() {
|
||||
json!(["blocked"])
|
||||
);
|
||||
assert_eq!(render_json["sampling_params"]["vllm_xargs"]["custom"], 1);
|
||||
assert_eq!(
|
||||
render_json["reasoning_parser_kwargs"],
|
||||
json!({"chat_template_kwargs": {"enable_thinking": false}})
|
||||
);
|
||||
|
||||
let (chat, engine_task) = test_models_with_engine_outputs_and_backend_inner(
|
||||
b"engine-render-generate-round-trip",
|
||||
default_stream_output_specs(),
|
||||
Some(token_ids),
|
||||
move |request| {
|
||||
assert_eq!(
|
||||
request.prompt_token_ids.as_deref(),
|
||||
Some(token_ids.as_slice())
|
||||
);
|
||||
assert_eq!(
|
||||
request
|
||||
.reasoning_parser_kwargs
|
||||
.as_ref()
|
||||
.map(|kwargs| &kwargs.chat_template_kwargs["enable_thinking"]),
|
||||
Some(&json!(false))
|
||||
);
|
||||
},
|
||||
Arc::new(FakeChatBackend::new()),
|
||||
)
|
||||
.await;
|
||||
|
||||
Reference in New Issue
Block a user