[Rust Frontend] Support thinking_token_budget for chat and completions (#46137)

Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Signed-off-by: RickyChen / 陳昭儒 <ricky.chen@infinirc.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Chao-Ju Chen
2026-06-22 16:00:02 +08:00
committed by GitHub
co-authored by Bugen Zhao mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent a9f7b2d41c
commit 80abe0de7d
13 changed files with 178 additions and 7 deletions
+13
View File
@@ -103,6 +103,7 @@ fn is_request_validation_error(error: &vllm_text::Error) -> bool {
| vllm_text::Error::EmptyPromptTokenIds { .. }
| vllm_text::Error::Logprobs(_)
| vllm_text::Error::OutOfVocab(_)
| vllm_text::Error::InvalidThinkingTokenBudget
// An empty tokenized prompt detected later, at request prepare
// time, surfaces through the transparent Llm wrapper.
| vllm_text::Error::Llm(vllm_llm::Error::EmptyPromptTokenIds { .. })
@@ -127,6 +128,18 @@ mod tests {
assert!(response.error.message.contains("9000"));
}
#[test]
fn invalid_thinking_token_budget_maps_to_invalid_request() {
let api_error = text_submit_error(
"failed to submit completion request",
vllm_text::Error::InvalidThinkingTokenBudget,
);
assert_eq!(api_error.status_code(), StatusCode::BAD_REQUEST);
let response = api_error.to_error_response();
assert_eq!(response.error.error_type, "invalid_request_error");
assert!(response.error.message.contains("thinking_token_budget"));
}
#[test]
fn chat_wrapped_prompt_too_long_maps_to_invalid_request() {
let error = vllm_chat::Error::Text(vllm_text::Error::PromptTooLong {
@@ -150,6 +150,33 @@ mod tests {
);
}
#[test]
fn prepare_generate_request_forwards_thinking_token_budget() {
let request: GenerateRequest = serde_json::from_value(json!({
"model": "Qwen/Qwen1.5-0.5B-Chat",
"token_ids": [11, 22, 33],
"sampling_params": {
"thinking_token_budget": 64
}
}))
.expect("parse request");
let prepared = prepare_generate_request(
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
)
.expect("prepare");
// The raw inference route shares `vllm_text::SamplingParams`, so the
// field is carried through to lowering exactly like the OpenAI routes
// (normalization/validation then happens in `lower_sampling_params`).
assert_eq!(
prepared.text_request.sampling_params.thinking_token_budget,
Some(64)
);
}
#[test]
fn prepare_generate_request_gates_continuous_usage_on_include_usage() {
let request: GenerateRequest = serde_json::from_value(json!({
@@ -115,6 +115,7 @@ pub(super) fn prepare_chat_request(
seed: request.seed,
max_tokens: request.max_completion_tokens,
min_tokens: request.min_tokens,
thinking_token_budget: request.thinking_token_budget,
logprobs: request.logprobs.then_some(top_logprobs),
prompt_logprobs,
min_p: request.min_p,
@@ -613,6 +614,31 @@ mod tests {
assert_eq!(prepared.chat_request.sampling_params, expected);
}
#[test]
fn prepare_chat_request_passes_through_thinking_token_budget() {
let prepare = |budget: Option<i64>| {
prepare_chat_request(
ChatCompletionRequest {
thinking_token_budget: budget,
..base_request()
},
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
)
.expect("request is valid")
.chat_request
.sampling_params
.thinking_token_budget
};
// The convert layer forwards the raw value verbatim (including the `-1`
// "unlimited" sentinel); normalization/validation happens during
// lowering (see `vllm_text::lower`).
assert_eq!(prepare(Some(64)), Some(64));
assert_eq!(prepare(Some(-1)), Some(-1));
assert_eq!(prepare(None), None);
}
#[test]
fn prepare_chat_request_accepts_developer_messages() {
let request = ChatCompletionRequest {
@@ -165,8 +165,10 @@ pub struct ChatCompletionRequest {
pub bad_words: Option<Vec<String>>,
// -------- Extra vLLM Parameters --------
/// Token budget for reasoning/thinking
pub thinking_token_budget: Option<u32>,
/// Token budget for reasoning/thinking. Accepts a non-negative integer, or
/// `-1` for unlimited (mirroring the Python frontend, which normalizes `-1`
/// to "no budget").
pub thinking_token_budget: Option<i64>,
/// Whether to include reasoning content in the response
#[serde(default = "default_true")]
@@ -108,11 +108,6 @@ pub(super) fn validate_request_compat(
"truncate_prompt_tokens",
"truncate_prompt_tokens is not supported.",
)?;
reject_non_default(
request.thinking_token_budget.as_ref(),
"thinking_token_budget",
"thinking_token_budget is not supported.",
)?;
reject_non_default(
request.media_io_kwargs.as_ref(),
"media_io_kwargs",
@@ -108,6 +108,7 @@ pub(super) fn prepare_completion_request(
seed: request.seed,
max_tokens,
min_tokens: request.min_tokens,
thinking_token_budget: request.thinking_token_budget,
logprobs,
prompt_logprobs,
min_p: request.min_p,
@@ -266,6 +267,34 @@ mod tests {
assert!(!prepared.text_request.decode_options.skip_special_tokens);
}
#[test]
fn prepare_completion_request_passes_through_thinking_token_budget() {
let prepare = |budget: serde_json::Value| {
let request: CompletionRequest = serde_json::from_value(json!({
"model": "Qwen/Qwen1.5-0.5B-Chat",
"prompt": "hello",
"thinking_token_budget": budget,
}))
.expect("parse request");
prepare_completion_request(
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
)
.expect("prepare")
.text_request
.sampling_params
.thinking_token_budget
};
// The convert layer forwards the raw value verbatim (including the `-1`
// "unlimited" sentinel); normalization/validation happens during
// lowering (see `vllm_text::lower`).
assert_eq!(prepare(json!(64)), Some(64));
assert_eq!(prepare(json!(-1)), Some(-1));
assert_eq!(prepare(json!(null)), None);
}
#[test]
fn prepare_completion_request_maps_stream_usage_and_token_format_options() {
let request: CompletionRequest = serde_json::from_value(json!({
@@ -146,6 +146,11 @@ pub struct CompletionRequest {
/// Additional kwargs for structured outputs
pub structured_outputs: Option<Value>,
/// Token budget for reasoning/thinking. Accepts a non-negative integer, or
/// `-1` for unlimited (mirroring the Python frontend, which normalizes `-1`
/// to "no budget").
pub thinking_token_budget: Option<i64>,
/// Request scheduling priority (lower means earlier; default 0)
pub priority: Option<i32>,