[Rust Frontend] Support thinking_token_budget for chat and completions (#46137)
Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Signed-off-by: RickyChen / 陳昭儒 <ricky.chen@infinirc.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
co-authored by
Bugen Zhao
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
a9f7b2d41c
commit
80abe0de7d
@@ -103,6 +103,7 @@ fn is_request_validation_error(error: &vllm_text::Error) -> bool {
|
||||
| vllm_text::Error::EmptyPromptTokenIds { .. }
|
||||
| vllm_text::Error::Logprobs(_)
|
||||
| vllm_text::Error::OutOfVocab(_)
|
||||
| vllm_text::Error::InvalidThinkingTokenBudget
|
||||
// An empty tokenized prompt detected later, at request prepare
|
||||
// time, surfaces through the transparent Llm wrapper.
|
||||
| vllm_text::Error::Llm(vllm_llm::Error::EmptyPromptTokenIds { .. })
|
||||
@@ -127,6 +128,18 @@ mod tests {
|
||||
assert!(response.error.message.contains("9000"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_thinking_token_budget_maps_to_invalid_request() {
|
||||
let api_error = text_submit_error(
|
||||
"failed to submit completion request",
|
||||
vllm_text::Error::InvalidThinkingTokenBudget,
|
||||
);
|
||||
assert_eq!(api_error.status_code(), StatusCode::BAD_REQUEST);
|
||||
let response = api_error.to_error_response();
|
||||
assert_eq!(response.error.error_type, "invalid_request_error");
|
||||
assert!(response.error.message.contains("thinking_token_budget"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chat_wrapped_prompt_too_long_maps_to_invalid_request() {
|
||||
let error = vllm_chat::Error::Text(vllm_text::Error::PromptTooLong {
|
||||
|
||||
@@ -150,6 +150,33 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_generate_request_forwards_thinking_token_budget() {
|
||||
let request: GenerateRequest = serde_json::from_value(json!({
|
||||
"model": "Qwen/Qwen1.5-0.5B-Chat",
|
||||
"token_ids": [11, 22, 33],
|
||||
"sampling_params": {
|
||||
"thinking_token_budget": 64
|
||||
}
|
||||
}))
|
||||
.expect("parse request");
|
||||
|
||||
let prepared = prepare_generate_request(
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
)
|
||||
.expect("prepare");
|
||||
|
||||
// The raw inference route shares `vllm_text::SamplingParams`, so the
|
||||
// field is carried through to lowering exactly like the OpenAI routes
|
||||
// (normalization/validation then happens in `lower_sampling_params`).
|
||||
assert_eq!(
|
||||
prepared.text_request.sampling_params.thinking_token_budget,
|
||||
Some(64)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_generate_request_gates_continuous_usage_on_include_usage() {
|
||||
let request: GenerateRequest = serde_json::from_value(json!({
|
||||
|
||||
@@ -115,6 +115,7 @@ pub(super) fn prepare_chat_request(
|
||||
seed: request.seed,
|
||||
max_tokens: request.max_completion_tokens,
|
||||
min_tokens: request.min_tokens,
|
||||
thinking_token_budget: request.thinking_token_budget,
|
||||
logprobs: request.logprobs.then_some(top_logprobs),
|
||||
prompt_logprobs,
|
||||
min_p: request.min_p,
|
||||
@@ -613,6 +614,31 @@ mod tests {
|
||||
assert_eq!(prepared.chat_request.sampling_params, expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_chat_request_passes_through_thinking_token_budget() {
|
||||
let prepare = |budget: Option<i64>| {
|
||||
prepare_chat_request(
|
||||
ChatCompletionRequest {
|
||||
thinking_token_budget: budget,
|
||||
..base_request()
|
||||
},
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
)
|
||||
.expect("request is valid")
|
||||
.chat_request
|
||||
.sampling_params
|
||||
.thinking_token_budget
|
||||
};
|
||||
|
||||
// The convert layer forwards the raw value verbatim (including the `-1`
|
||||
// "unlimited" sentinel); normalization/validation happens during
|
||||
// lowering (see `vllm_text::lower`).
|
||||
assert_eq!(prepare(Some(64)), Some(64));
|
||||
assert_eq!(prepare(Some(-1)), Some(-1));
|
||||
assert_eq!(prepare(None), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_chat_request_accepts_developer_messages() {
|
||||
let request = ChatCompletionRequest {
|
||||
|
||||
@@ -165,8 +165,10 @@ pub struct ChatCompletionRequest {
|
||||
pub bad_words: Option<Vec<String>>,
|
||||
|
||||
// -------- Extra vLLM Parameters --------
|
||||
/// Token budget for reasoning/thinking
|
||||
pub thinking_token_budget: Option<u32>,
|
||||
/// Token budget for reasoning/thinking. Accepts a non-negative integer, or
|
||||
/// `-1` for unlimited (mirroring the Python frontend, which normalizes `-1`
|
||||
/// to "no budget").
|
||||
pub thinking_token_budget: Option<i64>,
|
||||
|
||||
/// Whether to include reasoning content in the response
|
||||
#[serde(default = "default_true")]
|
||||
|
||||
@@ -108,11 +108,6 @@ pub(super) fn validate_request_compat(
|
||||
"truncate_prompt_tokens",
|
||||
"truncate_prompt_tokens is not supported.",
|
||||
)?;
|
||||
reject_non_default(
|
||||
request.thinking_token_budget.as_ref(),
|
||||
"thinking_token_budget",
|
||||
"thinking_token_budget is not supported.",
|
||||
)?;
|
||||
reject_non_default(
|
||||
request.media_io_kwargs.as_ref(),
|
||||
"media_io_kwargs",
|
||||
|
||||
@@ -108,6 +108,7 @@ pub(super) fn prepare_completion_request(
|
||||
seed: request.seed,
|
||||
max_tokens,
|
||||
min_tokens: request.min_tokens,
|
||||
thinking_token_budget: request.thinking_token_budget,
|
||||
logprobs,
|
||||
prompt_logprobs,
|
||||
min_p: request.min_p,
|
||||
@@ -266,6 +267,34 @@ mod tests {
|
||||
assert!(!prepared.text_request.decode_options.skip_special_tokens);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_completion_request_passes_through_thinking_token_budget() {
|
||||
let prepare = |budget: serde_json::Value| {
|
||||
let request: CompletionRequest = serde_json::from_value(json!({
|
||||
"model": "Qwen/Qwen1.5-0.5B-Chat",
|
||||
"prompt": "hello",
|
||||
"thinking_token_budget": budget,
|
||||
}))
|
||||
.expect("parse request");
|
||||
prepare_completion_request(
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
)
|
||||
.expect("prepare")
|
||||
.text_request
|
||||
.sampling_params
|
||||
.thinking_token_budget
|
||||
};
|
||||
|
||||
// The convert layer forwards the raw value verbatim (including the `-1`
|
||||
// "unlimited" sentinel); normalization/validation happens during
|
||||
// lowering (see `vllm_text::lower`).
|
||||
assert_eq!(prepare(json!(64)), Some(64));
|
||||
assert_eq!(prepare(json!(-1)), Some(-1));
|
||||
assert_eq!(prepare(json!(null)), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_completion_request_maps_stream_usage_and_token_format_options() {
|
||||
let request: CompletionRequest = serde_json::from_value(json!({
|
||||
|
||||
@@ -146,6 +146,11 @@ pub struct CompletionRequest {
|
||||
/// Additional kwargs for structured outputs
|
||||
pub structured_outputs: Option<Value>,
|
||||
|
||||
/// Token budget for reasoning/thinking. Accepts a non-negative integer, or
|
||||
/// `-1` for unlimited (mirroring the Python frontend, which normalizes `-1`
|
||||
/// to "no budget").
|
||||
pub thinking_token_budget: Option<i64>,
|
||||
|
||||
/// Request scheduling priority (lower means earlier; default 0)
|
||||
pub priority: Option<i32>,
|
||||
|
||||
|
||||
Reference in New Issue
Block a user