diff --git a/rust/src/engine-core-client/src/protocol/mod.rs b/rust/src/engine-core-client/src/protocol/mod.rs index 5e340b9117..8862b077c9 100644 --- a/rust/src/engine-core-client/src/protocol/mod.rs +++ b/rust/src/engine-core-client/src/protocol/mod.rs @@ -277,6 +277,12 @@ pub struct EngineCoreSamplingParams { pub max_tokens: u32, /// Minimum number of tokens to generate before EOS or stop-token handling. pub min_tokens: u32, + /// Maximum number of reasoning ("thinking") tokens to emit before the + /// reasoning section is force-closed. `None` means unlimited; the + /// user-facing `-1` sentinel is normalized to `None` by the frontend before + /// reaching this DTO, so only non-negative values are sent. Enforced + /// engine-side (and only when a reasoning parser is configured). + pub thinking_token_budget: Option, /// Number of log probabilities to return per generated token. /// /// `None` disables sample logprobs. `-1` requests the full vocabulary. @@ -345,6 +351,7 @@ impl EngineCoreSamplingParams { seed: None, max_tokens: 65536, min_tokens: 0, + thinking_token_budget: None, logprobs: None, prompt_logprobs: None, min_p: 0.0, diff --git a/rust/src/engine-core-client/src/tests/client.rs b/rust/src/engine-core-client/src/tests/client.rs index c00a422685..a7ccd59816 100644 --- a/rust/src/engine-core-client/src/tests/client.rs +++ b/rust/src/engine-core-client/src/tests/client.rs @@ -150,6 +150,7 @@ fn sample_request_with_id(request_id: &str) -> EngineCoreRequest { top_k: 8, max_tokens: 32, min_tokens: 1, + thinking_token_budget: Some(256), stop_token_ids: vec![151643], eos_token_id: Some(151645), all_stop_token_ids: BTreeSet::from([151643, 151645]), @@ -2502,6 +2503,7 @@ fn python_msgpack_fixtures_match_rust_encoding() { seed: None, max_tokens: 16, min_tokens: 0, + thinking_token_budget: None, logprobs: None, prompt_logprobs: None, min_p: 0.0, diff --git a/rust/src/engine-core-client/src/tests/python_compat.py b/rust/src/engine-core-client/src/tests/python_compat.py index ba4f7daa3d..a3f44ea7f0 100755 --- a/rust/src/engine-core-client/src/tests/python_compat.py +++ b/rust/src/engine-core-client/src/tests/python_compat.py @@ -39,6 +39,7 @@ class EngineCoreSamplingParams(msgspec.Struct, dict=True, omit_defaults=True): seed: int | None = None max_tokens: int = 16 min_tokens: int = 0 + thinking_token_budget: int | None = None min_p: float = 0.0 frequency_penalty: float = 0.0 presence_penalty: float = 0.0 @@ -122,6 +123,7 @@ request = EngineCoreRequest( seed=None, max_tokens=32, min_tokens=1, + thinking_token_budget=256, min_p=0.0, frequency_penalty=0.0, presence_penalty=0.0, diff --git a/rust/src/server/src/error.rs b/rust/src/server/src/error.rs index e5a5c1a40d..b32b281fa9 100644 --- a/rust/src/server/src/error.rs +++ b/rust/src/server/src/error.rs @@ -103,6 +103,7 @@ fn is_request_validation_error(error: &vllm_text::Error) -> bool { | vllm_text::Error::EmptyPromptTokenIds { .. } | vllm_text::Error::Logprobs(_) | vllm_text::Error::OutOfVocab(_) + | vllm_text::Error::InvalidThinkingTokenBudget // An empty tokenized prompt detected later, at request prepare // time, surfaces through the transparent Llm wrapper. | vllm_text::Error::Llm(vllm_llm::Error::EmptyPromptTokenIds { .. }) @@ -127,6 +128,18 @@ mod tests { assert!(response.error.message.contains("9000")); } + #[test] + fn invalid_thinking_token_budget_maps_to_invalid_request() { + let api_error = text_submit_error( + "failed to submit completion request", + vllm_text::Error::InvalidThinkingTokenBudget, + ); + assert_eq!(api_error.status_code(), StatusCode::BAD_REQUEST); + let response = api_error.to_error_response(); + assert_eq!(response.error.error_type, "invalid_request_error"); + assert!(response.error.message.contains("thinking_token_budget")); + } + #[test] fn chat_wrapped_prompt_too_long_maps_to_invalid_request() { let error = vllm_chat::Error::Text(vllm_text::Error::PromptTooLong { diff --git a/rust/src/server/src/routes/inference/generate/convert.rs b/rust/src/server/src/routes/inference/generate/convert.rs index 73bca4a1f8..df3ba33735 100644 --- a/rust/src/server/src/routes/inference/generate/convert.rs +++ b/rust/src/server/src/routes/inference/generate/convert.rs @@ -150,6 +150,33 @@ mod tests { ); } + #[test] + fn prepare_generate_request_forwards_thinking_token_budget() { + let request: GenerateRequest = serde_json::from_value(json!({ + "model": "Qwen/Qwen1.5-0.5B-Chat", + "token_ids": [11, 22, 33], + "sampling_params": { + "thinking_token_budget": 64 + } + })) + .expect("parse request"); + + let prepared = prepare_generate_request( + request, + &served(&["Qwen/Qwen1.5-0.5B-Chat"]), + ResolvedRequestContext::default(), + ) + .expect("prepare"); + + // The raw inference route shares `vllm_text::SamplingParams`, so the + // field is carried through to lowering exactly like the OpenAI routes + // (normalization/validation then happens in `lower_sampling_params`). + assert_eq!( + prepared.text_request.sampling_params.thinking_token_budget, + Some(64) + ); + } + #[test] fn prepare_generate_request_gates_continuous_usage_on_include_usage() { let request: GenerateRequest = serde_json::from_value(json!({ diff --git a/rust/src/server/src/routes/openai/chat_completions/convert.rs b/rust/src/server/src/routes/openai/chat_completions/convert.rs index bc581842da..0462294b0c 100644 --- a/rust/src/server/src/routes/openai/chat_completions/convert.rs +++ b/rust/src/server/src/routes/openai/chat_completions/convert.rs @@ -115,6 +115,7 @@ pub(super) fn prepare_chat_request( seed: request.seed, max_tokens: request.max_completion_tokens, min_tokens: request.min_tokens, + thinking_token_budget: request.thinking_token_budget, logprobs: request.logprobs.then_some(top_logprobs), prompt_logprobs, min_p: request.min_p, @@ -613,6 +614,31 @@ mod tests { assert_eq!(prepared.chat_request.sampling_params, expected); } + #[test] + fn prepare_chat_request_passes_through_thinking_token_budget() { + let prepare = |budget: Option| { + prepare_chat_request( + ChatCompletionRequest { + thinking_token_budget: budget, + ..base_request() + }, + &served(&["Qwen/Qwen1.5-0.5B-Chat"]), + ResolvedRequestContext::default(), + ) + .expect("request is valid") + .chat_request + .sampling_params + .thinking_token_budget + }; + + // The convert layer forwards the raw value verbatim (including the `-1` + // "unlimited" sentinel); normalization/validation happens during + // lowering (see `vllm_text::lower`). + assert_eq!(prepare(Some(64)), Some(64)); + assert_eq!(prepare(Some(-1)), Some(-1)); + assert_eq!(prepare(None), None); + } + #[test] fn prepare_chat_request_accepts_developer_messages() { let request = ChatCompletionRequest { diff --git a/rust/src/server/src/routes/openai/chat_completions/types.rs b/rust/src/server/src/routes/openai/chat_completions/types.rs index 3efef62213..a76a52b8ad 100644 --- a/rust/src/server/src/routes/openai/chat_completions/types.rs +++ b/rust/src/server/src/routes/openai/chat_completions/types.rs @@ -165,8 +165,10 @@ pub struct ChatCompletionRequest { pub bad_words: Option>, // -------- Extra vLLM Parameters -------- - /// Token budget for reasoning/thinking - pub thinking_token_budget: Option, + /// Token budget for reasoning/thinking. Accepts a non-negative integer, or + /// `-1` for unlimited (mirroring the Python frontend, which normalizes `-1` + /// to "no budget"). + pub thinking_token_budget: Option, /// Whether to include reasoning content in the response #[serde(default = "default_true")] diff --git a/rust/src/server/src/routes/openai/chat_completions/validate.rs b/rust/src/server/src/routes/openai/chat_completions/validate.rs index bbf32c6950..2e789573d2 100644 --- a/rust/src/server/src/routes/openai/chat_completions/validate.rs +++ b/rust/src/server/src/routes/openai/chat_completions/validate.rs @@ -108,11 +108,6 @@ pub(super) fn validate_request_compat( "truncate_prompt_tokens", "truncate_prompt_tokens is not supported.", )?; - reject_non_default( - request.thinking_token_budget.as_ref(), - "thinking_token_budget", - "thinking_token_budget is not supported.", - )?; reject_non_default( request.media_io_kwargs.as_ref(), "media_io_kwargs", diff --git a/rust/src/server/src/routes/openai/completions/convert.rs b/rust/src/server/src/routes/openai/completions/convert.rs index 9c30692859..517bb93b0d 100644 --- a/rust/src/server/src/routes/openai/completions/convert.rs +++ b/rust/src/server/src/routes/openai/completions/convert.rs @@ -108,6 +108,7 @@ pub(super) fn prepare_completion_request( seed: request.seed, max_tokens, min_tokens: request.min_tokens, + thinking_token_budget: request.thinking_token_budget, logprobs, prompt_logprobs, min_p: request.min_p, @@ -266,6 +267,34 @@ mod tests { assert!(!prepared.text_request.decode_options.skip_special_tokens); } + #[test] + fn prepare_completion_request_passes_through_thinking_token_budget() { + let prepare = |budget: serde_json::Value| { + let request: CompletionRequest = serde_json::from_value(json!({ + "model": "Qwen/Qwen1.5-0.5B-Chat", + "prompt": "hello", + "thinking_token_budget": budget, + })) + .expect("parse request"); + prepare_completion_request( + request, + &served(&["Qwen/Qwen1.5-0.5B-Chat"]), + ResolvedRequestContext::default(), + ) + .expect("prepare") + .text_request + .sampling_params + .thinking_token_budget + }; + + // The convert layer forwards the raw value verbatim (including the `-1` + // "unlimited" sentinel); normalization/validation happens during + // lowering (see `vllm_text::lower`). + assert_eq!(prepare(json!(64)), Some(64)); + assert_eq!(prepare(json!(-1)), Some(-1)); + assert_eq!(prepare(json!(null)), None); + } + #[test] fn prepare_completion_request_maps_stream_usage_and_token_format_options() { let request: CompletionRequest = serde_json::from_value(json!({ diff --git a/rust/src/server/src/routes/openai/completions/types.rs b/rust/src/server/src/routes/openai/completions/types.rs index adc8a7ba7c..93005cd8b8 100644 --- a/rust/src/server/src/routes/openai/completions/types.rs +++ b/rust/src/server/src/routes/openai/completions/types.rs @@ -146,6 +146,11 @@ pub struct CompletionRequest { /// Additional kwargs for structured outputs pub structured_outputs: Option, + /// Token budget for reasoning/thinking. Accepts a non-negative integer, or + /// `-1` for unlimited (mirroring the Python frontend, which normalizes `-1` + /// to "no budget"). + pub thinking_token_budget: Option, + /// Request scheduling priority (lower means earlier; default 0) pub priority: Option, diff --git a/rust/src/text/src/error.rs b/rust/src/text/src/error.rs index f686e56d52..b98d990b93 100644 --- a/rust/src/text/src/error.rs +++ b/rust/src/text/src/error.rs @@ -20,6 +20,8 @@ pub enum Error { Logprobs(#[from] LogprobsError), #[error(transparent)] OutOfVocab(#[from] OutOfVocabError), + #[error("`thinking_token_budget` must be a non-negative integer or -1 for unlimited.")] + InvalidThinkingTokenBudget, #[error("text request stream `{request_id}` closed before terminal output")] StreamClosedBeforeTerminalOutput { request_id: String }, #[error(transparent)] diff --git a/rust/src/text/src/lower.rs b/rust/src/text/src/lower.rs index 077dfcb980..c5f1e67568 100644 --- a/rust/src/text/src/lower.rs +++ b/rust/src/text/src/lower.rs @@ -87,6 +87,7 @@ pub fn lower_sampling_params( seed, max_tokens, min_tokens, + thinking_token_budget, logprobs, prompt_logprobs, min_p, @@ -128,6 +129,7 @@ pub fn lower_sampling_params( prompt_len, )?; let min_tokens = min_tokens.unwrap_or(0); + let thinking_token_budget = normalize_thinking_token_budget(thinking_token_budget)?; let frequency_penalty = frequency_penalty.unwrap_or(0.0); let presence_penalty = presence_penalty.unwrap_or(0.0); @@ -149,6 +151,7 @@ pub fn lower_sampling_params( seed, max_tokens, min_tokens, + thinking_token_budget, logprobs, prompt_logprobs, min_p, @@ -170,6 +173,21 @@ pub fn lower_sampling_params( Ok(params) } +/// Normalize the user-facing `thinking_token_budget` into the engine value. +/// +/// Mirrors Python's `validate_thinking_token_budget` +/// (): +/// `None` and the `-1` "unlimited" sentinel both map to `None`; any other +/// negative value is rejected; non-negative values pass through unchanged. Like +/// Python's `int`, no upper bound is imposed. +fn normalize_thinking_token_budget(value: Option) -> Result> { + match value { + None | Some(-1) => Ok(None), + Some(budget) if budget >= 0 => Ok(Some(budget as u64)), + Some(_) => Err(Error::InvalidThinkingTokenBudget), + } +} + /// Convert bad-word strings into token-ID sequences, following the Python vLLM /// logic in `SamplingParams.update_from_tokenizer()`. /// @@ -366,6 +384,36 @@ mod tests { ) } + #[test] + fn lower_sampling_params_normalizes_thinking_token_budget() { + let lower = |budget: Option| { + lower_sampling_params_with_limits( + SamplingParams { + thinking_token_budget: budget, + ..SamplingParams::default() + }, + sample_sampling_limits(), + ) + }; + + // Non-negative budgets (including 0) pass through unchanged. + assert_eq!(lower(Some(256)).unwrap().thinking_token_budget, Some(256)); + assert_eq!(lower(Some(0)).unwrap().thinking_token_budget, Some(0)); + // `None` and the `-1` "unlimited" sentinel both disable the budget. + assert_eq!(lower(None).unwrap().thinking_token_budget, None); + assert_eq!(lower(Some(-1)).unwrap().thinking_token_budget, None); + // No upper bound is imposed, matching Python's `int`. + assert_eq!( + lower(Some(i64::from(u32::MAX) + 1)).unwrap().thinking_token_budget, + Some(u64::from(u32::MAX) + 1) + ); + // Other negatives are rejected. + assert!(matches!( + lower(Some(-2)), + Err(Error::InvalidThinkingTokenBudget) + )); + } + #[test] fn lower_text_request_applies_python_style_eos_hints() { let prepared = lower_text_request( @@ -386,6 +434,7 @@ mod tests { seed: None, max_tokens: 999997, min_tokens: 0, + thinking_token_budget: None, logprobs: None, prompt_logprobs: None, min_p: 0.0, @@ -437,6 +486,7 @@ mod tests { seed: None, max_tokens: 999997, min_tokens: 0, + thinking_token_budget: None, logprobs: None, prompt_logprobs: None, min_p: 0.0, @@ -567,6 +617,7 @@ mod tests { seed: None, max_tokens: 40957, min_tokens: 0, + thinking_token_budget: None, logprobs: None, prompt_logprobs: None, min_p: 0.0, @@ -628,6 +679,7 @@ mod tests { seed: None, max_tokens: 999997, min_tokens: 0, + thinking_token_budget: None, logprobs: None, prompt_logprobs: None, min_p: 0.0, @@ -697,6 +749,7 @@ mod tests { seed: None, max_tokens: 32, min_tokens: 2, + thinking_token_budget: None, logprobs: None, prompt_logprobs: None, min_p: 0.1, @@ -929,6 +982,7 @@ mod tests { seed: None, max_tokens: 128, min_tokens: 0, + thinking_token_budget: None, logprobs: None, prompt_logprobs: None, min_p: 0.1, diff --git a/rust/src/text/src/request.rs b/rust/src/text/src/request.rs index 1ca8f8a924..682d85390d 100644 --- a/rust/src/text/src/request.rs +++ b/rust/src/text/src/request.rs @@ -56,6 +56,12 @@ pub struct SamplingParams { pub max_tokens: Option, /// Minimum number of tokens to generate before EOS or stop-token handling. pub min_tokens: Option, + /// Maximum number of reasoning ("thinking") tokens to emit before the + /// reasoning section is force-closed. `None` or the user-facing `-1` + /// "unlimited" sentinel both disable the budget. The raw value is carried + /// here; `-1` is normalized to `None` (and other negatives rejected) during + /// lowering (see `lower_sampling_params`). + pub thinking_token_budget: Option, /// Number of log probabilities to return per generated token. /// /// `None` disables sample logprobs. `-1` requests the full vocabulary. @@ -116,6 +122,7 @@ impl Default for SamplingParams { seed: None, max_tokens: None, min_tokens: None, + thinking_token_budget: None, logprobs: None, prompt_logprobs: None, min_p: None,