[Rust Frontend] Support thinking_token_budget for chat and completions (#46137)

Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Signed-off-by: RickyChen / 陳昭儒 <ricky.chen@infinirc.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Chao-Ju Chen
2026-06-22 16:00:02 +08:00
committed by GitHub
co-authored by Bugen Zhao mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent a9f7b2d41c
commit 80abe0de7d
13 changed files with 178 additions and 7 deletions
+2
View File
@@ -20,6 +20,8 @@ pub enum Error {
Logprobs(#[from] LogprobsError),
#[error(transparent)]
OutOfVocab(#[from] OutOfVocabError),
#[error("`thinking_token_budget` must be a non-negative integer or -1 for unlimited.")]
InvalidThinkingTokenBudget,
#[error("text request stream `{request_id}` closed before terminal output")]
StreamClosedBeforeTerminalOutput { request_id: String },
#[error(transparent)]
+54
View File
@@ -87,6 +87,7 @@ pub fn lower_sampling_params(
seed,
max_tokens,
min_tokens,
thinking_token_budget,
logprobs,
prompt_logprobs,
min_p,
@@ -128,6 +129,7 @@ pub fn lower_sampling_params(
prompt_len,
)?;
let min_tokens = min_tokens.unwrap_or(0);
let thinking_token_budget = normalize_thinking_token_budget(thinking_token_budget)?;
let frequency_penalty = frequency_penalty.unwrap_or(0.0);
let presence_penalty = presence_penalty.unwrap_or(0.0);
@@ -149,6 +151,7 @@ pub fn lower_sampling_params(
seed,
max_tokens,
min_tokens,
thinking_token_budget,
logprobs,
prompt_logprobs,
min_p,
@@ -170,6 +173,21 @@ pub fn lower_sampling_params(
Ok(params)
}
/// Normalize the user-facing `thinking_token_budget` into the engine value.
///
/// Mirrors Python's `validate_thinking_token_budget`
/// (<https://github.com/vllm-project/vllm/blob/ecf9d83520eb217401b47d8a5451a27c5231b8c2/vllm/sampling_params.py#L35-L55>):
/// `None` and the `-1` "unlimited" sentinel both map to `None`; any other
/// negative value is rejected; non-negative values pass through unchanged. Like
/// Python's `int`, no upper bound is imposed.
fn normalize_thinking_token_budget(value: Option<i64>) -> Result<Option<u64>> {
match value {
None | Some(-1) => Ok(None),
Some(budget) if budget >= 0 => Ok(Some(budget as u64)),
Some(_) => Err(Error::InvalidThinkingTokenBudget),
}
}
/// Convert bad-word strings into token-ID sequences, following the Python vLLM
/// logic in `SamplingParams.update_from_tokenizer()`.
///
@@ -366,6 +384,36 @@ mod tests {
)
}
#[test]
fn lower_sampling_params_normalizes_thinking_token_budget() {
let lower = |budget: Option<i64>| {
lower_sampling_params_with_limits(
SamplingParams {
thinking_token_budget: budget,
..SamplingParams::default()
},
sample_sampling_limits(),
)
};
// Non-negative budgets (including 0) pass through unchanged.
assert_eq!(lower(Some(256)).unwrap().thinking_token_budget, Some(256));
assert_eq!(lower(Some(0)).unwrap().thinking_token_budget, Some(0));
// `None` and the `-1` "unlimited" sentinel both disable the budget.
assert_eq!(lower(None).unwrap().thinking_token_budget, None);
assert_eq!(lower(Some(-1)).unwrap().thinking_token_budget, None);
// No upper bound is imposed, matching Python's `int`.
assert_eq!(
lower(Some(i64::from(u32::MAX) + 1)).unwrap().thinking_token_budget,
Some(u64::from(u32::MAX) + 1)
);
// Other negatives are rejected.
assert!(matches!(
lower(Some(-2)),
Err(Error::InvalidThinkingTokenBudget)
));
}
#[test]
fn lower_text_request_applies_python_style_eos_hints() {
let prepared = lower_text_request(
@@ -386,6 +434,7 @@ mod tests {
seed: None,
max_tokens: 999997,
min_tokens: 0,
thinking_token_budget: None,
logprobs: None,
prompt_logprobs: None,
min_p: 0.0,
@@ -437,6 +486,7 @@ mod tests {
seed: None,
max_tokens: 999997,
min_tokens: 0,
thinking_token_budget: None,
logprobs: None,
prompt_logprobs: None,
min_p: 0.0,
@@ -567,6 +617,7 @@ mod tests {
seed: None,
max_tokens: 40957,
min_tokens: 0,
thinking_token_budget: None,
logprobs: None,
prompt_logprobs: None,
min_p: 0.0,
@@ -628,6 +679,7 @@ mod tests {
seed: None,
max_tokens: 999997,
min_tokens: 0,
thinking_token_budget: None,
logprobs: None,
prompt_logprobs: None,
min_p: 0.0,
@@ -697,6 +749,7 @@ mod tests {
seed: None,
max_tokens: 32,
min_tokens: 2,
thinking_token_budget: None,
logprobs: None,
prompt_logprobs: None,
min_p: 0.1,
@@ -929,6 +982,7 @@ mod tests {
seed: None,
max_tokens: 128,
min_tokens: 0,
thinking_token_budget: None,
logprobs: None,
prompt_logprobs: None,
min_p: 0.1,
+7
View File
@@ -56,6 +56,12 @@ pub struct SamplingParams {
pub max_tokens: Option<u32>,
/// Minimum number of tokens to generate before EOS or stop-token handling.
pub min_tokens: Option<u32>,
/// Maximum number of reasoning ("thinking") tokens to emit before the
/// reasoning section is force-closed. `None` or the user-facing `-1`
/// "unlimited" sentinel both disable the budget. The raw value is carried
/// here; `-1` is normalized to `None` (and other negatives rejected) during
/// lowering (see `lower_sampling_params`).
pub thinking_token_budget: Option<i64>,
/// Number of log probabilities to return per generated token.
///
/// `None` disables sample logprobs. `-1` requests the full vocabulary.
@@ -116,6 +122,7 @@ impl Default for SamplingParams {
seed: None,
max_tokens: None,
min_tokens: None,
thinking_token_budget: None,
logprobs: None,
prompt_logprobs: None,
min_p: None,