[Rust Frontend] Support thinking_token_budget for chat and completions (#46137)
Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Signed-off-by: RickyChen / 陳昭儒 <ricky.chen@infinirc.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
co-authored by
Bugen Zhao
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
a9f7b2d41c
commit
80abe0de7d
@@ -20,6 +20,8 @@ pub enum Error {
|
||||
Logprobs(#[from] LogprobsError),
|
||||
#[error(transparent)]
|
||||
OutOfVocab(#[from] OutOfVocabError),
|
||||
#[error("`thinking_token_budget` must be a non-negative integer or -1 for unlimited.")]
|
||||
InvalidThinkingTokenBudget,
|
||||
#[error("text request stream `{request_id}` closed before terminal output")]
|
||||
StreamClosedBeforeTerminalOutput { request_id: String },
|
||||
#[error(transparent)]
|
||||
|
||||
@@ -87,6 +87,7 @@ pub fn lower_sampling_params(
|
||||
seed,
|
||||
max_tokens,
|
||||
min_tokens,
|
||||
thinking_token_budget,
|
||||
logprobs,
|
||||
prompt_logprobs,
|
||||
min_p,
|
||||
@@ -128,6 +129,7 @@ pub fn lower_sampling_params(
|
||||
prompt_len,
|
||||
)?;
|
||||
let min_tokens = min_tokens.unwrap_or(0);
|
||||
let thinking_token_budget = normalize_thinking_token_budget(thinking_token_budget)?;
|
||||
let frequency_penalty = frequency_penalty.unwrap_or(0.0);
|
||||
let presence_penalty = presence_penalty.unwrap_or(0.0);
|
||||
|
||||
@@ -149,6 +151,7 @@ pub fn lower_sampling_params(
|
||||
seed,
|
||||
max_tokens,
|
||||
min_tokens,
|
||||
thinking_token_budget,
|
||||
logprobs,
|
||||
prompt_logprobs,
|
||||
min_p,
|
||||
@@ -170,6 +173,21 @@ pub fn lower_sampling_params(
|
||||
Ok(params)
|
||||
}
|
||||
|
||||
/// Normalize the user-facing `thinking_token_budget` into the engine value.
|
||||
///
|
||||
/// Mirrors Python's `validate_thinking_token_budget`
|
||||
/// (<https://github.com/vllm-project/vllm/blob/ecf9d83520eb217401b47d8a5451a27c5231b8c2/vllm/sampling_params.py#L35-L55>):
|
||||
/// `None` and the `-1` "unlimited" sentinel both map to `None`; any other
|
||||
/// negative value is rejected; non-negative values pass through unchanged. Like
|
||||
/// Python's `int`, no upper bound is imposed.
|
||||
fn normalize_thinking_token_budget(value: Option<i64>) -> Result<Option<u64>> {
|
||||
match value {
|
||||
None | Some(-1) => Ok(None),
|
||||
Some(budget) if budget >= 0 => Ok(Some(budget as u64)),
|
||||
Some(_) => Err(Error::InvalidThinkingTokenBudget),
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert bad-word strings into token-ID sequences, following the Python vLLM
|
||||
/// logic in `SamplingParams.update_from_tokenizer()`.
|
||||
///
|
||||
@@ -366,6 +384,36 @@ mod tests {
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lower_sampling_params_normalizes_thinking_token_budget() {
|
||||
let lower = |budget: Option<i64>| {
|
||||
lower_sampling_params_with_limits(
|
||||
SamplingParams {
|
||||
thinking_token_budget: budget,
|
||||
..SamplingParams::default()
|
||||
},
|
||||
sample_sampling_limits(),
|
||||
)
|
||||
};
|
||||
|
||||
// Non-negative budgets (including 0) pass through unchanged.
|
||||
assert_eq!(lower(Some(256)).unwrap().thinking_token_budget, Some(256));
|
||||
assert_eq!(lower(Some(0)).unwrap().thinking_token_budget, Some(0));
|
||||
// `None` and the `-1` "unlimited" sentinel both disable the budget.
|
||||
assert_eq!(lower(None).unwrap().thinking_token_budget, None);
|
||||
assert_eq!(lower(Some(-1)).unwrap().thinking_token_budget, None);
|
||||
// No upper bound is imposed, matching Python's `int`.
|
||||
assert_eq!(
|
||||
lower(Some(i64::from(u32::MAX) + 1)).unwrap().thinking_token_budget,
|
||||
Some(u64::from(u32::MAX) + 1)
|
||||
);
|
||||
// Other negatives are rejected.
|
||||
assert!(matches!(
|
||||
lower(Some(-2)),
|
||||
Err(Error::InvalidThinkingTokenBudget)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lower_text_request_applies_python_style_eos_hints() {
|
||||
let prepared = lower_text_request(
|
||||
@@ -386,6 +434,7 @@ mod tests {
|
||||
seed: None,
|
||||
max_tokens: 999997,
|
||||
min_tokens: 0,
|
||||
thinking_token_budget: None,
|
||||
logprobs: None,
|
||||
prompt_logprobs: None,
|
||||
min_p: 0.0,
|
||||
@@ -437,6 +486,7 @@ mod tests {
|
||||
seed: None,
|
||||
max_tokens: 999997,
|
||||
min_tokens: 0,
|
||||
thinking_token_budget: None,
|
||||
logprobs: None,
|
||||
prompt_logprobs: None,
|
||||
min_p: 0.0,
|
||||
@@ -567,6 +617,7 @@ mod tests {
|
||||
seed: None,
|
||||
max_tokens: 40957,
|
||||
min_tokens: 0,
|
||||
thinking_token_budget: None,
|
||||
logprobs: None,
|
||||
prompt_logprobs: None,
|
||||
min_p: 0.0,
|
||||
@@ -628,6 +679,7 @@ mod tests {
|
||||
seed: None,
|
||||
max_tokens: 999997,
|
||||
min_tokens: 0,
|
||||
thinking_token_budget: None,
|
||||
logprobs: None,
|
||||
prompt_logprobs: None,
|
||||
min_p: 0.0,
|
||||
@@ -697,6 +749,7 @@ mod tests {
|
||||
seed: None,
|
||||
max_tokens: 32,
|
||||
min_tokens: 2,
|
||||
thinking_token_budget: None,
|
||||
logprobs: None,
|
||||
prompt_logprobs: None,
|
||||
min_p: 0.1,
|
||||
@@ -929,6 +982,7 @@ mod tests {
|
||||
seed: None,
|
||||
max_tokens: 128,
|
||||
min_tokens: 0,
|
||||
thinking_token_budget: None,
|
||||
logprobs: None,
|
||||
prompt_logprobs: None,
|
||||
min_p: 0.1,
|
||||
|
||||
@@ -56,6 +56,12 @@ pub struct SamplingParams {
|
||||
pub max_tokens: Option<u32>,
|
||||
/// Minimum number of tokens to generate before EOS or stop-token handling.
|
||||
pub min_tokens: Option<u32>,
|
||||
/// Maximum number of reasoning ("thinking") tokens to emit before the
|
||||
/// reasoning section is force-closed. `None` or the user-facing `-1`
|
||||
/// "unlimited" sentinel both disable the budget. The raw value is carried
|
||||
/// here; `-1` is normalized to `None` (and other negatives rejected) during
|
||||
/// lowering (see `lower_sampling_params`).
|
||||
pub thinking_token_budget: Option<i64>,
|
||||
/// Number of log probabilities to return per generated token.
|
||||
///
|
||||
/// `None` disables sample logprobs. `-1` requests the full vocabulary.
|
||||
@@ -116,6 +122,7 @@ impl Default for SamplingParams {
|
||||
seed: None,
|
||||
max_tokens: None,
|
||||
min_tokens: None,
|
||||
thinking_token_budget: None,
|
||||
logprobs: None,
|
||||
prompt_logprobs: None,
|
||||
min_p: None,
|
||||
|
||||
Reference in New Issue
Block a user