[Rust Frontend] Add support for truncate_prompt_tokens and truncation_side (#48584)

Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: pranavthakur0-0 <pranavthakurdew@gmail.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Pranav Thakur
2026-08-29 00:16:37 +00:00
committed by GitHub
co-authored by Bugen Zhao
parent 1a16c2ad2c
commit 085e9bb07e
17 changed files with 424 additions and 35 deletions
+12
View File
@@ -35,6 +35,15 @@ pub enum Error {
MinTokensExceedsMaxTokens { min_tokens: u32, max_tokens: u32 },
#[error("`thinking_token_budget` must be a non-negative integer or -1 for unlimited.")]
InvalidThinkingTokenBudget,
#[error(
"truncate_prompt_tokens={value} cannot be greater than max_model_len - \
max_tokens = {budget}. Please request a smaller truncation size."
)]
TruncatePromptTokensExceedsBudget { value: u64, budget: u32 },
#[error("invalid truncate_prompt_tokens={value}; must be >= -1")]
InvalidTruncatePromptTokens { value: i64 },
#[error("truncate_prompt_tokens is not supported for multimodal requests")]
TruncateUnsupportedWithMultimodal,
#[error("invalid repetition detection params: {message}")]
InvalidRepetitionDetection { message: String },
#[error("text request stream `{request_id}` closed before terminal output")]
@@ -59,6 +68,9 @@ impl Error {
| Self::SamplingParams(_)
| Self::MinTokensExceedsMaxTokens { .. }
| Self::InvalidThinkingTokenBudget
| Self::TruncatePromptTokensExceedsBudget { .. }
| Self::InvalidTruncatePromptTokens { .. }
| Self::TruncateUnsupportedWithMultimodal
| Self::InvalidRepetitionDetection { .. }
// An empty tokenized prompt detected later, at request prepare
// time, surfaces through the transparent Llm wrapper.
+64 -11
View File
@@ -33,6 +33,8 @@ mod error;
mod lower;
pub mod output;
mod request;
mod truncation;
pub use truncation::{PromptTruncation, PromptTruncationLimit, TruncationSide};
pub use vllm_tokenizer as tokenizer;
trait_set! {
@@ -80,26 +82,35 @@ impl TextRequestProcessor {
self.max_model_len
}
fn tokenize_prompt(
tokenizer: &DynTokenizer,
fn prepare_prompt_tokens(
&self,
prompt: Prompt,
add_special_tokens: bool,
prompt_truncation: Option<PromptTruncation>,
max_output_tokens: Option<u32>,
) -> Result<Vec<u32>> {
match prompt {
Prompt::Text(text) => tokenizer.encode(&text, add_special_tokens).map_err(Into::into),
let tokenizer = self.backend.tokenizer();
let mut prompt_token_ids = match prompt {
Prompt::Text(text) => tokenizer.encode(&text, add_special_tokens)?,
// Pre-tokenized prompts are the main completions-side escape hatch that lets benchmark
// and infra workloads bypass chat rendering and tokenizer overhead entirely.
Prompt::TokenIds(token_ids) => Ok(token_ids),
Prompt::TokenIds(token_ids) => token_ids,
};
if let Some(prompt_truncation) = prompt_truncation {
let input_budget = self.max_model_len.saturating_sub(max_output_tokens.unwrap_or(0));
prompt_truncation.apply(&mut prompt_token_ids, input_budget)?;
}
Ok(prompt_token_ids)
}
/// Tokenize one request without generation-specific lowering.
/// Prepare one request's prompt tokens without generation-specific lowering.
pub fn tokenize(&self, request: TextRequest) -> Result<Vec<u32>> {
request.validate()?;
Self::tokenize_prompt(
&self.backend.tokenizer(),
self.prepare_prompt_tokens(
request.prompt,
request.add_special_tokens,
request.prompt_truncation,
request.sampling_params.max_tokens,
)
}
@@ -111,12 +122,13 @@ impl TextRequestProcessor {
request.arrival_time = Some(vllm_llm::current_unix_timestamp_secs());
}
let tokenizer = self.backend.tokenizer();
let prompt_token_ids = Self::tokenize_prompt(
&tokenizer,
let prompt_token_ids = self.prepare_prompt_tokens(
take(&mut request.prompt),
request.add_special_tokens,
request.prompt_truncation,
request.sampling_params.max_tokens,
)?;
let tokenizer = self.backend.tokenizer();
let sampling_hints = self.backend.sampling_hints()?;
let sampling_limits = SamplingLimits {
max_model_len: self.max_model_len,
@@ -247,3 +259,44 @@ impl TextLlm {
Ok(())
}
}
#[cfg(test)]
mod tests {
use std::sync::Arc;
use vllm_tokenizer::test_utils::TestTokenizer;
use super::*;
#[derive(Debug)]
struct TestTextBackend;
impl TextBackend for TestTextBackend {
fn tokenizer(&self) -> DynTokenizer {
Arc::new(TestTokenizer::new())
}
fn model_id(&self) -> &str {
"test-model"
}
}
#[test]
fn tokenize_and_prepare_apply_the_same_prompt_truncation() {
let processor = TextRequestProcessor::new(Arc::new(TestTextBackend), 10);
let request = TextRequest {
prompt: Prompt::TokenIds(vec![1, 2, 3, 4, 5]),
prompt_truncation: Some(PromptTruncation {
limit: PromptTruncationLimit::Fixed(3),
side: TruncationSide::Left,
}),
..TextRequest::for_test()
};
assert_eq!(processor.tokenize(request.clone()).unwrap(), vec![3, 4, 5]);
assert_eq!(
processor.prepare(request).unwrap().generate_request.prompt_token_ids,
vec![3, 4, 5]
);
}
}
+8
View File
@@ -14,6 +14,7 @@ use vllm_engine_core_client::protocol::structured_outputs::StructuredOutputsPara
use crate::error::{Error, Result};
use crate::output::TextDecodeOptions;
use crate::truncation::PromptTruncation;
/// Normalize vLLM's signed `top_k` domain into [`SamplingParams::top_k`].
///
@@ -196,6 +197,9 @@ pub struct TextRequest {
pub priority: i32,
/// Salt for prefix cache isolation in multi-user environments.
pub cache_salt: Option<String>,
/// Prompt-truncation policy resolved by the caller.
#[serde(default)]
pub prompt_truncation: Option<PromptTruncation>,
/// Whether to add special tokens (e.g. BOS) during prompt tokenization.
pub add_special_tokens: bool,
/// Override data parallel rank.
@@ -231,6 +235,7 @@ impl TextRequest {
intermediate: true,
priority: 0,
cache_salt: None,
prompt_truncation: None,
add_special_tokens: false,
data_parallel_rank: None,
session_id: None,
@@ -257,6 +262,9 @@ impl TextRequest {
request_id: self.request_id.clone(),
});
}
if self.mm_features.is_some() && self.prompt_truncation.is_some() {
return Err(Error::TruncateUnsupportedWithMultimodal);
}
Ok(())
}
}
+178
View File
@@ -0,0 +1,178 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
use serde::{Deserialize, Serialize};
use crate::error::{Error, Result};
/// Which side of the prompt to discard when truncation is active.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum TruncationSide {
/// Keep the last N tokens, discarding the prompt prefix.
Left,
/// Keep the first N tokens, discarding the prompt suffix.
Right,
}
/// Maximum number of prompt tokens to retain.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub enum PromptTruncationLimit {
/// Use the input budget remaining after reserving output tokens.
InputBudget,
/// Retain at most the given number of prompt tokens.
Fixed(u64),
}
/// Typed prompt-truncation policy used inside the text frontend.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub struct PromptTruncation {
/// Maximum number of prompt tokens to retain.
pub limit: PromptTruncationLimit,
/// Side from which excess prompt tokens are discarded.
pub side: TruncationSide,
}
impl PromptTruncation {
/// Convert the OpenAI-compatible integer representation into a typed policy.
pub fn from_wire(limit: i64, side: TruncationSide) -> Result<Self> {
let limit = match limit {
-1 => PromptTruncationLimit::InputBudget,
0.. => PromptTruncationLimit::Fixed(limit as u64),
_ => return Err(Error::InvalidTruncatePromptTokens { value: limit }),
};
Ok(Self { limit, side })
}
pub(crate) fn apply(self, prompt_token_ids: &mut Vec<u32>, input_budget: u32) -> Result<()> {
let max_input_tokens = match self.limit {
PromptTruncationLimit::InputBudget => input_budget,
PromptTruncationLimit::Fixed(limit) => {
if limit > u64::from(input_budget) {
return Err(Error::TruncatePromptTokensExceedsBudget {
value: limit,
budget: input_budget,
});
}
limit as u32
}
} as usize;
if prompt_token_ids.len() <= max_input_tokens {
return Ok(());
}
match self.side {
TruncationSide::Left => {
let start = prompt_token_ids.len() - max_input_tokens;
prompt_token_ids.drain(0..start);
}
TruncationSide::Right => prompt_token_ids.truncate(max_input_tokens),
}
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn parses_wire_limits() {
assert_eq!(
PromptTruncation::from_wire(-1, TruncationSide::Left).unwrap(),
PromptTruncation {
limit: PromptTruncationLimit::InputBudget,
side: TruncationSide::Left,
}
);
assert_eq!(
PromptTruncation::from_wire(0, TruncationSide::Right).unwrap(),
PromptTruncation {
limit: PromptTruncationLimit::Fixed(0),
side: TruncationSide::Right,
}
);
assert!(matches!(
PromptTruncation::from_wire(-2, TruncationSide::Left),
Err(Error::InvalidTruncatePromptTokens { value: -2 })
));
}
#[test]
fn applies_fixed_limit_from_either_side() {
let mut left = vec![1, 2, 3, 4, 5];
PromptTruncation {
limit: PromptTruncationLimit::Fixed(3),
side: TruncationSide::Left,
}
.apply(&mut left, 90)
.unwrap();
assert_eq!(left, vec![3, 4, 5]);
let mut right = vec![1, 2, 3, 4, 5];
PromptTruncation {
limit: PromptTruncationLimit::Fixed(3),
side: TruncationSide::Right,
}
.apply(&mut right, 90)
.unwrap();
assert_eq!(right, vec![1, 2, 3]);
}
#[test]
fn input_budget_reserves_output_tokens() {
let mut prompt = vec![0; 100];
PromptTruncation {
limit: PromptTruncationLimit::InputBudget,
side: TruncationSide::Left,
}
.apply(&mut prompt, 70)
.unwrap();
assert_eq!(prompt.len(), 70);
}
#[test]
fn fixed_limit_cannot_exceed_input_budget() {
let mut prompt = vec![1, 2, 3];
let error = PromptTruncation {
limit: PromptTruncationLimit::Fixed(80),
side: TruncationSide::Left,
}
.apply(&mut prompt, 70)
.unwrap_err();
assert!(matches!(
error,
Error::TruncatePromptTokensExceedsBudget {
value: 80,
budget: 70
}
));
}
#[test]
fn zero_limit_clears_prompt() {
let mut prompt = vec![1, 2, 3];
PromptTruncation {
limit: PromptTruncationLimit::Fixed(0),
side: TruncationSide::Left,
}
.apply(&mut prompt, 90)
.unwrap();
assert!(prompt.is_empty());
}
#[test]
fn limit_at_or_above_prompt_length_is_a_noop() {
for limit in [3, 10] {
let mut prompt = vec![1, 2, 3];
PromptTruncation {
limit: PromptTruncationLimit::Fixed(limit),
side: TruncationSide::Left,
}
.apply(&mut prompt, 90)
.unwrap();
assert_eq!(prompt, vec![1, 2, 3]);
}
}
}