[Rust Frontend] Add support for truncate_prompt_tokens and truncation_side (#48584)
Co-authored-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: pranavthakur0-0 <pranavthakurdew@gmail.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
co-authored by
Bugen Zhao
parent
1a16c2ad2c
commit
085e9bb07e
@@ -35,6 +35,15 @@ pub enum Error {
|
||||
MinTokensExceedsMaxTokens { min_tokens: u32, max_tokens: u32 },
|
||||
#[error("`thinking_token_budget` must be a non-negative integer or -1 for unlimited.")]
|
||||
InvalidThinkingTokenBudget,
|
||||
#[error(
|
||||
"truncate_prompt_tokens={value} cannot be greater than max_model_len - \
|
||||
max_tokens = {budget}. Please request a smaller truncation size."
|
||||
)]
|
||||
TruncatePromptTokensExceedsBudget { value: u64, budget: u32 },
|
||||
#[error("invalid truncate_prompt_tokens={value}; must be >= -1")]
|
||||
InvalidTruncatePromptTokens { value: i64 },
|
||||
#[error("truncate_prompt_tokens is not supported for multimodal requests")]
|
||||
TruncateUnsupportedWithMultimodal,
|
||||
#[error("invalid repetition detection params: {message}")]
|
||||
InvalidRepetitionDetection { message: String },
|
||||
#[error("text request stream `{request_id}` closed before terminal output")]
|
||||
@@ -59,6 +68,9 @@ impl Error {
|
||||
| Self::SamplingParams(_)
|
||||
| Self::MinTokensExceedsMaxTokens { .. }
|
||||
| Self::InvalidThinkingTokenBudget
|
||||
| Self::TruncatePromptTokensExceedsBudget { .. }
|
||||
| Self::InvalidTruncatePromptTokens { .. }
|
||||
| Self::TruncateUnsupportedWithMultimodal
|
||||
| Self::InvalidRepetitionDetection { .. }
|
||||
// An empty tokenized prompt detected later, at request prepare
|
||||
// time, surfaces through the transparent Llm wrapper.
|
||||
|
||||
+64
-11
@@ -33,6 +33,8 @@ mod error;
|
||||
mod lower;
|
||||
pub mod output;
|
||||
mod request;
|
||||
mod truncation;
|
||||
pub use truncation::{PromptTruncation, PromptTruncationLimit, TruncationSide};
|
||||
pub use vllm_tokenizer as tokenizer;
|
||||
|
||||
trait_set! {
|
||||
@@ -80,26 +82,35 @@ impl TextRequestProcessor {
|
||||
self.max_model_len
|
||||
}
|
||||
|
||||
fn tokenize_prompt(
|
||||
tokenizer: &DynTokenizer,
|
||||
fn prepare_prompt_tokens(
|
||||
&self,
|
||||
prompt: Prompt,
|
||||
add_special_tokens: bool,
|
||||
prompt_truncation: Option<PromptTruncation>,
|
||||
max_output_tokens: Option<u32>,
|
||||
) -> Result<Vec<u32>> {
|
||||
match prompt {
|
||||
Prompt::Text(text) => tokenizer.encode(&text, add_special_tokens).map_err(Into::into),
|
||||
let tokenizer = self.backend.tokenizer();
|
||||
let mut prompt_token_ids = match prompt {
|
||||
Prompt::Text(text) => tokenizer.encode(&text, add_special_tokens)?,
|
||||
// Pre-tokenized prompts are the main completions-side escape hatch that lets benchmark
|
||||
// and infra workloads bypass chat rendering and tokenizer overhead entirely.
|
||||
Prompt::TokenIds(token_ids) => Ok(token_ids),
|
||||
Prompt::TokenIds(token_ids) => token_ids,
|
||||
};
|
||||
if let Some(prompt_truncation) = prompt_truncation {
|
||||
let input_budget = self.max_model_len.saturating_sub(max_output_tokens.unwrap_or(0));
|
||||
prompt_truncation.apply(&mut prompt_token_ids, input_budget)?;
|
||||
}
|
||||
Ok(prompt_token_ids)
|
||||
}
|
||||
|
||||
/// Tokenize one request without generation-specific lowering.
|
||||
/// Prepare one request's prompt tokens without generation-specific lowering.
|
||||
pub fn tokenize(&self, request: TextRequest) -> Result<Vec<u32>> {
|
||||
request.validate()?;
|
||||
Self::tokenize_prompt(
|
||||
&self.backend.tokenizer(),
|
||||
self.prepare_prompt_tokens(
|
||||
request.prompt,
|
||||
request.add_special_tokens,
|
||||
request.prompt_truncation,
|
||||
request.sampling_params.max_tokens,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -111,12 +122,13 @@ impl TextRequestProcessor {
|
||||
request.arrival_time = Some(vllm_llm::current_unix_timestamp_secs());
|
||||
}
|
||||
|
||||
let tokenizer = self.backend.tokenizer();
|
||||
let prompt_token_ids = Self::tokenize_prompt(
|
||||
&tokenizer,
|
||||
let prompt_token_ids = self.prepare_prompt_tokens(
|
||||
take(&mut request.prompt),
|
||||
request.add_special_tokens,
|
||||
request.prompt_truncation,
|
||||
request.sampling_params.max_tokens,
|
||||
)?;
|
||||
let tokenizer = self.backend.tokenizer();
|
||||
let sampling_hints = self.backend.sampling_hints()?;
|
||||
let sampling_limits = SamplingLimits {
|
||||
max_model_len: self.max_model_len,
|
||||
@@ -247,3 +259,44 @@ impl TextLlm {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::Arc;
|
||||
|
||||
use vllm_tokenizer::test_utils::TestTokenizer;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[derive(Debug)]
|
||||
struct TestTextBackend;
|
||||
|
||||
impl TextBackend for TestTextBackend {
|
||||
fn tokenizer(&self) -> DynTokenizer {
|
||||
Arc::new(TestTokenizer::new())
|
||||
}
|
||||
|
||||
fn model_id(&self) -> &str {
|
||||
"test-model"
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tokenize_and_prepare_apply_the_same_prompt_truncation() {
|
||||
let processor = TextRequestProcessor::new(Arc::new(TestTextBackend), 10);
|
||||
let request = TextRequest {
|
||||
prompt: Prompt::TokenIds(vec![1, 2, 3, 4, 5]),
|
||||
prompt_truncation: Some(PromptTruncation {
|
||||
limit: PromptTruncationLimit::Fixed(3),
|
||||
side: TruncationSide::Left,
|
||||
}),
|
||||
..TextRequest::for_test()
|
||||
};
|
||||
|
||||
assert_eq!(processor.tokenize(request.clone()).unwrap(), vec![3, 4, 5]);
|
||||
assert_eq!(
|
||||
processor.prepare(request).unwrap().generate_request.prompt_token_ids,
|
||||
vec![3, 4, 5]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,6 +14,7 @@ use vllm_engine_core_client::protocol::structured_outputs::StructuredOutputsPara
|
||||
|
||||
use crate::error::{Error, Result};
|
||||
use crate::output::TextDecodeOptions;
|
||||
use crate::truncation::PromptTruncation;
|
||||
|
||||
/// Normalize vLLM's signed `top_k` domain into [`SamplingParams::top_k`].
|
||||
///
|
||||
@@ -196,6 +197,9 @@ pub struct TextRequest {
|
||||
pub priority: i32,
|
||||
/// Salt for prefix cache isolation in multi-user environments.
|
||||
pub cache_salt: Option<String>,
|
||||
/// Prompt-truncation policy resolved by the caller.
|
||||
#[serde(default)]
|
||||
pub prompt_truncation: Option<PromptTruncation>,
|
||||
/// Whether to add special tokens (e.g. BOS) during prompt tokenization.
|
||||
pub add_special_tokens: bool,
|
||||
/// Override data parallel rank.
|
||||
@@ -231,6 +235,7 @@ impl TextRequest {
|
||||
intermediate: true,
|
||||
priority: 0,
|
||||
cache_salt: None,
|
||||
prompt_truncation: None,
|
||||
add_special_tokens: false,
|
||||
data_parallel_rank: None,
|
||||
session_id: None,
|
||||
@@ -257,6 +262,9 @@ impl TextRequest {
|
||||
request_id: self.request_id.clone(),
|
||||
});
|
||||
}
|
||||
if self.mm_features.is_some() && self.prompt_truncation.is_some() {
|
||||
return Err(Error::TruncateUnsupportedWithMultimodal);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,178 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::error::{Error, Result};
|
||||
|
||||
/// Which side of the prompt to discard when truncation is active.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum TruncationSide {
|
||||
/// Keep the last N tokens, discarding the prompt prefix.
|
||||
Left,
|
||||
/// Keep the first N tokens, discarding the prompt suffix.
|
||||
Right,
|
||||
}
|
||||
|
||||
/// Maximum number of prompt tokens to retain.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub enum PromptTruncationLimit {
|
||||
/// Use the input budget remaining after reserving output tokens.
|
||||
InputBudget,
|
||||
/// Retain at most the given number of prompt tokens.
|
||||
Fixed(u64),
|
||||
}
|
||||
|
||||
/// Typed prompt-truncation policy used inside the text frontend.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct PromptTruncation {
|
||||
/// Maximum number of prompt tokens to retain.
|
||||
pub limit: PromptTruncationLimit,
|
||||
/// Side from which excess prompt tokens are discarded.
|
||||
pub side: TruncationSide,
|
||||
}
|
||||
|
||||
impl PromptTruncation {
|
||||
/// Convert the OpenAI-compatible integer representation into a typed policy.
|
||||
pub fn from_wire(limit: i64, side: TruncationSide) -> Result<Self> {
|
||||
let limit = match limit {
|
||||
-1 => PromptTruncationLimit::InputBudget,
|
||||
0.. => PromptTruncationLimit::Fixed(limit as u64),
|
||||
_ => return Err(Error::InvalidTruncatePromptTokens { value: limit }),
|
||||
};
|
||||
Ok(Self { limit, side })
|
||||
}
|
||||
|
||||
pub(crate) fn apply(self, prompt_token_ids: &mut Vec<u32>, input_budget: u32) -> Result<()> {
|
||||
let max_input_tokens = match self.limit {
|
||||
PromptTruncationLimit::InputBudget => input_budget,
|
||||
PromptTruncationLimit::Fixed(limit) => {
|
||||
if limit > u64::from(input_budget) {
|
||||
return Err(Error::TruncatePromptTokensExceedsBudget {
|
||||
value: limit,
|
||||
budget: input_budget,
|
||||
});
|
||||
}
|
||||
limit as u32
|
||||
}
|
||||
} as usize;
|
||||
|
||||
if prompt_token_ids.len() <= max_input_tokens {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
match self.side {
|
||||
TruncationSide::Left => {
|
||||
let start = prompt_token_ids.len() - max_input_tokens;
|
||||
prompt_token_ids.drain(0..start);
|
||||
}
|
||||
TruncationSide::Right => prompt_token_ids.truncate(max_input_tokens),
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn parses_wire_limits() {
|
||||
assert_eq!(
|
||||
PromptTruncation::from_wire(-1, TruncationSide::Left).unwrap(),
|
||||
PromptTruncation {
|
||||
limit: PromptTruncationLimit::InputBudget,
|
||||
side: TruncationSide::Left,
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
PromptTruncation::from_wire(0, TruncationSide::Right).unwrap(),
|
||||
PromptTruncation {
|
||||
limit: PromptTruncationLimit::Fixed(0),
|
||||
side: TruncationSide::Right,
|
||||
}
|
||||
);
|
||||
assert!(matches!(
|
||||
PromptTruncation::from_wire(-2, TruncationSide::Left),
|
||||
Err(Error::InvalidTruncatePromptTokens { value: -2 })
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn applies_fixed_limit_from_either_side() {
|
||||
let mut left = vec![1, 2, 3, 4, 5];
|
||||
PromptTruncation {
|
||||
limit: PromptTruncationLimit::Fixed(3),
|
||||
side: TruncationSide::Left,
|
||||
}
|
||||
.apply(&mut left, 90)
|
||||
.unwrap();
|
||||
assert_eq!(left, vec![3, 4, 5]);
|
||||
|
||||
let mut right = vec![1, 2, 3, 4, 5];
|
||||
PromptTruncation {
|
||||
limit: PromptTruncationLimit::Fixed(3),
|
||||
side: TruncationSide::Right,
|
||||
}
|
||||
.apply(&mut right, 90)
|
||||
.unwrap();
|
||||
assert_eq!(right, vec![1, 2, 3]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn input_budget_reserves_output_tokens() {
|
||||
let mut prompt = vec![0; 100];
|
||||
PromptTruncation {
|
||||
limit: PromptTruncationLimit::InputBudget,
|
||||
side: TruncationSide::Left,
|
||||
}
|
||||
.apply(&mut prompt, 70)
|
||||
.unwrap();
|
||||
assert_eq!(prompt.len(), 70);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fixed_limit_cannot_exceed_input_budget() {
|
||||
let mut prompt = vec![1, 2, 3];
|
||||
let error = PromptTruncation {
|
||||
limit: PromptTruncationLimit::Fixed(80),
|
||||
side: TruncationSide::Left,
|
||||
}
|
||||
.apply(&mut prompt, 70)
|
||||
.unwrap_err();
|
||||
assert!(matches!(
|
||||
error,
|
||||
Error::TruncatePromptTokensExceedsBudget {
|
||||
value: 80,
|
||||
budget: 70
|
||||
}
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn zero_limit_clears_prompt() {
|
||||
let mut prompt = vec![1, 2, 3];
|
||||
PromptTruncation {
|
||||
limit: PromptTruncationLimit::Fixed(0),
|
||||
side: TruncationSide::Left,
|
||||
}
|
||||
.apply(&mut prompt, 90)
|
||||
.unwrap();
|
||||
assert!(prompt.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn limit_at_or_above_prompt_length_is_a_noop() {
|
||||
for limit in [3, 10] {
|
||||
let mut prompt = vec![1, 2, 3];
|
||||
PromptTruncation {
|
||||
limit: PromptTruncationLimit::Fixed(limit),
|
||||
side: TruncationSide::Left,
|
||||
}
|
||||
.apply(&mut prompt, 90)
|
||||
.unwrap();
|
||||
assert_eq!(prompt, vec![1, 2, 3]);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user