[Rust Frontend] Add support for truncate_prompt_tokens and truncation_side (#48584)

Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: pranavthakur0-0 <pranavthakurdew@gmail.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Pranav Thakur
2026-08-29 00:16:37 +00:00
committed by GitHub
co-authored by Bugen Zhao
parent 1a16c2ad2c
commit 085e9bb07e
17 changed files with 424 additions and 35 deletions
+25
View File
@@ -224,6 +224,31 @@ mod tests {
assert!(response.error.message.contains("allowed_token_ids"));
}
#[test]
fn truncate_exceeds_budget_maps_to_invalid_request() {
let error = vllm_text::Error::TruncatePromptTokensExceedsBudget {
value: 8000,
budget: 4000,
};
let api_error = text_submit_error("failed to submit completion request", error);
assert_eq!(api_error.status_code(), StatusCode::BAD_REQUEST);
let response = api_error.to_error_response();
assert_eq!(response.error.error_type, "invalid_request_error");
assert!(response.error.message.contains("truncate_prompt_tokens=8000"));
assert!(response.error.message.contains("max_tokens = 4000"));
}
#[test]
fn invalid_truncate_prompt_tokens_maps_to_invalid_request() {
let error = vllm_text::Error::InvalidTruncatePromptTokens { value: -2 };
let api_error = text_submit_error("failed to submit completion request", error);
assert_eq!(api_error.status_code(), StatusCode::BAD_REQUEST);
let response = api_error.to_error_response();
assert_eq!(response.error.error_type, "invalid_request_error");
assert!(response.error.message.contains("-2"));
assert!(response.error.message.contains("must be >= -1"));
}
#[test]
fn other_submit_errors_stay_internal() {
let error = vllm_text::Error::Tokenizer("backend exploded".to_string());
+9 -7
View File
@@ -11,8 +11,8 @@ use vllm_chat::MediaContentPart;
use vllm_engine_core_client::protocol::output::StopReason;
use vllm_engine_core_client::protocol::structured_outputs::StructuredOutputsParams;
use vllm_text::{
DecodedLogprobs, DecodedPromptLogprobs, FinishReason, Finished, Prompt, SamplingParams,
TextDecodeOptions, TextRequest,
DecodedLogprobs, DecodedPromptLogprobs, FinishReason, Finished, Prompt, PromptTruncation,
PromptTruncationLimit, SamplingParams, TextDecodeOptions, TextRequest, TruncationSide,
};
use super::pb;
@@ -110,11 +110,12 @@ pub fn to_text_request(
)));
}
if req.truncate_prompt_tokens != 0 {
return Err(Status::invalid_argument(
"truncate_prompt_tokens is not supported",
));
}
// Proto3 uses zero as unset; positive values select fixed left truncation.
// The -1 input-budget sentinel is outside the uint32 field domain.
let prompt_truncation = (req.truncate_prompt_tokens != 0).then_some(PromptTruncation {
limit: PromptTruncationLimit::Fixed(u64::from(req.truncate_prompt_tokens)),
side: TruncationSide::Left,
});
let prompt = match req.prompt {
Some(pb::generate_request::Prompt::Text(text)) => Prompt::Text(text),
@@ -173,6 +174,7 @@ pub fn to_text_request(
sampling_params,
decode_options,
intermediate: stream,
prompt_truncation,
priority: req.priority,
cache_salt: kv.map(|k| &k.cache_salt).filter(|s| !s.is_empty()).cloned(),
add_special_tokens: true,
@@ -73,6 +73,7 @@ pub(super) fn prepare_generate_request(
sampling_params,
decode_options: TextDecodeOptions::default(),
intermediate: false,
prompt_truncation: None,
priority: request.priority,
cache_salt: request.cache_salt,
add_special_tokens: false,
@@ -10,8 +10,9 @@ use vllm_chat::{
use super::types::ChatCompletionRequest;
use super::validate;
use crate::error::{ApiError, bail_invalid_request, chat_submit_error};
use crate::error::{ApiError, bail_invalid_request, chat_submit_error, text_submit_error};
use crate::lora::LoraModelResolution;
use crate::routes::openai::utils::resolve_generation_prompt_truncation;
use crate::routes::openai::utils::structured_outputs::convert_from_response_format;
use crate::routes::openai::utils::types::{
ChatMessage, ContentPart, MessageContent, Tool, ToolChoice, ToolChoiceValue, ToolReference,
@@ -71,6 +72,12 @@ pub(super) fn prepare_chat_request(
) -> Result<PreparedRequest, ApiError> {
validate::validate_request_compat(&request, &lora_resolution.model_names)?;
let prompt_truncation = resolve_generation_prompt_truncation(
request.truncate_prompt_tokens,
request.truncation_side,
)
.map_err(|error| text_submit_error("invalid prompt truncation", error))?;
let request_id = format!("chatcmpl-{}", ctx.request_id);
let response_model = lora_resolution
.lora_request
@@ -184,6 +191,7 @@ pub(super) fn prepare_chat_request(
min_tokens: request.min_tokens.unwrap_or(0),
},
intermediate: request.stream,
prompt_truncation,
priority: ctx.priority.or(request.priority).unwrap_or(0),
documents: request.documents,
cache_salt: request.cache_salt,
@@ -439,7 +447,9 @@ mod tests {
ChatRenderer, ChatTool as VllmChatTool, ChatToolChoice, GenerationPromptMode,
KimiK3ChatRenderer, SamplingParams as VllmSamplingParams,
};
use vllm_text::{Prompt, output::TextDecodeOptions};
use vllm_text::{
Prompt, PromptTruncation, PromptTruncationLimit, TruncationSide, output::TextDecodeOptions,
};
use vllm_tokenizer::{Tokenizer, test_utils::TestTokenizer};
use super::prepare_chat_request;
@@ -530,6 +540,27 @@ mod tests {
}
}
#[test]
fn prepare_chat_request_preserves_explicit_truncation_side() {
let mut request = base_request();
request.truncate_prompt_tokens = Some(2);
request.truncation_side = Some(TruncationSide::Right);
let prepared = prepare_chat_request(
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
)
.expect("prepare truncation");
assert_eq!(
prepared.chat_request.prompt_truncation,
Some(PromptTruncation {
limit: PromptTruncationLimit::Fixed(2),
side: TruncationSide::Right,
})
);
}
#[test]
fn prepare_chat_request_accepts_zero_min_tokens() {
let request: ChatCompletionRequest = serde_json::from_value(json!({
@@ -11,6 +11,7 @@ use serde_with::SerializeDisplay;
use validator::Validate;
use vllm_chat::ReasoningEffort;
use vllm_engine_core_client::protocol::sampling::RepetitionDetectionParams;
use vllm_text::TruncationSide;
use crate::routes::openai::utils::structured_outputs::ResponseFormat;
use crate::routes::openai::utils::types::{
@@ -159,6 +160,9 @@ pub struct ChatCompletionRequest {
/// Truncate prompt tokens to this length
pub truncate_prompt_tokens: Option<i64>,
/// Which side to truncate from when truncate_prompt_tokens is active
pub truncation_side: Option<TruncationSide>,
/// Number of prompt logprobs to return
pub prompt_logprobs: Option<i32>,
@@ -290,6 +294,7 @@ impl Default for ChatCompletionRequest {
skip_special_tokens: true,
spaces_between_special_tokens: true,
truncate_prompt_tokens: None,
truncation_side: None,
prompt_logprobs: None,
allowed_token_ids: None,
bad_words: None,
@@ -4,6 +4,7 @@
use super::types::ChatCompletionRequest;
use crate::error::{ApiError, bail_invalid_request};
use crate::routes::openai::utils::types::{ChatMessage, Tool};
use crate::routes::openai::utils::validate_generation_prompt_truncation;
/// Enforce the minimal compatibility contract for the Rust OpenAI server.
pub(super) fn validate_request_compat(
@@ -84,11 +85,7 @@ pub(super) fn validate_request_compat(
"spaces_between_special_tokens is not supported."
);
}
reject_non_default(
request.truncate_prompt_tokens.as_ref(),
"truncate_prompt_tokens",
"truncate_prompt_tokens is not supported.",
)?;
validate_generation_prompt_truncation(request.truncate_prompt_tokens, request.echo)?;
reject_non_default(
request.media_io_kwargs.as_ref(),
"media_io_kwargs",
@@ -6,9 +6,10 @@ use vllm_text::tokenizer::Tokenizer;
use vllm_text::{Prompt, SamplingParams, TextDecodeOptions, TextRequest};
use super::types::CompletionRequest;
use crate::error::ApiError;
use crate::error::{ApiError, text_submit_error};
use crate::lora::LoraModelResolution;
use crate::routes::openai::completions::validate;
use crate::routes::openai::utils::resolve_generation_prompt_truncation;
use crate::routes::openai::utils::structured_outputs::convert_from_response_format_value;
use crate::utils::{
ResolvedRequestContext, convert_logit_bias, merge_ec_transfer_params, merge_kv_transfer_params,
@@ -62,6 +63,12 @@ pub(super) fn prepare_completion_request(
) -> Result<PreparedRequest, ApiError> {
validate::validate_request_compat(&request, &lora_resolution.model_names)?;
let prompt_truncation = resolve_generation_prompt_truncation(
request.truncate_prompt_tokens,
request.truncation_side,
)
.map_err(|error| text_submit_error("invalid prompt truncation", error))?;
let request_id = format!("cmpl-{}", ctx.request_id);
let response_model = lora_resolution
.lora_request
@@ -141,6 +148,7 @@ pub(super) fn prepare_completion_request(
min_tokens: request.min_tokens.unwrap_or(0),
},
intermediate: request.stream,
prompt_truncation,
priority: ctx.priority.or(request.priority).unwrap_or(0),
cache_salt: request.cache_salt,
add_special_tokens: request.add_special_tokens,
@@ -200,7 +208,7 @@ mod tests {
use axum::http::HeaderMap;
use serde_json::json;
use validator::Validate;
use vllm_text::Prompt;
use vllm_text::{Prompt, PromptTruncation, PromptTruncationLimit, TruncationSide};
use vllm_tokenizer::test_utils::TestTokenizer;
use super::prepare_completion_request;
@@ -323,6 +331,31 @@ mod tests {
}
}
#[test]
fn prepare_completion_request_uses_left_when_side_is_omitted() {
let mut value = base_request_json();
value
.as_object_mut()
.expect("request object")
.insert("truncate_prompt_tokens".to_string(), json!(-1));
let request = serde_json::from_value(value).expect("parse truncation");
let prepared = prepare_completion_request(
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
&test_tokenizer(),
)
.expect("prepare truncation");
assert_eq!(
prepared.text_request.prompt_truncation,
Some(PromptTruncation {
limit: PromptTruncationLimit::InputBudget,
side: TruncationSide::Left,
})
);
}
#[test]
fn normalize_coerces_null_max_tokens_to_default() {
// An absent `max_tokens` already gets the serde default.
@@ -8,7 +8,7 @@ use serde::{Deserialize, Serialize};
use serde_json::{Map, Value};
use validator::Validate;
use vllm_engine_core_client::protocol::sampling::RepetitionDetectionParams;
use vllm_text::Prompt;
use vllm_text::{Prompt, TruncationSide};
use crate::routes::openai::utils::types::{
LogProbs, Normalizable, PromptLogprobs, StreamOptions, StreamResponseEnvelope, StringOrArray,
@@ -138,6 +138,9 @@ pub struct CompletionRequest {
/// Truncate prompt tokens to this length
pub truncate_prompt_tokens: Option<i64>,
/// Which side to truncate from when truncate_prompt_tokens is active
pub truncation_side: Option<TruncationSide>,
/// Restrict output to these token IDs only
pub allowed_token_ids: Option<Vec<u32>>,
@@ -3,6 +3,7 @@
use super::types::CompletionRequest;
use crate::error::{ApiError, bail_invalid_request};
use crate::routes::openai::utils::validate_generation_prompt_truncation;
/// Enforce the minimal compatibility contract for the Rust OpenAI server.
pub(super) fn validate_request_compat(
@@ -85,12 +86,7 @@ pub(super) fn validate_request_compat(
"spaces_between_special_tokens is not supported."
);
}
if request.truncate_prompt_tokens.is_some() {
bail_invalid_request!(
param = "truncate_prompt_tokens",
"truncate_prompt_tokens is not supported."
);
}
validate_generation_prompt_truncation(request.truncate_prompt_tokens, request.echo)?;
Ok(())
}
@@ -6,3 +6,37 @@ pub mod structured_outputs;
pub mod types;
pub mod usage;
pub mod validated_json;
use vllm_text::{PromptTruncation, TruncationSide};
use crate::error::{ApiError, bail_invalid_request};
pub(crate) fn validate_generation_prompt_truncation(
limit: Option<i64>,
echo: bool,
) -> Result<(), ApiError> {
if let Some(limit) = limit
&& limit < -1
{
bail_invalid_request!(
param = "truncate_prompt_tokens",
"truncate_prompt_tokens must be >= -1."
);
}
if echo && limit.is_some() {
bail_invalid_request!(
param = "echo",
"`echo=true` is not supported with `truncate_prompt_tokens`."
);
}
Ok(())
}
pub(crate) fn resolve_generation_prompt_truncation(
limit: Option<i64>,
side: Option<TruncationSide>,
) -> vllm_text::Result<Option<PromptTruncation>> {
limit
.map(|limit| PromptTruncation::from_wire(limit, side.unwrap_or(TruncationSide::Left)))
.transpose()
}
@@ -50,6 +50,7 @@ impl TokenizeCompletionRequest {
intermediate: false,
priority: 0,
cache_salt: None,
prompt_truncation: None,
add_special_tokens: self.add_special_tokens,
data_parallel_rank: None,
session_id: None,
@@ -116,6 +117,7 @@ impl TokenizeChatRequest {
tool_context,
decode_options: TextDecodeOptions::default(),
intermediate: false,
prompt_truncation: None,
priority: 0,
documents: None,
cache_salt: None,