[Rust Frontend] Add support for truncate_prompt_tokens and truncation_side (#48584)
Co-authored-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: pranavthakur0-0 <pranavthakurdew@gmail.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
co-authored by
Bugen Zhao
parent
1a16c2ad2c
commit
085e9bb07e
@@ -224,6 +224,31 @@ mod tests {
|
||||
assert!(response.error.message.contains("allowed_token_ids"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn truncate_exceeds_budget_maps_to_invalid_request() {
|
||||
let error = vllm_text::Error::TruncatePromptTokensExceedsBudget {
|
||||
value: 8000,
|
||||
budget: 4000,
|
||||
};
|
||||
let api_error = text_submit_error("failed to submit completion request", error);
|
||||
assert_eq!(api_error.status_code(), StatusCode::BAD_REQUEST);
|
||||
let response = api_error.to_error_response();
|
||||
assert_eq!(response.error.error_type, "invalid_request_error");
|
||||
assert!(response.error.message.contains("truncate_prompt_tokens=8000"));
|
||||
assert!(response.error.message.contains("max_tokens = 4000"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_truncate_prompt_tokens_maps_to_invalid_request() {
|
||||
let error = vllm_text::Error::InvalidTruncatePromptTokens { value: -2 };
|
||||
let api_error = text_submit_error("failed to submit completion request", error);
|
||||
assert_eq!(api_error.status_code(), StatusCode::BAD_REQUEST);
|
||||
let response = api_error.to_error_response();
|
||||
assert_eq!(response.error.error_type, "invalid_request_error");
|
||||
assert!(response.error.message.contains("-2"));
|
||||
assert!(response.error.message.contains("must be >= -1"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn other_submit_errors_stay_internal() {
|
||||
let error = vllm_text::Error::Tokenizer("backend exploded".to_string());
|
||||
|
||||
@@ -11,8 +11,8 @@ use vllm_chat::MediaContentPart;
|
||||
use vllm_engine_core_client::protocol::output::StopReason;
|
||||
use vllm_engine_core_client::protocol::structured_outputs::StructuredOutputsParams;
|
||||
use vllm_text::{
|
||||
DecodedLogprobs, DecodedPromptLogprobs, FinishReason, Finished, Prompt, SamplingParams,
|
||||
TextDecodeOptions, TextRequest,
|
||||
DecodedLogprobs, DecodedPromptLogprobs, FinishReason, Finished, Prompt, PromptTruncation,
|
||||
PromptTruncationLimit, SamplingParams, TextDecodeOptions, TextRequest, TruncationSide,
|
||||
};
|
||||
|
||||
use super::pb;
|
||||
@@ -110,11 +110,12 @@ pub fn to_text_request(
|
||||
)));
|
||||
}
|
||||
|
||||
if req.truncate_prompt_tokens != 0 {
|
||||
return Err(Status::invalid_argument(
|
||||
"truncate_prompt_tokens is not supported",
|
||||
));
|
||||
}
|
||||
// Proto3 uses zero as unset; positive values select fixed left truncation.
|
||||
// The -1 input-budget sentinel is outside the uint32 field domain.
|
||||
let prompt_truncation = (req.truncate_prompt_tokens != 0).then_some(PromptTruncation {
|
||||
limit: PromptTruncationLimit::Fixed(u64::from(req.truncate_prompt_tokens)),
|
||||
side: TruncationSide::Left,
|
||||
});
|
||||
|
||||
let prompt = match req.prompt {
|
||||
Some(pb::generate_request::Prompt::Text(text)) => Prompt::Text(text),
|
||||
@@ -173,6 +174,7 @@ pub fn to_text_request(
|
||||
sampling_params,
|
||||
decode_options,
|
||||
intermediate: stream,
|
||||
prompt_truncation,
|
||||
priority: req.priority,
|
||||
cache_salt: kv.map(|k| &k.cache_salt).filter(|s| !s.is_empty()).cloned(),
|
||||
add_special_tokens: true,
|
||||
|
||||
@@ -73,6 +73,7 @@ pub(super) fn prepare_generate_request(
|
||||
sampling_params,
|
||||
decode_options: TextDecodeOptions::default(),
|
||||
intermediate: false,
|
||||
prompt_truncation: None,
|
||||
priority: request.priority,
|
||||
cache_salt: request.cache_salt,
|
||||
add_special_tokens: false,
|
||||
|
||||
@@ -10,8 +10,9 @@ use vllm_chat::{
|
||||
|
||||
use super::types::ChatCompletionRequest;
|
||||
use super::validate;
|
||||
use crate::error::{ApiError, bail_invalid_request, chat_submit_error};
|
||||
use crate::error::{ApiError, bail_invalid_request, chat_submit_error, text_submit_error};
|
||||
use crate::lora::LoraModelResolution;
|
||||
use crate::routes::openai::utils::resolve_generation_prompt_truncation;
|
||||
use crate::routes::openai::utils::structured_outputs::convert_from_response_format;
|
||||
use crate::routes::openai::utils::types::{
|
||||
ChatMessage, ContentPart, MessageContent, Tool, ToolChoice, ToolChoiceValue, ToolReference,
|
||||
@@ -71,6 +72,12 @@ pub(super) fn prepare_chat_request(
|
||||
) -> Result<PreparedRequest, ApiError> {
|
||||
validate::validate_request_compat(&request, &lora_resolution.model_names)?;
|
||||
|
||||
let prompt_truncation = resolve_generation_prompt_truncation(
|
||||
request.truncate_prompt_tokens,
|
||||
request.truncation_side,
|
||||
)
|
||||
.map_err(|error| text_submit_error("invalid prompt truncation", error))?;
|
||||
|
||||
let request_id = format!("chatcmpl-{}", ctx.request_id);
|
||||
let response_model = lora_resolution
|
||||
.lora_request
|
||||
@@ -184,6 +191,7 @@ pub(super) fn prepare_chat_request(
|
||||
min_tokens: request.min_tokens.unwrap_or(0),
|
||||
},
|
||||
intermediate: request.stream,
|
||||
prompt_truncation,
|
||||
priority: ctx.priority.or(request.priority).unwrap_or(0),
|
||||
documents: request.documents,
|
||||
cache_salt: request.cache_salt,
|
||||
@@ -439,7 +447,9 @@ mod tests {
|
||||
ChatRenderer, ChatTool as VllmChatTool, ChatToolChoice, GenerationPromptMode,
|
||||
KimiK3ChatRenderer, SamplingParams as VllmSamplingParams,
|
||||
};
|
||||
use vllm_text::{Prompt, output::TextDecodeOptions};
|
||||
use vllm_text::{
|
||||
Prompt, PromptTruncation, PromptTruncationLimit, TruncationSide, output::TextDecodeOptions,
|
||||
};
|
||||
use vllm_tokenizer::{Tokenizer, test_utils::TestTokenizer};
|
||||
|
||||
use super::prepare_chat_request;
|
||||
@@ -530,6 +540,27 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_chat_request_preserves_explicit_truncation_side() {
|
||||
let mut request = base_request();
|
||||
request.truncate_prompt_tokens = Some(2);
|
||||
request.truncation_side = Some(TruncationSide::Right);
|
||||
let prepared = prepare_chat_request(
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
)
|
||||
.expect("prepare truncation");
|
||||
|
||||
assert_eq!(
|
||||
prepared.chat_request.prompt_truncation,
|
||||
Some(PromptTruncation {
|
||||
limit: PromptTruncationLimit::Fixed(2),
|
||||
side: TruncationSide::Right,
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_chat_request_accepts_zero_min_tokens() {
|
||||
let request: ChatCompletionRequest = serde_json::from_value(json!({
|
||||
|
||||
@@ -11,6 +11,7 @@ use serde_with::SerializeDisplay;
|
||||
use validator::Validate;
|
||||
use vllm_chat::ReasoningEffort;
|
||||
use vllm_engine_core_client::protocol::sampling::RepetitionDetectionParams;
|
||||
use vllm_text::TruncationSide;
|
||||
|
||||
use crate::routes::openai::utils::structured_outputs::ResponseFormat;
|
||||
use crate::routes::openai::utils::types::{
|
||||
@@ -159,6 +160,9 @@ pub struct ChatCompletionRequest {
|
||||
/// Truncate prompt tokens to this length
|
||||
pub truncate_prompt_tokens: Option<i64>,
|
||||
|
||||
/// Which side to truncate from when truncate_prompt_tokens is active
|
||||
pub truncation_side: Option<TruncationSide>,
|
||||
|
||||
/// Number of prompt logprobs to return
|
||||
pub prompt_logprobs: Option<i32>,
|
||||
|
||||
@@ -290,6 +294,7 @@ impl Default for ChatCompletionRequest {
|
||||
skip_special_tokens: true,
|
||||
spaces_between_special_tokens: true,
|
||||
truncate_prompt_tokens: None,
|
||||
truncation_side: None,
|
||||
prompt_logprobs: None,
|
||||
allowed_token_ids: None,
|
||||
bad_words: None,
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
use super::types::ChatCompletionRequest;
|
||||
use crate::error::{ApiError, bail_invalid_request};
|
||||
use crate::routes::openai::utils::types::{ChatMessage, Tool};
|
||||
use crate::routes::openai::utils::validate_generation_prompt_truncation;
|
||||
|
||||
/// Enforce the minimal compatibility contract for the Rust OpenAI server.
|
||||
pub(super) fn validate_request_compat(
|
||||
@@ -84,11 +85,7 @@ pub(super) fn validate_request_compat(
|
||||
"spaces_between_special_tokens is not supported."
|
||||
);
|
||||
}
|
||||
reject_non_default(
|
||||
request.truncate_prompt_tokens.as_ref(),
|
||||
"truncate_prompt_tokens",
|
||||
"truncate_prompt_tokens is not supported.",
|
||||
)?;
|
||||
validate_generation_prompt_truncation(request.truncate_prompt_tokens, request.echo)?;
|
||||
reject_non_default(
|
||||
request.media_io_kwargs.as_ref(),
|
||||
"media_io_kwargs",
|
||||
|
||||
@@ -6,9 +6,10 @@ use vllm_text::tokenizer::Tokenizer;
|
||||
use vllm_text::{Prompt, SamplingParams, TextDecodeOptions, TextRequest};
|
||||
|
||||
use super::types::CompletionRequest;
|
||||
use crate::error::ApiError;
|
||||
use crate::error::{ApiError, text_submit_error};
|
||||
use crate::lora::LoraModelResolution;
|
||||
use crate::routes::openai::completions::validate;
|
||||
use crate::routes::openai::utils::resolve_generation_prompt_truncation;
|
||||
use crate::routes::openai::utils::structured_outputs::convert_from_response_format_value;
|
||||
use crate::utils::{
|
||||
ResolvedRequestContext, convert_logit_bias, merge_ec_transfer_params, merge_kv_transfer_params,
|
||||
@@ -62,6 +63,12 @@ pub(super) fn prepare_completion_request(
|
||||
) -> Result<PreparedRequest, ApiError> {
|
||||
validate::validate_request_compat(&request, &lora_resolution.model_names)?;
|
||||
|
||||
let prompt_truncation = resolve_generation_prompt_truncation(
|
||||
request.truncate_prompt_tokens,
|
||||
request.truncation_side,
|
||||
)
|
||||
.map_err(|error| text_submit_error("invalid prompt truncation", error))?;
|
||||
|
||||
let request_id = format!("cmpl-{}", ctx.request_id);
|
||||
let response_model = lora_resolution
|
||||
.lora_request
|
||||
@@ -141,6 +148,7 @@ pub(super) fn prepare_completion_request(
|
||||
min_tokens: request.min_tokens.unwrap_or(0),
|
||||
},
|
||||
intermediate: request.stream,
|
||||
prompt_truncation,
|
||||
priority: ctx.priority.or(request.priority).unwrap_or(0),
|
||||
cache_salt: request.cache_salt,
|
||||
add_special_tokens: request.add_special_tokens,
|
||||
@@ -200,7 +208,7 @@ mod tests {
|
||||
use axum::http::HeaderMap;
|
||||
use serde_json::json;
|
||||
use validator::Validate;
|
||||
use vllm_text::Prompt;
|
||||
use vllm_text::{Prompt, PromptTruncation, PromptTruncationLimit, TruncationSide};
|
||||
use vllm_tokenizer::test_utils::TestTokenizer;
|
||||
|
||||
use super::prepare_completion_request;
|
||||
@@ -323,6 +331,31 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_completion_request_uses_left_when_side_is_omitted() {
|
||||
let mut value = base_request_json();
|
||||
value
|
||||
.as_object_mut()
|
||||
.expect("request object")
|
||||
.insert("truncate_prompt_tokens".to_string(), json!(-1));
|
||||
let request = serde_json::from_value(value).expect("parse truncation");
|
||||
let prepared = prepare_completion_request(
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
&test_tokenizer(),
|
||||
)
|
||||
.expect("prepare truncation");
|
||||
|
||||
assert_eq!(
|
||||
prepared.text_request.prompt_truncation,
|
||||
Some(PromptTruncation {
|
||||
limit: PromptTruncationLimit::InputBudget,
|
||||
side: TruncationSide::Left,
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalize_coerces_null_max_tokens_to_default() {
|
||||
// An absent `max_tokens` already gets the serde default.
|
||||
|
||||
@@ -8,7 +8,7 @@ use serde::{Deserialize, Serialize};
|
||||
use serde_json::{Map, Value};
|
||||
use validator::Validate;
|
||||
use vllm_engine_core_client::protocol::sampling::RepetitionDetectionParams;
|
||||
use vllm_text::Prompt;
|
||||
use vllm_text::{Prompt, TruncationSide};
|
||||
|
||||
use crate::routes::openai::utils::types::{
|
||||
LogProbs, Normalizable, PromptLogprobs, StreamOptions, StreamResponseEnvelope, StringOrArray,
|
||||
@@ -138,6 +138,9 @@ pub struct CompletionRequest {
|
||||
/// Truncate prompt tokens to this length
|
||||
pub truncate_prompt_tokens: Option<i64>,
|
||||
|
||||
/// Which side to truncate from when truncate_prompt_tokens is active
|
||||
pub truncation_side: Option<TruncationSide>,
|
||||
|
||||
/// Restrict output to these token IDs only
|
||||
pub allowed_token_ids: Option<Vec<u32>>,
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
use super::types::CompletionRequest;
|
||||
use crate::error::{ApiError, bail_invalid_request};
|
||||
use crate::routes::openai::utils::validate_generation_prompt_truncation;
|
||||
|
||||
/// Enforce the minimal compatibility contract for the Rust OpenAI server.
|
||||
pub(super) fn validate_request_compat(
|
||||
@@ -85,12 +86,7 @@ pub(super) fn validate_request_compat(
|
||||
"spaces_between_special_tokens is not supported."
|
||||
);
|
||||
}
|
||||
if request.truncate_prompt_tokens.is_some() {
|
||||
bail_invalid_request!(
|
||||
param = "truncate_prompt_tokens",
|
||||
"truncate_prompt_tokens is not supported."
|
||||
);
|
||||
}
|
||||
validate_generation_prompt_truncation(request.truncate_prompt_tokens, request.echo)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -6,3 +6,37 @@ pub mod structured_outputs;
|
||||
pub mod types;
|
||||
pub mod usage;
|
||||
pub mod validated_json;
|
||||
|
||||
use vllm_text::{PromptTruncation, TruncationSide};
|
||||
|
||||
use crate::error::{ApiError, bail_invalid_request};
|
||||
|
||||
pub(crate) fn validate_generation_prompt_truncation(
|
||||
limit: Option<i64>,
|
||||
echo: bool,
|
||||
) -> Result<(), ApiError> {
|
||||
if let Some(limit) = limit
|
||||
&& limit < -1
|
||||
{
|
||||
bail_invalid_request!(
|
||||
param = "truncate_prompt_tokens",
|
||||
"truncate_prompt_tokens must be >= -1."
|
||||
);
|
||||
}
|
||||
if echo && limit.is_some() {
|
||||
bail_invalid_request!(
|
||||
param = "echo",
|
||||
"`echo=true` is not supported with `truncate_prompt_tokens`."
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn resolve_generation_prompt_truncation(
|
||||
limit: Option<i64>,
|
||||
side: Option<TruncationSide>,
|
||||
) -> vllm_text::Result<Option<PromptTruncation>> {
|
||||
limit
|
||||
.map(|limit| PromptTruncation::from_wire(limit, side.unwrap_or(TruncationSide::Left)))
|
||||
.transpose()
|
||||
}
|
||||
|
||||
@@ -50,6 +50,7 @@ impl TokenizeCompletionRequest {
|
||||
intermediate: false,
|
||||
priority: 0,
|
||||
cache_salt: None,
|
||||
prompt_truncation: None,
|
||||
add_special_tokens: self.add_special_tokens,
|
||||
data_parallel_rank: None,
|
||||
session_id: None,
|
||||
@@ -116,6 +117,7 @@ impl TokenizeChatRequest {
|
||||
tool_context,
|
||||
decode_options: TextDecodeOptions::default(),
|
||||
intermediate: false,
|
||||
prompt_truncation: None,
|
||||
priority: 0,
|
||||
documents: None,
|
||||
cache_salt: None,
|
||||
|
||||
Reference in New Issue
Block a user