[Rust Frontend] Add support for truncate_prompt_tokens and truncation_side (#48584)

Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: pranavthakur0-0 <pranavthakurdew@gmail.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Pranav Thakur
2026-08-29 00:16:37 +00:00
committed by GitHub
co-authored by Bugen Zhao
parent 1a16c2ad2c
commit 085e9bb07e
17 changed files with 424 additions and 35 deletions
+1
View File
@@ -195,6 +195,7 @@ impl ChatRequestProcessor {
sampling_params: request.sampling_params,
decode_options: request.decode_options,
intermediate: request.intermediate,
prompt_truncation: request.prompt_truncation,
priority: request.priority,
cache_salt: request.cache_salt,
add_special_tokens: request.add_special_tokens,
+9 -1
View File
@@ -8,8 +8,8 @@ use serde::{Deserialize, Serialize};
use serde_json::Value;
use vllm_engine_core_client::protocol::lora::LoraRequest;
pub use vllm_parser::tool::Tool as ChatTool;
pub use vllm_text::SamplingParams;
use vllm_text::TextDecodeOptions;
pub use vllm_text::{PromptTruncation, SamplingParams};
use crate::AssistantMessageExt;
use crate::error::{Error, Result};
@@ -570,6 +570,8 @@ pub struct ChatRequest {
/// output. If `true`, callers may receive zero or more incremental
/// content events before the final terminal one.
pub intermediate: bool,
/// Prompt-truncation policy resolved by the caller.
pub prompt_truncation: Option<PromptTruncation>,
/// Request scheduling priority (lower means earlier handling; default 0).
pub priority: i32,
/// Documents for RAG (retrieval-augmented generation), passed to the chat
@@ -601,6 +603,7 @@ impl ChatRequest {
tool_context: ResolvedToolContext::default(),
decode_options: TextDecodeOptions::default(),
intermediate: true,
prompt_truncation: None,
priority: 0,
documents: None,
cache_salt: None,
@@ -627,6 +630,11 @@ impl ChatRequest {
(GenerationPromptMode::NoGenerationPrompt, _)
| (GenerationPromptMode::StartNewAssistant, _) => {}
}
if self.has_multimodal() && self.prompt_truncation.is_some() {
return Err(Error::Text(
vllm_text::Error::TruncateUnsupportedWithMultimodal,
));
}
Ok(())
}