[Rust Frontend] Add support for truncate_prompt_tokens and truncation_side (#48584)
Co-authored-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: pranavthakur0-0 <pranavthakurdew@gmail.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
co-authored by
Bugen Zhao
parent
1a16c2ad2c
commit
085e9bb07e
@@ -195,6 +195,7 @@ impl ChatRequestProcessor {
|
||||
sampling_params: request.sampling_params,
|
||||
decode_options: request.decode_options,
|
||||
intermediate: request.intermediate,
|
||||
prompt_truncation: request.prompt_truncation,
|
||||
priority: request.priority,
|
||||
cache_salt: request.cache_salt,
|
||||
add_special_tokens: request.add_special_tokens,
|
||||
|
||||
@@ -8,8 +8,8 @@ use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use vllm_engine_core_client::protocol::lora::LoraRequest;
|
||||
pub use vllm_parser::tool::Tool as ChatTool;
|
||||
pub use vllm_text::SamplingParams;
|
||||
use vllm_text::TextDecodeOptions;
|
||||
pub use vllm_text::{PromptTruncation, SamplingParams};
|
||||
|
||||
use crate::AssistantMessageExt;
|
||||
use crate::error::{Error, Result};
|
||||
@@ -570,6 +570,8 @@ pub struct ChatRequest {
|
||||
/// output. If `true`, callers may receive zero or more incremental
|
||||
/// content events before the final terminal one.
|
||||
pub intermediate: bool,
|
||||
/// Prompt-truncation policy resolved by the caller.
|
||||
pub prompt_truncation: Option<PromptTruncation>,
|
||||
/// Request scheduling priority (lower means earlier handling; default 0).
|
||||
pub priority: i32,
|
||||
/// Documents for RAG (retrieval-augmented generation), passed to the chat
|
||||
@@ -601,6 +603,7 @@ impl ChatRequest {
|
||||
tool_context: ResolvedToolContext::default(),
|
||||
decode_options: TextDecodeOptions::default(),
|
||||
intermediate: true,
|
||||
prompt_truncation: None,
|
||||
priority: 0,
|
||||
documents: None,
|
||||
cache_salt: None,
|
||||
@@ -627,6 +630,11 @@ impl ChatRequest {
|
||||
(GenerationPromptMode::NoGenerationPrompt, _)
|
||||
| (GenerationPromptMode::StartNewAssistant, _) => {}
|
||||
}
|
||||
if self.has_multimodal() && self.prompt_truncation.is_some() {
|
||||
return Err(Error::Text(
|
||||
vllm_text::Error::TruncateUnsupportedWithMultimodal,
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user