[Feature] Add first-class KV hints request envelope for programmatic KV management (#53423)
Signed-off-by: Karen Chung <karenc@nvidia.com>
This commit is contained in:
@@ -60,6 +60,7 @@ pub fn lower_text_request(
|
||||
priority: request.priority,
|
||||
data_parallel_rank: request.data_parallel_rank,
|
||||
session_id: request.session_id.clone(),
|
||||
kv_hints: request.kv_hints.clone(),
|
||||
reasoning_parser_kwargs: request.reasoning_parser_kwargs.clone(),
|
||||
lora_request: request.lora_request.clone(),
|
||||
arrival_time: request.arrival_time,
|
||||
@@ -320,6 +321,7 @@ mod tests {
|
||||
use std::collections::{BTreeSet, HashMap};
|
||||
|
||||
use serial_test::file_serial;
|
||||
use vllm_engine_core_client::protocol::kv_hints::{KvHintAction, KvHintsEnvelope};
|
||||
use vllm_engine_core_client::protocol::multimodal::{
|
||||
MmFeatureSpec, MmModality, PlaceholderRange,
|
||||
};
|
||||
@@ -1301,6 +1303,35 @@ mod tests {
|
||||
assert_eq!(prepared.generate_request.request_id, "text-1");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lower_text_request_passes_kv_hints_through() {
|
||||
let hints = KvHintsEnvelope {
|
||||
protocol_version: "0.1".to_string(),
|
||||
message_id: "msg-1".to_string(),
|
||||
actions: vec![KvHintAction {
|
||||
action_id: "action-1".to_string(),
|
||||
action_type: "example.action".to_string(),
|
||||
action_version: "1.0".to_string(),
|
||||
payload: Default::default(),
|
||||
}],
|
||||
};
|
||||
let request = TextRequest {
|
||||
kv_hints: Some(hints.clone()),
|
||||
..sample_request()
|
||||
};
|
||||
|
||||
let prepared = lower_text_request(
|
||||
request,
|
||||
vec![1, 2, 3],
|
||||
sample_sampling_hints(),
|
||||
sample_sampling_limits(),
|
||||
&stub_tokenizer(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(prepared.generate_request.kv_hints, Some(hints));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lower_text_request_passes_arrival_time_through() {
|
||||
let request = TextRequest {
|
||||
|
||||
@@ -6,6 +6,7 @@ use std::collections::HashMap;
|
||||
use enum_as_inner::EnumAsInner;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use vllm_engine_core_client::protocol::kv_hints::KvHintsEnvelope;
|
||||
use vllm_engine_core_client::protocol::lora::LoraRequest;
|
||||
use vllm_engine_core_client::protocol::multimodal::MmFeatures;
|
||||
use vllm_engine_core_client::protocol::request::ReasoningParserKwargs;
|
||||
@@ -211,6 +212,9 @@ pub struct TextRequest {
|
||||
/// Stable session identity shared by related requests.
|
||||
#[serde(default)]
|
||||
pub session_id: Option<String>,
|
||||
/// Optional orchestrator-originated KV hints.
|
||||
#[serde(default)]
|
||||
pub kv_hints: Option<KvHintsEnvelope>,
|
||||
/// Optional reasoning-parser kwargs forwarded to engine-side structured
|
||||
/// output logic.
|
||||
#[serde(default)]
|
||||
@@ -242,6 +246,7 @@ impl TextRequest {
|
||||
add_special_tokens: false,
|
||||
data_parallel_rank: None,
|
||||
session_id: None,
|
||||
kv_hints: None,
|
||||
reasoning_parser_kwargs: None,
|
||||
lora_request: None,
|
||||
arrival_time: None,
|
||||
|
||||
Reference in New Issue
Block a user