[Feature] Add first-class KV hints request envelope for programmatic KV management (#53423)

Signed-off-by: Karen Chung <karenc@nvidia.com>
This commit is contained in:
Karen Chung
2026-09-21 09:56:11 -07:00
committed by GitHub
parent 4f14516790
commit 986e217158
30 changed files with 340 additions and 18 deletions
+31
View File
@@ -60,6 +60,7 @@ pub fn lower_text_request(
priority: request.priority,
data_parallel_rank: request.data_parallel_rank,
session_id: request.session_id.clone(),
kv_hints: request.kv_hints.clone(),
reasoning_parser_kwargs: request.reasoning_parser_kwargs.clone(),
lora_request: request.lora_request.clone(),
arrival_time: request.arrival_time,
@@ -320,6 +321,7 @@ mod tests {
use std::collections::{BTreeSet, HashMap};
use serial_test::file_serial;
use vllm_engine_core_client::protocol::kv_hints::{KvHintAction, KvHintsEnvelope};
use vllm_engine_core_client::protocol::multimodal::{
MmFeatureSpec, MmModality, PlaceholderRange,
};
@@ -1301,6 +1303,35 @@ mod tests {
assert_eq!(prepared.generate_request.request_id, "text-1");
}
#[test]
fn lower_text_request_passes_kv_hints_through() {
let hints = KvHintsEnvelope {
protocol_version: "0.1".to_string(),
message_id: "msg-1".to_string(),
actions: vec![KvHintAction {
action_id: "action-1".to_string(),
action_type: "example.action".to_string(),
action_version: "1.0".to_string(),
payload: Default::default(),
}],
};
let request = TextRequest {
kv_hints: Some(hints.clone()),
..sample_request()
};
let prepared = lower_text_request(
request,
vec![1, 2, 3],
sample_sampling_hints(),
sample_sampling_limits(),
&stub_tokenizer(),
)
.unwrap();
assert_eq!(prepared.generate_request.kv_hints, Some(hints));
}
#[test]
fn lower_text_request_passes_arrival_time_through() {
let request = TextRequest {
+5
View File
@@ -6,6 +6,7 @@ use std::collections::HashMap;
use enum_as_inner::EnumAsInner;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use vllm_engine_core_client::protocol::kv_hints::KvHintsEnvelope;
use vllm_engine_core_client::protocol::lora::LoraRequest;
use vllm_engine_core_client::protocol::multimodal::MmFeatures;
use vllm_engine_core_client::protocol::request::ReasoningParserKwargs;
@@ -211,6 +212,9 @@ pub struct TextRequest {
/// Stable session identity shared by related requests.
#[serde(default)]
pub session_id: Option<String>,
/// Optional orchestrator-originated KV hints.
#[serde(default)]
pub kv_hints: Option<KvHintsEnvelope>,
/// Optional reasoning-parser kwargs forwarded to engine-side structured
/// output logic.
#[serde(default)]
@@ -242,6 +246,7 @@ impl TextRequest {
add_special_tokens: false,
data_parallel_rank: None,
session_id: None,
kv_hints: None,
reasoning_parser_kwargs: None,
lora_request: None,
arrival_time: None,