[Feature] Add first-class KV hints request envelope for programmatic KV management (#53423)

Signed-off-by: Karen Chung <karenc@nvidia.com>
This commit is contained in:
Karen Chung
2026-09-21 09:56:11 -07:00
committed by GitHub
parent 4f14516790
commit 986e217158
30 changed files with 340 additions and 18 deletions
+1
View File
@@ -194,6 +194,7 @@ impl ChatRequestProcessor {
add_special_tokens: request.add_special_tokens,
data_parallel_rank: request.data_parallel_rank,
session_id: request.session_id,
kv_hints: None,
reasoning_parser_kwargs,
lora_request: request.lora_request,
arrival_time: Some(arrival_time),
@@ -0,0 +1,21 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
use std::collections::BTreeMap;
use serde::{Deserialize, Serialize};
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct KvHintAction {
pub action_id: String,
pub action_type: String,
pub action_version: String,
pub payload: BTreeMap<String, serde_json::Value>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct KvHintsEnvelope {
pub protocol_version: String,
pub message_id: String,
pub actions: Vec<KvHintAction>,
}
@@ -24,6 +24,7 @@ fn is_false(v: &bool) -> bool {
pub mod dtype;
pub mod handshake;
pub mod kv_hints;
pub mod logprobs;
pub mod lora;
pub mod multimodal;
@@ -8,6 +8,7 @@ use serde::{Deserialize, Serialize};
use serde_default::DefaultFromSerde;
use serde_tuple::{Deserialize_tuple, Serialize_tuple};
use crate::protocol::kv_hints::KvHintsEnvelope;
use crate::protocol::multimodal::MmFeatures;
use crate::protocol::sampling::EngineCoreSamplingParams;
use crate::protocol::{OpaqueValue, lora};
@@ -127,6 +128,8 @@ pub struct EngineCoreRequest {
/// Stable session identity shared by related requests.
#[serde(default)]
pub session_id: Option<String>,
#[serde(default)]
pub kv_hints: Option<KvHintsEnvelope>,
}
impl EngineCoreRequest {
@@ -159,6 +162,7 @@ mod tests {
use super::*;
use crate::protocol::dtype::TensorDtype;
use crate::protocol::kv_hints::{KvHintAction, KvHintsEnvelope};
use crate::protocol::multimodal::{
MmBatchedField, MmFeatureSpec, MmField, MmFieldElem, MmKwargValue, MmModality,
PlaceholderRange,
@@ -181,6 +185,16 @@ mod tests {
arrival_time: 1234.5,
client_index: 7,
session_id: Some("session-1".to_string()),
kv_hints: Some(KvHintsEnvelope {
protocol_version: "0.1".to_string(),
message_id: "msg-1".to_string(),
actions: vec![KvHintAction {
action_id: "action-1".to_string(),
action_type: "example.action".to_string(),
action_version: "1.0".to_string(),
payload: BTreeMap::from([("key".to_string(), serde_json::json!("value"))]),
}],
}),
..EngineCoreRequest::default()
};
@@ -191,13 +205,14 @@ mod tests {
other => panic!("expected array, got {other:?}"),
};
assert_eq!(array.len(), 21);
assert_eq!(array.len(), 22);
assert_eq!(array[0], Value::from("req-1"));
assert_eq!(array[2], Value::Nil);
assert_eq!(array[4], Value::Nil);
assert_eq!(array[10], Value::Nil);
assert_eq!(array[11], Value::from(7));
assert_eq!(array[20], Value::from("session-1"));
assert!(matches!(&array[21], Value::Map(_)));
}
#[test]
@@ -20,6 +20,7 @@ use zeromq::util::PeerIdentity;
use zeromq::{DealerSocket, PushSocket, SocketOptions, SubSocket, XPubSocket, ZmqMessage};
use crate::protocol::handshake::{EngineCoreReadyResponse, HandshakeInitMessage, ReadyMessage};
use crate::protocol::kv_hints::{KvHintAction, KvHintsEnvelope};
use crate::protocol::logprobs::MaybeWireLogprobs;
use crate::protocol::multimodal::{
MmFeatureSpec, MmField, MmFieldElem, MmFlatField, MmKwargValue, MmModality, MmSlice,
@@ -165,6 +166,16 @@ fn sample_request_with_id(request_id: &str) -> EngineCoreRequest {
}),
arrival_time: 42.5,
session_id: Some("session-1".to_string()),
kv_hints: Some(KvHintsEnvelope {
protocol_version: "0.1".to_string(),
message_id: "msg-1".to_string(),
actions: vec![KvHintAction {
action_id: "action-1".to_string(),
action_type: "example.action".to_string(),
action_version: "1.0".to_string(),
payload: BTreeMap::from([("key".to_string(), serde_json::json!("value"))]),
}],
}),
..EngineCoreRequest::default()
}
}
@@ -52,6 +52,19 @@ class EngineCoreSamplingParams(msgspec.Struct, dict=True, omit_defaults=True):
output_kind: RequestOutputKind = RequestOutputKind.DELTA
class KvHintAction(msgspec.Struct, frozen=True):
action_id: str
action_type: str
action_version: str
payload: dict[str, object]
class KvHintsEnvelope(msgspec.Struct, frozen=True):
protocol_version: str
message_id: str
actions: list[KvHintAction]
class EngineCoreRequest(
msgspec.Struct,
array_like=True,
@@ -78,6 +91,7 @@ class EngineCoreRequest(
reasoning_parser_kwargs: dict[str, object] | None = None
abort_immediately: bool = False
session_id: str | None = None
kv_hints: KvHintsEnvelope | None = None
class EngineCoreOutput(
@@ -153,6 +167,18 @@ request = EngineCoreRequest(
arrival_time=42.5,
client_index=0,
session_id="session-1",
kv_hints=KvHintsEnvelope(
protocol_version="0.1",
message_id="msg-1",
actions=[
KvHintAction(
action_id="action-1",
action_type="example.action",
action_version="1.0",
payload={"key": "value"},
)
],
),
)
# All defaults -> empty map. Regression guard for the sparse-map decode.
@@ -60,6 +60,7 @@ fn build_request(request_id: String, max_tokens: u32) -> GenerateRequest {
priority: 0,
data_parallel_rank: None,
session_id: None,
kv_hints: None,
reasoning_parser_kwargs: None,
lora_request: None,
}
+20
View File
@@ -4,6 +4,7 @@
use std::collections::BTreeMap;
use uuid::Uuid;
use vllm_engine_core_client::protocol::kv_hints::KvHintsEnvelope;
use vllm_engine_core_client::protocol::lora::LoraRequest;
use vllm_engine_core_client::protocol::multimodal::MmFeatures;
use vllm_engine_core_client::protocol::request::{EngineCoreRequest, ReasoningParserKwargs};
@@ -48,6 +49,8 @@ pub struct GenerateRequest {
pub data_parallel_rank: Option<u32>,
/// Stable session identity shared by related requests.
pub session_id: Option<String>,
/// Optional orchestrator-originated KV hints.
pub kv_hints: Option<KvHintsEnvelope>,
/// Optional reasoning-parser kwargs forwarded to engine-side structured
/// output logic.
pub reasoning_parser_kwargs: Option<ReasoningParserKwargs>,
@@ -79,6 +82,7 @@ impl GenerateRequest {
priority,
data_parallel_rank,
session_id,
kv_hints,
reasoning_parser_kwargs,
lora_request,
} = self;
@@ -109,6 +113,7 @@ impl GenerateRequest {
trace_headers,
resumable: false,
session_id,
kv_hints,
external_req_id: Some(external_request_id),
// Rust parser doesn't expose this information, leave it unset and let the
// reasoning logic in engine-sided structured output manager handle it.
@@ -134,6 +139,7 @@ impl PreparedGenerateRequest {
mod tests {
use std::collections::BTreeMap;
use vllm_engine_core_client::protocol::kv_hints::{KvHintAction, KvHintsEnvelope};
use vllm_engine_core_client::protocol::request::ReasoningParserKwargs;
use vllm_engine_core_client::protocol::sampling::EngineCoreSamplingParams;
@@ -155,6 +161,16 @@ mod tests {
priority: 3,
data_parallel_rank: Some(2),
session_id: Some("session-1".to_string()),
kv_hints: Some(KvHintsEnvelope {
protocol_version: "0.1".to_string(),
message_id: "msg-1".to_string(),
actions: vec![KvHintAction {
action_id: "action-1".to_string(),
action_type: "example.action".to_string(),
action_version: "1.0".to_string(),
payload: BTreeMap::new(),
}],
}),
reasoning_parser_kwargs: Some(ReasoningParserKwargs {
chat_template_kwargs: [(
"chat_template_kwargs".to_string(),
@@ -183,6 +199,10 @@ mod tests {
assert_eq!(request.cache_salt.as_deref(), Some("salt"));
assert_eq!(request.data_parallel_rank, Some(2));
assert_eq!(request.session_id.as_deref(), Some("session-1"));
assert_eq!(
request.kv_hints.as_ref().map(|hints| hints.message_id.as_str()),
Some("msg-1")
);
assert_eq!(
request.trace_headers,
Some(BTreeMap::from([(
+1
View File
@@ -167,6 +167,7 @@ fn sample_generate_request(request_id: &str, max_tokens: u32) -> GenerateRequest
priority: 0,
data_parallel_rank: None,
session_id: None,
kv_hints: None,
reasoning_parser_kwargs: None,
lora_request: None,
}
+61 -1
View File
@@ -7,6 +7,7 @@
use thiserror_ext::AsReport as _;
use tonic::Status;
use uuid::Uuid;
use vllm_engine_core_client::protocol::kv_hints::{KvHintAction, KvHintsEnvelope};
use vllm_engine_core_client::protocol::output::StopReason;
use vllm_engine_core_client::protocol::structured_outputs::StructuredOutputsParams;
use vllm_text::{
@@ -16,6 +17,32 @@ use vllm_text::{
use super::pb;
fn kv_hints_from_proto(hints: pb::KvHintsEnvelope) -> KvHintsEnvelope {
KvHintsEnvelope {
protocol_version: hints.protocol_version,
message_id: hints.message_id,
actions: hints
.actions
.into_iter()
.map(|action| KvHintAction {
action_id: action.action_id,
action_type: action.action_type,
action_version: action.action_version,
payload: action
.payload
.map(|payload| {
payload
.fields
.into_iter()
.map(|(key, value)| (key, proto_value_to_json(&value)))
.collect()
})
.unwrap_or_default(),
})
.collect(),
}
}
// ========================================================================================
// Request conversion
// ========================================================================================
@@ -56,6 +83,7 @@ pub fn to_text_request(
req.request_id
};
let session_id = req.session_id.filter(|s| !s.is_empty());
let kv_hints = req.kv_hints.map(kv_hints_from_proto);
let sampling = req.sampling.as_ref();
let decoding = req.decoding.as_ref();
@@ -108,6 +136,7 @@ pub fn to_text_request(
add_special_tokens: true,
data_parallel_rank: None,
session_id,
kv_hints,
reasoning_parser_kwargs: None,
lora_request: None,
arrival_time: None,
@@ -537,7 +566,9 @@ mod tests {
use vllm_tokenizer::test_utils::TestTokenizer;
use super::pb::finish_info::{FinishReason as PbFinishReason, StopReason as PbStopReason};
use super::{ResponseOpts, pb, to_finish_info, to_sequence_output, to_text_request};
use super::{
ResponseOpts, json_to_proto_struct, pb, to_finish_info, to_sequence_output, to_text_request,
};
fn base_request() -> pb::GenerateRequest {
pb::GenerateRequest {
@@ -578,6 +609,35 @@ mod tests {
assert_eq!(text.sampling_params.temperature, Some(0.7));
}
#[test]
fn kv_hints_propagate_from_top_level_request_field() {
let req = pb::GenerateRequest {
kv_hints: Some(pb::KvHintsEnvelope {
protocol_version: "0.1".to_string(),
message_id: "msg-1".to_string(),
actions: vec![pb::KvHintAction {
action_id: "action-1".to_string(),
action_type: "example.action".to_string(),
action_version: "1.0".to_string(),
payload: json_to_proto_struct(&serde_json::json!({"source": "g2"})),
}],
}),
..base_request()
};
let text = to_text_request(req, false, &["test-model".to_string()]).expect("convert ok");
let hints = text.kv_hints.expect("kv hints");
assert_eq!(hints.protocol_version, "0.1");
assert_eq!(hints.message_id, "msg-1");
assert_eq!(hints.actions[0].action_id, "action-1");
assert_eq!(hints.actions[0].action_type, "example.action");
assert_eq!(hints.actions[0].action_version, "1.0");
assert_eq!(
hints.actions[0].payload.get("source"),
Some(&serde_json::json!("g2"))
);
}
#[test]
fn unset_temperature_defaults_to_greedy() {
let text = to_text_request(base_request(), false, &["test-model".to_string()])
@@ -82,6 +82,7 @@ pub(super) fn prepare_generate_request(
add_special_tokens: false,
data_parallel_rank: ctx.data_parallel_rank,
session_id: ctx.session_id,
kv_hints: None,
reasoning_parser_kwargs: None,
lora_request: lora_resolution.lora_request.clone(),
arrival_time: None,
@@ -155,6 +155,7 @@ pub(super) fn prepare_completion_request(
add_special_tokens: request.add_special_tokens,
data_parallel_rank: ctx.data_parallel_rank,
session_id,
kv_hints: None,
reasoning_parser_kwargs: None,
lora_request: lora_resolution.lora_request.clone(),
arrival_time: None,
@@ -54,6 +54,7 @@ impl TokenizeCompletionRequest {
add_special_tokens: self.add_special_tokens,
data_parallel_rank: None,
session_id: None,
kv_hints: None,
reasoning_parser_kwargs: None,
lora_request: None,
arrival_time: None,
+31
View File
@@ -60,6 +60,7 @@ pub fn lower_text_request(
priority: request.priority,
data_parallel_rank: request.data_parallel_rank,
session_id: request.session_id.clone(),
kv_hints: request.kv_hints.clone(),
reasoning_parser_kwargs: request.reasoning_parser_kwargs.clone(),
lora_request: request.lora_request.clone(),
arrival_time: request.arrival_time,
@@ -320,6 +321,7 @@ mod tests {
use std::collections::{BTreeSet, HashMap};
use serial_test::file_serial;
use vllm_engine_core_client::protocol::kv_hints::{KvHintAction, KvHintsEnvelope};
use vllm_engine_core_client::protocol::multimodal::{
MmFeatureSpec, MmModality, PlaceholderRange,
};
@@ -1301,6 +1303,35 @@ mod tests {
assert_eq!(prepared.generate_request.request_id, "text-1");
}
#[test]
fn lower_text_request_passes_kv_hints_through() {
let hints = KvHintsEnvelope {
protocol_version: "0.1".to_string(),
message_id: "msg-1".to_string(),
actions: vec![KvHintAction {
action_id: "action-1".to_string(),
action_type: "example.action".to_string(),
action_version: "1.0".to_string(),
payload: Default::default(),
}],
};
let request = TextRequest {
kv_hints: Some(hints.clone()),
..sample_request()
};
let prepared = lower_text_request(
request,
vec![1, 2, 3],
sample_sampling_hints(),
sample_sampling_limits(),
&stub_tokenizer(),
)
.unwrap();
assert_eq!(prepared.generate_request.kv_hints, Some(hints));
}
#[test]
fn lower_text_request_passes_arrival_time_through() {
let request = TextRequest {
+5
View File
@@ -6,6 +6,7 @@ use std::collections::HashMap;
use enum_as_inner::EnumAsInner;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use vllm_engine_core_client::protocol::kv_hints::KvHintsEnvelope;
use vllm_engine_core_client::protocol::lora::LoraRequest;
use vllm_engine_core_client::protocol::multimodal::MmFeatures;
use vllm_engine_core_client::protocol::request::ReasoningParserKwargs;
@@ -211,6 +212,9 @@ pub struct TextRequest {
/// Stable session identity shared by related requests.
#[serde(default)]
pub session_id: Option<String>,
/// Optional orchestrator-originated KV hints.
#[serde(default)]
pub kv_hints: Option<KvHintsEnvelope>,
/// Optional reasoning-parser kwargs forwarded to engine-side structured
/// output logic.
#[serde(default)]
@@ -242,6 +246,7 @@ impl TextRequest {
add_special_tokens: false,
data_parallel_rank: None,
session_id: None,
kv_hints: None,
reasoning_parser_kwargs: None,
lora_request: None,
arrival_time: None,