[Feature] Add first-class KV hints request envelope for programmatic KV management (#53423)
Signed-off-by: Karen Chung <karenc@nvidia.com>
This commit is contained in:
@@ -7,6 +7,7 @@
|
||||
use thiserror_ext::AsReport as _;
|
||||
use tonic::Status;
|
||||
use uuid::Uuid;
|
||||
use vllm_engine_core_client::protocol::kv_hints::{KvHintAction, KvHintsEnvelope};
|
||||
use vllm_engine_core_client::protocol::output::StopReason;
|
||||
use vllm_engine_core_client::protocol::structured_outputs::StructuredOutputsParams;
|
||||
use vllm_text::{
|
||||
@@ -16,6 +17,32 @@ use vllm_text::{
|
||||
|
||||
use super::pb;
|
||||
|
||||
fn kv_hints_from_proto(hints: pb::KvHintsEnvelope) -> KvHintsEnvelope {
|
||||
KvHintsEnvelope {
|
||||
protocol_version: hints.protocol_version,
|
||||
message_id: hints.message_id,
|
||||
actions: hints
|
||||
.actions
|
||||
.into_iter()
|
||||
.map(|action| KvHintAction {
|
||||
action_id: action.action_id,
|
||||
action_type: action.action_type,
|
||||
action_version: action.action_version,
|
||||
payload: action
|
||||
.payload
|
||||
.map(|payload| {
|
||||
payload
|
||||
.fields
|
||||
.into_iter()
|
||||
.map(|(key, value)| (key, proto_value_to_json(&value)))
|
||||
.collect()
|
||||
})
|
||||
.unwrap_or_default(),
|
||||
})
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
// ========================================================================================
|
||||
// Request conversion
|
||||
// ========================================================================================
|
||||
@@ -56,6 +83,7 @@ pub fn to_text_request(
|
||||
req.request_id
|
||||
};
|
||||
let session_id = req.session_id.filter(|s| !s.is_empty());
|
||||
let kv_hints = req.kv_hints.map(kv_hints_from_proto);
|
||||
|
||||
let sampling = req.sampling.as_ref();
|
||||
let decoding = req.decoding.as_ref();
|
||||
@@ -108,6 +136,7 @@ pub fn to_text_request(
|
||||
add_special_tokens: true,
|
||||
data_parallel_rank: None,
|
||||
session_id,
|
||||
kv_hints,
|
||||
reasoning_parser_kwargs: None,
|
||||
lora_request: None,
|
||||
arrival_time: None,
|
||||
@@ -537,7 +566,9 @@ mod tests {
|
||||
use vllm_tokenizer::test_utils::TestTokenizer;
|
||||
|
||||
use super::pb::finish_info::{FinishReason as PbFinishReason, StopReason as PbStopReason};
|
||||
use super::{ResponseOpts, pb, to_finish_info, to_sequence_output, to_text_request};
|
||||
use super::{
|
||||
ResponseOpts, json_to_proto_struct, pb, to_finish_info, to_sequence_output, to_text_request,
|
||||
};
|
||||
|
||||
fn base_request() -> pb::GenerateRequest {
|
||||
pb::GenerateRequest {
|
||||
@@ -578,6 +609,35 @@ mod tests {
|
||||
assert_eq!(text.sampling_params.temperature, Some(0.7));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn kv_hints_propagate_from_top_level_request_field() {
|
||||
let req = pb::GenerateRequest {
|
||||
kv_hints: Some(pb::KvHintsEnvelope {
|
||||
protocol_version: "0.1".to_string(),
|
||||
message_id: "msg-1".to_string(),
|
||||
actions: vec![pb::KvHintAction {
|
||||
action_id: "action-1".to_string(),
|
||||
action_type: "example.action".to_string(),
|
||||
action_version: "1.0".to_string(),
|
||||
payload: json_to_proto_struct(&serde_json::json!({"source": "g2"})),
|
||||
}],
|
||||
}),
|
||||
..base_request()
|
||||
};
|
||||
|
||||
let text = to_text_request(req, false, &["test-model".to_string()]).expect("convert ok");
|
||||
let hints = text.kv_hints.expect("kv hints");
|
||||
assert_eq!(hints.protocol_version, "0.1");
|
||||
assert_eq!(hints.message_id, "msg-1");
|
||||
assert_eq!(hints.actions[0].action_id, "action-1");
|
||||
assert_eq!(hints.actions[0].action_type, "example.action");
|
||||
assert_eq!(hints.actions[0].action_version, "1.0");
|
||||
assert_eq!(
|
||||
hints.actions[0].payload.get("source"),
|
||||
Some(&serde_json::json!("g2"))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unset_temperature_defaults_to_greedy() {
|
||||
let text = to_text_request(base_request(), false, &["test-model".to_string()])
|
||||
|
||||
@@ -82,6 +82,7 @@ pub(super) fn prepare_generate_request(
|
||||
add_special_tokens: false,
|
||||
data_parallel_rank: ctx.data_parallel_rank,
|
||||
session_id: ctx.session_id,
|
||||
kv_hints: None,
|
||||
reasoning_parser_kwargs: None,
|
||||
lora_request: lora_resolution.lora_request.clone(),
|
||||
arrival_time: None,
|
||||
|
||||
@@ -155,6 +155,7 @@ pub(super) fn prepare_completion_request(
|
||||
add_special_tokens: request.add_special_tokens,
|
||||
data_parallel_rank: ctx.data_parallel_rank,
|
||||
session_id,
|
||||
kv_hints: None,
|
||||
reasoning_parser_kwargs: None,
|
||||
lora_request: lora_resolution.lora_request.clone(),
|
||||
arrival_time: None,
|
||||
|
||||
@@ -54,6 +54,7 @@ impl TokenizeCompletionRequest {
|
||||
add_special_tokens: self.add_special_tokens,
|
||||
data_parallel_rank: None,
|
||||
session_id: None,
|
||||
kv_hints: None,
|
||||
reasoning_parser_kwargs: None,
|
||||
lora_request: None,
|
||||
arrival_time: None,
|
||||
|
||||
Reference in New Issue
Block a user