[Feature] Add first-class KV hints request envelope for programmatic KV management (#53423)
Signed-off-by: Karen Chung <karenc@nvidia.com>
This commit is contained in:
@@ -0,0 +1,21 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct KvHintAction {
|
||||
pub action_id: String,
|
||||
pub action_type: String,
|
||||
pub action_version: String,
|
||||
pub payload: BTreeMap<String, serde_json::Value>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct KvHintsEnvelope {
|
||||
pub protocol_version: String,
|
||||
pub message_id: String,
|
||||
pub actions: Vec<KvHintAction>,
|
||||
}
|
||||
@@ -24,6 +24,7 @@ fn is_false(v: &bool) -> bool {
|
||||
|
||||
pub mod dtype;
|
||||
pub mod handshake;
|
||||
pub mod kv_hints;
|
||||
pub mod logprobs;
|
||||
pub mod lora;
|
||||
pub mod multimodal;
|
||||
|
||||
@@ -8,6 +8,7 @@ use serde::{Deserialize, Serialize};
|
||||
use serde_default::DefaultFromSerde;
|
||||
use serde_tuple::{Deserialize_tuple, Serialize_tuple};
|
||||
|
||||
use crate::protocol::kv_hints::KvHintsEnvelope;
|
||||
use crate::protocol::multimodal::MmFeatures;
|
||||
use crate::protocol::sampling::EngineCoreSamplingParams;
|
||||
use crate::protocol::{OpaqueValue, lora};
|
||||
@@ -127,6 +128,8 @@ pub struct EngineCoreRequest {
|
||||
/// Stable session identity shared by related requests.
|
||||
#[serde(default)]
|
||||
pub session_id: Option<String>,
|
||||
#[serde(default)]
|
||||
pub kv_hints: Option<KvHintsEnvelope>,
|
||||
}
|
||||
|
||||
impl EngineCoreRequest {
|
||||
@@ -159,6 +162,7 @@ mod tests {
|
||||
|
||||
use super::*;
|
||||
use crate::protocol::dtype::TensorDtype;
|
||||
use crate::protocol::kv_hints::{KvHintAction, KvHintsEnvelope};
|
||||
use crate::protocol::multimodal::{
|
||||
MmBatchedField, MmFeatureSpec, MmField, MmFieldElem, MmKwargValue, MmModality,
|
||||
PlaceholderRange,
|
||||
@@ -181,6 +185,16 @@ mod tests {
|
||||
arrival_time: 1234.5,
|
||||
client_index: 7,
|
||||
session_id: Some("session-1".to_string()),
|
||||
kv_hints: Some(KvHintsEnvelope {
|
||||
protocol_version: "0.1".to_string(),
|
||||
message_id: "msg-1".to_string(),
|
||||
actions: vec![KvHintAction {
|
||||
action_id: "action-1".to_string(),
|
||||
action_type: "example.action".to_string(),
|
||||
action_version: "1.0".to_string(),
|
||||
payload: BTreeMap::from([("key".to_string(), serde_json::json!("value"))]),
|
||||
}],
|
||||
}),
|
||||
..EngineCoreRequest::default()
|
||||
};
|
||||
|
||||
@@ -191,13 +205,14 @@ mod tests {
|
||||
other => panic!("expected array, got {other:?}"),
|
||||
};
|
||||
|
||||
assert_eq!(array.len(), 21);
|
||||
assert_eq!(array.len(), 22);
|
||||
assert_eq!(array[0], Value::from("req-1"));
|
||||
assert_eq!(array[2], Value::Nil);
|
||||
assert_eq!(array[4], Value::Nil);
|
||||
assert_eq!(array[10], Value::Nil);
|
||||
assert_eq!(array[11], Value::from(7));
|
||||
assert_eq!(array[20], Value::from("session-1"));
|
||||
assert!(matches!(&array[21], Value::Map(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -20,6 +20,7 @@ use zeromq::util::PeerIdentity;
|
||||
use zeromq::{DealerSocket, PushSocket, SocketOptions, SubSocket, XPubSocket, ZmqMessage};
|
||||
|
||||
use crate::protocol::handshake::{EngineCoreReadyResponse, HandshakeInitMessage, ReadyMessage};
|
||||
use crate::protocol::kv_hints::{KvHintAction, KvHintsEnvelope};
|
||||
use crate::protocol::logprobs::MaybeWireLogprobs;
|
||||
use crate::protocol::multimodal::{
|
||||
MmFeatureSpec, MmField, MmFieldElem, MmFlatField, MmKwargValue, MmModality, MmSlice,
|
||||
@@ -165,6 +166,16 @@ fn sample_request_with_id(request_id: &str) -> EngineCoreRequest {
|
||||
}),
|
||||
arrival_time: 42.5,
|
||||
session_id: Some("session-1".to_string()),
|
||||
kv_hints: Some(KvHintsEnvelope {
|
||||
protocol_version: "0.1".to_string(),
|
||||
message_id: "msg-1".to_string(),
|
||||
actions: vec![KvHintAction {
|
||||
action_id: "action-1".to_string(),
|
||||
action_type: "example.action".to_string(),
|
||||
action_version: "1.0".to_string(),
|
||||
payload: BTreeMap::from([("key".to_string(), serde_json::json!("value"))]),
|
||||
}],
|
||||
}),
|
||||
..EngineCoreRequest::default()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -52,6 +52,19 @@ class EngineCoreSamplingParams(msgspec.Struct, dict=True, omit_defaults=True):
|
||||
output_kind: RequestOutputKind = RequestOutputKind.DELTA
|
||||
|
||||
|
||||
class KvHintAction(msgspec.Struct, frozen=True):
|
||||
action_id: str
|
||||
action_type: str
|
||||
action_version: str
|
||||
payload: dict[str, object]
|
||||
|
||||
|
||||
class KvHintsEnvelope(msgspec.Struct, frozen=True):
|
||||
protocol_version: str
|
||||
message_id: str
|
||||
actions: list[KvHintAction]
|
||||
|
||||
|
||||
class EngineCoreRequest(
|
||||
msgspec.Struct,
|
||||
array_like=True,
|
||||
@@ -78,6 +91,7 @@ class EngineCoreRequest(
|
||||
reasoning_parser_kwargs: dict[str, object] | None = None
|
||||
abort_immediately: bool = False
|
||||
session_id: str | None = None
|
||||
kv_hints: KvHintsEnvelope | None = None
|
||||
|
||||
|
||||
class EngineCoreOutput(
|
||||
@@ -153,6 +167,18 @@ request = EngineCoreRequest(
|
||||
arrival_time=42.5,
|
||||
client_index=0,
|
||||
session_id="session-1",
|
||||
kv_hints=KvHintsEnvelope(
|
||||
protocol_version="0.1",
|
||||
message_id="msg-1",
|
||||
actions=[
|
||||
KvHintAction(
|
||||
action_id="action-1",
|
||||
action_type="example.action",
|
||||
action_version="1.0",
|
||||
payload={"key": "value"},
|
||||
)
|
||||
],
|
||||
),
|
||||
)
|
||||
|
||||
# All defaults -> empty map. Regression guard for the sparse-map decode.
|
||||
|
||||
Reference in New Issue
Block a user