feat(grpc): add KV event source discovery (#50033)

Signed-off-by: Connor Carpenter <connorc@nvidia.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Connor Carpenter
2026-07-29 13:46:40 -07:00
committed by GitHub
co-authored by Nick Hill
parent 2ecd8645d8
commit 48fc2e2797
13 changed files with 179 additions and 5 deletions
@@ -66,6 +66,7 @@ pub fn default_ready_response() -> EngineCoreReadyResponse {
instance_id: "test-instance".to_string(),
kv_cache_size_tokens: None,
kv_cache_max_concurrency: None,
kv_events_config: None,
}
}
@@ -24,6 +24,19 @@ pub struct ReadyMessage {
pub parallel_config_hash: Option<String>,
}
/// KV-event publisher configuration reported by EngineCore.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct KvEventsConfig {
pub enable_kv_cache_events: bool,
pub publisher: String,
pub endpoint: String,
pub replay_endpoint: Option<String>,
pub buffer_steps: u32,
pub hwm: u32,
pub max_queue_size: u32,
pub topic: String,
}
/// Post-initialization configuration sent from each engine on the input socket
/// registration message, after the handshake completes.
///
@@ -71,6 +84,9 @@ pub struct EngineCoreReadyResponse {
pub kv_cache_size_tokens: Option<u64>,
/// Maximum achievable request concurrency given the KV cache, if reported.
pub kv_cache_max_concurrency: Option<f64>,
/// KV-event publisher configuration, if configured.
#[serde(default)]
pub kv_events_config: Option<KvEventsConfig>,
}
/// Frontend-owned ZMQ addresses that are sent to the engine during startup
@@ -19,7 +19,7 @@ use zeromq::prelude::{Socket, SocketRecv, SocketSend};
use zeromq::util::PeerIdentity;
use zeromq::{DealerSocket, PushSocket, SocketOptions, SubSocket, XPubSocket, ZmqMessage};
use crate::protocol::handshake::{HandshakeInitMessage, ReadyMessage};
use crate::protocol::handshake::{EngineCoreReadyResponse, HandshakeInitMessage, ReadyMessage};
use crate::protocol::logprobs::MaybeWireLogprobs;
use crate::protocol::multimodal::{
MmFeatureSpec, MmField, MmFieldElem, MmFlatField, MmKwargValue, MmSlice, PlaceholderRange,
@@ -2677,6 +2677,21 @@ fn python_msgpack_fixtures_match_rust_encoding() {
rust_ready_keys, python_ready_keys,
"EngineCoreReadyResponse drifted from the Python dataclass",
);
let ready_response: EngineCoreReadyResponse =
rmp_serde::from_slice(&hex::decode(ready_response_hex).unwrap()).unwrap();
let kv_events_config = ready_response.kv_events_config.expect("KV events config should decode");
assert!(kv_events_config.enable_kv_cache_events);
assert_eq!(kv_events_config.publisher, "zmq");
assert_eq!(kv_events_config.endpoint, "tcp://127.0.0.1:5557");
assert_eq!(
kv_events_config.replay_endpoint.as_deref(),
Some("tcp://127.0.0.1:5558")
);
assert_eq!(kv_events_config.topic, "kv");
assert_eq!(kv_events_config.buffer_steps, 10_000);
assert_eq!(kv_events_config.hwm, 100_000);
assert_eq!(kv_events_config.max_queue_size, 100_000);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
@@ -353,6 +353,18 @@ multipart_prompt_logprobs = engine_outputs_wire(
)
@dataclass
class KVEventsConfig:
enable_kv_cache_events: bool
publisher: str
endpoint: str
replay_endpoint: str | None
buffer_steps: int
hwm: int
max_queue_size: int
topic: str
@dataclass
class EngineCoreReadyResponse:
max_model_len: int
@@ -372,6 +384,7 @@ class EngineCoreReadyResponse:
instance_id: str
kv_cache_size_tokens: int | None = None
kv_cache_max_concurrency: float | None = None
kv_events_config: KVEventsConfig | None = None
ready_response = EngineCoreReadyResponse(
@@ -390,6 +403,16 @@ ready_response = EngineCoreReadyResponse(
max_num_seqs=256,
max_num_batched_tokens=8192,
instance_id="test-instance",
kv_events_config=KVEventsConfig(
enable_kv_cache_events=True,
publisher="zmq",
endpoint="tcp://127.0.0.1:5557",
replay_endpoint="tcp://127.0.0.1:5558",
buffer_steps=10_000,
hwm=100_000,
max_queue_size=100_000,
topic="kv",
),
)
print(msgspec.msgpack.encode(request).hex())