feat(grpc): add KV event source discovery (#50033)
Signed-off-by: Connor Carpenter <connorc@nvidia.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
Nick Hill
parent
2ecd8645d8
commit
48fc2e2797
@@ -66,6 +66,7 @@ pub fn default_ready_response() -> EngineCoreReadyResponse {
|
||||
instance_id: "test-instance".to_string(),
|
||||
kv_cache_size_tokens: None,
|
||||
kv_cache_max_concurrency: None,
|
||||
kv_events_config: None,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -24,6 +24,19 @@ pub struct ReadyMessage {
|
||||
pub parallel_config_hash: Option<String>,
|
||||
}
|
||||
|
||||
/// KV-event publisher configuration reported by EngineCore.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct KvEventsConfig {
|
||||
pub enable_kv_cache_events: bool,
|
||||
pub publisher: String,
|
||||
pub endpoint: String,
|
||||
pub replay_endpoint: Option<String>,
|
||||
pub buffer_steps: u32,
|
||||
pub hwm: u32,
|
||||
pub max_queue_size: u32,
|
||||
pub topic: String,
|
||||
}
|
||||
|
||||
/// Post-initialization configuration sent from each engine on the input socket
|
||||
/// registration message, after the handshake completes.
|
||||
///
|
||||
@@ -71,6 +84,9 @@ pub struct EngineCoreReadyResponse {
|
||||
pub kv_cache_size_tokens: Option<u64>,
|
||||
/// Maximum achievable request concurrency given the KV cache, if reported.
|
||||
pub kv_cache_max_concurrency: Option<f64>,
|
||||
/// KV-event publisher configuration, if configured.
|
||||
#[serde(default)]
|
||||
pub kv_events_config: Option<KvEventsConfig>,
|
||||
}
|
||||
|
||||
/// Frontend-owned ZMQ addresses that are sent to the engine during startup
|
||||
|
||||
@@ -19,7 +19,7 @@ use zeromq::prelude::{Socket, SocketRecv, SocketSend};
|
||||
use zeromq::util::PeerIdentity;
|
||||
use zeromq::{DealerSocket, PushSocket, SocketOptions, SubSocket, XPubSocket, ZmqMessage};
|
||||
|
||||
use crate::protocol::handshake::{HandshakeInitMessage, ReadyMessage};
|
||||
use crate::protocol::handshake::{EngineCoreReadyResponse, HandshakeInitMessage, ReadyMessage};
|
||||
use crate::protocol::logprobs::MaybeWireLogprobs;
|
||||
use crate::protocol::multimodal::{
|
||||
MmFeatureSpec, MmField, MmFieldElem, MmFlatField, MmKwargValue, MmSlice, PlaceholderRange,
|
||||
@@ -2677,6 +2677,21 @@ fn python_msgpack_fixtures_match_rust_encoding() {
|
||||
rust_ready_keys, python_ready_keys,
|
||||
"EngineCoreReadyResponse drifted from the Python dataclass",
|
||||
);
|
||||
|
||||
let ready_response: EngineCoreReadyResponse =
|
||||
rmp_serde::from_slice(&hex::decode(ready_response_hex).unwrap()).unwrap();
|
||||
let kv_events_config = ready_response.kv_events_config.expect("KV events config should decode");
|
||||
assert!(kv_events_config.enable_kv_cache_events);
|
||||
assert_eq!(kv_events_config.publisher, "zmq");
|
||||
assert_eq!(kv_events_config.endpoint, "tcp://127.0.0.1:5557");
|
||||
assert_eq!(
|
||||
kv_events_config.replay_endpoint.as_deref(),
|
||||
Some("tcp://127.0.0.1:5558")
|
||||
);
|
||||
assert_eq!(kv_events_config.topic, "kv");
|
||||
assert_eq!(kv_events_config.buffer_steps, 10_000);
|
||||
assert_eq!(kv_events_config.hwm, 100_000);
|
||||
assert_eq!(kv_events_config.max_queue_size, 100_000);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
|
||||
@@ -353,6 +353,18 @@ multipart_prompt_logprobs = engine_outputs_wire(
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class KVEventsConfig:
|
||||
enable_kv_cache_events: bool
|
||||
publisher: str
|
||||
endpoint: str
|
||||
replay_endpoint: str | None
|
||||
buffer_steps: int
|
||||
hwm: int
|
||||
max_queue_size: int
|
||||
topic: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class EngineCoreReadyResponse:
|
||||
max_model_len: int
|
||||
@@ -372,6 +384,7 @@ class EngineCoreReadyResponse:
|
||||
instance_id: str
|
||||
kv_cache_size_tokens: int | None = None
|
||||
kv_cache_max_concurrency: float | None = None
|
||||
kv_events_config: KVEventsConfig | None = None
|
||||
|
||||
|
||||
ready_response = EngineCoreReadyResponse(
|
||||
@@ -390,6 +403,16 @@ ready_response = EngineCoreReadyResponse(
|
||||
max_num_seqs=256,
|
||||
max_num_batched_tokens=8192,
|
||||
instance_id="test-instance",
|
||||
kv_events_config=KVEventsConfig(
|
||||
enable_kv_cache_events=True,
|
||||
publisher="zmq",
|
||||
endpoint="tcp://127.0.0.1:5557",
|
||||
replay_endpoint="tcp://127.0.0.1:5558",
|
||||
buffer_steps=10_000,
|
||||
hwm=100_000,
|
||||
max_queue_size=100_000,
|
||||
topic="kv",
|
||||
),
|
||||
)
|
||||
|
||||
print(msgspec.msgpack.encode(request).hex())
|
||||
|
||||
Reference in New Issue
Block a user