feat(grpc): add KV event source discovery (#50033)
Signed-off-by: Connor Carpenter <connorc@nvidia.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
Nick Hill
parent
2ecd8645d8
commit
48fc2e2797
@@ -103,4 +103,33 @@ impl pb::control_server::Control for ControlServiceImpl {
|
||||
.map_err(|error| Status::internal(error.to_report_string()))?;
|
||||
Ok(Response::new(pb::AbortResponse {}))
|
||||
}
|
||||
|
||||
async fn get_kv_event_sources(
|
||||
&self,
|
||||
_request: Request<pb::GetKvEventSourcesRequest>,
|
||||
) -> Result<Response<pb::GetKvEventSourcesResponse>, Status> {
|
||||
let client = self.state.engine_core_client();
|
||||
let sources = client.ready_responses().into_iter().filter_map(kv_event_source).collect();
|
||||
Ok(Response::new(pb::GetKvEventSourcesResponse { sources }))
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn kv_event_source(response: &EngineCoreReadyResponse) -> Option<pb::KvEventSource> {
|
||||
let config = response.kv_events_config.as_ref()?;
|
||||
if !config.enable_kv_cache_events || config.publisher != "zmq" {
|
||||
return None;
|
||||
}
|
||||
|
||||
Some(pb::KvEventSource {
|
||||
transport: "zmq".to_string(),
|
||||
endpoint: config.endpoint.clone(),
|
||||
topic: config.topic.clone(),
|
||||
replay_endpoint: config.replay_endpoint.clone().unwrap_or_default(),
|
||||
data_parallel_rank: Some(response.data_parallel_rank),
|
||||
encoding: "msgpack".to_string(),
|
||||
schema_version: 1,
|
||||
buffer_steps: config.buffer_steps,
|
||||
hwm: config.hwm,
|
||||
max_queue_size: config.max_queue_size,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -29,6 +29,7 @@ use vllm_engine_core_client::mock_engine::{
|
||||
DEFAULT_MOCK_BLOCK_SIZE, DEFAULT_MOCK_MAX_MODEL_LEN, DEFAULT_MOCK_NUM_GPU_BLOCKS,
|
||||
default_ready_response,
|
||||
};
|
||||
use vllm_engine_core_client::protocol::handshake::KvEventsConfig;
|
||||
use vllm_engine_core_client::protocol::output::{
|
||||
EngineCoreFinishReason, EngineCoreOutput, EngineCoreOutputs, RequestBatchOutputs,
|
||||
};
|
||||
@@ -44,6 +45,7 @@ use vllm_tokenizer::test_utils::TestTokenizer;
|
||||
use zeromq::prelude::{SocketRecv, SocketSend};
|
||||
use zeromq::{DealerSocket, PushSocket, ZmqMessage};
|
||||
|
||||
use super::control::kv_event_source;
|
||||
use super::pb::control_client::ControlClient;
|
||||
use super::pb::inference_client::InferenceClient;
|
||||
use super::{ControlServer, ControlServiceImpl, InferenceServer, InferenceServiceImpl, pb};
|
||||
@@ -1319,6 +1321,40 @@ async fn control_aggregates_multi_engine_capacity() {
|
||||
|
||||
drop(engine_tasks);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn kv_event_source_filters_and_exposes_zmq_publisher() {
|
||||
let mut ready = default_ready_response();
|
||||
ready.data_parallel_rank = 2;
|
||||
ready.kv_events_config = Some(KvEventsConfig {
|
||||
enable_kv_cache_events: false,
|
||||
publisher: "null".to_string(),
|
||||
endpoint: "tcp://*:5559".to_string(),
|
||||
replay_endpoint: Some("tcp://*:5560".to_string()),
|
||||
buffer_steps: 10_000,
|
||||
hwm: 100_000,
|
||||
max_queue_size: 100_000,
|
||||
topic: "kv".to_string(),
|
||||
});
|
||||
|
||||
assert!(kv_event_source(&ready).is_none());
|
||||
|
||||
let config = ready.kv_events_config.as_mut().unwrap();
|
||||
config.enable_kv_cache_events = true;
|
||||
config.publisher = "zmq".to_string();
|
||||
let source = kv_event_source(&ready).expect("configured ZMQ event source");
|
||||
assert_eq!(source.transport, "zmq");
|
||||
assert_eq!(source.endpoint, "tcp://*:5559");
|
||||
assert_eq!(source.topic, "kv");
|
||||
assert_eq!(source.replay_endpoint, "tcp://*:5560");
|
||||
assert_eq!(source.data_parallel_rank, Some(2));
|
||||
assert_eq!(source.encoding, "msgpack");
|
||||
assert_eq!(source.schema_version, 1);
|
||||
assert_eq!(source.buffer_steps, 10_000);
|
||||
assert_eq!(source.hwm, 100_000);
|
||||
assert_eq!(source.max_queue_size, 100_000);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn grpc_health_transitions_to_not_serving_when_engine_becomes_unhealthy() {
|
||||
|
||||
Reference in New Issue
Block a user