feat(grpc): add KV event source discovery (#50033)
Signed-off-by: Connor Carpenter <connorc@nvidia.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
Nick Hill
parent
2ecd8645d8
commit
48fc2e2797
@@ -8,6 +8,7 @@ service Control {
|
||||
rpc GetServerInfo (GetServerInfoRequest) returns (ServerInfo) {}
|
||||
rpc GetModelInfo (GetModelInfoRequest) returns (ModelInfo) {}
|
||||
rpc Abort (AbortRequest) returns (AbortResponse) {}
|
||||
rpc GetKvEventSources (GetKvEventSourcesRequest) returns (GetKvEventSourcesResponse) {}
|
||||
}
|
||||
|
||||
message GetServerInfoRequest {}
|
||||
@@ -51,3 +52,23 @@ message AbortRequest {
|
||||
}
|
||||
|
||||
message AbortResponse {}
|
||||
|
||||
// ======================================================================================
|
||||
// KV discovery
|
||||
// ======================================================================================
|
||||
|
||||
message GetKvEventSourcesRequest {}
|
||||
message GetKvEventSourcesResponse { repeated KvEventSource sources = 1; }
|
||||
|
||||
message KvEventSource {
|
||||
string transport = 1;
|
||||
string endpoint = 2;
|
||||
string topic = 3;
|
||||
string replay_endpoint = 4;
|
||||
optional uint32 data_parallel_rank = 5;
|
||||
string encoding = 6;
|
||||
uint32 schema_version = 7;
|
||||
uint32 buffer_steps = 8;
|
||||
uint32 hwm = 9;
|
||||
uint32 max_queue_size = 10;
|
||||
}
|
||||
|
||||
@@ -66,6 +66,7 @@ pub fn default_ready_response() -> EngineCoreReadyResponse {
|
||||
instance_id: "test-instance".to_string(),
|
||||
kv_cache_size_tokens: None,
|
||||
kv_cache_max_concurrency: None,
|
||||
kv_events_config: None,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -24,6 +24,19 @@ pub struct ReadyMessage {
|
||||
pub parallel_config_hash: Option<String>,
|
||||
}
|
||||
|
||||
/// KV-event publisher configuration reported by EngineCore.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct KvEventsConfig {
|
||||
pub enable_kv_cache_events: bool,
|
||||
pub publisher: String,
|
||||
pub endpoint: String,
|
||||
pub replay_endpoint: Option<String>,
|
||||
pub buffer_steps: u32,
|
||||
pub hwm: u32,
|
||||
pub max_queue_size: u32,
|
||||
pub topic: String,
|
||||
}
|
||||
|
||||
/// Post-initialization configuration sent from each engine on the input socket
|
||||
/// registration message, after the handshake completes.
|
||||
///
|
||||
@@ -71,6 +84,9 @@ pub struct EngineCoreReadyResponse {
|
||||
pub kv_cache_size_tokens: Option<u64>,
|
||||
/// Maximum achievable request concurrency given the KV cache, if reported.
|
||||
pub kv_cache_max_concurrency: Option<f64>,
|
||||
/// KV-event publisher configuration, if configured.
|
||||
#[serde(default)]
|
||||
pub kv_events_config: Option<KvEventsConfig>,
|
||||
}
|
||||
|
||||
/// Frontend-owned ZMQ addresses that are sent to the engine during startup
|
||||
|
||||
@@ -19,7 +19,7 @@ use zeromq::prelude::{Socket, SocketRecv, SocketSend};
|
||||
use zeromq::util::PeerIdentity;
|
||||
use zeromq::{DealerSocket, PushSocket, SocketOptions, SubSocket, XPubSocket, ZmqMessage};
|
||||
|
||||
use crate::protocol::handshake::{HandshakeInitMessage, ReadyMessage};
|
||||
use crate::protocol::handshake::{EngineCoreReadyResponse, HandshakeInitMessage, ReadyMessage};
|
||||
use crate::protocol::logprobs::MaybeWireLogprobs;
|
||||
use crate::protocol::multimodal::{
|
||||
MmFeatureSpec, MmField, MmFieldElem, MmFlatField, MmKwargValue, MmSlice, PlaceholderRange,
|
||||
@@ -2677,6 +2677,21 @@ fn python_msgpack_fixtures_match_rust_encoding() {
|
||||
rust_ready_keys, python_ready_keys,
|
||||
"EngineCoreReadyResponse drifted from the Python dataclass",
|
||||
);
|
||||
|
||||
let ready_response: EngineCoreReadyResponse =
|
||||
rmp_serde::from_slice(&hex::decode(ready_response_hex).unwrap()).unwrap();
|
||||
let kv_events_config = ready_response.kv_events_config.expect("KV events config should decode");
|
||||
assert!(kv_events_config.enable_kv_cache_events);
|
||||
assert_eq!(kv_events_config.publisher, "zmq");
|
||||
assert_eq!(kv_events_config.endpoint, "tcp://127.0.0.1:5557");
|
||||
assert_eq!(
|
||||
kv_events_config.replay_endpoint.as_deref(),
|
||||
Some("tcp://127.0.0.1:5558")
|
||||
);
|
||||
assert_eq!(kv_events_config.topic, "kv");
|
||||
assert_eq!(kv_events_config.buffer_steps, 10_000);
|
||||
assert_eq!(kv_events_config.hwm, 100_000);
|
||||
assert_eq!(kv_events_config.max_queue_size, 100_000);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
|
||||
@@ -353,6 +353,18 @@ multipart_prompt_logprobs = engine_outputs_wire(
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class KVEventsConfig:
|
||||
enable_kv_cache_events: bool
|
||||
publisher: str
|
||||
endpoint: str
|
||||
replay_endpoint: str | None
|
||||
buffer_steps: int
|
||||
hwm: int
|
||||
max_queue_size: int
|
||||
topic: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class EngineCoreReadyResponse:
|
||||
max_model_len: int
|
||||
@@ -372,6 +384,7 @@ class EngineCoreReadyResponse:
|
||||
instance_id: str
|
||||
kv_cache_size_tokens: int | None = None
|
||||
kv_cache_max_concurrency: float | None = None
|
||||
kv_events_config: KVEventsConfig | None = None
|
||||
|
||||
|
||||
ready_response = EngineCoreReadyResponse(
|
||||
@@ -390,6 +403,16 @@ ready_response = EngineCoreReadyResponse(
|
||||
max_num_seqs=256,
|
||||
max_num_batched_tokens=8192,
|
||||
instance_id="test-instance",
|
||||
kv_events_config=KVEventsConfig(
|
||||
enable_kv_cache_events=True,
|
||||
publisher="zmq",
|
||||
endpoint="tcp://127.0.0.1:5557",
|
||||
replay_endpoint="tcp://127.0.0.1:5558",
|
||||
buffer_steps=10_000,
|
||||
hwm=100_000,
|
||||
max_queue_size=100_000,
|
||||
topic="kv",
|
||||
),
|
||||
)
|
||||
|
||||
print(msgspec.msgpack.encode(request).hex())
|
||||
|
||||
@@ -103,4 +103,33 @@ impl pb::control_server::Control for ControlServiceImpl {
|
||||
.map_err(|error| Status::internal(error.to_report_string()))?;
|
||||
Ok(Response::new(pb::AbortResponse {}))
|
||||
}
|
||||
|
||||
async fn get_kv_event_sources(
|
||||
&self,
|
||||
_request: Request<pb::GetKvEventSourcesRequest>,
|
||||
) -> Result<Response<pb::GetKvEventSourcesResponse>, Status> {
|
||||
let client = self.state.engine_core_client();
|
||||
let sources = client.ready_responses().into_iter().filter_map(kv_event_source).collect();
|
||||
Ok(Response::new(pb::GetKvEventSourcesResponse { sources }))
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn kv_event_source(response: &EngineCoreReadyResponse) -> Option<pb::KvEventSource> {
|
||||
let config = response.kv_events_config.as_ref()?;
|
||||
if !config.enable_kv_cache_events || config.publisher != "zmq" {
|
||||
return None;
|
||||
}
|
||||
|
||||
Some(pb::KvEventSource {
|
||||
transport: "zmq".to_string(),
|
||||
endpoint: config.endpoint.clone(),
|
||||
topic: config.topic.clone(),
|
||||
replay_endpoint: config.replay_endpoint.clone().unwrap_or_default(),
|
||||
data_parallel_rank: Some(response.data_parallel_rank),
|
||||
encoding: "msgpack".to_string(),
|
||||
schema_version: 1,
|
||||
buffer_steps: config.buffer_steps,
|
||||
hwm: config.hwm,
|
||||
max_queue_size: config.max_queue_size,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -29,6 +29,7 @@ use vllm_engine_core_client::mock_engine::{
|
||||
DEFAULT_MOCK_BLOCK_SIZE, DEFAULT_MOCK_MAX_MODEL_LEN, DEFAULT_MOCK_NUM_GPU_BLOCKS,
|
||||
default_ready_response,
|
||||
};
|
||||
use vllm_engine_core_client::protocol::handshake::KvEventsConfig;
|
||||
use vllm_engine_core_client::protocol::output::{
|
||||
EngineCoreFinishReason, EngineCoreOutput, EngineCoreOutputs, RequestBatchOutputs,
|
||||
};
|
||||
@@ -44,6 +45,7 @@ use vllm_tokenizer::test_utils::TestTokenizer;
|
||||
use zeromq::prelude::{SocketRecv, SocketSend};
|
||||
use zeromq::{DealerSocket, PushSocket, ZmqMessage};
|
||||
|
||||
use super::control::kv_event_source;
|
||||
use super::pb::control_client::ControlClient;
|
||||
use super::pb::inference_client::InferenceClient;
|
||||
use super::{ControlServer, ControlServiceImpl, InferenceServer, InferenceServiceImpl, pb};
|
||||
@@ -1319,6 +1321,40 @@ async fn control_aggregates_multi_engine_capacity() {
|
||||
|
||||
drop(engine_tasks);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn kv_event_source_filters_and_exposes_zmq_publisher() {
|
||||
let mut ready = default_ready_response();
|
||||
ready.data_parallel_rank = 2;
|
||||
ready.kv_events_config = Some(KvEventsConfig {
|
||||
enable_kv_cache_events: false,
|
||||
publisher: "null".to_string(),
|
||||
endpoint: "tcp://*:5559".to_string(),
|
||||
replay_endpoint: Some("tcp://*:5560".to_string()),
|
||||
buffer_steps: 10_000,
|
||||
hwm: 100_000,
|
||||
max_queue_size: 100_000,
|
||||
topic: "kv".to_string(),
|
||||
});
|
||||
|
||||
assert!(kv_event_source(&ready).is_none());
|
||||
|
||||
let config = ready.kv_events_config.as_mut().unwrap();
|
||||
config.enable_kv_cache_events = true;
|
||||
config.publisher = "zmq".to_string();
|
||||
let source = kv_event_source(&ready).expect("configured ZMQ event source");
|
||||
assert_eq!(source.transport, "zmq");
|
||||
assert_eq!(source.endpoint, "tcp://*:5559");
|
||||
assert_eq!(source.topic, "kv");
|
||||
assert_eq!(source.replay_endpoint, "tcp://*:5560");
|
||||
assert_eq!(source.data_parallel_rank, Some(2));
|
||||
assert_eq!(source.encoding, "msgpack");
|
||||
assert_eq!(source.schema_version, 1);
|
||||
assert_eq!(source.buffer_steps, 10_000);
|
||||
assert_eq!(source.hwm, 100_000);
|
||||
assert_eq!(source.max_queue_size, 100_000);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn grpc_health_transitions_to_not_serving_when_engine_becomes_unhealthy() {
|
||||
|
||||
Reference in New Issue
Block a user