[Core][Frontend] Bind KV-event publishers at port 0 and expose the bound endpoints (#55844)

Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: mukeshbhandarkar <shell00035@gmail.com>
Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
hangangqiang
2026-09-18 18:11:05 -07:00
committed by GitHub
co-authored by mukeshbhandarkar aoshen02 Claude Fable 5.1 aoshen02
parent 729ebac498
commit 90531daf6d
11 changed files with 282 additions and 14 deletions
+2
View File
@@ -6,6 +6,7 @@ mod cache;
mod collective_rpc;
mod health;
mod inference;
mod kv_event_sources;
mod load;
mod lora;
mod metrics;
@@ -91,6 +92,7 @@ fn build_router_with_options(
.route("/metrics", get(metrics::scrape))
.route("/load", get(load::load))
.route("/version", get(version::version))
.route("/kv_event_sources", get(kv_event_sources::kv_event_sources))
// OpenAI-compatible endpoints
.route("/v1/models", get(openai::list_models))
.route("/v1/completions", post(openai::completions))
@@ -0,0 +1,25 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
use std::collections::BTreeMap;
use std::sync::Arc;
use axum::Json;
use axum::extract::State;
use vllm_engine_core_client::protocol::handshake::KvEventsConfig;
use crate::state::AppState;
/// KV-event publisher config of each data-parallel rank, keyed by rank.
pub async fn kv_event_sources(
State(state): State<Arc<AppState>>,
) -> Json<BTreeMap<u32, KvEventsConfig>> {
Json(
state
.engine_core_client()
.ready_responses()
.into_iter()
.filter_map(|ready| Some((ready.data_parallel_rank, ready.kv_events_config.clone()?)))
.collect(),
)
}
+50
View File
@@ -2054,6 +2054,56 @@ async fn version_returns_engine_vllm_version() {
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn kv_event_sources_reports_each_ready_engine() {
let ready = vllm_engine_core_client::protocol::handshake::EngineCoreReadyResponse {
data_parallel_rank: 1,
kv_events_config: Some(
vllm_engine_core_client::protocol::handshake::KvEventsConfig {
enable_kv_cache_events: true,
publisher: "zmq".to_string(),
endpoint: "tcp://10.0.0.2:41233".to_string(),
replay_endpoint: None,
buffer_steps: 10_000,
hwm: 100_000,
max_queue_size: 100_000,
topic: "kv".to_string(),
},
),
..default_ready_response()
};
let (mut app, _engine_task) = test_dev_mode_app_with_ready(ready).await;
let response = app
.call(
Request::builder()
.uri("/kv_event_sources")
.body(Body::empty())
.expect("build request"),
)
.await
.expect("call app");
assert_eq!(response.status(), StatusCode::OK);
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json");
assert_eq!(
json,
json!({
"1": {
"enable_kv_cache_events": true,
"publisher": "zmq",
"endpoint": "tcp://10.0.0.2:41233",
"replay_endpoint": null,
"buffer_steps": 10000,
"hwm": 100000,
"max_queue_size": 100000,
"topic": "kv",
}
})
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn server_info_endpoint_is_dev_mode_only() {