[Core][Frontend] Bind KV-event publishers at port 0 and expose the bound endpoints (#55844)
Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: mukeshbhandarkar <shell00035@gmail.com> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com> Co-authored-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
co-authored by
mukeshbhandarkar
aoshen02
Claude Fable 5.1
aoshen02
parent
729ebac498
commit
90531daf6d
@@ -6,6 +6,7 @@ mod cache;
|
||||
mod collective_rpc;
|
||||
mod health;
|
||||
mod inference;
|
||||
mod kv_event_sources;
|
||||
mod load;
|
||||
mod lora;
|
||||
mod metrics;
|
||||
@@ -91,6 +92,7 @@ fn build_router_with_options(
|
||||
.route("/metrics", get(metrics::scrape))
|
||||
.route("/load", get(load::load))
|
||||
.route("/version", get(version::version))
|
||||
.route("/kv_event_sources", get(kv_event_sources::kv_event_sources))
|
||||
// OpenAI-compatible endpoints
|
||||
.route("/v1/models", get(openai::list_models))
|
||||
.route("/v1/completions", post(openai::completions))
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use axum::Json;
|
||||
use axum::extract::State;
|
||||
use vllm_engine_core_client::protocol::handshake::KvEventsConfig;
|
||||
|
||||
use crate::state::AppState;
|
||||
|
||||
/// KV-event publisher config of each data-parallel rank, keyed by rank.
|
||||
pub async fn kv_event_sources(
|
||||
State(state): State<Arc<AppState>>,
|
||||
) -> Json<BTreeMap<u32, KvEventsConfig>> {
|
||||
Json(
|
||||
state
|
||||
.engine_core_client()
|
||||
.ready_responses()
|
||||
.into_iter()
|
||||
.filter_map(|ready| Some((ready.data_parallel_rank, ready.kv_events_config.clone()?)))
|
||||
.collect(),
|
||||
)
|
||||
}
|
||||
@@ -2054,6 +2054,56 @@ async fn version_returns_engine_vllm_version() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn kv_event_sources_reports_each_ready_engine() {
|
||||
let ready = vllm_engine_core_client::protocol::handshake::EngineCoreReadyResponse {
|
||||
data_parallel_rank: 1,
|
||||
kv_events_config: Some(
|
||||
vllm_engine_core_client::protocol::handshake::KvEventsConfig {
|
||||
enable_kv_cache_events: true,
|
||||
publisher: "zmq".to_string(),
|
||||
endpoint: "tcp://10.0.0.2:41233".to_string(),
|
||||
replay_endpoint: None,
|
||||
buffer_steps: 10_000,
|
||||
hwm: 100_000,
|
||||
max_queue_size: 100_000,
|
||||
topic: "kv".to_string(),
|
||||
},
|
||||
),
|
||||
..default_ready_response()
|
||||
};
|
||||
let (mut app, _engine_task) = test_dev_mode_app_with_ready(ready).await;
|
||||
let response = app
|
||||
.call(
|
||||
Request::builder()
|
||||
.uri("/kv_event_sources")
|
||||
.body(Body::empty())
|
||||
.expect("build request"),
|
||||
)
|
||||
.await
|
||||
.expect("call app");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
|
||||
let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json");
|
||||
assert_eq!(
|
||||
json,
|
||||
json!({
|
||||
"1": {
|
||||
"enable_kv_cache_events": true,
|
||||
"publisher": "zmq",
|
||||
"endpoint": "tcp://10.0.0.2:41233",
|
||||
"replay_endpoint": null,
|
||||
"buffer_steps": 10000,
|
||||
"hwm": 100000,
|
||||
"max_queue_size": 100000,
|
||||
"topic": "kv",
|
||||
}
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn server_info_endpoint_is_dev_mode_only() {
|
||||
|
||||
Reference in New Issue
Block a user