[Rust Frontend]: Add /get_world_size route with static parallel size (#44801)

This commit is contained in:
Ashar
2026-06-18 13:10:20 +00:00
committed by GitHub
parent 22cc891108
commit 4cb5e746b6
11 changed files with 258 additions and 2 deletions
+18
View File
@@ -414,6 +414,24 @@ impl EngineCoreClient {
.expect("engine core client requires at least one engine")
}
/// Return the world size (TP * PP) from the parallel config, if available.
pub fn world_size(&self) -> u64 {
self.engines
.first()
.expect("engine core client requires at least one engine")
.ready_response
.world_size
}
/// Return the data parallel size from the parallel config, if available.
pub fn data_parallel_size(&self) -> u64 {
self.engines
.first()
.expect("engine core client requires at least one engine")
.ready_response
.data_parallel_size
}
/// Get the model name associated with this client used for metrics
/// labeling.
pub fn model_name(&self) -> &str {
@@ -52,6 +52,8 @@ pub fn default_ready_response() -> EngineCoreReadyResponse {
dp_stats_address: None,
dtype: ModelDtype::Float32,
vllm_version: "test-vllm-version".to_string(),
world_size: 1,
data_parallel_size: 1,
kv_cache_size_tokens: None,
kv_cache_max_concurrency: None,
}
@@ -44,6 +44,10 @@ pub struct EngineCoreReadyResponse {
pub dtype: ModelDtype,
/// Python vLLM version reported by the engine process.
pub vllm_version: String,
/// World size (TP * PP) from the parallel config.
pub world_size: u64,
/// Data parallelism size from the parallel config.
pub data_parallel_size: u64,
/// Total KV cache capacity in tokens, if reported.
pub kv_cache_size_tokens: Option<u64>,
/// Maximum achievable request concurrency given the KV cache, if reported.
+56 -1
View File
@@ -12,7 +12,7 @@ use crate::mock_engine::{
MockEngineConfig, MockEngineDataSockets, connect_to_bootstrapped_frontend, connect_to_frontend,
default_ready_response,
};
use crate::protocol::handshake::HandshakeInitMessage;
use crate::protocol::handshake::{EngineCoreReadyResponse, HandshakeInitMessage};
/// Per-test IPC endpoint namespace backed by a unique temporary directory.
///
@@ -62,6 +62,15 @@ fn test_mock_engine_config() -> MockEngineConfig {
}
}
fn test_mock_engine_config_with_ready(ready_response: EngineCoreReadyResponse) -> MockEngineConfig {
MockEngineConfig {
local: true,
headless: true,
ready_response,
..Default::default()
}
}
/// Complete the engine-core handshake and connect mock input/output sockets
/// plus optional coordinator sockets.
pub async fn setup_mock_engine_sockets(
@@ -147,3 +156,49 @@ where
});
(shutdown_tx, engine_task)
}
/// Like [`setup_mock_engine`] but uses a custom ready response for the
/// handshake, allowing tests to control `world_size`, `data_parallel_size`,
/// etc.
async fn setup_mock_engine_with_ready(
engine_handshake: String,
engine_id: impl Into<EngineId>,
ready_response: EngineCoreReadyResponse,
) -> (DealerSocket, PushSocket) {
let config = test_mock_engine_config_with_ready(ready_response);
let MockEngineSockets { data_sockets, .. } =
connect_to_frontend(engine_handshake, engine_id, config)
.await
.expect("connect mock engine with custom ready response");
let MockEngineDataSockets { dealer, push } =
data_sockets.into_iter().next().expect("mock engine data socket");
(dealer, push)
}
/// Like [`spawn_mock_engine_task`] but uses a custom ready response for the
/// handshake, allowing tests to set `world_size` and `data_parallel_size` to
/// non-default values.
pub fn spawn_mock_engine_task_with_ready<F>(
engine_handshake: String,
engine_id: impl Into<EngineId>,
ready_response: EngineCoreReadyResponse,
run: F,
) -> (oneshot::Sender<()>, tokio::task::JoinHandle<()>)
where
F: for<'a> FnOnce(
&'a mut DealerSocket,
&'a mut PushSocket,
) -> Pin<Box<dyn Future<Output = ()> + Send + 'a>>
+ Send
+ 'static,
{
let (shutdown_tx, shutdown_rx) = oneshot::channel();
let engine_id = engine_id.into();
let engine_task = tokio::spawn(async move {
let (mut dealer, mut push) =
setup_mock_engine_with_ready(engine_handshake, engine_id, ready_response).await;
run(&mut dealer, &mut push).await;
let _ = shutdown_rx.await;
});
(shutdown_tx, engine_task)
}
@@ -358,6 +358,8 @@ class EngineCoreReadyResponse:
dp_stats_address: str | None
dtype: str
vllm_version: str
world_size: int
data_parallel_size: int
kv_cache_size_tokens: int | None = None
kv_cache_max_concurrency: float | None = None
@@ -369,6 +371,8 @@ ready_response = EngineCoreReadyResponse(
dp_stats_address=None,
dtype="float32",
vllm_version="0.0.0",
data_parallel_size=1,
world_size=1,
)
print(msgspec.msgpack.encode(request).hex())
+2
View File
@@ -12,6 +12,7 @@ mod server_info;
mod sleep;
mod tokenize;
mod version;
mod world_size;
use std::sync::Arc;
@@ -100,6 +101,7 @@ fn build_router_with_options(
.route("/resume", post(pause::resume))
.route("/is_paused", get(pause::is_paused))
.route("/server_info", get(server_info::server_info))
.route("/get_world_size", get(world_size::get_world_size))
}
let enable_request_id_headers = state.api_server_options.enable_request_id_headers;
+112 -1
View File
@@ -23,6 +23,7 @@ use vllm_chat::{
ChatTextBackend, DefaultChatOutputProcessor, DynChatOutputProcessor, DynChatRenderer,
NewChatOutputProcessorOptions,
};
use vllm_engine_core_client::mock_engine::default_ready_response;
use vllm_engine_core_client::protocol::logprobs::{
Logprobs, MaybeWireLogprobs, PositionLogprobs, TokenLogprob,
};
@@ -31,7 +32,9 @@ use vllm_engine_core_client::protocol::{
EngineCoreFinishReason, EngineCoreOutput, EngineCoreOutputs, EngineCoreRequest, StopReason,
decode_value,
};
use vllm_engine_core_client::test_utils::{IpcNamespace, spawn_mock_engine_task};
use vllm_engine_core_client::test_utils::{
IpcNamespace, spawn_mock_engine_task, spawn_mock_engine_task_with_ready,
};
use vllm_engine_core_client::{
ENGINE_CORE_DEAD_SENTINEL, EngineCoreClient, EngineCoreClientConfig, EngineId,
};
@@ -788,6 +791,45 @@ async fn test_app_with_dev_mode(dev_mode_enabled: bool) -> axum::Router {
)
}
/// Build a dev-mode router backed by a mock engine using a custom ready
/// response, returning the router and the engine task handle so the engine
/// stays alive for the duration of the test.
async fn test_dev_mode_app_with_ready(
ready_response: vllm_engine_core_client::protocol::handshake::EngineCoreReadyResponse,
) -> (axum::Router, MockEngineTask) {
let ipc = IpcNamespace::new().expect("create ipc namespace");
let handshake_address = ipc.handshake_endpoint();
let engine_id = b"engine-world-size".to_vec();
let engine_task = MockEngineTask::new(spawn_mock_engine_task_with_ready(
handshake_address.clone(),
engine_id.clone(),
ready_response,
|_dealer, _push| boxed_test_future(async {}),
));
let client = EngineCoreClient::connect(
EngineCoreClientConfig::new_single(handshake_address)
.with_model_name("test-model")
.with_local_input_output_addresses(
Some(ipc.input_endpoint()),
Some(ipc.output_endpoint()),
),
)
.await
.expect("connect client");
let chat = ChatLlm::from_shared_backend(test_llm(client), Arc::new(FakeChatBackend::new()));
let app = build_router_with_dev_mode(
Arc::new(AppState::new(
vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()],
chat,
)),
true,
);
(app, engine_task)
}
async fn test_app_with_request_id_headers() -> (axum::Router, MockEngineTask) {
let (chat, engine_task) = test_models_with_engine_outputs_and_backend(
b"engine-openai-request-id",
@@ -5876,3 +5918,72 @@ async fn tokenize_chat_continue_final_vs_new_assistant_differs() {
let new_len = new_assistant["tokens"].as_array().unwrap().len();
assert!(new_len > continue_len);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn world_size_endpoint_is_dev_mode_only() {
let mut app = test_app().await;
let response = app
.call(
Request::builder()
.uri("/get_world_size")
.body(Body::empty())
.expect("build request"),
)
.await
.expect("call app");
assert_eq!(response.status(), StatusCode::NOT_FOUND);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn world_size_includes_data_parallelism_by_default() {
let ready = vllm_engine_core_client::protocol::handshake::EngineCoreReadyResponse {
world_size: 2,
data_parallel_size: 4,
..default_ready_response()
};
let (mut app, _engine_task) = test_dev_mode_app_with_ready(ready).await;
let response = app
.call(
Request::builder()
.uri("/get_world_size")
.body(Body::empty())
.expect("build request"),
)
.await
.expect("call app");
assert_eq!(response.status(), StatusCode::OK);
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json");
assert_eq!(json, json!({"world_size": 8}));
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn world_size_excludes_data_parallelism_when_include_dp_false() {
let ready = vllm_engine_core_client::protocol::handshake::EngineCoreReadyResponse {
world_size: 2,
data_parallel_size: 4,
..default_ready_response()
};
let (mut app, _engine_task) = test_dev_mode_app_with_ready(ready).await;
let response = app
.call(
Request::builder()
.uri("/get_world_size?include_dp=false")
.body(Body::empty())
.expect("build request"),
)
.await
.expect("call app");
assert_eq!(response.status(), StatusCode::OK);
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json");
assert_eq!(json, json!({"world_size": 2}));
}
+54
View File
@@ -0,0 +1,54 @@
use std::sync::Arc;
use axum::Json;
use axum::extract::{Query, State};
use serde::{Deserialize, Serialize};
use crate::error::ApiError;
use crate::state::AppState;
#[derive(Debug, Deserialize)]
pub(crate) struct WorldSizeParams {
/// If true (default), returns the world size including data parallelism
/// (TP * PP * DP). If false, returns the world size without data
/// parallelism (TP * PP).
#[serde(default = "default_true")]
include_dp: bool,
}
const fn default_true() -> bool {
true
}
#[derive(Serialize)]
pub(crate) struct WorldSizeResponse {
world_size: u64,
}
/// Get the world size from the parallel config.
///
/// Currently reads static values captured during the engine startup handshake.
///
/// TODO: If the world size can change at runtime (e.g. elastic EP scaling,
/// DP rank recovery), this should be switched to either:
/// - A `call_utility("get_world_size", (include_dp,))` RPC to the Python
/// engine for live values (simple, adds one ZMQ round-trip per request), or
/// - A push-based approach where the engine sends config updates via the
/// output stream into shared state (zero per-request overhead, more complex).
pub async fn get_world_size(
State(state): State<Arc<AppState>>,
Query(params): Query<WorldSizeParams>,
) -> Result<Json<WorldSizeResponse>, ApiError> {
let client = state.engine_core_client();
let ws = client.world_size();
let world_size = if params.include_dp {
let dp = client.data_parallel_size();
ws * dp
} else {
ws
};
Ok(Json(WorldSizeResponse { world_size }))
}
@@ -255,6 +255,8 @@ def test_apply_ready_response_syncs_block_size():
dp_stats_address=None,
dtype="bfloat16",
vllm_version="test",
world_size=1,
data_parallel_size=1,
)
)
client._apply_ready_response(payload)
+2
View File
@@ -78,6 +78,8 @@ class EngineCoreReadyResponse:
dp_stats_address: str | None
dtype: str
vllm_version: str
world_size: int
data_parallel_size: int
# KV cache capacity (None for encoder-only/attention-free models).
kv_cache_size_tokens: int | None = None
kv_cache_max_concurrency: float | None = None
+2
View File
@@ -1527,6 +1527,8 @@ class EngineCoreProc(EngineCore):
dp_stats_address=self.frontend_stats_publish_address,
dtype=str(self.vllm_config.model_config.dtype).removeprefix("torch."),
vllm_version=VLLM_VERSION,
world_size=self.vllm_config.parallel_config.world_size,
data_parallel_size=self.vllm_config.parallel_config.data_parallel_size,
kv_cache_size_tokens=(
self.vllm_config.cache_config.kv_cache_size_tokens
),