From 4cb5e746b63707ed470f952cfb77778a3dd34400 Mon Sep 17 00:00:00 2001 From: Ashar Date: Thu, 18 Jun 2026 18:40:20 +0530 Subject: [PATCH] [Rust Frontend]: Add `/get_world_size` route with static parallel size (#44801) --- rust/src/engine-core-client/src/client.rs | 18 +++ .../src/engine-core-client/src/mock_engine.rs | 2 + .../src/protocol/handshake.rs | 4 + rust/src/engine-core-client/src/test_utils.rs | 57 ++++++++- .../src/tests/python_compat.py | 4 + rust/src/server/src/routes.rs | 2 + rust/src/server/src/routes/tests.rs | 113 +++++++++++++++++- rust/src/server/src/routes/world_size.rs | 54 +++++++++ tests/v1/engine/test_engine_core_client.py | 2 + vllm/v1/engine/__init__.py | 2 + vllm/v1/engine/core.py | 2 + 11 files changed, 258 insertions(+), 2 deletions(-) create mode 100644 rust/src/server/src/routes/world_size.rs diff --git a/rust/src/engine-core-client/src/client.rs b/rust/src/engine-core-client/src/client.rs index b4357f77c7..f7df2fd7bb 100644 --- a/rust/src/engine-core-client/src/client.rs +++ b/rust/src/engine-core-client/src/client.rs @@ -414,6 +414,24 @@ impl EngineCoreClient { .expect("engine core client requires at least one engine") } + /// Return the world size (TP * PP) from the parallel config, if available. + pub fn world_size(&self) -> u64 { + self.engines + .first() + .expect("engine core client requires at least one engine") + .ready_response + .world_size + } + + /// Return the data parallel size from the parallel config, if available. + pub fn data_parallel_size(&self) -> u64 { + self.engines + .first() + .expect("engine core client requires at least one engine") + .ready_response + .data_parallel_size + } + /// Get the model name associated with this client used for metrics /// labeling. pub fn model_name(&self) -> &str { diff --git a/rust/src/engine-core-client/src/mock_engine.rs b/rust/src/engine-core-client/src/mock_engine.rs index 11c012b1f1..be6947bd45 100644 --- a/rust/src/engine-core-client/src/mock_engine.rs +++ b/rust/src/engine-core-client/src/mock_engine.rs @@ -52,6 +52,8 @@ pub fn default_ready_response() -> EngineCoreReadyResponse { dp_stats_address: None, dtype: ModelDtype::Float32, vllm_version: "test-vllm-version".to_string(), + world_size: 1, + data_parallel_size: 1, kv_cache_size_tokens: None, kv_cache_max_concurrency: None, } diff --git a/rust/src/engine-core-client/src/protocol/handshake.rs b/rust/src/engine-core-client/src/protocol/handshake.rs index 3ca8774b2d..1eea663044 100644 --- a/rust/src/engine-core-client/src/protocol/handshake.rs +++ b/rust/src/engine-core-client/src/protocol/handshake.rs @@ -44,6 +44,10 @@ pub struct EngineCoreReadyResponse { pub dtype: ModelDtype, /// Python vLLM version reported by the engine process. pub vllm_version: String, + /// World size (TP * PP) from the parallel config. + pub world_size: u64, + /// Data parallelism size from the parallel config. + pub data_parallel_size: u64, /// Total KV cache capacity in tokens, if reported. pub kv_cache_size_tokens: Option, /// Maximum achievable request concurrency given the KV cache, if reported. diff --git a/rust/src/engine-core-client/src/test_utils.rs b/rust/src/engine-core-client/src/test_utils.rs index 06f56380ab..0d777c9121 100644 --- a/rust/src/engine-core-client/src/test_utils.rs +++ b/rust/src/engine-core-client/src/test_utils.rs @@ -12,7 +12,7 @@ use crate::mock_engine::{ MockEngineConfig, MockEngineDataSockets, connect_to_bootstrapped_frontend, connect_to_frontend, default_ready_response, }; -use crate::protocol::handshake::HandshakeInitMessage; +use crate::protocol::handshake::{EngineCoreReadyResponse, HandshakeInitMessage}; /// Per-test IPC endpoint namespace backed by a unique temporary directory. /// @@ -62,6 +62,15 @@ fn test_mock_engine_config() -> MockEngineConfig { } } +fn test_mock_engine_config_with_ready(ready_response: EngineCoreReadyResponse) -> MockEngineConfig { + MockEngineConfig { + local: true, + headless: true, + ready_response, + ..Default::default() + } +} + /// Complete the engine-core handshake and connect mock input/output sockets /// plus optional coordinator sockets. pub async fn setup_mock_engine_sockets( @@ -147,3 +156,49 @@ where }); (shutdown_tx, engine_task) } + +/// Like [`setup_mock_engine`] but uses a custom ready response for the +/// handshake, allowing tests to control `world_size`, `data_parallel_size`, +/// etc. +async fn setup_mock_engine_with_ready( + engine_handshake: String, + engine_id: impl Into, + ready_response: EngineCoreReadyResponse, +) -> (DealerSocket, PushSocket) { + let config = test_mock_engine_config_with_ready(ready_response); + let MockEngineSockets { data_sockets, .. } = + connect_to_frontend(engine_handshake, engine_id, config) + .await + .expect("connect mock engine with custom ready response"); + let MockEngineDataSockets { dealer, push } = + data_sockets.into_iter().next().expect("mock engine data socket"); + (dealer, push) +} + +/// Like [`spawn_mock_engine_task`] but uses a custom ready response for the +/// handshake, allowing tests to set `world_size` and `data_parallel_size` to +/// non-default values. +pub fn spawn_mock_engine_task_with_ready( + engine_handshake: String, + engine_id: impl Into, + ready_response: EngineCoreReadyResponse, + run: F, +) -> (oneshot::Sender<()>, tokio::task::JoinHandle<()>) +where + F: for<'a> FnOnce( + &'a mut DealerSocket, + &'a mut PushSocket, + ) -> Pin + Send + 'a>> + + Send + + 'static, +{ + let (shutdown_tx, shutdown_rx) = oneshot::channel(); + let engine_id = engine_id.into(); + let engine_task = tokio::spawn(async move { + let (mut dealer, mut push) = + setup_mock_engine_with_ready(engine_handshake, engine_id, ready_response).await; + run(&mut dealer, &mut push).await; + let _ = shutdown_rx.await; + }); + (shutdown_tx, engine_task) +} diff --git a/rust/src/engine-core-client/src/tests/python_compat.py b/rust/src/engine-core-client/src/tests/python_compat.py index 8398c874da..ba4f7daa3d 100755 --- a/rust/src/engine-core-client/src/tests/python_compat.py +++ b/rust/src/engine-core-client/src/tests/python_compat.py @@ -358,6 +358,8 @@ class EngineCoreReadyResponse: dp_stats_address: str | None dtype: str vllm_version: str + world_size: int + data_parallel_size: int kv_cache_size_tokens: int | None = None kv_cache_max_concurrency: float | None = None @@ -369,6 +371,8 @@ ready_response = EngineCoreReadyResponse( dp_stats_address=None, dtype="float32", vllm_version="0.0.0", + data_parallel_size=1, + world_size=1, ) print(msgspec.msgpack.encode(request).hex()) diff --git a/rust/src/server/src/routes.rs b/rust/src/server/src/routes.rs index 3826ad40db..1e83c42781 100644 --- a/rust/src/server/src/routes.rs +++ b/rust/src/server/src/routes.rs @@ -12,6 +12,7 @@ mod server_info; mod sleep; mod tokenize; mod version; +mod world_size; use std::sync::Arc; @@ -100,6 +101,7 @@ fn build_router_with_options( .route("/resume", post(pause::resume)) .route("/is_paused", get(pause::is_paused)) .route("/server_info", get(server_info::server_info)) + .route("/get_world_size", get(world_size::get_world_size)) } let enable_request_id_headers = state.api_server_options.enable_request_id_headers; diff --git a/rust/src/server/src/routes/tests.rs b/rust/src/server/src/routes/tests.rs index 9d05b1c8b8..164b938f02 100644 --- a/rust/src/server/src/routes/tests.rs +++ b/rust/src/server/src/routes/tests.rs @@ -23,6 +23,7 @@ use vllm_chat::{ ChatTextBackend, DefaultChatOutputProcessor, DynChatOutputProcessor, DynChatRenderer, NewChatOutputProcessorOptions, }; +use vllm_engine_core_client::mock_engine::default_ready_response; use vllm_engine_core_client::protocol::logprobs::{ Logprobs, MaybeWireLogprobs, PositionLogprobs, TokenLogprob, }; @@ -31,7 +32,9 @@ use vllm_engine_core_client::protocol::{ EngineCoreFinishReason, EngineCoreOutput, EngineCoreOutputs, EngineCoreRequest, StopReason, decode_value, }; -use vllm_engine_core_client::test_utils::{IpcNamespace, spawn_mock_engine_task}; +use vllm_engine_core_client::test_utils::{ + IpcNamespace, spawn_mock_engine_task, spawn_mock_engine_task_with_ready, +}; use vllm_engine_core_client::{ ENGINE_CORE_DEAD_SENTINEL, EngineCoreClient, EngineCoreClientConfig, EngineId, }; @@ -788,6 +791,45 @@ async fn test_app_with_dev_mode(dev_mode_enabled: bool) -> axum::Router { ) } +/// Build a dev-mode router backed by a mock engine using a custom ready +/// response, returning the router and the engine task handle so the engine +/// stays alive for the duration of the test. +async fn test_dev_mode_app_with_ready( + ready_response: vllm_engine_core_client::protocol::handshake::EngineCoreReadyResponse, +) -> (axum::Router, MockEngineTask) { + let ipc = IpcNamespace::new().expect("create ipc namespace"); + let handshake_address = ipc.handshake_endpoint(); + let engine_id = b"engine-world-size".to_vec(); + + let engine_task = MockEngineTask::new(spawn_mock_engine_task_with_ready( + handshake_address.clone(), + engine_id.clone(), + ready_response, + |_dealer, _push| boxed_test_future(async {}), + )); + + let client = EngineCoreClient::connect( + EngineCoreClientConfig::new_single(handshake_address) + .with_model_name("test-model") + .with_local_input_output_addresses( + Some(ipc.input_endpoint()), + Some(ipc.output_endpoint()), + ), + ) + .await + .expect("connect client"); + + let chat = ChatLlm::from_shared_backend(test_llm(client), Arc::new(FakeChatBackend::new())); + let app = build_router_with_dev_mode( + Arc::new(AppState::new( + vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()], + chat, + )), + true, + ); + (app, engine_task) +} + async fn test_app_with_request_id_headers() -> (axum::Router, MockEngineTask) { let (chat, engine_task) = test_models_with_engine_outputs_and_backend( b"engine-openai-request-id", @@ -5876,3 +5918,72 @@ async fn tokenize_chat_continue_final_vs_new_assistant_differs() { let new_len = new_assistant["tokens"].as_array().unwrap().len(); assert!(new_len > continue_len); } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn world_size_endpoint_is_dev_mode_only() { + let mut app = test_app().await; + let response = app + .call( + Request::builder() + .uri("/get_world_size") + .body(Body::empty()) + .expect("build request"), + ) + .await + .expect("call app"); + + assert_eq!(response.status(), StatusCode::NOT_FOUND); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn world_size_includes_data_parallelism_by_default() { + let ready = vllm_engine_core_client::protocol::handshake::EngineCoreReadyResponse { + world_size: 2, + data_parallel_size: 4, + ..default_ready_response() + }; + let (mut app, _engine_task) = test_dev_mode_app_with_ready(ready).await; + + let response = app + .call( + Request::builder() + .uri("/get_world_size") + .body(Body::empty()) + .expect("build request"), + ) + .await + .expect("call app"); + + assert_eq!(response.status(), StatusCode::OK); + let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body"); + let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json"); + assert_eq!(json, json!({"world_size": 8})); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn world_size_excludes_data_parallelism_when_include_dp_false() { + let ready = vllm_engine_core_client::protocol::handshake::EngineCoreReadyResponse { + world_size: 2, + data_parallel_size: 4, + ..default_ready_response() + }; + let (mut app, _engine_task) = test_dev_mode_app_with_ready(ready).await; + + let response = app + .call( + Request::builder() + .uri("/get_world_size?include_dp=false") + .body(Body::empty()) + .expect("build request"), + ) + .await + .expect("call app"); + + assert_eq!(response.status(), StatusCode::OK); + let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body"); + let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json"); + assert_eq!(json, json!({"world_size": 2})); +} diff --git a/rust/src/server/src/routes/world_size.rs b/rust/src/server/src/routes/world_size.rs new file mode 100644 index 0000000000..da15757e8a --- /dev/null +++ b/rust/src/server/src/routes/world_size.rs @@ -0,0 +1,54 @@ +use std::sync::Arc; + +use axum::Json; +use axum::extract::{Query, State}; +use serde::{Deserialize, Serialize}; + +use crate::error::ApiError; +use crate::state::AppState; + +#[derive(Debug, Deserialize)] +pub(crate) struct WorldSizeParams { + /// If true (default), returns the world size including data parallelism + /// (TP * PP * DP). If false, returns the world size without data + /// parallelism (TP * PP). + #[serde(default = "default_true")] + include_dp: bool, +} + +const fn default_true() -> bool { + true +} + +#[derive(Serialize)] +pub(crate) struct WorldSizeResponse { + world_size: u64, +} + +/// Get the world size from the parallel config. +/// +/// Currently reads static values captured during the engine startup handshake. +/// +/// TODO: If the world size can change at runtime (e.g. elastic EP scaling, +/// DP rank recovery), this should be switched to either: +/// - A `call_utility("get_world_size", (include_dp,))` RPC to the Python +/// engine for live values (simple, adds one ZMQ round-trip per request), or +/// - A push-based approach where the engine sends config updates via the +/// output stream into shared state (zero per-request overhead, more complex). +pub async fn get_world_size( + State(state): State>, + Query(params): Query, +) -> Result, ApiError> { + let client = state.engine_core_client(); + + let ws = client.world_size(); + + let world_size = if params.include_dp { + let dp = client.data_parallel_size(); + ws * dp + } else { + ws + }; + + Ok(Json(WorldSizeResponse { world_size })) +} diff --git a/tests/v1/engine/test_engine_core_client.py b/tests/v1/engine/test_engine_core_client.py index 36dc95eea4..0b44b205cd 100644 --- a/tests/v1/engine/test_engine_core_client.py +++ b/tests/v1/engine/test_engine_core_client.py @@ -255,6 +255,8 @@ def test_apply_ready_response_syncs_block_size(): dp_stats_address=None, dtype="bfloat16", vllm_version="test", + world_size=1, + data_parallel_size=1, ) ) client._apply_ready_response(payload) diff --git a/vllm/v1/engine/__init__.py b/vllm/v1/engine/__init__.py index fbfe1c144c..a04f080ea6 100644 --- a/vllm/v1/engine/__init__.py +++ b/vllm/v1/engine/__init__.py @@ -78,6 +78,8 @@ class EngineCoreReadyResponse: dp_stats_address: str | None dtype: str vllm_version: str + world_size: int + data_parallel_size: int # KV cache capacity (None for encoder-only/attention-free models). kv_cache_size_tokens: int | None = None kv_cache_max_concurrency: float | None = None diff --git a/vllm/v1/engine/core.py b/vllm/v1/engine/core.py index f4e1b40e98..ac7037800a 100644 --- a/vllm/v1/engine/core.py +++ b/vllm/v1/engine/core.py @@ -1527,6 +1527,8 @@ class EngineCoreProc(EngineCore): dp_stats_address=self.frontend_stats_publish_address, dtype=str(self.vllm_config.model_config.dtype).removeprefix("torch."), vllm_version=VLLM_VERSION, + world_size=self.vllm_config.parallel_config.world_size, + data_parallel_size=self.vllm_config.parallel_config.data_parallel_size, kv_cache_size_tokens=( self.vllm_config.cache_config.kv_cache_size_tokens ),