diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index 25eb3aa063..2f13ff83e9 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -2417,6 +2417,7 @@ steps: - vllm/v1/engine/ - vllm/v1/worker/ - tests/utils.py + - tests/v1/distributed/test_dense_dp_world_size.py - tests/v1/distributed/test_external_lb_dp.py - tests/v1/distributed/test_hybrid_lb_dp.py - tests/v1/distributed/test_internal_lb_dp.py @@ -2424,6 +2425,7 @@ steps: commands: - export VLLM_USE_RUST_FRONTEND=1 - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_dense_dp_world_size.py - TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_internal_lb_dp.py -k "not 4 and not server_info" - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py -k "not 4 and not server_info" - TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_hybrid_lb_dp.py -k "not 4 and not server_info" @@ -2882,6 +2884,7 @@ steps: - TP_SIZE=2 DP_SIZE=2 pytest -v -s v1/distributed/test_async_llm_dp.py - TP_SIZE=2 DP_SIZE=2 pytest -v -s v1/distributed/test_eagle_dp.py - TP_SIZE=2 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py + - TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_dense_dp_world_size.py - TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_internal_lb_dp.py - TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_hybrid_lb_dp.py - pytest -v -s v1/engine/test_engine_core_client.py::test_kv_cache_events_dp diff --git a/.buildkite/test_areas/distributed.yaml b/.buildkite/test_areas/distributed.yaml index fb0329d583..32d1ff0342 100644 --- a/.buildkite/test_areas/distributed.yaml +++ b/.buildkite/test_areas/distributed.yaml @@ -154,6 +154,7 @@ steps: - TP_SIZE=2 DP_SIZE=2 pytest -v -s v1/distributed/test_async_llm_dp.py - TP_SIZE=2 DP_SIZE=2 pytest -v -s v1/distributed/test_eagle_dp.py - TP_SIZE=2 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py + - TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_dense_dp_world_size.py - TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_internal_lb_dp.py - TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_hybrid_lb_dp.py - pytest -v -s v1/engine/test_engine_core_client.py::test_kv_cache_events_dp diff --git a/.buildkite/test_areas/rust_frontend.yaml b/.buildkite/test_areas/rust_frontend.yaml index 6486df1c00..d562716701 100644 --- a/.buildkite/test_areas/rust_frontend.yaml +++ b/.buildkite/test_areas/rust_frontend.yaml @@ -108,6 +108,7 @@ steps: - vllm/v1/engine/ - vllm/v1/worker/ - tests/utils.py + - tests/v1/distributed/test_dense_dp_world_size.py - tests/v1/distributed/test_external_lb_dp.py - tests/v1/distributed/test_hybrid_lb_dp.py - tests/v1/distributed/test_internal_lb_dp.py @@ -115,6 +116,7 @@ steps: - export VLLM_USE_RUST_FRONTEND=1 - export VLLM_WORKER_MULTIPROC_METHOD=spawn - export NCCL_CUMEM_HOST_ENABLE=0 + - TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_dense_dp_world_size.py - TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_internal_lb_dp.py -k "not 4 and not server_info" - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py -k "not 4 and not server_info" - TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_hybrid_lb_dp.py -k "not 4 and not server_info" diff --git a/rust/src/cmd/src/cli.rs b/rust/src/cmd/src/cli.rs index e851fad549..64b6dc9800 100644 --- a/rust/src/cmd/src/cli.rs +++ b/rust/src/cmd/src/cli.rs @@ -474,9 +474,9 @@ impl SharedRuntimeArgs { output_address, engine_start_index, engine_count, + data_parallel_size, ready_timeout, }, - data_parallel_size, coordinator_mode: match coordinator_address { Some(address) => CoordinatorMode::External { address }, None => CoordinatorMode::None, @@ -533,7 +533,6 @@ impl SharedRuntimeArgs { local_input_address, local_output_address, }, - data_parallel_size: engine_count, coordinator_mode: CoordinatorMode::MaybeInProc, model: self.model, served_model_name: self.served_model_name, diff --git a/rust/src/cmd/src/cli/tests.rs b/rust/src/cmd/src/cli/tests.rs index 98a22ad324..80436ec423 100644 --- a/rust/src/cmd/src/cli/tests.rs +++ b/rust/src/cmd/src/cli/tests.rs @@ -864,7 +864,7 @@ fn frontend_args_json_applies_defaults() { assert_eq!(args.runtime.renderer, RendererSelection::Auto); assert_eq!(args.runtime.max_logprobs, None); assert_eq!(args.runtime.shutdown_timeout, 0); - assert_eq!(args.into_config().data_parallel_size, 4); + assert_eq!(args.into_config().transport_mode.data_parallel_size(), 4); } #[test] @@ -1521,7 +1521,6 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() { "", ), }, - data_parallel_size: 4, coordinator_mode: MaybeInProc, model: "Qwen/Qwen3-0.6B", served_model_name: [], @@ -1607,7 +1606,6 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() { local_input_address: None, local_output_address: None, }, - data_parallel_size: 4, coordinator_mode: MaybeInProc, model: "Qwen/Qwen3-0.6B", served_model_name: [], @@ -1710,9 +1708,9 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present output_address: "ipc:///tmp/output.sock", engine_start_index: 3, engine_count: 1, + data_parallel_size: 4, ready_timeout: 600s, }, - data_parallel_size: 4, coordinator_mode: External { address: "tcp://127.0.0.1:7000", }, diff --git a/rust/src/engine-core-client/src/client.rs b/rust/src/engine-core-client/src/client.rs index 1efca89621..e7b6e42b6a 100644 --- a/rust/src/engine-core-client/src/client.rs +++ b/rust/src/engine-core-client/src/client.rs @@ -15,7 +15,7 @@ use tracing::{debug, info, trace}; use crate::client::imp::{ClientInner, run_abort_loop, run_output_dispatcher_loop}; use crate::coordinator::CoordinatorHandle; -use crate::error::{Error, Result}; +use crate::error::{Error, Result, bail_invalid_client_config}; use crate::protocol::dtype::ModelDtype; use crate::protocol::handshake::EngineCoreReadyResponse; use crate::protocol::lora::LoraRequest; @@ -68,11 +68,70 @@ pub enum TransportMode { engine_start_index: u32, /// Total number of engines expected to register on this transport. engine_count: usize, + /// Deployment-wide data-parallel size. This may be larger than + /// `engine_count` when a supervisor partitions ranks across frontends. + data_parallel_size: usize, /// Maximum time to wait for all expected engines to register. ready_timeout: Duration, }, } +impl TransportMode { + /// Return the deployment-wide data-parallel size for this transport. + pub fn data_parallel_size(&self) -> usize { + match self { + Self::HandshakeOwner { engine_count, .. } => *engine_count, + Self::Bootstrapped { + data_parallel_size, .. + } => *data_parallel_size, + } + } + + /// Validate the transport topology before opening any sockets. + pub fn validate(&self) -> Result<()> { + let data_parallel_size = self.data_parallel_size(); + if data_parallel_size == 0 { + bail_invalid_client_config!("data parallel size must be at least 1"); + } + if data_parallel_size > usize::from(u16::MAX) + 1 { + bail_invalid_client_config!( + "data parallel size ({data_parallel_size}) exceeds the two-byte engine identity limit" + ); + } + + match self { + Self::HandshakeOwner { .. } => {} + Self::Bootstrapped { + engine_start_index, + engine_count, + .. + } => { + if *engine_count == 0 { + bail_invalid_client_config!("engine count must be at least 1"); + } + let engine_start_index = usize::try_from(*engine_start_index).map_err(|_| { + Error::InvalidClientConfig { + message: "engine start index does not fit usize".to_string(), + } + })?; + let engine_end_index = + engine_start_index.checked_add(*engine_count).ok_or_else(|| { + Error::InvalidClientConfig { + message: "engine start index + engine count overflows".to_string(), + } + })?; + if engine_end_index > data_parallel_size { + bail_invalid_client_config!( + "connected engine range [{engine_start_index}, {engine_end_index}) exceeds data parallel size ({data_parallel_size})" + ); + } + } + } + + Ok(()) + } +} + /// Which coordinator implementation should be active when one is present for a /// frontend client. #[derive(Debug, Clone, PartialEq, Eq)] @@ -117,6 +176,11 @@ impl EngineCoreClientConfig { } } + /// Validate the client topology before opening any transport sockets. + pub fn validate(&self) -> Result<()> { + self.transport_mode.validate() + } + /// Set the model name used by frontend-side metrics and diagnostics. pub fn with_model_name(mut self, model_name: impl Into) -> Self { self.model_name = model_name.into(); @@ -226,6 +290,7 @@ impl EngineCoreClient { /// handshake. In bootstrapped mode it binds the provided frontend /// sockets and waits for the expected engine registration frames. pub async fn connect(config: EngineCoreClientConfig) -> Result { + config.validate()?; let connected = match &config.transport_mode { TransportMode::HandshakeOwner { handshake_address, @@ -261,6 +326,7 @@ impl EngineCoreClient { engine_start_index, engine_count, ready_timeout, + .. } => { if let Some(CoordinatorMode::InProc) = config.coordinator_mode { panic!("cannot use in-process coordinator with bootstrapped transport mode") @@ -378,6 +444,12 @@ impl EngineCoreClient { self.engines.len() } + /// Return the deployment-wide data-parallel size configured for this + /// client. + pub fn data_parallel_size(&self) -> usize { + self.config.transport_mode.data_parallel_size() + } + /// Return the engine-side indices connected to this client. pub fn engine_indices(&self) -> Vec { self.engines diff --git a/rust/src/engine-core-client/src/error.rs b/rust/src/engine-core-client/src/error.rs index 98a578061e..d43952f023 100644 --- a/rust/src/engine-core-client/src/error.rs +++ b/rust/src/engine-core-client/src/error.rs @@ -49,6 +49,8 @@ pub enum Error { UnexpectedHandshakeIdentity { expected: Vec, actual: Vec }, #[error("unexpected startup handshake message: {message}")] UnexpectedHandshakeMessage { message: String }, + #[error("invalid engine-core client configuration: {message}")] + InvalidClientConfig { message: String }, #[error("unexpected non-control output on coordinator path: {message}")] UnexpectedCoordinatorOutput { message: String }, #[error("unexpected output on main dispatcher path: {message}")] diff --git a/rust/src/engine-core-client/src/mock_engine.rs b/rust/src/engine-core-client/src/mock_engine.rs index 8bd6d77360..77d72b525f 100644 --- a/rust/src/engine-core-client/src/mock_engine.rs +++ b/rust/src/engine-core-client/src/mock_engine.rs @@ -56,7 +56,7 @@ pub fn default_ready_response() -> EngineCoreReadyResponse { dtype: ModelDtype::Float32, vllm_version: "test-vllm-version".to_string(), world_size: 1, - data_parallel_size: 1, + effective_data_parallel_size: 1, tensor_parallel_size: 1, pipeline_parallel_size: 1, decode_context_parallel_size: 1, diff --git a/rust/src/engine-core-client/src/protocol/handshake.rs b/rust/src/engine-core-client/src/protocol/handshake.rs index 5cb4c98046..c7146abdbb 100644 --- a/rust/src/engine-core-client/src/protocol/handshake.rs +++ b/rust/src/engine-core-client/src/protocol/handshake.rs @@ -63,8 +63,11 @@ pub struct EngineCoreReadyResponse { pub vllm_version: String, /// World size (TP * PP) from the parallel config. pub world_size: u64, - /// Data parallelism size from the parallel config. - pub data_parallel_size: u64, + /// Data-parallel size from this EngineCore's effective parallel config. + /// Dense independent-DP ranks are reconfigured to report `1`; the client + /// transport owns the deployment-wide data-parallel size. + #[serde(rename = "data_parallel_size")] + pub effective_data_parallel_size: u64, // Required discovery metadata; EngineCore and client versions must match. /// Tensor-parallel size of this engine. pub tensor_parallel_size: u32, diff --git a/rust/src/engine-core-client/src/tests/client.rs b/rust/src/engine-core-client/src/tests/client.rs index 913b894ada..2f988e2bfe 100644 --- a/rust/src/engine-core-client/src/tests/client.rs +++ b/rust/src/engine-core-client/src/tests/client.rs @@ -303,6 +303,7 @@ fn bootstrapped_test_config( output_address, engine_start_index: 0, engine_count, + data_parallel_size: engine_count, ready_timeout, }, coordinator_mode, @@ -330,15 +331,47 @@ fn bootstrapped_test_config_with_start_index( ); let TransportMode::Bootstrapped { engine_start_index: start, + data_parallel_size, .. } = &mut config.transport_mode else { unreachable!("bootstrapped_test_config returns bootstrapped transport") }; *start = engine_start_index; + *data_parallel_size = usize::try_from(engine_start_index) + .expect("test start index fits usize") + .checked_add(engine_count) + .expect("test engine range fits usize"); config } +#[test] +fn client_config_validates_bootstrapped_dp_range() { + let mut config = bootstrapped_test_config_with_start_index( + "ipc://unused-input".to_string(), + "ipc://unused-output".to_string(), + 1, + 2, + Duration::from_secs(1), + 0, + None, + ); + config.validate().expect("frontend may own a subset of global DP ranks"); + + let TransportMode::Bootstrapped { + data_parallel_size, .. + } = &mut config.transport_mode + else { + unreachable!("expected bootstrapped transport") + }; + *data_parallel_size = 2; + let error = config.validate().expect_err("engine range above DP size must fail"); + expect_test::expect![[ + "invalid engine-core client configuration: connected engine range [1, 3) exceeds data parallel size (2)" + ]] + .assert_eq(&error.to_string()); +} + async fn recv_xpub_message(xpub: &mut XPubSocket) -> Vec { xpub.recv().await.unwrap().into_vec() } diff --git a/rust/src/server/examples/external_engine_openai_qwen.rs b/rust/src/server/examples/external_engine_openai_qwen.rs index 591252699a..3877762630 100644 --- a/rust/src/server/examples/external_engine_openai_qwen.rs +++ b/rust/src/server/examples/external_engine_openai_qwen.rs @@ -58,7 +58,6 @@ async fn main() -> Result<()> { local_input_address: None, local_output_address: None, }, - data_parallel_size: args.engine_count, coordinator_mode: CoordinatorMode::MaybeInProc, model: args.model, served_model_name: vec![], diff --git a/rust/src/server/src/config.rs b/rust/src/server/src/config.rs index b67e87f45f..a33d7e25fe 100644 --- a/rust/src/server/src/config.rs +++ b/rust/src/server/src/config.rs @@ -161,8 +161,6 @@ impl TlsConfig { pub struct Config { /// Frontend-to-engine transport setup. pub transport_mode: TransportMode, - /// Deployment-wide data-parallel size retained by the frontend. - pub data_parallel_size: usize, /// Requested frontend-side coordinator behavior. pub coordinator_mode: CoordinatorMode, /// Backend model identifier used for engine-core loading. @@ -239,46 +237,7 @@ impl Config { max_logprobs ); } - if self.data_parallel_size == 0 { - bail!("data parallel size must be at least 1"); - } - if self.data_parallel_size > usize::from(u16::MAX) + 1 { - bail!( - "data parallel size ({}) exceeds the two-byte engine identity limit", - self.data_parallel_size - ); - } - match &self.transport_mode { - TransportMode::HandshakeOwner { engine_count, .. } => { - if *engine_count != self.data_parallel_size { - bail!( - "managed frontend engine count ({engine_count}) must equal data parallel size ({})", - self.data_parallel_size - ); - } - } - TransportMode::Bootstrapped { - engine_start_index, - engine_count, - .. - } => { - if *engine_count == 0 { - bail!("engine count must be at least 1"); - } - let engine_start_index = usize::try_from(*engine_start_index) - .map_err(|_| anyhow::anyhow!("engine start index does not fit usize"))?; - let engine_end_index = - engine_start_index.checked_add(*engine_count).ok_or_else(|| { - anyhow::anyhow!("engine start index + engine count overflows") - })?; - if engine_end_index > self.data_parallel_size { - bail!( - "connected engine range [{engine_start_index}, {engine_end_index}) exceeds data parallel size {}", - self.data_parallel_size - ); - } - } - } + self.transport_mode.validate()?; Ok(()) } diff --git a/rust/src/server/src/grpc/control.rs b/rust/src/server/src/grpc/control.rs index c08803306f..9637dcbf8e 100644 --- a/rust/src/server/src/grpc/control.rs +++ b/rust/src/server/src/grpc/control.rs @@ -43,7 +43,7 @@ impl ControlServiceImpl { pb::ParallelismInfo { tensor_parallel_size: ready.tensor_parallel_size, pipeline_parallel_size: ready.pipeline_parallel_size, - data_parallel_size: self.state.data_parallel_size().min(u32::MAX as usize) as u32, + data_parallel_size: self.client().data_parallel_size().min(u32::MAX as usize) as u32, data_parallel_rank: ready.data_parallel_rank, decode_context_parallel_size: ready.decode_context_parallel_size, } diff --git a/rust/src/server/src/grpc/tests.rs b/rust/src/server/src/grpc/tests.rs index d5ee87a87f..9c1ea33b6b 100644 --- a/rust/src/server/src/grpc/tests.rs +++ b/rust/src/server/src/grpc/tests.rs @@ -1625,7 +1625,7 @@ async fn control_aggregates_multi_engine_capacity() { let mut ready_0 = default_ready_response(); ready_0.max_model_len = 8_192; ready_0.num_gpu_blocks = 10; - ready_0.data_parallel_size = 2; + ready_0.effective_data_parallel_size = 2; ready_0.weight_transfer_backend = Some("nccl".to_string()); ready_0.enable_sleep_mode = true; ready_0.supports_draft_weight_updates = true; @@ -1633,7 +1633,7 @@ async fn control_aggregates_multi_engine_capacity() { let mut ready_1 = default_ready_response(); ready_1.max_model_len = 4_096; ready_1.num_gpu_blocks = 20; - ready_1.data_parallel_size = 2; + ready_1.effective_data_parallel_size = 2; ready_1.data_parallel_rank = 1; let engine_tasks = [ready_0, ready_1].map(|ready| { @@ -1667,7 +1667,7 @@ async fn control_aggregates_multi_engine_capacity() { Llm::new(client), Arc::new(FakeTextBackend) as Arc, ); - let state = AppState::new(vec!["test-model".to_string()], chat).with_data_parallel_size(4); + let state = AppState::new(vec!["test-model".to_string()], chat); let service = ControlServiceImpl::new(Arc::new(state)); let server = pb::control_server::Control::get_server_info( @@ -1684,7 +1684,7 @@ async fn control_aggregates_multi_engine_capacity() { assert!(rl.weight_transfer_backend.is_empty()); assert!(!rl.sleep_mode_enabled); assert!(!rl.draft_weight_updates_enabled); - assert_eq!(server.parallelism.unwrap().data_parallel_size, 4); + assert_eq!(server.parallelism.unwrap().data_parallel_size, 2); drop(engine_tasks); } diff --git a/rust/src/server/src/lib.rs b/rust/src/server/src/lib.rs index 4423287e51..f9f5c42350 100644 --- a/rust/src/server/src/lib.rs +++ b/rust/src/server/src/lib.rs @@ -141,7 +141,6 @@ async fn build_state(config: &Config) -> Result> { .with_model_path(config.model.clone()) .with_api_server_options(config.api_server_options) .with_server_info(ServerInfoSnapshot::from_config(config)) - .with_data_parallel_size(config.data_parallel_size) .with_api_keys(config.api_keys.clone()) .with_cors(config.cors.clone()) .with_profiler(config.profiler.clone()), diff --git a/rust/src/server/src/routes/tests.rs b/rust/src/server/src/routes/tests.rs index 260851b4ef..9ac21ae07f 100644 --- a/rust/src/server/src/routes/tests.rs +++ b/rust/src/server/src/routes/tests.rs @@ -704,14 +704,13 @@ async fn test_app_with_dev_mode(dev_mode_enabled: bool) -> axum::Router { /// stays alive for the duration of the test. async fn test_dev_mode_app_with_ready( ready_response: vllm_engine_core_client::protocol::handshake::EngineCoreReadyResponse, - data_parallel_size: usize, ) -> (axum::Router, MockEngineTask) { let ipc = IpcNamespace::new().expect("create ipc namespace"); let handshake_address = ipc.handshake_endpoint(); let engine_id = b"engine-world-size".to_vec(); let engine_task = MockEngineTask::new(spawn_mock_engine_task_with_ready( handshake_address.clone(), - engine_id.clone(), + engine_id, ready_response, |_dealer, _push| boxed_test_future(async {}), )); @@ -729,10 +728,10 @@ async fn test_dev_mode_app_with_ready( let chat = ChatLlm::from_shared_backend(test_llm(client), Arc::new(FakeChatBackend::new())); let app = build_router_with_dev_mode( - Arc::new( - AppState::new(vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()], chat) - .with_data_parallel_size(data_parallel_size), - ), + Arc::new(AppState::new( + vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()], + chat, + )), true, ); (app, engine_task) @@ -6292,7 +6291,7 @@ async fn tokenize_allows_prompts_at_or_above_max_model_len() { max_model_len: 4, ..default_ready_response() }; - let (mut app, _engine_task) = test_dev_mode_app_with_ready(ready, 1).await; + let (mut app, _engine_task) = test_dev_mode_app_with_ready(ready).await; let (completion_status, completion_json) = post_json( &mut app, @@ -6543,58 +6542,6 @@ async fn world_size_endpoint_is_dev_mode_only() { assert_eq!(response.status(), StatusCode::NOT_FOUND); } -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -#[serial] -async fn world_size_includes_data_parallelism_by_default() { - let ready = vllm_engine_core_client::protocol::handshake::EngineCoreReadyResponse { - world_size: 2, - data_parallel_size: 1, - ..default_ready_response() - }; - let (mut app, _engine_task) = test_dev_mode_app_with_ready(ready, 4).await; - - let response = app - .call( - Request::builder() - .uri("/get_world_size") - .body(Body::empty()) - .expect("build request"), - ) - .await - .expect("call app"); - - assert_eq!(response.status(), StatusCode::OK); - let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body"); - let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json"); - assert_eq!(json, json!({"world_size": 8})); -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -#[serial] -async fn world_size_excludes_data_parallelism_when_include_dp_false() { - let ready = vllm_engine_core_client::protocol::handshake::EngineCoreReadyResponse { - world_size: 2, - data_parallel_size: 4, - ..default_ready_response() - }; - let (mut app, _engine_task) = test_dev_mode_app_with_ready(ready, 4).await; - - let response = app - .call( - Request::builder() - .uri("/get_world_size?include_dp=false") - .body(Body::empty()) - .expect("build request"), - ) - .await - .expect("call app"); - - assert_eq!(response.status(), StatusCode::OK); - let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body"); - let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json"); - assert_eq!(json, json!({"world_size": 2})); -} - // ========================= Profiler route tests ========================= async fn test_profiling_app_with_engine_script(script: F) -> (axum::Router, MockEngineTask) diff --git a/rust/src/server/src/routes/world_size.rs b/rust/src/server/src/routes/world_size.rs index 2be1bf76b7..8f5d3b204c 100644 --- a/rust/src/server/src/routes/world_size.rs +++ b/rust/src/server/src/routes/world_size.rs @@ -47,7 +47,7 @@ pub async fn get_world_size( let ws = client.world_size(); let world_size = if params.include_dp { - let dp = state.data_parallel_size() as u64; + let dp = client.data_parallel_size() as u64; ws * dp } else { ws diff --git a/rust/src/server/src/state.rs b/rust/src/server/src/state.rs index f745cf7c00..722dede8dd 100644 --- a/rust/src/server/src/state.rs +++ b/rust/src/server/src/state.rs @@ -40,8 +40,6 @@ pub struct AppState { pub cors: CorsConfig, /// Runtime server information returned by `/server_info`, when available. server_info: Option, - /// Deployment-wide data-parallel size retained by the frontend. - data_parallel_size: usize, /// SHA-256 hashes of API keys accepted as bearer tokens for guarded routes. api_key_hashes: Vec, /// Number of in-flight inference requests currently owned by this frontend. @@ -71,14 +69,12 @@ impl AppState { !served_model_names.is_empty(), "served_model_names must not be empty" ); - let data_parallel_size = chat.engine_core_client().engine_count(); Self { served_model_names, chat, api_server_options: ApiServerOptions::default(), cors: CorsConfig::default(), server_info: None, - data_parallel_size, api_key_hashes: Vec::new(), server_load: AtomicU64::new(0), lora_manager: LoraManager::new(), @@ -118,17 +114,6 @@ impl AppState { self } - /// Set the deployment-wide data-parallel size reported by frontend APIs. - pub(crate) fn with_data_parallel_size(mut self, size: usize) -> Self { - self.data_parallel_size = size; - self - } - - /// Return the deployment-wide data-parallel size. - pub(crate) fn data_parallel_size(&self) -> usize { - self.data_parallel_size - } - /// Build a `/server_info` response payload. pub(crate) fn server_info_response( &self, diff --git a/tests/v1/distributed/test_dense_dp_world_size.py b/tests/v1/distributed/test_dense_dp_world_size.py new file mode 100644 index 0000000000..1fc3793646 --- /dev/null +++ b/tests/v1/distributed/test_dense_dp_world_size.py @@ -0,0 +1,52 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import os + +import requests + +from tests.utils import RemoteOpenAIServer +from vllm.platforms import current_platform + +MODEL_NAME = "Qwen/Qwen3-0.6B" +DP_SIZE = int(os.getenv("DP_SIZE", "2")) +TP_SIZE = int(os.getenv("TP_SIZE", "1")) + + +def test_dense_dp_world_size(): + server_args = [ + "--dtype", + "bfloat16", + "--max-model-len", + "2048", + "--max-num-seqs", + "128", + "--enforce-eager", + "--data-parallel-size", + str(DP_SIZE), + "--data-parallel-size-local", + str(DP_SIZE), + "--tensor-parallel-size", + str(TP_SIZE), + ] + env_dict = { + "VLLM_SERVER_DEV_MODE": "1", + current_platform.device_control_env_var: ",".join( + str(current_platform.device_id_to_physical_device_id(i)) + for i in range(DP_SIZE * TP_SIZE) + ), + } + + with RemoteOpenAIServer( + MODEL_NAME, + server_args, + env_dict=env_dict, + ) as server: + response = requests.get(server.url_for("get_world_size")) + response.raise_for_status() + assert response.json() == {"world_size": TP_SIZE * DP_SIZE} + + response = requests.get( + server.url_for("get_world_size"), params={"include_dp": "false"} + ) + response.raise_for_status() + assert response.json() == {"world_size": TP_SIZE}