From fa722b9f0188179a66728f2f882e9d5bc92c9b0c Mon Sep 17 00:00:00 2001 From: Connor Carpenter Date: Mon, 10 Aug 2026 15:05:24 -0700 Subject: [PATCH] [Rust Frontend][gRPC] Add explicit data-parallel rank routing (#51178) Signed-off-by: Connor Carpenter Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- rust/src/cmd/src/cli.rs | 8 ++++ rust/src/cmd/src/cli/tests.rs | 9 ++++ rust/src/engine-core-client/src/client.rs | 9 ---- .../engine-core-client/src/client/state.rs | 40 ++++++++++-------- rust/src/engine-core-client/src/error.rs | 9 +++- rust/src/engine-core-client/src/transport.rs | 26 ++++++++++-- rust/src/mock-engine/src/lib.rs | 15 ++++--- .../examples/external_engine_openai_qwen.rs | 1 + rust/src/server/src/config.rs | 42 +++++++++++++++++++ rust/src/server/src/grpc/control.rs | 2 +- rust/src/server/src/grpc/inference.rs | 27 +++++++++++- rust/src/server/src/grpc/tests.rs | 42 ++++++++++++++++--- rust/src/server/src/lib.rs | 1 + rust/src/server/src/routes/tests.rs | 18 ++++---- rust/src/server/src/routes/world_size.rs | 2 +- rust/src/server/src/state.rs | 15 +++++++ rust/src/text/src/error.rs | 5 ++- vllm/entrypoints/cli/serve.py | 1 + vllm/v1/utils.py | 3 ++ 19 files changed, 215 insertions(+), 60 deletions(-) diff --git a/rust/src/cmd/src/cli.rs b/rust/src/cmd/src/cli.rs index b62e75ebac..833b93ee43 100644 --- a/rust/src/cmd/src/cli.rs +++ b/rust/src/cmd/src/cli.rs @@ -457,6 +457,7 @@ impl SharedRuntimeArgs { coordinator_address: Option, engine_start_index: u32, engine_count: usize, + data_parallel_size: usize, ) -> Config { let ready_timeout = self.ready_timeout(); let shutdown_timeout = self.shutdown_timeout(); @@ -474,6 +475,7 @@ impl SharedRuntimeArgs { engine_count, ready_timeout, }, + data_parallel_size, coordinator_mode: match coordinator_address { Some(address) => CoordinatorMode::External { address }, None => CoordinatorMode::None, @@ -530,6 +532,7 @@ impl SharedRuntimeArgs { local_input_address, local_output_address, }, + data_parallel_size: engine_count, coordinator_mode: CoordinatorMode::MaybeInProc, model: self.model, served_model_name: self.served_model_name, @@ -661,6 +664,9 @@ pub struct FrontendArgs { /// Total number of data-parallel engines expected for this frontend. #[arg(long, default_value_t = 1)] pub engine_count: usize, + /// Deployment-wide configured data-parallel size. Defaults to engine count. + #[arg(long)] + pub data_parallel_size: Option, /// Shared frontend arguments as one JSON object. #[arg(long = "args-json", value_parser = parse_runtime_args_json, value_name = "JSON")] @@ -670,6 +676,7 @@ pub struct FrontendArgs { impl FrontendArgs { /// Convert the CLI arguments into the OpenAI server's runtime config. pub fn into_config(self) -> Config { + let data_parallel_size = self.data_parallel_size.unwrap_or(self.engine_count); self.runtime.into_bootstrapped_config( self.listen_fd, self.input_address, @@ -677,6 +684,7 @@ impl FrontendArgs { self.coordinator_address, self.engine_start_index, self.engine_count, + data_parallel_size, ) } } diff --git a/rust/src/cmd/src/cli/tests.rs b/rust/src/cmd/src/cli/tests.rs index 28cea8a371..98a22ad324 100644 --- a/rust/src/cmd/src/cli/tests.rs +++ b/rust/src/cmd/src/cli/tests.rs @@ -785,6 +785,7 @@ fn frontend_args_accept_json() { ), engine_start_index: 0, engine_count: 1, + data_parallel_size: None, runtime: SharedRuntimeArgs { model: "Qwen/Qwen3-0.6B", engine_ready_timeout_secs: 600, @@ -846,6 +847,8 @@ fn frontend_args_json_applies_defaults() { "ipc:///tmp/input.sock", "--output-address", "ipc:///tmp/output.sock", + "--engine-count", + "4", "--args-json", r#"{"model_tag":"Qwen/Qwen3-0.6B"}"#, ]) @@ -861,6 +864,7 @@ fn frontend_args_json_applies_defaults() { assert_eq!(args.runtime.renderer, RendererSelection::Auto); assert_eq!(args.runtime.max_logprobs, None); assert_eq!(args.runtime.shutdown_timeout, 0); + assert_eq!(args.into_config().data_parallel_size, 4); } #[test] @@ -1517,6 +1521,7 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() { "", ), }, + data_parallel_size: 4, coordinator_mode: MaybeInProc, model: "Qwen/Qwen3-0.6B", served_model_name: [], @@ -1602,6 +1607,7 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() { local_input_address: None, local_output_address: None, }, + data_parallel_size: 4, coordinator_mode: MaybeInProc, model: "Qwen/Qwen3-0.6B", served_model_name: [], @@ -1685,6 +1691,8 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present "3", "--engine-count", "1", + "--data-parallel-size", + "4", "--args-json", r#"{"model_tag":"Qwen/Qwen3-0.6B"}"#, ]) @@ -1704,6 +1712,7 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present engine_count: 1, ready_timeout: 600s, }, + data_parallel_size: 4, coordinator_mode: External { address: "tcp://127.0.0.1:7000", }, diff --git a/rust/src/engine-core-client/src/client.rs b/rust/src/engine-core-client/src/client.rs index a5709e52d6..d1c20ad582 100644 --- a/rust/src/engine-core-client/src/client.rs +++ b/rust/src/engine-core-client/src/client.rs @@ -451,15 +451,6 @@ impl EngineCoreClient { .world_size } - /// Return the data parallel size from the parallel config, if available. - pub fn data_parallel_size(&self) -> u64 { - self.engines - .first() - .expect("engine core client requires at least one engine") - .ready_response - .data_parallel_size - } - /// Get the model name associated with this client used for metrics /// labeling. pub fn model_name(&self) -> &str { diff --git a/rust/src/engine-core-client/src/client/state.rs b/rust/src/engine-core-client/src/client/state.rs index 28e701ba73..59f77bf71c 100644 --- a/rust/src/engine-core-client/src/client/state.rs +++ b/rust/src/engine-core-client/src/client/state.rs @@ -161,15 +161,16 @@ impl RequestRegistry { fn choose_engine_for_request(&mut self, data_parallel_rank: Option) -> Result { if let Some(rank) = data_parallel_rank { - // Route to the engine at the specified rank index. - let engine_id = EngineId::from_engine_index(rank); - return self - .routing_per_engine - .contains_key(&engine_id) - .then_some(engine_id) + let engine_id = u16::try_from(rank).ok().map(EngineId::from_engine_index); + return engine_id + .filter(|engine_id| self.routing_per_engine.contains_key(engine_id)) .ok_or_else(|| Error::InvalidDataParallelRank { rank, - num_engines: self.routing_per_engine.len() as u32, + connected_ranks: self + .routing_per_engine + .keys() + .filter_map(EngineId::engine_index) + .collect(), }); } @@ -343,6 +344,9 @@ impl RequestRegistry { } fn apply_scheduler_counts(&mut self, engine_index: u32, next: EngineLoadSnapshot) -> bool { + let Ok(engine_index) = u16::try_from(engine_index) else { + return false; + }; let engine_id = EngineId::from_engine_index(engine_index); let Some(state) = self.routing_per_engine.get_mut(&engine_id) else { return false; @@ -840,26 +844,26 @@ mod tests { error, crate::error::Error::InvalidDataParallelRank { rank: 2, - num_engines: 2, - } + connected_ranks, + } if connected_ranks == vec![0, 1] )); } #[test] - fn register_with_rank_on_single_engine_only_accepts_zero() { - let engine_0 = EngineId::from_engine_index(0); - let mut registry = RequestRegistry::new(&[connected_engine(engine_0.clone())]); + fn register_with_rank_uses_global_engine_identity() { + let engine_3 = EngineId::from_engine_index(3); + let mut registry = RequestRegistry::new(&[connected_engine(engine_3.clone())]); - let (chosen, _) = registry.register("req-ok".to_string(), None, Some(0)).unwrap(); - assert_eq!(chosen, engine_0); + let (chosen, _) = registry.register("req-ok".to_string(), None, Some(3)).unwrap(); + assert_eq!(chosen, engine_3); - let error = registry.register("req-bad".to_string(), None, Some(1)).unwrap_err(); + let error = registry.register("req-bad".to_string(), None, Some(0)).unwrap_err(); assert!(matches!( error, crate::error::Error::InvalidDataParallelRank { - rank: 1, - num_engines: 1, - } + rank: 0, + connected_ranks, + } if connected_ranks == vec![3] )); } diff --git a/rust/src/engine-core-client/src/error.rs b/rust/src/engine-core-client/src/error.rs index 94809a6786..98a578061e 100644 --- a/rust/src/engine-core-client/src/error.rs +++ b/rust/src/engine-core-client/src/error.rs @@ -68,8 +68,13 @@ pub enum Error { ControlClosed { message: String }, #[error("request `{request_id}` is already in flight")] DuplicateRequestId { request_id: String }, - #[error("data parallel rank {rank} is out of range for {num_engines} engine(s)")] - InvalidDataParallelRank { rank: u32, num_engines: u32 }, + #[error( + "data parallel rank {rank} is not connected to this frontend; connected ranks: {connected_ranks:?}" + )] + InvalidDataParallelRank { + rank: u32, + connected_ranks: Vec, + }, #[error("engine-core output dispatcher closed: {message}")] DispatcherClosed { message: String }, #[error("engine-core client is closed: {message}")] diff --git a/rust/src/engine-core-client/src/transport.rs b/rust/src/engine-core-client/src/transport.rs index d3e3ecae2b..7aac8774ca 100644 --- a/rust/src/engine-core-client/src/transport.rs +++ b/rust/src/engine-core-client/src/transport.rs @@ -66,8 +66,8 @@ impl EngineId { /// Construct an engine id from the Python-compatible engine index encoding /// (two-byte little-endian). - pub fn from_engine_index(value: u32) -> Self { - Self(Bytes::copy_from_slice(&(value as u16).to_le_bytes())) + pub fn from_engine_index(value: u16) -> Self { + Self(Bytes::copy_from_slice(&value.to_le_bytes())) } } @@ -336,6 +336,22 @@ pub async fn connect_bootstrapped( if engine_count == 0 { bail_unexpected_handshake_message!("expected engine_count >= 1"); } + let engine_start_index = + u16::try_from(engine_start_index).map_err(|_| Error::UnexpectedHandshakeMessage { + message: "engine_start_index exceeds the two-byte engine identity limit".to_string(), + })?; + let engine_end_index = + usize::from(engine_start_index).checked_add(engine_count).ok_or_else(|| { + Error::UnexpectedHandshakeMessage { + message: "engine_start_index + engine_count overflows".to_string(), + } + })?; + if engine_end_index > usize::from(u16::MAX) + 1 { + return Err(Error::UnexpectedHandshakeMessage { + message: "engine_start_index + engine_count exceeds the two-byte engine identity limit" + .to_string(), + }); + } let mut input_socket = RouterSocket::new(); let input_address = input_socket.bind(input_address).await?.to_string(); @@ -345,8 +361,10 @@ pub async fn connect_bootstrapped( let engines = wait_for_input_registrations( &mut input_socket, - (0..engine_count) - .map(|offset| EngineId::from_engine_index(engine_start_index + offset as u32)), + (0..engine_count).map(|offset| { + let offset = u16::try_from(offset).expect("validated engine offset fits u16"); + EngineId::from_engine_index(engine_start_index + offset) + }), ready_timeout, ) .await?; diff --git a/rust/src/mock-engine/src/lib.rs b/rust/src/mock-engine/src/lib.rs index d4e8581b2f..63b1d432ba 100644 --- a/rust/src/mock-engine/src/lib.rs +++ b/rust/src/mock-engine/src/lib.rs @@ -48,10 +48,13 @@ pub struct Opt { } /// Run one mock engine until shutdown or transport failure. -async fn run_engine(engine_index: u32, opt: Opt, shutdown: CancellationToken) -> Result<()> { +async fn run_engine(engine_index: usize, opt: Opt, shutdown: CancellationToken) -> Result<()> { + let engine_index = + u16::try_from(engine_index).context("mock engine index exceeds two-byte identity limit")?; + let engine_id = EngineId::from_engine_index(engine_index); let MockEngineSockets { data_sockets, .. } = connect_to_frontend( &opt.handshake_address, - EngineId::from_engine_index(engine_index), + engine_id, MockEngineConfig::default(), ) .await @@ -71,7 +74,7 @@ async fn run_engine(engine_index: u32, opt: Opt, shutdown: CancellationToken) -> )); // Engine loop: input_rx -> engine logic -> output_tx let mut engine_loop = tokio::spawn(engine::run_engine_loop( - engine_index, + u32::from(engine_index), opt, input_rx, output_tx, @@ -111,11 +114,7 @@ pub async fn run(opt: Opt, shutdown: CancellationToken) -> Result<()> { let mut engines = JoinSet::new(); for engine_index in 0..opt.engine_count { - engines.spawn(run_engine( - engine_index as u32, - opt.clone(), - shutdown.clone(), - )); + engines.spawn(run_engine(engine_index, opt.clone(), shutdown.clone())); } tokio::select! { diff --git a/rust/src/server/examples/external_engine_openai_qwen.rs b/rust/src/server/examples/external_engine_openai_qwen.rs index 3877762630..591252699a 100644 --- a/rust/src/server/examples/external_engine_openai_qwen.rs +++ b/rust/src/server/examples/external_engine_openai_qwen.rs @@ -58,6 +58,7 @@ async fn main() -> Result<()> { local_input_address: None, local_output_address: None, }, + data_parallel_size: args.engine_count, coordinator_mode: CoordinatorMode::MaybeInProc, model: args.model, served_model_name: vec![], diff --git a/rust/src/server/src/config.rs b/rust/src/server/src/config.rs index 76848b3caa..b67e87f45f 100644 --- a/rust/src/server/src/config.rs +++ b/rust/src/server/src/config.rs @@ -161,6 +161,8 @@ impl TlsConfig { pub struct Config { /// Frontend-to-engine transport setup. pub transport_mode: TransportMode, + /// Deployment-wide data-parallel size retained by the frontend. + pub data_parallel_size: usize, /// Requested frontend-side coordinator behavior. pub coordinator_mode: CoordinatorMode, /// Backend model identifier used for engine-core loading. @@ -237,6 +239,46 @@ impl Config { max_logprobs ); } + if self.data_parallel_size == 0 { + bail!("data parallel size must be at least 1"); + } + if self.data_parallel_size > usize::from(u16::MAX) + 1 { + bail!( + "data parallel size ({}) exceeds the two-byte engine identity limit", + self.data_parallel_size + ); + } + match &self.transport_mode { + TransportMode::HandshakeOwner { engine_count, .. } => { + if *engine_count != self.data_parallel_size { + bail!( + "managed frontend engine count ({engine_count}) must equal data parallel size ({})", + self.data_parallel_size + ); + } + } + TransportMode::Bootstrapped { + engine_start_index, + engine_count, + .. + } => { + if *engine_count == 0 { + bail!("engine count must be at least 1"); + } + let engine_start_index = usize::try_from(*engine_start_index) + .map_err(|_| anyhow::anyhow!("engine start index does not fit usize"))?; + let engine_end_index = + engine_start_index.checked_add(*engine_count).ok_or_else(|| { + anyhow::anyhow!("engine start index + engine count overflows") + })?; + if engine_end_index > self.data_parallel_size { + bail!( + "connected engine range [{engine_start_index}, {engine_end_index}) exceeds data parallel size {}", + self.data_parallel_size + ); + } + } + } Ok(()) } diff --git a/rust/src/server/src/grpc/control.rs b/rust/src/server/src/grpc/control.rs index 8972098222..201d8ace95 100644 --- a/rust/src/server/src/grpc/control.rs +++ b/rust/src/server/src/grpc/control.rs @@ -31,7 +31,7 @@ impl ControlServiceImpl { pb::ParallelismInfo { tensor_parallel_size: ready.tensor_parallel_size, pipeline_parallel_size: ready.pipeline_parallel_size, - data_parallel_size: ready.data_parallel_size.min(u64::from(u32::MAX)) as u32, + data_parallel_size: self.state.data_parallel_size().min(u32::MAX as usize) as u32, data_parallel_rank: ready.data_parallel_rank, decode_context_parallel_size: ready.decode_context_parallel_size, } diff --git a/rust/src/server/src/grpc/inference.rs b/rust/src/server/src/grpc/inference.rs index 3fc56fd0d9..8eebdb7210 100644 --- a/rust/src/server/src/grpc/inference.rs +++ b/rust/src/server/src/grpc/inference.rs @@ -22,6 +22,8 @@ use crate::state::AppState; pub(crate) type InferenceGrpcService = InferenceServer; +const DATA_PARALLEL_RANK_METADATA_KEY: &str = "x-data-parallel-rank"; + /// gRPC inference service backed by the shared application state. pub struct InferenceServiceImpl { state: Arc, @@ -41,6 +43,7 @@ impl InferenceServiceImpl { async fn prepare_request( &self, mut proto_request: pb::GenerateRequest, + data_parallel_rank: Option, stream: bool, rpc: &'static str, ) -> Result { @@ -62,6 +65,7 @@ impl InferenceServiceImpl { rpc, %model, media_count, + ?data_parallel_rank, ); info!(parent: &request_span, "gRPC inference request received"); @@ -70,6 +74,7 @@ impl InferenceServiceImpl { let mut text_request = convert::to_text_request(proto_request, stream, self.state.served_model_names())?; text_request.arrival_time = Some(arrival_time); + text_request.data_parallel_rank = data_parallel_rank; let media = convert::media_parts_from_request(media)?; if !media.is_empty() { @@ -115,6 +120,20 @@ impl InferenceServiceImpl { } } +fn data_parallel_rank_from_metadata( + request: &Request, +) -> Result, Status> { + let Some(value) = request.metadata().get(DATA_PARALLEL_RANK_METADATA_KEY) else { + return Ok(None); + }; + let value = value.to_str().map_err(|_| { + Status::invalid_argument("x-data-parallel-rank metadata must be an unsigned 32-bit integer") + })?; + value.trim().parse::().map(Some).map_err(|_| { + Status::invalid_argument("x-data-parallel-rank metadata must be an unsigned 32-bit integer") + }) +} + #[tonic::async_trait] impl pb::inference_server::Inference for InferenceServiceImpl { type GenerateStreamStream = @@ -125,13 +144,14 @@ impl pb::inference_server::Inference for InferenceServiceImpl { &self, request: Request, ) -> Result, Status> { + let data_parallel_rank = data_parallel_rank_from_metadata(&request)?; let proto_req = request.into_inner(); let response_opts = ResponseOpts::from_proto(proto_req.response.as_ref()); let PreparedGrpcRequest { text_request, request_span, started_at, - } = self.prepare_request(proto_req, false, "Generate").await?; + } = self.prepare_request(proto_req, data_parallel_rank, false, "Generate").await?; let stream = self .state @@ -186,13 +206,16 @@ impl pb::inference_server::Inference for InferenceServiceImpl { &self, request: Request, ) -> Result, Status> { + let data_parallel_rank = data_parallel_rank_from_metadata(&request)?; let proto_req = request.into_inner(); let response_opts = ResponseOpts::from_proto(proto_req.response.as_ref()); let PreparedGrpcRequest { text_request, request_span, started_at, - } = self.prepare_request(proto_req, true, "GenerateStream").await?; + } = self + .prepare_request(proto_req, data_parallel_rank, true, "GenerateStream") + .await?; let stream = self .state diff --git a/rust/src/server/src/grpc/tests.rs b/rust/src/server/src/grpc/tests.rs index fdc98333f1..274ac67fec 100644 --- a/rust/src/server/src/grpc/tests.rs +++ b/rust/src/server/src/grpc/tests.rs @@ -843,6 +843,37 @@ async fn unary_generate_missing_prompt_returns_invalid_argument() { server_task.abort(); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn unary_generate_unconnected_data_parallel_rank_returns_invalid_argument() { + let (mut client, server_task, _engine_task) = grpc_test_server( + EngineId::from_engine_index(3), + default_stream_output_specs(), + ) + .await; + + let mut request = tonic::Request::new(pb::GenerateRequest { + request_id: "test-unconnected-dp-rank".to_string(), + model: "test-model".to_string(), + prompt: Some(pb::generate_request::Prompt::Text("hello".to_string())), + ..Default::default() + }); + request.metadata_mut().insert( + "x-data-parallel-rank", + "0".parse().expect("valid metadata value"), + ); + + let status = client + .generate(request) + .await + .expect_err("rank 0 should not select globally ranked engine 3"); + + assert_eq!(status.code(), tonic::Code::InvalidArgument); + assert!(status.message().contains("connected ranks: [3]")); + + server_task.abort(); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[serial] async fn unary_generate_min_tokens_above_max_tokens_returns_invalid_argument() { @@ -1523,7 +1554,9 @@ async fn control_aggregates_multi_engine_capacity() { ready_1.data_parallel_rank = 1; let engine_tasks = [ready_0, ready_1].map(|ready| { - let engine_id = EngineId::from_engine_index(ready.data_parallel_rank); + let engine_id = EngineId::from_engine_index( + ready.data_parallel_rank.try_into().expect("test rank fits engine identity"), + ); MockEngineTask::new(spawn_mock_engine_task_with_ready( handshake_address.clone(), engine_id, @@ -1551,10 +1584,8 @@ async fn control_aggregates_multi_engine_capacity() { Llm::new(client), Arc::new(FakeTextBackend) as Arc, ); - let service = ControlServiceImpl::new(Arc::new(AppState::new( - vec!["test-model".to_string()], - chat, - ))); + let state = AppState::new(vec!["test-model".to_string()], chat).with_data_parallel_size(4); + let service = ControlServiceImpl::new(Arc::new(state)); let server = pb::control_server::Control::get_server_info( &service, @@ -1565,6 +1596,7 @@ async fn control_aggregates_multi_engine_capacity() { .into_inner(); assert_eq!(server.max_model_len, 4_096); assert_eq!(server.total_kv_blocks, 30); + assert_eq!(server.parallelism.unwrap().data_parallel_size, 4); drop(engine_tasks); } diff --git a/rust/src/server/src/lib.rs b/rust/src/server/src/lib.rs index c36aaac2a1..ca19d0ab9e 100644 --- a/rust/src/server/src/lib.rs +++ b/rust/src/server/src/lib.rs @@ -141,6 +141,7 @@ async fn build_state(config: &Config) -> Result> { .with_model_path(config.model.clone()) .with_api_server_options(config.api_server_options) .with_server_info(ServerInfoSnapshot::from_config(config)) + .with_data_parallel_size(config.data_parallel_size) .with_api_keys(config.api_keys.clone()) .with_cors(config.cors.clone()) .with_profiler(config.profiler.clone()), diff --git a/rust/src/server/src/routes/tests.rs b/rust/src/server/src/routes/tests.rs index 646d904ea9..92f1d44868 100644 --- a/rust/src/server/src/routes/tests.rs +++ b/rust/src/server/src/routes/tests.rs @@ -725,11 +725,11 @@ async fn test_app_with_dev_mode(dev_mode_enabled: bool) -> axum::Router { /// stays alive for the duration of the test. async fn test_dev_mode_app_with_ready( ready_response: vllm_engine_core_client::protocol::handshake::EngineCoreReadyResponse, + data_parallel_size: usize, ) -> (axum::Router, MockEngineTask) { let ipc = IpcNamespace::new().expect("create ipc namespace"); let handshake_address = ipc.handshake_endpoint(); let engine_id = b"engine-world-size".to_vec(); - let engine_task = MockEngineTask::new(spawn_mock_engine_task_with_ready( handshake_address.clone(), engine_id.clone(), @@ -750,10 +750,10 @@ async fn test_dev_mode_app_with_ready( let chat = ChatLlm::from_shared_backend(test_llm(client), Arc::new(FakeChatBackend::new())); let app = build_router_with_dev_mode( - Arc::new(AppState::new( - vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()], - chat, - )), + Arc::new( + AppState::new(vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()], chat) + .with_data_parallel_size(data_parallel_size), + ), true, ); (app, engine_task) @@ -6313,7 +6313,7 @@ async fn tokenize_allows_prompts_at_or_above_max_model_len() { max_model_len: 4, ..default_ready_response() }; - let (mut app, _engine_task) = test_dev_mode_app_with_ready(ready).await; + let (mut app, _engine_task) = test_dev_mode_app_with_ready(ready, 1).await; let (completion_status, completion_json) = post_json( &mut app, @@ -6569,10 +6569,10 @@ async fn world_size_endpoint_is_dev_mode_only() { async fn world_size_includes_data_parallelism_by_default() { let ready = vllm_engine_core_client::protocol::handshake::EngineCoreReadyResponse { world_size: 2, - data_parallel_size: 4, + data_parallel_size: 1, ..default_ready_response() }; - let (mut app, _engine_task) = test_dev_mode_app_with_ready(ready).await; + let (mut app, _engine_task) = test_dev_mode_app_with_ready(ready, 4).await; let response = app .call( @@ -6598,7 +6598,7 @@ async fn world_size_excludes_data_parallelism_when_include_dp_false() { data_parallel_size: 4, ..default_ready_response() }; - let (mut app, _engine_task) = test_dev_mode_app_with_ready(ready).await; + let (mut app, _engine_task) = test_dev_mode_app_with_ready(ready, 4).await; let response = app .call( diff --git a/rust/src/server/src/routes/world_size.rs b/rust/src/server/src/routes/world_size.rs index 2347158477..2be1bf76b7 100644 --- a/rust/src/server/src/routes/world_size.rs +++ b/rust/src/server/src/routes/world_size.rs @@ -47,7 +47,7 @@ pub async fn get_world_size( let ws = client.world_size(); let world_size = if params.include_dp { - let dp = client.data_parallel_size(); + let dp = state.data_parallel_size() as u64; ws * dp } else { ws diff --git a/rust/src/server/src/state.rs b/rust/src/server/src/state.rs index 722dede8dd..f745cf7c00 100644 --- a/rust/src/server/src/state.rs +++ b/rust/src/server/src/state.rs @@ -40,6 +40,8 @@ pub struct AppState { pub cors: CorsConfig, /// Runtime server information returned by `/server_info`, when available. server_info: Option, + /// Deployment-wide data-parallel size retained by the frontend. + data_parallel_size: usize, /// SHA-256 hashes of API keys accepted as bearer tokens for guarded routes. api_key_hashes: Vec, /// Number of in-flight inference requests currently owned by this frontend. @@ -69,12 +71,14 @@ impl AppState { !served_model_names.is_empty(), "served_model_names must not be empty" ); + let data_parallel_size = chat.engine_core_client().engine_count(); Self { served_model_names, chat, api_server_options: ApiServerOptions::default(), cors: CorsConfig::default(), server_info: None, + data_parallel_size, api_key_hashes: Vec::new(), server_load: AtomicU64::new(0), lora_manager: LoraManager::new(), @@ -114,6 +118,17 @@ impl AppState { self } + /// Set the deployment-wide data-parallel size reported by frontend APIs. + pub(crate) fn with_data_parallel_size(mut self, size: usize) -> Self { + self.data_parallel_size = size; + self + } + + /// Return the deployment-wide data-parallel size. + pub(crate) fn data_parallel_size(&self) -> usize { + self.data_parallel_size + } + /// Build a `/server_info` response payload. pub(crate) fn server_info_response( &self, diff --git a/rust/src/text/src/error.rs b/rust/src/text/src/error.rs index 551c2c9df8..58ed85242c 100644 --- a/rust/src/text/src/error.rs +++ b/rust/src/text/src/error.rs @@ -62,7 +62,10 @@ impl Error { | Self::InvalidRepetitionDetection { .. } // An empty tokenized prompt detected later, at request prepare // time, surfaces through the transparent Llm wrapper. - | Self::Llm(LlmError::EmptyPromptTokenIds { .. }) => true, + | Self::Llm(LlmError::EmptyPromptTokenIds { .. }) + | Self::Llm(LlmError::EngineCoreClient( + EngineCoreError::InvalidDataParallelRank { .. }, + )) => true, _ => false, } } diff --git a/vllm/entrypoints/cli/serve.py b/vllm/entrypoints/cli/serve.py index f90a4f8c1d..9f29cd545f 100644 --- a/vllm/entrypoints/cli/serve.py +++ b/vllm/entrypoints/cli/serve.py @@ -352,6 +352,7 @@ def run_multi_api_server(args: argparse.Namespace): output_address=addresses.outputs[0], engine_start_index=expected_engine_start_index, engine_count=expected_engine_count, + data_parallel_size=parallel_config.data_parallel_size, stats_update_address=stats_update_address, ) else: diff --git a/vllm/v1/utils.py b/vllm/v1/utils.py index f5e0caa818..8d482f58fd 100644 --- a/vllm/v1/utils.py +++ b/vllm/v1/utils.py @@ -339,6 +339,7 @@ class RustFrontendProcessManager: output_address: str, engine_start_index: int, engine_count: int, + data_parallel_size: int, stats_update_address: str | None = None, ): import os @@ -360,6 +361,8 @@ class RustFrontendProcessManager: str(engine_start_index), "--engine-count", str(engine_count), + "--data-parallel-size", + str(data_parallel_size), ] if stats_update_address is not None: cmd.extend(["--coordinator-address", stats_update_address])