[Rust Frontend] Simplify data-parallel size ownership (#52575)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Bugen Zhao
2026-08-18 09:06:05 +00:00
committed by GitHub
co-authored by Nick Hill
parent eab1cff5b0
commit 41f179b57a
19 changed files with 188 additions and 134 deletions
@@ -58,7 +58,6 @@ async fn main() -> Result<()> {
local_input_address: None,
local_output_address: None,
},
data_parallel_size: args.engine_count,
coordinator_mode: CoordinatorMode::MaybeInProc,
model: args.model,
served_model_name: vec![],
+1 -42
View File
@@ -161,8 +161,6 @@ impl TlsConfig {
pub struct Config {
/// Frontend-to-engine transport setup.
pub transport_mode: TransportMode,
/// Deployment-wide data-parallel size retained by the frontend.
pub data_parallel_size: usize,
/// Requested frontend-side coordinator behavior.
pub coordinator_mode: CoordinatorMode,
/// Backend model identifier used for engine-core loading.
@@ -239,46 +237,7 @@ impl Config {
max_logprobs
);
}
if self.data_parallel_size == 0 {
bail!("data parallel size must be at least 1");
}
if self.data_parallel_size > usize::from(u16::MAX) + 1 {
bail!(
"data parallel size ({}) exceeds the two-byte engine identity limit",
self.data_parallel_size
);
}
match &self.transport_mode {
TransportMode::HandshakeOwner { engine_count, .. } => {
if *engine_count != self.data_parallel_size {
bail!(
"managed frontend engine count ({engine_count}) must equal data parallel size ({})",
self.data_parallel_size
);
}
}
TransportMode::Bootstrapped {
engine_start_index,
engine_count,
..
} => {
if *engine_count == 0 {
bail!("engine count must be at least 1");
}
let engine_start_index = usize::try_from(*engine_start_index)
.map_err(|_| anyhow::anyhow!("engine start index does not fit usize"))?;
let engine_end_index =
engine_start_index.checked_add(*engine_count).ok_or_else(|| {
anyhow::anyhow!("engine start index + engine count overflows")
})?;
if engine_end_index > self.data_parallel_size {
bail!(
"connected engine range [{engine_start_index}, {engine_end_index}) exceeds data parallel size {}",
self.data_parallel_size
);
}
}
}
self.transport_mode.validate()?;
Ok(())
}
+1 -1
View File
@@ -43,7 +43,7 @@ impl ControlServiceImpl {
pb::ParallelismInfo {
tensor_parallel_size: ready.tensor_parallel_size,
pipeline_parallel_size: ready.pipeline_parallel_size,
data_parallel_size: self.state.data_parallel_size().min(u32::MAX as usize) as u32,
data_parallel_size: self.client().data_parallel_size().min(u32::MAX as usize) as u32,
data_parallel_rank: ready.data_parallel_rank,
decode_context_parallel_size: ready.decode_context_parallel_size,
}
+4 -4
View File
@@ -1625,7 +1625,7 @@ async fn control_aggregates_multi_engine_capacity() {
let mut ready_0 = default_ready_response();
ready_0.max_model_len = 8_192;
ready_0.num_gpu_blocks = 10;
ready_0.data_parallel_size = 2;
ready_0.effective_data_parallel_size = 2;
ready_0.weight_transfer_backend = Some("nccl".to_string());
ready_0.enable_sleep_mode = true;
ready_0.supports_draft_weight_updates = true;
@@ -1633,7 +1633,7 @@ async fn control_aggregates_multi_engine_capacity() {
let mut ready_1 = default_ready_response();
ready_1.max_model_len = 4_096;
ready_1.num_gpu_blocks = 20;
ready_1.data_parallel_size = 2;
ready_1.effective_data_parallel_size = 2;
ready_1.data_parallel_rank = 1;
let engine_tasks = [ready_0, ready_1].map(|ready| {
@@ -1667,7 +1667,7 @@ async fn control_aggregates_multi_engine_capacity() {
Llm::new(client),
Arc::new(FakeTextBackend) as Arc<dyn ChatTextBackend>,
);
let state = AppState::new(vec!["test-model".to_string()], chat).with_data_parallel_size(4);
let state = AppState::new(vec!["test-model".to_string()], chat);
let service = ControlServiceImpl::new(Arc::new(state));
let server = pb::control_server::Control::get_server_info(
@@ -1684,7 +1684,7 @@ async fn control_aggregates_multi_engine_capacity() {
assert!(rl.weight_transfer_backend.is_empty());
assert!(!rl.sleep_mode_enabled);
assert!(!rl.draft_weight_updates_enabled);
assert_eq!(server.parallelism.unwrap().data_parallel_size, 4);
assert_eq!(server.parallelism.unwrap().data_parallel_size, 2);
drop(engine_tasks);
}
-1
View File
@@ -141,7 +141,6 @@ async fn build_state(config: &Config) -> Result<Arc<AppState>> {
.with_model_path(config.model.clone())
.with_api_server_options(config.api_server_options)
.with_server_info(ServerInfoSnapshot::from_config(config))
.with_data_parallel_size(config.data_parallel_size)
.with_api_keys(config.api_keys.clone())
.with_cors(config.cors.clone())
.with_profiler(config.profiler.clone()),
+6 -59
View File
@@ -704,14 +704,13 @@ async fn test_app_with_dev_mode(dev_mode_enabled: bool) -> axum::Router {
/// stays alive for the duration of the test.
async fn test_dev_mode_app_with_ready(
ready_response: vllm_engine_core_client::protocol::handshake::EngineCoreReadyResponse,
data_parallel_size: usize,
) -> (axum::Router, MockEngineTask) {
let ipc = IpcNamespace::new().expect("create ipc namespace");
let handshake_address = ipc.handshake_endpoint();
let engine_id = b"engine-world-size".to_vec();
let engine_task = MockEngineTask::new(spawn_mock_engine_task_with_ready(
handshake_address.clone(),
engine_id.clone(),
engine_id,
ready_response,
|_dealer, _push| boxed_test_future(async {}),
));
@@ -729,10 +728,10 @@ async fn test_dev_mode_app_with_ready(
let chat = ChatLlm::from_shared_backend(test_llm(client), Arc::new(FakeChatBackend::new()));
let app = build_router_with_dev_mode(
Arc::new(
AppState::new(vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()], chat)
.with_data_parallel_size(data_parallel_size),
),
Arc::new(AppState::new(
vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()],
chat,
)),
true,
);
(app, engine_task)
@@ -6292,7 +6291,7 @@ async fn tokenize_allows_prompts_at_or_above_max_model_len() {
max_model_len: 4,
..default_ready_response()
};
let (mut app, _engine_task) = test_dev_mode_app_with_ready(ready, 1).await;
let (mut app, _engine_task) = test_dev_mode_app_with_ready(ready).await;
let (completion_status, completion_json) = post_json(
&mut app,
@@ -6543,58 +6542,6 @@ async fn world_size_endpoint_is_dev_mode_only() {
assert_eq!(response.status(), StatusCode::NOT_FOUND);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn world_size_includes_data_parallelism_by_default() {
let ready = vllm_engine_core_client::protocol::handshake::EngineCoreReadyResponse {
world_size: 2,
data_parallel_size: 1,
..default_ready_response()
};
let (mut app, _engine_task) = test_dev_mode_app_with_ready(ready, 4).await;
let response = app
.call(
Request::builder()
.uri("/get_world_size")
.body(Body::empty())
.expect("build request"),
)
.await
.expect("call app");
assert_eq!(response.status(), StatusCode::OK);
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json");
assert_eq!(json, json!({"world_size": 8}));
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn world_size_excludes_data_parallelism_when_include_dp_false() {
let ready = vllm_engine_core_client::protocol::handshake::EngineCoreReadyResponse {
world_size: 2,
data_parallel_size: 4,
..default_ready_response()
};
let (mut app, _engine_task) = test_dev_mode_app_with_ready(ready, 4).await;
let response = app
.call(
Request::builder()
.uri("/get_world_size?include_dp=false")
.body(Body::empty())
.expect("build request"),
)
.await
.expect("call app");
assert_eq!(response.status(), StatusCode::OK);
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json");
assert_eq!(json, json!({"world_size": 2}));
}
// ========================= Profiler route tests =========================
async fn test_profiling_app_with_engine_script<F>(script: F) -> (axum::Router, MockEngineTask)
+1 -1
View File
@@ -47,7 +47,7 @@ pub async fn get_world_size(
let ws = client.world_size();
let world_size = if params.include_dp {
let dp = state.data_parallel_size() as u64;
let dp = client.data_parallel_size() as u64;
ws * dp
} else {
ws
-15
View File
@@ -40,8 +40,6 @@ pub struct AppState {
pub cors: CorsConfig,
/// Runtime server information returned by `/server_info`, when available.
server_info: Option<ServerInfoSnapshot>,
/// Deployment-wide data-parallel size retained by the frontend.
data_parallel_size: usize,
/// SHA-256 hashes of API keys accepted as bearer tokens for guarded routes.
api_key_hashes: Vec<ApiKeyHash>,
/// Number of in-flight inference requests currently owned by this frontend.
@@ -71,14 +69,12 @@ impl AppState {
!served_model_names.is_empty(),
"served_model_names must not be empty"
);
let data_parallel_size = chat.engine_core_client().engine_count();
Self {
served_model_names,
chat,
api_server_options: ApiServerOptions::default(),
cors: CorsConfig::default(),
server_info: None,
data_parallel_size,
api_key_hashes: Vec::new(),
server_load: AtomicU64::new(0),
lora_manager: LoraManager::new(),
@@ -118,17 +114,6 @@ impl AppState {
self
}
/// Set the deployment-wide data-parallel size reported by frontend APIs.
pub(crate) fn with_data_parallel_size(mut self, size: usize) -> Self {
self.data_parallel_size = size;
self
}
/// Return the deployment-wide data-parallel size.
pub(crate) fn data_parallel_size(&self) -> usize {
self.data_parallel_size
}
/// Build a `/server_info` response payload.
pub(crate) fn server_info_response(
&self,