[Rust Frontend] Simplify data-parallel size ownership (#52575)
Signed-off-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -15,7 +15,7 @@ use tracing::{debug, info, trace};
|
||||
|
||||
use crate::client::imp::{ClientInner, run_abort_loop, run_output_dispatcher_loop};
|
||||
use crate::coordinator::CoordinatorHandle;
|
||||
use crate::error::{Error, Result};
|
||||
use crate::error::{Error, Result, bail_invalid_client_config};
|
||||
use crate::protocol::dtype::ModelDtype;
|
||||
use crate::protocol::handshake::EngineCoreReadyResponse;
|
||||
use crate::protocol::lora::LoraRequest;
|
||||
@@ -68,11 +68,70 @@ pub enum TransportMode {
|
||||
engine_start_index: u32,
|
||||
/// Total number of engines expected to register on this transport.
|
||||
engine_count: usize,
|
||||
/// Deployment-wide data-parallel size. This may be larger than
|
||||
/// `engine_count` when a supervisor partitions ranks across frontends.
|
||||
data_parallel_size: usize,
|
||||
/// Maximum time to wait for all expected engines to register.
|
||||
ready_timeout: Duration,
|
||||
},
|
||||
}
|
||||
|
||||
impl TransportMode {
|
||||
/// Return the deployment-wide data-parallel size for this transport.
|
||||
pub fn data_parallel_size(&self) -> usize {
|
||||
match self {
|
||||
Self::HandshakeOwner { engine_count, .. } => *engine_count,
|
||||
Self::Bootstrapped {
|
||||
data_parallel_size, ..
|
||||
} => *data_parallel_size,
|
||||
}
|
||||
}
|
||||
|
||||
/// Validate the transport topology before opening any sockets.
|
||||
pub fn validate(&self) -> Result<()> {
|
||||
let data_parallel_size = self.data_parallel_size();
|
||||
if data_parallel_size == 0 {
|
||||
bail_invalid_client_config!("data parallel size must be at least 1");
|
||||
}
|
||||
if data_parallel_size > usize::from(u16::MAX) + 1 {
|
||||
bail_invalid_client_config!(
|
||||
"data parallel size ({data_parallel_size}) exceeds the two-byte engine identity limit"
|
||||
);
|
||||
}
|
||||
|
||||
match self {
|
||||
Self::HandshakeOwner { .. } => {}
|
||||
Self::Bootstrapped {
|
||||
engine_start_index,
|
||||
engine_count,
|
||||
..
|
||||
} => {
|
||||
if *engine_count == 0 {
|
||||
bail_invalid_client_config!("engine count must be at least 1");
|
||||
}
|
||||
let engine_start_index = usize::try_from(*engine_start_index).map_err(|_| {
|
||||
Error::InvalidClientConfig {
|
||||
message: "engine start index does not fit usize".to_string(),
|
||||
}
|
||||
})?;
|
||||
let engine_end_index =
|
||||
engine_start_index.checked_add(*engine_count).ok_or_else(|| {
|
||||
Error::InvalidClientConfig {
|
||||
message: "engine start index + engine count overflows".to_string(),
|
||||
}
|
||||
})?;
|
||||
if engine_end_index > data_parallel_size {
|
||||
bail_invalid_client_config!(
|
||||
"connected engine range [{engine_start_index}, {engine_end_index}) exceeds data parallel size ({data_parallel_size})"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Which coordinator implementation should be active when one is present for a
|
||||
/// frontend client.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
@@ -117,6 +176,11 @@ impl EngineCoreClientConfig {
|
||||
}
|
||||
}
|
||||
|
||||
/// Validate the client topology before opening any transport sockets.
|
||||
pub fn validate(&self) -> Result<()> {
|
||||
self.transport_mode.validate()
|
||||
}
|
||||
|
||||
/// Set the model name used by frontend-side metrics and diagnostics.
|
||||
pub fn with_model_name(mut self, model_name: impl Into<String>) -> Self {
|
||||
self.model_name = model_name.into();
|
||||
@@ -226,6 +290,7 @@ impl EngineCoreClient {
|
||||
/// handshake. In bootstrapped mode it binds the provided frontend
|
||||
/// sockets and waits for the expected engine registration frames.
|
||||
pub async fn connect(config: EngineCoreClientConfig) -> Result<Self> {
|
||||
config.validate()?;
|
||||
let connected = match &config.transport_mode {
|
||||
TransportMode::HandshakeOwner {
|
||||
handshake_address,
|
||||
@@ -261,6 +326,7 @@ impl EngineCoreClient {
|
||||
engine_start_index,
|
||||
engine_count,
|
||||
ready_timeout,
|
||||
..
|
||||
} => {
|
||||
if let Some(CoordinatorMode::InProc) = config.coordinator_mode {
|
||||
panic!("cannot use in-process coordinator with bootstrapped transport mode")
|
||||
@@ -378,6 +444,12 @@ impl EngineCoreClient {
|
||||
self.engines.len()
|
||||
}
|
||||
|
||||
/// Return the deployment-wide data-parallel size configured for this
|
||||
/// client.
|
||||
pub fn data_parallel_size(&self) -> usize {
|
||||
self.config.transport_mode.data_parallel_size()
|
||||
}
|
||||
|
||||
/// Return the engine-side indices connected to this client.
|
||||
pub fn engine_indices(&self) -> Vec<u32> {
|
||||
self.engines
|
||||
|
||||
@@ -49,6 +49,8 @@ pub enum Error {
|
||||
UnexpectedHandshakeIdentity { expected: Vec<u8>, actual: Vec<u8> },
|
||||
#[error("unexpected startup handshake message: {message}")]
|
||||
UnexpectedHandshakeMessage { message: String },
|
||||
#[error("invalid engine-core client configuration: {message}")]
|
||||
InvalidClientConfig { message: String },
|
||||
#[error("unexpected non-control output on coordinator path: {message}")]
|
||||
UnexpectedCoordinatorOutput { message: String },
|
||||
#[error("unexpected output on main dispatcher path: {message}")]
|
||||
|
||||
@@ -56,7 +56,7 @@ pub fn default_ready_response() -> EngineCoreReadyResponse {
|
||||
dtype: ModelDtype::Float32,
|
||||
vllm_version: "test-vllm-version".to_string(),
|
||||
world_size: 1,
|
||||
data_parallel_size: 1,
|
||||
effective_data_parallel_size: 1,
|
||||
tensor_parallel_size: 1,
|
||||
pipeline_parallel_size: 1,
|
||||
decode_context_parallel_size: 1,
|
||||
|
||||
@@ -63,8 +63,11 @@ pub struct EngineCoreReadyResponse {
|
||||
pub vllm_version: String,
|
||||
/// World size (TP * PP) from the parallel config.
|
||||
pub world_size: u64,
|
||||
/// Data parallelism size from the parallel config.
|
||||
pub data_parallel_size: u64,
|
||||
/// Data-parallel size from this EngineCore's effective parallel config.
|
||||
/// Dense independent-DP ranks are reconfigured to report `1`; the client
|
||||
/// transport owns the deployment-wide data-parallel size.
|
||||
#[serde(rename = "data_parallel_size")]
|
||||
pub effective_data_parallel_size: u64,
|
||||
// Required discovery metadata; EngineCore and client versions must match.
|
||||
/// Tensor-parallel size of this engine.
|
||||
pub tensor_parallel_size: u32,
|
||||
|
||||
@@ -303,6 +303,7 @@ fn bootstrapped_test_config(
|
||||
output_address,
|
||||
engine_start_index: 0,
|
||||
engine_count,
|
||||
data_parallel_size: engine_count,
|
||||
ready_timeout,
|
||||
},
|
||||
coordinator_mode,
|
||||
@@ -330,15 +331,47 @@ fn bootstrapped_test_config_with_start_index(
|
||||
);
|
||||
let TransportMode::Bootstrapped {
|
||||
engine_start_index: start,
|
||||
data_parallel_size,
|
||||
..
|
||||
} = &mut config.transport_mode
|
||||
else {
|
||||
unreachable!("bootstrapped_test_config returns bootstrapped transport")
|
||||
};
|
||||
*start = engine_start_index;
|
||||
*data_parallel_size = usize::try_from(engine_start_index)
|
||||
.expect("test start index fits usize")
|
||||
.checked_add(engine_count)
|
||||
.expect("test engine range fits usize");
|
||||
config
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn client_config_validates_bootstrapped_dp_range() {
|
||||
let mut config = bootstrapped_test_config_with_start_index(
|
||||
"ipc://unused-input".to_string(),
|
||||
"ipc://unused-output".to_string(),
|
||||
1,
|
||||
2,
|
||||
Duration::from_secs(1),
|
||||
0,
|
||||
None,
|
||||
);
|
||||
config.validate().expect("frontend may own a subset of global DP ranks");
|
||||
|
||||
let TransportMode::Bootstrapped {
|
||||
data_parallel_size, ..
|
||||
} = &mut config.transport_mode
|
||||
else {
|
||||
unreachable!("expected bootstrapped transport")
|
||||
};
|
||||
*data_parallel_size = 2;
|
||||
let error = config.validate().expect_err("engine range above DP size must fail");
|
||||
expect_test::expect![[
|
||||
"invalid engine-core client configuration: connected engine range [1, 3) exceeds data parallel size (2)"
|
||||
]]
|
||||
.assert_eq(&error.to_string());
|
||||
}
|
||||
|
||||
async fn recv_xpub_message(xpub: &mut XPubSocket) -> Vec<bytes::Bytes> {
|
||||
xpub.recv().await.unwrap().into_vec()
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user