[Rust Frontend][gRPC] Add explicit data-parallel rank routing (#51178)

Signed-off-by: Connor Carpenter <connorc@nvidia.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Connor Carpenter
2026-08-10 15:05:24 -07:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 3c6b2e9c08
commit fa722b9f01
19 changed files with 215 additions and 60 deletions
@@ -451,15 +451,6 @@ impl EngineCoreClient {
.world_size
}
/// Return the data parallel size from the parallel config, if available.
pub fn data_parallel_size(&self) -> u64 {
self.engines
.first()
.expect("engine core client requires at least one engine")
.ready_response
.data_parallel_size
}
/// Get the model name associated with this client used for metrics
/// labeling.
pub fn model_name(&self) -> &str {
+22 -18
View File
@@ -161,15 +161,16 @@ impl RequestRegistry {
fn choose_engine_for_request(&mut self, data_parallel_rank: Option<u32>) -> Result<EngineId> {
if let Some(rank) = data_parallel_rank {
// Route to the engine at the specified rank index.
let engine_id = EngineId::from_engine_index(rank);
return self
.routing_per_engine
.contains_key(&engine_id)
.then_some(engine_id)
let engine_id = u16::try_from(rank).ok().map(EngineId::from_engine_index);
return engine_id
.filter(|engine_id| self.routing_per_engine.contains_key(engine_id))
.ok_or_else(|| Error::InvalidDataParallelRank {
rank,
num_engines: self.routing_per_engine.len() as u32,
connected_ranks: self
.routing_per_engine
.keys()
.filter_map(EngineId::engine_index)
.collect(),
});
}
@@ -343,6 +344,9 @@ impl RequestRegistry {
}
fn apply_scheduler_counts(&mut self, engine_index: u32, next: EngineLoadSnapshot) -> bool {
let Ok(engine_index) = u16::try_from(engine_index) else {
return false;
};
let engine_id = EngineId::from_engine_index(engine_index);
let Some(state) = self.routing_per_engine.get_mut(&engine_id) else {
return false;
@@ -840,26 +844,26 @@ mod tests {
error,
crate::error::Error::InvalidDataParallelRank {
rank: 2,
num_engines: 2,
}
connected_ranks,
} if connected_ranks == vec![0, 1]
));
}
#[test]
fn register_with_rank_on_single_engine_only_accepts_zero() {
let engine_0 = EngineId::from_engine_index(0);
let mut registry = RequestRegistry::new(&[connected_engine(engine_0.clone())]);
fn register_with_rank_uses_global_engine_identity() {
let engine_3 = EngineId::from_engine_index(3);
let mut registry = RequestRegistry::new(&[connected_engine(engine_3.clone())]);
let (chosen, _) = registry.register("req-ok".to_string(), None, Some(0)).unwrap();
assert_eq!(chosen, engine_0);
let (chosen, _) = registry.register("req-ok".to_string(), None, Some(3)).unwrap();
assert_eq!(chosen, engine_3);
let error = registry.register("req-bad".to_string(), None, Some(1)).unwrap_err();
let error = registry.register("req-bad".to_string(), None, Some(0)).unwrap_err();
assert!(matches!(
error,
crate::error::Error::InvalidDataParallelRank {
rank: 1,
num_engines: 1,
}
rank: 0,
connected_ranks,
} if connected_ranks == vec![3]
));
}
+7 -2
View File
@@ -68,8 +68,13 @@ pub enum Error {
ControlClosed { message: String },
#[error("request `{request_id}` is already in flight")]
DuplicateRequestId { request_id: String },
#[error("data parallel rank {rank} is out of range for {num_engines} engine(s)")]
InvalidDataParallelRank { rank: u32, num_engines: u32 },
#[error(
"data parallel rank {rank} is not connected to this frontend; connected ranks: {connected_ranks:?}"
)]
InvalidDataParallelRank {
rank: u32,
connected_ranks: Vec<u32>,
},
#[error("engine-core output dispatcher closed: {message}")]
DispatcherClosed { message: String },
#[error("engine-core client is closed: {message}")]
+22 -4
View File
@@ -66,8 +66,8 @@ impl EngineId {
/// Construct an engine id from the Python-compatible engine index encoding
/// (two-byte little-endian).
pub fn from_engine_index(value: u32) -> Self {
Self(Bytes::copy_from_slice(&(value as u16).to_le_bytes()))
pub fn from_engine_index(value: u16) -> Self {
Self(Bytes::copy_from_slice(&value.to_le_bytes()))
}
}
@@ -336,6 +336,22 @@ pub async fn connect_bootstrapped(
if engine_count == 0 {
bail_unexpected_handshake_message!("expected engine_count >= 1");
}
let engine_start_index =
u16::try_from(engine_start_index).map_err(|_| Error::UnexpectedHandshakeMessage {
message: "engine_start_index exceeds the two-byte engine identity limit".to_string(),
})?;
let engine_end_index =
usize::from(engine_start_index).checked_add(engine_count).ok_or_else(|| {
Error::UnexpectedHandshakeMessage {
message: "engine_start_index + engine_count overflows".to_string(),
}
})?;
if engine_end_index > usize::from(u16::MAX) + 1 {
return Err(Error::UnexpectedHandshakeMessage {
message: "engine_start_index + engine_count exceeds the two-byte engine identity limit"
.to_string(),
});
}
let mut input_socket = RouterSocket::new();
let input_address = input_socket.bind(input_address).await?.to_string();
@@ -345,8 +361,10 @@ pub async fn connect_bootstrapped(
let engines = wait_for_input_registrations(
&mut input_socket,
(0..engine_count)
.map(|offset| EngineId::from_engine_index(engine_start_index + offset as u32)),
(0..engine_count).map(|offset| {
let offset = u16::try_from(offset).expect("validated engine offset fits u16");
EngineId::from_engine_index(engine_start_index + offset)
}),
ready_timeout,
)
.await?;