[Rust Frontend][gRPC] Add explicit data-parallel rank routing (#51178)
Signed-off-by: Connor Carpenter <connorc@nvidia.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
3c6b2e9c08
commit
fa722b9f01
@@ -451,15 +451,6 @@ impl EngineCoreClient {
|
||||
.world_size
|
||||
}
|
||||
|
||||
/// Return the data parallel size from the parallel config, if available.
|
||||
pub fn data_parallel_size(&self) -> u64 {
|
||||
self.engines
|
||||
.first()
|
||||
.expect("engine core client requires at least one engine")
|
||||
.ready_response
|
||||
.data_parallel_size
|
||||
}
|
||||
|
||||
/// Get the model name associated with this client used for metrics
|
||||
/// labeling.
|
||||
pub fn model_name(&self) -> &str {
|
||||
|
||||
@@ -161,15 +161,16 @@ impl RequestRegistry {
|
||||
|
||||
fn choose_engine_for_request(&mut self, data_parallel_rank: Option<u32>) -> Result<EngineId> {
|
||||
if let Some(rank) = data_parallel_rank {
|
||||
// Route to the engine at the specified rank index.
|
||||
let engine_id = EngineId::from_engine_index(rank);
|
||||
return self
|
||||
.routing_per_engine
|
||||
.contains_key(&engine_id)
|
||||
.then_some(engine_id)
|
||||
let engine_id = u16::try_from(rank).ok().map(EngineId::from_engine_index);
|
||||
return engine_id
|
||||
.filter(|engine_id| self.routing_per_engine.contains_key(engine_id))
|
||||
.ok_or_else(|| Error::InvalidDataParallelRank {
|
||||
rank,
|
||||
num_engines: self.routing_per_engine.len() as u32,
|
||||
connected_ranks: self
|
||||
.routing_per_engine
|
||||
.keys()
|
||||
.filter_map(EngineId::engine_index)
|
||||
.collect(),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -343,6 +344,9 @@ impl RequestRegistry {
|
||||
}
|
||||
|
||||
fn apply_scheduler_counts(&mut self, engine_index: u32, next: EngineLoadSnapshot) -> bool {
|
||||
let Ok(engine_index) = u16::try_from(engine_index) else {
|
||||
return false;
|
||||
};
|
||||
let engine_id = EngineId::from_engine_index(engine_index);
|
||||
let Some(state) = self.routing_per_engine.get_mut(&engine_id) else {
|
||||
return false;
|
||||
@@ -840,26 +844,26 @@ mod tests {
|
||||
error,
|
||||
crate::error::Error::InvalidDataParallelRank {
|
||||
rank: 2,
|
||||
num_engines: 2,
|
||||
}
|
||||
connected_ranks,
|
||||
} if connected_ranks == vec![0, 1]
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn register_with_rank_on_single_engine_only_accepts_zero() {
|
||||
let engine_0 = EngineId::from_engine_index(0);
|
||||
let mut registry = RequestRegistry::new(&[connected_engine(engine_0.clone())]);
|
||||
fn register_with_rank_uses_global_engine_identity() {
|
||||
let engine_3 = EngineId::from_engine_index(3);
|
||||
let mut registry = RequestRegistry::new(&[connected_engine(engine_3.clone())]);
|
||||
|
||||
let (chosen, _) = registry.register("req-ok".to_string(), None, Some(0)).unwrap();
|
||||
assert_eq!(chosen, engine_0);
|
||||
let (chosen, _) = registry.register("req-ok".to_string(), None, Some(3)).unwrap();
|
||||
assert_eq!(chosen, engine_3);
|
||||
|
||||
let error = registry.register("req-bad".to_string(), None, Some(1)).unwrap_err();
|
||||
let error = registry.register("req-bad".to_string(), None, Some(0)).unwrap_err();
|
||||
assert!(matches!(
|
||||
error,
|
||||
crate::error::Error::InvalidDataParallelRank {
|
||||
rank: 1,
|
||||
num_engines: 1,
|
||||
}
|
||||
rank: 0,
|
||||
connected_ranks,
|
||||
} if connected_ranks == vec![3]
|
||||
));
|
||||
}
|
||||
|
||||
|
||||
@@ -68,8 +68,13 @@ pub enum Error {
|
||||
ControlClosed { message: String },
|
||||
#[error("request `{request_id}` is already in flight")]
|
||||
DuplicateRequestId { request_id: String },
|
||||
#[error("data parallel rank {rank} is out of range for {num_engines} engine(s)")]
|
||||
InvalidDataParallelRank { rank: u32, num_engines: u32 },
|
||||
#[error(
|
||||
"data parallel rank {rank} is not connected to this frontend; connected ranks: {connected_ranks:?}"
|
||||
)]
|
||||
InvalidDataParallelRank {
|
||||
rank: u32,
|
||||
connected_ranks: Vec<u32>,
|
||||
},
|
||||
#[error("engine-core output dispatcher closed: {message}")]
|
||||
DispatcherClosed { message: String },
|
||||
#[error("engine-core client is closed: {message}")]
|
||||
|
||||
@@ -66,8 +66,8 @@ impl EngineId {
|
||||
|
||||
/// Construct an engine id from the Python-compatible engine index encoding
|
||||
/// (two-byte little-endian).
|
||||
pub fn from_engine_index(value: u32) -> Self {
|
||||
Self(Bytes::copy_from_slice(&(value as u16).to_le_bytes()))
|
||||
pub fn from_engine_index(value: u16) -> Self {
|
||||
Self(Bytes::copy_from_slice(&value.to_le_bytes()))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -336,6 +336,22 @@ pub async fn connect_bootstrapped(
|
||||
if engine_count == 0 {
|
||||
bail_unexpected_handshake_message!("expected engine_count >= 1");
|
||||
}
|
||||
let engine_start_index =
|
||||
u16::try_from(engine_start_index).map_err(|_| Error::UnexpectedHandshakeMessage {
|
||||
message: "engine_start_index exceeds the two-byte engine identity limit".to_string(),
|
||||
})?;
|
||||
let engine_end_index =
|
||||
usize::from(engine_start_index).checked_add(engine_count).ok_or_else(|| {
|
||||
Error::UnexpectedHandshakeMessage {
|
||||
message: "engine_start_index + engine_count overflows".to_string(),
|
||||
}
|
||||
})?;
|
||||
if engine_end_index > usize::from(u16::MAX) + 1 {
|
||||
return Err(Error::UnexpectedHandshakeMessage {
|
||||
message: "engine_start_index + engine_count exceeds the two-byte engine identity limit"
|
||||
.to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
let mut input_socket = RouterSocket::new();
|
||||
let input_address = input_socket.bind(input_address).await?.to_string();
|
||||
@@ -345,8 +361,10 @@ pub async fn connect_bootstrapped(
|
||||
|
||||
let engines = wait_for_input_registrations(
|
||||
&mut input_socket,
|
||||
(0..engine_count)
|
||||
.map(|offset| EngineId::from_engine_index(engine_start_index + offset as u32)),
|
||||
(0..engine_count).map(|offset| {
|
||||
let offset = u16::try_from(offset).expect("validated engine offset fits u16");
|
||||
EngineId::from_engine_index(engine_start_index + offset)
|
||||
}),
|
||||
ready_timeout,
|
||||
)
|
||||
.await?;
|
||||
|
||||
Reference in New Issue
Block a user