[Bugfix] Fix external LB DP rank handling when replicas share nodes (#53743)
Signed-off-by: Tony Lin <tony.lin@intel.com>
This commit is contained in:
@@ -2232,12 +2232,22 @@ class EngineArgs:
|
||||
self.node_rank * local_world_size
|
||||
) // world_size_within_dp
|
||||
if self.data_parallel_size > 1 and self.data_parallel_external_lb:
|
||||
self.data_parallel_rank = inferred_data_parallel_rank
|
||||
logger.info(
|
||||
"Inferred data_parallel_rank %d from node_rank %d for external lb",
|
||||
self.data_parallel_rank,
|
||||
self.node_rank,
|
||||
)
|
||||
if self.data_parallel_rank is None:
|
||||
if self.nnodes % self.data_parallel_size != 0:
|
||||
raise ValueError(
|
||||
"Invalid data-parallel launch options: "
|
||||
"`--node-rank` cannot unambiguously identify external "
|
||||
"data-parallel ranks when `--nnodes` is not divisible "
|
||||
"by `--data-parallel-size`. Set a unique "
|
||||
"`--data-parallel-rank` for each external-LB process."
|
||||
)
|
||||
self.data_parallel_rank = inferred_data_parallel_rank
|
||||
logger.info(
|
||||
"Inferred data_parallel_rank %d from node_rank %d "
|
||||
"for external lb",
|
||||
self.data_parallel_rank,
|
||||
self.node_rank,
|
||||
)
|
||||
elif self.data_parallel_size_local is None:
|
||||
# Infer data parallel size local for internal dplb:
|
||||
self.data_parallel_size_local = max(
|
||||
|
||||
Reference in New Issue
Block a user