[V1] Only print cudagraph tqdm on rank 0 with is_global_first_rank (#19516)
Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
@@ -1315,6 +1315,37 @@ def in_the_same_node_as(pg: Union[ProcessGroup, StatelessProcessGroup],
|
||||
return [x == 1 for x in aggregated_data.tolist()]
|
||||
|
||||
|
||||
def is_global_first_rank() -> bool:
|
||||
"""
|
||||
Check if the current process is the first rank globally across all
|
||||
parallelism strategies (PP, TP, DP, EP, etc.).
|
||||
|
||||
Unlike group-specific checks like `get_tensor_model_parallel_rank() == 0`
|
||||
or `get_pp_group().is_first_rank`, this function checks the global rank
|
||||
across all parallelism dimensions.
|
||||
|
||||
Returns:
|
||||
bool: True if this is the global first rank (rank 0), False otherwise.
|
||||
Returns True if distributed is not initialized (single process).
|
||||
"""
|
||||
try:
|
||||
# If world group is available, use it for the most accurate check
|
||||
global _WORLD
|
||||
if _WORLD is not None:
|
||||
return _WORLD.is_first_rank
|
||||
|
||||
# If torch distributed is not initialized, assume single process
|
||||
if not torch.distributed.is_initialized():
|
||||
return True
|
||||
|
||||
# Fallback to torch's global rank
|
||||
return torch.distributed.get_rank() == 0
|
||||
|
||||
except Exception:
|
||||
# If anything goes wrong, assume this is the first rank
|
||||
return True
|
||||
|
||||
|
||||
def _node_count(pg: Union[ProcessGroup, StatelessProcessGroup]) -> int:
|
||||
"""
|
||||
Returns the total number of nodes in the process group.
|
||||
|
||||
Reference in New Issue
Block a user