[Core] avoid too many cuda context by caching p2p test (#4021)

This commit is contained in:
youkaichao
2024-04-13 23:40:21 -07:00
committed by GitHub
parent 711a000255
commit 2cd6b4f362
3 changed files with 116 additions and 33 deletions
+9
View File
@@ -41,6 +41,13 @@ _CPU_WORLD_GROUP = None
# source rank when broadcasting from the first or last pipeline stage.
_PIPELINE_GLOBAL_RANKS = None
_LOCAL_RANK = -1
def get_local_rank():
global _LOCAL_RANK
return _LOCAL_RANK
def init_distributed_environment(
world_size: int = -1,
@@ -66,6 +73,8 @@ def init_distributed_environment(
ranks = list(range(torch.distributed.get_world_size()))
_CPU_WORLD_GROUP = torch.distributed.new_group(ranks=ranks,
backend="gloo")
global _LOCAL_RANK
_LOCAL_RANK = local_rank
def initialize_model_parallel(