[Core] avoid too many cuda context by caching p2p test (#4021)
This commit is contained in:
@@ -41,6 +41,13 @@ _CPU_WORLD_GROUP = None
|
||||
# source rank when broadcasting from the first or last pipeline stage.
|
||||
_PIPELINE_GLOBAL_RANKS = None
|
||||
|
||||
_LOCAL_RANK = -1
|
||||
|
||||
|
||||
def get_local_rank():
|
||||
global _LOCAL_RANK
|
||||
return _LOCAL_RANK
|
||||
|
||||
|
||||
def init_distributed_environment(
|
||||
world_size: int = -1,
|
||||
@@ -66,6 +73,8 @@ def init_distributed_environment(
|
||||
ranks = list(range(torch.distributed.get_world_size()))
|
||||
_CPU_WORLD_GROUP = torch.distributed.new_group(ranks=ranks,
|
||||
backend="gloo")
|
||||
global _LOCAL_RANK
|
||||
_LOCAL_RANK = local_rank
|
||||
|
||||
|
||||
def initialize_model_parallel(
|
||||
|
||||
Reference in New Issue
Block a user