[EPLB] Reject NCCL-based EPLB communicators with async EPLB (#44978)
Signed-off-by: Markov Ilya <markovilya197@gmail.com>
This commit is contained in:
@@ -470,6 +470,9 @@ class ElasticEPScalingExecutor:
|
||||
eplb_model_state.expert_buffer = [
|
||||
torch.empty_like(w) for w in model.expert_weights[0]
|
||||
]
|
||||
assert parallel_config.eplb_config.communicator is not None, (
|
||||
"EPLB communicator backend must be set by ParallelConfig"
|
||||
)
|
||||
eplb_model_state.communicator = create_eplb_communicator(
|
||||
group_coordinator=get_eplb_group(),
|
||||
backend=parallel_config.eplb_config.communicator,
|
||||
|
||||
@@ -617,7 +617,7 @@ class PyNcclEplbCommunicator(EplbCommunicator):
|
||||
|
||||
def create_eplb_communicator(
|
||||
group_coordinator: GroupCoordinator,
|
||||
backend: str | None,
|
||||
backend: str,
|
||||
expert_weights: Sequence[Sequence[torch.Tensor]],
|
||||
expert_buffer: Sequence[torch.Tensor],
|
||||
) -> EplbCommunicator:
|
||||
@@ -628,7 +628,6 @@ def create_eplb_communicator(
|
||||
device and CPU communication groups.
|
||||
backend: Communicator backend name (``"torch_nccl"``,
|
||||
``"torch_gloo"``, ``"pynccl"``, or ``"nixl"``).
|
||||
Falls back to ``"torch_nccl"`` when *None*.
|
||||
Stateless (elastic EP) groups only support ``"torch_nccl"``
|
||||
and ``"pynccl"``; ``"torch_nccl"`` is silently promoted to
|
||||
``"pynccl"`` in that case. When tensors reside on CPU,
|
||||
@@ -641,9 +640,6 @@ def create_eplb_communicator(
|
||||
expert_buffer: Pre-allocated receive buffer tensors (one per
|
||||
weight tensor in a single layer).
|
||||
"""
|
||||
if backend is None:
|
||||
backend = "torch_nccl"
|
||||
|
||||
first_layer = expert_weights[0] if expert_weights else []
|
||||
tensor_device_type = first_layer[0].device.type if first_layer else "cpu"
|
||||
torch_group = (
|
||||
|
||||
@@ -447,6 +447,9 @@ class EplbState:
|
||||
self._init_should_record_tensor(model)
|
||||
expert_buffer = [torch.empty_like(w) for w in model.expert_weights[0]]
|
||||
|
||||
assert self.parallel_config.eplb_config.communicator is not None, (
|
||||
"EPLB communicator backend must be set by ParallelConfig"
|
||||
)
|
||||
communicator = create_eplb_communicator(
|
||||
group_coordinator=get_eplb_group(),
|
||||
backend=self.parallel_config.eplb_config.communicator,
|
||||
|
||||
@@ -74,8 +74,6 @@ def override_envs_for_eplb(
|
||||
"""
|
||||
is_data_parallel = parallel_config.data_parallel_size > 1
|
||||
is_eplb_enabled = parallel_config.enable_eplb
|
||||
async_eplb = parallel_config.eplb_config.use_async
|
||||
is_deepep_ll = parallel_config.all2all_backend == "deepep_low_latency"
|
||||
is_mega_moe = moe_backend == "deep_gemm_mega_moe"
|
||||
is_nccl_based_eplb_communicator = parallel_config.eplb_config.communicator in (
|
||||
"torch_nccl",
|
||||
@@ -85,29 +83,16 @@ def override_envs_for_eplb(
|
||||
# Override NCCL_MAX_CTAS to avoid hangs when EPLB's NCCL weight exchange
|
||||
# contends with MoE backend's cooperative-launch on GPU SMs.
|
||||
#
|
||||
# DeepEP low-latency:
|
||||
# The hang happens when two ranks interleave kernel launches differently
|
||||
# between NCCL collectives (used by async EPLB weight exchange) and DeepEP
|
||||
# low-latency (LL) kernels. DeepEP LL uses a cooperative launch and tries
|
||||
# to reserve a large fraction of the GPU's SMs; if those SMs are currently
|
||||
# occupied by NCCL, the DeepEP LL launch blocks until enough SMs are
|
||||
# freed.
|
||||
#
|
||||
# If rank A enters DeepEP LL in main thread while rank B is still executing
|
||||
# NCCL in async thread, rank A can block waiting for SMs, while rank B can
|
||||
# block inside NCCL waiting for rank A to participate in the collective.
|
||||
# This circular wait causes a deadlock.
|
||||
# Limiting NCCL occupancy via NCCL_MAX_CTAS leaves space for the DeepEP
|
||||
# cooperative kernel to launch and complete, breaking the deadlock.
|
||||
# See: https://github.com/deepseek-ai/DeepEP/issues/496
|
||||
#
|
||||
# DeepGEMM Mega MoE also uses cooperative launch and will cause hang even
|
||||
# with sync EPLB.
|
||||
# DeepGEMM Mega MoE uses cooperative launch, which tries to reserve a
|
||||
# large fraction of the GPU's SMs. If those SMs are occupied by NCCL,
|
||||
# the cooperative launch blocks until enough SMs are freed, causing a
|
||||
# deadlock. Limiting NCCL occupancy via NCCL_MAX_CTAS leaves space for
|
||||
# the cooperative kernel to launch and complete.
|
||||
if (
|
||||
is_data_parallel
|
||||
and is_eplb_enabled
|
||||
and is_nccl_based_eplb_communicator
|
||||
and ((is_deepep_ll and async_eplb) or is_mega_moe)
|
||||
and is_mega_moe
|
||||
):
|
||||
current_value_str = os.getenv("NCCL_MAX_CTAS")
|
||||
|
||||
@@ -116,10 +101,9 @@ def override_envs_for_eplb(
|
||||
|
||||
override_value = 8
|
||||
os.environ["NCCL_MAX_CTAS"] = str(override_value)
|
||||
backend = "deepep_low_latency" if is_deepep_ll else "deep_gemm_mega_moe"
|
||||
logger.info_once(
|
||||
f"EPLB: Setting NCCL_MAX_CTAS={override_value} "
|
||||
f"for expert parallel with NCCL-based EPLB communicator and "
|
||||
f"cooperative MoE backend ({backend})",
|
||||
f"cooperative MoE backend (deep_gemm_mega_moe)",
|
||||
scope="global",
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user