[EPLB] Reject NCCL-based EPLB communicators with async EPLB (#44978)

Signed-off-by: Markov Ilya <markovilya197@gmail.com>
This commit is contained in:
Ilya Markov
2026-06-10 19:51:27 +00:00
committed by GitHub
parent 3d300aecb1
commit 6471ec75bd
8 changed files with 37 additions and 32 deletions
@@ -470,6 +470,9 @@ class ElasticEPScalingExecutor:
eplb_model_state.expert_buffer = [
torch.empty_like(w) for w in model.expert_weights[0]
]
assert parallel_config.eplb_config.communicator is not None, (
"EPLB communicator backend must be set by ParallelConfig"
)
eplb_model_state.communicator = create_eplb_communicator(
group_coordinator=get_eplb_group(),
backend=parallel_config.eplb_config.communicator,
+1 -5
View File
@@ -617,7 +617,7 @@ class PyNcclEplbCommunicator(EplbCommunicator):
def create_eplb_communicator(
group_coordinator: GroupCoordinator,
backend: str | None,
backend: str,
expert_weights: Sequence[Sequence[torch.Tensor]],
expert_buffer: Sequence[torch.Tensor],
) -> EplbCommunicator:
@@ -628,7 +628,6 @@ def create_eplb_communicator(
device and CPU communication groups.
backend: Communicator backend name (``"torch_nccl"``,
``"torch_gloo"``, ``"pynccl"``, or ``"nixl"``).
Falls back to ``"torch_nccl"`` when *None*.
Stateless (elastic EP) groups only support ``"torch_nccl"``
and ``"pynccl"``; ``"torch_nccl"`` is silently promoted to
``"pynccl"`` in that case. When tensors reside on CPU,
@@ -641,9 +640,6 @@ def create_eplb_communicator(
expert_buffer: Pre-allocated receive buffer tensors (one per
weight tensor in a single layer).
"""
if backend is None:
backend = "torch_nccl"
first_layer = expert_weights[0] if expert_weights else []
tensor_device_type = first_layer[0].device.type if first_layer else "cpu"
torch_group = (
+3
View File
@@ -447,6 +447,9 @@ class EplbState:
self._init_should_record_tensor(model)
expert_buffer = [torch.empty_like(w) for w in model.expert_weights[0]]
assert self.parallel_config.eplb_config.communicator is not None, (
"EPLB communicator backend must be set by ParallelConfig"
)
communicator = create_eplb_communicator(
group_coordinator=get_eplb_group(),
backend=self.parallel_config.eplb_config.communicator,
+7 -23
View File
@@ -74,8 +74,6 @@ def override_envs_for_eplb(
"""
is_data_parallel = parallel_config.data_parallel_size > 1
is_eplb_enabled = parallel_config.enable_eplb
async_eplb = parallel_config.eplb_config.use_async
is_deepep_ll = parallel_config.all2all_backend == "deepep_low_latency"
is_mega_moe = moe_backend == "deep_gemm_mega_moe"
is_nccl_based_eplb_communicator = parallel_config.eplb_config.communicator in (
"torch_nccl",
@@ -85,29 +83,16 @@ def override_envs_for_eplb(
# Override NCCL_MAX_CTAS to avoid hangs when EPLB's NCCL weight exchange
# contends with MoE backend's cooperative-launch on GPU SMs.
#
# DeepEP low-latency:
# The hang happens when two ranks interleave kernel launches differently
# between NCCL collectives (used by async EPLB weight exchange) and DeepEP
# low-latency (LL) kernels. DeepEP LL uses a cooperative launch and tries
# to reserve a large fraction of the GPU's SMs; if those SMs are currently
# occupied by NCCL, the DeepEP LL launch blocks until enough SMs are
# freed.
#
# If rank A enters DeepEP LL in main thread while rank B is still executing
# NCCL in async thread, rank A can block waiting for SMs, while rank B can
# block inside NCCL waiting for rank A to participate in the collective.
# This circular wait causes a deadlock.
# Limiting NCCL occupancy via NCCL_MAX_CTAS leaves space for the DeepEP
# cooperative kernel to launch and complete, breaking the deadlock.
# See: https://github.com/deepseek-ai/DeepEP/issues/496
#
# DeepGEMM Mega MoE also uses cooperative launch and will cause hang even
# with sync EPLB.
# DeepGEMM Mega MoE uses cooperative launch, which tries to reserve a
# large fraction of the GPU's SMs. If those SMs are occupied by NCCL,
# the cooperative launch blocks until enough SMs are freed, causing a
# deadlock. Limiting NCCL occupancy via NCCL_MAX_CTAS leaves space for
# the cooperative kernel to launch and complete.
if (
is_data_parallel
and is_eplb_enabled
and is_nccl_based_eplb_communicator
and ((is_deepep_ll and async_eplb) or is_mega_moe)
and is_mega_moe
):
current_value_str = os.getenv("NCCL_MAX_CTAS")
@@ -116,10 +101,9 @@ def override_envs_for_eplb(
override_value = 8
os.environ["NCCL_MAX_CTAS"] = str(override_value)
backend = "deepep_low_latency" if is_deepep_ll else "deep_gemm_mega_moe"
logger.info_once(
f"EPLB: Setting NCCL_MAX_CTAS={override_value} "
f"for expert parallel with NCCL-based EPLB communicator and "
f"cooperative MoE backend ({backend})",
f"cooperative MoE backend (deep_gemm_mega_moe)",
scope="global",
)