[WideEP] Explain how to fix a too-old NCCL for DeepEP v2 (#60449)

Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Misha Goin
2026-10-09 12:34:58 -07:00
committed by GitHub
co-authored by Claude
parent 04462732a4
commit 3766a67a32
4 changed files with 70 additions and 31 deletions
+18 -10
View File
@@ -11,12 +11,16 @@ Step 2 is necessary for multi-node deployment.
All scripts accept a positional argument as workspace path for staging the build, defaulting to `$(pwd)/ep_kernels_workspace`. All scripts accept a positional argument as workspace path for staging the build, defaulting to `$(pwd)/ep_kernels_workspace`.
## NCCL version requirement (CUDA 13+) ## NCCL version requirement for DeepEPv2
DeepEPv2 uses the NCCL GIN (GPU-Initiated Networking) backend, which requires DeepEPv2 (`--all2all-backend deepep_v2`) uses the NCCL GIN (GPU-Initiated
NCCL >= 2.30.4 at both compile time and runtime. PyTorch 2.11 pins Networking) backend, which requires NCCL >= 2.30.4 both when DeepEP is built
`nvidia-nccl-cu13==2.28.9` as a transitive dependency, so you need to and at runtime. PyTorch pins an older release as a dependency (PyTorch 2.13
override it. pins `nvidia-nccl-cu13==2.29.7`), so a plain `pip install` or `uv pip install`
of vLLM is not enough. The vLLM Docker images already override it.
Upgrade NCCL before running `install_python_libraries.sh`. Use
`nvidia-nccl-cu12` on CUDA 12.
**With uv** (recommended): **With uv** (recommended):
@@ -29,18 +33,22 @@ export UV_OVERRIDE=/tmp/nccl-override.txt
uv pip install vllm uv pip install vllm
``` ```
**With pip**: Keep `UV_OVERRIDE` set for later installs into the same environment. Without
it, any `uv pip install` that resolves PyTorch again restores the pinned NCCL.
**With pip**, or to fix an existing environment:
```bash ```bash
pip install vllm
pip install "nvidia-nccl-cu13>=2.30.4" --no-deps pip install "nvidia-nccl-cu13>=2.30.4" --no-deps
``` ```
The override / reinstall must happen before building DeepEP (for GIN device `install_python_libraries.sh` warns when the installed NCCL is too old. If
headers) and must remain in place at runtime. You can verify with: DeepEP was built against an older NCCL, upgrade NCCL and run the script again.
When NCCL is too old at runtime, vLLM logs the reason at startup and
`deepep_v2` fails with the same message. You can check with:
```bash ```bash
python -c "from vllm.utils.import_utils import has_deep_ep_v2; print(has_deep_ep_v2())" python -c "from vllm.utils.import_utils import deep_ep_v2_unavailable_reason; print(deep_ep_v2_unavailable_reason() or 'deepep_v2 is available')"
``` ```
## Usage ## Usage
+23 -3
View File
@@ -78,10 +78,25 @@ mkdir -p "$WHEEL_DIR"
pushd "$WORKSPACE" pushd "$WORKSPACE"
# install dependencies if not installed # install dependencies if not installed
UV_PIP_TARGET=()
if [ -z "$VIRTUAL_ENV" ]; then if [ -z "$VIRTUAL_ENV" ]; then
uv pip install --system cmake torch ninja UV_PIP_TARGET=(--system)
else fi
uv pip install cmake torch ninja uv pip install "${UV_PIP_TARGET[@]}" cmake torch ninja
# DeepEPv2 needs NCCL >= 2.30.4 at both build time and runtime, but PyTorch
# pins an older release.
DEEPEP_V2_MIN_NCCL="2.30.4"
NCCL_PACKAGE="nvidia-nccl-cu${CUDA_VERSION_MAJOR}"
NCCL_WARNING=""
NCCL_INSTALLED=$(uv pip show "${UV_PIP_TARGET[@]}" "$NCCL_PACKAGE" 2>/dev/null | sed -n 's/^Version: //p')
if [ -n "$NCCL_INSTALLED" ] && \
[ "$(printf '%s\n' "$DEEPEP_V2_MIN_NCCL" "$NCCL_INSTALLED" | sort -V | head -n1)" != "$DEEPEP_V2_MIN_NCCL" ]; then
NCCL_WARNING="WARNING: ${NCCL_PACKAGE} ${NCCL_INSTALLED} is installed, but the deepep_v2 backend
requires NCCL >= ${DEEPEP_V2_MIN_NCCL} when DeepEP is built and when it runs. To use deepep_v2, run
uv pip install \"${NCCL_PACKAGE}>=${DEEPEP_V2_MIN_NCCL}\" --no-deps
and then run this script again. See tools/ep_kernels/README.md."
echo "$NCCL_WARNING" >&2
fi fi
# fetch nvshmem # fetch nvshmem
@@ -236,3 +251,8 @@ if [ "$MODE" = "wheel" ]; then
echo "All wheels written to $WHEEL_DIR" echo "All wheels written to $WHEEL_DIR"
ls -l "$WHEEL_DIR" ls -l "$WHEEL_DIR"
fi fi
# Repeat after the verbose build output, where the first copy is easy to miss.
if [ -n "$NCCL_WARNING" ]; then
echo "$NCCL_WARNING" >&2
fi
@@ -23,6 +23,7 @@ from vllm.utils.flashinfer import (
from vllm.utils.func_utils import supports_kw from vllm.utils.func_utils import supports_kw
from vllm.utils.import_utils import ( from vllm.utils.import_utils import (
check_moonep_system_support, check_moonep_system_support,
deep_ep_v2_unavailable_reason,
has_deep_ep, has_deep_ep,
has_deep_ep_v2, has_deep_ep_v2,
has_mori, has_mori,
@@ -1104,10 +1105,7 @@ class DeepEPV2All2AllManager(All2AllManagerBase):
""" """
def __init__(self, cpu_group, tcp_store_group=None, device_group=None): def __init__(self, cpu_group, tcp_store_group=None, device_group=None):
assert has_deep_ep_v2(), ( assert has_deep_ep_v2(), deep_ep_v2_unavailable_reason()
"DeepEP v2 (ElasticBuffer) not available. Requires DeepEP >= 2.0 "
"(https://github.com/deepseek-ai/DeepEP) and NCCL >= 2.30.4."
)
super().__init__(cpu_group, tcp_store_group) super().__init__(cpu_group, tcp_store_group)
self._device_group = device_group self._device_group = device_group
self.handle_cache = Cache() self.handle_cache = Cache()
+27 -14
View File
@@ -469,31 +469,44 @@ def _format_nccl_raw_version(raw: int) -> str:
return f"{s[0]}.{s[1:3].lstrip('0') or '0'}.{s[3:].lstrip('0') or '0'}" return f"{s[0]}.{s[1:3].lstrip('0') or '0'}.{s[3:].lstrip('0') or '0'}"
def has_deep_ep_v2() -> bool: def deep_ep_v2_unavailable_reason() -> str | None:
"""Whether deep_ep with ElasticBuffer (v2 API) is available. """Why the deep_ep v2 (ElasticBuffer) API cannot be used, or None if it can.
Requires both the ElasticBuffer class in the deep_ep module and Requires both the ElasticBuffer class in the deep_ep module and
NCCL >= 2.30.4 (GIN backend), checked against the runtime library. NCCL >= 2.30.4 (GIN backend), checked against the runtime library.
""" """
install_doc = "See tools/ep_kernels/README.md."
if not _has_module("deep_ep"): if not _has_module("deep_ep"):
return False return f"DeepEP is not installed. {install_doc}"
import deep_ep # type: ignore[import-not-found] import deep_ep # type: ignore[import-not-found]
if not hasattr(deep_ep, "ElasticBuffer"): if not hasattr(deep_ep, "ElasticBuffer"):
return False return f"The installed DeepEP predates v2 (no ElasticBuffer). {install_doc}"
nccl_ver = _get_runtime_nccl_version()
if nccl_ver is None or nccl_ver < DEEPEP_V2_MIN_NCCL_VERSION_RAW:
import torch
min_version = _format_nccl_raw_version(DEEPEP_V2_MIN_NCCL_VERSION_RAW)
cuda_major = (torch.version.cuda or "13").split(".")[0]
return (
f"DeepEP v2 requires NCCL >= {min_version} but found "
f"{_format_nccl_raw_version(nccl_ver) if nccl_ver else 'unknown'}. "
"PyTorch pins an older NCCL; upgrade it with "
f'`pip install "nvidia-nccl-cu{cuda_major}>={min_version}" --no-deps` '
f"and rebuild DeepEP against it. {install_doc}"
)
return None
def has_deep_ep_v2() -> bool:
"""Whether deep_ep with ElasticBuffer (v2 API) is available."""
try: try:
nccl_ver = _get_runtime_nccl_version() reason = deep_ep_v2_unavailable_reason()
if nccl_ver is None or nccl_ver < DEEPEP_V2_MIN_NCCL_VERSION_RAW:
logger.info_once(
"DeepEP v2 requires NCCL >= %s but found %s. "
"deepep_v2 backend will not be available.",
_format_nccl_raw_version(DEEPEP_V2_MIN_NCCL_VERSION_RAW),
_format_nccl_raw_version(nccl_ver) if nccl_ver else "unknown",
)
return False
except Exception: except Exception:
return False return False
return True if reason is not None and _has_module("deep_ep"):
logger.info_once("deepep_v2 backend will not be available: %s", reason)
return reason is None
def has_deep_gemm() -> bool: def has_deep_gemm() -> bool: