From 3766a67a32f7ea8130d84383f72883cee5e8b006 Mon Sep 17 00:00:00 2001 From: Misha Goin Date: Fri, 9 Oct 2026 15:34:58 -0400 Subject: [PATCH] [WideEP] Explain how to fix a too-old NCCL for DeepEP v2 (#60449) Signed-off-by: mgoin Co-authored-by: Claude --- tools/ep_kernels/README.md | 28 ++++++++----- tools/ep_kernels/install_python_libraries.sh | 26 ++++++++++-- .../device_communicators/all2all.py | 6 +-- vllm/utils/import_utils.py | 41 ++++++++++++------- 4 files changed, 70 insertions(+), 31 deletions(-) diff --git a/tools/ep_kernels/README.md b/tools/ep_kernels/README.md index 04bc3ef276..286a6db7bc 100644 --- a/tools/ep_kernels/README.md +++ b/tools/ep_kernels/README.md @@ -11,12 +11,16 @@ Step 2 is necessary for multi-node deployment. All scripts accept a positional argument as workspace path for staging the build, defaulting to `$(pwd)/ep_kernels_workspace`. -## NCCL version requirement (CUDA 13+) +## NCCL version requirement for DeepEPv2 -DeepEPv2 uses the NCCL GIN (GPU-Initiated Networking) backend, which requires -NCCL >= 2.30.4 at both compile time and runtime. PyTorch 2.11 pins -`nvidia-nccl-cu13==2.28.9` as a transitive dependency, so you need to -override it. +DeepEPv2 (`--all2all-backend deepep_v2`) uses the NCCL GIN (GPU-Initiated +Networking) backend, which requires NCCL >= 2.30.4 both when DeepEP is built +and at runtime. PyTorch pins an older release as a dependency (PyTorch 2.13 +pins `nvidia-nccl-cu13==2.29.7`), so a plain `pip install` or `uv pip install` +of vLLM is not enough. The vLLM Docker images already override it. + +Upgrade NCCL before running `install_python_libraries.sh`. Use +`nvidia-nccl-cu12` on CUDA 12. **With uv** (recommended): @@ -29,18 +33,22 @@ export UV_OVERRIDE=/tmp/nccl-override.txt uv pip install vllm ``` -**With pip**: +Keep `UV_OVERRIDE` set for later installs into the same environment. Without +it, any `uv pip install` that resolves PyTorch again restores the pinned NCCL. + +**With pip**, or to fix an existing environment: ```bash -pip install vllm pip install "nvidia-nccl-cu13>=2.30.4" --no-deps ``` -The override / reinstall must happen before building DeepEP (for GIN device -headers) and must remain in place at runtime. You can verify with: +`install_python_libraries.sh` warns when the installed NCCL is too old. If +DeepEP was built against an older NCCL, upgrade NCCL and run the script again. +When NCCL is too old at runtime, vLLM logs the reason at startup and +`deepep_v2` fails with the same message. You can check with: ```bash -python -c "from vllm.utils.import_utils import has_deep_ep_v2; print(has_deep_ep_v2())" +python -c "from vllm.utils.import_utils import deep_ep_v2_unavailable_reason; print(deep_ep_v2_unavailable_reason() or 'deepep_v2 is available')" ``` ## Usage diff --git a/tools/ep_kernels/install_python_libraries.sh b/tools/ep_kernels/install_python_libraries.sh index 9ca6a8d6a9..fdd7aa20e9 100755 --- a/tools/ep_kernels/install_python_libraries.sh +++ b/tools/ep_kernels/install_python_libraries.sh @@ -78,10 +78,25 @@ mkdir -p "$WHEEL_DIR" pushd "$WORKSPACE" # install dependencies if not installed +UV_PIP_TARGET=() if [ -z "$VIRTUAL_ENV" ]; then - uv pip install --system cmake torch ninja -else - uv pip install cmake torch ninja + UV_PIP_TARGET=(--system) +fi +uv pip install "${UV_PIP_TARGET[@]}" cmake torch ninja + +# DeepEPv2 needs NCCL >= 2.30.4 at both build time and runtime, but PyTorch +# pins an older release. +DEEPEP_V2_MIN_NCCL="2.30.4" +NCCL_PACKAGE="nvidia-nccl-cu${CUDA_VERSION_MAJOR}" +NCCL_WARNING="" +NCCL_INSTALLED=$(uv pip show "${UV_PIP_TARGET[@]}" "$NCCL_PACKAGE" 2>/dev/null | sed -n 's/^Version: //p') +if [ -n "$NCCL_INSTALLED" ] && \ + [ "$(printf '%s\n' "$DEEPEP_V2_MIN_NCCL" "$NCCL_INSTALLED" | sort -V | head -n1)" != "$DEEPEP_V2_MIN_NCCL" ]; then + NCCL_WARNING="WARNING: ${NCCL_PACKAGE} ${NCCL_INSTALLED} is installed, but the deepep_v2 backend +requires NCCL >= ${DEEPEP_V2_MIN_NCCL} when DeepEP is built and when it runs. To use deepep_v2, run + uv pip install \"${NCCL_PACKAGE}>=${DEEPEP_V2_MIN_NCCL}\" --no-deps +and then run this script again. See tools/ep_kernels/README.md." + echo "$NCCL_WARNING" >&2 fi # fetch nvshmem @@ -236,3 +251,8 @@ if [ "$MODE" = "wheel" ]; then echo "All wheels written to $WHEEL_DIR" ls -l "$WHEEL_DIR" fi + +# Repeat after the verbose build output, where the first copy is easy to miss. +if [ -n "$NCCL_WARNING" ]; then + echo "$NCCL_WARNING" >&2 +fi diff --git a/vllm/distributed/device_communicators/all2all.py b/vllm/distributed/device_communicators/all2all.py index ae31c0b1d3..cff34139f9 100644 --- a/vllm/distributed/device_communicators/all2all.py +++ b/vllm/distributed/device_communicators/all2all.py @@ -23,6 +23,7 @@ from vllm.utils.flashinfer import ( from vllm.utils.func_utils import supports_kw from vllm.utils.import_utils import ( check_moonep_system_support, + deep_ep_v2_unavailable_reason, has_deep_ep, has_deep_ep_v2, has_mori, @@ -1104,10 +1105,7 @@ class DeepEPV2All2AllManager(All2AllManagerBase): """ def __init__(self, cpu_group, tcp_store_group=None, device_group=None): - assert has_deep_ep_v2(), ( - "DeepEP v2 (ElasticBuffer) not available. Requires DeepEP >= 2.0 " - "(https://github.com/deepseek-ai/DeepEP) and NCCL >= 2.30.4." - ) + assert has_deep_ep_v2(), deep_ep_v2_unavailable_reason() super().__init__(cpu_group, tcp_store_group) self._device_group = device_group self.handle_cache = Cache() diff --git a/vllm/utils/import_utils.py b/vllm/utils/import_utils.py index 04a7ebec2e..277f987d48 100644 --- a/vllm/utils/import_utils.py +++ b/vllm/utils/import_utils.py @@ -469,31 +469,44 @@ def _format_nccl_raw_version(raw: int) -> str: return f"{s[0]}.{s[1:3].lstrip('0') or '0'}.{s[3:].lstrip('0') or '0'}" -def has_deep_ep_v2() -> bool: - """Whether deep_ep with ElasticBuffer (v2 API) is available. +def deep_ep_v2_unavailable_reason() -> str | None: + """Why the deep_ep v2 (ElasticBuffer) API cannot be used, or None if it can. Requires both the ElasticBuffer class in the deep_ep module and NCCL >= 2.30.4 (GIN backend), checked against the runtime library. """ + install_doc = "See tools/ep_kernels/README.md." if not _has_module("deep_ep"): - return False + return f"DeepEP is not installed. {install_doc}" import deep_ep # type: ignore[import-not-found] if not hasattr(deep_ep, "ElasticBuffer"): - return False + return f"The installed DeepEP predates v2 (no ElasticBuffer). {install_doc}" + nccl_ver = _get_runtime_nccl_version() + if nccl_ver is None or nccl_ver < DEEPEP_V2_MIN_NCCL_VERSION_RAW: + import torch + + min_version = _format_nccl_raw_version(DEEPEP_V2_MIN_NCCL_VERSION_RAW) + cuda_major = (torch.version.cuda or "13").split(".")[0] + return ( + f"DeepEP v2 requires NCCL >= {min_version} but found " + f"{_format_nccl_raw_version(nccl_ver) if nccl_ver else 'unknown'}. " + "PyTorch pins an older NCCL; upgrade it with " + f'`pip install "nvidia-nccl-cu{cuda_major}>={min_version}" --no-deps` ' + f"and rebuild DeepEP against it. {install_doc}" + ) + return None + + +def has_deep_ep_v2() -> bool: + """Whether deep_ep with ElasticBuffer (v2 API) is available.""" try: - nccl_ver = _get_runtime_nccl_version() - if nccl_ver is None or nccl_ver < DEEPEP_V2_MIN_NCCL_VERSION_RAW: - logger.info_once( - "DeepEP v2 requires NCCL >= %s but found %s. " - "deepep_v2 backend will not be available.", - _format_nccl_raw_version(DEEPEP_V2_MIN_NCCL_VERSION_RAW), - _format_nccl_raw_version(nccl_ver) if nccl_ver else "unknown", - ) - return False + reason = deep_ep_v2_unavailable_reason() except Exception: return False - return True + if reason is not None and _has_module("deep_ep"): + logger.info_once("deepep_v2 backend will not be available: %s", reason) + return reason is None def has_deep_gemm() -> bool: