[WideEP] Explain how to fix a too-old NCCL for DeepEP v2 (#60449)

Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Claude <noreply@anthropic.com>

tools/ep_kernels/README.md

@@ -11,12 +11,16 @@ Step 2 is necessary for multi-node deployment.
 
 All scripts accept a positional argument as workspace path for staging the build, defaulting to `$(pwd)/ep_kernels_workspace`.
 
-## NCCL version requirement (CUDA 13+)
+## NCCL version requirement for DeepEPv2
 
-DeepEPv2 uses the NCCL GIN (GPU-Initiated Networking) backend, which requires
-NCCL >= 2.30.4 at both compile time and runtime. PyTorch 2.11 pins
-`nvidia-nccl-cu13==2.28.9` as a transitive dependency, so you need to
-override it.
+DeepEPv2 (`--all2all-backend deepep_v2`) uses the NCCL GIN (GPU-Initiated
+Networking) backend, which requires NCCL >= 2.30.4 both when DeepEP is built
+and at runtime. PyTorch pins an older release as a dependency (PyTorch 2.13
+pins `nvidia-nccl-cu13==2.29.7`), so a plain `pip install` or `uv pip install`
+of vLLM is not enough. The vLLM Docker images already override it.
+
+Upgrade NCCL before running `install_python_libraries.sh`. Use
+`nvidia-nccl-cu12` on CUDA 12.
 
 **With uv** (recommended):
 
@@ -29,18 +33,22 @@ export UV_OVERRIDE=/tmp/nccl-override.txt
 uv pip install vllm
 ```
 
-**With pip**:
+Keep `UV_OVERRIDE` set for later installs into the same environment. Without
+it, any `uv pip install` that resolves PyTorch again restores the pinned NCCL.
+
+**With pip**, or to fix an existing environment:
 
 ```bash
-pip install vllm
 pip install "nvidia-nccl-cu13>=2.30.4" --no-deps
 ```
 
-The override / reinstall must happen before building DeepEP (for GIN device
-headers) and must remain in place at runtime. You can verify with:
+`install_python_libraries.sh` warns when the installed NCCL is too old. If
+DeepEP was built against an older NCCL, upgrade NCCL and run the script again.
+When NCCL is too old at runtime, vLLM logs the reason at startup and
+`deepep_v2` fails with the same message. You can check with:
 
 ```bash
-python -c "from vllm.utils.import_utils import has_deep_ep_v2; print(has_deep_ep_v2())"
+python -c "from vllm.utils.import_utils import deep_ep_v2_unavailable_reason; print(deep_ep_v2_unavailable_reason() or 'deepep_v2 is available')"
 ```
 
 ## Usage

tools/ep_kernels/install_python_libraries.sh

@@ -78,10 +78,25 @@ mkdir -p "$WHEEL_DIR"
 pushd "$WORKSPACE"
 
 # install dependencies if not installed
+UV_PIP_TARGET=()
 if [ -z "$VIRTUAL_ENV" ]; then
-  uv pip install --system cmake torch ninja
-else
-  uv pip install cmake torch ninja
+  UV_PIP_TARGET=(--system)
+fi
+uv pip install "${UV_PIP_TARGET[@]}" cmake torch ninja
+
+# DeepEPv2 needs NCCL >= 2.30.4 at both build time and runtime, but PyTorch
+# pins an older release.
+DEEPEP_V2_MIN_NCCL="2.30.4"
+NCCL_PACKAGE="nvidia-nccl-cu${CUDA_VERSION_MAJOR}"
+NCCL_WARNING=""
+NCCL_INSTALLED=$(uv pip show "${UV_PIP_TARGET[@]}" "$NCCL_PACKAGE" 2>/dev/null | sed -n 's/^Version: //p')
+if [ -n "$NCCL_INSTALLED" ] && \
+    [ "$(printf '%s\n' "$DEEPEP_V2_MIN_NCCL" "$NCCL_INSTALLED" | sort -V | head -n1)" != "$DEEPEP_V2_MIN_NCCL" ]; then
+    NCCL_WARNING="WARNING: ${NCCL_PACKAGE} ${NCCL_INSTALLED} is installed, but the deepep_v2 backend
+requires NCCL >= ${DEEPEP_V2_MIN_NCCL} when DeepEP is built and when it runs. To use deepep_v2, run
+    uv pip install \"${NCCL_PACKAGE}>=${DEEPEP_V2_MIN_NCCL}\" --no-deps
+and then run this script again. See tools/ep_kernels/README.md."
+    echo "$NCCL_WARNING" >&2
 fi
 
 # fetch nvshmem
@@ -236,3 +251,8 @@ if [ "$MODE" = "wheel" ]; then
     echo "All wheels written to $WHEEL_DIR"
     ls -l "$WHEEL_DIR"
 fi
+
+# Repeat after the verbose build output, where the first copy is easy to miss.
+if [ -n "$NCCL_WARNING" ]; then
+    echo "$NCCL_WARNING" >&2
+fi

vllm/distributed/device_communicators/all2all.py

@@ -23,6 +23,7 @@ from vllm.utils.flashinfer import (
 from vllm.utils.func_utils import supports_kw
 from vllm.utils.import_utils import (
     check_moonep_system_support,
+    deep_ep_v2_unavailable_reason,
     has_deep_ep,
     has_deep_ep_v2,
     has_mori,
@@ -1104,10 +1105,7 @@ class DeepEPV2All2AllManager(All2AllManagerBase):
     """
 
     def __init__(self, cpu_group, tcp_store_group=None, device_group=None):
-        assert has_deep_ep_v2(), (
-            "DeepEP v2 (ElasticBuffer) not available. Requires DeepEP >= 2.0 "
-            "(https://github.com/deepseek-ai/DeepEP) and NCCL >= 2.30.4."
-        )
+        assert has_deep_ep_v2(), deep_ep_v2_unavailable_reason()
         super().__init__(cpu_group, tcp_store_group)
         self._device_group = device_group
         self.handle_cache = Cache()

vllm/utils/import_utils.py

@@ -469,31 +469,44 @@ def _format_nccl_raw_version(raw: int) -> str:
     return f"{s[0]}.{s[1:3].lstrip('0') or '0'}.{s[3:].lstrip('0') or '0'}"
 
 
-def has_deep_ep_v2() -> bool:
-    """Whether deep_ep with ElasticBuffer (v2 API) is available.
+def deep_ep_v2_unavailable_reason() -> str | None:
+    """Why the deep_ep v2 (ElasticBuffer) API cannot be used, or None if it can.
 
     Requires both the ElasticBuffer class in the deep_ep module and
     NCCL >= 2.30.4 (GIN backend), checked against the runtime library.
     """
+    install_doc = "See tools/ep_kernels/README.md."
     if not _has_module("deep_ep"):
-        return False
+        return f"DeepEP is not installed. {install_doc}"
     import deep_ep  # type: ignore[import-not-found]
 
     if not hasattr(deep_ep, "ElasticBuffer"):
-        return False
+        return f"The installed DeepEP predates v2 (no ElasticBuffer). {install_doc}"
+    nccl_ver = _get_runtime_nccl_version()
+    if nccl_ver is None or nccl_ver < DEEPEP_V2_MIN_NCCL_VERSION_RAW:
+        import torch
+
+        min_version = _format_nccl_raw_version(DEEPEP_V2_MIN_NCCL_VERSION_RAW)
+        cuda_major = (torch.version.cuda or "13").split(".")[0]
+        return (
+            f"DeepEP v2 requires NCCL >= {min_version} but found "
+            f"{_format_nccl_raw_version(nccl_ver) if nccl_ver else 'unknown'}. "
+            "PyTorch pins an older NCCL; upgrade it with "
+            f'`pip install "nvidia-nccl-cu{cuda_major}>={min_version}" --no-deps` '
+            f"and rebuild DeepEP against it. {install_doc}"
+        )
+    return None
+
+
+def has_deep_ep_v2() -> bool:
+    """Whether deep_ep with ElasticBuffer (v2 API) is available."""
     try:
-        nccl_ver = _get_runtime_nccl_version()
-        if nccl_ver is None or nccl_ver < DEEPEP_V2_MIN_NCCL_VERSION_RAW:
-            logger.info_once(
-                "DeepEP v2 requires NCCL >= %s but found %s. "
-                "deepep_v2 backend will not be available.",
-                _format_nccl_raw_version(DEEPEP_V2_MIN_NCCL_VERSION_RAW),
-                _format_nccl_raw_version(nccl_ver) if nccl_ver else "unknown",
-            )
-            return False
+        reason = deep_ep_v2_unavailable_reason()
     except Exception:
         return False
-    return True
+    if reason is not None and _has_module("deep_ep"):
+        logger.info_once("deepep_v2 backend will not be available: %s", reason)
+    return reason is None
 
 
 def has_deep_gemm() -> bool: