[WideEP] Explain how to fix a too-old NCCL for DeepEP v2 (#60449)
Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
+18
-10
@@ -11,12 +11,16 @@ Step 2 is necessary for multi-node deployment.
|
|||||||
|
|
||||||
All scripts accept a positional argument as workspace path for staging the build, defaulting to `$(pwd)/ep_kernels_workspace`.
|
All scripts accept a positional argument as workspace path for staging the build, defaulting to `$(pwd)/ep_kernels_workspace`.
|
||||||
|
|
||||||
## NCCL version requirement (CUDA 13+)
|
## NCCL version requirement for DeepEPv2
|
||||||
|
|
||||||
DeepEPv2 uses the NCCL GIN (GPU-Initiated Networking) backend, which requires
|
DeepEPv2 (`--all2all-backend deepep_v2`) uses the NCCL GIN (GPU-Initiated
|
||||||
NCCL >= 2.30.4 at both compile time and runtime. PyTorch 2.11 pins
|
Networking) backend, which requires NCCL >= 2.30.4 both when DeepEP is built
|
||||||
`nvidia-nccl-cu13==2.28.9` as a transitive dependency, so you need to
|
and at runtime. PyTorch pins an older release as a dependency (PyTorch 2.13
|
||||||
override it.
|
pins `nvidia-nccl-cu13==2.29.7`), so a plain `pip install` or `uv pip install`
|
||||||
|
of vLLM is not enough. The vLLM Docker images already override it.
|
||||||
|
|
||||||
|
Upgrade NCCL before running `install_python_libraries.sh`. Use
|
||||||
|
`nvidia-nccl-cu12` on CUDA 12.
|
||||||
|
|
||||||
**With uv** (recommended):
|
**With uv** (recommended):
|
||||||
|
|
||||||
@@ -29,18 +33,22 @@ export UV_OVERRIDE=/tmp/nccl-override.txt
|
|||||||
uv pip install vllm
|
uv pip install vllm
|
||||||
```
|
```
|
||||||
|
|
||||||
**With pip**:
|
Keep `UV_OVERRIDE` set for later installs into the same environment. Without
|
||||||
|
it, any `uv pip install` that resolves PyTorch again restores the pinned NCCL.
|
||||||
|
|
||||||
|
**With pip**, or to fix an existing environment:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
pip install vllm
|
|
||||||
pip install "nvidia-nccl-cu13>=2.30.4" --no-deps
|
pip install "nvidia-nccl-cu13>=2.30.4" --no-deps
|
||||||
```
|
```
|
||||||
|
|
||||||
The override / reinstall must happen before building DeepEP (for GIN device
|
`install_python_libraries.sh` warns when the installed NCCL is too old. If
|
||||||
headers) and must remain in place at runtime. You can verify with:
|
DeepEP was built against an older NCCL, upgrade NCCL and run the script again.
|
||||||
|
When NCCL is too old at runtime, vLLM logs the reason at startup and
|
||||||
|
`deepep_v2` fails with the same message. You can check with:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
python -c "from vllm.utils.import_utils import has_deep_ep_v2; print(has_deep_ep_v2())"
|
python -c "from vllm.utils.import_utils import deep_ep_v2_unavailable_reason; print(deep_ep_v2_unavailable_reason() or 'deepep_v2 is available')"
|
||||||
```
|
```
|
||||||
|
|
||||||
## Usage
|
## Usage
|
||||||
|
|||||||
@@ -78,10 +78,25 @@ mkdir -p "$WHEEL_DIR"
|
|||||||
pushd "$WORKSPACE"
|
pushd "$WORKSPACE"
|
||||||
|
|
||||||
# install dependencies if not installed
|
# install dependencies if not installed
|
||||||
|
UV_PIP_TARGET=()
|
||||||
if [ -z "$VIRTUAL_ENV" ]; then
|
if [ -z "$VIRTUAL_ENV" ]; then
|
||||||
uv pip install --system cmake torch ninja
|
UV_PIP_TARGET=(--system)
|
||||||
else
|
fi
|
||||||
uv pip install cmake torch ninja
|
uv pip install "${UV_PIP_TARGET[@]}" cmake torch ninja
|
||||||
|
|
||||||
|
# DeepEPv2 needs NCCL >= 2.30.4 at both build time and runtime, but PyTorch
|
||||||
|
# pins an older release.
|
||||||
|
DEEPEP_V2_MIN_NCCL="2.30.4"
|
||||||
|
NCCL_PACKAGE="nvidia-nccl-cu${CUDA_VERSION_MAJOR}"
|
||||||
|
NCCL_WARNING=""
|
||||||
|
NCCL_INSTALLED=$(uv pip show "${UV_PIP_TARGET[@]}" "$NCCL_PACKAGE" 2>/dev/null | sed -n 's/^Version: //p')
|
||||||
|
if [ -n "$NCCL_INSTALLED" ] && \
|
||||||
|
[ "$(printf '%s\n' "$DEEPEP_V2_MIN_NCCL" "$NCCL_INSTALLED" | sort -V | head -n1)" != "$DEEPEP_V2_MIN_NCCL" ]; then
|
||||||
|
NCCL_WARNING="WARNING: ${NCCL_PACKAGE} ${NCCL_INSTALLED} is installed, but the deepep_v2 backend
|
||||||
|
requires NCCL >= ${DEEPEP_V2_MIN_NCCL} when DeepEP is built and when it runs. To use deepep_v2, run
|
||||||
|
uv pip install \"${NCCL_PACKAGE}>=${DEEPEP_V2_MIN_NCCL}\" --no-deps
|
||||||
|
and then run this script again. See tools/ep_kernels/README.md."
|
||||||
|
echo "$NCCL_WARNING" >&2
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# fetch nvshmem
|
# fetch nvshmem
|
||||||
@@ -236,3 +251,8 @@ if [ "$MODE" = "wheel" ]; then
|
|||||||
echo "All wheels written to $WHEEL_DIR"
|
echo "All wheels written to $WHEEL_DIR"
|
||||||
ls -l "$WHEEL_DIR"
|
ls -l "$WHEEL_DIR"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# Repeat after the verbose build output, where the first copy is easy to miss.
|
||||||
|
if [ -n "$NCCL_WARNING" ]; then
|
||||||
|
echo "$NCCL_WARNING" >&2
|
||||||
|
fi
|
||||||
|
|||||||
@@ -23,6 +23,7 @@ from vllm.utils.flashinfer import (
|
|||||||
from vllm.utils.func_utils import supports_kw
|
from vllm.utils.func_utils import supports_kw
|
||||||
from vllm.utils.import_utils import (
|
from vllm.utils.import_utils import (
|
||||||
check_moonep_system_support,
|
check_moonep_system_support,
|
||||||
|
deep_ep_v2_unavailable_reason,
|
||||||
has_deep_ep,
|
has_deep_ep,
|
||||||
has_deep_ep_v2,
|
has_deep_ep_v2,
|
||||||
has_mori,
|
has_mori,
|
||||||
@@ -1104,10 +1105,7 @@ class DeepEPV2All2AllManager(All2AllManagerBase):
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, cpu_group, tcp_store_group=None, device_group=None):
|
def __init__(self, cpu_group, tcp_store_group=None, device_group=None):
|
||||||
assert has_deep_ep_v2(), (
|
assert has_deep_ep_v2(), deep_ep_v2_unavailable_reason()
|
||||||
"DeepEP v2 (ElasticBuffer) not available. Requires DeepEP >= 2.0 "
|
|
||||||
"(https://github.com/deepseek-ai/DeepEP) and NCCL >= 2.30.4."
|
|
||||||
)
|
|
||||||
super().__init__(cpu_group, tcp_store_group)
|
super().__init__(cpu_group, tcp_store_group)
|
||||||
self._device_group = device_group
|
self._device_group = device_group
|
||||||
self.handle_cache = Cache()
|
self.handle_cache = Cache()
|
||||||
|
|||||||
+27
-14
@@ -469,31 +469,44 @@ def _format_nccl_raw_version(raw: int) -> str:
|
|||||||
return f"{s[0]}.{s[1:3].lstrip('0') or '0'}.{s[3:].lstrip('0') or '0'}"
|
return f"{s[0]}.{s[1:3].lstrip('0') or '0'}.{s[3:].lstrip('0') or '0'}"
|
||||||
|
|
||||||
|
|
||||||
def has_deep_ep_v2() -> bool:
|
def deep_ep_v2_unavailable_reason() -> str | None:
|
||||||
"""Whether deep_ep with ElasticBuffer (v2 API) is available.
|
"""Why the deep_ep v2 (ElasticBuffer) API cannot be used, or None if it can.
|
||||||
|
|
||||||
Requires both the ElasticBuffer class in the deep_ep module and
|
Requires both the ElasticBuffer class in the deep_ep module and
|
||||||
NCCL >= 2.30.4 (GIN backend), checked against the runtime library.
|
NCCL >= 2.30.4 (GIN backend), checked against the runtime library.
|
||||||
"""
|
"""
|
||||||
|
install_doc = "See tools/ep_kernels/README.md."
|
||||||
if not _has_module("deep_ep"):
|
if not _has_module("deep_ep"):
|
||||||
return False
|
return f"DeepEP is not installed. {install_doc}"
|
||||||
import deep_ep # type: ignore[import-not-found]
|
import deep_ep # type: ignore[import-not-found]
|
||||||
|
|
||||||
if not hasattr(deep_ep, "ElasticBuffer"):
|
if not hasattr(deep_ep, "ElasticBuffer"):
|
||||||
return False
|
return f"The installed DeepEP predates v2 (no ElasticBuffer). {install_doc}"
|
||||||
|
nccl_ver = _get_runtime_nccl_version()
|
||||||
|
if nccl_ver is None or nccl_ver < DEEPEP_V2_MIN_NCCL_VERSION_RAW:
|
||||||
|
import torch
|
||||||
|
|
||||||
|
min_version = _format_nccl_raw_version(DEEPEP_V2_MIN_NCCL_VERSION_RAW)
|
||||||
|
cuda_major = (torch.version.cuda or "13").split(".")[0]
|
||||||
|
return (
|
||||||
|
f"DeepEP v2 requires NCCL >= {min_version} but found "
|
||||||
|
f"{_format_nccl_raw_version(nccl_ver) if nccl_ver else 'unknown'}. "
|
||||||
|
"PyTorch pins an older NCCL; upgrade it with "
|
||||||
|
f'`pip install "nvidia-nccl-cu{cuda_major}>={min_version}" --no-deps` '
|
||||||
|
f"and rebuild DeepEP against it. {install_doc}"
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def has_deep_ep_v2() -> bool:
|
||||||
|
"""Whether deep_ep with ElasticBuffer (v2 API) is available."""
|
||||||
try:
|
try:
|
||||||
nccl_ver = _get_runtime_nccl_version()
|
reason = deep_ep_v2_unavailable_reason()
|
||||||
if nccl_ver is None or nccl_ver < DEEPEP_V2_MIN_NCCL_VERSION_RAW:
|
|
||||||
logger.info_once(
|
|
||||||
"DeepEP v2 requires NCCL >= %s but found %s. "
|
|
||||||
"deepep_v2 backend will not be available.",
|
|
||||||
_format_nccl_raw_version(DEEPEP_V2_MIN_NCCL_VERSION_RAW),
|
|
||||||
_format_nccl_raw_version(nccl_ver) if nccl_ver else "unknown",
|
|
||||||
)
|
|
||||||
return False
|
|
||||||
except Exception:
|
except Exception:
|
||||||
return False
|
return False
|
||||||
return True
|
if reason is not None and _has_module("deep_ep"):
|
||||||
|
logger.info_once("deepep_v2 backend will not be available: %s", reason)
|
||||||
|
return reason is None
|
||||||
|
|
||||||
|
|
||||||
def has_deep_gemm() -> bool:
|
def has_deep_gemm() -> bool:
|
||||||
|
|||||||
Reference in New Issue
Block a user