[WideEP] Explain how to fix a too-old NCCL for DeepEP v2 (#60449)
Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
+27
-14
@@ -469,31 +469,44 @@ def _format_nccl_raw_version(raw: int) -> str:
|
||||
return f"{s[0]}.{s[1:3].lstrip('0') or '0'}.{s[3:].lstrip('0') or '0'}"
|
||||
|
||||
|
||||
def has_deep_ep_v2() -> bool:
|
||||
"""Whether deep_ep with ElasticBuffer (v2 API) is available.
|
||||
def deep_ep_v2_unavailable_reason() -> str | None:
|
||||
"""Why the deep_ep v2 (ElasticBuffer) API cannot be used, or None if it can.
|
||||
|
||||
Requires both the ElasticBuffer class in the deep_ep module and
|
||||
NCCL >= 2.30.4 (GIN backend), checked against the runtime library.
|
||||
"""
|
||||
install_doc = "See tools/ep_kernels/README.md."
|
||||
if not _has_module("deep_ep"):
|
||||
return False
|
||||
return f"DeepEP is not installed. {install_doc}"
|
||||
import deep_ep # type: ignore[import-not-found]
|
||||
|
||||
if not hasattr(deep_ep, "ElasticBuffer"):
|
||||
return False
|
||||
return f"The installed DeepEP predates v2 (no ElasticBuffer). {install_doc}"
|
||||
nccl_ver = _get_runtime_nccl_version()
|
||||
if nccl_ver is None or nccl_ver < DEEPEP_V2_MIN_NCCL_VERSION_RAW:
|
||||
import torch
|
||||
|
||||
min_version = _format_nccl_raw_version(DEEPEP_V2_MIN_NCCL_VERSION_RAW)
|
||||
cuda_major = (torch.version.cuda or "13").split(".")[0]
|
||||
return (
|
||||
f"DeepEP v2 requires NCCL >= {min_version} but found "
|
||||
f"{_format_nccl_raw_version(nccl_ver) if nccl_ver else 'unknown'}. "
|
||||
"PyTorch pins an older NCCL; upgrade it with "
|
||||
f'`pip install "nvidia-nccl-cu{cuda_major}>={min_version}" --no-deps` '
|
||||
f"and rebuild DeepEP against it. {install_doc}"
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def has_deep_ep_v2() -> bool:
|
||||
"""Whether deep_ep with ElasticBuffer (v2 API) is available."""
|
||||
try:
|
||||
nccl_ver = _get_runtime_nccl_version()
|
||||
if nccl_ver is None or nccl_ver < DEEPEP_V2_MIN_NCCL_VERSION_RAW:
|
||||
logger.info_once(
|
||||
"DeepEP v2 requires NCCL >= %s but found %s. "
|
||||
"deepep_v2 backend will not be available.",
|
||||
_format_nccl_raw_version(DEEPEP_V2_MIN_NCCL_VERSION_RAW),
|
||||
_format_nccl_raw_version(nccl_ver) if nccl_ver else "unknown",
|
||||
)
|
||||
return False
|
||||
reason = deep_ep_v2_unavailable_reason()
|
||||
except Exception:
|
||||
return False
|
||||
return True
|
||||
if reason is not None and _has_module("deep_ep"):
|
||||
logger.info_once("deepep_v2 backend will not be available: %s", reason)
|
||||
return reason is None
|
||||
|
||||
|
||||
def has_deep_gemm() -> bool:
|
||||
|
||||
Reference in New Issue
Block a user