[WideEP] Update NCCL to 2.30.7 to enable DeepEPv2 in the vllm/vllm-openai image (#45321)
Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Signed-off-by: Tyler Michael Smith <tyler@vllm.ai> Signed-off-by: Tyler Michael Smith <tyler@tylermsmith.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Ilya Markov <ilmarkov@users.noreply.github.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
co-authored by
Claude
Ilya Markov
OpenAI Codex
Codex
parent
e222c33f2f
commit
5d8e90a966
@@ -978,6 +978,7 @@ class DeepEPV2All2AllManager(All2AllManagerBase):
|
||||
self._device_group = device_group
|
||||
self.handle_cache = Cache()
|
||||
self._num_sms: int | None = None
|
||||
self._gin_checked = False
|
||||
|
||||
def _make_all2all_kwargs(
|
||||
self,
|
||||
@@ -1000,11 +1001,37 @@ class DeepEPV2All2AllManager(All2AllManagerBase):
|
||||
explicitly_destroy=True,
|
||||
)
|
||||
|
||||
def _check_gin_support(self, group) -> None:
|
||||
from vllm.utils.nccl import query_nccl_gin_type
|
||||
|
||||
# ProcessGroupNCCL creates communicators lazily. Initialize this exact
|
||||
# group before querying so a null comm pointer is not mistaken for
|
||||
# missing GIN support.
|
||||
probe = torch.zeros(1, device="cuda")
|
||||
torch.distributed.all_reduce(probe, group=group)
|
||||
|
||||
gin_type = query_nccl_gin_type(group)
|
||||
if gin_type is None:
|
||||
raise RuntimeError(
|
||||
"DeepEPv2 communicator properties query failed; "
|
||||
"networking capability could not be determined."
|
||||
)
|
||||
if gin_type == 0:
|
||||
raise RuntimeError(
|
||||
"DeepEPv2 requires NCCL GIN (GPU-Initiated Networking). "
|
||||
"This usually means IBGDA-capable InfiniBand NICs or drivers "
|
||||
"are not available. See tools/ep_kernels/README.md for "
|
||||
"requirements."
|
||||
)
|
||||
|
||||
def get_handle(self, kwargs):
|
||||
import deep_ep # type: ignore[import-not-found]
|
||||
|
||||
num_experts = kwargs.pop("num_experts", 256)
|
||||
buffer_kwargs = self._make_all2all_kwargs(**kwargs)
|
||||
if not self._gin_checked:
|
||||
self._check_gin_support(buffer_kwargs["group"])
|
||||
self._gin_checked = True
|
||||
logger.debug("DeepEP v2 all2all args %s", buffer_kwargs)
|
||||
handle: deep_ep.ElasticBuffer = self.handle_cache.get_or_create(
|
||||
buffer_kwargs, deep_ep.ElasticBuffer
|
||||
|
||||
@@ -14,7 +14,7 @@ from vllm import envs
|
||||
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.nccl import find_nccl_include_paths
|
||||
from vllm.utils.nccl import find_nccl_include_paths, find_nccl_library_paths
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -74,11 +74,15 @@ def compile_nccl_allocator():
|
||||
out_dir = tempfile.gettempdir()
|
||||
nccl_allocator_libname = "nccl_allocator"
|
||||
nccl_include_paths = find_nccl_include_paths()
|
||||
ldflags = ["-l:libnccl.so.2"]
|
||||
nccl_lib_paths = find_nccl_library_paths()
|
||||
if nccl_lib_paths:
|
||||
ldflags = [f"-L{p}" for p in nccl_lib_paths] + ldflags
|
||||
load_inline(
|
||||
name=nccl_allocator_libname,
|
||||
cpp_sources=nccl_allocator_source,
|
||||
with_cuda=True,
|
||||
extra_ldflags=["-lnccl"],
|
||||
extra_ldflags=ldflags,
|
||||
verbose=envs.VLLM_LOGGING_LEVEL == "DEBUG",
|
||||
is_python_module=False,
|
||||
build_directory=out_dir,
|
||||
|
||||
@@ -51,6 +51,26 @@ class ncclUniqueId(ctypes.Structure):
|
||||
_fields_ = [("internal", ctypes.c_byte * 128)]
|
||||
|
||||
|
||||
# NCCL 2.30+ ncclCommProperties_t. Only fields through ginType are read;
|
||||
# trailing fields keep the layout aligned with NCCL's versioned structure.
|
||||
class ncclCommProperties(ctypes.Structure):
|
||||
_fields_ = [
|
||||
("size", ctypes.c_size_t),
|
||||
("magic", ctypes.c_uint),
|
||||
("version", ctypes.c_uint),
|
||||
("rank", ctypes.c_int),
|
||||
("nRanks", ctypes.c_int),
|
||||
("cudaDev", ctypes.c_int),
|
||||
("nvmlDev", ctypes.c_int),
|
||||
("deviceApiSupport", ctypes.c_bool),
|
||||
("multimemSupport", ctypes.c_bool),
|
||||
("ginType", ctypes.c_int),
|
||||
("nLsaTeams", ctypes.c_int),
|
||||
("hostRmaSupport", ctypes.c_bool),
|
||||
("railedGinType", ctypes.c_int),
|
||||
]
|
||||
|
||||
|
||||
cudaStream_t = ctypes.c_void_p
|
||||
buffer_type = ctypes.c_void_p
|
||||
|
||||
@@ -317,6 +337,12 @@ class NCCLLibrary:
|
||||
# ncclResult_t ncclCommWindowDeregister(
|
||||
# ncclComm_t comm, ncclWindow_t win);
|
||||
Function("ncclCommWindowDeregister", ncclResult_t, [ncclComm_t, ncclWindow_t]),
|
||||
# Query runtime properties of a specific initialized communicator.
|
||||
Function(
|
||||
"ncclCommQueryProperties",
|
||||
ncclResult_t,
|
||||
[ncclComm_t, ctypes.POINTER(ncclCommProperties)],
|
||||
),
|
||||
]
|
||||
|
||||
# class attribute to store the mapping from the path to the library
|
||||
@@ -375,6 +401,9 @@ class NCCLLibrary:
|
||||
# Having an exception here on ROCm platform is
|
||||
# not allowed during graph capturing
|
||||
continue
|
||||
elif func.name == "ncclCommQueryProperties":
|
||||
# Optional on NCCL versions older than 2.29.
|
||||
continue
|
||||
raise
|
||||
NCCLLibrary.path_to_dict_mapping[so_file] = _funcs
|
||||
self._funcs = NCCLLibrary.path_to_dict_mapping[so_file]
|
||||
|
||||
Reference in New Issue
Block a user