From 026d5af7f410e59e3910a4d66af1f691db29ca1d Mon Sep 17 00:00:00 2001 From: Xuan-1998 Date: Fri, 28 Aug 2026 17:25:00 -0700 Subject: [PATCH] [Bugfix] Fix ncclCommQueryProperties heap overflow with NCCL >= 2.31 (#53008) Signed-off-by: Xuan Jiang Co-authored-by: Vladimir Aerov --- .../device_communicators/pynccl_wrapper.py | 17 +++++++++++++++-- vllm/utils/nccl.py | 5 ++++- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/vllm/distributed/device_communicators/pynccl_wrapper.py b/vllm/distributed/device_communicators/pynccl_wrapper.py index 78ee81e7b5..7056bc04ec 100644 --- a/vllm/distributed/device_communicators/pynccl_wrapper.py +++ b/vllm/distributed/device_communicators/pynccl_wrapper.py @@ -51,8 +51,15 @@ class ncclUniqueId(ctypes.Structure): _fields_ = [("internal", ctypes.c_byte * 128)] -# NCCL 2.30+ ncclCommProperties_t. Only fields through ginType are read; -# trailing fields keep the layout aligned with NCCL's versioned structure. +# Mirror of NCCL's versioned ncclCommProperties_t (nccl_device/core.h, +# v2.31.2-1, 144 bytes). ncclCommQueryProperties fills fields gated by the +# version the caller declares in props.version, not by props.size, so never +# declare a version newer than this layout: clamp to +# NCCL_COMM_PROPERTIES_LAYOUT_VERSION. To use fields added in a newer NCCL, +# extend the layout and bump the constant. +NCCL_COMM_PROPERTIES_LAYOUT_VERSION = 23102 # NCCL_VERSION(2, 31, 2) + + class ncclCommProperties(ctypes.Structure): _fields_ = [ ("size", ctypes.c_size_t), @@ -68,6 +75,12 @@ class ncclCommProperties(ctypes.Structure): ("nLsaTeams", ctypes.c_int), ("hostRmaSupport", ctypes.c_bool), ("railedGinType", ctypes.c_int), + # Filled only when the declared version is >= NCCL_VERSION(2, 31, 0). + ("commHash", ctypes.c_uint64), + ("ginMinStride", ctypes.c_int), + ("ginConnectionType", ctypes.c_int), + ("ginSupport", ctypes.c_bool * 64), + ("devCommRuntimeVersionSize", ctypes.c_size_t), ] diff --git a/vllm/utils/nccl.py b/vllm/utils/nccl.py index aeadaf5d22..5a033f1852 100644 --- a/vllm/utils/nccl.py +++ b/vllm/utils/nccl.py @@ -86,6 +86,7 @@ def find_nccl_library_paths() -> list[str] | None: def query_nccl_gin_type(group: torch.distributed.ProcessGroup) -> int | None: """Return the GIN type for an initialized group, or ``None`` on failure.""" from vllm.distributed.device_communicators.pynccl_wrapper import ( + NCCL_COMM_PROPERTIES_LAYOUT_VERSION, NCCLLibrary, ncclCommProperties, ) @@ -114,7 +115,9 @@ def query_nccl_gin_type(group: torch.distributed.ProcessGroup) -> int | None: ctypes.memset(ctypes.addressof(props), 0, ctypes.sizeof(props)) props.size = ctypes.sizeof(props) props.magic = 0xCAFEBEEF - props.version = nccl.ncclGetRawVersion() + props.version = min( + nccl.ncclGetRawVersion(), NCCL_COMM_PROPERTIES_LAYOUT_VERSION + ) result = query_fn(ctypes.c_void_p(comm_ptr), ctypes.byref(props)) except Exception: logger.warning("Failed to query NCCL communicator properties", exc_info=True)