[Bugfix] Fix ncclCommQueryProperties heap overflow with NCCL >= 2.31 (#53008)
Signed-off-by: Xuan Jiang <xuanj@amazon.com> Co-authored-by: Vladimir Aerov <vaerov@amazon.com>
This commit is contained in:
co-authored by
Vladimir Aerov
parent
68c52b5a93
commit
026d5af7f4
+4
-1
@@ -86,6 +86,7 @@ def find_nccl_library_paths() -> list[str] | None:
|
||||
def query_nccl_gin_type(group: torch.distributed.ProcessGroup) -> int | None:
|
||||
"""Return the GIN type for an initialized group, or ``None`` on failure."""
|
||||
from vllm.distributed.device_communicators.pynccl_wrapper import (
|
||||
NCCL_COMM_PROPERTIES_LAYOUT_VERSION,
|
||||
NCCLLibrary,
|
||||
ncclCommProperties,
|
||||
)
|
||||
@@ -114,7 +115,9 @@ def query_nccl_gin_type(group: torch.distributed.ProcessGroup) -> int | None:
|
||||
ctypes.memset(ctypes.addressof(props), 0, ctypes.sizeof(props))
|
||||
props.size = ctypes.sizeof(props)
|
||||
props.magic = 0xCAFEBEEF
|
||||
props.version = nccl.ncclGetRawVersion()
|
||||
props.version = min(
|
||||
nccl.ncclGetRawVersion(), NCCL_COMM_PROPERTIES_LAYOUT_VERSION
|
||||
)
|
||||
result = query_fn(ctypes.c_void_p(comm_ptr), ctypes.byref(props))
|
||||
except Exception:
|
||||
logger.warning("Failed to query NCCL communicator properties", exc_info=True)
|
||||
|
||||
Reference in New Issue
Block a user