[Core] nccl integrity check and test (#4155)
[Core] Add integrity check during initialization; add test for it (#4155)
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
import asyncio
|
||||
import enum
|
||||
import gc
|
||||
import glob
|
||||
import os
|
||||
import socket
|
||||
import subprocess
|
||||
@@ -517,3 +518,53 @@ def init_cached_hf_modules():
|
||||
"""
|
||||
from transformers.dynamic_module_utils import init_hf_modules
|
||||
init_hf_modules()
|
||||
|
||||
|
||||
def nccl_integrity_check(filepath):
|
||||
"""
|
||||
when the library is corrupted, we cannot catch
|
||||
the exception in python. it will crash the process.
|
||||
instead, we use the exit code of `ldd` to check
|
||||
if the library is corrupted. if not, we will return
|
||||
the version of the library.
|
||||
"""
|
||||
exit_code = os.system(f"ldd {filepath} 2>&1 > /dev/null")
|
||||
if exit_code != 0:
|
||||
raise RuntimeError(f"Failed to load NCCL library from {filepath} .")
|
||||
import ctypes
|
||||
|
||||
nccl = ctypes.CDLL(filepath)
|
||||
version = ctypes.c_int()
|
||||
nccl.ncclGetVersion.restype = ctypes.c_int
|
||||
nccl.ncclGetVersion.argtypes = [ctypes.POINTER(ctypes.c_int)]
|
||||
result = nccl.ncclGetVersion(ctypes.byref(version))
|
||||
assert result == 0
|
||||
return version.value
|
||||
|
||||
|
||||
def find_nccl_library():
|
||||
so_file = os.environ.get("VLLM_NCCL_SO_PATH", "")
|
||||
|
||||
# check if we have vllm-managed nccl
|
||||
vllm_nccl_path = None
|
||||
if torch.version.cuda is not None:
|
||||
cuda_major = torch.version.cuda.split(".")[0]
|
||||
path = os.path.expanduser(
|
||||
f"~/.config/vllm/nccl/cu{cuda_major}/libnccl.so.*")
|
||||
files = glob.glob(path)
|
||||
vllm_nccl_path = files[0] if files else None
|
||||
|
||||
# manually load the nccl library
|
||||
if so_file:
|
||||
logger.info(
|
||||
f"Found nccl from environment variable VLLM_NCCL_SO_PATH={so_file}"
|
||||
)
|
||||
else:
|
||||
if torch.version.cuda is not None:
|
||||
so_file = vllm_nccl_path or "libnccl.so.2"
|
||||
elif torch.version.hip is not None:
|
||||
so_file = "librccl.so.1"
|
||||
else:
|
||||
raise ValueError("NCCL only supports CUDA and ROCm backends.")
|
||||
logger.info(f"Found nccl from library {so_file}")
|
||||
return so_file
|
||||
|
||||
Reference in New Issue
Block a user