[Core] nccl integrity check and test (#4155)

[Core] Add integrity check during initialization; add test for it (#4155)
This commit is contained in:
youkaichao
2024-04-17 22:28:52 -07:00
committed by GitHub
parent 533d2a1f39
commit 6dc1fc9cfe
4 changed files with 107 additions and 26 deletions
+51
View File
@@ -1,6 +1,7 @@
import asyncio
import enum
import gc
import glob
import os
import socket
import subprocess
@@ -517,3 +518,53 @@ def init_cached_hf_modules():
"""
from transformers.dynamic_module_utils import init_hf_modules
init_hf_modules()
def nccl_integrity_check(filepath):
"""
when the library is corrupted, we cannot catch
the exception in python. it will crash the process.
instead, we use the exit code of `ldd` to check
if the library is corrupted. if not, we will return
the version of the library.
"""
exit_code = os.system(f"ldd {filepath} 2>&1 > /dev/null")
if exit_code != 0:
raise RuntimeError(f"Failed to load NCCL library from {filepath} .")
import ctypes
nccl = ctypes.CDLL(filepath)
version = ctypes.c_int()
nccl.ncclGetVersion.restype = ctypes.c_int
nccl.ncclGetVersion.argtypes = [ctypes.POINTER(ctypes.c_int)]
result = nccl.ncclGetVersion(ctypes.byref(version))
assert result == 0
return version.value
def find_nccl_library():
so_file = os.environ.get("VLLM_NCCL_SO_PATH", "")
# check if we have vllm-managed nccl
vllm_nccl_path = None
if torch.version.cuda is not None:
cuda_major = torch.version.cuda.split(".")[0]
path = os.path.expanduser(
f"~/.config/vllm/nccl/cu{cuda_major}/libnccl.so.*")
files = glob.glob(path)
vllm_nccl_path = files[0] if files else None
# manually load the nccl library
if so_file:
logger.info(
f"Found nccl from environment variable VLLM_NCCL_SO_PATH={so_file}"
)
else:
if torch.version.cuda is not None:
so_file = vllm_nccl_path or "libnccl.so.2"
elif torch.version.hip is not None:
so_file = "librccl.so.1"
else:
raise ValueError("NCCL only supports CUDA and ROCm backends.")
logger.info(f"Found nccl from library {so_file}")
return so_file