Introduce VLLM_CUDART_SO_PATH to allow users specify the .so path (#12998)
Signed-off-by: Lu Fang <lufang@fb.com>
This commit is contained in:
@@ -5,12 +5,14 @@ convenient for use when we just need to call a few functions.
|
||||
"""
|
||||
|
||||
import ctypes
|
||||
import glob
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
# this line makes it possible to directly load `libcudart.so` using `ctypes`
|
||||
import torch # noqa
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -60,6 +62,29 @@ def find_loaded_library(lib_name) -> Optional[str]:
|
||||
return path
|
||||
|
||||
|
||||
def get_cudart_lib_path_from_env() -> Optional[str]:
|
||||
"""
|
||||
In some system, find_loaded_library() may not work. So we allow users to
|
||||
specify the path through environment variable VLLM_CUDART_SO_PATH.
|
||||
"""
|
||||
cudart_so_env = envs.VLLM_CUDART_SO_PATH
|
||||
if cudart_so_env is not None:
|
||||
cudart_paths = [
|
||||
cudart_so_env,
|
||||
]
|
||||
for path in cudart_paths:
|
||||
file_paths = glob.glob(path)
|
||||
if len(file_paths) > 0:
|
||||
logger.info(
|
||||
"Found cudart library at %s through env var"
|
||||
"VLLM_CUDART_SO_PATH=%s",
|
||||
file_paths[0],
|
||||
cudart_so_env,
|
||||
)
|
||||
return file_paths[0]
|
||||
return None
|
||||
|
||||
|
||||
class CudaRTLibrary:
|
||||
exported_functions = [
|
||||
# cudaError_t cudaSetDevice ( int device )
|
||||
@@ -105,8 +130,13 @@ class CudaRTLibrary:
|
||||
def __init__(self, so_file: Optional[str] = None):
|
||||
if so_file is None:
|
||||
so_file = find_loaded_library("libcudart")
|
||||
if so_file is None:
|
||||
so_file = get_cudart_lib_path_from_env()
|
||||
assert so_file is not None, \
|
||||
"libcudart is not loaded in the current process"
|
||||
(
|
||||
"libcudart is not loaded in the current process, "
|
||||
"try setting VLLM_CUDART_SO_PATH"
|
||||
)
|
||||
if so_file not in CudaRTLibrary.path_to_library_cache:
|
||||
lib = ctypes.CDLL(so_file)
|
||||
CudaRTLibrary.path_to_library_cache[so_file] = lib
|
||||
|
||||
Reference in New Issue
Block a user