[UX] Improve UX of CPU backend (#36968)
Signed-off-by: jiang1.li <jiang1.li@intel.com> Signed-off-by: Li, Jiang <bigpyj64@gmail.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
parent
f680dc1b39
commit
092ace9e3a
+24
-11
@@ -252,6 +252,8 @@ class CpuPlatform(Platform):
|
||||
if vllm_config.lora_config is not None:
|
||||
compilation_config.mode = CompilationMode.NONE
|
||||
|
||||
vllm_config.profiler_config.torch_profiler_dump_cuda_time_total = False
|
||||
|
||||
assert vllm_config.device_config.device_type == "cpu"
|
||||
|
||||
#
|
||||
@@ -470,21 +472,32 @@ class CpuPlatform(Platform):
|
||||
@classmethod
|
||||
def import_kernels(cls) -> None:
|
||||
if Platform.get_cpu_architecture() in (CpuArchEnum.X86,):
|
||||
if torch._C._cpu._is_avx512_supported():
|
||||
try:
|
||||
import vllm._C # noqa: F401
|
||||
except ImportError as e:
|
||||
logger.warning("Failed to import from vllm._C: %r", e)
|
||||
# Note: The lib name is _C_AVX2/AVX512, but the module name is _C.
|
||||
# This will cause a exception "dynamic module does define
|
||||
# module export function". But the library is imported
|
||||
# successfully. So ignore the exception for now, until we find
|
||||
# a solution.
|
||||
ignored_msg = "dynamic module does not define module export function"
|
||||
if torch.cpu._is_avx512_supported():
|
||||
if torch.cpu._is_avx512_bf16_supported():
|
||||
try:
|
||||
import vllm._C # noqa: F401
|
||||
except ImportError as e:
|
||||
logger.warning("Failed to import from vllm._C: %r", e)
|
||||
else:
|
||||
try:
|
||||
import vllm._C_AVX512 # noqa: F401
|
||||
except ImportError as e:
|
||||
if ignored_msg not in e.msg:
|
||||
logger.warning(
|
||||
"Failed to import from vllm._C_AVX512: %r", e
|
||||
)
|
||||
else:
|
||||
# Note: The lib name is _C_AVX2, but the module name is _C.
|
||||
# This will cause a exception "dynamic module does define
|
||||
# module export function". But the library is imported
|
||||
# successfully. So ignore the exception for now, until we find
|
||||
# a solution.
|
||||
try:
|
||||
import vllm._C_AVX2 # noqa: F401
|
||||
except ImportError as e:
|
||||
logger.warning("Failed to import from vllm._C_AVX2: %r", e)
|
||||
if ignored_msg not in e.msg:
|
||||
logger.warning("Failed to import from vllm._C_AVX2: %r", e)
|
||||
else:
|
||||
try:
|
||||
import vllm._C # noqa: F401
|
||||
|
||||
@@ -52,6 +52,21 @@ class CPUWorker(Worker):
|
||||
)
|
||||
|
||||
def init_device(self):
|
||||
# Check whether critical libraries are loaded
|
||||
def check_preloaded_libs(name: str):
|
||||
ld_preload_list = os.environ.get("LD_PRELOAD", "")
|
||||
if name not in ld_preload_list:
|
||||
raise RuntimeError(
|
||||
f"{name} is not found in LD_PRELOAD. "
|
||||
"Please follow the section `set LD_PRELOAD` in "
|
||||
"https://docs.vllm.ai/en/latest/getting_started/installation/cpu/ "
|
||||
"to setup required pre-loaded libraries."
|
||||
)
|
||||
|
||||
check_preloaded_libs("libtcmalloc")
|
||||
if current_platform.get_cpu_architecture() == CpuArchEnum.X86:
|
||||
check_preloaded_libs("libiomp")
|
||||
|
||||
# Setup OpenMP threads affinity.
|
||||
omp_cpuids = envs.VLLM_CPU_OMP_THREADS_BIND
|
||||
# Under numa binding some cores reserved for kv transfer in nixl_connector.py
|
||||
|
||||
Reference in New Issue
Block a user