[CPU][Spec Decode] Warn about throughput loss when libiomp5 is not preloaded (#44419)
Signed-off-by: jmamou <jonathan.mamou@intel.com> Signed-off-by: Jonathan Mamou <jonathan.mamou@intel.com> Co-authored-by: Li, Jiang <bigpyj64@gmail.com> Co-authored-by: Li, Jiang <jiang1.li@intel.com>
This commit is contained in:
co-authored by
Li, Jiang <bigpyj64@gmail.com>
Li, Jiang <jiang1.li@intel.com>
parent
fa662b1a8b
commit
d5fe994e79
@@ -108,7 +108,7 @@ class CPUWorker(Worker):
|
||||
self.device = torch.device("cpu")
|
||||
|
||||
# Check whether critical libraries are loaded
|
||||
def check_preloaded_libs(name: str):
|
||||
def check_preloaded_libs(name: str) -> bool:
|
||||
ld_preload_list = os.environ.get("LD_PRELOAD", "")
|
||||
if name not in ld_preload_list:
|
||||
logger.warning(
|
||||
@@ -119,11 +119,22 @@ class CPUWorker(Worker):
|
||||
"to setup required pre-loaded libraries.",
|
||||
name,
|
||||
)
|
||||
return False
|
||||
return True
|
||||
|
||||
if sys.platform.startswith("linux"):
|
||||
check_preloaded_libs("libtcmalloc")
|
||||
if current_platform.get_cpu_architecture() == CpuArchEnum.X86:
|
||||
check_preloaded_libs("libiomp")
|
||||
iomp_loaded = check_preloaded_libs("libiomp")
|
||||
if not iomp_loaded and self.vllm_config.speculative_config is not None:
|
||||
logger.warning(
|
||||
"Speculative decoding on CPU without Intel OpenMP in "
|
||||
"LD_PRELOAD will cause significant performance loss. "
|
||||
"Please follow the section `set LD_PRELOAD` in "
|
||||
"https://docs.vllm.ai/en/latest/getting_started/"
|
||||
"installation/cpu/ "
|
||||
"to setup libiomp5.",
|
||||
)
|
||||
|
||||
def skip_set_num_threads(x: int):
|
||||
logger.warning(
|
||||
|
||||
Reference in New Issue
Block a user