[CI/Build] CPU release supports both of AVX2 and AVX512 (#35466)
Signed-off-by: jiang1.li <jiang1.li@intel.com> Co-authored-by: jiang1.li <jiang1.li@intel.com>
This commit is contained in:
+1
-3
@@ -178,9 +178,7 @@ def mla_decode_kvcache_cpu(
|
||||
block_tables: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
) -> None:
|
||||
torch.ops._C_cpu.mla_decode_kvcache(
|
||||
out, query, kv_cache, scale, block_tables, seq_lens
|
||||
)
|
||||
torch.ops._C.mla_decode_kvcache(out, query, kv_cache, scale, block_tables, seq_lens)
|
||||
|
||||
|
||||
# merge attn states ops
|
||||
|
||||
@@ -483,3 +483,27 @@ class CpuPlatform(Platform):
|
||||
@classmethod
|
||||
def support_hybrid_kv_cache(cls) -> bool:
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def import_kernels(cls) -> None:
|
||||
if Platform.get_cpu_architecture() in (CpuArchEnum.X86,):
|
||||
if torch._C._cpu._is_avx512_supported():
|
||||
try:
|
||||
import vllm._C # noqa: F401
|
||||
except ImportError as e:
|
||||
logger.warning("Failed to import from vllm._C: %r", e)
|
||||
else:
|
||||
# Note: The lib name is _C_AVX2, but the module name is _C.
|
||||
# This will cause a exception "dynamic module does define
|
||||
# module export function". But the library is imported
|
||||
# successfully. So ignore the exception for now, until we find
|
||||
# a solution.
|
||||
try:
|
||||
import vllm._C_AVX2 # noqa: F401
|
||||
except ImportError as e:
|
||||
logger.warning("Failed to import from vllm._C_AVX2: %r", e)
|
||||
else:
|
||||
try:
|
||||
import vllm._C # noqa: F401
|
||||
except ImportError as e:
|
||||
logger.warning("Failed to import from vllm._C: %r", e)
|
||||
|
||||
@@ -85,7 +85,7 @@ class CPUWorker(Worker):
|
||||
self.local_omp_cpuid = omp_cpuids_list[self.rank]
|
||||
|
||||
if self.local_omp_cpuid != "nobind":
|
||||
ret = torch.ops._C_utils.init_cpu_threads_env(self.local_omp_cpuid)
|
||||
ret = torch.ops._C.init_cpu_threads_env(self.local_omp_cpuid)
|
||||
if ret:
|
||||
logger.info(ret)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user