[Bugfix][CPU] Enable C++ causal_conv1d GDN path and float32 SSM cache on non-AMX AVX-512BF16 CPUs (#49688)
Signed-off-by: Dinesh Chitlangia <dineshchitlangia@gmail.com>
This commit is contained in:
@@ -181,13 +181,13 @@ class CpuPlatform(Platform):
|
||||
"otherwise the performance is not optimized."
|
||||
)
|
||||
|
||||
# AMX GDN requires float32 state
|
||||
# Accelerated GDN (AMX tiles or AVX-512BF16 VDPBF16PS) requires float32 SSM state.
|
||||
if (
|
||||
torch.cpu._is_amx_tile_supported()
|
||||
torch.cpu._is_avx512_bf16_supported()
|
||||
and cache_config.mamba_ssm_cache_dtype != "float32"
|
||||
):
|
||||
cache_config.mamba_ssm_cache_dtype = "float32"
|
||||
logger.warning("Reset SSM cache type to float32 for AMX mamba attention.")
|
||||
logger.warning("Reset SSM cache type to float32 for accelerated GDN mamba attention.")
|
||||
|
||||
# Lagecy setting
|
||||
env_key = "VLLM_CPU_KVCACHE_SPACE"
|
||||
@@ -290,8 +290,10 @@ class CpuPlatform(Platform):
|
||||
# Avoid inductor generates num_thread() and breaks the thread binding
|
||||
os.environ["TORCHINDUCTOR_CPP_DYNAMIC_THREADS"] = "1"
|
||||
|
||||
# For efficient conv state memory access
|
||||
if torch.cpu._is_amx_tile_supported():
|
||||
# For efficient conv state memory access. The C++ causal_conv1d
|
||||
# kernels (VDPBF16PS, no AMX tiles) consume the SD layout on any
|
||||
# AVX-512BF16 CPU, so apply it beyond AMX (e.g. AMD Zen5/Turin).
|
||||
if torch.cpu._is_avx512_bf16_supported():
|
||||
os.environ["VLLM_SSM_CONV_STATE_LAYOUT"] = "SD"
|
||||
|
||||
ld_preload_str = os.getenv("LD_PRELOAD", "")
|
||||
|
||||
Reference in New Issue
Block a user