diff --git a/vllm/platforms/cpu.py b/vllm/platforms/cpu.py index 571a8c9c2c..ed7cec519a 100644 --- a/vllm/platforms/cpu.py +++ b/vllm/platforms/cpu.py @@ -125,6 +125,14 @@ class CpuPlatform(Platform): "otherwise the performance is not optimized." ) + # AMX GDN requires float32 state + if ( + torch.cpu._is_amx_tile_supported() + and cache_config.mamba_ssm_cache_dtype != "float32" + ): + cache_config.mamba_ssm_cache_dtype = "float32" + logger.warning("Reset SSM cache type to float32 for AMX mamba attention.") + # Lagecy setting env_key = "VLLM_CPU_KVCACHE_SPACE" if env_key in os.environ and os.environ[env_key] != "":