[Misc] Auto fallback to float16 for pre-Ampere GPUs when detected bfloat16 config (#17265)

Signed-off-by: Isotr0py <2037008807@qq.com>
This commit is contained in:
Isotr0py
2025-05-09 17:16:12 +00:00
committed by GitHub
parent c44c384b1c
commit 5c4c08f6f1
4 changed files with 57 additions and 26 deletions
+15 -1
View File
@@ -10,7 +10,7 @@ import torch
from vllm.logger import init_logger
from .interface import Platform, PlatformEnum, _Backend
from .interface import CpuArchEnum, Platform, PlatformEnum, _Backend
logger = init_logger(__name__)
@@ -26,6 +26,20 @@ class CpuPlatform(Platform):
device_type: str = "cpu"
dispatch_key: str = "CPU"
@property
def supported_dtypes(self) -> list:
if self.get_cpu_architecture() == CpuArchEnum.POWERPC:
return [torch.bfloat16, torch.float32]
elif sys.platform.startswith(
"darwin") and self.get_cpu_architecture() == CpuArchEnum.ARM:
# TODO: change this condition to check if the platform support bf16
# instead of checking the OS. For instance M2 shall supports bf16
# already. But we need to modify `cpu_extension.cmake` to activate
# the feature in the build.
return [torch.bfloat16, torch.float32]
# x86/aarch64 CPU has supported both bf16 and fp16 natively.
return [torch.bfloat16, torch.float16, torch.float32]
@classmethod
def get_device_name(cls, device_id: int = 0) -> str:
return "cpu"