diff --git a/.buildkite/hardware_tests/cpu.yaml b/.buildkite/hardware_tests/cpu.yaml index a29b44ed3c..64d5a654f5 100644 --- a/.buildkite/hardware_tests/cpu.yaml +++ b/.buildkite/hardware_tests/cpu.yaml @@ -277,7 +277,7 @@ steps: # `uv pip install "vllm[zen]"`, which forces uv to re-resolve vLLM's full # dependency closure and chokes on its transitive triton-cpu URL dependency. # Keep this pin in sync with the `zen` extra in setup.py. - ZENTORCH_VERSION: "2.13.0.1" + ZENTORCH_VERSION: "2.14.0.1" retry: automatic: - exit_status: 1 # Transient Docker/BuildKit failure diff --git a/docs/getting_started/installation/cpu.md b/docs/getting_started/installation/cpu.md index 1e58a68fcf..ebf9ebcdfd 100644 --- a/docs/getting_started/installation/cpu.md +++ b/docs/getting_started/installation/cpu.md @@ -159,7 +159,7 @@ VLLM_USE_PRECOMPILED=1 VLLM_PRECOMPILED_WHEEL_VARIANT=cpu VLLM_TARGET_DEVICE=cpu ### Which `dtype` should be used? - Currently, vLLM CPU uses model default settings as `dtype`. However, due to unstable float16 support in torch CPU, it is recommended to explicitly set `dtype=bfloat16` if there are any performance or accuracy problem. -- On AMD Zen CPUs (`ZenCpuPlatform`), `float16` is **not** supported. Only `bfloat16` and `float32` are accepted; models declared with `float16` are auto-downcast to `bfloat16` at model load time. See [AMD Zen optimizations](#amd-zen-optimizations). +- On AMD Zen CPUs (`ZenCpuPlatform`), `float16` requires AVX512-FP16, available from Venice (6th Gen EPYC) onwards. On Turin (5th Gen EPYC) and earlier only `bfloat16` and `float32` are accepted, and models declared with `float16` are auto-downcast to `bfloat16` at model load time. See [AMD Zen optimizations](#amd-zen-optimizations). ### How to launch a vLLM service on CPU? diff --git a/docs/getting_started/installation/cpu.x86.inc.md b/docs/getting_started/installation/cpu.x86.inc.md index ebae441f93..4c46bf45f2 100644 --- a/docs/getting_started/installation/cpu.x86.inc.md +++ b/docs/getting_started/installation/cpu.x86.inc.md @@ -273,7 +273,9 @@ Otherwise, vLLM falls back to the default `CpuPlatform` (oneDNN / sgl-kernel pat ### Supported dtypes -`float16` is **not** supported on `ZenCpuPlatform`. `ZenCpuPlatform.supported_dtypes` advertises only `bfloat16` and `float32`, so models declared with `torch_dtype=float16` are auto-downcast to `bfloat16` at load time with the standard `"Your device 'cpu' doesn't support torch.float16. Falling back to torch.bfloat16 for compatibility."` warning emitted from `vllm/config/model.py`. +`ZenCpuPlatform.supported_dtypes` always advertises `bfloat16` and `float32`, and adds `float16` when `torch.ops.zentorch.zentorch_is_fp16_supported()` reports AVX512-FP16, which AMD ships from Venice (6th Gen EPYC) onwards. On Turin (5th Gen EPYC) and earlier, models declared with `torch_dtype=float16` are auto-downcast to `bfloat16` at load time with the standard `"Your device 'cpu' doesn't support torch.float16. Falling back to torch.bfloat16 for compatibility."` warning emitted from `vllm/config/model.py`. + +`bfloat16` is listed first either way, so `--dtype auto` keeps resolving to it. ### Environment variables diff --git a/setup.py b/setup.py index abefefa35b..83b59e303a 100644 --- a/setup.py +++ b/setup.py @@ -1688,7 +1688,7 @@ setup( install_requires=get_requirements(), extras_require={ # AMD Zen CPU optimizations via zentorch - "zen": ["zentorch==2.14.0.0"], + "zen": ["zentorch==2.14.0.1"], "bench": ["pandas", "matplotlib", "seaborn", "datasets", "scipy", "plotly"], "tensorizer": ["tensorizer==2.10.1"], "fastsafetensors": ["fastsafetensors >= 0.3.3"], diff --git a/tests/test_zen_cpu_platform_detection.py b/tests/test_zen_cpu_platform_detection.py index b8a3949923..902e7f3624 100644 --- a/tests/test_zen_cpu_platform_detection.py +++ b/tests/test_zen_cpu_platform_detection.py @@ -2,9 +2,11 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import builtins import logging +from types import SimpleNamespace from unittest.mock import mock_open, patch import pytest +import torch from vllm.platforms import ( _is_amd_zen_cpu, @@ -139,3 +141,18 @@ def test_platform_plugin_failure_is_logged(caplog_vllm): assert platform == "vllm.platforms.interface.UnspecifiedPlatform" assert "Platform plugin cpu failed during detection" in caplog_vllm.text assert "RuntimeError: plugin exploded" in caplog_vllm.text + + +@pytest.mark.parametrize("fp16_supported", [True, False]) +def test_zen_cpu_supported_dtypes_track_fp16_capability(fp16_supported): + from vllm.platforms.zen_cpu import ZenCpuPlatform + + # zentorch is absent on most CI hosts, so torch.ops.zentorch has to be + # stood up rather than patched in place. + zentorch_ops = SimpleNamespace(zentorch_is_fp16_supported=lambda: fp16_supported) + with patch.object(torch.ops, "zentorch", zentorch_ops, create=True): + dtypes = ZenCpuPlatform().supported_dtypes + + # bf16 stays first so `--dtype auto` resolves to it on every Zen part. + assert dtypes[0] is torch.bfloat16 + assert (torch.float16 in dtypes) is fp16_supported diff --git a/vllm/platforms/zen_cpu.py b/vllm/platforms/zen_cpu.py index 2af64e5e9f..1ee45be385 100644 --- a/vllm/platforms/zen_cpu.py +++ b/vllm/platforms/zen_cpu.py @@ -25,8 +25,10 @@ class ZenCpuPlatform(CpuPlatform): # is_cpu() also returns True for this platform (inherited from CpuPlatform). return True - # Currently, AMD CPUs do not support float16 compute. - # Hence explicitly return bfloat16 and float32. + # fp16 compute needs AVX512-FP16, which AMD ships from Venice (6th Gen + # EPYC) onwards; Turin (5th Gen EPYC) and earlier have no fp16 kernels. @property def supported_dtypes(self) -> list[torch.dtype]: + if torch.ops.zentorch.zentorch_is_fp16_supported(): + return [torch.bfloat16, torch.float32, torch.float16] return [torch.bfloat16, torch.float32]