diff --git a/tests/v1/attention/test_cpu_mla_backend.py b/tests/v1/attention/test_cpu_mla_backend.py index fc3867099e..938babb262 100644 --- a/tests/v1/attention/test_cpu_mla_backend.py +++ b/tests/v1/attention/test_cpu_mla_backend.py @@ -25,6 +25,9 @@ from vllm.v1.attention.backends.mla.prefill.cpu_sdpa import ( from vllm.v1.attention.backends.mla.prefill.selector import ( get_mla_prefill_backend, ) +from vllm.v1.attention.backends.mla.prefill.zen_cpu_sdpa import ( + ZenCPUSDPAMLAPrefillBackend, +) def _reference_ragged_attention( @@ -157,8 +160,10 @@ def test_cpu_mla_backend_smoke(tmp_path) -> None: def test_cpu_mla_prefill_backend_selected() -> None: # On CPU (no device capability) the MLA prefill backend must be the # SDPA-based CPU backend, not flash-attn which is unavailable on CPU. + # Accelerator-specific subclasses (e.g. the zentorch-backed Zen CPU + # backend) are also valid selections. backend_cls = get_mla_prefill_backend(None) - assert backend_cls is CPUSDPAMLAPrefillBackend + assert issubclass(backend_cls, CPUSDPAMLAPrefillBackend) @pytest.mark.skipif(not current_platform.is_cpu(), reason="CPU only") @@ -248,3 +253,64 @@ def test_cpu_mla_prefill_context_chunk() -> None: assert lse.float().allclose(ref_lse.float(), atol=2e-2) assert out.shape == (5, 4, 16) assert lse.shape == (4, 5) + + +@pytest.mark.skipif(not current_platform.is_cpu(), reason="CPU only") +@pytest.mark.skipif( + not ZenCPUSDPAMLAPrefillBackend.is_available(), + reason="requires a Zen CPU with the zentorch_sdpa op registered", +) +def test_zen_cpu_mla_prefill_new_tokens() -> None: + # Runs the zentorch kernel itself. The head dims are deliberately + # asymmetric (v_head_dim < qk_head_dim, as in real MLA) so that the V + # padding and output slicing are exercised; the generic-CPU cases above + # use equal dims and never reach that code. + qk_head_dim = 24 + v_head_dim = 16 + backend = ZenCPUSDPAMLAPrefillBackend( + num_heads=4, + scale=0.25, + kv_lora_rank=16, + qk_nope_head_dim=16, + qk_rope_head_dim=8, + v_head_dim=v_head_dim, + vllm_config=None, + ) + + class _PrefillMeta: + query_start_loc = torch.tensor([0, 3, 6], dtype=torch.int32) + + backend.prepare_metadata(_PrefillMeta()) + q = torch.randn(6, 4, qk_head_dim, dtype=torch.bfloat16) + k = torch.randn(6, 4, qk_head_dim, dtype=torch.bfloat16) + v = torch.randn(6, 4, v_head_dim, dtype=torch.bfloat16) + + out = backend.run_prefill_new_tokens(q, k, v, return_softmax_lse=False) + ref, _ = _reference_ragged_attention( + q, + k, + v, + _PrefillMeta.query_start_loc, + _PrefillMeta.query_start_loc, + scale=0.25, + causal=True, + ) + assert out.shape == (6, 4, v_head_dim) + assert out.float().allclose(ref.float(), atol=2e-2) + + # zentorch_sdpa does not return a usable LSE, so that path must fall back + # to the fp32 reference and match the generic backend exactly. + generic = CPUSDPAMLAPrefillBackend( + num_heads=4, + scale=0.25, + kv_lora_rank=16, + qk_nope_head_dim=16, + qk_rope_head_dim=8, + v_head_dim=v_head_dim, + vllm_config=None, + ) + generic.prepare_metadata(_PrefillMeta()) + zen_out, zen_lse = backend.run_prefill_new_tokens(q, k, v, return_softmax_lse=True) + gen_out, gen_lse = generic.run_prefill_new_tokens(q, k, v, return_softmax_lse=True) + assert torch.equal(zen_out, gen_out) + assert torch.equal(zen_lse, gen_lse) diff --git a/tests/v1/attention/test_mla_prefill_selector.py b/tests/v1/attention/test_mla_prefill_selector.py index 098eddfbf6..87d188c6a0 100644 --- a/tests/v1/attention/test_mla_prefill_selector.py +++ b/tests/v1/attention/test_mla_prefill_selector.py @@ -60,10 +60,43 @@ class TestGetMLAPrefillBackend: def test_cpu_uses_sdpa_prefill(self): vllm_config = _make_vllm_config() + zen_cls = MLAPrefillBackendEnum.ZEN_CPU.get_class() + + # A Zen CPU with zentorch selects the Zen backend, so expect whichever + # one this host offers. + expected = ( + zen_cls if zen_cls.is_available() else MLAPrefillBackendEnum.CPU.get_class() + ) with patch("vllm.platforms.current_platform") as mock_platform: mock_platform.is_cpu.return_value = True + backend = get_mla_prefill_backend(vllm_config) + assert backend is expected + + def test_zen_cpu_prefers_zentorch_prefill(self): + vllm_config = _make_vllm_config() + zen_cls = MLAPrefillBackendEnum.ZEN_CPU.get_class() + + with ( + patch("vllm.platforms.current_platform") as mock_platform, + patch.object(zen_cls, "is_available", return_value=True), + ): + mock_platform.is_cpu.return_value = True + + backend = get_mla_prefill_backend(vllm_config) + assert backend is zen_cls + + def test_zen_cpu_falls_back_when_zentorch_unavailable(self): + vllm_config = _make_vllm_config() + zen_cls = MLAPrefillBackendEnum.ZEN_CPU.get_class() + + with ( + patch("vllm.platforms.current_platform") as mock_platform, + patch.object(zen_cls, "is_available", return_value=False), + ): + mock_platform.is_cpu.return_value = True + backend = get_mla_prefill_backend(vllm_config) assert backend is MLAPrefillBackendEnum.CPU.get_class() diff --git a/vllm/v1/attention/backends/mla/prefill/registry.py b/vllm/v1/attention/backends/mla/prefill/registry.py index 1cf1761a77..f7ec31e153 100644 --- a/vllm/v1/attention/backends/mla/prefill/registry.py +++ b/vllm/v1/attention/backends/mla/prefill/registry.py @@ -53,6 +53,10 @@ class MLAPrefillBackendEnum(Enum, metaclass=_MLAPrefillBackendEnumMeta): "AiterFlashAttnPrefillBackend" ) CPU = "vllm.v1.attention.backends.mla.prefill.cpu_sdpa.CPUSDPAMLAPrefillBackend" + ZEN_CPU = ( + "vllm.v1.attention.backends.mla.prefill.zen_cpu_sdpa." + "ZenCPUSDPAMLAPrefillBackend" + ) # Placeholder for third-party/custom backends - must be registered before use # set to None to avoid alias with other backend, whose value is an empty string CUSTOM = None diff --git a/vllm/v1/attention/backends/mla/prefill/selector.py b/vllm/v1/attention/backends/mla/prefill/selector.py index a2b74e4327..0cb29f51b4 100644 --- a/vllm/v1/attention/backends/mla/prefill/selector.py +++ b/vllm/v1/attention/backends/mla/prefill/selector.py @@ -110,8 +110,18 @@ def get_mla_prefill_backend( from vllm.platforms import current_platform if current_platform.is_cpu(): - logger.info_once("Using CPU SDPA MLA prefill backend.") - return MLAPrefillBackendEnum.CPU.get_class() + # CPUs have no compute capability, so the capability-driven priority + # path below does not apply. Prefer an accelerator-specific CPU backend + # when its kernels are present, else the generic SDPA one. + for backend_enum in (MLAPrefillBackendEnum.ZEN_CPU, MLAPrefillBackendEnum.CPU): + try: + cpu_backend_cls = backend_enum.get_class() + except ImportError: + continue + if cpu_backend_cls.is_available(): + logger.info_once("Using %s MLA prefill backend.", backend_enum.name) + return cpu_backend_cls + raise ValueError("No valid CPU MLA prefill backend found.") device_capability = current_platform.get_device_capability() if device_capability is None: diff --git a/vllm/v1/attention/backends/mla/prefill/zen_cpu_sdpa.py b/vllm/v1/attention/backends/mla/prefill/zen_cpu_sdpa.py new file mode 100644 index 0000000000..88c6ce98a0 --- /dev/null +++ b/vllm/v1/attention/backends/mla/prefill/zen_cpu_sdpa.py @@ -0,0 +1,106 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""CPU MLA prefill backend using zentorch SDPA on AMD Zen CPUs. + +Subclasses the generic CPU SDPA backend and replaces the per-request attention +with ``zentorch_sdpa``, which fuses the scale, causal mask, softmax and both +matmuls of the base implementation into a single call. + +``zentorch_sdpa`` does not return a usable log-sum-exp, so requests that need +one (context chunks, prefix/suffix merge) are delegated to the base backend. + +Availability is reported through :meth:`is_available`, so the prefill selector +gates this backend once at selection time rather than per attention call. +""" + +import torch + +from vllm.model_executor.kernels.linear.zentorch_utils import has_zentorch_op +from vllm.v1.attention.backends.mla.prefill.cpu_sdpa import CPUSDPAMLAPrefillBackend + + +class ZenCPUSDPAMLAPrefillBackend(CPUSDPAMLAPrefillBackend): + """MLA prefill backend for AMD Zen CPUs backed by ``zentorch_sdpa``.""" + + @staticmethod + def get_name() -> str: + return "ZEN_CPU_SDPA_MLA" + + @classmethod + def is_available(cls) -> bool: + return has_zentorch_op(["zentorch_sdpa"]) + + @staticmethod + def _sdpa_layout( + q_r: torch.Tensor, + k_r: torch.Tensor, + v_r: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, int]: + """Convert one request's [S, NH, D] tensors to the [1, NH, S, D] layout. + + MLA has ``v_head_dim < qk_head_dim``, which the fused kernel does not + accept, so V is padded up to the query head dim. The original V head + dim is returned so the caller can slice the padding off afterwards. + """ + h_q = q_r.shape[-1] + h_v = v_r.shape[-1] + pad = h_q - h_v + + qs = q_r.transpose(0, 1).unsqueeze(0).contiguous() + ks = k_r.transpose(0, 1).unsqueeze(0).contiguous() + vs = v_r.transpose(0, 1).unsqueeze(0).contiguous() + if pad > 0: + vs = torch.nn.functional.pad(vs, (0, pad)) + return qs, ks, vs, h_v + + def _ragged_sdpa( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + q_cu_seq_lens: torch.Tensor, + kv_cu_seq_lens: torch.Tensor, + causal: bool, + return_softmax_lse: bool, + ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: + if return_softmax_lse: + # zentorch_sdpa's second return is not a usable LSE, so fall back + # to the base fp32 implementation whenever one is required. + return super()._ragged_sdpa( + q, + k, + v, + q_cu_seq_lens, + kv_cu_seq_lens, + causal, + return_softmax_lse, + ) + + num_reqs = q_cu_seq_lens.numel() - 1 + assert num_reqs == kv_cu_seq_lens.numel() - 1 + + out_chunks: list[torch.Tensor] = [] + for r in range(num_reqs): + q_start = int(q_cu_seq_lens[r]) + q_end = int(q_cu_seq_lens[r + 1]) + kv_start = int(kv_cu_seq_lens[r]) + kv_end = int(kv_cu_seq_lens[r + 1]) + + qs, ks, vs, h_v = self._sdpa_layout( + q[q_start:q_end], + k[kv_start:kv_end], + v[kv_start:kv_end], + ) + + attn, _ = torch.ops.zentorch.zentorch_sdpa( + qs, + ks, + vs, + 0.0, + causal, + attn_mask=None, + scale=self.scale, + ) + out_chunks.append(attn[0].transpose(0, 1)[..., :h_v].to(v.dtype)) + + return torch.cat(out_chunks, dim=0)