[Attention][CPU] Use zentorch SDPA for CPU MLA prefill (#54967)

Signed-off-by: Rakul Chauhan <rakul.chauhan@amd.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Rakul-Chauhan
2026-09-27 08:30:55 +00:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Cursor
parent a9eafde59c
commit 24c9772d19
5 changed files with 222 additions and 3 deletions
+67 -1
View File
@@ -25,6 +25,9 @@ from vllm.v1.attention.backends.mla.prefill.cpu_sdpa import (
from vllm.v1.attention.backends.mla.prefill.selector import (
get_mla_prefill_backend,
)
from vllm.v1.attention.backends.mla.prefill.zen_cpu_sdpa import (
ZenCPUSDPAMLAPrefillBackend,
)
def _reference_ragged_attention(
@@ -157,8 +160,10 @@ def test_cpu_mla_backend_smoke(tmp_path) -> None:
def test_cpu_mla_prefill_backend_selected() -> None:
# On CPU (no device capability) the MLA prefill backend must be the
# SDPA-based CPU backend, not flash-attn which is unavailable on CPU.
# Accelerator-specific subclasses (e.g. the zentorch-backed Zen CPU
# backend) are also valid selections.
backend_cls = get_mla_prefill_backend(None)
assert backend_cls is CPUSDPAMLAPrefillBackend
assert issubclass(backend_cls, CPUSDPAMLAPrefillBackend)
@pytest.mark.skipif(not current_platform.is_cpu(), reason="CPU only")
@@ -248,3 +253,64 @@ def test_cpu_mla_prefill_context_chunk() -> None:
assert lse.float().allclose(ref_lse.float(), atol=2e-2)
assert out.shape == (5, 4, 16)
assert lse.shape == (4, 5)
@pytest.mark.skipif(not current_platform.is_cpu(), reason="CPU only")
@pytest.mark.skipif(
not ZenCPUSDPAMLAPrefillBackend.is_available(),
reason="requires a Zen CPU with the zentorch_sdpa op registered",
)
def test_zen_cpu_mla_prefill_new_tokens() -> None:
# Runs the zentorch kernel itself. The head dims are deliberately
# asymmetric (v_head_dim < qk_head_dim, as in real MLA) so that the V
# padding and output slicing are exercised; the generic-CPU cases above
# use equal dims and never reach that code.
qk_head_dim = 24
v_head_dim = 16
backend = ZenCPUSDPAMLAPrefillBackend(
num_heads=4,
scale=0.25,
kv_lora_rank=16,
qk_nope_head_dim=16,
qk_rope_head_dim=8,
v_head_dim=v_head_dim,
vllm_config=None,
)
class _PrefillMeta:
query_start_loc = torch.tensor([0, 3, 6], dtype=torch.int32)
backend.prepare_metadata(_PrefillMeta())
q = torch.randn(6, 4, qk_head_dim, dtype=torch.bfloat16)
k = torch.randn(6, 4, qk_head_dim, dtype=torch.bfloat16)
v = torch.randn(6, 4, v_head_dim, dtype=torch.bfloat16)
out = backend.run_prefill_new_tokens(q, k, v, return_softmax_lse=False)
ref, _ = _reference_ragged_attention(
q,
k,
v,
_PrefillMeta.query_start_loc,
_PrefillMeta.query_start_loc,
scale=0.25,
causal=True,
)
assert out.shape == (6, 4, v_head_dim)
assert out.float().allclose(ref.float(), atol=2e-2)
# zentorch_sdpa does not return a usable LSE, so that path must fall back
# to the fp32 reference and match the generic backend exactly.
generic = CPUSDPAMLAPrefillBackend(
num_heads=4,
scale=0.25,
kv_lora_rank=16,
qk_nope_head_dim=16,
qk_rope_head_dim=8,
v_head_dim=v_head_dim,
vllm_config=None,
)
generic.prepare_metadata(_PrefillMeta())
zen_out, zen_lse = backend.run_prefill_new_tokens(q, k, v, return_softmax_lse=True)
gen_out, gen_lse = generic.run_prefill_new_tokens(q, k, v, return_softmax_lse=True)
assert torch.equal(zen_out, gen_out)
assert torch.equal(zen_lse, gen_lse)
@@ -60,10 +60,43 @@ class TestGetMLAPrefillBackend:
def test_cpu_uses_sdpa_prefill(self):
vllm_config = _make_vllm_config()
zen_cls = MLAPrefillBackendEnum.ZEN_CPU.get_class()
# A Zen CPU with zentorch selects the Zen backend, so expect whichever
# one this host offers.
expected = (
zen_cls if zen_cls.is_available() else MLAPrefillBackendEnum.CPU.get_class()
)
with patch("vllm.platforms.current_platform") as mock_platform:
mock_platform.is_cpu.return_value = True
backend = get_mla_prefill_backend(vllm_config)
assert backend is expected
def test_zen_cpu_prefers_zentorch_prefill(self):
vllm_config = _make_vllm_config()
zen_cls = MLAPrefillBackendEnum.ZEN_CPU.get_class()
with (
patch("vllm.platforms.current_platform") as mock_platform,
patch.object(zen_cls, "is_available", return_value=True),
):
mock_platform.is_cpu.return_value = True
backend = get_mla_prefill_backend(vllm_config)
assert backend is zen_cls
def test_zen_cpu_falls_back_when_zentorch_unavailable(self):
vllm_config = _make_vllm_config()
zen_cls = MLAPrefillBackendEnum.ZEN_CPU.get_class()
with (
patch("vllm.platforms.current_platform") as mock_platform,
patch.object(zen_cls, "is_available", return_value=False),
):
mock_platform.is_cpu.return_value = True
backend = get_mla_prefill_backend(vllm_config)
assert backend is MLAPrefillBackendEnum.CPU.get_class()
@@ -53,6 +53,10 @@ class MLAPrefillBackendEnum(Enum, metaclass=_MLAPrefillBackendEnumMeta):
"AiterFlashAttnPrefillBackend"
)
CPU = "vllm.v1.attention.backends.mla.prefill.cpu_sdpa.CPUSDPAMLAPrefillBackend"
ZEN_CPU = (
"vllm.v1.attention.backends.mla.prefill.zen_cpu_sdpa."
"ZenCPUSDPAMLAPrefillBackend"
)
# Placeholder for third-party/custom backends - must be registered before use
# set to None to avoid alias with other backend, whose value is an empty string
CUSTOM = None
@@ -110,8 +110,18 @@ def get_mla_prefill_backend(
from vllm.platforms import current_platform
if current_platform.is_cpu():
logger.info_once("Using CPU SDPA MLA prefill backend.")
return MLAPrefillBackendEnum.CPU.get_class()
# CPUs have no compute capability, so the capability-driven priority
# path below does not apply. Prefer an accelerator-specific CPU backend
# when its kernels are present, else the generic SDPA one.
for backend_enum in (MLAPrefillBackendEnum.ZEN_CPU, MLAPrefillBackendEnum.CPU):
try:
cpu_backend_cls = backend_enum.get_class()
except ImportError:
continue
if cpu_backend_cls.is_available():
logger.info_once("Using %s MLA prefill backend.", backend_enum.name)
return cpu_backend_cls
raise ValueError("No valid CPU MLA prefill backend found.")
device_capability = current_platform.get_device_capability()
if device_capability is None:
@@ -0,0 +1,106 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""CPU MLA prefill backend using zentorch SDPA on AMD Zen CPUs.
Subclasses the generic CPU SDPA backend and replaces the per-request attention
with ``zentorch_sdpa``, which fuses the scale, causal mask, softmax and both
matmuls of the base implementation into a single call.
``zentorch_sdpa`` does not return a usable log-sum-exp, so requests that need
one (context chunks, prefix/suffix merge) are delegated to the base backend.
Availability is reported through :meth:`is_available`, so the prefill selector
gates this backend once at selection time rather than per attention call.
"""
import torch
from vllm.model_executor.kernels.linear.zentorch_utils import has_zentorch_op
from vllm.v1.attention.backends.mla.prefill.cpu_sdpa import CPUSDPAMLAPrefillBackend
class ZenCPUSDPAMLAPrefillBackend(CPUSDPAMLAPrefillBackend):
"""MLA prefill backend for AMD Zen CPUs backed by ``zentorch_sdpa``."""
@staticmethod
def get_name() -> str:
return "ZEN_CPU_SDPA_MLA"
@classmethod
def is_available(cls) -> bool:
return has_zentorch_op(["zentorch_sdpa"])
@staticmethod
def _sdpa_layout(
q_r: torch.Tensor,
k_r: torch.Tensor,
v_r: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, int]:
"""Convert one request's [S, NH, D] tensors to the [1, NH, S, D] layout.
MLA has ``v_head_dim < qk_head_dim``, which the fused kernel does not
accept, so V is padded up to the query head dim. The original V head
dim is returned so the caller can slice the padding off afterwards.
"""
h_q = q_r.shape[-1]
h_v = v_r.shape[-1]
pad = h_q - h_v
qs = q_r.transpose(0, 1).unsqueeze(0).contiguous()
ks = k_r.transpose(0, 1).unsqueeze(0).contiguous()
vs = v_r.transpose(0, 1).unsqueeze(0).contiguous()
if pad > 0:
vs = torch.nn.functional.pad(vs, (0, pad))
return qs, ks, vs, h_v
def _ragged_sdpa(
self,
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
q_cu_seq_lens: torch.Tensor,
kv_cu_seq_lens: torch.Tensor,
causal: bool,
return_softmax_lse: bool,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
if return_softmax_lse:
# zentorch_sdpa's second return is not a usable LSE, so fall back
# to the base fp32 implementation whenever one is required.
return super()._ragged_sdpa(
q,
k,
v,
q_cu_seq_lens,
kv_cu_seq_lens,
causal,
return_softmax_lse,
)
num_reqs = q_cu_seq_lens.numel() - 1
assert num_reqs == kv_cu_seq_lens.numel() - 1
out_chunks: list[torch.Tensor] = []
for r in range(num_reqs):
q_start = int(q_cu_seq_lens[r])
q_end = int(q_cu_seq_lens[r + 1])
kv_start = int(kv_cu_seq_lens[r])
kv_end = int(kv_cu_seq_lens[r + 1])
qs, ks, vs, h_v = self._sdpa_layout(
q[q_start:q_end],
k[kv_start:kv_end],
v[kv_start:kv_end],
)
attn, _ = torch.ops.zentorch.zentorch_sdpa(
qs,
ks,
vs,
0.0,
causal,
attn_mask=None,
scale=self.scale,
)
out_chunks.append(attn[0].transpose(0, 1)[..., :h_v].to(v.dtype))
return torch.cat(out_chunks, dim=0)