[Attention][CPU] Use zentorch SDPA for CPU MLA prefill (#54967)
Signed-off-by: Rakul Chauhan <rakul.chauhan@amd.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Cursor
parent
a9eafde59c
commit
24c9772d19
@@ -25,6 +25,9 @@ from vllm.v1.attention.backends.mla.prefill.cpu_sdpa import (
|
||||
from vllm.v1.attention.backends.mla.prefill.selector import (
|
||||
get_mla_prefill_backend,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.prefill.zen_cpu_sdpa import (
|
||||
ZenCPUSDPAMLAPrefillBackend,
|
||||
)
|
||||
|
||||
|
||||
def _reference_ragged_attention(
|
||||
@@ -157,8 +160,10 @@ def test_cpu_mla_backend_smoke(tmp_path) -> None:
|
||||
def test_cpu_mla_prefill_backend_selected() -> None:
|
||||
# On CPU (no device capability) the MLA prefill backend must be the
|
||||
# SDPA-based CPU backend, not flash-attn which is unavailable on CPU.
|
||||
# Accelerator-specific subclasses (e.g. the zentorch-backed Zen CPU
|
||||
# backend) are also valid selections.
|
||||
backend_cls = get_mla_prefill_backend(None)
|
||||
assert backend_cls is CPUSDPAMLAPrefillBackend
|
||||
assert issubclass(backend_cls, CPUSDPAMLAPrefillBackend)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cpu(), reason="CPU only")
|
||||
@@ -248,3 +253,64 @@ def test_cpu_mla_prefill_context_chunk() -> None:
|
||||
assert lse.float().allclose(ref_lse.float(), atol=2e-2)
|
||||
assert out.shape == (5, 4, 16)
|
||||
assert lse.shape == (4, 5)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cpu(), reason="CPU only")
|
||||
@pytest.mark.skipif(
|
||||
not ZenCPUSDPAMLAPrefillBackend.is_available(),
|
||||
reason="requires a Zen CPU with the zentorch_sdpa op registered",
|
||||
)
|
||||
def test_zen_cpu_mla_prefill_new_tokens() -> None:
|
||||
# Runs the zentorch kernel itself. The head dims are deliberately
|
||||
# asymmetric (v_head_dim < qk_head_dim, as in real MLA) so that the V
|
||||
# padding and output slicing are exercised; the generic-CPU cases above
|
||||
# use equal dims and never reach that code.
|
||||
qk_head_dim = 24
|
||||
v_head_dim = 16
|
||||
backend = ZenCPUSDPAMLAPrefillBackend(
|
||||
num_heads=4,
|
||||
scale=0.25,
|
||||
kv_lora_rank=16,
|
||||
qk_nope_head_dim=16,
|
||||
qk_rope_head_dim=8,
|
||||
v_head_dim=v_head_dim,
|
||||
vllm_config=None,
|
||||
)
|
||||
|
||||
class _PrefillMeta:
|
||||
query_start_loc = torch.tensor([0, 3, 6], dtype=torch.int32)
|
||||
|
||||
backend.prepare_metadata(_PrefillMeta())
|
||||
q = torch.randn(6, 4, qk_head_dim, dtype=torch.bfloat16)
|
||||
k = torch.randn(6, 4, qk_head_dim, dtype=torch.bfloat16)
|
||||
v = torch.randn(6, 4, v_head_dim, dtype=torch.bfloat16)
|
||||
|
||||
out = backend.run_prefill_new_tokens(q, k, v, return_softmax_lse=False)
|
||||
ref, _ = _reference_ragged_attention(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
_PrefillMeta.query_start_loc,
|
||||
_PrefillMeta.query_start_loc,
|
||||
scale=0.25,
|
||||
causal=True,
|
||||
)
|
||||
assert out.shape == (6, 4, v_head_dim)
|
||||
assert out.float().allclose(ref.float(), atol=2e-2)
|
||||
|
||||
# zentorch_sdpa does not return a usable LSE, so that path must fall back
|
||||
# to the fp32 reference and match the generic backend exactly.
|
||||
generic = CPUSDPAMLAPrefillBackend(
|
||||
num_heads=4,
|
||||
scale=0.25,
|
||||
kv_lora_rank=16,
|
||||
qk_nope_head_dim=16,
|
||||
qk_rope_head_dim=8,
|
||||
v_head_dim=v_head_dim,
|
||||
vllm_config=None,
|
||||
)
|
||||
generic.prepare_metadata(_PrefillMeta())
|
||||
zen_out, zen_lse = backend.run_prefill_new_tokens(q, k, v, return_softmax_lse=True)
|
||||
gen_out, gen_lse = generic.run_prefill_new_tokens(q, k, v, return_softmax_lse=True)
|
||||
assert torch.equal(zen_out, gen_out)
|
||||
assert torch.equal(zen_lse, gen_lse)
|
||||
|
||||
@@ -60,10 +60,43 @@ class TestGetMLAPrefillBackend:
|
||||
|
||||
def test_cpu_uses_sdpa_prefill(self):
|
||||
vllm_config = _make_vllm_config()
|
||||
zen_cls = MLAPrefillBackendEnum.ZEN_CPU.get_class()
|
||||
|
||||
# A Zen CPU with zentorch selects the Zen backend, so expect whichever
|
||||
# one this host offers.
|
||||
expected = (
|
||||
zen_cls if zen_cls.is_available() else MLAPrefillBackendEnum.CPU.get_class()
|
||||
)
|
||||
|
||||
with patch("vllm.platforms.current_platform") as mock_platform:
|
||||
mock_platform.is_cpu.return_value = True
|
||||
|
||||
backend = get_mla_prefill_backend(vllm_config)
|
||||
assert backend is expected
|
||||
|
||||
def test_zen_cpu_prefers_zentorch_prefill(self):
|
||||
vllm_config = _make_vllm_config()
|
||||
zen_cls = MLAPrefillBackendEnum.ZEN_CPU.get_class()
|
||||
|
||||
with (
|
||||
patch("vllm.platforms.current_platform") as mock_platform,
|
||||
patch.object(zen_cls, "is_available", return_value=True),
|
||||
):
|
||||
mock_platform.is_cpu.return_value = True
|
||||
|
||||
backend = get_mla_prefill_backend(vllm_config)
|
||||
assert backend is zen_cls
|
||||
|
||||
def test_zen_cpu_falls_back_when_zentorch_unavailable(self):
|
||||
vllm_config = _make_vllm_config()
|
||||
zen_cls = MLAPrefillBackendEnum.ZEN_CPU.get_class()
|
||||
|
||||
with (
|
||||
patch("vllm.platforms.current_platform") as mock_platform,
|
||||
patch.object(zen_cls, "is_available", return_value=False),
|
||||
):
|
||||
mock_platform.is_cpu.return_value = True
|
||||
|
||||
backend = get_mla_prefill_backend(vllm_config)
|
||||
assert backend is MLAPrefillBackendEnum.CPU.get_class()
|
||||
|
||||
|
||||
@@ -53,6 +53,10 @@ class MLAPrefillBackendEnum(Enum, metaclass=_MLAPrefillBackendEnumMeta):
|
||||
"AiterFlashAttnPrefillBackend"
|
||||
)
|
||||
CPU = "vllm.v1.attention.backends.mla.prefill.cpu_sdpa.CPUSDPAMLAPrefillBackend"
|
||||
ZEN_CPU = (
|
||||
"vllm.v1.attention.backends.mla.prefill.zen_cpu_sdpa."
|
||||
"ZenCPUSDPAMLAPrefillBackend"
|
||||
)
|
||||
# Placeholder for third-party/custom backends - must be registered before use
|
||||
# set to None to avoid alias with other backend, whose value is an empty string
|
||||
CUSTOM = None
|
||||
|
||||
@@ -110,8 +110,18 @@ def get_mla_prefill_backend(
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if current_platform.is_cpu():
|
||||
logger.info_once("Using CPU SDPA MLA prefill backend.")
|
||||
return MLAPrefillBackendEnum.CPU.get_class()
|
||||
# CPUs have no compute capability, so the capability-driven priority
|
||||
# path below does not apply. Prefer an accelerator-specific CPU backend
|
||||
# when its kernels are present, else the generic SDPA one.
|
||||
for backend_enum in (MLAPrefillBackendEnum.ZEN_CPU, MLAPrefillBackendEnum.CPU):
|
||||
try:
|
||||
cpu_backend_cls = backend_enum.get_class()
|
||||
except ImportError:
|
||||
continue
|
||||
if cpu_backend_cls.is_available():
|
||||
logger.info_once("Using %s MLA prefill backend.", backend_enum.name)
|
||||
return cpu_backend_cls
|
||||
raise ValueError("No valid CPU MLA prefill backend found.")
|
||||
|
||||
device_capability = current_platform.get_device_capability()
|
||||
if device_capability is None:
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""CPU MLA prefill backend using zentorch SDPA on AMD Zen CPUs.
|
||||
|
||||
Subclasses the generic CPU SDPA backend and replaces the per-request attention
|
||||
with ``zentorch_sdpa``, which fuses the scale, causal mask, softmax and both
|
||||
matmuls of the base implementation into a single call.
|
||||
|
||||
``zentorch_sdpa`` does not return a usable log-sum-exp, so requests that need
|
||||
one (context chunks, prefix/suffix merge) are delegated to the base backend.
|
||||
|
||||
Availability is reported through :meth:`is_available`, so the prefill selector
|
||||
gates this backend once at selection time rather than per attention call.
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.kernels.linear.zentorch_utils import has_zentorch_op
|
||||
from vllm.v1.attention.backends.mla.prefill.cpu_sdpa import CPUSDPAMLAPrefillBackend
|
||||
|
||||
|
||||
class ZenCPUSDPAMLAPrefillBackend(CPUSDPAMLAPrefillBackend):
|
||||
"""MLA prefill backend for AMD Zen CPUs backed by ``zentorch_sdpa``."""
|
||||
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "ZEN_CPU_SDPA_MLA"
|
||||
|
||||
@classmethod
|
||||
def is_available(cls) -> bool:
|
||||
return has_zentorch_op(["zentorch_sdpa"])
|
||||
|
||||
@staticmethod
|
||||
def _sdpa_layout(
|
||||
q_r: torch.Tensor,
|
||||
k_r: torch.Tensor,
|
||||
v_r: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, int]:
|
||||
"""Convert one request's [S, NH, D] tensors to the [1, NH, S, D] layout.
|
||||
|
||||
MLA has ``v_head_dim < qk_head_dim``, which the fused kernel does not
|
||||
accept, so V is padded up to the query head dim. The original V head
|
||||
dim is returned so the caller can slice the padding off afterwards.
|
||||
"""
|
||||
h_q = q_r.shape[-1]
|
||||
h_v = v_r.shape[-1]
|
||||
pad = h_q - h_v
|
||||
|
||||
qs = q_r.transpose(0, 1).unsqueeze(0).contiguous()
|
||||
ks = k_r.transpose(0, 1).unsqueeze(0).contiguous()
|
||||
vs = v_r.transpose(0, 1).unsqueeze(0).contiguous()
|
||||
if pad > 0:
|
||||
vs = torch.nn.functional.pad(vs, (0, pad))
|
||||
return qs, ks, vs, h_v
|
||||
|
||||
def _ragged_sdpa(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
q_cu_seq_lens: torch.Tensor,
|
||||
kv_cu_seq_lens: torch.Tensor,
|
||||
causal: bool,
|
||||
return_softmax_lse: bool,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
if return_softmax_lse:
|
||||
# zentorch_sdpa's second return is not a usable LSE, so fall back
|
||||
# to the base fp32 implementation whenever one is required.
|
||||
return super()._ragged_sdpa(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
q_cu_seq_lens,
|
||||
kv_cu_seq_lens,
|
||||
causal,
|
||||
return_softmax_lse,
|
||||
)
|
||||
|
||||
num_reqs = q_cu_seq_lens.numel() - 1
|
||||
assert num_reqs == kv_cu_seq_lens.numel() - 1
|
||||
|
||||
out_chunks: list[torch.Tensor] = []
|
||||
for r in range(num_reqs):
|
||||
q_start = int(q_cu_seq_lens[r])
|
||||
q_end = int(q_cu_seq_lens[r + 1])
|
||||
kv_start = int(kv_cu_seq_lens[r])
|
||||
kv_end = int(kv_cu_seq_lens[r + 1])
|
||||
|
||||
qs, ks, vs, h_v = self._sdpa_layout(
|
||||
q[q_start:q_end],
|
||||
k[kv_start:kv_end],
|
||||
v[kv_start:kv_end],
|
||||
)
|
||||
|
||||
attn, _ = torch.ops.zentorch.zentorch_sdpa(
|
||||
qs,
|
||||
ks,
|
||||
vs,
|
||||
0.0,
|
||||
causal,
|
||||
attn_mask=None,
|
||||
scale=self.scale,
|
||||
)
|
||||
out_chunks.append(attn[0].transpose(0, 1)[..., :h_v].to(v.dtype))
|
||||
|
||||
return torch.cat(out_chunks, dim=0)
|
||||
Reference in New Issue
Block a user