[Attention][CPU] Run Zen CPU encoder attention on zentorch SDPA (#54508)

Signed-off-by: priyansh jain <priyansh.jain2@amd.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Priyansh Jain
2026-09-24 02:05:44 -07:00
committed by GitHub
co-authored by Cursor Andreas Karatzas
parent 14f98cbe5d
commit 5963795ec7
5 changed files with 319 additions and 4 deletions
+133
View File
@@ -3,13 +3,20 @@
import functools
import math
from types import SimpleNamespace
import pytest
import torch
from vllm.model_executor.kernels.linear.zentorch_utils import has_zentorch_op
from vllm.platforms import CpuArchEnum, current_platform
from vllm.utils.torch_utils import set_random_seed
from vllm.v1.attention.backend import AttentionType
from vllm.v1.attention.backends.cpu_attn import _get_attn_isa
from vllm.v1.attention.backends.zentorch_sdpa import (
should_use_zentorch_sdpa,
zentorch_sdpa_attn,
)
if not current_platform.is_cpu():
pytest.skip("skipping CPU-only tests", allow_module_level=True)
@@ -47,6 +54,24 @@ _FP8_RTOL = 0.1
ENCODER_SEQ_LENS = [
[1, 678, 2367, 145, 4162, 36, 7812],
]
# Uniform (dense zentorch_sdpa call) and ragged (per-sequence loop).
ZEN_ENCODER_SEQ_LENS = [
[128, 128, 128],
[1, 67, 233, 5],
]
_skip_no_zentorch_sdpa = pytest.mark.skipif(
not current_platform.is_zen_cpu() or not has_zentorch_op(["zentorch_sdpa_attn"]),
reason="zentorch_sdpa requires a Zen CPU with the op registered",
)
# should_use_zentorch_sdpa additionally gates each dtype on its ISA.
_skip_no_avx512_bf16 = pytest.mark.skipif(
not torch.cpu._is_avx512_bf16_supported(),
reason="zentorch_sdpa bfloat16 requires AVX512-BF16",
)
_skip_no_avx512 = pytest.mark.skipif(
not torch.cpu._is_avx512_supported(),
reason="zentorch_sdpa float32 requires AVX512",
)
def get_attn_isa(
@@ -205,6 +230,7 @@ def ref_varlen_encoder_attn(
seq_lens: list[int],
scale: float,
sliding_window: int | None = None,
alibi_slopes: torch.Tensor | None = None,
) -> torch.Tensor:
num_seqs = len(seq_lens)
dtype = query.dtype
@@ -232,6 +258,11 @@ def ref_varlen_encoder_attn(
else:
mask = empty_mask.logical_not()
if alibi_slopes is not None:
q_pos = torch.arange(0, seq_len)[None, :, None]
kv_pos = torch.arange(0, seq_len)[None, None, :]
attn += -alibi_slopes[:, None, None].float() * (q_pos - kv_pos)
attn.masked_fill_(mask, float("-inf"))
attn = torch.softmax(attn, dim=-1)
out = torch.einsum("hqk,khd->qhd", attn, v).to(dtype=dtype)
@@ -772,6 +803,108 @@ def test_varlen_encoder_attention_amx(
)
@torch.inference_mode()
def varlen_encoder_zentorch_sdpa(
seq_lens: list[int],
num_heads: tuple[int, int],
head_size: int,
sliding_window: int | None,
dtype: torch.dtype,
use_alibi: bool,
) -> None:
set_random_seed(0)
num_query_heads, num_kv_heads = num_heads
assert num_query_heads % num_kv_heads == 0
scale = head_size**-0.5
token_num = sum(seq_lens)
query = tensor_cache(
token_num * num_query_heads * head_size, dtype, tag="query"
).view(token_num, num_query_heads, head_size)
key_value = tensor_cache(
2 * token_num * num_kv_heads * head_size, dtype, tag="kv"
).view(2, token_num, num_kv_heads, head_size)
key, value = key_value.unbind(0)
query_start_loc = torch.zeros(len(seq_lens) + 1, dtype=torch.int32)
torch.cumsum(torch.tensor(seq_lens, dtype=torch.int32), 0, out=query_start_loc[1:])
alibi_slopes = _get_alibi_slopes(num_query_heads) if use_alibi else None
output = torch.empty_like(query)
zentorch_sdpa_attn(
query,
key,
value,
output,
SimpleNamespace(query_start_loc=query_start_loc, causal=False),
scale,
sliding_window if sliding_window is not None else -1,
alibi_slopes,
)
ref_output = ref_varlen_encoder_attn(
query=query,
key=key,
value=value,
seq_lens=seq_lens,
scale=scale,
sliding_window=sliding_window,
alibi_slopes=alibi_slopes,
)
torch.testing.assert_close(output, ref_output, atol=1.5e-2, rtol=1e-2)
_ZENTORCH_SDPA_DTYPES = [
pytest.param(torch.bfloat16, marks=_skip_no_avx512_bf16),
pytest.param(torch.float32, marks=_skip_no_avx512),
]
@_skip_no_zentorch_sdpa
@pytest.mark.parametrize("seq_lens", ZEN_ENCODER_SEQ_LENS)
@pytest.mark.parametrize("num_heads", NUM_HEADS)
@pytest.mark.parametrize("head_size", [128])
@pytest.mark.parametrize("sliding_window", SLIDING_WINDOWS)
@pytest.mark.parametrize("dtype", _ZENTORCH_SDPA_DTYPES)
@pytest.mark.parametrize("use_alibi", [False, True])
def test_zentorch_sdpa_attn(
seq_lens: list[int],
num_heads: tuple[int, int],
head_size: int,
sliding_window: int | None,
dtype: torch.dtype,
use_alibi: bool,
) -> None:
varlen_encoder_zentorch_sdpa(
seq_lens=seq_lens,
num_heads=num_heads,
head_size=head_size,
sliding_window=sliding_window,
dtype=dtype,
use_alibi=use_alibi,
)
@_skip_no_zentorch_sdpa
@pytest.mark.parametrize(
"attn_type", [AttentionType.ENCODER_ONLY, AttentionType.ENCODER]
)
@pytest.mark.parametrize("dtype", _ZENTORCH_SDPA_DTYPES)
def test_should_use_zentorch_sdpa(attn_type: str, dtype: torch.dtype) -> None:
assert should_use_zentorch_sdpa(attn_type, dtype)
@_skip_no_zentorch_sdpa
@pytest.mark.parametrize(
"attn_type,dtype",
[
(AttentionType.DECODER, torch.bfloat16),
(AttentionType.ENCODER_ONLY, torch.float16),
],
)
def test_should_not_use_zentorch_sdpa(attn_type: str, dtype: torch.dtype) -> None:
assert not should_use_zentorch_sdpa(attn_type, dtype)
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8_e4m3", "fp8_e5m2"])
@pytest.mark.parametrize("seq_lens", SEQ_LENS)
@pytest.mark.parametrize("num_heads", NUM_HEADS)