[CPU] Refine CPU attention frontend (#45391)

Signed-off-by: jiang1.li <jiang1.li@intel.com>
This commit is contained in:
Li, Jiang
2026-06-14 19:26:54 -07:00
committed by GitHub
parent b675cb7d0f
commit 8760f972ca
4 changed files with 384 additions and 217 deletions
+287 -7
View File
@@ -25,7 +25,6 @@ from vllm._custom_ops import (
if torch.cpu._is_amx_tile_supported():
torch.cpu._init_amx()
NUM_HEADS = [
(4, 4),
(8, 2),
@@ -43,6 +42,11 @@ SEQ_LENS = [ # (q_len, kv_len)
[(2345, 2345), (5, 5), (3, 16), (134, 5131)], # prefill batch
[(992, 2456), (1, 1234), (98, 1145), (1, 4162), (2345, 2345)], # mixed batch
]
_FP8_ATOL = {"fp8_e4m3": 0.2, "fp8_e5m2": 0.3}
_FP8_RTOL = 0.1
ENCODER_SEQ_LENS = [
[1, 678, 2367, 145, 4162, 36, 7812],
]
def get_attn_isa(
@@ -61,10 +65,7 @@ def get_attn_isa(
# rand number generation takes too much time, cache rand tensors
@functools.lru_cache(maxsize=128, typed=False)
def tensor_cache(
elem_num: int,
dtype: torch.dtype,
) -> torch.Tensor:
def tensor_cache(elem_num: int, dtype: torch.dtype, tag: str = "none") -> torch.Tensor:
tensor = torch.randn(elem_num, dtype=dtype)
return tensor
@@ -183,8 +184,222 @@ def ref_paged_attn(
return torch.cat(outputs, dim=0)
_FP8_ATOL = {"fp8_e4m3": 0.2, "fp8_e5m2": 0.3}
_FP8_RTOL = 0.1
def ref_varlen_encoder_attn(
query: torch.Tensor, # [token, q_head_num, head_dim]
key: torch.Tensor, # [token, kv_head_num, head_dim]
value: torch.Tensor,
seq_lens: list[int],
scale: float,
sliding_window: int | None = None,
) -> torch.Tensor:
num_seqs = len(seq_lens)
dtype = query.dtype
output = torch.empty_like(query)
start_idx = 0
for i in range(num_seqs):
seq_len = seq_lens[i]
q = query[start_idx : start_idx + seq_len].float()
k = key[start_idx : start_idx + seq_len].float()
v = value[start_idx : start_idx + seq_len].float()
q *= scale
if q.shape[1] != k.shape[1]:
k = torch.repeat_interleave(k, q.shape[1] // k.shape[1], dim=1)
v = torch.repeat_interleave(v, q.shape[1] // v.shape[1], dim=1)
attn = torch.einsum("qhd,khd->hqk", q, k).float()
empty_mask = torch.ones(seq_len, seq_len)
if sliding_window is not None:
mask = (
torch.triu(empty_mask, diagonal=1 - sliding_window).bool()
^ torch.triu(empty_mask, diagonal=sliding_window).bool()
).logical_not()
else:
mask = empty_mask.logical_not()
attn.masked_fill_(mask, float("-inf"))
attn = torch.softmax(attn, dim=-1)
out = torch.einsum("hqk,khd->qhd", attn, v).to(dtype=dtype)
output[start_idx : start_idx + seq_len].copy_(out)
start_idx += seq_len
return output
@torch.inference_mode()
def varlen_encoder_attention(
seq_lens: list[int],
num_heads: tuple[int, int],
head_size: int,
sliding_window: int | None,
dtype: torch.dtype,
block_size: int,
isa: str,
) -> None:
set_random_seed(0)
num_seqs = len(seq_lens)
num_query_heads = num_heads[0]
num_kv_heads = num_heads[1]
assert num_query_heads % num_kv_heads == 0
window_size = (
(sliding_window - 1, sliding_window - 1)
if sliding_window is not None
else (-1, -1)
)
scale = head_size**-0.5
token_num = sum(seq_lens)
seq_lens_tensor = torch.tensor(seq_lens, dtype=torch.int32)
query_start_loc = torch.zeros(num_seqs, dtype=torch.int32)
torch.cumsum(seq_lens_tensor[:-1], 0, out=query_start_loc[1:])
block_nums = (seq_lens_tensor + block_size - 1) // block_size
start_block_ids = torch.zeros_like(seq_lens_tensor)
torch.cumsum(block_nums[:-1], 0, out=start_block_ids[1:])
total_block_num: int = block_nums.sum().item()
max_block_num = block_nums.max().item()
block_offsets = torch.arange(0, max_block_num, dtype=torch.int32)
encoder_block_table = start_block_ids[:, None] + block_offsets[None, :]
slot_mapping_list = []
slot_start_idx = 0
for i in range(num_seqs):
block_num = block_nums[i].item()
seq_len = seq_lens[i]
slot_mapping_list.append(torch.arange(slot_start_idx, slot_start_idx + seq_len))
slot_start_idx += block_num * block_size
slot_mapping = torch.cat(slot_mapping_list)
query = tensor_cache(
elem_num=token_num * num_query_heads * head_size,
dtype=dtype,
tag="query",
)
query = query.view(
token_num,
num_query_heads,
head_size,
)
key_value = tensor_cache(
elem_num=2 * token_num * num_kv_heads * head_size,
dtype=dtype,
tag="kv",
)
key_value = key_value.view(
2,
token_num,
num_kv_heads,
head_size,
)
key, value = key_value.unbind(0)
# KV cache for CPU attention
packed_key_value_cache = torch.zeros(
total_block_num, num_kv_heads, block_size, head_size * 2, dtype=dtype
)
packed_key_value_cache = packed_key_value_cache.view(
(total_block_num, num_kv_heads, block_size * 2, -1)
)
packed_key_cache, packed_value_cache = packed_key_value_cache.chunk(2, dim=2)
cu_query_lens = torch.tensor([0] + seq_lens, dtype=torch.int32).cumsum(
dim=0, dtype=torch.int32
)
kv_lens_tensor = torch.tensor(seq_lens, dtype=torch.int32)
# use reshape_and_cache to pack key_cache and value_cache
cpu_attn_reshape_and_cache(
key=key.view(-1, num_kv_heads, head_size),
value=value.view(-1, num_kv_heads, head_size),
key_cache=packed_key_cache,
value_cache=packed_value_cache,
slot_mapping=slot_mapping,
isa=isa,
)
metadata = cpu_attn_get_scheduler_metadata(
num_reqs=num_seqs,
num_heads=num_query_heads,
num_kv_heads=num_kv_heads,
head_dim=head_size,
seq_lens=kv_lens_tensor,
dtype=dtype,
query_start_loc=cu_query_lens,
causal=False,
sliding_window_size=sliding_window if sliding_window is not None else -1,
isa=isa,
enable_kv_split=False,
)
out_without_split = torch.empty_like(query)
cpu_attention_with_kv_cache(
query=query,
key_cache=packed_key_cache,
value_cache=packed_value_cache,
output=out_without_split,
query_start_loc=cu_query_lens,
seq_lens=kv_lens_tensor,
scale=scale,
causal=False,
alibi_slopes=None,
sliding_window=window_size,
block_table=encoder_block_table,
softcap=0,
scheduler_metadata=metadata,
s_aux=None,
)
metadata = cpu_attn_get_scheduler_metadata(
num_reqs=num_seqs,
num_heads=num_query_heads,
num_kv_heads=num_kv_heads,
head_dim=head_size,
seq_lens=kv_lens_tensor,
dtype=dtype,
query_start_loc=cu_query_lens,
causal=False,
sliding_window_size=sliding_window if sliding_window is not None else -1,
isa=isa,
enable_kv_split=True,
)
out_with_split = torch.empty_like(query)
cpu_attention_with_kv_cache(
query=query,
key_cache=packed_key_cache,
value_cache=packed_value_cache,
output=out_with_split,
query_start_loc=cu_query_lens,
seq_lens=kv_lens_tensor,
scale=scale,
causal=False,
alibi_slopes=None,
sliding_window=window_size,
block_table=encoder_block_table,
softcap=0,
scheduler_metadata=metadata,
s_aux=None,
)
ref_output = ref_varlen_encoder_attn(
query=query,
key=key,
value=value,
seq_lens=seq_lens,
scale=scale,
sliding_window=sliding_window,
)
atol, rtol = 1.5e-2, 1e-2
(
torch.testing.assert_close(out_with_split, ref_output, atol=atol, rtol=rtol),
f"{torch.max(torch.abs(out_with_split - ref_output))}",
)
(
torch.testing.assert_close(out_without_split, ref_output, atol=atol, rtol=rtol),
f"{torch.max(torch.abs(out_without_split - ref_output))}",
)
@torch.inference_mode()
@@ -418,6 +633,71 @@ def varlen_with_paged_kv(
)
@pytest.mark.parametrize("seq_lens", ENCODER_SEQ_LENS)
@pytest.mark.parametrize("num_heads", NUM_HEADS)
@pytest.mark.parametrize("head_size", HEAD_SIZES)
@pytest.mark.parametrize(
"block_size",
[
128,
],
)
@pytest.mark.parametrize("sliding_window", SLIDING_WINDOWS)
@pytest.mark.parametrize("dtype", QTYPES)
@pytest.mark.parametrize("isa", ["vec"])
def test_varlen_encoder_attention_vec(
seq_lens: list[int],
num_heads: tuple[int, int],
head_size: int,
sliding_window: int | None,
dtype: torch.dtype,
block_size: int,
isa: str,
) -> None:
varlen_encoder_attention(
seq_lens=seq_lens,
num_heads=num_heads,
head_size=head_size,
sliding_window=sliding_window,
dtype=dtype,
block_size=block_size,
isa=isa,
)
@pytest.mark.parametrize("seq_lens", ENCODER_SEQ_LENS)
@pytest.mark.parametrize("num_heads", NUM_HEADS)
@pytest.mark.parametrize("head_size", HEAD_SIZES)
@pytest.mark.parametrize(
"block_size",
[
128,
],
)
@pytest.mark.parametrize("sliding_window", SLIDING_WINDOWS)
@pytest.mark.parametrize("dtype", [torch.bfloat16])
@pytest.mark.parametrize("isa", ["amx"])
@pytest.mark.skipif(not torch.cpu._is_amx_tile_supported(), reason="no AMX support.")
def test_varlen_encoder_attention_amx(
seq_lens: list[int],
num_heads: tuple[int, int],
head_size: int,
sliding_window: int | None,
dtype: torch.dtype,
block_size: int,
isa: str,
) -> None:
varlen_encoder_attention(
seq_lens=seq_lens,
num_heads=num_heads,
head_size=head_size,
sliding_window=sliding_window,
dtype=dtype,
block_size=block_size,
isa=isa,
)
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8_e4m3", "fp8_e5m2"])
@pytest.mark.parametrize("seq_lens", SEQ_LENS)
@pytest.mark.parametrize("num_heads", NUM_HEADS)