[CPU] Refine CPU attention frontend (#45391)

Signed-off-by: jiang1.li <jiang1.li@intel.com>
This commit is contained in:
Li, Jiang
2026-06-14 19:26:54 -07:00
committed by GitHub
parent b675cb7d0f
commit 8760f972ca
4 changed files with 384 additions and 217 deletions
+8 -7
View File
@@ -822,8 +822,8 @@ struct AttentionInput {
logits_buffer_t *__restrict__ logits_buffer, \
float *__restrict__ partial_q_buffer, float *__restrict__ max_buffer, \
float *__restrict__ sum_buffer, int32_t *__restrict__ block_table, \
const int32_t kv_tile_start_pos, const int32_t kv_tile_end_pos, \
const int32_t kv_tile_token_num, \
const int32_t kv_end_pos, const int32_t kv_tile_start_pos, \
const int32_t kv_tile_end_pos, const int32_t kv_tile_token_num, \
const int64_t kv_cache_num_blocks_stride, const int32_t q_head_num, \
const int32_t q_token_num, const int32_t q_tile_start_pos, \
const int32_t q_heads_per_kv, const int32_t block_size, \
@@ -834,7 +834,7 @@ struct AttentionInput {
#define CPU_ATTENTION_PARAMS \
q_heads_buffer, k_head_cache_ptr, v_head_cache_ptr, logits_buffer, \
partial_q_buffer, max_buffer, sum_buffer, block_table, \
partial_q_buffer, max_buffer, sum_buffer, block_table, kv_end_pos, \
kv_tile_start_pos, kv_tile_end_pos, kv_tile_token_num, \
kv_cache_num_blocks_stride, q_head_num, q_token_num, q_tile_start_pos, \
q_heads_per_kv, block_size, left_window_size, right_window_size, scale, \
@@ -917,6 +917,7 @@ class AttentionMainLoop {
// - max_buffer: [MaxQHeadNumPerIteration, 1], store max logits
// - sum_buffer: [MaxQHeadNumPerIteration, 1], store sum of exp
// - block_table
// - kv_end_pos: un-aligned end position of KV cache
// - kv_tile_start_pos: start position of KV cache, aligned to
// BlockSizeAlignment
// - kv_tile_end_pos: end position of KV cache, aligned to
@@ -1043,7 +1044,7 @@ class AttentionMainLoop {
}
apply_mask(logits_buffer, kv_tile_token_num, q_tile_start_pos,
kv_tile_start_pos, kv_tile_end_pos, q_token_num,
kv_end_pos, kv_tile_start_pos, kv_tile_end_pos, q_token_num,
q_heads_per_kv, left_window_size, right_window_size);
// if (debug_info){
@@ -1126,7 +1127,7 @@ class AttentionMainLoop {
void apply_mask(logits_buffer_t* __restrict__ logits_buffer,
const int64_t logits_buffer_stride,
const int32_t q_tile_start_pos,
const int32_t q_tile_start_pos, const int32_t kv_end_pos,
const int32_t kv_tile_start_pos,
const int32_t kv_tile_end_pos, const int32_t q_token_num,
const int32_t q_heads_per_kv,
@@ -1154,7 +1155,7 @@ class AttentionMainLoop {
std::max(kv_tile_start_pos,
curr_token_pos + sliding_window_right + 1));
}
return pos;
return std::min(pos, kv_end_pos);
}();
int32_t left_invalid_token_num = left_kv_pos - kv_tile_start_pos;
@@ -1789,7 +1790,7 @@ class AttentionMainLoop {
attn_impl.template execute_attention<Attention>(
curr_q_heads_buffer, curr_k_cache, curr_v_cache,
logits_buffer, curr_partial_q_buffer, curr_max_buffer,
curr_sum_buffer, curr_block_table,
curr_sum_buffer, curr_block_table, kv_end_pos,
aligned_actual_kv_tile_pos_left,
aligned_actual_kv_tile_pos_right, actual_kv_token_num,
kv_cache_block_num_stride, q_tile_head_num,
+1 -1
View File
@@ -11,7 +11,7 @@ import os
HEAD_DIMS_32 = [32, 64, 96, 128, 160, 192, 224, 256, 512]
# Head dimensions divisible by 16 but not 32 (VEC16 only)
HEAD_DIMS_16 = [80, 112]
HEAD_DIMS_16 = [48, 80, 112]
# ISA types
ISA_TYPES = {
+287 -7
View File
@@ -25,7 +25,6 @@ from vllm._custom_ops import (
if torch.cpu._is_amx_tile_supported():
torch.cpu._init_amx()
NUM_HEADS = [
(4, 4),
(8, 2),
@@ -43,6 +42,11 @@ SEQ_LENS = [ # (q_len, kv_len)
[(2345, 2345), (5, 5), (3, 16), (134, 5131)], # prefill batch
[(992, 2456), (1, 1234), (98, 1145), (1, 4162), (2345, 2345)], # mixed batch
]
_FP8_ATOL = {"fp8_e4m3": 0.2, "fp8_e5m2": 0.3}
_FP8_RTOL = 0.1
ENCODER_SEQ_LENS = [
[1, 678, 2367, 145, 4162, 36, 7812],
]
def get_attn_isa(
@@ -61,10 +65,7 @@ def get_attn_isa(
# rand number generation takes too much time, cache rand tensors
@functools.lru_cache(maxsize=128, typed=False)
def tensor_cache(
elem_num: int,
dtype: torch.dtype,
) -> torch.Tensor:
def tensor_cache(elem_num: int, dtype: torch.dtype, tag: str = "none") -> torch.Tensor:
tensor = torch.randn(elem_num, dtype=dtype)
return tensor
@@ -183,8 +184,222 @@ def ref_paged_attn(
return torch.cat(outputs, dim=0)
_FP8_ATOL = {"fp8_e4m3": 0.2, "fp8_e5m2": 0.3}
_FP8_RTOL = 0.1
def ref_varlen_encoder_attn(
query: torch.Tensor, # [token, q_head_num, head_dim]
key: torch.Tensor, # [token, kv_head_num, head_dim]
value: torch.Tensor,
seq_lens: list[int],
scale: float,
sliding_window: int | None = None,
) -> torch.Tensor:
num_seqs = len(seq_lens)
dtype = query.dtype
output = torch.empty_like(query)
start_idx = 0
for i in range(num_seqs):
seq_len = seq_lens[i]
q = query[start_idx : start_idx + seq_len].float()
k = key[start_idx : start_idx + seq_len].float()
v = value[start_idx : start_idx + seq_len].float()
q *= scale
if q.shape[1] != k.shape[1]:
k = torch.repeat_interleave(k, q.shape[1] // k.shape[1], dim=1)
v = torch.repeat_interleave(v, q.shape[1] // v.shape[1], dim=1)
attn = torch.einsum("qhd,khd->hqk", q, k).float()
empty_mask = torch.ones(seq_len, seq_len)
if sliding_window is not None:
mask = (
torch.triu(empty_mask, diagonal=1 - sliding_window).bool()
^ torch.triu(empty_mask, diagonal=sliding_window).bool()
).logical_not()
else:
mask = empty_mask.logical_not()
attn.masked_fill_(mask, float("-inf"))
attn = torch.softmax(attn, dim=-1)
out = torch.einsum("hqk,khd->qhd", attn, v).to(dtype=dtype)
output[start_idx : start_idx + seq_len].copy_(out)
start_idx += seq_len
return output
@torch.inference_mode()
def varlen_encoder_attention(
seq_lens: list[int],
num_heads: tuple[int, int],
head_size: int,
sliding_window: int | None,
dtype: torch.dtype,
block_size: int,
isa: str,
) -> None:
set_random_seed(0)
num_seqs = len(seq_lens)
num_query_heads = num_heads[0]
num_kv_heads = num_heads[1]
assert num_query_heads % num_kv_heads == 0
window_size = (
(sliding_window - 1, sliding_window - 1)
if sliding_window is not None
else (-1, -1)
)
scale = head_size**-0.5
token_num = sum(seq_lens)
seq_lens_tensor = torch.tensor(seq_lens, dtype=torch.int32)
query_start_loc = torch.zeros(num_seqs, dtype=torch.int32)
torch.cumsum(seq_lens_tensor[:-1], 0, out=query_start_loc[1:])
block_nums = (seq_lens_tensor + block_size - 1) // block_size
start_block_ids = torch.zeros_like(seq_lens_tensor)
torch.cumsum(block_nums[:-1], 0, out=start_block_ids[1:])
total_block_num: int = block_nums.sum().item()
max_block_num = block_nums.max().item()
block_offsets = torch.arange(0, max_block_num, dtype=torch.int32)
encoder_block_table = start_block_ids[:, None] + block_offsets[None, :]
slot_mapping_list = []
slot_start_idx = 0
for i in range(num_seqs):
block_num = block_nums[i].item()
seq_len = seq_lens[i]
slot_mapping_list.append(torch.arange(slot_start_idx, slot_start_idx + seq_len))
slot_start_idx += block_num * block_size
slot_mapping = torch.cat(slot_mapping_list)
query = tensor_cache(
elem_num=token_num * num_query_heads * head_size,
dtype=dtype,
tag="query",
)
query = query.view(
token_num,
num_query_heads,
head_size,
)
key_value = tensor_cache(
elem_num=2 * token_num * num_kv_heads * head_size,
dtype=dtype,
tag="kv",
)
key_value = key_value.view(
2,
token_num,
num_kv_heads,
head_size,
)
key, value = key_value.unbind(0)
# KV cache for CPU attention
packed_key_value_cache = torch.zeros(
total_block_num, num_kv_heads, block_size, head_size * 2, dtype=dtype
)
packed_key_value_cache = packed_key_value_cache.view(
(total_block_num, num_kv_heads, block_size * 2, -1)
)
packed_key_cache, packed_value_cache = packed_key_value_cache.chunk(2, dim=2)
cu_query_lens = torch.tensor([0] + seq_lens, dtype=torch.int32).cumsum(
dim=0, dtype=torch.int32
)
kv_lens_tensor = torch.tensor(seq_lens, dtype=torch.int32)
# use reshape_and_cache to pack key_cache and value_cache
cpu_attn_reshape_and_cache(
key=key.view(-1, num_kv_heads, head_size),
value=value.view(-1, num_kv_heads, head_size),
key_cache=packed_key_cache,
value_cache=packed_value_cache,
slot_mapping=slot_mapping,
isa=isa,
)
metadata = cpu_attn_get_scheduler_metadata(
num_reqs=num_seqs,
num_heads=num_query_heads,
num_kv_heads=num_kv_heads,
head_dim=head_size,
seq_lens=kv_lens_tensor,
dtype=dtype,
query_start_loc=cu_query_lens,
causal=False,
sliding_window_size=sliding_window if sliding_window is not None else -1,
isa=isa,
enable_kv_split=False,
)
out_without_split = torch.empty_like(query)
cpu_attention_with_kv_cache(
query=query,
key_cache=packed_key_cache,
value_cache=packed_value_cache,
output=out_without_split,
query_start_loc=cu_query_lens,
seq_lens=kv_lens_tensor,
scale=scale,
causal=False,
alibi_slopes=None,
sliding_window=window_size,
block_table=encoder_block_table,
softcap=0,
scheduler_metadata=metadata,
s_aux=None,
)
metadata = cpu_attn_get_scheduler_metadata(
num_reqs=num_seqs,
num_heads=num_query_heads,
num_kv_heads=num_kv_heads,
head_dim=head_size,
seq_lens=kv_lens_tensor,
dtype=dtype,
query_start_loc=cu_query_lens,
causal=False,
sliding_window_size=sliding_window if sliding_window is not None else -1,
isa=isa,
enable_kv_split=True,
)
out_with_split = torch.empty_like(query)
cpu_attention_with_kv_cache(
query=query,
key_cache=packed_key_cache,
value_cache=packed_value_cache,
output=out_with_split,
query_start_loc=cu_query_lens,
seq_lens=kv_lens_tensor,
scale=scale,
causal=False,
alibi_slopes=None,
sliding_window=window_size,
block_table=encoder_block_table,
softcap=0,
scheduler_metadata=metadata,
s_aux=None,
)
ref_output = ref_varlen_encoder_attn(
query=query,
key=key,
value=value,
seq_lens=seq_lens,
scale=scale,
sliding_window=sliding_window,
)
atol, rtol = 1.5e-2, 1e-2
(
torch.testing.assert_close(out_with_split, ref_output, atol=atol, rtol=rtol),
f"{torch.max(torch.abs(out_with_split - ref_output))}",
)
(
torch.testing.assert_close(out_without_split, ref_output, atol=atol, rtol=rtol),
f"{torch.max(torch.abs(out_without_split - ref_output))}",
)
@torch.inference_mode()
@@ -418,6 +633,71 @@ def varlen_with_paged_kv(
)
@pytest.mark.parametrize("seq_lens", ENCODER_SEQ_LENS)
@pytest.mark.parametrize("num_heads", NUM_HEADS)
@pytest.mark.parametrize("head_size", HEAD_SIZES)
@pytest.mark.parametrize(
"block_size",
[
128,
],
)
@pytest.mark.parametrize("sliding_window", SLIDING_WINDOWS)
@pytest.mark.parametrize("dtype", QTYPES)
@pytest.mark.parametrize("isa", ["vec"])
def test_varlen_encoder_attention_vec(
seq_lens: list[int],
num_heads: tuple[int, int],
head_size: int,
sliding_window: int | None,
dtype: torch.dtype,
block_size: int,
isa: str,
) -> None:
varlen_encoder_attention(
seq_lens=seq_lens,
num_heads=num_heads,
head_size=head_size,
sliding_window=sliding_window,
dtype=dtype,
block_size=block_size,
isa=isa,
)
@pytest.mark.parametrize("seq_lens", ENCODER_SEQ_LENS)
@pytest.mark.parametrize("num_heads", NUM_HEADS)
@pytest.mark.parametrize("head_size", HEAD_SIZES)
@pytest.mark.parametrize(
"block_size",
[
128,
],
)
@pytest.mark.parametrize("sliding_window", SLIDING_WINDOWS)
@pytest.mark.parametrize("dtype", [torch.bfloat16])
@pytest.mark.parametrize("isa", ["amx"])
@pytest.mark.skipif(not torch.cpu._is_amx_tile_supported(), reason="no AMX support.")
def test_varlen_encoder_attention_amx(
seq_lens: list[int],
num_heads: tuple[int, int],
head_size: int,
sliding_window: int | None,
dtype: torch.dtype,
block_size: int,
isa: str,
) -> None:
varlen_encoder_attention(
seq_lens=seq_lens,
num_heads=num_heads,
head_size=head_size,
sliding_window=sliding_window,
dtype=dtype,
block_size=block_size,
isa=isa,
)
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8_e4m3", "fp8_e5m2"])
@pytest.mark.parametrize("seq_lens", SEQ_LENS)
@pytest.mark.parametrize("num_heads", NUM_HEADS)
+88 -202
View File
@@ -11,7 +11,7 @@ import torch
from vllm import _custom_ops as ops
from vllm import envs
from vllm.config import VllmConfig
from vllm.config import VllmConfig, get_current_vllm_config
from vllm.logger import init_logger
from vllm.platforms import CpuArchEnum, current_platform
from vllm.utils.torch_utils import is_quantized_kv_cache
@@ -26,20 +26,15 @@ from vllm.v1.attention.backend import (
)
from vllm.v1.attention.backends.utils import (
KVCacheLayoutType,
split_decodes_and_prefills,
)
from vllm.v1.kv_cache_interface import AttentionSpec, CrossAttentionSpec
from vllm.v1.kv_cache_interface import (
AttentionSpec,
CrossAttentionSpec,
EncoderOnlyAttentionSpec,
)
logger = init_logger(__name__)
_CPU_ARCH_PREFER_MIXED_BATCH = (
CpuArchEnum.X86,
CpuArchEnum.ARM,
CpuArchEnum.S390X,
CpuArchEnum.RISCV,
CpuArchEnum.POWERPC,
)
class CPUAttentionBackend(AttentionBackend):
forward_includes_kv_cache_update: bool = False
@@ -124,6 +119,8 @@ class CPUAttentionMetadata:
sdpa_attn_masks: list[torch.Tensor | None] | None = None
sdpa_start_loc: torch.Tensor | None = None
encoder_cache: torch.Tensor | None = None
class CPUAttentionMetadataBuilder(AttentionMetadataBuilder[CPUAttentionMetadata]):
def __init__(
@@ -135,17 +132,6 @@ class CPUAttentionMetadataBuilder(AttentionMetadataBuilder[CPUAttentionMetadata]
) -> None:
super().__init__(kv_cache_spec, layer_names, vllm_config, device)
self.use_sdpa_prefill = False
reorder_batch_threshold = None
if current_platform.get_cpu_architecture() not in _CPU_ARCH_PREFER_MIXED_BATCH:
# in this case, decode seqs are reordered to the front of prefill seqs
# to split decode and prefill. Then use SDPA for prefill and
# cpu_attention_with_kv_cache for decode
reorder_batch_threshold = 1
self.use_sdpa_prefill = True
self._init_reorder_batch_threshold(reorder_batch_threshold, False)
self.kv_cache_spec = kv_cache_spec
self.vllm_config = vllm_config
@@ -168,6 +154,9 @@ class CPUAttentionMetadataBuilder(AttentionMetadataBuilder[CPUAttentionMetadata]
kv_cache_dtype_str,
)
self.is_cross_attention = isinstance(kv_cache_spec, CrossAttentionSpec)
self.is_encoder_only_attention = isinstance(
kv_cache_spec, EncoderOnlyAttentionSpec
)
def build(
self,
@@ -185,23 +174,34 @@ class CPUAttentionMetadataBuilder(AttentionMetadataBuilder[CPUAttentionMetadata]
slot_mapping = common_attn_metadata.slot_mapping
causal = False if self.is_cross_attention else common_attn_metadata.causal
sdpa_start_loc = query_start_loc
num_decode_tokens = 0
if self.use_sdpa_prefill and causal:
# Decoder, need reorder and truncate
assert self.reorder_batch_threshold
(num_decodes, num_prefills, num_decode_tokens, num_prefill_tokens) = (
split_decodes_and_prefills(
common_attn_metadata,
decode_threshold=self.reorder_batch_threshold,
require_uniform=True,
)
encoder_cache_tensor = None
if self.is_encoder_only_attention:
block_nums = (seq_lens + self.block_size - 1) // self.block_size
start_block_ids = torch.zeros_like(seq_lens)
torch.cumsum(block_nums[:-1], 0, out=start_block_ids[1:])
total_block_num: int = block_nums.sum().item()
max_block_num = block_nums.max().item()
block_offsets = torch.arange(
0, max_block_num, dtype=block_table_tensor.dtype
)
num_reqs = num_decodes
sdpa_start_loc = sdpa_start_loc[num_decodes:] - num_decode_tokens
seq_lens = seq_lens[:num_decodes]
query_start_loc = query_start_loc[: num_decodes + 1]
block_table_tensor = block_table_tensor[:num_decodes]
encoder_block_table = start_block_ids[:, None] + block_offsets[None, :]
torch.ops._C.compute_slot_mapping_kernel_impl(
query_start_loc,
common_attn_metadata.positions,
encoder_block_table,
slot_mapping,
self.block_size,
)
encoder_cache_tensor = torch.zeros(
(
total_block_num,
self.num_kv_heads,
self.block_size,
2 * self.head_dim,
),
dtype=self.dtype,
)
block_table_tensor = encoder_block_table
scheduler_metadata = ops.cpu_attn_get_scheduler_metadata(
num_reqs=num_reqs,
@@ -227,9 +227,7 @@ class CPUAttentionMetadataBuilder(AttentionMetadataBuilder[CPUAttentionMetadata]
slot_mapping=slot_mapping,
scheduler_metadata=scheduler_metadata,
causal=causal,
use_sdpa_prefill=self.use_sdpa_prefill,
num_decode_tokens=num_decode_tokens,
sdpa_start_loc=sdpa_start_loc,
encoder_cache=encoder_cache_tensor,
)
return attn_metadata
@@ -289,6 +287,14 @@ class CPUAttentionBackendImpl(AttentionImpl):
"heads in the layer"
)
vllm_config = get_current_vllm_config()
self.isa = _get_attn_isa(
vllm_config.model_config.dtype,
vllm_config.cache_config.block_size,
self.head_size,
self.kv_cache_dtype,
)
def forward(
self,
layer: AttentionLayer,
@@ -325,60 +331,58 @@ class CPUAttentionBackendImpl(AttentionImpl):
num_actual_tokens = attn_metadata.num_actual_tokens
# Handle encoder attention differently - no KV cache needed
# For encoder attention
if self.attn_type in (AttentionType.ENCODER_ONLY, AttentionType.ENCODER):
# For encoder attention,
return self._run_sdpa_forward(
query[:num_actual_tokens],
key[:num_actual_tokens],
value[:num_actual_tokens],
output[:num_actual_tokens],
attn_metadata,
self.attn_type,
)
kv_cache = attn_metadata.encoder_cache
# For decoder and cross-attention, use KV cache, size are
# [num_blocks, num_kv_heads, block_size, 2 * head_size]
# Make a view [num_blocks, num_kv_heads, block_size * 2, head_size]
# Then slice KV at dim 2
# KV cache size are [num_blocks, num_kv_heads, block_size,
# 2 * head_size]. Make a view [num_blocks, num_kv_heads,
# block_size * 2, head_size]. Then slice KV at dim 2
num_blocks, num_kv_heads, block_size, _ = kv_cache.size()
kv_cache = kv_cache.view((num_blocks, num_kv_heads, block_size * 2, -1))
key_cache, value_cache = kv_cache.chunk(2, dim=2)
if attn_metadata.use_sdpa_prefill:
assert self.sinks is None, "Attention sink is unsupported in SDPA prefill"
num_decode_tokens = attn_metadata.num_decode_tokens
self._run_sdpa_forward(
query[num_decode_tokens:num_actual_tokens],
key[num_decode_tokens:num_actual_tokens],
value[num_decode_tokens:num_actual_tokens],
output[num_decode_tokens:num_actual_tokens],
attn_metadata,
self.attn_type,
)
num_actual_tokens = num_decode_tokens
if num_actual_tokens > 0:
ops.cpu_attention_with_kv_cache(
query=query[:num_actual_tokens],
key_cache=key_cache,
value_cache=value_cache,
output=output[:num_actual_tokens], # type: ignore
query_start_loc=attn_metadata.query_start_loc,
seq_lens=attn_metadata.seq_lens,
scale=self.scale,
causal=attn_metadata.causal,
alibi_slopes=self.alibi_slopes, # type: ignore
sliding_window=self.sliding_window,
block_table=attn_metadata.block_table,
softcap=self.logits_soft_cap,
scheduler_metadata=attn_metadata.scheduler_metadata,
s_aux=self.sinks,
# key and value may be None in the case of cross attention. They are
# calculated once based on the output from the encoder and then cached
# in KV cache.
if (
self.kv_sharing_target_layer_name is None
and key is not None
and value is not None
):
ops.cpu_attn_reshape_and_cache(
key,
value,
key_cache,
value_cache,
attn_metadata.slot_mapping,
self.isa,
k_scale=layer._k_scale_float,
v_scale=layer._v_scale_float,
kv_cache_dtype=self.kv_cache_dtype,
)
ops.cpu_attention_with_kv_cache(
query=query[:num_actual_tokens],
key_cache=key_cache,
value_cache=value_cache,
output=output[:num_actual_tokens], # type: ignore
query_start_loc=attn_metadata.query_start_loc,
seq_lens=attn_metadata.seq_lens,
scale=self.scale,
causal=attn_metadata.causal,
alibi_slopes=self.alibi_slopes, # type: ignore
sliding_window=self.sliding_window,
block_table=attn_metadata.block_table,
softcap=self.logits_soft_cap,
scheduler_metadata=attn_metadata.scheduler_metadata,
s_aux=self.sinks,
k_scale=layer._k_scale_float,
v_scale=layer._v_scale_float,
kv_cache_dtype=self.kv_cache_dtype,
)
return output
def do_kv_cache_update(
@@ -395,136 +399,18 @@ class CPUAttentionBackendImpl(AttentionImpl):
num_blocks, num_kv_heads, block_size, _ = kv_cache.size()
kv_cache = kv_cache.view((num_blocks, num_kv_heads, block_size * 2, -1))
key_cache, value_cache = kv_cache.chunk(2, dim=2)
isa = _get_attn_isa(
key.dtype, key_cache.shape[2], self.head_size, self.kv_cache_dtype
)
ops.cpu_attn_reshape_and_cache(
key,
value,
key_cache,
value_cache,
slot_mapping,
isa,
self.isa,
k_scale=layer._k_scale_float,
v_scale=layer._v_scale_float,
kv_cache_dtype=self.kv_cache_dtype,
)
def _run_sdpa_forward(
self,
query: torch.Tensor,
key: torch.Tensor,
value: torch.Tensor,
output: torch.Tensor,
attn_metadata: CPUAttentionMetadata,
attn_type: str,
) -> torch.Tensor:
attn_masks = attn_metadata.sdpa_attn_masks
if attn_masks is None:
if self.alibi_slopes is not None:
attn_masks = _make_alibi_bias(
self.alibi_slopes,
query.dtype,
attn_metadata.sdpa_start_loc,
)
elif self.sliding_window[0] != -1 or self.sliding_window[1] != -1:
assert attn_metadata.seq_lens is not None
attn_masks = _make_sliding_window_bias(
attn_metadata.sdpa_start_loc,
self.sliding_window[0],
self.sliding_window[1],
query.dtype,
)
else:
attn_masks = [None] * (attn_metadata.sdpa_start_loc.size(0) - 1) # type: ignore
attn_metadata.sdpa_attn_masks = attn_masks
query = query.movedim(0, query.dim() - 2)
key = key.movedim(0, key.dim() - 2)
value = value.movedim(0, value.dim() - 2)
causal_attn = attn_type == AttentionType.DECODER
sdpa_start_loc = attn_metadata.sdpa_start_loc.numpy() # type: ignore
for i in range(len(attn_masks)):
mask = attn_masks[i]
start_q = sdpa_start_loc[i]
end_q = sdpa_start_loc[i + 1]
sub_out = (
torch.nn.functional.scaled_dot_product_attention(
query[None, :, start_q:end_q, :],
key[None, :, start_q:end_q, :],
value[None, :, start_q:end_q, :],
attn_mask=mask,
dropout_p=0.0,
is_causal=causal_attn and mask is None,
scale=self.scale,
enable_gqa=self.num_heads > self.num_kv_heads,
)
.squeeze(0)
.movedim(query.dim() - 2, 0)
)
output[start_q:end_q, :, :] = sub_out
return output
def _make_alibi_bias(
alibi_slopes: torch.Tensor,
dtype: torch.dtype,
sdpa_start_loc: torch.Tensor,
) -> list[torch.Tensor]:
attn_biases: list[torch.Tensor] = []
seq_num = sdpa_start_loc.size(0) - 1
sdpa_start_loc = sdpa_start_loc.numpy() # type: ignore
for i in range(seq_num):
seq_len = sdpa_start_loc[i + 1] - sdpa_start_loc[i]
bias = torch.arange(seq_len, dtype=dtype) # type: ignore
# NOTE(zhuohan): HF uses
# `bias = bias[None, :].repeat(seq_len, 1)`
# here. We find that both biases give the same results, but
# the bias below more accurately follows the original ALiBi
# paper.
bias = bias[None, :] - bias[:, None]
num_heads = alibi_slopes.shape[0]
bias = bias[None, :].repeat((num_heads, 1, 1))
bias.mul_(alibi_slopes[:, None, None]).unsqueeze_(0)
inf_mask = (
torch.empty((1, seq_len, seq_len), dtype=bias.dtype) # type: ignore
.fill_(-torch.inf)
.triu_(diagonal=1)
)
attn_biases.append((bias + inf_mask).to(dtype))
return attn_biases
def _make_sliding_window_bias(
sdpa_start_loc: torch.Tensor,
left_window_size: int,
right_window_size: int,
dtype: torch.dtype,
) -> list[torch.Tensor]:
attn_biases: list[torch.Tensor] = []
seq_num = sdpa_start_loc.size(0) - 1
sdpa_start_loc = sdpa_start_loc.numpy() # type: ignore
for i in range(seq_num):
seq_len = sdpa_start_loc[i + 1] - sdpa_start_loc[i]
mask = torch.full( # type: ignore
(1, seq_len, seq_len), # type: ignore
fill_value=1,
dtype=dtype,
)
if right_window_size != -1:
mask = torch.tril(mask, diagonal=right_window_size)
if left_window_size != -1:
mask = torch.triu(mask, diagonal=-left_window_size)
mask = torch.log(mask)
attn_biases.append(mask)
return attn_biases
@functools.lru_cache(maxsize=1)
def _riscv_supports_rvv() -> bool: