[CPU] Refine CPU attention frontend (#45391)
Signed-off-by: jiang1.li <jiang1.li@intel.com>
This commit is contained in:
@@ -822,8 +822,8 @@ struct AttentionInput {
|
||||
logits_buffer_t *__restrict__ logits_buffer, \
|
||||
float *__restrict__ partial_q_buffer, float *__restrict__ max_buffer, \
|
||||
float *__restrict__ sum_buffer, int32_t *__restrict__ block_table, \
|
||||
const int32_t kv_tile_start_pos, const int32_t kv_tile_end_pos, \
|
||||
const int32_t kv_tile_token_num, \
|
||||
const int32_t kv_end_pos, const int32_t kv_tile_start_pos, \
|
||||
const int32_t kv_tile_end_pos, const int32_t kv_tile_token_num, \
|
||||
const int64_t kv_cache_num_blocks_stride, const int32_t q_head_num, \
|
||||
const int32_t q_token_num, const int32_t q_tile_start_pos, \
|
||||
const int32_t q_heads_per_kv, const int32_t block_size, \
|
||||
@@ -834,7 +834,7 @@ struct AttentionInput {
|
||||
|
||||
#define CPU_ATTENTION_PARAMS \
|
||||
q_heads_buffer, k_head_cache_ptr, v_head_cache_ptr, logits_buffer, \
|
||||
partial_q_buffer, max_buffer, sum_buffer, block_table, \
|
||||
partial_q_buffer, max_buffer, sum_buffer, block_table, kv_end_pos, \
|
||||
kv_tile_start_pos, kv_tile_end_pos, kv_tile_token_num, \
|
||||
kv_cache_num_blocks_stride, q_head_num, q_token_num, q_tile_start_pos, \
|
||||
q_heads_per_kv, block_size, left_window_size, right_window_size, scale, \
|
||||
@@ -917,6 +917,7 @@ class AttentionMainLoop {
|
||||
// - max_buffer: [MaxQHeadNumPerIteration, 1], store max logits
|
||||
// - sum_buffer: [MaxQHeadNumPerIteration, 1], store sum of exp
|
||||
// - block_table
|
||||
// - kv_end_pos: un-aligned end position of KV cache
|
||||
// - kv_tile_start_pos: start position of KV cache, aligned to
|
||||
// BlockSizeAlignment
|
||||
// - kv_tile_end_pos: end position of KV cache, aligned to
|
||||
@@ -1043,7 +1044,7 @@ class AttentionMainLoop {
|
||||
}
|
||||
|
||||
apply_mask(logits_buffer, kv_tile_token_num, q_tile_start_pos,
|
||||
kv_tile_start_pos, kv_tile_end_pos, q_token_num,
|
||||
kv_end_pos, kv_tile_start_pos, kv_tile_end_pos, q_token_num,
|
||||
q_heads_per_kv, left_window_size, right_window_size);
|
||||
|
||||
// if (debug_info){
|
||||
@@ -1126,7 +1127,7 @@ class AttentionMainLoop {
|
||||
|
||||
void apply_mask(logits_buffer_t* __restrict__ logits_buffer,
|
||||
const int64_t logits_buffer_stride,
|
||||
const int32_t q_tile_start_pos,
|
||||
const int32_t q_tile_start_pos, const int32_t kv_end_pos,
|
||||
const int32_t kv_tile_start_pos,
|
||||
const int32_t kv_tile_end_pos, const int32_t q_token_num,
|
||||
const int32_t q_heads_per_kv,
|
||||
@@ -1154,7 +1155,7 @@ class AttentionMainLoop {
|
||||
std::max(kv_tile_start_pos,
|
||||
curr_token_pos + sliding_window_right + 1));
|
||||
}
|
||||
return pos;
|
||||
return std::min(pos, kv_end_pos);
|
||||
}();
|
||||
|
||||
int32_t left_invalid_token_num = left_kv_pos - kv_tile_start_pos;
|
||||
@@ -1789,7 +1790,7 @@ class AttentionMainLoop {
|
||||
attn_impl.template execute_attention<Attention>(
|
||||
curr_q_heads_buffer, curr_k_cache, curr_v_cache,
|
||||
logits_buffer, curr_partial_q_buffer, curr_max_buffer,
|
||||
curr_sum_buffer, curr_block_table,
|
||||
curr_sum_buffer, curr_block_table, kv_end_pos,
|
||||
aligned_actual_kv_tile_pos_left,
|
||||
aligned_actual_kv_tile_pos_right, actual_kv_token_num,
|
||||
kv_cache_block_num_stride, q_tile_head_num,
|
||||
|
||||
@@ -11,7 +11,7 @@ import os
|
||||
HEAD_DIMS_32 = [32, 64, 96, 128, 160, 192, 224, 256, 512]
|
||||
|
||||
# Head dimensions divisible by 16 but not 32 (VEC16 only)
|
||||
HEAD_DIMS_16 = [80, 112]
|
||||
HEAD_DIMS_16 = [48, 80, 112]
|
||||
|
||||
# ISA types
|
||||
ISA_TYPES = {
|
||||
|
||||
@@ -25,7 +25,6 @@ from vllm._custom_ops import (
|
||||
if torch.cpu._is_amx_tile_supported():
|
||||
torch.cpu._init_amx()
|
||||
|
||||
|
||||
NUM_HEADS = [
|
||||
(4, 4),
|
||||
(8, 2),
|
||||
@@ -43,6 +42,11 @@ SEQ_LENS = [ # (q_len, kv_len)
|
||||
[(2345, 2345), (5, 5), (3, 16), (134, 5131)], # prefill batch
|
||||
[(992, 2456), (1, 1234), (98, 1145), (1, 4162), (2345, 2345)], # mixed batch
|
||||
]
|
||||
_FP8_ATOL = {"fp8_e4m3": 0.2, "fp8_e5m2": 0.3}
|
||||
_FP8_RTOL = 0.1
|
||||
ENCODER_SEQ_LENS = [
|
||||
[1, 678, 2367, 145, 4162, 36, 7812],
|
||||
]
|
||||
|
||||
|
||||
def get_attn_isa(
|
||||
@@ -61,10 +65,7 @@ def get_attn_isa(
|
||||
|
||||
# rand number generation takes too much time, cache rand tensors
|
||||
@functools.lru_cache(maxsize=128, typed=False)
|
||||
def tensor_cache(
|
||||
elem_num: int,
|
||||
dtype: torch.dtype,
|
||||
) -> torch.Tensor:
|
||||
def tensor_cache(elem_num: int, dtype: torch.dtype, tag: str = "none") -> torch.Tensor:
|
||||
tensor = torch.randn(elem_num, dtype=dtype)
|
||||
|
||||
return tensor
|
||||
@@ -183,8 +184,222 @@ def ref_paged_attn(
|
||||
return torch.cat(outputs, dim=0)
|
||||
|
||||
|
||||
_FP8_ATOL = {"fp8_e4m3": 0.2, "fp8_e5m2": 0.3}
|
||||
_FP8_RTOL = 0.1
|
||||
def ref_varlen_encoder_attn(
|
||||
query: torch.Tensor, # [token, q_head_num, head_dim]
|
||||
key: torch.Tensor, # [token, kv_head_num, head_dim]
|
||||
value: torch.Tensor,
|
||||
seq_lens: list[int],
|
||||
scale: float,
|
||||
sliding_window: int | None = None,
|
||||
) -> torch.Tensor:
|
||||
num_seqs = len(seq_lens)
|
||||
dtype = query.dtype
|
||||
|
||||
output = torch.empty_like(query)
|
||||
|
||||
start_idx = 0
|
||||
for i in range(num_seqs):
|
||||
seq_len = seq_lens[i]
|
||||
q = query[start_idx : start_idx + seq_len].float()
|
||||
k = key[start_idx : start_idx + seq_len].float()
|
||||
v = value[start_idx : start_idx + seq_len].float()
|
||||
q *= scale
|
||||
|
||||
if q.shape[1] != k.shape[1]:
|
||||
k = torch.repeat_interleave(k, q.shape[1] // k.shape[1], dim=1)
|
||||
v = torch.repeat_interleave(v, q.shape[1] // v.shape[1], dim=1)
|
||||
attn = torch.einsum("qhd,khd->hqk", q, k).float()
|
||||
empty_mask = torch.ones(seq_len, seq_len)
|
||||
if sliding_window is not None:
|
||||
mask = (
|
||||
torch.triu(empty_mask, diagonal=1 - sliding_window).bool()
|
||||
^ torch.triu(empty_mask, diagonal=sliding_window).bool()
|
||||
).logical_not()
|
||||
else:
|
||||
mask = empty_mask.logical_not()
|
||||
|
||||
attn.masked_fill_(mask, float("-inf"))
|
||||
attn = torch.softmax(attn, dim=-1)
|
||||
out = torch.einsum("hqk,khd->qhd", attn, v).to(dtype=dtype)
|
||||
output[start_idx : start_idx + seq_len].copy_(out)
|
||||
|
||||
start_idx += seq_len
|
||||
|
||||
return output
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def varlen_encoder_attention(
|
||||
seq_lens: list[int],
|
||||
num_heads: tuple[int, int],
|
||||
head_size: int,
|
||||
sliding_window: int | None,
|
||||
dtype: torch.dtype,
|
||||
block_size: int,
|
||||
isa: str,
|
||||
) -> None:
|
||||
set_random_seed(0)
|
||||
num_seqs = len(seq_lens)
|
||||
num_query_heads = num_heads[0]
|
||||
num_kv_heads = num_heads[1]
|
||||
assert num_query_heads % num_kv_heads == 0
|
||||
window_size = (
|
||||
(sliding_window - 1, sliding_window - 1)
|
||||
if sliding_window is not None
|
||||
else (-1, -1)
|
||||
)
|
||||
scale = head_size**-0.5
|
||||
token_num = sum(seq_lens)
|
||||
|
||||
seq_lens_tensor = torch.tensor(seq_lens, dtype=torch.int32)
|
||||
query_start_loc = torch.zeros(num_seqs, dtype=torch.int32)
|
||||
torch.cumsum(seq_lens_tensor[:-1], 0, out=query_start_loc[1:])
|
||||
block_nums = (seq_lens_tensor + block_size - 1) // block_size
|
||||
start_block_ids = torch.zeros_like(seq_lens_tensor)
|
||||
torch.cumsum(block_nums[:-1], 0, out=start_block_ids[1:])
|
||||
total_block_num: int = block_nums.sum().item()
|
||||
max_block_num = block_nums.max().item()
|
||||
block_offsets = torch.arange(0, max_block_num, dtype=torch.int32)
|
||||
encoder_block_table = start_block_ids[:, None] + block_offsets[None, :]
|
||||
slot_mapping_list = []
|
||||
slot_start_idx = 0
|
||||
for i in range(num_seqs):
|
||||
block_num = block_nums[i].item()
|
||||
seq_len = seq_lens[i]
|
||||
slot_mapping_list.append(torch.arange(slot_start_idx, slot_start_idx + seq_len))
|
||||
slot_start_idx += block_num * block_size
|
||||
slot_mapping = torch.cat(slot_mapping_list)
|
||||
|
||||
query = tensor_cache(
|
||||
elem_num=token_num * num_query_heads * head_size,
|
||||
dtype=dtype,
|
||||
tag="query",
|
||||
)
|
||||
query = query.view(
|
||||
token_num,
|
||||
num_query_heads,
|
||||
head_size,
|
||||
)
|
||||
|
||||
key_value = tensor_cache(
|
||||
elem_num=2 * token_num * num_kv_heads * head_size,
|
||||
dtype=dtype,
|
||||
tag="kv",
|
||||
)
|
||||
key_value = key_value.view(
|
||||
2,
|
||||
token_num,
|
||||
num_kv_heads,
|
||||
head_size,
|
||||
)
|
||||
key, value = key_value.unbind(0)
|
||||
|
||||
# KV cache for CPU attention
|
||||
packed_key_value_cache = torch.zeros(
|
||||
total_block_num, num_kv_heads, block_size, head_size * 2, dtype=dtype
|
||||
)
|
||||
packed_key_value_cache = packed_key_value_cache.view(
|
||||
(total_block_num, num_kv_heads, block_size * 2, -1)
|
||||
)
|
||||
packed_key_cache, packed_value_cache = packed_key_value_cache.chunk(2, dim=2)
|
||||
|
||||
cu_query_lens = torch.tensor([0] + seq_lens, dtype=torch.int32).cumsum(
|
||||
dim=0, dtype=torch.int32
|
||||
)
|
||||
kv_lens_tensor = torch.tensor(seq_lens, dtype=torch.int32)
|
||||
|
||||
# use reshape_and_cache to pack key_cache and value_cache
|
||||
cpu_attn_reshape_and_cache(
|
||||
key=key.view(-1, num_kv_heads, head_size),
|
||||
value=value.view(-1, num_kv_heads, head_size),
|
||||
key_cache=packed_key_cache,
|
||||
value_cache=packed_value_cache,
|
||||
slot_mapping=slot_mapping,
|
||||
isa=isa,
|
||||
)
|
||||
|
||||
metadata = cpu_attn_get_scheduler_metadata(
|
||||
num_reqs=num_seqs,
|
||||
num_heads=num_query_heads,
|
||||
num_kv_heads=num_kv_heads,
|
||||
head_dim=head_size,
|
||||
seq_lens=kv_lens_tensor,
|
||||
dtype=dtype,
|
||||
query_start_loc=cu_query_lens,
|
||||
causal=False,
|
||||
sliding_window_size=sliding_window if sliding_window is not None else -1,
|
||||
isa=isa,
|
||||
enable_kv_split=False,
|
||||
)
|
||||
|
||||
out_without_split = torch.empty_like(query)
|
||||
cpu_attention_with_kv_cache(
|
||||
query=query,
|
||||
key_cache=packed_key_cache,
|
||||
value_cache=packed_value_cache,
|
||||
output=out_without_split,
|
||||
query_start_loc=cu_query_lens,
|
||||
seq_lens=kv_lens_tensor,
|
||||
scale=scale,
|
||||
causal=False,
|
||||
alibi_slopes=None,
|
||||
sliding_window=window_size,
|
||||
block_table=encoder_block_table,
|
||||
softcap=0,
|
||||
scheduler_metadata=metadata,
|
||||
s_aux=None,
|
||||
)
|
||||
|
||||
metadata = cpu_attn_get_scheduler_metadata(
|
||||
num_reqs=num_seqs,
|
||||
num_heads=num_query_heads,
|
||||
num_kv_heads=num_kv_heads,
|
||||
head_dim=head_size,
|
||||
seq_lens=kv_lens_tensor,
|
||||
dtype=dtype,
|
||||
query_start_loc=cu_query_lens,
|
||||
causal=False,
|
||||
sliding_window_size=sliding_window if sliding_window is not None else -1,
|
||||
isa=isa,
|
||||
enable_kv_split=True,
|
||||
)
|
||||
|
||||
out_with_split = torch.empty_like(query)
|
||||
cpu_attention_with_kv_cache(
|
||||
query=query,
|
||||
key_cache=packed_key_cache,
|
||||
value_cache=packed_value_cache,
|
||||
output=out_with_split,
|
||||
query_start_loc=cu_query_lens,
|
||||
seq_lens=kv_lens_tensor,
|
||||
scale=scale,
|
||||
causal=False,
|
||||
alibi_slopes=None,
|
||||
sliding_window=window_size,
|
||||
block_table=encoder_block_table,
|
||||
softcap=0,
|
||||
scheduler_metadata=metadata,
|
||||
s_aux=None,
|
||||
)
|
||||
|
||||
ref_output = ref_varlen_encoder_attn(
|
||||
query=query,
|
||||
key=key,
|
||||
value=value,
|
||||
seq_lens=seq_lens,
|
||||
scale=scale,
|
||||
sliding_window=sliding_window,
|
||||
)
|
||||
atol, rtol = 1.5e-2, 1e-2
|
||||
|
||||
(
|
||||
torch.testing.assert_close(out_with_split, ref_output, atol=atol, rtol=rtol),
|
||||
f"{torch.max(torch.abs(out_with_split - ref_output))}",
|
||||
)
|
||||
(
|
||||
torch.testing.assert_close(out_without_split, ref_output, atol=atol, rtol=rtol),
|
||||
f"{torch.max(torch.abs(out_without_split - ref_output))}",
|
||||
)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
@@ -418,6 +633,71 @@ def varlen_with_paged_kv(
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("seq_lens", ENCODER_SEQ_LENS)
|
||||
@pytest.mark.parametrize("num_heads", NUM_HEADS)
|
||||
@pytest.mark.parametrize("head_size", HEAD_SIZES)
|
||||
@pytest.mark.parametrize(
|
||||
"block_size",
|
||||
[
|
||||
128,
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("sliding_window", SLIDING_WINDOWS)
|
||||
@pytest.mark.parametrize("dtype", QTYPES)
|
||||
@pytest.mark.parametrize("isa", ["vec"])
|
||||
def test_varlen_encoder_attention_vec(
|
||||
seq_lens: list[int],
|
||||
num_heads: tuple[int, int],
|
||||
head_size: int,
|
||||
sliding_window: int | None,
|
||||
dtype: torch.dtype,
|
||||
block_size: int,
|
||||
isa: str,
|
||||
) -> None:
|
||||
varlen_encoder_attention(
|
||||
seq_lens=seq_lens,
|
||||
num_heads=num_heads,
|
||||
head_size=head_size,
|
||||
sliding_window=sliding_window,
|
||||
dtype=dtype,
|
||||
block_size=block_size,
|
||||
isa=isa,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("seq_lens", ENCODER_SEQ_LENS)
|
||||
@pytest.mark.parametrize("num_heads", NUM_HEADS)
|
||||
@pytest.mark.parametrize("head_size", HEAD_SIZES)
|
||||
@pytest.mark.parametrize(
|
||||
"block_size",
|
||||
[
|
||||
128,
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("sliding_window", SLIDING_WINDOWS)
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16])
|
||||
@pytest.mark.parametrize("isa", ["amx"])
|
||||
@pytest.mark.skipif(not torch.cpu._is_amx_tile_supported(), reason="no AMX support.")
|
||||
def test_varlen_encoder_attention_amx(
|
||||
seq_lens: list[int],
|
||||
num_heads: tuple[int, int],
|
||||
head_size: int,
|
||||
sliding_window: int | None,
|
||||
dtype: torch.dtype,
|
||||
block_size: int,
|
||||
isa: str,
|
||||
) -> None:
|
||||
varlen_encoder_attention(
|
||||
seq_lens=seq_lens,
|
||||
num_heads=num_heads,
|
||||
head_size=head_size,
|
||||
sliding_window=sliding_window,
|
||||
dtype=dtype,
|
||||
block_size=block_size,
|
||||
isa=isa,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8_e4m3", "fp8_e5m2"])
|
||||
@pytest.mark.parametrize("seq_lens", SEQ_LENS)
|
||||
@pytest.mark.parametrize("num_heads", NUM_HEADS)
|
||||
|
||||
@@ -11,7 +11,7 @@ import torch
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm import envs
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config import VllmConfig, get_current_vllm_config
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import CpuArchEnum, current_platform
|
||||
from vllm.utils.torch_utils import is_quantized_kv_cache
|
||||
@@ -26,20 +26,15 @@ from vllm.v1.attention.backend import (
|
||||
)
|
||||
from vllm.v1.attention.backends.utils import (
|
||||
KVCacheLayoutType,
|
||||
split_decodes_and_prefills,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import AttentionSpec, CrossAttentionSpec
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
AttentionSpec,
|
||||
CrossAttentionSpec,
|
||||
EncoderOnlyAttentionSpec,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_CPU_ARCH_PREFER_MIXED_BATCH = (
|
||||
CpuArchEnum.X86,
|
||||
CpuArchEnum.ARM,
|
||||
CpuArchEnum.S390X,
|
||||
CpuArchEnum.RISCV,
|
||||
CpuArchEnum.POWERPC,
|
||||
)
|
||||
|
||||
|
||||
class CPUAttentionBackend(AttentionBackend):
|
||||
forward_includes_kv_cache_update: bool = False
|
||||
@@ -124,6 +119,8 @@ class CPUAttentionMetadata:
|
||||
sdpa_attn_masks: list[torch.Tensor | None] | None = None
|
||||
sdpa_start_loc: torch.Tensor | None = None
|
||||
|
||||
encoder_cache: torch.Tensor | None = None
|
||||
|
||||
|
||||
class CPUAttentionMetadataBuilder(AttentionMetadataBuilder[CPUAttentionMetadata]):
|
||||
def __init__(
|
||||
@@ -135,17 +132,6 @@ class CPUAttentionMetadataBuilder(AttentionMetadataBuilder[CPUAttentionMetadata]
|
||||
) -> None:
|
||||
super().__init__(kv_cache_spec, layer_names, vllm_config, device)
|
||||
|
||||
self.use_sdpa_prefill = False
|
||||
reorder_batch_threshold = None
|
||||
if current_platform.get_cpu_architecture() not in _CPU_ARCH_PREFER_MIXED_BATCH:
|
||||
# in this case, decode seqs are reordered to the front of prefill seqs
|
||||
# to split decode and prefill. Then use SDPA for prefill and
|
||||
# cpu_attention_with_kv_cache for decode
|
||||
reorder_batch_threshold = 1
|
||||
self.use_sdpa_prefill = True
|
||||
|
||||
self._init_reorder_batch_threshold(reorder_batch_threshold, False)
|
||||
|
||||
self.kv_cache_spec = kv_cache_spec
|
||||
self.vllm_config = vllm_config
|
||||
|
||||
@@ -168,6 +154,9 @@ class CPUAttentionMetadataBuilder(AttentionMetadataBuilder[CPUAttentionMetadata]
|
||||
kv_cache_dtype_str,
|
||||
)
|
||||
self.is_cross_attention = isinstance(kv_cache_spec, CrossAttentionSpec)
|
||||
self.is_encoder_only_attention = isinstance(
|
||||
kv_cache_spec, EncoderOnlyAttentionSpec
|
||||
)
|
||||
|
||||
def build(
|
||||
self,
|
||||
@@ -185,23 +174,34 @@ class CPUAttentionMetadataBuilder(AttentionMetadataBuilder[CPUAttentionMetadata]
|
||||
slot_mapping = common_attn_metadata.slot_mapping
|
||||
causal = False if self.is_cross_attention else common_attn_metadata.causal
|
||||
|
||||
sdpa_start_loc = query_start_loc
|
||||
num_decode_tokens = 0
|
||||
if self.use_sdpa_prefill and causal:
|
||||
# Decoder, need reorder and truncate
|
||||
assert self.reorder_batch_threshold
|
||||
(num_decodes, num_prefills, num_decode_tokens, num_prefill_tokens) = (
|
||||
split_decodes_and_prefills(
|
||||
common_attn_metadata,
|
||||
decode_threshold=self.reorder_batch_threshold,
|
||||
require_uniform=True,
|
||||
)
|
||||
encoder_cache_tensor = None
|
||||
if self.is_encoder_only_attention:
|
||||
block_nums = (seq_lens + self.block_size - 1) // self.block_size
|
||||
start_block_ids = torch.zeros_like(seq_lens)
|
||||
torch.cumsum(block_nums[:-1], 0, out=start_block_ids[1:])
|
||||
total_block_num: int = block_nums.sum().item()
|
||||
max_block_num = block_nums.max().item()
|
||||
block_offsets = torch.arange(
|
||||
0, max_block_num, dtype=block_table_tensor.dtype
|
||||
)
|
||||
num_reqs = num_decodes
|
||||
sdpa_start_loc = sdpa_start_loc[num_decodes:] - num_decode_tokens
|
||||
seq_lens = seq_lens[:num_decodes]
|
||||
query_start_loc = query_start_loc[: num_decodes + 1]
|
||||
block_table_tensor = block_table_tensor[:num_decodes]
|
||||
encoder_block_table = start_block_ids[:, None] + block_offsets[None, :]
|
||||
torch.ops._C.compute_slot_mapping_kernel_impl(
|
||||
query_start_loc,
|
||||
common_attn_metadata.positions,
|
||||
encoder_block_table,
|
||||
slot_mapping,
|
||||
self.block_size,
|
||||
)
|
||||
encoder_cache_tensor = torch.zeros(
|
||||
(
|
||||
total_block_num,
|
||||
self.num_kv_heads,
|
||||
self.block_size,
|
||||
2 * self.head_dim,
|
||||
),
|
||||
dtype=self.dtype,
|
||||
)
|
||||
block_table_tensor = encoder_block_table
|
||||
|
||||
scheduler_metadata = ops.cpu_attn_get_scheduler_metadata(
|
||||
num_reqs=num_reqs,
|
||||
@@ -227,9 +227,7 @@ class CPUAttentionMetadataBuilder(AttentionMetadataBuilder[CPUAttentionMetadata]
|
||||
slot_mapping=slot_mapping,
|
||||
scheduler_metadata=scheduler_metadata,
|
||||
causal=causal,
|
||||
use_sdpa_prefill=self.use_sdpa_prefill,
|
||||
num_decode_tokens=num_decode_tokens,
|
||||
sdpa_start_loc=sdpa_start_loc,
|
||||
encoder_cache=encoder_cache_tensor,
|
||||
)
|
||||
|
||||
return attn_metadata
|
||||
@@ -289,6 +287,14 @@ class CPUAttentionBackendImpl(AttentionImpl):
|
||||
"heads in the layer"
|
||||
)
|
||||
|
||||
vllm_config = get_current_vllm_config()
|
||||
self.isa = _get_attn_isa(
|
||||
vllm_config.model_config.dtype,
|
||||
vllm_config.cache_config.block_size,
|
||||
self.head_size,
|
||||
self.kv_cache_dtype,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
layer: AttentionLayer,
|
||||
@@ -325,60 +331,58 @@ class CPUAttentionBackendImpl(AttentionImpl):
|
||||
|
||||
num_actual_tokens = attn_metadata.num_actual_tokens
|
||||
|
||||
# Handle encoder attention differently - no KV cache needed
|
||||
# For encoder attention
|
||||
if self.attn_type in (AttentionType.ENCODER_ONLY, AttentionType.ENCODER):
|
||||
# For encoder attention,
|
||||
return self._run_sdpa_forward(
|
||||
query[:num_actual_tokens],
|
||||
key[:num_actual_tokens],
|
||||
value[:num_actual_tokens],
|
||||
output[:num_actual_tokens],
|
||||
attn_metadata,
|
||||
self.attn_type,
|
||||
)
|
||||
kv_cache = attn_metadata.encoder_cache
|
||||
|
||||
# For decoder and cross-attention, use KV cache, size are
|
||||
# [num_blocks, num_kv_heads, block_size, 2 * head_size]
|
||||
# Make a view [num_blocks, num_kv_heads, block_size * 2, head_size]
|
||||
# Then slice KV at dim 2
|
||||
# KV cache size are [num_blocks, num_kv_heads, block_size,
|
||||
# 2 * head_size]. Make a view [num_blocks, num_kv_heads,
|
||||
# block_size * 2, head_size]. Then slice KV at dim 2
|
||||
num_blocks, num_kv_heads, block_size, _ = kv_cache.size()
|
||||
kv_cache = kv_cache.view((num_blocks, num_kv_heads, block_size * 2, -1))
|
||||
key_cache, value_cache = kv_cache.chunk(2, dim=2)
|
||||
|
||||
if attn_metadata.use_sdpa_prefill:
|
||||
assert self.sinks is None, "Attention sink is unsupported in SDPA prefill"
|
||||
num_decode_tokens = attn_metadata.num_decode_tokens
|
||||
self._run_sdpa_forward(
|
||||
query[num_decode_tokens:num_actual_tokens],
|
||||
key[num_decode_tokens:num_actual_tokens],
|
||||
value[num_decode_tokens:num_actual_tokens],
|
||||
output[num_decode_tokens:num_actual_tokens],
|
||||
attn_metadata,
|
||||
self.attn_type,
|
||||
)
|
||||
num_actual_tokens = num_decode_tokens
|
||||
|
||||
if num_actual_tokens > 0:
|
||||
ops.cpu_attention_with_kv_cache(
|
||||
query=query[:num_actual_tokens],
|
||||
key_cache=key_cache,
|
||||
value_cache=value_cache,
|
||||
output=output[:num_actual_tokens], # type: ignore
|
||||
query_start_loc=attn_metadata.query_start_loc,
|
||||
seq_lens=attn_metadata.seq_lens,
|
||||
scale=self.scale,
|
||||
causal=attn_metadata.causal,
|
||||
alibi_slopes=self.alibi_slopes, # type: ignore
|
||||
sliding_window=self.sliding_window,
|
||||
block_table=attn_metadata.block_table,
|
||||
softcap=self.logits_soft_cap,
|
||||
scheduler_metadata=attn_metadata.scheduler_metadata,
|
||||
s_aux=self.sinks,
|
||||
# key and value may be None in the case of cross attention. They are
|
||||
# calculated once based on the output from the encoder and then cached
|
||||
# in KV cache.
|
||||
if (
|
||||
self.kv_sharing_target_layer_name is None
|
||||
and key is not None
|
||||
and value is not None
|
||||
):
|
||||
ops.cpu_attn_reshape_and_cache(
|
||||
key,
|
||||
value,
|
||||
key_cache,
|
||||
value_cache,
|
||||
attn_metadata.slot_mapping,
|
||||
self.isa,
|
||||
k_scale=layer._k_scale_float,
|
||||
v_scale=layer._v_scale_float,
|
||||
kv_cache_dtype=self.kv_cache_dtype,
|
||||
)
|
||||
|
||||
ops.cpu_attention_with_kv_cache(
|
||||
query=query[:num_actual_tokens],
|
||||
key_cache=key_cache,
|
||||
value_cache=value_cache,
|
||||
output=output[:num_actual_tokens], # type: ignore
|
||||
query_start_loc=attn_metadata.query_start_loc,
|
||||
seq_lens=attn_metadata.seq_lens,
|
||||
scale=self.scale,
|
||||
causal=attn_metadata.causal,
|
||||
alibi_slopes=self.alibi_slopes, # type: ignore
|
||||
sliding_window=self.sliding_window,
|
||||
block_table=attn_metadata.block_table,
|
||||
softcap=self.logits_soft_cap,
|
||||
scheduler_metadata=attn_metadata.scheduler_metadata,
|
||||
s_aux=self.sinks,
|
||||
k_scale=layer._k_scale_float,
|
||||
v_scale=layer._v_scale_float,
|
||||
kv_cache_dtype=self.kv_cache_dtype,
|
||||
)
|
||||
|
||||
return output
|
||||
|
||||
def do_kv_cache_update(
|
||||
@@ -395,136 +399,18 @@ class CPUAttentionBackendImpl(AttentionImpl):
|
||||
num_blocks, num_kv_heads, block_size, _ = kv_cache.size()
|
||||
kv_cache = kv_cache.view((num_blocks, num_kv_heads, block_size * 2, -1))
|
||||
key_cache, value_cache = kv_cache.chunk(2, dim=2)
|
||||
isa = _get_attn_isa(
|
||||
key.dtype, key_cache.shape[2], self.head_size, self.kv_cache_dtype
|
||||
)
|
||||
ops.cpu_attn_reshape_and_cache(
|
||||
key,
|
||||
value,
|
||||
key_cache,
|
||||
value_cache,
|
||||
slot_mapping,
|
||||
isa,
|
||||
self.isa,
|
||||
k_scale=layer._k_scale_float,
|
||||
v_scale=layer._v_scale_float,
|
||||
kv_cache_dtype=self.kv_cache_dtype,
|
||||
)
|
||||
|
||||
def _run_sdpa_forward(
|
||||
self,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
attn_metadata: CPUAttentionMetadata,
|
||||
attn_type: str,
|
||||
) -> torch.Tensor:
|
||||
attn_masks = attn_metadata.sdpa_attn_masks
|
||||
if attn_masks is None:
|
||||
if self.alibi_slopes is not None:
|
||||
attn_masks = _make_alibi_bias(
|
||||
self.alibi_slopes,
|
||||
query.dtype,
|
||||
attn_metadata.sdpa_start_loc,
|
||||
)
|
||||
elif self.sliding_window[0] != -1 or self.sliding_window[1] != -1:
|
||||
assert attn_metadata.seq_lens is not None
|
||||
attn_masks = _make_sliding_window_bias(
|
||||
attn_metadata.sdpa_start_loc,
|
||||
self.sliding_window[0],
|
||||
self.sliding_window[1],
|
||||
query.dtype,
|
||||
)
|
||||
else:
|
||||
attn_masks = [None] * (attn_metadata.sdpa_start_loc.size(0) - 1) # type: ignore
|
||||
attn_metadata.sdpa_attn_masks = attn_masks
|
||||
|
||||
query = query.movedim(0, query.dim() - 2)
|
||||
key = key.movedim(0, key.dim() - 2)
|
||||
value = value.movedim(0, value.dim() - 2)
|
||||
|
||||
causal_attn = attn_type == AttentionType.DECODER
|
||||
|
||||
sdpa_start_loc = attn_metadata.sdpa_start_loc.numpy() # type: ignore
|
||||
for i in range(len(attn_masks)):
|
||||
mask = attn_masks[i]
|
||||
start_q = sdpa_start_loc[i]
|
||||
end_q = sdpa_start_loc[i + 1]
|
||||
sub_out = (
|
||||
torch.nn.functional.scaled_dot_product_attention(
|
||||
query[None, :, start_q:end_q, :],
|
||||
key[None, :, start_q:end_q, :],
|
||||
value[None, :, start_q:end_q, :],
|
||||
attn_mask=mask,
|
||||
dropout_p=0.0,
|
||||
is_causal=causal_attn and mask is None,
|
||||
scale=self.scale,
|
||||
enable_gqa=self.num_heads > self.num_kv_heads,
|
||||
)
|
||||
.squeeze(0)
|
||||
.movedim(query.dim() - 2, 0)
|
||||
)
|
||||
output[start_q:end_q, :, :] = sub_out
|
||||
return output
|
||||
|
||||
|
||||
def _make_alibi_bias(
|
||||
alibi_slopes: torch.Tensor,
|
||||
dtype: torch.dtype,
|
||||
sdpa_start_loc: torch.Tensor,
|
||||
) -> list[torch.Tensor]:
|
||||
attn_biases: list[torch.Tensor] = []
|
||||
seq_num = sdpa_start_loc.size(0) - 1
|
||||
sdpa_start_loc = sdpa_start_loc.numpy() # type: ignore
|
||||
for i in range(seq_num):
|
||||
seq_len = sdpa_start_loc[i + 1] - sdpa_start_loc[i]
|
||||
bias = torch.arange(seq_len, dtype=dtype) # type: ignore
|
||||
# NOTE(zhuohan): HF uses
|
||||
# `bias = bias[None, :].repeat(seq_len, 1)`
|
||||
# here. We find that both biases give the same results, but
|
||||
# the bias below more accurately follows the original ALiBi
|
||||
# paper.
|
||||
bias = bias[None, :] - bias[:, None]
|
||||
|
||||
num_heads = alibi_slopes.shape[0]
|
||||
bias = bias[None, :].repeat((num_heads, 1, 1))
|
||||
bias.mul_(alibi_slopes[:, None, None]).unsqueeze_(0)
|
||||
inf_mask = (
|
||||
torch.empty((1, seq_len, seq_len), dtype=bias.dtype) # type: ignore
|
||||
.fill_(-torch.inf)
|
||||
.triu_(diagonal=1)
|
||||
)
|
||||
attn_biases.append((bias + inf_mask).to(dtype))
|
||||
|
||||
return attn_biases
|
||||
|
||||
|
||||
def _make_sliding_window_bias(
|
||||
sdpa_start_loc: torch.Tensor,
|
||||
left_window_size: int,
|
||||
right_window_size: int,
|
||||
dtype: torch.dtype,
|
||||
) -> list[torch.Tensor]:
|
||||
attn_biases: list[torch.Tensor] = []
|
||||
seq_num = sdpa_start_loc.size(0) - 1
|
||||
sdpa_start_loc = sdpa_start_loc.numpy() # type: ignore
|
||||
for i in range(seq_num):
|
||||
seq_len = sdpa_start_loc[i + 1] - sdpa_start_loc[i]
|
||||
mask = torch.full( # type: ignore
|
||||
(1, seq_len, seq_len), # type: ignore
|
||||
fill_value=1,
|
||||
dtype=dtype,
|
||||
)
|
||||
|
||||
if right_window_size != -1:
|
||||
mask = torch.tril(mask, diagonal=right_window_size)
|
||||
if left_window_size != -1:
|
||||
mask = torch.triu(mask, diagonal=-left_window_size)
|
||||
mask = torch.log(mask)
|
||||
attn_biases.append(mask)
|
||||
|
||||
return attn_biases
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=1)
|
||||
def _riscv_supports_rvv() -> bool:
|
||||
|
||||
Reference in New Issue
Block a user