diff --git a/csrc/cpu/cpu_attn_impl.hpp b/csrc/cpu/cpu_attn_impl.hpp index 70081b36ee..be7915303a 100644 --- a/csrc/cpu/cpu_attn_impl.hpp +++ b/csrc/cpu/cpu_attn_impl.hpp @@ -822,8 +822,8 @@ struct AttentionInput { logits_buffer_t *__restrict__ logits_buffer, \ float *__restrict__ partial_q_buffer, float *__restrict__ max_buffer, \ float *__restrict__ sum_buffer, int32_t *__restrict__ block_table, \ - const int32_t kv_tile_start_pos, const int32_t kv_tile_end_pos, \ - const int32_t kv_tile_token_num, \ + const int32_t kv_end_pos, const int32_t kv_tile_start_pos, \ + const int32_t kv_tile_end_pos, const int32_t kv_tile_token_num, \ const int64_t kv_cache_num_blocks_stride, const int32_t q_head_num, \ const int32_t q_token_num, const int32_t q_tile_start_pos, \ const int32_t q_heads_per_kv, const int32_t block_size, \ @@ -834,7 +834,7 @@ struct AttentionInput { #define CPU_ATTENTION_PARAMS \ q_heads_buffer, k_head_cache_ptr, v_head_cache_ptr, logits_buffer, \ - partial_q_buffer, max_buffer, sum_buffer, block_table, \ + partial_q_buffer, max_buffer, sum_buffer, block_table, kv_end_pos, \ kv_tile_start_pos, kv_tile_end_pos, kv_tile_token_num, \ kv_cache_num_blocks_stride, q_head_num, q_token_num, q_tile_start_pos, \ q_heads_per_kv, block_size, left_window_size, right_window_size, scale, \ @@ -917,6 +917,7 @@ class AttentionMainLoop { // - max_buffer: [MaxQHeadNumPerIteration, 1], store max logits // - sum_buffer: [MaxQHeadNumPerIteration, 1], store sum of exp // - block_table + // - kv_end_pos: un-aligned end position of KV cache // - kv_tile_start_pos: start position of KV cache, aligned to // BlockSizeAlignment // - kv_tile_end_pos: end position of KV cache, aligned to @@ -1043,7 +1044,7 @@ class AttentionMainLoop { } apply_mask(logits_buffer, kv_tile_token_num, q_tile_start_pos, - kv_tile_start_pos, kv_tile_end_pos, q_token_num, + kv_end_pos, kv_tile_start_pos, kv_tile_end_pos, q_token_num, q_heads_per_kv, left_window_size, right_window_size); // if (debug_info){ @@ -1126,7 +1127,7 @@ class AttentionMainLoop { void apply_mask(logits_buffer_t* __restrict__ logits_buffer, const int64_t logits_buffer_stride, - const int32_t q_tile_start_pos, + const int32_t q_tile_start_pos, const int32_t kv_end_pos, const int32_t kv_tile_start_pos, const int32_t kv_tile_end_pos, const int32_t q_token_num, const int32_t q_heads_per_kv, @@ -1154,7 +1155,7 @@ class AttentionMainLoop { std::max(kv_tile_start_pos, curr_token_pos + sliding_window_right + 1)); } - return pos; + return std::min(pos, kv_end_pos); }(); int32_t left_invalid_token_num = left_kv_pos - kv_tile_start_pos; @@ -1789,7 +1790,7 @@ class AttentionMainLoop { attn_impl.template execute_attention( curr_q_heads_buffer, curr_k_cache, curr_v_cache, logits_buffer, curr_partial_q_buffer, curr_max_buffer, - curr_sum_buffer, curr_block_table, + curr_sum_buffer, curr_block_table, kv_end_pos, aligned_actual_kv_tile_pos_left, aligned_actual_kv_tile_pos_right, actual_kv_token_num, kv_cache_block_num_stride, q_tile_head_num, diff --git a/csrc/cpu/generate_cpu_attn_dispatch.py b/csrc/cpu/generate_cpu_attn_dispatch.py index 7c7123a6de..95ce9e6692 100644 --- a/csrc/cpu/generate_cpu_attn_dispatch.py +++ b/csrc/cpu/generate_cpu_attn_dispatch.py @@ -11,7 +11,7 @@ import os HEAD_DIMS_32 = [32, 64, 96, 128, 160, 192, 224, 256, 512] # Head dimensions divisible by 16 but not 32 (VEC16 only) -HEAD_DIMS_16 = [80, 112] +HEAD_DIMS_16 = [48, 80, 112] # ISA types ISA_TYPES = { diff --git a/tests/kernels/attention/test_cpu_attn.py b/tests/kernels/attention/test_cpu_attn.py index c393950255..b79621075f 100644 --- a/tests/kernels/attention/test_cpu_attn.py +++ b/tests/kernels/attention/test_cpu_attn.py @@ -25,7 +25,6 @@ from vllm._custom_ops import ( if torch.cpu._is_amx_tile_supported(): torch.cpu._init_amx() - NUM_HEADS = [ (4, 4), (8, 2), @@ -43,6 +42,11 @@ SEQ_LENS = [ # (q_len, kv_len) [(2345, 2345), (5, 5), (3, 16), (134, 5131)], # prefill batch [(992, 2456), (1, 1234), (98, 1145), (1, 4162), (2345, 2345)], # mixed batch ] +_FP8_ATOL = {"fp8_e4m3": 0.2, "fp8_e5m2": 0.3} +_FP8_RTOL = 0.1 +ENCODER_SEQ_LENS = [ + [1, 678, 2367, 145, 4162, 36, 7812], +] def get_attn_isa( @@ -61,10 +65,7 @@ def get_attn_isa( # rand number generation takes too much time, cache rand tensors @functools.lru_cache(maxsize=128, typed=False) -def tensor_cache( - elem_num: int, - dtype: torch.dtype, -) -> torch.Tensor: +def tensor_cache(elem_num: int, dtype: torch.dtype, tag: str = "none") -> torch.Tensor: tensor = torch.randn(elem_num, dtype=dtype) return tensor @@ -183,8 +184,222 @@ def ref_paged_attn( return torch.cat(outputs, dim=0) -_FP8_ATOL = {"fp8_e4m3": 0.2, "fp8_e5m2": 0.3} -_FP8_RTOL = 0.1 +def ref_varlen_encoder_attn( + query: torch.Tensor, # [token, q_head_num, head_dim] + key: torch.Tensor, # [token, kv_head_num, head_dim] + value: torch.Tensor, + seq_lens: list[int], + scale: float, + sliding_window: int | None = None, +) -> torch.Tensor: + num_seqs = len(seq_lens) + dtype = query.dtype + + output = torch.empty_like(query) + + start_idx = 0 + for i in range(num_seqs): + seq_len = seq_lens[i] + q = query[start_idx : start_idx + seq_len].float() + k = key[start_idx : start_idx + seq_len].float() + v = value[start_idx : start_idx + seq_len].float() + q *= scale + + if q.shape[1] != k.shape[1]: + k = torch.repeat_interleave(k, q.shape[1] // k.shape[1], dim=1) + v = torch.repeat_interleave(v, q.shape[1] // v.shape[1], dim=1) + attn = torch.einsum("qhd,khd->hqk", q, k).float() + empty_mask = torch.ones(seq_len, seq_len) + if sliding_window is not None: + mask = ( + torch.triu(empty_mask, diagonal=1 - sliding_window).bool() + ^ torch.triu(empty_mask, diagonal=sliding_window).bool() + ).logical_not() + else: + mask = empty_mask.logical_not() + + attn.masked_fill_(mask, float("-inf")) + attn = torch.softmax(attn, dim=-1) + out = torch.einsum("hqk,khd->qhd", attn, v).to(dtype=dtype) + output[start_idx : start_idx + seq_len].copy_(out) + + start_idx += seq_len + + return output + + +@torch.inference_mode() +def varlen_encoder_attention( + seq_lens: list[int], + num_heads: tuple[int, int], + head_size: int, + sliding_window: int | None, + dtype: torch.dtype, + block_size: int, + isa: str, +) -> None: + set_random_seed(0) + num_seqs = len(seq_lens) + num_query_heads = num_heads[0] + num_kv_heads = num_heads[1] + assert num_query_heads % num_kv_heads == 0 + window_size = ( + (sliding_window - 1, sliding_window - 1) + if sliding_window is not None + else (-1, -1) + ) + scale = head_size**-0.5 + token_num = sum(seq_lens) + + seq_lens_tensor = torch.tensor(seq_lens, dtype=torch.int32) + query_start_loc = torch.zeros(num_seqs, dtype=torch.int32) + torch.cumsum(seq_lens_tensor[:-1], 0, out=query_start_loc[1:]) + block_nums = (seq_lens_tensor + block_size - 1) // block_size + start_block_ids = torch.zeros_like(seq_lens_tensor) + torch.cumsum(block_nums[:-1], 0, out=start_block_ids[1:]) + total_block_num: int = block_nums.sum().item() + max_block_num = block_nums.max().item() + block_offsets = torch.arange(0, max_block_num, dtype=torch.int32) + encoder_block_table = start_block_ids[:, None] + block_offsets[None, :] + slot_mapping_list = [] + slot_start_idx = 0 + for i in range(num_seqs): + block_num = block_nums[i].item() + seq_len = seq_lens[i] + slot_mapping_list.append(torch.arange(slot_start_idx, slot_start_idx + seq_len)) + slot_start_idx += block_num * block_size + slot_mapping = torch.cat(slot_mapping_list) + + query = tensor_cache( + elem_num=token_num * num_query_heads * head_size, + dtype=dtype, + tag="query", + ) + query = query.view( + token_num, + num_query_heads, + head_size, + ) + + key_value = tensor_cache( + elem_num=2 * token_num * num_kv_heads * head_size, + dtype=dtype, + tag="kv", + ) + key_value = key_value.view( + 2, + token_num, + num_kv_heads, + head_size, + ) + key, value = key_value.unbind(0) + + # KV cache for CPU attention + packed_key_value_cache = torch.zeros( + total_block_num, num_kv_heads, block_size, head_size * 2, dtype=dtype + ) + packed_key_value_cache = packed_key_value_cache.view( + (total_block_num, num_kv_heads, block_size * 2, -1) + ) + packed_key_cache, packed_value_cache = packed_key_value_cache.chunk(2, dim=2) + + cu_query_lens = torch.tensor([0] + seq_lens, dtype=torch.int32).cumsum( + dim=0, dtype=torch.int32 + ) + kv_lens_tensor = torch.tensor(seq_lens, dtype=torch.int32) + + # use reshape_and_cache to pack key_cache and value_cache + cpu_attn_reshape_and_cache( + key=key.view(-1, num_kv_heads, head_size), + value=value.view(-1, num_kv_heads, head_size), + key_cache=packed_key_cache, + value_cache=packed_value_cache, + slot_mapping=slot_mapping, + isa=isa, + ) + + metadata = cpu_attn_get_scheduler_metadata( + num_reqs=num_seqs, + num_heads=num_query_heads, + num_kv_heads=num_kv_heads, + head_dim=head_size, + seq_lens=kv_lens_tensor, + dtype=dtype, + query_start_loc=cu_query_lens, + causal=False, + sliding_window_size=sliding_window if sliding_window is not None else -1, + isa=isa, + enable_kv_split=False, + ) + + out_without_split = torch.empty_like(query) + cpu_attention_with_kv_cache( + query=query, + key_cache=packed_key_cache, + value_cache=packed_value_cache, + output=out_without_split, + query_start_loc=cu_query_lens, + seq_lens=kv_lens_tensor, + scale=scale, + causal=False, + alibi_slopes=None, + sliding_window=window_size, + block_table=encoder_block_table, + softcap=0, + scheduler_metadata=metadata, + s_aux=None, + ) + + metadata = cpu_attn_get_scheduler_metadata( + num_reqs=num_seqs, + num_heads=num_query_heads, + num_kv_heads=num_kv_heads, + head_dim=head_size, + seq_lens=kv_lens_tensor, + dtype=dtype, + query_start_loc=cu_query_lens, + causal=False, + sliding_window_size=sliding_window if sliding_window is not None else -1, + isa=isa, + enable_kv_split=True, + ) + + out_with_split = torch.empty_like(query) + cpu_attention_with_kv_cache( + query=query, + key_cache=packed_key_cache, + value_cache=packed_value_cache, + output=out_with_split, + query_start_loc=cu_query_lens, + seq_lens=kv_lens_tensor, + scale=scale, + causal=False, + alibi_slopes=None, + sliding_window=window_size, + block_table=encoder_block_table, + softcap=0, + scheduler_metadata=metadata, + s_aux=None, + ) + + ref_output = ref_varlen_encoder_attn( + query=query, + key=key, + value=value, + seq_lens=seq_lens, + scale=scale, + sliding_window=sliding_window, + ) + atol, rtol = 1.5e-2, 1e-2 + + ( + torch.testing.assert_close(out_with_split, ref_output, atol=atol, rtol=rtol), + f"{torch.max(torch.abs(out_with_split - ref_output))}", + ) + ( + torch.testing.assert_close(out_without_split, ref_output, atol=atol, rtol=rtol), + f"{torch.max(torch.abs(out_without_split - ref_output))}", + ) @torch.inference_mode() @@ -418,6 +633,71 @@ def varlen_with_paged_kv( ) +@pytest.mark.parametrize("seq_lens", ENCODER_SEQ_LENS) +@pytest.mark.parametrize("num_heads", NUM_HEADS) +@pytest.mark.parametrize("head_size", HEAD_SIZES) +@pytest.mark.parametrize( + "block_size", + [ + 128, + ], +) +@pytest.mark.parametrize("sliding_window", SLIDING_WINDOWS) +@pytest.mark.parametrize("dtype", QTYPES) +@pytest.mark.parametrize("isa", ["vec"]) +def test_varlen_encoder_attention_vec( + seq_lens: list[int], + num_heads: tuple[int, int], + head_size: int, + sliding_window: int | None, + dtype: torch.dtype, + block_size: int, + isa: str, +) -> None: + varlen_encoder_attention( + seq_lens=seq_lens, + num_heads=num_heads, + head_size=head_size, + sliding_window=sliding_window, + dtype=dtype, + block_size=block_size, + isa=isa, + ) + + +@pytest.mark.parametrize("seq_lens", ENCODER_SEQ_LENS) +@pytest.mark.parametrize("num_heads", NUM_HEADS) +@pytest.mark.parametrize("head_size", HEAD_SIZES) +@pytest.mark.parametrize( + "block_size", + [ + 128, + ], +) +@pytest.mark.parametrize("sliding_window", SLIDING_WINDOWS) +@pytest.mark.parametrize("dtype", [torch.bfloat16]) +@pytest.mark.parametrize("isa", ["amx"]) +@pytest.mark.skipif(not torch.cpu._is_amx_tile_supported(), reason="no AMX support.") +def test_varlen_encoder_attention_amx( + seq_lens: list[int], + num_heads: tuple[int, int], + head_size: int, + sliding_window: int | None, + dtype: torch.dtype, + block_size: int, + isa: str, +) -> None: + varlen_encoder_attention( + seq_lens=seq_lens, + num_heads=num_heads, + head_size=head_size, + sliding_window=sliding_window, + dtype=dtype, + block_size=block_size, + isa=isa, + ) + + @pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8_e4m3", "fp8_e5m2"]) @pytest.mark.parametrize("seq_lens", SEQ_LENS) @pytest.mark.parametrize("num_heads", NUM_HEADS) diff --git a/vllm/v1/attention/backends/cpu_attn.py b/vllm/v1/attention/backends/cpu_attn.py index 39a29086f9..ebaab1b30d 100644 --- a/vllm/v1/attention/backends/cpu_attn.py +++ b/vllm/v1/attention/backends/cpu_attn.py @@ -11,7 +11,7 @@ import torch from vllm import _custom_ops as ops from vllm import envs -from vllm.config import VllmConfig +from vllm.config import VllmConfig, get_current_vllm_config from vllm.logger import init_logger from vllm.platforms import CpuArchEnum, current_platform from vllm.utils.torch_utils import is_quantized_kv_cache @@ -26,20 +26,15 @@ from vllm.v1.attention.backend import ( ) from vllm.v1.attention.backends.utils import ( KVCacheLayoutType, - split_decodes_and_prefills, ) -from vllm.v1.kv_cache_interface import AttentionSpec, CrossAttentionSpec +from vllm.v1.kv_cache_interface import ( + AttentionSpec, + CrossAttentionSpec, + EncoderOnlyAttentionSpec, +) logger = init_logger(__name__) -_CPU_ARCH_PREFER_MIXED_BATCH = ( - CpuArchEnum.X86, - CpuArchEnum.ARM, - CpuArchEnum.S390X, - CpuArchEnum.RISCV, - CpuArchEnum.POWERPC, -) - class CPUAttentionBackend(AttentionBackend): forward_includes_kv_cache_update: bool = False @@ -124,6 +119,8 @@ class CPUAttentionMetadata: sdpa_attn_masks: list[torch.Tensor | None] | None = None sdpa_start_loc: torch.Tensor | None = None + encoder_cache: torch.Tensor | None = None + class CPUAttentionMetadataBuilder(AttentionMetadataBuilder[CPUAttentionMetadata]): def __init__( @@ -135,17 +132,6 @@ class CPUAttentionMetadataBuilder(AttentionMetadataBuilder[CPUAttentionMetadata] ) -> None: super().__init__(kv_cache_spec, layer_names, vllm_config, device) - self.use_sdpa_prefill = False - reorder_batch_threshold = None - if current_platform.get_cpu_architecture() not in _CPU_ARCH_PREFER_MIXED_BATCH: - # in this case, decode seqs are reordered to the front of prefill seqs - # to split decode and prefill. Then use SDPA for prefill and - # cpu_attention_with_kv_cache for decode - reorder_batch_threshold = 1 - self.use_sdpa_prefill = True - - self._init_reorder_batch_threshold(reorder_batch_threshold, False) - self.kv_cache_spec = kv_cache_spec self.vllm_config = vllm_config @@ -168,6 +154,9 @@ class CPUAttentionMetadataBuilder(AttentionMetadataBuilder[CPUAttentionMetadata] kv_cache_dtype_str, ) self.is_cross_attention = isinstance(kv_cache_spec, CrossAttentionSpec) + self.is_encoder_only_attention = isinstance( + kv_cache_spec, EncoderOnlyAttentionSpec + ) def build( self, @@ -185,23 +174,34 @@ class CPUAttentionMetadataBuilder(AttentionMetadataBuilder[CPUAttentionMetadata] slot_mapping = common_attn_metadata.slot_mapping causal = False if self.is_cross_attention else common_attn_metadata.causal - sdpa_start_loc = query_start_loc - num_decode_tokens = 0 - if self.use_sdpa_prefill and causal: - # Decoder, need reorder and truncate - assert self.reorder_batch_threshold - (num_decodes, num_prefills, num_decode_tokens, num_prefill_tokens) = ( - split_decodes_and_prefills( - common_attn_metadata, - decode_threshold=self.reorder_batch_threshold, - require_uniform=True, - ) + encoder_cache_tensor = None + if self.is_encoder_only_attention: + block_nums = (seq_lens + self.block_size - 1) // self.block_size + start_block_ids = torch.zeros_like(seq_lens) + torch.cumsum(block_nums[:-1], 0, out=start_block_ids[1:]) + total_block_num: int = block_nums.sum().item() + max_block_num = block_nums.max().item() + block_offsets = torch.arange( + 0, max_block_num, dtype=block_table_tensor.dtype ) - num_reqs = num_decodes - sdpa_start_loc = sdpa_start_loc[num_decodes:] - num_decode_tokens - seq_lens = seq_lens[:num_decodes] - query_start_loc = query_start_loc[: num_decodes + 1] - block_table_tensor = block_table_tensor[:num_decodes] + encoder_block_table = start_block_ids[:, None] + block_offsets[None, :] + torch.ops._C.compute_slot_mapping_kernel_impl( + query_start_loc, + common_attn_metadata.positions, + encoder_block_table, + slot_mapping, + self.block_size, + ) + encoder_cache_tensor = torch.zeros( + ( + total_block_num, + self.num_kv_heads, + self.block_size, + 2 * self.head_dim, + ), + dtype=self.dtype, + ) + block_table_tensor = encoder_block_table scheduler_metadata = ops.cpu_attn_get_scheduler_metadata( num_reqs=num_reqs, @@ -227,9 +227,7 @@ class CPUAttentionMetadataBuilder(AttentionMetadataBuilder[CPUAttentionMetadata] slot_mapping=slot_mapping, scheduler_metadata=scheduler_metadata, causal=causal, - use_sdpa_prefill=self.use_sdpa_prefill, - num_decode_tokens=num_decode_tokens, - sdpa_start_loc=sdpa_start_loc, + encoder_cache=encoder_cache_tensor, ) return attn_metadata @@ -289,6 +287,14 @@ class CPUAttentionBackendImpl(AttentionImpl): "heads in the layer" ) + vllm_config = get_current_vllm_config() + self.isa = _get_attn_isa( + vllm_config.model_config.dtype, + vllm_config.cache_config.block_size, + self.head_size, + self.kv_cache_dtype, + ) + def forward( self, layer: AttentionLayer, @@ -325,60 +331,58 @@ class CPUAttentionBackendImpl(AttentionImpl): num_actual_tokens = attn_metadata.num_actual_tokens - # Handle encoder attention differently - no KV cache needed + # For encoder attention if self.attn_type in (AttentionType.ENCODER_ONLY, AttentionType.ENCODER): # For encoder attention, - return self._run_sdpa_forward( - query[:num_actual_tokens], - key[:num_actual_tokens], - value[:num_actual_tokens], - output[:num_actual_tokens], - attn_metadata, - self.attn_type, - ) + kv_cache = attn_metadata.encoder_cache - # For decoder and cross-attention, use KV cache, size are - # [num_blocks, num_kv_heads, block_size, 2 * head_size] - # Make a view [num_blocks, num_kv_heads, block_size * 2, head_size] - # Then slice KV at dim 2 + # KV cache size are [num_blocks, num_kv_heads, block_size, + # 2 * head_size]. Make a view [num_blocks, num_kv_heads, + # block_size * 2, head_size]. Then slice KV at dim 2 num_blocks, num_kv_heads, block_size, _ = kv_cache.size() kv_cache = kv_cache.view((num_blocks, num_kv_heads, block_size * 2, -1)) key_cache, value_cache = kv_cache.chunk(2, dim=2) - if attn_metadata.use_sdpa_prefill: - assert self.sinks is None, "Attention sink is unsupported in SDPA prefill" - num_decode_tokens = attn_metadata.num_decode_tokens - self._run_sdpa_forward( - query[num_decode_tokens:num_actual_tokens], - key[num_decode_tokens:num_actual_tokens], - value[num_decode_tokens:num_actual_tokens], - output[num_decode_tokens:num_actual_tokens], - attn_metadata, - self.attn_type, - ) - num_actual_tokens = num_decode_tokens - - if num_actual_tokens > 0: - ops.cpu_attention_with_kv_cache( - query=query[:num_actual_tokens], - key_cache=key_cache, - value_cache=value_cache, - output=output[:num_actual_tokens], # type: ignore - query_start_loc=attn_metadata.query_start_loc, - seq_lens=attn_metadata.seq_lens, - scale=self.scale, - causal=attn_metadata.causal, - alibi_slopes=self.alibi_slopes, # type: ignore - sliding_window=self.sliding_window, - block_table=attn_metadata.block_table, - softcap=self.logits_soft_cap, - scheduler_metadata=attn_metadata.scheduler_metadata, - s_aux=self.sinks, + # key and value may be None in the case of cross attention. They are + # calculated once based on the output from the encoder and then cached + # in KV cache. + if ( + self.kv_sharing_target_layer_name is None + and key is not None + and value is not None + ): + ops.cpu_attn_reshape_and_cache( + key, + value, + key_cache, + value_cache, + attn_metadata.slot_mapping, + self.isa, k_scale=layer._k_scale_float, v_scale=layer._v_scale_float, kv_cache_dtype=self.kv_cache_dtype, ) + ops.cpu_attention_with_kv_cache( + query=query[:num_actual_tokens], + key_cache=key_cache, + value_cache=value_cache, + output=output[:num_actual_tokens], # type: ignore + query_start_loc=attn_metadata.query_start_loc, + seq_lens=attn_metadata.seq_lens, + scale=self.scale, + causal=attn_metadata.causal, + alibi_slopes=self.alibi_slopes, # type: ignore + sliding_window=self.sliding_window, + block_table=attn_metadata.block_table, + softcap=self.logits_soft_cap, + scheduler_metadata=attn_metadata.scheduler_metadata, + s_aux=self.sinks, + k_scale=layer._k_scale_float, + v_scale=layer._v_scale_float, + kv_cache_dtype=self.kv_cache_dtype, + ) + return output def do_kv_cache_update( @@ -395,136 +399,18 @@ class CPUAttentionBackendImpl(AttentionImpl): num_blocks, num_kv_heads, block_size, _ = kv_cache.size() kv_cache = kv_cache.view((num_blocks, num_kv_heads, block_size * 2, -1)) key_cache, value_cache = kv_cache.chunk(2, dim=2) - isa = _get_attn_isa( - key.dtype, key_cache.shape[2], self.head_size, self.kv_cache_dtype - ) ops.cpu_attn_reshape_and_cache( key, value, key_cache, value_cache, slot_mapping, - isa, + self.isa, k_scale=layer._k_scale_float, v_scale=layer._v_scale_float, kv_cache_dtype=self.kv_cache_dtype, ) - def _run_sdpa_forward( - self, - query: torch.Tensor, - key: torch.Tensor, - value: torch.Tensor, - output: torch.Tensor, - attn_metadata: CPUAttentionMetadata, - attn_type: str, - ) -> torch.Tensor: - attn_masks = attn_metadata.sdpa_attn_masks - if attn_masks is None: - if self.alibi_slopes is not None: - attn_masks = _make_alibi_bias( - self.alibi_slopes, - query.dtype, - attn_metadata.sdpa_start_loc, - ) - elif self.sliding_window[0] != -1 or self.sliding_window[1] != -1: - assert attn_metadata.seq_lens is not None - attn_masks = _make_sliding_window_bias( - attn_metadata.sdpa_start_loc, - self.sliding_window[0], - self.sliding_window[1], - query.dtype, - ) - else: - attn_masks = [None] * (attn_metadata.sdpa_start_loc.size(0) - 1) # type: ignore - attn_metadata.sdpa_attn_masks = attn_masks - - query = query.movedim(0, query.dim() - 2) - key = key.movedim(0, key.dim() - 2) - value = value.movedim(0, value.dim() - 2) - - causal_attn = attn_type == AttentionType.DECODER - - sdpa_start_loc = attn_metadata.sdpa_start_loc.numpy() # type: ignore - for i in range(len(attn_masks)): - mask = attn_masks[i] - start_q = sdpa_start_loc[i] - end_q = sdpa_start_loc[i + 1] - sub_out = ( - torch.nn.functional.scaled_dot_product_attention( - query[None, :, start_q:end_q, :], - key[None, :, start_q:end_q, :], - value[None, :, start_q:end_q, :], - attn_mask=mask, - dropout_p=0.0, - is_causal=causal_attn and mask is None, - scale=self.scale, - enable_gqa=self.num_heads > self.num_kv_heads, - ) - .squeeze(0) - .movedim(query.dim() - 2, 0) - ) - output[start_q:end_q, :, :] = sub_out - return output - - -def _make_alibi_bias( - alibi_slopes: torch.Tensor, - dtype: torch.dtype, - sdpa_start_loc: torch.Tensor, -) -> list[torch.Tensor]: - attn_biases: list[torch.Tensor] = [] - seq_num = sdpa_start_loc.size(0) - 1 - sdpa_start_loc = sdpa_start_loc.numpy() # type: ignore - for i in range(seq_num): - seq_len = sdpa_start_loc[i + 1] - sdpa_start_loc[i] - bias = torch.arange(seq_len, dtype=dtype) # type: ignore - # NOTE(zhuohan): HF uses - # `bias = bias[None, :].repeat(seq_len, 1)` - # here. We find that both biases give the same results, but - # the bias below more accurately follows the original ALiBi - # paper. - bias = bias[None, :] - bias[:, None] - - num_heads = alibi_slopes.shape[0] - bias = bias[None, :].repeat((num_heads, 1, 1)) - bias.mul_(alibi_slopes[:, None, None]).unsqueeze_(0) - inf_mask = ( - torch.empty((1, seq_len, seq_len), dtype=bias.dtype) # type: ignore - .fill_(-torch.inf) - .triu_(diagonal=1) - ) - attn_biases.append((bias + inf_mask).to(dtype)) - - return attn_biases - - -def _make_sliding_window_bias( - sdpa_start_loc: torch.Tensor, - left_window_size: int, - right_window_size: int, - dtype: torch.dtype, -) -> list[torch.Tensor]: - attn_biases: list[torch.Tensor] = [] - seq_num = sdpa_start_loc.size(0) - 1 - sdpa_start_loc = sdpa_start_loc.numpy() # type: ignore - for i in range(seq_num): - seq_len = sdpa_start_loc[i + 1] - sdpa_start_loc[i] - mask = torch.full( # type: ignore - (1, seq_len, seq_len), # type: ignore - fill_value=1, - dtype=dtype, - ) - - if right_window_size != -1: - mask = torch.tril(mask, diagonal=right_window_size) - if left_window_size != -1: - mask = torch.triu(mask, diagonal=-left_window_size) - mask = torch.log(mask) - attn_biases.append(mask) - - return attn_biases - @functools.lru_cache(maxsize=1) def _riscv_supports_rvv() -> bool: