[6/N][KV-Cache Layout Refactor] Standardize KV cache layout (#51718)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
This commit is contained in:
@@ -649,6 +649,10 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
compile_native=True,
|
||||
)
|
||||
|
||||
def bind_kv_cache(self, kv_cache: torch.Tensor) -> None:
|
||||
# [B, H=1, N, C] -> [B, N, C]
|
||||
self.kv_cache = kv_cache.squeeze(1)
|
||||
|
||||
@property
|
||||
def chunked_prefill_workspace_size(self) -> int:
|
||||
if self._chunked_prefill_workspace_size is None:
|
||||
@@ -1451,27 +1455,6 @@ class MLACommonBackend(AttentionBackend):
|
||||
def get_builder_cls() -> type["MLACommonMetadataBuilder"]:
|
||||
return MLACommonMetadataBuilder
|
||||
|
||||
@staticmethod
|
||||
def get_kv_cache_shape(
|
||||
num_blocks: int,
|
||||
block_size: int,
|
||||
num_kv_heads: int, # assumed to be 1 for MLA
|
||||
head_size: int,
|
||||
cache_dtype_str: str = "auto",
|
||||
) -> tuple[int, ...]:
|
||||
return (num_blocks, block_size, head_size)
|
||||
|
||||
@staticmethod
|
||||
def get_kv_cache_stride_order(
|
||||
include_num_layers_dimension: bool = False,
|
||||
) -> tuple[int, ...]:
|
||||
if include_num_layers_dimension:
|
||||
# Default to identity permutation to signal cross-layer allocation
|
||||
# is unsupported. Each MLA backend must opt in to support cross-layer
|
||||
# allocation by overriding this method.
|
||||
return (0, 1, 2, 3)
|
||||
return (0, 1, 2)
|
||||
|
||||
@classmethod
|
||||
def get_supported_head_sizes(cls) -> list[int]:
|
||||
return [320, 576]
|
||||
|
||||
@@ -61,16 +61,16 @@ def hpc_rope_norm_forward(
|
||||
return
|
||||
|
||||
attn_layer = forward_context.no_compile_layers[layer_name]
|
||||
# bind_kv_cache stores the per-layer KV cache as a single 5D tensor
|
||||
# (num_blocks, 2, block_size, num_kv_heads, head_size), so use it directly.
|
||||
# bind_kv_cache stores the per-layer KV cache as a single tensor
|
||||
# (num_blocks, num_kv_heads, block_size, 2 * head_size), so use it directly.
|
||||
kv_cache = attn_layer.kv_cache
|
||||
|
||||
if kv_cache.numel() == 0:
|
||||
output.zero_()
|
||||
return
|
||||
|
||||
assert kv_cache.dim() == 5, (
|
||||
f"Expected kv_cache to have 5 dims, got {tuple(kv_cache.shape)}"
|
||||
assert kv_cache.dim() == 4, (
|
||||
f"Expected kv_cache to have 4 dims, got {tuple(kv_cache.shape)}"
|
||||
)
|
||||
|
||||
rope_norm = _hpc_rope_norm_instances[layer_name]
|
||||
@@ -295,6 +295,9 @@ class HpcRopeNorm(CustomOp, HpcModule):
|
||||
"""
|
||||
import hpc
|
||||
|
||||
# (B, H, N, 2*hs) -> two (B, N, H, hs) K/V views, as in HpcAttentionImpl.
|
||||
key_cache, value_cache = kv_cache.transpose(1, 2).split(self.head_dim, dim=-1)
|
||||
|
||||
num_actual_tokens = attn_metadata.num_actual_tokens
|
||||
num_prefill_reqs = attn_metadata.num_prefills
|
||||
num_decode_reqs = attn_metadata.num_decodes
|
||||
@@ -333,8 +336,8 @@ class HpcRopeNorm(CustomOp, HpcModule):
|
||||
|
||||
if self.use_fp8:
|
||||
_, q_scale, split_k_flag = hpc.rope_norm_store_kv_fp8(
|
||||
key_cache=kv_cache[:, 0],
|
||||
value_cache=kv_cache[:, 1],
|
||||
key_cache=key_cache,
|
||||
value_cache=value_cache,
|
||||
qkv=qkv_prefill,
|
||||
cos_sin=self.cos_sin_cache,
|
||||
num_seqlen_per_req=seq_lens_prefill,
|
||||
@@ -353,8 +356,8 @@ class HpcRopeNorm(CustomOp, HpcModule):
|
||||
attn_metadata.hpc_prefill_q_scale = q_scale
|
||||
else:
|
||||
hpc.rope_norm_store_kv(
|
||||
kv_cache[:, 0],
|
||||
kv_cache[:, 1],
|
||||
key_cache,
|
||||
value_cache,
|
||||
qkv_prefill,
|
||||
self.cos_sin_cache,
|
||||
seq_lens_prefill,
|
||||
@@ -379,8 +382,8 @@ class HpcRopeNorm(CustomOp, HpcModule):
|
||||
|
||||
if self.use_fp8:
|
||||
_, q_scale, split_k_flag = hpc.rope_norm_store_kv_fp8(
|
||||
key_cache=kv_cache[:, 0],
|
||||
value_cache=kv_cache[:, 1],
|
||||
key_cache=key_cache,
|
||||
value_cache=value_cache,
|
||||
qkv=qkv_decode,
|
||||
cos_sin=self.cos_sin_cache,
|
||||
num_seqlen_per_req=num_seq_kvcache,
|
||||
@@ -401,8 +404,8 @@ class HpcRopeNorm(CustomOp, HpcModule):
|
||||
attn_metadata.hpc_split_k_flag = split_k_flag
|
||||
else:
|
||||
hpc.rope_norm_store_kv(
|
||||
kv_cache[:, 0],
|
||||
kv_cache[:, 1],
|
||||
key_cache,
|
||||
value_cache,
|
||||
qkv_decode,
|
||||
self.cos_sin_cache,
|
||||
num_seq_kvcache,
|
||||
|
||||
@@ -645,6 +645,11 @@ class DeepseekV32IndexerCache(torch.nn.Module, AttentionLayerBase):
|
||||
raise ValueError(f"Duplicate layer name: {prefix}")
|
||||
compilation_config.static_forward_context[prefix] = self
|
||||
|
||||
def bind_kv_cache(self, kv_cache: torch.Tensor) -> None:
|
||||
# [B, H=1, N, C] -> [B, N, C]: the indexer kernels and
|
||||
# kv_cache_as_quant_view index a 3-D block-major cache.
|
||||
self.kv_cache = kv_cache.squeeze(1)
|
||||
|
||||
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec:
|
||||
return MLAAttentionSpec(
|
||||
block_size=self.cache_config.block_size,
|
||||
|
||||
@@ -80,14 +80,14 @@ def dummy_attention(layer_name, _placeholder):
|
||||
|
||||
def basic_cache(
|
||||
to_cache: torch.Tensor, # shape: [seq_len, num_heads, head_size]
|
||||
kv_cache: torch.Tensor, # shape: [num_blocks, block_size, num_heads, head_size]
|
||||
kv_cache: torch.Tensor, # shape: [num_blocks, num_heads, block_size, head_size]
|
||||
slot_mapping: torch.Tensor, # shape: [seq_len]
|
||||
):
|
||||
# Padding slots are -1; redirect them to the null block (block 0, never
|
||||
# allocated to a request) so the scatter stays branch-free and sync-free.
|
||||
block_size = kv_cache.shape[1]
|
||||
block_size = kv_cache.shape[2]
|
||||
slot_mapping = slot_mapping.clamp_min(0)
|
||||
kv_cache[slot_mapping // block_size, slot_mapping % block_size] = to_cache
|
||||
kv_cache[slot_mapping // block_size, :, slot_mapping % block_size] = to_cache
|
||||
|
||||
|
||||
######### CacheOnlyAttentionBackend ########
|
||||
@@ -123,18 +123,6 @@ class CacheOnlyAttentionBackend(AttentionBackend):
|
||||
def get_impl_cls() -> type["CacheOnlyAttentionImpl"]:
|
||||
return CacheOnlyAttentionImpl
|
||||
|
||||
@staticmethod
|
||||
def get_kv_cache_shape(
|
||||
num_blocks: int,
|
||||
block_size: int,
|
||||
num_kv_heads: int,
|
||||
head_size: int,
|
||||
cache_dtype_str: str = "auto",
|
||||
) -> tuple[int, ...]:
|
||||
# We set `num_kv_heads = num_hidden_layers` and `head_size = hidden_size`
|
||||
# We also don't use a k/v (2) dim
|
||||
return (num_blocks, block_size, num_kv_heads, head_size)
|
||||
|
||||
@staticmethod
|
||||
def get_builder_cls() -> type["CacheOnlyAttentionMetadataBuilder"]:
|
||||
return CacheOnlyAttentionMetadataBuilder
|
||||
|
||||
@@ -5,6 +5,7 @@ import functools
|
||||
import logging
|
||||
import math
|
||||
from dataclasses import replace
|
||||
from fractions import Fraction
|
||||
from functools import partial
|
||||
|
||||
import torch
|
||||
@@ -128,14 +129,10 @@ def create_whisper_attention_backend_with_block_pooling(
|
||||
vllm_config: VllmConfig,
|
||||
device: torch.device,
|
||||
):
|
||||
assert kv_cache_spec.num_kv_heads % block_pool_size == 0
|
||||
# Scale pooled-unit quantities back to unpooled (encoder-token)
|
||||
# units for the underlying attention metadata: the KV cache spec
|
||||
# counts blocks in pooled units, but the kernel runs on the
|
||||
# `block_pool_size`x-expanded sequence built below.
|
||||
spec_overrides = dict(
|
||||
kv_cache_spec = replace(
|
||||
kv_cache_spec,
|
||||
block_size=kv_cache_spec.block_size * block_pool_size,
|
||||
num_kv_heads=kv_cache_spec.num_kv_heads // block_pool_size,
|
||||
tokens_per_state=1,
|
||||
)
|
||||
if isinstance(kv_cache_spec, SlidingWindowSpec):
|
||||
# The manager keeps `sliding_window` in pooled units; the kernel
|
||||
@@ -143,8 +140,7 @@ def create_whisper_attention_backend_with_block_pooling(
|
||||
# unpooled units (`get_kv_cache_spec` sizes the pooled window so
|
||||
# this window always stays resident).
|
||||
assert sliding_window is not None
|
||||
spec_overrides["sliding_window"] = sliding_window
|
||||
kv_cache_spec = replace(kv_cache_spec, **spec_overrides)
|
||||
kv_cache_spec = replace(kv_cache_spec, sliding_window=sliding_window)
|
||||
super().__init__(kv_cache_spec, layer_names, vllm_config, device)
|
||||
# Override model_config-derived values with the actual
|
||||
# encoder values from kv_cache_spec
|
||||
@@ -261,18 +257,6 @@ def create_whisper_attention_backend_with_block_pooling(
|
||||
overrides={
|
||||
"get_builder_cls": lambda: WhisperCausalAttentionWithBlockPoolingBuilder,
|
||||
"get_impl_cls": lambda: WhisperCausalAttentionWithBlockPoolingImpl,
|
||||
"get_kv_cache_shape": lambda num_blocks,
|
||||
block_size,
|
||||
num_kv_heads,
|
||||
head_size,
|
||||
cache_dtype_str: underlying_attn_backend.get_kv_cache_shape(
|
||||
num_blocks,
|
||||
# we stretch each block by `block_pool_size`
|
||||
block_size * block_pool_size,
|
||||
num_kv_heads // block_pool_size,
|
||||
head_size,
|
||||
cache_dtype_str,
|
||||
),
|
||||
"forward_includes_kv_cache_update": True,
|
||||
},
|
||||
)
|
||||
@@ -341,7 +325,7 @@ class WhisperCausalAttentionWithBlockPooling(Attention):
|
||||
assert isinstance(kv_cache_spec, AttentionSpec)
|
||||
kv_cache_spec = replace(
|
||||
kv_cache_spec,
|
||||
num_kv_heads=self.block_pool_size * kv_cache_spec.num_kv_heads,
|
||||
tokens_per_state=Fraction(1, self.block_pool_size),
|
||||
)
|
||||
if isinstance(kv_cache_spec, SlidingWindowSpec):
|
||||
# The manager counts blocks in pooled units, so express the window in
|
||||
|
||||
Reference in New Issue
Block a user