[6/N][KV-Cache Layout Refactor] Standardize KV cache layout (#51718)

Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
This commit is contained in:
Lucas Wilkinson
2026-08-21 17:47:49 -07:00
committed by GitHub
parent 9ff7041b55
commit 8bdc70ec7b
150 changed files with 4006 additions and 5815 deletions
@@ -645,6 +645,11 @@ class DeepseekV32IndexerCache(torch.nn.Module, AttentionLayerBase):
raise ValueError(f"Duplicate layer name: {prefix}")
compilation_config.static_forward_context[prefix] = self
def bind_kv_cache(self, kv_cache: torch.Tensor) -> None:
# [B, H=1, N, C] -> [B, N, C]: the indexer kernels and
# kv_cache_as_quant_view index a 3-D block-major cache.
self.kv_cache = kv_cache.squeeze(1)
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec:
return MLAAttentionSpec(
block_size=self.cache_config.block_size,
@@ -80,14 +80,14 @@ def dummy_attention(layer_name, _placeholder):
def basic_cache(
to_cache: torch.Tensor, # shape: [seq_len, num_heads, head_size]
kv_cache: torch.Tensor, # shape: [num_blocks, block_size, num_heads, head_size]
kv_cache: torch.Tensor, # shape: [num_blocks, num_heads, block_size, head_size]
slot_mapping: torch.Tensor, # shape: [seq_len]
):
# Padding slots are -1; redirect them to the null block (block 0, never
# allocated to a request) so the scatter stays branch-free and sync-free.
block_size = kv_cache.shape[1]
block_size = kv_cache.shape[2]
slot_mapping = slot_mapping.clamp_min(0)
kv_cache[slot_mapping // block_size, slot_mapping % block_size] = to_cache
kv_cache[slot_mapping // block_size, :, slot_mapping % block_size] = to_cache
######### CacheOnlyAttentionBackend ########
@@ -123,18 +123,6 @@ class CacheOnlyAttentionBackend(AttentionBackend):
def get_impl_cls() -> type["CacheOnlyAttentionImpl"]:
return CacheOnlyAttentionImpl
@staticmethod
def get_kv_cache_shape(
num_blocks: int,
block_size: int,
num_kv_heads: int,
head_size: int,
cache_dtype_str: str = "auto",
) -> tuple[int, ...]:
# We set `num_kv_heads = num_hidden_layers` and `head_size = hidden_size`
# We also don't use a k/v (2) dim
return (num_blocks, block_size, num_kv_heads, head_size)
@staticmethod
def get_builder_cls() -> type["CacheOnlyAttentionMetadataBuilder"]:
return CacheOnlyAttentionMetadataBuilder
+6 -22
View File
@@ -5,6 +5,7 @@ import functools
import logging
import math
from dataclasses import replace
from fractions import Fraction
from functools import partial
import torch
@@ -128,14 +129,10 @@ def create_whisper_attention_backend_with_block_pooling(
vllm_config: VllmConfig,
device: torch.device,
):
assert kv_cache_spec.num_kv_heads % block_pool_size == 0
# Scale pooled-unit quantities back to unpooled (encoder-token)
# units for the underlying attention metadata: the KV cache spec
# counts blocks in pooled units, but the kernel runs on the
# `block_pool_size`x-expanded sequence built below.
spec_overrides = dict(
kv_cache_spec = replace(
kv_cache_spec,
block_size=kv_cache_spec.block_size * block_pool_size,
num_kv_heads=kv_cache_spec.num_kv_heads // block_pool_size,
tokens_per_state=1,
)
if isinstance(kv_cache_spec, SlidingWindowSpec):
# The manager keeps `sliding_window` in pooled units; the kernel
@@ -143,8 +140,7 @@ def create_whisper_attention_backend_with_block_pooling(
# unpooled units (`get_kv_cache_spec` sizes the pooled window so
# this window always stays resident).
assert sliding_window is not None
spec_overrides["sliding_window"] = sliding_window
kv_cache_spec = replace(kv_cache_spec, **spec_overrides)
kv_cache_spec = replace(kv_cache_spec, sliding_window=sliding_window)
super().__init__(kv_cache_spec, layer_names, vllm_config, device)
# Override model_config-derived values with the actual
# encoder values from kv_cache_spec
@@ -261,18 +257,6 @@ def create_whisper_attention_backend_with_block_pooling(
overrides={
"get_builder_cls": lambda: WhisperCausalAttentionWithBlockPoolingBuilder,
"get_impl_cls": lambda: WhisperCausalAttentionWithBlockPoolingImpl,
"get_kv_cache_shape": lambda num_blocks,
block_size,
num_kv_heads,
head_size,
cache_dtype_str: underlying_attn_backend.get_kv_cache_shape(
num_blocks,
# we stretch each block by `block_pool_size`
block_size * block_pool_size,
num_kv_heads // block_pool_size,
head_size,
cache_dtype_str,
),
"forward_includes_kv_cache_update": True,
},
)
@@ -341,7 +325,7 @@ class WhisperCausalAttentionWithBlockPooling(Attention):
assert isinstance(kv_cache_spec, AttentionSpec)
kv_cache_spec = replace(
kv_cache_spec,
num_kv_heads=self.block_pool_size * kv_cache_spec.num_kv_heads,
tokens_per_state=Fraction(1, self.block_pool_size),
)
if isinstance(kv_cache_spec, SlidingWindowSpec):
# The manager counts blocks in pooled units, so express the window in