[6/N][KV-Cache Layout Refactor] Standardize KV cache layout (#51718)

Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
This commit is contained in:
Lucas Wilkinson
2026-08-21 17:47:49 -07:00
committed by GitHub
parent 9ff7041b55
commit 8bdc70ec7b
150 changed files with 4006 additions and 5815 deletions
@@ -649,6 +649,10 @@ class MLAAttention(nn.Module, AttentionLayerBase):
compile_native=True,
)
def bind_kv_cache(self, kv_cache: torch.Tensor) -> None:
# [B, H=1, N, C] -> [B, N, C]
self.kv_cache = kv_cache.squeeze(1)
@property
def chunked_prefill_workspace_size(self) -> int:
if self._chunked_prefill_workspace_size is None:
@@ -1451,27 +1455,6 @@ class MLACommonBackend(AttentionBackend):
def get_builder_cls() -> type["MLACommonMetadataBuilder"]:
return MLACommonMetadataBuilder
@staticmethod
def get_kv_cache_shape(
num_blocks: int,
block_size: int,
num_kv_heads: int, # assumed to be 1 for MLA
head_size: int,
cache_dtype_str: str = "auto",
) -> tuple[int, ...]:
return (num_blocks, block_size, head_size)
@staticmethod
def get_kv_cache_stride_order(
include_num_layers_dimension: bool = False,
) -> tuple[int, ...]:
if include_num_layers_dimension:
# Default to identity permutation to signal cross-layer allocation
# is unsupported. Each MLA backend must opt in to support cross-layer
# allocation by overriding this method.
return (0, 1, 2, 3)
return (0, 1, 2)
@classmethod
def get_supported_head_sizes(cls) -> list[int]:
return [320, 576]
+15 -12
View File
@@ -61,16 +61,16 @@ def hpc_rope_norm_forward(
return
attn_layer = forward_context.no_compile_layers[layer_name]
# bind_kv_cache stores the per-layer KV cache as a single 5D tensor
# (num_blocks, 2, block_size, num_kv_heads, head_size), so use it directly.
# bind_kv_cache stores the per-layer KV cache as a single tensor
# (num_blocks, num_kv_heads, block_size, 2 * head_size), so use it directly.
kv_cache = attn_layer.kv_cache
if kv_cache.numel() == 0:
output.zero_()
return
assert kv_cache.dim() == 5, (
f"Expected kv_cache to have 5 dims, got {tuple(kv_cache.shape)}"
assert kv_cache.dim() == 4, (
f"Expected kv_cache to have 4 dims, got {tuple(kv_cache.shape)}"
)
rope_norm = _hpc_rope_norm_instances[layer_name]
@@ -295,6 +295,9 @@ class HpcRopeNorm(CustomOp, HpcModule):
"""
import hpc
# (B, H, N, 2*hs) -> two (B, N, H, hs) K/V views, as in HpcAttentionImpl.
key_cache, value_cache = kv_cache.transpose(1, 2).split(self.head_dim, dim=-1)
num_actual_tokens = attn_metadata.num_actual_tokens
num_prefill_reqs = attn_metadata.num_prefills
num_decode_reqs = attn_metadata.num_decodes
@@ -333,8 +336,8 @@ class HpcRopeNorm(CustomOp, HpcModule):
if self.use_fp8:
_, q_scale, split_k_flag = hpc.rope_norm_store_kv_fp8(
key_cache=kv_cache[:, 0],
value_cache=kv_cache[:, 1],
key_cache=key_cache,
value_cache=value_cache,
qkv=qkv_prefill,
cos_sin=self.cos_sin_cache,
num_seqlen_per_req=seq_lens_prefill,
@@ -353,8 +356,8 @@ class HpcRopeNorm(CustomOp, HpcModule):
attn_metadata.hpc_prefill_q_scale = q_scale
else:
hpc.rope_norm_store_kv(
kv_cache[:, 0],
kv_cache[:, 1],
key_cache,
value_cache,
qkv_prefill,
self.cos_sin_cache,
seq_lens_prefill,
@@ -379,8 +382,8 @@ class HpcRopeNorm(CustomOp, HpcModule):
if self.use_fp8:
_, q_scale, split_k_flag = hpc.rope_norm_store_kv_fp8(
key_cache=kv_cache[:, 0],
value_cache=kv_cache[:, 1],
key_cache=key_cache,
value_cache=value_cache,
qkv=qkv_decode,
cos_sin=self.cos_sin_cache,
num_seqlen_per_req=num_seq_kvcache,
@@ -401,8 +404,8 @@ class HpcRopeNorm(CustomOp, HpcModule):
attn_metadata.hpc_split_k_flag = split_k_flag
else:
hpc.rope_norm_store_kv(
kv_cache[:, 0],
kv_cache[:, 1],
key_cache,
value_cache,
qkv_decode,
self.cos_sin_cache,
num_seq_kvcache,