[3/N][KV-Cache Layout Refactor] Standardize Mamba cache; drop get_transfer_cache_regions (#44456)

Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Lucas Wilkinson
2026-07-21 09:16:15 +00:00
committed by GitHub
co-authored by Claude OpenAI Codex
parent eb44b3aaa4
commit 6700813f86
10 changed files with 115 additions and 225 deletions
+11 -87
View File
@@ -262,7 +262,7 @@ def _reshape_kv_cache(
kv_cache_config: "KVCacheConfig | None" = None,
) -> dict[str, Any]:
kv_caches: dict[str, Any] = {}
has_attn, has_mamba = False, False
has_attn = False
layer_packing: dict[str, tuple[int, int]] = {}
if kv_cache_config is not None:
@@ -340,38 +340,21 @@ def _reshape_kv_cache(
)
elif isinstance(kv_cache_spec, MambaSpec):
has_mamba = True
state_tensors = []
storage_offset_bytes = 0
for shape, dtype in zip(kv_cache_spec.shapes, kv_cache_spec.dtypes):
dtype_size = get_dtype_size(dtype)
num_element_per_page = kv_cache_spec.page_size_bytes // dtype_size
target_shape = (num_blocks, *shape)
stride = torch.empty(target_shape).stride()
target_stride = (num_element_per_page, *stride[1:])
assert storage_offset_bytes % dtype_size == 0
tensor = torch.as_strided(
kv_raw_tensor.view(dtype),
size=target_shape,
stride=target_stride,
storage_offset=storage_offset_bytes // dtype_size,
)
state_tensors.append(tensor)
storage_offset_bytes += stride[0] * dtype_size
kv_caches[layer_name] = state_tensors
page_size_bytes = kv_cache_spec.page_size_bytes
# Hold a single contiguous [num_blocks, 1, 1, page_size_bytes]
# int8 page view per layer; the layer's bind_kv_cache unpacks
# each block's bytes into its conv/ssm state views. Keeping
# one tensor per layer lets the KV connector register it
# without special-casing Mamba.
kv_caches[layer_name] = kv_raw_tensor[
: num_blocks * page_size_bytes
].view(num_blocks, 1, 1, page_size_bytes)
else:
raise NotImplementedError(
f"Unsupported KV cache spec type: {type(kv_cache_spec)}"
)
if has_attn and has_mamba:
_update_hybrid_attention_layout(
attn_groups=attn_groups,
kv_caches=kv_caches,
kernel_block_sizes=kernel_block_sizes,
cache_dtype=cache_dtype,
)
elif has_attn and kv_cache_config is not None:
if has_attn and kv_cache_config is not None:
_align_mixed_attention_kv_cache_views(
attn_groups=attn_groups,
kv_caches=kv_caches,
@@ -458,65 +441,6 @@ def _restride_blocks_first_kv_cache_to_kv_first_storage(
)
def _update_hybrid_attention_layout(
attn_groups: Iterable[AttentionGroup],
kv_caches: dict[str, Any],
kernel_block_sizes: list[int],
cache_dtype: str,
) -> None:
for group in attn_groups:
if group.kv_cache_group_id >= len(kernel_block_sizes):
continue
kv_cache_spec = group.kv_cache_spec
if not isinstance(kv_cache_spec, AttentionSpec):
continue
# Mirror the per-layer dtype selection used when building the shape
# above. The block-dim index is dtype-independent for current backends
# (quantization only changes the last dim), so this is a no-op today,
# but it keeps both call sites consistent for skip layers.
layer_cache_dtype = (
"auto"
if kv_cache_spec.kv_quant_mode == KVQuantMode.NONE
and not isinstance(kv_cache_spec, TQFullAttentionSpec)
else cache_dtype
)
block_dim = group.backend.get_kv_cache_block_dim(
kernel_block_sizes[group.kv_cache_group_id],
kv_cache_spec.num_kv_heads,
kv_cache_spec.head_size,
cache_dtype_str=layer_cache_dtype,
)
# if the first dim of the kvcache's layout is already num_blocks, continue
if block_dim == 0:
continue
assert block_dim == 1, (
"Expected the dim `num_blocks` at the second dim when updating"
" the kvcache's layout of full attention layer"
)
for layer_name in group.layer_names:
if layer_name not in kv_caches:
# Shared layer — will be aliased to its target after this pass.
continue
kv_cache = kv_caches[layer_name]
if kv_cache.shape[0] == 2:
assert kv_cache.shape[1] != 2, (
f"Cannot determine layout for tensor of shape {kv_cache.shape}"
)
hidden_size = kv_cache.shape[2:].numel()
kv_cache.as_strided_(
size=kv_cache.shape,
stride=(
hidden_size,
2 * hidden_size,
*kv_cache.stride()[2:],
),
)
def init_kv_cache(
runner_kv_caches: list[torch.Tensor | list[torch.Tensor]],
forward_context: dict[str, Any],