[3/N][KV-Cache Layout Refactor] Standardize Mamba cache; drop get_transfer_cache_regions (#44456)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
Claude
OpenAI Codex
parent
eb44b3aaa4
commit
6700813f86
@@ -262,7 +262,7 @@ def _reshape_kv_cache(
|
||||
kv_cache_config: "KVCacheConfig | None" = None,
|
||||
) -> dict[str, Any]:
|
||||
kv_caches: dict[str, Any] = {}
|
||||
has_attn, has_mamba = False, False
|
||||
has_attn = False
|
||||
|
||||
layer_packing: dict[str, tuple[int, int]] = {}
|
||||
if kv_cache_config is not None:
|
||||
@@ -340,38 +340,21 @@ def _reshape_kv_cache(
|
||||
)
|
||||
|
||||
elif isinstance(kv_cache_spec, MambaSpec):
|
||||
has_mamba = True
|
||||
state_tensors = []
|
||||
storage_offset_bytes = 0
|
||||
for shape, dtype in zip(kv_cache_spec.shapes, kv_cache_spec.dtypes):
|
||||
dtype_size = get_dtype_size(dtype)
|
||||
num_element_per_page = kv_cache_spec.page_size_bytes // dtype_size
|
||||
target_shape = (num_blocks, *shape)
|
||||
stride = torch.empty(target_shape).stride()
|
||||
target_stride = (num_element_per_page, *stride[1:])
|
||||
assert storage_offset_bytes % dtype_size == 0
|
||||
tensor = torch.as_strided(
|
||||
kv_raw_tensor.view(dtype),
|
||||
size=target_shape,
|
||||
stride=target_stride,
|
||||
storage_offset=storage_offset_bytes // dtype_size,
|
||||
)
|
||||
state_tensors.append(tensor)
|
||||
storage_offset_bytes += stride[0] * dtype_size
|
||||
kv_caches[layer_name] = state_tensors
|
||||
page_size_bytes = kv_cache_spec.page_size_bytes
|
||||
# Hold a single contiguous [num_blocks, 1, 1, page_size_bytes]
|
||||
# int8 page view per layer; the layer's bind_kv_cache unpacks
|
||||
# each block's bytes into its conv/ssm state views. Keeping
|
||||
# one tensor per layer lets the KV connector register it
|
||||
# without special-casing Mamba.
|
||||
kv_caches[layer_name] = kv_raw_tensor[
|
||||
: num_blocks * page_size_bytes
|
||||
].view(num_blocks, 1, 1, page_size_bytes)
|
||||
else:
|
||||
raise NotImplementedError(
|
||||
f"Unsupported KV cache spec type: {type(kv_cache_spec)}"
|
||||
)
|
||||
|
||||
if has_attn and has_mamba:
|
||||
_update_hybrid_attention_layout(
|
||||
attn_groups=attn_groups,
|
||||
kv_caches=kv_caches,
|
||||
kernel_block_sizes=kernel_block_sizes,
|
||||
cache_dtype=cache_dtype,
|
||||
)
|
||||
elif has_attn and kv_cache_config is not None:
|
||||
if has_attn and kv_cache_config is not None:
|
||||
_align_mixed_attention_kv_cache_views(
|
||||
attn_groups=attn_groups,
|
||||
kv_caches=kv_caches,
|
||||
@@ -458,65 +441,6 @@ def _restride_blocks_first_kv_cache_to_kv_first_storage(
|
||||
)
|
||||
|
||||
|
||||
def _update_hybrid_attention_layout(
|
||||
attn_groups: Iterable[AttentionGroup],
|
||||
kv_caches: dict[str, Any],
|
||||
kernel_block_sizes: list[int],
|
||||
cache_dtype: str,
|
||||
) -> None:
|
||||
for group in attn_groups:
|
||||
if group.kv_cache_group_id >= len(kernel_block_sizes):
|
||||
continue
|
||||
|
||||
kv_cache_spec = group.kv_cache_spec
|
||||
if not isinstance(kv_cache_spec, AttentionSpec):
|
||||
continue
|
||||
# Mirror the per-layer dtype selection used when building the shape
|
||||
# above. The block-dim index is dtype-independent for current backends
|
||||
# (quantization only changes the last dim), so this is a no-op today,
|
||||
# but it keeps both call sites consistent for skip layers.
|
||||
layer_cache_dtype = (
|
||||
"auto"
|
||||
if kv_cache_spec.kv_quant_mode == KVQuantMode.NONE
|
||||
and not isinstance(kv_cache_spec, TQFullAttentionSpec)
|
||||
else cache_dtype
|
||||
)
|
||||
block_dim = group.backend.get_kv_cache_block_dim(
|
||||
kernel_block_sizes[group.kv_cache_group_id],
|
||||
kv_cache_spec.num_kv_heads,
|
||||
kv_cache_spec.head_size,
|
||||
cache_dtype_str=layer_cache_dtype,
|
||||
)
|
||||
# if the first dim of the kvcache's layout is already num_blocks, continue
|
||||
if block_dim == 0:
|
||||
continue
|
||||
|
||||
assert block_dim == 1, (
|
||||
"Expected the dim `num_blocks` at the second dim when updating"
|
||||
" the kvcache's layout of full attention layer"
|
||||
)
|
||||
|
||||
for layer_name in group.layer_names:
|
||||
if layer_name not in kv_caches:
|
||||
# Shared layer — will be aliased to its target after this pass.
|
||||
continue
|
||||
|
||||
kv_cache = kv_caches[layer_name]
|
||||
if kv_cache.shape[0] == 2:
|
||||
assert kv_cache.shape[1] != 2, (
|
||||
f"Cannot determine layout for tensor of shape {kv_cache.shape}"
|
||||
)
|
||||
hidden_size = kv_cache.shape[2:].numel()
|
||||
kv_cache.as_strided_(
|
||||
size=kv_cache.shape,
|
||||
stride=(
|
||||
hidden_size,
|
||||
2 * hidden_size,
|
||||
*kv_cache.stride()[2:],
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def init_kv_cache(
|
||||
runner_kv_caches: list[torch.Tensor | list[torch.Tensor]],
|
||||
forward_context: dict[str, Any],
|
||||
|
||||
Reference in New Issue
Block a user