[3/N][KV-Cache Layout Refactor] Standardize Mamba cache; drop get_transfer_cache_regions (#44456)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
Claude
OpenAI Codex
parent
eb44b3aaa4
commit
6700813f86
@@ -4,6 +4,8 @@
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.v1.attention.backend import AttentionBackend, AttentionImpl
|
||||
from vllm.v1.kv_cache_interface import KVCacheSpec
|
||||
@@ -21,6 +23,14 @@ class AttentionLayerBase(ABC):
|
||||
impl: "AttentionImpl"
|
||||
supports_dcp: bool = True
|
||||
|
||||
def bind_kv_cache(self, kv_cache: torch.Tensor) -> None:
|
||||
"""Bind the allocated KV cache tensor to this layer.
|
||||
|
||||
The default stores the cache view as-is; subclasses (e.g. Mamba)
|
||||
override this to unpack the raw buffer into per-state views.
|
||||
"""
|
||||
self.kv_cache = kv_cache
|
||||
|
||||
@abstractmethod
|
||||
def get_attn_backend(self) -> type[AttentionBackend]:
|
||||
"""Get the attention backend class for this layer."""
|
||||
|
||||
@@ -2,11 +2,13 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from abc import abstractmethod
|
||||
from collections.abc import Iterable
|
||||
from math import prod
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.utils.torch_utils import get_dtype_size
|
||||
from vllm.v1.attention.backend import AttentionBackend
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
from vllm.v1.attention.selector import get_mamba_attn_backend
|
||||
@@ -24,6 +26,22 @@ class MambaBase(AttentionLayerBase):
|
||||
kv_cache: tuple[torch.Tensor, ...]
|
||||
supports_dcp: bool = False
|
||||
|
||||
def bind_kv_cache(self, kv_cache: torch.Tensor) -> None:
|
||||
"""Unpack a raw ``[B, 1, 1, C]`` int8 page view into per-state views.
|
||||
|
||||
Each block's ``C`` bytes hold the layer's states (e.g. conv, ssm)
|
||||
packed contiguously; slice them out and reinterpret per dtype/shape.
|
||||
"""
|
||||
pages = kv_cache.squeeze(dim=(1, 2))
|
||||
states: list[torch.Tensor] = []
|
||||
offset = 0
|
||||
for shape, dtype in zip(self.get_state_shape(), self.get_state_dtype()):
|
||||
nbytes = prod(shape) * get_dtype_size(dtype)
|
||||
state = pages[:, offset : offset + nbytes].view(dtype)
|
||||
states.append(state.view(-1, *shape))
|
||||
offset += nbytes
|
||||
self.kv_cache = tuple(states)
|
||||
|
||||
@abstractmethod
|
||||
def get_state_shape(self) -> Iterable[tuple[int, ...]]:
|
||||
"""
|
||||
|
||||
Reference in New Issue
Block a user