[6/N][KV-Cache Layout Refactor] Standardize KV cache layout (#51718)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
This commit is contained in:
@@ -649,6 +649,10 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
compile_native=True,
|
||||
)
|
||||
|
||||
def bind_kv_cache(self, kv_cache: torch.Tensor) -> None:
|
||||
# [B, H=1, N, C] -> [B, N, C]
|
||||
self.kv_cache = kv_cache.squeeze(1)
|
||||
|
||||
@property
|
||||
def chunked_prefill_workspace_size(self) -> int:
|
||||
if self._chunked_prefill_workspace_size is None:
|
||||
@@ -1451,27 +1455,6 @@ class MLACommonBackend(AttentionBackend):
|
||||
def get_builder_cls() -> type["MLACommonMetadataBuilder"]:
|
||||
return MLACommonMetadataBuilder
|
||||
|
||||
@staticmethod
|
||||
def get_kv_cache_shape(
|
||||
num_blocks: int,
|
||||
block_size: int,
|
||||
num_kv_heads: int, # assumed to be 1 for MLA
|
||||
head_size: int,
|
||||
cache_dtype_str: str = "auto",
|
||||
) -> tuple[int, ...]:
|
||||
return (num_blocks, block_size, head_size)
|
||||
|
||||
@staticmethod
|
||||
def get_kv_cache_stride_order(
|
||||
include_num_layers_dimension: bool = False,
|
||||
) -> tuple[int, ...]:
|
||||
if include_num_layers_dimension:
|
||||
# Default to identity permutation to signal cross-layer allocation
|
||||
# is unsupported. Each MLA backend must opt in to support cross-layer
|
||||
# allocation by overriding this method.
|
||||
return (0, 1, 2, 3)
|
||||
return (0, 1, 2)
|
||||
|
||||
@classmethod
|
||||
def get_supported_head_sizes(cls) -> list[int]:
|
||||
return [320, 576]
|
||||
|
||||
Reference in New Issue
Block a user