[6/N][KV-Cache Layout Refactor] Standardize KV cache layout (#51718)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
This commit is contained in:
@@ -42,7 +42,9 @@ from vllm.config.model import PROCESSED_LOGPROBS_MODES
|
||||
from vllm.distributed.ec_transfer import get_ec_transfer, has_ec_transfer
|
||||
from vllm.distributed.eplb.eplb_state import EplbState
|
||||
from vllm.distributed.kv_transfer import get_kv_transfer_group, has_kv_transfer_group
|
||||
from vllm.distributed.kv_transfer.kv_connector.utils import copy_kv_blocks
|
||||
from vllm.distributed.kv_transfer.kv_connector.utils import (
|
||||
copy_kv_blocks,
|
||||
)
|
||||
from vllm.distributed.parallel_state import (
|
||||
GraphCaptureContext,
|
||||
get_dcp_group,
|
||||
@@ -164,8 +166,6 @@ from vllm.v1.kv_cache_interface import (
|
||||
KVCacheGroupSpec,
|
||||
KVCacheSpec,
|
||||
KVCacheSpecKind,
|
||||
KVQuantMode,
|
||||
MambaSpec,
|
||||
SlidingWindowSpec,
|
||||
UniformTypeKVCacheSpecs,
|
||||
get_kv_cache_spec_kind,
|
||||
@@ -221,7 +221,6 @@ from vllm.v1.worker.cp_utils import (
|
||||
)
|
||||
from vllm.v1.worker.dp_utils import coordinate_batch_across_dp
|
||||
from vllm.v1.worker.ec_connector_model_runner_mixin import ECConnectorModelRunnerMixin
|
||||
from vllm.v1.worker.gpu.attn_utils import _reshape_attention_kv_cache
|
||||
from vllm.v1.worker.gpu_input_batch import CachedRequestState, InputBatch
|
||||
from vllm.v1.worker.gpu_ubatch_wrapper import UBatchWrapper
|
||||
from vllm.v1.worker.kv_connector_model_runner_mixin import KVConnectorModelRunnerMixin
|
||||
@@ -243,6 +242,7 @@ from .utils import (
|
||||
AttentionGroup,
|
||||
KVBlockZeroer,
|
||||
add_kv_sharing_layers_to_kv_cache_groups,
|
||||
allocate_kv_cache,
|
||||
bind_kv_cache,
|
||||
copy_kv_cache_blocks_inplace,
|
||||
prepare_kernel_block_sizes,
|
||||
@@ -612,9 +612,6 @@ class GPUModelRunner(
|
||||
# self.model: nn.Module # Set after load_model
|
||||
# Initialize in initialize_kv_cache
|
||||
self.kv_caches: list[torch.Tensor] = []
|
||||
# Initialize in initialize_kv_cache_tensors
|
||||
self.cross_layers_kv_cache: torch.Tensor | None = None
|
||||
self.cross_layers_attn_backend: type[AttentionBackend] | None = None
|
||||
# indexes: [kv_cache_group_id][attn_group]
|
||||
self.attn_groups: list[list[AttentionGroup]] = []
|
||||
# self.kv_cache_config: KVCacheConfig
|
||||
@@ -1200,7 +1197,6 @@ class GPUModelRunner(
|
||||
self.device,
|
||||
attn_groups_iter=self._kv_cache_spec_attn_group_iterator(),
|
||||
kernel_block_sizes=self._kernel_block_sizes,
|
||||
cache_dtype=self.cache_config.cache_dtype,
|
||||
runner_only_attn_layers=self.runner_only_attn_layers,
|
||||
static_forward_context=self.compilation_config.static_forward_context,
|
||||
)
|
||||
@@ -6707,9 +6703,6 @@ class GPUModelRunner(
|
||||
for i in range(len(self.kv_caches)):
|
||||
self.kv_caches[i] = None # type: ignore
|
||||
self.kv_caches.clear()
|
||||
if hasattr(self, "cross_layers_kv_cache"):
|
||||
self.cross_layers_kv_cache = None
|
||||
self.cross_layers_attn_backend = None
|
||||
if hasattr(self, "attn_groups"):
|
||||
self.attn_groups.clear()
|
||||
if hasattr(self, "kv_cache_config"):
|
||||
@@ -7441,49 +7434,6 @@ class GPUModelRunner(
|
||||
f"!= kv_cache kernel_block_sizes {kernel_block_sizes}"
|
||||
)
|
||||
|
||||
def _allocate_kv_cache_tensors(
|
||||
self, kv_cache_config: KVCacheConfig
|
||||
) -> dict[str, torch.Tensor]:
|
||||
"""
|
||||
Initializes the KV cache buffer with the correct size. The buffer needs
|
||||
to be reshaped to the desired shape before being used by the models.
|
||||
|
||||
Args:
|
||||
kv_cache_config: The KV cache config
|
||||
Returns:
|
||||
dict[str, torch.Tensor]: A map between layer names to their
|
||||
corresponding memory buffer for KV cache.
|
||||
"""
|
||||
kv_cache_raw_tensors: dict[str, torch.Tensor] = {}
|
||||
packed_backing: torch.Tensor | None = None
|
||||
for kv_cache_tensor in kv_cache_config.kv_cache_tensors:
|
||||
if kv_cache_tensor.block_stride > 0:
|
||||
# Allocate once; all packed tensors alias the same backing.
|
||||
if packed_backing is None:
|
||||
packed_backing = torch.zeros(
|
||||
kv_cache_tensor.size,
|
||||
dtype=torch.int8,
|
||||
device=self.device,
|
||||
)
|
||||
tensor = packed_backing
|
||||
else:
|
||||
tensor = torch.zeros(
|
||||
kv_cache_tensor.size, dtype=torch.int8, device=self.device
|
||||
)
|
||||
for layer_name in kv_cache_tensor.shared_by:
|
||||
kv_cache_raw_tensors[layer_name] = tensor
|
||||
|
||||
layer_names = set()
|
||||
for group in kv_cache_config.kv_cache_groups:
|
||||
for layer_name in group.layer_names:
|
||||
if layer_name in self.runner_only_attn_layers:
|
||||
continue
|
||||
layer_names.add(layer_name)
|
||||
assert layer_names == set(kv_cache_raw_tensors.keys()), (
|
||||
"Some layers are not correctly initialized"
|
||||
)
|
||||
return kv_cache_raw_tensors
|
||||
|
||||
def _attn_group_iterator(self) -> Iterator[AttentionGroup]:
|
||||
return itertools.chain.from_iterable(self.attn_groups)
|
||||
|
||||
@@ -7493,183 +7443,6 @@ class GPUModelRunner(
|
||||
for attn_groups in self.attn_groups:
|
||||
yield from attn_groups
|
||||
|
||||
def _reshape_kv_cache_tensors(
|
||||
self,
|
||||
kv_cache_raw_tensors: dict[str, torch.Tensor],
|
||||
kernel_block_sizes: list[int],
|
||||
) -> dict[str, torch.Tensor]:
|
||||
"""
|
||||
Reshape the KV cache tensors to the desired shape and dtype.
|
||||
|
||||
Args:
|
||||
kv_cache_raw_tensors: The KV cache buffer of each layer, with
|
||||
correct size but uninitialized shape.
|
||||
kernel_block_sizes: The kernel block sizes for each KV cache group.
|
||||
Returns:
|
||||
Dict[str, torch.Tensor]: A map between layer names to their
|
||||
corresponding memory buffer for KV cache.
|
||||
"""
|
||||
kv_caches: dict[str, torch.Tensor] = {}
|
||||
has_attn, has_mamba = False, False
|
||||
|
||||
# Map layer names to (offset, block_stride) within the packed
|
||||
# backing tensor so we can create strided views per layer.
|
||||
layer_packing: dict[str, tuple[int, int]] = {}
|
||||
for kv_tensor in self.kv_cache_config.kv_cache_tensors:
|
||||
if kv_tensor.block_stride > 0:
|
||||
for ln in kv_tensor.shared_by:
|
||||
layer_packing[ln] = (kv_tensor.offset, kv_tensor.block_stride)
|
||||
for group in self._kv_cache_spec_attn_group_iterator():
|
||||
kv_cache_spec = group.kv_cache_spec
|
||||
attn_backend = group.backend
|
||||
if group.kv_cache_group_id == len(kernel_block_sizes):
|
||||
# There may be a last group for layers without kv cache.
|
||||
continue
|
||||
kernel_block_size = kernel_block_sizes[group.kv_cache_group_id]
|
||||
for layer_name in group.layer_names:
|
||||
if layer_name in self.runner_only_attn_layers:
|
||||
continue
|
||||
raw_tensor = kv_cache_raw_tensors[layer_name]
|
||||
packing = layer_packing.get(layer_name)
|
||||
if packing is not None:
|
||||
_, blk_stride = packing
|
||||
num_blocks = raw_tensor.numel() // blk_stride
|
||||
else:
|
||||
assert raw_tensor.numel() % kv_cache_spec.page_size_bytes == 0
|
||||
num_blocks = raw_tensor.numel() // kv_cache_spec.page_size_bytes
|
||||
if isinstance(kv_cache_spec, AttentionSpec):
|
||||
has_attn = True
|
||||
num_blocks_per_kv_block = (
|
||||
kv_cache_spec.block_size // kernel_block_size
|
||||
)
|
||||
kernel_num_blocks = num_blocks * num_blocks_per_kv_block
|
||||
|
||||
# For MLA with compression, storage_block_size != block_size
|
||||
if kv_cache_spec.storage_block_size != kv_cache_spec.block_size:
|
||||
shape_block_size = kv_cache_spec.storage_block_size
|
||||
else:
|
||||
shape_block_size = kernel_block_size
|
||||
|
||||
# Skipped layers (--kv-cache-dtype-skip-layers) need
|
||||
# the unquantized shape.
|
||||
layer_cache_dtype_str = (
|
||||
"auto"
|
||||
if kv_cache_spec.kv_quant_mode == KVQuantMode.NONE
|
||||
else getattr(
|
||||
kv_cache_spec,
|
||||
"cache_dtype_str",
|
||||
None,
|
||||
)
|
||||
or self.cache_config.cache_dtype
|
||||
)
|
||||
kv_cache_shape = attn_backend.get_kv_cache_shape(
|
||||
kernel_num_blocks,
|
||||
shape_block_size,
|
||||
kv_cache_spec.num_kv_heads,
|
||||
kv_cache_spec.head_size,
|
||||
cache_dtype_str=layer_cache_dtype_str,
|
||||
)
|
||||
try:
|
||||
kv_cache_stride_order = attn_backend.get_kv_cache_stride_order()
|
||||
assert len(kv_cache_stride_order) == len(kv_cache_shape)
|
||||
except (AttributeError, NotImplementedError):
|
||||
kv_cache_stride_order = tuple(range(len(kv_cache_shape)))
|
||||
raw_tensor = kv_cache_raw_tensors[layer_name]
|
||||
kv_caches[layer_name] = _reshape_attention_kv_cache(
|
||||
raw_tensor,
|
||||
kv_cache_spec,
|
||||
kv_cache_shape,
|
||||
kv_cache_stride_order,
|
||||
kernel_num_blocks,
|
||||
packing,
|
||||
)
|
||||
|
||||
elif isinstance(kv_cache_spec, MambaSpec):
|
||||
has_mamba = True
|
||||
raw_tensor = kv_cache_raw_tensors[layer_name]
|
||||
page_size_bytes = kv_cache_spec.page_size_bytes
|
||||
# Hold a single contiguous [num_blocks, 1, 1, page_size_bytes]
|
||||
# int8 page view per layer; the layer's bind_kv_cache unpacks
|
||||
# each block's bytes into its conv/ssm state views. Keeping
|
||||
# one tensor per layer lets the KV connector register it
|
||||
# without special-casing Mamba.
|
||||
kv_caches[layer_name] = raw_tensor[
|
||||
: num_blocks * page_size_bytes
|
||||
].view(num_blocks, 1, 1, page_size_bytes)
|
||||
else:
|
||||
raise NotImplementedError
|
||||
|
||||
# Reconcile divergent KV layouts to blocks-first. Triggered by hybrid
|
||||
# attention/mamba models, and by encoder-decoder models whose shared
|
||||
# decoder/cross-attention allocation mixes K/V-first and blocks-first
|
||||
# backends (see _has_mixed_attention_kv_layout).
|
||||
if has_attn and (
|
||||
has_mamba or self._has_mixed_attention_kv_layout(kernel_block_sizes)
|
||||
):
|
||||
self._update_hybrid_attention_mamba_layout(kv_caches, kernel_block_sizes)
|
||||
|
||||
return kv_caches
|
||||
|
||||
def _has_mixed_attention_kv_layout(self, kernel_block_sizes: list[int]) -> bool:
|
||||
"""Whether attention groups disagree on the physical KV cache layout.
|
||||
|
||||
Encoder-decoder models (e.g. Whisper) share one raw KV allocation
|
||||
between a decoder self-attention layer (K/V-first ROCM_ATTN, block dim
|
||||
1) and a cross-attention layer (blocks-first, block dim 0). Mixed block
|
||||
dims mean a block ID maps to different bytes per layer, so the shared
|
||||
buffer must be normalized to a single (blocks-first) layout.
|
||||
"""
|
||||
block_dims: set[int] = set()
|
||||
for group in self._kv_cache_spec_attn_group_iterator():
|
||||
kv_cache_spec = group.kv_cache_spec
|
||||
if not isinstance(kv_cache_spec, AttentionSpec):
|
||||
continue
|
||||
if group.kv_cache_group_id == len(kernel_block_sizes):
|
||||
continue
|
||||
block_dims.add(
|
||||
group.backend.get_kv_cache_block_dim(
|
||||
kernel_block_sizes[group.kv_cache_group_id],
|
||||
kv_cache_spec.num_kv_heads,
|
||||
kv_cache_spec.head_size,
|
||||
cache_dtype_str=self.cache_config.cache_dtype,
|
||||
)
|
||||
)
|
||||
return len(block_dims) > 1
|
||||
|
||||
def _update_hybrid_attention_mamba_layout(
|
||||
self, kv_caches: dict[str, torch.Tensor], kernel_block_sizes: list[int]
|
||||
) -> None:
|
||||
"""
|
||||
Update the layout of attention layers from (2, num_blocks, ...) to
|
||||
(num_blocks, 2, ...).
|
||||
|
||||
Args:
|
||||
kv_caches: The KV cache buffer of each layer.
|
||||
kernel_block_sizes: The kernel block sizes for each KV cache group.
|
||||
"""
|
||||
|
||||
for group in self._kv_cache_spec_attn_group_iterator():
|
||||
kv_cache_spec = group.kv_cache_spec
|
||||
if not isinstance(kv_cache_spec, AttentionSpec):
|
||||
continue
|
||||
block_dim = group.backend.get_kv_cache_block_dim(
|
||||
kernel_block_sizes[group.kv_cache_group_id],
|
||||
kv_cache_spec.num_kv_heads,
|
||||
kv_cache_spec.head_size,
|
||||
cache_dtype_str=self.cache_config.cache_dtype,
|
||||
)
|
||||
# block_dim: 0 means (num_blocks, 2, ...); 1 means (2, num_blocks, ...).
|
||||
if block_dim == 0:
|
||||
continue
|
||||
assert block_dim == 1
|
||||
for layer_name in group.layer_names:
|
||||
kv_cache = kv_caches[layer_name]
|
||||
hidden_size = kv_cache.shape[2:].numel()
|
||||
kv_cache.as_strided_(
|
||||
size=kv_cache.shape,
|
||||
stride=(hidden_size, 2 * hidden_size, *kv_cache.stride()[2:]),
|
||||
)
|
||||
|
||||
def initialize_kv_cache_tensors(
|
||||
self, kv_cache_config: KVCacheConfig, kernel_block_sizes: list[int]
|
||||
) -> dict[str, torch.Tensor]:
|
||||
@@ -7685,29 +7458,12 @@ class GPUModelRunner(
|
||||
corresponding memory buffer for KV cache.
|
||||
"""
|
||||
|
||||
# Try creating KV caches optimized for kv-connector transfers
|
||||
cache_dtype = self.cache_config.cache_dtype
|
||||
if self.use_uniform_kv_cache(self.attn_groups):
|
||||
kv_caches, cross_layers_kv_cache, attn_backend = (
|
||||
self.allocate_uniform_kv_caches(
|
||||
kv_cache_config,
|
||||
self.attn_groups,
|
||||
cache_dtype,
|
||||
self.device,
|
||||
kernel_block_sizes,
|
||||
)
|
||||
)
|
||||
self.cross_layers_kv_cache = cross_layers_kv_cache
|
||||
self.cross_layers_attn_backend = attn_backend
|
||||
else:
|
||||
# Fallback to the general case
|
||||
# Initialize the memory buffer for KV cache
|
||||
kv_cache_raw_tensors = self._allocate_kv_cache_tensors(kv_cache_config)
|
||||
|
||||
# Change the memory buffer to the desired shape
|
||||
kv_caches = self._reshape_kv_cache_tensors(
|
||||
kv_cache_raw_tensors, kernel_block_sizes
|
||||
)
|
||||
kv_caches = allocate_kv_cache(
|
||||
kv_cache_config,
|
||||
self.device,
|
||||
self.cache_config.get_resolved_kv_cache_layout(),
|
||||
kernel_block_sizes,
|
||||
)
|
||||
|
||||
# Set up cross-layer KV cache sharing
|
||||
for layer_name, target_layer_name in self.shared_kv_cache_layers.items():
|
||||
@@ -7803,13 +7559,7 @@ class GPUModelRunner(
|
||||
|
||||
if has_kv_transfer_group() and not is_profiling:
|
||||
kv_transfer_group = get_kv_transfer_group()
|
||||
if self.cross_layers_kv_cache is not None:
|
||||
assert self.cross_layers_attn_backend is not None
|
||||
kv_transfer_group.register_cross_layers_kv_cache(
|
||||
self.cross_layers_kv_cache, self.cross_layers_attn_backend
|
||||
)
|
||||
else:
|
||||
kv_transfer_group.register_kv_caches(kv_caches)
|
||||
kv_transfer_group.register_kv_caches(kv_caches)
|
||||
kv_transfer_group.set_host_xfer_buffer_ops(copy_kv_blocks)
|
||||
|
||||
def get_routed_experts(
|
||||
@@ -7922,13 +7672,7 @@ class GPUModelRunner(
|
||||
# Skip modules that don't need KV cache (eg encoder-only attention)
|
||||
if spec := attn_module.get_kv_cache_spec(self.vllm_config):
|
||||
if isinstance(spec, AttentionSpec):
|
||||
backend = attn_module.get_attn_backend()
|
||||
# indexes_kv_by_block_stride() -> get_kv_cache_stride_order()
|
||||
# -> get_kv_cache_layout() needs the current vLLM config.
|
||||
with set_current_vllm_config(self.vllm_config):
|
||||
indexes = backend.indexes_kv_by_block_stride()
|
||||
spec = replace(spec, indexes_kv_by_block_stride=indexes)
|
||||
spec = backend.customize_spec(spec)
|
||||
spec = attn_module.get_attn_backend().customize_spec(spec)
|
||||
kv_cache_spec[layer_name] = spec
|
||||
|
||||
return kv_cache_spec
|
||||
|
||||
Reference in New Issue
Block a user