diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index 552dd28c9f..04aee9d48b 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -1946,8 +1946,12 @@ class VllmConfig: model_config = self.model_config speculative_config = self.speculative_config - if model_config is not None and model_config.has_inner_state: - unsupported.append("hybrid/mamba models") + if ( + model_config is not None + and model_config.has_inner_state + and self.cache_config.mamba_cache_mode == "align" + ): + unsupported.append("hybrid/mamba models with align cache mode") if self.parallel_config.prefill_context_parallel_size > 1: unsupported.append("prefill context parallelism") diff --git a/vllm/v1/worker/gpu/attn_utils.py b/vllm/v1/worker/gpu/attn_utils.py index 8844f06a88..c2ba12437f 100644 --- a/vllm/v1/worker/gpu/attn_utils.py +++ b/vllm/v1/worker/gpu/attn_utils.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from collections.abc import Sequence +from collections.abc import Iterable, Sequence from dataclasses import dataclass from typing import Any, cast @@ -11,7 +11,6 @@ from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase from vllm.utils.torch_utils import get_dtype_size from vllm.v1.attention.backend import ( - AttentionBackend, AttentionCGSupport, CommonAttentionMetadata, ) @@ -65,13 +64,8 @@ def init_attn_backend( vllm_config: VllmConfig, device: torch.device, active_layer_names: set[str] | None = None, -) -> tuple[ - dict[str, type[AttentionBackend]], - list[list[AttentionGroup]], - AttentionCGSupportInfo, - list[int], -]: - attn_backends: dict[str, type[AttentionBackend]] = {} +) -> tuple[list[list[AttentionGroup]], AttentionCGSupportInfo, list[int]]: + # Phase 1: discover attention groups for each kv cache group. attn_groups: list[list[AttentionGroup]] = [] # Add KV-sharing layers to their target's kv cache group so they are @@ -96,7 +90,6 @@ def init_attn_backend( for layer_name in layer_names: attn_backend = attn_layers[layer_name].get_attn_backend() - attn_backends[layer_name] = attn_backend layer_kv_cache_spec: KVCacheSpec = kv_cache_group_spec.kv_cache_spec if isinstance(layer_kv_cache_spec, UniformTypeKVCacheSpecs): @@ -122,7 +115,6 @@ def init_attn_backend( min_cg_support = AttentionCGSupport.ALWAYS min_cg_attn_backend = None for kv_cache_group_id, groups in enumerate(attn_groups): - kv_cache_group_spec = kv_cache_config.kv_cache_groups[kv_cache_group_id] kernel_block_size = None if kv_cache_group_id < len(kernel_block_sizes): kernel_block_size = kernel_block_sizes[kv_cache_group_id] @@ -143,21 +135,16 @@ def init_attn_backend( # Check cudagraph support for the attention backend cg_support = builder.get_cudagraph_support( vllm_config, - cast(AttentionSpec, kv_cache_group_spec.kv_cache_spec), + cast(AttentionSpec, group.kv_cache_spec), ) if cg_support.value < min_cg_support.value: min_cg_support = cg_support min_cg_attn_backend = group.backend.__name__ - return ( - attn_backends, - attn_groups, - AttentionCGSupportInfo( - min_cg_support=min_cg_support, - min_cg_attn_backend=min_cg_attn_backend, - ), - kernel_block_sizes, + attn_cg_support_info = AttentionCGSupportInfo( + min_cg_support=min_cg_support, min_cg_attn_backend=min_cg_attn_backend ) + return attn_groups, attn_cg_support_info, kernel_block_sizes def _allocate_kv_cache( @@ -180,25 +167,31 @@ def _allocate_kv_cache( def _reshape_kv_cache( - kv_cache_config: KVCacheConfig, + attn_groups: Sequence[AttentionGroup], kv_cache_raw_tensors: dict[str, torch.Tensor], - attn_backends: dict[str, type[AttentionBackend]], cache_dtype: str, kernel_block_sizes: list[int], shared_kv_cache_layers: dict[str, str], ) -> dict[str, Any]: kv_caches: dict[str, Any] = {} has_attn, has_mamba = False, False - for kv_cache_group_id, kv_cache_group_spec in enumerate( - kv_cache_config.kv_cache_groups - ): - for layer_name in kv_cache_group_spec.layer_names: + + for group in attn_groups: + if group.kv_cache_group_id >= len(kernel_block_sizes): + continue + + kv_cache_spec = group.kv_cache_spec + if kv_cache_spec.storage_block_size != kv_cache_spec.block_size: + # use storage_block_size as the kernel block size for groups + # that apply a compression on block size (eg. DeepSeek V4). + kernel_block_size = kv_cache_spec.storage_block_size + else: + kernel_block_size = kernel_block_sizes[group.kv_cache_group_id] + + for layer_name in group.layer_names: if layer_name in shared_kv_cache_layers: # Shared layer — tensor will be aliased to its target later. continue - kv_cache_spec = kv_cache_group_spec.kv_cache_spec - if isinstance(kv_cache_spec, UniformTypeKVCacheSpecs): - kv_cache_spec = kv_cache_spec.kv_cache_specs[layer_name] kv_raw_tensor = kv_cache_raw_tensors[layer_name] assert kv_raw_tensor.numel() % kv_cache_spec.page_size_bytes == 0 @@ -206,22 +199,11 @@ def _reshape_kv_cache( if isinstance(kv_cache_spec, AttentionSpec): has_attn = True - attn_backend = attn_backends[layer_name] - - if kv_cache_group_id < len(kernel_block_sizes): - kernel_block_size = kernel_block_sizes[kv_cache_group_id] - num_blocks *= kv_cache_spec.block_size // kernel_block_size - else: - kernel_block_size = kv_cache_spec.block_size - - if kv_cache_spec.storage_block_size != kv_cache_spec.block_size: - shape_block_size = kv_cache_spec.storage_block_size - else: - shape_block_size = kernel_block_size - - kv_cache_shape = attn_backend.get_kv_cache_shape( - num_blocks, - shape_block_size, + num_blocks_per_kv_block = kv_cache_spec.block_size // kernel_block_size + kernel_num_blocks = num_blocks * num_blocks_per_kv_block + kv_cache_shape = group.backend.get_kv_cache_shape( + kernel_num_blocks, + kernel_block_size, kv_cache_spec.num_kv_heads, kv_cache_spec.head_size, cache_dtype_str=cache_dtype, @@ -229,7 +211,7 @@ def _reshape_kv_cache( # FIXME(woosuk): Add kv_cache_stride_order to all attention backends. try: - kv_cache_stride_order = attn_backend.get_kv_cache_stride_order() + kv_cache_stride_order = group.backend.get_kv_cache_stride_order() assert len(kv_cache_stride_order) == len(kv_cache_shape) except (AttributeError, NotImplementedError): kv_cache_stride_order = tuple(range(len(kv_cache_shape))) @@ -290,7 +272,12 @@ def _reshape_kv_cache( ) if has_attn and has_mamba: - _update_hybrid_attention_layout(kv_caches, kv_cache_config) + _update_hybrid_attention_layout( + attn_groups=attn_groups, + kv_caches=kv_caches, + kernel_block_sizes=kernel_block_sizes, + cache_dtype=cache_dtype, + ) # Map any sharing layers to their target layer's KV cache. for layer_name, target_layer_name in shared_kv_cache_layers.items(): @@ -300,18 +287,38 @@ def _reshape_kv_cache( def _update_hybrid_attention_layout( - kv_caches: dict[str, Any], kv_cache_config: KVCacheConfig + attn_groups: Iterable[AttentionGroup], + kv_caches: dict[str, Any], + kernel_block_sizes: list[int], + cache_dtype: str, ) -> None: - for kv_cache_group_spec in kv_cache_config.kv_cache_groups: - for layer_name in kv_cache_group_spec.layer_names: + for group in attn_groups: + if group.kv_cache_group_id >= len(kernel_block_sizes): + continue + + kv_cache_spec = group.kv_cache_spec + if not isinstance(kv_cache_spec, AttentionSpec): + continue + block_dim = group.backend.get_kv_cache_block_dim( + kernel_block_sizes[group.kv_cache_group_id], + kv_cache_spec.num_kv_heads, + kv_cache_spec.head_size, + cache_dtype_str=cache_dtype, + ) + # if the first dim of the kvcache's layout is already num_blocks, continue + if block_dim == 0: + continue + + assert block_dim == 1, ( + "Expected the dim `num_blocks` at the second dim when updating" + " the kvcache's layout of full attention layer" + ) + + for layer_name in group.layer_names: if layer_name not in kv_caches: # Shared layer — will be aliased to its target after this pass. continue - kv_cache_spec = kv_cache_group_spec.kv_cache_spec - if isinstance(kv_cache_spec, UniformTypeKVCacheSpecs): - kv_cache_spec = kv_cache_spec.kv_cache_specs[layer_name] - if not isinstance(kv_cache_spec, AttentionSpec): - continue + kv_cache = kv_caches[layer_name] if kv_cache.shape[0] == 2: assert kv_cache.shape[1] != 2, ( @@ -329,10 +336,10 @@ def _update_hybrid_attention_layout( def init_kv_cache( - runner_kv_caches: list[torch.Tensor], + runner_kv_caches: list[torch.Tensor | list[torch.Tensor]], forward_context: dict[str, Any], kv_cache_config: KVCacheConfig, - attn_backends: dict[str, type[AttentionBackend]], + attn_groups: list[list[AttentionGroup]], device: torch.device, cache_dtype: str, kernel_block_sizes: list[int], @@ -342,13 +349,13 @@ def init_kv_cache( kv_cache_raw_tensors = _allocate_kv_cache( kv_cache_config, shared_kv_cache_layers, device ) + flattened_attn_groups = list(group for groups in attn_groups for group in groups) kv_caches = _reshape_kv_cache( - kv_cache_config, - kv_cache_raw_tensors, - attn_backends, - cache_dtype, - kernel_block_sizes, - shared_kv_cache_layers, + attn_groups=flattened_attn_groups, + kv_cache_raw_tensors=kv_cache_raw_tensors, + kernel_block_sizes=kernel_block_sizes, + cache_dtype=cache_dtype, + shared_kv_cache_layers=shared_kv_cache_layers, ) bind_kv_cache(kv_caches, forward_context, runner_kv_caches) return kv_caches diff --git a/vllm/v1/worker/gpu/block_table.py b/vllm/v1/worker/gpu/block_table.py index 2c8c551a48..41692f58e7 100644 --- a/vllm/v1/worker/gpu/block_table.py +++ b/vllm/v1/worker/gpu/block_table.py @@ -44,9 +44,7 @@ class BlockTables: for i in range(self.num_kv_cache_groups): max_num_blocks = max_num_blocks_per_group[i] * self.blocks_per_kv_block[i] block_table = StagedWriteTensor( - (self.max_num_reqs, max_num_blocks), - dtype=torch.int32, - device=device, + (self.max_num_reqs, max_num_blocks), dtype=torch.int32, device=device ) self.block_tables.append(block_table) @@ -132,7 +130,6 @@ class BlockTables: self.num_blocks.gpu, self.num_blocks.gpu.stride(0), num_reqs, - self.input_block_tables[0].shape[1], # max_num_blocks BLOCK_SIZE=1024, # type: ignore ) return tuple(bt[:num_reqs_padded] for bt in self.input_block_tables) @@ -192,7 +189,6 @@ def _gather_block_tables_kernel( num_blocks_ptr, # [num_kv_cache_groups, max_num_reqs] num_blocks_stride, num_reqs, # actual number of requests (for padding) - max_num_blocks, # stride for zeroing padded rows BLOCK_SIZE: tl.constexpr, ): # kv cache group id @@ -200,6 +196,7 @@ def _gather_block_tables_kernel( batch_idx = tl.program_id(1) stride = tl.load(block_table_strides + group_id) + max_num_blocks = stride # stride equals max_num_blocks for this group. dst_block_table_ptr = _load_ptr(dst_block_table_ptrs + group_id, tl.int32) dst_row_ptr = dst_block_table_ptr + batch_idx * stride diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 52b4475508..eae47bf423 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -393,10 +393,9 @@ class GPUModelRunner(LoRAModelRunnerMixin): ) + spec.num_speculative_blocks max_num_blocks_per_group.append(max_num_blocks) - (self.attn_backends, self.attn_groups, attn_cg_support, kernel_block_sizes) = ( - init_attn_backend(self.kv_cache_config, self.vllm_config, self.device) + self.attn_groups, attn_cg_support, kernel_block_sizes = init_attn_backend( + self.kv_cache_config, self.vllm_config, self.device ) - self.block_tables = BlockTables( block_sizes=block_sizes, max_num_reqs=self.max_num_reqs, @@ -440,7 +439,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): self.kv_caches, self.compilation_config.static_forward_context, self.kv_cache_config, - self.attn_backends, + self.attn_groups, self.device, self.cache_config.cache_dtype, kernel_block_sizes, diff --git a/vllm/v1/worker/gpu/spec_decode/eagle/speculator.py b/vllm/v1/worker/gpu/spec_decode/eagle/speculator.py index af1c3608da..6ae3fe793b 100644 --- a/vllm/v1/worker/gpu/spec_decode/eagle/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/eagle/speculator.py @@ -173,7 +173,7 @@ class EagleSpeculator: ) -> None: self.model_state = model_state self.kv_cache_config = kv_cache_config - _, self.attn_groups, _, _ = init_attn_backend( + self.attn_groups, _, _ = init_attn_backend( kv_cache_config, self.vllm_config, self.device,