[ModelRunnerV2][Hybrid model] Support kernel block size in hybrid model (#38831)

Signed-off-by: MengqingCao <cmq0113@163.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Signed-off-by: Mengqing Cao <cmq0113@163.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Mengqing Cao
2026-05-28 00:55:55 +00:00
committed by GitHub
co-authored by Nick Hill
parent 381edde1b9
commit 1223732dda
5 changed files with 82 additions and 75 deletions
+6 -2
View File
@@ -1946,8 +1946,12 @@ class VllmConfig:
model_config = self.model_config
speculative_config = self.speculative_config
if model_config is not None and model_config.has_inner_state:
unsupported.append("hybrid/mamba models")
if (
model_config is not None
and model_config.has_inner_state
and self.cache_config.mamba_cache_mode == "align"
):
unsupported.append("hybrid/mamba models with align cache mode")
if self.parallel_config.prefill_context_parallel_size > 1:
unsupported.append("prefill context parallelism")
+70 -63
View File
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Sequence
from collections.abc import Iterable, Sequence
from dataclasses import dataclass
from typing import Any, cast
@@ -11,7 +11,6 @@ from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
from vllm.utils.torch_utils import get_dtype_size
from vllm.v1.attention.backend import (
AttentionBackend,
AttentionCGSupport,
CommonAttentionMetadata,
)
@@ -65,13 +64,8 @@ def init_attn_backend(
vllm_config: VllmConfig,
device: torch.device,
active_layer_names: set[str] | None = None,
) -> tuple[
dict[str, type[AttentionBackend]],
list[list[AttentionGroup]],
AttentionCGSupportInfo,
list[int],
]:
attn_backends: dict[str, type[AttentionBackend]] = {}
) -> tuple[list[list[AttentionGroup]], AttentionCGSupportInfo, list[int]]:
# Phase 1: discover attention groups for each kv cache group.
attn_groups: list[list[AttentionGroup]] = []
# Add KV-sharing layers to their target's kv cache group so they are
@@ -96,7 +90,6 @@ def init_attn_backend(
for layer_name in layer_names:
attn_backend = attn_layers[layer_name].get_attn_backend()
attn_backends[layer_name] = attn_backend
layer_kv_cache_spec: KVCacheSpec = kv_cache_group_spec.kv_cache_spec
if isinstance(layer_kv_cache_spec, UniformTypeKVCacheSpecs):
@@ -122,7 +115,6 @@ def init_attn_backend(
min_cg_support = AttentionCGSupport.ALWAYS
min_cg_attn_backend = None
for kv_cache_group_id, groups in enumerate(attn_groups):
kv_cache_group_spec = kv_cache_config.kv_cache_groups[kv_cache_group_id]
kernel_block_size = None
if kv_cache_group_id < len(kernel_block_sizes):
kernel_block_size = kernel_block_sizes[kv_cache_group_id]
@@ -143,21 +135,16 @@ def init_attn_backend(
# Check cudagraph support for the attention backend
cg_support = builder.get_cudagraph_support(
vllm_config,
cast(AttentionSpec, kv_cache_group_spec.kv_cache_spec),
cast(AttentionSpec, group.kv_cache_spec),
)
if cg_support.value < min_cg_support.value:
min_cg_support = cg_support
min_cg_attn_backend = group.backend.__name__
return (
attn_backends,
attn_groups,
AttentionCGSupportInfo(
min_cg_support=min_cg_support,
min_cg_attn_backend=min_cg_attn_backend,
),
kernel_block_sizes,
attn_cg_support_info = AttentionCGSupportInfo(
min_cg_support=min_cg_support, min_cg_attn_backend=min_cg_attn_backend
)
return attn_groups, attn_cg_support_info, kernel_block_sizes
def _allocate_kv_cache(
@@ -180,25 +167,31 @@ def _allocate_kv_cache(
def _reshape_kv_cache(
kv_cache_config: KVCacheConfig,
attn_groups: Sequence[AttentionGroup],
kv_cache_raw_tensors: dict[str, torch.Tensor],
attn_backends: dict[str, type[AttentionBackend]],
cache_dtype: str,
kernel_block_sizes: list[int],
shared_kv_cache_layers: dict[str, str],
) -> dict[str, Any]:
kv_caches: dict[str, Any] = {}
has_attn, has_mamba = False, False
for kv_cache_group_id, kv_cache_group_spec in enumerate(
kv_cache_config.kv_cache_groups
):
for layer_name in kv_cache_group_spec.layer_names:
for group in attn_groups:
if group.kv_cache_group_id >= len(kernel_block_sizes):
continue
kv_cache_spec = group.kv_cache_spec
if kv_cache_spec.storage_block_size != kv_cache_spec.block_size:
# use storage_block_size as the kernel block size for groups
# that apply a compression on block size (eg. DeepSeek V4).
kernel_block_size = kv_cache_spec.storage_block_size
else:
kernel_block_size = kernel_block_sizes[group.kv_cache_group_id]
for layer_name in group.layer_names:
if layer_name in shared_kv_cache_layers:
# Shared layer — tensor will be aliased to its target later.
continue
kv_cache_spec = kv_cache_group_spec.kv_cache_spec
if isinstance(kv_cache_spec, UniformTypeKVCacheSpecs):
kv_cache_spec = kv_cache_spec.kv_cache_specs[layer_name]
kv_raw_tensor = kv_cache_raw_tensors[layer_name]
assert kv_raw_tensor.numel() % kv_cache_spec.page_size_bytes == 0
@@ -206,22 +199,11 @@ def _reshape_kv_cache(
if isinstance(kv_cache_spec, AttentionSpec):
has_attn = True
attn_backend = attn_backends[layer_name]
if kv_cache_group_id < len(kernel_block_sizes):
kernel_block_size = kernel_block_sizes[kv_cache_group_id]
num_blocks *= kv_cache_spec.block_size // kernel_block_size
else:
kernel_block_size = kv_cache_spec.block_size
if kv_cache_spec.storage_block_size != kv_cache_spec.block_size:
shape_block_size = kv_cache_spec.storage_block_size
else:
shape_block_size = kernel_block_size
kv_cache_shape = attn_backend.get_kv_cache_shape(
num_blocks,
shape_block_size,
num_blocks_per_kv_block = kv_cache_spec.block_size // kernel_block_size
kernel_num_blocks = num_blocks * num_blocks_per_kv_block
kv_cache_shape = group.backend.get_kv_cache_shape(
kernel_num_blocks,
kernel_block_size,
kv_cache_spec.num_kv_heads,
kv_cache_spec.head_size,
cache_dtype_str=cache_dtype,
@@ -229,7 +211,7 @@ def _reshape_kv_cache(
# FIXME(woosuk): Add kv_cache_stride_order to all attention backends.
try:
kv_cache_stride_order = attn_backend.get_kv_cache_stride_order()
kv_cache_stride_order = group.backend.get_kv_cache_stride_order()
assert len(kv_cache_stride_order) == len(kv_cache_shape)
except (AttributeError, NotImplementedError):
kv_cache_stride_order = tuple(range(len(kv_cache_shape)))
@@ -290,7 +272,12 @@ def _reshape_kv_cache(
)
if has_attn and has_mamba:
_update_hybrid_attention_layout(kv_caches, kv_cache_config)
_update_hybrid_attention_layout(
attn_groups=attn_groups,
kv_caches=kv_caches,
kernel_block_sizes=kernel_block_sizes,
cache_dtype=cache_dtype,
)
# Map any sharing layers to their target layer's KV cache.
for layer_name, target_layer_name in shared_kv_cache_layers.items():
@@ -300,18 +287,38 @@ def _reshape_kv_cache(
def _update_hybrid_attention_layout(
kv_caches: dict[str, Any], kv_cache_config: KVCacheConfig
attn_groups: Iterable[AttentionGroup],
kv_caches: dict[str, Any],
kernel_block_sizes: list[int],
cache_dtype: str,
) -> None:
for kv_cache_group_spec in kv_cache_config.kv_cache_groups:
for layer_name in kv_cache_group_spec.layer_names:
for group in attn_groups:
if group.kv_cache_group_id >= len(kernel_block_sizes):
continue
kv_cache_spec = group.kv_cache_spec
if not isinstance(kv_cache_spec, AttentionSpec):
continue
block_dim = group.backend.get_kv_cache_block_dim(
kernel_block_sizes[group.kv_cache_group_id],
kv_cache_spec.num_kv_heads,
kv_cache_spec.head_size,
cache_dtype_str=cache_dtype,
)
# if the first dim of the kvcache's layout is already num_blocks, continue
if block_dim == 0:
continue
assert block_dim == 1, (
"Expected the dim `num_blocks` at the second dim when updating"
" the kvcache's layout of full attention layer"
)
for layer_name in group.layer_names:
if layer_name not in kv_caches:
# Shared layer — will be aliased to its target after this pass.
continue
kv_cache_spec = kv_cache_group_spec.kv_cache_spec
if isinstance(kv_cache_spec, UniformTypeKVCacheSpecs):
kv_cache_spec = kv_cache_spec.kv_cache_specs[layer_name]
if not isinstance(kv_cache_spec, AttentionSpec):
continue
kv_cache = kv_caches[layer_name]
if kv_cache.shape[0] == 2:
assert kv_cache.shape[1] != 2, (
@@ -329,10 +336,10 @@ def _update_hybrid_attention_layout(
def init_kv_cache(
runner_kv_caches: list[torch.Tensor],
runner_kv_caches: list[torch.Tensor | list[torch.Tensor]],
forward_context: dict[str, Any],
kv_cache_config: KVCacheConfig,
attn_backends: dict[str, type[AttentionBackend]],
attn_groups: list[list[AttentionGroup]],
device: torch.device,
cache_dtype: str,
kernel_block_sizes: list[int],
@@ -342,13 +349,13 @@ def init_kv_cache(
kv_cache_raw_tensors = _allocate_kv_cache(
kv_cache_config, shared_kv_cache_layers, device
)
flattened_attn_groups = list(group for groups in attn_groups for group in groups)
kv_caches = _reshape_kv_cache(
kv_cache_config,
kv_cache_raw_tensors,
attn_backends,
cache_dtype,
kernel_block_sizes,
shared_kv_cache_layers,
attn_groups=flattened_attn_groups,
kv_cache_raw_tensors=kv_cache_raw_tensors,
kernel_block_sizes=kernel_block_sizes,
cache_dtype=cache_dtype,
shared_kv_cache_layers=shared_kv_cache_layers,
)
bind_kv_cache(kv_caches, forward_context, runner_kv_caches)
return kv_caches
+2 -5
View File
@@ -44,9 +44,7 @@ class BlockTables:
for i in range(self.num_kv_cache_groups):
max_num_blocks = max_num_blocks_per_group[i] * self.blocks_per_kv_block[i]
block_table = StagedWriteTensor(
(self.max_num_reqs, max_num_blocks),
dtype=torch.int32,
device=device,
(self.max_num_reqs, max_num_blocks), dtype=torch.int32, device=device
)
self.block_tables.append(block_table)
@@ -132,7 +130,6 @@ class BlockTables:
self.num_blocks.gpu,
self.num_blocks.gpu.stride(0),
num_reqs,
self.input_block_tables[0].shape[1], # max_num_blocks
BLOCK_SIZE=1024, # type: ignore
)
return tuple(bt[:num_reqs_padded] for bt in self.input_block_tables)
@@ -192,7 +189,6 @@ def _gather_block_tables_kernel(
num_blocks_ptr, # [num_kv_cache_groups, max_num_reqs]
num_blocks_stride,
num_reqs, # actual number of requests (for padding)
max_num_blocks, # stride for zeroing padded rows
BLOCK_SIZE: tl.constexpr,
):
# kv cache group id
@@ -200,6 +196,7 @@ def _gather_block_tables_kernel(
batch_idx = tl.program_id(1)
stride = tl.load(block_table_strides + group_id)
max_num_blocks = stride # stride equals max_num_blocks for this group.
dst_block_table_ptr = _load_ptr(dst_block_table_ptrs + group_id, tl.int32)
dst_row_ptr = dst_block_table_ptr + batch_idx * stride
+3 -4
View File
@@ -393,10 +393,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
) + spec.num_speculative_blocks
max_num_blocks_per_group.append(max_num_blocks)
(self.attn_backends, self.attn_groups, attn_cg_support, kernel_block_sizes) = (
init_attn_backend(self.kv_cache_config, self.vllm_config, self.device)
self.attn_groups, attn_cg_support, kernel_block_sizes = init_attn_backend(
self.kv_cache_config, self.vllm_config, self.device
)
self.block_tables = BlockTables(
block_sizes=block_sizes,
max_num_reqs=self.max_num_reqs,
@@ -440,7 +439,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.kv_caches,
self.compilation_config.static_forward_context,
self.kv_cache_config,
self.attn_backends,
self.attn_groups,
self.device,
self.cache_config.cache_dtype,
kernel_block_sizes,
@@ -173,7 +173,7 @@ class EagleSpeculator:
) -> None:
self.model_state = model_state
self.kv_cache_config = kv_cache_config
_, self.attn_groups, _, _ = init_attn_backend(
self.attn_groups, _, _ = init_attn_backend(
kv_cache_config,
self.vllm_config,
self.device,