[ModelRunnerV2][Hybrid model] Support kernel block size in hybrid model (#38831)
Signed-off-by: MengqingCao <cmq0113@163.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Signed-off-by: Mengqing Cao <cmq0113@163.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
Nick Hill
parent
381edde1b9
commit
1223732dda
+6
-2
@@ -1946,8 +1946,12 @@ class VllmConfig:
|
||||
model_config = self.model_config
|
||||
speculative_config = self.speculative_config
|
||||
|
||||
if model_config is not None and model_config.has_inner_state:
|
||||
unsupported.append("hybrid/mamba models")
|
||||
if (
|
||||
model_config is not None
|
||||
and model_config.has_inner_state
|
||||
and self.cache_config.mamba_cache_mode == "align"
|
||||
):
|
||||
unsupported.append("hybrid/mamba models with align cache mode")
|
||||
|
||||
if self.parallel_config.prefill_context_parallel_size > 1:
|
||||
unsupported.append("prefill context parallelism")
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Sequence
|
||||
from collections.abc import Iterable, Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, cast
|
||||
|
||||
@@ -11,7 +11,6 @@ from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.utils.torch_utils import get_dtype_size
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionBackend,
|
||||
AttentionCGSupport,
|
||||
CommonAttentionMetadata,
|
||||
)
|
||||
@@ -65,13 +64,8 @@ def init_attn_backend(
|
||||
vllm_config: VllmConfig,
|
||||
device: torch.device,
|
||||
active_layer_names: set[str] | None = None,
|
||||
) -> tuple[
|
||||
dict[str, type[AttentionBackend]],
|
||||
list[list[AttentionGroup]],
|
||||
AttentionCGSupportInfo,
|
||||
list[int],
|
||||
]:
|
||||
attn_backends: dict[str, type[AttentionBackend]] = {}
|
||||
) -> tuple[list[list[AttentionGroup]], AttentionCGSupportInfo, list[int]]:
|
||||
# Phase 1: discover attention groups for each kv cache group.
|
||||
attn_groups: list[list[AttentionGroup]] = []
|
||||
|
||||
# Add KV-sharing layers to their target's kv cache group so they are
|
||||
@@ -96,7 +90,6 @@ def init_attn_backend(
|
||||
|
||||
for layer_name in layer_names:
|
||||
attn_backend = attn_layers[layer_name].get_attn_backend()
|
||||
attn_backends[layer_name] = attn_backend
|
||||
|
||||
layer_kv_cache_spec: KVCacheSpec = kv_cache_group_spec.kv_cache_spec
|
||||
if isinstance(layer_kv_cache_spec, UniformTypeKVCacheSpecs):
|
||||
@@ -122,7 +115,6 @@ def init_attn_backend(
|
||||
min_cg_support = AttentionCGSupport.ALWAYS
|
||||
min_cg_attn_backend = None
|
||||
for kv_cache_group_id, groups in enumerate(attn_groups):
|
||||
kv_cache_group_spec = kv_cache_config.kv_cache_groups[kv_cache_group_id]
|
||||
kernel_block_size = None
|
||||
if kv_cache_group_id < len(kernel_block_sizes):
|
||||
kernel_block_size = kernel_block_sizes[kv_cache_group_id]
|
||||
@@ -143,21 +135,16 @@ def init_attn_backend(
|
||||
# Check cudagraph support for the attention backend
|
||||
cg_support = builder.get_cudagraph_support(
|
||||
vllm_config,
|
||||
cast(AttentionSpec, kv_cache_group_spec.kv_cache_spec),
|
||||
cast(AttentionSpec, group.kv_cache_spec),
|
||||
)
|
||||
if cg_support.value < min_cg_support.value:
|
||||
min_cg_support = cg_support
|
||||
min_cg_attn_backend = group.backend.__name__
|
||||
|
||||
return (
|
||||
attn_backends,
|
||||
attn_groups,
|
||||
AttentionCGSupportInfo(
|
||||
min_cg_support=min_cg_support,
|
||||
min_cg_attn_backend=min_cg_attn_backend,
|
||||
),
|
||||
kernel_block_sizes,
|
||||
attn_cg_support_info = AttentionCGSupportInfo(
|
||||
min_cg_support=min_cg_support, min_cg_attn_backend=min_cg_attn_backend
|
||||
)
|
||||
return attn_groups, attn_cg_support_info, kernel_block_sizes
|
||||
|
||||
|
||||
def _allocate_kv_cache(
|
||||
@@ -180,25 +167,31 @@ def _allocate_kv_cache(
|
||||
|
||||
|
||||
def _reshape_kv_cache(
|
||||
kv_cache_config: KVCacheConfig,
|
||||
attn_groups: Sequence[AttentionGroup],
|
||||
kv_cache_raw_tensors: dict[str, torch.Tensor],
|
||||
attn_backends: dict[str, type[AttentionBackend]],
|
||||
cache_dtype: str,
|
||||
kernel_block_sizes: list[int],
|
||||
shared_kv_cache_layers: dict[str, str],
|
||||
) -> dict[str, Any]:
|
||||
kv_caches: dict[str, Any] = {}
|
||||
has_attn, has_mamba = False, False
|
||||
for kv_cache_group_id, kv_cache_group_spec in enumerate(
|
||||
kv_cache_config.kv_cache_groups
|
||||
):
|
||||
for layer_name in kv_cache_group_spec.layer_names:
|
||||
|
||||
for group in attn_groups:
|
||||
if group.kv_cache_group_id >= len(kernel_block_sizes):
|
||||
continue
|
||||
|
||||
kv_cache_spec = group.kv_cache_spec
|
||||
if kv_cache_spec.storage_block_size != kv_cache_spec.block_size:
|
||||
# use storage_block_size as the kernel block size for groups
|
||||
# that apply a compression on block size (eg. DeepSeek V4).
|
||||
kernel_block_size = kv_cache_spec.storage_block_size
|
||||
else:
|
||||
kernel_block_size = kernel_block_sizes[group.kv_cache_group_id]
|
||||
|
||||
for layer_name in group.layer_names:
|
||||
if layer_name in shared_kv_cache_layers:
|
||||
# Shared layer — tensor will be aliased to its target later.
|
||||
continue
|
||||
kv_cache_spec = kv_cache_group_spec.kv_cache_spec
|
||||
if isinstance(kv_cache_spec, UniformTypeKVCacheSpecs):
|
||||
kv_cache_spec = kv_cache_spec.kv_cache_specs[layer_name]
|
||||
|
||||
kv_raw_tensor = kv_cache_raw_tensors[layer_name]
|
||||
assert kv_raw_tensor.numel() % kv_cache_spec.page_size_bytes == 0
|
||||
@@ -206,22 +199,11 @@ def _reshape_kv_cache(
|
||||
|
||||
if isinstance(kv_cache_spec, AttentionSpec):
|
||||
has_attn = True
|
||||
attn_backend = attn_backends[layer_name]
|
||||
|
||||
if kv_cache_group_id < len(kernel_block_sizes):
|
||||
kernel_block_size = kernel_block_sizes[kv_cache_group_id]
|
||||
num_blocks *= kv_cache_spec.block_size // kernel_block_size
|
||||
else:
|
||||
kernel_block_size = kv_cache_spec.block_size
|
||||
|
||||
if kv_cache_spec.storage_block_size != kv_cache_spec.block_size:
|
||||
shape_block_size = kv_cache_spec.storage_block_size
|
||||
else:
|
||||
shape_block_size = kernel_block_size
|
||||
|
||||
kv_cache_shape = attn_backend.get_kv_cache_shape(
|
||||
num_blocks,
|
||||
shape_block_size,
|
||||
num_blocks_per_kv_block = kv_cache_spec.block_size // kernel_block_size
|
||||
kernel_num_blocks = num_blocks * num_blocks_per_kv_block
|
||||
kv_cache_shape = group.backend.get_kv_cache_shape(
|
||||
kernel_num_blocks,
|
||||
kernel_block_size,
|
||||
kv_cache_spec.num_kv_heads,
|
||||
kv_cache_spec.head_size,
|
||||
cache_dtype_str=cache_dtype,
|
||||
@@ -229,7 +211,7 @@ def _reshape_kv_cache(
|
||||
|
||||
# FIXME(woosuk): Add kv_cache_stride_order to all attention backends.
|
||||
try:
|
||||
kv_cache_stride_order = attn_backend.get_kv_cache_stride_order()
|
||||
kv_cache_stride_order = group.backend.get_kv_cache_stride_order()
|
||||
assert len(kv_cache_stride_order) == len(kv_cache_shape)
|
||||
except (AttributeError, NotImplementedError):
|
||||
kv_cache_stride_order = tuple(range(len(kv_cache_shape)))
|
||||
@@ -290,7 +272,12 @@ def _reshape_kv_cache(
|
||||
)
|
||||
|
||||
if has_attn and has_mamba:
|
||||
_update_hybrid_attention_layout(kv_caches, kv_cache_config)
|
||||
_update_hybrid_attention_layout(
|
||||
attn_groups=attn_groups,
|
||||
kv_caches=kv_caches,
|
||||
kernel_block_sizes=kernel_block_sizes,
|
||||
cache_dtype=cache_dtype,
|
||||
)
|
||||
|
||||
# Map any sharing layers to their target layer's KV cache.
|
||||
for layer_name, target_layer_name in shared_kv_cache_layers.items():
|
||||
@@ -300,18 +287,38 @@ def _reshape_kv_cache(
|
||||
|
||||
|
||||
def _update_hybrid_attention_layout(
|
||||
kv_caches: dict[str, Any], kv_cache_config: KVCacheConfig
|
||||
attn_groups: Iterable[AttentionGroup],
|
||||
kv_caches: dict[str, Any],
|
||||
kernel_block_sizes: list[int],
|
||||
cache_dtype: str,
|
||||
) -> None:
|
||||
for kv_cache_group_spec in kv_cache_config.kv_cache_groups:
|
||||
for layer_name in kv_cache_group_spec.layer_names:
|
||||
for group in attn_groups:
|
||||
if group.kv_cache_group_id >= len(kernel_block_sizes):
|
||||
continue
|
||||
|
||||
kv_cache_spec = group.kv_cache_spec
|
||||
if not isinstance(kv_cache_spec, AttentionSpec):
|
||||
continue
|
||||
block_dim = group.backend.get_kv_cache_block_dim(
|
||||
kernel_block_sizes[group.kv_cache_group_id],
|
||||
kv_cache_spec.num_kv_heads,
|
||||
kv_cache_spec.head_size,
|
||||
cache_dtype_str=cache_dtype,
|
||||
)
|
||||
# if the first dim of the kvcache's layout is already num_blocks, continue
|
||||
if block_dim == 0:
|
||||
continue
|
||||
|
||||
assert block_dim == 1, (
|
||||
"Expected the dim `num_blocks` at the second dim when updating"
|
||||
" the kvcache's layout of full attention layer"
|
||||
)
|
||||
|
||||
for layer_name in group.layer_names:
|
||||
if layer_name not in kv_caches:
|
||||
# Shared layer — will be aliased to its target after this pass.
|
||||
continue
|
||||
kv_cache_spec = kv_cache_group_spec.kv_cache_spec
|
||||
if isinstance(kv_cache_spec, UniformTypeKVCacheSpecs):
|
||||
kv_cache_spec = kv_cache_spec.kv_cache_specs[layer_name]
|
||||
if not isinstance(kv_cache_spec, AttentionSpec):
|
||||
continue
|
||||
|
||||
kv_cache = kv_caches[layer_name]
|
||||
if kv_cache.shape[0] == 2:
|
||||
assert kv_cache.shape[1] != 2, (
|
||||
@@ -329,10 +336,10 @@ def _update_hybrid_attention_layout(
|
||||
|
||||
|
||||
def init_kv_cache(
|
||||
runner_kv_caches: list[torch.Tensor],
|
||||
runner_kv_caches: list[torch.Tensor | list[torch.Tensor]],
|
||||
forward_context: dict[str, Any],
|
||||
kv_cache_config: KVCacheConfig,
|
||||
attn_backends: dict[str, type[AttentionBackend]],
|
||||
attn_groups: list[list[AttentionGroup]],
|
||||
device: torch.device,
|
||||
cache_dtype: str,
|
||||
kernel_block_sizes: list[int],
|
||||
@@ -342,13 +349,13 @@ def init_kv_cache(
|
||||
kv_cache_raw_tensors = _allocate_kv_cache(
|
||||
kv_cache_config, shared_kv_cache_layers, device
|
||||
)
|
||||
flattened_attn_groups = list(group for groups in attn_groups for group in groups)
|
||||
kv_caches = _reshape_kv_cache(
|
||||
kv_cache_config,
|
||||
kv_cache_raw_tensors,
|
||||
attn_backends,
|
||||
cache_dtype,
|
||||
kernel_block_sizes,
|
||||
shared_kv_cache_layers,
|
||||
attn_groups=flattened_attn_groups,
|
||||
kv_cache_raw_tensors=kv_cache_raw_tensors,
|
||||
kernel_block_sizes=kernel_block_sizes,
|
||||
cache_dtype=cache_dtype,
|
||||
shared_kv_cache_layers=shared_kv_cache_layers,
|
||||
)
|
||||
bind_kv_cache(kv_caches, forward_context, runner_kv_caches)
|
||||
return kv_caches
|
||||
|
||||
@@ -44,9 +44,7 @@ class BlockTables:
|
||||
for i in range(self.num_kv_cache_groups):
|
||||
max_num_blocks = max_num_blocks_per_group[i] * self.blocks_per_kv_block[i]
|
||||
block_table = StagedWriteTensor(
|
||||
(self.max_num_reqs, max_num_blocks),
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
(self.max_num_reqs, max_num_blocks), dtype=torch.int32, device=device
|
||||
)
|
||||
self.block_tables.append(block_table)
|
||||
|
||||
@@ -132,7 +130,6 @@ class BlockTables:
|
||||
self.num_blocks.gpu,
|
||||
self.num_blocks.gpu.stride(0),
|
||||
num_reqs,
|
||||
self.input_block_tables[0].shape[1], # max_num_blocks
|
||||
BLOCK_SIZE=1024, # type: ignore
|
||||
)
|
||||
return tuple(bt[:num_reqs_padded] for bt in self.input_block_tables)
|
||||
@@ -192,7 +189,6 @@ def _gather_block_tables_kernel(
|
||||
num_blocks_ptr, # [num_kv_cache_groups, max_num_reqs]
|
||||
num_blocks_stride,
|
||||
num_reqs, # actual number of requests (for padding)
|
||||
max_num_blocks, # stride for zeroing padded rows
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
# kv cache group id
|
||||
@@ -200,6 +196,7 @@ def _gather_block_tables_kernel(
|
||||
batch_idx = tl.program_id(1)
|
||||
|
||||
stride = tl.load(block_table_strides + group_id)
|
||||
max_num_blocks = stride # stride equals max_num_blocks for this group.
|
||||
dst_block_table_ptr = _load_ptr(dst_block_table_ptrs + group_id, tl.int32)
|
||||
dst_row_ptr = dst_block_table_ptr + batch_idx * stride
|
||||
|
||||
|
||||
@@ -393,10 +393,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
) + spec.num_speculative_blocks
|
||||
max_num_blocks_per_group.append(max_num_blocks)
|
||||
|
||||
(self.attn_backends, self.attn_groups, attn_cg_support, kernel_block_sizes) = (
|
||||
init_attn_backend(self.kv_cache_config, self.vllm_config, self.device)
|
||||
self.attn_groups, attn_cg_support, kernel_block_sizes = init_attn_backend(
|
||||
self.kv_cache_config, self.vllm_config, self.device
|
||||
)
|
||||
|
||||
self.block_tables = BlockTables(
|
||||
block_sizes=block_sizes,
|
||||
max_num_reqs=self.max_num_reqs,
|
||||
@@ -440,7 +439,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.kv_caches,
|
||||
self.compilation_config.static_forward_context,
|
||||
self.kv_cache_config,
|
||||
self.attn_backends,
|
||||
self.attn_groups,
|
||||
self.device,
|
||||
self.cache_config.cache_dtype,
|
||||
kernel_block_sizes,
|
||||
|
||||
@@ -173,7 +173,7 @@ class EagleSpeculator:
|
||||
) -> None:
|
||||
self.model_state = model_state
|
||||
self.kv_cache_config = kv_cache_config
|
||||
_, self.attn_groups, _, _ = init_attn_backend(
|
||||
self.attn_groups, _, _ = init_attn_backend(
|
||||
kv_cache_config,
|
||||
self.vllm_config,
|
||||
self.device,
|
||||
|
||||
Reference in New Issue
Block a user