[Model Runner V2] support qwen35 / mamba hybrid model (#35520)
Signed-off-by: zhuhaoran <zhuhaoran.zhr@alibaba-inc.com>
This commit is contained in:
@@ -1892,6 +1892,10 @@ class VllmConfig:
|
|||||||
"to schedule a multiple of block_size tokens even if they are "
|
"to schedule a multiple of block_size tokens even if they are "
|
||||||
"in the middle of a mm input"
|
"in the middle of a mm input"
|
||||||
)
|
)
|
||||||
|
# TODO: support align mamba cache mode for model runner v2
|
||||||
|
assert not envs.VLLM_USE_V2_MODEL_RUNNER, (
|
||||||
|
"Model Runner V2 has not yet supported mamba_cache_mode='align'. "
|
||||||
|
)
|
||||||
|
|
||||||
@model_validator(mode="after")
|
@model_validator(mode="after")
|
||||||
def validate_nvfp4_kv_cache_with_mla(self) -> "VllmConfig":
|
def validate_nvfp4_kv_cache_with_mla(self) -> "VllmConfig":
|
||||||
|
|||||||
@@ -111,7 +111,15 @@ class MultiModalRegistry:
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
mm_config = model_config.get_multimodal_config()
|
mm_config = model_config.get_multimodal_config()
|
||||||
info = self._create_processing_info(model_config, tokenizer=None)
|
try:
|
||||||
|
info = self._create_processing_info(model_config, tokenizer=None)
|
||||||
|
except ValueError:
|
||||||
|
logger.warning_once(
|
||||||
|
"Model %s is treated as multimodal but has no registered "
|
||||||
|
"multimodal processor; running in text-only mode.",
|
||||||
|
model_config.model,
|
||||||
|
)
|
||||||
|
return False
|
||||||
|
|
||||||
# Check if all supported modalities have limit == 0
|
# Check if all supported modalities have limit == 0
|
||||||
if all(
|
if all(
|
||||||
@@ -170,7 +178,11 @@ class MultiModalRegistry:
|
|||||||
from vllm.model_executor.model_loader import get_model_architecture
|
from vllm.model_executor.model_loader import get_model_architecture
|
||||||
|
|
||||||
model_cls, _ = get_model_architecture(model_config)
|
model_cls, _ = get_model_architecture(model_config)
|
||||||
assert hasattr(model_cls, "_processor_factory")
|
if not hasattr(model_cls, "_processor_factory"):
|
||||||
|
raise ValueError(
|
||||||
|
f"Model class {model_cls.__name__} has no registered "
|
||||||
|
"multimodal processor"
|
||||||
|
)
|
||||||
return cast("SupportsMultiModal", model_cls)
|
return cast("SupportsMultiModal", model_cls)
|
||||||
|
|
||||||
def _create_processing_ctx(
|
def _create_processing_ctx(
|
||||||
|
|||||||
@@ -4,7 +4,6 @@ from collections.abc import Sequence
|
|||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from typing import Any, cast
|
from typing import Any, cast
|
||||||
|
|
||||||
import numpy as np
|
|
||||||
import torch
|
import torch
|
||||||
|
|
||||||
from vllm.config import VllmConfig, get_layers_from_vllm_config
|
from vllm.config import VllmConfig, get_layers_from_vllm_config
|
||||||
@@ -19,8 +18,10 @@ from vllm.v1.kv_cache_interface import (
|
|||||||
AttentionSpec,
|
AttentionSpec,
|
||||||
KVCacheConfig,
|
KVCacheConfig,
|
||||||
KVCacheSpec,
|
KVCacheSpec,
|
||||||
|
MambaSpec,
|
||||||
UniformTypeKVCacheSpecs,
|
UniformTypeKVCacheSpecs,
|
||||||
)
|
)
|
||||||
|
from vllm.v1.worker.gpu.model_states.interface import ModelSpecificAttnMetadata
|
||||||
from vllm.v1.worker.utils import AttentionGroup, bind_kv_cache
|
from vllm.v1.worker.utils import AttentionGroup, bind_kv_cache
|
||||||
|
|
||||||
|
|
||||||
@@ -147,68 +148,126 @@ def _reshape_kv_cache(
|
|||||||
kv_cache_raw_tensors: dict[str, torch.Tensor],
|
kv_cache_raw_tensors: dict[str, torch.Tensor],
|
||||||
attn_backends: dict[str, type[AttentionBackend]],
|
attn_backends: dict[str, type[AttentionBackend]],
|
||||||
cache_dtype: str,
|
cache_dtype: str,
|
||||||
) -> dict[str, torch.Tensor]:
|
) -> dict[str, Any]:
|
||||||
kv_caches: dict[str, torch.Tensor] = {}
|
kv_caches: dict[str, Any] = {}
|
||||||
|
has_attn, has_mamba = False, False
|
||||||
for kv_cache_group_spec in kv_cache_config.kv_cache_groups:
|
for kv_cache_group_spec in kv_cache_config.kv_cache_groups:
|
||||||
for layer_name in kv_cache_group_spec.layer_names:
|
for layer_name in kv_cache_group_spec.layer_names:
|
||||||
kv_cache_spec = kv_cache_group_spec.kv_cache_spec
|
kv_cache_spec = kv_cache_group_spec.kv_cache_spec
|
||||||
if isinstance(kv_cache_spec, UniformTypeKVCacheSpecs):
|
if isinstance(kv_cache_spec, UniformTypeKVCacheSpecs):
|
||||||
kv_cache_spec = kv_cache_spec.kv_cache_specs[layer_name]
|
kv_cache_spec = kv_cache_spec.kv_cache_specs[layer_name]
|
||||||
assert isinstance(kv_cache_spec, AttentionSpec)
|
|
||||||
|
|
||||||
raw_tensor = kv_cache_raw_tensors[layer_name]
|
kv_raw_tensor = kv_cache_raw_tensors[layer_name]
|
||||||
assert raw_tensor.numel() % kv_cache_spec.page_size_bytes == 0
|
assert kv_raw_tensor.numel() % kv_cache_spec.page_size_bytes == 0
|
||||||
num_blocks = raw_tensor.numel() // kv_cache_spec.page_size_bytes
|
num_blocks = kv_raw_tensor.numel() // kv_cache_spec.page_size_bytes
|
||||||
|
|
||||||
attn_backend = attn_backends[layer_name]
|
if isinstance(kv_cache_spec, AttentionSpec):
|
||||||
kv_cache_shape = attn_backend.get_kv_cache_shape(
|
has_attn = True
|
||||||
num_blocks,
|
attn_backend = attn_backends[layer_name]
|
||||||
kv_cache_spec.storage_block_size,
|
kv_cache_shape = attn_backend.get_kv_cache_shape(
|
||||||
kv_cache_spec.num_kv_heads,
|
num_blocks,
|
||||||
kv_cache_spec.head_size,
|
kv_cache_spec.storage_block_size,
|
||||||
cache_dtype,
|
kv_cache_spec.num_kv_heads,
|
||||||
)
|
kv_cache_spec.head_size,
|
||||||
|
cache_dtype_str=cache_dtype,
|
||||||
# FIXME(woosuk): Add kv_cache_stride_order to all attention backends.
|
|
||||||
try:
|
|
||||||
kv_cache_stride_order = attn_backend.get_kv_cache_stride_order()
|
|
||||||
assert len(kv_cache_stride_order) == len(kv_cache_shape)
|
|
||||||
except (AttributeError, NotImplementedError):
|
|
||||||
kv_cache_stride_order = tuple(range(len(kv_cache_shape)))
|
|
||||||
|
|
||||||
kv_cache_shape = tuple(kv_cache_shape[i] for i in kv_cache_stride_order)
|
|
||||||
inv_order = [
|
|
||||||
kv_cache_stride_order.index(i)
|
|
||||||
for i in range(len(kv_cache_stride_order))
|
|
||||||
]
|
|
||||||
|
|
||||||
dtype = kv_cache_spec.dtype
|
|
||||||
raw_tensor = raw_tensor.view(dtype)
|
|
||||||
if kv_cache_spec.page_size_padded is not None:
|
|
||||||
# Use strided view to handle page_size_bytes that
|
|
||||||
# include padding. This follows the same pattern as
|
|
||||||
# MambaSpec handling in gpu_model_runner.py.
|
|
||||||
# NOTE: This assumes kv_cache_shape[0] == num_blocks
|
|
||||||
# (i.e. the first physical dimension is the block
|
|
||||||
# index), which holds for MLA backends but NOT for
|
|
||||||
# standard attention backends whose shape starts with
|
|
||||||
# a K/V dimension of size 2.
|
|
||||||
dtype_size = get_dtype_size(dtype)
|
|
||||||
page_stride = kv_cache_spec.page_size_bytes // dtype_size
|
|
||||||
strides = list(torch.empty(kv_cache_shape).stride())
|
|
||||||
strides[inv_order[0]] = page_stride
|
|
||||||
kv_cache = torch.as_strided(
|
|
||||||
raw_tensor,
|
|
||||||
size=kv_cache_shape,
|
|
||||||
stride=tuple(strides),
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# FIXME(woosuk): Add kv_cache_stride_order to all attention backends.
|
||||||
|
try:
|
||||||
|
kv_cache_stride_order = attn_backend.get_kv_cache_stride_order()
|
||||||
|
assert len(kv_cache_stride_order) == len(kv_cache_shape)
|
||||||
|
except (AttributeError, NotImplementedError):
|
||||||
|
kv_cache_stride_order = tuple(range(len(kv_cache_shape)))
|
||||||
|
|
||||||
|
kv_cache_shape = tuple(kv_cache_shape[i] for i in kv_cache_stride_order)
|
||||||
|
inv_order = [
|
||||||
|
kv_cache_stride_order.index(i)
|
||||||
|
for i in range(len(kv_cache_stride_order))
|
||||||
|
]
|
||||||
|
|
||||||
|
dtype = kv_cache_spec.dtype
|
||||||
|
kv_tensor = kv_raw_tensor.view(dtype)
|
||||||
|
if kv_cache_spec.page_size_padded is not None:
|
||||||
|
# Use strided view to handle page_size_bytes that
|
||||||
|
# include padding. This follows the same pattern as
|
||||||
|
# MambaSpec handling in gpu_model_runner.py.
|
||||||
|
# NOTE: This assumes kv_cache_shape[0] == num_blocks
|
||||||
|
# (i.e. the first physical dimension is the block
|
||||||
|
# index), which holds for MLA backends but NOT for
|
||||||
|
# standard attention backends whose shape starts with
|
||||||
|
# a K/V dimension of size 2.
|
||||||
|
dtype_size = get_dtype_size(dtype)
|
||||||
|
page_stride = kv_cache_spec.page_size_bytes // dtype_size
|
||||||
|
strides = list(torch.empty(kv_cache_shape).stride())
|
||||||
|
strides[inv_order[0]] = page_stride
|
||||||
|
kv_cache = torch.as_strided(
|
||||||
|
kv_tensor,
|
||||||
|
size=kv_cache_shape,
|
||||||
|
stride=tuple(strides),
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
# No padding — safe to use a contiguous view.
|
||||||
|
kv_cache = kv_tensor.view(kv_cache_shape)
|
||||||
|
kv_caches[layer_name] = kv_cache.permute(*inv_order)
|
||||||
|
|
||||||
|
elif isinstance(kv_cache_spec, MambaSpec):
|
||||||
|
has_mamba = True
|
||||||
|
state_tensors = []
|
||||||
|
storage_offset_bytes = 0
|
||||||
|
for shape, dtype in zip(kv_cache_spec.shapes, kv_cache_spec.dtypes):
|
||||||
|
dtype_size = get_dtype_size(dtype)
|
||||||
|
num_element_per_page = kv_cache_spec.page_size_bytes // dtype_size
|
||||||
|
target_shape = (num_blocks, *shape)
|
||||||
|
stride = torch.empty(target_shape).stride()
|
||||||
|
target_stride = (num_element_per_page, *stride[1:])
|
||||||
|
assert storage_offset_bytes % dtype_size == 0
|
||||||
|
tensor = torch.as_strided(
|
||||||
|
kv_raw_tensor.view(dtype),
|
||||||
|
size=target_shape,
|
||||||
|
stride=target_stride,
|
||||||
|
storage_offset=storage_offset_bytes // dtype_size,
|
||||||
|
)
|
||||||
|
state_tensors.append(tensor)
|
||||||
|
storage_offset_bytes += stride[0] * dtype_size
|
||||||
|
kv_caches[layer_name] = state_tensors
|
||||||
else:
|
else:
|
||||||
# No padding — safe to use a contiguous view.
|
raise NotImplementedError(
|
||||||
kv_cache = raw_tensor.view(kv_cache_shape)
|
f"Unsupported KV cache spec type: {type(kv_cache_spec)}"
|
||||||
kv_caches[layer_name] = kv_cache.permute(*inv_order)
|
)
|
||||||
|
|
||||||
|
if has_attn and has_mamba:
|
||||||
|
_update_hybrid_attention_layout(kv_caches, kv_cache_config)
|
||||||
|
|
||||||
return kv_caches
|
return kv_caches
|
||||||
|
|
||||||
|
|
||||||
|
def _update_hybrid_attention_layout(
|
||||||
|
kv_caches: dict[str, Any],
|
||||||
|
kv_cache_config: KVCacheConfig,
|
||||||
|
) -> None:
|
||||||
|
for kv_cache_group_spec in kv_cache_config.kv_cache_groups:
|
||||||
|
for layer_name in kv_cache_group_spec.layer_names:
|
||||||
|
kv_cache_spec = kv_cache_group_spec.kv_cache_spec
|
||||||
|
if isinstance(kv_cache_spec, UniformTypeKVCacheSpecs):
|
||||||
|
kv_cache_spec = kv_cache_spec.kv_cache_specs[layer_name]
|
||||||
|
if not isinstance(kv_cache_spec, AttentionSpec):
|
||||||
|
continue
|
||||||
|
kv_cache = kv_caches[layer_name]
|
||||||
|
if kv_cache.shape[0] == 2:
|
||||||
|
assert kv_cache.shape[1] != 2, (
|
||||||
|
f"Cannot determine layout for tensor of shape {kv_cache.shape}"
|
||||||
|
)
|
||||||
|
hidden_size = kv_cache.shape[2:].numel()
|
||||||
|
kv_cache.as_strided_(
|
||||||
|
size=kv_cache.shape,
|
||||||
|
stride=(
|
||||||
|
hidden_size,
|
||||||
|
2 * hidden_size,
|
||||||
|
*kv_cache.stride()[2:],
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def init_kv_cache(
|
def init_kv_cache(
|
||||||
runner_kv_caches: list[torch.Tensor],
|
runner_kv_caches: list[torch.Tensor],
|
||||||
forward_context: dict[str, Any],
|
forward_context: dict[str, Any],
|
||||||
@@ -216,7 +275,7 @@ def init_kv_cache(
|
|||||||
attn_backends: dict[str, type[AttentionBackend]],
|
attn_backends: dict[str, type[AttentionBackend]],
|
||||||
device: torch.device,
|
device: torch.device,
|
||||||
cache_dtype: str,
|
cache_dtype: str,
|
||||||
) -> dict[str, torch.Tensor]:
|
) -> dict[str, Any]:
|
||||||
kv_cache_raw_tensors = _allocate_kv_cache(kv_cache_config, device)
|
kv_cache_raw_tensors = _allocate_kv_cache(kv_cache_config, device)
|
||||||
kv_caches = _reshape_kv_cache(
|
kv_caches = _reshape_kv_cache(
|
||||||
kv_cache_config, kv_cache_raw_tensors, attn_backends, cache_dtype
|
kv_cache_config, kv_cache_raw_tensors, attn_backends, cache_dtype
|
||||||
@@ -250,8 +309,9 @@ def build_attn_metadata(
|
|||||||
kv_cache_config: KVCacheConfig,
|
kv_cache_config: KVCacheConfig,
|
||||||
seq_lens_cpu_upper_bound: torch.Tensor | None = None,
|
seq_lens_cpu_upper_bound: torch.Tensor | None = None,
|
||||||
dcp_local_seq_lens: torch.Tensor | None = None,
|
dcp_local_seq_lens: torch.Tensor | None = None,
|
||||||
encoder_seq_lens: dict[int, tuple[torch.Tensor, np.ndarray]] | None = None,
|
|
||||||
positions: torch.Tensor | None = None,
|
positions: torch.Tensor | None = None,
|
||||||
|
model_specific_attn_metadata: ModelSpecificAttnMetadata | None = None,
|
||||||
|
for_cudagraph_capture: bool = False,
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
seq_lens = seq_lens[:num_reqs]
|
seq_lens = seq_lens[:num_reqs]
|
||||||
if dcp_local_seq_lens is not None:
|
if dcp_local_seq_lens is not None:
|
||||||
@@ -265,6 +325,11 @@ def build_attn_metadata(
|
|||||||
block_table = block_tables[i]
|
block_table = block_tables[i]
|
||||||
slot_mapping = slot_mappings[i]
|
slot_mapping = slot_mappings[i]
|
||||||
|
|
||||||
|
common_attn_metadata_extra_kwargs = (
|
||||||
|
model_specific_attn_metadata.get_extra_common_attn_kwargs(i, num_reqs)
|
||||||
|
if model_specific_attn_metadata is not None
|
||||||
|
else {}
|
||||||
|
)
|
||||||
common_attn_metadata = CommonAttentionMetadata(
|
common_attn_metadata = CommonAttentionMetadata(
|
||||||
query_start_loc=query_start_loc_gpu,
|
query_start_loc=query_start_loc_gpu,
|
||||||
query_start_loc_cpu=query_start_loc_cpu,
|
query_start_loc_cpu=query_start_loc_cpu,
|
||||||
@@ -279,17 +344,29 @@ def build_attn_metadata(
|
|||||||
causal=True,
|
causal=True,
|
||||||
dcp_local_seq_lens=dcp_local_seq_lens,
|
dcp_local_seq_lens=dcp_local_seq_lens,
|
||||||
positions=positions,
|
positions=positions,
|
||||||
|
**common_attn_metadata_extra_kwargs,
|
||||||
)
|
)
|
||||||
if encoder_seq_lens and i in encoder_seq_lens:
|
|
||||||
encoder_seq_lens_gpu, encoder_seq_lens_cpu = encoder_seq_lens[i]
|
|
||||||
common_attn_metadata.encoder_seq_lens = encoder_seq_lens_gpu
|
|
||||||
common_attn_metadata.encoder_seq_lens_cpu = encoder_seq_lens_cpu
|
|
||||||
|
|
||||||
for attn_group in attn_groups[i]:
|
for attn_group in attn_groups[i]:
|
||||||
attn_metadata_builder = attn_group.get_metadata_builder(0)
|
attn_metadata_builder = attn_group.get_metadata_builder(0)
|
||||||
metadata = attn_metadata_builder.build(
|
if for_cudagraph_capture:
|
||||||
common_prefix_len=0, common_attn_metadata=common_attn_metadata
|
metadata = attn_metadata_builder.build_for_cudagraph_capture(
|
||||||
)
|
common_attn_metadata
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
attn_metadata_extra_kwargs = (
|
||||||
|
model_specific_attn_metadata.get_extra_attn_kwargs(
|
||||||
|
attn_metadata_builder,
|
||||||
|
num_reqs,
|
||||||
|
)
|
||||||
|
if model_specific_attn_metadata is not None
|
||||||
|
else {}
|
||||||
|
)
|
||||||
|
metadata = attn_metadata_builder.build(
|
||||||
|
common_prefix_len=0,
|
||||||
|
common_attn_metadata=common_attn_metadata,
|
||||||
|
**attn_metadata_extra_kwargs,
|
||||||
|
)
|
||||||
for layer_name in attn_group.layer_names:
|
for layer_name in attn_group.layer_names:
|
||||||
attn_metadata[layer_name] = metadata
|
attn_metadata[layer_name] = metadata
|
||||||
return attn_metadata
|
return attn_metadata
|
||||||
|
|||||||
@@ -5,7 +5,6 @@ from collections.abc import Iterable
|
|||||||
import torch
|
import torch
|
||||||
|
|
||||||
from vllm.triton_utils import tl, triton
|
from vllm.triton_utils import tl, triton
|
||||||
from vllm.utils.math_utils import cdiv
|
|
||||||
from vllm.v1.attention.backends.utils import PAD_SLOT_ID
|
from vllm.v1.attention.backends.utils import PAD_SLOT_ID
|
||||||
from vllm.v1.worker.gpu.buffer_utils import StagedWriteTensor, UvaBackedTensor
|
from vllm.v1.worker.gpu.buffer_utils import StagedWriteTensor, UvaBackedTensor
|
||||||
|
|
||||||
@@ -16,7 +15,7 @@ class BlockTables:
|
|||||||
block_sizes: list[int],
|
block_sizes: list[int],
|
||||||
max_num_reqs: int,
|
max_num_reqs: int,
|
||||||
max_num_batched_tokens: int,
|
max_num_batched_tokens: int,
|
||||||
max_model_len: int,
|
max_num_blocks_per_group: list[int],
|
||||||
device: torch.device,
|
device: torch.device,
|
||||||
cp_size: int = 1,
|
cp_size: int = 1,
|
||||||
cp_rank: int = 0,
|
cp_rank: int = 0,
|
||||||
@@ -25,7 +24,6 @@ class BlockTables:
|
|||||||
self.block_sizes = block_sizes
|
self.block_sizes = block_sizes
|
||||||
self.max_num_reqs = max_num_reqs
|
self.max_num_reqs = max_num_reqs
|
||||||
self.max_num_batched_tokens = max_num_batched_tokens
|
self.max_num_batched_tokens = max_num_batched_tokens
|
||||||
self.max_model_len = max_model_len
|
|
||||||
self.device = device
|
self.device = device
|
||||||
|
|
||||||
self.cp_size = cp_size
|
self.cp_size = cp_size
|
||||||
@@ -33,19 +31,11 @@ class BlockTables:
|
|||||||
self.cp_interleave = cp_interleave
|
self.cp_interleave = cp_interleave
|
||||||
|
|
||||||
self.num_kv_cache_groups = len(self.block_sizes)
|
self.num_kv_cache_groups = len(self.block_sizes)
|
||||||
|
assert len(max_num_blocks_per_group) == self.num_kv_cache_groups
|
||||||
# num_kv_cache_groups x [max_num_reqs, max_num_blocks]
|
# num_kv_cache_groups x [max_num_reqs, max_num_blocks]
|
||||||
self.block_tables: list[StagedWriteTensor] = []
|
self.block_tables: list[StagedWriteTensor] = []
|
||||||
for i in range(self.num_kv_cache_groups):
|
for i in range(self.num_kv_cache_groups):
|
||||||
block_size = self.block_sizes[i]
|
max_num_blocks = max_num_blocks_per_group[i]
|
||||||
# When using DCP, each request's KV cache is sharded among different ranks.
|
|
||||||
# As a result, one block on the current rank covers `block_size * cp_size`
|
|
||||||
# tokens in the full, global (unsharded) sequence.
|
|
||||||
max_num_blocks = cdiv(self.max_model_len, block_size * self.cp_size)
|
|
||||||
# Align to a multiple of (128 / block_size) as required
|
|
||||||
# by some attention backends such as TRTLLM (#39324)
|
|
||||||
if block_size <= 128:
|
|
||||||
alignment = 128 // block_size
|
|
||||||
max_num_blocks = cdiv(max_num_blocks, alignment) * alignment
|
|
||||||
block_table = StagedWriteTensor(
|
block_table = StagedWriteTensor(
|
||||||
(self.max_num_reqs, max_num_blocks),
|
(self.max_num_reqs, max_num_blocks),
|
||||||
dtype=torch.int32,
|
dtype=torch.int32,
|
||||||
|
|||||||
@@ -34,7 +34,7 @@ logger = init_logger(__name__)
|
|||||||
|
|
||||||
|
|
||||||
class CapturedAttentionState(NamedTuple):
|
class CapturedAttentionState(NamedTuple):
|
||||||
attn_metadata: dict[str, Any]
|
attn_metadata: dict[str, Any] | None
|
||||||
slot_mappings: dict[str, torch.Tensor]
|
slot_mappings: dict[str, torch.Tensor]
|
||||||
|
|
||||||
|
|
||||||
@@ -100,6 +100,7 @@ class CudaGraphManager:
|
|||||||
self.decode_query_len = decode_query_len
|
self.decode_query_len = decode_query_len
|
||||||
|
|
||||||
self.dp_size = vllm_config.parallel_config.data_parallel_size
|
self.dp_size = vllm_config.parallel_config.data_parallel_size
|
||||||
|
self.tp_size = vllm_config.parallel_config.tensor_parallel_size
|
||||||
self.is_first_pp_rank = get_pp_group().is_first_rank
|
self.is_first_pp_rank = get_pp_group().is_first_rank
|
||||||
self.is_last_pp_rank = get_pp_group().is_last_rank
|
self.is_last_pp_rank = get_pp_group().is_last_rank
|
||||||
|
|
||||||
@@ -109,6 +110,10 @@ class CudaGraphManager:
|
|||||||
self._graphs_captured = False
|
self._graphs_captured = False
|
||||||
self._candidates: list[list[BatchExecutionDescriptor]] = []
|
self._candidates: list[list[BatchExecutionDescriptor]] = []
|
||||||
self._capture_descs: dict[CUDAGraphMode, list[BatchExecutionDescriptor]] = {}
|
self._capture_descs: dict[CUDAGraphMode, list[BatchExecutionDescriptor]] = {}
|
||||||
|
# adjust the cudagraph sizes to be a multiple of the uniform decode query length
|
||||||
|
self.compilation_config.adjust_cudagraph_sizes_for_spec_decode(
|
||||||
|
self.decode_query_len, self.tp_size
|
||||||
|
)
|
||||||
self._init_candidates()
|
self._init_candidates()
|
||||||
|
|
||||||
def _init_candidates(self) -> None:
|
def _init_candidates(self) -> None:
|
||||||
@@ -340,16 +345,16 @@ class ModelCudaGraphManager(CudaGraphManager):
|
|||||||
block_tables,
|
block_tables,
|
||||||
attn_groups,
|
attn_groups,
|
||||||
kv_cache_config,
|
kv_cache_config,
|
||||||
|
skip_attn=(desc.cg_mode == CUDAGraphMode.PIECEWISE),
|
||||||
)
|
)
|
||||||
|
|
||||||
def forward_fn(cg_mode: CUDAGraphMode) -> None:
|
def forward_fn(cg_mode: CUDAGraphMode) -> None:
|
||||||
batch_descriptor = (
|
batch_descriptor = None
|
||||||
BatchDescriptor(num_tokens=num_tokens)
|
if cg_mode == CUDAGraphMode.PIECEWISE:
|
||||||
if cg_mode == CUDAGraphMode.PIECEWISE
|
assert attn_metadata is None
|
||||||
else None
|
batch_descriptor = BatchDescriptor(num_tokens=num_tokens)
|
||||||
)
|
|
||||||
with set_forward_context(
|
with set_forward_context(
|
||||||
attn_metadata if cg_mode != CUDAGraphMode.PIECEWISE else None,
|
attn_metadata,
|
||||||
self.vllm_config,
|
self.vllm_config,
|
||||||
num_tokens=num_tokens,
|
num_tokens=num_tokens,
|
||||||
cudagraph_runtime_mode=cg_mode,
|
cudagraph_runtime_mode=cg_mode,
|
||||||
@@ -419,6 +424,7 @@ def prepare_inputs_to_capture(
|
|||||||
block_tables: BlockTables,
|
block_tables: BlockTables,
|
||||||
attn_groups: list[list[AttentionGroup]],
|
attn_groups: list[list[AttentionGroup]],
|
||||||
kv_cache_config: KVCacheConfig,
|
kv_cache_config: KVCacheConfig,
|
||||||
|
skip_attn: bool = False,
|
||||||
) -> CapturedAttentionState:
|
) -> CapturedAttentionState:
|
||||||
input_batch = InputBatch.make_dummy(num_reqs, num_tokens, input_buffers)
|
input_batch = InputBatch.make_dummy(num_reqs, num_tokens, input_buffers)
|
||||||
input_block_tables = block_tables.get_dummy_block_tables(num_reqs)
|
input_block_tables = block_tables.get_dummy_block_tables(num_reqs)
|
||||||
@@ -439,13 +445,15 @@ def prepare_inputs_to_capture(
|
|||||||
)
|
)
|
||||||
input_batch.dcp_local_seq_lens = input_buffers.dcp_local_seq_lens[:num_reqs]
|
input_batch.dcp_local_seq_lens = input_buffers.dcp_local_seq_lens[:num_reqs]
|
||||||
|
|
||||||
attn_metadata = model_state.prepare_attn(
|
attn_metadata = None
|
||||||
input_batch,
|
if not skip_attn:
|
||||||
CUDAGraphMode.NONE,
|
attn_metadata = model_state.prepare_attn(
|
||||||
input_block_tables,
|
input_batch,
|
||||||
slot_mappings,
|
CUDAGraphMode.NONE,
|
||||||
attn_groups,
|
input_block_tables,
|
||||||
kv_cache_config,
|
slot_mappings,
|
||||||
for_capture=True,
|
attn_groups,
|
||||||
)
|
kv_cache_config,
|
||||||
|
for_capture=True,
|
||||||
|
)
|
||||||
return CapturedAttentionState(attn_metadata, slot_mappings_by_layer)
|
return CapturedAttentionState(attn_metadata, slot_mappings_by_layer)
|
||||||
|
|||||||
@@ -53,7 +53,10 @@ class InputBatch:
|
|||||||
# sum(num_scheduled_tokens)
|
# sum(num_scheduled_tokens)
|
||||||
num_tokens: int
|
num_tokens: int
|
||||||
num_tokens_after_padding: int
|
num_tokens_after_padding: int
|
||||||
|
# Sum of draft tokens scheduled across requests.
|
||||||
num_draft_tokens: int
|
num_draft_tokens: int
|
||||||
|
# [num_reqs] number of draft tokens scheduled for each request, if any.
|
||||||
|
num_draft_tokens_per_req: np.ndarray | None
|
||||||
|
|
||||||
# [num_reqs + 1]
|
# [num_reqs + 1]
|
||||||
query_start_loc: torch.Tensor
|
query_start_loc: torch.Tensor
|
||||||
@@ -64,6 +67,8 @@ class InputBatch:
|
|||||||
seq_lens_cpu_upper_bound: torch.Tensor
|
seq_lens_cpu_upper_bound: torch.Tensor
|
||||||
# [num_reqs]
|
# [num_reqs]
|
||||||
dcp_local_seq_lens: torch.Tensor | None
|
dcp_local_seq_lens: torch.Tensor | None
|
||||||
|
# [num_reqs] CPU bool array.
|
||||||
|
is_prefilling_np: np.ndarray
|
||||||
|
|
||||||
# [num_tokens_after_padding]
|
# [num_tokens_after_padding]
|
||||||
input_ids: torch.Tensor
|
input_ids: torch.Tensor
|
||||||
@@ -137,11 +142,13 @@ class InputBatch:
|
|||||||
num_tokens=num_tokens,
|
num_tokens=num_tokens,
|
||||||
num_tokens_after_padding=num_tokens,
|
num_tokens_after_padding=num_tokens,
|
||||||
num_draft_tokens=0,
|
num_draft_tokens=0,
|
||||||
|
num_draft_tokens_per_req=None,
|
||||||
query_start_loc=query_start_loc,
|
query_start_loc=query_start_loc,
|
||||||
query_start_loc_np=query_start_loc_np,
|
query_start_loc_np=query_start_loc_np,
|
||||||
seq_lens=seq_lens,
|
seq_lens=seq_lens,
|
||||||
seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,
|
seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,
|
||||||
dcp_local_seq_lens=None,
|
dcp_local_seq_lens=None,
|
||||||
|
is_prefilling_np=np.zeros(num_reqs, dtype=np.bool_),
|
||||||
input_ids=input_ids,
|
input_ids=input_ids,
|
||||||
positions=positions,
|
positions=positions,
|
||||||
logits_indices=logits_indices,
|
logits_indices=logits_indices,
|
||||||
|
|||||||
@@ -44,10 +44,11 @@ from vllm.model_executor.model_loader import get_model_loader
|
|||||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
from vllm.sequence import IntermediateTensors
|
from vllm.sequence import IntermediateTensors
|
||||||
from vllm.tasks import SupportedTask
|
from vllm.tasks import SupportedTask
|
||||||
|
from vllm.utils.math_utils import cdiv
|
||||||
from vllm.utils.mem_utils import DeviceMemoryProfiler, format_gib
|
from vllm.utils.mem_utils import DeviceMemoryProfiler, format_gib
|
||||||
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE
|
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE
|
||||||
from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
|
from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
|
||||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
from vllm.v1.kv_cache_interface import KVCacheConfig, MambaSpec
|
||||||
from vllm.v1.outputs import DraftTokenIds, KVConnectorOutput, ModelRunnerOutput
|
from vllm.v1.outputs import DraftTokenIds, KVConnectorOutput, ModelRunnerOutput
|
||||||
from vllm.v1.worker.cp_utils import check_attention_cp_compatibility
|
from vllm.v1.worker.cp_utils import check_attention_cp_compatibility
|
||||||
from vllm.v1.worker.gpu.async_utils import AsyncOutput, AsyncPoolingOutput
|
from vllm.v1.worker.gpu.async_utils import AsyncOutput, AsyncPoolingOutput
|
||||||
@@ -338,10 +339,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
|||||||
def initialize_kv_cache(self, kv_cache_config: KVCacheConfig) -> None:
|
def initialize_kv_cache(self, kv_cache_config: KVCacheConfig) -> None:
|
||||||
kv_cache_config = deepcopy(kv_cache_config)
|
kv_cache_config = deepcopy(kv_cache_config)
|
||||||
self.kv_cache_config = kv_cache_config
|
self.kv_cache_config = kv_cache_config
|
||||||
block_sizes = [
|
|
||||||
kv_cache_group.kv_cache_spec.block_size
|
|
||||||
for kv_cache_group in kv_cache_config.kv_cache_groups
|
|
||||||
]
|
|
||||||
|
|
||||||
block_table_max_model_len = self.max_model_len
|
block_table_max_model_len = self.max_model_len
|
||||||
if self.is_encoder_decoder:
|
if self.is_encoder_decoder:
|
||||||
@@ -352,11 +349,34 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
|||||||
getattr(self.model_config.hf_config, "max_source_positions", 0),
|
getattr(self.model_config.hf_config, "max_source_positions", 0),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
block_sizes = []
|
||||||
|
max_num_blocks_per_group = []
|
||||||
|
for kv_cache_group in kv_cache_config.kv_cache_groups:
|
||||||
|
spec = kv_cache_group.kv_cache_spec
|
||||||
|
block_sizes.append(spec.block_size)
|
||||||
|
# When using DCP, each request's KV cache is sharded among different ranks.
|
||||||
|
# As a result, one block on the current rank covers `block_size * cp_size`
|
||||||
|
# tokens in the full, global (unsharded) sequence.
|
||||||
|
max_num_blocks = cdiv(
|
||||||
|
block_table_max_model_len, spec.block_size * self.dcp_size
|
||||||
|
)
|
||||||
|
# Align to a multiple of (128 / block_size) as required by some attention
|
||||||
|
# backends such as TRTLLM (#39324)
|
||||||
|
if spec.block_size <= 128:
|
||||||
|
alignment = 128 // spec.block_size
|
||||||
|
max_num_blocks = cdiv(max_num_blocks, alignment) * alignment
|
||||||
|
# For Mamba/Hybrid Model, KVCaches need extra blocks for speculative tokens
|
||||||
|
if isinstance(spec, MambaSpec):
|
||||||
|
max_num_blocks = (
|
||||||
|
max_num_blocks if self.cache_config.enable_prefix_caching else 1
|
||||||
|
) + spec.num_speculative_blocks
|
||||||
|
max_num_blocks_per_group.append(max_num_blocks)
|
||||||
|
|
||||||
self.block_tables = BlockTables(
|
self.block_tables = BlockTables(
|
||||||
block_sizes=block_sizes,
|
block_sizes=block_sizes,
|
||||||
max_num_reqs=self.max_num_reqs,
|
max_num_reqs=self.max_num_reqs,
|
||||||
max_num_batched_tokens=self.max_num_tokens,
|
max_num_batched_tokens=self.max_num_tokens,
|
||||||
max_model_len=block_table_max_model_len,
|
max_num_blocks_per_group=max_num_blocks_per_group,
|
||||||
device=self.device,
|
device=self.device,
|
||||||
cp_size=self.dcp_size,
|
cp_size=self.dcp_size,
|
||||||
cp_rank=self.dcp_rank,
|
cp_rank=self.dcp_rank,
|
||||||
@@ -722,6 +742,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
|||||||
|
|
||||||
# Get the number of draft tokens for each request.
|
# Get the number of draft tokens for each request.
|
||||||
draft_tokens = scheduler_output.scheduled_spec_decode_tokens
|
draft_tokens = scheduler_output.scheduled_spec_decode_tokens
|
||||||
|
num_draft_tokens_per_req = None
|
||||||
if not draft_tokens:
|
if not draft_tokens:
|
||||||
# No draft token scheduled (common case).
|
# No draft token scheduled (common case).
|
||||||
total_num_draft_tokens = 0
|
total_num_draft_tokens = 0
|
||||||
@@ -735,15 +756,15 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
|||||||
num_reqs, dtype=torch.int32, device=self.device
|
num_reqs, dtype=torch.int32, device=self.device
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
num_draft_tokens = np.fromiter(
|
num_draft_tokens_per_req = np.fromiter(
|
||||||
(len(draft_tokens.get(req_id, ())) for req_id in req_ids),
|
(len(draft_tokens.get(req_id, ())) for req_id in req_ids),
|
||||||
dtype=np.int32,
|
dtype=np.int32,
|
||||||
count=num_reqs,
|
count=num_reqs,
|
||||||
)
|
)
|
||||||
total_num_draft_tokens = int(num_draft_tokens.sum())
|
total_num_draft_tokens = int(num_draft_tokens_per_req.sum())
|
||||||
total_num_logits = num_reqs + total_num_draft_tokens
|
total_num_logits = num_reqs + total_num_draft_tokens
|
||||||
|
|
||||||
num_logits = num_draft_tokens + 1
|
num_logits = num_draft_tokens_per_req + 1
|
||||||
cu_num_logits_np = np.empty(num_reqs + 1, dtype=np.int32)
|
cu_num_logits_np = np.empty(num_reqs + 1, dtype=np.int32)
|
||||||
cu_num_logits_np[0] = 0
|
cu_num_logits_np[0] = 0
|
||||||
np.cumsum(num_logits, out=cu_num_logits_np[1:])
|
np.cumsum(num_logits, out=cu_num_logits_np[1:])
|
||||||
@@ -766,9 +787,13 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
|||||||
async_copy_to_gpu(query_start_loc_np, out=self.input_buffers.query_start_loc)
|
async_copy_to_gpu(query_start_loc_np, out=self.input_buffers.query_start_loc)
|
||||||
query_start_loc_np = query_start_loc_np[: num_reqs_padded + 1]
|
query_start_loc_np = query_start_loc_np[: num_reqs_padded + 1]
|
||||||
query_start_loc = self.input_buffers.query_start_loc[: num_reqs_padded + 1]
|
query_start_loc = self.input_buffers.query_start_loc[: num_reqs_padded + 1]
|
||||||
|
is_prefilling_np = (
|
||||||
|
self.req_states.num_computed_prefill_tokens[idx_mapping_np]
|
||||||
|
< self.req_states.prefill_len.np[idx_mapping_np]
|
||||||
|
)
|
||||||
|
|
||||||
# Get prefill tokens if any.
|
# Get prefill tokens if any.
|
||||||
if self.req_states.any_prefills(idx_mapping_np):
|
if np.any(is_prefilling_np):
|
||||||
prepare_prefill_inputs(
|
prepare_prefill_inputs(
|
||||||
self.input_buffers.input_ids,
|
self.input_buffers.input_ids,
|
||||||
self.req_states.next_prefill_tokens,
|
self.req_states.next_prefill_tokens,
|
||||||
@@ -836,11 +861,13 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
|||||||
num_tokens=num_tokens,
|
num_tokens=num_tokens,
|
||||||
num_tokens_after_padding=num_tokens_after_padding,
|
num_tokens_after_padding=num_tokens_after_padding,
|
||||||
num_draft_tokens=total_num_draft_tokens,
|
num_draft_tokens=total_num_draft_tokens,
|
||||||
|
num_draft_tokens_per_req=num_draft_tokens_per_req,
|
||||||
query_start_loc=query_start_loc,
|
query_start_loc=query_start_loc,
|
||||||
query_start_loc_np=query_start_loc_np,
|
query_start_loc_np=query_start_loc_np,
|
||||||
seq_lens=seq_lens,
|
seq_lens=seq_lens,
|
||||||
seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,
|
seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,
|
||||||
dcp_local_seq_lens=dcp_local_seq_lens,
|
dcp_local_seq_lens=dcp_local_seq_lens,
|
||||||
|
is_prefilling_np=is_prefilling_np,
|
||||||
input_ids=self.input_buffers.input_ids[:num_tokens_after_padding],
|
input_ids=self.input_buffers.input_ids[:num_tokens_after_padding],
|
||||||
positions=self.input_buffers.positions[:num_tokens_after_padding],
|
positions=self.input_buffers.positions[:num_tokens_after_padding],
|
||||||
logits_indices=logits_indices,
|
logits_indices=logits_indices,
|
||||||
@@ -958,6 +985,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
|||||||
input_batch.num_scheduled_tokens
|
input_batch.num_scheduled_tokens
|
||||||
)
|
)
|
||||||
|
|
||||||
|
self.model_state.postprocess_state(input_batch, num_sampled)
|
||||||
|
|
||||||
@torch.inference_mode()
|
@torch.inference_mode()
|
||||||
def execute_model(
|
def execute_model(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -18,6 +18,11 @@ def init_model_state(
|
|||||||
|
|
||||||
return WhisperModelState(vllm_config, model, encoder_cache, device)
|
return WhisperModelState(vllm_config, model, encoder_cache, device)
|
||||||
|
|
||||||
|
if vllm_config.model_config.is_hybrid:
|
||||||
|
from vllm.v1.worker.gpu.model_states.mamba_hybrid import MambaHybridModelState
|
||||||
|
|
||||||
|
return MambaHybridModelState(vllm_config, model, encoder_cache, device)
|
||||||
|
|
||||||
from vllm.v1.worker.gpu.model_states.default import DefaultModelState
|
from vllm.v1.worker.gpu.model_states.default import DefaultModelState
|
||||||
|
|
||||||
return DefaultModelState(vllm_config, model, encoder_cache, device)
|
return DefaultModelState(vllm_config, model, encoder_cache, device)
|
||||||
|
|||||||
@@ -194,5 +194,6 @@ class DefaultModelState(ModelState):
|
|||||||
seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,
|
seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,
|
||||||
dcp_local_seq_lens=input_batch.dcp_local_seq_lens,
|
dcp_local_seq_lens=input_batch.dcp_local_seq_lens,
|
||||||
positions=input_batch.positions,
|
positions=input_batch.positions,
|
||||||
|
for_cudagraph_capture=for_capture,
|
||||||
)
|
)
|
||||||
return attn_metadata
|
return attn_metadata
|
||||||
|
|||||||
@@ -17,6 +17,24 @@ from vllm.v1.worker.gpu.states import RequestState
|
|||||||
from vllm.v1.worker.utils import AttentionGroup
|
from vllm.v1.worker.utils import AttentionGroup
|
||||||
|
|
||||||
|
|
||||||
|
class ModelSpecificAttnMetadata:
|
||||||
|
"""Base class for model-specific attention metadata."""
|
||||||
|
|
||||||
|
def get_extra_common_attn_kwargs(
|
||||||
|
self,
|
||||||
|
kv_cache_group_id: int,
|
||||||
|
num_reqs: int,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
return {}
|
||||||
|
|
||||||
|
def get_extra_attn_kwargs(
|
||||||
|
self,
|
||||||
|
attn_metadata_builder: Any,
|
||||||
|
num_reqs: int,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
return {}
|
||||||
|
|
||||||
|
|
||||||
class ModelState(ABC):
|
class ModelState(ABC):
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def __init__(
|
def __init__(
|
||||||
@@ -38,6 +56,13 @@ class ModelState(ABC):
|
|||||||
def apply_staged_writes(self) -> None:
|
def apply_staged_writes(self) -> None:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
def postprocess_state(
|
||||||
|
self,
|
||||||
|
input_batch: InputBatch,
|
||||||
|
num_sampled: torch.Tensor,
|
||||||
|
) -> None:
|
||||||
|
return None
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def get_mm_embeddings(
|
def get_mm_embeddings(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -0,0 +1,150 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
import torch
|
||||||
|
import torch.nn as nn
|
||||||
|
|
||||||
|
from vllm.config import VllmConfig
|
||||||
|
from vllm.config.compilation import CUDAGraphMode
|
||||||
|
from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadataBuilder
|
||||||
|
from vllm.v1.attention.backends.mamba2_attn import Mamba2AttentionMetadataBuilder
|
||||||
|
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||||
|
from vllm.v1.worker.gpu.attn_utils import build_attn_metadata
|
||||||
|
from vllm.v1.worker.gpu.input_batch import InputBatch
|
||||||
|
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
|
||||||
|
from vllm.v1.worker.gpu.model_states.default import DefaultModelState
|
||||||
|
from vllm.v1.worker.gpu.model_states.interface import ModelSpecificAttnMetadata
|
||||||
|
from vllm.v1.worker.utils import AttentionGroup
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class MambaHybridAttnMetadata(ModelSpecificAttnMetadata):
|
||||||
|
is_prefilling: torch.Tensor
|
||||||
|
num_accepted_tokens: torch.Tensor | None = None
|
||||||
|
num_decode_draft_tokens_cpu: torch.Tensor | None = None
|
||||||
|
|
||||||
|
def get_extra_common_attn_kwargs(
|
||||||
|
self,
|
||||||
|
kv_cache_group_id: int,
|
||||||
|
num_reqs: int,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
return {"is_prefilling": self.is_prefilling[:num_reqs]}
|
||||||
|
|
||||||
|
def get_extra_attn_kwargs(
|
||||||
|
self,
|
||||||
|
attn_metadata_builder: Any,
|
||||||
|
num_reqs: int,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
if not isinstance(
|
||||||
|
attn_metadata_builder,
|
||||||
|
(Mamba2AttentionMetadataBuilder, GDNAttentionMetadataBuilder),
|
||||||
|
):
|
||||||
|
return {}
|
||||||
|
return {
|
||||||
|
"num_accepted_tokens": None
|
||||||
|
if self.num_accepted_tokens is None
|
||||||
|
else self.num_accepted_tokens[:num_reqs],
|
||||||
|
"num_decode_draft_tokens_cpu": None
|
||||||
|
if self.num_decode_draft_tokens_cpu is None
|
||||||
|
else self.num_decode_draft_tokens_cpu[:num_reqs],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class MambaHybridModelState(DefaultModelState):
|
||||||
|
"""Model state for hybrid attention + Mamba / linear-attention models."""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
vllm_config: VllmConfig,
|
||||||
|
model: nn.Module,
|
||||||
|
encoder_cache: EncoderCache | None,
|
||||||
|
device: torch.device,
|
||||||
|
) -> None:
|
||||||
|
super().__init__(vllm_config, model, encoder_cache, device)
|
||||||
|
self.num_accepted_tokens_gpu = torch.ones(
|
||||||
|
self.max_num_reqs, dtype=torch.int32, device=self.device
|
||||||
|
)
|
||||||
|
|
||||||
|
def prepare_attn(
|
||||||
|
self,
|
||||||
|
input_batch: InputBatch,
|
||||||
|
cudagraph_mode: CUDAGraphMode,
|
||||||
|
block_tables: tuple[torch.Tensor, ...],
|
||||||
|
slot_mappings: torch.Tensor,
|
||||||
|
attn_groups: list[list[AttentionGroup]],
|
||||||
|
kv_cache_config: KVCacheConfig,
|
||||||
|
for_capture: bool = False,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
if cudagraph_mode == CUDAGraphMode.FULL:
|
||||||
|
num_reqs = input_batch.num_reqs_after_padding
|
||||||
|
num_tokens = input_batch.num_tokens_after_padding
|
||||||
|
else:
|
||||||
|
num_reqs = input_batch.num_reqs
|
||||||
|
num_tokens = input_batch.num_tokens
|
||||||
|
query_start_loc_cpu = torch.from_numpy(input_batch.query_start_loc_np)
|
||||||
|
max_query_len = input_batch.num_scheduled_tokens.max().item()
|
||||||
|
|
||||||
|
is_prefilling = torch.zeros(num_reqs, dtype=torch.bool, device="cpu")
|
||||||
|
is_prefilling[: input_batch.num_reqs] = torch.from_numpy(
|
||||||
|
input_batch.is_prefilling_np
|
||||||
|
)
|
||||||
|
# During CUDAGraph capture, num_decode_draft_tokens_cpu and num_accepted_tokens
|
||||||
|
# are created by attn_metadata_builder.build_for_cudagraph_capture, so we only
|
||||||
|
# compute them during actual (non-capture) forward execution.
|
||||||
|
num_accepted_tokens = None
|
||||||
|
num_decode_draft_tokens_cpu = None
|
||||||
|
if not for_capture:
|
||||||
|
num_accepted_tokens = self.num_accepted_tokens_gpu.new_ones(num_reqs)
|
||||||
|
num_accepted_tokens[: input_batch.num_reqs] = self.num_accepted_tokens_gpu[
|
||||||
|
input_batch.idx_mapping
|
||||||
|
]
|
||||||
|
|
||||||
|
# GDN uses >= 0 to select spec-decode rows, so non-decode rows
|
||||||
|
# need the -1 sentinel rather than a raw zero draft count.
|
||||||
|
num_decode_draft_tokens_np = np.full(num_reqs, -1, dtype=np.int32)
|
||||||
|
if input_batch.num_draft_tokens_per_req is not None:
|
||||||
|
spec_decode_mask = (
|
||||||
|
input_batch.num_draft_tokens_per_req > 0
|
||||||
|
) & ~input_batch.is_prefilling_np
|
||||||
|
num_decode_draft_tokens_np[: input_batch.num_reqs] = np.where(
|
||||||
|
spec_decode_mask,
|
||||||
|
input_batch.num_draft_tokens_per_req,
|
||||||
|
-1,
|
||||||
|
)
|
||||||
|
num_decode_draft_tokens_cpu = torch.from_numpy(num_decode_draft_tokens_np)
|
||||||
|
|
||||||
|
mamba_attn_metadata = MambaHybridAttnMetadata(
|
||||||
|
is_prefilling=is_prefilling,
|
||||||
|
num_accepted_tokens=num_accepted_tokens,
|
||||||
|
num_decode_draft_tokens_cpu=num_decode_draft_tokens_cpu,
|
||||||
|
)
|
||||||
|
return build_attn_metadata(
|
||||||
|
attn_groups=attn_groups,
|
||||||
|
num_reqs=num_reqs,
|
||||||
|
num_tokens=num_tokens,
|
||||||
|
query_start_loc_gpu=input_batch.query_start_loc,
|
||||||
|
query_start_loc_cpu=query_start_loc_cpu,
|
||||||
|
max_query_len=max_query_len,
|
||||||
|
seq_lens=input_batch.seq_lens,
|
||||||
|
max_seq_len=self.max_model_len,
|
||||||
|
block_tables=block_tables,
|
||||||
|
slot_mappings=slot_mappings,
|
||||||
|
kv_cache_config=kv_cache_config,
|
||||||
|
dcp_local_seq_lens=input_batch.dcp_local_seq_lens,
|
||||||
|
model_specific_attn_metadata=mamba_attn_metadata,
|
||||||
|
for_cudagraph_capture=for_capture,
|
||||||
|
)
|
||||||
|
|
||||||
|
def postprocess_state(
|
||||||
|
self,
|
||||||
|
input_batch: InputBatch,
|
||||||
|
num_sampled: torch.Tensor,
|
||||||
|
) -> None:
|
||||||
|
# Chunked prefill does not sample a token, so num_sampled can be 0.
|
||||||
|
# Mamba treats num_accepted_tokens=1 as the neutral non-spec value.
|
||||||
|
self.num_accepted_tokens_gpu[input_batch.idx_mapping] = torch.clamp(
|
||||||
|
num_sampled, min=1
|
||||||
|
)
|
||||||
@@ -1,5 +1,6 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
from dataclasses import dataclass
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
import numpy as np
|
import numpy as np
|
||||||
@@ -13,11 +14,33 @@ from vllm.v1.worker.gpu.attn_utils import build_attn_metadata
|
|||||||
from vllm.v1.worker.gpu.input_batch import InputBatch
|
from vllm.v1.worker.gpu.input_batch import InputBatch
|
||||||
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
|
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
|
||||||
from vllm.v1.worker.gpu.mm.encoder_runner import EncoderRunner
|
from vllm.v1.worker.gpu.mm.encoder_runner import EncoderRunner
|
||||||
from vllm.v1.worker.gpu.model_states.interface import ModelState
|
from vllm.v1.worker.gpu.model_states.interface import (
|
||||||
|
ModelSpecificAttnMetadata,
|
||||||
|
ModelState,
|
||||||
|
)
|
||||||
from vllm.v1.worker.gpu.states import RequestState
|
from vllm.v1.worker.gpu.states import RequestState
|
||||||
from vllm.v1.worker.utils import AttentionGroup
|
from vllm.v1.worker.utils import AttentionGroup
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class WhisperAttnMetadata(ModelSpecificAttnMetadata):
|
||||||
|
encoder_seq_lens: dict[int, tuple[torch.Tensor, np.ndarray]]
|
||||||
|
|
||||||
|
def get_extra_common_attn_kwargs(
|
||||||
|
self,
|
||||||
|
kv_cache_group_id: int,
|
||||||
|
num_reqs: int,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
encoder_seq_lens = self.encoder_seq_lens.get(kv_cache_group_id)
|
||||||
|
if encoder_seq_lens is None:
|
||||||
|
return {}
|
||||||
|
encoder_seq_lens_gpu, encoder_seq_lens_cpu = encoder_seq_lens
|
||||||
|
return {
|
||||||
|
"encoder_seq_lens": encoder_seq_lens_gpu[:num_reqs],
|
||||||
|
"encoder_seq_lens_cpu": encoder_seq_lens_cpu[:num_reqs],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
class WhisperModelState(ModelState):
|
class WhisperModelState(ModelState):
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
@@ -111,8 +134,8 @@ class WhisperModelState(ModelState):
|
|||||||
else:
|
else:
|
||||||
num_reqs = input_batch.num_reqs
|
num_reqs = input_batch.num_reqs
|
||||||
num_tokens = input_batch.num_tokens
|
num_tokens = input_batch.num_tokens
|
||||||
encoder_seq_lens = self._get_encoder_seq_lens(
|
whisper_attn_metadata = WhisperAttnMetadata(
|
||||||
input_batch.req_ids, attn_groups, for_capture
|
self._get_encoder_seq_lens(input_batch.req_ids, attn_groups, for_capture)
|
||||||
)
|
)
|
||||||
|
|
||||||
query_start_loc_cpu = torch.from_numpy(input_batch.query_start_loc_np)
|
query_start_loc_cpu = torch.from_numpy(input_batch.query_start_loc_np)
|
||||||
@@ -136,7 +159,8 @@ class WhisperModelState(ModelState):
|
|||||||
kv_cache_config=kv_cache_config,
|
kv_cache_config=kv_cache_config,
|
||||||
seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,
|
seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,
|
||||||
dcp_local_seq_lens=input_batch.dcp_local_seq_lens,
|
dcp_local_seq_lens=input_batch.dcp_local_seq_lens,
|
||||||
encoder_seq_lens=encoder_seq_lens,
|
model_specific_attn_metadata=whisper_attn_metadata,
|
||||||
|
for_cudagraph_capture=for_capture,
|
||||||
)
|
)
|
||||||
return attn_metadata
|
return attn_metadata
|
||||||
|
|
||||||
|
|||||||
@@ -106,6 +106,7 @@ class DecodeEagleCudaGraphManager(EagleCudaGraphManagerBase):
|
|||||||
block_tables,
|
block_tables,
|
||||||
attn_groups,
|
attn_groups,
|
||||||
kv_cache_config,
|
kv_cache_config,
|
||||||
|
skip_attn=(desc.cg_mode == CUDAGraphMode.PIECEWISE),
|
||||||
)
|
)
|
||||||
attn_metadata, slot_mappings = attn_state
|
attn_metadata, slot_mappings = attn_state
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user