[Model Runner V2] support qwen35 / mamba hybrid model (#35520)

Signed-off-by: zhuhaoran <zhuhaoran.zhr@alibaba-inc.com>
This commit is contained in:
zhrrr
2026-05-07 09:31:05 -07:00
committed by GitHub
parent 06a60d3dd0
commit 7a08b34fbf
14 changed files with 439 additions and 106 deletions
+1 -1
View File
@@ -142,7 +142,7 @@ class KernelConfig:
"""Backend for MoE expert computation kernels. Available options:
- "auto": Automatically select the best backend based on model and hardware
- "triton": Use Triton-based fused MoE kernels
- "triton": Use Triton-based fused MoE kernels
- "deep_gemm": Use DeepGEMM kernels (FP8 block-quantized only)
- "deep_gemm_mega_moe": Use DeepGEMM mega MoE kernels
- "cutlass": Use vLLM CUTLASS kernels
+4
View File
@@ -1892,6 +1892,10 @@ class VllmConfig:
"to schedule a multiple of block_size tokens even if they are "
"in the middle of a mm input"
)
# TODO: support align mamba cache mode for model runner v2
assert not envs.VLLM_USE_V2_MODEL_RUNNER, (
"Model Runner V2 has not yet supported mamba_cache_mode='align'. "
)
@model_validator(mode="after")
def validate_nvfp4_kv_cache_with_mla(self) -> "VllmConfig":
+14 -2
View File
@@ -111,7 +111,15 @@ class MultiModalRegistry:
return False
mm_config = model_config.get_multimodal_config()
info = self._create_processing_info(model_config, tokenizer=None)
try:
info = self._create_processing_info(model_config, tokenizer=None)
except ValueError:
logger.warning_once(
"Model %s is treated as multimodal but has no registered "
"multimodal processor; running in text-only mode.",
model_config.model,
)
return False
# Check if all supported modalities have limit == 0
if all(
@@ -170,7 +178,11 @@ class MultiModalRegistry:
from vllm.model_executor.model_loader import get_model_architecture
model_cls, _ = get_model_architecture(model_config)
assert hasattr(model_cls, "_processor_factory")
if not hasattr(model_cls, "_processor_factory"):
raise ValueError(
f"Model class {model_cls.__name__} has no registered "
"multimodal processor"
)
return cast("SupportsMultiModal", model_cls)
def _create_processing_ctx(
+137 -60
View File
@@ -4,7 +4,6 @@ from collections.abc import Sequence
from dataclasses import dataclass
from typing import Any, cast
import numpy as np
import torch
from vllm.config import VllmConfig, get_layers_from_vllm_config
@@ -19,8 +18,10 @@ from vllm.v1.kv_cache_interface import (
AttentionSpec,
KVCacheConfig,
KVCacheSpec,
MambaSpec,
UniformTypeKVCacheSpecs,
)
from vllm.v1.worker.gpu.model_states.interface import ModelSpecificAttnMetadata
from vllm.v1.worker.utils import AttentionGroup, bind_kv_cache
@@ -147,68 +148,126 @@ def _reshape_kv_cache(
kv_cache_raw_tensors: dict[str, torch.Tensor],
attn_backends: dict[str, type[AttentionBackend]],
cache_dtype: str,
) -> dict[str, torch.Tensor]:
kv_caches: dict[str, torch.Tensor] = {}
) -> dict[str, Any]:
kv_caches: dict[str, Any] = {}
has_attn, has_mamba = False, False
for kv_cache_group_spec in kv_cache_config.kv_cache_groups:
for layer_name in kv_cache_group_spec.layer_names:
kv_cache_spec = kv_cache_group_spec.kv_cache_spec
if isinstance(kv_cache_spec, UniformTypeKVCacheSpecs):
kv_cache_spec = kv_cache_spec.kv_cache_specs[layer_name]
assert isinstance(kv_cache_spec, AttentionSpec)
raw_tensor = kv_cache_raw_tensors[layer_name]
assert raw_tensor.numel() % kv_cache_spec.page_size_bytes == 0
num_blocks = raw_tensor.numel() // kv_cache_spec.page_size_bytes
kv_raw_tensor = kv_cache_raw_tensors[layer_name]
assert kv_raw_tensor.numel() % kv_cache_spec.page_size_bytes == 0
num_blocks = kv_raw_tensor.numel() // kv_cache_spec.page_size_bytes
attn_backend = attn_backends[layer_name]
kv_cache_shape = attn_backend.get_kv_cache_shape(
num_blocks,
kv_cache_spec.storage_block_size,
kv_cache_spec.num_kv_heads,
kv_cache_spec.head_size,
cache_dtype,
)
# FIXME(woosuk): Add kv_cache_stride_order to all attention backends.
try:
kv_cache_stride_order = attn_backend.get_kv_cache_stride_order()
assert len(kv_cache_stride_order) == len(kv_cache_shape)
except (AttributeError, NotImplementedError):
kv_cache_stride_order = tuple(range(len(kv_cache_shape)))
kv_cache_shape = tuple(kv_cache_shape[i] for i in kv_cache_stride_order)
inv_order = [
kv_cache_stride_order.index(i)
for i in range(len(kv_cache_stride_order))
]
dtype = kv_cache_spec.dtype
raw_tensor = raw_tensor.view(dtype)
if kv_cache_spec.page_size_padded is not None:
# Use strided view to handle page_size_bytes that
# include padding. This follows the same pattern as
# MambaSpec handling in gpu_model_runner.py.
# NOTE: This assumes kv_cache_shape[0] == num_blocks
# (i.e. the first physical dimension is the block
# index), which holds for MLA backends but NOT for
# standard attention backends whose shape starts with
# a K/V dimension of size 2.
dtype_size = get_dtype_size(dtype)
page_stride = kv_cache_spec.page_size_bytes // dtype_size
strides = list(torch.empty(kv_cache_shape).stride())
strides[inv_order[0]] = page_stride
kv_cache = torch.as_strided(
raw_tensor,
size=kv_cache_shape,
stride=tuple(strides),
if isinstance(kv_cache_spec, AttentionSpec):
has_attn = True
attn_backend = attn_backends[layer_name]
kv_cache_shape = attn_backend.get_kv_cache_shape(
num_blocks,
kv_cache_spec.storage_block_size,
kv_cache_spec.num_kv_heads,
kv_cache_spec.head_size,
cache_dtype_str=cache_dtype,
)
# FIXME(woosuk): Add kv_cache_stride_order to all attention backends.
try:
kv_cache_stride_order = attn_backend.get_kv_cache_stride_order()
assert len(kv_cache_stride_order) == len(kv_cache_shape)
except (AttributeError, NotImplementedError):
kv_cache_stride_order = tuple(range(len(kv_cache_shape)))
kv_cache_shape = tuple(kv_cache_shape[i] for i in kv_cache_stride_order)
inv_order = [
kv_cache_stride_order.index(i)
for i in range(len(kv_cache_stride_order))
]
dtype = kv_cache_spec.dtype
kv_tensor = kv_raw_tensor.view(dtype)
if kv_cache_spec.page_size_padded is not None:
# Use strided view to handle page_size_bytes that
# include padding. This follows the same pattern as
# MambaSpec handling in gpu_model_runner.py.
# NOTE: This assumes kv_cache_shape[0] == num_blocks
# (i.e. the first physical dimension is the block
# index), which holds for MLA backends but NOT for
# standard attention backends whose shape starts with
# a K/V dimension of size 2.
dtype_size = get_dtype_size(dtype)
page_stride = kv_cache_spec.page_size_bytes // dtype_size
strides = list(torch.empty(kv_cache_shape).stride())
strides[inv_order[0]] = page_stride
kv_cache = torch.as_strided(
kv_tensor,
size=kv_cache_shape,
stride=tuple(strides),
)
else:
# No padding — safe to use a contiguous view.
kv_cache = kv_tensor.view(kv_cache_shape)
kv_caches[layer_name] = kv_cache.permute(*inv_order)
elif isinstance(kv_cache_spec, MambaSpec):
has_mamba = True
state_tensors = []
storage_offset_bytes = 0
for shape, dtype in zip(kv_cache_spec.shapes, kv_cache_spec.dtypes):
dtype_size = get_dtype_size(dtype)
num_element_per_page = kv_cache_spec.page_size_bytes // dtype_size
target_shape = (num_blocks, *shape)
stride = torch.empty(target_shape).stride()
target_stride = (num_element_per_page, *stride[1:])
assert storage_offset_bytes % dtype_size == 0
tensor = torch.as_strided(
kv_raw_tensor.view(dtype),
size=target_shape,
stride=target_stride,
storage_offset=storage_offset_bytes // dtype_size,
)
state_tensors.append(tensor)
storage_offset_bytes += stride[0] * dtype_size
kv_caches[layer_name] = state_tensors
else:
# No padding — safe to use a contiguous view.
kv_cache = raw_tensor.view(kv_cache_shape)
kv_caches[layer_name] = kv_cache.permute(*inv_order)
raise NotImplementedError(
f"Unsupported KV cache spec type: {type(kv_cache_spec)}"
)
if has_attn and has_mamba:
_update_hybrid_attention_layout(kv_caches, kv_cache_config)
return kv_caches
def _update_hybrid_attention_layout(
kv_caches: dict[str, Any],
kv_cache_config: KVCacheConfig,
) -> None:
for kv_cache_group_spec in kv_cache_config.kv_cache_groups:
for layer_name in kv_cache_group_spec.layer_names:
kv_cache_spec = kv_cache_group_spec.kv_cache_spec
if isinstance(kv_cache_spec, UniformTypeKVCacheSpecs):
kv_cache_spec = kv_cache_spec.kv_cache_specs[layer_name]
if not isinstance(kv_cache_spec, AttentionSpec):
continue
kv_cache = kv_caches[layer_name]
if kv_cache.shape[0] == 2:
assert kv_cache.shape[1] != 2, (
f"Cannot determine layout for tensor of shape {kv_cache.shape}"
)
hidden_size = kv_cache.shape[2:].numel()
kv_cache.as_strided_(
size=kv_cache.shape,
stride=(
hidden_size,
2 * hidden_size,
*kv_cache.stride()[2:],
),
)
def init_kv_cache(
runner_kv_caches: list[torch.Tensor],
forward_context: dict[str, Any],
@@ -216,7 +275,7 @@ def init_kv_cache(
attn_backends: dict[str, type[AttentionBackend]],
device: torch.device,
cache_dtype: str,
) -> dict[str, torch.Tensor]:
) -> dict[str, Any]:
kv_cache_raw_tensors = _allocate_kv_cache(kv_cache_config, device)
kv_caches = _reshape_kv_cache(
kv_cache_config, kv_cache_raw_tensors, attn_backends, cache_dtype
@@ -250,8 +309,9 @@ def build_attn_metadata(
kv_cache_config: KVCacheConfig,
seq_lens_cpu_upper_bound: torch.Tensor | None = None,
dcp_local_seq_lens: torch.Tensor | None = None,
encoder_seq_lens: dict[int, tuple[torch.Tensor, np.ndarray]] | None = None,
positions: torch.Tensor | None = None,
model_specific_attn_metadata: ModelSpecificAttnMetadata | None = None,
for_cudagraph_capture: bool = False,
) -> dict[str, Any]:
seq_lens = seq_lens[:num_reqs]
if dcp_local_seq_lens is not None:
@@ -265,6 +325,11 @@ def build_attn_metadata(
block_table = block_tables[i]
slot_mapping = slot_mappings[i]
common_attn_metadata_extra_kwargs = (
model_specific_attn_metadata.get_extra_common_attn_kwargs(i, num_reqs)
if model_specific_attn_metadata is not None
else {}
)
common_attn_metadata = CommonAttentionMetadata(
query_start_loc=query_start_loc_gpu,
query_start_loc_cpu=query_start_loc_cpu,
@@ -279,17 +344,29 @@ def build_attn_metadata(
causal=True,
dcp_local_seq_lens=dcp_local_seq_lens,
positions=positions,
**common_attn_metadata_extra_kwargs,
)
if encoder_seq_lens and i in encoder_seq_lens:
encoder_seq_lens_gpu, encoder_seq_lens_cpu = encoder_seq_lens[i]
common_attn_metadata.encoder_seq_lens = encoder_seq_lens_gpu
common_attn_metadata.encoder_seq_lens_cpu = encoder_seq_lens_cpu
for attn_group in attn_groups[i]:
attn_metadata_builder = attn_group.get_metadata_builder(0)
metadata = attn_metadata_builder.build(
common_prefix_len=0, common_attn_metadata=common_attn_metadata
)
if for_cudagraph_capture:
metadata = attn_metadata_builder.build_for_cudagraph_capture(
common_attn_metadata
)
else:
attn_metadata_extra_kwargs = (
model_specific_attn_metadata.get_extra_attn_kwargs(
attn_metadata_builder,
num_reqs,
)
if model_specific_attn_metadata is not None
else {}
)
metadata = attn_metadata_builder.build(
common_prefix_len=0,
common_attn_metadata=common_attn_metadata,
**attn_metadata_extra_kwargs,
)
for layer_name in attn_group.layer_names:
attn_metadata[layer_name] = metadata
return attn_metadata
+3 -13
View File
@@ -5,7 +5,6 @@ from collections.abc import Iterable
import torch
from vllm.triton_utils import tl, triton
from vllm.utils.math_utils import cdiv
from vllm.v1.attention.backends.utils import PAD_SLOT_ID
from vllm.v1.worker.gpu.buffer_utils import StagedWriteTensor, UvaBackedTensor
@@ -16,7 +15,7 @@ class BlockTables:
block_sizes: list[int],
max_num_reqs: int,
max_num_batched_tokens: int,
max_model_len: int,
max_num_blocks_per_group: list[int],
device: torch.device,
cp_size: int = 1,
cp_rank: int = 0,
@@ -25,7 +24,6 @@ class BlockTables:
self.block_sizes = block_sizes
self.max_num_reqs = max_num_reqs
self.max_num_batched_tokens = max_num_batched_tokens
self.max_model_len = max_model_len
self.device = device
self.cp_size = cp_size
@@ -33,19 +31,11 @@ class BlockTables:
self.cp_interleave = cp_interleave
self.num_kv_cache_groups = len(self.block_sizes)
assert len(max_num_blocks_per_group) == self.num_kv_cache_groups
# num_kv_cache_groups x [max_num_reqs, max_num_blocks]
self.block_tables: list[StagedWriteTensor] = []
for i in range(self.num_kv_cache_groups):
block_size = self.block_sizes[i]
# When using DCP, each request's KV cache is sharded among different ranks.
# As a result, one block on the current rank covers `block_size * cp_size`
# tokens in the full, global (unsharded) sequence.
max_num_blocks = cdiv(self.max_model_len, block_size * self.cp_size)
# Align to a multiple of (128 / block_size) as required
# by some attention backends such as TRTLLM (#39324)
if block_size <= 128:
alignment = 128 // block_size
max_num_blocks = cdiv(max_num_blocks, alignment) * alignment
max_num_blocks = max_num_blocks_per_group[i]
block_table = StagedWriteTensor(
(self.max_num_reqs, max_num_blocks),
dtype=torch.int32,
+24 -16
View File
@@ -34,7 +34,7 @@ logger = init_logger(__name__)
class CapturedAttentionState(NamedTuple):
attn_metadata: dict[str, Any]
attn_metadata: dict[str, Any] | None
slot_mappings: dict[str, torch.Tensor]
@@ -100,6 +100,7 @@ class CudaGraphManager:
self.decode_query_len = decode_query_len
self.dp_size = vllm_config.parallel_config.data_parallel_size
self.tp_size = vllm_config.parallel_config.tensor_parallel_size
self.is_first_pp_rank = get_pp_group().is_first_rank
self.is_last_pp_rank = get_pp_group().is_last_rank
@@ -109,6 +110,10 @@ class CudaGraphManager:
self._graphs_captured = False
self._candidates: list[list[BatchExecutionDescriptor]] = []
self._capture_descs: dict[CUDAGraphMode, list[BatchExecutionDescriptor]] = {}
# adjust the cudagraph sizes to be a multiple of the uniform decode query length
self.compilation_config.adjust_cudagraph_sizes_for_spec_decode(
self.decode_query_len, self.tp_size
)
self._init_candidates()
def _init_candidates(self) -> None:
@@ -340,16 +345,16 @@ class ModelCudaGraphManager(CudaGraphManager):
block_tables,
attn_groups,
kv_cache_config,
skip_attn=(desc.cg_mode == CUDAGraphMode.PIECEWISE),
)
def forward_fn(cg_mode: CUDAGraphMode) -> None:
batch_descriptor = (
BatchDescriptor(num_tokens=num_tokens)
if cg_mode == CUDAGraphMode.PIECEWISE
else None
)
batch_descriptor = None
if cg_mode == CUDAGraphMode.PIECEWISE:
assert attn_metadata is None
batch_descriptor = BatchDescriptor(num_tokens=num_tokens)
with set_forward_context(
attn_metadata if cg_mode != CUDAGraphMode.PIECEWISE else None,
attn_metadata,
self.vllm_config,
num_tokens=num_tokens,
cudagraph_runtime_mode=cg_mode,
@@ -419,6 +424,7 @@ def prepare_inputs_to_capture(
block_tables: BlockTables,
attn_groups: list[list[AttentionGroup]],
kv_cache_config: KVCacheConfig,
skip_attn: bool = False,
) -> CapturedAttentionState:
input_batch = InputBatch.make_dummy(num_reqs, num_tokens, input_buffers)
input_block_tables = block_tables.get_dummy_block_tables(num_reqs)
@@ -439,13 +445,15 @@ def prepare_inputs_to_capture(
)
input_batch.dcp_local_seq_lens = input_buffers.dcp_local_seq_lens[:num_reqs]
attn_metadata = model_state.prepare_attn(
input_batch,
CUDAGraphMode.NONE,
input_block_tables,
slot_mappings,
attn_groups,
kv_cache_config,
for_capture=True,
)
attn_metadata = None
if not skip_attn:
attn_metadata = model_state.prepare_attn(
input_batch,
CUDAGraphMode.NONE,
input_block_tables,
slot_mappings,
attn_groups,
kv_cache_config,
for_capture=True,
)
return CapturedAttentionState(attn_metadata, slot_mappings_by_layer)
+7
View File
@@ -53,7 +53,10 @@ class InputBatch:
# sum(num_scheduled_tokens)
num_tokens: int
num_tokens_after_padding: int
# Sum of draft tokens scheduled across requests.
num_draft_tokens: int
# [num_reqs] number of draft tokens scheduled for each request, if any.
num_draft_tokens_per_req: np.ndarray | None
# [num_reqs + 1]
query_start_loc: torch.Tensor
@@ -64,6 +67,8 @@ class InputBatch:
seq_lens_cpu_upper_bound: torch.Tensor
# [num_reqs]
dcp_local_seq_lens: torch.Tensor | None
# [num_reqs] CPU bool array.
is_prefilling_np: np.ndarray
# [num_tokens_after_padding]
input_ids: torch.Tensor
@@ -137,11 +142,13 @@ class InputBatch:
num_tokens=num_tokens,
num_tokens_after_padding=num_tokens,
num_draft_tokens=0,
num_draft_tokens_per_req=None,
query_start_loc=query_start_loc,
query_start_loc_np=query_start_loc_np,
seq_lens=seq_lens,
seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,
dcp_local_seq_lens=None,
is_prefilling_np=np.zeros(num_reqs, dtype=np.bool_),
input_ids=input_ids,
positions=positions,
logits_indices=logits_indices,
+39 -10
View File
@@ -44,10 +44,11 @@ from vllm.model_executor.model_loader import get_model_loader
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.sequence import IntermediateTensors
from vllm.tasks import SupportedTask
from vllm.utils.math_utils import cdiv
from vllm.utils.mem_utils import DeviceMemoryProfiler, format_gib
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE
from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.kv_cache_interface import KVCacheConfig, MambaSpec
from vllm.v1.outputs import DraftTokenIds, KVConnectorOutput, ModelRunnerOutput
from vllm.v1.worker.cp_utils import check_attention_cp_compatibility
from vllm.v1.worker.gpu.async_utils import AsyncOutput, AsyncPoolingOutput
@@ -338,10 +339,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
def initialize_kv_cache(self, kv_cache_config: KVCacheConfig) -> None:
kv_cache_config = deepcopy(kv_cache_config)
self.kv_cache_config = kv_cache_config
block_sizes = [
kv_cache_group.kv_cache_spec.block_size
for kv_cache_group in kv_cache_config.kv_cache_groups
]
block_table_max_model_len = self.max_model_len
if self.is_encoder_decoder:
@@ -352,11 +349,34 @@ class GPUModelRunner(LoRAModelRunnerMixin):
getattr(self.model_config.hf_config, "max_source_positions", 0),
)
block_sizes = []
max_num_blocks_per_group = []
for kv_cache_group in kv_cache_config.kv_cache_groups:
spec = kv_cache_group.kv_cache_spec
block_sizes.append(spec.block_size)
# When using DCP, each request's KV cache is sharded among different ranks.
# As a result, one block on the current rank covers `block_size * cp_size`
# tokens in the full, global (unsharded) sequence.
max_num_blocks = cdiv(
block_table_max_model_len, spec.block_size * self.dcp_size
)
# Align to a multiple of (128 / block_size) as required by some attention
# backends such as TRTLLM (#39324)
if spec.block_size <= 128:
alignment = 128 // spec.block_size
max_num_blocks = cdiv(max_num_blocks, alignment) * alignment
# For Mamba/Hybrid Model, KVCaches need extra blocks for speculative tokens
if isinstance(spec, MambaSpec):
max_num_blocks = (
max_num_blocks if self.cache_config.enable_prefix_caching else 1
) + spec.num_speculative_blocks
max_num_blocks_per_group.append(max_num_blocks)
self.block_tables = BlockTables(
block_sizes=block_sizes,
max_num_reqs=self.max_num_reqs,
max_num_batched_tokens=self.max_num_tokens,
max_model_len=block_table_max_model_len,
max_num_blocks_per_group=max_num_blocks_per_group,
device=self.device,
cp_size=self.dcp_size,
cp_rank=self.dcp_rank,
@@ -722,6 +742,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
# Get the number of draft tokens for each request.
draft_tokens = scheduler_output.scheduled_spec_decode_tokens
num_draft_tokens_per_req = None
if not draft_tokens:
# No draft token scheduled (common case).
total_num_draft_tokens = 0
@@ -735,15 +756,15 @@ class GPUModelRunner(LoRAModelRunnerMixin):
num_reqs, dtype=torch.int32, device=self.device
)
else:
num_draft_tokens = np.fromiter(
num_draft_tokens_per_req = np.fromiter(
(len(draft_tokens.get(req_id, ())) for req_id in req_ids),
dtype=np.int32,
count=num_reqs,
)
total_num_draft_tokens = int(num_draft_tokens.sum())
total_num_draft_tokens = int(num_draft_tokens_per_req.sum())
total_num_logits = num_reqs + total_num_draft_tokens
num_logits = num_draft_tokens + 1
num_logits = num_draft_tokens_per_req + 1
cu_num_logits_np = np.empty(num_reqs + 1, dtype=np.int32)
cu_num_logits_np[0] = 0
np.cumsum(num_logits, out=cu_num_logits_np[1:])
@@ -766,9 +787,13 @@ class GPUModelRunner(LoRAModelRunnerMixin):
async_copy_to_gpu(query_start_loc_np, out=self.input_buffers.query_start_loc)
query_start_loc_np = query_start_loc_np[: num_reqs_padded + 1]
query_start_loc = self.input_buffers.query_start_loc[: num_reqs_padded + 1]
is_prefilling_np = (
self.req_states.num_computed_prefill_tokens[idx_mapping_np]
< self.req_states.prefill_len.np[idx_mapping_np]
)
# Get prefill tokens if any.
if self.req_states.any_prefills(idx_mapping_np):
if np.any(is_prefilling_np):
prepare_prefill_inputs(
self.input_buffers.input_ids,
self.req_states.next_prefill_tokens,
@@ -836,11 +861,13 @@ class GPUModelRunner(LoRAModelRunnerMixin):
num_tokens=num_tokens,
num_tokens_after_padding=num_tokens_after_padding,
num_draft_tokens=total_num_draft_tokens,
num_draft_tokens_per_req=num_draft_tokens_per_req,
query_start_loc=query_start_loc,
query_start_loc_np=query_start_loc_np,
seq_lens=seq_lens,
seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,
dcp_local_seq_lens=dcp_local_seq_lens,
is_prefilling_np=is_prefilling_np,
input_ids=self.input_buffers.input_ids[:num_tokens_after_padding],
positions=self.input_buffers.positions[:num_tokens_after_padding],
logits_indices=logits_indices,
@@ -958,6 +985,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
input_batch.num_scheduled_tokens
)
self.model_state.postprocess_state(input_batch, num_sampled)
@torch.inference_mode()
def execute_model(
self,
@@ -18,6 +18,11 @@ def init_model_state(
return WhisperModelState(vllm_config, model, encoder_cache, device)
if vllm_config.model_config.is_hybrid:
from vllm.v1.worker.gpu.model_states.mamba_hybrid import MambaHybridModelState
return MambaHybridModelState(vllm_config, model, encoder_cache, device)
from vllm.v1.worker.gpu.model_states.default import DefaultModelState
return DefaultModelState(vllm_config, model, encoder_cache, device)
@@ -194,5 +194,6 @@ class DefaultModelState(ModelState):
seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,
dcp_local_seq_lens=input_batch.dcp_local_seq_lens,
positions=input_batch.positions,
for_cudagraph_capture=for_capture,
)
return attn_metadata
@@ -17,6 +17,24 @@ from vllm.v1.worker.gpu.states import RequestState
from vllm.v1.worker.utils import AttentionGroup
class ModelSpecificAttnMetadata:
"""Base class for model-specific attention metadata."""
def get_extra_common_attn_kwargs(
self,
kv_cache_group_id: int,
num_reqs: int,
) -> dict[str, Any]:
return {}
def get_extra_attn_kwargs(
self,
attn_metadata_builder: Any,
num_reqs: int,
) -> dict[str, Any]:
return {}
class ModelState(ABC):
@abstractmethod
def __init__(
@@ -38,6 +56,13 @@ class ModelState(ABC):
def apply_staged_writes(self) -> None:
return None
def postprocess_state(
self,
input_batch: InputBatch,
num_sampled: torch.Tensor,
) -> None:
return None
@abstractmethod
def get_mm_embeddings(
self,
@@ -0,0 +1,150 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass
from typing import Any
import numpy as np
import torch
import torch.nn as nn
from vllm.config import VllmConfig
from vllm.config.compilation import CUDAGraphMode
from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadataBuilder
from vllm.v1.attention.backends.mamba2_attn import Mamba2AttentionMetadataBuilder
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.worker.gpu.attn_utils import build_attn_metadata
from vllm.v1.worker.gpu.input_batch import InputBatch
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
from vllm.v1.worker.gpu.model_states.default import DefaultModelState
from vllm.v1.worker.gpu.model_states.interface import ModelSpecificAttnMetadata
from vllm.v1.worker.utils import AttentionGroup
@dataclass
class MambaHybridAttnMetadata(ModelSpecificAttnMetadata):
is_prefilling: torch.Tensor
num_accepted_tokens: torch.Tensor | None = None
num_decode_draft_tokens_cpu: torch.Tensor | None = None
def get_extra_common_attn_kwargs(
self,
kv_cache_group_id: int,
num_reqs: int,
) -> dict[str, Any]:
return {"is_prefilling": self.is_prefilling[:num_reqs]}
def get_extra_attn_kwargs(
self,
attn_metadata_builder: Any,
num_reqs: int,
) -> dict[str, Any]:
if not isinstance(
attn_metadata_builder,
(Mamba2AttentionMetadataBuilder, GDNAttentionMetadataBuilder),
):
return {}
return {
"num_accepted_tokens": None
if self.num_accepted_tokens is None
else self.num_accepted_tokens[:num_reqs],
"num_decode_draft_tokens_cpu": None
if self.num_decode_draft_tokens_cpu is None
else self.num_decode_draft_tokens_cpu[:num_reqs],
}
class MambaHybridModelState(DefaultModelState):
"""Model state for hybrid attention + Mamba / linear-attention models."""
def __init__(
self,
vllm_config: VllmConfig,
model: nn.Module,
encoder_cache: EncoderCache | None,
device: torch.device,
) -> None:
super().__init__(vllm_config, model, encoder_cache, device)
self.num_accepted_tokens_gpu = torch.ones(
self.max_num_reqs, dtype=torch.int32, device=self.device
)
def prepare_attn(
self,
input_batch: InputBatch,
cudagraph_mode: CUDAGraphMode,
block_tables: tuple[torch.Tensor, ...],
slot_mappings: torch.Tensor,
attn_groups: list[list[AttentionGroup]],
kv_cache_config: KVCacheConfig,
for_capture: bool = False,
) -> dict[str, Any]:
if cudagraph_mode == CUDAGraphMode.FULL:
num_reqs = input_batch.num_reqs_after_padding
num_tokens = input_batch.num_tokens_after_padding
else:
num_reqs = input_batch.num_reqs
num_tokens = input_batch.num_tokens
query_start_loc_cpu = torch.from_numpy(input_batch.query_start_loc_np)
max_query_len = input_batch.num_scheduled_tokens.max().item()
is_prefilling = torch.zeros(num_reqs, dtype=torch.bool, device="cpu")
is_prefilling[: input_batch.num_reqs] = torch.from_numpy(
input_batch.is_prefilling_np
)
# During CUDAGraph capture, num_decode_draft_tokens_cpu and num_accepted_tokens
# are created by attn_metadata_builder.build_for_cudagraph_capture, so we only
# compute them during actual (non-capture) forward execution.
num_accepted_tokens = None
num_decode_draft_tokens_cpu = None
if not for_capture:
num_accepted_tokens = self.num_accepted_tokens_gpu.new_ones(num_reqs)
num_accepted_tokens[: input_batch.num_reqs] = self.num_accepted_tokens_gpu[
input_batch.idx_mapping
]
# GDN uses >= 0 to select spec-decode rows, so non-decode rows
# need the -1 sentinel rather than a raw zero draft count.
num_decode_draft_tokens_np = np.full(num_reqs, -1, dtype=np.int32)
if input_batch.num_draft_tokens_per_req is not None:
spec_decode_mask = (
input_batch.num_draft_tokens_per_req > 0
) & ~input_batch.is_prefilling_np
num_decode_draft_tokens_np[: input_batch.num_reqs] = np.where(
spec_decode_mask,
input_batch.num_draft_tokens_per_req,
-1,
)
num_decode_draft_tokens_cpu = torch.from_numpy(num_decode_draft_tokens_np)
mamba_attn_metadata = MambaHybridAttnMetadata(
is_prefilling=is_prefilling,
num_accepted_tokens=num_accepted_tokens,
num_decode_draft_tokens_cpu=num_decode_draft_tokens_cpu,
)
return build_attn_metadata(
attn_groups=attn_groups,
num_reqs=num_reqs,
num_tokens=num_tokens,
query_start_loc_gpu=input_batch.query_start_loc,
query_start_loc_cpu=query_start_loc_cpu,
max_query_len=max_query_len,
seq_lens=input_batch.seq_lens,
max_seq_len=self.max_model_len,
block_tables=block_tables,
slot_mappings=slot_mappings,
kv_cache_config=kv_cache_config,
dcp_local_seq_lens=input_batch.dcp_local_seq_lens,
model_specific_attn_metadata=mamba_attn_metadata,
for_cudagraph_capture=for_capture,
)
def postprocess_state(
self,
input_batch: InputBatch,
num_sampled: torch.Tensor,
) -> None:
# Chunked prefill does not sample a token, so num_sampled can be 0.
# Mamba treats num_accepted_tokens=1 as the neutral non-spec value.
self.num_accepted_tokens_gpu[input_batch.idx_mapping] = torch.clamp(
num_sampled, min=1
)
+28 -4
View File
@@ -1,5 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass
from typing import Any
import numpy as np
@@ -13,11 +14,33 @@ from vllm.v1.worker.gpu.attn_utils import build_attn_metadata
from vllm.v1.worker.gpu.input_batch import InputBatch
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
from vllm.v1.worker.gpu.mm.encoder_runner import EncoderRunner
from vllm.v1.worker.gpu.model_states.interface import ModelState
from vllm.v1.worker.gpu.model_states.interface import (
ModelSpecificAttnMetadata,
ModelState,
)
from vllm.v1.worker.gpu.states import RequestState
from vllm.v1.worker.utils import AttentionGroup
@dataclass
class WhisperAttnMetadata(ModelSpecificAttnMetadata):
encoder_seq_lens: dict[int, tuple[torch.Tensor, np.ndarray]]
def get_extra_common_attn_kwargs(
self,
kv_cache_group_id: int,
num_reqs: int,
) -> dict[str, Any]:
encoder_seq_lens = self.encoder_seq_lens.get(kv_cache_group_id)
if encoder_seq_lens is None:
return {}
encoder_seq_lens_gpu, encoder_seq_lens_cpu = encoder_seq_lens
return {
"encoder_seq_lens": encoder_seq_lens_gpu[:num_reqs],
"encoder_seq_lens_cpu": encoder_seq_lens_cpu[:num_reqs],
}
class WhisperModelState(ModelState):
def __init__(
self,
@@ -111,8 +134,8 @@ class WhisperModelState(ModelState):
else:
num_reqs = input_batch.num_reqs
num_tokens = input_batch.num_tokens
encoder_seq_lens = self._get_encoder_seq_lens(
input_batch.req_ids, attn_groups, for_capture
whisper_attn_metadata = WhisperAttnMetadata(
self._get_encoder_seq_lens(input_batch.req_ids, attn_groups, for_capture)
)
query_start_loc_cpu = torch.from_numpy(input_batch.query_start_loc_np)
@@ -136,7 +159,8 @@ class WhisperModelState(ModelState):
kv_cache_config=kv_cache_config,
seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,
dcp_local_seq_lens=input_batch.dcp_local_seq_lens,
encoder_seq_lens=encoder_seq_lens,
model_specific_attn_metadata=whisper_attn_metadata,
for_cudagraph_capture=for_capture,
)
return attn_metadata
@@ -106,6 +106,7 @@ class DecodeEagleCudaGraphManager(EagleCudaGraphManagerBase):
block_tables,
attn_groups,
kv_cache_config,
skip_attn=(desc.cg_mode == CUDAGraphMode.PIECEWISE),
)
attn_metadata, slot_mappings = attn_state