From 7a08b34fbf17a491ccf1f7e2cff3b8184a9e4454 Mon Sep 17 00:00:00 2001 From: zhrrr <43847754+izhuhaoran@users.noreply.github.com> Date: Fri, 8 May 2026 00:31:05 +0800 Subject: [PATCH] [Model Runner V2] support qwen35 / mamba hybrid model (#35520) Signed-off-by: zhuhaoran --- vllm/config/kernel.py | 2 +- vllm/config/vllm.py | 4 + vllm/multimodal/registry.py | 16 +- vllm/v1/worker/gpu/attn_utils.py | 197 ++++++++++++------ vllm/v1/worker/gpu/block_table.py | 16 +- vllm/v1/worker/gpu/cudagraph_utils.py | 40 ++-- vllm/v1/worker/gpu/input_batch.py | 7 + vllm/v1/worker/gpu/model_runner.py | 49 ++++- vllm/v1/worker/gpu/model_states/__init__.py | 5 + vllm/v1/worker/gpu/model_states/default.py | 1 + vllm/v1/worker/gpu/model_states/interface.py | 25 +++ .../worker/gpu/model_states/mamba_hybrid.py | 150 +++++++++++++ vllm/v1/worker/gpu/model_states/whisper.py | 32 ++- .../worker/gpu/spec_decode/eagle/cudagraph.py | 1 + 14 files changed, 439 insertions(+), 106 deletions(-) create mode 100644 vllm/v1/worker/gpu/model_states/mamba_hybrid.py diff --git a/vllm/config/kernel.py b/vllm/config/kernel.py index da1b1f9f1b..ec9c878525 100644 --- a/vllm/config/kernel.py +++ b/vllm/config/kernel.py @@ -142,7 +142,7 @@ class KernelConfig: """Backend for MoE expert computation kernels. Available options: - "auto": Automatically select the best backend based on model and hardware - - "triton": Use Triton-based fused MoE kernels + - "triton": Use Triton-based fused MoE kernels - "deep_gemm": Use DeepGEMM kernels (FP8 block-quantized only) - "deep_gemm_mega_moe": Use DeepGEMM mega MoE kernels - "cutlass": Use vLLM CUTLASS kernels diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index 52c04509b2..4bde11a798 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -1892,6 +1892,10 @@ class VllmConfig: "to schedule a multiple of block_size tokens even if they are " "in the middle of a mm input" ) + # TODO: support align mamba cache mode for model runner v2 + assert not envs.VLLM_USE_V2_MODEL_RUNNER, ( + "Model Runner V2 has not yet supported mamba_cache_mode='align'. " + ) @model_validator(mode="after") def validate_nvfp4_kv_cache_with_mla(self) -> "VllmConfig": diff --git a/vllm/multimodal/registry.py b/vllm/multimodal/registry.py index a2c8c24ec0..6fdae47083 100644 --- a/vllm/multimodal/registry.py +++ b/vllm/multimodal/registry.py @@ -111,7 +111,15 @@ class MultiModalRegistry: return False mm_config = model_config.get_multimodal_config() - info = self._create_processing_info(model_config, tokenizer=None) + try: + info = self._create_processing_info(model_config, tokenizer=None) + except ValueError: + logger.warning_once( + "Model %s is treated as multimodal but has no registered " + "multimodal processor; running in text-only mode.", + model_config.model, + ) + return False # Check if all supported modalities have limit == 0 if all( @@ -170,7 +178,11 @@ class MultiModalRegistry: from vllm.model_executor.model_loader import get_model_architecture model_cls, _ = get_model_architecture(model_config) - assert hasattr(model_cls, "_processor_factory") + if not hasattr(model_cls, "_processor_factory"): + raise ValueError( + f"Model class {model_cls.__name__} has no registered " + "multimodal processor" + ) return cast("SupportsMultiModal", model_cls) def _create_processing_ctx( diff --git a/vllm/v1/worker/gpu/attn_utils.py b/vllm/v1/worker/gpu/attn_utils.py index 2262575812..5930777de6 100644 --- a/vllm/v1/worker/gpu/attn_utils.py +++ b/vllm/v1/worker/gpu/attn_utils.py @@ -4,7 +4,6 @@ from collections.abc import Sequence from dataclasses import dataclass from typing import Any, cast -import numpy as np import torch from vllm.config import VllmConfig, get_layers_from_vllm_config @@ -19,8 +18,10 @@ from vllm.v1.kv_cache_interface import ( AttentionSpec, KVCacheConfig, KVCacheSpec, + MambaSpec, UniformTypeKVCacheSpecs, ) +from vllm.v1.worker.gpu.model_states.interface import ModelSpecificAttnMetadata from vllm.v1.worker.utils import AttentionGroup, bind_kv_cache @@ -147,68 +148,126 @@ def _reshape_kv_cache( kv_cache_raw_tensors: dict[str, torch.Tensor], attn_backends: dict[str, type[AttentionBackend]], cache_dtype: str, -) -> dict[str, torch.Tensor]: - kv_caches: dict[str, torch.Tensor] = {} +) -> dict[str, Any]: + kv_caches: dict[str, Any] = {} + has_attn, has_mamba = False, False for kv_cache_group_spec in kv_cache_config.kv_cache_groups: for layer_name in kv_cache_group_spec.layer_names: kv_cache_spec = kv_cache_group_spec.kv_cache_spec if isinstance(kv_cache_spec, UniformTypeKVCacheSpecs): kv_cache_spec = kv_cache_spec.kv_cache_specs[layer_name] - assert isinstance(kv_cache_spec, AttentionSpec) - raw_tensor = kv_cache_raw_tensors[layer_name] - assert raw_tensor.numel() % kv_cache_spec.page_size_bytes == 0 - num_blocks = raw_tensor.numel() // kv_cache_spec.page_size_bytes + kv_raw_tensor = kv_cache_raw_tensors[layer_name] + assert kv_raw_tensor.numel() % kv_cache_spec.page_size_bytes == 0 + num_blocks = kv_raw_tensor.numel() // kv_cache_spec.page_size_bytes - attn_backend = attn_backends[layer_name] - kv_cache_shape = attn_backend.get_kv_cache_shape( - num_blocks, - kv_cache_spec.storage_block_size, - kv_cache_spec.num_kv_heads, - kv_cache_spec.head_size, - cache_dtype, - ) - - # FIXME(woosuk): Add kv_cache_stride_order to all attention backends. - try: - kv_cache_stride_order = attn_backend.get_kv_cache_stride_order() - assert len(kv_cache_stride_order) == len(kv_cache_shape) - except (AttributeError, NotImplementedError): - kv_cache_stride_order = tuple(range(len(kv_cache_shape))) - - kv_cache_shape = tuple(kv_cache_shape[i] for i in kv_cache_stride_order) - inv_order = [ - kv_cache_stride_order.index(i) - for i in range(len(kv_cache_stride_order)) - ] - - dtype = kv_cache_spec.dtype - raw_tensor = raw_tensor.view(dtype) - if kv_cache_spec.page_size_padded is not None: - # Use strided view to handle page_size_bytes that - # include padding. This follows the same pattern as - # MambaSpec handling in gpu_model_runner.py. - # NOTE: This assumes kv_cache_shape[0] == num_blocks - # (i.e. the first physical dimension is the block - # index), which holds for MLA backends but NOT for - # standard attention backends whose shape starts with - # a K/V dimension of size 2. - dtype_size = get_dtype_size(dtype) - page_stride = kv_cache_spec.page_size_bytes // dtype_size - strides = list(torch.empty(kv_cache_shape).stride()) - strides[inv_order[0]] = page_stride - kv_cache = torch.as_strided( - raw_tensor, - size=kv_cache_shape, - stride=tuple(strides), + if isinstance(kv_cache_spec, AttentionSpec): + has_attn = True + attn_backend = attn_backends[layer_name] + kv_cache_shape = attn_backend.get_kv_cache_shape( + num_blocks, + kv_cache_spec.storage_block_size, + kv_cache_spec.num_kv_heads, + kv_cache_spec.head_size, + cache_dtype_str=cache_dtype, ) + + # FIXME(woosuk): Add kv_cache_stride_order to all attention backends. + try: + kv_cache_stride_order = attn_backend.get_kv_cache_stride_order() + assert len(kv_cache_stride_order) == len(kv_cache_shape) + except (AttributeError, NotImplementedError): + kv_cache_stride_order = tuple(range(len(kv_cache_shape))) + + kv_cache_shape = tuple(kv_cache_shape[i] for i in kv_cache_stride_order) + inv_order = [ + kv_cache_stride_order.index(i) + for i in range(len(kv_cache_stride_order)) + ] + + dtype = kv_cache_spec.dtype + kv_tensor = kv_raw_tensor.view(dtype) + if kv_cache_spec.page_size_padded is not None: + # Use strided view to handle page_size_bytes that + # include padding. This follows the same pattern as + # MambaSpec handling in gpu_model_runner.py. + # NOTE: This assumes kv_cache_shape[0] == num_blocks + # (i.e. the first physical dimension is the block + # index), which holds for MLA backends but NOT for + # standard attention backends whose shape starts with + # a K/V dimension of size 2. + dtype_size = get_dtype_size(dtype) + page_stride = kv_cache_spec.page_size_bytes // dtype_size + strides = list(torch.empty(kv_cache_shape).stride()) + strides[inv_order[0]] = page_stride + kv_cache = torch.as_strided( + kv_tensor, + size=kv_cache_shape, + stride=tuple(strides), + ) + else: + # No padding — safe to use a contiguous view. + kv_cache = kv_tensor.view(kv_cache_shape) + kv_caches[layer_name] = kv_cache.permute(*inv_order) + + elif isinstance(kv_cache_spec, MambaSpec): + has_mamba = True + state_tensors = [] + storage_offset_bytes = 0 + for shape, dtype in zip(kv_cache_spec.shapes, kv_cache_spec.dtypes): + dtype_size = get_dtype_size(dtype) + num_element_per_page = kv_cache_spec.page_size_bytes // dtype_size + target_shape = (num_blocks, *shape) + stride = torch.empty(target_shape).stride() + target_stride = (num_element_per_page, *stride[1:]) + assert storage_offset_bytes % dtype_size == 0 + tensor = torch.as_strided( + kv_raw_tensor.view(dtype), + size=target_shape, + stride=target_stride, + storage_offset=storage_offset_bytes // dtype_size, + ) + state_tensors.append(tensor) + storage_offset_bytes += stride[0] * dtype_size + kv_caches[layer_name] = state_tensors else: - # No padding — safe to use a contiguous view. - kv_cache = raw_tensor.view(kv_cache_shape) - kv_caches[layer_name] = kv_cache.permute(*inv_order) + raise NotImplementedError( + f"Unsupported KV cache spec type: {type(kv_cache_spec)}" + ) + + if has_attn and has_mamba: + _update_hybrid_attention_layout(kv_caches, kv_cache_config) + return kv_caches +def _update_hybrid_attention_layout( + kv_caches: dict[str, Any], + kv_cache_config: KVCacheConfig, +) -> None: + for kv_cache_group_spec in kv_cache_config.kv_cache_groups: + for layer_name in kv_cache_group_spec.layer_names: + kv_cache_spec = kv_cache_group_spec.kv_cache_spec + if isinstance(kv_cache_spec, UniformTypeKVCacheSpecs): + kv_cache_spec = kv_cache_spec.kv_cache_specs[layer_name] + if not isinstance(kv_cache_spec, AttentionSpec): + continue + kv_cache = kv_caches[layer_name] + if kv_cache.shape[0] == 2: + assert kv_cache.shape[1] != 2, ( + f"Cannot determine layout for tensor of shape {kv_cache.shape}" + ) + hidden_size = kv_cache.shape[2:].numel() + kv_cache.as_strided_( + size=kv_cache.shape, + stride=( + hidden_size, + 2 * hidden_size, + *kv_cache.stride()[2:], + ), + ) + + def init_kv_cache( runner_kv_caches: list[torch.Tensor], forward_context: dict[str, Any], @@ -216,7 +275,7 @@ def init_kv_cache( attn_backends: dict[str, type[AttentionBackend]], device: torch.device, cache_dtype: str, -) -> dict[str, torch.Tensor]: +) -> dict[str, Any]: kv_cache_raw_tensors = _allocate_kv_cache(kv_cache_config, device) kv_caches = _reshape_kv_cache( kv_cache_config, kv_cache_raw_tensors, attn_backends, cache_dtype @@ -250,8 +309,9 @@ def build_attn_metadata( kv_cache_config: KVCacheConfig, seq_lens_cpu_upper_bound: torch.Tensor | None = None, dcp_local_seq_lens: torch.Tensor | None = None, - encoder_seq_lens: dict[int, tuple[torch.Tensor, np.ndarray]] | None = None, positions: torch.Tensor | None = None, + model_specific_attn_metadata: ModelSpecificAttnMetadata | None = None, + for_cudagraph_capture: bool = False, ) -> dict[str, Any]: seq_lens = seq_lens[:num_reqs] if dcp_local_seq_lens is not None: @@ -265,6 +325,11 @@ def build_attn_metadata( block_table = block_tables[i] slot_mapping = slot_mappings[i] + common_attn_metadata_extra_kwargs = ( + model_specific_attn_metadata.get_extra_common_attn_kwargs(i, num_reqs) + if model_specific_attn_metadata is not None + else {} + ) common_attn_metadata = CommonAttentionMetadata( query_start_loc=query_start_loc_gpu, query_start_loc_cpu=query_start_loc_cpu, @@ -279,17 +344,29 @@ def build_attn_metadata( causal=True, dcp_local_seq_lens=dcp_local_seq_lens, positions=positions, + **common_attn_metadata_extra_kwargs, ) - if encoder_seq_lens and i in encoder_seq_lens: - encoder_seq_lens_gpu, encoder_seq_lens_cpu = encoder_seq_lens[i] - common_attn_metadata.encoder_seq_lens = encoder_seq_lens_gpu - common_attn_metadata.encoder_seq_lens_cpu = encoder_seq_lens_cpu for attn_group in attn_groups[i]: attn_metadata_builder = attn_group.get_metadata_builder(0) - metadata = attn_metadata_builder.build( - common_prefix_len=0, common_attn_metadata=common_attn_metadata - ) + if for_cudagraph_capture: + metadata = attn_metadata_builder.build_for_cudagraph_capture( + common_attn_metadata + ) + else: + attn_metadata_extra_kwargs = ( + model_specific_attn_metadata.get_extra_attn_kwargs( + attn_metadata_builder, + num_reqs, + ) + if model_specific_attn_metadata is not None + else {} + ) + metadata = attn_metadata_builder.build( + common_prefix_len=0, + common_attn_metadata=common_attn_metadata, + **attn_metadata_extra_kwargs, + ) for layer_name in attn_group.layer_names: attn_metadata[layer_name] = metadata return attn_metadata diff --git a/vllm/v1/worker/gpu/block_table.py b/vllm/v1/worker/gpu/block_table.py index 3061278b01..62454f7030 100644 --- a/vllm/v1/worker/gpu/block_table.py +++ b/vllm/v1/worker/gpu/block_table.py @@ -5,7 +5,6 @@ from collections.abc import Iterable import torch from vllm.triton_utils import tl, triton -from vllm.utils.math_utils import cdiv from vllm.v1.attention.backends.utils import PAD_SLOT_ID from vllm.v1.worker.gpu.buffer_utils import StagedWriteTensor, UvaBackedTensor @@ -16,7 +15,7 @@ class BlockTables: block_sizes: list[int], max_num_reqs: int, max_num_batched_tokens: int, - max_model_len: int, + max_num_blocks_per_group: list[int], device: torch.device, cp_size: int = 1, cp_rank: int = 0, @@ -25,7 +24,6 @@ class BlockTables: self.block_sizes = block_sizes self.max_num_reqs = max_num_reqs self.max_num_batched_tokens = max_num_batched_tokens - self.max_model_len = max_model_len self.device = device self.cp_size = cp_size @@ -33,19 +31,11 @@ class BlockTables: self.cp_interleave = cp_interleave self.num_kv_cache_groups = len(self.block_sizes) + assert len(max_num_blocks_per_group) == self.num_kv_cache_groups # num_kv_cache_groups x [max_num_reqs, max_num_blocks] self.block_tables: list[StagedWriteTensor] = [] for i in range(self.num_kv_cache_groups): - block_size = self.block_sizes[i] - # When using DCP, each request's KV cache is sharded among different ranks. - # As a result, one block on the current rank covers `block_size * cp_size` - # tokens in the full, global (unsharded) sequence. - max_num_blocks = cdiv(self.max_model_len, block_size * self.cp_size) - # Align to a multiple of (128 / block_size) as required - # by some attention backends such as TRTLLM (#39324) - if block_size <= 128: - alignment = 128 // block_size - max_num_blocks = cdiv(max_num_blocks, alignment) * alignment + max_num_blocks = max_num_blocks_per_group[i] block_table = StagedWriteTensor( (self.max_num_reqs, max_num_blocks), dtype=torch.int32, diff --git a/vllm/v1/worker/gpu/cudagraph_utils.py b/vllm/v1/worker/gpu/cudagraph_utils.py index 662f92e581..71a15c3bd9 100644 --- a/vllm/v1/worker/gpu/cudagraph_utils.py +++ b/vllm/v1/worker/gpu/cudagraph_utils.py @@ -34,7 +34,7 @@ logger = init_logger(__name__) class CapturedAttentionState(NamedTuple): - attn_metadata: dict[str, Any] + attn_metadata: dict[str, Any] | None slot_mappings: dict[str, torch.Tensor] @@ -100,6 +100,7 @@ class CudaGraphManager: self.decode_query_len = decode_query_len self.dp_size = vllm_config.parallel_config.data_parallel_size + self.tp_size = vllm_config.parallel_config.tensor_parallel_size self.is_first_pp_rank = get_pp_group().is_first_rank self.is_last_pp_rank = get_pp_group().is_last_rank @@ -109,6 +110,10 @@ class CudaGraphManager: self._graphs_captured = False self._candidates: list[list[BatchExecutionDescriptor]] = [] self._capture_descs: dict[CUDAGraphMode, list[BatchExecutionDescriptor]] = {} + # adjust the cudagraph sizes to be a multiple of the uniform decode query length + self.compilation_config.adjust_cudagraph_sizes_for_spec_decode( + self.decode_query_len, self.tp_size + ) self._init_candidates() def _init_candidates(self) -> None: @@ -340,16 +345,16 @@ class ModelCudaGraphManager(CudaGraphManager): block_tables, attn_groups, kv_cache_config, + skip_attn=(desc.cg_mode == CUDAGraphMode.PIECEWISE), ) def forward_fn(cg_mode: CUDAGraphMode) -> None: - batch_descriptor = ( - BatchDescriptor(num_tokens=num_tokens) - if cg_mode == CUDAGraphMode.PIECEWISE - else None - ) + batch_descriptor = None + if cg_mode == CUDAGraphMode.PIECEWISE: + assert attn_metadata is None + batch_descriptor = BatchDescriptor(num_tokens=num_tokens) with set_forward_context( - attn_metadata if cg_mode != CUDAGraphMode.PIECEWISE else None, + attn_metadata, self.vllm_config, num_tokens=num_tokens, cudagraph_runtime_mode=cg_mode, @@ -419,6 +424,7 @@ def prepare_inputs_to_capture( block_tables: BlockTables, attn_groups: list[list[AttentionGroup]], kv_cache_config: KVCacheConfig, + skip_attn: bool = False, ) -> CapturedAttentionState: input_batch = InputBatch.make_dummy(num_reqs, num_tokens, input_buffers) input_block_tables = block_tables.get_dummy_block_tables(num_reqs) @@ -439,13 +445,15 @@ def prepare_inputs_to_capture( ) input_batch.dcp_local_seq_lens = input_buffers.dcp_local_seq_lens[:num_reqs] - attn_metadata = model_state.prepare_attn( - input_batch, - CUDAGraphMode.NONE, - input_block_tables, - slot_mappings, - attn_groups, - kv_cache_config, - for_capture=True, - ) + attn_metadata = None + if not skip_attn: + attn_metadata = model_state.prepare_attn( + input_batch, + CUDAGraphMode.NONE, + input_block_tables, + slot_mappings, + attn_groups, + kv_cache_config, + for_capture=True, + ) return CapturedAttentionState(attn_metadata, slot_mappings_by_layer) diff --git a/vllm/v1/worker/gpu/input_batch.py b/vllm/v1/worker/gpu/input_batch.py index be14de272a..b253d7d8c0 100644 --- a/vllm/v1/worker/gpu/input_batch.py +++ b/vllm/v1/worker/gpu/input_batch.py @@ -53,7 +53,10 @@ class InputBatch: # sum(num_scheduled_tokens) num_tokens: int num_tokens_after_padding: int + # Sum of draft tokens scheduled across requests. num_draft_tokens: int + # [num_reqs] number of draft tokens scheduled for each request, if any. + num_draft_tokens_per_req: np.ndarray | None # [num_reqs + 1] query_start_loc: torch.Tensor @@ -64,6 +67,8 @@ class InputBatch: seq_lens_cpu_upper_bound: torch.Tensor # [num_reqs] dcp_local_seq_lens: torch.Tensor | None + # [num_reqs] CPU bool array. + is_prefilling_np: np.ndarray # [num_tokens_after_padding] input_ids: torch.Tensor @@ -137,11 +142,13 @@ class InputBatch: num_tokens=num_tokens, num_tokens_after_padding=num_tokens, num_draft_tokens=0, + num_draft_tokens_per_req=None, query_start_loc=query_start_loc, query_start_loc_np=query_start_loc_np, seq_lens=seq_lens, seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound, dcp_local_seq_lens=None, + is_prefilling_np=np.zeros(num_reqs, dtype=np.bool_), input_ids=input_ids, positions=positions, logits_indices=logits_indices, diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index bf882a8af3..b81bd0dc59 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -44,10 +44,11 @@ from vllm.model_executor.model_loader import get_model_loader from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.sequence import IntermediateTensors from vllm.tasks import SupportedTask +from vllm.utils.math_utils import cdiv from vllm.utils.mem_utils import DeviceMemoryProfiler, format_gib from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput -from vllm.v1.kv_cache_interface import KVCacheConfig +from vllm.v1.kv_cache_interface import KVCacheConfig, MambaSpec from vllm.v1.outputs import DraftTokenIds, KVConnectorOutput, ModelRunnerOutput from vllm.v1.worker.cp_utils import check_attention_cp_compatibility from vllm.v1.worker.gpu.async_utils import AsyncOutput, AsyncPoolingOutput @@ -338,10 +339,6 @@ class GPUModelRunner(LoRAModelRunnerMixin): def initialize_kv_cache(self, kv_cache_config: KVCacheConfig) -> None: kv_cache_config = deepcopy(kv_cache_config) self.kv_cache_config = kv_cache_config - block_sizes = [ - kv_cache_group.kv_cache_spec.block_size - for kv_cache_group in kv_cache_config.kv_cache_groups - ] block_table_max_model_len = self.max_model_len if self.is_encoder_decoder: @@ -352,11 +349,34 @@ class GPUModelRunner(LoRAModelRunnerMixin): getattr(self.model_config.hf_config, "max_source_positions", 0), ) + block_sizes = [] + max_num_blocks_per_group = [] + for kv_cache_group in kv_cache_config.kv_cache_groups: + spec = kv_cache_group.kv_cache_spec + block_sizes.append(spec.block_size) + # When using DCP, each request's KV cache is sharded among different ranks. + # As a result, one block on the current rank covers `block_size * cp_size` + # tokens in the full, global (unsharded) sequence. + max_num_blocks = cdiv( + block_table_max_model_len, spec.block_size * self.dcp_size + ) + # Align to a multiple of (128 / block_size) as required by some attention + # backends such as TRTLLM (#39324) + if spec.block_size <= 128: + alignment = 128 // spec.block_size + max_num_blocks = cdiv(max_num_blocks, alignment) * alignment + # For Mamba/Hybrid Model, KVCaches need extra blocks for speculative tokens + if isinstance(spec, MambaSpec): + max_num_blocks = ( + max_num_blocks if self.cache_config.enable_prefix_caching else 1 + ) + spec.num_speculative_blocks + max_num_blocks_per_group.append(max_num_blocks) + self.block_tables = BlockTables( block_sizes=block_sizes, max_num_reqs=self.max_num_reqs, max_num_batched_tokens=self.max_num_tokens, - max_model_len=block_table_max_model_len, + max_num_blocks_per_group=max_num_blocks_per_group, device=self.device, cp_size=self.dcp_size, cp_rank=self.dcp_rank, @@ -722,6 +742,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): # Get the number of draft tokens for each request. draft_tokens = scheduler_output.scheduled_spec_decode_tokens + num_draft_tokens_per_req = None if not draft_tokens: # No draft token scheduled (common case). total_num_draft_tokens = 0 @@ -735,15 +756,15 @@ class GPUModelRunner(LoRAModelRunnerMixin): num_reqs, dtype=torch.int32, device=self.device ) else: - num_draft_tokens = np.fromiter( + num_draft_tokens_per_req = np.fromiter( (len(draft_tokens.get(req_id, ())) for req_id in req_ids), dtype=np.int32, count=num_reqs, ) - total_num_draft_tokens = int(num_draft_tokens.sum()) + total_num_draft_tokens = int(num_draft_tokens_per_req.sum()) total_num_logits = num_reqs + total_num_draft_tokens - num_logits = num_draft_tokens + 1 + num_logits = num_draft_tokens_per_req + 1 cu_num_logits_np = np.empty(num_reqs + 1, dtype=np.int32) cu_num_logits_np[0] = 0 np.cumsum(num_logits, out=cu_num_logits_np[1:]) @@ -766,9 +787,13 @@ class GPUModelRunner(LoRAModelRunnerMixin): async_copy_to_gpu(query_start_loc_np, out=self.input_buffers.query_start_loc) query_start_loc_np = query_start_loc_np[: num_reqs_padded + 1] query_start_loc = self.input_buffers.query_start_loc[: num_reqs_padded + 1] + is_prefilling_np = ( + self.req_states.num_computed_prefill_tokens[idx_mapping_np] + < self.req_states.prefill_len.np[idx_mapping_np] + ) # Get prefill tokens if any. - if self.req_states.any_prefills(idx_mapping_np): + if np.any(is_prefilling_np): prepare_prefill_inputs( self.input_buffers.input_ids, self.req_states.next_prefill_tokens, @@ -836,11 +861,13 @@ class GPUModelRunner(LoRAModelRunnerMixin): num_tokens=num_tokens, num_tokens_after_padding=num_tokens_after_padding, num_draft_tokens=total_num_draft_tokens, + num_draft_tokens_per_req=num_draft_tokens_per_req, query_start_loc=query_start_loc, query_start_loc_np=query_start_loc_np, seq_lens=seq_lens, seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound, dcp_local_seq_lens=dcp_local_seq_lens, + is_prefilling_np=is_prefilling_np, input_ids=self.input_buffers.input_ids[:num_tokens_after_padding], positions=self.input_buffers.positions[:num_tokens_after_padding], logits_indices=logits_indices, @@ -958,6 +985,8 @@ class GPUModelRunner(LoRAModelRunnerMixin): input_batch.num_scheduled_tokens ) + self.model_state.postprocess_state(input_batch, num_sampled) + @torch.inference_mode() def execute_model( self, diff --git a/vllm/v1/worker/gpu/model_states/__init__.py b/vllm/v1/worker/gpu/model_states/__init__.py index 6514525533..06b5a92c39 100644 --- a/vllm/v1/worker/gpu/model_states/__init__.py +++ b/vllm/v1/worker/gpu/model_states/__init__.py @@ -18,6 +18,11 @@ def init_model_state( return WhisperModelState(vllm_config, model, encoder_cache, device) + if vllm_config.model_config.is_hybrid: + from vllm.v1.worker.gpu.model_states.mamba_hybrid import MambaHybridModelState + + return MambaHybridModelState(vllm_config, model, encoder_cache, device) + from vllm.v1.worker.gpu.model_states.default import DefaultModelState return DefaultModelState(vllm_config, model, encoder_cache, device) diff --git a/vllm/v1/worker/gpu/model_states/default.py b/vllm/v1/worker/gpu/model_states/default.py index 1b8ee066ee..7f7955a58a 100644 --- a/vllm/v1/worker/gpu/model_states/default.py +++ b/vllm/v1/worker/gpu/model_states/default.py @@ -194,5 +194,6 @@ class DefaultModelState(ModelState): seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound, dcp_local_seq_lens=input_batch.dcp_local_seq_lens, positions=input_batch.positions, + for_cudagraph_capture=for_capture, ) return attn_metadata diff --git a/vllm/v1/worker/gpu/model_states/interface.py b/vllm/v1/worker/gpu/model_states/interface.py index d83ab2fc51..721e5c2013 100644 --- a/vllm/v1/worker/gpu/model_states/interface.py +++ b/vllm/v1/worker/gpu/model_states/interface.py @@ -17,6 +17,24 @@ from vllm.v1.worker.gpu.states import RequestState from vllm.v1.worker.utils import AttentionGroup +class ModelSpecificAttnMetadata: + """Base class for model-specific attention metadata.""" + + def get_extra_common_attn_kwargs( + self, + kv_cache_group_id: int, + num_reqs: int, + ) -> dict[str, Any]: + return {} + + def get_extra_attn_kwargs( + self, + attn_metadata_builder: Any, + num_reqs: int, + ) -> dict[str, Any]: + return {} + + class ModelState(ABC): @abstractmethod def __init__( @@ -38,6 +56,13 @@ class ModelState(ABC): def apply_staged_writes(self) -> None: return None + def postprocess_state( + self, + input_batch: InputBatch, + num_sampled: torch.Tensor, + ) -> None: + return None + @abstractmethod def get_mm_embeddings( self, diff --git a/vllm/v1/worker/gpu/model_states/mamba_hybrid.py b/vllm/v1/worker/gpu/model_states/mamba_hybrid.py new file mode 100644 index 0000000000..93115fdf64 --- /dev/null +++ b/vllm/v1/worker/gpu/model_states/mamba_hybrid.py @@ -0,0 +1,150 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from dataclasses import dataclass +from typing import Any + +import numpy as np +import torch +import torch.nn as nn + +from vllm.config import VllmConfig +from vllm.config.compilation import CUDAGraphMode +from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadataBuilder +from vllm.v1.attention.backends.mamba2_attn import Mamba2AttentionMetadataBuilder +from vllm.v1.kv_cache_interface import KVCacheConfig +from vllm.v1.worker.gpu.attn_utils import build_attn_metadata +from vllm.v1.worker.gpu.input_batch import InputBatch +from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache +from vllm.v1.worker.gpu.model_states.default import DefaultModelState +from vllm.v1.worker.gpu.model_states.interface import ModelSpecificAttnMetadata +from vllm.v1.worker.utils import AttentionGroup + + +@dataclass +class MambaHybridAttnMetadata(ModelSpecificAttnMetadata): + is_prefilling: torch.Tensor + num_accepted_tokens: torch.Tensor | None = None + num_decode_draft_tokens_cpu: torch.Tensor | None = None + + def get_extra_common_attn_kwargs( + self, + kv_cache_group_id: int, + num_reqs: int, + ) -> dict[str, Any]: + return {"is_prefilling": self.is_prefilling[:num_reqs]} + + def get_extra_attn_kwargs( + self, + attn_metadata_builder: Any, + num_reqs: int, + ) -> dict[str, Any]: + if not isinstance( + attn_metadata_builder, + (Mamba2AttentionMetadataBuilder, GDNAttentionMetadataBuilder), + ): + return {} + return { + "num_accepted_tokens": None + if self.num_accepted_tokens is None + else self.num_accepted_tokens[:num_reqs], + "num_decode_draft_tokens_cpu": None + if self.num_decode_draft_tokens_cpu is None + else self.num_decode_draft_tokens_cpu[:num_reqs], + } + + +class MambaHybridModelState(DefaultModelState): + """Model state for hybrid attention + Mamba / linear-attention models.""" + + def __init__( + self, + vllm_config: VllmConfig, + model: nn.Module, + encoder_cache: EncoderCache | None, + device: torch.device, + ) -> None: + super().__init__(vllm_config, model, encoder_cache, device) + self.num_accepted_tokens_gpu = torch.ones( + self.max_num_reqs, dtype=torch.int32, device=self.device + ) + + def prepare_attn( + self, + input_batch: InputBatch, + cudagraph_mode: CUDAGraphMode, + block_tables: tuple[torch.Tensor, ...], + slot_mappings: torch.Tensor, + attn_groups: list[list[AttentionGroup]], + kv_cache_config: KVCacheConfig, + for_capture: bool = False, + ) -> dict[str, Any]: + if cudagraph_mode == CUDAGraphMode.FULL: + num_reqs = input_batch.num_reqs_after_padding + num_tokens = input_batch.num_tokens_after_padding + else: + num_reqs = input_batch.num_reqs + num_tokens = input_batch.num_tokens + query_start_loc_cpu = torch.from_numpy(input_batch.query_start_loc_np) + max_query_len = input_batch.num_scheduled_tokens.max().item() + + is_prefilling = torch.zeros(num_reqs, dtype=torch.bool, device="cpu") + is_prefilling[: input_batch.num_reqs] = torch.from_numpy( + input_batch.is_prefilling_np + ) + # During CUDAGraph capture, num_decode_draft_tokens_cpu and num_accepted_tokens + # are created by attn_metadata_builder.build_for_cudagraph_capture, so we only + # compute them during actual (non-capture) forward execution. + num_accepted_tokens = None + num_decode_draft_tokens_cpu = None + if not for_capture: + num_accepted_tokens = self.num_accepted_tokens_gpu.new_ones(num_reqs) + num_accepted_tokens[: input_batch.num_reqs] = self.num_accepted_tokens_gpu[ + input_batch.idx_mapping + ] + + # GDN uses >= 0 to select spec-decode rows, so non-decode rows + # need the -1 sentinel rather than a raw zero draft count. + num_decode_draft_tokens_np = np.full(num_reqs, -1, dtype=np.int32) + if input_batch.num_draft_tokens_per_req is not None: + spec_decode_mask = ( + input_batch.num_draft_tokens_per_req > 0 + ) & ~input_batch.is_prefilling_np + num_decode_draft_tokens_np[: input_batch.num_reqs] = np.where( + spec_decode_mask, + input_batch.num_draft_tokens_per_req, + -1, + ) + num_decode_draft_tokens_cpu = torch.from_numpy(num_decode_draft_tokens_np) + + mamba_attn_metadata = MambaHybridAttnMetadata( + is_prefilling=is_prefilling, + num_accepted_tokens=num_accepted_tokens, + num_decode_draft_tokens_cpu=num_decode_draft_tokens_cpu, + ) + return build_attn_metadata( + attn_groups=attn_groups, + num_reqs=num_reqs, + num_tokens=num_tokens, + query_start_loc_gpu=input_batch.query_start_loc, + query_start_loc_cpu=query_start_loc_cpu, + max_query_len=max_query_len, + seq_lens=input_batch.seq_lens, + max_seq_len=self.max_model_len, + block_tables=block_tables, + slot_mappings=slot_mappings, + kv_cache_config=kv_cache_config, + dcp_local_seq_lens=input_batch.dcp_local_seq_lens, + model_specific_attn_metadata=mamba_attn_metadata, + for_cudagraph_capture=for_capture, + ) + + def postprocess_state( + self, + input_batch: InputBatch, + num_sampled: torch.Tensor, + ) -> None: + # Chunked prefill does not sample a token, so num_sampled can be 0. + # Mamba treats num_accepted_tokens=1 as the neutral non-spec value. + self.num_accepted_tokens_gpu[input_batch.idx_mapping] = torch.clamp( + num_sampled, min=1 + ) diff --git a/vllm/v1/worker/gpu/model_states/whisper.py b/vllm/v1/worker/gpu/model_states/whisper.py index a6faea482c..0ef3cadc87 100644 --- a/vllm/v1/worker/gpu/model_states/whisper.py +++ b/vllm/v1/worker/gpu/model_states/whisper.py @@ -1,5 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from dataclasses import dataclass from typing import Any import numpy as np @@ -13,11 +14,33 @@ from vllm.v1.worker.gpu.attn_utils import build_attn_metadata from vllm.v1.worker.gpu.input_batch import InputBatch from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache from vllm.v1.worker.gpu.mm.encoder_runner import EncoderRunner -from vllm.v1.worker.gpu.model_states.interface import ModelState +from vllm.v1.worker.gpu.model_states.interface import ( + ModelSpecificAttnMetadata, + ModelState, +) from vllm.v1.worker.gpu.states import RequestState from vllm.v1.worker.utils import AttentionGroup +@dataclass +class WhisperAttnMetadata(ModelSpecificAttnMetadata): + encoder_seq_lens: dict[int, tuple[torch.Tensor, np.ndarray]] + + def get_extra_common_attn_kwargs( + self, + kv_cache_group_id: int, + num_reqs: int, + ) -> dict[str, Any]: + encoder_seq_lens = self.encoder_seq_lens.get(kv_cache_group_id) + if encoder_seq_lens is None: + return {} + encoder_seq_lens_gpu, encoder_seq_lens_cpu = encoder_seq_lens + return { + "encoder_seq_lens": encoder_seq_lens_gpu[:num_reqs], + "encoder_seq_lens_cpu": encoder_seq_lens_cpu[:num_reqs], + } + + class WhisperModelState(ModelState): def __init__( self, @@ -111,8 +134,8 @@ class WhisperModelState(ModelState): else: num_reqs = input_batch.num_reqs num_tokens = input_batch.num_tokens - encoder_seq_lens = self._get_encoder_seq_lens( - input_batch.req_ids, attn_groups, for_capture + whisper_attn_metadata = WhisperAttnMetadata( + self._get_encoder_seq_lens(input_batch.req_ids, attn_groups, for_capture) ) query_start_loc_cpu = torch.from_numpy(input_batch.query_start_loc_np) @@ -136,7 +159,8 @@ class WhisperModelState(ModelState): kv_cache_config=kv_cache_config, seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound, dcp_local_seq_lens=input_batch.dcp_local_seq_lens, - encoder_seq_lens=encoder_seq_lens, + model_specific_attn_metadata=whisper_attn_metadata, + for_cudagraph_capture=for_capture, ) return attn_metadata diff --git a/vllm/v1/worker/gpu/spec_decode/eagle/cudagraph.py b/vllm/v1/worker/gpu/spec_decode/eagle/cudagraph.py index 8a29ccb398..43bece01d0 100644 --- a/vllm/v1/worker/gpu/spec_decode/eagle/cudagraph.py +++ b/vllm/v1/worker/gpu/spec_decode/eagle/cudagraph.py @@ -106,6 +106,7 @@ class DecodeEagleCudaGraphManager(EagleCudaGraphManagerBase): block_tables, attn_groups, kv_cache_config, + skip_attn=(desc.cg_mode == CUDAGraphMode.PIECEWISE), ) attn_metadata, slot_mappings = attn_state