[Model Runner V2] support qwen35 / mamba hybrid model (#35520)
Signed-off-by: zhuhaoran <zhuhaoran.zhr@alibaba-inc.com>
This commit is contained in:
@@ -142,7 +142,7 @@ class KernelConfig:
|
||||
"""Backend for MoE expert computation kernels. Available options:
|
||||
|
||||
- "auto": Automatically select the best backend based on model and hardware
|
||||
- "triton": Use Triton-based fused MoE kernels
|
||||
- "triton": Use Triton-based fused MoE kernels
|
||||
- "deep_gemm": Use DeepGEMM kernels (FP8 block-quantized only)
|
||||
- "deep_gemm_mega_moe": Use DeepGEMM mega MoE kernels
|
||||
- "cutlass": Use vLLM CUTLASS kernels
|
||||
|
||||
@@ -1892,6 +1892,10 @@ class VllmConfig:
|
||||
"to schedule a multiple of block_size tokens even if they are "
|
||||
"in the middle of a mm input"
|
||||
)
|
||||
# TODO: support align mamba cache mode for model runner v2
|
||||
assert not envs.VLLM_USE_V2_MODEL_RUNNER, (
|
||||
"Model Runner V2 has not yet supported mamba_cache_mode='align'. "
|
||||
)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_nvfp4_kv_cache_with_mla(self) -> "VllmConfig":
|
||||
|
||||
@@ -111,7 +111,15 @@ class MultiModalRegistry:
|
||||
return False
|
||||
|
||||
mm_config = model_config.get_multimodal_config()
|
||||
info = self._create_processing_info(model_config, tokenizer=None)
|
||||
try:
|
||||
info = self._create_processing_info(model_config, tokenizer=None)
|
||||
except ValueError:
|
||||
logger.warning_once(
|
||||
"Model %s is treated as multimodal but has no registered "
|
||||
"multimodal processor; running in text-only mode.",
|
||||
model_config.model,
|
||||
)
|
||||
return False
|
||||
|
||||
# Check if all supported modalities have limit == 0
|
||||
if all(
|
||||
@@ -170,7 +178,11 @@ class MultiModalRegistry:
|
||||
from vllm.model_executor.model_loader import get_model_architecture
|
||||
|
||||
model_cls, _ = get_model_architecture(model_config)
|
||||
assert hasattr(model_cls, "_processor_factory")
|
||||
if not hasattr(model_cls, "_processor_factory"):
|
||||
raise ValueError(
|
||||
f"Model class {model_cls.__name__} has no registered "
|
||||
"multimodal processor"
|
||||
)
|
||||
return cast("SupportsMultiModal", model_cls)
|
||||
|
||||
def _create_processing_ctx(
|
||||
|
||||
@@ -4,7 +4,6 @@ from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, cast
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig, get_layers_from_vllm_config
|
||||
@@ -19,8 +18,10 @@ from vllm.v1.kv_cache_interface import (
|
||||
AttentionSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheSpec,
|
||||
MambaSpec,
|
||||
UniformTypeKVCacheSpecs,
|
||||
)
|
||||
from vllm.v1.worker.gpu.model_states.interface import ModelSpecificAttnMetadata
|
||||
from vllm.v1.worker.utils import AttentionGroup, bind_kv_cache
|
||||
|
||||
|
||||
@@ -147,68 +148,126 @@ def _reshape_kv_cache(
|
||||
kv_cache_raw_tensors: dict[str, torch.Tensor],
|
||||
attn_backends: dict[str, type[AttentionBackend]],
|
||||
cache_dtype: str,
|
||||
) -> dict[str, torch.Tensor]:
|
||||
kv_caches: dict[str, torch.Tensor] = {}
|
||||
) -> dict[str, Any]:
|
||||
kv_caches: dict[str, Any] = {}
|
||||
has_attn, has_mamba = False, False
|
||||
for kv_cache_group_spec in kv_cache_config.kv_cache_groups:
|
||||
for layer_name in kv_cache_group_spec.layer_names:
|
||||
kv_cache_spec = kv_cache_group_spec.kv_cache_spec
|
||||
if isinstance(kv_cache_spec, UniformTypeKVCacheSpecs):
|
||||
kv_cache_spec = kv_cache_spec.kv_cache_specs[layer_name]
|
||||
assert isinstance(kv_cache_spec, AttentionSpec)
|
||||
|
||||
raw_tensor = kv_cache_raw_tensors[layer_name]
|
||||
assert raw_tensor.numel() % kv_cache_spec.page_size_bytes == 0
|
||||
num_blocks = raw_tensor.numel() // kv_cache_spec.page_size_bytes
|
||||
kv_raw_tensor = kv_cache_raw_tensors[layer_name]
|
||||
assert kv_raw_tensor.numel() % kv_cache_spec.page_size_bytes == 0
|
||||
num_blocks = kv_raw_tensor.numel() // kv_cache_spec.page_size_bytes
|
||||
|
||||
attn_backend = attn_backends[layer_name]
|
||||
kv_cache_shape = attn_backend.get_kv_cache_shape(
|
||||
num_blocks,
|
||||
kv_cache_spec.storage_block_size,
|
||||
kv_cache_spec.num_kv_heads,
|
||||
kv_cache_spec.head_size,
|
||||
cache_dtype,
|
||||
)
|
||||
|
||||
# FIXME(woosuk): Add kv_cache_stride_order to all attention backends.
|
||||
try:
|
||||
kv_cache_stride_order = attn_backend.get_kv_cache_stride_order()
|
||||
assert len(kv_cache_stride_order) == len(kv_cache_shape)
|
||||
except (AttributeError, NotImplementedError):
|
||||
kv_cache_stride_order = tuple(range(len(kv_cache_shape)))
|
||||
|
||||
kv_cache_shape = tuple(kv_cache_shape[i] for i in kv_cache_stride_order)
|
||||
inv_order = [
|
||||
kv_cache_stride_order.index(i)
|
||||
for i in range(len(kv_cache_stride_order))
|
||||
]
|
||||
|
||||
dtype = kv_cache_spec.dtype
|
||||
raw_tensor = raw_tensor.view(dtype)
|
||||
if kv_cache_spec.page_size_padded is not None:
|
||||
# Use strided view to handle page_size_bytes that
|
||||
# include padding. This follows the same pattern as
|
||||
# MambaSpec handling in gpu_model_runner.py.
|
||||
# NOTE: This assumes kv_cache_shape[0] == num_blocks
|
||||
# (i.e. the first physical dimension is the block
|
||||
# index), which holds for MLA backends but NOT for
|
||||
# standard attention backends whose shape starts with
|
||||
# a K/V dimension of size 2.
|
||||
dtype_size = get_dtype_size(dtype)
|
||||
page_stride = kv_cache_spec.page_size_bytes // dtype_size
|
||||
strides = list(torch.empty(kv_cache_shape).stride())
|
||||
strides[inv_order[0]] = page_stride
|
||||
kv_cache = torch.as_strided(
|
||||
raw_tensor,
|
||||
size=kv_cache_shape,
|
||||
stride=tuple(strides),
|
||||
if isinstance(kv_cache_spec, AttentionSpec):
|
||||
has_attn = True
|
||||
attn_backend = attn_backends[layer_name]
|
||||
kv_cache_shape = attn_backend.get_kv_cache_shape(
|
||||
num_blocks,
|
||||
kv_cache_spec.storage_block_size,
|
||||
kv_cache_spec.num_kv_heads,
|
||||
kv_cache_spec.head_size,
|
||||
cache_dtype_str=cache_dtype,
|
||||
)
|
||||
|
||||
# FIXME(woosuk): Add kv_cache_stride_order to all attention backends.
|
||||
try:
|
||||
kv_cache_stride_order = attn_backend.get_kv_cache_stride_order()
|
||||
assert len(kv_cache_stride_order) == len(kv_cache_shape)
|
||||
except (AttributeError, NotImplementedError):
|
||||
kv_cache_stride_order = tuple(range(len(kv_cache_shape)))
|
||||
|
||||
kv_cache_shape = tuple(kv_cache_shape[i] for i in kv_cache_stride_order)
|
||||
inv_order = [
|
||||
kv_cache_stride_order.index(i)
|
||||
for i in range(len(kv_cache_stride_order))
|
||||
]
|
||||
|
||||
dtype = kv_cache_spec.dtype
|
||||
kv_tensor = kv_raw_tensor.view(dtype)
|
||||
if kv_cache_spec.page_size_padded is not None:
|
||||
# Use strided view to handle page_size_bytes that
|
||||
# include padding. This follows the same pattern as
|
||||
# MambaSpec handling in gpu_model_runner.py.
|
||||
# NOTE: This assumes kv_cache_shape[0] == num_blocks
|
||||
# (i.e. the first physical dimension is the block
|
||||
# index), which holds for MLA backends but NOT for
|
||||
# standard attention backends whose shape starts with
|
||||
# a K/V dimension of size 2.
|
||||
dtype_size = get_dtype_size(dtype)
|
||||
page_stride = kv_cache_spec.page_size_bytes // dtype_size
|
||||
strides = list(torch.empty(kv_cache_shape).stride())
|
||||
strides[inv_order[0]] = page_stride
|
||||
kv_cache = torch.as_strided(
|
||||
kv_tensor,
|
||||
size=kv_cache_shape,
|
||||
stride=tuple(strides),
|
||||
)
|
||||
else:
|
||||
# No padding — safe to use a contiguous view.
|
||||
kv_cache = kv_tensor.view(kv_cache_shape)
|
||||
kv_caches[layer_name] = kv_cache.permute(*inv_order)
|
||||
|
||||
elif isinstance(kv_cache_spec, MambaSpec):
|
||||
has_mamba = True
|
||||
state_tensors = []
|
||||
storage_offset_bytes = 0
|
||||
for shape, dtype in zip(kv_cache_spec.shapes, kv_cache_spec.dtypes):
|
||||
dtype_size = get_dtype_size(dtype)
|
||||
num_element_per_page = kv_cache_spec.page_size_bytes // dtype_size
|
||||
target_shape = (num_blocks, *shape)
|
||||
stride = torch.empty(target_shape).stride()
|
||||
target_stride = (num_element_per_page, *stride[1:])
|
||||
assert storage_offset_bytes % dtype_size == 0
|
||||
tensor = torch.as_strided(
|
||||
kv_raw_tensor.view(dtype),
|
||||
size=target_shape,
|
||||
stride=target_stride,
|
||||
storage_offset=storage_offset_bytes // dtype_size,
|
||||
)
|
||||
state_tensors.append(tensor)
|
||||
storage_offset_bytes += stride[0] * dtype_size
|
||||
kv_caches[layer_name] = state_tensors
|
||||
else:
|
||||
# No padding — safe to use a contiguous view.
|
||||
kv_cache = raw_tensor.view(kv_cache_shape)
|
||||
kv_caches[layer_name] = kv_cache.permute(*inv_order)
|
||||
raise NotImplementedError(
|
||||
f"Unsupported KV cache spec type: {type(kv_cache_spec)}"
|
||||
)
|
||||
|
||||
if has_attn and has_mamba:
|
||||
_update_hybrid_attention_layout(kv_caches, kv_cache_config)
|
||||
|
||||
return kv_caches
|
||||
|
||||
|
||||
def _update_hybrid_attention_layout(
|
||||
kv_caches: dict[str, Any],
|
||||
kv_cache_config: KVCacheConfig,
|
||||
) -> None:
|
||||
for kv_cache_group_spec in kv_cache_config.kv_cache_groups:
|
||||
for layer_name in kv_cache_group_spec.layer_names:
|
||||
kv_cache_spec = kv_cache_group_spec.kv_cache_spec
|
||||
if isinstance(kv_cache_spec, UniformTypeKVCacheSpecs):
|
||||
kv_cache_spec = kv_cache_spec.kv_cache_specs[layer_name]
|
||||
if not isinstance(kv_cache_spec, AttentionSpec):
|
||||
continue
|
||||
kv_cache = kv_caches[layer_name]
|
||||
if kv_cache.shape[0] == 2:
|
||||
assert kv_cache.shape[1] != 2, (
|
||||
f"Cannot determine layout for tensor of shape {kv_cache.shape}"
|
||||
)
|
||||
hidden_size = kv_cache.shape[2:].numel()
|
||||
kv_cache.as_strided_(
|
||||
size=kv_cache.shape,
|
||||
stride=(
|
||||
hidden_size,
|
||||
2 * hidden_size,
|
||||
*kv_cache.stride()[2:],
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def init_kv_cache(
|
||||
runner_kv_caches: list[torch.Tensor],
|
||||
forward_context: dict[str, Any],
|
||||
@@ -216,7 +275,7 @@ def init_kv_cache(
|
||||
attn_backends: dict[str, type[AttentionBackend]],
|
||||
device: torch.device,
|
||||
cache_dtype: str,
|
||||
) -> dict[str, torch.Tensor]:
|
||||
) -> dict[str, Any]:
|
||||
kv_cache_raw_tensors = _allocate_kv_cache(kv_cache_config, device)
|
||||
kv_caches = _reshape_kv_cache(
|
||||
kv_cache_config, kv_cache_raw_tensors, attn_backends, cache_dtype
|
||||
@@ -250,8 +309,9 @@ def build_attn_metadata(
|
||||
kv_cache_config: KVCacheConfig,
|
||||
seq_lens_cpu_upper_bound: torch.Tensor | None = None,
|
||||
dcp_local_seq_lens: torch.Tensor | None = None,
|
||||
encoder_seq_lens: dict[int, tuple[torch.Tensor, np.ndarray]] | None = None,
|
||||
positions: torch.Tensor | None = None,
|
||||
model_specific_attn_metadata: ModelSpecificAttnMetadata | None = None,
|
||||
for_cudagraph_capture: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
seq_lens = seq_lens[:num_reqs]
|
||||
if dcp_local_seq_lens is not None:
|
||||
@@ -265,6 +325,11 @@ def build_attn_metadata(
|
||||
block_table = block_tables[i]
|
||||
slot_mapping = slot_mappings[i]
|
||||
|
||||
common_attn_metadata_extra_kwargs = (
|
||||
model_specific_attn_metadata.get_extra_common_attn_kwargs(i, num_reqs)
|
||||
if model_specific_attn_metadata is not None
|
||||
else {}
|
||||
)
|
||||
common_attn_metadata = CommonAttentionMetadata(
|
||||
query_start_loc=query_start_loc_gpu,
|
||||
query_start_loc_cpu=query_start_loc_cpu,
|
||||
@@ -279,17 +344,29 @@ def build_attn_metadata(
|
||||
causal=True,
|
||||
dcp_local_seq_lens=dcp_local_seq_lens,
|
||||
positions=positions,
|
||||
**common_attn_metadata_extra_kwargs,
|
||||
)
|
||||
if encoder_seq_lens and i in encoder_seq_lens:
|
||||
encoder_seq_lens_gpu, encoder_seq_lens_cpu = encoder_seq_lens[i]
|
||||
common_attn_metadata.encoder_seq_lens = encoder_seq_lens_gpu
|
||||
common_attn_metadata.encoder_seq_lens_cpu = encoder_seq_lens_cpu
|
||||
|
||||
for attn_group in attn_groups[i]:
|
||||
attn_metadata_builder = attn_group.get_metadata_builder(0)
|
||||
metadata = attn_metadata_builder.build(
|
||||
common_prefix_len=0, common_attn_metadata=common_attn_metadata
|
||||
)
|
||||
if for_cudagraph_capture:
|
||||
metadata = attn_metadata_builder.build_for_cudagraph_capture(
|
||||
common_attn_metadata
|
||||
)
|
||||
else:
|
||||
attn_metadata_extra_kwargs = (
|
||||
model_specific_attn_metadata.get_extra_attn_kwargs(
|
||||
attn_metadata_builder,
|
||||
num_reqs,
|
||||
)
|
||||
if model_specific_attn_metadata is not None
|
||||
else {}
|
||||
)
|
||||
metadata = attn_metadata_builder.build(
|
||||
common_prefix_len=0,
|
||||
common_attn_metadata=common_attn_metadata,
|
||||
**attn_metadata_extra_kwargs,
|
||||
)
|
||||
for layer_name in attn_group.layer_names:
|
||||
attn_metadata[layer_name] = metadata
|
||||
return attn_metadata
|
||||
|
||||
@@ -5,7 +5,6 @@ from collections.abc import Iterable
|
||||
import torch
|
||||
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.v1.attention.backends.utils import PAD_SLOT_ID
|
||||
from vllm.v1.worker.gpu.buffer_utils import StagedWriteTensor, UvaBackedTensor
|
||||
|
||||
@@ -16,7 +15,7 @@ class BlockTables:
|
||||
block_sizes: list[int],
|
||||
max_num_reqs: int,
|
||||
max_num_batched_tokens: int,
|
||||
max_model_len: int,
|
||||
max_num_blocks_per_group: list[int],
|
||||
device: torch.device,
|
||||
cp_size: int = 1,
|
||||
cp_rank: int = 0,
|
||||
@@ -25,7 +24,6 @@ class BlockTables:
|
||||
self.block_sizes = block_sizes
|
||||
self.max_num_reqs = max_num_reqs
|
||||
self.max_num_batched_tokens = max_num_batched_tokens
|
||||
self.max_model_len = max_model_len
|
||||
self.device = device
|
||||
|
||||
self.cp_size = cp_size
|
||||
@@ -33,19 +31,11 @@ class BlockTables:
|
||||
self.cp_interleave = cp_interleave
|
||||
|
||||
self.num_kv_cache_groups = len(self.block_sizes)
|
||||
assert len(max_num_blocks_per_group) == self.num_kv_cache_groups
|
||||
# num_kv_cache_groups x [max_num_reqs, max_num_blocks]
|
||||
self.block_tables: list[StagedWriteTensor] = []
|
||||
for i in range(self.num_kv_cache_groups):
|
||||
block_size = self.block_sizes[i]
|
||||
# When using DCP, each request's KV cache is sharded among different ranks.
|
||||
# As a result, one block on the current rank covers `block_size * cp_size`
|
||||
# tokens in the full, global (unsharded) sequence.
|
||||
max_num_blocks = cdiv(self.max_model_len, block_size * self.cp_size)
|
||||
# Align to a multiple of (128 / block_size) as required
|
||||
# by some attention backends such as TRTLLM (#39324)
|
||||
if block_size <= 128:
|
||||
alignment = 128 // block_size
|
||||
max_num_blocks = cdiv(max_num_blocks, alignment) * alignment
|
||||
max_num_blocks = max_num_blocks_per_group[i]
|
||||
block_table = StagedWriteTensor(
|
||||
(self.max_num_reqs, max_num_blocks),
|
||||
dtype=torch.int32,
|
||||
|
||||
@@ -34,7 +34,7 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
class CapturedAttentionState(NamedTuple):
|
||||
attn_metadata: dict[str, Any]
|
||||
attn_metadata: dict[str, Any] | None
|
||||
slot_mappings: dict[str, torch.Tensor]
|
||||
|
||||
|
||||
@@ -100,6 +100,7 @@ class CudaGraphManager:
|
||||
self.decode_query_len = decode_query_len
|
||||
|
||||
self.dp_size = vllm_config.parallel_config.data_parallel_size
|
||||
self.tp_size = vllm_config.parallel_config.tensor_parallel_size
|
||||
self.is_first_pp_rank = get_pp_group().is_first_rank
|
||||
self.is_last_pp_rank = get_pp_group().is_last_rank
|
||||
|
||||
@@ -109,6 +110,10 @@ class CudaGraphManager:
|
||||
self._graphs_captured = False
|
||||
self._candidates: list[list[BatchExecutionDescriptor]] = []
|
||||
self._capture_descs: dict[CUDAGraphMode, list[BatchExecutionDescriptor]] = {}
|
||||
# adjust the cudagraph sizes to be a multiple of the uniform decode query length
|
||||
self.compilation_config.adjust_cudagraph_sizes_for_spec_decode(
|
||||
self.decode_query_len, self.tp_size
|
||||
)
|
||||
self._init_candidates()
|
||||
|
||||
def _init_candidates(self) -> None:
|
||||
@@ -340,16 +345,16 @@ class ModelCudaGraphManager(CudaGraphManager):
|
||||
block_tables,
|
||||
attn_groups,
|
||||
kv_cache_config,
|
||||
skip_attn=(desc.cg_mode == CUDAGraphMode.PIECEWISE),
|
||||
)
|
||||
|
||||
def forward_fn(cg_mode: CUDAGraphMode) -> None:
|
||||
batch_descriptor = (
|
||||
BatchDescriptor(num_tokens=num_tokens)
|
||||
if cg_mode == CUDAGraphMode.PIECEWISE
|
||||
else None
|
||||
)
|
||||
batch_descriptor = None
|
||||
if cg_mode == CUDAGraphMode.PIECEWISE:
|
||||
assert attn_metadata is None
|
||||
batch_descriptor = BatchDescriptor(num_tokens=num_tokens)
|
||||
with set_forward_context(
|
||||
attn_metadata if cg_mode != CUDAGraphMode.PIECEWISE else None,
|
||||
attn_metadata,
|
||||
self.vllm_config,
|
||||
num_tokens=num_tokens,
|
||||
cudagraph_runtime_mode=cg_mode,
|
||||
@@ -419,6 +424,7 @@ def prepare_inputs_to_capture(
|
||||
block_tables: BlockTables,
|
||||
attn_groups: list[list[AttentionGroup]],
|
||||
kv_cache_config: KVCacheConfig,
|
||||
skip_attn: bool = False,
|
||||
) -> CapturedAttentionState:
|
||||
input_batch = InputBatch.make_dummy(num_reqs, num_tokens, input_buffers)
|
||||
input_block_tables = block_tables.get_dummy_block_tables(num_reqs)
|
||||
@@ -439,13 +445,15 @@ def prepare_inputs_to_capture(
|
||||
)
|
||||
input_batch.dcp_local_seq_lens = input_buffers.dcp_local_seq_lens[:num_reqs]
|
||||
|
||||
attn_metadata = model_state.prepare_attn(
|
||||
input_batch,
|
||||
CUDAGraphMode.NONE,
|
||||
input_block_tables,
|
||||
slot_mappings,
|
||||
attn_groups,
|
||||
kv_cache_config,
|
||||
for_capture=True,
|
||||
)
|
||||
attn_metadata = None
|
||||
if not skip_attn:
|
||||
attn_metadata = model_state.prepare_attn(
|
||||
input_batch,
|
||||
CUDAGraphMode.NONE,
|
||||
input_block_tables,
|
||||
slot_mappings,
|
||||
attn_groups,
|
||||
kv_cache_config,
|
||||
for_capture=True,
|
||||
)
|
||||
return CapturedAttentionState(attn_metadata, slot_mappings_by_layer)
|
||||
|
||||
@@ -53,7 +53,10 @@ class InputBatch:
|
||||
# sum(num_scheduled_tokens)
|
||||
num_tokens: int
|
||||
num_tokens_after_padding: int
|
||||
# Sum of draft tokens scheduled across requests.
|
||||
num_draft_tokens: int
|
||||
# [num_reqs] number of draft tokens scheduled for each request, if any.
|
||||
num_draft_tokens_per_req: np.ndarray | None
|
||||
|
||||
# [num_reqs + 1]
|
||||
query_start_loc: torch.Tensor
|
||||
@@ -64,6 +67,8 @@ class InputBatch:
|
||||
seq_lens_cpu_upper_bound: torch.Tensor
|
||||
# [num_reqs]
|
||||
dcp_local_seq_lens: torch.Tensor | None
|
||||
# [num_reqs] CPU bool array.
|
||||
is_prefilling_np: np.ndarray
|
||||
|
||||
# [num_tokens_after_padding]
|
||||
input_ids: torch.Tensor
|
||||
@@ -137,11 +142,13 @@ class InputBatch:
|
||||
num_tokens=num_tokens,
|
||||
num_tokens_after_padding=num_tokens,
|
||||
num_draft_tokens=0,
|
||||
num_draft_tokens_per_req=None,
|
||||
query_start_loc=query_start_loc,
|
||||
query_start_loc_np=query_start_loc_np,
|
||||
seq_lens=seq_lens,
|
||||
seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,
|
||||
dcp_local_seq_lens=None,
|
||||
is_prefilling_np=np.zeros(num_reqs, dtype=np.bool_),
|
||||
input_ids=input_ids,
|
||||
positions=positions,
|
||||
logits_indices=logits_indices,
|
||||
|
||||
@@ -44,10 +44,11 @@ from vllm.model_executor.model_loader import get_model_loader
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.tasks import SupportedTask
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.mem_utils import DeviceMemoryProfiler, format_gib
|
||||
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE
|
||||
from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig, MambaSpec
|
||||
from vllm.v1.outputs import DraftTokenIds, KVConnectorOutput, ModelRunnerOutput
|
||||
from vllm.v1.worker.cp_utils import check_attention_cp_compatibility
|
||||
from vllm.v1.worker.gpu.async_utils import AsyncOutput, AsyncPoolingOutput
|
||||
@@ -338,10 +339,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
def initialize_kv_cache(self, kv_cache_config: KVCacheConfig) -> None:
|
||||
kv_cache_config = deepcopy(kv_cache_config)
|
||||
self.kv_cache_config = kv_cache_config
|
||||
block_sizes = [
|
||||
kv_cache_group.kv_cache_spec.block_size
|
||||
for kv_cache_group in kv_cache_config.kv_cache_groups
|
||||
]
|
||||
|
||||
block_table_max_model_len = self.max_model_len
|
||||
if self.is_encoder_decoder:
|
||||
@@ -352,11 +349,34 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
getattr(self.model_config.hf_config, "max_source_positions", 0),
|
||||
)
|
||||
|
||||
block_sizes = []
|
||||
max_num_blocks_per_group = []
|
||||
for kv_cache_group in kv_cache_config.kv_cache_groups:
|
||||
spec = kv_cache_group.kv_cache_spec
|
||||
block_sizes.append(spec.block_size)
|
||||
# When using DCP, each request's KV cache is sharded among different ranks.
|
||||
# As a result, one block on the current rank covers `block_size * cp_size`
|
||||
# tokens in the full, global (unsharded) sequence.
|
||||
max_num_blocks = cdiv(
|
||||
block_table_max_model_len, spec.block_size * self.dcp_size
|
||||
)
|
||||
# Align to a multiple of (128 / block_size) as required by some attention
|
||||
# backends such as TRTLLM (#39324)
|
||||
if spec.block_size <= 128:
|
||||
alignment = 128 // spec.block_size
|
||||
max_num_blocks = cdiv(max_num_blocks, alignment) * alignment
|
||||
# For Mamba/Hybrid Model, KVCaches need extra blocks for speculative tokens
|
||||
if isinstance(spec, MambaSpec):
|
||||
max_num_blocks = (
|
||||
max_num_blocks if self.cache_config.enable_prefix_caching else 1
|
||||
) + spec.num_speculative_blocks
|
||||
max_num_blocks_per_group.append(max_num_blocks)
|
||||
|
||||
self.block_tables = BlockTables(
|
||||
block_sizes=block_sizes,
|
||||
max_num_reqs=self.max_num_reqs,
|
||||
max_num_batched_tokens=self.max_num_tokens,
|
||||
max_model_len=block_table_max_model_len,
|
||||
max_num_blocks_per_group=max_num_blocks_per_group,
|
||||
device=self.device,
|
||||
cp_size=self.dcp_size,
|
||||
cp_rank=self.dcp_rank,
|
||||
@@ -722,6 +742,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
|
||||
# Get the number of draft tokens for each request.
|
||||
draft_tokens = scheduler_output.scheduled_spec_decode_tokens
|
||||
num_draft_tokens_per_req = None
|
||||
if not draft_tokens:
|
||||
# No draft token scheduled (common case).
|
||||
total_num_draft_tokens = 0
|
||||
@@ -735,15 +756,15 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
num_reqs, dtype=torch.int32, device=self.device
|
||||
)
|
||||
else:
|
||||
num_draft_tokens = np.fromiter(
|
||||
num_draft_tokens_per_req = np.fromiter(
|
||||
(len(draft_tokens.get(req_id, ())) for req_id in req_ids),
|
||||
dtype=np.int32,
|
||||
count=num_reqs,
|
||||
)
|
||||
total_num_draft_tokens = int(num_draft_tokens.sum())
|
||||
total_num_draft_tokens = int(num_draft_tokens_per_req.sum())
|
||||
total_num_logits = num_reqs + total_num_draft_tokens
|
||||
|
||||
num_logits = num_draft_tokens + 1
|
||||
num_logits = num_draft_tokens_per_req + 1
|
||||
cu_num_logits_np = np.empty(num_reqs + 1, dtype=np.int32)
|
||||
cu_num_logits_np[0] = 0
|
||||
np.cumsum(num_logits, out=cu_num_logits_np[1:])
|
||||
@@ -766,9 +787,13 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
async_copy_to_gpu(query_start_loc_np, out=self.input_buffers.query_start_loc)
|
||||
query_start_loc_np = query_start_loc_np[: num_reqs_padded + 1]
|
||||
query_start_loc = self.input_buffers.query_start_loc[: num_reqs_padded + 1]
|
||||
is_prefilling_np = (
|
||||
self.req_states.num_computed_prefill_tokens[idx_mapping_np]
|
||||
< self.req_states.prefill_len.np[idx_mapping_np]
|
||||
)
|
||||
|
||||
# Get prefill tokens if any.
|
||||
if self.req_states.any_prefills(idx_mapping_np):
|
||||
if np.any(is_prefilling_np):
|
||||
prepare_prefill_inputs(
|
||||
self.input_buffers.input_ids,
|
||||
self.req_states.next_prefill_tokens,
|
||||
@@ -836,11 +861,13 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
num_tokens=num_tokens,
|
||||
num_tokens_after_padding=num_tokens_after_padding,
|
||||
num_draft_tokens=total_num_draft_tokens,
|
||||
num_draft_tokens_per_req=num_draft_tokens_per_req,
|
||||
query_start_loc=query_start_loc,
|
||||
query_start_loc_np=query_start_loc_np,
|
||||
seq_lens=seq_lens,
|
||||
seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,
|
||||
dcp_local_seq_lens=dcp_local_seq_lens,
|
||||
is_prefilling_np=is_prefilling_np,
|
||||
input_ids=self.input_buffers.input_ids[:num_tokens_after_padding],
|
||||
positions=self.input_buffers.positions[:num_tokens_after_padding],
|
||||
logits_indices=logits_indices,
|
||||
@@ -958,6 +985,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
input_batch.num_scheduled_tokens
|
||||
)
|
||||
|
||||
self.model_state.postprocess_state(input_batch, num_sampled)
|
||||
|
||||
@torch.inference_mode()
|
||||
def execute_model(
|
||||
self,
|
||||
|
||||
@@ -18,6 +18,11 @@ def init_model_state(
|
||||
|
||||
return WhisperModelState(vllm_config, model, encoder_cache, device)
|
||||
|
||||
if vllm_config.model_config.is_hybrid:
|
||||
from vllm.v1.worker.gpu.model_states.mamba_hybrid import MambaHybridModelState
|
||||
|
||||
return MambaHybridModelState(vllm_config, model, encoder_cache, device)
|
||||
|
||||
from vllm.v1.worker.gpu.model_states.default import DefaultModelState
|
||||
|
||||
return DefaultModelState(vllm_config, model, encoder_cache, device)
|
||||
|
||||
@@ -194,5 +194,6 @@ class DefaultModelState(ModelState):
|
||||
seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,
|
||||
dcp_local_seq_lens=input_batch.dcp_local_seq_lens,
|
||||
positions=input_batch.positions,
|
||||
for_cudagraph_capture=for_capture,
|
||||
)
|
||||
return attn_metadata
|
||||
|
||||
@@ -17,6 +17,24 @@ from vllm.v1.worker.gpu.states import RequestState
|
||||
from vllm.v1.worker.utils import AttentionGroup
|
||||
|
||||
|
||||
class ModelSpecificAttnMetadata:
|
||||
"""Base class for model-specific attention metadata."""
|
||||
|
||||
def get_extra_common_attn_kwargs(
|
||||
self,
|
||||
kv_cache_group_id: int,
|
||||
num_reqs: int,
|
||||
) -> dict[str, Any]:
|
||||
return {}
|
||||
|
||||
def get_extra_attn_kwargs(
|
||||
self,
|
||||
attn_metadata_builder: Any,
|
||||
num_reqs: int,
|
||||
) -> dict[str, Any]:
|
||||
return {}
|
||||
|
||||
|
||||
class ModelState(ABC):
|
||||
@abstractmethod
|
||||
def __init__(
|
||||
@@ -38,6 +56,13 @@ class ModelState(ABC):
|
||||
def apply_staged_writes(self) -> None:
|
||||
return None
|
||||
|
||||
def postprocess_state(
|
||||
self,
|
||||
input_batch: InputBatch,
|
||||
num_sampled: torch.Tensor,
|
||||
) -> None:
|
||||
return None
|
||||
|
||||
@abstractmethod
|
||||
def get_mm_embeddings(
|
||||
self,
|
||||
|
||||
@@ -0,0 +1,150 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadataBuilder
|
||||
from vllm.v1.attention.backends.mamba2_attn import Mamba2AttentionMetadataBuilder
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.worker.gpu.attn_utils import build_attn_metadata
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch
|
||||
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
|
||||
from vllm.v1.worker.gpu.model_states.default import DefaultModelState
|
||||
from vllm.v1.worker.gpu.model_states.interface import ModelSpecificAttnMetadata
|
||||
from vllm.v1.worker.utils import AttentionGroup
|
||||
|
||||
|
||||
@dataclass
|
||||
class MambaHybridAttnMetadata(ModelSpecificAttnMetadata):
|
||||
is_prefilling: torch.Tensor
|
||||
num_accepted_tokens: torch.Tensor | None = None
|
||||
num_decode_draft_tokens_cpu: torch.Tensor | None = None
|
||||
|
||||
def get_extra_common_attn_kwargs(
|
||||
self,
|
||||
kv_cache_group_id: int,
|
||||
num_reqs: int,
|
||||
) -> dict[str, Any]:
|
||||
return {"is_prefilling": self.is_prefilling[:num_reqs]}
|
||||
|
||||
def get_extra_attn_kwargs(
|
||||
self,
|
||||
attn_metadata_builder: Any,
|
||||
num_reqs: int,
|
||||
) -> dict[str, Any]:
|
||||
if not isinstance(
|
||||
attn_metadata_builder,
|
||||
(Mamba2AttentionMetadataBuilder, GDNAttentionMetadataBuilder),
|
||||
):
|
||||
return {}
|
||||
return {
|
||||
"num_accepted_tokens": None
|
||||
if self.num_accepted_tokens is None
|
||||
else self.num_accepted_tokens[:num_reqs],
|
||||
"num_decode_draft_tokens_cpu": None
|
||||
if self.num_decode_draft_tokens_cpu is None
|
||||
else self.num_decode_draft_tokens_cpu[:num_reqs],
|
||||
}
|
||||
|
||||
|
||||
class MambaHybridModelState(DefaultModelState):
|
||||
"""Model state for hybrid attention + Mamba / linear-attention models."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
model: nn.Module,
|
||||
encoder_cache: EncoderCache | None,
|
||||
device: torch.device,
|
||||
) -> None:
|
||||
super().__init__(vllm_config, model, encoder_cache, device)
|
||||
self.num_accepted_tokens_gpu = torch.ones(
|
||||
self.max_num_reqs, dtype=torch.int32, device=self.device
|
||||
)
|
||||
|
||||
def prepare_attn(
|
||||
self,
|
||||
input_batch: InputBatch,
|
||||
cudagraph_mode: CUDAGraphMode,
|
||||
block_tables: tuple[torch.Tensor, ...],
|
||||
slot_mappings: torch.Tensor,
|
||||
attn_groups: list[list[AttentionGroup]],
|
||||
kv_cache_config: KVCacheConfig,
|
||||
for_capture: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
if cudagraph_mode == CUDAGraphMode.FULL:
|
||||
num_reqs = input_batch.num_reqs_after_padding
|
||||
num_tokens = input_batch.num_tokens_after_padding
|
||||
else:
|
||||
num_reqs = input_batch.num_reqs
|
||||
num_tokens = input_batch.num_tokens
|
||||
query_start_loc_cpu = torch.from_numpy(input_batch.query_start_loc_np)
|
||||
max_query_len = input_batch.num_scheduled_tokens.max().item()
|
||||
|
||||
is_prefilling = torch.zeros(num_reqs, dtype=torch.bool, device="cpu")
|
||||
is_prefilling[: input_batch.num_reqs] = torch.from_numpy(
|
||||
input_batch.is_prefilling_np
|
||||
)
|
||||
# During CUDAGraph capture, num_decode_draft_tokens_cpu and num_accepted_tokens
|
||||
# are created by attn_metadata_builder.build_for_cudagraph_capture, so we only
|
||||
# compute them during actual (non-capture) forward execution.
|
||||
num_accepted_tokens = None
|
||||
num_decode_draft_tokens_cpu = None
|
||||
if not for_capture:
|
||||
num_accepted_tokens = self.num_accepted_tokens_gpu.new_ones(num_reqs)
|
||||
num_accepted_tokens[: input_batch.num_reqs] = self.num_accepted_tokens_gpu[
|
||||
input_batch.idx_mapping
|
||||
]
|
||||
|
||||
# GDN uses >= 0 to select spec-decode rows, so non-decode rows
|
||||
# need the -1 sentinel rather than a raw zero draft count.
|
||||
num_decode_draft_tokens_np = np.full(num_reqs, -1, dtype=np.int32)
|
||||
if input_batch.num_draft_tokens_per_req is not None:
|
||||
spec_decode_mask = (
|
||||
input_batch.num_draft_tokens_per_req > 0
|
||||
) & ~input_batch.is_prefilling_np
|
||||
num_decode_draft_tokens_np[: input_batch.num_reqs] = np.where(
|
||||
spec_decode_mask,
|
||||
input_batch.num_draft_tokens_per_req,
|
||||
-1,
|
||||
)
|
||||
num_decode_draft_tokens_cpu = torch.from_numpy(num_decode_draft_tokens_np)
|
||||
|
||||
mamba_attn_metadata = MambaHybridAttnMetadata(
|
||||
is_prefilling=is_prefilling,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
num_decode_draft_tokens_cpu=num_decode_draft_tokens_cpu,
|
||||
)
|
||||
return build_attn_metadata(
|
||||
attn_groups=attn_groups,
|
||||
num_reqs=num_reqs,
|
||||
num_tokens=num_tokens,
|
||||
query_start_loc_gpu=input_batch.query_start_loc,
|
||||
query_start_loc_cpu=query_start_loc_cpu,
|
||||
max_query_len=max_query_len,
|
||||
seq_lens=input_batch.seq_lens,
|
||||
max_seq_len=self.max_model_len,
|
||||
block_tables=block_tables,
|
||||
slot_mappings=slot_mappings,
|
||||
kv_cache_config=kv_cache_config,
|
||||
dcp_local_seq_lens=input_batch.dcp_local_seq_lens,
|
||||
model_specific_attn_metadata=mamba_attn_metadata,
|
||||
for_cudagraph_capture=for_capture,
|
||||
)
|
||||
|
||||
def postprocess_state(
|
||||
self,
|
||||
input_batch: InputBatch,
|
||||
num_sampled: torch.Tensor,
|
||||
) -> None:
|
||||
# Chunked prefill does not sample a token, so num_sampled can be 0.
|
||||
# Mamba treats num_accepted_tokens=1 as the neutral non-spec value.
|
||||
self.num_accepted_tokens_gpu[input_batch.idx_mapping] = torch.clamp(
|
||||
num_sampled, min=1
|
||||
)
|
||||
@@ -1,5 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
@@ -13,11 +14,33 @@ from vllm.v1.worker.gpu.attn_utils import build_attn_metadata
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch
|
||||
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
|
||||
from vllm.v1.worker.gpu.mm.encoder_runner import EncoderRunner
|
||||
from vllm.v1.worker.gpu.model_states.interface import ModelState
|
||||
from vllm.v1.worker.gpu.model_states.interface import (
|
||||
ModelSpecificAttnMetadata,
|
||||
ModelState,
|
||||
)
|
||||
from vllm.v1.worker.gpu.states import RequestState
|
||||
from vllm.v1.worker.utils import AttentionGroup
|
||||
|
||||
|
||||
@dataclass
|
||||
class WhisperAttnMetadata(ModelSpecificAttnMetadata):
|
||||
encoder_seq_lens: dict[int, tuple[torch.Tensor, np.ndarray]]
|
||||
|
||||
def get_extra_common_attn_kwargs(
|
||||
self,
|
||||
kv_cache_group_id: int,
|
||||
num_reqs: int,
|
||||
) -> dict[str, Any]:
|
||||
encoder_seq_lens = self.encoder_seq_lens.get(kv_cache_group_id)
|
||||
if encoder_seq_lens is None:
|
||||
return {}
|
||||
encoder_seq_lens_gpu, encoder_seq_lens_cpu = encoder_seq_lens
|
||||
return {
|
||||
"encoder_seq_lens": encoder_seq_lens_gpu[:num_reqs],
|
||||
"encoder_seq_lens_cpu": encoder_seq_lens_cpu[:num_reqs],
|
||||
}
|
||||
|
||||
|
||||
class WhisperModelState(ModelState):
|
||||
def __init__(
|
||||
self,
|
||||
@@ -111,8 +134,8 @@ class WhisperModelState(ModelState):
|
||||
else:
|
||||
num_reqs = input_batch.num_reqs
|
||||
num_tokens = input_batch.num_tokens
|
||||
encoder_seq_lens = self._get_encoder_seq_lens(
|
||||
input_batch.req_ids, attn_groups, for_capture
|
||||
whisper_attn_metadata = WhisperAttnMetadata(
|
||||
self._get_encoder_seq_lens(input_batch.req_ids, attn_groups, for_capture)
|
||||
)
|
||||
|
||||
query_start_loc_cpu = torch.from_numpy(input_batch.query_start_loc_np)
|
||||
@@ -136,7 +159,8 @@ class WhisperModelState(ModelState):
|
||||
kv_cache_config=kv_cache_config,
|
||||
seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,
|
||||
dcp_local_seq_lens=input_batch.dcp_local_seq_lens,
|
||||
encoder_seq_lens=encoder_seq_lens,
|
||||
model_specific_attn_metadata=whisper_attn_metadata,
|
||||
for_cudagraph_capture=for_capture,
|
||||
)
|
||||
return attn_metadata
|
||||
|
||||
|
||||
@@ -106,6 +106,7 @@ class DecodeEagleCudaGraphManager(EagleCudaGraphManagerBase):
|
||||
block_tables,
|
||||
attn_groups,
|
||||
kv_cache_config,
|
||||
skip_attn=(desc.cg_mode == CUDAGraphMode.PIECEWISE),
|
||||
)
|
||||
attn_metadata, slot_mappings = attn_state
|
||||
|
||||
|
||||
Reference in New Issue
Block a user