diff --git a/vllm/model_executor/layers/mamba/mamba_utils.py b/vllm/model_executor/layers/mamba/mamba_utils.py index 18a73c6554..71872de47c 100644 --- a/vllm/model_executor/layers/mamba/mamba_utils.py +++ b/vllm/model_executor/layers/mamba/mamba_utils.py @@ -235,9 +235,10 @@ class MambaStateShapeCalculator: tp_world_size: int, intermediate_size: int, conv_kernel: int, + num_spec: int = 0, ) -> tuple[tuple[int, int]]: conv_dim = divide(intermediate_size, tp_world_size) - conv_state_shape = cls._orient_conv_shape(conv_dim, conv_kernel - 1) + conv_state_shape = cls._orient_conv_shape(conv_dim, conv_kernel - 1 + num_spec) return (conv_state_shape,) @classmethod diff --git a/vllm/model_executor/layers/mamba/short_conv.py b/vllm/model_executor/layers/mamba/short_conv.py index f5548cc7de..de2efea50f 100644 --- a/vllm/model_executor/layers/mamba/short_conv.py +++ b/vllm/model_executor/layers/mamba/short_conv.py @@ -80,7 +80,9 @@ class ShortConv(MambaBase, PluggableLayer): prefix=f"{prefix}.out_proj", ) - compilation_config = get_current_vllm_config().compilation_config + vllm_config = get_current_vllm_config() + self.num_spec = vllm_config.num_speculative_tokens + compilation_config = vllm_config.compilation_config if prefix in compilation_config.static_forward_context: raise ValueError(f"Duplicate layer name: {prefix}") compilation_config.static_forward_context[prefix] = self @@ -234,6 +236,8 @@ class ShortConv(MambaBase, PluggableLayer): state_indices_tensor_d = attn_metadata.state_indices_tensor_d has_initial_states_p = attn_metadata.has_initial_states_p query_start_loc_p = attn_metadata.query_start_loc_p + num_accepted_tokens = attn_metadata.num_accepted_tokens + query_start_loc_d = attn_metadata.query_start_loc_d BCx, _ = self.in_proj(hidden_states) @@ -296,14 +300,33 @@ class ShortConv(MambaBase, PluggableLayer): if has_decode: Bx_d = (B_d * x_d).contiguous() - Bx = causal_conv1d_update( - Bx_d, - conv_state, - conv_weights, - self.conv.bias, - activation=None, - conv_state_indices=state_indices_tensor_d, - ) + if num_accepted_tokens is not None: + # Speculative decode: the verify step feeds >1 query token per + # decode request, so use the spec-aware conv update path + # (mirrors mamba_mixer2). state_indices_tensor_d is + # (num_decodes, 1 + num_spec_tokens) here. + assert state_indices_tensor_d is not None + Bx = causal_conv1d_update( + Bx_d, + conv_state, + conv_weights, + self.conv.bias, + activation=None, + conv_state_indices=state_indices_tensor_d, + num_accepted_tokens=num_accepted_tokens, + query_start_loc=query_start_loc_d, + max_query_len=state_indices_tensor_d.size(-1), + ) + else: + # Non-spec decode + Bx = causal_conv1d_update( + Bx_d, + conv_state, + conv_weights, + self.conv.bias, + activation=None, + conv_state_indices=state_indices_tensor_d, + ) y = C_d * Bx conv_output_list.insert(0, y) @@ -326,6 +349,7 @@ class ShortConv(MambaBase, PluggableLayer): tp_world_size=get_tensor_model_parallel_world_size(), intermediate_size=self.conv_dim, conv_kernel=self.L_cache, + num_spec=self.num_spec, ) @property diff --git a/vllm/v1/worker/gpu/model_states/mamba_hybrid.py b/vllm/v1/worker/gpu/model_states/mamba_hybrid.py index c61a56f9aa..420bf5d6dd 100644 --- a/vllm/v1/worker/gpu/model_states/mamba_hybrid.py +++ b/vllm/v1/worker/gpu/model_states/mamba_hybrid.py @@ -12,6 +12,7 @@ from vllm.config.compilation import CUDAGraphMode from vllm.triton_utils import tl, triton from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadataBuilder from vllm.v1.attention.backends.mamba2_attn import Mamba2AttentionMetadataBuilder +from vllm.v1.attention.backends.short_conv_attn import ShortConvAttentionMetadataBuilder from vllm.v1.core.sched.output import NewRequestData from vllm.v1.kv_cache_interface import KVCacheConfig, MambaSpec from vllm.v1.utils import CpuGpuBuffer @@ -48,7 +49,11 @@ class MambaHybridAttnMetadata(ModelSpecificAttnMetadata): ) -> dict[str, Any]: if not isinstance( attn_metadata_builder, - (Mamba2AttentionMetadataBuilder, GDNAttentionMetadataBuilder), + ( + Mamba2AttentionMetadataBuilder, + GDNAttentionMetadataBuilder, + ShortConvAttentionMetadataBuilder, + ), ): return {} return { diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index 4ef7f5860a..8a0e27d3c8 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -148,6 +148,7 @@ from vllm.v1.attention.backends.linear_attn import ( BailingLinearAttentionMetadataBuilder, ) from vllm.v1.attention.backends.mamba2_attn import Mamba2AttentionMetadataBuilder +from vllm.v1.attention.backends.short_conv_attn import ShortConvAttentionMetadataBuilder from vllm.v1.attention.backends.utils import ( NULL_BLOCK_ID, create_fast_prefill_custom_backend, @@ -2572,6 +2573,7 @@ class GPUModelRunner( Mamba2AttentionMetadataBuilder, GDNAttentionMetadataBuilder, BailingLinearAttentionMetadataBuilder, + ShortConvAttentionMetadataBuilder, ), ): assert ubid is None, (