From d0d6e5f3a26f484786e7f20490152e911fd4e220 Mon Sep 17 00:00:00 2001 From: Vadim Gimpelson <156319763+vadiklyutiy@users.noreply.github.com> Date: Mon, 5 Oct 2026 16:49:47 +0400 Subject: [PATCH] [Bugfix] Fix Mamba page size AssertionError with spec decoding on GraniteMoeHybrid, FalconH1 and Zamba2 (#59975) Signed-off-by: Vadim Gimpelson Co-authored-by: Claude --- .../test_attention_backends_selection.py | 43 +++++++++++++++++++ vllm/model_executor/models/falcon_h1.py | 1 + .../model_executor/models/granitemoehybrid.py | 1 + vllm/model_executor/models/zamba2.py | 1 + 4 files changed, 46 insertions(+) diff --git a/tests/v1/attention/test_attention_backends_selection.py b/tests/v1/attention/test_attention_backends_selection.py index 2695517e14..1d89027317 100644 --- a/tests/v1/attention/test_attention_backends_selection.py +++ b/tests/v1/attention/test_attention_backends_selection.py @@ -13,7 +13,11 @@ from vllm.model_executor.layers.mamba.linear.minimax_linear_attn import ( ) from vllm.model_executor.layers.mamba.mamba_mixer import MambaMixer from vllm.model_executor.layers.mamba.mamba_mixer2 import MambaMixer2 +from vllm.model_executor.layers.mamba.mamba_utils import MambaStateShapeCalculator from vllm.model_executor.layers.mamba.short_conv import ShortConv +from vllm.model_executor.models.falcon_h1 import FalconH1ForCausalLM +from vllm.model_executor.models.granitemoehybrid import GraniteMoeHybridForCausalLM +from vllm.model_executor.models.zamba2 import Zamba2ForCausalLM from vllm.v1.attention.backends.linear_attn import LinearAttentionBackend from vllm.v1.attention.backends.mamba1_attn import Mamba1AttentionBackend from vllm.v1.attention.backends.mamba2_attn import Mamba2AttentionBackend @@ -47,6 +51,45 @@ def test_replayssm_does_not_reserve_speculative_state_blocks( assert spec.num_speculative_blocks == expected_blocks +@pytest.mark.parametrize( + "model_cls", [FalconH1ForCausalLM, GraniteMoeHybridForCausalLM, Zamba2ForCausalLM] +) +def test_mamba2_config_state_shape_includes_speculative_tokens(model_cls): + """The config-time shape sizes the padded Mamba page, so it must match + MambaMixer2.get_state_shape, whose conv state grows with spec tokens.""" + hf_config = SimpleNamespace( + hidden_size=128, + mamba_expand=2, + mamba_d_ssm=None, + mamba_n_groups=1, + mamba_ngroups=1, + mamba_n_heads=8, + n_mamba_heads=8, + mamba_d_head=32, + mamba_headdim=32, + mamba_d_state=16, + mamba_d_conv=4, + ) + vllm_config = SimpleNamespace( + model_config=SimpleNamespace(hf_config=hf_config), + parallel_config=SimpleNamespace(tensor_parallel_size=1), + num_speculative_tokens=2, + ) + + assert model_cls.get_mamba_state_shape_from_config( + vllm_config + ) == MambaStateShapeCalculator.mamba2_state_shape( + tp_world_size=1, + intermediate_size=256, + n_groups=1, + num_heads=8, + head_dim=32, + state_size=16, + conv_kernel=4, + num_spec=2, + ) + + @pytest.mark.parametrize( "layer_class, init_kwargs, expected_backend, expected_mamba_type", [ diff --git a/vllm/model_executor/models/falcon_h1.py b/vllm/model_executor/models/falcon_h1.py index f18f938774..203a3f3ec3 100644 --- a/vllm/model_executor/models/falcon_h1.py +++ b/vllm/model_executor/models/falcon_h1.py @@ -569,6 +569,7 @@ class FalconH1ForCausalLM( head_dim=hf_config.mamba_d_head, state_size=hf_config.mamba_d_state, conv_kernel=hf_config.mamba_d_conv, + num_spec=vllm_config.num_speculative_tokens, ) @classmethod diff --git a/vllm/model_executor/models/granitemoehybrid.py b/vllm/model_executor/models/granitemoehybrid.py index da7cd0478b..84a981e6bb 100644 --- a/vllm/model_executor/models/granitemoehybrid.py +++ b/vllm/model_executor/models/granitemoehybrid.py @@ -647,6 +647,7 @@ class GraniteMoeHybridForCausalLM( head_dim=hf_config.mamba_d_head, state_size=hf_config.mamba_d_state, conv_kernel=hf_config.mamba_d_conv, + num_spec=vllm_config.num_speculative_tokens, ) @classmethod diff --git a/vllm/model_executor/models/zamba2.py b/vllm/model_executor/models/zamba2.py index 6506c0776e..2e90da86ea 100644 --- a/vllm/model_executor/models/zamba2.py +++ b/vllm/model_executor/models/zamba2.py @@ -896,6 +896,7 @@ class Zamba2ForCausalLM(nn.Module, HasInnerState, IsHybrid, SupportsMambaPrefixC head_dim=hf_config.mamba_headdim, state_size=hf_config.mamba_d_state, conv_kernel=hf_config.mamba_d_conv, + num_spec=vllm_config.num_speculative_tokens, ) @classmethod