[Bugfix] Fix Mamba page size AssertionError with spec decoding on GraniteMoeHybrid, FalconH1 and Zamba2 (#59975)

Signed-off-by: Vadim Gimpelson <vadim.gimpelson@gmail.com>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Vadim Gimpelson
2026-10-05 12:49:47 +00:00
committed by GitHub
co-authored by Claude
parent 51eeb0c58f
commit d0d6e5f3a2
4 changed files with 46 additions and 0 deletions
@@ -13,7 +13,11 @@ from vllm.model_executor.layers.mamba.linear.minimax_linear_attn import (
)
from vllm.model_executor.layers.mamba.mamba_mixer import MambaMixer
from vllm.model_executor.layers.mamba.mamba_mixer2 import MambaMixer2
from vllm.model_executor.layers.mamba.mamba_utils import MambaStateShapeCalculator
from vllm.model_executor.layers.mamba.short_conv import ShortConv
from vllm.model_executor.models.falcon_h1 import FalconH1ForCausalLM
from vllm.model_executor.models.granitemoehybrid import GraniteMoeHybridForCausalLM
from vllm.model_executor.models.zamba2 import Zamba2ForCausalLM
from vllm.v1.attention.backends.linear_attn import LinearAttentionBackend
from vllm.v1.attention.backends.mamba1_attn import Mamba1AttentionBackend
from vllm.v1.attention.backends.mamba2_attn import Mamba2AttentionBackend
@@ -47,6 +51,45 @@ def test_replayssm_does_not_reserve_speculative_state_blocks(
assert spec.num_speculative_blocks == expected_blocks
@pytest.mark.parametrize(
"model_cls", [FalconH1ForCausalLM, GraniteMoeHybridForCausalLM, Zamba2ForCausalLM]
)
def test_mamba2_config_state_shape_includes_speculative_tokens(model_cls):
"""The config-time shape sizes the padded Mamba page, so it must match
MambaMixer2.get_state_shape, whose conv state grows with spec tokens."""
hf_config = SimpleNamespace(
hidden_size=128,
mamba_expand=2,
mamba_d_ssm=None,
mamba_n_groups=1,
mamba_ngroups=1,
mamba_n_heads=8,
n_mamba_heads=8,
mamba_d_head=32,
mamba_headdim=32,
mamba_d_state=16,
mamba_d_conv=4,
)
vllm_config = SimpleNamespace(
model_config=SimpleNamespace(hf_config=hf_config),
parallel_config=SimpleNamespace(tensor_parallel_size=1),
num_speculative_tokens=2,
)
assert model_cls.get_mamba_state_shape_from_config(
vllm_config
) == MambaStateShapeCalculator.mamba2_state_shape(
tp_world_size=1,
intermediate_size=256,
n_groups=1,
num_heads=8,
head_dim=32,
state_size=16,
conv_kernel=4,
num_spec=2,
)
@pytest.mark.parametrize(
"layer_class, init_kwargs, expected_backend, expected_mamba_type",
[
+1
View File
@@ -569,6 +569,7 @@ class FalconH1ForCausalLM(
head_dim=hf_config.mamba_d_head,
state_size=hf_config.mamba_d_state,
conv_kernel=hf_config.mamba_d_conv,
num_spec=vllm_config.num_speculative_tokens,
)
@classmethod
@@ -647,6 +647,7 @@ class GraniteMoeHybridForCausalLM(
head_dim=hf_config.mamba_d_head,
state_size=hf_config.mamba_d_state,
conv_kernel=hf_config.mamba_d_conv,
num_spec=vllm_config.num_speculative_tokens,
)
@classmethod
+1
View File
@@ -896,6 +896,7 @@ class Zamba2ForCausalLM(nn.Module, HasInnerState, IsHybrid, SupportsMambaPrefixC
head_dim=hf_config.mamba_headdim,
state_size=hf_config.mamba_d_state,
conv_kernel=hf_config.mamba_d_conv,
num_spec=vllm_config.num_speculative_tokens,
)
@classmethod