[Bugfix] Fix Mamba page size AssertionError with spec decoding on GraniteMoeHybrid, FalconH1 and Zamba2 (#59975)
Signed-off-by: Vadim Gimpelson <vadim.gimpelson@gmail.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude
parent
51eeb0c58f
commit
d0d6e5f3a2
@@ -13,7 +13,11 @@ from vllm.model_executor.layers.mamba.linear.minimax_linear_attn import (
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.mamba_mixer import MambaMixer
|
||||
from vllm.model_executor.layers.mamba.mamba_mixer2 import MambaMixer2
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import MambaStateShapeCalculator
|
||||
from vllm.model_executor.layers.mamba.short_conv import ShortConv
|
||||
from vllm.model_executor.models.falcon_h1 import FalconH1ForCausalLM
|
||||
from vllm.model_executor.models.granitemoehybrid import GraniteMoeHybridForCausalLM
|
||||
from vllm.model_executor.models.zamba2 import Zamba2ForCausalLM
|
||||
from vllm.v1.attention.backends.linear_attn import LinearAttentionBackend
|
||||
from vllm.v1.attention.backends.mamba1_attn import Mamba1AttentionBackend
|
||||
from vllm.v1.attention.backends.mamba2_attn import Mamba2AttentionBackend
|
||||
@@ -47,6 +51,45 @@ def test_replayssm_does_not_reserve_speculative_state_blocks(
|
||||
assert spec.num_speculative_blocks == expected_blocks
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_cls", [FalconH1ForCausalLM, GraniteMoeHybridForCausalLM, Zamba2ForCausalLM]
|
||||
)
|
||||
def test_mamba2_config_state_shape_includes_speculative_tokens(model_cls):
|
||||
"""The config-time shape sizes the padded Mamba page, so it must match
|
||||
MambaMixer2.get_state_shape, whose conv state grows with spec tokens."""
|
||||
hf_config = SimpleNamespace(
|
||||
hidden_size=128,
|
||||
mamba_expand=2,
|
||||
mamba_d_ssm=None,
|
||||
mamba_n_groups=1,
|
||||
mamba_ngroups=1,
|
||||
mamba_n_heads=8,
|
||||
n_mamba_heads=8,
|
||||
mamba_d_head=32,
|
||||
mamba_headdim=32,
|
||||
mamba_d_state=16,
|
||||
mamba_d_conv=4,
|
||||
)
|
||||
vllm_config = SimpleNamespace(
|
||||
model_config=SimpleNamespace(hf_config=hf_config),
|
||||
parallel_config=SimpleNamespace(tensor_parallel_size=1),
|
||||
num_speculative_tokens=2,
|
||||
)
|
||||
|
||||
assert model_cls.get_mamba_state_shape_from_config(
|
||||
vllm_config
|
||||
) == MambaStateShapeCalculator.mamba2_state_shape(
|
||||
tp_world_size=1,
|
||||
intermediate_size=256,
|
||||
n_groups=1,
|
||||
num_heads=8,
|
||||
head_dim=32,
|
||||
state_size=16,
|
||||
conv_kernel=4,
|
||||
num_spec=2,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"layer_class, init_kwargs, expected_backend, expected_mamba_type",
|
||||
[
|
||||
|
||||
@@ -569,6 +569,7 @@ class FalconH1ForCausalLM(
|
||||
head_dim=hf_config.mamba_d_head,
|
||||
state_size=hf_config.mamba_d_state,
|
||||
conv_kernel=hf_config.mamba_d_conv,
|
||||
num_spec=vllm_config.num_speculative_tokens,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
|
||||
@@ -647,6 +647,7 @@ class GraniteMoeHybridForCausalLM(
|
||||
head_dim=hf_config.mamba_d_head,
|
||||
state_size=hf_config.mamba_d_state,
|
||||
conv_kernel=hf_config.mamba_d_conv,
|
||||
num_spec=vllm_config.num_speculative_tokens,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
|
||||
@@ -896,6 +896,7 @@ class Zamba2ForCausalLM(nn.Module, HasInnerState, IsHybrid, SupportsMambaPrefixC
|
||||
head_dim=hf_config.mamba_headdim,
|
||||
state_size=hf_config.mamba_d_state,
|
||||
conv_kernel=hf_config.mamba_d_conv,
|
||||
num_spec=vllm_config.num_speculative_tokens,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
|
||||
Reference in New Issue
Block a user