[Bugfix] Fix Mamba page size AssertionError with spec decoding on GraniteMoeHybrid, FalconH1 and Zamba2 (#59975)
Signed-off-by: Vadim Gimpelson <vadim.gimpelson@gmail.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude
parent
51eeb0c58f
commit
d0d6e5f3a2
@@ -569,6 +569,7 @@ class FalconH1ForCausalLM(
|
||||
head_dim=hf_config.mamba_d_head,
|
||||
state_size=hf_config.mamba_d_state,
|
||||
conv_kernel=hf_config.mamba_d_conv,
|
||||
num_spec=vllm_config.num_speculative_tokens,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
|
||||
@@ -647,6 +647,7 @@ class GraniteMoeHybridForCausalLM(
|
||||
head_dim=hf_config.mamba_d_head,
|
||||
state_size=hf_config.mamba_d_state,
|
||||
conv_kernel=hf_config.mamba_d_conv,
|
||||
num_spec=vllm_config.num_speculative_tokens,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
|
||||
@@ -896,6 +896,7 @@ class Zamba2ForCausalLM(nn.Module, HasInnerState, IsHybrid, SupportsMambaPrefixC
|
||||
head_dim=hf_config.mamba_headdim,
|
||||
state_size=hf_config.mamba_d_state,
|
||||
conv_kernel=hf_config.mamba_d_conv,
|
||||
num_spec=vllm_config.num_speculative_tokens,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
|
||||
Reference in New Issue
Block a user