[Model][Spec Decode] Nemotron-H MTP and Mamba Speculative Decoding Support (#33726)
Signed-off-by: Shahar Mor <smor@nvidia.com> Signed-off-by: Benjamin Chislett <bchislett@nvidia.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Shahar Mor <smor@nvidia.com> Co-authored-by: Roi Koren <roik@nvidia.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
This commit is contained in:
co-authored by
Shahar Mor
Roi Koren
Lucas Wilkinson
parent
a9e15e040d
commit
f5972a872f
@@ -41,14 +41,6 @@ class MambaBase(AttentionLayerBase):
|
||||
pass
|
||||
|
||||
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec | None:
|
||||
if (
|
||||
vllm_config.speculative_config is not None
|
||||
and vllm_config.model_config.hf_config.model_type
|
||||
not in ["qwen3_next", "qwen3_5", "qwen3_5_moe"]
|
||||
):
|
||||
raise NotImplementedError(
|
||||
"Mamba with speculative decoding is not supported yet."
|
||||
)
|
||||
mamba_block_size = vllm_config.cache_config.mamba_block_size
|
||||
page_size_padded = vllm_config.cache_config.mamba_page_size_padded
|
||||
return MambaSpec(
|
||||
|
||||
Reference in New Issue
Block a user