[Bugfix] Fix speculative decoding for short_conv (LFM2) models (#50272)
Signed-off-by: jens.luecke <jens.luecke@liquid.ai> Co-authored-by: Tonoken3 <sakamakismile@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Tonoken3
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
704f12aa1e
commit
da329cc303
@@ -12,6 +12,7 @@ from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadataBuilder
|
||||
from vllm.v1.attention.backends.mamba2_attn import Mamba2AttentionMetadataBuilder
|
||||
from vllm.v1.attention.backends.short_conv_attn import ShortConvAttentionMetadataBuilder
|
||||
from vllm.v1.core.sched.output import NewRequestData
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig, MambaSpec
|
||||
from vllm.v1.utils import CpuGpuBuffer
|
||||
@@ -48,7 +49,11 @@ class MambaHybridAttnMetadata(ModelSpecificAttnMetadata):
|
||||
) -> dict[str, Any]:
|
||||
if not isinstance(
|
||||
attn_metadata_builder,
|
||||
(Mamba2AttentionMetadataBuilder, GDNAttentionMetadataBuilder),
|
||||
(
|
||||
Mamba2AttentionMetadataBuilder,
|
||||
GDNAttentionMetadataBuilder,
|
||||
ShortConvAttentionMetadataBuilder,
|
||||
),
|
||||
):
|
||||
return {}
|
||||
return {
|
||||
|
||||
Reference in New Issue
Block a user