[Bugfix][Spec Decode] Fix multimodal embedding merge in Llama 4 and Mistral Large 3 EAGLE drafters (#57633)

Signed-off-by: Nishant Revur <kumrfni@amazon.com>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Nishant Revur
2026-10-09 01:44:50 +00:00
committed by GitHub
co-authored by Claude
parent 4274ae956c
commit 581692cc0f
3 changed files with 71 additions and 5 deletions
+21 -2
View File
@@ -36,10 +36,15 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
from vllm.model_executor.models.llama4 import Llama4DecoderLayer, Llama4ForCausalLM
from vllm.model_executor.models.utils import extract_layer_index
from .interfaces import SupportsMultiModal, SupportsMultiModalEmbeddings
from .interfaces import (
MultiModalEmbeddings,
SupportsMultiModalEmbeddings,
_require_is_multimodal,
)
from .utils import (
AutoWeightsLoader,
WeightsMapper,
_merge_multimodal_embeddings,
maybe_prefix,
process_eagle_weight,
)
@@ -204,7 +209,21 @@ class EagleLlama4ForCausalLM(_EagleLlama4ForCausalLMBase, SupportsMultiModalEmbe
def get_language_model(self) -> torch.nn.Module:
return self.model
embed_input_ids = SupportsMultiModal.embed_input_ids # type: ignore
def embed_input_ids(
self,
input_ids: torch.Tensor,
multimodal_embeddings: MultiModalEmbeddings | None = None,
*,
is_multimodal: torch.Tensor | None = None,
) -> torch.Tensor:
inputs_embeds = self.model.embed_input_ids(input_ids)
if multimodal_embeddings is None or len(multimodal_embeddings) == 0:
return inputs_embeds
return _merge_multimodal_embeddings(
inputs_embeds=inputs_embeds,
multimodal_embeddings=multimodal_embeddings,
is_multimodal=_require_is_multimodal(is_multimodal),
)
def forward(
self,
@@ -23,8 +23,17 @@ from vllm.model_executor.models.deepseek_v2 import (
)
from vllm.model_executor.models.mistral_large_3 import MistralLarge3ForCausalLM
from .interfaces import SupportsMultiModal, SupportsMultiModalEmbeddings
from .utils import WeightsMapper, make_empty_intermediate_tensors_factory, maybe_prefix
from .interfaces import (
MultiModalEmbeddings,
SupportsMultiModalEmbeddings,
_require_is_multimodal,
)
from .utils import (
WeightsMapper,
_merge_multimodal_embeddings,
make_empty_intermediate_tensors_factory,
maybe_prefix,
)
logger = init_logger(__name__)
@@ -148,7 +157,21 @@ class EagleMistralLarge3ForCausalLM(
def get_language_model(self) -> torch.nn.Module:
return self.model
embed_input_ids = SupportsMultiModal.embed_input_ids # type: ignore
def embed_input_ids(
self,
input_ids: torch.Tensor,
multimodal_embeddings: MultiModalEmbeddings | None = None,
*,
is_multimodal: torch.Tensor | None = None,
) -> torch.Tensor:
inputs_embeds = self.model.embed_input_ids(input_ids)
if multimodal_embeddings is None or len(multimodal_embeddings) == 0:
return inputs_embeds
return _merge_multimodal_embeddings(
inputs_embeds=inputs_embeds,
multimodal_embeddings=multimodal_embeddings,
is_multimodal=_require_is_multimodal(is_multimodal),
)
def forward(
self,