[Bugfix][Spec Decode] Fix multimodal embedding merge in Llama 4 and Mistral Large 3 EAGLE drafters (#57633)
Signed-off-by: Nishant Revur <kumrfni@amazon.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -36,10 +36,15 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
from vllm.model_executor.models.llama4 import Llama4DecoderLayer, Llama4ForCausalLM
|
||||
from vllm.model_executor.models.utils import extract_layer_index
|
||||
|
||||
from .interfaces import SupportsMultiModal, SupportsMultiModalEmbeddings
|
||||
from .interfaces import (
|
||||
MultiModalEmbeddings,
|
||||
SupportsMultiModalEmbeddings,
|
||||
_require_is_multimodal,
|
||||
)
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
WeightsMapper,
|
||||
_merge_multimodal_embeddings,
|
||||
maybe_prefix,
|
||||
process_eagle_weight,
|
||||
)
|
||||
@@ -204,7 +209,21 @@ class EagleLlama4ForCausalLM(_EagleLlama4ForCausalLMBase, SupportsMultiModalEmbe
|
||||
def get_language_model(self) -> torch.nn.Module:
|
||||
return self.model
|
||||
|
||||
embed_input_ids = SupportsMultiModal.embed_input_ids # type: ignore
|
||||
def embed_input_ids(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
multimodal_embeddings: MultiModalEmbeddings | None = None,
|
||||
*,
|
||||
is_multimodal: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
inputs_embeds = self.model.embed_input_ids(input_ids)
|
||||
if multimodal_embeddings is None or len(multimodal_embeddings) == 0:
|
||||
return inputs_embeds
|
||||
return _merge_multimodal_embeddings(
|
||||
inputs_embeds=inputs_embeds,
|
||||
multimodal_embeddings=multimodal_embeddings,
|
||||
is_multimodal=_require_is_multimodal(is_multimodal),
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
|
||||
@@ -23,8 +23,17 @@ from vllm.model_executor.models.deepseek_v2 import (
|
||||
)
|
||||
from vllm.model_executor.models.mistral_large_3 import MistralLarge3ForCausalLM
|
||||
|
||||
from .interfaces import SupportsMultiModal, SupportsMultiModalEmbeddings
|
||||
from .utils import WeightsMapper, make_empty_intermediate_tensors_factory, maybe_prefix
|
||||
from .interfaces import (
|
||||
MultiModalEmbeddings,
|
||||
SupportsMultiModalEmbeddings,
|
||||
_require_is_multimodal,
|
||||
)
|
||||
from .utils import (
|
||||
WeightsMapper,
|
||||
_merge_multimodal_embeddings,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
maybe_prefix,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -148,7 +157,21 @@ class EagleMistralLarge3ForCausalLM(
|
||||
def get_language_model(self) -> torch.nn.Module:
|
||||
return self.model
|
||||
|
||||
embed_input_ids = SupportsMultiModal.embed_input_ids # type: ignore
|
||||
def embed_input_ids(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
multimodal_embeddings: MultiModalEmbeddings | None = None,
|
||||
*,
|
||||
is_multimodal: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
inputs_embeds = self.model.embed_input_ids(input_ids)
|
||||
if multimodal_embeddings is None or len(multimodal_embeddings) == 0:
|
||||
return inputs_embeds
|
||||
return _merge_multimodal_embeddings(
|
||||
inputs_embeds=inputs_embeds,
|
||||
multimodal_embeddings=multimodal_embeddings,
|
||||
is_multimodal=_require_is_multimodal(is_multimodal),
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user