[Bugfix][Model Runner V2] Restore multimodal draft capability detection (#50417)

Signed-off-by: Raphael Rialland <raphael.rialland@mistral.ai>
This commit is contained in:
Raphaël Rialland
2026-08-03 14:18:14 -07:00
committed by GitHub
parent 42ab184ea7
commit 4a3447d200
12 changed files with 317 additions and 58 deletions
+4
View File
@@ -6,12 +6,14 @@ from .interfaces import (
SupportsLoRA,
SupportsMRoPE,
SupportsMultiModal,
SupportsMultiModalEmbeddings,
SupportsPP,
SupportsTranscription,
has_inner_state,
supports_lora,
supports_mrope,
supports_multimodal,
supports_multimodal_embeddings,
supports_pp,
supports_transcription,
)
@@ -35,6 +37,8 @@ __all__ = [
"supports_lora",
"SupportsMultiModal",
"supports_multimodal",
"SupportsMultiModalEmbeddings",
"supports_multimodal_embeddings",
"SupportsMRoPE",
"supports_mrope",
"SupportsPP",
+34 -1
View File
@@ -110,7 +110,22 @@ _language_model_by_module = dict[nn.Module, "VllmModel"]()
@runtime_checkable
class SupportsMultiModal(Protocol):
class SupportsMultiModalEmbeddings(Protocol):
"""The interface for models that can merge external multimodal embeddings."""
supports_multimodal_embeddings: ClassVar[Literal[True]] = True
def embed_input_ids(
self,
input_ids: Tensor,
multimodal_embeddings: MultiModalEmbeddings | None = None,
*,
is_multimodal: Tensor | None = None,
) -> Tensor: ...
@runtime_checkable
class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
"""The interface required for all multi-modal models."""
supports_multimodal: ClassVar[Literal[True]] = True
@@ -488,6 +503,24 @@ def supports_multimodal(
return getattr(model, "supports_multimodal", False)
@overload
def supports_multimodal_embeddings(
model: type[object],
) -> TypeIs[type[SupportsMultiModalEmbeddings]]: ...
@overload
def supports_multimodal_embeddings(
model: object,
) -> TypeIs[SupportsMultiModalEmbeddings]: ...
def supports_multimodal_embeddings(
model: type[object] | object,
) -> TypeIs[type[SupportsMultiModalEmbeddings]] | TypeIs[SupportsMultiModalEmbeddings]:
return getattr(model, "supports_multimodal_embeddings", False)
def supports_multimodal_raw_input_only(model: type[object] | object) -> bool:
return getattr(model, "supports_multimodal_raw_input_only", False)
+2 -2
View File
@@ -35,7 +35,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
from vllm.model_executor.models.llama4 import Llama4DecoderLayer, Llama4ForCausalLM
from vllm.model_executor.models.utils import extract_layer_index
from .interfaces import SupportsMultiModal
from .interfaces import SupportsMultiModal, SupportsMultiModalEmbeddings
from .utils import (
AutoWeightsLoader,
WeightsMapper,
@@ -152,7 +152,7 @@ class LlamaModel(nn.Module):
}
class EagleLlama4ForCausalLM(Llama4ForCausalLM):
class EagleLlama4ForCausalLM(Llama4ForCausalLM, SupportsMultiModalEmbeddings):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
nn.Module.__init__(self)
self.config = vllm_config.speculative_config.draft_model_config.hf_config
+5 -2
View File
@@ -14,7 +14,10 @@ from vllm.model_executor.layers.linear import RowParallelLinear
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
from vllm.model_executor.models.interfaces import MultiModalEmbeddings
from vllm.model_executor.models.interfaces import (
MultiModalEmbeddings,
SupportsMultiModalEmbeddings,
)
from vllm.model_executor.models.llama import LlamaConfig
from vllm.model_executor.models.mistral import (
MistralDecoderLayer,
@@ -109,7 +112,7 @@ class EagleMistralModel(MistralModel):
return hidden_states, hidden_states
class EagleMistralForCausalLM(MistralForCausalLM):
class EagleMistralForCausalLM(MistralForCausalLM, SupportsMultiModalEmbeddings):
mistral_mapping = MistralForCausalLM.mistral_mapping | {
"eagle_linear": "model.fc",
}
@@ -22,7 +22,7 @@ from vllm.model_executor.models.deepseek_v2 import (
)
from vllm.model_executor.models.mistral_large_3 import MistralLarge3ForCausalLM
from .interfaces import SupportsMultiModal
from .interfaces import SupportsMultiModal, SupportsMultiModalEmbeddings
from .utils import WeightsMapper, make_empty_intermediate_tensors_factory, maybe_prefix
logger = init_logger(__name__)
@@ -107,7 +107,9 @@ class EagleMistralLarge3Model(DeepseekV2Model):
return output
class EagleMistralLarge3ForCausalLM(MistralLarge3ForCausalLM):
class EagleMistralLarge3ForCausalLM(
MistralLarge3ForCausalLM, SupportsMultiModalEmbeddings
):
hf_to_vllm_mapper = MistralLarge3ForCausalLM.hf_to_vllm_mapper | WeightsMapper(
orig_to_new_regex={
regex.compile(r"\Aeagle_linear\.weight\Z"): r"model.fc.weight",