[Bugfix][Model Runner V2] Restore multimodal draft capability detection (#50417)
Signed-off-by: Raphael Rialland <raphael.rialland@mistral.ai>
This commit is contained in:
@@ -6,12 +6,14 @@ from .interfaces import (
|
||||
SupportsLoRA,
|
||||
SupportsMRoPE,
|
||||
SupportsMultiModal,
|
||||
SupportsMultiModalEmbeddings,
|
||||
SupportsPP,
|
||||
SupportsTranscription,
|
||||
has_inner_state,
|
||||
supports_lora,
|
||||
supports_mrope,
|
||||
supports_multimodal,
|
||||
supports_multimodal_embeddings,
|
||||
supports_pp,
|
||||
supports_transcription,
|
||||
)
|
||||
@@ -35,6 +37,8 @@ __all__ = [
|
||||
"supports_lora",
|
||||
"SupportsMultiModal",
|
||||
"supports_multimodal",
|
||||
"SupportsMultiModalEmbeddings",
|
||||
"supports_multimodal_embeddings",
|
||||
"SupportsMRoPE",
|
||||
"supports_mrope",
|
||||
"SupportsPP",
|
||||
|
||||
@@ -110,7 +110,22 @@ _language_model_by_module = dict[nn.Module, "VllmModel"]()
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class SupportsMultiModal(Protocol):
|
||||
class SupportsMultiModalEmbeddings(Protocol):
|
||||
"""The interface for models that can merge external multimodal embeddings."""
|
||||
|
||||
supports_multimodal_embeddings: ClassVar[Literal[True]] = True
|
||||
|
||||
def embed_input_ids(
|
||||
self,
|
||||
input_ids: Tensor,
|
||||
multimodal_embeddings: MultiModalEmbeddings | None = None,
|
||||
*,
|
||||
is_multimodal: Tensor | None = None,
|
||||
) -> Tensor: ...
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
|
||||
"""The interface required for all multi-modal models."""
|
||||
|
||||
supports_multimodal: ClassVar[Literal[True]] = True
|
||||
@@ -488,6 +503,24 @@ def supports_multimodal(
|
||||
return getattr(model, "supports_multimodal", False)
|
||||
|
||||
|
||||
@overload
|
||||
def supports_multimodal_embeddings(
|
||||
model: type[object],
|
||||
) -> TypeIs[type[SupportsMultiModalEmbeddings]]: ...
|
||||
|
||||
|
||||
@overload
|
||||
def supports_multimodal_embeddings(
|
||||
model: object,
|
||||
) -> TypeIs[SupportsMultiModalEmbeddings]: ...
|
||||
|
||||
|
||||
def supports_multimodal_embeddings(
|
||||
model: type[object] | object,
|
||||
) -> TypeIs[type[SupportsMultiModalEmbeddings]] | TypeIs[SupportsMultiModalEmbeddings]:
|
||||
return getattr(model, "supports_multimodal_embeddings", False)
|
||||
|
||||
|
||||
def supports_multimodal_raw_input_only(model: type[object] | object) -> bool:
|
||||
return getattr(model, "supports_multimodal_raw_input_only", False)
|
||||
|
||||
|
||||
@@ -35,7 +35,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
from vllm.model_executor.models.llama4 import Llama4DecoderLayer, Llama4ForCausalLM
|
||||
from vllm.model_executor.models.utils import extract_layer_index
|
||||
|
||||
from .interfaces import SupportsMultiModal
|
||||
from .interfaces import SupportsMultiModal, SupportsMultiModalEmbeddings
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
WeightsMapper,
|
||||
@@ -152,7 +152,7 @@ class LlamaModel(nn.Module):
|
||||
}
|
||||
|
||||
|
||||
class EagleLlama4ForCausalLM(Llama4ForCausalLM):
|
||||
class EagleLlama4ForCausalLM(Llama4ForCausalLM, SupportsMultiModalEmbeddings):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
nn.Module.__init__(self)
|
||||
self.config = vllm_config.speculative_config.draft_model_config.hf_config
|
||||
|
||||
@@ -14,7 +14,10 @@ from vllm.model_executor.layers.linear import RowParallelLinear
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
|
||||
from vllm.model_executor.models.interfaces import MultiModalEmbeddings
|
||||
from vllm.model_executor.models.interfaces import (
|
||||
MultiModalEmbeddings,
|
||||
SupportsMultiModalEmbeddings,
|
||||
)
|
||||
from vllm.model_executor.models.llama import LlamaConfig
|
||||
from vllm.model_executor.models.mistral import (
|
||||
MistralDecoderLayer,
|
||||
@@ -109,7 +112,7 @@ class EagleMistralModel(MistralModel):
|
||||
return hidden_states, hidden_states
|
||||
|
||||
|
||||
class EagleMistralForCausalLM(MistralForCausalLM):
|
||||
class EagleMistralForCausalLM(MistralForCausalLM, SupportsMultiModalEmbeddings):
|
||||
mistral_mapping = MistralForCausalLM.mistral_mapping | {
|
||||
"eagle_linear": "model.fc",
|
||||
}
|
||||
|
||||
@@ -22,7 +22,7 @@ from vllm.model_executor.models.deepseek_v2 import (
|
||||
)
|
||||
from vllm.model_executor.models.mistral_large_3 import MistralLarge3ForCausalLM
|
||||
|
||||
from .interfaces import SupportsMultiModal
|
||||
from .interfaces import SupportsMultiModal, SupportsMultiModalEmbeddings
|
||||
from .utils import WeightsMapper, make_empty_intermediate_tensors_factory, maybe_prefix
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -107,7 +107,9 @@ class EagleMistralLarge3Model(DeepseekV2Model):
|
||||
return output
|
||||
|
||||
|
||||
class EagleMistralLarge3ForCausalLM(MistralLarge3ForCausalLM):
|
||||
class EagleMistralLarge3ForCausalLM(
|
||||
MistralLarge3ForCausalLM, SupportsMultiModalEmbeddings
|
||||
):
|
||||
hf_to_vllm_mapper = MistralLarge3ForCausalLM.hf_to_vllm_mapper | WeightsMapper(
|
||||
orig_to_new_regex={
|
||||
regex.compile(r"\Aeagle_linear\.weight\Z"): r"model.fc.weight",
|
||||
|
||||
Reference in New Issue
Block a user