[Bugfix][Model Runner V2] Restore multimodal draft capability detection (#50417)

Signed-off-by: Raphael Rialland <raphael.rialland@mistral.ai>
This commit is contained in:
Raphaël Rialland
2026-08-03 14:18:14 -07:00
committed by GitHub
parent 42ab184ea7
commit 4a3447d200
12 changed files with 317 additions and 58 deletions
@@ -4,18 +4,32 @@
from contextlib import nullcontext
from types import SimpleNamespace
import pytest
import torch
from vllm.config.compilation import CUDAGraphMode
from vllm.model_executor.models import supports_multimodal_embeddings
from vllm.model_executor.models.exaone4_5_mtp import Exaone4_5_MTP
from vllm.model_executor.models.llama4_eagle import EagleLlama4ForCausalLM
from vllm.model_executor.models.llama_eagle3 import Eagle3LlamaForCausalLM
from vllm.model_executor.models.mistral_eagle import EagleMistralForCausalLM
from vllm.model_executor.models.mistral_large_3_eagle import (
EagleMistralLarge3ForCausalLM,
)
from vllm.v1.worker.gpu.spec_decode import speculator as base_spec_module
from vllm.v1.worker.gpu.spec_decode.autoregressive import speculator as spec_module
from vllm.v1.worker.gpu.spec_decode.autoregressive.speculator import (
AutoRegressiveSpeculator,
)
from vllm.v1.worker.gpu.spec_decode.multi_module_mtp.speculator import (
MultiModuleMTPSpeculator,
)
from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator
class _TestSpeculator(AutoRegressiveSpeculator):
def load_draft_model(self, target_model, target_attn_layer_names):
raise NotImplementedError
return self.test_draft_model
class _DraftModel(torch.nn.Module):
@@ -27,6 +41,43 @@ class _DraftModel(torch.nn.Module):
return self.output
class _MultimodalDraftModel(torch.nn.Module):
supports_multimodal_embeddings = True
def embed_input_ids(
self,
input_ids,
multimodal_embeddings=None,
*,
is_multimodal=None,
):
raise AssertionError("embed_input_ids should not be called during loading")
class _TextOnlyDraftModel(torch.nn.Module):
def embed_input_ids(
self,
input_ids,
multimodal_embeddings=None,
*,
is_multimodal=None,
):
raise AssertionError("embed_input_ids should not be called during loading")
def _mock_base_model_load(monkeypatch):
monkeypatch.setattr(
base_spec_module,
"get_layers_from_vllm_config",
lambda *args, **kwargs: {},
)
monkeypatch.setattr(
DraftModelSpeculator,
"_validate_local_argmax_reduction",
lambda self: None,
)
def _make_speculator(
monkeypatch,
output: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
@@ -49,6 +100,153 @@ def _make_speculator(
return speculator
def test_mm_support_configured_after_model_load(monkeypatch):
target_model_config = object()
draft_model_config = object()
vllm_config = SimpleNamespace(model_config=target_model_config)
draft_model = _MultimodalDraftModel()
def init_base(speculator, vllm_config, device):
speculator.vllm_config = vllm_config
speculator.device = device
speculator.max_num_tokens = 4
speculator.max_num_reqs = 2
speculator.hidden_size = 3
speculator.dtype = torch.float32
speculator.draft_model_config = draft_model_config
speculator.supports_mm_inputs = False
checked_configs = []
def supports_multimodal_inputs(model_config):
checked_configs.append(model_config)
return True
monkeypatch.setattr(DraftModelSpeculator, "__init__", init_base)
_mock_base_model_load(monkeypatch)
monkeypatch.setattr(
base_spec_module.MULTIMODAL_REGISTRY,
"supports_multimodal_inputs",
supports_multimodal_inputs,
)
speculator = _TestSpeculator(vllm_config, torch.device("cpu"))
assert checked_configs == []
assert not speculator.supports_mm_inputs
assert speculator.inputs_embeds is None
speculator.test_draft_model = draft_model
speculator.load_model(torch.nn.Module())
assert checked_configs == [target_model_config]
assert speculator.supports_mm_inputs
assert speculator.inputs_embeds is not None
assert speculator.inputs_embeds.shape == (4, 3)
def test_load_model_keeps_mm_support_for_capable_drafter(monkeypatch):
speculator = object.__new__(_TestSpeculator)
speculator.supports_mm_inputs = False
speculator.inputs_embeds = None
speculator.vllm_config = SimpleNamespace(model_config=object())
speculator.max_num_tokens = 4
speculator.hidden_size = 3
speculator.dtype = torch.float32
speculator.device = torch.device("cpu")
draft_model = _MultimodalDraftModel()
speculator.test_draft_model = draft_model
_mock_base_model_load(monkeypatch)
monkeypatch.setattr(
base_spec_module.MULTIMODAL_REGISTRY,
"supports_multimodal_inputs",
lambda model_config: True,
)
speculator.load_model(torch.nn.Module())
assert speculator.supports_mm_inputs
assert speculator.inputs_embeds is not None
def test_load_model_disables_mm_support_for_text_only_drafter(monkeypatch):
speculator = object.__new__(_TestSpeculator)
speculator.supports_mm_inputs = False
speculator.inputs_embeds = None
speculator.vllm_config = SimpleNamespace(model_config=object())
draft_model = _TextOnlyDraftModel()
speculator.test_draft_model = draft_model
warning_messages = []
_mock_base_model_load(monkeypatch)
monkeypatch.setattr(
base_spec_module.MULTIMODAL_REGISTRY,
"supports_multimodal_inputs",
lambda model_config: True,
)
monkeypatch.setattr(
base_spec_module.logger,
"warning_once",
lambda message, *args: warning_messages.append(message % args),
)
speculator.load_model(torch.nn.Module())
assert not speculator.supports_mm_inputs
assert warning_messages == [
"Draft model _TextOnlyDraftModel does not support external multimodal "
"embeddings. Embeddings from the target model will not be passed to the "
"drafter; using text-only draft inputs instead."
]
def test_multi_module_mm_support_configured_after_model_load(monkeypatch):
speculator = object.__new__(MultiModuleMTPSpeculator)
speculator.supports_mm_inputs = False
speculator.inputs_embeds = None
speculator.cached_draft_input_embeds = None
speculator.vllm_config = SimpleNamespace(model_config=object())
speculator.max_num_tokens = 4
speculator.max_num_reqs = 2
speculator.num_speculative_steps = 3
speculator.hidden_size = 3
speculator.dtype = torch.float32
speculator.device = torch.device("cpu")
draft_model = _MultimodalDraftModel()
_mock_base_model_load(monkeypatch)
monkeypatch.setattr(
MultiModuleMTPSpeculator,
"load_draft_model",
lambda self, target_model, target_attn_layer_names: draft_model,
)
monkeypatch.setattr(
base_spec_module.MULTIMODAL_REGISTRY,
"supports_multimodal_inputs",
lambda model_config: True,
)
speculator.load_model(torch.nn.Module())
assert speculator.supports_mm_inputs
assert speculator.inputs_embeds is not None
assert speculator.inputs_embeds.shape == (4, 3)
assert speculator.cached_draft_input_embeds is not None
assert speculator.cached_draft_input_embeds.shape == (2, 2, 3)
@pytest.mark.parametrize(
("model_cls", "expected"),
[
(EagleLlama4ForCausalLM, True),
(EagleMistralForCausalLM, True),
(EagleMistralLarge3ForCausalLM, True),
(Exaone4_5_MTP, True),
(Eagle3LlamaForCausalLM, False),
],
)
def test_draft_model_multimodal_embedding_capability(model_cls, expected):
assert supports_multimodal_embeddings(model_cls) is expected
def test_run_model_unpacks_tuple_return_for_mtp(monkeypatch):
logits_hidden = torch.full((4, 3), 1.0)
feedback_hidden = torch.full((4, 3), 2.0)
+4
View File
@@ -6,12 +6,14 @@ from .interfaces import (
SupportsLoRA,
SupportsMRoPE,
SupportsMultiModal,
SupportsMultiModalEmbeddings,
SupportsPP,
SupportsTranscription,
has_inner_state,
supports_lora,
supports_mrope,
supports_multimodal,
supports_multimodal_embeddings,
supports_pp,
supports_transcription,
)
@@ -35,6 +37,8 @@ __all__ = [
"supports_lora",
"SupportsMultiModal",
"supports_multimodal",
"SupportsMultiModalEmbeddings",
"supports_multimodal_embeddings",
"SupportsMRoPE",
"supports_mrope",
"SupportsPP",
+34 -1
View File
@@ -110,7 +110,22 @@ _language_model_by_module = dict[nn.Module, "VllmModel"]()
@runtime_checkable
class SupportsMultiModal(Protocol):
class SupportsMultiModalEmbeddings(Protocol):
"""The interface for models that can merge external multimodal embeddings."""
supports_multimodal_embeddings: ClassVar[Literal[True]] = True
def embed_input_ids(
self,
input_ids: Tensor,
multimodal_embeddings: MultiModalEmbeddings | None = None,
*,
is_multimodal: Tensor | None = None,
) -> Tensor: ...
@runtime_checkable
class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
"""The interface required for all multi-modal models."""
supports_multimodal: ClassVar[Literal[True]] = True
@@ -488,6 +503,24 @@ def supports_multimodal(
return getattr(model, "supports_multimodal", False)
@overload
def supports_multimodal_embeddings(
model: type[object],
) -> TypeIs[type[SupportsMultiModalEmbeddings]]: ...
@overload
def supports_multimodal_embeddings(
model: object,
) -> TypeIs[SupportsMultiModalEmbeddings]: ...
def supports_multimodal_embeddings(
model: type[object] | object,
) -> TypeIs[type[SupportsMultiModalEmbeddings]] | TypeIs[SupportsMultiModalEmbeddings]:
return getattr(model, "supports_multimodal_embeddings", False)
def supports_multimodal_raw_input_only(model: type[object] | object) -> bool:
return getattr(model, "supports_multimodal_raw_input_only", False)
+2 -2
View File
@@ -35,7 +35,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
from vllm.model_executor.models.llama4 import Llama4DecoderLayer, Llama4ForCausalLM
from vllm.model_executor.models.utils import extract_layer_index
from .interfaces import SupportsMultiModal
from .interfaces import SupportsMultiModal, SupportsMultiModalEmbeddings
from .utils import (
AutoWeightsLoader,
WeightsMapper,
@@ -152,7 +152,7 @@ class LlamaModel(nn.Module):
}
class EagleLlama4ForCausalLM(Llama4ForCausalLM):
class EagleLlama4ForCausalLM(Llama4ForCausalLM, SupportsMultiModalEmbeddings):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
nn.Module.__init__(self)
self.config = vllm_config.speculative_config.draft_model_config.hf_config
+5 -2
View File
@@ -14,7 +14,10 @@ from vllm.model_executor.layers.linear import RowParallelLinear
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
from vllm.model_executor.models.interfaces import MultiModalEmbeddings
from vllm.model_executor.models.interfaces import (
MultiModalEmbeddings,
SupportsMultiModalEmbeddings,
)
from vllm.model_executor.models.llama import LlamaConfig
from vllm.model_executor.models.mistral import (
MistralDecoderLayer,
@@ -109,7 +112,7 @@ class EagleMistralModel(MistralModel):
return hidden_states, hidden_states
class EagleMistralForCausalLM(MistralForCausalLM):
class EagleMistralForCausalLM(MistralForCausalLM, SupportsMultiModalEmbeddings):
mistral_mapping = MistralForCausalLM.mistral_mapping | {
"eagle_linear": "model.fc",
}
@@ -22,7 +22,7 @@ from vllm.model_executor.models.deepseek_v2 import (
)
from vllm.model_executor.models.mistral_large_3 import MistralLarge3ForCausalLM
from .interfaces import SupportsMultiModal
from .interfaces import SupportsMultiModal, SupportsMultiModalEmbeddings
from .utils import WeightsMapper, make_empty_intermediate_tensors_factory, maybe_prefix
logger = init_logger(__name__)
@@ -107,7 +107,9 @@ class EagleMistralLarge3Model(DeepseekV2Model):
return output
class EagleMistralLarge3ForCausalLM(MistralLarge3ForCausalLM):
class EagleMistralLarge3ForCausalLM(
MistralLarge3ForCausalLM, SupportsMultiModalEmbeddings
):
hf_to_vllm_mapper = MistralLarge3ForCausalLM.hf_to_vllm_mapper | WeightsMapper(
orig_to_new_regex={
regex.compile(r"\Aeagle_linear\.weight\Z"): r"model.fc.weight",
+2 -1
View File
@@ -29,6 +29,7 @@ from vllm.model_executor.model_loader.mtp_validation import (
is_mtp_completeness_check_enabled,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.models.interfaces import SupportsMultiModalEmbeddings
from vllm.model_executor.models.utils import maybe_prefix
from vllm.sequence import IntermediateTensors
@@ -213,7 +214,7 @@ class InklingMultiTokenPredictor(nn.Module):
return hidden
class InklingMTP(nn.Module):
class InklingMTP(nn.Module, SupportsMultiModalEmbeddings):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
super().__init__()
assert vllm_config.speculative_config is not None
+2 -1
View File
@@ -33,6 +33,7 @@ from vllm.model_executor.model_loader.mtp_validation import (
is_mtp_completeness_check_enabled,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.models.interfaces import SupportsMultiModalEmbeddings
from vllm.model_executor.models.utils import maybe_prefix
from vllm.sequence import IntermediateTensors
@@ -242,7 +243,7 @@ class InklingMultiTokenPredictor(nn.Module):
return hidden
class InklingMTP(nn.Module):
class InklingMTP(nn.Module, SupportsMultiModalEmbeddings):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
super().__init__()
assert vllm_config.speculative_config is not None
+14 -13
View File
@@ -25,7 +25,10 @@ from vllm.forward_context import set_forward_context
from vllm.logger import init_logger
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
from vllm.model_executor.model_loader import get_model
from vllm.model_executor.models import supports_multimodal
from vllm.model_executor.models import (
supports_multimodal,
supports_multimodal_embeddings,
)
from vllm.model_executor.models.deepseek_eagle3 import Eagle3DeepseekV2ForCausalLM
from vllm.model_executor.models.interfaces import SupportsMultiModal
from vllm.model_executor.models.laguna_dflash import DFlashLagunaForCausalLM
@@ -1349,18 +1352,16 @@ class SpecDecodeBaseProposer:
if all_attn_layers[name].get_kv_cache_spec(self.vllm_config) is not None
}
if self.supports_mm_inputs:
# Even if the target model is multimodal, we can also use
# text-only draft models
try:
dummy_input_ids = torch.tensor([[1]], device=self.input_ids.device)
self.model.embed_input_ids(dummy_input_ids, multimodal_embeddings=None)
except (NotImplementedError, AttributeError, TypeError):
logger.warning(
"Draft model does not support multimodal inputs, "
"falling back to text-only mode"
)
self.supports_mm_inputs = False
# Even if the target model is multimodal, we can also use
# text-only draft models
if self.supports_mm_inputs and not supports_multimodal_embeddings(self.model):
logger.warning_once(
"Draft model %s does not support external multimodal embeddings. "
"Embeddings from the target model will not be passed to the "
"drafter; using text-only draft inputs instead.",
type(self.model).__name__,
)
self.supports_mm_inputs = False
if supports_multimodal(target_model):
# handle multimodality
@@ -3,12 +3,12 @@
from typing import Any
import torch
import torch.nn as nn
from vllm.config import VllmConfig
from vllm.config.compilation import CUDAGraphMode
from vllm.forward_context import BatchDescriptor, set_forward_context
from vllm.logger import init_logger
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.triton_utils import tl, triton
from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer
from vllm.v1.worker.gpu.cudagraph_utils import (
@@ -37,17 +37,23 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
self.max_num_reqs, dtype=torch.int64, device=device
)
self.supports_mm_inputs = MULTIMODAL_REGISTRY.supports_multimodal_inputs(
self.draft_model_config
)
if self.supports_mm_inputs:
self.inputs_embeds = torch.zeros(
self.max_num_tokens, self.hidden_size, dtype=self.dtype, device=device
)
self.inputs_embeds: torch.Tensor | None = None
self.prefill_cudagraph_manager: SpeculatorCudaGraphManager | None = None
self.decode_cudagraph_manager: SpeculatorCudaGraphManager | None = None
def load_model(self, target_model: nn.Module) -> None:
super().load_model(target_model)
if not self.supports_mm_inputs:
return
self.inputs_embeds = torch.zeros(
self.max_num_tokens,
self.hidden_size,
dtype=self.dtype,
device=self.device,
)
@property
def advance_draft_positions(self) -> bool:
"""
@@ -295,6 +301,7 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
):
inputs_embeds = None
if self.supports_mm_inputs:
assert self.inputs_embeds is not None
# Merge multimodal embeddings with input ids.
mm_embeds, is_mm_embed = mm_inputs or (None, None)
num_input_tokens = (
@@ -9,7 +9,6 @@ from vllm.config import VllmConfig
from vllm.config.compilation import CUDAGraphMode
from vllm.forward_context import BatchDescriptor, set_forward_context
from vllm.logger import init_logger
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.triton_utils import tl, triton
from vllm.v1.attention.backends.utils import PAD_SLOT_ID
from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer
@@ -39,25 +38,7 @@ class MultiModuleMTPSpeculator(DraftModelSpeculator):
self.max_num_reqs, dtype=torch.int64, device=device
)
self.supports_mm_inputs = MULTIMODAL_REGISTRY.supports_multimodal_inputs(
self.draft_model_config
)
# HACK: the Inkling MTP draft has no MM processor of its own (its draft
# config is flattened text-only), but it consumes the target's merged
# embeddings at draft prefill — treat it as MM-capable whenever the
# target is.
if (
not self.supports_mm_inputs
and self.draft_model_config.hf_config.model_type == "inkling_mtp"
):
self.supports_mm_inputs = MULTIMODAL_REGISTRY.supports_multimodal_inputs(
vllm_config.model_config
)
self.inputs_embeds: torch.Tensor | None = None
if self.supports_mm_inputs:
self.inputs_embeds = torch.zeros(
self.max_num_tokens, self.hidden_size, dtype=self.dtype, device=device
)
# Input id overrides for the last num_speculative_steps - 1 draft steps.
# Used by chunked-prefilling requests to swap in the future prefill token
@@ -80,14 +61,6 @@ class MultiModuleMTPSpeculator(DraftModelSpeculator):
device=self.device,
)
self.cached_draft_input_embeds: torch.Tensor | None = None
if self.supports_mm_inputs:
self.cached_draft_input_embeds = torch.zeros(
self.max_num_reqs,
self.num_speculative_steps - 1,
self.hidden_size,
dtype=self.dtype,
device=self.device,
)
self.cached_target_hidden_states = torch.zeros(
self.max_num_reqs,
self.num_speculative_steps - 1,
@@ -105,6 +78,25 @@ class MultiModuleMTPSpeculator(DraftModelSpeculator):
) -> nn.Module:
return load_eagle_model(target_model, self.vllm_config)
def load_model(self, target_model: nn.Module) -> None:
super().load_model(target_model)
if not self.supports_mm_inputs:
return
self.inputs_embeds = torch.zeros(
self.max_num_tokens,
self.hidden_size,
dtype=self.dtype,
device=self.device,
)
self.cached_draft_input_embeds = torch.zeros(
self.max_num_reqs,
self.num_speculative_steps - 1,
self.hidden_size,
dtype=self.dtype,
device=self.device,
)
def init_cudagraph_manager(self, cudagraph_mode: CUDAGraphMode) -> None:
# TODO(TheEpicDolphin): Support piecewise cudagraph for multi-module MTP.
if cudagraph_mode.has_piecewise_cudagraphs():
@@ -13,6 +13,8 @@ from vllm.config.compilation import CUDAGraphMode
from vllm.distributed.eplb.eplb_state import EplbState
from vllm.logger import init_logger
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
from vllm.model_executor.models import supports_multimodal_embeddings
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.worker.gpu.attn_utils import (
build_attn_metadata,
@@ -137,6 +139,8 @@ class DraftModelSpeculator(BaseSpeculator):
device=device,
)
self.supports_mm_inputs = False
@abstractmethod
def load_draft_model(
self,
@@ -164,6 +168,19 @@ class DraftModelSpeculator(BaseSpeculator):
)
self.draft_attn_layer_names = all_attn_layers - target_attn_layer_names
target_supports_mm = MULTIMODAL_REGISTRY.supports_multimodal_inputs(
self.vllm_config.model_config
)
draft_supports_mm = supports_multimodal_embeddings(self.model)
self.supports_mm_inputs = target_supports_mm and draft_supports_mm
if target_supports_mm and not draft_supports_mm:
logger.warning_once(
"Draft model %s does not support external multimodal embeddings. "
"Embeddings from the target model will not be passed to the "
"drafter; using text-only draft inputs instead.",
type(self.model).__name__,
)
def set_eplb_state(self, eplb_state: EplbState) -> None:
"""Inject EPLB state after construction."""
self.eplb_state = eplb_state