[Bugfix][Model Runner V2] Restore multimodal draft capability detection (#50417)

Signed-off-by: Raphael Rialland <raphael.rialland@mistral.ai>
This commit is contained in:
Raphaël Rialland
2026-08-03 14:18:14 -07:00
committed by GitHub
parent 42ab184ea7
commit 4a3447d200
12 changed files with 317 additions and 58 deletions
@@ -4,18 +4,32 @@
from contextlib import nullcontext
from types import SimpleNamespace
import pytest
import torch
from vllm.config.compilation import CUDAGraphMode
from vllm.model_executor.models import supports_multimodal_embeddings
from vllm.model_executor.models.exaone4_5_mtp import Exaone4_5_MTP
from vllm.model_executor.models.llama4_eagle import EagleLlama4ForCausalLM
from vllm.model_executor.models.llama_eagle3 import Eagle3LlamaForCausalLM
from vllm.model_executor.models.mistral_eagle import EagleMistralForCausalLM
from vllm.model_executor.models.mistral_large_3_eagle import (
EagleMistralLarge3ForCausalLM,
)
from vllm.v1.worker.gpu.spec_decode import speculator as base_spec_module
from vllm.v1.worker.gpu.spec_decode.autoregressive import speculator as spec_module
from vllm.v1.worker.gpu.spec_decode.autoregressive.speculator import (
AutoRegressiveSpeculator,
)
from vllm.v1.worker.gpu.spec_decode.multi_module_mtp.speculator import (
MultiModuleMTPSpeculator,
)
from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator
class _TestSpeculator(AutoRegressiveSpeculator):
def load_draft_model(self, target_model, target_attn_layer_names):
raise NotImplementedError
return self.test_draft_model
class _DraftModel(torch.nn.Module):
@@ -27,6 +41,43 @@ class _DraftModel(torch.nn.Module):
return self.output
class _MultimodalDraftModel(torch.nn.Module):
supports_multimodal_embeddings = True
def embed_input_ids(
self,
input_ids,
multimodal_embeddings=None,
*,
is_multimodal=None,
):
raise AssertionError("embed_input_ids should not be called during loading")
class _TextOnlyDraftModel(torch.nn.Module):
def embed_input_ids(
self,
input_ids,
multimodal_embeddings=None,
*,
is_multimodal=None,
):
raise AssertionError("embed_input_ids should not be called during loading")
def _mock_base_model_load(monkeypatch):
monkeypatch.setattr(
base_spec_module,
"get_layers_from_vllm_config",
lambda *args, **kwargs: {},
)
monkeypatch.setattr(
DraftModelSpeculator,
"_validate_local_argmax_reduction",
lambda self: None,
)
def _make_speculator(
monkeypatch,
output: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
@@ -49,6 +100,153 @@ def _make_speculator(
return speculator
def test_mm_support_configured_after_model_load(monkeypatch):
target_model_config = object()
draft_model_config = object()
vllm_config = SimpleNamespace(model_config=target_model_config)
draft_model = _MultimodalDraftModel()
def init_base(speculator, vllm_config, device):
speculator.vllm_config = vllm_config
speculator.device = device
speculator.max_num_tokens = 4
speculator.max_num_reqs = 2
speculator.hidden_size = 3
speculator.dtype = torch.float32
speculator.draft_model_config = draft_model_config
speculator.supports_mm_inputs = False
checked_configs = []
def supports_multimodal_inputs(model_config):
checked_configs.append(model_config)
return True
monkeypatch.setattr(DraftModelSpeculator, "__init__", init_base)
_mock_base_model_load(monkeypatch)
monkeypatch.setattr(
base_spec_module.MULTIMODAL_REGISTRY,
"supports_multimodal_inputs",
supports_multimodal_inputs,
)
speculator = _TestSpeculator(vllm_config, torch.device("cpu"))
assert checked_configs == []
assert not speculator.supports_mm_inputs
assert speculator.inputs_embeds is None
speculator.test_draft_model = draft_model
speculator.load_model(torch.nn.Module())
assert checked_configs == [target_model_config]
assert speculator.supports_mm_inputs
assert speculator.inputs_embeds is not None
assert speculator.inputs_embeds.shape == (4, 3)
def test_load_model_keeps_mm_support_for_capable_drafter(monkeypatch):
speculator = object.__new__(_TestSpeculator)
speculator.supports_mm_inputs = False
speculator.inputs_embeds = None
speculator.vllm_config = SimpleNamespace(model_config=object())
speculator.max_num_tokens = 4
speculator.hidden_size = 3
speculator.dtype = torch.float32
speculator.device = torch.device("cpu")
draft_model = _MultimodalDraftModel()
speculator.test_draft_model = draft_model
_mock_base_model_load(monkeypatch)
monkeypatch.setattr(
base_spec_module.MULTIMODAL_REGISTRY,
"supports_multimodal_inputs",
lambda model_config: True,
)
speculator.load_model(torch.nn.Module())
assert speculator.supports_mm_inputs
assert speculator.inputs_embeds is not None
def test_load_model_disables_mm_support_for_text_only_drafter(monkeypatch):
speculator = object.__new__(_TestSpeculator)
speculator.supports_mm_inputs = False
speculator.inputs_embeds = None
speculator.vllm_config = SimpleNamespace(model_config=object())
draft_model = _TextOnlyDraftModel()
speculator.test_draft_model = draft_model
warning_messages = []
_mock_base_model_load(monkeypatch)
monkeypatch.setattr(
base_spec_module.MULTIMODAL_REGISTRY,
"supports_multimodal_inputs",
lambda model_config: True,
)
monkeypatch.setattr(
base_spec_module.logger,
"warning_once",
lambda message, *args: warning_messages.append(message % args),
)
speculator.load_model(torch.nn.Module())
assert not speculator.supports_mm_inputs
assert warning_messages == [
"Draft model _TextOnlyDraftModel does not support external multimodal "
"embeddings. Embeddings from the target model will not be passed to the "
"drafter; using text-only draft inputs instead."
]
def test_multi_module_mm_support_configured_after_model_load(monkeypatch):
speculator = object.__new__(MultiModuleMTPSpeculator)
speculator.supports_mm_inputs = False
speculator.inputs_embeds = None
speculator.cached_draft_input_embeds = None
speculator.vllm_config = SimpleNamespace(model_config=object())
speculator.max_num_tokens = 4
speculator.max_num_reqs = 2
speculator.num_speculative_steps = 3
speculator.hidden_size = 3
speculator.dtype = torch.float32
speculator.device = torch.device("cpu")
draft_model = _MultimodalDraftModel()
_mock_base_model_load(monkeypatch)
monkeypatch.setattr(
MultiModuleMTPSpeculator,
"load_draft_model",
lambda self, target_model, target_attn_layer_names: draft_model,
)
monkeypatch.setattr(
base_spec_module.MULTIMODAL_REGISTRY,
"supports_multimodal_inputs",
lambda model_config: True,
)
speculator.load_model(torch.nn.Module())
assert speculator.supports_mm_inputs
assert speculator.inputs_embeds is not None
assert speculator.inputs_embeds.shape == (4, 3)
assert speculator.cached_draft_input_embeds is not None
assert speculator.cached_draft_input_embeds.shape == (2, 2, 3)
@pytest.mark.parametrize(
("model_cls", "expected"),
[
(EagleLlama4ForCausalLM, True),
(EagleMistralForCausalLM, True),
(EagleMistralLarge3ForCausalLM, True),
(Exaone4_5_MTP, True),
(Eagle3LlamaForCausalLM, False),
],
)
def test_draft_model_multimodal_embedding_capability(model_cls, expected):
assert supports_multimodal_embeddings(model_cls) is expected
def test_run_model_unpacks_tuple_return_for_mtp(monkeypatch):
logits_hidden = torch.full((4, 3), 1.0)
feedback_hidden = torch.full((4, 3), 2.0)