[Bugfix][Model Runner V2] Restore multimodal draft capability detection (#50417)
Signed-off-by: Raphael Rialland <raphael.rialland@mistral.ai>
This commit is contained in:
@@ -4,18 +4,32 @@
|
||||
from contextlib import nullcontext
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.model_executor.models import supports_multimodal_embeddings
|
||||
from vllm.model_executor.models.exaone4_5_mtp import Exaone4_5_MTP
|
||||
from vllm.model_executor.models.llama4_eagle import EagleLlama4ForCausalLM
|
||||
from vllm.model_executor.models.llama_eagle3 import Eagle3LlamaForCausalLM
|
||||
from vllm.model_executor.models.mistral_eagle import EagleMistralForCausalLM
|
||||
from vllm.model_executor.models.mistral_large_3_eagle import (
|
||||
EagleMistralLarge3ForCausalLM,
|
||||
)
|
||||
from vllm.v1.worker.gpu.spec_decode import speculator as base_spec_module
|
||||
from vllm.v1.worker.gpu.spec_decode.autoregressive import speculator as spec_module
|
||||
from vllm.v1.worker.gpu.spec_decode.autoregressive.speculator import (
|
||||
AutoRegressiveSpeculator,
|
||||
)
|
||||
from vllm.v1.worker.gpu.spec_decode.multi_module_mtp.speculator import (
|
||||
MultiModuleMTPSpeculator,
|
||||
)
|
||||
from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator
|
||||
|
||||
|
||||
class _TestSpeculator(AutoRegressiveSpeculator):
|
||||
def load_draft_model(self, target_model, target_attn_layer_names):
|
||||
raise NotImplementedError
|
||||
return self.test_draft_model
|
||||
|
||||
|
||||
class _DraftModel(torch.nn.Module):
|
||||
@@ -27,6 +41,43 @@ class _DraftModel(torch.nn.Module):
|
||||
return self.output
|
||||
|
||||
|
||||
class _MultimodalDraftModel(torch.nn.Module):
|
||||
supports_multimodal_embeddings = True
|
||||
|
||||
def embed_input_ids(
|
||||
self,
|
||||
input_ids,
|
||||
multimodal_embeddings=None,
|
||||
*,
|
||||
is_multimodal=None,
|
||||
):
|
||||
raise AssertionError("embed_input_ids should not be called during loading")
|
||||
|
||||
|
||||
class _TextOnlyDraftModel(torch.nn.Module):
|
||||
def embed_input_ids(
|
||||
self,
|
||||
input_ids,
|
||||
multimodal_embeddings=None,
|
||||
*,
|
||||
is_multimodal=None,
|
||||
):
|
||||
raise AssertionError("embed_input_ids should not be called during loading")
|
||||
|
||||
|
||||
def _mock_base_model_load(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
base_spec_module,
|
||||
"get_layers_from_vllm_config",
|
||||
lambda *args, **kwargs: {},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
DraftModelSpeculator,
|
||||
"_validate_local_argmax_reduction",
|
||||
lambda self: None,
|
||||
)
|
||||
|
||||
|
||||
def _make_speculator(
|
||||
monkeypatch,
|
||||
output: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
|
||||
@@ -49,6 +100,153 @@ def _make_speculator(
|
||||
return speculator
|
||||
|
||||
|
||||
def test_mm_support_configured_after_model_load(monkeypatch):
|
||||
target_model_config = object()
|
||||
draft_model_config = object()
|
||||
vllm_config = SimpleNamespace(model_config=target_model_config)
|
||||
draft_model = _MultimodalDraftModel()
|
||||
|
||||
def init_base(speculator, vllm_config, device):
|
||||
speculator.vllm_config = vllm_config
|
||||
speculator.device = device
|
||||
speculator.max_num_tokens = 4
|
||||
speculator.max_num_reqs = 2
|
||||
speculator.hidden_size = 3
|
||||
speculator.dtype = torch.float32
|
||||
speculator.draft_model_config = draft_model_config
|
||||
speculator.supports_mm_inputs = False
|
||||
|
||||
checked_configs = []
|
||||
|
||||
def supports_multimodal_inputs(model_config):
|
||||
checked_configs.append(model_config)
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(DraftModelSpeculator, "__init__", init_base)
|
||||
_mock_base_model_load(monkeypatch)
|
||||
monkeypatch.setattr(
|
||||
base_spec_module.MULTIMODAL_REGISTRY,
|
||||
"supports_multimodal_inputs",
|
||||
supports_multimodal_inputs,
|
||||
)
|
||||
|
||||
speculator = _TestSpeculator(vllm_config, torch.device("cpu"))
|
||||
|
||||
assert checked_configs == []
|
||||
assert not speculator.supports_mm_inputs
|
||||
assert speculator.inputs_embeds is None
|
||||
|
||||
speculator.test_draft_model = draft_model
|
||||
speculator.load_model(torch.nn.Module())
|
||||
|
||||
assert checked_configs == [target_model_config]
|
||||
assert speculator.supports_mm_inputs
|
||||
assert speculator.inputs_embeds is not None
|
||||
assert speculator.inputs_embeds.shape == (4, 3)
|
||||
|
||||
|
||||
def test_load_model_keeps_mm_support_for_capable_drafter(monkeypatch):
|
||||
speculator = object.__new__(_TestSpeculator)
|
||||
speculator.supports_mm_inputs = False
|
||||
speculator.inputs_embeds = None
|
||||
speculator.vllm_config = SimpleNamespace(model_config=object())
|
||||
speculator.max_num_tokens = 4
|
||||
speculator.hidden_size = 3
|
||||
speculator.dtype = torch.float32
|
||||
speculator.device = torch.device("cpu")
|
||||
draft_model = _MultimodalDraftModel()
|
||||
speculator.test_draft_model = draft_model
|
||||
_mock_base_model_load(monkeypatch)
|
||||
monkeypatch.setattr(
|
||||
base_spec_module.MULTIMODAL_REGISTRY,
|
||||
"supports_multimodal_inputs",
|
||||
lambda model_config: True,
|
||||
)
|
||||
|
||||
speculator.load_model(torch.nn.Module())
|
||||
|
||||
assert speculator.supports_mm_inputs
|
||||
assert speculator.inputs_embeds is not None
|
||||
|
||||
|
||||
def test_load_model_disables_mm_support_for_text_only_drafter(monkeypatch):
|
||||
speculator = object.__new__(_TestSpeculator)
|
||||
speculator.supports_mm_inputs = False
|
||||
speculator.inputs_embeds = None
|
||||
speculator.vllm_config = SimpleNamespace(model_config=object())
|
||||
draft_model = _TextOnlyDraftModel()
|
||||
speculator.test_draft_model = draft_model
|
||||
warning_messages = []
|
||||
_mock_base_model_load(monkeypatch)
|
||||
monkeypatch.setattr(
|
||||
base_spec_module.MULTIMODAL_REGISTRY,
|
||||
"supports_multimodal_inputs",
|
||||
lambda model_config: True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
base_spec_module.logger,
|
||||
"warning_once",
|
||||
lambda message, *args: warning_messages.append(message % args),
|
||||
)
|
||||
|
||||
speculator.load_model(torch.nn.Module())
|
||||
|
||||
assert not speculator.supports_mm_inputs
|
||||
assert warning_messages == [
|
||||
"Draft model _TextOnlyDraftModel does not support external multimodal "
|
||||
"embeddings. Embeddings from the target model will not be passed to the "
|
||||
"drafter; using text-only draft inputs instead."
|
||||
]
|
||||
|
||||
|
||||
def test_multi_module_mm_support_configured_after_model_load(monkeypatch):
|
||||
speculator = object.__new__(MultiModuleMTPSpeculator)
|
||||
speculator.supports_mm_inputs = False
|
||||
speculator.inputs_embeds = None
|
||||
speculator.cached_draft_input_embeds = None
|
||||
speculator.vllm_config = SimpleNamespace(model_config=object())
|
||||
speculator.max_num_tokens = 4
|
||||
speculator.max_num_reqs = 2
|
||||
speculator.num_speculative_steps = 3
|
||||
speculator.hidden_size = 3
|
||||
speculator.dtype = torch.float32
|
||||
speculator.device = torch.device("cpu")
|
||||
draft_model = _MultimodalDraftModel()
|
||||
_mock_base_model_load(monkeypatch)
|
||||
monkeypatch.setattr(
|
||||
MultiModuleMTPSpeculator,
|
||||
"load_draft_model",
|
||||
lambda self, target_model, target_attn_layer_names: draft_model,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
base_spec_module.MULTIMODAL_REGISTRY,
|
||||
"supports_multimodal_inputs",
|
||||
lambda model_config: True,
|
||||
)
|
||||
|
||||
speculator.load_model(torch.nn.Module())
|
||||
|
||||
assert speculator.supports_mm_inputs
|
||||
assert speculator.inputs_embeds is not None
|
||||
assert speculator.inputs_embeds.shape == (4, 3)
|
||||
assert speculator.cached_draft_input_embeds is not None
|
||||
assert speculator.cached_draft_input_embeds.shape == (2, 2, 3)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model_cls", "expected"),
|
||||
[
|
||||
(EagleLlama4ForCausalLM, True),
|
||||
(EagleMistralForCausalLM, True),
|
||||
(EagleMistralLarge3ForCausalLM, True),
|
||||
(Exaone4_5_MTP, True),
|
||||
(Eagle3LlamaForCausalLM, False),
|
||||
],
|
||||
)
|
||||
def test_draft_model_multimodal_embedding_capability(model_cls, expected):
|
||||
assert supports_multimodal_embeddings(model_cls) is expected
|
||||
|
||||
|
||||
def test_run_model_unpacks_tuple_return_for_mtp(monkeypatch):
|
||||
logits_hidden = torch.full((4, 3), 1.0)
|
||||
feedback_hidden = torch.full((4, 3), 2.0)
|
||||
|
||||
Reference in New Issue
Block a user