From 4a3447d200e5aa428d68d1a00aa00f1a19a1a729 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Rialland?= <36076211+TQCB@users.noreply.github.com> Date: Mon, 3 Aug 2026 23:18:14 +0200 Subject: [PATCH] [Bugfix][Model Runner V2] Restore multimodal draft capability detection (#50417) Signed-off-by: Raphael Rialland --- .../test_gpu_autoregressive_speculator.py | 200 +++++++++++++++++- vllm/model_executor/models/__init__.py | 4 + vllm/model_executor/models/interfaces.py | 35 ++- vllm/model_executor/models/llama4_eagle.py | 4 +- vllm/model_executor/models/mistral_eagle.py | 7 +- .../models/mistral_large_3_eagle.py | 6 +- vllm/models/inkling/amd/mtp.py | 3 +- vllm/models/inkling/nvidia/mtp.py | 3 +- vllm/v1/spec_decode/llm_base_proposer.py | 27 +-- .../spec_decode/autoregressive/speculator.py | 23 +- .../multi_module_mtp/speculator.py | 46 ++-- vllm/v1/worker/gpu/spec_decode/speculator.py | 17 ++ 12 files changed, 317 insertions(+), 58 deletions(-) diff --git a/tests/v1/worker/test_gpu_autoregressive_speculator.py b/tests/v1/worker/test_gpu_autoregressive_speculator.py index 940fb375a9..9f5f90a977 100644 --- a/tests/v1/worker/test_gpu_autoregressive_speculator.py +++ b/tests/v1/worker/test_gpu_autoregressive_speculator.py @@ -4,18 +4,32 @@ from contextlib import nullcontext from types import SimpleNamespace +import pytest import torch from vllm.config.compilation import CUDAGraphMode +from vllm.model_executor.models import supports_multimodal_embeddings +from vllm.model_executor.models.exaone4_5_mtp import Exaone4_5_MTP +from vllm.model_executor.models.llama4_eagle import EagleLlama4ForCausalLM +from vllm.model_executor.models.llama_eagle3 import Eagle3LlamaForCausalLM +from vllm.model_executor.models.mistral_eagle import EagleMistralForCausalLM +from vllm.model_executor.models.mistral_large_3_eagle import ( + EagleMistralLarge3ForCausalLM, +) +from vllm.v1.worker.gpu.spec_decode import speculator as base_spec_module from vllm.v1.worker.gpu.spec_decode.autoregressive import speculator as spec_module from vllm.v1.worker.gpu.spec_decode.autoregressive.speculator import ( AutoRegressiveSpeculator, ) +from vllm.v1.worker.gpu.spec_decode.multi_module_mtp.speculator import ( + MultiModuleMTPSpeculator, +) +from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator class _TestSpeculator(AutoRegressiveSpeculator): def load_draft_model(self, target_model, target_attn_layer_names): - raise NotImplementedError + return self.test_draft_model class _DraftModel(torch.nn.Module): @@ -27,6 +41,43 @@ class _DraftModel(torch.nn.Module): return self.output +class _MultimodalDraftModel(torch.nn.Module): + supports_multimodal_embeddings = True + + def embed_input_ids( + self, + input_ids, + multimodal_embeddings=None, + *, + is_multimodal=None, + ): + raise AssertionError("embed_input_ids should not be called during loading") + + +class _TextOnlyDraftModel(torch.nn.Module): + def embed_input_ids( + self, + input_ids, + multimodal_embeddings=None, + *, + is_multimodal=None, + ): + raise AssertionError("embed_input_ids should not be called during loading") + + +def _mock_base_model_load(monkeypatch): + monkeypatch.setattr( + base_spec_module, + "get_layers_from_vllm_config", + lambda *args, **kwargs: {}, + ) + monkeypatch.setattr( + DraftModelSpeculator, + "_validate_local_argmax_reduction", + lambda self: None, + ) + + def _make_speculator( monkeypatch, output: torch.Tensor | tuple[torch.Tensor, torch.Tensor], @@ -49,6 +100,153 @@ def _make_speculator( return speculator +def test_mm_support_configured_after_model_load(monkeypatch): + target_model_config = object() + draft_model_config = object() + vllm_config = SimpleNamespace(model_config=target_model_config) + draft_model = _MultimodalDraftModel() + + def init_base(speculator, vllm_config, device): + speculator.vllm_config = vllm_config + speculator.device = device + speculator.max_num_tokens = 4 + speculator.max_num_reqs = 2 + speculator.hidden_size = 3 + speculator.dtype = torch.float32 + speculator.draft_model_config = draft_model_config + speculator.supports_mm_inputs = False + + checked_configs = [] + + def supports_multimodal_inputs(model_config): + checked_configs.append(model_config) + return True + + monkeypatch.setattr(DraftModelSpeculator, "__init__", init_base) + _mock_base_model_load(monkeypatch) + monkeypatch.setattr( + base_spec_module.MULTIMODAL_REGISTRY, + "supports_multimodal_inputs", + supports_multimodal_inputs, + ) + + speculator = _TestSpeculator(vllm_config, torch.device("cpu")) + + assert checked_configs == [] + assert not speculator.supports_mm_inputs + assert speculator.inputs_embeds is None + + speculator.test_draft_model = draft_model + speculator.load_model(torch.nn.Module()) + + assert checked_configs == [target_model_config] + assert speculator.supports_mm_inputs + assert speculator.inputs_embeds is not None + assert speculator.inputs_embeds.shape == (4, 3) + + +def test_load_model_keeps_mm_support_for_capable_drafter(monkeypatch): + speculator = object.__new__(_TestSpeculator) + speculator.supports_mm_inputs = False + speculator.inputs_embeds = None + speculator.vllm_config = SimpleNamespace(model_config=object()) + speculator.max_num_tokens = 4 + speculator.hidden_size = 3 + speculator.dtype = torch.float32 + speculator.device = torch.device("cpu") + draft_model = _MultimodalDraftModel() + speculator.test_draft_model = draft_model + _mock_base_model_load(monkeypatch) + monkeypatch.setattr( + base_spec_module.MULTIMODAL_REGISTRY, + "supports_multimodal_inputs", + lambda model_config: True, + ) + + speculator.load_model(torch.nn.Module()) + + assert speculator.supports_mm_inputs + assert speculator.inputs_embeds is not None + + +def test_load_model_disables_mm_support_for_text_only_drafter(monkeypatch): + speculator = object.__new__(_TestSpeculator) + speculator.supports_mm_inputs = False + speculator.inputs_embeds = None + speculator.vllm_config = SimpleNamespace(model_config=object()) + draft_model = _TextOnlyDraftModel() + speculator.test_draft_model = draft_model + warning_messages = [] + _mock_base_model_load(monkeypatch) + monkeypatch.setattr( + base_spec_module.MULTIMODAL_REGISTRY, + "supports_multimodal_inputs", + lambda model_config: True, + ) + monkeypatch.setattr( + base_spec_module.logger, + "warning_once", + lambda message, *args: warning_messages.append(message % args), + ) + + speculator.load_model(torch.nn.Module()) + + assert not speculator.supports_mm_inputs + assert warning_messages == [ + "Draft model _TextOnlyDraftModel does not support external multimodal " + "embeddings. Embeddings from the target model will not be passed to the " + "drafter; using text-only draft inputs instead." + ] + + +def test_multi_module_mm_support_configured_after_model_load(monkeypatch): + speculator = object.__new__(MultiModuleMTPSpeculator) + speculator.supports_mm_inputs = False + speculator.inputs_embeds = None + speculator.cached_draft_input_embeds = None + speculator.vllm_config = SimpleNamespace(model_config=object()) + speculator.max_num_tokens = 4 + speculator.max_num_reqs = 2 + speculator.num_speculative_steps = 3 + speculator.hidden_size = 3 + speculator.dtype = torch.float32 + speculator.device = torch.device("cpu") + draft_model = _MultimodalDraftModel() + _mock_base_model_load(monkeypatch) + monkeypatch.setattr( + MultiModuleMTPSpeculator, + "load_draft_model", + lambda self, target_model, target_attn_layer_names: draft_model, + ) + monkeypatch.setattr( + base_spec_module.MULTIMODAL_REGISTRY, + "supports_multimodal_inputs", + lambda model_config: True, + ) + + speculator.load_model(torch.nn.Module()) + + assert speculator.supports_mm_inputs + assert speculator.inputs_embeds is not None + assert speculator.inputs_embeds.shape == (4, 3) + assert speculator.cached_draft_input_embeds is not None + assert speculator.cached_draft_input_embeds.shape == (2, 2, 3) + + +@pytest.mark.parametrize( + ("model_cls", "expected"), + [ + (EagleLlama4ForCausalLM, True), + (EagleMistralForCausalLM, True), + (EagleMistralLarge3ForCausalLM, True), + (Exaone4_5_MTP, True), + (Eagle3LlamaForCausalLM, False), + ], +) +def test_draft_model_multimodal_embedding_capability(model_cls, expected): + assert supports_multimodal_embeddings(model_cls) is expected + + def test_run_model_unpacks_tuple_return_for_mtp(monkeypatch): logits_hidden = torch.full((4, 3), 1.0) feedback_hidden = torch.full((4, 3), 2.0) diff --git a/vllm/model_executor/models/__init__.py b/vllm/model_executor/models/__init__.py index 9f8dd042bf..478b00d0af 100644 --- a/vllm/model_executor/models/__init__.py +++ b/vllm/model_executor/models/__init__.py @@ -6,12 +6,14 @@ from .interfaces import ( SupportsLoRA, SupportsMRoPE, SupportsMultiModal, + SupportsMultiModalEmbeddings, SupportsPP, SupportsTranscription, has_inner_state, supports_lora, supports_mrope, supports_multimodal, + supports_multimodal_embeddings, supports_pp, supports_transcription, ) @@ -35,6 +37,8 @@ __all__ = [ "supports_lora", "SupportsMultiModal", "supports_multimodal", + "SupportsMultiModalEmbeddings", + "supports_multimodal_embeddings", "SupportsMRoPE", "supports_mrope", "SupportsPP", diff --git a/vllm/model_executor/models/interfaces.py b/vllm/model_executor/models/interfaces.py index efdab8c4d1..de2dbf2a47 100644 --- a/vllm/model_executor/models/interfaces.py +++ b/vllm/model_executor/models/interfaces.py @@ -110,7 +110,22 @@ _language_model_by_module = dict[nn.Module, "VllmModel"]() @runtime_checkable -class SupportsMultiModal(Protocol): +class SupportsMultiModalEmbeddings(Protocol): + """The interface for models that can merge external multimodal embeddings.""" + + supports_multimodal_embeddings: ClassVar[Literal[True]] = True + + def embed_input_ids( + self, + input_ids: Tensor, + multimodal_embeddings: MultiModalEmbeddings | None = None, + *, + is_multimodal: Tensor | None = None, + ) -> Tensor: ... + + +@runtime_checkable +class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol): """The interface required for all multi-modal models.""" supports_multimodal: ClassVar[Literal[True]] = True @@ -488,6 +503,24 @@ def supports_multimodal( return getattr(model, "supports_multimodal", False) +@overload +def supports_multimodal_embeddings( + model: type[object], +) -> TypeIs[type[SupportsMultiModalEmbeddings]]: ... + + +@overload +def supports_multimodal_embeddings( + model: object, +) -> TypeIs[SupportsMultiModalEmbeddings]: ... + + +def supports_multimodal_embeddings( + model: type[object] | object, +) -> TypeIs[type[SupportsMultiModalEmbeddings]] | TypeIs[SupportsMultiModalEmbeddings]: + return getattr(model, "supports_multimodal_embeddings", False) + + def supports_multimodal_raw_input_only(model: type[object] | object) -> bool: return getattr(model, "supports_multimodal_raw_input_only", False) diff --git a/vllm/model_executor/models/llama4_eagle.py b/vllm/model_executor/models/llama4_eagle.py index d94dbb0496..e41b0c5dd2 100644 --- a/vllm/model_executor/models/llama4_eagle.py +++ b/vllm/model_executor/models/llama4_eagle.py @@ -35,7 +35,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( from vllm.model_executor.models.llama4 import Llama4DecoderLayer, Llama4ForCausalLM from vllm.model_executor.models.utils import extract_layer_index -from .interfaces import SupportsMultiModal +from .interfaces import SupportsMultiModal, SupportsMultiModalEmbeddings from .utils import ( AutoWeightsLoader, WeightsMapper, @@ -152,7 +152,7 @@ class LlamaModel(nn.Module): } -class EagleLlama4ForCausalLM(Llama4ForCausalLM): +class EagleLlama4ForCausalLM(Llama4ForCausalLM, SupportsMultiModalEmbeddings): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): nn.Module.__init__(self) self.config = vllm_config.speculative_config.draft_model_config.hf_config diff --git a/vllm/model_executor/models/mistral_eagle.py b/vllm/model_executor/models/mistral_eagle.py index 75d1ebb91a..6a85f970ae 100644 --- a/vllm/model_executor/models/mistral_eagle.py +++ b/vllm/model_executor/models/mistral_eagle.py @@ -14,7 +14,10 @@ from vllm.model_executor.layers.linear import RowParallelLinear from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.quantization.base_config import QuantizationConfig from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding -from vllm.model_executor.models.interfaces import MultiModalEmbeddings +from vllm.model_executor.models.interfaces import ( + MultiModalEmbeddings, + SupportsMultiModalEmbeddings, +) from vllm.model_executor.models.llama import LlamaConfig from vllm.model_executor.models.mistral import ( MistralDecoderLayer, @@ -109,7 +112,7 @@ class EagleMistralModel(MistralModel): return hidden_states, hidden_states -class EagleMistralForCausalLM(MistralForCausalLM): +class EagleMistralForCausalLM(MistralForCausalLM, SupportsMultiModalEmbeddings): mistral_mapping = MistralForCausalLM.mistral_mapping | { "eagle_linear": "model.fc", } diff --git a/vllm/model_executor/models/mistral_large_3_eagle.py b/vllm/model_executor/models/mistral_large_3_eagle.py index 8ace01205d..d194536beb 100644 --- a/vllm/model_executor/models/mistral_large_3_eagle.py +++ b/vllm/model_executor/models/mistral_large_3_eagle.py @@ -22,7 +22,7 @@ from vllm.model_executor.models.deepseek_v2 import ( ) from vllm.model_executor.models.mistral_large_3 import MistralLarge3ForCausalLM -from .interfaces import SupportsMultiModal +from .interfaces import SupportsMultiModal, SupportsMultiModalEmbeddings from .utils import WeightsMapper, make_empty_intermediate_tensors_factory, maybe_prefix logger = init_logger(__name__) @@ -107,7 +107,9 @@ class EagleMistralLarge3Model(DeepseekV2Model): return output -class EagleMistralLarge3ForCausalLM(MistralLarge3ForCausalLM): +class EagleMistralLarge3ForCausalLM( + MistralLarge3ForCausalLM, SupportsMultiModalEmbeddings +): hf_to_vllm_mapper = MistralLarge3ForCausalLM.hf_to_vllm_mapper | WeightsMapper( orig_to_new_regex={ regex.compile(r"\Aeagle_linear\.weight\Z"): r"model.fc.weight", diff --git a/vllm/models/inkling/amd/mtp.py b/vllm/models/inkling/amd/mtp.py index 34cde020ed..5a6ba23b4f 100644 --- a/vllm/models/inkling/amd/mtp.py +++ b/vllm/models/inkling/amd/mtp.py @@ -29,6 +29,7 @@ from vllm.model_executor.model_loader.mtp_validation import ( is_mtp_completeness_check_enabled, ) from vllm.model_executor.model_loader.weight_utils import default_weight_loader +from vllm.model_executor.models.interfaces import SupportsMultiModalEmbeddings from vllm.model_executor.models.utils import maybe_prefix from vllm.sequence import IntermediateTensors @@ -213,7 +214,7 @@ class InklingMultiTokenPredictor(nn.Module): return hidden -class InklingMTP(nn.Module): +class InklingMTP(nn.Module, SupportsMultiModalEmbeddings): def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: super().__init__() assert vllm_config.speculative_config is not None diff --git a/vllm/models/inkling/nvidia/mtp.py b/vllm/models/inkling/nvidia/mtp.py index 197894beb9..f9cba37f3c 100644 --- a/vllm/models/inkling/nvidia/mtp.py +++ b/vllm/models/inkling/nvidia/mtp.py @@ -33,6 +33,7 @@ from vllm.model_executor.model_loader.mtp_validation import ( is_mtp_completeness_check_enabled, ) from vllm.model_executor.model_loader.weight_utils import default_weight_loader +from vllm.model_executor.models.interfaces import SupportsMultiModalEmbeddings from vllm.model_executor.models.utils import maybe_prefix from vllm.sequence import IntermediateTensors @@ -242,7 +243,7 @@ class InklingMultiTokenPredictor(nn.Module): return hidden -class InklingMTP(nn.Module): +class InklingMTP(nn.Module, SupportsMultiModalEmbeddings): def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: super().__init__() assert vllm_config.speculative_config is not None diff --git a/vllm/v1/spec_decode/llm_base_proposer.py b/vllm/v1/spec_decode/llm_base_proposer.py index c02c3350d8..675bc4bbe5 100644 --- a/vllm/v1/spec_decode/llm_base_proposer.py +++ b/vllm/v1/spec_decode/llm_base_proposer.py @@ -25,7 +25,10 @@ from vllm.forward_context import set_forward_context from vllm.logger import init_logger from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase from vllm.model_executor.model_loader import get_model -from vllm.model_executor.models import supports_multimodal +from vllm.model_executor.models import ( + supports_multimodal, + supports_multimodal_embeddings, +) from vllm.model_executor.models.deepseek_eagle3 import Eagle3DeepseekV2ForCausalLM from vllm.model_executor.models.interfaces import SupportsMultiModal from vllm.model_executor.models.laguna_dflash import DFlashLagunaForCausalLM @@ -1349,18 +1352,16 @@ class SpecDecodeBaseProposer: if all_attn_layers[name].get_kv_cache_spec(self.vllm_config) is not None } - if self.supports_mm_inputs: - # Even if the target model is multimodal, we can also use - # text-only draft models - try: - dummy_input_ids = torch.tensor([[1]], device=self.input_ids.device) - self.model.embed_input_ids(dummy_input_ids, multimodal_embeddings=None) - except (NotImplementedError, AttributeError, TypeError): - logger.warning( - "Draft model does not support multimodal inputs, " - "falling back to text-only mode" - ) - self.supports_mm_inputs = False + # Even if the target model is multimodal, we can also use + # text-only draft models + if self.supports_mm_inputs and not supports_multimodal_embeddings(self.model): + logger.warning_once( + "Draft model %s does not support external multimodal embeddings. " + "Embeddings from the target model will not be passed to the " + "drafter; using text-only draft inputs instead.", + type(self.model).__name__, + ) + self.supports_mm_inputs = False if supports_multimodal(target_model): # handle multimodality diff --git a/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py b/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py index 9035ea38b2..7dbd811253 100644 --- a/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py @@ -3,12 +3,12 @@ from typing import Any import torch +import torch.nn as nn from vllm.config import VllmConfig from vllm.config.compilation import CUDAGraphMode from vllm.forward_context import BatchDescriptor, set_forward_context from vllm.logger import init_logger -from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.triton_utils import tl, triton from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer from vllm.v1.worker.gpu.cudagraph_utils import ( @@ -37,17 +37,23 @@ class AutoRegressiveSpeculator(DraftModelSpeculator): self.max_num_reqs, dtype=torch.int64, device=device ) - self.supports_mm_inputs = MULTIMODAL_REGISTRY.supports_multimodal_inputs( - self.draft_model_config - ) - if self.supports_mm_inputs: - self.inputs_embeds = torch.zeros( - self.max_num_tokens, self.hidden_size, dtype=self.dtype, device=device - ) + self.inputs_embeds: torch.Tensor | None = None self.prefill_cudagraph_manager: SpeculatorCudaGraphManager | None = None self.decode_cudagraph_manager: SpeculatorCudaGraphManager | None = None + def load_model(self, target_model: nn.Module) -> None: + super().load_model(target_model) + if not self.supports_mm_inputs: + return + + self.inputs_embeds = torch.zeros( + self.max_num_tokens, + self.hidden_size, + dtype=self.dtype, + device=self.device, + ) + @property def advance_draft_positions(self) -> bool: """ @@ -295,6 +301,7 @@ class AutoRegressiveSpeculator(DraftModelSpeculator): ): inputs_embeds = None if self.supports_mm_inputs: + assert self.inputs_embeds is not None # Merge multimodal embeddings with input ids. mm_embeds, is_mm_embed = mm_inputs or (None, None) num_input_tokens = ( diff --git a/vllm/v1/worker/gpu/spec_decode/multi_module_mtp/speculator.py b/vllm/v1/worker/gpu/spec_decode/multi_module_mtp/speculator.py index 7fda70a801..4be22b4474 100644 --- a/vllm/v1/worker/gpu/spec_decode/multi_module_mtp/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/multi_module_mtp/speculator.py @@ -9,7 +9,6 @@ from vllm.config import VllmConfig from vllm.config.compilation import CUDAGraphMode from vllm.forward_context import BatchDescriptor, set_forward_context from vllm.logger import init_logger -from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.triton_utils import tl, triton from vllm.v1.attention.backends.utils import PAD_SLOT_ID from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer @@ -39,25 +38,7 @@ class MultiModuleMTPSpeculator(DraftModelSpeculator): self.max_num_reqs, dtype=torch.int64, device=device ) - self.supports_mm_inputs = MULTIMODAL_REGISTRY.supports_multimodal_inputs( - self.draft_model_config - ) - # HACK: the Inkling MTP draft has no MM processor of its own (its draft - # config is flattened text-only), but it consumes the target's merged - # embeddings at draft prefill — treat it as MM-capable whenever the - # target is. - if ( - not self.supports_mm_inputs - and self.draft_model_config.hf_config.model_type == "inkling_mtp" - ): - self.supports_mm_inputs = MULTIMODAL_REGISTRY.supports_multimodal_inputs( - vllm_config.model_config - ) self.inputs_embeds: torch.Tensor | None = None - if self.supports_mm_inputs: - self.inputs_embeds = torch.zeros( - self.max_num_tokens, self.hidden_size, dtype=self.dtype, device=device - ) # Input id overrides for the last num_speculative_steps - 1 draft steps. # Used by chunked-prefilling requests to swap in the future prefill token @@ -80,14 +61,6 @@ class MultiModuleMTPSpeculator(DraftModelSpeculator): device=self.device, ) self.cached_draft_input_embeds: torch.Tensor | None = None - if self.supports_mm_inputs: - self.cached_draft_input_embeds = torch.zeros( - self.max_num_reqs, - self.num_speculative_steps - 1, - self.hidden_size, - dtype=self.dtype, - device=self.device, - ) self.cached_target_hidden_states = torch.zeros( self.max_num_reqs, self.num_speculative_steps - 1, @@ -105,6 +78,25 @@ class MultiModuleMTPSpeculator(DraftModelSpeculator): ) -> nn.Module: return load_eagle_model(target_model, self.vllm_config) + def load_model(self, target_model: nn.Module) -> None: + super().load_model(target_model) + if not self.supports_mm_inputs: + return + + self.inputs_embeds = torch.zeros( + self.max_num_tokens, + self.hidden_size, + dtype=self.dtype, + device=self.device, + ) + self.cached_draft_input_embeds = torch.zeros( + self.max_num_reqs, + self.num_speculative_steps - 1, + self.hidden_size, + dtype=self.dtype, + device=self.device, + ) + def init_cudagraph_manager(self, cudagraph_mode: CUDAGraphMode) -> None: # TODO(TheEpicDolphin): Support piecewise cudagraph for multi-module MTP. if cudagraph_mode.has_piecewise_cudagraphs(): diff --git a/vllm/v1/worker/gpu/spec_decode/speculator.py b/vllm/v1/worker/gpu/spec_decode/speculator.py index 34f9925940..71eb412ba2 100644 --- a/vllm/v1/worker/gpu/spec_decode/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/speculator.py @@ -13,6 +13,8 @@ from vllm.config.compilation import CUDAGraphMode from vllm.distributed.eplb.eplb_state import EplbState from vllm.logger import init_logger from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase +from vllm.model_executor.models import supports_multimodal_embeddings +from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.v1.kv_cache_interface import KVCacheConfig from vllm.v1.worker.gpu.attn_utils import ( build_attn_metadata, @@ -137,6 +139,8 @@ class DraftModelSpeculator(BaseSpeculator): device=device, ) + self.supports_mm_inputs = False + @abstractmethod def load_draft_model( self, @@ -164,6 +168,19 @@ class DraftModelSpeculator(BaseSpeculator): ) self.draft_attn_layer_names = all_attn_layers - target_attn_layer_names + target_supports_mm = MULTIMODAL_REGISTRY.supports_multimodal_inputs( + self.vllm_config.model_config + ) + draft_supports_mm = supports_multimodal_embeddings(self.model) + self.supports_mm_inputs = target_supports_mm and draft_supports_mm + if target_supports_mm and not draft_supports_mm: + logger.warning_once( + "Draft model %s does not support external multimodal embeddings. " + "Embeddings from the target model will not be passed to the " + "drafter; using text-only draft inputs instead.", + type(self.model).__name__, + ) + def set_eplb_state(self, eplb_state: EplbState) -> None: """Inject EPLB state after construction.""" self.eplb_state = eplb_state