[Bugfix][Model Runner V2] Restore multimodal draft capability detection (#50417)
Signed-off-by: Raphael Rialland <raphael.rialland@mistral.ai>
This commit is contained in:
@@ -4,18 +4,32 @@
|
||||
from contextlib import nullcontext
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.model_executor.models import supports_multimodal_embeddings
|
||||
from vllm.model_executor.models.exaone4_5_mtp import Exaone4_5_MTP
|
||||
from vllm.model_executor.models.llama4_eagle import EagleLlama4ForCausalLM
|
||||
from vllm.model_executor.models.llama_eagle3 import Eagle3LlamaForCausalLM
|
||||
from vllm.model_executor.models.mistral_eagle import EagleMistralForCausalLM
|
||||
from vllm.model_executor.models.mistral_large_3_eagle import (
|
||||
EagleMistralLarge3ForCausalLM,
|
||||
)
|
||||
from vllm.v1.worker.gpu.spec_decode import speculator as base_spec_module
|
||||
from vllm.v1.worker.gpu.spec_decode.autoregressive import speculator as spec_module
|
||||
from vllm.v1.worker.gpu.spec_decode.autoregressive.speculator import (
|
||||
AutoRegressiveSpeculator,
|
||||
)
|
||||
from vllm.v1.worker.gpu.spec_decode.multi_module_mtp.speculator import (
|
||||
MultiModuleMTPSpeculator,
|
||||
)
|
||||
from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator
|
||||
|
||||
|
||||
class _TestSpeculator(AutoRegressiveSpeculator):
|
||||
def load_draft_model(self, target_model, target_attn_layer_names):
|
||||
raise NotImplementedError
|
||||
return self.test_draft_model
|
||||
|
||||
|
||||
class _DraftModel(torch.nn.Module):
|
||||
@@ -27,6 +41,43 @@ class _DraftModel(torch.nn.Module):
|
||||
return self.output
|
||||
|
||||
|
||||
class _MultimodalDraftModel(torch.nn.Module):
|
||||
supports_multimodal_embeddings = True
|
||||
|
||||
def embed_input_ids(
|
||||
self,
|
||||
input_ids,
|
||||
multimodal_embeddings=None,
|
||||
*,
|
||||
is_multimodal=None,
|
||||
):
|
||||
raise AssertionError("embed_input_ids should not be called during loading")
|
||||
|
||||
|
||||
class _TextOnlyDraftModel(torch.nn.Module):
|
||||
def embed_input_ids(
|
||||
self,
|
||||
input_ids,
|
||||
multimodal_embeddings=None,
|
||||
*,
|
||||
is_multimodal=None,
|
||||
):
|
||||
raise AssertionError("embed_input_ids should not be called during loading")
|
||||
|
||||
|
||||
def _mock_base_model_load(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
base_spec_module,
|
||||
"get_layers_from_vllm_config",
|
||||
lambda *args, **kwargs: {},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
DraftModelSpeculator,
|
||||
"_validate_local_argmax_reduction",
|
||||
lambda self: None,
|
||||
)
|
||||
|
||||
|
||||
def _make_speculator(
|
||||
monkeypatch,
|
||||
output: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
|
||||
@@ -49,6 +100,153 @@ def _make_speculator(
|
||||
return speculator
|
||||
|
||||
|
||||
def test_mm_support_configured_after_model_load(monkeypatch):
|
||||
target_model_config = object()
|
||||
draft_model_config = object()
|
||||
vllm_config = SimpleNamespace(model_config=target_model_config)
|
||||
draft_model = _MultimodalDraftModel()
|
||||
|
||||
def init_base(speculator, vllm_config, device):
|
||||
speculator.vllm_config = vllm_config
|
||||
speculator.device = device
|
||||
speculator.max_num_tokens = 4
|
||||
speculator.max_num_reqs = 2
|
||||
speculator.hidden_size = 3
|
||||
speculator.dtype = torch.float32
|
||||
speculator.draft_model_config = draft_model_config
|
||||
speculator.supports_mm_inputs = False
|
||||
|
||||
checked_configs = []
|
||||
|
||||
def supports_multimodal_inputs(model_config):
|
||||
checked_configs.append(model_config)
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(DraftModelSpeculator, "__init__", init_base)
|
||||
_mock_base_model_load(monkeypatch)
|
||||
monkeypatch.setattr(
|
||||
base_spec_module.MULTIMODAL_REGISTRY,
|
||||
"supports_multimodal_inputs",
|
||||
supports_multimodal_inputs,
|
||||
)
|
||||
|
||||
speculator = _TestSpeculator(vllm_config, torch.device("cpu"))
|
||||
|
||||
assert checked_configs == []
|
||||
assert not speculator.supports_mm_inputs
|
||||
assert speculator.inputs_embeds is None
|
||||
|
||||
speculator.test_draft_model = draft_model
|
||||
speculator.load_model(torch.nn.Module())
|
||||
|
||||
assert checked_configs == [target_model_config]
|
||||
assert speculator.supports_mm_inputs
|
||||
assert speculator.inputs_embeds is not None
|
||||
assert speculator.inputs_embeds.shape == (4, 3)
|
||||
|
||||
|
||||
def test_load_model_keeps_mm_support_for_capable_drafter(monkeypatch):
|
||||
speculator = object.__new__(_TestSpeculator)
|
||||
speculator.supports_mm_inputs = False
|
||||
speculator.inputs_embeds = None
|
||||
speculator.vllm_config = SimpleNamespace(model_config=object())
|
||||
speculator.max_num_tokens = 4
|
||||
speculator.hidden_size = 3
|
||||
speculator.dtype = torch.float32
|
||||
speculator.device = torch.device("cpu")
|
||||
draft_model = _MultimodalDraftModel()
|
||||
speculator.test_draft_model = draft_model
|
||||
_mock_base_model_load(monkeypatch)
|
||||
monkeypatch.setattr(
|
||||
base_spec_module.MULTIMODAL_REGISTRY,
|
||||
"supports_multimodal_inputs",
|
||||
lambda model_config: True,
|
||||
)
|
||||
|
||||
speculator.load_model(torch.nn.Module())
|
||||
|
||||
assert speculator.supports_mm_inputs
|
||||
assert speculator.inputs_embeds is not None
|
||||
|
||||
|
||||
def test_load_model_disables_mm_support_for_text_only_drafter(monkeypatch):
|
||||
speculator = object.__new__(_TestSpeculator)
|
||||
speculator.supports_mm_inputs = False
|
||||
speculator.inputs_embeds = None
|
||||
speculator.vllm_config = SimpleNamespace(model_config=object())
|
||||
draft_model = _TextOnlyDraftModel()
|
||||
speculator.test_draft_model = draft_model
|
||||
warning_messages = []
|
||||
_mock_base_model_load(monkeypatch)
|
||||
monkeypatch.setattr(
|
||||
base_spec_module.MULTIMODAL_REGISTRY,
|
||||
"supports_multimodal_inputs",
|
||||
lambda model_config: True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
base_spec_module.logger,
|
||||
"warning_once",
|
||||
lambda message, *args: warning_messages.append(message % args),
|
||||
)
|
||||
|
||||
speculator.load_model(torch.nn.Module())
|
||||
|
||||
assert not speculator.supports_mm_inputs
|
||||
assert warning_messages == [
|
||||
"Draft model _TextOnlyDraftModel does not support external multimodal "
|
||||
"embeddings. Embeddings from the target model will not be passed to the "
|
||||
"drafter; using text-only draft inputs instead."
|
||||
]
|
||||
|
||||
|
||||
def test_multi_module_mm_support_configured_after_model_load(monkeypatch):
|
||||
speculator = object.__new__(MultiModuleMTPSpeculator)
|
||||
speculator.supports_mm_inputs = False
|
||||
speculator.inputs_embeds = None
|
||||
speculator.cached_draft_input_embeds = None
|
||||
speculator.vllm_config = SimpleNamespace(model_config=object())
|
||||
speculator.max_num_tokens = 4
|
||||
speculator.max_num_reqs = 2
|
||||
speculator.num_speculative_steps = 3
|
||||
speculator.hidden_size = 3
|
||||
speculator.dtype = torch.float32
|
||||
speculator.device = torch.device("cpu")
|
||||
draft_model = _MultimodalDraftModel()
|
||||
_mock_base_model_load(monkeypatch)
|
||||
monkeypatch.setattr(
|
||||
MultiModuleMTPSpeculator,
|
||||
"load_draft_model",
|
||||
lambda self, target_model, target_attn_layer_names: draft_model,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
base_spec_module.MULTIMODAL_REGISTRY,
|
||||
"supports_multimodal_inputs",
|
||||
lambda model_config: True,
|
||||
)
|
||||
|
||||
speculator.load_model(torch.nn.Module())
|
||||
|
||||
assert speculator.supports_mm_inputs
|
||||
assert speculator.inputs_embeds is not None
|
||||
assert speculator.inputs_embeds.shape == (4, 3)
|
||||
assert speculator.cached_draft_input_embeds is not None
|
||||
assert speculator.cached_draft_input_embeds.shape == (2, 2, 3)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model_cls", "expected"),
|
||||
[
|
||||
(EagleLlama4ForCausalLM, True),
|
||||
(EagleMistralForCausalLM, True),
|
||||
(EagleMistralLarge3ForCausalLM, True),
|
||||
(Exaone4_5_MTP, True),
|
||||
(Eagle3LlamaForCausalLM, False),
|
||||
],
|
||||
)
|
||||
def test_draft_model_multimodal_embedding_capability(model_cls, expected):
|
||||
assert supports_multimodal_embeddings(model_cls) is expected
|
||||
|
||||
|
||||
def test_run_model_unpacks_tuple_return_for_mtp(monkeypatch):
|
||||
logits_hidden = torch.full((4, 3), 1.0)
|
||||
feedback_hidden = torch.full((4, 3), 2.0)
|
||||
|
||||
@@ -6,12 +6,14 @@ from .interfaces import (
|
||||
SupportsLoRA,
|
||||
SupportsMRoPE,
|
||||
SupportsMultiModal,
|
||||
SupportsMultiModalEmbeddings,
|
||||
SupportsPP,
|
||||
SupportsTranscription,
|
||||
has_inner_state,
|
||||
supports_lora,
|
||||
supports_mrope,
|
||||
supports_multimodal,
|
||||
supports_multimodal_embeddings,
|
||||
supports_pp,
|
||||
supports_transcription,
|
||||
)
|
||||
@@ -35,6 +37,8 @@ __all__ = [
|
||||
"supports_lora",
|
||||
"SupportsMultiModal",
|
||||
"supports_multimodal",
|
||||
"SupportsMultiModalEmbeddings",
|
||||
"supports_multimodal_embeddings",
|
||||
"SupportsMRoPE",
|
||||
"supports_mrope",
|
||||
"SupportsPP",
|
||||
|
||||
@@ -110,7 +110,22 @@ _language_model_by_module = dict[nn.Module, "VllmModel"]()
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class SupportsMultiModal(Protocol):
|
||||
class SupportsMultiModalEmbeddings(Protocol):
|
||||
"""The interface for models that can merge external multimodal embeddings."""
|
||||
|
||||
supports_multimodal_embeddings: ClassVar[Literal[True]] = True
|
||||
|
||||
def embed_input_ids(
|
||||
self,
|
||||
input_ids: Tensor,
|
||||
multimodal_embeddings: MultiModalEmbeddings | None = None,
|
||||
*,
|
||||
is_multimodal: Tensor | None = None,
|
||||
) -> Tensor: ...
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
|
||||
"""The interface required for all multi-modal models."""
|
||||
|
||||
supports_multimodal: ClassVar[Literal[True]] = True
|
||||
@@ -488,6 +503,24 @@ def supports_multimodal(
|
||||
return getattr(model, "supports_multimodal", False)
|
||||
|
||||
|
||||
@overload
|
||||
def supports_multimodal_embeddings(
|
||||
model: type[object],
|
||||
) -> TypeIs[type[SupportsMultiModalEmbeddings]]: ...
|
||||
|
||||
|
||||
@overload
|
||||
def supports_multimodal_embeddings(
|
||||
model: object,
|
||||
) -> TypeIs[SupportsMultiModalEmbeddings]: ...
|
||||
|
||||
|
||||
def supports_multimodal_embeddings(
|
||||
model: type[object] | object,
|
||||
) -> TypeIs[type[SupportsMultiModalEmbeddings]] | TypeIs[SupportsMultiModalEmbeddings]:
|
||||
return getattr(model, "supports_multimodal_embeddings", False)
|
||||
|
||||
|
||||
def supports_multimodal_raw_input_only(model: type[object] | object) -> bool:
|
||||
return getattr(model, "supports_multimodal_raw_input_only", False)
|
||||
|
||||
|
||||
@@ -35,7 +35,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
from vllm.model_executor.models.llama4 import Llama4DecoderLayer, Llama4ForCausalLM
|
||||
from vllm.model_executor.models.utils import extract_layer_index
|
||||
|
||||
from .interfaces import SupportsMultiModal
|
||||
from .interfaces import SupportsMultiModal, SupportsMultiModalEmbeddings
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
WeightsMapper,
|
||||
@@ -152,7 +152,7 @@ class LlamaModel(nn.Module):
|
||||
}
|
||||
|
||||
|
||||
class EagleLlama4ForCausalLM(Llama4ForCausalLM):
|
||||
class EagleLlama4ForCausalLM(Llama4ForCausalLM, SupportsMultiModalEmbeddings):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
nn.Module.__init__(self)
|
||||
self.config = vllm_config.speculative_config.draft_model_config.hf_config
|
||||
|
||||
@@ -14,7 +14,10 @@ from vllm.model_executor.layers.linear import RowParallelLinear
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
|
||||
from vllm.model_executor.models.interfaces import MultiModalEmbeddings
|
||||
from vllm.model_executor.models.interfaces import (
|
||||
MultiModalEmbeddings,
|
||||
SupportsMultiModalEmbeddings,
|
||||
)
|
||||
from vllm.model_executor.models.llama import LlamaConfig
|
||||
from vllm.model_executor.models.mistral import (
|
||||
MistralDecoderLayer,
|
||||
@@ -109,7 +112,7 @@ class EagleMistralModel(MistralModel):
|
||||
return hidden_states, hidden_states
|
||||
|
||||
|
||||
class EagleMistralForCausalLM(MistralForCausalLM):
|
||||
class EagleMistralForCausalLM(MistralForCausalLM, SupportsMultiModalEmbeddings):
|
||||
mistral_mapping = MistralForCausalLM.mistral_mapping | {
|
||||
"eagle_linear": "model.fc",
|
||||
}
|
||||
|
||||
@@ -22,7 +22,7 @@ from vllm.model_executor.models.deepseek_v2 import (
|
||||
)
|
||||
from vllm.model_executor.models.mistral_large_3 import MistralLarge3ForCausalLM
|
||||
|
||||
from .interfaces import SupportsMultiModal
|
||||
from .interfaces import SupportsMultiModal, SupportsMultiModalEmbeddings
|
||||
from .utils import WeightsMapper, make_empty_intermediate_tensors_factory, maybe_prefix
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -107,7 +107,9 @@ class EagleMistralLarge3Model(DeepseekV2Model):
|
||||
return output
|
||||
|
||||
|
||||
class EagleMistralLarge3ForCausalLM(MistralLarge3ForCausalLM):
|
||||
class EagleMistralLarge3ForCausalLM(
|
||||
MistralLarge3ForCausalLM, SupportsMultiModalEmbeddings
|
||||
):
|
||||
hf_to_vllm_mapper = MistralLarge3ForCausalLM.hf_to_vllm_mapper | WeightsMapper(
|
||||
orig_to_new_regex={
|
||||
regex.compile(r"\Aeagle_linear\.weight\Z"): r"model.fc.weight",
|
||||
|
||||
@@ -29,6 +29,7 @@ from vllm.model_executor.model_loader.mtp_validation import (
|
||||
is_mtp_completeness_check_enabled,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.models.interfaces import SupportsMultiModalEmbeddings
|
||||
from vllm.model_executor.models.utils import maybe_prefix
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
@@ -213,7 +214,7 @@ class InklingMultiTokenPredictor(nn.Module):
|
||||
return hidden
|
||||
|
||||
|
||||
class InklingMTP(nn.Module):
|
||||
class InklingMTP(nn.Module, SupportsMultiModalEmbeddings):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
super().__init__()
|
||||
assert vllm_config.speculative_config is not None
|
||||
|
||||
@@ -33,6 +33,7 @@ from vllm.model_executor.model_loader.mtp_validation import (
|
||||
is_mtp_completeness_check_enabled,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.models.interfaces import SupportsMultiModalEmbeddings
|
||||
from vllm.model_executor.models.utils import maybe_prefix
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
@@ -242,7 +243,7 @@ class InklingMultiTokenPredictor(nn.Module):
|
||||
return hidden
|
||||
|
||||
|
||||
class InklingMTP(nn.Module):
|
||||
class InklingMTP(nn.Module, SupportsMultiModalEmbeddings):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
super().__init__()
|
||||
assert vllm_config.speculative_config is not None
|
||||
|
||||
@@ -25,7 +25,10 @@ from vllm.forward_context import set_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.model_executor.model_loader import get_model
|
||||
from vllm.model_executor.models import supports_multimodal
|
||||
from vllm.model_executor.models import (
|
||||
supports_multimodal,
|
||||
supports_multimodal_embeddings,
|
||||
)
|
||||
from vllm.model_executor.models.deepseek_eagle3 import Eagle3DeepseekV2ForCausalLM
|
||||
from vllm.model_executor.models.interfaces import SupportsMultiModal
|
||||
from vllm.model_executor.models.laguna_dflash import DFlashLagunaForCausalLM
|
||||
@@ -1349,18 +1352,16 @@ class SpecDecodeBaseProposer:
|
||||
if all_attn_layers[name].get_kv_cache_spec(self.vllm_config) is not None
|
||||
}
|
||||
|
||||
if self.supports_mm_inputs:
|
||||
# Even if the target model is multimodal, we can also use
|
||||
# text-only draft models
|
||||
try:
|
||||
dummy_input_ids = torch.tensor([[1]], device=self.input_ids.device)
|
||||
self.model.embed_input_ids(dummy_input_ids, multimodal_embeddings=None)
|
||||
except (NotImplementedError, AttributeError, TypeError):
|
||||
logger.warning(
|
||||
"Draft model does not support multimodal inputs, "
|
||||
"falling back to text-only mode"
|
||||
)
|
||||
self.supports_mm_inputs = False
|
||||
# Even if the target model is multimodal, we can also use
|
||||
# text-only draft models
|
||||
if self.supports_mm_inputs and not supports_multimodal_embeddings(self.model):
|
||||
logger.warning_once(
|
||||
"Draft model %s does not support external multimodal embeddings. "
|
||||
"Embeddings from the target model will not be passed to the "
|
||||
"drafter; using text-only draft inputs instead.",
|
||||
type(self.model).__name__,
|
||||
)
|
||||
self.supports_mm_inputs = False
|
||||
|
||||
if supports_multimodal(target_model):
|
||||
# handle multimodality
|
||||
|
||||
@@ -3,12 +3,12 @@
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.forward_context import BatchDescriptor, set_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer
|
||||
from vllm.v1.worker.gpu.cudagraph_utils import (
|
||||
@@ -37,17 +37,23 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
||||
self.max_num_reqs, dtype=torch.int64, device=device
|
||||
)
|
||||
|
||||
self.supports_mm_inputs = MULTIMODAL_REGISTRY.supports_multimodal_inputs(
|
||||
self.draft_model_config
|
||||
)
|
||||
if self.supports_mm_inputs:
|
||||
self.inputs_embeds = torch.zeros(
|
||||
self.max_num_tokens, self.hidden_size, dtype=self.dtype, device=device
|
||||
)
|
||||
self.inputs_embeds: torch.Tensor | None = None
|
||||
|
||||
self.prefill_cudagraph_manager: SpeculatorCudaGraphManager | None = None
|
||||
self.decode_cudagraph_manager: SpeculatorCudaGraphManager | None = None
|
||||
|
||||
def load_model(self, target_model: nn.Module) -> None:
|
||||
super().load_model(target_model)
|
||||
if not self.supports_mm_inputs:
|
||||
return
|
||||
|
||||
self.inputs_embeds = torch.zeros(
|
||||
self.max_num_tokens,
|
||||
self.hidden_size,
|
||||
dtype=self.dtype,
|
||||
device=self.device,
|
||||
)
|
||||
|
||||
@property
|
||||
def advance_draft_positions(self) -> bool:
|
||||
"""
|
||||
@@ -295,6 +301,7 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
||||
):
|
||||
inputs_embeds = None
|
||||
if self.supports_mm_inputs:
|
||||
assert self.inputs_embeds is not None
|
||||
# Merge multimodal embeddings with input ids.
|
||||
mm_embeds, is_mm_embed = mm_inputs or (None, None)
|
||||
num_input_tokens = (
|
||||
|
||||
@@ -9,7 +9,6 @@ from vllm.config import VllmConfig
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.forward_context import BatchDescriptor, set_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.attention.backends.utils import PAD_SLOT_ID
|
||||
from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer
|
||||
@@ -39,25 +38,7 @@ class MultiModuleMTPSpeculator(DraftModelSpeculator):
|
||||
self.max_num_reqs, dtype=torch.int64, device=device
|
||||
)
|
||||
|
||||
self.supports_mm_inputs = MULTIMODAL_REGISTRY.supports_multimodal_inputs(
|
||||
self.draft_model_config
|
||||
)
|
||||
# HACK: the Inkling MTP draft has no MM processor of its own (its draft
|
||||
# config is flattened text-only), but it consumes the target's merged
|
||||
# embeddings at draft prefill — treat it as MM-capable whenever the
|
||||
# target is.
|
||||
if (
|
||||
not self.supports_mm_inputs
|
||||
and self.draft_model_config.hf_config.model_type == "inkling_mtp"
|
||||
):
|
||||
self.supports_mm_inputs = MULTIMODAL_REGISTRY.supports_multimodal_inputs(
|
||||
vllm_config.model_config
|
||||
)
|
||||
self.inputs_embeds: torch.Tensor | None = None
|
||||
if self.supports_mm_inputs:
|
||||
self.inputs_embeds = torch.zeros(
|
||||
self.max_num_tokens, self.hidden_size, dtype=self.dtype, device=device
|
||||
)
|
||||
|
||||
# Input id overrides for the last num_speculative_steps - 1 draft steps.
|
||||
# Used by chunked-prefilling requests to swap in the future prefill token
|
||||
@@ -80,14 +61,6 @@ class MultiModuleMTPSpeculator(DraftModelSpeculator):
|
||||
device=self.device,
|
||||
)
|
||||
self.cached_draft_input_embeds: torch.Tensor | None = None
|
||||
if self.supports_mm_inputs:
|
||||
self.cached_draft_input_embeds = torch.zeros(
|
||||
self.max_num_reqs,
|
||||
self.num_speculative_steps - 1,
|
||||
self.hidden_size,
|
||||
dtype=self.dtype,
|
||||
device=self.device,
|
||||
)
|
||||
self.cached_target_hidden_states = torch.zeros(
|
||||
self.max_num_reqs,
|
||||
self.num_speculative_steps - 1,
|
||||
@@ -105,6 +78,25 @@ class MultiModuleMTPSpeculator(DraftModelSpeculator):
|
||||
) -> nn.Module:
|
||||
return load_eagle_model(target_model, self.vllm_config)
|
||||
|
||||
def load_model(self, target_model: nn.Module) -> None:
|
||||
super().load_model(target_model)
|
||||
if not self.supports_mm_inputs:
|
||||
return
|
||||
|
||||
self.inputs_embeds = torch.zeros(
|
||||
self.max_num_tokens,
|
||||
self.hidden_size,
|
||||
dtype=self.dtype,
|
||||
device=self.device,
|
||||
)
|
||||
self.cached_draft_input_embeds = torch.zeros(
|
||||
self.max_num_reqs,
|
||||
self.num_speculative_steps - 1,
|
||||
self.hidden_size,
|
||||
dtype=self.dtype,
|
||||
device=self.device,
|
||||
)
|
||||
|
||||
def init_cudagraph_manager(self, cudagraph_mode: CUDAGraphMode) -> None:
|
||||
# TODO(TheEpicDolphin): Support piecewise cudagraph for multi-module MTP.
|
||||
if cudagraph_mode.has_piecewise_cudagraphs():
|
||||
|
||||
@@ -13,6 +13,8 @@ from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.distributed.eplb.eplb_state import EplbState
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.model_executor.models import supports_multimodal_embeddings
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.worker.gpu.attn_utils import (
|
||||
build_attn_metadata,
|
||||
@@ -137,6 +139,8 @@ class DraftModelSpeculator(BaseSpeculator):
|
||||
device=device,
|
||||
)
|
||||
|
||||
self.supports_mm_inputs = False
|
||||
|
||||
@abstractmethod
|
||||
def load_draft_model(
|
||||
self,
|
||||
@@ -164,6 +168,19 @@ class DraftModelSpeculator(BaseSpeculator):
|
||||
)
|
||||
self.draft_attn_layer_names = all_attn_layers - target_attn_layer_names
|
||||
|
||||
target_supports_mm = MULTIMODAL_REGISTRY.supports_multimodal_inputs(
|
||||
self.vllm_config.model_config
|
||||
)
|
||||
draft_supports_mm = supports_multimodal_embeddings(self.model)
|
||||
self.supports_mm_inputs = target_supports_mm and draft_supports_mm
|
||||
if target_supports_mm and not draft_supports_mm:
|
||||
logger.warning_once(
|
||||
"Draft model %s does not support external multimodal embeddings. "
|
||||
"Embeddings from the target model will not be passed to the "
|
||||
"drafter; using text-only draft inputs instead.",
|
||||
type(self.model).__name__,
|
||||
)
|
||||
|
||||
def set_eplb_state(self, eplb_state: EplbState) -> None:
|
||||
"""Inject EPLB state after construction."""
|
||||
self.eplb_state = eplb_state
|
||||
|
||||
Reference in New Issue
Block a user