diff --git a/tests/models/multimodal/processing/test_audioflamingo3.py b/tests/models/multimodal/processing/test_audioflamingo3.py index f5920a3be4..f223145d98 100644 --- a/tests/models/multimodal/processing/test_audioflamingo3.py +++ b/tests/models/multimodal/processing/test_audioflamingo3.py @@ -25,6 +25,7 @@ import torch from transformers import PretrainedConfig from tests.models.registry import HF_EXAMPLE_MODELS +from vllm.config.multimodal import MultiModalDummyOptions class MockAudioFlamingo3Config(PretrainedConfig): @@ -127,7 +128,7 @@ def test_dummy_data_generation(mock_ctx): builder = AudioFlamingo3DummyInputsBuilder(info) mm_counts = {"audio": 2} - dummy_data = builder.get_dummy_mm_data(100, mm_counts, {}) + dummy_data = builder.get_dummy_mm_data(100, mm_counts, MultiModalDummyOptions()) assert "audio" in dummy_data assert len(dummy_data["audio"]) == 2 diff --git a/tests/models/multimodal/processing/test_common.py b/tests/models/multimodal/processing/test_common.py index 1fbb2149a6..ea4305976c 100644 --- a/tests/models/multimodal/processing/test_common.py +++ b/tests/models/multimodal/processing/test_common.py @@ -13,6 +13,7 @@ from vllm.config.multimodal import ( AudioDummyOptions, BaseDummyOptions, ImageDummyOptions, + MultiModalDummyOptions, VideoDummyOptions, ) from vllm.inputs import MultiModalDataDict, MultiModalInput @@ -174,7 +175,7 @@ def get_token_prompt( inputs = dummy_inputs.get_dummy_processor_inputs( model_config.max_model_len, mm_counts, - mm_options={}, + mm_options=MultiModalDummyOptions(), # Assume all Mistral models define this extra argument mm_data=mm_data, # type: ignore[call-arg] ) @@ -182,7 +183,7 @@ def get_token_prompt( inputs = dummy_inputs.get_dummy_processor_inputs( model_config.max_model_len, mm_counts, - mm_options={}, + mm_options=MultiModalDummyOptions(), ) if not isinstance(inputs.prompt, list): @@ -278,10 +279,12 @@ def _test_processing_correctness( return BaseDummyOptions(count=count) # Assign normalized DummyOptions to the model config - model_config.get_multimodal_config().limit_per_prompt = { - modality: _to_dummy_options(modality, count) - for modality, count in limit_mm_per_prompt_ints.items() - } + model_config.get_multimodal_config().limit_per_prompt = MultiModalDummyOptions( + { + modality: _to_dummy_options(modality, count) + for modality, count in limit_mm_per_prompt_ints.items() + } + ) baseline_processor = factories.build_processor(ctx, cache=None) cached_processor = factories.build_processor(ctx, cache=cache) diff --git a/tests/models/multimodal/processing/test_molmo2.py b/tests/models/multimodal/processing/test_molmo2.py index 5633d47a97..68d3c8163c 100644 --- a/tests/models/multimodal/processing/test_molmo2.py +++ b/tests/models/multimodal/processing/test_molmo2.py @@ -6,7 +6,7 @@ from types import SimpleNamespace import pytest import torch -from vllm.config.multimodal import VideoDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions, VideoDummyOptions from vllm.model_executor.models.molmo2 import ( Molmo2DummyInputsBuilder, build_flat_image_bool_length, @@ -83,7 +83,9 @@ def test_dummy_video_num_frames_override_honors_min_of_two( data = builder.get_dummy_mm_data( seq_len=128, mm_counts={"image": 0, "video": 1}, - mm_options={"video": VideoDummyOptions(num_frames=num_frames_override)}, + mm_options=MultiModalDummyOptions( + video=VideoDummyOptions(num_frames=num_frames_override) + ), ) video, _metadata = data["video"][0] diff --git a/tests/models/multimodal/processing/test_qwen3_vl.py b/tests/models/multimodal/processing/test_qwen3_vl.py index c58904a9e2..04f87049c9 100644 --- a/tests/models/multimodal/processing/test_qwen3_vl.py +++ b/tests/models/multimodal/processing/test_qwen3_vl.py @@ -12,6 +12,7 @@ import numpy as np import pytest from vllm.config import ModelConfig +from vllm.config.multimodal import MultiModalDummyOptions from vllm.multimodal import MULTIMODAL_REGISTRY from ...registry import HF_EXAMPLE_MODELS @@ -339,7 +340,9 @@ def test_dummy_video_spreads_budget_when_frame_cap_enabled(model_id: str) -> Non limit_mm_per_prompt={"image": 0, "video": 1}, ) capped = MULTIMODAL_REGISTRY.create_processor(capped_ctx.model_config) - capped_dummy = capped.dummy_inputs.get_dummy_mm_data(1024, {"video": 1}, {}) + capped_dummy = capped.dummy_inputs.get_dummy_mm_data( + 1024, {"video": 1}, MultiModalDummyOptions() + ) capped_frames = capped_dummy["video"][0][0].shape[0] assert capped_frames == 16, ( f"Expected the dummy to spread the budget over 16 frames, got {capped_frames}" @@ -351,7 +354,9 @@ def test_dummy_video_spreads_budget_when_frame_cap_enabled(model_id: str) -> Non limit_mm_per_prompt={"image": 0, "video": 1}, ) uncapped = MULTIMODAL_REGISTRY.create_processor(uncapped_ctx.model_config) - uncapped_dummy = uncapped.dummy_inputs.get_dummy_mm_data(1024, {"video": 1}, {}) + uncapped_dummy = uncapped.dummy_inputs.get_dummy_mm_data( + 1024, {"video": 1}, MultiModalDummyOptions() + ) uncapped_frames = uncapped_dummy["video"][0][0].shape[0] assert uncapped_frames == 2, ( f"Expected the uncapped dummy to keep 2 frames, got {uncapped_frames}" diff --git a/tests/models/multimodal/processing/test_tensor_schema.py b/tests/models/multimodal/processing/test_tensor_schema.py index e1fb0cef74..f249a25269 100644 --- a/tests/models/multimodal/processing/test_tensor_schema.py +++ b/tests/models/multimodal/processing/test_tensor_schema.py @@ -18,6 +18,7 @@ from vllm.config.multimodal import ( AudioDummyOptions, BaseDummyOptions, ImageDummyOptions, + MultiModalDummyOptions, VideoDummyOptions, ) from vllm.distributed import ( @@ -98,7 +99,7 @@ def create_batched_mm_kwargs( processor_inputs = dummy_inputs.get_dummy_processor_inputs( seq_len=model_config.max_model_len, mm_counts=mm_counts, - mm_options={}, + mm_options=MultiModalDummyOptions(), ) mm_items = processor_inputs.mm_data_items resized_mm_data = { @@ -248,10 +249,12 @@ def test_model_tensor_schema(model_id: str): return AudioDummyOptions(count=count) return BaseDummyOptions(count=count) - model_config.get_multimodal_config().limit_per_prompt = { - modality: _to_dummy_options(modality, count) - for modality, count in limit_mm_per_prompt.items() - } + model_config.get_multimodal_config().limit_per_prompt = MultiModalDummyOptions( + { + modality: _to_dummy_options(modality, count) + for modality, count in limit_mm_per_prompt.items() + } + ) processor = factories.build_processor(ctx, cache=None) with initialize_dummy_model(model_cls, model_config) as model: diff --git a/vllm/config/multimodal.py b/vllm/config/multimodal.py index 09e184b8ce..4eca42430c 100644 --- a/vllm/config/multimodal.py +++ b/vllm/config/multimodal.py @@ -3,11 +3,18 @@ from collections.abc import Mapping from pathlib import Path -from typing import Any, Literal, TypeAlias, TypedDict, final +from typing import Any, Literal, TypeAlias, TypeVar, final, overload import torch -from pydantic import ConfigDict, Field, field_validator, model_validator +from pydantic import ( + ConfigDict, + Field, + GetCoreSchemaHandler, + field_validator, + model_validator, +) from pydantic.dataclasses import dataclass +from pydantic_core import core_schema import vllm.envs as envs from vllm.config.ec_transfer import ECTransferConfig @@ -18,6 +25,8 @@ from vllm.v1.attention.backends.registry import AttentionBackendEnum logger = init_logger(__name__) +_T = TypeVar("_T") + @dataclass class BaseDummyOptions: @@ -51,17 +60,50 @@ class AudioDummyOptions(BaseDummyOptions): @final -class MultiModalDummyOptionsBuiltins(TypedDict, total=False): - """Type annotations for modality types predefined by vLLM.""" +class MultiModalDummyOptions(dict[str, BaseDummyOptions]): + """Dummy data options for each modality. - image: ImageDummyOptions - """Options for dummy images.""" + Lookups of the modalities predefined by vLLM return their own options + class, while any other modality returns + [`BaseDummyOptions`][vllm.config.multimodal.BaseDummyOptions]. + """ - video: VideoDummyOptions - """Options for dummy videos.""" + @overload # type: ignore[override] + def get(self, key: Literal["image"], /) -> ImageDummyOptions | None: ... - audio: AudioDummyOptions - """Options for dummy audios.""" + @overload + def get(self, key: Literal["image"], default: _T, /) -> ImageDummyOptions | _T: ... + + @overload + def get(self, key: Literal["video"], /) -> VideoDummyOptions | None: ... + + @overload + def get(self, key: Literal["video"], default: _T, /) -> VideoDummyOptions | _T: ... + + @overload + def get(self, key: Literal["audio"], /) -> AudioDummyOptions | None: ... + + @overload + def get(self, key: Literal["audio"], default: _T, /) -> AudioDummyOptions | _T: ... + + @overload + def get(self, key: str, /) -> BaseDummyOptions | None: ... + + @overload + def get(self, key: str, default: _T, /) -> BaseDummyOptions | _T: ... + + def get(self, key: str, default: object = None, /) -> object: + return super().get(key, default) + + @classmethod + def __get_pydantic_core_schema__( + cls, + source_type: Any, + handler: GetCoreSchemaHandler, + ) -> core_schema.CoreSchema: + return core_schema.no_info_after_validator_function( + cls, handler.generate_schema(dict[str, BaseDummyOptions]) + ) MMEncoderTPMode = Literal["weights", "data"] @@ -75,15 +117,6 @@ MMProcessorDevice: TypeAlias = str `"xpu"` on XPU). Validated against that set by the CLI.""" -MMDummyOptions: TypeAlias = dict[str, BaseDummyOptions] -""" -A dictionary containing an entry for each modality type of dummy data. - -The built-in modalities are defined by -[`MultiModalDummyOptionsBuiltins`][vllm.config.multimodal.MultiModalDummyOptionsBuiltins]. -""" - - @config class MultiModalConfig: """Controls the behavior of multimodal models.""" @@ -91,7 +124,9 @@ class MultiModalConfig: language_model_only: bool = False """If True, disables all multimodal inputs by setting all modality limits to 0. Equivalent to setting `--limit-mm-per-prompt` to 0 for every modality.""" - limit_per_prompt: MMDummyOptions = Field(default_factory=dict) + limit_per_prompt: MultiModalDummyOptions = Field( + default_factory=MultiModalDummyOptions + ) """The maximum number of input items and options allowed per prompt for each modality. @@ -257,8 +292,8 @@ class MultiModalConfig: def _validate_limit_per_prompt( cls, value: dict[str, int | dict[str, int]], - ) -> MMDummyOptions: - out: MMDummyOptions = {} + ) -> MultiModalDummyOptions: + out = MultiModalDummyOptions() for k, v in value.items(): # Handle legacy format where only count is specified diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index d637391cd3..2dd4b04f2b 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -22,6 +22,7 @@ from typing import ( cast, get_args, get_origin, + is_typeddict, ) import huggingface_hub @@ -183,6 +184,18 @@ def is_type(type_hint: TypeHint, type: TypeHintT) -> TypeIs[TypeHintT]: return type_hint is type or get_origin(type_hint) is type +def is_dict_subclass(type_hint: TypeHint) -> bool: + """Check if the type hint is a subclass of `dict`. + + `TypedDict`s are excluded because they do not support class checks. + """ + return ( + isinstance(type_hint, type) + and not is_typeddict(type_hint) + and issubclass(type_hint, dict) + ) + + def contains_type(type_hints: set[TypeHint], type: TypeHintT) -> bool: """Check if the type hints contain a specific type.""" return any(is_type(type_hint, type) for type_hint in type_hints) @@ -305,6 +318,8 @@ def _compute_kwargs(cls: ConfigType) -> dict[str, dict[str, Any]]: for field in fields(cls): # Get the set of possible types for the field type_hints: set[TypeHint] = get_type_hints(field.type) + # Subclasses of dict (e.g. MultiModalDummyOptions) are CLI dicts + type_hints = {dict if is_dict_subclass(th) else th for th in type_hints} # If the field is a dataclass, we can use the model_validate_json generator = (th for th in type_hints if is_dataclass(th)) diff --git a/vllm/model_executor/models/aria.py b/vllm/model_executor/models/aria.py index 4240406f7b..01119b4a00 100644 --- a/vllm/model_executor/models/aria.py +++ b/vllm/model_executor/models/aria.py @@ -10,7 +10,7 @@ from transformers.models.aria.modeling_aria import AriaCrossAttention from transformers.models.aria.processing_aria import AriaProcessor from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions, ImageDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.inputs import MultiModalDataDict from vllm.model_executor.layers.activation import get_act_fn from vllm.model_executor.layers.fused_moe import FusedMoEFactory @@ -315,22 +315,18 @@ class AriaDummyInputsBuilder(BaseDummyInputsBuilder[AriaProcessingInfo]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: vision_config = self.info.get_vision_config() max_image_size = vision_config.image_size - num_images = mm_counts.get("image", 0) - - image_overrides = mm_options.get("image") - assert image_overrides is None or isinstance(image_overrides, ImageDummyOptions) return { "image": self._get_dummy_images( width=max_image_size, height=max_image_size, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ) } diff --git a/vllm/model_executor/models/audioflamingo3.py b/vllm/model_executor/models/audioflamingo3.py index 36f2569277..4be996d278 100644 --- a/vllm/model_executor/models/audioflamingo3.py +++ b/vllm/model_executor/models/audioflamingo3.py @@ -30,7 +30,7 @@ from transformers.models.audioflamingo3 import ( from transformers.models.qwen2_audio import Qwen2AudioEncoder from vllm.config import VllmConfig -from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.inputs import ModalityData, MultiModalDataDict from vllm.model_executor.layers.activation import get_act_fn from vllm.model_executor.models.module_mapping import MultiModelKeys @@ -216,21 +216,18 @@ class AudioFlamingo3DummyInputsBuilder( self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: hf_processor = self.info.get_hf_processor() feature_extractor = self.info.get_feature_extractor() sampling_rate = feature_extractor.sampling_rate audio_len = int(hf_processor.max_audio_len * sampling_rate) - num_audios = mm_counts.get("audio", 0) - audio_overrides = mm_options.get("audio") - assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions) return { "audio": self._get_dummy_audios( length=audio_len, - num_audios=num_audios, - overrides=audio_overrides, + num_audios=mm_counts.get("audio", 0), + overrides=mm_options.get("audio"), ) } diff --git a/vllm/model_executor/models/bagel.py b/vllm/model_executor/models/bagel.py index 9e0bd4ccbf..e9ebb0e5ed 100644 --- a/vllm/model_executor/models/bagel.py +++ b/vllm/model_executor/models/bagel.py @@ -14,7 +14,7 @@ import torch import torch.nn as nn from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions, ImageDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.inputs import MultiModalDataDict from vllm.logger import init_logger from vllm.model_executor.layers.activation import get_act_fn @@ -248,23 +248,20 @@ class BagelDummyInputsBuilder(BaseDummyInputsBuilder[BagelProcessingInfo]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) hf_config = self.info.get_hf_config() vit_config = hf_config.vit_config # Use the configured image size image_size = vit_config.image_size - image_overrides = mm_options.get("image") - assert image_overrides is None or isinstance(image_overrides, ImageDummyOptions) return { "image": self._get_dummy_images( width=image_size, height=image_size, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ), } diff --git a/vllm/model_executor/models/bailing_moe_v3_vl.py b/vllm/model_executor/models/bailing_moe_v3_vl.py index 7cd0568530..a236d0722b 100644 --- a/vllm/model_executor/models/bailing_moe_v3_vl.py +++ b/vllm/model_executor/models/bailing_moe_v3_vl.py @@ -13,7 +13,7 @@ import torch.nn as nn from transformers import AutoProcessor, BatchFeature from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.inputs import MultiModalDataDict from vllm.model_executor.layers.linear import ( ColumnParallelLinear, @@ -103,7 +103,7 @@ class BailingMoeV3VLDummyInputsBuilder( self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: target_width, target_height = self.info.get_image_size_with_most_features() return { diff --git a/vllm/model_executor/models/bee.py b/vllm/model_executor/models/bee.py index 4a57413b6e..cc6c0f09ab 100644 --- a/vllm/model_executor/models/bee.py +++ b/vllm/model_executor/models/bee.py @@ -8,7 +8,7 @@ import torch.nn as nn from transformers.activations import GELUActivation from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions, ImageDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.inputs import MultiModalDataDict from vllm.multimodal import MULTIMODAL_REGISTRY @@ -90,21 +90,16 @@ class BeeDummyInputsBuilder(LlavaDummyInputsBuilder[BeeProcessingInfo]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) - target_width, target_height = self.info.get_image_size_with_most_features() - image_overrides = mm_options.get("image") - assert image_overrides is None or isinstance(image_overrides, ImageDummyOptions) - return { "image": self._get_dummy_images( width=target_width, height=target_height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ), } diff --git a/vllm/model_executor/models/blip2.py b/vllm/model_executor/models/blip2.py index 48da4f63e8..cd019e41ec 100644 --- a/vllm/model_executor/models/blip2.py +++ b/vllm/model_executor/models/blip2.py @@ -15,7 +15,7 @@ from transformers import ( ) from vllm.config import CacheConfig, VllmConfig -from vllm.config.multimodal import BaseDummyOptions, ImageDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.inputs import MultiModalDataDict from vllm.model_executor.layers.activation import get_act_fn from vllm.model_executor.layers.quantization import QuantizationConfig @@ -444,23 +444,19 @@ class Blip2DummyInputsBuilder(BaseDummyInputsBuilder[Blip2ProcessingInfo]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: hf_config = self.info.get_hf_config() vision_config = hf_config.vision_config max_image_size = vision_config.image_size - num_images = mm_counts.get("image", 0) - - image_overrides = mm_options.get("image") - assert image_overrides is None or isinstance(image_overrides, ImageDummyOptions) return { "image": self._get_dummy_images( width=max_image_size, height=max_image_size, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ) } diff --git a/vllm/model_executor/models/clip.py b/vllm/model_executor/models/clip.py index 6fb311fb14..7c5ba496ea 100644 --- a/vllm/model_executor/models/clip.py +++ b/vllm/model_executor/models/clip.py @@ -15,7 +15,7 @@ from transformers import ( ) from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.distributed import divide, get_tensor_model_parallel_world_size from vllm.inputs import MultiModalDataDict, MultiModalInput from vllm.model_executor.layers.activation import get_act_fn @@ -205,20 +205,16 @@ class CLIPDummyInputsBuilder(BaseDummyInputsBuilder[CLIPProcessingInfo]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) - target_width, target_height = self.info.get_image_size_with_most_features() - image_overrides = mm_options.get("image") - return { "image": self._get_dummy_images( width=target_width, height=target_height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ) } diff --git a/vllm/model_executor/models/cohere2_vision.py b/vllm/model_executor/models/cohere2_vision.py index a4748932d6..4fdaff4444 100644 --- a/vllm/model_executor/models/cohere2_vision.py +++ b/vllm/model_executor/models/cohere2_vision.py @@ -18,7 +18,7 @@ from transformers.models.cohere2_vision.processing_cohere2_vision import ( ) from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.inputs import MultiModalDataDict from vllm.model_executor.layers.activation import MulAndSilu from vllm.model_executor.layers.linear import ( @@ -202,19 +202,16 @@ class Cohere2VisionDummyInputsBuilder( self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) image_size = self.info.get_image_size_with_most_features() - image_overrides = mm_options.get("image") - return { "image": self._get_dummy_images( width=image_size.width, height=image_size.height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ) } diff --git a/vllm/model_executor/models/cohere_asr.py b/vllm/model_executor/models/cohere_asr.py index 33b4aa7b0d..c8a196ffdc 100644 --- a/vllm/model_executor/models/cohere_asr.py +++ b/vllm/model_executor/models/cohere_asr.py @@ -12,7 +12,7 @@ from transformers import PretrainedConfig from vllm.compilation.decorators import support_torch_compile from vllm.config import CacheConfig, ModelConfig, SpeechToTextConfig, VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.config.speech_to_text import SpeechToTextParams from vllm.distributed import get_tensor_model_parallel_world_size from vllm.inputs import MultiModalDataDict, PromptType, TokensPrompt @@ -1944,7 +1944,7 @@ class CohereASRDummyInputsBuilder(BaseDummyInputsBuilder[CohereASRProcessingInfo self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, mm_processor_kwargs=None, ) -> MultiModalDataDict: feature_extractor = self.info.get_feature_extractor() diff --git a/vllm/model_executor/models/cohere_compass.py b/vllm/model_executor/models/cohere_compass.py index 1d0dd8c35b..9e8adabb80 100644 --- a/vllm/model_executor/models/cohere_compass.py +++ b/vllm/model_executor/models/cohere_compass.py @@ -51,7 +51,7 @@ from transformers.models.cohere_compass.image_processing_cohere_compass import ( from vllm.compilation.decorators import support_torch_compile from vllm.config import CacheConfig, VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.distributed import ( get_pp_group, get_tensor_model_parallel_world_size, @@ -1238,11 +1238,8 @@ class CohereCompassDummyInputsBuilder( self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) - image_overrides = mm_options.get("image") - target_image_width, target_image_height = ( self.info.get_image_size_with_most_features() ) @@ -1251,8 +1248,8 @@ class CohereCompassDummyInputsBuilder( "image": self._get_dummy_images( width=target_image_width, height=target_image_height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ), } diff --git a/vllm/model_executor/models/colmodernvbert.py b/vllm/model_executor/models/colmodernvbert.py index 0c93e7ef52..a14476f774 100644 --- a/vllm/model_executor/models/colmodernvbert.py +++ b/vllm/model_executor/models/colmodernvbert.py @@ -15,7 +15,7 @@ from torch import nn from transformers import BatchFeature from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.inputs import MultiModalDataDict from vllm.model_executor.layers.pooler.tokwise import pooler_for_token_embed from vllm.multimodal import MULTIMODAL_REGISTRY @@ -135,18 +135,16 @@ class ColModernVBertDummyInputsBuilder( self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) target_width, target_height = self.info.get_image_size_with_most_features() - image_overrides = mm_options.get("image") return { "image": self._get_dummy_images( width=target_width, height=target_height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ) } diff --git a/vllm/model_executor/models/deepseek_ocr.py b/vllm/model_executor/models/deepseek_ocr.py index 3d1921dc2b..62a46fc498 100644 --- a/vllm/model_executor/models/deepseek_ocr.py +++ b/vllm/model_executor/models/deepseek_ocr.py @@ -11,7 +11,7 @@ import torch.nn as nn from transformers import BatchFeature, CLIPVisionConfig from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.inputs import MultiModalDataDict from vllm.model_executor.models.interfaces import ( MultiModalEmbeddings, @@ -268,17 +268,15 @@ class DeepseekOCRDummyInputsBuilder(BaseDummyInputsBuilder[DeepseekOCRProcessing self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) - max_image_size = self.info.get_image_size_with_most_features() return { "image": self._get_dummy_images( width=max_image_size.width, height=max_image_size.height, - num_images=num_images, + num_images=mm_counts.get("image", 0), ) } diff --git a/vllm/model_executor/models/deepseek_ocr2.py b/vllm/model_executor/models/deepseek_ocr2.py index a64b71b64e..938598a68b 100644 --- a/vllm/model_executor/models/deepseek_ocr2.py +++ b/vllm/model_executor/models/deepseek_ocr2.py @@ -11,7 +11,7 @@ import torch.nn as nn from transformers import BatchFeature from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.inputs import MultiModalDataDict from vllm.model_executor.models.interfaces import ( MultiModalEmbeddings, @@ -140,17 +140,15 @@ class DeepseekOCR2DummyInputsBuilder( self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) - max_image_size = self.info.get_image_size_with_most_features() return { "image": self._get_dummy_images( width=max_image_size.width, height=max_image_size.height, - num_images=num_images, + num_images=mm_counts.get("image", 0), ) } diff --git a/vllm/model_executor/models/deepseek_vl2.py b/vllm/model_executor/models/deepseek_vl2.py index 6f798a87e6..c720f62335 100644 --- a/vllm/model_executor/models/deepseek_vl2.py +++ b/vllm/model_executor/models/deepseek_vl2.py @@ -15,7 +15,7 @@ from einops import rearrange, repeat from transformers import BatchFeature from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.distributed import get_tensor_model_parallel_world_size from vllm.inputs import MultiModalDataDict from vllm.model_executor.layers.quantization import QuantizationConfig @@ -212,20 +212,16 @@ class DeepseekVL2DummyInputsBuilder(BaseDummyInputsBuilder[DeepseekVL2Processing self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) - max_image_size = self.info.get_image_size_with_most_features() - image_overrides = mm_options.get("image") - return { "image": self._get_dummy_images( width=max_image_size.width, height=max_image_size.height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ) } diff --git a/vllm/model_executor/models/dots_ocr.py b/vllm/model_executor/models/dots_ocr.py index 36b1cd530d..adcc560ba5 100644 --- a/vllm/model_executor/models/dots_ocr.py +++ b/vllm/model_executor/models/dots_ocr.py @@ -9,7 +9,7 @@ from torch.nn import LayerNorm from transformers.models.qwen2_vl import Qwen2VLProcessor from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.distributed import utils as dist_utils from vllm.distributed.parallel_state import ( get_tensor_model_parallel_rank, @@ -104,20 +104,16 @@ class DotsOCRDummyInputsBuilder(Qwen2VLDummyInputsBuilder): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) - target_width, target_height = self.info.get_image_size_with_most_features() - image_overrides = mm_options.get("image") - return { "image": self._get_dummy_images( width=target_width, height=target_height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ), } diff --git a/vllm/model_executor/models/ernie45_vl.py b/vllm/model_executor/models/ernie45_vl.py index e1b847c294..a800458bbf 100644 --- a/vllm/model_executor/models/ernie45_vl.py +++ b/vllm/model_executor/models/ernie45_vl.py @@ -36,7 +36,7 @@ from einops import rearrange from transformers import BaseImageProcessor, BatchFeature from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions, VideoDummyOptions from vllm.distributed import parallel_state from vllm.distributed import utils as dist_utils from vllm.inputs import MultiModalDataDict @@ -1276,33 +1276,26 @@ class Ernie4_5_VLDummyInputsBuilder(BaseDummyInputsBuilder[Ernie4_5_VLProcessing self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) - num_videos = mm_counts.get("video", 0) - target_width, target_height = self.info.get_image_size_with_most_features() target_num_frames = self.info.get_num_frames_with_most_features( seq_len, mm_counts ) - image_overrides = mm_options.get("image") - video_overrides = mm_options.get("video") - assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions) - return { "image": self._get_dummy_images( width=target_width, height=target_height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ), "video": self._get_dummy_videos( width=target_width, height=target_height, num_frames=target_num_frames, - num_videos=num_videos, - overrides=video_overrides, + num_videos=mm_counts.get("video", 0), + overrides=mm_options.get("video"), ), } diff --git a/vllm/model_executor/models/fireredasr2.py b/vllm/model_executor/models/fireredasr2.py index 82a4a6b506..be5f5efbff 100644 --- a/vllm/model_executor/models/fireredasr2.py +++ b/vllm/model_executor/models/fireredasr2.py @@ -12,7 +12,7 @@ from transformers import ( ) from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig -from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.config.speech_to_text import SpeechToTextParams from vllm.inputs import MultiModalDataDict, PromptType from vllm.logger import init_logger @@ -207,20 +207,18 @@ class FireRedASR2DummyInputsBuilder(BaseDummyInputsBuilder[FireRedASR2Processing self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: feature_extractor = self.info.get_feature_extractor() sampling_rate = feature_extractor.sampling_rate audio_len = feature_extractor.chunk_length * sampling_rate - num_audios = mm_counts.get("audio", 0) - - audio_overrides = mm_options.get("audio") - assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions) ret = { "audio": self._get_dummy_audios( - length=audio_len, num_audios=num_audios, overrides=audio_overrides + length=audio_len, + num_audios=mm_counts.get("audio", 0), + overrides=mm_options.get("audio"), ) } return ret diff --git a/vllm/model_executor/models/funasr.py b/vllm/model_executor/models/funasr.py index 7bf253ab40..f8db523785 100644 --- a/vllm/model_executor/models/funasr.py +++ b/vllm/model_executor/models/funasr.py @@ -14,7 +14,7 @@ from transformers import ( ) from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig -from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.config.speech_to_text import SpeechToTextParams from vllm.distributed import get_tensor_model_parallel_world_size from vllm.inputs import MultiModalDataDict, PromptType @@ -734,22 +734,18 @@ class FunASRDummyInputsBuilder(BaseDummyInputsBuilder[FunASRProcessingInfo]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: feature_extractor = self.info.get_feature_extractor() sampling_rate = feature_extractor.sampling_rate audio_len = feature_extractor.chunk_length * sampling_rate - num_audios = mm_counts.get("audio", 0) - - audio_overrides = mm_options.get("audio") - assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions) return { "audio": self._get_dummy_audios( length=audio_len, - num_audios=num_audios, - overrides=audio_overrides, + num_audios=mm_counts.get("audio", 0), + overrides=mm_options.get("audio"), ), } diff --git a/vllm/model_executor/models/funaudiochat.py b/vllm/model_executor/models/funaudiochat.py index 7a093cc2e7..8b40ddf0cd 100644 --- a/vllm/model_executor/models/funaudiochat.py +++ b/vllm/model_executor/models/funaudiochat.py @@ -26,7 +26,7 @@ from transformers.feature_extraction_utils import BatchFeature from transformers.modeling_outputs import BaseModelOutput from vllm.config import VllmConfig -from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.inputs import MultiModalDataDict from vllm.model_executor.layers.attention.mm_encoder_attention import MMEncoderAttention from vllm.model_executor.layers.linear import QKVParallelLinear, RowParallelLinear @@ -580,7 +580,7 @@ class FunAudioChatDummyInputsBuilder( self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: feature_extractor = self.info.get_feature_extractor() sampling_rate = int(feature_extractor.sampling_rate) @@ -597,15 +597,12 @@ class FunAudioChatDummyInputsBuilder( 1, (target_num_frames * sampling_rate + token_fps - 1) // token_fps, ) - num_audios = int(mm_counts.get("audio", 0)) - audio_overrides = mm_options.get("audio") - assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions) return { "audio": self._get_dummy_audios( length=audio_len, - num_audios=num_audios, - overrides=audio_overrides, + num_audios=int(mm_counts.get("audio", 0)), + overrides=mm_options.get("audio"), ) } diff --git a/vllm/model_executor/models/gemma3_mm.py b/vllm/model_executor/models/gemma3_mm.py index 5bccd9aec0..c6166bc2f4 100644 --- a/vllm/model_executor/models/gemma3_mm.py +++ b/vllm/model_executor/models/gemma3_mm.py @@ -11,7 +11,7 @@ from transformers.models.gemma3.image_processing_gemma3 import Gemma3ImageProces from transformers.models.gemma3.processing_gemma3 import Gemma3ProcessorKwargs from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.inputs import MultiModalDataDict from vllm.logger import init_logger from vllm.model_executor.layers.layernorm import GemmaRMSNorm @@ -244,20 +244,16 @@ class Gemma3DummyInputsBuilder(BaseDummyInputsBuilder[Gemma3ProcessingInfo]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) - target_width, target_height = self.info.get_image_size_with_most_features() - image_overrides = mm_options.get("image") - return { "image": self._get_dummy_images( width=target_width, height=target_height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ) } diff --git a/vllm/model_executor/models/gemma3n_mm.py b/vllm/model_executor/models/gemma3n_mm.py index d0f536afe8..be2e8edeb2 100644 --- a/vllm/model_executor/models/gemma3n_mm.py +++ b/vllm/model_executor/models/gemma3n_mm.py @@ -18,7 +18,7 @@ from transformers.models.gemma3n import ( from transformers.models.siglip import SiglipImageProcessorFast from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig -from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.config.speech_to_text import SpeechToTextParams from vllm.inputs import MultiModalDataDict, PromptType, TextPrompt from vllm.logger import init_logger @@ -232,10 +232,8 @@ class Gemma3nDummyInputsBuilder(BaseDummyInputsBuilder[Gemma3nProcessingInfo]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) - num_audios = mm_counts.get("audio", 0) processor = self.info.get_hf_processor() audio_feature_extractor: Gemma3nAudioFeatureExtractor = ( processor.feature_extractor @@ -245,21 +243,17 @@ class Gemma3nDummyInputsBuilder(BaseDummyInputsBuilder[Gemma3nProcessingInfo]): img_width = image_processor.size.get("width", 224) img_height = image_processor.size.get("height", 224) - image_overrides = mm_options.get("image") - audio_overrides = mm_options.get("audio") - assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions) - return { "image": self._get_dummy_images( width=img_width, height=img_height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ), "audio": self._get_dummy_audios( length=audio_len, - num_audios=num_audios, - overrides=audio_overrides, + num_audios=mm_counts.get("audio", 0), + overrides=mm_options.get("audio"), ), } diff --git a/vllm/model_executor/models/gemma4_mm.py b/vllm/model_executor/models/gemma4_mm.py index f8cb9752c8..796dfbb6e8 100644 --- a/vllm/model_executor/models/gemma4_mm.py +++ b/vllm/model_executor/models/gemma4_mm.py @@ -36,11 +36,7 @@ from transformers.models.gemma4.configuration_gemma4 import ( from vllm.config import VllmConfig from vllm.config.model import get_served_model_name -from vllm.config.multimodal import ( - AudioDummyOptions, - BaseDummyOptions, - VideoDummyOptions, -) +from vllm.config.multimodal import MultiModalDummyOptions, VideoDummyOptions from vllm.inputs import MultiModalDataDict from vllm.logger import init_logger from vllm.model_executor.layers.layernorm import RMSNorm @@ -273,10 +269,7 @@ class Gemma4ProcessingInfo(BaseProcessingInfo): num_frames = _VIDEO_MAX_FRAMES mm_config = self.ctx.model_config.get_multimodal_config() video_opts = mm_config.limit_per_prompt.get("video") - if ( - isinstance(video_opts, VideoDummyOptions) - and video_opts.num_frames is not None - ): + if video_opts is not None and video_opts.num_frames is not None: num_frames = min(num_frames, video_opts.num_frames) tokens["video"] = num_frames * (_VIDEO_MAX_SOFT_TOKENS + 2 + 6) return tokens @@ -502,9 +495,8 @@ class Gemma4DummyInputsBuilder(BaseDummyInputsBuilder[Gemma4ProcessingInfo]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions] | None = None, + mm_options: MultiModalDummyOptions | None = None, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) num_audios = mm_counts.get("audio", 0) num_videos = mm_counts.get("video", 0) processor = self.info.get_hf_processor() @@ -517,18 +509,12 @@ class Gemma4DummyInputsBuilder(BaseDummyInputsBuilder[Gemma4ProcessingInfo]): img_width = size.get("width", 224) img_height = size.get("height", 224) - image_overrides = mm_options.get("image") if mm_options else None - audio_overrides = mm_options.get("audio") if mm_options else None - video_overrides = mm_options.get("video") if mm_options else None - assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions) - assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions) - data: dict[str, Any] = { "image": self._get_dummy_images( width=img_width, height=img_height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image") if mm_options else None, ), } @@ -537,7 +523,7 @@ class Gemma4DummyInputsBuilder(BaseDummyInputsBuilder[Gemma4ProcessingInfo]): data["audio"] = self._get_dummy_audios( length=audio_len, num_audios=num_audios, - overrides=audio_overrides, + overrides=mm_options.get("audio") if mm_options else None, ) if num_videos > 0: @@ -546,7 +532,7 @@ class Gemma4DummyInputsBuilder(BaseDummyInputsBuilder[Gemma4ProcessingInfo]): height=img_height, num_frames=_VIDEO_MAX_FRAMES, num_videos=num_videos, - overrides=video_overrides, + overrides=mm_options.get("video") if mm_options else None, ) return data diff --git a/vllm/model_executor/models/gemma4_unified.py b/vllm/model_executor/models/gemma4_unified.py index 7c9aaa8499..1a26246515 100644 --- a/vllm/model_executor/models/gemma4_unified.py +++ b/vllm/model_executor/models/gemma4_unified.py @@ -28,7 +28,6 @@ from transformers.models.gemma4_unified.processing_gemma4_unified import ( ) from vllm.config import VllmConfig -from vllm.config.multimodal import VideoDummyOptions from vllm.model_executor.layers.linear import ColumnParallelLinear from vllm.model_executor.models.gemma3n_mm import batch_audio_features from vllm.model_executor.models.gemma4 import Gemma4ForCausalLM @@ -177,10 +176,7 @@ class Gemma4UnifiedProcessingInfo(Gemma4ProcessingInfo): num_frames = _VIDEO_MAX_FRAMES mm_config = self.ctx.model_config.get_multimodal_config() video_opts = mm_config.limit_per_prompt.get("video") - if ( - isinstance(video_opts, VideoDummyOptions) - and video_opts.num_frames is not None - ): + if video_opts is not None and video_opts.num_frames is not None: num_frames = min(num_frames, video_opts.num_frames) tokens["video"] = num_frames * (_VIDEO_MAX_SOFT_TOKENS + 2 + 6) return tokens diff --git a/vllm/model_executor/models/glm4_1v.py b/vllm/model_executor/models/glm4_1v.py index 1537ae7d77..95cdcc5a20 100644 --- a/vllm/model_executor/models/glm4_1v.py +++ b/vllm/model_executor/models/glm4_1v.py @@ -52,7 +52,7 @@ from transformers.video_processing_utils import BaseVideoProcessor from transformers.video_utils import VideoMetadata from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions, VideoDummyOptions from vllm.distributed import get_tensor_model_parallel_world_size, parallel_state from vllm.distributed import utils as dist_utils from vllm.inputs import MultiModalDataDict @@ -1455,33 +1455,26 @@ class Glm4vDummyInputsBuilder(BaseDummyInputsBuilder[Glm4vProcessingInfo]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) - num_videos = mm_counts.get("video", 0) - target_width, target_height = self.info.get_image_size_with_most_features() target_num_frames = self.info.get_num_frames_with_most_features( seq_len, mm_counts ) - image_overrides = mm_options.get("image") - video_overrides = mm_options.get("video") - assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions) - return { "image": self._get_dummy_images( width=target_width, height=target_height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ), "video": self._get_dummy_videos( width=target_width, height=target_height, num_frames=target_num_frames, - num_videos=num_videos, - overrides=video_overrides, + num_videos=mm_counts.get("video", 0), + overrides=mm_options.get("video"), ), } diff --git a/vllm/model_executor/models/glm4v.py b/vllm/model_executor/models/glm4v.py index 4fc6446ad3..7b42d5734f 100644 --- a/vllm/model_executor/models/glm4v.py +++ b/vllm/model_executor/models/glm4v.py @@ -16,7 +16,7 @@ from torch.nn import LayerNorm from transformers import BatchFeature from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.distributed import get_tensor_model_parallel_world_size from vllm.inputs import MultiModalDataDict from vllm.model_executor.layers.activation import SiluAndMul, get_act_fn @@ -445,22 +445,19 @@ class GLM4VDummyInputsBuilder(BaseDummyInputsBuilder[GLM4VProcessingInfo]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: hf_config = self.info.get_hf_config() vision_config = hf_config.vision_config target_width = target_height = vision_config["image_size"] - num_images = mm_counts.get("image", 0) - - image_overrides = mm_options.get("image") return { "image": self._get_dummy_images( width=target_width, height=target_height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ) } diff --git a/vllm/model_executor/models/glmasr.py b/vllm/model_executor/models/glmasr.py index 003b02c28e..cf12dedbe5 100644 --- a/vllm/model_executor/models/glmasr.py +++ b/vllm/model_executor/models/glmasr.py @@ -11,7 +11,7 @@ from transformers.models.glmasr import GlmAsrConfig, GlmAsrProcessor from transformers.models.whisper import WhisperFeatureExtractor from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig -from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.config.speech_to_text import SpeechToTextParams from vllm.distributed.parallel_state import get_tensor_model_parallel_world_size from vllm.inputs import ModalityData, MultiModalDataDict, PromptType, TokensPrompt @@ -694,13 +694,10 @@ class GlmAsrDummyInputsBuilder(BaseDummyInputsBuilder[GlmAsrProcessingInfo]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: feature_extractor = self.info.get_feature_extractor() sampling_rate = feature_extractor.sampling_rate - num_audios = mm_counts.get("audio", 0) - audio_overrides = mm_options.get("audio") - assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions) max_audio_len = getattr( self.info.get_hf_processor(), "max_audio_len", DEFAULT_MAX_AUDIO_LEN_S @@ -710,8 +707,8 @@ class GlmAsrDummyInputsBuilder(BaseDummyInputsBuilder[GlmAsrProcessingInfo]): return { "audio": self._get_dummy_audios( length=audio_len, - num_audios=num_audios, - overrides=audio_overrides, + num_audios=mm_counts.get("audio", 0), + overrides=mm_options.get("audio"), ) } diff --git a/vllm/model_executor/models/granite_speech.py b/vllm/model_executor/models/granite_speech.py index 1c1e98be04..53cbe7df43 100644 --- a/vllm/model_executor/models/granite_speech.py +++ b/vllm/model_executor/models/granite_speech.py @@ -34,7 +34,7 @@ from torch import nn from transformers import BatchFeature, PretrainedConfig from vllm.config import CacheConfig, ModelConfig, SpeechToTextConfig, VllmConfig -from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.config.speech_to_text import SpeechToTextParams from vllm.inputs import MultiModalDataDict, PromptType, TokensPrompt from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear @@ -208,17 +208,13 @@ class GraniteSpeechDummyInputsBuilder( self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_audios = mm_counts.get("audio", 0) - audio_overrides = mm_options.get("audio") - assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions) - return { "audio": self._get_dummy_audios( length=self.info.get_max_audio_len(), - num_audios=num_audios, - overrides=audio_overrides, + num_audios=mm_counts.get("audio", 0), + overrides=mm_options.get("audio"), ) } diff --git a/vllm/model_executor/models/hyperclovax_vision_v2.py b/vllm/model_executor/models/hyperclovax_vision_v2.py index 07106c9471..e269e2a059 100644 --- a/vllm/model_executor/models/hyperclovax_vision_v2.py +++ b/vllm/model_executor/models/hyperclovax_vision_v2.py @@ -18,7 +18,7 @@ import torch.nn as nn from transformers import BatchFeature from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.forward_context import set_forward_context from vllm.inputs import MultiModalDataDict from vllm.multimodal import MULTIMODAL_REGISTRY @@ -193,7 +193,7 @@ class HCXVisionV2DummyInputsBuilder(BaseDummyInputsBuilder[HCXVisionV2Processing self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions] | None = None, + mm_options: MultiModalDummyOptions | None = None, mm_processor_kwargs: Mapping[str, object] | None = None, ) -> ProcessorInputs: """Build dummy processor inputs for memory profiling.""" @@ -225,31 +225,25 @@ class HCXVisionV2DummyInputsBuilder(BaseDummyInputsBuilder[HCXVisionV2Processing self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions] | None = None, + mm_options: MultiModalDummyOptions | None = None, mm_processor_kwargs: Mapping[str, object] | None = None, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) - num_videos = mm_counts.get("video", 0) - target_width, target_height = self.info.get_image_size_with_most_features() target_num_frames = 16 # Default for video - image_overrides = mm_options.get("image") if mm_options else None - video_overrides = mm_options.get("video") if mm_options else None - result: MultiModalDataDict = { "image": self._get_dummy_images( width=target_width, height=target_height, - num_images=num_images, - overrides=image_overrides, # type: ignore + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image") if mm_options else None, ), "video": self._get_dummy_videos( width=target_width, height=target_height, num_frames=target_num_frames, - num_videos=num_videos, - overrides=video_overrides, # type: ignore + num_videos=mm_counts.get("video", 0), + overrides=mm_options.get("video") if mm_options else None, ), } diff --git a/vllm/model_executor/models/idefics3.py b/vllm/model_executor/models/idefics3.py index dd03e17d1e..80c71cfe79 100644 --- a/vllm/model_executor/models/idefics3.py +++ b/vllm/model_executor/models/idefics3.py @@ -29,7 +29,7 @@ from transformers import ( ) from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.inputs import MultiModalDataDict from vllm.model_executor.layers.linear import ReplicatedLinear from vllm.model_executor.layers.logits_processor import LogitsProcessor @@ -221,21 +221,18 @@ class Idefics3DummyInputsBuilder(BaseDummyInputsBuilder[Idefics3ProcessingInfo]) self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) hf_processor = self.info.get_hf_processor() image_processor: Idefics3ImageProcessor = hf_processor.image_processor longest_edge = image_processor.max_image_size["longest_edge"] - image_overrides = mm_options.get("image") - return { "image": self._get_dummy_images( width=longest_edge, height=longest_edge, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ) } diff --git a/vllm/model_executor/models/interns1.py b/vllm/model_executor/models/interns1.py index 4a4264660d..8d818e9a78 100644 --- a/vllm/model_executor/models/interns1.py +++ b/vllm/model_executor/models/interns1.py @@ -22,7 +22,7 @@ from transformers.models.internvl.video_processing_internvl import ( ) from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.inputs import MultiModalDataDict from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.model_executor.models.interns1_vit import InternS1VisionModel @@ -299,35 +299,29 @@ class InternS1DummyInputsBuilder(BaseDummyInputsBuilder[InternS1ProcessingInfo]) self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: target_width, target_height = self.info.get_image_size_with_most_features() target_num_frames = self.info.get_num_frames_with_most_features( seq_len, mm_counts ) - num_images = mm_counts.get("image", 0) - num_videos = mm_counts.get("video", 0) config = self.info.get_hf_config() image_size_h, image_size_w = config.vision_config.image_size - image_overrides = mm_options.get("image") - video_overrides = mm_options.get("video") - assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions) - return { "image": self._get_dummy_images( width=target_width, height=target_height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ), "video": self._get_dummy_videos( width=image_size_w, height=image_size_h, num_frames=target_num_frames, - num_videos=num_videos, - overrides=video_overrides, + num_videos=mm_counts.get("video", 0), + overrides=mm_options.get("video"), ), } diff --git a/vllm/model_executor/models/internvl.py b/vllm/model_executor/models/internvl.py index 873c594c54..3640f8d6a3 100644 --- a/vllm/model_executor/models/internvl.py +++ b/vllm/model_executor/models/internvl.py @@ -17,7 +17,7 @@ import torch.nn as nn from transformers import BatchFeature, PretrainedConfig from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.inputs import MultiModalDataDict from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig @@ -203,19 +203,16 @@ class BaseInternVLDummyInputsBuilder(BaseDummyInputsBuilder[_I]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: target_width, target_height = self.info.get_image_size_with_most_features() - num_images = mm_counts.get("image", 0) - - image_overrides = mm_options.get("image") return { "image": self._get_dummy_images( width=target_width, height=target_height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ) } @@ -425,7 +422,7 @@ class InternVLDummyInputsBuilder( self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: dummy_image = super().get_dummy_mm_data(seq_len, mm_counts, mm_options) if self.info.ctx_video_token: @@ -436,9 +433,6 @@ class InternVLDummyInputsBuilder( ) num_videos = mm_counts.get("video", 0) video_overrides = mm_options.get("video") - assert video_overrides is None or isinstance( - video_overrides, VideoDummyOptions - ) dummy_video = { "video": self._get_dummy_videos( width=image_size, diff --git a/vllm/model_executor/models/isaac.py b/vllm/model_executor/models/isaac.py index f7c1ee2dd1..f8f6dee32c 100644 --- a/vllm/model_executor/models/isaac.py +++ b/vllm/model_executor/models/isaac.py @@ -13,7 +13,7 @@ from einops import rearrange from transformers.image_processing_utils import BatchFeature from vllm.config import ModelConfig, VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.distributed import parallel_state from vllm.distributed import utils as dist_utils from vllm.inputs import MultiModalDataDict @@ -386,19 +386,16 @@ class IsaacDummyInputsBuilder(BaseDummyInputsBuilder[IsaacProcessingInfo]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) - target_width, target_height = self.info.get_image_size_with_most_features() - image_overrides = mm_options.get("image") return { "image": self._get_dummy_images( width=target_width, height=target_height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ), } diff --git a/vllm/model_executor/models/kanana_v.py b/vllm/model_executor/models/kanana_v.py index e3c27e479b..fd076d66e7 100644 --- a/vllm/model_executor/models/kanana_v.py +++ b/vllm/model_executor/models/kanana_v.py @@ -18,7 +18,7 @@ from transformers.modeling_outputs import BaseModelOutput from transformers.models.qwen2_vl.configuration_qwen2_vl import Qwen2VLVisionConfig from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.inputs import MultiModalDataDict from vllm.logger import init_logger from vllm.multimodal import MULTIMODAL_REGISTRY @@ -442,7 +442,7 @@ class KananaVDummyInputsBuilder(BaseDummyInputsBuilder[KananaVProcessingInfo]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: num_images = mm_counts.get("image", 0) return { diff --git a/vllm/model_executor/models/keye.py b/vllm/model_executor/models/keye.py index 1322920900..bfd2af714b 100644 --- a/vllm/model_executor/models/keye.py +++ b/vllm/model_executor/models/keye.py @@ -17,7 +17,7 @@ from transformers.modeling_outputs import BaseModelOutput, BaseModelOutputWithPo from transformers.utils import torch_int from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.distributed import get_tensor_model_parallel_world_size from vllm.inputs import ModalityData, MultiModalDataDict from vllm.logger import init_logger @@ -1124,30 +1124,24 @@ class KeyeBaseDummyInputsBuilder(BaseDummyInputsBuilder[_I]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) - num_videos = mm_counts.get("video", 0) - target_width, target_height = self.info.get_image_size_with_most_features() target_num_frames = self.info.get_num_frames_with_most_features(seq_len) - image_overrides = mm_options.get("image") - video_overrides = mm_options.get("video") - mm_data = { "image": self._get_dummy_images( width=target_width, height=target_height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ), "video": self._get_dummy_videos( width=target_width, height=target_height, num_frames=target_num_frames, - num_videos=num_videos, - overrides=video_overrides, + num_videos=mm_counts.get("video", 0), + overrides=mm_options.get("video"), ), } diff --git a/vllm/model_executor/models/kimi_audio.py b/vllm/model_executor/models/kimi_audio.py index 219c7f0e7b..3541fcf371 100644 --- a/vllm/model_executor/models/kimi_audio.py +++ b/vllm/model_executor/models/kimi_audio.py @@ -13,7 +13,7 @@ from transformers import BatchFeature from transformers import WhisperConfig as HFWhisperConfig from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.config.speech_to_text import SpeechToTextParams from vllm.inputs import PromptType, TokensPrompt from vllm.model_executor.model_loader import DefaultModelLoader @@ -180,7 +180,7 @@ class KimiAudioDummyInputsBuilder(BaseDummyInputsBuilder[KimiAudioProcessingInfo self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> ProcessorInputs: dummy_mm_data = self.get_dummy_mm_data(seq_len, mm_counts, mm_options) dummy_mm_items = self.info.parse_mm_data(dummy_mm_data) diff --git a/vllm/model_executor/models/kimi_k25.py b/vllm/model_executor/models/kimi_k25.py index 9af46e441a..81b5ec1646 100644 --- a/vllm/model_executor/models/kimi_k25.py +++ b/vllm/model_executor/models/kimi_k25.py @@ -14,7 +14,7 @@ from torch import nn from transformers import BatchFeature from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.inputs import MultiModalDataDict from vllm.logger import init_logger from vllm.model_executor.layers.quantization import QuantizationConfig @@ -216,7 +216,7 @@ class KimiK25DummyInputsBuilder(BaseDummyInputsBuilder[KimiK25ProcessingInfo]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: # TODO: Support mm_options for vision_chunk to allow user configuration dummy_items = self.get_dummy_mm_items() diff --git a/vllm/model_executor/models/kimi_vl.py b/vllm/model_executor/models/kimi_vl.py index 344dfdaaf0..13779042b6 100644 --- a/vllm/model_executor/models/kimi_vl.py +++ b/vllm/model_executor/models/kimi_vl.py @@ -53,7 +53,7 @@ from transformers import BatchFeature from transformers.activations import GELUActivation from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.inputs import MultiModalDataDict from vllm.model_executor.layers.linear import ReplicatedLinear from vllm.model_executor.models.interfaces import ( @@ -219,18 +219,14 @@ class KimiVLDummyInputsBuilder(BaseDummyInputsBuilder[KimiVLProcessingInfo]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) - - image_overrides = mm_options.get("image") - return { "image": self._get_dummy_images( width=MaxImageTokenMeta.width, height=MaxImageTokenMeta.height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ) } diff --git a/vllm/model_executor/models/lfm2_vl.py b/vllm/model_executor/models/lfm2_vl.py index 1cd51b93a2..644da00c81 100644 --- a/vllm/model_executor/models/lfm2_vl.py +++ b/vllm/model_executor/models/lfm2_vl.py @@ -21,7 +21,7 @@ from transformers.models.lfm2_vl.image_processing_lfm2_vl_fast import ( from typing_extensions import Buffer from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.forward_context import set_forward_context from vllm.inputs import MultiModalDataDict from vllm.model_executor.layers.linear import ReplicatedLinear @@ -387,20 +387,16 @@ class Lfm2VLDummyInputsBuilder(BaseDummyInputsBuilder[Lfm2VLProcessingInfo]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) - target_width, target_height = self.info.get_image_size_with_most_features() - image_overrides = mm_options.get("image") - return { "image": self._get_dummy_images( width=target_width, height=target_height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ), } diff --git a/vllm/model_executor/models/llava.py b/vllm/model_executor/models/llava.py index c3a437eb36..3ea0df9d16 100644 --- a/vllm/model_executor/models/llava.py +++ b/vllm/model_executor/models/llava.py @@ -19,7 +19,7 @@ from transformers.models.llava import LlavaProcessor from transformers.models.pixtral import PixtralProcessor from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.inputs import MultiModalDataDict from vllm.model_executor.layers.activation import get_act_fn from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear @@ -228,20 +228,16 @@ class LlavaDummyInputsBuilder(BaseDummyInputsBuilder[_I]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) - target_width, target_height = self.info.get_image_size_with_most_features() - image_overrides = mm_options.get("image") - return { "image": self._get_dummy_images( width=target_width, height=target_height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ) } diff --git a/vllm/model_executor/models/llava_next_video.py b/vllm/model_executor/models/llava_next_video.py index 235d1adbac..a33480ef08 100644 --- a/vllm/model_executor/models/llava_next_video.py +++ b/vllm/model_executor/models/llava_next_video.py @@ -10,7 +10,7 @@ import torch.nn as nn from transformers import BatchFeature, LlavaNextVideoConfig, LlavaNextVideoProcessor from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.inputs import MultiModalDataDict from vllm.model_executor.layers.activation import get_act_fn from vllm.model_executor.models.clip import CLIPVisionModel @@ -171,25 +171,20 @@ class LlavaNextVideoDummyInputsBuilder( self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_videos = mm_counts.get("video", 0) - target_width, target_height = self.info.get_image_size_with_most_features() target_num_frames = self.info.get_num_frames_with_most_features( seq_len, mm_counts ) - video_overrides = mm_options.get("video") - assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions) - return { "video": self._get_dummy_videos( width=target_width, height=target_height, num_frames=target_num_frames, - num_videos=num_videos, - overrides=video_overrides, + num_videos=mm_counts.get("video", 0), + overrides=mm_options.get("video"), ) } diff --git a/vllm/model_executor/models/llava_onevision.py b/vllm/model_executor/models/llava_onevision.py index 66503d7741..6fc96c8e67 100644 --- a/vllm/model_executor/models/llava_onevision.py +++ b/vllm/model_executor/models/llava_onevision.py @@ -14,7 +14,7 @@ from transformers.models.llava_onevision.modeling_llava_onevision import ( ) from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.inputs import MultiModalDataDict from vllm.model_executor.layers.activation import get_act_fn from vllm.multimodal import MULTIMODAL_REGISTRY @@ -278,33 +278,26 @@ class LlavaOnevisionDummyInputsBuilder( self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) - num_videos = mm_counts.get("video", 0) - target_width, target_height = self.info.get_image_size_with_most_features() target_num_frames = self.info.get_num_frames_with_most_features( seq_len, mm_counts ) - image_overrides = mm_options.get("image") - video_overrides = mm_options.get("video") - assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions) - return { "image": self._get_dummy_images( width=target_width, height=target_height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ), "video": self._get_dummy_videos( width=target_width, height=target_height, num_frames=target_num_frames, - num_videos=num_videos, - overrides=video_overrides, + num_videos=mm_counts.get("video", 0), + overrides=mm_options.get("video"), ), } diff --git a/vllm/model_executor/models/llava_onevision2.py b/vllm/model_executor/models/llava_onevision2.py index 6988982c23..d8d514be23 100644 --- a/vllm/model_executor/models/llava_onevision2.py +++ b/vllm/model_executor/models/llava_onevision2.py @@ -54,7 +54,7 @@ from vllm.compilation.decorators import ( support_torch_compile, ) from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.distributed import parallel_state from vllm.distributed import utils as dist_utils from vllm.inputs import ModalityData, MultiModalDataDict @@ -1471,7 +1471,7 @@ class LlavaOnevision2DummyInputsBuilder( self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions] | None = None, + mm_options: MultiModalDummyOptions | None = None, ) -> MultiModalDataDict: n_img = mm_counts.get("image", 0) n_vid = mm_counts.get("video", 0) diff --git a/vllm/model_executor/models/midashenglm.py b/vllm/model_executor/models/midashenglm.py index 7de6ab0e7e..6dae25ffb2 100644 --- a/vllm/model_executor/models/midashenglm.py +++ b/vllm/model_executor/models/midashenglm.py @@ -36,7 +36,7 @@ from torch.nn.functional import scaled_dot_product_attention from transformers import BatchFeature from vllm.config import VllmConfig -from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.distributed import get_tensor_model_parallel_world_size from vllm.inputs import MultiModalDataDict from vllm.model_executor.layers.activation import get_act_fn @@ -564,18 +564,13 @@ class MiDashengLMDummyInputsBuilder(BaseDummyInputsBuilder[MiDashengLMProcessing self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_audios = mm_counts.get("audio", 0) - - audio_overrides = mm_options.get("audio") - assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions) - return { "audio": self._get_dummy_audios( length=self.info.get_max_audio_len(), - num_audios=num_audios, - overrides=audio_overrides, + num_audios=mm_counts.get("audio", 0), + overrides=mm_options.get("audio"), ) } diff --git a/vllm/model_executor/models/mimo_v2_omni.py b/vllm/model_executor/models/mimo_v2_omni.py index d74edd6d4a..91a3177b7f 100644 --- a/vllm/model_executor/models/mimo_v2_omni.py +++ b/vllm/model_executor/models/mimo_v2_omni.py @@ -15,11 +15,7 @@ from transformers.models.qwen2_vl.image_processing_qwen2_vl import smart_resize from typing_extensions import TypedDict from vllm.config import VllmConfig -from vllm.config.multimodal import ( - BaseDummyOptions, - ImageDummyOptions, - VideoDummyOptions, -) +from vllm.config.multimodal import MultiModalDummyOptions from vllm.distributed import parallel_state from vllm.distributed import utils as dist_utils from vllm.inputs import MultiModalDataDict @@ -1184,33 +1180,26 @@ class MiMoV2OmniDummyInputsBuilder(BaseDummyInputsBuilder[MiMoV2OmniProcessingIn self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) - num_videos = mm_counts.get("video", 0) - target_width, target_height = self.info.get_image_size_with_most_features() target_num_frames = self.info.get_num_frames_with_most_features( seq_len, mm_counts ) - image_overrides = mm_options.get("image") - video_overrides = mm_options.get("video") - assert image_overrides is None or isinstance(image_overrides, ImageDummyOptions) - assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions) return { "image": self._get_dummy_images( width=target_width, height=target_height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ), "video": self._get_dummy_videos( width=target_width, height=target_height, num_frames=target_num_frames, - num_videos=num_videos, - overrides=video_overrides, + num_videos=mm_counts.get("video", 0), + overrides=mm_options.get("video"), ), } diff --git a/vllm/model_executor/models/minicpmo.py b/vllm/model_executor/models/minicpmo.py index 2ec8567921..cc8a1066f2 100644 --- a/vllm/model_executor/models/minicpmo.py +++ b/vllm/model_executor/models/minicpmo.py @@ -40,7 +40,7 @@ from transformers.models.whisper.modeling_whisper import ( ) from vllm.config import VllmConfig -from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.inputs import ModalityData, MultiModalDataDict from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargsItems from vllm.multimodal.inputs import ( @@ -402,22 +402,18 @@ class MiniCPMODummyInputsBuilder(MiniCPMVDummyInputsBuilder[MiniCPMOProcessingIn self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_audios = mm_counts.get("audio", 0) audio_len = ( self.info.get_max_audio_chunks_with_most_features() * self.info.get_default_audio_sampling_rate() ) - audio_overrides = mm_options.get("audio") - assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions) - audio_mm_data = { "audio": self._get_dummy_audios( length=audio_len, - num_audios=num_audios, - overrides=audio_overrides, + num_audios=mm_counts.get("audio", 0), + overrides=mm_options.get("audio"), ) } diff --git a/vllm/model_executor/models/minicpmv.py b/vllm/model_executor/models/minicpmv.py index 14720c5c86..8027dc1027 100644 --- a/vllm/model_executor/models/minicpmv.py +++ b/vllm/model_executor/models/minicpmv.py @@ -45,9 +45,8 @@ from typing_extensions import TypedDict, TypeVar from vllm.config import VllmConfig from vllm.config.multimodal import ( - BaseDummyOptions, ImageDummyOptions, - VideoDummyOptions, + MultiModalDummyOptions, ) from vllm.inputs import ModalityData, MultiModalDataDict from vllm.model_executor.layers.quantization import QuantizationConfig @@ -848,9 +847,8 @@ class MiniCPMVDummyInputsBuilder(BaseDummyInputsBuilder[_I]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) num_videos = mm_counts.get("video", 0) image_width, image_height = self.info.get_image_size_with_most_features() @@ -859,14 +857,12 @@ class MiniCPMVDummyInputsBuilder(BaseDummyInputsBuilder[_I]): seq_len, mm_counts ) - image_overrides = mm_options.get("image") video_overrides = mm_options.get("video") - assert image_overrides is None or isinstance(image_overrides, ImageDummyOptions) # Convert video overrides to image overrides for per-frame image generation, # and apply num_frames override to num_video_frames. video_frame_overrides: ImageDummyOptions | None = None - if isinstance(video_overrides, VideoDummyOptions): + if video_overrides is not None: if video_overrides.num_frames: num_video_frames = min(num_video_frames, video_overrides.num_frames) if video_overrides.width or video_overrides.height: @@ -880,8 +876,8 @@ class MiniCPMVDummyInputsBuilder(BaseDummyInputsBuilder[_I]): "image": self._get_dummy_images( width=image_width, height=image_height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ), "video": [ self._get_dummy_images( diff --git a/vllm/model_executor/models/mistral3.py b/vllm/model_executor/models/mistral3.py index f9d7dafe70..41352dc427 100644 --- a/vllm/model_executor/models/mistral3.py +++ b/vllm/model_executor/models/mistral3.py @@ -11,7 +11,7 @@ from transformers import BatchFeature, Mistral3Config, PixtralVisionConfig from transformers.models.pixtral import PixtralProcessor from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.inputs import MultiModalDataDict from vllm.model_executor.layers.activation import get_act_fn from vllm.model_executor.layers.layernorm import RMSNorm @@ -261,20 +261,16 @@ class Mistral3DummyInputsBuilder(BaseDummyInputsBuilder): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) - target_width, target_height = self.info.get_image_size_with_most_features() - image_overrides = mm_options.get("image") - return { "image": self._get_dummy_images( width=target_width, height=target_height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ) } diff --git a/vllm/model_executor/models/mllama4.py b/vllm/model_executor/models/mllama4.py index 797c06d180..89d4edbe86 100644 --- a/vllm/model_executor/models/mllama4.py +++ b/vllm/model_executor/models/mllama4.py @@ -36,7 +36,7 @@ from vllm.compilation.decorators import ( support_torch_compile, ) from vllm.config import VllmConfig, set_current_vllm_config -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.distributed import get_tensor_model_parallel_world_size from vllm.inputs import MultiModalDataDict from vllm.model_executor.layers.attention import MMEncoderAttention @@ -709,20 +709,16 @@ class Mllama4DummyInputsBuilder(BaseDummyInputsBuilder[Mllama4ProcessingInfo]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) - (target_width, target_height) = self.info.get_image_size_with_most_features() - image_overrides = mm_options.get("image") - return { "image": self._get_dummy_images( width=target_width, height=target_height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ) } diff --git a/vllm/model_executor/models/molmo.py b/vllm/model_executor/models/molmo.py index 9aeef3715c..17ad7f0be1 100644 --- a/vllm/model_executor/models/molmo.py +++ b/vllm/model_executor/models/molmo.py @@ -21,7 +21,7 @@ from transformers import ( from vllm.compilation.decorators import support_torch_compile from vllm.config import CacheConfig, VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.distributed import ( get_pp_group, get_tensor_model_parallel_rank, @@ -1104,19 +1104,16 @@ class MolmoDummyInputsBuilder(BaseDummyInputsBuilder[MolmoProcessingInfo]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: target_width, target_height = self.info.get_image_size_with_most_features() - num_images = mm_counts.get("image", 0) - - image_overrides = mm_options.get("image") return { "image": self._get_dummy_images( width=target_width, height=target_height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ) } diff --git a/vllm/model_executor/models/molmo2.py b/vllm/model_executor/models/molmo2.py index 4c0ce0868e..881be146e7 100644 --- a/vllm/model_executor/models/molmo2.py +++ b/vllm/model_executor/models/molmo2.py @@ -26,7 +26,7 @@ from typing_extensions import TypedDict from vllm.compilation.decorators import support_torch_compile from vllm.config import CacheConfig, VllmConfig -from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions, VideoDummyOptions from vllm.distributed import ( get_pp_group, get_tensor_model_parallel_rank, @@ -1834,7 +1834,7 @@ class Molmo2DummyInputsBuilder(BaseDummyInputsBuilder[Molmo2ProcessingInfo]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: num_images = mm_counts.get("image", 0) num_videos = mm_counts.get("video", 0) @@ -1845,13 +1845,11 @@ class Molmo2DummyInputsBuilder(BaseDummyInputsBuilder[Molmo2ProcessingInfo]): if num_images > 0: target_width, target_height = self.info.get_image_size_with_most_features() - image_overrides = mm_options.get("image") - dummy_images = self._get_dummy_images( width=target_width, height=target_height, num_images=num_images, - overrides=image_overrides, + overrides=mm_options.get("image"), ) if num_videos > 0: @@ -1864,7 +1862,6 @@ class Molmo2DummyInputsBuilder(BaseDummyInputsBuilder[Molmo2ProcessingInfo]): video_overrides = mm_options.get("video") if video_overrides: - assert isinstance(video_overrides, VideoDummyOptions) num_frames_override = video_overrides.num_frames if num_frames_override: if num_frames_override > target_num_frames: diff --git a/vllm/model_executor/models/moondream3.py b/vllm/model_executor/models/moondream3.py index 45b9420184..1e750fc9d0 100644 --- a/vllm/model_executor/models/moondream3.py +++ b/vllm/model_executor/models/moondream3.py @@ -13,7 +13,7 @@ import torch.nn.functional as F from transformers import BatchFeature from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.distributed import ( get_pp_group, get_tensor_model_parallel_rank, @@ -943,15 +943,14 @@ class Moondream3DummyInputsBuilder(BaseDummyInputsBuilder[Moondream3ProcessingIn self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions] | None = None, + mm_options: MultiModalDummyOptions | None = None, mm_processor_kwargs: Mapping[str, object] | None = None, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) return { "image": self._get_dummy_images( width=378, height=378, - num_images=num_images, + num_images=mm_counts.get("image", 0), ) } diff --git a/vllm/model_executor/models/moss_audio.py b/vllm/model_executor/models/moss_audio.py index 6092812585..02551d5663 100644 --- a/vllm/model_executor/models/moss_audio.py +++ b/vllm/model_executor/models/moss_audio.py @@ -19,7 +19,7 @@ from typing_extensions import TypedDict from vllm.compilation.decorators import support_torch_compile from vllm.config import VllmConfig -from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.distributed import ( get_pp_group, get_tensor_model_parallel_world_size, @@ -1341,16 +1341,13 @@ class MossAudioDummyInputsBuilder(BaseDummyInputsBuilder[MossAudioProcessingInfo self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_audios = mm_counts.get("audio", 0) - audio_overrides = mm_options.get("audio") - assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions) return { "audio": self._get_dummy_audios( length=16000, - num_audios=num_audios, - overrides=audio_overrides, + num_audios=mm_counts.get("audio", 0), + overrides=mm_options.get("audio"), ) } diff --git a/vllm/model_executor/models/moss_transcribe_diarize.py b/vllm/model_executor/models/moss_transcribe_diarize.py index 32ac69c9af..208d5d4bc1 100644 --- a/vllm/model_executor/models/moss_transcribe_diarize.py +++ b/vllm/model_executor/models/moss_transcribe_diarize.py @@ -20,7 +20,7 @@ from torch import nn from transformers import BatchFeature from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig -from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.config.speech_to_text import SpeechToTextParams from vllm.inputs import ModalityData, MultiModalDataDict, PromptType, TextPrompt from vllm.model_executor.models.interfaces import ( @@ -423,20 +423,18 @@ class MossTranscribeDiarizeDummyInputsBuilder( self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: num_audios = mm_counts.get("audio", 0) if num_audios == 0: return {} feature_extractor = self.info.get_feature_extractor() - audio_overrides = mm_options.get("audio") - assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions) return { "audio": self._get_dummy_audios( length=_get_max_audio_samples(feature_extractor), num_audios=num_audios, - overrides=audio_overrides, + overrides=mm_options.get("audio"), ) } @@ -444,7 +442,7 @@ class MossTranscribeDiarizeDummyInputsBuilder( self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> ProcessorInputs: dummy_mm_data = self.get_dummy_mm_data(seq_len, mm_counts, mm_options) dummy_mm_items = self.info.parse_mm_data(dummy_mm_data) diff --git a/vllm/model_executor/models/muse_glimmer.py b/vllm/model_executor/models/muse_glimmer.py index ed8064542b..a7938bb56b 100644 --- a/vllm/model_executor/models/muse_glimmer.py +++ b/vllm/model_executor/models/muse_glimmer.py @@ -45,11 +45,7 @@ from transformers import BatchFeature from vllm.compilation.decorators import support_torch_compile from vllm.config import CacheConfig, VllmConfig -from vllm.config.multimodal import ( - BaseDummyOptions, - ImageDummyOptions, - VideoDummyOptions, -) +from vllm.config.multimodal import MultiModalDummyOptions from vllm.distributed import ( divide, get_pp_group, @@ -234,7 +230,7 @@ class MuseGlimmerDummyInputsBuilder(BaseDummyInputsBuilder[MuseGlimmerProcessing self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: processor = self.info.get_hf_processor() video_processor = processor.video_processor @@ -245,16 +241,12 @@ class MuseGlimmerDummyInputsBuilder(BaseDummyInputsBuilder[MuseGlimmerProcessing * int(video_processor.downsample_factor) * video_grid ) - image_overrides = mm_options.get("image") - video_overrides = mm_options.get("video") - assert image_overrides is None or isinstance(image_overrides, ImageDummyOptions) - assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions) return { "image": self._get_dummy_images( width=image_width, height=image_height, num_images=mm_counts.get("image", 0), - overrides=image_overrides, + overrides=mm_options.get("image"), ), "video": self._get_dummy_videos( width=video_size, @@ -263,7 +255,7 @@ class MuseGlimmerDummyInputsBuilder(BaseDummyInputsBuilder[MuseGlimmerProcessing seq_len, mm_counts ), num_videos=mm_counts.get("video", 0), - overrides=video_overrides, + overrides=mm_options.get("video"), ), } diff --git a/vllm/model_executor/models/nano_nemotron_vl.py b/vllm/model_executor/models/nano_nemotron_vl.py index fee431617c..de61880b5f 100644 --- a/vllm/model_executor/models/nano_nemotron_vl.py +++ b/vllm/model_executor/models/nano_nemotron_vl.py @@ -21,8 +21,7 @@ from transformers import BatchFeature, PretrainedConfig from vllm import envs from vllm.config import VllmConfig from vllm.config.multimodal import ( - AudioDummyOptions, - BaseDummyOptions, + MultiModalDummyOptions, VideoDummyOptions, ) from vllm.inputs import MultiModalDataDict, MultiModalInput @@ -839,22 +838,19 @@ class NanoNemotronVLDummyInputsBuilder( self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) (target_width, target_height), _ = ( self.info.get_dummy_image_size_and_max_tokens(mm_counts) ) processor = self.info.get_hf_processor() - image_overrides = mm_options.get("image") - dummy_image = { "image": self._get_dummy_images( width=target_width, height=target_height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ) } @@ -889,9 +885,6 @@ class NanoNemotronVLDummyInputsBuilder( target_num_frames = num_frames num_videos = mm_counts.get("video", 0) video_overrides = mm_options.get("video") - assert video_overrides is None or isinstance( - video_overrides, VideoDummyOptions - ) dummy_video = { "video": self._get_dummy_videos( width=video_width, @@ -907,9 +900,6 @@ class NanoNemotronVLDummyInputsBuilder( if sound_config := self.info.sound_config: num_audios = mm_counts.get("audio", 0) audio_overrides = mm_options.get("audio") if mm_options else None - assert audio_overrides is None or isinstance( - audio_overrides, AudioDummyOptions - ) tokens_per_audio = max(1, seq_len // max(num_audios, 1)) max_audio_num_samples = MAX_AUDIO_LEN_S * sound_config.sampling_rate calculated_max_audio_num_samples = ParakeetExtractor.audio_length( diff --git a/vllm/model_executor/models/nemotron_parse.py b/vllm/model_executor/models/nemotron_parse.py index 01b5578531..5c4375b600 100644 --- a/vllm/model_executor/models/nemotron_parse.py +++ b/vllm/model_executor/models/nemotron_parse.py @@ -22,7 +22,7 @@ from transformers import ( from vllm.config import CacheConfig, VllmConfig from vllm.config.lora import LoRAConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.inputs import MultiModalDataDict from vllm.logger import init_logger from vllm.model_executor.layers.activation import get_act_fn @@ -388,7 +388,7 @@ class NemotronParseDummyInputsBuilder( self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: num_images = mm_counts.get("image", 0) diff --git a/vllm/model_executor/models/nvlm_d.py b/vllm/model_executor/models/nvlm_d.py index a69f024a01..5efcdb6f53 100644 --- a/vllm/model_executor/models/nvlm_d.py +++ b/vllm/model_executor/models/nvlm_d.py @@ -13,7 +13,7 @@ import torch import torch.nn as nn from transformers import PretrainedConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.inputs import MultiModalDataDict from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.multimodal import MULTIMODAL_REGISTRY @@ -83,19 +83,16 @@ class NVLMDummyInputsBuilder(BaseInternVLDummyInputsBuilder[NVLMProcessingInfo]) self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: target_width, target_height = self.info.get_image_size_with_most_features() - num_images = mm_counts.get("image", 0) - - image_overrides = mm_options.get("image") return { "image": self._get_dummy_images( width=target_width, height=target_height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ) } diff --git a/vllm/model_executor/models/openvla.py b/vllm/model_executor/models/openvla.py index add7eac41f..9242f7e1f5 100644 --- a/vllm/model_executor/models/openvla.py +++ b/vllm/model_executor/models/openvla.py @@ -9,7 +9,7 @@ import torch.nn as nn from transformers import BatchFeature from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.inputs import MultiModalDataDict from vllm.model_executor.layers.activation import get_act_fn from vllm.model_executor.layers.linear import ( @@ -299,18 +299,16 @@ class OpenVLADummyInputsBuilder(BaseDummyInputsBuilder[OpenVLAProcessingInfo]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) - image_overrides = mm_options.get("image") image_size = self.info.get_image_size_with_most_features() return { "image": self._get_dummy_images( width=image_size.width, height=image_size.height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ) } diff --git a/vllm/model_executor/models/ovis.py b/vllm/model_executor/models/ovis.py index bd00686b41..ed0ea7ab7b 100644 --- a/vllm/model_executor/models/ovis.py +++ b/vllm/model_executor/models/ovis.py @@ -29,7 +29,7 @@ from torch.nn.functional import gumbel_softmax, pad, softmax from transformers import BatchFeature, PretrainedConfig from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.inputs import MultiModalDataDict from vllm.model_executor.layers.linear import ReplicatedLinear from vllm.model_executor.layers.quantization import QuantizationConfig @@ -311,20 +311,16 @@ class OvisDummyInputsBuilder(BaseDummyInputsBuilder[OvisProcessingInfo]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) - target_width, target_height = self.info.get_image_size_with_most_features() - image_overrides = mm_options.get("image") - mm_data = { "image": self._get_dummy_images( width=target_width, height=target_height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ), } return mm_data diff --git a/vllm/model_executor/models/ovis2_5.py b/vllm/model_executor/models/ovis2_5.py index 588c06b8de..2520b38ae3 100644 --- a/vllm/model_executor/models/ovis2_5.py +++ b/vllm/model_executor/models/ovis2_5.py @@ -11,7 +11,7 @@ import torch.nn as nn from transformers import BaseImageProcessor, BatchFeature, PretrainedConfig from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.inputs import MultiModalDataDict from vllm.model_executor.layers.linear import ReplicatedLinear from vllm.model_executor.layers.quantization import QuantizationConfig @@ -290,33 +290,26 @@ class Ovis2_5DummyInputsBuilder(BaseDummyInputsBuilder[Ovis2_5ProcessingInfo]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) - num_videos = mm_counts.get("video", 0) - target_width, target_height = self.info.get_image_size_with_most_features() target_num_frames = self.info.get_num_frames_with_most_features( seq_len, mm_counts ) - image_overrides = mm_options.get("image") - video_overrides = mm_options.get("video") - assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions) - mm_data = { "image": self._get_dummy_images( width=target_width, height=target_height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ), "video": self._get_dummy_videos( width=target_width, height=target_height, num_frames=target_num_frames, - num_videos=num_videos, - overrides=video_overrides, + num_videos=mm_counts.get("video", 0), + overrides=mm_options.get("video"), ), } return mm_data diff --git a/vllm/model_executor/models/paddleocr_vl.py b/vllm/model_executor/models/paddleocr_vl.py index 976d82032f..69dae97d8d 100644 --- a/vllm/model_executor/models/paddleocr_vl.py +++ b/vllm/model_executor/models/paddleocr_vl.py @@ -32,7 +32,7 @@ from transformers.modeling_outputs import ( from transformers.utils import torch_int from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.distributed import parallel_state from vllm.distributed import utils as dist_utils from vllm.inputs import MultiModalDataDict @@ -223,19 +223,16 @@ class PaddleOCRVLDummyInputsBuilder(BaseDummyInputsBuilder[PaddleOCRVLProcessing self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) - max_image_size = self.info.get_image_size_with_most_features() - image_overrides = mm_options.get("image") return { "image": self._get_dummy_images( width=max_image_size.width, height=max_image_size.height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ) } diff --git a/vllm/model_executor/models/paligemma.py b/vllm/model_executor/models/paligemma.py index c1eafa2f09..d0173e4c11 100644 --- a/vllm/model_executor/models/paligemma.py +++ b/vllm/model_executor/models/paligemma.py @@ -8,7 +8,7 @@ from torch import nn from transformers import BatchFeature, PaliGemmaConfig from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.inputs import MultiModalDataDict, MultiModalInput from vllm.logger import init_logger from vllm.multimodal import MULTIMODAL_REGISTRY @@ -130,22 +130,18 @@ class PaliGemmaDummyInputsBuilder(BaseDummyInputsBuilder[PaliGemmaProcessingInfo self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: hf_config = self.info.get_hf_config() vision_config = hf_config.vision_config max_image_size = vision_config.image_size - num_images = mm_counts.get("image", 0) - - image_overrides = mm_options.get("image") - return { "image": self._get_dummy_images( width=max_image_size, height=max_image_size, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ) } diff --git a/vllm/model_executor/models/phi3v.py b/vllm/model_executor/models/phi3v.py index a75b4afd22..2486dc3e8c 100644 --- a/vllm/model_executor/models/phi3v.py +++ b/vllm/model_executor/models/phi3v.py @@ -29,7 +29,7 @@ from transformers import ( ) from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.inputs import MultiModalDataDict from vllm.logger import init_logger from vllm.model_executor.layers.quantization import QuantizationConfig @@ -371,20 +371,16 @@ class Phi3VDummyInputsBuilder(BaseDummyInputsBuilder[Phi3VProcessingInfo]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) - target_width, target_height = self.info.get_image_size_with_most_features() - image_overrides = mm_options.get("image") - return { "image": self._get_dummy_images( width=target_width, height=target_height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ) } diff --git a/vllm/model_executor/models/phi4mm.py b/vllm/model_executor/models/phi4mm.py index c08eda612d..86cb485967 100644 --- a/vllm/model_executor/models/phi4mm.py +++ b/vllm/model_executor/models/phi4mm.py @@ -17,7 +17,7 @@ from transformers import ( ) from vllm.config import VllmConfig -from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.distributed import get_pp_group from vllm.inputs import MultiModalDataDict from vllm.model_executor.layers.logits_processor import LogitsProcessor @@ -839,28 +839,21 @@ class Phi4MMDummyInputsBuilder(BaseDummyInputsBuilder[Phi4MMProcessingInfo]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_audios = mm_counts.get("audio", 0) - num_images = mm_counts.get("image", 0) - target_width, target_height = self.info.get_image_size_with_most_features() - image_overrides = mm_options.get("image") - audio_overrides = mm_options.get("audio") - assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions) - mm_data = { "image": self._get_dummy_images( width=target_width, height=target_height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ), "audio": self._get_dummy_audios( length=_AUDIO_MAX_SOUNDFILE_SIZE, - num_audios=num_audios, - overrides=audio_overrides, + num_audios=mm_counts.get("audio", 0), + overrides=mm_options.get("audio"), ), } diff --git a/vllm/model_executor/models/phi4siglip.py b/vllm/model_executor/models/phi4siglip.py index fd9fa94c7d..078f3155c2 100644 --- a/vllm/model_executor/models/phi4siglip.py +++ b/vllm/model_executor/models/phi4siglip.py @@ -14,7 +14,7 @@ import torch.nn as nn from transformers import BatchFeature, PretrainedConfig, Siglip2VisionConfig from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.inputs import MultiModalDataDict from vllm.logger import init_logger from vllm.multimodal import MULTIMODAL_REGISTRY @@ -126,15 +126,14 @@ class Phi4SiglipDummyInputsBuilder( self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) size = self.info.get_image_size_with_most_features() return { "image": self._get_dummy_images( width=size.width, height=size.height, - num_images=num_images, + num_images=mm_counts.get("image", 0), overrides=mm_options.get("image"), ), } diff --git a/vllm/model_executor/models/pixtral.py b/vllm/model_executor/models/pixtral.py index 425bba74fd..f5c167aec3 100644 --- a/vllm/model_executor/models/pixtral.py +++ b/vllm/model_executor/models/pixtral.py @@ -21,7 +21,7 @@ from transformers.models.pixtral.image_processing_pixtral import ( from transformers.models.pixtral.modeling_pixtral import apply_rotary_pos_emb from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.distributed import divide, get_tensor_model_parallel_world_size from vllm.inputs import MultiModalDataDict from vllm.model_executor.layers.activation import SiluAndMul, get_act_and_mul_fn @@ -208,20 +208,16 @@ class PixtralDummyInputsBuilder(BaseDummyInputsBuilder[PixtralProcessingInfo]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) - target_width, target_height = self.info.get_image_size_with_most_features() - image_overrides = mm_options.get("image") - return { "image": self._get_dummy_images( width=target_width, height=target_height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ) } @@ -229,7 +225,7 @@ class PixtralDummyInputsBuilder(BaseDummyInputsBuilder[PixtralProcessingInfo]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, mm_data: MultiModalDataDict | None = None, ) -> ProcessorInputs: tokenizer = self.info.get_tokenizer() diff --git a/vllm/model_executor/models/qwen2_5_omni_thinker.py b/vllm/model_executor/models/qwen2_5_omni_thinker.py index cecc8345e5..ce7cb058b7 100644 --- a/vllm/model_executor/models/qwen2_5_omni_thinker.py +++ b/vllm/model_executor/models/qwen2_5_omni_thinker.py @@ -44,7 +44,7 @@ from transformers.models.qwen2_5_omni.processing_qwen2_5_omni import ( from transformers.models.whisper import WhisperFeatureExtractor from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.forward_context import set_forward_context from vllm.inputs import ModalityData, MultiModalDataDict from vllm.logger import init_logger @@ -448,12 +448,8 @@ class Qwen2_5OmniThinkerDummyInputsBuilder( self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_audios = mm_counts.get("audio", 0) - num_images = mm_counts.get("image", 0) - num_videos = mm_counts.get("video", 0) - feature_extractor = self.info.get_feature_extractor() target_audio_length = ( @@ -469,28 +465,24 @@ class Qwen2_5OmniThinkerDummyInputsBuilder( seq_len, mm_counts ) - image_overrides = mm_options.get("image") - video_overrides = mm_options.get("video") - audio_overrides = mm_options.get("audio") - mm_data = { "audio": self._get_dummy_audios( length=target_audio_length, - num_audios=num_audios, - overrides=audio_overrides, + num_audios=mm_counts.get("audio", 0), + overrides=mm_options.get("audio"), ), "image": self._get_dummy_images( width=target_width, height=target_height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ), "video": self._get_dummy_videos( width=target_width, height=target_height, num_frames=target_num_frames, - num_videos=num_videos, - overrides=video_overrides, + num_videos=mm_counts.get("video", 0), + overrides=mm_options.get("video"), ), } diff --git a/vllm/model_executor/models/qwen2_audio.py b/vllm/model_executor/models/qwen2_audio.py index 7e7ffd5cfa..b643fe4227 100644 --- a/vllm/model_executor/models/qwen2_audio.py +++ b/vllm/model_executor/models/qwen2_audio.py @@ -37,7 +37,7 @@ from transformers.models.qwen2_audio import ( from transformers.models.whisper import WhisperFeatureExtractor from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.inputs import ModalityData, MultiModalDataDict from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.inputs import ( @@ -214,21 +214,18 @@ class Qwen2AudioDummyInputsBuilder(BaseDummyInputsBuilder[Qwen2AudioProcessingIn self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: feature_extractor = self.info.get_feature_extractor() sampling_rate = feature_extractor.sampling_rate audio_len = feature_extractor.chunk_length * sampling_rate - num_audios = mm_counts.get("audio", 0) - - audio_overrides = mm_options.get("audio") return { "audio": self._get_dummy_audios( length=audio_len, - num_audios=num_audios, - overrides=audio_overrides, + num_audios=mm_counts.get("audio", 0), + overrides=mm_options.get("audio"), ) } diff --git a/vllm/model_executor/models/qwen2_vl.py b/vllm/model_executor/models/qwen2_vl.py index d5e9d0a28b..4bbddb0167 100644 --- a/vllm/model_executor/models/qwen2_vl.py +++ b/vllm/model_executor/models/qwen2_vl.py @@ -43,7 +43,7 @@ from transformers.models.qwen2_vl.configuration_qwen2_vl import ( from transformers.models.qwen2_vl.image_processing_qwen2_vl import smart_resize from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.distributed import parallel_state, tensor_model_parallel_all_gather from vllm.distributed import utils as dist_utils from vllm.inputs import ModalityData, MultiModalDataDict @@ -1096,32 +1096,26 @@ class Qwen2VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen2VLProcessingInfo]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) - num_videos = mm_counts.get("video", 0) - target_width, target_height = self.info.get_image_size_with_most_features() target_num_frames = self.info.get_num_frames_with_most_features( seq_len, mm_counts ) - image_overrides = mm_options.get("image") - video_overrides = mm_options.get("video") - return { "image": self._get_dummy_images( width=target_width, height=target_height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ), "video": self._get_dummy_videos( width=target_width, height=target_height, num_frames=target_num_frames, - num_videos=num_videos, - overrides=video_overrides, + num_videos=mm_counts.get("video", 0), + overrides=mm_options.get("video"), ), } diff --git a/vllm/model_executor/models/qwen3_asr.py b/vllm/model_executor/models/qwen3_asr.py index 14925ea550..241e26fa4e 100644 --- a/vllm/model_executor/models/qwen3_asr.py +++ b/vllm/model_executor/models/qwen3_asr.py @@ -32,7 +32,7 @@ from transformers.feature_extraction_utils import BatchFeature from transformers.models.whisper import WhisperFeatureExtractor from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.config.speech_to_text import SpeechToTextParams from vllm.inputs import ModalityData, MultiModalDataDict, PromptType, TokensPrompt from vllm.logger import init_logger @@ -226,10 +226,8 @@ class Qwen3ASRDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3ASRProcessingInfo]) self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_audios = mm_counts.get("audio", 0) - feature_extractor = self.info.get_feature_extractor() target_audio_length = ( @@ -240,13 +238,11 @@ class Qwen3ASRDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3ASRProcessingInfo]) * feature_extractor.sampling_rate ) - audio_overrides = mm_options.get("audio") - return { "audio": self._get_dummy_audios( length=target_audio_length, - num_audios=num_audios, - overrides=audio_overrides, + num_audios=mm_counts.get("audio", 0), + overrides=mm_options.get("audio"), ), } diff --git a/vllm/model_executor/models/qwen3_vl.py b/vllm/model_executor/models/qwen3_vl.py index dc2baa1c5e..f7b553926c 100644 --- a/vllm/model_executor/models/qwen3_vl.py +++ b/vllm/model_executor/models/qwen3_vl.py @@ -54,8 +54,8 @@ from vllm.compilation.decorators import ( ) from vllm.config import VllmConfig from vllm.config.multimodal import ( - BaseDummyOptions, MultiModalConfig, + MultiModalDummyOptions, VideoDummyOptions, VideoPruningMethod, ) @@ -1119,11 +1119,8 @@ class Qwen3VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3VLProcessingInfo]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) - num_videos = mm_counts.get("video", 0) - image_overrides = mm_options.get("image") video_overrides = mm_options.get("video") target_image_width, target_image_height = ( @@ -1133,7 +1130,6 @@ class Qwen3VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3VLProcessingInfo]): # treat videos as special images target_num_frames = 2 if video_overrides: - assert isinstance(video_overrides, VideoDummyOptions) num_frames_override = video_overrides.num_frames if num_frames_override: if num_frames_override > target_num_frames: @@ -1204,7 +1200,6 @@ class Qwen3VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3VLProcessingInfo]): target_video_size.height, ) if video_overrides: - assert isinstance(video_overrides, VideoDummyOptions) width_override = video_overrides.width if width_override: if width_override > target_video_width: @@ -1230,14 +1225,14 @@ class Qwen3VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3VLProcessingInfo]): "image": self._get_dummy_images( width=target_image_width, height=target_image_height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ), "video": self._get_dummy_videos( width=target_video_width, height=target_video_height, num_frames=target_num_frames, - num_videos=num_videos, + num_videos=mm_counts.get("video", 0), ), } diff --git a/vllm/model_executor/models/rvl.py b/vllm/model_executor/models/rvl.py index bf9661bcaf..5e822edae1 100644 --- a/vllm/model_executor/models/rvl.py +++ b/vllm/model_executor/models/rvl.py @@ -8,7 +8,7 @@ import torch.nn as nn from transformers.activations import GELUActivation from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.inputs import MultiModalDataDict from vllm.multimodal import MULTIMODAL_REGISTRY @@ -40,20 +40,16 @@ class RVLDummyInputsBuilder(LlavaDummyInputsBuilder[RVLProcessingInfo]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) - target_width, target_height = self.info.get_image_size_with_most_features() - image_overrides = mm_options.get("image") - return { "image": self._get_dummy_images( width=target_width, height=target_height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ), } diff --git a/vllm/model_executor/models/siglip.py b/vllm/model_executor/models/siglip.py index 8a5b226b25..ab7146c598 100644 --- a/vllm/model_executor/models/siglip.py +++ b/vllm/model_executor/models/siglip.py @@ -16,7 +16,7 @@ from transformers import ( ) from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.distributed import divide, get_tensor_model_parallel_world_size from vllm.inputs import MultiModalDataDict, MultiModalInput from vllm.model_executor.layers.activation import get_act_fn @@ -158,20 +158,16 @@ class SiglipDummyInputsBuilder(BaseDummyInputsBuilder[SiglipProcessingInfo]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) - target_width, target_height = self.info.get_image_size_with_most_features() - image_overrides = mm_options.get("image") - return { "image": self._get_dummy_images( width=target_width, height=target_height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ) } diff --git a/vllm/model_executor/models/step3_vl.py b/vllm/model_executor/models/step3_vl.py index aed99c84cb..ac747a7ecb 100644 --- a/vllm/model_executor/models/step3_vl.py +++ b/vllm/model_executor/models/step3_vl.py @@ -11,7 +11,7 @@ import torch.nn.functional as F from transformers import BatchFeature from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.distributed import get_tensor_model_parallel_world_size from vllm.inputs import MultiModalDataDict from vllm.model_executor.layers.activation import get_act_fn @@ -139,19 +139,16 @@ class Step3VLDummyInputsBuilder(BaseDummyInputsBuilder[Step3VLProcessingInfo]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: target_width, target_height = self.info.get_image_size_with_most_features() - num_images = mm_counts.get("image", 0) - - image_overrides = mm_options.get("image") return { "image": self._get_dummy_images( width=target_width, height=target_height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ) } diff --git a/vllm/model_executor/models/terratorch.py b/vllm/model_executor/models/terratorch.py index ddd1ac81ce..ce04acc72d 100644 --- a/vllm/model_executor/models/terratorch.py +++ b/vllm/model_executor/models/terratorch.py @@ -33,7 +33,7 @@ from terratorch.vllm import ( from transformers import BatchFeature from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.inputs import ModalityData, MultiModalDataDict, MultiModalInput, mm_input from vllm.logger import init_logger from vllm.model_executor.layers.pooler import IdentityPooler @@ -153,7 +153,7 @@ class TerratorchInputBuilder(BaseDummyInputsBuilder[TerratorchProcessingInfo]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: # Dummy data is generated based on the 'input' section # defined in the HF configuration file diff --git a/vllm/model_executor/models/transformers/multimodal.py b/vllm/model_executor/models/transformers/multimodal.py index 164a4dbd83..cecb4f564a 100644 --- a/vllm/model_executor/models/transformers/multimodal.py +++ b/vllm/model_executor/models/transformers/multimodal.py @@ -27,7 +27,6 @@ from packaging.version import Version from transformers.utils.generic import ModelOutput from vllm.compilation.decorators import should_torch_compile_mm_encoder -from vllm.config.multimodal import AudioDummyOptions from vllm.config.utils import getattr_iter from vllm.inputs import ( MultiModalDataBuiltins, @@ -75,7 +74,7 @@ if TYPE_CHECKING: from transformers import BatchFeature, PreTrainedModel from vllm.config import VllmConfig - from vllm.config.multimodal import BaseDummyOptions + from vllm.config.multimodal import MultiModalDummyOptions logger = init_logger(__name__) @@ -195,7 +194,7 @@ class MultiModalDummyInputsBuilder(BaseDummyInputsBuilder[MultiModalProcessingIn self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, "BaseDummyOptions"], + mm_options: "MultiModalDummyOptions", ) -> MultiModalDataDict: data = MultiModalDataBuiltins() if self.info._is_audio_model() and (num_audios := mm_counts.get("audio", 0)): @@ -204,21 +203,16 @@ class MultiModalDummyInputsBuilder(BaseDummyInputsBuilder[MultiModalProcessingIn chunk_length = getattr(sub, "chunk_length", None) if sub else None if chunk_length is None: chunk_length = 30 - audio_len = int(chunk_length * sampling_rate) - audio_overrides = mm_options.get("audio") - assert audio_overrides is None or isinstance( - audio_overrides, AudioDummyOptions - ) data["audio"] = self._get_dummy_audios( - length=audio_len, + length=int(chunk_length * sampling_rate), num_audios=num_audios, - overrides=audio_overrides, + overrides=mm_options.get("audio"), ) if self.info._is_image_model() and (num_images := mm_counts.get("image", 0)): - target_width, target_height = self.info.get_image_size_with_most_features() + width, height = self.info.get_image_size_with_most_features() data["image"] = self._get_dummy_images( - width=target_width, - height=target_height, + width=width, + height=height, num_images=num_images, overrides=mm_options.get("image"), ) diff --git a/vllm/model_executor/models/ultravox.py b/vllm/model_executor/models/ultravox.py index 3a1d017f4c..bb92833f77 100644 --- a/vllm/model_executor/models/ultravox.py +++ b/vllm/model_executor/models/ultravox.py @@ -16,7 +16,7 @@ from transformers import BatchFeature, ProcessorMixin from transformers.models.whisper import WhisperFeatureExtractor from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.distributed import get_tensor_model_parallel_world_size from vllm.inputs import MultiModalDataDict from vllm.model_executor.layers.activation import MulAndSilu, get_act_fn @@ -169,7 +169,7 @@ class UltravoxDummyInputsBuilder(BaseDummyInputsBuilder[UltravoxProcessingInfo]) self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: feature_extractor = self.info.get_feature_extractor() @@ -177,15 +177,12 @@ class UltravoxDummyInputsBuilder(BaseDummyInputsBuilder[UltravoxProcessingInfo]) audio_len = ( feature_extractor.chunk_length * sampling_rate * _MAX_ENCODER_BATCH_SIZE ) - num_audios = mm_counts.get("audio", 0) - - audio_overrides = mm_options.get("audio") return { "audio": self._get_dummy_audios( length=audio_len, - num_audios=num_audios, - overrides=audio_overrides, + num_audios=mm_counts.get("audio", 0), + overrides=mm_options.get("audio"), ) } diff --git a/vllm/model_executor/models/voxtral.py b/vllm/model_executor/models/voxtral.py index ad54e8eb20..0b8d889bbc 100644 --- a/vllm/model_executor/models/voxtral.py +++ b/vllm/model_executor/models/voxtral.py @@ -19,7 +19,7 @@ from mistral_common.tokens.tokenizers.audio import Audio from transformers import WhisperConfig from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.config.speech_to_text import SpeechToTextParams from vllm.inputs import MultiModalDataDict, PromptType, TokensPrompt from vllm.logger import init_logger @@ -137,19 +137,15 @@ class VoxtralDummyInputsBuilder(BaseDummyInputsBuilder[VoxtralProcessingInfo]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_audios = mm_counts.get("audio", 0) - target_length = self.info.get_max_audio_array_len() - audio_overrides = mm_options.get("audio") - return { "audio": self._get_dummy_audios( length=target_length, - num_audios=num_audios, - overrides=audio_overrides, + num_audios=mm_counts.get("audio", 0), + overrides=mm_options.get("audio"), ) } @@ -157,7 +153,7 @@ class VoxtralDummyInputsBuilder(BaseDummyInputsBuilder[VoxtralProcessingInfo]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, mm_data: MultiModalDataDict | None = None, ) -> ProcessorInputs: tokenizer = self.info.get_tokenizer() diff --git a/vllm/model_executor/models/whisper.py b/vllm/model_executor/models/whisper.py index b50d04d04b..59268619b0 100644 --- a/vllm/model_executor/models/whisper.py +++ b/vllm/model_executor/models/whisper.py @@ -19,7 +19,7 @@ from transformers.models.whisper.modeling_whisper import sinusoids from vllm.compilation.decorators import support_torch_compile from vllm.config import CacheConfig, ModelConfig, SpeechToTextConfig, VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.config.speech_to_text import SpeechToTextParams from vllm.distributed import get_tensor_model_parallel_world_size from vllm.inputs import ( @@ -724,21 +724,18 @@ class WhisperDummyInputsBuilder(BaseDummyInputsBuilder[WhisperProcessingInfo]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: feature_extractor = self.info.get_feature_extractor() sampling_rate = feature_extractor.sampling_rate audio_len = feature_extractor.chunk_length * sampling_rate - num_audios = mm_counts.get("audio", 0) - - audio_overrides = mm_options.get("audio") return { "audio": self._get_dummy_audios( length=audio_len, - num_audios=num_audios, - overrides=audio_overrides, + num_audios=mm_counts.get("audio", 0), + overrides=mm_options.get("audio"), ) } diff --git a/vllm/models/deepseek_v4/common/mm_preprocess.py b/vllm/models/deepseek_v4/common/mm_preprocess.py index 919146c551..d28848bb8f 100644 --- a/vllm/models/deepseek_v4/common/mm_preprocess.py +++ b/vllm/models/deepseek_v4/common/mm_preprocess.py @@ -22,7 +22,7 @@ reference's out-of-vocab scheme. import math from collections.abc import Mapping, Sequence -from typing import Any, cast +from typing import Any import numpy as np import torch @@ -30,7 +30,7 @@ from PIL import Image, ImageOps from transformers import BatchFeature from typing_extensions import assert_never -from vllm.config.multimodal import BaseDummyOptions, ImageDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.inputs import MultiModalDataDict from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargsItems from vllm.multimodal.parse import ImageSize, MultiModalDataItems @@ -375,7 +375,7 @@ class DeepseekV4VLDummyInputsBuilder( self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: size = self.info.get_image_size_with_most_features() return { @@ -383,7 +383,7 @@ class DeepseekV4VLDummyInputsBuilder( width=size.width, height=size.height, num_images=mm_counts.get("image", 0), - overrides=cast(ImageDummyOptions | None, mm_options.get("image")), + overrides=mm_options.get("image"), ), } diff --git a/vllm/models/deepseek_v41/common/mm_preprocess.py b/vllm/models/deepseek_v41/common/mm_preprocess.py index 4f87b816c9..7710edb43f 100644 --- a/vllm/models/deepseek_v41/common/mm_preprocess.py +++ b/vllm/models/deepseek_v41/common/mm_preprocess.py @@ -19,14 +19,14 @@ boundaries freely). import math from collections.abc import Mapping, Sequence -from typing import Any, cast +from typing import Any import numpy as np import torch from PIL import Image, ImageOps from transformers import BatchFeature -from vllm.config.multimodal import BaseDummyOptions, ImageDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.inputs import MultiModalDataDict from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargsItems from vllm.multimodal.parse import ImageSize, MultiModalDataItems @@ -296,7 +296,7 @@ class DeepseekV4VLDummyInputsBuilder( self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: size = self.info.get_image_size_with_most_features() return { @@ -304,7 +304,7 @@ class DeepseekV4VLDummyInputsBuilder( width=size.width, height=size.height, num_images=mm_counts.get("image", 0), - overrides=cast(ImageDummyOptions | None, mm_options.get("image")), + overrides=mm_options.get("image"), ), } diff --git a/vllm/models/dots3_note/common/processor.py b/vllm/models/dots3_note/common/processor.py index addaf77c81..4589874ec0 100644 --- a/vllm/models/dots3_note/common/processor.py +++ b/vllm/models/dots3_note/common/processor.py @@ -13,10 +13,7 @@ from PIL import Image from transformers import BatchFeature from vllm.config.multimodal import ( - AudioDummyOptions, - BaseDummyOptions, - ImageDummyOptions, - VideoDummyOptions, + MultiModalDummyOptions, ) from vllm.inputs import MultiModalDataDict from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargsItems @@ -584,7 +581,7 @@ class Dots3NoteDummyInputsBuilder(BaseDummyInputsBuilder[Dots3NoteProcessingInfo self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: data: dict[str, Any] = {} num_images = mm_counts.get("image", 0) @@ -594,7 +591,7 @@ class Dots3NoteDummyInputsBuilder(BaseDummyInputsBuilder[Dots3NoteProcessingInfo width=width, height=height, num_images=num_images, - overrides=cast(ImageDummyOptions | None, mm_options.get("image")), + overrides=mm_options.get("image"), ) num_audios = mm_counts.get("audio", 0) if num_audios: @@ -611,7 +608,7 @@ class Dots3NoteDummyInputsBuilder(BaseDummyInputsBuilder[Dots3NoteProcessingInfo data["audio"] = self._get_dummy_audios( length=min(chunk_samples, max(1, seq_len) * stride), num_audios=num_audios, - overrides=cast(AudioDummyOptions | None, mm_options.get("audio")), + overrides=mm_options.get("audio"), ) num_videos = mm_counts.get("video", 0) if num_videos: @@ -625,7 +622,7 @@ class Dots3NoteDummyInputsBuilder(BaseDummyInputsBuilder[Dots3NoteProcessingInfo height=height, num_frames=num_frames, num_videos=num_videos, - overrides=cast(VideoDummyOptions | None, mm_options.get("video")), + overrides=mm_options.get("video"), ) return data diff --git a/vllm/models/inkling/common/mm_preprocess.py b/vllm/models/inkling/common/mm_preprocess.py index cc88c12529..a949e4ec74 100644 --- a/vllm/models/inkling/common/mm_preprocess.py +++ b/vllm/models/inkling/common/mm_preprocess.py @@ -13,9 +13,7 @@ import torch from transformers.feature_extraction_utils import BatchFeature from vllm.config.multimodal import ( - AudioDummyOptions, - BaseDummyOptions, - ImageDummyOptions, + MultiModalDummyOptions, ) from vllm.inputs import MultiModalDataDict from vllm.multimodal.inputs import ( @@ -145,7 +143,7 @@ class InklingDummyInputsBuilder(BaseDummyInputsBuilder[InklingProcessingInfo]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: config = self.info.get_hf_config() num_images = mm_counts.get("image", 0) @@ -156,12 +154,11 @@ class InklingDummyInputsBuilder(BaseDummyInputsBuilder[InklingProcessingInfo]): patch_size = getattr(config.vision_config, "patch_size", 40) # A square image ~4 patches wide so the dummy emits several patches. side = patch_size * 4 - image_overrides = mm_options.get("image") mm_data["image"] = self._get_dummy_images( width=side, height=side, num_images=num_images, - overrides=cast(ImageDummyOptions | None, image_overrides), + overrides=mm_options.get("image"), ) if num_audios: # Size the dummy at the maximum allowed audio so memory/encoder @@ -169,11 +166,10 @@ class InklingDummyInputsBuilder(BaseDummyInputsBuilder[InklingProcessingInfo]): params = self.info.get_hf_processor().audio_feature_extractor.params hop = int(round(params.audio_token_duration_s * params.sample_rate)) audio_len = MAX_AUDIO_TOKENS * hop - audio_overrides = mm_options.get("audio") mm_data["audio"] = self._get_dummy_audios( length=audio_len, num_audios=num_audios, - overrides=cast(AudioDummyOptions | None, audio_overrides), + overrides=mm_options.get("audio"), ) return mm_data diff --git a/vllm/models/kimi_k3/common/mm_preprocess.py b/vllm/models/kimi_k3/common/mm_preprocess.py index cfe603b47b..dfd90fe9a9 100644 --- a/vllm/models/kimi_k3/common/mm_preprocess.py +++ b/vllm/models/kimi_k3/common/mm_preprocess.py @@ -9,7 +9,7 @@ from typing import Any, cast import torch from transformers import BatchFeature -from vllm.config.multimodal import BaseDummyOptions, ImageDummyOptions +from vllm.config.multimodal import ImageDummyOptions, MultiModalDummyOptions from vllm.inputs import MultiModalDataDict from vllm.logger import init_logger from vllm.multimodal.inputs import ( @@ -204,7 +204,7 @@ class KimiK3DummyInputsBuilder(BaseDummyInputsBuilder[KimiK3ProcessingInfo]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions] | None = None, + mm_options: MultiModalDummyOptions | None = None, ) -> MultiModalDataDict: media_proc_cfg = self.info.image_processor.media_proc_cfg max_size = self.info.get_max_image_size( @@ -214,7 +214,6 @@ class KimiK3DummyInputsBuilder(BaseDummyInputsBuilder[KimiK3ProcessingInfo]): media_proc_cfg["patch_limit_on_one_side"], media_proc_cfg["fixed_output_tokens"], ) - num_images = mm_counts.get("image", 0) image_overrides = cast( ImageDummyOptions | None, mm_options.get("image") if mm_options else None, @@ -223,7 +222,7 @@ class KimiK3DummyInputsBuilder(BaseDummyInputsBuilder[KimiK3ProcessingInfo]): "image": self._get_dummy_images( width=max_size.width, height=max_size.height, - num_images=num_images, + num_images=mm_counts.get("image", 0), overrides=image_overrides, ) } diff --git a/vllm/models/minimax_m3/common/mm_preprocess.py b/vllm/models/minimax_m3/common/mm_preprocess.py index 4e8b791093..958e4f189b 100644 --- a/vllm/models/minimax_m3/common/mm_preprocess.py +++ b/vllm/models/minimax_m3/common/mm_preprocess.py @@ -11,9 +11,7 @@ from transformers import BatchFeature from transformers.video_utils import VideoMetadata from vllm.config.multimodal import ( - BaseDummyOptions, - ImageDummyOptions, - VideoDummyOptions, + MultiModalDummyOptions, ) from vllm.inputs import MultiModalDataDict from vllm.multimodal.inputs import ( @@ -283,7 +281,7 @@ class MiniMaxM3VLDummyInputsBuilder(BaseDummyInputsBuilder[MiniMaxM3VLProcessing self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: size = self.info.get_image_size_with_most_features() video_size = self.info.get_video_size_with_most_features() @@ -293,14 +291,14 @@ class MiniMaxM3VLDummyInputsBuilder(BaseDummyInputsBuilder[MiniMaxM3VLProcessing width=size.width, height=size.height, num_images=mm_counts.get("image", 0), - overrides=cast(ImageDummyOptions | None, mm_options.get("image")), + overrides=mm_options.get("image"), ), "video": self._get_dummy_videos( width=video_size.width, height=video_size.height, num_frames=num_frames, num_videos=mm_counts.get("video", 0), - overrides=cast(VideoDummyOptions | None, mm_options.get("video")), + overrides=mm_options.get("video"), ), } diff --git a/vllm/multimodal/processing/dummy_inputs.py b/vllm/multimodal/processing/dummy_inputs.py index 7ec90d030c..53e9f4f5b6 100644 --- a/vllm/multimodal/processing/dummy_inputs.py +++ b/vllm/multimodal/processing/dummy_inputs.py @@ -10,8 +10,8 @@ from PIL import Image from vllm.config.multimodal import ( AudioDummyOptions, - BaseDummyOptions, ImageDummyOptions, + MultiModalDummyOptions, VideoDummyOptions, ) from vllm.inputs import MultiModalDataDict @@ -45,7 +45,7 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: """Build the multimodal input which, after processing, results in the maximum possible number of placeholder tokens. @@ -65,7 +65,7 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> ProcessorInputs: """Build the input which, after processing, results in the maximum possible number of placeholder tokens. @@ -124,9 +124,8 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]): width: int, height: int, num_images: int, - overrides: BaseDummyOptions | None = None, + overrides: ImageDummyOptions | None = None, ) -> list[Image.Image]: - assert overrides is None or isinstance(overrides, ImageDummyOptions) if num_images == 0: return [] if overrides: