iHore/vLLM
8
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[Multimodal] Type dummy options per modality (#57576)

Browse Source 
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Harry Mellor
2026-09-18 17:29:11 +01:00
committed by GitHub
co-authored by Claude Opus 5
parent 56cac80b14
commit 8a5cf54387
93 changed files with 461 additions and 720 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/models/multimodal/processing/test_audioflamingo3.py
+2 -1
View File
@@ -25,6 +25,7 @@ import torch
from transformers import PretrainedConfig
from tests.models.registry import HF_EXAMPLE_MODELS
from vllm.config.multimodal import MultiModalDummyOptions
class MockAudioFlamingo3Config(PretrainedConfig):
@@ -127,7 +128,7 @@ def test_dummy_data_generation(mock_ctx):
builder = AudioFlamingo3DummyInputsBuilder(info)
mm_counts = {"audio": 2}
dummy_data = builder.get_dummy_mm_data(100, mm_counts, {})
dummy_data = builder.get_dummy_mm_data(100, mm_counts, MultiModalDummyOptions())
assert "audio" in dummy_data
assert len(dummy_data["audio"]) == 2
tests/models/multimodal/processing/test_common.py
+9 -6
View File
@@ -13,6 +13,7 @@ from vllm.config.multimodal import (
AudioDummyOptions,
BaseDummyOptions,
ImageDummyOptions,
MultiModalDummyOptions,
VideoDummyOptions,
)
from vllm.inputs import MultiModalDataDict, MultiModalInput
@@ -174,7 +175,7 @@ def get_token_prompt(
inputs = dummy_inputs.get_dummy_processor_inputs(
model_config.max_model_len,
mm_counts,
mm_options={},
mm_options=MultiModalDummyOptions(),
# Assume all Mistral models define this extra argument
mm_data=mm_data, # type: ignore[call-arg]
)
@@ -182,7 +183,7 @@ def get_token_prompt(
inputs = dummy_inputs.get_dummy_processor_inputs(
model_config.max_model_len,
mm_counts,
mm_options={},
mm_options=MultiModalDummyOptions(),
)
if not isinstance(inputs.prompt, list):
@@ -278,10 +279,12 @@ def _test_processing_correctness(
return BaseDummyOptions(count=count)
# Assign normalized DummyOptions to the model config
model_config.get_multimodal_config().limit_per_prompt = {
modality: _to_dummy_options(modality, count)
for modality, count in limit_mm_per_prompt_ints.items()
}
model_config.get_multimodal_config().limit_per_prompt = MultiModalDummyOptions(
{
modality: _to_dummy_options(modality, count)
for modality, count in limit_mm_per_prompt_ints.items()
}
)
baseline_processor = factories.build_processor(ctx, cache=None)
cached_processor = factories.build_processor(ctx, cache=cache)
tests/models/multimodal/processing/test_molmo2.py
+4 -2
View File
@@ -6,7 +6,7 @@ from types import SimpleNamespace
import pytest
import torch
from vllm.config.multimodal import VideoDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions, VideoDummyOptions
from vllm.model_executor.models.molmo2 import (
Molmo2DummyInputsBuilder,
build_flat_image_bool_length,
@@ -83,7 +83,9 @@ def test_dummy_video_num_frames_override_honors_min_of_two(
data = builder.get_dummy_mm_data(
seq_len=128,
mm_counts={"image": 0, "video": 1},
mm_options={"video": VideoDummyOptions(num_frames=num_frames_override)},
mm_options=MultiModalDummyOptions(
video=VideoDummyOptions(num_frames=num_frames_override)
),
)
video, _metadata = data["video"][0]
tests/models/multimodal/processing/test_qwen3_vl.py
+7 -2
View File
@@ -12,6 +12,7 @@ import numpy as np
import pytest
from vllm.config import ModelConfig
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.multimodal import MULTIMODAL_REGISTRY
from ...registry import HF_EXAMPLE_MODELS
@@ -339,7 +340,9 @@ def test_dummy_video_spreads_budget_when_frame_cap_enabled(model_id: str) -> Non
limit_mm_per_prompt={"image": 0, "video": 1},
)
capped = MULTIMODAL_REGISTRY.create_processor(capped_ctx.model_config)
capped_dummy = capped.dummy_inputs.get_dummy_mm_data(1024, {"video": 1}, {})
capped_dummy = capped.dummy_inputs.get_dummy_mm_data(
1024, {"video": 1}, MultiModalDummyOptions()
)
capped_frames = capped_dummy["video"][0][0].shape[0]
assert capped_frames == 16, (
f"Expected the dummy to spread the budget over 16 frames, got {capped_frames}"
@@ -351,7 +354,9 @@ def test_dummy_video_spreads_budget_when_frame_cap_enabled(model_id: str) -> Non
limit_mm_per_prompt={"image": 0, "video": 1},
)
uncapped = MULTIMODAL_REGISTRY.create_processor(uncapped_ctx.model_config)
uncapped_dummy = uncapped.dummy_inputs.get_dummy_mm_data(1024, {"video": 1}, {})
uncapped_dummy = uncapped.dummy_inputs.get_dummy_mm_data(
1024, {"video": 1}, MultiModalDummyOptions()
)
uncapped_frames = uncapped_dummy["video"][0][0].shape[0]
assert uncapped_frames == 2, (
f"Expected the uncapped dummy to keep 2 frames, got {uncapped_frames}"
tests/models/multimodal/processing/test_tensor_schema.py
+8 -5
View File
@@ -18,6 +18,7 @@ from vllm.config.multimodal import (
AudioDummyOptions,
BaseDummyOptions,
ImageDummyOptions,
MultiModalDummyOptions,
VideoDummyOptions,
)
from vllm.distributed import (
@@ -98,7 +99,7 @@ def create_batched_mm_kwargs(
processor_inputs = dummy_inputs.get_dummy_processor_inputs(
seq_len=model_config.max_model_len,
mm_counts=mm_counts,
mm_options={},
mm_options=MultiModalDummyOptions(),
)
mm_items = processor_inputs.mm_data_items
resized_mm_data = {
@@ -248,10 +249,12 @@ def test_model_tensor_schema(model_id: str):
return AudioDummyOptions(count=count)
return BaseDummyOptions(count=count)
model_config.get_multimodal_config().limit_per_prompt = {
modality: _to_dummy_options(modality, count)
for modality, count in limit_mm_per_prompt.items()
}
model_config.get_multimodal_config().limit_per_prompt = MultiModalDummyOptions(
{
modality: _to_dummy_options(modality, count)
for modality, count in limit_mm_per_prompt.items()
}
)
processor = factories.build_processor(ctx, cache=None)
with initialize_dummy_model(model_cls, model_config) as model:
vllm/config/multimodal.py
+57 -22
View File
@@ -3,11 +3,18 @@
from collections.abc import Mapping
from pathlib import Path
from typing import Any, Literal, TypeAlias, TypedDict, final
from typing import Any, Literal, TypeAlias, TypeVar, final, overload
import torch
from pydantic import ConfigDict, Field, field_validator, model_validator
from pydantic import (
ConfigDict,
Field,
GetCoreSchemaHandler,
field_validator,
model_validator,
)
from pydantic.dataclasses import dataclass
from pydantic_core import core_schema
import vllm.envs as envs
from vllm.config.ec_transfer import ECTransferConfig
@@ -18,6 +25,8 @@ from vllm.v1.attention.backends.registry import AttentionBackendEnum
logger = init_logger(__name__)
_T = TypeVar("_T")
@dataclass
class BaseDummyOptions:
@@ -51,17 +60,50 @@ class AudioDummyOptions(BaseDummyOptions):
@final
class MultiModalDummyOptionsBuiltins(TypedDict, total=False):
"""Type annotations for modality types predefined by vLLM."""
class MultiModalDummyOptions(dict[str, BaseDummyOptions]):
"""Dummy data options for each modality.
image: ImageDummyOptions
"""Options for dummy images."""
Lookups of the modalities predefined by vLLM return their own options
class, while any other modality returns
[`BaseDummyOptions`][vllm.config.multimodal.BaseDummyOptions].
"""
video: VideoDummyOptions
"""Options for dummy videos."""
@overload # type: ignore[override]
def get(self, key: Literal["image"], /) -> ImageDummyOptions | None: ...
audio: AudioDummyOptions
"""Options for dummy audios."""
@overload
def get(self, key: Literal["image"], default: _T, /) -> ImageDummyOptions | _T: ...
@overload
def get(self, key: Literal["video"], /) -> VideoDummyOptions | None: ...
@overload
def get(self, key: Literal["video"], default: _T, /) -> VideoDummyOptions | _T: ...
@overload
def get(self, key: Literal["audio"], /) -> AudioDummyOptions | None: ...
@overload
def get(self, key: Literal["audio"], default: _T, /) -> AudioDummyOptions | _T: ...
@overload
def get(self, key: str, /) -> BaseDummyOptions | None: ...
@overload
def get(self, key: str, default: _T, /) -> BaseDummyOptions | _T: ...
def get(self, key: str, default: object = None, /) -> object:
return super().get(key, default)
@classmethod
def __get_pydantic_core_schema__(
cls,
source_type: Any,
handler: GetCoreSchemaHandler,
) -> core_schema.CoreSchema:
return core_schema.no_info_after_validator_function(
cls, handler.generate_schema(dict[str, BaseDummyOptions])
)
MMEncoderTPMode = Literal["weights", "data"]
@@ -75,15 +117,6 @@ MMProcessorDevice: TypeAlias = str
`"xpu"` on XPU). Validated against that set by the CLI."""
MMDummyOptions: TypeAlias = dict[str, BaseDummyOptions]
"""
A dictionary containing an entry for each modality type of dummy data.
The built-in modalities are defined by
[`MultiModalDummyOptionsBuiltins`][vllm.config.multimodal.MultiModalDummyOptionsBuiltins].
"""
@config
class MultiModalConfig:
"""Controls the behavior of multimodal models."""
@@ -91,7 +124,9 @@ class MultiModalConfig:
language_model_only: bool = False
"""If True, disables all multimodal inputs by setting all modality limits to 0.
Equivalent to setting `--limit-mm-per-prompt` to 0 for every modality."""
limit_per_prompt: MMDummyOptions = Field(default_factory=dict)
limit_per_prompt: MultiModalDummyOptions = Field(
default_factory=MultiModalDummyOptions
)
"""The maximum number of input items and options allowed per
prompt for each modality.
@@ -257,8 +292,8 @@ class MultiModalConfig:
def _validate_limit_per_prompt(
cls,
value: dict[str, int | dict[str, int]],
) -> MMDummyOptions:
out: MMDummyOptions = {}
) -> MultiModalDummyOptions:
out = MultiModalDummyOptions()
for k, v in value.items():
# Handle legacy format where only count is specified
vllm/engine/arg_utils.py
+15
View File
@@ -22,6 +22,7 @@ from typing import (
cast,
get_args,
get_origin,
is_typeddict,
)
import huggingface_hub
@@ -183,6 +184,18 @@ def is_type(type_hint: TypeHint, type: TypeHintT) -> TypeIs[TypeHintT]:
return type_hint is type or get_origin(type_hint) is type
def is_dict_subclass(type_hint: TypeHint) -> bool:
"""Check if the type hint is a subclass of `dict`.
`TypedDict`s are excluded because they do not support class checks.
"""
return (
isinstance(type_hint, type)
and not is_typeddict(type_hint)
and issubclass(type_hint, dict)
)
def contains_type(type_hints: set[TypeHint], type: TypeHintT) -> bool:
"""Check if the type hints contain a specific type."""
return any(is_type(type_hint, type) for type_hint in type_hints)
@@ -305,6 +318,8 @@ def _compute_kwargs(cls: ConfigType) -> dict[str, dict[str, Any]]:
for field in fields(cls):
# Get the set of possible types for the field
type_hints: set[TypeHint] = get_type_hints(field.type)
# Subclasses of dict (e.g. MultiModalDummyOptions) are CLI dicts
type_hints = {dict if is_dict_subclass(th) else th for th in type_hints}
# If the field is a dataclass, we can use the model_validate_json
generator = (th for th in type_hints if is_dataclass(th))
vllm/model_executor/models/aria.py
+4 -8
View File
@@ -10,7 +10,7 @@ from transformers.models.aria.modeling_aria import AriaCrossAttention
from transformers.models.aria.processing_aria import AriaProcessor
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions, ImageDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.activation import get_act_fn
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
@@ -315,22 +315,18 @@ class AriaDummyInputsBuilder(BaseDummyInputsBuilder[AriaProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
vision_config = self.info.get_vision_config()
max_image_size = vision_config.image_size
num_images = mm_counts.get("image", 0)
image_overrides = mm_options.get("image")
assert image_overrides is None or isinstance(image_overrides, ImageDummyOptions)
return {
"image": self._get_dummy_images(
width=max_image_size,
height=max_image_size,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
)
}
vllm/model_executor/models/audioflamingo3.py
+4 -7
View File
@@ -30,7 +30,7 @@ from transformers.models.audioflamingo3 import (
from transformers.models.qwen2_audio import Qwen2AudioEncoder
from vllm.config import VllmConfig
from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import ModalityData, MultiModalDataDict
from vllm.model_executor.layers.activation import get_act_fn
from vllm.model_executor.models.module_mapping import MultiModelKeys
@@ -216,21 +216,18 @@ class AudioFlamingo3DummyInputsBuilder(
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
hf_processor = self.info.get_hf_processor()
feature_extractor = self.info.get_feature_extractor()
sampling_rate = feature_extractor.sampling_rate
audio_len = int(hf_processor.max_audio_len * sampling_rate)
num_audios = mm_counts.get("audio", 0)
audio_overrides = mm_options.get("audio")
assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions)
return {
"audio": self._get_dummy_audios(
length=audio_len,
num_audios=num_audios,
overrides=audio_overrides,
num_audios=mm_counts.get("audio", 0),
overrides=mm_options.get("audio"),
)
}
vllm/model_executor/models/bagel.py
+4 -7
View File
@@ -14,7 +14,7 @@ import torch
import torch.nn as nn
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions, ImageDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.logger import init_logger
from vllm.model_executor.layers.activation import get_act_fn
@@ -248,23 +248,20 @@ class BagelDummyInputsBuilder(BaseDummyInputsBuilder[BagelProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
hf_config = self.info.get_hf_config()
vit_config = hf_config.vit_config
# Use the configured image size
image_size = vit_config.image_size
image_overrides = mm_options.get("image")
assert image_overrides is None or isinstance(image_overrides, ImageDummyOptions)
return {
"image": self._get_dummy_images(
width=image_size,
height=image_size,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
),
}
vllm/model_executor/models/bailing_moe_v3_vl.py
+2 -2
View File
@@ -13,7 +13,7 @@ import torch.nn as nn
from transformers import AutoProcessor, BatchFeature
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
@@ -103,7 +103,7 @@ class BailingMoeV3VLDummyInputsBuilder(
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
target_width, target_height = self.info.get_image_size_with_most_features()
return {
vllm/model_executor/models/bee.py
+4 -9
View File
@@ -8,7 +8,7 @@ import torch.nn as nn
from transformers.activations import GELUActivation
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions, ImageDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.multimodal import MULTIMODAL_REGISTRY
@@ -90,21 +90,16 @@ class BeeDummyInputsBuilder(LlavaDummyInputsBuilder[BeeProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
target_width, target_height = self.info.get_image_size_with_most_features()
image_overrides = mm_options.get("image")
assert image_overrides is None or isinstance(image_overrides, ImageDummyOptions)
return {
"image": self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
),
}
vllm/model_executor/models/blip2.py
+4 -8
View File
@@ -15,7 +15,7 @@ from transformers import (
)
from vllm.config import CacheConfig, VllmConfig
from vllm.config.multimodal import BaseDummyOptions, ImageDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.activation import get_act_fn
from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -444,23 +444,19 @@ class Blip2DummyInputsBuilder(BaseDummyInputsBuilder[Blip2ProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
hf_config = self.info.get_hf_config()
vision_config = hf_config.vision_config
max_image_size = vision_config.image_size
num_images = mm_counts.get("image", 0)
image_overrides = mm_options.get("image")
assert image_overrides is None or isinstance(image_overrides, ImageDummyOptions)
return {
"image": self._get_dummy_images(
width=max_image_size,
height=max_image_size,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
)
}
vllm/model_executor/models/clip.py
+4 -8
View File
@@ -15,7 +15,7 @@ from transformers import (
)
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.distributed import divide, get_tensor_model_parallel_world_size
from vllm.inputs import MultiModalDataDict, MultiModalInput
from vllm.model_executor.layers.activation import get_act_fn
@@ -205,20 +205,16 @@ class CLIPDummyInputsBuilder(BaseDummyInputsBuilder[CLIPProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
target_width, target_height = self.info.get_image_size_with_most_features()
image_overrides = mm_options.get("image")
return {
"image": self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
)
}
vllm/model_executor/models/cohere2_vision.py
+4 -7
View File
@@ -18,7 +18,7 @@ from transformers.models.cohere2_vision.processing_cohere2_vision import (
)
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.activation import MulAndSilu
from vllm.model_executor.layers.linear import (
@@ -202,19 +202,16 @@ class Cohere2VisionDummyInputsBuilder(
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
image_size = self.info.get_image_size_with_most_features()
image_overrides = mm_options.get("image")
return {
"image": self._get_dummy_images(
width=image_size.width,
height=image_size.height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
)
}
vllm/model_executor/models/cohere_asr.py
+2 -2
View File
@@ -12,7 +12,7 @@ from transformers import PretrainedConfig
from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, ModelConfig, SpeechToTextConfig, VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.config.speech_to_text import SpeechToTextParams
from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.inputs import MultiModalDataDict, PromptType, TokensPrompt
@@ -1944,7 +1944,7 @@ class CohereASRDummyInputsBuilder(BaseDummyInputsBuilder[CohereASRProcessingInfo
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
mm_processor_kwargs=None,
) -> MultiModalDataDict:
feature_extractor = self.info.get_feature_extractor()
vllm/model_executor/models/cohere_compass.py
+4 -7
View File
@@ -51,7 +51,7 @@ from transformers.models.cohere_compass.image_processing_cohere_compass import (
from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.distributed import (
get_pp_group,
get_tensor_model_parallel_world_size,
@@ -1238,11 +1238,8 @@ class CohereCompassDummyInputsBuilder(
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
image_overrides = mm_options.get("image")
target_image_width, target_image_height = (
self.info.get_image_size_with_most_features()
)
@@ -1251,8 +1248,8 @@ class CohereCompassDummyInputsBuilder(
"image": self._get_dummy_images(
width=target_image_width,
height=target_image_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
),
}
vllm/model_executor/models/colmodernvbert.py
+4 -6
View File
@@ -15,7 +15,7 @@ from torch import nn
from transformers import BatchFeature
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.pooler.tokwise import pooler_for_token_embed
from vllm.multimodal import MULTIMODAL_REGISTRY
@@ -135,18 +135,16 @@ class ColModernVBertDummyInputsBuilder(
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
target_width, target_height = self.info.get_image_size_with_most_features()
image_overrides = mm_options.get("image")
return {
"image": self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
)
}
vllm/model_executor/models/deepseek_ocr.py
+3 -5
View File
@@ -11,7 +11,7 @@ import torch.nn as nn
from transformers import BatchFeature, CLIPVisionConfig
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.models.interfaces import (
MultiModalEmbeddings,
@@ -268,17 +268,15 @@ class DeepseekOCRDummyInputsBuilder(BaseDummyInputsBuilder[DeepseekOCRProcessing
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
max_image_size = self.info.get_image_size_with_most_features()
return {
"image": self._get_dummy_images(
width=max_image_size.width,
height=max_image_size.height,
num_images=num_images,
num_images=mm_counts.get("image", 0),
)
}
vllm/model_executor/models/deepseek_ocr2.py
+3 -5
View File
@@ -11,7 +11,7 @@ import torch.nn as nn
from transformers import BatchFeature
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.models.interfaces import (
MultiModalEmbeddings,
@@ -140,17 +140,15 @@ class DeepseekOCR2DummyInputsBuilder(
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
max_image_size = self.info.get_image_size_with_most_features()
return {
"image": self._get_dummy_images(
width=max_image_size.width,
height=max_image_size.height,
num_images=num_images,
num_images=mm_counts.get("image", 0),
)
}
vllm/model_executor/models/deepseek_vl2.py
+4 -8
View File
@@ -15,7 +15,7 @@ from einops import rearrange, repeat
from transformers import BatchFeature
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -212,20 +212,16 @@ class DeepseekVL2DummyInputsBuilder(BaseDummyInputsBuilder[DeepseekVL2Processing
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
max_image_size = self.info.get_image_size_with_most_features()
image_overrides = mm_options.get("image")
return {
"image": self._get_dummy_images(
width=max_image_size.width,
height=max_image_size.height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
)
}
vllm/model_executor/models/dots_ocr.py
+4 -8
View File
@@ -9,7 +9,7 @@ from torch.nn import LayerNorm
from transformers.models.qwen2_vl import Qwen2VLProcessor
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.distributed import utils as dist_utils
from vllm.distributed.parallel_state import (
get_tensor_model_parallel_rank,
@@ -104,20 +104,16 @@ class DotsOCRDummyInputsBuilder(Qwen2VLDummyInputsBuilder):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
target_width, target_height = self.info.get_image_size_with_most_features()
image_overrides = mm_options.get("image")
return {
"image": self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
),
}
vllm/model_executor/models/ernie45_vl.py
+6 -13
View File
@@ -36,7 +36,7 @@ from einops import rearrange
from transformers import BaseImageProcessor, BatchFeature
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions, VideoDummyOptions
from vllm.distributed import parallel_state
from vllm.distributed import utils as dist_utils
from vllm.inputs import MultiModalDataDict
@@ -1276,33 +1276,26 @@ class Ernie4_5_VLDummyInputsBuilder(BaseDummyInputsBuilder[Ernie4_5_VLProcessing
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
num_videos = mm_counts.get("video", 0)
target_width, target_height = self.info.get_image_size_with_most_features()
target_num_frames = self.info.get_num_frames_with_most_features(
seq_len, mm_counts
)
image_overrides = mm_options.get("image")
video_overrides = mm_options.get("video")
assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions)
return {
"image": self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
),
"video": self._get_dummy_videos(
width=target_width,
height=target_height,
num_frames=target_num_frames,
num_videos=num_videos,
overrides=video_overrides,
num_videos=mm_counts.get("video", 0),
overrides=mm_options.get("video"),
),
}
vllm/model_executor/models/fireredasr2.py
+5 -7
View File
@@ -12,7 +12,7 @@ from transformers import (
)
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.config.speech_to_text import SpeechToTextParams
from vllm.inputs import MultiModalDataDict, PromptType
from vllm.logger import init_logger
@@ -207,20 +207,18 @@ class FireRedASR2DummyInputsBuilder(BaseDummyInputsBuilder[FireRedASR2Processing
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
feature_extractor = self.info.get_feature_extractor()
sampling_rate = feature_extractor.sampling_rate
audio_len = feature_extractor.chunk_length * sampling_rate
num_audios = mm_counts.get("audio", 0)
audio_overrides = mm_options.get("audio")
assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions)
ret = {
"audio": self._get_dummy_audios(
length=audio_len, num_audios=num_audios, overrides=audio_overrides
length=audio_len,
num_audios=mm_counts.get("audio", 0),
overrides=mm_options.get("audio"),
)
}
return ret
vllm/model_executor/models/funasr.py
+4 -8
View File
@@ -14,7 +14,7 @@ from transformers import (
)
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.config.speech_to_text import SpeechToTextParams
from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.inputs import MultiModalDataDict, PromptType
@@ -734,22 +734,18 @@ class FunASRDummyInputsBuilder(BaseDummyInputsBuilder[FunASRProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
feature_extractor = self.info.get_feature_extractor()
sampling_rate = feature_extractor.sampling_rate
audio_len = feature_extractor.chunk_length * sampling_rate
num_audios = mm_counts.get("audio", 0)
audio_overrides = mm_options.get("audio")
assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions)
return {
"audio": self._get_dummy_audios(
length=audio_len,
num_audios=num_audios,
overrides=audio_overrides,
num_audios=mm_counts.get("audio", 0),
overrides=mm_options.get("audio"),
),
}
vllm/model_executor/models/funaudiochat.py
+4 -7
View File
@@ -26,7 +26,7 @@ from transformers.feature_extraction_utils import BatchFeature
from transformers.modeling_outputs import BaseModelOutput
from vllm.config import VllmConfig
from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.attention.mm_encoder_attention import MMEncoderAttention
from vllm.model_executor.layers.linear import QKVParallelLinear, RowParallelLinear
@@ -580,7 +580,7 @@ class FunAudioChatDummyInputsBuilder(
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
feature_extractor = self.info.get_feature_extractor()
sampling_rate = int(feature_extractor.sampling_rate)
@@ -597,15 +597,12 @@ class FunAudioChatDummyInputsBuilder(
1,
(target_num_frames * sampling_rate + token_fps - 1) // token_fps,
)
num_audios = int(mm_counts.get("audio", 0))
audio_overrides = mm_options.get("audio")
assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions)
return {
"audio": self._get_dummy_audios(
length=audio_len,
num_audios=num_audios,
overrides=audio_overrides,
num_audios=int(mm_counts.get("audio", 0)),
overrides=mm_options.get("audio"),
)
}
vllm/model_executor/models/gemma3_mm.py
+4 -8
View File
@@ -11,7 +11,7 @@ from transformers.models.gemma3.image_processing_gemma3 import Gemma3ImageProces
from transformers.models.gemma3.processing_gemma3 import Gemma3ProcessorKwargs
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.logger import init_logger
from vllm.model_executor.layers.layernorm import GemmaRMSNorm
@@ -244,20 +244,16 @@ class Gemma3DummyInputsBuilder(BaseDummyInputsBuilder[Gemma3ProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
target_width, target_height = self.info.get_image_size_with_most_features()
image_overrides = mm_options.get("image")
return {
"image": self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
)
}
vllm/model_executor/models/gemma3n_mm.py
+6 -12
View File
@@ -18,7 +18,7 @@ from transformers.models.gemma3n import (
from transformers.models.siglip import SiglipImageProcessorFast
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.config.speech_to_text import SpeechToTextParams
from vllm.inputs import MultiModalDataDict, PromptType, TextPrompt
from vllm.logger import init_logger
@@ -232,10 +232,8 @@ class Gemma3nDummyInputsBuilder(BaseDummyInputsBuilder[Gemma3nProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
num_audios = mm_counts.get("audio", 0)
processor = self.info.get_hf_processor()
audio_feature_extractor: Gemma3nAudioFeatureExtractor = (
processor.feature_extractor
@@ -245,21 +243,17 @@ class Gemma3nDummyInputsBuilder(BaseDummyInputsBuilder[Gemma3nProcessingInfo]):
img_width = image_processor.size.get("width", 224)
img_height = image_processor.size.get("height", 224)
image_overrides = mm_options.get("image")
audio_overrides = mm_options.get("audio")
assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions)
return {
"image": self._get_dummy_images(
width=img_width,
height=img_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
),
"audio": self._get_dummy_audios(
length=audio_len,
num_audios=num_audios,
overrides=audio_overrides,
num_audios=mm_counts.get("audio", 0),
overrides=mm_options.get("audio"),
),
}
vllm/model_executor/models/gemma4_mm.py
+7 -21
View File
@@ -36,11 +36,7 @@ from transformers.models.gemma4.configuration_gemma4 import (
from vllm.config import VllmConfig
from vllm.config.model import get_served_model_name
from vllm.config.multimodal import (
AudioDummyOptions,
BaseDummyOptions,
VideoDummyOptions,
)
from vllm.config.multimodal import MultiModalDummyOptions, VideoDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.logger import init_logger
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -273,10 +269,7 @@ class Gemma4ProcessingInfo(BaseProcessingInfo):
num_frames = _VIDEO_MAX_FRAMES
mm_config = self.ctx.model_config.get_multimodal_config()
video_opts = mm_config.limit_per_prompt.get("video")
if (
isinstance(video_opts, VideoDummyOptions)
and video_opts.num_frames is not None
):
if video_opts is not None and video_opts.num_frames is not None:
num_frames = min(num_frames, video_opts.num_frames)
tokens["video"] = num_frames * (_VIDEO_MAX_SOFT_TOKENS + 2 + 6)
return tokens
@@ -502,9 +495,8 @@ class Gemma4DummyInputsBuilder(BaseDummyInputsBuilder[Gemma4ProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions] | None = None,
mm_options: MultiModalDummyOptions | None = None,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
num_audios = mm_counts.get("audio", 0)
num_videos = mm_counts.get("video", 0)
processor = self.info.get_hf_processor()
@@ -517,18 +509,12 @@ class Gemma4DummyInputsBuilder(BaseDummyInputsBuilder[Gemma4ProcessingInfo]):
img_width = size.get("width", 224)
img_height = size.get("height", 224)
image_overrides = mm_options.get("image") if mm_options else None
audio_overrides = mm_options.get("audio") if mm_options else None
video_overrides = mm_options.get("video") if mm_options else None
assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions)
assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions)
data: dict[str, Any] = {
"image": self._get_dummy_images(
width=img_width,
height=img_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image") if mm_options else None,
),
}
@@ -537,7 +523,7 @@ class Gemma4DummyInputsBuilder(BaseDummyInputsBuilder[Gemma4ProcessingInfo]):
data["audio"] = self._get_dummy_audios(
length=audio_len,
num_audios=num_audios,
overrides=audio_overrides,
overrides=mm_options.get("audio") if mm_options else None,
)
if num_videos > 0:
@@ -546,7 +532,7 @@ class Gemma4DummyInputsBuilder(BaseDummyInputsBuilder[Gemma4ProcessingInfo]):
height=img_height,
num_frames=_VIDEO_MAX_FRAMES,
num_videos=num_videos,
overrides=video_overrides,
overrides=mm_options.get("video") if mm_options else None,
)
return data
vllm/model_executor/models/gemma4_unified.py
+1 -5
View File
@@ -28,7 +28,6 @@ from transformers.models.gemma4_unified.processing_gemma4_unified import (
)
from vllm.config import VllmConfig
from vllm.config.multimodal import VideoDummyOptions
from vllm.model_executor.layers.linear import ColumnParallelLinear
from vllm.model_executor.models.gemma3n_mm import batch_audio_features
from vllm.model_executor.models.gemma4 import Gemma4ForCausalLM
@@ -177,10 +176,7 @@ class Gemma4UnifiedProcessingInfo(Gemma4ProcessingInfo):
num_frames = _VIDEO_MAX_FRAMES
mm_config = self.ctx.model_config.get_multimodal_config()
video_opts = mm_config.limit_per_prompt.get("video")
if (
isinstance(video_opts, VideoDummyOptions)
and video_opts.num_frames is not None
):
if video_opts is not None and video_opts.num_frames is not None:
num_frames = min(num_frames, video_opts.num_frames)
tokens["video"] = num_frames * (_VIDEO_MAX_SOFT_TOKENS + 2 + 6)
return tokens
vllm/model_executor/models/glm4_1v.py
+6 -13
View File
@@ -52,7 +52,7 @@ from transformers.video_processing_utils import BaseVideoProcessor
from transformers.video_utils import VideoMetadata
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions, VideoDummyOptions
from vllm.distributed import get_tensor_model_parallel_world_size, parallel_state
from vllm.distributed import utils as dist_utils
from vllm.inputs import MultiModalDataDict
@@ -1455,33 +1455,26 @@ class Glm4vDummyInputsBuilder(BaseDummyInputsBuilder[Glm4vProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
num_videos = mm_counts.get("video", 0)
target_width, target_height = self.info.get_image_size_with_most_features()
target_num_frames = self.info.get_num_frames_with_most_features(
seq_len, mm_counts
)
image_overrides = mm_options.get("image")
video_overrides = mm_options.get("video")
assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions)
return {
"image": self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
),
"video": self._get_dummy_videos(
width=target_width,
height=target_height,
num_frames=target_num_frames,
num_videos=num_videos,
overrides=video_overrides,
num_videos=mm_counts.get("video", 0),
overrides=mm_options.get("video"),
),
}
vllm/model_executor/models/glm4v.py
+4 -7
View File
@@ -16,7 +16,7 @@ from torch.nn import LayerNorm
from transformers import BatchFeature
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.activation import SiluAndMul, get_act_fn
@@ -445,22 +445,19 @@ class GLM4VDummyInputsBuilder(BaseDummyInputsBuilder[GLM4VProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
hf_config = self.info.get_hf_config()
vision_config = hf_config.vision_config
target_width = target_height = vision_config["image_size"]
num_images = mm_counts.get("image", 0)
image_overrides = mm_options.get("image")
return {
"image": self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
)
}
vllm/model_executor/models/glmasr.py
+4 -7
View File
@@ -11,7 +11,7 @@ from transformers.models.glmasr import GlmAsrConfig, GlmAsrProcessor
from transformers.models.whisper import WhisperFeatureExtractor
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.config.speech_to_text import SpeechToTextParams
from vllm.distributed.parallel_state import get_tensor_model_parallel_world_size
from vllm.inputs import ModalityData, MultiModalDataDict, PromptType, TokensPrompt
@@ -694,13 +694,10 @@ class GlmAsrDummyInputsBuilder(BaseDummyInputsBuilder[GlmAsrProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
feature_extractor = self.info.get_feature_extractor()
sampling_rate = feature_extractor.sampling_rate
num_audios = mm_counts.get("audio", 0)
audio_overrides = mm_options.get("audio")
assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions)
max_audio_len = getattr(
self.info.get_hf_processor(), "max_audio_len", DEFAULT_MAX_AUDIO_LEN_S
@@ -710,8 +707,8 @@ class GlmAsrDummyInputsBuilder(BaseDummyInputsBuilder[GlmAsrProcessingInfo]):
return {
"audio": self._get_dummy_audios(
length=audio_len,
num_audios=num_audios,
overrides=audio_overrides,
num_audios=mm_counts.get("audio", 0),
overrides=mm_options.get("audio"),
)
}
vllm/model_executor/models/granite_speech.py
+4 -8
View File
@@ -34,7 +34,7 @@ from torch import nn
from transformers import BatchFeature, PretrainedConfig
from vllm.config import CacheConfig, ModelConfig, SpeechToTextConfig, VllmConfig
from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.config.speech_to_text import SpeechToTextParams
from vllm.inputs import MultiModalDataDict, PromptType, TokensPrompt
from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
@@ -208,17 +208,13 @@ class GraniteSpeechDummyInputsBuilder(
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_audios = mm_counts.get("audio", 0)
audio_overrides = mm_options.get("audio")
assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions)
return {
"audio": self._get_dummy_audios(
length=self.info.get_max_audio_len(),
num_audios=num_audios,
overrides=audio_overrides,
num_audios=mm_counts.get("audio", 0),
overrides=mm_options.get("audio"),
)
}
vllm/model_executor/models/hyperclovax_vision_v2.py
+7 -13
View File
@@ -18,7 +18,7 @@ import torch.nn as nn
from transformers import BatchFeature
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.forward_context import set_forward_context
from vllm.inputs import MultiModalDataDict
from vllm.multimodal import MULTIMODAL_REGISTRY
@@ -193,7 +193,7 @@ class HCXVisionV2DummyInputsBuilder(BaseDummyInputsBuilder[HCXVisionV2Processing
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions] | None = None,
mm_options: MultiModalDummyOptions | None = None,
mm_processor_kwargs: Mapping[str, object] | None = None,
) -> ProcessorInputs:
"""Build dummy processor inputs for memory profiling."""
@@ -225,31 +225,25 @@ class HCXVisionV2DummyInputsBuilder(BaseDummyInputsBuilder[HCXVisionV2Processing
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions] | None = None,
mm_options: MultiModalDummyOptions | None = None,
mm_processor_kwargs: Mapping[str, object] | None = None,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
num_videos = mm_counts.get("video", 0)
target_width, target_height = self.info.get_image_size_with_most_features()
target_num_frames = 16 # Default for video
image_overrides = mm_options.get("image") if mm_options else None
video_overrides = mm_options.get("video") if mm_options else None
result: MultiModalDataDict = {
"image": self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides, # type: ignore
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image") if mm_options else None,
),
"video": self._get_dummy_videos(
width=target_width,
height=target_height,
num_frames=target_num_frames,
num_videos=num_videos,
overrides=video_overrides, # type: ignore
num_videos=mm_counts.get("video", 0),
overrides=mm_options.get("video") if mm_options else None,
),
}
vllm/model_executor/models/idefics3.py
+4 -7
View File
@@ -29,7 +29,7 @@ from transformers import (
)
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.layers.logits_processor import LogitsProcessor
@@ -221,21 +221,18 @@ class Idefics3DummyInputsBuilder(BaseDummyInputsBuilder[Idefics3ProcessingInfo])
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
hf_processor = self.info.get_hf_processor()
image_processor: Idefics3ImageProcessor = hf_processor.image_processor
longest_edge = image_processor.max_image_size["longest_edge"]
image_overrides = mm_options.get("image")
return {
"image": self._get_dummy_images(
width=longest_edge,
height=longest_edge,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
)
}
vllm/model_executor/models/interns1.py
+6 -12
View File
@@ -22,7 +22,7 @@ from transformers.models.internvl.video_processing_internvl import (
)
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.models.interns1_vit import InternS1VisionModel
@@ -299,35 +299,29 @@ class InternS1DummyInputsBuilder(BaseDummyInputsBuilder[InternS1ProcessingInfo])
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
target_width, target_height = self.info.get_image_size_with_most_features()
target_num_frames = self.info.get_num_frames_with_most_features(
seq_len, mm_counts
)
num_images = mm_counts.get("image", 0)
num_videos = mm_counts.get("video", 0)
config = self.info.get_hf_config()
image_size_h, image_size_w = config.vision_config.image_size
image_overrides = mm_options.get("image")
video_overrides = mm_options.get("video")
assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions)
return {
"image": self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
),
"video": self._get_dummy_videos(
width=image_size_w,
height=image_size_h,
num_frames=target_num_frames,
num_videos=num_videos,
overrides=video_overrides,
num_videos=mm_counts.get("video", 0),
overrides=mm_options.get("video"),
),
}
vllm/model_executor/models/internvl.py
+5 -11
View File
@@ -17,7 +17,7 @@ import torch.nn as nn
from transformers import BatchFeature, PretrainedConfig
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
@@ -203,19 +203,16 @@ class BaseInternVLDummyInputsBuilder(BaseDummyInputsBuilder[_I]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
target_width, target_height = self.info.get_image_size_with_most_features()
num_images = mm_counts.get("image", 0)
image_overrides = mm_options.get("image")
return {
"image": self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
)
}
@@ -425,7 +422,7 @@ class InternVLDummyInputsBuilder(
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
dummy_image = super().get_dummy_mm_data(seq_len, mm_counts, mm_options)
if self.info.ctx_video_token:
@@ -436,9 +433,6 @@ class InternVLDummyInputsBuilder(
)
num_videos = mm_counts.get("video", 0)
video_overrides = mm_options.get("video")
assert video_overrides is None or isinstance(
video_overrides, VideoDummyOptions
)
dummy_video = {
"video": self._get_dummy_videos(
width=image_size,
vllm/model_executor/models/isaac.py
+4 -7
View File
@@ -13,7 +13,7 @@ from einops import rearrange
from transformers.image_processing_utils import BatchFeature
from vllm.config import ModelConfig, VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.distributed import parallel_state
from vllm.distributed import utils as dist_utils
from vllm.inputs import MultiModalDataDict
@@ -386,19 +386,16 @@ class IsaacDummyInputsBuilder(BaseDummyInputsBuilder[IsaacProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
target_width, target_height = self.info.get_image_size_with_most_features()
image_overrides = mm_options.get("image")
return {
"image": self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
),
}
vllm/model_executor/models/kanana_v.py
+2 -2
View File
@@ -18,7 +18,7 @@ from transformers.modeling_outputs import BaseModelOutput
from transformers.models.qwen2_vl.configuration_qwen2_vl import Qwen2VLVisionConfig
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.logger import init_logger
from vllm.multimodal import MULTIMODAL_REGISTRY
@@ -442,7 +442,7 @@ class KananaVDummyInputsBuilder(BaseDummyInputsBuilder[KananaVProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
return {
vllm/model_executor/models/keye.py
+6 -12
View File
@@ -17,7 +17,7 @@ from transformers.modeling_outputs import BaseModelOutput, BaseModelOutputWithPo
from transformers.utils import torch_int
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.inputs import ModalityData, MultiModalDataDict
from vllm.logger import init_logger
@@ -1124,30 +1124,24 @@ class KeyeBaseDummyInputsBuilder(BaseDummyInputsBuilder[_I]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
num_videos = mm_counts.get("video", 0)
target_width, target_height = self.info.get_image_size_with_most_features()
target_num_frames = self.info.get_num_frames_with_most_features(seq_len)
image_overrides = mm_options.get("image")
video_overrides = mm_options.get("video")
mm_data = {
"image": self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
),
"video": self._get_dummy_videos(
width=target_width,
height=target_height,
num_frames=target_num_frames,
num_videos=num_videos,
overrides=video_overrides,
num_videos=mm_counts.get("video", 0),
overrides=mm_options.get("video"),
),
}
vllm/model_executor/models/kimi_audio.py
+2 -2
View File
@@ -13,7 +13,7 @@ from transformers import BatchFeature
from transformers import WhisperConfig as HFWhisperConfig
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.config.speech_to_text import SpeechToTextParams
from vllm.inputs import PromptType, TokensPrompt
from vllm.model_executor.model_loader import DefaultModelLoader
@@ -180,7 +180,7 @@ class KimiAudioDummyInputsBuilder(BaseDummyInputsBuilder[KimiAudioProcessingInfo
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> ProcessorInputs:
dummy_mm_data = self.get_dummy_mm_data(seq_len, mm_counts, mm_options)
dummy_mm_items = self.info.parse_mm_data(dummy_mm_data)
vllm/model_executor/models/kimi_k25.py
+2 -2
View File
@@ -14,7 +14,7 @@ from torch import nn
from transformers import BatchFeature
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -216,7 +216,7 @@ class KimiK25DummyInputsBuilder(BaseDummyInputsBuilder[KimiK25ProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
# TODO: Support mm_options for vision_chunk to allow user configuration
dummy_items = self.get_dummy_mm_items()
vllm/model_executor/models/kimi_vl.py
+4 -8
View File
@@ -53,7 +53,7 @@ from transformers import BatchFeature
from transformers.activations import GELUActivation
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.models.interfaces import (
@@ -219,18 +219,14 @@ class KimiVLDummyInputsBuilder(BaseDummyInputsBuilder[KimiVLProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
image_overrides = mm_options.get("image")
return {
"image": self._get_dummy_images(
width=MaxImageTokenMeta.width,
height=MaxImageTokenMeta.height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
)
}
vllm/model_executor/models/lfm2_vl.py
+4 -8
View File
@@ -21,7 +21,7 @@ from transformers.models.lfm2_vl.image_processing_lfm2_vl_fast import (
from typing_extensions import Buffer
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.forward_context import set_forward_context
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.linear import ReplicatedLinear
@@ -387,20 +387,16 @@ class Lfm2VLDummyInputsBuilder(BaseDummyInputsBuilder[Lfm2VLProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
target_width, target_height = self.info.get_image_size_with_most_features()
image_overrides = mm_options.get("image")
return {
"image": self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
),
}
vllm/model_executor/models/llava.py
+4 -8
View File
@@ -19,7 +19,7 @@ from transformers.models.llava import LlavaProcessor
from transformers.models.pixtral import PixtralProcessor
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.activation import get_act_fn
from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
@@ -228,20 +228,16 @@ class LlavaDummyInputsBuilder(BaseDummyInputsBuilder[_I]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
target_width, target_height = self.info.get_image_size_with_most_features()
image_overrides = mm_options.get("image")
return {
"image": self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
)
}
vllm/model_executor/models/llava_next_video.py
+4 -9
View File
@@ -10,7 +10,7 @@ import torch.nn as nn
from transformers import BatchFeature, LlavaNextVideoConfig, LlavaNextVideoProcessor
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.activation import get_act_fn
from vllm.model_executor.models.clip import CLIPVisionModel
@@ -171,25 +171,20 @@ class LlavaNextVideoDummyInputsBuilder(
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_videos = mm_counts.get("video", 0)
target_width, target_height = self.info.get_image_size_with_most_features()
target_num_frames = self.info.get_num_frames_with_most_features(
seq_len, mm_counts
)
video_overrides = mm_options.get("video")
assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions)
return {
"video": self._get_dummy_videos(
width=target_width,
height=target_height,
num_frames=target_num_frames,
num_videos=num_videos,
overrides=video_overrides,
num_videos=mm_counts.get("video", 0),
overrides=mm_options.get("video"),
)
}
vllm/model_executor/models/llava_onevision.py
+6 -13
View File
@@ -14,7 +14,7 @@ from transformers.models.llava_onevision.modeling_llava_onevision import (
)
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.activation import get_act_fn
from vllm.multimodal import MULTIMODAL_REGISTRY
@@ -278,33 +278,26 @@ class LlavaOnevisionDummyInputsBuilder(
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
num_videos = mm_counts.get("video", 0)
target_width, target_height = self.info.get_image_size_with_most_features()
target_num_frames = self.info.get_num_frames_with_most_features(
seq_len, mm_counts
)
image_overrides = mm_options.get("image")
video_overrides = mm_options.get("video")
assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions)
return {
"image": self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
),
"video": self._get_dummy_videos(
width=target_width,
height=target_height,
num_frames=target_num_frames,
num_videos=num_videos,
overrides=video_overrides,
num_videos=mm_counts.get("video", 0),
overrides=mm_options.get("video"),
),
}
vllm/model_executor/models/llava_onevision2.py
+2 -2
View File
@@ -54,7 +54,7 @@ from vllm.compilation.decorators import (
support_torch_compile,
)
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.distributed import parallel_state
from vllm.distributed import utils as dist_utils
from vllm.inputs import ModalityData, MultiModalDataDict
@@ -1471,7 +1471,7 @@ class LlavaOnevision2DummyInputsBuilder(
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions] | None = None,
mm_options: MultiModalDummyOptions | None = None,
) -> MultiModalDataDict:
n_img = mm_counts.get("image", 0)
n_vid = mm_counts.get("video", 0)
vllm/model_executor/models/midashenglm.py
+4 -9
View File
@@ -36,7 +36,7 @@ from torch.nn.functional import scaled_dot_product_attention
from transformers import BatchFeature
from vllm.config import VllmConfig
from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.activation import get_act_fn
@@ -564,18 +564,13 @@ class MiDashengLMDummyInputsBuilder(BaseDummyInputsBuilder[MiDashengLMProcessing
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_audios = mm_counts.get("audio", 0)
audio_overrides = mm_options.get("audio")
assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions)
return {
"audio": self._get_dummy_audios(
length=self.info.get_max_audio_len(),
num_audios=num_audios,
overrides=audio_overrides,
num_audios=mm_counts.get("audio", 0),
overrides=mm_options.get("audio"),
)
}
vllm/model_executor/models/mimo_v2_omni.py
+6 -17
View File
@@ -15,11 +15,7 @@ from transformers.models.qwen2_vl.image_processing_qwen2_vl import smart_resize
from typing_extensions import TypedDict
from vllm.config import VllmConfig
from vllm.config.multimodal import (
BaseDummyOptions,
ImageDummyOptions,
VideoDummyOptions,
)
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.distributed import parallel_state
from vllm.distributed import utils as dist_utils
from vllm.inputs import MultiModalDataDict
@@ -1184,33 +1180,26 @@ class MiMoV2OmniDummyInputsBuilder(BaseDummyInputsBuilder[MiMoV2OmniProcessingIn
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
num_videos = mm_counts.get("video", 0)
target_width, target_height = self.info.get_image_size_with_most_features()
target_num_frames = self.info.get_num_frames_with_most_features(
seq_len, mm_counts
)
image_overrides = mm_options.get("image")
video_overrides = mm_options.get("video")
assert image_overrides is None or isinstance(image_overrides, ImageDummyOptions)
assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions)
return {
"image": self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
),
"video": self._get_dummy_videos(
width=target_width,
height=target_height,
num_frames=target_num_frames,
num_videos=num_videos,
overrides=video_overrides,
num_videos=mm_counts.get("video", 0),
overrides=mm_options.get("video"),
),
}
vllm/model_executor/models/minicpmo.py
+4 -8
View File
@@ -40,7 +40,7 @@ from transformers.models.whisper.modeling_whisper import (
)
from vllm.config import VllmConfig
from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import ModalityData, MultiModalDataDict
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargsItems
from vllm.multimodal.inputs import (
@@ -402,22 +402,18 @@ class MiniCPMODummyInputsBuilder(MiniCPMVDummyInputsBuilder[MiniCPMOProcessingIn
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_audios = mm_counts.get("audio", 0)
audio_len = (
self.info.get_max_audio_chunks_with_most_features()
* self.info.get_default_audio_sampling_rate()
)
audio_overrides = mm_options.get("audio")
assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions)
audio_mm_data = {
"audio": self._get_dummy_audios(
length=audio_len,
num_audios=num_audios,
overrides=audio_overrides,
num_audios=mm_counts.get("audio", 0),
overrides=mm_options.get("audio"),
)
}
vllm/model_executor/models/minicpmv.py
+5 -9
View File
@@ -45,9 +45,8 @@ from typing_extensions import TypedDict, TypeVar
from vllm.config import VllmConfig
from vllm.config.multimodal import (
BaseDummyOptions,
ImageDummyOptions,
VideoDummyOptions,
MultiModalDummyOptions,
)
from vllm.inputs import ModalityData, MultiModalDataDict
from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -848,9 +847,8 @@ class MiniCPMVDummyInputsBuilder(BaseDummyInputsBuilder[_I]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
num_videos = mm_counts.get("video", 0)
image_width, image_height = self.info.get_image_size_with_most_features()
@@ -859,14 +857,12 @@ class MiniCPMVDummyInputsBuilder(BaseDummyInputsBuilder[_I]):
seq_len, mm_counts
)
image_overrides = mm_options.get("image")
video_overrides = mm_options.get("video")
assert image_overrides is None or isinstance(image_overrides, ImageDummyOptions)
# Convert video overrides to image overrides for per-frame image generation,
# and apply num_frames override to num_video_frames.
video_frame_overrides: ImageDummyOptions | None = None
if isinstance(video_overrides, VideoDummyOptions):
if video_overrides is not None:
if video_overrides.num_frames:
num_video_frames = min(num_video_frames, video_overrides.num_frames)
if video_overrides.width or video_overrides.height:
@@ -880,8 +876,8 @@ class MiniCPMVDummyInputsBuilder(BaseDummyInputsBuilder[_I]):
"image": self._get_dummy_images(
width=image_width,
height=image_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
),
"video": [
self._get_dummy_images(
vllm/model_executor/models/mistral3.py
+4 -8
View File
@@ -11,7 +11,7 @@ from transformers import BatchFeature, Mistral3Config, PixtralVisionConfig
from transformers.models.pixtral import PixtralProcessor
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.activation import get_act_fn
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -261,20 +261,16 @@ class Mistral3DummyInputsBuilder(BaseDummyInputsBuilder):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
target_width, target_height = self.info.get_image_size_with_most_features()
image_overrides = mm_options.get("image")
return {
"image": self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
)
}
vllm/model_executor/models/mllama4.py
+4 -8
View File
@@ -36,7 +36,7 @@ from vllm.compilation.decorators import (
support_torch_compile,
)
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.attention import MMEncoderAttention
@@ -709,20 +709,16 @@ class Mllama4DummyInputsBuilder(BaseDummyInputsBuilder[Mllama4ProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
(target_width, target_height) = self.info.get_image_size_with_most_features()
image_overrides = mm_options.get("image")
return {
"image": self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
)
}
vllm/model_executor/models/molmo.py
+4 -7
View File
@@ -21,7 +21,7 @@ from transformers import (
from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.distributed import (
get_pp_group,
get_tensor_model_parallel_rank,
@@ -1104,19 +1104,16 @@ class MolmoDummyInputsBuilder(BaseDummyInputsBuilder[MolmoProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
target_width, target_height = self.info.get_image_size_with_most_features()
num_images = mm_counts.get("image", 0)
image_overrides = mm_options.get("image")
return {
"image": self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
)
}
vllm/model_executor/models/molmo2.py
+3 -6
View File
@@ -26,7 +26,7 @@ from typing_extensions import TypedDict
from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, VllmConfig
from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions, VideoDummyOptions
from vllm.distributed import (
get_pp_group,
get_tensor_model_parallel_rank,
@@ -1834,7 +1834,7 @@ class Molmo2DummyInputsBuilder(BaseDummyInputsBuilder[Molmo2ProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
num_videos = mm_counts.get("video", 0)
@@ -1845,13 +1845,11 @@ class Molmo2DummyInputsBuilder(BaseDummyInputsBuilder[Molmo2ProcessingInfo]):
if num_images > 0:
target_width, target_height = self.info.get_image_size_with_most_features()
image_overrides = mm_options.get("image")
dummy_images = self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides,
overrides=mm_options.get("image"),
)
if num_videos > 0:
@@ -1864,7 +1862,6 @@ class Molmo2DummyInputsBuilder(BaseDummyInputsBuilder[Molmo2ProcessingInfo]):
video_overrides = mm_options.get("video")
if video_overrides:
assert isinstance(video_overrides, VideoDummyOptions)
num_frames_override = video_overrides.num_frames
if num_frames_override:
if num_frames_override > target_num_frames:
vllm/model_executor/models/moondream3.py
+3 -4
View File
@@ -13,7 +13,7 @@ import torch.nn.functional as F
from transformers import BatchFeature
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.distributed import (
get_pp_group,
get_tensor_model_parallel_rank,
@@ -943,15 +943,14 @@ class Moondream3DummyInputsBuilder(BaseDummyInputsBuilder[Moondream3ProcessingIn
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions] | None = None,
mm_options: MultiModalDummyOptions | None = None,
mm_processor_kwargs: Mapping[str, object] | None = None,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
return {
"image": self._get_dummy_images(
width=378,
height=378,
num_images=num_images,
num_images=mm_counts.get("image", 0),
)
}
vllm/model_executor/models/moss_audio.py
+4 -7
View File
@@ -19,7 +19,7 @@ from typing_extensions import TypedDict
from vllm.compilation.decorators import support_torch_compile
from vllm.config import VllmConfig
from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.distributed import (
get_pp_group,
get_tensor_model_parallel_world_size,
@@ -1341,16 +1341,13 @@ class MossAudioDummyInputsBuilder(BaseDummyInputsBuilder[MossAudioProcessingInfo
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_audios = mm_counts.get("audio", 0)
audio_overrides = mm_options.get("audio")
assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions)
return {
"audio": self._get_dummy_audios(
length=16000,
num_audios=num_audios,
overrides=audio_overrides,
num_audios=mm_counts.get("audio", 0),
overrides=mm_options.get("audio"),
)
}
vllm/model_executor/models/moss_transcribe_diarize.py
+4 -6
View File
@@ -20,7 +20,7 @@ from torch import nn
from transformers import BatchFeature
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.config.speech_to_text import SpeechToTextParams
from vllm.inputs import ModalityData, MultiModalDataDict, PromptType, TextPrompt
from vllm.model_executor.models.interfaces import (
@@ -423,20 +423,18 @@ class MossTranscribeDiarizeDummyInputsBuilder(
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_audios = mm_counts.get("audio", 0)
if num_audios == 0:
return {}
feature_extractor = self.info.get_feature_extractor()
audio_overrides = mm_options.get("audio")
assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions)
return {
"audio": self._get_dummy_audios(
length=_get_max_audio_samples(feature_extractor),
num_audios=num_audios,
overrides=audio_overrides,
overrides=mm_options.get("audio"),
)
}
@@ -444,7 +442,7 @@ class MossTranscribeDiarizeDummyInputsBuilder(
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> ProcessorInputs:
dummy_mm_data = self.get_dummy_mm_data(seq_len, mm_counts, mm_options)
dummy_mm_items = self.info.parse_mm_data(dummy_mm_data)
vllm/model_executor/models/muse_glimmer.py
+4 -12
View File
@@ -45,11 +45,7 @@ from transformers import BatchFeature
from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, VllmConfig
from vllm.config.multimodal import (
BaseDummyOptions,
ImageDummyOptions,
VideoDummyOptions,
)
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.distributed import (
divide,
get_pp_group,
@@ -234,7 +230,7 @@ class MuseGlimmerDummyInputsBuilder(BaseDummyInputsBuilder[MuseGlimmerProcessing
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
processor = self.info.get_hf_processor()
video_processor = processor.video_processor
@@ -245,16 +241,12 @@ class MuseGlimmerDummyInputsBuilder(BaseDummyInputsBuilder[MuseGlimmerProcessing
* int(video_processor.downsample_factor)
* video_grid
)
image_overrides = mm_options.get("image")
video_overrides = mm_options.get("video")
assert image_overrides is None or isinstance(image_overrides, ImageDummyOptions)
assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions)
return {
"image": self._get_dummy_images(
width=image_width,
height=image_height,
num_images=mm_counts.get("image", 0),
overrides=image_overrides,
overrides=mm_options.get("image"),
),
"video": self._get_dummy_videos(
width=video_size,
@@ -263,7 +255,7 @@ class MuseGlimmerDummyInputsBuilder(BaseDummyInputsBuilder[MuseGlimmerProcessing
seq_len, mm_counts
),
num_videos=mm_counts.get("video", 0),
overrides=video_overrides,
overrides=mm_options.get("video"),
),
}
vllm/model_executor/models/nano_nemotron_vl.py
+4 -14
View File
@@ -21,8 +21,7 @@ from transformers import BatchFeature, PretrainedConfig
from vllm import envs
from vllm.config import VllmConfig
from vllm.config.multimodal import (
AudioDummyOptions,
BaseDummyOptions,
MultiModalDummyOptions,
VideoDummyOptions,
)
from vllm.inputs import MultiModalDataDict, MultiModalInput
@@ -839,22 +838,19 @@ class NanoNemotronVLDummyInputsBuilder(
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
(target_width, target_height), _ = (
self.info.get_dummy_image_size_and_max_tokens(mm_counts)
)
processor = self.info.get_hf_processor()
image_overrides = mm_options.get("image")
dummy_image = {
"image": self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
)
}
@@ -889,9 +885,6 @@ class NanoNemotronVLDummyInputsBuilder(
target_num_frames = num_frames
num_videos = mm_counts.get("video", 0)
video_overrides = mm_options.get("video")
assert video_overrides is None or isinstance(
video_overrides, VideoDummyOptions
)
dummy_video = {
"video": self._get_dummy_videos(
width=video_width,
@@ -907,9 +900,6 @@ class NanoNemotronVLDummyInputsBuilder(
if sound_config := self.info.sound_config:
num_audios = mm_counts.get("audio", 0)
audio_overrides = mm_options.get("audio") if mm_options else None
assert audio_overrides is None or isinstance(
audio_overrides, AudioDummyOptions
)
tokens_per_audio = max(1, seq_len // max(num_audios, 1))
max_audio_num_samples = MAX_AUDIO_LEN_S * sound_config.sampling_rate
calculated_max_audio_num_samples = ParakeetExtractor.audio_length(
vllm/model_executor/models/nemotron_parse.py
+2 -2
View File
@@ -22,7 +22,7 @@ from transformers import (
from vllm.config import CacheConfig, VllmConfig
from vllm.config.lora import LoRAConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.logger import init_logger
from vllm.model_executor.layers.activation import get_act_fn
@@ -388,7 +388,7 @@ class NemotronParseDummyInputsBuilder(
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
vllm/model_executor/models/nvlm_d.py
+4 -7
View File
@@ -13,7 +13,7 @@ import torch
import torch.nn as nn
from transformers import PretrainedConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.multimodal import MULTIMODAL_REGISTRY
@@ -83,19 +83,16 @@ class NVLMDummyInputsBuilder(BaseInternVLDummyInputsBuilder[NVLMProcessingInfo])
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
target_width, target_height = self.info.get_image_size_with_most_features()
num_images = mm_counts.get("image", 0)
image_overrides = mm_options.get("image")
return {
"image": self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
)
}
vllm/model_executor/models/openvla.py
+4 -6
View File
@@ -9,7 +9,7 @@ import torch.nn as nn
from transformers import BatchFeature
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.activation import get_act_fn
from vllm.model_executor.layers.linear import (
@@ -299,18 +299,16 @@ class OpenVLADummyInputsBuilder(BaseDummyInputsBuilder[OpenVLAProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
image_overrides = mm_options.get("image")
image_size = self.info.get_image_size_with_most_features()
return {
"image": self._get_dummy_images(
width=image_size.width,
height=image_size.height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
)
}
vllm/model_executor/models/ovis.py
+4 -8
View File
@@ -29,7 +29,7 @@ from torch.nn.functional import gumbel_softmax, pad, softmax
from transformers import BatchFeature, PretrainedConfig
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -311,20 +311,16 @@ class OvisDummyInputsBuilder(BaseDummyInputsBuilder[OvisProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
target_width, target_height = self.info.get_image_size_with_most_features()
image_overrides = mm_options.get("image")
mm_data = {
"image": self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
),
}
return mm_data
vllm/model_executor/models/ovis2_5.py
+6 -13
View File
@@ -11,7 +11,7 @@ import torch.nn as nn
from transformers import BaseImageProcessor, BatchFeature, PretrainedConfig
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -290,33 +290,26 @@ class Ovis2_5DummyInputsBuilder(BaseDummyInputsBuilder[Ovis2_5ProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
num_videos = mm_counts.get("video", 0)
target_width, target_height = self.info.get_image_size_with_most_features()
target_num_frames = self.info.get_num_frames_with_most_features(
seq_len, mm_counts
)
image_overrides = mm_options.get("image")
video_overrides = mm_options.get("video")
assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions)
mm_data = {
"image": self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
),
"video": self._get_dummy_videos(
width=target_width,
height=target_height,
num_frames=target_num_frames,
num_videos=num_videos,
overrides=video_overrides,
num_videos=mm_counts.get("video", 0),
overrides=mm_options.get("video"),
),
}
return mm_data
vllm/model_executor/models/paddleocr_vl.py
+4 -7
View File
@@ -32,7 +32,7 @@ from transformers.modeling_outputs import (
from transformers.utils import torch_int
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.distributed import parallel_state
from vllm.distributed import utils as dist_utils
from vllm.inputs import MultiModalDataDict
@@ -223,19 +223,16 @@ class PaddleOCRVLDummyInputsBuilder(BaseDummyInputsBuilder[PaddleOCRVLProcessing
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
max_image_size = self.info.get_image_size_with_most_features()
image_overrides = mm_options.get("image")
return {
"image": self._get_dummy_images(
width=max_image_size.width,
height=max_image_size.height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
)
}
vllm/model_executor/models/paligemma.py
+4 -8
View File
@@ -8,7 +8,7 @@ from torch import nn
from transformers import BatchFeature, PaliGemmaConfig
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict, MultiModalInput
from vllm.logger import init_logger
from vllm.multimodal import MULTIMODAL_REGISTRY
@@ -130,22 +130,18 @@ class PaliGemmaDummyInputsBuilder(BaseDummyInputsBuilder[PaliGemmaProcessingInfo
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
hf_config = self.info.get_hf_config()
vision_config = hf_config.vision_config
max_image_size = vision_config.image_size
num_images = mm_counts.get("image", 0)
image_overrides = mm_options.get("image")
return {
"image": self._get_dummy_images(
width=max_image_size,
height=max_image_size,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
)
}
vllm/model_executor/models/phi3v.py
+4 -8
View File
@@ -29,7 +29,7 @@ from transformers import (
)
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -371,20 +371,16 @@ class Phi3VDummyInputsBuilder(BaseDummyInputsBuilder[Phi3VProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
target_width, target_height = self.info.get_image_size_with_most_features()
image_overrides = mm_options.get("image")
return {
"image": self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
)
}
vllm/model_executor/models/phi4mm.py
+6 -13
View File
@@ -17,7 +17,7 @@ from transformers import (
)
from vllm.config import VllmConfig
from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.distributed import get_pp_group
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.logits_processor import LogitsProcessor
@@ -839,28 +839,21 @@ class Phi4MMDummyInputsBuilder(BaseDummyInputsBuilder[Phi4MMProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_audios = mm_counts.get("audio", 0)
num_images = mm_counts.get("image", 0)
target_width, target_height = self.info.get_image_size_with_most_features()
image_overrides = mm_options.get("image")
audio_overrides = mm_options.get("audio")
assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions)
mm_data = {
"image": self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
),
"audio": self._get_dummy_audios(
length=_AUDIO_MAX_SOUNDFILE_SIZE,
num_audios=num_audios,
overrides=audio_overrides,
num_audios=mm_counts.get("audio", 0),
overrides=mm_options.get("audio"),
),
}
vllm/model_executor/models/phi4siglip.py
+3 -4
View File
@@ -14,7 +14,7 @@ import torch.nn as nn
from transformers import BatchFeature, PretrainedConfig, Siglip2VisionConfig
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.logger import init_logger
from vllm.multimodal import MULTIMODAL_REGISTRY
@@ -126,15 +126,14 @@ class Phi4SiglipDummyInputsBuilder(
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
size = self.info.get_image_size_with_most_features()
return {
"image": self._get_dummy_images(
width=size.width,
height=size.height,
num_images=num_images,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
),
}
vllm/model_executor/models/pixtral.py
+5 -9
View File
@@ -21,7 +21,7 @@ from transformers.models.pixtral.image_processing_pixtral import (
from transformers.models.pixtral.modeling_pixtral import apply_rotary_pos_emb
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.distributed import divide, get_tensor_model_parallel_world_size
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.activation import SiluAndMul, get_act_and_mul_fn
@@ -208,20 +208,16 @@ class PixtralDummyInputsBuilder(BaseDummyInputsBuilder[PixtralProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
target_width, target_height = self.info.get_image_size_with_most_features()
image_overrides = mm_options.get("image")
return {
"image": self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
)
}
@@ -229,7 +225,7 @@ class PixtralDummyInputsBuilder(BaseDummyInputsBuilder[PixtralProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
mm_data: MultiModalDataDict | None = None,
) -> ProcessorInputs:
tokenizer = self.info.get_tokenizer()
vllm/model_executor/models/qwen2_5_omni_thinker.py
+8 -16
View File
@@ -44,7 +44,7 @@ from transformers.models.qwen2_5_omni.processing_qwen2_5_omni import (
from transformers.models.whisper import WhisperFeatureExtractor
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.forward_context import set_forward_context
from vllm.inputs import ModalityData, MultiModalDataDict
from vllm.logger import init_logger
@@ -448,12 +448,8 @@ class Qwen2_5OmniThinkerDummyInputsBuilder(
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_audios = mm_counts.get("audio", 0)
num_images = mm_counts.get("image", 0)
num_videos = mm_counts.get("video", 0)
feature_extractor = self.info.get_feature_extractor()
target_audio_length = (
@@ -469,28 +465,24 @@ class Qwen2_5OmniThinkerDummyInputsBuilder(
seq_len, mm_counts
)
image_overrides = mm_options.get("image")
video_overrides = mm_options.get("video")
audio_overrides = mm_options.get("audio")
mm_data = {
"audio": self._get_dummy_audios(
length=target_audio_length,
num_audios=num_audios,
overrides=audio_overrides,
num_audios=mm_counts.get("audio", 0),
overrides=mm_options.get("audio"),
),
"image": self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
),
"video": self._get_dummy_videos(
width=target_width,
height=target_height,
num_frames=target_num_frames,
num_videos=num_videos,
overrides=video_overrides,
num_videos=mm_counts.get("video", 0),
overrides=mm_options.get("video"),
),
}
vllm/model_executor/models/qwen2_audio.py
+4 -7
View File
@@ -37,7 +37,7 @@ from transformers.models.qwen2_audio import (
from transformers.models.whisper import WhisperFeatureExtractor
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import ModalityData, MultiModalDataDict
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import (
@@ -214,21 +214,18 @@ class Qwen2AudioDummyInputsBuilder(BaseDummyInputsBuilder[Qwen2AudioProcessingIn
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
feature_extractor = self.info.get_feature_extractor()
sampling_rate = feature_extractor.sampling_rate
audio_len = feature_extractor.chunk_length * sampling_rate
num_audios = mm_counts.get("audio", 0)
audio_overrides = mm_options.get("audio")
return {
"audio": self._get_dummy_audios(
length=audio_len,
num_audios=num_audios,
overrides=audio_overrides,
num_audios=mm_counts.get("audio", 0),
overrides=mm_options.get("audio"),
)
}
vllm/model_executor/models/qwen2_vl.py
+6 -12
View File
@@ -43,7 +43,7 @@ from transformers.models.qwen2_vl.configuration_qwen2_vl import (
from transformers.models.qwen2_vl.image_processing_qwen2_vl import smart_resize
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.distributed import parallel_state, tensor_model_parallel_all_gather
from vllm.distributed import utils as dist_utils
from vllm.inputs import ModalityData, MultiModalDataDict
@@ -1096,32 +1096,26 @@ class Qwen2VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen2VLProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
num_videos = mm_counts.get("video", 0)
target_width, target_height = self.info.get_image_size_with_most_features()
target_num_frames = self.info.get_num_frames_with_most_features(
seq_len, mm_counts
)
image_overrides = mm_options.get("image")
video_overrides = mm_options.get("video")
return {
"image": self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
),
"video": self._get_dummy_videos(
width=target_width,
height=target_height,
num_frames=target_num_frames,
num_videos=num_videos,
overrides=video_overrides,
num_videos=mm_counts.get("video", 0),
overrides=mm_options.get("video"),
),
}
vllm/model_executor/models/qwen3_asr.py
+4 -8
View File
@@ -32,7 +32,7 @@ from transformers.feature_extraction_utils import BatchFeature
from transformers.models.whisper import WhisperFeatureExtractor
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.config.speech_to_text import SpeechToTextParams
from vllm.inputs import ModalityData, MultiModalDataDict, PromptType, TokensPrompt
from vllm.logger import init_logger
@@ -226,10 +226,8 @@ class Qwen3ASRDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3ASRProcessingInfo])
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_audios = mm_counts.get("audio", 0)
feature_extractor = self.info.get_feature_extractor()
target_audio_length = (
@@ -240,13 +238,11 @@ class Qwen3ASRDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3ASRProcessingInfo])
* feature_extractor.sampling_rate
)
audio_overrides = mm_options.get("audio")
return {
"audio": self._get_dummy_audios(
length=target_audio_length,
num_audios=num_audios,
overrides=audio_overrides,
num_audios=mm_counts.get("audio", 0),
overrides=mm_options.get("audio"),
),
}
vllm/model_executor/models/qwen3_vl.py
+5 -10
View File
@@ -54,8 +54,8 @@ from vllm.compilation.decorators import (
)
from vllm.config import VllmConfig
from vllm.config.multimodal import (
BaseDummyOptions,
MultiModalConfig,
MultiModalDummyOptions,
VideoDummyOptions,
VideoPruningMethod,
)
@@ -1119,11 +1119,8 @@ class Qwen3VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3VLProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
num_videos = mm_counts.get("video", 0)
image_overrides = mm_options.get("image")
video_overrides = mm_options.get("video")
target_image_width, target_image_height = (
@@ -1133,7 +1130,6 @@ class Qwen3VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3VLProcessingInfo]):
# treat videos as special images
target_num_frames = 2
if video_overrides:
assert isinstance(video_overrides, VideoDummyOptions)
num_frames_override = video_overrides.num_frames
if num_frames_override:
if num_frames_override > target_num_frames:
@@ -1204,7 +1200,6 @@ class Qwen3VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3VLProcessingInfo]):
target_video_size.height,
)
if video_overrides:
assert isinstance(video_overrides, VideoDummyOptions)
width_override = video_overrides.width
if width_override:
if width_override > target_video_width:
@@ -1230,14 +1225,14 @@ class Qwen3VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3VLProcessingInfo]):
"image": self._get_dummy_images(
width=target_image_width,
height=target_image_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
),
"video": self._get_dummy_videos(
width=target_video_width,
height=target_video_height,
num_frames=target_num_frames,
num_videos=num_videos,
num_videos=mm_counts.get("video", 0),
),
}
vllm/model_executor/models/rvl.py
+4 -8
View File
@@ -8,7 +8,7 @@ import torch.nn as nn
from transformers.activations import GELUActivation
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.multimodal import MULTIMODAL_REGISTRY
@@ -40,20 +40,16 @@ class RVLDummyInputsBuilder(LlavaDummyInputsBuilder[RVLProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
target_width, target_height = self.info.get_image_size_with_most_features()
image_overrides = mm_options.get("image")
return {
"image": self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
),
}
vllm/model_executor/models/siglip.py
+4 -8
View File
@@ -16,7 +16,7 @@ from transformers import (
)
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.distributed import divide, get_tensor_model_parallel_world_size
from vllm.inputs import MultiModalDataDict, MultiModalInput
from vllm.model_executor.layers.activation import get_act_fn
@@ -158,20 +158,16 @@ class SiglipDummyInputsBuilder(BaseDummyInputsBuilder[SiglipProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
target_width, target_height = self.info.get_image_size_with_most_features()
image_overrides = mm_options.get("image")
return {
"image": self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
)
}
vllm/model_executor/models/step3_vl.py
+4 -7
View File
@@ -11,7 +11,7 @@ import torch.nn.functional as F
from transformers import BatchFeature
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.activation import get_act_fn
@@ -139,19 +139,16 @@ class Step3VLDummyInputsBuilder(BaseDummyInputsBuilder[Step3VLProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
target_width, target_height = self.info.get_image_size_with_most_features()
num_images = mm_counts.get("image", 0)
image_overrides = mm_options.get("image")
return {
"image": self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
)
}
vllm/model_executor/models/terratorch.py
+2 -2
View File
@@ -33,7 +33,7 @@ from terratorch.vllm import (
from transformers import BatchFeature
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import ModalityData, MultiModalDataDict, MultiModalInput, mm_input
from vllm.logger import init_logger
from vllm.model_executor.layers.pooler import IdentityPooler
@@ -153,7 +153,7 @@ class TerratorchInputBuilder(BaseDummyInputsBuilder[TerratorchProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
# Dummy data is generated based on the 'input' section
# defined in the HF configuration file
vllm/model_executor/models/transformers/multimodal.py
+7 -13
View File
@@ -27,7 +27,6 @@ from packaging.version import Version
from transformers.utils.generic import ModelOutput
from vllm.compilation.decorators import should_torch_compile_mm_encoder
from vllm.config.multimodal import AudioDummyOptions
from vllm.config.utils import getattr_iter
from vllm.inputs import (
MultiModalDataBuiltins,
@@ -75,7 +74,7 @@ if TYPE_CHECKING:
from transformers import BatchFeature, PreTrainedModel
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
logger = init_logger(__name__)
@@ -195,7 +194,7 @@ class MultiModalDummyInputsBuilder(BaseDummyInputsBuilder[MultiModalProcessingIn
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, "BaseDummyOptions"],
mm_options: "MultiModalDummyOptions",
) -> MultiModalDataDict:
data = MultiModalDataBuiltins()
if self.info._is_audio_model() and (num_audios := mm_counts.get("audio", 0)):
@@ -204,21 +203,16 @@ class MultiModalDummyInputsBuilder(BaseDummyInputsBuilder[MultiModalProcessingIn
chunk_length = getattr(sub, "chunk_length", None) if sub else None
if chunk_length is None:
chunk_length = 30
audio_len = int(chunk_length * sampling_rate)
audio_overrides = mm_options.get("audio")
assert audio_overrides is None or isinstance(
audio_overrides, AudioDummyOptions
)
data["audio"] = self._get_dummy_audios(
length=audio_len,
length=int(chunk_length * sampling_rate),
num_audios=num_audios,
overrides=audio_overrides,
overrides=mm_options.get("audio"),
)
if self.info._is_image_model() and (num_images := mm_counts.get("image", 0)):
target_width, target_height = self.info.get_image_size_with_most_features()
width, height = self.info.get_image_size_with_most_features()
data["image"] = self._get_dummy_images(
width=target_width,
height=target_height,
width=width,
height=height,
num_images=num_images,
overrides=mm_options.get("image"),
)
vllm/model_executor/models/ultravox.py
+4 -7
View File
@@ -16,7 +16,7 @@ from transformers import BatchFeature, ProcessorMixin
from transformers.models.whisper import WhisperFeatureExtractor
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.activation import MulAndSilu, get_act_fn
@@ -169,7 +169,7 @@ class UltravoxDummyInputsBuilder(BaseDummyInputsBuilder[UltravoxProcessingInfo])
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
feature_extractor = self.info.get_feature_extractor()
@@ -177,15 +177,12 @@ class UltravoxDummyInputsBuilder(BaseDummyInputsBuilder[UltravoxProcessingInfo])
audio_len = (
feature_extractor.chunk_length * sampling_rate * _MAX_ENCODER_BATCH_SIZE
)
num_audios = mm_counts.get("audio", 0)
audio_overrides = mm_options.get("audio")
return {
"audio": self._get_dummy_audios(
length=audio_len,
num_audios=num_audios,
overrides=audio_overrides,
num_audios=mm_counts.get("audio", 0),
overrides=mm_options.get("audio"),
)
}
vllm/model_executor/models/voxtral.py
+5 -9
View File
@@ -19,7 +19,7 @@ from mistral_common.tokens.tokenizers.audio import Audio
from transformers import WhisperConfig
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.config.speech_to_text import SpeechToTextParams
from vllm.inputs import MultiModalDataDict, PromptType, TokensPrompt
from vllm.logger import init_logger
@@ -137,19 +137,15 @@ class VoxtralDummyInputsBuilder(BaseDummyInputsBuilder[VoxtralProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_audios = mm_counts.get("audio", 0)
target_length = self.info.get_max_audio_array_len()
audio_overrides = mm_options.get("audio")
return {
"audio": self._get_dummy_audios(
length=target_length,
num_audios=num_audios,
overrides=audio_overrides,
num_audios=mm_counts.get("audio", 0),
overrides=mm_options.get("audio"),
)
}
@@ -157,7 +153,7 @@ class VoxtralDummyInputsBuilder(BaseDummyInputsBuilder[VoxtralProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
mm_data: MultiModalDataDict | None = None,
) -> ProcessorInputs:
tokenizer = self.info.get_tokenizer()
vllm/model_executor/models/whisper.py
+4 -7
View File
@@ -19,7 +19,7 @@ from transformers.models.whisper.modeling_whisper import sinusoids
from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, ModelConfig, SpeechToTextConfig, VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.config.speech_to_text import SpeechToTextParams
from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.inputs import (
@@ -724,21 +724,18 @@ class WhisperDummyInputsBuilder(BaseDummyInputsBuilder[WhisperProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
feature_extractor = self.info.get_feature_extractor()
sampling_rate = feature_extractor.sampling_rate
audio_len = feature_extractor.chunk_length * sampling_rate
num_audios = mm_counts.get("audio", 0)
audio_overrides = mm_options.get("audio")
return {
"audio": self._get_dummy_audios(
length=audio_len,
num_audios=num_audios,
overrides=audio_overrides,
num_audios=mm_counts.get("audio", 0),
overrides=mm_options.get("audio"),
)
}
vllm/models/deepseek_v4/common/mm_preprocess.py
+4 -4
View File
@@ -22,7 +22,7 @@ reference's out-of-vocab scheme.
import math
from collections.abc import Mapping, Sequence
from typing import Any, cast
from typing import Any
import numpy as np
import torch
@@ -30,7 +30,7 @@ from PIL import Image, ImageOps
from transformers import BatchFeature
from typing_extensions import assert_never
from vllm.config.multimodal import BaseDummyOptions, ImageDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargsItems
from vllm.multimodal.parse import ImageSize, MultiModalDataItems
@@ -375,7 +375,7 @@ class DeepseekV4VLDummyInputsBuilder(
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
size = self.info.get_image_size_with_most_features()
return {
@@ -383,7 +383,7 @@ class DeepseekV4VLDummyInputsBuilder(
width=size.width,
height=size.height,
num_images=mm_counts.get("image", 0),
overrides=cast(ImageDummyOptions | None, mm_options.get("image")),
overrides=mm_options.get("image"),
),
}
vllm/models/deepseek_v41/common/mm_preprocess.py
+4 -4
View File
@@ -19,14 +19,14 @@ boundaries freely).
import math
from collections.abc import Mapping, Sequence
from typing import Any, cast
from typing import Any
import numpy as np
import torch
from PIL import Image, ImageOps
from transformers import BatchFeature
from vllm.config.multimodal import BaseDummyOptions, ImageDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargsItems
from vllm.multimodal.parse import ImageSize, MultiModalDataItems
@@ -296,7 +296,7 @@ class DeepseekV4VLDummyInputsBuilder(
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
size = self.info.get_image_size_with_most_features()
return {
@@ -304,7 +304,7 @@ class DeepseekV4VLDummyInputsBuilder(
width=size.width,
height=size.height,
num_images=mm_counts.get("image", 0),
overrides=cast(ImageDummyOptions | None, mm_options.get("image")),
overrides=mm_options.get("image"),
),
}
vllm/models/dots3_note/common/processor.py
+5 -8
View File
@@ -13,10 +13,7 @@ from PIL import Image
from transformers import BatchFeature
from vllm.config.multimodal import (
AudioDummyOptions,
BaseDummyOptions,
ImageDummyOptions,
VideoDummyOptions,
MultiModalDummyOptions,
)
from vllm.inputs import MultiModalDataDict
from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargsItems
@@ -584,7 +581,7 @@ class Dots3NoteDummyInputsBuilder(BaseDummyInputsBuilder[Dots3NoteProcessingInfo
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
data: dict[str, Any] = {}
num_images = mm_counts.get("image", 0)
@@ -594,7 +591,7 @@ class Dots3NoteDummyInputsBuilder(BaseDummyInputsBuilder[Dots3NoteProcessingInfo
width=width,
height=height,
num_images=num_images,
overrides=cast(ImageDummyOptions | None, mm_options.get("image")),
overrides=mm_options.get("image"),
)
num_audios = mm_counts.get("audio", 0)
if num_audios:
@@ -611,7 +608,7 @@ class Dots3NoteDummyInputsBuilder(BaseDummyInputsBuilder[Dots3NoteProcessingInfo
data["audio"] = self._get_dummy_audios(
length=min(chunk_samples, max(1, seq_len) * stride),
num_audios=num_audios,
overrides=cast(AudioDummyOptions | None, mm_options.get("audio")),
overrides=mm_options.get("audio"),
)
num_videos = mm_counts.get("video", 0)
if num_videos:
@@ -625,7 +622,7 @@ class Dots3NoteDummyInputsBuilder(BaseDummyInputsBuilder[Dots3NoteProcessingInfo
height=height,
num_frames=num_frames,
num_videos=num_videos,
overrides=cast(VideoDummyOptions | None, mm_options.get("video")),
overrides=mm_options.get("video"),
)
return data
vllm/models/inkling/common/mm_preprocess.py
+4 -8
View File
@@ -13,9 +13,7 @@ import torch
from transformers.feature_extraction_utils import BatchFeature
from vllm.config.multimodal import (
AudioDummyOptions,
BaseDummyOptions,
ImageDummyOptions,
MultiModalDummyOptions,
)
from vllm.inputs import MultiModalDataDict
from vllm.multimodal.inputs import (
@@ -145,7 +143,7 @@ class InklingDummyInputsBuilder(BaseDummyInputsBuilder[InklingProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
config = self.info.get_hf_config()
num_images = mm_counts.get("image", 0)
@@ -156,12 +154,11 @@ class InklingDummyInputsBuilder(BaseDummyInputsBuilder[InklingProcessingInfo]):
patch_size = getattr(config.vision_config, "patch_size", 40)
# A square image ~4 patches wide so the dummy emits several patches.
side = patch_size * 4
image_overrides = mm_options.get("image")
mm_data["image"] = self._get_dummy_images(
width=side,
height=side,
num_images=num_images,
overrides=cast(ImageDummyOptions | None, image_overrides),
overrides=mm_options.get("image"),
)
if num_audios:
# Size the dummy at the maximum allowed audio so memory/encoder
@@ -169,11 +166,10 @@ class InklingDummyInputsBuilder(BaseDummyInputsBuilder[InklingProcessingInfo]):
params = self.info.get_hf_processor().audio_feature_extractor.params
hop = int(round(params.audio_token_duration_s * params.sample_rate))
audio_len = MAX_AUDIO_TOKENS * hop
audio_overrides = mm_options.get("audio")
mm_data["audio"] = self._get_dummy_audios(
length=audio_len,
num_audios=num_audios,
overrides=cast(AudioDummyOptions | None, audio_overrides),
overrides=mm_options.get("audio"),
)
return mm_data
vllm/models/kimi_k3/common/mm_preprocess.py
+3 -4
View File
@@ -9,7 +9,7 @@ from typing import Any, cast
import torch
from transformers import BatchFeature
from vllm.config.multimodal import BaseDummyOptions, ImageDummyOptions
from vllm.config.multimodal import ImageDummyOptions, MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.logger import init_logger
from vllm.multimodal.inputs import (
@@ -204,7 +204,7 @@ class KimiK3DummyInputsBuilder(BaseDummyInputsBuilder[KimiK3ProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions] | None = None,
mm_options: MultiModalDummyOptions | None = None,
) -> MultiModalDataDict:
media_proc_cfg = self.info.image_processor.media_proc_cfg
max_size = self.info.get_max_image_size(
@@ -214,7 +214,6 @@ class KimiK3DummyInputsBuilder(BaseDummyInputsBuilder[KimiK3ProcessingInfo]):
media_proc_cfg["patch_limit_on_one_side"],
media_proc_cfg["fixed_output_tokens"],
)
num_images = mm_counts.get("image", 0)
image_overrides = cast(
ImageDummyOptions | None,
mm_options.get("image") if mm_options else None,
@@ -223,7 +222,7 @@ class KimiK3DummyInputsBuilder(BaseDummyInputsBuilder[KimiK3ProcessingInfo]):
"image": self._get_dummy_images(
width=max_size.width,
height=max_size.height,
num_images=num_images,
num_images=mm_counts.get("image", 0),
overrides=image_overrides,
)
}
vllm/models/minimax_m3/common/mm_preprocess.py
+4 -6
View File
@@ -11,9 +11,7 @@ from transformers import BatchFeature
from transformers.video_utils import VideoMetadata
from vllm.config.multimodal import (
BaseDummyOptions,
ImageDummyOptions,
VideoDummyOptions,
MultiModalDummyOptions,
)
from vllm.inputs import MultiModalDataDict
from vllm.multimodal.inputs import (
@@ -283,7 +281,7 @@ class MiniMaxM3VLDummyInputsBuilder(BaseDummyInputsBuilder[MiniMaxM3VLProcessing
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
size = self.info.get_image_size_with_most_features()
video_size = self.info.get_video_size_with_most_features()
@@ -293,14 +291,14 @@ class MiniMaxM3VLDummyInputsBuilder(BaseDummyInputsBuilder[MiniMaxM3VLProcessing
width=size.width,
height=size.height,
num_images=mm_counts.get("image", 0),
overrides=cast(ImageDummyOptions | None, mm_options.get("image")),
overrides=mm_options.get("image"),
),
"video": self._get_dummy_videos(
width=video_size.width,
height=video_size.height,
num_frames=num_frames,
num_videos=mm_counts.get("video", 0),
overrides=cast(VideoDummyOptions | None, mm_options.get("video")),
overrides=mm_options.get("video"),
),
}
vllm/multimodal/processing/dummy_inputs.py
+4 -5
View File
@@ -10,8 +10,8 @@ from PIL import Image
from vllm.config.multimodal import (
AudioDummyOptions,
BaseDummyOptions,
ImageDummyOptions,
MultiModalDummyOptions,
VideoDummyOptions,
)
from vllm.inputs import MultiModalDataDict
@@ -45,7 +45,7 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
"""Build the multimodal input which, after processing, results in
the maximum possible number of placeholder tokens.
@@ -65,7 +65,7 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> ProcessorInputs:
"""Build the input which, after processing, results in
the maximum possible number of placeholder tokens.
@@ -124,9 +124,8 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]):
width: int,
height: int,
num_images: int,
overrides: BaseDummyOptions | None = None,
overrides: ImageDummyOptions | None = None,
) -> list[Image.Image]:
assert overrides is None or isinstance(overrides, ImageDummyOptions)
if num_images == 0:
return []
if overrides: