[Multimodal] Type dummy options per modality (#57576)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Harry Mellor
2026-09-18 17:29:11 +01:00
committed by GitHub
co-authored by Claude Opus 5
parent 56cac80b14
commit 8a5cf54387
93 changed files with 461 additions and 720 deletions
+4 -8
View File
@@ -10,7 +10,7 @@ from transformers.models.aria.modeling_aria import AriaCrossAttention
from transformers.models.aria.processing_aria import AriaProcessor
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions, ImageDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.activation import get_act_fn
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
@@ -315,22 +315,18 @@ class AriaDummyInputsBuilder(BaseDummyInputsBuilder[AriaProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
vision_config = self.info.get_vision_config()
max_image_size = vision_config.image_size
num_images = mm_counts.get("image", 0)
image_overrides = mm_options.get("image")
assert image_overrides is None or isinstance(image_overrides, ImageDummyOptions)
return {
"image": self._get_dummy_images(
width=max_image_size,
height=max_image_size,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
)
}
+4 -7
View File
@@ -30,7 +30,7 @@ from transformers.models.audioflamingo3 import (
from transformers.models.qwen2_audio import Qwen2AudioEncoder
from vllm.config import VllmConfig
from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import ModalityData, MultiModalDataDict
from vllm.model_executor.layers.activation import get_act_fn
from vllm.model_executor.models.module_mapping import MultiModelKeys
@@ -216,21 +216,18 @@ class AudioFlamingo3DummyInputsBuilder(
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
hf_processor = self.info.get_hf_processor()
feature_extractor = self.info.get_feature_extractor()
sampling_rate = feature_extractor.sampling_rate
audio_len = int(hf_processor.max_audio_len * sampling_rate)
num_audios = mm_counts.get("audio", 0)
audio_overrides = mm_options.get("audio")
assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions)
return {
"audio": self._get_dummy_audios(
length=audio_len,
num_audios=num_audios,
overrides=audio_overrides,
num_audios=mm_counts.get("audio", 0),
overrides=mm_options.get("audio"),
)
}
+4 -7
View File
@@ -14,7 +14,7 @@ import torch
import torch.nn as nn
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions, ImageDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.logger import init_logger
from vllm.model_executor.layers.activation import get_act_fn
@@ -248,23 +248,20 @@ class BagelDummyInputsBuilder(BaseDummyInputsBuilder[BagelProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
hf_config = self.info.get_hf_config()
vit_config = hf_config.vit_config
# Use the configured image size
image_size = vit_config.image_size
image_overrides = mm_options.get("image")
assert image_overrides is None or isinstance(image_overrides, ImageDummyOptions)
return {
"image": self._get_dummy_images(
width=image_size,
height=image_size,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
),
}
@@ -13,7 +13,7 @@ import torch.nn as nn
from transformers import AutoProcessor, BatchFeature
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
@@ -103,7 +103,7 @@ class BailingMoeV3VLDummyInputsBuilder(
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
target_width, target_height = self.info.get_image_size_with_most_features()
return {
+4 -9
View File
@@ -8,7 +8,7 @@ import torch.nn as nn
from transformers.activations import GELUActivation
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions, ImageDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.multimodal import MULTIMODAL_REGISTRY
@@ -90,21 +90,16 @@ class BeeDummyInputsBuilder(LlavaDummyInputsBuilder[BeeProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
target_width, target_height = self.info.get_image_size_with_most_features()
image_overrides = mm_options.get("image")
assert image_overrides is None or isinstance(image_overrides, ImageDummyOptions)
return {
"image": self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
),
}
+4 -8
View File
@@ -15,7 +15,7 @@ from transformers import (
)
from vllm.config import CacheConfig, VllmConfig
from vllm.config.multimodal import BaseDummyOptions, ImageDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.activation import get_act_fn
from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -444,23 +444,19 @@ class Blip2DummyInputsBuilder(BaseDummyInputsBuilder[Blip2ProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
hf_config = self.info.get_hf_config()
vision_config = hf_config.vision_config
max_image_size = vision_config.image_size
num_images = mm_counts.get("image", 0)
image_overrides = mm_options.get("image")
assert image_overrides is None or isinstance(image_overrides, ImageDummyOptions)
return {
"image": self._get_dummy_images(
width=max_image_size,
height=max_image_size,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
)
}
+4 -8
View File
@@ -15,7 +15,7 @@ from transformers import (
)
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.distributed import divide, get_tensor_model_parallel_world_size
from vllm.inputs import MultiModalDataDict, MultiModalInput
from vllm.model_executor.layers.activation import get_act_fn
@@ -205,20 +205,16 @@ class CLIPDummyInputsBuilder(BaseDummyInputsBuilder[CLIPProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
target_width, target_height = self.info.get_image_size_with_most_features()
image_overrides = mm_options.get("image")
return {
"image": self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
)
}
+4 -7
View File
@@ -18,7 +18,7 @@ from transformers.models.cohere2_vision.processing_cohere2_vision import (
)
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.activation import MulAndSilu
from vllm.model_executor.layers.linear import (
@@ -202,19 +202,16 @@ class Cohere2VisionDummyInputsBuilder(
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
image_size = self.info.get_image_size_with_most_features()
image_overrides = mm_options.get("image")
return {
"image": self._get_dummy_images(
width=image_size.width,
height=image_size.height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
)
}
+2 -2
View File
@@ -12,7 +12,7 @@ from transformers import PretrainedConfig
from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, ModelConfig, SpeechToTextConfig, VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.config.speech_to_text import SpeechToTextParams
from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.inputs import MultiModalDataDict, PromptType, TokensPrompt
@@ -1944,7 +1944,7 @@ class CohereASRDummyInputsBuilder(BaseDummyInputsBuilder[CohereASRProcessingInfo
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
mm_processor_kwargs=None,
) -> MultiModalDataDict:
feature_extractor = self.info.get_feature_extractor()
+4 -7
View File
@@ -51,7 +51,7 @@ from transformers.models.cohere_compass.image_processing_cohere_compass import (
from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.distributed import (
get_pp_group,
get_tensor_model_parallel_world_size,
@@ -1238,11 +1238,8 @@ class CohereCompassDummyInputsBuilder(
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
image_overrides = mm_options.get("image")
target_image_width, target_image_height = (
self.info.get_image_size_with_most_features()
)
@@ -1251,8 +1248,8 @@ class CohereCompassDummyInputsBuilder(
"image": self._get_dummy_images(
width=target_image_width,
height=target_image_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
),
}
+4 -6
View File
@@ -15,7 +15,7 @@ from torch import nn
from transformers import BatchFeature
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.pooler.tokwise import pooler_for_token_embed
from vllm.multimodal import MULTIMODAL_REGISTRY
@@ -135,18 +135,16 @@ class ColModernVBertDummyInputsBuilder(
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
target_width, target_height = self.info.get_image_size_with_most_features()
image_overrides = mm_options.get("image")
return {
"image": self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
)
}
+3 -5
View File
@@ -11,7 +11,7 @@ import torch.nn as nn
from transformers import BatchFeature, CLIPVisionConfig
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.models.interfaces import (
MultiModalEmbeddings,
@@ -268,17 +268,15 @@ class DeepseekOCRDummyInputsBuilder(BaseDummyInputsBuilder[DeepseekOCRProcessing
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
max_image_size = self.info.get_image_size_with_most_features()
return {
"image": self._get_dummy_images(
width=max_image_size.width,
height=max_image_size.height,
num_images=num_images,
num_images=mm_counts.get("image", 0),
)
}
+3 -5
View File
@@ -11,7 +11,7 @@ import torch.nn as nn
from transformers import BatchFeature
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.models.interfaces import (
MultiModalEmbeddings,
@@ -140,17 +140,15 @@ class DeepseekOCR2DummyInputsBuilder(
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
max_image_size = self.info.get_image_size_with_most_features()
return {
"image": self._get_dummy_images(
width=max_image_size.width,
height=max_image_size.height,
num_images=num_images,
num_images=mm_counts.get("image", 0),
)
}
+4 -8
View File
@@ -15,7 +15,7 @@ from einops import rearrange, repeat
from transformers import BatchFeature
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -212,20 +212,16 @@ class DeepseekVL2DummyInputsBuilder(BaseDummyInputsBuilder[DeepseekVL2Processing
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
max_image_size = self.info.get_image_size_with_most_features()
image_overrides = mm_options.get("image")
return {
"image": self._get_dummy_images(
width=max_image_size.width,
height=max_image_size.height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
)
}
+4 -8
View File
@@ -9,7 +9,7 @@ from torch.nn import LayerNorm
from transformers.models.qwen2_vl import Qwen2VLProcessor
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.distributed import utils as dist_utils
from vllm.distributed.parallel_state import (
get_tensor_model_parallel_rank,
@@ -104,20 +104,16 @@ class DotsOCRDummyInputsBuilder(Qwen2VLDummyInputsBuilder):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
target_width, target_height = self.info.get_image_size_with_most_features()
image_overrides = mm_options.get("image")
return {
"image": self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
),
}
+6 -13
View File
@@ -36,7 +36,7 @@ from einops import rearrange
from transformers import BaseImageProcessor, BatchFeature
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions, VideoDummyOptions
from vllm.distributed import parallel_state
from vllm.distributed import utils as dist_utils
from vllm.inputs import MultiModalDataDict
@@ -1276,33 +1276,26 @@ class Ernie4_5_VLDummyInputsBuilder(BaseDummyInputsBuilder[Ernie4_5_VLProcessing
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
num_videos = mm_counts.get("video", 0)
target_width, target_height = self.info.get_image_size_with_most_features()
target_num_frames = self.info.get_num_frames_with_most_features(
seq_len, mm_counts
)
image_overrides = mm_options.get("image")
video_overrides = mm_options.get("video")
assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions)
return {
"image": self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
),
"video": self._get_dummy_videos(
width=target_width,
height=target_height,
num_frames=target_num_frames,
num_videos=num_videos,
overrides=video_overrides,
num_videos=mm_counts.get("video", 0),
overrides=mm_options.get("video"),
),
}
+5 -7
View File
@@ -12,7 +12,7 @@ from transformers import (
)
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.config.speech_to_text import SpeechToTextParams
from vllm.inputs import MultiModalDataDict, PromptType
from vllm.logger import init_logger
@@ -207,20 +207,18 @@ class FireRedASR2DummyInputsBuilder(BaseDummyInputsBuilder[FireRedASR2Processing
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
feature_extractor = self.info.get_feature_extractor()
sampling_rate = feature_extractor.sampling_rate
audio_len = feature_extractor.chunk_length * sampling_rate
num_audios = mm_counts.get("audio", 0)
audio_overrides = mm_options.get("audio")
assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions)
ret = {
"audio": self._get_dummy_audios(
length=audio_len, num_audios=num_audios, overrides=audio_overrides
length=audio_len,
num_audios=mm_counts.get("audio", 0),
overrides=mm_options.get("audio"),
)
}
return ret
+4 -8
View File
@@ -14,7 +14,7 @@ from transformers import (
)
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.config.speech_to_text import SpeechToTextParams
from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.inputs import MultiModalDataDict, PromptType
@@ -734,22 +734,18 @@ class FunASRDummyInputsBuilder(BaseDummyInputsBuilder[FunASRProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
feature_extractor = self.info.get_feature_extractor()
sampling_rate = feature_extractor.sampling_rate
audio_len = feature_extractor.chunk_length * sampling_rate
num_audios = mm_counts.get("audio", 0)
audio_overrides = mm_options.get("audio")
assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions)
return {
"audio": self._get_dummy_audios(
length=audio_len,
num_audios=num_audios,
overrides=audio_overrides,
num_audios=mm_counts.get("audio", 0),
overrides=mm_options.get("audio"),
),
}
+4 -7
View File
@@ -26,7 +26,7 @@ from transformers.feature_extraction_utils import BatchFeature
from transformers.modeling_outputs import BaseModelOutput
from vllm.config import VllmConfig
from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.attention.mm_encoder_attention import MMEncoderAttention
from vllm.model_executor.layers.linear import QKVParallelLinear, RowParallelLinear
@@ -580,7 +580,7 @@ class FunAudioChatDummyInputsBuilder(
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
feature_extractor = self.info.get_feature_extractor()
sampling_rate = int(feature_extractor.sampling_rate)
@@ -597,15 +597,12 @@ class FunAudioChatDummyInputsBuilder(
1,
(target_num_frames * sampling_rate + token_fps - 1) // token_fps,
)
num_audios = int(mm_counts.get("audio", 0))
audio_overrides = mm_options.get("audio")
assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions)
return {
"audio": self._get_dummy_audios(
length=audio_len,
num_audios=num_audios,
overrides=audio_overrides,
num_audios=int(mm_counts.get("audio", 0)),
overrides=mm_options.get("audio"),
)
}
+4 -8
View File
@@ -11,7 +11,7 @@ from transformers.models.gemma3.image_processing_gemma3 import Gemma3ImageProces
from transformers.models.gemma3.processing_gemma3 import Gemma3ProcessorKwargs
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.logger import init_logger
from vllm.model_executor.layers.layernorm import GemmaRMSNorm
@@ -244,20 +244,16 @@ class Gemma3DummyInputsBuilder(BaseDummyInputsBuilder[Gemma3ProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
target_width, target_height = self.info.get_image_size_with_most_features()
image_overrides = mm_options.get("image")
return {
"image": self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
)
}
+6 -12
View File
@@ -18,7 +18,7 @@ from transformers.models.gemma3n import (
from transformers.models.siglip import SiglipImageProcessorFast
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.config.speech_to_text import SpeechToTextParams
from vllm.inputs import MultiModalDataDict, PromptType, TextPrompt
from vllm.logger import init_logger
@@ -232,10 +232,8 @@ class Gemma3nDummyInputsBuilder(BaseDummyInputsBuilder[Gemma3nProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
num_audios = mm_counts.get("audio", 0)
processor = self.info.get_hf_processor()
audio_feature_extractor: Gemma3nAudioFeatureExtractor = (
processor.feature_extractor
@@ -245,21 +243,17 @@ class Gemma3nDummyInputsBuilder(BaseDummyInputsBuilder[Gemma3nProcessingInfo]):
img_width = image_processor.size.get("width", 224)
img_height = image_processor.size.get("height", 224)
image_overrides = mm_options.get("image")
audio_overrides = mm_options.get("audio")
assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions)
return {
"image": self._get_dummy_images(
width=img_width,
height=img_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
),
"audio": self._get_dummy_audios(
length=audio_len,
num_audios=num_audios,
overrides=audio_overrides,
num_audios=mm_counts.get("audio", 0),
overrides=mm_options.get("audio"),
),
}
+7 -21
View File
@@ -36,11 +36,7 @@ from transformers.models.gemma4.configuration_gemma4 import (
from vllm.config import VllmConfig
from vllm.config.model import get_served_model_name
from vllm.config.multimodal import (
AudioDummyOptions,
BaseDummyOptions,
VideoDummyOptions,
)
from vllm.config.multimodal import MultiModalDummyOptions, VideoDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.logger import init_logger
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -273,10 +269,7 @@ class Gemma4ProcessingInfo(BaseProcessingInfo):
num_frames = _VIDEO_MAX_FRAMES
mm_config = self.ctx.model_config.get_multimodal_config()
video_opts = mm_config.limit_per_prompt.get("video")
if (
isinstance(video_opts, VideoDummyOptions)
and video_opts.num_frames is not None
):
if video_opts is not None and video_opts.num_frames is not None:
num_frames = min(num_frames, video_opts.num_frames)
tokens["video"] = num_frames * (_VIDEO_MAX_SOFT_TOKENS + 2 + 6)
return tokens
@@ -502,9 +495,8 @@ class Gemma4DummyInputsBuilder(BaseDummyInputsBuilder[Gemma4ProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions] | None = None,
mm_options: MultiModalDummyOptions | None = None,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
num_audios = mm_counts.get("audio", 0)
num_videos = mm_counts.get("video", 0)
processor = self.info.get_hf_processor()
@@ -517,18 +509,12 @@ class Gemma4DummyInputsBuilder(BaseDummyInputsBuilder[Gemma4ProcessingInfo]):
img_width = size.get("width", 224)
img_height = size.get("height", 224)
image_overrides = mm_options.get("image") if mm_options else None
audio_overrides = mm_options.get("audio") if mm_options else None
video_overrides = mm_options.get("video") if mm_options else None
assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions)
assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions)
data: dict[str, Any] = {
"image": self._get_dummy_images(
width=img_width,
height=img_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image") if mm_options else None,
),
}
@@ -537,7 +523,7 @@ class Gemma4DummyInputsBuilder(BaseDummyInputsBuilder[Gemma4ProcessingInfo]):
data["audio"] = self._get_dummy_audios(
length=audio_len,
num_audios=num_audios,
overrides=audio_overrides,
overrides=mm_options.get("audio") if mm_options else None,
)
if num_videos > 0:
@@ -546,7 +532,7 @@ class Gemma4DummyInputsBuilder(BaseDummyInputsBuilder[Gemma4ProcessingInfo]):
height=img_height,
num_frames=_VIDEO_MAX_FRAMES,
num_videos=num_videos,
overrides=video_overrides,
overrides=mm_options.get("video") if mm_options else None,
)
return data
+1 -5
View File
@@ -28,7 +28,6 @@ from transformers.models.gemma4_unified.processing_gemma4_unified import (
)
from vllm.config import VllmConfig
from vllm.config.multimodal import VideoDummyOptions
from vllm.model_executor.layers.linear import ColumnParallelLinear
from vllm.model_executor.models.gemma3n_mm import batch_audio_features
from vllm.model_executor.models.gemma4 import Gemma4ForCausalLM
@@ -177,10 +176,7 @@ class Gemma4UnifiedProcessingInfo(Gemma4ProcessingInfo):
num_frames = _VIDEO_MAX_FRAMES
mm_config = self.ctx.model_config.get_multimodal_config()
video_opts = mm_config.limit_per_prompt.get("video")
if (
isinstance(video_opts, VideoDummyOptions)
and video_opts.num_frames is not None
):
if video_opts is not None and video_opts.num_frames is not None:
num_frames = min(num_frames, video_opts.num_frames)
tokens["video"] = num_frames * (_VIDEO_MAX_SOFT_TOKENS + 2 + 6)
return tokens
+6 -13
View File
@@ -52,7 +52,7 @@ from transformers.video_processing_utils import BaseVideoProcessor
from transformers.video_utils import VideoMetadata
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions, VideoDummyOptions
from vllm.distributed import get_tensor_model_parallel_world_size, parallel_state
from vllm.distributed import utils as dist_utils
from vllm.inputs import MultiModalDataDict
@@ -1455,33 +1455,26 @@ class Glm4vDummyInputsBuilder(BaseDummyInputsBuilder[Glm4vProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
num_videos = mm_counts.get("video", 0)
target_width, target_height = self.info.get_image_size_with_most_features()
target_num_frames = self.info.get_num_frames_with_most_features(
seq_len, mm_counts
)
image_overrides = mm_options.get("image")
video_overrides = mm_options.get("video")
assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions)
return {
"image": self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
),
"video": self._get_dummy_videos(
width=target_width,
height=target_height,
num_frames=target_num_frames,
num_videos=num_videos,
overrides=video_overrides,
num_videos=mm_counts.get("video", 0),
overrides=mm_options.get("video"),
),
}
+4 -7
View File
@@ -16,7 +16,7 @@ from torch.nn import LayerNorm
from transformers import BatchFeature
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.activation import SiluAndMul, get_act_fn
@@ -445,22 +445,19 @@ class GLM4VDummyInputsBuilder(BaseDummyInputsBuilder[GLM4VProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
hf_config = self.info.get_hf_config()
vision_config = hf_config.vision_config
target_width = target_height = vision_config["image_size"]
num_images = mm_counts.get("image", 0)
image_overrides = mm_options.get("image")
return {
"image": self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
)
}
+4 -7
View File
@@ -11,7 +11,7 @@ from transformers.models.glmasr import GlmAsrConfig, GlmAsrProcessor
from transformers.models.whisper import WhisperFeatureExtractor
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.config.speech_to_text import SpeechToTextParams
from vllm.distributed.parallel_state import get_tensor_model_parallel_world_size
from vllm.inputs import ModalityData, MultiModalDataDict, PromptType, TokensPrompt
@@ -694,13 +694,10 @@ class GlmAsrDummyInputsBuilder(BaseDummyInputsBuilder[GlmAsrProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
feature_extractor = self.info.get_feature_extractor()
sampling_rate = feature_extractor.sampling_rate
num_audios = mm_counts.get("audio", 0)
audio_overrides = mm_options.get("audio")
assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions)
max_audio_len = getattr(
self.info.get_hf_processor(), "max_audio_len", DEFAULT_MAX_AUDIO_LEN_S
@@ -710,8 +707,8 @@ class GlmAsrDummyInputsBuilder(BaseDummyInputsBuilder[GlmAsrProcessingInfo]):
return {
"audio": self._get_dummy_audios(
length=audio_len,
num_audios=num_audios,
overrides=audio_overrides,
num_audios=mm_counts.get("audio", 0),
overrides=mm_options.get("audio"),
)
}
+4 -8
View File
@@ -34,7 +34,7 @@ from torch import nn
from transformers import BatchFeature, PretrainedConfig
from vllm.config import CacheConfig, ModelConfig, SpeechToTextConfig, VllmConfig
from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.config.speech_to_text import SpeechToTextParams
from vllm.inputs import MultiModalDataDict, PromptType, TokensPrompt
from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
@@ -208,17 +208,13 @@ class GraniteSpeechDummyInputsBuilder(
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_audios = mm_counts.get("audio", 0)
audio_overrides = mm_options.get("audio")
assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions)
return {
"audio": self._get_dummy_audios(
length=self.info.get_max_audio_len(),
num_audios=num_audios,
overrides=audio_overrides,
num_audios=mm_counts.get("audio", 0),
overrides=mm_options.get("audio"),
)
}
@@ -18,7 +18,7 @@ import torch.nn as nn
from transformers import BatchFeature
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.forward_context import set_forward_context
from vllm.inputs import MultiModalDataDict
from vllm.multimodal import MULTIMODAL_REGISTRY
@@ -193,7 +193,7 @@ class HCXVisionV2DummyInputsBuilder(BaseDummyInputsBuilder[HCXVisionV2Processing
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions] | None = None,
mm_options: MultiModalDummyOptions | None = None,
mm_processor_kwargs: Mapping[str, object] | None = None,
) -> ProcessorInputs:
"""Build dummy processor inputs for memory profiling."""
@@ -225,31 +225,25 @@ class HCXVisionV2DummyInputsBuilder(BaseDummyInputsBuilder[HCXVisionV2Processing
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions] | None = None,
mm_options: MultiModalDummyOptions | None = None,
mm_processor_kwargs: Mapping[str, object] | None = None,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
num_videos = mm_counts.get("video", 0)
target_width, target_height = self.info.get_image_size_with_most_features()
target_num_frames = 16 # Default for video
image_overrides = mm_options.get("image") if mm_options else None
video_overrides = mm_options.get("video") if mm_options else None
result: MultiModalDataDict = {
"image": self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides, # type: ignore
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image") if mm_options else None,
),
"video": self._get_dummy_videos(
width=target_width,
height=target_height,
num_frames=target_num_frames,
num_videos=num_videos,
overrides=video_overrides, # type: ignore
num_videos=mm_counts.get("video", 0),
overrides=mm_options.get("video") if mm_options else None,
),
}
+4 -7
View File
@@ -29,7 +29,7 @@ from transformers import (
)
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.layers.logits_processor import LogitsProcessor
@@ -221,21 +221,18 @@ class Idefics3DummyInputsBuilder(BaseDummyInputsBuilder[Idefics3ProcessingInfo])
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
hf_processor = self.info.get_hf_processor()
image_processor: Idefics3ImageProcessor = hf_processor.image_processor
longest_edge = image_processor.max_image_size["longest_edge"]
image_overrides = mm_options.get("image")
return {
"image": self._get_dummy_images(
width=longest_edge,
height=longest_edge,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
)
}
+6 -12
View File
@@ -22,7 +22,7 @@ from transformers.models.internvl.video_processing_internvl import (
)
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.models.interns1_vit import InternS1VisionModel
@@ -299,35 +299,29 @@ class InternS1DummyInputsBuilder(BaseDummyInputsBuilder[InternS1ProcessingInfo])
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
target_width, target_height = self.info.get_image_size_with_most_features()
target_num_frames = self.info.get_num_frames_with_most_features(
seq_len, mm_counts
)
num_images = mm_counts.get("image", 0)
num_videos = mm_counts.get("video", 0)
config = self.info.get_hf_config()
image_size_h, image_size_w = config.vision_config.image_size
image_overrides = mm_options.get("image")
video_overrides = mm_options.get("video")
assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions)
return {
"image": self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
),
"video": self._get_dummy_videos(
width=image_size_w,
height=image_size_h,
num_frames=target_num_frames,
num_videos=num_videos,
overrides=video_overrides,
num_videos=mm_counts.get("video", 0),
overrides=mm_options.get("video"),
),
}
+5 -11
View File
@@ -17,7 +17,7 @@ import torch.nn as nn
from transformers import BatchFeature, PretrainedConfig
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
@@ -203,19 +203,16 @@ class BaseInternVLDummyInputsBuilder(BaseDummyInputsBuilder[_I]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
target_width, target_height = self.info.get_image_size_with_most_features()
num_images = mm_counts.get("image", 0)
image_overrides = mm_options.get("image")
return {
"image": self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
)
}
@@ -425,7 +422,7 @@ class InternVLDummyInputsBuilder(
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
dummy_image = super().get_dummy_mm_data(seq_len, mm_counts, mm_options)
if self.info.ctx_video_token:
@@ -436,9 +433,6 @@ class InternVLDummyInputsBuilder(
)
num_videos = mm_counts.get("video", 0)
video_overrides = mm_options.get("video")
assert video_overrides is None or isinstance(
video_overrides, VideoDummyOptions
)
dummy_video = {
"video": self._get_dummy_videos(
width=image_size,
+4 -7
View File
@@ -13,7 +13,7 @@ from einops import rearrange
from transformers.image_processing_utils import BatchFeature
from vllm.config import ModelConfig, VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.distributed import parallel_state
from vllm.distributed import utils as dist_utils
from vllm.inputs import MultiModalDataDict
@@ -386,19 +386,16 @@ class IsaacDummyInputsBuilder(BaseDummyInputsBuilder[IsaacProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
target_width, target_height = self.info.get_image_size_with_most_features()
image_overrides = mm_options.get("image")
return {
"image": self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
),
}
+2 -2
View File
@@ -18,7 +18,7 @@ from transformers.modeling_outputs import BaseModelOutput
from transformers.models.qwen2_vl.configuration_qwen2_vl import Qwen2VLVisionConfig
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.logger import init_logger
from vllm.multimodal import MULTIMODAL_REGISTRY
@@ -442,7 +442,7 @@ class KananaVDummyInputsBuilder(BaseDummyInputsBuilder[KananaVProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
return {
+6 -12
View File
@@ -17,7 +17,7 @@ from transformers.modeling_outputs import BaseModelOutput, BaseModelOutputWithPo
from transformers.utils import torch_int
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.inputs import ModalityData, MultiModalDataDict
from vllm.logger import init_logger
@@ -1124,30 +1124,24 @@ class KeyeBaseDummyInputsBuilder(BaseDummyInputsBuilder[_I]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
num_videos = mm_counts.get("video", 0)
target_width, target_height = self.info.get_image_size_with_most_features()
target_num_frames = self.info.get_num_frames_with_most_features(seq_len)
image_overrides = mm_options.get("image")
video_overrides = mm_options.get("video")
mm_data = {
"image": self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
),
"video": self._get_dummy_videos(
width=target_width,
height=target_height,
num_frames=target_num_frames,
num_videos=num_videos,
overrides=video_overrides,
num_videos=mm_counts.get("video", 0),
overrides=mm_options.get("video"),
),
}
+2 -2
View File
@@ -13,7 +13,7 @@ from transformers import BatchFeature
from transformers import WhisperConfig as HFWhisperConfig
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.config.speech_to_text import SpeechToTextParams
from vllm.inputs import PromptType, TokensPrompt
from vllm.model_executor.model_loader import DefaultModelLoader
@@ -180,7 +180,7 @@ class KimiAudioDummyInputsBuilder(BaseDummyInputsBuilder[KimiAudioProcessingInfo
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> ProcessorInputs:
dummy_mm_data = self.get_dummy_mm_data(seq_len, mm_counts, mm_options)
dummy_mm_items = self.info.parse_mm_data(dummy_mm_data)
+2 -2
View File
@@ -14,7 +14,7 @@ from torch import nn
from transformers import BatchFeature
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -216,7 +216,7 @@ class KimiK25DummyInputsBuilder(BaseDummyInputsBuilder[KimiK25ProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
# TODO: Support mm_options for vision_chunk to allow user configuration
dummy_items = self.get_dummy_mm_items()
+4 -8
View File
@@ -53,7 +53,7 @@ from transformers import BatchFeature
from transformers.activations import GELUActivation
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.models.interfaces import (
@@ -219,18 +219,14 @@ class KimiVLDummyInputsBuilder(BaseDummyInputsBuilder[KimiVLProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
image_overrides = mm_options.get("image")
return {
"image": self._get_dummy_images(
width=MaxImageTokenMeta.width,
height=MaxImageTokenMeta.height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
)
}
+4 -8
View File
@@ -21,7 +21,7 @@ from transformers.models.lfm2_vl.image_processing_lfm2_vl_fast import (
from typing_extensions import Buffer
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.forward_context import set_forward_context
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.linear import ReplicatedLinear
@@ -387,20 +387,16 @@ class Lfm2VLDummyInputsBuilder(BaseDummyInputsBuilder[Lfm2VLProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
target_width, target_height = self.info.get_image_size_with_most_features()
image_overrides = mm_options.get("image")
return {
"image": self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
),
}
+4 -8
View File
@@ -19,7 +19,7 @@ from transformers.models.llava import LlavaProcessor
from transformers.models.pixtral import PixtralProcessor
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.activation import get_act_fn
from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
@@ -228,20 +228,16 @@ class LlavaDummyInputsBuilder(BaseDummyInputsBuilder[_I]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
target_width, target_height = self.info.get_image_size_with_most_features()
image_overrides = mm_options.get("image")
return {
"image": self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
)
}
@@ -10,7 +10,7 @@ import torch.nn as nn
from transformers import BatchFeature, LlavaNextVideoConfig, LlavaNextVideoProcessor
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.activation import get_act_fn
from vllm.model_executor.models.clip import CLIPVisionModel
@@ -171,25 +171,20 @@ class LlavaNextVideoDummyInputsBuilder(
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_videos = mm_counts.get("video", 0)
target_width, target_height = self.info.get_image_size_with_most_features()
target_num_frames = self.info.get_num_frames_with_most_features(
seq_len, mm_counts
)
video_overrides = mm_options.get("video")
assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions)
return {
"video": self._get_dummy_videos(
width=target_width,
height=target_height,
num_frames=target_num_frames,
num_videos=num_videos,
overrides=video_overrides,
num_videos=mm_counts.get("video", 0),
overrides=mm_options.get("video"),
)
}
+6 -13
View File
@@ -14,7 +14,7 @@ from transformers.models.llava_onevision.modeling_llava_onevision import (
)
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.activation import get_act_fn
from vllm.multimodal import MULTIMODAL_REGISTRY
@@ -278,33 +278,26 @@ class LlavaOnevisionDummyInputsBuilder(
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
num_videos = mm_counts.get("video", 0)
target_width, target_height = self.info.get_image_size_with_most_features()
target_num_frames = self.info.get_num_frames_with_most_features(
seq_len, mm_counts
)
image_overrides = mm_options.get("image")
video_overrides = mm_options.get("video")
assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions)
return {
"image": self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
),
"video": self._get_dummy_videos(
width=target_width,
height=target_height,
num_frames=target_num_frames,
num_videos=num_videos,
overrides=video_overrides,
num_videos=mm_counts.get("video", 0),
overrides=mm_options.get("video"),
),
}
@@ -54,7 +54,7 @@ from vllm.compilation.decorators import (
support_torch_compile,
)
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.distributed import parallel_state
from vllm.distributed import utils as dist_utils
from vllm.inputs import ModalityData, MultiModalDataDict
@@ -1471,7 +1471,7 @@ class LlavaOnevision2DummyInputsBuilder(
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions] | None = None,
mm_options: MultiModalDummyOptions | None = None,
) -> MultiModalDataDict:
n_img = mm_counts.get("image", 0)
n_vid = mm_counts.get("video", 0)
+4 -9
View File
@@ -36,7 +36,7 @@ from torch.nn.functional import scaled_dot_product_attention
from transformers import BatchFeature
from vllm.config import VllmConfig
from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.activation import get_act_fn
@@ -564,18 +564,13 @@ class MiDashengLMDummyInputsBuilder(BaseDummyInputsBuilder[MiDashengLMProcessing
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_audios = mm_counts.get("audio", 0)
audio_overrides = mm_options.get("audio")
assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions)
return {
"audio": self._get_dummy_audios(
length=self.info.get_max_audio_len(),
num_audios=num_audios,
overrides=audio_overrides,
num_audios=mm_counts.get("audio", 0),
overrides=mm_options.get("audio"),
)
}
+6 -17
View File
@@ -15,11 +15,7 @@ from transformers.models.qwen2_vl.image_processing_qwen2_vl import smart_resize
from typing_extensions import TypedDict
from vllm.config import VllmConfig
from vllm.config.multimodal import (
BaseDummyOptions,
ImageDummyOptions,
VideoDummyOptions,
)
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.distributed import parallel_state
from vllm.distributed import utils as dist_utils
from vllm.inputs import MultiModalDataDict
@@ -1184,33 +1180,26 @@ class MiMoV2OmniDummyInputsBuilder(BaseDummyInputsBuilder[MiMoV2OmniProcessingIn
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
num_videos = mm_counts.get("video", 0)
target_width, target_height = self.info.get_image_size_with_most_features()
target_num_frames = self.info.get_num_frames_with_most_features(
seq_len, mm_counts
)
image_overrides = mm_options.get("image")
video_overrides = mm_options.get("video")
assert image_overrides is None or isinstance(image_overrides, ImageDummyOptions)
assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions)
return {
"image": self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
),
"video": self._get_dummy_videos(
width=target_width,
height=target_height,
num_frames=target_num_frames,
num_videos=num_videos,
overrides=video_overrides,
num_videos=mm_counts.get("video", 0),
overrides=mm_options.get("video"),
),
}
+4 -8
View File
@@ -40,7 +40,7 @@ from transformers.models.whisper.modeling_whisper import (
)
from vllm.config import VllmConfig
from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import ModalityData, MultiModalDataDict
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargsItems
from vllm.multimodal.inputs import (
@@ -402,22 +402,18 @@ class MiniCPMODummyInputsBuilder(MiniCPMVDummyInputsBuilder[MiniCPMOProcessingIn
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_audios = mm_counts.get("audio", 0)
audio_len = (
self.info.get_max_audio_chunks_with_most_features()
* self.info.get_default_audio_sampling_rate()
)
audio_overrides = mm_options.get("audio")
assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions)
audio_mm_data = {
"audio": self._get_dummy_audios(
length=audio_len,
num_audios=num_audios,
overrides=audio_overrides,
num_audios=mm_counts.get("audio", 0),
overrides=mm_options.get("audio"),
)
}
+5 -9
View File
@@ -45,9 +45,8 @@ from typing_extensions import TypedDict, TypeVar
from vllm.config import VllmConfig
from vllm.config.multimodal import (
BaseDummyOptions,
ImageDummyOptions,
VideoDummyOptions,
MultiModalDummyOptions,
)
from vllm.inputs import ModalityData, MultiModalDataDict
from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -848,9 +847,8 @@ class MiniCPMVDummyInputsBuilder(BaseDummyInputsBuilder[_I]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
num_videos = mm_counts.get("video", 0)
image_width, image_height = self.info.get_image_size_with_most_features()
@@ -859,14 +857,12 @@ class MiniCPMVDummyInputsBuilder(BaseDummyInputsBuilder[_I]):
seq_len, mm_counts
)
image_overrides = mm_options.get("image")
video_overrides = mm_options.get("video")
assert image_overrides is None or isinstance(image_overrides, ImageDummyOptions)
# Convert video overrides to image overrides for per-frame image generation,
# and apply num_frames override to num_video_frames.
video_frame_overrides: ImageDummyOptions | None = None
if isinstance(video_overrides, VideoDummyOptions):
if video_overrides is not None:
if video_overrides.num_frames:
num_video_frames = min(num_video_frames, video_overrides.num_frames)
if video_overrides.width or video_overrides.height:
@@ -880,8 +876,8 @@ class MiniCPMVDummyInputsBuilder(BaseDummyInputsBuilder[_I]):
"image": self._get_dummy_images(
width=image_width,
height=image_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
),
"video": [
self._get_dummy_images(
+4 -8
View File
@@ -11,7 +11,7 @@ from transformers import BatchFeature, Mistral3Config, PixtralVisionConfig
from transformers.models.pixtral import PixtralProcessor
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.activation import get_act_fn
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -261,20 +261,16 @@ class Mistral3DummyInputsBuilder(BaseDummyInputsBuilder):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
target_width, target_height = self.info.get_image_size_with_most_features()
image_overrides = mm_options.get("image")
return {
"image": self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
)
}
+4 -8
View File
@@ -36,7 +36,7 @@ from vllm.compilation.decorators import (
support_torch_compile,
)
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.attention import MMEncoderAttention
@@ -709,20 +709,16 @@ class Mllama4DummyInputsBuilder(BaseDummyInputsBuilder[Mllama4ProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
(target_width, target_height) = self.info.get_image_size_with_most_features()
image_overrides = mm_options.get("image")
return {
"image": self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
)
}
+4 -7
View File
@@ -21,7 +21,7 @@ from transformers import (
from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.distributed import (
get_pp_group,
get_tensor_model_parallel_rank,
@@ -1104,19 +1104,16 @@ class MolmoDummyInputsBuilder(BaseDummyInputsBuilder[MolmoProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
target_width, target_height = self.info.get_image_size_with_most_features()
num_images = mm_counts.get("image", 0)
image_overrides = mm_options.get("image")
return {
"image": self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
)
}
+3 -6
View File
@@ -26,7 +26,7 @@ from typing_extensions import TypedDict
from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, VllmConfig
from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions, VideoDummyOptions
from vllm.distributed import (
get_pp_group,
get_tensor_model_parallel_rank,
@@ -1834,7 +1834,7 @@ class Molmo2DummyInputsBuilder(BaseDummyInputsBuilder[Molmo2ProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
num_videos = mm_counts.get("video", 0)
@@ -1845,13 +1845,11 @@ class Molmo2DummyInputsBuilder(BaseDummyInputsBuilder[Molmo2ProcessingInfo]):
if num_images > 0:
target_width, target_height = self.info.get_image_size_with_most_features()
image_overrides = mm_options.get("image")
dummy_images = self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides,
overrides=mm_options.get("image"),
)
if num_videos > 0:
@@ -1864,7 +1862,6 @@ class Molmo2DummyInputsBuilder(BaseDummyInputsBuilder[Molmo2ProcessingInfo]):
video_overrides = mm_options.get("video")
if video_overrides:
assert isinstance(video_overrides, VideoDummyOptions)
num_frames_override = video_overrides.num_frames
if num_frames_override:
if num_frames_override > target_num_frames:
+3 -4
View File
@@ -13,7 +13,7 @@ import torch.nn.functional as F
from transformers import BatchFeature
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.distributed import (
get_pp_group,
get_tensor_model_parallel_rank,
@@ -943,15 +943,14 @@ class Moondream3DummyInputsBuilder(BaseDummyInputsBuilder[Moondream3ProcessingIn
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions] | None = None,
mm_options: MultiModalDummyOptions | None = None,
mm_processor_kwargs: Mapping[str, object] | None = None,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
return {
"image": self._get_dummy_images(
width=378,
height=378,
num_images=num_images,
num_images=mm_counts.get("image", 0),
)
}
+4 -7
View File
@@ -19,7 +19,7 @@ from typing_extensions import TypedDict
from vllm.compilation.decorators import support_torch_compile
from vllm.config import VllmConfig
from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.distributed import (
get_pp_group,
get_tensor_model_parallel_world_size,
@@ -1341,16 +1341,13 @@ class MossAudioDummyInputsBuilder(BaseDummyInputsBuilder[MossAudioProcessingInfo
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_audios = mm_counts.get("audio", 0)
audio_overrides = mm_options.get("audio")
assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions)
return {
"audio": self._get_dummy_audios(
length=16000,
num_audios=num_audios,
overrides=audio_overrides,
num_audios=mm_counts.get("audio", 0),
overrides=mm_options.get("audio"),
)
}
@@ -20,7 +20,7 @@ from torch import nn
from transformers import BatchFeature
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.config.speech_to_text import SpeechToTextParams
from vllm.inputs import ModalityData, MultiModalDataDict, PromptType, TextPrompt
from vllm.model_executor.models.interfaces import (
@@ -423,20 +423,18 @@ class MossTranscribeDiarizeDummyInputsBuilder(
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_audios = mm_counts.get("audio", 0)
if num_audios == 0:
return {}
feature_extractor = self.info.get_feature_extractor()
audio_overrides = mm_options.get("audio")
assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions)
return {
"audio": self._get_dummy_audios(
length=_get_max_audio_samples(feature_extractor),
num_audios=num_audios,
overrides=audio_overrides,
overrides=mm_options.get("audio"),
)
}
@@ -444,7 +442,7 @@ class MossTranscribeDiarizeDummyInputsBuilder(
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> ProcessorInputs:
dummy_mm_data = self.get_dummy_mm_data(seq_len, mm_counts, mm_options)
dummy_mm_items = self.info.parse_mm_data(dummy_mm_data)
+4 -12
View File
@@ -45,11 +45,7 @@ from transformers import BatchFeature
from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, VllmConfig
from vllm.config.multimodal import (
BaseDummyOptions,
ImageDummyOptions,
VideoDummyOptions,
)
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.distributed import (
divide,
get_pp_group,
@@ -234,7 +230,7 @@ class MuseGlimmerDummyInputsBuilder(BaseDummyInputsBuilder[MuseGlimmerProcessing
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
processor = self.info.get_hf_processor()
video_processor = processor.video_processor
@@ -245,16 +241,12 @@ class MuseGlimmerDummyInputsBuilder(BaseDummyInputsBuilder[MuseGlimmerProcessing
* int(video_processor.downsample_factor)
* video_grid
)
image_overrides = mm_options.get("image")
video_overrides = mm_options.get("video")
assert image_overrides is None or isinstance(image_overrides, ImageDummyOptions)
assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions)
return {
"image": self._get_dummy_images(
width=image_width,
height=image_height,
num_images=mm_counts.get("image", 0),
overrides=image_overrides,
overrides=mm_options.get("image"),
),
"video": self._get_dummy_videos(
width=video_size,
@@ -263,7 +255,7 @@ class MuseGlimmerDummyInputsBuilder(BaseDummyInputsBuilder[MuseGlimmerProcessing
seq_len, mm_counts
),
num_videos=mm_counts.get("video", 0),
overrides=video_overrides,
overrides=mm_options.get("video"),
),
}
+4 -14
View File
@@ -21,8 +21,7 @@ from transformers import BatchFeature, PretrainedConfig
from vllm import envs
from vllm.config import VllmConfig
from vllm.config.multimodal import (
AudioDummyOptions,
BaseDummyOptions,
MultiModalDummyOptions,
VideoDummyOptions,
)
from vllm.inputs import MultiModalDataDict, MultiModalInput
@@ -839,22 +838,19 @@ class NanoNemotronVLDummyInputsBuilder(
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
(target_width, target_height), _ = (
self.info.get_dummy_image_size_and_max_tokens(mm_counts)
)
processor = self.info.get_hf_processor()
image_overrides = mm_options.get("image")
dummy_image = {
"image": self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
)
}
@@ -889,9 +885,6 @@ class NanoNemotronVLDummyInputsBuilder(
target_num_frames = num_frames
num_videos = mm_counts.get("video", 0)
video_overrides = mm_options.get("video")
assert video_overrides is None or isinstance(
video_overrides, VideoDummyOptions
)
dummy_video = {
"video": self._get_dummy_videos(
width=video_width,
@@ -907,9 +900,6 @@ class NanoNemotronVLDummyInputsBuilder(
if sound_config := self.info.sound_config:
num_audios = mm_counts.get("audio", 0)
audio_overrides = mm_options.get("audio") if mm_options else None
assert audio_overrides is None or isinstance(
audio_overrides, AudioDummyOptions
)
tokens_per_audio = max(1, seq_len // max(num_audios, 1))
max_audio_num_samples = MAX_AUDIO_LEN_S * sound_config.sampling_rate
calculated_max_audio_num_samples = ParakeetExtractor.audio_length(
+2 -2
View File
@@ -22,7 +22,7 @@ from transformers import (
from vllm.config import CacheConfig, VllmConfig
from vllm.config.lora import LoRAConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.logger import init_logger
from vllm.model_executor.layers.activation import get_act_fn
@@ -388,7 +388,7 @@ class NemotronParseDummyInputsBuilder(
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
+4 -7
View File
@@ -13,7 +13,7 @@ import torch
import torch.nn as nn
from transformers import PretrainedConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.multimodal import MULTIMODAL_REGISTRY
@@ -83,19 +83,16 @@ class NVLMDummyInputsBuilder(BaseInternVLDummyInputsBuilder[NVLMProcessingInfo])
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
target_width, target_height = self.info.get_image_size_with_most_features()
num_images = mm_counts.get("image", 0)
image_overrides = mm_options.get("image")
return {
"image": self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
)
}
+4 -6
View File
@@ -9,7 +9,7 @@ import torch.nn as nn
from transformers import BatchFeature
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.activation import get_act_fn
from vllm.model_executor.layers.linear import (
@@ -299,18 +299,16 @@ class OpenVLADummyInputsBuilder(BaseDummyInputsBuilder[OpenVLAProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
image_overrides = mm_options.get("image")
image_size = self.info.get_image_size_with_most_features()
return {
"image": self._get_dummy_images(
width=image_size.width,
height=image_size.height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
)
}
+4 -8
View File
@@ -29,7 +29,7 @@ from torch.nn.functional import gumbel_softmax, pad, softmax
from transformers import BatchFeature, PretrainedConfig
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -311,20 +311,16 @@ class OvisDummyInputsBuilder(BaseDummyInputsBuilder[OvisProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
target_width, target_height = self.info.get_image_size_with_most_features()
image_overrides = mm_options.get("image")
mm_data = {
"image": self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
),
}
return mm_data
+6 -13
View File
@@ -11,7 +11,7 @@ import torch.nn as nn
from transformers import BaseImageProcessor, BatchFeature, PretrainedConfig
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -290,33 +290,26 @@ class Ovis2_5DummyInputsBuilder(BaseDummyInputsBuilder[Ovis2_5ProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
num_videos = mm_counts.get("video", 0)
target_width, target_height = self.info.get_image_size_with_most_features()
target_num_frames = self.info.get_num_frames_with_most_features(
seq_len, mm_counts
)
image_overrides = mm_options.get("image")
video_overrides = mm_options.get("video")
assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions)
mm_data = {
"image": self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
),
"video": self._get_dummy_videos(
width=target_width,
height=target_height,
num_frames=target_num_frames,
num_videos=num_videos,
overrides=video_overrides,
num_videos=mm_counts.get("video", 0),
overrides=mm_options.get("video"),
),
}
return mm_data
+4 -7
View File
@@ -32,7 +32,7 @@ from transformers.modeling_outputs import (
from transformers.utils import torch_int
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.distributed import parallel_state
from vllm.distributed import utils as dist_utils
from vllm.inputs import MultiModalDataDict
@@ -223,19 +223,16 @@ class PaddleOCRVLDummyInputsBuilder(BaseDummyInputsBuilder[PaddleOCRVLProcessing
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
max_image_size = self.info.get_image_size_with_most_features()
image_overrides = mm_options.get("image")
return {
"image": self._get_dummy_images(
width=max_image_size.width,
height=max_image_size.height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
)
}
+4 -8
View File
@@ -8,7 +8,7 @@ from torch import nn
from transformers import BatchFeature, PaliGemmaConfig
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict, MultiModalInput
from vllm.logger import init_logger
from vllm.multimodal import MULTIMODAL_REGISTRY
@@ -130,22 +130,18 @@ class PaliGemmaDummyInputsBuilder(BaseDummyInputsBuilder[PaliGemmaProcessingInfo
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
hf_config = self.info.get_hf_config()
vision_config = hf_config.vision_config
max_image_size = vision_config.image_size
num_images = mm_counts.get("image", 0)
image_overrides = mm_options.get("image")
return {
"image": self._get_dummy_images(
width=max_image_size,
height=max_image_size,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
)
}
+4 -8
View File
@@ -29,7 +29,7 @@ from transformers import (
)
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -371,20 +371,16 @@ class Phi3VDummyInputsBuilder(BaseDummyInputsBuilder[Phi3VProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
target_width, target_height = self.info.get_image_size_with_most_features()
image_overrides = mm_options.get("image")
return {
"image": self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
)
}
+6 -13
View File
@@ -17,7 +17,7 @@ from transformers import (
)
from vllm.config import VllmConfig
from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.distributed import get_pp_group
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.logits_processor import LogitsProcessor
@@ -839,28 +839,21 @@ class Phi4MMDummyInputsBuilder(BaseDummyInputsBuilder[Phi4MMProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_audios = mm_counts.get("audio", 0)
num_images = mm_counts.get("image", 0)
target_width, target_height = self.info.get_image_size_with_most_features()
image_overrides = mm_options.get("image")
audio_overrides = mm_options.get("audio")
assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions)
mm_data = {
"image": self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
),
"audio": self._get_dummy_audios(
length=_AUDIO_MAX_SOUNDFILE_SIZE,
num_audios=num_audios,
overrides=audio_overrides,
num_audios=mm_counts.get("audio", 0),
overrides=mm_options.get("audio"),
),
}
+3 -4
View File
@@ -14,7 +14,7 @@ import torch.nn as nn
from transformers import BatchFeature, PretrainedConfig, Siglip2VisionConfig
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.logger import init_logger
from vllm.multimodal import MULTIMODAL_REGISTRY
@@ -126,15 +126,14 @@ class Phi4SiglipDummyInputsBuilder(
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
size = self.info.get_image_size_with_most_features()
return {
"image": self._get_dummy_images(
width=size.width,
height=size.height,
num_images=num_images,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
),
}
+5 -9
View File
@@ -21,7 +21,7 @@ from transformers.models.pixtral.image_processing_pixtral import (
from transformers.models.pixtral.modeling_pixtral import apply_rotary_pos_emb
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.distributed import divide, get_tensor_model_parallel_world_size
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.activation import SiluAndMul, get_act_and_mul_fn
@@ -208,20 +208,16 @@ class PixtralDummyInputsBuilder(BaseDummyInputsBuilder[PixtralProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
target_width, target_height = self.info.get_image_size_with_most_features()
image_overrides = mm_options.get("image")
return {
"image": self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
)
}
@@ -229,7 +225,7 @@ class PixtralDummyInputsBuilder(BaseDummyInputsBuilder[PixtralProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
mm_data: MultiModalDataDict | None = None,
) -> ProcessorInputs:
tokenizer = self.info.get_tokenizer()
@@ -44,7 +44,7 @@ from transformers.models.qwen2_5_omni.processing_qwen2_5_omni import (
from transformers.models.whisper import WhisperFeatureExtractor
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.forward_context import set_forward_context
from vllm.inputs import ModalityData, MultiModalDataDict
from vllm.logger import init_logger
@@ -448,12 +448,8 @@ class Qwen2_5OmniThinkerDummyInputsBuilder(
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_audios = mm_counts.get("audio", 0)
num_images = mm_counts.get("image", 0)
num_videos = mm_counts.get("video", 0)
feature_extractor = self.info.get_feature_extractor()
target_audio_length = (
@@ -469,28 +465,24 @@ class Qwen2_5OmniThinkerDummyInputsBuilder(
seq_len, mm_counts
)
image_overrides = mm_options.get("image")
video_overrides = mm_options.get("video")
audio_overrides = mm_options.get("audio")
mm_data = {
"audio": self._get_dummy_audios(
length=target_audio_length,
num_audios=num_audios,
overrides=audio_overrides,
num_audios=mm_counts.get("audio", 0),
overrides=mm_options.get("audio"),
),
"image": self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
),
"video": self._get_dummy_videos(
width=target_width,
height=target_height,
num_frames=target_num_frames,
num_videos=num_videos,
overrides=video_overrides,
num_videos=mm_counts.get("video", 0),
overrides=mm_options.get("video"),
),
}
+4 -7
View File
@@ -37,7 +37,7 @@ from transformers.models.qwen2_audio import (
from transformers.models.whisper import WhisperFeatureExtractor
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import ModalityData, MultiModalDataDict
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import (
@@ -214,21 +214,18 @@ class Qwen2AudioDummyInputsBuilder(BaseDummyInputsBuilder[Qwen2AudioProcessingIn
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
feature_extractor = self.info.get_feature_extractor()
sampling_rate = feature_extractor.sampling_rate
audio_len = feature_extractor.chunk_length * sampling_rate
num_audios = mm_counts.get("audio", 0)
audio_overrides = mm_options.get("audio")
return {
"audio": self._get_dummy_audios(
length=audio_len,
num_audios=num_audios,
overrides=audio_overrides,
num_audios=mm_counts.get("audio", 0),
overrides=mm_options.get("audio"),
)
}
+6 -12
View File
@@ -43,7 +43,7 @@ from transformers.models.qwen2_vl.configuration_qwen2_vl import (
from transformers.models.qwen2_vl.image_processing_qwen2_vl import smart_resize
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.distributed import parallel_state, tensor_model_parallel_all_gather
from vllm.distributed import utils as dist_utils
from vllm.inputs import ModalityData, MultiModalDataDict
@@ -1096,32 +1096,26 @@ class Qwen2VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen2VLProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
num_videos = mm_counts.get("video", 0)
target_width, target_height = self.info.get_image_size_with_most_features()
target_num_frames = self.info.get_num_frames_with_most_features(
seq_len, mm_counts
)
image_overrides = mm_options.get("image")
video_overrides = mm_options.get("video")
return {
"image": self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
),
"video": self._get_dummy_videos(
width=target_width,
height=target_height,
num_frames=target_num_frames,
num_videos=num_videos,
overrides=video_overrides,
num_videos=mm_counts.get("video", 0),
overrides=mm_options.get("video"),
),
}
+4 -8
View File
@@ -32,7 +32,7 @@ from transformers.feature_extraction_utils import BatchFeature
from transformers.models.whisper import WhisperFeatureExtractor
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.config.speech_to_text import SpeechToTextParams
from vllm.inputs import ModalityData, MultiModalDataDict, PromptType, TokensPrompt
from vllm.logger import init_logger
@@ -226,10 +226,8 @@ class Qwen3ASRDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3ASRProcessingInfo])
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_audios = mm_counts.get("audio", 0)
feature_extractor = self.info.get_feature_extractor()
target_audio_length = (
@@ -240,13 +238,11 @@ class Qwen3ASRDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3ASRProcessingInfo])
* feature_extractor.sampling_rate
)
audio_overrides = mm_options.get("audio")
return {
"audio": self._get_dummy_audios(
length=target_audio_length,
num_audios=num_audios,
overrides=audio_overrides,
num_audios=mm_counts.get("audio", 0),
overrides=mm_options.get("audio"),
),
}
+5 -10
View File
@@ -54,8 +54,8 @@ from vllm.compilation.decorators import (
)
from vllm.config import VllmConfig
from vllm.config.multimodal import (
BaseDummyOptions,
MultiModalConfig,
MultiModalDummyOptions,
VideoDummyOptions,
VideoPruningMethod,
)
@@ -1119,11 +1119,8 @@ class Qwen3VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3VLProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
num_videos = mm_counts.get("video", 0)
image_overrides = mm_options.get("image")
video_overrides = mm_options.get("video")
target_image_width, target_image_height = (
@@ -1133,7 +1130,6 @@ class Qwen3VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3VLProcessingInfo]):
# treat videos as special images
target_num_frames = 2
if video_overrides:
assert isinstance(video_overrides, VideoDummyOptions)
num_frames_override = video_overrides.num_frames
if num_frames_override:
if num_frames_override > target_num_frames:
@@ -1204,7 +1200,6 @@ class Qwen3VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3VLProcessingInfo]):
target_video_size.height,
)
if video_overrides:
assert isinstance(video_overrides, VideoDummyOptions)
width_override = video_overrides.width
if width_override:
if width_override > target_video_width:
@@ -1230,14 +1225,14 @@ class Qwen3VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3VLProcessingInfo]):
"image": self._get_dummy_images(
width=target_image_width,
height=target_image_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
),
"video": self._get_dummy_videos(
width=target_video_width,
height=target_video_height,
num_frames=target_num_frames,
num_videos=num_videos,
num_videos=mm_counts.get("video", 0),
),
}
+4 -8
View File
@@ -8,7 +8,7 @@ import torch.nn as nn
from transformers.activations import GELUActivation
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.multimodal import MULTIMODAL_REGISTRY
@@ -40,20 +40,16 @@ class RVLDummyInputsBuilder(LlavaDummyInputsBuilder[RVLProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
target_width, target_height = self.info.get_image_size_with_most_features()
image_overrides = mm_options.get("image")
return {
"image": self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
),
}
+4 -8
View File
@@ -16,7 +16,7 @@ from transformers import (
)
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.distributed import divide, get_tensor_model_parallel_world_size
from vllm.inputs import MultiModalDataDict, MultiModalInput
from vllm.model_executor.layers.activation import get_act_fn
@@ -158,20 +158,16 @@ class SiglipDummyInputsBuilder(BaseDummyInputsBuilder[SiglipProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
target_width, target_height = self.info.get_image_size_with_most_features()
image_overrides = mm_options.get("image")
return {
"image": self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
)
}
+4 -7
View File
@@ -11,7 +11,7 @@ import torch.nn.functional as F
from transformers import BatchFeature
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.activation import get_act_fn
@@ -139,19 +139,16 @@ class Step3VLDummyInputsBuilder(BaseDummyInputsBuilder[Step3VLProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
target_width, target_height = self.info.get_image_size_with_most_features()
num_images = mm_counts.get("image", 0)
image_overrides = mm_options.get("image")
return {
"image": self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
)
}
+2 -2
View File
@@ -33,7 +33,7 @@ from terratorch.vllm import (
from transformers import BatchFeature
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import ModalityData, MultiModalDataDict, MultiModalInput, mm_input
from vllm.logger import init_logger
from vllm.model_executor.layers.pooler import IdentityPooler
@@ -153,7 +153,7 @@ class TerratorchInputBuilder(BaseDummyInputsBuilder[TerratorchProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
# Dummy data is generated based on the 'input' section
# defined in the HF configuration file
@@ -27,7 +27,6 @@ from packaging.version import Version
from transformers.utils.generic import ModelOutput
from vllm.compilation.decorators import should_torch_compile_mm_encoder
from vllm.config.multimodal import AudioDummyOptions
from vllm.config.utils import getattr_iter
from vllm.inputs import (
MultiModalDataBuiltins,
@@ -75,7 +74,7 @@ if TYPE_CHECKING:
from transformers import BatchFeature, PreTrainedModel
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
logger = init_logger(__name__)
@@ -195,7 +194,7 @@ class MultiModalDummyInputsBuilder(BaseDummyInputsBuilder[MultiModalProcessingIn
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, "BaseDummyOptions"],
mm_options: "MultiModalDummyOptions",
) -> MultiModalDataDict:
data = MultiModalDataBuiltins()
if self.info._is_audio_model() and (num_audios := mm_counts.get("audio", 0)):
@@ -204,21 +203,16 @@ class MultiModalDummyInputsBuilder(BaseDummyInputsBuilder[MultiModalProcessingIn
chunk_length = getattr(sub, "chunk_length", None) if sub else None
if chunk_length is None:
chunk_length = 30
audio_len = int(chunk_length * sampling_rate)
audio_overrides = mm_options.get("audio")
assert audio_overrides is None or isinstance(
audio_overrides, AudioDummyOptions
)
data["audio"] = self._get_dummy_audios(
length=audio_len,
length=int(chunk_length * sampling_rate),
num_audios=num_audios,
overrides=audio_overrides,
overrides=mm_options.get("audio"),
)
if self.info._is_image_model() and (num_images := mm_counts.get("image", 0)):
target_width, target_height = self.info.get_image_size_with_most_features()
width, height = self.info.get_image_size_with_most_features()
data["image"] = self._get_dummy_images(
width=target_width,
height=target_height,
width=width,
height=height,
num_images=num_images,
overrides=mm_options.get("image"),
)
+4 -7
View File
@@ -16,7 +16,7 @@ from transformers import BatchFeature, ProcessorMixin
from transformers.models.whisper import WhisperFeatureExtractor
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.activation import MulAndSilu, get_act_fn
@@ -169,7 +169,7 @@ class UltravoxDummyInputsBuilder(BaseDummyInputsBuilder[UltravoxProcessingInfo])
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
feature_extractor = self.info.get_feature_extractor()
@@ -177,15 +177,12 @@ class UltravoxDummyInputsBuilder(BaseDummyInputsBuilder[UltravoxProcessingInfo])
audio_len = (
feature_extractor.chunk_length * sampling_rate * _MAX_ENCODER_BATCH_SIZE
)
num_audios = mm_counts.get("audio", 0)
audio_overrides = mm_options.get("audio")
return {
"audio": self._get_dummy_audios(
length=audio_len,
num_audios=num_audios,
overrides=audio_overrides,
num_audios=mm_counts.get("audio", 0),
overrides=mm_options.get("audio"),
)
}
+5 -9
View File
@@ -19,7 +19,7 @@ from mistral_common.tokens.tokenizers.audio import Audio
from transformers import WhisperConfig
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.config.speech_to_text import SpeechToTextParams
from vllm.inputs import MultiModalDataDict, PromptType, TokensPrompt
from vllm.logger import init_logger
@@ -137,19 +137,15 @@ class VoxtralDummyInputsBuilder(BaseDummyInputsBuilder[VoxtralProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
num_audios = mm_counts.get("audio", 0)
target_length = self.info.get_max_audio_array_len()
audio_overrides = mm_options.get("audio")
return {
"audio": self._get_dummy_audios(
length=target_length,
num_audios=num_audios,
overrides=audio_overrides,
num_audios=mm_counts.get("audio", 0),
overrides=mm_options.get("audio"),
)
}
@@ -157,7 +153,7 @@ class VoxtralDummyInputsBuilder(BaseDummyInputsBuilder[VoxtralProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
mm_data: MultiModalDataDict | None = None,
) -> ProcessorInputs:
tokenizer = self.info.get_tokenizer()
+4 -7
View File
@@ -19,7 +19,7 @@ from transformers.models.whisper.modeling_whisper import sinusoids
from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, ModelConfig, SpeechToTextConfig, VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.config.speech_to_text import SpeechToTextParams
from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.inputs import (
@@ -724,21 +724,18 @@ class WhisperDummyInputsBuilder(BaseDummyInputsBuilder[WhisperProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
feature_extractor = self.info.get_feature_extractor()
sampling_rate = feature_extractor.sampling_rate
audio_len = feature_extractor.chunk_length * sampling_rate
num_audios = mm_counts.get("audio", 0)
audio_overrides = mm_options.get("audio")
return {
"audio": self._get_dummy_audios(
length=audio_len,
num_audios=num_audios,
overrides=audio_overrides,
num_audios=mm_counts.get("audio", 0),
overrides=mm_options.get("audio"),
)
}