[Multimodal] Type dummy options per modality (#57576)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Harry Mellor
2026-09-18 17:29:11 +01:00
committed by GitHub
co-authored by Claude Opus 5
parent 56cac80b14
commit 8a5cf54387
93 changed files with 461 additions and 720 deletions
@@ -22,7 +22,7 @@ reference's out-of-vocab scheme.
import math
from collections.abc import Mapping, Sequence
from typing import Any, cast
from typing import Any
import numpy as np
import torch
@@ -30,7 +30,7 @@ from PIL import Image, ImageOps
from transformers import BatchFeature
from typing_extensions import assert_never
from vllm.config.multimodal import BaseDummyOptions, ImageDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargsItems
from vllm.multimodal.parse import ImageSize, MultiModalDataItems
@@ -375,7 +375,7 @@ class DeepseekV4VLDummyInputsBuilder(
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
size = self.info.get_image_size_with_most_features()
return {
@@ -383,7 +383,7 @@ class DeepseekV4VLDummyInputsBuilder(
width=size.width,
height=size.height,
num_images=mm_counts.get("image", 0),
overrides=cast(ImageDummyOptions | None, mm_options.get("image")),
overrides=mm_options.get("image"),
),
}
@@ -19,14 +19,14 @@ boundaries freely).
import math
from collections.abc import Mapping, Sequence
from typing import Any, cast
from typing import Any
import numpy as np
import torch
from PIL import Image, ImageOps
from transformers import BatchFeature
from vllm.config.multimodal import BaseDummyOptions, ImageDummyOptions
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargsItems
from vllm.multimodal.parse import ImageSize, MultiModalDataItems
@@ -296,7 +296,7 @@ class DeepseekV4VLDummyInputsBuilder(
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
size = self.info.get_image_size_with_most_features()
return {
@@ -304,7 +304,7 @@ class DeepseekV4VLDummyInputsBuilder(
width=size.width,
height=size.height,
num_images=mm_counts.get("image", 0),
overrides=cast(ImageDummyOptions | None, mm_options.get("image")),
overrides=mm_options.get("image"),
),
}
+5 -8
View File
@@ -13,10 +13,7 @@ from PIL import Image
from transformers import BatchFeature
from vllm.config.multimodal import (
AudioDummyOptions,
BaseDummyOptions,
ImageDummyOptions,
VideoDummyOptions,
MultiModalDummyOptions,
)
from vllm.inputs import MultiModalDataDict
from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargsItems
@@ -584,7 +581,7 @@ class Dots3NoteDummyInputsBuilder(BaseDummyInputsBuilder[Dots3NoteProcessingInfo
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
data: dict[str, Any] = {}
num_images = mm_counts.get("image", 0)
@@ -594,7 +591,7 @@ class Dots3NoteDummyInputsBuilder(BaseDummyInputsBuilder[Dots3NoteProcessingInfo
width=width,
height=height,
num_images=num_images,
overrides=cast(ImageDummyOptions | None, mm_options.get("image")),
overrides=mm_options.get("image"),
)
num_audios = mm_counts.get("audio", 0)
if num_audios:
@@ -611,7 +608,7 @@ class Dots3NoteDummyInputsBuilder(BaseDummyInputsBuilder[Dots3NoteProcessingInfo
data["audio"] = self._get_dummy_audios(
length=min(chunk_samples, max(1, seq_len) * stride),
num_audios=num_audios,
overrides=cast(AudioDummyOptions | None, mm_options.get("audio")),
overrides=mm_options.get("audio"),
)
num_videos = mm_counts.get("video", 0)
if num_videos:
@@ -625,7 +622,7 @@ class Dots3NoteDummyInputsBuilder(BaseDummyInputsBuilder[Dots3NoteProcessingInfo
height=height,
num_frames=num_frames,
num_videos=num_videos,
overrides=cast(VideoDummyOptions | None, mm_options.get("video")),
overrides=mm_options.get("video"),
)
return data
+4 -8
View File
@@ -13,9 +13,7 @@ import torch
from transformers.feature_extraction_utils import BatchFeature
from vllm.config.multimodal import (
AudioDummyOptions,
BaseDummyOptions,
ImageDummyOptions,
MultiModalDummyOptions,
)
from vllm.inputs import MultiModalDataDict
from vllm.multimodal.inputs import (
@@ -145,7 +143,7 @@ class InklingDummyInputsBuilder(BaseDummyInputsBuilder[InklingProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
config = self.info.get_hf_config()
num_images = mm_counts.get("image", 0)
@@ -156,12 +154,11 @@ class InklingDummyInputsBuilder(BaseDummyInputsBuilder[InklingProcessingInfo]):
patch_size = getattr(config.vision_config, "patch_size", 40)
# A square image ~4 patches wide so the dummy emits several patches.
side = patch_size * 4
image_overrides = mm_options.get("image")
mm_data["image"] = self._get_dummy_images(
width=side,
height=side,
num_images=num_images,
overrides=cast(ImageDummyOptions | None, image_overrides),
overrides=mm_options.get("image"),
)
if num_audios:
# Size the dummy at the maximum allowed audio so memory/encoder
@@ -169,11 +166,10 @@ class InklingDummyInputsBuilder(BaseDummyInputsBuilder[InklingProcessingInfo]):
params = self.info.get_hf_processor().audio_feature_extractor.params
hop = int(round(params.audio_token_duration_s * params.sample_rate))
audio_len = MAX_AUDIO_TOKENS * hop
audio_overrides = mm_options.get("audio")
mm_data["audio"] = self._get_dummy_audios(
length=audio_len,
num_audios=num_audios,
overrides=cast(AudioDummyOptions | None, audio_overrides),
overrides=mm_options.get("audio"),
)
return mm_data
+3 -4
View File
@@ -9,7 +9,7 @@ from typing import Any, cast
import torch
from transformers import BatchFeature
from vllm.config.multimodal import BaseDummyOptions, ImageDummyOptions
from vllm.config.multimodal import ImageDummyOptions, MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.logger import init_logger
from vllm.multimodal.inputs import (
@@ -204,7 +204,7 @@ class KimiK3DummyInputsBuilder(BaseDummyInputsBuilder[KimiK3ProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions] | None = None,
mm_options: MultiModalDummyOptions | None = None,
) -> MultiModalDataDict:
media_proc_cfg = self.info.image_processor.media_proc_cfg
max_size = self.info.get_max_image_size(
@@ -214,7 +214,6 @@ class KimiK3DummyInputsBuilder(BaseDummyInputsBuilder[KimiK3ProcessingInfo]):
media_proc_cfg["patch_limit_on_one_side"],
media_proc_cfg["fixed_output_tokens"],
)
num_images = mm_counts.get("image", 0)
image_overrides = cast(
ImageDummyOptions | None,
mm_options.get("image") if mm_options else None,
@@ -223,7 +222,7 @@ class KimiK3DummyInputsBuilder(BaseDummyInputsBuilder[KimiK3ProcessingInfo]):
"image": self._get_dummy_images(
width=max_size.width,
height=max_size.height,
num_images=num_images,
num_images=mm_counts.get("image", 0),
overrides=image_overrides,
)
}
@@ -11,9 +11,7 @@ from transformers import BatchFeature
from transformers.video_utils import VideoMetadata
from vllm.config.multimodal import (
BaseDummyOptions,
ImageDummyOptions,
VideoDummyOptions,
MultiModalDummyOptions,
)
from vllm.inputs import MultiModalDataDict
from vllm.multimodal.inputs import (
@@ -283,7 +281,7 @@ class MiniMaxM3VLDummyInputsBuilder(BaseDummyInputsBuilder[MiniMaxM3VLProcessing
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
size = self.info.get_image_size_with_most_features()
video_size = self.info.get_video_size_with_most_features()
@@ -293,14 +291,14 @@ class MiniMaxM3VLDummyInputsBuilder(BaseDummyInputsBuilder[MiniMaxM3VLProcessing
width=size.width,
height=size.height,
num_images=mm_counts.get("image", 0),
overrides=cast(ImageDummyOptions | None, mm_options.get("image")),
overrides=mm_options.get("image"),
),
"video": self._get_dummy_videos(
width=video_size.width,
height=video_size.height,
num_frames=num_frames,
num_videos=mm_counts.get("video", 0),
overrides=cast(VideoDummyOptions | None, mm_options.get("video")),
overrides=mm_options.get("video"),
),
}