[Multimodal] Type dummy options per modality (#57576)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
56cac80b14
commit
8a5cf54387
@@ -22,7 +22,7 @@ reference's out-of-vocab scheme.
|
||||
|
||||
import math
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import Any, cast
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
@@ -30,7 +30,7 @@ from PIL import Image, ImageOps
|
||||
from transformers import BatchFeature
|
||||
from typing_extensions import assert_never
|
||||
|
||||
from vllm.config.multimodal import BaseDummyOptions, ImageDummyOptions
|
||||
from vllm.config.multimodal import MultiModalDummyOptions
|
||||
from vllm.inputs import MultiModalDataDict
|
||||
from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargsItems
|
||||
from vllm.multimodal.parse import ImageSize, MultiModalDataItems
|
||||
@@ -375,7 +375,7 @@ class DeepseekV4VLDummyInputsBuilder(
|
||||
self,
|
||||
seq_len: int,
|
||||
mm_counts: Mapping[str, int],
|
||||
mm_options: Mapping[str, BaseDummyOptions],
|
||||
mm_options: MultiModalDummyOptions,
|
||||
) -> MultiModalDataDict:
|
||||
size = self.info.get_image_size_with_most_features()
|
||||
return {
|
||||
@@ -383,7 +383,7 @@ class DeepseekV4VLDummyInputsBuilder(
|
||||
width=size.width,
|
||||
height=size.height,
|
||||
num_images=mm_counts.get("image", 0),
|
||||
overrides=cast(ImageDummyOptions | None, mm_options.get("image")),
|
||||
overrides=mm_options.get("image"),
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
@@ -19,14 +19,14 @@ boundaries freely).
|
||||
|
||||
import math
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import Any, cast
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from PIL import Image, ImageOps
|
||||
from transformers import BatchFeature
|
||||
|
||||
from vllm.config.multimodal import BaseDummyOptions, ImageDummyOptions
|
||||
from vllm.config.multimodal import MultiModalDummyOptions
|
||||
from vllm.inputs import MultiModalDataDict
|
||||
from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargsItems
|
||||
from vllm.multimodal.parse import ImageSize, MultiModalDataItems
|
||||
@@ -296,7 +296,7 @@ class DeepseekV4VLDummyInputsBuilder(
|
||||
self,
|
||||
seq_len: int,
|
||||
mm_counts: Mapping[str, int],
|
||||
mm_options: Mapping[str, BaseDummyOptions],
|
||||
mm_options: MultiModalDummyOptions,
|
||||
) -> MultiModalDataDict:
|
||||
size = self.info.get_image_size_with_most_features()
|
||||
return {
|
||||
@@ -304,7 +304,7 @@ class DeepseekV4VLDummyInputsBuilder(
|
||||
width=size.width,
|
||||
height=size.height,
|
||||
num_images=mm_counts.get("image", 0),
|
||||
overrides=cast(ImageDummyOptions | None, mm_options.get("image")),
|
||||
overrides=mm_options.get("image"),
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
@@ -13,10 +13,7 @@ from PIL import Image
|
||||
from transformers import BatchFeature
|
||||
|
||||
from vllm.config.multimodal import (
|
||||
AudioDummyOptions,
|
||||
BaseDummyOptions,
|
||||
ImageDummyOptions,
|
||||
VideoDummyOptions,
|
||||
MultiModalDummyOptions,
|
||||
)
|
||||
from vllm.inputs import MultiModalDataDict
|
||||
from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargsItems
|
||||
@@ -584,7 +581,7 @@ class Dots3NoteDummyInputsBuilder(BaseDummyInputsBuilder[Dots3NoteProcessingInfo
|
||||
self,
|
||||
seq_len: int,
|
||||
mm_counts: Mapping[str, int],
|
||||
mm_options: Mapping[str, BaseDummyOptions],
|
||||
mm_options: MultiModalDummyOptions,
|
||||
) -> MultiModalDataDict:
|
||||
data: dict[str, Any] = {}
|
||||
num_images = mm_counts.get("image", 0)
|
||||
@@ -594,7 +591,7 @@ class Dots3NoteDummyInputsBuilder(BaseDummyInputsBuilder[Dots3NoteProcessingInfo
|
||||
width=width,
|
||||
height=height,
|
||||
num_images=num_images,
|
||||
overrides=cast(ImageDummyOptions | None, mm_options.get("image")),
|
||||
overrides=mm_options.get("image"),
|
||||
)
|
||||
num_audios = mm_counts.get("audio", 0)
|
||||
if num_audios:
|
||||
@@ -611,7 +608,7 @@ class Dots3NoteDummyInputsBuilder(BaseDummyInputsBuilder[Dots3NoteProcessingInfo
|
||||
data["audio"] = self._get_dummy_audios(
|
||||
length=min(chunk_samples, max(1, seq_len) * stride),
|
||||
num_audios=num_audios,
|
||||
overrides=cast(AudioDummyOptions | None, mm_options.get("audio")),
|
||||
overrides=mm_options.get("audio"),
|
||||
)
|
||||
num_videos = mm_counts.get("video", 0)
|
||||
if num_videos:
|
||||
@@ -625,7 +622,7 @@ class Dots3NoteDummyInputsBuilder(BaseDummyInputsBuilder[Dots3NoteProcessingInfo
|
||||
height=height,
|
||||
num_frames=num_frames,
|
||||
num_videos=num_videos,
|
||||
overrides=cast(VideoDummyOptions | None, mm_options.get("video")),
|
||||
overrides=mm_options.get("video"),
|
||||
)
|
||||
return data
|
||||
|
||||
|
||||
@@ -13,9 +13,7 @@ import torch
|
||||
from transformers.feature_extraction_utils import BatchFeature
|
||||
|
||||
from vllm.config.multimodal import (
|
||||
AudioDummyOptions,
|
||||
BaseDummyOptions,
|
||||
ImageDummyOptions,
|
||||
MultiModalDummyOptions,
|
||||
)
|
||||
from vllm.inputs import MultiModalDataDict
|
||||
from vllm.multimodal.inputs import (
|
||||
@@ -145,7 +143,7 @@ class InklingDummyInputsBuilder(BaseDummyInputsBuilder[InklingProcessingInfo]):
|
||||
self,
|
||||
seq_len: int,
|
||||
mm_counts: Mapping[str, int],
|
||||
mm_options: Mapping[str, BaseDummyOptions],
|
||||
mm_options: MultiModalDummyOptions,
|
||||
) -> MultiModalDataDict:
|
||||
config = self.info.get_hf_config()
|
||||
num_images = mm_counts.get("image", 0)
|
||||
@@ -156,12 +154,11 @@ class InklingDummyInputsBuilder(BaseDummyInputsBuilder[InklingProcessingInfo]):
|
||||
patch_size = getattr(config.vision_config, "patch_size", 40)
|
||||
# A square image ~4 patches wide so the dummy emits several patches.
|
||||
side = patch_size * 4
|
||||
image_overrides = mm_options.get("image")
|
||||
mm_data["image"] = self._get_dummy_images(
|
||||
width=side,
|
||||
height=side,
|
||||
num_images=num_images,
|
||||
overrides=cast(ImageDummyOptions | None, image_overrides),
|
||||
overrides=mm_options.get("image"),
|
||||
)
|
||||
if num_audios:
|
||||
# Size the dummy at the maximum allowed audio so memory/encoder
|
||||
@@ -169,11 +166,10 @@ class InklingDummyInputsBuilder(BaseDummyInputsBuilder[InklingProcessingInfo]):
|
||||
params = self.info.get_hf_processor().audio_feature_extractor.params
|
||||
hop = int(round(params.audio_token_duration_s * params.sample_rate))
|
||||
audio_len = MAX_AUDIO_TOKENS * hop
|
||||
audio_overrides = mm_options.get("audio")
|
||||
mm_data["audio"] = self._get_dummy_audios(
|
||||
length=audio_len,
|
||||
num_audios=num_audios,
|
||||
overrides=cast(AudioDummyOptions | None, audio_overrides),
|
||||
overrides=mm_options.get("audio"),
|
||||
)
|
||||
return mm_data
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ from typing import Any, cast
|
||||
import torch
|
||||
from transformers import BatchFeature
|
||||
|
||||
from vllm.config.multimodal import BaseDummyOptions, ImageDummyOptions
|
||||
from vllm.config.multimodal import ImageDummyOptions, MultiModalDummyOptions
|
||||
from vllm.inputs import MultiModalDataDict
|
||||
from vllm.logger import init_logger
|
||||
from vllm.multimodal.inputs import (
|
||||
@@ -204,7 +204,7 @@ class KimiK3DummyInputsBuilder(BaseDummyInputsBuilder[KimiK3ProcessingInfo]):
|
||||
self,
|
||||
seq_len: int,
|
||||
mm_counts: Mapping[str, int],
|
||||
mm_options: Mapping[str, BaseDummyOptions] | None = None,
|
||||
mm_options: MultiModalDummyOptions | None = None,
|
||||
) -> MultiModalDataDict:
|
||||
media_proc_cfg = self.info.image_processor.media_proc_cfg
|
||||
max_size = self.info.get_max_image_size(
|
||||
@@ -214,7 +214,6 @@ class KimiK3DummyInputsBuilder(BaseDummyInputsBuilder[KimiK3ProcessingInfo]):
|
||||
media_proc_cfg["patch_limit_on_one_side"],
|
||||
media_proc_cfg["fixed_output_tokens"],
|
||||
)
|
||||
num_images = mm_counts.get("image", 0)
|
||||
image_overrides = cast(
|
||||
ImageDummyOptions | None,
|
||||
mm_options.get("image") if mm_options else None,
|
||||
@@ -223,7 +222,7 @@ class KimiK3DummyInputsBuilder(BaseDummyInputsBuilder[KimiK3ProcessingInfo]):
|
||||
"image": self._get_dummy_images(
|
||||
width=max_size.width,
|
||||
height=max_size.height,
|
||||
num_images=num_images,
|
||||
num_images=mm_counts.get("image", 0),
|
||||
overrides=image_overrides,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -11,9 +11,7 @@ from transformers import BatchFeature
|
||||
from transformers.video_utils import VideoMetadata
|
||||
|
||||
from vllm.config.multimodal import (
|
||||
BaseDummyOptions,
|
||||
ImageDummyOptions,
|
||||
VideoDummyOptions,
|
||||
MultiModalDummyOptions,
|
||||
)
|
||||
from vllm.inputs import MultiModalDataDict
|
||||
from vllm.multimodal.inputs import (
|
||||
@@ -283,7 +281,7 @@ class MiniMaxM3VLDummyInputsBuilder(BaseDummyInputsBuilder[MiniMaxM3VLProcessing
|
||||
self,
|
||||
seq_len: int,
|
||||
mm_counts: Mapping[str, int],
|
||||
mm_options: Mapping[str, BaseDummyOptions],
|
||||
mm_options: MultiModalDummyOptions,
|
||||
) -> MultiModalDataDict:
|
||||
size = self.info.get_image_size_with_most_features()
|
||||
video_size = self.info.get_video_size_with_most_features()
|
||||
@@ -293,14 +291,14 @@ class MiniMaxM3VLDummyInputsBuilder(BaseDummyInputsBuilder[MiniMaxM3VLProcessing
|
||||
width=size.width,
|
||||
height=size.height,
|
||||
num_images=mm_counts.get("image", 0),
|
||||
overrides=cast(ImageDummyOptions | None, mm_options.get("image")),
|
||||
overrides=mm_options.get("image"),
|
||||
),
|
||||
"video": self._get_dummy_videos(
|
||||
width=video_size.width,
|
||||
height=video_size.height,
|
||||
num_frames=num_frames,
|
||||
num_videos=mm_counts.get("video", 0),
|
||||
overrides=cast(VideoDummyOptions | None, mm_options.get("video")),
|
||||
overrides=mm_options.get("video"),
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user