From 7853700d4cdc2b84c728c840114ed42cda7454a7 Mon Sep 17 00:00:00 2001 From: Cyrus Leung Date: Tue, 22 Sep 2026 00:39:20 +0800 Subject: [PATCH] [MM] Move get_dummy_processor_inputs into MM processor (#57967) Signed-off-by: DarkLight1337 --- .../multimodal/processing/test_common.py | 5 +- .../processing/test_tensor_schema.py | 3 +- tests/renderers/test_warmup.py | 14 ++--- .../models/hyperclovax_vision_v2.py | 56 ++++++++----------- vllm/model_executor/models/kimi_audio.py | 47 ++++++++-------- .../models/moss_transcribe_diarize.py | 41 ++++++++------ vllm/model_executor/models/pixtral.py | 12 ++-- vllm/model_executor/models/voxtral.py | 12 ++-- vllm/multimodal/processing/dummy_inputs.py | 37 ------------ vllm/multimodal/processing/processor.py | 39 ++++++++++++- 10 files changed, 131 insertions(+), 135 deletions(-) diff --git a/tests/models/multimodal/processing/test_common.py b/tests/models/multimodal/processing/test_common.py index db6f37dc3d..fa23956017 100644 --- a/tests/models/multimodal/processing/test_common.py +++ b/tests/models/multimodal/processing/test_common.py @@ -159,7 +159,6 @@ def get_token_prompt( processor: BaseMultiModalProcessor, mm_data: MultiModalDataDict, ) -> list[int]: - dummy_inputs = processor.dummy_inputs tokenizer: TokenizerLike = processor.info.get_tokenizer() model_config = processor.info.ctx.model_config @@ -174,7 +173,7 @@ def get_token_prompt( mm_counts = {k: len(vs) for k, vs in parsed_data.items()} if is_mistral_tokenizer(tokenizer): - inputs = dummy_inputs.get_dummy_processor_inputs( + inputs = processor.get_dummy_inputs( model_config.max_model_len, mm_counts, mm_options=MultiModalDummyOptions(), @@ -182,7 +181,7 @@ def get_token_prompt( mm_data=mm_data, # type: ignore[call-arg] ) else: - inputs = dummy_inputs.get_dummy_processor_inputs( + inputs = processor.get_dummy_inputs( model_config.max_model_len, mm_counts, mm_options=MultiModalDummyOptions(), diff --git a/tests/models/multimodal/processing/test_tensor_schema.py b/tests/models/multimodal/processing/test_tensor_schema.py index 1e1722537d..73be3ab76c 100644 --- a/tests/models/multimodal/processing/test_tensor_schema.py +++ b/tests/models/multimodal/processing/test_tensor_schema.py @@ -90,13 +90,12 @@ def create_batched_mm_kwargs( size_factors: tuple[float, ...] = (1.0, 0.5, 0.25), ) -> Iterable[tuple[str, int, BatchedTensorInputs]]: processing_info = processor.info - dummy_inputs = processor.dummy_inputs supported_mm_limits = processing_info.get_supported_mm_limits() mm_counts = { modality: 3 if limit is None else limit for modality, limit in supported_mm_limits.items() } - processor_inputs = dummy_inputs.get_dummy_processor_inputs( + processor_inputs = processor.get_dummy_inputs( seq_len=model_config.max_model_len, mm_counts=mm_counts, mm_options=MultiModalDummyOptions(), diff --git a/tests/renderers/test_warmup.py b/tests/renderers/test_warmup.py index 2627cdd985..b0e3dcb547 100644 --- a/tests/renderers/test_warmup.py +++ b/tests/renderers/test_warmup.py @@ -4,7 +4,7 @@ These tests exercise: - Zero-limit modalities are filtered from mm_counts passed to - get_dummy_processor_inputs (e.g. --limit-mm-per-prompt image=0 ...) + get_dummy_inputs (e.g. --limit-mm-per-prompt image=0 ...) - The warmup runs through the processor-only cache and clears it afterwards, so its dummy inputs never land in the sender cache used by the serving path - MM warmup is skipped entirely when mm_processor is None @@ -92,7 +92,7 @@ class TestMmWarmupZeroLimitFiltering: BaseRenderer.warmup(renderer, ChatParams()) - get_inputs = renderer.mm_processor.dummy_inputs.get_dummy_processor_inputs + get_inputs = renderer.mm_processor.get_dummy_inputs get_inputs.assert_called_once() _, kwargs = get_inputs.call_args assert "video" not in kwargs["mm_counts"] @@ -104,7 +104,7 @@ class TestMmWarmupZeroLimitFiltering: BaseRenderer.warmup(renderer, ChatParams()) - get_inputs = renderer.mm_processor.dummy_inputs.get_dummy_processor_inputs + get_inputs = renderer.mm_processor.get_dummy_inputs get_inputs.assert_called_once() _, kwargs = get_inputs.call_args assert kwargs["mm_counts"] == {} @@ -115,7 +115,7 @@ class TestMmWarmupZeroLimitFiltering: BaseRenderer.warmup(renderer, ChatParams()) - get_inputs = renderer.mm_processor.dummy_inputs.get_dummy_processor_inputs + get_inputs = renderer.mm_processor.get_dummy_inputs get_inputs.assert_called_once() _, kwargs = get_inputs.call_args assert kwargs["mm_counts"] == {"image": 1, "video": 1} @@ -134,7 +134,7 @@ class TestMmWarmupRunsNormally: BaseRenderer.warmup(renderer, ChatParams()) - get_inputs = renderer.mm_processor.dummy_inputs.get_dummy_processor_inputs + get_inputs = renderer.mm_processor.get_dummy_inputs assert get_inputs.call_args.kwargs["seq_len"] == expected_seq_len def test_processor_apply_called(self): @@ -207,9 +207,7 @@ class TestWarmupFaultIsolation: def test_mm_failure_is_swallowed_and_cache_cleared(self): renderer = _make_renderer_mock({"image": 1}) # the warmup blows up before apply() - renderer.mm_processor.dummy_inputs.get_dummy_processor_inputs.side_effect = ( - RuntimeError("mm boom") - ) + renderer.mm_processor.get_dummy_inputs.side_effect = RuntimeError("mm boom") BaseRenderer.warmup(renderer, ChatParams()) # must not raise diff --git a/vllm/model_executor/models/hyperclovax_vision_v2.py b/vllm/model_executor/models/hyperclovax_vision_v2.py index e269e2a059..ee23d2c4df 100644 --- a/vllm/model_executor/models/hyperclovax_vision_v2.py +++ b/vllm/model_executor/models/hyperclovax_vision_v2.py @@ -189,38 +189,6 @@ class HCXVisionV2DummyInputsBuilder(BaseDummyInputsBuilder[HCXVisionV2Processing num_videos = mm_counts.get("video", 0) return V2_IMAGE_TOKEN * num_images + V2_VIDEO_TOKEN * num_videos - def get_dummy_processor_inputs( - self, - seq_len: int, - mm_counts: Mapping[str, int], - mm_options: MultiModalDummyOptions | None = None, - mm_processor_kwargs: Mapping[str, object] | None = None, - ) -> ProcessorInputs: - """Build dummy processor inputs for memory profiling.""" - num_images = mm_counts.get("image", 0) - num_videos = mm_counts.get("video", 0) - prompt_text = V2_IMAGE_TOKEN * num_images + V2_VIDEO_TOKEN * num_videos - - dummy_mm_data = self.get_dummy_mm_data( - seq_len, - mm_counts, - mm_options, - mm_processor_kwargs=mm_processor_kwargs, - ) - dummy_mm_items = self.info.parse_mm_data(dummy_mm_data, validate=False) - - tokenizer = self.info.get_tokenizer() - prompt = tokenizer.encode( - prompt_text, - **self.info.default_tok_params.get_encode_kwargs(), - ) - - return ProcessorInputs( - prompt=prompt, - mm_data_items=dummy_mm_items, - hf_processor_mm_kwargs=mm_processor_kwargs or {}, - ) - def get_dummy_mm_data( self, seq_len: int, @@ -255,6 +223,28 @@ class HCXVisionV2MultiModalProcessor( ): """Multimodal processor for HyperCLOVAX V2 (32B Think model).""" + def get_dummy_inputs( + self, + seq_len: int, + mm_counts: Mapping[str, int], + mm_options: MultiModalDummyOptions, + ) -> ProcessorInputs: + num_images = mm_counts.get("image", 0) + num_videos = mm_counts.get("video", 0) + prompt_text = V2_IMAGE_TOKEN * num_images + V2_VIDEO_TOKEN * num_videos + + builder = self.dummy_inputs + dummy_mm_data = builder.get_dummy_mm_data(seq_len, mm_counts, mm_options) + dummy_mm_items = self.info.parse_mm_data(dummy_mm_data, validate=False) + + tokenizer = self.info.get_tokenizer() + prompt = tokenizer.encode( + prompt_text, + **self.info.default_tok_params.get_encode_kwargs(), + ) + + return ProcessorInputs(prompt=prompt, mm_data_items=dummy_mm_items) + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str: return self.dummy_inputs.get_dummy_text(mm_counts) @@ -267,7 +257,7 @@ class HCXVisionV2MultiModalProcessor( hf_config = self.info.get_hf_config() # Use token IDs directly from config. - # This matches what get_dummy_processor_inputs uses, ensuring consistency. + # This matches what get_dummy_inputs uses, ensuring consistency. placeholder: dict[str, int] = { "image": hf_config.image_token_id, # 128060 for <|IMAGE_PAD|> "video": hf_config.video_token_id, # 128061 for <|VIDEO_PAD|> diff --git a/vllm/model_executor/models/kimi_audio.py b/vllm/model_executor/models/kimi_audio.py index 3541fcf371..a2357836f1 100644 --- a/vllm/model_executor/models/kimi_audio.py +++ b/vllm/model_executor/models/kimi_audio.py @@ -176,29 +176,6 @@ class KimiAudioDummyInputsBuilder(BaseDummyInputsBuilder[KimiAudioProcessingInfo ), } - def get_dummy_processor_inputs( - self, - seq_len: int, - mm_counts: Mapping[str, int], - mm_options: MultiModalDummyOptions, - ) -> ProcessorInputs: - dummy_mm_data = self.get_dummy_mm_data(seq_len, mm_counts, mm_options) - dummy_mm_items = self.info.parse_mm_data(dummy_mm_data) - - num_audios = mm_counts.get("audio", 0) - dummy_tokens = ( - [198] - if num_audios == 0 - else [ - KimiAudioProcessor.KIMIA_MEDIA_BEGIN, - KimiAudioProcessor.KIMIA_TEXT_BLANK, - KimiAudioProcessor.KIMIA_MEDIA_END, - ] - * num_audios - ) - - return ProcessorInputs(prompt=dummy_tokens, mm_data_items=dummy_mm_items) - # Field config for Kimi-Audio multimodal data _KIMIAUDIO_FIELD_CONFIG = { @@ -228,6 +205,30 @@ class KimiAudioMultiModalDataParser(MultiModalDataParser): class KimiAudioMultiModalProcessor(BaseMultiModalProcessor[KimiAudioProcessingInfo]): """vLLM multi-modal processor wrapper for Kimi-Audio.""" + def get_dummy_inputs( + self, + seq_len: int, + mm_counts: Mapping[str, int], + mm_options: MultiModalDummyOptions, + ) -> ProcessorInputs: + builder = self.dummy_inputs + dummy_mm_data = builder.get_dummy_mm_data(seq_len, mm_counts, mm_options) + dummy_mm_items = self.info.parse_mm_data(dummy_mm_data) + + num_audios = mm_counts.get("audio", 0) + dummy_tokens = ( + [198] + if num_audios == 0 + else [ + KimiAudioProcessor.KIMIA_MEDIA_BEGIN, + KimiAudioProcessor.KIMIA_TEXT_BLANK, + KimiAudioProcessor.KIMIA_MEDIA_END, + ] + * num_audios + ) + + return ProcessorInputs(prompt=dummy_tokens, mm_data_items=dummy_mm_items) + def _get_hf_mm_inputs( self, mm_items: MultiModalDataItems, diff --git a/vllm/model_executor/models/moss_transcribe_diarize.py b/vllm/model_executor/models/moss_transcribe_diarize.py index 208d5d4bc1..e3fdab76ff 100644 --- a/vllm/model_executor/models/moss_transcribe_diarize.py +++ b/vllm/model_executor/models/moss_transcribe_diarize.py @@ -438,31 +438,38 @@ class MossTranscribeDiarizeDummyInputsBuilder( ) } - def get_dummy_processor_inputs( + +class MossTranscribeDiarizeMultiModalProcessor( + BaseMultiModalProcessor[MossTranscribeDiarizeProcessingInfo] +): + def get_dummy_inputs( self, seq_len: int, mm_counts: Mapping[str, int], mm_options: MultiModalDummyOptions, ) -> ProcessorInputs: - dummy_mm_data = self.get_dummy_mm_data(seq_len, mm_counts, mm_options) - dummy_mm_items = self.info.parse_mm_data(dummy_mm_data) - num_audios = mm_counts.get("audio", 0) + builder = self.dummy_inputs tokenizer = self.info.get_tokenizer() - prompt = cached_encode( - tokenizer, - AUDIO_PLACEHOLDER * num_audios, - add_special_tokens=False, - ) or cached_encode( - tokenizer, - "\n", - add_special_tokens=False, - ) + + dummy_mm_data = builder.get_dummy_mm_data(seq_len, mm_counts, mm_options) + dummy_mm_items = self.info.parse_mm_data(dummy_mm_data) + + num_audios = mm_counts.get("audio", 0) + if num_audios > 0: + prompt = cached_encode( + tokenizer, + AUDIO_PLACEHOLDER * num_audios, + add_special_tokens=False, + ) + else: + prompt = cached_encode( + tokenizer, + "\n", + add_special_tokens=False, + ) + return ProcessorInputs(prompt=prompt, mm_data_items=dummy_mm_items) - -class MossTranscribeDiarizeMultiModalProcessor( - BaseMultiModalProcessor[MossTranscribeDiarizeProcessingInfo] -): def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str: return self.dummy_inputs.get_dummy_text(mm_counts) diff --git a/vllm/model_executor/models/pixtral.py b/vllm/model_executor/models/pixtral.py index f5c167aec3..91ac38e110 100644 --- a/vllm/model_executor/models/pixtral.py +++ b/vllm/model_executor/models/pixtral.py @@ -221,18 +221,22 @@ class PixtralDummyInputsBuilder(BaseDummyInputsBuilder[PixtralProcessingInfo]): ) } - def get_dummy_processor_inputs( + +class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo]): + def get_dummy_inputs( self, seq_len: int, mm_counts: Mapping[str, int], mm_options: MultiModalDummyOptions, + # For test_common.py only mm_data: MultiModalDataDict | None = None, ) -> ProcessorInputs: + builder = self.dummy_inputs tokenizer = self.info.get_tokenizer() - dummy_text = self.get_dummy_text(mm_counts) + dummy_text = builder.get_dummy_text(mm_counts) dummy_mm_data = ( - self.get_dummy_mm_data(seq_len, mm_counts, mm_options) + builder.get_dummy_mm_data(seq_len, mm_counts, mm_options) if mm_data is None else mm_data ) @@ -256,8 +260,6 @@ class PixtralDummyInputsBuilder(BaseDummyInputsBuilder[PixtralProcessingInfo]): return ProcessorInputs(prompt=dummy_tokens, mm_data_items=dummy_mm_items) - -class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo]): # The tokens are already inserted by the chat template, # so we just double check that they exist def _maybe_apply_prompt_updates( diff --git a/vllm/model_executor/models/voxtral.py b/vllm/model_executor/models/voxtral.py index 0b8d889bbc..21b291344d 100644 --- a/vllm/model_executor/models/voxtral.py +++ b/vllm/model_executor/models/voxtral.py @@ -149,19 +149,23 @@ class VoxtralDummyInputsBuilder(BaseDummyInputsBuilder[VoxtralProcessingInfo]): ) } - def get_dummy_processor_inputs( + +class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo]): + def get_dummy_inputs( self, seq_len: int, mm_counts: Mapping[str, int], mm_options: MultiModalDummyOptions, + # For test_common.py only mm_data: MultiModalDataDict | None = None, ) -> ProcessorInputs: + builder = self.dummy_inputs tokenizer = self.info.get_tokenizer() feature_extractor = self.info.get_feature_extractor() - dummy_text = self.get_dummy_text(mm_counts) + dummy_text = builder.get_dummy_text(mm_counts) dummy_mm_data = ( - self.get_dummy_mm_data(seq_len, mm_counts, mm_options) + builder.get_dummy_mm_data(seq_len, mm_counts, mm_options) if mm_data is None else mm_data ) @@ -197,8 +201,6 @@ class VoxtralDummyInputsBuilder(BaseDummyInputsBuilder[VoxtralProcessingInfo]): return ProcessorInputs(prompt=dummy_tokens, mm_data_items=dummy_mm_items) - -class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo]): # The tokens are already inserted by the chat template, # so we just double check that they exist def _maybe_apply_prompt_updates( diff --git a/vllm/multimodal/processing/dummy_inputs.py b/vllm/multimodal/processing/dummy_inputs.py index 53e9f4f5b6..dd49b6c4fa 100644 --- a/vllm/multimodal/processing/dummy_inputs.py +++ b/vllm/multimodal/processing/dummy_inputs.py @@ -18,7 +18,6 @@ from vllm.inputs import MultiModalDataDict from vllm.logger import init_logger from .context import BaseProcessingInfo -from .inputs import ProcessorInputs _I = TypeVar("_I", bound=BaseProcessingInfo) @@ -61,42 +60,6 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]): """ raise NotImplementedError - def get_dummy_processor_inputs( - self, - seq_len: int, - mm_counts: Mapping[str, int], - mm_options: MultiModalDummyOptions, - ) -> ProcessorInputs: - """Build the input which, after processing, results in - the maximum possible number of placeholder tokens. - - Args: - seq_len: Sequence length - mm_counts: Count of items per modality - mm_options: Configurable options per modality (optional) - - """ - dummy_text = self.get_dummy_text(mm_counts) - dummy_mm_data = self.get_dummy_mm_data(seq_len, mm_counts, mm_options) - dummy_mm_items = self.info.parse_mm_data(dummy_mm_data, validate=False) - - tokenizer = self.info.ctx.tokenizer - dummy_prompt: list[int] - if tokenizer is None: - # Tokenizer-less models (e.g. `skip_tokenizer_init=True`) only - # accept embeddings and have an empty dummy text, so there are no - # prompt tokens. - dummy_prompt = [] - else: - from .processor import cached_encode - - dummy_prompt = cached_encode(tokenizer, dummy_text, truncation=False) - - return ProcessorInputs( - prompt=dummy_prompt, - mm_data_items=dummy_mm_items, - ) - def _get_dummy_audios( self, *, diff --git a/vllm/multimodal/processing/processor.py b/vllm/multimodal/processing/processor.py index b8e51b2053..eb53023f0f 100644 --- a/vllm/multimodal/processing/processor.py +++ b/vllm/multimodal/processing/processor.py @@ -39,7 +39,7 @@ from ..inputs import ( ) from ..parse import MultiModalDataItems, MultiModalUUIDItems from .context import BaseProcessingInfo, TimingContext -from .dummy_inputs import BaseDummyInputsBuilder +from .dummy_inputs import BaseDummyInputsBuilder, MultiModalDummyOptions from .inputs import ProcessorInputs if TYPE_CHECKING: @@ -1055,6 +1055,41 @@ class BaseMultiModalProcessor(ABC, Generic[_I]): self.data_parser = self.info.get_data_parser() + def get_dummy_inputs( + self, + seq_len: int, + mm_counts: Mapping[str, int], + mm_options: MultiModalDummyOptions, + ) -> ProcessorInputs: + """Build the input which, after processing, results in + the maximum possible number of placeholder tokens. + + Args: + seq_len: Sequence length + mm_counts: Count of items per modality + mm_options: Configurable options per modality (optional) + + """ + builder = self.dummy_inputs + dummy_text = builder.get_dummy_text(mm_counts) + dummy_mm_data = builder.get_dummy_mm_data(seq_len, mm_counts, mm_options) + dummy_mm_items = self.info.parse_mm_data(dummy_mm_data, validate=False) + + tokenizer = self.info.ctx.tokenizer + dummy_prompt: list[int] + if tokenizer is None: + # Tokenizer-less models (e.g. `skip_tokenizer_init=True`) only + # accept embeddings and have an empty dummy text, so there are no + # prompt tokens. + dummy_prompt = [] + else: + dummy_prompt = cached_encode(tokenizer, dummy_text, truncation=False) + + return ProcessorInputs( + prompt=dummy_prompt, + mm_data_items=dummy_mm_items, + ) + def get_dummy_mm_inputs( self, mm_counts: Mapping[str, int], @@ -1070,7 +1105,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]): mm_config = model_config.get_multimodal_config() - processor_inputs = self.dummy_inputs.get_dummy_processor_inputs( + processor_inputs = self.get_dummy_inputs( seq_len=seq_len, mm_counts=mm_counts, mm_options=mm_config.limit_per_prompt,